AHjDdZddlZddlZddlmZddlmZmZmZm Z m Z m Z ddl Z ddl Z ddlmZddlmZddlmZddlmZdd lmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$dd l%m&Z&dd l'm(Z(m)Z)eZ*d Z+e) d5d ee,efde ee,efde e,de ee,efde ee,efde ee,efde ee,efde e e-e j\fde e,defdZ/e) d5d ee,efde ee,efde e,de ee,efde ee,efde ee,efde ee,efde e e-e j\fde e,de eeeeefffdZ0e) d6de e1de e,de e,de e,de e,de ee,efde ee,efde e e-e j\fde e,defdZ2e) d6de e1de e,de e,de e,de e,de ee,efde ee,efde e e-e j\fde e,de eeeeefffdZ3e) d7d e,de ee,efde ee,efde e e-e j\fde e,def d!Z4e) d7d e,de ee,efde ee,efde e e-e j\fde e,de eeeeefff d"Z5e) d5d e,de e,de ee,efde ee,efde ee,efde ee,efde e e-e j\fde e,defd#Z6e) d5d e,de e,de ee,efde ee,efde ee,efde ee,efde e e-e j\fde e,de eeeeefffd$Z7e) d7d e,de ee,efde ee,efde e e-e j\fde e,def d%Z8e) d7d e,de ee,efde ee,efde e e-e j\fde e,de eeeeefff d&Z9e) d7d e,de ee,efde ee,efde e e-e j\fde e,def d'Z:e) d7d e,de ee,efde ee,efde e e-e j\fde e,de eeeeefff d(Z;e) d5d e,d)ee,efde e,de ee,efde ee,efde ee,efde ee,efde e e-e j\fde e,de"fd*Ze) d8d e,de e1de e,de e,de e,de ee,efde ee,efde e e-e j\fde e,de e!eeee!fffd-Z?e) d7d e,d.e,de ee,efde ee,efde e e-e j\fde e,de"fd/Z@e) d7d e,d.e,de ee,efde ee,efde e e-e j\fde e,de e"eeee"fffd0ZAe) d7d e,d.e,de ee,efde ee,efde e e-e j\fde e,defd1ZBe) d7d e,d.e,de ee,efde ee,efde e e-e j\fde e,de eeeeefffd2ZCe) d7d e,d.e,de ee,efde ee,efde e e-e j\fde e,de#fd3ZDe) d7d e,d.e,de ee,efde ee,efde e e-e j\fde e,de e#eeee#fffd4ZEy)9zw Main entry point for Evals API operations Provides create, list, get, update, delete, and cancel operations for evals N)partial)Any CoroutineDictListOptionalUnion)request_timeout)Logging)BaseEvalsAPIConfig)BaseLLMHTTPHandler) CancelEvalResponseCancelRunResponseCreateEvalRequestCreateRunRequestDeleteEvalResponseEvalListEvalsParamsListEvalsResponseListRunsParamsListRunsResponseRunRunDeleteResponseUpdateEvalRequest)GenericLiteLLMParams)ProviderConfigManagerclientzhttps://api.openai.comdata_source_configtesting_criterianamemetadata extra_headers extra_query extra_bodytimeoutcustom_llm_providerreturnc Kt} tj} d| d<ttf|||||||||d | } t j } t| j| }| jd|d{}tj|r |d{}|S|}|S7)7 #t$r}tjd||| | d}~wwxYww)ae Async: Create a new evaluation Args: data_source_config: Configuration for the data source testing_criteria: List of graders for all eval runs name: Optional name for the evaluation metadata: Optional additional metadata (max 16 key-value pairs) extra_headers: Additional headers for the request extra_query: Additional query parameters extra_body: Additional body parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: Eval object T acreate_eval) rrr r!r"r#r$r%r&Nmodelr&original_exceptioncompletion_kwargs extra_kwargs) localsasyncioget_event_loopr create_eval contextvars copy_contextrunrun_in_executor iscoroutine Exceptionlitellmexception_type)rrr r!r"r#r$r%r&kwargs local_varsloopfuncctxfunc_with_context init_responseresponsees J/root/tools/cai/cai_env/lib/python3.12/site-packages/litellm/evals/main.pyr)r))s>J" %%'!%~   1-'#! 3    &&(#CGGT2"2249JKK   } -**H%H L+  $$ 3 (    Y CA2B.B*B.B, B.%C&B.)C*B.,B.. C7CCCc  t} | jd} | jdd} | jdddu} tdi| }|d}t j t j|}|td |||d }|||d <|r|j||xsi}|j|| }|j||| }|jxst}|j|d}| j| d|d| i|t j#|||||| ||xst$| | jd| jd }|S#t&$r}t j(d||| | d}~wwxYw)a^ Create a new evaluation Args: data_source_config: Configuration for the data source testing_criteria: List of graders for all eval runs name: Optional name for the evaluation metadata: Optional additional metadata (max 16 key-value pairs) extra_headers: Additional headers for the request extra_query: Additional query parameters extra_body: Additional body parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: Eval object litellm_logging_objlitellm_call_idNr)FTopenaiproviderz!CREATE eval is not supported for )rrr headerslitellm_params)create_requestrNrMevals)api_baseendpointr;r+optional_paramsrNr&rshared_session url request_bodyevals_api_provider_configr&rN logging_objr"r% _is_asyncrrUr*)r/getpoprrget_provider_evals_api_configr9 LlmProviders ValueErrorupdatevalidate_environmenttransform_create_eval_requestrQDEFAULT_OPENAI_API_BASEget_complete_urlupdate_from_kwargsbase_llm_http_handlercreate_eval_handlerr r8r:)rrr r!r"r#r$r%r&r;r<rGrHr[rNrYrOrMrXrQrWrBrCs rDr2r2ns>JW 17  " % ,@AT@UVW W#5 0-   %)N6 "   ! !* - %2+@@NA  1NN))O "**E.E'889  ..(!?!4 / )<<%&? 3)+!.::h'!::&67=    $$ 3 (    sEE F'FFlimitafterbeforeorderorder_byc Kt} tj} d| d<ttf|||||||||d | } t j } t| j| }| jd|d{}tj|r |d{}|S|}|S7)7 #t$r}tjd||| | d}~wwxYww)a Async: List all evaluations Args: limit: Number of results to return per page (max 100, default 20) after: Cursor for pagination - returns evals after this ID before: Cursor for pagination - returns evals before this ID order: Sort order ('asc' or 'desc', default 'desc') order_by: Field to sort by ('created_at' or 'updated_at', default 'created_at') extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: ListEvalsResponse object T alist_evals) rjrkrlrmrnr"r#r%r&Nr*) r/r0r1r list_evalsr3r4r5r6r7r8r9r:)rjrkrlrmrnr"r#r%r&r;r<r=r>r?r@rArBrCs rDrprps>J" %%' $}   '# 3    &&(#CGGT2"2249JKK   } -**H%H L+  $$ 3 (    rEc  t} | jd} | jdd} | jdddu} tdi| }|d}t j t j|}|td |i}|||d <|||d <|||d <|||d <|||d<|r|j||xsi}|j||}|j|||\}}| j| d|d| i|tj|||||| ||xst| | jd| jd }|S#t $r}t j"d||| | d}~wwxYw)a List all evaluations Args: limit: Number of results to return per page (max 100, default 20) after: Cursor for pagination - returns evals after this ID before: Cursor for pagination - returns evals before this ID order: Sort order ('asc' or 'desc', default 'desc') order_by: Field to sort by ('created_at' or 'updated_at', default 'created_at') extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: ListEvalsResponse object rGrHNrpFTrIrJz LIST evals is not supported for rjrkrlrmrnrL) list_paramsrNrMrSrrU rW query_paramsrYr&rNrZr"r%r[rrUr*r\)r/r]r^rrr_r9r`rarbrctransform_list_evals_requestrgrhlist_evals_handlerr r8r:)rjrkrlrmrnr"r#r%r&r;r<rGrHr[rNrYrsrMrWrurBrCs rDrqrq-s>JV 17  " % ,?@S?TUV V(*  #(K  #(K  $*K !  #(K  &.K #    { + %2+@@NA  6RR#)S \ ..(!?!4 / );;%&? 3)+!.::h'!::&67<    $$ 3 (    sEE E;E66E;eval_idc Kt} tj}d|d<ttf|||||d|}t j } t| j|} |jd| d{} tj| r | d{} | S| } | S7)7 #t$r} tjd|| ||d} ~ wwxYww)ax Async: Get an evaluation by ID Args: eval_id: The ID of the evaluation to fetch extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: Eval object T aget_evalrxr"r#r%r&Nr*) r/r0r1rget_evalr3r4r5r6r7r8r9r:rxr"r#r%r&r;r<r=r>r?r@rArBrCs rDrzrzs.J %%'"{  '# 3   &&(#CGGT2"2249JKK   } -**H%H L+  $$ 3 (    Y CA.B*<B&=B*B(B*!C"B*%C&B*(B** C3C  CCc t} |jd}|jdd}|jdddu} tdi|} |d}t j t j|} | td ||xsi} | j| | } | jxst} | j|| | | \}} |j|dd |id|i| tj|| || || |xst | |jd|jd }|S#t"$r}t j$d||||d}~wwxYw)aq Get an evaluation by ID Args: eval_id: The ID of the evaluation to fetch extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: Eval object rGrHNrzFTrIrJzGET eval is not supported for rLrxrQrNrMrxrSrrU rWrYr&rNrZr"r%r[rrUr*r\)r/r]r^rrr_r9r`rarcrQretransform_get_eval_requestrgrhget_eval_handlerr r8r:rxr"r#r%r&r;r<rGrHr[rNrYrMrQrWrBrCs rDr|r|s.JF 17  " % ,=>Q=RST T %2+@@NA  "**E.E0KK) L W ..&0!?!4 / )99&? 3)+!.::h'!::&67:    $$ 3 (    D$D11 E:EEc Kt} tj} d|d<ttf||||||||d|} t j } t| j| } | jd| d{}tj|r |d{}|S|}|S7)7 #t$r}tjd||| |d}~wwxYww)a Async: Update an evaluation Args: eval_id: The ID of the evaluation to update name: Updated name metadata: Updated metadata extra_headers: Additional headers for the request extra_query: Additional query parameters extra_body: Additional body parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: Eval object T aupdate_eval)rxr r!r"r#r$r%r&Nr*) r/r0r1r update_evalr3r4r5r6r7r8r9r:)rxr r!r"r#r$r%r&r;r<r=r>r?r@rArBrCs rDrr@s:J! %%'!%~   '#! 3    &&(#CGGT2"2249JKK   } -**H%H L+  $$ 3 (    sY CA1B-?B)B-B+B-$C%B-(C)B-+B-- C6CCCc ht} |jd} |jdd} |jdddu} tdi|} |d}t j t j|}|td |i}|||d <|3hd }|jDcic] \}}||vs ||}}}|r||d <|r|j||xsi}|j|| }| jxst}|j|||| |\}}}| j|d|d| i|t j#||||| | ||xst$| |jd|jd }|Scc}}w#t&$r}t j(d||| |d}~wwxYw)a Update an evaluation Args: eval_id: The ID of the evaluation to update name: Updated name metadata: Updated metadata extra_headers: Additional headers for the request extra_query: Additional query parameters extra_body: Additional body parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: Eval object rGrHNrFTrIrJz!UPDATE eval is not supported for r >rMrR user_agent user_api_keyuser_api_key_authuser_api_key_hashrequester_metadatauser_api_key_aliasuser_api_key_spendlitellm_api_versionuser_api_key_org_idrequester_ip_addressuser_api_key_team_iduser_api_key_user_iduser_api_key_metadatauser_api_key_max_budgetuser_api_key_team_aliasuser_api_key_team_spenduser_api_key_user_emailuser_api_key_user_spendlitellm_parent_otel_spanuser_api_key_end_user_iduser_api_key_auth_metadatauser_api_key_request_routeglobal_max_parallel_requestsuser_api_end_user_max_budgetuser_api_key_budget_reset_atuser_api_key_team_max_budgetuser_api_key_user_max_budgetuser_api_key_model_max_budgetr!rL)rxupdate_requestrQrNrMrSrrUrVr*r\)r/r]r^rrr_r9r`raitemsrbrcrQretransform_update_eval_requestrgrhupdate_eval_handlerr r8r:)rxr r!r"r#r$r%r&r;r<rGrHr[rNrYr internal_keyskvfiltered_metadatarMrQrWrXrBrCs rDrrsM:J 17  " % ,@AT@UVW W-/  %)N6 "  MD"*!1!AQm5K1! !!->z*   ! !* - %2+@@NA  "**E.E & C C)) D     ..(!?!4 / )<<%&? 3)+!.::h'!::&67=  q!r  $$ 3 (    s1B!F - F:F?CF F F1F,,F1c Kt} tj}d|d<ttf|||||d|}t j } t| j|} |jd| d{} tj| r | d{} | S| } | S7)7 #t$r} tjd|| ||d} ~ wwxYww)a Async: Delete an evaluation Args: eval_id: The ID of the evaluation to delete extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: DeleteEvalResponse object T adelete_evalr{Nr*) r/r0r1r delete_evalr3r4r5r6r7r8r9r:r}s rDrr".J %%'!%~  '# 3   &&(#CGGT2"2249JKK   } -**H%H L+  $$ 3 (    r~c t} |jd}|jdd}|jdddu} tdi|} |d}t j t j|} | td ||xsi} | j| | } | jxst} | j|| | | \}} |j|dd |id|i| tj|| || || |xst | |jd|jd }|S#t"$r}t j$d||||d}~wwxYw)a} Delete an evaluation Args: eval_id: The ID of the evaluation to delete extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: DeleteEvalResponse object rGrHNrFTrIrJz!DELETE eval is not supported for rLrrxrSrrUrr*r\)r/r]r^rrr_r9r`rarcrQretransform_delete_eval_requestrgrhdelete_eval_handlerr r8r:rs rDrr[s.JF 17  " % ,@AT@UVW W %2+@@NA  "**E.E0NN) O W ..&0!?!4 / )<<&? 3)+!.::h'!::&67=    $$ 3 (    rc Kt} tj}d|d<ttf|||||d|}t j } t| j|} |jd| d{} tj| r | d{} | S| } | S7)7 #t$r} tjd|| ||d} ~ wwxYww)a Async: Cancel a running evaluation Args: eval_id: The ID of the evaluation to cancel extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: CancelEvalResponse object T acancel_evalr{Nr*) r/r0r1r cancel_evalr3r4r5r6r7r8r9r:r}s rDrrrr~c t} |jd}|jdd}|jdddu} tdi|} |d}t j t j|} | td ||xsi} | j| | } | jxst} | j|| | | \}} }|j|dd |id|i| tj|| || || |xst | |jd|jd }|S#t"$r}t j$d||||d}~wwxYw)a Cancel a running evaluation Args: eval_id: The ID of the evaluation to cancel extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: CancelEvalResponse object rGrHNrFTrIrJz!CANCEL eval is not supported for rLrrxrSrrUrr*r\)r/r]r^rrr_r9r`rarcrQretransform_cancel_eval_requestrgrhcancel_eval_handlerr r8r:)rxr"r#r%r&r;r<rGrHr[rNrYrMrQrWrXrBrCs rDrrs.JJ 17  " % ,@AT@UVW W %2+@@NA  "**E.E & C C) D     ..&0!?!4 / )<<&? 3)+!.::h'!::&67=    $$ 3 (    sD%D22 E;EE data_sourcec Kt} tj} d| d<ttf|||||||||d | } t j } t| j| }| jd|d{}tj|r |d{}|S|}|S7)7 #t$r}tjd||| | d}~wwxYww)ap Async: Create a new run for an evaluation Args: eval_id: The ID of the evaluation to run data_source: Data source configuration for the run (can be jsonl, completions, or responses type) name: Optional name for the run metadata: Optional additional metadata extra_headers: Additional headers for the request extra_query: Additional query parameters extra_body: Additional body parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: Run object T acreate_run) rxrr r!r"r#r$r%r&Nr*) r/r0r1r create_runr3r4r5r6r7r8r9r:)rxrr r!r"r#r$r%r&r;r<r=r>r?r@rArBrCs rDrr_s>J" %%' $}   #'#! 3    &&(#CGGT2"2249JKK   } -**H%H L+  $$ 3 (    rEc  t} | jd} | jdd} | jdddu} tdi| }|d}t j t j|}|td |d |i}|||d <|r|j||xsi}|j|| }|jxst}|j|||| \}}| j| d|d| i|tj!|||||| ||xst#j$dd| | jd| jd }|S#t&$r}t j(d||| | d}~wwxYw)a Create a new run for an evaluation Args: eval_id: The ID of the evaluation to run data_source: Data source configuration for the run (can be jsonl, completions, or responses type) name: Optional name for the run metadata: Optional additional metadata extra_headers: Additional headers for the request extra_query: Additional query parameters extra_body: Additional body parameters timeout: Request timeout (default 600s for long-running operations) custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: Run object rGrHNrFTrIrJz CREATE run is not supported for rr rL)rxrOrNrMrSg@g@)r%connectrrUrVr*r\)r/r]r^rrr_r9r`rarbrcrQretransform_create_run_requestrgrhcreate_run_handlerhttpxTimeoutr8r:)rxrr r!r"r#r$r%r&r;r<rGrHr[rNrYrOrMrQrWrXrBrCs rDrrs>JT 17  " % ,?@S?TUV V ;,   %)N6 "   ! !* - %2+@@NA  "**E.E5RR)) S \ ..(!?!4 / );;%&? 3)+!Hu}}UCH::h'!::&67<    $$ 3 (    sEE F(FFc Kt} tj} d| d<ttf|||||||||d | } t j } t| j| }| jd|d{}tj|r |d{}|S|}|S7)7 #t$r}tjd||| | d}~wwxYww)a Async: List all runs for an evaluation Args: eval_id: The ID of the evaluation limit: Number of results to return per page (max 100, default 20) after: Cursor for pagination - returns runs after this ID before: Cursor for pagination - returns runs before this ID order: Sort order ('asc' or 'desc', default 'desc') extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: ListRunsResponse object T alist_runs) rxrjrkrlrmr"r#r%r&Nr*) r/r0r1r list_runsr3r4r5r6r7r8r9r:)rxrjrkrlrmr"r#r%r&r;r<r=r>r?r@rArBrCs rDrrs>J" %%'#|   '# 3    &&(#CGGT2"2249JKK   } -**H%H L+  $$ 3 (    rEc  t} | jd} | jdd} | jdddu} tdi| }|d}t j t j|}|td |i}|||d <|||d <|||d <|||d <|r|j||xsi}|j||}|j||||\}}| j| dd|i|d| i|tj|||||| ||xst| | jd| jd }|S#t $r}t j"d||| | d}~wwxYw)a List all runs for an evaluation Args: eval_id: The ID of the evaluation limit: Number of results to return per page (max 100, default 20) after: Cursor for pagination - returns runs after this ID before: Cursor for pagination - returns runs before this ID order: Sort order ('asc' or 'desc', default 'desc') extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: ListRunsResponse object rGrHNrFTrIrJzLIST runs is not supported for rjrkrlrmrL)rxrsrNrMrxrSrrUrtr*r\)r/r]r^rrr_r9r`rarbrctransform_list_runs_requestrgrhlist_runs_handlerr r8r:)rxrjrkrlrmr"r#r%r&r;r<rGrHr[rNrYrsrMrWrurBrCs rDrr`s>JU 17  " % ,>?R>STU U')  #(K  #(K  $*K !  #(K     { + %2+@@NA  6QQ#) R \ ..&@<@!?!4 / )::%&? 3)+!.::h'!::&67;    $$ 3 (    sEE E9E44E9run_idc Kt} tj}d|d<ttf||||||d|} t j } t| j| } |jd| d{} tj| r | d{} | S| } | S7)7 #t$r}tjd||||d}~wwxYww)a Async: Get a specific run Args: eval_id: The ID of the evaluation run_id: The ID of the run to retrieve extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: Run object Taget_runrxrr"r#r%r&Nr*) r/r0r1rget_runr3r4r5r6r7r8r9r:rxrr"r#r%r&r;r<r=r>r?r@rArBrCs rDrrs2J %%'!z   '# 3    &&(#CGGT2"2249JKK   } -**H%H L+  $$ 3 (    Y CA/B+=B'>B+B)B+"C#B+&C'B+)B++ C4CCCc t} |jd}|jdd} |jdddu} tdi|} |d}t j t j|} | td ||xsi} | j| | } | jxst}| j|||| | \}} |j|d||d d| i| tj|| || || |xst | |jd|jd }|S#t"$r}t j$d||||d}~wwxYw)a Get a specific run Args: eval_id: The ID of the evaluation run_id: The ID of the run to retrieve extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: Run object rGrHNrFTrIrJzGET run is not supported for rLrxrrQrNrMrxrrSrrUrr*r\)r/r]r^rrr_r9r`rarcrQretransform_get_run_requestrgrhget_run_handlerr r8r:)rxrr"r#r%r&r;r<rGrHr[rNrYrMrQrWrBrCs rDrrs2JG 17  " % ,<=P  " % ,?@S?TUV V %2+@@NA  "**E.E & B B) C     ..(/6B!?!4 / );;&? 3)+!.::h'!::&67<    $$ 3 (    D'D44 E=EEc Kt} tj}d|d<ttf||||||d|} t j } t| j| } |jd| d{} tj| r | d{} | S| } | S7)7 #t$r}tjd||||d}~wwxYww)a Async: Delete a run Args: eval_id: The ID of the evaluation run_id: The ID of the run to delete extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: RunDeleteResponse object T adelete_runrNr*) r/r0r1r delete_runr3r4r5r6r7r8r9r:rs rDrr!rrc t} |jd}|jdd} |jdddu} tdi|} |d}t j t j|} | td ||xsi} | j| | } | jxst}| j|||| | \}} }|j|d||d d| i| tj|| || || |xst | |jd|jd }|S#t"$r}t j$d||||d}~wwxYw)a Delete a run Args: eval_id: The ID of the evaluation run_id: The ID of the run to delete extra_headers: Additional headers for the request extra_query: Additional query parameters timeout: Request timeout custom_llm_provider: Provider name (e.g., 'openai') **kwargs: Additional parameters Returns: RunDeleteResponse object rGrHNrFTrIrJz DELETE run is not supported for rLrrrSrrUrr*r\)r/r]r^rrr_r9r`rarcrQretransform_delete_run_requestrgrhdelete_run_handlerr r8r:rs rDrr]rr)NNNNNNN) NNNNNNNNN)NNNN)NNNNNNNN)F__doc__r0r3 functoolsrtypingrrrrrr rr9litellm.constantsr *litellm.litellm_core_utils.litellm_loggingr LiteLLMLoggingObj*litellm.llms.base_llm.evals.transformationr *litellm.llms.custom_httpx.llm_http_handlerr litellm.types.llms.openai_evalsrrrrrrrrrrrrrlitellm.types.routerr litellm.utilsrrrhrestrfloatrr)r2intrprqrzr|rrrrrrrrrrrrrrrrr\rDrsN >> -SII67+,2)-.2,0+/59)-A S#XA 4S>*A  3-A tCH~& A DcN+ A $sCx.) A c3h(A eE5==01 2A "#A  A A H)-.2,0+/59)-v S#Xv 4S>*v  3-v tCH~& v DcN+ v $sCx.) v c3h(v eE5==01 2v "#v  43T>* *+v v r ".2,059)-A C=A C=A  SMA  C= A sm A DcN+ A $sCx.)A eE5==01 2A "#A A A H ".2,059)-u C=u C=u  SMu  C= u sm u DcN+ u $sCx.)u eE5==01 2u "#u  iS2C(CD DEu u p/3,059)- 5 5 DcN+5 $sCx.)5 eE5==01 2 5 "# 5  5 5 p/3,059)- ] ] DcN+] $sCx.)] eE5==01 2 ] "# ]  43T>* *+] ] @)-.2,0+/59)-> > 3-> tCH~&> DcN+ > $sCx.) > c3h( > eE5==01 2> "#>  > > B)-.2,0+/59)-\ \ 3-\ tCH~&\ DcN+ \ $sCx.) \ c3h( \ eE5==01 2\ "#\  43T>* *+\ \ ~/3,059)- 5 5 DcN+5 $sCx.)5 eE5==01 2 5 "# 5 5 5 p/3,059)- ] ] DcN+] $sCx.)] eE5==01 2 ] "# ]  yc3E)EF FG] ] @/3,059)- 5 5 DcN+5 $sCx.)5 eE5==01 2 5 "# 5 5 5 p/3,059)- a a DcN+a $sCx.)a eE5==01 2 a "# a  yc3E)EF FGa a R)-.2,0+/59)-A A c3hA  3-A tCH~& A DcN+ A $sCx.) A c3h(A eE5==01 2A "#A  A A H)-.2,0+/59)-s s c3hs  3-s tCH~& s DcN+ s $sCx.) s c3h(s eE5==01 2s "#s  3 #sC-( ()s s l  .2,059)-A A C=A  C=A  SM A C= A DcN+ A $sCx.)A eE5==01 2A "#A A A H  .2,059)-t t C=t  C=t  SM t C= t DcN+ t $sCx.)t eE5==01 2t "#t  YsC1A'AB BCt t n/3,059)- 8 8 8 DcN+8 $sCx.) 8 eE5==01 2 8 "# 8  8 8 v/3,059)- ` ` ` DcN+` $sCx.) ` eE5==01 2 ` "# `  3 #sC-( ()` ` F/3,059)- 8 8 8 DcN+8 $sCx.) 8 eE5==01 2 8 "# 8 8 8 v/3,059)- d d d DcN+d $sCx.) d eE5==01 2 d "# d  iS2C(CD DEd d X/3,059)- 8 8 8 DcN+8 $sCx.) 8 eE5==01 2 8 "# 8 8 8 v/3,059)- d d d DcN+d $sCx.) d eE5==01 2 d "# d  iS2C(CD DEd d r