
    AHj             U          U d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
mZ d dlmZmZmZ d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$ d dl%m&Z& erd dl'm(Z( d dl)Z)d dl*Z*d dl+Z+d dl,Z,d d	l-m.Z. d d
l/m0Z0 d dl1Z1d dl1m2Z2 d dl3m4Z4m5Z5m6Z6 erd dl7m8Z8 d dl9m:Z: d dl;m<Z<m=Z= d dl>m?Z? d dl@mAZA d dlBmCZC d dlDmEZEmFZF d dlGmHZH d dlImJZJ d dlKmLZLmMZM d dl7m8ZN d dlOmPZPmQZQ d dlRmSZS d dlTmUZUmVZV d dlWmXZX d dlYmZZZ d dl[m\Z\ d dl]m^Z^m_Z_ d dl`maZa d dlbmcZc d d ldmeZemfZf d d!lgmhZh d d"limjZjmkZk d d#llmmZm d d$l9mnZnmoZompZpmqZq d d%l3mrZrmsZsmtZtmuZumvZvmwZwmxZxmyZymzZzm{Z{m|Z|m}Z}m~Z~mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ d&d'lmZ d&d(lmZmZmZ d&d)lmZ d&d*lmZmZ d&d+lmZmZmZ d&d,lmZmZmZmZmZmZ d&d-lmZ d&d.lmZ d&d/lmZ d&d0lmZmZ d&d1lmZ d&d2lmZ d&d3lmZ d&d4lmZ d&d5lmZmZ d&d6lmZ d&d7lmZ d&d8lmZ d&d9lmZ d&d:lmZ d&d;lmZ d&d<lmZ d&d=lmZ d&d>lmZ d&d?lmZmZ d&d@lmZ d&dAlmZmZ d&dBlmZ d&dClmZ d&dDlmZ d&dElmZ d&dFlmZ d&dGlmZ d&dHlmZ d&d<lmZ d&dIlmZ d&dJlmZ d&dKlmZ d&dLlmZ d&dMlmZ d&dNlmZ d&dOlmZ d&dPlmZ d&d<lmАZ  d&dQlmZ d&dGlmZ d&dRlmZ d&dSlmZ d&dTl	m
Z
 d&dUlmZ d&dVlmZ d&dWlmZ d&dXlmZ d&dYlmZ d&dZlmZ d&d[lmZ d&d\lmZ d&d]lmZ d&d<lmАZ d&d^lm Z  d&d_l!m"Z" d&d`l#m$Z$ d&dal%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z- d&dbl.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7  e       Z8 e       Z9 e       Z: e       Z; e       Z< e
       Z= e
       Z> e       Z? e       Z@ e       ZA e       ZB e       ZC e       ZD e       ZE e       ZF e       ZG eΫ       ZH e       ZI eī       ZJ eȫ       ZK eƫ       ZL e       ZM e       ZN e       ZO e       ZP e       ZQ e       ZR e       ZS e       ZT e       ZU e        ZV e       ZW e       ZX eګ       ZY eի       ZZ eӫ       Z[ e       Z\ eʫ       Z] e       Z^ e       Z_ e       Z` e       Zae"ebecedeveof   Ze G dc dd      Zf G de df      Zg G dg dh      Zh G di dj      Zi eHj                         e2g dddddddddddddddddddddddddddddddddddddddf(dkebdledmee   dneeb   doee"ekelf      dpeek   dqeek   dreel   dseem   dteed   dueel   dveel   dweee(      dxee)   dyee'   dzeek   d{eek   d|eed   d}eeb   d~ee"ede!e.   f      deel   dee   dee"ebedf      deem   deem   deel   deed      deed      deeb   deeb   deeb   deeb   deeb   deen   deed   dee$   dee-   ded   deem   de"evesf   fPd              Zod Zp	 dde"ebecf   dkebdeeb   fdZqdeem   doee"ekebe*j                  f      dkebfdZsdeem   doee"ekebe*j                  f      dkebfdZtdoe"ekebe*j                  f   fdZudoe"ekebe*j                  f   fdZv	 	 	 	 	 	 	 	 ddkebdledseem   dreel   deee   dee   deem   doee"ekebe*j                  f      fdZw	 	 	 ddkebdebdee-   deee      dee   de edebf   fdZxdeeb   dkebdemfdZydeddedfdZzdeeed      deeed      fdZ{	 ddebdeddeed   dedfdZ| eHj                         e2g dddddddddddddddddddddddddddddddddddddddf(dkebdledoee"ekebe*j                  f      dpeek   dqeek   dreel   dseem   dteed   dveel   dueel   dweee(      dxee)   dyee'   dzeek   d{eek   d|eed   d}eeb   deed      deed      d~ee"ede!e.   f      deel   dee   dee"ebedf      deem   deel   deem   dee-   deed   deeb   deeb   dmee   dneeb   deeb   deeb   deeb   deen   dee$   ded   deem   de"evesf   fPd              Zd Z}d Z~d Zd Ze2detfd       Ze0g ddddddddddddfdeel   deeb   deeb   deeb   deeb   deeb   demd}eeb   ded   deeeetf   fd       Ze0g ddddddddddddfdddeel   deeb   deeb   deeb   deeb   deeb   demd}eeb   ded   detfd       Ze2g ddddddddddddfdeel   deeb   deeb   deeb   deeb   deeb   demd}eeb   de"eteeeetf   f   fd       Ze2de"eyezf   fd       Ze2	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dde"ebee"ebee"ebeel   f      f      f   dkeeb   deel   deem   d{eek   d|eeelelf      deel   dueel   dreel   dzeek   dee"ebeeb   f      dseem   dteed   deeb   dpeek   dqeek   d}eeb   deeb   deeb   deeb   deen   deeb   f,dÄ       ZdĐebdee"e.e/f      fdńZde"eebef   ee   f   fdƄZdĐebdee"e.e/f      fdǄZ	 ddȐebdkeeb   deeb   de,fdɄZe2	 	 	 ddȐebdkeeb   deeb   deeb   de,f
dʄ       Ze2de{fd˄       Ze2	 	 	 	 	 	 	 	 	 	 	 	 ddkebd̐e2deeb   deeb   d~eed      deeed         dpeel   d}eeb   deeb   deeb   deeb   deel   de"e{eeee{f   f   fd҄       Ze2de+fdӄ       Ze2	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddkebdȐebdee"ebedf      deeb   deeb   deeb   deeb   deeb   deel   deed   doee"eke*j                  f      d~eeb   deel   deeb   deed   deeb   deem   de"e+eeee+f   f   f$d܄       Z	 	 	 ddݐeddeed      deeb   dee   fdZd Zd Z	 ddendlee   deyfdZ	 	 	 	 ddendleen   ded   dee"eveyf      fdZ	 	 	 	 	 ddkebdleeeebef         deeeebef         deeb   deeb   deeb   ddfdZdaee   ed<   d ZdebdefdZy)    N)futures)FIRST_COMPLETEDThreadPoolExecutorwait)deepcopy)partial)TYPE_CHECKINGAnyAsyncIteratorCallable	CoroutineDictIterableListLiteralMappingOptionalTupleTypeUnioncastget_args)uuid)ClientSession)	BaseModel)overload)client)exception_typeget_litellm_paramsget_optional_params)Logging)TokenCountResponse),DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT(DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT)LiteLLMUnknownProvider)CustomLogger)run_async_function)calculate_request_durationget_audio_file_for_health_check)tracer)ProviderSpecificHeaderUtils)_create_health_check_response_filter_model_params)mock_embeddingmock_image_generation)get_content_from_model_response)
BaseConfigBaseImageGenerationConfig)#convert_model_response_to_streaming)BedrockModelInfo)CohereModelInfo)AsyncHTTPHandlerHTTPHandler)OpenAIGPT5Config)JSONProviderRegistry)VertexAIModelRouteget_vertex_ai_model_route)_realtime_health_check)get_secret_boolget_secret_str)GenericLiteLLMParams)CustomPricingLiteLLMParamsModelResponseStreamRawRequestTypedDictStreamingChoices)'ChoicesCustomStreamWrapperEmbeddingResponseMessageModelResponseProviderConfigManagerTextChoicesTextCompletionResponseTextCompletionStreamWrapperTranscriptionResponseUsage_get_model_info_helper/add_provider_specific_params_to_optional_params#async_mock_completion_streaming_obj convert_to_model_response_objectcreate_pretrained_tokenizercreate_tokenizerget_api_keyget_llm_providerget_model_info!get_non_default_completion_params$get_non_default_transcription_paramsget_optional_params_embeddingsget_optional_params_image_gen!get_optional_params_transcriptionget_requester_metadata
get_secretget_standard_openai_paramsmock_completion_streaming_objpre_process_non_default_paramsread_config_argsshould_run_mock_completionsupports_httpx_timeouttoken_counter validate_and_fix_openai_messagesvalidate_and_fix_openai_toolsvalidate_and_fix_thinking_param$validate_chat_completion_tool_choicevalidate_openai_optional_params   )verbose_logger)disable_cacheenable_cacheupdate_cache)safe_deep_copy)async_completion_with_fallbackscompletion_with_fallbacks)add_system_prompt_to_messagesget_completion_messages#update_messages_with_model_file_ids)custom_promptfunction_call_promptmap_system_message_pt	ollama_ptprompt_factory stringify_json_tool_call_content)ChunkProcessor)AnthropicChatCompletion)AzureAudioTranscription)AzureChatCompletion_check_dynamic_azure_params)AzureOpenAIO1ChatCompletion)AzureTextCompletion)AzureAnthropicChatCompletion)AzureAIEmbedding)BedrockConverseLLM
BedrockLLM)BedrockEmbedding)BedrockImageEdit)BedrockImageGeneration)BytezChatConfig)ClarifaiConfig)CodestralTextCompletion)handler)BaseLLMAIOHTTPHandler)BaseLLMHTTPHandler)	CustomLLMcustom_chat_llm_router)DatabricksEmbeddingHandler)aleph_alphapalm)get_api_key_from_env)GroqChatCompletion)HerokuChatConfig)HuggingFaceEmbedding)LemonadeChatConfig)
completion)OCIChatConfig)	oobabooga)OpenAITextCompletion)OpenAIImageVariationsHandler)OpenAIChatCompletion)OpenAIAudioTranscription)OpenAILikeChatHandler)OpenAILikeEmbeddingHandler)OVHCloudChatConfig)PredibaseChatCompletion)SagemakerChatHandler)SagemakerLLM)GenAIHubOrchestration)vertex_ai_non_gemini)	VertexLLM)GoogleBatchEmbeddings)VertexImageGeneration)VertexMultimodalEmbedding)VertexAIPartnerModels)VertexEmbedding)VertexAIGemmaModels)VertexAIModelGardenModels)WatsonXChatHandler)IBMWatsonXMixin)AnthropicThinkingParam)ChatCompletionAssistantMessageChatCompletionAudioParamChatCompletionModality$ChatCompletionPredictionContentParamChatCompletionUserMessageHttpxBinaryResponseContentOpenAIModerationResponseOpenAIWebSearchOptions)	AdapterCompletionStreamWrapperChatCompletionMessageToolCallCompletionTokensDetails	FileTypesHiddenParamsLlmProvidersPromptTokensDetailsProviderSpecificHeaderall_litellm_paramsc                   t    e Zd Zddddej                  dddee   dee   dee   dee   dee	eef      f
d	Z
y)
LiteLLMNX  )api_keyorganizationbase_urltimeoutmax_retriesdefault_headersr   r   r   r   r   c                Z    t               | _        t        | j                  d       | _        y )N
router_obj)localsparamsChatchat)selfr   r   r   r   r   r   s          D/root/tools/cai/cai_env/lib/python3.12/site-packages/litellm/main.py__init__zLiteLLM.__init__9  s     h6	    )__name__
__module____qualname__litellmnum_retriesr   strfloatintr   r    r   r   r   r   8  sq     &*"&#&%,%8%87;7 sm	7
 3-7 %7 c]7 "'#s("347r   r   c                       e Zd Zdee   fdZy)r   r   c                     || _         | j                   j                  dd      du r8| j                   j                  d       t        | j                   |      | _        y t        | j                   |      | _        y )NacompletionFTr   )r   getpopAsyncCompletionscompletionsCompletionsr   r   r   s      r   r   zChat.__init__H  s\    ;;??=%0D8KKOOM*EU
FD  +4;;:NDr   N)r   r   r   r   r
   r   r   r   r   r   r   G  s    O8C= Or   r   c                   &    e Zd Zdee   fdZddZy)r   r   c                      || _         || _        y Nr   r   r   s      r   r   zCompletions.__init__T      $r   Nc                 0   |j                         D ]  \  }}|| j                  |<    |xs | j                  j                  d      }| j                  + | j                  j                  d||d| j                  }|S t	        d||d| j                  }|S Nmodelr   messagesr   )itemsr   r   r   r   r   r   r   kwargskvresponses          r   createzCompletions.createX  s    LLN 	DAqDKKN	11??&1t11 h26++H
  "PPDKKPHr   r   r   r   r   r   r
   r   r   r   r   r   r   r   S      %8C= %
r   r   c                   &    e Zd Zdee   fdZddZy)r   r   c                      || _         || _        y r   r   r   s      r   r   zAsyncCompletions.__init__f  r   r   Nc                 `  K   |j                         D ]  \  }}|| j                  |<    |xs | j                  j                  d      }| j                  3 | j                  j                  d||d| j                   d {   }|S t	        d||d| j                   d {   }|S 7 (7 wr   )r   r   r   r   r   r   s          r   r   zAsyncCompletions.createj  s     LLN 	DAqDKKN	11??&8T__88 h26++ H
  )WuxW4;;WWH Xs$   A?B.B*!B.#B,$B.,B.r   r   r   r   r   r   r   e  r   r   r   r   r   	functionsfunction_callr   temperaturetop_pnstreamstream_options
max_tokensmax_completion_tokens
modalities
predictionaudiopresence_penaltyfrequency_penalty
logit_biasuserresponse_formatseedtoolstool_choiceparallel_tool_callslogprobstop_logprobsreasoning_effort)noneminimallowmediumhighxhighdefault	verbosity)r  r  r  safety_identifierservice_tierr   api_versionr   
model_listextra_headersthinkingweb_search_optionsshared_sessionr   enable_json_schema_validationreturnc)                   K   |)j                  dd      }*|)j                  dd      }+|+du rt        |+||        d{    t        j                         },|)j                  dd      }-|)j                  dd      }.t	        |.t
              r|.j                  |)j                  dd      |)|      rw|.j                  | ||)|)j                  dd      |)j                  d	d      ||)j                  d
d      |)j                  dd             d{   \  } }}/|t        |      dk(  rd}|'#t        j                  dt        |'       d       nt        j                  d       i d| d|d|d|d|d|d|d|d|d|	d|
d|d|d|d|d |d!|i d"|d#|d$|d%|d&|d'|d(|d)|d*|d+|d,|d-| d.|!d/|"d0|#d1|d2|||$d|%|&|'|(d3}0|-#t        | |-|0j                  d-d      4      \  }/}-}/}/|*xs t        j                  }*|**t        dAi |0d5d|*i|)i d{   }1|1t!        d6      |1S |)j                  d7      }2|)j                  d8      }3|)j                  d9      }4|)j                  d      }+|2r+t#        |3|4|+:      rt        j$                  |2       d{    	 t'        t(        fi |0|)}5t+        j,                         }6t'        |6j.                  |5      }7|,j1                  d|7       d{   }8t	        |8t2              st	        |8t4              rt	        |8t2              rt5        dAi |8}1|8}1n"t        j6                  |8      r|8 d{   }1n|8}1|-d;k(  s|-d<k(  rGt	        |1t8              r7t        j:                         j=                  |1t        j4                         =      }1t	        |1t>              r|1jA                  |,>       |1S 7 7 7 7 :7 7 # t         $ r}9|-xs d?}-tC        | |-|9|0|)@      d}9~9ww xY ww)Ba  
    Asynchronously executes a litellm.completion() call for any of litellm supported llms (example gpt-4, gpt-3.5-turbo, claude-2, command-nightly)

    Parameters:
        model (str): The name of the language model to use for text completion. see all supported LLMs: https://docs.litellm.ai/docs/providers/
        messages (List): A list of message objects representing the conversation context (default is an empty list).

        OPTIONAL PARAMS
        functions (List, optional): A list of functions to apply to the conversation messages (default is an empty list).
        function_call (str, optional): The name of the function to call within the conversation (default is an empty string).
        temperature (float, optional): The temperature parameter for controlling the randomness of the output (default is 1.0).
        top_p (float, optional): The top-p parameter for nucleus sampling (default is 1.0).
        n (int, optional): The number of completions to generate (default is 1).
        stream (bool, optional): If True, return a streaming response (default is False).
        stream_options (dict, optional): A dictionary containing options for the streaming response. Only use this if stream is True.
        stop(string/list, optional): - Up to 4 sequences where the LLM API will stop generating further tokens.
        max_tokens (integer, optional): The maximum number of tokens in the generated completion (default is infinity).
        max_completion_tokens (integer, optional): An upper bound for the number of tokens that can be generated for a completion, including visible output tokens and reasoning tokens.
        modalities (List[ChatCompletionModality], optional): Output types that you would like the model to generate for this request. You can use `["text", "audio"]`
        prediction (ChatCompletionPredictionContentParam, optional): Configuration for a Predicted Output, which can greatly improve response times when large parts of the model response are known ahead of time. This is most common when you are regenerating a file with only minor changes to most of the content.
        audio (ChatCompletionAudioParam, optional): Parameters for audio output. Required when audio output is requested with modalities: ["audio"]
        presence_penalty (float, optional): It is used to penalize new tokens based on their existence in the text so far.
        frequency_penalty: It is used to penalize new tokens based on their frequency in the text so far.
        logit_bias (dict, optional): Used to modify the probability of specific tokens appearing in the completion.
        user (str, optional):  A unique identifier representing your end-user. This can help the LLM provider to monitor and detect abuse.
        metadata (dict, optional): Pass in additional metadata to tag your completion calls - eg. prompt version, details, etc.
        api_base (str, optional): Base URL for the API (default is None).
        api_version (str, optional): API version (default is None).
        api_key (str, optional): API key (default is None).
        model_list (list, optional): List of api base, version, keys
        timeout (float, optional): The maximum execution time in seconds for the completion request.

        LITELLM Specific Params
        mock_response (str, optional): If provided, return a mock completion response for testing or debugging purposes (default is None).
        custom_llm_provider (str, optional): Used for Non-OpenAI LLMs, Example usage for bedrock, set model="amazon.titan-tg1-large" and custom_llm_provider="bedrock"
    Returns:
        ModelResponse: A response object containing the generated completion and associated metadata.

    Notes:
        - This function is an asynchronous version of the `completion` function.
        - The `completion` function is called using `run_in_executor` to execute synchronously in the event loop.
        - If `stream` is True, the function returns an async generator that yields completion lines.
    	fallbacksNmock_timeoutTcustom_llm_providerlitellm_logging_obj	prompt_id)r0  non_default_paramsr  prompt_variablesprompt_labelprompt_version)r   r   r1  r0  r2  r  r3  r4  r   uA   🔄 SHARED SESSION: acompletion called with shared_session (ID: )uA   🔄 NO SHARED SESSION: acompletion called without shared_sessionr   r   r  r  r   r  r  r  r  r  stopr  r	  r
  r  r  r  r  r  r  r  r  r  r  r  r  r  deployment_idr   r#  r   r$  r  r!  )r"  r%  r   r&  r'  r(  r)  r   r.  api_baser   z]No response from fallbacks. Got none. Turn on `litellm.set_verbose=True` to see more details.
mock_delaymock_responsemock_tool_calls)r;  r<  r-  text-completion-openaitext-completion-codestralresponse_objectmodel_response_object)loopopenair   r.  original_exceptioncompletion_kwargsextra_kwargsr   )"r   _handle_mock_timeout_asyncasyncioget_event_loop
isinstanceLiteLLMLoggingObj"should_run_prompt_management_hooks async_get_chat_completion_promptlenrl   debugidrV   r   model_fallbacksrq   	Exceptionrc   sleepr   r   contextvarscopy_contextrunrun_in_executordictrH   iscoroutinerK   OpenAITextCompletionConfig%convert_to_chat_model_response_objectrE   set_logging_event_loopr   ):r   r   r  r  r   r  r  r  r  r  r6  r  r	  r
  r  r  r  r  r  r  r  r  r  r  r  r  r  r7  r  r   r!  r"  r   r#  r   r$  r%  r&  r'  r(  r)  r   r,  r-  rB  r.  r/  _rF  r   r:  r;  r<  funcctxfunc_with_contextinit_responsees:                                                             r   r   r   w  s    B 

;-I::nd3Lt(wFFF!!#D **%:DA
 !**%:DA%'89>>jjd3% 	? 	
 &FF%jjd3#ZZ(:DAND9!::&6= G 	
 	
		
$ UqE !OPRSaPbOccde	
 	O	

**H* 	Y* 		*
 	7* 	{* 	* 	Q* 	&* 	.* 	* 	j* 	 !6* 	j* 	j*  	!*" 	,#*$ 	.%*& 	j'*( 	)** 	?+*, 	-*. 	/*0 	{1*2 	23*4 	H5*6 	7*8 	9*: 	H;*< 	{=*> 	7?*@ 	jA*B 	,C*D 	.E*F %&0()FS*V "'7 3&**:t<(
$1 4W44I8 

)4i(J6(J
 
 o   L)JJJ/Mjj!23O::n-L0#'!
 mmJ''')
zA%6A&A &&(#CGGT2"2249JKKmT*j=/
 -.(9=9$H  /**H$H  #;;"&AA#9:99;aa (&-&;&;&= b H h 34++ ,  A 	G(	
l
( 	( L +"  
1=X 3 /
 	

s   9P
OCP
O	DP
"O#A?P
"O#P
(AO# 9O:AO# O!A;O# P
P
P
P
O# !O# #	P,PPP
c                   K   	 t        d|         t        j                  |       r
|  d {   } | 2 3 d {   }t        d|        | 7 #7 6 y # t        $ r}|xs d}t	        |||      d }~ww xY ww)Nz'received response in _async_streaming: zline in async streaming: rC  )r   r.  rE  )print_verboserI  rZ  rS  r   )r   r   r.  argslinerc  s         r   _async_streamingrh    s     
?zJKx(%~H" 	 	$5dV<=J &	(  
1=X 3 
 	

sR   A;(A AA AAAA AA A;	A8A33A88A;r;  r.  c           
          t        | t              ryt        | t        j                        r| t	        j
                  t        | dd      t        | dt        |             t        | d|xs d      |t        j                  dd      	      t        | t              r,| d
k(  r't	        j                  dt        | d|xs d      |      t        | t              r,| dk(  r't	        j                  dt        | d|xs d      |      t        | t              r,| dk(  r't	        j                  dt        | d|xs d      |      t        | t              rA| j                  d      r/t	        j
                  d| d|t        j                  dd      	      y y )Nstatus_code  textllm_providerrC  POSTzhttps://api.openai.com/v1/)methodurl)rj  messagerm  r   requestzlitellm.RateLimitErrorzthis is a mock rate limit errorrq  rm  r   z"litellm.ContextWindowExceededErrorz,this is a mock context window exceeded errorzlitellm.InternalServerErrorz$this is a mock internal server errorz Exception: content_filter_policyi  azure)rK  rS  rC  APIErrorr   MockExceptiongetattrr   httpxRequestRateLimitErrorContextWindowExceededErrorInternalServerError
startswithr;  r   r.  s      r   !_handle_mock_potential_exceptionsr    s   
 -+mV__5##}cBM63}3EF ~/B/Nh MM5QR
 	
 
M3	'M=U,U$$5 ~/B/Nh 
 	
 	=#&AA00B ~/B/Nh 
 	
 	=#&::)): ~/B/Nh 
 	
 
M3	'M,D,D*- ##! MM5QR
 	
-	'r   r-  c                 X    | du r&|#t        |       t        j                  dd|      y y NTzThis is a mock timeout errorrC  rs  )_sleep_for_timeoutr   Timeoutr-  r   r   s      r   _handle_mock_timeoutr    s;    
 t 37#oo2!
 	
 !4r   c                 t   K   | du r.|+t        |       d {    t        j                  dd|      y y 7 wr  )_sleep_for_timeout_asyncr   r  r  s      r   rH  rH    sH     
 t 3&w///oo2!
 	
 !4/s   868c                 <   t        | t              rt        j                  |        y t        | t              rt        j                  t        |              y t        | t
        j                        r-| j                   t        j                  | j                         y y y r   )rK  r   timerT  r   rx  r  connectr   s    r   r  r    sb    '5!

7	GS	!

5>"	GU]]	+0K

7??# 1L	+r   c                   K   t        | t              rt        j                  |        d {    y t        | t              r't        j                  t        |              d {    y t        | t
        j                        r5| j                  (t        j                  | j                         d {    y y y 7 7 W7 wr   )rK  r   rI  rT  r   rx  r  r  r  s    r   r  r    s     '5!mmG$$$	GS	!mmE'N+++	GU]]	+0KmmGOO,,, 1L	+ 	%+,s4   )B>B86B>"B:#AB>0B<1B>:B><B>Fr<  c
           
         	 |
j                  d      xs d}|d}t        ||	|        ||j                  |d       t        |t              st        |t
              rt        || |       t        t        t        t        t        t        f   |      }t        |t              r*|j                  d	      rt        j                  d
d| d      }|
j                  dd      }||st        j                   |       t        |t              rt        di |S t        |t              r|s|S t#        |      }t               }|du rVt               }|
j                  dd      du rt%        t'        ||| |      | d|      S t%        t)        ||| |      | d|      S t        |t        j                        r|t        t        |      }|||j*                  d   j,                  _        njg }t1        |      D ]S  }t        j2                  j5                  |t        j2                  j7                  |d            }|j9                  |       U ||_        t;        t        j                               |_        | |_        |r5|D cg c]  }tA        di | c}|j*                  d   j,                  _!        tE        |dtG        tH        tJ        tH        tJ        z                	 t        j2                  jM                  |       \  }}}}||jN                  d<   ||jQ                  |dd       |S c c}w # t
        $ r Y (w xY w# t
        $ r;}t        |tR        jT                        r|t        djW                  |            d}~ww xY w) aS  
    Generate a mock completion response for testing or debugging purposes.

    This is a helper function that simulates the response structure of the OpenAI completion API.

    Parameters:
        model (str): The name of the language model for which the mock response is generated.
        messages (List): A list of message objects representing the conversation context.
        stream (bool, optional): If True, returns a mock streaming response (default is False).
        mock_response (str, optional): The content of the mock response (default is "This is a mock request").
        mock_timeout (bool, optional): If True, the mock response will be a timeout error (default is False).
        timeout (float, optional): The timeout value to use for the mock response (default is None).
        **kwargs: Additional keyword arguments that can be used but are not required.

    Returns:
        litellm.ModelResponse: A ModelResponse simulating a completion response with the specified model, messages, and mock response.

    Raises:
        Exception: If an error occurs during the generation of the mock completion response.
    Note:
        - This function is intended for testing or debugging purposes to generate mock completion responses.
        - If 'stream' is True, it returns a response that mimics the behavior of a streaming completion.
    r   FNThis is a mock requestr  zmock-key)inputr   r~  zException: mock_streaming_errorz&This is a mock error raised mid-stream	anthropici  )rq  rm  r   rj  r:  T)r;  r   r  rC  completion_streamr   r.  logging_objr   	assistant)contentrole)indexrq  usageprompt_tokenscompletion_tokenstotal_tokensr   r.  zmy-secret-keyzmy-original-responser  r   original_responsez$Mock completion response failed - {}r   ),r   r  pre_callrK  r   rS  r  r   r   rY  rH   rA   r}  r   rv  r  rT  r3   rE   rQ   r`   choicesrq  r  rangeutilsrD   rG   appendr   createdr   r   
tool_callssetattrrN   r$   r#   rV   _hidden_params	post_callrC  ru  format)r   r   r  r  r;  r<  r-  loggingr.  r   r   is_acompletion
time_delaymodel_response_all_choicesi_choice	tool_callr^  rc  s                       r   mock_completionr    s   H}JM2;e 4M,uU "  
 mS)Zy-Q-+$7 #t],??@-
 mS)m.F.F-/
 $11@(	M ZZd3
!.JJz"mT* 1=11m]3$$?NMDQOT>02Nzz-/47*&I&m5TU'  (0 '  '"?"-uPQ# $,#  mW%:%:;S-098EN""1%--5L1X -!--//#MM11 -K 2  0  ##G,- &2N"!$TYY[!1$ "1< .:	:<N""1%--8
 	F"NE>?		
	+2==+I+IPU+I+V(A"AqCVN))*?@
 '"8   =<&  		  Ja)G>EEaHIIJsi   DL: L: 'AL: :L: C&L: <L&AL: 4L+ L: &L: +	L74L: 6L77L: :	M>6M99M>c                 P   i }	 t        t        t        | |            }|j                  d      *| j	                  d      r| j                  dd      } d}||d<   ||dk(  rd|d<   | j                  dd      } |dv rDt        j                  |       r/|r-|+|j                  d      dk7  rd|d<   | j                  dd      } || fS # t        $ rX}t        j                  dj                  |             | j	                  d      r| j                  dd      } d}||d<   Y d }~d }~ww xY w)	Nr   r.  modez
responses/ 	responsesxaizError getting model info: {})rC  rt  )r   rY  rO   r   r}  replacerS  rl   rP  r  r8   is_model_gpt_5_4_plus_model)r   r.  r'  r  r  
model_infor  rc  s           r   responses_api_bridge_checkr    sE    "$J&"1D

 >>&!)e.>.>|.LMM,3ED!%Jv).AU.J!,JvMM,3E 	2288?(NN6"k1(
6lB/u-  &;BB1EF
 MM,3ED!%Jv&s   A4C 	D%AD  D%c                 R    | dk(  ry| dk(  s
| dk(  s| dk(  rd|j                         v S y)Nr  Tazure_aibedrock	vertex_aiclaudeF)lowerr.  r   s     r   _should_allow_input_examplesr    s;     k)z))++-5;;=((r   toolc                     | j                         }|j                  dd        |j                  d      }t        |t              r'|j                         }|j                  dd        ||d<   |S )Ninput_examplesfunction)copyr   r   rK  rY  )r  	tool_copyr  s      r   _drop_input_examples_from_toolr    s^    		IMM"D)}}Z(H(D!==?%t, (	*r   c                     | y g }| D ]>  }t        |t              r|j                  t        |             .|j                  |       @ |S r   )rK  rY  r  r  )r  cleaned_toolsr  s      r   _drop_input_examples_from_toolsr    sT     } "M 'dD!  !?!EF  &	'
 r   r   r  c                     d| i}t         j                  D ]  }|j                  |      }||||<    |r7t        |t              r'dD ]"  }||v s||   |j                  |||          $ |S )a"  Build a complete model cost entry from kwargs and model_info.

    Collects all CustomPricingLiteLLMParams fields present in kwargs and
    merges metadata from model_info (mode, supports_prompt_caching, max_tokens)
    so that register_model() receives the full pricing configuration.
    litellm_provider)r  supports_prompt_cachingr  )r@   model_fieldsr   rK  rY  
setdefault)r.  r   r  entry
field_namevaluekeys          r   _build_custom_pricing_entryr    s     &':;E0== &


:& %E*&
 jT2D 	7Cj Z_%@  jo6	7 Lr   c)                 4   | t        d      t        |      }t        |      }t        |      }t	        |      }t        |&      }&t               }*|)j                  dd	      }+|+s|rd
dlm	}, d
dl
m}- d
dlm}. t        t        t         |.      |      }/|-j#                  |/      r |,di d| d|d| d|!d|d|d|d|d|d|d|d|
d|	d|d|d|d|d|d |d!|d"|d#|d$|d%|d&|d'|d(|d)|d*|d+|d,|d-|d.|"d/|#d0|$d1|%d2|d3|&d4|d5|'d6|(|)S |)j%                  d7d      }0|)j%                  d8d      }1|)j%                  d9d      }2t        t        t&           |)j%                  d:d            }3|)j%                  d;d<      }4|)j%                  d=d      }5|)j%                  d>d	      }6|)j%                  d?d      }7|)j%                  d@d      }8|)j%                  dAd      }9|)j%                  dBd      }:|)j%                  dCd      };|)j%                  dDd      }<|)j%                  dEd      }=t        t        t(           |)j%                  dFd            }>|)j%                  dGd      xs |}?|)j%                  dHd      }@|)j%                  dId      }A|)j%                  dJd      }B|?i }?||?j+                  |       t,        j.                  0	 t,        j.                  j1                         }C|?j+                  |C       |)j%                  dLd      }E|)j%                  dMd      }F|)j%                  dNd      }G|)j%                  dOd      }H|)j%                  dPd      }I|)j%                  dQd      }J|)j%                  dRd      }K|)j%                  dSd      }L|)j%                  dTd      }M|)j%                  dUd      }N|)j%                  dVd      }O|)j%                  dWd      }P|)j%                  dXd      }Q|)j%                  dYd      }R|)j%                  dZd      }S|)j%                  d[d      }T|)j%                  d\d      }U|)j%                  d]d      }V|)j%                  d^d      }W|)j%                  d_d      }X|)j%                  d`d	      }Y|)j%                  dad	      }Z|)j%                  dbd	      }[|)j%                  dcd      }\|)j%                  ddd	      }]t        t        t8           |)j%                  ded            }^t        t        t:           |)j%                  dfd            }_|)j%                  dgd      }`t=        |@xs d	ABh      }t?        |)i      }ai }btA        |8tB              rO|8jE                  ^aj      r<|8jG                  | |a^_|)j%                  dkd      |)j%                  dld      m      \  } }}c`rtI        |`dno      }	 |"|"}0EE}Ft        tB        |8      }d|=xs t,        jJ                  }=|=tM        di |*S |%3|%D ecg c]  }e|edp   | k(  sedq    }f}et-        jN                  ddr|fi|*S t,        jP                  r%| t,        jP                  v rt,        jP                  |    } tS               }gtU        |gdst-        jV                                |)j%                  dtd	      dnu rdt}7||} dt}7tY        | |7|0|$u      \  } }7}h}0t[        | |7|v      \  }i} t]        |7| w      st_        |      }|>&|?j+                  ta        jb                  |>|7x             g:te        gdy      r.|7gjf                  d?<   |)j%                  dzd      |gjf                  d{<   |xs |)j%                  d|d<      xs d<}tA        |th        jj                        rtm        |7      s|jn                  xs d<}n%tA        |th        jj                        stq        |      }KLM(t-        jr                  |7 d}|  tu        |7|)|;~      i       i }jOsPsQsRsSr6| i i}jOrOj|    dV<   PrPj|    dW<   QrQj|    dX<   RrRj|    dY<   SrSj|    dZ<   |)j%                  d      retw        ||)j%                  dCi       j%                  dAd      t        tx        t8        tx        t8        t8        f   f   |)j%                  d                  }d}k|7@|7tz        D lcg c]  }l|lj|                   c}lv r t        j                  | t{        |7            }kkkj                  |      }V tA        Vt&              rVd	u rt        |      }hh}$i d| d|!d|d|d|d|d|d|d|
d|	d|d|d|d|d|d |d!|i d| d?|7d"|d#|d$|d%|dMFd'|d(|d/|#d&|d|d*|d3|&d4|d,|d-|d|)j%                  d      i}mt        di |ma}ct        | |m|a|7|)j%                  d      dndnk      }nt,        j                  r0cj%                  dd      rcj                  d      }ot        ||o      }t        di db[d0|$d;|4d=|5d>|6d?|7d7|0d|)j%                  dd      dt,        jP                  d|9dB|:dC|;dD|<d[Td]dTMdRKdUNdSLdNGd`|)j%                  d`      d|)j%                  d      dI|)j%                  dI      d^Wd|)j%                  d      d|)j%                  d      d\Udjd|)j%                  d      d_Xd|)j%                  d      de^df_dQJd|)j%                  dd      d|)j%                  dd	      d/|#d|)j%                  d      d|)j%                  d      d|)j%                  d      d|)j%                  d      d|)j%                  d      d|)j%                  d      d|)j%                  d      dMFd|d|)j%                  dd	      d|)j%                  d      d|)j%                  d      }bt        tB        d      j                  | |n|b|7       |1s|2s|3r9|)j                  d:d       t        | ||||1|2d[|)j%                  dd      |7|3|      S ij%                  d      dk7  rt[        | |7|||      \  }i} ij%                  d      dk(  rRd
dlJmK}p tA        |t:              rd|v rt;        c      }c||cd*<   pj                  | ||?g|$|0[dcb|\|7t               |      S |7dtk(  rvd	}q\CtA        \t        j                        stA        \t        j                        rt        d/|#i\      }qt        d      xs dt}r|0xs t,        j                  xs t        d      }0|#xs/ t,        j                  xs t        d      xs t,        j                  }#|$xs< t,        j                  xs* t,        j                  xs t        d      xs t        d      }$cj%                  di       j                  dd      xs t        d      }sbj%                  dd      }t|?xs t,        j                  }?||cd2<   FFcdM<   t-        j                         j                  |       rlt,        j                  j                         }u|uj                         D ]  \  }v}w|vcvswcv<    t        j                  | ||?|$|0|#qsgt        cb|5d[|\|7      }xnlt,        j                  j                         }u|uj                         D ]  \  }v}w|vcvswcv<    t        j                  | ||?|$|0|#rqstgt        cb|5d[|\      }xcj%                  dd	      rdj                  ||$x|?|#|0d       xS |7dk(  rt        d      xs dt}r|0xs t,        j                  xs t        d      }0|0t        d      |#xs t,        j                  xs t        d      }#|$xs< t,        j                  xs* t,        j                  xs t        d      xs t        d      }$cj%                  di       j                  dd      xs t        d      }sbj%                  dd      }t|?xs t,        j                  }?||cd2<   t,        j                  j                         }u|uj                         D ]  \  }v}w|vcvswcv<    t        j                  | ||?|$|0t        t8        |#      rstgt        cb|5d[|\      }x|cj%                  dd	      s[dnu rdj                  ||$x|?|#|0d       xS |7dk(  r1	 t        j                  | ||?g|$|0[dcb|'|\|7t               |k      }x|xS |7dk(  r$d
dlemf}y |yj                  |       }z|zdk(  rfd
dlhmi}{ yj                  |0      }0|0t        d      yj                  |$      }${j                  | ||0|$gdcb|[||?xs t,        j                        }x|xS d| j                         v ryj                  |0      }0|0t        d      yj                  |$      }$|0rK|0j                  d}      }0|0j                  d      s)d|0v r|0j                  dd«      }|||d
   dz   }0n|0dz   }0|0dz   }0t        j                  | ||0[t,        j                  gt        cb|5t               |$d|?|\|7ë      }x|cj%                  dd	      s[dnu rdj                  ||$xī       x}x|xS yj                  |0      }0|yj                  |$      }$|?xs t,        j                  }?||cd2<   d| v rt        |      }	 t        j                  | ||?g|$|0[dcb|'|\|7t               |ƫ      }xcj%                  dd	      rdj                  ||$xdG|?i       xS |7dk(  s/d| v s+d| v s'|7t,        j                  v r|)j%                  d`      dnu rdt        _t        |0xs. t,        j                  xs t        d˫      xs t        d̫      xs d}0dt        _T        |$xs/ t,        j                  xs t,        j                  xs t        dΫ      }$|?xs t,        j                  }?||cd2<   t,        j                  j                         }u|uj                         D ]  \  }v}w|vcvswcv<    t,        j                  rt,        j                  t        _w        t        |      d
kD  r&d|d
   v rtA        |d
   d   t              r	|d
   d   }}n"dj                  |D ~cg c]  }~|~d   	 c}~      }}t        j                  | |gt        |$|7|0[\dcb|5|ѫ      }|cj%                  dd	      d	u r-[d	u r)Yd	u r%t-        j                         j                  gҫ      }cj%                  dd	      s[dnu rdj                  ||$dG|?i       }x|xS |7dk(  r1	 t        j                  | ||?g|$|0[dcb|'|\|7t               |k      }x|xS |7dk(  r1	 t        j                  | ||?g|$|0[dcb|'|\|7t               |k      }x|xS |7dk(  r1	 t        j                  | ||?g|$|0[dcb|'|\|7t               |k      }x|xS |7dk(  r1	 t        j                  | ||?g|$|0[dcb|'|\|7t               |k      }x|xS |7dk(  r|0xs! t,        j                  xs t        dث      xs d}0|$xs/ t,        j                  xs t,        j                  xs t        dګ      }$|?xs t,        j                  }?t,        j                  j                         }u|uj                         D ]  \  }v}w|vcvswcv<    t        j                  | ||[|0gcb|'|7||?t               |$d\۫      }x|xS |7dk(  r|0xs t,        j                  xs t        dݫ      }0|$xs t,        j                  xs t        dޫ      }$|?xs t,        j                  }?t,        j                  j                         }u|uj                         D ]  \  }v}w|vcvswcv<    t        j                  | ||[|0gcb|'|7||?t               |$d\۫      }x|xS |7dk(  rt,        j                   j                  | |0|$|?c      \  }0}$}?|0xs t,        j                  xs t        d      }0|0t3        d      |?xs t,        j                  }?t        j                  | ||[|0gcb|'|7||?t               |$d\k      }x|xS |7dk(  r|$xs= t,        j                  xs+ t,        j                  xs t        d      xs t        d      }$|?xs t,        j                  xs i }?	 t        j                  | ||?g|$|0[dcb|'|\|7t               |k      }x|xS |7dk(  r|?xs t,        j                  }?t,        j                  j                         }u|uj                         D ]  \  }v}w|vcvswcv<    t        j                  | ||?g[dcb||'\|7t               |$|0|      }x|xS |7dk(  r|0xs. t,        j                  xs t        d˫      xs t        d̫      xs d}0|$xs/ t,        j                  xs t,        j                  xs t        dΫ      }$|?xs t,        j                  }?||cd2<   t
        j                  | ||?g|$|0[dcb|\|7t               |      }x|xS |7dk(  r|$xs0 t,        j                  xs t        d      xs t,        j                  }$|0xs! t,        j                  xs t        d      xs d}0t        j                  | ||?g|$|0[dcb|'|\|7t               |k      }x|dj                  ||$|xī       |xS |7dk(  r|$xs t        d      xs t,        j                  }$|0xs! t,        j                  xs t        d      xs d}0t        j                  | ||0|7gt               dc|b|'[||$|?\k      }x|dj                  ||$|xī       |xS |7dk(  re|0xs t,        j                  xs t        d      }0t        j                  | ||0|7gt               dc|b|'[||$|?\k      }x|dj                  ||$|xī       |xS | t,        j                  v sx|7dk(  ss|7dk(  sn|7dk(  si|7dk(  sd|7dk(  s_|7dk(  sZ|7dk(  sU|7dk(  sP|7dk(  sK|7dk(  sF|7dk(  sA|7dk(  s<|7d k(  s6|7dk(  s0|7t,        j                  v st        j                  |7      sd| v r|0xs. t,        j                  xs t        d˫      xs t        d̫      xs d}0Ixs" t,        j                  xs t        d      xs d}I|It        _w        |$xs/ t,        j                  xs t,        j                  xs t        dΫ      }$|?xs t,        j                  }?|7dk(  rBd
dlm} d
dlm}  |       }|j                         } ||      }|rj+                  |       }||cd2<   t,        j                  r|:t!        |:      }|cdB<   t,        j"                  j                         }u|uj                         D ]  \  }v}w|vcvswcv<    t%        d      }	 r/t        j                  | ||0|7gt               dc|b|'[||$|?\k      }xn-t&        j                  | ||?gt        |$|0[dcb|5|j\I|7|'      }xcj%                  dd	      rdj                  ||$xdG|?i       xS |7d	k(  ry|$xs t,        j                  xs t        d
      }$|0xs# t,        j                  xs t        d      xs d}0t        j                  | ||0|7gt               dc|b|'[||$|?\k      }x|xS d| v s|7dk(  s| t,        j(                  v r|$xs? t,        j*                  xs, t,        j                  xs t        d      xs t        d      }|0xs# t,        j                  xs t        d      xs d}0jxs t,        j                  }jt-        | ||0gt        cb|5t               d|j[|?      }g|cj%                  dd	      dnu rdj                  |gī       g}x|xS d| v s|7dk(  s| t,        j.                  v r	 xS |7dk(  r|$xsG t,        j0                  xs4 t,        j                  xs" t2        j4                  j%                  d      }$jxs t,        j                  }j|0xs1 t,        j                  xs t        d      xs t        d      xs d}0t%        d      }|0s|0j                  d      s|0dz  }0nrt5        j6                  d       t        j                  | ||[|0gcb|'d||?t               |$d      }x|xS |7dk(  rD|$xsG t,        j0                  xs4 t,        j                  xs" t2        j4                  j%                  d      }$jxs t,        j                  }j|0xs1 t,        j                  xs t        d      xs t        d      xs d}0t%        d      }|0s|0j                  d      s|0dz  }0nrt5        j6                  d       t8        j                  | ||0[t,        j                  gt        cb|5t               |$d|?|\|7ë      }x|cj%                  dd	      s[dnu rdj                  ||$xī       x}x|xS |7dk(  r|$xs1 t,        j:                  xs t        d       xs t,        j                  }|0xs# t,        j                  xs t        d!      xs d"}0t=        | ||0gt        cb|5t               d#      }xd|cv rcd   dnu rt?        x| dd$      }xcj%                  dd	      s[dnu rdj                  ||$xī       x}x|xS |7d%k(  r|$xs? t,        j@                  xs, t        d&      xs t        d'      xs t,        j                  }|0xs# t,        j                  xs t        d(      xs d)}0tC        j                  | ||0gt        cb|5t               t,        jD                  d*      }gd|cv rcd   dnu rt?        g| d%d$      }x|xS g}x|xS |7d+k(  s|7d,k(  r{|$xs? t,        jF                  xs, t        d-      xs t        d.      xs t,        j                  }tI        jJ                  |       }t5        j6                  d/|        |d0k(  rB|0xs# t,        j                  xs t        d1      xs d2}0d3| v r=| jM                  d3d4      } n'|0xs# t,        j                  xs t        d1      xs d5}0|?xs t,        j                  xs i }?|?i }?||?j+                  |       t5        j6                  d6|  d7|0        t5        j6                  d8k        t        j                  | ||[|0gcb|'d+||?t               |kd9      }x|xS |7d:k(  r|$xs1 t,        jN                  xs t        d;      xs t,        j                  }|0xs# t,        j                  xs t        d<      xs d=}0tP        j                  | ||0gt        cb|5t               dd:j>      }g|g}x|xS |7d?k(  r|$xs1 t,        jR                  xs t        d@      xs t,        j                  }$|0xs# t,        j                  xs t        dA      xs dB}0tP        j                  | ||0gt        cb|5t               |$d||7jC      }x|xS |7dDk(  r|$xsk t,        jT                  xsX t2        j4                  j%                  dE      xs4 t2        j4                  j%                  dF      xs t,        j                  }|?xs t,        j                  }t        j                  | ||g|0[dcb|\|7t               |      }x|xS |7dGk(  r.t        j                  | ||?g|$|0[dcb|\|7t               |      }x|xS |7dHk(  rY|$xs t        dI      xs t,        j                  }$|0xs dJ}0t        j                  | ||?g|$|0[dcb|\|7t               |kK      }x|xS |7dLk(  rTdL}7tW        j                  | |g|0t        cbd|5t               dM      }gd|cv rcd   dnu rt?        g| dLd$      }x|xS g}x|xS |7dNk(  r|0xs* t,        j                  xs t3        jX                  dO      }0|$xs1 t,        j                  xs t,        jZ                  xs t        dP      }$|?xs t,        j                  }?	 t        j                  | ||[|0gcbdN||?t               |$d\Q      }xcj%                  dd	      rdj                  ||$xdG|?i       xS |7dRk(  r0t        j                  | ||?g|$|0[dcb|\|7t               |kK      }x|xS |7dSk(  rz|0xs# t,        j                  xs t        dT      xs dU}0|$xs? t,        j                  xs- t,        j\                  xs t        dV      xs t        dW      }$t        dX      xs dY}t        dZ      xs d[}|d\}|?xs t,        j                  }|rj+                         }?t,        j^                  j                         }u|uj                         D ]3  \  }v}w|vdk(  rdcv rcv   j+                  w       $wcv<   *vcvs/wcv<   5 | |d]c}t        j                  | ||[|0g|cb|'dS||?t               |$d\۫      }x|dj                  |t        j                  |xī       |xS |7d^k(  rY|0xs# t,        j                  xs t        d_      xs d`}0|$xs t,        j                  xs t        da      }$t        db      xs dY}t        dc      xs d[}|dd}|?xs t,        j                  }|rj+                         }?t,        j`                  j                         }u|uj                         D ]3  \  }v}w|vdk(  rdcv rcv   j+                  w       $wcv<   *vcvs/wcv<   5 | |d]c}t        j                  | ||[|0g|cb|'d^||?t               |$d\۫      }x|dj                  |t        j                  |xī       |xS |7dk(  sde| v s| t,        jb                  v r	 	 xS |7dfk(  rt        dg      |7dhk(  s|7dik(  racj                  djd      xs4 cj                  dkd      xs t,        jd                  xs t        dl      }cj                  dmd      xs4 cj                  dnd      xs t,        jf                  xs t        do      }cj                  dpd      xs! cj                  dqd      xs t        dr      }|$xs+ ti               xs t        ds      xs t,        j                  }|0xs t,        j                  xs t        dt      }0tk        cxs i       }tl        j                  | |gt        |b|5t               d[||7\|0|?u      }x|xS |7dvk(  rcj                  djd      xs4 cj                  dkd      xs t,        jd                  xs t        dl      }cj                  dmd      xs4 cj                  dnd      xs t,        jf                  xs t        do      }cj                  dpd      xs! cj                  dqd      xs t        dr      }|0xs t,        j                  xs t        dw      }0tk        cxs i       }to        | bx      }|tp        jr                  k(  r7tt        j                  | |gt        b|5t               |0d[|?j|\y      }gntp        jv                  k(  r8tl        j                  | |gt        b|5t               dd[||7\|0|?u      }gnStp        jx                  k(  r7tz        j                  | |gt        b|5t               |0d[|?j|\y      }gntp        j|                  k(  r6t~        j                  | |gt        b|5t               |0d[|?j|\y      }gntp        j                  k(  rTd
dzlm}  |       }bdj<   |bdm<   |bdp<   t        j                  | ||g|bt               d|0d[|\dv||?xs i {      }gnSt        j                  | |gt        b|5t               d[|      }gdcv rcd   dnu r[d	u rt?        g| dvd$      }x|xS g}x|xS |7d}k(  rcj                  dd      xs4 cj                  d~d      xs t,        j                  xs t        d      }|t        d      |0xsF cj                  d7d      xs2 cj                  d.d      xs t,        j                  xs t        d      }0|$xs1 t,        j                  xs t,        j                  xs t        d      }$t        j                  | |gt        cb|5t               d[|0j|$|      }d|cv rcd   dnu r[d	u rS }x|xS |7dk(  r|0xs= cj                  d7d      xs) cj                  d.d      xs t,        j                  xs d}0|$xs t,        j                  xs t        d      }$t-        j                  |      }t        j                  | ||t        cb|5t               d[|0j|$|      }d|cv rcd   dnu r[d	u rS }x|xS |7dv r1t        j                  | ||[|0gcb|7||?t               |$d\Q      }g|g}x|xS |7dk(  r3t        j                  | |gt        cbjU|5t               d[      }g|g}x|xS |7dk(  rjxs t,        j                  }jdcv rt5        j6                  d       cj                  d      }|j                         j                         }|j                  rj                  cd<   j                  rj                  cd<   j                  rj                  cd<   dzcvscdz   j                  j                  cdz<   t        j                  |       }|dk(  rE| jM                  dd4      } t        j                  | |jgcb|5t               d|?|[\|0|$      }x|xS dk(  rE| jM                  dd4      } t        j                  | ||[|0gcbd||?t               |$d\Q      }x|xS t        j                  | ||[|0gcbd||?t               |$d\Q      }xxS |7dk(  r7t        j                  | ||?gt        |$|0[dcb|5|j\t               d      }x|xS |7dk(  rj|$xs= cj                  dd      xs( t        d      xs t        d      xs t        d      }$|0xsk cj                  d|cj                  d7|cj                  d.d                  xs6 t        d      xs( t        d      xs t        d      xs t        d      }0cj                  d|cj                  dd            }d}|Zj%                  d|0      }0|j%                  d|j%                  d0|$            }$|j%                  d|j%                  dd            }cd<   t        j                  | ||[|0gcb|'d||?t               |$d\۫      }x|xS |7dk(  rdjxs t,        j                  }jt        j                  | ||jgt        cb|5t               d
      }gd|cv rcd   dnu rt?        g| dd$      }x|xS g}x|xS |7dk(  rht,        j                  xs |0xs t        d      xs d}0|$d|?vr
d|$ |?d<   t        j                  | ||[|0gcb|'d||?t               |$d\۫      }x|xS |7dk(  rt,        j                  xs |0xs t        d      xs d}0|$xsG t,        j                  xs4 t2        j4                  j%                  d      xs t,        j                  }$|$d|?vr
d|$ |?d<   t        j                  | ||[|0gcb|'d||?t               |$d\۫      }x|xS |7dk(  rCt,        j                  xs |0}0t        j                  | ||[|0gcb|'|7||?t               |$d      }x|xS |7dk(  r|$xs1 t,        j                  xs t,        j                  xs t        d      }$t        d      }|0xs( t,        j                  xs t        d      xs d d}0jxs t,        j                  }jt        j                  | ||[|0gcb|'d||?t               |$d      }x|xS |7dk(  s| t,        j                  v r|0xs t,        j                  }0d}7cj                  dd	      }t        j                  | ||0gt        |cb|5t               d\      }g|dnu r$gd   d
   d   d   }t?        || dd$      }x|xS g}x|xS |7dk(  s| t,        j                  v rD	 |d	u rt        |      nd}\t        j                  | ||?g|$|0[dcb|'||\|7t               |ƫ      }x|xS |7dk(  rDt,        j                  xs |0}0t        j                  | ||[|0gcb|'d||?t               |$d      }x|xS |7dk(  rk|$xs1 t,        j                  xs t        d      xs t,        j                  }$t        j                  | ||?g|$|0[dcb|\|7t               |t        K      }x	 |xS |7dk(  rk|$xs1 t,        j                  xs t        d      xs t,        j                  }$t        j                  | ||?g|$|0[dcb|\|7t               |t        K      }x	 |xS |7dk(  s| t,        j                  v r|$xs1 t,        j                  xs t        dë      xs t,        j                  }$|0xs# t,        j                  xs t        dī      xs d}0t        j                  | ||?g|$|0[dcb|\|7t               |t        K      }x	 |xS |7dk(  rt,        j                  xs |0xs d4}|d4k(  rt        dǫ      	 dj                  |D ~cg c]  }~|~d   	 c}~      }}t,        j                  j                  |?| |}g|
|||)j%                  dȫ      dɜdʜ|)j%                  di       ˫      }|j                         }	 |d   d
   d   d
   }|gj                  d
   j                  _        t        t        j                               |g_        | |g_        |g}x|xS |7t,        j                  v rd}t,        j                  D ]  }|d   |7k(  sd   } t        | |7Ы      t        [|ѫ      }|?xs t,        j                  xs i }? di d| d|dG|?dgdt        d0|$d7|0db[dddcdqbd=|5d|djdc\dt               }x|dnu rt?        x| |7d׫      S xS |7dk(  rd
dlm}  |       j                  |0xs t,        j                  |$xs t,        j                  ګ      \  }0}$|?xs t,        j                  }?t        j                  | ||[|0gcb|'|7||?t               |$d\۫      }x|xS t        | |7Ы      # t2        $ r#}Dt5        j6                  dKD        Y d}D~D;d}D~Dww xY wc c}ew c c}lw # t2        $ r'}Ddj                  ||$t9        D      dG|?i       |Dd}D~Dww xY w# t2        $ r'}Ddj                  ||$t9        D      dG|?i       |Dd}D~Dww xY wc c}~w # t2        $ r'}Ddj                  ||$t9        D      dG|?i       |Dd}D~Dww xY w# t2        $ r'}Ddj                  ||$t9        D      dG|?i       |Dd}D~Dww xY w# t2        $ r'}Ddj                  ||$t9        D      dG|?i       |Dd}D~Dww xY w# t2        $ r'}Ddj                  ||$t9        D      dG|?i       |Dd}D~Dww xY w# t2        $ r'}Ddj                  ||$t9        D      dG|?i       |Dd}D~Dww xY w# t2        $ r'}Ddj                  ||$t9        D      dG|?i       |Dd}D~Dww xY w# t2        $ r'}Ddj                  ||$t9        D      dG|?i       |Dd}D~Dww xY w# t2        $ r'}Ddj                  ||$t9        D      dG|?i       |Dd}D~Dww xY wc c}~w # t2        $ r}Dt        | |7D|*|)۫      d}D~Dww xY w(  a  
    Perform a completion() using any of litellm supported llms (example gpt-4, gpt-3.5-turbo, claude-2, command-nightly)
    Parameters:
        model (str): The name of the language model to use for text completion. see all supported LLMs: https://docs.litellm.ai/docs/providers/
        messages (List): A list of message objects representing the conversation context (default is an empty list).

        OPTIONAL PARAMS
        functions (List, optional): A list of functions to apply to the conversation messages (default is an empty list).
        function_call (str, optional): The name of the function to call within the conversation (default is an empty string).
        temperature (float, optional): The temperature parameter for controlling the randomness of the output (default is 1.0).
        top_p (float, optional): The top-p parameter for nucleus sampling (default is 1.0).
        n (int, optional): The number of completions to generate (default is 1).
        stream (bool, optional): If True, return a streaming response (default is False).
        stream_options (dict, optional): A dictionary containing options for the streaming response. Only set this when you set stream: true.
        stop(string/list, optional): - Up to 4 sequences where the LLM API will stop generating further tokens.
        max_tokens (integer, optional): The maximum number of tokens in the generated completion (default is infinity).
        max_completion_tokens (integer, optional): An upper bound for the number of tokens that can be generated for a completion, including visible output tokens and reasoning tokens.
        modalities (List[ChatCompletionModality], optional): Output types that you would like the model to generate for this request.. You can use `["text", "audio"]`
        prediction (ChatCompletionPredictionContentParam, optional): Configuration for a Predicted Output, which can greatly improve response times when large parts of the model response are known ahead of time. This is most common when you are regenerating a file with only minor changes to most of the content.
        audio (ChatCompletionAudioParam, optional): Parameters for audio output. Required when audio output is requested with modalities: ["audio"]
        presence_penalty (float, optional): It is used to penalize new tokens based on their existence in the text so far.
        frequency_penalty: It is used to penalize new tokens based on their frequency in the text so far.
        logit_bias (dict, optional): Used to modify the probability of specific tokens appearing in the completion.
        user (str, optional):  A unique identifier representing your end-user. This can help the LLM provider to monitor and detect abuse.
        logprobs (bool, optional): Whether to return log probabilities of the output tokens or not. If true, returns the log probabilities of each output token returned in the content of message
        top_logprobs (int, optional): An integer between 0 and 5 specifying the number of most likely tokens to return at each token position, each with an associated log probability. logprobs must be set to true if this parameter is used.
        metadata (dict, optional): Pass in additional metadata to tag your completion calls - eg. prompt version, details, etc.
        api_base (str, optional): Base URL for the API (default is None).
        api_version (str, optional): API version (default is None).
        api_key (str, optional): API key (default is None).
        model_list (list, optional): List of api base, version, keys
        extra_headers (dict, optional): Additional headers to include in the request.

        LITELLM Specific Params
        mock_response (str, optional): If provided, return a mock completion response for testing or debugging purposes (default is None).
        custom_llm_provider (str, optional): Used for Non-OpenAI LLMs, Example usage for bedrock, set model="amazon.titan-tg1-large" and custom_llm_provider="bedrock"
        max_retries (int, optional): The number of retries to attempt (default is 0).
    Returns:
        ModelResponse: A response object containing the generated completion and associated metadata.

    Note:
        - This function is used to perform completions() using the specified language model.
        - It supports various optional parameters for customizing the completion behavior.
        - If 'mock_response' is provided, a mock completion response is returned for testing or debugging.
    Nzmodel param not passed in.)r   )r  )r  )r6  )r&  _skip_mcp_handlerFr   )acompletion_with_mcp)LiteLLM_Proxy_MCP_Handler)	ToolParamr   r   r  r  r   r  r  r  r  r  r6  r  r	  r
  r  r  r  r  r  r  r  r  r  r  r  r  r  r7  r  r   r!  r"  r   r#  r   r$  r%  r&  r'  r(  r)  r9  r;  r<  r-  force_timeoutr   	logger_fnverboser.  r/  rQ  metadatar  proxy_server_requestr,  provider_specific_headerheadersensure_alternating_rolesuser_continue_messageassistant_continue_message"Failed to get proxy auth headers: r   r   cooldown_timecontext_window_fallback_dictr   
ssl_verifyinput_cost_per_tokenoutput_cost_per_tokeninput_cost_per_secondoutput_cost_per_secondinitial_prompt_valuerolesfinal_prompt_value	bos_token	eos_tokenpreset_cache_keyhf_model_namesupports_system_message
base_model(disable_add_transform_inline_image_blocktext_completionatext_completionr   r   zno-logr0  r2  litellm_system_prompt)r   r  r  r  r   )r0  r1  r3  r4  )r   r   r1  r0  r2  r3  r4  T)r   system_promptmerge_with_first_system
model_namelitellm_paramsdeploymentsr  rt  r   r.  r9  r   )r   r.  r'  r  )r  r.  r  aws_region_nameregion_namerequest_timeout/r.  r   r  model_file_id_mapping)r   model_idr
  r   providerallowed_openai_paramsadditional_drop_params)r   passed_paramsspecial_paramsr.  r  remove_sensitive_keysadd_provider_specific_paramsprovider_configfunctions_unsupported_model)r   r  litellm_call_idmodel_alias_mapcompletion_call_idno_logazure_ad_token_providerlitellm_trace_idlitellm_session_idcustom_prompt_dictlitellm_metadatadrop_params"merge_reasoning_content_in_choicesuse_litellm_proxyazure_ad_token	tenant_id	client_idclient_secretazure_usernameazure_passwordazure_scopelitellm_request_debugtpmrpmr   r  optional_paramsr  r.  r:  )
r  r  r;  r<  r  r   r:  r.  r-  r   r  r  )r   r.  r'  r  r  )responses_api_bridgesummary)r   r   r  r  r   r9  r   r  r-  r  r   r   r.  encodingr  )azure_client_paramsazure_clientAZURE_API_TYPEAZURE_API_BASEAZURE_API_VERSIONAZURE_OPENAI_API_KEYAZURE_API_KEY
extra_bodyAZURE_AD_TOKENr  )r   r   r  r   r9  r#  dynamic_paramsr"  r  re  r-  r  r  r  r   r   r   r.  )r   r   r  r   r9  r#  api_typer:  r"  r  r  re  r-  r  r  r  r   r   r   )r  r#  r9  )r  r   r  additional_args
azure_textz}api_base is required for Azure OpenAI LLM provider. Either set it dynamically or set the AZURE_API_BASE environment variable.)r   r   r  r   r9  r#  r;  r"  r  r  re  r-  r  r  r  r   r   r   deepseek)r   r   r  r  r   r9  r   r  r-  r  r(  r   r   r.  r0  r  r  r  )AzureFoundryModelInfoagents)AzureAIAgentsConfigz^Azure AI Agents requests require an api_base. Set `api_base` or the AZURE_AI_API_BASE env var.)r   r   r9  r   r  r  r-  r  r   r   r  r  r  z^Azure Anthropic requests require an api_base. Set `api_base` or the AZURE_AI_API_BASE env var.z/v1/messagesz
/anthropicrk   )r   r   r9  r   r  r  re  r-  r  r  r0  r   r  r  r   r   r.  r  z	command-r)r   r   r  r  r   r9  r   r  r-  r  r(  r   r   r.  r0  r  r=  zft:babbage-002zft:davinci-002rC  OPENAI_BASE_URLOPENAI_API_BASEhttps://api.openai.com/v1OPENAI_API_KEYr   )r   r   r  re  r   r.  r9  r   r   r  r-  r  r  r   r?  fireworks_aiherokuragflowr  groqGROQ_API_BASEzhttps://api.groq.com/openai/v1GROQ_API_KEY)r   r  r   r   r9  r  r-  r  r(  r.  r   r  r0  r   r  r   bedrock_mantleBEDROCK_MANTLE_API_BASEBEDROCK_MANTLE_API_KEYa2a)r   r9  r   r  r-  A2A_API_BASEzapi_base is required for A2A provider. Either provide api_base parameter, set A2A_API_BASE environment variable, or register the agent in the proxy with model='a2a/<agent-name>'.)r   r  r   r   r9  r  r-  r  r(  r.  r   r  r0  r   r  r   r  gigachatGIGACHAT_API_KEYGIGACHAT_CREDENTIALSsap)r   r   r  r  r   r  r-  r  r   r(  r   r.  r0  r   r9  r  aiohttp_openaicometapiCOMETAPI_KEYCOMETAPI_API_BASEhttps://api.cometapi.com/v1minimaxMINIMAX_API_KEYMINIMAX_API_BASEzhttps://api.minimax.io/v1)r   r   r9  r.  r  r0  r  r-  r   r  r(  r   r  r   r  r   r  hosted_vllmHOSTED_VLLM_API_BASEcustom_openai	deepinfra
perplexity
nvidia_nimcerebrasbaseten	sambanova
volcengineanyscaletogether_ainebiuswandbclarifaizft:gpt-3.5-turboOPENAI_ORGANIZATIONgithub_copilot)Authenticator)get_copilot_default_headers)EXPERIMENTAL_OPENAI_BASE_LLM_HTTP_HANDLER)r   r   r  r  re  r   r9  r   r  r-  r  r  r   r  r   r   r.  r(  mistralMISTRAL_API_KEYMISTRAL_API_BASEzhttps://api.mistral.ai/v1	replicateREPLICATE_API_KEYREPLICATE_API_TOKENREPLICATE_API_BASEzhttps://api.replicate.com/v1)r   r   r9  r  re  r-  r  r  r0  r   r  r  r   r  anthropic_textANTHROPIC_API_KEYANTHROPIC_API_BASEANTHROPIC_BASE_URLz%https://api.anthropic.com/v1/complete$LITELLM_ANTHROPIC_DISABLE_URL_SUFFIXz/v1/completezILITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/complete suffix)r   r  r   r   r9  r  r-  r  r(  r.  r   r  r0  r   r  r  z%https://api.anthropic.com/v1/messageszILITELLM_ANTHROPIC_DISABLE_URL_SUFFIX is set, skipping /v1/messages suffix	nlp_cloudNLP_CLOUD_API_KEYNLP_CLOUD_API_BASEzhttps://api.nlpcloud.io/v1/gpu/)r   r   r9  r  re  r-  r  r  r0  r   r  )r.  r  r   ALEPH_ALPHA_API_KEYALEPHALPHA_API_KEYALEPH_ALPHA_API_BASEz$https://api.aleph-alpha.com/complete)r   r   r9  r  re  r-  r  r  r0  default_max_tokens_to_sampler   r  cohere_chatcohereCOHERE_API_KEY
CO_API_KEYzCohere route: v2COHERE_API_BASEzhttps://api.cohere.com/v2/chatzv2/r  zhttps://api.cohere.ai/v1/chatzModel: z, API Base: zProvider Config: )r   r  r   r   r9  r  r-  r  r(  r.  r   r  r0  r   r  r  maritalkMARITALK_API_KEYMARITALK_API_BASEzhttps://chat.maritaca.ai/api)r   r   r9  r  re  r-  r  r  r0  r   r  r.  r  amazon_novaAMAZON_NOVA_API_KEYAMAZON_NOVA_API_BASEzhttps://api.nova.amazon.com/v1)r   r   r9  r  re  r-  r  r  r0  r   r  r   r.  r  huggingfaceHF_TOKENHUGGINGFACE_API_KEYocicompactifaiCOMPACTIFAI_API_KEYzhttps://api.compactif.ai/v1)r   r   r  r  r   r9  r   r  r-  r  r   r   r.  r0  r  r  r   )r   r   r  r9  re  r-  r  r   r  r0  r  
databricksDATABRICKS_API_BASEDATABRICKS_API_KEY)r   r  r   r   r9  r  r-  r  r.  r   r  r0  r   r  r   	datarobot
openrouterOPENROUTER_API_BASEhttps://openrouter.ai/api/v1OPENROUTER_API_KEY
OR_API_KEYOR_SITE_URLhttps://litellm.aiOR_APP_NAMEliteLLMzHTTP-RefererzX-Titler   vercel_ai_gatewayVERCEL_AI_GATEWAY_API_BASEhttps://ai-gateway.vercel.sh/v1VERCEL_AI_GATEWAY_API_KEYVERCEL_SITE_URLVERCEL_APP_NAME)zhttp-refererzx-titletogethercomputerr   zPalm was decommisioned on October 2024. Please use the `gemini/` route for Gemini Google AI Studio Models. Announcement: https://ai.google.dev/palm_docs/palm?hl=envertex_ai_betageminivertex_projectvertex_ai_projectVERTEXAI_PROJECTvertex_locationvertex_ai_locationVERTEXAI_LOCATIONvertex_credentialsvertex_ai_credentialsVERTEXAI_CREDENTIALSPALM_API_KEYGEMINI_API_BASE)r   r   r  re  r-  r  r  r0  r  r  r  gemini_api_keyr  r   r   r.  r   r9  r%  r  VERTEXAI_API_BASE)r   r  )r   r   r  re  r-  r  r  r0  r9  r  r  r  r  r   r  r  r   r   )VertexAgentEngineConfig)r   r  r   r  r-  r  r0  r   r9  r  r   r   r   r.  r  r  )r   r   r  re  r-  r  r  r0  r  r  r  r  r   	predibasepredibase_tenant_idPREDIBASE_TENANT_IDzMissing Predibase Tenant ID - Required for making the request. Set dynamically (e.g. `completion(..tenant_id=<MY-ID>)`) or in env - `PREDIBASE_TENANT_ID`.PREDIBASE_API_BASEPREDIBASE_API_KEY)r   r   r  re  r-  r  r  r0  r  r   r9  r  r   r#  r   r>  z/https://codestral.mistral.ai/v1/fim/completionsCODESTRAL_API_KEY)r  )r   r   r  re  r-  r  r  r0  r  r   r9  r  r   r   )sagemaker_chatsagemaker_nova	sagemaker)r   r   r  re  r-  r  r  r  r  r0  r  r   r  aws_bedrock_clientz'aws_bedrock_client' is a deprecated param. Please move to another auth method - https://docs.litellm.ai/docs/providers/bedrock#boto3---authentication.aws_access_key_idaws_secret_access_keyaws_session_tokenconversez	converse/)r   r   r  r  r-  r  r  r0  r  r%  r   r   r   r9  r   converse_likezconverse_like/watsonx)r   r   r  r  re  r   r9  r   r  r-  r  r  r   r  r   r0  r.  watsonx_textapikeyWATSONX_APIKEYWATSONX_API_KEY
WX_API_KEYrp  WATSONX_API_BASEWATSONX_URLWX_URLWML_URLwx_credentialswatsonx_credentialstokenwatsonx_tokenvllm)
r   r   r  r  re  r-  r  r  r0  r  ollamaOLLAMA_API_BASEhttp://localhost:11434AuthorizationzBearer ollama_chatOLLAMA_API_KEYtriton
cloudflareCLOUDFLARE_API_KEYCLOUDFLARE_ACCOUNT_IDCLOUDFLARE_API_BASEz.https://api.cloudflare.com/client/v4/accounts/z/ai/run/petals)r   r   r9  r  re  r-  r  r  r0  r  r   r  rq  	snowflaker  gradient_aibytezBYTEZ_API_KEYlemonadeLEMONADE_API_KEYovhcloudOVHCLOUD_API_KEYOVHCLOUD_API_BASE0https://oai.endpoints.kepler.ai.cloud.ovh.net/v1customzGapi_base not set. Set api_base or litellm.api_base for custom endpointstop_k)promptr  r  r  r  )r   r   )r  jsondataoutputr  custom_handlerr  )async_fnr  
custom_llmr  re  r  r-  r0  r  	langgraph)LangGraphConfig)r9  r   rD  r   )
ValueErrorrf   rg   ri   rj   rh   r   r   .litellm.responses.mcp.chat_completions_handlerr  /litellm.responses.mcp.litellm_proxy_mcp_handlerr  litellm.types.llms.openair  r   r   r   _should_use_litellm_mcp_gatewayr   boolr   updater   
proxy_authget_auth_headersrS  rl   warningr   rY  rt   rX   rK  rL  rM  get_chat_completion_promptrs   rR  rr   batch_completion_modelsr  rH   r  rN   rV   r  r  r  r+   get_provider_specific_headershasattrr  rx  r  rd   readr   register_modelr  ru   r   r   r  rI   get_provider_chat_config'translate_developer_role_to_system_rolerx   r    ra   add_function_to_promptrw   r   update_environment_variablesr  litellm.completion_extrasr.  r   _get_encodingrC  AzureOpenAIAsyncAzureOpenAIr   r^   r9  r#  r>   AZURE_DEFAULT_API_VERSIONr   	azure_keyr  AzureOpenAIO1Configis_o_series_model
get_configr   azure_o1_chat_completionsre  AzureOpenAIConfigazure_chat_completionsr  azure_text_completionsbase_llm_http_handler"litellm.llms.azure_ai.common_utilsr?  get_azure_ai_routelitellm.llms.azure_ai.agentsrA  get_api_baserU   r  rstripendswithsplit azure_anthropic_chat_completionsr  r{   +openai_text_completion_compatible_providersr;  
openai_keyr[  r   rO  listjoinopenai_text_completionsr\  groq_keyGroqChatConfigBedrockMantleChatConfig	A2AConfig"resolve_agent_config_from_registrygigachat_keyGenAIHubOrchestrationConfigsap_gen_ai_hub_chat_completionsbase_llm_aiohttp_handlercometapi_keyopen_ai_chat_completion_modelsopenai_compatible_providersr9   exists)litellm.llms.github_copilot.authenticatorro  (litellm.llms.github_copilot.common_utilsrp  enable_preview_featuresr]   OpenAIConfigr=   openai_chat_completionsreplicate_modelsreplicate_keyreplicate_chat_completionclarifai_modelsanthropic_keyosenvironrP  anthropic_chat_completionsnlp_cloud_keynlp_cloud_chat_completionrE   aleph_alpha_keyr   r  
cohere_keyr5   get_cohere_router  maritalk_keyopenai_like_chat_completionamazon_nova_api_keyhuggingface_keyr   getenvdatabricks_keyopenrouter_keyOpenrouterConfigVercelAIGatewayConfigtogether_ai_modelsr  r  r   rp   vertex_chat_completionr;   r:   PARTNER_MODELS%vertex_partner_models_chat_completionGEMINIGEMMAvertex_gemma_chat_completionMODEL_GARDEN#vertex_model_garden_chat_completionAGENT_ENGINE2litellm.llms.vertex_ai.agent_engine.transformationr  r   r  predibase_keypredibase_chat_completionsrK   codestral_text_completionssagemaker_llm_get_credentialsget_frozen_credentials
access_key
secret_keyr  metar  r4   get_bedrock_route bedrock_converse_chat_completionwatsonx_chat_completionvllm_handler
ollama_keycloudflare_api_keypetals_modelspetals_handlersnowflake_modelsr7   	bytez_keybytez_transformationlemonade_keylemonade_transformationovhcloud_modelsovhcloud_keyovhcloud_transformationmodule_level_clientpostr  r  rq  r  r   r  r  r   _custom_providerscustom_provider_mapr%   r   *litellm.llms.langgraph.chat.transformationr  $_get_openai_compatible_provider_infor   )r   r   r   r  r  r  r  r  r6  r	  r  r
  r  r  r  r  r  r  r  r   r  r  r  r  r  r  r  r'  r7  r%  r!  r"  r  r  r   r#  r   r$  r&  r(  r)  r   rf  skip_mcp_handlerr  r  r  tools_for_mcpr9  r;  r<  r-  r  r  r  r.  r/  rQ  r  r  r  r,  r  r  r  r  r  proxy_headersrc  r   r   r  r  r   r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r   r  r0  r2  r  r1  r  r-  r  mr  r  dynamic_api_keyresponses_api_model_infor  r  r  optional_param_argsprocessed_non_default_paramsr  r.  r:  r;  r"  r  configr   r   r   r?  azure_ai_routerA  partsr  rq  	_responsero  rp  copilot_authcopilot_api_keycopilot_headersopenai_metadatause_base_llm_http_handlerr=  disable_url_suffixrD  rF  rG  cohere_routerI  rL  
hf_headersopenrouter_site_urlopenrouter_app_nameopenrouter_headers_headersr  vercel_site_urlvercel_app_namevercel_headersr  r  r  r  
new_paramsmodel_router  vertex_agent_engine_configr#  _model_responsetext_completion_model_responser  credsbedrock_router  r  
account_idresp_stringrp  respresponse_jsonstring_responser  item
handler_fnr  s                                                                                                                                                                                     r   r   r     sR   J }566/BH)6E6;OK*5D.AH 8Dzz"5u=W	
 	8 Xhy&9:EB$DD E 

 ( ++!+ $+ ,	+
  + (+ + + +  .+ + &+ '<+ &+ &+  !+" "2#+$ #4%+& &'+( )+* !0++, -+. /+0 (1+2 %83+4 "5+6 *7+8 ,9+: "2;+< $=+> #4?+@ *A+B "C+D (E+F  G+H &I+J ,K+L "M+N $6O+P  .Q+R /LU+ +X zz*d+H28**_d2SMjj!2D9O

>4(HILJJ4M

;-IjjE*G **%:DA **%:DA	D$	Bzz*d+HL$/J!::&<dC

;-I#'(&**5OQU*V  jjD):]G/5zz"D0 BHB LR::$dL  }%%	M#..??AMNN=) **tK **]D1KJJ5M#)::.Ld#S ::nd3LL$/J!::&<dC"JJ'>E"JJ'>E#ZZ(@$G!::&<dCJJw%E$8$?

;-I

;-Izz"4d;JJ5M$jj)BDIL$/J/5zz2D0, jj!2E:Ozz"4e<**]E2KZZ$'FZZ%(FXc]FJJ{D$ABIHTNFJJ7I4,PQ"JJ'>E&!9!BU3#=	H ;&IN %'89>>4F 	? 	
  ::1-ND9!::&6= ; 
		
 0/$(
C0
H"%K%)*;=P%Q8!8!8	 ,4t44!-7()1\?e;S"#K  22S{SdSS""u0G0G'G++E '9JJw&$.")$!E")@P 3	A
="OX +E 31+
' % , 35
 4%@E#/NN+II-E(; %'.BR*SCVN))*?@;A::!4<N))-8
 FVZZ(93?F3gu}}-6L7
 ll)cGGU]]3GnG !,1F1R".""*+1UG46Q,?%#-7   !"'#DX"5)*@A5:"5)'2!BT"5)*>?9B"5)+69B"5)+6::-.:!L"599$E&*d38n,-vzz:Q/R'H 15*/B+7G
'HNNG
 0
 4LLl3F&GO &&NN! O H
 $/2D9'50,h?H&%G%
%
]%
 ;%
 U	%

 %
 f%
 n%
 D%
 *%
 $%:%
 *%
 *%
 U%
  0%
  !2%
  *!%
" D#%
& U'%
( "#6)%
* +%
, D-%
. U/%
0 ;1%
2 ;3%
4 5%
6 L7%
8 ;9%
: "#6;%
< =%
>  0?%
@ A%
B !"4C%
D  !2E%
F LG%
H $VZZ0G%HI%
L . 
!
%7
 (F-- 3#)::.F#G"&)-+	(
$ ))o.A.A)4/
 +:*=*=-+' ,!-HH
 , 4
#4
4
 (4
  	4

 4
 !44
 4
 #JJ'8$?4
 $334
  "4
 4
 "4
 "64
 .4
 4
  #8!4
" "6#4
$ $:%4
& #8'4
( ()4
* #JJ'89+4
, %+JJ/H$I-4
. #)**-D"E/4
0 "14
2 $ZZ(:;34
4  &zz*>?54
6 (74
8  294
: $ZZ(:;;4
< 6^=4
> 

=1?4
@  A4
B .C4
D "E4
F 06zz4d0G4
L %jj)<eDM4
N $O4
P "::&67Q4
R jj-S4
T jj-U4
V !**_5W4
X "::&67Y4
Z "::&67[4
\ 

=1]4
^ $_4
` a4
b #)**-De"Lc4
d 

5!e4
f 

5!g4
j 	(EE8) 3 	F 	
 O|JJ~t,"+ /'!::lD9$7) ( $''/;>.H$7#5!1/+$e $''/;>F*D1iCS6S"&"76F 23'22!-!'# /-$7& 3  $ ') #N!66#5#56ff&=&=>!<)6(D!'"
 ""23>wHS7#3#3SzBR7SH  5&&5!"565 44	   3??3$$3 ""893 "/2  -00rBFF $ 2 01  '5&8&8)4'# 0G(3@0&1<.**,>>U>K 44??A"LLN /DAq0-.*	/ 5??%##% +#1#1#1"/$3#1' ' +#!(;% @ , !22==?"LLN /DAq0-.*	/ 2<<%##% +%#1#1,C#1"/$3#1' ' +#!' = , ""8U3!!"#&.#*'2$,%	 " 	bR OR !L0%&67B7HW7#3#3W~FV7WH  T 
  7&&7!"56   3??3$$3 ""893 "/2  -00rBFF $ 2 01  '5&8&8)4'# 0G(3@0 ..99;F +1_,)*OA&	+ .88!! k2!-(?-+ /-##'% 9 H* ""8U3{d7J!!"#&.#*'2$,%	 " 	BP oO !J.0;;%##1#% + '$3#1#1#!(;*_!$3# < fO mN !J.P2EEeLN )L0==hG#$K  0;;GD.99%%##1 '$3#1# +!#6w : JN kM U[[]*0==hG#$K  0;;GD 's3H#,,^<'83$,NN<$CE',Qx,'>H'/,'>H#+n#<;FF%% +'.'A'A#1"/$3#1'*_# '##!(;# G & #&&x7;$;N%%& '*2 & 
 $LL GL 1==hG/;;GD!4W__ ,7DOO4 %'?RH4??#!) ''5 '!)$/$+(7'5'5 '%,?!.%!  @  H8 #&&x7%%& '*2)2G(<	 & lJ _J  #;;5(5("BBC

,-5&FO  /##//0/ /0/ /  "&F  0??0%%0 ./	  0G(3@0 77BBDF +1_,)*OA&	+
 ##&-&:&:# H!!,x{95t< "!Y/X"N'79#5"NO 0::!-+$7!'# /-# ; I$  ##He4=5(#u, $>>@ff$-^ g 	 ""8U3{d7J!!"#&/%.$8	 "  !HlG kG !N20;;%##1#% + '$3#1#1#!(;*_!$3# < dG mF !H,0;;%##1#% + '$3#1#1#!(;*_!$3# < hF qE !I-0;;%##1#% + '$3#1#1#!(;*_!$3# < jE uD !E)0;;%##1#% + '$3#1#1#!(;*_!$3# < nD wC !F* 4##4o.4 4	   .??.##. n-	  0G ++668F +1_,)*OA&	+ -77!'!- /--$7&#! 8 HBC _B !$44UG,,U
;T0U  XXJ?W4XG0G44??AF +1O+)*OA&+ -77!'!- /--$7&#! 8 HJB gA !E) !!DD! / E 	  U7#3#3U~n7UHX  0G,77!'!- /--$7&# /# 8 Hp@ K@ !J.  6??6''6 016 45  66BG0;;%##1#% + '$3#1#1#!(;*_!$3# < n? u> !E)0G88CCEF +1_,)*OA&	+ 7AA!-'# /--$7&!! B H`> }= !$44  /##//0/ /0/ /   0??0%%0 ./	  0G(3@0/::!-!'# /-$7& ; HR= q< !J. #''#!.1# ??	   1##1!"561 1	  -77!-!'# /--$7& /# 8 H* 8  f; a; !I-U0A!BUgooG  /##/!"45/ /	  -77!!$7-&# /--' /# 8 H& 8  f: a: !M1VG,,V?U0V  -77!!$7-&# /--' /# 8 H& 8  p9 i9 W;;;"o5"k1"l2"l2"j0"i/"k1"l2"j0"h."m3"h."g-"j0"g&I&II#**# "U*
  /##//0/ /0/ /   ''34 	  #/F  0??0%%0 ./	  0G #&66S  -".":":"<"=o"N #**=9 /(3@0 //H4H"8"B".2AOJ/ ))446F +1_,)*OA&	+ )8;)%2,4??#!)!),?'5!.$+(7 ''5'5$/% ' '%(7#  @  H(  7AA#!) ''5&3 '!)$/$+(7'5"+ '+=%%1,?'5%  B  H< ""8U3!!"#&.%.$8	 " T5 G5 !I-QQJ?P4QG /##/01/ /	  -77!!$7-&# /--' /# 8 Ht4 M4 5 "k1000  5((5??5 125 34   2##2232 2	  "4!Qw7Q7Q6!!-+ /-#&%##5'N" ""8U3t;!!")&4 "  &Hn2 k2 %"j0///b2 a2 !$44 7((7??7 ::>>"56	  "4!Qw7Q7Q ;##;23; 23; ;  "11W!X$* )).9N*#$$_ -77!'!- /--$4&# 8 Hf1 E1 !K/ 7((7??7 ::>>"56	  "4!Qw7Q7Q  ;##;23; 23; ;  "11W!X$* )).9N*#$$_ 2<<!!'#*#=#=-+ /-#&#$7# = H& ""8U3{d7J!!"#&. " 
  HR/ Q/ !K/ #((#12# ??	   5##5235 5	  1!!-+ /-#&%#H ?*x/HD/P.(3 '	 ""8U3{d7J!!"#&. "   Ht- s- !M1 #**#34# 23# ??   :##:45: :	  )33!!-+ /-#&-4-?-?'#N ?*x/HD/P."(5 '	  %Hb, a, !M15HH5T #%%#!"23# ",/# ??  +;;EBL  >,!@At# 8''8%&788 8	  E>!MM%4E  7''7%&787 7	  66BG(}-  75'hZ!HI  #4_4E!FG,77!'!- /--$1&" /#! 8 HR+ o* !J. #''#01# ??	   2##2122 2	  9CC!!-+ /-#&$#$.#5 D N  &Hp) o) !M1 #..#!"78# ??	   4##4!"894 4	  3==!!-+ /-#&#$7#5 > HT) u( !M1 #**#::>>*-# ::>>"78# ??  !3GOOJ,77!"-'!'# /-$7& 8 Hb( A( !E),77!-!'# /-$7& 8 H~' ]' !M1S>*?@SGOO   @#@H -77!-!'# /-$7& /! 8 HL' i& !K/"-&11!-!+ /-#&#N ?*x/HD/P."(3 '	  %Hx% w% !L0 4##49923   4??4))4 23	  0G0;;!% +%#1$3#1(4##*_# '! < 6 ""8U3!!"#&.%.$8	 " V$ I$ !K/,77!-!'# /-$7& /! 8 HF$ c# !L0 2##2!"782 2	   0??0))0 ""670 ",/  #-]";"S?S",]";"Hy !4."
 1'//H"))(3(G --88:F +1$#6'*11!4-.*o-)*OA&+ #LOLD -77!'!- /--$0&#! 8 H& (  b! ]! !$77 5##5!">?5 5	  U7??Uj9T.U  )):;S?SO():;HyO !0*N
 1'//H%%h/$G 22==?F +1$#6'*11!4-.*o-)*OA&+ #LOLD -77!'!- /--$7&#! 8 H& (  d ]  =0"e+333 N M !F* v  !$448Kx8W##$4d; 2"&&':DA2))2 01	   ##$5t< 3"&&';TB3**3 12	   ##$8$? 6"&&'>E645   #')#n-# ??	   T7#3#3TzBS7TH'(=2>J-88!-+ *-#& 20#5-#'$7!%' 9 HL a !K/##$4d; 2"&&':DA2))2 01	   ##$5t< 3"&&';TB3**3 12	   ##$8$? 6"&&'>E645   V7#3#3VzBU7VH'(=2>J3NK 0???!F!Q!Q%#1"/$.#1'*_%$6#4'9 ' +#'9#!% "R "(  2 9 99!7!B!B%#1"/$.#1'*_$6#4'9#' ' +#(;!%")' "C "*  2 8 88!=!H!H%#1"/$.#1'*_%$6#4'9 ' +#'9#!% "I "(  2 ? ??!D!O!O%#1"/$.#1'*_%$6#4'9 ' +#'9#!% "P "(  2 ? ?? .E-F* 4E/04F017I34!6!A!A!%#1$.#1*_ % ' +#!(3$>#Mr! "B "& "6!@!@%#1"/$.#1'*_$6#4'9 ' +"" /'1T9#u,2&,7$+	 H $O%HF E !K/##K6 5"&&'<dC5..5 34	     q 
  4"&&z484"&&z484 ##4 23   3??3((3 12	  9CC!-+ /-#&#'!#5# D O& O+#H-55(&&&H\ [ !$?? E"&&z48E"&&z48E ##E E  SSJ?R4SG-4-K-K.* 9CC!=+ /-#&#'!#5 D O$ O+#H-55(&&&HN M !$HH 3==!'!- /-$7&# > N& &H^ ] !K/*55!-+ /-#5+#&#' 6 N  &Hx w !I-!3!Qw7Q7Q#6&& n &5%8%89M%N"*;;=TTV##;@;K;KO$78##?D?O?OO$;<;;;@;;O$78%_<&'89A +//;; $) ->>uEM
*k26;FF%'9#1$3#1'*_ '")# +!%# G | [ /1&6;0;;!% +%#1$3#1(1##*_# '! < V s 1;;!% +%#1$3#1(1##*_# '! < r Q !I-.99!-+!'# /-##5&$-# : HN i !N2 0"&&x60!"230 ""340 ",/   
-"&&#''"O$7$7
D$I
- ""45
- "-0
- "(+
- "),  -00 ##)4N $(E))--eX>(,,n00GD '**"&&'  +0(,77!'!- /--$2&#! 8 HL i !F*!3!Qw7Q7Q)44!#5-+ /-#&#N O+0IT0Q /"(. '	   &Hp o !H,   ,,/0, ,	  "g'E-4WI+>(,77!'!- /--$,&#! 8 HZ u !M1   ,,/0, ,	   #%%#::>>"23# ??	  "g'E-4WI+>(,77!'!- /--$1&#! 8 HR m !H,''38H,77!'!- /--$7&# 8 Hh G !L0 4--4??4 23	  $$;<J Y##Y34Y DJ<xX	  "4!Qw7Q7Q,77!'!- /--$0&# 8 Hf
 C
 !H,9N9N0N37#3#3H"*$((59F+66!!-+ /-#&#N ~,Y7:9EiP.(0 '	  %HJ	 I	 !K/5G<T<T3T4:eOK0  1;;%##1#% + '$3#1#1#!(;*_!! < ~ G !M1''38H,77!'!- /--$1&# 8 HB _ !G+ #$$#!/2# ??	  -77!-!'# /-$7& 4! 8 H& h g !J. #''#!"45# ??	  -77!-!'# /-$7& 7! 8 H& p m !J.%7;R;R2R #''#!"45# ??	   F##F!"56F F	  -77!-!'# /-$7& 7! 8 H& h e !H,""4h4"C{cRi ] " XXJgwy1JKF..33"#)(&0'2!&!'G!4
 jjr2
 4 D !IIKM ,F3A6x@CO8GN""1%--5%(%5N"#(N %Hn i  7#<#<< 37N33 <
#'::%)*:%;N< %,5H 
 0$VJ 66BG " !    .	
 ,   " ( $ !0  . $   $6   '!H$ ~*&.(; '	 X K !K/R
  !FF!5W%5%527?? G  0G,77!'!- /--$7&#! 8 H.  )1D Qb  	M""%Gs#KLL	MvPG
P  !!"#&)!f%.$8	 "  | ! %%& '*-a&)2G(<	 &  GL #Oz  !!"#&)!f%.$8	 "  <  !!"#&)!f%.$8	 "  >  !!"#&)!f%.$8	 "  <  !!"#&)!f%.$8	 "  @  !!"#&)!f%.$8	 "  \	  !!"#&)!f%.$8	 "  ~  !!"#&)!f%.$8	 "  V  !!"#&)!f%.$8	 "  d Kv  
 3 "
 	

sb  /Cz6 :9DC7 4DC7 :C{%C{%DC7 )JDC7 -C{* NDC7 	BDC7 F/DC7 A+DC7 2ADC7 D8DC7 BA0DC7 C<DC7 D.C{/ D0DC7 D2BDC7 F:C;DC7 J6ADC7 L-C|" L1*DC7 MDDC7 QA3DC7 SC}SBDC7 U>DC7 V.C} V2DC7 V4DC7 V:.C~ W(DC7 W*DC7 W0.C  XDC7 X DC7 X&.C3 YDC7 YB)DC7 \ 5DC7 \6BDC7 _
5DC7 ` B DC7 b!ADC7 d .D@& d.DC7 d0ADC7 f6DC7 f:B3DC7 i.B!DC7 lBDC7 nA)DC7 p	G#DC7 w-DC7 xADA y *DC7 zA>DC7 |
C.DC7 9 DC7 @DDC7 D(E
DC7 I3CDC7 MCDC7 PDC7 PFDC7 VBDC7 X4BDC7 [	B6DC7 ^ 3DC7 ^4ADC7 `ADC7 a)DC7 a-A=DC7 c+.DB d*DC7 e5DC7 e:DDC7 jA DC7 k;C>DC7 o:A DC7 q DC7 q<E?DC7 w<K;DC7 C8DC7 C<DDC7 HDC7 HCDC7 K"DC7 K&5DC7 L8DC7 MEDC7 RA
DC7 S(/DC7 T<DC7 UE0DC7 [A%DC7 \,DC7 \0A-DC7 ^B8DC7 aADC7 b B6DC7 eBDC7 g/DC7 g3DC7 hADB? iDC7 iA	DC7 jA0DC7 lA0DC7 m<B*DC7 p'ADC7 q+DC2q7CDC7 u2DC7 u6BDC7 xDC7 xBDC7 z'DC7 z6	C{"z?C{{C{"{%
DC7 {/	C|{8"C||C||DC7 |"	C}|+"C}}C}}DC7 }	C~
}#"C~~C~
~
DC7 ~	C~=~"C~8~8C~=~=DC7  	C0	"C++C00DC7 3	D@#<"D@@D@#@#DC7 @&	DA@/"DAADAADC7 A	DB	A""DBBDB	B	DC7 B	DB<B"DB7B7DB<B<DC7 B?	DC/C"DC*C*DC/C/DC7 C7	DDD DDDDDc                     	 ddl }|j                  dd      }d|d<   d|d<   |j                  dd      }|j                  d	t              }|d
k(  r5|j	                  |j                  dd      |j                  |      d      }n"|j	                  |j                  |      d      } ||g| i |S # t        $ r}t        d|       d}~ww xY w)z8
    Executes a litellm.completion() with 3 retries
    r   N?tenacity import failed please run `pip install tenacity`. Errorr      r   retry_strategyconstant_retryoriginal_functionexponential_backoff_retryrk   
   
multipliermaxTr   r6  reraiser6  r  )tenacityrS  r   r   Retryingwait_exponentialstop_after_attemptrf  r   r  rc  r   r  r  retryers           r   completion_with_retriesr  D  s   
 **]A.KF=F=MSZZ*NN 

#6
C44##**aR*@,,[9 $ 
 ##,,[94 $ 
 $6t6v66/  
MaSQ
 	

s   B2 2	C;C		Cc                    K   	 ddl }|j                  dd      }d|d<   d|d<   |j                  dd      }|j                  d	t              }|d
k(  r5|j	                  |j                  dd      |j                  |      d      }n"|j	                  |j                  |      d      } ||g| i | d{   S # t        $ r}t        d|       d}~ww xY w7 #w)z{
    [DEPRECATED]. Use 'acompletion' or router.acompletion instead!
    Executes a litellm.completion() with 3 retries
    r   Nr  r   r  r   r  r  r  r  rk   r  r  Tr  r  )r  rS  r   r   AsyncRetryingr  r  r  s           r   acompletion_with_retriesr  d  s
    

 **]A.KF=F=ZZ 02BCN

#6
C44((**aR*@,,[9 ) 
 ((,,[94 ) 
 *<T<V<<<)  
MaSQ
 	

( =s4   CB< B/C7C8C<	CCCCc                     	 ddl }ddlm} |j	                  dd      }d|d<   d|d<   |j	                  dd	      }|j	                  d
|      }|dk(  r5|j                  |j                  dd      |j                  |      d      }n"|j                  |j                  |      d      } ||g| i |S # t        $ r}t        d|       d}~ww xY w)z5
    Executes a litellm.responses() with retries
    r   Nr  )r  r   r  r   r  r  r  r  rk   r  r  Tr  r  )r  rS  litellm.responses.mainr  r   r  r  r  )	rf  r   r  rc  r  r   r  r  r  s	            r   responses_with_retriesr    s   
 1**]A.KF=F=MSZZ*NN 

#6	B44##**aR*@,,[9 $ 
 ##,,[94 $ 
 $6t6v663  
MaSQ
 	

s   B4 4	C=CCc                    K   	 ddl }ddlm} |j	                  dd      }d|d<   d|d<   |j	                  dd	      }|j	                  d
|      }|dk(  r5|j                  |j                  dd      |j                  |      d      }n"|j                  |j                  |      d      } ||g| i | d{   S # t        $ r}t        d|       d}~ww xY w7 #w)z6
    Executes a litellm.aresponses() with retries
    r   Nr  )
aresponsesr   r  r   r  r  r  r  rk   r  r  Tr  r  )r  rS  r  r  r   r  r  r  )	rf  r   r  rc  r  r   r  r  r  s	            r   aresponses_with_retriesr    s    
 2**]A.KF=F=ZZ 02BCN

#6
C44((**aR*@,,[9 ) 
 ((,,[94 ) 
 *<T<V<<<-  
MaSQ
 	

, =s4   CB> B1C9C:C>	CCCCc                    K   t        j                         }t        |       dkD  r| d   n|d   }d|d<   |j                  dd      }	 t	        t
        g| i |}t        j                         }t	        |j                  |      }t        |||j                  dd            \  }}}}|j                  d|       d{   }	d}
t        |	t              rt        di |	}
n2t        |	t              r|	}
nt        j                  |	      r
|	 d{   }
|
+t        |
t              rt        |
d	      r||
j                   d<   |
t#        d
      |
S 7 7 B# t$        $ r}|xs d}t'        |||| |      d}~ww xY ww)ay  
    Asynchronously calls the `embedding` function with the given arguments and keyword arguments.

    Parameters:
    - `args` (tuple): Positional arguments to be passed to the `embedding` function.
    - `kwargs` (dict): Keyword arguments to be passed to the `embedding` function.

    Returns:
    - `response` (Any): The response returned by the `embedding` function.
    r   r   T
aembeddingr.  Nr9  r8  r  zCUnable to get Embedding Response. Please pass a valid llm_provider.rC  rD  r   )rI  rJ  rO  r   r   	embeddingrU  rV  rW  rV   rX  rK  rY  rF   rZ  r  r  r  rS  r   rf  r   rB  r   r.  r_  r`  ra  r^  rb  r   rc  s               r   r  r    s     !!#D4y1}DG&/EF< **%:DA,
y24262 &&(#CGGT2'7 3ZZ
D1(
$1 #2249JKK04mT*(9=9H'89$H  /**H 8%67"23=PH##$9:U  ) L +  
1=X 3 "
 	

sP   AE8A5E <E=AE E?E E8E E 	E5E00E55E8r   
dimensionsencoding_formatr9  r;  cachingr  Tc                     y r   r   r   r  r  r  r   r9  r#  r   r;  r  r  r.  r  r  r  r   s                   r   r  r        , r   )r  c                     y r   r   r  s                   r   r  r  !  r  r   c                 (   |j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  d      }|j                  dd      }|j                  dd      }|j                  d	d      }|j                  d
d      }|j                  dd      xs |}|i }||j                  |       t        j                  0	 t        j                  j                         }|j                  |       |j                  dd      }|j                  dd      }|j                  dd      }g d}d	d
gt        z   }||z   } |j                         D !"ci c]  \  }!}"|!| vs|!|" }#}!}"t        | |||      \  } }}$}|$|$}|j                  dd      }%t        d| |
||||%d|#}&|||7t        j                  | d|  t        |||j                  d            i       t        di |}'|}(|(j!                  | |
|&|'|       |t#        | |      S 	 d})|du s|dk(  r|xs t        j$                  xs t'        d      }|xs/ t        j(                  xs t'        d      xs t        j*                  }|&j                  dd      xs t'        d      }*|xs/ t        j,                  xs t        j.                  xs t'        d      }|t1        d      t2        j5                  | |||||*||(|t7               |&||||xs ||'       })nr|d!k(  r?|xs t        j,                  }t8        j5                  | |||||(|t7               |&|||'"      })n.|d#k(  s#|d$k(  s|d%k(  s|d&k(  s| t        j:                  v r||xs. t        j$                  xs t'        d'      xs t'        d(      xs d)}t        j<                  xs t'        d*      xs dt>        _        |xs/ t        j,                  xs t        j@                  xs t'        d+      }|
|i k7  r||&d
<   |||&d,<   nd|&d,<   d}tB        j5                  | ||||(|t7               |&||||-      })n1|d.k(  r}|xs t        j$                  xs tE        d/      }|xs/ t        j,                  xs t        jF                  xs tE        d0      }tH        j5                  | ||||(|t7               |&||1
      })n|d2k(  rp|xs t        j$                  xs t'        d3      }|t        j,                  xs t'        d4      }t8        j5                  | |||||(|t7               |&|||'|xs i 5      })n:|d6k(  s
|d7k(  s|d8k(  r|xs t        j$                  xs t'        d9      }|3|xs/ t        j,                  xs t        jJ                  xs t'        d:      }|
|i k7  r||&d
<   tL        j5                  | ||||(|t7               |&||1
      })n|d;k(  s|d<k(  rp|xs< t        jN                  xs* t'        d=      xs t'        d>      xs t        j,                  }+|i }t8        j5                  | ||||+|(|t7               |&|||'|5      })n&|d?k(  r|xs! t        j$                  xs t'        d@      xs dA}|xs< t        j,                  xs* t        jP                  xs t'        dB      xs t'        dC      }tE        dD      xs dE},tE        dF      xs dG}-|,|-dH}.|xs t        jR                  }/|/r|.j                  |/       |.}t8        j5                  | |||||(|t7               |&|||'|5      })nD|dIk(  r|xs! t        j$                  xs t'        dJ      xs dK}|xs* t        j,                  xs t'        dL      xs t'        dM      }t8        j5                  | |||||(|t7               |&|||'|5      })n|dNk(  rf|xs/ t        jT                  xs tE        dO      xs t        j,                  }tV        j5                  | |tY               |||(t7               |&|||'|P      })nU|dQk(  rOt[        |t\              r|g}0n|}0t^        ja                  | |0tY               |(|&t7               |||i |tb        ||R      })n|dSk(  r8|t1        dT      t8        j5                  | |||||(|t7               |&||i "      })
n|dUk(  r||xs te               xs t        j,                  }1|xs t        j$                  xs t'        dV      }tf        ji                  | |tY               |(|&t7               ddd|tb        dU|1|||W      })
nC|dXk(  r|&j                  dYd      xs> |&j                  dZd      xs* t        jj                  xs t'        d[      xs t'        d\      }2|&j                  d]d      xs> |&j                  d^d      xs* t        jl                  xs t'        d_      xs t'        d`      }3|&j                  dad      xs, |&j                  dbd      xs t'        dc      xs t'        dd      }4|xs* t        j$                  xs t'        de      xs t'        df      }	 to        | dXg      }5|5j                  dhdi      }6|6r;tf        ji                  | |tY               |(|&t7               |2|3|4|tb        dXd|||W      })ndj|&v sdk|&v s| tp        jr                  v r:tp        ju                  | |tY               |(|&|'t7               |2|3|4|tb        dX||l      })nztv        j5                  | |tY               |(|&t7               |2|3|4dX||tb        |||m      })n?|dnk(  r/ty        j4                  | |tY               ||(|&t7               |o      })n|dpk(  rt        j$                  xs |xs t'        dq      xs dr}t[        |t\              r|g}t{        ds |D              st        j|                  dt| | dpu      |du rt~        j                  nt~        j                  }7 |7|| |tY               |(|&t7               v      })n\|dwk(  r2t        j5                  | |tY               |(|&t7               tb        x      })n%|dyk(  rJ|xs t        j,                  xs t'        dz      }tB        j5                  | ||||(|t7               |&||1
      })n|d{k(  rJ|xs t        j,                  xs t'        d|      }tB        j5                  | ||||(|t7               |&||1
      })n|d}k(  ro|xs t        j,                  xs t'        d~      }|xs! t        j$                  xs t'        d      xs d}tB        j5                  | ||||(|t7               |&||1
      })n|dk(  ro|xs t        j,                  xs t'        d      }|xs! t        j$                  xs t'        d      xs d}tB        j5                  | ||||(|t7               |&||1
      })n|dk(  rq|xs t        j,                  xs t'        d      }|xs! t        j$                  xs t'        d      xs d}t8        j5                  | |||||(|t7               |&||i "      })n)|dk(  r+t8        j5                  | |||||(|t7               |&||i "      })n|dk(  r+t8        j5                  | |||||(|t7               |&||i "      })n|dk(  rYt        j                  |&||      }8|8d   }|8d   }d|8v r|8d   |&d<   t8        j5                  | |||||(|t7               |&i ||      })nk|dk(  rs|xs! t        j,                  xs t'        d      xs d}|xs! t        j$                  xs t'        d      xs d}tB        j5                  | ||||(|t7               |&||1
      })n|dk(  r+t8        j5                  | |||||(|t7               |&i ||      })n|dk(  r}|xs t        j$                  xs t'        d      }|xs/ t        j,                  xs t        j@                  xs t'        d      }t        j5                  | ||||(|t7               |&||1
      })nA|dk(  rAt[        |t\              r|g}0n|}0t8        j5                  | |0||||(|t7               |&i ||      })n|dk(  r~|xs* t        j,                  xs t'        d      xs t'        d      }9|9t1        d      |t[        |t              r|}ni }t8        j5                  | ||||(||&i t7               |9|||      })nx|dk(  rq|xs t        j,                  xs t'        d      }|xs! t        j$                  xs t'        d      xs d}t8        j5                  | |||||(|t7               |&||i "      })n|dk(  r|xs/ t        j                  xs t'        d      xs t        j,                  }|xs! t        j$                  xs t'        d      xs d}t8        j5                  | |||||(|t7               |&||i "      })nz|t        j                  v rmd}:t        j                  D ]  };|;d   |k(  s|;d   }: |:t        | |g      |s|:j4                  n|:j                  }< |<| ||(||||&t7               tb        |'
      })n|dk(  r9|xs t'        d      }t8        j5                  | |||||(|t7               |&||i "      })n|dk(  r||xs< t        j,                  xs* t        j                  xs t'        d      xs t'        d      }t8        j5                  | |||||(|t7               |&||d|j                  dd      i"      })n<|dk(  r*t8        j5                  | |||||(|t7               |&||i "      })nt        | |g      |)+t        |)d      rt[        |)t6              r||)j                  d<   |)t        | |g      |)S # t        $ r#}t        j                  d|        Y d}~d}~ww xY wc c}"}!w # t        $ r di}6Y 	w xY w# t        $ r1}|j                  ||t]        |             t        | |||      d}~ww xY w)aw  
    Embedding function that calls an API to generate embeddings for the given input.

    Parameters:
    - model: The embedding model to use.
    - input: The input for which embeddings are to be generated.
    - encoding_format: Optional[str] The format to return the embeddings in. Can be either `float` or `base64`
    - dimensions: The number of dimensions the resulting output embeddings should have. Only supported in text-embedding-3 and later models.
    - timeout: The timeout value for the API call, default 10 mins
    - litellm_call_id: The call ID for litellm logging.
    - litellm_logging_obj: The litellm logging object.
    - logger_fn: The logger function.
    - api_base: Optional. The base URL for the API.
    - api_version: Optional. The version of the API.
    - api_key: Optional. The API key to use.
    - api_type: Optional. The type of the API.
    - caching: A boolean indicating whether to enable caching.
    - custom_llm_provider: The custom llm provider.

    Returns:
    - response: The response received from the API call.

    Raises:
    - exception_type: If an exception occurs during the API call.
    rt  Nr   r(  r   r/  r;  r  r  r%  r  r  r  r  r  )r  r  r  r9  r#  r   r7  r   r   r   r   r   r  r  r  )r   r  r  r  r.  r  r  r  r	  r,  )r   r;  Tr4  r5  r"  r9  r7  zPNo API Base provided for Azure OpenAI LLM provider. Set 'AZURE_API_BASE' in .env)r   r  r9  r   r#  r"  r  r  r   r  r-  r   r  r   r  r  rn  )r   r  r.  r9  r   r  r   r  r-  r   r  r  rC  ri  rc  litellm_proxyrB  rC  rD  rm  rE  r  )r   r  r9  r   r  r   r  r-  r   r  r   r(  r  r  r  )
r   r  r9  r   r  r   r  r-  r   r  r^  r_  HOSTED_VLLM_API_KEY)r   r  r.  r9  r   r  r   r  r-  r   r  r  r  openai_like	llamafile	lm_studioOPENAI_LIKE_API_BASEOPENAI_LIKE_API_KEYr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  VERCEL_OIDC_TOKENr  r  )r   r  r0  r   r9  r  r  r-  r   r  r  r  r  )r   r  r0  r  r-  r  r   r   r  r  r9  re  r%  r   r  z7api_base is required for triton. Please pass `api_base`r  r  )r   r  r0  r  r-  r  r  r  r  r  re  r.  r   r9  r   r%  r  r  r  r  VERTEX_PROJECTr  r  r  VERTEX_LOCATIONr  r  r  VERTEX_CREDENTIALSr  VERTEX_API_BASEr  uses_embed_contentFimagevideo)r   r  r0  r  r-  r  r  r  r  r  r  re  r.  r   r9  )r   r  r0  r  r-  r  r  r  r  r.  r   r  re  r   r9  r   r   )r   r  r0  r9  r  r-  r  r   r  r  r  c              3   <   K   | ]  }t        |t                y wr   )rK  r   ).0r  s     r   	<genexpr>zembedding.<locals>.<genexpr>  s     ?z$,?s   z+Invalid input for ollama embeddings. input=rq  r   rm  )r9  r   promptsr0  r  r-  r  r  )r   r  r0  r  r-  r  re  rr  rs  rG  FIREWORKS_AI_API_KEYrj  NEBIUS_API_KEYNEBIUS_API_BASEzapi.studio.nebius.ai/v1rk  WANDB_API_KEYWANDB_API_BASEz!https://api.inference.wandb.ai/v1rf  SAMBANOVA_API_KEYSAMBANOVA_API_BASEzhttps://api.sambanova.ai/v1voyageinfinityr  )r-  r   r9  r   r9  r  )r   r  r.  r9  r   r  r   r  r-  r  r   r  
xinferenceXINFERENCE_API_KEYzstub-xinference-keyXINFERENCE_API_BASEzhttp://127.0.0.1:9997/v1rU  r  AZURE_AI_API_BASEAZURE_AI_API_KEYjina_airg  ARK_API_KEYVOLCENGINE_API_KEYzuMissing API key for Volcengine. Set ARK_API_KEY or VOLCENGINE_API_KEY environment variable or pass api_key parameter.)r   r  r   r.  r  r9  r-  r  r  r   r   r  r  r  r  r  r  rW  rX  rY  rZ  r  r  )
r   r  r  r9  r   r   r-  r  re  r  r  SNOWFLAKE_JWTrR  rT  rS  r  rb  r  r.  r  )r   rE  r.  rG  r   )Qr   r   r  r   r  r  rS  rl   r  r   r   rV   rZ   r
  r  r   r  r.   r9  r>   r#  r  r   r  r  r  r  rF   r  open_ai_embedding_modelsr   rC  r&  r;  r^   rN  databricks_embeddingopenai_like_keyopenai_like_embeddingrG  rO  r  rL  huggingface_embedr  rK  r   bedrock_embedding
embeddingsre  r   google_batch_embeddingsbatch_embeddingsr  r  rW   vertex_multimodal_embedding%SUPPORTED_MULTIMODAL_EMBEDDING_MODELSmultimodal_embeddingvertex_embeddingr   allBadRequestErrorr  ollama_aembeddingsollama_embeddingsr`  r   get_watsonx_credentialsazure_ai_embeddingrY  r3  rx  ry  r%   r  r/  r  r  r  r   )=r   r  r  r  r   r9  r#  r   r;  r  r  r.  r  r  r   rt  r   r(  r   r/  r;  r  r  r%  r  r~  rc  r  r  r  openai_paramsr  default_paramsr   r   r1  r  r  r-  litellm_params_dictr  r   r"  rG  r  r  r  r  transformed_inputr  r  r  r  r  r  ollama_embeddings_fncredentialsvolcengine_keyr  r  r  s=                                                                r   r  r  <  s*   Z JJw%EZZ$'FZZ 0$7N**]D1K-3ZZ8M-N+1::ot+LM$jj)BDI!'L$!?JJJ5MjjD):]G }%%	M#..??AMNN=) "::&<dC"JJ'>E"JJ'>EM  	 	N
 #^3NA1N+B1  =M/	=9E "!172 5 '/3 O 	(-B-N		*&'q02M(;!%zz,73	
 -6v6!4G(('*/ )   EGGI
  	 D=/7:  W7#3#3W~FV7WH  5&&5!"565 44	  -00 $ 2 01 
  3??3$$3 "/2	   f 
 .77!'-(?#02 /%'0=2! 8 H$ !$440G,66$7!#02 /%2 7 H  8+"m3"l2"o5999'/  /##/!"34/ ""34/ /  $$ !"78   4??4%%4 ""23	  "w"}3:0*5D 12 6: 12K /88!#02 /%'- 9 H !L0X7#3#3XzBW7XH  4??4))4 23	  ,55!#02 /% 6 H !M1VG,,V?U0V 
 !//R^<Q-R,66$7!#02 /%22 7 H   =0"k1"k1 VG,,V?U0V 
  ==..= &&;<	  "w"}3:0 -66!#02 /% 7 H !H,0C}0T #%%#!"23# ",/# ??  ,66$7!"#02 /%2 7 H !L0 2##2!"782 2	   0??0))0 ""670 ",/  #-]";"S?S",]";"Hy !4."
 1'//H"))(3(G,66$7!#02 /%2 7 H !$77 5##5!">?5 5	   7??7!"=>7 ""56	  -66$7!#02 /%2 7 H !M1 #**#34# ??	  )22&!#02 /%2 3 H !I-%%%*G!$)!(33'&# /02%!!+% 4 H  !H, M  -66$7!#02 /%! 7 H !H,$Q(<(>Q'//NX7#3#3X~FW7XH.??&# /02# $#'%+$,&!%! @ H& !K/##$4d; 4"&&':DA4))4 ""454 ""23   ##$5t< 5"&&';TB5**5 ""565 ""34   ##$8$? 8"&&'>E8!"898 ""67	   5##5!"565 ""34	 ++[
 &0^^4H%%P" "2CC*_ '$3#4#6#4$6'9)"/(3 %!")! D & ?*o-.TTU 7KK*_ '$3#6#4#6#4$6'9)"/(3!% L $ ,55*_ '$3#4#6#4$6'9(3#)"/#%!! 6 $ !K/ **&!# /02	H !H,   ,,!"34, ,	  %%???--I%Q!)  % ))-- !
 ,!&# /02H !K/$..&# /02+ / H !I-UUNCT4UG.88!#02 /% 9 H !N2T7??Tn=S.T  /88!#02 /% 9 H !H,TTNCS4TG -##-!"34- -	  /88!#02 /% 9 H !G+SSN?4SG 7##7!"237 7	  /88!#02 /% 9 H !K/WWNCV4WG 1##1!"671 1	  -66$7!#02 /%! 7 H !H,,66$7!#02 /%! 7 H !J.,66$7!#02 /%! 7 H !I-)AA /8K "),G":.H+%+6w+?(,66$7!#02 /!% 7 H !L0 )??)!"67) )	   .##.!"78. .	  /88!#02 /% 9 H !E),66$7!#02 /!% 7 H !J. 7##7!"56   6??6%%6 ""45	  *33!#02 /% 4 H !I-%%%*G!$)!,66'$7!#02 /!% 7 H !L0 8??8!-08 ""67	  %  L  (Zt-L',66$7#! /!02&% 7 H !J.VVNCU4VG F##F!"56F F	  -66$7!#02 /%! 7 H !J. #''#!.1# ??	   1##1!"561 1	  -66$7!#02 /%! 7 H !G$=$==26N33 <
#'::%)*:%;N< %,5H  " ((#..  "#! /02+2H !K/@!@G,66$7!#02 /%! 7 H !J. 6??6''6 ""896 ""45  -66$7!#02 /% ,fjjt.LM 7 H !L0,66$7!#02 /%! 7 H )1D   "238%67=PH##$9:(1D  Y   	M""%Gs#KLL	M6F  +%*"+^  
%%!!f 	& 	
   3	
 	

s    /AO ,AO=9AO=-`8AP &AP _ AP IFAP O	AO:OAO5O5AO:PAPPAP PAPPAP P	AQP,AQ
Q
AQc                    K   t        j                         }t        |       dkD  r| d   n|d   }d|d<   d}	 t        t        g| i |}t        j                         }t        |j                  |      }|j                  d|       d{   }t        |t              st        |t              rt        |t              rt        di |}	n%|}	n"t        j                  |      r| d{   }	n|}	|j                  dd      du s t        |	t              st        |	t              r+t        t!        |	|||       |||j                  d	      
      S t        |	t              r|	S t        j                  |	      r
|	 d{   }	t               }
t"        j$                  j&                  j)                  |
|	|      }
|
S 7 .7 7 A# t*        $ r}|xs d}t-        |||| |      d}~ww xY ww)zP
    Implemented to handle async streaming for the text completion endpoint
    r   r   Tr   Nr  F)r   r   r.  rf  r  )r  r   r.  r  )text_completion_responser   r.  rC  rD  r   )rI  rJ  rO  r   r  rU  rV  rW  rX  rK  rY  rK   rZ  r   rL   rE   rh  r   r  LiteLLMResponseObjectHandlerconvert_chat_to_text_completionrS  r   )rf  r   rB  r   r.  r_  r`  ra  rb  r   r  rc  s               r   r  r    s     !!#D4y1}DG&/E F=;
888 &&(#CGGT2"2249JKKmT*j1/
 -.1BMB(  /**H$H JJx'4/($?@($78."2%(;	# $7%zz*:;
 
 ($:;$$X.!)>'='?$'.}}'Q'Q'q'q)A!$7 (r ($
 ,+U L +2 *  
1=X 3 "
 	

sy   4G%AF> 	F7
AF> 'F:(A%F> G%F> G% F> :F<;;F> 6G%7F> :F> <F> >	G"GG""G%r  best_ofechor6  suffixc           
        )*+ dd l }	 dv r*d   }t        |t              r|j                  d       t	               *i )||)d<   ||)d<   ||)d<   ||)d<   ||)d<   ||)d<   ||)d	<   |	|	)d
<   |
|
)d<   ||)d<   ||)d<   ||)d<   ||)d<   ||)d<   ||)d<   ||)d<   ||)d<   ||)d<   ||)d<   t        ||      \  }}}}|dk(  r	|du r	dvrdd<   t        | t              rdd l}t        j                  d      +t        |       dkD  rt        | d   t              r| D cg c]  }d  }})*+fd}|j                  j                         5 } t        |       D !"cg c]  \  }!}"| j                  ||!|"       }#}!}"t        |j                  j                  |#            D ]  \  }!}$|$j!                         ||!<    |*_        d d d        *S g }%t        | t              r?t        |       dkD  r1t        | d   t              r| D ]  }&d|&d}'|%j%                  |'        nt        | t              rd| dg}%ns|dk(  s|dk(  s|d k(  s
|d!k(  s|d"k(  rKt        | t              r;t        |       dkD  r-t        | d   t              st        | d   t&              rd| dg}%nt)        d#|  d$      j                  d%d        |.|dk(  r)|t*        j,                  vrd&|z   )j                  dd        t/        d'      dd(<   t1        |%d))}(j3                  d*d+      du r|(S |du s$j3                  dd+      du st        |(t4              rt7        |(||,      }(|(S t        |(t6              r|(S t        |(t              r|(S t*        j8                  j:                  j=                  |(*-      **S c c}w c c}"}!w # 1 sw Y   *S xY w).Nr   enginer  r  r  r  r  r  r  r  r6  r  r  r  r  r  r  r9  r#  r   r.  r8  r  Ttop_n_tokensr  ztext-davinci-003c                     
j                  |      }i }t        |dd|}|j                  dd       	d<   d	d<   |j                  dd       	d<   |j                  dd       	d<   |d   d	   S )
Nr  )r   r  r   rQ  r  objectr  r   r  r   )decoder  r   )r  individual_promptdecoded_prompt
all_paramsr   rf  r   r   r-  r  	tokenizers        r   process_promptz'text_completion.<locals>.process_prompt~  s    %.%5%56G%HN!>F!>o!>J7F  $-$%8
 %8H 6>\\$5M,T29J,X6:B,,yRV:W,Y78@Wd8S,W5#I.q11r   r  r  rC  rt  r=  r>  r=  zVUnmapped prompt format. Your prompt is neither a list of strings nor a string. prompt=z;. File an issue - https://github.com/BerriAI/litellm/issuesr  ztext-completion-openai/z;model is not set. Set either via 'model' or 'engine' param.r  r   r   F)r  r   r  r.  )r   r  )r  rK  r   r   rK   rV   r'  concurrent.futurestiktokenencoding_for_modelrO  r   r   	enumeratesubmitas_completedresultr  r  r   rS  r   r4  r  r   r   rE   rL   r  r  r  ),r  r   r  r  r  r  r  r  r  r  r6  r  r  r  r  r  r  r9  r#  r   r$  r.  rf  r   r  _engine_modelr  
concurrentxr  r   executorr  r  completed_futuresfuturer   prq  r   r-  r  r  s,    `                    ``                 @@@r   r  r    s   j 4 6"=Z5E

857&(O%,	""&$/@+,(2%&.
#(2%} #.>*+"&$*!!,:()$*!)4&#( "&&.
#)4&%,	"&1D-. >N/>:F( m+4<V+)*~& fd#% 334FGI6{Q:fQi#>+12aT2	22 2"  ''::< 	A 5>f4E)0A0 !;LM)% ) &/"**778IJ& 7	6 (.}}	!7 8A,4	A 0/ H&$CK!O
6!9c8R 	%A%!4GOOG$	% 
FC	 #78  8+"g-"l2"&AA"&>>vt$K!Oq	4(Jvay#,F $78dekdl  mh  i
 	
 JJxx'???-6E 5t<}VWW $F 
  	
 H zz-'4/$::h&$.h 34.&) 3	
 	H9	: (23 	22RR%= 	S 	
  $#] 3()	A 0/s%   ?	O.O=OA	OOO!
adapter_idc                   K   	 d}t         j                  D ]  }|d   | k(  s|d   } |)t        dj                  | t         j                              |j	                  |      }t        di | d{   }d}t        |t              r|j                  |      }t        |t              r|j                  |      }|S 7 L# t        $ r}|d}~ww xY ww)	zD
    Implemented to handle async calls for adapter_completion()
    NrQ  adapterHNo matching adapter given. Received 'adapter_id'={}, litellm.adapters={}r  r   r  r   )r   adaptersr  r  !translate_completion_input_paramsr   rK  rH   "translate_completion_output_paramsrE   ,translate_completion_output_params_streamingrS  )r1  r   translation_objr  
new_kwargsr   translated_responserc  s           r   aadapter_completionr>    s     26$$ 	2DDzZ'"&y/	2 "Zaa 0 0  %FFfFU
DOD]R\D]>]  	 h."1"T"T! #U # h 34LL&. M    #" ?^   sA   CC AC 4C5AC  CC 	CCCCc            	         K   ddl m} t        t        t        t        t
        t        t        t        f   t        t           f   f    |j                  di | ddi      }| d {   S 7 w)Nr   )"GenerateContentToCompletionHandler	_is_asyncTr   )%litellm.google_genai.adapters.handlerr@  r   r   r
   r   r   r   r   bytesgenerate_content_handler)r   r@  coros      r   aadapter_generate_contentrF    si      Y#sE$sCx.-2F"FGGHC*CC 	
	
 $	
D ::s   A#A,%A*&A,c                    d }t         j                  D ]  }|d   | k(  s|d   } |)t        dj                  | t         j                              |j	                  |      }t        di |}d }t        |t              r|j                  |      }|S t        |t              st        j                  |      r|j                  |      }|S )NrQ  r3  r4  r  r5  r6  r   )r   r7  r  r  r8  r   rK  rH   r9  rE   inspectisgeneratorr:  )r1  r   r;  r  r<  r   r=  s          r   adapter_completionrJ  '  s     /3O   .:#"9oO. V]]G,,
 	
 !BB&BQJ:D:Rz:RH 	  (M*-PP Q 
  
H1	2g6I6I(6SHH"* I  	 r   r  c                     |xs/ t         j                  xs t         j                  xs t        d      }|j	                  dd       }|j	                  dd       }|0|t        j                  ||      }nt        j                  |      }||j                  j                  | |      }n|j                  j                  |       }|j                         }t         j                  j                  j                  |      S )	NrE  r9  r   )r   r   )r   r  r   r  r@  )r   r   r&  r>   r   rC  OpenAImoderationsr   
model_dumpr  r  convert_to_moderation_response)r  r   r   r   r9  openai_clientr   response_dicts           r   
moderationrU  M  s    
 	 	,??	,	, *+	  zz*d+HJJx.M"MM'HMM"MM':M ,,33%u3M ,,33%3@"--/M==55TT% U  r   c                   K   ddl m} |xs/ t        j                  xs t        j                  xs t        d      }t        di |}|j                  dd       }d }	 t        j                  |xs d||j                  |j                        \  }}}	}|j                  dd       }
|
t        |
|      s't        j                  d||j                  xs |	      }n|
}|xs$ t        j                  j                  j                   }|(|j#                  ||j                  d
d       i i ||       |&|j$                  j'                  | |       d {   }n$|j$                  j'                  |        d {   }|j)                         }t        j*                  j,                  j/                  |      S # t        j                  $ r Y 7w xY w7 }7 Zw)Nr   )AsyncOpenAIrE  r/  r  r  r   T)is_asyncr   r9  r  r,  rL  rM  rN  r   )rC  rW  r   r   r&  r>   r?   r   rV   r9  r  rK  r;  _get_openai_clientr   OPENAIr  r  rP  r   rQ  r  r  rR  )r  r   r   r.  r   rW  r-  r/  _dynamic_api_base_dynamic_api_keyrS  _openai_clientr   rT  s                 r   amoderationr^  m  s     # 	 	,??	,	, *+	  +4V4O7=zzt8  $$+2 3$--#++	
	
 JJx.MJ}k$J '>&P&P$--B1B 'Q '
 ' .R1E1E1L1L1R1R&88FD) !4 	9 	
 '33::e:TT'33:::GG"--/M==55TT% U  E "" < UGsI   AG6F3 B=GG$G4G5>G3G
G	G

GGc                    K   t        j                         }t        |       dkD  r| d   n|d   }d|d<   |j                  dd      }d}	 t	        t
        g| i |}t        j                         }t	        |j                  |      }t        ||j                  dd            \  }	}}	}	|j                  d|       d{   }
t        |
t              rt        di |
}nMt        |
t              r|
}n:t        j                  |
      r|
 d{   }n|j                  d|       d{   }t        |t              st        d	t!        |             |=t        |t"              s-|+t%        |d
d      }|t'        |      }|||j(                  d<   |S 7 7 7 p# t*        $ r}|xs d}t-        |||| |      d}~ww xY ww)a
    Calls openai + azure whisper endpoints.

    Allows router to load balance between them
    r   r   TatranscriptionfileNr9  r   r9  zVInvalid response from transcription provider, expected TranscriptionResponse, but got durationaudio_transcription_durationrC  rD  r   )rI  rJ  rO  r   r   transcriptionrU  rV  rW  rV   rX  rK  rY  rM   rZ  r  typer   rw  r(   r  rS  r   )rf  r   rB  r   rb  r.  r_  r`  ra  r^  rb  r   existing_durationcalculated_durationrc  s                  r   ra  ra    s     !!#D4y1}DG&/E#F::fd#D6
}6t6v6 &&(#CGGT2'7&**Z">(
$1
 #2249JKKmT*,=}=H'<=$H  /**H "11$8IJJH($9:himnviwhxy   x3  '*d C (&@&F#&2 , ++6 A L + K0  
1=X 3 "
 	

sc   AG 	A4F =F>AF FF &F'A+F G F F F 	F="F88F==G rb  language)r  rl  srtverbose_jsonvtttimestamp_granularities)wordsegmentr   c                 H   |j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  dd      }|j                  d      }|j                  d	d      }|j                  d
d      }|j                  dg        t        |      }|j                  dd      }|rt        |      |j                  d<   |t
        j                  }t        j                  j                         }t        | ||
|	      \  } }}}
||}	t        d$| ||||||d|}t        d$i |}|j                  | |i ||||di d||       d}t        j                  | t!        |            }|dk(  r|
xs t        j"                  xs t%        d      }
|xs t        j&                  xs t%        d      }|j                  dd      xs t%        d      }|	xs/ t        j(                  xs t        j*                  xs t%        d      }	||d	<   t,        j/                  | ||||||||
|	||||      }n|dk(  s|t        j0                  v r|
xs. t        j"                  xs t3        d      xs t3        d      xs d}
t        j4                  xs t3        d      xs dt
        _        |	xs/ t        j(                  xs t        j6                  xs t3        d      }	t8        j/                  | |||||||||
|	|||      }nK|It:        j/                  | ||||||"t=        |t>              st=        |t@              r|nd||||
|	|i ||       }|;t=        |tB              s+tE        |d!d      } | tG        |      }!|!|!|jH                  d"<   |tK        d#      |S )%r`  r  Nr  r  r  ra  Fr/  r%  r(  tagsr   r  )r   rj  r  r  rn  r  r.  r  r  r  r  r  stream_responser,  r  rt  r4  r5  r"  r9  r7  )r   
audio_filer-  r  ra  r   r   r  r9  r   r#  r"  r   r  rC  rB  rC  rD  rm  rE  )r   ru  r-  r  ra  r   r   r  r   r9  r   r  r  r(  )r   ru  r-  r  r  ra  r   r   r   r  r9  r   r.  r  r  r(  rd  re  z8Unmapped provider passed in. Unable to get the response.r   )&r   r   rY   r   model_call_detailsrC  DEFAULT_MAX_RETRIESr   r  rM   rV   r\   r   r  rI   'get_provider_audio_transcription_configr   r9  r>   r#  r   r  azure_audio_transcriptionsaudio_transcriptionsr5  r^   r   r&  openai_audio_transcriptionsr  rK  r7   r6   r   rw  r(   r  r  )"r   rb  rj  r  r  rn  r  r  r   r   r9  r#  r   r.  r   r  r  r  r  ra  r/  r%  r(  r1  r   r  r  r-  r
  r   r  r"  rh  ri  s"                                     r   rf  rf    sp   6 jj!2D9O!::&<dCL$/Jzz*d+HZZ 0%8N-3ZZ8M-NJJ5MZZ 0$7N
JJvr=fE 	

8T"  ;>v;..x800]]88:N<L/	=9E "!7 	' 7/	 	O -6v644.$8$  $!
 
 0 5 $ 	  ,SS12O
 g%Sw//S>BR3S U7..U.AT2U 	  $4d; 
~@

  //  / o.	 	 ,9(-BB+))+#)#. C 
  
	(wBBB  +++,+ +,+ + 	    /0 	 bW__b0B0BbjQaFb.CC+))+#+.) D 
  
	$(==+.)) %v{3!&*:;	  #+ 3+)1 > 
: Jx$C#Hj$?$"<T"B". ( ''2 STTOr   c                  X  K   t        j                         }t        |       dkD  r| d   n|d   }d|d<   |j                  dd      }	 t	        t
        g| i |}t        j                         }t	        |j                  |      }t        ||j                  dd            \  }}}}|j                  d|       d{   }	t        j                  |	      r|	 d{   }
|
S |j                  d|       d{   }
|
S 7 A7 $7 
# t        $ r}|xs d	}t        |||| |
      d}~ww xY ww)z%
    Calls openai tts endpoints.
    r   r   Taspeechr.  Nr9  rc  rC  rD  )rI  rJ  rO  r   r   speechrU  rV  rW  rV   rX  rZ  rS  r   r  s               r   r}  r}    sB    
 !!#D4y1}DG&/EF9 **%:DA
v/// &&(#CGGT2'7&**Z">(
$1
 #2249JKK}-**H  "11$8IJJH L* K 
1=X 3 "
 	

sm   AD*A4D ;C=<D C?D  D*!D 6D7D <D*=D ?D D 	D'D""D''D*voicer   projectr  speedinstructionsr  r}  c                    |j                  dd       }|j                  dd       }|j                  dd       }|j                  dd       }|j                  dd       }|j                  dd       }t        | ||      \  } }}}|j                  dg        i }|||d	<   |||d
<   |||d<   |
t        j                  }
|"t        j
                  xs t        j                  }t        dEi |}t        j                  | t        j                  |            }||j                  | ||d|      \  }}t        t        |j                  d            }|j                  | ||||||	d i d||       d }|dk(  s|t        j                   v r|t#        |t$              st        j&                  d| |      |xs. t        j(                  xs t+        d      xs t+        d      xs d}|xs/ t        j,                  xs t        j.                  xs t+        d      }|xs! t        j0                  xs t+        d      xs d }|xs! t        j2                  xs t+        d      xs d }|xs t        j4                  }t6        j9                  | |||||||||
|||      }n|dk(  r| j;                  d      rUddlm} |t        j&                  d | |      t        ||      }  | j@                  dE| ||||||
|tB        |xs d||d!|}n<|t#        |t$              st        j&                  d"| |      |xs t        j(                  xs t+        d#      }|xs t        jD                  xs t+        d$      }|xs< t        j,                  xs* t        jF                  xs t+        d%      xs t+        d&      }|j                  d'i       j                  d(d       xs t+        d)      }!|j                  d*d       }"|r||d<   tH        j9                  | |||||||!|"|||
|||+      }n"|d,k(  rdd-l%m&}# | |#       }t        |#|      }$t#        |t$              r|nd }%|%|%jO                         st        j&                  d.| |      |%jO                         }%|j                  |#jP                  d       }&t#        |&tR              r|&||#jP                  <   |%||#jT                  <   |||d/<   |||d0<   tB        jW                  | ||%|$|||||
|||xs d1      }n5|d2k(  s|d3k(  rdd4l,m-}' t]        dEi |}(d5| v r#d6d7l/m0}) |)jc                  | |||||xs i ||8      S | |'       }t        |'|      }*|je                  |(jf                  |(jh                  |(jj                  d9        |*j@                  dE| ||||||
|tB        |xs d|(j(                  d d!|}nx|d5k(  r#d6d7l/m0}) |)jc                  | |||||xs i ||8      S |d:k(  rTdd;l6m7}+ |t        j&                  d<| |      t        |+|      }, |,j@                  dE| ||||||
|tB        |xs d||d!|}n|d=k(  rdd>l8m9}- | |-       }t        |-|      }.|||d/<   |||d0<   d }/t#        |t$              r|}/nGt#        |tR              r7|j                  d?      xs$ |j                  d@      xs |j                  dA      }/tB        jW                  | ||/|.|||||
|||xs d1      }nG|dBk(  rBddCl:m;}0 | |0       }t        |0|      }1 |1j@                  dE| ||||||
|tB        |xs d||d!|}|)ty        dDj{                  |t        j|                              |S )FNr  r  r  r%  r  r(  r8  rr  r  r  r  r  F)r   r-  r  r  r   r/  rs  r,  rC  z;'voice' is required to be passed as a string for OpenAI TTSr  rB  rC  rD  rE  rm  OPENAI_PROJECT)r   r  r  r-  r   r9  r   r  r   r   r   r}  r(  rt  zspeech/r   )AzureAVATextToSpeechConfigz,Azure Speech Service configuration not found)r   r  r  r-  r
  r  r   r%  r  r}  r9  r   z:'voice' is required to be passed as a string for Azure TTSr4  r5  r6  r7  r8  r"  r9  r  )r   r  r  r-  r   r9  r#  r"  r  r   r   r   r   r}  r  
elevenlabs)ElevenLabsTextToSpeechConfigzA'voice' must resolve to an ElevenLabs voice id for ElevenLabs TTSr9  r   )r   r  r  text_to_speech_provider_configtext_to_speech_optional_paramsr.  r  r  r   r%  r   rA  r  r  )VertexAITextToSpeechConfigr  rk   )#speech_to_completion_bridge_handler)r   r  r  r-  r  r  r  r.  )r  r  r  runwayml)RunwayMLTextToSpeechConfigz/RunwayML Text-to-Speech configuration not foundr[  )MinimaxTextToSpeechConfigvoice_idrQ  name	aws_polly)AWSPollyTextToSpeechConfigz@Unable to map the custom llm provider={} to a known provider={}.r   )?r   rV   r   r   r  r   rC  rw  r   rI   "get_provider_text_to_speech_configr   map_openai_paramsr   rL  r  r5  rK  r   r  r9  r^   r   r&  r   r  r  r;  audio_speechr}  0litellm.llms.azure.text_to_speech.transformationr  dispatch_text_to_speechr  r#  r  r  5litellm.llms.elevenlabs.text_to_speech.transformationr  stripELEVENLABS_QUERY_PARAMS_KEYrY  ELEVENLABS_VOICE_ID_KEYtext_to_speech_handler4litellm.llms.vertex_ai.text_to_speech.transformationr  r?   4endpoints.speech.speech_to_completion_bridge.handlerr  r~  r  r  r  r  3litellm.llms.runwayml.text_to_speech.transformationr  2litellm.llms.minimax.text_to_speech.transformationr  4litellm.llms.aws_polly.text_to_speech.transformationr  rS  r  provider_list)2r   r  r  r   r9  r#  r   r  r   r  r   r  r  r  r   r  r.  r}  r   r  r  r  r%  r  r(  r  r-  r
  r  r  r   r  azure_configr"  r  r  elevenlabs_configr  query_paramsr  generic_optional_paramsr  vertex_configr  runwayml_configr  minimax_config	voice_strr  aws_polly_configs2                                                     r   r~  r~    s	   , ::fd#D%+ZZ0A4%HO!::&<dCJJ5ML$/JZZ 0$7N<L)<x=9E JJvrO"-<)*#( *6'))))GV-G-G,6v6 	@@))*=>	
 # &1!?!Q!Q+ "R "
 &*6::&;<&K ,,'.$8$  $!
 
 0 - & 		  	x''"E"EE=E3!7))U0   +++,+ +,+ + 	  ,,!!, *+	 	  ##/0 	 	  *+ 	 	 ,W__*77+%#) 8 
 
	'I&
 .5--J!4   *,JL <|;;  /$7'+&;(5! H" }Zs%;--X!4 
  S7#3#3SzBR7SH%_)<)<_
K^@_K  /??/$$/ 45/ o.  -<,?,?b,Q,U,U $- -  
 '-jj1JD&Q#3@0-:: /!'-(?)'2 ; H" 
	,	
 *1-I-K* (*H
 'uc258>>#3))[0 
 >>#zz(DDd
 lD)   ,HH  	(@@	
 .6
+-4	*(??+<+: 3.#'& @ 
 
	+/BFV/V	
 #7"@"@ u 7== /22'$7 > 	 	 *1-G-I* 79WX 	"""9"H"H#:#J#J&=&P&P	
 9=88 
+ 3#!"7$u,55
 
 
	(	
 399+.Mr# 3 : 	
 		
 

	*	

 *1))I0  &(F
 ;?:: 
+ 3#'"7$u
 
 
		)	

 *1-F-H*79WX.6
+-4	* $(	eS!It$		*-U4UEIIfDUI(??+9+: 3.#'& @ 
 
	+	

 *1-G-I*  &(F
 <#;; 
+ 3#'"7$u
 
  NUU#W%:%:
 	

 Or   model_paramsr  )r   r   r  r  audio_transcriptionimage_generationvideo_generationbatchrerankrealtimer  ocrc                   K   ddl m} ddlm}  |       } |dg ddt	        t        j                               t        j                  j                         t	        t        j                               d      }|| d	<   |j                  | 
      } 	 | j                  dd      }|t        d      |t        j                  v r$|"t        j                  |   j                  d      }| j                  dd      }	| j                  dd      }
| j                  dd      }t        ||	|
|      \  }}}}|t        j                  v r$|"t        j                  |   j                  d      }ddi| d<   |xs d}d|v r|j                  ||| |       d{   S |j!                  ||| ||      }||v r> ||           d{   }t#        |di       j                  di       xs i }t%        |      S t        d| d      7 k7 A# t        $ r~}t'        j(                         }t+        |t              r|dd }|ddt	        |       d | icY d}~S t	        |      d!z   |z   }|j,                  j                  d"      }||d#cY d}~S d}~ww xY ww)$a   
    Support health checks for different providers. Return remaining rate limit, etc.

    Returns:
        {
            "x-ratelimit-remaining-requests": int,
            "x-ratelimit-remaining-tokens": int,
            "x-ms-region": str,
        }
    r   )get_litellm_logging_class)HealthCheckHelpersr  Fr   T)r   r   r  	call_typer  
start_timefunction_idlog_raw_request_responser/  )r  r   Nzmodel not setr  r.  r9  r   r  zno-cachecacher   *)r   r.  r  r/  )r   r.  r  r  r  r  r  zMode zI not supported. See modes here: https://docs.litellm.ai/docs/proxy/healthi  errorzerror:zz. Missing `mode`. Set the `mode` for the model - https://docs.litellm.ai/docs/proxy/health#embedding-models  
stacktrace: z
stack trace: raw_request_typed_dict)r  r  ))litellm.litellm_core_utils.cached_importsr  /litellm.litellm_core_utils.health_check_helpersr  r   r   uuid4datetimenow;_update_model_params_with_health_check_tracking_informationr   rS  r   
model_costrV   ahealth_check_wildcard_modelsget_mode_handlersrw  r,   	traceback
format_excrK  rv  )r  r  r  r  r  r  r!   r/  r   custom_llm_provider_from_paramsapi_base_from_paramsapi_key_from_paramsr.  r^  mode_handlersr  _response_headersrc  stack_traceerror_to_returnr  s                        r   ahealth_checkr    s    > TR ()G "DJJL)$$((*

%!%	 +>L&'VV% 	W 	
 G
+//>=O,,G&&&4<%%e,008D*6*:*:;PRV*W'+//
DA*..y$?+; ?)'	,
("Aq G&&&4<%%e,008D !
W ~v%<+II$7)$7	 J    +<< 3% = 
 = 1mD133I 	#3R8<<YKQr  11BCCvfg /  4  
**,k3'%et,K<6#a&  *e  fq  er  s  a&#44{B!4!G!G!K!K$"

 %&<
 	
!
sy   BJC>G> G:G> J'G> :G<;/G> *J+G> <G> >	J=J JJ
0J :J;J JJc                     	 t        j                  |        t        j                  rt	        |        y y # t
        $ r Y y w xY wr   )rl   rP  r   set_verboseprintrS  )print_statements    r   re  re  V  s=    _-/"  s   04 	A A c                      t         j                  &t        t         j                        }t        di | |S t	        d      )Nz`No config path set, please set a config path using `litellm.config_path = 'path/to/config.json'`r   )r   config_pathrb   r   r  )r   config_argss     r   config_completionr  _  sA    &&w':':;2F2k22n
 	
r   chunksc           
         | d   d   }| d   d   }| d   d   }| d   d   }| d   j                  dd       }| d   d   d   d	   }| d   d   d   d
   }|||||d d||dgd d d dd}	g }
| D ]R  }|d   }|D ]F  }|t        |d      s|j                  d      %|j                  d      }|
j                  |       H T dj                  |
      }||	d   d   d<   t	        |      dkD  rn	 	 t        ||      |	d   d<   t        ||d      |	d   d<   |	d   d   |	d   d   z   |	d   d<   t        di |	S # t        $ r t        d       d|	d   d<   Y Tw xY w)Nr   rQ  r  r  r   system_fingerprintr  finish_reasonr  )rl  r  r  r  r  )rQ  r  r  r   r  r  r  rl  r  r   r  r  z1token_counter failed, assuming prompt tokens is 0T)r   rl  count_response_tokensr  r  r   )	r   r  r  r(  rO  re   rS  re  rK   )r  r   rQ  r  r  r   r  r  r  r   content_listchunkr  choicer  combined_contents                   r   $stream_chunk_builder_text_completionr  j  s    
4BAYx FQi	"G1IgE';TB2Jy)!,_=Mbz)$Q'
3H 0 $!.	
 "!% 
H( L 	-	" 	-F"FF+JJv&2 **V,##G,	-	- ww|, &6HY6"
q /-:(.
/* .;".HW)* 	/*Xg->?R-SS Wn% "-H-- 	/ 	IJ-./*	/s   ,D5 5EEr  r!   c                    	 | t        j                  dddd      | sy t        | |      }|j                  } t	        |       dk(  ry t        | d   d   d   t         j                  j                        rt        | |      S | d   d   }|j                  |       }g }d	}	| D ]]  }
t	        |
d         dk(  r|
d   d   }t        |t              r|j                  d
i       nt        |d
i       }t        |t              r|}n:t        |d      r,t        t        t         t"        f   |j%                               }ni }|j                  d      w|j                  d      f|j                  d      U|j                  d      D|j                  d      3|j                  d      "|j                  d      |j                  d      d}	 n;|j                  d      }t        |t               sI|sM|j'                  |       ` |	r|rdj)                  |      |d   d   d   d<   t+        |      }|j-                  | |||d      }t/        |d|       t1        |       D ]v  }
t        |
t              r|
j                  d      }nt        |
dd       }t        |t              sCd|v sH|j2                  j5                  di       j7                  |d           n t         j8                  r|t/        |d|j;                  |             |S | D 
cg c]0  }
t	        |
d         dkD  rd|
d   d   d
   v r|
d   d   d
   d   |
2 }}
t	        |      dkD  rP|j=                  |      }t        t>        |j@                  d         }d |jB                  _"        ||jB                  _#        | D 
cg c]0  }
t	        |
d         dkD  rd|
d   d   d
   v r|
d   d   d
   d   |
2 }}
t	        |      dkD  rNt        t>        |j@                  d         }d |jB                  _"        |jI                  |      |jB                  _%        | D 
cg c]0  }
t	        |
d         dkD  rd|
d   d   d
   v r|
d   d   d
   d   |
2 }}
t	        |      dkD  r|jM                  |      |d   d   d   d<   | D 
cg c]0  }
t	        |
d         dkD  rd|
d   d   d
   v r|
d   d   d
   d   |
2 }}
t	        |      dkD  r|jO                  |      |d   d   d   d<   | D 
cg c]0  }
t	        |
d         dkD  rd|
d   d   d
   v r|
d   d   d
   d   |
2 }}
t	        |      dkD  r|jQ                  |      |d   d   d   d<   | D 
cg c]0  }
t	        |
d         dkD  rd|
d   d   d
   v r|
d   d   d
   d   |
2 }}
t	        |      dkD  r4g }|D ]  }|jS                  |d   d   d
   d          ! ||d   d   d   d<   | D 
cg c]0  }
t	        |
d         dkD  rd|
d   d   d
   v r|
d   d   d
   d   |
2 }}
t	        |      dkD  r=t        t>        |j@                  d         }|jU                  |      |jB                  _+        | D 
cg c]0  }
t	        |
d         dkD  rd|
d   d   d
   v r|
d   d   d
   d   |
2 }}
t	        |      dkD  r4g }|D ]  }
|jS                  |
d   d   d
   d          ! ||d   d   d   d<   | D 
cg c]0  }
t	        |
d         dkD  rd|
d   d   d
   v r|
d   d   d
   d   |
2 }}
t	        |      dkD  ri } |D ]q  }
|
d   d   d
   d   }!t        |!t              s"|!jY                         D ]=  \  }"}#|"| vr|#| |"<   t        |#tZ              rt        | |"   tZ              r|#| |"<   9|#| |"<   ? s | r.t        t>        |j@                  d         }| |jB                  _.        t+        |      }|j_                  |      }$|j-                  | ||||$      }t/        |d|       t1        |       D ]v  }
t        |
t              r|
j                  d      }nt        |
dd       }t        |t              sCd|v sH|j2                  j5                  di       j7                  |d           n t         j8                  r|t/        |d|j;                  |             |S c c}
w c c}
w c c}
w c c}
w c c}
w c c}
w c c}
w c c}
w c c}
w # t`        $ rK}%tc        jd                  djg                  t!        |%                   t        j                  dddd      d }%~%ww xY w)Nrk  z=Error building chunks for logging/streaming usage calculationr  )rj  rq  rm  r   r   r  )r  r   r   TdeltarQ  r  r  reasoning_contentthinking_blocksannotationsr  imagesprovider_specific_fieldsFr  rq  )r  r   completion_outputr   reasoning_tokensr  r  cost)r(  zAlitellm.main.py::stream_chunk_builder() - Exception occurred - {})4r   ru  r|   r  rO  rK  r  rJ   r  build_base_responserY  r   rw  r  r   r   r   r
   rQ  r  r(  r0   calculate_usager  reversedr  r  r  include_cost_in_streaming_usage_response_cost_calculatorget_combined_tool_contentrD   r  rq  r  r  "get_combined_function_call_contentr  get_combined_contentget_combined_thinking_contentget_combined_reasoning_contentextendget_combined_audio_contentr  r   r'  r  count_reasoning_tokensrS  rl   	exceptionr  )&r  r   r  end_timer  	processorr   r   simple_content_partsis_simple_text_streamr  r  	delta_objr  r  r  r  hiddentool_call_chunkstool_calls_listr  function_call_chunkscontent_chunksr  reasoning_chunksannotation_chunksall_annotationsacaudio_chunksimage_chunks
all_imagesprovider_specific_chunkscombined_provider_fieldsfieldsr  r  r  rc  s&                                         r   stream_chunk_builderr    sS   c
>""W	  "684	!! v;!1Ii #W]]%>%>
 8  q	'"008 +- $  	5E5#$)9%a(F fd+ 

7B'VWb1 
 )T*!L1T#s(^Y-A-A-CD 		,'399_-99901=99./;99]+799W%199X&29978D(-%ii	*G'3'G$++G4A 	5D !#?Aww(@#A&y1)< !@ I--"3!!" . E Hgu- "&) 	eT*"YY'78F$U,<dCFfd+0Jf0T++662BfV$>?@	 66;;R999J
 O  
5#$q(i 0 3G <<i #G,\:F	 
 
  1$'AABRSO7H$4$4Q$78G&*GOO#)8GOO&   
5#$q(5#3A#6w#??i #G,_=I	  
  
 #$q(7H$4$4Q$78G&*GOO#<<=QR OO)  
5#$q(U9-a099i #G,Y7C	 
 
 ~" ..~> Y"9-  
5#$q(!U9%5a%8%AAi #G,->?K	 
 
 !# 77H Y"9-!  
5#$q(#uY'7':7'CCi #G,-@AM	 
 
  1$ 889IJ Y"9-#  
5#$q(y!1!!4W!==i #G,];G	 
 
  !A% %'O' Q&&r)}Q'7'@'OPQ?NHY"9-m<  
5#$q(5+A.w77i #G,W5A	 
 
 |q 7H$4$4Q$78G$-$H$H$VGOO!  
5#$q(E),Q/88i #G,X6B	 
 
 |q J% J!!%	"21"5g">x"HIJ:DHY"9-h7  $
5#$q(*eI.>q.A'.JJi #G,-GHT	 $
  $
 '(1,79$1 By)!,W56PQfd+&,lln 	B
U&>><A4S9't44S94: =B4S9<A4S9	BB (w(8(8(;<;S8;HE$;;HE))/- * 
 	'5) f% 		E%&#34 (8$?&$',F&,P''22.& :;<		 22{7Nv{DDHDU c
 




"

"$
|  
  OVVA	

 S	
 	

s   c: &c: 9c: E&c: )c: -B1c: c: $A!c: c: 
5c?A$c: #5cA"c: :5c/1c:  5c1c: 5c!;1c: ,5c&!Ac: )5c+Ac: /5c0$Ac: ,5c5!5c: Dc: &c: +A!c: -c: :	eAe		esystemr"   c           	      *  K   ddl m} ddlm}m} ddlm}	  || ||      \  }
}}}||}||}d| ||di}	  ||      } |	j                  | |	      }|S|j                         }|A|j                  |      r0|j                  |
|d|| ||
       d{   }||j                  s|S |xs g }|r|r	d|dg|z   }t        j                   | ||      } ||| |
d      S 7 K# t        $ r%}t        j                  d|  d|        Y d}~bd}~ww xY ww)a  
    Count tokens for a given model and messages using provider-specific APIs.

    Routes to the appropriate provider's token counting API (OpenAI, Anthropic, etc.)
    for exact token counts. Falls back to local tiktoken-based counting for unsupported providers.

    Args:
        model: The model identifier (e.g., "openai/gpt-4o", "anthropic/claude-3-5-sonnet-20241022")
        messages: The messages to count tokens for (standard chat format)
        tools: Optional tools/functions to include in token count
        system: Optional system message/instructions
        api_key: Optional API key (falls back to environment variable)
        api_base: Optional custom API base URL

    Returns:
        TokenCountResponse with total_tokens and metadata
    r   )rV   )r   r"   )rI   )r   r9  r   Nr  )r   r   r9  r  )model_to_user   contents
deploymentrequest_modelr  r  z)Provider token counting failed for model=z, falling back to local: r  r!  )r   r   r  local_tokenizer)r  r  
model_usedtokenizer_type)1litellm.litellm_core_utils.get_llm_provider_logicrV   litellm.types.utilsr   r"   litellm.utilsrI   get_provider_model_infoget_token_countershould_use_token_counting_apicount_tokensr  rS  rl   rP  r   re   )r   r   r  r  r   r9  rV   r   r"   rI   resolved_modelr.  r  dynamic_api_baser  llm_provider_enumprovider_model_infotoken_counter_instancer(  rc  fallback_messageslocal_counts                         r   acount_tokensr)    s    2 SD3 		 !# 	 
"J
()<=K3KK"3
 *%8%J%J%L"&2*HH'  6BB!/%!)"'!  C    %fll!M !B#&.6BCFWW''"K  !(	 5  
7w>WXYWZ[	
 	

sA   5DAC" C C" (8D C" "	D+DDDD_encoding_cachec                  \    t         !ddl} | j                  t           j                  a t         S )z*Get encoding, loading it lazily if needed.Nr   )r*  sysmodulesr   r0  )r,  s    r   r  r  Q  s)      ++h/88r   r  c                     | dk(  r5ddl m}  |       }ddl}||j                  t           j
                  d<   |a|S t        dt        d|       )z#Lazy import handler for main moduler0  r   )_get_default_encodingNzmodule z has no attribute )litellm._lazy_importsr/  r,  r-  r   __dict__r*  AttributeError)r  r/  	_encodingr,  s       r   __getattr__r4  \  sX    z 	@)+	5>H&&z2#
78,.@I
JJr   r   )FNr  NFNNN)NNN)NNNNNNNNNNNNNNNNNNNNN)NN)NNNNNNr   NNNNN)NNNNNNNNNNNNNNNN)r   NN)NNNN)NNNNN(  rI  rU  r  rH  r  rA  randomr,  r  r  r+  r   r"  r   r   r   r  r   	functoolsr   typingr	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   litellm._uuidr   aiohttpr   dotenvrx  rC  r#  pydanticr   typing_extensionsr   r   r   r  r   r   r    *litellm.litellm_core_utils.litellm_loggingr!   r  r"   litellm.constantsr#   r$   litellm.exceptionsr%   "litellm.integrations.custom_loggerr&   #litellm.litellm_core_utils.asyncifyr'   ,litellm.litellm_core_utils.audio_utils.utilsr(   r)   %litellm.litellm_core_utils.dd_tracingr*   8litellm.litellm_core_utils.get_provider_specific_headersr+   -litellm.litellm_core_utils.health_check_utilsr,   r-   rL  )litellm.litellm_core_utils.mock_functionsr.   r/   8litellm.litellm_core_utils.prompt_templates.common_utilsr0   litellm.llms.base_llmr1   r2   )litellm.llms.base_llm.base_model_iteratorr3   !litellm.llms.bedrock.common_utilsr4    litellm.llms.cohere.common_utilsr5   &litellm.llms.custom_httpx.http_handlerr6   r7   -litellm.llms.openai.chat.gpt_5_transformationr8   $litellm.llms.openai_like.json_loaderr9   #litellm.llms.vertex_ai.common_utilsr:   r;   litellm.realtime_api.mainr<   litellm.secret_managers.mainr=   r>   litellm.types.routerr?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   r`   ra   rb   rc   rd   re   rf   rg   rh   ri   rj   _loggingrl   caching.cachingrm   rn   ro   litellm_core_utils.core_helpersrp   !litellm_core_utils.fallback_utilsrq   rr   0litellm_core_utils.prompt_templates.common_utilsrs   rt   ru   +litellm_core_utils.prompt_templates.factoryrv   rw   rx   ry   rz   r{   0litellm_core_utils.streaming_chunk_builder_utilsr|   llms.anthropic.chatr}   llms.azure.audio_transcriptionsr~   llms.azure.azurer   r    llms.azure.chat.o_series_handlerr   llms.azure.completion.handlerr   llms.azure_ai.anthropic.handlerr   llms.azure_ai.embedr   llms.bedrock.chatr   r   llms.bedrock.embed.embeddingr   llms.bedrock.image_edit.handlerr   +llms.bedrock.image_generation.image_handlerr   llms.bytez.chat.transformationr   !llms.clarifai.chat.transformationr   !llms.codestral.completion.handlerr   llms.cohere.embedr   cohere_embed!llms.custom_httpx.aiohttp_handlerr   "llms.custom_httpx.llm_http_handlerr   llms.custom_llmr   r   llms.databricks.embed.handlerr   llms.deprecated_providersr   r   llms.gemini.common_utilsr   llms.groq.chat.handlerr   llms.heroku.chat.transformationr   "llms.huggingface.embedding.handlerr   !llms.lemonade.chat.transformationr   llms.nlp_cloud.chat.handlerr   rE  llms.oci.chat.transformationr   llms.ollama.completionr  llms.oobabooga.chatr   llms.openai.completion.handlerr   $llms.openai.image_variations.handlerr   llms.openai.openair   "llms.openai.transcriptions.handlerr   llms.openai_like.chat.handlerr   "llms.openai_like.embedding.handlerr   !llms.ovhcloud.chat.transformationr   llms.petals.completionrm  llms.predibase.chat.handlerr   llms.replicate.chat.handlerr>  llms.sagemaker.chat.handlerr   !llms.sagemaker.completion.handlerr   llms.sap.chat.handlerr   llms.vertex_air   8llms.vertex_ai.gemini.vertex_and_google_ai_studio_geminir   <llms.vertex_ai.gemini_embeddings.batch_embed_content_handlerr   8llms.vertex_ai.image_generation.image_generation_handlerr   6llms.vertex_ai.multimodal_embeddings.embedding_handlerr   ,llms.vertex_ai.vertex_ai_partner_models.mainr   2llms.vertex_ai.vertex_embeddings.embedding_handlerr   'llms.vertex_ai.vertex_gemma_models.mainr   'llms.vertex_ai.vertex_model_garden.mainr   llms.vllm.completionri  llms.watsonx.chat.handlerr   llms.watsonx.common_utilsr   types.llms.anthropicr   types.llms.openair   r   r   r   r   r   r   r   types.utilsr   r   r   r   r   r   r   r   r   r;  r)  r{  openai_image_variationsgroq_chat_completionsr1  sap_gen_ai_hub_embr  rC  r$  r  r  r  ry  r  r^  r_  rg  r  bedrock_image_generationbedrock_image_editrS  r  r  vertex_image_generationr  rU  rX  rZ  r`  rh  r  rJ  r  r  r2  sagemaker_chat_completionrp  heroku_transformationoci_transformationru  rr  r   rS  rY  MOCK_RESPONSE_TYPEr   r   r   r   wrapr   r   r   r'  r   rh  r  r  r  rH  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r>  rC  rF  rJ  rU  r^  ra  rf  r}  r~  r  re  r  r  r  r)  r*  __annotations__r  r4  r   r   r   <module>r     s        	  
    H H      & %      &   R Q B6 6 ; B 9 T H ? < P J E = H 5 ( ( ( ( ( ( ( ( ( ( (T % F F ; 
  M 8 D N I > I 1 = : = O ; = F 6 D B > E 8 : 6 = D A P 7 5 * @ N 4 H @ J A = @ @ P = = ; ; 8 8 0 0 O O    P O O O H H N N 9 9 9 6 6 8 8	 	 	 	 	
 
 
 
 
 
 /0 .0 68 68 *, "7"9 *, %' 46 #?#A  ,. 79 ,. 46 (* 46 46 #5#7  $& 13 %' " "$ 79 /1 /1 (=(? %24 &?&A #,. 24 35 13 *, 02 02 &( (* "_ ,. ,. 3	4@SST 7 7	O 	O $ $   $#'+/#'!!%)	 $+/9=AE04(,)-!%>B .2*.#"& 	<@'+"&"!%!!%$(15;?0448cL
L
 L
 ~	L

 C=L
 eE3J'(L
 %L
 E?L
 }L
 TNL
 TNL
 L
 $C=L
 456L
  =>!L
" ,-#L
$ uo%L
&  'L
( )L
* 3-+L
. eD$y/$9:;/L
0 3-1L
2 D>3L
4 %T	*+5L
6 "$7L
8 tn9L
: 3-;L
> NO?L
D  789EL
F  }GL
H 3-IL
L smML
N #OL
P c]QL
R SL
T D>UL
X -.YL
Z !!78[L
^ _-_L
b $,D>cL
f =--.gL
  L
^
( *.8
i(8
8
 "#8
v
4.
eE3567
 

4.
eE3567
 
$eS%--&? @ $-E%emm2K,L - #2J&*#(:>aJaJaJ TNaJ }	aJ
 ./aJ d^aJ 4.aJ eE3567aJN <@!%&*... !!78. DI	.
 sm. 49.b!#/2	 $ DJd4j" "&  
	4  :>#'!!%)	+/ $9=AE04(,)-!% 	<@>B .2#"&*.;?$('+"& $#'"!%!!%150448eg4
g4
 g4
 eE3567	g4

 %g4
 E?g4
 }g4
 TNg4
 TNg4
 $C=g4
 g4
 456g4
 =>g4
 ,-g4
  uo!g4
"  #g4
$ %g4
& 3-'g4
* NO+g4
0  7891g4
2 eD$y/$9:;3g4
4 3-5g4
6 D>7g4
8 %T	*+9g4
: tn;g4
< 3-=g4
> "$?g4
@ !!78Ag4
D D>Eg4
F  }Gg4
H 3-Ig4
L ~Mg4
N C=Og4
R smSg4
T #Ug4
V c]Wg4
X Yg4
\ -.]g4
` _-ag4
d $,D>eg4
h =--.ig4
  g4
Ti7@=<7D=@ <
): <
 <
D 
  $%)"!%!"! 	
 c] sm # c] sm  3-$ %( sC**+) 
2 
  $%)"!%!"!$ "'% 	
 c] sm # c] sm  3-$ %( ) 
4   $%)"!%!"!Y
 	Y

 c]Y
 smY
 #Y
 c]Y
 smY
 Y
 3-Y
$ iS2C(CDDE%Y
 Y
z" F

!#>>?F
 F
R 
   	 	 	 	 	 	 	 	 	!%) 	#'! 	"!%!!%)-aE$T%T%T#Y"7889::E$ C=	E$
 E$ E$  E$ S#XE$" #E$( )E$. /E$4 5E$: c49n;E$@ TNAE$B TNCE$D EE$J %KE$L E?ME$N OE$V smWE$X #YE$Z c][E$\ ]E$` "#aE$ E$V&&eI==>?&R
4S>=//0  eI==>? N GK}6>sm@   !)-	EEC=E c]E "#	E E EV B
-B B
 B
J 
 #  	JN!%!"!%!%%VV
V sm	V
 SMV <=V &d73D+E&FGV #V 3-V c]V smV  #!V" ##V(  )C6K,K"LLM)V Vr %
&@ %
 %
P  )-!"!%"&!!%#59%)"&")-"%GGG E#t)$%G c]	G
 smG #G 3-G c]G #G tnG eE5==012G c]G C=G 3-G  d^!G" "##G$ d^%G( %yc;U1U'VVW)G G| 	  'A
A

	
A
$ SM%A
& D>'A
L
 .2G.G.$TNG.G.X  $'+j
j
tnj

 )$j
 eM#99:;j
d	 04,0 !"fftDcN+,f Dc3h()f SM	f
 c]f smf fT "&# %Kc Kc Kr   