
    AHjY                       d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZmZmZ d dlZd dlZd dlZd dlmZ d dl m!Z! d dl"m#Z# d dl$Z$d dl%Z$d dl&Z$d dl$m'Z' d d	l(m)Z) d d
l*m+Z+ d dl,m-Z-m.Z.m/Z/m0Z0 d dl1m2Z2m3Z3m4Z4 d dl5m6Z6 d dl7m8Z8 d dl9m:Z:m;Z; d dl<m=Z= d dl>m?Z? d dl@mAZA d dlBmCZD d dlEmFZF d dlGmHZH d dlImJZJ d dlKmLZL d dlMmNZN d dlOmPZP d dlQmRZR d dlSmTZT d dlUmVZV d dlWmXZX d dlYmZZZm[Z[ d dl\m]Z]m^Z^m_Z_ d d l`maZa d d!lbmcZcmdZd d d"lemfZfmgZg d d#lhmiZi d d$ljmkZkmlZlmmZmmnZnmoZo d d%lpmqZqmrZrmsZs d d&ltmuZv d d'lwmxZxmyZy d d(lzm{Z{ d d)l|m}Z} d d*l~mZ d d+lmZ d d,lmZmZ d d-lmZmZ d d.lmZmZmZmZ d d/lmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ d d0lmZ d d1lmZmZmZ d d2lmZ d d2lmZ d d3lmZmZmZ d d4lmZmZmZmZmZmZmZmZmZmZ d5d6lmZ erd d7lmZ d d8lmZmZ d d9lmZ eeef   ZneZeZeZeZ G d: d;ej                        Z G d< d=      Zy)>    Ndefaultdict)	lru_cache)TYPE_CHECKINGAnyAsyncGeneratorCallableDict	GeneratorListLiteralOptionalTupleUnioncast)AsyncOpenAI)	BaseModel)overload)get_secret_str)verbose_router_logger)uuid)	DualCacheInMemoryCache
RedisCacheRedisClusterCache)DEFAULT_HEALTH_CHECK_INTERVAL)DEFAULT_HEALTH_CHECK_STALENESS_MULTIPLIERDEFAULT_MAX_LRU_CACHE_SIZE)CustomLogger)run_async_function)!_get_parent_otel_span_from_kwargs&get_metadata_variable_name_from_kwargs)coroutine_checker)CredentialAccessor)tracer)Logging)SensitiveDataMasker)JSONProviderRegistry)RouterBudgetLimiting)LeastBusyLoggingHandler)LowestCostLoggingHandler)LowestLatencyLoggingHandler)LowestTPMLoggingHandler)LowestTPMLoggingHandler_v2simple_shuffle)get_deployments_for_tag) add_fallback_headers_to_responseadd_retry_headers_to_response)"_get_router_metadata_variable_namereplace_model_in_jsonlshould_replace_model_in_jsonl)InitalizeCachedClient)get_dynamic_litellm_paramsis_clientside_credential)filter_team_based_modelsfilter_web_search_deployments)CooldownCache)DEFAULT_COOLDOWN_TIME_SECONDS_async_get_cooldown_deployments/_async_get_cooldown_deployments_with_debug_info_get_cooldown_deployments_set_cooldown_deployments)#_check_non_standard_fallback_formatget_fallback_model_grouprun_async_fallback)!get_num_retries_from_retry_policy)#async_raise_no_deployment_exceptionsend_llm_exception_alert)DeploymentHealthCache)DeploymentAffinityCheck)ModelRateLimitingCheck)PromptCachingDeploymentCheck)0increment_deployment_failures_for_current_minute1increment_deployment_successes_for_current_minute)FlowItem	Scheduler)AllMessageValues	FileTypesOpenAIFileObjectOpenAIFilesPurpose)#CONFIGURABLE_CLIENTSIDE_AUTH_PARAMSVALID_LITELLM_ENVIRONMENTSAlertingConfigAllowedFailsPolicyAssistantsTypedDictCredentialLiteLLMParamsCustomRoutingStrategyBase
DeploymentDeploymentTypedDictGuardrailTypedDictLiteLLM_ParamsMockRouterTestingParamsModelGroupInfoOptionalPreCallChecksRetryPolicyRouterCacheEnumRouterGeneralSettingsRouterModelGroupAliasItemRouterRateLimitErrorRouterRateLimitErrorBasicRoutingStrategySearchToolTypedDict)ServiceTypes)CustomPricingLiteLLMParamsGenericBudgetConfigTypeLiteLLMBatch)	ModelInfo)ModelResponseStreamStandardLoggingPayloadUsage)
CustomStreamWrapperEmbeddingResponseModelResponseRulesfunction_setupget_llm_provider!get_non_default_completion_params
get_secretget_utc_datetimeis_region_allowed   )PatternMatchRouter)Span)
AutoRouterPreRoutingHookResponseComplexityRouterc                       e Zd ZdZy)RoutingArgs<   N)__name__
__module____qualname__ttl     F/root/tools/cai/cai_env/lib/python3.12/site-packages/litellm/router.pyr   r      s    
Cr   r   c            e       x   e Zd ZU  e       Zeed<   dZee   ed<   dZ	e
ed<   dZdZee   ed<   dZee   ed<   dZeeeeeef         ed	<   ddddddddddi dddddddddddd
dg g g i dddddi ddddddi dd e       dddddf1deeee   eeeef      f      dee   deee      deee      dee   dee   dee
   dee   dee
   dee   dedeee      de
dee   dee
   dee
   dee
   dee   dee   d ee   d!ee
   d"ed#e d$   d%eee      d&ed'ed(ed)eeeeee!f   f      d*ed+ed,ed-e
d.eee"ef      d/eee"f   d0ee
   d1ee#   d2ee   d3ee   d4e d5   d6ee$   d7ed8ee%   d9ee&   d:ee   d;e
d<eeeee   f      d=ed>ed?ee
   d@dfddAZ'e(d@ee   fdB       Z)dC Z*dD Z+e(dEeeef   d@ee,e-f   fdF       Z.dGe,fdHZ/d4ee0ef   d7efdIZ1dJ Z2dK Z3dL Z4dM Z5dN Z6dO Z7dP Z8dQ Z9dR Z:dS Z;dT Z<dU Z=dVee   fdWZ>d6ee$   fdXZ?dYefdZZ@d[ed\eeeef      d@eeAeBf   fd]ZCd[ed\eeeef      d@eeAeBf   fd^ZDd@efd_ZEd`ed\ee   fdaZFeGd[ed\eeH   dbe d   d@eBfdc       ZIeG	 dTd[ed\eeH   dbe d   d@eAfdd       ZIeG	 dTd[ed\eeH   dbee d   e d   f   d@eeBeAf   fde       ZI	 dTd[ed\eeH   dbefdfZIe(dgeJdheeK   d@dfdi       ZLdjeBd\eeeef      dked@eBfdlZMdjeBd\eeeef      dked@eBfdmZNd`ed\ee   fdnZOd[ed\eeeef      d@eeAeBf   fdoZP	 dUd[edpedqee   d@dfdrZQdseRdYed@dfdtZS	 dUdpedqee   d@dfduZT	 dVdYedpedvee   d@eUfdwZVe(dYedped@dfdx       ZW	 dVdYedpedvee   d@dfdyZXdYedpefdzZYdped{ed@eeee
f      fd|ZZdped{ed@eeee
f      fd}Z[dped{ed@eeee
f      fd~Z\dee   d\eeeeef      eeeeef         f   fdZ]d[ed\eeeH      fdZ^eGd[ed\eeeef      dbe d   d@eBfd       Z_eG	 dTd[ed\eeeef      dbe d   d@eAfd       Z_	 dTd[ed\eeeef      dbefdZ_eG	 dTd[ed\eeH   de
dbe d   d@eAf
d       Z`eGd[ed\eeH   de
dbe d   d@eBf
d       Z`	 dTd[ed\eeH   de
fdZ`d[ede
dedeaedf   dpeeef   f
dZbd[ed@efdZcd[ed\eeH   dpeeef   fdZdded[efdZeded[efdZfded[efdZgded[efdZhdeid[efdZjdeid[efdZkd[ededefdZld[efdZmd[efdZn	 	 	 dWd[ededee   dee   dee   f
dZo	 	 	 dWd[ededee   dee   dee   f
dZpd[edefdZq	 	 	 dWded[edee   dee   dee   f
dZrded[efdZsdefdZtd[edefdZudedefdZvd[edefdZwded@dfdZxd[edefdZydpeeef   d[eded@eeef   fdZzd[edefdZ{d[edefdZ|	 dTd[edeeef   dee   d@e}fdZ~deeef   d[efdZ	 dXd[edeeef   dee   d@e}fdZdeeef   d[efdZd[ed@efdZd[ed@efdZd[eedf   fdZd Zd[ed@efdZd[ed@efdZ	 dVd[ee   d@efdZd[ed@efdZd[ed@efdZd[efdZ	 dVdedee   fdZ	 dYdede d   fdĄZ	 dVdedee   fdńZ	 dVdedee   fdƄZdefdǄZ	 dVdedee   fdȄZ	 	 dZdedee   dee   fdʄZdeRdee   d&ee   d'ee   d(ee   dee   dedpefd΄Z ej0                         dτ        Z	 	 	 	 d[dpedee   d&ee   d'ee   d(ee   f
dЄZ ej0                         dф        Zdefd҄Z	 dVdpedee   fdӄZ	 	 	 	 	 d\deRdee   dee   d'ee   d(ee   dee   fd؄Zdل Z	 dVd&eeeee   f      dee   d@eee      fdڄZd@ee   fdۄZ	 	 dZdeRde
de
dee   dee   d@ee
ef   fd݄Zdބ Zd@ee   fd߄Zd@efdZdee   fdZdped@e d   fdZdpedeRd@efdZdedee   d@e
fdZd@efdZd[edeAdped@efdZd[edee   fdZd[edee   d@eaee   ee   f   fdZdYefdZ	 dVdYedee   dee   fdZ	 	 dZd[edee   d\eeeH      dee   dee   dee   fdZdedefdZdedededed@eeU   f
dZded@efdZdYeUfdZded@efdZdYeUfdZdYeUd@efdZdefdZdYeUd@eUfd ZdYeUded[efdZdYeUd@eeU   fdZdede
d@dfdZd[ede
d@dfdZ	 dVd[edee   d@dfdZdYeUd@eeU   fd	ZÐd
ed@eeU   fdZĐded@eeU   fdZŐded@ee   fdZƐded@eeU   fdZǐded@eeeef      fdZeG	 dVdYeedf   ded
dd@efd       ZeGdYdded
ed@efd       Z	 dVdYeeedf      ded
ee   d@efdZʐd
ed@ee   fdZːd
ed@ee   fdZ̐deded@ee   fdZdeded@ee   fdZded@ee   fdZded@eaee
   ee
   f   fdZ eeԐ      ded@ee   fd       Zded@eee
f   fdZ	 dVdedee   d@efd Zded@dfd!Zdefd"Z	 d]dee   d#ed@ee   fd$Zڐd%ed@efd&Zېdee   d@ee   fd'Zܐd(ed)ed@ee   fd*Z	 dVded[ed)ee   d@efd+Z	 	 dZded,ee   d)ee   d@ee   fd-ZߐdVd)ee   d@ee   fd.Z	 dVdYed)ee   d@ee   fd/Zd0ee   d@efd1Z	 dVdee   d@ee   fd2Z	 dZdee   d)ee   d@eee      fd3Zd^d4Zd^d5Z	 	 	 d_dee   d6ee   d)ee   d@eeee   f   fd7Zd6ed@efd8Zd9 Zd: ZdVd;Z	 dVd[eded\eeeef      dee   fd<Zd[ed@ee   fd=Zd[ed@efd>Z	 	 	 	 d`d[ed\eeeeef         deeeef      d?ee   dee   d@eaeeeef   f   fd@Z	 	 	 	 d`d[eded\eeeeef         deeeef      d?ee   dee   d@eee   ef   fdAZ	 	 	 dad[eded\eeeeef         deeeef      d?ee   f
dBZ	 	 	 dad[eded\eeeeef         deeeef      d?ee   f
dCZ	 	 	 dad[eded\eeeeef         deeeef      d?ee   d@ee   fdDZ	 	 	 	 d`d[ed\eeeeef         deeeef      d?ee   dee   f
dEZ	 	 	 	 d`d[ed\eeeeef         deeeef      d?ee   dee   f
dFZdee   dGee   d@ee   fdHZ	 dVdee   dee   d@ee   fdIZ	 dVdee   dee   d@ee   fdJZdee   d@ee   fdKZ	 dVdee   fdLZ	 dVdseRdee   fdMZdseRfdNZdO ZdPefdQZ dR ZdS Zy(b  Routermodel_namesFcache_responsesi  default_cache_time_secondsNleastbusy_loggerlowesttpm_loggeroptional_callbacksINFOTr   simple-shuffle
model_listassistants_configsearch_toolsguardrail_list	redis_url
redis_host
redis_portredis_passwordredis_dbcache_kwargscaching_groups
client_ttlpolling_intervaldefault_prioritynum_retriesmax_fallbackstimeoutstream_timeoutdefault_litellm_paramsdefault_max_parallel_requestsset_verbosedebug_level)DEBUGr   default_fallbacks	fallbackscontext_window_fallbackscontent_policy_fallbacksmodel_group_aliasenable_pre_call_checksenable_tag_filteringtag_filtering_match_anyretry_afterretry_policymodel_group_retry_policyallowed_failsallowed_fails_policycooldown_timedisable_cooldownsrouting_strategy)r   
least-busyusage-based-routinglatency-based-routingcost-based-routingusage-based-routing-v2optional_pre_call_checksrouting_strategy_argsprovider_budget_configalerting_configrouter_general_settingsdeployment_affinity_ttl_secondsmodel_group_affinity_configignore_invalid_deploymentsenable_health_check_routing health_check_staleness_thresholdreturnc2                 x   || _         |/| _        || _        || _        || _        || _        ddlm}2  |2       | _        dt        _
        | j                   du rQ|dk(  r$t        j                  t        j                         n(|dk(  r#t        j                  t        j                         |,xs
 t!               | _        || _        |xs g | _        |xs g | _        g | _        i | _        d}3d}4i }5|| _        ||p|nd}3|||5d	<   |||5d
<   |t1        |      |5d<   |||5d<   |	#t        j2                  d       t1        |	      |5d<   |5j5                  |       | j7                  |5      }4|
r7t        j8                   t        j:                  d&d|3i|5t        _        |
| _        t?        |4tA                     | _        tC        ||4      | _"        || _#        d| _$        || _%        g | _&        tO               | _(        i | _)        i | _*        i | _+        |xs i | _,        i | _-        i | _.        i | _/        |G| ja                  |       | jb                  | _2        |D ]  }6d|6d   v sd| j,                  |6d   d   <   ! ng | _1        d| _3        |#|#| _4        nt        jh                  | _4        |%xs tj        | _6        to        | j8                  | jl                        | _8        |&| _9        |0| _:        |1xs tv        tx        z  }7t{        | j8                  t}        |7            | _?        tA               | _@        ||| _A        n;t        j                  t        j                  | _A        nt        j                  | _A        ||| _D        n;t        j                  t        j                  | _D        nt        j                  | _D        |xs t        j                  | _G        || _H        | | _I        |'| _J        |xs t        j                  }8| j                  |8       |8| _K        |t        j                  H|xs t        j                  }8| j                  | j                  j                  d|8i       n
d|8ig| _K        |xs t        j                  | _O        |xs t        j                  }9| j                  |9       |9| _P        t        t              | _S        t        t              | _T        t        t              | _U        g | _V        |xs i }t        j                  ||       | _X        || _Y        | j                  j                  d|       | j                  j                  dd       | j                  j                  di       j5                  d|i       i | _[        	 | j                  |'|)       d| _]        t        t        j                  t              r*t        j                  j                  | j                         n)t        j                  j                  | j                         t        t        j                  t              r*t        j                  j                  | j                         n| j                  gt        _d        t        t        j                  t              r*t        j                  j                  | j                         n| j                  gt        _g        t        t        j                  t              r*t        j                  j                  | j                         n| j                  gt        _j        |)| _m        |*| _n        |-| _o        d| _p        t        j                  || j                        r|(|(j                  d       ndg}(d| _s        |!t        |!t              rt        d&i |!| _s        nt        |!t              r|!| _s        | j                  >t        j                  d j                  | j                  j                  d!                   |"| _y        |.| _z        d| _{        |$t        |$t              rt        d&i |$| _{        nt        |$t              r|$| _{        | j                  >t        j                  d"j                  | j                  j                  d!                   |+| _}        |(| j                  |(       | j                  rt        d# | j                   xs g D              s| j                   g | _        t        | j8                  | j                  d$d$d$| j                  %      }:| j                   j                  |:       t        j                  j                  |:       | j                  | j                          | j	                          | j                          | j                          y)'a  
        Initialize the Router class with the given parameters for caching, reliability, and routing strategy.

        Args:
            model_list (Optional[list]): List of models to be used. Defaults to None.
            redis_url (Optional[str]): URL of the Redis server. Defaults to None.
            redis_host (Optional[str]): Hostname of the Redis server. Defaults to None.
            redis_port (Optional[int]): Port of the Redis server. Defaults to None.
            redis_password (Optional[str]): Password of the Redis server. Defaults to None.
            cache_responses (Optional[bool]): Flag to enable caching of responses. Defaults to False.
            cache_kwargs (dict): Additional kwargs to pass to RedisCache. Defaults to {}.
            caching_groups (Optional[List[tuple]]): List of model groups for caching across model groups. Defaults to None.
            client_ttl (int): Time-to-live for cached clients in seconds. Defaults to 3600.
            polling_interval: (Optional[float]): frequency of polling queue. Only for '.scheduler_acompletion()'. Default is 3ms.
            default_priority: (Optional[int]): the default priority for a request. Only for '.scheduler_acompletion()'. Default is None.
            num_retries (Optional[int]): Number of retries for failed requests. Defaults to 2.
            timeout (Optional[float]): Timeout for requests. Defaults to None.
            default_litellm_params (dict): Default parameters for Router.chat.completion.create. Defaults to {}.
            set_verbose (bool): Flag to set verbose mode. Defaults to False.
            debug_level (Literal["DEBUG", "INFO"]): Debug level for logging. Defaults to "INFO".
            fallbacks (List): List of fallback options. Defaults to [].
            context_window_fallbacks (List): List of context window fallback options. Defaults to [].
            enable_pre_call_checks (boolean): Filter out deployments which are outside context window limits for a given prompt
            model_group_alias (Optional[dict]): Alias for model groups. Defaults to {}.
            retry_after (int): Minimum time to wait before retrying a failed request. Defaults to 0.
            allowed_fails (Optional[int]): Number of allowed fails before adding to cooldown. Defaults to None.
            cooldown_time (float): Time to cooldown a deployment after failure in seconds. Defaults to 1.
            routing_strategy (Literal["simple-shuffle", "least-busy", "usage-based-routing", "latency-based-routing", "cost-based-routing"]): Routing strategy. Defaults to "simple-shuffle".
            routing_strategy_args (dict): Additional args for latency-based routing. Defaults to {}.
            alerting_config (AlertingConfig): Slack alerting configuration. Defaults to None.
            provider_budget_config (ProviderBudgetConfig): Provider budget configuration. Use this to set llm_provider budget limits. example $100/day to OpenAI, $100/day to Azure, etc. Defaults to None.
            deployment_affinity_ttl_seconds (int): TTL for user-key -> deployment affinity mapping. Defaults to 3600.
            ignore_invalid_deployments (bool): Ignores invalid deployments, and continues with other deployments. Default is to raise an error.
        Returns:
            Router: An instance of the litellm.Router class.

        Example Usage:
        ```python
        from litellm import Router
        model_list = [
        {
            "model_name": "azure-gpt-3.5-turbo", # model alias
            "litellm_params": { # params for litellm completion/embedding call
                "model": "azure/<your-deployment-name-1>",
                "api_key": <your-api-key>,
                "api_version": <your-api-version>,
                "api_base": <your-api-base>
            },
        },
        {
            "model_name": "azure-gpt-3.5-turbo", # model alias
            "litellm_params": { # params for litellm completion/embedding call
                "model": "azure/<your-deployment-name-2>",
                "api_key": <your-api-key>,
                "api_version": <your-api-version>,
                "api_base": <your-api-base>
            },
        },
        {
            "model_name": "openai-gpt-3.5-turbo", # model alias
            "litellm_params": { # params for litellm completion/embedding call
                "model": "gpt-3.5-turbo",
                "api_key": <your-api-key>,
            },
        ]

        router = Router(model_list=model_list, fallbacks=[{"azure-gpt-3.5-turbo": "openai-gpt-3.5-turbo"}])
        ```
        r   )ServiceLoggingTr   r   localNredisurlhostportpasswordzwDeprecated 'redis_db' argument used. Please remove 'redis_db' from your config/database and use 'cache_kwargs' instead.dbtype)redis_cachein_memory_cache)r   r   modellitellm_params)cachedefault_cooldown_time)r   staleness_threshold)fallback_param*)params
router_objr   max_retriesmetadatar   r   r   )r   r   router_budget_limitingz+[32mRouter Custom Retry Policy Set:
{}[0mexclude_nonez3[32mRouter Custom Allowed Fails Policy Set:
{}[0mc              3   <   K   | ]  }t        |t                y wN)
isinstancerI   ).0cbs     r   	<genexpr>z"Router.__init__.<locals>.<genexpr>  s       8
 r238
   Fr   ttl_secondsenable_user_key_affinityenable_responses_api_affinityenable_session_id_affinityr   r   )r   r   r   r   r   r   litellm._service_loggerr   service_logger_objlitellmsuppress_debug_infor   setLevelloggingr   r   rd   r   r   r   r   deployment_namesdeployment_latency_mapr   strwarningupdate_create_redis_cacher   Cacher   r   r   rO   	schedulerr   default_deploymentr   provider_default_deployment_idsr}   pattern_routerteam_pattern_routersauto_routerscomplexity_routersr    model_id_to_deployment_index_map model_name_to_deployment_indices team_model_to_deployment_indicesset_model_listr   healthy_deployments_access_groups_cacher   r=   r   r<   cooldown_cacher   r   r   r   rH   floathealth_state_cachefailed_callsr   openaiDEFAULT_MAX_RETRIESr   ROUTER_MAX_FALLBACKSrequest_timeoutr   r   r   r   r   validate_fallbacksr   appendr   r   r   inttotal_calls
fail_callssuccess_callsprevious_modelsChatchatr   
setdefaultdeployment_statsrouting_strategy_initaccess_groupsr   _async_success_callbacklistlogging_callback_manager"add_litellm_async_success_callbackdeployment_callback_on_successsuccess_callbackadd_litellm_success_callback#sync_deployment_callback_on_success_async_failure_callback"add_litellm_async_failure_callback$async_deployment_callback_on_failurefailure_callbackadd_litellm_failure_callbackdeployment_callback_on_failurer   r   r   router_budget_loggerr)   !should_init_router_budget_limiterr   dictrb   infoformat
model_dumpr   r   r   rW   r   add_optional_pre_call_checksanyr   rI   add_litellm_callback_initialize_alertinginitialize_assistants_endpointinitialize_router_endpointsapply_default_settings);selfr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   
cache_typer   cache_configm
_staleness
_fallbacks_content_policy_fallbacksaffinity_callbacks;                                                              r   __init__zRouter.__init__   s:	   ~ '*D'&&<#$8!'>$:2@2B&*#t#f$%..w||<'%..w}}=#>'<'> 	$ "3(.B,2 	 ')#  	 ')$ Z%;
@V J$&/U#%'1V$%'*:V$)+9Z(#%-- N &)]T" -22<@K}}$ ' N: N N#2D #]_


 #-;
 !1"&-J*:<,02  	! 68AC # 	
 AC- GI-RT-!
+-1__D$ Ra 011PQD//2B0CG0LMR
  O EI!$!.D!(!6!6D*K.K+**D4F4F
 "3+F(5 
),UU 	 #8**%
2C#
 O 	 "*D  ,&22D%99D$!.D"".!(!6!6D!(!=!=D9'"9"9,& 0 3'"3"3
z:#(G,E,E,Q*Gg.G.GJ~~)%%sJ&78#&
"3!4 %H(H(H 	%
 %H(H(H 	" 	/HI(A%(3)
 (3(
 +6+
  	
 "8!=2LL(>4P	 '=###..y'B##..}a@##..z2>EE~.	
 ')	 	""-"7 	# 	
 "g55t<,,OO33 ,,OO33 g..5,,II88 )-(P(P'QG$g55t<,,OO99
 99/G+ g..5,,II33 )-(K(K'LG$%:"&<#/N,DH!AA!$:U:U
 (3(//0HI,D+E(37#,-$/$?,$?!L+6$0!  ,%**HOO))44$4G % 	%
 ( 	( CG!+.5,>,VAU,V)02DE,@)((4%**PWW11<<$<O :I#/--.FG
 ++C 8
..4"8
 5
 &&.*,' 7jj @@)..3+0,0,L,L! ##**+<=,,AABST+%%'++-((*##%r   c                      t        j                  t        j                        } | j                  | j
                  z   }d|v r|j                  d       |S )zS
        Returns a list of valid arguments for the Router.__init__ method.
        r?  )inspectgetfullargspecr   rG  args
kwonlyargsremove)arg_spec
valid_argss     r   get_valid_argszRouter.get_valid_args  sH    
 ))&//:]]X%8%88
Zf%r   c                 *    g }| j                  |       y)z;
        Apply the default settings to the router.
        N)r8  )r?  default_pre_call_checkss     r   r>  zRouter.apply_default_settings  s    
 :<))*ABr   c                 $   t         j                  j                  t         j                  |        t         j                  j                  t         j                  |        t         j                  j                  t         j
                  |        t         j                  j                  t         j                  |        t         j                  j                  t         j                  |        t         j                  j                  t         j                  |        t         j                  j                  t         j                  |        | j                  B| j                  D ]2  }t         j                  j                  t         j                  |d       4 yy)z
        Pseudo-destructor to be invoked to clean up global data structures when router is no longer used.
        For now, unhook router's callbacks from all lists
        NF)require_self)r   r&  #remove_callback_from_list_by_objectr$  r)  r,  r/  input_callbackservice_callback	callbacksr   )r?  callbacks     r   discardzRouter.discard  s8   
 	((LL++T	
 	((LL$$d	
 	((LL++T	
 	((LL$$d	
 	((LL""D	
 	((LL$$d	
 	((LLt	

 "". 33 00TT%%xe U  /r   rA  c                     | j                  d      }|s2t        d      }|%t        |t              rt	        j
                  |      }|rt        di i | d|iS t        di | S )za
        Initializes either a RedisCache or RedisClusterCache based on the cache_config.
        startup_nodesREDIS_CLUSTER_NODESr   )getry   r   r   jsonloadsr   r   )rA  r\  _env_cluster_nodess      r   r   zRouter._create_redis_cache  so     %((9!+,A!B!-*=OQT2U $

+= >$X'W,'W'WXX---r   r   c                 T    | j                   j                  || j                   _        yy)z
        Update the redis cache for the router, if none set.

        Allows proxy user to just do
        ```yaml
        litellm_settings:
            cache: true
        ```
        and caching to just work.
        N)r   r   )r?  r   s     r   _update_redis_cachezRouter._update_redis_cache  s%     ::!!)%*DJJ" *r   c                    t        j                  d|        dgt        D cg c]  }|j                   c}z   }|<t	        |t
              xr ||v }t	        |t              }|s|st        d| d| d      |t        j                  j                  k(  s|t        j                  k(  rt        | j                        | _
        t	        t        j                  t              r*t        j                  j                  | j                         n| j                  gt        _        t	        t        j                  t              r*t        j                   j#                  | j                         y y |t        j$                  j                  k(  s|t        j$                  k(  ret'        | j                  |      | _        t	        t        j                  t              r*t        j                   j#                  | j(                         y y |t        j*                  j                  k(  s|t        j*                  k(  ret-        | j                  |      | _        t	        t        j                  t              r*t        j                   j#                  | j.                         y y |t        j0                  j                  k(  s|t        j0                  k(  ret3        | j                  |      | _        t	        t        j                  t              r*t        j                   j#                  | j4                         y y |t        j6                  j                  k(  s|t        j6                  k(  ret9        | j                  i       | _        t	        t        j                  t              r*t        j                   j#                  | j:                         y y y c c}w )NzRouting strategy: r   zInvalid routing_strategy: 'z'. Valid options: z. Check 'router_settings.routing_strategy' in your config.yaml or the 'routing_strategy' parameter if using the Router SDK directly.)router_cache)re  routing_args)r   r5  rh   valuer   r   
ValueError
LEAST_BUSYr*   r   r   r   rV  r%  r  rX  r&  r:  USAGE_BASED_ROUTINGr-   r   USAGE_BASED_ROUTING_V2r.   lowesttpm_logger_v2LATENCY_BASEDr,   lowestlatency_logger
COST_BASEDr+   lowestcost_logger)r?  r   r   svalid_strategy_stringsis_valid_stringis_valid_enums          r   r"  zRouter.routing_strategy_init  s    	""%78H7I#JK
 #3!36X1qww6X!X'+S1 ?$(>>  ''7IM"= 12B1C D&&<%= >\]   : : @ @@?#=#==$;$TD!'00$7&&--d.C.CD*.*?*?)@&'++T200EEdF[F[\ 3  C C I II?#F#FF$;!ZZ2%D! '++T200EEdF[F[\ 3  F F L LL?#I#II'A!ZZ2(D$ '++T200EEdF^F^_ 3  = = C CC?#@#@@(C!ZZ2)D% '++T200EEdF_F_` 3  : : @ @@?#=#==%=!ZZ&D" '++T200EEdF\F\] 3 K 7Ys   O c                 D   | j                  t        j                        | _        | j                  t        j                        | _        | j                  t        j                        | _        | j                  t        j
                        | _        | j                  t        j                        | _        | j                  t        j                        | _        | j                  t        j                        | _        | j                  t        j                        | _	        y r   )
factory_functionr   acreate_assistantsadelete_assistantaget_assistantsacreate_threadaget_threada_add_messageaget_messagesarun_threadr?  s    r   r<  z%Router.initialize_assistants_endpointl  s    "&"7"78R8R"S!%!6!6w7P7P!Q#44W5L5LM"33G4J4JK001D1DE!2273H3HI!2273H3HI001D1DEr   c                    | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j
                  d      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _	        | j                  t        j                  d	      | _
        | j                  t        j                  d
      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j                   d      | _        | j                  t        j"                  d      | _        | j                  t        j$                  d      | _        | j                  t        j&                  d      | _        | j                  t        j(                  d      | _        | j                  t        j*                  d      | _        | j                  t        j,                  d      | _        | j                  t        j.                  d      | _        | j                  t        j0                  d      | _        y)z+Helper to initialize core router endpoints.
moderation	call_typeanthropic_messagesagenerate_contentaadapter_generate_content
aresponsesafile_deleteafile_content	responsesaget_responsesacancel_responsesacompact_responsesadelete_responsesalist_input_items
_arealtime_aresponses_websocketacreate_fine_tuning_jobacancel_fine_tuning_jobalist_fine_tuning_jobsaretrieve_fine_tuning_jobalist_filesaimage_editallm_passthrough_routeN)rv  r   amoderationr  aanthropic_messagesr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  
afile_listr  r  r  s    r   _initialize_core_endpointsz!Router._initialize_core_endpointsw  sW   00< 1 
 $(#8#8&&2F $9 $
  #'"7"7&&2F #8 #
 "&!6!6%%1D "7 "
 *.)>)>--9T *? *
& //, 0 
 !11  N 2 
 "22!!_ 3 
 ..w/@/@K.X"33"".> 4 
 "&!6!6%%1D "7 "
 #'"7"7&&2F #8 #
 "&!6!6%%1D "7 "
 "&!6!6%%1D "7 "
 //, 0 
 &*%:%:))5L &; &
" (,'<'<++7P (= (
$ (,'<'<++7P (= (
$ '+&;&;**6N '< '
# *.)>)>--9T *? *
& //- 0 
  00= 1 
 '+&;&;**6N '< '
#r   c                 H   ddl m}m}m}m}m}m}m}m}m	}	m
}
m} | j                  |d      | _        | j                  |
d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |	d      | _        | j                  |d	      | _        | j                  |d
      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        y)z"Initialize vector store endpoints.r   )adeletealist	aretrieveasearchaupdatecreatedeleter%  retrievesearchr   avector_store_searchr  vector_store_searchvector_store_createavector_store_retrievevector_store_retrieveavector_store_listvector_store_listavector_store_updatevector_store_updateavector_store_deletevector_store_deleteN)litellm.vector_stores.mainr  r  r  r  r  r  r  r%  r  r  r   rv  r  r  r  r  r  r  r  r  r  r  r  )r?  r  r  r  r  r  r  r  r%  r  r  r   s               r   "_initialize_vector_store_endpointsz)Router._initialize_vector_store_endpoints  sv   	
 	
 	
 	
 %)$9$95 %: %
! $(#8#83 $9 $
  $(#8#83 $9 $
  '+&;&;!9 '< '
# &*%:%: 7 &; &
" #'"7"71 #8 #
 "&!6!6/ "7 "
 %)$9$95 %: %
! $(#8#83 $9 $
  %)$9$95 %: %
! $(#8#83 $9 $
 r   c                    ddl m} ddl m} ddl m} ddl m} ddl m} ddl m} ddl m} dd	l m} dd
l m	}	 ddl m
}
 ddl m} ddl m} | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |	d      | _        | j                  |d      | _        | j                  |
d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        y)z'Initialize vector store file endpoints.r   acreater  )r  )r  )aretrieve_content)r  r  r  )r%  )r  )retrieve_content)r   avector_store_file_creater  vector_store_file_createavector_store_file_listvector_store_file_listavector_store_file_retrievevector_store_file_retrieveavector_store_file_contentvector_store_file_contentavector_store_file_updatevector_store_file_updateavector_store_file_deletevector_store_file_deleteN)litellm.vector_store_files.mainr  r  r  r  r  r  r  r  r%  r  r  r   rv  r  r  r  r  r  r  r  r  r  r  r  r  )r?  avector_store_file_create_fnavector_store_file_delete_fnavector_store_file_list_fnavector_store_file_retrieve_fnavector_store_file_content_fnavector_store_file_update_fnvector_store_file_create_fnvector_store_file_delete_fnvector_store_file_list_fnvector_store_file_retrieve_fnvector_store_file_content_fnvector_store_file_update_fns                r   '_initialize_vector_store_file_endpointsz.Router._initialize_vector_store_file_endpoints  s   	
	
 	X	
	
	
	
	
 	V	
	
	
 *.)>)>(4O *? *
& )-(=(='3M )> )
% (,'<'<&2K (= (
$ '+&;&;%1I '< '
# ,0+@+@*6S ,A ,
( +/*?*?)5Q +@ +
' +/*?*?)5Q +@ +
' *.)>)>(4O *? *
& *.)>)>(4O *? *
& )-(=(='3M )> )
% *.)>)>(4O *? *
& )-(=(='3M )> )
%r   c                     ddl m}m}m}m} | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        y)	z"Initialize Google GenAI endpoints.r   )r  agenerate_content_streamgenerate_contentgenerate_content_streamr  r  r  r  r  N)litellm.google_genair  r  r  r  rv  )r?  r  r  r  r  s        r   "_initialize_google_genai_endpointsz)Router._initialize_google_genai_endpoints9  s    	
 	
 "&!6!6)< "7 "
 !% 5 5(: !6 !
 )-(=(=$0J )> )
% (,'<'<#/H (= (
$r   c                     ddl m}m} | j                  |d      | _        | j                  |d      | _        ddlm}m} | j                  |d      | _        | j                  |d      | _        y	)
z$Initialize OCR and search endpoints.r   )aocrocrr  r  r  r  r  r  r  N)litellm.ocrr  r  rv  litellm.searchr  r  )r?  r  r  r  r  s        r    _initialize_ocr_search_endpointsz'Router._initialize_ocr_search_endpointsO  sd    )))$&)A	(((>2,,W	,J++Fh+Gr   c                    ddl m}m}m}m}m}m}m}m}m	}	m
}
m}m}m}m}m}m}m}m} | j'                  |d      | _        | j'                  |d      | _        | j'                  |d      | _        | j'                  |d      | _        | j'                  |	d      | _	        | j'                  |d	      | _        | j'                  |d
      | _        | j'                  |
d      | _
        | j'                  |d      | _        | j'                  |d      | _        | j'                  |d      | _        | j'                  |d      | _        | j'                  |d      | _        | j'                  |d      | _        | j'                  |d      | _        | j'                  |d      | _        | j'                  |d      | _        | j'                  |d      | _        y)zInitialize video endpoints.r   )avideo_contentavideo_create_characteravideo_editavideo_extensionavideo_generationavideo_get_characteravideo_listavideo_remixavideo_statusvideo_contentvideo_create_character
video_editvideo_extensionvideo_generationvideo_get_character
video_listvideo_remixvideo_statusr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  N)litellm.videosr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rv  )r?  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  s                      r   _initialize_video_endpointsz"Router._initialize_video_endpoints[  s-   	
 	
 	
 	
 	
* "&!6!6)< "7 "
 !% 5 5(: !6 !
  000V//
l/S!22_ 3 
 !11N 2 
 #33&6 4 
 "22_ 3 
 !11N 2 
  000V'+'<'<#/H (= (
$ '+&;&;".F '< '
# %)$9$9 ,B %: %
! $(#8#8+@ $9 $
   000V//
l/S $ 5 5(: !6 !
  $44'8  5  
r   c           	      $   ddl m}m}m}m}m}m}m}m} ddl	m
}	 | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d	      | _        | j                  |d
      | _        | j                  |d      | _        | j                  |d      | _        |	j                         D ]#  \  }
}t        | |
| j                  ||
             % y)zInitialize container endpoints.r   )acreate_containeradelete_containeralist_containersaretrieve_containercreate_containerdelete_containerlist_containersretrieve_container)_generated_endpointsr  r  r  r  r  r  r   r  r  N)litellm.containersr  r  r  r  r  r  r  r   #litellm.containers.endpoint_factoryr  rv  itemssetattr)r?  r  r  r  r  r  r  r  r   container_file_endpointsnamefuncs               r   _initialize_container_endpointsz&Router._initialize_container_endpoints  sR   		
 		
 		
	
 "&!6!6)< "7 "
 !% 5 5(: !6 !
 !% 5 5(: !6 !
  $44'8  5  
 $(#8#8+@ $9 $
  #'"7"7*> #8 #
 "&!6!6)< "7 "
 !% 5 5(: !6 !

 388: 	MJD$D$ 5 5dd 5 KL	Mr   c                 4   | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j                  d      | _        | j                  t        j
                  d      | _        y)z*Initialize Anthropic Skills API endpoints.acreate_skillr  alist_skills
aget_skilladelete_skillN)rv  r   r  r  r  r  r  s    r   _initialize_skills_endpointsz#Router._initialize_skills_endpoints  s    !22!!_ 3 
 !11  N 2 
 //, 0 
 "22!!_ 3 
r   c                    ddl m} ddl m} ddl m} ddl m} ddl m} ddl m} ddl m} dd	l m} | j                  |d
      | _
        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        | j                  |d      | _        y)z-Initialize Google Interactions API endpoints.r   )acancelr  r  )aget)cancelr  r  r^  acreate_interactionr  create_interactionaget_interactionget_interactionadelete_interactiondelete_interactionacancel_interactioncancel_interactionN)litellm.interactionsr  r  r  r  r  r  r  r^  rv  r  r  r  r  r  r  r  r  )	r?  r  r  r  r  r  r  r  r  s	            r   "_initialize_interactions_endpointsz)Router._initialize_interactions_endpoints  s   GGGAEEE?#'#8#8+@ $9 $
  #'"7"7*> #8 #
 !% 5 5(: !6 !
  $44'8  5  
 $(#8#8+@ $9 $
  #'"7"7*> #8 #
 $(#8#8+@ $9 $
  #'"7"7*> #8 #
r   c                 $   | j                          | j                          | j                          | j                          | j	                          | j                          | j                          | j                          | j                          y)zvHelper to initialize specialized router endpoints (vector store, OCR, search, video, container, skills, interactions).N)	r  r  r  r  )_override_vector_store_methods_for_routerr  r	  r  r  r  s    r   !_initialize_specialized_endpointsz(Router._initialize_specialized_endpoints  sn    //1446//1--/668((*,,.))+//1r   c                 D    | j                          | j                          y r   )r  r!  r  s    r   r=  z"Router.initialize_router_endpoints  s    '')..0r   r   c           	          |y|D ]J  }t        |t              st        d| d      t        |      dk7  s1t        d| dt        |       d       y)z3
        Validate the fallbacks parameter.
        NzItem 'z' is not a dictionary.r|   zDictionary 'z%' must have exactly one key, but has z keys.)r   r4  rh  len)r?  r   fallback_dicts      r   r  zRouter.validate_fallbacks  sq     !+ 	MmT2 6-8N!OPP=!Q& "=/1VWZ[hWiVjjpq 		r   c                    |y d|v }d|v }d|v }|s|s|r| j                   g | _         d }| j                   D ]  }t        |t              s|} n |n|j                  xs ||_        |j                  xs ||_        |j
                  xs ||_        | j                  |_        | j                  r{| j                  |_        nit        | j                  | j                  |||| j                        }| j                   j                  |       t        j                  j                  |       d|v rzddlm | j                   g | _         t!        fd| j                   D              }|sA        }	| j                   j                  |	       t        j                  j                  |	       |D ]  }
d }|
d	v r
|
d
k(  rt#        | j                        }nM|
dk(  r-t%        | j                  | j&                  | j(                        }n|
dk(  rt+        | j                        }|v| j                   g | _         | j                   j                  |       t        j                  j                  |        y )Ndeployment_affinityresponses_api_deployment_checksession_affinityr   encrypted_content_affinityr   )EncryptedContentAffinityCheckc              3   6   K   | ]  }t        |        y wr   )r   )r   r   r+  s     r   r   z6Router.add_optional_pre_call_checks.<locals>.<genexpr>e  s!      % 2<=%s   )r'  r(  r)  r*  prompt_caching)r   r   )
dual_cacher   r   enforce_model_rate_limits)r.  )r   r   rI   r   r   r   r   r   r   r   r  r   r&  r:  Elitellm.router_utils.pre_call_checks.encrypted_content_affinity_checkr+  r9  rK   r)   r   r   rJ   )r?  r   r   r   r   existing_affinity_callbackr   rF  already_registeredec_callbackpre_call_check	_callbackr+  s               @r   r8  z#Router.add_optional_pre_call_checks  s    $+
 $9<T#T ,0HH 	& &8;S%S"$,)&&.*,'LP&-- b"9:13.
 *5.GG 0/ +C
 /LL 54 +H
 /II 21 +E
 88 +6 3388 /J %<** $ D D-E2O/I040P0P%! ''../@A00EEFWX
 (+CC &&.*,'!$ %11% " &;=''..{;00EEkR
 7 	MN04I "  !118tzzJ	#;;0#zz+/+F+F#	
  #>>2djjI	 &&.*,'##**95,,AA)L5	Mr   
deploymentc                 .   	 t        j                  |      }|d   }t        j                  r#t	        dd      }|j                  |      |d<   |S d|v r|d   dd dz   |d<   |S # t        $ r(}t        j                  dt        |              |d}~ww xY w)	z
        returns a copy of the deployment with the api key masked

        Only returns 2 characters of the api key and masks the rest with * (10 *).
        r      r   )visible_prefixvisible_suffixapi_keyNz
**********z+Error occurred while printing deployment - )
copydeepcopyr   redact_user_api_key_infor'   	mask_dict	Exceptionr   debugr   )r?  r6  _deployment_copyr   maskeres         r   print_deploymentzRouter.print_deployment  s    	#}}Z8#34D#EN//,AaP5;5E5En5U !12 $# n,,:9,Ebq,IH,Ty)## 	!''=c!fXF G		s   AA# A# #	B,#BBr   messagesc                     	 t        j                  d| d       ||d<   ||d<   | j                  |d<   | j                  ||        | j                  di |}|S # t
        $ r}|d}~ww xY w)	z
        Example usage:
        response = router.completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hey, how's it going?"}]
        zrouter.completion(model=z,..)r   rF  original_functionr   kwargsNr   )r   rA  _completion_update_kwargs_before_fallbacksfunction_with_fallbacksr@  )r?  r   rF  rJ  responserD  s         r   
completionzRouter.completion  s    
	!''*B5'(NO#F7O!)F:*.*:*:F&'00uV0L3t33=f=HO 	G	s   AA 	A+$A&&A+c           	      4   d }d }	 |j                         }||d<   | j                  |||j                  dd       |      }|d   j                         }|j                  dd       }|5t        j                  | j
                  ||f|d      }	|	j                          | j                  ||       |j                  dd        |d   }| j                  ||      }
|j                  d	d       }||
||
j                  k7  rd }n|
}| j                  |      s| j                  |
       i ||| j                  |d|}t        j                  di |}t!        j"                  d| d       t%        |t&              r.| j)                  |||      }|rt        j*                  d|d      t%        |t,              r| j/                  |||      S |S # t0        $ r@}t!        j"                  d| dt3        |       d       || j5                  ||       |d }~ww xY w)Nr   specific_deploymentr   rF  rQ  request_kwargsr   silent_modelT)targetrK  rJ  daemonr6  rJ  r;  r6  rF  cachingclientzlitellm.completion(model=)[32m 200 OK[0mr   rN  rJ  Response output was blocked. messager   llm_providermodel_responserF  initial_kwargs)[31m Exception [0mr   )r<  get_available_deploymentpop	threadingThread_silent_experiment_completionstart_update_kwargs_with_deployment_get_clientr^  r;  has_model_id routing_strategy_pre_call_checksr   r   rO  r   r5  r   rt   "_should_raise_content_policy_errorContentPolicyViolationErrorrr   _completion_streaming_iteratorr@  r   (_set_deployment_num_retries_on_exception)r?  r   rF  rJ  
model_namer6  #input_kwargs_for_streaming_fallbackr   rT  threadpotential_model_clientdynamic_api_keymodel_clientinput_kwargsrN  _should_raiserD  s                    r   rK  zRouter._completion  s    

^	 39++-/;@/8 66!$*JJ/Dd$K%	 7 J ((89>>@N)--ndCL'
 #))==&1!	 //:f/UJJ~t,'0J%)%5%5%f &6 &" %jjD9O+*6#'='E'EE#5 $$U+555L $//&	
 L ))9L9H!&&+J<7NO
 (M2 $ G G(6 !H ! !!== >#%'  ($78::#+%#F ;   O 	!&&+J<7KCPQF8SZ[ %==aLG	s   GG G 	H;HHc                 L   ddl m}  ||      }|j                  d      }|!|j                  d      |u rt        |      |d<   d|vri |d<   |d   j	                  dd       d|d   d<   d|d	<   |j	                  d
d       |j	                  dd       |j	                  dd       |S )u  
        Prepare kwargs for a silent experiment by ensuring isolation from the primary call.

        Guarantee metadata isolation: safe_deep_copy falls back to the original
        reference when deepcopy fails (e.g. metadata contains UserAPIKeyAuth with
        parent_otel_span — an OTel Span that is not deepcopy-able). Force a shallow
        copy of the metadata dict so mutations (model_group, is_silent_experiment)
        never corrupt the main call's metadata.
        r   safe_deep_copyr   Nlitellm_parent_otel_spanTis_silent_experimentFstreamlitellm_call_idlitellm_logging_objstandard_logging_object)'litellm.litellm_core_utils.core_helpersr  r^  r4  ri  )r?  rJ  r  silent_kwargsoriginal_metadatas        r   _get_silent_experiment_kwargsz$Router._get_silent_experiment_kwargs  s     	K&v. #JJz2)!!*-1BB(,->(?M*%]*(*M*%
 	j!%%&@$G<@j!"89 #(h 	+T2/63T: r   rT  c           	      0    	 |j                  di       j                  dd      ryt        j                        t        j                  d          j
                  d
i |d   d<   t        j                         }t        j                  |       	  fd}|j                   |              |j                          y# |j                          w xY w# t        $ r.}t        j                  d d	t        |              Y d}~yd}~ww xY w)zE
        Run a silent experiment in the background (thread).
        r   r  FN%Starting silent experiment for model model_groupc                  0  K    j                   dt        t        t                 d d {    t	        j
                         } t	        j                         }|| j                  |       | rt	        j                  | ddi d {    y y 7 `7 w)Nr   rF  return_exceptionsTr   )	acompletionr   r   rP   asyncio	all_taskscurrent_taskrZ  gather)pendingcurrentrF  r?  r  rT  s     r   _run_silent_completionzDRouter._silent_experiment_completion.<locals>._run_silent_completionj  s     *$** *!%d+;&<h!G (   &//1G%224G*0%nngNNNN  Os"   .BBABBBB#Silent experiment failed for model : r   )r^  r<  r=  r   r5  r  r  new_event_loopset_event_looprun_until_completecloser@  errorr   )r?  rT  rF  rJ  loopr  rD  r  s   ```    @r   rl  z$Router._silent_experiment_completionN  s    +	zz*b)--.DeL}}X.H!&&7~F ?D>>HHM 8DM*%m4 ))+D""4(O ''(>(@A



 	!''5l^2c!fXN 	s5   "C A0C C	 8C 	CC 	D'$DDr  c                    K   y wr   r   r?  r   rF  r  rJ  s        r   r  zRouter.acompletion        	   c                    K   y wr   r   r  s        r   r  zRouter.acompletion  r  r  c                    K   y wr   r   r  s        r   r  zRouter.acompletion  r  r  c                 &  K   	 ||d<   ||d<   ||d<   | j                   |d<   | j                  ||       |j                  d      xs | j                  }t	        j                         }| j                  |      }|r| j                  |||       d {   S |+t        |t              r | j                  di | d {   }n | j                  di | d {   }t	        j                         }	|	|z
  }
t        j                  | j                  j                  t        j                   |
d||	t#        |      	             |S 7 7 7 n# t$        $ r;}t        j                  t'        | |t)        j*                         |
             |d }~ww xY ww)Nr   rF  r  rH  rI  priority)r   rF  rJ  r  servicedurationr  
start_timeend_timeparent_otel_spanlitellm_router_instancerS  error_traceback_stroriginal_exceptionr   )_acompletionrL  r^  r   time_is_prompt_management_model_prompt_management_factoryr   r  schedule_acompletionasync_function_with_fallbacksr  create_taskr   async_service_success_hookrj   ROUTERr!   r@  rG   	traceback
format_exc)r?  r   rF  r  rJ  request_priorityr  r  rN  r  	_durationrD  s               r   r  zRouter.acompletion  s    ,	#F7O!)F:%F8*.*;*;F&'00uV0L%zz*5N9N9NJ*.*J*J5*Q'*!<<%! =   
  +
;KS0Q!:!:!:!DV!DD!C!C!C!Mf!MMyy{H :-I''BB(//&+)%%Fv%N C 	 O- EM  		(,0#)(1(<(<(>'(	 G		sl   FBE
 EE
 F'E
 >E?E
 EA)E
 FE
 E
 E
 
	F6F		FFfallback_itemcomplete_response_object_usagec                     ddl m} t        t        t           t        | dd            }||gng }|)t        |d      r|j                  |j                  |       |j                  |      }t        | d|       y)zCMerge partial-stream usage with fallback-stream usage on the chunk.r   )BaseTokenUsageProcessorusageN)usage_objects)litellm.cost_calculatorr  r   r   rq   getattrhasattrr  r  combine_usage_objectsr  )r  r  r  r  r  combined_usages         r   _combine_fallback_usagezRouter._combine_fallback_usage  s     	DXe_gmWd&KL#(#4"*66@.44@  !?@0FF' G 
 	w7r   rd  re  c                 n    K   ddl m  G fddt              } fd} | |             S w)z
        Helper to iterate over a streaming response.

        Catches errors for fallbacks using the router's fallback system
        r   MidStreamFallbackErrorc                   4     e Zd Zdef fdZd Zd Z xZS )ERouter._acompletion_streaming_iterator.<locals>.FallbackStreamWrapperasync_generatorc                     t         |   |j                  j                  j                         || _        t        d      r j                  j                         | _        y y N)completion_streamr   custom_llm_providerlogging_obj_hidden_params)	superrG  r   r  r  _async_generatorr  r  r<  )r?  r  	__class__rd  s     r   rG  zNRouter._acompletion_streaming_iterator.<locals>.FallbackStreamWrapper.__init__  sd     &5(..(6(J(J . : :	 !  )8%>+;<*8*G*G*L*L*ND' =r   c                     | S r   r   r  s    r   	__aiter__zORouter._acompletion_streaming_iterator.<locals>.FallbackStreamWrapper.__aiter__      r   c                 R   K   | j                   j                          d {   S 7 wr   )r  	__anext__r  s    r   r  zORouter._acompletion_streaming_iterator.<locals>.FallbackStreamWrapper.__anext__   s!     !22<<>>>>s   '%')r   r   r   r   rG  r  r  __classcell__r  rd  s   @r   FallbackStreamWrapperr    s    O O?r   r  c                   K   d } 	 2 3 d {   }| 7 
6 n# $ r}ddl m}  |j                        }t        t        t
           t        |dd             }	 t        t        j                  d            }j                  dj                        }j                  dj                        }j                  dj                        }	j                  d	<   |j                  s|j                  sd
<   ndddd|j                  ddgz   d
<   j                  |       j!                  |d|||	|d       d {  7  } t#        | d      rD| 2 3 d {  7  }
|
r.t%        |
t&              rt#        |
d      rj)                  |
|       |
 @6 nd  n+# t*        $ r}t-        j.                  d|        |d }~ww xY wY d }~nd }~ww xY wt1        j2                  d      5  t#        d      rG	 j5                          d {  7   n,# t6        $ r }t-        j8                  d|       Y d }~nd }~ww xY w| St#        | d      rG	 | j5                          d {  7   n,# t6        $ r }t-        j8                  d|       Y d }~nd }~ww xY wd d d        y # 1 sw Y   y xY w# t1        j2                  d      5  t#        d      rG	 j5                          d {  7   n,# t6        $ r }t-        j8                  d|       Y d }~nd }~ww xY w| St#        | d      rG	 | j5                          d {  7   n,# t6        $ r }t-        j8                  d|       Y d }~nd }~ww xY wd d d        w # 1 sw Y   w xY wxY ww)Nr   stream_chunk_builderchunksr  r   r   r   r   rH  rF  system  You are a helpful assistant. You are given a message and you need to respond to it. You are also given a generated content. You need to respond to the message in continuation of the generated content. Do not repeat the same content. Your response should be in continuation of this text: rolecontent	assistantTr  r  prefixrI  Fr   )rD  disable_fallbacksr   r   r   r  rK  rJ  r  Fallback also failed: )shieldaclose7stream_with_fallbacks: error closing model_response: %s:stream_with_fallbacks: error closing fallback_response: %s)litellm.mainr  r  r   r   rq   r  r   r^  r   r   r   r  is_pre_first_chunkgenerated_contentrL  *async_function_with_fallbacks_common_utilsr  r   ro   r  r@  r   r  anyioCancelScoper  BaseExceptionrA  )fallback_responseitemrD  r  complete_response_objectr  r  r   r   r   r  fallback_errorr  re  rF  rd  r?  s               r   stream_with_fallbackszERouter._acompletion_streaming_iterator.<locals>.stream_with_fallbacks  s     $i"0  $J.) N)=+?)00,( 26UO4gtD2.D)"&sN,>,>w,G"HK0>0B0B#T^^1I @N?Q?Q2D4Q4Q@, @N?Q?Q2D4Q4Q@, ;?:K:KN#67++13F3F
 6>z25=(0 ,M
 )4+,+>+>*.
A 
6z2 88). 9  #MM.3&/5M5M(3!##1 N 	 	 	 & 0+>3D 	0 	0- -$.}>Q$R$+M7$C $ < <$13Q!" #0/ 4E #
  ))//00@A )()SN)h &&d3 ~x8"0"7"7"999, 177 Y ! 
 )4)8:"3":":"<<<, 177 \ !   U&&d3 ~x8"0"7"7"999, 177 Y ! 
 )4)8:"3":":"<<<, 177 \ !   s  N	   J1 G;GC*F%	E
F%F!E$"F&>F%$G%	G.GGGJ1 GJ1 N	3J%HHHJ%	I$H?:J%?IJ%I0)I,*I0/J%0	J9JJ%JJ%	N	%J.*N	1NM:K0)K,
*K0/M:0	L	9L	M:L	M:+M>M
?MM:	M.	M)	$M:)M.	.M:1	N:N?NN	litellm.exceptionsr  rr   )r?  rd  rF  re  r  r  r  s   ````  @r   _acompletion_streaming_iteratorz&Router._acompletion_streaming_iterator  s7      	>	?$7 	?(k	 k	Z %%:%<==s   .5c                 j    ddl m  G fddt              }| fd} | |             S )a  
        Sync equivalent of _acompletion_streaming_iterator.

        Wraps a sync streaming response so that MidStreamFallbackError
        (raised by CustomStreamWrapper.__next__) triggers the Router's
        fallback chain instead of surfacing directly to the caller.
        r   r  c                   4     e Zd Zdef fdZd Zd Z xZS )HRouter._completion_streaming_iterator.<locals>.SyncFallbackStreamWrappersync_generatorc                     t         |   |j                  j                  j                         || _        t        d      r j                  j                         | _        y y r  )	r  rG  r   r  r  _sync_generatorr  r  r<  )r?  r  r  rd  s     r   rG  zQRouter._completion_streaming_iterator.<locals>.SyncFallbackStreamWrapper.__init__  sd     &4(..(6(J(J . : :	 !  (6$>+;<*8*G*G*L*L*ND' =r   c                     | S r   r   r  s    r   __iter__zQRouter._completion_streaming_iterator.<locals>.SyncFallbackStreamWrapper.__iter__  r  r   c                 ,    t        | j                        S r   )nextr
  r  s    r   __next__zQRouter._completion_streaming_iterator.<locals>.SyncFallbackStreamWrapper.__next__  s    D0011r   )r   r   r   r   rG  r  r  r  r  s   @r   SyncFallbackStreamWrapperr    s    	Oy 	O2r   r  c            	   3     K   d } 	 D ]  }|  	 t#        d      r	 j1                          | t#        | d      r	 | j1                          y y y # $ r}ddl m}  |j                        }t        t        t
           t        |dd             }	 t        t        j                  d            }j                  dj                        }j                  dj                        }j                  dj                        }	j                  d	<   |j                  s|j                  sd
<   ndddd|j                  ddgz   d
<   j                  |        j                   di |||	d} t#        | d      r<| D ]6  }
|
r.t%        |
t&              rt#        |
d      rj)                  |
|       |
 8 nd  n+# t*        $ r}t-        j.                  d|        |d }~ww xY wY d }~d }~ww xY w# t2        $ r!}t-        j4                  d|       Y d }~d }~ww xY w# t2        $ r }t-        j4                  d|       Y d }~y d }~ww xY w# t#        d      r>	 j1                          n,# t2        $ r }t-        j4                  d|       Y d }~nd }~ww xY w| Kt#        | d      r>	 | j1                          w # t2        $ r }t-        j4                  d|       Y d }~w d }~ww xY ww w xY ww)Nr   r  r  r  r   r   r   r   rH  rF  r  r  r  r  Tr  rI  )r   r   r   r  r  r  r  r  r   )r  r  r  r   r   rq   r  r   r^  r   r   r   rK  r  r  rL  rM  r  r   ro   r  r@  r   r  r  r  rA  )r  r  rD  r  r  r  r  r   r   r   r  r   	close_errr  re  rF  rd  router_selfs                r   r  zDRouter._completion_streaming_iterator.<locals>.stream_with_fallbacks  sc     $X* DJL >73&,,. %0W%w6)//1	60Y * B)=+?)00,( 26UO4gtD2.8)"&sN,>,>w,G"HK0>0B0B#[%:%:1I @N?Q?Q2#<<@, @N?Q?Q2#<<@, ;F:Q:QN#67++13F3F5=z25=(0 ,M
 )4+,+>+>*.
A 
6z2  ??). @  )L(K(K )()"+1I1I	)% 0*=-> 	0M -$.}>Q$R$+M7$C + C C$13Q!" #0/	0 #
  ))//00@A )(	)}B)N ) -33U%  ) -33X%  >73&,,.( -33U% 
 %0W%w6)//1( -33X% 60s  K5A I K5H  K5H- K5G=;G8D2GG8	G/G**G//G82I 8G==I  	H*	H%K5%H**K5-	I6IK5IK5K2'I87K28	J!JK2J!!K23KK2	K-K(#K2(K--K22K5r  )r?  rd  rF  re  r  r  r  r  s    ```  @@r   rt  z%Router._completion_streaming_iteratorr  s:     	>	2(; 	2$ Z	 Z	x ))>)@AAr   c           	        K   	 |j                  di       j                  dd      ryt        j                  |      }t        j                  d|         | j
                  d
i |}||d   d<    | j                  d
|t        t        t           |      d| d{    y7 # t        $ r.}t        j                  d| d	t        |              Y d}~yd}~ww xY ww)z<
        Run a silent experiment in the background.
        r   r  FNr  r  r  r  r  r   )r^  r<  r=  r   r5  r  r  r   r   rP   r@  r  r   )r?  rT  rF  rJ  r  rD  s         r   _silent_experiment_acompletionz%Router._silent_experiment_acompletion  s     	zz*b)--.DeL}}X.H!&&7~F ?D>>HHM7CM*%m4 #$"" "d#34h?    
  	!''5l^2c!fXN 	sK   C"B# CA4B# B!B#  C!B# #	C,$CCCCc                 p  K   d}d}i }	 |j                         }||d<   t        |      }t        j                         }	| j                  |||j	                  dd      |       d{   }|}t        j                         }
|
|	z
  }t        j                  | j                  j                  t        j                  |d|	|
t        |                   | j                  ||       |d   j                         }|j	                  d	d      }|(t        j                   | j                  d ||d
|       | j                  ||       |j	                  d	d       |d   }| j                  ||      }| j                  |xx   dz  cc<   i ||| j                   |d|}|j	                  d	d       t#        j$                  d i |}|j'                  dd      }| j)                  ||d      }|]t+        |t
        j,                        rC|4 d{    	 | j/                  |||       d{    | d{   }ddd      d{    n&| j/                  |||       d{    | d{   }t+        t0              r.| j3                  |||      }|rt#        j4                  d|d      | j6                  |xx   dz  cc<   t9        j:                  d| d       | j                  |||       t+        |t<              r| j?                  |||       d{   S |S 7 7 7 7 7 # 1 d{  7  sw Y   xY w7 7 7 +# t"        j@                  $ rz}|j'                  di       j'                  dd      }|j'                  di       j'                  dd      }|xjB                  d| d| z  c_!        || jE                  ||       |d}~wtF        $ rY}t9        j:                  d| dtI        |       d       || jJ                  |xx   dz  cc<   || jE                  ||       |d}~ww xY ww)!z
        - Get an available deployment
        - call it with a semaphore over the call
        - semaphore specific to it's rpm
        - in the semaphore,  make a check against it's local rpm before running
        Nr   rQ  rR  async_get_available_deploymentr  r6  r  r   rT  )rT  rF  rW  r|   rY  r  max_parallel_requestsr6  rJ  client_type)r6  r  r  r]  r^  r_  r`  zlitellm.acompletion(model=r\  )r6  rN  r  rc  r  r   z$

Deployment Info: request_timeout: z

timeout: rf  rg  r   )&r<  r!   r  r  ri  r  r  r   r  rj   r  _track_deployment_metricsr  rn  _get_async_openai_model_clientr  r   r   r  r^  ro  r   	Semaphore&async_routing_strategy_pre_call_checksrt   rr  rs  r  r   r5  rr   r  Timeoutra  ru  r@  r   r  )r?  r   rF  rJ  rv  r6  _timeout_debug_deployment_dictrw  r  r  r  r  r   rT  r{  r|  	_responser  rpm_semaphorerN  r}  rD   deployment_request_timeout_paramdeployment_timeout_params                           r   r  zRouter._acompletion  s     

 	'[	28++-/;@/8@HJ#BB!$*JJ/Dd$K%	  C   J .8*yy{H :-I''BB(//&>)%%Fv%N C 	 **%8H +  ((89>>@N)--ndCL' ##7D77 %1!) ! //:f/UJJ~t,'0J>>% ? L Z(A-( $//&	
 L ^T2++;l;I4:JJ%t5K !,,%3 - M
 (Zw00. ) 
/ 
/ EE#-$/)9 F   
 &/H
/ 
/ 
/ AA) +%5 B    "+? (M2 $ G G(6 !H ! !!== >#%'  z*a/*!&&,ZL8OP **%!!1 +  ($78!AA#+%#F B    OZ
/

  /
/ 
/ 
/ 
/ +4  	/M/Q/Q "0c#T* - (F'I'I "(c)T" % IIABbAccn  pH  oI  J  JI%==aLG 		!&&,ZL8LSQRVHT[\ %
+q0+%==aLG		s   P6AM &L'F%M L"M L+)L%*	L+3L'4L+8M L)M  M !	M *M+B-M MM P6M P6M "M %L+'L+)M +L=1L42L=9M M M P3A5OP3AP..P33P6rJ  metadata_variable_namec                    |j                  d| j                        |d<   |j                  dt        t	        j
                                      d}| j                  |      r|}|j                  |i       j                  ||d       y)zm
        Adds/updates to kwargs:
        - num_retries
        - litellm_trace_id
        - metadata
        r   litellm_trace_idNr   )r  r   )r^  r   r   r   r   uuid4_get_model_from_aliasr   )r?  r   rJ  r&  r   s        r   rL  z&Router._update_kwargs_before_fallbacks  s}     !'

=$:J:J K},c$**,.?@+/%%E%2 %0"5<<!8IJ	
r   	exceptionc                     t        |d      r|j                  y|j                  di       }|j                  d      }|	 t        |      |_        yy# t        t
        f$ r Y yw xY w)z
        Set num_retries from deployment litellm_params on the exception.

        This allows the retry logic in async_function_with_retries to use
        per-deployment retry settings instead of the global setting.
        r   Nr   )r  r   r^  r  rh  	TypeError)r?  r,  r6  r   dep_num_retriess        r   ru  z/Router._set_deployment_num_retries_on_exception  ss     9m,1F1F1R#(8"=(,,];&(+O(<	% ' 	* s    A A$#A$c                     | j                   j                         }|j                  di       xs i }|j                         D ]  \  }}|	|j	                  ||        |j	                  |i       j                  |       y)z
        Adds default litellm params to kwargs, if set.

        Handles inserting this as either "metadata" or "litellm_metadata" depending on the metadata_variable_name
        r   N)r   r<  ri  r  r   r   )r?  rJ  r&  defaultsmetadata_defaultskeyrg  s          r   *_update_kwargs_with_default_litellm_paramsz1Router._update_kwargs_with_default_litellm_params  s     ..335$LLR8>B #..* 	*JC}c5)	* 	0"5<<=NOr   function_namec                    |j                  di       j                         }|d   j                         }t        ||      }t        |      }|j                  |i       j                  d      }| j	                  ||      }	|j                  d      }
|	|d<   |
|d<   t        |t        di ||	      }| j                  |
       |S )z.
        Handle clientside credential
        
model_infor   )r   rS  r5  r  )r  r   idoriginal_model_idrv  r   r7  rX  r   )r^  r<  r8   r4   _generate_model_idr[   r^   upsert_deployment)r?  r6  rJ  r5  r7  r   dynamic_litellm_paramsr&  r  	_model_idr:  deployment_pydantic_objs               r   _handle_clientside_credentialz$Router._handle_clientside_credential  s      ^^L"5::<
#$45::<!;)&"
 "D'"
 jj!7<@@O++#4J , 
	 'NN40$
4*;
&'",")C,BC!#

 	. 	 	
 '&r   c                 L   | j                  di       xs i }t        |t              r|}n|j                  d      }|j                  d      xs g }|j                  d      xs g }|s|rt	        |      t	        |      z   }||d<   d|vr|j                  d      	|d   |d<   yyy)z
        Merge tools from deployment litellm_params with request kwargs.
        When both have tools, concatenate them (deployment tools first, then request tools).
        tool_choice: use request value if provided, else deployment's.
        r   Tr   toolstool_choiceN)r^  r   r4  r7  r%  )r6  rJ  dep_params_raw
dep_params	dep_tools	req_toolsmergeds          r   _merge_tools_from_deploymentz#Router._merge_tools_from_deployment	  s     $(8"=Cnd+'J'222EJNN7+1r	JJw'-2		)_tI6F$F7O&:>>-+H+T$.}$=F=! ,U&r   c                    | j                  ||       |j                  di       j                         }|d   d   }|d   j                  d      }|d   }t        |      rZ| j	                  |||      }|j
                  j                         }|j                  j                  }|j                  j                  }t        |	      }	|j                  |	i       j                  ||||d
       |j                  di       j                  d      }
|
rH||	   j                  d      xs g }t        |      }|
D ]  }||vs|j                  |        |||	   d<   |j                  di       j                  d      }|r:d| }||	   j                  d      xs g }||vr|j                  |       |||	   d<   ||d<   | j                  ||d         |d<   | j!                  ||	       y)a  
        3 jobs:
        - Adds selected deployment, model_info and api_base to kwargs["metadata"] (used for logging)
        - Adds default litellm params to kwargs, if set.
        - Merges tools from deployment with request (proxy-configured tools + request tools).
        rW  r7  r   r   api_baserv  )rS  r6  rJ  r5  r8  )r6  r7  rL  deployment_model_nametagslitellm_credential_namezCredential: rJ  datar   )rJ  r&  N)rJ  r^  r<  r9   rA  r7  r7  r   r   rL  r4   r   r   r%  r  _get_timeoutr4  )r?  r6  rJ  r5  r7  deployment_litellm_model_namedeployment_api_baserN  r@  r&  deployment_tagsexisting_tagsmerged_tagstagcredential_namecredential_tags                   r   rn  z%Router._update_kwargs_with_deployment1	  s<    	))Z)O^^L"5::<
(23C(DW(M%()9:>>zJ *< 8#6:&*&H&H%fM 'I '# 1;;FFHJ,C,R,R,X,X)"9"H"H"Q"Q!C'"
 	0"5<<;(/)>		
 %..)92>BB6J"#9:>>vFL"M}-K& ,k)&&s+, 6AF)*62 %..)92>BB%
 +O+<=N"#9:>>vFL"M]2$$^45BF)*62)| --
+; < . 
y 	772H 	8 	
r   c                     | j                  ||d      }|j                  dd      }||||j                  k7  rd}|S |}|S )a  
        Helper to get AsyncOpenAI or AsyncAzureOpenAI client that was created for the deployment

        The same OpenAI client is re-used to optimize latency / performance in production

        If dynamic api key is provided:
            Do not re-use the client. Pass model_client=None. The OpenAI/ AzureOpenAI client will be recreated in the handler for the llm provider
        asyncr  r;  N)ro  r^  r;  )r?  r6  rJ  ry  rz  r{  s         r   r  z%Router._get_async_openai_model_clientw	  sh     "&!1!1!&g "2 "

 !**Y5'&2#9#A#AAL  2Lr   rR  c                     |j                  dd      xs> |j                  dd      xs* | j                  xs | j                  j                  dd      S )z=Helper to get stream timeout from kwargs or deployment paramsr   N)r^  r   r   )r?  rJ  rR  s      r   _get_stream_timeoutzRouter._get_stream_timeout	  s_    
 JJ'. Gxx $G ""	G
 **../?F	
r   c                     |j                  dd      xsf |j                  dd      xsR |j                  dd      xs> |j                  dd      xs* | j                  xs | j                  j                  dd      }|S )zAHelper to get non-stream timeout from kwargs or deployment paramsr   Nr  )r^  r   r   r?  rJ  rR  r   s       r   _get_non_stream_timeoutzRouter._get_non_stream_timeout	  s    
 JJy$' 	@zz+T2	@xx4	@
 xx!4	@ ||	@ **..y$? 	 r   c                 ~    d}|j                  dd      r| j                  ||      }|| j                  ||      }|S )z6Helper to get timeout from kwargs or deployment paramsNr  FrQ  )r^  r_  rb  ra  s       r   rS  zRouter._get_timeout	  sP    /3::h&..f4.HG?22D 3 G r   modelsc                    K   dt         dt        t           f fd}dt         dt        t           dt        f fd}t	        |t
              rQt        d |D              r?g }|D ]  }|j                   |d||d|        t        j                  |  d{   }|S t	        |t
              rt        d	 |D              rg }t        |      D ]'  \  }	}
|D ]  }|j                   |d||	|
d
|        ) t        j                  |  d{   }t        t        |            D cg c]  }g  }}|D ]A  }t	        |t              r||d      j                  |d          .|d   j                  |       C |S yy7 7 qc c}w w)a6  
        Async Batch Completion. Used for 2 scenarios:
        1. Batch Process 1 request to N models on litellm.Router. Pass messages as List[Dict[str, str]] to use this
        2. Batch Process N requests to M models on litellm.Router. Pass messages as List[List[Dict[str, str]]] to use this

        Example Request for 1 request to N models:
        ```
            response = await router.abatch_completion(
                models=["gpt-3.5-turbo", "groq-llama"],
                messages=[
                    {"role": "user", "content": "is litellm becoming a better product ?"}
                ],
                max_tokens=15,
            )
        ```


        Example Request for N requests to M models:
        ```
            response = await router.abatch_completion(
                models=["gpt-3.5-turbo", "groq-llama"],
                messages=[
                    [{"role": "user", "content": "is litellm becoming a better product ?"}],
                    [{"role": "user", "content": "who is this"}],
                ],
            )
        ```
        r   rF  c                 |   K   	  j                   d| |d| d{   S 7 # t        $ r}|cY d}~S d}~ww xY wwzs
            Wrapper around self.async_completion that catches exceptions and returns them as a result
            r  Nr   r  r@  r   rF  rJ  rD  r?  s       r   _async_completion_no_exceptionszARouter.abatch_completion.<locals>._async_completion_no_exceptions	  D     -T--WEHWPVWWWW 0   <$ "$ <$ 	949<9<idxc                    K   	  j                   d| |d| d{   |fS 7 # t        $ r}||fcY d}~S d}~ww xY wwrg  rh  )r   rF  rm  rJ  rD  r?  s        r   *_async_completion_no_exceptions_return_idxzLRouter.abatch_completion.<locals>._async_completion_no_exceptions_return_idx	  sS     *$**TTVTT T  #vs4   A & $& A & 	=8=A =A c              3   <   K   | ]  }t        |t                y wr   )r   r4  r   rB  s     r   r   z+Router.abatch_completion.<locals>.<genexpr>	  s     -TajD.A-Tr   r  Nc              3   <   K   | ]  }t        |t                y wr   )r   r%  rq  s     r   r   z+Router.abatch_completion.<locals>.<genexpr>
  s     /V
1d0C/Vr   )r   rm  rF  r|   r   r   )r   r   rP   r  r   r%  allr  r  r  	enumerateranger$  tuple)r?  rd  rF  rJ  rj  ro  _tasksr   rN  rm  ra  r  _final_responsess   `             r   abatch_completionzRouter.abatch_completion	  s    H				"&'7"8				+,	 	& h%#-T8-T*TF i=gET\g`fghi %^^V44HO$'C/VX/V,VF )( 3 W# EMMB "'S7FL &nnf55I<A#h-<P/Qq/QO/Q% 8h.#HQK077D#A&--h7	8
 #"# -W' 5 6/Qs8   BFE;A4FE=F&	E?/AF=F?Fc           	          K   dt         dt        t           f fd}g }|D ]  }|j                   |d||d|        t	        j
                  |  d{   }|S 7 w)a  
        Async Batch Completion - Batch Process multiple Messages to one model_group on litellm.Router

        Use this for sending multiple requests to 1 model

        Args:
            model (List[str]): model group
            messages (List[List[Dict[str, str]]]): list of messages. Each element in the list is one request
            **kwargs: additional kwargs
        Usage:
            response = await self.abatch_completion_one_model_multiple_requests(
                model="gpt-3.5-turbo",
                messages=[
                    [{"role": "user", "content": "hello"}, {"role": "user", "content": "tell me something funny"}],
                    [{"role": "user", "content": "hello good mornign"}],
                ]
            )
        r   rF  c                 |   K   	  j                   d| |d| d{   S 7 # t        $ r}|cY d}~S d}~ww xY wwrg  rh  ri  s       r   rj  z]Router.abatch_completion_one_model_multiple_requests.<locals>._async_completion_no_exceptions-
  rk  rl  r  Nr   )r   r   rP   r  r  r  )r?  r   rF  rJ  rj  rw  message_requestrN  s   `       r   -abatch_completion_one_model_multiple_requestsz4Router.abatch_completion_one_model_multiple_requests
  s{     ,				"&'7"8		 ' 	OMM/ /=C	 !00 1s   AAAAc                    K   y wr   r   r  s        r   "abatch_completion_fastest_responsez)Router.abatch_completion_fastest_responseF
  r  r  c                    K   y wr   r   r  s        r   r  z)Router.abatch_completion_fastest_responseN
  r  r  c                    K   |j                  d      D cg c]  }|j                          }}dt        dt        t        t        t        f      dt
        dt        dt        t        t        t        f   f
 fd}g dt        j                  ffd	}|D ]2  }t        j                   |d|||d
|      }	j                  |	       4 r_t        j                  t        j                          d{   \  }
|
D ]'  } ||       d{   }|d|j"                  d<   |c S  r_t        d      c c}w 7 F7 0w)z
        model - List of comma-separated model names. E.g. model="gpt-4, gpt-3.5-turbo"

        Returns fastest response from list of model names. OpenAI-compatible endpoint.
        ,r   rF  r  rJ  r   c                    K   	  j                   d| ||d| d{   }|S 7 # t        j                  $ r& t        j                  dj                  |               t        $ r}|cY d}~S d}~ww xY ww)zn
            Wrapper around self.acompletion that catches exceptions and returns them as a result
            r   rF  r  Nz4Received 'task.cancel'. Cancelling call w/ model={}.r   )r  r  CancelledErrorr   rA  r6  r@  )r   rF  r  rJ  resultrD  r?  s         r   rj  zRRouter.abatch_completion_fastest_response.<locals>._async_completion_no_exceptionsd
  s~     	/t//hehW]haghh i)) %++JQQRWX  s>   A6' %' A6' A A3'A.(A3)A6.A33A6taskc                   K   	 |  d {   }t        |t        t        f      r@t        j                  d       D ]  }|j                           |	 j                  |        S 	 	 j                  |        y 7 n# t        $ r Y S w xY w# t        $ r Y /w xY w# t        $ r Y y w xY w# 	 j                  |        w # t        $ r Y w w xY wxY ww)Nz=Received successful response. Cancelling other LLM API calls.)	r   rt   rr   r   rA  r  rM  KeyErrorr@  )r  r  tpending_taskss      r   check_responsezARouter.abatch_completion_fastest_response.<locals>.check_responsew
  s     #f}6I&JK)//W + #
#!!((. L!((. $         !((. s   CB
 A9AB
 A;$C%B( 'B 8C9B
 ;	BCBC
	BB( BB( 	B%"C$B%%C(C*B<;C<	CCCCCr  )return_whenNT!fastest_response_batch_completionzAll tasks failedr   )splitstripr   r   r
   boolr   r   rt   rr   r@  r  Taskr  r  waitFIRST_COMPLETEDr  )r?  r   rF  r  rJ  rB  rd  rj  r  r  donecompleted_taskr  r  s   `            @r   r  z)Router.abatch_completion_fastest_responseV
  s]     &+[[%56!'')66		"&tCH~"6	@D	PS	="5y@A	" 	w|| 	,  	'E&&/ (6EKD
   &	' (/7+B+B) #D- #' "-n==%QUF))*MN!M"	  *++C 7j# >s:   D?D6CD?4D;5D?D=D?D?+D?=D?r  c                    K   y wr   r   r?  r   rF  r  r  rJ  s         r   r  zRouter.schedule_acompletion
  r  r  c                    K   y wr   r   r  s         r   r  zRouter.schedule_acompletion
  r  r  c                 B  K   t        |      }t        t        j                               }t	        |||      }| j
                  j                  |       d {    t        j                         | j                  z   }	t        j                         }
| j
                  j                  }d}|
|	k  r| j                  ||       d {   \  }}| j
                  j                  |j                  |j                  |       d {   }|rn7t        j                   |       d {    t        j                         }
|
|	k  r|r]	  | j"                  d|||d| d {   }|j$                  j'                  di        |j$                  d   j)                  dd	i       |S | j
                  j/                  |j                  |j                         d {    t1        j2                  d|d      7 7 /7 7 7 # t*        $ r}t-        |d
|       |d }~ww xY w7 Hw)Nr  
request_idrv  requestFr   r  r9  rv  health_deploymentsr  additional_headers%x-litellm-request-prioritization-usedTr  r  rv  %Request timed out while polling queuer  r`  r   )r!   r   r   r*  rN   r  add_requestr  	monotonicr   r   _async_get_healthy_deploymentspollr  rv  r  sleepr  r  r   r   r@  r  remove_requestr   r   )r?  r   rF  r  r  rJ  r  _request_idr  r  	curr_timepoll_intervalmake_request_healthy_deploymentsrx  r"  rD  s                    r   r  zRouter.schedule_acompletion
  s$     =VD$**,'"
 nn(((666 >>#dll2NN$	77(",0,O,O.> -P - '# ! "&!4!4????#7 "5 " L
 mmM222 NN,	 (" "2$"2"2 #(6#EK# 	 ((334H"M(()=>EE<dC !  ..//??t 0    //?% S 	7' 3
  :x0
s   AHG1A,HG4<HG7H"G9#HHG= G;AG= 5HHH4H7H9H;G= =	HHHHrH  rK  .c                 \  K   t        |      }t        t        j                               }t	        |||      }| j
                  j                  |       d {    t        j                         | j                  z   }	t        j                         }
| j
                  j                  }d}|
|	k  r| j                  ||       d {   \  }}| j
                  j                  |j                  |j                  |       d {   }|rn7t        j                   |       d {    t        j                         }
|
|	k  r|ri	  ||i | d {   }t#        |j$                  t&              r<|j$                  j)                  di        |j$                  d   j+                  ddi       |S | j
                  j1                  |j                  |j                  
       d {    t3        j4                  d|d      7 7 ;7 7 7 # t,        $ r}t/        |d	|       |d }~ww xY w7 Iw)Nr  r  Fr  r  r  r  Tr  r  r  r  r`  )r!   r   r   r*  rN   r  r  r  r  r   r   r  r  r  rv  r  r  r   r  r4  r   r   r@  r  r  r   r   )r?  r   r  rH  rK  rJ  r  r  r  r  r  r  r  r  rx  r"  rD  s                    r   _schedule_factoryzRouter._schedule_factory
  s     =VD$**,'"
 nn(((666 >>#dll2NN$	77(",0,O,O.> -P - '# ! "&!4!4????#7 "5 " L
 mmM222 NN,	 (" 
"3T"DV"DD	i66=,,778LbQ,,-ABII@$G !  ..//??t 0    //?% Q 	7' 3
 E  :x0
s   AH,G=A,H,H <H,HH,"H#H,H,H
 HAH
 +5H, H*!H, H,H,H,H
 
	H'H""H''H,c                     | j                  |      }|t        |      dk7  ry|d   d   j                  dd       }|d|vry|j                  d      d   }|t        j
                  v S )Nrv  r|   Fr   r   r   /)get_model_listr$  r^  r  r   )_known_custom_logger_compatible_callbacks)r?  r   r   litellm_modelsplit_litellm_models        r   r  z"Router._is_prompt_management_model:  s|    ((E(:
ZA!5"1&67;;GTJ C}$<+11#6q9"g&W&WWWr   c                   K   |j                  dd       }|$t        di dt               t               d|\  }}t	        t
        |      }| j                  |dddg|j                  dd             }| j                  ||	       |d
   j                         }|j                  dd       }t        |t              xr |j                  d      }|j                  d      xs |d
   j                  dd       }	|j                  d      xs |d
   j                  dd       }
|j                  dd       xs |d
   j                  dd       }|s,|	t        |	t              st        d|	 dt        |	             |
*t        |
t              st        d|
 dt        |
             |j!                  ||t#        |      |	|
|      \  }}}h d}|j%                         D ci c]  \  }}||vs|| }}}i |||}||d<   ||d<   ||d<   |	|d<   |
|d<   ||d<   | j'                  |      }|t)        |      dk(  r.|j                  d       t+        j,                  di | d {   S  | j.                  di | d {   S c c}}w 7 $7 w)Nr  r  )rH  	rules_objr  userpromptr  rQ  r   rF  rQ  rW  r   r   zlitellm_agent/	prompt_idprompt_variablesprompt_labelz*Prompt ID is not set or not a string. Got=z, type=z2Prompt variables is set but not a dictionary. Got=)rJ  )r   rF  non_default_paramsr  r  r  >   r  r  prompt_versionbitbucket_configdotprompt_configr  rF  r  r   rH  r   )r^  rv   ru   rz   r   LiteLLMLoggingrh  ri  rn  r<  r   r   
startswithrh  r   r4  get_chat_completion_promptrx   r  r  r$  r   r  r  )r?  r   rF  rJ  litellm_logging_objectprompt_management_deploymentrR  r  is_litellm_agent_modelr  r  r  optional_paramsprompt_management_paramskvfiltered_data_model_lists                     r   r  z!Router._prompt_management_factoryF  s;     "(,A4!H!)-; .)6!&"2"4 	.*"F "&n6L!M'+'D'D%(;< &

+@$ G (E (
$ 	++3F 	, 	
 ,,<=BBD$/ ",3"
 "9&&'78 	 JJ{+ !/K0

#k4
  	 "::
 
)*:;??
 	
 zz.$7 $;W<

#nd
# 	 &Z	3%?<YKwtT]N_`  '
;KT0RDEUDVV]^bcs^t]uv  #==@O-% > 
		
$
  "ZZ\
QQ6N-NAqD
 
 @M?V??w%z(>$%'{%5!"!-~))U);#k"2a"7JJ*+ ,,6v6667T77A&AAA#
 7As7   GJJ*J/A4J#J$J=J
>J
Jr  c                    	 ||d<   ||d<   | j                   |d<   |j                  d| j                        |d<   |j                  di       j	                  d|i        | j
                  di |}|S # t        $ r}|d }~ww xY w)Nr   r  rH  r   r   r  r   )_image_generationr^  r   r   r   rM  r@  r?  r  r   rJ  rN  rD  s         r   image_generationzRouter.image_generation  s    
	#F7O%F8*.*@*@F&'$*JJ}d>N>N$OF=!j"-44mU5KL3t33=f=HO 	G	s   A.A1 1	B:A<<Bc           	         d}	 t        j                  d| d|        | j                  |dddg|j                  dd             }| j	                  ||	       |d
   j                         }| j                  ||	      }| j                  |xx   dz  cc<   | j                  |       t        j                  di i ||| j                  |d|}| j                  |xx   dz  cc<   t        j                  d| d       |S # t        $ rE}	t        j                  d| dt        |	       d       || j                   |xx   dz  cc<   |	d }	~	ww xY w)Nr_  #Inside _image_generation()- model: 
; kwargs: r  r  r  rQ  r  rW  r   r|   rX  r  rZ  r[  zlitellm.image_generation(model=r\  rf  rg  r   )r   rA  rh  ri  rn  r<  r  r  rq  r   r  r   r  r5  r@  r   r  )
r?  r  r   rJ  rv  r6  rR  r{  rN  rD  s
             r   r  zRouter._image_generation  s   
*	!''5eWJvhO 66#)h?@$*JJ/Dd$K 7 J
 //:f/U./446D>>% ? L
 Z(A-( 11Z1H// $#33*	
 H z*a/*!&&1*=TU O 	!&&1*=QRUVWRXQYY`a %
+q0+G	s   C?D 	EA EEc           	      l  K   	 ||d<   ||d<   | j                   |d<   |j                  d| j                        |d<   | j                  ||        | j                  di | d {   }|S 7 # t
        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY ww)Nr   r  rH  r   rI  r  r   )_aimage_generationr^  r   rL  r  r@  r  r  rG   r  r  r  s         r   aimage_generationzRouter.aimage_generation  s     	#F7O%F8*.*A*AF&'$*JJ}d>N>N$OF=!00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   B4A A- $A+%A- *B4+A- -	B166B,,B11B4c           	      6  K   |}	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd       |       d {   }| j                  ||       |d	   j                         }|d
   }| j                  ||      }| j                  |xx   dz  cc<   t        j                  di i ||| j                  |d|}	| j                  ||d      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   dz  cc<   t        j$                  d| d       |	S 7 L7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   dz  cc<   |d }~ww xY ww)Nr  r  r  r  r  rQ  rR  rW  r   r   r|   r  r  r  r  z litellm.aimage_generation(model=r\  rf  rg  r   )r   rA  r!   r  ri  rn  r<  r  r  r   r  r   ro  r   r  r  r  r  r5  r@  r   r  )r?  r  r   rJ  rv  r  r6  rR  r{  rN  r#  rD  s               r   r  zRouter._aimage_generation  s    
D	!''5eWJvhO  AH#BB#)h?@$*JJ/Dd$K%	  C   J //:f/U./446DgJ>>% ? L
 Z(A-(00 $#33*	
 H !,,%3 - M (Zw00. ) . . EE#-@P F    &.~H. . . AA)<L B    "*>z*a/*!&&2:,>UV OqJ.
  .. . . . *  	!&&2:,>RSVWXSYRZZab %
+q0+G	s   HAG F$B8G F'G F/-F).	F/7F+8F/<G F-G #G$	G -G.5G #H$G 'G )F/+F/-G /G5F86G=G G 	HA HHHfilec           	      .  K   	 ||d<   ||d<   | j                   |d<   | j                  ||        | j                  di | d{   }|S 7 # t        $ r;}t	        j
                  t        | |t        j                         |             |d}~ww xY ww)a  
        Example Usage:

        ```
        from litellm import Router
        client = Router(model_list = [
            {
                "model_name": "whisper",
                "litellm_params": {
                    "model": "whisper-1",
                },
            },
        ])

        audio_file = open("speech.mp3", "rb")
        transcript = await client.atranscription(
        model="whisper",
        file=audio_file
        )

        ```
        r   r  rH  rI  Nr  r   )	_atranscriptionrL  r  r@  r  r  rG   r  r  )r?  r  r   rJ  rN  rD  s         r   atranscriptionzRouter.atranscriptionA  s     .	#F7O!F6N*.*>*>F&'00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		;   BAA AA BA 	B6BBBc           	      ,  K   |}	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd       |       d {   }| j                  ||       |d	   j                         }| j                  ||      }| j                  |xx   d
z  cc<   t        j                  di i ||| j                  |d|}	| j                  ||d      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   d
z  cc<   t        j$                  d| d       |	S 7 G7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   d
z  cc<   |d }~ww xY ww)Nz!Inside _atranscription()- model: r  r  r  r  rQ  rR  rW  r   r|   )r  rZ  r[  r  r  r  zlitellm.atranscription(model=r\  rf  rg  r   )r   rA  r!   r  ri  rn  r<  r  r  r   r  r   ro  r   r  r  r  r  r5  r@  r   r  )r?  r  r   rJ  rv  r  r6  rR  r{  rN  r#  rD  s               r   r  zRouter._atranscriptionk  s    
B	!''3E7*VHM  AH#BB#)h?@$*JJ/Dd$K%	  C   J //:f/U./446D>>% ? L
 Z(A-(--  #33*	
 H !,,%3 - M (Zw00. ) . . EE#-@P F    &.~H. . . AA)<L B    "*>z*a/*!&&/
|;RS OmF.
  .. . . . *  	!&&/
|;OPSTUPVxW^_ %
+q0+G	s   HAG FB3G F"G F*(F$)	F*2F&3F*7G F(G F?	G (G)5G HG "G $F*&F*(G *F<0F31F<8G G 	HA HHHinputvoicec           	        K   	 ||d<   ||d<   | j                  |dddg|j                  dd      |       d{   }| j                  ||	       |d
   j                         }|d    | j                  j                         D ])  \  }}||vr|||<   |dk(  s||   j                  |       + | j                  ||d      }	|j                  dd      }
|
|	|
|	j                  k7  rd}n|	}t        j                  di i |d|i| d{   }|S 7 7 # t        $ r;}t        j                  t        | |t!        j"                         |             |d}~ww xY ww)a  
        Example Usage:

        ```
        from litellm import Router
        client = Router(model_list = [
            {
                "model_name": "tts",
                "litellm_params": {
                    "model": "tts-1",
                },
            },
        ])

        async with client.aspeech(
            model="tts",
            voice="alloy",
            input="the quick brown fox jumped over the lazy dogs",
            api_base=None,
            api_key=None,
            organization=None,
            project=None,
            max_retries=1,
            timeout=600,
            client=None,
            optional_params={},
        ) as response:
            response.stream_to_file(speech_file_path)

        ```
        r  r  r  r  r  rQ  NrR  rI  r   r   r   r]  r  r;  r[  r  r   )r  ri  rL  r<  r   r  r   ro  r^  r;  r   aspeechr@  r  r  rG   r  r  )r?  r   r  r  rJ  r6  rR  r  r  ry  rz  r{  rN  rD  s                 r   r  zRouter.aspeech  s    @4	#F7O#F7O#BB#)h?@$*JJ/Dd$K%	  C   J 00uV0L./446DM3399; (1VO !F1I*_1I$$Q'( &*%5%5%f' &6 &" %jjD9O+*6#'='E'EE#5$__ l  H OM>  		(,0#)(1(<(<(>'(	 G		sR   E 6D DAD A4D DD E D D 	E"6EEE c           	      6  K   	 ||d<   t         |d<   | j                  |d<   | j                  ||        | j                  di | d {   }|S 7 # t        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY wwNr   r  rH  rI  r  r   )
r  _arerankrL  r  r@  r  r  rG   r  r  r?  r   rJ  rN  rD  s        r   arerankzRouter.arerank  s     	#F7O#F7O*.--F&'00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   BAA 	A
A BA 	B6BBBc           	        K   d }	 t        j                  d| d|        | j                  ||j                  dd       |       d {   }| j	                  ||       |d   j                         }|d   }| j                  ||      }| j                  |xx   dz  cc<   t        j                  di i || j                  |d	| d {   }| j                  |xx   dz  cc<   t        j                  d
| d       |S 7 7 8# t        $ rE}t        j                  d
| dt        |       d       || j                  |xx   dz  cc<   |d }~ww xY ww)NzInside _rerank()- model: r  rQ  )r   rQ  rS  rW  r   r   r|   )rZ  r[  zlitellm.arerank(model=r\  rf  rg  r   )r   rA  r  ri  rn  r<  r  r  r   r  r   r  r5  r@  r   r  )	r?  r   rJ  rv  r6  rR  r{  rN  rD  s	            r   r  zRouter._arerank  s    
'	!''+E7*VHE  $BB$*JJ/Dd$K%  C   J
 //:f/U./446DgJ>>% ? L Z(A-($__ #33* 	 H z*a/*!&&(4KL O9  	!&&(4HQPWX %
+q0+G	sO   EAD D	BD D5D ED D 	EA EEEis_retryis_fallbackis_asyncc                    d|dg}	 ||d<   ||d<   |j                  d| j                        |d<   |j                  di       j                  d|i       | j	                  |||j                  dd       	      }|d
   j                         }	| j                  j                         D ])  \  }
}|
|vr|||
<   |
dk(  s||
   j                  |       + t        j                  di i |	|| j                  d|S # t        $ r}|d }~ww xY w)Nr  r  r   r  r   r   r  rQ  r  r   )r  rZ  r   )r^  r   r   r   rh  ri  r<  r   r  r   text_completionr   r@  )r?  r   r  r  r  r  rJ  rF  r6  rR  r  r  rD  s                r   r  zRouter.text_completionG  s6    $78	#F7O%F8$*JJ}d>N>N$OF=!j"-44mU5KL 66!$*JJ/Dd$K 7 J ./446D3399; (1VO !F1I*_1I$$Q'( **s-r-rSWSgSg-rkq-rss 	G	s   B2C9 ;=C9 9	D	DD	c           	        K   |j                  dd       9| j                  ||j                  d      | j                  ||f|       d {   S 	 ||d<   ||d<   | j                  |d<   | j                  ||        | j                  di | d {   }|S 7 M7 # t        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY ww)	Nr  )r   r  rH  rK  rJ  r   r  rH  rI  r  r   )r^  r  ri  atext_completion_atext_completionrL  r  r@  r  r  rG   r  r  )	r?  r   r  r  r  r  rJ  rN  rD  s	            r   r  zRouter.atext_completionl  s      ::j$'3//J/"&"7"7V_ 0   	#F7O%F8*.*@*@F&'00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		sI   AC"	B
C"AB BB C"B 	C$6CCC"c           	      2  K   	 t        j                  d| d|        t        |      }| j                  |d|dg|j	                  dd       |       d {   }| j                  ||       |d   j                         }|d	   }| j                  ||      }| j                  |xx   d
z  cc<   t        j                  di i ||| j                  |d|}	| j                  ||d      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   d
z  cc<   t        j$                  d| d       |	S 7 L7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   d
z  cc<   |d }~ww xY ww)N#Inside _atext_completion()- model: r  r  r  rQ  rR  rW  r   r   r|   r  r  r  r  zlitellm.atext_completion(model=r\  rf  rg  r   )r   rA  r!   r  ri  rn  r<  r  r  r   r  r   ro  r   r  r  r  r  r5  r@  r   r  )r?  r   r  rJ  r  r6  rR  rv  r{  rN  r#  rD  s               r   r  zRouter._atext_completion  s    C	!''5eWJvhO  AH#BB#)f=>$*JJ/Dd$K%	  C   J //:f/U./446DgJ>>% ? L Z(A-(// $#33*	
 H !,,%3 - M (Zw00. ) . . EE#-@P F    &.~H. . . AA)<L B    "*>z*a/*!&&1*=TU OoH.
  .. . . . *  	!&&1%8LSQRVHT[\  &!+&G	   HAG F"B8G F%G F-+F',	F-5F)6F-:G F+G !G"	G +G,5G !H"G %G 'F-)F-+G -F?3F64F?;G G 	HA HHH
adapter_idc           	        K   	 ||d<   ||d<   | j                   |d<   |j                  d| j                        |d<   |j                  di       j	                  d|i        | j
                  di | d {   }|S 7 # t        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY ww)	Nr   r  rH  r   r   r  r  r   )_aadapter_completionr^  r   r   r   r  r@  r  r  rG   r  r  )	r?  r  r   r  r  r  rJ  rN  rD  s	            r   aadapter_completionzRouter.aadapter_completion  s     	#F7O#-F< *.*C*CF&'$*JJ}d>N>N$OF=!j"-44mU5KL?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   CA0A= 4A;5A= :C;A= =	C6B<<CCc           	      2  K   	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd       |       d {   }| j                  ||       |d	   j                         }|d
   }| j                  ||      }| j                  |xx   dz  cc<   t        j                  di i ||| j                  |d|}	| j                  ||d      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   dz  cc<   t        j$                  d| d       |	S 7 L7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   dz  cc<   |d }~ww xY ww)Nz&Inside _aadapter_completion()- model: r  r  zdefault textr  rQ  rR  rW  r   r   r|   )r  rZ  r[  r  r  r  z"litellm.aadapter_completion(model=r\  rf  rg  r   )r   rA  r!   r  ri  rn  r<  r  r  r   r  r   ro  r   r  r  r  r  r5  r@  r   r  )r?  r  r   rJ  r  r6  rR  rv  r{  rN  r#  rD  s               r   r  zRouter._aadapter_completion  s    C	!''8z&R  AH#BB#)nEF$*JJ/Dd$K%	  C   J //:f/U./446DgJ>>% ? L Z(A-(22 ",#33*	
 H !,,%3 - M (Zw00. ) . . EE#-@P F    &.~H. . . AA)<L B    "*>z*a/*!&&4ZL@WX OoH.
  .. . . . *  	!&&4UG;OPSTUPVxW^_  &!+&G	r  c                 T   K   ddl m}  |j                  d| |d| d{   S 7 w)z
        Helper function to make a search API call through the router with load balancing and fallbacks.
        Reuses the router's retry/fallback infrastructure.
        r   SearchAPIRouter)router_instancerH  Nr   )&litellm.router_utils.search_api_routerr  async_search_with_fallbacks)r?  rH  rJ  r  s       r   _asearch_with_fallbackszRouter._asearch_with_fallbacks:  s@     
 	K@_@@ 
 /
 
 
 	
 
s   (&(original_generic_functionc                 V   K   ddl m}  |j                  d| ||d| d{   S 7 w)z
        Helper function for search API calls - selects a search tool and calls the original function.
        Called by async_function_with_fallbacks for each retry attempt.
        r   r   )r  r   r  Nr   )r  r  "async_search_with_fallbacks_helper)r?  r   r  rJ  r  s        r   _asearch_with_fallbacks_helperz%Router._asearch_with_fallbacks_helperG  sC      	KG_GG 
 &?
 	
 
 	
 
    )')guardrail_namec                    K   ||d<   ||d<   | j                   |d<   | j                  ||d       t        j                  d| d|         | j                  d	i | d{   }|S 7 w)
aw  
        Execute a guardrail with load balancing and fallbacks.

        Args:
            guardrail_name: Name of the guardrail to execute
            original_function: The guardrail's execution function (e.g., async_pre_call_hook)
            **kwargs: Additional arguments passed to the guardrail

        Returns:
            Result from the guardrail execution
        r   r  rH  litellm_metadatar   rJ  r&  z&Inside aguardrail() - guardrail_name: r  Nr   )_aguardrail_helperrL  r   rA  r  )r?  r  rH  rJ  rN  s        r   
aguardrailzRouter.aguardrailW  s     " )w.?*+&*&=&="#,, #5 	- 	

 	##4^4DJvhW	
 <;;EfEE Fs   AA) A'!A)c                    K   |}| j                  |      }t        j                  d|j                  di       j                  d              ||d<    |di | d{   }|S 7 w)a}  
        Helper for aguardrail - selects a guardrail deployment and executes it.
        Called by async_function_with_fallbacks for each retry attempt.

        Args:
            model: The guardrail_name (named 'model' for fallback system compatibility)
            original_generic_function: The guardrail's execution function
            **kwargs: Additional arguments
        )r  zSelected guardrail deployment: r   	guardrailselected_guardrailNr   )get_available_guardrailr   rA  r^  )r?  r   r  rJ  r  r  rN  s          r   r  zRouter._aguardrail_helperv  s      !99) : 
 	##-.@.D.DEUWY.Z.^.^_j.k-lm	

 (:#$2<V<< =s   AA'A%A'r]   c                     ddl m} | j                  D cg c]  }|j                  d      |k(  s| }}|st	        d|       t        |      dk(  r|d   S t        t         || ||            S c c}w )z
        Select a guardrail deployment using the router's load balancing strategy.

        Args:
            guardrail_name: Name of the guardrail to select

        Returns:
            Selected guardrail configuration dict
        r   r/   r  zNo guardrail found with name: r|   llm_router_instancer  r   )&litellm.router_strategy.simple_shuffler0   r   r^  rh  r$  r   r]   )r?  r  r0   gr  s        r   r  zRouter.get_available_guardrail  s     	J **
aee4D.E.WA
 
 #=n=MNOO"#q(&q)) $($7$
 	

s
   A1A1c           	      f  K   	 ||d<   ||d<   | j                   |d<   | j                  ||d       t        j                  d| d|         | j                  d
i | d{   }|S 7 # t
        $ r;}t        j                  t        | |t        j                         |	             |d}~ww xY ww)
        Helper function to make a generic LLM API call through the router, this allows you to use retries/fallbacks with litellm router
        r   r  rH  r  r  z3Inside ageneric_api_call_with_fallbacks() - model: r  Nr  r   )(_ageneric_api_call_with_fallbacks_helperrL  r   rA  r  r@  r  r  rG   r  r  )r?  r   rH  rJ  rN  rD  s         r   !_ageneric_api_call_with_fallbacksz(Router._ageneric_api_call_with_fallbacks  s     	#F7O2CF./*.*W*WF&'00FCU 1  "''EeWJW]V^_ @T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   B1AA* !A("A* 'B1(A* *	B.36B))B..B1rv  c                     d|v rT|d   rOddl m} 	  |||j                  d      |j                  d            \  }}}}|}|d   j	                  ||      |d<   |S # t        $ r |}Y 'w xY w)a  
        Add the deployment model to the endpoint for LLM passthrough route.

        e.g for bedrock invoke users can pass endpoint as /model/special-bedrock-model/invoke
          it should be actually sent as /model/us.anthropic.claude-3-5-sonnet-20240620-v1:0/invoke
        endpointr   )rw   r  rL  )r   r  rL  )r   rw   r^  r@  replace)r?  rJ  r   rv  rw   stripped_model_namerx  replacement_model_names           r   ;_add_deployment_model_to_endpoint_for_llm_passthrough_routezBRouter._add_deployment_model_to_endpoint_for_llm_passthrough_route  s     F:$6 1
4/?$(.

3H(I#ZZ
30,#Q1
 *=&
 "(
!3!;!;-"F:   4)3&4s   0A A)(A)c           	      ~  K   |j                  dd      }d}	 t        |      }	 | j                  |||j                  dd      |j                  dd             d{   }| j                  |||	       |d
   j                         }	|	d   }
| j                  |
xx   dz  cc<   | j                  |||
        |di i |	d| j                  i|}| j                  ||d      }|\t        |t        j                        rB|4 d{    	 | j                  ||       d{    | d{   }ddd      d{    n%| j                  ||       d{    | d{   }| j                  |
xx   dz  cc<   t!        j"                  d|
 d       |S 7 ?# t        $ r!}|r |dd|i| d{  7  cY d}~S |d}~ww xY w7 7 7 7 # 1 d{  7  sw Y   |xY w7 7 # t        $ rE}t!        j"                  d| dt%        |       d       || j&                  |xx   dz  cc<   |d}~ww xY ww)r  passthrough_on_no_deploymentFr  rF  NrQ  )r   rS  rF  rQ  r   rM  r   r|   )rJ  r   rv  rZ  r  r  r  z'ageneric_api_call_with_fallbacks(model=r\  rf  rg  r   )ri  r!   r  r^  r@  rn  r<  r  r#  r   ro  r   r  r  r  r  r   r5  r   r  )r?  r   r  rJ  r%  r5  r  r6  rD  rR  rv  rN  r#  s                r   r  z/Router._ageneric_api_call_with_fallbacks_helper  s     (.zz2PRW'X$;F	@H
#'#F#F#)#ZZ
D9(.

3H$(O	 $G $ 
 //%fM 0  ./446DgJZ(A-(LLUz M  1 t33 H !,,%3 - M (Zw00. ) . . EE#-@P F    &.~H. . . AA)<L B    "*>z*a/*!&&9*E\] Oy  /!:!Q!Q&!QQQQD.
  .. . . . *  	!&&9%@TUXYZU[T\\cd  &!+&G	s  H=G, 8F FF !B'G, G	G, G$G%	G.G/G3G, >G?G, G(	G, $G*%5G, H=F 	G'G6F97G;G<G,  H=GGG, GGG, G%GG%!G, *G, ,	H:5A H55H::H=c           
         |j                   }t        d      }	 t        j                  d| d| d|        | j	                  |||       | j                  ||j                  dd      |j                  d	d      |
      }| j                  ||d       |d   j                         }|d   }| j                  |xx   dz  cc<   d|v r|d   r|d   j                  ||      |d<   ||d<   | j                  |       	 t        |d         \  }	}
}	}	 |di i ||
| j                  d|}| j                   |xx   dz  cc<   t        j"                  | d| d       |S # t        $ r d}
Y \w xY w# t        $ rG}t        j"                  | d| dt%        |       d       || j&                  |xx   dz  cc<   |d}~ww xY w)a  
        Make a generic LLM API call through the router, this allows you to use retries/fallbacks with litellm router
        Args:
            model: The model to use
            original_function: The handler function to call (e.g., litellm.completion)
            **kwargs: Additional arguments to pass to the handler function
        Returns:
            The response from the handler function
        generic_api_callr8  z&Inside _generic_api_call() - handler: z	, model: r  r  rF  NrQ  rR  rM  r   r   r|   r  rX  r)  )r  rZ  z(model=r\  rf  rg  r   )r   r4   r   rA  rL  rh  r^  ri  rn  r<  r  r   rq  rw   r@  r   r  r5  r   r  )r?  r   rH  rJ  handler_namer&  r6  rR  rv  rx  r  rN  rD  s                r    _generic_api_call_with_fallbacksz'Router._generic_api_call_with_fallbacksE  sC    )11!C,"
:	!''8iPUwV`ag`hi 00'= 1 
 66J5$*JJ/Dd$K%	 7 J //%fDV 0  ./446DgJZ(A-( V#z(:%+J%7%?%?z%Rz"(F7O 11Z1H+/?d7m/T,&1 ) +>#33 	H z*a/*!&&.
|3JK O!  +&*#+"  	!&&.w.B3q6('R  &!+&G	s>   C"E1 =E  AE1  E.+E1 -E..E1 1	G:AF<<Gc                     	 ||d<   ||d<   | j                   |d<   | j                  ||        | j                  di |}|S # t        $ r}|d }~ww xY w)Nr   r  rH  rI  r   )
_embeddingrL  rM  r@  r?  r   r  r  rJ  rN  rD  s          r   	embeddingzRouter.embedding  sj    	#F7O#F7O*.//F&'00uV0L3t33=f=HO 	G	s   ?A 	AAAc           	         d }	 t        j                  d| d|        | j                  |||j                  dd             }| j	                  ||       |d   j                         }|d   }| j                  ||d	      }|j                  d
d       }||||j                  k7  rd }	n|}	| j                  |xx   dz  cc<   | j                  |       t        j                  di i ||| j                  |	d|}
| j                  |xx   dz  cc<   t        j                  d| d       |
S # t         $ rE}t        j                  d| dt#        |       d       || j$                  |xx   dz  cc<   |d }~ww xY w)NzInside embedding()- model: r  rQ  )r   r  rQ  rW  r   r   syncr  r;  r|   rX  r  rZ  r[  zlitellm.embedding(model=r\  rf  rg  r   )r   rA  rh  ri  rn  r<  ro  r^  r;  r  rq  r   r-  r   r  r5  r@  r   r  )r?  r  r   rJ  rv  r6  rR  ry  rz  r{  rN  rD  s               r   r+  zRouter._embedding  s   
4	!''-eWJvhG 66$*JJ/Dd$K 7 J
 //:f/U./446DgJ%)%5%5%f& &6 &" %jjD9O+*6#'='E'EE#5Z(A-( 11Z1H(( "#33*	
 H z*a/*!&&*:,6MN O 	!&&*:,6J3q6(RYZ %
+q0+G	s   D+D0 0	E>9A E99E>c           	      .  K   	 ||d<   ||d<   | j                   |d<   | j                  ||        | j                  di | d {   }|S 7 # t        $ r;}t	        j
                  t        | |t        j                         |             |d }~ww xY wwr  )	_aembeddingrL  r  r@  r  r  rG   r  r  r,  s          r   
aembeddingzRouter.aembedding  s     	#F7O#F7O*.*:*:F&'00uV0L?T??I&IIHO J 		(,0#)(1(<(<(>'(	 G		r  c           	      .  K   d }	 t        j                  d| d|        t        |      }| j                  |||j	                  dd       |       d {   }| j                  ||       |d   j                         }|d   }| j                  ||      }| j                  |xx   dz  cc<   t        j                  di i ||| j                  |d	|}	| j                  ||d
      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   dz  cc<   t        j$                  d| d       |	S 7 L7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   dz  cc<   |d }~ww xY ww)NzInside _aembedding()- model: r  rQ  )r   r  rQ  rS  rW  r   r   r|   r0  r  r  r  zlitellm.aembedding(model=r\  rf  rg  r   )r   rA  r!   r  ri  rn  r<  r  r  r   r3  r   ro  r   r  r  r  r  r5  r@  r   r  )r?  r  r   rJ  rv  r  r6  rR  r{  rN  r#  rD  s               r   r2  zRouter._aembedding  s    
B	!''/wjI  AH#BB$*JJ/Dd$K%	  C   J //:f/U./446DgJ>>% ? L
 Z(A-()) "#33*	
 H !,,%3 - M (Zw00. ) . . EE#-@P F    &.~H. . . AA)<L B    "*>z*a/*!&&+J<7NO OmF.
  .. . . . *  	!&&+J<7KCPQF8SZ[ %
+q0+G	s   HAG F B8G F#G F+)F%*	F+3F'4F+8G F)G G  	G )G*5G H G #G %F+'F+)G +F=1F42F=9G G 	HA HHHc           	      b  K   	 ||d<   | j                   |d<   |j                  d| j                        |d<   | j                  ||        | j                  di | d {   }|S 7 # t
        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY ww)Nr   rH  r   rI  r  r   )_acreate_filer^  r   rL  r  r@  r  r  rG   r  r  r  s        r   acreate_filezRouter.acreate_file:  s     
	#F7O*.*<*<F&'$*JJ}d>N>N$OF=!00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   B/AA( A& A( %B/&A( (	B,16B''B,,B/c                    K   	 ddl m} t        j                  d| d        t	               j                  |dddgj                  dd       	       d {   }d
t        dt        f fd}g }t        |t              r|j                   ||             n|D ]  }|j                   ||              t        j                  |  d {   }t        |      dk(  rt        d       |||      }	t        t        |d         }
|	|
j                   d<   |
S 7 7 M# t        $ rH}t        j"                  d| d dt%        |       d       | j&                  |xx   dz  cc<   |d }~ww xY ww)Nr   )add_model_file_id_mappingsr  r  r  files-api-fake-textr  rQ  r   rF  rQ  rS  r  r6  r   c           	      P  K   ddl m}  |      }j                  | |d       | d   j                         }|d   }j	                  | |      }j
                  |xx   dz  cc<   t        |d   	      \  }}}}t        t        t           j                  d
            }	t        t        t           j                  d            }
|
r|	st        d      t        |	      }|rt        |
|      }
|
|d<   d|v r|d   |j                  di       d<   t!        j"                  di i ||j$                  |d|}j'                  | |d      }|\t)        |t*        j,                        rB|4 d {    	 j/                  |        d {    | d {   }d d d       d {    n%j/                  |        d {    | d {   }j0                  |xx   dz  cc<   t3        j4                  d| d       |S 7 7 x7 p7 b# 1 d {  7  sw Y   LxY w7 ]7 Uw)Nr   r  r7  rM  r   r   rW  r|   r)  purposer  z2file and file_purpose are required for create_file)r=  )file_contentnew_model_namegcs_bucket_namer  r  rZ  r[  r  r  r  litellm.acreate_file(model=r\  r   )r  r  rn  r<  r  r  rw   r   r   rS   r^  rQ   r@  r6   r5   r   r   r7  r   ro  r   r  r  r  r  r   r5  )r6  r  kwargs_copyrR  rv  r{  stripped_modelr  rx  r=  r  replace_model_in_jsonl_boolrN  r#  rJ  r  r?  s                 r   create_file_for_deploymentz8Router._acreate_file.<locals>.create_file_for_deploymentf  s    R,V433)&"0 4 
 ""2388:!']
#BB)&  C     ,1, =Mw-=9 3Q
 x(:;VZZ	=RSHY/F1CD7#L  /L#/+ /1%)'5D
 +/K'%- ./  **+=rB) #// /B#'#7#7".	
 & !% 0 0)& 7 !1 ! !,!7#4#42  - 2 2 #II'1DT J    *2>2 2 2 EE#-@P F    &.~H"":.!3.%**1*=TU  )2
 $22 2 2 2  .s   E/H&2H3H&6HH	HH	HH&(H)H&H"	H&H$7H&H	HH&HHHH&$H&No healthy deployments found.)r  r  model_file_id_mappingrB  , rf  rg  r|   )!litellm.router_utils.common_utilsr9  r   rA  r!   async_get_healthy_deploymentsri  r4  rR   r   r  r  r  r$  r@  r   r  r,  r   r  )r?  r   rJ  r9  r  rF  tasksr6  r  rH  returned_responserD  r  s   ` `         @r   r6  zRouter._acreate_fileR  s    
B	T!''5eWJvhO  AH(,(J(J#)6KLM$*JJ/Dd$K%!1 )K ) #V T V FV V p E-t478KLM"5 IJLL!;J!GHI &nne44I9~" ?@@$>$79%! !%%5y| D & ,,' %$i#P 5  	!++-eWBvh>RSVWXSYRZZab  &!+&G	sP   E:AD&  D"!A5D& D$A
D& !E:"D& $D& &	E7/AE22E77E:c           	      r  K   	 |)ddl m} | j                  |d      } |di | d{   S ddlm} t	        |      }| j                  |dddg|j                  d	d      |
       d{   }|d   j                         }|d   }	| j                  ||d       | j                  ||      }
| j                  |	xx   dz  cc<   t        |d         \  }}}} |di i ||| j                  |
d|}| j                  ||d      }|[t        |t        j                         rA|4 d{    | j#                  ||       d{    | d{   }ddd      d{    n%| j#                  ||       d{    | d{   }| j$                  |	xx   dz  cc<   t'        j(                  d|	 d       |S 7 7 V7 7 ~7 v7 h# 1 d{  7  sw Y   RxY w7 c7 [# t*        $ rE}t'        j,                  d| dt/        |       d       || j0                  |xx   dz  cc<   |d}~ww xY ww)z
        Create a vector store for a specific model.

        Args:
            model: Model name from router config
            **kwargs: Vector store creation parameters

        Returns:
            VectorStoreCreateResponse
        Nr   r  avector_store_creater  r  zvector-store-api-fake-textr  rQ  rR  r   r   rM  rW  r|   r)  rA  r  r  r  z#litellm.avector_store_create(model=r\  rf  rg  r   )r  r  rv  litellm.vector_storesr!   r  ri  r<  rn  r  r  rw   r   ro  r   r  r  r  r  r   r5  r@  r,  r   r  )r?  r   rJ  r  
factory_fnavector_store_create_sdkr  r6  rR  rv  r{  rx  r  rN  r#  rD  s                   r   rO  zRouter.avector_store_create  s    O	}> "22'= 3 
 (1&111Q@H#BB#)6RST$*JJ/Dd$K%	  C   J ./446DgJ//%4 0   >>% ? L Z(A-( ,<$w-+P(A"Aq/ +>#33*	
 H !,,%3 - M (Zw00. ) . .EE#-@P F    &.~H	. . . AA)<L B    "*>z*a/*!&&5j\AXY O 2
T.  .	. . . . *  	!++5eW<PQTUVQWPXX_`  &!+&G	s   H7&G& F?G& H7=G& ,G-C G& -G.G& 1GG		GG	GG& "G#G& >G"?	G& G$	5G& >H7?G& G& G& G	GG& GGGG& $G& &	H4/A H//H44H7c                      y)aE  
        Override factory-generated vector store methods with router-aware implementations.
        This is called after _initialize_vector_store_endpoints() to ensure our custom
        methods that handle deployment selection and credential injection are used instead
        of the generic factory-generated ones.
        Nr   r  s    r   r   z0Router._override_vector_store_methods_for_router<  s     	r   c           	      |  K   	 ||d<   | j                   |d<   |j                  d| j                        |d<   t        d      }| j	                  |||        | j
                  di | d {   }|S 7 # t        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY ww)	Nr   rH  r   _acreate_batchr8  r  r  r   )rU  r^  r   r4   rL  r  r@  r  r  rG   r  r  r?  r   rJ  r&  rN  rD  s         r   acreate_batchzRouter.acreate_batchG  s     
	#F7O*.*=*=F&'$*JJ}d>N>N$OF=!%G.&" 00'= 1 
 @T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		;   B<A(A5 ,A3-A5 2B<3A5 5	B9>6B44B99B<c                 b  K   	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd       |       d {   }|d   j                         }|d	   }| j                  ||d
       | j                  ||      }| j                  |xx   dz  cc<   t        |d	         \  }}	}}t        j                  di i ||	| j                  |d|}
| j                  ||d      }|\t        |t        j                         rB|4 d {    	 | j#                  ||       d {    |
 d {   }
d d d       d {    n%| j#                  ||       d {    |
 d {   }
| j$                  |xx   dz  cc<   t        j&                  d| d       |
S 7 a7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t(        $ rH}t        j*                  d| d| dt-        |       d       || j.                  |xx   dz  cc<   |d }~ww xY ww)Nz Inside _acreate_batch()- model: r  r  r:  r  rQ  rR  r   r   rU  rM  rW  r|   r)  rA  r  r  r  zlitellm.acreate_batch(model=r\  zlitellm._acreate_batch(model=rI  rf  rg  r   )r   rA  r!   r  ri  r<  rn  r  r  rw   r   rW  r   ro  r   r  r  r  r  r5  r@  r,  r   r  r?  r   rJ  r  r6  rR  rv  r{  rx  r  rN  r#  rD  s                r   rU  zRouter._acreate_batchf      
I	!''25'F8L  AH#BB#)6KLM$*JJ/Dd$K%	  C   J ./446DgJ//%fDT 0   >>% ? L Z(A-( ,<$w-+P(A"Aq,, +>#33*	
 H !,,%3 - M (Zw00. ) . . EE#-@P F    &.~H. . . AA)<L B    "*>z*a/*!&&.zl:QR O{R.
  .. . . . *  	!++/wb@TUXYZU[T\\cd  &!+&G	   H/AG F7CG $F:%G (G F<	G
F>GG G G 6G7	G  G5G 6H/7G :G <G>G G GG	GG G 	H,$AH''H,,H/c                    	K   	 t              }|3 j                  |dddgj                  dd      |       d{   }n j                         }|t	        d      g 	dt
        f	 fd	}|^t        |t              rNt        |      d
kD  r@t        j                  |D cg c]  } |t        t
        |             c}ddi d{   }n<|/t        |t              r |t        t
        |             d{   }nt	        d      |=t        |t              r|S t        |t              r|D ]  }t        |t              s|c S  	r	d
   t	        dj                  	            7 /c c}w 7 7 z# t        $ r;}t        j                  t!         t#        j$                         |             |d}~ww xY ww)z
        Iterate through all models in a model group to check for batch

        Future Improvement - cache the result.
        Nr  zretrieve-api-fake-textr  rQ  r;  Router not yet initialized.rv  c                 2  K   	 ddl m} | d   j                  d      }| d   j                         }|j                  d      }|t	        d|        |st        |      \  }}}} |	      }j                  t        t        |       |d	       |j                  dd        |j                  dd        t        j                  d
i i |d|i| d {   S 7 # t        $ r0}dd l} |j                          
j                  |       Y d }~y d }~ww xY ww)Nr   r  r   r   r  z2Model not found in litellm_params for deployment: r)  aretrieve_batchrM  r   )r  r  r^  r<  r@  rw   rn  r   r4  ri  r   r`  r  	print_excr  )rv  r  r   rR  r  rx  
new_kwargsrD  r  rJ  receieved_exceptionsr?  s            r   try_retrieve_batchz2Router.aretrieve_batch.<locals>.try_retrieve_batch  sI    $ V&'78<<WEE%&67<<>D*.((3H*I'}'PQ[P\] 
 /7G"'84.1 "0!7J77#'j#9)&7 8 
 NN#8$?HH2D9!(!8!8 ""13F )"    !  $'I'')(//2 sA   DCC CC DC 	D$&D
DDDr   r  TrG  z7Unable to find batch in any model. Received errors - {}r  )r!   rK  ri  r  r@  r\   r   r%  r$  r  r  r   r4  rm   r6  r  rG   r  r  )
r?  r   rJ  r  filtered_model_listrd  resultsr  rD  rc  s
   ` `      @r   r`  zRouter.aretrieve_batch  s    l	@H  <<'-:RST(.

3H$(O#)%5 =   $ '+&9&9&;#"* =>>#% % 5H % R $/2D9+,q0 ' &9! +40CU+KL!
 '+!  %0Z#T6 !3,.AB!    ?@@ "g|4"N.") *%fl;#)M*
 $*1-- IPP( sx2  		(,0#)(1(<(<(>'(	 G		s}   G:E< E0A%E< 'E3
E< E80E< ?E: #E< #G$%E< 
E< G#E< 3E< :E< <	G 6F;;G  Gc           	      |  K   	 ||d<   | j                   |d<   |j                  d| j                        |d<   t        d      }| j	                  |||        | j
                  d	i | d{   }|S 7 # t        $ r;}t        j                  t        | |t        j                         |             |d}~ww xY ww)
zZ
        Cancel a batch through the router with proper model-to-provider mapping.
        r   rH  r   _acancel_batchr8  r  Nr  r   )rh  r^  r   r4   rL  r  r@  r  r  rG   r  r  rV  s         r   acancel_batchzRouter.acancel_batch.  s     	#F7O*.*=*=F&'$*JJ}d>N>N$OF=!%G.&" 00'= 1 
 @T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		rX  c                 b  K   	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd       |       d {   }|d   j                         }|d	   }| j                  ||d
       | j                  ||      }| j                  |xx   dz  cc<   t        |d	         \  }}	}}t        j                  di i ||	| j                  |d|}
| j                  ||d      }|\t        |t        j                         rB|4 d {    	 | j#                  ||       d {    |
 d {   }
d d d       d {    n%| j#                  ||       d {    |
 d {   }
| j$                  |xx   dz  cc<   t        j&                  d| d       |
S 7 a7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t(        $ rH}t        j*                  d| d| dt-        |       d       || j.                  |xx   dz  cc<   |d }~ww xY ww)Nz Inside _acancel_batch()- model: r  r  zbatch-api-fake-textr  rQ  rR  r   r   rh  rM  rW  r|   r)  rA  r  r  r  zlitellm.acancel_batch(model=r\  zlitellm._acancel_batch(model=rI  rf  rg  r   )r   rA  r!   r  ri  r<  rn  r  r  rw   r   ri  r   ro  r   r  r  r  r  r5  r@  r,  r   r  rZ  s                r   rh  zRouter._acancel_batchP  r[  r\  c                   K   | j                  |      }|t        d      dt        ffd}t        j                  |D cg c]
  } ||       c}  d{   }dg dddd}|D ]g  }||d	   t        |d	      rt        |d	      |d	<   t        |d
      |d
<   |d   j                  |j                         t        |dd      du scd|d<   i |S c c}w 7 w)zQ
        Return all the batches across all deployments of a model group.
        r  Nr^  r   c                 x   K   	 t        j                  di i | d    d {   S 7 # t        $ r Y y w xY ww)Nr   r   )r   alist_batchesr@  rI  s    r   rd  z0Router.alist_batches.<locals>.try_retrieve_batch  sT     $22 ;/0;F;     s(   :+ )+ :+ 	7:7:r%  F)objectrR  first_idlast_idhas_morero  rp  rR  rq  T)	r  r@  r\   r  r  r  r  extendrR  )r?  r   rJ  re  rd  rf  final_resultsr  s     `     r   rm  zRouter.alist_batches  s     #11U1C&9::	,? 	  5HIE 'I
 

 
  
	5F! ,49T07
0KM*-+269+Ei(f%,,V[[9 6:u5=04M*-
	5 / J
s*   ?CCCCC-AC
Cr  c                   K   | j                  |j                  dd      |       |j                  d      ru| j                  |d         r`| j                  |d   |       d {   }|d   d   |d<   |d   j	                         }| j                  ||       |j                  |        |di | d {   S 7 V7 w)	Nr   r_  rI  r  )r   rS  r   rW  r   )rL  r^  r  r  r<  rn  r   )r?  rH  r  rJ  r6  rR  s         r   )_pass_through_moderation_endpoint_factoryz0Router._pass_through_moderation_endpoint_factory  s      	,,**Wb) 	- 	
 ::g4#6#6&/#6#R#BBWo%  C   J ))9:7CF7O./446D//% 0  MM$&0000 1s%   A#C %B<&AC 7B>8C >C r  )d
assistantsr  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  rO  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  aupload_container_fileupload_container_filealist_container_fileslist_container_filesaretrieve_container_fileretrieve_container_fileadelete_container_filedelete_container_filer  r  r  r  r  r  r  r  r  r  r  r  c                 P    dv r&	 	 d
dt         t           dt         t           f fd}|S dv r&	 	 d
dt         t           dt         t           f fd}|S dv r%	 	 d
dt         t           dt         t           ffd}|S 	 	 d
dt         t           dt         t           f fd	}|S )z
        Creates appropriate wrapper functions for different API call types.

        Returns:
            - A synchronous function for synchronous call types
            - An asynchronous function for asynchronous call types
        )r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r[  c                 ,     j                   ddi|S )NrH  r   )r)  r  r[  rJ  rH  r?  s      r   sync_wrapperz-Router.factory_function.<locals>.sync_wrapperr  s*    
 =t<< &7;A r   )r  r  r  r  c                 t    | r	d|vr| |d<   |j                  d      r j                  ddi|S  di |S )Nr  r   rH  r   )r^  r)  r  s      r   vector_store_sync_wrapperz:Router.factory_function.<locals>.vector_store_sync_wrapper  s\    
 '+@+N4GF01::g&@4@@ *;?E  )2622r   )r  r  r  r  r  r  c                      d| |d|S )Nr  r[  r   r   )r  r[  rJ  rH  s      r   vector_store_file_sync_wrapperz?Router.factory_function.<locals>.vector_store_file_sync_wrapper  s'    
 ) (;!  r   c                   K   dk(  r j                   d| |d| d {   S dk(  r j                  ddi| d {   S dv r j                  ddi| d {   S dv r j                  d| d| d {   S dv r j                  ddi| d {   S d	v r j
                  d| d| d {   S d
k(  r j                  ddd| d {   S dv r j                  ddi| d {   S dv r j                  d| d| d {   S dv r j                  d| |d| d {   S dv r j                  d| d| d {   S y 7 O7 17 7 7 7 7 7 z7 [7 ;7 w)Nrv  )rH  r  r[  r  rH  r  )r  r  r  r  r  r  )rH  r  )r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  )	r  r  r  r  rw  ry  r{  r}   aretrieve_container_file_contentr  T)rH  r%  )r  r  r  r  r  )r  rO  r  r  r  r  )r  r  )r  r  r  r   ))_pass_through_assistants_endpoint_factoryru  r   _init_vector_store_api_endpointsr  _init_containers_api_endpoints_init_responses_api_endpoints _init_interactions_api_endpoints)r  r[  rJ  r  rH  r?  s      r   async_wrapperz.Router.factory_function.<locals>.async_wrapper  s    
 L(KTKK &7(;! 	   l*KTKK &7;A   339T99 &7      CTBB &7(;   
   > DTCC &7    
 
 AT@@ &7(;   
 66CTCC &715   
    @T?? &7      CTBB &7(;   
 ??CTCC &7(;! 	     
 CTBB &7(;   iHs   F
E1 F
E4F
#E7$ F
E:F
$E<% F
E>!F
'F (F
F F
(F)!F

F F
+F,F
4F
7F
:F
<F
>F
 F
F
F
F
F
NN)r   r   r   )r?  rH  r  r  r  r  r  s   ```    r   rv  zRouter.factory_function  s    d  
 
( 6:(,%-c]    
 
 6:(,3%-c]3 3 -, 
 
 6:(,	%-c]	 	 21 26$(C	!)#C	SMC	J r   c                    K   |r	d|vr||d<   |j                  d      r | j                  dd|i| d{   S  |di | d{   S 7 7 w)z
        Initialize the Vector Store API endpoints on the router.

        If a model is provided in kwargs, use model-based routing to get
        the deployment credentials. Otherwise, call the original function directly.
        r  r   rH  Nr   )r^  r  r?  rH  r  rJ  s       r   r  z'Router._init_vector_store_api_endpoints1  st      #8#F,?F() ::g??? "3   '0000 1s!   4AAAAAAc                 D   K   |r	d|vr||d<    |di | d{   S 7 w)z
        Initialize the Containers API endpoints on the router.

        Container operations don't need model-based routing, so we call the
        original function directly with the custom_llm_provider.
        r  Nr   r   r  s       r   r  z%Router._init_containers_api_endpointsJ  s2      #8#F,?F()&00000s     c                    K   ddl m} |j                  |j                  d            }|||d<    | j                  dd|i| d{   S 7 w)z
        Initialize the Responses API endpoints on the router.

        GET, DELETE, CANCEL Responses API Requests encode the model_id in the response_id, this function decodes the response_id and sets the model to the model_id.
        r   )ResponsesAPIRequestUtilsresponse_idNr   rH  r   )litellm.responses.utilsr  get_model_id_from_response_idr^  r  )r?  rH  rJ  r  model_ids        r   r  z$Router._init_responses_api_endpointsZ  sh      	E+IIJJ}%
 &F7O;T;; 
/

 
 	
 
s   AAAAc                 V   K   |r	d|vr||d<   d|vrd|d<    |di | d{   S 7 w)z
        Initialize the Interactions API endpoints on the router.

        GET, DELETE, CANCEL Interactions API Requests don't need model-based routing,
        so we call the original function directly with the custom_llm_provider.
        r  geminiNr   r   r  s       r   r  z'Router._init_interactions_api_endpointsp  sE      #8#F,?F() .,4F()&00000r
  r[  c                    K   |E| j                   .| j                   d   }|j                  | j                   d          nt        d       |d||d| d{   S 7 w)z@Internal helper function to pass through the assistants endpointNr  r   z'custom_llm_provider' must be set. Either via:
 `Router(assistants_config={'custom_llm_provider': ..})` 
or
 `router.arun_thread(custom_llm_provider=..)`r  r   )r   r   r@  )r?  rH  r  r[  rJ  s        r   r  z0Router._pass_through_assistants_endpoint_factory  s      &%%1&*&<&<=R&S#d445EFG s  ' 
 3F
FL
 
 	
 
s   AAAArD  r  r  c	                 X
  K   t        j                  dt        j                                 |}	d}
|j	                  d      }d}|du s||| |	d|}d|vr| j
                  |d<   d|vrd	|d<   t        |t        j                  t        j                  f      }| j                  |
      }|D ch c]D  }|j	                  di       j	                  d      !|j	                  di       j	                  d      F }}t        |      }t        |      dkD  r|s|j	                  d      }||n|d	   }|D cg c]  }||kD  r||d }}d}|;|9t        |      r|}n*t        |t        t         |            \  }}|
|||   d   }||xs g z   }|r)|j#                  ||d       t%        |i | d{   }|S 	 t        j&                  d       t        |      }|r)|j#                  ||d       t%        |i | d{   }|S t        |t        j                        r|@| j)                  ||      }||	|j#                  ||d       t%        |i | d{   }|S dj+                  |||      }t        j&                  dj+                  |             |xj,                  dj+                  |      z  c_        nt        |t        j                        r|@| j)                  ||      }||	|j#                  ||d       t%        |i | d{   }|S dj+                  |||      }t        j&                  dj+                  |             |xj,                  dj+                  |      z  c_        ||t        j                  d|        t        |t        t         |            \  }
}|

|||   d   }
|
Dt        j&                  d| d|        t/        |	d      r|	xj,                  d| d| z  c_        |	|j#                  |
|d       t%        |i | d{   }|S t/        |	d      rW|	xj,                  d!j+                  ||
      z  c_        t        |      d	kD  r$|	xj,                  d"j+                  |      z  c_        |	c c}w c c}w 7 7 7 F7 7 # t0        $ ru} t3        |      }!t        j4                  dj+                  t!        |       t        j                         t7        | |!        d{  7               t!        |       }Y d} ~ d} ~ ww xY ww)#zD
        Common utilities for async_function_with_fallbacks
        	TracebackNr   r_  T)litellm_routerr  r   fallback_depthr   r  r   orderr|   _target_order)r   r  )r   )r   r  r   )fallback_model_grouporiginal_model_groupzTrying to fallback b/w modelszmodel={}. context_window_fallbacks={}. fallbacks={}.

Set 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbackszGot 'ContextWindowExceededError'. No context_window_fallback set. Defaulting                         to fallbacks, if available.{})msgz
{}zmodel={}. content_policy_fallback={}. fallbacks={}.

Set 'content_policy_fallback' - https://docs.litellm.ai/docs/routing#fallbackszGot 'ContentPolicyViolationError'. No content_policy_fallback set. Defaulting                         to fallbacks, if available.{}zinside model fallbacks: z7No fallback model group found for original model_group=z. Fallbacks=ra  zlitellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - {}
{}

Debug Information:
Cooldown Deployments={}r  r  z<. Received Model Group={}
Available Model Group Fallbacks={}z
Error doing the fallback: {})r   rA  r  r  r^  r   r   r   ContextWindowExceededErrorrs  _get_all_deploymentssortedr$  rB   rC   r   r   r   rD   r5  (_get_fallback_model_group_from_fallbacksr6  ra  r  r@  r!   r  r?   )"r?  rD  r  r   r   r   r  rK  rJ  r  r  r  fallback_failure_exception_strr|  _skip_order_fallbackall_deploymentsd
_order_setorder_valuescurrent_target
skip_up_tooorder_fallback_entriesexternal_fallback_groupgeneric_idxcombined_fallbacksrN  is_non_standard_fallback_format#context_window_fallback_model_grouperror_message#content_policy_fallback_model_groupgeneric_fallback_idxnew_exceptionr  s"                                     r   r  z1Router.async_function_with_fallbacks_common_utils  s     	##i	0D0D0F/G$HI#.4jj.A)+&$(<(DG #"4
 
 ,.,0,>,>L)</-.L)*  *//1T1TU 
 33?S3T %
uu%r*..w7C EE"B'++G4

 

 $J/|q )=#ZZ8N"0"<,q/  &,z> /C," , 7;#$)@6K /8+;S"+$(k$:<8+[ /6;;R2;K2H2M/ "8'-2" "##0B0D "4"""   L	@!&&'FG /R#/+ /##090D "4""" 
  !W??@+7 EE":$/ F  8 ;B00 ''4W4H &8&&&  H $O %m  %t  %t#%=y%M *..77=v)8 II}!==IAwBBC+7 EE":$/ F  8 ;B00 ''4W4H &8&&&  H $O %l  %s  %s#%=y%M *..77=v)8 II}!==I$)@%++.Fyk,RS -' $S+ 6(( (/4H4T+45I+J3+O('/)..QR]Q^^jktjuv 19=*228op{o|  }I  JS  IT  7U  U2,,##0D0D "4""" 
   %y1&&*i*p*p$+ & 12Q6"**4;;6* ! ]
,@,2 H Z  	@@H!'' k  r  r&((*I04)9  	 .1-?*	@s   B-T*/A	R8;T*3RA.T*2R3T*:AR) ?R R) T*AR) R!R) !T*"B2R) R$R) T*D
R) %R'&R) +A1T*R) !R) $R) 'R) )	T'2AT"TT"T*"T''T*c                   K   |j                  d      }|j                  dd      }|j                  d| j                        }|j                  d| j                        }|j                  d| j                        }|j                  dd      }	 | j                  |||||	       | | j                  |i |d|i d{   }	n | j                  |i | d{   }	t        j                  t        j                        rt        j                  d
|	        t        |	d      }	|	S 7 i7 P# t        $ r+}
| j                  |
|||||||       d{  7  cY d}
~
S d}
~
ww xY ww)z
        Try calling the function_with_retries
        If it fails after num_retries, fall back to another model group
        r   r  Fr   r   r   mock_timeoutN)rJ  r  r   r   r   zAsync Response: r   )rN  attempted_fallbacks)r^  ri  r   r   r   _handle_mock_testing_fallbacksasync_function_with_retriesr   isEnabledForr   r   rA  r2   r@  r  )r?  rK  rJ  r  r  r   r   r   r  rN  rD  s              r   r  z$Router.async_function_with_fallbacks  s     &,ZZ%8,2JJ7JE,R$*JJ{DNN$K	39::&(E(E4
  4:::&(E(E4
  zz.$7 	//'#)A)A 0  '!A!A!A"#"2>"  "B!A!A4!R6!RR$11'--@%++.>xj,IJ7!$%H O S  
	HH!((	 	 	 	
	sg   B
E#1D, >D(?D, D*AD, 'E#(D, *D, ,	E 5EEEE E#E  E#c                    t        j                  |      }|j                  ,|j                  du rt        j                  |dd| d|       |j
                  ,|j
                  du rt        j                  |dd| d|       |j                  -|j                  du rt        j                  |dd| d|       yy)	a  
        Helper function to raise a litellm Error for mock testing purposes.

        Raises:
            litellm.InternalServerError: when `mock_testing_fallbacks=True` passed in request params
            litellm.ContextWindowExceededError: when `mock_testing_context_fallbacks=True` passed in request params
            litellm.ContentPolicyViolationError: when `mock_testing_content_policy_fallbacks=True` passed in request params
        NTr_  #This is a mock exception for model=z#, to trigger a fallback. Fallbacks=)r   rb  ra  zF, to trigger a fallback.                     Context_Window_Fallbacks=zF, to trigger a fallback.                     Context_Policy_Fallbacks=)	r_   from_kwargsmock_testing_fallbacksr   InternalServerErrormock_testing_context_fallbacksr  %mock_testing_content_policy_fallbacksrs  )r?  rJ  r  r   r   r   mock_testing_paramss          r   r  z%Router._handle_mock_testing_fallbacks  s     6AA&I66B#::dB--!=k]Jmnwmxy   >>J#BBdJ44!=k] K..F-GI   EEQ#IITQ55!=k] K..F-GI  R Rr   c                   K   t        j                  d       |j                  d      }|j                  d| j                        }t	        |      }|j                  d| j
                        }|j                  d| j                        }|j                  d| j                        }|j                  d      }	|j                  d      }
|j                  d	|j                  d
            xs i }d|v rFt        |d   t              r3| j                  |d         }||j                  dt        |      i       t        j                  d| d|
        d|d<   |
|d<   	 | j                  |	|        | j                  |g|i | d {   }t!        |dd       }|S 7 # t"        $ r}d }|}t%        |dd       }|t        |t&              r|}
	 | j)                  |j                  d      xs d|       d {  7  \  }}d}| j*                  |G|	xs$ |j                  d      xs |j                  d      }t-        |||| j*                        }||}
d}|s| j/                  ||||||       |
|d<   |
dkD  r| j1                  ||      }n t        j                  d|
        | j3                  ||
|
||      }t5        j6                  |       d {  7   t9        |
      D ]/  }	 |dz   |d<   |
|d<    | j                  |g|i | d {  7  }t;        j<                  |      r| d {  7  }t!        ||dz   |
      }|c cY d }~S # t"        $ r}|}| j1                  ||      }|
|z
  dz
  }|j                  d      }| | j)                  ||       d {  7  \  }}ng }|s(	 | j/                  ||||||       n# t"        $ r |w xY w| j3                  |||
||      }t5        j6                  |       d {  7   Y d }~*d }~ww xY w t?        |      t@        jB                  v r#tE        |d|
       ||dz   n|
}tE        |d|       |d }~ww xY ww)Nz#Inside async function with retries.rH  r   r   r   r   r   r   r  r   r  r  model_group_sizez/async function w/ retries: original_function - z, num_retries - r   attempted_retriesr   )r  rJ  )rN  r  r   r_  r  Fr,  r  r   r   T)r  r  r  r   regular_fallbacksr   )rJ  rD  z#Retrying request with num_retries: )rD  remaining_retriesr   r  r  r|   )#r   rA  ri  r   r!   r   r   r   r^  r   r   r  r   r$  %_handle_mock_testing_rate_limit_error	make_callr3   r@  r  r  r  r   "_get_num_retries_from_retry_policyshould_retry_this_error	log_retry_time_to_sleep_before_retryr  r  ru  r#   is_async_callabler   r   LITELLM_EXCEPTION_TYPESr  )r?  rK  rJ  rH  r   r  r   r   r   r  r   	_metadatar   rN  rD  current_attemptr  deployment_num_retriesr  _all_deployments_retry_policy_applies_model_group_for_retry_policy_retry_policy_retriesr   r  _modelrx  _timeoutactual_retries_attempteds                                r   r  z"Router.async_function_with_retries  sN    ##$IJ"JJ':;JJ{DNN;	<VD#)::&(E(E$
  $*::&(E(E$
  $*::&(E(E$
  &,ZZ%8jj/ !**%7J9OPVTV	I%*Y}5Ms*S,,	-8P,QJ%  "4c*o!FG##=>O=PP`al`mn	
 *+	%&  		
b	%66' 7  ,T^^,=OOOOH4!QDH O	 P
  X	%"O!"%,Qt%D"%1j&7 5 99jj)/R!1 :   $  %*!  ,0H0T
  V9==#?V6::gCV . )K0 =-E!%!2!2	)% )4"7K,0) ),,(<$4-E&/-E -  (3Im$ Qv9KL!''5k]C ::$"-'$8 0 ; K --,,,#(#5 ?2>25Dq5HI12/:Im,%3T^^4E%W%WPV%WWWH(::  *2>><!)*9A*=$/ H
 $O  ,2 *+& "^^6Q^?F(3o(E(I%,2JJw,?F) #'"E"E"(-= #F #  0 02, 1
$ 88&'4H0@9Q2;9Q 9   ) $"#G$  $??*;$/,@(8  @  H "--111Y,2'?2B &'7+J+JJ*M;G ,;+FOa'K ) *M;ST$$qX	%s   EQ!,F FF Q!F 
Q(AQ7G:8CQKQ+&MLM1L42M	QQQ!	PAP N#
!P/OPO	1PP
P
QPAQQQ!c                    K   |j                  d      } ||i |}t        j                  |      st        j                  |      r
| d{   }| j                  ||       d{   }|S 7 !7 w)z^
        Handler for making a call to the .completion()/.embeddings()/etc. functions.
        r   N)rN  r  )r^  r#   r  rI  isawaitableset_response_headers)r?  rH  rK  rJ  r  rN  s         r   r  zRouter.make_call  s}      jj)$d5f5..x8G<O<O=
 &~H22; 3 
 
  &
s$   A	A1A-A1&A/'A1/A1c                 @   |j                  dd      }| j                  |      }d}|:t        |      dk(  r,t        t        t
           |d   d   j                  d            }|:|du r5t        j                  d	|        t        j                  |d
d| d|      yy)z
        Helper function to raise a mock litellm.RateLimitError error for testing purposes.

        Raises:
            litellm.RateLimitError error when `mock_testing_rate_limit_error=True` passed in request params
        mock_testing_rate_limit_errorNr  r|   r   r   r   TzTlitellm.router.py::_mock_rate_limit_error() - Raising mock RateLimitError for model=r_  r  z , to trigger a rate limit error.)r   rb  ra  r   )ri  r  r$  r   r   r  r^  r   r5  r   RateLimitError)r?  rJ  r  r  available_modelsr   s         r   r  z,Router._handle_mock_testing_rate_limit_error  s     9?

+T9
%  ..+.F%)'C0@,AQ,F/23CDHHWK
 *5-5!&&fgrfst ((!=k]Jjk'	  6 6r   r  r  r  r  c                 8   d}|t        |t              rt        |      }d}|t        |t              rt        |      }t        |t        j                        r||t        |t        j
                        r||t        |dd      }	|	t        j                  |	      s|	dvr|t        |t        j                        r|t        |t        j                        r|dk  r|t        |      dkD  r|t        |t        j                        r	 |dk  r||dk  r|y)au  
        1. raise an exception for ContextWindowExceededError if context_window_fallbacks is not None
        2. raise an exception for ContentPolicyViolationError if content_policy_fallbacks is not None

        2. raise an exception for RateLimitError if
            - there are no fallbacks
            - there are no healthy deployments in the same model group
        r   Nstatus_code)i  i  r|   T)r   r%  r$  r   r  rs  r  _should_retryNotFoundErrorr  r  AuthenticationError)
r?  r  r  r  r   r   r  _num_healthy_deployments_num_all_deploymentsr  s
             r   r  zRouter.should_retry_this_error  s*   " $% *z:Mt/T'*+>'?$ &:ot+L#&#7  ug@@A(4K ugAAB(4Ke]D9"7+@+@+M*,eW223KeV223(A-%1)*Q.eV778
 %) $q(Kr   c                 4    t        | j                  g|i |S )z~
        Sync wrapper for async_function_with_fallbacks

        Wrapped to reduce code duplication and prevent bugs.
        )r    r  )r?  rK  rJ  s      r   rM  zRouter.function_with_fallbacks7  s     "$"D"DVtVvVVr   c                 l    |yd}|D ])  }t        |j                               d   |k(  s#||   } |S  |S )a@  
        Returns the list of fallback models to use for a given model group

        If no fallback model group is found, returns None

        Example:
            fallbacks = [{"gpt-3.5-turbo": ["gpt-4"]}, {"gpt-4o": ["gpt-3.5-turbo"]}]
            model_group = "gpt-3.5-turbo"
            returns: ["gpt-4"]
        Nr   )r%  keys)r?  r   r  r  r  s        r   r  z/Router._get_fallback_model_group_from_fallbacks?  sV     48 	DDIIK #{2'+K'8$##		 $#r   c                     | j                   y| j                   D ]C  }t        |t              sd|v s|d   }t        |t              s/t	        |      dkD  s>|d   c S  y)zX
        Returns the first model from the default_fallbacks list, if it exists.
        Nr   r   )r   r   r4  r%  r$  )r?  fallbackdefault_lists      r   _get_first_default_fallbackz"Router._get_first_default_fallbackX  sb     >>! 	+H(D)cXo'}lD1c,6G!6K'?*		+
 r   r  c                    |t        |      dk(  rn!|t        |t              rt        |      dkD  ryd}t        |d      r,t        |j                  d      r|j                  j
                  }t        |d      r|j                  }|%t        j                  |||| j                        }|S t        j                  ||| j                        }|S )	z
        Calculate back-off, then retry

        It should instantly retry only when:
            1. there are healthy deployments in the same model group
            2. there are fallbacks for the completion call
        Nr|   r   rN  headerslitellm_response_headers)r  r   response_headersmin_timeout)r  r   r  )
r$  r   r%  r  rN  r  r  r   _calculate_retry_afterr   )r?  rD  r  r   r  r  r  r   s           r   r  z"Router._time_to_sleep_before_retrye  s    " &3+?1+D+.5'(1,481j!gajj)&D zz11101 99'44"3'!1 ,,	G  44"3' ,,G r   c                   K   ddl m} 	 |j                  dd      }|t        d      |d   j                  d      y|d   d   j                  dd      }|d   d   j                  d	d      }|d   j                  d
i       xs i }	|	j                  dd      }
||
yt	        |
t
              rt        |
      }
| j                  |
      }|y| j                  ||      }|j                  dd      }|j                  dd      }|j                  j                  }|j                  j                  }|j                  dd      }|j                  dd      }t        | |
       ||	||||yt        |      }|j                  dd      }t               }|j                  d      }t         j"                  j$                  j'                  |
||      }g }|j)                   |||t*        j,                  j$                               t         j.                  j$                  j'                  |
||      }|j)                   ||dt*        j,                  j$                               | j0                  j3                  ||       d{    |S 7 # t4        $ r7}t7        j8                  dj'                  t        |                   Y d}~yd}~ww xY ww)zG
        Track remaining tpm/rpm quota for model in model_list
        r   )RedisPipelineIncrementOperationr  Nzstandard_logging_object is Noner   r   r6  r  r7  r9  r  r6  received_model_nametpmrpmr  deployment_idtotal_tokens%H-%Mr9  current_minuter   )r3  increment_valuer   r|   )increment_listr  zOlitellm.router.Router::deployment_callback_on_success(): Exception occured - {})litellm.types.cachingr   r^  rh  r   r  r   get_deploymentget_router_model_infor   r  r  rM   r!   rz   strftimerc   TPMrg  r6  r  r   r   RPMr   async_increment_cache_pipeliner@  r   rA  )r?  rJ  completion_responser  r  r   r  deployment_namer  r7  r9  deployment_infodeployment_model_infor  r  tpm_litellm_paramsrpm_litellm_paramstpm_model_inforpm_model_infor  r  dtr  tpm_keypipeline_operationsrpm_keyrD  s                              r   r(  z%Router.deployment_callback_on_success  s'     	Jt	HN

)4I# '. !BCC&'++J7?"()9"::"F"J"J $# %%56zBFF!4 $$4599,KQr
^^D$/&"*C(RB #'"5"5r"5"B"*,0,F,F#2,7 -G -)
 *--eT:C)--eT:C *9)G)G)K)K&)8)G)G)K)K& &;%>%>ud%KN%:%>%>ud%KN B,0"$ K*2*2&.&.#DV#L &=&A&A.RS&T
 &'!#" *--33::. ;  NP# $**3#(4'OO11 *--33::. ;  $**3#()'OO11 jj??#6%5 @   
   	!''ahhF
 	sq   K3J KAJ K/J KB J .K/DJ JJ KJ 	K-K	K	KKc                    d}|d   j                  d      nc|d   d   j                  dd      }|d   j                  di       xs i }|j                  dd      }||yt        |t              rt        |      }|t	        | |      }|S y)z
        Tracks the number of successes for a deployment in the current minute (using in-memory cache)

        Returns:
        - key: str - The key used to increment the cache
        - None: if no key is found
        Nr   r   r  r7  r9  r  )r^  r   r  r   rM   )	r?  rJ  r  r  r  r9  r  r7  r3  s	            r   r+  z*Router.sync_deployment_callback_on_success  s     "#''
3; !12:>BB=RVWK 0155lBGM2Jd+B"bjB$W>C(, C Jr   c                    t        j                  d       	 |j                  dd      }t        |dd      }|j                  di       }|j                  di       }t        j
                  j                  j                  |      }	|j                  d	d      }
d}|	 t        j                  j                  |	
      }|
|
dk\  r|
}n||dk\  r|}n| j                  }t        |t              r3|j                  d      }|yt        | |       t        | ||||      }|S t        j                  d       y# t        $ r}|d}~ww xY w)a`  
        2 jobs:
        - Tracks the number of failures for a deployment in the current minute (using in-memory cache)
        - Puts the deployment in cooldown if it exceeds the allowed fails / minute

        Returns:
        - True if the deployment should be put in cooldown
        - False if the deployment should not be put in cooldown
        z1Router: Entering 'deployment_callback_on_failure'r,  Nr  r_  r   r7  )r  r   )r  r   r9  Fr  r  exception_statusr  r6  time_to_cooldownzWRouter: Exiting 'deployment_callback_on_failure' without cooldown. No model_info found.)r   rA  r^  r  r   litellm_core_utilsexception_mapping_utils_get_response_headersutils&_get_retry_after_from_exception_headerr   r   r4  rL   rA   r@  )r?  rJ  r  r  r  r,  r$  r   _model_infoexception_headersdeployment_cooldownheader_cooldown_time_to_cooldownr  r  rD  s                   r   r1  z%Router.deployment_callback_on_failure=  sp     	##$WX9	

;5I&y-D $ZZ(8"=N(,,\2>K ' : : R R h h#, !i !
 #1"4"4_d"K"O ,")--"V"V%6 #W # #.3F!3K$7! ,A1E$3!$($6$6!+t,/:t/D ( @,0"/ 3,0%5'0,%6 %++m  	G	s$   C&D3 >D3 D3 3	E<D>>Er  c                 (  K   |d   d   j                  dd      }|d   d   j                  dd      }|d   j                  di       xs i }|j                  dd      }||yt        |t              rt        |      }t	        |      }	t               }
|
j                  d      }t        j                  j                  j                  |||	      }| j                  j                  |d
|	t        j                  j                         d{    y7 w)z3
        Update RPM usage for a deployment
        r   r   r6  Nr  r7  r9  r	  r
  r|   )r3  rg  r  r   )r^  r   r  r   r!   rz   r  rc   r  rg  r6  r   async_increment_cacher   r   )r?  rJ  r  r  r  r  r  r7  r9  r  r  r  r   s                r   r.  z+Router.async_deployment_callback_on_failure  s     !!12:>BB$
 -.z:>>}dS,-11,CIr
^^D$'"*C RB<VD

 "%%++22. 3 
 jj..-%%	 / 
 	
 	
s   DD
DD)r   r  c                     t        |      S )a  
        Helper to return what the "metadata" field should be called in the request data

        - New endpoints return `litellm_metadata`
        - Old endpoints return `metadata`

        Context:
        - LiteLLM used `metadata` as an internal field for storing metadata
        - OpenAI then started using this field for their metadata
        - LiteLLM is now moving to using `litellm_metadata` for our metadata
        )r"   )r?  rJ  s     r   '_get_metadata_variable_name_from_kwargsz.Router._get_metadata_variable_name_from_kwargs  s     6f==r   c                    	 d|v rdnd}t        |      j                  t        |      d}|j                         D ]W  \  }}||ddfvr|||<   ||k(  st	        |t
              s*i ||<   ||   j                         D ]  \  }}|dk7  s|||   |<    Y t        | j                        dkD  r| j                  j                  d       | j                  j                  |       | j                  ||   d<   |S # t        $ r}|d	}~ww xY w)
z
        When a retry or fallback happens, log the details of the just failed model call - similar to Sentry breadcrumbing
        r  r   )exception_typeexception_stringrF  rH  r     r   N)r   r   r   r  r   r4  r$  r  ri  r  r@  )	r?  rJ  rD  _metadata_varprevious_modelr  r  
metadata_k
metadata_vs	            r   r  zRouter.log_retry  s8   	&8F&B"
 
 #'q'"2"2$'FN 	G  ]J8KLL()N1%-'Jq$,?46N=1282G2M2M2O G.
J%)::<FN1-j9GG 4''(1,$$((+  ''77;7K7KF=!"34M 	G	s+   AC7 C7 %#C7 	A-C7 7	D DDr  r  c                     |}| j                   j                  ||d      }|#d}| j                   j                  ||dd       |S |dz  }| j                   j                  ||d       |S )zf
        Update deployment rpm for that minute

        Returns:
        - int: request count
        T)r3  r  
local_onlyr|   r   )r3  rg  r=  r   )r3  rg  r=  )r   	get_cache	set_cache)r?  r  r  r   request_counts        r   _update_usagezRouter._update_usage  s      

,,*:t - 
  MJJ  =Tr !   QMJJ  =T !  r   c                 n    | j                   y| j                   D ]  }t        |t              sd|v s y y)NFr   T)r   r   r4  )r?  r  s     r   _has_default_fallbackszRouter._has_default_fallbacks  s;    >>! 	 H(D)(?	  r   rN  c                    |j                   r5t        |j                         dkD  r|j                   d   j                  dk7  ry|j                  d| j                        }|2d}|D ](  }t        |j                               d   |k(  s#||   } n |y| j                         ryt        j                  dj                  ||             y)z
        Determines if a content policy error should be raised.

        Only raised if a fallback is available.

        Else, original response is returned.
        r   content_filterFr   NTzyContent Policy Error occurred. No available fallbacks. Returning original response. model={}, content_policy_fallbacks={})choicesr$  finish_reasonr^  r   r%  r  rC  r   rA  r6  )r?  r   rN  rJ  r   r  r  s          r   rr  z)Router._should_raise_content_policy_error  s     H$4$4 5 9"004DD#)::&(E(E$
 
 $/#' 0 		$Q'50+/;(
 $/((*## H  O  O/	

 r   c                     g }	 | j                  |      \  }}t        |t              rg S 	 t	        | |      }g }|D ]  }|d   d   |v r|j                  |         ||fS # t        $ r Y Aw xY w)Nr)  r  r7  r9  )#_common_checks_available_deploymentr   r4  r@  r@   r  )r?  r   r  r  rx  unhealthy_deploymentsr  r6  s           r   _get_healthy_deploymentszRouter._get_healthy_deployments*  s    !#	"&"J"J #K #A *D1	 2
 !:$(;K!
 %'* 	7J,'-1FF#**:6		7 #$444  		s   &A" "	A.-A.c                   K   g }	 | j                  |      \  }}t        |t              rg |fS 	 t	        | |       d{   }t        |      }g }|D ]  }|d   d   |vs|j                  |         ||fS # t        $ r Y Tw xY w7 Gw)z
        Returns Tuple of:
        - Tuple[List[Dict], List[Dict]]:
            1. healthy_deployments: list of healthy deployments
            2. all_deployments: list of all deployments
        r)  r  Nr7  r9  )rI  r   r4  r@  r>   setr  )	r?  r   r  r  rx  rJ  unhealthy_deployments_setr  r6  s	            r   r  z%Router._async_get_healthy_deploymentsA  s      "$	"&"J"J #K #A *D1+++ 2
 'F$(;K'
 !
 %((=$>!$&* 	7J,'-5NN#**:6	7 #$444  		!
s9   B
(A9 B
 B B
"B
9	BB
BB
c                 r    t         j                  D ]$  }t        |t              s|j	                  |       & y)a  
        Mimics 'async_routing_strategy_pre_call_checks'

        Ensures consistent update rpm implementation for 'usage-based-routing-v2'

        Returns:
        - None

        Raises:
        - Rate Limit Exception - If the deployment is over it's tpm/rpm limits
        N)r   rX  r   r   r4  )r?  r6  r5  s      r   rq  z'Router.routing_strategy_pre_call_checks_  s1     !** 	5I)\2((4	5r   r  c           
      |  K   t         j                  D ].  }t        |t              s	 |j	                  ||       d{    0 y7 # t         j
                  $ r}|t        j                  |j                  |t        j                         t        j                                      t        j                  |j                  |t        j                         f      j                          t!        | |j"                  ||d   d   | j$                         |d}~wt&        $ r}|t        j                  |j                  |t        j                         t        j                                      t        j                  |j                  |t        j                         f      j                          |d}~ww xY ww)O  
        For usage-based-routing-v2, enables running rpm checks before the call is made, inside the semaphore.

        -> makes the calls concurrency-safe, when rpm limits are set for a deployment

        Returns:
        - None

        Raises:
        - Rate Limit Exception - If the deployment is over it's tpm/rpm limits
        Nr,  traceback_exceptionr  rU  rK  r7  r9  r#  )r   rX  r   r   async_pre_call_checkr  r  r  async_failure_handlerr  r  r  rj  rk  failure_handlerrm  rA   r  r   r@  )r?  r6  r  r  r5  rD  s         r   r  z-Router.async_routing_strategy_pre_call_checkso  s    " !** *	I)\2(#88EUVVV*	 W-- ".++'==*+4=4H4H4J)- >  "((#.#>#>"#Y%9%9%;!<  %'-04)*+,#-l#;D#A)-);); G  ".++'==*+4=4H4H4J)- >  "((#.#>#>"#Y%9%9%;!<  %'GsF   $F<AAAF<AF9B<DF9"BF44F99F<rS  c           
        K   |}t         j                  D ]2  }t        |t              s	 |j	                  |||||       d{   }4 |S 7 # t
        $ r}	|t        j                  |j                  |	t        j                         t        j                                      t        j                  |j                  |	t        j                         f      j                          |	d}	~	ww xY ww)rQ  r   r  rF  rS  r  NrR  rT  )r   rX  r   r   async_filter_deploymentsr@  r  r  rV  r  r  r  rj  rk  rW  rm  )
r?  r   r  rF  r  rS  r  returned_healthy_deploymentsr5  rD  s
             r   !async_callback_filter_deploymentsz(Router.async_callback_filter_deployments  s     ( (;$ ** 	I)\2'@@"'0L%-+9-= A   1	8 ,+1 ! ".++'==*+4=4H4H4J)- >  "((#.#>#>"#Y%9%9%;!<  %'Gs;   &C1AAAC1A	C.BC))C..C1r   c                    |g}|j                         D ]  \  }}t        |t              r|j                  |       nOt        |t              r%|j                  t        j                  |             n|j                  t        |             t        |t              r|j                  |       t        |t              r%|j                  t        j                  |             |j                  t        |              dj                  |      }t        j                  |j                               }|j                         S )z
        Helper function to consistently generate the same id for a deployment

        - create a string from all the litellm params
        - hash
        - use hash as id
        r_  )r  r   r   r  r4  r_  dumpsjoinhashlibsha256encode	hexdigest)r?  r  r   partsr  r  
concat_strhash_objects           r   r<  zRouter._generate_model_id  s     "((* 	%DAq!S!QAt$TZZ]+SV$!S!QAt$TZZ]+SV$	% WWU^
nnZ%6%6%89$$&&r   r  _model_name_litellm_paramsr+  c           
      $   	 t        di |}t        di ||||d}t        j                  j	                         D ]0  }|j
                  j                  |      |j
                  |   ||<   2 |j                  j                  }|t        j                  ||i       |j
                  j                  }|j
                  j                  |j
                  j                  dz   |z   }t        j                  j	                         }	|j                         D 
ci c]  \  }
}|
|	vs|
| }}
}t        j                  ||i       | j                  |      dur3t        j                   d|j"                   d|j                  d	           y|j
                  j                  d
      xs g }|D ]  }	 t%        j&                  |        | j-                  |      }|j/                  d      }| j1                  ||j                  j                         |S c c}}
w # t$        j(                  $ r$}t+        d|j"                   d|d|       |d}~ww xY w# t2        $ r1}| j4                  rt        j6                  d| d       Y d}~y|d}~ww xY w)a^  
        Create a deployment object and add it to the model list

        If the deployment is not active for the current environment, it is ignored

        Returns:
        - Deployment: The deployment object
        - None: If the deployment is not active for the current environment (if 'supported_environments' is set in litellm_params)
        r;  N
model_costr  rX  TzIgnoring deployment z% as it is not active for environment supported_environments	tag_regexz&Invalid regex in tag_regex for model 'z': u    — r   r   r  zError creating deployment: 1, ignoring and continuing with other deployments.r   )r^   r[   rk   model_fieldsr  r   r^  r7  r9  r   register_modelr   r  r  $deployment_is_active_for_environmentr   r   rv  recompiler  rh  _add_deploymentto_json _add_model_to_list_and_index_mapr@  r   r,  )r?  r  rg  rh  r+  r   r6  fieldr  _custom_pricing_fieldsr  r  _shared_model_info
_tag_regexpatternexcr   rD  s                     r   _create_deploymentzRouter._create_deployment  s    P	-;-No-NN# !&-&	J 4@@EEG J,,007C)3)B)B5)IK&J
 ",,//H#&& +  %3399K((<<H--AACG+U  &@%L%L%Q%Q%S"!,!2!2!4"AAW8W1" " ""!3 99Z9P &--*:+@+@*AAfgqg|g|  ~V  hW  gX  Y  $2266{CIrJ% JJw' ---DJ&&D&9E11j&;&;&>&> 2  M"2 xx $@AVAV@WWZ"+U3%1   	..%//1!4ef 	si   AI B=I H%H*A!I $I 1HAI I I.III 	J%J
J

Jc                 t    |j                   j                  d      ry|j                   j                  d      ryy)z
        Check if the deployment is an auto-router deployment (semantic router).

        Returns True if the litellm_params model starts with "auto_router/"
        but NOT "auto_router/complexity_router" (which uses complexity routing).
        auto_router/complexity_routerFzauto_router/Tr   r  r?  r   s     r   _is_auto_router_deploymentz!Router._is_auto_router_deployment`  s6     **+JK**>:r   c                    ddl m} |j                  j                  }|j                  j                  }||t        d      |j                  j                  }|t        d      |j                  j                  }|t        d       ||j                  |||||       }|j                  | j                  v rt        d|j                   d	      || j                  |j                  <   y)
z
        Initialize the auto-router deployment.

        This will initialize the auto-router and add it to the auto-routers dictionary.
        r   )r   Nzzauto_router_config_path or auto_router_config is required for auto-router deployments. Please set it in the litellm_paramszfauto_router_default_model is required for auto-router deployments. Please set it in the litellm_paramszhauto_router_embedding_model is required for auto-router deployments. Please set it in the litellm_params)rv  auto_router_config_pathauto_router_configdefault_modelembedding_modelr  zAuto-router deployment 3 already exists. Please use a different model name.)
/litellm.router_strategy.auto_router.auto_routerr   r   r  r  rh  auto_router_default_modelauto_router_embedding_modelrv  r  )r?  r6  r   r  r  r  r  autor_routers           r   init_auto_router_deploymentz"Router.init_auto_router_deploymentm  s!    	O %%== 	  -7,E,E,X,X"*/A/I M  %%?? 	  x  %%AA 	 "z  $.!,,$;1'+$($
   D$5$55)**?*?)@@st  4@*//0r   c                 <    |j                   j                  d      ryy)z
        Check if the deployment is a complexity-router deployment.

        Returns True if the litellm_params model starts with "auto_router/complexity_router"
        r  TFr  r  s     r    _is_complexity_router_deploymentz'Router._is_complexity_router_deployment  s     **+JKr   c                    ddl m} |j                  j                  }|j                  j                  }|8|r6|j                  di       }|j                  d      xs |j                  d      }|t        d       ||j                  || |      }|j                  | j                  v rt        d	|j                   d
      || j                  |j                  <   y)z
        Initialize the complexity-router deployment.

        This will initialize the complexity-router and add it to the complexity-routers dictionary.
        r   r   NtiersMEDIUMSIMPLEzcomplexity_router_default_model is required for complexity-router deployments, or configure tiers in complexity_router_config. Please set it in the litellm_params)rv  r  r  complexity_router_configzComplexity-router deployment r  )	;litellm.router_strategy.complexity_router.complexity_routerr   r   r  complexity_router_default_modelr^  rh  rv  r  )r?  r6  r   r  r  r  complexity_routers          r   !init_complexity_router_deploymentz(Router.init_complexity_router_deployment  s    	
 %%>> 	! %%EE 	
  %=,00"=E!IIh/F599X3FM f 
 /?!,,'$(%=	/
   D$;$;;/
0E0E/FFyz  :K
 5 56r   c           	      P   |j                   d|j                   vs|j                   d   yt        d      }|t        d      |t        vrt        dt         d|       |j                   d   D ]"  }|t        vst        dt         d| d	|        ||j                   d   v ryy
)a  
        Function to check if a llm deployment is active for a given environment. Allows using the same config.yaml across multople environments

        Requires `LITELLM_ENVIRONMENT` to be set in .env. Valid values for environment:
            - development
            - staging
            - production

        Raises:
        - ValueError: If LITELLM_ENVIRONMENT is not set in .env or not one of the valid values
        - ValueError: If supported_environments is not set in model_info or not one of the valid values
        rl  TLITELLM_ENVIRONMENT)secret_namezPSet 'supported_environments' for model but not 'LITELLM_ENVIRONMENT' set in .envz#LITELLM_ENVIRONMENT must be one of z. but set as: z&supported_environments must be one of z for deployment: F)r7  r   rh  rU   )r?  r6  litellm_environment_envs       r   rr  z+Router.deployment_is_active_for_environment  s     !!)'z/D/DD$$%=>F,9NO&b  &@@56P5QQ_`s_tu  ))*BC 	D55 <=W<XXfgkfll}  I  ~J  K 	 *"7"78P"QQr   c                 L   t        j                  |      }g | _        i | _        i | _        i | _        | j                          | j                          |D ]
  }|j                  d      }|j                  d      }t        |t              rI|j                         D ]6  \  }}t        |t              s|j                  d      s)t        |      ||<   8 |j                  di       }d|vr| j                  ||      }	|	|d<   |j!                  dd       8t        |d   t"              r%|d   D ]  }
|
|d<   | j%                  ||||        | j%                  ||||        t'        j(                  d| j+                                 |D ch c]  }|d   	 c}| _        y c c}w )	Nrv  r   os.environ/r7  r9  organization)r  rg  rh  r+  z
Initialized Model List )r<  r=  r   r	  r
  r  "_invalidate_model_group_info_cache_invalidate_access_groups_cacheri  r   r4  r  r   r  ry   r<  r^  r%  r~  r   rA  get_model_namesr   )r?  r   original_model_listr   rg  rh  r  r  r+  _idorgrB  s               r   r  zRouter.set_model_list  s   "mmJ702-02-02-//1,,. ) !	E))L1K#ii(89O/40+113 ;DAq!!S)all=.I-7]*; !&		, ;K ;&--k?K$'D!"">48D/J +>: C69ON3++(-$/(7$/	 ,  ''$) +$3 +	 ( 9!	F 	##'(<(<(>'?@	
 6@@AlO@@s   F!c                 ^   dd l }|j                  j                  }d}d|v r(|j                  d      d   }|t        j
                  v rd}|r	|}d }d }d }	nt	        j                  |j                  j                  |j                  j                  dd             \  }}}}	|t        j                  vr#t        j                  |      st        d|       | j                  j                  |j                  j                         |j                  j                  (t        |dd       t        |d      |j                  _        |j                  j                   (t        |d	d       t        |d	      |j                  _        d
|j"                  v r{| j$                  j'                  |j"                  |j)                  d             |j*                  j,                  r/| j.                  j                  |j*                  j,                         |j*                  j                  d      }
|j*                  j                  d      }|
Z|Xd
|v rT|
| j0                  vrt3               | j0                  |
<   | j0                  |
   j'                  ||j)                  d             |j                  j                  dg       xs g }|D ]i  }|j                  di       }dD ]P  }||v s||   j5                  d      s||   j7                  dd      }|j8                  j                  |d      ||<   R k |(| j;                  |||j                  j                         | j=                  |j                        r| j?                  |       | jA                  |j                        r| jC                  |       |S )Nr   Fr  Tr  r   r  zUnsupported provider - r  r  r   r   team_idteam_public_model_namedataSources
parameters)r  r3  r  r_  )r6  r  r   )r   rX  )"osr   r   r  r   r  rw   r^  provider_listr(   existsr@  r   r  r  r  r  rv  r  add_patternrv  r7  r9  r  r  r}   r  r   environ'_initialize_deployment_for_pass_throughr  r  r  r  )r?  r6  r  r  is_prompt_management_modelr  r  r  rz  rL  _team_id_team_public_model_namedata_sourcesdata_sourcer   	param_keyenv_names                    r   ru  zRouter._add_deployment0  s    #1177%*"-"/"5"5c":1"="g&W&WW-1*% #F"&"OH (( //55$.$=$=$A$A)4%# $7+@+@@,334GH"9:M9N OPP 	$$Z%>%>%D%DE
 %%))1
E40<,3J,FJ%%) %%))1
E40<,3J,FJ%%) *''' ++%%z'9'9t'9'L $$''44;;J<Q<Q<T<TU((,,Y7","7"7";";<T"U '3..t8886H6J))(3%%h/;;');););)N
 "0044]BGM2' 	EK __\26F0 E	&6)+<+G+G+V%i088KH(*

x(DF9%	E	E *88%$7 //55 9  ***:S:S*T,,
,C
 00%44 1 
 22j2Ir   c                    |j                   j                  du rSddlm} |j                   j                  *t        j                  |j                   j                        }ni }|dk(  r|j                  d      xs |j                   j                  }|j                  d      xs |j                   j                  }|j                  d      xs |j                   j                  }||t        d	      |j                  |||
       nf|j                  d      xs |j                   j                  }	|j                  d      xs |j                   j                  }
|j                  ||	|
       	 y)a*  
        Optional: Initialize deployment for pass-through endpoints if `deployment.litellm_params.use_in_pass_through` is True

        Each provider uses diff .env vars for pass-through endpoints, this helper uses the deployment credentials to set the .env vars for pass-through endpoints
        Tr   )passthrough_endpoint_routerN	vertex_aivertex_projectvertex_locationvertex_credentialsz]vertex_project, and vertex_location must be set in litellm_params for pass-through endpoints.)
project_idlocationr  rL  r;  )r  rL  r;  )r   use_in_pass_through>litellm.proxy.pass_through_endpoints.llm_passthrough_endpointsr  rP  r$   get_credential_valuesr^  r  r  r  rh  add_vertex_credentialsrL  r;  set_pass_through_credentials)r?  r6  r  r   r  credential_valuesr  r  r  rL  r;  s              r   r  z.Router._initialize_deployment_for_pass_through  s    $$88D@ ((@@L$6$L$L--EE%! %'!"k1%))*:; @!00?? 
 &))*;< A!00@@  
 &))*>? D!00CC #
 ")_-D$w  ,BB-,'9 C  &))*5 :!0099 
 &)))4 9!0088  ,HH(;%# I 
 r   c                 ^   |j                   j                  }|r| j                  |      ry|j                  d      }| j	                  |       |j                   j                  }|z|j                   j                  d      }t        j                  j                         D ]%  }|j                  j                  |      }|!|||<   ' t        j                  ||i       | j                  ||j                   j                         | j                  j                  |j                          |S )z
        Parameters:
        - deployment: Deployment - the deployment to be added to the Router

        Returns:
        - The added deployment
        - OR None (if deployment already exists)
        NTr   rX  rj  rn  )r7  r9  rp  rv  ru  r7  rk   rp  r  r   r^  r   rq  rw  r   addrv  )r?  r6  _deployment_model_id_deploymentr?  _model_info_dictrx  field_values           r   add_deploymentzRouter.add_deployment  s'     *4477D$5$56J$K !((d(;
3 )),,	 %/%:%:%E%ESW%E%X3@@EEG :(77;;EB*.9$U+: ""y:J.KL 	--
(=(=(@(@ 	. 	
 	Z223r   r  removal_idxc                    | j                   j                         D ]  \  }}||kD  s|dz
  | j                   |<    || j                   v r| j                   |= t        | j                  j                               D ]j  \  }}g }|D ]3  }||k(  r	||kD  r|j	                  |dz
         #|j	                  |       5 t        |      dkD  r|| j                  |<   ^| j                  |= l t        | j                  j                               D ]j  \  }}g }|D ]3  }||k(  r	||kD  r|j	                  |dz
         #|j	                  |       5 t        |      dkD  r|| j                  |<   ^| j                  |= l y)a"  
        Helper method to update deployment indices after a deployment has been removed from model_list.

        Parameters:
        - model_id: str - the id of the deployment that was removed
        - removal_idx: int - the index where the deployment was removed from model_list
        r|   r   N)r	  r  r%  r
  r  r$  r  )	r?  r  r  r  rm  rv  indicesupdated_indicesr3  s	            r   (_update_deployment_indices_after_removalz/Router._update_deployment_indices_after_removal  s    #'"G"G"M"M"O 	OM3[ GJQw55mD	O t<<<55h? $((M(M(S(S(U#V 	FJ O 	0+%;&#**373 $**3/	0 ?#a'DS55jA99*E%	F* !!F!F!L!L!NO 	?LC O 	0+%;&#**373 $**3/	0 ?#a'=L55c:99#>%	?r   rm  c                 B   |j                  d      xs i j                  d      }|j                  d      xs i j                  d      }|rU|rR||f}|| j                  vrg | j                  |<   || j                  |   vr| j                  |   j                  |       yyyy)z
        Helper to update team_model_to_deployment_indices for a single deployment.

        Parameters:
        - model: dict - the deployment to index
        - idx: int - the index in model_list
        r7  r  r  N)r^  r  r  )r?  r   rm  r  r  r3  s         r   _update_team_model_indexzRouter._update_team_model_indexR  s     99\*0b55i@"'))L"9"?R!D!D$"
 -23C$???=?55c:$??DD55c:AA#F E	 .7r   c                    t        | j                        }| j                  j                  |       | j                          | j	                          ||| j
                  |<   n6|j                  di       j                  d      || j
                  |d   d   <   |j                  d      }|r;|| j                  vrg | j                  |<   | j                  |   j                  |       | j                  ||       y)a!  
        Helper method to add a model to the model_list and update both indices.

        Parameters:
        - model: dict - the model to add to the list
        - model_id: Optional[str] - the model ID to use for indexing. If None, will try to get from model["model_info"]["id"]
        Nr7  r9  rv  )	r$  r   r  r  r  r	  r^  r
  r  )r?  r   r  rm  rv  s        r   rw  z'Router._add_model_to_list_and_index_mape  s     $//"u%//1,,. >AD11(;YY|R(,,T2>ORD11%2Ed2KL YY|,
!F!FFDF55jA11*=DDSI 	%%eS1r   c                 b   	 |j                   j                  xs d}| j                  |      }||j                  |j                  k(  r|j                   |j                   k(  ryd}|j                   j                  }| j                  }||v rU||   }|N| j
                  j                  |       | j                          | j                          | j                  ||       | j                  |       |S # t        $ r1}| j                  rt        j                  d| d       Y d}~y|d}~ww xY w)z
        Add or update deployment
        Parameters:
        - deployment: Deployment - the deployment to be added to the Router

        Returns:
        - The added/updated deployment
        r_  r  Nr  r  rX  zError upserting deployment: ro  )r7  r9  r  r   r	  r   ri  r  r  r  r  r@  r   r   rA  )r?  r6  r  _deployment_on_routerr  r  deployment_fast_mappingrD  s           r   r=  zRouter.upsert_deployment  sH   +	#-#8#8#;#;#Ar :>:M:M- ;N ;! %0 --1F1U1UU"--1F1Q1QQ   .2 * 5 5 8 8*.*O*O' $;;"9-"HK".++K8??A<<>EE%2 F 
 :6 	..%++21#5fg 	s%   A C4 #BC4 4	D.=%D)'D))D.r9  c                    d}|| j                   v r| j                   |   }	 |P| j                  j                  |      }| j                          | j	                          | j                  ||       |S y# t        $ r Y yw xY w)z
        Parameters:
        - id: str - the id of the deployment to be deleted

        Returns:
        - The deleted deployment
        - OR None (if deleted deployment not found)
        Nr  )r	  r   ri  r  r  r  r@  )r?  r9  deployment_idxr  s       r   delete_deploymentzRouter.delete_deployment  s     666!BB2FN	)**>:779446==^ >   		s   AA4 4	B ?B c                     || j                   v rn| j                   |   }| j                  |   }t        |t              rt	        di |S t        |t              r|S t        dj                  t        |                  y)zl
        Returns -> Deployment or None

        Raise Exception -> if model found in invalid format
        zModel invalid format - {}Nr   )r	  r   r   r4  r[   r@  r6  r   )r?  r  rm  r   s       r   r  zRouter.get_deployment  su     t<<<77ACOOC(E%&!*E**E:. ; B B4; OPPr   c                     | j                  |      }|yt        di |j                  j                  d      j                  d      S )zE
        Returns -> dict of credentials for a given model id
        r  NTr   r   )r  rY   r   r7  )r?  r  r6  s      r   get_deployment_credentialsz!Router.get_deployment_credentials  sU     ((((;
& 
''222E

*$*
'	(r   model_group_namec                    || j                   v rs| j                   |   }|rb| j                  |d      }t        |t              rt	        di |S t        |t              r|S t        dj                  t        |                  y)z
        Returns -> Deployment or None

        Raise Exception -> if model found in invalid format

        Optimized with O(1) index lookup instead of O(n) linear scan.
        r   zModel Name invalid - {}Nr   )r
  r   r   r4  r[   r@  r6  r   )r?  r  r  r   s       r   "get_deployment_by_model_group_namez)Router.get_deployment_by_model_group_name  s}     tDDD;;<LMG
3eT*%...z2 L#$=$D$DT%[$QRRr   c                    | j                  |      }|| j                  |      }|T| j                  j                  |      xs g }|r3|d   }t	        |t
              rt        di |}nt	        |t              r|}|yt        di |j                  j                  d      j                  d      }|j                  j                  {t        j                  |j                  j                        }|s-t        j                  d|j                  j                   d       |j                  |       |j!                  d	d       |j                  j"                  r|j                  j"                  |d
<   |S d|j                  j$                  v r-|j                  j$                  j'                  d      d   |d
<   |S d|d
<   |S )a  
        Get API credentials and provider info from a model name in model_list.
        Useful for passthrough endpoints (files, batches, etc.) that need credentials.

        This method tries to find a deployment by model_id first, and if not found,
        it tries to find by model_group_name (model_name).

        Args:
            model_id: Model ID or model name from model_list (e.g., "gpt-4o-litellm")

        Returns:
            Dictionary containing api_key, api_base, custom_llm_provider, etc.
            Returns None if model not found.

        Example:
            credentials = router.get_deployment_credentials_with_provider("gpt-4o-litellm")
            # Returns: {"api_key": "sk-...", "custom_llm_provider": "openai", ...}
        r  N)r  r   Tr   zCredential 'z' not found in credential_listrP  r  r  r  r   )r  r  r  router   r4  r[   rY   r   r7  rP  r$   r  r   r   r   ri  r  r   r  )r?  r  r6  potential_wildcard_modelsdeployment_dictcredentialsr  s          r   (get_deployment_credentials_with_providerz/Router.get_deployment_credentials_with_provider  s   , ((((;
 @@!) A J
 (,(;(;(A(A((K(Qr%(";A">ot4!+!>o!>J<!0J . 
''222E

*$*
' 	
 $$<<H 2 H H))AA! %%--":#<#<#T#T"UUst 01OO5t< $$88 ))== %  J--3331;1J1J1P1P1V1V22K-.  2:K-.r   r[   r  c                      y r   r   r?  r6  r  r9  s       r   r  zRouter.get_router_model_infoY  s     	r   c                      y r   r   r  s       r   r  zRouter.get_router_model_infob  s     	r   c           	         || j                  |      }||}|t        d      |j                  d      xs i j                  dd      }|%|j                  d      xs i j                  dd      }|}|j                  d      }t        |t              r|}ngt        |t
              rd|v rt	        di |}nGt        dt        |      j                   d	|j                  d      xs i j                  d
d             t        j                  |j                  |      \  }	}
}}|
dk(  r|t        j                  d|	 d       n|
dk7  r|	}d|v r| j                  j                  |      }|w|D ]r  }	 |j                  d      xs i j                  d
      |j                  d      xs i j                  d
      k(  r&|j                  d      xs i j                  d      } nt ||	}|j!                  dj#                  |
            sdj#                  |
|      }n|}t        j$                  |      }|j                  d      xs i }||j'                  t)        t*        |             |S # t        $ r Y w xY w)a  
        For a given model id, return the model info (max tokens, input cost, output cost, etc.).

        Augment litellm info with additional params set in `model_info`.

        For azure models, ignore the `model:`. Only set max tokens, cost values if base_model is set.

        Returns
        - ModelInfo - If found -> typed dict with max tokens, input cost, etc.

        Raises:
        - ValueError -> If model is not mapped yet
        Nr  zDeployment not foundr7  
base_modelr   r   z.Deployment missing valid litellm_params. Got: z, deployment_id: r9  unknownr   r   azurez Could not identify azure model 'z'. Set azure 'base_model' for accurate max tokens, cost tracking, etc.- https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-modelsr   z{}/z{}/{}r)  r   )r  rh  r^  r   r^   r4  r   r   r   rw   r   r   r  r  r  r@  r  r6  get_model_infor   r   rn   )r?  r6  r  r9  r  r  r   litellm_params_datar   r  r  rx  potential_modelspotential_modelmodel_info_namer7  user_model_infos                    r   r  zRouter.get_router_model_infoh  s   & >--r-:K&(
344 !nn\28b==lDQ
$..)9:@bEEdJ  )nn-=>)>:0N+T2wBU7U+B.ABN01::; <"#->>,#?#E2"J"J4QZ"[!\^ 
 -4,D,D &&)-
)#Q ')j.@!''26(  ;Z  [ !G+Ee|#'#6#6#<#<=P#Q #/+; 
!	! / 3 3L A GRLLTR *| < B!c$i ( %4$7$78H$I$OR"%#g, !& !& (
! = E-@ AB%nn-@%HO#O++/B
 %..6<"!d9o>?-  ) ! !s   4A-I,,	I98I9c                 \    || j                   v r| j                   |   }| j                  |   S y)a%  
        For a given model id, return the model info

        Returns
        - dict: the model in list with 'model_name', 'litellm_params', Optional['model_info']
        - None: could not find deployment in list

        Optimized with O(1) index lookup instead of O(n) linear scan.
        N)r	  r   )r?  r9  rm  s      r   r  zRouter.get_model_info  s4     66677;C??3''r   c                 Z    | j                  |      }|y|d   }| j                  |      S )zT
        Return list of all models in the same model group as that model id
        )r9  Nrv  r  )r  r  )r?  r9  r7  rv  s       r   get_model_groupzRouter.get_model_group  s>    
 ((B(/
-
""j"99r   c           	         ddl m} d}d}d}	 t        j                  j	                  |      }	 t        j                  |      }	 |I|j	                  dd      }|5t        j                  |      }||xs i } |t        t        |      |      }|7|5t        t         |t        t        |      j                         |            }|S ||}|S # t
        $ r Y w xY w# t
        $ r Y w xY w# t
        $ r Y hw xY w)z
        For a given model id, return the model info

        1. Check if model_id is in model info
        2. If not, check if litellm model name is in model info
        3. If not, return None
        r   )_update_dictionaryNr)  r  )litellm.utilsr  r   rk  r^  r@  r  r   r4  rn   r<  )	r?  r  rv  r  r7  custom_model_infolitellm_model_name_model_infor  base_model_infos	            r   get_deployment_model_infoz Router.get_deployment_model_info  s6    	5*.
,0=A%	 ' 2 2 6 6x @	,3,B,B,T)
	 ,.22<F
)&-&<&<:&NO&2,=,C),> 7--) (-J-V"<=BBD%J  +66JI  		
  		"  		s5   C C AC- 	CC	C*)C*-	C98C9user_facing_model_group_namec                 	   d}d}d}d}| j                  |      }|y|D ]'  }d}	d|v r|d   |k(  rd}	n!d|v r| j                  j                  |      d}	|	s9t        di |d   }
|
j                  }|j                  di       }|j                  di       }d}||j                  dd      }||j                  dd      }||j                  dd      }d}||j                  d	d      }||j                  d	d      }||j                  d	d      }	 |j                  d
d      }|| j                  ||
j                        }nd}d\  }}	 t        j                  |
j                  |
j                        \  }}}}|Tt        j$                  ||      }|g }|j                  di       }|j                  dd      }t'        |ddddd|||d
      }|t)        di ||gd|}nG||j*                  vr|j*                  j-                  |       |j                  dd      -|d   (|j.                  |d   |j.                  kD  r
|d   |_        |j                  dd      -|d   (|j0                  |d   |j0                  kD  r
|d   |_        |j                  dd      (|j2                  |d   |j2                  kD  r
|d   |_        |j                  dd      (|j4                  |d   |j4                  kD  r
|d   |_        |j                  dd      	 |d   du rd|_        |j                  dd      |d   du rd|_        |j                  dd      |d   du rd|_        |j                  dd      |d   du rd|_        |j                  dd      |d   du rd|_        |j                  dd      |d   du rd|_         |j                  dd      |d   
|d   |_!        |j                  dd      ||j                  d      }|j                  d	d      ||j                  d	      }|	|d}||z  }||d}||z  }* ||||_"        |||_#        |||_        |S # t        $ r d}Y 0w xY w# t        j                  j                  $ r8}t        j                  dj!                  t#        |                   Y d}~Nd}~ww xY w) z
        For a given model group name, return the combined model info

        Returns:
        - ModelGroupInfo if able to construct a model group
        - None if error constructing model group info
        Nr  Frv  Tr   r7  r  r  r9  )r  rv  )r_  r_  r  z.litellm.router.py::get_model_group_info() - {}moder  r   )
r3  
max_tokensmax_input_tokensmax_output_tokensinput_cost_per_tokenoutput_cost_per_tokenlitellm_providerr  supported_openai_paramssupports_system_messages)r  	providersr  r  r	  r
  "supports_parallel_function_callingsupports_visionsupports_function_callingsupports_web_searchsupports_url_contextsupports_reasoningr  r   )$r  r  r  r^   #configurable_clientside_auth_paramsr^  r  r   r@  r   rw   r  
exceptionsBadRequestErrorr   r  r6  r   get_supported_openai_paramsModelMapInfor`   r  r  r  r  r	  r
  r  r  r  r  r  r  r  r  r  )r?  r  r  model_group_info	total_tpm	total_rpmr  r   r   is_matchr   model_litellm_paramsmodel_info_dict_deployment_tpm_deployment_rpmr  r7  r  rb  rx  rD  r  db_model_infor  s                           r   _set_model_group_infozRouter._set_model_group_info  sC    6:#'	#'	SW+((K(@
 E	-EH%%*=*L%''--k:F+Fe4D.EFN BB 0
 $)99-=r#B #iib9O .2O&"'))E4"8&"6":":5$"G&"1"5"5eT"B .2O&"'))E4"8&"6":":5$"G&"1"5"5eT"B	"*..tT:'!%!?!?!)n6J6J "@ "J "&J
 +1'M<4;4L4L(..(6(J(J51|Q !*1*M*M'\+' +2.0+ !&		, ;$((8)##%)&*)**+%1,C-1
  '#1 $'C&2^ %$   '7'A'AA$..55lCNN#5t<H"#56B(99A%&89*;;< 9CCU8V$5NN#6=I"#67C(::B%&9:*<<= :DDW9X$6>>"8$?K$99A!"89&;;< =G.=$9 >>"94@L$::B!"9:&<<= >H/>$: NN#GN "#GHDPJN$GNN#4d;G"#45=7;$4NN#>EQ"#>?4GAE$>NN#8$?K"#89TA;?$8NN#94@L"#9:dB<@$9 NN#7>J"#78D@:>$7NN#<dCO"#<=I?I1@$< >>%.:?V&0nnU&;O>>%.:?V&0nnU&;O*$ !I_,	*$ !I_,	KE	-L '$'0 $$'0 $ 3>7 !D  A  "!
" %%55 %++DKKCPQFS s*   4Q*0Q<*Q98Q9<S-SSc                     || j                   v rT| j                   |   }t        |t              r|}nt        |t              r|d   du ry|d   }ny| j	                  ||      S | j	                  ||      S )z
        For a given model group name, return the combined model info

        Returns:
        - ModelGroupInfo if able to construct a model group
        - None if error constructing model group info or hidden model group
        hiddenTNr   )r  r  )r   r   r   r4  r#  )r?  r  r  _router_model_groups       r   get_model_group_infozRouter.get_model_group_info  s     $000))+6D$$&*#D$'>T)*.w-'--/-8 .   ))#+ * 
 	
r   c                   K   t               }|j                  d      }g }g }| j                  |      }|y|D ]  }|j                  di       j                  d      }|d   j                  d      }	||	=|j	                  t
        j                  j                  j                  ||	|	             |j	                  t
        j                  j                  j                  ||	|	              ||z   }
| j                  j                  |

       d{   }|y|dt        |       }|t        |      d }d}|!|D ]  }t        |t              s|d}||z  } d}|!|D ]  }t        |t              s|d}||z  } ||fS 7 qw)z
        Returns current tpm/rpm usage for model group

        Parameters:
        - model_group: str - the received model name from the user (can be a wildcard route).

        Returns:
        - usage: Tuple[tpm, rpm]
        r	  r  Nr  r7  r9  r   r   )r9  r   r  )r  r   )rz   r  r  r^  r  rc   r  rg  r6  r  r   async_batch_get_cacher$  r   r  )r?  r  r  r  tpm_keysrpm_keysr   r   r9  r  combined_tpm_rpm_keyscombined_tpm_rpm_valuestpm_usage_listrpm_usage_list	tpm_usager  	rpm_usages                    r   get_model_group_usagezRouter.get_model_group_usage%  s     
 ! ((K(@
 	E %		, ; ? ? EB+01A+B+F+F,M z]2OO##))00'#1 1  OO##))00'#1 1 	* !)8 3(,

(H(H& )I )
 #
 #*)@3x=)Q)@X)Q $(	%# #a% ($%	NI	# $(	%# #a% ($%	NI	#
 )##3#
s   DFF<F$F5F)maxsizec                 $    | j                  |      S )z
        Cached version of get_model_group_info, uses @lru_cache wrapper

        This is a speed optimization, since set_response_headers makes a call to get_model_group_info on every request
        )r'  )r?  r  s     r   _cached_get_model_group_infoz#Router._cached_get_model_group_infon  s     ((55r   c                 <  K   | j                  |      }||j                  |j                  }nd }||j                  |j                  }nd }||i S | j                  |       d {   \  }}i }|||xs dz
  |d<   ||d<   |||xs dz
  |d<   ||d<   |S 7 1w)Nr   x-ratelimit-remaining-tokenszx-ratelimit-limit-tokensx-ratelimit-remaining-requestszx-ratelimit-limit-requests)r5  r  r  r2  )r?  r  r  	tpm_limit	rpm_limitcurrent_tpmcurrent_rpmreturned_dicts           r   get_remaining_model_group_usagez&Router.get_remaining_model_group_usagey  s     <<[I',<,@,@,L(,,II',<,@,@,L(,,II!2I)-)C)CK)P#P [ <E q=M89 9BM45 >G q?M:; ;DM67 $Qs   A&B(B)2Bc                 ~  K   t        |t              rt        |d      rt        |j                  t              r|j                  j                  di        ||j                  d   d<   |j                  d   }d|vr?d|vr;|9| j                  |       d{   }|j                         D ]  \  }}|	|||<    |S 7 &w)a  
        Add the most accurate rate limit headers for a given model response.

        ## TODO: add model group rate limit headers
        # - if healthy_deployments > 1, return model group rate limit headers
        # - else return the model's rate limit headers
        r  r  zx-litellm-model-groupr7  r8  N)r   r   r  r  r4  r   r>  r  )r?  rN  r  r  remaining_usageheaderrg  s          r   r  zRouter.set_response_headers  s      x+"23822D9##../CRH  ##$89' "*!8!89M!N /6HH4<NN+(,(L(L) # &5%:%:%< ;MFE(5:*62; #s   BB=B;B=2
B=c                 R   | j                   j                          | j                  j                          t        |      D ]e  \  }}|j	                  d      }|r;|| j                   vrg | j                   |<   | j                   |   j                  |       | j                  ||       g y)z
        Build model_name -> deployment indices mapping for O(1) lookups.

        This index allows us to find all deployments for a given model_name in O(1) time
        instead of O(n) linear scan through the entire model_list.
        rv  N)r
  clearr  rt  r^  r  r  )r?  r   rm  r   rv  s        r   _build_model_name_indexzRouter._build_model_name_index  s     	--335--335#J/ 	6JC<0JT%J%JJHJD99*E55jAHHM))%5	6r   c                 v   g | _         | j                          | j                          t        |      D ]  \  }}|j	                  di       }|j	                  d      }|G|j	                  dd      }|j	                  di       }| j                  ||      }d|vri |d<   ||d   d<   | j                  ||        y)z
        Build model index from model list to enable O(1) lookups immediately.
        This is called during initialization to avoid the race condition where
        requests arrive before model_id_to_deployment_index_map is populated.
        r7  r9  Nrv  r_  r   rn  )r   r  r  rt  r^  r<  rw  )r?  r   rx  r   r7  r  rv  r   s           r   '_build_model_id_to_deployment_index_mapz.Router._build_model_id_to_deployment_index_map  s     //1,,.!*- 	RHAu<4J!~~d+H "YY|R8
!&+;R!@22:~Nu,*,E,',4l#D)111Q	Rr   exclude_team_modelsc                    g }|p|| j                   v r`| j                   |   }|D ]L  }| j                  |   }d|v sd|d   v s|r|d   j                  d      r6|j                  |d   d          N |S | j                  j                         D ]U  }| j                  |   }| j                  |   }d|v s&d|d   v s.|r|d   j                  d      rE|j                  |       W |S )z
        if 'model_name' is none, returns all.

        Returns list of model id's.

        Optimized with O(1) or O(k) index lookup when model_name provided,
        instead of O(n) linear scan.
        r7  r9  r  )r
  r   r^  r  r	  r  )r?  rv  rG  idsr  rm  r   r  s           r   get_model_idszRouter.get_model_ids  s    !TBBB??
K" >C OOC0E#u,|9L1L.53F3J3J93U$

5#6t#<=>" 
 !AAFFH );;HE,5(TU<5H-H*u\/B/F/Fy/Q JJx() 
r   candidate_idc                     || j                   v S )a  
        O(1) membership check for a deployment ID without allocating large lists.

        Note: Call sites may pass a variable named `model` when it actually
        contains a deployment ID. This helper expects the deployment ID string.

        Uses the existing `model_id_to_deployment_index_map` which is kept
        in sync by `_build_model_id_to_deployment_index_map` and model-list
        mutation helpers.
        )r	  )r?  rK  s     r   rp  zRouter.has_model_id   s     tDDDDr   c                 p   |sy|| j                   v s| j                  |      r|S | j                  d      }|sy|D ]x  }|j                  di       }|j                  d      }||k(  xs2 |xr |j	                  d|       xs |xr |j	                  d|       }|sb|j                  d      }|sv|c S  y)a  
        Resolve model_name from model_id.

        This method attempts to find the correct model_name to use with the router
        so that litellm_params can be automatically injected from the model config.

        Strategy:
        1. First, check if model_id directly matches a model_name or deployment ID
        2. If not, search through router's model_list to find a match by litellm_params.model
        3. Return the model_name if found, None otherwise

        Args:
            model_id: The model_id extracted from decoded video_id
                     (could be model_name or litellm_params.model value)

        Returns:
            model_name if found, None otherwise. If None, the request will fall through
            to normal flow using environment variables.
        Nr  r   r   r  :rv  )r   rp  r  r^  endswith)r?  r  
all_modelsr6  r   actual_modelmatchesrv  s           r    resolve_model_name_from_model_idz'Router.resolve_model_name_from_model_id   s    ,  t'''4+<+<X+FO ((D(9
$ 	&J'^^,<bAN)--g6L
 ( L J\%:%:Qxj>%JL J\%:%:Qxj>%J  '^^L9
%%	&$ r   team_model_namer  c                     | j                  ||      }|sy|D ]*  }|j                  di       j                  d      |k(  s(|c S  y)a  
        Check if team_model_name resolves to team-specific deployments.

        Returns the public model name (unchanged) so the router can find all
        sibling deployments via team_id filtering, instead of collapsing to a
        single internal model_name.

        Returns:
        - str: the team_model_name if team deployments exist for this team
        - None: if no team-specific model is found
        rv  r  Nr7  r  )r  r^  )r?  rT  r  rd  r   s        r   map_team_modelzRouter.map_team_modelR   sW     $$$Q 	'Eyyr*..y9WD&&	' r   c                    |O|j                  d      xs i j                  d      |k(  r(||j                  d      xs i j                  d      k(  ry|6|d   |k(  r.|j                  d      xs i j                  d      }||||k(  ryy)zU
        Get the team-specific model name if team_id matches the deployment.
        r7  r  r  Trv  Fr  )r?  rv  r   r  model_team_ids        r   should_include_deploymentz Router.should_include_deploymenti   s     <(.B33I>'I		,'-2223KLM #l(;z(I #YY|4:??	JM ( G+r   model_aliasc                    g }|||f}|| j                   v rx| j                   |   }|D ]`  }| j                  |   }| j                  |||      s'|'|j                         }	||	d<   |j	                  |	       P|j	                  |       b |r|S || j
                  v rv| j
                  |   }|D ]`  }| j                  |   }| j                  |||      s'|'|j                         }	||	d<   |j	                  |	       P|j	                  |       b |S |lt        | j                        D ]T  \  }}| j                  |||      s|'|j                         }	||	d<   |j	                  |	       D|j	                  |       V |S )aM  
        Return all deployments of a model name

        Used for accurate 'get_model_list'.

        if team_id specified, only return team-specific models

        Optimized with O(1) index lookup instead of O(n) linear scan.

        Note: when team_id is provided, O(1) lookup in
        `team_model_to_deployment_indices` only applies when `model_name` is the
        team public model name. If a caller passes an internal deployment model
        name (for example, `model_name_<team_id>_<uuid>`), this method falls back
        to the standard model-name index / scan path.
        )rv  r   r  rv  )r  r   rZ  r<  r  r
  rt  )
r?  rv  r[  r  returned_modelsr3  r  rm  r   alias_models
             r   r  zRouter._get_all_deployments   s   * 68 J'Cd;;;??D" 6C OOC0E99#-UG :  !".&+jjl4?L1'..{;'..u56 #** >>>;;JGG  6,11) 2  #. ',jjl4?L1'..{;'..u56: !   (8 
6
U11) 2  #.&+jjl4?L1'..{;'..u5
6 r   c                    | j                         xs g }g }|D ]m  }|j                  d      }| j                  |      r(| j                  ||      }|s;|j	                  |       M|j	                  |j                  dd             o |S )u  
        Returns all possible model names for the router, including models defined via model_group_alias.

        If a team_id is provided, only deployments configured with that team_id (i.e. team‐specific models)
        will yield their team public name.
        r7  )r6  r  rv  r_  )r  r^  _is_team_specific_model_get_team_specific_modelr  )r?  r  deploymentsr   r6  r7  rT  s          r   r  zRouter.get_model_names   s     ))+1r% 		EJ#5J++J7"&"?"?)7 #@ # #&&7"":>>,#CD		E r   c                 ~    |j                  d      xs i }|y||j                  d      k(  r|j                  d      S y)a  
        Get the team-specific model name if team_id matches the deployment.

        Args:
            deployment: DeploymentTypedDict - The model deployment
            team_id: Optional[str] - If passed, will return router models set with a `team_id` matching the passed `team_id`.

        Returns:
            str: The `team_public_model_name` if team_id matches
            None: If team_id doesn't match or no team info exists
        r7  Nr  r  r  )r?  r6  r  r7  s       r   ra  zRouter._get_team_specific_model   sD     &0^^L%A%GR
jnnY//>>":;;r   r7  c                 >    t        |xr |j                  d            S )z
        Check if model info contains team-specific configuration.

        Args:
            model_info: Model information dictionary

        Returns:
            bool: True if model has team-specific configuration
        r  )r  r^  )r?  r7  s     r   r`  zRouter._is_team_specific_model   s     J<:>>)#<==r   c                 l   g }|#|| j                   vr|S || j                   |   fg}n#t        | j                   j                               }|D ]d  \  }}t        |t              r|}n*t        |t
              rt        di |}|d   du r<|d   }nC|j                  | j                  ||             f |S )z
        Helper function to get model list from model alias.

        Used by `.get_model_list` to get model list from model alias.
        r%  Tr   )rv  r[  r   )	r   r%  r  r   r   r4  re   rr  r  )r?  rv  r]  alias_itemsr[  model_value_router_model_name_model_values           r   get_model_list_from_model_aliasz&Router.get_model_list_from_model_alias!  s     68!!7!77&&&(>(>z(JKLKt55;;=>K(3 	$K+s+*5"K.8G;G)T1)5g)>&""))1{ * 	$ r   c                    g }|"|j                  | j                  ||             |j                  | j                  |             t        |      dk(  r| j                  j                  |      xs g }|A|| j                  v r3| j                  |   j                  |      xs g }|j                  |       |*|(|D ]#  }t        di |}||d<   |j                  |       % ||| j                  z  }|S )z
        Includes router model_group_alias'es as well

        if team_id specified, returns matching team-specific models
        rV  r  r   rv  r   )
rr  r  rj  r$  r  r  r  r\   r  r   )r?  rv  r  r]  r  #potential_team_only_wildcard_modelsrB  deployment_typed_dicts           r   r  zRouter.get_model_list1!  s'    68!""))Z)Q 	00J0G	
 1$(,(;(;(A(A*(M(SQS% "w$2K2K'K--g6<<ZHNB 4 *001TU%*C*O2 BA,?,D!,D):D),7#**+@AB
 t.Or   c                 8    | j                   j                          y)zInvalidate the cached model group info.

        Call this whenever self.model_list is modified to ensure the cache is rebuilt.
        N)r5  cache_clearr  s    r   r  z)Router._invalidate_model_group_info_cache[!  s    
 	))557r   c                     d| _         y)zInvalidate the cached access groups.

        Call this whenever self.model_list is modified to ensure the cache is rebuilt.
        N)r  r  s    r   r  z&Router._invalidate_access_groups_cacheb!  s    
 %)!r   model_access_groupc                    |du xr
 |du xr |du }|r| j                   | j                   S ddlm}  |t              }| j	                  ||      }|rq|D ]l  }|j                  d      }	|	s|	j                  dg       xs g D ]=  }
| |
|k(  s|d   }||
   j                  |       %|d   }||
   j                  |       ? n |rt        |      | _         | j                   S |S )a  
        If model_name is provided, only return access groups for that model.

        Parameters:
        - model_name: Optional[str] - the received model name from the user (can be a wildcard route). If set, will only return access groups for that model.
        - model_access_group: Optional[str] - the received model access group from the user. If set, will only return models for that access group.
        - team_id: Optional[str] - the team id, to resolve team-specific models
        Nr   r   rV  r7  r#  rv  )r  collectionsr   r%  r  r^  r  r4  )r?  rv  rq  r  
_use_cacher   r#  r   rB  r+  groups              r   get_model_access_groupszRouter.get_model_access_groupsi!  s     $Q#5#=Q'T/ 	
 $33?,,,+#D)((J(P
 
DeeL1!,"!E!K D-9$(::-.|_
 -e 4 ; ;J G)*<J)%077
CD
D (,](;D%,,,r   c                     | j                  |      }t        |      dk(  ry|j                  |g       }|D ]   }| j                  j	                  |        y y)zG
        Return True if model access group is a wildcard route
        )rq  r   Fr  T)rv  r$  r^  r  r  )r?  rq  r#  rd  r   s        r   )_is_model_access_group_for_wildcard_routez0Router._is_model_access_group_for_wildcard_route!  sr     441 5 
 }"""#5r: 	E""(((7C	
 r   c                     t        |       }i }g d}|D ]K  }||v r||   ||<   |dk(  s| j                  dk(  s%| j                  j                  j	                         ||<   M |S )a  
        Get router settings method, returns a dictionary of the settings and their values.
        For example get the set values for routing_strategy_args, routing_strategy, allowed_fails, cooldown_time, num_retries, timeout, max_retries, retry_after
        )r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   )varsr   rn  rf  r_  )r?  	_all_vars_settings_to_returnvars_to_includevars        r   get_settingszRouter.get_settings!  s    
 J	 
  # 	YCi+4S>#C(..))-DD+/+D+D+Q+Q+V+V+X#C(	Y #"r   c                    g d}g d}| j                         }|D ]  }||v rg||v rt        ||         }t        | ||       '|dk(  r1|d   ||   k7  r&| j                  ||   |j	                  di              t        | |||          nt        j                  dj                  |              t        j                  d| j                                 y)	z-
        Update the router settings.
        )r   r   r   r   r   r   r   r   r   r   r   r   )r   r   r   r   r   r   r   r   zSetting {} is not allowedzUpdated Router settings: N)r  r  r  r"  r^  r   rA  r6  )r?  rJ  _allowed_settings_int_settings_existing_router_settingsr~  _casted_values          r   update_settingszRouter.update_settings!  s    


 %)$5$5$7! 	UC''-'$'s$4MD#}5 1156HIVTW[X22-3C[28** 73 3  D#vc{3%++,G,N,Ns,ST'	U( 	##&?@Q@Q@S?T$UVr   c                 |   |d   d   }t        |      }|dk(  rhdj                  |      }| j                  j                  |d|      }|5t	        j
                  | |       | j                  j                  |d|      }|S |dk(  r]|j                  d	      du r%| d
}| j                  j                  |d|      }|S | d}| j                  j                  |d|      }|S |j                  d	      du r$| d}| j                  j                  ||      }|S | d}| j                  j                  ||      }|S )a  
        Returns the appropriate client based on the given deployment, kwargs, and client_type.

        Parameters:
            deployment (dict): The deployment dictionary containing the clients.
            kwargs (dict): The keyword arguments passed to the function.
            client_type (str): The type of client to return.

        Returns:
            The appropriate client based on the given client_type and kwargs.
        r7  r9  r  z{}_max_parallel_requests_clientTr3  r=  r  )r  r   r]  r  _stream_async_client_async_client_stream_client)r3  r  _client)r!   r6  r   r>  r7    set_max_parallel_requests_clientr^  )r?  r6  rJ  r  r  r  	cache_keyr[  s           r   ro  zRouter._get_client"  s    l+D1+LV+T119@@JIZZ))$AQ * F ~%FF,0
 --!dEU .  MG#zz(#t+'j(<=	--!dEU .  'j6	--!dEU .  zz(#t+'j7	--!4D .  'j0	--!4D .  r   c           	      	   t        j                  d|        t        |      }t               }	 t	        j
                  |      }d}	d}
d}t        |      }t               }|j                  d      }| d| }| j                  j                  |d	|
      xs i }t        |      D ]  \  }}|j!                  di       }|j!                  di       }d}	 |j!                  dd      }||j!                  dd      }| j#                  ||      }|xs |j!                  dd      }t%        |t&              rZ|j!                  dd      Ht%        |d   t(              r5||d   kD  r-|j+                  |       d	}	|
dj                  ||d   |      z  }
|j!                  dd      }| j                  j                  |d	|
      xs d}t%        |t&              r| j.                  dk7  rv|j!                  |d      ||<   t1        |||         }t%        |t&              rB|j!                  dd      0t%        |d   t(              r|d   |k  r|j+                  |       d	}|M|j!                  d      <|j!                  d      }|)t3        t5        d!i ||      s|j+                  |       |t        j6                  du s|xs |}t	        j8                  |t5        d!i |      \  }}}}t	        j:                  ||      }|]t        j<                  j?                  |      } dg}!| jA                         D ]8  \  }"}#|"|vs|"|!v st        j                  d|"        |j+                  |       :  tC        |      tC        |      k(  r<	 |d	u rtE        |      |	d	u r't	        jF                  dj                  |
      |d       tC        |      dkD  r!t        |      D $%cg c]  \  }$}%|$|vr|% }}$}%|S # t        $ r9}t        j                  dj                  t        |                   |cY d}~S d}~ww xY w# t        $ r8}t        j,                  dj                  t        |                   Y d}~d}~ww xY wc c}%}$w )"a  
        Filter out model in model group, if:

        - model context window < message length. For azure openai models, requires 'base_model' is set. - https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models
        - filter models above rpm limits
        - if region given, filter out models not in that region / unknown region
        - [TODO] function call and model doesn't support function calling
        z2Starting Pre-call checks for deployments in model=)rF  zllitellm.router.py::_pre_call_checks: failed to count tokens. Returning initial list of deployments. Got - {}NFr_  r	  z:rpm:Tr  r   r7  r  r  r   r  z%Model={}, Max Input Tokens={}, Got={}zAn error occurs - {}r9  r   r   r  allowed_model_region)r   r  r  r  )passed_paramsresponse_formatz1INVALID MODEL INDEX @ REQUEST KWARG FILTERING, k=r)  z~litellm._pre_call_checks: Context Window exceeded for given call. No models have context window large enough for this call.
{}r`  r   )$r   rA  r%  rM  r   token_counterr@  r  r6  r   r!   rz   r  r   r>  rt  r^  r  r   r4  r  r  r,  r   maxr{   r^   drop_paramsrw   r  r)  get_non_default_paramsr  r$  rg   r  )&r?  r   r  rF  rS  _returned_deploymentsinvalid_model_indicesinput_tokensrD  _context_window_error_potential_error_str_rate_limit_errorr  r  r  r   model_group_cacherm  r6  rh  r+  _deployment_modelr  r7  r  current_request_cache_localcurrent_requestr  _dep_model_for_paramsr  rx  r  r  special_paramsr  r  ir  s&                                         r   _pre_call_checkszRouter._pre_call_checks:"  sQ     	##@H	
 !%%8 9 #	)"00(CL !&!!<^L W-G5 01JJ  ?O !   	 	  ))>? v	;OC(nn-=rBO$..r:K !%W(__\4@
%!0!4!4\4!HJ!77)u 8 
 %/$T/2E2Egt2T! z40"'94@L #:.@#A3G(:6H+II-11#604-,CJJ 1 *+= > ,, ! #tR0H 

$$ TDT %   	 ( ,d3))-EE.?.C.CHa.P!(+"%/1B81L#
 5'++E48D #?5#93?+E2oE-11#6,0)  *"&&'=>J'5'9'9:P'Q$'3,'5'H'H-A .11#6  )g.A.AU.J(9(BU% ,,/#1#DO#D)' +2*M*M/(;+'
 +2 *1)M)M&4 *N *& '8%8N 2 8 8 : ;1$;;^@S177"STUSV W 255c:;av	;p $%-B)CC !D(/  '$.88 ^  e  e,  !#  $%) &&;<%Aq11 %! % %$W  	)!''~  F  FF
 )(	)r  W%//0F0M0McRSf0UVVWX%s<   O7 B8P<"R 7	P9 .P4.P94P9<	Q=-Q88Q=c                 t    || j                   vry| j                   |   }t        |t              r|}|S |d   }|S )z
        Get the model from the alias.

        Returns:
        - str, the litellm model name
        - None, if model is not in model group alias
        Nr   )r   r   r   )r?  r   _items      r   r+  zRouter._get_model_from_alias#  sK     ...&&u-eS!E  'NEr   c                 X    | j                   D cg c]  }|d   d   |k(  s| c}S c c}w )z6
        Get the deployment by litellm model.
        r   r   )r   )r?  r   rB  s      r    _get_deployment_by_litellm_modelz'Router._get_deployment_by_litellm_model#  s.      ??Taa0@.A'.Je.STTTs   ''rQ  c                 4   d}|N|j                  d      xs i }|j                  d      xs i }|j                  d      xs |j                  d      }|du r|| j                  |      fS | j                  |      rM| j                  |      }	|	*|	j                  j
                  }
|
|	j                  d      fS t        d	| d
      | j                  |      }||}|| j                  vr|| j                  ||      }|r||fS | j                  j                  |      }|r||fS |3|| j                  v r%| j                  |   j                  |      }|r||fS | j                  F| j                  j                         }| j                  d   j                         |d<   ||d   d<   ||fS | j                  |      }t!        |      dk(  r| j                  |      }t#        j$                  t&        j(                        rt#        j*                  d|        t!        |      dk(  r| j-                         rB| j/                         }|r0t#        j0                  d| d| d       |}| j                  |      }t!        |      dk(  rU| j3                  |      d| dj5                  |      }nd| dj5                  |      }t7        j8                  ||d      t6        j:                  r%|t6        j:                  v rt6        j:                  |   }||fS )aF  
        Common checks for 'get_available_deployment' across sync + async call.

        If 'healthy_deployments' returned is None, this means the user chose a specific deployment

        Returns
        - str, the litellm model name
        - List, if multiple models chosen
        - Dict, if specific model chosen
        Nr   r  user_api_key_team_idTr)  r  r   zBLiteLLM Router: Trying to call specific deployment, but Model ID :z does not exist in Model ID maprV  r   r   r  r   zinitial list of deployments: zModel 'z7' not found. Attempting to use default fallback model 'z'.zYou passed in model=z+. There is no 'model_name' with this stringz1. There are no healthy deployments for this modelr_  r`  )r^  r  rp  r  r   r   r7  rh  r+  r   r  r  get_deployments_by_patternr  r  r<  r$  r   r  r   r   rA  rC  r  r5  r  r6  r   r  model_alias_map)r?  r   rF  r  rQ  rS  request_team_idr   r  r6  deployment_model_model_from_aliasteam_deploymentspattern_deploymentsupdated_deploymentr  fallback_modelra  s                     r   rI  z*Router._common_checks_available_deployment#  s   & *.%%))*5;H-112DEK&ll& >!%%&<=  $&$??e?LLLu%,,e,<J%#-#<#<#B#B ')>)>D)>)QQQTUZT[[z{  !66U6C(%E((( *#'#<#<$o $= $  $ "222 #'"5"5"P"P #Q # #111  +#t'@'@@&*&?&?#',, -  $
 ' "555 &&2%)%<%<%A%A%C"7;7N7N$8$& ##34 AF"#34W=000 #7757I"#q("&"G"Ge"G"T --gmm<!''/0C/DE "#q(**,!%!A!A!C!)..!%(_`n_ooqr +E*.*C*Cu*C*U' &'1,&&%&8@ 4UG;fgnnG !5UG;lmttG --#!#  ""u0G0G'G++E )))r   c           	        K   | j                  |||||      \  }}t        ||      }t        j                  t        j
                        rt        j                  d|        t        ||      }t        j                  t        j
                        rt        j                  d|        t        |t              r|S | j                  ||       d{   }t        | |       d{   }t        j                  t        j
                        rt        j                  d|        | j                  ||	      }| j                  |||t        t        t            |      nd||
       d{   }| j"                  r,|*| j%                  |t        t        t&           |      ||      }t)        | |||| j+                  |             d{   }|xs i j-                  dd      }	t.        j0                  j3                  t        t        t&           |      |	      }t5        |      dk(  rt7        | ||       d{   }
|
|S 7 r7 `7 7 7 w)z
        Get the healthy deployments for a model.

        Returns:
        - List[Dict], if multiple models chosen
        *OR*
        - Dict, if specific model chosen
        r   rF  r  rQ  rS  )r  rS  z'healthy_deployments after team filter: z-healthy_deployments after web search filter: r  r  Nr  cooldown deployments: r  cooldown_deploymentsrY  r   r  rF  rS  )r  r   rS  r  r&  r  target_orderr   r  r   r  )rI  r:   r   r  r   r   rA  r;   r   r4  0_async_filter_health_check_unhealthy_deploymentsr>   _filter_cooldown_deploymentsr\  r   r   rP   r   r  r
   r1   r3  ri  r   r)  _get_order_filtered_deploymentsr$  rF   )r?  r   rS  rF  r  rQ  r  r  r  r  r,  s              r   rK  z$Router.async_get_healthy_deployments#  s    $ &*%M%M 3) &N &
"" 7 3)

 !--gmm<!''9:M9NO < 3)

 !--gmm<!''?@S?TU )40&& GG$7!1 H   	 &E$(;K&
  
 !--gmm<!''*@AU@V(WX"?? 3!5 @ 

 %)$J$J 3:B:NT*+X6TX)- %K %
 
 &&8+?"&"7"7$(d5H$I!-	 #8 # %< $) 3#'#O#O$%
 
 (-222?DI%mmKKd01 L 
 "#q(A(,!1 I
 O""u 

$
"s]   CIII)I	*BI.I/AIIA/I=I>	I	IIIIc           
        K   | j                   dk7  rR| j                   dk7  rC| j                   dk7  r4| j                   dk7  r%| j                   dk7  r| j                  |||||      S 	 t        |      }| j                  |||||       d{   }||j                  }|j
                  }| j                  ||||||	       d{   }t        |t              r|S |j                  d
      rt        |      dk(  r|d   S t        j                         }	| j                   dk(  r4| j                  (| j                  j                  ||||       d{   }
n| j                   dk(  r4| j                  (| j                  j                  ||||       d{   }
n| j                   dk(  r5| j                  )| j                  j                  |||||       d{   }
n`| j                   dk(  rt!        | ||      S | j                   dk(  r2| j"                  &| j"                  j                  ||       d{   }
nd}
|
t%        | ||       d{   }|t'        j(                  d| d| j+                  |
       d|        t        j                         }||	z
  }t-        j.                  | j0                  j3                  t4        j6                  |d||	|             |
S 7 V7  7 7 W7 7 7 # t8        $ r}t;        j<                         }|j|j                  dd      }|Vt?        j@                  |jB                  ||f      jE                          t-        j.                  |jG                  ||             |d}~ww xY ww)z
        Async implementation of 'get_available_deployments'.

        Allows all cache calls to be made async => 10x perf impact (8rps -> 100 rps).
        r   r   r   r   r   r  r   rS  rF  r  rQ  Nr   rS  rF  r  rQ  r  "_encrypted_content_affinity_pinnedr|   r   r  r  rF  r  r  r  rF  r  rS  r  r  r  r  $get_available_deployment for model: , Selected deployment:  for model: 2<routing_strategy>.async_get_available_deploymentsr  r  r  r  r  r  r  rT  )$r   rh  r!   async_pre_routing_hookr   rF  rK  r   r4  r^  r$  r  rl  async_get_available_deploymentsrp  rn  r0   r   rF   r   r5  rE  r  r  r   r  rj   r  r@  r  r  rj  rk  rW  rm  rV  )r?  r   rS  rF  r  rQ  r  pre_routing_hook_responser  r  r6  r,  r  r  rD  rS  r  s                    r   r  z%Router.async_get_available_deployment$  s2     !!%==%%)99%%)==%%)@@%%500!$7- 1  F	@P /3.I.I-!$7 /J / )% )41774== )-(J(J-!$7!1 )K ) # -t4** ""#GH+,1*1--J%%)AA,,8 22RR$),?!)#	 S    %%)==**6 00PP$),?!)#	 Q    %%)@@--9 33SS$),?!)#'5 T    &&*::%(,(;  %%5))5 //OO$),? P    "
!"E,0%5# 	
  !&&6ug=TUYUjUjkuUvTw  xD  EJ  DK  L yy{H :-I''BB(//&R%5)% C 	 ])#0&0  	"+"6"6"8),001FM*$$*::!45 eg''#99!=PQ G!	s   A"N%$K2 	K
8K2 K"K2 N#K2 <N=AK2 K%AK2 K(AK2 K+!K2 7N8;K2 3K.4K2 K0BK2 NK2 "K2 %K2 (K2 +K2 .K2 0K2 2	N;BM==NNc           
        K   	 t        |      }| j                  |||||       d{   }||j                  }|j                  }| j	                  ||||||       d{   }t        |t              r`|j                  di       }	|	j                  d      r|S t        j                  d|j                  di       j                  d       d	|d
      | j                  |      }
t        |
      dk(  rt        j                  d| d|d
      t        j                         }| j                  dk(  r4| j                  (| j                  j!                  ||
||       d{   }n| j                  dk(  r5| j"                  )| j"                  j!                  ||
|||       d{   }n`| j                  dk(  rt%        | |
|      S | j                  dk(  r2| j&                  &| j&                  j!                  ||
       d{   }nd}|t)        | ||       d{   }|t+        j,                  d| d| j/                  |              t        j                         }||z
  }t1        j2                  | j4                  j7                  t8        j:                  |d|||             |S 7 7 P7 Q7 7 7 # t<        $ r}t?        j@                         }|j|j                  dd      }|VtC        jD                  |jF                  ||f      jI                          t1        j2                  |jK                  ||             |d}~ww xY ww)z
        Async version of get_available_deployment_for_pass_through

        Only returns deployments configured with use_in_pass_through=True
        r  Nr  r   r  Deployment r7  r9  C does not support pass-through endpoint (use_in_pass_through=False)r_  r`  r  r   Model z has no deployments configured with use_in_pass_through=True. Please add use_in_pass_through: true to the deployment configurationr   r  r   r  r   r  r   r  r  z7async_get_available_deployment_for_pass_through model: , selected deployment: r  r  r  rT  )&r!   r  r   rF  rK  r   r4  r^  r   r   _filter_pass_through_deploymentsr$  r  perf_counterr   rl  r  rn  r0   r   rF   r   r5  rE  r  r  r   r  rj   r  r@  r  r  rj  rk  rW  rm  rV  )r?  r   rS  rF  r  rQ  r  r  r  r   pass_through_deploymentsr  r6  r,  r  r  rD  rS  r  s                      r   /async_get_available_deployment_for_pass_throughz6Router.async_get_available_deployment_for_pass_through$  s    E	@P /3.I.I-!$7 /J / )% )41774== )-(J(J-!$7!1 )K ) # -t4!4!8!89I2!N!%%&;<..!11"-.A.E.ElTV.W.[.[\`.a-b  cf  !g#%'  (,'L'L$7 (M ($ +,1--$UG  ,n  o!#  **,J%%)AA,,8 22RR$),D!)#	 S    %%)@@--9 33SS$),D!)#'5 T    &&*::%(,(@  %%5))5 //OO$),D P    "
!"E,0%5# 	
  !&&I%Pghlh}h}  I  iJ  hK  L ((*H :-I''BB(//&R%5)% C 	 i)#P&2  	"+"6"6"8),001FM*$$*::!45 eg''#99!=PQ G	s   M$J> J.8J> !J1"8J> MCJ> #J4$AJ> 'J7(!J> 	M
;J> J:J>  J<!BJ> -M.J> 1J> 4J> 7J> :J> <J> >	MBM		MMc                    K   || j                   v r+| j                   |   j                  |||||       d{   S || j                  v r+| j                  |   j                  |||||       d{   S y7 >7 w)z
        This hook is called before the routing decision is made.

        Used for the litellm auto-router to modify the request before the routing decision is made.
        r  N)r  r  r  )r?  r   rS  rF  r  rQ  s         r   r  zRouter.async_pre_routing_hookH%  s       D%%%**51HH-!$7 I    D+++007NN-!$7 O    )s!   5A:A68A:0A81A:8A:c                    | j                  ||||      \  }}t        |t              r|S t        |      }| j	                  ||      }t        | |      }| j                  ||      }| j                  r|| j                  ||||      }|xs i j                  dd      }	t        j                  j                  ||	      }t        |      d	k(  rU| j                  |
      }
| j                  j!                  |
|      }t        | |      }t#        ||| j                  |      | j$                  dk(  r*| j&                  | j&                  j)                  ||      }n| j$                  dk(  rt+        | ||      S | j$                  dk(  r+| j,                  | j,                  j)                  |||      }nx| j$                  dk(  r,| j.                   | j.                  j)                  ||||      }n=| j$                  dk(  r,| j0                   | j0                  j)                  ||||      }nd}|nt3        j4                  d| d       | j                  |
      }
| j                  j!                  |
|      }t        | |      }t#        ||| j                  |      t3        j4                  d| d| j7                  |       d|        |S )zB
        Returns the deployment based on routing strategy
        r   rF  r  rQ  r  r  r  Nr  r  r  r   r  	model_idsr  r   r   r   cooldown_listr   r  r   r  r   r  r  rS  r   r  r   r  z, No deployment availabler  r  )rI  r   r4  r!   *_filter_health_check_unhealthy_deploymentsr@   r  r   r  ri  r   r)  r  r$  rJ  r  get_min_cooldownrf   r   r   get_available_deploymentsr0   rn  r   rl  r   r5  rE  )r?  r   rF  r  rQ  rS  r  r  r  r  r  _cooldown_time_cooldown_listr6  s                 r   rh  zRouter.get_available_deploymento%  s    &*%M%M 3	 &N &
"" )40&&+L,

 #MM 3- N 

  9$(;K 
 #?? 3!5 @ 
 &&8+?"&"7"7$7!-	 #8 # (-222?DI%mmKKm L 
 "#q(**e*<I!00AA#6F B N 7(,?ON ','+'B'B,	    L0T5J5J5V..HH!7J I J ""&66 "$($7  !!%<<))522LL!$7- M J !!%::%%1..HH!$7!	 I J !!%==((411KK!$7!	 L J J!&&6ug=VW **e*<I!00AA#6F B N 7(,?ON ','+'B'B,	  	""25'9PQUQfQfgqQrPss  AF  @G  H	
 r   c                 Z   | j                  ||||      \  }}t        |t              r`|j                  di       }|j                  d      r|S t	        j
                  d|j                  di       j                  d       d|d	      | j                  |
      }t        |      dk(  rt	        j
                  d| d|d	      t        |      }	| j                  ||	      }t        | |	      }
| j                  ||
      }| j                  r|| j                  ||||      }t        |      dk(  rU| j                  |      }| j                  j!                  ||	      }t        | |	      }t#        ||| j                  |      | j$                  dk(  r*| j&                  | j&                  j)                  ||      }n| j$                  dk(  rt+        | ||      S | j$                  dk(  r+| j,                  | j,                  j)                  |||      }nx| j$                  dk(  r,| j.                   | j.                  j)                  ||||      }n=| j$                  dk(  r,| j0                   | j0                  j)                  ||||      }nd}|nt3        j4                  d| d        | j                  |      }| j                  j!                  ||	      }t        | |	      }t#        ||| j                  |      t3        j4                  d| d!| j7                  |              |S )"a  
        Returns deployments available for pass-through endpoints (based on load balancing strategy)

        Similar to get_available_deployment, but only returns deployments with use_in_pass_through=True

        Args:
            model: Model name
            messages: Optional list of messages
            input: Optional input data
            specific_deployment: Whether to find a specific deployment
            request_kwargs: Optional request parameters

        Returns:
            Dict: Selected deployment configuration

        Raises:
            BadRequestError: If no deployment is configured with use_in_pass_through=True
            RouterRateLimitError: If no pass-through deployments are available
        r  r   r  r  r7  r9  r  r_  r`  r  r   r  z has no deployment configured with use_in_pass_through=True. Please add use_in_pass_through: true in the deployment configurationr  r  r  Nr  r  r  r  r   r  r   r  r   r  r   r  r   z1get_available_deployment_for_pass_through model: z, no available deploymentsr  )rI  r   r4  r^  r   r  r  r$  r!   r  r@   r  r   r  rJ  r  r  rf   r   r   r  r0   rn  r   rl  r   r5  rE  )r?  r   rF  r  rQ  rS  r  r   r  r  r  r  r  r  r6  s                  r   )get_available_deployment_for_pass_throughz0Router.get_available_deployment_for_pass_through%  s   8 &*%M%M 3	 &N &
"" )400445ErJN!!"78** --)*=*A*A,PR*S*W*WX\*])^  _b  c!#  $(#H#H 3 $I $
  '(A-))   (i  j  ,M,
 $(#R#R 8- $S $
   9$(;K 
 $(#D#D 8!5 $E $
  &&8+?'+'<'<$<!-	 (= ($ '(A-**e*<I!00AA#6F B N 7(,?ON ','+'B'B,	    L0T5J5J5V..HH!7O I J ""&66!$($<  !!%<<))522LL!$<- M J !!%::%%1..HH!$<!	 I J !!%==((411KK!$<!	 L J J!&&CE7Jde **e*<I!00AA#6F B N 7(,?ON ','+'B'B,	  	""?wF]^b^s^st~^  ^A  B	
 r   r  c                     t        j                  t        j                        rt        j                  d|        t        |      }|D cg c]  }|d   d   |vr| c}S c c}w )a  
        Filters out the deployments currently cooling down from the list of healthy deployments

        Args:
            healthy_deployments: List of healthy deployments
            cooldown_deployments: List of model_ids cooling down. cooldown_deployments is a list of model_id's cooling down, cooldown_deployments = ["16700539-b3cd-42f4-b426-6a12a1bb706a", "16700539-b3cd-42f4-b426-7899"]

        Returns:
            List of healthy deployments
        r  r7  r9  )r   r  r   r   rA  rM  )r?  r  r  cooldown_setr6  s        r   r  z#Router._filter_cooldown_deployments&  sm     !--gmm<!''*@AU@V(WX/0 2
,'-\A 
 	
 
s   A!c                    K   | j                   s|S | j                  j                  |       d{   }|s|S |D cg c]  }|d   d   |vs| }}|st        j                  d       |S |S 7 =c c}w w)a  
        Filter out deployments marked unhealthy by background health checks.
        No-op when enable_health_check_routing is False.
        Returns all deployments if health state is unavailable, stale, or would
        exclude every candidate (safety net).
        r  Nr7  r9  JAll deployments marked unhealthy by health checks, bypassing health filter)r   r  "async_get_unhealthy_deployment_idsr   r   r?  r  r  unhealthy_idsr  filtereds         r   r  z7Router._async_filter_health_check_unhealthy_deployments&  s      //&& ))LL!1 M   	
 && +
aod.C=.XA
 
 !))\ '&#
s&   .A5A.A5A0A0A50A5c                     | j                   s|S | j                  j                  |      }|s|S |D cg c]  }|d   d   |vs| }}|st        j                  d       |S |S c c}w )zASync version of _async_filter_health_check_unhealthy_deployments.r  r7  r9  r  )r   r  get_unhealthy_deployment_idsr   r   r  s         r   r  z1Router._filter_health_check_unhealthy_deployments&  s     //&&//LL- M 
 && +
aod.C=.XA
 
 !))\ '&
s   A$A$c                     t        j                  dt        |       d       |D cg c]&  }|j                  di       j                  dd      r|( }}t        j                  dt        |       d       |S c c}w )a  
        Filter out deployments configured with use_in_pass_through=True

        Args:
            healthy_deployments: List of healthy deployments

        Returns:
            List[Dict]: Only includes a list of deployments that support pass-through
        z%Filter pass-through deployments from z healthy deploymentsr   r  FzFound z& deployments with pass-through enabled)r   rA  r$  r^  )r?  r  r6  r  s       r   r  z'Router._filter_pass_through_deployments&  s     	##3C8K4L3MMab	
 2$
~~.3778MuU $
  $
 	##S1233YZ	
 ('$
s   +A8c                     	 |j                  di       j                  dd      }||| j                  ||       yyy# t        $ r+}t        j                  dt        |              Y d}~yd}~ww xY w)z7
        Tracks successful requests rpm usage.
        r7  r9  Nz$Error in _track_deployment_metrics: )r^  rA  r@  r   r  r   )r?  r6  r  rN  r  rD  s         r   r  z Router._track_deployment_metrics'  s    		Y!~~lB7;;D$GH'&& "2 (    	Y!''*NsSTvh(WXX	Ys   8= 	A1!A,,A1c                 H    t        ||| j                  | j                        S )Nr  )r  r   r   )r?  r,  r  s      r   rE   z(Router.get_num_retries_from_retry_policy!'  s)     2#%)%B%B**	
 	
r   c                    | j                   }|yt        |t        j                        r|j                  |j                  S t        |t        j
                        r|j                  |j                  S t        |t        j                        r|j                  |j                  S t        |t        j                        r|j                  |j                  S t        |t        j                        r|j                  |j                  S yy)a  
        BadRequestErrorRetries: Optional[int] = None
        AuthenticationErrorRetries: Optional[int] = None
        TimeoutErrorRetries: Optional[int] = None
        RateLimitErrorRetries: Optional[int] = None
        ContentPolicyViolationErrorRetries: Optional[int] = None
        N)r   r   r   r  AuthenticationErrorAllowedFailsr   TimeoutErrorAllowedFailsr  RateLimitErrorAllowedFailsrs  'ContentPolicyViolationErrorAllowedFailsr  BadRequestErrorAllowedFails)r?  r,  r   s      r   get_allowed_fails_from_policyz$Router.get_allowed_fails_from_policy+'  s     >B=V=V' y'"="=>$DDP'GGGy'//2$==I'@@@y'"8"89$??K'BBBy'"E"EF$LLX'OOOy'"9"9:$@@L'CCC M ;r   c                 J   ddl m} | j                  y | j                  } ||j                  dg|j                        }|| _        t        j                  j                  |       t        j                  j                  |j                         t        j                  d       y )Nr   )SlackAlertingslack)alerting_thresholdalertingdefault_webhook_urlz2[94m
Initialized Alerting for litellm.Router[0m
)1litellm.integrations.SlackAlerting.slack_alertingr  r   r   webhook_urlslack_alerting_loggerr   r&  r:  r*  !response_taking_too_long_callbackr   r5  )r?  r  router_alerting_config_slack_alerting_loggers       r   r;  zRouter._initialize_alertingS'  s    S'151E1E!.5HHY 6 B B"
 &<"((==>TU((EE"DD	
 	""H	
r   CustomRoutingStrategyc                 `    t        | d|j                         t        | d|j                         y)a  
        Sets get_available_deployment and async_get_available_deployment on an instanced of litellm.Router

        Use this to set your custom routing strategy

        Args:
            CustomRoutingStrategy: litellm.router.CustomRoutingStrategyBase
        rh  r  N)r  rh  r  )r?  r	  s     r   set_custom_routing_strategyz"Router.set_custom_routing_strategyk'  s4     	&!::	

 	,!@@	
r   c                 N    d t         _        | j                  j                          y r   )r   r   flush_cacher  s    r   r  zRouter.flush_cache'  s    

 r   c                     g t         _        g t         _        g t         _        g t         _        d | _        | j                          y r   )r   r)  r$  r/  r,  r   r  r  s    r   resetzRouter.reset'  s6    #% *,'#% *,' r   )F)r   r   )FFF)T)rv  r  )NNNN)NNNNN)NF)r   N)NNN)NNFN)NNF(  r   r   r   rM  r   __annotations__r   r   r  r   r  tenacityr   r*   r   r-   r   r   r   r   r	   r   rd   r\   r
   r   rX   ri   r]   r4  rv  r  r   re   rb   rW   ra   rl   rV   rG  staticmethodrP  r>  rZ  r   r   r   rc  rh   r"  r<  r  r  r  r  r  r  r	  r  r  r!  r=  r  r8  rE  rt   rr   rO  rK  r  rl  r   rP   r  ro   rq   r  r  rt  r  r  rL  r@  ru  r4  r[   rA  rJ  rn  r  r_  rb  rS  rz  r~  r  r  r   r  r  r  r  r  r  r  rQ   r  r  r  r  r  r  r  r  r  r  r  r	  r  r  r  r  r#  r  r)  rs   r-  r+  r3  r2  rR   r7  r6  rO  r   rm   rW  rU  r`  ri  rh  rm  ru  rv  r  r  r  r  r   r  r  r%   wrapr  r  r  r  r  r  rM  r  r  r  r(  r+  r1  r.  r3  r  r~   rA  rC  rr  rK  r  rq  r  r  r\  r<  r~  r^   r  r  r  r  rr  r%  r  ru  r  r  r  r  rw  r=  r  r  r  r  r  r  r  r  r  rn   r  r`   r#  r'  r2  r   r   r5  r>  r  rD  rF  rJ  rp  rS  rW  rZ  r  r  ra  r`  rj  r  r  r  rv  rx  r  r  ro  r  r+  r  rI  rK  r  r  r   r  rh  r  r  r  r  r  r  rE   r  r;  rZ   r  r  r  r   r   r   r   r      s    uK&+OXd^+&11H:>h67>:>h67>MQeL(C,G&H!IJQ ;?<@=A#'$($((,"&*/ ,0*.%) #'*. 7;!06 )+)+ ',%*(,     ,0 DH&(DH48 "#/3FJ+0,1:>od&$*+T$sCx.-AAB
d& $$78d& t$789d& !&8!9:d& C=d& SMd& SMd& !d&  3-!d&" "$#d&$ %d&& !K
'd&, -d&0 #5/1d&2 #3-3d&6 c]7d&8  
9d&> %?d&@ !Ad&B !)!
Cd&H (0}Id&J Kd&L _-Md&N $I
Od&T Ud&V #'Wd&X #'Yd&Z $eC!::;;<
[d&` !%ad&b #cd&d "&ed&f gd&h +t#$
id&n #'#
od&t  
ud&z '
{d&@  
Ad&F $D>Gd&H "&
Id&X #++@"AYd&Z  $[d&\ !))@ A]d&^ ".1_d&` "*!"
ad&f *-gd&h &.d3S	>.B%Cid&j %)kd&l &*md&n +33-od&p 
qd&L DI   D .38n.	z,,	-. ."+ +M %os&: ;MTXM^	FD
P0
dF
P
,
HB
H+MZ
"
H21$ mM(01F(GmM^4 2$(c3h$8	}11	2&cc$(c3h$8c	}11	2cJ. .`11+/91j $()9$:DKDM	 
 UZ$()9$:DKEN	 
 kp$()9$:DI'RV-Y`afYgJgDh	"M1	2  	33 '(3 	3j 8*8(08 
8 8(N>+N> tCH~&N> 	N>
 
N>`B+B tCH~&B 	B
 
BB+/9@ii$(c3h$8i	}12	3i^ 1;	

 
 !)	

 

*"04	, EOPP4<SMP	P, NR''(,'=Ec]'	'@ > >t > > >. (,	D
D
 D
  }	D

 
D
L t 4

"&
	%s
#	$
"&	%s
#	$$	4 	t 	uczAR8S 	Z#S	Z# T#s(^,d4S#X3G.HHIZ#x++$(.>)?$@+^ $(c3h$8BI$-	  SX$(c3h$8BI%.	  	M,M, tCH~&M, 	M,f di$()9$:FISZ[`Sa	 
 $()9$:FISZ[_S`	  @@ '(@ 	@D?? ? $	?
 CHo? S#X?B
X 
X 
XaBaB '(aB S#X	aBFs 3 , ,C ,\c # *Fs F3 FP( (3 (TD) DC DLT3 Ts T3 Tl3 *)C )^ $)&+#(## # 4.	#
 d^# 4.#R $)&+#(## # 4.	#
 d^# 4.#JDS D# DT $)&+#(  4.	
 d^ 4.:DS D DL
x 


5=
  $> $,:!
!
 
!
F-5>38n-0>A	c3h>OO5=ObJJ-5J` $)	 S$Y 4.	 
"6c4i 0 6 6x $(	 S$Y 4.	 
2DuS$Y'7 D DN 
	0GG 
	GT^S$Y^@	 
	>NN 
	Nd  $v}v 
	vp   
	 DNN 
	N`//l .21#1 &c]1D QB#B c"e
BN
 .21#1 &c]18 .21#1 &c]1 
#
2 .21#1 &c]1, .2(,	
#
 &c]
 %	
,w!w! $D>w! D>	w!
 #+4.w! #+4.w! c]w! w! w!r V[[]1 1l &*$(3737-- c]- D>	-
 #+4.- #+4.-^ V[[]C% C%J $ :>!!)1#!L /3*.3737,0FF &d^F "$	F
 #+4.F #+4.F $D>FPW &*$S$s)^,-$ c]$ 
$s)		$2Xc] $ /3*.// / 	/
 &d^/ "$/ 
sEz	/f@D! 
#!FJ 
JX 
+3C= 
D>>	/	0> # # #t #J 4<TN	6 $$$1$;?$	$L5c 5Xd^ 5.55,4TN5	tDz4:%	&5<54 5( 15	;; #4.; n-	;F *.041,1, "$Z1, 4 012	1,
 #4.1, !1, n-1,f'c '4 '>`` ` 	`
 ` 
*	`D D -@j -@^~ RV +KJ +KZ&z &d &P0A 0Aj*  B:$:;>:GJ:x' '8L 'R:?:?*-:?	:?xGd G G G( 6:22%-c]2	2>4J 48J;O 4lC HZ,@ 8s x
/C &	(3 	(8D> 	( #	*	0KK	$sCx.	!KZ 
 	$,- ! 	
 
  58>A	  !	`U4#567` !` SM	`
 
`D $  
:# 
:(4. 
:66),6	)	6pf f >Af 	.	!f P
 
8P 
>G$G$	x}hsm+	,G$R 1266	.	!6 36 cSVh B ;?""*23-"	"H6$ 64 6&R$ R8 MR""3-"EI"	c"HE E E4 4	#4lc C HSM 0 FJ&*5=c]	: &*!%	NN c]N #	N
 
!	"N`x} S	 0 IM-8@	#*
>(4. 
>T 
> +/$"3-$	!	"$N JN("3-(9A#(	$*+	,(T8) %),0!%	.SM. %SM. #	.
 
c49n	.`"%	.#B1Wf4v *.G%G% "G% tCH~&	G%
 !G%R3 8C= &Uc Ud U 48,0.3)-B*B* 4S#X/0B* c4i()	B*
 &d^B* !B* 
sE$*%%	&B*P 48,0.3+/o#o# o# 4S#X/0	o#
 c4i()o# &d^o# #4.o# 
tDz4	 o#j 48,0.3aa a 4S#X/0	a
 c4i()a &d^aN 48,0.3RR R 4S#X/0	R
 c4i()R &d^Rp 48,0.3%% % 4S#X/0	%
 c4i()% &d^% 
(	)%T 48,0.3)-HH 4S#X/0H c4i()	H
 &d^H !HZ 48,0.3)-ee 4S#X/0e c4i()	e
 &d^e !eN
#':
EI#Y
	d
4 ,0 !$Z  #4.  
d	 J ,0!$Z #4. 
d	6(#':(	d(: FJY,4TNY$ BF
"
19#
&Dy &DP
0
%>
.!r   r   )r  r<  enumr`  rI  r_  r   rs  rj  r  r  rs  r   	functoolsr   typingr   r   r   r	   r
   r   r   r   r   r   r   r   r  httpxr  r   pydanticr   typing_extensionsr   r   litellm.litellm_core_utils2litellm.litellm_core_utils.exception_mapping_utilsr   litellm._loggingr   litellm._uuidr   litellm.caching.cachingr   r   r   r   litellm.constantsr   r   r   "litellm.integrations.custom_loggerr   #litellm.litellm_core_utils.asyncifyr    r  r!   r"   ,litellm.litellm_core_utils.coroutine_checkerr#   .litellm.litellm_core_utils.credential_accessorr$   %litellm.litellm_core_utils.dd_tracingr%   *litellm.litellm_core_utils.litellm_loggingr&   r  0litellm.litellm_core_utils.sensitive_data_maskerr'   $litellm.llms.openai_like.json_loaderr(   &litellm.router_strategy.budget_limiterr)   "litellm.router_strategy.least_busyr*   #litellm.router_strategy.lowest_costr+   &litellm.router_strategy.lowest_latencyr,   &litellm.router_strategy.lowest_tpm_rpmr-   )litellm.router_strategy.lowest_tpm_rpm_v2r.   r  r0   )litellm.router_strategy.tag_based_routingr1   /litellm.router_utils.add_retry_fallback_headersr2   r3    litellm.router_utils.batch_utilsr4   r5   r6   /litellm.router_utils.client_initalization_utilsr7   2litellm.router_utils.clientside_credential_handlerr8   r9   rJ  r:   r;   #litellm.router_utils.cooldown_cacher<   &litellm.router_utils.cooldown_handlersr=   r>   r?   r@   rA   ,litellm.router_utils.fallback_event_handlersrB   rC   rD   *litellm.router_utils.get_retry_from_policyrE   r  !litellm.router_utils.handle_errorrF   rG   'litellm.router_utils.health_state_cacherH   >litellm.router_utils.pre_call_checks.deployment_affinity_checkrI   ;litellm.router_utils.pre_call_checks.model_rate_limit_checkrJ   Dlitellm.router_utils.pre_call_checks.prompt_caching_deployment_checkrK   >litellm.router_utils.router_callbacks.track_deployment_metricsrL   rM   litellm.schedulerrN   rO   litellm.types.llms.openairP   rQ   rR   rS   litellm.types.routerrT   rU   rV   rW   rX   rY   rZ   r[   r\   r]   r^   r_   r`   ra   rb   rc   rd   re   rf   rg   rh   ri   litellm.types.servicesrj   litellm.types.utilsrk   rl   rm   rn   r  ro   rp   rq   r  rr   rs   rt   ru   rv   rw   rx   ry   rz   r{   &router_utils.pattern_match_deploymentsr}   opentelemetry.tracer~   _Spanr  r   r   r  r   Enumr   r   r   r   r   <module>rF     s          	    #          &  ! 9 " 2   
 < B K M 8 P P E G F H N J P A M 
 R >  
 J 2      0 0 
 * 9 
   G1 DDJ $)) uZ uZr   