AHj`;ddlZddlZddlZddlZddlZddlZddlmZmZddl Z eje Z ddl mZmZgdZddgZdeefdZdefd Zd Zd!d ed eefd ZdedefdZdZdefdZdededeeeffdZ d"ded eedeedeefdZ!dededefdZ"dededefdZ# d#dedeedeed eedeedeedeefd Z$y)$N)ListOptional)DEFAULT_HEALTH_CHECK_PROMPTHEALTH_CHECK_TIMEOUT_SECONDS)messagesapi_keypromptinputvertex_credentialsaws_access_key_idaws_secret_access_keymodel mode_errorreturnc ddl}|j|jj}tj dk(rt |dz St |dz S#t$rYywxYw)zj Get process RSS memory in MB. On Linux, ru_maxrss is in KB. On macOS, ru_maxrss is in bytes. rNdarwinii)resource getrusage RUSAGE_SELF ru_maxrsssysplatformfloat Exception)rrs R/root/tools/cai/cai_env/lib/python3.12/site-packages/litellm/proxy/health_check.py_get_process_rss_mbrsd &&x';';<FF <<8 ##{3 3Y$&& sA A A A&%A&c$t}|y|dS)Nunknownz.2f)r)rss_mbs r_rss_mb_for_logr -s "F ~S\c<ddg}dtj|dgS)z, Get a random message from the LLM. zHey how's it going?z What's 1 + 1?user)rolecontent)randomchoice)rs r_get_random_llm_messager(4s%&7H h(? @ AAr! endpoint_datadetailsc|jdd|dur-|jDcic]\}}|tvs||c}}S|jDcic]\}}|tvs||c}}Scc}}wcc}}w)z7 Clean the endpoint data for display to users. litellm_logging_objNF)popitemsILLEGAL_DISPLAY_PARAMSMINIMAL_DISPLAY_PARAMS)r)r*kvs r_clean_endpoint_datar3=s+T2 % (--/S$!Q1A>Ac K|d}|jdi}|jdd}t||}|jdxst}tt j ||t dg|d{S7w)Nlitellm_paramsr6modehealth_check_timeoutztest from litellm)rLr r )r9'_update_litellm_params_for_health_checkrrIlitellm ahealth_checkr)rrKr6rLrHs r_run_model_health_checkrQes+,N<,J >>&$ 'D._schedule_nexts` ":.OC""#::#FG"t^S-@A   sA AA) return_when)rZiter enumerateboolrangeminrDwaitr8keysFIRST_COMPLETEDr-resultr) rRrSresultsr`_donerGr[er]r^r_s @@@r+_run_health_checks_with_bounded_concurrencyrows FS[(G.0Ni'(JN D 3(#f+6 7  ##% &//  a D $$T*C !#{{}       N ""  !   !sBB!D&C('DC* D$D* D3C=8D=DDmax_concurrencyinstrumentation_contextc K|xsi}t|jdd}|jdd}|jdd}d}d}t|tr|dkDrd}t ||d {\} }nS|D cgc] } t j t| "} } t| }t j| d d id {} |r@tjd |||t|||tjt g} g} t| |D]\}} | d }| jdxsijd}t|t r-d|vr)t#i|||}|r||d<| j%|lt|t r)t#i|||}|r||d<| j%|t#||}|r||d<| j%|| | fS7}cc} w7.w)z Perform a health check for each model in the list. max_concurrency: Optional limit on concurrent health check requests. enabledFcycle_idrsource unboundedrboundedNreturn_exceptionsTzhealth_check_dispatch_summary source=%s cycle_id=%s mode=%s model_count=%d max_concurrency=%s peak_in_flight=%d thread_count=%d rss_mb=%srKr6r7rCmodel_id)rdr9 isinstanceintrorDrXrQrZgatherloggerdebug threading active_countr zipdictr3r;)r4r*rprqinstrumentation_enabledrtru dispatch_moder^rkrtaskshealthy_endpointsunhealthy_endpoints is_healthyrK _model_idcleaneds r_perform_health_checkrs-6;"#:#>#>y%#PQ&**:yAH $ ( (9 =FMN/3'Oa,?! (S ) #  NX DIG   7 > ?  UFFF  X     O    " " $    *50 E/0YY|,277= j$ 'G:,E*+Kn+K +KWUG&/ #  $ $W -  D )*+Kn+K +KWUG&/ #  & &w /*>7CG&/ #  & &w /%0( 1 11]#  Gs1A0G:2G/3 G:?%G2$%G: G7 D&G:2G:rrctj}i}|D]}|jd}|sd|dd||<!|D]}|jd}|sd|dd||<!|S)a Build a dict mapping deployment_id -> DeploymentHealthStateValue from health check endpoint results. Each endpoint dict includes a 'model_id' field (added by _perform_health_check) that maps back to the deployment's model_info.id. Used by the background health check loop to feed health state into the router's DeploymentHealthCache for health-check-driven routing. ryT)r timestampreasonFbackground_health_check_failed)timer9)rrnowstateseprys rbuild_deployment_health_statesrs ))+CF66*% "  F8 "66*% # : F8  Mr!r6rKcBt|d<|jdd}|||d<n/d|jdxs|jdxsdvrd |d<|jdd}|||d<|jd dd k(r|jd d |d<|djdrqddlm}|d}|jdr|dd}|j d}g}|D]"}||j vs|j|$dj|}||d<|S)a Update the litellm params for health check. - gets a short `messages` param for health check - updates the `model` param with the `health_check_model` if it exists Doc: https://docs.litellm.ai/docs/proxy/health#wildcard-routes - updates the `voice` param with the `health_check_voice` for `audio_speech` mode if it exists Doc: https://docs.litellm.ai/docs/proxy/health#text-to-speech-models - for Bedrock models with region routing (bedrock/region/model), strips the litellm routing prefix but preserves the model ID rhealth_check_max_tokensN max_tokens*health_check_modelrrrL audio_speechhealth_check_voicealloyvoicezbedrock/r)BedrockModelInfo/) r(r9 startswith!litellm.llms.bedrock.common_utilsrsplitall_global_regionsr;join) r6rK_health_check_max_tokens_health_check_modelrrpartsfiltered_partsparts rrNrN sP"9!:N:)~~.GN+'?|$ +,Q0B0B70KQr ()|$$..)=tD&"5w~~fd#~5",..1Ew"Owg))*5Fw'   J '!"IE C  ,D+>>>%%d+ , ("'w r! cli_modelryc K|xsi}t|jdd}|jdd}|jdd} |s(|r |d|idg}n|rtjd| |ggfSt j } t |} |<|D cgc],} | jd xsijd |k(s+| .} } | r?| }n<|:|D cgc]} | d d|k(s| }} |gk(r|D cgc] } | d |k(s | }} |}t |}t|}t |}|r7tjd| || |||tjt  t||||d {\}}|ratjd| ||t |t |t j | z dztjt ||fScc} wcc} wcc} w7}#t$rQ|rMtjd| ||t j | z dztjtwxYww)av Perform a health check on the system. When model_id is provided, only the deployment with that id is checked (so models that share the same name but have different ids are checked separately). When model (name) is provided, all deployments matching that name are checked. Returns: (bool): True if the health check passes, False otherwise. rsFrtrrur) model_namerKzAhealth_check_cycle_skipped source=%s cycle_id=%s reason=no_modelsNr6r7rKr)r4zhealth_check_cycle_start source=%s cycle_id=%s requested_model_count=%d post_model_filter_count=%d deduped_model_count=%d max_concurrency=%s thread_count=%d rss_mb=%s)rprqzihealth_check_cycle_failed source=%s cycle_id=%s model_count=%d duration_ms=%.2f thread_count=%d rss_mb=%sizhealth_check_cycle_complete source=%s cycle_id=%s model_count=%d healthy_count=%d unhealthy_count=%d duration_ms=%.2f thread_count=%d rss_mb=%s)rdr9r}r~r monotonicrZrArrr rr exception)r4rrr*ryrprqrrtrucycle_start_timerequested_model_countx_by_id_new_model_listpost_filter_model_countdeduped_model_countrrs rperform_health_checkrMs&6;"#:#>#>y%#PQ&**:yAH $ ( (9 =F  (Wirs  !   8 $W"<0 Xe_  B  x~     &-$'# '#%('# 49'#X#%).2 C2C2 d^C2c]C2&d^ C2L### #L==&*= =D #""%).2l2l2 C=l2}l2d^ l2 sm l2 c] l2&d^l2r!