AHjA/ ddlZddlZddlZddlZddlZddlZddlmZddl m Z m Z m Z m Z ddlZddlZddlmZddlZddlmZddlmZe rddlmZne Zej2j5ej6dZej:dd Zed k(redd lm Z dZ!Gd d e Z"d e e#de$de#fdZ%GddZ&ejNejPddddejPddddejPdeddejPddd !ejPd"ejRd#!ejPd$d%dd&!ejPd'dd(!ejPd)dd&!ejPd*dd+!ejPd,d-e*d./ejPd0d1d-e*d2d34ejPd5d1d-e*d6d74ejPd8d1d-e*d9:ejPd;de+d<=ejPd>de,d?=ejPd@de,dA=ejPdBd-dCDejPdEd-dFDejPdGdHddI!ejPdJde+dK=ejPdLd-e*dM=ejPdNde#dO=ejPdPd-d1dQRejPdSdTd1d-e*dU:ejPdVd-dWXejPdYd-dZXejPd[d1d-d\]ejPd^d1d-d_]ejPd`dae,db=ejPdcd1d-dd]ejPded1d-df]ejPdgde#dhdijejPdkde#dldmjejPdnde#do=ejPdpd-d1dqRejPdrd-d1dsRejPdtd-d1duRejPdvde,dwdxjejPdyde,dzd{jejPd|d-d1d}d~de*de*fdZ-e.dk(re-yy)N) TYPE_CHECKINGAnyOptionalUnion) load_dotenv)!DEFAULT_NUM_WORKERS_LITELLM_PROXY)get_secret_bool)FastAPIzlitellm.secrets LITELLM_MODEDEV)EnumceZdZdZdZy)LiteLLMDatabaseConnectionPool <N)__name__ __module__ __qualname__database_connection_pool_limit database_connection_pool_timeoutO/root/tools/cai/cai_env/lib/python3.12/site-packages/litellm/proxy/proxy_cli.pyrr$s%'"')$rrurlparamsreturncddlm}|jd||jd|t|tr|dk(r|j dyt j |}t j|j}|j|t j|d}t j|j| }|S) Nr)verbose_proxy_loggerzurl: zparams: zLappend_query_params received empty or non-string URL, returning empty stringT)doseq)query) litellm._loggingrdebug isinstancestrwarningurlparseparse_qsr!update urlencode urlunparse_replace)rrr parsed_url parsed_query encoded_query modified_urls rappend_query_paramsr1)s5se}-&23 c3 3"9 $$ Z ""3'J$$Z%5%56L&&|4@M&&z':':':'OPL rcPeZdZedZedZededededee effdZ e ddeded e ed e ed e f d Z e dd ededededede ef dZe ddeded edededede efdZedZedZedZedede e d dfdZy)ProxyInitializationHelperscttjjd}tjd|dy)Nlitellmz LiteLLM: Current Version =  ) importlibmetadataversionclickecho) pkg_versions r_echo_litellm_versionz0ProxyInitializationHelpers._echo_litellm_version>s.((00;  2;-rBCrctdtjd|d|d}ttj|jdy)Nz) LiteLLM: Health Testing models in confighttp://:z/health)r)indent)printhttpxgetjsondumps)hostportresponses r_run_health_checkz,ProxyInitializationHelpers._run_health_checkCsA :;9974&$w!?@ djj34rrHrImodeltestcZ|xsd}tjd|ddl}d|d|}t|tr|}n t d|j d|}|jjj|d d d gd }tjd|td||jjj|d d d gd} | D]} tjd| td|jj|d } t| y)Nz gpt-3.5-turbozE LiteLLM: Making a test ChatCompletions request to your proxy. Model=rr?r@zInvalid test valuez My API Key)api_keybase_urluserz*this is a test request, write a short poem)rolecontent)rLmessages max_tokensz LiteLLM: response from proxy zN LiteLLM: Making a test ChatCompletions + streaming r equest to proxy. Model=T)rLrUstreamz'LiteLLM: streaming response from proxy z$ making completion request to proxy)rLprompt) r:r;openair$r% ValueErrorOpenAIchat completionscreaterC) rHrIrLrM request_modelrYapi_baseclientrJstream_responsechunkcompletion_responses r_run_test_chat_completionz4ProxyInitializationHelpers._run_test_chat_completionIse0  TUbTc d  TF!D6* dC H12 2|hG;;**11#K 2    4XJ?@ ]^k]l m !++1188#K 9  % JE JJ@H I J 56$0077(T8  !"rN log_configkeepalive_timeoutrcddl}ddlm}d||d}|td|||d<n|jr ||d<|||d<|S) z8 Get the arguments for `uvicorn` worker rN)_get_uvicorn_json_log_configzlitellm.proxy.proxy_server:app)apprHrIzUsing log_config: rftimeout_keep_alive)r5r"rirC json_logs)rHrIrfrgr5ri uvicorn_argss r_get_default_unvicorn_init_argsz:ProxyInitializationHelpers._get_default_unvicorn_init_argssk A4  ! &zl3 4)3L &   )E)GL &  (1BL- .rrjssl_certfile_pathssl_keyfile_pathcipherscddl}ddlm}ddlm}t d|d|d|} |d|g| _|+|)t d|d |d || _|| _||| _ |j||| y) z5 Initialize litellm with `hypercorn` rN)serve)Config)LiteLLM Proxy: Starting server on r@z using Hypercorn /LiteLLM Proxy: Using SSL with certfile:  and keyfile:  ) asynciohypercorn.asynciorshypercorn.configrtrCbindcertfilekeyfilerqrun) rjrHrIrorprqryrsrtconfigs r_init_hypercorn_serverz1ProxyInitializationHelpers._init_hypercorn_servers ++ :4&$G` a q'(  (-=-I DEVDWWefvewxAB 0FO-FN"!(  E#v&'r num_workersmax_requests_before_restartc  tjdk(rnddl}Gfddjjj }t d|dd|d |d|d d d d dd} ||| d<tjjdrddl m  fd} | | d<||t d|d|d|| d<|| d<||| jy)z- Run litellm with `gunicorn` ntrNc0eZdZdfd ZdZdZxZS)NProxyInitializationHelpers._run_gunicorn_server..StandaloneApplicationc|xsi|_||_t| dd}|dz}t t dt d|dt dt ddy) Nz curl --location 'http://0.0.0.0:z/chat/completions' \ai --header 'Content-Type: application/json' \ --data ' { "model": "gpt-3.5-turbo", "messages": [ { "role": "user", "content": "what llm are you" } ] }' zLiteLLM: Test your local proxy with: "litellm --test" This runs an openai.ChatCompletion request to your proxy [In a new terminal tab] z8LiteLLM: Curl Command Test for your local proxy z  z;Docs: https://docs.litellm.ai/docs/simple_proxy z5See all Router/Swagger docs on http://0.0.0.0:)options applicationsuper__init__rC)selfrjr _endpoint_str curl_command __class__rIs rrzWProxyInitializationHelpers._run_gunicorn_server..StandaloneApplication.__init__s&}" #&  "7tfycc}}wN)cfgritemssettingssetlower)rkeyvaluers r load_configzZProxyInitializationHelpers._run_gunicorn_server..StandaloneApplication.load_configs88'+/,,*<*<*>&C$(("3"338IU F  F"(,,.9JCxx+ SYY[%89s'B'c|jSr)r)rs rloadzSProxyInitializationHelpers._run_gunicorn_server..StandaloneApplication.loads'''rr)rrrrrr __classcell__)rrIs@rStandaloneApplicationrs$ L 9 (rrrur@z with z workers zuvicorn.workers.UvicornWorkerT-iXz+%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s)r|workers worker_classpreload accesslogtimeoutaccess_log_format max_requestsPROMETHEUS_MULTIPROC_DIR)mark_worker_exitc*|jyr)pid)serverworkerrs r child_exitzCProxyInitializationHelpers._run_gunicorn_server..child_exits ,rrrvrwrxr}r~)rjr) osnamegunicorn.app.baserjbaseBaseApplicationrCenvironrE litellm.proxy.prometheus_cleanuprr) rHrIrjrrorprgunicornrgunicorn_optionsrrs ` @r_run_gunicorn_serverz/ProxyInitializationHelpers._run_gunicorn_servers 77d?  $7 (HLL$5$5$E$E7 (r :4&$vk]Zk l fAdV$";!N  ' 2/J ^ , ::>>4 5 I -.8 \ *  (-=-I DEVDWWefvewxAB ,= Z (*: Y '#/?@DDFrc ddg}ttjd5}tj|||dddy#1swYyxYw#t $r}t d|dYd}~yd}~wwxYw)Nollamarsw)stdoutstderrzh LiteLLM Warning: proxy started with `ollama` model `ollama serve` failed with Exceptionz-. Ensure you run `ollama serve` )openrdevnull subprocessPopen ExceptionrC)commandres r_run_ollama_servez,ProxyInitializationHelpers._run_ollama_serve-s )Gbjj#& J'  I J J J  iijhkl    s3AAAA A A A0A++A0cddl}|j|j|j5}|jd|fdk(cdddS#1swYyxYw)Nr localhost)socketAF_INET SOCK_STREAM connect_ex)rIrss r_is_port_in_usez*ProxyInitializationHelpers._is_port_in_use;sJ ]]6>>6+=+= > :!<<d 349 : : :s A  Ac*tjdvryy)zBHelper function to determine the event loop type based on platform)win32cygwincliNuvloop)sysplatformrrr_get_loop_typez)ProxyInitializationHelpers._get_loop_typeBs <<5 5rlitellm_settingscddl}|dks|y|jdxsg}|jdxsg}|jdxsg}t|tr|g}t|tr|g}t|tr|g}||z|z}d|vryddlm}t jjd xst jjd }| } |sAt jj|jd }|t jd <t j|d ||| rdnd} td| d|y)z Auto-create PROMETHEUS_MULTIPROC_DIR when running with multiple workers and prometheus is configured as a callback. rN callbackssuccess_callbackfailure_callback prometheus)wipe_directoryrprometheus_multiproc_dirlitellm_prometheus_multiprocT)exist_okz Auto-createdzUsing existingz LiteLLM: z PROMETHEUS_MULTIPROC_DIR=) tempfilerEr$r%rrrrpathjoin gettempdirmakedirsrC) rrrrsuccess_callbacksfailure_callbacks all_callbacksr multiproc_dir auto_createdactions r%_maybe_setup_prometheus_multiproc_dirz@ProxyInitializationHelpers._maybe_setup_prometheus_multiproc_dirIsV  ! /7 %((5; ,001CDJ,001CDJ i %" I ' -!2 3  ' -!2 3 !$558II } , C 'AB bjjnn &G )( GGLL##%'EM6CBJJ1 2 MD1}%#/5E &!;M?KLr)NNr)rrr staticmethodr=rKr%intrboolrerdictrnr rrrrrrrrrr3r3=sDD55 3#3#3#3#D#I 3#3#j%)+/ SM$C=   6"& (  ( ( ( (  ( # ( (D6:lGlGlGlG lG  lG  lG&.c]lGlG\  ::  ,M,M"4.,M ,M,Mrr3z--hostz0.0.0.0z!Host for the server to listen on.HOST)defaulthelpenvvarz--portzPort to bind the server to.PORTz --num_workerszfNumber of uvicorn / gunicorn workers to spin up. Default is 1 (from DEFAULT_NUM_WORKERS_LITELLM_PROXY) NUM_WORKERSz --api_basez API base URL.)rrz --api_versionz$For azure - pass in the api version.z--modelz-mz)The model name to pass to litellm expectsz--aliaszThe alias for the model - use this to give a litellm model name (e.g. "huggingface/codellama/CodeLlama-7b-Instruct-hf") a more user-friendly name ("codellama")z --add_keyz --headerszheaders for the API callz--saveTzSave the model-specific config)is_flagtyperz--debugFzTo debug the inputDEBUG)rrrrrz--detailed_debugzTo view detailed debug logsDETAILED_DEBUGz --use_queuez)To use celery workers for async endpoints)rrrrz --temperaturezSet temperature for the model)rrrz --max_tokenszSet max tokens for the modelz--request_timeoutz+Set timeout in seconds for completion callsz --drop_paramszDrop any unmapped params)rrz--add_function_to_promptz5If function passed but unsupported, pass it as promptz--configz-cz]Path to the proxy configuration file (e.g. config.yaml). Usage `litellm --config config.yaml`z --max_budgetz`Set max budget for API calls - works for hosted models like OpenAI, TogetherAI, Anthropic, etc.`z --telemetryzZHelps us know if people are using this feature. Turn this off by doing `--telemetry False`z --log_configz&Path to the logging configuration filez--setupzRRun the interactive setup wizard to configure providers and generate a config file)rrrz --versionz-vzPrint LiteLLM versionz--healthz:Make a chat/completions request to all llms in config.yaml) flag_valuerz--testz4proxy chat completions url to make a test request toz --test_asyncz9Calls async endpoints /queue/requests and /queue/response)rrrz--iam_token_db_authz!Connects to RDS DB with IAM tokenz--num_requestsrz-Number of requests to hit async endpoint withz--run_gunicornzTStarts proxy via gunicorn, instead of uvicorn (better for managing multiple workers)z--run_hypercornz@Starts proxy via hypercorn, instead of uvicorn (supports HTTP/2)z--ssl_keyfile_pathz^Path to the SSL keyfile. Use this when you want to provide SSL certificate when starting proxySSL_KEYFILE_PATH)rrrrz--ssl_certfile_pathz_Path to the SSL certfile. Use this when you want to provide SSL certificate when starting proxySSL_CERTFILE_PATHz --ciphersz!Ciphers to use for the SSL setup.z--use_prisma_db_pushzHUse prisma db push instead of prisma migrate for database schema updatesz--localzfor local debuggingz--skip_server_startupzASkip starting the server after setup (useful for migrations only)z--keepalive_timeoutzSSet the uvicorn keepalive timeout in seconds (uvicorn timeout_keep_alive parameter)KEEPALIVE_TIMEOUTz--max_requests_before_restartz]Restart worker after this many requests (uvicorn: limit_max_requests, gunicorn: max_requests)MAX_REQUESTS_BEFORE_RESTARTz --enforce_prisma_migration_checkz7Exit with error if database migration fails on startup.ENFORCE_PRISMA_MIGRATION_CHECK)rrrruse_prisma_db_pushenforce_prisma_migration_checkc( |rddlm}(|(yt})|r ddlm}*m}+m},m}-n ddlm}*m}+m},m}-|durtjy|rd |vr|tj|durtj||y|durtj||||y|rt!j"|}|-d^id |d |d |d |d| d| d| d| d| d|d|d|d|d|d|d|d| ddl}/d}0d}1i}2|s t)drddlm}3t/j0d }4t/j0d!}5t/j0d"}6t/j0d#}7t/j0d$}8|3|4|5|6%}9d&|6d'|9d(|4d'|5d)|7 }:|8r|:d*|8z }:|:t.j2d+<d,t.j2d<dd-lm};t/j0d.dJt/j0d.d,k(r2|;}<||>t.j2|=<d}?| ddl}@|+}A@j=|Aj?|0}B|BjAd1d}?|?'d2|?vr#|?d2durddl!}Cd|C_"|CjFBjAd3i}2|2i}2|2jAd4d}D|Dddl!}C|*d^iD|C_$|2r%|2jAd5d}EAjK|E|6|2jAd7d}F|F7t/j0d+"dd8l&m'}G|G}F|FrFt.j2d+<|2jAd9tPjRjT}0|2jAd:tPjVjT}1FrFjYd;rzt/jZ}Ht\j^jadt.j^jcd<ddl!}Cdd=l!m2}I|IFd>}Ft/jf|HF#tiFtrFt.j2d+<|7t/j0d+"dd8l&m'}G|G}F|FrFt.j2d+<|4tPjRjT}0tPjVjT}1t/j0d+dt/j0d?dA dd@l5m6}Jt/j0d+d;|0|1dA}KJd+d>}Fto|Fr tFndK}L|Lt.j2d+<t/j0d?d9|0|1dA}Kt/j0d?}Fto|F|K}L|Lt.j2d?<tqj<dBgdCd}MMrsddEl:m;}NddFl}P|P|2jAdGdDur NdHnMOj|# Is:|'r!tdJt]jdntdKn tdL|dMk(r+tj|rtjdNdO}ddl!}C| durCjddPlFm},tj||r|?ndQ|$r tdRytj|||"|%S}Q|&|&QdT<|dDurV|dDurR| |tdU| dV|dW|QdX<| |QdY<tj}R|RrRQdZ<|/j<d^iQd[|iy|durtj|||,|| ||&\y|durtj|,||| ||!]yy#t$r}.td|.dd}.~.wt$r&}.dt|.vr|.ddlm}*m}+m},m}-Yd}.~.md}.~.wwxYw#t&$r tdwxYw#t&$r td/wxYw#tr$rdD}MY?wxYw)_Nr)run_setup_wizard)KeyManagementSettings ProxyConfigrjsave_worker_configrzMissing dependency z$. Run `pip install 'litellm[proxy]'`zlitellm[proxy]TrrLaliasr` api_versionr#detailed_debug temperaturerVrequest_timeout max_budget telemetry drop_paramsadd_function_to_promptheaderssaver use_queuezLuvicorn, gunicorn needs to be imported. Run - `pip install 'litellm[proxy]'`drIAM_TOKEN_DB_AUTH)generate_iam_auth_token DATABASE_HOST DATABASE_PORT DATABASE_USER DATABASE_NAMEDATABASE_SCHEMA)db_hostdb_portdb_userz postgresql://r@@/z?schema= DATABASE_URLTrue)decrypt_env_var USE_AWS_KMSz?yaml needs to be imported. Run - `pip install 'litellm[proxy]'`)config_file_pathrrlgeneral_settingskey_management_settingskey_management_system)r(r% database_url)$construct_database_url_from_env_varsrrz os.environ/z../..)get_secret_str) default_value DIRECT_URL) get_secret)connection_limit pool_timeoutprisma)capture_outputF)check_prisma_schema_diff) PrismaManagershould_update_prisma_schemadisable_prisma_schema_update)db_url) use_migratezLiteLLM Proxy: Database setup failed after multiple retries. The proxy cannot start safely. Please check your database connection and migration status.zLiteLLM Proxy: Database migration failed but continuing startup. Set --enforce_prisma_migration_check or ENFORCE_PRISMA_MIGRATION_CHECK=true to exit on failure.zYUnable to connect to DB. DATABASE_URL found in environment, but prisma package not found.rii)rj)rrz>LiteLLM: Setup complete. Skipping server startup as requested.)rHrIrfrglimit_max_requestsrvrwrx ssl_keyfile ssl_certfileloopr)rHrIrjrrorpr)rjrHrIrorprqr)Llitellm.setup_wizardrlocals proxy_serverrrrjrModuleNotFoundError ImportErrorr%r3r=rrKrerFloadsuvicornrr litellm.proxy.auth.rds_iam_tokenrrgetenvr*litellm.secret_managers.aws_secret_managerr#rryr get_configrEr5rl _turn_on_json_key_management_settingsinitialize_secret_managerlitellm.proxy.utilsr*rrrr startswithgetcwdrrinsertabspathr+chdirr$litellm.secret_managers.mainr.r1rFileNotFoundError litellm.proxy.db.check_migrationr3litellm.proxy.db.prisma_clientr4r5setup_databaserCexitrrandomrandint_turn_on_debuglitellm.proxy.proxy_serverrrnrrr)SrHrIr`r rLradd_keyrrr#r r rVr rrrr rrMlocalr test_asynciam_token_db_auth num_requestsrhealthsetupr9 run_gunicorn run_hypercornrprorqrfrskip_server_startuprgrrrargsrrrjrrrCdb_connection_pool_limitdb_connection_timeoutr&rrrrdb_name db_schematoken_db_urlr# new_env_varkvrry proxy_config_configr5r'r(r)r* original_dirr+r.rr0is_prisma_runnabler3r4r5rm loop_typesS r run_serverrtysj 9 8D      ,$"88: U"x'7"446 ~"44T4@ t|"<* >bii7? P?AL-9 >* >-LLRR %.NNTT " IInd + 7yyt,8 +C99^T2>-E(=F$.nD#QL#6-9L)tV$L2>BJJ~.99\40<-E(=F$&99\#:L#6|V#LL/;BJJ|,z$?%)""U 0(,,-KL -D9(77(:$:8:!!D HHQK!!I o 4<6FFtL>>$.D T ! "G " " $ 3 #HH#17-T I   P  1QQ!/ R  ' 21LL- . 5 ]e%; ,1A1MHIZH[[ijzi{|EF/? ]+/@ ^,2AACI'0 V$ GKK  # T ! & ; ;'"3!1,G < d " & = ="3!1 > #O # %%aS(LM  3q6) ^ ^  n !U r% +%*" +sN ]5^,4_+C_ ^)(]77 ^)^$$^),__ _+*_+__main__)/r7rFrrWrr urllib.parseparser'typingrrrrr:rDdotenvrr5litellm.constantsrrQr fastapir rappendrMconfig_filenamerE litellm_modeenumr rrr%rr1r3roptionAZURE_DEFAULT_API_VERSIONrfloatrrtrrrrrs  66 ?8G  #ryy/ 5M  *D* Xc]DS(yMyMx   i&IRXh+HQWX  - q   lD?   - - /  tT(S  k $Ok4.HI h46VW         &     4 T4SDs1O  6  ot2LM   @   h     k    e    1     ]      E   ?    D     ,    8     _     K    i    j    ,     S  iu;PQ    L    ^  #  h ( &   B + OHIOP%)QO R               N XJ  @ YVOd  zLr