
    )#h                    6   U d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlmZmZ d dlmZmZmZmZmZmZmZmZmZ er	d dlmZ eZneZddZee_         ej:                  de       g Ze e!d<   ejD                  jG                  d ejD                  jI                  d	             	 d dl%Z%d dl&Z&d dl'Z'd dl(Z(d dl)Z)d d
l*m+Z+ g dZ.d Z/d dl0m1Z1 d dl2m3Z3 d dl4Z4d dl4m5Z5 d dl6m7Z7m8Z8 d dl9m:Z:m;Z; d dl<m=Z= d dl>m?Z? d dl@mAZAmBZB d dlCmDZDmEZE d dlF d dlGmHZI d dlJmKZK d dlLmMZM d dlNmOZO d dlPmQZQ d dlRmSZSmTZTmUZU d dlVmWZWmXZX d dlYmHZZ d dl[mHZ\ d d l]m^Z^ d d!l_m`Z`maZambZb d d"lcmdZd d dlcmHZe d d#lfmgZgmhZh d d$limjZjmkZk d d%llmmZmmnZn d d&lompZp d d'lqmrZr d d(lsmtZt d dlumHZv d d)lumwZw d dlxmHZy d d*lzm{Z{m|Z| d d+l}m~Z~ d dlmHZ d d,lmZ d d-lmZ d d.lmZ d d/lmZ d d0lmZ d dlmHZ d dlmHZ d dlmHZ d d1lmZ d d2lmZmZmZ d dlmHZ d dlmHZ d dlmHZ d dlmHZ d d3lmZ d d4lmZ d dlmHZ d d5lmZ d dlmHZ d d6lmZ d dlmHZ d d7lmZ d dlmHZ d dlmHZ d d8lmZ d dlmHZ d d9lmZ d dlmHZ d d:lmZmZmZmZmZmZmZmZmZmZmZmZmZ d dlmHZ d dlmHZ d d;lmZ d d<lmZmZmZmZ d d=lmZmZmZ d d>lmZ d d?lmZ d d@lmZmZmZmZ d dAlmZ d dBlmZmZmZmZ d dClmZ d dDlmZ d dElmZ d dFlmZmZ d dGlmZ d dElmZ d dHlmZ d dIlmZ 	 d dJlmZ dLe4_        d dlZd dMlmZ d dNl'mZmZmZmZmZmZmZmZmZmZmZm Z  d dOlmZ d dPlmZ d dQlmZ d dRlmZm	Z	m
Z
mZmZ d dSlmZ d dTlmZ d dUlmZ d dVlmZ 	 d dlmc mZ  ej0                  dWdX      Z eQ       at4        j7                         aee!dY<    ej0                  dZ      Zee   e!d[<    er       Z e
d\Z!e"e!d]<   n e"e      Z! ej0                  d^      Z#ee   e!d_<   e#
d`Z$e%e!da<   n e%e#      Z$e dbZ&dce& ddZ'e'dez  Z'dfZ(t8        r ej0                  dgdh      ndhZ)t8        r ej0                  didje( dke'       n	dle( dke' Z*dm Z+dn Z,e3doefdp       Z- e e        e       e)e*eee-q      Z.d drlm/Z/ ds Z0dt Z1 ej0                  dudv      dwk(  rt8        r
e1e._2         G dx dye3jh                        Z5e.jm                  e7      dzed{e7fd|       Z8 e       ZHd}gZ9	 ejD                  ju                  ejD                  jI                  e;            Z<ejD                  j{                  e<d~d      Z>e.j                  d ee>dL      d        ej                  e>      D ]  ZAeAj                  d      seAdk7  sejD                  j                  eA      d    ZDejD                  j{                  e>eD      ZE ej                  eEdL       ejD                  j{                  e>eA      ZGejD                  j{                  eEd      ZH ej                  eGeH        edXk7  rO eJde d        ej0                  d      eej                  d<   e.j                  d      dzefd       ZMe.j                  ee9dLd}gd}g       d dlmOZO daPdaQdaRdaSdaTdaUdLaVdZWdaXdaYee   e!d<   dLZZda[da\ee5   e!d<   da]ee    e!d<   i a^e_e!d<   i a`e_e!d<   dZadZbdacee   e!d<   daddaeee   e!d<    e:e5j                  j                        Zh eeh      Zie4j                  j                  ei       dalee;   e!d<   damdandaodapdaqdardasi atg Zuee!d<   dZvdawdaxeyd   e!d<   dazda{da|da}da~ eO       Zdaee   e!d<   daee!d<   daee   e!d<    eeht8              adZdZdZdaeeD   e!d<   dddddddddi d
dedee   dee   dee   dee   dee   dee   deee%ef      dee_   dee   dee_   de_fdZdzefdZd ZdefdZddefdZd Zd Z	 ddee_ef   dedee   fdZ	 	 	 	 	 	 	 	 ddZdee   dee   dee   dee   dee%   dee   fdńZdƄ ZdǄ Z G dȄ dɫ      Z e       adʄ Z	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd˄Zd̄ Zdede_fd̈́Zdede_fd΄Zdede_fdτZdede_fdЄZi fdѐe_fd҄Zdӄ ZdԄ Z G dՄ d֫      ZeHjG                  d eeW      gdg٫      eHjG                  d eeW      gdg٫       eeW      fdefdۄ              ZeHjK                  d eeW      gdg٫      eHjK                  d eeW      gdg٫      eHjK                  d eeW      gdg٫      eHjK                  d eeW      gdgdddiiet       e&jL                  e&jN                  ee!e$eee7      d eeW      fdzededee   defd                                   ZeHjK                  d eeW      gdg٫      eHjK                  d eeW      gdg٫      eHjK                  d eeW      gdg٫      eHjK                  d eeW      gdg٫      d eeW      fdzededee   defd                            ZeHjK                  d eeW      ge
dg      eHjK                  d eeW      ge
dg      eHjK                  d eeW      ge
dg      eHjK                  d eeW      ge
dg      d eeW      fdzededee   defd                            ZeHjK                  d eeW      ge
dg      eHjK                  d eeW      ge
dg       eeW      fdzededefd              ZeHjK                  d eeW      gdg٫      eHjK                  d eeW      gdg٫       eeW      fdzededefd              ZeHjK                  d eeW      gdg٫      eHjK                  d eeW      gdg٫       ed       eeW      fdzeded edefd              Zd dl'mZmZmZ d dl4mZ e.jc                  d      e.jc                  d       eeX      fdedѐefd              ZeHjG                  d eeW      gd	g٫      eHjG                  d
 eeW      gd	g٫       eeW      fdzededefd              ZeHjK                  d eeW      gd	g٫      eHjK                  d
 eeW      gd	g٫       eeW      fdzededefd              ZeHjk                  d eeW      gd	g٫      eHjk                  d eeW      gd	g٫       eeW      fdzedededefd              ZeHjK                  d eeW      gd	g٫      eHjK                  d eeW      gd	g٫       eeW      fdzededefd              ZeHjG                  d eeW      gd	g٫      eHjG                  d eeW      gd	g٫       eeW      fdzedededefd              ZeHjK                  d eeW      gd	g٫      eHjK                  d eeW      gd	g٫       eeW      fdzedededefd              ZeHjG                  d eeW      gd	g٫      eHjG                  d eeW      gd	g٫       eeW      fdzedededefd              ZeHjK                  d eeW      gd	g٫      eHjK                  d eeW      gd	g٫       eeW      fdzedededefd              ZeHjK                  d eeW      ge
d g      eHjK                  d! eeW      ge
d g       eeW      fdzededefd"              ZeHjK                  d#d$g eeW      ged%       eeW      fd&ededzedefd'       ZeHjK                  d(d)g eeW      ge*      dzefd+       ZeHjG                  d,d)g eeW      g-      dѐefd.       Zd/ededefd0Zd/ededefd1Zdee   dedefd2ZeHjK                  d3d4dg eeW      g5       eeW      fd/edefd6       ZeHjK                  d7d8dg eeW      g5       eeW      fd/edefd9       ZeHjG                  d:d;dg eeW      gd<       eeW       e'j                  dd=>      dfdedee   d?ee   fd@       ZeHjG                  dAdBdgd eeW      gC       eeW      dddfdedDee   dEee   dFee   fdG       ZeHjG                  dHdIdgd eeW      gC       eeW      dJddddfdedDee   dEee   dFee   dKee   dLee   fdM       ZeHjG                  dNdOdgd eeW      gC       eeW      dJddddfdedDee   dEee   dFee   dKee   dLee   fdP       ZeHjG                  dQdRdgd eeW      gC       eeW      dddddfdedDee   dEee   dFee   dKee   dLee   fdS       Zdѐe_de_fdTZeHjG                  dUdg eeW      g-      eHjG                  dVdg eeW      g-       eeW      dfdedWee   fdX              Zde5dYee   dZee   dee   fd[ZeHjG                  d\dg eeW      g-       eeW      dfdedZee   fd]       ZeHjK                  d^d_dg eeW      g5      d`efda       ZeHjG                  dbdcdg eeW      gd<      dd        ZeHjG                  dedfdgg eeW      gd<       eeW      fdefdh       ZeHjK                  didjg eeW      gdk      d eeW      fdzededee   defdl       ZԐe.jG                  dmdjgdn      dzefdo       ZeHjK                  dpdq      dzefdr       Z֐e.jG                  dsdq      dtefdu       Zאe.jK                  dvdq      dwefdx       Zِe.jG                  dydq      dz        ZeHjK                  d{d|g eeW      ged%       eeW      fdwedefd}       ZeHjG                  d~d|g eeW      ged%       eeW      fdedefd       ZeHjK                  dd|g eeW      ged%       eeW      fdwedefd       ZeHjK                  dd|g eeW      ged%       eeW      fdwedefd       ZeHjK                  ddg eeW      gdk      defd       Z	 eHjK                  ddg eeW      gdk       eeW      fdwedefd       ZeHjG                  ddg eeW      ged%       eeW      fdedefd       ZeHjG                  ddg eeW      gdk       eeW      fdeyd   dedee   fd       ZeHjK                  ddg eeW      gdk       eeW      fdwedefd       ZeHjG                  ddgd eeW      g      d        ZeHjG                  ddg eeW      gdk      defd       ZeHjG                  dd eeW      g      d        ZeHjG                  d eeW      g      dzefd       ZeHjG                  d eeW      g      d        Ze.j                  eH       e.j                  eZ       e.j                  e       e.j                  ev       e.j                  eƫ       e.j                  e       e.j                  eī       e.j                  e       e.j                  e       e.j                  e       e.j                  e       e.j                  e       e.j                  e       e.j                  e       e.j                  e       e.j                  e       e.j                  e\       e.j                  eI       e.j                  ey       e.j                  ee       e.j                  e       e.j                  e       e.j                  e       e.j                  e       y# e,$ rZ- e,de- d      dZ-[-ww xY w# e$ r dKZY w xY w# e$ r 	 d dlZn# e$ r Y nw xY wY @w xY w# e$ r Y w xY w(      N)datetime	timedelta)	TYPE_CHECKINGAnyListOptionalTuplecastget_args
get_originget_type_hints)Spanc                 \    | d| d|j                    d|  d}||j                  |       y y )N:: 
)__name__write)messagecategoryfilenamelinenofilelinetraceback_infos          W/var/www/html/sandstorm/venv/lib/python3.12/site-packages/litellm/proxy/proxy_server.pyshowwarningr   #   s>     z6("X->->,?r'"MN

>"     default)r   messagesz../..)AsyncIOSchedulerzMissing dependency z$. Run `pip install 'litellm[proxy]'`)z%'The thing I wish you improved is...'z'A feature I really want is...'z*'The worst thing about this product is...'z$'This product would be better if...'z 'I don't like how this works...'z&'It would help me if you could add...'z/'This feature doesn't meet my needs because...'z&'I get frustrated when the product...'c                     d} t        j                  t              }t                t        dd| z  z   dz          t        dd| z  z   dz          t        ddj	                  |      z          t        ddj	                  d      z          t        dd| z  z   dz          t        dd| z  z   dz          t                t        d	       t                t                t                t        d
       t                t                y )N<   z[1;37m#-z#[0m z[1;37mz# {:^59} #[0mz-https://github.com/BerriAI/litellm/issues/newz/ Thank you for using LiteLLM! - Krrish & IshaanzR[1;31mGive Feedback / Get Help: https://github.com/BerriAI/litellm/issues/new[0m)randomchoicelist_of_messagesprintformat)	box_widthr   s     r   generate_feedback_boxr,   K   s    I mm,-G	G	
sY
.
;<	
sY
.
;<	,,33G<
<=	

$
$%T
U	V 

sY
.
;<	
sY
.
;<	G	
;<	G	G	G	b 
G	Gr   )defaultdict)asynccontextmanager)Routerverbose_proxy_loggerverbose_router_logger)	DualCache
RedisCache)RejectedRequestError)SlackAlerting)!_get_parent_otel_span_from_kwargs get_litellm_metadata_from_kwargs)AsyncHTTPHandlerHTTPHandler)*router)log_db_metrics)check_response_size_is_safe)
JWTHandler)LicenseCheck)get_complete_model_listget_key_modelsget_team_models)user_api_key_authuser_api_key_auth_websocket)	html_form)get_logging_caching_headers3get_remaining_tokens_and_requests_from_request_datainitialize_callbacks_on_proxy)init_verbose_loggers)decrypt_value_helperencrypt_value_helper)_read_request_bodycheck_file_size_under_limit)!get_config_file_contents_from_gcsget_file_contents_from_s3)%remove_sensitive_info_from_deployment)
ProxyState)ERROR_RESPONSES)set_fine_tuning_config)init_guardrails_v2initialize_guardrails)perform_health_check)&_PROXY_VirtualKeyModelMaxBudgetLimiter)"_OPTIONAL_PromptInjectionDetection)_PROXY_failure_handler)_PROXY_track_cost_callback)add_litellm_data_to_request)user_update)delete_verification_tokensduration_in_secondsgenerate_key_helper_fn)update_team),get_disabled_non_admin_personal_key_creation)create_audit_log_for_update)set_files_config)!initialize_pass_through_endpoints)route_request)get_logging_payload)PrismaClientProxyLogging_cache_user_row_get_docs_url_get_projected_spend_over_limit_get_redoc_url_is_projected_spend_over_limit_is_valid_team_configsget_error_message_strget_instance_fn
hash_tokenreset_budgetupdate_spend)set_default_vertex_config)AssistantsTypedDict
DeploymentLiteLLM_ParamsModelGroupInfo)DefaultPrioritiesFlowItem	Scheduler)load_aws_kms)load_google_kms)
get_secretget_secret_boolget_secret_strstr_to_bool)SlackAlertingArgs)AnthropicMessagesRequestAnthropicResponse!AnthropicResponseContentBlockTextAnthropicResponseUsageBlock)HttpxBinaryResponseContent)DeploymentTypedDict)	ModelInfo)RouterGeneralSettingsupdateDeployment)CustomHuggingfaceTokenizer)StandardLoggingPayload)!get_end_user_id_for_cost_tracking)versionz0.0.0T)Union)DependsFastAPIFileFormHeaderHTTPExceptionPathQueryRequestResponse
UploadFilestatus)jsonable_encoder)CORSMiddleware)get_openapi)FileResponseJSONResponseORJSONResponseRedirectResponseStreamingResponse)	APIRouter)OAuth2PasswordBearer)APIKeyHeader)StaticFilesSERVER_ROOT_PATH premium_user+LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRIES'global_max_parallel_request_retries_env   #global_max_parallel_request_retries1LITELLM_GLOBAL_MAX_PARALLEL_REQUEST_RETRY_TIMEOUT-global_max_parallel_request_retry_timeout_envg      N@)global_max_parallel_request_retry_timeout/ui/u(   👉 [```LiteLLM Admin Panel on /ui```](z). Create, Edit Keys with SSOuB   

💸 [```LiteLLM Model Cost Map```](https://models.litellm.ai/).zr[**Customize Swagger Docs**](https://docs.litellm.ai/docs/proxy/enterprise#swagger-docs---custom-routes--branding)
DOCS_TITLEzLiteLLM APIDOCS_DESCRIPTIONzJEnterprise Edition 

Proxy Server to call 100+ LLMs in the OpenAI format. 

z5Proxy Server to call 100+ LLMs in the OpenAI format. c                  ,    d a d ad ad ad ad ad ad ad ad a	y N)

master_keyuser_config_file_pathotel_logginguser_custom_authuser_custom_auth_pathuser_custom_key_generateuser_custom_ssouse_background_health_checkshealth_check_intervalprisma_client r   r   cleanup_router_config_variablesr   w  s<     J L #O#'  Mr   c                  b  K   t        j                  d       t        r1t        j                  d       t        j	                          d {    t
        j                  &t
        j                  j	                          d {    t        j                          d {    t        t        j                          d {    dt
        j                  v r.	 ddlm}  | | j                  j                          t!                y t!                y 7 7 7 u7 U# t        $ r Y t!                y w xY ww)Nz"Shutting down LiteLLM Proxy ServerzDisconnecting from Prismalangfuser   langFuseLogger)r1   infor   debug
disconnectlitellmcachejwt_handlerclosedb_writer_clientsuccess_callbacklitellm.utilsr   Langfuseflush	Exceptionr   r   s    r   proxy_shutdown_eventr     s     BC""#>?&&(((}} mm&&(((



#$$&&& W---	4)''--/ $%#%/ 	) 	) 	'  	 $%	sl   AD/
D5D/ DD/D!D/>D?D/"D 8D/D/D/D/	D,D/+D,,D/appc                  K   dd l }t                t        dd       at        4t        dd       }t
        j                  |t        t               d {   at        j                  dj                  t                     t        du rt        j                         at        d      }t        d      }t        j                  d	|       |`t         j"                  j%                  |      r`t&        j)                  |
      rIt&        j+                  t,        |       d {   \  aaan |t3        |t4              r]t         j"                  j%                  |      r>t&        j)                  |
      r(t&        j+                  t,        |       d {   \  aaant         j6                  j9                  d      8t3        |t4              r(t&        j+                  t,        |       d {   \  aaanYt3        |t:              rt=        di | d {    n5 |j>                  |      }t3        |t:              rt=        di | d {    t
        jA                  t,        t        tB               t
        jE                  t0        t        t               tF        rtI        jJ                  tM                      tN        tN        jQ                  t,               t        j                  dt               t        /t        )t
        jS                  t        t        tT        t0               t        -tV        jX                  dkD  rt
        j[                  tT               t        ;t
        j]                  t0        t        t^        t`        tb        t               d {    t
        je                          d  tg                d {    y 7 q7 7 :7 7 7 7 B7 w)Nr   LITELLM_MASTER_KEYDATABASE_URL)database_urlproxy_logging_objuser_api_key_cachezElitellm.proxy.proxy_server.py::startup() - CHECKING PREMIUM USER - {}FWORKER_CONFIGCONFIG_FILE_PATHzworker_config: %sconfig_file_pathr=   r   LITELLM_CONFIG_BUCKET_NAME)
llm_routerr   redis_usage_cache)general_settingsr   r   r<   zprisma_client: %s)r   r   litellm_proxy_admin_namer   )litellm_proxy_budget_name)r   r   !proxy_budget_rescheduler_min_time!proxy_budget_rescheduler_max_timeproxy_batch_write_atr   r   )4jsonrK   r   r   r   ProxyStartupEvent_setup_prisma_clientr   r   r1   r   r*   r   _license_check
is_premiumr   ospathisfileproxy_configis_yamlload_configr   llm_model_listr   
isinstancestrenvirongetdict
initializeloads_initialize_startup_loggingr   _initialize_jwt_authr   asynciocreate_task_run_background_health_checkprompt_injection_detection_objupdate_environment_add_master_key_hash_to_dbr   r   
max_budget_add_proxy_budget_to_db$initialize_scheduled_background_jobsr   r   r   _init_dd_tracerr   )r   r   _db_urlworker_configenv_config_yamls        r   proxy_startup_eventr    sf      0$7J!+ND!A/DD /1 E 
 
 OVV	

 u%002 1;?0KM%34F%GO2MB"77>>/*|/C/C, 0D 0
 #..!O /  	  
	"}c*}-$$m$D #..!M /  	  ZZ^^89E*3K
 #..!M /  	  t,-}--- 'DJJ}5M-. 1=11111++ 2  **)#- +  $(*	
 &1&999L2MB Z%;44!'%=-	 	5 	
  W%7%7!%;11&> 	2 	

  DD-'.O.O!5/ E 
 	
 	
 %%' 
 
   e
8 .
 2N	
 !s   AOOCOOA0OO	AO&O''OO5OOEOO*O=O>OO	OOOOOO)docs_url	redoc_urltitledescriptionr   	root_pathlifespan)APIWebSocketRoutec            	         t         j                  rt         j                  S t        t         j                  t         j                  t         j
                  t         j                        } t         j                  D cg c]  }t        |t              s| }}|D ]  }|j                  j                  d      d   j                  d      }g }t        |d      rG|j                  j                  D ].  }|j                  |j                   d|j"                  ddid	       0 d
d|j                   xs | dd|j                   xs |j%                  dd       |dddiidgdi| d   |<    | t         _        t         j                  S c c}w )N)r  r   r  routes{r   ?	dependantquerytypestring)nameinrequiredschemar   zWebSocket: zWebSocket connection endpoint
websocket_/_101r  zWebSocket Protocol Switched	WebSocket)summaryr  operationId
parameters	responsestagspaths)r   openapi_schemar   r  r   r  r  r   r  r   splitrstriphasattrr  query_paramsappendr   r"  replace)r/  routewebsocket_routes	base_pathr+  params         r   get_openapi_schemar:  8  s{   
!!! iiOOzz	N ::E;L)M 
 " 
JJ$$S)!,33C8	 
5+&55 
!! %

%$)NN"H#		
 ()@y(AB>!+EJJ,U):K:KCQT:U+VW(#m5R%ST$	.
w	*'
< (CIs   :FFc                     t         j                  rt         j                  S t               } t        j                  j
                  }i }|D ]  }|| d   v s| d   |   ||<    || d<   | t         _        t         j                  S )Nr.  )r   r/  r:  LiteLLMRoutesopenai_routesvalue)r/  r=  paths_to_includer6  s       r   custom_openapir@  k  s    
!!!')N "//55M EN7++&4W&=e&DU#E /N7'Cr   DOCS_FILTEREDFalseTruec                       e Zd ZdZy)UserAPIKeyCacheTTLEnumr#   N)r   
__module____qualname__in_memory_cache_ttlr   r   r   rE  rE    s    r   rE  requestexcc           	         K   |j                   }t        |j                  rt        |j                        nt        j
                  d|j                  |j                  |j                  |j                  di|      S w)Nerrorr   r  r9  code)status_codecontentheaders)	rQ  r   rN  intr   HTTP_500_INTERNAL_SERVER_ERRORr   r  r9  )rI  rJ  rQ  s      r   openai_exception_handlerrT    sg      kkG XXCM6+P+P ;;	
  s   A9A;r;   _experimentalout/ui)	directoryhtmlui)r   z.htmlz
index.html)exist_okz8server_root_path is set, forwarding any /ui requests to PROXY_BASE_URLhttpc                    K   | j                   j                  j                  d      r9| j                   j                  j                  dt         dd      }t        |      S  ||        d {   S 7 w)NrW     )urlr   
startswithr5  server_root_pathr   )rI  	call_nextnew_paths      r   redirect_ui_middlewarere    sb     {{**51";;++33E>N=Os;SUVW'11"7++++s   A*A3,A1-A3)allow_originsallow_credentialsallow_methodsallow_headers)DictFr   r   r   r   callback_settingszapi_log.jsonr   r   )default_in_memory_ttl)
dual_cacher   queuezlitellm-proxy-budgetdefault_user_idall)adminrp  ui_access_modeiU  i]  
   r  store_model_in_dbopen_telemetry_logger)r   r   r   )
call_idmodel_id	cache_keyapi_baser   model_regionresponse_costhidden_params!fastest_response_batch_completionrequest_datauser_api_key_dictrv  rw  rx  ry  r   rz  r{  r|  r}  r~  returnc                    dd h}|xs i }||||||t        |      t        | j                        t        | j                        t        | j                        t        | j                        t        |j                  dd             t        |j                  dd             |	t        |	      nd d|j                         D ci c]  \  }}|t        |       c}}}|
r:t        |
      }|j                  |       t        |
      }|r|j                  |       	 |j                         D ci c]  \  }}||vr|t        |       c}}S c c}}w c c}}w # t        $ r$}t        j                  d|        i cY d }~S d }~ww xY w)Nr   _response_mslitellm_overhead_time_ms)x-litellm-call-idzx-litellm-model-idzx-litellm-cache-keyzx-litellm-model-api-basezx-litellm-versionzx-litellm-model-regionzx-litellm-response-costzx-litellm-key-tpm-limitzx-litellm-key-rpm-limitzx-litellm-key-max-budgetzx-litellm-key-spendzx-litellm-response-duration-mszx-litellm-overhead-duration-msz+x-litellm-fastest_response_batch_completionzError setting custom headers: )r   	tpm_limit	rpm_limitr	  spendr   itemsrI   updaterH   r   r1   rL  )r  rv  rw  rx  ry  r   rz  r{  r|  r}  r~  kwargsexclude_valueskvrQ  remaining_tokens_headerlogging_caching_headerskeyr>  es                        r   get_custom_headersr  	  s    $ZN!'RM$&($,$".#&}#5#&'8'B'B#C#&'8'B'B#C$'(9(D(D$E"#4#:#:;*-m.?.?PT.U*V*-8$?+

 1< 12'* "(
0A1c!f9
0+G. "U#
 	./"=l"K"NN23 &mmo
UN* UO
 	
 1

  ""%CA3#GH	s6   EE -EE E 	F E?9F?Fc                 L  K   t        j                          }t        j                          |z
  dk  rnt        j                  d       d{    | j                          d{   r|j	                          t        dd      t        j                          |z
  dk  rmyy7 V7 @w)am  
    Asynchronously checks if the request is disconnected at regular intervals.
    If the request is disconnected
    - cancel the litellm.router task
    - raises an HTTPException with status code 499 and detail "Client disconnected the request".

    Parameters:
    - request: Request: The request object to check for disconnection.
    Returns:
    - None
    X  r_  Ni  zClient disconnected the requestrO  detail)timer  sleepis_disconnectedcancelr   )rI  llm_api_call_task
start_times      r   check_request_disconnectionr  F  s      J
))+

"S
(mmA((*** $$&8  ))+

"S
(*s*   AB$	B 
B$!B"";B$B$"B$c                     ddl m} t        |       }|t        u r!t	        |       D ]  }t        ||      s|c S  yt        | t              rt        | t              r| S y)CResolve the actual TypedDict class from a potentially wrapped type.r   )_TypedDictMetaN)typing_extensionsr  r   r   r   r   r  r   )typr  originargs       r   _resolve_typed_dict_typer  c  sY    0_FC= 	C#~.
	
  
C	:c4#8
r   c                 
   t        |       }g }|t        u rJt        |       D ]:  }|t        |t	        d            rdt        |      vs*|j                  |       < |S t        | t              rt        | t              r| gS |S )r  NNoneType)r   r   r   r   r  r   r4  	BaseModel)r  r  typsr  s       r   _resolve_pydantic_typer  q  s}    _FDC= 	!C"3T
3c#h.C 	! K 
C	:c9#=uKr   use_azure_key_vaultc                 F   | du ry 	 ddl m} ddlm} t	        j
                  dd       }|t        d       |       } |||      }|t        _        t        j                  t        _        y # t        $ r+}t        |      }t        j                  d|       Y d }~y d }~ww xY w)	NFr   )DefaultAzureCredential)SecretClientAZURE_KEY_VAULT_URIzMError when loading keys from Azure Key Vault: AZURE_KEY_VAULT_URI is not set.)	vault_url
credentialztError when loading keys from Azure Key Vault: %s .Ensure you run `pip install azure-identity azure-keyvault-secrets`)azure.identityr  azure.keyvault.secretsr  r   getenvr   r   secret_manager_clientKeyManagementSystemAZURE_KEY_VAULT_key_management_systemr   r1   	exception)r  r  r  KVUrir  clientr  
_error_strs           r   load_from_azure_key_vaultr    s    e#
97 		/6=_  ,-
 *E(.%)<)L)L& 
V
&& C	
 	

s   A$A, ,	B 5!BB c                      t         ot        t        j                  t              rPt        j                  d       t        t        j                  vr$t        j                  j                  t               y y y y )Nz.setting litellm success callback to track cost)	r   r   r   _async_success_callbacklistr1   r   r\   r4  r   r   r   cost_trackingr    sZ     g55t< &&'WX*73R3RR//667QR S = !r   c                      t         ot        t        j                  t              rPt        j                  d       t        t        j                  vr$t        j                  j                  t               y y y y )Nz.setting litellm failure callback to track cost)	r   r   r   failure_callbackr  r1   r   r[   r4  r   r   r   error_trackingr    sZ     g..5 &&'WX&w/G/GG((//0FG H 6 !r   payloadspend_logs_urlc                 &   |q|ot        | d   t              r| d   j                         | d<   t        | d   t              r| d   j                         | d<   |j                  j	                  |        |S ||j                  j	                  |        |S )N	startTimeendTime)r   r   	isoformatspend_log_transactionsr4  )r  r   r  s      r   _set_spend_logs_payloadr    s    
  ^%?gk*H5#*;#7#A#A#CGK gi((3!(!3!=!=!?GI,,33G<  
	",,33G<r   c
           
        	K   	 t        j                  d d|  d d        | .t        | t              r| j	                  d      rt        |       n| fd}
fd}fd	}fd
}	fd}t        j                   |
              t        j                   |              t        j                   |              t        j                   |              t        du r |        d {    nt        j                  d       t        j                  d       y 7 0# t        $ r- t        j                  dt        j                                 Y y w xY ww)Nz&Enters prisma db call, response_cost: z	, token: z; user_id: z; team_id: sk-tokenc            	      v  K   t         j                         d{   } | t        | t              rt	        di | } 	 t
        g}t        j                  dkD  r|j                  t               |D ]9  }|t
        j                  j                  |d      z   t
        j                  |<   ; 5t
        j                  j                  d      z   t
        j                  <   yyy7 # t        $ rC}t        j                  ddt!        |       dt#        j$                          z          Y d}~yd}~ww xY ww)zu
            - Update that user's row
            - Update litellm-proxy-budget row (global proxy spend)
            r  Nr   z[91mz&Update User DB call failed to execute r   r   )r   async_get_cacher   r   LiteLLM_UserTabler   r   r	  r4  r   user_list_transactonsr   end_user_list_transactonsr   r1   r   r   	traceback
format_exc)existing_user_objuser_ids_idr  end_user_idr{  user_ids       r   _update_user_dbz(update_database.<locals>._update_user_db  sC     '9&H&HW&H&U U ,<Mt1T$5$J8I$J! , 'yH**Q. (AB' ? -"/"E"E"I"I#q"Q!R *??D #.)+EEII +Q &??L / -	 !V0  $))>s1vhbI]I]I_H`ab s@   D9C(!D9 8C* 9A,C* %D9*	D639D1,D91D66D9c                  ,  K   	 t        j                  d d d       y t        5t        j                  j	                  d      z   t        j                  <   y y # t
        $ r(} t        j                  dt        |               | d } ~ ww xY ww)Nz'adding spend to key db. Response cost: z	. Token: .r   z'Update Key DB Call failed to execute - )r1   r   r   key_list_transactonsr   r   r  r   )r  hashed_tokenr{  s    r   _update_key_dbz'update_database.<locals>._update_key_db  s     $**=m_IVbUccde  ' ,%'<<@@qQR "66|D -
  $..=c!fXF 	s2   BA  B:A  B 	B)#BBBc                  $  K   	 t         ;t              } | d<   t        | t        j                  d      t               a y y # t
        $ r=}t        j                  dt        |       dt        j                                 |d }~ww xY ww)N)r  response_objr  end_timer  r  SPEND_LOGS_URL)r  r  r   z)Update Spend Logs DB failed to execute - r   )r   rh   r  r   r  r   r1   r   r   r  r  )r  r  completion_responser  r  r  r{  r  s     r   _insert_spend_log_to_dbz0update_database.<locals>._insert_spend_log_to_db  s      ,1%%8#-!)$/G (5GG$$; '')yy1A'B&3%M -   $**?Axr)J^J^J`Iab 	s)   BA A B	B8BBBc                    K   	 t        j                  d d d       t        j                  d       y t        rt        j                  j	                  d      z   t        j                  <   	 d d } t        j
                  j	                  | d      z   t        j
                  | <   y y # t        $ r Y y w xY w# t        $ r=}t        j                  dt        |       d	t        j                                 |d }~ww xY ww)
Nz(adding spend to team db. Response cost: z. team_id: r  zAtrack_cost_callback: team_id is None. Not tracking spend for teamr   z	team_id::z::user_id::z#Update Team DB failed to execute - r   )r1   r   r   team_list_transactonsr   team_member_list_transactonsr   r   r   r  r  )team_member_keyr  r{  team_idr  s     r   _update_team_dbz(update_database.<locals>._update_team_db,  s%    $**>}o[Y`Xaabc ?(..[  ,%'==AA'1MN "77@
 -6gYk'*S)+HHLL / &BB?S -  %  $))9#a&IDXDXDZC[\ 	sV   D
3C D
:C 4<B2 0D
2	B>;C <D
=B>>C 	D
8DDD
c                    K   	 t        j                  dj                               t        j                  d       y t        5t        j                  j                  d      z   t        j                  <   y y # t        $ r=} t        j                  dt        |        dt        j                                 | d } ~ ww xY ww)Nz6adding spend to org db. Response cost: {}. org_id: {}.z?track_cost_callback: org_id is None. Not tracking spend for orgr   z"Update Org DB failed to execute - r   )r1   r   r*   r   org_list_transactonsr   r   r   r   r  r  )r  org_idr{  s    r   _update_org_dbz'update_database.<locals>._update_org_dbO  s     $**LSS%v
 >(..Y  ,%'<<@@KL "66v> -
  $))8Q9CWCWCYBZ[ 	s4   C<A> C:A> <C>	C8B??CCFzwdisable_spend_logs=True. Skipping writing spend logs to db. Other spend updates - Key/User/Team table will still occur.zRuns spend update on all tablesz Error updating Prisma database: )r1   r   r   r   ra  rs   r  r  disable_spend_logsr   r   r  r  )r  r{  r  r  r  r  r  r  r  r  r  r  r  r  r  r  s    `````````     @r   update_databaser    sA    g
""4]O9UGS^_f^ggrszr{|	
 E3!7E<L<LU<S%E2L L"	J	&	 	4 	F	. 	O-.N,-O-.N,-&)+++ %% J 	""#DE ,  
"".y/C/C/E.FG	

s;   E+C1D2 ?D0 /D2 /E+0D2 23E(%E+'E((E+r  r  r  r  parent_otel_spanc                 `  
K   g 
dt         dt        f
fd}
fd}
fd}
fd}	|  ||        d{     |        d{     |        d{     |	        d{    t        j                  t        j                  
d	|
             y7 c7 T7 E7 6w)z`
    Use this to update the cache with new user spend.

    Put any alerting logic in here.
    r  r{  c           	        K   t        | t              r| j                  d      rt        |       }n| }t	        j
                  d|       t        j                  |       d {   }t	        j
                  d|        t	        j
                  d|        |y |j                  }||z   }|j                  du r|j                  t        ||j                  d   	      d
u rt        ||j                  j                  dd       	      \  }}|j                  j                  dt        d            }t        |j                   xs d||j"                  ||j$                  ||      }	t'        j(                  t*        j-                  d|	             |'t/        |dd       |j0                  xs d}
|
|z   |_        |'t/        |dd       |j2                  xs d}||z   |_        ||_        j5                  ||f       y 7 w)Nr  r  z"_update_key_cache: hashed_token=%sr  z&_update_key_cache: existing_spend_obj=z#_update_key_cache: existing spend: Fsoft_budget)current_spendsoft_budget_limitTinfr   )r  r  	key_aliasr	  r  projected_spendprojected_exceeded_dateprojected_limit_exceeded)r  	user_info
team_spendr   team_member_spend)r   r   ra  rs   r1   r   r   r  r  soft_budget_cooldownlitellm_budget_tablero   rm   r   floatCallInfor  r  r  r  r  r   budget_alertsgetattrr  r  r4  )r  r{  r  existing_spend_objexisting_spend	new_spendr  r  
soft_limit	call_infoexisting_team_spendexisting_team_member_spendvalues_to_update_in_caches               r   _update_key_cachez'update_cache.<locals>._update_key_cache  s9    eS!e&6&6u&=%E2L L""#GVBTBdBdiuBdBv<v""45G4HI	
 	""12D1EF	
 %/55N"]2	 33u<"77C."+&8&M&M%'  8W'"4"I"I"M"M!4#84O4 ,@@DDuU|J !(..4",66%*22 /(?I !//3' 0  **L$?K"4"?"?"D1,?-,O) **,?FR);)M)M)RQR& +]: 0
 $- !((,8J)KLW =ws   A G?#G<$FG?c                     K   	g} 	 | D ]  }|t         j                  |       d {   }| y t        j                  d| d        t	        |t
              r|d   }n|j                  }|z   }t	        |t
              r||d<   
j                  ||f       ||_        
j                  ||j                         f        t         j                  dj                  t                     d {   }|y /|,|z   }
j                  dj                  t              |f       y y y 7 7 <# t        $ r@}t        j                  dt        |       dt        j                                 Y d }~y d }~ww xY ww)Nr  z!_update_user_db: existing spend: ; response_cost: r  z{}:spendz'An error occurred updating user cache: r   )r   r  r1   r   r   r   r  r4  r   r*   r   r   r   r  r  )r  r  r
  r  r  global_proxy_spend	incrementr  r{  r  r  s           r   _update_user_cachez(update_cache.<locals>._update_user_cache  s    9*	 W;+=+M+MRU+M+V%V"%-$**78J7KK\]j\kl 0$7%7%@N%7%=%=N*]:	 0$72;&w/-44c;M5NO/8&,-44c;M;R;R;T5UV3W6 (:'I'I%%&>? (J ( " ")*/A/M.>	)00&&'?@)L 0N*; &W."  	 &&9#a&iFZFZF\E]^ 	sg   E>!D2 D-D2 E>CD2 3D04D2 :E>;/D2 *E>-D2 0D2 2	E;;6E61E>6E;;E>c                     K   y dj                        } 	 t        j                  |        d {   }|y t        j                  d| d        |d}n"t        |t              r|d   }n|j                  }|z   }t        |t              r||d<   j                  | |f       y ||_        j                  | |j                         f       y 7 # t        $ r+}t        j                  dt        |              Y d }~y d }~ww xY ww)Nzend_user_id:{}r  z%_update_end_user_db: existing spend: r  r   r  +An error occurred updating end user cache: )r*   r   r  r1   r   r   r   r  r4  r   r   r  r   )r  r
  r  r  r  r  r{  r  s        r   _update_end_user_cachez,update_cache.<locals>._update_end_user_cache  s0    -"7%%k2	'9'I'Ic'I'R!R!)  &&78J7KK\]j\kl ")!"0$7%7%@N%7%=%=N&6I ,d3.7"7+)00#7I1JK+4"()00#7I7N7N7P1QR1 "S2  	 **=c!fXF 	sU   DC CC DA/C +D,(C DC 	D !DDDDc                    K   y dj                        } 	 t        j                  |        d {   }|y t        j                  d| d        |d}n"t        |t              r|d   }n|j                  }|d}|z   }t        |t              r||d<   j                  | |f       y ||_        j                  | |f       y 7 # t        $ r+}t        j                  dt        |              Y d }~y d }~ww xY ww)Nz
team_id:{}r  z!_update_team_db: existing spend: r  g        r  r  )r*   r   r  r1   r   r   r   r  r4  r   r  r   )r  r
  r  r  r  r{  r  r  s        r   _update_team_cachez(update_cache.<locals>._update_team_cache5  s5    ?m3!!'*"	 )88S8AA  ") &&34F3GGXYfXgh ")250$7%7%@N%7%=%=N%!$&6I ,d3.7"7+)00#7I1JK+4"()00#7I1JK7 B8  	 **=c!fXF 	sU   DC CC DA3C /D0C 
DC 	D!C<7D<DDN)r  r{  r#   )
cache_listttllitellm_parent_otel_span)r   r  r  r  r   async_set_cache_pipeline)r  r  r  r  r{  r  r  r  r  r  r  s    ````     @r   update_cacher!  y  s      8:RMs RM5 RMj-`#L'R ]6e=III """$&&& """330%5 	4 	
 	J 	# 	' 	#sG   ;B.B&B.B(B.$B*%B.5B,61B.(B.*B.,B.c                      	 ddg} t        t        j                  d      5 }t        j                  | ||       d d d        y # 1 sw Y   y xY w# t
        $ r#}t        j                  d| d       Y d }~y d }~ww xY w)Nollamaservew)stdoutstderrzd
            LiteLLM Warning: proxy started with `ollama` model
`ollama serve` failed with Exceptionz). 
Ensure you run `ollama serve`
        )openr   devnull
subprocessPopenr   r1   r   )commandr)  r  s      r   run_ollama_server-  s  s    

W%"**c" 	FgWWWE	F 	F 	F 
""eefdg h		
 	

s3   A AA AA A 	A:A55A:c                  h  K   t        j                  t              } | y	 t        | t               d{   \  }}|t
        d<   |t
        d<   t        |      t
        d<   t        |      t
        d<   t        5t        t        t              r!t        j                  t               d{    7 y7 w)z}
    Periodically run health checks in the background on the endpoints.

    Update health_check_results, based on this.
    N)
model_listdetailshealthy_endpointsunhealthy_endpointshealthy_countunhealthy_count)copydeepcopyr   rX   health_check_detailshealth_check_resultslenr   r   r  r  r  )_llm_model_listr1  r2  s      r   r  r    s      mmN3O
7K&0D8
 2
..
 5F016I23034E0F_-256I2J./ ,!52
 -- 5666 2
 7s"   2B2B.A3B2(B0)B20B2c            	          e Zd ZdZd3dZdedefdZdedefdZ		 d4de
e   defd	Zd
ededefdZdefdZ	 d5d
edededefdZdedee   defdZdefdZdefdZd4de
e   defdZd
efdZd Zde
ej2                     defdZdefdZde
e   fd Zd6defd!Zd"edefd#Z d"edefd$Z!d%ed&e"fd'Z#d(e
e$   fd)Z%d*ed+ed,e&defd-Z'd.e(d
ed/e
e   fd0Z)d.e(defd1Z*d.e(d&e"fd2Z+y)7ProxyConfigz}
    Abstraction class on top of config loading/updating logic. Gives us one place to control all config updating logic.
    r  Nc                     i | _         y r   config)selfs    r   __init__zProxyConfig.__init__  s	    &(r   r   c                     t         j                  j                  |      syt         j                  j                  |      \  }}|j	                         dk(  xs |j	                         dk(  S )NFz.yamlz.yml)r   r   r   splitextlower)r@  r   r&  file_extensions       r   r   zProxyConfig.is_yaml  sW    ww~~./GG,,-=>>##%0TN4H4H4Jf4TTr   	file_pathc           	          	 t        |d      5 }t        j                  |      xs i cddd       S # 1 sw Y   yxY w# t        $ r}t        d| dt	        |             d}~ww xY w)z,
        Load and parse a YAML file
        rNzError loading yaml file r   )r(  yaml	safe_loadr   r   )r@  rF  r   r  s       r   _load_yaml_filezProxyConfig._load_yaml_file  sh    	Ni% 2~~d+1r2 2 2 	N6ykCF8LMM	Ns)   = 1	= := = 	A%A  A%c           	        K   |xs t         }||a t        j                  j                  |       r,t	        | d      5 }t        j                  |      }ddd       n|t        d|       g i i i d}| j                  t        j                  j                  t        j                  j                  |xs d                  }t        j                  dt        j                  |d	              |S # 1 sw Y   xY ww)
z
        Given a config file path, load the config from the file.
        Args:
            config_file_path (str): path to the config file
        Returns:
            dict: config
        NrH  zConfig file not found: )r/  r   router_settingslitellm_settingsr   )r?  base_dirzloaded config=   indent)r   r   r   existsr(  rI  rJ  r   _process_includesdirnameabspathr1   r   r   dumps)r@  r   rF  config_filer?  s        r   _get_config_from_filez!ProxyConfig._get_config_from_file  s      %=(=	'$4! 77>>YK)c* 5k45 5"5i[ABB !$&#%$&	F ''BGGOOBGGOOIOQS4T$U ( 
 	""^DJJva4P3Q#RS%5 5s   <D C4B D 4C=9D r?  rO  c                    d|vr|S t        |d   t              st        d      |d   D ]  }t        j                  j                  ||      }t        j                  j                  |      st        d|       | j                  |      }|j                         D ]3  \  }}t        |t              r||v r||   j                  |       /|||<   5  |d= |S )au  
        Process includes by appending their contents to the main config

        Handles nested config.yamls with `include` section

        Example config: This will get the contents from files in `include` and append it
        ```yaml
        include:
            - model_config.yaml

        litellm_settings:
            callbacks: ["prometheus"]
        ```
        includez&'include' must be a list of file pathszIncluded file not found: )r   r  
ValueErrorr   r   joinrS  FileNotFoundErrorrK  r  extend)r@  r?  rO  include_filerF  included_configr  r>  s           r   rT  zProxyConfig._process_includes  s     F"M&+T2EFF #9- 	(LX|<I77>>),'*CI;(OPP"229=O-335 (
UeT*sf}3K&&u-"'F3K	(	( 9r   
new_configc                 :  K   	 t         Pt        j                  dd      du st        r2|j	                  dd        t         j                  |d       d {    y t        t         d      5 }t        j                  ||d       d d d        y 7 8# 1 sw Y   y xY ww)	Nrt  FTr/  r?  )data
table_namer%  )default_flow_style)
r   r   r   rt  popinsert_datar(  r   rI  dump)r@  rb  rX  s      r   save_configzProxyConfig.save_config  s     	 $  !4e<D  NN<.+++QQQ ./#6 M+		*keLM M	 RM Ms*   ABBB+B
BBBdepth	max_depthc                    ||kD  rt        j                  d| d       |S |j                         D ]  \  }}t        |t              r| j                  ||dz   |      ||<   1t        |t              r0|D ]*  }t        |t              s| j                  ||dz   |      }, qt        |t              s|j                  d      st        |      ||<    |S )a  
        Check for os.environ/ variables in the config and replace them with the actual values.
        Includes a depth limit to prevent infinite recursion.

        Args:
            config (dict): The configuration dictionary to process.
            depth (int): Current recursion depth.
            max_depth (int): Maximum allowed recursion depth.

        Returns:
            dict: Processed configuration dictionary.
        zMaximum recursion depth (z") reached while processing config.r_  )r?  rk  rl  os.environ/)
r1   warningr  r   r   _check_for_os_environ_varsr  r   ra  r   )r@  r?  rk  rl  r  r>  items          r   rp  z&ProxyConfig._check_for_os_environ_vars  s     9 ((+I;6XY M ,,. 	0JC%&"== 	Y > s E4(! D!$-#>>#'uqyI  ?   E3'E,<,<],K(/s	0 r   r  all_teams_configc                     i }|D ]   }d|vrt        d|       ||d   k(  s|} n |j                         D ]6  \  }}t        |t              s|j	                  d      s)t        |      ||<   8 |S )Nr  zteam_id missing from team: rn  )r   r  r   r   ra  r   )r@  r  rr  team_configteamr  r  s          r   _get_team_configzProxyConfig._get_team_config@  s    $ 	D$"=dV DEE$y/)"	  %%' 	/DAq!S!all=&A!+AA	/ r   c                     | j                         }|j                  di       }|j                  dd      }|i S | j                  ||      }|S )z^
        - for a given team id
        - return the relevant completion() call params
        rN  default_team_settingsN)r  rr  )get_config_stater   rv  )r@  r  r?  rN  rr  rt  s         r   load_team_configzProxyConfig.load_team_configM  sh     &&( "::&8"=+//0GN#I++.> , 
 r   cache_paramsc                 "   ddl m} d|v r|d   t         _        d|v r|d   t         _         |di |t         _        t         j                  Dt        t         j                  j                  t              rt         j                  j                  ay y y )Nr   Cacherl  default_redis_ttldefault_in_redis_ttlr   )r   r~  rl  r  r   r   r4   r   )r@  r{  r~  s      r   _init_cachezProxyConfig._init_cache`  sx    
 	""l2,89P,QG),.(45K(LG%--==$GMM4G4G)T ' 3 3 *U$r   c                   K   t         j                  j                  d      t         j                  j                  d      }t         j                  j                  d      }t         j                  j                  d      }t        j                  d||       |dk(  rt        ||       d{   }nt        ||      }|%t        d      | j                  |	       d{   }t        $| j                  |t        t        
       d{   }t        j                  |      }|j                  dd       | j                  |      }| j!                  |       |S 7 7 }7 Uw)a  
        Load config file
        Supports reading from:
        - .yaml file paths
        - LiteLLM connected DB
        - GCS
        - S3

        Args:
            config_file_path (str): path to the config file
        Returns:
            dict: config

        r   N LITELLM_CONFIG_BUCKET_OBJECT_KEYLITELLM_CONFIG_BUCKET_TYPEzbucket_name: %s, object_key: %sgcs)bucket_name
object_keyz(Unable to load config from given source.r   )r?  r   rt  environment_variablesr>  )r   r   r   r1   r   rP   rQ   r   rY  r   _update_config_from_dbrt  r5  r6  rg  rp  update_config_state)r@  r   r  r  bucket_typer?  printed_yamls          r   
get_configzProxyConfig.get_configs  sJ    $ ::>>67C**..)EFK(JKJ**..)EFK &&1;
 e#@ +
   3 +
 ~ JKK  55GW5XXF$66+"3 7  F }}V,0$7000?   /9 Ys7   B)E"+E,4E" E!)E"
E AE"E" E"c                     || _         y r   r>  )r@  r?  s     r   r  zProxyConfig.update_config_state  s	    r   c                     	 t        j                  | j                        S # t        $ r;}t	        j
                  dj                  | j                  |             i cY d}~S d}~ww xY w)z
        Returns a deep copy of the config,

        Do this, to avoid mutating the config state outside of allowed methods
        z^ProxyConfig:get_config_state(): Error returning copy of config state. self.config={}
Error: {}N)r5  r6  r?  r   r1   r   r*   )r@  r  s     r   ry  zProxyConfig.get_config_state  sV    	==-- 	 &&qxxKK
 I	s   ! 	A%0A A% A%r=   c                   K   | j                  |       d{   }|j                  dd      }|rv|j                         D ],  \  }}t        t	        ||            t
        j                  |<   . d|v r3t        j                  dd      t        _	        t        j                         a|j                  dd      a|j                  dd      }|i }|rd}d	}	|j                         D ]  \  }}|d
k(  r|du rt        | d       ddlm}
 i }d|v r|d   }|j!                  |       |j                  dd      }t#        j$                  d|       |dk(  s|dk(  rt'        |j)                               dk(  rt	        dd      }t	        dd      }d}|j!                  |||d       t	        dd      t	        dd      }|j!                  d|i       t#        j$                  d||	|       t#        j$                  d||	|       t#        j$                  d||	|       t#        j$                  d||	|       |dk(  r|j!                  ddi       |j                         D ]7  \  }}t+        |      t        u s|j-                  d      s*t	        |      ||<   9 | j/                  |       t0        j2                  t#        j$                  | d |	        |d
k(  r|d!u r|d"k(  r t5        |t        ||#      }|t0        _        |d$k(  rt9        |t        ||%       6|d&k(  r?t;        ||'      gt0        _        t#        j$                  d(t0        j<                          z|d)k(  rt?        |      t0        _         |d*k(  rAt?        |      t0        _!        t0        j@                  t0        jB                  t0        _         |d+k(  r<dd,l"m#} |D cg c]  }|d-   t;        |d.   |'      d/ c}t0        _$         |        |d0k(  rg t0        _%        |D ]  }d1|v r*t0        jJ                  jM                  t;        |2             1t0        jJ                  jM                  |       d3|v sUt        s#tO        tP        jR                  jT                        t#        j$                  d4       dd5l+m,}  |       }tZ        j]                  d6|        t        | d7t0        jJ                   d8|	        	|d9k(  rg t0        _/        |D ]O  }d1|v r*t0        j^                  jM                  t;        |2             1t0        j^                  jM                  |       Q t        | d:t0        j^                   d8|	        |dk(  r|d;k(  rQta        |      D ]  \  }}	 tc        d|i |  t#        j$                  | d>| d?| |	        ti        t0        ||       |d@k(  r|ote        |tf              r_|j                         D ]6  \  }}te        |t              s|j-                  d      s)t	        |      ||<   8 tk        d|i |t0        _6        ctO        dA|       t#        j$                  | d>| d?| |	        ti        t0        ||        |j                  dBi       }|i }|r|j                  dCd      }| jo                  |D       |j                  dEd      }|tq        d|i |t0        _9        |j                  dFd!      }tu        |G       |j                  dHd!      }tw        |I       | jy                  |J       |j                  dKd      } | rG| j-                  d      r6t#        j$                  dL       t	        |       } t#        j$                  dM|        |j                  dNt	        dOd            a=tz        r_tz        j-                  d      rJt	        tz              a=te        tz        t              s'tO        dPj}                  t+        tz                          tz        #te        tz        t              rt        tz              a@|j                  dQd      }!|! t        j                  t?        |!      dR       |j                  dSd!      aCt        d!aC|j                  dTd      }"|"t;        |"|'      aD|j                  dUd      }#|#t;        |#|'      aE|j                  dVd      }$|$t;        |$|'      aF|j                  dWd      t        |dW   X       d{    |j                  dYdZ      aH|j                  d[d      }%|%t        d!u rt        d\      |j                  d]t              aJ|j                  d^t              aK|j                  d_t              aL|j                  d`t              aM|j                  dad!      aN|j                  dbdc      aO|j                  ddd      aP|j                  de      .t        dur&t        dftP        jR                  jT                  z         dg|v r"|dg   t        _	        t        j                         adht0        j2                  dui}&|j                  did      }'|'r|'|&di<   t        dj       |'D ]  }(|(dk   j                         D ]9  \  })}*te        |*t              s|*j-                  d      s)t	        |*      |(dk   |)<   ; t        dl|(j                  dmdn       d	       |(dk   do   }+|(dk   j                  dpd      },dq|+v s|,t                 d}-|j                  drd      }.|.rx|.dk   j                         D ]W  \  })}*te        |*t              s|*j-                  d      s)|*j                  ddn      }t        j                  |      }*|*|.dk   |)<   Y t        d|i |.}-|j                  dsd      }/t        |/t       |j                  dud      }0t        |0t       |j                  dvd      }1t        |1t       |j                  dwd      }2|2rwte        |2tf              rgt        j                  t0        j                        }3dxdih}4|3j                  D 5cg c]	  }5|5|4vs|5 }6}5|2j                         D ]  \  })}*|)|6v s|*|&|)<    t1        j                  d|i |&|-t        dy      dz}d}7||j                  d"d      }7|7rt        |7|{       ||j                         |fS 7 c c}w # tN        $ rJ te        |tf              rtO        d<| d=|j)                                tO        d<| d=t+        |             w xY w7 :c c}5w w)}z<
        Load config values into proxy global state
        r   Nr  )secret_namedefault_valueLITELLM_LICENSErk  rN  z[94mz[0mr   Tz
Setting Cache on Proxyr   r}  r{  r  rediszpassed cache type=%szredis-semantic
REDIS_HOST
REDIS_PORT)r  hostportREDIS_PASSWORDpasswordz%sCache Type:%s %sz%sCache Host:%s %sz%sCache Port:%s %sz%sCache Password:%s %sredis_semantic_cache_use_asyncrn  )r{  zSet Cache on LiteLLM ProxyF
guardrails)guardrails_configr   r   rN  	callbacks)r>  r   r   rN  post_call_rules)r>  r   zlitellm.post_call_rules: max_internal_user_budget default_max_internal_user_budgetcustom_provider_map)custom_llm_setupprovidercustom_handler)r  r  r   r  r>  
prometheusz'Starting Prometheus Metrics on /metrics)make_asgi_appz/metricsz! Initialized Success Callbacks - r%   r  z! Initialized Failure Callbacks - rx  z4team_id missing from default_team_settings at index=z
passed in value=z setting litellm.=upperbound_key_generate_paramsz=Invalid value set for upperbound_key_generate_params - value=r   key_management_system)r  key_management_settingsuse_google_kmsr  r  r  )r   r   zGOING INTO LITELLM.GET_SECRET!zRETRIEVED DB URL: %sr   r   z.Master key must be a string. Current type - {}user_api_key_cache_ttl)rl  r  rt  custom_authcustom_key_generate
custom_ssopass_through_endpointsr  rr  rp  allowed_ipsz\allowed_ips is an Enterprise Feature. Please add a valid LITELLM_LICENSE to your envionment.r   r   r   r  background_health_checksr   ,  r7  enforced_paramszTrying to use `enforced_params`litellm_licensecache_responsesr/  z<[32mLiteLLM: Proxy initialized with Config, Set models:[0mlitellm_paramsz	[32m    
model_namer   modelry  r#  assistant_settingsfinetune_settingsr>  files_settingsdefault_vertex_configrM  r@  async_only_mode)assistants_configrouter_general_settings)all_guardrailsr   r   )^r  r   r  r   r   r   r   r  r   license_strr   r   rk  r)   litellm.caching.cachingr~  r  r1   r   r9  keysr  ra  r  r   r   rW   guardrail_name_config_maprJ   rr   r  r  r  r  r   r  r  r   r4  r   CommonProxyErrorsnot_premium_userr>  prometheus_clientr  r   mountr  	enumerateTeamDefaultSettingsr   r   setattr#LiteLLM_UpperboundKeyGenerateParamsr  initialize_secret_managerKeyManagementSettings_key_management_settingsr   r  _load_alerting_settingsr   r*   rs   litellm_master_key_hashr   update_cache_ttlrt  r   r   r   rf   rr  r\  r   r   r   r  r   r   r7  r-  r5  rw   rU   re   rv   inspectgetfullargspecr/   argsr   rV   get_model_list)8r@  r=   r   r?  r  r  r>  rN  blue_color_codereset_color_coder~  r{  cache_params_in_config
cache_type
cache_host
cache_portcache_passwordr  r  rq  callbackr  metrics_appidxteam_setting_k_vr   r  r  r  r  r   r  r  r  r  r  router_paramsr/  r  r  r  litellm_model_namelitellm_model_api_baser  r  finetuning_configfiles_configr  rM  arg_specexclude_argsxavailable_argsguardrails_v2s8                                                           r   r   zProxyConfig.load_config  s     "__>N_OO !'

+BD I 399; X
U"%jSPU&V"W

3X !$99-/YY7H$-O*-88: #JJ':DA "::&8$?#!(O(.446 R1
U'>etm_--EFG=#%L%)991A.1Q.$++,BC!-!1!1&'!BJ(../EzR #g-?O1Ol//12a7%/d%C
%/d%C
)-$++(2(2(2 &&6=I-78H$-ON(//$.!" -220+,&	 -220+,&	 -220+,&	 -224+,*	 "%55$++-Mt,TU '3&8&8&: B
U;#-%2B2B=2Q0:50AL-B
 $$,$?}}0,22.//IJZI[\ G^L(0E*/%1)9)9	1- 9RG5K'1#%1)9)9	 --'eFVW/G+ )..3G4K4K3LM 667<U|G4>>?DU|G<77?#DD  8 11> %*	3 ! )-Z(8.=&*+;&<1A/	3G/ %&../1G, %* C(?#44;; /h ?
 $44;;HE+x7'3*3(9(J(J(P(P+& %& !5 : :$M!" !L /<o #		*k B+C, *++LWMeMeLffghxgyz ../1G, %* F(?#44;; /h ?
 $44;;HEF *++LWMeMeLffghxgyz N* 33-6. )\	/?,?	 )..*++<SE5'JZI[\ GS%0<<(Zt-D&+kkm ;FB)"c2r}}]7S,6rNb	; @H%H  > ([\a[bc  )..*++<SE5'JZI[\ GS%0eR1j "::&8"=#!$4$8$89PRV$W!**AV*W&6&:&:)4'# '23H 4-40 .112BEJN>:"2"6"67Le"T%:MN((:J(K+//EL 7 7 F$**+KL),7$**+A<P)--j)=tDJ j33MB'
3
!*c2#HOO ,  %*Z*E*4Z*@'%5%9%9($&" &1"33*/0F*G&* 4 
 !1 4 45H% P ($)! +..}dCK&#2%8H$  #3"6"67Ld"S".+:-@P,( *--lDAJ%"1$7G#
  ##$<dCO7+;<T+U   .11 %N +..}dCK&<5+@ r  1A0D0D35V1- 1A0D0D35V1- $4#7#7&(<$  "2!5!5$&8"
 ,<+?+?*E,( %5$8$89PRU$V!#3#7#78NPT#U  !$$%67C , 5'88>>?  !$44-=>O-P*-88: w}} 

 ZZd3
*4M,'T $ 	'!"2399; CDAq!!S)all=.I5?]./2C UYY|R%@$AIJ%*+;%<W%E")./?)@)D)DZQU)V&116L6T$&	' <@#ZZ(<dC*+;<BBD @1a%!,,}*E="5B		"A>?&'78;	@
 !4 I6H I #JJ':DA&78 zz"2D9- !'

+BD I!)>? !**%6=z/4@--gnn=H L
 *2PA!<:OaPNP'--/ )1&'(M!$)  

/$9 $%
 /3"JJ|T:M,?O v,,.0@@@y PV	3F  ) ),=&/&Z[^Z__qr~  sD  sD  sF  rG  %H'" !" #,"VWZV[[mnrs  oA  nB  !C# \T Qs   us3Iu0u2u6C.u&*us6-A;u)D*us;A1uu#K4uuFu/uAuuAu$u6C/u%	u/u3uA$u6u;Auuur   c                    ddl m} |j                  dd      }t        j                  d|        |y|D ]  }|dk(  rpt
        j                  |j                  dd      |j                  dd      |j                  d	d      |j                  d
d      |j                  dd      t               x|t        j                  v s ||ddd|j                  dd      i      }|t        j                  j                  |        y)z.
        Initialize alerting settings
        r   )$_init_custom_logger_compatible_classalertingNz_alerting_callbacks: slackalerting_thresholdr  alert_typesalert_to_webhook_urlalerting_args)r  r  r  r  r  redis_cache)logging_integrationinternal_usage_cacher   custom_logger_init_args)*litellm.litellm_core_utils.litellm_loggingr  r   r1   r   r   update_valuesr   r   )_known_custom_logger_compatible_callbacksr  r4  )r@  r   r  _alerting_callbacks_alert_loggers         r   r  z#ProxyConfig._load_alerting_settings  s   	
 /22:tD""%:;K:L#MN&) 	:F !//-11*dC'7';';<PRU'V 0 4 4]D I)9)=)=.* #3"6"6"M 1 0 	 WNNNB,2-1#'+-=-A-A/SW-X1	G *))0093	:4 	r   r  c                 4   ||t         j                  j                  k(  rt        d       y|t         j                  j                  k(  rt        d       y|t         j                  j                  k(  rddlm} |j                  d       y|t         j                  j                  k(  rt        d       y|t         j                  j                  k(  rdd	lm}  |        y|t         j                  j                  k(  rdd
lm}  |        yt%        d      y)z_
        Initialize the relevant secret manager if `key_management_system` is provided
        NTr  r  r   )AWSSecretsManagerV2)use_aws_secret_manager)use_aws_kms)GoogleSecretManager)HashicorpSecretManagerz&Invalid Key Management System selected)r  r  r>  r  
GOOGLE_KMSr   AWS_SECRET_MANAGER-litellm.secret_managers.aws_secret_manager_v2r  load_aws_secret_managerAWS_KMSr~   GOOGLE_SECRET_MANAGER-litellm.secret_managers.google_secret_managerr  HASHICORP_VAULT0litellm.secret_managers.hashicorp_secret_managerr  r\  )r@  r  r  r  r  s        r   r  z%ProxyConfig.initialize_secret_manager  s     !,$(;(K(K(Q(QQ)dC&*=*H*H*N*NNt4%&99??@ $;;SW;X&*=*E*E*K*KK.%)<)R)R)X)XX $%&*=*M*M*S*SS '( !IJJC -r   c                    t        |dd      }|||j                  d<   d|j                  d<   t        du rht        |dd      |j                  d<   t        |dd      |j                  d<   t        |dd      |j                  d<   t        |d	d      |j                  d	<   |j                  t        |j                  t              rld|j                  vr|j
                  |j                  d<   d|j                  v r |j                  d   d
u r||j                  d<   t        di |j                  }|S t        |j
                  |      }|S )z
        Common logic across add + delete router models
        Parameters:
        - deployment
        - db_model -> flag for differentiating model stored in db vs. config -> used on UI

        Return model info w/ id
        rw  NidTdb_model
created_at
updated_at
created_by
updated_byF)r  r  r   )r	  
model_infor   r   r   rw  RouterModelInfo)r@  r  r  r  _model_infos        r   get_model_info_with_idz"ProxyConfig.get_model_info_with_id  sD    %UJ=?%(ET"+/EZ(4-4UL$-OE\*-4UL$-OE\*-4UL$-OE\*-4UL$-OE\*'Ju7G7G,N5+++).  &U---%2B2B:2NRW2W/7  ,)=E,<,<=K  *U^^hOKr   	db_modelsc                   K   g }t         t        |      dk(  ry|D ]<  }| j                  |      }|j                  "|j	                  |j                         > | j                  t               d{   }|j                  dd      }|r|D ]  }|d   j                         D ]9  \  }}	t        |	t              s|	j                  d      s)t        |	      |d   |<   ; |j                  di       j                  d	d      }
|
t         j                  |d
   |d         }
|j	                  |
        t         j                         }d}|D ]%  }
|
|vst         j                  |
      }|!|dz  }' |S 7 w)a5  
        (Helper function of add deployment) -> combined to reduce prisma db calls

        - Create all up list of model id's (db + config)
        - Compare all up list to router model id's
        - Remove any that are missing

        Return:
        - int - returns number of deleted deployments
        Nr   r  r   r/  r  rn  r  r  r  )model_groupr  r  r_  )r   r9  r   r  r4  r  r   r   r  r   r   ra  r   _generate_model_idget_model_idsdelete_deployment)r@  r!  combined_id_listmr  r?  r/  r  r  r  rw  router_model_idsdeleted_deployments
is_deleteds                 r   _delete_deploymentzProxyConfig._delete_deployment  s      Y1!4  	7A4414=J}}( ''
6	7 8MNNZZd3
# 2!"2399; CDAq!!S)all=.I5?]./2C
 !99\26::4F#)<<$),$7',-='>  =  H !''12  &335  ( 	-H//'99X9F
)'1,'		-
 #"9 Os0   ;E:6E:4E75AE:<E:BE:E:.
E:c                 `   ddl }t        t        t        t              st	        dt               t
        yd}|D ]  }|j                  }t        |t              rq|j                         D ]R  \  }}t        |t              st        |      }|t	        dj                  |            t        |      dkD  sN|||<   T t        di |}nt        j                  d|        | j                  |d      }	t
        j!                  t#        |j$                  ||		      
      }
|
|dz  } |S )z
        Iterate through db models

        for any not in router - add them.

        Return - number of deployments added
        r   N7Master key is not initialized or formatted. master_key=r  zUnable to decrypt value={}HInvalid model added to proxy db. Invalid litellm params. litellm_params=T)r  r  r  r  r  )
deploymentr_  r   )base64r   r   r   r   r   r  r   r  rL   r*   r9  ry   r1   rL  r   upsert_deploymentrx   r  )r@  r!  r4  added_modelsr*  _litellm_paramsr  r  _valuer  addeds              r   _add_deploymentzProxyConfig._add_deployment+	  sU    	Z
C%@I*V   !	"A..O/40+113 8DAq!!S)!5A!>!>"+,H,O,OPQ,R"SSv;?17OA.8 #1"C?"C %**^_n^op 55$ 6 K 00% ||#2* 1 E  !C!	"D r   
new_modelsr   c           	      .  K   dd l }	 t        Ht        At        j                  dt        |              g }|D ]  }|j                  }t        |t              r5|j                         D ]  \  }}t        |      }	|	||<    t        di |}nt        j                  d|        m| j                  |      }
|j                  t        |j                   ||
      j#                  d              t        |      dkD  rt        j                  d	|        t%        j&                  |t)        d
            at        j                  dt                nMt        j                  dt        |              | j+                  |       d {    | j-                  |       t        t        j5                         at8        j;                          d {   }|j=                  di       xs i }|j=                  dd       }|j=                  dd       }|It        |t>              r9|D ]4  }|t$        j@                  vst$        j@                  j                  |       6 |It        |t>              r9|D ]4  }|t$        jB                  vst$        jB                  j                  |       6 |j=                  di       }|j                         D ]'  \  }}	 t        |      }	|	|	tD        jF                  |<   ) t        itH        ctH        jJ                  jL                  jO                  ddi       d {   }|-|jP                  !|jP                  }t        jR                  di | |j=                  di       }d|v r!tT        tT        j=                  dd       t        tT        d   t>              rm|j=                  dd       [t        |d   t>              rHt        j                  d       |d   tT        d<   tT        d   |_+        tT        d   |jX                  _+        ntT        5i a*|d   tT        d<   tT        d   |_+        tT        d   |jX                  _+        nFt        tT        t              r2|d   tT        d<   tT        d   |_+        tT        d   |jX                  _+        d|v rB|d   tT        d<   tT        d   |_-        |jX                  j]                  tT        d   t               d|v r5|d   tT        d<   |jX                  j]                  tT        d   t               y y 7 # t.        $ r,}t        j0                  dt3        |              Y d }~d }~ww xY w7 # t.        $ r+}t        j                  d|t3        |             Y d }~d }~ww xY w7 aw)Nr   zlen new_models: r  r1  r#  r2  Texclude_nonez_model_list: r  )r/  r  zupdated llm_router: )r!  z+Error adding/deleting model to llm_router: rN  r   r  r  z#Error setting env variable: %s - %s
param_namerM  wherer   r  z?Overriding Default 'alerting' values with db 'alerting' values.r  )r  r   r  )r  r   r   )/r4  r   r   r1   r   r9  r  r   r   r  rL   ry   rL  r   r4  rx   r  to_jsonr   r/   r   r.  r:  r   r  r   r  r   r   r  r   r  r   r  r   r   r   dblitellm_config
find_firstparam_valueupdate_settingsr   r  slack_alerting_instancer  r  )r@  r;  r   r4  _model_listr*  r7  r  r  decrypted_valuer  r  config_datarN  success_callbacksfailure_callbacksr   r  r  db_router_settings_router_settings_general_settingss                         r   _update_llm_routerzProxyConfig._update_llm_routerc	  s     	/	!j&<$**-=c*o=N+OP$&# A&'&6&6O!/48$3$9$9$; ADAq.B.KO1@OA.A +9*K?*K,22fgvfwx !"&"="=A"="FK&&"'(||+:'2 "'t'4, {#a'(..{m/LM!(#.0E,01"J )..1Ej\/RS$**-=c*o=N+OP--
-CCC $$z$: !'668N )3355&??+=rBHb,001CTJ,001CTJ(Z8I4-P$5 F #7+C+CC,,334DEF
 (Z8I4-P$5 F #7+C+CC,,334DEF !,0G L)//1 	DAq"6Q"?".$3BJJqM		 !m&?'4'7'7'F'F'Q'Q#%67 (R ( " #.&22>#5#A#A **>-=> (OO,>C** ,$((T:F/
;TB%))*d;G0<dC$**U 0A0 , .>j-I!*EUF!99B ")#% /@/L ,-=j-I!*EUF!99B ,d3/@/L ,-=j-I!*EUF!99B --.?.N]+,<],K)55CC,];
 D  "%667H&834 55CC%56L%M% D 	 7} D
  	 **=c!fXF 	 6,  $**91c!f "s   VFT  TT  $1VUA'V>A
V	A	V!U4=V1V2G+VT   	U)!U
VUV	V$ V
V
VVdb_general_settingsc                 2  K   |yt        |      }d|v r|d   t        d<   d|v r|d   t        d<   d|v r3|d   t        d<   t        j                  j	                  t        d          d|v r(|d   t        d<   t        t        d          d{    yy7 w)z;
        Pull from DB, read general settings value
        Nmax_parallel_requestsglobal_max_parallel_requestsr  )r  r  r  )r   r   r   rH  r  rf   )r@  rR  rP  s      r   _update_general_settingsz$ProxyConfig._update_general_settings	  s     
 & !45"&778I'945 *->>?P.@;<
 //0A/0R_-55CC.? D 
 $'889J(:56 4'78P'Q  	 9s   BBBBcurrent_configr?  db_param_valuec                     t        ||   t              r%|j                         D ]  \  }}|s	|||   |<    |S |||<   |S r   )r   r   r  )r@  rW  r?  rX  r  r>  s         r   _update_config_fieldsz!ProxyConfig._update_config_fields
  s]     nZ0$7 -224 <
U6;N:.s3<  *8N:&r   r   rt  c                   K   |durt        j                  d       |S g }g d}|D ]'  }|j                  d|d      }|j                  |       ) t	        j
                  |  d {   }|D ]j  }|t        |dd       }	|	dk(  r"t        j                  d|j                          t        |d	d       }
|	J|
M|	|v r| j                  ||	|

      }f|
||	<   l |S 7 uw)NTz4'store_model_in_db' is not True, skipping db updates)r   rM  rN  r  r?  r?  )r  r>  re  rN  zlitellm_settings: rF  )rW  r?  rX  )	r1   r   get_generic_datar4  r  gatherr	  rF  rZ  )r@  r   r?  rt  _tasksr  r  responser,  r?  rF  s              r   r  z"ProxyConfig._update_config_from_db.
  s&     D( %%F M
  	$A$55 h 6 H MM(#		$ "..&11	! 	9H#$X|TB
!33(--,X-A-A,BC &htD)k.E!V+!%!;!;+1'1+6 "< " .9z*%	9( + 2s%   A#C%C&C2AC6C9#Cc                    K   	 |j                   j                  j                          d {   }|S 7 # t        $ r:}t	        j
                  dj                  t        |                   g }Y d }~|S d }~ww xY ww)NzQlitellm.proxy_server.py::add_deployment() - Error getting new models from DB - {})rC  litellm_proxymodeltable	find_manyr   r1   r  r*   r   )r@  r   r;  r  s       r   _get_models_from_dbzProxyConfig._get_models_from_db_
  su     	,//GGQQSSJ  T 	 **cjjF
 J	s6   A:'4 24 A:4 	A7/A2,A:2A77A:c                   K   	 t         t        t         t              st        dt                | j	                  |       d{   }| j                  ||       d{    |j                  j                  j                  ddi       d{   }|%| j                  |j                         d{    yy7 w7 ^7 07 # t        $ r7}t        j                  d	j                  t        |                   Y d}~yd}~ww xY ww)
z{
        - Check db for new models
        - Check if model id's in router already
        - If not, add to router
        Nr0  )r   )r;  r   r?  r   r@  )rR  z>litellm.proxy.proxy_server.py::ProxyConfig:add_deployment - {})r   r   r   r\  rc  rQ  rC  rD  rE  rV  rF  r   r1   r  r*   )r@  r   r   r;  rR  r  s         r   add_deploymentzProxyConfig.add_deploymentl
  s%    	!J)D Mj\Z   $77m7TTJ ))%9J *    )6(8(8(G(G(R(R#%78 )S ) #
 #.33(;(G(G 4    / U#  	 **PWWF 	sw   DAC B=C  B?!/C C%C 6C7C ;D=C ?C C C 	D-D ;D DD)r  Nr   )r   rs  F),r   rF  rG  __doc__rA  r   boolr   r   rK  r   rY  rT  rj  rR  rp  r   rj  rv  rz  r  r  r  ry  r   r/   r   r  r  r  r   r  r.  r:  rj   rQ  JsonrV  r   rZ  ri   r  rc  re  r   r   r   r<  r<    s-   )U U UN N N 15$ ($	$L% % % %NMD M, >@###&#7:#	#J tDz d  &44&6# 6$ 6p$  DAw~~.DABEDAL& &P%Kx} %KN @6#$ 6#3 6#p6 6# 6pTT (Tl!(4. !F  	
 
$/#/ / $D>	/b|  (#( ((r   r<  c                  R    dd l } |j                  |       t        j                  d<   y )Nr   r   )r   rW  r   r   )rd  r   s     r   save_worker_configrk  
  s    ",$**T"2BJJr   c                   K   t        j                  dd      j                         dk7  r
t                | a|a|du rddd l}ddlm}m	}m
} |j                  |j                          |j                  |j                          |j                  |j                         |du redd l}ddlm}m	}m
} |j                  |j                          |j                  |j                          |j                  |j                         n|du r|du rt         j                  j                  d	d      }||j!                         d
k(  rGdd l}ddlm	}m
}  |j                  |j                          |j                  |j                         nY|j!                         dk(  rFdd l}ddlm	}m
}  |j                  |j                          |j                  |j                         di t        i i}|r't"        j%                  t&        |       d {   \  aaa|r|a||t           d<   |r|a||t           d<   |r|t         j                  d<   |r||t           d<   |r|a||t           d<   |r|a||t           d<   |r||t           d<   |du rdt4        _        d|d   d<   |du rdt4        _        d|d   d<   |	r|	t4        _        |	|d   d<   t<        r	 |
ay 7 ƭw)NLITELLM_DONT_SHOW_FEEDBACK_BOXr   trueTr   )verbose_loggerr1   r2   )levelFLITELLM_LOGINFOr0   DEBUGgeneralr   rQ  ry  AZURE_API_VERSION
max_tokenstemperaturerequest_timeoutaliasdrop_paramsadd_function_to_promptr	  ) r   r  rD  r,   
user_model
user_debuglogginglitellm._loggingro  r1   r2   setLevelrr  rs  r   r   upperr   r   r   r   r   user_headersuser_api_baseuser_temperatureuser_request_timeoutr   rz  r{  r	  experimentaluser_telemetry)r  ry  ry  api_versionr   detailed_debugrw  rv  rx  r	  	telemetryrz  r{  rQ  save	use_queuer?  r~  ro  r1   r2   litellm_log_settingdynamic_configs                          r   r   r   
  s    ( 
yy126<<>&HJJ}	
 	
 	gll3&&&W\\:%%%GLL9	
 	
 	gmm4&&&W]];%%%GMM:	%Ne3 jjnn]B?*"((*f4X /%..!,, .$--!,, %**,7X.%..!-- .$--!--  Z4N
 ***v*VV		
07z"9- 19z":. 	

&' 3=z"<0&4?z"=1.8Gz"#45.3z"7+d"37y!-0%)-&>By!":;'2<y!,/NA Ws   HKKCKc              #     K   t        j                  d       | D ]B  }t        j                  d|       	 dt        j                  |j	                                d D y # t
        $ r dt        j                  |       d Y lw xY ww)Ninside generatorzreturned chunk: %sdata: r   )r1   r   r   rW  r   r   )r_  chunks     r   data_generatorr    s     12 3""#7?	34::ejjl34D993  	34::e,-T22	3s(   1B
)A B
 $BB
BB
c                   K   t        j                  d       	 t        j                          | 4 d {   }t        j	                  ||       d {   }|2 3 d {   }|j                  d      }	 d| d &7 K7 .7 %# t        $ r}dt        |       d Y d }~Ld }~ww xY w6 d d d       d {  7   n# 1 d {  7  sw Y   nxY wd}d| d y # t        $ r}t        j                  dj                  t        |                   t        j                  |||	       d {  7   t        j                  d
| d       t        |t              r|t        j                         }t        |       d| }t        t!        |d|      t!        |dd      t!        |dd      t!        |dd            }	t#        j$                  d|	j'                         i      }
d|
 d Y d }~y d }~ww xY ww)Nr  r  r_  Tr=  r  r   [DONE]zTlitellm.proxy.proxy_server.async_assistants_data_generator(): Exception occured - {}r  original_exceptionr~  [1;31mAn error occurred: P

 Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`r   r  Noner9  rO    rM  rL  )r1   r   r  r   async_post_call_streaming_hookmodel_dump_jsonr   r   r  r*   post_call_failure_hookr   r   r  r  ProxyExceptionr	  r   rW  to_dict)r_  r  r~  r  cr  done_messageerror_traceback	error_msgproxy_exceptionerror_returneds              r   async_assistants_data_generatorr  !  s     12/,		 	0 	0u ,JJ"3e K  E
 ! 0 0a%%4%80"1#T**	0
0 ! 0"3q6($///0	 !	0 	0 	0 	0 	0   |nD)) ,&&biiA	

  66/ % 7 
 	
 	

 	""+A3  /A  B	
 a'G'224Oq6($&78I(Ay)4FF+!Wf-M3/	
 Wo.E.E.G$HI~&d+++7,s   G>C! A?C! C BC B-BB-!C 5	B>C ?C! C B-	B*B% C %B**C .C! 9B<:C!  CC	CC!  G>!
G;+AG63D64B=G61G>6G;;G>c                  K   t        j                  d       	 t        j                          | 2 3 d {   }t        j                  dj                  |             t        j                  ||       d {   }t        |t              r|j                  dd      }	 d| d z7 u7 4# t        $ r}dt        |       d Y d }~d }~ww xY w6 d}d| d y # t        $ r}t        j                  d	j                  t        |                   t        j                  |||
       d {  7   t        j                  d| d       t        |t              r|t        j                         }t        |       d| }t!        t#        |d|      t#        |dd      t#        |dd      t#        |dd            }t%        j&                  d|j)                         i      }	d|	 d Y d }~y d }~ww xY ww)Nr  3async_data_generator: received streaming chunk - {}r  T)r>  exclude_unsetr  r   r  Ilitellm.proxy.proxy_server.async_data_generator(): Exception occured - {}r  r  r  r   r  r  r9  rO  r  rM  rL  )r1   r   r  r*   r   r  r   r  r  r   r   r  r  r   r  r  r  r	  r   rW  r  )
r_  r  r~  r  r  r  r  r  r  r  s
             r   async_data_generatorr  W  s     122,		# 	, 	,% &&ELLUS ,JJ"3e K  E %+--4t-T,ugT**	,
  ,s1vhd+++, $$  |nD)) ,&&W^^A	

  66/ % 7 
 	
 	

 	""+A3  /A  B	
 a'G'224Oq6($&78I(Ay)4FF+!Wf-M3/	
 Wo.E.E.G$HI~&d+++9,s   G?C" CB)C?C" 6B+7'C" 	B-(C" )C+C" -	C6CC" CC" !G?"
G<,AG74D75B=G72G?7G<<G?c                  K   t        j                  d       	 t        j                          | 2 3 d {   }t        j                  dj                  |             t        j                  ||       d {   }|j                  d      }	 d| dt        j                  |       d ~7 y7 8# t        $ r}d| dt        |       d Y d }~d }~ww xY w6 y # t        $ r}t        j                  dj                  t        |                   t        j                  |||	       d {  7   t        j                  d
| d       t        |t              r|t        j                          }t        |       d| }t#        t%        |d|      t%        |dd      t%        |dd      t%        |dd            }t        j                  d|j'                         i      }	d|	 d Y d }~y d }~ww xY ww)Nr  r  r  r  zevent: z
data:r   r  r  r  r  r   r  r9  rO  r  rM  rL  r  )r1   r   r  r*   r   r  r   r   rW  r   r   r  r  r   r   r  r  r  r	  r  )
r_  r  r~  r  
event_typer  r  r  r  r  s
             r   async_data_generator_anthropicr    s     12-,		# 	@ 	@% &&ELLUS ,JJ"3e K  E 6*J@
|74::e3D2ETJJ	@
  @
|73q6($???@ $  ,&&W^^A	

  66/ % 7 
 	
 	

 	""+A3  /A  B	
 a'G'224Oq6($&78I(Ay)4FF+!Wf-M3/	
 Wo.E.E.G$HI~&d+++9,s   G;C CB-C?C 6B/7C B1,C -C/C 1	C:CC CC G;
G8(AG30D31B=G3.G;3G88G;c                     t        | ||      S )Nr_  r  r~  )r  r  s      r   select_data_generatorr    s      +! r   r  c                     | j                  di       }| j                  di       j                  dd       }	 d|v r|j                  dd       }t        j                  |      }|S # t        $ r i cY S w xY w)Nr  r  r  azure
base_model)r   r   get_model_infor   )r  r  model_to_lookuplitellm_model_infos       r   get_litellm_model_infor    sz    <,Jii 0"599'4HOo%(nn\4@O$33OD!!  	s   ,A# #A10A1c                 6    t        j                  d| d          y )Nz$Backing off... this was attempt # %stries)r1   r   )r0  s    r   
on_backoffr    s    EwwGWXr   c                 >   t        | t              xr; t        | dd       d uxr* t        | j                  t              xr d| j                  v  }t
        j                  d      du ry|r4t        j                  t        j                  dt	        |       d             |S )Nr   z"Max parallel request limit reached1disable_retry_on_max_parallel_request_limit_errorTgiveup)eventr  )r   r  r	  r   r   r   r   r1   r   r   rW  )r  results     r   r  r    s    1n% 	>Ay$'t3	>qyy#&	> 1AII=	F 	PQ	 !!$**xcRSf-U"VWMr   c                       e Zd Zedee   dedee   fd       Zede	dee
   defd       Zed	ede
d
ede	fd       Zedefd       Zede	de
dedededefd       Zedee   dededee
   fd       Zed        Zy)r   r   r   r   c                 R    t                t                |j                  ||       y)z*Initialize logging and alerting on startup)r   r   N)r  r  startup_event)clsr   r   r   s       r   r  z-ProxyStartupEvent._initialize_startup_logging  s*     	 	''!5F 	( 	
r   r   r   r   c                 (   |j                  dd      ^|d   j                         D ]9  \  }}t        |t              s|j	                  d      s)t        |      |d   |<   ; t        di |d   }n
t               }t        j                  |||       y)zInitialize JWT auth on startuplitellm_jwtauthNrn  )r   r   r  r   )	r   r  r   r   ra  r   LiteLLM_JWTAuthr   r  )r  r   r   r   r  r  r  s          r   r  z&ProxyStartupEvent._initialize_jwt_auth  s      148D():;AAC K1a%!,,}*E=G]$%67:K .T0@AR0STO-/O&&'1+ 	' 	
r   r   r   c                 @   t        j                  dd      t        j                  d|      }|j                  d      du rt        j                  d       yt        ddg i i d||t        j                  dd	t        j                  i
      }t        j                  |       y)z,Adds master key hash to db for cost trackingPROXY_ADMIN_IDN$disable_adding_master_key_hash_to_dbTz&Skipping writing master key hash to dbuserr   update_data	user_role)request_typedurationmodelsaliasesr?  r  r  r  r  
query_typeupdate_key_values)
r   r  r   r1   r   ra   LitellmUserRolesPROXY_ADMINr  r  )r  r   r   r   r   task_1s         r   r  z,ProxyStartupEvent._add_master_key_hash_to_db  s     99%t,8')yy ":($  FG4O %%&NO ,# 0*66(#.0@0L0L"MF 'r   r   c                    t         j                  t        d      t        j                  t        d|dg i i dt         j                  t         j                  dt         j                  t         j                  d             y)z Adds a global proxy budget to dbNzPbudget_duration not set on Proxy. budget_duration is required to use max_budget.r  r   r  )r	  budget_duration)r  r  r  r  r  r?  r  r	  r  r  r  )r   r  r   r  r  ra   r	  )r  r   s     r   r
  z)ProxyStartupEvent._add_proxy_budget_to_db=  sy     ""*b 
 	"#1"-- ' 7 7(")"4"4'.'>'>#	
r   r   r   r   c           	        K   t               }t        j                  ||      }t        j                  |dz
  |dz         }	|j                  dd      du r|j	                  t
        d||g       |j	                  t        d|	|t        |g       t        dt              xs t        a	t        du rD|j	                  t        j                  dd||g       t        j                  ||	       d
{    |I|j                  j                  2|/t        d       |j                  dd      xs d}
t        |
d
d       }|
d   j!                         dk7  rt#        d      |j	                  |j                  j$                  d|t'        j(                         t+        d      z   |
g       |j	                  |j                  j,                  dd       t/        j0                  d      rXddlm} |j	                  |j                  j6                  ddd |d             |j                  j7                          d
{    |j9                          y
7 b7 w)z%Initializes scheduled background jobsr   disable_reset_budgetFinterval)secondsr  STORE_MODEL_IN_DBTrs  r   r   Nz3Alerting: Initializing Weekly/Monthly Spend Reportsspend_report_frequency7ddzBspend_report_frequency must be specified in days, e.g., '1d', '7d')r  )daysnext_run_timer  cronr_  )dayPROMETHEUS_URLr   )ZoneInfo	   zAmerica/Los_Angeles)hourminutetimezone)r!   r&   randintr   add_jobrt   ru   r   r   rt  r   re  rH  r  r)   rR  rD  r\  send_weekly_spend_reportr   nowr   send_monthly_spend_reportr   r  zoneinfor  #send_fallback_stats_from_prometheusstart)r  r   r   r   r   r   r   	schedulerr  batch_writing_intervalr  r  r  s                r   r  z6ProxyStartupEvent.initialize_scheduled_background_jobsY  s     %&	>>-/P
 "( 1$&:Q&>"

  6>%Gj(-  
 	*!13DE	 	 	
 /1BCXGX 	 $++#%67	   --+?P .   
 )!99BBN)GH !$$%=tDL #
 -cr23D%b)//1S8 X  !99RR&llnB'(,-   !99SS   yy)*-!!%==aa%&;< "  (??cceeegb fs%   C*I,I-EI8I9IIr   r  c                 h  K   d}|	 t        ||      }|j                          d{    t        j                  |j                                t        j                  |j                                t        dd      dur|j                          d{    |S # t        $ r}|d}~ww xY w7 7 w)zQ
        - Sets up prisma client
        - Adds necessary views to proxy
        N)r   r   &DISABLE_PRISMA_HEALTH_CHECK_ON_STARTUPFT)	ri   r   connectr  r  check_view_exists(_set_spend_logs_row_count_in_proxy_stater   health_check)r  r   r   r   r   r  s         r   r   z&ProxyStartupEvent._setup_prisma_client  s      15# ,!-AR!  ''))) //1 FFH   H%P $00222)   *  3sD   B2B B2B.A+B2B0B2	B+$B&&B++B20B2c                 P    t        dd      du rddl}|j                  dd       yy)z
        Initialize dd tracer - if `USE_DDTRACE=true` in .env

        DD tracer is used to trace Python applications.
        Doc: https://docs.datadoghq.com/tracing/trace_collection/automatic_instrumentation/dd_libraries/python/
        USE_DDTRACEFTr   N)r~  openai)r   ddtrace	patch_all)r  r  s     r   r  z!ProxyStartupEvent._init_dd_tracer  s.     =%0D8d59 9r   N)r   rF  rG  classmethodr   r/   rj   r4   r  r   ri   r3   r  r   r  r
  rR  r  r   r  r   r   r   r   r     st   
V$
 (
 $J/	
 
" 

  -
 &	
 
( (( $( #&	(
 ( (@ 
 
 
6 bb $b ,/	b
 ,/b "b (b bH $sm$ ($ &	$
 
,	$ $L 
: 
:r   r   z
/v1/modelszmodel management)dependenciesr-  z/modelsc           
      `  K   g }t        t              }t        g }n(t        j                         }t        j	                         }t        | ||      }t        | ||      }t        |||t        t        j                  dd            }t        |D cg c]	  }|dddd c}d	
      S c c}w w)z
    Use `/model/info` - to get detailed model information, example - pricing, mode, etc.

    This is just for compatibility with openai projects like aider.
    r  proxy_model_listmodel_access_groupsinfer_model_from_keysF
key_modelsteam_modelsr  r|  r
  r  ijNcr  )r  objectcreatedowned_byr  )rd  r  )r-   r  r   get_model_namesget_model_access_groupsrC   rD   rB   r|  r   r   r   )r  
all_modelsr	  r  r  r  r  s          r   r/  r/    s      J0;D0A%557(@@B+)/J "+)/K
 )).223JERJ  $
  !%$	
  
s   BB.B) B.z/v1/chat/completionszchat/completionsz/chat/completionsz&/engines/{model:path}/chat/completionsz1/openai/deployments/{model:path}/chat/completions   r  zSuccessful response)r  r-  r,  )	max_triesmax_timer  r  loggerfastapi_responsec                 d  K   i }	 t        |        d{   }t        j                  dj                  t	        j
                  |d                   t        || t        |t        t               d{   }t        j                  dd      xs t        xs	 |xs |d   |d<   t        r	t        |d	<   t        r	t        |d
<   t        r	t        |d<   t        r	t        |d<   t!        |d   t"              r.|d   t$        j&                  v rt$        j&                  |d      |d<   t(        j+                  ||d       d{   }| j,                  j                  dt#        t/        j0                                     |d<   t%        j2                  j4                  d>dt$        j2                  j7                         t9        j:                         d|\  }}||d<   g }|j=                  t(        j?                  ||d             tA        |dtB        t               d{   }|j=                  |       tE        jF                  | }| d{   }	|	d   }
tI        |
di       xs i }|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}|j                  dd      }|j                  di       xs i }tB        tB        jK                  |      |d<   tE        jL                  t(        jO                  |j                  dd      d !             d"|v rS|d"   d#u rLtQ        d>||jR                  |||t        |tI        |d$d      |||d%|}tU        |
||&      }tW        |d'|(      S t(        jY                  |||
)       d{   }
tI        |
di       xs i }|j                  di       xs i }|j,                  j[                  tQ        d>||jR                  |||t        |tI        |d$d      |||d%|       t]        |
*       d{    |
S 7  7 7 7 '7 7 7 # t^        $ r'}|j`                  }t(        jc                  |||+       d{  7   t%        jd                         }|jf                  |jh                  d,   jf                  _5        |j                  d"d      |d"   d#u rzt$        j2                  jm                  |d#-      }t%        jn                  ||j                  dd      d.|j                  dd      /      }tU        |||&      }tW        |d'0      cY d}~S t%        jp                  d,d,d,1      }||_9        |cY d}~S d}~wtt        $ r,}t        jv                  d2t#        |              t(        jc                  |||+       d{  7   tI        |d3d      }t        j                  d4||       t!        |tx              r^t{        tI        |d5t#        |            tI        |d6d7      tI        |d8d7      tI        |d9t|        j~                        tI        |d:i       ;      t#        |       }t{        tI        |d<|      tI        |d6d7      tI        |d8d7      tI        |d9d=      tI        |d:i       ;      d}~ww xY ww)?a  

    Follows the exact same API spec as `OpenAI's Chat API https://platform.openai.com/docs/api-reference/chat`

    ```bash
    curl -X POST http://localhost:4000/v1/chat/completions 
    -H "Content-Type: application/json" 
    -H "Authorization: Bearer sk-1234" 
    -d '{
        "model": "gpt-4o",
        "messages": [
            {
                "role": "user",
                "content": "Hello!"
            }
        ]
    }'
    ```

    rI  NzRequest received by LiteLLM:
{}rP  rQ  rd  rI  r   r  r   r   completion_modelr  rw  rx  rv  ry  
completionr  rd  	call_typer  litellm_call_idacompletion)original_function	rules_objr  litellm_logging_objrd  r  r  rd  
route_typer   r|  r_  _hidden_paramsrw  r   rx  r{  r}  additional_headersrw  r3  successr   r   streamTallowed_model_region)r  rv  rw  rx  ry  r   r{  rz  r}  r~  r|  r  text/event-stream
media_typerQ  rd  r  r_  r_  r  r   model_responseconvert_to_deltacached_response)completion_streamr  custom_llm_providerlogging_objr1  prompt_tokenscompletion_tokenstotal_tokenszBlitellm.proxy.proxy_server.chat_completion(): Exception occured - litellm_debug_infoo[1;31mAn error occurred: %s %s

 Debug this by setting `--debug`, e.g. `litellm --model gpt-3.5-turbo --debug`r  r  r  r9  rO  rQ  )r   r  r9  rN  rQ  r   r  r   )@rN   r1   r   r*   r   rW  r]   r   r   r   r   r|  r  r  user_max_tokensr  r   r   r   model_alias_mapr   pre_call_hookrQ  uuiduuid4utilsfunction_setupRulesr   r  r4  during_call_hookrg   r   r  r]  r	  get_deploymentr  update_request_statusr  r   r  r   post_call_success_hookr  r?   r5   r~  r  ModelResponser   choicesrP  ModelResponseIteratorCustomStreamWrapperUsageusager   r  r   r  r   HTTP_400_BAD_REQUEST)rI  r  r  r  rd  r:  tasksllm_callllm_responsesr,  r_  r|  rw  rx  ry  r{  r}  r)  custom_headersselected_data_generatorr  _data_chat_response	_iterator_streaming_response_usager@  r  s                               r   chat_completionr_  +  s    z D`
'88"".55djja6PQ	
 1-/%
 
   !3T:  G}	 	W "2D&:D"#!0D,D
 d7mS)d7mw?V?V.V#33DMBDM '44/dl 5 
 
 #*//"5"5TZZ\!2#
 $MM88 
+mm))+||~
 	
T '2"#.."3& / 	
 '$!!	
 
 	X  
 ('	Q<*:B?E2 $$Z6<"!%%k48>B	 $$Z6<"%))/4@FB,9,=,=/-
) $1#4#45I2#N#TRT !!+!:!:H!:!MD33 $):B ?	 4 	
 h4!7/ "3#33!#!+$%68NPRS2S!+ %N '<!"3!'#
 %'.&  +AA):X B 
 

 H.39r 	 +../CRHNB  '' "3#33!#!+$%68NPRS2S!+ %	
  *8<<< 9

D
:
 (f
2 	=    66/  7 
 	
 	

 !..045IIq!))188Hd#/DNd4J;;- < I #*"="="+hhw+$5 HH%:DA	# '<,"3"'# %'.  Q!RST% 
&&PQTUVQWPXY	
  66/ATX 7 
 	
 	
 %Q(<bA"" C	
 a' 8SV4Q/a&1Qv/J/JK9b1  1vh	Ay)4FF+!Wf-M3/Ay"-
 	
1
s   Z0Q P5AQ 3P84CQ <P;=CQ P>,Q QE	Q Z0Q *Q+BQ .Q/Q 4Z05Q 8Q ;Q >Q Q Q Q 
Z-'U49Q<:CU4Z-	Z0 U4.Z-/Z04Z-<Z(=W >C*Z((Z--Z0z/v1/completionscompletionsz/completionsz!/engines/{model:path}/completionsz,/openai/deployments/{model:path}/completionsc                 Z	  K   i }	 t        |        d{   }t        j                  dd      xs t        xs	 |xs |d   |d<   t        r	t        |d<   t	        || t        |t
        t               d{   }t        r	t        |d<   t        r	t        |d<   t        r	t        |d<   t        r	t        |d	<   |d   t        j                  v rt        j                  |d      |d<   t        j                  ||d
       d{   }t        |dt         t               d{   }| d{   }t#        |di       xs i }|j                  dd      xs d}|j                  dd      xs d}	|j                  d	d      xs d}
|j                  dd      xs d}|j                  dd      xs d}t%        j&                  t        j)                  |j                  dd      d             t+        j,                  d|       d|v r;|d   du r4t/        ||||	|
t
        |||	      }t1        |||      }t3        |d|      S t        j5                  |||       d{   }|j6                  j9                  t/        ||||	|
t
        |||	             t;        |       d{    |S 7 7 07 7 7 7 Z7 # t<        $ rH}|j>                  }t        jA                  |||        d{  7   |j                  dd      |d   du rt        jB                         }t        jD                  d!d!d!"      }||_#        |jH                  |jJ                  d!   jH                  _&        t        jN                  jQ                  |d#      }t        jR                  ||j                  dd      $      }t1        |||      }t3        |di       cY d}~S t        jT                         }|jH                  |jJ                  d!   _+        |cY d}~S d}~wtX        $ r}t        jA                  |||        d{  7   t+        jZ                  d%j]                  t_        |                   t_        |       }ta        t#        |d&|      t#        |d'd(      t#        |d)d(      t#        |d*d+      ,      d}~ww xY ww)-a  
    Follows the exact same API spec as `OpenAI's Completions API https://platform.openai.com/docs/api-reference/completions`

    ```bash
    curl -X POST http://localhost:4000/v1/completions 
    -H "Content-Type: application/json" 
    -H "Authorization: Bearer sk-1234" 
    -d '{
        "model": "gpt-3.5-turbo-instruct",
        "prompt": "Once upon a time",
        "max_tokens": 50,
        "temperature": 0.7
    }'
    ```
    r  Nr  r  r  rw  rx  rv  ry  text_completionr  atext_completionr&  r(  rw  r   rx  r{  r   r+  r,  final response: %sr-  T)	r  rv  rw  rx  ry  r   r{  r|  r~  r  r/  r0  r2  )	r  rv  rw  rx  ry  r   r{  r~  r|  r3  r  r   r<  r4  r8  r  z?litellm.proxy.proxy_server.completion(): Exception occured - {}r   r  r  r9  rO  r  rM  )1rN   r   r   r|  r]   r   r   r  r  rB  r  r   rC  r   rD  rg   r   r	  r  r  rL  r1   r   r  r  r   rM  rQ  r  r?   r5   r~  r  rN  rR  rS  r   rO  rP  rG  rP  TextCompletionStreamWrapperTextCompletionResponsetextr   r  r*   r   r  )rI  r  r  r  rd  rV  r_  r|  rw  rx  ry  r{  r   rX  rY  r  rZ  r[  r^  r\  r]  	_responser  s                          r   r  r  L  s    R Dg
'88   !3T:  G}	 	W &DM0-/%
 
 "2D&:D"#!0D,D
 =G333#33DMBDM '44/dFW 5 
 

 ')!!	
 
 ">*:B?E2 $$Z6<"!%%k48>B	 $$Z6<"%))/4@FB'++,=tDJ 	33 $):B ?	 4 	
 	""#7Bh4!7/"3'!#!++!
N '<!"3!'# %'.&  +AA):X B 
 
 	  ''"3'!#!+!+
	
 *8<<<_ 9
4


 "V
" 	= &66/  7 
 	
 	

 99Xt$0U8_5L$224N]]"#F
 $*N 89		N""1%--5;;- < I #*"E"E"+ii,#
 '<,"3!'# %'.   668I()		Ia % 
66/ATX 7 
 	
 	
 	&&MTTA	

 1vh	Ay)4FF+!Wf-M3/	
 	

s   R+J5 J"AJ5 5J%6B	J5 ?J( J5 J+	J5 (J.)DJ5 :R+;J5 J1AJ5 J3J5 !R+"J5 %J5 (J5 +J5 .J5 1J5 3J5 5
R(?'P&K)'CPR(R+	3P<R(=R+R(R#)P,*A9R##R((R+z/v1/embeddings
embeddings)r  response_classr-  z/embeddingsz /engines/{model:path}/embeddingsz+/openai/deployments/{model:path}/embeddingsc                 	  K   i }	 | j                          d{   }t        j                  |      }t        j                  dt        j                  |d             t        || t        |t        t               d{   }t        j                  dd      xs t        xs	 |xs |d   |d<   t        r	t        |d<   |d   t        j                  v rt        j                  |d      |d<   t        t        j                   ng }d|v rt#        |d   t$              rt'        |d         d	kD  rt#        |d   d	   t$              rt#        |d   d	   d	   t(              rt*        |d   |v r~t*        D ]u  }|d
   |d   k(  r0|d   d   t        j,                  v s|d   d   j/                  d      r>g }|d   D ](  }	|j1                  t        j2                  d|	             * ||d<    n t4        j7                  ||d       d{   }g }
|
j1                  t4        j9                  ||d             t;        |dt        t               d{   }|
j1                  |       t=        j>                  |
 }| d{   }|d   }t=        j@                  t4        jC                  |j                  dd      d             tE        |di       xs i }|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}|j                  di       xs i }|jF                  jI                  tK        d-||||t        |tE        |dd      |||d 
|       tM        |!       d{    |S 7 7 (7 7 c7 97 # tN        $ r}t4        jQ                  |||"       d{  7   tE        |d#d      }t        j                  d$||       t        jR                  d%jU                  tW        |                   t#        |tX              rIt[        |      }t]        |tE        |d&d'      tE        |d(d'      tE        |d)t^        j`                        *      tW        |       }t]        tE        |d+|      tE        |d&d'      tE        |d(d'      tE        |d)d,      *      d}~ww xY ww).a  
    Follows the exact same API spec as `OpenAI's Embeddings API https://platform.openai.com/docs/api-reference/embeddings`

    ```bash
    curl -X POST http://localhost:4000/v1/embeddings 
    -H "Content-Type: application/json" 
    -H "Authorization: Bearer sk-1234" 
    -d '{
        "model": "text-embedding-ada-002",
        "input": "The quick brown fox jumps over the lazy dog"
    }'
    ```

NzRequest received by LiteLLM:
%srP  rQ  r  embedding_modelr  inputr   r  r  zazure/zgpt-3.5-turbo)r  tokensrj  r  r%  
aembeddingr&  r_  r   r   r+  r,  r(  rw  rx  ry  r{  r)  r.  
r  rw  rx  ry  r   r{  rz  rv  r~  r|  r3  r  r@  rA  z?litellm.proxy.proxy_server.embeddings(): Exception occured - {}r  r  r9  rO  rM  r   r  r   )1bodyorjsonr   r1   r   r   rW  r]   r   r   r   r   r|  r   rC  r   model_namesr   r  r9  rR  r   open_ai_embedding_modelsra  r4  decoder   rD  rJ  rg   r  r]  r  rL  r	  rQ  r  r  r?   r   r  r  r*   r   r   rq   r  r   rT  )rI  r  r  r  rd  rr  router_model_namesr*  
input_listirU  rV  rW  r,  r_  r|  rw  rx  ry  r{  r   r)  r  r@  r   r  s                             r   rj  rj     s#    ` D]\\^#||D!"".JJtA&	
 1-/%
 
   !2D9  G}	 	W &DM
 =G333#33DMBDM7A7MZ33SUtO4=$/DM"Q&4=+T24=+A.4 )d7m?Q.Q' A$w-7*+G48X8XX-.w7BB8L &(
!%g A&-- '_Q O )3W" '44/dl 5 
 
 .."3& / 	
 '#!!	
 
 	X  
 ('	Q< 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"%))/4@FB'++,=tDJ#0#4#45I2#N#TRT  '' "3!#!+$%68NPRS'!+ %	
 *8<<<y $
f

 (F 	=  66/ATX 7 
 	
 	
 %Q(<bA"" C	

 	&&MTTA	

 a'+A.G Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 3s   SN4 N#A$N4 >N&?FN4 N)AN4 	N,
,N4 6N/7D%N4 N2N4 "S#N4 &N4 )N4 ,N4 /N4 2N4 4
S>SOC6SSSz/v1/images/generationsimagesz/images/generationsc                 ~  K   i }	 | j                          d {   }t        j                  |      }t        || t        |t
        t               d {   }t        j                  dd       xs t        xs |d   |d<   t        r	t        |d<   |d   t        j                  v rt        j                  |d      |d<   t        j                  ||d       d {   }t        |dt        t               d {   }| d {   }t        j                   t        j#                  |j                  dd	      d
             t%        |di       xs i }|j                  dd       xs d	}|j                  dd       xs d	}	|j                  dd       xs d	}
|j                  dd       xs d	}|j                  dd       xs d	}|j&                  j)                  t+        |||	|
t
        |t%        |dd	      |||
             |S 7 7 7 ,7 7 	# t,        $ r#}t        j/                  |||       d {  7   t1        j2                  dj5                  t7        |                   t1        j8                  t;        j<                                t?        |t@              rRtC        t%        |dt7        |            t%        |dd      t%        |dd      t%        |dtD        jF                              t7        |       }tC        t%        |d|      t%        |dd      t%        |dd      t%        |dd            d }~ww xY ww)Nr  image_generation_modelr  image_generationr  aimage_generationr&  r   r   r+  r,  r(  rw  rx  ry  r{  r.  rq  r  zElitellm.proxy.proxy_server.image_generation(): Exception occured - {}r   r  r  r9  rO  rM  r  )$rr  rs  r   r]   r   r   r   r   r|  r   rC  r   rD  rg   r   r  r  rL  r	  rQ  r  r  r   r  r1   rL  r*   r   r   r  r  r   r   r  r   rT  )rI  r  r  rd  rr  rV  r_  r|  rw  rx  ry  r{  r   r  r  s                  r   r}  r}    s?    $ Da\\^#||D! 1-/%
 
   !94@ G} 	W
 &DM
 =G333#33DMBDM '44/dFX 5 
 

 '*!!	
 
 "> 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"%))/4@FB'++,=tDJ  ''"3!#!+$%68NPRS'!+	
 M $
0


 "@  66/ATX 7 
 	
 	
 	""SZZA	

 	""9#7#7#9:a' 9c!f5Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 's   L=H G>9H HBH HH 7H8	H H
C;H =L=>H H H H 
H 
L:L52H53DL55L::L=z/v1/audio/speechaudioz/audio/speechc                   K   i }	 | j                          d{   }t        j                  |      }t        || t        |t
        t               d{   }|j                  dd      |j                  |j                  |d<   t        r	t        |d<   t        j                  ||d       d{   }t        |dt        t               d{   }| d{   }t        j                  t        j!                  |j                  d	d
      d             t#        |di       xs i }|j                  dd      xs d
}|j                  dd      xs d
}	|j                  dd      xs d
}
|j                  dd      xs d
}|j                  d	d      xs d
}dt$        fd}t'        |||	|
t
        |t#        |dd
      d|||      }t)        |||       t+         ||      d|      S 7 7 7 >7 "7 # t,        $ r[}t/        j0                  dj3                  t5        |                   t/        j6                  t9        j:                                |d}~ww xY ww)z`
    Same params as:

    https://platform.openai.com/docs/api-reference/audio/createSpeech
    Nr  r  r  r}  r  aspeechr&  r   r   r+  r,  r(  rw  rx  ry  r{  ri  c                l   K   | j                  d       d {   }|2 3 d {   }| 7 7 6 y w)Ni   )
chunk_size)aiter_bytes)ri  
_generatorr  s      r   generatezaudio_speech.<locals>.generate  s?     (444EEJ)  e Fzs$   4.4202424r.  )r  rw  rx  ry  r   r{  rz  r}  rv  r~  r|  r  z
audio/mpegr0  zAlitellm.proxy.proxy_server.audio_speech(): Exception occured - {})rr  rs  r   r]   r   r   r   r   r  r|  r   rD  rg   r   r  r  rL  r	  r   r  r  r   r   r1   rL  r*   r   r   r  r  )rI  r  r  rd  rr  rV  r_  r|  rw  rx  ry  r{  r   r  rX  r  s                   r   audio_speechr  h  sx    * DV\\^#||D! 1-/%
 
 88FD!).?.G.G.S,44DL&DM '44/dFX 5 
 

 ' !!	
 
 "> 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"%))/4@FB'++,=tDJ	&@ 	
 ,/' !24JBO.2#'
 	/	

 !X<
 	
S $
 


 "\  ""OVVA	

 	""9#7#7#9:s   IG6 G'9G6 G*AG6 /G-0G6 G0	G6 G3DG6 &I'G6 *G6 -G6 0G6 3G6 6	I?AIIIz/v1/audio/transcriptionsz/audio/transcriptions.r   c                 j  K   i }	 | j                          d{   }|j                         D ci c]  \  }}|dk7  s|| }}}t        || t        |t        t
               d{   }|j                  dd      |j                  |j                  |d<   t        j                  dd      xs t        xs |d   |d<   t        r	t        |d<   t        t        j                  ng }|j                  t        dt        j                  dd	      t        |||
       |j!                          d{   }	t#        j$                  |	      }
|j                  |
_        |
|d<   	 t(        j+                  ||d       d{   }t-        |dt        t               d{   }| d{   }	 |
j5                          t7        j8                  t(        j;                  |j                  dd      d             t=        |di       xs i }|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}|j                  di       xs i }|j>                  jA                  tC        d%||||t        |t=        |dd      |||d
|       |S 7 c c}}w 7 n7 7 ]7 A7 :# t.        $ r}t1        dt3        |            d}~ww xY w# |
j5                          w xY w# t.        $ r}t(        jE                  |||       d{  7   tG        jH                  djK                  t3        |                   tM        |t0              r\t        t=        |dt3        |jN                              t=        |d d!      t=        |d"d!      t=        |d#t        j                        $      t3        |       }t        t=        |d|      t=        |d d!      t=        |d"d!      t=        |d#d      $      d}~ww xY ww)&zq
    Same params as:

    https://platform.openai.com/docs/api-reference/audio/createTranscription?lang=curl
    Nr   r  r  moderation_modelr  z.File name is None. Please check your file namebad_request)r   rN  r  r9  )r~  r   rw  audio_transcriptionr  atranscriptionr&  r  r  r   r   r+  r,  r(  rw  rx  ry  r{  r)  r.  rq  r  zHlitellm.proxy.proxy_server.audio_transcription(): Exception occured - {}r   r  r  r9  rO  rM  r   )(formr  r]   r   r   r   r   r  r|  r   rt  r   r  r   rT  rO   readioBytesIOr   r   rD  rg   r   r   r   r   r  r  rL  r	  rQ  r  r  r  r1   r  r*   r   r  )rI  r  r   r  rd  	form_datar  r>  rw  file_contentfile_objectrV  r_  r  r|  rw  rx  ry  r{  r   r)  r  s                         r   audio_transcriptionsr    s@    , D|!,,.(	-6__->PzsE#-U
PP 1-/%
 
 88FD!).?.G.G.S,44DL   !3T: G} 	W
 &DM7A7MZ33SU==  H00"	  	$1	
 "YY[(jj.=="V	 *88"3/ 9  D ++%%	 H &~H  	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"%))/4@FB'++,=tDJ#0#4#45I2#N#TRT  '' "3!#!+$%68NPRS'!+ %	
  E )P
J ) & 	@CA??	@ F  66/ATX 7 
 	
 	
 	&&V]]A	

 a' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 %s   P3L  KL  KK#L  'K(C
L  2K3/L  #K$ >K?K$ K	K$ 'K!(K$ -DL  P3L  L  L  K$ K$ !K$ $	L-LLL LL   
P0*P+MC%P++P00P3)r   r(  WebSocketDisconnect)
_arealtimez/v1/realtimez	/realtime	websocketc                   K   dd l }| j                          d {    || d}	 t        |dt        t               d {   }| d {    y 7 47 7 	# |j
                  j                  $ rE}t        j                  d       | j                  |j                  d       d {  7   Y d }~y d }~wt        $ r4 t        j                  d       | j                  dd       d {  7   Y y w xY ww)	Nr   )r  r  r  r&  zInvalid status code)rN  reasonzInternal server errori  )
websocketsacceptrg   r   r|  
exceptionsInvalidStatusCoder1   r  r   rO  r   )r  r  r  r  rd  rV  r  s          r   websocket_endpointr  w  s      



 DI&#!!	
 
 ! 
 	  22 P&&'<=oo1==9NoOOO I&&'>?oo40GoHHHIs   C1A	C1A A 	A 	A
A C1A A C..5B.#B&$B.)C1.7C.%C(&C.+C1-C..C1z/v1/assistants
assistantsz/assistantsc                 6  K   i }	 | j                          d{    t        || t        |t        t               d{   }t
        't        ddt        j                  j                  i      t        j                  di | d{   }t        j                  t        j                  |j                  dd      d	             t!        |d
i       xs i }|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}|j"                  j%                  t'        ||||t        t!        |dd      ||             |S 7 >7 7 # t(        $ r-}	t        j+                  ||	|       d{  7   t-        j.                  dj1                  t3        |	                   t-        j4                  t7        j8                                t;        |	t              r\t=        t!        |	dt3        |	j>                              t!        |	dd      t!        |	dd      t!        |	dt@        jB                              t3        |	       }
t=        t!        |	d|
      t!        |	dd      t!        |	dd      t!        |	dd            d}	~	ww xY ww)z
    Returns a list of assistants.

    API Reference docs - https://platform.openai.com/docs/api-reference/assistants/listAssistants
    Nr  r  rL  r  r   r   r+  r,  r(  rw  rx  ry  r.  r  rw  rx  ry  r   rz  r~  r|  r  zClitellm.proxy.proxy_server.get_assistants(): Exception occured - {}r   r  r  r9  rO  rM  r   )"rr  r]   r   r   r   r   r   r  no_llm_routerr>  aget_assistantsr  r  r   rL  r   r	  rQ  r  r  r   r  r1   rL  r*   r   r   r  r  r   r  r  r   rT  rI  r  r  rd  r_  r|  rw  rx  ry  r  r  s              r   get_assistantsr    s    * DHlln 1-/%
 
 2C2Q2Q2W2W(X  $33;d;; 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!+		
 [ 	
 <:  66/ATX 7 
 	
 	
 	""QXXA	

 	""9#7#7#9:a' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 'i   JE E$E EA	E E	CE JE E E 
J)JFDJJJc                 `  K   i }	 | j                          d{   }t        j                  |      }t        || t        |t
        t               d{   }t        't        ddt        j                  j                  i      t        j                  di | d{   }t        j                  t        j!                  |j#                  dd      d	             t%        |d
i       xs i }|j#                  dd      xs d}|j#                  dd      xs d}|j#                  dd      xs d}	|j&                  j)                  t+        ||||	t
        t%        |dd      ||             |S 7 S7 7 # t,        $ r-}
t        j/                  ||
|       d{  7   t1        j2                  dj5                  t7        |
                   t1        j8                  t;        j<                                t?        |
t              r\tA        t%        |
dt7        |
jB                              t%        |
dd      t%        |
dd      t%        |
dtD        jF                              t7        |
       }tA        t%        |
d|      t%        |
dd      t%        |
dd      t%        |
dd            d}
~
ww xY ww)z~
    Create assistant

    API Reference docs - https://platform.openai.com/docs/api-reference/assistants/createAssistant
    Nr  r  rL  r  r   r   r+  r,  r(  rw  rx  ry  r.  r  r  zElitellm.proxy.proxy_server.create_assistant(): Exception occured - {}r   r  r  r9  rO  rM  r   )$rr  rs  r   r]   r   r   r   r   r   r  r  r>  acreate_assistantsr  r  r   rL  r   r	  rQ  r  r  r   r  r1   rL  r*   r   r   r  r  r   r  r  r   rT  )rI  r  r  rd  rr  r_  r|  rw  rx  ry  r  r  s               r   create_assistantr    s    * DI\\^#||D! 1-/%
 
 2C2Q2Q2W2W(X  $66>>> 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!+		
 ] $
 ?:  66/ATX 7 
 	
 	
 	""SZZA	

 	""9#7#7#9:a' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 'sk   J.E4 E,9E4 E/A	E4 E2CE4 +J.,E4 /E4 2E4 4
J+>J&FDJ&&J++J.z"/v1/assistants/{assistant_id:path}z/assistants/{assistant_id:path}assistant_idc                   K   i }	 t        || t        |t        t               d{   }t        't        ddt        j                  j                  i      t	        j                  dd|i| d{   }t        j                  t        j                  |j                  dd      d	
             t        |di       xs i }|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}	|j                   j#                  t%        ||||	t        t        |dd      ||             |S 7 7 # t&        $ r-}
t        j)                  ||
|       d{  7   t+        j,                  dj/                  t1        |
                   t+        j2                  t5        j6                                t9        |
t
              r\t;        t        |
dt1        |
j<                              t        |
dd      t        |
dd      t        |
dt>        j@                              t1        |
       }t;        t        |
d|      t        |
dd      t        |
dd      t        |
dd            d}
~
ww xY ww)z~
    Delete assistant

    API Reference docs - https://platform.openai.com/docs/api-reference/assistants/createAssistant
    r  Nr  rL  r  r  r   r   r+  r,  r(  rw  rx  ry  r.  r  r  zElitellm.proxy.proxy_server.delete_assistant(): Exception occured - {}r   r  r  r9  rO  rM  r   )!r]   r   r   r   r   r   r  r  r>  adelete_assistantr  r  r   rL  r   r	  rQ  r  r  r   r  r1   rL  r*   r   r   r  r  r   r  r  r   rT  )rI  r  r  r  rd  r_  r|  rw  rx  ry  r  r  s               r   delete_assistantr  d  sv    , DG 1-/%
 
 2C2Q2Q2W2W(X  $55X<XSWXX 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!+		
 U
 Y:  66/ATX 7 
 	
 	
 	""SZZA	

 	""9#7#7#9:a' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 'Y   J  E EAE 2E3CE  J E E 
I=I8+E.,DI88I==J z/v1/threadsz/threadsc                 6  K   i }	 | j                          d{    t        || t        |t        t               d{   }t
        't        ddt        j                  j                  i      t        j                  di | d{   }t        j                  t        j                  |j                  dd      d	             t!        |d
i       xs i }|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}|j"                  j%                  t'        ||||t        t!        |dd      ||             |S 7 >7 7 # t(        $ r-}	t        j+                  ||	|       d{  7   t-        j.                  dj1                  t3        |	                   t-        j4                  t7        j8                                t;        |	t              r\t=        t!        |	dt3        |	j>                              t!        |	dd      t!        |	dd      t!        |	dt@        jB                              t3        |	       }
t=        t!        |	d|
      t!        |	dd      t!        |	dd      t!        |	dd            d}	~	ww xY ww)zs
    Create a thread.

    API Reference - https://platform.openai.com/docs/api-reference/threads/createThread
    Nr  r  rL  r  r   r   r+  r,  r(  rw  rx  ry  r.  r  r  zClitellm.proxy.proxy_server.create_threads(): Exception occured - {}r   r  r  r9  rO  rM  r   )"rr  r]   r   r   r   r   r   r  r  r>  acreate_threadr  r  r   rL  r   r	  rQ  r  r  r   r  r1   rL  r*   r   r   r  r  r   r  r  r   rT  r  s              r   create_threadsr    s    * DHlln 1-/%
 
 2C2Q2Q2W2W(X  $22:T:: 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!+		
 [ 	
 ;:  66/ATX 7 
 	
 	
 	""QXXA	

 	""9#7#7#9:a' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 'r  z/v1/threads/{thread_id}z/threads/{thread_id}	thread_idc                   K   i }	 t        || t        |t        t               d{   }t        't        ddt        j                  j                  i      t	        j                  dd|i| d{   }t        j                  t        j                  |j                  dd      d	
             t        |di       xs i }|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}	|j                   j#                  t%        ||||	t        t        |dd      ||             |S 7 7 # t&        $ r-}
t        j)                  ||
|       d{  7   t+        j,                  dj/                  t1        |
                   t+        j2                  t5        j6                                t9        |
t
              r\t;        t        |
dt1        |
j<                              t        |
dd      t        |
dd      t        |
dt>        j@                              t1        |
       }t;        t        |
d|      t        |
dd      t        |
dd      t        |
dd            d}
~
ww xY ww)zs
    Retrieves a thread.

    API Reference - https://platform.openai.com/docs/api-reference/threads/getThread
    r  Nr  rL  r  r  r   r   r+  r,  r(  rw  rx  ry  r.  r  r  z?litellm.proxy.proxy_server.get_thread(): Exception occured - {}r   r  r  r9  rO  rM  r   )!r]   r   r   r   r   r   r  r  r>  aget_threadr  r  r   rL  r   r	  rQ  r  r  r   r  r1   rL  r*   r   r   r  r  r   r  r  r   rT  rI  r  r  r  rd  r_  r|  rw  rx  ry  r  r  s               r   
get_threadr  &  su    , DF 1-/%
 
 2C2Q2Q2W2W(X  $//L)LtLL 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!+		
 U
 M:  66/ATX 7 
 	
 	
 	""MTTA	

 	""9#7#7#9:a' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 'r  z /v1/threads/{thread_id}/messagesz/threads/{thread_id}/messagesc                 d  K   i }	 | j                          d{   }t        j                  |      }t        || t        |t
        t               d{   }t        't        ddt        j                  j                  i      t        j                  dd|i| d{   }t        j                  t        j!                  |j#                  dd      d	
             t%        |di       xs i }|j#                  dd      xs d}|j#                  dd      xs d}	|j#                  dd      xs d}
|j&                  j)                  t+        |||	|
t
        t%        |dd      ||             |S 7 U7 7 # t,        $ r-}t        j/                  |||       d{  7   t1        j2                  dj5                  t7        |                   t1        j8                  t;        j<                                t?        |t              r\tA        t%        |dt7        |jB                              t%        |dd      t%        |dd      t%        |dtD        jF                              t7        |       }tA        t%        |d|      t%        |dd      t%        |dd      t%        |dd            d}~ww xY ww)zv
    Create a message.

    API Reference - https://platform.openai.com/docs/api-reference/messages/createMessage
    Nr  r  rL  r  r  r   r   r+  r,  r(  rw  rx  ry  r.  r  r  zAlitellm.proxy.proxy_server.add_messages(): Exception occured - {}r   r  r  r9  rO  rM  r   )$rr  rs  r   r]   r   r   r   r   r   r  r  r>  a_add_messager  r  r   rL  r   r	  rQ  r  r  r   r  r1   rL  r*   r   r   r  r  r   r  r  r   rT  rI  r  r  r  rd  rr  r_  r|  rw  rx  ry  r  r  s                r   add_messagesr    s    , DI\\^#||D! 1-/%
 
 2C2Q2Q2W2W(X  $11NINNN 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!+		
 ] $
 O:  66/ATX 7 
 	
 	
 	""OVVA	

 	""9#7#7#9:a' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 'sk   J0E6 E.9E6 E1AE6 E4 CE6 -J0.E6 1E6 4E6 6
J- J(FDJ((J--J0c                   K   i }	 t        || t        |t        t               d{   }t        't        ddt        j                  j                  i      t	        j                  dd|i| d{   }t        j                  t        j                  |j                  dd      d	
             t        |di       xs i }|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}	|j                   j#                  t%        ||||	t        t        |dd      ||             |S 7 7 # t&        $ r-}
t        j)                  ||
|       d{  7   t+        j,                  dj/                  t1        |
                   t+        j2                  t5        j6                                t9        |
t
              r\t;        t        |
dt1        |
j<                              t        |
dd      t        |
dd      t        |
dt>        j@                              t1        |
       }t;        t        |
d|      t        |
dd      t        |
dd      t        |
dd            d}
~
ww xY ww)z
    Returns a list of messages for a given thread.

    API Reference - https://platform.openai.com/docs/api-reference/messages/listMessages
    r  Nr  rL  r  r  r   r   r+  r,  r(  rw  rx  ry  r.  r  r  zAlitellm.proxy.proxy_server.get_messages(): Exception occured - {}r   r  r  r9  rO  rM  r   )!r]   r   r   r   r   r   r  r  r>  aget_messagesr  r  r   rL  r   r	  rQ  r  r  r   r  r1   rL  r*   r   r   r  r  r   r  r  r   rT  r  s               r   get_messagesr    ss    , DE0-/%
 
 2C2Q2Q2W2W(X  $11NINNN 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!+		
 U
 O:  66/ATX 7 
 	
 	
 	""OVVA	

 	""9#7#7#9:a' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 'r  z/v1/threads/{thread_id}/runsz/threads/{thread_id}/runsc                   K   i }	 | j                          d{   }t        j                  |      }t        || t        |t
        t               d{   }t        't        ddt        j                  j                  i      t        j                  dd|i| d{   }d|v r |d   du rt        t        |||	      d
      S t        j                   t"        j%                  |j'                  dd      d             t)        |di       xs i }|j'                  dd      xs d}|j'                  dd      xs d}	|j'                  dd      xs d}
|j*                  j-                  t/        |||	|
t
        t)        |dd      ||             |S 7 y7 B7 # t0        $ r-}t"        j3                  |||       d{  7   t5        j6                  dj9                  t;        |                   t5        j<                  t?        j@                                tC        |t              r\tE        t)        |dt;        |jF                              t)        |dd      t)        |dd      t)        |dtH        jJ                              t;        |       }tE        t)        |d|      t)        |dd      t)        |dd      t)        |dd            d}~ww xY ww)zi
    Create a run.

    API Reference: https://platform.openai.com/docs/api-reference/runs/createRun
    Nr  r  rL  r  r  r-  Tr  r_  r~  r/  r;  r   r   r+  r,  r(  rw  rx  ry  r.  r  r  z?litellm.proxy.proxy_server.run_thread(): Exception occured - {}r   r  r  r9  rO  rM  r   )&rr  rs  r   r]   r   r   r   r   r   r  r  r>  arun_threadr   r  r  r  r   rL  r   r	  rQ  r  r  r   r  r1   rL  r*   r   r   r  r  r   r  r  r   rT  r  s                r   
run_threadr  H  s    , DS\\^#||D!0-/%
 
 2C2Q2Q2W2W(X  $//L)LtLL h4!7$/&7%!%
 /  	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!+		
 s $
 MR  66/ATX 7 
 	
 	
 	""MTTA	

 	""9#7#7#9:a' 9c!((m<Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 'sw   KF F9F FAF F 'F KC	F KF F F 
K$K?G DKKKz/v1/moderationsmoderationsz/moderationsc                   K   i }	 | j                          d{   }t        j                  |      }t        || t        |t
        t               d{   }t        j                  dd      xs t        xs |j                  d      |d<   t        r	t        |d<   t        j                  ||d       d{   }t        j                          t        |dt        t               d{   }| d{   }t        j                  t        j!                  |j                  d	d
      d             t#        |di       xs i }|j                  dd      xs d
}|j                  dd      xs d
}	|j                  dd      xs d
}
|j$                  j'                  t)        |||	|
t
        t#        |dd
      ||             |S 7 7 r7 7 7 # t*        $ r}t        j-                  |||       d{  7   t/        j0                  dj3                  t5        |                   t7        |t8              rRt;        t#        |dt5        |            t#        |dd      t#        |dd      t#        |dt<        j>                              t5        |       }t;        t#        |d|      t#        |dd      t#        |dd      t#        |dd            d}~ww xY ww)a  
    The moderations endpoint is a tool you can use to check whether content complies with an LLM Providers policies.

    Quick Start
    ```
    curl --location 'http://0.0.0.0:4000/moderations'     --header 'Content-Type: application/json'     --header 'Authorization: Bearer sk-1234'     --data '{"input": "Sample text goes here", "model": "text-moderation-stable"}'
    ```
    Nr  r  r  
moderationr  amoderationr&  r   r   r+  r,  r(  rw  rx  ry  r.  r  r  z@litellm.proxy.proxy_server.moderations(): Exception occured - {}r   r  r  r9  rO  rM  r  ) rr  rs  r   r]   r   r   r   r   r|  r   rD  r  rg   r   r  r  rL  r	  rQ  r  r  r   r  r1   r  r*   r   r   r   r  r   rT  )rI  r  r  rd  rr  rV  r_  r|  rw  rx  ry  r  r  s                r   r  r    s    : DY\\^#||D! 1-/%
 
   !3T: !!xx  	W
 &DM '44/dl 5 
 
 			 '$!!	
 
 "> 	33 $):B ?	 4 	
  *:B?E2 $$Z6<"!%%k48>B	 $$Z6<"  ''"3!#!$%68NPRS!+		
  $
$

 ":  66/ATX 7 
 	
 	
 	&&NUUA	

 a' 9c!f5Q/a&1Qv/J/JK	  q6(I 9i8Q/a&1Qs3	 %s   KG G9G GA"G 6G72G )G*	G 3G4CG KG G G G G 	KK3G64CKKKz/v1/messagesz[beta] Anthropic `/v1/messages`)r-  r  response_modelinclude_in_schemaanthropic_datac                 $  K   ddl m} ddlm} d|dgt         _        t        |       d{   }i |ddi}	 t        j                  d	d      xs t        xs |d
   |d
<   t        r	t        |d
<   t        ||t        |t        t               d{   }t        r	t        |d<   t        r	t        |d<   t        r	t        |d<   t        r	t        |d<   |d
   t         j                   v rt         j                   |d
      |d
<   t"        j%                  ||d       d{   }t&        t&        j(                  ng }d|v r*t+        j,                  t        j.                  d4i |      }	nt&        1|d
   |v r*t+        j,                  t'        j.                  d4i |      }	nt&        Ot&        j0                  ?|d
   t&        j0                  v r*t+        j,                  t'        j.                  d4i |      }	n\t&        C|d
   t&        j2                  v r.t+        j,                  t'        j.                  d4i |ddi      }	nt&        B|d
   t&        j5                         v r)t+        j,                  t'        j.                  d4i |      }	nt&        f|d
   |vr_t&        j6                  &t9        t&        j:                  j<                        dkD  r)t+        j,                  t'        j.                  d4i |      }	n_t        )t+        j,                  t        j.                  d4i |      }	n0t?        t@        jB                  dd|j                  d
d      z   i      |	 d{   }
tE        |
di       xs i }|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}|j                  dd      xs d}t+        j,                  t"        jG                  |j                  dd      d             tI        jJ                  d |
       |jL                  jO                  tQ        ||||t        |||!             d"|v r"|d"   du rtS        |
||#      }tU        |d$%      S tI        jV                  d&jY                  |
             |
S 7 a7 7 x7 C# tZ        $ rH}|j\                  }t"        j_                  |||'       d{  7   |j                  d"d      |d"   du rt        j`                         }t        jb                  ddd(      }||_2        |jf                  |jh                  d   jf                  _5        t         jl                  jo                  |d)      }t        jp                  ||j                  d
d      *      }ts        |||#      }tU        |d$i +      cY d}~S t        jt                         }|jf                  |jh                  d   _;        |cY d}~S d}~wtx        $ r}t"        j_                  |||'       d{  7   tI        jz                  d,jY                  t}        |                   t}        |       }t        tE        |d-|      tE        |d.d/      tE        |d0d/      tE        |d1d2      3      d}~ww xY ww)5u   
    🚨 DEPRECATED ENDPOINT🚨

    Use `{PROXY_BASE_URL}/anthropic/v1/messages` instead - [Docs](https://docs.litellm.ai/docs/anthropic_completion).

    This was a BETA endpoint that calls 100+ LLMs in the anthropic format.
    r   )adapter_completion)anthropic_adapter	anthropic)r  adapterr  N
adapter_idr  r  r  rw  rx  rv  ry  rb  r  api_keyspecific_deploymentTrL  z/completion: Invalid model name passed in model=r   r  r(  rw  rx  r{  r   r+  r,  rd  )r  rw  rx  ry  r   r{  r~  r|  r-  r  r/  r;  z
Response from Litellm:
{}r  r<  r4  re  r0  zGlitellm.proxy.proxy_server.anthropic_response(): Exception occured - {}r   r  r  r9  rO  r  rM  r   )@r   r  "litellm.adapters.anthropic_adapterr  adaptersrN   r   r   r|  r]   r   r   r  r  rB  r  rC  r   rD  r   rt  r  r  aadapter_completionmodel_group_aliasdeployment_namesr'  default_deploymentr9  pattern_routerpatternsr   r   rT  r	  rL  r1   r   rQ  r  r  r  r   r   r*   r5   r~  r  rN  rR  rS  r   rO  rP  rG  rP  rf  r  rg  rh  r   r  r   r  )r  r  rI  r  r  r  r~  rd  rw  llm_responser_  r|  rw  rx  ry  r{  rY  r  rZ  r[  r^  r\  r]  ri  r  s                            r   anthropic_responser  2  sU    ( +D*7HIJG ,G<<L<L<,<Dx
  !3T: G} 	W
 &DM0-/%
 
 "2D&:D"#!0D,D
 =G333#33DMBDM '44/dFW 5 
 

 8B7MZ33SU"..w/J/J/RT/RSL"tG}8J'J"..z/M/M/UPT/UVL",,8W!=!=="..z/M/M/UPT/UVL"tG}
8S8S'S"....PP4PL "tG}
8P8P8R'R"..z/M/M/UPT/UVL"W%77--9z0099:Q> #..z/M/M/UPT/UVL#"..w/J/J/RT/RSL"77Nhhw+,  &%*:B?E2 $$Z6<"!%%k48>B	 $$Z6<"%))/4@FB 	33 $):B ?	 4 	
 	""#7B  ''"3!#!+!+		
 h4!7&D!"3!'#
 %'. 
 	!!"@"G"G"QRG =
4
d &Z   &66/  7 
 	
 	

 99Xt$0U8_5L$224N]]"#F
 $*N 89		N""1%--5;;- < I #*"E"E"+ii,#
 '<,"3!'# %'.   668I()		Ia % 
66/ATX 7 
 	
 	
 	&&U\\A	

 1vh	Ay)4FF+!Wf-M3/	
 	

s   +ZRZAR RB	R RH7R RDR 'Z(%R ZR R R 
Z$'W'SCW'(Z)Z.3W'!Z"Z'Z3ZXA9ZZZz/utils/token_counterz	llm utils)r-  r  r  c                   K   ddl m} | j                  }| j                  }||t	        dd      d}d}d}t
        Lt
        j                  D ]9  }|d   | j                  k(  s|}t
        j                  || j                        } n |:|j                  d	i       j                  d
      }d|v r|j                  dd      d   }|xs | j                  }d}	|'t        t        t           |j                  dd            }	t         j                  j                  ||	      }
t!        |
d         } |||||
      }t#        || j                  ||      S w)r%   r   )token_counterN  z#prompt or messages must be providedr  r  )r3  received_model_namer  r  r%  r_  custom_tokenizer)r  r  r  )r  rh  r    r  )r?  request_model
model_usedtokenizer_type)r   r  promptr    r   r   r/  r  get_router_model_infor   r0  r
   r   r   rG  _select_tokenizerr   TokenCountResponse)rI  r  r  r    r3  r  r  _modelmodel_to_user  _tokenizer_usedtokenizer_usedr?  s                r   r  r    s     & ^^FH~(*$I
 	
 J)-J ++ 	Fl#w}}4#
'==)(/ > 
 	 '^^,<bAEEgN$$!3!9!9#q!A!!D 	+gmm  >B/0NN-t4
 mm55-= 6 O 01N (	L !mm%	 s   A!E$C1Ez/utils/supported_openai_params)r-  r  c                    K   	 t        j                  |       \  } }}}dt        j                  | |      iS # t        $ r t	        dddj                  |       i      w xY ww)a-  
    Returns supported openai params for a given litellm model name 

    e.g. `gpt-4` vs `gpt-3.5-turbo` 

    Example curl: 
    ```
    curl -X GET --location 'http://localhost:4000/utils/supported_openai_params?model=gpt-3.5-turbo-16k'         --header 'Authorization: Bearer sk-1234'
    ```
    r#  supported_openai_paramsr  r9  r  rL  zCould not map model={}r  )r   get_llm_providerget_supported_openai_paramsr   r   r*   )r  r9  r&  s      r   r  r  V  sx     "

+2+C+C%+P("Aq%w'J'J1D(
 	

  
W.F.M.Me.T$U
 	

s   A#38 A#(A  A#model_paramsc                   K   | j                   j                  d      }| j                   j                  }|j                         D ]   \  }}t	        |      }|| j                   |<   " | j
                  j                  | j                  | j                   j                  d      | j
                  j                  d      |j                  xs t        |j                  xs t        d}| j
                  j                  | j
                  j                  |d<   |j                  j                  j                  |       d {   }	|	S 7 w)NTr=  r  )rw  r  r  r  r  r  rw  rd  )r  r   r  r  rM   r  r  r  r  r  r   rC  ra  create)
r  r  r   _litellm_params_dict_orignal_litellm_model_namer  r  encrypted_valuerZ  r5  s
             r   _add_model_to_dbr  w  s@     (66;;;N"."="="C"C$**, 91.Q7)8##A&9 !++.."--&55EESWEX"--== > 
 (//K3K'//K3K	E !!-(3366j(++CCJJ K  N s   D=E?E Ec                 ,  K   | j                   j                  }| j                  }|yd| dt        j                          }|| _        t        | ||       d{   }t        t        |||i      |t        ddi      	       d{    |S 7 57 w)
a  
    If 'team_id' is provided,

    - generate a unique 'model_name' for the model (e.g. 'model_name_{team_id}_{uuid})
    - store the model in the db with the unique 'model_name'
    - store a team model alias mapping {"model_name": "model_name_{team_id}_{uuid}"}
    Nmodel_name_r&  r  r  r   )r  model_aliasesr  r]  )scope)rd  r  http_request)	r  r  r  rE  rF  r  rb   UpdateTeamRequestr   )r  r  r   _team_idoriginal_model_nameunique_model_namer5  s          r   _add_team_model_to_dbr    s      &&..H&11%hZq?/L ,!+# N .0AB
 ,FF#34   !s$   ABB.B	B
BBc                     d}|j                   r|j                   t        j                  k(  ry| !|j                   t        j                  k7  rd}|S |j                  | k7  rd}|S )NTF)r  r  r  r  )r  r  can_make_calls      r   check_if_team_id_matches_keyr    sl     M##''+;+G+GG&&*:*F*FF!M  $$/!Mr   z
/model/newz=Allows adding new models to the model list in the config.yaml)r  r-  r  c                   K   	 dd l }t        t        dddi      | j                  j                  /t
        dur't        ddt        j                  j                  i      t        | j                  j                  |      st        ddd	i      d }t        du r	 	 | j                  }| j                  j                  t        | |t        
       d {   }nt        | |t        
       d {   }t        j                  t        t                d {    t"        j%                  dg       xs g }d|v r>t         j&                  j)                  | j                  || j                         d {    |S t        dddi      7 7 7 u7 # t*        $ r#}t-        j.                  d|        Y d }~|S d }~ww xY w# t*        $ r}t-        j0                  dj3                  t5        |                   t-        j6                  t9        j:                                t=        |t              rYt?        tA        |ddt5        |       d      tB        jD                  tA        |dd      tA        |dtF        jH                              t=        |t>              r|t?        dt5        |      z   tB        jD                  tA        |dd      tF        jH                        d }~ww xY ww)Nr   r  rL  VNo DB Connected. Here's how to do it - https://docs.litellm.ai/docs/proxy/virtual_keysr  Ti  )r  r  z,Team ID does not match the API key's team IDr  r  r  r  )r  r  passed_model_infozException in add_new_model: CSet `'STORE_MODEL_IN_DB='True'` in your env to enable this feature.zBlitellm.proxy.proxy_server.add_new_model(): Exception occured - {}r  Authentication Error()r9  r  rO  rM  Authentication Error, )%r4  r   r   r  r  r   r  r  r>  r  rt  r  r  r  r   re  r   r   r   rH  model_added_alertr   r1   r  rL  r*   r   r   r  r  r   r  r	  ProxyErrorTypes
auth_errorr   rT  )r  r  r4  r5  _original_litellm_model_name	_alertingr  s          r   add_new_modelr    s    ]
 u  ""**6<t;S!2!C!C!I!IJ 
 , ++33GX
  !OP 
 $
S/;/F/F,**22:+;%1*;&3, &N ,A%1*;&3, &N
 #11"/CT 2    -00R@FB	i'+CCUU#/#:#:+G*6*A*A V       b 7&&

  S$..1MaS/QRR S  
""PWWA	

 	""9#7#7#9:a' 8/DSVHA-NO$//a&1Qv/J/JK	  >*G,s1v5 ++!Wf-,,	
 	
!
s   K#BG  7F F	F 0F1&F FAF 3F4F 8G  9K#:G  	F F F F 	F=F82G  7K#8F==G   
K 
DKK  K#z/model/updatezEdit existing model paramsc                 8  K   	 dd l }t        t        dddi      t        du rd }t	        | dd       }|t        d      |j                  }|t        d	      t        j                  j                  j                  d
|i       d {   }|6t        %t        j                  |      t        dddi      t        d      t        |j                        }| j                  t        d      | j                  j                  d      }|j                         D ]   \  }}	t        |	      }
|
| j                  |<   " i }| j                  j                         }|j                         D ]  \  }}||||<   ||v r||   	||   ||<    ! t!        j"                  |      |j$                  xs t&        d}t        j                  j                  j)                  d
|i|       d {   }|S y 7 h7 
# t
        $ r}t+        j,                  dj/                  t1        |                   t+        j2                  t5        j6                                t9        |t              rYt;        t	        |ddt1        |       d      t<        j>                  t	        |dd      t	        |dt@        jB                              t9        |t:              r|t;        dt1        |      z   t<        j>                  t	        |dd      t@        jB                        d }~ww xY ww)Nr   r  rL  r  r  Tr  zmodel_info not providedzmodel_info.id not providedrw  r@  r*  r  zOCan't edit model. Model in config. Store model in db via `/model/new`. to edit.zmodel not foundzlitellm_params not providedr=  r  )r  r  rA  rd  zAlitellm.proxy.proxy_server.update_model(): Exception occured - {}r  r
  r  r9  r  rO  rM  r  )"r4  r   r   rt  r	  r   r  rC  ra  find_uniquer   rK  r   r  r  rM   r   rW  r  r   r  r1   rL  r*   r   r   r  r  r   r  r  r  r   rT  )r  r  r4  	_model_idr  _existing_litellm_params_existing_litellm_params_dict_new_litellm_params_dictr  r  r  merged_dictionary_mpr  r>  rZ  r5  r  s                     r   update_modelr  <  sN    e
  u  $I!,dCK" 9::#I  <==#&&>>JJ%y1 K   %
 (/*"1191EQ'$'#%v     122,0(77-) **2 =>>'3'B'B'G'G! (H ($
 1668 A1"6Q"?1@++A.A
 !#--224C!iik 	
U$-2%c*885c:F-J3-O%c*	 #'**->"?/77S;SE $1#3#3#K#K#R#R!9- $S $ N
 "!E %f  
""OVVA	

 	""9#7#7#9:a' 8/DSVHA-NO$//a&1Qv/J/JK	  >*G,s1v5 ++!Wf-,,	
 	
!
sO   LBG7 
G2EG7 *G5+G7 0L2G7 5G7 7
LDLLLz/v2/model/infozv2 - returns all the models set on the config.yaml, shows 'user_access' = True if the user has access to the model. Provides more info about each model in /models, including config.yaml descriptions (except api key and api base))r  r-  r  r  z!Specify the model name (optional))r  r   c                   K   t         t        t         t              st        dddt          di      t        j                          d{    t        j                  t               }t        
|t        gz  }t        | j                        dkD  r	 ||D cg c]  }|d   |k(  s| }}|D ]  }|j                  d	i       }|d
u r7d}t        t        j                  |i d      xs d}nd}t        |      }||d<   t        |      }	|	i k(  r;|j                  di       }
|
j                  dd      }	 t!        j"                  |      }	|	i k(  rc|j                  di       }
|
j                  dd      }|j'                  d      }t        |      dkD  r|d   }	 t!        j"                  ||d         }	|	j)                         D ]  \  }}||vs|||<    ||d	<   |d   j+                  dd       |d   j+                  dd       |d   j+                  dd       |d   j+                  dd        t-        j.                  d|       d|iS 7  c c}w # t$        $ r i }	Y w xY w# t$        $ r i }	Y w xY ww)zQ
    BETA ENDPOINT. Might change unexpectedly. Use `/v1/model/info` for now.
    Nr  rL  zNo model list passed, models=zI. You can add a model through the config.yaml or on the LiteLLM Admin UI.r  r   r  r  Tr  async)r3  r  client_typellm_router_is_Noneopenai_clientr#  r  r  r%  r  r  r  vertex_credentialsaws_access_key_idaws_secret_access_keyall_models: %srd  )r   r   r  r   r   r  r5  r6  r|  r9  r  r   r   _get_clientr   r  r   r  r   r0  r  rg  r1   r   )r  r  r   r  r*  r  r  _openai_clientr!  r  r  litellm_modelsplit_modelr  r  s                  r   model_info_v2r*    s    & Z%E88H  IR  S
 	
 
!
!
###~.Jzl"
 ##$q(!+HAq%/GaH
H  5DZZb1
D=#N%**#)"' +   	  "6/M&3F?# 4&A ##ZZ(8"=N*..w=M(%,%;%;-%P" ##ZZ(8"=N*..w=M'--c2K;!# +B(%,%;%;'[^&"
 ',,. 	"DAq
" !
1	"  *| 	 $$Y5 $$%94@ $$%8$? $$%<dCk5Dn /<JO $ I>  (%'"(  (%'"(s|   A	I6IAI6I&I*B
I65IAI6I%3I6A>I6I6I"I6!I""I6%I30I62I33I6z/model/streaming_metricsz1View time to first token for models in spend logs)r  r-  r  r  _selected_model_groupr  r  c                   K   t         5t        t        j                  j                  ddt
        j                        |xs! t        j                         t        d      z
  }|xs t        j                         }|j                         |j                         k(  }|rd}nd}t               }t         j                  j                  ||||       d {   }i }||D ]  }	|	d   }
|	d	   }|	d
   }d}|r|	d   }|}||vri ||<   n|	d   }|}|	d
   }||vri ||<   t        |      }d|
v rt        |
      }d|v r|j                  d      d   }|j!                  |       |||   |<    	 g }t#        t%        |j'                         d             }|D ]@  }dt        |      i}||   j'                         D ]
  \  }}|||<    |j)                  |       B |t+        |      dS y 7 w)Ninternal_errorr  rM     r  a  
            SELECT
                api_base,
                model_group,
                model,
                "startTime",
                request_id,
                EXTRACT(epoch FROM ("completionStartTime" - "startTime")) AS time_to_first_token
            FROM
                "LiteLLM_SpendLogs"
            WHERE
                "model_group" = $1 AND "cache_hit" != 'True'
                AND "completionStartTime" IS NOT NULL
                AND "completionStartTime" != "endTime"
                AND DATE("startTime") = DATE($2::timestamp)
            GROUP BY
                api_base,
                model_group,
                model,
                request_id
            ORDER BY
                time_to_first_token DESC;
        a  
            SELECT
                api_base,
                model_group,
                model,
                DATE_TRUNC('day', "startTime")::DATE AS day,
                AVG(EXTRACT(epoch FROM ("completionStartTime" - "startTime"))) AS time_to_first_token
            FROM
                "LiteLLM_SpendLogs"
            WHERE
                "startTime" BETWEEN $2::timestamp AND $3::timestamp
                AND "model_group" = $1 AND "cache_hit" != 'True'
                AND "completionStartTime" IS NOT NULL
                AND "completionStartTime" != "endTime"
            GROUP BY
                api_base,
                model_group,
                model,
                day
            ORDER BY
                time_to_first_token DESC;
        ry  r  time_to_first_tokenr   
request_idr  https:///openai/r   c                     | d   S Nr   r   rq  s    r   <lambda>z)model_streaming_metrics.<locals>.<lambda>      dSTg r   r  daterd  all_api_bases)r   r  r  db_not_connected_errorr>  r   rS  r   r  r   r9  setrC  	query_rawr   r0  addr   sortedr  r4  r  )r  r+  r  r  is_same_day	sql_query_all_api_basesdb_response_daily_entries
model_data	_api_baser  r0  
unique_key_request_id_day_combined_model_namer_  r  entry	model_keylatencys                         r   model_streaming_metricsrO    sI     %<<BB!66	
 	
 ?X\\^iQ.??I'G.."glln4K	0	. UN%((22()W K N% 	SJ":.I(F",-B"CJ(6(
n424N;/!%(!
&01F&G#~-+-N4(#&v; Y&'*9~$11';'A'A*'Ma'P$34?RN:&';<1	S4	  " f^%9%9%;AUVW! 	#CSX&E&4S&9&?&?&A +"	7#*i +OOE"		# !.1
 	
_ 	s   CG&G#DG&z/model/metricsz>View number of requests & avg latency per model on config.yamlz	gpt-4-32kr  customerc                 @  K   t         t        dddt        j                        |xs! t	        j
                         t        d      z
  }|xs t	        j
                         }||dk(  rd}||dk(  rd}d	}t               }t         j                  j                  ||||||       d {   }i }	||D ]j  }
|
d
   }|
d   }|
d   }|
d   }||	vri |	|<   t        |      }d|v rt        |      }d|v r|j                  d      d   }|j                  |       ||	|   |<   l 	 g }t        t        |	j                         d             }	|	D ]@  }dt        |      i}|	|   j                         D ]
  \  }}|||<    |j!                  |       B |t#        |      dS y 7 w)N Prisma Client is not initializedr-  r  rM     r/  	undefinednulla  
        SELECT
            api_base,
            model_group,
            model,
            DATE_TRUNC('day', "startTime")::DATE AS day,
            AVG(EXTRACT(epoch FROM ("endTime" - "startTime")) / NULLIF("completion_tokens", 0)) AS avg_latency_per_token
        FROM
            "LiteLLM_SpendLogs"
        WHERE
            "startTime" >= $2::timestamp AND "startTime" <= $3::timestamp
            AND "model_group" = $1 AND "cache_hit" != 'True'
            AND (
                CASE
                    WHEN $4 != 'null' THEN "api_key" = $4
                    ELSE TRUE
                END
            )
            AND (
                CASE
                    WHEN $5 != 'null' THEN "end_user" = $5
                    ELSE TRUE
                END
            )
        GROUP BY
            api_base,
            model_group,
            model,
            day
        HAVING
            SUM(completion_tokens) > 0
        ORDER BY
            avg_latency_per_token DESC;
    ry  r  r  avg_latency_per_tokenr2  r3  r   c                     | d   S r5  r   r6  s    r   r7  zmodel_metrics.<locals>.<lambda>  r8  r   r  r9  r:  )r   r  r   rS  r   r  r   r=  rC  r>  r   r0  r?  r   r@  r  r4  r  )r  r+  r  r  r  rP  rB  rC  rD  rE  rF  rG  r  rJ  _avg_latency_per_tokenrK  r_  r  rL  rM  rN  s                        r   model_metricsrY    s
      6!66	
 	
 @X\\^iR.@@I'G'[08{2!ID UN%((22()Wgx K N% 	PJ":.I(Fe$D%/0G%H">)')t$#&v; Y&'*9~$11';'A'A*'Ma'P$349ON4 !56	P 	  " f^%9%9%;AUVW! 	#CSX&E&4S&9&?&?&A +"	7#*i +OOE"		# !.1
 	
K s   B&F(F)C4Fz/model/metrics/slow_responsesz/View number of hanging requests per model_groupc           	        K   t         t        dddt        j                        ||dk(  rd}||dk(  rd}|xs! t	        j
                         t        d      z
  }|xs t	        j
                         }t        j                  j                  xs d	}t        |      }d
}t         j                  j                  |||||||       d {   }|9|D ]4  }	|	j                  d      xs d}
d|
v r|
j                  d      d   }
|
|	d<   6 |S 7 Aw)NrR  r-  r  rM  rT  rU  rS  r/  r  a  
SELECT
    api_base,
    COUNT(*) AS total_count,
    SUM(CASE
        WHEN ("endTime" - "startTime") >= (INTERVAL '1 SECOND' * CAST($1 AS INTEGER)) THEN 1
        ELSE 0
    END) AS slow_count
FROM
    "LiteLLM_SpendLogs"
WHERE
    "model_group" = $2
    AND "cache_hit" != 'True'
    AND "startTime" >= $3::timestamp
    AND "startTime" <= $4::timestamp
    AND (
        CASE
            WHEN $5 != 'null' THEN "api_key" = $5
            ELSE TRUE
        END
    )
    AND (
        CASE
            WHEN $6 != 'null' THEN "end_user" = $6
            ELSE TRUE
        END
    )
GROUP BY
    api_base
ORDER BY
    slow_count DESC;
    ry  r   r3  r   )r   r  r   rS  r   r  r   r   rH  r  rR  rC  r>  r   r0  )r  r+  r  r  r  rP  r  rB  rD  rowrG  s              r   model_metrics_slow_responsesr\    s:      6!66	
 	
 '[08{2@X\\^iR.@@I'G 	11DDK  /0IB &((22 K  	(C
+1rIY&%OOJ7:	'C
O		(
 !s   CDD
	ADz/model/metrics/exceptionsz7View number of failed requests per model on config.yamlc                   K   t         t        dddt        j                        |xs! t	        j
                         t        d      z
  }|xs t	        j
                         }||dk(  rd}	 d	}t         j                  j                  |||||       d {   }g }t               }		 ||D ]  }
|
j                  d
d      }|
j                  dd      }|
j                  di       }||d}|j                  |       |j                  |       |j                         D ]  \  }}|	j                  |         |t        |	      dS 7 w)NrR  r-  r  rM  rS  r/  rT  rU  aD  
        WITH cte AS (
            SELECT 
                CASE WHEN api_base = '' THEN litellm_model_name ELSE CONCAT(litellm_model_name, '-', api_base) END AS combined_model_api_base,
                exception_type,
                COUNT(*) AS num_rate_limit_exceptions
            FROM "LiteLLM_ErrorLogs"
            WHERE 
                "startTime" >= $1::timestamp 
                AND "endTime" <= $2::timestamp 
                AND model_group = $3
            GROUP BY combined_model_api_base, exception_type
        )
        SELECT 
            combined_model_api_base,
            COUNT(*) AS total_exceptions,
            json_object_agg(exception_type, num_rate_limit_exceptions) AS exception_counts
        FROM cte
        GROUP BY combined_model_api_base
        ORDER BY total_exceptions DESC
        LIMIT 200;
    combined_model_api_baser   total_exceptionsr   exception_counts)r  r_  )rd  exception_types)r   r  r   rS  r   r  r   rC  r>  r=  r   r  r4  r  r?  r  )r  r+  r  r  r  rP  rB  rD  r_  ra  rF  r  r_  r`  curr_rowr  r  s                    r   model_metrics_exceptionsrc  e  sg      6!66	
 	
 @X\\^iR.@@I'G'[0I, &((229g'<g K HeO % 	'JNN#<bAE)~~.@!D)~~.@"E$4H OO,-OOH%(..0 '1##A&'	' o1FGG?s   BEEB/Ec                 <   | j                  di       }t        |       }|i k(  r;| j                  di       }|j                  dd       }	 t        j                  |      }|i k(  rc| j                  di       }|j                  dd       }|j                  d      }t        |      dkD  r|d   }	 t        j                  ||d         }|j                         D ]  \  }}||vs|||<    || d<   t        | 	      } | S # t        $ r i }Y w xY w# t        $ r i }Y Sw xY w)
Nr  r#  r  r  r%  r   r  r  deployment_dict)	r   r  r   r  r   r0  r9  r  rR   )r  r  r  r  r(  r)  r  r  s           r   _get_proxy_model_inforg    sK   <,J 0e< R#3R8&**7D9	$!(!7!7m!L R#3R8&**7D9#))#.{a'OM	$!(!7!7#Q"
 #((* 1JJqM %E,1%HEL/  	$!#	$  	$!#	$s$   	C< -D <D
	D
DDz/model/infoz/v1/model/infolitellm_model_idc           
        K   t         e	 t        t        t        j                  t                     }t        dt        t               |      }|j                         }t        |      }d|iS t        t        dddi	      t        t        ddd
i	      |Lt        j                  |      }|t        ddd| di	      t        |j                  d            }d|giS g }t        t               }t        g }n(t        j#                         }t        j%                         }t'        | ||      }	t)        | ||      }
t+        |	|
|t         t,        j/                  dd            }t1        |      dkD  rO|}t        j3                         a
t        1t        D cg c]  }|d   |v s| }}t5        j6                  |      }ng }|D ]  }t        |      } t9        j:                  d|       d|iS # t
        $ r i }Y w xY wc c}w w)a  
    Provides more info about each model in /models, including config.yaml descriptions (except api key and api base)

    Parameters:
        litellm_model_id: Optional[str] = None (this is the value of `x-litellm-model-id` returned in response headers)

        - When litellm_model_id is passed, it will return the info for that specific model
        - When litellm_model_id is not passed, it will return the info for all models

    Returns:
        Returns a dictionary containing information about each model.

    Example Response:
    ```json
    {
        "data": [
                    {
                        "model_name": "fake-openai-endpoint",
                        "litellm_params": {
                            "api_base": "https://exampleopenaiendpoint-production.up.railway.app/",
                            "model": "openai/fake"
                        },
                        "model_info": {
                            "id": "112f74fab24a7a5245d2ced3536dd8f5f9192c57ee6e332af0f0512e08bed5af",
                            "db_model": false
                        }
                    }
                ]
    }

    ```
    r#  r;   r2  re  rd  r  rL  zLLM Model List not loaded in. Make sure you passed models in your config.yaml or on the LiteLLM Admin UI. - https://docs.litellm.ai/docs/proxy/configsr  zLLM Router is not loaded in. Make sure you passed models in your config.yaml or on the LiteLLM Admin UI. - https://docs.litellm.ai/docs/proxy/configsr*  r  zModel id = z not found on litellm proxyTr=  r  r
  Fr  r   r  r%  )r|  r
   rj  r   r  r   rx   ry   
model_dumprR   r   r   r   rK  rg  r-   r  r  r  rC   rD   rB   r   r   r9  r  r5  r6  r1   r   )r  rh  r  _deployment_info_deployment_info_dictdeployment_infor  r	  r  r  r  all_models_strrt  r*  _relevant_modelsr  s                   r   model_info_v1rp    s{    ` 	#D'*@*@z*RSJ &)  "
 !1 ; ; = E1!
 -..  r
 	
   q
 	
 #$33=M3N"{+;*<<WX  !6!,,$,?!
 ./00J0;D0A%557(@@B+)/J
 "+)/K
 -).223JERN >Q$#224%) Q|_-K    '78JJ 3%E23 /<Jo  	J	X s;   H(G6 E,HH+H/AH6HHHHrn  r$  c                 t    g }|D ]0  }|||k7  r| j                  |      }| |j                  |       2 |S )N)r$  )get_model_group_infor4  )r   rn  r$  model_groupsr  _model_group_infos         r   _get_model_group_inforu  t  sY     *,L 3"{e';&;;;N( 123 r   z/model_group/infoc           
        K   t         t        dddi      t        t        dddi      t        t              }t        g }n(t        j                         }t        j                         }t        | ||      }t        | ||      }t        |||t        t        j                  dd      	      }t        t        ||
      }d|iS w)a#  
    Get information about all the deployments on litellm proxy, including config.yaml descriptions (except api key and api base)

    - /model_group/info returns all model groups. End users of proxy should use /model_group/info since those models will be used for /chat/completions, /embeddings, etc.
    - /model_group/info?model_group=rerank-english-v3.0 returns all model groups for a specific model group (`model_name` in config.yaml)

    

    Example Request (All Models):
    ```shell
    curl -X 'GET'     'http://localhost:4000/model_group/info'     -H 'accept: application/json'     -H 'x-api-key: sk-1234'
    ```

    Example Request (Specific Model Group):
    ```shell
    curl -X 'GET'     'http://localhost:4000/model_group/info?model_group=rerank-english-v3.0'     -H 'accept: application/json'     -H 'Authorization: Bearer sk-1234'
    ```

    Example Request (Specific Wildcard Model Group): (e.g. `model_name: openai/*` on config.yaml) 
    ```shell
    curl -X 'GET'     'http://localhost:4000/model_group/info?model_group=openai/tts-1' 
    -H 'accept: application/json'     -H 'Authorization: Bearersk-1234'
    ```

    Learn how to use and set wildcard models [here](https://docs.litellm.ai/docs/wildcard_routing)

    Example Response:
    ```json
        {
            "data": [
                {
                "model_group": "rerank-english-v3.0",
                "providers": [
                    "cohere"
                ],
                "max_input_tokens": null,
                "max_output_tokens": null,
                "input_cost_per_token": 0.0,
                "output_cost_per_token": 0.0,
                "mode": null,
                "tpm": null,
                "rpm": null,
                "supports_parallel_function_calling": false,
                "supports_vision": false,
                "supports_function_calling": false,
                "supported_openai_params": [
                    "stream",
                    "temperature",
                    "max_tokens",
                    "logit_bias",
                    "top_p",
                    "frequency_penalty",
                    "presence_penalty",
                    "stop",
                    "n",
                    "extra_headers"
                ]
                },
                {
                "model_group": "gpt-3.5-turbo",
                "providers": [
                    "openai"
                ],
                "max_input_tokens": 16385.0,
                "max_output_tokens": 4096.0,
                "input_cost_per_token": 1.5e-06,
                "output_cost_per_token": 2e-06,
                "mode": "chat",
                "tpm": null,
                "rpm": null,
                "supports_parallel_function_calling": false,
                "supports_vision": false,
                "supports_function_calling": true,
                "supported_openai_params": [
                    "frequency_penalty",
                    "logit_bias",
                    "logprobs",
                    "top_logprobs",
                    "max_tokens",
                    "max_completion_tokens",
                    "n",
                    "presence_penalty",
                    "seed",
                    "stop",
                    "stream",
                    "stream_options",
                    "temperature",
                    "top_p",
                    "tools",
                    "tool_choice",
                    "function_call",
                    "functions",
                    "max_retries",
                    "extra_headers",
                    "parallel_tool_calls",
                    "response_format"
                ]
                },
                {
                "model_group": "llava-hf",
                "providers": [
                    "openai"
                ],
                "max_input_tokens": null,
                "max_output_tokens": null,
                "input_cost_per_token": 0.0,
                "output_cost_per_token": 0.0,
                "mode": null,
                "tpm": null,
                "rpm": null,
                "supports_parallel_function_calling": false,
                "supports_vision": true,
                "supports_function_calling": false,
                "supported_openai_params": [
                    "frequency_penalty",
                    "logit_bias",
                    "logprobs",
                    "top_logprobs",
                    "max_tokens",
                    "max_completion_tokens",
                    "n",
                    "presence_penalty",
                    "seed",
                    "stop",
                    "stream",
                    "stream_options",
                    "temperature",
                    "top_p",
                    "tools",
                    "tool_choice",
                    "function_call",
                    "functions",
                    "max_retries",
                    "extra_headers",
                    "parallel_tool_calls",
                    "response_format"
                ]
                }
            ]
            }
    ```
    r  rL  zLLM Model List not loaded inr  zLLM Router is not loaded inr  r
  Fr  )r   rn  r$  rd  )r   r   r   r-   r  r  r  rC   rD   rB   r|  r   r   ru  )r  r$  r	  r  r  r  rn  rs  s           r   model_group_inforw    s     B W.L$M
 	
 W.K$L
 	
 1<D0A%557(@@B+)/J
 "+)/K
 -).223JERN *?n+*L L!!s   CCz/model/deletez;Allows deleting models in the model list in the config.yamlr  c                 2  K   	 	 t         t        dddi      t        du r	 t         j                  j                  j                  d| j                  i       d {   }|t        ddd	| j                   d
i      t         t        j                  | j                         dd|j                   diS t        dddi      7 j# t        $ r}t        |t              rYt        t        |ddt        |       d      t        j                   t        |dd      t        |dt"        j$                              t        |t              r|t        dt        |      z   t        j                   t        |dd      t"        j$                        d }~ww xY ww)Nr  rL  r  r  Trw  r@  r  zModel with id=z not found in dbr%  r   zModel: z deleted successfullyr	  r  r
  r  r9  r  rO  rM  r  )r   r   rt  rC  ra  deleter  r   r(  rw  r   r   r  r	  r   r  r  r   rT  )r  r  r  s      r   delete_modelrz  M  s    ?
	  u  $
 )++CCJJ!:==1 K  F ~# ##~jmm_DT%UV  %,,
,>(99NOPPb !.  
a' 8/DSVHA-NO$//a&1Qv/J/JK	  >*G,s1v5 ++!Wf-,,	
 	

s=   FAC	 CAC	 7F8C	 		FB=FFFz/model/settingszMReturns provider name, description, and required parameters for each providerc            	         K   g } t         j                  D ]2  }| j                  t        |t        j                  |                   4 | S w)a?  
    Used by UI to generate 'model add' page
    {
        field_name=field_name,
        field_type=allowed_args[field_name]["type"], # string/int
        field_description=field_info.description or "", # human-friendly description
        field_value=general_settings.get(field_name, None), # example value
    }
    )r9  )r   fields)r   provider_listr4  ProviderInfoget_provider_fields)returned_listr  s     r   model_settingsr    sQ     $ M)) 
22xP	

 s   A
Az/alerting/settingszFReturn the configurable alerting param, description, and current valuer  c                   K   	 t         't        ddt        j                  j                  i      | j
                  t        j                  k7  rAt        dddj                  t        j                  j                  | j
                        i      t         j                  j                  j                  ddi       d {   }|E|j                  9t        |j                        }|j                  di       }|j                  d	      }ni }d }d
did
did
did
did
did
did
did
did
did	}t         j"                  }|j$                  j'                         }g }d}	t(        j                  d	      r$t+        t(        d	   t,              rdt(        d	   v rd}	t/        d|d   d
   d|	|dndd d      }
|j1                  |
       t2        j4                  j7                         D ]m  \  }}||v sd }||v rd}nd}t/        |||   d
   |j8                  xs d|j                  |d       ||j:                  |dk(  rdnd      }
|j1                  |
       o |S 7 w)Nr  rL  r  {}, your role={}r?  r   r@  r  r  r  BooleanInteger)	slack_alertingdaily_report_frequencyreport_check_intervalbudget_alert_ttloutage_alert_ttlregion_outage_alert_ttlminor_outage_alert_thresholdmajor_outage_alert_thresholdmax_outage_alert_list_sizeFr  Tr  zhEnable slack alerting for monitoring proxy in production: llm outages, budgets, spend tracking failures.)
field_name
field_typefield_descriptionfield_valuestored_in_dbfield_default_valuepremium_fieldr   r  )r   r   r  r<  r>  r  r  r  r*   not_allowed_accessrC  rD  rE  rF  r   r   r   rH  r  rj  r   r   r  
ConfigListr4  r   model_fieldsr  r  r   )r  rR  db_general_settings_dictalerting_args_dictalerting_valuesallowed_args_slack_alerting_slack_alerting_args_dict
return_valis_slack_enabled_response_objr  
field_info_stored_in_dbs                 r   alerting_settingsr    s     .EEKKL
 	

 ""&6&B&BB+22%88>>%//
 	
 !. 0 0 ? ? J J/0 !K !  &+>+J+J+V#'(;(G(G#H #;#?#?QS#T*B*F*Fz*R "9-#)9"5"()!4#Y/#Y/$*I#6)/(;)/(;'-y&9
L &7%N%NO / = = H H JJJ'J$d- &z22## 01&9 E$,8Te M m$"3"@"@"F"F"H -
J%,0M// $ %&%'
3F;","8"8">B599*dK*$.$6$6&*CCD
M m,)-* Ks   B<I>I?D)I)A(Iz/queue/chat/completionsr  )r-  r  r  c                   K   	 i }	 | j                          d {   }t        | j                        | j                  t	        | j
                        t        j                  |      d|d<   t        j                  d|       t        j                  dd       xs t        xs	 |xs |d   |d<   |j                  dd       |j                  |j                  |d<   d|vri |d<   |j                  |d   d<   |j                  |d   d	<   t	        | j
                        }|j                  d
d        ||d   d<   t!        |dd       |d   d<   |j                  |d   d<   t!        |dd       |d   d<   t        | j                        |d   d<   t"        r	t"        |d<   t$        r	t$        |d<   t&        r	t&        |d<   t(        r	t(        |d<   t*        't-        ddt.        j0                  j2                  i      t+        j4                  d)i | d {   }d|v r |d   du rt7        t9        |||      d      S |j
                  j;                  dt        |d         i       |S 7 E7 V# t<        $ r}t>        jA                  |||        d {  7   tC        |t,              rYtE        t!        |d!d"t        |       d#      tF        jH                  t!        |d$d%      t!        |d&tJ        jL                        '      tC        |tD              r|tE        d(t        |      z   tF        jH                  t!        |d$d%      tJ        jL                  '      d }~ww xY ww)*N)r`  methodrQ  rr  proxy_server_requestzreceiving data: %sr  r  r  metadatauser_api_keyuser_api_key_metadataauthorizationrQ  r  user_api_key_aliasuser_api_key_user_idr  user_api_key_team_idendpointrw  rx  rv  ry  r  rL  r  r-  Tr  r/  r;  zx-litellm-prioritypriorityr  r  r
  r  r9  r  rO  rM  r  r   )'r   r   r`  r  r   rQ  r5  r1   r   r   r   r|  r  r  r  rg  r	  r  r  rB  r  r   r   r  r  r>  schedule_acompletionr   r  r  r   r   r  r   r  r  r  r   rT  )rI  r  r  r  rd  _headersr_  r  s           r   async_queue_requestr  '  sr     D\
\\^# w{{#nnGOO,IIdO	(
#$ 	""#7>  !3T:  G}	 	W 88FD!).?.G.G.S,44DLT!!D+<+D+DZ(4E4N4NZ01(T	
 '/Z#18{D2
Z-. 4E3L3LZ/03:y$4
Z/0 (+7;;'7Z$ "2D&:D"#!0D,D2C2Q2Q2W2W(X  $88@4@@ h4!7$$&7%!%
 /  	  '')=s4
CS?T(UVQ $r A   
66/ATX 7 
 	
 	
 a' 8/DSVHA-NO$//a&1Qv/J/JK	  >*G,s1v5 ++!Wf-,,	
 	

sd   MI$ IG0I$ I"'I$ 3M4*I$ MI$ "I$ $	M-MJ	CMMMz/fallback/login)r-  r  c                   K   t        j                  dt        | j                              }t        j                  d      }|j	                  d      r|dz  }n|dz  }|ddlm}  |t        d	      S ddlm}  |t        d	      S w)
z
    Create Proxy API Keys using Google Workspace SSO. Requires setting PROXY_BASE_URL in .env
    PROXY_BASE_URL should be the your deployed proxy endpoint, e.g. PROXY_BASE_URL="https://litellm-production-7002.up.railway.app/"
    Example:
    r\  UI_USERNAMEr%  zsso/callbackz/sso/callbackr   )HTMLResponser  )rP  rO  )r   r  r   base_urlendswithfastapi.responsesr  rG   )rI  redirect_urlui_usernamer  s       r   fallback_loginr    s|      99-s73C3C/DEL))M*KS!&' 	3I3??2I3??s   BBz/login)r  c                 `
  K   	 dd l }t        +t        dt        j                  dt        j                        | j                          d {   }t        |j                  d            }t        |j                  d            }t        j                  dd	      }t        j                  d
d       }|t        t        t              nd }|+t        dt        j                  d
t        j                        d }d }t        6t        j                   j"                  j%                  dd|ii       d {   }t'               }		 t)        j*                  ||      rt)        j*                  ||      rt,        j.                  }t0        }
t0        }t        j                  dd       t        j2                  d   |
k(  s	|
t0        k(  rt        j                  dt0              }t5        t7        ||             d {    t        j                  d      <t9        d0ddit,        j.                  dt:        j<                  g i i d|dd	 d {   }n+t        dt        j                  dt        j                        |d   }t        j                  dd      }|j?                  d      r|dz  }n|dz  }dd l }|jC                  |
|d |dtD        tF        j                  d d!      |	d"t        d#$      }|d%|
z   z  }tI        |d&'      }|jK                  d|(       |S |	 tM        |d)d*      }
tM        |d+t,        jN                        }tM        |dd*      }tM        |dd*      }tQ        |,      }t)        j*                  ||      st)        j*                  ||      rt        j                  d      .t9        d0ddi|dt:        j<                  g i i d|
dd	 d {   }n+t        dt        j                  dt        j                        |d   }t        j                  dd      }|j?                  d      r|dz  }n|dz  }dd l }|jC                  |
|||dtD        tF        j                  d d!      |	d"t        d#$      }|d%|
z   z  }tI        |d&'      }|jK                  d|(       |S t        d-| t        j                  d.t        jR                        t        d/t        j                  d.t        jR                        # t        $ r t        j                  g d       Y w xY w7 7 7 07 7 [w)1Nr   )pipinstallzpython-multipartMaster Key not set for Proxy. Please set Master Key to use Admin UI. Set `LITELLM_MASTER_KEY` in .env or set general_settings:master_key in config.yaml.  https://docs.litellm.ai/docs/proxy/virtual_keys. If set, use `--detailed_debug` to debug issue.r   rM  usernamer  r  rq  UI_PASSWORDzset Proxy master key to use UI. https://docs.litellm.ai/docs/proxy/virtual_keys. If set, use `--detailed_debug` to debug issue.
user_emailequalsr@  r  )r  r  r  r   r  r  24hrlitellm-dashboard	r  r  key_max_budgetr  r  r?  r  r  r  z_No Database connected. Set DATABASE_URL in .env. If set, use `--detailed_debug` to debug issue.r  r\  r   r%  zui/r   username_passwordlitellm_key_header_nameAuthorizationr  r  r  r  login_methodr   auth_header_name(disabled_non_admin_personal_key_creationHS256	algorithmz?userID=i/  )r`  rO  )r  r>  r  unknownr  r  zAInvalid credentials used to access UI.
Not valid credentials for invalid_credentialszVInvalid credentials used to access UI.
Check 'UI_USERNAME', 'UI_PASSWORD' in .env filer   )*	multipartImportErrorr*  runr   r  r  r  r   rS  r  r   r   r   r  r   rC  litellm_usertablerE  rc   secretscompare_digestr  r  r   r   r^   UpdateUserRequestra   r   max_ui_session_budgetr  jwtencoder   r   r   
set_cookier	  INTERNAL_USER_VIEW_ONLYrs   HTTP_401_UNAUTHORIZED)rI  r  r  r  r  r  ui_password	_user_rowr  r  r  key_user_idr_  r  litellm_dashboard_uir  	jwt_tokenredirect_responser  	_passwordhash_passwords                        r   loginr    s|    
?  P ++66	
 	
 D488J'(H488J'(H))M73K))M40K)3)?c*oT V ++66	
 	
 I 	   '**<<GG(H!56 H 
 
	 	56 -
 h49O9O+: %00	* /II&-9

+,700))$46NOK
 "##
 	
 	
 99^$03 " "2!=!= &&-&C&C ! *2
 H !y$//$::	  w!yy)92>((- E)  F* JJ""& 3 ,$4$8$8-% =e   
	  	
W 44,1ESVW$$	$B  			
 )Y	:{$4$L$L
	 Yi@
Iz9=	 #2!!(I6':P:P9;
 yy(4!7 "!&" &/$**1*G*G"$#%"$!"#*#6
"  %}(33(>>	  7#C#%99-=r#B #,,S1$-$$.$

&",!*$7$0(8(<(<1?) Ai ! # I  !J$88  0(c! ((WI(F$$ \]e\fg$//+11	  m ++'--	
 	
Q  ?=>?  4
<	
Hs   T.S8 AT.TC%T.2T"3CT.5T%6AT.T(F
T.T+D'T.8 TT.TT."T.%T.(T.+T.z/onboarding/get_tokeninvite_linkc                   K   t         +t        dt        j                  dt        j
                        t        't        ddt        j                  j                  i      t        j                  j                  j                  d| i	       d{   }|t        d
ddi      t        j                  j!                         j#                         }|j$                  j#                         }||k  rt        d
ddi      t        j                  j!                         }t        j                  j                  j'                  d| i||d|j(                  d       d{   }t        j                  j*                  j                  d|j(                  i	       d{   }|t        d
ddi      |j,                  }t/        d&ddi|j0                  dt        j2                  g i i d|j(                  dd	 d{   }|d   }	t5        j6                  dd      }
|
j9                  d      r|
dz  }
n|
dz  }
ddl}t=               }|j?                  |j(                  |	|j,                  |j0                  dt@        tB        jE                  dd       |d!t         d"#      }|
d$jG                  ||      z  }
|
||d%S 7 %7 [7  7 ĭw)'z
    - Get the invite link
    - Validate it's still 'valid'
    - Invalidate the link (prevents abuse)
    - Get user from db
    - Pass in user_email if set
    Nr  r   rM  r  rL  r  r  r@    %Invitation link does not exist in db.Invitation link has expired.T)accepted_atr  is_acceptedr  r  r  User does not exist in db.r  r  r  r   r  r  r  r\  r   r%  zui/onboardingz/ui/onboardingr  r  r  r  r  r  z?token={}&user_email={})	login_urlr  r  r   )$r   r  r  r  r   rS  r   r   r  r<  r>  rC  litellm_invitationlinkr  r   rG  get_utc_datetimer9  
expires_atr  r  r  r  ra   r  r  r   r  r  r  rc   r  r   r   r   r*   )r  
invite_objutc_now_dateexpires_at_datecurrent_timer&  user_objr  r_  r  r  r  r  r  s                 r   
onboardingr    s
      P ++66	
 	
 .EEKKL
 	

 %''>>JJ[! K  J W.U$V
 	

 ==11388:L ++002O%W.L$M
 	

 ==113L55<<[!'&$,,	
 =  	A #%%77CC*,,- D  H W.J$K
 	
 $$J+  "++%;;''*

 H 7
C99%5r:$$S)/ 00 	56 - 

''"--!++/( 0 4 4)?! 9a	
 	  I" 5<<Y
SS)  m&	sK   BK J5CK J8=K J;AK 9J>:B<K 8K ;K >K z/onboarding/claim_tokenrd  c                 h  K   t         't        ddt        j                  j                  i      t         j
                  j                  j                  d| j                  i       d{   }|t        dddi      t        j                  j                         j                         }|j                  j                         }||k  rt        ddd	i      |j                  d
u rnt        dddi      |j                  | j                  k7  r3t        dddj!                  | j                  |j                        i      t#        | j$                        }t         j
                  j&                  j)                  d|j                  id|i       d{   }|t        dddi      |S 7 D7 w)a&  
    Special route. Allows UI link share user to update their password.

    - Get the invite link
    - Validate it's still 'valid'
    - Check if user within initial session (prevents abuse)
    - Get user from db
    - Update user password

    This route can only update user password.
    Nr  rL  r  r  r@  r  r  r  TzThe invitation link was never validated. Please file an issue, if this is not intended - https://github.com/BerriAI/litellm/issues.zwInvalid invitation link. The user id submitted does not match the user id this link is attached to. Got={}, Expected={}r  r  r  r  r  )r   r   r  r<  r>  rC  r  r  invitation_linkr   rG  r  r9  r  r  r  r*   rs   r  r  r  )rd  r  r  r  r  r  s         r   claim_onboarding_linkr    s     .EEKKL
 	

 %''>>JJT))* K  J W.U$V
 	

 ==11388:L ++002O%W.L$M
 	
 %  _
 	
 T\\)  S  Z  ZLL*"4"4
 	
 T]]3M"%%77>>*,,-Z4O ?  H W.J$K
 	
 OiVs%   A'F2)F-*D+F2F0F20F2z
/get_imagec                     t         j                  j                  t         j                  j                  t                    } t         j                  j                  | d      }t        j                  d|      }t        j                  d|       |j                  d      rt               }|j                  |      }|j                  dk(  r]t         j                  j                  | d      }t        |d      5 }|j                  |j                         ddd       t!        |d	
      S t!        |d	
      S t!        |d	
      S # 1 sw Y   0xY w)zGet logo to show on admin UIzlogo.jpgUI_LOGO_PATHzReading logo from path: %s)zhttp://r2  r  zcached_logo.jpgwbNz
image/jpegr;  )r   r   rU  rV  __file__r]  r  r1   r   ra  r:   r   rO  r(  r   rP  r   )current_dirdefault_logo	logo_pathr  r_  
cache_pathfs          r   	get_imager	  N  s   
 ''//"''//(";<K77<<Z8L		.,7I;YG 34::i(3&k3DEJj$' *1(()*  
|DD  FF I,??* *s   4D??Ez/invitation/newzInvite Linksc           	        K   t         't        ddt        j                  j                  i      |j
                  t        j                  k7  rAt        dddj                  t        j                  j                  |j
                        i      t        j                  j                         }|t        d      z   }	 t         j                  j                  j!                  | j"                  |||j"                  xs t$        ||j"                  xs t$        d	       d{   }|S 7 # t&        $ r9}d
t)        |      v rt        dddi      t        ddt)        |      i      d}~ww xY ww)%  
    Allow admin to create invite links, to onboard new users to Admin UI.

    ```
    curl -X POST 'http://localhost:4000/invitation/new'         -H 'Content-Type: application/json'         -d '{
            "user_id": "1234" // 👈 id of user in 'LiteLLM_UserTable'
        }'
    ```
    Nr  rL  r  r  r.  r/  )r  r  r  r  r  r  r  z*Foreign key constraint failed on the fieldzYUser id does not exist in 'LiteLLM_UserTable'. Fix this by creating user via `/user/new`.r  )r   r   r  r<  r>  r  r  r  r*   r  r   rG  r  r   rC  r  r  r  r   r   r   )rd  r  r  r  r_  r  s         r   new_invitationr  q  si    . .EEKKL
 	

 ""&6&B&BB+22%88>>%//
 	
 ==113L	q 11JG&))@@GG<<*(/77S;S*/77S;S H 	
 	
 	
  G73q6Ax  Wc!f4EFFGs=   B9E,<A"D' D%D' $E,%D' '	E)04E$$E))E,z/invitation/infoinvitation_idc                   K   t         't        ddt        j                  j                  i      |j
                  t        j                  k7  rAt        dddj                  t        j                  j                  |j
                        i      t         j                  j                  j                  d| i       d{   }|t        dddi      |S 7 w)	r  Nr  rL  r  r  r  r@  -Invitation id does not exist in the database.)r   r   r  r<  r>  r  r  r  r*   r  rC  r  r  )r  r  r_  s      r   invitation_infor    s     . .EEKKL
 	

 ""&6&B&BB+22%88>>%//
 	
 #%%<<HH]# I  H LM
 	
 Os   B;C=C>Cz/invitation/updatec           	        K   t         't        ddt        j                  j                  i      |j
                  (t        dddj                  |j
                        i      t        j                  j                         }t         j                  j                  j                  d| j                  i| j                  | j                  |||j
                  d	       d{   }|t        ddd
i      |S 7 w)uZ  
    Update when invitation is accepted
    
    ```
    curl -X POST 'http://localhost:4000/invitation/update'         -H 'Content-Type: application/json'         -d '{
            "invitation_id": "1234" // 👈 id of invitation in 'LiteLLM_InvitationTable'
            "is_accepted": True // when invitation is accepted
        }'
    ```
    Nr  rL  r  r  z'Unable to identify user id. Received={}r  )r  r  r  r  r  r  r  )r   r   r  r<  r>  r  r*   r   rG  r  rC  r  r  r  r  )rd  r  r  r_  s       r   invitation_updater    s    2 .EEKKL
 	

   (BII%--
 	
 ==113L"%%<<CCT''($$++'&+33
 D 	 	H LM
 	
 O!	s   CC: C8!C:z/invitation/deletec                   K   t         't        ddt        j                  j                  i      |j
                  t        j                  k7  rAt        dddj                  t        j                  j                  |j
                        i      t         j                  j                  j                  d| j                  i       d{   }|t        dddi      |S 7 w)	u  
    Delete invitation link
    
    ```
    curl -X POST 'http://localhost:4000/invitation/delete'         -H 'Content-Type: application/json'         -d '{
            "invitation_id": "1234" // 👈 id of invitation in 'LiteLLM_InvitationTable'
        }'
    ```
    Nr  rL  r  r  r  r@  r  )r   r   r  r<  r>  r  r  r  r*   r  rC  r  ry  r  r2  s      r   invitation_deleter  $  s     0 .EEKKL
 	

 ""&6&B&BB+22%88>>%//
 	
 #%%<<CCT''( D  H LM
 	
 Os   CC!CC!z/config/updatezconfig.yamlconfig_infoc                   K   	 ddl }	 t        t        d      t        durt	        dddi      | j                  d	      }t        j                  |      }|j                         D ]H  \  }}|d
k(  st        j                  j                  j                  d|i||dd|id       d{    J t        j                          d{   }t        j                  d|       | j                  |j!                  di        | j                  j#                  d	      }|d   }|j                         D ]G  \  }}|dk(  r8d|vrddgi}n.t%        |d   t&              rd|d   vr|d   j)                  d       |||<   I ||d<   | j*                  l|j!                  di        | j*                  }|j                         D ]  \  }}t-        |      }	|	||<    |d   }
|j                         D ]  \  }}||   |
|<    | j.                  |j!                  di        | j.                  }i ||d   |d<   d|v rZd|d   v rSt%        |d   d   t&              r=t%        |d   t&              r*|d   d   |d   z   }t'        t1        |            }||d   d<   t        j3                  |       d{    t        j5                  t        t6               d{    ddiS 7 ,7 7 57 # t        $ r}t        j8                  dj;                  t=        |                   t        j                  t?        j@                                t%        |t              rYtC        tE        |ddt=        |       d       tF        jH                  tE        |d!d"      tE        |d#tJ        jL                        $      t%        |tB              r|tC        d%t=        |      z   tF        jH                  tE        |d!d"      tJ        jL                  $      d}~ww xY ww)&z
    For Admin UI - allows admin to update config via UI

    Currently supports modifying General Settings + LiteLLM settings
    r   NzNo DB ConnectedTr  rL  r	  r  r=  rM  r?  r?  rF  rF  r  r  r  zLoaded config: %sr   r  r  r  r  r  rN  r   )rb  r  r   zConfig updated successfullyzBlitellm.proxy.proxy_server.update_config(): Exception occured - {}r  r
  r  r9  r  rO  rM  r  )'r4  r   r   rt  r   r   jsonify_objectr  rC  rD  upsertr   r  r1   r   r   
setdefaultr   r   r  r4  r  rM   rN  r=  rj  re  r   rL  r*   r   r  r  r  r	  r  r  r   rT  )r  r4  updated_settingsr  r  r?  updated_general_settings_existing_settings_updated_environment_variablesr  _existing_env_variablesupdated_litellm_settingscombined_success_callbackr  s                 r   update_configr#  Z  s    @
	  -..D(b  '+++>(778HI$**, 	DAq%%#&&55<<'+121"E#0!"4 =   	 $..00""#6? ''30"5'2'C'C'H'H! (I ($ "((:!;0668 	*1..!);;.87)-D*#$6z$BDI"*<Z*HH.z:AA'J()"1%	* *<F%&,,85r:-8-N-N* 7<<> D1"6Q"?4C.q1D '--D&E#6<<> O1-KA-N'*O
 ''30"5'2'C'C$***+,*F%& #&>>&&1C*DD -./ABD !9:L!MtT123EF23EFG . 15S9R5S0T- 2 -.*
 &&&&999))';L * 
 	
 	
 899i 1F 	:	

  
""PWWA	

 	""9#7#7#9:a' 8/DSVHA-NO$//a&1Qv/J/JK	  >*G,s1v5 ++!Wf-,,	
 	
!
sy   PA/K" 48K" ,K-K" 
KGK" (K)&K" K K" PK" K" K"  K" "
P,DO==PPz/config/field/updatec           
        K   	 t         't        ddt        j                  j                  i      |j
                  t        j                  k7  r't        ddt        j                  j                  i      | j                  t        j                  vr(t        dddj                  | j                        i      	 t        di | j                  | j                  i t         j                   j"                  j%                  ddi	       d{   }||j&                  i }nt)        |j&                        }| j                  || j                  <   t         j                   j"                  j+                  ddidt-        j.                  |      d
dt-        j.                  |      id       d{   }|S # t        $ r2 t        dddj                  t        | j                              i      w xY w7 7 Fw)z=
    Update a specific field in litellm general settings
    Nr  rL  r  Invalid field={} passed in.z)Invalid type of field value={} passed in.r?  r   r@  r  rF  r  r  r   )r   r   r  r<  r>  r  r  r  r  r  ConfigGeneralSettingsr  r*   r  r   r  rC  rD  rE  rF  r   r  r   rW  rd  r  rR  r   r_  s        r   update_config_general_settingsr(    s     .EEKKL
 	

 ""&6&B&BB.AAGGH
 	

 3@@@:AA$//RS
 	



D$2B2B CD !. 0 0 ? ? J J/0 !K ! 
 "&9&E&E&M 3 ? ?@ )-(8(8T__%"%%44;;/0%7

ScHde$djj1A&BC
 <  H OC  
DKK))*
 	

s=   B7G5:!F3 /G5
G1B!G5,G3-G53;G..G53G5z/config/field/infor  c                   K   	 t         't        ddt        j                  j                  i      |j
                  t        j                  k7  r't        ddt        j                  j                  i      | t        j                  vrt        dddj                  |       i      t         j                  j                  j                  ddi       d {   }||j                  t        dddj                  |       i      t!        |j                        }| |v rt#        | ||    	      S t        dddj                  |       i      7 ww)
Nr  rL  r  r%  r?  r   r@  zField name={} not in DB)r  r  )r   r   r  r<  r>  r  r  r  r  r&  r  r*   rC  rD  rE  rF  r   ConfigFieldInfo)r  r  rR  r   s       r   get_config_general_settingsr+  A   st    
 .EEKKL
 	

 ""&6&B&BB.AAGGH
 	

 .;;;:AA*MN
 	
 !. 0 0 ? ? J J/0 !K ! 
 "&9&E&E&M6==jIJ
 	

   3 ? ?@))"%3CJ3O   !:!A!A*!MN %s   CEEA8Ez/config/listconfig_typec                   K   	 t         't        ddt        j                  j                  i      |j
                  t        j                  k7  rAt        dddj                  t        j                  j                  |j
                        i      t         j                  j                  j                  ddi       d{   }|"|j                  t        |j                        }ni }d	d
id	d
id	d
id	d
id	did}g }t        j                   j#                         D ]  \  }}||v s||   d	   }|dk(  r/|dk(  rt$        g}	ng }	|	D ]  }
|
j&                  j#                         D cg c]3  \  }}t)        ||j*                  dt,        j/                  |d      d      5 }}}d}|
j                   j#                         D ]6  \  }}t1        |d      r |j2                  |j2                  ||   _        |dz  }8 d}||v rd}n
|t,        v rd}t7        |||   d	   |j2                  xs dt,        j/                  |d      ||j8                  |      }|j;                  |        Id}d}||v rd}n
|t,        v rd}t7        |||   d	   |j2                  xs dt,        j/                  |d      ||j8                  |      }|j;                  |        |S 7 c c}}w w)z
    List the available fields + current values for a given type of setting (currently just 'general_settings'user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),)
    Nr  rL  r  r  r?  r   r@  r  r  PydanticModel)rT  rU  max_request_size_mbmax_response_size_mbr  r  r   )r  r  r  r  r  r   r  r_  TF)r  r  r  r  r  r  nested_fields)r   r   r  r<  r>  r  r  r  r*   r  rC  rD  rE  rF  r   r&  r  r  PassThroughGenericEndpoint__annotations__FieldDetailr   r   r   r2  r  r  r  r   r4  )r,  r  rR  r  r  r  r  r  typed_dict_typepydantic_class_listpydantic_class	sub_fieldsub_field_typer1  r  sub_field_infor  r  s                     r   get_config_listr;     sj     
 .EEKKL
 	

 ""&6&B&BB+22%88>>%//
 	
 !. 0 0 ? ? J J/0 !K !  &+>+J+J+V#'(;(G(G#H #%  #))!4)/(; &	2!' 3#)?";L J"7"D"D"J"J"L K1
J% +:6v>O/1!99+E*F'*,'&9 *5N :H9W9W9]9]9_	% 6I~ $'0'5'>'>.00@0D0DYPT0U)-	%M 	% C (44::<! !& $NMB . : : F !/ : : *#.@ q! %)M!%==(,#'77(-$.#-#/
#;F#C*4*@*@*FB$4$8$8T$J%2,6,>,>&3%M %%m4U*5Z !% $!99$(M#33$)M *)+J7?&0&<&<&B 0 4 4Z F!.(2(:(:"/! !!-0WK1Z AD	%s-   B<K>K?A'K'A K'8K
D5KKz/config/field/deletec                   K   	 t         't        ddt        j                  j                  i      |j
                  t        j                  k7  rAt        dddj                  t        j                  j                  |j
                        i      | j                  t        j                  vr(t        dddj                  | j                        i      t         j                  j                  j                  ddi	       d{   }||j                   (t        ddd
j                  | j                        i      t#        |j                         }|j%                  | j                  d       t         j                  j                  j'                  ddidt)        j*                  |      ddt)        j*                  |      id       d{   }|S 7 7 w)z|
    Delete the db value of this field in litellm general settings. Resets it to it's initial default value on litellm.
    Nr  rL  r  r  r%  r?  r   r@  zField name={} not in configr  rF  r  r  )r   r   r  r<  r>  r  r  r  r*   r  r  r&  r  rC  rD  rE  rF  r   rg  r  r   rW  r'  s        r   delete_config_general_settingsr=  	!  s    
 .EEKKL
 	

 ""&6&B&BB+22%88>>%//
 	
 3@@@:AA$//RS
 	
 !. 0 0 ? ? J J/0 !K ! 
 "&9&E&E&M:AA$//RS
 	

   3 ? ?@ $/"%%44;;/0%7

ScHde$djj1A&BC
 <  H O3"s%   D GGC	GGGGz/get/config/callbacks)r-  r  r  c                  0  K   	 ddl } t               }t        j                          d{   }|j	                  di       }|j	                  di       }|j	                  di       }|j	                  dg       }g }	 |D ]  }|dk7  r|dk(  rd	g}	n1|d
k(  rdg}	n(|dk(  rdg}	n|dk(  rdg}	n|dk(  rg d}	n|dk(  rg d}	ng }	i }
|	D ]-  }|j	                  |d      }|d|
|<   t        |      }||
|<   / |j                  ||
d       |dk(  sg d}i }|D ]-  }|j	                  |d      }|d||<   t        |      }|||<   / |j                  ||d        |j	                  dg       }g }d|v rdg}i }|D ]C  }|j	                  |d      }|t        j                  dd      }|||<   3t        |      }|||<   E t        j                  j                  }t        j                  j                         }t        j                  j                  }|j                  d|||d       g d}i }|D ]-  }|j	                  |d      }|d||<   t        |      }|||<   / |j                  d|d       t        i }nt        j                         }d||||dS 7 o# t         $ r}t#        j$                  dj'                  t)        |                   t+        |t,              rYt/        t1        |d d!t)        |       d"      t2        j4                  t1        |d#d$      t1        |d%t6        j8                        &      t+        |t.              r|t/        d't)        |      z   t2        j4                  t1        |d#d$      t6        j8                  &      d}~ww xY ww)(z~
    For Admin UI - allows admin to view config via UI
    # return the callbacks and the env variables for the callback

    r   NrN  r   r  r   r   	openmeterOPENMETER_API_KEY
braintrustBRAINTRUST_API_KEY	traceloopTRACELOOP_API_KEYcustom_callback_apiGENERIC_LOGGER_ENDPOINTotel)OTEL_EXPORTEROTEL_ENDPOINTOTEL_HEADERS	langsmith)LANGSMITH_API_KEYLANGSMITH_PROJECTLANGSMITH_DEFAULT_RUN_NAMEr  )r   	variables)LANGFUSE_PUBLIC_KEYLANGFUSE_SECRET_KEYLANGFUSE_HOSTr  r  SLACK_WEBHOOK_URL)r   rO  active_alertsalerts_to_webhook)	SMTP_HOST	SMTP_PORTSMTP_USERNAMESMTP_PASSWORDSMTP_SENDER_EMAILTEST_EMAIL_ADDRESSEMAIL_LOGO_URLEMAIL_SUPPORT_CONTACTemailr+  )r   r  alertsrM  available_callbacksz?litellm.proxy.proxy_server.get_config(): Exception occured - {}r  r
  r  r9  r  rO  rM  r  )r4  AllCallbacksr   r  r   rL   r4  r   r  r   rH  r  _all_possible_alert_typesr  r   get_settingsr   r1   r  r*   r   r   r   r  r	  r  r  r   rT  )r4  all_available_callbacksrK  _litellm_settingsrP  r  _success_callbacks_data_to_return	_callbackenv_varsenv_vars_dict_varenv_variablerJ  _langfuse_vars_langfuse_env_varsr  alerting_data_slack_vars_slack_env_varsr8  _decrypted_value_alerting_types_all_alert_types_alerts_to_webhook_email_vars_email_env_varsrO  r  s                                r   r  r  Q!  sS    o
"..(3355'OO,>C'OO,>C +0G L /223ErJ	 , 6	IJ&++ H ,., H +- 34H"77 9:H&(QH+- H  "H "$ >D#8#<#<T4#HL#+.2d+ +?\*R.=d+>  &&	'VWj("
 &("* CD#8#<#<T4#HL#+37*40 +?\*R3B*40C  &&&5GHi6	r &))*b9	i#K !O# =488tD'YY':DAF,2OD) (<,'O$,<OD)= 0GGSSO!99SSU  "99NN    #!0%4);		
  	9D044T4@L#(,% $8l#K (8%	9 	,	
 !)668  (#/#:
 	
_ 6l  
&&MTTA	

 a' 8/DSVHA-NO$//a&1Qv/J/JK	  >*G,s1v5 ++!Wf-,,	
 	

sA   N%J JC!J FJ NJ 	N$C*NNNz/config/yamlc                    K   ddiS w)a  
    This is a mock endpoint, to show what you can set in config.yaml details in the Swagger UI.

    Parameters:

    The config.yaml object has the following attributes:
    - **model_list**: *Optional[List[ModelParams]]* - A list of supported models on the server, along with model-specific configurations. ModelParams includes "model_name" (name of the model), "litellm_params" (litellm-specific parameters for the model), and "model_info" (additional info about the model such as id, mode, cost per token, etc).

    - **litellm_settings**: *Optional[dict]*: Settings for the litellm module. You can specify multiple properties like "drop_params", "set_verbose", "api_base", "cache".

    - **general_settings**: *Optional[ConfigGeneralSettings]*: General settings for the server like "completion_model" (default model for chat completion calls), "use_azure_key_vault" (option to load keys from azure key vault), "master_key" (key required for all calls to proxy), and others.

    Please, refer to each class's description for a better understanding of the specific attributes within them.

    Note: This is a mock endpoint primarily meant for demonstration purposes, and does not actually provide or change any configurations.
    helloworldr   )r  s    r   config_yaml_endpointr{  "  s     . Ws   z/get/litellm_model_cost_map)r  r  c                     K   	 t         j                  } | S # t        $ r}t        ddt	        |       d      d }~ww xY ww)Nr  zInternal Server Error (r  r  )r   
model_costr   r   r   )_model_cost_mapr  s     r   get_litellm_model_cost_mapr  *"  sK     
!,, 
,SVHA6
 	

s   A A	>9>Ar%  )r  c                    K   yw)NzLiteLLM: RUNNINGr   r  s    r   homer  :"  s	     s   z/routesc            	        K   g } t         j                  D ]d  }t        |dd      }|t        |dd      t        |dd      t        |dd      t        |dd      r|j                  ndd}| j	                  |       f d| iS w)zD
    Get a list of available routes in the FastAPI application.
    r  Nr   methodsr   )r   r  r   r  r  )r   r  r	  r   r4  )r  r6  endpoint_route
route_infos       r   
get_routesr  ?"  s     
 F & 
D9%vt4"5)T:vt4 uj$7 #++	J MM*%& fs   %B AB )NNrf  r   )NNNNNNNN)NNNNFFNNr  NFTTNFFN(  r  r5  r  r  r   r&   r  r*  sysr  r  rE  warningsr   r   typingr   r   r   r   r	   r
   r   r   r   opentelemetry.tracer   _Spanr   filterwarningsUserWarningr    r  r3  r   insertrV  r~  backofffastapirs  rI  apscheduler.schedulers.asyncior!   r  r  r(   r,   collectionsr-   
contextlibr.   r   r/   r  r1   r2   r  r3   r4   litellm.exceptionsr5   1litellm.integrations.SlackAlerting.slack_alertingr6   'litellm.litellm_core_utils.core_helpersr7   r8   &litellm.llms.custom_httpx.http_handlerr9   r:   litellm.proxy._types5litellm.proxy.analytics_endpoints.analytics_endpointsr=   analytics_routerlitellm.proxy.auth.auth_checksr>   litellm.proxy.auth.auth_utilsr?   litellm.proxy.auth.handle_jwtr@   "litellm.proxy.auth.litellm_licenserA   litellm.proxy.auth.model_checksrB   rC   rD   $litellm.proxy.auth.user_api_key_authrE   rF   )litellm.proxy.batches_endpoints.endpointsbatches_routerlitellm.proxy.caching_routescaching_router)litellm.proxy.common_utils.admin_ui_utilsrG   )litellm.proxy.common_utils.callback_utilsrH   rI   rJ   &litellm.proxy.common_utils.debug_utilsrK   debugging_endpoints_router0litellm.proxy.common_utils.encrypt_decrypt_utilsrL   rM   -litellm.proxy.common_utils.http_parsing_utilsrN   rO   ,litellm.proxy.common_utils.load_config_utilsrP   rQ   0litellm.proxy.common_utils.openai_endpoint_utilsrR   &litellm.proxy.common_utils.proxy_staterS   (litellm.proxy.common_utils.swagger_utilsrT   -litellm.proxy.fine_tuning_endpoints.endpointsfine_tuning_routerrU   ,litellm.proxy.guardrails.guardrail_endpointsguardrails_router(litellm.proxy.guardrails.init_guardrailsrV   rW   litellm.proxy.health_checkrX   0litellm.proxy.health_endpoints._health_endpointshealth_router,litellm.proxy.hooks.model_max_budget_limiterrY   .litellm.proxy.hooks.prompt_injection_detectionrZ   )litellm.proxy.hooks.proxy_failure_handlerr[   -litellm.proxy.hooks.proxy_track_cost_callbackr\   $litellm.proxy.litellm_pre_call_utilsr]   >litellm.proxy.management_endpoints.budget_management_endpointsbudget_management_router5litellm.proxy.management_endpoints.customer_endpointscustomer_router:litellm.proxy.management_endpoints.internal_user_endpointsinternal_user_routerr^   ;litellm.proxy.management_endpoints.key_management_endpointsr_   r`   ra   key_management_router9litellm.proxy.management_endpoints.organization_endpointsorganization_router:litellm.proxy.management_endpoints.team_callback_endpointsteam_callback_router1litellm.proxy.management_endpoints.team_endpointsteam_routerrb   )litellm.proxy.management_endpoints.ui_ssorc   ui_sso_router+litellm.proxy.management_helpers.audit_logsrd   4litellm.proxy.openai_files_endpoints.files_endpointsopenai_files_routerre   >litellm.proxy.pass_through_endpoints.llm_passthrough_endpointsllm_passthrough_router;litellm.proxy.pass_through_endpoints.pass_through_endpointsrf   pass_through_router(litellm.proxy.rerank_endpoints.endpointsrerank_routerlitellm.proxy.route_llm_requestrg   7litellm.proxy.spend_tracking.spend_management_endpointsspend_management_router1litellm.proxy.spend_tracking.spend_tracking_utilsrh   7litellm.proxy.ui_crud_endpoints.proxy_setting_endpointsui_crud_endpoints_routerlitellm.proxy.utilsri   rj   rk   rl   rm   rn   ro   rp   rq   rr   rs   rt   ru   4litellm.proxy.vertex_ai_endpoints.langfuse_endpointslangfuse_router2litellm.proxy.vertex_ai_endpoints.vertex_endpointsvertex_routerrv   litellm.routerrw   rx   ry   rz   litellm.schedulerr{   r|   r}   *litellm.secret_managers.aws_secret_managerr~   "litellm.secret_managers.google_kmsr   litellm.secret_managers.mainr   r   r   r   )litellm.types.integrations.slack_alertingr   litellm.types.llms.anthropicr   r   r   r   litellm.types.llms.openair   litellm.types.routerr   r   r  r   r   litellm.types.utilsr   ModelMapInfor   r   r   litellm._versionr   r   suppress_debug_infor   r   r   r   r   r   r   r   r   r   r   r   r   r   fastapi.encodersr   fastapi.middleware.corsr   fastapi.openapi.utilsr   r  r   r   r   r   r   fastapi.routingr   fastapi.securityr   fastapi.security.api_keyr   fastapi.staticfilesr   litellm.proxy.enterpriseproxy
enterpriser  rb  r   r   r   rh  r   r   proxy_stater   rR  r   r   r  ui_link
ui_messagecustom_swagger_message_title_descriptionr   r   r  r   r  r:  r@  openapienumEnumrE  exception_handlerr  rT  originsrU  r  r  r]  ui_pathr  listdirr   r  rC  folder_namefolder_pathmakedirssrcdstrenamer)   r   
middlewarere  add_middlewarerj  r  r|  r}  rB  r  r  r  user_configr  r   local_loggingr  r   r   r   r   rk  log_filer  r   r   r   rH  r>  r   model_max_budget_limiterr  r4  r   r   r   r   r   r  r   r7  r8  rn  r   r   rr  Literalr   r   r   r  r  r   r  rt  ru  r   async_resultcelery_app_conn	celery_fnr   UserAPIKeyAuthr  r  r  r  r  r  r  SpendLogsPayloadr  r  r!  r-  r  r<  r   rk  r   r  r  r  r  r  r  r  r  r   r   r/  poston_exceptionexpor_  r  rj  r}  r  r  r(  r  r  r  r  r  r  ry  r  r  r  r  r  r  r  r  r  TokenCountRequestr  r  r  r  r  r  r  r*  rO  rY  r\  rc  rg  rp  ru  rw  ModelInfoDeleterz  r  r  r  r  r  r  InvitationClaimr  r	  InvitationModelInvitationNewr  r  InvitationUpdater  InvitationDeleter  
ConfigYAMLr#  ConfigFieldUpdater(  r*  r+  r  r;  ConfigFieldDeleter=  r  r{  r  r  r  include_routerr   r   r   <module>r$     s$      	 	    
     (
 
 
 1DD# #    	K 8
 $  rwww	U?	 8 $ *   H 9 3 K Q " : E 4 ; 
 O B ? 
 H W > D V P T < T M T L S 
 T I N S R M 9 R    W X  E D C >  H  A 4 = H : 9 6 ;( #       . - 2 2 - -    & % 1 1 1 1 + +11 299/4 #..0d 09B1: '#  l*2/0'0*-.U*V'?Hryy7@ -x}  187;-u;0551- d
#.wi7TU  T T
 N  4@<	/] 	 BII
VWmVnnrs}r~
 AAW@XX\]g\h	i  &> ~!7 ~! ~!B _
  . -0f  299_g&&0\ CKTYY  ~&G .  '& 
% 	''//"''//(";<Kggll;?GIIe[7>TIJBJJw' 
 W%(l*B''**84Q7K'',,w<KBKKd3 '',,w1C'',,{L9CBIIc3
  2FGWFXX[\	
 299%&.+;BJJ'(			,' 	, 
 	,   %%   

  '+ x} +#
HV #!% % $  4  
HSM  (,x% ,0DDJJ  B!      1 2 8J'    # 	   t 2 , */' /$' !$' !   lOS )K L S 4 '+ x} + )  	/3 (+, 3 ""#"!"&15$(8<#%:%: c]: sm	:
 }: sm: c]: 3-: E%*-.: D>: (0~: 4.: 
:zw :4 "
4 
>SH %)4))* SM& s
lwC=wc]w #w c]	w
 E?w tnwt
7>t tn' }3 

	#sn33,!/3,?C3,l6,!/6,?C6,r1,!/1,?C1,h!/?C *, $ Y
&|: |:@ (9 :;CUBV   W%678@R?S   )00A(B.%..b +,-
	  
 +,-
	  
 ,+,-
	  
 7+,-
	]$9:NoN	   LL16  (/0A(B	@
@
@
 C=@
 &	@
<@
F W->%?$@   '*;"<!=]O   '+,-
  
 2+,-
    (/0A(B	A
A
A
 C=A
 &	A
 A
H +,-!
	   +,-!
	   &+,-!
	   1+,-!
	    (/0A(B	vvv C=v &	v0vr +,-!
	   +,-!
	   )00A(Bhhh &hhV +,-
  
 +,-
   )00A(Bbbb &bbJ +,-
  
 +,-
   Cy(/0A(B	III I &	IIb < ;   ~{ 9:III  IP +,-
  
 +,-
   )00A(BTTT &TTn +,-
  
 +,-
   )00A(BUUU &UUp (+,-
  
 %+,-
   )00A(B	TTT T &	TTn +,-
  
 +,-
   )00A(BTTT &TTn +,-
  
 +,-
   )00A(B	SSS S &	SSl &+,-
  
 #+,-
   )00A(B	VVV V &	VVr &+,-
  
 #+,-
   )00A(B	RRR R &	RRj "+,-
  
 +,-
   )00A(B	``` ` &	``F +,-!
	   +,-!
	   )00A(Bkkk &kkb 
+	,+,-$   )00A(B	M
,M
M
 M
 &	M
M
t 
+,-%	  :!2 ::z $
+,-  

 


8%  :&&%&  &Rc]/=	& O
	+,-	   )00A(Bb
b
%b
b
L ,
	+,-	   )00A(Bj
"j
%j
j
Z  w
	+,-   )00A(B(7=== "\ %\ C=\ 
 D>\ \ ~ C
	+,-   )00A(B+/$("&	z
%z
#C=z
 !z
 h	z
z
z P
	+,-   )00A(B+6$("&!"i
%i
#C=i
 !i
 h	i

 c]i
 smi
i
X #A
	+,-   )00A(B+6$("&!"O%O#C=O !O h	O
 c]O smOOd I
	+,-   )00A(B+/$("&!"NH%NH#C=NH !NH h	NH
 c]NH smNHNHb& &$ &R 
	+,-  
 
	+,-   )00A(B&*A %A smA A H(,S	@H	. 
	+,-   )00A(B!%B"%B"#B"
B"L M
	+,-	  A
? A
A
H _
	+,-  4 X
+,-   )00A(Bd%ddP 
	+,-	    (/0A(B	k
k
k
 C=k
 &	k
k
\ 	.!1UK@' @ L@4   Q
 Q
Q
h 	 E:q# q ;qh 
#u=Ho H >HV /@ 0@D 
	+,-"   >EEV=W8G
8G,:8G8Gv 
	+,-"   =DDU<V**+9**Z 
	+,-"   )00A(B3
3%33l 
	+,-"   )00A(B+
+%++^ 
+,-	  G
Z G
G
V 
+,-	   )00A(BE
E%EEP 
+,-"   )00A(B44%44n 
+,-	   )00A(BA+,A%A 
*AAH 
+,-	   )00A(B?
?%??D 
+,-	  v
v
r 
+,-	  J ( !+,-  



 Cw'89:;  < IW->%?$@A B`   6    > "   = !   % &   = !   ) *   ? #   & '   = !   ( )   ' (   ;    = !   & '   ? #   * +   > "   # $   $ %   - .   + ,   & '   ' (   + ,WR  U
+A3.RS
TTUF  GN   	h  		s   >Aj6 Ak +Ak :B"Ak< Ak< $DAk< j6Akj;AkkAkkAkkAkkAk9k#Ak*k)Ak9k*Ak2k/Ak9k1Ak2k2Ak9k8Ak9k<AllAl