
    )#h                    z   d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlmZ d dlmZ d dlmZmZmZmZmZmZmZmZmZmZ d dlZd dlZd dlmZ d dlmZ d dlm Z  d dl!Z!d dl"Z!d dl#Z!d dl!m$Z$ d d	l%m&Z& d d
l'm(Z(m)Z)m*Z* d dl+m,Z, d dl-m.Z. d dl/m0Z0 d dl1m2Z3 d dl4m5Z5 d dl6m7Z7 d dl8m9Z9 d dl:m;Z; d dl<m=Z= d dl>m?Z? d dl@mAZA d dlBmCZC d dlDmEZEmFZF d dlGmHZH d dlImJZJ d dlKmLZLmMZMmNZNmOZOmPZP d dlQmRZRmSZSmTZT d dlUmVZW d dlXmYZYmZZZ d dl[m\Z\ d dl]m^Z^m_Z_ d d l`maZambZb d d!lcmdZdmeZemfZfmgZg d d"lhmiZimjZjmkZkmlZlmmZmmnZnmoZompZpmqZqmrZrmsZsmtZtmuZumvZvmwZwmxZxmyZymzZz d d#l{m|Z| d d$l}m~Z~ d d%l}mZ d d&l}mZ d d'lmZmZmZmZmZmZmZmZmZmZ d(d)lmZ er	d d*lmZ eZneZ G d+ d,ej$                        Z G d- d.      Zy)/    Ndefaultdict)	lru_cache)
TYPE_CHECKINGAnyCallableDictListLiteralOptionalTupleUnioncast)AsyncOpenAI)	BaseModel)overload)get_secret_str)verbose_router_logger)	DualCacheInMemoryCache
RedisCache)CustomLogger)run_async_function)!_get_parent_otel_span_from_kwargs)Logging)RouterBudgetLimiting)LeastBusyLoggingHandler)LowestCostLoggingHandler)LowestLatencyLoggingHandler)LowestTPMLoggingHandler)LowestTPMLoggingHandler_v2)simple_shuffle)get_deployments_for_tag)"_get_router_metadata_variable_namereplace_model_in_jsonl)InitalizeOpenAISDKClient)CooldownCache)DEFAULT_COOLDOWN_TIME_SECONDS_async_get_cooldown_deployments/_async_get_cooldown_deployments_with_debug_info_get_cooldown_deployments_set_cooldown_deployments)#_check_non_standard_fallback_formatget_fallback_model_grouprun_async_fallback)!get_num_retries_from_retry_policy)#async_raise_no_deployment_exceptionsend_llm_exception_alert)PromptCachingDeploymentCheck)0increment_deployment_failures_for_current_minute1increment_deployment_successes_for_current_minute)FlowItem	Scheduler)AllMessageValuesBatch
FileObject	FileTypes)#CONFIGURABLE_CLIENTSIDE_AUTH_PARAMSVALID_LITELLM_ENVIRONMENTSAlertingConfigAllowedFailsPolicyAssistantsTypedDictCustomRoutingStrategyBase
DeploymentDeploymentTypedDictLiteLLM_ParamsModelGroupInfoOptionalPreCallChecksRetryPolicyRouterCacheEnumRouterGeneralSettingsRouterModelGroupAliasItemRouterRateLimitErrorRouterRateLimitErrorBasicRoutingStrategy)ServiceTypes)GenericBudgetConfigType)	ModelInfo)StandardLoggingPayload)
CustomStreamWrapperEmbeddingResponseModelResponseRulesfunction_setupget_llm_provider!get_non_default_completion_params
get_secretget_utc_datetimeis_region_allowed   )PatternMatchRouter)Spanc                       e Zd ZdZy)RoutingArgs<   N)__name__
__module____qualname__ttl     K/var/www/html/sandstorm/venv/lib/python3.12/site-packages/litellm/router.pyr`   r`      s    
Crg   r`   c            S          e Zd ZU g Zeed<   dZee   ed<   dZ	e
ed<   dZdZee   ed<   dZee   ed<   dddddddi dddddddddddd	dg g g i ddd
di ddddddi dd e       f(deeee   eeeef      f      dee   dee   dee   dee
   dee   dee   dedeee      de
dee   dee
   dee
   dee
   dee   dee   dee   dee
   deded   d eee      d!ed"ed#ed$eeeeeef   f      d%ed&ed'e
d(eeeef      d)eeef   d*ee
   d+ee   d,ee   d-ee   d.ed/   d0ee   d1ed2ee   d3ee    d4ee   d5dfRd6Z!d7 Z"d8ee   fd9Z#d0ee   fd:Z$d.ee%ef   d1efd;Z&d<efd=Z'd>ed?eeeef      d5ee(e)f   fd@Z*d>ed?eeeef      d5ee(e)f   fdAZ+e,d>ed?ee-   dBedC   d5e)fdD       Z.e,	 dd>ed?ee-   dBed   d5e(fdE       Z.e,	 dd>ed?ee-   dBeedC   ed   f   d5ee)e(f   fdF       Z.	 dd>ed?ee-   dBefdGZ.d>ed?eeeef      d5ee(e)f   fdHZ/d>edIed5dfdJZ0dIed5dfdKZ1d<edIed5dfdLZ2d<edIefdMZ3dIedNed5eeee
f      fdOZ4dIedNed5eeee
f      fdPZ5dIedNed5eeee
f      fdQZ6dRee   d?eeeeef      eeeeef         f   fdSZ7d>ed?eee-      fdTZ8e,d>ed?eeeef      dBedC   d5e)fdU       Z9e,	 dd>ed?eeeef      dBed   d5e(fdV       Z9	 dd>ed?eeeef      dBefdWZ9e,	 dd>ed?ee-   dXe
dBed   d5e(f
dY       Z:e,d>ed?ee-   dXe
dBedC   d5e)f
dZ       Z:	 dd>ed?ee-   dXe
fd[Z:d>edXe
d\e;d]e<ed^f   dIeeef   f
d_Z=d>ed5efd`Z>d>ed?ee-   dIeeef   fdaZ?dbed>efdcZ@dbed>efddZAdbed>efdeZBdbed>efdfZCdgeDd>efdhZEdgeDd>efdiZFd>edjedkefdlZGd>efdmZHd>efdnZId>efdoZJ	 	 	 dd>edbedpee   dqee   dree   f
dsZK	 	 	 dd>edbedpee   dqee   dree   f
dtZLd>edbefduZM	 	 	 ddved>edpee   dqee   dree   f
dwZNdved>efdxZO	 dd>edjeeef   dree   d5ePfdyZQdjeeef   d>efdzZR	 dd>edjeeef   dree   d5ePfd{ZSdjeeef   d>efd|ZTd>ed5eUfd}ZVd>ed5eUfd~ZWd>ed5eXfdZYd>ed5eXfdZZd5eXfdZ[d>efdZ\d\e;fdZ]	 dd\e;ded   fdZ^	 	 dd\e;deed      dee_   fdZ`d Za	 	 	 	 ddIedee   d!ee   d"ee   d#ee   f
dZbd Zcd\efdZd	 ddIedee   fdZe	 	 	 	 	 ddefdee   dee   d"ee   d#ee   dee   fdZgd Zh	 dd!eeeee   f      dee   d5eee      fdZi	 	 ddefde
de
dee   dee   d5ee
ef   fdZjd Zkd5ee   fdZld5efdZmdee   fdZndIedefd5efdZodedeep   d5e
fdZqd5efdZrd>ede(dIed5efdZsd>edeep   fdZtd>edeep   d5e<ee   ee   f   fdZud<efdZv	 dd<edeep   deew   fdZx	 	 dd>edee   d?eee-      deep   dee   deew   fdZydedefdZzdedededed5ee{   f
dZ|d<e{d5efdZ}de~fdZd<e{d5e{fdZd<e{d5ee{   fdZd<e{d5ee{   fdZded5ee{   fdZded5ee{   fdZded5ee{   fdZe,	 dd<ededdd5efd       Ze,d<ddeded5efd       Z	 dd<ee   dedee   d5efdÄZded5ee   fdĄZded5ee   fdńZdeded5ee   fdǄZded5ee   fdȄZded5e<ee
   ee
   f   fdɄZ edʬ˫      ded5ee   fd̄       Zded5eee
f   fd̈́Z	 ddedee   d5efd΄Zddee   d5ee   fdЄZ	 ddedee   d5ee   fd҄Zd5ee   fdӄZ	 ddee   d5eee      fdԄZ	 ddee   dee   d5eeee   f   fdքZded5efdׄZd؄ Zdل ZddڄZ	 dd>eded?eeeef      dee   fdۄZd>ed5ee   fd܄Zd>ed5efd݄Z	 	 	 dd>ed?eeeeef         djeeeef      dee   d5e<eeeef   f   f
d߄Z	 	 	 	 dd>ed?eeeeef         djeeeef      dee   dee   f
dZ	 	 	 	 dd>ed?eeeeef         djeeeef      dee   dee   f
dZdee   dee   d5ee   fdZ	 ddeep   fdZ	 ddefdee   fdZdeffdZd ZdefdZd Zd Zy)Routermodel_namesFcache_responsesi  default_cache_time_secondsNleastbusy_loggerlowesttpm_loggerINFOr   simple-shuffle
model_listassistants_config	redis_url
redis_host
redis_portredis_passwordcache_kwargscaching_groups
client_ttlpolling_intervaldefault_prioritynum_retriesmax_fallbackstimeoutstream_timeoutdefault_litellm_paramsdefault_max_parallel_requestsset_verbosedebug_level)DEBUGrp   default_fallbacks	fallbackscontext_window_fallbackscontent_policy_fallbacksmodel_group_aliasenable_pre_call_checksenable_tag_filteringretry_afterretry_policymodel_group_retry_policyallowed_failsallowed_fails_policycooldown_timedisable_cooldownsrouting_strategy)rq   
least-busyusage-based-routinglatency-based-routingcost-based-routingusage-based-routing-v2optional_pre_call_checksrouting_strategy_argsprovider_budget_configalerting_configrouter_general_settingsreturnc)                 b   ddl m}) || _        || _        || _        || _        dt        _        | j                  du rQ|dk(  r$t        j                  t        j                         n(|dk(  r#t        j                  t        j                         |(xs
 t               | _        || _        g | _        i | _        d}*d}+i },|
| _        ||E|Cd}*|||,d	<   |||,d
<   |t'        |      |,d<   |||,d<   |,j)                  |       t+        d)i |,}+|r7t        j,                   t        j.                  d)d|*i|,t        _        || _        t3        |+t5                     | _        t7        ||+      | _        || _        d| _        || _        g | _         tC               | _"        |\tG        jH                  |      }| jK                  |       | jL                  | _'        |D ]  }-d|-d   v sd| j"                  |-d   d   <   ! ng | _&        ||| _(        nt        jP                  | _(        |!xs tR        | _*        tW        | j,                  | jT                        | _,        |"| _-        t5               | _.        ||| _/        n;t        j^                  t        j^                  | _/        nt`        jb                  | _/        ||| _2        n;t        jd                  t        jd                  | _2        nt        jf                  | _2        |xs t        jh                  | _5        || _6        || _7        |#| _8        |xs t        jr                  }.| ju                  |.       |.| _9        |t        jv                  H|xs t        jv                  }.| jr                  | jr                  jy                  d|.i       n
d|.ig| _9        |xs t        jz                  | _=        |xs t        j|                  }/| ju                  |/       |/| _>        t        t              | _A        t        t              | _B        t        t              | _C        g | _D        |xs i | _E        |xs i }t        j                  ||       | _G        || _H        | j                  j                  d|       | j                  j                  dd       | j                  j                  di       j)                  d|	i       i | _J        	 | j                  |#|%       d| _L        t        t        j                  t              r*t        j                  jy                  | j                         n)t        j                  jy                  | j                         t        t        j                  t              r*t        j                  jy                  | j                         n| j                  gt        _Q        t        t        j                  t              r*t        j                  jy                  | j                         n| j                  gt        _S        t        t        j                  t              r*t        j                  jy                  | j                         n| j                  gt        _U        t        j                  d| jp                   d| j                   d| jr                   d| j|                   d| jz                   d | j,                  j                   d!        |)       | _Y        |%| _Z        |&| _[        d| _\        t        j                  || j                  "      r|$|$jy                  d#       nd#g}$d| __        |vt        |t              rt        d)i || __        nt        |t              r|| __        t        j                  d$j                  | j                  j                  d%                   || _e        d| _f        | vt        | t              rt        d)i | | _f        nt        | t              r| | _f        t        j                  d&j                  | j                  j                  d%                   |'| _h        |$| j                  |$       | j                  | j                          | j                          | j                  t        j                  d'(      | _m        y)*a  
        Initialize the Router class with the given parameters for caching, reliability, and routing strategy.

        Args:
            model_list (Optional[list]): List of models to be used. Defaults to None.
            redis_url (Optional[str]): URL of the Redis server. Defaults to None.
            redis_host (Optional[str]): Hostname of the Redis server. Defaults to None.
            redis_port (Optional[int]): Port of the Redis server. Defaults to None.
            redis_password (Optional[str]): Password of the Redis server. Defaults to None.
            cache_responses (Optional[bool]): Flag to enable caching of responses. Defaults to False.
            cache_kwargs (dict): Additional kwargs to pass to RedisCache. Defaults to {}.
            caching_groups (Optional[List[tuple]]): List of model groups for caching across model groups. Defaults to None.
            client_ttl (int): Time-to-live for cached clients in seconds. Defaults to 3600.
            polling_interval: (Optional[float]): frequency of polling queue. Only for '.scheduler_acompletion()'. Default is 3ms.
            default_priority: (Optional[int]): the default priority for a request. Only for '.scheduler_acompletion()'. Default is None.
            num_retries (Optional[int]): Number of retries for failed requests. Defaults to 2.
            timeout (Optional[float]): Timeout for requests. Defaults to None.
            default_litellm_params (dict): Default parameters for Router.chat.completion.create. Defaults to {}.
            set_verbose (bool): Flag to set verbose mode. Defaults to False.
            debug_level (Literal["DEBUG", "INFO"]): Debug level for logging. Defaults to "INFO".
            fallbacks (List): List of fallback options. Defaults to [].
            context_window_fallbacks (List): List of context window fallback options. Defaults to [].
            enable_pre_call_checks (boolean): Filter out deployments which are outside context window limits for a given prompt
            model_group_alias (Optional[dict]): Alias for model groups. Defaults to {}.
            retry_after (int): Minimum time to wait before retrying a failed request. Defaults to 0.
            allowed_fails (Optional[int]): Number of allowed fails before adding to cooldown. Defaults to None.
            cooldown_time (float): Time to cooldown a deployment after failure in seconds. Defaults to 1.
            routing_strategy (Literal["simple-shuffle", "least-busy", "usage-based-routing", "latency-based-routing", "cost-based-routing"]): Routing strategy. Defaults to "simple-shuffle".
            routing_strategy_args (dict): Additional args for latency-based routing. Defaults to {}.
            alerting_config (AlertingConfig): Slack alerting configuration. Defaults to None.
            provider_budget_config (ProviderBudgetConfig): Provider budget configuration. Use this to set llm_provider budget limits. example $100/day to OpenAI, $100/day to Azure, etc. Defaults to None.
        Returns:
            Router: An instance of the litellm.Router class.

        Example Usage:
        ```python
        from litellm import Router
        model_list = [
        {
            "model_name": "azure-gpt-3.5-turbo", # model alias
            "litellm_params": { # params for litellm completion/embedding call
                "model": "azure/<your-deployment-name-1>",
                "api_key": <your-api-key>,
                "api_version": <your-api-version>,
                "api_base": <your-api-base>
            },
        },
        {
            "model_name": "azure-gpt-3.5-turbo", # model alias
            "litellm_params": { # params for litellm completion/embedding call
                "model": "azure/<your-deployment-name-2>",
                "api_key": <your-api-key>,
                "api_version": <your-api-version>,
                "api_base": <your-api-base>
            },
        },
        {
            "model_name": "openai-gpt-3.5-turbo", # model alias
            "litellm_params": { # params for litellm completion/embedding call
                "model": "gpt-3.5-turbo",
                "api_key": <your-api-key>,
            },
        ]

        router = Router(model_list=model_list, fallbacks=[{"azure-gpt-3.5-turbo": "openai-gpt-3.5-turbo"}])
        ```
        r   )ServiceLoggingTrp   r   localNredisurlhostportpasswordtype)redis_cachein_memory_cache)r{   r   modellitellm_params)cachedefault_cooldown_time)fallback_param*)params
router_objr   max_retriesmetadatary   r   r   z)Intialized router with Routing strategy: z"

Routing enable_pre_call_checks: z

Routing fallbacks: z

Routing content fallbacks: z$

Routing context window fallbacks: z

Router Redis Caching=
)rr   r   router_budget_limitingz+[32mRouter Custom Retry Policy Set:
{}[0mexclude_nonez3[32mRouter Custom Allowed Fails Policy Set:
{}[0m
moderation)	call_typerf   )nlitellm._service_loggerr   r   r   r   r   litellmsuppress_debug_infor   setLevelloggingrp   r   rI   r   rs   deployment_namesdeployment_latency_maprz   strupdater   r   Cacherl   r   r   r7   	schedulerr|   default_deploymentr   provider_default_deployment_idsr]   pattern_routercopydeepcopyset_model_listrr   healthy_deploymentsr   r(   r   r'   cooldown_cacher   failed_callsr}   openaiDEFAULT_MAX_RETRIESr~   ROUTER_MAX_FALLBACKSrequest_timeoutr   r   r   r   r   validate_fallbacksr   appendr   r   r   inttotal_calls
fail_callssuccess_callsprevious_modelsr   Chatchatr   
setdefaultdeployment_statsrouting_strategy_initaccess_groups
isinstance_async_success_callbacklistdeployment_callback_on_successsuccess_callback#sync_deployment_callback_on_success_async_failure_callback$async_deployment_callback_on_failurefailure_callbackdeployment_callback_on_failuredebugr   service_logger_objr   r   router_budget_loggerr   !should_init_router_budget_limiterr   dictrG   infoformat
model_dumpr   r   r?   r   add_optional_pre_call_checks_initialize_alertinginitialize_assistants_endpointfactory_functionamoderation)0selfrr   rs   rt   ru   rv   rw   rl   rx   ry   rz   r{   r|   r}   r~   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   
cache_typer   cache_configm
_fallbacks_content_policy_fallbackss0                                                   rh   __init__zRouter.__init__   s   d 	;&&&<#$8!&*#t#f$%..w||<'%..w}}=#>'<'> 	$ "3 	 ')#  	 ')$ Z%;
@V J$&/U#%'1V$%'*:V$)+9Z( -$4|4K}}$ ' N: N N#2D #]_


 #-;
 !1"&-J*:<,02!z2J
+-1__D$ Ra 011PQD//2B0CG0LMR
  O $!.D!(!6!6D*K.K+**D4F4F
 "3O 	 "*D  ,&22D%99D$!.D"".!(!6!6D!(!=!=D9'"9"9,& 0 3'"3"3
z:#(G,E,E,Q*Gg.G.GJ~~)%%sJ&78#&
"3!4 %H(H(H 	%
 %H(H(H 	" 	/HI(A%(3)
 (3(
 +6+
  	 # 	
 "8!=2LL(>4P	 '=###..y'B##..}a@##..z2>EE~.	
 ')	 	""-"7 	# 	
 "g55t<++2243V3VW++2243V3VWg..5$$++D,T,TU(,(P(P'QG$g55t<++2299
 99/G+ g..5$$++D,O,OP(,(K(K'LG$##78M8M7N O//3/J/J.K L""&..!1 2**.*G*G)H I1151N1N0O P$$(JJ$:$:#;2?	
 #1"2%:"&<#DH!AA!$:U:U
 (3(//0HI,D+E(37#,-$/$?,$?!L+6$0!!&&DKK%%00d0C % 	% CG!+.5,>,VAU,V)02DE,@)!&&LSS--88d8K :I#/--.FG+%%'++-00< 1 
rg   c                 D   | j                  t        j                        | _        | j                  t        j                        | _        | j                  t        j                        | _        | j                  t        j
                        | _        | j                  t        j                        | _        | j                  t        j                        | _        | j                  t        j                        | _        | j                  t        j                        | _	        y N)
r   r   acreate_assistantsadelete_assistantaget_assistantsacreate_threadaget_threada_add_messageaget_messagesarun_threadr   s    rh   r   z%Router.initialize_assistants_endpoint7  s    "&"7"78R8R"S!%!6!6w7P7P!Q#44W5L5LM"33G4J4JK001D1DE!2273H3HI!2273H3HI001D1DErg   r   c           	          |y|D ]J  }t        |t              st        d| d      t        |      dk7  s1t        d| dt        |       d       y)z3
        Validate the fallbacks parameter.
        NzItem 'z' is not a dictionary.r\   zDictionary 'z%' must have exactly one key, but has z keys.)r   r   
ValueErrorlen)r   r   fallback_dicts      rh   r   zRouter.validate_fallbacksB  sq     !+ 	MmT2 6-8N!OPP=!Q& "=/1VWZ[hWiVjjpq 		rg   c                     |y|D ]s  }d }|dk(  rt        | j                        }n1|dk(  r,t        | j                  | j                  | j                        }|Ut
        j                  j                  |       u y y )Nprompt_caching)r   r   )
dual_cacher   rr   )r3   r   r   r   rr   r   	callbacksr   )r   r   pre_call_check	_callbacks       rh   r   z#Router.add_optional_pre_call_checksQ  s     $/": 848	!%55 <4:: NI#'?? 4#'::/3/J/J#'??!I
 (%%,,Y78 0rg   c                 .   t        j                  d|        |t        j                  j                  k(  s|t        j                  k(  rt        | j                  | j                        | _        t        t        j                  t              r*t        j                  j                  | j                         n| j                  gt        _        t        t        j                  t              r*t        j                  j                  | j                         y y |t        j                  j                  k(  s|t        j                  k(  rpt!        | j                  | j                  |      | _        t        t        j                  t              r*t        j                  j                  | j"                         y y |t        j$                  j                  k(  s|t        j$                  k(  rpt'        | j                  | j                  |      | _        t        t        j                  t              r*t        j                  j                  | j(                         y y |t        j*                  j                  k(  s|t        j*                  k(  rpt-        | j                  | j                  |      | _        t        t        j                  t              r*t        j                  j                  | j.                         y y |t        j0                  j                  k(  s|t        j0                  k(  rpt3        | j                  | j                  i       | _        t        t        j                  t              r*t        j                  j                  | j4                         y y y )NzRouting strategy: )router_cacherr   )r  rr   routing_args)r   r   rM   
LEAST_BUSYvaluer   r   rr   rn   r   r   input_callbackr   r   r  USAGE_BASED_ROUTINGr    ro   USAGE_BASED_ROUTING_V2r!   lowesttpm_logger_v2LATENCY_BASEDr   lowestlatency_logger
COST_BASEDr   lowestcost_logger)r   r   r   s      rh   r   zRouter.routing_strategy_initb  s    	""%78H7I#JK : : @ @@?#=#==$;!ZZDOO%D! '00$7&&--d.C.CD*.*?*?)@&'++T2!!(()>)>? 3  C C I II?#F#FF$;!ZZ??2%D!
 '++T2!!(()>)>? 3  F F L LL?#I#II'A!ZZ??2(D$
 '++T2!!(()A)AB 3  = = C CC?#@#@@(C!ZZ??2)D%
 '++T2!!(()B)BC 3  : : @ @@?#=#==%=!ZZ??&D"
 '++T2!!(()?)?@ 3 rg   
deploymentc                     	 t        j                  |      }|d   }d|v r|d   dd dz   |d<   |S # t        $ r(}t        j                  dt        |              |d}~ww xY w)z
        returns a copy of the deployment with the api key masked

        Only returns 2 characters of the api key and masks the rest with * (10 *).
        r   api_keyN   z
**********z+Error occurred while printing deployment - )r   r   	Exceptionr   r   r   )r   r!  _deployment_copyr   es        rh   print_deploymentzRouter.print_deployment  s}    
	#}}Z8#34D#ENN*,:9,Ebq,IH,Ty)## 	!''=c!fXF G		s   -0 	A!#AA!r   messagesc                     	 t        j                  d| d       ||d<   ||d<   | j                  |d<   | j                  ||        | j                  di |}|S # t
        $ r}|d}~ww xY w)	z
        Example usage:
        response = router.completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hey, how's it going?"}]
        zrouter.completion(model=z,..)r   r)  original_functionr   kwargsNrf   )r   r   _completion_update_kwargs_before_fallbacksfunction_with_fallbacksr%  )r   r   r)  r-  responser'  s         rh   
completionzRouter.completion  s    
	!''*B5'(NO#F7O!)F:*.*:*:F&'00uV0L3t33=f=HO 	G	s   AA 	A+$A&&A+c           	         d }	 | j                  |||j                  dd             }| j                  ||       |d   j                         }|d   }| j	                  ||      }|j                  dd       }||||j                  k7  rd }	n|}	|| j                         vr| j                  |       t        j                  di i ||| j                  |	d|}
t        j                  d	| d
       t        |
t              r.| j!                  ||
|      }|rt        j"                  d|d      |
S # t$        $ r,}t        j                  d	| dt'        |       d       |d }~ww xY w)Nspecific_deploymentr   r)  r4  r!  r-  r   r   r#  r!  r)  cachingclientzlitellm.completion(model=)[32m 200 OK[0mr   r1  r-  Response output was blocked. messager   llm_provider)[31m Exception [0mrf   )get_available_deploymentpop_update_kwargs_with_deploymentr   _get_clientgetr#  get_model_ids routing_strategy_pre_call_checksr   r2  rl   r   r   r   rT   "_should_raise_content_policy_errorContentPolicyViolationErrorr%  r   )r   r   r)  r-  
model_namer!  datapotential_model_clientdynamic_api_keymodel_clientr1  _should_raiser'  s                rh   r.  zRouter._completion  s    
<	66!$*JJ/Dd$K 7 J
 //:f/U./446DgJ%)%5%5%f &6 &" %jjD9O+*6#'='E'EE#5 D..00555L))  (#33*	
 H "&&+J<7NO
 (M2 $ G G(6 !H ! !!== >#%'  O 	!&&+J<7KCPQF8SZ[ G		s   D1D6 6	E+?'E&&E+streamTc                    K   y wr   rf   r   r   r)  rS  r-  s        rh   acompletionzRouter.acompletion        	   c                    K   y wr   rf   rU  s        rh   rV  zRouter.acompletion  rW  rX  c                    K   y wr   rf   rU  s        rh   rV  zRouter.acompletion  rW  rX  c                 &  K   	 ||d<   ||d<   ||d<   | j                   |d<   | j                  ||       |j                  d      xs | j                  }t	        j                         }| j                  |      }|r| j                  |||       d {   S |+t        |t              r | j                  di | d {   }n | j                  di | d {   }t	        j                         }	|	|z
  }
t        j                  | j                  j                  t        j                   |
d||	t#        |      	             |S 7 7 7 n# t$        $ r;}t        j                  t'        | |t)        j*                         |
             |d }~ww xY ww)Nr   r)  rS  r+  r,  priority)r   r)  r-  rV  servicedurationr   
start_timeend_timeparent_otel_spanlitellm_router_instancerequest_kwargserror_traceback_stroriginal_exceptionrf   )_acompletionr/  rH  r|   time_is_prompt_management_model_prompt_management_factoryr   r   schedule_acompletionasync_function_with_fallbacksasynciocreate_taskr   async_service_success_hookrN   ROUTERr   r%  r2   	traceback
format_exc)r   r   r)  rS  r-  request_priorityr`  rj  r1  ra  	_durationr'  s               rh   rV  zRouter.acompletion#  s    ,	#F7O!)F:%F8*.*;*;F&'00uV0L%zz*5N9N9NJ*.*J*J5*Q'*!<<%! =   
  +
;KS0Q!:!:!:!DV!DD!C!C!C!Mf!MMyy{H :-I''BB(//&+)%%Fv%N C 	 O- EM  		(,0#)(1(<(<(>'(	 G		sl   FBE
 EE
 F'E
 >E?E
 EA)E
 FE
 E
 E
 
	F6F		FFc                   K   d}	 t        j                  d| d|        t        |      }t        j                         }| j	                  |||j                  dd      |       d{   }t        j                         }||z
  }	t        j                  | j                  j                  t        j                  |	d||t        |                   | j                  ||       | j                  ||	       |d
   j                         }
|
d   }| j                  ||	      }| j                   |xx   dz  cc<   t#        j$                  di i |
|| j&                  |d|}|j)                  dd      }| j+                  ||d      }|]t-        |t        j.                        rC|4 d{    	 | j1                  |||       d{    | d{   }ddd      d{    n&| j1                  |||       d{    | d{   }t-        t2              r.| j5                  |||      }|rt#        j6                  d|d      | j8                  |xx   dz  cc<   t        j:                  d| d       | j                  |||       |S 7 )7 7 7 7 # 1 d{  7  sw Y   xY w7 7 # t<        $ rE}t        j:                  d| dt?        |       d       || j@                  |xx   dz  cc<   |d}~ww xY ww)z
        - Get an available deployment
        - call it with a semaphore over the call
        - semaphore specific to it's rpm
        - in the semaphore,  make a check against it's local rpm before running
        NzInside _acompletion()- model: 
; kwargs: r4  )r   r)  r4  re  async_get_available_deploymentr]  r!  rb  r6  r   r   r\   r8  litellm_logging_objmax_parallel_requestsr!  r-  client_type)r!  logging_objrb  r<  r=  r>  r?  zlitellm.acompletion(model=r;  )r!  r1  rb  rB  rC  rf   )!r   r   r   ri  rx  rE  rn  ro  r   rp  rN   rq  _track_deployment_metricsrF  r   _get_async_openai_model_clientr   r   rV  rl   rH  rG  r   	Semaphore&async_routing_strategy_pre_call_checksrT   rK  rL  r   r   r%  r   r   )r   r   r)  r-  rM  rb  r`  r!  ra  ru  rN  rQ  	_responser~  rpm_semaphorer1  rR  r'  s                     rh   rh  zRouter._acompletionX  s     
q	!''0z&J  AHJ#BB!$*JJ/Dd$K%	  C   J yy{H :-I''BB(//&>)%%Fv%N C 	 **%8H +  //:f/U./446DgJ>>% ? L Z(A-(++  (#33*	
 I 5;JJ%t5K !,,%3 - M
 (Zw00. ) 
/ 
/ EE#-$/)9 F   
 &/H
/ 
/ 
/ AA) +%5 B    "+? (M2 $ G G(6 !H ! !!== >#%'  z*a/*!&&,ZL8OP **%!!1 +  OIr
/

  /
/ 
/ 
/ 
/ +4  	!&&,ZL8LSQRVHT[\ %
+q0+G	s   LA"J5 (J)EJ5 *J+J5 .JJ	JJJJ5 !J"J5 >J1?	J5 J3	BJ5 LJ5 J5 JJJ5 J."J%#J.*J5 3J5 5	L>A K>>LLr-  c                     |j                  d| j                        |d<   |j                  dt        t	        j
                                      |j                  di       j                  d|i       y)zm
        Adds/updates to kwargs:
        - num_retries
        - litellm_trace_id
        - metadata
        r}   litellm_trace_idr   model_groupN)rH  r}   r   r   uuiduuid4r   )r   r   r-  s      rh   r/  z&Router._update_kwargs_before_fallbacks  s[     !'

=$:J:J K},c$**,.?@*b)00-1GHrg   c                     | j                   j                         D ]+  \  }}||vr||||<   |dk(  s||   j                  |       - y)z@
        Adds default litellm params to kwargs, if set.
        Nr   )r   itemsr   )r   r-  kvs       rh   *_update_kwargs_with_default_litellm_paramsz1Router._update_kwargs_with_default_litellm_params  sS     //557 	$DAqAMq	jq	  #	$rg   c                 8   |j                  di       j                  |d   d   |j                  di       |j                  di       j                  d      d       |j                  di       |d<   | j                  ||d         |d<   | j	                  |	       y
)z
        2 jobs:
        - Adds selected deployment, model_info and api_base to kwargs["metadata"] (used for logging)
        - Adds default litellm params to kwargs, if set.
        r   r   r   
model_infoapi_baser!  r  r  r-  rN  r   r-  N)r   r   rH  _get_timeoutr  )r   r!  r-  s      rh   rF  z%Router._update_kwargs_with_deployment  s     	*b)00()9:7C(nn\2>&NN+;R@DDZP	
  *~~lB?| --
+; < . 
y 	77v7Frg   c                     | j                  ||d      }|j                  dd      }||||j                  k7  rd}|S |}|S )a  
        Helper to get AsyncOpenAI or AsyncAzureOpenAI client that was created for the deployment

        The same OpenAI client is re-used to optimize latency / performance in production

        If dynamic api key is provided:
            Do not re-use the client. Pass model_client=None. The OpenAI/ AzureOpenAI client will be recreated in the handler for the llm provider
        asyncr|  r#  N)rG  rH  r#  )r   r!  r-  rO  rP  rQ  s         rh   r  z%Router._get_async_openai_model_client   sh     "&!1!1!&g "2 "

 !**Y5'&2#9#A#AAL  2Lrg   rN  c                     |j                  dd      xs> |j                  dd      xs* | j                  xs | j                  j                  dd      S )z=Helper to get stream timeout from kwargs or deployment paramsr   N)rH  r   r   )r   r-  rN  s      rh   _get_stream_timeoutzRouter._get_stream_timeout  s_    
 JJ'. Gxx $G ""	G
 **../?F	
rg   c                     |j                  dd      xsf |j                  dd      xsR |j                  dd      xs> |j                  dd      xs* | j                  xs | j                  j                  dd      }|S )zAHelper to get non-stream timeout from kwargs or deployment paramsr   Nr   )rH  r   r   r   r-  rN  r   s       rh   _get_non_stream_timeoutzRouter._get_non_stream_timeout'  s    
 JJy$' 	@zz+T2	@xx4	@
 xx!4	@ ||	@ **..y$? 	 rg   c                 ~    d}|j                  dd      r| j                  ||      }|| j                  ||      }|S )z6Helper to get timeout from kwargs or deployment paramsNrS  Fr  )rH  r  r  r  s       rh   r  zRouter._get_timeout9  sP    /3::h&..f4.HG?22D 3 G rg   modelsc                    K   dt         dt        t           f fd}dt         dt        t           dt        f fd}t	        |t
              rQt        d |D              r?g }|D ]  }|j                   |d||d|        t        j                  |  d{   }|S t	        |t
              rt        d	 |D              rg }t        |      D ]'  \  }	}
|D ]  }|j                   |d||	|
d
|        ) t        j                  |  d{   }t        t        |            D cg c]  }g  }}|D ]A  }t	        |t              r||d      j                  |d          .|d   j                  |       C |S yy7 7 qc c}w w)a6  
        Async Batch Completion. Used for 2 scenarios:
        1. Batch Process 1 request to N models on litellm.Router. Pass messages as List[Dict[str, str]] to use this
        2. Batch Process N requests to M models on litellm.Router. Pass messages as List[List[Dict[str, str]]] to use this

        Example Request for 1 request to N models:
        ```
            response = await router.abatch_completion(
                models=["gpt-3.5-turbo", "groq-llama"],
                messages=[
                    {"role": "user", "content": "is litellm becoming a better product ?"}
                ],
                max_tokens=15,
            )
        ```


        Example Request for N requests to M models:
        ```
            response = await router.abatch_completion(
                models=["gpt-3.5-turbo", "groq-llama"],
                messages=[
                    [{"role": "user", "content": "is litellm becoming a better product ?"}],
                    [{"role": "user", "content": "who is this"}],
                ],
            )
        ```
        r   r)  c                 |   K   	  j                   d| |d| d{   S 7 # t        $ r}|cY d}~S d}~ww xY wwzs
            Wrapper around self.async_completion that catches exceptions and returns them as a result
            r   r)  Nrf   rV  r%  r   r)  r-  r'  r   s       rh   _async_completion_no_exceptionszARouter.abatch_completion.<locals>._async_completion_no_exceptionsh  D     -T--WEHWPVWWWW 0   <$ "$ <$ 	949<9<idxc                    K   	  j                   d| |d| d{   |fS 7 # t        $ r}||fcY d}~S d}~ww xY wwr  r  )r   r)  r  r-  r'  r   s        rh   *_async_completion_no_exceptions_return_idxzLRouter.abatch_completion.<locals>._async_completion_no_exceptions_return_idxs  sS     *$**TTVTT T  #vs4   A & $& A & 	=8=A =A c              3   <   K   | ]  }t        |t                y wr   )r   r   .0r   s     rh   	<genexpr>z+Router.abatch_completion.<locals>.<genexpr>  s     -TajD.A-T   r  Nc              3   <   K   | ]  }t        |t                y wr   )r   r   r  s     rh   r  z+Router.abatch_completion.<locals>.<genexpr>  s     /V
1d0C/Vr  )r   r  r)  r\   r   rf   )r   r
   r8   r   r   r   allr   rn  gather	enumerateranger  tuple)r   r  r)  r-  r  r  _tasksr   r1  r  r@  	responses_final_responsess   `             rh   abatch_completionzRouter.abatch_completionD  s    H				"&'7"8				+,	 	& h%#-T8-T*TF i=gET\g`fghi %^^V44HO$'C/VX/V,VF )( 3 W# EMMB "'S7FL &nnf55I<A#h-<P/Qq/QO/Q% 8h.#HQK077D#A&--h7	8
 #"# -W' 5 6/Qs8   BFE;A4FE=F&	E?/AF=F?Fc           	          K   dt         dt        t           f fd}g }|D ]  }|j                   |d||d|        t	        j
                  |  d{   }|S 7 w)a  
        Async Batch Completion - Batch Process multiple Messages to one model_group on litellm.Router

        Use this for sending multiple requests to 1 model

        Args:
            model (List[str]): model group
            messages (List[List[Dict[str, str]]]): list of messages. Each element in the list is one request
            **kwargs: additional kwargs
        Usage:
            response = await self.abatch_completion_one_model_multiple_requests(
                model="gpt-3.5-turbo",
                messages=[
                    [{"role": "user", "content": "hello"}, {"role": "user", "content": "tell me something funny"}],
                    [{"role": "user", "content": "hello good mornign"}],
                ]
            )
        r   r)  c                 |   K   	  j                   d| |d| d{   S 7 # t        $ r}|cY d}~S d}~ww xY wwr  r  r  s       rh   r  z]Router.abatch_completion_one_model_multiple_requests.<locals>._async_completion_no_exceptions  r  r  r  Nrf   )r   r
   r8   r   rn  r  )r   r   r)  r-  r  r  message_requestr1  s   `       rh   -abatch_completion_one_model_multiple_requestsz4Router.abatch_completion_one_model_multiple_requests  s{     ,				"&'7"8		 ' 	OMM/ /=C	 !00 1s   AAAAc                    K   y wr   rf   rU  s        rh   "abatch_completion_fastest_responsez)Router.abatch_completion_fastest_response  rW  rX  c                    K   y wr   rf   rU  s        rh   r  z)Router.abatch_completion_fastest_response  rW  rX  c                    K   |j                  d      D cg c]  }|j                          }}dt        dt        t        t        t        f      dt
        dt        dt        t        t        t        f   f
 fd}g dt        j                  ffd	}|D ]2  }t        j                   |d|||d
|      }	j                  |	       4 r_t        j                  t        j                          d{   \  }
|
D ]'  } ||       d{   }|d|j"                  d<   |c S  r_t        d      c c}w 7 F7 0w)z
        model - List of comma-separated model names. E.g. model="gpt-4, gpt-3.5-turbo"

        Returns fastest response from list of model names. OpenAI-compatible endpoint.
        ,r   r)  rS  r-  r   c                    K   	  j                   d| ||d| d{   S 7 # t        j                  $ r& t        j                  dj                  |               t        $ r}|cY d}~S d}~ww xY ww)zn
            Wrapper around self.acompletion that catches exceptions and returns them as a result
            r   r)  rS  Nz4Received 'task.cancel'. Cancelling call w/ model={}.rf   )rV  rn  CancelledErrorr   r   r   r%  )r   r)  rS  r-  r'  r   s        rh   r  zRRouter.abatch_completion_fastest_response.<locals>._async_completion_no_exceptions  sw     -T--fEHU[f_effff)) %++JQQRWX  s>   A4% #% A4% A A1%A,&A1'A4,A11A4taskc                   K   	 |  d {   }t        |t        t        f      r@t        j                  d       D ]  }|j                           |	 j                  |        S 	 	 j                  |        y 7 n# t        $ r Y S w xY w# t        $ r Y /w xY w# t        $ r Y y w xY w# 	 j                  |        w # t        $ r Y w w xY wxY ww)Nz=Received successful response. Cancelling other LLM API calls.)	r   rT   rR   r   r   cancelremoveKeyErrorr%  )r  resulttpending_taskss      rh   check_responsezARouter.abatch_completion_fastest_response.<locals>.check_response  s     #f}6I&JK)//W + #
#!!((. L!((. $         !((. s   CB
 A9AB
 A;$C%B( 'B 8C9B
 ;	BCBC
	BB( BB( 	B%"C$B%%C(C*B<;C<	CCCCCr  )return_whenNT!fastest_response_batch_completionzAll tasks failedrf   )splitstripr   r
   r	   boolr   r   rT   rR   r%  rn  Taskro  r   waitFIRST_COMPLETED_hidden_params)r   r   r)  rS  r-  r   r  r  r  r  donecompleted_taskr  r  s   `            @rh   r  z)Router.abatch_completion_fastest_response  s]     &+[[%56!'')66		"&tCH~"6	@D	PS	="5y@A	  	w|| 	,  	'E&&/ (6EKD
   &	' (/7+B+B) #D- #' "-n==%QUF))*MN!M"	  *++ 7h# >s:   D?D6CD?4D;5D?D=D?D?+D?=D?r\  c                    K   y wr   rf   r   r   r)  r\  rS  r-  s         rh   rl  zRouter.schedule_acompletion0  rW  rX  c                    K   y wr   rf   r  s         rh   rl  zRouter.schedule_acompletion6  rW  rX  c                   K   t        |      }t        t        j                               }t	        ||d      }| j
                  j                  |       d {    t        j                         | j                  z   }	t        j                         }
| j
                  j                  }d}|
|	k  r| j                  ||       d {   \  }}| j
                  j                  |j                  |j                  |       d {   }|rn7t        j                  |       d {    t        j                         }
|
|	k  r|r]	  | j                   d|||d| d {   }|j"                  j%                  di        |j"                  d   j'                  d	d
i       |S t-        j.                  d|d      7 `7 7 7 7 c# t(        $ r}t+        |d|       |d }~ww xY ww)Nzgpt-3.5-turbor\  
request_idrM  requestFr   rb  idrM  health_deploymentsr  additional_headers%x-litellm-request-prioritization-usedTr\  %Request timed out while polling queuer   r?  rf   )r   r   r  r  r6   r   add_requestri  r   r{   _async_get_healthy_deploymentspollr  rM  rn  sleeprV  r  r   r   r%  setattrr   Timeout)r   r   r)  r\  rS  r-  rb  _request_iditemra  	curr_timepoll_intervalmake_request_healthy_deploymentsr  r  r'  s                    rh   rl  zRouter.schedule_acompletion>  s     =VD$**,'"&
 nn(((666 99;-IIK	77(",0,O,O.> -P - '# ! "&!4!4????#7 "5 " L
 mmM222 IIK	 (" "2$"2"2 #(6#EK# 	 ((334H"M(()=>EE<dC ! 
 //?% K 	7' 3
  :x0s   AG#F8A,G#F;<G#F=G#"F?#G#G#G GAG G#;G#=G#?G#G 	G GG  G#r+  args.c                   K   t        |      }t        t        j                               }t	        |||      }| j
                  j                  |       d {    t        j                         | j                  z   }	t        j                         }
| j
                  j                  }d}|
|	k  r| j                  ||       d {   \  }}| j
                  j                  |j                  |j                  |       d {   }|rn7t        j                  |       d {    t        j                         }
|
|	k  r|ri	  ||i | d {   }t!        |j"                  t$              r<|j"                  j'                  di        |j"                  d   j)                  ddi       |S t/        j0                  d
|d      7 l7 7 7 7 ~# t*        $ r}t-        |d	|       |d }~ww xY ww)Nr  r  Fr  r  r  r  Tr\  r  r   r?  )r   r   r  r  r6   r   r  ri  r   r{   r  r  r  rM  rn  r  r   r  r   r   r   r%  r  r   r  )r   r   r\  r+  r  r-  rb  r  r  ra  r  r  r  r  r  r  r'  s                    rh   _schedule_factoryzRouter._schedule_factory|  s     =VD$**,'"
 nn(((666 99;-IIK	77(",0,O,O.> -P - '# ! "&!4!4????#7 "5 " L
 mmM222 IIK	 (" 
"3T"DV"DD	i66=,,778LbQ,,-ABII@$G ! 
 //?% I 	7' 3
 E  :x0s   AG0GA,G0G<G0G
G0"G#G0G0G GAG +G0G0
G0G0G 	G-G((G--G0c                     | j                  |      }|yt        |      dk7  ry|d   d   j                  dd       }|yd|v r'|j                  d      d   }|t        j
                  v ryy)	NrM  Fr\   r   r   r   /T)get_model_listr  rH  r  r   )_known_custom_logger_compatible_callbacks)r   r   rr   litellm_modelsplit_litellm_models        rh   rj  z"Router._is_prompt_management_model  s    ((E(:
z?a"1&67;;GTJ -"/"5"5c":1"="g&W&WWrg   c                   K   |j                  dd       }|$t        di dt               t               d|\  }}t	        t
        |      }| j                  |dddg|j                  dd             }|d	   j                  d
d       }|j                  d      xs |d	   j                  dd       }|j                  d      xs |d	   j                  dd       }|t        |t              st        d| dt        |             |*t        |t              st        d| dt        |             |j                  ||t        |      ||      \  }}}	i ||	}||d
<   ||d<   ||d<   ||d<   ||d<   | j                  |      }
|
t!        |
      dk(  r.|j                  d       t#        j$                  di | d {   S  | j&                  di | d {   S 7 7 w)Nrz  rV  )r+  	rules_objr`  userpromptrolecontentr4  r5  r   r   	prompt_idprompt_variablesz*Prompt ID is not set or not a string. Got=z, type=z2Prompt variables is set but not a dictionary. Got=r  )r   r)  non_default_paramsr  r  r)  r  r   r+  rf   )rH  rV   rU   rZ   r   LiteLLMLoggingrD  rE  r   r   r  r   r   get_chat_completion_promptrX   r  r  r   rV  rm  )r   r   r)  r-  litellm_logging_objectprompt_management_deploymentr  r  r  optional_params_model_lists              rh   rk  z!Router._prompt_management_factory  sg     "(,A4!H!)-; .)6!&"2"4 	.*"F "&n6L!M'+'D'D%(;< &

+@$ G (E (
$ 55EFJJT
 JJ{+ !/K0

#k4
  	 "::
 
)*:;??
 	 Jy#$><YKwtT]N_`  '
;KT0RDEUDVV]^bcs^t]uv 
 #==#!#DF#S#!1 >  	)x /F.o.w%z(>$%'{%5!"))U);#k"2a"7JJ*+ ,,6v6667T77A&AAA 7As$   F<G!>G?G!GG!G!r  c                    	 ||d<   ||d<   | j                   |d<   |j                  d| j                        |d<   |j                  di       j	                  d|i        | j
                  di |}|S # t        $ r}|d }~ww xY w)Nr   r  r+  r}   r   r  rf   )_image_generationrH  r}   r   r   r0  r%  r   r  r   r-  r1  r'  s         rh   image_generationzRouter.image_generation  s    
	#F7O%F8*.*@*@F&'$*JJ}d>N>N$OF=!j"-44mU5KL3t33=f=HO 	G	   A.A1 1	B:A<<Bc           	         d}	 t        j                  d| d|        | j                  |dddg|j                  dd             }| j	                  ||	       |d
   j                         }| j                  ||	      }| j                  |xx   dz  cc<   | j                  |       t        j                  di i ||| j                  |d|}| j                  |xx   dz  cc<   t        j                  d| d       |S # t        $ rE}	t        j                  d| dt        |	       d       || j                   |xx   dz  cc<   |	d }	~	ww xY w)Nr>  #Inside _image_generation()- model: rw  r  r  r  r4  r5  r6  r   r\   r7  r  r9  r:  zlitellm.image_generation(model=r;  rB  rC  rf   )r   r   rD  rE  rF  r   r  r   rJ  r   r  rl   r   r   r%  r   r   )
r   r  r   r-  rM  r!  rN  rQ  r1  r'  s
             rh   r  zRouter._image_generation  s   
*	!''5eWJvhO 66#)h?@$*JJ/Dd$K 7 J
 //:f/U./446D>>% ? L
 Z(A-( 11Z1H// $#33*	
 H z*a/*!&&1*=TU O 	!&&1*=QRUVWRXQYY`a %
+q0+G	s   C?D 	EA EEc           	      l  K   	 ||d<   ||d<   | j                   |d<   |j                  d| j                        |d<   | j                  ||        | j                  di | d {   }|S 7 # t
        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY ww)Nr   r  r+  r}   r,  rc  rf   )_aimage_generationrH  r}   r/  rm  r%  rn  ro  r2   rr  rs  r  s         rh   aimage_generationzRouter.aimage_generationJ  s     	#F7O%F8*.*A*AF&'$*JJ}d>N>N$OF=!00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   B4A A- $A+%A- *B4+A- -	B166B,,B11B4c           	      4  K   |}	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd              d {   }| j                  ||       |d	   j                         }|d
   }| j                  ||      }| j                  |xx   dz  cc<   t        j                  di i ||| j                  |d|}	| j                  ||d      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   dz  cc<   t        j$                  d| d       |	S 7 L7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   dz  cc<   |d }~ww xY ww)Nr  rw  r  r  r  r4  r5  r6  r   r   r\   r  r{  r|  ry  z litellm.aimage_generation(model=r;  rB  rC  rf   )r   r   r   rx  rE  rF  r   r  r   r   r  rl   rG  r   rn  r  r  r   r   r%  r   r   )r   r  r   r-  rM  rb  r!  rN  rQ  r1  r  r'  s               rh   r  zRouter._aimage_generation_  s    
C	!''5eWJvhO  AH#BB#)h?@$*JJ/Dd$K  C   J
 //:f/U./446DgJ>>% ? L
 Z(A-(00 $#33*	
 H !,,%3 - M (Zw00. ) . . EE#-@P F    &.~H. . . AA)<L B    "*>z*a/*!&&2:,>UV OoH.
  .. . . . *  	!&&2:,>RSVWXSYRZZab %
+q0+G	s   HAG F#B8G F&G F.,F(-	F.6F*7F.;G F,G "G#	G ,G-5G "H#G &G (F.*F.,G .G 4F75G <G G 	HA HHHfilec           	      .  K   	 ||d<   ||d<   | j                   |d<   | j                  ||        | j                  di | d{   }|S 7 # t        $ r;}t	        j
                  t        | |t        j                         |             |d}~ww xY ww)a  
        Example Usage:

        ```
        from litellm import Router
        client = Router(model_list = [
            {
                "model_name": "whisper",
                "litellm_params": {
                    "model": "whisper-1",
                },
            },
        ])

        audio_file = open("speech.mp3", "rb")
        transcript = await client.atranscription(
        model="whisper",
        file=audio_file
        )

        ```
        r   r  r+  r,  Nrc  rf   )	_atranscriptionr/  rm  r%  rn  ro  r2   rr  rs  )r   r  r   r-  r1  r'  s         rh   atranscriptionzRouter.atranscription  s     .	#F7O!F6N*.*>*>F&'00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		;   BAA AA BA 	B6BBBc           	      *  K   |}	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd              d {   }| j                  ||       |d	   j                         }| j                  ||      }| j                  |xx   d
z  cc<   t        j                  di i ||| j                  |d|}	| j                  ||d      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   d
z  cc<   t        j$                  d| d       |	S 7 G7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   d
z  cc<   |d }~ww xY ww)Nz!Inside _atranscription()- model: rw  r  r  r  r4  r5  r6  r   r\   )r  r9  r:  r{  r|  ry  zlitellm.atranscription(model=r;  rB  rC  rf   )r   r   r   rx  rE  rF  r   r  r   r   r  rl   rG  r   rn  r  r  r   r   r%  r   r   )r   r  r   r-  rM  rb  r!  rN  rQ  r1  r  r'  s               rh   r  zRouter._atranscription  s    
A	!''3E7*VHM  AH#BB#)h?@$*JJ/Dd$K  C   J //:f/U./446D>>% ? L
 Z(A-(--  #33*	
 H !,,%3 - M (Zw00. ) . . EE#-@P F    &.~H. . . AA)<L B    "*>z*a/*!&&/
|;RS OkD.
  .. . . . *  	!&&/
|;OPSTUPVxW^_ %
+q0+G	s   HAG FB3G F!G F)'F#(	F)1F%2F)6G F'G F>	G 'G (5G HG !G #F)%F)'G )F;/F20F;7G  G 	HA HHHinputvoicec           	        K   	 ||d<   ||d<   | j                  |dddg|j                  dd             d{   }| j                  ||	       |d
   j                         }|d    | j                  j                         D ])  \  }}||vr|||<   |dk(  s||   j                  |       + | j                  ||d      }	|j                  dd      }
|
|	|
|	j                  k7  rd}n|	}t        j                  di i |d|i| d{   }|S 7 7 # t        $ r;}t        j                  t        | |t!        j"                         |             |d}~ww xY ww)a  
        Example Usage:

        ```
        from litellm import Router
        client = Router(model_list = [
            {
                "model_name": "tts",
                "litellm_params": {
                    "model": "tts-1",
                },
            },
        ])

        async with client.aspeech(
            model="tts",
            voice="alloy",
            input="the quick brown fox jumped over the lazy dogs",
            api_base=None,
            api_key=None,
            organization=None,
            project=None,
            max_retries=1,
            timeout=600,
            client=None,
            optional_params={},
        ) as response:
            response.stream_to_file(speech_file_path)

        ```
        r"  r#  r  r  r  r4  Nr5  r,  r   r   r   r  r|  r#  r:  rc  rf   )rx  rE  r/  r   r   r  r   rG  rH  r#  r   aspeechr%  rn  ro  r2   rr  rs  )r   r   r"  r#  r-  r!  rN  r  r  rO  rP  rQ  r1  r'  s                 rh   r%  zRouter.aspeech  s    @3	#F7O#F7O#BB#)h?@$*JJ/Dd$K  C   J
 00uV0L./446DM3399; (1VO !F1I*_1I$$Q'( &*%5%5%f' &6 &" %jjD9O+*6#'='E'EE#5$__ l  H OK<  		(,0#)(1(<(<(>'(	 G		sR   E5D DAD A4D DD ED D 	E!6EEEc           	      6  K   	 ||d<   t         |d<   | j                  |d<   | j                  ||        | j                  di | d {   }|S 7 # t        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY wwNr   r"  r+  r,  rc  rf   )
r"  _arerankr/  rm  r%  rn  ro  r2   rr  rs  r   r   r-  r1  r'  s        rh   arerankzRouter.arerankj  s     	#F7O#F7O*.--F&'00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   BAA 	A
A BA 	B6BBBc           	        K   d }	 t        j                  d| d|        | j                  ||j                  dd              d {   }| j	                  ||       |d   j                         }|d   }| j                  ||      }| j                  |xx   dz  cc<   t        j                  di i || j                  |d	| d {   }| j                  |xx   dz  cc<   t        j                  d
| d       |S 7 7 8# t        $ rE}t        j                  d
| dt        |       d       || j                  |xx   dz  cc<   |d }~ww xY ww)NzInside _rerank()- model: rw  r4  )r   r4  r6  r   r   r\   )r9  r:  zlitellm.arerank(model=r;  rB  rC  rf   )r   r   rx  rE  rF  r   r  r   r   r*  rl   r   r   r%  r   r   )	r   r   r-  rM  r!  rN  rQ  r1  r'  s	            rh   r(  zRouter._arerank  s    
&	!''+E7*VHE  $BB$*JJ/Dd$K  C   J //:f/U./446DgJ>>% ? L Z(A-($__ #33* 	 H z*a/*!&&(4KL O7  	!&&(4HQPWX %
+q0+G	sO   EAD DBD D5D ED D 	EA EEEc                   K   dddg}	 |j                  d| j                        |d<   | j                  ||       | j                  |||j	                  dd              d {   }|d   j                         }| j                  j                         D ])  \  }}||vr|||<   |d	k(  s||   j                  |       + t        j                  di i |d
| j                  i| d {   S 7 7 # t        $ rO}| j                  dkD  r9||d<   ||d<   | j                  |d<    | j                  di | d {  7  cY d }~S |d }~ww xY ww)Nr  z
dummy-textr  r}   r,  r4  r5  r   r   r9  r   r   r)  r+  rf   )rH  r}   r/  rx  rE  r   r   r  r   r   
_arealtimerl   r%  async_function_with_retries)	r   r   r-  r)  r!  rN  r  r  r'  s	            rh   r-  zRouter._arealtime  sw    #=>	$*JJ}d>N>N$OF=!00uV0L  $BB!$*JJ/Dd$K  C   J ./446D3399; (1VO !F1I*_1I$$Q'( !++b.a.ay$BVBV.aZ`.abbb c 	!#"'w%-z".2oo*+=T==GGGGG	ss   EAC4 #C0$AC4 +A C4 +C2,C4 /E0C4 2C4 4	E==E:D=;E?E EEEEis_retryis_fallbackis_asyncc                    d|dg}	 ||d<   ||d<   |j                  d| j                        |d<   |j                  di       j                  d|i       | j	                  |||j                  dd       	      }|d
   j                         }	| j                  j                         D ])  \  }
}|
|vr|||
<   |
dk(  s||
   j                  |       + t        j                  di i |	|| j                  d|S # t        $ r}|d }~ww xY w)Nr  r  r   r  r}   r   r  r4  r5  r   )r  r9  rf   )rH  r}   r   r   rD  rE  r   r   r  r   text_completionrl   r%  )r   r   r  r/  r0  r1  r-  r)  r!  rN  r  r  r'  s                rh   r3  zRouter.text_completion  s6    $78	#F7O%F8$*JJ}d>N>N$OF=!j"-44mU5KL 66!$*JJ/Dd$K 7 J ./446D3399; (1VO !F1I*_1I$$Q'( **s-r-rSWSgSg-rkq-rss 	G	s   B2C9 ;=C9 9	D	DD	c           	        K   |j                  dd       9| j                  ||j                  d      | j                  ||f|       d {   S 	 ||d<   ||d<   | j                  |d<   | j                  ||        | j                  di | d {   }|S 7 M7 # t        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY ww)	Nr\  )r   r\  r+  r  r-  r   r  r+  r,  rc  rf   )rH  r  rE  atext_completion_atext_completionr/  rm  r%  rn  ro  r2   rr  rs  )	r   r   r  r/  r0  r1  r-  r1  r'  s	            rh   r5  zRouter.atext_completion  s      ::j$'3//J/"&"7"7V_ 0   	#F7O%F8*.*@*@F&'00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		sI   AC"	B
C"AB BB C"B 	C$6CCC"c           	      0  K   	 t        j                  d| d|        t        |      }| j                  |d|dg|j	                  dd              d {   }| j                  ||       |d   j                         }|d	   }| j                  ||      }| j                  |xx   d
z  cc<   t        j                  di i ||| j                  |d|}	| j                  ||d      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   d
z  cc<   t        j$                  d| d       |	S 7 L7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   d
z  cc<   |d }~ww xY ww)N#Inside _atext_completion()- model: rw  r  r  r4  r5  r6  r   r   r\   r  r{  r|  ry  zlitellm.atext_completion(model=r;  rB  rC  rf   )r   r   r   rx  rE  rF  r   r  r   r   r5  rl   rG  r   rn  r  r  r   r   r%  r   r   )r   r   r  r-  rb  r!  rN  rM  rQ  r1  r  r'  s               rh   r6  zRouter._atext_completion  s    B	!''5eWJvhO  AH#BB#)f=>$*JJ/Dd$K  C   J
 //:f/U./446DgJ>>% ? L Z(A-(// $#33*	
 H !,,%3 - M (Zw00. ) . . EE#-@P F    &.~H. . . AA)<L B    "*>z*a/*!&&1*=TU OmF.
  .. . . . *  	!&&1%8LSQRVHT[\  &!+&G	   HAG F!B8G F$G F,*F&+	F,4F(5F,9G F*G  G!	G *G+5G  H!G $G &F,(F,*G ,F>2F53F>:G G 	HA HHH
adapter_idc           	        K   	 ||d<   ||d<   | j                   |d<   |j                  d| j                        |d<   |j                  di       j	                  d|i        | j
                  di | d {   }|S 7 # t        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY ww)	Nr   r:  r+  r}   r   r  rc  rf   )_aadapter_completionrH  r}   r   r   rm  r%  rn  ro  r2   rr  rs  )	r   r:  r   r/  r0  r1  r-  r1  r'  s	            rh   aadapter_completionzRouter.aadapter_completionX  s     	#F7O#-F< *.*C*CF&'$*JJ}d>N>N$OF=!j"-44mU5KL?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		s;   CA0A= 4A;5A= :C;A= =	C6B<<CCc           	      0  K   	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd              d {   }| j                  ||       |d	   j                         }|d
   }| j                  ||      }| j                  |xx   dz  cc<   t        j                  di i ||| j                  |d|}	| j                  ||d      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   dz  cc<   t        j$                  d| d       |	S 7 L7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   dz  cc<   |d }~ww xY ww)Nz&Inside _aadapter_completion()- model: rw  r  zdefault textr  r4  r5  r6  r   r   r\   )r:  r9  r:  r{  r|  ry  z"litellm.aadapter_completion(model=r;  rB  rC  rf   )r   r   r   rx  rE  rF  r   r  r   r   r=  rl   rG  r   rn  r  r  r   r   r%  r   r   )r   r:  r   r-  rb  r!  rN  rM  rQ  r1  r  r'  s               rh   r<  zRouter._aadapter_completionu  s    B	!''8z&R  AH#BB#)nEF$*JJ/Dd$K  C   J
 //:f/U./446DgJ>>% ? L Z(A-(22 ",#33*	
 H !,,%3 - M (Zw00. ) . . EE#-@P F    &.~H. . . AA)<L B    "*>z*a/*!&&4ZL@WX OmF.
  .. . . . *  	!&&4UG;OPSTUPVxW^_  &!+&G	r9  c                    	 ||d<   ||d<   | j                   |d<   |j                  d| j                        |d<   |j                  di       j	                  d|i        | j
                  di |}|S # t        $ r}|d }~ww xY w)Nr   r"  r+  r}   r   r  rf   )
_embeddingrH  r}   r   r   r0  r%  r   r   r"  r1  r-  r1  r'  s          rh   	embeddingzRouter.embedding  s    		#F7O#F7O*.//F&'$*JJ}d>N>N$OF=!j"-44mU5KL3t33=f=HO 	G	r  c           	         d }	 t        j                  d| d|        | j                  |||j                  dd             }| j	                  ||       |d   j                         }|d   }| j                  ||d	      }|j                  d
d       }||||j                  k7  rd }	n|}	| j                  |xx   dz  cc<   | j                  |       t        j                  di i ||| j                  |	d|}
| j                  |xx   dz  cc<   t        j                  d| d       |
S # t         $ rE}t        j                  d| dt#        |       d       || j$                  |xx   dz  cc<   |d }~ww xY w)NzInside embedding()- model: rw  r4  r   r"  r4  r6  r   r   syncr|  r#  r\   r7  r"  r9  r:  zlitellm.embedding(model=r;  rB  rC  rf   )r   r   rD  rE  rF  r   rG  rH  r#  r   rJ  r   rB  rl   r   r   r%  r   r   )r   r"  r   r-  rM  r!  rN  rO  rP  rQ  r1  r'  s               rh   r@  zRouter._embedding  s   
4	!''-eWJvhG 66$*JJ/Dd$K 7 J
 //:f/U./446DgJ%)%5%5%f& &6 &" %jjD9O+*6#'='E'EE#5Z(A-( 11Z1H(( "#33*	
 H z*a/*!&&*:,6MN O 	!&&*:,6J3q6(RYZ %
+q0+G	s   D+D0 0	E>9A E99E>c           	      .  K   	 ||d<   ||d<   | j                   |d<   | j                  ||        | j                  di | d {   }|S 7 # t        $ r;}t	        j
                  t        | |t        j                         |             |d }~ww xY wwr'  )	_aembeddingr/  rm  r%  rn  ro  r2   rr  rs  rA  s          rh   
aembeddingzRouter.aembedding	  s     	#F7O#F7O*.*:*:F&'00uV0L?T??I&IIHO J 		(,0#)(1(<(<(>'(	 G		r   c           	      ,  K   d }	 t        j                  d| d|        t        |      }| j                  |||j	                  dd              d {   }| j                  ||       |d   j                         }|d   }| j                  ||      }| j                  |xx   dz  cc<   t        j                  di i ||| j                  |d	|}	| j                  ||d
      }
|
\t        |
t        j                        rB|
4 d {    	 | j!                  ||       d {    |	 d {   }	d d d       d {    n%| j!                  ||       d {    |	 d {   }	| j"                  |xx   dz  cc<   t        j$                  d| d       |	S 7 L7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t&        $ rE}t        j$                  d| dt)        |       d       || j*                  |xx   dz  cc<   |d }~ww xY ww)NzInside _aembedding()- model: rw  r4  rD  r6  r   r   r\   rF  r{  r|  ry  zlitellm.aembedding(model=r;  rB  rC  rf   )r   r   r   rx  rE  rF  r   r  r   r   rI  rl   rG  r   rn  r  r  r   r   r%  r   r   )r   r"  r   r-  rM  rb  r!  rN  rQ  r1  r  r'  s               rh   rH  zRouter._aembedding	  s    
A	!''/wjI  AH#BB$*JJ/Dd$K  C   J
 //:f/U./446DgJ>>% ? L
 Z(A-()) "#33*	
 H !,,%3 - M (Zw00. ) . . EE#-@P F    &.~H. . . AA)<L B    "*>z*a/*!&&+J<7NO OkD.
  .. . . . *  	!&&+J<7KCPQF8SZ[ %
+q0+G	s   HAG FB8G F"G F*(F$)	F*2F&3F*7G F(G F?	G (G)5G HG "G $F*&F*(G *F<0F31F<8G G 	HA HHHc           	      b  K   	 ||d<   | j                   |d<   |j                  d| j                        |d<   | j                  ||        | j                  di | d {   }|S 7 # t
        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY wwNr   r+  r}   r,  rc  rf   )_acreate_filerH  r}   r/  rm  r%  rn  ro  r2   rr  rs  r)  s        rh   acreate_filezRouter.acreate_filec	  s     
	#F7O*.*<*<F&'$*JJ}d>N>N$OF=!00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		;   B/AA( A& A( %B/&A( (	B,16B''B,,B/c                   K   	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd              d {   }| j                  ||       |d	   j                         }|d
   }| j                  ||      }| j                  |xx   dz  cc<   t        |d
         \  }}	}
}
t        |d   |      |d<   t        j                  di i ||	| j                  |d|}| j                  ||d      }|\t        |t         j"                        rB|4 d {    	 | j%                  ||       d {    | d {   }d d d       d {    n%| j%                  ||       d {    | d {   }| j&                  |xx   dz  cc<   t        j(                  d| d       |S 7 s7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t*        $ rH}t        j,                  d| d| dt/        |       d       || j0                  |xx   dz  cc<   |d }~ww xY ww)Nr8  rw  r  files-api-fake-textr  r4  r5  r6  r   r   r\   r   r  )file_contentnew_model_namecustom_llm_providerr9  r:  r{  r|  ry  litellm.acreate_file(model=r;  , rB  rC  rf   )r   r   r   rx  rE  rF  r   r  r   rW   r%   r   rN  rl   rG  r   rn  r  r  r   r   r%  	exceptionr   r   )r   r   r-  rb  r!  rN  rM  rQ  stripped_modelrV  r  r1  r  r'  s                 rh   rM  zRouter._acreate_file{	  s    
J	!''5eWJvhO  AH#BB#)6KLM$*JJ/Dd$K  C   J
 //:f/U./446DgJ>>% ? L Z(A-( 9I7m95N/A 4#F^NF6N ++ +>#33*	
 H !,,%3 - M (Zw00. ) . . EE#-@P F    &.~H. . . AA)<L B    "*>z*a/*!&&-j\9PQ O}V.
  .. . . . *  	!++-eWBvh>RSVWXSYRZZab  &!+&G	s   I AG, GCG, 5G6G, 9GG	GGG G, +G,G, G(	G, G*5G, I G, G, GGG, G%GG%!G, *G, ,	H=5AH88H==I c           	      b  K   	 ||d<   | j                   |d<   |j                  d| j                        |d<   | j                  ||        | j                  di | d {   }|S 7 # t
        $ r;}t        j                  t        | |t        j                         |             |d }~ww xY wwrL  )_acreate_batchrH  r}   r/  rm  r%  rn  ro  r2   rr  rs  r)  s        rh   acreate_batchzRouter.acreate_batch	  s     
	#F7O*.*=*=F&'$*JJ}d>N>N$OF=!00uV0L?T??I&IIHO J  		(,0#)(1(<(<(>'(	 G		rO  c                 T  K   	 t        j                  d| d|        t        |      }| j                  |dddg|j	                  dd              d {   }t        d	      }|j                  |i       j                  |d
   d   |j                  di       |j                  d
i       j                  d      d       |j                  di       |d<   |d
   j                         }|d   }| j                  ||       | j                  ||      }| j                  |xx   dz  cc<   t        |d         \  }	}
}	}	t        j                  di i ||
| j                   |d|}| j#                  ||d      }|\t%        |t&        j(                        rB|4 d {    	 | j+                  ||       d {    | d {   }d d d       d {    n%| j+                  ||       d {    | d {   }| j,                  |xx   dz  cc<   t        j.                  d| d       |S 7 7 7 {7 s7 e# 1 d {  7  sw Y   OxY w7 `7 X# t0        $ rH}t        j2                  d| d| dt5        |       d       || j6                  |xx   dz  cc<   |d }~ww xY ww)Nz Inside _acreate_batch()- model: rw  r  rQ  r  r4  r5  r\  )function_namer   r   r  r  r  r6  r\   rR  rU  r{  r|  ry  rW  r;  zlitellm._acreate_batch(model=rX  rB  rC  rf   )r   r   r   rx  rE  r$   r   r   rH  r   rF  r  r   rW   r   r]  rl   rG  r   rn  r  r  r   r   r%  rY  r   r   )r   r   r-  rb  r!  metadata_variable_namerN  rM  rQ  r  rV  r1  r  r'  s                 rh   r\  zRouter._acreate_batch	  s?    
P	!''25'F8L  AH#BB#)6KLM$*JJ/Dd$K  C   J
 &H.&" 4b9@@",-=">w"G",..r"B */? D H H T $.>>,#CF< ./446DgJ//:f/U>>% ? L Z(A-( ,<$w-+P(A"Aq,, +>#33*	
 H !,,%3 - M (Zw00. ) . . EE#-@P F    &.~H. . . AA)<L B    "*>z*a/*!&&-j\9PQ OIb.
  .. . . . *  	!++/wb@TUXYZU[T\\cd  &!+&G	s   J(AI H0EI H3I !H;9H5:	H;H7H;I H9I /I0	I 9I:5I /J(0I 3I 5H;7H;9I ;III	I I 	J%AJ  J%%J(c           	        K   	 | j                         }|t        d      g fd}t        j                  |D cg c]
  } ||       c}ddi d{   }|D ]  }t	        |t
              s|c S  rd   t        dj                              c c}w 7 E# t        $ r;}t        j                  t        | t        j                         |             |d}~ww xY ww)	z
        Iterate through all models in a model group to check for batch

        Future Improvement - cache the result.
        NRouter not yet initialized.c                   K   	 t        | d   d         \  }}}}t        j                        }|j                  dd        t	        j
                  dd|i| d {   S 7 # t        $ r}j                  |       Y d }~y d }~ww xY ww)Nr   r   rR  rV  rf   )rW   r   r   rE  r   aretrieve_batchr%  r   )rM  r  rV  
new_kwargsr'  r-  receieved_exceptionss        rh   try_retrieve_batchz2Router.aretrieve_batch.<locals>.try_retrieve_batchL
  s       4D()9:7C40A*Aq "&v!6JNN#8$?!(!8!8 ",?"CM"    !  (//2 sA   BAA$ A"A$ !B"A$ $	B-B>BBBreturn_exceptionsTr   z7Unable to find batch in any model. Received errors - {}rc  )r  r%  rn  r  r   r9   r   ro  r2   rr  rs  )	r   r-  filtered_model_listrg  r   resultsr  r'  rf  s	    `      @rh   rd  zRouter.aretrieve_batch;
  s    6	"&"5"5"7"* =>>#%  " $NN9LM$U+M"& G " "fe,!M"
 $*1-- IPP(  N(  		(,0#)(1(<(<(>'(	 G		sK   C&8B B
B BB 2B 5C&6(B 	C#(6CC##C&c                   K   | j                  |      }|t        d      dt        ffd}t        j                  |D cg c]
  } ||       c}  d{   }dg dddd}|D ]g  }||d	   t        |d	      rt        |d	      |d	<   t        |d
      |d
<   |d   j                  |j                         t        |dd      du scd|d<   i |S c c}w 7 w)zQ
        Return all the batches across all deployments of a model group.
        r  Nrb  r   c                 x   K   	 t        j                  di i | d    d {   S 7 # t        $ r Y y w xY ww)Nr   rf   )r   alist_batchesr%  r,  s    rh   rg  z0Router.alist_batches.<locals>.try_retrieve_batch
  sT     $22 ;/0;F;     s(   :+ )+ :+ 	7:7:r   F)objectrN  first_idlast_idhas_morero  rp  rN  rq  T)	r  r%  rC   rn  r  hasattrgetattrextendrN  )r   r   r-  ri  rg  rj  final_resultsr  s     `     rh   rm  zRouter.alist_batches|
  s     #11U1C&9::	,? 	  5HIE 'I
 

 
  
	5F! ,49T07
0KM*-+269+Ei(f%,,V[[9 6:u5=04M*-
	5 / J
s*   ?CCCCC-AC
Cc                    K   |j                  d      r=| j                  |d         r(| j                  |d          d {   }|d   d   |d<    |di | d {   S 7 7 w)Nr   r  rR  r   rf   )rH  r  rx  )r   r+  r-  r!  s       rh   )_pass_through_moderation_endpoint_factoryz0Router._pass_through_moderation_endpoint_factory
  s|     
 ::g4#6#6&/#6#R#BBWo  C   J ))9:7CF7O&0000	 1s#   ?A%A!A%A#A%#A%r   )
assistantsr   c                 T     	 	 ddt         t        d      dt         d   f fd}|S )NrV  r   azurer:  r   c                    K   dk(  r j                   d| |d| d {   S dk(  r j                  ddi| d {   S y 7 &7 w)Nrx  )r+  rV  r:  r   r+  rf   ))_pass_through_assistants_endpoint_factoryrw  )rV  r:  r-  r   r+  r   s      rh   new_functionz-Router.factory_function.<locals>.new_function
  s     
 L(KTKK &7(;! 	   l*KTKK &7   +s!   AA	 AAAANN)r   r   )r   r+  r   r~  s   ``` rh   r   zRouter.factory_function
  s:     IM.2	!)'2C*D!E	]+	& rg   rV  rz  r:  c                    K   |E| j                   .| j                   d   }|j                  | j                   d          nt        d       |d||d| d{   S 7 w)z@Internal helper function to pass through the assistants endpointNrV  r   z'custom_llm_provider' must be set. Either via:
 `Router(assistants_config={'custom_llm_provider': ..})` 
or
 `router.arun_thread(custom_llm_provider=..)`)rV  r:  rf   )rs   r   r%  )r   r+  rV  r:  r-  s        rh   r}  z0Router._pass_through_assistants_endpoint_factory
  s      &%%1&*&<&<=R&S#d445EFG s  ' 
 3F
FL
 
 	
 
s   AAAAc                 
  K   |j                  d      }|j                  dd      }|j                  d| j                        }|j                  d| j                        }|j                  d| j                        }|j                  dd      }	 | j                  |||||	       | | j                  |i |d|i d{   }	n | j                  |i | d{   }	t        j                  d
|	        |	S 7 97  # t        $ r}
t        j                  dt        j                                 |
}d}|j                  d      }d}|du s||
| |d|}d|vr| j                  |d<   d|vrd|d<   	 t        j                  d       t        |      }|r/|j                  ||d       t!        |i | d{  7  }	|	cY d}
~
S t#        |
t$        j&                        r|F| j)                  ||      }|||j                  ||d       t!        |i | d{  7  }	|	cY d}
~
S dj+                  |||      }t        j                  dj+                  |             |
xj,                  dj+                  |      z  c_        nt#        |
t$        j.                        r|F| j)                  ||      }|||j                  ||d       t!        |i | d{  7  }	|	cY d}
~
S dj+                  |||      }t        j                  dj+                  |             |
xj,                  dj+                  |      z  c_        ||t        j                  d|        t1        |t3        t4        |            \  }}|
|||   d   }|Dt        j                  d| d|        t7        |d       r|xj,                  d| d| z  c_        ||j                  ||d       t!        |i | d{  7  }	|	cY d}
~
S n# t        $ r}t        j8                          t;        |      }t        j<                  d!j+                  t5        |      t        j                         t?        | |"       d{  7               t5        |      }Y d}~nd}~ww xY wt7        |d       rW|xj,                  d#j+                  ||      z  c_        tA        |      dkD  r$|xj,                  d$j+                  |      z  c_        |d}
~
ww xY ww)%z
        Try calling the function_with_retries
        If it fails after num_retries, fall back to another model group
        r   disable_fallbacksFr   r   r   mock_timeoutN)r-  r  r   r   r   zAsync Response: 	Tracebackr>  T)litellm_routerrg  r~   fallback_depthr   zTrying to fallback b/w models)r   )fallback_model_grouporiginal_model_group)r   r  zmodel={}. context_window_fallbacks={}. fallbacks={}.

Set 'context_window_fallback' - https://docs.litellm.ai/docs/routing#fallbackszGot 'ContextWindowExceededError'. No context_window_fallback set. Defaulting                             to fallbacks, if available.{})msgz
{}zmodel={}. content_policy_fallback={}. fallbacks={}.

Set 'content_policy_fallback' - https://docs.litellm.ai/docs/routing#fallbackszGot 'ContentPolicyViolationError'. No content_policy_fallback set. Defaulting                             to fallbacks, if available.{}zinside model fallbacks: r   z7No fallback model group found for original model_group=z. Fallbacks=r@  zlitellm.router.py::async_function_with_fallbacks() - Error occurred while trying to do fallbacks - {}
{}

Debug Information:
Cooldown Deployments={}rd  rb  z;
Received Model Group={}
Available Model Group Fallbacks={}z
Error doing the fallback: {})!rH  rE  r   r   r   _handle_mock_testing_fallbacksr.  r   r   r%  rr  rs  r~   r   r-   r   r/   r   r   ContextWindowExceededError(_get_fallback_model_group_from_fallbacksr   r@  rL  r.   r   r   rr  	print_excr   errorr*   r  )r   r  r-  r  r  r   r   r   r  r1  r'  rg  r  r  fallback_failure_exception_strinput_kwargsis_non_standard_fallback_formaterror_messagegeneric_fallback_idxnew_exceptionrb  s                        rh   rm  z$Router.async_function_with_fallbacks
  s    
 &,ZZ%8,2JJ7JE,R$*JJ{DNN$K	39::&(E(E4
  4:::&(E(E4
  zz.$7D	%//'#)A)A 0  '!A!A!A"#"2>"  "B!A!A4!R6!RR!''*:8*(EFO S  s	%!'')I4H4H4J3K(LM!"#' 28**W2E -/* D(,@,H #'&8 L l2040B0B_-|312-.OD%**+JK 3V'3/ 3 ''4=4H &8&&&    H
 $Oa!C!CD/; II*B,7 J  - 07"44$++8L8L *<!*** $ $  ( )q  )x  )x')A9) .22!;;A6 -< 		V]]=%AA	7#F#FG/; II*B,7 J  - 07"44$++8L8L *<!*** $ $  ( )p  )w  )w')A9) .22!;;A6 -< 		V]]=%AA	([-D)//2J9+0VW0&/(,S+(> ?(*> -40</89M/Ns/S,+3-22UVaUbbnoxnyz ##5yA.66<st  tA  AM  NW  MX  ;Y  Y600 ''4H4H &8&&&    H
 $O D##%#DV#L %++ o  v  vM*!,,.M48-=  	 25]1C.D )95"**.m.t.t(/ * 56:&..8??:. %$gs	%s   B
T 1C< >C8?C< C:C< 7T 8C< :C< <
S=A,S83AO>8F;9O>?S= T AO>HO>!S="T 'B2O>KO> S=!T &D
O>0O31O>7S=8T =S8>	RA&R-Q0.RS8RA(S88S==T r  c                 J   |j                  dd      }|j                  dd      }|j                  dd      }|"|du rt        j                  |dd| d| 	      |"|du rt        j                  |dd| d
| 	      |#|du rt        j                  |dd| d| 	      yy)a  
        Helper function to raise a litellm Error for mock testing purposes.

        Raises:
            litellm.InternalServerError: when `mock_testing_fallbacks=True` passed in request params
            litellm.ContextWindowExceededError: when `mock_testing_context_fallbacks=True` passed in request params
            litellm.ContentPolicyViolationError: when `mock_testing_content_policy_fallbacks=True` passed in request params
        mock_testing_fallbacksNmock_testing_context_fallbacks%mock_testing_content_policy_fallbacksTr>  #This is a mock exception for model=z#, to trigger a fallback. Fallbacks=r   rA  r@  zF, to trigger a fallback.                     Context_Window_Fallbacks=zF, to trigger a fallback.                     Context_Policy_Fallbacks=)rE  r   InternalServerErrorr  rL  )	r   r-  r  r   r   r   r  r  r  s	            rh   r  z%Router._handle_mock_testing_fallbacks  s
     "(,Dd!K)/,d*
& 17

3T1
- "-2HD2P--!=k]Jmnwmxy  +6.$644!=k] K..F-GI  2=5=55!=k] K..F-GI  > >rg   c           
      *  K   t        j                  d| d|        |j                  d      }|j                  d| j                        }t	        |      }|j                  d| j
                        }|j                  d| j                        }|j                  d      }|j                  d      }	|j                  d	      xs i }
d
|
v rFt        |
d
   t              r3| j                  |
d
         }||
j                  dt        |      i       t        j                  d| d|	        	 | j                  ||        | j                  |g|i | d {   }|S 7 # t        $ r&}d }|}t!        |dd       }|t        |t"              r|}		 | j%                  |j                  d      xs d|       d {  7  \  }}| j'                  ||||||       | j(                  | j*                  &| j-                  ||j                  d            }||}	|	dkD  r| j/                  ||      }n | j1                  ||	|	||      }t3        j4                  |       d {  7   t7        |	      D ]  }	  | j                  |g|i | d {  7  }t9        j:                  |      r| d {  7  }|c cY d }~S # t        $ r}| j/                  ||      }|	|z
  }|j                  d      }| | j%                  ||       d {  7  \  }}ng }| j1                  |||	||      }t3        j4                  |       d {  7   Y d }~d }~ww xY w t=        |      t>        j@                  v rtC        |d|	       tC        |d|       |d }~ww xY ww)Nz+Inside async function with retries: args - z; kwargs - r+  r   r   r   r   r}   r   r  r  model_group_sizez/async function w/ retries: original_function - z, num_retries - )r  r-  r>  r  )r  r   all_deploymentsr   regular_fallbacksr   )rY  r  r   )r-  r'  )r'  remaining_retriesr}   r   r  r   )"r   r   rE  r   r   r   r   rH  r   r   r  r   r  %_handle_mock_testing_rate_limit_error	make_callr%  rs  r   r  should_retry_this_errorr   r   r0   	log_retry_time_to_sleep_before_retryrn  r  r  inspectiscoroutinefunctionr   r   LITELLM_EXCEPTION_TYPESr  )r   r  r-  r+  r   rb  r   r   r  r}   	_metadatarr   r1  r'  current_attemptrg  deployment_num_retriesr  _all_deployments_retry_policy_retriesr   r  _modelr  _timeouts                            rh   r.  z"Router.async_function_with_retries  s    ##9${6(S	
 #JJ':;JJ{DNN;	<VD#)::&(E(E$
  $*::&(E(E$
  &,ZZ%8jj/ !**Z06B	I%*Y}5Ms*S,,	-8P,QJ%  "4c*o!FG##=>O=PP`al`mn	
e	%66' 7  ,T^^,=OOOOHO P  ]	%"O!"%,Qt%D"%1j&7 5 99 **W-3%5 :    3 "2 (($8 0)A"+)A )  !!-00< )-(N(N0fjj>Q )O )% )4"7KQv9KL ::$"-'$8 0 ; K --,,,#(#5 22%3T^^4E%W%WPV%WWWH22  *2>>#O  2!^^6Q^?F(3o(E%,2JJw,?F)"&"E"E&,1A #F #   0,a 02,#??,*;$/,@(8  @  H "--111+22B &'7+J+JJ*M;G*M?K$${]	%s   D(N+,E  EE  NE   
N*AN9F<:B%NI" N4J<JJ<-J0.J<4N6N7N<	MAML	
9M M
MNM;NNNc                    K   |j                  d      } ||i |}t        j                  |      st        j                  |      r
| d{   }| j	                  ||       d{   }|S 7 !7 w)z^
        Handler for making a call to the .completion()/.embeddings()/etc. functions.
        r   N)r1  r  )rH  r  r  isawaitableset_response_headers)r   r+  r  r-  r  r1  s         rh   r  zRouter.make_callv  sx      jj)$d5f5&&x0G4G4G4Q%~H22; 3 
 
  &
s$   A	A1A-A1&A/'A1/A1c                     |j                  dd      }|9|du r4t        j                  d|        t        j                  |dd| d      yy)	z
        Helper function to raise a mock litellm.RateLimitError error for testing purposes.

        Raises:
            litellm.RateLimitError error when `mock_testing_rate_limit_error=True` passed in request params
        mock_testing_rate_limit_errorNTzTlitellm.router.py::_mock_rate_limit_error() - Raising mock RateLimitError for model=r>  r  z , to trigger a rate limit error.r  )rE  r   r   r   RateLimitError)r   r-  r  r  s       rh   r  z,Router._handle_mock_testing_rate_limit_error  st     9?

+T9
% *5-5!&&fgrfst ((!=k]Jjk  6 6rg   r  r   r  r  c                    d}|t        |t              rt        |      }d}|t        |t              rt        |      }t        |t        j                        r||t        |t        j
                        r||t        |t        j                        r|t        |t        j                        r|dk  r|t        |      dkD  r|t        |t        j                        r	 |dk  r||dk  r|y)au  
        1. raise an exception for ContextWindowExceededError if context_window_fallbacks is not None
        2. raise an exception for ContentPolicyViolationError if content_policy_fallbacks is not None

        2. raise an exception for RateLimitError if
            - there are no fallbacks
            - there are no healthy deployments in the same model group
        r   r\   T)
r   r   r  r   r  rL  NotFoundErrorr   r  AuthenticationError)	r   r  r   r  r   r   r  _num_healthy_deployments_num_all_deploymentss	            rh   r  zRouter.should_retry_this_error  s    " $% *z:Mt/T'*+>'?$ &:ot+L#&#7  ug@@A(4K ugAAB(4KeW223KeV223(A-%1)*Q.eV778
 %) $q(Krg   c                 4    t        | j                  g|i |S )z~
        Sync wrapper for async_function_with_fallbacks

        Wrapped to reduce code duplication and prevent bugs.
        )r   rm  )r   r  r-  s      rh   r0  zRouter.function_with_fallbacks  s     "$"D"DVtVvVVrg   c                 l    |yd}|D ])  }t        |j                               d   |k(  s#||   } |S  |S )a@  
        Returns the list of fallback models to use for a given model group

        If no fallback model group is found, returns None

        Example:
            fallbacks = [{"gpt-3.5-turbo": ["gpt-4"]}, {"gpt-4o": ["gpt-3.5-turbo"]}]
            model_group = "gpt-3.5-turbo"
            returns: ["gpt-4"]
        Nr   )r   keys)r   r   r  r  r  s        rh   r  z/Router._get_fallback_model_group_from_fallbacks  sV     48 	DDIIK #{2'+K'8$##		 $#rg   r'  r  c                    |t        |      dk(  rn!|t        |t              rt        |      dkD  ryd}t        |d      r,t        |j                  d      r|j                  j
                  }t        |d      r|j                  }|%t        j                  |||| j                        }|S t        j                  ||| j                        }|S )	z
        Calculate back-off, then retry

        It should instantly retry only when:
            1. there are healthy deployments in the same model group
            2. there are fallbacks for the completion call
        Nr\   r   r1  headerslitellm_response_headers)r  r   response_headersmin_timeout)r  r   r  )
r  r   r   rr  r1  r  r  r   _calculate_retry_afterr   )r   r'  r  r}   r   r  r  r   s           rh   r  z"Router._time_to_sleep_before_retry   s    " &3+?1+D+.5'(1,481j!gajj)&D zz11101 99'44"3'!1 ,,	G  44"3' ,,G rg   c                   K   	 |j                  dd      }|t        d      |d   j                  d      y|d   d   j                  dd      }|j                  dd      }|j                  dd      }||yt        |t              rt	        |      }t        |      }	|j                  d	d
      }
t               }|j                  d      }t        j                  j                  j                  |||      }| j                  j                  ||
|	t        j                  j                         d{    t        j                   j                  j                  |||      }| j                  j                  |d|	t        j                  j                         d{    t#        | |       |S 7 ~7 # t$        $ r7}t'        j(                  dj                  t	        |                   Y d}~yd}~ww xY ww)zG
        Track remaining tpm/rpm quota for model in model_list
        standard_logging_objectNzstandard_logging_object is Noner   r   r!  r  model_idtotal_tokensr   %H-%Mr  current_minuter   keyr  rb  re   r\   rd  deployment_idzOlitellm.router.Router::deployment_callback_on_success(): Exception occured - {})rH  r  r   r   r   r   rZ   strftimerH   TPMr  r   r   async_increment_cacher`   re   RPMr5   r%  r   rY  )r   r-  completion_responser`  ra  r  deployment_namer  r  rb  r  dtr  tpm_keyrpm_keyr'  s                   rh   r   z%Router.deployment_callback_on_success3  s    F	HN

)4I# '. !BCC&'++J7?"()9"::"F"J"J $# 699-N,00TB&"*C(RB#DV#L &=&A&A.RS&T
 &'!#" *--33::. ;  jj66&%5#--	 7    *--33::. ;  jj66%5#--	 7    B,0"$
 /  	!++ahhF
 	sk   G83F5 G8A F5 8G89B9F5 2F13A*F5 F3F5 0G81F5 3F5 5	G5>-G0+G80G55G8c                    d}|d   j                  d      nc|d   d   j                  dd      }|d   j                  di       xs i }|j                  dd      }||yt        |t              rt        |      }|t	        | |      }|S y)z
        Tracks the number of successes for a deployment in the current minute (using in-memory cache)

        Returns:
        - key: str - The key used to increment the cache
        - None: if no key is found
        Nr   r   r  r  r  r  )rH  r   r   r   r5   )	r   r-  r  r`  ra  r  r  r  r  s	            rh   r   z*Router.sync_deployment_callback_on_success  s     "#''
3; !12:>BB=RVWK 0155lBGM2Jd+B"bjB$W>C(, C Jrg   c                 F   	 |j                  dd      }t        |dd      }|j                  di       j                  di       }t        j                  j                  j                  |      }|j                  di       j                  d| j                        }	|3t        j                  j                  |	      }	|	|	d
k  r| j                  }	t        |t              r1|j                  dd      }
t        | |
       t        | |||
|	      }|S y# t        $ r}|d}~ww xY w)a`  
        2 jobs:
        - Tracks the number of failures for a deployment in the current minute (using in-memory cache)
        - Puts the deployment in cooldown if it exceeds the allowed fails / minute

        Returns:
        - True if the deployment should be put in cooldown
        - False if the deployment should not be put in cooldown
        rY  Nstatus_coder>  r   r  )rg  r   )r  r   r  r  rd  exception_statusrg  r!  time_to_cooldownF)rH  rs  r   litellm_core_utilsexception_mapping_utils_get_response_headersr   utils&_get_retry_after_from_exception_headerr   r   r4   r,   r%  )r   r-  r  r`  ra  rY  r  _model_infoexception_headers_time_to_cooldownr  r  r'  s                rh   r   z%Router.deployment_callback_on_failure  s<    ,	

;5I&y-D **%5r:>>|RPK ' : : R R h h#, !i ! !'

+;R @ D D!3!3! !, MMHH): I  " %,0AA0E(,(:(:%+t, +d ;@,0"/ 3,0%5'0,%6  	G	s   DD 	D DD r  c                 (  K   |d   d   j                  dd      }|d   d   j                  dd      }|d   j                  di       xs i }|j                  dd      }||yt        |t              rt        |      }t	        |      }	t               }
|
j                  d      }t        j                  j                  j                  |||	      }| j                  j                  |d
|	t        j                  j                         d{    y7 w)z3
        Update RPM usage for a deployment
        r   r   r!  Nr  r  r  r  r  r\   r  )rH  r   r   r   r   rZ   r  rH   r  r  r   r   r  r`   re   )r   r-  r  r`  ra  r  r  r  r  rb  r  r  r  s                rh   r   z+Router.async_deployment_callback_on_failure  s     !!12:>BB$
 -.z:>>}dS,-11,CIr
^^D$'"*C RB<VD

 "%%++22. 3 
 jj..-%%	 / 
 	
 	
s   DD
DDc                    	 t        |      j                  t        |      d}|j                         D ]T  \  }}|dvr|||<   |dk(  st	        |t
              s'i |d<   |d   j                         D ]  \  }}|dk7  s|||   |<    V t        | j                        dkD  r| j                  j                  d       | j                  j                  |       | j                  |d   d<   |S # t        $ r}|d}~ww xY w)z
        When a retry or fallback happens, log the details of the just failed model call - similar to Sentry breadcrumbing
        )exception_typeexception_string)r   r)  r+  r   r      r   N)r   rb   r   r  r   r   r  r   rE  r   r%  )r   r-  r'  previous_modelr  r  
metadata_k
metadata_vs           rh   r  zRouter.log_retry  s   	 #'q'"2"2$'FN 	G  II()N1%*_At)<13N:.282D2J2J2L G.
J%)::<FN1-j9GG 4''(1,$$((+  ''7484H4HF:01M 	G	s+   AC, 	C, #C, >A-C, ,	C<5C77C<r  rb  c                     |}| j                   j                  ||d      }|#d}| j                   j                  ||dd       |S |dz  }| j                   j                  ||d       |S )zf
        Update deployment rpm for that minute

        Returns:
        - int: request count
        T)r  rb  
local_onlyr\   ra   )r  r  r  re   )r  r  r  )r   	get_cache	set_cache)r   r  rb  r  request_counts        rh   _update_usagezRouter._update_usage*  s      

,,*:t - 
  MJJ  =Tr !   QMJJ  =T !  rg   c                 n    | j                   y| j                   D ]  }t        |t              sd|v s y y)NFr   T)r   r   r   )r   fallbacks     rh   _has_default_fallbackszRouter._has_default_fallbacksE  s;    >>! 	 H(D)(?	  rg   r1  c                 J   |j                   d   j                  dk7  ry|j                  d| j                        }|2d}|D ](  }t	        |j                               d   |k(  s#||   } n |y| j                         ryt        j                  dj                  ||             y)z
        Determines if a content policy error should be raised.

        Only raised if a fallback is available.

        Else, original response is returned.
        r   content_filterFr   NTzyContent Policy Error occurred. No available fallbacks. Returning original response. model={}, content_policy_fallbacks={})
choicesfinish_reasonrH  r   r   r  r  r   r   r   )r   r   r1  r-  r   r  r  s          rh   rK  z)Router._should_raise_content_policy_errorN  s     A,,0@@#)::&(E(E$
 
 $/#' 0 		$Q'50+/;(
 $/((*"" H  O  O/	

 rg   c                     g }	 | j                  |      \  }}t        |t              rg S 	 t	        | |      }g }|D ]  }|d   d   |v r|j                  |         ||fS # t        $ r Y Aw xY w)NrR  r  r  r  )#_common_checks_available_deploymentr   r   r%  r+   r   r   r   rb  r  r  unhealthy_deploymentsr   r!  s           rh   _get_healthy_deploymentszRouter._get_healthy_deploymentss  s    !#	"&"J"J #K #A *D1	 2
 !:$(;K!
 %'* 	7J,'-1FF#**:6		7 #$444  		s   &A" "	A.-A.c                   K   g }	 | j                  |      \  }}t        |t              rg |fS 	 t	        | |       d{   }g }|D ]  }|d   d   |v r|j                  |         ||fS # t        $ r Y Iw xY w7 <w)z
        Returns Tuple of:
        - Tuple[List[Dict], List[Dict]]:
            1. healthy_deployments: list of healthy deployments
            2. all_deployments: list of all deployments
        rR  r  Nr  r  )r  r   r   r%  r)   r   r  s           rh   r  z%Router._async_get_healthy_deployments  s      "$	"&"J"J #K #A *D1+++ 2
 'F$(;K'
 !
 %'* 	7J,'-1FF#**:6		7
 #$444  		!
s3   A?(A. A? A=-A?.	A:7A?9A::A?c                 r    t         j                  D ]$  }t        |t              s|j	                  |       & y)a  
        Mimics 'async_routing_strategy_pre_call_checks'

        Ensures consistent update rpm implementation for 'usage-based-routing-v2'

        Returns:
        - None

        Raises:
        - Rate Limit Exception - If the deployment is over it's tpm/rpm limits
        N)r   r  r   r   r  )r   r!  r  s      rh   rJ  z'Router.routing_strategy_pre_call_checks  s1     !** 	5I)\2((4	5rg   r~  c           
      |  K   t         j                  D ].  }t        |t              s	 |j	                  ||       d{    0 y7 # t         j
                  $ r}|t        j                  |j                  |t        j                         t        j                                      t        j                  |j                  |t        j                         f      j                          t!        | |j"                  ||d   d   | j$                         |d}~wt&        $ r}|t        j                  |j                  |t        j                         t        j                                      t        j                  |j                  |t        j                         f      j                          |d}~ww xY ww)O  
        For usage-based-routing-v2, enables running rpm checks before the call is made, inside the semaphore.

        -> makes the calls concurrency-safe, when rpm limits are set for a deployment

        Returns:
        - None

        Raises:
        - Rate Limit Exception - If the deployment is over it's tpm/rpm limits
        NrY  traceback_exceptionra  targetr  r  r  r  )r   r  r   r   async_pre_call_checkr  rn  ro  async_failure_handlerrr  rs  ri  	threadingThreadfailure_handlerstartr,   r  r   r%  )r   r!  rb  r~  r  r'  s         rh   r  z-Router.async_routing_strategy_pre_call_checks  s    " !** *	I)\2(#88EUVVV*	 W-- ".++'==*+4=4H4H4J)- >  "((#.#>#>"#Y%9%9%;!<  %'-04)*+,#-l#;D#A)-);); G  ".++'==*+4=4H4H4J)- >  "((#.#>#>"#Y%9%9%;!<  %'GsF   $F<AAAF<AF9B<DF9"BF44F99F<re  c           
        K   |}t         j                  D ]2  }t        |t              s	 |j	                  |||||       d{   }4 |S 7 # t
        $ r}	|t        j                  |j                  |	t        j                         t        j                                      t        j                  |j                  |	t        j                         f      j                          |	d}	~	ww xY ww)r  r   r   r)  re  rb  Nr  r  )r   r  r   r   async_filter_deploymentsr%  rn  ro  r
  rr  rs  ri  r  r  r  r  )
r   r   r   r)  rb  re  r~  returned_healthy_deploymentsr  r'  s
             rh   !async_callback_filter_deploymentsz(Router.async_callback_filter_deployments  s     ( (;$ ** 	I)\2'@@"'0L%-+9-= A   1	8 ,+1 ! ".++'==*+4=4H4H4J)- >  "((#.#>#>"#Y%9%9%;!<  %'Gs;   &C1AAAC1A	C.BC))C..C1r   c                    |}|j                         D ]  \  }}t        |t              r||z  }n7t        |t              r|t	        j
                  |      z  }n|t        |      z  }t        |t              r||z  }it        |t              r|t	        j
                  |      z  }|t        |      z  } t        j                  |j                               }|j                         S )z
        Helper function to consistently generate the same id for a deployment

        - create a string from all the litellm params
        - hash
        - use hash as id
        )
r  r   r   r   jsondumpshashlibsha256encode	hexdigest)r   r  r   
concat_strr  r  hash_objects          rh   _generate_model_idzRouter._generate_model_id(  s     !
"((* 	%DAq!S!a
At$djjm+
c!f$
!S!a
At$djjm+
c!f$
	% nnZ%6%6%89$$&&rg   deployment_info_model_name_litellm_paramsr  c           
         t        di ||t        di ||d}|j                  j                  }|j                  j                  |j                  j                  dz   |z   }t        j                  ||i       | j                  |      dur3t        j                  d|j                   d|j                  d	           y| j                  |      }|j                  d
      }| j                  j                  |       |S )a^  
        Create a deployment object and add it to the model list

        If the deployment is not active for the current environment, it is ignored

        Returns:
        - Deployment: The deployment object
        - None: If the deployment is not active for the current environment (if 'supported_environments' is set in litellm_params)
        )rM  r   r  Nr  )
model_costr7  TzIgnoring deployment z% as it is not active for environment supported_environmentsr   rf   )rB   rD   r   r   rV  r   register_model$deployment_is_active_for_environmentr   warningrM  r  _add_deploymentto_jsonrr   r   )r   r  r  r   r  r!  r   s          rh   _create_deploymentzRouter._create_deploymentD  s(       

")<O<"	

 !//55$$88D))==CkQ  	[	
 44
4KSWW!))&z'<'<&==bcmcxcx  zR  dS  cT  U ))Z)@
"""5u%rg   c           	      P   |j                   d|j                   vs|j                   d   yt        d      }|t        d      |t        vrt        dt         d|       |j                   d   D ]"  }|t        vst        dt         d| d	|        ||j                   d   v ryy
)a  
        Function to check if a llm deployment is active for a given environment. Allows using the same config.yaml across multople environments

        Requires `LITELLM_ENVIRONMENT` to be set in .env. Valid values for environment:
            - development
            - staging
            - production

        Raises:
        - ValueError: If LITELLM_ENVIRONMENT is not set in .env or not one of the valid values
        - ValueError: If supported_environments is not set in model_info or not one of the valid values
        r#  TLITELLM_ENVIRONMENT)secret_namezPSet 'supported_environments' for model but not 'LITELLM_ENVIRONMENT' set in .envz#LITELLM_ENVIRONMENT must be one of z. but set as: z&supported_environments must be one of z for deployment: F)r  r   r  r=   )r   r!  litellm_environment_envs       rh   r%  z+Router.deployment_is_active_for_environmentv  s     !!)'z/D/DD$$%=>F,9NO&b  &@@56P5QQ_`s_tu  ))*BC 	D55 <=W<XXfgkfll}  I  ~J  K 	 *"7"78P"QQrg   c                    t        j                  |      }g | _        |D ]
  }|j                  d      }|j                  d      }t	        |t
              rI|j                         D ]6  \  }}t	        |t              s|j                  d      s)t        |      ||<   8 |j                  di       }d|vr| j                  ||      }	|	|d<   |j                  dd       8t	        |d   t              r%|d   D ]  }
|
|d<   | j                  ||||        | j                  ||||        t        j                  d| j!                                 |D cg c]  }|d   	 c}| _        y c c}w )	NrM  r   os.environ/r  r  organization)r  r  r   r  z
Initialized Model List )r   r   rr   rE  r   r   r  r   
startswithrY   r  rH  r   r)  r   r   get_model_namesrk   )r   rr   original_model_listr   r  r   r  r  r  _idorgr   s               rh   r   zRouter.set_model_list  s   "mmJ7 ) !	E))L1K#ii(89O/40+113 ;DAq!!S)all=.I-7]*; !&		, ;K ;&--k?K$'D!"">48D/J +>: C69ON3++(-$/(7$/	 ,  ''$) +$3 +	 ( 9!	F 	##'(<(<(>'?@	
 6@@AlO@@s   E,c                    dd l }| j                  j                  |j                  j                         |j                  j
                  (t        |dd       t        |d      |j                  _        |j                  j                  (t        |dd       t        |d      |j                  _        t        j                  |j                  j                  |j                  j                  dd             \  }}}}d|j                  v r{| j                  j                  |j                  |j                  d             |j                  j                   r/| j"                  j                  |j                  j                          |j                  j                  d	g       xs g }|D ]i  }|j                  d
i       }	dD ]P  }
|
|	v s|	|
   j%                  d      s|	|
   j'                  dd      }|j(                  j                  |d      |	|
<   R k |t        j*                  vrt-        d|       t/        j0                  | |j                  d             |S )Nr   rpmtpmrV  r   rV  r   Tr   dataSources
parameters)endpointr  r0  r>  zUnsupported provider - rd  r   )osr   r   r   r   r8  rs  r9  r   rW   rH  rM  r   add_patternr(  r  r  r   r2  replaceenvironprovider_listr%  r&   
set_client)r   r!  r?  r  rV  rP  r  data_sourcesdata_sourcer   	param_keyenv_names               rh   r'  zRouter._add_deployment  s<    	$$Z%>%>%D%DE
 %%))1
E40<,3J,FJ%%) %%))1
E40<,3J,FJ%%) $$++11 * 9 9 = =%t!
	
$ *''' ++%%z'9'9t'9'L $$''44;;J<Q<Q<T<TU "0044]BGM2' 	EK __\26F0 E	&6)+<+G+G+V%i088KH(*

x(DF9%	E	E g&;&;;56I5JKLL 	!++$(
0B0BPT0B0U	
 rg   c                    |j                   j                  | j                         v ry|j                  d      }| j                  j                  |       | j                  |       | j                  j                  |j                         |S )z
        Parameters:
        - deployment: Deployment - the deployment to be added to the Router

        Returns:
        - The added deployment
        - OR None (if deployment already exists)
        NTr   r7  )	r  r  rI  r(  rr   r   r'  rk   rM  )r   r!  _deployments      rh   add_deploymentzRouter.add_deployment  s       ##t'9'9';; !((d(;{+ 	
3 	
 5 56rg   c                 v   |j                   j                  xs d}| j                  |      }|x|j                  |j                  k(  ryd}t	        | j
                        D ]'  \  }}|d   d   |j                   j                  k(  s&|}) || j
                  j                  |       | j                  |       |S )z
        Add or update deployment
        Parameters:
        - deployment: Deployment - the deployment to be added to the Router

        Returns:
        - The added/updated deployment
        r>  r  Nr  r  r7  )r  r  get_deploymentr   r  rr   rE  rK  )r   r!  _deployment_model_id_deployment_on_routerremoval_idxr  r   s          rh   upsert_deploymentzRouter.upsert_deployment/  s      *4477=26:6I6I) 7J 7
 !,((,A,P,PP *.K'8 &
U&t,
0E0E0H0HH"%K& &##K0 	z2rg   r  c                     d}t        | j                        D ]  \  }}|d   d   |k(  s|} 	 || j                  j                  |      }|S y# t        $ r Y yw xY w)z
        Parameters:
        - id: str - the id of the deployment to be deleted

        Returns:
        - The deleted deployment
        - OR None (if deleted deployment not found)
        Nr  r  )r  rr   rE  r%  )r   r  deployment_idxr  r   r  s         rh   delete_deploymentzRouter.delete_deploymentQ  st     0 	%FCt$*!$	%	)**>: 		s   A 	AAr  c                     | j                   D ]n  }d|v sd|d   v s||d   d   k(  st        |t              rt        di |c S t        |t              r|c S t	        dj                  t        |                   y)l
        Returns -> Deployment or None

        Raise Exception -> if model found in invalid format
        r  r  zModel invalid format - {}Nrf   rr   r   r   rB   r%  r   r   )r   r  r   s      rh   rN  zRouter.get_deploymenth  s     __ 	YEu$|1D)Du\2488!%.)2E22#E:6$'(C(J(J4PU;(WXX	Y rg   model_group_namec                     | j                   D ]^  }|d   |k(  st        |t              rt        di |c S t        |t              r|c S t	        dj                  t        |                   y)rW  rM  zModel Name invalid - {}Nrf   rX  )r   rY  r   s      rh   "get_deployment_by_model_group_namez)Router.get_deployment_by_model_group_namey  sn     __ 	SE\"&66eT*%...z2 L#$=$D$DT%[$QRR	S rg   received_model_namec                      y r   rf   r   r!  r\  r  s       rh   get_router_model_infozRouter.get_router_model_info       	rg   c                      y r   rf   r^  s       rh   r_  zRouter.get_router_model_info  r`  rg   c                 
   |&| j                  |      }||j                  d      }|t        d      |j                  di       j                  dd      }|"|j                  di       j                  dd      }|}t	        j
                  |j                  di       j                  d	d
      t        di |j                  di             \  }}}	}	|dk(  r|t        j                  d       n|dk7  r|}| j                  j                  |      }
d|v rp|
n|
D ]i  }	 |j                  di       j                  d      |j                  di       j                  d      k(  r#|j                  di       j                  d	      } nk |j                  dj                  |            sdj                  ||      }n|}t	        j                  |      }|j                  di       }|j                  |       |S # t        $ r Y w xY w)a  
        For a given model id, return the model info (max tokens, input cost, output cost, etc.).

        Augment litellm info with additional params set in `model_info`.

        For azure models, ignore the `model:`. Only set max tokens, cost values if base_model is set.

        Returns
        - ModelInfo - If found -> typed dict with max tokens, input cost, etc.

        Raises:
        - ValueError -> If model is not mapped yet
        NrM  Tr   zDeployment not foundr  
base_modelr   r   r>  r   r   r{  zCould not identify azure model. Set azure 'base_model' for accurate max tokens, cost tracking, etc.- https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-modelsr   r  z{}/z{}/{}rR  rf   )rN  r   r  rH  r   rW   rD   r   r  r   router%  r2  r   get_model_infor   )r   r!  r\  r  rJ  rc  r   r  rV  r  potential_modelspotential_modelmodel_info_namer  user_model_infos                  rh   r_  zRouter.get_router_model_info  s/   & >--r-:K&(333F
344  ^^L"599,M
#(8"=AA,PTUJ -4,D,D..!126::7BG)QJNN;KR,PQ-
)#Q ')j.@!'' O !G+E#22889LMe| 0 <'7 
O	*..|R@DD '^^L"=AA$GH %4$7$78H"$M$Q$Q '%E "H
 -@ AB%nn-@%HO#O++/B
 %..r:/*! % s   A$G66	HHc                 ^    | j                   D ]  }d|v sd|d   v s||d   d   k(  s|c S  y)z
        For a given model id, return the model info

        Returns
        - dict: the model in list with 'model_name', 'litellm_params', Optional['model_info']
        - None: could not find deployment in list
        r  r  Nrr   )r   r  r   s      rh   rf  zRouter.get_model_info  sG     __ 	!Eu$|1D)D|,T22 L	! rg   c                 Z    | j                  |      }|y|d   }| j                  |      S )zT
        Return list of all models in the same model group as that model id
        )r  NrM  r  )rf  r  )r   r  r  rM  s       rh   get_model_groupzRouter.get_model_group  s>    
 ((B(/
-
""j"99rg   user_facing_model_group_namec                    d}d}d}d}| j                  |      }|y|D ]  }d}	d|v r|d   |k(  rd}	n!d|v r| j                  j                  |      d}	|	s9t        di |d   }
|
j                  }d}||j                  dd      }|"|j                  di       j                  dd      }|"|j                  di       j                  dd      }d}||j                  d	d      }|"|j                  di       j                  d	d      }|"|j                  di       j                  d	d      }	 t        j                  |
j                  
      }d\  }}	 t        j                  |
j                  |
j                        \  }}}}|0t        j$                  ||      }|g }t'        |ddddd|d|d
      }|t)        d||gd|}n||j*                  vr|j*                  j-                  |       |j                  dd      -|d   (|j.                  |d   |j.                  kD  r
|d   |_        |j                  dd      -|d   (|j0                  |d   |j0                  kD  r
|d   |_        |j                  dd      (|j2                  |d   |j2                  kD  r
|d   |_        |j                  dd      (|j4                  |d   |j4                  kD  r
|d   |_        |j                  dd      	 |d   du rd|_        |j                  dd      |d   du rd|_        |j                  dd      |d   du rd|_        |j                  dd      |d   
|d   |_        |j                  dd      ||j                  d      }|j                  d	d      ||j                  d	      }|	|d}||z  }||d}||z  } ||||_        |||_         |||_        |S # t        $ r d}Y w xY w# t        j                  j                  $ r8}t        j                  dj!                  t#        |                   Y d}~d}~ww xY w)z
        For a given model group name, return the combined model info

        Returns:
        - ModelGroupInfo if able to construct a model group
        - None if error constructing model group info
        Nr  FrM  Tr   r9  r  r8  rR  )r>  r>  r:  z.litellm.router.py::get_model_group_info() - {}r   r   )
r  
max_tokensmax_input_tokensmax_output_tokensinput_cost_per_tokenoutput_cost_per_tokenlitellm_providermodesupported_openai_paramssupports_system_messages)r  	providersrr  rs  rt  ru  "supports_parallel_function_callingsupports_visionsupports_function_callingrx  rf   )!r  r   re  rD   #configurable_clientside_auth_paramsrH  r   rf  r   r%  rW   rV  
exceptionsBadRequestErrorr   r  r   r   get_supported_openai_paramsModelMapInforE   rz  r   rr  rs  rt  ru  r{  r|  r}  rx  r9  r8  )r   r  ro  model_group_info	total_tpm	total_rpmr~  rr   r   is_matchr   _deployment_tpm_deployment_rpmr  r  rA  r  r'  rx  s                      rh   _set_model_group_infozRouter._set_model_group_info  sz    6:#'	#'	SW+((K(@
 `	-EH%%*=*L%''--k:F+Fe4D.EFN BB 0 .2O&"'))E4"8&"')),<b"A"E"EeT"R&"'))L""="A"A%"N .2O&"'))E4"8&"')),<b"A"E"EeT"R&"'))L""="A"A%"N"$33.:N:NO
 +1'M<4;4L4L(..(6(J(J51|Q !*1*M*M'\+' +2.0+)##%)&*)**+%1,C-1
  '#1 $ <$Zd$   '7'A'AA$..55lCNN#5t<H"#56B(99A%&89*;;< 9CCU8V$5NN#6=I"#67C(::B%&9:*<<= :DDW9X$6>>"8$?K$99A!"89&;;< =G.=$9 >>"94@L$::B!"9:&<<= >H/>$: NN#GN "#GHDPJN$GNN#4d;G"#45=7;$4NN#>EQ"#>?4GAE$>NN#<dCO"#<=I?I1@$< >>%.:?V&0nnU&;O>>%.:?V&0nnU&;O*$ !I_,	*$ !I_,	A`	-B '$'0 $$'0 $ 3>7 !D  M  "!
" %%55 %++DKKCPQFS s*   0 O,0O>,O;:O;>Q-QQc                     || j                   v rT| j                   |   }t        |t              r|}nt        |t              r|d   du ry|d   }ny| j	                  ||      S | j	                  ||      S )z
        For a given model group name, return the combined model info

        Returns:
        - ModelGroupInfo if able to construct a model group
        - None if error constructing model group info or hidden model group
        hiddenTNr   )r  ro  )r   r   r   r   r  )r   r  r  _router_model_groups       rh   get_model_group_infozRouter.get_model_group_info  s     $000))+6D$$&*#D$'>T)*.w-'--/-8 .   ))#+ * 
 	
rg   c                   K   t               }|j                  d      }g }g }| j                  |      }|y|D ]  }|j                  di       j                  d      }|d   j                  d      }	||	=|j	                  t
        j                  j                  j                  ||	|	             |j	                  t
        j                  j                  j                  ||	|	              ||z   }
| j                  j                  |

       d{   }|y|dt        |       }|t        |      d }d}|!|D ]  }t        |t              s|d}||z  } d}|!|D ]  }t        |t              s|d}||z  } ||fS 7 qw)z
        Returns current tpm/rpm usage for model group

        Parameters:
        - model_group: str - the received model name from the user (can be a wildcard route).

        Returns:
        - usage: Tuple[tpm, rpm]
        r  r  Nr  r  r  r   r   )r  r   r  )r  r   )rZ   r  r  rH  r   rH   r  r  r   r  r   async_batch_get_cacher  r   r   )r   r  r  r  tpm_keysrpm_keysrr   r   r  r  combined_tpm_rpm_keyscombined_tpm_rpm_valuestpm_usage_listrpm_usage_list	tpm_usager  	rpm_usages                    rh   get_model_group_usagezRouter.get_model_group_usage  s     
 ! ((K(@
 	E %		, ; ? ? EB+01A+B+F+F,M z]2OO##))00'#1 1  OO##))00'#1 1 	* !)8 3(,

(H(H& )I )
 #
 #*)@3x=)Q)@X)Q $(	%# #a% ($%	NI	# $(	%# #a% ($%	NI	#
 )##3#
s   DFF<F$F5F@   )maxsizec                 $    | j                  |      S )z
        Cached version of get_model_group_info, uses @lru_cache wrapper

        This is a speed optimization, since set_response_headers makes a call to get_model_group_info on every request
        )r  )r   r  s     rh   _cached_get_model_group_infoz#Router._cached_get_model_group_info)  s     ((55rg   c                 <  K   | j                  |      }||j                  |j                  }nd }||j                  |j                  }nd }||i S | j                  |       d {   \  }}i }|||xs dz
  |d<   ||d<   |||xs dz
  |d<   ||d<   |S 7 1w)Nr   x-ratelimit-remaining-tokenszx-ratelimit-limit-tokensx-ratelimit-remaining-requestszx-ratelimit-limit-requests)r  r9  r8  r  )r   r  r  	tpm_limit	rpm_limitcurrent_tpmcurrent_rpmreturned_dicts           rh   get_remaining_model_group_usagez&Router.get_remaining_model_group_usage4  s     <<[I',<,@,@,L(,,II',<,@,@,L(,,II!2I)-)C)CK)P#P [ <E q=M89 9BM45 >G q?M:; ;DM67 $Qs   A&B(B)2Bc                 ~  K   t        |t              rt        |d      rt        |j                  t              r|j                  j                  di        ||j                  d   d<   |j                  d   }d|vr?d|vr;|9| j                  |       d{   }|j                         D ]  \  }}|	|||<    |S 7 &w)a  
        Add the most accurate rate limit headers for a given model response.

        ## TODO: add model group rate limit headers
        # - if healthy_deployments > 1, return model group rate limit headers
        # - else return the model's rate limit headers
        r  r  zx-litellm-model-groupr  r  N)r   r   rr  r  r   r   r  r  )r   r1  r  r  remaining_usageheaderr  s          rh   r  zRouter.set_response_headersT  s      x+"23822D9##../CRH  ##$89' "*!8!89M!N /6HH4<NN+(,(L(L) # &5%:%:%< ;MFE(5:*62; #s   BB=B;B=2
B=rM  c                     g }| j                   D ]G  }d|v sd|d   v s|d   d   }||d   |k(  r|j                  |       4|7|j                  |       I |S )z\
        if 'model_name' is none, returns all.

        Returns list of model id's.
        r  r  rM  )rr   r   )r   rM  idsr   r  s        rh   rI  zRouter.get_model_idsx  st     __ 	#Eu$|1D)D<(.)eL.AZ.OJJrN'JJrN	# 
rg   model_aliasc                     g }| j                   D ]M  }||d   |k(  s|,t        j                  |      }||d<   |j                  |       =|j                  |       O |S )ze
        Return all deployments of a model name

        Used for accurate 'get_model_list'.
        rM  )rr   r   r   r   )r   rM  r  returned_modelsr   alias_models         rh   _get_all_deploymentszRouter._get_all_deployments  sp     68__ 	2E%%*=*K*"&--"6K0;K-#**;7#**51	2 rg   c                 h    | j                         }|g S g }|D ]  }|j                  |d           |S )zn
        Returns all possible model names for router.

        Includes model_group_alias models too.
        rM  )r  r   )r   rr   rk   r   s       rh   r3  zRouter.get_model_names  sI     ((*
I 	0Aq/	0rg   c           
      \   t        | d      rg }|!|j                  | j                  |             t        | d      r| j                  j	                         D ]d  \  }}t        |t              r|}n*t        |t              rt        d
i |}|d   du r<|d   }nC|j                  | j                  ||             f t        |      d	k(  rE| j                  j                  |      }|(|j                  |D cg c]  }t        d
i | c}       ||| j                  z  }|S |S yc c}w )z>
        Includes router model_group_alias'es as well
        rr   Nr  r   r  Tr   )rM  r  r   rf   )rr  rt  r  r   r  r   r   r   rJ   r  r   re  rC   rr   )	r   rM  r  r  model_value_router_model_name_model_valuepotential_wildcard_modelsr   s	            rh   r  zRouter.get_model_list  sJ    4&9;O%&&t'@'@J'@'WXt01040F0F0L0L0N ,K!+s32=*#K6'@'O;'O'1T9$1=g1F. #**11'9{ 2 & ?#q(,0,?,?,E,Ej,Q),8#**;TUa,1q1U !4??2&&"" Vs   :D)model_access_groupc                 $   ddl m}  |t              }| j                  |      }|ri|D ]d  }|j	                  di       j	                  dg       D ]=  }| ||k(  s|d   }||   j                  |       %|d   }||   j                  |       ? f |S )a  
        If model_name is provided, only return access groups for that model.

        Parameters:
        - model_name: Optional[str] - the received model name from the user (can be a wildcard route). If set, will only return access groups for that model.
        - model_access_group: Optional[str] - the received model access group from the user. If set, will only return models for that access group.
        r   r   r  r  r   rM  )collectionsr   r   r  rH  r   )r   rM  r  r   r   rr   r   groups           rh   get_model_access_groupszRouter.get_model_access_groups  s     	,#D)((J(?
 @UU<488"M @E)5 $66)*<J)%077
C%&|_
%e,33J?@@ rg   c                     | j                  |      }t        |      dk(  ry|j                  |g       }|D ]   }| j                  j	                  |        y y)zG
        Return True if model access group is a wildcard route
        )r  r   Fr  T)r  r  rH  r   re  )r   r  r   r  r   s        rh   )_is_model_access_group_for_wildcard_routez0Router._is_model_access_group_for_wildcard_route  sr     441 5 
 }"""#5r: 	E""(((7C	
 rg   c                     t        |       }i }g d}|D ]K  }||v r||   ||<   |dk(  s| j                  dk(  s%| j                  j                  j	                         ||<   M |S )a  
        Get router settings method, returns a dictionary of the settings and their values.
        For example get the set values for routing_strategy_args, routing_strategy, allowed_fails, cooldown_time, num_retries, timeout, max_retries, retry_after
        r   r   r   r   r}   r   r   r   r   r   r   r   r   )varsr   r  r  r  )r   	_all_vars_settings_to_returnvars_to_includevars        rh   get_settingszRouter.get_settings  s    
 J	 
 # 	YCi+4S>#C(..))-DD+/+D+D+Q+Q+V+V+X#C(	Y #"rg   c                    g d}g d}| j                         }|D ]  }||v rg||v rt        ||         }t        | ||       '|dk(  r1|d   ||   k7  r&| j                  ||   |j	                  di              t        | |||          nt        j                  dj                  |              t        j                  d| j                                 y)	z-
        Update the router settings.
        r  )r   r}   r   r   r   r   r   r   zSetting {} is not allowedzUpdated Router settings: N)r  r   r  r   rH  r   r   r   )r   r-  _allowed_settings_int_settings_existing_router_settingsr  _casted_values          rh   update_settingszRouter.update_settings,  s    


 %)$5$5$7! 	UC''-'$'s$4MD#}5 1156HIVTW[X22-3C[28** 73 3  D#vc{3%++,G,N,Ns,ST'	U( 	##&?@Q@Q@S?T$UVrg   c                    |d   d   }t        |      }|dk(  r1dj                  |      }| j                  j                  |d|      }|S |dk(  r|j	                  d      du r]| d	}| j                  j                  |d|      }|6	 t        j                  | |
       | j                  j                  |d|      }|S | d}| j                  j                  |d|      }|6	 t        j                  | |
       | j                  j                  |d|      }|S |j	                  d      du r[| d}| j                  j                  ||      }|5	 t        j                  | |
       | j                  j                  ||      }|S | d}| j                  j                  ||      }|5	 t        j                  | |
       | j                  j                  ||      }|S )a  
        Returns the appropriate client based on the given deployment, kwargs, and client_type.

        Parameters:
            deployment (dict): The deployment dictionary containing the clients.
            kwargs (dict): The keyword arguments passed to the function.
            client_type (str): The type of client to return.

        Returns:
            The appropriate client based on the given client_type and kwargs.
        r  r  r{  z{}_max_parallel_requests_clientTr  r  rb  r  rS  _stream_async_clientr>  _async_client_stream_client)r  rb  _client)r   r   r   r  rH  r&   rD  )r   r!  r-  r}  r  rb  	cache_keyr:  s           rh   rG  zRouter._get_client^  sd    l+D1+LV+T119@@JIZZ))$AQ * F MG#zz(#t+'j(<=	--!dEU .  > -7704J "ZZ11%#')9 2 F
 'j6	--!dEU .  > -7704J "ZZ11%#')9 2 F
 zz(#t+'j7	--!4D .  > -7704J "ZZ11%8H 2 F 'j0	--!4D .  > -7704J "ZZ11%8H 2 F rg   c           	      	   t        j                  d|        t        j                  |      }g }	 t	        j
                  |      }d}	d}
d}t        |      }t               }|j                  d      }| d| }| j                  j                  |d	|
      xs i }t        |      D ]  \  }}	 |j!                  di       j!                  dd      }|"|j!                  di       j!                  dd      }| j#                  ||      }|xs" |j!                  di       j!                  dd      }t%        |t&              rZ|j!                  dd      Ht%        |d   t(              r5||d   kD  r-|j+                  |       d	}	|
dj                  ||d   |      z  }
|j!                  di       }|j!                  di       j!                  dd      }| j                  j                  |d	|
      xs d}t%        |t&              r| j.                  dk7  rv|j!                  |d      ||<   t1        |||         }t%        |t&              rB|j!                  dd      0t%        |d   t(              r|d   |k  r|j+                  |       d	}|M|j!                  d      <|j!                  d      }|)t3        t5        d!i ||      s|j+                  |       +|/t        j6                  du sCt	        j8                  |t5        d!i |      \  }}}}t	        j:                  ||      }|t        j<                  j?                  |      }dg}|jA                         D ]8  \  }} ||vs||v st        j                  d|        |j+                  |       :  tC        |      tC        |      k(  r<	 |d	u rtE        |      |	d	u r't	        jF                  dj                  |
      |d       tC        |      dkD  r!tI        |      D ]  }|jK                  |        tC        |      dkD  rt        j<                  jM                  |      }|S # t        $ r9}t        j                  dj                  t        |                   |cY d}~S d}~ww xY w# t        $ r8}t        j,                  dj                  t        |                   Y d}~5d}~ww xY w)"a  
        Filter out model in model group, if:

        - model context window < message length. For azure openai models, requires 'base_model' is set. - https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models
        - filter models above rpm limits
        - if region given, filter out models not in that region / unknown region
        - [TODO] function call and model doesn't support function calling
        z2Starting Pre-call checks for deployments in model=)r)  zllitellm.router.py::_pre_call_checks: failed to count tokens. Returning initial list of deployments. Got - {}NFr>  r  z:rpm:Tr  r  rc  r   )r!  r\  r   rr  z%Model={}, Max Input Tokens={}, Got={}zAn error occurs - {}r  r   r   r8  allowed_model_region)r   r  rd  r:  )passed_paramsresponse_formatz1INVALID MODEL INDEX @ REQUEST KWARG FILTERING, k=rR  z~litellm._pre_call_checks: Context Window exceeded for given call. No models have context window large enough for this call.
{}r?  rf   )'r   r   r   r   r   token_counterr%  r  r   r   r   rZ   r  r   r  r  rH  r_  r   r   r   r   rY  r   maxr[   rD   drop_paramsrW   r  r  get_non_default_paramsr  r  rL   r  reversedrE  _get_order_filtered_deployments)!r   r   r   r)  re  _returned_deploymentsinvalid_model_indicesinput_tokensr'  _context_window_error_potential_error_str_rate_limit_errorrb  r  r  r  model_group_cacher  r!  rc  r  r   r  current_request_cache_localcurrent_requestr  rV  r  rx  r	  special_paramsr  r  s!                                    rh   _pre_call_checkszRouter._pre_call_checks  su     	##@H	
 !%.A B "	)"00(CL !&!!<^L W-G5 01JJ  ?O !   	 	  ))>? l	>OCW'^^L"=AA,PTU
%!+0@"!E!I!I$d"J "77)u 8 
 # jnn5Er&J&N&NT'
 z40"'94@L #:.@#A3G(:6H+II-44S904-,CJJ %z2D'E|,
 ! )nn-=rBO!~~lB7;;D"EH 

$$ TDT %   	 ( ,d3))-EE.?.C.CHa.P!(+"%/1B81L#
 5'++E48D #?5#93?+E2oE-44S9,0)  *"&&'=>J'5'9'9:P'Q$'3,'5'H'H-A .44S9  )g.A.AU.J3:3K3K0Q0Q40*Aq +2*M*M5H+' +2 *1)M)M&4 *N *& '8%8N 2 8 8 : >1$;;^@S177"STUSV W 288=>Ml	>\ $%-B)CC !D(/  '$.88 ^  e  e,  !#  $%) 56 /%))#./ $%)$+MM$Q$Q%%! %$G  	)!''~  F  FF
 )(	)l  W%//0F0M0McRSf0UVVWs6   Q /C(R	R.R	R	R	S-SSc                 t    || j                   vry| j                   |   }t        |t              r|}|S |d   }|S )z
        Get the model from the alias.

        Returns:
        - str, the litellm model name
        - None, if model is not in model group alias
        Nr   )r   r   r   )r   r   _items      rh   _get_model_from_aliaszRouter._get_model_from_aliasx  sK     ...&&u-eS!E  'NErg   c                 X    | j                   D cg c]  }|d   d   |k(  s| c}S c c}w )z6
        Get the deployment by litellm model.
        r   r   rl  )r   r   r   s      rh    _get_deployment_by_litellm_modelz'Router._get_deployment_by_litellm_model  s.      ??Taa0@.A'.Je.STTTs   ''r4  c                 r   |du r|| j                  |      fS || j                         v rY| j                  |      }|*|j                  j                  }||j                  d      fS t        d| d| j                         | j                  |      }||}|| j                  vrY| j                  j                  |      }|r||fS | j                  +t        j                  | j                        }	||	d   d<   ||	fS | j                  |	      }
t        |
      d
k(  r| j                  |      }
t!        j"                  d|
        t        |
      d
k(  r't%        j&                  dj)                  |      |d      t$        j*                  r%|t$        j*                  v rt$        j*                  |   }||
fS )aF  
        Common checks for 'get_available_deployment' across sync + async call.

        If 'healthy_deployments' returned is None, this means the user chose a specific deployment

        Returns
        - str, the litellm model name
        - List, if multiple models chosen
        - Dict, if specific model chosen
        TrR  rM  r   zBLiteLLM Router: Trying to call specific deployment, but Model ID :z6 does not exist in                     Model ID List: r   r   r  r   zinitial list of deployments: zBYou passed in model={}. There is no 'model_name' with this string r>  r?  )r  rI  rN  r   r   r   r  r  rk   r   get_deployments_by_patternr   r   r   r  r  r   r   r   r  r   model_alias_map)r   r   r)  r"  r4  r!  deployment_model_model_from_aliaspattern_deploymentsupdated_deploymentr   s              rh   r  z*Router._common_checks_available_deployment  s   $ $&$??e?LLLd((**,,e,<J%#-#<#<#B#B ')>)>D)>)QQQTUZT[ \$$($6$6#79 
 !66U6C(%E((("&"5"5"P"P #Q # #111 &&2%)]]++&" AF"#34W=000 #7757I"#q("&"G"Ge"G"T##+,?+@A	
 "#q())\cc   ""u0G0G'G++E )))rg   c           
      \  K   | j                   dk7  rR| j                   dk7  rC| j                   dk7  r4| j                   dk7  r%| j                   dk7  r| j                  |||||      S 	 t        |      }| j                  ||||      \  }}t	        |t
              r|S t        | |       d	{   }t        j                  d
|        t        j                  d|        | j                  ||      }| j                  |||t        t        t           |      nd	||       d	{   }| j                  r,|*| j                  |t        t        t            |      ||      }t#        | |||       d	{   }t%        |      dk(  rt'        | ||       d	{   }	|	t)        j(                         }
| j                   dk(  r4| j*                  (| j*                  j-                  ||||       d	{   }n| j                   dk(  r4| j.                  (| j.                  j-                  ||||       d	{   }n| j                   dk(  r5| j0                  )| j0                  j-                  |||||       d	{   }n`| j                   dk(  rt3        | ||      S | j                   dk(  r2| j4                  &| j4                  j-                  ||       d	{   }nd	}|t'        | ||       d	{   }	|	t        j6                  d| d| j9                  |       d|        t)        j(                         }||
z
  }t;        j<                  | j>                  jA                  tB        jD                  |d||
|             |S 7 7 _7 7 7 7 ]7 7 7 # tF        $ r}tI        jJ                         }|j|jM                  dd	      }|VtO        jP                  |jR                  ||f      jU                          t;        j<                  |jW                  ||             |d	}~ww xY ww)z
        Async implementation of 'get_available_deployments'.

        Allows all cache calls to be made async => 10x perf impact (8rps -> 100 rps).
        r   rq   r   r   r   )r   r)  r"  r4  re  r   r)  r"  r4  r  Nzasync cooldown deployments: zcooldown_deployments: r   cooldown_deploymentsr  r   r   r)  re  )llm_router_instancer   re  r   r   )rd  r   rb  r  r   r)  r"  )r  r   r)  r"  re  r  r   r   r  r   $get_available_deployment for model: , Selected deployment:  for model: z2<routing_strategy>.async_get_available_deployments)r^  r_  r   rb  r`  ra  rz  r  ),r   rD  r   r  r   r   r)   r   r   _filter_cooldown_deploymentsr  r   r
   r8   r   r  r	   r#   r  r1   ri  r  async_get_available_deploymentsr   r  r"   rn   r   r(  rn  ro  r   rp  rN   rq  r%  rr  rs  rH  r  r  r  r  r
  )r   r   r)  r"  r4  re  rb  r   r  rY  r`  r!  ra  ru  r'  r  r~  s                    rh   rx  z%Router.async_get_available_deployment  s     !!%==%%)99%%)==%%)@@%%500!$7- 1  Z	@P)-)Q)Q!$7	 *R *&E& -t4**)H(,?O* $  "''./C.DE "''*@AU@V(WX"&"C"C$7%9 #D #
 )-(N(N$7  + ./:-!1 )O 
) 
# **x/C&*&;&;(,T$Z9L(M%#1	 '< '# )@$(-$7	) # &'1,"E,0%5# 	
  J%%)AA,,8 22RR$),?!)#	 S    %%)==**6 00PP$),?!)#	 Q    %%)@@--9 33SS$),?!)#'5 T    &&*::%(,(;  %%5))5 //OO$),? P    "
!"E,0%5# 	
  !&&6ug=TUYUjUjkuUvTw  xD  EJ  DK  L yy{H :-I''BB(//&R%5)% C 	 }$
#(#&0  	"+"6"6"8),001FM*$$*::!45 eg''#99!=PQ G!	s   A"P,%4N P,N *N +A9N $N%AN 3N4#N N	AN /N0AN 2N3AN 6N7!N P,;N NN /N0BN ?P, N N N 	N N N N N N 	P)"BP$$P))P,c                    | j                  ||||      \  }}t        |t              r|S t        |      }t	        | |      }| j                  ||      }| j                  r|| j                  ||||      }t        |      dk(  rU| j                  |      }	| j                  j                  |	|      }
t	        | |      }t        ||
| j                  |	      | j                  d
k(  r*| j                  | j                  j                  ||      }n| j                  dk(  rt!        | ||      S | j                  dk(  r+| j"                  | j"                  j                  |||      }nx| j                  dk(  r,| j$                   | j$                  j                  ||||      }n=| j                  dk(  r,| j&                   | j&                  j                  ||||      }nd}|nt)        j*                  d| d       | j                  |      }	| j                  j                  |	|      }
t	        | |      }t        ||
| j                  |	      t)        j*                  d| d| j-                  |       d|        |S )zB
        Returns the deployment based on routing strategy
        r  r  r  Nr  r   r  )	model_idsrb  )r   r   r   cooldown_listr   r  rq   r  r   )r  r   re  r   r  r   r  z, No deployment availabler  r  )r  r   r   r   r+   r  r   r  r  rI  r   get_min_cooldownrK   r   rn   get_available_deploymentsr"   r  ro   r  r   r   r(  )r   r   r)  r"  r4  re  r   rb  r  r  _cooldown_time_cooldown_listr!  s                rh   rD  zRouter.get_available_deployment  s8    &*%M%M 3	 &N &
"" )40&&+L,
  9$(;K 
 #?? 3!5 @ 
 &&8+?"&"7"7$7!-	 #8 # "#q(**e*<I!00AA#6F B N 7(,?ON ','+'B'B,	    L0T5J5J5V..HH!7J I J ""&66 "$($7  !!%<<))522LL!$7- M J !!%::%%1..HH!$7!	 I J !!%==((411KK!$7!	 L J J!&&6ug=VW **e*<I!00AA#6F B N 7(,?ON ','+'B'B,	  	""25'9PQUQfQfgqQrPss  AF  @G  H	
 rg   r  c                     g }t        j                  d|        |D ]   }|d   d   }||v s|j                  |       " |D ]  }|j                  |        |S )a  
        Filters out the deployments currently cooling down from the list of healthy deployments

        Args:
            healthy_deployments: List of healthy deployments
            cooldown_deployments: List of model_ids cooling down. cooldown_deployments is a list of model_id's cooling down, cooldown_deployments = ["16700539-b3cd-42f4-b426-6a12a1bb706a", "16700539-b3cd-42f4-b426-7899"]

        Returns:
            List of healthy deployments
        zcooldown deployments: r  r  )r   r   r   r  )r   r   r  deployments_to_remover!  r  s         rh   r  z#Router._filter_cooldown_deployments  s|     !###&<=Q<R$ST- 	9J&|4T:M 44%,,Z8	9 0 	3J&&z2	3""rg   c                     	 |j                  di       j                  dd      }||| j                  ||       yyy# t        $ r+}t        j                  dt        |              Y d}~yd}~ww xY w)z7
        Tracks successful requests rpm usage.
        r  r  Nz$Error in _track_deployment_metrics: )rH  r  r%  r   r  r   )r   r!  rb  r1  r  r'  s         rh   r  z Router._track_deployment_metrics.  s    
	Y!~~lB7;;D$GH '&& "2 (    	Y!''*NsSTvh(WXX	Ys   8= 	A1!A,,A1rY  c                 H    t        ||| j                  | j                        S )N)rY  r  r   r   )"_get_num_retries_from_retry_policyr   r   )r   rY  r  s      rh   r0   z(Router.get_num_retries_from_retry_policy@  s)     2#%)%B%B**	
 	
rg   c                    | j                   }|yt        |t        j                        r|j                  |j                  S t        |t        j
                        r|j                  |j                  S t        |t        j                        r|j                  |j                  S t        |t        j                        r|j                  |j                  S t        |t        j                        r|j                  |j                  S yy)a  
        BadRequestErrorRetries: Optional[int] = None
        AuthenticationErrorRetries: Optional[int] = None
        TimeoutErrorRetries: Optional[int] = None
        RateLimitErrorRetries: Optional[int] = None
        ContentPolicyViolationErrorRetries: Optional[int] = None
        N)r   r   r   r  BadRequestErrorAllowedFailsr  AuthenticationErrorAllowedFailsr  TimeoutErrorAllowedFailsr  RateLimitErrorAllowedFailsrL  'ContentPolicyViolationErrorAllowedFails)r   rY  r   s      rh   get_allowed_fails_from_policyz$Router.get_allowed_fails_from_policyJ  s     >B=V=V' y'"9"9:$@@L'CCCy'"="=>$DDP'GGGy'//2$==I'@@@y'"8"89$??K'BBBy'"E"EF$LLX'OOO Y Grg   c                 J   ddl m} | j                  y | j                  } ||j                  dg|j                        }|| _        t        j                  j                  |       t        j                  j                  |j                         t        j                  d       y )Nr   )SlackAlertingslack)alerting_thresholdalertingdefault_webhook_urlz2[94m
Initialized Alerting for litellm.Router[0m
)1litellm.integrations.SlackAlerting.slack_alertingr  r   r  webhook_urlslack_alerting_loggerr   r  r   r   !response_taking_too_long_callbackr   r   )r   r  router_alerting_config_slack_alerting_loggers       rh   r   zRouter._initialize_alertingr  s    S'151E1E!.5HHY 6 B B"
 &<"  !78  ''"DD	
 	""H	
rg   CustomRoutingStrategyc                 `    t        | d|j                         t        | d|j                         y)a  
        Sets get_available_deployment and async_get_available_deployment on an instanced of litellm.Router

        Use this to set your custom routing strategy

        Args:
            CustomRoutingStrategy: litellm.router.CustomRoutingStrategyBase
        rD  rx  N)r  rD  rx  )r   r   s     rh   set_custom_routing_strategyz"Router.set_custom_routing_strategy  s4     	&!::	

 	,!@@	
rg   c                 N    d t         _        | j                  j                          y r   )r   r   flush_cacher	  s    rh   r$  zRouter.flush_cache  s    

 rg   c                     g t         _        g t         _        g t         _        g t         _        d | _        | j                          y r   )r   r   r   r   r   r   r$  r	  s    rh   resetzRouter.reset  s6    #% *,'#% *,' rg   )F)FFF)T)rx  r  )NNNNr   )NNNNN)NNF)NNFN)rb   rc   rd   rk   r
   __annotations__rl   r   r  rm   r   tenacityrn   r   ro   r    rI   r   rC   r	   r   r   r@   r   r  floatr   rJ   rG   r?   rF   rO   r>   r   r   r   r   rM   r   r(  rT   rR   r2  r.  r   r8   rV  rh  r/  r  rF  r  r  r  r  r  r  r  rl  r   r   r  rj  rk  r  r  r  r  r;   r  r  r%  r*  r(  r-  r3  r5  r6  r=  r<  rS   rB  r@  rI  rH  r:   rN  rM  r9   r]  r\  rd  rm  rw  r   r   r}  rm  r  r.  r  r  r%  r  r0  r  r  r   r   r   r   r  r^   r  r  rK  r   r  rJ  r
  r  r  r  rB   r)  r%  r   r   r'  rK  rR  rU  rN  r[  r  r_  rf  rn  rE   r  r  r  r   r  r  r  rI  r  r3  r  r  r  r  r  rG  r  r  r  r  rx  rD  r  r  r0   r  r   rA   r"  r$  r&  rf   rg   rh   rj   rj      s   K&+OXd^+&11H:>h67>:>h67> ;?#'$($((,*/ ,0*.%) #'*. 7;!06 )+)+ ',%*     ,0 DH&(DH48 "#Y\
$*+T$sCx.-AAB
\
 $$78\
 C=\
 SM\
 SM\
 !\
 "$\
 \
 !K
\
" #\
& #5/'\
( #3-)\
, c]-\
.  
/\
4 %5\
6 !7\
8 !)!
9\
> (0}?\
@ A\
B _-C\
D $I
E\
J K\
L #'M\
N #'O\
P $eC!::;;<
Q\
V !%W\
X #Y\
Z [\
\ +t#$
]\
b #'#
c\
h  
i\
n '
o\
t  
u\
z $D>{\
| "&
}\
L #++@"AM\
N  $O\
P !))@ AQ\
R ".1S\
T "*!"
U\
Z 
[\
|	F$ 8(01F(G8"? %os&: ;?TX?B4 ($(c3h$8	}11	2&@@$(c3h$8@	}11	2@H $()9$:DKDM	 
 UZ$()9$:DKEN	 
 kp$()9$:DI'RV-Y`afYgJgDh	"M1	2  	33 '(3 	3j{{$(c3h$8{	}11	2{z	IS 	I$ 	I4 	I
$ 
$$ 
$G Gt GPT G( t 4

"&
	%s
#	$
"&	%s
#	$$	4 	t 	uczAR8S 	Z#S	Z# T#s(^,d4S#X3G.HHIZ#x++$(.>)?$@+^ $(c3h$8BI$-	  SX$(c3h$8BI%.	  	K,K, tCH~&K, 	K,b di$()9$:FISZ[`Sa	 
 $()9$:FISZ[_S`	  << '(< 	<|;; ; $	;
 CHo; S#X;z  $BBBB '(BB S#X	BBHs 3 , ,C ,\c # *Es E3 EN( (3 (TC) CC CJS3 Ss S3 Sj3 *(C (Tc H $)&+#(## # 4.	#
 d^# 4.#R $)&+#(## # 4.	#
 d^# 4.#JCS C# CR $)&+#(  4.	
 d^ 4.:CS C CR $)	 S$Y 4.	 
$6c4i 0 6 6x $(	 S$Y 4.	 
2CuS$Y'7 C CL 
	0OO 
	Ob 
	0UU 
	Un? 
?B//f
1#
1 :F# 56: EI(,	
#
 &g.?&@A
 %	
,U%t &*$(373711 c]1 D>	1
 #+4.1 #+4.1f%B   :>)1#8 /3*.3737,0?? &d^? "$	?
 #+4.? #+4.? $D>?BW &*$S$s)^,-$ c]$ 
$s)		$< /3*.// / 	/
 &d^/ "$/ 
sEz	/fPd! 
#!F< 
<| 
+3C= 
D     t  D 4<TN	6 ##$1#;?#	#J5c 5Xd^ 5.55,4TN5	tDz4:%	&5<54 5( 15	;; #4.; n-	;F *.041,1, "$Z1, 4 012	1,
 #4.1, !1, n-1,f'c '4 '800 0 	0
 0 
*	0d&z &d &P+A +AZI* I IV 8L 2 J  8J;O  DC HZ,@ .s x
/C " #	*	$ EI58>B	 
 58>A	  !	KTNK !K SM	K
 
KZ $ 
:# 
:(4. 
:A A >AA 	.	!A F
 
8P 
>G$G$	x}hsm+	,G$R r66	.	!6 6 cSVh B ;?""*23-"	"H c " =A,4SM	!	"(c   +/-"3--	$*+	,-` UY"3-DLSM	c49n	8"%	.#>0WdY@ *.}%}% "}% tCH~&	}%
 !}%~3 8C= &Uc Ud U 48,0.3L*L* 4S#X/0L* c4i()	L*
 &d^L* 
sE$*%%	&L*b 48,0.3)-uu 4S#X/0u c4i()	u
 &d^u !ut 48,0.3)-{{ 4S#X/0{ c4i()	{
 &d^{ !{z##':#EI#Y#	d#8 FJY,4TNY& BF
"
19#
&Py &PP
0
%>
.!rg   rj   )rn  r   enumr  r  r  r   r  ri  rr  r  r  r   	functoolsr   typingr   r   r   r	   r
   r   r   r   r   r   httpxr   r   pydanticr   typing_extensionsr   r   litellm.litellm_core_utils2litellm.litellm_core_utils.exception_mapping_utilsr   litellm._loggingr   litellm.caching.cachingr   r   r   "litellm.integrations.custom_loggerr   #litellm.litellm_core_utils.asyncifyr   'litellm.litellm_core_utils.core_helpersr   *litellm.litellm_core_utils.litellm_loggingr   r
  &litellm.router_strategy.budget_limiterr   "litellm.router_strategy.least_busyr   #litellm.router_strategy.lowest_costr   &litellm.router_strategy.lowest_latencyr   &litellm.router_strategy.lowest_tpm_rpmr    )litellm.router_strategy.lowest_tpm_rpm_v2r!   &litellm.router_strategy.simple_shuffler"   )litellm.router_strategy.tag_based_routingr#    litellm.router_utils.batch_utilsr$   r%   /litellm.router_utils.client_initalization_utilsr&   #litellm.router_utils.cooldown_cacher'   &litellm.router_utils.cooldown_handlersr(   r)   r*   r+   r,   ,litellm.router_utils.fallback_event_handlersr-   r.   r/   *litellm.router_utils.get_retry_from_policyr0   r  !litellm.router_utils.handle_errorr1   r2   Dlitellm.router_utils.pre_call_checks.prompt_caching_deployment_checkr3   >litellm.router_utils.router_callbacks.track_deployment_metricsr4   r5   litellm.schedulerr6   r7   litellm.types.llms.openair8   r9   r:   r;   litellm.types.routerr<   r=   r>   r?   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   litellm.types.servicesrN   litellm.types.utilsrO   rP   r  rQ   litellm.utilsrR   rS   rT   rU   rV   rW   rX   rY   rZ   r[   &router_utils.pattern_match_deploymentsr]   opentelemetry.tracer^   _SpanEnumr`   rj   rf   rg   rh   <module>rS     s2              #        &  ! 9 " 2 H H ; B U P G F H N J P A M U =  
 2 T T    ( 0 7 9 6   G1DD$)) [X [Xrg   