
    )#h                    <   U d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlmZmZ d dlmZmZ d dlmZ d dlmZ d dl m!Z!m"Z"m#Z# d dl$Z$d dl%Z%d dl&Z&d dl'Z'd dl(Z(d dl&m)Z) d dl*m+Z+ d d	l,m-Z-m.Z. d d
l/m0Z0 d dl1m2Z2 d dl(m3Z3 d dl4m5Z5 d dl6Z6d dl7Z6d dl8Z6d dl9Z6d dl:Z6d dl;m<Z< d dl=m>Z> d dl?m@Z@mAZA d dlBmCZC d dlDmEZEmFZF d dlGmHZH d dlImJZJmKZKmLZL d dlMmNZNmOZO d dlPmQZQ d dlRmSZS d dlTmUZUmVZVmWZWmXZXmYZY d dlZm[Z[ d dl\m]Z] d dl^m_Z_ d dl`maZa d dlbmcZcmdZd d dlemfZf d dlgmhZh d d limjZjmkZk d d!llmmZmmnZn d d"lompZpmqZq d d#lrmsZs d d$ltmuZu d d%lvmwZwmxZxmyZymzZzm{Z{m|Z|m}Z} d d&l~mZ d d'lmZ d d(lmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ  ej8                  d)d*      5 Z e
j<                  e      Zddd        e
j@                  e      Zd dlZd d+lmZ d d,lmZmZmZmZmZmZmZmZmZmZmZmZmZ d d-l'mZ d d.lmZ d d/lmZmZ d d0lmZ d d1lmZ d d2lmZ d d3lmZ d d4lmZ d5d6lmZmZ d5d7lmZmZmZmZmZ d5d8lmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ d5d9lmZmZ d5d:lmZmZmZ d5d;lmZ d<Z ee=      ZdZdZdadZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZdZg a eee      ed><   dai Zeeeef      ed?<   i Zeeeef      ed@<   dZdZ	 	 d%dAedBedC   fdDZ	dE Z
dFefdGZdHecdIefdJZdKeeef   dLedMeee   eeef   f   fdNZdO Z	 d&dKee   dPedMefdQZdRedHecdSee   dKedTej
                  dUej
                  dMdfdVZ	 d'dSedWee   fdXZ edYZ      dSefd[       Zd(dWee   fd]Zd\g dfd^ee   dWee   fd_Z	 	 	 	 	 	 	 	 d)d`ee   daee   dbee   dceee{      ddeez   deee   dfee   fdgZ	 d*dhediee   fdjZdkefdlZdm Zdn Zdo Z	 	 	 	 	 	 	 	 d+dWee   daeeeee   f      d`ee   deee   dceee{      ddeez   dfee   dMefdpZdqedMefdrZ dSedqee   dMefdsZ!	 d'dSedqee   dMefdtZ"	 d'dSedqee   dMefduZ#dSedqee   dvedMefdwZ$d'dSedqee   dMefdxZ%d'dSedqee   dMefdyZ&	 d'dSedqee   dMefdzZ'	 d'dSedqee   dMefd{Z(d'dSedqee   dMefd|Z)	 d'dSedqee   dMefd}Z*dSefd~Z+dededMefdZ,deeef   fdZ-	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d,dee   dqee   dee   dee   dee   dee   dee   dee   dee   dee   dee   dee   dee   dee   dMefdZ.dMefdZ/dededee   dMefdZ0	 	 	 	 	 	 	 d-dSedee   dee   dee   dee   deeed         dqee   dee   fdZ1	 	 	 	 	 	 	 	 	 d.dSee   dee   dee   dee   dee   dee   dee   dqee   dee   fdZ2	 	 	 	 	 	 d/dSedee   dee   dee   dee   dee   fdZ3d Z4d Z5dedeee      dMefdZ6dedqedMefdZ7	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d0dSed`eeew      fdZ8dedMefdZ9dee   dee   dee   dee   dMee   f
dZ:dee   dMefdZ;dqededMee   fdZ<dedMee   fdZ=dedMefdZ>dedMefdZ?dededMefdZ@dedee   dMee   fdZAdKedMefdZBdaedMefdZCdedMefdÄZDdĐedee   fdƄZEdǄ ZFdSedMee   fdȄZGdMefdɄZHdMefdʄZIdːedMefd̄ZJdSedqee   dMefd̈́ZKdvedMefd΄ZLdϐedqee   dMefdЄZMd dlmNZN  G d҄ dӐeN      ZOdSedqee   dMeOfdԄZPdːedMee   fdՄZQ e<d֬Z      dSedqee   dMefdׄ       ZR	 d'dSedqee   dMefd؄ZSd'dSedqee   dMefdلZTdڐefdۄZUd܄ ZVd݄ ZW	 	 	 	 	 d1dSedqedededސedߐefdZXdqedMee   fdZYd ZZ	 	 	 d2dSee   dee   dee   dMefdZ[d Z\d Z]d Z^dSedefdZ_defdZ`	 d'dee&j                     fdZb	 	 d3dededee&j                     dedMeecef   f
dZd	 	 d4dSedededefdZe G d d      Zf	 d'dee   fdZg	 d'dee   fdZhdMefdZid Zjd Zkd Zld Zmd ZndSee   fdZo	 	 	 	 d5dSee   decd efdZpd&dedMee   fdZqd Zrd Zsdee   dMee   fdZtd'dZu G d d	      Zv G d
 d      Zw G d de      ZxdewdMefdZydedMefdZzdedMefdZ{d`eew   dMefdZ|dqedMee{   fdZ}d dlvm~Z~mZmZmZmZmZ dee2ef   dMefdZd`eew   fdZd`eew   fdZddeeeef      dMeeeef      fdZ G d d      Z	 d6deded   dMee   fdZ	 	 d7dSed`eeew      dceee{      dqee   dMef
dZd edMeec   fd!Zd dl&Z&ded"edMefd#ZdKedMefd$Zy# 1 sw Y   	xY w(8      N)	dataclassfield)	lru_cachewraps)	resources)iscoroutine)abspathdirnamejoin)Proxy)get_environment_proxies)_parsing	_pydantic)ResponseFormat)	BaseModel)Encoding)	Tokenizer)lru_cache_wrapper)	DualCache)CachingHandlerResponseLLMCachingHandler)CustomLogger)map_finish_reasonprocess_response_headers)encoding)_get_response_headersexception_typeget_error_message)_is_non_openai_azure_modelget_llm_provider)get_supported_openai_params)_ensure_extra_body_is_safe)LiteLLMResponseObjectHandler#_handle_invalid_parallel_tool_calls convert_to_model_response_objectconvert_to_streaming_response#convert_to_streaming_response_async)get_api_base)get_formatted_prompt)get_response_headers)ResponseMetadata)LiteLLMLoggingObject(redact_message_input_output_from_logging)Rules)CustomStreamWrapper)calculate_img_tokensget_modified_max_tokens)AsyncHTTPHandlerHTTPHandler)!get_num_retries_from_retry_policyreset_retry_policy)
get_secret)ANTHROPIC_API_ONLY_HEADERS)AllMessageValuesAllPromptValuesChatCompletionAssistantToolCall"ChatCompletionNamedToolChoiceParamChatCompletionToolParam$ChatCompletionToolParamFunctionChunkOpenAITextCompletionUserMessage)RerankResponse)	FileTypes)OPENAI_RESPONSE_HEADERS	CallTypesChatCompletionDeltaToolCallChatCompletionMessageToolCallChoicesCostPerTokenCustomHuggingfaceTokenizerDelta	EmbeddingEmbeddingResponseFunctionImageResponseLlmProvidersLlmProvidersSetMessage	ModelInfoModelInfoBaseModelResponseModelResponseStreamProviderFieldStreamingChoicesTextChoicesTextCompletionResponseTranscriptionResponseUsageall_litellm_paramsz%litellm.litellm_core_utils.tokenizerszanthropic_tokenizer.json)ThreadPoolExecutor)TYPE_CHECKINGAnyCallableDictIterableListLiteralOptionalTupleTypeUnioncastget_args)OpenAIError)BaseAudioTranscriptionConfig)BaseLLMModelInfotype_to_response_format_param)
BaseConfig)BaseTextCompletionConfig)BaseEmbeddingConfig)BaseImageVariationConfig)BaseRerankConfig   )_is_debugging_onverbose_logger)CacheQdrantSemanticCache
RedisCacheRedisSemanticCacheS3Cache)APIConnectionErrorAPIErrorAuthenticationErrorBadRequestErrorBudgetExceededErrorContentPolicyViolationErrorContextWindowExceededErrorNotFoundErrorri   PermissionDeniedErrorRateLimitErrorServiceUnavailableErrorTimeoutUnprocessableEntityErrorUnsupportedParamsError)AllowedModelRegionKeyManagementSystem) ChatCompletionDeltaToolCallChunkChatCompletionToolCallChunk#ChatCompletionToolCallFunctionChunk)LiteLLM_Paramsd   )max_workerscallback_listadditional_detailslocal_cachelogger_only	log_level)DEBUGINFOERRORc                 
   	 |dk(  rt        j                  |        n5|dk(  rt        j                  |        n|dk(  rt        j                  |        t        j
                  du r|du rt        |        y y y # t        $ r Y y w xY w)Nr   r   r   TF)rt   debuginfoerrorlitellmset_verboseprint	Exception)print_statementr   r   s      J/var/www/html/sandstorm/venv/lib/python3.12/site-packages/litellm/utils.pyprint_verboser   !  s    

  1& 0'!  1$&;%+?/" ,@& s   A1A6 6	BBc                     t         j                  D ]q  } | d   t         j                  vr"t         j                  j                  | d          | d   t         j                  vsPt         j                  j                  | d          s y)z2
    Add custom_llm provider to provider list
    providerN)r   custom_provider_mapprovider_listappend_custom_providers)
custom_llms    r   custom_llm_setupr   5  ss     11 E
j!)>)>>!!((J)?@j!)B)BB%%,,Z
-CDE    original_functionc                    ddl m} ddlm} t         j                  du rt        j                  d       	 t                d|v r|d   nd }t        t         j                        dkD  rt         j                  D ][  t        t              rQt         j                  j                  j                  d d       "t        fdt         j                   D              ret         j"                  vrt         j"                  j%                         t         j&                  vrt         j&                  j%                         t         j(                  vrt         j(                  j%                         t         j                   vrt         j                   j%                         t         j*                  vs=t         j*                  j%                         ^ t-        d	t         j                           t        t         j"                        dkD  s8t        t         j&                        dkD  st        t         j(                        dkD  rdt        t.              dk(  rRt1        t3        t         j"                  t         j&                  z   t         j(                  z               a |t.        |
       t        t         j"                        dkD  rg }t5        t         j"                        D ]K  \  }	t7        j8                        st         j:                  j%                         |j%                  |	       M t=        |      D ]!  }	t         j"                  j?                  |	       # t        t         j&                        dkD  rg }t5        t         j&                        D ]  \  }	t7        j8                        r1t         j                   j%                         |j%                  |	       Mdk(  sdk(  r1t         j                   j%                         |j%                  |	       t         j@                  v sddlm}
  |
d d       t        fdt         j                  D              rt         j                  j%                         t         j"                  j%                         t         j&                  j%                         t         j(                  j%                         t         j                   j%                         t         j*                  j%                          t=        |      D ]!  }	t         j&                  j?                  |	       # t        t         j(                        dkD  rg }t5        t         j(                        D ]K  \  }	t7        j8                        st         j*                  j%                         |j%                  |	       M t=        |      D ]!  }	t         j(                  j?                  |	       # d }d }d }d }|jC                  dd       t        |d   t0              rg }t5        |d         D ]]  \  }	t7        j8                        sdk(  sdk(  s&|"t        |t0              r|j%                         ng}|j%                  |	       _ t=        |      D ]  }	|d   j?                  |	        |j?                  d      }|jC                  dd       $t        |d   t0              r|j?                  d      }tD        rp	 tG        jH                  |      }t         jL                  r6|j?                  dd        |j?                  dd        |j?                  dd        tE        dd| d| d       d|v r|d   a't        |      dkD  r|d   n|jC                  dd       }| }|tP        jR                  jT                  k(  s|tP        jV                  jT                  k(  rd }t        |      dkD  r|d   }n|jC                  dd       r|d   }t        |t0              rt        |      dkD  rt        |d   tX              rd|d   v r|j[                  dj]                  d |D              |        n|tP        j^                  jT                  k(  s|tP        j`                  jT                  k(  r't        |      dkD  r|d   n|jC                  dd       }n4|tP        jb                  jT                  k(  s|tP        jd                  jT                  k(  rt        |      dkD  r|d   n|d   }n|tP        jf                  jT                  k(  s|tP        jh                  jT                  k(  rt        |      dkD  r|d   n|d   }n|tP        jj                  jT                  k(  s|tP        jl                  jT                  k(  rt        |      dkD  r|d   n|d   }n8|tP        jn                  jT                  k(  s|tP        jp                  jT                  k(  r|jC                  d!      }n|tP        jr                  jT                  k(  s|tP        jt                  jT                  k(  rct        |      dkD  r|d   n|d"   }t         j                  jv                  jx                  j{                  |#      }d$|v r	||d$   d%<   nd%|i|d$<   |}nO|tP        j|                  jT                  k(  s|tP        j~                  jT                  k(  r|jC                  dd&      }nd'}d(|v r	|d(   du rdnd)} |||||d*   |jC                  d+      |xs d|||||||,      }d-di}d$|v r|d$   |d$<   |j                  |di ||jC                  d.d       /       ||fS # tJ        $ r |}Y ow xY w# tJ        $ r7}t        j                  d0t        j                          d1| d2|        |d }~ww xY w)3Nr   )Logging)set_callbacksTze`litellm.set_verbose` is deprecated. Please set `os.environ['LITELLM_LOG'] = 'DEBUG'` for debug logs.id)internal_usage_cache
llm_routerc              3   H   K   | ]  }t        |t                      y wN
isinstancetype).0cbcallbacks     r   	<genexpr>z!function_setup.<locals>.<genexpr>\  s$      / #2tH~6/   "z8Initialized litellm callbacks, Async Success Callbacks: )r   function_iddynamodb	openmeter)$_init_custom_logger_compatible_classc              3   H   K   | ]  }t        |t                      y wr   r   )r   r   callback_classs     r   r   z!function_setup.<locals>.<genexpr>  s"      >AC
2tN';<>r   success_callbacks3failure_callbackmessagesinputpromptzlitellm.llm_callzPositional Args: z, Keyword Args: r   )categorymessagelevel	logger_fnmodelrr   content c              3   n   K   | ]-  }d |v r't        |d    t              r|j                  d d       / yw)r   r   Nr   strget)r   ms     r   r   z!function_setup.<locals>.<genexpr>  s7      "$>j9s.K i,"s   35r   r   queryfile)file_objmetadatafile_checksumspeechzdefault-message-valuestreamFlitellm_call_idlitellm_trace_id)r   r   r   r   r   r   	call_type
start_timedynamic_success_callbacksdynamic_failure_callbacksdynamic_async_success_callbacksdynamic_async_failure_callbackskwargsapi_basestream_options)r   useroptional_paramslitellm_paramsr   z4litellm.utils.py::function_setup() - [Non-Blocking] z	; args - z; kwargs - )Dr   r   *litellm.litellm_core_utils.litellm_loggingr   r   rt   warningr   len	callbacksr   r   litellm_core_utilslitellm_loggingr   any_async_success_callbackinput_callbackr   r   r   _async_failure_callbackr   r   listset	enumerateinspectiscoroutinefunction_async_input_callbackreversedpop)_known_custom_logger_compatible_callbacksr   add_breadcrumbcopydeepcopyr   turn_off_message_logginguser_logger_fnrB   
completionvalueacompletiondictpre_call_rulesr   	embedding
aembeddingimage_generationaimage_generation
moderationamoderationatext_completiontext_completionrerankarerankatranscriptiontranscriptionaudio_utilsutilsget_audio_file_nameaspeechr   update_environment_variablesr   	traceback
format_exc)r   	rules_objr   argsr   LiteLLMLoggingr   r   removed_async_itemsindexr   r   r   r   r   details_to_logr   r   r   	_file_objr   r   logging_objr   er   r   s                            @@r   function_setupr$  A  s
    2Hd"s	
N 	 6:V^VD\w  !A%#-- Eh,&99IInn t  o  H  '3 /")"A"A/ , !7#9#99**11(;7#;#;;,,33H=7#;#;;,,33H=7#B#BB33::8D7#B#BB33::8D)E* J7KjKjJkl
 &&'!+7++,q07++,q0
 !**../../M ;Ow%%&*"$#,W-C-C#D 6x..x81188B'..u56 ""56 2&&**512w''(1,"$#,W-E-E#F Ox..x833::8D'..u5+x;/F 33::8D'..u5!R!RR &J -1#'&N &1# >GNGXGX> ;  ))00@..55nE0077G0077G77>>~N77>>~N;O@ ""56 4((,,U34 w''(1,"$#,W-E-E#F 6x..x833::8D'..u56 ""56 4((,,U34
  	"
  	(
  	"
  	( ::($/;
%&A
 #%#,V4F-G#H 6x//9:-4'6Bz7H 8>>xH;C*7'..u56 ""56 6)*..u56(.

3E(F%::($/;
%&A
 )/

3E(F%(!%v!6 //"":t4""7D1""8T2++D61A.AQR
 & #K0Nt9q=Qfjj$.G%	--333I11777H4y1}7J-!*- 8T*MA%x{D1!,(('' "!)" 
   )  ,,222I00666"%d)a-tAwVZZ5NH33999I77==="%d)a-tAwVH5EH--333I11777"%d)a-tAwVG_H33999I55;;;"%d)a-tAwVH5EH))///9	@Q@Q@W@W3Wzz'*H11777I33999.1$i!m47I**66<<PP& Q  
 V#6Cz"?3&5}%Ez"$H**000IAQAQAWAW4Wzz'84H.H!V+x0@D0He$"#45#ZZ(:;#)r!&?&?,K,K
" +5b)9)/
);N:&00)!::&6= 	1 	
 F""g  (!'(h  B9CWCWCYBZZcdhciitu{t|}	
 	sl   E=n! 6En! ?D3n! 3n! "n! *D>n! )Cn! -B+n! n .Q n! nn! nn! !	o!*2oo!r"  is_completion_with_fallbacksc                    K   |du rSt        d| j                          t        j                  | j                  |||             | j	                  |||       y y w)NFz>Async Wrapper: Completed Call, calling async_success_handler: resultr   end_time)r   async_success_handlerasynciocreate_task-handle_sync_success_callbacks_for_async_callsr"  r(  r   r)  r%  s        r   _client_async_logging_helperr/  ]  so      	%-L[MnMnLop	
 	--fj(K	
 	AA! 	B 	
 	.   AAr   	exceptionreturnc                     | j                  dd      }|t        j                  }| j                  dd      r-t        || j                  d            }t	               | d<   ||}|| fS )zm
    Get the number of retries from the kwargs and the retry policy.
    Used for the wrapper functions.
    num_retriesNretry_policyr1  r5  )r   r   r4  r4   r5   )r   r1  r4  retry_policy_num_retriess       r   _get_wrapper_num_retriesr8  u  sr     **]D1K))zz.$'#DN3$
  "4!5~#/2Kr   c                      t               dt        fddt        t           f fdt	                fd       }t	                fd       }t        j                         }|r|S |S )Nr2  c                 \    t        j                  |       ryt        j                  |       ryyNTF)r   r   r   )r  s    r   check_coroutinezclient.<locals>.check_coroutine  s'    u%((/r   r   c                 T   	 | y 	j                   }|t        j                  j                  k(  s|t        j                  j                  k(  r |       }|du ry t        | t              rt        | j                        dkD  r| j                  d   j                  j                  }|Z
j                  ||       t        j                  du r	 |d|v r|d   	 d }t        |d   t              r|d   j                  d      	 |d   }n1t         j"                  j%                  |d         rt'        |d         }|1t        j(                  j*                  j-                  |d   d   |       |d|v rt        |d   t              rxd	|d   v rp|d   d	   d
k(  rdd|d   v r\t        |d   d   t              rEd|d   v r=|d   d   du r2t        j(                  j*                  j-                  |d   d   |       y y y y y y y y y y y y y y # t.        $ r Y w xY w# t0        $ r}|d }~ww xY w)NTr   r   response_formatjson_schemar>  schema)rA  responser   json_objectresponse_schemaenforce_validation)__name__rB   r  r  r  r   rR   r   choicesr   r   post_call_rulesr   enable_json_schema_validationr  r   r   _completionsis_basemodel_typerl   r   json_validation_rulevalidate_schema	TypeErrorr   )original_responser   r   r   is_coroutinemodel_responsejson_response_formatr#  r<  r   r  s           r   post_call_processingz$client.<locals>.post_call_processing  s   d	 (-66	!5!5!;!;; I$9$9$?$??#23D#EL#t+ ''8-H #$5$=$= > B<M<U<U !=%ggg +  .9 ) 9 9*8 !: !" $+#H#HD#P'%-,;,G0A_0T0?@Q0R3718 TX,@0:4CDU4V4812 5D4E5225#m2D7;5<
 HW4EH20D 2:1F1F1X1X0?@Q0R2. 5RHW<MI:56 1E 0D/O070J0J0_0_0o0o;O8E<66><@ >L	 1p 12 %4$?(9_(L(2(78I(JD)& )//BS2T(T(78I(J6(R'4)5(9'67H'I)J(2(78I(J,=)* )-	)& )='67H'I)J(78I(J(<)& (,), %,$>$>$S$S$c$c/>?P/Q,=0* 2@	 %d %&),)J)&)J)5 )U)& )M %@a  : !C I @t ,5 %-(,%-D  	G	sJ   H AH A5H B H 0B
H 	HH HH 	H' H""H'c                     j                   }t        |      rt        j                  K|j	                  di       j	                  dd       }|'t        j                  t        |      k  rt        d       | i |}d|v ra|d   du rZd|v rT|d   du rMg }t        |      D ]  \  }}|j                  |        t        j                  ||j	                  dd             S |S |S t        | |       t        j                  j                         }d }|j	                  d	d       }	d
|vr t        t        j                               |d
<   t        |       dkD  r| d   n|j	                  dd       }
	 |	t!        j                   |g| i |\  }	}|	|d	<   t#        ||      }||	_        |j'                         D ]C  \  }}|	t)        |t              s|j+                  d      s,t        j,                  |      ||<   E t        j.                  rJt        j0                  t        j.                  kD  r)t3        t        j0                  t        j.                        t        j                  K|j	                  di       j	                  dd       }|'t        j                  t        |      k  rt        d      t5        d|j	                  dd       dt        j6                   d|j	                  di       j	                  dd              |j	                  dd       t        j6                  |j	                  dd      du r|j	                  di       j	                  dd      dur|j	                  dd      dur|j	                  dd      dur|j	                  dd      dur|j	                  dd      dur|j	                  dd      durq|j	                  dd      dur]|j	                  dd      durIt9        j:                  d       |j=                  |
xs d|	||||        }|j>                  |j>                  S |j	                  d!d       |
t        j@                  du r|tB        jD                  jF                  k(  s|tB        jH                  jF                  k(  r}	 |
}|j	                  d"d       d#|j	                  d"       }d }t        |       d$kD  r| d$   }n|j	                  dd       r|d   }|j	                  d!      }tK        |
|||d d %      }||d!<    | i |}t        j                  j                         }d|v rr|d   du rkd|v rT|d   du rMg }t        |      D ]  \  }}|j                  |        t        j                  ||j	                  dd             S tM        ||	|
|||'       |S d|v r	|d   du r|S d|v r	|d   du r|S d|v r	|d   du r|S d|v r	|d   du r|S d(|v r	|d(   du r|S tO        jP                  |      r|S  ||
xs d |)       |jS                  || |*       t9        jT                  d+       tV        jY                  |	jZ                  |||       tM        ||	|
|||'       |S # t        $ r"}t5        d&t        |              Y d }~d }~ww xY w# t        $ r}j                   }|tB        jH                  jF                  k(  r]|j	                  d,d       xs t        j\                  xs d }|j	                  d-d       r)t_        ||j	                  d-      .      }ta               |d-<   d t        _.        |j	                  d/i       }d0|j	                  di       v }|ro|smt)        |tb        jd                        s4t)        |tb        jf                        st)        |tb        jh                        rw||d,<   t        jj                  | i |cY d }~S t)        |t        jl                  jn                        r4|r2|
|v r.|s,t        |       dkD  r	||
   | d<   n||
   |d<    | i |cY d }~S tq        jr                         }t        j                  j                         }|	r|	ju                  ||||       |d }~ww xY w)1Nr   previous_modelszMax retries per request hit!r   Tcomplete_responser   r   litellm_logging_objr   r   r   r   request_kwargsr   zos.environ/current_cost
max_budgetzSYNC kwargs[caching]: cachingF; litellm.cache: z#; kwargs.get('cache')['no-cache']: cachezno-cacher	  r  r  aimg_generationr  r  
_arealtimezINSIDE CHECKING SYNC CACHEr   r   r   r"  r   r   r   r  
max_tokenshf_model_namezhuggingface/rr   )r   
base_modelr   user_max_tokens
buffer_numbuffer_percz&Error while checking max token limit: r(  r"  r   r   r   r)  r  rO  r   r   )r(  r  r   z0Wrapper: Completed Call, calling success_handlerr4  r5  r6  context_window_fallback_dictmodel_group);rF  _is_async_requestr   num_retries_per_requestr   r   r   r   r   stream_chunk_builderprint_args_passed_to_litellmdatetimenowr   uuiduuid4r$  r   _llm_caching_handleritemsr   
startswithr6   r]  _current_costr~   r   r`  rt   r   _sync_get_cachecached_resultmodify_paramsrB   r  r  r  r1   update_response_metadatar+  r   sync_set_cacher   executorsubmitsuccess_handlerr4  r4   r5   openair{   r   rz   completion_with_retries
exceptionsr   r  r  failure_handler)r  r   r   rU  r(  chunksidxchunkr   r"  r   rv  kvcaching_handler_responserf  r   rg  modified_max_tokensr#  r)  r4  rl  _is_litellm_router_calltraceback_exceptionr   rS  r  s                            r   wrapperzclient.<locals>.wrapper  s4	    &..	V$..:"(**Z"<"@"@%t# #.66#o:NN'(FGG '77F6!fX&6$&>'6123t;F&/&7 -
Ue,-"77J)E  "MM 	%%6fE&&**,
6<jj!47

 F*(+DJJL(9F$%*-d)a-tAwVZZQU=Vg	"&4%..	:'HL'PV'#V -8F()6G"3%%7 
 0DK,  61=Z3%7ALL<W ' 2 21 5F1I6 !!((7+=+==-%,%:%:#*#5#5  ..:"(**Z"<"@"@%t# #.66#o:NN'(FGG (Iu)E(FFWX_XeXeWf  gJ  KQ  KU  KU  V]  _a  Kb  Kf  Kf  gq  sx  Ky  Jz  { #JJy$7? ' 9!::i74?

7B/33JFdRJJ|U34?JJ159EJJ}e4D@JJ0%8DJJ/7tCJJy%0<JJ|U34? $$%AB(88#kr*;$/#-"+%! 9  ) ,99E3AAA 

<.:%)) !6!6!<!<< I$8$8$>$>>U!&Jzz/48D'3FJJ4O3P%Q
#H4y1}#'7J5#)*#5&,jj&>O*A##-!)(7#'$(+' ,?F<( '77F((,,.H6!fX&6$&>'6123t;F&/&7 -
Ue,-"77J)E 
 -%$/#%#-!) "M&(VM-Bd-J'F<,@D,H"f,8I1Jd1R!V+7G0HD0Pf$	):d)B$$V, !"(mt & !// 0   RSOO++	 %'%! MO ! U!$J3q6("STTUP  5	)22II00666JJ}d3Rw7J7JRd  ::nd3"C"#%+ZZ%?#K
 +, >*  # 06zz2B0, +86::< +'  (? #1foo6%a8%a)B)BC0;}-&>>OOOq'"4"4"O"OP4!==34y1}">u"EQ*Fu*Mw,d=f=="+"6"6"8((,,.H ++*J Gk5	s   ,AZ ?Z Z "IZ 2A Z A<Y0 BZ Z +Z 8Z Z Z Z ,Z A,Z 0	Z9ZZ ZZ 
a>(D)a9a>Aa9)a>/A
a99a>c                  
  K   t        | |       t        j                  j                         }d }|j                  dd       }t	        ||      }j
                  }d|vr t        t        j                               |d<   t        |       dkD  r| d   n|j                  dd       }|j                  d      d u}	 |t        j
                  |g| i |\  }}||d<   ||_        t        j                  rJt        j                  t        j                  kD  r)t        t        j                  t        j                        t!        d|j                  d	d
       dt        j"                   d|j                  dd               |j%                  |xs d|||||        d {   }	|	j&                  |	j(                  |	j&                  S |	j*                  du r|	j(                  S  | i | d {   }t        j                  j                         }
d|v rr|d   du rkd|v rT|d   du rMg }t-        |      D ]  \  }}|j/                  |        t        j0                  ||j                  dd             S t3        ||||||
       |S |t4        j6                  j8                  k(  r|S  |||       |j;                  |||        d {    t=        j>                  tA        ||||
|             |jC                  |||
       tE        |tF              r"|	j(                  	 |jI                  |	|||
      S t3        ||||||
       |S 7 7 _7 # tJ        $ r}tM        jN                         }t        j                  j                         }
|r[	 |jQ                  ||||
       n# tJ        $ r}|d }~ww xY w	 |jS                  ||||
       d {  7   n# tJ        $ r}|d }~ww xY wj
                  }tU        ||      \  }}|t4        jV                  j8                  k(  r|j                  di       }d|j                  di       v }|r|s	 d t        _,        ||d<   |d <   tE        |tZ        j\                        rd!|d"<   ntE        |tZ        j^                        rd#|d"<   t        j`                  | i | d {  7  cY d }~S # tJ        $ r Y ncw xY wtE        |t        jb                  jd                        r;|r9||v r5t        |       dkD  r	||   | d<   n||   |d<    | i | d {  7  cY d }~S tg        |d|       |d }~ww xY ww)$NrX  rY  r   r   r   	fallbacksr[  zASYNC kwargs[caching]: r^  Fr_  z; kwargs.get('cache'): r`  r   rc  Tr   rV  r   rW  rj  rk  )r(  r   r   r  r.  r'  )_caching_handler_responseembedding_responser   r)  )r   r1  rl  rm  r   r4  r   exponential_backoff_retryretry_strategyconstant_retry)4rq  rr  rs  r   r   rF  r   rt  ru  r   r$  rv  r   r]  ry  r~   r   r`  _async_get_cacher{  final_embedding_cached_response embedding_all_elements_cache_hitr   r   rp  r}  rB   	arealtimer  async_set_cacher+  r,  r/  r-  r   rJ   2_combine_cached_embedding_response_with_api_resultr   r  r  r  async_failure_handlerr8  r  r4  r  r   r{   acompletion_with_retriesr  r   setattr)r  r   r   r(  r"  rv  r   r   r%  r  r)  r  r  r  r#  r  r4  rl  r  r   rS  r  s                      r   wrapper_asynczclient.<locals>.wrapper_async  s    $%6fE&&**,
6<jj!47
 3D/!!3
 &..	F*(+DJJL(9F$%*-d)a-tAwVZZQU=V'-zz+'>d'J$o	"&4%..	:'HL'PV'#V -8F()/CK,!!((7+=+==-%,%:%:#*#5#5  )&**Y*F)GGXY`YfYfXgg~  @F  @J  @J  KR  TX  @Y  Z  [ +;;+2&7 +)'! <   & *77C-MMU0>>>*KKtS0PPP -d=f==F((,,.H6!fX&6$&>'6123t;F&/&7 -
Ue,-"77J)E  -%$/#%#-!) "Mi11777 "(v
 '66"3	 7    , +!)%1M EE%! F  6#45-MM ,^^.G'-)%	 _   %'%! M( >>Z  ;	"+"6"6"8((,,.H//.
H ! G%;;.
H   ! G *22I":&TU"VKI11777/5zz2B0, +86::< +'  (?   + 1<}-6G23%v44 8SF#34'6??;7GF#34%,%E%Et%Vv%VVVV$  q'"4"4"O"OP4!==4y1}">u"EQ*Fu*Mw!2D!CF!CCCC=+ Gw;	sV  B6U:C'M !M"'M 	U
M #U$M /M0BM 4U5M UM &U'#M 
MA,M 7U8M 
UM M M 
U
4UN'&U'	N70N22N77U;OOOU	O)"O$$O))A'UA,S=S >SU
U	SUSAU+T.,U0U
1U6UU

U)r.   boolrc   r  r   r   r   )r   r  r  rP  r<  rS  r  s   `   @@@r   clientr    s    I$ eQU eN V Vp C CJ ../@AL r   is_pass_throughc                    | y| j                  dd      du s| j                  dd      du s| j                  dd      du s| j                  dd      du s|| j                  dd      du sh| j                  dd      du sT| j                  d	d      du s@| j                  d
d      du s,| j                  dd      du s| j                  dd      du s|du ryy)aj  
    Returns True if the call type is an internal async request.

    eg. litellm.acompletion, litellm.aimage_generation, litellm.acreate_batch, litellm._arealtime

    Args:
        kwargs (dict): The kwargs passed to the litellm function
        is_pass_through (bool): Whether the call is a pass-through call. By default all pass through calls are async.
    Fr  Tr	  ra  r  r  r  r  rb  acreate_batchacreate_fine_tuning_jobr   )r   r  s     r   rn  rn    s     ~

=%(D0::lE*d2::'/47::mU+t3::(%0D8::&.$6::i'4/::lE*d2::ou-5::/74?d"r   r(  r   r   r)  c                     | yt        |       }|j                  |||       |j                  |||       |j                          y)z
    Updates response metadata, adds the following:
        - response._hidden_params
        - response._hidden_params["litellm_overhead_time_ms"]
        - response.response_time_ms
    N)r"  r   r   )r   r)  r"  )r+   set_hidden_paramsset_timing_metricsapply)r(  r"  r   r   r   r)  r   s          r   r}  r}    sQ     ~'H;eFSk    NNr   custom_tokenizerc                 P    |t        |d   |d   |d         }|S t        |       S )N
identifierrevision
auth_token)r  r  r  r   )create_pretrained_tokenizer_select_tokenizer_helper)r   r  
_tokenizers      r   _select_tokenizerr    s?     #0'5%j1'5


 #%00r      )maxsizec                    | t         j                  v rd| v rt        j                  d      }d|dS | t         j                  v r"d| vrt        j
                  t              }d|dS d| j                         v sd| j                         v rt        j                  d      }d|dS d	| j                         v rt        j                  d
      }d|dS dt        dS )Nz	command-rz#Xenova/c4ai-command-r-v01-tokenizerhuggingface_tokenizerr   	tokenizerzclaude-3zllama-2	replicatez#hf-internal-testing/llama-tokenizerzllama-3zXenova/llama-3-tokenizeropenai_tokenizer)	r   cohere_modelsr   from_pretrainedanthropic_modelsfrom_strclaude_json_strlowerr   )r   cohere_tokenizerclaude_tokenizerr  s       r   r  r  +  s    %%%+*>$441
 0>NOO	'**	*z/F$--o>/>NOO	ekkm	#{ekkm'C--.ST	/iHH	ekkm	#--.HI	/iHH '!
 	
r   r   c                     |xs t        |       }t        |d   t              r|d   j                  |d      }|S |d   j                  |      }|S )a  
    Encodes the given text using the specified model.

    Args:
        model (str): The name of the model to use for tokenization.
        custom_tokenizer (Optional[dict]): A custom tokenizer created with the `create_pretrained_tokenizer` or `create_tokenizer` method. Must be a dictionary with a string value for `type` and Tokenizer for `tokenizer`. Default is None.
        text (str): The text to be encoded.

    Returns:
        enc: The encoded text.
    r  r   disallowed_special)r  r   r   encode)r   textr  tokenizer_jsonencs        r   r  r  G  s`     &G):)GN.-x8[)00"0M J [)006Jr   tokensc                 N    |xs t        |       }|d   j                  |      }|S )Nr  r  )r  decode)r   r  r  r  decs        r   r  r  [  s,    %G):)GN

%
,
,V
4CJr   r   r  is_tool_calltoolstool_choicecount_response_tokensuse_default_image_token_countc           
         t        d|        	 d|v rt        j                  d      }nt        j                  |      }|dk(  rd}	d}
n=|t
        j                  v rd	}	d
}
n&|t
        j                  v rd	}	d
}
nt        d| d      d}d}|r!|t         |j                  |d            }n| P| D ]I  }||	z  }|j                  dd      dk(  rd}|j                         D ]  \  }}t        |t              r,|t         |j                  |d            z  }|dk(  s=||
z  }Ct        |t              sT|D ]  }|d   dk(  r,||d   z  }|t         |j                  |d   d            z  }7|d   dk(  s@t        |d   t               r>|d   }|j                  dd      }|j                  d      }|t#        |||xs d      z  }t        |d   t              s|d   }|t#        |d|xs d      z  }  L nD|#|du rt         |j                  |d            }|S |t         |j                  |d            }|d	z  }|r,|t         |j                  t%        |                  z  }|dz  }|r|r|dz  }|dk(  r|d
z  }|S t        |t               r)|dz  }|t         |j                  |d    d               z  }|S # t        $ r$ t        d       t        j                  d      }Y w xY w)!z
    Return the number of tokens used by a list of messages.

    Borrowed from https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb.
    z2LiteLLM: Utils - Counting tokens for OpenAI model=zgpt-4o
o200k_basez5Warning: model not found. Using cl100k_base encoding.cl100k_basezgpt-3.5-turbo-0301      rr   z8num_tokens_from_messages() is not implemented for model zz. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens.r   FNr  r  rolesystemTnamer   r  	image_urldetailautourldatamoder  	   none   function)r   tiktokenget_encodingencoding_for_modelKeyErrorr   open_ai_chat_completion_models
azure_llmsNotImplementedErrorr   r  r   rw  r   r   ra   r  r0   _format_function_definitions)r   r   r  r  r  r  r  r  r   tokens_per_messagetokens_per_name
num_tokensincludes_system_messager   keyr  cimage_url_dictr  r  image_url_strs                        r   openai_token_counterr  a  sx   " FugNO8u,,\:H2259H $$ 	 	'88	8	'$$	$!H  PJ  M
 	
 J#("EF
		 "	"G,,J{{64(H4*.'%mmo "
UeS)#ohooePR&S"TTJf}"o5
t," "V9. AfI-D&# /&	b Q+ J vY+5)!K.$?12;)7););Hf)M&4&8&8&? *.B),)/B_ C-',	/" !"
 ",AkNC!@01+ *.B)6)/B_ C-',	/" !"
'""	"	"F 
	3t;"EF
		"EF
!OJc/(//*Fu*MNOO
a
(a
 fa

 	 
K	&a
c/(//+j*A&*IJKK
g  8MN((78s   /K )K43K4r  r  c                     	 t        j                  | ||      }d|dS # t        $ r:}t        j                  d| d       t        j                  | |      }Y d}~Cd}~ww xY w)a  
    Creates a tokenizer from an existing file on a HuggingFace repository to be used with `token_counter`.

    Args:
    identifier (str): The identifier of a Model on the Hugging Face Hub, that contains a tokenizer.json file
    revision (str, defaults to main): A branch or commit id
    auth_token (str, optional, defaults to None): An optional auth token used to access private repositories on the Hugging Face Hub

    Returns:
    dict: A dictionary with the tokenizer and its type.
    )r  r  z%Error creating pretrained tokenizer: z-. Defaulting to version without 'auth_token'.)r  Nr  r  )r   r  r   rt   r   )r  r  r  r  r#  s        r   r  r    so    M--j
	 ,)DD  M3A36cd	
 --j8L		Ms    	A"0AA"jsonc                 6    t        j                  |       }d|dS )z
    Creates a tokenizer from a valid JSON string for use with `token_counter`.

    Args:
    json (str): A valid JSON string representing a previously serialized tokenizer

    Returns:
    dict: A dictionary with the tokenizer and its type.
    r  r  )r   r  )r  r  s     r   create_tokenizerr    s     ""4(I+)DDr   c                 h   g }|j                  d       |j                  d       | D ]  }|j                  d      }|j                  d      x}r|j                  d|        |j                  d      }|j                  di       }|j                  d      }|rR|j                         rB|j                  d	| d
       |j                  t        |d             |j                  d       n|j                  d	| d       |j                  d        |j                  d       dj	                  |      S )zFormats tool definitions in the format that OpenAI appears to use.
    Based on https://github.com/forestwanglin/openai-java/blob/main/jtokkit/src/main/java/xyz/felh/openai/jtokkit/utils/TikTokenUtils.java
    znamespace functions {r   r  description// r  
parameters
propertiesztype z = (_: {r   z
}) => any;z = () => any;z} // namespace functions
)r   r   keys_format_object_parametersr   )r  linestoolr  function_descriptionfunction_namer  r  s           r   r  r    s    E	LL()	LL 88J'#+<<#>>>LL33456 V,\\,3
^^L1
*//+LL5y9:LL2:qABLL&LL5}=>R 
LL+,99Ur   c           
         | j                  d      }|sy| j                  dg       }g }|j                         D ]Y  \  }}|j                  d      }|r|j                  d|        d}|r||v rd}|j                  | | dt        ||       d       [ d	j	                  |D 	cg c]  }	d
t        d|      z  |	z    c}	      S c c}	w )Nr  r   requiredr  r  ?z: ,r   r   )r   rw  r   _format_typer   max)
r  indentr  required_paramsr  r  propsr  questionlines
             r   r  r    s    -J nnZ4OE &&( I
Uii.LL3{m,-so5HuXJbeV)D(EQGHI 99eDdcC6N*T1DEEDs   $Cc                 n   | j                  d      }|dk(  r+d| v r&dj                  | d   D cg c]  }d| d
 c}      S y|dk(  rt        | d   |       dS |d	k(  rd
t        | |dz          dS |dv r+d| v r&dj                  | d   D cg c]  }d| d
 c}      S y|dk(  ry|dk(  ryyc c}w c c}w )Nr   stringenumz | "arrayrw  z[]objectz{
   z
})integernumberr   booleannullr   )r   r   r  r  )r  r  r   items       r   r  r  "  s    99VDxU?::uV}Et4&{EFF	uW~v67r::		/vzBC4HH	&	&U?::uV}Et4&{EFF			 # F Fs   B-B2c                    d}d}	|E|7t        d|        d}|D ]   }
|
j                  dd      |
j                  d      }t        |t              r	||
d   z  }nt        |t              r|D ]  }|d   dk(  r	||d   z  }|d   d	k(  st        |d	   t
              r>|d	   }|j                  d
d      }|j                  d      }|	t        |||xs d      z  }	nt        |d	   t              s|d	   }|	t        |d|xs d      z  }	 |
j                  d      sd}|
d   D ]  }d|v s|d   d   }||z  } # nFt        d      t        |t              rdj                  d |D              }nt        |t              rd}| ||xs t        |       }|d   dk(  r+|d   j                  |      }t        |j                        }	|	S |d   dk(  r| t        j                  v s| t        j                  v rK| t        j                  v r| j!                  dd      } t        d|         t#        || ||||||xs d      }	|	S t        d|         t#        |d||||||xs d      }	|	S t        t%        j                  |d            }	|	S ) a  
    Count the number of tokens in a given text using a specified model.

    Args:
    model (str): The name of the model to use for tokenization. Default is an empty string.
    custom_tokenizer (Optional[dict]): A custom tokenizer created with the `create_pretrained_tokenizer` or `create_tokenizer` method. Must be a dictionary with a string value for `type` and Tokenizer for `tokenizer`. Default is None.
    text (str): The raw text string to be passed to the model. Default is None.
    messages (Optional[List[Dict[str, str]]]): Alternative to passing in text. A list of dictionaries representing messages with "role" and "content" keys. Default is None.

    Returns:
    int: The number of tokens in the text.
    Fr   Nz!token_counter messages received: r   r   r   r  r  r  r  r  r  
tool_callsTr  	argumentsz%text and messages cannot both be Nonec              3   B   K   | ]  }t        |t              s|  y wr   )r   r   )r   ts     r   r   z token_counter.<locals>.<genexpr>|  s     =Q*Q*<q=   r  r  r  r  z-35z-3.5z6Token Counter - using OpenAI token counter, for model=)r  r   r   r  r  r  r  r  z7Token Counter - using generic token counter, for model=zgpt-3.5-turbor  r  )r   r   r   r   ra   r  r0   
ValueErrorr   r  r  r   idsr   r  r  replacer  r   )r   r  r  r   r  r  r  r  r  r  r   r   r  r  r  r  r  	tool_callfunction_argumentsr  r  s                        r   token_counterr/  :  s    . LJ|=hZHID# !7;;y$/;%kk)4G!'3/	 22#GT2!( &A yF2 $&	 1!"6k!9#-and#C56{^N-;-?-?&-QF*8*<*<U*CC$.2F-0-3Fc G1+0	3& %&J &0+%D45kNM$.2F-:-3Fc G1+0	3& %&J!&, ;;|,#'L%,\%: 7	%21::1F{1S. $66D7=!7F DEE	D$	ww=$==	D#	 $,8)K->U-K&!%<< -44T:CSWWJR Q F#'99???G...G...!MM%8ELUGT 2%!-*? +2O 3

8 ! MeWU 2)%!-*? +2O 3

  "EF
r   custom_llm_providerc                     g d}| |v ryy)zU
    Helper function to know if a provider implementation supports httpx timeout
    )r  azurebedrockTFr  )r0  supported_providerss     r   supports_httpx_timeoutr5    s     911r   c                     t        | |d      S )a  
    Check if the given model supports system messages and return a boolean value.

    Parameters:
    model (str): The model name to be checked.
    custom_llm_provider (str): The provider to be checked.

    Returns:
    bool: True if the model supports system messages, False otherwise.

    Raises:
    Exception: If the given model is not found in model_prices_and_context_window.json.
    supports_system_messagesr   r0  r  _supports_factoryr   r0  s     r   r7  r7    s     /& r   c                 0   	 t        | |      \  } }}}t
        j                  j                  t
        j                  j                  g}||v ryt        | |d	      S # t        $ r1}t        j                  d|  d| dt	        |              Y d}~yd}~ww xY w)
ak  
    Check if the given model + provider supports 'response_schema' as a param.

    Parameters:
    model (str): The model name to be checked.
    custom_llm_provider (str): The provider to be checked.

    Returns:
    bool: True if the model supports response_schema, False otherwise.

    Does not raise error. Defaults to 'False'. Outputs logging.error.
    r;  zOModel not found or error in checking response schema support. You passed model=, custom_llm_provider=	. Error: NFTsupports_response_schemar8  )
r    r   rt   r   r   r   rM   	PREDIBASEFIREWORKS_AIr:  )r   r0  _r#  *PROVIDERS_GLOBALLY_SUPPORT_RESPONSE_SCHEMAs        r   r?  r?    s     +;-@,
("Aq 	&&))2.
 HH/&   ]^c]ddz  |O  {P  PY  Z]  ^_  Z`  Ya  b	
 	s   A 	B$'BBc                     t        | |d      S )  
    Check if the given model supports function calling and return a boolean value.

    Parameters:
    model (str): The model name to be checked.
    custom_llm_provider (Optional[str]): The provider to be checked.

    Returns:
    bool: True if the model supports function calling, False otherwise.

    Raises:
    Exception: If the given model is not found or there's an error in retrieval.
    supports_function_callingr8  r9  r;  s     r   rF  rF    s      /' r   r  c                    	 t        j                  | |      \  } }}}t        | |      }|j                  |d      du ryy# t        $ r4}t        j                  d| d|  d| dt        |              Y d}~yd}~ww xY w)	rE  r;  FTz%Model not found or error in checking z support. You passed model=r=  r>  N)r   r    _get_model_info_helperr   r   rt   r   r   )r   r0  r  rB  
model_infor#  s         r   r:  r:    s    +2+C+C-@,
("Aq ,-@

 >>#u%- 3C58STYSZZp  rE  qF  FO  PS  TU  PV  OW  X	
 	s   =A 	A>
*A99A>c                     t        | |d      S )zECheck if a given model supports audio input in a chat completion callsupports_audio_inputr8  r9  r;  s     r   rK  rK  2  s    )<BX r   c                     t        | |d      S )zCCheck if a given model supports pdf input in a chat completion callsupports_pdf_inputr8  r9  r;  s     r   rM  rM  9  s    )<BV r   c                     t        | |d      S )zFCheck if a given model supports audio output in a chat completion callrK  r8  r9  r;  s     r   supports_audio_outputrO  @  s     )<BX r   c                     t        | |d      S )a  
    Check if the given model supports prompt caching and return a boolean value.

    Parameters:
    model (str): The model name to be checked.
    custom_llm_provider (Optional[str]): The provider to be checked.

    Returns:
    bool: True if the model supports prompt caching, False otherwise.

    Raises:
    Exception: If the given model is not found or there's an error in retrieval.
    supports_prompt_cachingr8  r9  r;  s     r   rQ  rQ  I  s      /% r   c                    	 t        j                  | |      \  } }}}t        j                  | |      }|j                  dd      du ryy# t        $ r1}t        j                  d|  d| dt        |              Y d}~yd}~ww xY w)	a  
    Check if the given model supports vision and return a boolean value.

    Parameters:
    model (str): The model name to be checked.
    custom_llm_provider (Optional[str]): The provider to be checked.

    Returns:
    bool: True if the model supports vision, False otherwise.
    r;  supports_visionFTzFModel not found or error in checking vision support. You passed model=r=  r>  N)r   r    get_model_infor   r   rt   r   r   )r   r0  rB  rI  r#  s        r   rS  rS  `  s    +2+C+C-@,
("Aq ++-@

 >>+U3t; TUZT[[q  sF  rG  GP  QT  UV  QW  PX  Y	
 	s   AA 	B'B  Bc                     t        | |d      S )z]
    Check if the given model supports embedding image input and return a boolean value.
    supports_embedding_image_inputr8  r9  r;  s     r   rV  rV  ~  s     /, r   c                     | t         j                  v r)t         j                  |    }|j                  dd      du ryyt        d|  d      )a  
    Check if the given model supports parallel function calling and return True if it does, False otherwise.

    Parameters:
        model (str): The model to check for support of parallel function calling.

    Returns:
        bool: True if the model supports parallel function calling, False otherwise.

    Raises:
        Exception: If the model is not found in the model_cost dictionary.
    "supports_parallel_function_callingFTz?Model not supports parallel function calling. You passed model=.)r   
model_costr   r   )r   rI  s     r   rX  rX    sV     """''.
>>>F$NMeWTUV
 	
r   existing_dictnew_dictc                 @    |j                         D ]
  \  }}|| |<    | S r   rw  )r[  r\  r  r  s       r   _update_dictionaryr_    s.      1a r   rZ  c                 	   i }t        | t              r| }n&t        | t              rt        j                  |       }|j                         D ]'  \  }}	 t        t        t        |            }|d   }t        ||      }t        j                  j                  |i       j                  |       |j                  d      dk(  r3|t        j                  vst        j                  j                  |       |j                  d      dk(  r3|t        j                   vst        j                   j                  |       |j                  d      dk(  r5|t        j"                  vst        j"                  j                  |       9|j                  d      dk(  r5|t        j$                  vsat        j$                  j                  |       |j                  d      d	k(  rJ|j'                  d
d      }|t        j(                  vst        j(                  j                  |d          |j                  d      dk(  r5|t        j*                  vst        j*                  j                  |       )|j                  d      dk(  r5|t        j,                  vsQt        j,                  j                  |       r|j                  d      dk(  r5|t        j.                  vst        j.                  j                  |       |j                  d      dk(  r5|t        j0                  vst        j0                  j                  |       |j                  d      dk(  r5|t        j2                  vs,t        j2                  j                  |       M|j                  d      dk(  r5|t        j4                  vsut        j4                  j                  |       |j                  d      dk(  r5|t        j6                  vst        j6                  j                  |       |j                  d      dk(  s|t        j8                  vs	t        j8                  j                  |       * | S # t        $ r i }|}Y w xY w)a  
    Register new / Override existing models (and their pricing) to specific providers.
    Provide EITHER a model cost dictionary or a url to a hosted json blob
    Example usage:
    model_cost_dict = {
        "gpt-4": {
            "max_tokens": 8192,
            "input_cost_per_token": 0.00003,
            "output_cost_per_token": 0.00006,
            "litellm_provider": "openai",
            "mode": "chat"
        },
    }
    r  r  r  litellm_providerr  text-completion-openaicohere	anthropic
openrouter/rr   zvertex_ai-text-modelszvertex_ai-code-text-modelszvertex_ai-chat-modelszvertex_ai-code-chat-modelsai21	nlp_cloudaleph_alphar3  )r   r  r   r   get_model_cost_maprw  rg   rT  r   r_  rZ  
setdefaultupdater   r  r   open_ai_text_completion_modelsr  r  splitopenrouter_modelsvertex_text_modelsvertex_code_text_modelsvertex_chat_modelsvertex_code_chat_modelsai21_modelsnlp_cloud_modelsaleph_alpha_modelsbedrock_models)rZ  loaded_model_costr  r  existing_modelmodel_cost_keyupdated_dictionarysplit_strings           r   register_modelr~    s    *d#&	J	$#66:F'--/ 33
U	!#'n3.G#HN+E2N
 0F%%nb9@@AST99'(H4'@@@66==cBYY)*.FF'@@@66==cBYY)*h6'///%%,,S1YY)*k9'222((//4YY)*l:99S!,L'333))00aAYY)*.EE'444**11#6YY)*.JJ'999//66s;YY)*.EE'444**11#6YY)*.JJ'999//66s;YY)*f4'---##**3/YY)*k9'222((//4YY)*m;'444**11#6YY)*i7'000&&--c2g33h _  	!N N	!s   Q99R
	R
api_keyr   r   rf  r   re  custom_prompt_dictlitellm_metadata(disable_add_transform_inline_image_blockdrop_params	prompt_idprompt_variables
async_callc%                     i d|d| d|d|d|d|d|	d|
d	|d
|d|d|d|d|d|di d|i d|d|d|d|d|d|d|d|xs t        |      d|d|d|d|d| d |!d!|"d"|#d#|$}&|&S )$Nr  r  force_timeoutr   verboser0  r   r   model_alias_mapcompletion_call_idr   rI  proxy_server_requestpreset_cache_keyzno-logstream_responseinput_cost_per_tokeninput_cost_per_secondoutput_cost_per_tokenoutput_cost_per_secondcooldown_timer  azure_ad_token_provideruser_continue_messagerf  r   r   re  r  r  r  r  r  r  r  )*_get_base_model_from_litellm_call_metadata)'r  r  r2  r   r  hugging_facer  together_air0  r   r   r  r  r   rI  r  r  r  no_logr  r  r  r  r  r  r  r  rf  r   re  r  r  r  r  r  r  r  r   r   s'                                          r   get_litellm_paramsr    s   P${$7$ 	$ 	Y	$
 	7$ 	2$ 	H$ 	?$ 	?$ 	0$ 	H$ 	j$ 	 4$ 	,$ 	&$  	2!$" 	 4#$$ 	 !6%$& 	 !6'$( 	!"8)$* 	+$, 	?-$. 	"#:/$0 	 !61$2 	j I5xH5$6 	,7$8 	9$: 	0;$< 	,=$> 	34\?$@ 	{A$B 	YC$D 	,E$F 	jG$NJ r   c                 2    |t        |t              r| |v ryyr;  )r   r   r  additional_drop_paramss     r   _should_drop_paramr  G  s"    *-t4''r   passed_paramsdefault_paramsr  c                     i }| j                         D ](  \  }}||v s|||   k7  st        ||      du s$|||<   * |S )Nr  F)rw  r  )r  r  r  non_default_paramsr  r  s         r   _get_non_default_paramsr  R  sc     ##% &1^A&&"Q?UV %&q!& r   languager   r>  temperaturetimestamp_granularities)wordsegmentc                 2   t               }	|	j                  d      |	j                  d      |	j                  d      }
|
j                         D ]
  \  }}||	|<    d d d d d}|	j                         D ci c]  \  }}||v r|||   k7  r|| c}}i }fd}d } t        j	                  | t                    }dk(  sdk(  r}nd	k(  rWt        j                         j                         } ||
       t        j                         j                  || nd      }n6|4|j                  |       } ||
       |j                  || nd      }|	j                         D ]  }||j                         vs|	|   ||<    |S c c}}w )Nr0  r  r   )r  r   r>  r  c                    t        j                               dkD  rdt        j                               }|D ]D  }du st        j                  du r|| vrj                  |d        0|| vs5t        dd d       S y )Nr   T  z1Setting user/encoding format is not supported by =. To drop it from the call, set `litellm.drop_params = True`.status_coder   r   r  r   r   r  r   r   )supported_paramsr  r  r0  r  r  s      r   _check_valid_argz;get_optional_params_transcription.<locals>._check_valid_arg  s    !&&()A-*//12D 	4'7+>+>$+F//&**1d3..0$'"STgSh  if  !g 	 &% .r   r   r   r  r2  groqr  Fr  r   r   r  r  )localsr   rw  ProviderConfigManager'get_provider_audio_transcription_configrM   r   GroqSTTConfigget_supported_openai_params_sttmap_openai_params_sttr!   map_openai_paramsr  )r   r  r   r>  r  r  r0  r  r   r  special_paramsr  r  r  r   r  provider_configr  r  s         ``          @r   !get_optional_params_transcriptionr  b  s    HM'++,AB##M2K"&&x0N$$& 1a 	N "'')AqA):$: 	
1
 O& ?CO&/WW!"56 X 

 h&*=*H,		&"002RRT*:;!//1GG1+'2'>E	 H 
 
	$*FFUFS*:;);;1+'2'>E	 < 
 ! 2N''))!.q!1OA2 ks   8Fnqualitysizestyler   c	                 r   t               }
|
j                  dd       } |
j                  d      |
j                  dd       }|
j                  d      }|j                         D ]M  \  }}|j                  d      rdk7  rdk7  r"|dk(  rdk7  r-|j                  d	      rd
k7  rdk7  rI||
|<   O d d d d d d d}t	        |
||      i }fd}dk(  sdk(  st
        j                  v r}ndk(  rot
        j                  j                  |       rt
        j                  nt
        j                  }|j                  |       } ||       |j                  i       }n"d
k(  rdg}	  ||       |t        |      |d<   |
j                         D ]  }||j                         vs|
|   ||<    |S )Nr   r0  r  r   aws_r3  	sagemakerre  vertex_	vertex_aivertex_ai_beta)r  r  r>  r  r  r   r  r  r  c           	         t        j                               dkD  rct        j                               }|D ]C  }t        j                  du r|| vrj                  |d        ,|| vs1t        dd| d d       S y )Nr   Tr  z	Setting `z` is not supported by r  r  r  )r  r  r  r0  r  s      r   r  z7get_optional_params_image_gen.<locals>._check_valid_arg  s    !&&()A-*//12D 	''4/A=M4M&**1d3..0$'"+A3.DEXDY  ZW  !X 	 &% .r   r  r2  r  r  r  r   r  sampleCount)r  r   rw  rx  r  r   openai_compatible_providersAmazonStability3Config_is_stability_3_modelAmazonStabilityConfigr!   r  intr  )r   r  r  r>  r  r  r   r0  r  r   r  r  r  r  r  r   r  config_classr  r  s          `           @r   get_optional_params_image_genr    s    HMgt,E'++,AB*../GN"&&x0N$$& 1<<9,1D1S/!&9[&HLL##{2#'77a  N 1#%5
 O&  	x'')'"E"EE,			) --CC%CP **.. 	
 (CC%CP*:;&8812 9 
 
	+5	 	*:;=-0VOM*! 2N''))!.q!1OA2 r   encoding_format
dimensionsc                 	    t               }|j                  dd       |j                  d      }	|	j                         D ]
  \  }
}|||
<    |j                  dd       |j                  dd       }d d d d}dt        t           f fd}t        |||      d	k(  r( d
 vrdj                         v rt        dd      dk(  rJt         d      } ||       t        j                         j                  i  nd      }i ||}|S dk(  rHt         xs ddd      } ||       t        j                         j                  i       }i ||}|S dk(  r?t         xs ddd      } ||       t        j                  j                  i |      }|S dk(  rHt         dd      } ||       t        j                         j                  i |      \  }}i ||}|S dk(  rXt        j                         j                         } ||       t        j                         j                  i       }i ||}|S dk(  rd v rt        j                          }n\d v rt        j"                         }nCd v rt        j$                         }n*d  v rt        j&                         }ng } ||       i |}|S |j                         } ||       |j                  i       }i ||}|S d!k(  rDt         d!d      } ||       t        j(                         j                  i       }i ||}|S d"k(  rDt         d"d      } ||       t        j*                         j                  i       }i ||}|S d#k(  rJt         d#d      } ||       t        j,                         j                  i  nd      }i ||}|S d$k(  rEt         d$d      } ||       t        j.                         j                  i  %      }i ||}|S d	k7  rd&k7  rt        j0                  vryt3        j                               d'kD  r]t        j4                  d(u sd(u r3t	        j                               }|D ]  }
j                  |
d         nt        dd) d* d+      i |}|S ),Nr0  r   r  r  )r   r  r  r  c           	          | y i }j                         D ]  }|| vs|   ||<    |r/t        j                  du sdu ry t        d d| d d      y )NTr   does not support parameters: , for model=f. To drop these, set `litellm.drop_params=True` or for proxy:

`litellm_settings:
 drop_params: true`
r  )r  r   r  r   )r  unsupported_paramsr  r0  r  r   r  s      r   r  z8get_optional_params_embeddings.<locals>._check_valid_arg-	  s    ##((* 	>A(((:1(="1%	> ""d*'K4,?, #233QRdQeeqrwqx  yc  d  r   r  r  ztext-embedding-3r  r  zSetting dimensions is not supported for OpenAI `text-embedding-3` and later models. To drop it from the call, set `litellm.drop_params = True`.r  triton
embeddings)r   r0  request_typer  Fr  
databricksr   r  
nvidia_nim)r  r   r   r  	lm_studior3  zamazon.titan-embed-text-v1zamazon.titan-embed-image-v1zamazon.titan-embed-text-v2:0zcohere.embed-multilingual-v3mistraljina_aivoyagefireworks_ai)r  r   r   r2  r   TzSetting z is not supported by r  )r  r   rw  rc   r   r  r  r   r!   r   TritonEmbeddingConfigr  DatabricksEmbeddingConfignvidiaNimEmbeddingConfigVertexAITextEmbeddingConfigLmStudioEmbeddingConfigAmazonTitanG1Config&AmazonTitanMultimodalEmbeddingG1ConfigAmazonTitanV2ConfigBedrockCohereEmbeddingConfigMistralEmbeddingConfigJinaAIEmbeddingConfigVoyageEmbeddingConfigFireworksAIEmbeddingConfigr  r   r  )r   r   r  r  r0  r  r  r   r  r  r  r  r  r  r  r   final_paramsr  r  r  s   `   ``             @r   get_optional_params_embeddingsr  	  s#    HM'++,A4H"&&x0N$$& 1a  ##M48K*../GN"t4PN8D>  $ 1#%5 h& "%/ 2 7 7 99( j  
	(6 3%

 	*:;!779KK1'2'>E	 L 
 5/4V4		,6+2 ,%

 	*:;!;;=OO12 P 
 5/4V4		,6+2 ,%

 	*:;!::LL12f M 
 		+6 +%

 	*:; //1CC12f D 
	
 5/4V4		+++-IIK 	 	*:;!99;MM12 N 
 5/4V4			)'50!557F*e3CCEF+u4002F+u499;F!.>?%f:L!==?*:; 2212 3 
 5/4V4			)6 )%

 	*:;!88:LL12 M 
 5/4V4			)6 )%

 	*:;!779KK12 L 
 5/4V4		(6 (%

 	*:;!779KK1'2'>E	 L 
 5/4V4		.6 .%

 	*:;!<<>PP12U Q 
 5/4V4 	x'7*w'J'JJ!&&()A-##t+{d/B.3356 4A&**1d34 - #&'9&::OPcOd  eb  c  4(3F3Lr   c                     t        | t              r3d| v r
| d   du r| d= | j                         D ]  \  }}t        |        | S t        | t              r| D ]  }t        |        | S )a  
    clean out 'additionalProperties = False'. Causes vertexai/gemini OpenAI API Schema errors - https://github.com/langchain-ai/langchainjs/issues/5240

    Relevant Issues: https://github.com/BerriAI/litellm/issues/6136, https://github.com/BerriAI/litellm/issues/6088
    additionalPropertiesF)r   r  rw  _remove_additional_propertiesr   rA  r  r  r#  s       r   r  r  	  s     &$!V+7M0NRW0W-. !,,. 	1JC)%0	1 M 
FD	! 	0D)$/	0 Mr   c                     t        | t              r,d| v r| d= | j                         D ]  \  }}t        |        | S t        | t              r| D ]  }t        |        | S )zy
    Relevant Issues: https://github.com/BerriAI/litellm/issues/6136, https://github.com/BerriAI/litellm/issues/6088
    strict)r   r  rw  _remove_strict_from_schemar   r  s       r   r  r  
  sr     &$vx  !,,. 	.JC&u-	. M 
FD	! 	-D&t,	- Mr   r  supported_openai_paramsc                     g }|i S | j                         D ]  }||vs|j                  |        |D ]  }| j                  |d        | S )z;
    Remove unsupported params from non_default_params
    N)r  r   r   )r  r  remove_keysparamr  s        r   _remove_unsupported_paramsr  
  sj     K&	#((* &//u%&  *sD)*r   extra_headersc                     t         j                  dur| S i }| j                         D ]4  \  }}|t        v r"|dk7  rt	        j
                  d| d| d       0|||<   6 |S )z
    For `anthropic-beta` headers, ensure provider is anthropic.

    Vertex AI raises an exception if `anthropic-beta` is passed in.
    Tre  	Provider z does not support z2 header. Dropping from request, to prevent errors.)r   filter_invalid_headersrw  r7   rt   r   )r  r0  clean_extra_headersr  r  s        r   get_clean_extra_headersr  -
  s     %%T1##% '1**/Bk/Q  /00B1#Ewx &'"' r   c                  (   23 t               j                         33j                  d      }!|!j                         D ]M  \  }"}#|"j	                  d      rdk7  rdk7  r"|"dk(  rdk7  r-|"j	                  d      rdk7  rdk7  rI|#3|"<   O i }$t
        j                  }%|%d	   v r	 d
k(  r&t        j                         j                  3|$      }$ndk(  r&t        j                         j                  3|$      }$nZdk(  sdk(  r&t        j                         j                  3|$      }$n*dk(  r%t        j                         j                  3|$      }$i dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd dd d dd d d d d d d d d d d d d d}&3j                         D "#ci c]A  \  }"}#|"d k7  r7|"d!k7  r2|"d"k7  r-|"d#k7  r(|"d$k7  r#|"d%k7  r|"|&v r|#|&|"   k7  rt        |"|&      d'u r|"|#C c}#}"2|t        |(      }d2v s	d2v sd)2v r d*k(  rd+k7  rd
k7  rdk7  rd,k7  rd-k7  rd.k7  rd/k7  rd0k7  rd1k7  rd2k7  rd3k7  r߉d4k7  rډd5k7  rՉd6k7  rЉd7k7  rˉd8k7  rƉd9k7  rdk7  rd:k7  rd;k7  rt
        j                  vrd*k(  rTd<|$d=<   d>t
        _        d)2v r'2j                  d)      |$d?<   2j                  d@d        n`d2v r\2j                  d      |$d?<   nGt
        j                  r&2j                  d)2j                  dd             |$d?<   nt!        dAdB dCD      d }'@t"        D (cg c]  }(|(j$                   c}(v r t&        j)                   t#              E      }'dF2v r-|'|'j+                  2dF   G      2dF<   nt-        2dF   G      2dF<   d)2v rzt/        2t0              rj2d)   }|D ]`  })|)j3                  dHi       }*|*j3                  dId       }+|+*t        j4                  |+      },dJ|,v r|,dJ   d'u r|,j                  dJd        |,|*dI<   b dKt6        t8           f 23fdL}-t;         M      }.|.t;         dNM      }. |-|.xs g O       d7k(  r=t        j<                         j?                   2|$t/        t@              rnd'P      }$nMdQk(  rxt        jB                         j?                   2|$t/        t@              rnd'P      }$t        jB                         j?                   2|$t/        t@              rnd'P      }$nЉd9k(  r=t        jD                         j?                  2|$ t/        t@              rnd'R      }$nd8k(  r=t        jF                         j?                  2|$ t/        t@              rnd'R      }$nLdSk(  r-t        jH                         j?                  2|$ nd'R      }$ndTk(  r=t        jJ                         j?                  2|$ t/        t@              rnd'R      }$n؉dUk(  r=t        jL                         j?                  2|$ t/        t@              rnd'R      }$ndVk(  r=t        jN                         j?                  2|$ t/        t@              rnd'R      }$nTdWk(  r=t        jP                         j?                  2|$ t/        t@              rnd'R      }$nd-k(  r=t        jR                         j?                  2|$ t/        t@              rnd'R      }$nЉdk(  r t
        jT                  v sZ t
        jV                  v sH t
        jX                  v s6 t
        jZ                  v s$ t
        j\                  v s t
        j^                  v r=t        j`                         j?                  2|$ t/        t@              rnd'R      }$n"dXk(  r=t        jb                         j?                  2|$ t/        t@              rnd'R      }$
ndk(  s	dk(  rAdX v r=t        j`                         j?                  2|$ t/        t@              rnd'R      }$
nt
        jd                  jg                   M      r=t        jd                         j?                   2|$t/        t@              rnd'P      }$
n7dk(  rO t
        jh                  v r=t        jj                         j?                  2|$ t/        t@              rnd'R      }$	ndk(  r t
        jl                  v r~d5 v r=t        jn                         j?                   2|$t/        t@              rnd'P      }$	nt        jp                         j?                   2|$t/        t@              rnd'P      }$	nNdk(  rO t
        jr                  v r=t        jt                         j?                  2|$ t/        t@              rnd'R      }$ndk(  r=t        jv                         j?                  2|$ t/        t@              rnd'R      }$ndk(  r	t        jx                         j{                         }/|/t
        j|                  v r>t        jx                         j?                   2|$t/        t@              rnd'|Y      }$n?d7 v rddZ3v r9 j	                  d[      r%t        j~                         j?                  2|$      }$nt        j                         j?                  2|$      }$n|'|'j?                  2|$ t/        t@              rnd'R      }$nd\k(  r=t        j                         j?                   2|$t/        t@              rnd'P      }$ngd*k(  r=t        j                         j?                  2|$ t/        t@              rnd'R      }$n%d:k(  r=t        j                         j?                   2|$t/        t@              rnd'P      }$nd]k(  r=t        j                         j?                  2|$ t/        t@              rnd'R      }$nd^k(  r=t        j                         j?                  2|$ t/        t@              rnd'R      }$n_d_k(  r=t        j                         j?                  2|$ t/        t@              rnd'R      }$nd`k(  r-|'+|'j?                  2|$ t/        t@              rnd'R      }$nd6k(  sd5k(  r=t        jp                         j?                  2|$ t/        t@              rnd'R      }$ndak(  r=t        jn                         j?                  2|$ t/        t@              rnd'R      }$nbdbk(  r=t        j                         j?                  2|$ t/        t@              rnd'R      }$n d/k(  r=t        j                         j?                   2|$t/        t@              rnd'P      }$nމd0k(  r=t        j                         j?                  2|$ t/        t@              rnd'R      }$nd1k(  r(t        j                         j?                   2|$c      }$nod2k(  sddk(  r=t        j                         j?                  2|$ t/        t@              rnd'R      }$n(dek(  r=t        j                         j?                  2|$ t/        t@              rnd'R      }$nd3k(  r=t        j                         j?                  2|$ t/        t@              rnd'R      }$ndfk(  r=t        j                         j?                  2|$ t/        t@              rnd'R      }$nbdgk(  r=t        j                         j?                  2|$ t/        t@              rnd'R      }$n d.k(  r=t        j                         j?                  2|$ t/        t@              rnd'R      }$nމd4k(  r=t        j                         j?                  2|$ t/        t@              rnd'R      }$nd;k(  r=t        j                         j?                  2|$ t/        t@              rnd'R      }$nZdk(  rt        j                         j?                  2|$ t/        t@              rnd'R      }$3j                         D ]3  }0t        j                         j                  |0      s't        dh|0        n҉dik(  r=t        j                         j?                  2|$ t/        t@              rnd'R      }$ndNk(  r=t        j                         j?                  2|$ t/        t@              rnd'R      }$nNd
k(  rt        j                         j                   j      r<t        j                         j?                  2|$ t/        t@              rnd'R      }$nt        j                  dkj                  |t
        j                  t        dl                   |xs/ t
        j                  xs t        dl      xs t
        j                  }t        j                         j?                  2|$ |t/        t@              rnd'm      }$n;t        j                         j?                  2|$ t/        t@              rnd'R      }$g dnt
        j                  z   v rt        do|&      d'u r3j                  doi       }13j                         D ]  }"|"|&j                         vs3|"   |1|"<    |$j                  doi        i |$do   |1|$do<   t        |$do   p      |$do<   n03j                         D ]  }"|"|&j                         vs3|"   |$|"<    ||$j                  dqi        i |$dq   ||$dq<   t        dr|$        |$S c c}#}"w c c}(w )sNr   r  r3  r  re  r  r  r  	providersr2  r  watsonx	functionsfunction_callr  top_pr  r   r   stoprd  max_completion_tokens
modalities
predictionaudiopresence_penaltyfrequency_penalty
logit_biasr   r   )r   r0  r>  seedr  r  max_retrieslogprobstop_logprobsr  api_versionparallel_tool_callsr  r  r   r   r0  r  r  r  r   r  F)r  r0  r  ollamarc  anyscaler  r  r  cerebrasxai	ai21_chat
volcenginedeepseek	codestralr  re  cohere_chatrd  ollama_chatrf  r  formatTfunctions_unsupported_modelr  r  z%Function calling is not supported by rY  r  r  r>  r@  r  r  r  r  c           	         t        j                  d d        t        j                  d        t        j                  d        i }j                         D ]0  }|| vs|dk(  s
|dk(  s|dk(  r|dk(  rd	k(  r#|d
k(  r)|   ||<   2 |rVt        j
                  du s,du r(|j                         D ]  }j                  |d         y t        d d| d d      y )Nz
LiteLLM completion() model= z; provider = z(
LiteLLM: Params passed to completion() z4
LiteLLM: Non-Default params passed to completion() r   r   r   r  rr   r  Tr  r  r  r  r  )rt   r   r   r  r   r  r   r   )	r  r  r  r0  r  r   r  r  r  s	      r   r  z-get_optional_params.<locals>._check_valid_arg3  sM   ,UG=AT@UV	
 	7G	
 	CDVCWX	
  #((* 	BA((;!'7"71=8Q& -?q,A&q)	B ""d*'K4,?+002 4A&**1d34 - #233QRdQeeqrwqx  yc  d  r   r;  r  r  )r   r  r   r  anthropic_textr  r  maritalkr  	predibasehuggingfacegemini)r   r  r   r  r   aws_bedrock_clientzanthropic.claude-3
cloudflareri  petals	deepinfra
perplexitytext-completion-codestralr  )r   r  r   rh  r  hosted_vllmvllmzLiteLLM now defaults to Watsonx's `/text/chat` endpoint. Please use the `watsonx_text` provider instead, to call the `/text/generation` endpoint. Param: watsonx_textr  zoAzure optional params - api_version: api_version={}, litellm.api_version={}, os.environ['AZURE_API_VERSION']={}AZURE_API_VERSION)r  r   r   r  r  )r  r2  rc  
extra_body)r9  r  z Final returned optional params: )cr  r   r   rw  rx  r   !common_cloud_provider_auth_paramsAzureOpenAIConfigmap_special_auth_paramsAmazonBedrockGlobalConfigVertexAIConfigIBMWatsonXAIConfigr  r  r  add_function_to_promptr   rM   r  r  get_provider_chat_config$get_json_schema_from_pydantic_objectrl   r   r   r   r   ra   r   r!   AnthropicConfigr  r  AnthropicTextConfigCohereConfigCohereChatConfigTritonConfigMaritalkConfigReplicateConfigPredibaseConfigHuggingfaceConfigTogetherAIConfigrs  rt  rq  rr  vertex_language_modelsvertex_vision_modelsVertexGeminiConfigGoogleAIStudioGeminiConfigVertexAIAnthropicConfigis_supported_modelvertex_llama3_modelsVertexAILlama3Configvertex_mistral_modelsCodestralTextCompletionConfigMistralConfigvertex_ai_ai21_modelsVertexAIAi21ConfigSagemakerConfigAmazonConverseConfig_get_base_modelbedrock_converse_modelsAmazonAnthropicClaude3ConfigAmazonAnthropicConfigCloudflareChatConfigOllamaConfigOllamaChatConfigNLPCloudConfigPetalsConfigDeepInfraConfigDatabricksConfigNvidiaNimConfigCerebrasConfigXAIChatConfigAI21ChatConfigFireworksAIConfigVolcEngineConfigHostedVLLMChatConfig
VLLMConfigGroqChatConfigOpenAIConfigOpenrouterConfigIBMWatsonXChatConfigr  is_watsonx_text_paramr*  AzureOpenAIO1Configis_o1_modelrt   r   r'  r  r6   AZURE_DEFAULT_API_VERSIONOpenAILikeChatConfigrl  r"   r   )4r   r  r  r  r  r  r   r   r  rd  r  r  r  r  r  r  r  r   r0  r>  r  r  r  r  r  r  r  r  r  r  r  r   r   r  r  r  r   common_auth_dictr  r  r   r	  tool_functionr  new_parametersr  r  rf  r  r9  r  r  s4   `    `            `          `                    @@r   get_optional_paramsr{  @
  s   L HMMOM"&&x0N$$& 1<<9,1D1S/!&9[&HLL##{2#'77a O@@.{;;	 ')%779QQ#0/ R O !I-113KK'4o L    ;."&66%446NN#0/ O O !I-%88:RR#0/ S O!T!! 	t! 		!
 	T! 	$! 	$! 	! 	d! 	 ! 	d! 	d! 	! 	D! 	T!  	d!!" 	#!$ !#"&A!NL "'')AqL**]"]"--Z^#^A&&"Q?UV 	
1$  /'=P
 	))00((  8+#'??#w.#{2#z1#}4#v-#|3#z1#u,#{2#|3#z1#{2#y0#{2#}4#x/#y0#}4#|3#7+N+NN"h.,2) . 00*..w7 $$AB '**%t !$66*..{; $$AB ..ASAWAW/33KFB => - #CDWCXXYZ 
 -1O&+>'3C#C , 0HH,/B"C I 
 ..&DD$67H$I E  01 5R 23D E501 $$D* #7+ 	= !HHZ4M&**<>J%!%z!:*n<&'=>%G"&&'=tD.<l+	= #49 # #J 3)< 6X
 &6&<"=k)!113EE1+ *z+t/L  F 	
 
 0	0!557II1+ *z+t/L  J 	
 "557II1+ *z+t/L  J 	
 
	( "..0BB1+ *z+t/L  C 	
 
	-!224FF1+ *z+t/L  G 	
 
	(!..0BB1+'2'>E	 C 
 

	*!002DD1+ *z+t/L  E 	
 
	+!113EE1+ *z+t/L  F 	
 
	+!113EE1+ *z+t/L  F 	
 
	-!335GG1+ *z+t/L  H 	
 
	-!224FF1+ *z+t/L  G 	
 
	++++G333G...G333G222G000!446HH1+ *z+t/L  I 	
 
	(!<<>PP1+ *z+t/L  Q 	
 
 0	0{*x5/@!446HH1+ *z+t/L  I 	
 
	(	(	;	;)< 
< 
 "99;MM1+ *z+t/L  N 	
 
	+9U9U0U!668JJ1+ *z+t/L  K 	
 
	+9V9V0V%%CCEWW#5 / #.:k43P   X 	O &335GG#5 / #.:k43P   H 	O 
	+9V9V0V!446HH1+ *z+t/L  I 	
 
	+!113EE1+ *z+t/L  F 	
 
		)113CCEJ
888%::<NN#5 / #.:k43P  ! O 
O E!#}4##$89<<>PP/A,; Q  $ #*"?"?"A"S"S'9$3 #T # (-??#5 / #.:k43P   @ 	O 
	,!668JJ1+ *z+t/L  K 	
 
	(!..0BB1+ *z+t/L  C 	
 
	-!224FF1+ *z+t/L  G 	
 
	+!002DD1+ *z+t/L  E 	
 
	(!..0BB1+ *z+t/L  C 	
 
	+!113EE1+ *z+t/L  F 	
 
	,1L);;1+ *z+t/L  < 	
 
		)-@K-O!//1CC1+ *z+t/L  D 	
 
 ;	;!??ASS1+ *z+t/L  T 	
 
	,!224FF1+ *z+t/L  G 	
 
	,!113EE1+ *z+t/L  F 	
 

	*!002DD1+ *z+t/L  E 	
 
	%!//1CC1+ D 

 
	+/Bf/L!002DD1+ *z+t/L  E 	
 
	.!335GG1+ *z+t/L  H 	
 
	,!224FF1+ *z+t/L  G 	
 
	-!668JJ1+ *z+t/L  K 	
 
	&!,,.@@1+ *z+t/L  A 	
 
	&!002DD1+ *z+t/L  E 	
 

	*!..0BB1+ *z+t/L  C 	
 
	,!224FF1+ *z+t/L  G 	
 
		)!668JJ1+ *z+t/L  K 	
 #'') 	E))+AA%H  p  qv  pw  x 	
 
	.!446HH1+ *z+t/L  I 	
 
	(!..0BB1+ *z+t/L  C 	
 
	'&&(4454A%99;MM#5 / #.:k43P   N 	O    B  I  I!4!4jAT6U  5&&5125 44	  &779KK#5 /' #.:k43P   L 
O "668JJ1+ *z+t/L  K 	
 	8

-
-.	. 7M 
 '**<<J"'') 5N//11$1!$4JqM5 &&|R8-!,/--OL)
 -G*<8-OL)
 ##% 	6A++--%21%5"	6  ""?B7,
o.,
,
( 4_4EFGfCs   4AAO6AO<c                     i dd dd dd dd dd dd dd dd d	d d
d dd dd dd dd dddd dd d d d d d d d}| j                         D ci c]  \  }}|dk7  r|dk7  r||v r|||   k7  r||  }}}|S c c}}w )Nr  r  r  r  r  r   r   r  rd  r  r  r  r   r   r0  r   r>  r  )r  r  r  r  r  r  r^  )r  r  r  r  r  s        r   get_non_default_paramsr}    sM   T 	t 		
 	T 	$ 	$ 	 	d 	D 	T 	d 	 	 	r  	4!" 	#$ /N8 "'')	AqL**^#^A&& 	
1	 	 	s   #A:max_parallel_requestsrpmtpmdefault_max_parallel_requestsc                 T    | | S ||S |t        |dz  dz        }|dk(  rd}|S ||S y)a  
    Returns the max parallel requests to send to a deployment.

    Used in semaphore for async requests on router.

    Parameters:
    - max_parallel_requests - Optional[int] - max_parallel_requests allowed for that deployment
    - rpm - Optional[int] - requests per minute allowed for that deployment
    - tpm - Optional[int] - tokens per minute allowed for that deployment
    - default_max_parallel_requests - Optional[int] - default_max_parallel_requests allowed for any deployment

    Returns:
    - int or None (if all params are None)

    Order:
    max_parallel_requests > rpm > tpm / 6 (azure formula) > default max_parallel_requests

    Azure RPM formula:
    6 rpm per 1000 TPM
    https://learn.microsoft.com/en-us/azure/ai-services/openai/quotas-limits


    Ni     r   rr   )r  )r~  r  r  r  calculated_rpms        r   calculate_max_parallel_requestsr    sS    : ($$	
	S4Z!^,QN	&	2,,r   healthy_deploymentsc                     t        d | D        d       }|(| D cg c]  }|d   j                  d      |k(  r| }}|S | S c c}w )Nc              3   :   K   | ]  }d |d   v r
|d   d      yw)orderr   Nr  )r   
deployments     r   r   z2_get_order_filtered_deployments.<locals>.<genexpr>1  s1      	
*%566 '(1	
s   )defaultr   r  )minr   )r  	min_orderr  filtered_deploymentss       r   _get_order_filtered_deploymentsr  /  sr    	
1	

 I  2 
*+//8IE  
  
 $# 
s    Ar   c                 @   | dk(  rX|j                   xs* t        j                   xs t        d      xs t        d      }|t        |t              r|S |j                  S | dk(  r|j
                  }||S |j                  S | dk(  r|j                  }||S |j                  S )z=
    Return the region for a model, for a given provider
    r  VERTEXAI_LOCATIONVERTEX_LOCATIONr3  r  )vertex_locationr   r6   r   r   aws_region_namewatsonx_region_nameregion_name)r0  r   vertex_ai_locationr  r  s        r   _get_model_regionr  D  s     k) ** -&&--.- +,	 	 )j9KS.Q%% %%% 
		)(88&""
 %%%	 
		),@@*&&%%%r   c                    t        j                  | j                  |       \  }}}}t        ||       }|/t	        j
                  dj                  | j                               y|dk(  rEt        j                         j                         }t        j                         j                         }n|dk(  rEt        j                         j                         }t        j                         j                         }n|dk(  rEt        j                         j                         }t        j                         j                         }nN|dk(  rEt        j                         j                         }t        j                         j                         }ng }g }|D ]  }||j                         v s y	 |D ]  }||j                         v s y
 y)z
    Infer if a model is in the EU or US region

    Returns:
    - str (region) - "eu" or "us"
    - None (if region not found)
    )r   r   )r0  r   Nz'Cannot infer model region for model: {}r2  r  r3  r  euus)r   r    r   r  rt   r   r'  r;  get_eu_regionsget_us_regionsr>  r=  r?  r  )r   r   r0  rB  model_region
eu_regions
us_regionsregions           r   _infer_model_regionr  _  s    (/'?'?"">($E1 %/L 5<<^=Q=QR	
 g%..0??A
..0??A
		+++-<<>
++-<<>
			)668GGI
668GGI
			)//1@@B
//1@@B


 \''))  \'')) r   c                 L    | j                   dk(  ryt        |       }||dk(  ryy)8
    Return true/false if a deployment is in the EU
    r  Tr   Fr  r  r   r  s     r   _is_region_eur    4     !!T) 'nELLD$8r   c                 L    | j                   dk(  ryt        |       }||dk(  ryy)z8
    Return true/false if a deployment is in the US
    r  Tr  Fr  r  s     r   _is_region_usr    r  r   allowed_model_regionc                 $    | j                   |k(  ryy)r  TF)r  )r   r  s     r   is_region_allowedr    s     !!%99r   r  c           	         d| j                   v rt        | j                  t              rt        | j                  t              r| j                   j                  dd      }t        j                         j                  || j                  | j                  | j                  xs t        j                  d|xs d      }|j                  dd      }|S y)	zM
    Pass the litellm params for an azure model, and get back the region
    r2  zazure/r   
   chat)r   r  r   r  timeoutr  zx-ms-regionN)r   r   r  r   r   r,  r   AzureChatCompletionget_headersr  rv  r   )r   r  _modelrB  r  s        r   get_model_regionr    s     	>'''~--s3~..4%%--h; 446BB"**#,,&22Wg6W6W C 
 !)]D Ar   c                 d    	 | j                  d      }t        |      d d }|S # t        $ r Y yw xY w)Nr   r   r   )r   r   r   )r   	_messagess     r   get_first_chars_messagesr    s=    JJz*		N4C(	 s    # 	//c                 H    dj                  d | D              }t        |      S )Nr   c              3   B   K   | ]  }|j                         r|  y wr   )isspace)r   chars     r   r   z$_count_characters.<locals>.<genexpr>  s     HTDHr)  )r   r   )r  filtered_texts     r   _count_charactersr    s!    GGHTHHM}r   response_objc                 4   | j                   }d}|D ]  }t        |t              r1|j                  j                  *||j                  j                  z  }Dt        |t
              sU|j                  j                  l||j                  j                  z  } |S )Nr   )rG  r   rE   r   r   rU   delta)r  _choicesresponse_strchoices       r   get_response_stringr    s    7C7K7KHL 5fg&~~%%1 6 66 01||##/ 4 445 r   llm_providerdynamic_api_keyc                    |xs t         j                  }| dk(  s| dk(  r#|xs t         j                  xs t        d      }|S | dk(  s| dk(  r#|xs t         j                  xs t        d      }|S | dk(  r#|xs t         j
                  xs t        d      }|S | d	k(  r#|xs t         j                  xs t        d
      }|S | dk(  r#|xs t         j                  xs t        d      }|S | dk(  s| dk(  r#|xs t         j                  xs t        d      }|S | dk(  r#|xs t         j                  xs t        d      }|S | dk(  r#|xs t         j                  xs t        d      }|S | dk(  r#|xs t         j                  xs t        d      }|S | dk(  r.|xs* t         j                  xs t        d      xs t        d      }|S )Nr  rc  OPENAI_API_KEYre  r*  ANTHROPIC_API_KEYrh  AI211_API_KEYrj  ALEPH_ALPHA_API_KEYbasetenBASETEN_API_KEYrd  r%  COHERE_API_KEYr-  HUGGINGFACE_API_KEYri  NLP_CLOUD_API_KEYr  REPLICATE_API_KEYr  TOGETHERAI_API_KEYTOGETHER_AI_TOKEN)r   r  
openai_keyr6   anthropic_keyai21_keyaleph_alpha_keybaseten_key
cohere_keyhuggingface_keynlp_cloud_keyreplicate_keytogetherai_api_key)r  r  r  s      r   get_api_keyr    s   0Gx<3K#KOW//O:>N3OJ NG 
	$8H(HUW22UjAT6UD NA 
	LW--LO1L> N; 
	&Sw..S*=R2S 	8 N1 
	"QW00QJ?P4Q. N+ 
	!\]%BOW//O:>N3O( N% 
	&Sw..S*=R2S 	" N 
	$UW22UjAT6U N 
	$UW22UjAT6U N 
	& /))/.// -.	 	 Nr   c                      dd l } ddl m } t        | d      r |j                  | j                        S  |j                         S )Nr   )rr  UTC)rr  hasattrrs  r  utcnow)dtrr  s     r   get_utc_datetimer    s6    !r5x||BFF##x  r   c                 :   d }	 | t         j                  v rVdt         j                  |    v rt         j                  |    d   S dt         j                  |    v rt         j                  |    d   S t        |       \  } }}}|dk(  r ||       }|S | t         j                  v rWdt         j                  |    v rt         j                  |    d   S dt         j                  |    v rt         j                  |    d   S yt               # t        $ r t        d|  d	      w xY w)
aY  
    Get the maximum number of output tokens allowed for a given model.

    Parameters:
    model (str): The name of the model.

    Returns:
        int: The maximum number of tokens allowed for the given model.

    Raises:
        Exception: If the model is not mapped yet.

    Example:
        >>> get_max_tokens("gpt-4")
        8192
    c                     d|  d}	 t         j                  j                  |      }|j                          |j	                         }|j                  d      }||S y # t
        $ r Y y w xY wNzhttps://huggingface.co/z/raw/main/config.jsonmax_position_embeddingsr   module_level_clientr   raise_for_statusr  r   
model_name
config_urlrB  config_jsonr  s        r   _get_max_position_embeddingsz4get_max_tokens.<locals>._get_max_position_embeddings5  sv    .zl:OP
	2266zBH%%' #--/K&1oo6O&P#&2.. 		   AA 	A)(A)max_output_tokensrd  r  r-  r  NzModel zr isn't mapped yet. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json)r   rZ  r    r   )r   r  r0  rB  rd  s        r   get_max_tokensr  #  s8   $&
G&&&"g&8&8&??))%01DEE!3!3E!::))%0>>+;%+H("Aq-/5GJG&&&"g&8&8&??))%01DEE!3!3E!::))%0>>  + 
UG  N  O
 	

s)   <D *D - D <D *D 7
D Dc                 0    t        j                  dd|       S )Nz-\d+$r   resubr  s    r   _strip_stable_vertex_versionr  `  s    66(B
++r   c                 H    t        j                         j                  |       S r   )r   r[  r\  r  s    r   _strip_bedrock_regionr  d  s    '')99*EEr   r  c                 0    t        j                  dd|       S )a  
    Strips the organization, custom suffix, and ID from an OpenAI fine-tuned model name.

    input: ft:gpt-3.5-turbo:my-org:custom_suffix:id
    output: ft:gpt-3.5-turbo

    Args:
    model_name (str): The full model name

    Returns:
    str: The stripped model name
    z(:[^:]+){3}$r   r  r  s    r   !_strip_openai_finetune_model_namer  h  s     66/2z22r   c                     |r|dk(  rt        |       }|S |r|dk(  s|dk(  rt        |       }|S |r|dk(  rt        |       }|S d| v rt        |       }|S | S )Nr3  r  r  r.  r  zft:)r  r  r  )r   r0  strip_bedrock_regionstrip_versionstrip_finetunes        r   _strip_model_namer  x  sx    2i?4F##	{*.AX.M4F	"5"E4F	%:eLr   c                 (    t         j                  |    S r   )r   rZ  r  s    r   _get_model_info_from_model_costr    s    c""r   rI  c                     |r[d| v rW| d   |k7  rO|dk(  r| d   j                  d      ry|dk(  r| d   j                  d      ry|dk(  r| d   j                  d      ryyy)zG
    Check if the model info provider matches the custom provider.
    rb  r  Tr  r3  F)rx  rI  r0  s     r   _check_provider_matchr
    s     j()*.AA+-*3

*[
!3"  N2z8

*^
$8%  I-*3

*Y
3  r   )	TypedDictc                   @    e Zd ZU eed<   eed<   eed<   eed<   eed<   y)'PotentialModelNamesAndCustomLLMProvidersplit_modelcombined_model_namestripped_model_namecombined_stripped_model_namer0  N)rF  
__module____qualname__r   __annotations__r  r   r   r  r    s     "%%r   r  c           
         |$	 t        |       \  }}}}| }t        | |      }|}n|rJ| j                  |dz         r6| j	                  d      d   }| }t        ||      }dj                  ||      }n3| }dj                  ||       }t        | |      }dj                  ||      }t        ||||t        t        |            S # t        $ r | }Y w xY w)Nr  r;  rg  rr   z{}/{})r  r  r  r  r0  )	r    r   r  rx  ro  r'  r  rg   r   )r   r0  r  rB  r  r  r  s          r   _get_potential_model_namesr    s    "	 5EE5R2K,a $/-@
 (;$	!1!1c!" kk#&q)#/3F
 (/~~!4(
$ %nn-@%H/-@
 (/~~(
$
 3//%A &9: ;  	 K	 s   C CCc                     d|  d}	 t         j                  j                  |      }|j                          |j	                         }|j                  d      }||S y # t
        $ r Y y w xY wr  r  r  s        r   r  r    sx    *:,6KLJ..22:>!!# mmo #.//2K"L".** r     c                     t        | |      S )z_
    _get_model_info_helper wrapped with lru_cache

    Speed Optimization to hit high RPS
    r;  )rH  r;  s     r   _cached_get_model_info_helperr    s     "CVWWr   c                 Z   	 i t         j                  t         j                  }| |v r||    } ||dk(  rd}|U|dk(  rPd| z   t         j                  v rd| z   } n5| dz   t         j                  v r| dz   } n| dz   t         j
                  v r| dz   } t        | |      }|d   }|d   }|d	   }|d
   }|d   }|dk(  r%t        |       }t        | |dddddddddddd      S |dk(  s|dk(  r#t        j                         j                  |       S 	 d}	d}
d}|t         j                  v r|}
t        |
      }	t        |	|      sd}	|	/| t         j                  v r| }
t        |
      }	t        |	|      sd}	|	/|t         j                  v r|}
t        |
      }	t        |	|      sd}	|	/|t         j                  v r|}
t        |
      }	t        |	|      sd}	|	/|t         j                  v r|}
t        |
      }	t        |	|      sd}	|r(|t        v r t        j!                  | t#        |            }|	,|*t%        t&        t(           |j                  | |	            }	d}
|	|
t+        d      |dk(  rd|	d<   |	j-                  d      }|'t/        j0                  dj3                  | |             d}|	j-                  d      }|'t/        j0                  dj3                  | |             d}t        dDi d |
d!|	j-                  d!d      d"|	j-                  d"d      d#|	j-                  d#d      d|d$|	j-                  d$d      d%|	j-                  d%d      d&|	j-                  d&d      d'|	j-                  d'd      d(|	j-                  d(d      d)|	j-                  d)d      d*|	j-                  d*d      d|d+|	j-                  d+d      d,|	j-                  d,d      d-|	j-                  d-d      d.|	j-                  d.d      d/|	j-                  d/d      d0|	j-                  d0d      d1|	j-                  d1d      d2|	j-                  d2|      d3|	j-                  d3      d4|	j-                  d4d      d|	j-                  dd      d5|	j-                  d5d6      d7|	j-                  d7d6      d8|	j-                  d8d6      d9|	j-                  d9d6      d:|	j-                  d:d6      d;|	j-                  d;d6      d<|	j-                  d<d6      d=|	j-                  d=d6      d>|	j-                  d>d      d?|	j-                  d?d      d@|	j-                  d@d      S # t4        $ rG}t/        j0                  dA|        dBt7        |      v r|t5        dCj3                  | |            d}~ww xY w)Ezy
    Helper for 'get_model_info'. Separated out to avoid infinite loop caused by returning 'supported_openai_param's
    Nr  r  zmeta/z@latestr;  r  r  r  r  r0  r-  r  r   r  )r  rd  max_input_tokensr  r  r  rb  r  r7  r?  rF  supports_assistant_prefillrQ  rM  r  r&  r  r	  r  )r   existing_model_infoprovider_specific_model_infoz|This model isn't mapped yet. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.jsonr,  Tr?  r  z`model={}, custom_llm_provider={} has no input_cost_per_token in model_cost_map. Defaulting to 0.r  zamodel={}, custom_llm_provider={} has no output_cost_per_token in model_cost_map. Defaulting to 0.r  rd  r  r  cache_creation_input_token_costcache_read_input_token_costinput_cost_per_character&input_cost_per_token_above_128k_tokensinput_cost_per_queryr  input_cost_per_audio_tokenoutput_cost_per_audio_tokenoutput_cost_per_character'output_cost_per_token_above_128k_tokens+output_cost_per_character_above_128k_tokensr  output_cost_per_imageoutput_vector_sizerb  r  r7  rS  FrF  r  rQ  rK  rO  rM  rV  supports_native_streamingr  r  zError getting model info: OllamaErrorzThis model isn't mapped yet. model={}, custom_llm_provider={}. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json.r  )r   r  azure_embedding_modelsrS  rU  rX  r  r  rQ   ra  rT  rZ  r  r
  rN   r  get_provider_model_inforM   rg   rc   r_   r*  r   rt   r   r'  r   r   )r   r0  r  potential_model_namesr  r  r  r  rd  _model_infor  r  _input_cost_per_token_output_cost_per_tokenr#  s                  r   rH  rH    ss   `
M**Mg.L.LM
Ju%E*/BFV/V"-*/Bk/Q'">">>%"g&C&CC	)"g&C&CC	) :-@!
 44IJ34IJ'<*(
$ ,M:34IJ-/5GJ %!%"&%&&'!.)-)-*.+/(,#'   !H,0C}0T'')88?? 59K!%C:>O"g&8&88)=#F,*@S #'K"u0B0B'B=#F,*@S #'K#0G4F4FF2=#F,*@S #'K"':g>P>P'P)=#F,*@S #'K"{g6H6H'H!=#F,*@S #'K"':o'M"7"O"O,7J*K #P # "'B"TN#22# 3  5"ck  S 
 #k1:>675@__&6! %,$$v}}2
 )*%6Aoo'7" &-$$w~~2
 *+&  FF&??<>F "-1CT!JF #.//2Et"L	F
 &;F 1<5t1F -8OO14-F *5.*F 8C<d8F$ &1__5KT%R%F& '2oo6Mt&T'F( ,7??0$,)F. '=/F0 -8OO14-1F6 +6///+7F< 9D=t9=FB =HOOA4=CFH (37OQU'VIFJ '2oo6Mt&TKFL $/??3G#NMFN "-&(;"OFT !__V,UFV *5.*WF\ *5.*]Fb !,0A5 IcFd +6///+eFj ,7??0%,kFp )4-u)qFv &1__5KU%SwFx '2oo6Mu&UyFz $/??3G#O{F| 0;4e0}FB +6///+CFH  OOE40IFJ  OOE40KF FN  
9!=>CF"G n  u  u*
 	
	
s&   CU ,U 
QU 	V*#AV%%V*c                 l    t        j                  | |      }t        | |      }t        di |d|i}|S )a  
    Get a dict for the maximum tokens (context window), input_cost_per_token, output_cost_per_token  for a given model.

    Parameters:
    - model (str): The name of the model.
    - custom_llm_provider (str | null): the provider used for the model. If provided, used to check if the litellm model info is for that provider.

    Returns:
        dict: A dictionary containing the following information:
            key: Required[str] # the key in litellm.model_cost which is returned
            max_tokens: Required[Optional[int]]
            max_input_tokens: Required[Optional[int]]
            max_output_tokens: Required[Optional[int]]
            input_cost_per_token: Required[float]
            input_cost_per_character: Optional[float]  # only for vertex ai models
            input_cost_per_token_above_128k_tokens: Optional[float]  # only for vertex ai models
            input_cost_per_character_above_128k_tokens: Optional[
                float
            ]  # only for vertex ai models
            input_cost_per_query: Optional[float] # only for rerank models
            input_cost_per_image: Optional[float]  # only for vertex ai models
            input_cost_per_audio_token: Optional[float]
            input_cost_per_audio_per_second: Optional[float]  # only for vertex ai models
            input_cost_per_video_per_second: Optional[float]  # only for vertex ai models
            output_cost_per_token: Required[float]
            output_cost_per_audio_token: Optional[float]
            output_cost_per_character: Optional[float]  # only for vertex ai models
            output_cost_per_token_above_128k_tokens: Optional[
                float
            ]  # only for vertex ai models
            output_cost_per_character_above_128k_tokens: Optional[
                float
            ]  # only for vertex ai models
            output_cost_per_image: Optional[float]
            output_vector_size: Optional[int]
            output_cost_per_video_per_second: Optional[float]  # only for vertex ai models
            output_cost_per_audio_per_second: Optional[float]  # only for vertex ai models
            litellm_provider: Required[str]
            mode: Required[
                Literal[
                    "completion", "embedding", "image_generation", "chat", "audio_transcription"
                ]
            ]
            supported_openai_params: Required[Optional[List[str]]]
            supports_system_messages: Optional[bool]
            supports_response_schema: Optional[bool]
            supports_vision: Optional[bool]
            supports_function_calling: Optional[bool]
            supports_prompt_caching: Optional[bool]
            supports_audio_input: Optional[bool]
            supports_audio_output: Optional[bool]
            supports_pdf_input: Optional[bool]
    Raises:
        Exception: If the model is not mapped yet.

    Example:
        >>> get_model_info("gpt-4")
        {
            "max_tokens": 8192,
            "input_cost_per_token": 0.00003,
            "output_cost_per_token": 0.00006,
            "litellm_provider": "openai",
            "mode": "chat",
            "supported_openai_params": ["temperature", "max_tokens", "top_p", "frequency_penalty", "presence_penalty"]
        }
    r;  r  r  )r   r!   rH  rP   )r   r0  r  r1  returned_model_infos        r   rT  rT    sS    F &AA)< )/K
 $ 
/F r   python_type_namec                     t         j                  dt        j                  dt        j                  dt        j                  dt
        j                  dt        j                  dddi}|j                  | d      S )	zConverts standard python types to json schema types

    Parameters
    ----------
    python_type_name : str
        __name__ of type

    Returns
    -------
    str
        a standard JSON schema type, "string" if not recognized.
    r  r  r   r!  r  r  NoneTyper"  )r   rF  r  floatr  r   r  r   )r6  python_to_json_schema_typess     r   json_schema_typer;  @  s]     	hiywxF# '**+;XFFr   c                 z   	 ddl }ddlm} ddlm} | j                  } |j                  |       } ||      }dj                  |d   D cg c]  }|j                          c}      }	i }
g } |j                  |       j                  }|j                         D ]Z  \  }}t        |d      r t        |j                  j                        }nd}d}d}|d   D ]  }|j                   |k(  st        |d	      rQ|j"                  }d
|v r|j%                  d      d   }n!d|v r	 t'        t)         ||                  }d}t        |      }dj                  |j*                  D cg c]  }|j                          c}      } |||d}t-        |j                         D cg c]  \  }}t/        |t&              s||f c}}      |
|<   |j0                  |j2                  k(  sJ|j5                  |       ] ||	d|
dd}|r||d   d<   |S # t
        $ r}|d}~ww xY wc c}w # t
        $ r Y w xY wc c}w c c}}w )ae  Using type hints and numpy-styled docstring,
    produce a dictionnary usable for OpenAI function calling

    Parameters
    ----------
    input_function : function
        A function with a numpy-style docstring

    Returns
    -------
    dictionnary
        A dictionnary to add to the list passed to `functions` parameter of `litellm.completion`
    r   N)literal_eval)NumpyDocStringr  Summary
annotation
Parametersr   optionalr  {r  )r   r  r  r  r   r  r  r  r  r  r  )r   astr=  numpydoc.docscraper>  r   rF  getdocr   strip	signaturer  rw  r  r;  r@  r  r   ro  r   r   descr  r   r  emptyr   )input_functionr   r=  r>  r#  r  	docstringnumpydocsr  r  r  
param_info
param_namer  
param_typeparam_description
param_enum
param_data
param_dictr  r  r(  s                          r   function_to_dictrX  Z  sR   $5 ""D~.Ii(H))0CD1QWWYDEK JO""">2==J'--/ '/
E5,')%*:*:*C*CDJJ 
 #<0 	TJ*,:v.!+J!Z/%/%5%5c%:1%=

*!),T,z2J-K)LJ)1J "2*!=J$(II*//.RQqwwy.R$S!!	T& ,

 "& * 0 0 2I1jC6HaVI"

:
 ==EKK'"":.O'/V "$
F +:|Z(MI   E:  ) ! ! /S JsA   H 
HH#H2H76H7	HHH#	H/.H/c                 f    t        j                  |       }|j                  |      }t        |      S )Npath)httpxURL	copy_withr   )original_urlnew_pathr  modified_urls       r   
modify_urlrb    s+    
))L
!C==h=/L|r   	num_callsr  c                 f   d}d}|r|}|r|}t        |      D cg c]  }d|dg
 }	}t        j                         }
	 t        j                  | |	|||       t        j                         }||
z
  }|ddd dS c c}w # t        $ r*}t        j                         }||
z
  }|dd|dcY d }~S d }~ww xY w)	NzHey, how's it goingr   r   r  r   )r   r   r0  r   r  success)total_response_time
calls_madestatusr1  failed)rangetimer   batch_completionr   )r   r0  r   r   rc  r  test_prompt
test_callsrB  r   r   r)  response_timer#  s                 r   load_test_modelrq    s     (KJ
DI*DUVq&[9:VHVJ
   3'	
 99; :-#0	
 	
 W$  
99; :-#0	
 	

s"   A89A= =	B0B+%B0+B0c                     | dk(  r"t        j                         j                         S | dk(  r"t        j                         j                         S | dk(  r"t        j                         j                         S g S )z,Return the fields required for each providerr  r  azure_ai)r   rf  get_required_paramsra  AzureAIStudioConfigr0  s    r   get_provider_fieldsrw    sk     l*'')==??		(##%99;;	
	***,@@BB 	r   c                  F   t               j                         D  ci c]  \  } }| |d nt        |       }} }i }i }t        j                  dd       }|r|j                  d      ng }|j                         D ]h  \  } }|/t        j                         || <   t        j                         || <   7t        j                  |      || <   t        j                  |      || <   j |D ]0  }t        j                         ||<   t        j                         ||<   2 ||fS c c}} w )Nra  NO_PROXYr  )proxy)	r   rw  r   osgetenvro  r\  HTTPTransportAsyncHTTPTransport)r  r  proxiessync_proxy_mountsasync_proxy_mountsno_proxyno_proxy_urlsrz  s           r   !create_proxy_transport_and_mountsr    s2    01779C 	S[Ten4G 
  yyT*H+3HNN3'Mmmo L
U=%*%8%8%:c"&+&>&>&@s#%*%8%8u%Ec"&+&>&>U&Ks#L  =!&!4!4!6#"'":":"<3= 0001s   Dc                    d}g }| ||dS 	 t        |       \  }}}}|r|dk(  r)dt        j                  v rd}	n|j	                  d       	n|dk(  rOd	t        j                  v r(d
t        j                  v rdt        j                  v rd}	nO|j                  g d       	n:|dk(  r)dt        j                  v rd}	n|j	                  d       	n|dk(  r)dt        j                  v rd}n|j	                  d       n|dk(  r)dt        j                  v rd}n|j	                  d       n|dk(  r)dt        j                  v rd}n|j	                  d       n|dk(  r=dt        j                  v rdt        j                  v rd}nU|j                  ddg       n@|dk(  r)dt        j                  v rd}n%|j	                  d       n|dk(  r)dt        j                  v rd}n|j	                  d       n|dk(  r)dt        j                  v rd}n|j	                  d       n|dk(  r)dt        j                  v rd}n|j	                  d       n|d k(  r)d!t        j                  v rd}nm|j	                  d!       nZ|d"k(  r)d#t        j                  v rd}n?|j	                  d#       n,|d$k(  s|d%k(  rLd&t        j                  v rd't        j                  v rd}n|j	                  d&       |j	                  d'       n|d(v r)d)t        j                  v rd}n|j	                  d)       n|d*k(  r)d+t        j                  v rd}n|j	                  d+       n{|d,k(  r)d-t        j                  v rd}n`|j	                  d-       nM|d.k(  r)d/t        j                  v rd}n2|j	                  d/       n|d0k(  r)d1t        j                  v rd}n|j	                  d1       n|d2k(  r)d3t        j                  v rd}n|j	                  d3       n|d4k(  r)d5t        j                  v rd}n|j	                  d5       n|d6k(  r)d7t        j                  v rd}nz|j	                  d7       ng|d8k(  r)dt        j                  v rd}nL|j	                  d       n9|d9k(  r)d:t        j                  v rd}n|j	                  d:       n|d;k(  s|d<k(  r)d=t        j                  v rd}n|j	                  d=       n|d>k(  r)d?t        j                  v rd}n|j	                  d?       n|d@k(  r)dAt        j                  v rd}n|j	                  dA       n||dBk(  r)dCt        j                  v rd}na|j	                  dC       nN|dDk(  r)dEt        j                  v rd}n3|j	                  dE       n |dFk(  r)dGt        j                  v rd}n|j	                  dG       n|dHk(  r_dIt        j                  v s6dJt        j                  v s$dKt        j                  v sdLt        j                  v rd}n|j	                  dI       n|dMk(  rdNt        j                  v r(dOt        j                  v sdPt        j                  v rd}nN|j	                  dN       |j	                  dP       n*| t        j                  v s6| t        j                  v s$| t        j                  v s| t        j                  v r)dt        j                  v rd}n|j	                  d       n| t        j                  v r)dt        j                  v rd}n|j	                  d       n~| t        j                  v r)dt        j                  v rd}nV|j	                  d       nC| t        j                  v r)dt        j                  v rd}n|j	                  d       n| t        j                  v r)dt        j                  v rd}n|j	                  d       n| t        j                  v s'| t        j                   v s| t        j"                  d   v r=dt        j                  v rdt        j                  v rd}nl|j                  ddg       nW| t        j$                  v r)dt        j                  v rd}n/|j	                  d       n| t        j&                  v r'dt        j                  v rd}n|j	                  d       n| t        j(                  v r'dt        j                  v rd}n|j	                  d       n| t        j*                  v r'dt        j                  v rd}n|j	                  d       nq| t        j,                  v r'd!t        j                  v rd}nJ|j	                  d!       n8| t        j.                  v r&d#t        j                  v rd}n|j	                  d#       |/g }|D ]&  }dQ|j1                         vs|j	                  |       ( |}|/g }|D ]&  }dR|j1                         vs|j	                  |       ( |}t3        |      dSk(  rd}||dS # t        $ r d}Y 
Dw xY w)Ta  
    Checks if the environment variables are valid for the given model.

    Args:
        model (Optional[str]): The name of the model. Defaults to None.
        api_key (Optional[str]): If the user passed in an api key, of their own.

    Returns:
        dict: A dictionary containing the following keys:
            - keys_in_environment (bool): True if all the required keys are present in the environment, False otherwise.
            - missing_keys (List[str]): A list of missing keys in the environment.
    FN)keys_in_environmentmissing_keysr  r  r  Tr2  AZURE_API_BASEr8  AZURE_API_KEY)r  r8  r  re  r  rd  r  r  r  rf  OPENROUTER_API_KEYr  VERTEXAI_PROJECTr  r-  r  rh  AI21_API_KEYr  r  rj  r  r  r  ri  r  r3  r  AWS_ACCESS_KEY_IDAWS_SECRET_ACCESS_KEY)r  r&  OLLAMA_API_BASEr  ANYSCALE_API_KEYr2  DEEPINFRA_API_KEYr.  GEMINI_API_KEYr  GROQ_API_KEYr  NVIDIA_NIM_API_KEYr  CEREBRAS_API_KEYr   XAI_API_KEYr!  r"  VOLCENGINE_API_KEYr$  r4  CODESTRAL_API_KEYr#  DEEPSEEK_API_KEYr  MISTRAL_API_KEYpalmPALM_API_KEYr3  PERPLEXITYAI_API_KEYr  VOYAGE_API_KEYr  FIREWORKS_AI_API_KEYFIREWORKS_API_KEYFIREWORKSAI_API_KEYFIREWORKS_AI_TOKENr0  CLOUDFLARE_API_KEYCLOUDFLARE_ACCOUNT_IDCLOUDFLARE_API_BASEr  r   r   )r    r   r{  environr   extendr   r  rn  open_ai_embedding_modelsopenai_image_generation_modelsr  r  replicate_modelsrp  rs  rq  models_by_providerhuggingface_modelsru  together_ai_modelsrw  baseten_modelsrv  r  r   )	r   r  r   r  r  rB  r0  new_missing_keysr  s	            r   validate_environmentr    sI	   "   L}#6(
 	

#'7e'D$1 (*2::-&*###$45 G+ BJJ.'2::5#rzz1&*###L !K/"bjj0&*###$78 H,2::-&*###$45 K/"bjj0&*###$78 L0#rzz1&*###$89 K/!RZZ/4G2::4U&*###%79L$MN M1$

2&*###$9: F*+&*###N3 M1#rzz1&*###$89 M1$

2&*###$9: I- BJJ.&*###$56 K/"bjj0&*###$78 I-1D1S#rzz1+rzz9&*###$78##$;< $== BJJ.&*###$56 J.!RZZ/&*###$67 K/"bjj0&*###$78 H,2::-&*###$45 F*+&*###N3 L0#rzz1&*###$89 J.!RZZ/&*###$67 E)

*&*###M2 K/+&*###N3 L0#rzz1&*###$89;."&AA"bjj0&*###$78 J.!RZZ/&*###$67 I- BJJ.&*###$56 F*+&*###N3 L0%3&*###$:; H,2::-&*###$45 N2&"**4&"**4(BJJ6'2::5&*###$:; L0#rzz1'2::5(BJJ6&*###$89##$9: W;;;>>>888>>>2::-&*###$45g..."bjj0&*###$78g+++2::-&*###$45g..."bjj0&*###$78g///#rzz1&*###$89 W///22222;??!RZZ/4G2::4U&*###%79L$MNg000$

2&*###$9:g)))+&*###N3g000#rzz1&*###$89g000$

2&*###$9:g,,, BJJ.&*###$56g..."bjj0&*###$78 	-C		+ '',	- ( 	-C, '',	- (
<A"#6UU}  #"#s   i i#"i#c                  ,    t        j                  | i |S r   )r   r  )r  r   s     r   acreater  O  s    ///r   c                    dj                  d |D              }d}d| v r)	 dd l}ddlm}m}m}  |       }|j                  |      }|S t        t        j                  |            }|S # t        $ r t        d       Y Zw xY w)Nr  c              3   &   K   | ]	  }|d      yw)r   Nr  )r   r   s     r   r   z*prompt_token_calculator.<locals>.<genexpr>U  s     ?7GI&?s   r   claudez:Anthropic import failed please run `pip install anthropic`)	AI_PROMPTHUMAN_PROMPT	Anthropic)
r   re  r   r  r  r  count_tokensr   r   r  )	r   r   r  r  re  r  r  r  anthropic_objs	            r   prompt_token_calculatorr  S  s    88?h??DJ5	T 	A@!"//5
  ./
  	TRS	Ts   A( (A?>A?c                     	 | t         j                  v s| t         j                  v r t        j                  j                  |        y dddg}t        j                  | |       y # t        $ r t        d| d      w xY w)Nr   zHello Worldre  r   r   r   )r   r   r  )	r   r  rn  r  modelsretriever  r   r}   r  s     r   valid_modelr  e  su    H W;;;>>>MM""5)!'MBCHUX> HbBGGHs   AA$ A$ $A<c                 v    dddg}	 t        j                  | ||d       y# t        $ r Y yt        $ r Y yw xY w)aW  
    Checks if a given API key is valid for a specific model by making a litellm.completion call with max_tokens=10

    Args:
        model (str): The name of the model to check the API key against.
        api_key (str): The API key to be checked.

    Returns:
        bool: True if the API key is valid for the model, False otherwise.
    r   zHey, how's it going?re  r  )r   r   r  rd  TF)r   r  r|   r   )r   r  r   s      r   check_valid_keyr  t  sP      ,BCDH(G	
   s   " 	888r  c                 4    | dk(  ry| dk(  ry| dk(  ry| dk\  ryy)a  
    Retries on 408, 409, 429 and 500 errors.

    Any client error in the 400-499 range that isn't explicitly handled (such as 400 Bad Request, 401 Unauthorized, 403 Forbidden, 404 Not Found, etc.) would not trigger a retry.

    Reimplementation of openai's should retry logic, since that one can't be imported.
    https://github.com/openai/openai-python/blob/af67cfab4210d8e497c05390ce14f39105c77519/src/openai/_base_client.py#L639
    i  Ti  i  r  Fr  )r  s    r   _should_retryr    s9     c c c cr   response_headersc                 N   	 ddl }| | j                  d      }	 t        |      }|S d}|S # t        $ r_ |j                  j                  |      }|d}n;|j                  j                  |      }t        |t        j                         z
        }Y |S w xY w# t        $ r d}Y yw xY w)z
    Reimplementation of openai's calculate retry after, since that one can't be imported.
    https://github.com/openai/openai-python/blob/af67cfab4210d8e497c05390ce14f39105c77519/src/openai/_base_client.py#L631
    r   Nzretry-afterr  )emailr   r  r   r  parsedate_tz	mktime_tzrl  )r  r  retry_headerretry_afterretry_date_tuple
retry_dates         r   &_get_retry_after_from_exception_headerr    s     '+//>L@!,/  K  @#(;;#;#;L#I #+"$K!&!6!67G!HJ"%j499;&>"?K @  s7   B + B B A$BB BB B$#B$remaining_retriesr  min_timeoutc                     t        |      }|d|cxk  rdk  r|S  d}d}|| z
  }t        |t        d|      z  |      }ddt        j                         z  z
  }	||	z  }
|
|k\  r|
S |S )Nr   <   g      ?g       @g       @rr   g      ?)r  r  powrandom)r  r  r  r  r  initial_retry_delaymax_retry_delay
nb_retriessleep_secondsjitterr  s              r   _calculate_retry_afterr    s     99IJK 1{#8b#8 $9 O00J +c#z.BBOTM ''Ff$G,7=+=r   rolesinitial_prompt_valuefinal_prompt_valuec                 n    t        |       d   } |||dt        j                  | <   t        j                  S )a  
    Register a prompt template to follow your custom format for a given model

    Args:
        model (str): The name of the model.
        roles (dict): A dictionary mapping roles to their respective prompt values.
        initial_prompt_value (str, optional): The initial prompt value. Defaults to "".
        final_prompt_value (str, optional): The final prompt value. Defaults to "".

    Returns:
        dict: The updated custom prompt dictionary.
    Example usage:
    ```
    import litellm
    litellm.register_prompt_template(
            model="llama-2",
        initial_prompt_value="You are a good assistant" # [OPTIONAL]
            roles={
            "system": {
                "pre_message": "[INST] <<SYS>>
", # [OPTIONAL]
                "post_message": "
<</SYS>>
 [/INST]
" # [OPTIONAL]
            },
            "user": {
                "pre_message": "[INST] ", # [OPTIONAL]
                "post_message": " [/INST]" # [OPTIONAL]
            },
            "assistant": {
                "pre_message": "
" # [OPTIONAL]
                "post_message": "
" # [OPTIONAL]
            }
        }
        final_prompt_value="Now answer as best you can:" # [OPTIONAL]
    )
    ```
    r  r   )r  r  r  )r    r   r  )r   r  r  r  s       r   register_prompt_templater    s=    R 5)!,E 40)Gu%
 %%%r   c                   R    e Zd Z	 	 ddee   dee   fdZd Zd Zde	fdZ
d	 Zd
 Zy)TextCompletionStreamWrapperNr   r0  c                 <    || _         || _        || _        || _        y r   )completion_streamr   r   r0  )selfr  r   r   r0  s        r   __init__z$TextCompletionStreamWrapper.__init__  s#     "3
,#6 r   c                     | S r   r  r  s    r   __iter__z$TextCompletionStreamWrapper.__iter__#      r   c                     | S r   r  r  s    r   	__aiter__z%TextCompletionStreamWrapper.__aiter__&  r  r   r  c           	         	 t               }|j                  dd       |d<   d|d<   |j                  dd       |d<   |j                  dd       |d<   t               }t        |t              rt
        |d   d   d   d	   |d
<   |d   d   d   |d<   |d   d   d   |d<   |g|d<   | j                  r3| j                  j                  dd      du r|j                  dd       |d<   |S # t
        $ r}t        d| dt        |             d }~ww xY w)Nr   r  r  createdr   rG  r   r  r   r  r  finish_reasoninclude_usageFTusagez=Error occurred converting to text completion object - chunk: z	; Error: )rW   r   rV   r   rE   r   r   r   )r  r  rB  text_choicesr#  s        r   !convert_to_text_completion_objectz=TextCompletionStreamWrapper.convert_to_text_completion_object)  sE   	-/H"YYtT2HTN!2HX"'))It"<HY %		'4 8HW&=Lw  #(#3A#6w#?	#JL $))$4Q$7$@L!,1),<Q,?,PL)#/.HY ##''++OUCtK$)IIgt$<!O 	OPUwV_`cde`f_gh 	s   C!C$ $	D-DDc                    t                	 | j                  D ]$  }|dk(  s|t        | j                  |      }|c S  t        # t        $ r t        t        $ r-}t        | j                  | j                  xs d|i i       d }~ww xY w)NNoner  r   )r   r0  original_exceptioncompletion_kwargsextra_kwargs)rW   r  r   r  StopIterationr   r   r0  )r  r  processed_chunkr#  s       r   __next__z$TextCompletionStreamWrapper.__next__G  s     	// 'F?em#O"&"H"Hu"H"U&&	'
   	  	 jj$($<$<$B#$"$ 	s   1A A B	(BB	c                    K   	 | j                   2 3 d {   }|dk(  s|t        | j                  |      }|c S 7 '6 t        # t        $ r t        w xY ww)Nr  r  )r  r   r  r  StopAsyncIteration)r  r  r  s      r   	__anext__z%TextCompletionStreamWrapper.__anext__\  sn     	%#55 ' 'eF?em#O"&"H"Hu"H"U&&	'5
   	%$$	%s5   AA ><>#A A>A AANN)rF  r  r  rc   r  r   r  r  r  rR   r  r  r  r  r   r   r  r    sK    
 *.-1
7 !	
7
 &c]
7} <*	%r   r  c              #     K   t        |t        j                        r|t        dt	        |      d      D ]  }t        d|||dz          }||| j                  d   _        npg }t        |      D ]Y  }t        j                  j                  |t        j                  j                  d|||dz                }|j                  |       [ || _        |   y wNr   r  	assistantre  )r  r  r   r   MockExceptionrk  r   rH   rG  r  r  rU   r   	rQ  mock_responser   r  icompletion_obj_all_choicesj_streaming_choices	            r   mock_completion_streaming_objr  h  s      -!6!671c-(!, Kq1q59QR9.<N""1%+L1X 7$+MM$B$B!----(-AE2J .  %C %! ##$567 &2N"s   CCc                  K   t        |t        j                        r|t        dt	        |      d      D ]  }t        d|||dz          }||| j                  d   _        npg }t        |      D ]Y  }t        j                  j                  |t        j                  j                  d|||dz                }|j                  |       [ || _        |   y wr  r  r  s	            r   #async_mock_completion_streaming_objr    s      -!6!671c-(!, Kq1q59QR9.<N""1%+L1X 7$+MM$B$B!----(-AE2J .  %C %! ##$567 &2N"s   CCc                     	 dd l } |j                          t        | d      5 }t        j                  |      }d d d        |S # 1 sw Y   S xY w# t
        $ r}|d }~ww xY w)Nr   r)r{  getcwdopenr  loadr   )config_pathr{  config_fileconfigr#  s        r   read_config_argsr    sa    
		+s# 	,{YY{+F	, 		,  s3   !A A	A AA A 	A AA c                     d| d}t        |g|      }||kD  r%t        d       t        |||      }t        |g|      }|||z
  fS )Nr  re  zH`tokentrimmer`: Warning, system message exceeds token limit. Trimming...)get_token_countr   shorten_message_to_fit_limit)system_messagerd  r   system_message_eventsystem_message_tokensnew_system_messages         r   process_system_messager    si    $,H+-A,BEJz)V	
 : *e
 !01C0De L.C!CCCr   c                 z    | d d d   } g }| D ]+  }t        ||      }||z
  }|dk  r |S t        |||||      }- |S )Nr  r  )final_messagesr   available_tokensrd  r   )r  attempt_message_addition)r   rd  r   r!  r   used_tokensr"  s          r   process_messagesr%    sp    "~HN 
%ne<%3q   2)-!

 r   c                     |g| z   }t        ||      }||k  r|S d|vr!t        |||      }t        || ||      r|g| z   S | S )Nr   r   r  )r  r  can_add_message)r!  r   r"  rd  r   temp_messagestemp_message_tokensupdated_messages           r   r#  r#    sg     I.M)=Nj( 
	'6w@PRWX?NJN#$~55r   c                 ,    t        || gz   |      |k  ryyr;  )r  )r   r   rd  r   s       r   r(  r(    s    x7)+U3zAr   c                     t        ||       S )Nr  )r/  r'  s     r   r  r    s    ux88r   c                     |d|v r|dk  r| S | d   }	 t        | g|      }||k  r	 | S ||z  }t        t        |      |z        dz
  }t        d|      }|dz  }|d| }|| d }	|dz   |	z   }
|
| d<   |
}`)	z_
    Shorten a message to fit within a token limit by removing characters from the middle.
    Ngptr  r   rr   r   r  z..)r  r  r   r  )r   tokens_neededr   r   total_tokensratio
new_lengthhalf_length	left_half
right_halftrimmed_contents              r   r  r    s     Ue^0Ci G
&y%8=( N ,.W-.2
J'
 AoL[)	k\]+
#d*Z7,	!# r   
trim_ratioreturn_response_tokensc                    t        j                  |       } 	 |Z|t        j                  v rFt        j                  |   j	                  dt        j                  |   d         }t        ||z        }n| S d}| D ]  }|d   dk(  s||rdndz  }||d   z  } g }t        |       D ]  }|d   d	k7  r n|j                  |        t        |      r| dt        |        } t        |xs d| 
      }	t        d|	 d|        |	|k  r| S t        d|  d|	 d|        d}
|r1t        |||      \  }
}|dk(  r|
gS | D cg c]  }|d   dk7  s| } }t        | ||      }|
r|
g|z   }t        |      dkD  r|j                  |       |r|t        ||      z
  }||fS |S c c}w # t        $ r9}t!        j"                  dj%                  t'        |                   | cY d}~S d}~ww xY w)a  
    Trim a list of messages to fit within a model's token limit.

    Args:
        messages: Input messages to be trimmed. Each message is a dictionary with 'role' and 'content'.
        model: The LiteLLM model being used (determines the token limit).
        trim_ratio: Target ratio of tokens to use after trimming. Default is 0.75, meaning it will trim messages so they use about 75% of the model's token limit.
        return_response_tokens: If True, also return the number of tokens left available for the response after trimming.
        max_tokens: Instead of specifying a model or trim_ratio, you can specify this directly.

    Returns:
        Trimmed messages and optionally the number of tokens available for response.
    Nr  rd  r   r  r  r  r   r	  r  zCurrent tokens: z, max tokens: zNeed to trim input messages: z, current_tokensz, max_tokens: )r  rd  r   r   )r   rd  r   z'Got exception while token trimming - {})r   r   r   rZ  r   r  r   r   r   r/  r   r  r%  r  r  r   rt   r1  r'  r   )r   r   r8  r9  rd  max_tokens_for_modelr  r   tool_messagescurrent_tokensr  r!  response_tokensr#  s                 r   trim_messagesr?    s]   , }}X&HL***'.'9'9%'@'D'D&(:(:5(A,(O($ !!5
!BC

   	5Gv(*.$b@')"44	5 ) 	*Gv&(  )	*
 } 53}#5"56H&U[b8L((8zlST J&O 	+H:5EnEUUcdncop	
 04/E-*E0, * Q,-- 08WG76?h;VWHW)*E

  23nDN}!!!-0 #(?>5+QQO!?22% X&    5<<SVD	
 	sV   AF  3F  BF  0F  7F  ;F	FAF  F  F   	G").GG"G"check_provider_endpointc                    	 t         j                  j                         }g }g }t        j                  D ]V  }|j                  dd      }|}|j                          d}|j                          d}||v s||v sF|j                  |       X |D ]  }t        j                  dt        |            }	|dk(  r|j                  d       :|	"| r |j                  |	j                                ^t        j                  j                  |g       }
|j                  |
        |S # t        $ r$}t!        j"                  d|        g cY d}~S d}~ww xY w)	z
    Returns a list of valid LLMs based on the set environment variables

    Args:
        check_provider_endpoint: If True, will check the provider's endpoint for valid models.

    Returns:
        A list of valid LLMs
    rB  r   _API_KEYNr  r2  z	Azure-LLMzError getting valid models: )r{  r  r  r   r   r,  upperr   r  r/  rM   r  
get_modelsr  r   r   rt   r   )r@  environ_keysvalid_providersvalid_modelsr   env_provider_1env_provider_2expected_provider_key_1expected_provider_key_2r  models_for_providerr#  s               r   get_valid_modelsrM  w  sb   'zz(-- 	1H%--c26N%N *8)=)=)?(@&I#)7)=)=)?(@&I#'<7*l:  &&x0	1  ( 	9H3KK%h/ L O
 7"##K0 ,1H##O$>$>$@A&-&@&@&D&DXr&R###$78	9  ;A3?@	s%   A7D" :B'D" "	E+E
E
Ec           	         t               sy 	 d|v r|d   du r| j                  dk(  ry d|v r|d   du r| j                  dk(  ry d|v r|d   du r| j                  dk(  ry dj                  t        t        |            }dj                  d	 |j                         D              }t        d
       t        d       |r"|r t        d| j                   d| d| d       nW|rt        d| j                   d| d       n8|rt        d| j                   d| d       nt        d| j                   d       t        d
       y # t        $ r Y y w xY w)Nr  Tr  r	  r  ra  img_generationz, c              3   B   K   | ]  \  }}| d t        |         yw)=N)repr)r   r  r  s      r   r   z/print_args_passed_to_litellm.<locals>.<genexpr>  s#     W*#u#aU}5Ws   r  z[92mRequest to litellm:[0mz[92mlitellm.(z)[0mz()[0m)rs   rF  r   maprR  rw  r   r   )r   r  r   args_str
kwargs_strs        r   rq  rq    s   . V#}%-!**l:F"|$,!**k9'()T1!**.>>99St_-YYWWW
	
 	0	
 F"#4#=#=">azJ<W_` "#4#=#=">azR "#4#=#=">a
|8T ,->-G-G,H	RSd s#   E  E  E  C!E   	EEc                 x    	 d| j                  d      z   dz   |j                  d      z   }|S # t        $ r Y y w xY w)Nztime-z%H-%M-%S-%frB  r   )strftimer   r   )r   r  response_ids      r   get_logging_idrZ    sO    j))-883>AQAQRVAWW 	  s   *- 	99c                 b    | y | *| j                  di       }||j                  dd       }||S y )NrI  rf  r  )r   rI  rf  s      r   r  r    sH     \\,3
!#d;J%!!r   c                     | y | j                  di       }|4|j                  dd       }||S |j                  di       }t        |      S y )Nr   rf  r   r  )r   r  )model_call_detailsr   _base_modelr   s       r   _get_base_model_from_metadatar_    sb    !'++,<bAN!$((t<"!%%j"598LLr   c                   6    e Zd Zd	dedefdZd Zd Zd Zd Z	y)
ModelResponseIteratorrQ  convert_to_deltac                     |du rdt        d      | _        | j                  j                  d   j                  }|j                  d   j                  j
                  |_        d| _        y || _        d| _        y )NT)r   r   F)rR   rQ  rG  r  r   r   is_done)r  rQ  rb  _deltas       r   r  zModelResponseIterator.__init__  sk    t#"/t"<D((00399F+33A6>>FFFN  #1Dr   c                     | S r   r  r  s    r   r  zModelResponseIterator.__iter__  r  r   c                 L    | j                   rt        d| _         | j                  S NT)rd  r  rQ  r  s    r   r  zModelResponseIterator.__next__  s"    <<"""r   c                     | S r   r  r  s    r   r  zModelResponseIterator.__aiter__  r  r   c                 T   K   | j                   rt        d| _         | j                  S wrh  )rd  r  rQ  r  s    r   r  zModelResponseIterator.__anext__  s&     <<$$"""s   &(NF)
rF  r  r  rR   r  r  r  r  r  r  r  r   r   ra  ra    s)    }  ##r   ra  c                   *    e Zd Zd Zd Zd Zd Zd Zy)ModelResponseListIteratorc                      || _         d| _        y Nr   )model_responsesr  )r  rp  s     r   r  z"ModelResponseListIterator.__init__'  s    .
r   c                     | S r   r  r  s    r   r  z"ModelResponseListIterator.__iter__,  r  r   c                     | j                   t        | j                        k\  rt        | j                  | j                      }| xj                   dz  c_         |S Nrr   )r  r   rp  r  r  rQ  s     r   r  z"ModelResponseListIterator.__next__/  sF    ::T1122--djj9

a
r   c                     | S r   r  r  s    r   r  z#ModelResponseListIterator.__aiter__7  r  r   c                    K   | j                   t        | j                        k\  rt        | j                  | j                      }| xj                   dz  c_         |S wrs  )r  r   rp  r  rt  s     r   r  z#ModelResponseListIterator.__anext__:  sJ     ::T1122$$--djj9

a
r0  N)rF  r  r  r  r  r  r  r  r  r   r   rm  rm  &  s    
r   rm  c                         e Zd Zd fdZ xZS )CustomModelResponseIteratorc                 "    t         |           y r   )superr  )r  	__class__s    r   r  z$CustomModelResponseIterator.__init__C  s    r   )r2  N)rF  r  r  r  __classcell__)r{  s   @r   rx  rx  B  s     r   rx  r   c                     d| vry| d   t        | d   t              ry| d   D ]*  }|d   dk(  s|j                  d      |d   d   dk(  s* y y)z
    Returns true, if message is marked as needing to be cached.

    Used for anthropic/gemini context caching.

    Follows the anthropic format {"cache_control": {"type": "ephemeral"}}
    r   Fr   r  cache_control	ephemeralTr   )r   r   s     r   is_cached_messager  G  st     y!Z	0BC%H9% FOv%O,8(0K? r   rP  c                     	 | j                  d      sy| j                  d      d   } t        j                  | d      }t        j                  |      j                  d      | k(  S # t        $ r Y yw xY w)Nzdata:Fr  rr   T)validatezutf-8)rx  ro  base64	b64decode	b64encoder  r   )rP  decoded_bytess     r   is_base64_encodedr  _  su    ||
 GGCLO ((T: .55g>!CC s   A& AA& &	A21A2c                 6    d| v r| j                  d      d   S | S )z3
    s: b64str OR data:image/png;base64,b64str
    r  rr   )ro  )rP  s    r   get_base64_strr  r  s"     axwws|AHr   c                 8    | D ]  }|j                  d       y y)zm
    Returns true, if messages has tool call blocks.

    Used for anthropic/bedrock message validation.
    r%  TFr  )r   r   s     r   has_tool_call_blocksr  {  s*      ;;|$0 r   c           
      <    t        dt        dddi d            gS )z
    Prevent Anthropic from raising error when tool_use block exists but no tools are provided.

    Relevent Issues: https://github.com/BerriAI/litellm/issues/5388, https://github.com/BerriAI/litellm/issues/5747
    r  
dummy_toolzThis is a dummy tool callr  rD  rE  )r   r  )r<   r=   rv  s    r   add_dummy_toolr    s2     	 9!7$"$
	
 r   )ChatCompletionAudioObjectChatCompletionImageObjectChatCompletionTextObjectChatCompletionUserMessageOpenAIMessageContentValidUserMessageContentTypesc                     t        | t              r| j                  d      S t        | t              r| S t	        dt        |        d      )z
    Converts a message to a dictionary if it's a Pydantic model.

    Args:
        message: The message, which may be a Pydantic model or a dictionary.

    Returns:
        dict: The converted message.
    T)exclude_nonezInvalid message type: z". Expected dict or Pydantic model.)r   r   
model_dumpr  rN  r   )r   s    r   convert_to_dictr    sO     '9%!!t!44	GT	"$T']O3UV
 	
r   c                     | D cg c])  }t        t        t        t        t        |                  + } }t	        |       S c c}w )zI
    Ensures all messages are valid OpenAI chat completion messages.
    rW  )rg   r8   r  r  &validate_chat_completion_user_messages)r   r   s     r   !validate_chat_completion_messagesr    sE     IQCDtD!}=>H  28DD	s   .Ac           	      ~   t        |       D ]|  \  }}	 |d   dk(  rn|j                  d      }|[t        |t              r3t        |t              r:|D ]5  }t        |t
              s|j                  d      t        vs,t        d       ~ | S # t        $ r&}dt        |      v rt        d| d| d	      |d}~ww xY w)
a;  
    Ensures all user messages are valid OpenAI chat completion messages.

    Args:
        messages: List of message dictionaries
        message_content_type: Type to validate content against

    Returns:
        List[dict]: The validated messages

    Raises:
        ValueError: If any message is invalid
    r  r   r   Nr   zinvalid content typezInvalid user message=z
 at index zL. Please ensure all user messages are valid OpenAI chat completion messages.)r   r   r   r   r   r  r  r   )r   r  r   user_contentr#  r#  s         r   r  r    s     H% Q	yF" uuY/+!,4 #L$7$0 LD)$5#'88F#3;W#W*34J*K$KL( O  	%Q/+A3j  >J  K  	s(   +B%B%B=B	B<!B77B<c                     ddl m}m} | | S t        | t              r| S t        | t
              r3| j                  d      | j                  d      t        d|  d      | S t        d|  dt        |        d      )	z
    Confirm the tool choice is passed in the OpenAI format.

    Prevents user errors like: https://github.com/BerriAI/litellm/issues/7483
    r   )#ChatCompletionToolChoiceObjectParam$ChatCompletionToolChoiceStringValuesr   r  z!Invalid tool choice, tool_choice=z3. Please ensure tool_choice follows the OpenAI specz. Got=zW. Expecting str, or dict. Please ensure tool_choice follows the OpenAI tool_choice spec)	litellm.types.llms.openair  r  r   r   r  r   r   r   )r  r  r  s      r   $validate_chat_completion_tool_choicer    s    
 	K	%	K	&??6"*kooj.I.Q3K=@st  

+K=tK?P>Q  Ri  	j r   c                       e Zd Zedededefd       Zedededefd       Z	ededede
fd       Zedededee   fd       Zedededefd       Zedee   dedee   fd	       Zedededee   fd
       Zy)r  r   r   r2  c                    |t         j                  k(  r4t        j                  j	                  |       rt        j
                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                   |k(  rt        j"                         S t        j                   j$                  |k(  rt        j&                         S t        j                   j(                  |k(  rt        j*                         S t        j                   j,                  |k(  rt        j.                         S t        j                   j0                  |k(  rt        j2                         S t        j                   j4                  |k(  rd| v 	rt        j6                         S t        j                   j8                  |k(  rt        j:                         S t        j                   j<                  |k(  rt        j>                         S t        j                   j@                  |k(  rt        jB                         S t        j                   jD                  |k(  rt        jF                         S t        j                   jH                  |k(  rt        jJ                         S t        j                   jL                  |k(  rt        jN                         S t        j                   jP                  |k(  rt        jR                         S t        j                   jT                  |k(  rt        jV                         S t        j                   jX                  |k(  rt        jZ                         S t        j                   j\                  |k(  sWt        j                   j^                  |k(  s:t        j                   j`                  |k(  st        j                   jb                  |k(  rt        jd                         S t        j                   jf                  |k(  rt        jh                         S t        j                   jj                  |k(  rt        jl                         S t        j                   jn                  |k(  rt        jp                         S t        j                   jr                  |k(  rt        jt                         S t        j                   jv                  |k(  rt        jx                         S t        j                   jz                  |k(  rt        j|                         S t        j                   j~                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  st        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rLt        j                         j                  |       rt        j                         S t        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   jj                  |k(  rt        jl                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  st        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j8                  |k(  rt        j:                         S t        j                   j0                  |k(  rt        j2                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         S t        j                   j                  |k(  rt        j                         j                  |       }|t        j                  v r	 t        j                         S d| v rt        j                         S d| v rt        j                         S d| v rt        j                         S d| v rt        j                         S d| v rt        j                         S t        j                         S )zC
        Returns the provider config for a given provider.
        r  r  amazonmetarh  rd  r  )}rM   OPENAIr   openAIO1Configis_model_o1_reasoning_modelOpenAIO1ConfigDEEPSEEKDeepSeekChatConfigGROQro  
DATABRICKSrf  XAIri  TEXT_COMPLETION_OPENAIOpenAITextCompletionConfigCOHERE_CHATrF  COHERErE  CLARIFAIClarifaiConfig	ANTHROPICrC  ANTHROPIC_TEXTrD  	VERTEX_AIrQ  
CLOUDFLAREr`  SAGEMAKER_CHATSagemakerChatConfig	SAGEMAKERrZ  rA  rk  
FRIENDLIAIFriendliaiChatConfigWATSONXrr  WATSONX_TEXTr?  EMPOWEREmpowerChatConfigGITHUBGithubChatConfigCUSTOMCUSTOM_OPENAIOPENAI_LIKELITELLM_PROXYrw  AIOHTTP_OPENAIAiohttpOpenAIChatConfigHOSTED_VLLMrm  	LM_STUDIOLMStudioChatConfig	GALADRIELGaladrielChatConfig	REPLICATErI  HUGGINGFACErK  TOGETHER_AIrL  
OPENROUTERrq  GEMINIrP  AI21	AI21_CHATrj  AZURErt  ru  r;  AZURE_AIru  
AZURE_TEXTAzureOpenAITextConfig	NLP_CLOUDrc  	OOBABOOGAOobaboogaConfigOLLAMA_CHATrb  	DEEPINFRAre  
PERPLEXITYPerplexityChatConfigMISTRAL	CODESTRALrW  
NVIDIA_NIMrg  CEREBRASrh  
VOLCENGINErl  TEXT_COMPLETION_CODESTRALrV  	SAMBANOVASambanovaConfigMARITALKrH  VLLMrn  OLLAMAra  r@  rJ  TRITONrG  PETALSrd  BEDROCKr[  r\  r]  AmazonTitanConfigAmazonLlamaConfigAmazonAI21ConfigAmazonCohereConfigAmazonMistralConfigOpenAIGPTConfig)r   r   rf  s      r   rA  z.ProviderConfigManager.get_provider_chat_config  s 	    +++&&BBBO))++!!**h6--//!!&&(2))++!!,,8++--!!%%1((**!!88HD5577!!--9++--!!((H4''))!!**h6))++!!++x7**,,!!00H<..00!!++x75 6688!!,,8//11!!00H<..00!!++x7**,,!!..(:,,..!!,,8//11!!))X5//11!!..(:--//!!))X5,,..!!((H4++--  ''83##11X=##//8;##11X=//11!!00H<2244!!--9//11!!++x7--//!!++x7..00!!++x7**,,!!--9,,..!!--9++--!!,,8++--!!((H45577  %%1##--9))++!!''83**,88u8E2244,,..!!**h6..00!!,,80022!!--9//11!!++x7))++!!++x7**,,!!--9++--!!++x7**,,!!,,8//11  ((H4##--9((**!!,,8**,,!!**h6))++!!,,8++--!!;;xG88::!!++x7**,,!!**h6))++!!,,8//11!!00H<..00!!&&(2%%''!!((H4''))!!++x7**,,!!((H4''))!!((H4''))!!))X5 557GGNJW<<< &&(( U"0022500225//11U"1133e#2244&&((r   c                 Z   t         j                  j                  |k(  rt        j                         S t         j                  j                  |k(  rt        j
                         S t         j                  j                  |k(  rt        j                         S t        d|j                   d      )Nr  z" does not support embedding config)
r   rM   VOYAGEr  r  r  r  IBMWatsonXEmbeddingConfigr*  r  r  s     r   get_provider_embedding_configz3ProviderConfigManager.get_provider_embedding_config  s    
 &&(20022!!((H40022!!))X544669X^^$44VWXXr   c                 P   t         j                  j                  |k(  rt        j                         S t         j                  j                  |k(  rt        j
                         S t         j                  j                  |k(  rt        j                         S t        j                         S r   )r   rM   r  CohereRerankConfigr  AzureAIRerankConfigINFINITYInfinityRerankConfigr  s     r   get_provider_rerank_configz0ProviderConfigManager.get_provider_rerank_config  s|    
 &&(2--//!!**h6..00!!**h6//11))++r   c                     t         j                  j                  |k(  rt        j                         S t         j                  j                  |k(  rt        j
                         S y r   )r   rM   rA  #FireworksAIAudioTranscriptionConfigDEEPGRAM DeepgramAudioTranscriptionConfigr  s     r   r  z=ProviderConfigManager.get_provider_audio_transcription_config  sM    
 ,,8>>@@!!**h6;;==r   c                     t         j                  |k(  rt        j                         S t         j                  |k(  rt        j
                         S t        j                         S r   )rM   rA  r   FireworksAITextCompletionConfigr  TogetherAITextCompletionConfigr  r  s     r   #get_provider_text_completion_configz9ProviderConfigManager.get_provider_text_completion_config  sM    
 $$0::<<%%199;;1133r   c                 <   t         j                  |k(  rt        j                         S t         j                  |k(  rt        j
                         S t         j                  |k(  rt        j                         S t         j                  |k(  rt        j                         S y r   )
rM   rA  r   rk  r  r  r  LiteLLMProxyChatConfigTOPAZTopazModelInfor  s     r   r/  z-ProviderConfigManager.get_provider_model_info  s{    
 $$0,,..  H,**,,''8311338+))++r   c                     t         j                  |k(  rt        j                         S t         j                  |k(  rt        j
                         S y r   )rM   r  r   OpenAIImageVariationConfigr
  TopazImageVariationConfigr  s     r   #get_provider_image_variation_configz9ProviderConfigManager.get_provider_image_variation_config  sA    
 (*55778+4466r   N)rF  r  r  staticmethodr   rM   rm   rA  ro   r  rq   r  rc   rj   r  rn   r  rk   r/  rp   r  r  r   r   r  r    sa   S)S)*S)	S) S)j 
Y
Y
Y 

Y 
Y 
,
,
, 

, 
,  
.	/  444 
"4 4 } 
"	#   
*	+ r   r  service_type)r   
prometheusc                    t        t        | j                  di       xs i       }t        t        t           | j                  d      xs |j                  d            }t
        j                  ry|dk(  rt
        j                  ry|S )z
    Used for enforcing `disable_end_user_cost_tracking` param.

    service_type: "litellm_logging" or "prometheus" - used to allow prometheus only disable cost tracking.
    r   user_api_key_end_user_idNr  )rg   r  r   rc   r   r   disable_end_user_cost_tracking.disable_end_user_cost_tracking_prometheus_only)r   r  	_metadataend_user_ids       r   !get_end_user_id_for_cost_trackingr    s|     T>--j"=CDI56 	5==34K
 --$BBr   c                     	 ||y|| j                  |      s|dz   | z   } t        ||| d      }|dk\  S # t        $ r"}t        j                  d|        Y d}~yd}~ww xY w)z
    Returns true if the prompt is valid for prompt caching.

    OpenAI + Anthropic providers have a minimum token count of 1024 for prompt caching.
    NFrg  T)r   r  r   r  i   z)Error in is_prompt_caching_valid_prompt: )rx  r/  r   rt   r   )r   r   r  r0  token_countr#  s         r   is_prompt_caching_valid_promptr    s    *53C3C4
 (#-5E#*.	
 d"" HLMs   6 .6 	A!AA!srt_or_vtt_contentc                     d}t        j                  ||       }|syg }|D ]<  }t        t        |      \  }}}}|dz  |dz  z   |z   |dz  z   }	|j	                  |	       > |rt        |      S dS )a  
    Extracts the total duration (in seconds) from SRT or VTT content.

    Args:
        srt_or_vtt_content (str): The content of an SRT or VTT file as a string.

    Returns:
        Optional[float]: The total duration in seconds, or None if no timestamps are found.
    z"(\d{2}):(\d{2}):(\d{2})[.,](\d{3})Ni  r  g     @@)r  findallrT  r  r   r  )
r  timestamp_pattern
timestamps	durationsmatchhoursminutessecondsmillisecondstotal_secondss
             r    extract_duration_from_srt_or_vttr)  "  s     >-/ABJ I (03C-ww|3g=v@UU'(
 '3y>0D0r   ending_pathc                 X   t        j                  |       }|j                  i       }|j                  j	                  d      }|j                  d      }|j                  d      D cg c]  }|s|	 }}|j                  d      D cg c]  }|s|	 }}g }	t        t        |            D ]#  }
||
d |dt        |      |
z
   k(  s|d|
 |z   }	 n ||z   }	ddj                  |	      z   }|j                  |      }t        |j                  |j                              S c c}w c c}w )z
    Adds an ending path to an API base URL while preventing duplicate path segments.

    Args:
        api_base: Base URL string
        ending_path: Path to append to the base URL

    Returns:
        Modified URL string with proper path handling
    )paramsrg  NrZ  )r\  r]  r^  r[  rstriplstripro  rk  r   r   r   r,  )r   r*  r_  base_url	base_pathend_pathrP  base_segmentsend_segmentsfinal_segmentsr  modified_pathra  s                r   _add_path_to_api_baser6  A  s;    99X&L%%R%0H!!((-I!!#&H !* 4:1Q:M:'~~c28!aA8L8 N3}%& 6-Es=/AA/E FF*2A.=N6 '5 #((>22M%%=%9L |%%\-@-@%ABB% ;8s   'D"/D"D'D'c                     t         j                  }|t        z   }| j                         D ci c]  \  }}||vs|| }}}|S c c}}w r   )r   OPENAI_CHAT_COMPLETION_PARAMSrZ   rw  )r   openai_paramsr  r  r  r  s         r   !get_non_default_completion_paramsr:  g  sU    99M"%77NA1N+B1  s
   AA)Fr   rk  r   )r   r   N)Nzgpt-3.5-turbo-0613NFNNFF)mainN)r   NNNFNNF)%NiX  FNFFFFNNNNNNNNNNNNNNNNNNNNNNNNNNNNN)NNNNNNN)	NNNNNNNNN)NNNr   NN)NNNNNFNNNNNNNNNNNr   NNNNNNNNNNNNN)r   r   r   r   r   )NNNro  )r   r   )Ng      ?FN)r   r  (  rF  r+  r  binasciir   rr  hashlibr   io	itertoolsr  loggingr{  r  r  struct
subprocesssystextwrap	threadingrl  r  rt  dataclassesr   r   	functoolsr   r   	importlibr   r   os.pathr	   r
   r   aiohttpdotenvr\  r  r  r   httpx._utilsr   
openai.libr   r   *openai.types.chat.completion_create_paramsr   pydanticr   r   
tokenizersr   r   litellm._service_loggerlitellm.litellm_core_utils,litellm.litellm_core_utils.audio_utils.utils/litellm.litellm_core_utils.json_validation_rule#litellm.caching._internal_lru_cacher   litellm.caching.cachingr   litellm.caching.caching_handlerr   r   "litellm.integrations.custom_loggerr   'litellm.litellm_core_utils.core_helpersr   r   +litellm.litellm_core_utils.default_encodingr   2litellm.litellm_core_utils.exception_mapping_utilsr   r   r   1litellm.litellm_core_utils.get_llm_provider_logicr   r    6litellm.litellm_core_utils.get_supported_openai_paramsr!   ,litellm.litellm_core_utils.llm_request_utilsr"   Flitellm.litellm_core_utils.llm_response_utils.convert_dict_to_responser#   r$   r%   r&   r'   :litellm.litellm_core_utils.llm_response_utils.get_api_baser(   Blitellm.litellm_core_utils.llm_response_utils.get_formatted_promptr)   9litellm.litellm_core_utils.llm_response_utils.get_headersr*   ?litellm.litellm_core_utils.llm_response_utils.response_metadatar+   *litellm.litellm_core_utils.redact_messagesr,   r-    litellm.litellm_core_utils.rulesr.   ,litellm.litellm_core_utils.streaming_handlerr/   (litellm.litellm_core_utils.token_counterr0   r1   &litellm.llms.custom_httpx.http_handlerr2   r3   *litellm.router_utils.get_retry_from_policyr4   r5   litellm.secret_managers.mainr6   litellm.types.llms.anthropicr7   r  r8   r9   r:   r;   r<   r=   r>   litellm.types.rerankr?   litellm.types.utilsr@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   rS   rT   rU   rV   rW   rX   rY   rZ   	open_textfr  	json_datadumpsr  importlib.metadataconcurrent.futuresr[   typingr\   r]   r^   r_   r`   ra   rb   rc   rd   re   rf   rg   rh   ri   OriginalError8litellm.llms.base_llm.audio_transcription.transformationrj    litellm.llms.base_llm.base_utilsrk   rl   )litellm.llms.base_llm.chat.transformationrm   /litellm.llms.base_llm.completion.transformationrn   .litellm.llms.base_llm.embedding.transformationro   5litellm.llms.base_llm.image_variations.transformationrp   +litellm.llms.base_llm.rerank.transformationrq   _loggingrs   rt   caching.cachingru   rv   rw   rx   ry   r  rz   r{   r|   r}   r~   r   r   r   r   r   r   r   r   r   proxy._typesr   r   types.llms.openair   r   r   types.routerr   MAX_THREADSr  sentry_sdk_instancecapture_exceptionr   posthog	slack_appalerts_channelheliconeLoggerathinaLoggerpromptLayerLoggerlangsmithLoggerlogfireLoggerweightsBiasesLoggercustomLoggerlangFuseLoggeropenMeterLogger
lagoLoggerdataDogLoggerprometheusLoggerdynamoLoggers3LoggergenericAPILoggergreenscaleLoggerlunaryLoggeraispendLoggersupabaseClientr   r   r  r  r   r   last_fetched_atlast_fetched_at_keysr  r   r   r$  r/  r   r  r8  r  r  rn  r}  r  r  r  r  r   r  r  r  r  r  r  r/  r5  r7  r?  rF  r:  rK  rM  rO  rQ  rS  rV  rX  r_  r~  r  r  r  r  r  r  r  r  r  r  r{  r}  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r
  r  r  r  r  r  rH  rT  r;  rX  rb  rq  rw  r  r  r  r  r  r  r  Headersr  r9  r  r  r  r  r  r  r  r%  r#  r(  r  r  r?  rM  rq  rZ  r  r_  ra  rm  rx  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r)  r6  r:  r  r   r   <module>r     s$           	    	  	         ( &   * *       0 * E       ! 3 6 A - U ; A 
 T  T 3 L Q 4 C   0 )      : Y+-G 		!I
 $**Y'  1     0 A T N I 6     " B 
 )  +6  
	  
   %'xS	" '/1 HT#s(^, 1(*Xd38n% * 2 3: /0(	EYYx
%

 #'
0cN'0
8C=$sCx.(),W	x "TN 
@% C= 	
 !!  
4 JN
1
1"*+E"F
1 3
C 
 
6 ( PT T#Y x~   $
#(59@D 	49jtnj 3-j 4.	j
 D012j <=j $j $,D>j\ CGEE2:3-E6E3 E2F"2 '+,0#,159@D49stns 5d3i(
)s tn	s
 $D>s D012s <=s $,D>s 	sl	 	 	C hsm PT , 6:&&%-c]&	&T 6:%-c]	.S x} 3 SW B (3- SW c  QU  6:%-c]	 6:%-c]	.3 Xc] d > 6:

%-c]
	

c 
2d d t IuS$Y/ IZ "
)-"#  $&*#')-'+?C"&#'+!%KMc]M "#M smM tnM8 9M: sm;M< C==M> !?M@ tnAMB /7tnCMD $EMF }GMH tnIMJ KMN 
OM`T )-GOPT~	$ # %)!%JN)-"&OOsmO SMO c]	O
 #O &d73D+E&FGO "#O $Of  !%))--1_C=_}_ c]_ c]	_
 3-_ C=_ 3-_ "#_ %TN_J %) $"&-1WW 3-W c]	W
 W $W %TNWt.*7?S	7J	"4 c d . 

	
		
15EX XD t,-.EXv&$ &4 &R(#C=(	#( 
#( $,C=	(
 c](Vd  *&&.<&c]&6, ,8DV;W ,^. T . T ":=	"*23-c]4T c C C m  )c )HSM )X!:
# :
(3- :
z, ,F F3# 3# 3 S x}  &# # #d # SW 4 i **%-c]*,*ZS Xc] . 2XX%-c]XX X 6:f
f
%-c]f
f
RP# PHSM PY PfGs G4Xv  "(
(
(
 (
 	(

 (
 (
VS T-5H  1:  !"zVC=zVc]zV smzV 
	zVz	0$H3  .s < 15u}}-J 15	>>> u}}-> 	>
 5#:>: !# 	/&/&/& /& 	/&dO% O%f >B-5c]0 >B-5c]0T "D"*&9   P  #(cC=c c !	cL1d 1tCy 1h1htnc] # #> 8( 
/ D 0  &c c 	4(8#9 	d 	 5L0M *   
U9d?3 
 
(	E5E0F 	E"T:J5K "J%c	*+eD#I8` `J >O9: c]: 6:)-	t,-. D012 "#	
 
<1 1% 18 #CC #Cc #Cc #CLd t W~ s   &nn