##hE\ddlZddlZddlZddlZddlZddlZddlmZmZddl m Z m Z ddl m Z ddlmZddlZddlmZmZddlmZej,d k7rddlZerdd lmZdd lmZdd lmZerddlZdd lm Z m!Z!m"Z"m#Z#e$ejJejLzZ'e$ejJejPzejLzdzZ)dZ*gdZ+ddiddiddiddiddiddidZ,dZ-dZ.dZ/dZ0dZ1dedfd Z2d!Z3d"Z4Gd#d$Z5e Gd%d&Z6d'efd(Z7Gd)d*eZ8y)+N)ArgumentParser Namespace) dataclassfield)Thread)Optional)is_rich_availableis_torch_available)BaseTransformersCLICommandWindows)Console)Live)Markdown)AutoModelForCausalLM AutoTokenizerBitsAndBytesConfigTextIteratorStreamerz .!\"#$%&'()*+,\-/:<=>?@[]^_`{|}~a# **TRANSFORMERS CHAT INTERFACE** The chat interface is a simple tool to try out a chat model. Besides talking to the model there are several commands: - **help**: show this help message - **clear**: clears the current conversation and start a new one - **example {NAME}**: load example named `{NAME}` from the config and use it as the user input - **set {SETTING_NAME}={SETTING_VALUE};**: change the system prompt or generation settings (multiple settings are separated by a ';'). - **reset**: same as clear but also resets the generation configs to defaults if they have been changed by **set** - **save {SAVE_NAME} (optional)**: save the current chat and settings to file by default to `./chat_history/{MODEL_NAME}/chat_{DATETIME}.yaml` or `{SAVE_NAME}` if provided - **exit**: closes the interface )max_new_tokens do_sample num_beams temperaturetop_ptop_krepetition_penaltytextz5There is a Llama in my lawn, how can I get rid of it?zyWrite a Python function that integrates any Python function f(x) numerically over an arbitrary interval [x_start, x_end].z4How many helicopters can a human eat in one sitting?z4Count to 10 but skip every number ending with an 'e'zWhy aren't birds real?z2Why is it important to eat socks after meditating?)llamacode helicopternumbersbirdssocksctjdk(rtjSt j tj jS)Nr )platformsystemosgetloginpwdgetpwuidgetuidpw_nameW/var/www/html/sandstorm/venv/lib/python3.12/site-packages/transformers/commands/chat.py get_usernamer/Zs8I%{{}||BIIK(000r-c<tjd}|d|dS)Nz%Y-%m-%d_%H-%M-%Sz/chat_z.json)timestrftime) model_nametime_strs r.create_default_filenamer5as$}}01H\z //r-ci}t||d<||d<|j}|5t|j}tj j ||}t jtj j|dt|d5}tj||ddddtj j|S#1swY(xYw)Nsettings chat_historyT)exist_okw)indent) vars save_folderr5model_name_or_pathr&pathjoinmakedirsdirnameopenjsondumpabspath)chatargsfilename output_dictfolderfs r. save_chatrNfsK"4jK "&K   F*4+B+BC77<<1KK)D9 h , +q+, 77??8 $$,,s CC"c|g}|Sd|dg}|S)Nr%rolecontentr,) system_promptrHs r.clear_chat_historyrTws( K"m<= Kr-cF|ddjjd}|Dcgc]9}|jdd|t|jdddzdf;}}t|}d}|D]}t ||r_ t t ||tr"||dk(rd||<n5||d k(rd||<n'ttt ||||||<n|jd|d|r|jd|dfS|D]-}t|||||jd|d||d /tjd|dfScc}w#t$r5|jd |d ||d tt ||d Y)wxYw)Nr;;=rr FTrueTFalsezCannot cast setting z (=z) to .z There is no 'z ' setting.zGThere was an issue parsing the settings. No settings have been changed.zSet z to g?)stripsplitlendicthasattr isinstancegetattrbool ValueErrortype print_redsetattr print_greenr1sleep) user_input current_args interfacer7settingerrornames r.parse_settingsros!"~##%++C0HbjkW^s#A&GMM#4Fq4I0JQ0N0P(QRkHkH~H EB < & glD94@~/)-!$72).((%FT',*E%FxPT~%VHTN   -vZ @ A#B& efU"" FD L$ 7  ! !Dd8D>2B!"D E F 3T!!Al  ##*4&HTN3C5gVbdhNiIjHkklm s>EAE"":F F returnrc|jr:td|j|j|j|j}|S|j rtd}|Sd}|S)NT) load_in_4bitbnb_4bit_compute_dtypebnb_4bit_quant_typebnb_4bit_use_double_quantbnb_4bit_quant_storage) load_in_8bit)rrr torch_dtypertuse_bnb_nested_quantrw) model_argsquantization_configs r.get_quantization_configr|st0#-#9#9 * > >&0&E&E#-#9#9     0  # r-ctj|j|j|j}|j dvr |j nt t|j }t|}|j|j|d|d}tj|jfd|ji|}t |dd|j|j}||fS)N)revisiontrust_remote_code)autoNr)r~attn_implementationrx device_mapr{r hf_device_map) rfrom_pretrainedr?model_revisionrrxratorchr|rrtodevice)rI tokenizerrxr{ model_kwargsmodels r.load_model_and_tokenizerrs-- $$00I '+&6&6.&H$""gV[]a]m]mNnK1$7''#77"2 L ! 0 0  373I3I MY Euot,4% ) r-c|j |j}n |j}g}|/|j|j|j d|7|j|j dDcgc] }t |c}t |dk(r|j|j||fScc}w)N,r) pad_token_id eos_token_idextendconvert_tokens_to_idsr\intr]append)r eos_tokens eos_token_idsrall_eos_token_idstoken_ids r.parse_eos_tokensrs% --  --   !@!@AQAQRUAV!WX   @S@STW@X!YH#h-!YZ "  !7!78 * ** "Zs9B?c>eZdZd dZdZdZdZdZdZdZ d Z y) RichInterfaceNcft|_|d|_n||_|d|_y||_y)N assistantuser)r_consoler3 user_name)selfr3rs r.__init__zRichInterface.__init__s3   )DO(DO  #DN&DNr-cXd}|jjd|jdt|jd5}t |D]\}}|r|dk(r||z }g}|j D]G}|j ||jdr|j d7|j d Itdj|jd }|j| d d d |jj|S#1swY%xYw) zStream output from a role.z [bold blue]:r;)consolerefresh_per_secondrz``` z z github-dark) code_themeN) rprintr3r enumerate splitlinesr startswithrrAr[update) r output_streamrliveioutputslineslinemarkdowns r. stream_outputzRichInterface.stream_outputs  l4??*;2>? $--A > &$' 6 & 7!q& OO--DLL&u- T* V,-$BGGEN$8$8$:}U H%5 & &:  = & &s B8D  D)c|jjd|jd}|jj|S)N [bold red]: )rinputrr)rrs r.rzRichInterface.inputs9 ##k$..1A$FG  r-c8|jjyN)rclearrs r.rzRichInterface.clears r-c|jjd|jd||jjy)Nrz>:[/ bold red] )rrrrrs r.print_user_messagez RichInterface.print_user_messages7 k$..)99I$PQ r-ct|jjd||jjy)Nz [bold green]rrrs r.rgzRichInterface.print_green#s+ l4&12 r-ct|jjd||jjy)Nz [bold red]rrs r.rezRichInterface.print_red's+ j/0 r-c|jjtt|jjyr)rrr HELP_STRINGrs r. print_helpzRichInterface.print_help+s) H[12 r-)NN) __name__ __module__ __qualname__rrrrrrgrerr,r-r.rrs+ '$L r-rcTeZdZUdZeddiZeed<edddiZe eed <eddd iZ e eed <ed dd iZ eed<edddiZ eed<edddiZ e eed<edddiZeed<edddiZeed<edddiZeed<edddiZeed<ed dd!iZeed"<eddd#iZeed$<eddd%iZeed&<eddd'iZe eed(<eddd)iZe eed*<ed+dd,iZeed-<ed.d/gd0d1Ze eed2<ed3dd4iZeed5<eddd6iZe eed7<ed3dd8iZeed9<ed3dd:iZeed;<eddcpuzDevice to use for inference.rz"Path to a yaml file with examples. examples_pathz%Maximum number of tokens to generate.rTz,Whether to sample outputs during generation.rr z Number of beams for beam search.rg?z%Temperature parameter for generation.r2zValue of k for top-k sampling.rz Value of p for nucleus sampling.rzRepetition penalty.rzNEOS tokens to stop the generation. If multiple they should be comma separated.rzQEOS token IDs to stop the generation. If multiple they should be comma separated.rmainzLSpecific model version to use (can be a branch name, tag name or commit id).rrzOverride the default `torch.dtype` and load the model under this dtype. If `'auto'` is passed, the dtype will be automatically derived from the model's weights.)rbfloat16float16float32)rchoicesrxFz2Whether to trust remote code when loading a model.rzWhich attention implementation to use; you can run --attn_implementation=flash_attention_2, in which case you must install this manually by running `pip install flash-attn --no-build-isolation`.rzIWhether to use 8 bit precision for the base model - works only with LoRA.rwzIWhether to use 4 bit precision for the base model - works only with LoRA.rrnf4zQuantization type.fp4rtz#Whether to use nested quantization.ry)"rrr__doc__rr?str__annotations__rrrSr>rrrrrrbrrfloatrrrrrrrxrrrwrrrtryr,r-r.rr0s4n$f6V-WXX@h7ijD(3-j#(IY@Z#[M8C=[%6&JhAijKj9W0XYFCY#(Im@n#oM8C=o v?f6ghNChDFe5fgKgrV5U,VWE3W8Z/[\E5\ %cVEZ<[ \\ %jk!J $)mn$M8C=  hiNC"'PA "K#$)] ^t*/ r *#efL$efL$ %UFZhmotgu=vww!&uHm?n!o$or-rrIct|S)z; Factory function used to chat with a local model. ) ChatCommandrIs r.chat_command_factoryrs t r-cHeZdZedefdZdZededefdZ dZ y) rparsercntf}|jdt|}|jty)z Register this command to argparse so it's available for the transformer-cli Args: parser: Root parser to register command-specific arguments rH)rdataclass_types)funcN)r add_parserr set_defaultsr)rr chat_parsers r.register_subcommandzChatCommand.register_subcommands6)*''[Ra'b   &: ;r-c||_yrr)rrIs r.rzChatCommand.__init__s  r-srpc|jdrd|vry|ddjdDcgc]#}|js|j%}}|D]}|jddk7ry|jdd\}}|j}|j}|r|syt |j t syt |j tryycc}w)Nzset rWFr;rVr T)rr\r[countsetissubsetALLOWED_KEY_CHARSALLOWED_VALUE_CHARS)ra assignments assignmentkeyvalues r.is_valid_setting_commandz$ChatCommand.is_valid_setting_commands||F#s!|+,AB%++c*:HQaggiqwwyH H% J$)#))#q1JC))+CKKMEes8$$%67u:&&':;# &+Is C0C0c ts tdts td|j}|jt }n3t |j5}tj|}dddtj|}|j t}n |j}t|\}}t|dd}t||j |j"\} } t%|j&|} | j)t+|j,} | j/} | dk(r&t+|j,} | j)=| dk(r| j1S| dk(ry| d k(r;| j)tj|}t+|j,} | j3d rft5| j7d krJ| j7}t5|d k(r|d }nd}t9| ||}| j;d |d|j=| r&t?| || \}}|rg} | j)G| j3drt5| j7d k(rz| j7d }|vr2| j)g} | jA||d||d} n1| jCd|dtE|jGd| jId| d|jK| ddjM|jN}tQjR|}||||jT|jV|jX|jZ|j\|j^|j`| | d }tc|jd|}|jg| ji|}|jk| jId|d#1swYxYw#tl$rYywxYw)NzHYou need to install rich to use the chat interface. (`pip install rich`)zJYou need to install torch to use the chat interface. (`pip install torch`)T)skip_special_tokens skip_prompt)r3rrrexitresetsaver zChat saved in !examplerzExample z* not found in list of available examples: rZrrPpt)return_tensorsadd_generation_prompt) inputsattention_maskstreamerrrrrrrrrr)targetkwargsr)7r ImportErrorr rIrDEFAULT_EXAMPLESrDyaml safe_loadcopydeepcopyrr/rrrrrrr?rrTrSrrrr]r\rNrgrrorrelistkeysrapply_chat_templaterrr ones_likerrrrrrrrgeneratestartrrAKeyboardInterrupt)rrIexamplesrMrjrrrgeneration_streamerrrrkrHri split_inputrJsuccess example_namerrgeneration_kwargsthread model_outputs r.runzChatCommand.runs "hi i!#jk kyy    %'Hd(() -Q>>!, -}}T* 99 >D99D3D9y29RVdhi&6y$//SWSeSe&f# m!T-D-DPTU !,"<"<=P &__. (-l.H.HIDOO%'((*'(OO%#'==#6L-l.H.HID((0S9I9I9K5Lq5P","2"2"4K;'1,#.q>#'(|XFH))N8*A*FG00<,::|U^,_)L'!!) ((3JOP (667JK   [\JK]% - -D%  sKP(3:P5.P5P5 ?P5 A5P55P58B%P5DP5(P25 QQN) rrr staticmethodrrrrrbrrr,r-r.rrsH r.s   .( D(8?? !$&jj,,v/@/@@A 6==(6+<+<