
    Ig]c                        d Z ddlmZ ddlZddlmZmZmZmZ ddl	m
Z
 ddlmZ ddlmZmZmZ dd	lmZ dd
lmZ  ej*                  e      Zddddddddddddddddddddddddddddddddddddddddddi ddddddddddd d!d"dd#d$d%dd&d'd(d)d*dd+dd,dd-dd.dd/dd0diddd1dd1ddd2dddd3d4d5d6d7d8ddd9d:dd;d<d=d>d7d8dd9d:d?d@dA	d=d>d7d8dBdCd9d:d?d@dD
dddddddddddddddEd6dFd;dGdHdIdJdKdLdMdNdddLdMdddddddddOdPZdQdRdSdTdUdVdWdXdYdZd[d\d]d^
dTdUdVdWdXdYdZd[d\d]d^
dTdUdVdWddYdZd[d\d]d^
dTdUdVdWddYdZd[d\d]d_d`dadTdUdVdWddYdZd[d\d]d^
dbdcdddedfdTdUdVdWddYdZd[d\d]d^
dgdWdhd]didjdTdUdVdWddZd[dkd]dl	dgdmdndodhdidpdUdTdWdVdZd[dqdrdsZdtdudvdwdxdbdcdddedydz
d{dQdbdcddded|d}Zd~ Z G d d      Z G d de      Z G d de      Z G d de      Z G d de      Z eeeee e e e e d	Z!defdZ"y)z
Integration with GGML / The file is copied and adapted from https://github.com/99991/pygguf
with extra methods beings exposed
    )arrayN)	Tokenizerdecodersnormalizerspre_tokenizers)BPE   )
AddedToken)GPT2ConverterLlamaConverterQwen2Converter)logging)tqdmzmodel.embed_tokenszmodel.layerszmlp.up_projzmlp.down_projzmlp.gate_projpost_attention_layernorminput_layernormzself_attn.q_projzself_attn.v_projzself_attn.k_projzself_attn.o_projzlm_head.weightz
model.norm)
token_embdblkffn_upffn_downffn_gateffn_norm	attn_normattn_qattn_vattn_kattn_outputoutput.weightoutput_normr   r   ffn_up_expszmlp.expertsffn_up_shexpzmlp.shared_expert.up_projffn_down_expsffn_down_shexpzmlp.shared_expert.down_projr   zffn_gate_inp.weightzmlp.gate.weightffn_gate_expsffn_gate_shexpzmlp.shared_expert.gate_projffn_gate_inp_shexpzmlp.shared_expert_gater   r   r   r   r   r   r   zmlp.gate_up_projzself_attn.qkv_proj)r   r   r   r   r   r   r   attn_qkvr   r   r   z"transformer.word_embeddings.weightz%transformer.word_embeddings_layernormztransformer.hzmlp.dense_h_to_4hzmlp.dense_4h_to_hzself_attention.query_key_valuezself_attention.denseztransformer.ln_f)ztoken_embd.weighttoken_embd_normr   r   r   r   r   r&   r   r   r   word_embeddingshz	.lm_head.ln_f)	r   r   r   r   r   r&   r   .output.r   z.ln_mlp.ln_attn)
r   r   r   r   z.attn_norm.attn_norm_2r&   r   r+   r   ztransformer.wteztransformer.wpeln_1zattn.c_attnzattn.c_proj.weightzattn.c_proj.biasln_2zmlp.c_fcz
mlp.c_proj)r   r   position_embdr   r   r&   zattn_output.weightzattn_output.biasr   r   r   )r   r   r   r   r   r   r   r   r   r   r   r   )llamamistralqwen2qwen2moephi3bloomfalcon7b	falcon40bstablelmgpt2
starcoder2
model_type_model_name_or_path)architecturenamemax_position_embeddingsnum_hidden_layersintermediate_sizehidden_sizehead_dim
rope_thetanum_attention_headsnum_key_value_headsrms_norm_eps
vocab_size)
context_lengthblock_countfeed_forward_lengthembedding_lengthrope.dimension_countrope.freq_baseattention.head_countattention.head_count_kv attention.layer_norm_rms_epsilonrI   num_expertsnum_experts_per_tok)rJ   rK   rL   rM   rN   rO   rP   rQ   rR   rI   expert_countexpert_used_countbos_token_ideos_token_idunk_token_idpad_token_id)ggml.bos_token_idggml.eos_token_idggml.unknown_token_idggml.padding_token_idn_layern_headlayer_norm_epsilon)rK   rM   rP   rI   attention.layer_norm_epsilonlayer_norm_eps)	rJ   rK   rL   rM   rN   rP   rQ   rb   rI   n_ctxn_embdrL   )rK   rJ   rM   rL   rP   rb   norm_epsilon)rK   rJ   rM   rL   rP   rQ   rb   )generalr1   r2   r3   r4   falcon	tokenizerr5   r6   r9   r:   r;   tokenizer_typetokensscores
token_typemergesadd_prefix_space)

ggml.modelzggml.tokenszggml.scoreszggml.token_typezggml.mergesr[   r\   r]   r^   zggml.add_space_prefixchat_template)rq   rp   r[   r\   r]   r^   )ri   tokenizer_configc                    t        |t              s|g}t        |      dk(  r|d   }d }n|d   dk7  rt        d      |\  }}|dv rt	        | d         } | S |dv rt        | d         } | S |dv rt        | d         } | S |dv r3t        d	t        |             j                         j                         } | S |d
v rt        | |      } | S )N   r   	   zPReceived multiple types, therefore expected the first type to indicate an array.)r   rt   r	            
      )      )   )   B)ru   )
isinstancelistlen
ValueErrorintfloatboolr   tobytesdecode_gguf_parse_value)_value	data_typearray_data_types      [/var/www/html/answerous/venv/lib/python3.12/site-packages/transformers/integrations/ggml.pyr   r   X  s    i&K	
9~aL	Q<1opp%."	?..VAY M 
g	vay! M 
c	fQi
 M	 
c	sDL)113::< M 
c	"6?;M    c                       e Zd Zd Zy)GGUFTokenizerSkeletonc                 p   |j                         D ]  \  }}t        | ||        t        | d      s)t        | d      rt        | d      st        d      | j                  }| j
                  }t        |      D ci c]  \  }}|||    c}}t        j                  d       g }t        j                               D ]k  \  }	}
g }t        dt        |	            D ]*  }|	d | |	|d  }}||v s||v s|j                  |||
f       , t        |fdd	      }|j                  |       m t        |d
 d	      }|D cg c]  }|d   |d   f }}|| _        nt| j                  D 	cg c]  }	t!        |	j#                  d             c}	| _        t        | d      s1t        t        | j                              D cg c]  }d  c}| _        t        | d      sg | _        t        | d      sd | _        t        | d      r| j&                  | j(                  | _        y y y c c}}w c c}w c c}	w c c}w )Nrn   rk   rl   z\tokens and scores need to be passed for a LLaMa tokenizer without merges to be instantiated.z:Merges were not in checkpoint, building merges on the fly.rt   c                 $    | d      | d      fS )Nr   rt    )xvocabs    r   <lambda>z0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>  s    U1Q4[%!+4N r   T)keyreversec                     | d   S )Nr	   r   )vals    r   r   z0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>  s
    CF r   r    added_tokensrY   unknown_token_id)itemssetattrhasattrr   rk   rl   	enumerateloggerwarningr   ranger   appendsortedextendrn   tuplesplitr   rY   r   )selfdict_kvrk   rl   itrn   mergepiece_scorelocalindexpiece_lpiece_rr   _r   s                    @r   __init__zGGUFTokenizerSkeleton.__init__q  s    KKM 	 DAqD!Q	  tX&4*'$2I r  [[F[[F.7.?@daQq	\@ENNWXF&*5;;=&9 %"{"1c%j1 FE',Ve}eEFmWG&(W->gw%DEF u*NX\]e$% F(:DIF2893s1vs1v&9F9 DK@DLu5S!12LDK4*-23t{{3C-DEtEt^, "Dt^, $D 4+,1B1B1J $ 5 5D 2K,7 A : MEs   =H#H)+!H. 	H3N)__name__
__module____qualname__r   r   r   r   r   r   p  s    '6r   r   c                   0    e Zd Zd Zd Zd Zd Zd Zd Zy)GGUFLlamaConverterc                     t        |      | _        | j                  | _        i | _        t	        | j                  dd      dk7  | _        y )Nrj   r1   )r   protooriginal_tokenizeradditional_kwargsgetattris_llama_3_tokenizerr   tokenizer_dicts     r   r   zGGUFLlamaConverter.__init__  s>    *>:
"&**!#$+DJJ8H'$RV]$]!r   c                 T    t        t        |j                  |j                              S Nr   ziprk   rl   r   r   s     r   r   zGGUFLlamaConverter.vocab      Cell344r   c                     |j                   S r   rn   r   s     r   rn   zGGUFLlamaConverter.merges      ||r   c           
         | j                  | j                        }| j                  | j                        }t        |      D ci c]  \  }\  }}|| }}}}|j                  |j
                  |j                     nd }t        |dd       |j
                  |j                     nd }	t        |dd       |j
                  |j                     nd }
t        t        |||dd            }g }t        | j                  d      s^||j                  t        |dd             |	|j                  t        |	dd             |
|j                  t        |
dd             n}t        j                  t        j                  | j                  j                         dk(        d	   }|D ]6  }|j                  t        | j                  j
                  |   dd             8 t#        |      d	k7  r|j%                  |       t#        | j                  j&                        d	k7  r?|j)                  | j                  j&                  D cg c]  }t        |dd       c}       || j*                  d
<   |	| j*                  d<   |
| j*                  d<   | j,                  r>d | j*                  d<   d| j*                  d<   d| j*                  d<   d| j.                  _        |S c c}}}w c c}w )NrW   rX   T)	unk_tokenfuse_unkbyte_fallbackrm   F
normalizedspecialrv   r   r   	eos_token	bos_tokenro   clean_up_tokenization_spaceslegacy)r   r   rn   r   rY   rk   r   rW   r   r   r   r   r
   npwherer   rm   r   add_special_tokensr   
add_tokensr   r   r   r   )r   r   vocab_scoresrn   r   word_score	bpe_vocabr   r   r   ri   special_tokensspecial_tokens_idxidxadded_tokens                   r   ri   zGGUFLlamaConverter.tokenizer  s   zz$**-TZZ(6?6MNN!2NT6T1WN	N8=8J8J8VELL!3!34\`	8?~W[8\8hELL!3!34nr	8?~W[8\8hELL!3!34nr	#"
	 tzz<0$%%juVZ&[\$%%juVZ&[\$%%juVZ&[\ "$"((4::3H3H*IQ*N!OPQ!R) j%%j1B1B31GTYcg&hij ~!#((8tzz&&'1,  ]a]g]g]t]tukKE5Iu /8{+.7{+.7{+$$9=D""#56EID""#AB/4D""8,-2D##*k ON vs   KKc                 0   t        j                         t        j                         t        j                  dd      g}| j                  r|t        j
                  ddd      gz  }|r|t        j                  dd      gz  }t        j                  |      S )N   ▁r   FTro   trim_offsets	use_regexrt   contentleft)r   ByteFallbackFuseReplacer   	ByteLevelStripSequencer   replacementro   sequences       r   decoderzGGUFLlamaConverter.decoder  s    !!#MMOUC(
 $$++UQVbfghhH!<==H  **r   c                     | j                  | j                        }| j                  | j                        }|||_        d}d}t        | j                  d      r| j                  j
                  }| j                  ||      }|||_        | j                  ||      |_        | j                         }|r||_        | j                  r7t        j                  ddd      |_        t        j                  g       |_        |S )Nr   Tro   Fr   )ri   r   
normalizerr   r   ro   pre_tokenizerr   post_processorr   r   r   r   r   )r   ri   r   r   ro   r   r   s          r   	convertedzGGUFLlamaConverter.converted  s    NN4::.	 __TZZ0
!#-I 4**,>?#66GG**;8HI$&3I# LL6FG	,,.'5I$ $$&4&>&>!&Ud'I#
 $/#7#7#;I r   N)	r   r   r   r   r   rn   ri   r   r   r   r   r   r   r     s"    ^58t+!r   r   c                   *     e Zd Zd Zdef fdZ xZS )GGUFQwen2Converterc                 2    t        |      | _        i | _        y r   r   r   r   r   s     r   r   zGGUFQwen2Converter.__init__      "7"G!#r   returnc           	      4   t        | j                  j                        D ci c]  \  }}||
 }}}| j                  j                  }t        |   ||      }|j                  t        ddd      t        ddd      t        ddd      g       |S c c}}w )N<|endoftext|>FTr   z<|im_start|>z
<|im_end|>)r   r   rk   rn   superr   r   r
   r   r   r   r   rn   ri   	__class__s         r   r   zGGUFQwen2Converter.converted  s    (1$2I2I2P2P(QRWQqRR((//G%eV4	$$?udK>eTJ<E4H	
  Ss   Br   r   r   r   r   r   __classcell__r  s   @r   r   r     s    $9  r   r   c                   6    e Zd Zd Zd Zd Zd Zd ZdefdZ	y)	GGUFPhi3Converterc                 T    t        |      | _        | j                  | _        i | _        y r   )r   r   r   r   r   s     r   r   zGGUFPhi3Converter.__init__)  s"    *>:
"&**!#r   c                 T    t        t        |j                  |j                              S r   r   r   s     r   r   zGGUFPhi3Converter.vocab.  r   r   c                     |j                   S r   r   r   s     r   rn   zGGUFPhi3Converter.merges1  r   r   c                    | j                  | j                        }| j                  | j                        }t        |      D ci c]  \  }\  }}|| }}}}t	        t        ||            }|j                  t        ddddd      t        ddd      t        dddd      t        d	ddd      t        d
ddd      t        dddd      t        dddd      t        dddd      t        dddd      t        dddd      t        dddd      t        dddd      g       |j                  |j                  |j                     nd | j                  d<   |j                  |j                  |j                     nd | j                  d<   |j                  |j                  |j                     nd | j                  d<   |j                  |j                  |j                     nd | j                  d<   |S c c}}}w )Nz</s>TF)rstriplstripr   r   r  r   z<|assistant|>)r  r   r   z<|placeholder1|>z<|placeholder2|>z<|placeholder3|>z<|placeholder4|>z
<|system|>z<|end|>z<|placeholder5|>z<|placeholder6|>z<|user|>r   r   r   	pad_token)r   r   rn   r   r   r   r   r
   rY   rk   r   rX   rW   rZ   )	r   r   r   rn   r   r   r   r   ri   s	            r   ri   zGGUFPhi3Converter.tokenizer4  s    zz$**-TZZ(6?6MNN!2NT6T1WN	Nc)V45	$$6$uX\]?udK?4ESWX-duVZ[-duVZ[-duVZ[-duVZ[<PTU9TeTR-duVZ[-duVZ[:dudS	
$ 160B0B0NELL++,TX 	{+ 160B0B0NELL++,TX 	{+ 160B0B0NELL++,TX 	{+ 160B0B0NELL++,TX 	{+ E Os   G:c                     t        j                         t        j                         t        j                  |d      g}|r|t        j                  dd      gz  }t        j
                  |      S )Nr   rt   r   )r   r   r   r   r   r   r   s       r   r   zGGUFPhi3Converter.decoder[  s\    !!#MMO[#.
 !<==H  **r   r  c                     | j                  | j                        }d}d}t        | j                  d      r| j                  j                  }| j                  ||      |_        |S )Nr   Tro   )ri   r   r   r   ro   r   )r   ri   r   ro   s       r   r   zGGUFPhi3Converter.convertedf  s\    NN4::.	4**,>?#66GG LL6FG	r   N)
r   r   r   r   r   rn   ri   r   r   r   r   r   r   r  r  (  s(    $
5%N	+
9 
r   r  c                   *     e Zd Zd Zdef fdZ xZS )GGUFGPTConverterc                 2    t        |      | _        i | _        y r   r   r   s     r   r   zGGUFGPTConverter.__init__t  r   r   r  c                     t        | j                  j                        D ci c]  \  }}||
 }}}| j                  j                  }t        |   ||      }|S c c}}w r   )r   r   rk   rn   r  r   r  s         r   r   zGGUFGPTConverter.convertedx  s]    (1$2I2I2P2P(QRWQqRR((//G%eV4	 Ss   Ar  r	  s   @r   r  r  s  s    $9  r   r  )	r1   r3   	qwen2_moer5   r6   rh   r9   r:   r;   r  c                 ^    | }t        |   |      }|j                         }||j                  fS )a6  
    Utilities to convert a slow tokenizer instance in a fast tokenizer instance.

    Args:
        architecture (`str`): The model architecture derived from gguf file.
        transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]):
            Instance of a slow tokenizer to convert in the backend tokenizer for
            [`~tokenization_utils_base.PreTrainedTokenizerFast`].

    Return:
        A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a
        [`~tokenization_utils_base.PreTrainedTokenizerFast`]
    )GGUF_TO_FAST_CONVERTERSr   r   )r>   r   tokenizer_class_name	converterfast_tokenizers        r   convert_gguf_tokenizerr    s9     ('(<=nMI((*N96666r   )#__doc__r   numpyr   
tokenizersr   r   r   r   tokenizers.modelsr    r
   convert_slow_tokenizerr   r   r   utilsr   utils.loggingr   
get_loggerr   r   GGUF_TENSOR_MAPPINGGGUF_CONFIG_MAPPINGGGUF_TOKENIZER_MAPPINGr   r   r   r   r  r  r  r  r   r   r   <module>r,     s   
   G G !  R R    
		H	%
 +##.&$$$))#  +##.&$$$))#  +##.&$$$))#*~ 	} 	3	
 	 	7 	. 	0 	 	7 	6 	& 	$ 	$ 	$  	)!" 	)#$ 	|%* +$#&.&())# BB%'.&4-)) (%'&4-
 (%'! 4- +##.&$$$))#  (*)!2.  + .&$$$))#_] D %%
 4*2) *& 5#8,:" 4*2) *& 5#8,:" 4*2) $& 5#8,:" 4*2) $& 5#8,:"%2 4*2) $& 5#8,:" ,+!/!/	 4*2) $& 5#8,:" !) ("(< 4*2) $ 5#8(8"
 !!$4 ((< +3)2 5#8(6cz | ''++!/!/!3 )"++!/!/ 00(6 (6Vv vr (H HV	} 	  # "
 7I 7r   