
    ##hv                     &   d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlmZ ddlmZ ddlmZmZmZ ddlmZ ddlmZmZmZmZmZmZ  ej<                  e      Z  ejB                         Z"d	 Z#d
ee$ejJ                  f   ddfdZ&dee$ejJ                  f   de'e$   fdZ(dee$ejJ                  f   de'e$   fdZ)dee$ejJ                  f   de'e$   fdZ*dee$ejJ                  f   de'e$   fdZ+ddde$dee$ejJ                  f   de,de-fdZ.	 	 	 	 	 	 	 	 	 d.dee$ejJ                  f   de$deee$ejJ                  f      de,dee,   dee/e$e$f      deee,e$f      dee$   d e,d!ee$   d"ee$   de$fd#Z0	 	 	 	 	 	 	 	 	 d.d$e$dee$ejJ                  f   deee$ejJ                  f      de,dee,   dee/e$e$f      deee,e$f      dee$   d e,d!ee$   d%ee$   de-fd&Z1d/d'ed(ee$ejJ                  f   d)ee/   de'e$   fd*Z2d+ Z3d,Z4d- Z5y)0z3Utilities to dynamically load objects from the Hub.    N)Path)
ModuleType)AnyOptionalUnion)try_to_load_from_cache   )HF_MODULES_CACHE TRANSFORMERS_DYNAMIC_MODULE_NAMEcached_fileextract_commit_hashis_offline_modeloggingc                  <   t         t        j                  v ryt        j                  j                  t                t	        j
                  t         d       t        t               dz  } | j                         s%| j                          t        j                          yy)z_
    Creates the cache directory for modules with an init, and adds it to the Python path.
    NTexist_ok__init__.py)r
   syspathappendosmakedirsr   existstouch	importlibinvalidate_caches)	init_paths    ^/var/www/html/sandstorm/venv/lib/python3.12/site-packages/transformers/dynamic_module_utils.pyinit_hf_modulesr   0   sj    
 388#HHOO$%KK 40%&6I##%     namereturnc                 X   t                t        t              | z  j                         }|j                  j                         st        |j                         t        j                  |d       |dz  }|j                         s%|j                          t        j                          yy)z
    Creates a dynamic module in the cache directory for modules.

    Args:
        name (`str` or `os.PathLike`):
            The name of the dynamic module to create.
    Tr   r   N)r   r   r
   resolveparentr   create_dynamic_moduler   r   r   r   r   )r!   dynamic_module_pathr   s      r   r&   r&   @   s      01D8AAC%%,,.1889KK#d3#m3I 	##%	 r    module_filec                 ,   t        | d      5 }|j                         }ddd       t        j                  dt        j                        }|t        j                  d|t        j                        z  }t        t        |            S # 1 sw Y   lxY w)z
    Get the list of modules that are relatively imported in a module file.

    Args:
        module_file (`str` or `os.PathLike`): The module file to inspect.

    Returns:
        `list[str]`: The list of relative imports in the module.
    utf-8encodingNz^\s*import\s+\.(\S+)\s*$)flagsz^\s*from\s+\.(\S+)\s+import)openreadrefindall	MULTILINElistset)r(   fcontentrelative_importss       r   get_relative_importsr8   V   s{     
kG	, &&( zz"=wbll[

#A7RTR^R^__$%&& s   B

Bc                 l   d}| g}g }|sg }|D ]  }|j                  t        |              t        |       j                  }|D cg c]  }t	        ||z         }}|D cg c]	  }||vs| }}|D cg c]  }| d	 }}t        |      dk(  }|j                  |       |s|S c c}w c c}w c c}w )a  
    Get the list of all files that are needed for a given module. Note that this function recurses through the relative
    imports (if a imports b and b imports c, it will return module files for b and c).

    Args:
        module_file (`str` or `os.PathLike`): The module file to inspect.

    Returns:
        `list[str]`: The list of all relative imports a given module needs (recursively), which will give us the list
        of module files a given module needs.
    F.pyr   )extendr8   r   r%   strlen)	r(   	no_changefiles_to_checkall_relative_importsnew_importsr5   module_pathmnew_import_filess	            r   get_relative_import_filesrE   k   s     I!]N  	8A3A67	8 ;'..:EFQCa0FF'7Y!1DX;XAYY-=>QCs)>>()Q.	##N3    GY>s   B'!	B,+B,5B1filenamec                     t        | d      5 }|j                         }ddd       t               fdt        j                        } |       t              S # 1 sw Y   AxY w)a  
    Extracts all the libraries (not relative imports this time) that are imported in a file.

    Args:
        filename (`str` or `os.PathLike`): The module file to inspect.

    Returns:
        `list[str]`: The list of all packages required to use the input module.
    r*   r+   Nc                    t        | t        j                        ry t        | t        j                        rj| j                  }t        j
                  |      D ]E  }t        |t        j                        st        |j                  dd      j                  d      sE y  nt        | t        j                        rD| j                  D ]4  }|j                  j                  d      d   }|s$j                  |       6 nft        | t        j                        rL| j                   dk(  r=| j"                  r1| j"                  j                  d      d   }|rj                  |       t        j$                  |       D ]
  } |        y )Nid is_flash_attn.r   )
isinstanceastTryIftestwalkCallgetattrfunc
startswithImportnamesr!   splitadd
ImportFromlevelmoduleiter_child_nodes)noderQ   condition_nodealias
top_modulechildimported_modulesrecursive_look_for_importss         r   re   z/get_imports.<locals>.recursive_look_for_imports   s2   dCGG$cff%99D"%((4. nchh7GNDWDWY]_a<b<m<m#=  cjj) 5"ZZ--c215
$((45 cnn-zzQ4;;![[..s3A6
$((4 ))$/ 	.E&u-	.r    )r.   r/   r4   rN   parsesorted)rF   r5   r6   treerd   re   s       @@r   get_importsri      sc     
h	) Q&&(u.8 99WDt$"##G s   A!!A*c                    t        |       }g }|D ]  }	 t        j                  |        t        |      dkD  r0t        ddj                  |       dd	j                  |       d
      t        |       S # t        $ rE}t        j                  d| d|        dt        |      v r|j                  |       n Y d}~d}~ww xY w)a.  
    Check if the current Python environment contains all the libraries that are imported in a file. Will raise if a
    library is missing.

    Args:
        filename (`str` or `os.PathLike`): The module file to check.

    Returns:
        `list[str]`: The list of relative imports in the file.
    z&Encountered exception while importing z: zNo module namedNr   z\This modeling file requires the following packages that were not found in your environment: z, z. Run `pip install  `)ri   r   import_moduleImportErrorloggerwarningr<   r   r=   joinr8   )rF   importsmissing_packagesimp	exceptions        r   check_importsrv      s     (#G 
	##C( q jyy)*++>sxxHX?Y>ZZ[]
 	

  ))!  	NNCC59+VW !C	N2 '', -	s   A44	C=;B==CFforce_reload
class_namerB   rx   c          
         t         j                  j                  |      }|j                  d      r|dd }|j	                  t         j                  j
                  d      }t        t              |z  }t        5  |r4t        j                  j                  |d       t        j                          t        j                  j                  |      }t        j                  j!                  ||      }|gt#        t%        t        t'        |                  z   }t)        j*                  dj-                  d |D                    j/                         }|3t        j                  j1                  |      }	|	t        j                  |<   n|}	t3        |	dd	      |k7  r"|j4                  j7                  |	       ||	_        t3        |	|       cddd       S # 1 sw Y   yxY w)
a  
    Import a module on the cache directory for modules and extract a class from it.

    Args:
        class_name (`str`): The name of the class to import.
        module_path (`str` or `os.PathLike`): The path to the module to import.
        force_reload (`bool`, *optional*, defaults to `False`):
            Whether to reload the dynamic module from file if it already exists in `sys.modules`.
            Otherwise, the module is only reloaded if the file has changed.

    Returns:
        `typing.Type`: The class looked for.
    r:   NrL   )locationr    c              3   T   K   | ]   }t        |      |j                         z    " y wN)bytes
read_bytes).0r5   s     r   	<genexpr>z&get_class_in_module.<locals>.<genexpr>   s      2cQR58alln3L2cs   &(__transformers_module_hash__rJ   )r   r   normpathendswithreplacesepr   r
   _HF_REMOTE_CODE_LOCKr   modulespopr   r   getutilspec_from_file_locationrg   maprE   hashlibsha256rq   	hexdigestmodule_from_specrT   loaderexec_moduler   )
ry   rB   rx   r!   r(   cached_modulemodule_specmodule_filesmodule_hashr]   s
             r   get_class_in_moduler      sj   & 77K(D}}UCRy<<S)D-.<K	 +KKOOD$''').1kkood.Cnn<<TK<X %0=6#dD]^iDj:k3l#l">>#((2cVb2c*cdnnp  ^^44[AF &CKK"F692>+M**622=F/vz*-+ + +s   8E
GGpretrained_model_name_or_path	cache_dirforce_downloadresume_downloadproxiestokenrevisionlocal_files_only	repo_type_commit_hashc                    |j                  dd      }|)t        j                  dt               |t	        d      |}t               r|st        j                  d       d}t        |       } t        j                  j                  |       }|r t        j                  j                  |       }n:| j                  dt        j                  j                        }t        | |||
|	      }g }	 t!        | |||||||||	|
	      }|s|k7  r|j#                  |       t)        |      }t*        t        j                  j                  z   |z   }t-        |       t/        t0              |z  }|t        j                  j                  |       k(  r||z  j3                         r"t5        j6                  |t        ||z              s-t9        j:                  |||z         t=        j>                          |D ]  }| d}t        j                  jA                  | |      }||z  j3                         r#t5        j6                  |t        ||z              r^t9        j:                  |||z         t=        j>                           ntC        ||
      }||z  }|t        j                  j                  z   |z   }t-        |       ||z  j3                         s-t9        j:                  |||z         t=        j>                          |D ]E  }|| dz  j3                         rtE        | | d||||||||
       |j#                  | d       G tG        |      dkD  rP|NdjA                  |D cg c]  }d| 	 c}      }|	dn|	 d}d| |  }t        jI                  d| d| d       t        j                  jA                  ||      S # t$        $ r t        j'                  d
| d|  d        w xY wc c}w )a	  
    Prepares Downloads a module from a local folder or a distant repo and returns its path inside the cached
    Transformers module.

    Args:
        pretrained_model_name_or_path (`str` or `os.PathLike`):
            This can be either:

            - a string, the *model id* of a pretrained model configuration hosted inside a model repo on
              huggingface.co.
            - a path to a *directory* containing a configuration file saved using the
              [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`.

        module_file (`str`):
            The name of the module file containing the class to look for.
        cache_dir (`str` or `os.PathLike`, *optional*):
            Path to a directory in which a downloaded pretrained model configuration should be cached if the standard
            cache should not be used.
        force_download (`bool`, *optional*, defaults to `False`):
            Whether or not to force to (re-)download the configuration files and override the cached versions if they
            exist.
        resume_download:
            Deprecated and ignored. All downloads are now resumed by default when possible.
            Will be removed in v5 of Transformers.
        proxies (`Dict[str, str]`, *optional*):
            A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
            'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
        token (`str` or *bool*, *optional*):
            The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
            when running `huggingface-cli login` (stored in `~/.huggingface`).
        revision (`str`, *optional*, defaults to `"main"`):
            The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
            git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
            identifier allowed by git.
        local_files_only (`bool`, *optional*, defaults to `False`):
            If `True`, will only try to load the tokenizer configuration from local files.
        repo_type (`str`, *optional*):
            Specify the repo type (useful when downloading from a space for instance).

    <Tip>

    Passing `token=True` is required when you want to use a private model.

    </Tip>

    Returns:
        `str`: The path to the module inside the cache.
    use_auth_tokenNrThe `use_auth_token` argument is deprecated and will be removed in v5 of Transformers. Please use `token` instead.V`token` and `use_auth_token` are both specified. Please set only the argument `token`.z+Offline mode: forcing local_files_only=TrueT/)r   r   r   )	r   r   r   r   r   r   r   r   r   zCould not locate the z inside rL   r:   )r   r   r   r   r   r   r   r   r   
z- rJ   zs/zhttps://huggingface.co/z9A new version of the following files was downloaded from z:
z
. Make sure to double-check they do not contain any added malicious code. To avoid downloading new versions of the code file, you can pin a revision.)%r   warningswarnFutureWarning
ValueErrorr   ro   infor<   r   r   isdirbasenamer   r   r   r   r   OSErrorerrorrv   r   r&   r   r
   r   filecmpcmpshutilcopyr   r   rq   r   get_cached_module_filer=   rp   )r   r(   r   r   r   r   r   r   r   r   r   deprecated_kwargsr   is_local	submoduler   	new_filesresolved_module_filemodules_neededfull_submodulesubmodule_pathmodule_neededmodule_needed_filecommit_hashr5   repo_type_strurls                              r   r   r     s   | '**+;TBN! A	
 uvv!1AB %((E$F!ww}}:;HGG$$%BC	199#rww{{K	.);)Vbnw
 I*))+-% 
 M-AA[) ##78N 6CiON.)*+n<NBGG$$%BCC ,446gkk #n{&B"C?
 KK,n{.JK'')+ 	.M,oS1M!#.K]!["]2::<GKK"C(F$GE .0NO++-	. **>M (+5'"''++5Cn-,446KK,n{.JK'')+ 	8M"c%::BBD&1$oS)'#1$3#%%5!,   M?#!67	8  9~h.II;AA3x;<	'/	{"5E'7T6UVGuCPY{AA	
 77<<44G  ,[MB_A``abct <s   *.N< O&<'O#class_referencecode_revisionc                 8   |j                  dd      }|)t        j                  dt               |t	        d      |}d| v r| j                  d      \  }} n|}| j                  d      \  }}|
||k(  r|}
t        ||dz   ||||||
||	
      }t        |||	      S )
a>  
    Extracts a class from a module file, present in the local folder or repository of a model.

    <Tip warning={true}>

    Calling this function will execute the code in the module file found locally or downloaded from the Hub. It should
    therefore only be called on trusted repos.

    </Tip>



    Args:
        class_reference (`str`):
            The full name of the class to load, including its module and optionally its repo.
        pretrained_model_name_or_path (`str` or `os.PathLike`):
            This can be either:

            - a string, the *model id* of a pretrained model configuration hosted inside a model repo on
              huggingface.co.
            - a path to a *directory* containing a configuration file saved using the
              [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`.

            This is used when `class_reference` does not specify another repo.
        module_file (`str`):
            The name of the module file containing the class to look for.
        class_name (`str`):
            The name of the class to import in the module.
        cache_dir (`str` or `os.PathLike`, *optional*):
            Path to a directory in which a downloaded pretrained model configuration should be cached if the standard
            cache should not be used.
        force_download (`bool`, *optional*, defaults to `False`):
            Whether or not to force to (re-)download the configuration files and override the cached versions if they
            exist.
        resume_download:
            Deprecated and ignored. All downloads are now resumed by default when possible.
            Will be removed in v5 of Transformers.
        proxies (`Dict[str, str]`, *optional*):
            A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',
            'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request.
        token (`str` or `bool`, *optional*):
            The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated
            when running `huggingface-cli login` (stored in `~/.huggingface`).
        revision (`str`, *optional*, defaults to `"main"`):
            The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a
            git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any
            identifier allowed by git.
        local_files_only (`bool`, *optional*, defaults to `False`):
            If `True`, will only try to load the tokenizer configuration from local files.
        repo_type (`str`, *optional*):
            Specify the repo type (useful when downloading from a space for instance).
        code_revision (`str`, *optional*, defaults to `"main"`):
            The specific revision to use for the code on the Hub, if the code leaves in a different repository than the
            rest of the model. It can be a branch name, a tag name, or a commit id, since we use a git-based system for
            storing models and other artifacts on huggingface.co, so `revision` can be any identifier allowed by git.

    <Tip>

    Passing `token=True` is required when you want to use a private model.

    </Tip>

    Returns:
        `typing.Type`: The class, dynamically imported from the module.

    Examples:

    ```python
    # Download module `modeling.py` from huggingface.co and cache then extract the class `MyBertModel` from this
    # module.
    cls = get_class_from_dynamic_module("modeling.MyBertModel", "sgugger/my-bert-model")

    # Download module `modeling.py` from a given repo and cache then extract the class `MyBertModel` from this
    # module.
    cls = get_class_from_dynamic_module("sgugger/my-bert-model--modeling.MyBertModel", "sgugger/another-bert-model")
    ```r   Nr   r   z--rL   r:   )r   r   r   r   r   r   r   r   rw   )r   r   r   r   r   rY   r   r   )r   r   r   r   r   r   r   r   r   r   r   kwargsr   repo_idr(   ry   final_modules                    r   get_class_from_dynamic_moduler     s    t ZZ 0$7N! A	
 uvv #2#8#8#> /-33C8K!>'!I )e%')L z<nUUr    objfolderconfigc                 \     j                   dk(  rt        j                  d  d| d       y fd}t        |t        t
        f      r|D ]
  } ||        n
| ||       g }t        j                   j                      j                  }t        |      t        |      j                  z  }t        j                  ||       |j                  |       t        |      D ]J  }t        |      t        |      j                  z  }t        j                  ||       |j                  |       L |S )a  
    Save the modeling files corresponding to a custom model/configuration/tokenizer etc. in a given folder. Optionally
    adds the proper fields in a config.

    Args:
        obj (`Any`): The object for which to save the module files.
        folder (`str` or `os.PathLike`): The folder where to save.
        config (`PretrainedConfig` or dictionary, `optional`):
            A config in which to register the auto_map corresponding to this custom object.

    Returns:
        `List[str]`: The list of files saved.
    __main__z We can't save the code defining z in z as it's been defined in __main__. You should put this code in a separate module so we can include it in the saved folder and make it easier to share via the Hub.Nc                    
j                   j                  }|j                  d      d   }| d
j                   j                   }d|v rd }d }
j                   j                  j	                  d      rf| d
j                   j                   }t        
dd       Yt        
d      }|j                  }|j                  d      d   }| d|j                   }n| d
j                   j                   }||f}t        | t              r'| j                  di       }	||	
j                  <   |	| d<   y t        | dd       || j                  
j                  <   y 
j                  |i| _
        y )NrL   	TokenizerFastslow_tokenizer_classauto_map)	__class__
__module__rY   __name__r   rT   rM   dictr   _auto_classr   )_configmodule_namelast_module	full_namer   fast_tokenizer_classslow_tokenizerslow_tok_module_namelast_slow_tok_moduler   r   s             r   _set_auto_map_in_configz3custom_object_save.<locals>._set_auto_map_in_configS  sq   mm..!'',R0"m1S]]%;%;$<=	)##' #' }}%%..v6*5a8N8N7O'P$3 6=I%,S2H%IN+9+D+D(+?+E+Ec+J2+N(.B-C1^E\E\D]+^( +6a8N8N7O'P$-/CDIgt${{:r2H(1HS__%"*GJWj$/;09GS__- #;Gr    )r   ro   rp   rM   r3   tupler   r   __file__r   r!   r   r   r   rE   )	r   r   r   r   cfgresultobject_file	dest_fileneeded_files	   `        r   custom_object_saver   =  s    ~~#.se4x @ 	

 	<@ &4-( 	)C#C(	)		'F++cnn-66KV[ 1 6 67I
KKY'
MM) 1= !LD$5$:$:;	K+i !
 Mr    c                     t        d      )NzLoading this model requires you to execute custom code contained in the model repository on your local machine. Please set the option `trust_remote_code=True` to permit loading of this model.)r   )signumframes     r   _raise_timeout_errorr     s    
	c r       c                    | |rd} n|rt         dkD  rd }	 t        j                  t        j                  t              }t        j                  t                | >t        d| d| d      }|j                         dv rd} n|j                         dv rd} | >t        j                  d       	 |Ht        j                  t        j                  |       t        j                  d       n|rt        d d        |r|s| st        d
| d      | S # t        $ r t        d| d| d	      w xY w# |:t        j                  t        j                  |       t        j                  d       w w xY w)NFr   zThe repository for z contains custom code which must be executed to correctly load the model. You can inspect the repository content at https://hf.co/z.
You can avoid this prompt in future by passing the argument `trust_remote_code=True`.

Do you wish to run the custom code? [y/N] )yesy1T)non0rJ   zS.
Please pass the argument `trust_remote_code=True` to allow custom code to be run.zLoading z requires you to execute the configuration file in that repo on your local machine. Make sure you have read the code there to avoid malicious use, then set the option `trust_remote_code=True` to remove this error.)	TIME_OUT_REMOTE_CODEsignalSIGALRMr   alarminputlower	Exceptionr   )trust_remote_code
model_namehas_local_codehas_remote_codeprev_sig_handleranswers         r   resolve_trust_remote_coder    s     %!5!9#$#)==AU#V 12'/"-j\ :ccmbn oEFF ||~)::,0)+??,1) (/ Q $/MM&..2BCLLO t,~6Gzl #M M
 	
 -   )* 6__i^j khi  $/MM&..2BCLLO 0s   B D D D**D- ->E+)	NFNNNNFNNr~   )6__doc__rN   r   r   r   importlib.utilr   r0   r   r   r   	threadingr   pathlibr   typesr   typingr   r   r   huggingface_hubr   utilsr
   r   r   r   r   r   
get_loggerr   ro   Lockr   r   r<   PathLiker&   r3   r8   rE   ri   rv   booltyper   r   r   r   r   r   r   r   r    r   <module>r     s   : 
     	 	   
     ' ' 2  
		H	%%y~~' & &c2;;&6 7 &D &,'eC,<&= '$s) '* 5bkk1A+B  tCy  B-$%R[[ 01 -$d3i -$` *E#r{{"23  *S	  *N 	.+.+sBKK'(.+ 	.+
 
.+h 48 &*(,(,""#"&m5#(bkk)9#:m5m5 c2;;./0m5 	m5
 d^m5 d38n%m5 E$)$%m5 smm5 m5 }m5 3-m5 	m5f 48 &*(,(,""##'zVzV#(bkk)9#:zV c2;;./0zV 	zV
 d^zV d38n%zV E$)$%zV smzV zV }zV C=zV 
zVzIC IsBKK/?)@ I(SW. Idhildm IX  +r    