
    |IgA                        d Z ddlZddlZddlZddlZddlmZ ddlmZ ddl	m
Z
 ddlmZ ddlmZmZmZmZmZmZmZmZ ddlmZ dd	lmZ d
dlmZmZmZmZmZm Z m!Z!m"Z" d
dl#m$Z$ d
dl%m&Z&m'Z' erd
dl(m)Z)  e jT                  e+      Z, ejZ                  d      Z.dZ/dddZ0e G d d             Z1e"	 	 	 d9dee1   dee2   de2de2dee2   dee2   deee2e2f      deee3   ee3   f   fd       Z4 G d de      Z5 G d  d!e      Z6	 	 	 d9d"d#d$edee2   deee2e2f      dee2   ddfd%Z7d&e3fd'Z8d(e3fd)Z9d*e3fd+Z:d"d#d,e2ddfd-Z;d"d#d.ed/e<d,e2ddf
d0Z=d.ed1e1d/e<dee2   fd2Z>d3ee   d4e2de6fd5Z?d"d#d6ee2   d/e<dee   fd7Z@d"d#d6ee2   d/e<dee   fd8ZAy):z.Git LFS related type definitions and utilities    N)	dataclass)ceil)getsize)Path)TYPE_CHECKINGBinaryIODictIterableListOptionalTuple	TypedDict)unquote)	constants   )build_hf_headersfix_hf_endpoint_in_urlget_sessionhf_raise_for_statushttp_backoffloggingtqdmvalidate_hf_hub_args)SliceFileObj)sha256sha_fileobj)CommitOperationAddz^[0-9a-f]{40}$zlfs-multipart-uploadzapplication/vnd.git-lfs+json)AcceptzContent-Typec                   r    e Zd ZU dZeed<   eed<   eed<   edefd       Z	edefd       Z
ed	efd
       Zy)
UploadInfoad  
    Dataclass holding required information to determine whether a blob
    should be uploaded to the hub using the LFS protocol or the regular protocol

    Args:
        sha256 (`bytes`):
            SHA256 hash of the blob
        size (`int`):
            Size in bytes of the blob
        sample (`bytes`):
            First 512 bytes of the blob
    r   sizesamplepathc                     t        |      }t        j                  |d      5 }|j                  d      d d }t	        |      }d d d         | |      S # 1 sw Y   xY w)Nrb   r!   r   r"   )r   ioopenpeekr   )clsr#   r!   filer"   shas         P/var/www/html/answerous/venv/lib/python3.12/site-packages/huggingface_hub/lfs.py	from_pathzUploadInfo.from_pathN   s_    t}WWT4  	$DYYs^DS)Fd#C	$ S88	$ 	$s    AAdatac                 b    t        |      j                         } | t        |      |d d |      S )Nr&   )r!   r"   r   )r   digestlen)r+   r0   r-   s      r.   
from_byteszUploadInfo.from_bytesV   s-    Tl!!#D	$t*SAA    fileobjc                     |j                  d      }|j                  dt        j                         t	        |      }|j                         }|j                  dt        j                          | |||      S )Nr&   r   r'   )readseekr(   SEEK_SETr   tell)r+   r6   r"   r-   r!   s        r.   from_fileobjzUploadInfo.from_fileobj[   sX    c"Q$'"||~Q$S88r5   N)__name__
__module____qualname____doc__bytes__annotations__intclassmethodstrr/   r4   r   r<    r5   r.   r    r    ;   sj     M
IM9S 9 9 Be B B 98 9 9r5   r    upload_infostoken	repo_typerepo_idrevisionendpointheadersreturnc                    ||nt         j                  }d}|t         j                  v rt         j                  |   }| d| | d}dddg| D 	cg c])  }	|	j                  j	                         |	j
                  d+ c}	d	d
}
|dt        |      i|
d<   i t        t        |      |xs i }t               j                  |||
      }t        |       |j                         }|j                  dd      }t        |t              st!        d      |D cg c]  }d|vst#        |       c}|D cg c]  }d|v st%        |       c}fS c c}	w c c}w c c}w )a  
    Requests the LFS batch endpoint to retrieve upload instructions

    Learn more: https://github.com/git-lfs/git-lfs/blob/main/docs/api/batch.md

    Args:
        upload_infos (`Iterable` of `UploadInfo`):
            `UploadInfo` for the files that are being uploaded, typically obtained
            from `CommitOperationAdd.upload_info`
        repo_type (`str`):
            Type of the repo to upload to: `"model"`, `"dataset"` or `"space"`.
        repo_id (`str`):
            A namespace (user or an organization) and a repo name separated
            by a `/`.
        revision (`str`, *optional*):
            The git revision to upload to.
        headers (`dict`, *optional*):
            Additional headers to include in the request

    Returns:
        `LfsBatchInfo`: 2-tuple:
            - First element is the list of upload instructions from the server
            - Second element is an list of errors, if any

    Raises:
        [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
            If an argument is invalid or the server response is malformed.
        [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
            If the server returned an error.
    N /z.git/info/lfs/objects/batchuploadbasic	multipartoidr!   r   )	operation	transfersobjects	hash_algonameref)rH   rM   jsonrY   zMalformed response from servererror)r   ENDPOINTREPO_TYPES_URL_PREFIXESr   hexr!   r   LFS_HEADERSr   r   postr   r^   get
isinstancelist
ValueError_validate_batch_actions_validate_batch_error)rG   rH   rI   rJ   rK   rL   rM   
url_prefix	batch_urlrR   payloadresp
batch_inforY   objs                  r.   post_lfs_batch_inforq   e   s   P $/xY5G5GHJI55566yA
*Aj\'2MNI{+ '

  }}((*
 G  '("34


' =bG
 =iwGDJnnY-Ggt$9:: 29O#G3<N	 	%O/6I'S.	s	#I 3
4 	PIs   .E	E E3	E=Ec                   "    e Zd ZU eed<   eed<   y)PayloadPartT
partNumberetagN)r=   r>   r?   rC   rB   rE   rF   r5   r.   rs   rs      s    O
Ir5   rs   c                   ,    e Zd ZU dZeed<   ee   ed<   y)CompletionPayloadTz?Payload that will be sent to the Hub when uploading multi-part.rV   partsN)r=   r>   r?   r@   rE   rB   r   rs   rF   r5   r.   rw   rw      s    I	Hr5   rw   rW   r   lfs_batch_actionc                 B   t        |       |j                  d      }|$t        j                  d| j                   d       y|d   d   }t        |       |d   j                  d      }|t        |       |j                  di       }|j                  d      }	t        |d	   |
      }
|		 t        |	      }	t        | ||	|
       nt        | |
       |t        |       t        |d	   |      }t               j                  |t        ||      | j                  j                   j#                         | j                  j$                  d      }t'        |       t        j                  | j                   d       y# t        t        f$ r t        d|	 d      w xY w)aw  
    Handles uploading a given object to the Hub with the LFS protocol.

    Can be a No-op if the content of the file is already present on the hub large file storage.

    Args:
        operation (`CommitOperationAdd`):
            The add operation triggering this upload.
        lfs_batch_action (`dict`):
            Upload instructions from the LFS batch endpoint for this object. See [`~utils.lfs.post_lfs_batch_info`] for
            more details.
        headers (`dict`, *optional*):
            Headers to include in the request, including authentication and user agent headers.

    Raises:
        [`ValueError`](https://docs.python.org/3/library/exceptions.html#ValueError)
            If `lfs_batch_action` is improperly formatted
        [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
            If the upload resulted in an error
    actionsNzContent of file z. is already present upstream - skipping uploadrR   verifyheader
chunk_sizehref)rL   zTMalformed response from LFS batch endpoint: `chunk_size` should be an integer. Got 'z'.)rW   r}   r~   
upload_url)rW   r   )rH   rM   rU   r]   z: Upload successful)ri   re   loggerdebugpath_in_repo_validate_lfs_actionr   rC   rh   	TypeError_upload_multi_part_upload_single_partr   rd   r   upload_infor   rb   r!   r   )rW   ry   rH   rM   rL   r{   upload_actionverify_actionr}   r~   r   
verify_urlverify_resps                r.   
lfs_uploadr      s   8 ,-""9-G'	(>(>'??mno %Y/9M'$Y/33H=M ]+ x,FL)J'f(=QJ	ZJ
 	Yv*akliJG  ]++M&,A8L
!m(($5'B"..5599;YEZEZE_E_` ) 

 	K(
LLI**++>?@% I& 	fgqfrrtu 	s   -E? ?F
lfs_actionc                     t        | j                  d      t              r0| j                  d      *t        | j                  d      t              st	        d      | S ).validates response from the LFS batch endpointr   r}   z"lfs_action is improperly formatted)rf   re   rE   dictrh   )r   s    r.   r   r     sK     	:>>&)3/^^H%-JNN8<TVZ1[=>>r5   lfs_batch_actionsc                 P   t        | j                  d      t              rt        | j                  d      t              st	        d      | j                  di       j                  d      }| j                  di       j                  d      }|t        |       |t        |       | S )r   rV   r!   z)lfs_batch_actions is improperly formattedr{   rR   r|   )rf   re   rE   rC   rh   r   )r   r   r   s      r.   ri   ri     s    (,,U3S9jIZI^I^_eIfhk>lDEE%)))R8<<XFM%)))R8<<XFM ]+ ]+r5   lfs_batch_errorc                 l   t        | j                  d      t              rt        | j                  d      t              st	        d      | j                  d      }t        |t
              r>t        |j                  d      t              rt        |j                  d      t              st	        d      | S )r   rV   r!   z'lfs_batch_error is improperly formattedr_   messagecode)rf   re   rE   rC   rh   r   )r   
error_infos     r.   rj   rj     s    **5137JGZGZ[aGbdg<hBCC $$W-J:t$z~~i0#6z~~f-s3BCCr5   r   c                     | j                  d      5 }t        d||d      }t        |       ddd       y# 1 sw Y   yxY w)a  
    Uploads `fileobj` as a single PUT HTTP request (basic LFS transfer protocol)

    Args:
        upload_url (`str`):
            The URL to PUT the file to.
        fileobj:
            The file-like object holding the data to upload.

    Returns: `requests.Response`

    Raises:
     [`HTTPError`](https://requests.readthedocs.io/en/latest/api/#requests.HTTPError)
        If the upload resulted in an error.
    T	with_tqdmPUTi  i  i  i  r0   retry_on_status_codesN)as_filer   r   )rW   r   r6   responses       r.   r   r   ,  sE      
		T		* &gzWklH%& & &s	   7A r}   r~   c                    t        || j                  |      }t        j                  }t        j                  rKt	        | j
                  t              s1t	        | j
                  t              st        j                  d       d}|rt        | ||      nt        | ||      }t               j                  |t        || j                  j                  j!                               t"              }t%        |       y)z@
    Uploads file using HF multipart LFS transfer protocol.
    )r}   r   r~   zlhf_transfer is enabled but does not support uploading from bytes or BinaryIO, falling back to regular uploadF)rW   sorted_parts_urlsr~   )r^   rM   N)_get_sorted_parts_urlsr   r   HF_HUB_ENABLE_HF_TRANSFERrf   path_or_fileobjrE   r   warningswarn_upload_parts_hf_transfer_upload_parts_iterativelyr   rd   _get_completion_payloadr   rb   rc   r   )rW   r}   r~   r   r   use_hf_transferresponse_headerscompletion_ress           r.   r   r   B  s    
 /f)J_J_lvw  99O++944c:944d;	
    	"IIZgqr&N_lvw  !]''$%5y7L7L7S7S7W7W7YZ ( N
 'r5   r   c           
      V   t        | j                         D cg c]1  \  }}|j                         rt        |      dkD  rt	        |d      |f3 c}}d       D cg c]  \  }}|	 }}}t        |      }|t        |j                  |z        k7  rt        d      |S c c}}w c c}}w )Nr   
   c                     | d   S )Nr   rF   )ts    r.   <lambda>z(_get_sorted_parts_urls.<locals>.<lambda>n  s
    !A$ r5   )keyz0Invalid server response to upload large LFS file)sorteditemsisdigitr3   rC   r   r!   rh   )r}   r   r~   part_numr   _sorted_part_upload_urls	num_partss           r.   r   r   e  s     $ -3LLN(Hj##%#h-!*; Xr"J/
 

Az 	
 
 +,ID))J677KLL""
s   6B
B%r   rV   c                     g }t        |       D ]H  \  }}|j                  d      }||dk(  rt        d| d|dz          |j                  |dz   |d       J ||dS )Nru   rP   zInvalid etag (`z`) returned for part r   )rt   ru   )rV   rx   )	enumeratere   rh   append)r   rV   rx   part_numberr}   ru   s         r.   r   r   w  s     "E()9: 	
Vzz&!<42:tf4I+XY/IZ[\\)Ao	
		
 ''r5   r   c           	      8   g }| j                  d      5 }t        |      D ]T  \  }}t        |||z  |      5 }t        d||d      }t	        |       |j                  |j                         d d d        V 	 d d d        |S # 1 sw Y   kxY w# 1 sw Y   |S xY w)NTr   )	seek_from
read_limitr   r   r   )r   r   r   r   r   r   rM   )	rW   r   r~   rM   r6   part_idxpart_upload_urlfileobj_slicepart_upload_ress	            r.   r   r     s     G			T		* 8g)23D)E 	8%Ho$x/% 
8 ".?Vj# $O4667
8 
8	88 N
8 
88 Ns"   #B6B.
BBBBc                 z   	 ddl m} dt	        j
                  |      j                  v }|st        j                  d       | j                  j                  }| j                  }t        |      dkD  rd|dd   }t        j                         t        j                   k(  rd	nd }t#        d
d	|d||d      5 }	  |d| j$                  ||dddd|rd|j&                  ini }	|s|j'                  |       |	cd d d        S # t        $ r t        d      w xY w# t(        $ r}
t+        d      |
d }
~
ww xY w# 1 sw Y   y xY w)Nr   )multipart_uploadzFast uploading using 'hf_transfer' is enabled (HF_HUB_ENABLE_HF_TRANSFER=1) but 'hf_transfer' package is not available in your environment. Try `pip install hf_transfer`.callbackzYou are using an outdated version of `hf_transfer`. Consider upgrading to latest version to enable progress bars using `pip install -U hf_transfer`.(   u   (…)iTBzhuggingface_hub.lfs_upload)unit
unit_scaletotalinitialdescdisabler[            )	file_path
parts_urlsr~   	max_filesparallel_failuresmax_retriesz~An error occurred while uploading using `hf_transfer`. Consider disabling HF_HUB_ENABLE_HF_TRANSFER for better error handling.rF   )hf_transferr   ImportErrorrh   inspect	signature
parametersr   r   r   r!   r   r3   r   getEffectiveLevelr   NOTSETr   r   update	ExceptionRuntimeError)rW   r   r~   r   supports_callbackr   r   r   progressoutputes              r.   r   r     sw   
0 #g&7&78H&I&T&TT c	
 !!&&E!!D
4y2~tCDzl# //1W^^Cd$G	)
  
	% #33,%"% 5FJ02F !OOE"5 +  
Q
 	

P  	* 	' s;   C< 3D15)DD1<D	D.D))D..D11D:)NNN)Br@   r   r(   rer   dataclassesr   mathr   os.pathr   pathlibr   typingr   r   r	   r
   r   r   r   r   urllib.parser   huggingface_hubr   utilsr   r   r   r   r   r   r   r   
utils._lfsr   	utils.shar   r   _commit_apir   
get_loggerr=   r   compile	OID_REGEXLFS_MULTIPART_UPLOAD_COMMANDrc   r    rE   r   rq   rs   rw   r   r   ri   rj   r   rC   r   r   r   r   r   rF   r5   r.   <module>r      s   5  	 	  !    \ \ \   %	 	 	 % * /			H	%BJJ()	5  -2 &9 &9 &9R  #"(,K:&KC=K K 	K
 smK smK d38n%K 4:tDz!"K K\9 
   (,"CA#CACA C=CA d38n%	CA
 smCA 
CALT t 4 &#7 &S &T &, ("6  (  (RU  (cf  (ko  (F#4 #j #c #VZ[^V_ #$(d4j (s (GY (#8<S	OR	$Z(5#58<S	5OR5	$Z5r5   