*#he dZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl m Z mZddlmZddlmZddlmZddlmZmZmZmZmZddlmZmZddlZddlZddl Z dd lm!Z!dd l"m#Z#d d l$m%Z%m&Z&d d l'm(Z(ddl$m)Z)ddl*m+Z+ddl,m-Z-e)j\e/Z0dZ1ede2eZ3dPdeeee2fde2fdZ4de2de5fdZ6de2de5fdZ7de2de5fdZ8de3de3fdZ9dQde2de2de2fdZ: dRde2de2dee jved$e2d%e2de2fd&Z?d'e2de2fd(Z@dPd)ZA dPde2fd*ZBdPd+eee2eCfde2fd,ZD dTd-e2d.eee2e5fd/eee2e5fdeCfd0ZEGd1d2eFZGdPd3ee2fd4ZHdde jjddd5d fd6eeKd7eeCd8ee jjd9eeeLd:eLd;eMde2d-e2d:eLd;eMdywrGN)r"replacer&seplstrip).0pathnames r+ z#url_or_path_join..qs2*t^f3x=+@+@+M+T+TUX+Y*tsAA)r5 posixpathr)ras_posix)r_r`s r+url_or_path_joinrlos>Y~~iu*tjs*tuuI* *3355r6 url_or_pathc~t|r|d|jdStjj |SrF)r5rindexr&r$dirname)rms r+url_or_path_parentrqvs6[!4[//455ww{++r6c|jd}t|}|j}|r2|jd}t|}|d|jzz }|jdr|dz }|S)a Convert `url` into a hashed filename in a repeatable way. If `etag` is specified, append its hash to the url's, delimited by a period. If the url ends with .h5 (Keras HDF5 weights) adds '.h5' to the name so that TF 2.0 can identify it as a HDF5 file (see https://github.com/tensorflow/tensorflow/blob/00fad90125b18b80fe054de1055770cfb8fe4ba3/tensorflow/python/keras/engine/network.py#L1380) utf-8.z.py)encoder hexdigestendswith)urletag url_bytesurl_hashrD etag_bytes etag_hashs r+hash_url_to_filenamer~}sx 7#Ii H!!#H [[) :& C)--/// ||EE Or6c | tdi|}|jxstj}t |t r t |}t |t r t |}t|rt|||j|j|j|j|j|j|j|j |j"|j$|j& }nKt(j*j-|r|}n)t/|rt1d|dt3d|d||S|j4r1t7|jj9||j:}|S) a Given something that might be a URL (or might be a local path), determine which. If it's a URL, download the file and cache it, and return the path to the cached file. If it's already a local path, make sure the file exists and then return the path. Return: Local path (string) Raises: FileNotFoundError: in case of non-recoverable file (non-existent or no cache on disk) ConnectionError: in case of unreachable url and no cache on disk ValueError: if it couldn't parse the url or filename correctly requests.exceptions.ConnectionError: in case of internet connection issue ) cache_dirforce_downloadproxiesresume_download user_agentlocal_files_onlyuse_etagrQtokenignore_url_paramsstorage_options download_descz Local file z doesn't existzunable to parse z as a URL or as a local path)r) force_extract)rrrDOWNLOADED_DATASETS_PATHr@rr"r5get_from_cacherrrrrrrQrrrrr&r$r(r8FileNotFoundError ValueErrorextract_compressed_filerextractr)r-download_configdownload_kwargsr output_paths r+ cached_pathrse,(;?;))LV-L-LI)T" N /4(o._%$ *99#+++;;&11,==$--'33!''-??+;;)77   (%  '+o->n MNN+O+<.sGDAqs!A3ZGs)rr PY_VERSIONhuggingface_hubPYARROW_VERSIONTORCH_AVAILABLE TORCH_VERSION TF_AVAILABLE TF_VERSION JAX_AVAILABLE JAX_VERSIONBEAM_AVAILABLE BEAM_VERSIONr@dictr)itemsr")ruas r+get_datasets_user_agentrs6 [M "BIf''( ))B ::; < !!"*-+ &  ;VY\]`Ya;a  "r6? func_args func_kwargs exceptions status_codesrQbase_wait_time max_wait_timec 4|xsd}|xsi}d} ||i|S#|$rx} ||k\s|r| jj|vr| t||d|zz} tj |d| d||z dt j | |dz }Yd} ~ nd} ~ wwxYw)Nrrrz timed out, retrying in zs... []r)response status_codeminloggerinfotimesleep) funcrrrrrQrrretryerr sleep_times r+_retryrsRI#K E  2k2 2  # 9Q9QYe9e  E0IJ  tf$NE7   '''bv||~3PWb[abHG  O##22H4G4G4W4WX '{"  vhl3%7PQVWbQbPccdef uqyAQ0QR  :&&  's)A 7CACCctd|tj||\}}}t|dkDrt d||j |dS)Nrrrz=HEAD can be called with at most one path but was called with r)rfsspecget_fs_token_pathslenrr)rxrfs_pathss r+ fsspec_headrTs\%u&=>,,S/RLB5 5zA~XY^X_`aa 7758 r6c eZdZdfd ZxZS) TqdmCallbackcBt||g|i|t|_yN)super__init__r_tqdm)self tqdm_kwargsargskwargs __class__s r+rzTqdmCallback.__init__]s! 6t6v6 r6r)rrrr __classcell__)rs@r+rr\s r6rc td|tj||\}}}t|dkDrt d|t |xsdt j ddd }|j|d |j| y) Nrrrz,,S/RLB5 5zA~WX]W^_``)M"::<<  HKKa)..8K V^^++C+A B a FF1I      s.*A!A A!AA!A!! A-,A-cxtd| tjd|d|jt t j j||5}tj||dddy#1swYyxYw#t jj$r}t|dd}~wwxYw)NrzGetting through FTP z into r) rrrrTrrrrshutil copyfileobjerrorURLErrorr)rxrrr es r+ftp_getr|s%u&=>+ *3%vinn5EFG V^^++C+A B -a   q) , - - - << +a d*+s6AB A?6B ?BB B B9( B44B9c Dtj|xsi}t|jd|d<|dkDr d|dd|d<t d|d ||||| } | j d k(ry| j jd } | |t| znd} tjd d | ||xsdtj 5} | jdD]-} | jt| |j| / dddy#1swYyxYw)N user-agentrrzbytes=d-RangeGETT)rrxstreamrrcookiesrQrizContent-Lengthrr)rrtotalinitialrri) chunk_size)copydeepcopyrgetrrrintrtqdmr iter_contentupdaterwrite)rxrr resume_sizerrrrQrrcontent_lengthrprogresschunks r+http_getr,s0mmG$*G3w{{a Given a URL, look for the corresponding file in the local cache. If it's not there, download it. Then return the path to the cached file. Return: Local path (string) Raises: FileNotFoundError: in case of non-recoverable file (non-existent or no cache on disk) ConnectionError: in case of unreachable url and no cache on disk rrrNTrF)ryr4rftpr1rr6ry)r/rrrQrdownload_warningzdrive.google.comz &confirm=zconfirm=z &confirm=tizfirebasestorage.googleapis.comiiz7^https?://github.com/.*?/.*?/releases/download/.*?/.*?$z#^https://.*?s3.*?amazonaws.com/.*?$zndownloader.figstatic.comz"Couldn't get ETag version for url izUnauthorized for URL zU. Please use the parameter `token=True` after logging in with `huggingface-cli login`z6Cannot find the requested files in the cached path at zi and outgoing traffic has been disabled. To enable file online look-ups, set 'local_files_only' to False.izCouldn't find file at rzCouldn't reach z ()z (error z.lockrc3XKt|5}|dddy#1swYyxYwwr)r*)modefincomplete_paths r+temp_file_managerz)get_from_cache..temp_file_managerns,ot,    s * *'*rza+b)r@zB not found in cache or force_download set to True, downloading to )rr)rr(rrrQrzstoring z in cache at izcreating metadata file for )rxryz.jsonr rs)encoding)zw+b)9rrrrHF_DATASETS_CACHEr@rr"r&r'r rr$r~r)r(rr2r rr"rRrrrrrrematchrxrrrrOSErrorrrTimeoutrrreprrrrstatst_sizerTrrr,rmoveumaskchmodr*jsondump)%rxrrr etag_timeoutrrrrrQrrrrr cached_url connectedrrry head_errorr2rD cache_pathrrrr lock_pathrCr(rrNmeta meta_path meta_filerBs% @r+rrs<%  ==K>* %n  #H U^^  !h&:&:c&A#&>* %n2'2' %}4      '(9 F ww~~o. ggo6>>   I KK3%abkbpbpaqr sY'003 ?Q^_# +## +&  (  hse= => INNJ/  UeV^, 1*>?D)( )S7 3 'y IIdI & 'c2'h o,,445 J  n  : ' 'c2'h sk(A0V/V/C1V/#W;A*W;1AW"B7W;W/W;/#WWW"W, 'W;/W8 4W;;Xcfd}|S)Ncpdjdz|j |jndz|_|SNr0z )r)__doc__fndocstrs r+docstring_decoratorz1add_start_docstrings..docstring_decorators1WWV_v-rzz?U[]^  r6rrarbs` r+add_start_docstringsrd r6cfd}|S)Ncp|j |jnddzdjz|_|Sr])r^r)r_s r+rbz/add_end_docstrings..docstring_decorators1$&JJ$:bjjfLrwwW]^  r6rrcs` r+add_end_docstringsrhrer6c&td|DS)Nc3PK|]}|jj ywr)rKrL)rgr$s r+riz(estimate_dataset_size..s5ttyy{""5s$&)sum)rs r+estimate_dataset_sizerls 5u5 55r6rAct} |jd}|s t|S||z }|jdr t|SB)Nr ) bytearrayr rwbytes)rAresbs r+readlinerssT +C  FF1I  : q <<   : r6r)FT)FTr)TN)Nr)rrr$@)NN)rt)NrNNgY@rN)NNNTrtr)NFNdFNFTrNrFNN)_r^r iorPr&rjrFrr#rrr contextlibrr functoolsrhashlibrpathlibrtypingrr r r r urllib.parser rrrrr packagingrr0rrdownload.download_configrrrrfilelockr get_loggerrrINCOMPLETE_SUFFIXr"rr,boolr5r8r;rBrOResponser rSr^rlrqr~rrrrrrrrRequestExceptiontupler#floatrrr callbacksrrr rr,rRr9rrdrhrl RawIOBasersrr6r+rs7   .77* $"5#   H %! CdhuT3Y/?&@C&s3s4sn3n4nYcYdYJAJ!J 6c6S6RU6NO" 8  i '(WW3W WY\W6666,C,C,4B BJsDy1A(Bc*fr eCI./HPQVWZ\`W`QaHb 2 ?  8C= "&"&=E=P=P=a=a(,$X((99:  49%   : $ $ $$ $  $  $$N6##00 = +qu#Fde &fr eCI./HPQVWZ\`W`QaHb c]*   N  !Nb6  r6