*#h~bddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl m Z ddlm Z ddl mZddlmZmZddlmZmZmZmZmZmZmZmZmZddlmZddlZddl m!Z!d d l"m#Z#d d l$m%Z%d d l&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,d d l-m.Z.d dl/m0Z0ddl1m2Z2e.e3Z4gdZ5ie%Dcic](}|jljod|jp*c}ddiZ9e%Dchc]}|jpc}Z:ejvdZZYd?dd@d*ee2fdAZZdUd*ee2fdBZ[GdCdDe\eZ]d5eeGee]ffdEZ^dd;d*ee2fdFZ_dd;d*ee2fdGZ`dUd*ee2fdHZadUd*ee2fdIZbdUd*ee2fdJZcdWd*ee2fdKZddUd*ee2fdLZeGdMdNeZfGdOdPefZgGdQdRefZhGdSdTZiycc}wcc}w)XN) TimeoutError)BytesIO)chain)Path PurePosixPath) AnyCallableDict GeneratorIterableListOptionalTupleUnion) ElementTree) ClientError)config)COMPRESSION_FILESYSTEMS)"get_authentication_headers_for_urlget_datasets_user_agent http_head is_local_pathis_relative_pathurl_or_path_join) get_logger) map_nested)DownloadConfig) txtcsvjsonjsonltsvconllconlluorigparquetpklpicklerelxml.zipz(?rCKs sc eZdZy)NonStreamableDatasetErrorN)__name__ __module__ __qualname__rBrErEQsrJrEct|jd^}}t|r tjj |g|St j |g|}dj |g|zS)u# This function extends os.path.join to support the "::" hop separator. It supports both paths and urls. A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". This is used to access files inside a zip file over http for example. Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. Then you can just chain the url this way: zip://folder1/file.txt::https://host.com/archive.zip The xjoin function allows you to apply the join on the first path of the chain. Example:: >>> xjoin("zip://folder1::https://host.com/archive.zip", "file.txt") zip://folder1/file.txt::https://host.com/archive.zip ::)strsplitrospathjoin posixpath)apbs rBxjoinrVUsb& FLL EAQww||A""" NN1 !q !yy!q!!rJc>t|jd^}}t|r7tjj t |j}ntj |}|jdr|dz }dj|g|zS)u# This function extends os.path.dirname to support the "::" hop separator. It supports both paths and urls. A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". This is used to access files inside a zip file over http for example. Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. Then you can just chain the url this way: zip://folder1/file.txt::https://host.com/archive.zip The xdirname function allows you to apply the dirname on the first path of the chain. Example:: >>> xdirname("zip://folder1/file.txt::https://host.com/archive.zip") zip://folder1::https://host.com/archive.zip rL://) rMrNrrOrPdirnameras_posixrRendswithrQrSrUs rBxdirnamer^ps|& FLL EAQ GGOODG,,. /   a  zz# T  99aS1W rJurlpathdownload_configct|jd^}}t|rtjj |St ||\}}tj||^}}|j |S)aExtend `os.path.exists` function to support both local and remote files. Args: urlpath (`str`): URL path. download_config : mainly use token or storage_options to support different platforms and auth types. Returns: `bool` rLr`storage_options) _as_strrNrrOrPexists!_prepare_path_and_storage_optionsfsspecget_fs_token_paths)r_r`main_hop rest_hopsrdfs_s rBxexistsrnsp#7+11$7HyXww~~h''#DW^m#n **7OTQyy""rJct|jd^}}t|r6tjj t |jStj |S)u This function extends os.path.basename to support the "::" hop separator. It supports both paths and urls. A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". This is used to access files inside a zip file over http for example. Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. Then you can just chain the url this way: zip://folder1/file.txt::https://host.com/archive.zip The xbasename function allows you to apply the basename on the first path of the chain. Example:: >>> xbasename("zip://folder1/file.txt::https://host.com/archive.zip") file.txt rL) rMrNrrOrPbasenamerr[rRr]s rB xbasenamerqsU& FLL EAQwwQ 0 0 233!!!$$rJcFt|jd^}}t|r6tjjt |j Stj|\}}dj|jdr|dzn|g|z|fS)u, This function extends os.path.split to support the "::" hop separator. It supports both paths and urls. A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". This is used to access files inside a zip file over http for example. Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. Then you can just chain the url this way: zip://folder1/file.txt::https://host.com/archive.zip The xsplit function allows you to apply the xsplit on the first path of the chain. Example:: >>> xsplit("zip://folder1/file.txt::https://host.com/archive.zip") ('zip://folder1::https://host.com/archive.zip', 'file.txt') rLrXrY) rMrNrrOrPrr[rRrQr\)rSrUtails rBxsplitrts& FLL EAQww}}T!W--/00//!$4yyajjo!d(1=ABDHHrJct|jd^}}t|r6tjj t |jStj |\}}dj|g|z|fS)u8 This function extends os.path.splitext to support the "::" hop separator. It supports both paths and urls. A shorthand, particularly useful where you have multiple hops, is to “chain” the URLs with the special separator "::". This is used to access files inside a zip file over http for example. Let's say you have a zip file at https://host.com/archive.zip, and you want to access the file inside the zip file at /folder1/file.txt. Then you can just chain the url this way: zip://folder1/file.txt::https://host.com/archive.zip The xsplitext function allows you to apply the splitext on the first path of the chain. Example:: >>> xsplitext("zip://folder1/file.txt::https://host.com/archive.zip") ('zip://folder1/file::https://host.com/archive.zip', '.txt') rL) rMrNrrOrPsplitextrr[rRrQ)rSrUexts rB xsplitextrxsr& FLL EAQwwQ 0 0 233##A&3yy!q!3&&rJreturnct|jd^}}t|rtjj |St ||\}}tj||^}}|j |S)zExtend `os.path.isfile` function to support remote files. Args: path (`str`): URL path. download_config : mainly use token or storage_options to support different platforms and auth types. Returns: `bool` rLrbrc) rMrNrrOrPisfilergrhri)rPr`rjrkrdrlrms rBxisfiler|smt9??40HyXww~~d## A$Xg ho**4QQyy""rJct|jd^}}t|rtjj |St ||\}}tj||^}}|j|}|1t||5}t|j}ddd|S|S#1swY|SxYw)zExtend `os.path.getsize` function to support remote files. Args: path (`str`): URL path. download_config : mainly use token or storage_options to support different platforms and auth types. Returns: `int`: optional rLrbrcN) rMrNrrOrPgetsizergrhrisizexopenr?read) rPr`rjrkrdrlrmrfs rBxgetsizerst9??40HyXwwt$$ A$Xg ho**4QQwwx  <t_= %1668} % t  % s B66CcNt|jd^}}t|rtjj |St ||\}}tj||^}}|jdd}|jdsy|j |S)zExtend `os.path.isdir` function to support remote files. Args: path (`str`): URL path. download_config : mainly use token or storage_options to support different platforms and auth types. Returns: `bool` rLrbrc://r/T) rMrNrrOrPisdirrgrhristrip)rPr`rjrkrdrlrm inner_paths rBxisdirrst9??40HyXww}}T"" A$Xg ho**4QQ^^E*1- $xx ##rJczt|jd^}}t|rB|r!tjj ||Stjj |S|r2t j |t|jddStjj |S)zExtend `os.path.relpath` function to support remote files. Args: path (`str`): URL path. start (`str`): Start URL directory path. Returns: `str` rL)startr)rMrNrrOrPrelpathrR)rPrrjrks rBxrelpathr4st9??40HyX9>rwwxu5]BGGOOT\D]]OTy  U1A1A$1G1JKsZ\ZaZaZiZijrZssrJcZ|jtjfd}||_y)Nc :d}tddzD] } |i|}|Std|#ttf$r\}|}tj dt j d|ddtjt j Yd}~d}~wwxYw)Nrz4Got disconnected from remote data host. Retrying in zsec [r]zServer Disconnected) rangerrloggerwarningrSTREAMING_READ_RETRY_INTERVALtimesleepConnectionError)argskwargsdisconnect_errretryouterr max_retriesrs rBread_with_retriesz?_add_retries_to_file_obj_read_method..read_with_retriesIs1kAo. ME AD+F+  M""78n L . A!$J6KoKoJppuv{u||}J~KKLM 6??@@  As/BABB)rrSTREAMING_READ_MAX_RETRIES)file_objrrrs @@rB$_add_retries_to_file_obj_read_methodrEs% ==D33K &HMrJrPcd|jdd}dD]}|j|d}|S)Nr-z?-_r)rN)rP extensionsymbs rB_get_path_extensionr\s> 3#I-OOD)!, - rJc |jd|j t }|jdt t D]W}tj|dt |z }||cStj|dt |z }|Jtd|dy#ttjf$rYywxYw)zQread the magic number from a file-like object and return the compression protocolrNzCompression protocol 'z' not implemented.) seekAttributeErrorioUnsupportedOperationrMAGIC_NUMBER_MAX_LENGTHr$MAGIC_NUMBER_TO_COMPRESSION_PROTOCOLget0MAGIC_NUMBER_TO_UNSUPPORTED_COMPRESSION_PROTOCOLNotImplementedError)rrAi compressions rB*_get_extraction_protocol_with_magic_numberrfs q 6612LFF1I * +`:>>|LiNehiNi?jk  " FJJ{mK]&^_ _ ` B33 4sB##B?>B?ct|}|jdd}t|}|tvs|dvs|j dry|t vr t |St ||\}} tj|fi|xsi5}t|cdddS#1swYyxYw#t$r0|jtjrt|dzdwxYw)NrLrtgztarz.tar.gzz.tar.bz2z.tar.xzrbS If the repo is private or gated, make sure to log in with `huggingface-cli login`.)rMrNrBASE_KNOWN_EXTENSIONSr\!COMPRESSION_EXTENSION_TO_PROTOCOLrgrhopenrFileNotFoundError startswithr HF_ENDPOINT)r_r`rPrrdrs rB_get_extraction_protocolrxs'lG == q !D#D)I**  & ==; < 7 70;;@ZijG_  [[ <_%: < A=a@ A A A    f00 1#pp   s*+B& B B&B#B&#B&&9Ccg}i}|jdD]4}t||\}}|j||j|6dj |fS)NrLrb)rN,_prepare_single_hop_path_and_storage_optionsappendupdaterQ)r_r`prepared_urlpathprepared_storage_optionshoprds rBrgrgsm!}}T"9KCapq_$ ''89 99% & 77rJcl|dn |j}d|vr|jddnd}|||jvr|j|}nV|R||jvrD|jjDcic]\}}|t j vr|| }}}ni}|r||i}|dvrit ||dtidd id |j|i||<d |vrnt|}d}|jjD]D\} } | jd s|d | zz }|j}d|i|j|i||<Fd |vr d|vr|dz }|jdr d||dd<||fS|dk(r(|tjd|j|i||<||fScc}}w)a\ Prepare the URL and the kwargs that must be passed to the HttpFileSystem or to requests.get/head In particular it resolves google drive URLs It also adds the authentication headers for the Hugging Face Hub, for both https:// and hf:// paths. Storage options are formatted in the form {protocol: storage_options_for_protocol} Nrrfile)httphttps)tokenz user-agent trust_envT)headers client_kwargszdrive.google.comdownload_warningz &confirm=cookieszconfirm=z &confirm=tz"https://raw.githubusercontent.com/identityrzAccept-Encodinghf)rendpoint)rrNrditemsrhavailable_protocolsrrrrrrrr) r_r`rprotocolrd option_name option_valueresponserkvs rBrrs:$+D1F1FE*/7*:w}}U#A&H"x?3R3R'R)99(C  $9X9X)X.=-L-L-R-R-T ) \&"<"<">>  %  #_5$$4WEJ57*40 % ""8R0 % !  ( )HG ((..0 j1<< 23{Q.G&..G1:G0iGZGZ[cegGh0iOH-  j  (Zw-F | #G   B CFPOH %i 01B C O ## T **% !!(B/% ! O ##M s8#F0rbrct|}|jd^}}t|rt||g|i|St ||\}}i||xsi} t j|g|d|i|j} t| | S#t $r } t| dk(r td| d} ~ wt$r0|jtjrt|dzdwxYw)aExtend `open` function to support remote files using `fsspec`. It also has a retry mechanism in case connection fails. The `args` and `kwargs` are passed to `fsspec.open`, except `token` which is used for queries to private repos on huggingface.co Args: file (`str`): Path name of the file to be opened. mode (`str`, *optional*, default "r"): Mode in which the file is opened. *args: Arguments to be passed to `fsspec.open`. download_config : mainly use token or storage_options to support different platforms and auth types. **kwargs: Keyword arguments to be passed to `fsspec.open`. Returns: file object rLrbmodezCannot seek streaming HTTP filezStreaming is not possible for this dataset because data host server doesn't support HTTP range requests. You can still load this dataset in non-streaming mode by passing `streaming=False` (default)Nr)rerNrrrgrh ValueErrorrMrErrrrr) rrr`rrfile_strrjrkrdres rBrrs"t}H#>>$/HyXHd4T4V44=hXghD/ 2 2?0b 2F;;t@@$@@EEG )2 O!  q66 6+y    ??6-- .#mm   s)B C+B//2;;BKKPRO\ijL hZs<.9:YFGjjjs!C*c+>Kt|jd^}}t|rtj|fi|Ed{yt ||\}}t j||^}}|jdd}|jdr|j|sgSt|jtr |jn|jd} |j|fi|D]%\} } } dj| d| g|z| | f'y7׭w) auExtend `os.walk` function to support remote files. Args: urlpath (`str`): URL root path. download_config : mainly use token or storage_options to support different platforms and auth types. **kwargs: Additional keyword arguments forwarded to the underlying filesystem. Yields: `tuple`: 3-tuple (dirpath, dirnames, filenames). rLNrbrcrrrr)rerNrrOwalkrgrhrirrrrrMrQ) r_r`rrjrkrdrlrmrrdirpathdirnames filenamess rBxwalkr9s#7+11$7HyX778.v...$EW^m#n **7OTQ^^E*1-   C *)=I",R[[#">2;;BKKPRO,3BGGJ,I&,I Z (GXy))zWI67)CDhPYY Y Z /sADDCDceZdZdZfdZddeefdZddeefdZdZ e ddZ e de fd Z e de fd Ze de fd Zd Zd ee dfddfdZd e ddfdZfdZxZS)xPathzHExtension of `pathlib.Path` to support both local paths and remote URLs.ct|}|jd^}}t|r|S|j dd}t j d|}||jdrdz }|Sdz }|S)NrL\rrrXrY)super__str__rNrreplace#SINGLE_SLASH_AFTER_PROTOCOL_PATTERNsubr\)selfpath_strrjrk path_as_posix __class__s rBrz xPath.__str__Ws7?$'~~d39  "O ((s3 ;??}U !7!7!<D CED rJr`c.tt||S)zExtend `pathlib.Path.exists` method to support both local and remote files. Args: download_config : mainly use token or storage_options to support different platforms and auth types. Returns: `bool` rb)rnrM)rr`s rBrfz xPath.existsass4y/BBrJc#6K|j}|jd^}}t|r#t|j |Ed{y|rD|d}t ||\}}|jdd|i}dj ||g|dd}nd}tjt|||^}} |j t||} | D]7} t|dj |jd| g|z9y7ȭw)a]Glob function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. Args: pattern (`str`): Pattern that resulting paths must match. download_config : mainly use token or storage_options to support different platforms and auth types. Yields: [`xPath`] rLNrrbrrrc) r[rNrrrrgrQrhrirVtyper) rpatternr` posix_pathrjrkr_rdrlrmrrs rBrz xPath.globls ]]_ )//59  "H~**73 3 3#A,+LWfu+v(#*==#7#:O"L!YY''JIabM'JK "&..uZ/I[jkFB GGE(G$<=M - ]  d4jr{{m3|n,M+NQZ+Z![\\ ]! 4sA DDC Dc .|jd|zfi|S)zRglob function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. Args: pattern (`str`): Pattern that resulting paths must match. Yields: [`xPath`] z**/)r)rrrs rBrglobz xPath.rglobstyy3F33rJrycRt|t|jS)zName function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. Returns: [`xPath`] )rr^r[rs rBparentz xPath.parents tDz(4==?344rJclt|jjddjS)zName function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. Returns: `str` rLr)rr[rNrr s rBrz xPath.name+T]]_2248;<AAArJclt|jjddjS)zStem function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. Returns: `str` rLr)rr[rNstemr s rBrz xPath.stemrrJclt|jjddjS)zSuffix function for argument of type :obj:`~pathlib.Path` that supports both local paths end remote URLs. Returns: `str` rLr)rr[rNsuffixr s rBrz xPath.suffixs+T]]_2248;<CCCrJc2tt|g|i|S)aExtend :func:`xopen` to support argument of type :obj:`~pathlib.Path`. Args: **args: Arguments passed to :func:`fsspec.open`. **kwargs: Keyword arguments passed to :func:`fsspec.open`. Returns: `io.FileIO`: File-like object. )rrM)rrrs rBrz xPath.opensSY0000rJrT.cTt|t|jg|S)zExtend :func:`xjoin` to support argument of type :obj:`~pathlib.Path`. Args: *p (`tuple` of `str`): Other path components. Returns: [`xPath`] )rrVr[rrTs rBjoinpathzxPath.joinpaths%tDz% 4!455rJc$|j|Sr>)rrs rB __truediv__zxPath.__truediv__s}}QrJc Rt|jd^}}t|r't|tt||St|dj t|t|j |jg|zS)NrL) rMrNrrr with_suffixrQrr[)rrrjrkrs rBrzxPath.with_suffixs"4yt49  "4:c%'"5f"=>? ?tDz$))ZT$Z h0G0S0STZ0[%\%e%e%g$hkt$tuvvrJr>)ryr)rFrGrH__doc__rrrrfrr propertyrrMrrrrrrrr __classcell__)rs@rBrrTsR Ch~&> C]Xn-E]@ 455BcBBBcBBDDD 1 65c? 6w 6 S W wwrJrcrt|tr t|Sttt|Sr>)rrrM)rPs rBreres("4/3t9JSs4y9I5JJrJcddl}t|dr|j|g|i|St|}|jt |d|g|i|SNrrrbrb)r5hasattrrrMr)filepath_or_bufferr`rrr5s rB xgzip_openr&s_!6*tyy+=d=f== !34tyy14Yk\`kdjkkrJcddl}t|dr|j|g|i|St|}|jt |d|g|i|Sr")numpyr$loadrMr)r%r`rrnps rB xnumpy_loadr+s_!6*rww);D;F;; !34rwwu/WiZ^ibhiirJc ddl}t|dr|j|fi|St|}|j dddk(rt |||d<|jt |d|fi|S)Nrrrinferrbr#)pandasr$read_csvrMrrrr%r`rpds rBxpandas_read_csvr2sz!6*r{{-888 !34 ::mW - 8$<=Oap$qF= !r{{5!3T?[f_effrJc ddl}t|dr |j|fi|St |} |jt|d|fi|S#t$r-|jt |j fi|cYSwxYw#t$r9|jt t|d|j fi|cYSwxYwr")r.r$ read_excelrrrrMrr0s rBxpandas_read_excelr5s!6* O 2==!3>v> >!!34  2=='94Q`!alekl l  O 2==);)@)@)B!CNvN N O   2==0$X]]_`dj  s"AB 3BB ?C  C c ddlm}t|dr|j|fi|S|jt |d|fi|Sr")scipy.iorr$loadmatr)r%r`rsios rB xsio_loadmatr:sH!6*s{{-888s{{5!3T?[f_effrJct|drtj||St|d|5}tj||cdddS#1swYyxYw)aExtend `xml.etree.ElementTree.parse` function to support remote files. Args: source: File path or file object. parser (`XMLParser`, *optional*, default `XMLParser`): Parser instance. download_config : mainly use token or storage_options to support different platforms and auth types. Returns: `xml.etree.ElementTree.Element`: Root element of the given source document. r)parserr#rbN)r$ETparser)sourcer<r`rs rB xet_parser@sPvvxxv.. 64 A .Q88Af- . . .s AAc t|dr*tjjj|fi|St |d|5}tjjj|fi|cdddS#1swYyxYw)aExtend `xml.dom.minidom.parse` function to support remote files. Args: filename_or_file (`str` or file): File path or file object. download_config : mainly use token or storage_options to support different platforms and auth types. **kwargs (optional): Additional keyword arguments passed to `xml.dom.minidom.parse`. Returns: :obj:`xml.dom.minidom.Document`: Parsed document. rr#rbN)r$r,domminidomr>r)filename_or_filer`rrs rBxxml_dom_minidom_parserE-sm(ww$$%5@@@ #T? K 6q77??((5f5 6 6 6s *A99Bc"eZdZdZdefdZdZy)_IterableFromGeneratorzkUtility class to create an iterable from a generator function, in order to reset the generator when needed. generatorc.||_||_||_yr>rHrr)rrHrrs rB__init__z_IterableFromGenerator.__init__Bs"  rJc#lK|j|ji|jEd{y7wr>rJr s rB__iter__z_IterableFromGenerator.__iter__Gs'!4>>499< <<A0A$A A$A"A$ A0 A$"A$$A-)A0c(||j|Sr>)rg)rfrQs rBfrom_bufzArchiveIterable.from_bufs3))733rJc*||j||Sr>)ri)rfurlpath_or_bufr`s rB from_urlpathzArchiveIterable.from_urlpaths3))>?KKrJr>)ryrO)rFrGrHr staticmethodr[rd classmethodr rrgrMrrrirkrnrIrJrBrOrOKsS  & &(itT0A&B((GK,,,4^,D, 5$$ %,,44L8N;SL_pLLrJrOc veZdZdZe ddeeeefdee de eddffdZ eddee ddfdZ y) FilesIterablez8An iterable of paths from a list of directories or filesNurlpathsr`ryc #Kt|ts|g}|D]}t||r t|j dr+|0t ||rt ||D]\}}}t|Dcgc]}|j dr|c}|ddt|j drPt|D]"}|j drt||$t|ycc}ww)NrbrR) rlistr|rqrrrsortedrVr) rfrsr`r_rrrrZr`s rB_iter_from_urlpathsz!FilesIterable._iter_from_urlpathss(D) zH 1Gw@W%00= A49'Sb4c 70GXy#))qgQXQcQcdoQp')q"rHQK )44[A $*9$57#..{;$#GX66 7 7(00+ 1*rsA0C72C2 C2 A*C7c*||j||Sr>)rw)rfrsr`s rB from_urlpathszFilesIterable.from_urlpathss3**HoFFrJr>) rFrGrHrrprrMr rrr rwryrIrJrBrrrrs{BZ^1S$s)^,1?G?W1 3d? #118Gh~6NGZiGGrJrrc eZdZdZdZ ddeedeedeedeefdZe d Z d Z d ed efd Z dZ d ed efdZdZdeeej$fd eefdZdeeeefd eefdZy)StreamingDownloadManagera Download manager that uses the "::" separator to navigate through (possibly remote) compressed archives. Contrary to the regular `DownloadManager`, the `download` and `extract` methods don't actually download nor extract data, but they rather return the path or url that could be opened using the `xopen` function which extends the built-in `open` function to stream data from remote files. TN dataset_namedata_dirr` base_pathc||_||_|xstjj d|_|xs t |_y)Nr-) _dataset_name _data_dirrOrPabspath _base_pathrr`)rr|r}r`r~s rBrKz!StreamingDownloadManager.__init__s<*!#;rwws';.B.2BrJc|jSr>)rr s rB manual_dirz#StreamingDownloadManager.manual_dirs ~~rJc6t|j|d}|S)aUNormalize URL(s) of files to stream data from. This is the lazy version of `DownloadManager.download` for streaming. Args: url_or_urls (`str` or `list` or `dict`): URL(s) of files to stream data from. Each url is a `str`. Returns: url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input url_or_urls. Example: ```py >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') ``` T map_tuple)r _downloadr url_or_urlss rBdownloadz!StreamingDownloadManager.downloads"!M rJr_ryc^t|}t|rt|j|}|Sr>)rMrrr)rr_s rBrz"StreamingDownloadManager._downloads(g, G $&t@GrJc6t|j|d}|S)aAdd extraction protocol for given url(s) for streaming. This is the lazy version of `DownloadManager.extract` for streaming. Args: url_or_urls (`str` or `list` or `dict`): URL(s) of files to stream data from. Each url is a `str`. Returns: url(s): (`str` or `list` or `dict`), URL(s) to stream data from matching the given input `url_or_urls`. Example: ```py >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') >>> extracted_files = dl_manager.extract(downloaded_files) ``` Tr)r_extract)rrrss rBextractz StreamingDownloadManager.extracts&dmm[DIrJct|}t||j}|jdd}t |}|dvs|j drt d|d||S|tvrUtjj|jdd}d|vr|d|jdn|}|d |d|S|d |S) NrbrLrrrz+Extraction protocol for TAR archives like 'z' is not implemented in streaming mode. Please use `dl_manager.iter_archive` instead. Example usage: url = dl_manager.download(url) tar_archive_iterator = dl_manager.iter_archive(url) for filename, file in tar_archive_iterator: ...r-rz://::) rMrr`rNrr\r!SINGLE_FILE_COMPRESSION_PROTOCOLSrOrPrprindex)rr_rrPr inner_files rBrz!StreamingDownloadManager._extractsg,+GTEYEYZ}}T"1%'-  &$--8Z*[%=gYG   N : :))'--*=a*@AJAD AR$>> archive = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') >>> files = dl_manager.iter_archive(archive) ``` rrb)r$rOrkrnr`)rrms rB iter_archivez%StreamingDownloadManager.iter_archive)s<( >6 *"++N; ;"//PTPdPd/e erJrscDtj||jS)aIterate over files. Args: urlpaths (`str` or `list` of `str`): Root paths. Yields: str: File URL path. Example: ```py >>> files = dl_manager.download_and_extract('https://huggingface.co/datasets/beans/resolve/main/data/train.zip') >>> files = dl_manager.iter_files(files) ``` rb)rrryr`)rrss rB iter_filesz#StreamingDownloadManager.iter_filesBs "**8TEYEY*ZZrJ)NNNN)rFrGrHr is_streamingrrMrrKrrrrrrrrrBufferedReaderr rrr rrIrJrBr{r{sL'+"&48#' Csm C3- C".1 C C= C(,///68(f5b6G6G1G+HfXV[_f2[5d3i#8[Xc][rJr{r>)r)NN)jrrrOrRrerTrxml.dom.minidomr,r]asynciorr itertoolsrpathlibrrtypingrr r r r r rrr xml.etreerr=rhaiohttp.client_exceptionsrrr filesystemsrutils.file_utilsrrrrrr utils.loggingrutils.py_utilsrr`rrFrrrlstriprrrcompilerbytesfromhexrrmaxr ExceptionrErVr^rMrnrqrtrxboolr|intrrrrrrrrgrrrrrrrrer&r+r2r5r:r@rErGrOrrr{)fs_classs0rBrsi    'YYY' 11''+ H %I`aXx  %x'8'88a% 5 %! H_$_8X%6%6$_!&0bjj&=# MM*u MM*u MM*u MM(U MM&6 MM.!4 MM*u MM*v ($ U40BDtu   "6>#S#8N+C#(%4I6'6#8N#;#t#&H^$<0$(>":$d$,t"&.cc`Xc]`$cH^:R gH^V6$ =X =<L,<L~"G*"GJd[d[sb%`s $-L'L,