*#h}V2dZddlZddlZddlZddlZddlZddlZddlZddlmZddl m Z ddl m Z ddl mZmZmZmZmZmZmZmZddlmZdd lmZmZdd lmZmZmZmZm Z dd l!m"Z"dd l#m$Z$m%Z%m&Z&dd l'm(Z(m)Z)m*Z*ddl+m,Z,e$e-Z.gdZ/e0jcdde0jcdde0jcdde0jcdde0jcdde0jcdde0jcdde0jcddiZ2dd iZ3e4d!e e2e3DZ5Gd"d#ejlZ7Gd$d%eZ8d&e9d'e9fd(Z:d'ee9fd)Z;d&e9d'ee9fd*Z<Gd+d,eZ=Gd-d.e=Z>Gd/d0e=Z?Gd1d2Z@y)3zDownload manager interface.N)datetime)partial)chain)CallableDict GeneratorIterableListOptionalTupleUnion)config)DeprecatedEnum deprecated) cached_pathget_from_cachehash_url_to_filenameis_relative_pathurl_or_path_join)get_size_checksum_dict) get_loggeris_progress_bar_enabledtqdm)NestedDataStructure map_nestedsize_str)DownloadConfig) txtcsvjsonjsonltsvconllconlluorigparquetpklpicklerelxml504B0304zip504B0506504B0708425A68bz21F8Bgzip FD377A585A00xz04224D18lz428B52FFDzstdsRar!rarc#2K|]}t|ywN)len).0 magic_numbers _/var/www/html/sandstorm/venv/lib/python3.12/site-packages/datasets/download/download_manager.py rBFs sceZdZdZdZdZdZy) DownloadModea)`Enum` for how to treat pre-existing downloads and data. The default mode is `REUSE_DATASET_IF_EXISTS`, which will reuse both raw downloads and the prepared dataset if they exist. The generations modes: | | Downloads | Dataset | |-------------------------------------|-----------|---------| | `REUSE_DATASET_IF_EXISTS` (default) | Reuse | Reuse | | `REUSE_CACHE_IF_EXISTS` | Reuse | Fresh | | `FORCE_REDOWNLOAD` | Fresh | Fresh | reuse_dataset_if_existsreuse_cache_if_existsforce_redownloadN)__name__ __module__ __qualname____doc__REUSE_DATASET_IF_EXISTSREUSE_CACHE_IF_EXISTSFORCE_REDOWNLOADrArDrDLs 83)rPrDc(eZdZdZdZdZedZy) GenerateModerErFrGcy)NzUse 'DownloadMode' instead.rOselfs rA help_messagezGenerateMode.help_messagefs,rPN)rHrIrJrLrMrNpropertyrVrOrPrArRrRas&73) --rPrRpathreturncd|jdd}dD]}|j|d}|S)N.z?-_r)split)rX extensionsymbs rA_get_path_extensionr`ks> 3#I-OOD)!, - rPc |jd|j t }|jdt t D]W}tj|dt |z }||cStj|dt |z }|Jtd|dy#ttjf$rYywxYw)zQread the magic number from a file-like object and return the compression protocolrNzCompression protocol 'z' not implemented.) seekAttributeErrorioUnsupportedOperationreadMAGIC_NUMBER_MAX_LENGTHrange$MAGIC_NUMBER_TO_COMPRESSION_PROTOCOLget0MAGIC_NUMBER_TO_UNSUPPORTED_COMPRESSION_PROTOCOLNotImplementedError)fr@i compressions rA*_get_extraction_protocol_with_magic_numberrpus q 6612LFF1I * +`:>>|LiNehiNi?jk  " FJJ{mK]&^_ _ ` B33 4sB##B?>B?ct|}t|}|tvs|dvs|jdryt |d5}t |cdddS#1swYyxYw)N)tgztar)z.tar.gzz.tar.bz2z.tar.xzrb)strr`BASE_KNOWN_EXTENSIONSendswithopenrp)rXr^rms rA_get_extraction_protocolrysa t9D#D)I **  & ==; < dD =Q9!<===s  AAc"eZdZdZdefdZdZy)_IterableFromGeneratorzkUtility class to create an iterable from a generator function, in order to reset the generator when needed. generatorc.||_||_||_yr=r|argskwargs)rUr|rrs rA__init__z_IterableFromGenerator.__init__s"  rPc#lK|j|ji|jEd{y7wr=r~rTs rA__iter__z_IterableFromGenerator.__iter__s'!4>>499< <<A,A AA AA  A,A A  A)%A,c(||j|Sr=)r)rrs rAfrom_bufzArchiveIterable.from_bufs3))733rPc(||j|Sr=)r)rurlpath_or_bufs rA from_pathzArchiveIterable.from_paths3&&77rP)rYr)rHrIrJrK staticmethodrr classmethodrr rrurrrrOrPrArrsS  & &(itT0A&B((,c,itT8I.J,,4488rPrcXeZdZdZedeeeefdeeddffdZ eddZ y) FilesIterablez8An iterable of paths from a list of directories or filesurlpathsrYNc #rKt|ts|g}|D]}tjj |r4tjj |j drR|Wtj|D]\}}}t|Dcgc]}|j dr|c}|ddtjj |j drdt|D]6}|j drtjj||8ycc}wwr) isinstancelistrrXisfilerrwalksortedjoin)rrrdirpathdirnames filenamesdirnamers rA_iter_from_pathszFilesIterable._iter_from_pathss(D) zH >Gww~~g&77##G,77 D 46GGG4D >0GXy#))qgQXQcQcdoQp')q"rHQKww''0;;KH $*9$5>#..{;$ ggll7H== > > >*rsBD7D2 +D2 /BD7c(||j|Sr=)r)rrs rA from_pathszFilesIterable.from_pathss3''22rP)rYr) rHrIrJrKrr rur rrrrOrPrArrsSB>c49n(=>)CQUW[OB\>>.33rPrc (eZdZdZ ddeedeedeedeefdZedZ ed Z e d Z d e d e fd ZeddZdZdededefdZdeeej*ffdZdeeeeffdZddZdZdZdZdZy)DownloadManagerFN dataset_namedata_dirdownload_config base_pathc||_||_|xstjj d|_i|_||_|xs t|_ i|_ i|_ y)a4Download manager constructor. Args: data_dir: can be used to specify a manual directory to get the files from. dataset_name (`str`): name of dataset this instance will be used for. If provided, downloads will contain which datasets they were used for. download_config (`DownloadConfig`): to specify the cache directory and other download options base_path (`str`): base path that is used when relative paths are used to download files. This can be a remote url. record_checksums (`bool`, defaults to `True`): Whether to record the checksums of the downloaded files. If None, the value is inferred from the builder. r[N) _dataset_name _data_dirrrXabspath _base_path_recorded_sizes_checksumsrecord_checksumsrrdownloaded_pathsextracted_paths)rUrrrrrs rArzDownloadManager.__init__s\2*!#;rwws';Z\& 0.B.2B "!rPc|jSr=)rrTs rA manual_dirzDownloadManager.manual_dir$s ~~rPcVtd|jjDS)z+Returns the total size of downloaded files.c3&K|] }|d yw) num_bytesNrO)r?checksums_dicts rArBz2DownloadManager.downloaded_size..+sm>>+.ms)sumrvaluesrTs rAdownloaded_sizezDownloadManager.downloaded_size(s$mTEcEcEjEjElmmmrPcddlm|jjj d t dfdt fd|t }|S)aShip the files using Beam FileSystems to the pipeline temp dir. Args: downloaded_path_or_paths (`str` or `list[str]` or `dict[str, str]`): Nested structure containing the downloaded path(s). pipeline ([`utils.beam_utils.BeamPipeline`]): Apache Beam Pipeline. Returns: `str` or `list[str]` or `dict[str, str]` r)upload_local_to_remote temp_locationzFYou need to specify 'temp_location' in PipelineOptions to upload filesc (tjtjtj j |}tjd|dttj j|d|d|||S)Nz Uploading z (z) to r[) posixpathrrDOWNLOADED_DATASETS_DIRrrXrloggerinforgetsize)local_file_pathremote_file_path remote_dirrs rAuploadz8DownloadManager.ship_files_with_pipeline..uploadAs(~~F::BGGz:DownloadManager.ship_files_with_pipeline..Ls F?$;rP disable_tqdm)utils.beam_utilsr_optionsget_all_optionsrj ValueErrorrr)downloaded_path_or_pathspipelineuploaded_path_or_pathsrrrs @@@rAship_files_with_pipelinez(DownloadManager.ship_files_with_pipeline-sb >&&668<<_M  ef f $", ; $466"  &%rP url_or_urlsrc d}ttt|j|j|dt D]2\}}t ||j |jt|<4y)z)Record size/checksum of downloaded files.zComputing checksums)delaydescdisable)record_checksumN) rrr.flattenrrrrru)rUrrrurlrXs rA_record_sizes_checksumsz'DownloadManager._record_sizes_checksumsRsw [((*,D,L,L,NO P&/11  IC8Nd&;&;8D * *3s8 4 rPzCUse `.download`/`.download_and_extract` with `fsspec` URLs instead.c, |jjxstj |jj} fd}t ||t }t|}t|}t|j|jD]J\}} t| dd|d}|r|jjs2|||t| dd|L|j|||jS#t$rd}Y_wxYw)a Download given urls(s) by calling `custom_download`. Args: url_or_urls (`str` or `list` or `dict`): URL or `list` or `dict` of URLs to download and extract. Each URL is a `str`. custom_download (`Callable[src_url, dst_path]`): The source URL and destination path. For example `tf.io.gfile.copy`, that lets you download from Google storage. Returns: downloaded_path(s): `str`, The downloaded paths matching the given input `url_or_urls`. Example: ```py >>> downloaded_files = dl_manager.download_custom('s3://my-bucket/data.zip', custom_download_for_my_private_bucket) ``` cVtjjt|Sr=)rrXrr)r cache_dirs rAurl_to_downloaded_pathz?DownloadManager.download_custom..url_to_downloaded_pathys77<< +?+DE ErPrTF)rlocal_files_onlyuse_etag max_retries)rrrDOWNLOADED_DATASETS_PATHrrrrr.rrFileNotFoundErrorforce_downloadrdata) rUrcustom_downloadrrrrrXcachedrs @rAdownload_customzDownloadManager.download_custom`s,((22Uf6U6U **66  F$. "KBYB[>[$  *+6 #67O#P [0024L4T4T4VW IC 9tealT11@@T*9teal  $$[2JK',,,%  s!D DDc 0|jj}d|_|jd|_t |j |}t j}t||d|jt d}t j|z }tjd|jdzd t|}t|}|jj!t#t%|j'|j't j}|j)||t j|z }tjd |jdzd |j*S) ayDownload given URL(s). By default, only one process is used for download. Pass customized `download_config.num_proc` to change this behavior. Args: url_or_urls (`str` or `list` or `dict`): URL or `list` or `dict` of URLs to download. Each URL is a `str`. Returns: `str` or `list` or `dict`: The downloaded paths matching the given input `url_or_urls`. Example: ```py >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') ``` FDownloading datarTzDownloading data files) map_tuplenum_procrrzDownloading took <z minzChecksum Computation took )rcopyextract_compressed_file download_descr _downloadrnowrr rrr total_secondsrrupdatedictr.rrr)rUrr download_func start_timerdurations rAdownloadzDownloadManager.downloadsV&..33527/  ( ( 0,>O )P \\^ #-  $--466) $  <<>J. '(>(>(@B(F'GtLM)+6 #67O#P  $$T#k.A.A.CE]EeEeEg*h%ij\\^  $$[2JK<<>J. 01G1G1IR1O0PPTUV',,,rPurl_or_filenamerYctt|}t|rt|j|}t ||S)Nr )rurrrr)rUrrs rArzDownloadManager._downloads1o. O ,.tPO?OLLrP path_or_bufcnt|drtj|Stj|S)aKIterate over files within an archive. Args: path_or_buf (`str` or `io.BufferedReader`): Archive path or archive binary file object. Yields: `tuple[str, io.BufferedReader]`: 2-tuple (path_within_archive, file_object). File object is opened in binary mode. Example: ```py >>> archive = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') >>> files = dl_manager.iter_archive(archive) ``` rf)hasattrrrr)rUrs rA iter_archivezDownloadManager.iter_archives0( ; '"++K8 8",,[9 9rPpathsc,tj|S)aIterate over file paths. Args: paths (`str` or `list` of `str`): Root paths. Yields: `str`: File path. Example: ```py >>> files = dl_manager.download_and_extract('https://huggingface.co/datasets/beans/resolve/main/data/train.zip') >>> files = dl_manager.iter_files(files) ``` )rr)rUr!s rA iter_fileszDownloadManager.iter_filess"''..rPc |dk7rtjdt|jj }d|_|j d|_ttt|||jt d}t|}t|}|jjtt!|j#|j#|j$S)akExtract given path(s). Args: path_or_paths (path or `list` or `dict`): Path of file to extract. Each path is a `str`. num_proc (`int`): Use multi-processing if `num_proc` > 1 and the length of `path_or_paths` is larger than `num_proc`. Pass `DownloadConfig(num_proc=)` to the initializer instead. Returns: extracted_path(s): `str`, The extracted paths matching the given input path_or_paths. Example: ```py >>> downloaded_files = dl_manager.download('https://storage.googleapis.com/seldon-datasets/sentence_polarity_v1/rt-polaritydata.tar.gz') >>> extracted_files = dl_manager.extract(downloaded_files) ``` rz'num_proc' was deprecated in version 2.6.2 and will be removed in 3.0.0. Pass `DownloadConfig(num_proc=)` to the initializer instead.Tr r zExtracting data files)r rr)warningswarn FutureWarningrrrrrrrr rrrrrr.rr)rU path_or_pathsr rrs rAextractzDownloadManager.extracts6 | # MMb ..33526/  ( ( 0,>O )$ K A $--466(  ,M: -o> ##D]-B-B-DoF]F]F_)`$ab###rPcB|j|j|S)aDownload and extract given `url_or_urls`. Is roughly equivalent to: ``` extracted_paths = dl_manager.extract(dl_manager.download(url_or_urls)) ``` Args: url_or_urls (`str` or `list` or `dict`): URL or `list` or `dict` of URLs to download and extract. Each URL is a `str`. Returns: extracted_path(s): `str`, extracted paths of given URL(s). )r)r)rUrs rAdownload_and_extractz$DownloadManager.download_and_extract%s ||DMM+677rPc6|jjSr=)rrrTs rAget_recorded_sizes_checksumsz,DownloadManager.get_recorded_sizes_checksums7s--2244rPcvt|jjt|jjz }t |jj D]L\}}||vs t jj|s+t j||j|=Nyr=) setrrrritemsrrXrremove)rUpaths_to_deletekeyrXs rAdelete_extracted_filesz&DownloadManager.delete_extracted_files:sd2299;>$+? $((- .rPcR|jjr|jyyr=)rdelete_extractedr4rTs rAmanage_extracted_filesz&DownloadManager.manage_extracted_filesAs"    0 0  ' ' ) 1rP)NNNNT)r)rHrIrJ is_streamingr rurrrWrrrrrrrrrrr rdBufferedReaderr r r#r)r+r-r4r7rOrPrArrs9L'+"&48#' !"sm!"3-!"".1 !" C= !"Fnn"&"&H 3F bu UV.-W.-`.-`MM~MRUM:c23D3D.D(E:2/c49n 5/&/$b8$5.*rPr)ArKenumrdrrrr%rr functoolsr itertoolsrtypingrrrr r r r r rutils.deprecation_utilsrrutils.file_utilsrrrrrutils.info_utilsr utils.loggingrrrutils.py_utilsrrrrrrHrrvbytesfromhexrirkmaxrgEnumrDrRrur`rpryr{rrrrOrPrArHs " TTT@tt5EEFF+ H   MM*u MM*u MM*u MM(U MM&6 MM.!4 MM*u MM*v ($ U40BDtu *499**->-cc`Xc]`$ =3 =8C= = =X =:8,:8z3*3@E*E*rP