*#h(i dZddlZddlZddlZddlZddlZddlmZddlm Z ddl m Z m Z m Z mZddlZddlmZddlmZddlmZmZdd lmZmZmZmZdd lmZdd l m!Z!e rdd l"m#Z#dd l$m%Z%m&Z&ejNe(Z)dZ*ejVdedddejXZ-ejVdZ.Gdde/Z0Gdde/Z1edGddZ2 d3de3de ddee3dfd e e3d!e e3d"e2f d#Z4Gd$d%Z5Gd&d'e5Z6Gd(d)e5Z7edGd*d+Z8edGd,d-Z9d.Z:d/Z;d0Zy)4z Arrow ArrowReader.N) dataclass)Path) TYPE_CHECKINGListOptionalUnion)DownloadConfig) _split_refilenames_for_dataset_split) InMemoryTableMemoryMappedTableTable concat_tables)logging) cached_path) DatasetInfo)Split SplitInfoz=https://storage.googleapis.com/huggingface-nlp/cache/datasetsz ^ (?Pz) (\[ ((?P-?\d+) (?P%)?)? : ((?P-?\d+) (?P%)?)? \])?(\((?P[^\)]*)\))? $ z\s*\+\s*ceZdZdZy)DatasetNotOnHfGcsErrorz?When you can't get the dataset from the Hf google cloud storageN__name__ __module__ __qualname____doc__R/var/www/html/sandstorm/venv/lib/python3.12/site-packages/datasets/arrow_reader.pyrrAsIrrceZdZdZy)MissingFilesOnHfGcsErrorz9When some files are missing on the Hf oogle cloud storageNrrrr r"r"GsCrr"T)frozenc,eZdZUdZeed<eeed<y)FileInstructionsa}The file instructions associated with a split ReadInstruction. Attributes: num_examples: `int`, The total number of examples file_instructions: List[dict(filename, skip, take)], the files information. The filenames contains the relative path, not absolute. skip/take indicates which example read in the file: `ds.slice(skip, take)` num_examplesfile_instructionsN)rrrrint__annotations__rdictrrr r%r%MsDz!rr%name split_infosr instructionReadInstructionfiletype_suffix prefix_pathreturnc @t|ts!tdt|j|s t d|Dcic]}|j |j}}|Dcic]}|j |j}}|Dcic]4}|j t|||j |||j 6}}t|tstj|}|j|} g} d} | D]} || j} || j}|| j}| jdn | j}| j| n | j}|(|D]"}| ||z z } | j!||||z d$d}d}t#||D]^\}}||z }||krJ||kDrE||kDr||z nd}||kr||z |z nd}|dk(r6| j!|||d| |dk(r||z n|z } ||z }`t%| | Scc}wcc}wcc}w)aReturns instructions of the split dict. Args: name (`str`): Name of the dataset. split_infos (`list` of `[SplitInfo]`): Dataset splits information. instruction ([`ReadInstruction`] or `str`): Reading instruction for a dataset. filetype_suffix (`str`, *optional*): Suffix of dataset files, e.g. 'arrow' or 'parquet'. prefix_path (`str`, *optional*): Prefix of dataset files, e.g. directory name. Returns: [`FileInstructions`] zExpected str 'name', but got: zExpected non-empty str 'name')path dataset_namesplitr/ shard_lengthsr)filenameskiptaker)r&r') isinstancestr TypeErrortyper ValueErrorr+r&r6r r. from_spec to_absolute splitnamefrom_toappendzipr%)r+r,r-r/r0infoname2lenname2shard_lengthsname2filenamesabsolute_instructionsr'r& abs_instr split_length filenamesr6rBrCr7 index_start index_end shard_lengthr8r9s r make_file_instructionsrQ\s& dC 8d9L9L8MNOO 8999DE 4,,,EHEDOPD$))T%7%77PP    .))+,TYY7    N  k? 3%// < '33H=L*,  3 34 "9#6#67 *9+>+>? __,)//&\\1\y||  % dU * !((hWY\aWa)bc dKI*-i*G ,&,\) 9$k)927+2E5;.1D689n2 +d2"Dqy %,,(DZ^-_` 42:L4$74OL|+  ,,. !+ WFP sH'H 9HceZdZdZdededfdZddefdZddefdZ d Z dd Z dd e e d edfdZdefdZy ) BaseReaderz@ Build a Dataset object out of Instruction instance(s). r3rFrc.||_||_d|_y)zInitializes ArrowReader. Args: path (str): path where tfrecords are stored. info (DatasetInfo): info about the dataset. N)_path_info_filetype_suffix)selfr3rFs r __init__zBaseReader.__init__s .2 /3rr1ct)=Returns a Dataset instance from given (filename, skip, take).)NotImplementedError)rXfilename_skip_take in_memorys r _get_table_from_filenamez#BaseReader._get_table_from_filenames!!rct|dk(std|Ds tdg}tj|}|D]2}t j j|j|d|d<4|D]&}|j||}|j|(|Dcgc]}t|dkDs|}}|s-|j|jj td|xsItjgtj |jjj"g}t|dk7r t%|}|S|d}|Scc}w) aReturns Dataset for given file instructions. Args: files: List[dict(filename, skip, take)], the files information. The filenames contain the absolute path, not relative. skip/take indicates which example read in the file: `ds.slice(skip, take)` in_memory (bool, default False): Whether to copy the data in-memory. rc3<K|]}t|tywN)r:r*).0fs r z)BaseReader._read_files..s%IajD&9%Isz&please provide valid file informationsr7r^zqTried to read an empty table. Please specify at least info.features to create an empty table with the right type.)schemar )lenallr>copydeepcopyosr3joinrUr_rDrVfeaturesr from_batchespargr=r)rXfilesr^ pa_tablesrdf_dictpa_tablets r _read_fileszBaseReader._read_filessU u:?#%I5%I"IEF F  e$ DAGGLLQz]CAjM D 'F";;Fi;XH   X & '!*81SVaZQ8 8djj0DJJ4G4G4OD m-"<" read_files)rXr+ instructionsr,r^rqmsgs r readzBaseReader.readsJ***4{K!,/HICS/ !U,ZcddrNrqrz)Nr.rc|j||}|ddlm}|t|}nd}||j|d}|S)aJReturns single Dataset instance for the set of file instructions. Args: files: List[dict(filename, skip, take)], the files information. The filenames contains the relative path, not absolute. skip/take indicates which example read in the file: `ds.skip().take()` original_instructions: store the original instructions used to build the dataset split in the dataset. in_memory (bool, default False): Whether to copy the data in-memory. Returns: kwargs to build a Dataset instance. rfNr )r) arrow_tablerFr5)rvsplitsrr;rV)rXrqrzr^rtrr5dataset_kwargss r r{zBaseReader.read_filessN&##EY#? , %#345EE)14::PUVrdownload_configcrtdz|jtjdz} tjj |d}t |jtjd}tj|tjj |jd|j>|jj|jj|j |jjD]}|j!|jj"||jjj%}|D]} t't)| dj+|j} tjj || } t | jtjd|} tj| | dy#t$r}t|dd}~wwxYw#t$r}t-|dd}~wwxYw)a% Download the dataset files from the Hf GCS Args: dl_cache_dir: `str`, the local cache directory used to download files relative_data_dir: `str`, the relative directory of the remote files from the `datasets` directory on GCS. /zdataset_info.jsonN)r+r-r,r7)r)HF_GCP_BASE_URLreplacerlsepr3rmrshutilmoverUrVupdatefrom_directoryFileNotFoundErrorrrrx builder_namevaluesr;r relative_tor") rXrrelative_data_dirremote_cache_dirremote_dataset_infodownloaded_dataset_infoerrr5r'file_instructionfile_to_downloadremote_prepared_filenamedownloaded_prepared_filenames r download_from_hf_gcszBaseReader.download_from_hf_gcss+S03D3L3LRVVUX3YY 8"$'',,/?AT"U &12E2M2MbffVY2Z&[ # KK/djjJ]1^ _zz% !!$**";";DJJ"GH :** \$($>$>00 % $ 1 1 8 8 :%?%! ):\$'*40@0L+M+Y+YZ^ZdZd+e'f$/1ww||088EWf40KK <>Nz>Z[ \ \! 8(-4 7 8 ! :*3/T 9 :s1CG?>DH? H HH H6% H11H6F)NF)rrrrr;rrYrr_rvrxr~rr*rr{r rrrr rSrSs~ 4S 4(? 4"u"U6 e<JN Dz %%EF<":N":rrScXeZdZdZdededffd Zd defdZe d defdZ xZ S) ArrowReaderz Build a Dataset object out of Instruction instance(s). This Reader uses either memory mapping or file descriptors (in-memory) on arrow files. r3rFrc4t|||d|_y)zInitializes ArrowReader. Args: path (str): path where Arrow files are stored. info (DatasetInfo): info about the dataset. arrowNsuperrYrWrXr3rF __class__s r rYzArrowReader.__init__?s t$ 'rr1c|dd|vr|dndd|vr|dnd}}}tj||}|dk(rt||z }|'|%|dk(r|t|k(s|j||}|S)r[r7r8Nr9rfrr)r read_tablerhslice)rXr]r^r7r8r9tables r r_z$ArrowReader._get_table_from_filenameIs z **04F*F v &D*04F*F v &D$ &&x9&E 2:u:$D   0$!)PSTYPZHZKKd+E rc@|rtnt}|j|S)z Read table from file. Args: filename (str): File name of the table. in_memory (bool, default=False): Whether to copy the data in-memory. Returns: pyarrow.Table )r r from_file)r7r^ table_clss r rzArrowReader.read_tableXs&/M4E ""8,,rr) rrrrr;rrYrr_ staticmethodr __classcell__rs@r rr9sG (S((?( u  - - -rrc8eZdZdZdededffd ZdZxZS) ParquetReaderzv Build a Dataset object out of Instruction instance(s). This Reader uses memory mapping on parquet files. r3rFrc4t|||d|_y)zInitializes ParquetReader. Args: path (str): path where tfrecords are stored. info (DatasetInfo): info about the dataset. parquetNrrs r rYzParquetReader.__init__ns t$ )rc |dd|vr|dndd|vr|dnd}}}tj|d}|'|%|dk(r|t|k(s|j||}|S)r[r7r8Nr9T) memory_mapr)pqrrhr)rXr]kwargsr7r8r9rts r r_z&ParquetReader._get_table_from_filenamexs z **04F*F v &D*04F*F v &D$ ==d;   0$!)PST\P]H]~~dD1Hr) rrrrr;rrYr_rrs@r rrhs% *S*(?* rrc0eZdZUdZeed<eed<eed<y)_AbsoluteInstructionz?A machine friendly slice: defined absolute positive boundaries.rArBrCN)rrrrr;r)r(rrr rrsIN J GrrcreZdZUdZeed<dZeeed<dZ eeed<dZ eeed<dZ eeed<dZ y) _RelativeInstructionzHRepresents a single parsed slicing instruction, can use % and negatives.rANrBrCunitroundingcH|j|jdvr td|j|jdvr td|jdk7r|j td|jdk(r/|j#t |jdkDr td|jdk(r/|j #t |j dkDr td|j|jdk(rd n |j|j d <y) N)%abszunit must be either % or abs)closestpct1_dropremainderz5rounding must be either closest or pct1_dropremainderrzAIt is forbidden to specify rounding if not using percent slicing.dz2Percent slice boundaries must be > -100 and < 100.rr)rr>rrBrrC__dict__rXs r __post_init__z"_RelativeInstruction.__post_init__s 99 TYYl%B;< < == $>_)_TU U 99  9`a a 99  63tzz?S;PQR R 99  3DGG s8JQR R151F499X[K[Iaeanan j!r) rrrrr;r)rBrr(rCrrrrrr rrsHRNE8C=B D(3-"Hhsm" orrc tj|}|std||jds|jdrdnd}t |jd|jd|jdrt |jdnd |jd r t |jd | Sd | S) z)Returns ReadInstruction for given string.z!Unrecognized instruction format: from_pctto_pctrrr5rfromNrC) split_namerrBrCr) _SUB_SPEC_REmatchr>groupr.r()specresrs r _str_to_read_instructionrs   T "C 3ED 99W%:&(+ &(9c#))F#$t#&99T?3syy    9=   rc\|dkr d}t||tj|dz zS)NrzUsing "pct1_dropremainder" rounding on a split with less than 100 elements is forbidden: it always results in an empty dataset.Y@)r>mathtrunc)boundaryr&r}s r _pct_to_abs_pct1rs:c L o djj!56 66rc6tt||zdz S)Nr)r(round)rr&s r _pct_to_abs_closestrs uX ,u45 66rc |jdk(rtnt}|j}||vrt d|dt |d||}|j }|j}|jdk(r|dn|||}||n|||}n |dn|}||n|}t||kDst||kDrd|xsd d |xsd d |d }t ||dkr||z}n|dk(rd}|dkr||z}n||k(rd}t|||S) zReturns _AbsoluteInstruction instance for given RelativeInstruction. Args: rel_instr: RelativeInstruction instance. name2len: dict {split_name: num_examples}. rzUnknown split "z". Should be one of .rNrzRequested slice [:z] incompatible with z examples.) rrrrAr>listrBrCrrr) rel_instrrG pct_to_absr5r&rBrCr}s r _rel_to_abs_instrrs>)2(:(:i(G$M]J   E H?5'1Ed8nEUUVWXXE?L OOE B~~] 5,(GZ\ZL-I]Z\R 5zL CGl$:!%+2abz9Ml^[efo qyu$ ! Av B  |   ub 11rc\eZdZdZdZedZd dZedZdZ dZ d Z d Z d Z y) r.aReading instruction for a dataset. Examples:: # The following lines are equivalent: ds = datasets.load_dataset('mnist', split='test[:33%]') ds = datasets.load_dataset('mnist', split=datasets.ReadInstruction.from_spec('test[:33%]')) ds = datasets.load_dataset('mnist', split=datasets.ReadInstruction('test', to=33, unit='%')) ds = datasets.load_dataset('mnist', split=datasets.ReadInstruction( 'test', from_=0, to=33, unit='%')) # The following lines are equivalent: ds = datasets.load_dataset('mnist', split='test[:33%]+train[1:-1]') ds = datasets.load_dataset('mnist', split=datasets.ReadInstruction.from_spec( 'test[:33%]+train[1:-1]')) ds = datasets.load_dataset('mnist', split=( datasets.ReadInstruction('test', to=33, unit='%') + datasets.ReadInstruction('train', from_=1, to=-1, unit='abs'))) # The following lines are equivalent: ds = datasets.load_dataset('mnist', split='test[:33%](pct1_dropremainder)') ds = datasets.load_dataset('mnist', split=datasets.ReadInstruction.from_spec( 'test[:33%](pct1_dropremainder)')) ds = datasets.load_dataset('mnist', split=datasets.ReadInstruction( 'test', from_=0, to=33, unit='%', rounding="pct1_dropremainder")) # 10-fold validation: tests = datasets.load_dataset( 'mnist', [datasets.ReadInstruction('train', from_=k, to=k+10, unit='%') for k in range(0, 100, 10)]) trains = datasets.load_dataset( 'mnist', [datasets.ReadInstruction('train', to=k, unit='%') + datasets.ReadInstruction('train', from_=k+10, unit='%') for k in range(0, 100, 10)]) c||_yrb_relative_instructions)rXrelative_instructionss r _initzReadInstruction._inits &;#rcJ|j|}|j||S)zCReturns ReadInstruction obj initialized with relative_instructions.)__new__r)clsrresults r ,_read_instruction_from_relative_instructionsz.VsFc,S1Fsr N)r;_ADDITION_SEP_REr5r>rsum)rrsubsr-s r r?zReadInstruction.from_spec9sZ04y%%d+EdVLM M.tAw7 FT!"XF TTrcg}|jD]}|j}|j |j|j}|j}|j}|j }|dk(r|nd}|t ||znd}|t ||znd}d|d|d}|dk(r | |dk7rd|dnd} ||| zz }|j|d j|S) Nrr[r]r()+) rrArBrCrrr;rDrm) rXrel_instr_specsrrel_instr_specrBrCrr slice_str rounding_strs r to_speczReadInstruction.to_specXs44 3I&00N*ill.F!\\ ~~$--#s{t-2->E T)B')~SWt^2wat1- '+s{x7KPX\ePeazOkm)l"::  " "> 2 3 xx((rcDt|ts d}t||j}|j}|djdk7rF|djdk7r4|jdj |dj k7r t d|j||zS)zEReturns a new ReadInstruction obj, result of appending other to self.zAReadInstruction can only be added to another ReadInstruction obj.rrzPIt is forbidden to sum ReadInstruction instances with different rounding values.)r:r.r<rrrr>r)rXotherr}self_ris other_riss r __add__zReadInstruction.__add__ls%1UCC. ..00 QK   %! !!U*++A.779Q<;P;PPop p@@IAUVVrc"|jSrb)rrs r __str__zReadInstruction.__str__{s||~rc"d|jdS)NzReadInstruction(rrrs r __repr__zReadInstruction.__repr__~s!$"="=!>a@@rcT|jDcgc]}t||c}Scc}w)aZTranslate instruction into a list of absolute instructions. Those absolute instructions are then to be added together. Args: name2len (`dict`): Associating split names to number of examples. Returns: list of _AbsoluteInstruction instances (corresponds to the + in spec). )rr)rXrGrs r r@zReadInstruction.to_absolutes(IMHcHcd9!)X6ddds%)NNNN)rrrrr classmethodrrYr?rrrrr@rrr r.r.sY$L<R:UU<)( WA er)NN)?rrjrrlrer dataclassesrpathlibrtypingrrrrpyarrowrppyarrow.parquetrrdownload.download_configr namingr r rr rrrutilsrutils.file_utilsrrFrrrr get_loggerrloggerrcompileXrrConnectionErrorrr"r%r;rQrSrrrrrrrrr.rrr rs  !774:II)!(   H %Qrzz aO   DD 2::k* _    $ " " "$&*!% E Ek"Es--.Ec] E # E  EPR:R:j,-*,-^J> $    $ooo0 772Ddeder