*#hBZ2ddlZddlZddlZddlZddlZddlmcmZddl m Z ddl m Z ddl mZddlmZddlmZddlmZddlmZdd lmZdd lmZmZdd lmZdd lmZm Z dd l!m"Z"ee#Z$dZ%dZ&GddeZ'edGddeZ(y)N)ArgumentParser)Path)Optional)config)BaseDatasetsCLICommand)DownloadConfig)DownloadManager)MockDownloadManager)dataset_module_factoryimport_main_class) deprecated) get_loggerset_verbosity_warning) map_nestedzutf-8c t|j|j|j|j|j |j |j|j|j SN) DummyDataCommandpath_to_dataset auto_generaten_lines json_fieldxml_tagmatch_text_fileskeep_uncompressed cache_direncoding)argss Y/var/www/html/sandstorm/venv/lib/python3.12/site-packages/datasets/commands/dummy_data.pydummy_data_command_factoryrsV            ceZdZfdZfdZfdZ ddedeedeedeed eed e f d Z dd ed ededeedeedeed eed efdZ e de fdZdZxZS)!DummyDataGeneratorDownloadManagercNt||i|||_g|_g|_yr)super__init__mock_download_managerdownloaded_dummy_pathsexpected_dummy_paths)selfr&rkwargs __class__s rr%z*DummyDataGeneratorDownloadManager.__init__*s- $)&)%:"&(#$&!r ct||}|jj|}t|jj |dt|j j |d|SNT) map_tuple)r$downloadr&rr'appendr(r) url_or_urlsoutput dummy_outputr+s rr/z*DummyDataGeneratorDownloadManager.download0s]!+.11::;G 4..55vN4,,33\TR r ct|t| |}|jj|}t |j j |dt |jj |d|Sr-)r$extractr/r&rr'r0r(r1s rdownload_and_extractz6DummyDataGeneratorDownloadManager.download_and_extract7sf!1+!>?11::;G 4..55vN4,,33\TR r rrrrrreturnc tjtjj|jj |jj |jjddd}d|j_t|j|jD]~\}}tjj|jj |jj |jj|} ||j|| |||||z }|dk(rtjd|dkDS)N dummy_dataTexist_okrFrrrrrzDummy data generation failed: no dummy files were created. Make sure the data files format is supported by the auto-generation.)osmakedirspathjoinr&datasets_scripts_dir dataset_namedummy_data_folderload_existing_dummy_datazipr'r(_create_dummy_dataloggererror) r)rrrrrtotalsrc_pathrelative_dst_pathdst_paths rauto_generate_dummy_data_folderzADummyDataGeneratorDownloadManager.auto_generate_dummy_data_folder>s. GGLL**??**77**<<    >C"";+.t/J/JDLeLe+f  'H'ww||**??**77**<<! H T,,%!1!- E  A: LLW qyr rLrNc |xst}tjj|rtj d|t |jgd}tfd|D} |Ntjj|} |jdD]} | tj| | z} | rt |jjddt||5} t|d|5} g}t| D]\}}||k\rn|j!|| j#d j%|j'ddddddy d vrt||5} t)j*| }|||}t-|t.rmt1d |j3Ds&t5d t7|j9d|j;Dcic] \}}||d| }}}n|d|}|||i}t |jjddt|d|5} t)j<|| ddddddy tfddDr/|tj?dy |jA|||||y tj?d|dytjjC|rd}tjD|D]\}}}|D]}|jGdrtjj%||}tjj%|t |jI|}||jK|||||||z }|Sy#1swYQxYw#1swYy xYwcc}}w#1swY]xYw#1swYy xYw)Nz#Trying to generate dummy data file )z.txtz.csvz.jsonlz.tsvc3&K|]}|v ywr.0 extensiondst_path_extensionss r zGDummyDataGeneratorDownloadManager._create_dummy_data..ws+vQZI9L,L+v,Tr=parentsrwz.jsonc3<K|]}t|tywr) isinstancelist)rTvs rrWzGDummyDataGeneratorDownloadManager._create_dummy_data..s"S1:a#6"SszCouldn't parse columns z\. Maybe specify which json field must be used to read the data with --json_field .c3&K|]}|v ywrrRrSs rrWzGDummyDataGeneratorDownloadManager._create_dummy_data..sX)Y"55XrX)z.xmlz.txmzEFound xml file but 'xml_tag' is set to None. Please provide --xml_tag)rrzCouldn't generate dummy file 'z9'. Ignore that if this file is not useful for dummy data.r.r>)&DEFAULT_ENCODINGr?rAisfilerIdebugrsuffixesanybasenamesplitfnmatchparentmkdiropen enumerater0writerBstripjsonloadradictallvalues ValueErrorrbkeysitemsdumpwarning_create_xml_dummy_dataisdirwalk startswith relative_torH)r)rLrNrrrrrline_by_line_extensionsis_line_by_line_text_file file_namepatternsrc_filedst_file first_linesiline json_datakrcfirst_json_datarKrA_filesname src_file_path dst_file_pathrVs @rrHz4DummyDataGeneratorDownloadManager._create_dummy_datahs// 77>>( # LL>xjI J"&x."9"9 &H #(++v^u+v(v %+GG,,X6 /55c:UG-G1TT-U)X%%++T4+H(X6E(hh?E8&( '0':5GAt G| %'..t45!rww{';'A'A'CD EE//(X6=( $ ( 3I!-$-j$9 !)T2""S @P@P@R"SS","9$y~~?O:P9QRQ!Q# GPooFW*Xda1ak>*X*X*3HW*=!-+5*GN))//t/Lhh?=8 /8<=#=&XGWXX?NN#jk//(GU\go/p NN0 :ww  WW]]8 $E"$''("3 a! D??3/(* T4(@ (* XtM?R?^?^_g?h(i !8!8))$+'1$+-=%-"9"  L!%UEEE+Y ==#=&s]6N=AN0!N=BO O AOO4O0N: 5N==O OO OO%ct|jjddt||5}d}g}t j |dD]^\}} |dk(r|j | |j} | j|k(s=||kr|dz }H|sK|d j| `t j j||dddy#1swYyxYw) NTrZr\r)startend)eventsrr_)element) rrnrorpET iterparser0poptagremove ElementTreerr) rLrNrrrrn_liner[eventelemrs rr~z8DummyDataGeneratorDownloadManager._create_xml_dummy_datas X##T4#@ (X . L(FG!||H=MN 9 tG#NN4( Axx7*!G+"aKF& ' 2 24 8 9 NN4 ( . .x( . K L L LsAC! C!=C!!C*c<tjj||jj}tjj|d}d}t j d|dtj|d||tj|y)Nr;z"Compressing dummy data folder to 'z.zip'rG) r?rArBr&rErIinfoshutil make_archivermtree)r)rroot_dir base_namebase_dirs r!compress_autogenerated_dummy_datazCDummyDataGeneratorDownloadManager.compress_autogenerated_dummy_datasq77<<1K1K1]1]^GGLL<8  8 5IJIuhA i r )r8NNNN)NNNN)__name__ __module__ __qualname__r%r/r7intrstrboolrOrH staticmethodrfr~r __classcell__)r+s@rr"r")s' $(!%*."& ((SM(# ( #3- ( 3- ( (^%)!%*."&RRR R SM R # R#3-R3-R RhDEP`LL$!r r"zThe `datasets` repository does not host the dataset scripts anymore. Therefore, dummy data is no longer needed to test their loading with CI.ceZdZedefdZdededede ede ede ed ed e ed e efd Z d Z de efdZ dZ y)rparserc|jdd}|jddd|jdtd d |jd tdd |jdtdd |jdtdd |jddd|jdtdd |jdtddt |jdtd|j t y)Nr;zGenerate dummy data.)helpz--auto_generate store_truez!Automatically generate dummy data)actionrz --n_linesr8zBNumber of lines or samples to keep when auto-generating dummy data)typedefaultrz --json_fieldzOptional, json field to read the data from when auto-generating dummy data. In the json data files, this field must point to a list of samples as json objects (ex: the 'data' field for squad-like files)z --xml_tagz[Optional, xml tag name of the samples inside the xml files when auto-generating dummy data.z--match_text_fileszOptional, a comma separated list of file patterns that looks for line-by-line text files other than *.txt or *.csv. Example: --match_text_files *.labelz--keep_uncompressedzWhether to leave the dummy data folders uncompressed when auto-generating dummy data. Useful for debugging for to do manual adjustements before compressing.z --cache_dirzKCache directory to download and cache files when auto-generating dummy dataz --encodingz=Encoding to use when auto-generating dummy data. Defaults to rz/Path to the dataset (example: ./datasets/squad))rr)func) add_parser add_argumentrrrf set_defaultsr)r test_parsers rregister_subcommandz$DummyDataCommand.register_subcommandsL'' ;Q'R   !2t'|rt)d|jdnt)d |jddddy#1swYyxYw) N) config_namehashrTF)rDrversionuse_local_dummy_datarF)dataset_buildermock_dl_managerr)rrz>Automatic dummy data generation succeeded for all configs of ''zTr<)check_duplicate_keysc3&K|] }|dkD yw)rNrR)rT n_exampless rrWz.qsV*zA~VszEDummy data generation done and dummy data test succeeded for config 'z''.rzCDummy data generation done but dummy data test failed since splits z have 0 examples for config 'z&Failed to load dummy data for config 'z''. Original error: z#Dummy data generated in directory 'zg' but kept uncompressed. Please compress this directory into a zip file to use it for dummy data tests.)$rr?rArBrDOWNLOADED_DATASETS_DIRDOWNLOADED_DATASETS_PATHrr"r_split_generatorsrFrOrrrrrrCrDrr@_prepare_split split_info num_examplesrrwrxrIr}OSErrorrJrrrEr)r)rrr dl_cache_dirr dl_managerpath_do_datasetn_examples_per_splitsplit_generatorssplit_generator split_name empty_splitsegenerated_dummy_data_dirs rrz)DummyDataCommand._autogenerate_dummy_dataKs GGLL&*H*H I00  )<@6++?ds  ))*538022MM''MM!33^^ 3 ! ggll?+O+OQ`QmQmnO  8 8 I7;O 4#% KK22T B !#2#D#D_#U '7iO#22?Y^2_APA[A[AhAh()=)=>iV8L8S8S8UVVNN_`o`v`v`{`{_||A 6J$'1MablMmqrMr $L$NN]^j]klIJYJ`J`JeJeIffij!')ww||D4I4I?KlKl'm $ KK56N5OPaa $  <_=S=S=X=X}td|jd|jd|jdYd}~Fd}~wwxYw#t$r&}|j)|jYd}~^d}~wwxYw)6Nz$Creating dummy folder structure for z... Tr<zDataset z with config a seems to already open files in the method `_split_generators(...)`. You might consider to instead only open files in the method `_generate_examples(...)` instead. If this is not possible the dummy data has to be created with less guidance. Make sure you create the file rez/Collecting dummy data file paths to create for zU ==============================DUMMY DATA INSTRUCTIONS============================== zconfig z of r^z(- In order to create the dummy data for z, please go into the folder 'z ' with `cd z` . za- It appears that the function `_generate_examples(...)` expects one or more files in the folder z using the function `glob.glob(...)`. In this case, please refer to the `_generate_examples(...)` method to see under which filename the dummy data files should be created. z, rr_z1- Please create a single dummy data file called 'z' from the folder 'zV'. Make sure that the dummy data file provides at least one example for the split(s) 'z' z0- Please create the following dummy data files 'z' z- For each of the splits 'zU', make sure that one or more of the dummy data files provide at least one example z- If the method `_generate_examples(...)` includes multiple `open()` statements, you might have to create other files in addition to 'zG'. In this case please refer to the `_generate_examples(...)` method z@- After the dummy data file is created, it should be zipped to 'z.zip' with the command `zip z.zip z` z- You can now delete the file 'z' with the command `rm z- To get the file 'z;' back for further changes to the dummy data, simply unzip z.zip with the command `unzip z.zip` zP- After all dummy data files are created, they should be zipped recursively to 'z.zip' with the command `zip -r z/` z!- You can now delete the folder 'z' with the command `rm -r z- To get the folder 'z'- Make sure you have created the file 'z .zip' in 'z' zT=================================================================================== rR)r?rArBrrErIrr@rFileNotFoundErrorrrrfilenamesetdummy_file_namerr0 gen_kwargs_generate_examplesaddlennextiter)r)rrrEgenerator_splitsrfiles_to_create split_namesrrlr generatordummy_data_guidance_print config_stringkeyrecord files_strings rrz/DummyDataCommand._print_dummy_data_instructionss(GGLL)>)>@a@ab :;L:MTRS %5 .@@Q  % )99% 0E KKI%**V W   uzz *))J:::HZHI 0,i)CRCYCYCego4499:$?km*>#$++,,IJ[I\\ghygz{CDD)$-KC*0QRaQbbS.TT)+ 04ii ,  ! #?#q(T$2G-HO-[)/`aefjkzf{a|`}~QRcQddz{FzGGM.NN). #yy9 )/_`l_mnABSATTY.ZZ))/I+Wn.oo) %,rsr@@I*J J %  1 $d?.C)D)W %+kl{k|}YZiYjjopo@@F*G G % %1/1BBYZiYjjpq  % &+>>OPKL[K\\yzIyJJT*U U % %+{}L|MMlm|l}}BCRBSSZ*[ [ % %3O3DD^_n^oouv  % &+@@QRMN]M^^{|K{LLV*W W %!5o5FjQbPccg h ! "_4! '(Q!  4--.mOr!s ""#4<>GC7D. H  l!l!^ Ty)-y)y)r