*#h~ddlZddlZddlZddlmZmZddlmZddlm Z dZ dZ gdZ gdZ d efd ZGd d eZy) N)ArgumentParser Namespace)BaseDatasetsCLICommand) get_loggerz><<<<<<< This should probably be modified because it mentions: z======= >>>>>>> )TextEncoderConfigByteTextEncoderSubwordTextEncoderencoder_configmaybe_build_from_corpus manual_dir) )z tfds\.coredatasets)ztf\.io\.gfile\.GFileopen)z tf\.([\w\d]+)zdatasets.Value('\1'))ztfds\.features\.Text\(\)zdatasets.Value('string'))ztfds\.features\.Text\(zdatasets.Value('string'),)z+features\s*=\s*tfds.features.FeaturesDict\(zfeatures=datasets.Features()ztfds\.features\.FeaturesDict\(zdict()zThe TensorFlow Datasets AuthorszDThe TensorFlow Datasets Authors and the HuggingFace Datasets Authors)ztfds\.z datasets.)zdl_manager\.manual_dirzself.config.data_dir)zself\.builder_configz self.configargscBt|j|jS)zz Factory function used to convert a model TF 1.0 checkpoint in a PyTorch checkpoint. Returns: ConvertCommand )ConvertCommand tfds_pathdatasets_directory)rs V/var/www/html/sandstorm/venv/lib/python3.12/site-packages/datasets/commands/convert.pyconvert_command_factoryr*s $..$*A*A BBc8eZdZedefdZdedefdZdZy)rparserc|jdd}|jdtdd|jdtdd |jt y ) z Register this command to argparse so it's available for the datasets-cli Args: parser: Root parser to register command-specific arguments convertzHConvert a TensorFlow Datasets dataset to a HuggingFace Datasets dataset.)helpz --tfds_pathTzQPath to a TensorFlow Datasets folder to convert or a single tfds file to convert.)typerequiredrz--datasets_directoryz(Path to the HuggingFace Datasets folder.)funcN) add_parser add_argumentstr set_defaultsr)r train_parsers rregister_subcommandz"ConvertCommand.register_subcommand4ss(( [)  !! d " !! "tBl "  !!'>!?rrrc@td|_||_||_y)Nzdatasets-cli/converting)r_logger _tfds_path_datasets_directory)selfrrrs r__init__zConvertCommand.__init__Ks!";< ##5 rc tjj|jr*tjj |j}n^tjj |jr*tjj |j}n tdtjj |j}|jjd|d|g}g}i}tjj|jrtj|}n*tjj|jg}|D]}|jjd|tjj||}tjj||} tjj |r d|vsd|vsd|vr|jjdt|d 5} | j} dddg} d } d }g} D]}|d vr d vrdvrdvrdndvrd#dvrdndvrj!ddnt#fdt$Drid}t't)fdt$}| j+t,t/|zdz| j+| j+t0t2D]\}}t5j6||dvr`t5j8d}|j;d|j=dj?dDd |j=dzd!vsd"vsdvrtd#jAd$vsd%vrd} | j+| sd&|vr|j!dd}tjj||}tjj||} tjB|d'|jjd(||jE|Dcic]}||c}n|j+| |r|j+| t| d)d 5} | jG| ddd|jjd*| |D]n} tjj|}||j!dd}|jjd+|d|tIjJ||p|r'|D]!}|jjQd.|d/#yy#1swY]xYwcc}w#1swYxYw#tL$r"|jjOd,|d-YwxYw)0NzA--tfds_path is neither a directory nor a file. Please check path.zConverting datasets from z to zLooking at file r*_testz.pyz Skipping filezutf-8)encodingFz!import tensorflow.compat.v2 as tfz @tfds.corez builder=selfz-import tensorflow_datasets.public_api as tfdszimport datasets zimport tensorflowzfrom absl import loggingzfrom datasets import logging getLoggerrc3&K|]}|v ywN).0 expressionout_lines r z%ConvertCommand.run..sOJx/OsTc |vSr1r2)er5s rz$ConvertCommand.run..s a8mr tensorflow_datasetsz/from\stensorflow_datasets.*import\s([^\.\r\n]+)c3<K|]}|jywr1)strip)r3imps rr6z%ConvertCommand.run..s'Y 'Ys,zfrom . import ztf.ztfds.zError converting GeneratorBasedBuilderBeamBasedBuilderwmt)exist_okzAdding directory wz Converted in zMoving z#Cannot find destination folder for z. Please copy manually.z!You need to manually update file z4 to remove configurations using 'TextEncoderConfig'.))ospathisdirr'abspathisfiledirname ValueErrorr(r&infolistdirbasenamejoinr readlinesreplaceany TO_HIGHLIGHTlistfilterappendHIGHLIGHT_MESSAGE_PREr!HIGHLIGHT_MESSAGE_POST TO_CONVERTresubmatchextendgroupsplitr=makedirsupdate writelinesshutilcopyKeyErrorerrorwarning)r) abs_tfds_pathabs_datasets_path utils_fileswith_manual_updateimports_to_builder_map file_namesf_name input_file output_fileflines out_lines is_builderneeds_manual_update tfds_importsline to_removepattern replacementr]dir_name output_dirr> utils_file dest_folder file_pathr5s @rrunzConvertCommand.runQsH 77== )GGOODOO>*-v1ETZIZ^ckq^q !!/2j73 &q  &IJ"' L+ +7(B!X-#x/DP2H(H4!H/8;?H H,'// \JHO,OO*.' $V,C\%R SI$$%:S^%Kd%RS$$X.$$%;<0:J,#%66';#IJ)H4HH%WYabE '''Yu{{1~?S?STW?X'YY/%++a.@HH$8(;?TX`?`$'89I8J%KLL*h6:LPX:X!%J  *W+ +ZUf_!>>%4WW\\*;XF  ggll:v>  J6 !!$5j\"BC&--,.W3sJ.WX"";/""))+6k39 (Q Y' ( LL   k]; <]N =`& nJ n))*54V^^E25NO  !!GK=ZL"IJ J 4  n /   $$7 {Bvw  a & &v/X ( ( n ""%H Tk#lm ns1$V4 V 8V"9A+V.V "V+ .(WWN) __name__ __module__ __qualname__ staticmethodrr$r!r*rr2rrrr3s7@N@@,6#636 rrr)rFr[rdargparserrdatasets.commandsrdatasets.utils.loggingrrXrYrTrZrrr2rrrsS .4-]  "C)CP+Pr