
    *#h:!                         d dl Z d dlmZ d dlmZ d dlmZmZ d dlm	Z	 d dl
Zd dlmZ d dlmZ d dlmZ d d	lmZmZ d d
lmZ d dlmZ d dlmZmZ  ee      Zd Z G d de      Zy)    N)ArgumentParser)Path)copyfilermtree)	Generator)DatasetBuilder)BaseDatasetsCLICommand)DownloadMode)dataset_module_factoryimport_main_class)logger)VerificationMode)ERROR
get_loggerc                     t        | j                  | j                  | j                  | j                  | j
                  | j                  xs | j                  | j                  | j                  | j                  	      S )N)TestCommanddatasetname	cache_dirdata_dirall_configs	save_info
save_infosignore_verificationsforce_redownloadclear_cache)argss    S/var/www/html/sandstorm/venv/lib/python3.12/site-packages/datasets/commands/test.py_test_command_factoryr      s_    		)$//!!
 
    c                   X    e Zd ZdZedefd       Zdededededed	ed
ededefdZ	d Z
y)r   Fparserc                    | j                  dd      }|j                  dt        d d       |j                  dt        d d       |j                  d	t        d d
       |j                  ddd       |j                  ddd       |j                  ddd       |j                  ddd       |j                  ddd       |j                  ddd       |j                  dt        d       |j                  t               y )NtestzTest dataset implementation.)helpz--namezDataset processing name)typedefaultr%   z--cache_dirz.Cache directory where the datasets are stored.z
--data_dirz@Can be used to specify a manual directory to get the files from.z--all_configs
store_truezTest all dataset configurations)actionr%   z--save_infoz6Save the dataset infos in the dataset card (README.md)z--ignore_verificationsz1Run the test without checksums and splits checks.z--force_redownloadzForce dataset redownloadz--clear_cachezBRemove downloaded files and cached datasets after each config testz--save_infoszalias to save_infor   zName of the dataset to download)r&   r%   )func)
add_parseradd_argumentstrset_defaultsr   )r"   test_parsers     r   register_subcommandzTestCommand.register_subcommand%   s6   ''5S'T  THa b  A	 	! 	
 	  S	 	! 	
 	  Lm n  ,5m 	! 	
 	  $D 	! 	

 	  !5lQk l  U 	! 	
 	  K_ `  ;\ ]  &; <r    r   r   r   r   r   r   r   r   r   c
                     || _         || _        || _        || _        || _        || _        || _        || _        |	| _        |	r|st        d       t        d       |rd| _        y y )Na  When --clear_cache is used, specifying a cache directory is mandatory.
The 'download' folder of the cache directory and the dataset builder cache will be deleted after each configuration test.
Please provide a --cache_dir that will be used to test the dataset script.   T)_dataset_name
_cache_dir	_data_dir_all_configs_save_infos_ignore_verifications_force_redownload_clear_cacheprintexit)
selfr   r   r   r   r   r   r   r   r   s
             r   __init__zTestCommand.__init__I   st      
#!'%%9"!1'y]
 G)-D& r    c           
          t               j                  t                j                  " j                  rt        d       t        d        j                   j                  c}t        |      t        j                         j                  r!j                  rt        j                        nd}dt        t        d d f   f fd}t         |             D ]d  \  }}t        d|j                   j"                   d|dz    d| d       t$        j&                  j)                  t$        j&                  j+                  |j-                         t.        j                   j0                              |_        |j5                   j6                  st8        j:                  nt8        j<                   j>                  rt@        jB                  nt@        jD                  d	
       |jG                           jH                  r|jI                           jH                  r5t$        j&                  j+                  j-                         d      }tK        |      j"                  dz   }t$        j&                  j+                  ||      }t$        j&                  jM                  |      r t$        j&                  jO                  |      }	nTt$        j&                  jM                  |      r|}	n2t$        j&                  jQ                  |      r|}	nd }	t        d|        |	:t$        j&                  j+                  |	d      }
tS        ||
       t        d|
         jT                  st$        j&                  jQ                  |jV                        r7tX        j[                  d|jV                          t]        |jV                         t$        j&                  j+                   jV                  t.        j                   j^                        }t$        j&                  jQ                  |      sBtX        j[                  d|        t]        |       g t        d       y )NzABoth parameters `config` and `all_configs` can't be used at once.r2   returnc               3   &  K   j                   rj                  rt        j                        D ]u  \  } }dj                  v r, dj                  j
                  dj                   @ d|j                  j                  j
                  dj                   w y dj                  v r, dj                  j
                  dj                   y  dj                  j
                  dj                   y w)Nconfig_name)r   r   )rC   r   r    )r7   BUILDER_CONFIGS	enumeratebuilder_kwargsr5   r6   r   )iconfigbuilder_clsrC   moduler>   s     r   get_buildersz%TestCommand.run.<locals>.get_buildersr   s      [%@%@!*;+F+F!G IAv$(=(==) &*oo%)^^ %33  * (.&*oo%)^^ %33	  !F$9$99%r$..r\b\q\qrr% $/"&//!% !//	 s   DDzTesting builder 'z' (/)F)download_modeverification_modetry_from_hf_gcsz	README.mdz.pyzDataset card saved at zClearing cache at zTest successful.)0	fl_loggersetLevelr   r4   r7   r<   r=   r3   r   r   module_pathrE   lenr   r   rF   rI   r   ospathexistsjoinget_imported_module_dirdatasetsDATASETDICT_INFOS_FILENAME_record_infosdownload_and_preparer:   r
   REUSE_CACHE_IF_EXISTSFORCE_REDOWNLOADr9   r   	NO_CHECKS
ALL_CHECKS
as_datasetr8   r   isfiledirnameisdirr   r;   r5   r   warningr   DOWNLOADED_DATASETS_DIR)r>   rW   
n_buildersrL   jbuilderdataset_readme_pathr   combined_pathdataset_diruser_dataset_readme_pathdownload_dirrJ   rC   rK   s   `           @@@r   runzTestCommand.runh   s/   U#::!d&7&7UVG MM4::k'-'(:(:;9=9J9J{OjOjS445pq
	id(BC 	 	6 $LN3 3	)JAw%gnn&9&9%:#a!eWAj\QRST$&GGNNW<<>@j@jk%G! ((-- +@@!22-- #3"<"<%00 % )   ##% &(ggll;3V3V3XZe&f#Dz. "T4 877>>$'"$''//$"7KWW^^M2"&KWW]]4("&K"&K23F2GHI */1ww||K/U,02JK23K2LMN   77==!3!34NN%78J8J7K#LM7--.!ww||DOOX__=d=de77==.NN%7~#FG<(g3	)j 	 !r    N)__name__
__module____qualname____test__staticmethodr   r0   r-   boolr?   rq   rD   r    r   r   r   "   s    H!=N != !=F.. . 	.
 . . . #. . .>Z"r    r   ) rV   argparser   pathlibr   shutilr   r   typingr   datasets.configr[   datasets.builderr   datasets.commandsr	   "datasets.download.download_managerr
   datasets.loadr   r   datasets.utils.filelockr   rR   datasets.utils.info_utilsr   datasets.utils.loggingr   r   rr   r   r   rD   r    r   <module>r      sL    	 #  #   + 4 ; C 7 6 4 
H	`"( `"r    