
    *#h(                         d Z ddlZddlZddlZddlmZ ddlmZm	Z	m
Z
mZ ddlmZ ddlmZmZ ddlmZ dd	lmZ  ee      Z G d
 d      Zy)z Mock download manager interface.    N)Path)CallableListOptionalUnion)ZipFile   )cached_pathhf_github_url)
get_logger)Versionc                      e Zd ZdZdZdZ	 	 	 	 d dededeeef   de	e   d	e
d
e
de	ee      fdZed        Zed        Zed        Zd Zed        Zed        Zed        Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Z d Z!y)!MockDownloadManager
dummy_datadatasetsFNdataset_nameconfigversion	cache_diruse_local_dummy_dataload_existing_dummy_datadownload_callbacksc                     d| _         || _        || _        || _        || _        |xs g | _        || _        t        |      | _        d | _	        d | _
        y )Nr   )downloaded_sizer   r   r   r   r   r   strversion_name_dummy_file_bucket_url)selfr   r   r   r   r   r   r   s           d/var/www/html/sandstorm/venv/lib/python3.12/site-packages/datasets/download/mock_download_manager.py__init__zMockDownloadManager.__init__&   s`      !("$8!2D2J )A%  L    c                 \    | j                   | j                         | _         | j                   S N)r   download_dummy_datar   s    r    
dummy_filezMockDownloadManager.dummy_fileA   s+    ##779Dr"   c                     | j                   ?t        j                  j                  d| j                   j                  | j
                        S t        j                  j                  d| j
                        S )Ndummy)r   ospathjoinnamer   r&   s    r    dummy_data_folderz%MockDownloadManager.dummy_data_folderG   sN    ;;"77<<)9)94;L;LMMww||GT%6%677r"   c                 V    t         j                  j                  | j                  d      S )Nzdummy_data.zip)r*   r+   r,   r.   r&   s    r    dummy_zip_filez"MockDownloadManager.dummy_zip_fileO   s    ww||D224DEEr"   c                     | j                   du r| j                  n| j                  }t        || j                  dd      }t
        j                  j                  || j                        S )NT)r   extract_compressed_fileforce_extract)	r   local_path_to_dummy_datagithub_path_to_dummy_datar
   r   r*   r+   r,   dummy_file_name)r   path_to_dummy_data_dir
local_paths      r    r%   z'MockDownloadManager.download_dummy_dataS   s^    -1-F-F$-ND))TXTrTr 	 !"dnnVZjn

 ww||J(<(<==r"   c                     t         j                  j                  | j                  | j                  | j
                        S r$   )r*   r+   r,   datasets_scripts_dirr   r0   r&   s    r    r4   z,MockDownloadManager.local_path_to_dummy_data^   s+    ww||D55t7H7H$J]J]^^r"   c                     | j                   Ct        | j                  | j                  j	                  t
        j                  d            | _         | j                   S )N/)r   r   r   r0   replacer*   sepr&   s    r    r5   z-MockDownloadManager.github_path_to_dummy_datab   sJ    #,T->->@S@S@[@[\^\b\bdg@hiDr"   c                    t         j                  j                  | j                        r| j                  S dj	                  | j                  j                  t         j                  d      j                  d      d d       S Nr<   )r*   r+   isdirr'   r,   r=   r>   splitr&   s    r    
manual_dirzMockDownloadManager.manual_dirh   sX     77==)??"xx//<BB3GLMMr"   c                    | j                   r| j                  }n| j                  }t        |t              r| j                  ||      S t        |t        t        f      r| j                  ||      S | j                  ||      S r$   )
r   r'   r6   
isinstancedictcreate_dummy_data_dictlisttuplecreate_dummy_data_listcreate_dummy_data_single)r   data_urlargsr'   s       r    download_and_extractz(MockDownloadManager.download_and_extractq   st    ((J --J h%..z8DD4-0..z8DD00XFFr"   c                 $    | j                  |      S r$   rO   )r   rM   rN   s      r    downloadzMockDownloadManager.download       ((22r"   c                 $    | j                  |      S r$   rQ   )r   rM   custom_downloads      r    download_customz#MockDownloadManager.download_custom   rS   r"   c                     |S r$    )r   r+   rN   kwargss       r    extractzMockDownloadManager.extract   s    r"   c                     i S r$   rX   r&   s    r    get_recorded_sizes_checksumsz0MockDownloadManager.get_recorded_sizes_checksums   s    	r"   c                 X   i }|j                         D ]  \  }}| j                  D ],  }t        |t              r|D ]
  } ||        #|} ||       . t        |t              r^|D cg c]R  }t        j
                  j                  |t        j                  j                  t        |      j                              T }	}nR|}t        j
                  j                  |t        j                  j                  t        |      j                              }	|	||<   	 t        d |j                         D              rbt        t        |j                                     t        |j                               k  r&|j                         D 	ci c]  \  }}	||	|z    }}}	|S c c}w c c}	}w )Nc              3   <   K   | ]  }t        |t                y wr$   )rF   r   ).0is     r    	<genexpr>z=MockDownloadManager.create_dummy_data_dict.<locals>.<genexpr>   s     Daz!S!Ds   )itemsr   rF   rI   r*   r+   r,   urllibparse
quote_plusr   r-   allvalueslenset)
r   path_to_dummy_datarM   dummy_data_dictkeysingle_urlsdownload_callback
single_urlxvalues
             r    rH   z*MockDownloadManager.create_dummy_data_dict   s}    ( 0 	)C%)%<%< 2!k40&1 6
)*56 "-J%j12 +t,juvef&8&,,:Q:QRVWXRYR^R^:_`vv(
%79P9PQUV`QaQfQf9gh#(OC 	)$ D?+A+A+CDDSQ`QgQgQiMjIknq""$o
 J
 CRBWBWBYZJCsECK/ZOZ w [s   )AF!F&c           	         g }t        d |D              }t        d |D              }|r|s|r|d   gt        |      z  }|D ]{  }| j                  D ]
  } ||        t        j                  j                  |t        j                  j                  |j                  d      d               }|j                  |       } |S )Nc              3   Z   K   | ]#  }t        t        j                  d |             % yw)z[0-9]{3,}-of-[0-9]{3,}N)boolrefindallr_   urls     r    ra   z=MockDownloadManager.create_dummy_data_list.<locals>.<genexpr>   s"     `PSD,Dc!JK`s   )+c              3   >   K   | ]  }|j                  d         yw)z3https://ftp.ncbi.nlm.nih.gov/pubmed/baseline/pubmedN)
startswithrw   s     r    ra   z=MockDownloadManager.create_dummy_data_list.<locals>.<genexpr>   s       
VYCNNPQ 
s   r   r<   rA   )rf   rh   r   r*   r+   r,   rc   rd   re   rC   append)	r   rj   rM   dummy_data_listis_tf_recordsis_pubmed_recordsro   rn   rq   s	            r    rK   z*MockDownloadManager.create_dummy_data_list   s    `W_``  
]e 
 
 *; }s8}4H" 	*J%)%<%< .!!*-. GGLL!3V\\5L5LZM]M]^aMbceMf5ghE""5)	* r"   c                 0   | j                   D ]
  } ||        t        j                  j                  |t        j
                  j                  |j                  d      d               }t        j                  j                  |      s| j                  s|S |S r@   )
r   r*   r+   r,   rc   rd   re   rC   existsr   )r   rj   rM   rn   rq   s        r    rL   z,MockDownloadManager.create_dummy_data_single   s{    !%!8!8 	(h'	( /1H1HX[I\]_I`1ab77>>% (E(EL &%r"   c                      y r$   rX   r&   s    r    delete_extracted_filesz*MockDownloadManager.delete_extracted_files       r"   c                      y r$   rX   r&   s    r    manage_extracted_filesz*MockDownloadManager.manage_extracted_files   r   r"   c              #   D   K    fd}t        |      } j                  r ||      n|j                  d      }|D ]a  }|j                         s|j                  j                  d      r0|j                  |      j                         |j                  d      f c y w)Nc              3   V  K   t        j                        j                  }| j                  |      }t	        j
                        5 }|j                         }d d d        D ]5  }|j                  |j                               s#|j                  |       7 y # 1 sw Y   DxY wwr$   )
r   r'   parentrelative_tor   r4   namelistrz   as_posixjoinpath)r+   dummy_parent_pathrelative_pathzip_filemembersmemberr   s         r    _iter_archive_membersz?MockDownloadManager.iter_archive.<locals>._iter_archive_members   s      $T__ 5 < < ,,->?M667 .8"++-.! =$$]%;%;%=>+44V<<=. .s$   AB)	B,B)B)B&"B)*.__rb)	r   r   rglobis_filer-   rz   r   r   open)r   r+   r   
file_paths	file_paths   `    r    iter_archivez MockDownloadManager.iter_archive   s     	= Dz484M4M*40SWS]S]^aSb
# 	SI  "9>>+D+D[+Q++D1::<innT>RRR	Ss   AB B +5B c              #   *  K   t        |t              s|g}|D ]  }t        j                  j	                  |      r5t        j                  j                  |      j                  d      r y | Wt        j                  |      D ]  \  }}}t        j                  j                  |      j                  d      r6|j                          t        |      D ]6  }|j                  d      rt        j                  j                  ||       8   y w)Nr   )rF   rI   r*   r+   isfilebasenamerz   walksortsortedr,   )r   pathsr+   dirpathdirnames	filenamesfilenames          r    
iter_fileszMockDownloadManager.iter_files   s     %&GE 	>Dww~~d#77##D)44[A
46GGDM >0GXyww''0;;KH MMO$*9$5 >#..{;$ ggll7H==>	>	>s   DD)NFTN)"__name__
__module____qualname__r6   r:   is_streamingr   r   r   r   rt   r   r   r!   propertyr'   r.   r0   r%   r4   r5   rD   rO   rR   rV   rZ   r\   rH   rK   rL   r   r   r   r   rX   r"   r    r   r   !   s@   "O%L $(%*)-7;     w|$	 
 C=  #  #'  %T(^4 6    
 8 8 F F	> _ _    
 N NG"338$& S">r"   r   )__doc__r*   ru   urllib.parserc   pathlibr   typingr   r   r   r   zipfiler   utils.file_utilsr
   r   utils.loggingr   utils.versionr   r   loggerr   rX   r"   r    <module>r      s@     ' 	 	   2 2  9 & # 
H	U> U>r"   