
    *#h                         d dl Z d dlZd dlmZ d dlmZmZ ddlmZ ddl	m
Z
mZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZ ddl m!Z! ddl"m#Z# dd	l$m%Z%  e!e&      Z'erdd
l(m)Z) ddee   fdZ*ddZ+y)    N)wraps)TYPE_CHECKINGOptional   )DownloadConfig)	xbasenamexdirname	xet_parsexexistsxgetsizexglob
xgzip_openxisdirxisfilexjoinxlistdirxnumpy_loadxopenxpandas_read_csvxpandas_read_excelxPathxrelpathxsio_loadmatxsplit	xsplitextxwalkxxml_dom_minidom_parse)
get_logger)patch_submodule)get_importsDatasetBuilderdownload_configc                 4   t        j                  |       }t        |d      r]|j                  rQt	        |j                  t
              r6j                  |j                  _        j                  |j                  _        yfd}t        |d |t                    j                          t        |d |t                    j                          t        |d |t                    j                          t        |d |t                    j                          t        |dt              j                          t        |d	t              j                          t        |d
t               j                          t        |dt"              j                          t        |dt$              j                          t        |dt&              j                          t        |d |t(                    j                          t        |d |t*                    j                          t        |d |t,                    j                          t        |d |t.                    j                          t        |dt0              j                          t        |d |t2                    j                          t        |d |t4                    j                          t        |d |t6              dg      j                          t        |d |t8              dg      j                          t        |d |t:              dg      j                          t        |d |t<                    j                          t        |d |t>                    j                          |_        y)aE  Extend the module to support streaming.

    We patch some functions in the module to use `fsspec` to support data streaming:
    - We use `fsspec.open` to open and read remote files. We patch the module function:
      - `open`
    - We use the "::" hop separator to join paths and navigate remote compressed/archive files. We patch the module
      functions:
      - `os.path.join`
      - `pathlib.Path.joinpath` and `pathlib.Path.__truediv__` (called when using the "/" operator)

    The patched functions are replaced with custom functions defined to work with the
    :class:`~download.streaming_download_manager.StreamingDownloadManager`.

    Args:
        module_path: Path to the module to be extended.
        download_config : mainly use use_auth_token or storage_options to support different platforms and auth types.
    _patched_for_streamingNc                 @     t                fd       }d|_        |S )Nc                       | di|S )Nr#    )argskwargsr#   functions     O/var/www/html/sandstorm/venv/lib/python3.12/site-packages/datasets/streaming.pywrapperz?extend_module_for_streaming.<locals>.wrap_auth.<locals>.wrapperH   s    TM?MfMM    	wrap_auth)r   _decorator_name_)r+   r-   r#   s   ` r,   r/   z.extend_module_for_streaming.<locals>.wrap_authG   s*    	x	N 
	N $/ r.   openz
os.listdirzos.walkz	glob.globzos.path.joinzos.path.dirnamezos.path.basenamezos.path.relpathzos.path.splitzos.path.splitextzos.path.existszos.path.isdirzos.path.isfilezos.path.getsizezpathlib.Pathz	gzip.openz
numpy.loadzpandas.read_csv__version__)attrszpandas.read_excelzscipy.io.loadmatzxml.etree.ElementTree.parsezxml.dom.minidom.parse) 	importlibimport_modulehasattrr%   
isinstancer   tokenstorage_optionsr   r   startr   r   r   r   r	   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r
   r   )module_pathr#   moduler/   s    `  r,   extend_module_for_streamingr=   +   s   & $$[1F v/0V5R5Rf33^D2A2G2GF))/<K<[<[F))9 FFIe$45;;=FL)H*=>DDFFIy'78>>@FK5)9:@@BFNE288:F-x8>>@F.	:@@BF-x8>>@FOV4::<F.	:@@BF,i.@AGGIFOYv->?EEGF,i.@AGGIF-y/BCIIKFNE288:FK:)>?EEGFL)K*@AGGIF-y9I/JS`RabhhjF/;M1NWdVefllnF.	,0GP]_eegF99Y;OPVVXF3Y?U5VW]]_$3F!r.   c                    t        | j                  | j                        }t        | j                  |       | j                  j                  d      syt        t        j                  | j                              D ]N  }|d   dk(  s|d   }dj                  | j                  j                  d      dd	 |gz         }t        ||       P dd
lm} t        |       j                  dd D cg c]3  }t!        ||      r%|j                  |j                  k7  r|j                  5 }}|D ]  }t        ||        yc c}w )zExtend the dataset builder module and the modules imported by it to support streaming.

    Args:
        builder (:class:`DatasetBuilder`): Dataset builder instance.
    )r9   r8   )r#   z	datasets.r   internalr   .Nr!   )r   r9   r8   r=   
__module__
startswithr    inspectgetfile	__class__joinsplitbuilderr"   type__mro__
issubclass)	rI   r#   importsinternal_import_nameinternal_module_namer"   clsparent_builder_modulesr<   s	            r,   $extend_dataset_builder_for_streamingrR   l   s/    %W5L5LT[TaTabO 2 2OT((5"7??73D3D#EF 	cGqzZ''.qz$'*xx0B0B0H0H0Mcr0RVjUk0k'l$+,@Rab		c ( =((,c>*s~~AZAZ/Z 	 
 ) M#FOLMs   -8D<)N)rI   r"   ),r4   rD   	functoolsr   typingr   r   download.download_configr   #download.streaming_download_managerr   r	   r
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   utils.loggingr   utils.patchingr   utils.py_utilsr    __name__loggerrI   r"   r=   rR   r(   r.   r,   <module>r\      si       * 4     0 & + ' 
H	 '>4h~>V >4BMr.   