
    *#h
                        U d dl Z d dlZd dlZd dlZd dlmZ d dlZd dlZddlm	Z	 e j                  j                  d      duZerddlmZ e	j                  e	j                   e	j"                  e	j$                  e	j&                  gZee	j*                     ed<   eD ]t  Zej0                  ej2                  v r;ej2                  ej0                     eur  ej4                  dej0                   d	        ej6                  ej0                  ed
       v dedefdZdej<                  defdZ dej<                  dedefdZ!ddZ"y)    N)List   )compressions3fs)S3FileSystemCOMPRESSION_FILESYSTEMSz*A filesystem protocol was already set for z and will be overwritten.T)clobberdataset_pathreturnc                 6    d| v r| j                  d      d   } | S )z
    Preprocesses `dataset_path` and removes remote filesystem (e.g. removing `s3://`).

    Args:
        dataset_path (`str`):
            Path (e.g. `dataset/train`) or remote uri (e.g. `s3://my-bucket/dataset/train`) of the dataset directory.
    z://r   )split)r
   s    Z/var/www/html/sandstorm/venv/lib/python3.12/site-packages/datasets/filesystems/__init__.pyextract_path_from_urir   !   s'     #))%03    fsc                 T    | &t        | j                  x}t              r|fn|}d|vryy)z
    Validates if filesystem has remote protocol.

    Args:
        fs (`fsspec.spec.AbstractFileSystem`):
            An abstract super-class for pythonic file-systems, e.g. `fsspec.filesystem('file')` or [`datasets.filesystems.S3FileSystem`].
    fileTF)
isinstanceprotocolstr)r   p	protocolss      r   is_remote_filesystemr   .   s2     
~&BKK'7q=QD1	"r   srcdstc                     t        |        }|r5t        j                  | j                  |      | j                  |             y| j	                  ||d       y)z2
    Renames the file `src` in `fs` to `dst`.
    T)	recursiveN)r   shutilmove_strip_protocolmv)r   r   r   is_locals       r   renamer#   =   sJ     (++HB&&s+R-?-?-DE
c3$'r   c                  8   t        t        j                  d      rt        j                  j                          ydt        j                  j                  d<   dt        j                  j
                  d<   t        j                         t        j                  _        y)z
    Clear reference to the loop and thread.
    This is necessary otherwise HTTPFileSystem hangs in the ML training loop.
    Only required for fsspec >= 0.9.0
    See https://github.com/fsspec/gcsfs/issues/379
    
reset_lockNr   )	hasattrfsspecasynr%   iothreadloop	threadingLocklock r   r   _reset_fsspec_lockr/   I   s]     v{{L) "&Q"$>>+r   )r   N)#	importlibr   r+   warningstypingr   r'   fsspec.asyn r   util	find_spec	_has_s3fss3filesystemr   Bz2FileSystemGzipFileSystemLz4FileSystemXzFileSystemZstdFileSystemr   BaseCompressedFileFileSystem__annotations__fs_classr   registrywarnregister_implementationr   r   AbstractFileSystemboolr   r#   r/   r.   r   r   <module>rF      sO           NN$$V,D8	* K kFFG  ( NHFOO+@Q@Q0RZb0bB8CTCTBUUnop"F""8#4#4hMN
 
 
V66 4 	(v(( 	(s 	( 	(,r   