
    *#h                     j    d dl Z d dlmZ d dlmZ ddlmZ dZ ee      Z	 G d de      Z
dd	Zdd
Zy)    N)FileSystems)Pipeline   )
get_loggeri    c                       e Zd ZdZd Zy)BeamPipelinez<Wrapper over `apache_beam.pipeline.Pipeline` for conveniencec                 \    | j                   j                         j                  d      }|dv S )Nrunner)NDirectRunnerPortableRunner)_optionsget_all_optionsget)selfr
   s     V/var/www/html/sandstorm/venv/lib/python3.12/site-packages/datasets/utils/beam_utils.pyis_localzBeamPipeline.is_local   s+    ..044X>AAA    N)__name__
__module____qualname____doc__r    r   r   r   r      s    FBr   r   c                    t         }|j                  |      r6|rt        j                  d| d       nt        j                  d| d       y|j	                  |      5 }t        | d      5 }|j                  t              }|r)|j                  |       |j                  t              }|r)ddd       ddd       y# 1 sw Y   xY w# 1 sw Y   yxY w)zJUse the Beam Filesystems to upload to a remote directory on gcs/s3/hdfs...zRemote path already exist: &. Overwriting it as force_upload=True.$. Skipping it as force_upload=False.Nrb)	r   existsloggerinfocreateopenread
CHUNK_SIZEwrite)local_file_pathremote_file_pathforce_uploadfsremote_file
local_filechunks          r   upload_local_to_remoter,      s    	B	yy!"KK56F5GGmnoKK56F5GGklm	#	$ 4/4( 	4JOOJ/E!!%("
3 	44 4	4 	44 4s%   C
,A B>-C
>C	C

Cc                    t         }t        j                  j                  |      r6|rt        j                  d|  d       nt        j                  d|  d       y|j                  |       5 }t        |d      5 }|j                  t              }|r)|j                  |       |j                  t              }|r)ddd       ddd       y# 1 sw Y   xY w# 1 sw Y   yxY w)zNUse the Beam Filesystems to download from a remote directory on gcs/s3/hdfs...zLocal path already exist: r   r   Nwb)
r   ospathr   r   r   r!   r"   r#   r$   )r&   r%   force_downloadr(   r)   r*   r+   s          r   download_remote_to_localr2   &   s    	B	ww~~o&KK45E4FFlmnKK45E4FFjkl	!	" 5k/4( 	5J$$Z0E  '#((4 	55 5	5 	55 5s%   -C:A C;CC	CC!)F)r/   apache_beam.io.filesystemsr   apache_beam.pipeliner   loggingr   r#   r   r   r   r,   r2   r   r   r   <module>r6      s:    	 2 )  
	H	B8 B4"5r   