
    *#h                         d dl Z d dlmZ d dlmZ d dlZd dlZd dlm	Z	 ej                  j                  j                  e      Ze G d dej                               Z G d dej"                        Zy)	    N)	dataclass)Optional)
table_castc                   :    e Zd ZU dZdZeej                     ed<   y)ArrowConfigzBuilderConfig for Arrow.Nfeatures)	__name__
__module____qualname____doc__r   r   datasetsFeatures__annotations__     b/var/www/html/sandstorm/venv/lib/python3.12/site-packages/datasets/packaged_modules/arrow/arrow.pyr   r      s    ",0Hhx(()0r   r   c                   Z    e Zd ZeZd Zd Zdej                  dej                  fdZ	d Z
y)Arrowc                 V    t        j                  | j                  j                        S )N)r   )r   DatasetInfoconfigr   )selfs    r   _infozArrow._info   s    ##T[[-A-ABBr   c                     | j                   j                  s"t        d| j                   j                         |j                  | j                   j                        }t	        |t
        t        t        f      re|}t	        |t
              r|g}|D cg c]  }|j                  |       }}t        j                  t        j                  j                  d|i      gS g }|j                         D ]  \  }}t	        |t
              r|g}|D cg c]  }|j                  |       }}| j                  j                  t         j"                  j%                  |      D ]m  }t'        |d      5 }t        j(                  j+                  t,        j.                  j1                  |      j2                        | j                  _        ddd        n |j5                  t        j                  |d|i              |S c c}w c c}w # 1 sw Y   CxY w)z-We handle string, list and dicts in datafilesz=At least one data file must be specified, but got data_files=files)name
gen_kwargsNrb)r   
data_files
ValueErrordownload_and_extract
isinstancestrlisttuple
iter_filesr   SplitGeneratorSplitTRAINitemsinfor   	itertoolschainfrom_iterableopenr   from_arrow_schemapaipcopen_streamschemaappend)r   
dl_managerr   r   filesplits
split_namefs           r   _split_generatorszArrow._split_generators   s   {{%%\]a]h]h]s]s\tuvv44T[[5K5KL
j3e"45E%%=BCTZ**40CEC++1E1ESZ\aRbcdd!+!1!1!3 	aJ%%=BCTZ**40CECyy!!)%OO99%@ DdD) oQ-5->->-P-PQSQWQWQcQcdeQfQmQm-n		*o MM(11zwX]N^_`	a  D Do os   G:G?,AHHpa_tablereturnc                     | j                   j                  *t        || j                   j                  j                        }|S )N)r+   r   r   arrow_schema)r   r<   s     r   _cast_tablezArrow._cast_table6   s5    99) "(DII,>,>,K,KLHr   c              #     K   t        t        j                  j                  |            D ]  \  }}t	        |d      5 }	 t        t
        j                  j                  |            D ]?  \  }}t
        j                  j                  |g      }| d| | j                  |      f A 	 d d d         y # t        $ r-}t        j                  d| dt        |       d|         d }~ww xY w# 1 sw Y   xY ww)Nr   _zFailed to read file 'z' with error z: )	enumerater,   r-   r.   r/   r1   r2   r3   Tablefrom_batchesr@   r    loggererrortype)	r   r   file_idxr7   r:   	batch_idxrecord_batchr<   es	            r   _generate_tableszArrow._generate_tables=   s     '	(E(Ee(LM 	NHddD! 
Q	3<RVV=O=OPQ=R3S T/	<#%88#8#8,#H "*
!I;79I9I(9SSST
 
	 " LL#8mDQRG9TVWXVY!Z[
 
s:   ;C:C.A*B5*C:5	C+>(C&&C++C..C7	3C:N)r	   r
   r   r   BUILDER_CONFIG_CLASSr   r;   r1   rD   r@   rM   r   r   r   r   r      s3    &C6BHH  r   r   )r,   dataclassesr   typingr   pyarrowr1   r   datasets.tabler   utilslogging
get_loggerr	   rF   BuilderConfigr   ArrowBasedBuilderr   r   r   r   <module>rX      sf     !    % 
			*	*8	4 1((( 1 14H&& 4r   