
    *#hP                         d dl Z d dlZd dlmZmZmZ d dlZddlmZm	Z	m
Z
mZ ddlmZ ddlmZ ddlmZ ddlmZmZ d	d
lmZ  G d de      Z G d d      Zy)    N)BinaryIOOptionalUnion   )DatasetFeatures
NamedSplitconfig)query_table)Json)logging)NestedDataStructureLikePathLike   )AbstractDatasetReaderc                   t     e Zd Z	 	 	 	 	 	 	 ddee   dee   dee   dede	de	dee   dee
   f fd	Zd
 Z xZS )JsonDatasetReaderpath_or_pathssplitfeatures	cache_dirkeep_in_memory	streamingfieldnum_procc	           
          t        
|   |f||||||d|	 || _        t        |t              r|n| j
                  |i}t        d||||d|	| _        y )N)r   r   r   r   r   r   )r   
data_filesr   r    )super__init__r   
isinstancedictr   r   builder)selfr   r   r   r   r   r   r   r   kwargs	__class__s             M/var/www/html/sandstorm/venv/lib/python3.12/site-packages/datasets/io/json.pyr    zJsonDatasetReader.__init__   s     			
)		
 		
 
)3M4)Htzz[hNi 
$	

 
    c                 6   | j                   r(| j                  j                  | j                        }|S d }d }d }d }| j                  j	                  ||||| j
                         | j                  j                  | j                  || j                        }|S )N)r   )download_configdownload_modeverification_mode	base_pathr   )r   r,   	in_memory)r   r#   as_streaming_datasetr   download_and_preparer   
as_datasetr   )r$   datasetr*   r+   r,   r-   s         r'   readzJsonDatasetReader.read0   s    >>ll77djj7IG& ! #O M $ILL-- /+"3# .  ll--jj4EQUQdQd . G r(   )NNNFFNN)__name__
__module____qualname__r   r   r   r	   r   strboolintr    r3   __classcell__)r&   s   @r'   r   r      s     '+'+$#"&
.x8
 
#
 8$	

 
 
 
 }
 3-
@r(   r   c            
       b    e Zd Z	 	 ddedeeef   dee   dee   fdZ	defdZ
d	 Zd
edefdZy)JsonDatasetWriterNr2   path_or_buf
batch_sizer   c                     ||dk  rt        d| d      || _        || _        |r|nt        j                  | _        || _        d| _        || _        y )Nr   z	num_proc z must be an integer > 0.zutf-8)	
ValueErrorr2   r=   r
   DEFAULT_MAX_BATCH_SIZEr>   r   encodingto_json_kwargs)r$   r2   r=   r>   r   rC   s         r'   r    zJsonDatasetWriter.__init__J   s[     HMy
2JKLL&(2*8U8U ,r(   returnc                    | j                   j                  dd       }| j                   j                  dd      }| j                   j                  d|dk(  rdnd      }d| j                   vr|dv rd| j                   d<   | j                   j                  d	d       }|d
vrt        d| d      t        | j                  t
        t        t        j                  f      rMt        j                  | j                  d|      5 } | j                  d|||d| j                   }d d d        |S |rt        d| d       | j                  d| j                  ||d| j                   }|S # 1 sw Y   S xY w)Nr=   orientrecordslinesTFindex)r   tablecompression)Ninfergzipbz2xzz&`datasets` currently does not support z compressionwb)rK   )file_objrF   rH   zUThe compression parameter is not supported when writing to a buffer, but compression=z1 was passed. Please provide a local path instead.r   )rC   popNotImplementedErrorr!   r=   r7   bytesosr   fsspecopen_write)r$   _rF   rH   rK   bufferwrittens          r'   writezJsonDatasetWriter.write\   s~   ##M48$$((9=##''99LRWX$---&<N2N+0D())--mTBBB%(N{m[g&hiid&&eR[[(ABT--tM jQW%$++ivfEiUYUhUhij  )klwkxGH  "dkko4+;+;FRWo[_[n[noGj s   =!E%%E/c                 T   |\  }}}}t        | j                  j                  t        ||| j                  z         | j                  j
                        } |j                         j                  dd ||d|}|j                  d      s|dz  }|j                  | j                        S )N)rJ   keyindices)r=   rF   rH   
r   )r   r2   dataslicer>   _indices	to_pandasto_jsonendswithencoderB   )r$   argsoffsetrF   rH   rC   batchjson_strs           r'   _batch_jsonzJsonDatasetWriter._batch_jsons   s    04-~,,##fft67LL))

 -5??$,,lfTYl]kl  &Ht}}--r(   rQ   c                    d}| j                   | j                   dk(  rt        j                  t        dt	        | j
                        | j                        dt        j                          d      D ]+  }| j                  ||||f      }||j                  |      z  }- |S t	        | j
                        | j                  }	}t        j                  | j                         5 }
t        j                  |
j                  | j                  t        d||	      D cg c]  }||||f
 c}      ||	z  r||	z  dz   n||	z  dt        j                          d      D ]  }||j                  |      z  } 	 ddd       |S c c}w # 1 sw Y   |S xY w)zWrites the pyarrow table as JSON lines to a binary file handle.

        Caller is responsible for opening and closing the handle.
        r   Nr   bazCreating json from Arrow format)unitdisabledesc)totalro   rp   rq   )r   r   tqdmrangelenr2   r>   is_progress_bar_enabledrl   r\   multiprocessingPoolimap)r$   rQ   rF   rH   rC   r[   ri   rk   num_rowsr>   pools              r'   rX   zJsonDatasetWriter._write   s    == DMMQ$6!,,aT\\*DOO<#;;==6	 4  ++VVUN,ST8>>(334.  $'t||#4doojH %%dmm4 8 'II((OTUVX`blOmnV&&%@n ;CZ:O8z1Q6U]akUk ' ? ? AA:	! 
8H x~~h77G
88  o	8 s   "5F E;$AF ;F  F
)NN)r4   r5   r6   r   r   r   r   r   r9   r    r\   rl   rX   r   r(   r'   r<   r<   I   so    
 %)"&-- 8X-.- SM	-
 3--$s ..%% 
%r(   r<   )rw   rU   typingr   r   r   rV    r   r   r	   r
   
formattingr   packaged_modules.json.jsonr   utilsr   utils.typingr   r   abcr   r   r<   r   r(   r'   <module>r      sC     	 , ,  4 4 $ -  < &7- 7t\ \r(   