
    *#h=                         d dl Z d dlmZmZmZ ddlmZmZmZ ddl	m
Z
 ddlmZ ddlmZ dd	lmZ erd dlZd dlZ G d
 de      Z G d d      Zy)    N)TYPE_CHECKINGOptionalUnion   )DatasetFeaturesconfig)query_table)Sql)logging   )AbstractDatasetInputStreamc                   `     e Zd Z	 	 	 ddeedf   deedddf   dee   ded	ef
 fd
Zd Z	 xZ
S )SqlDatasetReadersqlzsqlalchemy.sql.Selectableconsqlalchemy.engine.Connectionsqlalchemy.engine.Enginesqlite3.Connectionfeatures	cache_dirkeep_in_memoryc                 V    t        |   d|||d| t        d||||d|| _        y )N)r   r   r   )r   r   r   r    )super__init__r   builder)selfr   r   r   r   r   kwargs	__class__s          L/var/www/html/sandstorm/venv/lib/python3.12/site-packages/datasets/io/sql.pyr   zSqlDatasetReader.__init__   sG     	i(iP^ibhi 
	

 
    c                     d }d }d }d }| j                   j                  ||||       | j                   j                  d|| j                        }|S )N)download_configdownload_modeverification_mode	base_pathtrain)splitr&   	in_memory)r   download_and_prepare
as_datasetr   )r   r$   r%   r&   r'   datasets         r!   readzSqlDatasetReader.read$   sj     	))+'/ 	* 	
 ,,))->$J]J] * 
 r"   )NNF)__name__
__module____qualname__r   strr   r   boolr   r.   __classcell__)r    s   @r!   r   r      si    
 (,$
3334
 368RThhi
 8$	

 
 
$r"   r   c                   f    e Zd Z	 	 ddededeedddf   dee   d	ee   f
d
ZdefdZ	d Z
defdZy)SqlDatasetWriterNr-   namer   r   r   r   
batch_sizenum_procc                     ||dk  rt        d| d      || _        || _        || _        |r|nt        j
                  | _        || _        || _        y )Nr   z	num_proc z must be an integer > 0.)	
ValueErrorr-   r7   r   r	   DEFAULT_MAX_BATCH_SIZEr8   r9   to_sql_kwargs)r   r-   r7   r   r8   r9   r=   s          r!   r   zSqlDatasetWriter.__init__:   sZ     HMy
2JKLL	(2*8U8U *r"   returnc                     | j                   j                  dd       }| j                   j                  dd       }| j                   j                  dd      } | j                  dd|i| j                   }|S )Nr   r   indexFr   )r=   pop_write)r   _r@   writtens       r!   writezSqlDatasetWriter.writeM   sm    ""5$/""5$/""&&w6$++@E@T-?-?@r"   c                 X   |\  }}}|dkD  ri |ddin|}t        | j                  j                  t        ||| j                  z         | j                  j
                        }|j                         } |j                  | j                  | j                  fd|i|}|xs t        |      S )Nr   	if_existsappend)tablekeyindicesr@   )r
   r-   dataslicer8   _indices	to_pandasto_sqlr7   r   len)r   argsoffsetr@   r=   batchdfnum_rowss           r!   
_batch_sqlzSqlDatasetWriter._batch_sqlU   s    '+$}DJQJ@=@+x@Ta,,##fft67LL))

 __299TYYOOO"3r7"r"   c                    d}| j                   | j                   dk(  rrt        j                  t        dt	        | j
                        | j                        dt        j                          d      D ]  }|| j                  |||f      z  } |S t	        | j
                        | j                  }}t        j                  | j                         5 }t        j                  |j                  | j                  t        d||      D cg c]  }|||f	 c}      ||z  r||z  dz   n||z  dt        j                          d      D ]  }||z  }	 	 ddd       |S c c}w # 1 sw Y   |S xY w)zWrites the pyarrow table as SQL to a database.

        Caller is responsible for opening and closing the SQL connection.
        r   Nr   bazCreating SQL from Arrow format)unitdisabledesc)totalrZ   r[   r\   )r9   r   tqdmrangerQ   r-   r8   is_progress_bar_enabledrW   multiprocessingPoolimap)r   r@   r=   rD   rS   rV   r8   pools           r!   rB   zSqlDatasetWriter._writea   sd   
 == DMMQ$6!,,aT\\*DOO<#;;==5	 K 4??FE=+IJJK,  $'t||#4doojH %%dmm4 ( 'IIFKAxYcFdeF&%7e ;CZ:O8z1Q6U]akUk ' ? ? AA9	! 
(H x'G
((  f	( s   5EE=EEE()NN)r/   r0   r1   r   r2   r   r   intr   rE   rW   rB   r   r"   r!   r6   r6   9   su     %)"&++ + 368RThhi	+
 SM+ 3-+&s 
# r"   r6   )ra   typingr   r   r    r   r   r	   
formattingr
   packaged_modules.sql.sqlr   utilsr   abcr   sqlite3
sqlalchemyr   r6   r   r"   r!   <module>rn      sE     1 1 ( ( $ *  + %1 %PF Fr"   