*#h\6ddlZddlZddlZddlmZddlmZmZmZm Z m Z m Z ddl Z ddlZddlZddlmZmZddlmZddlmZmZddlmZddlmZej:j<j?e Z!erddl"Z"eGd d ejFZ$d d d ee%fdZ&d d dee%fdZ'GddeZ(GddejRZ*y)N) dataclass) TYPE_CHECKINGIterableListOptionalTupleUnion) ArrowWriter ParquetWriter)MAX_SHARD_SIZE)is_remote_filesystemrename)_BaseExamplesIterable)convert_file_size_to_intc:eZdZUdZdZeejed<y) SparkConfigzBuilderConfig for Spark.Nfeatures) __name__ __module__ __qualname____doc__rrdatasetsFeatures__annotations__b/var/www/html/sandstorm/venv/lib/python3.12/site-packages/datasets/packaged_modules/spark/spark.pyrrs",0Hhx(()0rrdfpyspark.sql.DataFramenew_partition_orderc|jdjd|d}|ddD]6}|jdjd|}|j|}8|S)N*z part_id = r)selectwhereunion)rr df_combined partition_id partition_dfs r_reorder_dataframe_by_partitionr*"su))C.&&4G4J3K'LMK+AB/6 yy~++j,GH !'' 5 6 rpartition_orderc"ddlfd}|S)Nrc3dKjd jjjj d}t | }d}|j d}d}|D]A}|j}|d}|jd||k7r|}d}|d||f|dz }Cyw) Nr"part_idrT)prefetchPartitions_r#) r$sql functionsspark_partition_idaliasr*toLocalIteratorasDictpop) df_with_partition_idr)row_idrowscurr_partitionrow row_as_dictr.rr+pysparks r generate_fnz0_generate_iterable_examples..generate_fn0s!yygkk.C.C.V.V.X.^.^_h.ij67K_] ++t+D C**,K!),G OOI &(!(IQvh'4 4 aKF sB-B0)r?)rr+r@r?s`` @r_generate_iterable_examplesrA*s" rceZdZ d d dZdZdej jddfdZde de ddfd Z e de fd Z y) SparkExamplesIterableNc||_|xs-t|jjj|_t |j|j|_yN)rrangerddgetNumPartitionsr+rAgenerate_examples_fn)selfrr+s r__init__zSparkExamplesIterable.__init__EsG .W% 8T8T8V2W$?I]I]$^!rc#@K|jEd{y7wrE)rIrJs r__iter__zSparkExamplesIterable.__iter__Ns,,...s  generatorreturnctt|jjj }|j |t |j|SN)r+)listrFrrGrHshufflerC)rJrOr+s rshuffle_data_sourcesz*SparkExamplesIterable.shuffle_data_sourcesQsAuTWW[[%A%A%CDE/*$TWWoNNr worker_id num_workerscT|j||}t|j|SrR)split_shard_indices_by_workerrCr)rJrVrWr+s rshard_data_sourcesz(SparkExamplesIterable.shard_data_sourcesVs%<.create_cache_and_write_probe{sL KK D 1iTZZ\=M=M1MNJ S !< rz spark.masterlocalr#rztWhen using Dataset.from_spark on a multi-node cluster, the driver and all workers should be able to access cache_dir) _cache_dirrlconfget startswith sparkContext parallelizerF mapPartitionscollectrxrzisfile ValueError)rJrproberes @r_validate_cache_dirzSpark._validate_cache_dirusOO   ;;     3 > >w G  ?? ((44U1XqAOOPlmuuw ww~~eAh' C  rcVtj|jjS)N)r)r DatasetInfoconfigrrMs r_infoz Spark._infos##T[[-A-ABBr dl_managerc`tjtjjgS)N)name)rSplitGeneratorSplitTRAIN)rJrs r_split_generatorszSpark._split_generatorss ''X^^-A-ABCCrcddl}d}|jj}|dkr|nd}|jj|j dj |dj |jjjdjdjdj|z }||z}||kDr9t|t||z }|jj ||_yy)Nrc3vK|D]0}tjjd|jgi2yw)N batch_bytes)pa RecordBatch from_pydictnbytes)itbatchs rget_arrow_batch_sizez=Spark._repartition_df_if_needed..get_arrow_batch_sizes7 Rnn00-%,,1PQQ Rs79dr#zbatch_bytes: longr sample_bytes)r?rcountlimit repartition mapInArrowaggr2r3sumr5rrminra) rJmax_shard_sizer?r df_num_rowssample_num_rowsapprox_bytes_per_rowapprox_total_sizenew_num_partitionss r_repartition_df_if_neededzSpark._repartition_df_if_neededs Rggmmo )4);+ GGMM/ * [^ Z,.A B S&&**=9??O P WYq   \   1;> ~ -!$[#6G.6X2Y!Z gg))*<=DG .rfpath file_formatrrPc #2 Kddl |dk(rtnt |jrGtj j |jtj jn |dk(|jj|j |jj  f d}|jj|djdj! j"j$j'dj)d j"j$j'dj)d  j"j$j+dj)d  j"j$j-dj)d j/}|D]>}|j0|j2|j4|j6|j8ff@yw) Nrparquetc 3D Kjj}t|d}|)tjj |gdgdgggdSd} j d|dj d|d }tjj|g}|j||D]}|jk\r|j\}}|jtjj |g|g|gggd|dz }|jj d|dj d|d }tjj|g}|j||jdkDrN|j\}}|jtjj |g|g|gggd k7rtjtj j#D]r} tj j%tj j# tj j'| } t(j+| | tyyw) Nr)task_id num_examples num_bytes)namesSSSSS05dTTTTT)rrzwriter_batch_sizestorage_optionsembed_local_filesr#) TaskContext taskAttemptIdnextrr from_arraysreplaceTable from_batches write_table _num_bytesfinalizeclose _featuresrxlistdirrzdirnamer{basenameshutilmove)rr first_batchshard_idwritertablerrrfiledestrrrrr?r working_fpathr writer_classs r write_arrowz0Spark._prepare_split_single..write_arrowsx)g))+99;Gr4.K"~~11YaS)B2H!!"**7xnFNNw[bcfZgi"3 /"3 FHH));-8E   u % *!-&2C2C~2U.4oo.?+L)LLN..44!\NYK@F5MH)!'!1!1*227xnNVVW^cjknboq*;(7*; F--ug6""5)# *&  1$*0//*;' i nn00Y <B1 %JJrww}'EF,D77<<(>@P@PQU@VWDKKd+,&sJJ z2task_id: long, num_examples: long, num_bytes: longrrtotal_num_examplesrtotal_num_bytes num_shards shard_lengths)r?r r rmrxrzr{rrr_writer_batch_size_fsrrrgroupByrr2r3rr5r collect_listrrrrrr)rJrrrrstatsr=rrr?rrrrs ` ` @@@@@@@r_prepare_split_singlezSpark._prepare_split_singles (3y(@}k TXTeTe T%6%68H8H8OPkp '94;;'' 33((222 ,2 ,j GG  {,` a WY  S %%)).9??@TU %%))+6<<=NO %%++K8>>|L %%22>BHHY  WY  pC++ 6 68K8KS^^]`]n]noo o psH Hsplit_generatorzdatasets.SplitGeneratornum_procc |jt|xst}|j|t |j  }|rt jjntj}d}|jd|j|d|} ||j| d} d} dg} g} |j||D]E\}}|\}}}}|dkDs| |z } | |z } |z | j||f| j|G| |j_| |j_t$j'dddkDr| |j_|j dt*d t*d t*ffd g}d}t-t/| D]3}| |\}}t-|D]}|j|||g|dz }5|j0j2j5|t/|j7fd j9yd}| dd}|j;j=d |dj=d|dj=|dy)Nz-TTTTT-SSSSS-of-NNNNN-.rz Renaming z shards.r#rrglobal_shard_idc tjd|djd|djd|djddy)Nrrrz TTTTT-SSSSSNNNNN)rr)rrrrfs total_shardss r _rename_shardz+Spark._prepare_split.._rename_shard=s^ MM'hs^=EEgRYZ]Q^`MM-OC3HJRRSZ_klo^prrc|SrEr)argsrs rz&Spark._prepare_split..OsS`bfSgrrrrr)rrr rr rrxrzr{ posixpathr _output_dirrappendextend split_inforrloggerdebugrrarFr\rlrrmapr_renamer)rJrrrrkwargsis_local path_joinSUFFIXfnamerrtask_id_and_num_shardsall_shard_lengthsrcontentrrrrrrirrrrrs @@@@r_prepare_splitzSpark._prepare_split s   "1.2RNS &&~6+DHH55$,BGGLL).. (99+Q334VHAk]K$**E2 !# $ : :5+~ ^ 8 GW   1}"l2"9, * &--w .CD!((7 83E""//>"",  yh78 ! 7HO & & 4B   "% DO3567 )&2RpRpRp Rp %T5e#445 6 Rpn#48"& N2NN!sCx1 N 3- N`.2. .rrd)+rxrr| dataclassesrtypingrrrrrr numpyr^pyarrowrrdatasets.arrow_writerr r datasets.configr datasets.filesystemsr rdatasets.iterable_datasetrdatasets.utils.py_utilsrutilslogging get_loggerrrr? BuilderConfigrrar*rArCDatasetBuilderrdrrrr"s  !HH<*<<    * *8 4 1(((1 1 (?VZ[^V_#Y4)1)6~.H # #~.r