*#h d Z ddlZddlZddlZddlZddlZddlZddlZddl Z ddl
Z
ddl m Z ddl
mZmZ ddlmZ ddlmZ ddlmZ ddlmZ dd l mZmZ dd
lmZmZm Z m Z m Z m Z m Z m!Z!m"Z"m#Z# dd l$m%Z% ddl&Z&ddl'Z'ddl(Z'ddl)Z*dd l+m,Z, dd
l-m.Z. ddl/m0Z0 ddl1m2Z2 ddl/m3Z3 ddl4Z5ddl4m6Z6m7Z7 e3jr e: Z;ejx Z=d Z>de#e?e@f de?fdZAd ZBde@de@de e@e@f fdZCd ZDe d ZEe dVd e?fd ZFd ZGd! ZHd" ZId# ZJ G d$ d%eK ZL G d& d'eM ZNd( ZO dWd)eegef d*ed+ePd,ePd-ePd.ePd/e e? d0e?d1e eQ d2ePd3e e@ defd4ZR G d5 d6 ZSdXd7ZTd8e@de!e@e e@ f fd9ZUd:e@de!e@e@e@e@f fd;ZV G d< d=e&j ZWd> ZXe d? ZYd@ ZZdA Z[e0j e,j dB k r e[e dC Z^n_e0j j ddD e,j dB j e,j dE j fv r e[e dF Z`e0j e,j dG k r e[e dH Zane0j j ddD e,j dG j k( r e[e dI Zan_e0j j ddD e,j dB j e,j dE j fv r e[e dJ ZadK Zb e"dL ZcdMej dNedOe ec f dPeKde?fdQZedRe#ej j e'j j f de e? fdSZhdRe#ej j e'j j f dNedOe ec f dTe eK de ec fdUZiy# e8$ r
dxZ5xZ7Z6Y w xY w)Yz)Some python utils function and classes.
N)contextmanager)fields is_dataclass)BytesIO)Manager)Empty)
disk_usage)CodeType FunctionType)
AnyCallableDictIterableListOptionalSetTupleTypeVarUnion)urlparse)version)tqdm )config) parallel_map )logging)FinalLiteralc ~ | syg d}t | } |D ] \ }}| |z }|dk\ s|dd| c S t | dS )a6 Returns a human readable size string.
If size_in_bytes is None, then returns "Unknown size".
For example `size_str(1.5 * datasets.units.GiB) == "1.50 GiB"`.
Args:
size_in_bytes: `int` or `None`, the size, in bytes, that we want to
format as a human-readable size string.
z Unknown size))PiB )TiB )GiB @)MiB )KiB g ?z.2f z bytes)floatint)
size_in_bytes
_NAME_LISTname
size_bytesvalues T/var/www/html/sandstorm/venv/lib/python3.12/site-packages/datasets/utils/py_utils.pysize_strr4 E sf aJ-(M & )j
* C<C[$( () -
! (( sizereturnc t | t r| S | j j d rt | dd dz S | j j d rt | dd dz S | j j d rt | dd dz S | j j d rt | dd d
z S | j j d rt | dd d z S | j j d
r)t | dd dz }| j d r|dz S |S | j j d r)t | dd dz }| j d r|dz S |S | j j d r)t | dd dz }| j d r|dz S |S | j j d r)t | dd dz }| j d r|dz S |S | j j d r)t | dd dz }| j d r|dz S |S t d| d ) a)
Converts a size expressed as a string with digits an unit (like `"50MB"`) to an integer (in bytes).
Args:
size (`int` or `str`): The size to convert. Will be directly returned if an `int`.
Example:
```py
>>> convert_file_size_to_int("1MiB")
1048576
```
PIBNr" TIBr$ GIBr& MIBr( KIBr* PBl I5 b TBl J)GBi ʚ;MBi@B KBi z`size=zM` is not in a valid format. Use an integer followed by the unit, e.g., '5GB'.)
isinstancer- upperendswith
ValueError)r6 int_sizes r3 convert_file_size_to_intrL ] s? $ zz| U#49~'' zz| U#49~'' zz| U#49~'' zz| U#49~'' zz| U#49~'' zz| T"tCRy>V, $
c 2x1}@@ zz| T"tCRy>V, $
c 2x1}@@ zz| T"tCRy>U+ $
c 2x1}@@ zz| T"tCRy>U+ $
c 2x1}@@ zz| T"tCRy>U+ $
c 2x1}@@
vdV#pq
rrr5 c | j dd j dd j dd j dd j d d
j d d j d
d j dd j dd j dd j d
j dd S )N\z\\.z\.*z.*+z\+z///(z\()z\)|z\|^z\^$z\$?)replacerstrip)patterns r3 glob_pattern_to_regexr\ s e$ e d e s e e e e e c
r5 stringr[ c t j dd| }t j || }|t d| d| t |j }t j d| }t t || }|S )a Un-format a string using a python f-string pattern.
From https://stackoverflow.com/a/36838374
Example::
>>> p = 'hello, my name is {name} and I am a {age} year old {what}'
>>> s = p.format(name='cody', age=18, what='quarterback')
>>> s
'hello, my name is cody and I am a 18 year old quarterback'
>>> string_to_dict(s, p)
{'age': '18', 'name': 'cody', 'what': 'quarterback'}
Args:
string (str): input string
pattern (str): pattern formatted like a python f-string
Returns:
Dict[str, str]: dictionary of variable -> value, retrieved from the input using the pattern
Raises:
ValueError: if the string doesn't match the pattern
z{(.+?)}z (?P<_\1>.+)zString z doesn't match the pattern ) resubsearchrJ listgroupsfindalldictzip)r] r[ regexresultvalueskeys_dicts r3 string_to_dictrl sz .
FF:~w 7E
YYuf
%F
~76(*EgYOPP
&--/
"F
::j' *D T6" #E Lr5 c t d fdt | t s | st | d | S )zbConvert an object to its dictionary representation recursively.
c > t | xr t | t S N)r rG typeobjs r3 _is_dataclass_instancez&asdict.._is_dataclass_instance s C >C)>%>>r5 c | rwi }t | D ]e } t | |j }|j r,||j k7 s |j
j
dd sW|||j < g |S t | t r.t | d r" t | | D cg c]
} | c} S t | t t f r t | fd| D S t | t r0| j D ci c] \ }} | | c}}S t j | S c c}w c c}}w )N$include_in_asdict_even_if_is_defaultF_fieldsc 3 . K | ] } | y wro ).0v
_asdict_inners r3 z0asdict.._asdict_inner.. s ;!]1 - ;s )r getattrr0 initdefaultmetadatagetrG tuplehasattrrp rb re itemscopydeepcopy)rr rh fr2 rz kr{ rs s r3 r{ z asdict.._asdict_inner s !# &FC[
+%gc166&:;vv!))!3qzz~~Flns7t%*F166N
+ M
U
#Y(? 49 =A}Q / => >
dE]
+ 49 ;s ;; ;
T
"CF99;O41aM!$mA&66O O==% % > Ps ,EE z is not a dict or a dataclass)rG re TypeError)rr r{ rs s @@r3 asdictr s? ?&* c4 )?)D3% <=>> r5 c # K t | |d }t | || d t | || y# t | || w xY ww)z%Temporarily assign obj.attr to value.N)r} setattr)rr attrr2 originals r3 temporary_assignmentr sB sD$'H Cu %
T8$T8$s A0 A?Aseedc # K t j j }t j j | |rt j
rddl}|j j }|j j | |j j r5|j j }|j j | |rt j rddl}ddlm} |j j# } |j j$ j' | }
|j j) |
|j+ s t- d |j! } | j. } t1 | d }
|
r | j2 }| j5 | d t j j7 | |r`t j
rPj j9 |j j r|j j; |rJt j r9j j) _
r| _ yt= | d yyy# t j j7 | |r`t j
rPj j9 |j j r|j j; |rJt j r9j j) _
r| _ w t= | d w w w xY ww)zUTemporarily set the random seed. This works for python numpy, pytorch and tensorflow.r N)contextzBSetting random seed for TensorFlow is only available in eager mode_rng) nprandom get_stater r TORCH_AVAILABLEtorch
get_rng_state manual_seedcuda is_availableget_rng_state_allmanual_seed_all TF_AVAILABLE
tensorflowtensorflow.pythonr get_global_generator Generator from_seedset_global_generatorexecuting_eagerlyrJ _seedr r _set_global_seed set_state
set_rng_stateset_rng_state_alldelattr)r set_pytorchset_tensorflownp_stater torch_statetorch_cuda_statestf tfpycontexttf_statetemp_gen
tf_contexttf_seedtf_rng_initializedtf_rngs r3 temp_seedr sg yy""$HIINN4v--ll002
& :: " " $ %
< < > JJ & &t ,&-- <9911399&&006
&&x0##% ab b ((*
"" $Z 8 __F##D),
H% 611 LL & &{ 3zz&&(
,,->? f 1 1 II * *8 4 &J !"(
F+ 2> H% 611 LL & &{ 3zz&&(
,,->? f 1 1 II * *8 4 &J !"(
F+ 2>s F0M3J 7CMCMMc # b K t }| D ] }||vs|j | | yw)z=Iterate over iterable and return only unique values in order.N)setadd)ri seenr2 s r3
unique_valuesr % s4 5D HHUOKs //c fd}|S )z4If the value is None, return None, else call `func`.c | | S d S ro rx )r2 funcs r3 wrapperz'no_op_if_value_is_null..wrapper1 s # /tE{9T9r5 rx )r r s ` r3 no_op_if_value_is_nullr . s : Nr5 c : t | D ]
\ }}| ||fc S y)zwReturn the index and the value of the first non-null value in the iterable. If all values are None, return -1 as index.)N) enumerate)iterableir2 s r3 first_non_null_valuer 7 s. h' 5 e8O r5 c ' ~ K t t j | D ] t fd| D f yw)z9Iterate over items of dictionaries grouped by their keys.c 3 ( K | ] }| y wro rx )ry dkeys r3 r| zzip_dict..C s /A3/s N)r itertoolschainr )dictsr s @r3 zip_dictr ? s: Y__e 45 05/////0s :=c 6 e Zd ZdZ fdZ fdZ fdZ xZS )NonMutableDictzDict where keys can only be added but not modified.
Will raise an error if the user try to overwrite one key. The error message
can be customized during construction. It will be formatted using {key} for
the overwritten key.
c l |j dd | _ |r t d t | |i | y )N error_msgz$Try to overwrite existing key: {key}z1NonMutableDict cannot be initialized with kwargs.)pop
_error_msgrJ super__init__)selfargskwargs __class__s r3 r zNonMutableDict.__init__N s< ** 2
PQ Q
$)&)r5 c v || v r%t | j j | t | || S )Nr )rJ r formatr __setitem__)r r r2 r s r3 r zNonMutableDict.__setitem__W s9 $; T__ 3 3 3 <= =w"3..r5 c t fd|D r:t j j t t | z t
| S )Nc 3 & K | ] }|v
y wro rx )ry r r s r3 r| z(NonMutableDict.update..] s (QqDy(s r )anyrJ r r r r update)r otherr s ` r3 r zNonMutableDict.update\ sJ (%( ( T__ 3 3D CJ8N 3 OP Pw~e$$r5 )__name__
__module__ __qualname____doc__r r r
__classcell__)r s @r3 r r F s */
% %r5 r c e Zd ZdZddZy)
classpropertyz5Descriptor to be used as decorator for @classmethods.Nc D | j j d | S ro )fget__get__)r rr objtypes r3 r zclassproperty.__get__e s /tyy w/11r5 ro )r r r r r rx r5 r3 r r b s
?2r5 r c | \ }}}}}}t |t st || s || S |9t j t j k rt j
|0|s.t
d t j D rt ddd t |t r|j n|}|||dz nddz t | z n|}t j |||d| 5 } t |t r.| D
ci c] \ }
} |
t || |dddf c} }
cddd S | D cg c] } t || |dddf } } t |t r | cddd S t |t rt | cddd S t j | cddd S c c} }
w c c} w # 1 sw Y yxY w)
zEApply a function recursively to each element of a nested data struct.Nc 3 8 K | ] }d |j v yw)notebookN)r )ry tqdm_clss r3 r| z%_single_map_nested..v s 4rYaZ8CTCT5T4rs r+ T)endflush#rr )disablepositionunitdesc)rG re r
get_verbosityWARNINGset_verbosity_warningr r __mro__printr str_single_map_nestedrb r r array)
r function data_structtypesrank disable_tqdmr
pbar_iterable pbar_descpbarr rz mappeds
r3 r r i s =A:Hk5$ d k4 (K1O $$ G 1 1 3goo E%%' 4reieqeq4r1r
cr& ,6k4+HK%%'kMNRN^t /R3>TJdhI m\Du[d e
(im k4 (^bcVZVWYZA)8QtT4*PQQc
(
( [__UV((AudD$)OP_F_+t,
(
( K/ V}
(
( xx'
(
(c_
(
(s< 3F8 F-%F82F86F3F8+F8F8- F88Gr r dict_onlymap_list map_tuple map_numpynum_procparallel_min_lengthr r r c |ag }|sR|r |j t |r |j t |r |j t j t | }t |t st || s | | S | xs t j } t |t rt |j n|} |d}|dk7 r|dk st | |k r5t j | | |
D cg c] } t | | |dddf }
} nNt j 5 t j ddt t# | | ||| |
t }
ddd t |t r#t
t% |j'
S t |t r
S t |t r t
S t j(
S c c} w # 1 sw Y xY w) a Apply a function recursively to each element of a nested data struct.
Use multiprocessing if num_proc > 1 and the length of data_struct is greater than or equal to
`parallel_min_length`.
Before version 2.5.0, multiprocessing was not used if `num_proc` was greater than or equal to ``len(iterable)``.
Now, if `num_proc` is greater than or equal to ``len(iterable)``, `num_proc` is set to ``len(iterable)`` and
multiprocessing is used.
Args:
function (`Callable`): Function to be applied to `data_struct`.
data_struct (`Any`): Data structure to apply `function` to.
dict_only (`bool`, default `False`): Whether only apply `function` recursively to `dict` values in
`data_struct`.
map_list (`bool`, default `True`): Whether also apply `function` recursively to `list` elements (besides `dict`
values).
map_tuple (`bool`, default `False`): Whether also apply `function` recursively to `tuple` elements (besides
`dict` values).
map_numpy (`bool, default `False`): Whether also apply `function` recursively to `numpy.array` elements (besides
`dict` values).
num_proc (`int`, *optional*): Number of processes.
parallel_min_length (`int`, default `2`): Minimum length of `data_struct` required for parallel
processing.
types (`tuple`, *optional*): Additional types (besides `dict` values) to apply `function` recursively to their
elements.
disable_tqdm (`bool`, default `True`): Whether to disable the tqdm progressbar.
desc (`str`, *optional*): Prefix for the tqdm progressbar.
Returns:
`Any`
Nr r )r r TignorezL.* is experimental and might be subject to breaking changes in the future\.$)messagecategory)appendrb r r ndarrayrG re r is_progress_bar_enabledri lenr r warningscatch_warningsfilterwarnings UserWarningr rf rj r )r r r r r r r r r r r r rr r s r3
map_nestedr" s d
} T" U# RZZ(e k4 (K1O $$ Hw'F'F'H#HL-7 T-JtK&&()P[H2~(a-3x=;N+N ||HlN
#udD$ G H
$
$
& o # #g$
"(Hh|UY[mnF
o +t$C ((*F344 k4 (M
U
+= 88F# #+
o os G 12G%%G.c e Zd ZddZddZy)NestedDataStructureNc & ||| _ y g | _ y ro )data)r r& s r3 r z NestedDataStructure.__init__ s ,D " r5 c ||n | j }t |t r(| j t |j S t |t t f r(|D cg c] }| j | D ] }| c}}S |gS c c}}w ro )r& rG re flattenrb ri r )r r& item flatteneds r3 r( zNestedDataStructure.flatten sw 'tTYY dD !<<T[[] 34 4
tUm
,*.S$ T@RS9ISIS S6M Ts $ B
ro )r r r r r( rx r5 r3 r$ r$ s 5r5 r$ c t t j j | j }| |k S # t
$ r Y yw xY w)NT)r ospathabspathfreeOSError) needed_bytes directory
free_bytess r3 has_sufficient_disk_spacer4 sE :;@@
* $$ s 29 AAurl_pathc v t | }d}|j dv r|j dk( rd| v r6| j d st d| d | j dd } | |fS |j d d }d
|v r|j d
n|d f\ }}|j d \ }}d
| d | d| d} | d| }| |fS )zMConvert a link to a file on a github repo in a link to the raw github object.N)httphttpss3z
github.comblobz.pyz External import from github at z) should point to a file ending with '.py'rawr z/tree/masterrR zhttps://github.com/z /archive/z.zip-)r schemenetlocrI rJ rY r- split)r5 parsed
sub_directory github_path repo_infobranch
repo_owner repo_names r3 _convert_github_urlrH s
h
FM
}}//FMM\4Q X $$U+ #B8*Lu!vww ''6H ] "" !++ab/K?G;?V 1 1( ;]hjr\s Iv$-OOC$8 !J ,ZL)IfXUYZH (k6( 3M ] ""r5 file_pathc N g }t | d 5 }|j |j ddd t j d| d g }d}|D ] }t j d| }t | dk( r| }|r.t j d |t
j
)t j d |t
j
j d rt fd |D r j d
rD j d
}t | \ }}|j d j d ||f j d s
|j d j d j d df B j d
r7 j d
}|j d j d |df |j d j d j d df |S # 1 sw Y xY w)a Find whether we should import or clone additional files for a given processing script.
And list the import.
We allow:
- library dependencies,
- local dependencies and
- external dependencies whose url is specified with a comment starting from "# From:' followed by the raw url to a file, an archive or a github repository.
external dependencies will be downloaded (and extracted if needed in the dataset folder).
We also add an `__init__.py` to each sub-folder of a downloaded folder so the user can import from them in the script.
Note that only direct import in the dataset processing script will be handled
We don't recursively explore the additional import to download further files.
Example::
import tensorflow
import .c4_utils
import .clicr.dataset-code.build_json_dataset # From: https://raw.githubusercontent.com/clips/clicr/master/dataset-code/build_json_dataset
zutf-8)encodingNz Checking z for additional imports.Fz[\s\S]*?"""[\s\S]*?r z=^import\s+(\.?)([^\s\.]+)[^#\r\n]*(?:#\s+From:\s+)?([^\r\n]*))flagszQ^from\s+(\.?)([^\s\.]+)(?:[^\s]*)\s+import\s+[^#\r\n]*(?:#\s+From:\s+)?([^\r\n]*)c 3 L K | ] }|d j d k( yw)r r N)group)ry impmatchs r3 r| z get_imports..D s" ?3q6U[[^+?s !$ externalr internallibrary)openextend readlinesloggerdebugr_ rd r rP MULTILINErN r rH r )
rI linesr importsis_in_docstringlinedocstr_start_matchr5 rB rP s
@r3 get_importsr` s (
E
i' * $a
Q[[]#$ LL9YK'?@A9;GO )R ZZ(> E ! "a ' #2 1O
Y[_gigsgst =HHdllE
} ;;q>?w??{{1~ ;;q>*=h*G'-
EKKNHm TUQ
EKKNEKKND QR{{1~ ;;q> 5;;q>8T JK 5;;q>5;;q>4 PQS)RV Nc$ $s HH$c e Zd ZdZej
j
ej j j ZddZ
d Z y)PicklerzHSame Pickler as the one from dill, but improved for notebooks and shellsc
t | }|t j vrwt j t j d k rd n^t j j d d t j d j t j d j fv rd |j |j fdk( r dd l
t | fd }n|j |j fd k( r dd l
t | fd
}n|j |j fd k( r dd l t | fd }nP|j j d
r5t! d |j" D r dd l
t |
fd }t& j j) | || y # t $ r Y .w xY w# t $ r Y =w xY w# t $ r Y Lw xY w# t $ r Y [w xY w)N0.3.6c V t j j j | y ro )dill_dillloginfopicklermsgs r3 dill_logz Pickler.save..dill_loge s JJNN'',r5 rQ 0.3.7c X t j j j | | y ro )rf rg rX tracerj s r3 rm z Pickler.save..dill_logj s JJ%%++GS9r5 )_regexPatternr c | d| |j |j f}| j j || | d y )NzRe: rq z# Re)r[ rL save_reducecompile)rk rr r rm rg s r3 _save_regexz!Pickler.save.._save_regexq sP D,7 KK II ++EMM4S+I &1 r5 )r Tensorc fd} | d| |j j j f}| j ||| | d y )Nc & j | S ro )
from_numpy)np_arrayr s r3 _create_tensorz:Pickler.save.._save_tensor.._create_tensor s #(#3#3H#= =r5 zTo: rq z# To)detachcpunumpyrt )rk rr r| r rm r s r3 _save_tensorz"Pickler.save.._save_tensor s\ > !D,7 #
0 0 2 8 8 : < ++NDc+J &1 r5 )z
tiktoken.coreEncodingc | d| |j |j |j |j f}| j j
|| | d y )NzEnc: rq z# Enc)r0 _pat_str_mergeable_ranks_special_tokensrt r )rk rr r rm tiktokens r3 _save_encodingz$Pickler.save.._save_encoding s] E#-8 ##,,8L8LcNaNa b ++H,=,=t+M '2 r5 z
spacy.langc 3 R K | ] }|j |j fd k( ! yw))zspacy.languageLanguageN)r r )ry clss r3 r| z Pickler.save.. s* FUX .2PPFs %'c fd} | d| |j |j f}| j ||| | d y )Nc j j | d d }|j | }|j | S )Nnlplang)utilget_lang_class from_config
from_bytes)r
bytes_datalang_clsr spacys r3 _create_langz6Pickler.save.._save_lang.._create_lang s? ',zz'@'@ vAV'WH"*"6"6v">C#&>>*#= =r5 zSp: rq z# Sp)r to_bytesrt )rk rr r r rm r s r3
_save_langz Pickler.save.._save_lang sP >
!D,7 #
CLLN ; ++L$C+H &1 r5 )save_persistent_id)rp rb dispatchr DILL_VERSIONr parsereleaser r rg pklregister ImportErrorr r
startswithr r r rf save)
r rr r obj_typerv r r r rm rg r r r s
@@@@@r3 r z Pickler.save_ s 9 7++ +""W]]7%;;- $$,,Ra0W]]75K5S5SU\UbUbcjUkUsUs4tt: ##X%6%67;PP * + %%x'8'89=PP * + %%x'8'89=ZZ # * + $$// =# F\d\l\lF C *
+
$8JKq # $ # # * # sH F> =G
2G G+ > G
G
GG G('G(+ G76G7c j t | t k7 r!t j j | | y y ro )rp r rf rb memoize)r rr s r3 r zPickler.memoize s' 9 LL s + r5 N)T) r r r r rf rg MetaCatchingDictrb r r r r rx r5 r3 rb rb Z s9 Rzz**4<<+@+@+E+E+GHHULn,r5 rb c < t |d j | y)zpickle an object to a fileT)recurseN)rb dumprr files r3 r r s D$ $$S)
r5 c # > K dt | j D cg c] }|j c}v rAt | d r5t | j
t rt | di 5 d d d d y d y c c}w # 1 sw Y y xY w# t $ r d Y y w xY ww)NPreTrainedTokenizerBasecache) rp r r r rG r re r r )rr
base_classs r3 _no_cache_fieldsr s %PTUXPYPaPa)b***=*=)b bW%399d+%c7B7
*c
sh B B
A97B
&A>+B
3B 4B
8B 9B
>BB
B B
BB BB c t }t | 5 t | | ddd |j S # 1 sw Y |j S xY w)z pickle an object to a stringN)StringIOr r getvaluer s r3 dumpsr sB :D # S$ ==? ==?s
;Ac fd}|S )Nc . | t j < | S ro )rb r )r ts r3 proxyzpklregister..proxy s " r5 rx )r r s ` r3 r r s Lr5 rd c Z t j j j d| |j j d st
|j j t j j dkD rE|j j t j j d j d s|j dk( rdn(t j j |j }d}t j j r@t |d r|j |j |j" |j$ |j& |j( |j* |j, |j. |j0 ||j ||j2 |j4 |j6 f}n |j |j" |j$ |j& |j( |j* |j, |j. |j0 ||j ||j2 |j4 |j6 f}n|j |j$ |j& |j( |j* |j, |j. |j0 ||j ||j2 |j4 |j6 f}| j9 t: || t j j j d
y ) z
From dill._dill.save_code
This is a modified version that removes the origin (filename + line no.)
of functions created in notebooks or shells for example.
zCo: r co_posonlyargcountrq # CoN) rf rg rh ri co_filenamer r r@ r, r- sepco_namebasenamePY3r co_argcountr co_kwonlyargcount
co_nlocals co_stacksizeco_flagsco_code co_constsco_names co_varnames co_lnotab co_freevars co_cellvarsrt r
rk rr r co_firstlinenor s r3
_save_coder sE
d3%L)& ))#.COO))"''++67!;OO))"''++6r:EElS{{j(
!!#//2 ::>>s 01OO**))NN$$LLKKMMLLOO KK"MMOOOO!( OO))NN$$LLKKMMLLOO KK"MMOOOO &
D Hd4
F#r5 rQ rn c
t j j j | d| |j j d st
|j j t j j dkD rE|j j t j j d j d s|j dk( rdn(t j j |j }d}t |d r|j |j |j |j" |j$ |j& |j( |j* |j, |j. |j0 ||j |j2 ||j4 |j6 |j8 |j: |j< |j> f}nt |d r|j |j |j |j" |j$ |j& |j( |j* |j, |j. |j0 ||j |j2 ||j4 |j: |j< |j> f}nt |d
r|j |j |j |j" |j$ |j& |j( |j* |j, |j. |j0 ||j ||j4 |j< |j> f}n>t |d r|j |j |j" |j$ |j& |j( |j* |j, |j. |j0 ||j ||j |j< |j> f}n|j |j" |j$ |j& |j( |j* |j, |j. |j0 ||j ||j |j< |j> f}| jA t j jB || t j j j | d
y )NzCo: %sr r r@ r r r co_endlinetableco_exceptiontable co_linetabler rq r )"rf rg rX rp r r r r@ r, r- r r r r r r r r r r r r r r r co_qualnamer r co_columntabler r r rt _create_coder s r3 save_coder F s
37, ))#.COO))"''++67!;OO))"''++6r:EElS{{j(
!!#//2 3) *
&&%%
##""%%+D. S-
.
&&%%
%%'D* S.
)
&&%%
#D& S.
/&&%%
!D( %%
D$ DJJ 3 3TsC
0r5 z0.3.5c
R t j j | snt j j j d| t | dd r}t j j } ||dd }t | t j j v ret j j r |j n |j }n2t j j r |j n |j }t t |j }t | dd }t | dd }t | t j j v xr |du}t! t j j |ft j j t | < t j j rdt |j" d d
v xr |du}|rd| _ |rd| _ t |d d }| j) t j j* |j" ||j, |j. |j0 |j2 |f| ndt |j4 d d
v xr |duxr
t | dd }|rd| _ |rd| _ | j) t j j* |j4 ||j6 |j8 |j: |j2 f| |r|| _ |r|| _ t j j< r |s |s
|s|s|s|s|r| j? t j j j d
yt j j j d| t |dt |dd } t j j@ jC | || t j j j d y)z
From dill._dill.save_function
This is a modified version that make globs deterministic since the order of
the keys in the output dictionary of globalvars can change.
zF1: _recurseFTr builtin_byrefNr r rx __kwdefaults__rq # F1zF2: r r r0 # F2)"rf rg _locate_functionrh ri r} detect
globalvarsidstackr __globals__ func_globalsre sortedr r __code__r r rt _create_functionr __defaults__ __closure____dict__ func_code func_name
func_defaults func_closureOLDER
clear_memo StockPickler save_global)
rk rr r globsr r _memo_super fkwdefaultsr0 s
r3
save_functionr s1 zz**3/ JJNN $se -w
E2 ![[ 3 3
"3dCc7djj . ../3zz~~COO3CSCSE+/::>>s?O?O
./E Wh5F w
D9HW
0 00Lxt7KE(+DJJ,<,<(=s(BDJJ RW %zz~~!WS\\:r%JJdQW_cQc%)GN',G$ %c+;T B ##JJ//\\5#,,8H8H#//[^[g[gitu $
z2 FF < t+< U;
%)GN',G$##JJ//]]E3==#:K:KSM]M]_b_k_kl $
!'#+
FufUW_""$ JJNN '
JJNN $se -3Z0NOD JJ # # / /4 / H JJNN 'r5 c x t j j || sft j j j d|z t | dd }t | dd }t | dd }t | dt j j }g }|r ddlm} ||dd }d
|j i} nt j j r |j n |j }|r(||j u rt | d | j }|} nV|F|j :t t j j |j d d d |u r|} nd
|j i} | |u }
t t! | j# } |
r| }n$|"t t! |j# }|| |urt j j r'|j% D ch c]
} t' | } } n&|j) D ch c]
} t' | } } |D ]8 }
|
| v s||
j+ t j j, | |ff n. |j+ t j j, | |ff t j j r|j. }i }d
D ] }t ||d }||||< |j0 |j2 k7 r|j0 |d< d
| vs|j | d
k7 r|j |d< |j }t5 | t ur||d < d }|r||f}t j j7 | t j j8 |j: | |j2 |j< |f|f|| n|j> }|j@ $|j+ tB |d|j@ ff d
| vs|j | d
k7 r$|j+ tB |d|j ff |j r$|j+ tB |d |j ff t j j7 | t j j8 |jD | |jF |jH |ff|| |rtK tM |j% d }|r||rz|D ]u }tB |d|ff} |jO | | jR | t j j r | jU tW dd e| jU d w t j j j d y t j j j d|z t |dt |d
d }t j jX j[ | || t j j j d y c c} w c c} w # tP $ r Y Rw xY w)NF1: %sr _postproc_main_modified_original_mainr r Tr r _mainr r r __annotations__r r rr
postproc_listr
cell_contents0UTF-8r F2: %sr r ).rf rg r rh ri r} __builtin__ dill.detectr r r r r r _import_modulere r r ri r
itervaluesr _setitemsr r r rp _save_with_postprocr r r r r r r r r nextiterremoverJ rt writebytesr r )rk rr r r r r r r
globs_copyr globs_is_globs_copygglob_ids
stack_elementclosure
state_dict fattrnamefattrstatetopmost_postproccellpossible_postprocr0 s r3 r r & sn zz**38 JJNN 3 / w
D9H d;I $W.> EN $W.>
@V@V WN M2 'T4 H
$S^^404
S__CDTDT
"jN4K4K&K!('>!J!S!SJ &E *2
9 9#..$ OQ[]abfpp &E ' 8E #(:"5
./E" "
' !&)9)9);"< =
%%z*A ::>>/9/@/@/B C!1 CH C/9/D/D/F G!1 GH G%. VM$0!-0779M9MPUWaOb8cd V
"(($***>*>
@S)TUzz~~ //
!Q 6I #CD 9E (05
9-6 ##s||3141A1AJ~. U*cnnj@Q.Q/2~~J|, ;d*-2Jz* E !: -E
..
33 ucllC? ::>> #MM%W*= > #MM# .
/
JJNN '
JJNN 3 /3Z0NOD JJ # # / /4 / H JJNN 'a D Gz * % $%s 1X"X'5X,, X98X9c h t j j || st |j t
urt
|dd }|$t j j j }t j j |d }d} t j j ||j \ }}t
|dd |u rd}|rpt j j j | d| | j t df| t j j j | d y t j j j | d | t
| d
d }t
| d d }t
| d d } t
| d
t j j }
g } |r ddlm} | |dd }
d|j$ i}n|j& }
| r(|
|
j( u rt
| d|
j( }
|
}nV|
F|j$ :t
t j j |j$ d dd |
u r|
}nd|j$ i}||
u }t+ t- |j/ }|r|}
n$|
"t+ t- |
j/ }
|
||
ur|
j1 D ch c]
}t3 | }}|D ]8 }||v s|| j5 t j j6 ||
ff n. | j5 t j j6 ||
ff |j8 }i }dD ] }t
||d }||||< |j |j k7 r|j |d< d|vs|j$ |d k7 r|j$ |d< |j( }t | t* ur||d< d }|r||f}t j j; | t j j< |j ||j |j> |f|f|| |rutA tC |j1 d }|rP|rN|D ]I }tD |d|ff} |jG | | j | | jK tM dd K t j j j | d y t j j j | d| t
|dt
|dd }t j jN jQ | || t j j j | d y # t $ r Y w xY wc c}w # tH $ r Y 2w xY w) Nr T)safeF__func__zF3: %srq z# F3r r r r r r r r r r r r r r r r r
r r r r ))rf rg r rp r r
r} r r r
_getattributer AttributeErrorrX rp rt r
r r r r re r r ri r r r r r r r r r r r rJ r r r r ) rk rr module_namemodule
_pypy_builtinfound_r r r r r r r r r r r r r r r r r r! r" r# r0 s r3 r r s9 zz**38CLL!1 &c< > &"&**"8"8"A"AK 22;T2J %
#zz 7 7@P@P QHE1 uj$73>(,
!JJ%%++GXsC''%1D#'NJJ%%++GV< JJ # #GXs ; w
D9H d;I $W.> EN $W.>
@V@V WN M2 'T4 H
$S^^4 __
"jN4K4K&K!('>!J!S!SJ &E *2
9 9#..$ OQ[]abfpp &E ' 8E #(:"5
./E" "
' !&)9)9);"< =
%%z*A ,6+<+<+>?aBqE??%. VM$0!-0779M9MPUWaOb8cd V
"(($***>*>
@S)TUooGJ M
2 Y5$,1Jy)
2 3<</-0-=-=
>*&#..E*>
<(LLEE{$&).
:& z) JJ * *,,s||UCLLRURbRbdk.lnst +
+
#'Y-=-=-?(@$#G / '
;-4t_c6R,S)% , 3 34E F
,++->?
eC&9:
;
JJ # #GV 4
JJ # #GXs ;3Z0NOD JJ # # / /4 / H JJ # #GV 4s & D @R * % $%s* >V V
V$ V V $ V10V1c t j | j | j | j | j
| j }| j |_ |S ro )r r r r r r r r )r rh s r3 copyfuncr/ @ sI
t/?/? PTPaPacgcscs
tF / /F Mr5 Yqueuer .r c \ t |di | D ] \ }} | j | S )Nrx )r put)r1 r r r rh s r3 _write_generator_to_queuer4 I s3 t~f~. 6 & Hr5 poolc T | j D ch c] }|j c}S c c}w ro )_poolpid)r5 r s r3
_get_pool_pidr9 O s :: &aAEE && &s %kwargs_iterablec
# K t | }d}t | t j j rt
nt j
} | 5 }|j }|D cg c] }| j t |||f } } |j d t | |k7 r
d}t d 2c c}w # t $ r( t d | D r |j rY nY Qw xY w |sU| D
cg c] }
|
j d nc c}
w c}
n.# |s'| D
cg c] }
|
j d nc c}
w c}
w w xY wd d d y # 1 sw Y y xY ww)NFTg?)timeoutc 3 < K | ] }|j y wro )ready)ry async_results r3 r| z%iflatmap_unordered..f s RL<--/Rs zkOne of the subprocesses has abruptly died during map operation.To debug the error, disable multiprocessing.)r9 rG multiprocessingr5 Poolr multiprocessQueue apply_asyncr4 r r allempty RuntimeError) r5 r r: initial_pool_pid pool_changed manager_clsmanagerr1 r
async_resultsr? s r3 iflatmap_unorderedrM S sh %T*L 'o.B.B.G.G H'lNbNbK S'
]l
SYD 6f8M N
S #%))D11 !&*::#'L&G
RMRRWbW\WbWbWd DQRL !!$!/RR DQRL !!$!/RR )S S Ss AE"E& B<E D C D <E,C2-D 1C22D 6E<D
E E
&E
? E
E
E"E E")FF) FTFFNr NTN)rO )jr r functoolsr multiprocessing.poolr@ r, r1 r_ r r
contextlibr dataclassesr r ior r r r shutilr r
r typingr r
r r r r r r r r urllib.parser rf rB multiprocess.poolr r packagingr tqdm.autor r r parallelr r typing_extensions_typing_extensionsr r r
get_loggerr rX lru_cacher r4 r- r rL r\ rl r r r r r r r re r propertyr r boolr r" r$ r4 rH r` rb r r r r r r r r r r r/ r0 rC r4 r5 rA r9 rM rx r5 r3 r` s % , " # ( \ \ \ ! # 020
H %
)0)s5c? )ss )sX& 3 c3h B$ N % % 2,C 2, 2,j0%T %82H 2 (F " ! \$ucz"\$\$ \$ \$
\$
\$ sm\$ \$ E?\$ \$ 3-\$ \$~ %## #%Xc]0B*C #&F3 F5c3);#< FR_,dll _,D
w//\ \| !$ w)?)G)G W^I_IgIg(hhO Od
w// F FP !$
g(>(F(FF J JX !$ w)?)G)G W^I_IgIg(hh G GT CL
U[[
hqkAQ8R
\`
eh
'o 2 2 7 7 9J9J9O9O OP 'UXY\U] ' S
$$))<+<+<+A+AA
B S
3 #
$ S d^ S
a[ S{( 0+///50s " N: : O O