*#hddlmZmZddlmZmZmZmZmZm Z m Z ddl Z erddl mZeGddZeGdd Zy) ) dataclassfield) TYPE_CHECKINGAnyClassVarDictListOptionalUnionN) FeatureTypeceZdZUdZeeed<dZeeed<dZ e eed<dZ e e ed<e ddd Zeed <d Zd ed eed fffdZy) Translationa`FeatureConnector` for translations with fixed languages per example. Here for compatiblity with tfds. Args: languages (`dict`): A dictionary for each example mapping string language codes to string translations. Example: ```python >>> # At construction time: >>> datasets.features.Translation(languages=['en', 'fr', 'de']) >>> # During data generation: >>> yield { ... 'en': 'the cat', ... 'fr': 'le chat', ... 'de': 'die katze' ... } ``` languagesNiddictdtypepa_typeFdefaultinitrepr_typectjt|jDcic]}|tjc}Scc}wN)pastructsortedrstring)selflangs Z/var/www/html/sandstorm/venv/lib/python3.12/site-packages/datasets/features/translation.py__call__zTranslation.__call__)s3yyt~~8NO$ +OPPOsA returnr clddlm}t|jDcic] }||d c}Scc}w)z2Flatten the Translation feature into a dictionary.r )Valuer)featuresr&rr)r r&ks r"flattenzTranslation.flatten,s+#,24>>,BCq5?"CCCs1)__name__ __module__ __qualname____doc__r str__annotations__rr rrrrrrr#r rr)r"rr sx*CyB !E8C=!!GXc]!}5uEE3EQD}d3 3E.FFGDr1rceZdZUdZdZeeed<dZee ed<dZ ee ed<dZ e e ed<dZe eed<edd d Ze ed <d Zd ZdZdedee dfffdZy)TranslationVariableLanguagesa$`FeatureConnector` for translations with variable languages per example. Here for compatiblity with tfds. Args: languages (`dict`): A dictionary for each example mapping string language codes to one or more string translations. The languages present may vary from example to example. Returns: - `language` or `translation` (variable-length 1D `tf.Tensor` of `tf.string`): Language codes sorted in ascending order or plain text translations, sorted to align with language codes. Example: ```python >>> # At construction time: >>> datasets.features.TranslationVariableLanguages(languages=['en', 'fr', 'de']) >>> # During data generation: >>> yield { ... 'en': 'the cat', ... 'fr': ['le chat', 'la chatte,'] ... 'de': 'die katze' ... } >>> # Tensor returned : >>> { ... 'language': ['en', 'de', 'fr', 'fr'], ... 'translation': ['the cat', 'die katze', 'la chatte', 'le chat'], ... } ``` Nr num_languagesrrrrFrrc|jrtt|jnd|_|jrt|j|_yd|_yr)rrsetlenr4r s r" __post_init__z*TranslationVariableLanguages.__post_init__\s=8<DNN 34D48NNS0r1ctjtjtjtjtjdS)Nlanguage translation)rrlist_rr8s r"r#z%TranslationVariableLanguages.__call__`s8yybhhryy{&;BHHUWU^U^U`Labccr1c t|j}|jrSt||z rEtddjt t||z ddj|dg}|j D]J\}}t |tr|j||f*|j|Dcgc]}||fc}Ltt |\}}||dScc}w)NzSome languages in example (z, z) are not in valid set (z).r;) r6r ValueErrorjoinritems isinstancer.appendextendzip) r translation_dictlang_settranslation_tuplesr!textelr translationss r"encode_examplez+TranslationVariableLanguages.encode_examplecst~~& >>c"23h>-diisCS?TW_?_8`.a-bbz{|E|EFN|O{PPRS   *002 GJD$$$"))4,7"))*E"D":*EF  G#&v.@'A"B <%lCC +Fs C- r$r cLddlm}m}||d||ddS)zCFlatten the TranslationVariableLanguages feature into a dictionary.r )Sequencer&rr;)r'rOr&)r rOr&s r"r)z$TranslationVariableLanguages.flattenxs)-!x1#E(O4  r1)r*r+r,r-rr r r/r4intrr.rrrrrrr9r#rMr rr)r0r1r"r3r33s>!%Ix~$#'M8C='B !E8C=!!GXc]!=EPUVE3VMdD* }d3 3E.FFG r1r3) dataclassesrrtypingrrrrr r r pyarrowrr'r rr3r0r1r"rTsU(LLL% $D$D $DN K K  K r1