
    +#h                       d dl mZ d dlZd dlZd dlZd dlZd dlmZmZ d dl	m	Z	 d dl
mZmZmZ d dlmZ d dlmZ d dlmZ d d	lmZmZmZ d dlZd dlZd d
lmZmZ d dlmZ d dlmZ  d dl!m"Z"m#Z# d dl$m%Z% d dlm&Z& d dl'm(Z( d dlm)Z) d dl*m+Z+ d dl,m-Z- d dl.m/Z/m0Z0 d dl1m2Z2 d dl3m4Z5 d dl6m7Z7m8Z8 d dl9m:Z: d dl;m<Z<m=Z=m>Z>  e>       rd dl?m@Z@mAZAmBZBmCZCmDZD  ej                  eF      ZGerd dlHmIZI d dlJmKZK d dlLmMZM  G d de)      ZN e2d        G d! d"eN             ZOg d#ZPg d$ZQd*d%ZRd+d&ZSe G d' d(e             ZTd,d)ZUy)-    )annotationsN)Counterdefaultdict)copy)	dataclassfieldfields)Path)python_versionindent)TYPE_CHECKINGAnyLiteral)CardData	ModelCard)dataset_info)
model_info)
EvalResulteval_results_to_model_index)	yaml_dump)nn)tqdm)TrainerCallback)CodeCarbonCallback)make_markdown_table)TrainerControlTrainerState)
deprecated__version__)StaticEmbeddingTransformer)$SentenceTransformerTrainingArguments)fullnameis_accelerate_availableis_datasets_available)DatasetDatasetDictIterableDatasetIterableDatasetDictValue)SentenceEvaluator)SentenceTransformer)SentenceTransformerTrainerc                       e Zd Zd fdZ	 	 	 	 	 	 	 	 	 	 	 	 ddZ	 	 	 	 	 	 	 	 	 	 ddZ	 	 	 	 	 	 	 	 	 	 	 	 d	dZ	 	 	 	 	 	 	 	 	 	 	 	 d
dZ xZS )$SentenceTransformerModelCardCallbackc                0    t         |           || _        y N)super__init__default_args_dict)selfr6   	__class__s     ]/var/www/html/sandstorm/venv/lib/python3.12/site-packages/sentence_transformers/model_card.pyr5   z-SentenceTransformerModelCardCallback.__init__/   s    !2    c                   ddl m}m}m}	 |j                  j                  d       |j                  j                  D 
cg c]  }
t        |
t              s|
 }}
|r|d   |j                  _
        |j                  rU|j                  j                  |j                  |j                  j                  |j                  d      |j                  _        |j                  rU|j                  j                  |j                  |j                  j                   |j                  d      |j                  _        t        |j                  t"              r$t%        |j                  j'                               }n|j                  g}d}|t)        |      k  r]||   }t        ||	||f      r5t+        |d      r)|j                  |vr|j-                  |j                         |dz  }|t)        |      k  r]|j                  j/                  |       |j                  j0                  s9|j                  xs |j                  x}r|j                  j3                  |       y y y c c}
w )Nr   )AdaptiveLayerLossMatryoshka2dLossMatryoshkaLossgenerated_from_trainertrainevalloss   )sentence_transformers.lossesr<   r=   r>   model_card_dataadd_tagscallback_handler	callbacks
isinstancer   code_carbon_callbacktrain_datasetextract_dataset_metadatatrain_datasetsrB   eval_dataseteval_datasetsdictlistvalueslenhasattrappend
set_losseswidgetset_widget_examples)r7   argsstatecontrolmodeltrainerkwargsr<   r=   r>   callbackrH   lossesloss_idxrB   datasets                   r9   on_init_endz0SentenceTransformerModelCardCallback.on_init_end3   s    	ed&&'?@ &-%=%=%G%G
!:V^`rKsH
	 
 9B1E!!6   383H3H3a3a%%u'<'<'K'KW\\[b4E!!0 272G2G2`2`$$e&;&;&I&I7<<Y_3E!!/ gllD)',,--/0Fll^F V$(#D4.2CEU!VWD&)IIV+dii(MH V$ 	((0 $$++G<P<P<iT[TiTi1i1i!!55g> 2j+K
s   IIc                b   h d}|j                         }|j                         D 	ci c]  \  }}	||vs||	 c}	}|j                  _        |j                         D 	ci c],  \  }}	||vr#|| j                  v r|	| j                  |   k7  r||	. c}	}|j                  _        y c c}	}w c c}	}w )N>   do_evaldo_testdo_trainrun_name	hub_token	report_to
eval_delay
eval_steps
output_dir
save_stepslogging_dirlogging_stepssave_strategylogging_strategysave_total_limitgreater_is_betterpush_to_hub_tokensamples_per_labelshow_progress_barlogging_first_stepevaluation_strategymetric_for_best_model)to_dictitemsrE   all_hyperparametersr6   non_default_hyperparameters)
r7   rY   rZ   r[   r\   r^   ignore_keys	args_dictkeyvalues
             r9   on_train_beginz3SentenceTransformerModelCardCallback.on_train_begini   s    
0 LLN	)2):5
%3c>TCJ5
1
 (oo/=
U+%#1G1G*GEUYUkUkloUpLp J=
95
=
s   B%B% 1B+c                   |D ci c];  }|j                  d      sdj                  |j                  d      dd        ||   = }}t        |      dk(  rd|v rd|d   i}|j                  j
                  rR|j                  j
                  d   d   |j                  k(  r)|j                  j
                  d   j                  |       y |j                  j
                  j                  |j                  |j                  d	|       y c c}w )
N_loss _rC   rB   Validation LossStepEpochr   )
endswithjoinsplitrS   rE   training_logsglobal_stepupdaterU   epoch)	r7   rY   rZ   r[   r\   metricsr^   r   	loss_dicts	            r9   on_evaluatez0SentenceTransformerModelCardCallback.on_evaluate   s     LSlCVYVbVbcjVkSXXciinQR0173<?l	ly>Q6Y#6*If,=>I!!//%%33B7?5CTCTT!!//3::9E!!//66"[[!--   ms
   D)Dc                   dht        |      z  }|r|j                  j                  rW|j                  j                  d   d   |j                  k(  r.||j	                            |j                  j                  d   d<   y |j                  j                  j                  |j                  |j                  ||j	                            d       y y )NrB   r   r   Training Loss)r   r   r   )setrE   r   r   poprU   r   )r7   rY   rZ   r[   r\   logsr^   keyss           r9   on_logz+SentenceTransformerModelCardCallback.on_log   s     x#d)#%%33))77;FCuGXGXXKOPTPXPXPZK[%%33B7H%%33::!& % 1 1)-dhhj)9 r:   )r6   dict[str, Any]returnNone)rY   r$   rZ   r   r[   r   r\   r.   r]   r/   r   r   )
rY   r$   rZ   r   r[   r   r\   r.   r   r   )rY   r$   rZ   r   r[   r   r\   r.   r   dict[str, float]r   r   )rY   r$   rZ   r   r[   r   r\   r.   r   r   r   r   )	__name__
__module____qualname__r5   rc   r   r   r   __classcell__r8   s   @r9   r1   r1   .   s    34?24? 4?  	4?
 #4? ,4? 
4?l(
2(
 (
  	(

 #(
 
(
T2   	
 # " 
42   	
 #  
r:   r1   zThe `ModelCardCallback` has been renamed to `SentenceTransformerModelCardCallback` and the former is now deprecated. Please use `SentenceTransformerModelCardCallback` instead.c                       e Zd Z fdZ xZS )ModelCardCallbackc                $    t        |   |i | y r3   )r4   r5   )r7   rY   r^   r8   s      r9   r5   zModelCardCallback.__init__   s    $)&)r:   )r   r   r   r5   r   r   s   @r9   r   r      s    * *r:   r   )languagelicenselibrary_nametagsdatasetsr   pipeline_tagrW   model-indexco2_eq_emissions
base_model)r\   r]   eval_results_dictc                     t               t        t        j                  t        j                  d} t               rddlm} || d<   t               rddlm} || d<   ddl	m} || d<   | S )N)pythonsentence_transformerstransformerstorchr   r    
accelerater   
tokenizers)
r   sentence_transformers_versionr   r!   r   r&   r   r'   r   r   )versionsaccelerate_versiondatasets_versiontokenizers_versions       r9   get_versionsr      s^     "!>$00""	H  @!3</</H\Or:   c                >    t        | t              rt        | d      S | S )N   )rI   floatroundr   s    r9   
format_logr      s    %UALr:   c                  X   e Zd ZU dZ ee      Zded<   dZded<   dZ	ded<   dZ
ded	<    ee      Zd
ed<    ee      Zd
ed<   dZded<    ed       Zded<   dZded<    edd      Zded<    edd      Zded<    eed      Zded<    eed      Zded<    eed      Zded<    eed      Zd ed!<    eed      Zd
ed"<    edd      Zded#<    eed      Zd
ed$<    edd      Zd%ed&<    eed      Zd'ed(<    edd      Zd)ed*<    eedd+      Zd,ed-<    ed.d      Zd/ed0<    ed1d      Zd2ed3<    ed4d      Z ded5<    ed6d      Z!ded7<    ee"d      Z#d'ed8<    e e$e%      jL                  d9z  d      Z'd:ed;<    eddd<      Z(d=ed><   dWd?Z)	 dX	 	 	 	 	 dYd@Z*dZdAZ+d[dBZ,d\dCZ-	 d]	 	 	 	 	 	 	 	 	 d^dDZ.d_dEZ/d`dadFZ0dbdGZ1	 	 	 	 	 	 	 	 dcdHZ2	 	 	 	 	 	 	 	 	 	 dddIZ3dedJZ4dfdKZ5d`dgdLZ6dhdMZ7didNZ8djdOZ9dWdPZ:dkdQZ;dR Z<dldSZ=dkdTZ>dkdUZ?d`dmdVZ@y)n SentenceTransformerModelCardDataa  A dataclass storing data used in the model card.

    Args:
        language (`Optional[Union[str, List[str]]]`): The model language, either a string or a list,
            e.g. "en" or ["en", "de", "nl"]
        license (`Optional[str]`): The license of the model, e.g. "apache-2.0", "mit",
            or "cc-by-nc-sa-4.0"
        model_name (`Optional[str]`): The pretty name of the model, e.g. "SentenceTransformer based on microsoft/mpnet-base".
        model_id (`Optional[str]`): The model ID when pushing the model to the Hub,
            e.g. "tomaarsen/sbert-mpnet-base-allnli".
        train_datasets (`List[Dict[str, str]]`): A list of the names and/or Hugging Face dataset IDs of the training datasets.
            e.g. [{"name": "SNLI", "id": "stanfordnlp/snli"}, {"name": "MultiNLI", "id": "nyu-mll/multi_nli"}, {"name": "STSB"}]
        eval_datasets (`List[Dict[str, str]]`): A list of the names and/or Hugging Face dataset IDs of the evaluation datasets.
            e.g. [{"name": "SNLI", "id": "stanfordnlp/snli"}, {"id": "mteb/stsbenchmark-sts"}]
        task_name (`str`): The human-readable task the model is trained on,
            e.g. "semantic textual similarity, semantic search, paraphrase mining, text classification, clustering, and more".
        tags (`Optional[List[str]]`): A list of tags for the model,
            e.g. ["sentence-transformers", "sentence-similarity", "feature-extraction"].

    .. tip::

        Install `codecarbon <https://github.com/mlco2/codecarbon>`_ to automatically track carbon emission usage and
        include it in your model cards.

    Example::

        >>> model = SentenceTransformer(
        ...     "microsoft/mpnet-base",
        ...     model_card_data=SentenceTransformerModelCardData(
        ...         model_id="tomaarsen/sbert-mpnet-base-allnli",
        ...         train_datasets=[{"name": "SNLI", "id": "stanfordnlp/snli"}, {"name": "MultiNLI", "id": "nyu-mll/multi_nli"}],
        ...         eval_datasets=[{"name": "SNLI", "id": "stanfordnlp/snli"}, {"name": "MultiNLI", "id": "nyu-mll/multi_nli"}],
        ...         license="apache-2.0",
        ...         language="en",
        ...     ),
        ... )
    )default_factoryzstr | list[str] | Noner   N
str | Noner   
model_namemodel_idlist[dict[str, str]]rM   rO   zjsemantic textual similarity, semantic search, paraphrase mining, text classification, clustering, and morestr	task_namec                 
    g dS )N)sentence-transformerssentence-similarityzfeature-extraction r   r:   r9   <lambda>z)SentenceTransformerModelCardData.<lambda>.  s	     !
 r:   zlist[str] | Noner   r   zLiteral['deprecated']generate_widget_examplesF)defaultinitr   base_model_revision)r   r   r   r~   r}   z.dict[SentenceEvaluator, dict[str, Any]] | Noner   zlist[dict[str, float]]r   rW   predict_examplelabel_example_listzCodeCarbonCallback | NonerJ   dict[str, str]	citationsz
int | Nonebest_model_step)r   r   repr	list[str]r   Tbool
first_saver   intwidget_stepr   r   r   r   versionzmodel_card_template.mdr
   template_path)r   r   r   zSentenceTransformer | Noner\   c                   | j                    }t        | j                   t              r| j                   g| _         | j                  | j                  |      | _        | j                  | j
                  |      | _        | j                  rJ| j                  j                  d      dk7  r+t        j                  d| j                  d       d | _        y y y )N)infer_languages/rC   zThe provided z} model ID should include the organization or user, such as "tomaarsen/mpnet-base-nli-matryoshka". Setting `model_id` to None.)
r   rI   r   validate_datasetsrM   rO   r   countloggerwarning)r7   r   s     r9   __post_init__z.SentenceTransformerModelCardData.__post_init__R  s    "mm+dmmS)!]]ODM"44T5H5HZi4j!33D4F4FXg3h==T]]005:NN0 1^ ^ !DM ;=r:   c                R   g }|D ]  }d|vrd|v r|d   |d<   d|v r	 t        |d         }|j                  rq|rod|j                  v ra|j                  j                  d      }|Dt        |t              r|g}|D ],  }|| j
                  vs| j
                  j                  |       . |j                  | j                  vr&| j                  j                  |j                         	 |j                  |        |S # t        $ r" t        j                  d|d   d       |d= Y ?w xY w)Nnameidr   zThe dataset `id` z5 does not exist on the Hub. Setting the `id` to None.)get_dataset_infocardDatagetrI   r   r   rU   r   r   	Exceptionr   r   )r7   dataset_listr   output_dataset_listrb   infodataset_languager   s           r9   r   z2SentenceTransformerModelCardData.validate_datasetsb  s9    !# 	0GW$7?&-dmGFOw6+GDM:D }}Z4===X+/==+<+<Z+H(+7)*:C@4D3E 0,< C#+4==#@$(MM$8$8$BC
 wwdmm3,,TWW5&&w/9	0: #") ! &NN+GDM+<<qr  	&s   C;;(D&%D&c                   ddi}|D ]&  }	 |j                   ||j                  j                  <   ( t	        t
              }|j                         D ]  \  }}||   j                  |        dd}|j                         D ci c]  \  }} ||      | c}}| _        | j                  D ci c]  }|j                  j                  | c}D cg c]  }d| 	 c}       y # t        $ r Y w xY wc c}}w c c}w c c}w )NzSentence Transformersa  
@inproceedings{reimers-2019-sentence-bert,
    title = "Sentence-BERT: Sentence Embeddings using Siamese BERT-Networks",
    author = "Reimers, Nils and Gurevych, Iryna",
    booktitle = "Proceedings of the 2019 Conference on Empirical Methods in Natural Language Processing",
    month = "11",
    year = "2019",
    publisher = "Association for Computational Linguistics",
    url = "https://arxiv.org/abs/1908.10084",
}
c                b    t        |       dkD  rdj                  | d d       dz   | d   z   S | d   S )NrC   z, r   z and r   )rS   r   )r`   s    r9   	join_listz>SentenceTransformerModelCardData.set_losses.<locals>.join_list  s:    6{Qyy-7&*DD!9r:   zloss:)r`   r   r   r   )
citationr8   r   r   r   rQ   r|   rU   r   rF   )r7   r`   r   rB   inverted_citationsr   r   s          r9   rV   z+SentenceTransformerModelCardData.set_losses  s   # 
&
	  	D59]]	$..112	
 ).'oo/ 	6ND(x(//5	6	
 OaNfNfNhi:J(F)F+X5i]c2dUY4>>3J3JD3P2de$tf~ef   j2des#   #C$C3-C9C>$	C0/C0c                    || _         y r3   )r   )r7   steps     r9   set_best_model_stepz4SentenceTransformerModelCardData.set_best_model_step  s
    #r:   c                f   t        |t        t        f      ry t        |t              rt	        |      }g | _        t        t        j                  t        |j                               d            }d}t        |j                         ddd      D ]  \  }}t        ||   t              r||   j                  j                         D cg c]5  \  }}t        |t              st        |t              r|j                   d	v r|7 }}}||   j#                  |      }	t%        |	      }
|
d
k(  ri }t'        |	j)                  t        j*                  t-        |
      t/        ||
                        D ](  \  }}t1        d |j                         D              ||<   * t3        t5        |j                         d        \  }}|d | t        ||d  d d d         }}|D ]<  }|	|   j                         D cg c]  \  }}|dk7  r|j7                  d      s| }}}t%        |      dk  r|r|j9                         }|	|   j                         D cg c]  \  }}|dk7  r|j7                  d      s| }}}t%        |      dk(  r|j;                  |       n|j=                  |d          t%        |      dk  r|rt%        |      dk  r| j
                  j=                  |d
   t        j*                  |dd  t%        |      dz
        d       |d d | _        ?  y c c}}w c c}}w c c}}w )N)rb      )k  zComputing widget examplesexampleF)descunitleave>   stringlarge_stringr   c              3  d   K   | ](  \  }}|d k7  r|j                  d      st        |       * yw)dataset_name_prompt_lengthN)r   rS   .0r   r   s      r9   	<genexpr>zGSentenceTransformerModelCardData.set_widget_examples.<locals>.<genexpr>  s4      #"Un,S\\BR5S J#s   .0c                    | d   S )NrC   r   )xs    r9   r   zFSentenceTransformerModelCardData.set_widget_examples.<locals>.<lambda>  s
    AaD r:   r   r   r  r  r   rC   )source_sentence	sentences   ) rI   r*   r+   r(   r)   rW   r   randomchoicesrQ   r   r   r|   featuresrP   r,   dtypeselect_columnsrS   	enumerateselectsamplerangeminsumzipsortedr   r   extendrU   r   )r7   rb   dataset_namesnum_samples_to_checkr  num_samplescolumnfeaturecolumnsstr_datasetdataset_sizelengthsidxr  indicesr   target_indicesbackup_indicesr   sentencer  
backup_idxbackup_samples                          r9   rX   z4SentenceTransformerModelCardData.set_widget_examples  s4   g1DEFgw'!'2GtGLLN/Cq IJ#)-!(C)[`*
 =	5%L+ ',/A (/|'<'E'E'K'K'M#FGgt,w.7==D^3^ G  ",/>>wGK{+Lq G(""6==|1DL`bnHo#pq  V  # #&,lln#   fW]]_.IJJGQ-4\k-BDQ\Q]I^_cac_cIdDeNN & 5 *5S)9)?)?)A%Xn,S\\BR5S 	 
 )nq(^!/!3!3!5J .9-D-J-J-L%)C.0FV9W !%M %
 =)Q.!((7 "((q)9: )nq(^ y>A%""(1!6==QZ[\[]Q^befobpstbtCuv (1!}$75E=	56%s   :L!7"L'
"L-
c                $   ddl m} t        |      | j                  |<   t	        |d      r|j
                  x}rt        ||      r$|j                  D cg c]  }|j
                   }}nt        |t              r|g}|j                         D 	ci c]  \  }}	||v s||	 }
}}	| j                  r4| j                  d   d   |k(  r| j                  d   j                  |
       y | j                  j                  ||d|
       y y y c c}w c c}	}w )Nr   )SequentialEvaluatorprimary_metricr   r   r   ) sentence_transformers.evaluationr6  r   r   rT   r7  rI   
evaluatorsr   r|   r   r   rU   )r7   	evaluatorr   r   r   r6  primary_metricssub_evaluatorr   r   training_log_metricss              r9   set_evaluation_metricsz7SentenceTransformerModelCardData.set_evaluation_metrics  s    	I,0My) 9./	H`H`5`_5`)%89U^UiUi"jM=#?#?"j"jOS1#2"3AH#k:3TW[jTjCJ#k #k!!d&8&8&<V&D&L""2&--.BC""))!& $ / 6a/"j $ls   DDDc           	     (   d}t        t              }t               }|D ]m  }|d   }|d   }||vrC||   j                  dt	        |       d       t        ||         |k\  r|j                  |       t        |      | j                  k(  sm n |j                         D cg c]^  \  }}| j                  j                  r)t        |t              r| j                  j                  |   n|ddj                  |      z   dz   d	` c}}| _        y c c}}w )
Nr  textlabelz<li>z</li>z<ul> z</ul>)LabelExamples)r   rQ   r   rU   r   rS   addnum_classesr|   r\   labelsrI   r   r   r   )	r7   rb   num_examples_per_labelexamplesfinished_labelsr  r@  rA  example_sets	            r9   set_label_examplesz3SentenceTransformerModelCardData.set_label_examples  s
   !"t$% 	F&>D7OEO+&&d4j\'?@x'+AA#''.?#t'7'77	 '/nn&6#

 #{ 6:ZZ5F5F:V[]`Ka**51gl"RWW[%99GC#
 #
s   !A#Dc           	     4   t        |t              r=|j                         D cg c]  \  }}| j                  ||      D ]  }| ! c}}}S |rt	        j
                  d|      rd }|xs |j                  j                  t        |j                        d}|j                  j                  rR|j                  |j                  j                  v r0|j                  j                  |j                     j                  |d<   |j                  x}rt        |j                               d   }|j                  d      rUd|v rQ|t!        d      d  j                  d      }|d   |d<   |d	   j                  d
      d   x}rt!        |      dk(  r||d<   |gS c c}}}w )N)r  z_dataset_\d+)r   r   sizer   zhf://datasets/@r   rC   r   (   revision)rI   r)   r|   infer_datasetsrematchr   r  r   r   splitsnum_examplesdownload_checksumsrQ   r   
startswithrS   )	r7   rb   r  sub_datasetdataset_output	checksumssourcesource_partsrQ  s	            r9   rR  z/SentenceTransformerModelCardData.infer_datasets"  s   g{+ 29 -L+#22;\2Z    BHH_lCL !=GLL$=$='
 <<7==GLL4G4G#G%,\\%8%8%G%T%TN6"  22292)..*+A.F  !12sf}%c*:&;&=>DDSI'3At$ ,Q 5 5c :1 ==H=3x=TVCV19N:.7s   $Fc                8    | j                   j                  |      S r3   )r\   tokenize)r7   r@  s     r9   r_  z)SentenceTransformerModelCardData.tokenizeA  s    zz""4((r:   c                

   |si S t        |t              rt        |      |d<   |j                  D cg c]  }d| d
 c}|d<   i |d<   t        |t              r|j                  D ]j  }|dd |   }|d   }t        |t              r| j                  |      }t        |t              r*d	|v r&|d	   j                  d
      j                         }d}	n|D 
cg c]  }
t        |
       }}
d}	dt        t        |      d       d|	 t        t        |      t        |      z  d       d|	 t        t        |      d       d|	 dd|d   |<   t        |t        t        f      rTt        |      }dt        |      D ci c])  }|t        |      d
kD  rdnd ||   t        |      z  d+ c}d|d   |<   Tt        |t               rVdt        t        |      d      t        t        |      t        |      z  d      t        t        |      d      dd|d   |<   t        |t"              rt        |D cg c]  }t        |       c}      }t        |      d
k(  rddt        |       did|d   |<   dt        |       dt        |      t        |      z  ddt        |       ddd|d   |<   Wt%        |      i d|d   |<   m d5d}ddi|d   j'                         D ci c]  \  }}||d    c}}ddi|d   j'                         D ci c]  \  }}| ||d          c}}g}t)        t+        |      j-                  d d!      d"      |d#<   |dd$ |d%<   t        |d%   t#        |d%         d            }g }t/        |      D ]  }i }|j                  D ]  }|d%   |   |   }t        |t"              r"t        |      d&kD  rt	        |dd&       dd' d(z   }t        |t              rt        |      dkD  r|dd d)z   }t	        |      j-                  d*d+      }d| d||<    |j1                  |        t)        t+        |      j-                  d d!      d"      |d,<   d-t%        |      i|d.<   t3        |d/      r>|j5                         }	 t7        j8                  |d01      }t)        d2| d3d"      |d.   d4<   |S c c}w c c}
w c c}w c c}w c c}}w c c}}w # t:        $ r t	        |      }Y Nw xY w)6a  
        Given a dataset, compute the following:
        * Dataset Size
        * Dataset Columns
        * Dataset Stats
            - Strings: min, mean, max word count/token length
            - Integers: Counter() instance
            - Floats: min, mean, max range
            - List: number of elements or min, mean, max number of elements
        * 3 Example samples
        * Loss function name
            - Loss function config
        rN  z<code>z</code>r*  statsNr  r   attention_maskrC   )dimtokens
charactersr	     r   )r   meanmax)r  datar   ~rB  z.2%r   rQ   z	 elementsz.2fri  c                Z    ddj                  d | j                         D              z   dz   S )Nz<ul><li>z	</li><li>c              3  0   K   | ]  \  }}| d |   yw)z: Nr   r  s      r9   r  zaSentenceTransformerModelCardData.compute_dataset_metrics.<locals>.to_html_list.<locals>.<genexpr>  s      4f:3PUuBug5F4fs   z
</li></ul>)r   r|   )ri  s    r9   to_html_listzNSentenceTransformerModelCardData.compute_dataset_metrics.<locals>.to_html_list  s.    !K$4$44fY]YcYcYe4f$ffiuuur:   typer  details-:|--|  stats_tabler  rI  r  r   z, ...]z...
z<br>examples_tabler%   rB   get_config_dictr   r   ```json

```config_code)ri  rP   )rI   r(   rS   column_namesr   r_  rP   r!  tolistr   r   rh  r   r   r   r#  r   rQ   r%   r|   r   r   replacer  rU   rT   rv  jsondumps	TypeError)r7   rb   r   rB   r(  
subsectionfirst	tokenizedr-  suffixr2  counterr   lstrm  r   stats_linesr'  examples_lines
sample_idxr*  config
str_configs                          r9   compute_dataset_metricsz8SentenceTransformerModelCardData.compute_dataset_metricsD  s   & Igw'#&w<L JQJ^J^"_VF87#;"_Y "Wgw'!.. 8[$Ud^F3
"1eS) $j 9I!)T27G97T"+,<"="A"Aa"A"H"O"O"Q!)AK"LX3x="L"L!-!)&+CL!&<%=Qvh#G',S\CL-H!'L&MQvh$W&+CL!&<%=Qvh#G!5L)&1  T{3%j1G!& (.g! #  3w<!+;C#DWS\TWXbTcEcdgDh!ii!5L)&1  u-!(#(Z!#<$)#j/C
O*KQ$O#(Z!#<!5L)&1  t,%:&FCs3x&FGG7|q(%+ &3u:,i(@%9W-f5 &,*-g,y'A+.w<#g,+Fs*K9(U*-g,y'A%9W-f5 ?GuoWY4ZL)&1q8[tv VelSZF[FaFaFcd
UU7^ 3deYuVbcjVkVqVqVs"t
U3U6](C#C"tuK +11D[1Q1Y1YZ_af1gim*nL''.r{L$l:6tL<T7UVW7XYZKN#K0 /
%22 
>F(4V<ZHE!%.3u:> #E"1Is 3h >!%-#e*t2C %etu 4J..tV<E(.ugW&=GFO
> %%g./ .44G4W4_4_`egl4mos-tL)*  
V 4*+))+F)!ZZq9
 399ZLPU9VX\2]L /G #` #M! 'G0  e"t<  ) [
)s5   SS?.S9S
1SS%
S+ +TTc                   |r|rft        |t              rt        |      t        |      k7  st        |t              r/t        |      dk7  r!t        j                  d| d| d| d       g }|s| j                  |      }t        |t              ret        |j                         |j                         |      D cg c].  \  }}}| j                  ||t        |t              r||   n|      0 }}}}n| j                  ||d   |      g}|dk(  r?t        |D cg c]  }|j                  dd       c}      }	|	r| j                  d	|	        | j                  |      S c c}}}w c c}w )
NrC   zThe number of `z?_datasets` in the model card data does not match the number of z1 datasets in the Trainer. Removing the provided `z$_datasets` from the model card data.r   r@   rN  zdataset_size:)rI   r)   rS   r(   r   r   rR  r"  r   rR   r  rP   r!  r   rF   r   )
r7   rb   dataset_metadatarB   dataset_typer  dataset_valuer   metadatanum_training_sampless
             r9   rL   z9SentenceTransformerModelCardData.extract_dataset_metadata  s    G[1c:J6KsSZ|6[w0S9I5Ja5O%l^3rs  sA A..:^;_a $& ##'#6#6w#? ';/ FI(8:JF	$ 	$ Bm\ 00%$.8t.D\*$	$  	$ %)$@$@JZ[\J]_c$d#e  7"#&P`'aHVQ(?'a#b #.B-CDE%%&677'	$ (bs   <3E E'c                    || _         y r3   )r\   )r7   r\   s     r9   register_modelz/SentenceTransformerModelCardData.register_model  s	    
r:   c                    || _         y r3   )r   )r7   r   s     r9   set_model_idz-SentenceTransformerModelCardData.set_model_id  s	     r:   c                    	 t        |      }|j                  | _        ||dk(  r|j                  }|| _        y# t        $ r Y yw xY w)NFmainT)get_model_infor   r   r   shar   )r7   r   rQ  r   s       r9   set_base_modelz/SentenceTransformerModelCardData.set_base_model  sS    	'1J %--x61!~~H#+   		s   9 	AAc                8    t        |t              r|g}|| _        y r3   )rI   r   r   )r7   r   s     r9   set_languagez-SentenceTransformerModelCardData.set_language  s    h$ zH r:   c                    || _         y r3   )r   )r7   r   s     r9   set_licensez,SentenceTransformerModelCardData.set_license  s	    r:   c                    t        |t              r|g}|D ],  }|| j                  vs| j                  j                  |       . y r3   )rI   r   r   rU   )r7   r   tags      r9   rF   z)SentenceTransformerModelCardData.add_tags  s@    dC 6D 	&C$))#		  %	&r:   c           
        t        | j                  d   t              r| j                  d   j                  j                  j
                  }t        |      }dj                  |j                  dd        g}|j                  j                  d      }|t        dt        |            D cg c].  }dj                  |d |       dz   dj                  ||d        z   0 c}z  }|D ]  }| j                  |      s y  y t        | j                  d   t              rC| j                  d   j                  r)| j                  | j                  d   j                         y y y c c}w )Nr   r   r   rC   )rI   r\   r#   
auto_modelr  _name_or_pathr
   r   partsr   r   r  rS   r  r"   r   )r7   r   base_model_pathcandidate_model_idsrU  r.  r   s          r9   try_to_set_base_modelz6SentenceTransformerModelCardData.try_to_set_base_model  s=   djjm[1A1188FFJ":.O $'88O,A,A"#,F#G"H
 %))//4FQVWXZ]^dZeQf$JM&,sxxst/EE$  0 &&x0 

17zz!}''##DJJqM$<$<= ( 8$s   (3Ec                   g }i }g }| j                   j                         D ]^  \  }}t        |dd      t        |dd      }r{t        fd|j	                         D              rY|j                         D ci c]  \  }}|t              dz   d | }}}|r%|j                  dz         r|t              dz   d }d#d}	|j                         D ci c]  \  }}| |	|       }}}|j                         D 
cg c]2  \  }
}|
|k(  rd|
 dn|
|
|k(  rdt        |       dn
t        |      d	4 }}
}|j                  }t        |dd      }d
}t        |d      r:|j                         x}r(	 t        j                  |d      }t        d| dd      }|j!                  t#        |      ||||d       fd|j%                  |j                         D 
cg c]  \  }
} |      x}vt'        ||j)                         j+                  dd      |xs d|r"|j+                  dd      j+                  dd      nd|
j+                  dd      j-                         |
|       c}}
       |j/                  |       a g }|D ]  }|d   D ci c]  }|d   |d    }}t1        |      }|D ]  }t1        d |d   D              }|d   |d   k(  s$||k(  s*|d   |d   k7  s6|d   |d   k(  sB|d   D ]+  }d|v r|j3                  d      ||d   <   ||d      ||d   <   - t5        |d   t6              s	|d   g|d<   |d   j!                  |d            |j!                  |        |D ]/  }t9        |j3                  d            j+                  dd       |d!<   1 |t7        |j	                               t;        | j<                  |      d"S c c}}w c c}}w c c}}
w # t        $ r t        |      }Y rw xY wc c}}
w c c}w )$au  Format the evaluation metrics for the model card.

        The following keys will be returned:
        - eval_metrics: A list of dictionaries containing the class name, description, dataset name, and a markdown table
          This is used to display the evaluation metrics in the model card.
        - metrics: A list of all metric keys. This is used in the model card metadata.
        - model-index: A list of dictionaries containing the task name, task type, dataset type, dataset name, metric name,
          metric type, and metric value. This is used to display the evaluation metrics in the model card metadata.
        r   Nr7  c              3  F   K   | ]  }|j                  d z           yw)r   N)rX  )r  r   r   s     r9   r  zGSentenceTransformerModelCardData.format_eval_metrics.<locals>.<genexpr>4  s     Q3CNN4#:6Qs   !rC   r   c                b    	 t        | d      r| j                         S 	 | S # t        $ r Y | S w xY w)z^Try to convert a value from a Numpy or Torch scalar to pure Python, if not already pure Pythonr  )rT   itemr   r   s    r9   try_to_pure_pythonzPSentenceTransformerModelCardData.format_eval_metrics.<locals>.try_to_pure_python9  sB    ug.$zz|+ /  ! s   ! 	..**)Metricr,   rB  rv  r   r   rw  rx  rr  )
class_namedescriptionr  table_linesry  c                    	 t        |       S # t        $ r Y nw xY wt        | t              rd| v r | j	                         d         S y )Nr   r   )r   r   rI   r   r   )metric_valuetry_to_floats    r9   r  zJSentenceTransformerModelCardData.format_eval_metrics.<locals>.try_to_floatc  sT     ..   lC0SL5H'(:(:(<Q(?@@s   
 	r   -unknownUnknown)r   	task_typer  r  metric_namemetric_typer  r  r  r,   c              3  &   K   | ]	  }|d      yw)r  Nr   )r  lines     r9   r  zGSentenceTransformerModelCardData.format_eval_metrics.<locals>.<genexpr>  s     1pT$x.1ps   r  r  ry  rp  rq  table)eval_metricsr   r   )r   r   r   r   )r   r|   getattrallr   rS   rX  r   r  rT   rv  r}  r~  r  r   r   rU   r%   r$  r   lowerr|  titler   r   r   rI   rQ   r   r   r   )r7   r  all_metricseval_resultsr:  r   r7  r   r   r  
metric_keyr  r  r  r  ry  r  r  metric_value_floatgrouped_eval_metricseval_metricr  eval_metric_mappingeval_metric_metricsgrouped_eval_metricgrouped_eval_metric_metricsr   r  s                             @@r9   format_eval_metricsz4SentenceTransformerModelCardData.format_eval_metrics$  s    "&"8"8">">"@ L	(Iw9fd3D$Y0@$GNQ',,.QQIPY:33s4y1}/6YY!n&?&?s
&K%3CIMO%DN IPX*#us.u55XGX 18 -J 6@>5Q:,b1Wa!^3  "*\":!;2>#L1	K  $//K"9fd;LKy"34ID]D]D_:_&:_-!%F1!=J %yE%BDI"*9"5#.$0#.#.	  5<MMO 1
L.:<.HH*U "-"-"3"3"5"="=c3"G%1%>YYe\%9%9#s%C%K%KCQT%Ukt$.$6$6sC$@$F$F$H$.%7 w'YL	(^  "' 	9KMXYfMg"hT4>4=#@"h"h"%&9":'; 9#.11pM`anMo1p.p+-1D\1RR+/JJ#N37J>7ZZ#M26I-6XX !4M B `"d?HLQXHYD!4^!DE<OPTU]P^<_[89	` &&9.&I4P?RSa?b>c+N;'7>>{>?Z[%9( %++K8/	92 $8 	+>?R?V?VWd?e+f+n+nu,(	 1K,,./6tU
 	
U Z Y" ! -!$VJ-4& #is1   ;OO7O!5O'B
PP'O?>O?c                   g | j                   D ]-  }|j                         D ]  }|vsj                  |        / dfd}t        |      }| j                   D cg c]M  }|D ci c]?  }||d   | j                  k(  rd||v rt        ||         nd dn|j                  |d      A c}O }}}t        |      }|d|v dS c c}w c c}}w )Nc                    | dk(  ry| dk(  ry| dk(  ry| dk(  ry| j                  d	      ry
j                  |       dz   S )Nr   r   r   rC   r   rf  r   r  rB   r   r  )r   index)r   eval_lines_keyss    r9   sort_metricszKSentenceTransformerModelCardData.format_training_logs.<locals>.sort_metrics  sS    g~f}o%''||F#"((-11r:   r  r   r  r  )
eval_linesexplain_bold_in_eval)r   r   r   r   )r   r   rU   r#  r   r   r   r   )	r7   linesr   r  sorted_eval_lines_keysr  r   r  r  s	           @r9   format_training_logsz5SentenceTransformerModelCardData.format_training_logs  s   '' 	0Ezz| 0o-#**3/0	0	2 "(\!J **
  2	  <4#7#77 3$;*T#Y/CHKXXc3'(
 
 )7
$$(J$6
 	

s   "	C+AC	/C	Cc                f   | j                   j                  j                         }dt        |j                        dz  t        |j
                        dd|j                  dk(  |j                  |j                  t        |j                  dz  d      di}|j                  r|j                  |d   d	<   |S )
Nr   r  
codecarbonzfine-tuningYi  r  )	emissionsenergy_consumedr\  training_typeon_cloud	cpu_modelram_total_size
hours_usedhardware_used)rJ   tracker_prepare_emissions_datar   r  r  r  r  r  r   duration	gpu_model)r7   emissions_dataresultss      r9   get_codecarbon_dataz4SentenceTransformerModelCardData.get_codecarbon_data  s    22::RRT">#;#;<tC#()G)G#H&!.*33s:+55"0"?"?#N$;$;d$BAF
!
 ##;I;S;SG&'8r:   c                   d}| j                   j                  r]dddddj                  | j                   j                  | j                   j                  j                  dd      j	                               }| j                   j                         | j                   j                         t        | j                         |dS )	NzCosine SimilarityzDot ProductzEuclidean DistancezManhattan Distance)cosinedot	euclidean	manhattanr   r   )model_max_lengthoutput_dimensionalitymodel_stringsimilarity_fn_name)r\   r  r   r|  r  get_max_seq_length get_sentence_embedding_dimensionr   )r7   r  s     r9   get_model_specific_metadataz<SentenceTransformerModelCardData.get_model_specific_metadata  s    0::((-$11	"
 c$**//1N1N1V1VWZ\_1`1f1f1hi  !%

 = = ?%)ZZ%P%P%R

O"4	
 	
r:   c                   | j                   r| j                  s	 | j                          | j                  sf| j                  r5| j
                  j                  j                   d| j                   | _        n%| j
                  j                  j                  | _        t        |       D ci c]#  }|j                  t        | |j                        % }}| j                  r 	 |j                  | j                                | j                   r 	 |j                  | j#                                t%        | j                         dkD  |d<   | j&                  rU| j&                  j(                  r?| j&                  j(                  j*                  |j                  | j-                                |j                  | j/                                d| _         t0        D ]  }|j3                  |d         |S # t        $ r Y w xY wc c}w # t        $ r}t        j                  d|        |d }~ww xY w# t        $ r#}t        j                  d|        Y d }~(d }~ww xY w)Nz
 based on z+Error while formatting evaluation metrics: z&Error while formatting training logs: d   hide_eval_linesF)r   r   r  r   r   r\   r8   r   r	   r   r  r   r   r  r   r   r   r  rS   rJ   r  _start_timer  r  IGNORED_FIELDSr   )r7   r   
super_dictexcr   s        r9   r{   z(SentenceTransformerModelCardData.to_dict  s   ??4??**,
 %)ZZ%9%9%B%B$C:dooM^"_"&**"6"6"?"?IOPTVejj'$

";;V
V !!!!$":":"<= O!!$";";"=> ),D,>,>(?#(E
$% %%))11))11==Id6689 	$::<=! 	&CNN3%	&W   W  !LSERS	  O!GuMNNOsG   G, )(G< H H, ,	G98G9	H)
H$$H),	I5IIc           	         t        | j                         j                         D ci c]  \  }}|t        v s|d g fvs|| c}}d|      j	                         S c c}}w )NF)	sort_keys
line_break)r   r{   r|   YAML_FIELDSstrip)r7   r   r   r   s       r9   to_yamlz(SentenceTransformerModelCardData.to_yaml   s\    *.,,.*>*>*@sJCC;DV[`imoqhr[rS%Zs!
 %'		ss   A
A
A
)r   r   )T)r   list[dict[str, Any]]r   r   r   r  )r`   zlist[nn.Module]r   r   )r   r   r   r   )rb   Dataset | DatasetDictr   r   )r   r   )
r:  r-   r   r   r   r   r   r   r   r   )rb   r(   r   r   r3   )rb   r  r  r   r   r   )r@  str | list[str]r   r   )rb   z Dataset | IterableDataset | Noner   r   rB   z'dict[str, nn.Module] | nn.Module | Noner   r   )
rb   r  r  r  rB   z nn.Module | dict[str, nn.Module]r  zLiteral['train', 'eval']r   r  )r\   r.   r   r   )r   r   r   r   )r   r   rQ  r   r   r   )r   r  r   r   )r   r   r   r   )r   r  r   r   r   r   )r   z1dict[Literal['co2_eq_emissions'], dict[str, Any]])r   r   )Ar   r   r   __doc__r   rQ   r   __annotations__r   r   r   rM   rO   r   r   r   r   r   rP   r~   r}   r   r   rW   r   r   rJ   r   r   r   r   r   r   r   r   r   r
   __file__parentr   r\   r   r   rV   r   rX   r>  rL  rR  r_  r  rL   r  r  r  r  r  rF   r  r  r  r  r  r{   r  r   r:   r9   r   r      s>   $N (-T'BH$BGZ!J
!Hj+0+FN(F*/*EM'Et s  #
D
  7C3B #4e<J
<&+Du&EE27SX2YY*/5*QQHM^binHoEo,1$U,SM)S#(E#JF J(-d(GO%G/4TPU/V,V6;Du6U3U %d GI~G"'5"AOZA5uMHiM T6J6Re4K4 &;%HL#H&=EJL#J#LuMG^MX(=(=@X(X_deM4e ).dU(SE%S!" KO!#0!#CG!#	!#Fg>$I5X bc*5CLO[^	6
* >)|1| %| 6	|
 
||)8&)8 /)8 /	)8
 /)8 
)8V!
!
&>,
B$
L&
 0dr:   r   c                    t        j                  | j                  | j                  j                  d      }|j                  S )Nu   🤗)	card_datar   hf_emoji)r   from_templaterE   r   content)r\   
model_cards     r9   generate_model_cardr  (  s:    ((''u7L7L7Z7ZekJ r:   r  )r   zfloat | int | strr   r   )r\   r.   r   r   )V
__future__r   r}  loggingr  rS  collectionsr   r   r   dataclassesr   r   r	   pathlibr
   platformr   textwrapr   typingr   r   r   r   r   huggingface_hubr   r   r   r   r   r  huggingface_hub.repocard_datar   r   huggingface_hub.utilsr   r   tqdm.autonotebookr   r   transformers.integrationsr   transformers.modelcardr   transformers.trainer_callbackr   r   typing_extensionsr   r   r!   r   sentence_transformers.modelsr"   r#   #sentence_transformers.training_argsr$   sentence_transformers.utilr%   r&   r'   r   r(   r)   r*   r+   r,   	getLoggerr   r   2sentence_transformers.evaluation.SentenceEvaluatorr-   )sentence_transformers.SentenceTransformerr.   sentence_transformers.trainerr/   r1   r   r  r  r   r   r   r  r   r:   r9   <module>r*     s   "    	 ,  0 0  #  . .   / < 8 Q +  " ( 8 6 F ( N E T _ _ZZ			8	$TMHV? Vr  v*< **
 ;, ix i iXr:   