+#hkddlZddlmZmZddlZddlZGddejZGddeZ GddeZ Gd d eZ e ejjddk(ZGd d eZy#e$rd ZYwxYw)N)ListOptionalczeZdZdZdZd dZejdedde jfdZ dedee fd Z y) BaseSentenceVectorizera[ Base Class for Vectorizers. The main purpose is to vectorize text (doc/query) for ANN/KNN indexes. `__call__` method takes `List[Example]` as a single input, then extracts `field_to_vectorize` from every Example and convert them into embeddings. You can customize extraction logic in the `_extract_text_from_examples` method. text_to_vectorizereturnNcyN)selfs \/var/www/html/sandstorm/venv/lib/python3.12/site-packages/dsp/modules/sentence_vectorizer.py__init__zBaseSentenceVectorizer.__init__s inp_examplesExamplecyr r )r rs r __call__zBaseSentenceVectorizer.__call__s rc t|dtr|S|Dcgc].}dj|jDcgc]}|| c}0c}}Scc}wcc}}w)Nr ) isinstancestrjoin _input_keys)r rexamplekeys r _extract_text_from_examplesz2BaseSentenceVectorizer._extract_text_from_examplessK l1os + WcdG'2E2EF3'#,FGddFdsA A AA)rN)__name__ __module__ __qualname____doc__field_to_vectorizerabcabstractmethodrnpndarrayrrrr rr rrs^-   T)_   eecerrc VeZdZdZ d dedededefdZdede jfd Z y ) SentenceTransformersVectorizerz Vectorizer based on `SentenceTransformers` models. You can pick any model from this link: https://huggingface.co/models?library=sentence-transformers More details about models: https://www.sbert.net/docs/pretrained_models.html model_name_or_path vectorize_bsmax_gpu_devicesnormalize_embeddingsc ddlm}ddlm}||\|_|_|j rdnd|_|||j|_||_ ||_ ||_ y#t$r tdwxYw)Nr)SentenceTransformerzYou need to install sentence_transformers library to use pretrained embedders. Please check the official doc https://www.sbert.net/ or simply run `pip install sentence-transformers)determine_devicescudacpu)device) sentence_transformersr- ImportErrordsp.utils.ann_utilsr. num_devicesis_gpu proxy_devicemodelr(r)r+)r r(r)r*r+r-r.s r rz'SentenceTransformersVectorizer.__init__&s  A :(9/(J%$+&*kkFu();DDUDUV "4($8! C  s A''A<rrc|j|}|jr|jdkDrtt |j}|j j |}|j j|||j}|j j||jr"|tjj|z }|S|j j||j|j}|S)N)target_devices) sentencespool batch_size)r<r>r+)rr6r5listranger8start_multi_process_poolencode_multi_processr)stop_multi_process_poolr+r$linalgnormencode)r rrr;r=embs r rz'SentenceTransformersVectorizer.__call__Ds <<\J ;;4++a/!%(8(8"9:N::66n6UD**11+,,2C JJ . .t 4((BIINN3//J**##+,,%)%>%>$C JrN)zall-MiniLM-L6-v2r:F) rrrr rintboolrrr$r%rr rr r'r'sW#5 %* 999 9 # 9<Tbjjrr'cHeZdZdZd defdZdeddejfdZ y) NaiveGetFieldVectorizerz If embeddings were precomputed, then we could just extract them from the proper field (set by `field_with_embedding`) from each `Example`. field_with_embeddingc||_yr )rM)r rMs r rz NaiveGetFieldVectorizer.__init__ds $8!rrrrc|Dcgc](}t||jjdd*}}tj|dj tj }|Scc}w)Nr:r)axis)getattrrMreshaper$ concatenateastypefloat32)r r cur_example embeddingss r rz NaiveGetFieldVectorizer.__call__gsg ,  K!:!: ; C CAr J  ^^JQ7>>rzzJ   s-A*N) vectorized) rrrr rrrr$r%rr rr rLrL_s.9S9T)_rrLc ZeZdZdZ d dedededefdZdedd ejfd Z y ) CohereVectorizera  This vectorizer uses the Cohere API to convert texts to embeddings. More about the available models: https://docs.cohere.com/reference/embed `api_key` should be passed as an argument and can be retrieved from https://dashboard.cohere.com/api-keys api_keyr8embed_batch_sizeembedding_typecb||_||_||_ddl}|j ||_y)Nr)r8r]r^cohereClientclient)r r\r8r]r^r`s r rzCohereVectorizer.__init__ws. 0,mmG, rrrrc|j|}g}t|dz |jzdz}t|D]u}||jz}|dz|jz}|||}|jj ||j |j} |j| jwtj|tj} | S)Nr:)textsr8 input_typedtype) rlenr]r@rbembedr8r^extendrXr$arrayrV) r rrembeddings_list n_batches cur_batch_idx start_idxend_idx cur_batchresponserXs r rzCohereVectorizer.__call__s <<\J*+a/D4I4IIAM "9- 8M%(=(==I$q(D,A,AAG))W=I{{((jj..)H  " "8#6#6 7 8XXoRZZ@ rN)zembed-english-v3.0`search_document) rrrr rrIrrr$r%rr rr r[r[psX* "/ - - - -  -T)_rr[Tc\eZdZdZ d dededeefdZdedd e jfd Z y) OpenAIVectorizera This vectorizer uses OpenAI API to convert texts to embeddings. Changing `model` is not recommended. More about the model: https://openai.com/blog/new-and-improved-embedding-model/ `api_key` should be passed as an argument or as env variable (`OPENAI_API_KEY`). Nr8r]r\c||_||_trtj|_ntj |_|r |t_yyr )r8r] OPENAI_LEGACYopenai EmbeddingrXr\)r r8r]r\s r rzOpenAIVectorizer.__init__s?  0 #--DN#..DN $FN rrrrc|j|}g}t|dz |jzdz}t|D]u}||jz}|dz|jz}|||}|jj |j |} | dD cgc]} | d } } |j| wtj|tj} | Scc} w)Nr:)r8inputdata embeddingrf) rrhr]r@rzcreater8rjr$rkrV) r rrrlrmrnrorprqrrcur_objcur_batch_embeddingsrXs r rzOpenAIVectorizer.__call__s <<\J*+a/D4I4IIAM "9- 9M%(=(==I$q(D,A,AAG))W=I~~,,jj-H IQQWHX#YWGK$8#Y #Y  " "#7 8 9XXoRZZ@  $Zs C)ztext-embedding-ada-002iN) rrrr rrIrrrr$r%rr rr rvrvsR. $!% %%%# %"T)_rrv)r"typingrrnumpyr$ryABCrr'rLr[rIversion __version__rx Exceptionrvr rr rs ! eSWWe.=%;=@4")-)X22156!;M +-+ Ms "A::BB