IgUddlmZddlZddlZddlZddlZddlmZddlm Z m Z m Z m Z m Z mZmZmZmZmZmZmZmZddlZddlmZddlmZddlmZddlmZe rddl Z ed Z!e"ee!Z#d e$d <ed Z%e"ee%Z&d e$d<dZ'dZ(dZ)e)dzZ*gdZ+gdZ,ddigZ-ej\e/Z0d*dZ1d+dZ2d,dZ3d-d.dZ4GddeZ5 d/ d0dZ6 d1 d2dZ7 d3dZ8dgdf d4d Z9d5d!Z:d6d"Z;d7d#Zd:d&Z? d; dd)ZBy)?) annotationsN)deepcopy) TYPE_CHECKINGAnyCallableDictIterableListLiteralOptionalSizedTupleTypeUnionget_args)Document) Embeddings) VectorStore)maximal_marginal_relevance)L2IPzList[DISTANCE_METRICS]AVAILABLE_DISTANCE_METRICS) TileDBDense TileDBSparse FaissFlat FaissIVFFlatFlinngz List[ENGINES]AVAILABLE_ENGINES langchain ) _distanceidcontent)r#r%blobzMissing propertyct|trUt|trEt|t|k7r.t|d|d|dt|d|dt| y)z Check that sizes of two variables are the same Args: x: Variable to compare y: Variable to compare x_name: Name for variable x y_name: Name for variable y z and z% expected to be equal length but len(z)=z and len(N) isinstancer len ValueError)xyx_namey_names b/var/www/html/answerous/venv/lib/python3.12/site-packages/langchain_community/vectorstores/vdms.py_len_check_if_sizedr0>sm!U 1e 4Q3q69IheF8$("SVHIfXRAx A   cJt|Dcgc]\}}| c}}Scc}}wN)_results_to_docs_and_scores)resultsdoc_s r/_results_to_docsr8Ps9'B CFCC CC Cs cg} |d\}}t|dkDrd|dvrd|ddvr|ddd}|D]r}t|dd}|d}tD] }||vs||= |jD cic]\} } | tvr| | } } } |j t || |ft|Scc} } w#t$r#} tjd| Yd} ~ |Sd} ~ wwxYw) NrFindDescriptorentitiesr# r%) page_contentmetadataz2No results returned. Error while parsing results: ) r)roundINVALID_DOC_METADATA_KEYSitemsINVALID_METADATA_VALUEappendr Exceptionloggerwarning) r5 final_res responsesblobsresult_entitiesentdistance txt_contentspmkeymvalpropses r/r4r4Ts2IQ"1: 5  NQ  IaL0il+;<<'l+;CC!c ddl}|j}|j|||S#t$r tdwxYw)zVDMS client for the VDMS server. Args: host: IP or hostname of VDMS server port: Port to connect to VDMS server rNzOCould not import vdms python package. Please install it with `pip install vdms.)vdms ImportErrorconnect)hostportrTclients r/ VDMS_ClientrZssO TYY[F NN4 M   8   s +AcDeZdZdZdeddddd d3dZed4dZd5dZd6d Z d7d Z d8d Z d9d Z e edf d:d Z d; ddZ d; d?dZ d@ dAdZ dB dCdZgdf dDdZ dE dFdZ dGdZddedf dHdZ dI dJdZddef dKdZdef dLdZ dMdZdNdZdOdZ dP dQd Z dR dSd!Z!e eddddf dTd"Z"dUd#Z#e$ddeef dVd$Z%e$dddeef dWd%Z&dddgf dXd&Z'e ed'df dYd(Z(e ed'df dZd)Z)e ed'df d[d*Z*e ed'df d\d+Z+dde edddf d]d,Z,e edf d^d-Z-e edf d_d.Z.e edf d`d/Z/e edf dad0Z0 dbd1Z1 dcd2Z2y)dVDMSaIntel Lab's VDMS for vector-store workloads. To use, you should have both: - the ``vdms`` python package installed - a host (str) and port (int) associated with a deployed VDMS Server Visit https://github.com/IntelLabs/vdms/wiki more information. IT IS HIGHLY SUGGESTED TO NORMALIZE YOUR DATA. Args: client: VDMS Client used to connect to VDMS server collection_name: Name of data collection [Default: langchain] distance_strategy: Method used to calculate distances. VDMS supports "L2" (euclidean distance) or "IP" (inner product) [Default: L2] engine: Underlying implementation for indexing and computing distances. VDMS supports TileDBDense, TileDBSparse, FaissFlat, FaissIVFFlat, and Flinng [Default: FaissFlat] embedding: Any embedding function implementing `langchain_core.embeddings.Embeddings` interface. relevance_score_fn: Function for obtaining relevance score Example: .. code-block:: python from langchain_huggingface import HuggingFaceEmbeddings from langchain_community.vectorstores.vdms import VDMS, VDMS_Client model_name = "sentence-transformers/all-mpnet-base-v2" vectorstore = VDMS( client=VDMS_Client("localhost", 55555), embedding=HuggingFaceEmbeddings(model_name=model_name), collection_name="langchain-demo", distance_strategy="L2", engine="FaissFlat", ) Nrr) embeddingcollection_namedistance_strategyenginerelevance_score_fnembedding_dimensionsc||_||_||_||_|j ||||_|j ||j|j|_y)Nr`metric)_clientsimilarity_search_enginer_r]_check_required_inputsoverride_relevance_score_fnadd_set_collection_name)selfrYr]r^r_r`rarbs r/__init__z VDMS.__init__sj (.%!2" ##O5IJ,>(!% 00))!-! r1c|jSr3)r]rls r/ embeddingszVDMS.embeddingss ~~r1ct|jtr|jj|Sd}|dz }t |)Nz*Must provide `embedding` which is expectedz to be an Embeddings object)r(r]rembed_documentsr*)rltextsp_strs r/_embed_documentszVDMS._embed_documentss@ dnnj 1>>11%8 8@E 2 2EU# #r1c |j4t|jdr|jjdd|i|Std)N embed_videopathsz:Must provide `embedding` which has attribute `embed_video`)r]hasattrrwr*)rlrxkwargss r/ _embed_videozVDMS._embed_videosH >> %'$..-*P-4>>--DEDVD DL r1c|j2t|jdr|jj|Std)N embed_imageurisz:Must provide `embedding` which has attribute `embed_image`)r]rzr~r*)rlrs r/ _embed_imagezVDMS._embed_imagesA >> %'$..-*P>>--4-8 8L r1ct|jtr|jj|St d)NzEMust provide `embedding` which is expected to be an Embeddings object)r(r]r embed_queryr*)rltexts r/ _embed_queryzVDMS._embed_querys7 dnnj 1>>--d3 3. r1c|j |jS|jjdvrdStd|jd)a8 The 'correct' relevance function may differ depending on a few things, including: - the distance / similarity metric used by the VectorStore - the scale of your embeddings (OpenAI's are unit normed. Many others are not!) - embedding dimensionality - etc. )ipl2c|Sr3ry)r+s r/z1VDMS._select_relevance_score_fn..sQr1z=No supported normalization function for distance_strategy of z;.Consider providing relevance_score_fn to VDMS constructor.)rir_lowerr*ros r/_select_relevance_score_fnzVDMS._select_relevance_score_fnsf  + + 733 3  ! ! ' ' )\ 9 --1-C-C,DEMM r1c |jd|d<|jd||||d|}g}|D]G\}} |j|j|| f&|j||j| fI|S)z?Return docs and their similarity scores on a scale from 0 to 1.Tnormalize_distance)querykfetch_kfilterry)risimilarity_search_with_scorerC) rlrrrrr{docs_and_scoresdocs_and_rel_scoresr6scores r/(_similarity_search_with_relevance_scoresz-VDMS._similarity_search_with_relevance_scoress  + + 3+/F' (;$;;     *,) JC//7#**C<8#**$::5AB  #"r1ct||dd||n|Dcgc]}dc}}t||dd||n*|Dcgc]}ttj!c}}t||ddg}g}g} t ||||D]V\} } } } |j || | | | \}}|$|j ||j || j | X|j||\}}| Scc}wcc}w)Nrsrp metadatasidsr>r]documentr$)r0struuiduuid4zip_VDMS__get_add_queryrC_VDMS__run_vdms_query)rlr^rsrprrr7 all_queries all_blobs inserted_idsmetaembr6r$rr&responseresponse_arrays r/addzVDMS.add#s E:w E!*!6I5c |i}ntd|i}t|j||\}}|rT|ddjD cic] \} } | | d } } } d|d} | dz } t| td| |dfS|r|j ||d vr||d <|j D],} | |j vs|j j| .td |dd|dddd }t|}||fScc} } w) Nr$r:rz[!] Embedding with id (z) exists in DB;z#Therefore, skipped and not insertedz Skipped values are: )Nr% AddDescriptorr) _check_descriptor_exists_by_idrfrAprintupdatekeysrrCrembedding2bytes)rlr^r>r]rr$rQ id_existsrprop_keyprop_val skipped_valuepstrrr&s r/__get_add_queryzVDMS.__get_add_querysV :$&E2JE= or  Iu/44D.E%/eg !*(hrl*! ! 1OD==d .}o>?d{"  LL " : %'E)  5A222**11!4 5    y)    G!sC8Fct|||}|j|g\}}t|dkDrt|dj d}|St t }|S)N) unique_entitydeletionr,)_find_property_entityrr) _bytes2strsplitrDEFAULT_PROPERTIES)rlr^rr find_queryr response_blobrs r/__get_propertieszVDMS.__get_propertiessr + =8 #'"7"7 "E- }  !$.}Q/?$@$F$Fs$K !%$%--?$@ !$$r1c|jj||\}}t||}|r|jj||fSr3)rfr_check_valid_responseprint_last_response)rlrrrrrr7s r/__run_vdms_queryzVDMS.__run_vdms_querysI $(<<#5#5k9#M . !+x 8  LL , , .''r1c .t||ddt||dd||n|Dcgc]}dc}}t||dd|j|}g}t||||D]\} } } } d|ji} ddd gi}| d| g|d <t d |ddddd|| }|j |g\}}|j || | | | \}}|j|j |g|g\}}|j| |j|||jycc}w)z Updates (find, delete, add) a collection based on id. If more than one collection returned with id, error occuers r documentsrpNrrrrrr$r:rr) r0rrrrrrrC_VDMS__update_properties)rlr^rrrprr7 orig_props updated_idsrrr6r$r5rrrrr&s r/__updatez VDMS.__updatesj CE;?CULA!*!6I3 ?-0II(=#!)#@)% X #33K(L ! J 1r1Tc |Dcgc]}|j|}}|r|rt|D] \} }||| d<n|rg}|D]}|jd|i||n*|D cgc]} tt j !c} }|j |} ||D cgc]} i}} n|D cgc] } t| }} |jd|| |||d||Scc}wcc} wcc} wcc} w)aRun more images through the embeddings and add to the vectorstore. Images are added as embeddings (AddDescriptor) instead of separate entity (AddImage) within VDMS to leverage similarity search capability Args: uris: List of paths to the images to add to the vectorstore. metadatas: Optional list of metadatas associated with the images. ids: Optional list of unique IDs. batch_size (int): Number of concurrent requests to send to the server. add_path: Bool to add image path as metadata Returns: List of ids from adding images into the vectorstore. ) image_pathrrrsrprr batch_sizery) encode_image enumeraterCrrrr_validate_vdms_propertiesadd_from) rlrrrradd_pathr{uri b64_textsmidxr7rpms r/ add_imageszVDMS.add_images*s,2CGG3T&&#&6G G  &t_ 4 c03 $ - 4 I 6  ,!45 6_cd*K3tzz|+<*K&&D&1  %)**I*?HI!215III  !!     ;H+L +IsC#&$C(% C-5C2c ||Dcgc]}d}}|r|rt|D] \} } | || d<n|rg}|D]} |jd| i||n*|Dcgc]}ttj!c}}|j dd|i|} ||Dcgc]}i}}|j d|| |||d||Scc}wcc}wcc}w)aRun videos through the embeddings and add to the vectorstore. Videos are added as embeddings (AddDescriptor) instead of separate entity (AddVideo) within VDMS to leverage similarity search capability Args: paths: List of paths to the videos to add to the vectorstore. metadatas: Optional list of text associated with the videos. metadatas: Optional list of metadatas associated with the videos. ids: Optional list of unique IDs. batch_size (int): Number of concurrent requests to send to the server. add_path: Bool to add video path as metadata Returns: List of ids from adding videos into the vectorstore. r video_pathrxrry)rrCrrrr|r) rlrxrsrrrrr{r7rpathrps r/ add_videoszVDMS.add_videosbs 4 =!&'AR'E'  '. 5 d04 $ - 5 I 7  ,!56 7_ce*L3tzz|+<*L'T&&=U=f=  %*++I+  !!     7(+M ,s B>$C Cc 8t|}|*|Dcgc]}ttj!}}|j |}||Dcgc]}i}}n|Dcgc] }t |}}|j d|||||d|} | Scc}wcc}wcc}w)aRun more texts through the embeddings and add to the vectorstore. Args: texts: List of strings to add to the vectorstore. metadatas: Optional list of metadatas associated with the texts. ids: Optional list of unique IDs. batch_size (int): Number of concurrent requests to send to the server. Returns: List of ids from adding the texts into the vectorstore. rry)rrrrrurr) rlrsrrrr{r7rprrs r/ add_textszVDMS.add_textss(U  ;.343tzz|$4C4**51  %*++I+?HI!215III$t}} !!    #5 ,Is$B  BBc |j|j}g}tdt||D]a} t | |zt|} || | } || | } || | } |r|| | }|j |j| | | }|j |c|j|j||j|S)Nr)rprsrr) rrkranger)minrextendrr)rlrsrprrrr{rr start_idxend_idx batch_textsbatch_embedding_vectors batch_idsbatch_metadatasresults r/rz VDMS.add_froms**4+@+@A "$ q#e*j9 (I)j0#e*=G '2K&07&C #Ig.I"+Ig">XX%%2!) F    '# ((    ! !:t/I/I r1cL|jjs td|jtvr td|j t vr td|j td|||_n|j6t|jdr t|jd|_n|j~t|jdst|jdrRt|jd r1 |jjjj|_n td |j|}t|d r|j j#|y||_y#t$r td wxYw) Nz_VDMS client must be connected to a VDMS server.Please use VDMS_Client to establish a connectionz-distance_strategy must be either 'L2' or 'IP'z]engine must be either 'TileDBDense', 'TileDBSparse', 'FaissFlat', 'FaissIVFFlat', or 'Flinng'Must provide embedding functionrzThis is a sample sentence.r~rwmodelz>Embedding dimension needed. Please define embedding_dimensionsr)rf is_connectedr*r_rrgrr]rrzr)rrtoken_embedding embedding_dimrrr )rlr^rb current_propss r/rhzVDMS._check_required_inputss||((*E   ! !)C CLM M  ( (0A A=  >> !>? ?  +';D $ ^^ 'GDNNM,R'*!!">?(D $^^ ' DNNM 2t~~}5t~~w/,,<<JJ,!T --o> 40 1  & & - -m <4AD &"$Xs /FF#c g}g}ddgd}td|dddddd| }|j||j||\}}|dddS)Nrr$)countrr:rrreturned)rrCr)rlr^rrr5rrrs r/rz VDMS.count sy!# ! /     5!#'#8#8i#P .{+,Z88r1c,tj|Sr3)base64 b64decode)rl base64_images r/ decode_imagezVDMS.decode_image6s --r1c J||n |j}|j|||S)zDelete by ID. These are the IDs in the vectorstore. Args: ids: List of ids to delete. Returns: Optional[bool]: True if deletion is successful, False otherwise, None if not implemented. )rr)rk _VDMS__delete)rlrr^rr{names r/deletez VDMS.delete9s+ #2"=4CXCX}}Ts }DDr1cd}d}t||||}|j|g|\} } |r|| dvr| d|ddd}| | |fS)Nrr:)rr5rr;rr#)rr) rlsetnamerr5r normalizemax_dist command_strrrrs r/get_k_candidateszVDMS.get_k_candidatesLsw&      $(#8#8%)#L .  3{;/ ;B? LH11r1c g} t|} | | j| ||j|||| |\} } } n5|ddgi}n&d|vrdg|d<nd|dvr|djdt||||}|j |g\} } || dvr*| d|ddkDr| d|dDcgc]}|d }}nggfS|j|||| |\} } } || dvs|| dvr| d|ddk(rggfSg}| d|dD]*}|d|vr|j|t ||k(s*n|| d|d<t || d|d<t ||kr d}t ||rS| dtjfvrd n| } t| d|dD]$\}}|d | z |d <|d | d|d|d <&| | fScc}w) N)r&rr$rr5rrr;z4Returned items < k_neighbors; Try increasing fetch_kg?r#) rrCr)rrr)rnpinfr)rlr(r%rrrr5query_embeddingrrr&rrr'rrKids_of_interest new_entitiesrtent_idxs r/get_descriptor_responsezVDMS.get_descriptor_responsecs " /     T "  151F1FgyDV2G2 .Hnh!D6*w&#'&WV_,&&t,$' E (,'<'K MPN K(4W=kJ  ''I#s3 G ct|d5}|j}tj|j dcdddS#1swYyxYw)Nrbzutf-8)openreadr b64encodedecode)rlrfr&s r/rzVDMS.encode_imagesG *d # :q668D##D)009 : : :s 4A  Ac |d}|j||Dcgc]}|jc}|Dcgc]}|jc}||||Scc}wcc}w)aCreate a VDMS vectorstore from a list of documents. Args: collection_name (str): Name of the collection to create. documents (List[Document]): List of documents to add to vectorstore. embedding (Embeddings): Embedding function. Defaults to None. ids (Optional[List[str]]): List of document IDs. Defaults to None. batch_size (int): Number of concurrent requests to send to the server. Returns: VDMS: VDMS vectorstore. rY)rYrsrr]rrr^) from_textsr=r>) clsrr]rrr^r{rYr6s r/from_documentszVDMS.from_documentss_,#8,~~/893##9/89s||9!+  99s A A c |d}||||} |*|D cgc]} ttj!}} | j||||| Scc} w)aHCreate a VDMS vectorstore from a raw documents. Args: texts (List[str]): List of texts to add to the collection. embedding (Embeddings): Embedding function. Defaults to None. metadatas (Optional[List[dict]]): List of metadatas. Defaults to None. ids (Optional[List[str]]): List of document IDs. Defaults to None. batch_size (int): Number of concurrent requests to send to the server. collection_name (str): Name of the collection to create. Returns: VDMS: VDMS vectorstore. rY)r^r]rY)rsrrr)rrrr) r<rsr]rrrr^r{rYvdms_collectionr7s r/r;zVDMS.from_textssr0#8,+  ;.343tzz|$4C4!!! " 5s$Acg}g}ddi}|||d<d|vr|j|}||d<d|vrd|d <td |d|| } |j| |j||\} } | | fS) aGets the collection. Get embeddings and their associated data from the data store. If no constraints provided returns all embeddings up to limit. Args: constraints: A dict used to filter results by. E.g. `{"color" : ["==", "red"], "price": [">", 4.00]}`. Optional. limit: The number of documents to return. Optional. include: A list of what to include in the results. Can contain `"embeddings"`, `"metadatas"`, `"documents"`. Ids are always included. Defaults to `["metadatas", "documents"]`. Optional. rrNlimitr>rrpTr&r:)rrr5)rrrCr) rlr^rrAincluderrr5rrrrs r/getzVDMS.gets("$ ! #*B-  $GG   $($9$9/$J !3GFO 7 ""GFO  #   5!#'#8#8i#P .''r1g?c T|j tdtjj |s(t |jdr|j |}ntjj |r-t |jdr|j|gd}ngtjj |r-t |jdr|j|gd}nd|d }|d z }|d z }t||j||||| } | S aRReturn docs selected using the maximal marginal relevance. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents. Args: query (str): Query to look up. Text or path for image or video. k: Number of Documents to return. Defaults to 4. fetch_k: Number of Documents to fetch to pass to MMR algorithm. lambda_mult: Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. Defaults to 0.5. filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. Returns: List of Documents selected by maximal marginal relevance. zBFor MMR search, you must specify an embedding function oncreation.rr~rrrwrx(Could not generate embedding for query ''.9If using path for image or video, verify embedding model 6has callable functions 'embed_image' or 'embed_video'.) lambda_multr) r]r*osrisfilerzrrr|'max_marginal_relevance_search_by_vector) rlrrrrKrr{embedding_vector error_msgdocss r/max_marginal_relevance_searchz"VDMS.max_marginal_relevance_search2s4 >> !W  ww~~e$)O#007  WW^^E "wt~~}'M#00ug0>qA  WW^^E "wt~~}'M#00w0?B B5'LI T TI Q QIY' ';;  # <  r1c |j|g||gd}t|dddk(rgS|ddDcgc]}tt|} }t t j |t j| ||} t|} t| D cgc] \} } | | vs | }} } |Scc}wcc} } waHReturn docs selected using the maximal marginal relevance. Maximal marginal relevance optimizes for similarity to query AND diversity among selected documents. Args: embedding: Embedding to look up documents similar to. k: Number of Documents to return. Defaults to 4. fetch_k: Number of Documents to fetch to pass to MMR algorithm. lambda_mult: Number between 0 and 1 that determines the degree of diversity among the results with 0 corresponding to maximum diversity and 1 to minimum diversity. Defaults to 0.5. filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. Returns: List of Documents selected by maximal marginal relevance. )rr distancesrp)query_embeddings n_resultsrrBrrdtype)rrK) query_collection_embeddingsr)r_bytes2embeddingrr,arrayfloat32r8r)rlr]rrrKrr{r5rembedding_list mmr_selected candidatesirselected_resultss r/rNz,VDMS.max_marginal_relevance_search_by_vectorhs422'[I 3  wqz!}  "I>EQZ]39%f-.N6"**5' L*'2J( 3 aqL7H   $ #! sB7! B<.B<c T|j tdtjj |s(t |jdr|j |}ntjj |r-t |jdr|j|gd}ngtjj |r-t |jdr|j|gd}nd|d }|d z }|d z }t||j||||| } | SrE) r]r*rLrrMrzrrr|2max_marginal_relevance_search_with_score_by_vector) rlrrrrKrr{r]rPrQs r/(max_marginal_relevance_search_with_scorez-VDMS.max_marginal_relevance_search_with_scores4 >> !W ww~~e$)O))%0I WW^^E "wt~~}'M))w)7:I WW^^E "wt~~}'M)))8;IB5'LI T TI Q QIY' 'FF  # G  r1c |j|g||gd}t|dddk(rgS|ddDcgc]}tt|} }t t j |t j| ||} t|} t| D cgc]\} \} }| | vs| |f}} } }|Scc}wcc}} } wrT) rZr)rr[rr,r\r]r4r)rlr]rrrKrr{r5rr^r_r`rarbsrcs r/rez7VDMS.max_marginal_relevance_search_with_score_by_vectors422'[I 3  wqz!}  "I>EQZ]39%f-.N6"**5' L5W=J)2*(=  $1fq!lARA   $ #! sB>"C2Cc g} | |j}|| S|jddg} |d| vr|jd| vd}|D]1} |jd||||||| \} } | j | | g3| S)NrBrrp)rr&r:)rrrr5rr.)rkrCrr2rC)rlrVr^rWrrr5rr{ all_responsesrBqembrrs r/rZz VDMS.query_collection_embeddings s$&  ""33O  # **Y 6 ?{g522$/G % =D'+'C'C %"#5 $(D ( $Hn  (N!; < =r1c f|j|f|||d|}|Dcgc]\}}| c}}Scc}}w)aRun similarity search with VDMS. Args: query (str): Query to look up. Text or path for image or video. k (int): Number of results to return. Defaults to 3. fetch_k (int): Number of candidates to fetch for knn (>= k). filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. Returns: List[Document]: List of documents most similar to the query text. )rrr)r) rlrrrrr{rr6r7s r/similarity_searchzVDMS.similarity_search3sG&<$;;   :@ #22Q222s -c H|jd|g|||d|}t|S)aReturn docs most similar to embedding vector. Args: embedding (List[float]): Embedding to look up documents similar to. k (int): Number of Documents to return. Defaults to 3. fetch_k (int): Number of candidates to fetch for knn (>= k). filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. Returns: List of Documents most similar to the query vector. rVrWrrry)rZr8rlr]rrrr{r5s r/similarity_search_by_vectorz VDMS.similarity_search_by_vectorKs@"3$22 '[     ((r1c j|j tdtjj |s(t |jdr|j |}ntjj |r-t |jdr|j|gd}ngtjj |r-t |jdr|j|gd}nd|d }|d z }|d z }t||jd |g|||d |}t|S)aERun similarity search with VDMS with distance. Args: query (str): Query to look up. Text or path for image or video. k (int): Number of results to return. Defaults to 3. fetch_k (int): Number of candidates to fetch for knn (>= k). filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. Returns: List[Tuple[Document, float]]: List of documents most similar to the query text and cosine distance in float for each. Lower score represents more similarity. rrr~rrrwrFrGrHrIrJrory) r]r*rLrrMrzrrr|rZr4) rlrrrrr{r.rPr5s r/rz!VDMS.similarity_search_with_scorefs!* >> !>? ?77>>%(WT^^]-S/3/@/@/G&74>>=+Q"&"3"3%"3"A!"D&74>>=+Q"&"3"35'"3"B1"EFugRP XX UU  ++6d66"1!2   G+733r1c H|jd|g|||d|}t|S)a6 Return docs most similar to embedding vector and similarity score. Args: embedding (List[float]): Embedding to look up documents similar to. k (int): Number of Documents to return. Defaults to 3. fetch_k (int): Number of candidates to fetch for knn (>= k). filter (Optional[Dict[str, str]]): Filter by metadata. Defaults to None. Returns: List[Tuple[Document, float]]: List of documents most similar to the query text. Lower score represents more similarity. rory)rZr4rps r/&similarity_search_with_score_by_vectorz+VDMS.similarity_search_with_score_by_vectors@03$22 '[    +733r1c,|j||g|gS)zUpdate a document in the collection. Args: document_id (str): ID of the document to update. document (Document): Document to update. )update_documents)rlr^ document_idrs r/update_documentzVDMS.update_documents$$_{mhZPPr1c|Dcgc]}|j}}|Dcgc]}t|j}}|j|}|j |||||ycc}wcc}w)zUpdate a document in the collection. Args: ids (List[str]): List of ids of the document to update. documents (List[Document]): List of documents to update. )rrprN)r=rr>ru _VDMS__update)rlr^rrrrr>rps r/rvzVDMS.update_documentss7@@(%%@@IR =E %h&7&7 8  **40   !  A s A$A))rY vdms.vdmsr]Optional[Embeddings]r^rr_DISTANCE_METRICSr`ENGINESraz"Optional[Callable[[float], float]]rb Optional[int]returnNone)rr|)rs List[str]rList[List[float]])rxrr{rrr)rrrr)rrr List[float])rzCallable[[float], float]) rrrintrrrOptional[Dict[str, Any]]r{rrList[Tuple[Document, float]])NN) r^rrsrrprr1Optional[Union[List[None], List[Dict[str, Any]]]]rOptional[List[str]]rr )rr)r^rr`r~rer}rr)r^rrzUnion[None, List[str]]rUnion[None, Dict[str, Any]]rbool)NNNN) r^rr> Optional[Any]r]Union[List[float], None]rrr$ Optional[str]rz4Tuple[Dict[str, Dict[str, Any]], Union[bytes, None]]FF)r^rrOptional[bool]rrrr)rz List[Dict]rOptional[List]rrrTuple[Any, Any]r3) r^rrrrrrprrrrr)r^rrr rrrr)rrrOptional[List[dict]]rrrrrrr{rrr)NNNrT)rxrrsrrrrrrrrrr{rrr) rsz Iterable[str]rrrrrrr{rrr)rsrrprrrrrrrr{rrr)r^rrbzUnion[int, None]rr)r^rrr)rrrbytes)NNN) rrr^rrOptional[Dict]r{rrr)NNF) r%rrrr5rrrr&rrz(Tuple[List[Dict[str, Any]], List, float])r(rr%rrrrrrOptional[dict]r5rr.zOptional[List[float]]rrrz!Tuple[List[Dict[str, Any]], List])rrrr)r< Type[VDMS]rList[Document]r]r|rrrrr^rr{rrr\)r<rrsrr]r|rrrrrrr^rr{rrr\) r^rrrrArrBrrr)rrrrrrrKfloatrOptional[Dict[str, List]]r{rrr)r]rrrrrrKrrrr{rrr)rrrrrrrKrrrr{rrr)r]rrrrrrKrrrr{rrr)rVzOptional[List[List[float]]]r^rrWrrrrrr5rrrr{rrz!List[Tuple[Dict[str, Any], List]]) rrrrrrrrr{rrr) r]rrrrrrrr{rrr) rrrrrrrrr{rrr) r]rrrrrrrr{rrr)r^rrwrrrrr)r^rrrrrrr)3__name__ __module__ __qualname____doc__DEFAULT_COLLECTION_NAMErmpropertyrprur|rrr DEFAULT_KDEFAULT_FETCH_Krrrjr!rrrrzrDEFAULT_INSERT_BATCH_SIZErrrrrhrrr#r)r2r classmethodr=r;rCrRrNrfrerZrmrqrrtrxrvryr1r/r\r\s $T+/6.2%AE.2  (    ,   ? ,   8$4&+/ ## # # ) #  # &#DHL#' &  E  !  H&#' !  0'+37 +/+/$+/1 +/  +/`#'.2"& 6 6  6 , 6  6  6  >6 v).#( %%&%! %  %&%'.3 ( (" (, (  (&HL 9 9 9  9 & 9 E 9  9 vMM(,M$2 M  M.+/#'3#' 66(6! 6  6 ! 66 6v&**.#'#'66#6( 6 ! 6  6!66 6v+/#'3 ''('! '  '  ' '\+/3 ##&# # ( #  ## #J6B"6B:J6B 6Bp9,. $()-&* E E'E$ E  E  E.-1$($) 222* 2 " 2 " 2 226%&&*,015#(J(J(J( J(  J( $ J(*J(/J(!J( +J(X: +/#'36    !  (  !             D+/*.#'36& &&(&( & ! &  &&& &&V'+#(\ 0(0($0( 0(  0(  0(j& ,0 44 4 4  4 * 44 4r& ,0 5$5$ 5$ 5$  5$ * 5$5$ 5$t& ,0 22 2 2  2 * 22 &2n& ,0 5$5$ 5$ 5$  5$ * 5$5$ &5$r9=)-"&.2/3#('5''' '  ' , '-'!'' +'X&,0 33 3 3 * 3  3 36&,0 )) ) ) * )  ) )<&,0 ,4,4 ,4 ,4 * ,4  ,4 &,4b&,0 44 4 4 * 4  4 &4B Q" Q14 Q@H Q  Q " )2 ?M  r1r\c d|i} d|vr|r|| d<||| d<|tvr|| d<d|vr||| d<d|vr|t|| d<d|vr |tvr|| d <d|vr |tvr|| d <|| i} | S) NsetAddr_ref propertiesrFindrrr5)rBr) r(r%rrrQrrrr5entityrs r/rrs$W-F w v **$|  0v !8 #K 0} 4J!J +} 0F!F#y & !E Lr1c |dk(rAtd||fDr-||d} ||| d<||| d<||| d<|difvr|| d<|7|| d<n1|d k(rd |i} |r|| d <| difvr| | d <| | | d <ntd||| i} | S)Nrc3$K|]}|du ywr3ry).0vars r/ z%_add_descriptorset..s141s)r" dimensionsr`rerrrrrrrr5zUnknown command: )allr*) r(r"num_dimsr`rerrQrrrr5rrs r/rrs((S1$((#31.""   %F8   %F8  ? F6N r "#(F<  !F6N + + #-F< tRj ($/F= !   'F9 ,[M:;; & !E Lr1ct|dkDrdj|nd}d}i}d|d<d|d<d |i}d |d <||d <||d<t|}i}|||<||fS) Nrrr AddEntityrclassTr&r"zqueryable propertiestyper%)r)join _str2bytes)r^rall_properties_str querytyperrQ byte_datars r/_add_entity_with_blobr8s695H15L.1RTIF"F7OF6N#_5E*E&M)E) F<-.IEE) ) r1findcNg}g}gd}|j|vr)tdjdj||jdk(r"t |d}|j |||fS|jdk(r5t ||\}}|j ||j |||fS|jdk(rOt |d }|j |t ||\}}|j ||j |||fS) N)rrrz"[!] Invalid type. Choices are : {}rrT)rrr)r)rr*formatrrrCr) r^rrrrrchoicesrrs r/rrNs,  KH'G7*=DDSXXgEVWXXv%%oTJ5!"        &0.Qy5! "        )%oE5!1.Qy5! "   r1c4tj|d}|S)Nr]rX)r, frombuffer)r&rs r/r[r[qs --I .C Jr1c"|jSr3)r8)in_bytess r/rrvs ?? r1c tt|Dcgc]}|jD]}|c}}Scc}}wr3)rrr)rqrs r/_get_cmds_from_queryrzs2  >1QVVX>Q>Q>? @@>s:cjt|}ttxrtfd|D}|S)Nc3dK|]'}|dvxrdd|vxrd|ddkD)yw)rrNry)rcmdrs r/rz(_check_valid_response..sZ3  x{ - (1+c* * - QK Z (1 , -3s-0)rr(rany)rrcmd_list valid_ress ` r/rr~s<#K0H8T*s3 30I r1cdd|gi}td||dgdd}|g}|j|\}}t||}||fS)Nr$rr:r)rrr+)rrr) rYr%r$rfindDescriptorrresr7rs r/rrsb $$K$"- N ""K \\+ &FC%k37I n $$r1c\d}|'tj|d}|j}|S)zConvert embedding to bytes.Nr]rX)r,r\tobytes)r]r&rs r/rrs/ Dhhy 2{{} Kr1cd}i}d|d<|r||d<i}d|d<d|d<d g|d <||d <i}|rd d g|d<d |g|d<||d<i}|||<|S)N FindEntityrruniqueTr&rrr%rr5rrrr"rry)r^rrrrr5rrs r/rrs IF"F7O(x GGFOGG kGFOF9"$K$(!9 K 1K'F=EE) Lr1c,tj|Sr3)rencode)in_strs r/rrs ::f r1cxi}|jD]$\}}t|tr||t|<&|Sr3)rAr(rr)r> new_metadatakeyrs r/rrsA#%Lnn&+ U%&%*LS "+ r1) r+rr,rr-rr.rrr)r5rrr)r5rrr) localhosti)rWrrXrrr{)NNNNNNN)r(rr%rrrrrrQrrrrrrrr5rrDict[str, Dict[str, Any]]) NNNNNNFNN)r(rr"rrrr`rrerrrrQrrrrrrrr5rrDict[str, Any])r^rrr rzTuple[Dict[str, Any], bytes]) r^rrrrr rrrr)r&rrr)rrrr)rrrr)rz List[dict]rrrr)rYr{r%rr$rrzTuple[bool, Any])r]rrzUnion[bytes, None]r)r^rrrrrrr)rrrr)r>rrr)C __future__rrloggingrLrcopyrtypingrrrrr r r r r rrrrnumpyr,langchain_core.documentsrlangchain_core.embeddingsrlangchain_core.vectorstoresr&langchain_community.vectorstores.utilsrrTr}rr__annotations__r~rrrrrrr@rB getLoggerrrEr0r8r4rZr\rrrrr[rrrrrrrrryr1r/rs"   -03M 6:(CS:T5U2U  $((9#:=:% a-3<,dB7   8 $ $D>(N ;N n*  !%"&"## # #  #  #  ## ###R#   "&"44 44  4  4  4 4 44 444n*.!0 ! ! ! !  !  !F A% % % % %&%*$! 8r1