Ig + 4ddlZddlmZddlmZddlmZddlmZm Z m Z m Z m Z m Z mZmZddlZddlZddlmZddlmZmZmZmZddlmZmZdd lmZmZdd l m!Z!gd Z"d d dddd ide"Dcgc]}|d c}ddgZ#GddeZ$GddeZ%ycc}w)N) defaultdict)get_all_start_methods)Path)AnyDictIterableListOptionalTupleTypeUnion)SnowballStemmer)define_cache_dir iter_batchget_all_punctuationremove_non_alphanumeric)ParallelWorkerPoolWorker)SparseEmbeddingSparseTextEmbeddingBase)SimpleTokenizer)arabic azerbaijanibasquebengalicatalanchinesedanishdutchenglishfinnishfrenchgermangreekhebrewhinglish hungarian indonesianitaliankazakhnepali norwegian portugueseromanianrussianslovenespanishswedishtajikturkishz Qdrant/bm25z6BM25 as sparse embeddings meant to be used with Qdrantz apache-2.0g{Gz?hfz mock.file.txtT)model descriptionlicense size_in_GBsources model_fileadditional_files requires_idfceZdZdZ ddedeedededededeffd Ze d e e ee ffd Z e d eded e efd Z ddededeeeefdedeed eef dZ ddeeeefdedeed eefdZde ed e efdZde ed e efdZde ed e eeffdZe ded efdZdeeeefd eefdZe d edfdZxZS)Bm25aImplements traditional BM25 in a form of sparse embeddings. Uses a count of tokens in the document to evaluate the importance of the token. WARNING: This model is expected to be used with `modifier="idf"` in the sparse vector index of Qdrant. BM25 formula: score(q, d) = SUM[ IDF(q_i) * (f(q_i, d) * (k + 1)) / (f(q_i, d) + k * (1 - b + b * (|d| / avg_len))) ], where IDF is the inverse document frequency, computed on Qdrant's side f(q_i, d) is the term frequency of the token q_i in the document d k, b, avg_len are hyperparameters, described below. Args: model_name (str): The name of the model to use. cache_dir (str, optional): The path to the cache directory. Can be set using the `FASTEMBED_CACHE_PATH` env variable. Defaults to `fastembed_cache` in the system's temp directory. k (float, optional): The k parameter in the BM25 formula. Defines the saturation of the term frequency. I.e. defines how fast the moment when additional terms stop to increase the score. Defaults to 1.2. b (float, optional): The b parameter in the BM25 formula. Defines the importance of the document length. Defaults to 0.75. avg_len (float, optional): The average length of the documents in the corpus. Defaults to 256.0. Raises: ValueError: If the model_name is not in the format / e.g. BAAI/bge-base-en. model_name cache_dirkbavg_lenlanguagetoken_max_lengthc t |||fi||tvrt|d||_||_||_||_|j|} t||_ |j| |j|j|_ ||_tt!|_t|j%|j|j|_t)||_t,|_y)Nz language is not supported)local_files_only)super__init__supported_languages ValueErrorrFrCrDrE_get_model_descriptionrrBdownload_model_local_files_only _model_dirrGsetr punctuation_load_stopwords stopwordsrstemmerr tokenizer) selfrArBrCrDrErFrGkwargsmodel_description __class__s R/var/www/html/answerous/venv/lib/python3.12/site-packages/fastembed/sparse/bm25.pyrKz Bm25.__init__cs Y9&9 . .z)CDE E$DM  77 C))4-- t~~@V@V. !1245T11$//4==QR&x0 (returnctS)zLists the supported models. Returns: List[Dict[str, Any]]: A list of dictionaries containing the model information. )supported_bm25_modelsclss r\list_supported_modelszBm25.list_supported_modelss %$r] model_dirc||dz }|jsgSt|d5}|jjcdddS#1swYyxYw)Nr6r)existsopenread splitlines)rbrdrFstopwords_pathfs r\rTzBm25._load_stopwordssW"z%66$$&I .# & )!668&&( ) ) )s AA documents batch_sizeparallelc#Kd}t|tr|g}d}t|trt||krd}||r+t ||D]}|j |Ed{y|dk(rt j}dtvrdnd}|||j|j|jd} t|xsd|j|} | jt ||fi| D] }|D]} | y7w) NFTr forkserverspawn)rArBrCrDrE) num_workersworker start_method) isinstancestrlistlenr raw_embedos cpu_countrrCrDrEr_get_worker_class ordered_map) rXrArBrmrnrois_smallbatchrvparamspoolrecords r\_embed_documentszBm25._embed_documentss i %" IH i &9~ *  x#Iz: 1>>%000 11}<<>+7;P;R+R 1, data-parallel encoding will be used, recommended for offline encoding of large datasets. If 0, use all available cores. If None, don't use data-parallel processing, use default onnxruntime threading instead. Returns: List of embeddings, one per document )rArBrmrnroN)rrArxrB)rXrmrnrorYs r\embedz Bm25.embedsA,(($..)! )   s7A?Atokenscg}|D]}||jvr|j|jvr/t||jkDrH|j j |j}|st|j||SN)rSlowerrUrzrGrV stem_wordappend)rXrstemmed_tokenstoken stemmed_tokens r\_stemz Bm25._stems 5E((({{}.5zD111 LL225;;=AM%%m4 5r]cg}|D]n}t|}|jj|}|j|}|j |}|j t j|p|Sr)rrWtokenizer_term_frequencyrr from_dict)rXrm embeddingsdocumentrrtoken_id2values r\r{zBm25.raw_embedsw ! IH.x8H^^,,X6F!ZZ/N!11.AN   o77G H  I r]c Xi}tt}|D]}||xxdz cc<t|}|D]t}|j|}||}||jdzz||<||xx||jd|j z |j |z|j z zzzzcc<v|S)agCalculate the term frequency part of the BM25 formula. ( f(q_i, d) * (k + 1) ) / ( f(q_i, d) + k * (1 - b + b * (|d| / avg_len)) ) Args: tokens (List[str]): The list of tokens in the document. Returns: Dict[int, float]: The token_id to term frequency mapping. rs)rintrzcompute_token_idrCrDrE)rXrtf_mapcounterrdoc_lentoken_idnum_occurrencess r\rzBm25._term_frequencysc"# (M M "a ' " (f+$ M,,];H%m4O.$&&1*=F8  8 $&&DFF TVVg- <<3!     r]rc>ttj|Sr)absmmh3hash)rbrs r\rzBm25.compute_token_ids499U#$$r]queryc +xKt|tr|g}|D]}t|}jj |}j |}t jttfd|Dt j}t j|}t||yw)zTo emulate BM25 behaviour, we don't need to use weights in the query, and it's enough to just hash the tokens and assign a weight of 1.0 to them. c3@K|]}j|ywr)r).0rrXs r\ z#Bm25.query_embed...sR%..u5Rs)dtype)indicesvaluesN) rwrxrrWrrnparrayryrRint32 ones_liker)rXrrYtextrr token_idsrs` r\ query_embedzBm25.query_embed"s eS !GE DD*40D^^,,T2F!ZZ/NSR>RRShhI\\),F!)FC C DsB7B: Bm25WorkerctSr)rras r\r~zBm25._get_worker_class4sr])Ng333333?g?gp@r ()N)__name__ __module__ __qualname____doc__rxr floatrrK classmethodr rrrcrrTr rrrrrr{rrrr r~ __classcell__)r[s@r\r@r@Gs/<$(! "")")C=")  ")  ")  ")")")H%d4S>&:%%)))S ))"& (!(!(!hsm+, (!  (! 3- (! / "(!Z"&  hsm+,  3-  / "  <DI$s)$ 9  o  d3iDe4D<%S%S%%DsHSM'9!:DRaIbD$$|"4r]r@c eZdZdedefdZededededdfdZdee e efdee e effdZ e dedede fd Zy ) rrArBc 6|j||fi||_yr)init_embeddingr7)rXrArBrYs r\rKzBm25Worker.__init__:s )T((YI&I r]rYr^c |d||d|SN)rArBr)rbrArBrYs r\startzBm25Worker.startBsHjIHHHr]itemsc#bK|D]&\}}|jj|}||f(ywr)r7r{)rXridxr onnx_outputs r\processzBm25Worker.processFs8 #JC**..u5K{" " #s-/c td||d|Sr)r@)rArBrYs r\rzBm25Worker.init_embeddingKsIzYI&IIr]N)rrrrxrKrrrrr rr staticmethodr@rrr]r\rr9sJJJIsIsIcIlII#XeCHo6#8E#s(O;T# J3J3JTJJr]r)&r| collectionsrmultiprocessingrpathlibrtypingrrrr r r r r rnumpyrpy_rust_stemmersrfastembed.common.utilsrrrrfastembed.parallel_processorrr&fastembed.sparse.sparse_embedding_baserr fastembed.sparse.utils.tokenizerrrLr`r@r)langs0r\rs #1JJJ , D=DO - "7JKtvT]K   o "odJJqLs+ B