
    :Qg
                         d dl mZmZ d dlmZ d dlZd dlmZmZ d dl	m
Z
 d Zd Zdee   d	eeeef      fd
Zd Zd Zd Zd Zd Zy)    )ListTuple)patchN)mock_sent_tokenizemock_word_tokenize)tokenizec                  P   t         j                  j                          t        j                  t
        dt              5  t        j                  t         d      5 } t        j                          d d d        d d d         j                          y # 1 sw Y   "xY w# 1 sw Y   &xY w)Nfind)side_effectdownload_nltk_packages)r   &_download_nltk_packages_if_not_presentcache_clearr   objectnltkLookupErrorassert_called_oncemock_downloads    `/var/www/html/answerous/venv/lib/python3.12/site-packages/test_unstructured/nlp/test_tokenize.py*test_nltk_packages_download_if_not_presentr   
   sy    33??A	dF	< >\\($<= 	>;;=	>> $$&	> 	>> >s#   BB/BB	BB%c                  D   t         j                  j                          t        j                  t
        d      5  t        j                  t
        d      5 } t        j                          d d d        d d d         j                          y # 1 sw Y   "xY w# 1 sw Y   &xY w)Nr
   download)r   r   r   r   r   r   assert_not_calledr   s    r   %test_nltk_packages_do_not_download_ifr      su    33??A	dF	# :U\\$
%C :}779: : ##%: : : :s#   BB
)B
B	BBtokensreturnc                     g }| D ]<  }|j                         dk(  r|j                  |df       *|j                  |df       > |S )NaskVB )lowerappend)r   pos_tagstokens      r   mock_pos_tagr%      sK    &(H );;=E!OOUDM*OOUBK(	)
 O    c                  @    t        j                  d      } | g dk(  sJ y )NzITEM 2A. PROPERTIES))ITEMNNP)2ACD).r,   )
PROPERTIESNN)r   pos_tag)parts_of_speechs    r   test_pos_tagr1   %   s*    &&'<=O    r&   c                    | j                  t        dt               | j                  t        dt               t        j                  j                          t        j                  j                         j                  dk(  sJ t        j                  d       t        j                  j                         j                  dk(  sJ y N_word_tokenize_pos_tagr   !Greetings! I am from outer space.   )setattrr   r   r%   word_tokenizer   
cache_infocurrsizemonkeypatchs    r   test_word_tokenize_cachesr>   /   s    "24FG*l;&&(!!,,.771<<<>?!!,,.771<<<r&   c                    | j                  t        dt               | j                  t        dt               | j                  t        dt               t        j
                  j                          t        j
                  j                         j                  dk(  sJ t        j
                  d       t        j
                  j                         j                  dk(  sJ y )N_sent_tokenizer4   r5   r   r6   r7   )	r8   r   r   r   r%   sent_tokenizer   r:   r;   r<   s    r   test_sent_tokenize_cachesrB   8   s    "24FG"24FG*l;&&(!!,,.771<<<>?!!,,.771<<<r&   c                    | j                  t        dt               | j                  t        dt               t        j                  j                          t        j                  j                         j                  dk(  sJ t        j                  d       t        j                  j                         j                  dk(  sJ y r3   )r8   r   r   r%   r/   r   r:   r;   r<   s    r   test_pos_tag_cachesrD   B   s    "24FG*l;  "&&(11Q66689&&(11Q666r&   c                      d} t        j                  |        t        j                  |        t        j                  |        y )Nz$I am a big brown bear. What are you?)r   rA   r9   r/   )sentences    r   test_tokenizers_functions_runrG   K   s1    5H8$8$Xr&   )typingr   r   unittest.mockr   r   test_unstructured.nlp.mock_nltkr   r   unstructured.nlpr   r   r   strr%   r1   r>   rB   rD   rG    r&   r   <module>rN      s[       R %'&c tE#s(O'< ==7r&   