
    *+h|                     b   d dl Zd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlmZmZ d dlmZ d dlmZ d dlmZ ddlmZ  ej,                  d      j/                  ej0                         dd	Zd
 Zd Z G d d      Z G d d      Z G d d      Z G d d      Zd Z d Z!y)    N)ListDict)tqdmRecursiveCharacterTextSplitter)extract   )LitellmModelhttpxc                 b    t        |       |kD  r | d| }t        j                  d| d       |S | S )zTruncate filename to max_length to ensure the filename won't exceed the file system limit.

    Args:
        filename: str
        max_length: int, default to 125 (usual path length limit is 255 chars)
    Nz/Filename is too long. Filename is truncated to .)lenloggingwarning)filename
max_lengthtruncated_filenames      6/var/www/html/sandstorm/storm/knowledge_storm/utils.pytruncate_filenamer      sE     8}z!%kz2=>P=QQRS	
 "!O    c                    	 t        | d      5 }t        j                  |      }d d d        j                         D ]!  \  }}t        |      t        j                  |<   # y # 1 sw Y   >xY w# t        $ r! t	        d|  t
        j                         Y y t        j                  $ r! t	        d|  t
        j                         Y y w xY w)NrzFile not found: )filezError decoding TOML file: )opentomlloadFileNotFoundErrorprintsysstderrTomlDecodeErroritemsstrosenviron)toml_file_pathr   datakeyvalues        r   load_api_keyr*   )   s    .#& 	#$99T?D	# jjl %
Ue*

3%	# 	#   01

C *>*:;#**Ms-   A- A!A- !A*&A- -'C	0C	C	c                     d|  dS )Nz[91m z[00m )messages    r   makeStringRedr.   8   s    wix((r   c                        e Zd ZdZedddeddfd       Zeded	ededdfd
       Zedededdfd       Ze	 	 	 	 	 	 	 	 	 	 	 ddededededededede	de	de	dededededefd       Z
y)QdrantVectorStoreManageraK  
    Helper class for managing the Qdrant vector store, can be used with `VectorRM` in rm.py.

    Before you initialize `VectorRM`, call `create_or_update_vector_store` to create or update the vector store.
    Once you have the vector store, you can initialize `VectorRM` with the vector store path or the Qdrant server URL.
    clientQdrantClientcollection_namemodelHuggingFaceEmbeddingsc                 6   ddl m} ddlm} 	 | t	        d      | j                  |       rt        d| d        || ||      S t        d| d	       | j                  | |j                  d
|j                  j                                || ||      S )Nr   )Qdrant)models!Qdrant client is not initialized.)r3   zCollection z" exists. Loading the collection...)r1   r3   
embeddingsz+ does not exist. Creating the collection...i   )sizedistance)r3   vectors_config)langchain_qdrantr7   qdrant_clientr8   
ValueErrorcollection_existsr   create_collectionVectorParamsDistanceCOSINE)r1   r3   r4   r7   r8   s        r   _check_create_collectionz1QdrantVectorStoreManager._check_create_collectionD   s     	,(Q>@AA##6G#IK00RST /   o..YZ $$#2"3%22(>(>  3   %   /  r   urlapi_keyc                    ddl m} 	 |5t        j                  d      st	        d      t        j                  d      }| t	        d      	  || |      }t
        j                  |||      S # t        $ r}t	        d|       d }~ww xY w)	Nr   r2   QDRANT_API_KEYzPlease provide an api key.z+Please provide a url for the Qdrant server.)rG   rH   r1   r3   r4   z,Error occurs when connecting to the server: )r?   r2   r$   getenvr@   r0   rF   	Exception)rG   rH   r3   r4   r2   r1   es          r   _init_online_vector_dbz/QdrantVectorStoreManager._init_online_vector_dbf   s     	/	 ?99-. !=>>ii 01G;JKK	Q!c7;F+DDe E    	QKA3OPP	Qs   !A/ /	B8BBvector_store_pathc                     ddl m} 	 | t        d      	  ||       }t        j	                  |||      S # t
        $ r}t        d|       d }~ww xY w)Nr   rJ   zPlease provide a folder path.)pathrL   z,Error occurs when loading the vector store: )r?   r2   r@   r0   rF   rN   )rQ   r3   r4   r2   r1   rO   s         r   _init_offline_vector_dbz0QdrantVectorStoreManager._init_offline_vector_db   sx     	/	
 $<==	Q!'89F+DDe E    	QKA3OPP	Qs    7 	A AANvector_db_mode	file_pathcontent_columntitle_column
url_columndesc_column
batch_size
chunk_sizechunk_overlapqdrant_api_keyembedding_modeldevicec                 *   ddl m} 	 | t        d      d|i}ddi}ddlm}  ||||      }|t        d	      |j                  d
      st        d      |t        d      |t        d      d }|dk(  rt        j                  ||| |      }n)|dk(  rt        j                  |
| |      }nt        d      |t        d      dd l	}|j                  |      }||j                  vrt        d| d      ||j                  vrt        d| d      |j                  d      D cg c]6  } |||   |j                  |d      ||   |j                  |d      d      8 }}ddlm}  |||	t         dg d      }|j#                  |      }t!        |      |z   dz
  |z  }t%        t'        |            D ]8  }||z  }t)        |dz   |z  t!        |            }|j+                  ||| |        : |j,                  j/                          y c c}w )!Nr   )Documentz!Please provide a collection name.r`   normalize_embeddingsT)r5   )
model_namemodel_kwargsencode_kwargszPlease provide a file path.z.csvz1Not valid file format. Please provide a csv file.z.Please provide the name of the content column.z*Please provide the name of the url column.online)rG   rH   r3   r4   offline)rQ   r3   r4   zDInvalid vector_db_mode. Please provide either 'online' or 'offline'.r9   zContent column z not found in the csv file.zURL column records)orient )titlerG   description)page_contentmetadatar   


r   u   ．u   。,u   ，u   、 u   ​rk   )r\   r]   length_functionadd_start_index
separatorsr	   )	documentsr[   )r?   rb   r@   langchain_huggingfacer5   endswithr0   rP   rT   pandasread_csvcolumnsto_dictgetlangchain_text_splittersr   r   split_documentsr   rangeminadd_documentsr1   close) r3   rU   rV   rW   rX   rY   rZ   r[   r\   r]   rQ   rG   r^   r_   r`   rb   re   rf   r5   r4   qdrantpddfrowrx   r   text_splitterr   num_batchesi	start_idxend_idxs                                    r   create_or_update_vector_storez6QdrantVectorStoreManager.create_or_update_vector_store   s   $ 	+	. "@AA &)/6?%&%'
 :;;!!&)PRR!MNNIJJ X%-DD& /	 E F y(-EE"3 / F F V  >@AA 	[[#+!.!11LM  RZZ'{:,6QRSS zzz3

   0 WW\26z?#&77;#;

	 

 	L6!' 
& (77	B ?+j81<KeK() 	AJI1q5J.O0DEG  ))G<% ! 	 	_

s   +;H)rl   rG   rm   @   i  d   NNNzBAAI/bge-m3mps)__name__
__module____qualname____doc__staticmethodr#   rF   rP   rT   intr   r,   r   r   r0   r0   <   sy    14=T B QQQ14Q=TQ Q4 QQ14Q=TQ Q*  $( !%",RRR R 	R
 R R R R R R R R R R R Rr   r0   c                       e Zd Zed        Zed        Zed        Zed        Zed        Zedd       Z	ed        Z
ed        Zed	        Zy
)ArticleTextProcessingc                     d}d}| j                  d      D ]V  }|j                         }|D ]  }||k  r||dz   z  }|dz  } n ||k\  r |j                         S |j                         dz   }X |j                         S )a   
        Limit the word count of an input string to a specified maximum, while preserving the integrity of complete lines.

        The function truncates the input string at the nearest word that does not exceed the maximum word count,
        ensuring that no partial lines are included in the output. Words are defined as text separated by spaces,
        and lines are defined as text separated by newline characters.

        Args:
            input_string (str): The string to be truncated. This string may contain multiple lines.
            max_word_count (int): The maximum number of words allowed in the truncated string.

        Returns:
            str: The truncated string with word count limited to `max_word_count`, preserving complete lines.
        r   rk   rr   rt   r	   )splitstrip)input_stringmax_word_count
word_countlimited_stringword
line_wordslws          r   !limit_word_count_preserve_newlinez7ArticleTextProcessing.limit_word_count_preserve_newline.  s    " 
 &&t, 
	;DJ  ."b3h.N!OJ ^+ ##%% ,113d:N
	; ##%%r   c                 T    t        | t              s| S t        j                  dd|       S )a  
        Removes all citations from a given string. Citations are assumed to be in the format
        of numbers enclosed in square brackets, such as [1], [2], or [1, 2], etc. This function searches
        for all occurrences of such patterns and removes them, returning the cleaned string.

        Args:
            s (str): The string from which citations are to be removed.

        Returns:
            str: The string with all citation patterns removed.
        z\[\d+(?:,\s*\d+)*\]rk   )
isinstancer#   resub)ss    r   remove_citationsz&ArticleTextProcessing.remove_citationsP  s'     !S!Hvv,b!44r   c                 p    t        j                  d|       }|D cg c]  }t        |dd        c}S c c}w )a_  
        Extracts citation indexes from the provided content string and returns them as a list of integers.

        Args:
            content (str): The content string containing citations in the format [number].

        Returns:
            List[int]: A list of unique citation indexes extracted from the content, in the order they appear.
        \[\d+\]r	   )r   findallr   )r   matchesindexs      r   parse_citation_indicesz,ArticleTextProcessing.parse_citation_indicesa  s3     **Z+.56UE!BK 666s   3c                     d }d }t        j                  d||       } t        j                  d||       } d}t        t        j                  ||             }|r&|d   }| d|j	                          j                         } | S )a  
        Removes uncompleted sentences and standalone citations from the input text. Sentences are identified
        by their ending punctuation (.!?), optionally followed by a citation in square brackets (e.g., "[1]").
        Grouped citations (e.g., "[1, 2]") are split into individual ones (e.g., "[1] [2]"). Only text up to
        and including the last complete sentence and its citation is retained.

        Args:
            text (str): The input text from which uncompleted sentences and their citations are to be removed.

        Returns:
            str: The processed string with uncompleted sentences and standalone citations removed, leaving only
            complete sentences and their associated citations if present.
        c                 r    | j                  d      j                  d      }dj                  d |D              S )Nr	   z, rt   c              3   (   K   | ]
  }d | d  yw)[]Nr,   ).0ns     r   	<genexpr>z~ArticleTextProcessing.remove_uncompleted_sentences_with_citations.<locals>.replace_with_individual_brackets.<locals>.<genexpr>  s     6as!H6s   )groupr   join)matchnumberss     r    replace_with_individual_bracketszkArticleTextProcessing.remove_uncompleted_sentences_with_citations.<locals>.replace_with_individual_brackets  s/    kk!n**40G886g666r   c                     | j                  d      }t        t        t        j                  d|                  }t        |d       }dj                  |      S )Nr   r   c                 6    t        | j                  d            S )Nz[])r   r   )xs    r   <lambda>znArticleTextProcessing.remove_uncompleted_sentences_with_citations.<locals>.deduplicate_group.<locals>.<lambda>  s    AGGDM0B r   )r(   rk   )r   listsetr   r   sortedr   )r   	citationsunique_citationssorted_citationss       r   deduplicate_groupz\ArticleTextProcessing.remove_uncompleted_sentences_with_citations.<locals>.deduplicate_group  sM    AI#C

:y(I$JK% &B  77+,,r   z\[([0-9, ]+)\]z
(\[\d+\])+z([.!?])\s*(\[\d+\])?\s*r   N)r   r   r   finditerendr   )textr   r   eos_patternr   
last_matchs         r   +remove_uncompleted_sentences_with_citationszAArticleTextProcessing.remove_uncompleted_sentences_with_citationso  s    "	7
	- vv')I4Pvvm%6=* 1r{{;56 J**..*+113Dr   c           	      B   | j                   D ]q  }d|j                  v r-|j                  d |j                  j                  d       |_        d|j                  v r-|j                  d |j                  j                  d       |_        |j                  j                  dd      j	                         |_        	 t        t        j                  d|j                        D cg c]  }t        |       c}      }|t        |j                        kD  rLt        t        |j                        |dz         D ]'  }|j                  j                  d| d	d      |_        ) t        j                  |j                        |_        t | S c c}w # t        $ r}d}Y d }~d }~ww xY w)
NzReferences:zSources:zAnswer:rk   z	\[(\d+)\]r   r	   r   r   )dlg_historyagent_utterancefindreplacer   maxr   r   r   rN   r   search_resultsr   r   r   )convturnr   max_ref_numrO   r   s         r   clean_up_citationz'ArticleTextProcessing.clean_up_citation  s   $$ 	D 4 44'+';';>d**//>($ T111'+';';;d**//
;($ $(#7#7#?#?	2#N#T#T#VD  !%'ZZd>R>R%STSVT
 S!4!455s4#6#67qI VA+/+?+?+G+G!A3aRT+UD(V &QQ((  '	2  U   s*   7'F	F
0F	F			FFFc                    g }d}| j                  d      D ]  }|j                         }|dk7  r%d|j                          |j                         v rg }|j                  d      r#|j	                  d      }|j                  |       q|j                  d      sd|dz   z  dz   |dd  j                         z   }|j                  |        dj                  |      } t        j                  d	d| t        j                  
      } t        j                  dd| t        j                  
      } t        j                  dd| t        j                  
      } t        j                  dd| t        j                  
      } t        j                  dd| t        j                  
      } t        j                  dd| t        j                  
      } t        j                  dd| t        j                  
      } t        j                  dd| t        j                  
      } t        j                  dd| t        j                  
      } t        j                  dd| t        j                  
      } t        j                  dd| t        j                  
      } t        j                  dd| t        j                  
      } t        j                  dd|       } | S )Nr   rr   rk   # #-r	   rt   z#[#]? See also.*?(?=##|$))flagsz#[#]? See Also.*?(?=##|$)z#[#]? Notes.*?(?=##|$)z#[#]? References.*?(?=##|$)z#[#]? External links.*?(?=##|$)z#[#]? External Links.*?(?=##|$)z#[#]? Bibliography.*?(?=##|$)z#[#]? Further reading*?(?=##|$)z#[#]? Further Reading*?(?=##|$)z#[#]? Summary.*?(?=##|$)z#[#]? Appendices.*?(?=##|$)z#[#]? Appendix.*?(?=##|$)z\[.*?\])
r   r   lower
startswithcountappendr   r   r   DOTALL)outlinetopicoutput_linescurrent_levellinestripped_linesubsection_headers          r   clean_up_outlinez&ArticleTextProcessing.clean_up_outline  s6   MM$' 	7D JJLM{EKKM?3}7J7J7LL! '', - 3 3C 8##M2))#.=1,-3mAB6G6M6M6OO " ##$56	7" ))L) &&5r7"))T&&5r7"))T&&2BryyQ&&7WBIIV&&.G299
 &&.G299
 &&92wbiiX&&.G299
 &&.G299
 &&4b'S&&7WBIIV&&5r7"))T&&R1r   c                    | j                  d      }g }d}|D ]  }|j                         }t        |      dk(  r"|j                  d      st        j                  |      }|r|j                  d      rd}n_|j                  d      s"|j                  d      s|j                  d      rd|v sd	|v rd
}|j                  |        dj                  |      S )zClean up a section:
        1. Remove uncompleted sentences (usually due to output token limitation).
        2. Deduplicate individual groups of citations.
        3. Remove unnecessary summary.rr   Fr   r   Overallz
In summaryzIn conclusionz	# Summaryz# ConclusionTrq   )r   r   r   r   r   r   r   r   )r   
paragraphsoutput_paragraphssummary_sec_flagps        r   clean_up_sectionz&ArticleTextProcessing.clean_up_section  s     ZZ%
  	(A	A1v{<<$)UUVWX<<$',$Y'<<-<<0a>Q#6#' $$Q')	(. {{,--r   c                     |D ]  }| j                  d| dd| d      }  |j                         D ]  \  }}| j                  d| dd| d      } ! | S )z>Update citation index in the string based on the citation map.r   r   __PLACEHOLDER___)r   r"   )r   citation_maporiginal_citationunify_citations       r   update_citation_indexz+ArticleTextProcessing.update_citation_index  s     ". 			%&a(N;L:MR*PA	 2>1C1C1E 	Y-~		N+<*=R@AnEUUVBWXA	Y r   c                    | j                  d      }|D cg c]  }|j                         s| }}di d}|dfg}|D ]  }|j                  d      r|j                  d      }|j                  d      j                         }di d}|r)|d   d   |k\  r|j	                          |r|d   d   |k\  r||d   d   d	   |<   |j                  ||f       |d   d   d
xx   |dz   z  cc<    |d	   S c c}w )a  
        Parses a structured text into a nested dictionary. The structure of the text
        is defined by markdown-like headers (using '#' symbols) to denote sections
        and subsections. Each section can contain content and further nested subsections.

        The resulting dictionary captures the hierarchical structure of sections, where
        each section is represented as a key (the section's title) mapping to a value
        that is another dictionary. This dictionary contains two keys:
        - 'content': content of the section
        - 'subsections': a list of dictionaries, each representing a nested subsection
        following the same structure.

        Args:
            input_string (str): A string containing the structured text to parse.

        Returns:
            A dictionary representing contains the section title as the key, and another dictionary
        as the value, which includes the 'content' and 'subsections' keys as described above.
        rr   rk   )contentsubsectionsr   r   r   r	   r   r   r   )r   r   r   r   popr   )r   linesr   rootcurrent_pathlevelrl   new_sections           r   parse_article_into_dictz-ArticleTextProcessing.parse_article_into_dict)  s$   * ""4("'8$4::<88b1r
| 	>Ds#

3

4(..0*,R@ #|B'7':e'C $$& #|B'7':e'C =HR #M259##[%$89R #I.$+=.	>  M"") 9s
   C1C1N)rk   )r   r   r   r   r   r   r   r   r   r   r   r   r  r,   r   r   r   r   -  s    & &B 5 5  7 7 : :x  8 . .`  .  .D 	 	 )# )#r   r   c                       e Zd Zed	d       Zed        Zed	d       Zed        Zed        Zed        Z	ed        Z
y)
FileIOHelperc                     t        |d|      5 }t        j                  | |t        j                         d d d        y # 1 sw Y   y xY w)Nwencoding)default)r   jsondumpr  handle_non_serializable)obj	file_namer  fws       r   	dump_jsonzFileIOHelper.dump_jsonW  s?    )S84 	MIIc2|'K'KL	M 	M 	Ms	   '?Ac                      y)Nznon-serializable contentsr,   )r  s    r   r  z$FileIOHelper.handle_non_serializable\  s    *r   c                 t    t        | d|      5 }t        j                  |      cd d d        S # 1 sw Y   y xY w)Nr   r
  )r   r  r   )r  r  frs      r   	load_jsonzFileIOHelper.load_json`  s0    )S84 	!99R=	! 	! 	!s   .7c                 h    t        |d      5 }|j                  |        d d d        y # 1 sw Y   y xY w)Nr	  )r   write)r   rS   fs      r   	write_strzFileIOHelper.write_stre  s-    $_ 	GGAJ	 	 	s   (1c                     t        | d      5 }dj                  |j                               cd d d        S # 1 sw Y   y xY w)Nr   rr   )r   r   	readlinesrS   r  s     r   load_strzFileIOHelper.load_strj  s4    $_ 	,99Q[[]+	, 	, 	,s   6?c                 r    t        |d      5 }t        j                  | |       d d d        y # 1 sw Y   y xY w)Nwb)r   pickler  )r  rS   r  s      r   dump_picklezFileIOHelper.dump_pickleo  s1    $ 	 KKQ	  	  	 s   -6c                 p    t        | d      5 }t        j                  |      cd d d        S # 1 sw Y   y xY w)Nrb)r   r"  r   r  s     r   load_picklezFileIOHelper.load_picklet  s.    $ 	";;q>	" 	" 	"s   ,5N)zutf-8)r   r   r   r   r  r  r  r  r  r#  r&  r,   r   r   r  r  V  s    M M + + ! !   , ,     " "r   r  c                   d    e Zd ZdZ	 	 	 ddededefdZdefdZdee   d	e	fd
Z
dee   d	e	fdZy)WebPageHelperzHelper class to process web pages.

    Acknowledgement: Part of the code is adapted from https://github.com/stanford-oval/WikiChat project.
    min_char_countsnippet_chunk_sizemax_thread_numc                     t        j                  d      | _        || _        || _        t        |dt        dg d      | _        y)a0  
        Args:
            min_char_count: Minimum character count for the article to be considered valid.
            snippet_chunk_size: Maximum character count for each snippet.
            max_thread_num: Maximum number of threads to use for concurrent requests (e.g., downloading webpages).
        F)verifyr   rp   )r\   r]   ru   is_separator_regexrw   N)r   Clienthttpx_clientr)  r+  r   r   r   )selfr)  r*  r+  s       r   __init__zWebPageHelper.__init__  sC     "LL6,,;)$
r   rG   c                    	 | j                   j                  |d      }|j                  dk\  r|j                          |j                  S # t
        j                  $ r/}t        d|j                  j                  d|       Y d }~y d }~ww xY w)N   )timeouti  zError while requesting z - )
r0  r   status_coderaise_for_statusr   r   	HTTPErrorr   requestrG   )r1  rG   resexcs       r   download_webpagezWebPageHelper.download_webpage  sz    	##''Q'7C#%$$&;; 	+CKKOO+>c#IJ	s   AA
 
B%BBurlsreturnc                 d   t         j                  j                  | j                        5 }t	        |j                  | j                  |            }d d d        i }t        |      D ]:  \  }}|	t        |ddd      }|t        |      | j                  kD  s4d|i||<   < |S # 1 sw Y   VxY w)N)max_workersFtxt)include_tablesinclude_commentsoutput_formatr   )
concurrentfuturesThreadPoolExecutorr+  r   mapr<  zipr   r   r)  )r1  r=  executorhtmlsarticleshuarticle_texts           r   urls_to_articleszWebPageHelper.urls_to_articles  s    22++ 3 
 	Dd&;&;TBCE	D
 t$ 
	5DAqy"$!&#	L 'C,=@S@S,S%|4
	5 '	D 	Ds   &B&&B/c                     | j                  |      }|D ])  }| j                  j                  ||   d         ||   d<   + |S )Nr   snippets)rP  r   
split_text)r1  r=  rL  rN  s       r   urls_to_snippetszWebPageHelper.urls_to_snippets  sS    ((. 	YA&*&8&8&C&CHQKPVDW&XHQK
#	Y r   N)   i  
   )r   r   r   r   r   r2  r#   r<  r   r   rP  rT  r,   r   r   r(  r(  z  sn     ""& 	!
!
  !
 	!
FC T#Y 4 ,T#Y 4 r   r(  c                    t        dddd      }t        | j                               dkD  ryt        j                  d|       sy	d
|  }ddddd}	  ||      d   j                  dd      j                  dd      }|j                  d      r>t        j                  d|      }|r%t        |j                  d            }||v r||   S 	 yy	 y# t        $ r
}Y d }~yd }~ww xY w)Nazure/gpt-4o-minirV          ?r4   
max_tokenstemperaturetop_p   zAThe input is too long. Please make your input topic more concise!z^[a-zA-Z0-9\s\-"\,\.?\']*$zfThe input contains invalid characters. The input should only contain a-z, A-Z, 0-9, space, -/"/,./?/'.a  Here is a topic input into a knowledge curation engine that can write a Wikipedia-like article for the topic. Please judge whether it is appropriate or not for the engine to curate information for this topic based on English search engine. The following types of inputs are inappropriate:
1. Inputs that may be related to illegal, harmful, violent, racist, or sexual purposes.
2. Inputs that are given using languages other than English. Currently, the engine can only support English.
3. Inputs that are related to personal experience or personal information. Currently, the engine can only use information from the search engine.
4. Inputs that are not aimed at topic research or inquiry. For example, asks requiring detailed execution, such as calculations, programming, or specific service searches fall outside the engine's scope of capabilities.
If the topic is appropriate for the engine to process, output "Yes."; otherwise, output "No. The input violates reason [1/2/3/4]".
User input: zSorry, this input may be related to sensitive topics. Please try another topic. (Our input filtering uses OpenAI GPT-4o-mini, which may result in false positives. We apologize for any inconvenience.)zSorry, the current engine can only support English. Please try another topic. (Our input filtering uses OpenAI GPT-4o-mini, which may result in false positives. We apologize for any inconvenience.)zSorry, the current engine cannot process topics related to personal experience. Please try another topic. (Our input filtering uses OpenAI GPT-4o-mini, which may result in false positives. We apologize for any inconvenience.)zSorry, STORM cannot follow arbitrary instruction. Please input a topic you want to learn about. (Our input filtering uses OpenAI GPT-4o-mini, which may result in false positives. We apologize for any inconvenience.))r	         r4  r   r   rk   r   Nozreason\s(\d+)r	   z<Sorry, the input is inappropriate. Please try another topic!Approved)r
   r   r   r   r   r   r   regexsearchr   r   rN   )
user_inputmy_openai_modelpromptreject_reason_inforesponser   reject_reasonrO   s           r    user_input_appropriateness_checkrl    s   "!	O :#R881:>x LF////N"6*1-55c2>FFsBOt$LL!18<E #EKKN 3 $66-m<< W R %   NMNs   A6C C 	C C c                     t        dddd      }d|  d}	  ||      d   j                  d	d
      j                  dd
      }|j                  d      ry	 y# t        $ r
}Y d }~yd }~ww xY w)NrX  rV  rY  rZ  r[  a  
    Here is a purpose input into a report generation engine that can create a long-form report on any topic of interest. 
    Please judge whether the provided purpose is valid for using this service. 
    Try to judge if given purpose is non-sense like random words or just try to get around the sanity check.
    You should not make the rule too strict.
    
    If the purpose is valid, output "Yes."; otherwise, output "No" followed by reason.
    User input: z
    r   r   rk   r   rb  zVPlease provide a more detailed explanation on your purpose of requesting this article.rc  )r
   r   r   rN   )rf  rg  rh  rj  rO   s        r   purpose_appropriateness_checkrn    s    "!	O  Fh"6*1-55c2>FFsBOt$k %
   hghs   <A 	A)$A))}   )"concurrent.futuresrE  dspyr   r  r   r$   r"  r   rd  r   r   typingr   r   r   r   r   trafilaturar   lmr
   	getLoggersetLevelWARNINGr   r*   r.   r0   r   r  r(  rl  rn  r,   r   r   <module>rx     s         	  	  
    C     '  # #GOO 4$%)n nbf# f#R	!" !"HN Nb4nr   