:QgdddlmZddlmZmZmZmZmZmZm Z ddl Z ddl m Z mZddlmZddlmZddlmZddlmZmZdd lmZdd lmZdd lmZmZmZm Z dd l!m"Z"dd l#m$Z$m%Z%ddl&m'Z'ddl(m)Z)er ddlm*Z*ddl+m,Z,dZ-dZ. d+ d,dZ/e)d d- d.dZ0dZ1e.fd/dZ2e.f d0dZ3de.f d1dZ4de.fd1dZ5e)d d2dZ6d3dZ7e)d d4 d5dZ8 d6d Z9d7d!Z: d8d"Z; d9d#Z dd(ZAd?d)ZBd@d*ZCy)A) annotations) TYPE_CHECKINGAnyBinaryIOListOptionalUnioncastN)LTChar LTTextBox) PDFObjRef) open_filename) Rectangle) PixelSpace PointSpace)CoordinatesMetadata)remove_control_characters)extract_image_objectsextract_text_objectsopen_pdfminer_pages_generator rect_to_bbox) env_config)SORT_MODE_BASICSource)sort_text_regions)requires_dependencies) TextRegion)DocumentLayoutg{Gz?ct|d5}tt|}t||\}}||fcdddS#1swYyxYw)Nrb)filedpi)rr rprocess_data_with_pdfminer)filenamer#fpextracted_layout layouts_linkss q/var/www/html/answerous/venv/lib/python3.12/site-packages/unstructured/partition/pdf_image/pdfminer_processing.pyprocess_file_with_pdfminerr*#sQ x &/" (B *D+ '- . ///s $;Aunstructured_inferencec ddlm}m}g}g}|dz }tt |D]\}\}} | j | j } } g} g} g}t| | }|jrt|j| ||}tj}g}| D]}t|j| \}}}}||||f}t|dkDrOt|t r?t#||||}t%|| \}}|D]}|j't)||t+|drt-|}|D]}|j/}t1gt|j| ||t2j4|}|jU|jj6dkDso| j'|t9|} | D]o}!t1gt|!j| |dt2j4|}|jE|jj6dkDs_| j'|q|D"#cgc]&}"|"dD#cgc]}#|#|z c}#|"d|"d |"d d (}$}"}#t;| tj<}%t;| tj>}&g|%|&}'tA|'tB}'tA|'}'|j'|'|j'|$||fScc}#wcc}#}"w) zzLoads the image and word objects from a pdf using pdfplumber and the image renderings of the pdf pages using pdf2imager)EmbeddedTextRegionImageTextRegionH)widthheightget_textNbboxtexturi start_index)r3r4urlr6)")unstructured_inference.inference.elementsr-r. enumeraterr0r1rannotsget_urisrPDF_ANNOTATION_THRESHOLDrr3len isinstancer check_annotations_within_elementget_words_from_objappendmap_bbox_and_indexhasattrrr2_create_text_regionrPDFMINERarearremove_duplicate_elements#EMBEDDED_TEXT_SAME_REGION_THRESHOLD$EMBEDDED_IMAGE_SAME_REGION_THRESHOLDrr)(r"r#r-r.layoutsr(coef page_numberpage page_layoutr0r1 text_layout image_layoutannotation_listcoordinate_systemannotation_threshold urls_metadataobjx1y1x2y2r3annotations_within_element_wordsannotinner_text_objects inner_obj_text text_regioninner_image_objectsimg_objmetadataxlinksclean_text_layoutclean_image_layoutlayouts( r)r$r$0sb GM 8D,56STX6Y,ZQ$( (dK#));+=+=v  &  ;;&t{{F"!3 8I%..0E"5#%innf=### # + #K#''3 8H8H8M8MPQ8Q#**;7 8'O.P PST TWX BJJsBLL$56CVYIZ9[[^__bcdeJsQ39q=1IsQ39q=1I   H %yx'@)//RZB[ [[rn?ct||}t||}t|||\}}}|tj|tz |kD||j kzS)zacompute if each element from bboxes1 is almost a subregion of one or more elements in bboxes2rwrzrrrr EPSILON_AREAT) bboxes1bboxes2 thresholdrwrrrrrs r)&bboxes1_is_almost_subregion_of_bboxes2rsd %Wx@G$Wx@G'K8($J 9 I|< F'K($J 9 L)ikk2IJ2VW W[d ddrncddlm}ddlm}|j}t t ||D]\}\}}|j} |j} | jd} | jd} | | f} i}t|j|rd|jjvrddd}|d | || d |}tttd |t }g}|D]S}|j"t%td || }n |j"}t'||_|j)|U||jd d |S)z1Merge an inferred layout with an extracted layoutr)+merge_inferred_layout_with_extracted_layout)UnstructuredDetectronONNXModelr0r1R_50r)same_region_thresholdsubregion_threshold)inferred_layoutr'page_image_sizerN)ra pdf_objects).unstructured_inference.inference.layoutelementr,unstructured_inference.models.detectron2onnxrpagesr9zipelementsimage_metadatagetr>detection_model model_pathrr rrr4 aggregate_embedded_text_by_blockrrA)inferred_document_layoutr'hi_res_model_name"merge_inferred_with_extracted_pagerinferred_pagesry inferred_pageextracted_page_layoutrrwh image_sizethreshold_kwargs merged_layoutr layout_elr4s r)$merge_inferred_with_extracted_layoutrsm\-33N5> N,-6'-1 1M0(00&55   w '   x (V  }446T Um;;FFF9VWX>YZ   + < O;&d( s"D$D$>D)cg}t|D] \}}|j|j"t||}g}t|D]1\}}|||dzdfj r!|j|3|S)zMRemoves duplicate text elements extracted by PDFMiner from a document layout.r|N)r9rAr3rany)rrrvryrioufiltered_elementss r)rGrG:sF)$ 7 gll#$  +C)* 7 q!a%'z?     )* rncLt|Dcgc]}|jc}|jgtjj d}dj t |Dcgc]$\}}||s |js|j&c}}}|Scc}wcc}}w)zgExtracts the text aggregated from the elements of the given layout that lie within the given block.r|r} )rr3rrrjoinr9r4)rarrUmaskryr4s r)rrPs 2()c)   @@  cqck  88Ik,B]&!StAwSVS[S[SXX] ^D K * ^sB( B 6 B B c B|Dcgc]}t|jd}}t||g}t|Dcgc]L\}}t |r<||jd||jd||jddN}}}|Scc}wcc}}w)Nr3r4r7r6)r4r7r6)rrrr9r) page_linksregionlink links_bboxesresultsidxresultrfs r)get_links_in_elementras=GHTItxx/0HLH4\F8LG%W-   C v; sO''/c?&&u-%c?..}=  E  LI s BABct|trt||||S|j}|gSt||||S)a Extracts URI annotations from a single or a list of PDF object references on a specific page. The type of annots (list or not) depends on the pdf formatting. The function detectes the type of annots and then pass on to get_uris_from_annots function as a list. Args: annots (PDFObjRef | list[PDFObjRef]): A single or a list of PDF object references representing annotations on the page. height (float): The height of the page in the specified coordinate system. coordinate_system (PixelSpace | PointSpace): The coordinate system used to represent the annotations' coordinates. page_number (int): The page number from which to extract annotations. Returns: list[dict]: A list of dictionaries, each containing information about a URI annotation, including its coordinates, bounding box, type, URI link, and page number. )r>listget_uris_from_annotsresolve)r:r1rRrLresolved_annotss r)r;r;rsI.&$#FF4E{SSnn&O 9JK XXrnc g}|D]p}t|}t|ts |jdd}|rt|tst |dk7rS|jdd}|rt|tst |dk7rt||\} } } } | | f| | f| | f| | ff} t| |}d|vrt|d}t|tsd}d|vr!t|dtst |d}d} |d k(r&tt|d jd }|d k(r&tt|d jd }|j|| | | | f|||ds|S#t$rY+wxYw)a Extracts URI annotations from a list of PDF object references. Args: annots (list[PDFObjRef]): A list of PDF object references representing annotations on a page. height (int | float): The height of the page in the specified coordinate system. coordinate_system (PixelSpace | PointSpace): The coordinate system used to represent the annotations' coordinates. page_number (int): The page number from which to extract annotations. Returns: list[dict]: A list of dictionaries, each containing information about a URI annotation, including its coordinates, bounding box, type, URI link, and page number. SubtypeNz/'Link'Rectrp)pointssystemASz/'URI'URIzutf-8z/'GoTo'D) coordinatesr3typer5rL) try_resolver>dictrr strr=rrdecode ExceptionrA)r:r1rRrLrQ annotationannotation_dictsubtyperectrVrWrXrYrcoordinates_metadatauri_dicturi_typer5s r)rrs*O- %j1/40 !%%i6*Wi8CLI?FFwO9$!+hsm"<=DDWM 3RR( *   K- \    sAE;; FFcF |jS#t$r|cYSwxYw)z Attempt to resolve a PDF object reference. If successful, returns the resolved object; otherwise, returns the original reference. )rr)r]s r)rrs' }}  s   cg}|D]C}|d|k(s t|d}|st||d|z |kDs3|j|E|S)a Filter annotations that are within or highly overlap with a specified element on a page. Args: annotation_list (list[dict[str,Any]]): A list of dictionaries, each containing information about an annotation. element_bbox (tuple[float, float, float, float]): The bounding box coordinates of the specified element in the bbox format (x1, y1, x2, y2). page_number (int): The page number to which the annotations and element belong. annotation_threshold (float, optional): The threshold value (between 0.0 and 1.0) that determines the minimum overlap required for an annotation to be considered within the element. Default is 0.9. Returns: list[dict[str,Any]]: A list of dictionaries containing information about annotations that are within or highly overlap with the specified element on the given page, based on the specified threshold. rLr3)calculate_bbox_areacalculate_intersection_arearA)rQ element_bboxrLrSrZrannotation_bbox_sizes r)r?r?sn0"$%> m $ 3#6z&7I#J #+L*V:LMPdd&'+11*=> &%rnc g}g}d}|D]j}d}d\}}} } d} t|D]?\} } t| tr |j| | j }|r,|j s|j|||| | f| dd}g|s|j }|r>|j k7r+|j }|j|||| | f| dd}t|dk(r<|| z} | j}|| jz } | j} || jz }n| j} || jz } ||z }%|j|||| | f| dd}B|t|z }m||fS)a| Extracts characters and word bounding boxes from a PDF text element. Args: obj (LTTextBox): The PDF text element from which to extract characters and words. height (float): The height of the page in the specified coordinate system. Returns: tuple[list[LTChar], list[dict[str,Any]]]: A tuple containing two lists: - list[LTChar]: A list of LTChar objects representing individual characters. - list[dict[str,Any]]]: A list of dictionaries, each containing information about a word, including its text, bounding box, and start index in the element's text. r)NNNN)r4r3r6) r9r>r rAr2stripisalnumr=x0y0rVrW)rUr1 charactersr\text_len text_linewordrVrWrXrYr6index charactercharrs r)r@r@s"J EH+# /BB )) 4&  E9)V,!!), ))+ LL!%BB/?P[\D"llnGDLLNg5"llnGLL!%BB/?P[\Dt9>"*U"2K"B),,.B"B),,.B"B),,.B  !BB+;KXM& N C N"W+#X u rnc *t|dk(r d|d<d|d<|Stj|ddtj|Dcgc] }|dd c}z dz|ddtj|Dcgc] }|dd c}z dzz}tj|ddtj|Dcgc] }|dd c}z dz|dd tj|Dcgc] }|dd  c}z dzz}t |}t |}d}||k\r%t ||dzD]}|d z }|||dz }n||d}|j |d<||d|d<|Scc}wcc}wcc}wcc}w) aq Maps a bounding box annotation to the corresponding text and start index within a list of words. Args: words (list[dict[str,Any]]): A list of dictionaries, each containing information about a word, including its text, bounding box, and start index. annot (dict[str,Any]): The annotation dictionary to be mapped, which will be updated with "text" and "start_index" fields. Returns: dict: The updated annotation dictionary with "text" representing the mapped text and "start_index" representing the start index of the mapped text in the list of words. rrr4r6r3r|r)r=rrsqrtarray try_argminranger) r\r]rdistance_from_bbox_startdistance_from_bbox_end closest_start closest_endr4r[s r)rBrBKs 5zQf !m !ww vq BHH%%H$d6l1o%HI IaO = bhhE'JDV Q'JK KPQ Q R  WW vq BHH%%H$d6l1o%HI IaO = bhhE'JDV Q'JK KPQ Q R78M34K Dm#}kAo6 %A CKD E!HV$ $D %]#F+JJLE&M / >E- L+&I'J&I'JsF8FF 6Fc|\}}}}|\}}}} t||} t||} t||} t|| } | | kr| | krt| | | | f}|Sy)a Calculate the area of intersection between two bounding boxes. Args: bbox1 (tuple[float, float, float, float]): The coordinates of the first bounding box in the format (x1, y1, x2, y2). bbox2 (tuple[float, float, float, float]): The coordinates of the second bounding box in the format (x1, y1, x2, y2). Returns: float: The area of intersection between the two bounding boxes. If there is no intersection, the function returns 0.0. g)maxminr)bbox1bbox2x1_1y1_1x2_1y2_1x1_2y1_2x2_2y2_2x_intersectiony_intersectionx2_intersectiony2_intersectionintersection_areas r)rrvs}"#D$d"D$dt_Nt_N$oO$oO'N_,L/ ^_o N ! rnc*|\}}}}||z ||z z}|S)a( Calculate the area of a bounding box. Args: bbox (tuple[float, float, float, float]): The coordinates of the bounding box in the format (x1, y1, x2, y2). Returns: float: The area of the bounding box, computed as the product of its width and height. r)r3rVrWrXrYrFs r)rrs(NBB GR D Krnc^ ttj|S#t$rYywxYw)a; Attempt to find the index of the minimum value in a NumPy array. Args: array (np.ndarray): The NumPy array in which to find the minimum value's index. Returns: int: The index of the minimum value in the array. If the array is empty or an IndexError occurs, it returns -1. r )intrrargmin IndexError)rs r)rrs-299U#$$ s  ,,)r)r%rr#r*return+tuple[List[List['TextRegion']], List[List]])Nr-)r"z Optional[Union[bytes, BinaryIO]]r#r*r.r/)rwr*r. np.ndarray)rr0rr0rwr*)rfloatrwr*r.r0)r'DocumentLayout'r'zList[List['TextRegion']]rrr.r2)rr2r.r2)r)rlist['TextRegion']rr1r.r3)raz 'TextRegion'rr3r.r)rrrrr.r) r:zPDFObjRef | list[PDFObjRef]r1r1rRPixelSpace | PointSpacerLr*r.list[dict[str, Any]]) r:zlist[PDFObjRef]r1z int | floatrRr4rLr*r.r5)r]r ) rQr5r!tuple[float, float, float, float]rLr*rSr1r.r5)rUr r1r1r.z)tuple[list[LTChar], list[dict[str, Any]]])r\r5r]zdict[str, Any])rr6rr6r.r1)r3r6r.r1)rr0r.r*)D __future__rtypingrrrrrr r numpyrrpdfminer.layoutr r pdfminer.pdftypesr pdfminer.utilsrr8r"unstructured.documents.coordinatesrrunstructured.documents.elementsr0unstructured.partition.pdf_image.pdf_image_utilsr/unstructured.partition.pdf_image.pdfminer_utilsrrrr#unstructured.partition.utils.configr&unstructured.partition.utils.constantsrr$unstructured.partition.utils.sortingrunstructured.utilsrr'unstructured_inference.inference.layoutrr DEFAULT_ROUNDr*r$rDrzrrrrrrGrrr;rrr?r@rBrrrrrnr)rGs"LLL-'(?E?V ;JB4DF   / /  /1 //0-1b" *b" b"1b"1b"J 4A"?L \  \", \8; \"*-m!&8;"/2=e/06$.6$.6$6$ 6$16$r$N/0 1*# ""Y 'Y Y/Y Y  Y>D D D/D D  DN!&)!&3!&!& !&  !&HA A A/AH(V , , D  rn