:Qgi4ddlZddlZddlZddlZddlZddlZddlZddlmZddl m Z ddl Z ddl mZddlmZddlmZddlmZddlmZdd lmZdd lmZdd lmZmZmZmZmZmZdd l m!Z!m"Z"dd l#m$Z%ddl#m&Z&m'Z'm(Z(e)e&e'ze(zZ*e*jWddZ,dZ-dZ.gdZ/gdZ0d.dZ1d.dZ2Gdde3Z4dZ5dZ6dZ7dZ8dZ9dZ:dZ;d ed e?fd#Z@d e?fd$ZAd/d%ZBd0d&ZCd0d'ZDd1d(ZE d2d)ZFd*ZGd+ZH d3d,ZId4d-ZJy)5N)groupby) itemgetter) PDFParser) PDFDocument)PDFPage)PDFTextExtractionNotAllowed)PDFResourceManager)PDFPageInterpreter)PDFPageAggregator)LAParamsLTAnnoLTCharLTTextLineHorizontalLTTextLineVerticalLTImage)Requesturlopen)urlparse) uses_relative uses_netloc uses_paramscX t|jtvS#t$rYywxYw)zCheck to see if a URL has a valid protocol. Parameters ---------- url : str or unicode Returns ------- isurl : bool If url has a valid protocol return True otherwise False. F) parse_urlscheme _VALID_URLS Exception)urls J/var/www/html/answerous/venv/lib/python3.12/site-packages/camelot/utils.pyis_urlr (s.~$$ 33 s  ))cd}|rP|tjtjtjztj zz }|dz}|rP|S)Nr)randomchoicestringdigitsascii_lowercaseascii_uppercase)lengthrets r random_stringr+;sS C  v}} MMF22 2V5K5K K   !   Jctdd}tjdd5}ddi}t|d|}t |}|j j }|d k7r td |j|jdddtjjtjjj|}tj |j||S#1swYrxYw) zDownload file from specified URL. Parameters ---------- url : str or unicode Returns ------- filepath : str or unicode Temporary filepath. z.pdfwbF)deletez User-Agentz Mozilla/5.0Nzapplication/pdfzFile format not supported)r+tempfileNamedTemporaryFilerrinfoget_content_typeNotImplementedErrorwritereadospathjoindirnamenameshutilmove)rfilenamefheadersrequestobj content_typefilepaths r download_urlrFEs "#4(H  $ $T% 8A/#tW-gxxz224 , ,%&AB B  ww||BGGOOAFF3X>H KK! Os A*DD )columnsedge_tolrow_tol column_tol) process_background line_scale copy_text shift_textline_tol joint_tolthreshold_blocksizethreshold_constant iterations resolutioncPfd}dk(r|t|y|t|y)Nct|jt|j}|r)tdj t |ddy)N,z cannot be used with flavor='')set intersectionkeys ValueErrorr:sorted) parser_kwargs input_kwargsisecflavors rcheck_intersectionz*validate_input..check_intersectionpsZ=!..s<3D3D3F/GH 88F4L)**GxqQ  r,lattice) stream_kwargslattice_kwargs)kwargsrarbs ` rvalidate_inputrgos&=&1>62r,c|dk(r1|jD]}|tvs |j||S|jD]}|tvs |j||S)Nrc)r[rdpopre)rfrakeys r remove_extrark}sj ;;= Cm# 3  M;;= Cn$ 3  Mr,ceZdZdZdZy)TemporaryDirectorycLtj|_|jSN)r1mkdtempr<)selfs r __enter__zTemporaryDirectory.__enter__s$$& yyr,cBtj|jyro)r=rmtreer<)rqexc_type exc_value tracebacks r__exit__zTemporaryDirectory.__exit__s dii r,N)__name__ __module__ __qualname__rrrxr,rrmrms !r,rmc||z }|S)zTranslates x2 by x1. Parameters ---------- x1 : float x2 : float Returns ------- x2 : float r|)x1x2s r translaters"HB Ir,c||z}|S)zScales x by scaling factor s. Parameters ---------- x : float s : float Returns ------- x : float r|)xss rscalersFA Hr,c$|\}}}}|\}}}t||}ttt| ||}t||}ttt| ||}t|t|t|t|f} | S)ayTranslates and scales pdf coordinate space to image coordinate space. Parameters ---------- k : tuple Tuple (x1, y1, x2, y2) representing table bounding box where (x1, y1) -> lt and (x2, y2) -> rb in PDFMiner coordinate space. factors : tuple Tuple (scaling_factor_x, scaling_factor_y, pdf_y) where the first two elements are scaling factors and pdf_y is height of pdf. Returns ------- knew : tuple Tuple (x1, y1, x2, y2) representing table bounding box where (x1, y1) -> lt and (x2, y2) -> rb in OpenCV coordinate space. )rabsrint) kfactorsr~y1ry2scaling_factor_xscaling_factor_ypdf_yknews r scale_pdfrs.NBB07-& r# $B s9eVR()+; zget_rotation..+s0K a;AHHQK!O;K02c3hK|]*}|jddkDxr|jddk,ywrrrs rrzget_rotation..,s0OAAHHQK!O? a?Or anticlockwise clockwise)lenget_textstripsum) charshorizontal_text vertical_textrotationrhlenvlenrrs r get_rotationrs*H ?Caajjl.@.@.BC DD =AaAJJL,>,>,@A BD d{KUKK OOO &/-&??[ O DAs#B'B'#B,(B,cf|d|df}|d|df}|Dcgc]=}|d|ddz kDr-|d|ddzkr|ddz |dcxkr |ddzkrnn|?}}|Dcgc]=}|d|ddz kDr-|d|ddzkr|ddz |dcxkr |ddzkrnn|?}}||fScc}wcc}w)a6Returns all line segments present inside a bounding box. Parameters ---------- bbox : tuple Tuple (x1, y1, x2, y2) representing a bounding box where (x1, y1) -> lb and (x2, y2) -> rt in PDFMiner coordinate space. v_segments : list List of vertical line segments. h_segments : list List of vertical horizontal segments. Returns ------- v_s : list List of vertical line segments that lie inside table. h_s : list List of horizontal line segments that lie inside table. rr"rrr|) bboxrrlbrtrv_srh_ss rsegments_in_bboxr1s , q'47 B q'47 B  Q4"Q%!) !r!uqy 0RUQY!A$5S"Q%RS)5S  C   Q4"Q%!) !r!uqy 0RUQY!A$5S"Q%RS)5S  C  8O   s AB)!AB.c6|d|df}|d|df}|Dcgc]l}|ddz |j|jzdz cxkr |ddzkr9nn6|ddz |j|jzdz cxkr |ddzkrnn|n}}|Dchc]}|}}|D]d}|j D]O}||k(r t ||st ||t|z dkDs2t||s?|j|Qft|} | Scc}wcc}w)aReturns all text objects present inside a bounding box. Parameters ---------- bbox : tuple Tuple (x1, y1, x2, y2) representing a bounding box where (x1, y1) -> lb and (x2, y2) -> rt in the PDF coordinate space. text : List of PDFMiner text objects. Returns ------- t_bbox : list List of PDFMiner text objects that lie inside table, discarding the overlapping ones rr"rr@g?) x0r~y0rcopybbox_intersectbbox_intersection_area bbox_area bbox_longerdiscardlist) rtextrrrt_bboxrestbabb unique_boxess r text_in_bboxrVs9" q'47 B q'47 B a519+ 8r!uqy 8 qEAI!$$+, 91 9 F !A D )))+ )BRxb"%*2r2Yr]BcI"2r* R( )):L ) s A1D Dreturnc2t|j|j}t|j|j}t|j|j}t|j |j }||ks||kDry||z ||z z}|S)a.Returns area of the intersection of the bounding boxes of two PDFMiner objects. Parameters ---------- ba : PDFMiner text object bb : PDFMiner text object Returns ------- intersection_area : float Area of the intersection of the bounding boxes of both objects )maxrminrr~r)rrx_lefty_topx_righty_bottomintersection_areas rrrs F ruu E"%%G255"%% H8e+ 6)eh.>? r,ch|j|jz |j|jz zS)zReturns area of the bounding box of a PDFMiner object. Parameters ---------- bb : PDFMiner text object Returns ------- area : float Area of the bounding box of the object r~rrr)rs rrrs' EEBEEMbeebeem ,,r,c|j|jk\xrO|j|jk\xr4|j|jk\xr|j|jk\S)aReturns True if the bounding boxes of two PDFMiner objects intersect. Parameters ---------- ba : PDFMiner text object bb : PDFMiner text object Returns ------- overlaps : bool True if the bounding boxes intersect rrrs rrrsI 55BEE> Rbeeruun R"%% RBEERUUNRr,ch|j|jz |j|jz k\S)a3Returns True if the bounding box of the first PDFMiner object is longer or equal to the second. Parameters ---------- ba : PDFMiner text object bb : PDFMiner text object Returns ------- longer : bool True if the bounding box of the first object is longer or equal )r~rrs rrrs' EEBEEMruuruu} --r,cg}|D]R}|s|j||d}tj|||r||zdz }||d<B|j|T|S)zMerges lines which are within a tolerance by calculating a moving mean, based on their x or y axis projections. Parameters ---------- ar : list line_tol : int, optional (default: 2) Returns ------- ret : list )atolr)rnpisclose)arrOr*atemps rmerge_close_linesrsh C   JJqMr7Dzz$1qC'B 1   Jr,c |s|Stjddjttj|dd|tj }|S)aStrips any characters in `strip` that are present in `text`. Parameters ---------- text : str Text to process and strip. strip : str, optional (default: '') Characters that should be stripped from `text`. Returns ------- stripped : str [r])flags)resubr:mapescapeUNICODE)rrstrippeds r text_striprsL  vv RWWSE* + ,A.D H Or,c|dk(rO|Dcgc]C}t|ts1|jtj|j dfE}}nS|dk(rN|Dcgc]C}t|ts1|jtj|j dfE}}Dcgc]\}}tj|d}}}tt|dkDr g}t|} t|tdD]\} } | | k(rv| Dcgc]}|d } }dj| js=| jdd| jd |jdj| | Dcgc]}|d } }dj| js|jdj| dj|} n-dj|Dcgc]}|jc}} t!| |Scc}wcc}wcc}}wcc}wcc}wcc}w) aFlags super/subscripts in text by enclosing them with . May give false positives. Parameters ---------- textline : list List of PDFMiner LTChar objects. direction : string Direction of the PDFMiner LTTextLine object. strip_text : str, optional (default: '') Characters that should be stripped from a string before assigning it to a cell. Returns ------- fstring : string horizontalr.)decimalsverticalr"rrzz) isinstancer rrroundheightwidthrrYrrrr:rinsertrr)textline direction strip_textrdrsizelflistmin_sizerjrfcharsfstrings rflag_font_sizer s&L  a(ZZ\288AHHq9 :  j  a(ZZ\288AGGa8 9  788 d$ #8A8 3q6{Qq6!!Z]3 2JCh(-.1!A$..776?((*MM!U+MM&)LL1(-.1!A$..776?((*LL1 2''%.'':A1::<:; gz **;  9 / / ;s%AH+AH0-!H5 H; IIc  d}g}|j} |dk(r|jst|jD cgc]\}} | d|dkr |d| dkr|} }} t|jD cgc]'\} } | d|d|dzdz cxkr| dkrnn| )} } } | d} | Dcgc]=}|j | |j s ||j | |jf?}}|s"| d|j | djfg}|jD]}|j| }|D]}t|tr|d|j|jzdz cxkr|dkr>nn;|j|jzdz |dkr|j| |d|f||dk(s|j| |ddz|ft|t s|j| |d|fn|dk(r|jst|jD cgc]\} }|d|dkr |d|dkr| }} }t|jDcgc]'\}}|d|d|dzdz cxkr|dkrnn|)}}}|d}|D cgc]=} |j | |j"s | |j | |jf?}} |s"|d|j d|jfg}|jD]}|j|}|D]}t|tr|d|j|jzdz cxkr|dkr>nn;|j|jzdz |dk\r|j|d||f||dk(s|j|ddz ||ft|t s|j|d||fg}t)|t+ddD]\}}|r8|j|d|dt-|Dcgc]}|d c}||f@|Dcgc]}|dj'}}|j|d|dt/d j1||f|Scc} }wcc} } wcc}wcc}} wcc}}wcc} w#t$$rdd|j'fgcYSwxYwcc}wcc}w) a9Splits PDFMiner LTTextLine into substrings if it spans across multiple rows/columns. Parameters ---------- table : camelot.core.Table textline : object PDFMiner LTTextLine object. direction : string Direction of the PDFMiner LTTextLine object. flag_size : bool, optional (default: False) Whether or not to highlight a substring using if its size is different from rest of the string. (Useful for super and subscripts.) strip_text : str, optional (default: '') Characters that should be stripped from a string before assigning it to a cell. Returns ------- grouped_chars : list List of tuples of the form (idx, text) where idx is the index of row/column and text is the an lttextline substring. rrrr"rrrrr)ris_empty enumeratecolsrowscellsrightr_objsrrrrrr~rr bottom IndexErrorrrrr rr:)tablerr flag_sizeridxcut_textrir x_overlaprrr_idxcx_cutsrCrowcuty y_overlapc_idxy_cutscol grouped_charsrjrrgcharss rsplit_textliner*6sE4 CH ==DD/  $X->->-@&ejj1AqQ447?tAw!A$I&ejj1AqQ4DGd1g-2:ad:E aA3<./ Aq@Q@W@WEKKN1%(()F$Q<Q);)>)>?@~~ :jjm! :C!#v.Fsvv!&;Es1vE!$#&&A 5Q ?$OOQA,<=! #fRj0 (CFQJ0D E#C0 CFC(89 : : * $X->->-@&ejj1AqQ447?tAw!A$I&ejj1AqQ4DGd1g-2:ad:E aA3<./ Aq@Q@X@XEKKN1%(()F$Q<R);)>)>?@~~ :jjm! :C!#v.Fsvv!&;Es1vE!$#&&A 5Q ?$OOSVQ,<=! #fRj0 (Q!Q0D E#C0 QC(89 : :$Mh 1a(89 U   FF"',-!1-yZ 055!admmo5F5  QQBGGFOZ!HI  i  ,  * /R**,-../. 6s/S"R3#S>,R9* S6!R? R?8C S+S/A S;"SS8,S $ S0!S S2C S=+S)S S8 S=3"SS54S5c Xdgdz\}}tt|jD]}|j|jzdz |j|dks4|j|jzdz |j|dkDsdg} |j D]} | d|j kr| d|jk\ry| d|jkr |jn| d} | d|j k\r |j n| d} | jt| | z t| d| dz z | jdtttd| dk(rx|jjd} |j|j f}|j dd|j ddf}tj| d|d ||}| j!t#| }ndgd z\}}}}|j|j|dkDr(t|j|j|dz }|j|j|dkr(t|j|j|dz }|j|j |dkr(t|j|j |dz }|j |j |dkDr(t|j |j |dz }t|j|j z d k(rd n!t|j|j z }t|j|jz d k(rd n!t|j|jz }||z}|||zz|||zzz|z }|rt%||||| |fS|r||t'|j(||fg|fS||t+|j|fg|fS)a}Gets indices of the table cell where given text object lies by comparing their y and x-coordinates. Parameters ---------- table : camelot.core.Table t : object PDFMiner LTTextLine object. direction : string Direction of the PDFMiner LTTextLine object. split_text : bool, optional (default: False) Whether or not to split a text line if it spans across multiple cells. flag_size : bool, optional (default: False) Whether or not to highlight a substring using if its size is different from rest of the string. (Useful for super and subscripts) strip_text : str, optional (default: '') Characters that should be stripped from a string before assigning it to a cell. Returns ------- indices : list List of tuples of the form (r_idx, c_idx, text) where r_idx and c_idx are row and column indices. error : float Assignment error, percentage of text area that lies outside a cell. +-------+ | | | [Text bounding box] | | +-------+ rrrrr"c |dk7S)Nrr|)rs rz!get_table_index..s br,  z does not lie in column range r?)rrr )rangerrrrrr~rrrrfilterrrwarningswarnindexrr*r rr)rrr split_textrrrr%rlt_col_overlaprleftrr text_range col_range y0_offset y1_offset x0_offset x1_offsetXYchareaerrors rget_table_indexrDsN4!8LE5 3uzz? # DD144K3 Aq!1 1qttadd{c6IEJJ M M 7  NZZ .Q4144N*NO"))"-  .40.ABCqHzz|))$/ddADD\ "ZZ]1-uzz"~a/@A  fAj\)G {SE"((^)<=E +034q.Iy)Yttejj""uzz%0334 ttejj""uzz%0334 ttejj""uzz%0334 ttejj""uzz%0334 144!$$;3&Cqtt ,>?J r,c t|d5}t|} t| } | jst d|t |||||||} t } t| | } t| | }tj| D]E}|j|| j}|jd}|jd}||f}GfcdddS#1swYyxYw)aMReturns a PDFMiner LTPage object and page dimension of a single page pdf. To get the definitions of kwargs, see https://pdfminersix.rtfd.io/en/latest/reference/composable.html. Parameters ---------- filename : string Path to pdf file. line_overlap : float char_margin : float line_margin : float word_margin : float boxes_flow : float detect_vertical : bool all_texts : bool Returns ------- layout : object PDFMiner LTPage object. dim : tuple Dimension of pdf page in the form (width, height). rbz Text extraction is not allowed: ) line_overlap char_margin line_margin word_margin boxes_flowdetect_vertical all_texts)laparamsrrN)openrris_extractablerr r r r r create_pages process_page get_resultr)r?rWrXrYrZr[r\r]r@parserdocumentr^rsrcmgrdevice interpreterpagelayoutrrdims rget_page_layoutrlGsD h 1v&&&-28*= %###!+ %&"7X>(&9 ((2 "D  $ $T *&&(FKKNE[[^F&/C  " s{3s CCC&c|dk(rt}n#|dk(rt}n|dk(rt}n |dk(rt}|g} |jD]0}t |r|j |!|t||z }2 |S#t$rY|SwxYw)aRecursively parses pdf layout to get a list of PDFMiner text objects. Parameters ---------- layout : object PDFMiner LTPage object. ltype : string Specify 'char', 'lh', 'lv' to get LTChar, LTTextLineHorizontal, and LTTextLineVertical objects respectively. t : list Returns ------- t : list List of PDFMiner text objects. charimagerr)ltype) rrrrrrrget_text_objectsAttributeError)rjrprLTObjectrCs rrqrqs&  '  # #' / !%y  << 8C#x( %c77  8 H   H s?A77 BB)rc)r)r)Fr)FFr)?r1rtg?rtTT)rnN)Kr8rr#r=r%r1r4 itertoolsroperatorrnumpyrpdfminer.pdfparserrpdfminer.pdfdocumentrpdfminer.pdfpagerrpdfminer.pdfinterpr r pdfminer.converterr pdfminer.layoutr r rrrrurllib.requestrr urllib.parserrrrrrYrrr r+rFrdrergrkobjectrmrrrrrrrrPrrboolrrrrr r*rDrNrTrlrqr|r,rrsU (,$8110,.@@-+- ;< B&6A   3 !!" "B96x>"J'Te4 -U - SdS".4."841+hsnHJbQJ<4;|% r,