"#hhzdZddlZddlZddlmZddlmZddlmZddlm Z m Z ddl m Z m Z mZmZmZddlmZmZd d lmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9d d l:m;Z;mm?Z?m@Z@mAZAmBZBd d lCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLejeNZOdedePfdZQhdZRhdZShdZThdZUddhZVeUjeVZXhdZYejdZ[e'e(fe)e*ffZ\dePde?de ePfdZ]dedePde?de ePfdZ^dede?de ePfdZ_d eePd!eePd"eePde?de e ePf d#Z`d$ePd%eePd!eePd"eePde?de e ePf d&Zae e;'d(ebdePde?debfd)Zcdede?de ePfd*Zddede?de ePfd+Zed,e e ePdePfd-Zfd$ePde?de ePfd.Zg d9d/eehePefd0eid1eid2ePd3e ePd4eid5e eeePfd6e eeePfd7eide ePfd8Zjy):zZModule bundling all functions needed to determine the date of HTML strings or LXML trees. N)Counter)deepcopy)datetime) lru_cachepartial)MatchOptionalPatternUnionr) HtmlElementtostring)%discard_unwantedextract_url_dateidiosyncrasies_search img_search json_search regex_parsepattern_search try_date_exprDATE_EXPRESSIONS FAST_PREPEND SLOW_PREPENDFREE_TEXT_EXPRESSIONSMAX_SEGMENT_LENMIN_SEGMENT_LEN YEAR_PATTERN YMD_PATTERNCOPYRIGHT_PATTERNTIMESTAMP_PATTERN THREE_PATTERN THREE_CATCHTHREE_LOOSE_PATTERNTHREE_LOOSE_CATCHSELECT_YMD_PATTERNSELECT_YMD_YEARYMD_YEARDATESTRINGS_PATTERNDATESTRINGS_CATCHSLASHES_PATTERN SLASHES_YEARYYYYMM_PATTERN YYYYMM_CATCHMMYYYY_PATTERN MMYYYY_YEARSIMPLE_PATTERNTHREE_COMP_REGEX_ATHREE_COMP_REGEX_BTWO_COMP_REGEX) CACHE_SIZE CLEANING_LISTMAX_POSSIBLE_CANDIDATES) Extractor clean_html load_html trim_text) check_extracted_referencecompare_valuesfilter_ymd_candidate get_min_date get_max_date is_valid_dateis_valid_formatplausible_year_filtervalidate_and_convertelementreturnc:t|ddjS)z,Format the element to be logged to a string.Funicode pretty_printencoding)r strip)rDs J/var/www/html/sandstorm/venv/lib/python3.12/site-packages/htmldate/core.py logstringrMKs G%) D J J LL>Edc.datedc:date bt:pubdate dc.created dc:created og:pubdate og:regdate dcterms.date publish-date sailthru.datedc.date.issueddcterms.issuedpublished-datearticle.createddc.date.createddcterms.createdog:publish_dateog:datepublishedparsely-pub-datetwt-published-atarticle.publishedarticle:post_datearticle:publishedog:published_timevr:published_timevideo:release_datedc.date.publicationrnews:datepublishedfield-name-post-dateog:article:publishedarticle:published_datearticle:published_timearticle:publicationdateog:article:published_timeog:question:published_timecxenseparse:recs:publishtimedcsext.articlefirstpublishedanalyticsattributes.articledate"shareaholic:article_published_timedatemetapdateptimecreatedgentimepubdatedoc_date rbpubdate timestamp dateposted datecreated displaydate publishdate date_created publish_date publish_time release_date citation_date datepublished publisheddatedate_publishedpublished_datepublished_timepublication_datecontent_create_datearticle_date_originalmediator_published_timeoriginalpublicationdatecitation_publication_date>utimelastmodlastdatemodified lastmodified last-modified> bt:moddate dc.modifiedog:updated_timearticle:modifieddcterms.modifiedog:modified_timearticle:modified_datearticle:modified_timearticle:post_modifiedog:article:modified_time datemodifiedrr updated_time modified_time revision_datemodificationdate>pubyearrrr dateupdate>date-publishedtime published publishedz\D+$textoptionsct|}t|tkrytj d|dt }t ||j|j|j|jS)z'Prepare text and try to extract a date.N) r:lenrNON_DIGITS_REGEXsubrrformat extensiveminmax)rrs rL examine_textrs` T?D 4yO#   D)9/$: ;D  gnng//gkk rNtree expressionc|j|}|rt|tkDry|D]=}|j|j ddfD]}t ||}|s|ccS?y)z3Check HTML elements one by one for date expressionsNtitler)xpathrr6 text_contentgetr)rrrelementselemrattempts rLexamine_date_elementsrst zz*%H s8}'>>&&($((7B*?@ D"41G  rNc d\}}tt|j|j|j|j }|j dD]U}|jrd|jvrd|jvr-d|jvr|jddj}|dk(rt|jd|}n|tvr8tjd t|||jd}n|tvrtjd t||j s||jd}nH||jd}n/d |jvr|jd dj}|tvs |t"vrtjd t|||jd}||tvr |j s|t"vr|j s|}n|}nd |jvr"|jd dj}|t$vrztjd t|||jdxs|jd}||t&vr |j s|t(vr|j s|}n|dk(rtjd t|d|jvrdj+|jddddg}t-|d|j|j rZ|}nVd|jvr[|jddjdk(r$tjdt|||jd}nd|jvr|jddj}|dk(r[tjdt||j r||jd}nw||jd}n_|dk(rZtjdt||j s||jd}n||jd}|Vn||tjd|}|S)a Parse header elements to find date cues :param tree: LXML parsed tree object :type tree: LXML tree :param options: Options for extraction :type options: Extractor :return: Returns a valid date expression as a string, or None )NN) outputformatextensive_searchmin_datemax_datez.//metacontentrnamerzog:urlzexamining meta name: %spropertyzexamining meta property: %sitempropzexamining meta itemprop: %s copyrightyear-01%Y-%m-%dearliestlatestr|zexamining meta pubdate: %sz http-equivrvzexamining meta http-equiv: %srz-opting for reserve date with less granularity)rrrrrriterfindattribrlowerrDATE_ATTRIBUTESLOGGERdebugrM NAME_MODIFIEDoriginalPROPERTY_MODIFIEDITEMPROP_ATTRSITEMPROP_ATTRS_ORIGINALITEMPROP_ATTRS_MODIFIEDjoinr@)rr headerdatereservetryfuncr attributers rLexamine_headerrs %J^^ ** G i(V  +$++-  T[[ ,224IH$*488I+>Ho- 6 $H$TXXi%89 m+ 6 $H''!()) L  3U1XuQxH L/s>G! htmlstringrcft||||j|j}t||||S)z)Chained candidate filtering and selectionrrrr)rBrrr)rrrrr candidatess rLsearch_patternr s8'{{ J Jw @@rN)maxsize referencect||j|j|j|j}| t |||S|S)z[Compare candidate to current date reference (includes date validation and older/newer test))rrrrrr<)r rrrs rLcompare_referencer sHGNNG$5$5w{{GKKGi':: rNc|jd}dt|cxkr tkrny d}|D]}d|jvra t |j dd}tjd||jr |dk(s||kr|}]|jrj||kDsp|}s|j dtvsd|jvr|j d}tjd||jr?t||j|j|j|j}||cSt!|||}|dkDsnb|j"s/t|j"d kDsItjd |j"t!||j"|}t%||xs t'|d|Sy #t $rYwxYw) zTScan the page for abbr elements and check if their content contains an eligible datez.//abbrrz data-utimerzdata-utime found: %sclassrzabbr published-title found: %sNrzabbr published found: %s)findallrr6rrr ValueErrorrrr CLASS_ATTRSrrrrrr rr;r)rrrr r candidatetrytextrs rLexamine_abbr_elementsrs ||I&H3x=222` _ ' QDt{{* #DHH\2$> ?I 3Y?##a9y;P )I ))i).C )I'"k1dkk)"hhw/GLL!A7K''"/##NN#--#KK#KK ##.#*N$5i'$R $q=!YY3tyy>B#6LL!;TYYG 1)TYY PIO' QR)G< @U   A  S"sG GGcF|jd}dt|cxkr tkrnyd}|D]}d}|jdd}t|dkDr[d|jvr9|jddk(r%|j rd}t jd |nd |jvr|j r[|jd djd s!|jd djd rd}t jd |nO|j sC|jd dk(r/d}t jd|nt jd||r?t||j|j|j|j}|o|cSt|||}|jt|jdkDst jd|jt||j|}t!||Sy)zTScan the page for time elements and check if their content contains an eligible datez.//timerFrrr|Tz#shortcut for time pubdate found: %srz entry-datez entry-timez$shortcut for time/datetime found: %supdatedz,shortcut for updated time/datetime found: %sztime/datetime found: %sNztime/datetime found in text: %s)rrr6rrrrr startswithrrrrrr rr;)rrrr r shortcut_flag datetime_attrrs rLexamine_time_elementsrs ||I&H3x=222p m 1 MD!M HHZ4M=!A%,+y8(($(MLL!F V +''"-88F88GR0;;LI(,  BM%--$((72Cy2P(,  J) LL!:MJ +%))  G*& 1)]G TI&3tyy>A+= > J-iGL c1 Mh)G<< rNrcd|jD\}}}t|dk(r|ddk(rd|nd|}|d|d|S)zoNormalize string output by adding "0" if necessary, and optionally expand the year from two to four digits.c3DK|]}|s|jdyw)rN)zfill).0gs rL z"normalize_match..8s@qa @s  rr91920r)groupsr)rdaymonthrs rLnormalize_matchr)5s[AELLN@C 4yA~"1gnD6{Btf+V1UG1SE ""rNc  tjdd}t|ttt|}|Tt |d}t t|ddd|j|jrtjd||}tjdtD]]}t||d|dt|}t||d|j||j|j|j}|[|cSt|tt |j|j }i}|D]*} t#j$| } t'| } || || <,t)|}t+|t,t.|}t|t|j||j|j|j}||St|t0t2t|}t|t0|j||j|j|j}||St|t4t6|j|jd }i}|D]*} t9j$| } t'| } || || <,t)|}t+|t,t.|}t|t4|j||j|j|j}||Stjd t|t:t<t|}|tt |dt |d d} |dk(s| j>|k\rStA| |j|j|j}|$tjdt:|d|d |St|tBtD|j|j|j }i}|D]N} tGj$| } | d} tI| dk(rd| } djK| d | dg} || || <Pt)|}t+|t,t.|}t|tB|j||j|j|j}||StM|} |dk(s| r@| j>|k\r1tA| |j|j|j}||S|dk7rFtjdtt |dd} | jO|jStjdt|tPtt|}|tt |ddd} t | d|j|jrH| j>|k\r9tjdtP|d| jO|jSy)a Opportunistically search the HTML text for common text patterns :param htmlstring: The HTML document in string format, potentially cleaned and stripped to the core (much faster) :type htmlstring: string :param options: Define extraction options :type options: Extractor :return: Returns a valid date expression as a string, or None z(looking for copyright/footer informationrNrrrz'copyright year/footer pattern found: %sz 3 componentsrT)rrrr incompletezswitching to two componentsrz#date found for pattern "%s": %s, %s0rrzusing copyright year as defaultzswitching to one componentrzdate found for pattern "%s": %s))rrr rrrr@rrrTHREE_COMP_PATTERNSr=rrrBr%r&r1rr)rrrr'r(r)r*r+r2r,r-rrCr.r/r3rrrstrftimer0)rrcopyear bestmatchrrresultr replacementitemrr dateobjectr(s rL search_pager5>s  LL;<G I9Q<   T1a $W[[  LLBD IG LL ("  QK QK    &  QK     NN KK KK   M%*'"{{ JK2"((.#E* !+D!1 I2%J [(GLI !  F  I"  F '{{ JK2"((.#E* !+D!1 I2%J [(GLI !  F  LL./ Ic)A,/Yq\1BAF a<:??g5)GNNW[[F! 9"aLaL   '{{## JK2$$T*a u:?wKEHHeAht45 !+D!1 I 2%J [(GLI !  F Z(J!| z''A% W[[   M!| 67c'lAq1 ""7>>22 LL-. Ic)A,/A6 JW[[ 7* LL1>9Q< &&w~~6 6 rN htmlobjectr original_daterurlverboserrdeferred_url_extractorc |r$tjtjt|} | y|dk7r t |syt |t |t|||} d} |$| jd} | | jd}t|| } | |s| St| | xs t| | } | | S|r| | St| | }||S ttt!| t"\}}|rt*t,z}n t.t,z}t1||| xst1|d| xs t3|| } | | S t5|dd }t;|t<| xst?|| xs tA|| } | | S|rt&jCdd}tE|D]>}|jG}tHtK|cxkr tLksn2tO||| }@tQ|| }|xs tS|| Sy#t$$r| }t&j)dY0wxYw#t6$r!t5|d j9d d }YwxYw)a Extract dates from HTML documents using markup analysis and text patterns :param htmlobject: Two possibilities: 1. HTML document (e.g. body of HTTP request or .html-file) in text string form or LXML parsed tree or 2. URL string (gets detected automatically) :type htmlobject: string or lxml tree :param extensive_search: Activate pattern-based opportunistic text search :type extensive_search: boolean :param original_date: Look for original date (e.g. publication date) instead of most recent one (e.g. last modified, updated time) :type original_date: boolean :param outputformat: Provide a valid datetime format for the returned string (see datetime.strftime()) :type outputformat: string :param url: Provide an URL manually for pattern-searching in URL (in some cases much faster) :type url: string :param verbose: Set verbosity level for debugging :type verbose: boolean :param min_date: Set the earliest acceptable date manually (ISO 8601 YMD format) :type min_date: datetime, string :param max_date: Set the latest acceptable date manually (ISO 8601 YMD format) :type max_date: datetime, string :param deferred_url_extractor: Use url extractor as backup only to prioritize full expressions, e.g. of the type `%Y-%m-%d %H:%M:%S` :type deferred_url_extractor: boolean :return: Returns a valid date expression as a string, or None )levelNrz.//link[@rel="canonical"]hrefzlxml cleaner errorz.//title|.//h1FrGrH)rIzutf-8ignorezextensive search startedr)*logging basicConfigDEBUGr9rAr7r?r>findrrrrrrr8rr5rrerrorrrrrrr UnicodeDecodeErrordecoderr rrrrrKrrrr r;r5)r6rr7rr8r9rrr:rr url_resulturlelemr1 abbr_result search_tree discarded date_exprrr segment converteds rL find_daterN(sd'--0 Z D |z!/,*GXX GJ {))78  ++f%C"#w/J&<D' * Hk$.HF  *"8( K+!1 x~} 5"  Y #33  #33     7 !    7 !g 6  Yk R  z#4g> 6 k7 + 6 W 5   /0 ,[9 GGmmoG"S\COC))WgFI  G .iA EEgxX Ys$%HH/ H,+H,/&II)TFrNFNNF)k__doc__r?re collectionsrcopyrr functoolsrrtypingrr r r Counter_Type lxml.htmlr r extractorsrrrrrrrrrrrrrrrrrr r!r"r#r$r%r&r'r(r)r*r+r,r-r.r/r0r1r2r3settingsr4r5r6utilsr7r8r9r: validatorsr;r<r=r>r?r@rArBrC getLogger__name__rstrrMrrrrrunionrrcompilerr-rrrrr rr rrr)r5bytesboolrNrNrLrcs (KK+&&&&&&&&&&NIH>>      8 $M{MsM HV *F)<8(../FG? 2::g&K +,     c]   c] (u u uc]up4c"4 3<4S\4 4 eCj 4nAA S\A 3<AS\ A  A eCj A$ :      6 6 6c]6r> > >c]>B#8E#J/#C#gCg)g gX""/3/3#(oeS+-.ooo o # o  ouXs]+,ouXs]+,o!oc]orN