#hZ dZddlZddlZddlmZmZmZmZddlm Z ddl m Z m Z ddl mZmZmZmZmZmZmZmZmZddlmZdd lmZdd lmZmZmZmZm Z ejBe"Z#ejHd ejJZ&ejHd ejJZ'ejHd ejJZ( d&de)de*de*dee)de*de*deee)e)ffdZ+ d'dddddddddde)dee)de*de*dee)de*de*de*de*dee)dee)dee)fdZ,ddddddd d!e)dee)d"ee)d#ee d$e*de*de*dee)deee)ee)ffd%Z-y)(z0 Core functions needed to make the module work. N)ListOptionalSetTuple)RobotFileParser) normalize_url scrub_url) basic_filter domain_filterextension_filteris_navigation_pageis_not_crawlable lang_filter path_filter type_filter validate_url)redirection_test) BLACKLIST)extract_domain get_base_urlfix_relative_urls is_external is_known_linkz ]+?>zhreflang=["\']?([a-z-]+)zhref=["\']?([^ ]+?)(["\' >])FTurlstrictwith_redirectslanguagewith_navtrailing_slashreturnc t|durtjd|tt |}|r t |}t |||durtjd|t|,t||||durtjd|tt|\}}|durtjd|tt|jdurtjd|tt|jdurtjd |t|r>t|j|jdurtjd |tt||||}|rt!|t"d }n t!|d }|tjd |y ||fS#t$tf$rtjd|YywxYw)a(Check links for appropriateness and sanity Args: url: url to check strict: set to True for stricter filtering with_redirects: set to True for redirection test (per HTTP HEAD request) language: set target language (ISO 639-1 codes) with_nav: set to True to include navigation pages instead of discarding them trailing_slash: set to False to trim trailing slashes Returns: A tuple consisting of canonical URL and extracted domain Raises: ValueError, handled in exception. Fzrejected, basic filter: %s)rrzrejected, type filter: %sNzrejected, lang filter: %szrejected, validation test: %szrejected, extension filter: %szrejected, domain name: %szrejected, path filter: %sT) blacklistfast)r$zdiscarded URL: %s)r LOGGERdebug ValueErrorr rrrrr pathr netlocrqueryr rrAttributeError) rrrrrr validation_test parsed_urldomains I/var/www/html/sandstorm/venv/lib/python3.12/site-packages/courlan/core.py check_urlr0*s4>   % LL5s ; n "3'C s6H = F LL4c :   C6>BeK LL4c : '33&7# e # LL8# >  JOO , 5 LL93 ?  ** +u 4 LL4c :  k*//:3C3CDM LL4c : J.I #C94HF#Cd3F > LL4c :  ; J ' (#.sF7F??%G'&G') no_filterrrr r redirects referencebase_url pagecontent external_boolr1r2r3r4c 6| r tdt|} |xs| }tt} } |s| S| xs| } dtj |DD]} d| vrd| vr |durh|fd| vrbt j | }|s.|dj|s |ddk(sKtj | }|sc| j|dxtj | }|s| j|d| D]l} | jd s t|| } |dur+t| ||||| }|8|d } |t| | d k7rOt| | r\| j| ntjdt!| t!| | S)azFilter links in a HTML document using a series of heuristics Args: pagecontent: whole page in binary format url: full URL of the original page external_bool: set to True for external links only, False for internal links only no_filter: override settings and bypass checks to return all possible URLs language: set target language (ISO 639-1 codes) strict: set to True for stricter filtering trailing_slash: set to False to trim trailing slashes with_nav: set to True to include navigation pages instead of discarding them redirects: set to True for redirection test (per HTTP HEAD request) reference: provide a host reference for external/internal evaluation Returns: A set containing filtered HTTP links checked for sanity and consistency. Raises: Nothing. ,'base_url' is deprecated, use 'url' instead.c3&K|] }|d yw)rN).0ms r/ z extract_links..sF!1FsrelnofollowFhreflangrz x-defaulthttp)rr rrrrT)rr3 ignore_suffixu!%s links found – %s valid links)r'rsetFIND_LINKS_REGEXfinditerHREFLANG_REGEXsearch startswith LINK_REGEXaddrr0rrr%infolen)r5rr6r1rrr rr2r3r4 candidates validlinkslink langmatch linkmatchcheckeds r/ extract_linksrSsDGHHC H /C UCE J %XIG/88EF- D=Z4/   ("6:;M&--d3I! ''1Yq\[5P&--d3 NN9Q<0#))$/Iy|,!-&v&$S$/D  -!(! G1:D IT! z * t14 KK3S_c*oV )langrulesexternalrrr4 htmlstringrUrVrWc|r tdgg} }t||||||D]P} t| s||jd| s#t | r| j | @|j | R|| fS)zPFind links in a HTML document, filter and prioritize them for crawling purposes.r8)r5rr6rrr*)r'rSr can_fetchrappend) rXrrUrVrWrrr4linkslinks_priorityrOs r/ filter_linksr_sGHH>E   D !  eooc4&@  d #  ! !$ ' LL #& .  rT)FFNFT)NF).__doc__loggingretypingrrrrurllib.robotparserrcleanr r filtersr r r rrrrrrnetworkrsettingsrurlutilsrrrrr getLogger__name__r%compileIrDrFrIstrboolr0rSr_r:rTr/rps_  --.+   &   8 $2::nbdd37> RZZ7 >  " Z Z ZZsm Z  Z  ZeCHoZ~] "#"]] #]]  ] sm ] ]]]]}]sm] X]H'+"%!%! #%! 3- %! O $ %!  %! %!%!sm%! 49d3i  %!rT