AHjm.dZddlZddlmZddlmZddlmZddlm Z m Z m Z m Z m Z mZ ddlZddlmZmZdd lmZdd lmZmZmZdd lmZdd lmZdd lmZmZm Z ejBe"Z#e ddZ$dZ%dZ&dZ'GddZ(de)de)de*e)dze)dzffdZ+de)de*e)dze)dze)dzffdZ,de)de)dedzfdZ-de)dedzfdZ.de)de)dzde/fd Z0d!e1e)de/fd"Z2 d5de)d#e(d$e)dzddfd%Z3d&edzd#e(ddfd'Z4 d6d(e)d)e)dzd*edzd!e1e)dzd+e1e)dzd,e)dzde(fd-Z5 d7d#e(d.e/de(fd/Z6e&e'dddeddfde)d0e7d1e7d!e1e)dzd2e1e)dzd)e)dzd3ed*edzd,e)dzde*e1e)e1e)ffd4Z8y#e$rYWwxYw)8zC Functions dedicated to website navigation and crawling/spidering. N) ConfigParser)sleep)RobotFileParser)UrlStore extract_linksfix_relative_urls get_base_urlis_navigation_pageis_not_crawlable)XPathtostring)baseline)Responsefetch_response fetch_url)prune_unwanted_nodes)DEFAULT_CONFIG) LANGID_FLAG decode_file load_htmlF) compressedstrictz /robots.txt ic eZdZdZgdZ ddededzdedzdedzddf d Zdedefd Zdedefd Z d e ddfd Z de edzde efdZ dedefdZy)CrawlParametersz6Store necessary information to manage a focused crawl.) startbaselangrulesrefi known_numis_on prune_xpathNrrr r%returnc||_|j||_|j||_||_|xst |j|_d|_d|_ d|_ ||_ y)NrT) r _get_base_urlr_get_referencer!r get_rulesr r"r#r$r%)selfrrr r%s J/root/tools/cai/cai_env/lib/python3.12/site-packages/trafilatura/spider.py__init__zCrawlParameters.__init__/sh  ++E2 ++E2 $ -2-Ji 6J  '2c<t|}|std||S)z#Set reference domain for the crawl.zcannot start crawl: )r ValueError)r+rrs r,r(zCrawlParameters._get_base_url@s' '3E7;< < r.cX|jddk\r|jdddS|S)zDetermine the reference URL./rr)countrsplit)r+rs r,r)zCrawlParameters._get_referenceGs-*/++c*:a*?u||C#A&JUJr. url_storect|j|j|_t |j |j|_y)z*Adjust crawl data based on URL store info.N)boolfind_unvisited_urlsrr$lenfind_known_urlsr#)r+r6s r,update_metadatazCrawlParameters.update_metadataKs8)77 BC Y66tyyABr.todocr|sgS|Dcgc]#}||jk7s|j|vs"|%c}Scc}w)z.Prepare the todo list, excluding invalid URLs.)rr!)r+r=us r, filter_listzCrawlParameters.filter_listPs2IEa1 ?txx1}EEEs 444linkc|j xs|jjd|xr|j|vxr t| S)z9Run checks: robots.txt rules, URL type and crawl breadth.*)r can_fetchr!r )r+rAs r, is_valid_linkzCrawlParameters.is_valid_linkVsBJJA$**"6"6sD"AvtxxSWGWv`pqu`v\vvr.NNN)__name__ __module__ __qualname____doc__ __slots__strrr-r(r)rr<listr@r8rEr.r,rr*s<cI  (,"& 33Dj3% 3 4Z 3  3"33KCKCKCCdC FS D 0FT#YF w#w$wr.r htmlstringhomepager&cd|vrd|vr||fSt|}|||fS|jd}|r|dnd}|rd|vrtjd|||fS|j ddj j jd d}|jd st|}t||}t|}|tjd |y tjd |||fS)z:Check if there could be a redirection by meta-refresh tag.z "refresh"z "REFRESH"z@.//meta[@http-equiv="refresh" or @http-equiv="REFRESH"]/@contentr;zno redirect found: %srzurl=httpzfailed redirect: %s)NNzsuccessful redirect: %s) rxpathlogginginfosplitstriplowerreplace startswithr rrwarning)rOrP html_treeresultsresulturl2base_url newhtmlstrings r,refresh_detectionrd[s*$J)F8##*%I8##oo`aG"WQZF S& ,h78## << Q  % % ' - - / 7 7 CD ??6 "% 40dOM-t4 LL*D1 $ r.cHt|d}|r |jsy|j|dfvr,tjd|j|j}t |j}t ||\}}|ytjd|||t|fS)zBCheck if the homepage is redirected and return appropriate values.FdecoderFr2zfollowed homepage redirect: %szfetching homepage OK: %s) rdataurlrVrWrrddebugr )rPresponserOnew_htmlstring new_homepages r,probe_alternative_homepagern}shu5H 8==||Hc?* 5x||D<<X]]+J$5Z#J NL MM,l; <l)C CCr. robots_urlrhct}|j| |j|j|S#t$r }t j d|Yd}~yd}~wwxYw)zEParse a robots.txt file with the standard library urllib.robotparser.zcannot read robots.txt: %sN)rset_urlparse splitlines ExceptionLOGGERerror)rorhr excs r, parse_robotsrxs[  E MM* DOO%& L  137s> A'A""A'rbcJ|tz}t|}|r t||SdS)z?Attempt to fetch and parse robots.txt file for a given website.N)ROBOTS_TXT_URLrrx)rbrorhs r,r*r*s*N*J Z D-1< D );t;r.languagec|r=|r;tr5t|\}}}tj|\}}t ||k(Sy)zRun a baseline extraction and use a language detector to check if the content matches the target language. Return True if language checks are bypassed.T)rr py3langidclassifyr8)rOr{_textr`s r,is_target_languagersBh;j) 4&&t, Fh&'' r.r=c&td|DS)z6Probe if there are still navigation URLs in the queue.c32K|]}t|yw)N)r ).0ris r, z&is_still_navigation..s73!#&7s)any)r=s r,is_still_navigationrs 7$7 77r.paramsric bt||jsy|r|j|t|jtr |jgn |j}t |}|E  6;;  ##D)  d #  ! !$ ' LL  En='/Is0D, rkc| |jsytj|jgdt t |j||j y)z2Convert urllib3 response object and extract links.NT)visited)rhrrrirrr)rkrs r,process_responsersG x}}  ~t4+hmm,ffkkBr.rrr knownr%ct||||}tj|xsgdtj|j|tj |j |j |s1tj|jgdt|d}|S|jt|S)zInitialize crawl by setting variables, copying values to the URL store and retrieving the initial page if the crawl starts.T)rr)rF)initial) rrrr@ store_rulesrr r crawl_pager<)rrr r=rr%rs r, init_crawlrsUD% =FEKR6 F..t45 &++v||4 >FD1 M y) Mr.rctj|j}|s6d|_t tj |j|_|S|xjdz c_|r:t|\}}}|rB|r@|r>tj|gt|||nt|d}t|||jt|S)z6Examine a webpage, extract navigation links and links.Fr)rirf)rget_urlrr$r:r;r#r"rnrrrrr<)rrrirOrP new_base_urlrks r,rr s   FKK (C  y88EF  HHMH-G-L* Hl (|   z * *f# 6!#e46* 9% Mr. max_seen_urlsmax_known_urls known_linksconfigc Nt||||||} tj| j|j dd} | j r_| j |krP| j|krAt| } t| | j r| j |kr| j|krAttjtj| j}ttjtj| j}||fS)aBasic crawler targeting pages of interest within a website. Args: homepage: URL of the page to first page to fetch, preferably the homepage of a website. max_seen_urls: maximum number of pages to visit, stop iterations at this number or at the exhaustion of pages on the website, whichever comes first. max_known_urls: stop if the total number of pages "known" exceeds this number. todo: provide a previously generated list of pages to visit / crawl frontier. known_links: provide a list of previously known pages. lang: try to target links according to language heuristics. config: use a different configuration (configparser format). rules: provide politeness rules (urllib.robotparser.RobotFileParser() format). prune_xpath: remove unwanted elements from the HTML pages using XPath. Returns: List of pages to visit, deque format, possibly empty if there are no further pages to visit. Set of known links. DEFAULT SLEEP_TIME)default)rrget_crawl_delayrgetfloatr$r"r#rrrMdictfromkeysr9r;) rPrrr=rrrr r%r sleep_times r,focused_crawlerr(s:$t[+ NF**6;;PY[g@h*iJ ,,688m38H8H>8YF# j ,,688m38H8H>8Y  i;;FKKHI JDt}}Y%>%>v{{%KLMK  r.)rR)NNNNN)F)9rJrV configparserrtimerurllib.robotparserrcourlanrrrr r r r} ImportError lxml.etreer r r downloadsrrrhtmlprocessingrsettingsrutilsrrr getLoggerrGrurrz MAX_SEEN_URLSMAX_KNOWN_URLSrrLtuplerdrnrxr*r8rrMrrrrrintrrNr.r,rs%. '::0$66   8 $ e 4  .w.wb#sTz3QU:?U9VDDD 3:sTz3: -.D2 S  $0F <<$ 6<3#*8d3i8D8">"> "> t"> ">J Co C  C  C $(!""   * T ! s)d   9t   t :  B'(!$()$("**** s)d  * cT! * * * * T !*t* 49d3i  *e  sE==FF