AHjM bdZ ddlZdZddlZddlZ ddlZdZddlm Z ddl m Z ddl m Z mZmZmZddlmZ ddlZdZ ddlZdZ ddlZdZ dd lmZdd lmZdd lmZdd l m!Z!m"Z"m#Z#dd l$m%Z%e rddl&m'Z'm(Z(ejRe*Z+ddhZ,ejZdej\Z/ejZdej\Z0ejZdZ1ejZdZ2e"dddddZ3ejZdejhejjzZ6ejZdZ7ejZdZ8hdZ9ddhZ:dZ;ejZdZ<ejZd ejzZ>d!e?d"e?fd#Z@d$e?d"eAfd%ZBd&e?d"eCeDfd'ZEd!e?eDzd"eDfd(ZFd)eDd"eAfd*ZGd+eDd)eDd"eDfd,ZHd-eDd"e!dzfd.ZId-ed"e!dzfd/ZJe d01d2eDd"eDfd3ZKd4eDd"eDfd5ZLdbd4eDd6ed7d"eDfd8ZMe d91dcd:eDd;eAdeDd;eAd]*/[^<]*>z(z(|<[^>]*>)z([\x00-\x08\x0b\x0c\x0e-\x1f\ufffe\uffff]) collect_idsdefault_doctypeencodingremove_comments remove_pisz(?])\n)flagsz^https?://|/+$z3[^\s]+\.(avif|bmp|gif|hei[cf]|jpe?g|png|webp)(\b|$)>phitdrefcellheaditemquotecodepre)zhttp-equiv="content-language"zproperty="og:locale"z ([a-z]{2})z\W*(Drucken|E-?Mail|Facebook|Flipboard|Google|Instagram|Linkedin|Mail|PDF|Pinterest|Pocket|Print|QQ|Reddit|Twitter|WeChat|WeiBo|Whatsapp|Xing|Mehr zum Thema:?|More on this.{,8}$)$ filecontentreturnc4t|ts|Str|dddk(r tj|Str|dddk(r tj|Str tj|Str tj|S|S#t $rt jdY~wxYw#tj$rt jdYwxYw#tj$rYwxYw#tj$rY|SwxYw)z Don't trust response headers and try to decompress a binary string with a cascade of installed packages. Use magic numbers when available. Nszinvalid GZ files(/zinvalid ZSTD file) isinstancebytesHAS_GZIPgzip decompress ExceptionLOGGERwarningHAS_ZSTD zstandard ZstdError HAS_BROTLIbrotlierrorHAS_ZLIBzlib)r's I/root/tools/cai/cai_env/lib/python3.12/site-packages/trafilatura/utils.pyhandle_compressed_filer=js k5 )KO6 .??;/ /KO':: 0'' 4 4 $$[1 1 ??;/ / - . NN, - . "" 0 NN. / 0 ||    zz     sGBB9"C'>DB65B69(C$#C$'C=<C=DDdatacF |jdy#t$rYywxYw)zLSimple heuristic to determine if a bytestring uses standard unicode encodingzUTF-8FT)decodeUnicodeDecodeError)r>s r<isutf8rBs, G  s    bytesobjectct|rdgSg}t/t|d}||j|jt |dkr t |}n!t |dd|ddzxs t |}t |dkDr)|j |Dcgc]}|jc}|Dcgc] }|tvs |c}Scc}wcc}w)z> "Read all input or first chunk and return a list of encodingsrNri'iixr) rBcchardet_detectappendlowerlenr extendrUNICODE_ALIASES)rCguessescchardet_guessdetection_resultsrgs r<detect_encodingrPskyG"(5jA  % NN>//1 2 ;%&{3&{5D'9Kq >? ;!!?":A ;;? ;sC< C Cct|tr|Sd}t|}t|D]} |j |}n|xst|ddS#t t f$rtjd|d}YSwxYw)zCheck if the bytestring could be GZip and eventually decompress it, guess bytestring encoding and try to decode to Unicode string. Resort to destructive conversion otherwise.Nzwrong encoding detected: %srreplace)rerrors) r,strr=rPr@ LookupErrorrAr2r3)r'htmltextguessed_encodings r< decode_filerXs+s#H)5K+K8 "))*:;H   Ks;KK/0  NN8:J KH sA'A?>A? beginningc d|vS)zOAssess if the object is proper HTML (awith a corresponding tag or declaration).html)rYs r<is_dubious_htmlr]s  "" htmlstringcdtjd|}d|vr3|jd\}}}tjd|ddz|z}t t |j D]=\}}d|vr,|jdrtjd|d}|S|d kDs<|S|S) z>Repair faulty HTML strings to make then palatable for libxml2.doctype r)countzz\1>) INVALID_XML_CHARSsub partition DOCTYPE_TAG enumerateiter splitlinesendswith FAULTY_HTML)r_rY firstline_restilines r<repair_faulty_htmlrts"&&r:6JI'11$7 1d __R!_zincompatible input typeN2FryTzlxml parsing failed: %srrez9parsed tree length: %s, wrong data type or not valid HTML)r,rrhasattrr>r-rT TypeErrortyperXrGr]rtrr| ValueErrorrr1r2r9rH)rur}rY check_flagfallback_parser~s r< load_htmlrs1 *k**l+wz6/J__ j5#, /14 3CDD DZ(J3B%%'I +J#J :JN5*[9 D A ~ + J$.3t9q= PRUVZR[\ K  + 5 .445s DEE%EEi@)maxsizecharcJ|js|jr|SdS)z3Return a character if it belongs to certain classesra) isprintableisspace)rs r<return_printables_and_spacesrs"##%4?R?r^stringcd|jr|Sdjtt|S)z6Prevent non-printable and XML invalid character errorsra)rjoinmaprrs r<remove_control_charactersrs, 7733V< ==r^ unicodeform)NFCNFDNFKCNFKDct||S)z;Normalize the given string to the specified unicode format.r )rrs r<normalize_unicoder"s [& ))r^irspreserve_spacetrailing_spacect|jddjddjdd}|sttj d|}t t tj|rd}|S|rB|d jrdnd }|d jrdnd }d j|||g}|S) zjRemove HTML space entities, then discard incompatible unicode and invalid XML characters on line levelz  z rcz   Nrra) rrRtrimLINES_TRIMMINGrgallrrTrr)rsrrnew_line space_before space_afters r<line_processingr's )gt)D)L)LWVZ)[)c)cdlnv)wxH **4:; s3;;) *H O "&q'//"33L!%b!1!1!3#Kww h DEH Or^textc|r t|dS djtdfd|jDj ddS#t $rYywxYw)zzsanitize..Cs&kQUt^'L&ksu␤ra)rrfilterrlrRAttributeError)rrrs ` r<sanitizer<sht^T::yy&kY]YhYhYj&klmuu b   sAA A A r}cR|jD]}|j}| |jnd}|jtvxs|tv}|jtvxs |tvxs|}|j D]S}d|vs|j |r"|j ddd|jvs9|j j|U|jrt|j|||_ |jst|j|||_ |S)z?Trims spaces, removes control characters and normalizes unicodera:rr) rk getparenttagSPACING_PROTECTEDFORMATTING_PROTECTEDattribsplitnsmappoprrtail)r}elemparent parent_tagrr attributes r< sanitize_treerJs L!#)#5VZZ2 %66Y*HY:Y%99qZK_=_qcq /Ii{{9-a1H1KSWS]S]1]KKOOI. / 99 NNKDI 99 NNKDI%L& Kr^c dj|jjS#ttf$rYywxYw)z/Remove unnecessary spaces within a text string.rra)rrstriprrrs r<rrbs<xx '--// I &s,/AAelementcdD]!}|j|d}t|s!y|jjD]$\}}|j dst|s$yy)z*Check if an element is a valid img element)data-srcsrcraTrF)get is_image_fileritems startswith)rattrrvalues r<is_image_elementrlsi#kk$#   #>>//1 KD%z*}U/C  r^imagesrcc`|t|dkDryttj|S)zCheck if the observed string corresponds to a valid image extension. Use a length threshold and apply a regex on the content.i F)rHboolIMAGE_EXTENSIONsearch)rs r<rrzs.3x=4/ &&x0 11r^iterablenc#Kt|}tt||x}r|tt||x}ryyw)zChunk data into smaller pieces.N)rktupler)rriteratorbatchs r< make_chunksrsFH~H!,- -% - !,- -% -s >AAmy_lenoptionsrc||jkr!tjd|jy||jkDr"tjd||jyy)z=Check if the document length is within acceptable boundaries.ztoo small/incorrect for URL %sFztoo large: length %s for URL %sT) min_file_sizer2r9url max_file_size)rrs r<is_acceptable_lengthrsO %%% 5w{{C %%% 6 L r^target_languagestrictcPtD]G}|jd|d}|stfd|Drytj d|y|r>|j d}|r+tfd|Drytj d ytj d y) zoCheck HTML meta-elements for language information and split the result in case there are several languages.z .//meta[@z ][@content]c3K|]9}tj|jddjv;yw)contentraN RE_HTML_LANGrrrGrrrs r<rz"check_html_lang..s6l^b?l&8&8)R9P9V9V9X&YYl?ATz%s lang attr failedFz //html[@lang]c3K|]9}tj|jddjv;yw)langraNrrs r<rz"check_html_lang..s6i[_?l&8&8&"9M9S9S9U&VVirzHTML lang failedzNo relevant lang elements found)TARGET_LANG_ATTRSfindallanyr2debugxpath)r}rrrelemss ` r<check_html_langrs" yk:; lfkll LL. 5  ?+ ichii LL+ , LL23 r^ temp_text temp_commentsctdurFt|t|kDrtj|ntj|\}}|Stj dd}|S)zARun external component (if installed) for language identificationTz3Language detector not installed, skipping detectionN) LANGID_FLAGrH py3langidclassifyr2r3)rrresultrps r<language_classifierrs]d58^c-FX5XI&&y1^g^p^pq~^  M LM Mr^docmetarc|[t|||_|j>|j|k7r/tjd|j|jd|fSd|fS)zFFilter text based on language detection and store relevant informationzwrong language: %s %sTF)rlanguager2r3r)rrrrs r<language_filterrsc".y-H    'G,<,<,O NN2G4D4Dgkk R= '>r^c|j |jn |j}| xsC|jxs1tt t j |jS)zFilter out unwanted text)rrrrr RE_FILTERmatchrl)rtesttexts r< textfilterrsN&||3w||H< a8++- aS(J]J]J_5`1aar^c>t|xr|j S)zJDetermine if a string is only composed of spaces and/or control characters)rrrs r<text_chars_testrs < 0 000r^ dest_elemsrc_elemcl|jD]!}|j||j|#y)z0Copy attributes from src element to dest elementN)keyssetr)rrkeys r<copy_attributesr s.}}1 c8??3/01r^rct|jy|}|#|jdk(ry|j}|#y)z+Check whether an element is in a table cellFr!T)rr)rcurrents r<is_in_table_cellrsG ~~#G   ;;& ##%   r^ct|sy|jdk(r|j}| xs|d|k(Stt|j }|j}| xs|d|k(S)z:Check whether an element is the last element in table cellFr!r)rr getchildrenrr r)rchildrenrs r<is_last_element_in_cellrst D ! xx6##%|3x|t33h 01%%'|3x|t33r^cR|}|#|jdk(ry|j}|#y)z=Check whether an element is a list item or within a list itemr#TF)rr)rrs r<is_element_in_itemrs7&G   ;;& ##%   r^c|jdk(r |jry|}d}|%|jdk(r|}n|j}|%|y|jsyy)z:Check whether an element is the first element in list itemr#TNF)rrr)rr item_ancestors r<is_first_element_in_itemrsk{{f&GM   ;;& #M ##%       r^ct|sy|jdk(rt|jdk(S|j }|y|jdk(S)z9Check whether an element is the last element in list itemFr#rT)rrrHrgetnext)r next_elements r<is_last_element_in_itemrsZ g &{{f7&&()Q..??$L6))r^)r)FF)F)c__doc__r/r. ImportErrorloggingrer;r: functoolsr itertoolsrtypingrrrr unicodedatar r8r7r5r4rrcchardetr rEcharset_normalizerr lxml.etreer lxml.htmlrrrurllib3.responsersettingsrr getLogger__name__r2rJcompileIrirnHTML_STRIP_TAGSrfr|UNICODE MULTILINErURL_BLACKLIST_REGEXrrrrr IGNORECASErr-r=rrBlistrTrPrXr]rtrrrrrrrrrrrintrrrrrrrrr rrrrrr\r^r<r7sD H H 44!JH K 2*99*-   8 $G$bjj5rtt< bjj+RTT2 "**45BJJJKUEGeivz{ -RZZ",,5NO bjj!23"**STPUONrzz-(  BJJG --   ""%"J4<<49<0LUS[LSL2#s#t# 333"t);(#(+"4(V 5@s@s@@ >c>c>*c*8T0U*be*  4#tT^adh^h( 3  d WZ]aWa X0 4 h 4 2C$J242###{t+TVZ03ssTzs3Wafklpr|l|f}"bbTb1C$J1411x1811 8   4( 4t 4Th4( *X *$ *SHHJHK OshKK+K9LL L#K('K(+K65K69LLLLL L #L.-L.