
    AHju                        d Z ddlZddlZddlmZ ddlmZ ddlmZ ddl	m
Z
mZmZmZmZmZ ddlmZ dd	lmZmZmZmZmZ dd
lmZmZ ddlmZmZmZmZm Z  ddl!m"Z" ddl#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*  ejV                  e,      Z-ddhZ.ddhZ/h dZ0h dZ1ddhZ2ddhZ3de4dede5e4z  dz  ddfdZ6dedededz  fdZ7dedededz  fdZ8d ed!ed"eddfd#Z9d$ed ededdfd%Z:d&ed'eddfd(Z;d&ede<fd)Z=d*ed+eddfd,Z>dedededz  fd-Z?dede<fd.Z@dedefd/ZAdedededz  fd0ZBded1eCe4   dededz  fd2ZDded1eCe4   dededz  fd3ZEd4e<defd5ZFd6ed1eCe4   dededz  fd7ZGdCdedz  dedz  dedz  fd8ZHded1eCe4   dededz  fd9ZIefd:ed;eded1edef
d<ZJd:ed1eCe4   dedefd=ZKd:ededeLee4eCe4   f   fd>ZMd?ededeLee4eNf   fd@ZOd&ed1eCe4   dededz  fdAZPd:ededeLee4eNef   fdBZQy)Dz6
Functions related to the main Trafilatura extractor.
    N)deepcopy)Any)urljoin)Element
SubElement_Elementstrip_elements
strip_tagstostring)HtmlElement   )delete_by_link_densityhandle_textnodelink_density_test_tablesprocess_nodeprune_unwanted_nodes)TAG_CATALOG	Extractor)FORMATTING_PROTECTEDcopy_attributesis_image_filetext_chars_testtrim)delete_element)
BODY_XPATHCOMMENTS_DISCARD_XPATHCOMMENTS_XPATHDISCARD_IMAGE_ELEMENTSOVERALL_DISCARD_XPATHPRECISION_DISCARD_XPATHTEASER_DISCARD_XPATHhireftdth>   r"   r$   r%   r#   >   r"   r#   spancodequoteheadmsgtagtextreturnc           	      V    t         j                  d| |t        |xs d      xs d       y)z/Format extraction event for debugging purposes.z	%s: %s %s NoneN)LOGGERdebugr   )r*   r+   r,   s      R/root/tools/cai/cai_env/lib/python3.12/site-packages/trafilatura/main_extractor.py
_log_eventr4   /   s"    
LLc3TZR(8(BFC    elementoptionsc                    t        |       dk(  rt        | |      }nCt        |       }t        |       D ]*  }t	        ||d      }||j                  |       d|_        , |,t        dj                  |j                                     du r|S y)zProcess head elements (titles)r   Fcomments_fixNdoner/   T)
lenr   r   listr   appendr+   r   joinitertext)r6   r7   titlechildprocessed_childs        r3   handle_titlesrD   4   s    
7|q
 Wg. ! '] 	E .eW5QO*_-EI	 _RWWU^^5E-FG4Or5   c                     t        | |      }|y| j                         }|| j                         }||j                  t        vrt        d      }|j                  d|       |S |}|S )zXProcess formatting elements (b, i, etc. converted to hi) found
    outside of paragraphsNpr   )r   	getparentgetpreviousr+   r   r   insert)r6   r7   
formattingparentprocessed_elements        r3   handle_formattingrM   N   s~     gw/JD  F~$$&~+??#CL  J/  'r5   new_child_elemsubelemprocessed_subchildc                     t        | |j                        }|j                  |j                  c|_        |_        |j                  D ]!  }|j                  ||j                  |          # y)z/Add a sub-element to an existing child element.N)r   r+   r,   tailattribset)rN   rO   rP   sub_child_elemattrs        r3   add_sub_elementrW      s`    0B0F0FGN/A/F/FHZH_H_,N, 74!567r5   rB   c                     | j                   |_         | j                  d      D ]U  }|j                  dk(  r t        ||      }|/|j	                  |       nt        ||d      }|t        |||       d|_        W y)z<Iterate through an element child and rewire its descendants.*r=   NFr9   r;   )r,   iterdescendantsr+   handle_listsr>   r   rW   )rB   rN   r7   rO   rP   s        r3   process_nested_elementsr\      s    **N((- 	;;& !-gw!?!-%%&89!0'PU!V!-9KL	r5   elemnew_elemc                 P    | j                  d      x}r|j                  d|       yy)z>Copy the rend attribute from an existing element to a new one.rendN)getrT   )r]   r^   	rend_attrs      r3   update_elem_renditionrc      s)    HHV$$y$VY' %r5   c                 b    | duxr* t        dj                  | j                                     du S )z"Find if the element contains text.Nr/   T)r   r?   r@   )r]   s    r3   is_text_elementre      s+    tQ0H IT QQr5   processed_elem	orig_elemc                     | Ut        || j                        }| j                  | j                  c|_        |_        | j                  dv rt	        ||        yyy)z&Create a new sub-element if necessary.N)graphicr#   )r   r+   r,   rR   r   )rf   rg   	childelems      r3   define_newelemrk      sY    !y.*<*<=	)7)<)<n>Q>Q&		!33I~6 4 "r5   c                 >   t        | j                        }| j                  7| j                  j                         rt	        |d      }| j                  |_        | j                  d      D ]  }t        d      }t        |      dk(  r~t        ||      }|"|j                  xs d|_        |j                  r<|j                  j                         r"|xj                  d|j                  z   z  c_        |j                  |       nt        |||       |j                  |j                  j                         r|D cg c]  }|j                  dk7  s| }}|r_|d   }|j                  |j                  j                         s|j                  |_        n"|xj                  d|j                  z   z  c_        |j                  st        |      dkD  rt        ||       |j                  |       d|_         d| _        t        |      rt        | |       |S yc c}w )z3Process lists elements including their descendants.Nitemr   r/    r;   )r   r+   r,   stripr   rZ   r<   r   rR   r>   r\   rc   re   )	r6   r7   rL   rN   rB   rC   elnew_child_elem_childrenlast_subchilds	            r3   r[   r[      s   ,||GLL$6$6$8#$5v>%ll ((0  u:?*5':O*&5&:&:&@b#"''O,@,@,F,F,H"''31E1E+EE'!((8#E>7Czz%%***:*:*<8F*[""&&TZJZ2*['*[*$;B$?M$))19K9K9Q9Q9S-2ZZ*%**cEJJ.>>*#n"5"9!%8$$^4	-. GK()g'89  ! +\s   <HHc                     | j                  d      s| j                  dk(  ry| j                         }|d|j                  dd      v ry| j                  d      }|t	        |       dk(  ryy)	zECheck if it is a code element according to common structural markers.langr'   T	highlightclassr/   r   F)ra   r+   rG   findr<   )r6   rK   r'   s      r3   is_code_block_elementry      sl     {{6gkkV3 FkVZZ-DD<<DCLA-r5   c                 d    t        |       }| j                  d      D ]	  }d|_         d|_        |S )z/Turn element into a properly tagged code block.rY   r;   r'   )r   iterr+   )r6   rL   rB   s      r3   handle_code_blocksr|      s;     )c" 	"r5   c                     t        |       rt        |       S t        | j                        }| j	                  d      D ]#  }t        ||      }|t        ||       d|_        % t        |      rt        |d       |S y)zProcess quotes elements.rY   Nr;   r(   )	ry   r|   r   r+   r{   r   rk   re   r
   )r6   r7   rL   rB   rC   s        r3   handle_quotesr~      s    W%!'**,c" &ug6&?,=>		
 ()$g.  r5   potential_tagsc                    | j                   dk(  rd| j                  dd      v rt        |       S | j                   |vr1| j                   dk7  r!t        d| j                   | j                         y| j                   dk(  rZt        | |dd	
      }|It        |j                        d	u r2|j                  j                          |j                   dk(  rd|_         |S y)zAHandle diverse or unknown elements in the scope of relevant tags.divzw3-coderw   r/   r;   zdiscarding elementNFTr:   preserve_spacesrF   )	r+   ra   r|   r4   r,   r   r   rS   clear)r6   r   r7   rL   s       r3   handle_other_elementsr      s     {{e	W[["-E E!'** {{.(;;& +W[[',,G{{e ,GW5bfg(_=N=S=S-TX\-\$$**, $$-(+!%$$r5   c                    | j                   j                          t        |       dk(  rt        | |      S t	        | j
                        }| j                  d      D ]  }|j
                  |vr1|j
                  dk7  r"t        d|j
                  |j                         Ct        ||dd      }||j
                  d	k(  rct        d
d	|j                         |j                  r'|xj                  d|j                  xs dz   z  c_        n|j                  |_        d|_        t	        |j
                        }|j
                  t        v rt        |      dkD  rH|D ]C  }t        |j                        du rd|j                  z   |_        t        ||j
                         E |j
                  dk(  r#|j                  d|j                  dd             nB|j
                  dk(  r3|j                  d      "|j                  d|j                  dd             |j                  |j                  c|_        |_        |j
                  dk(  rt!        ||      }||}|j#                  |       d|_         t        |      dkD  r-|d   }	|	j
                  dk(  r|	j                  t%        |	       |S |j                  r|S t        dd	t'        |             y)zIProcess paragraphs along with their children, trim and clean the content.r   rY   r;   zunexpected in pFTr   NrF   z
extra in prn   r/   r"   r`   r#   targetri   ro   lbzdiscarding element:)rS   r   r<   r   r   r+   r{   r4   r,   r   P_FORMATTINGr   r
   rT   ra   rR   handle_imager>   r   r   )
r6   r   r7   rL   rB   rC   newsubrm   
image_elem	last_elems
             r3   handle_paragraphsr     s~   NN 7|qGW--  ,c" 999N*uyyF/B(%))UZZ@ *%u^bc&""c)<o.B.BC$))%**c_5I5I5OR.PP*-<-A-A%*"	UYY'F""l2'!+ / >*4995=(+diiDI"?DHH=>
 99$JJvuyy'<=YY%'yy*6

8UYYx-DE& (7';';_=Q=Q$FK""i/)/7C
)'F$$V,	s9v !%b)	==D Y^^%;9%    $c84E+FGr5   	is_headerc                 D    t        d      }| r|j                  dd       |S )z1Determine cell element type and mint new element.cellroler)   )r   rT   )r   cell_elements     r3   define_cell_typer   m  s&     6?L(r5   
table_elemc                    t        d      }t        | ddd       d}t               }| j                  d      D ]x  }d}|j                  t              D ]P  }|j                  dd      }	t        |	      j                         rt        |	      nd}	|j                  |	       ||	z  }R t        ||      }z d	}
d	}|dkD  rt        |      nd
}t        d      }|r|j                  d|       | j                         D ]  }|j                  dk(  rGt        |      dkD  r|j                  |       t        d      }|r|j                  d|       |
xs |}
n|j                  t        v r~|j                  dk(  xr |
 }|xs |}t        |      }t        |      dk(  r3t!        ||      }||j"                  |j$                  c|_        |_        n|j"                  |j$                  c|_        |_        d|_        |j                         D ]  }|j                  t&        v r)|j                  t        v rd|_        t)        ||dd      }n]|j                  dk(  r1|j*                  dk(  r"t-        ||      }|1|j                  |       d}nt/        ||j1                  dg      |      }|t3        ||       d|_         |j"                  st        |      dkD  r#|j                  |       n|j                  dk(  r n
d|_         t        |      dk(  r|j4                  j7                  dd       t        |      dkD  r|j                  |       t        |      dkD  r|S y)zProcess single table element.tabletheadtbodytfootr   trcolspanr   Fr/   rowr&   r%   Nr;   r   T)r   r:   r=   recallr   )r   r
   rT   r{   TABLE_ELEMSra   strisdigitintaddmaxrZ   r+   r<   r>   r   r   r,   rR   	TABLE_ALLr   focusr[   handle_textelemunionrk   rS   pop)r   r   r7   newtablemax_colsdiff_colspansr   total_colspansr$   r   seen_header_rowseen_header	span_attrnewrow
subelementr   rN   processed_cellrB   rP   s                       r3   handle_tabler   v  s)   wH z7GW5 HEMood# 1''+& 	&BffY*G&)'l&:&:&<c'l!Gg&g%N		&
 x01 OK!)AH2IU^F

69% 002 1 
>>T!6{Q' JJvy1"1"@[^^{*"$.F3FI%2K-i8N:!#!-j'!B!-?M?R?RTbTgTg<N')< <F??JOO8#^%8!'
'779 'EyyI- 993(.EI-<UG]apt-u*f,(1J-9%-I*-9*112DE15. .=UNDXDXZ_Y`Dacj-k*)5&'9>J &EI''* ""c.&9A&=n-^^w&
c1 h =Q&$' 6{Q
8}qr5   c                     | yt        | j                        }dD ]3  }| j                  |d      }t        |      s!|j	                  d|        nT | j
                  j                         D ]6  \  }}|j                  d      st        |      s$|j	                  d|        n | j                  d      x}r|j	                  d|       | j                  d      x}r|j	                  d|       |j
                  r|j                  d      sy|j                  dd      }|j                  d      sN|#|j                  t        |j                  |      }nt        j                  d	d
|      }|j	                  d|       | j                  |_        |S )z5Process image elements and their relevant attributes.N)data-srcsrcr/   r   r   altrA   httpz^//zhttp://)r   r+   ra   r   rT   rS   items
startswithurlr   resubrR   )	r6   r7   rL   rV   r   valuealt_attr
title_attrlinks	            r3   r   r     su   ,# 
kk$#!!%-	
 #>>//1 	KD%z*}U/C!%%eU3	 ;;u%%x%eX.[[))z)gz2 ##+<+@+@+G   +D??6"7;;#:7;;-D66&)T2DeT*$\\r5   c                 ~   d}| j                   dk(  rt        | |      }|S | j                   t        v rt        | |      }|S | j                   dk(  rt	        | |      }|S | j                   dk(  rt        | ||      }|S | j                   dk(  rCt        | j                        du r*t        | |      }|t        d      }|j                  |_
        |S | j                   t        v rt        | |      }|S | j                   dk(  rd|v rt        | ||      }|S | j                   dk(  rd|v rt        | |      }|S t        | ||      }|S )	z?Process text element and determine how to deal with its contentNr=   r)   rF   r   Tr   ri   )r+   r[   CODES_QUOTESr~   rD   r   r   rR   r   r   r,   
FORMATTINGrM   r   r   r   )r6   r   r7   new_elementthis_elements        r3   r   r     sk   K{{f"7G4, + 
	$#GW5( ' 
	#GW5$ # 
	'I   
	7<<(D0'9L'%cl#/#4#4   

	"'9  
	G~$="7NGD  
		!i>&A"7G4  ,G^WMr5   treeresult_bodyc                 L   t         j                  d       d}j                  dk(  rj                  ddg       |dz  }t	        |       }dvrt        |ddd	       nt        |d	       |j                  |      }|j                  t        d
 fd|D                     |S )zLook for all previously unconsidered wild elements, including outside of the determined
    frame and throughout the document to recover potentially missing text partszRecovering wild text elementsz\.//blockquote|.//code|.//p|.//pre|.//q|.//quote|.//table|.//div[contains(@class, 'w3-code')]r   r   r   z|.//div|.//lb|.//listr#   ar&   c                 
    | d uS N xs    r3   <lambda>z#recover_wild_text.<locals>.<lambda>,  s
    atm r5   c              3   8   K   | ]  }t        |        y wr   r   .0er7   r   s     r3   	<genexpr>z$recover_wild_text.<locals>.<genexpr>-  s     KQ_Q8K   )	r1   r2   r   updateprune_unwanted_sectionsr
   xpathextendfilter)r   r   r7   r   search_exprsearch_treesubelemss     ``   r3   recover_wild_textr     s    
 LL01pK}} udm,..)$HKN";UF3;'  -H#K(K	
 r5   c                    |j                   dk(  }t        | t        d      } d|vrt        | t              } |j                   dk7  r"t        | t              } |rt        | t
              } t        d      D ]/  }t        | dd|      } t        | d	d
|      } t        | dd
|      } 1 d|v s|r1| j                  d      D ]  }t        |      du st        |d
        |rot        |       dkD  rC| d   j                  dk(  r1t        | d   d
       t        |       dkD  r| d   j                  dk(  r1t        | dd
d      } t        | dd
d      } | S )z1Rule-based deletion of targeted document sections	precisionT)with_backupri   r      r   )backtrackingfavor_precisionr=   FrF   r   	keep_tailr   ro   r)   r(   )r   r   r   r   r!   r    ranger   r{   r   r   r<   r+   )r   r   r7   r   _r]   s         r3   r   r   3  sP   mm{2O&;ND&#D*@A}} #D*>?'.EFD1X f%dEVef%dFXgh%dCeUdef
 . OIIg& 	6D'-5tu5	6 $i!mb!748u5 $i!mb!7%dFX\]%dG%Y]^Kr5   c                    t        t              j                  du rj                  g d       j                  du rj                  d       j                  du rj                  d       t        d      }t        D ]  }t        d  ||       D        d       }| t        |      }t        |      dk(  r<|j                  d      }j                  d	k(  rd
}nd}|r*t        dj                  |            j                  |z  k  rj                  d       dvrt!        |d       dvrt!        |d       t"        j%                  t'                     |j                  d      }|D ch c]  }|j(                   c}dhk(  r|g}|j+                  fd|D        D 	cg c]  }	|	|		 c}	       t        |      dkD  rI|d   j(                  t,        v r4t/        |d   d       t        |      dkD  r|d   j(                  t,        v r4t        |      d
kD  st"        j%                  t1        t3        |                    n dj                  |j5                               j7                         }
||
fS c c}w c c}	w )NT)r   r$   r%   r   ri   r#   bodyc              3   &   K   | ]	  }||  y wr   r   r   ss     r3   r   z_extract.<locals>.<genexpr>b       ?a?   r   z//p//text()r   r      r/   r   r&   .//*r   c              3   8   K   | ]  }t        |        y wr   r   r   s     r3   r   z_extract.<locals>.<genexpr>~  s     )hZ[/!^W*U)hr   ro   Fr   rn   )rT   r   tablesr   imagesr   linksr   r   nextr   r<   r   r   r?   min_extracted_sizer
   r1   r2   sortedr+   r   NOT_AT_THE_ENDr   r   r   r@   rp   )r   r7   r   exprsubtreeptestfactorr   r   rq   	temp_textr   s    `         @r3   _extractr  U  s:   %N~~9:~~9%}}5!&/K %?4:?F?)'>7Kw<1m,==K'FFBGGEN+g.H.H6.QQu%&w&'w'VN+,==(#$aAEE$.yH)h_g)h{2lnlzB{|+"B(;(;~(M;r?e< +"B(;(;~(M {aLLc$i)K%L --/0668I	>11 % |s   3I?)J
1J
cleaned_treec                 F   t        |       }t        | |      \  }}}t        |      dk(  st        |      |j                  k  r;t	        ||||      }dj                  |j                               j                         }t        |d       t        |d       ||t        |      fS )zFind the main content of a page using a set of XPath expressions,
    then extract relevant elements, strip them of unwanted subparts and
    convert themr   rn   r;   r   )
r   r  r<   r   r   r?   r@   rp   r	   r
   )r  r7   backup_treer   r  r   s         r3   extract_contentr    s    
 <(K-5lG-L*KN ;1I1K1K K'['>ZHH[1134::<	;'{E"	3y>11r5   c                 x    | j                   |v r,t        | |d      }||j                  j                          |S y)z?Process comment node and determine how to deal with its contentTr9   N)r+   r   rS   r   )r]   r   r7   rL   s       r3   process_comments_noder    s@    xx>!+D'M($$**, %$r5   c           
      
   t        d      }t        t              t        D ]  }t	        d  ||       D        d      }|t        |t              }t        |ddd       |j                  t        d fd|j                  d	      D                     t        |      d
kD  st        j                  |       t        |d        n dj                  |j!                               j#                         }||t        |      | fS )z>Try to extract comments out of potential sections in the HTML.r   c              3   &   K   | ]	  }||  y wr   r   r   s     r3   r   z#extract_comments.<locals>.<genexpr>  r   r   Nr   r#   r&   c                 
    | d uS r   r   r   s    r3   r   z"extract_comments.<locals>.<lambda>  s
    Qd] r5   c              3   8   K   | ]  }t        |        y wr   )r  r   s     r3   r   z#extract_comments.<locals>.<genexpr>  s     ,~cd-B1nV]-^,~r   r   r   Fr   rn   )r   rT   r   r   r   r   r   r
   r   r   r   r<   r1   r2   r   r?   r@   rp   )r   r7   comments_bodyr   r   temp_commentsr   s    `    @r3   extract_commentsr    s    FOM%N ?4:?F?&w0FG7C/ 	*,~hohuhuv|h},~	
 }!LL7e414 HH]3356<<>M-]);TAAr5   r   )R__doc__loggingr   copyr   typingr   urllib.parser   
lxml.etreer   r   r   r	   r
   r   	lxml.htmlr   htmlprocessingr   r   r   r   r   settingsr   r   utilsr   r   r   r   r   xmlr   xpathsr   r   r   r   r   r    r!   	getLogger__name__r1   r   r   r   r   r   r   r   bytesr4   rD   rM   rW   r\   rc   boolre   rk   r[   ry   r|   r~   rT   r   r   r   r   r   r   r   r   tupler  r   r  r  r  r   r5   r3   <module>r!     s    	     Z Z !  - ^ ^    
		8	$ e}Tl%	"
 %DC Dc Dt); D D
8 i HtO 4/x /) /4 /d7H 7x 7U] 7bf 78 X PY ^b  ( (H ( (R( Rt R
78 7 7T 7&( &Y &8d? &R8    X 8 i HtO $8 SX PY ^fim^m 6Ox OS OI OZbeiZi Od  WX Ws3x W) WX`cgXg Wt'(T/ 'I4D 'PX[_P_ 'TX s3x ) X`cgXg > Yd
$,7@RU6+ s3x R[ `k D22; 22 22uXsCPSH=T7U 22j2+ 2	 2eHVY[^L^F_ 2. #c( Y [cfj[j !B; !B !BuXsTWYdEd?e !Br5   