
    AHja                     h   d Z ddlZddlZddlmZ ddlmZ ddlmZ ddl	m
Z ddlmZ ddlmZmZmZmZmZmZmZ d	d
lmZmZ d	dlmZmZmZmZmZmZm Z m!Z!  ejD                  e#      Z$ ed      Z% e& ee'      jP                  dz  dz        Z)h dZ*h dZ+da,ddhZ-h dZ. ed      Z/h dZ0h dZ1h dZ2h dZ3g dZ4dddddZ5d Z6dMd!ed"e7d#dfd$Z8dNd!ed%e7d#dfd&Z9d'ed#efd(Z:d'ed#efd)Z;dMd*ed+e7d#e&fd,Z<d'ed#efd-Z=d*ed#efd.Z>d/ed0ed#e&fd1Z?d2ed*ed#dfd3Z@d*ed#efd4ZAd5ed6e&dz  d#efd7ZBd5ed#e7fd8ZCd!ed%e7d#e&fd9ZDd!ed:eEe&   d%e7d#dfd;ZFd<edz  d%e7d#e&fd=ZGd>d?d@d/ed%e7dAe&d?e&d#e&f
dBZHd*ed#efdCZId*ed#e&fdDZJdEed*ed#efdFZKd!ed#dfdGZLd!ed#dfdHZMd!ed#efdIZNdJed#dfdKZOd!ed#dfdLZPy)OzE
All functions related to XML generation, processing and validation.
    N)unescape)version)StringIO)dumps)Path)DTDElement
SubElement	XMLParser_Element
fromstringtostring   )Document	Extractor)is_element_in_itemis_first_element_in_itemis_in_table_cellis_last_element_in_cellis_last_element_in_itemsanitizesanitize_treetext_chars_testtrafilaturadataztei_corpus.dtd>   pabhilbdeldivrefrowbodycellcodeheaditemlistquotetablegraphic>   rendroletypetarget	renditionr   r   >   r   r   r)   r*   r+   T)remove_blank_text>   r   r   r#   r'   r)   r*   r+   r,   >   r   r    r"   r%   r&   r'   r(   >	   r   r    r"   r#   r%   r'   r(   r)   r,   >   r%   r(   noter*   figure)sitenametitleauthordateurlhostnamedescription
categoriestagslicenseidfingerprintlanguagez***__`)z#bz#iz#uz#ti  element	keep_tailreturnc                    | j                         }|y|rc| j                  rW| j                         }|#|j                  xs d| j                  z   |_        n"|j                  xs d| j                  z   |_        |j	                  |        y)z
    Removes this element from the tree, including its children and
    text. The tail text is joined to the previous element or parent.
    N )	getparenttailgetprevioustextremove)rE   rF   parentpreviouss       G/root/tools/cai/cai_env/lib/python3.12/site-packages/trafilatura/xml.pydelete_elementrR   V   su    
  F~W\\&&(!;;,"<FK%]]0bGLL@HM
MM'    include_formattingc                 \   | j                         }|yt        | |      }| j                  || j                  z  }| j                         }|%|j                  r|j                   d| n||_        n*|j                  |j                   d| |_        n||_        |j                  |        y)zAMerge element with its parent and convert formatting to markdown.N )rJ   replace_element_textrK   rL   rM   rN   )rE   rT   rO   	full_textrP   s        rQ   merge_with_parentrY   i   s     F~$W.@AI||W\\!	""$H:B--8==/9+6Y		 Qyk2
MM'rS   treec                 8   | j                  d      D ]  }t        |      dk(  st        |j                        du s*t        |j                        du sB|j                         }|U|j                  dk7  se|j                  dk7  su|j                  |        | S )z"Remove text elements without text.rB   r   Fr,   r&   )iterlenr   rM   rK   rJ   tagrN   )rZ   rE   rO   s      rQ   remove_empty_elementsr_   ~   s    99S> 'w<1!>%!GO\c\h\hLimrLr&&(F !gkkY&>6::QWCWg&' KrS   c                     t        | j                  d            D ]`  }|j                  ddd      D ]H  }|j                  |j                  k(  s|j	                         j                  t
        vs>t        |       J b | S )z/Prevent nested tags among a fixed list of tags.z.//head | .//code | .//pr&   r'   r   )reversedxpathiterdescendantsr^   rJ   NESTING_WHITELISTrY   )rZ   elemsubelems      rQ   strip_double_tagsrg      st    $>?@ +++FFC@ 	+G{{dhh&7+<+<+>+B+BJ[+[!'*	++ KrS   docmetawith_metadatac                 H   |r| j                   D ci c]  }|t        | |d       }}|j                  |j                  d      |j                  d      |j                  d      dj	                  |j                  d      xs g       dj	                  |j                  d      xs g       t        |j                  d      d	
      d       |j                  d      }n%dt        | j                  d	
      i}| j                  }t        |d	
      |d<   t        |d	      S c c}w )z0Build JSON output based on extracted informationNr9   r5   r;   ;r<   r=   r$   F)rT   )sourcezsource-hostnameexcerptr<   r=   rM   commentsbodyrM   comments)ensure_ascii)		__slots__getattrupdatepopjoinxmltotxtr$   rn   
json_dumps)rh   ri   slot
outputdictrn   s        rQ   build_json_outputrz      s    ELEVEVWTdGGT488W
W$../#->>*#=%>>-8!hhz~~l'C'IrJ!7!=2> !7ER		
 "~~n5hw||NO
++%luMJzju55% Xs   Dc                     | j                  d      D ]/  }|j                  t        vs|j                  j	                          1 | S )zRemove unnecessary attributes.rB   )r\   r^   WITH_ATTRIBUTESattribclear)rZ   re   s     rQ   clean_attributesr      s<    		#  88?*KK  KrS   c                    t        d      }t        ||        d| j                  _        |j	                  t        | j                               d| j                  _        |j	                  t        | j                               |S )z4Build XML output tree based on extracted informationdocmainro   )r	   add_xml_metar$   r^   appendr   rn   rh   outputs     rQ   build_xml_outputr      sd    U^F!GLL MM"7<<01)G
MM"7#7#789MrS   documentoptionsc                    t        | j                         t        | j                         |j                  dk(  rt        nt
        } ||       }t        |      }t        t        |d      t              }|j                  dk(  r6|j                  r*t        j                  dt        |      |j                         t        |dd      j                         S )z9Make sure the XML output is conform and valid if requiredxmlunicode)encodingxmlteizTEI validation result: %s %sT)pretty_printr   )rg   r$   r_   formatr   build_tei_outputr   r   r   CONTROL_PARSERtei_validationLOGGERdebugvalidate_teirl   strip)r   r   funcoutput_trees       rQ   control_xml_outputr      s    hmm$(--(&~~6<LDx.K,KXkIFWK ~~!g&<&<3\+5NPWP^P^_KdYGMMOOrS   r   c                     t         D ]E  }t        ||d      }|s| j                  |t        |t              r|ndj                  |             G y)z-Add extracted metadata to the XML output treeNrk   )META_ATTRIBUTESrr   set
isinstancestrru   )r   rh   	attributevalues       rQ   r   r      sF    $ X	D1JJy:eS+A%sxxPUWXrS   c                 H    t        |       }t        || j                        }|S )z8Build TEI-XML output tree based on extracted information)write_teitree	check_teir9   r   s     rQ   r   r      s%     7#F vw{{+FMrS   xmldocr9   c                    | j                  d      D ]v  }d|_        |j                  dd       |j                         }|/t	        |      dkD  rt        |      }|j                  ||       |}|j                  dk(  slt        |       x | j                  d      D ]J  }|j                  s|j                  j                         s+d|j                  dc|_        |_        |_	        L | j                  d	      D ]  }|j                  t        vr-t        j                  d
|j                  |       t        |       B|j                  t         v rt#        |       n%|j                  dk(  rt%        |       t'        |       |j(                  D cg c]  }|t*        vs| c}D ]?  }t        j                  d||j                  |       |j(                  j-                  |       A  | S c c}w )zCCheck if the resulting XML file is conform and scrub remaining tagsr'   r   r/   headerNr   r   z.//text/body//div/lbz.//text/body//*z"not a TEI element, removing: %s %sr!   z0not a valid TEI attribute, removing: %s in %s %s)r\   r^   r   rJ   r]   _tei_handle_complex_headreplace_move_element_one_level_upfindallrK   r   rM   TEI_VALID_TAGSr   warningrY   TEI_REMOVE_TAIL_handle_unwanted_tails!_handle_text_content_of_div_nodes_wrap_unwanted_siblings_of_divr}   TEI_VALID_ATTRSrt   )r   r9   re   rO   new_elemar   s          rQ   r   r      s    F# -"!>t9q=/5HNN4*D::&t,- 56 B99*-0$))T*DHdiB 01 '88>) NN?3Od#88&"4(XX-d3*40 &*[[MA_4L!M 	'INNMyZ^ZbZbdghKKOOI&	'!'& M Ns   G(G(c                     t         t        t              a t         j                  |       }|du r.t        j                  dt         j                  j                         |S )zUCheck if an XML document is conform to the guidelines of the Text Encoding InitiativeFznot a valid TEI document: %s)TEI_DTDr   
TEI_SCHEMAvalidater   r   	error_log
last_error)r   results     rQ   r   r     sI     j/f%F5w7H7H7S7STMrS   c                 N   | j                   xs d}|r5| j                   r(| j                  dv r|j                         }n| j                  dk(  r)	 t        | j	                  d      d         }d|z   d| }n| j                  d	k(  rd
| d
}n| j                  dk(  r0| j	                  d      }|t        v rt        |    | t        |    }n{| j                  dk(  rld|v s| j                  d      rQ| j                  d      D ]6  }| d|j                  xs d }|j                         j                  |       8 d| d}nd| d}| j                  dk(  ri|rFd| d}| j	                  d      }|r	| d| d}nEt        j                  d|| j                         |}n!t        j                  d|| j                         | j                  dk(  r"|j                         }|rt        |       s| d}t!        |       rt#        |       sd| }|S # t
        t        f$ r d}Y w xY w)zeDetermine element text based on just the text of the element. One must deal with the tail separately.rI   )articler)   r+   r'   r-   r      #rV   r    z~~r   r&   
z.//lbz```
z
```
rD   r"   []r0   ()zmissing link attribute: %s %s'zempty link: %s %sr%   z- )rM   r^   r   intget	TypeError
ValueErrorHI_FORMATTINGrb   rK   rJ   rN   r   r   r}   r   r   r   )rE   rT   	elem_textnumberr-   r   	link_textr0   s           rQ   rW   rW     s)   "Igll;;66!)I[[F"W[[034 <.)5I[[E!YKr*I[[D ;;v&D}$,T23I;}T?R>ST	[[F"y GMM'$:!--0 .B#,+R2 ?ILLN))"-. $I;g6		{!,	{{eI;a(I[[*F(k6(!4	?GNN[%	NN.	7>>J{{fOO%	4W=$+QI  (1A'1J$	S z* s   H H$#H$
returnlistc           	      B   | j                   dk(  r!| j                         |j                  d       | j                  r|j                  t	        | |             | j
                  rC| j                   dk7  r4t        |       r)|j                  | j
                  j                                | D ]  }t        |||        | j                  s| j                   dk(  r| j                  dd       d| j                  dd       }|j                  d	|j                          d
| j                  dd       d       | j
                  r5|j                  d| j
                  j                                 n| j                   t        v r| j                   dk(  rt        | j                  d            }| j                  d      xs | j                  d      }|r|j                         sd}nt        t        |      t               }||k  r|j                  d||z
  z   d       | j                  d      rJ|j                  dd|z   d       n1|j                  d       n| j                   dk7  r| j                   dk7  ry| j                   t        v rA| j                  d      s0t#        |       s%|j                  |r| j                   dk7  rdnd       nO| j                   dk(  r|j                  d       n.| j                   t$        vrt'        |       s|j                  d       t        |       }| j
                  rQ|sO|j                  t#        |       s| j                   dk(  r| j
                  j                         n| j
                         t)        |       r|s|j                  d       yyy)zYRecursively convert a LXML element and its children to a flattened string representation.r%   Nz| r,   r6   rI   rV   altz![z](srcr   r#   z.//cellcolspanspanr   |r   z./cell[@role='head']z
|z---|r(   zancestor::cellu   
␤
z | r)   )r^   rL   r   rM   rW   rK   r   r   process_elementr   NEWLINE_ELEMSr]   rb   isdigitminr   MAX_TABLE_WIDTHr   SPECIAL_FORMATTINGr   r   )	rE   r   rT   childrM   
cell_count	span_infomax_span
is_in_cells	            rQ   r   r   U  s	   {{f!4!4!6!>$||.w8JKL||y05Eg5N 	',,,,./ ?z+=>? <<;;)#kk'2./qUB1G0HID4::<.7;;ub3I2J!LM||!!Agll&8&8&:%;"<=[[M){{e# y!9:
#KK	2Igkk&6I	 	(9(9(; H"3y>?CH(%%:0E)F(Gr&JK==!78%%FX,=+>b&AB!!$'[[F"w{{f'< 
 {{m#GMM:J,KTfgnTo*<PUAU,[_`		% 	.	.7Nw7W# "'*J||J2DW2MQXQ\Q\`fQf',,,,.lslxlxy w'
$ 1;'rS   	xmloutputc                 v    | yg }t        | ||       t        t        dj                  |      d      xs d      S )zLConvert to plain text format and optionally preserve formatting as markdown.rI   T)r   r   r   ru   )r   rT   r   s      rQ   rv   rv     s=    JIz+=>HRWWZ0$7=2>>rS   	null)delimr   r   c                   t        | j                  |      xs |}t        | j                  |      xs |}t               }t	        j
                  ||t        j                        }|j                  | j                  | j                  | j                  | j                  | j                  | j                  | j                  ||| j                  | j                   fD cg c]  }|r|n|
 c}       |j#                         S c c}w )zAConvert the internal XML document representation to a CSV string.)	delimiterquoting)rv   r$   rn   r   csvwriterQUOTE_MINIMALwriterowr9   r?   r@   r:   r6   imager8   r>   pagetypegetvalue)	r   rT   r   r   posttextcommentstextr   outputwriterds	            rQ   xmltocsvr     s     '9:BdHH113EFN$L ZF::fs?P?PQL  $$!!  !!	
 A	
$ ??#	
s   C=c                 d   t        dd      }t        ||        t        |d      }t        |d      }t        | j                        }d|_        |j                  dd       |j                  |       t        | j                        }d|_        |j                  dd	       |j                  |       |S )
z6Bundle the extracted post and comments into a TEI treeTEIzhttp://www.tei-c.org/ns/1.0)xmlnsrM   r$   r!   r/   entryro   )	r	   write_fullheaderr
   r   r$   r^   r   r   rn   )rh   teidoctextelemtextbodypostbodyrn   s         rQ   r   r     s    U"?@FVW%&&)H(F+H-HHLLL!OOH#G$8$89LLVZ(OOL!MrS   c                 Z   | j                   r8| j                  r,| j                  j                          d| j                    d}|S | j                   xs | j                  xs d}t        j	                  t
        j                        r%|dk(  r t        j                  d| j                         |S )z5Construct a publisher string to include in TEI headerz (r   zN/Azno publisher for URL %s)	r:   r5   r   r   isEnabledForloggingWARNINGr   r9   )rh   	publishers     rQ   _define_publisher_stringr    s    G,,''--/073C3C2DAF	
  $$A(8(8AE	w/I4FNN4gkkBrS   r   c                    t        | d      }t        |d      }t        |d      }|j                  t        |dd      _        |j                  r|j                  t        |d      _        t        |d      }t	        |      }|j
                  r9|t        |d	      _        t        |d
      }|j
                  t        |d      _        nt        |d       t        |d      }|j                  r|j                  t        |dd      _        |j                  t        |dd      _        t        |d      }	t        |	d      }
dj                  t        d|j                  |j                  g            }|s t        j                  d|j                         dj                  t        d|j                  |g            |
_        |t        |	dd      _        t        |	d      }t        |d      }|j                  t        |dd      _        |j                  r|j                  t        |d      _        t        |d      }|t        |d	      _        |j                  rt        |dd|j                         |j                  t        |d      _        t        |d      }t        |d      }|j                  t        |d      _        |j                   s|j"                  rt        |d      }t        |d      }|j                   r,dj                  |j                         t        |d d!      _        |j"                  r,dj                  |j"                        t        |d d"      _        t        |d#      }|j$                  t        |dd$      _        t        |d%      }t        |d&      }t        |d't&        d()      }d(t        |d*      _        t        |dd+,       |S )-z+Write TEI header based on gathered metadata	teiHeaderfileDesc	titleStmtr6   r   )r/   r7   publicationStmtr   availabilityr   	notesStmtr3   r?   r@   
sourceDescbiblz, Nzno sigle for URL %ssiglebiblFullptrURL)r/   r0   r8   profileDescabstract	textClasskeywords,termr<   r=   creationdownloadencodingDescappInfoapplicationTrafilatura)r   identlabelz$https://github.com/adbar/trafilatura)r0   )r
   r6   rM   r7   r  r>   r?   r@   ru   filterr5   r8   r   r   r9   r;   r<   r=   filedatePKG_VERSION)r   rh   r   filedescbib_titlestmtpublicationstmt_apublisher_stringr  	notesstmt
sourcedescsource_biblr  biblfullpublicationstmtprofiledescr  	textclassr  r  encodingdescappinfor  s                         rQ   r   r     s    ,F&*-Hx5M;B==J}gF38~~3:>>
=(+0"8->?/8:J
$k27!"3^D-4__
<%* 	$c*8[1Izz8?


9f405=D=P=PJy&}5:Hl3JZ0KIIfTG$4$4gll#CDEE,gkk:yyw}}e.D!EFK8=Jz605*j1Hx5M;B==J}gF38~~3:>>
=(+0 +<=O4DJ,1{{?EgkkJ/6||J',V]3K+z2H%,%8%8Jx"W\\{K8	i4CF88GL^L^C_Jxl;@<<=@XXgll=SJxf5:+z2H9@9I9IJxj16fn5Ly1GWm[P]^K,9J{G$){E*PQMrS   c                    | j                   r| j                   j                         rt        |       dkD  rK| d   j                  dk(  r9| j                    d| d   j                   xs d j                         | d   _         n.t	        d      }| j                   |_         | j                  d|       d| _         | j                  r| j                  j                         rt        |       dkD  rK| d   j                  dk(  r9| d   j                   xs d d| j                   j                         | d   _         n-t	        d      }| j                  |_         | j                  |       d| _        yyy)z@Wrap loose text in <div> within <p> elements for TEI conformity.r   r   rV   rI   N)rM   r   r]   r^   r	   insertrK   r   )rE   	new_childs     rQ   r   r   -  s    ||**,w<!
# 5!(a
0E2/FGMMOGAJOI$\\INNN1i(||**,w<!3 6")"+"2"2"8b!97<<.IOOQGBKI$\\INNN9% -|rS   c                    | j                   r| j                   j                         nd| _         | j                   sy| j                  dk(  r>dj                  t	        d| j
                  | j                   g            | _        d| _         yt        d      }| j                   |_        | j                         }|$|j                  |j                  |       dz   |       d| _         y)z Handle tail on p and ab elementsNr   rV   r   )
rK   r   r^   ru   r  rM   r	   rJ   r/  index)rE   new_siblingrO   s      rQ   r   r   B  s    +2<<7<<%%'TGL<<{{cxxtgllGLL-I JK GL cl"<<""$MM&,,w/!3[AGLrS   c                 .   t        d| j                        }| j                  r| j                  j                         nd|_        | j	                         D ]  }|j
                  dk(  rjt        |      dkD  s|j                  r>t        |      dk(  s|d   j                  rt        |d       |j                  |d   _        j|j                  |_        ||j                  |        | j                  r| j                  j                         nd}|r||_        |S )z0Convert certain child elements to <ab> and <lb>.r   )r}   Nr   r   r.  r   )
r	   r}   rM   r   iterchildrenr^   r]   rK   r
   r   )rE   new_elementr   rK   s       rQ   r   r   S  s    $w~~6K/6||w||))+K%%' 
&99;!#{'7'7{#q(KO,@,@{D1',zzB$#(:: u%
& $+<<7<<TDrS   div_elementc                    t        d      }d}| j                         }|y| j                         D ]}  }|j                  dk(  r nl|j                  t        v r'|xs |j                  |      }|j                  |       M|sPt        |      dkD  s_|j                  ||       t        d      }d} |r"t        |      dk7  r|j                  ||       yyy)z=Wrap unwanted siblings of a div element in a new div element.r!   Nr   )	r	   rJ   itersiblingsr^   TEI_DIV_SIBLINGSr2  r   r]   r/  )r7  r3  new_sibling_indexrO   siblings        rQ   r   r   h  s    %.K""$F~++- );;%;;** 1 JV\\'5Jw' !S%5%9/=%en$(!) S-2'5 3rS   c                    | j                         }||j                         nd}||yt        d      }|j                  t        | j	                                      |j                  |j                  |      dz   |        | j                  r| j                  j                         nd}|r||_	        d| _        |j                  r|j                  j                         nd}|r||_        d|_        t        |      dkD  s|j                  s|j                  r$|j                  |j                  |       dz   |       t        |      dk(  r|j                  s|j                  |       yyy)z
    Fix TEI compatibility issues by moving certain p-elems up in the XML tree.
    There is always a n+2 nesting for p-elements with the minimal structure ./TEI/text/body/p
    Nr   r   r   )rJ   r	   extendr)   r9  r/  r2  rK   r   rM   r]   rN   )rE   rO   grand_parentr   rK   s        rQ   r   r     s-   
  F)/);6##%L~-s|HOOD--/01**62Q6@#*<<7<<TD"(++6;;4D
8}qHMMX]]L..w7!;XF
6{aF# !,rS   )T)F)Q__doc__r   r   htmlr   importlib.metadatar   ior   jsonr   rw   pathlibr   
lxml.etreer   r	   r
   r   r   r   r   settingsr   r   utilsr   r   r   r   r   r   r   r   	getLogger__name__r   r  r   __file__rO   r   r   r   r   r   r:  r   r   r   r|   rd   r   r   r   boolrR   rY   r_   rg   rz   r   r   r   r   r   r   r   rW   r)   r   rv   r   r   r  r   r   r   r   r   r    rS   rQ   <module>rN     s4      &  $  Z Z Z )	 	 	 
		8	$m$ h&&/2BBC
& B
+8 T2OI X?   3d#>H   &x T d *	 	X 	H  6x 6 6 608  h 8 P PI P# P&X XH X Xh 8 'h 'S4Z 'H 'T d 3( 3 3 3lD X D 49 D RV D [_ D N	?4 	?T 	?c 	? LP]c x T S WZ hk @8  &h 3 DX D DX DNx D *H  "h 8 *6 6T 62$ $T $rS   