
    AHja7                        d Z ddlZddlZddlmZ ddlmZ ddlmZ ddlm	Z	 ddl
mZ ddlmZmZmZ d	d
lmZ  eedd      Ze e ed            n ej*                         xs d	Zg dZ ee      dhz  Zddedz  dedz  defdZ e       ZdddddddddZdedededz  fdZ G d d      Z ddejB                  d edz  de fd!Z"dd"e#de$eef   fd#Z% G d$ d%      Z& e'ed&      Z(d'Z)d(Z*d)Z+d*Z,d+Z-h d,Z.g d-Z/g d.Z0 ed/      Z1 e2g d0      Z3i d1d2d3d4d5d6d7d8d9d:d;d<d=d>d?d@dAdBdCdDdEdFdGdHdIdJdKdLdMdNdOdPdQdRi dSdTdUdVdWdXdYdZd[d\d]d^d_d`dadbdcdddedfdgdhdidjdkdldmdndodpdqdrdsdti dudvdwdxdydzd{d|d}d~ddddddddddddddddddddddddi dddddddddddddddddddddddddddddddddddddddddddd	Z4y)z<
Listing a series of settings that are applied module-wide.
    N)ConfigParser)datetime)unescape)Path)Any)ElementXPath_Element   )line_processingsched_getaffinity)csvjsonhtmlmarkdowntxtxmlxmlteipythonfilenameconfigreturnc                     ||S | &t        t        t              j                  dz        } n$t        |       j	                         st        d      t               }|j                  |        |S )zE
    Use configuration object or read and parse a settings file.
    zsettings.cfgz$The given config file does not exist)strr   __file__parentis_fileFileNotFoundErrorr   read)r   r   s     L/root/tools/cai/cai_env/lib/python3.12/site-packages/trafilatura/settings.py
use_configr!      sb     tH~,,~=>(^##% FGG^F
KKM    MIN_EXTRACTED_SIZEMIN_OUTPUT_SIZEMIN_OUTPUT_COMM_SIZEMIN_EXTRACTED_COMM_SIZEMIN_DUPLCHECK_SIZEMAX_REPETITIONSMAX_FILE_SIZEMIN_FILE_SIZE)min_extracted_sizemin_output_sizemin_output_comm_sizemin_extracted_comm_sizemin_duplcheck_sizemax_repetitionsmax_file_sizemin_file_sizeoptionc                     | j                  d|d      j                         }|j                         rt        |      S dS )zJRead an optional positive integer setting; None when empty or non-numeric.DEFAULT )fallbackN)getstripisdigitint)r   r3   values      r    _get_optional_intr=   :   s6    JJy&2J6<<>E3u:2d2r"   c            +       h   e Zd ZU dZg dZeed<   eed<   eed<   eed<   eed<   eed<   eed	<   eed
<   edddddddddddddddddddddede	de
de
de
de
de
de
de
de
de
de	dz  de	dz  de	dz  de
de
d e
d!ee	   dz  d"ee	   dz  d#ee	e	f   dz  f(d$Zde	dz  de	dz  d%dfd&Zd'e	d%dfd(Zded%dfd)Zy)*	Extractorz0Defines a class to store all extraction options.)r   formatfastfocuscomments
formattinglinksimagestablesdeduplangr+   r,   r-   r.   r/   r0   r1   r2   max_tree_sizesourceurlwith_metadataonly_with_metadatatei_validationdate_paramsauthor_blacklisturl_blacklistr+   r,   r-   r.   r/   r0   r1   r2   r   FTN)r   output_formatrA   	precisionrecallrC   rD   rE   rF   rG   rH   rI   rL   rK   rM   rN   rO   rQ   rR   rP   r   rS   rA   rT   rU   rC   rD   rE   rF   rG   rH   rI   rL   rK   rM   rN   rO   rQ   rR   rP   c                r   | j                  ||       | j                  |       | j                  |xs t               || _        |rdn|rdnd| _        || _        |xs | j                  dk(  | _        || _	        |	| _
        |
| _        || _        || _        || _        || _        || _        |xs
 t#               | _        |xs
 t#               | _        |xs |xs t)        |      xs |dk(  | _        |xs% t-        | j.                  j1                  dd            | _        t5        | j.                  d      | _        y )	NrU   rT   balancedr   r   r5   EXTENSIVE_DATE_SEARCHMAX_TREE_SIZE)_set_source_set_format_add_configDEFAULT_CONFIGrA   rB   rC   r@   rD   rE   rF   rG   rH   rI   rL   rN   rO   setrQ   rR   boolrM   set_date_paramsr   
getbooleanrP   r=   rJ   )selfr   rS   rA   rT   rU   rC   rD   rE   rF   rG   rH   rI   rL   rK   rM   rN   rO   rQ   rR   rP   s                        r    __init__zExtractor.__init__r   s"   0 	f%'1>2	&,(+PZ
& * GdkkZ.G 
"" 
 $	"(:$2*:*Cce'4'=#0#z4F#z$}J]#zanrzaz+6 ,
/KK""9.EF;
 *;4;;)Xr"   r   c                 d    |xs |}|xr! |j                  dd      j                  d      | _        y)z)Set the source attribute in a robust way.zutf-8replaceN)encodedecoderK   )rb   rL   rK   s      r    rZ   zExtractor._set_source   s-    Rw	!B!I!I'!Rr"   chosen_formatc                 v    |t         vr*t        ddj                  t        t                            || _        y)z;Store the format if supported and raise an error otherwise.z#Cannot set format, must be one of: z, N)SUPPORTED_FORMATSAttributeErrorjoinsortedr@   )rb   rh   s     r    r[   zExtractor._set_format   s6     11 #FtyyQWXiQjGkFl!mnn#r"   c           	          t         j                         D ]"  \  }}t        | ||j                  d|             $ || _        y)z&Store options loaded from config file.r5   N)CONFIG_MAPPINGitemssetattrgetintr   )rb   r   keyr<   s       r    r\   zExtractor._add_config   s>    (..0 	@JCD#v}}Y>?	@r"   )__name__
__module____qualname____doc__	__slots__r;   __annotations__r]   r   r   r_   r^   dictrc   rZ   r[   r\    r"   r    r?   r?   A   s   6"IJ   
  ." !##($,0)--1-.Y .Y 	.Y
 .Y .Y .Y .Y .Y .Y .Y .Y .Y Dj.Y 4Z.Y  d
!.Y" #.Y$ !%.Y& '.Y( c(T/).Y* 3x$+.Y, #s(^d*-.Y`SsTz S3: S$ S
$ $ $, 4 r"   r?   argsrL   c                 v   t        t        | j                        | j                  | j                  | j
                  | j                  | j                  | j                  | j                  | j                  || j                  | j                  | j                        }dD ]  }t        ||t        | |              |S )z-Derive extractor configuration from CLI args.)r   )r   rS   rD   rT   rU   rC   rG   rH   rI   rL   rM   rN   rO   )rA   rF   rE   )r?   r!   config_filerS   rD   rT   rU   no_comments	no_tablesdeduplicatetarget_languagerM   rN   validate_teirq   getattr)r|   rL   optionsattrs       r    args_to_extractorr      s    4#3#34((??..{{!!~~!!((22((G , 4wtT234Nr"   	extensivec                 P    d| t        j                         j                  d      dS )z/Provide default parameters for date extraction.Tz%Y-%m-%d)original_dateextensive_searchmax_date)r   nowstrftime)r   s    r    r`   r`      s(     %LLN++J7 r"   c            ,       t   e Zd ZdZg dZdddddddddddddddddddddddedz  dedz  dedz  dedz  d	edz  d
edz  dedz  dee   dz  dee   dz  dedz  dedz  dedz  dedz  dedz  dedz  dedz  dedz  dedz  dedz  dedz  dedz  f*dZe	de
eef   dd fd       Zd dZde
eedz  f   fdZy)!DocumentzZDefines a class to store all necessary data and metadata fields for extracted information.)titleauthorrL   hostnamedescriptionsitenamedate
categoriestagsfingerprintidlicensebodyrC   commentsbodyraw_texttextlanguageimagepagetypefiledateN)r   r   rL   r   r   r   r   r   r   r   idvallicense_valr   rC   r   r   r   r   r   r   r   r   r   rL   r   r   r   r   r   r   r   r   r   r   rC   r   r   r   r   r   r   r   c                ^   || _         || _        || _        || _        || _        || _        || _        || _        |	| _        |
| _	        || _
        || _        ||n
t        d      | _        || _        ||n
t        d      | _        || _        || _        || _        || _        || _        || _        y )Nr   )r   r   rL   r   r   r   r   r   r   r   r   r   r   r   rC   r   r   r   r   r   r   r   )rb   r   r   rL   r   r   r   r   r   r   r   r   r   r   rC   r   r   r   r   r   r   r   s                         r    rc   zDocument.__init__   s    2 "'
"("$,'2$, $	,6&*	'2##.&*&6dGFO	$,6B6NlT[\bTc$, $	$,!&
$,$,r"   datar   c                 ^     |        }|j                         D ]  \  }}t        |||        |S )z.Set a series of attributes using a dictionary.)rp   rq   )clsr   docrs   r<   s        r    	from_dictzDocument.from_dict  s5     e**, 	%JCCe$	%
r"   c                     | j                   D ]V  }t        | |      }t        |t              s t	        |      dkD  r|dd dz   }t        t        |            }t        | ||       X y)z*Limit text length and trim the attributes.'  Ni'  u   …)rx   r   
isinstancer   lenr   r   rq   )rb   slotr<   s      r    clean_and_trimzDocument.clean_and_trim'  sa    NN 	+DD$'E%%u:%!%4L50E'8dE*	+r"   c           	      X    | j                   D ci c]  }|t        | |d       c}S c c}w )z%Convert the document to a dictionary.N)rx   r   )rb   r   s     r    as_dictzDocument.as_dict3  s(    <@NNKDgdD$//KKKs   ')r   N)rt   ru   rv   rw   rx   r   listr
   rc   classmethodrz   r   r   r   r   r{   r"   r    r   r      s   `I8 !!#"&#'+!%"& "& $#(,## ##/-- Tz-- d
	--
 4Z-- *-- 4Z-- *-- Dj-- I$-- 3i$-- 4Z-- Tz-- 4Z-- o--  *!--" o#--$ *%--& Dj'--( *)--* Tz+--, *---. */--^ T#s(^ 
  
+Lc3:o. Lr"   r      i   i     i@B r   >   bipqdddtemh1h2h3h4h5h6lidivpremainspanstrongarticlesection
blockquote)4asideembedfencedframefooterformheadiframemenuobjectscriptappletaudiocanvasfiguremappicturesvgvideoareablinkbuttondatalistdialogframeframesetfieldsetlinkinputinslabellegendmarqueemathmenuitemnavnoindexnoscriptoptgroupr3   outputparamprogressrprtrtcselectrK   styletracktextareatimeuse)abbracronymaddressbdibdobigciter   dfnfonthgroupimgr   markmetarubysmalltbodytemplatetfoottheadz[.//aside|.//div[contains(@class|@id, 'footer')]|.//fencedframe|.//footer|.//script|.//style)
r   codedelr   hilbr   r   r   quoteaf	Afrikaansan	AragonesearArabicazAzerbaijanibe
Belarusianbg	BulgarianbnBengalibrBretonbsBosniancaCatalancsCzechcyWelshdaDanishdeGermanelGreekenEnglisheo	EsperantoesSpanishetEstonianeuBasquefaPersianfiFinnishfrFrenchgaIrishglGalicianguGujaratiheHebrewr  HindihrCroatianhtHaitianhu	HungarianhyArmenianr   
Indonesianis	IcelandicitItalianjvJavanesekaGeorgiankkKazakhknKannadakoKoreankuKurdishkyKyrgyzlaLatinr  Luxembourgishlt
LithuanianlvLatvianmk
Macedonianml	MalayalammrMarathimsMalaymtMaltesenbNorwegian_BokmalneNepalinlDutchnnNorwegian_NynorsknoocOccitanplPolishpt
PortuguesequQuechuaroRomanianruRussianskSlovaksl	SloveniansqAlbaniansrSerbiansvSwedishswSwahiliTamilTeluguTagalogTurkish	UkrainianUrdu
VietnameseVolapukWalloon)	tatetltrukurvivowa)NN)N)T)5rw   argparseosconfigparserr   r   r   r   pathlibr   typingr   
lxml.etreer   r	   r
   utilsr   r   _get_affinityr   	cpu_count	CPU_COUNTSUPPORTED_FMT_CLIr^   rj   r   r!   r]   ro   r;   r=   r?   	Namespacer   r_   rz   r`   r   minPARALLEL_CORESLRU_SIZEMAX_FILES_PER_DIRECTORYFILENAME_LEN	MAX_LINKSMAX_SITEMAPS_SEENCUT_EMPTY_ELEMSMANUALLY_CLEANEDMANUALLY_STRIPPEDBASIC_CLEAN_XPATH	frozensetTAG_CATALOGJUSTEXT_LANGUAGESr{   r"   r    <module>r     s    	 %     / / " /6%2%>Ca !\R\\^EXWX	 P )*hZ7 t L44G S_ "  /(28.($$	3l 3C 3C$J 3p pfH.. S4Z 9 ,t tCH~ aL aLJ Y#   	 :8 v 2 wx fgO+O+O 	(O 	-	O
 	,O 	+O 	)O 	(O 	)O 	)O 	'O 	'O 	(O 	(O 	'O  	)!O" 	+#O$ 	)%O& 	*'O( 	()O* 	)+O, 	)-O. 	(/O0 	'1O2 	*3O4 	*5O6 	(7O8 	'9O: 	*;O< 	)=O> 	+?O@ 	*AOB 	,COD 	+EOF 	)GOH 	*IOJ 	*KOL 	(MON 	)OOP 	(QOR 	)SOT 	(UOV 	'WOX 	/YOZ 	,[O\ 	)]O^ 	,_O` 	+aOb 	)cOd 	'eOf 	)gOh 	
iOj 	(kOl 	'mOn 	
oOp 	
qOr 	)sOt 	(uOv 	,wOx 	)yOz 	*{O| 	)}O~ 	(O@ 	+AOB 	*COD 	)EOF 	)GOH 	)IOJ 







[O r"   