
    AHjhC                     >   d Z 	 ddlZdZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlmZ ddlmZ ddlmZmZmZ ddlmZ dd	lmZ dd
lmZmZmZmZ ddlmZ ddlmZmZm Z  ddl!m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z( ddl)m*Z*m+Z+m,Z,m-Z-m.Z. ddl/m0Z0 ddl1m2Z2 ddl3m4Z4m5Z5m6Z6m7Z7 ddl8m9Z9 ddl:m;Z;m<Z<m=Z=m>Z>m?Z?  ej                  eA      ZB ej                  d       ej                  ej                  z   ZF ej                  d      ZH ej                  d      ZI ej                  d      ZJg dZKddddd ZLd!ej                  d"eNeO   fd#ZPd$eOd"eQeO   fd%ZRd!ej                  d"efd&ZSd'eOd"eTfd(ZUd)eOd*eVd"eOfd+ZWd,eOd-eOd"eXeOeOf   fd.ZYd/eOd"eOfd0ZZ	 	 dMd!ej                  d1eOd/eOd2eVd3eOdz  d"eXeOeOf   fd4Z[dNd5eOd!ej                  d2eVd"eOfd6Z\	 	 	 dOd7eOdz  d!ej                  d1eOd2eVd3eOdz  d"dfd8Z]d9eOd"eeOddf   fd:Z^dMd$eOd!ej                  d2eVd;e6dz  d"df
d<Z_d5eOd!ej                  d2eVd;e6dz  d"eVf
d=Z`d>ed!ej                  d2eVd;e6d"eXeNeO   eVf   f
d?Zad!ej                  d"eVfd@Zbd>edAeNeO   d!ej                  d"efdBZc	 	 	 dPd!ej                  dCeVd>edz  d;e6dz  d"df
dDZdd!ej                  d"dfdEZedFeNeO   dGeVd"eVfdHZfd!ej                  d>ed"eVfdIZgd!ej                  d"dfdJZh	 	 dQd5eOeiz  dz  d!ej                  dKeOdz  d;e6dz  d"eOdz  f
dLZjy# e$ r dZY w xY w)Rz1
Functions dedicated to command-line processing.
    NTF)urlsafe_b64encode)	Generator)ProcessPoolExecutorThreadPoolExecutoras_completed)datetime)partial)makedirspathstatwalk)RLock)UrlStoreextract_domainget_base_url)spider   )html2txt)extract)generate_bow_hash)Responseadd_to_compressed_dictbuffered_downloadsbuffered_response_downloadsload_download_buffer)find_feed_urls)reset_caches)FILENAME_LENMAX_FILES_PER_DIRECTORY	Extractorargs_to_extractor)sitemap_search)LANGID_FLAGURL_BLACKLIST_REGEXis_acceptable_lengthlanguage_classifiermake_chunksiY  z[^/]+$z\.[a-z]{2,5}$z<[^<]+?>)URLcrawlexploreprobefeedsitemapz.csvz.jsonz.xml)csvjsonxmlxmlteiargsreturnc                    g }| j                   r:	 t        | j                   d      5 }|j                  d |D               ddd       n&t        D ]  }t        | |      st        | |      g} n |st        j                  d       t        t        j                  |            S # 1 sw Y   exY w# t        $ r t	        j
                  d       Y aw xY w)zGRead list of URLs to process or derive one from command-line arguments.utf-8encodingc              3   <   K   | ]  }|j                           y wN)strip).0lines     M/root/tools/cai/cai_env/lib/python3.12/site-packages/trafilatura/cli_utils.py	<genexpr>z"load_input_urls.<locals>.<genexpr>P   s     !E4$**,!E   Nz+ERROR: system, file type or buffer encodingzNo input provided)
input_fileopenextendUnicodeDecodeErrorsysexitINPUT_URLS_ARGSgetattrLOGGERwarninglistdictfromkeys)r2   
input_urls	inputfileargs       r=   load_input_urlsrP   H   s    J	Ddoo8 FI!!!E9!EEF
 # 	CtS!%dC01
	
 *+ j)**F F! 	DHHBC	Ds(   B0 B$ B0 $B-)B0 0CCfilenamec           	          t        | d      5 }|D ch c]&  }t        j                  d|j                               ( }}ddd       |S c c}w # 1 sw Y   S xY w)zRead list of unwanted URLs.r5   r6    N)rA   r$   subr:   )rQ   inputfhr<   	blacklists       r=   load_blacklistrW   `   s^    	h	) TWKRS4(,,R>S	ST  TT s   A+A
A
AAc                     t        |       }t        || j                  | j                  xr | j                   | j
                  | j                        S )zGRead input list of URLs to process and build a domain-aware dictionary.)rV   compression
url_filterverbose)rP   r   rV   r-   rJ   rZ   r[   )r2   	inputlists     r=   load_input_dictr]   h   sC    %I!..\\3$))m??     	directoryc                     t        j                  |       rt        j                  |       s	 t        | d       yy# t        $ r( t
        j                  j                  d| z   dz          Y yw xY w)z;Check if the output directory is within reach and writable.T)exist_okz0ERROR: Destination directory cannot be created: 
F)r   existsisdirr
   OSErrorrD   stderrwrite)r_   s    r=   check_outputdir_statusrh   u   sd     ;;y!I)>	Y.   	 JJOR[[^bbc	s   ; .A,+A,dirnamecc                 x    |dk\  rt        t        |t        z        dz         nd}t        j                  | |      S )z7Return a destination directory based on a file counter.r   r   rS   )strintr   r   join)ri   rj   c_dirs      r=   determine_counter_dirrp      s5    9:aCA//0145RE99We$$r^   destdir	extensionc                     d}|t        j                  |      rWdj                  d t        t              D              }t        j                  | ||z         }|At        j                  |      rW|fS )zCFind a writable path and return it along with its random file name.NrS   c              3   N   K   | ]  }t        j                  t                y wr9   )randomchoice
CHAR_CLASS)r;   _s     r=   r>   z$get_writable_path.<locals>.<genexpr>   s     R6==4Rs   #%)r   rc   rn   ranger   )rq   rr   output_pathrQ   s       r=   get_writable_pathr{      sf    K

[!977ReL>QRRiiI)=> 
[!9   r^   contentc                 p    t        t        t        j                  d|       d            j	                         S )zaCreate a filename-safe string by hashing the given content
    after deleting potential XML tags.rS      )r   r   	CLEAN_XMLrT   decode)r|   s    r=   generate_hash_filenamer      s+     .y}}R/I2NOVVXXr^   orig_filenamecounternew_filenamec                 x   t         j                  | j                  d      }| j                  rMt        j                  d|      }t        j                  | j                  |      }t        j                  d|      }n%t        | j                  |      }|xs t        |      }t        j                  |||z         }	|	|fS )zPPick a directory based on selected options and a file name based on output type.z.txtrS   )EXTENSION_MAPPINGgetoutput_format	keep_dirs	STRIP_DIRrT   r   rn   
output_dirSTRIP_EXTENSIONrp   r   )
r2   r   r|   r   r   rr   original_dirdestination_dirrQ   rz   s
             r=   determine_output_pathr      s     "%%d&8&8&AI~~ }}R7))DOO\B"&&r=9/IB#9'#B))OX	-ABK''r^   
htmlstringc                    t        |j                  |      }t        |d      \  }}t        |      du rGt        rAt        j                  |d      5 }|j                  | j                  d             ddd       |S |S # 1 sw Y   |S xY w)z-Write a copy of raw HTML in backup directory.z.html.gzTwbr5   N)	rp   
backup_dirr{   rh   HAS_GZIPgziprA   rg   encode)r   r2   r   destination_directoryrz   rQ   
outputfiles          r=   archive_htmlr      s|    1$//7K-.CZPK34<YY{D) 	9ZZ..w78	9O8O	9Os   !A<<Bresultc                    | y|j                   #t        j                  j                  | dz          yt	        ||| ||      \  }}t        |      du r)t        |dd      5 }|j                  |        ddd       yy# 1 sw Y   yxY w)z-Deal with result (write to STDOUT or to file)Nrb   Twr5   )moder7   )r   rD   stdoutrg   r   rh   rA   )r   r2   r   r   r   destination_pathr   r   s           r=   write_resultr      s     ~

$',A$W]_fht,u)/!/2d:&S7C )z  () ) ;) )s    A<<Binputdirc              #   r   K   t        |       D ]%  \  }}}|D ]  }t        j                  ||        ' yw)z2Walk the directory tree and output all file names.N)r   r   rn   )r   rootrx   
inputfilesfnames        r=   generate_filelistr      s@     #H~ )a 	)E))D%((	))s   57optionsc                    |st        |      }| |_        t        | d      5 }|j                         }ddd       t	        |       }t        |j                  |j                        }t        j                  |      j                  d      |j                  d<   t        ||      }t        ||| |d       y# 1 sw Y   xY w)z1Aggregated functions to process a file in a list.rbNz%Y-%m-%dmax_dater   )r   )r!   sourcerA   readr   minst_ctimest_mtimer   fromtimestampstrftimedate_paramsexaminer   )	rQ   r2   r   r   inputfr   	file_statref_timestampr   s	            r=   file_processingr      s    #D)GN	h	 #[[]
# XI	**I,>,>?M&.&<&<]&K&T&TU_&`G
#Zw7FxtD# #s   B55B>c                     |j                   rt        | ||      nd}t        | ||      }t        |||||       |dk\  r|r|dz  }|S )zVExtract text and metadata from a download webpage and eventually write out the result.rS   r   )r   r   r   r   r   )r   r   r   r   )r   r2   r   r   fileslugr   s         r=   process_resultr      sP     ;?//|Jg6rHZw7FXwU]^!|1Nr^   	url_storec                 t   g }|j                   j                  dd      }| j                  st        | |      \  }} t	        ||j
                  |      D ]T  \  }}|r&t        |t              r||_        t        ||||      }.t        j                  d|       |j                  |       V | j                  s||fS )z?Implement a download queue consumer, single- or multi-threaded.DEFAULT
SLEEP_TIMEr   zNo result for URL: %s)configgetfloatdoner   r   parallel
isinstancerl   urlr   rH   rI   append)	r   r2   r   r   errors
sleep_time
bufferlistr   r   s	            r=   download_queue_processingr      s     F((LAJnn 4Y
 K
I-j$--QXY 	#KC*VS1!(wH6<c"	# nn 7?r^   c           	        	
 t        |       }|j                         }| j                  r|j                          t	        |       }t        | j                  rt        nt        | j                  |j                  j                  dd      |j                  j                  dd            
t               }t        | j                        5 		
fd|D        }t!        |      D ]  }|j#                         |j%                  |j#                                | j                  s@t'        |j)                               | j                  k\  sg|5  |j+                          |j                          t-                ddd        	 ddd       t/        | |      }| j0                  rt3        |||       }t5        | ||       |S # 1 sw Y   xY w# 1 sw Y   JxY w)	z/Group CLI functions dedicated to URL discovery.r   EXTERNAL_URLSr   )target_langexternalr   max_workersc              3   B   K   | ]  }j                  |        y wr9   )submit)r;   r   executorfuncs     r=   r>   z cli_discovery.<locals>.<genexpr>!  s     D#8??4-Ds   N)r   r   )r]   	dump_urlsrJ   resetr!   r	   r,   r   r"   target_languager   
getbooleanr   r   r   r   r   r   add_urlslenget_known_domainsprint_unvisited_urlsr   url_processing_pipeliner*   build_exploration_dictcli_crawler)r2   r   rM   r   lockfuturesfuture	exit_codecontrol_dictr   r   s            @@r=   cli_discoveryr     sy   %I$$&Jyy%G))((**9oF>>**9lC	D 7D 
	6 '(DD #7+ 	'F}}*""6==?399Y%@%@%B!Ct}}!T '!668!)$' '	'	' (i8I ||-iTJDL'B' '' 's6   +G4+G &GG	+F<4
G<GGGrM   c                 8   |D ch c]  }t        |       }}|| j                         D ch c]  }t        |       c}z
  }|D cg c]  }t        |      |v s| }}t        ||j                  |j                  |j
                        S c c}w c c}w c c}w )zMFind domains for which nothing has been found and add info to the crawl dict.)rV   rZ   r[   )r   r   r   rV   rZ   r[   )r   rM   r2   uinput_domainsstill_to_crawlnew_input_urlss          r=   r   r   :  s    0:;1^A&;M;"A\A\A^%_AnQ&7%__N!+SA~a/@N/RaSNS!..??	  <%_Ss   BBBBnc                 0   |xs t        |       }|j                  j                  dd      }i }|)t        j                  j                  t        |              n|t        _        t        j                  j                         D ]r  }t        j                  j                  |   j                  s+t        j                  j                  |d      }|sOt        j                  || j                        ||<   t t        j                  j                  st        t        j                  |      \  }t        _        t        || j                   |      D ];  \  }	}
|
s	t#        |
t$              st        j&                  |
|t)        |	                = t+        fdt        j                  j-                         D              rnt        j                  j                  st/        d	j1                  d
 t        j                  j3                         D                     y)z~Start a focused crawler which downloads a fixed number of URLs within a website
    and prints the links found in the process.r   r   NF)
as_visited)langr   c              3   (   K   | ]	  }|k\    y wr9    )r;   rj   r   s     r=   r>   zcli_crawler.<locals>.<genexpr>k  s     A!qAvAs   rb   c              3       K   | ]  }|  y wr9   r   )r;   r   s     r=   r>   zcli_crawler.<locals>.<genexpr>n  s     <!A<s   )r!   r   r   r   	URL_STOREr   rP   r   urldicttuplesget_url
init_crawlr   r   r   r   r   r   r   process_responser   anyget_all_countsprintrn   r   )r2   r   r   r   r   
param_dicthostname	startpager   r   r   s    `         r=   r   r   G  s    0*40G((LAJJ !!/$"78$ $$668 _##H-44((00e0LI'-'8'8I]I]'^
8$	_ ##';F<L<Lj'Y$
F$6z4==Zab 	OKC*VX6''
<;L0MN	O Av//>>@AA ## 
$))<v//99;<
<=r^   c                 \   t        |       }t        |       }t        || j                  |      D ]{  \  }}|	t	        |      }|st        |      |j                  kD  s0t        d |D              sCt        r&| j                  rt        |d      | j                  k(  sot        |d       } y)zBProbe websites for extractable content and print the fitting ones.r   Nc              3   <   K   | ]  }|j                           y wr9   )isalpha)r;   rj   s     r=   r>   z!probe_homepage.<locals>.<genexpr>y  s     Jg[\199;Jgr?   rS   T)flush)rP   r!   r   r   r   r   min_extracted_sizer   r#   r   r&   r   )r2   rM   r   r   r   s        r=   probe_homepager  q  s     &J%G)*dmmWU +Vf%F#f+(B(BBsJg`fJgGg"$*>*>BUV\^`BaeieyeyBy#T*+r^   r   totalc                 @    |dkD  rt        |       |z  nd}|dkD  ry| ryy)zvCompute exit code based on the number of errors:
    0 if there are no errors, 126 if there are too many, 1 otherwise.r   gGz?~   r   )r   )r   r  ratios      r=   _define_exit_coder  ~  s-     $)19CK%!Et|r^   c                 f   | j                   r|j                          yt        |       }|j                         }|t        kD  rdnd}t        || ||      \  }}t        j                  dt        |      |       | j                  du rt               }|j                  |D cg c]  }d|z   	 c}       t        |j                  d            dkD  rTt        || ||      \  }}t        j                  dt        |      t        |             t        ||j                               S t        ||      S c c}w )	zKAggregated functions to show a list and download and process an input list.Fr   z%s / %s URLs could not be foundTzhttps://web.archive.org/web/20/zhttps://web.archive.orgz-%s archived URLs out of %s could not be found)rJ   r   r!   total_url_numberr   r   rH   debugr   archivedr   r   find_known_urlsr  )	r2   r   r   	url_countr   r   earchived_errorsrx   s	            r=   r   r     s   yy&&(%G**,I66aBG 0	4'ROFG
LL2CKK}}J	6Ra=ARSy(()BCDqH!:9dGU\!]OQLL?O$F %_i6P6P6RSSVY// Ss   D.c                    d}t        |       }|j                  j                  dd      }t        | j                        5 }t        t        | j                        t              D ]W  }|dk  rt        |      t        k\  rd}t        t        | ||      }|j                  ||d|       |dk\  sJ|t        |      z  }Y 	 d	d	d	       y	# 1 sw Y   y	xY w)
zGDefine batches for parallel file processing and perform the extraction.r
  r   EXTRACTION_TIMEOUTr   r   )r2   r   r   
   )	chunksizetimeoutN)r!   r   getintr   r   r'   r   	input_dirr   r   r	   r   map)r2   filecounterr   r  r   	filebatchworkers          r=   file_processing_pipeliner    s    K%Gnn##I/CDG 
	7 	.8$%6t~~%FH_` 	.IQ3y>5L#L_4V]^FLLb'LJas9~-	.	. 	. 	.s    A-C.CCr   c                    d}|st        ||      }| !t        j                  j                  d       |S t	        t        |       |      s!t        j                  j                  d       |S 	 t        | |      }|S # t        $ rL}t        j                  j                  dt        |       dt        j                          d       Y d}~|S d}~ww xY w)z;Generic safeguards and triggers around extraction function.NzERROR: empty document
zERROR: file size
r   zERROR: rb   )r!   rD   rf   rg   r%   r   r   	Exceptionrl   	traceback
format_exc)r   r2   r   r   r   errs         r=   r   r     s     F#D#.

23 M "#j/7;

-. M	OZ9F M  	OJJws3xj93G3G3I2J"MNNM	Os   +A: :	CAC

C)r
  N)r
  )rS   r
  N)   NN)NN)k__doc__r   r   ImportErrorargparseloggingru   restringrD   r   base64r   collections.abcr   concurrent.futuresr   r   r   r   	functoolsr	   osr
   r   r   r   	threadingr   courlanr   r   r   trafilaturar   baseliner   corer   deduplicationr   	downloadsr   r   r   r   r   feedsr   metar   settingsr   r   r    r!   sitemapsr"   utilsr#   r$   r%   r&   r'   	getLogger__name__rH   seedascii_lettersdigitsrw   compiler   r   r   rF   r   	NamespacerJ   rl   rP   setrW   r]   boolrh   rm   rp   tupler{   r   r   r   r   r   r   r   r   r   r   r   r  r  r   r  bytesr   r   r^   r=   <module>rF     s  H    	  
  $ % T T   ) )  : :    , ~ ~ !   %  
		8	$ C !!FMM1
BJJy!	"**-.BJJ{#	I 	 +(,, +c +0S SX 
(,, 
 
c d  %3 %3 %3 %!s !s !uS#X !YC YC Y #(


(( ( 	(
 *( 38_(4	S 	(:(: 	S 	RU 	 #)$J)


) ) 	)
 *) 
)() )	#tT/(B )Ec E);); Ec EYbeiYi Euy E"
s 
(*<*< 
s 
U^aeUe 
jm 
'11<?JS
49c>*(** (s (V
h 
DI 
XM_M_ 
dl 
 !% $	'>


'>
'> $'> 	'>
 
'>T
+++ 
+ 
+	d3i 	 	 	0("4"4 0 0c 0<.8#5#5 .$ .,  $	ed"


 
t 	
 	4Ze  Hs   L LL