
    AHj/P                        d Z ddlZddlZddlZddlZddlZ	 ddlZdZ	 ddl	Z	dZ
ddlmZmZ ddlmZ ddlmZmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZ ddlmZ ddl m!Z!m"Z" ddl#m$Z$ ddl%m&Z&m'Z'm(Z(  ejR                  e*      Z+ G d d      Z, e,       Z- G d de      Z. G d d      Z/ G d d      Z0 G d d      Z1de2de1fdZ3y# e$ r dZY w xY w# e$ r dZ
Y w xY w) zd
Defines a URL store which holds URLs along with relevant information and entails crawling helpers.
    NTF)defaultdictdeque)Callable)datetime	timedelta)Enum)
itemgetter)Lock)Any)RobotFileParser   )normalize_url)filter_links)lang_filtervalidate_url)clear_caches)get_base_urlget_host_and_pathis_known_linkc                   `    e Zd ZdZdZddeddfdZededefd       Z	dedefd	Z
dedefd
Zy)
CompressorzYUse system information on available compression modules and define corresponding methods.)
compressordecompressorcompressionreturnNc                    |  |  |r2t         r,t        j                  t        j                  c| _        | _        y |r2t        r,t        j                  t        j                  c| _        | _        y | j                  x| _        | _        y N)	HAS_BZ2bz2compress
decompressr   r   HAS_ZLIBzlib
_identical)selfr   s     H/root/tools/cai/cai_env/lib/python3.12/site-packages/courlan/urlstore.py__init__zCompressor.__init__1   sW    714s~~.DOT.X15.DOT.26//ADOd/    datac                     | S )zReturn unchanged data. )r)   s    r&   r$   zCompressor._identical;   s	     r(   c                 N    | j                  t        j                  |d            S )z9Pickle the data and compress it if a method is available.   )protocol)r   pickledumpsr%   r)   s     r&   r    zCompressor.compress@   s    v||D1=>>r(   c                 J    t        j                  | j                  |            S )zADecompress the data if a method is available and load the object.)r/   loadsr   r1   s     r&   r!   zCompressor.decompressD   s    ||D--d344r(   T)__name__
__module____qualname____doc__	__slots__boolr'   staticmethodr   r$   r    bytesr!   r+   r(   r&   r   r   ,   sh    _.IBD BD B    ?S ?S ?5u 5 5r(   r   c                       e Zd ZdZdZdZdZy)Statez0Record state information about a domain or host.r         N)r5   r6   r7   r8   OPENALL_VISITEDBUSTEDr+   r(   r&   r>   r>   L   s    6DKFr(   r>   c                   <    e Zd ZdZdZej                  fdeddfdZy)DomainEntryz7Class to record host-related information and URL paths.)countrulesstate	timestamptotaltuplesrH   r   Nc                 h    d| _         d | _        || _        d | _        d| _        t               | _        y )Nr   )rF   rG   rH   rI   rJ   r   rK   )r%   rH   s     r&   r'   zDomainEntry.__init__Y   s.    
-1
!
*.
+07r(   )r5   r6   r7   r8   r9   r>   rA   r'   r+   r(   r&   rE   rE   T   s$    =KI&+jj 3e 3T 3r(   rE   c                   4    e Zd ZdZdZdededdfdZdefdZy)	UrlPathTuplezBClass storing information for URL paths relative to a domain/host.)urlpathvisitedrO   rP   r   Nc                 >    |j                  d      | _        || _        y )Nutf-8)encoderO   rP   )r%   rO   rP   s      r&   r'   zUrlPathTuple.__init__g   s    %nnW5$r(   c                 8    | j                   j                  d      S )zGet the URL path as string.rR   )rO   decoder%   s    r&   pathzUrlPathTuple.pathk   s    ||""7++r(   )	r5   r6   r7   r8   r9   strr:   r'   rW   r+   r(   r&   rN   rN   b   s0    H&I% %d %t %,c ,r(   rN   c                   t   e Zd ZdZdZ	 	 	 	 	 dEdededz  dededed	dfd
Zd	eee	f   fdZ
deee	f   d	dfdZ	 dFdee   ded	eeee   f   fdZded	ee   fdZdGdZ	 	 	 dHdedee   dz  dedz  dee   dz  d	df
dZdIdee   dedz  d	ee   fdZ	 	 	 dJdee   dz  dee   dz  ded	dfdZ	 	 	 dKdededed edz  d!ed	dfd"Zd#ee   d	dfd$ZdGd%Zd	ee   fd&Zd	ee   fd'Zded	efd(Zd	efd)Zded	ee   fd*Zded	ee   fd+Z dee   d	ee   fd,Z!dee   d	ee   fd-Z"ded	efd.Z#ded	efd/Z$dLded0ed	edz  fd1Z%	 	 dMd2e&d3ed	ee   fd4Z'	 dNd3ed2ed	ee   fd5Z(d6ed7e)dz  d	dfd8Z*d6ed	e)dz  fd9Z+dOd6ed:e&d	e&fd;Z,d	ee   fd<Z-d	efd=Z.d>e&d	efd?Z/d	ee   fd@Z0dGdAZ1dGdBZ2dCed	dfdDZ3y)PUrlStorezNDefines a class to store domain-classified URLs and perform checks against it.)
compresseddonelanguagestricttrailing_slashurldict_lockNr[   r]   r^   r_   verboser   c                     | _         d _        | _        | _        | _        t        t               _        t                _	        dt        dt        dd f fd}|rjt        j                  j                  d      sJ	 t        j                  t        j                  |       t        j                  t        j                   |       y y y # t"        $ r t$        j'                  d       Y y w xY w)NFnumframer   c                     t         j                  dt        j                               j	                          t        j                  d       y )Nz<Processing interrupted, dumping unvisited URLs from %s hostsr   )LOGGERdebuglenr`   print_unvisited_urlssysexit)rd   re   r%   s     r&   dump_unvisited_urlsz.UrlStore.__init__.<locals>.dump_unvisited_urls   s7    LLNDLL! %%'HHQKr(   winz2Cannot set signal handlers outside the main thread)r[   r\   r]   r^   r_   r   rE   r`   r
   ra   r   rk   platform
startswithsignalSIGINTSIGTERM
ValueErrorrg   warning)r%   r[   r]   r^   r_   rb   rm   s   `      r&   r'   zUrlStore.__init__}   s     !+	$,"$26A+6N6
	S 	 	 	 3<<2259Ufmm-@Afnn.AB :7  USTUs   >AC	 	C*)C*c                 b    | j                   D ci c]  }|dk7  s	|t        | |       c}S c c}w )z;Return the picklable state, excluding the unpicklable lock.ra   )r9   getattr)r%   slots     r&   __getstate__zUrlStore.__getstate__   s-    6:nnXdPWgdD))XXXs   
,,rH   c                 l    |j                         D ]  \  }}t        | ||        t               | _        y)z6Restore state after unpickling and re-create the lock.N)itemssetattrr
   ra   )r%   rH   rx   values       r&   __setstate__zUrlStore.__setstate__   s0     ;;= 	'KD%D$&	'V
r(   r)   rP   c                 p   t        t              }t        j                  |      D ]  }	 t	        |      \  }}|du s|t
        j                  d|       t        | j                  Jt        || j                  | j                  | j                        du rt
        j                  d|       t        t        || j                  | j                  | j                        }t        |      \  }}	||   j                  t        |	|              |S # t         t        f$ r t
        j#                  d|       Y w xY w)NFzInvalid URL: %szWrong language: %s)r^   r]   r_   zDiscarding URL: %s)r   r   dictfromkeysr   rg   rh   rt   r]   r   r^   r_   r   r   appendrN   	TypeErrorru   )
r%   r)   rP   	inputdicturlvalidation_result
parsed_url
normalizedhostinforO   s
             r&   _buffer_urlszUrlStore._buffer_urls   s&    <Gu;M	==& 	:C:0<S0A-!:$-1CLL!2C8$$ MM-#T]]DKK9L9L 
 LL!5s;$$*;;!]]#'#6#6	
 %6j$A!'(#**<+IJ3	:8  z* :3S9:s   CD%D54D5domainc                     || j                   v rQ| j                  r,t        j                  | j                   |   j                        S | j                   |   j                  S t               S r   )r`   r[   
COMPRESSORr!   rK   r   r%   r   s     r&   
_load_urlszUrlStore._load_urls   sQ    T\\!!,,T\\&-A-H-HII<<'...wr(   c                     | j                   sHt        d | j                  j                         D              r| j                  5  d| _         d d d        y y y # 1 sw Y   y xY w)Nc              3   V   K   | ]!  }|j                   t        j                  k7   # y wr   )rH   r>   rA   .0vs     r&   	<genexpr>z%UrlStore._set_done.<locals>.<genexpr>   s      V1EJJ!6 Vs   ')T)r\   allr`   valuesra   rV   s    r&   	_set_donezUrlStore._set_done   sS    yyS V@S@S@U VV ! 	! ! Wy! !s   AAto_rightrI   to_leftc                 x   |j                  d      rd|dd  z   }|| j                  v rS|}nP|j                  d      r?d|dd  z   }|| j                  v r)| j                  |   | j                  |<   | j                  |= || j                  v rZ| j                  |   j                  t        j                  u ry | j                  |      }|D ch c]  }|j                          c}nt               }t               ||j                  fd|D               ||j                  fd|D               | j                  5  | j                  r(t        j                  |      | j                  |   _        n|| j                  |   _        t!        |      | j                  |   _        ||| j                  |   _        t'        d	 |D              r#t        j(                  | j                  |   _        n5t        j*                  | j                  |   _        | j,                  rd
| _        d d d        y c c}w # 1 sw Y   y xY w)Nzhttp://https   zhttps://httpr-   c              3   X   K   | ]!  }t        |j                               r| # y wr   r   rW   r   tknowns     r&   r   z'UrlStore._store_urls.<locals>.<genexpr>   s      Ra=53QR   **c              3   X   K   | ]!  }t        |j                               r| # y wr   r   r   s     r&   r   z'UrlStore._store_urls.<locals>.<genexpr>   s      U!mAFFHe6TAUr   c              3   4   K   | ]  }|j                     y wr   )rP   )r   us     r&   r   z'UrlStore._store_urls.<locals>.<genexpr>  s     +199+   F)rp   r`   rH   r>   rC   r   rW   r   setextend
extendleftra   r[   r   r    rK   ri   rJ   rI   r   rB   rA   r\   )	r%   r   r   rI   r   	candidateurlsr   r   s	           @r&   _store_urlszUrlStore._store_urls   s    Y'&*,IDLL("z*+IDLL('+||I'>V$LL+ T\\!||F#))U\\9??6*D'+,!QVVX,E7DEE KKR8RROOUwUUZZ 	&.8.A.A$.GV$+.2V$+),TDLL &$1:V$.+d++-2->->V$*-2ZZV$*99 %DI	& 	& -	& 	&s   	H+<C&H00H9r   switchc                 >   d }i }t         j                  |      }t        |      D ]g  }t        |      \  }}||k7  r9|}| j	                  |      D 	ci c]  }	|	j                         |	j                   }}	||v sT|dk(  s|dk(  s_||   se||= i t        |      S c c}	w )Nr   r?   )r   r   sortedr   r   rW   rP   list)
r%   r   r   last_domainknown_pathsremaining_urlsr   r   rO   r   s
             r&   _search_urlszUrlStore._search_urls  s    "&.0t,.) 
	(C 1# 6Hg;&&<@OOH<UVqqvvx2VV+%!!G0D"3'
	( N## Ws   "B
appendleftc                     |r;| j                  ||      j                         D ]  \  }}| j                  ||        |r<| j                  ||      j                         D ]  \  }}| j                  ||        yy)zAdd a list of URLs to the (possibly) existing one.
        Optional: append certain URLs to the left,
        specify if the URLs have already been visited.)r   )r   N)r   r{   r   )r%   r   r   rP   host	urltupless         r&   add_urlszUrlStore.add_urls"  s     #'#4#4T7#C#I#I#K ;i  	 :;#'#4#4Z#I#O#O#Q :i  y 9: r(   
htmlstringr   externallangwith_navc           	          t        |      }| j                  |      }t        ||||xs | j                  || j                  |      \  }}	| j                  ||	       y)zJFind links in a HTML document, filter them and add them to the data store.)r   r   r   r   rG   r^   r   )r   r   N)r   	get_rulesr   r]   r^   r   )
r%   r   r   r   r   r   base_urlrG   linkslinks_prioritys
             r&   add_from_htmlzUrlStore.add_from_html2  s_      $x( ,!&;;!
~ 	5^<r(   domainsc                    | j                   5  |D ])  }t        t        j                        | j                  |<   + 	 ddd       | j                          t        j                         }t        j                  d|       y# 1 sw Y   DxY w)z)Declare domains void and prune the store.)rH   Nz'%s objects in GC after UrlStore.discard)
ra   rE   r>   rC   r`   r   gccollectrg   rh   )r%   r   drd   s       r&   discardzUrlStore.discardI  so    ZZ 	B B"-ELL"AQB	B 	jjl>D	B 	Bs   /B  B	c                     | j                   5  t        t              | _        ddd       t	                t        j                         }t        j                  d|       y# 1 sw Y   >xY w)zRe-initialize the URL store.Nz UrlStore reset, %s objects in GC)	ra   r   rE   r`   r   r   r   rg   rh   )r%   rd   s     r&   resetzUrlStore.resetR  sI    ZZ 	4&{3DL	4jjl7=		4 	4s   AA(c                 H    t        | j                  j                               S )z#Return all known domains as a list.)r   r`   keysrV   s    r&   get_known_domainszUrlStore.get_known_domains\  s    DLL%%'((r(   c                     | j                   j                         D cg c]%  \  }}|j                  t        j                  k(  s$|' c}}S c c}}w )ziFind all domains for which there are unvisited URLs
        and potentially adjust done meta-information.)r`   r{   rH   r>   rA   )r%   r   r   s      r&   get_unvisited_domainszUrlStore.get_unvisited_domains`  s8     #ll002Ldaagg6KLLLs   %AAc                 t    || j                   v r*| j                   |   j                  t        j                  k7  S y)z9Tell if all known URLs for the website have been visited.F)r`   rH   r>   rA   r   s     r&   is_exhausted_domainzUrlStore.is_exhausted_domaine  s/    T\\!<<'--;;r(   c                 4    t        | j                               S )zFReturn the number of websites for which there are still URLs to visit.)ri   r   rV   s    r&   unvisited_websites_numberz"UrlStore.unvisited_websites_numberl  s    4--/00r(   c                 l    | j                  |      D cg c]  }||j                         z    c}S c c}w )zLGet all already known URLs for the given domain (ex. "https://example.org").)r   rW   r%   r   r   s      r&   find_known_urlszUrlStore.find_known_urlsr  s*    +/??6+BCa!CCCs   1c                     | j                  |      s=| j                  |      D cg c]"  }|j                  r||j                         z   $ c}S g S c c}w )z,Get all unvisited URLs for the given domain.)r   r   rP   rW   r   s      r&   find_unvisited_urlszUrlStore.find_unvisited_urlsv  sF    ''//3v/FX!aiiFQVVX%XX	 Ys
   AAc                 (    | j                  |d      S )z:Take a list of URLs and return the currently unknown ones.r   r   r   r%   r   s     r&   filter_unknown_urlszUrlStore.filter_unknown_urls|        a 00r(   c                 (    | j                  |d      S )z<Take a list of URLs and return the currently unvisited ones.r?   r   r   r   s     r&   filter_unvisited_urlszUrlStore.filter_unvisited_urls  r   r(   c                 :    t        | j                  |g             S )z0Check if the given URL has already been visited.)r:   r   )r%   r   s     r&   has_been_visitedzUrlStore.has_been_visited  s    22C59:::r(   c                     t        |      \  }}|| j                  |      D ch c]  }|j                          c}v S c c}w )z/Check if the given URL has already been stored.)r   r   rW   )r%   r   r   rO   r   s        r&   is_knownzUrlStore.is_known  s9    -c2'T__X-FG1668GGGGs   >
as_visitedc                 "   | j                  |      s| j                  |      }|D ]  }|j                  r|rdd|_        | j                  5  | j                  |   xj
                  dz  c_        ddd       | j                  ||t        j                                ||j                         z   c S  | j                  5  t        j                  | j                  |   _        ddd       | j                          y# 1 sw Y   xY w# 1 sw Y   &xY w)zSRetrieve a single URL and consider it to be visited (with corresponding timestamp).Tr   NrI   )r   r   rP   ra   r`   rF   r   r   nowrW   r>   rB   rH   r   )r%   r   r   
url_tuplesr   s        r&   get_urlzUrlStore.get_url  s     ''/0J! /{{!&*!ZZ < LL066!;6<((x||~(V!CHHJ../ ZZ 	;).):):DLL &	;< <
	; 	;s   
#C9=#D9D	D
time_limitmax_urlsc                    g }| j                   j                         D ]  \  }}|j                  t        j                  k7  r$|j
                  r3t        j                         |j
                  z
  j                         |kD  sc| j                  |      }|w|j                  |       t        |      |k\  s n | j                          |S )zaGet a list of immediately downloadable URLs according to the given
        time limit per domain.)r`   r{   rH   r>   rA   rI   r   r   total_secondsr   r   ri   r   )r%   r   r   r   websiteentryr   s          r&   get_download_urlszUrlStore.get_download_urls  s     "ll002 	NGU{{ejj(OOLLNU__4CCE
Rll7+?KK$4yH,	 	r(   c                    | j                         }|sg S |t        |      z  xs d}g }|D ]b  }| j                  |      }g }|D ]  }	t        |      |k\  st        |      t        |      z   |k\  r nl|	j                  r:|j	                  |	j                                d|	_        | j                  5  | j                  |   xj                  dz  c_        ddd        t        j                         }
| j                  |   j                  }|r|
|z
  j                         |kD  rd}n!|t        |
|z
  j                         d      z
  }|D ]  }|j	                  |||z   f       ||z  } |
t        d||z
        z   }| j                  |||       e | j!                          t#        |t%        d            S # 1 sw Y   }xY w)	zcGet up to the specified number of URLs along with a suitable
        backoff schedule (in seconds).r   TNg        z.2fr   r   )key)r   ri   r   rP   r   rW   ra   r`   rF   r   r   rI   r   floatr   r   r   r   r	   )r%   r   r   	potential
per_domaintargetsr   r   urlpathsr   r   original_timestampschedule_secsrO   
total_diffs                  r&   establish_download_schedulez$UrlStore.establish_download_schedule  s    ..0	IY/41
+- "	GF0J"$H! 
8MZ/Gs8}4A{{OOCHHJ/"&CK 8V,22a728 8
8 ,,.C!%f!5!?!?&,,;;=
J # *U00??A#F. ! $ ,v/?@A+, yMJ,FGGJVZ:FE"	GH 	g:a=11/8 8s   0#F77Gr   rG   c                 n    | j                   rt        j                  |      }|| j                  |   _        y)z)Store crawling rules for a given website.N)r[   r   r    r`   rG   )r%   r   rG   s      r&   store_ruleszUrlStore.store_rules  s*    ??''.E&+W#r(   c                     || j                   v rQ| j                  r,t        j                  | j                   |   j                        S | j                   |   j                  S y)z7Return the stored crawling rules for the given website.N)r`   r[   r   r!   rG   )r%   r   s     r&   r   zUrlStore.get_rules  sM    dll"!,,T\\'-B-H-HII<<(...r(   defaultc                 v    d}| j                  |      }	 |j                  d      }|xs |S # t        $ r Y w xY w)zBReturn the delay as extracted from robots.txt, or a given default.N*)r   crawl_delayAttributeError)r%   r   r  delayrG   s        r&   get_crawl_delayzUrlStore.get_crawl_delay   sK    w'	%%c*E   		s   , 	88c                 p    | j                   j                         D cg c]  }|j                   c}S c c}w )z2Return all download counts for the hosts in store.)r`   r   rF   )r%   r   s     r&   get_all_countszUrlStore.get_all_counts  s'    !%!4!4!67A777s   3c                 V    t        d | j                  j                         D              S )z!Find number of all URLs in store.c              3   4   K   | ]  }|j                     y wr   )rJ   r   s     r&   r   z,UrlStore.total_url_number.<locals>.<genexpr>  s     :q177:r   )sumr`   r   rV   s    r&   total_url_numberzUrlStore.total_url_number  s     :DLL$7$7$9:::r(   	thresholdc                 \    t        fd| j                  j                         D              S )z^Find out if the download limit (in seconds) has been reached for one of the websites in store.c              3   <   K   | ]  }|j                   k\    y wr   )rF   )r   r   r  s     r&   r   z6UrlStore.download_threshold_reached.<locals>.<genexpr>  s     GA177i'Gs   )anyr`   r   )r%   r  s    `r&   download_threshold_reachedz#UrlStore.download_threshold_reached  s!    G1D1D1FGGGr(   c                 l    g }| j                   D ]"  }|j                  | j                  |             $ |S )z Return a list of all known URLs.)r`   r   r   )r%   r   r   s      r&   	dump_urlszUrlStore.dump_urls  s6    ll 	6FKK,,V45	6r(   c                 |    | j                   D ]-  }t        dj                  | j                  |            d       / y)z"Print all unvisited URLs in store.
TflushN)r`   printjoinr   r   s     r&   rj   zUrlStore.print_unvisited_urls   s5    ll 	KF$))D44V<=TJ	Kr(   c                     | j                   D ]c  }t        dj                  | j                  |      D cg c],  }| |j	                          dt        |j                         . c}      d       e yc c}w )z5Print all URLs in store (URL + TAB + visited or not).r  	Tr  N)r`   r  r  r   rW   rX   rP   r   s      r&   
print_urlszUrlStore.print_urls%  sr    ll 		F		 "&!8 "(1668*Bs199~.>? 		s   1A4filenamec                 r    t        |d      5 }t        j                  | |       ddd       y# 1 sw Y   yxY w)zWrite the URL store to disk.wbN)openr/   dump)r%   r   outputs      r&   writezUrlStore.write4  s1    (D! 	&VKKf%	& 	& 	&s   -6)FNFTF)F)r   N)NNNr   )NNF)FNTr4   )g      $@i'  )d   
   )r-   )4r5   r6   r7   r8   r9   r:   rX   r'   r   r   ry   r~   r   r   r   rN   r   r   r   r   r   intr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r  r   r	  r  r  r  r  rj   r  r&  r+   r(   r&   rZ   rZ   p   s   TI !##UU *U 	U
 U U 
UBYd38n Y$sCx. T  05 I (, 	S%--	. D |)< ! 04%).24&4& %,4& d?	4&
 |$t+4& 
4&l$c $C$J $$s) $0 "&'+	:3i$: I$: 	:
 
:( == = 	=
 Dj= = 
=.EtCy ET E>)49 )MtCy M
# $ 13 1Dc Dd3i D# $s) 1S	 1d3i 11$s) 1S	 1;C ;D ;HC HD Hc t sTz . !  
c	0 682222/222	c22l,3 ,/E ,$ , 4)? 	 s 	 U 	 5 	 8S	 8;# ;HE Hd H49 K
&c &d &r(   rZ   r   r   c                 t    t        | d      5 }t        j                  |      }ddd       |S # 1 sw Y   S xY w)zLoad a URL store from disk.rbN)r#  r/   load)r   r%  	url_stores      r&   
load_storer.  :  s8    	h	 (KK'	((s   -7)4r8   r   loggingr/   rq   rk   r   r   ImportErrorr#   r"   collectionsr   r   collections.abcr   r   r   enumr   operatorr	   	threadingr
   typingr   urllib.robotparserr   cleanr   corer   filtersr   r   metar   urlutilsr   r   r   	getLoggerr5   rg   r   r   r>   rE   rN   rZ   rX   r.  r+   r(   r&   <module>r>     s    
    
GH
 + $ (     .    .  D D			8	$5 5: \
D 3 3, ,G& G&T  W  G  Hs"   C C CCC)(C)