
    AHj5                        d Z ddlZddlZddlZddlmZmZ ddlmZm	Z	 ddl
mZ ddlmZ ddlmZ dd	lmZ dd
lmZ  ej(                  e      Zdee   dz  dej0                  fdZ	 	 	 	 ddee   dedededz  dedeeeef      fdZdedeee      fdZdej0                  ddfdZdej0                  ddfdZ dej0                  ddfdZ!ddZ"edk(  r e"        yy)z,
Implements a basic command-line interface.
    N)IteratorSequence)ProcessPoolExecutoras_completed)	ExitStack)islice   )	check_url)_make_sample)UrlStoreargsreturnc                    t        j                  d      }|j                  dd      }|j                  dddt        d	       |j                  d
ddt        d	       |j                  dddt               |j                  dddd       |j                  dddt
               |j                  dd      }|j                  ddd       |j                  dddt               |j                  d d!d"d       |j                  d#d$      }|j                  d%d&t
               |j                  d'd(t
               |j                  d)d*t
               |j                  |       S )+z(Define parser for command-line argumentsz"Command-line interface for Courlan)descriptionzI/OzManage input and outputz-iz--inputfilezname of input file (required)T)helptyperequiredz-oz--outputfilezname of output file (required)z-dz--discardedfilez/name of file to store discarded URLs (optional))r   r   z-vz	--verbosezincrease output verbosity
store_true)r   actionz-pz
--parallelz4number of parallel processes (not used for sampling)	FilteringzConfigure URL filtersz--strictzperform more restrictive testsz-lz
--languagez$use language filter (ISO 639-1 code)z-rz--redirectszcheck redirectsSamplingz+Use sampling by host, configure sample sizez--samplezsize of sample per domainz--exclude-maxz%exclude domains with more than n URLsz--exclude-minz%exclude domains with less than n URLs)argparseArgumentParseradd_argument_groupadd_argumentstrint
parse_args)r   
argsparsergroup1group2group3s        C/root/tools/cai/cai_env/lib/python3.12/site-packages/courlan/cli.pyr   r      s   ((8J **52KLF
,   -   >	   k ;L   C	   **;8OPF
9,   l!Gc   m"3L   **AF 
)D3O
EC   EC     &&    urlsstrictwith_redirectslanguagewith_navc                     g }| D ]>  }t        |||||      }||j                  d|d   f       ,|j                  d|f       @ |S )z6Internal function to be used with CLI multiprocessing.)r&   r'   r(   r)   Tr   F)r
   append)r%   r&   r'   r(   r)   resultsurlresults           r#   _cli_check_urlsr/   N   se     G ))
 NND&),-NNE3<() Nr$   	inputfilec              #      K   t        | dd      5 }	 t        |d      D cg c]  }|j                          }}|s
	 ddd       y| 8c c}w # 1 sw Y   yxY ww)zRead input line in batchesutf-8ignoreencodingerrorsi N)openr   strip)r0   inputfhlinebatchs       r#   _batch_linesr<   f   se     	i'(	; w.4We.DEdTZZ\EEE	 
 K	 E s+   AAAA	A
AAAc                 .   | j                   r$t        j                  t        j                         n#t        j                  t        j
                         t        dd| j                  | j                         }t        | j                        D ]  }|j                  |        t        | j                  dd      5 }t        || j                  | j                  | j                         D ]  }|j#                  |dz           	 ddd       y# 1 sw Y   yxY w)	zSample URLs on the CLI.TN)
compressedr(   r&   verbosewr2   r5   )exclude_minexclude_max
)r?   LOGGERsetLevelloggingDEBUGERRORr   r&   r<   r0   add_urlsr7   
outputfiler   samplerB   rC   write)r   urlstorer;   outputfhr-   s        r#   _cli_samplerP   p   s    ||&&$t{{DLLH dnn- !% ! 
doosW	5 'KK((((	
 	'C NN3:&	'' ' 's   :ADDc                 $    t               5 }|j                  t         j                  dd            }|j                  t	         j
                              |j                  t         j                  dd            } j                  '|j                  t         j                  dd            nd}	 g }t        |      dk  rJt        |d      D cg c]  }|j                          }}|sn |j                  |       t        |      dk  rJ|snc fd	|D        }t        |      D ]H  }	|	j                         D ]3  \  }
}|
r|j                  |d
z          | |j                  |d
z          5 J 	 ddd       yc c}w # 1 sw Y   yxY w)z7Read input file bit by bit and process URLs in batches.r2   r3   r4   )max_workersr@   rA   Ni  c              3      K   | ]<  }j                  t        |j                  j                  j                          > yw))r&   r'   r(   N)submitr/   r&   	redirectsr(   ).0r;   r   executors     r#   	<genexpr>z_cli_process.<locals>.<genexpr>   sD      	  #;;#'>>!]]   	s   AArD   )r   enter_contextr7   r0   r   parallelrK   discardedfilelenr   r8   r+   r   r.   rM   )r   stackr9   rO   	discardfhbatchesr:   
line_batchfuturesfuturevalidr-   rW   s   `           @r#   _cli_processrd      s   	 '4%%'(C
 &&':t}}'UV&&tDOOS7'ST !!- T%7%7w OP 	
 ')Gg,%7=gt7LMtdjjlM
M!z*	 g,% 	 %	G 'w/ 4"(--/ 4JE3 sTz2".!d
3	44-  +'4 '4 N'4 '4s+   CFF,$FAFFFFc                 J    | j                   rt        |        yt        |        y)z+Start processing according to the argumentsN)rL   rP   rd   r   s    r#   process_argsrg      s    {{DTr$   c                  R    t        t        j                  dd       } t        |        y)zRun as a command-line utility.r	   N)r   sysargvrg   rf   s    r#   mainrk      s    chhqrl#Dr$   __main__)FFNF)r   N)#__doc__r   rG   ri   collections.abcr   r   concurrent.futuresr   r   
contextlibr   	itertoolsr   corer
   samplingr   rN   r   	getLogger__name__rE   r   	Namespacer   listbooltupler/   r<   rP   rd   rg   rk    r$   r#   <module>r{      sA     
 . @     " 			8	$7'Xc]T) 7'h.@.@ 7'x  
s)  Dj	
  
%c	
0C HT#Y$7 'h(( 'T '.)4x)) )4d )4Xx)) d  zF r$   