AHjCdZddlZddlZddlZddlmZmZddlmZm Z ddl m Z ddl m Z ddlmZddlmZdd lmZdd lmZddlZddlZdd lmZdd lmZd dlmZmZd dlm Z m!Z!m"Z"m#Z#m$Z$ ddl%m&Z&ejNjQdZ) ddl+Z+e+jXZ-e-j]e+j^e+j`e-j]e+j^e+jbdZ2ejfe4Z5ejlejnjpda9da:da;dedejxezfdZ=dJdZ>ej~jdZAe rdeAdvr eAdxxdz cc<dedzdzZBeBeAd<gd ZChd!ZDGd"d#ZEd$e deFeGeHdzeHdzffd%ZIdKd$e d&eJeHeHfdzdeJeHeHffd'ZKd$e dej~jfd(ZMdLd$e d)eNdejxezfd*ZOd+eHd)eNd,eNd$e deEdzf d-ZPd+eHd.eEd/edeNfd0ZQd+eHd.eEd1eNd/edeEeHzdzf d2ZRdedfd+eHd)eNd$e d/edzdeHdzf d3ZSddded4d+eHd1eNd)eNd,eNd$e deEdzf d5ZTd+eHdeNfd6ZUd+eHdeNfd7ZVd+eHdeNfd8ZW dMd9eGeHd:eXeHdzd;eHdzdeNdefd?ZYdNd #6:;<@BMRS[ceZdZdZgdZdedededdfdZde fd Z defd Z d e eefddfd Z d e ddfdZde eeffdZy)Responsez5Store information gathered in a HTTP response object.dataheadershtmlstatusurlr@rCrDrNcJ||_d|_d|_||_||_yNr?)selfr@rCrDs r%__init__zResponse.__init__ps% .2 $  r'c|jduSrF)r@rGs r%__bool__zResponse.__bool__wsyy$$r'cH|jxst|jSrF)rBrr@rJs r%__repr__zResponse.__repr__zsyy2K 22r' headerdictc~|jDcic]\}}|j|c}}|_ycc}}w)z#Store response headers if required.N)itemslowerrA)rGrNkvs r% store_headerszResponse.store_headers}s02<1A1A1CDA 1 D Ds9decodecX|r(|jrt|j|_yyy)z9Decode the bytestring in data and store a string in html.N)r@rrB)rGrUs r% decode_datazResponse.decode_datas" dii#DII.DI 6r'cV|jDcic]}|t||c}Scc}w)z,Convert the response object to a dictionary.) __slots__getattr)rGattrs r%as_dictzResponse.as_dicts&6:nnEdgdD))EEEs&)__name__ __module__ __qualname____doc__rYbytesintstrrHboolrKrMdictrTrWr\rr'r%r>r>ks;configc|jdddj}|r|jnd}|jddxsd}||fS)zARead and extract HTTP header strings from the configuration file.DEFAULT USER_AGENTS)fallbackNCOOKIE)getstrip splitlines)rfmyagents agent_listmycookies r% _parse_configrssUzz)]Rz@FFHH*2$$&Jzz)X.6$H x r'rAc|tk7r1t|\}}i}|rtj||d<|r||d<|xstS)z1Internal function to decide on user-agent string.r1Cookie)rrsrandomchoiceDEFAULT_HEADERS)rfrArprrs r%_determine_headersrysL *62( $*MM($;GL !  (GH   %o%r'c ts[tjj|j dd|j ddd|j dddz t atS)z5Define a retry strategy according to the config file.rh MAX_REDIRECTSrDOWNLOAD_TIMEOUT)totalredirectconnectbackoff_factorstatus_forcelist)RETRY_STRATEGYr!utilRetrygetint FORCE_STATUSrfs r%_get_retry_strategyrs^  ++-- ?;]]9o>!==4FG!K) ,  r'no_sslc|rtnt}|s@t|jdd|rdnt j |rdnd}|r|a|S|a|S)zXCreate a urllib3 pool manager according to options in the config file and HTTPS setting.rhr|N CERT_NONE CERT_REQUIRED)timeoutca_certs cert_reqs) NO_CERT_POOL HTTP_POOLr&rcertifiwhere)rfrpools r%_initiate_poolrsZ"rarCgeturlrTrAr! exceptionsSSLErrorLOGGERwarning_send_urllib_request Exceptionerror) rDrrrf pool_managerresponser@ max_file_sizechunkresperrs r%rrs=8%fV<  ''  &v.'/! ( { iA __U+ ;E KK 4y=( !9:: ; d X__hoo6GH    x// 0    & &E4c:#C|VDD 8 ,c377 8s&BC0A(C00AE3E;EEroptionsct|jxs|jxsd}|jdk7r"tj d|j|yt ||syy)z2Check if the response conforms to formal criteria.rjz!not a 200 response: %s for URL %sFT)rrBr@rCrrr)rDrrlentests r%_is_suitable_responsersR(--68==6B7G# 8(//3O  1 r'rUc>t|||r|r |jS|Sy)z:Internal function to run safety checks on response result.N)rrB)rDrrUrs r%_handle_responsers$S(G4 &x}}4H4 r'c|r |jn|}t|d||}|r3|jr'|s t|}t |||r |j Sy)aDownloads a web page and seamlessly decodes the response. Args: url: URL of the page to fetch. no_ssl: Do not try to establish a secure connection (to prevent SSLError). config: Pass configuration values for output control. options: Extraction options (supersedes config). Returns: Unicode string or None in case of failed downloads and invalid results. T)rUrrfrN)rffetch_responser@rrrB)rDrrfrrs r% fetch_urlrsP$ 'W^^FFc$vfMHHMMv.G h 8== r')rUrrrfctstnt}tj d||||||}|stj d|y|j ||S)aDownloads a web page and returns a full response object. Args: url: URL of the page to fetch. decode: Use html attribute to decode the data (boolean). no_ssl: Don't try to establish a secure connection (to prevent SSLError). with_headers: Keep track of the response headers. config: Pass configuration values for output control. Returns: Response object or None in case of failed downloads and invalid results. zsending request: %szrequest failed: %sN) HAS_PYCURLr_send_pycurl_requestrdebugrW)rDrUrrrf dl_functionrs r%rr sV*/9&>RK LL&,3 f=H  )3/  Or'cd}tj}|jtj|j d|jtj d|jtj d|jtjd|jtjdt| |j|jtjdk}|j!|S#tj$r#}tjd||d}Yd}~Cd}~wwxYw) z+Send a basic HTTP HEAD request with pycurl.Futf-8 rTizpycurl HEAD error: %s %sN)r*Curlr)URLencodeCONNECTTIMEOUTSSL_VERIFYPEERSSL_VERIFYHOSTNOBODYr-performgetinfo RESPONSE_CODErrrclose)rD page_existsr,rs r%_pycurl_is_live_pager?sK ;;=DKK CJJw/0KK%%r*KK%%q)KK%%q)KK t$d ll6#7#783>  JJL  << /c: s2DE )EE cv t|}y#t$r!}tjd||Yd}~yd}~wwxYw)zGUse courlan redirection test (based on urllib3) to send a HEAD request.zurllib3 HEAD error: %s %sNFT)rrrr)rD_rs r%_urllib3_is_live_pagerZs< S !   0#s;s  838cFtr t|nd}|xs t|S)zCSend a HTTP HEAD request without taking anything else into account.F)rrr)rDresults r% is_live_pagerds"*4 !# &%F  /*3//r' inputlist blacklist url_filter url_store compressionverbosec<|t|d|}ttj|}|r(|Dcgc]}t j d||vs|}}|r%|Dcgc]t fd|Ds}}|j||Scc}wcc}w)zMFilter, convert input URLs and add them to domain-aware processing dictionaryF) compressedstrictrrjc3&K|]}|v ywrFr).0fus r% z)add_to_compressed_dict..}s0LAa0Ls)r listrefromkeysrsubanyadd_urls)rrrrrrrs `r%add_to_compressed_dictrks E7S T]]9-.I )]1-@-D-DR-KS\-\Q] ] )M1S0L0L-LQM M y!  ^NsBB!B;B sleep_timech |j|d}|s |jr ||fSt|2)zRDetermine threading strategy and draw URLs respecting domain-based back-off rules.i) time_limitmax_urls)get_download_urlsdoner )rr bufferlists r%load_download_bufferrsA 00JQV0W   y   j r'rdownload_threadsworker chunksizec # Kt|5}t||D]I}|Dcic]}|j|||}}t|D]}|||j fK dddycc}w#1swYyxYww)z3Use a thread pool to perform a series of downloads.) max_workersN)rrsubmitrr) rrrrexecutorrrD future_to_urlfutures r%_buffered_downloadsrs (8 9=X Y7 =EJOP3X__VS93>PMP&}5 =#F+V]]_<< = ===P==s, BA7A2+A7) B2A77B<Bc>tt|}t|||S)z3Download queue consumer, single- or multi-threaded.)r)rrr)rrrrs r%buffered_downloadsrs Y 0F z+;V DDr'cf|r |jnt}tt|}t |||S)z7Download queue consumer, returns full Response objects.r)rfrrrr)rrrrfrs r%buffered_response_downloadsrs- 'W^^NF ^F 3F z+;V DDr'ct|jDcgc] \}}|d|}}}tj}|j tj |j d|j tjt|j tj||j tjd|j tj|jdd|j tj|jdd|j tj|jdd|j tj|jdd|j tj d|durA|j tj"d |j tj$d n2|j tj&t)j*|r4t-}|j tj.|j0t3| |j5} tC| |jEtjF|jEtjH} |jK|ri} jMjOddjQD]=} d| vr| jSdd\}}|jU| |jU<?| jW| | Scc}}w#tj6$rd} t8j7d || |d ur?| j:d t<vr*t8j?d || tA|d||cYd } ~ SYd } ~ y d } ~ wwxYw)zDExperimental function using libcurl and pycurl to speed up downloadsz: rrrhr{r|rTrzpycurl error: %s %sFzretrying after SSL error: %s %sNz iso-8859-1replace)errors:),ryrPr*rr)rrSHARE CURL_SHARE HTTPHEADERFOLLOWLOCATION MAXREDIRSrrTIMEOUT MAXFILESIZENOSIGNALrrCAINFOrrr HEADERFUNCTIONwriter- perform_rbrrrCURL_SSL_ERRORSrrr>rr EFFECTIVE_URLrgetvaluerUrosplitrnrT)rDrrrfheadercontent headerlistr, headerbytes bufferbytesrr respheaderslinenamevalues r%rrs ASSY@Z@`@`@bc_VWVHBwi(cJc ;;=DKK CJJw/0KK j)KK!!:.KK%%q)KK  &-- ?"KLKK%%v}}Y@R'STKK i9K LMKK""FMM)_$MNKK# ~ F))1- F))1- FMM7==?3i  F));+<+<=d oo' " Kf.B.B!CT\\RXRfRfEg hDJJL ((*11,y1Q\\^ 6D$**S!,KD%(- K % 6 ;' KKdH <<  *C5 U?sxx{o= LL:C E'T<H H s$M1M$$O7AO OO)r,z pycurl.CurlrNrF)F)NNNFF)g@)i')ar`loggingosrvcollections.abcrrconcurrent.futuresrr configparserr functoolsrimportlib.metadatar ior timer typingr rr!courlanr courlan.networkrsettingsrrutilsrrrrrurllib3.contrib.socksrenvironrmr ImportErrorr* CurlSharerr)SH_SHARE LOCK_DATA_DNSLOCK_DATA_SSL_SESSIONr getLoggerr]rdisable_warningsrInsecureRequestWarningrrrr"r&r-r make_headersrx USER_AGENTrrr>tuplerrcrsreryrrrdrrrrrrrrrsetrfloatrrbrrrrrr'r%r;s} /?%&,/7 |,I !!!#Jfoov';';<foov'C'CDJ   8 $++BBC   11 3 3c 911 ,,++D+A o.?@@%&'1& gm4 47a a *  (?FFF , 5cT1A3:1M+N  &| &d38nt6K &W[\_ad\dWe &  1C1C  <'BUBUX[B[*!c!4!t!\!^fim^m!H s h  t   *.9B ^d) $        4Z >)         _>cd6st0c0d0"&!!% Cy3x$d $     0!H!%!%PTUXPY[cPcJd! =S  = = cUCZ  = = uS#Xd*+ ="!%ES EE EuS#Xd*+ E!% ES  E E  EuS(]#T4/0 EJcJ4JtJ\J^fim^mJY IJs%.!L+A#L9+L65L69MM