
    AHj"                     N   U d Z ddlZddlZddlmZ ddlmZ ddlmZ ddlZ	 ddl	m
Z ddlmZ ddlmZmZmZ d	d
lmZ  ej*                  e      ZddhZee   ed<    ej8                  ej:                  j<                         ej>                  jA                  ddg d      Z! ejD                  e!      Z# edddd      Z$ ejJ                  dejL                        Z' ejJ                  dejL                        Z( edd       G d d             Z)dee*z  ez  dz  de+fdZ,de*de+fdZ-de*de.e   fdZ/d e*ez  defd!Z0d"edefd#Z1d$ededz  fd%Z2d&ede+fd'Z3d(ed&edefd)Z4d*ededz  fd+Z5d*e*ez  ez  dedz  fd,Z6d-ed.e.e   defd/Z7d0edefd1Z8y# e$ r dZY zw xY w)2z7
Module bundling functions related to HTML processing.
    N)	dataclass)datetime)Any)detect)
from_bytes)HtmlElement
HTMLParser
fromstring   )MAX_FILE_SIZEutf-8utf_8UNICODE_ALIASES   )i  i  i  i  i  )totalconnectstatus_forcelist)retriesFT)collect_idsdefault_doctypeencoding
remove_pisz^< ?! ?DOCTYPE.+?/ ?>z(<html.*?)\s*/>)slotseqc                   D    e Zd ZU dZeed<   eed<   eed<   eed<   eed<   y)	Extractorz0Defines a class to store all extraction options.	extensivemaxminoriginalformatN)__name__
__module____qualname____doc__bool__annotations__r   str     F/root/tools/cai/cai_env/lib/python3.12/site-packages/htmldate/utils.pyr   r   /   s    6O	M	MNKr*   r   datareturnc                 0    |  xs t        |       t        kD  S )z6Check if the input object is suitable to be processed.)lenr   r,   s    r+   is_wrong_documentr1   :   s    80s4y=00r*   c                 F    	 | j                  d       y# t        $ r Y yw xY w)zLSimple heuristic to determine if a bytestring uses standard unicode encodingzUTF-8FT)decodeUnicodeDecodeErrorr0   s    r+   isutf8r5   ?   s,    G   s    	  bytesobjectc                 Z   t        |       rdgS g }t        /t        |       d   }||j                  |j                                t	        | dd       xs t	        |       }|j                  |D cg c]  }|j                   c}       |D cg c]  }|t        vs| c}S c c}w c c}w )z<Read all input or first chunk and return a list of encodingsr   Nr   i:  )r5   cchardet_detectappendlowerr   extendr   r   )r6   guessescchardet_guessdetection_resultsrgs         r+   detect_encodingrA   H   s     kyG"(5jA%NN>//12 #;v#67R:k;RNN(9:1AJJ:; ;!!?":A;; ; <s   0B#B(B(filecontentc                     t        | t              r| S d}t        |       D ]  }	 | j                  |      } n |xs t        | dd      S # t        t
        f$ r t        j                  d|       Y Qw xY w)znGuess bytestring encoding and try to decode to Unicode string.
    Resort to destructive conversion otherwise.Nzwrong encoding detected: %sr   replace)r   errors)
isinstancer(   rA   r3   LookupErrorr4   LOGGERwarning)rB   htmltextguessed_encodings      r+   decode_filerL   ^   s     +s#H+K8 	"))*:;H  Ks;KK /0 	LNN8:JK	Ls   A

%A21A2responsec                 T    t        | d      r| j                  n| }|rt        |      S dS )zRead the data from a response object exposing the body via ``.data``
    (e.g. urllib3 or a compatible response) or from a bytestring, then guess
    its encoding and decode it to return a unicode string.r,    )hasattrr,   rL   )rM   r,   s     r+   decode_responserQ   q   s*     $Hf58==8D $;t,",r*   urlc                 r   	 t         j                  d| d      }|j                  dk7  r"t        j	                  d|j                  |        yt        |j                        rt        j	                  d|        yt        |j                        S # t        $ r!}t        j	                  d| |       Y d}~yd}~ww xY w)	a,  Fetches page using urllib3 and decodes the response.

    Args:
        url: URL of the page to fetch.

    Returns:
        HTML code as string, or Urllib3 response object (headers + body), or empty string in case
        the result is invalid, or None if there was a problem with the network.

    GET   )timeout   z!not a 200 response: %s for URL %szincorrect input data for URL %szdownload error: %s %sN)		HTTP_POOLrequeststatusrH   errorr1   r,   rQ   	Exception)rR   rM   errs      r+   	fetch_urlr^   {   s    2 $$UC$<
 ??c!LL<hoosS
 	 x}}-LL:C@  #8==11  8,c377 8s   B 	B6B11B6	beginningc                 
    d| vS )zOAssess if the object is proper HTML (awith a corresponding tag or declaration).htmlr)   )r_   s    r+   is_dubious_htmlrb      s    ""r*   
htmlstringc                 &   d|v r3| j                  d      \  }}}t        j                  d|d      dz   |z   } t        | j	                               D ]=  \  }}d|v r,|j                  d      rt        j                  d| d      }  | S |d	kD  s< | S  | S )
z>Repair faulty HTML strings to make then palatable for libxml2.doctype
rO   r   )countz<htmlz/>z\1>   )	partitionDOCTYPE_TAGsub	enumerate
splitlinesendswithFAULTY_HTML)rc   r_   	firstline_restilines          r+   repair_faulty_htmlru      s     I'11$7	1d __R!_<tCdJ
Z2245 4d?t}}T2$1EJ  q5 r*   
htmlobjectc                     	 t        | j                  d      t              S # t        $ r }t        j                  d|       Y d}~yd}~ww xY w)z!Try to pass bytes to LXML parser.utf8parserzlxml parser bytestring %sN)r
   encodeHTML_PARSERr\   rH   r[   )rv   r]   s     r+   fromstring_bytesr}      sD    7*++F3KHH 70#667s   " 	AAAc                    t        | t              r| S t        | t        t        f      st	        dt        |              t        | t              rH| j                  d      r7d| vr3t        j                  d|        t        |       }|t        d|        |} d}t        |       } | dd j                         }t        | |      } d}	 t        | t        	      }|t'        |      dk  r|st!        |       }|:t)        |      r/t'        |      dk  r!t        j%                  dt'        |             d}|S # t        $ r d
}t!        |       }Y st"        $ r }t        j%                  d|       Y d}~d}~ww xY w)zkLoad object given as input and validate its type
    (accepted: lxml.html tree, bytestring and string)
    zincompatible input type: http zURL detected, downloading: %sNzURL couldn't be processed: 2   Fry   Tzlxml parsing failed: %sr   rh   z9parsed tree length: %s, wrong data type or not valid HTML)rF   r   bytesr(   	TypeErrortype
startswithrH   debugr^   
ValueErrorrL   r:   ru   r
   r|   r}   r\   r[   r/   rb   )rv   
downloadedtreer_   fallback_parser]   s         r+   	load_htmlr      sk   
 *k*j5#,/3D4D3EFGG 	:s#!!&)z!4jAz*
::,GHH
DZ(J3B%%'I#J	:JN5*[9 	D	A~
+ OI63t9q=GT	
 K!  ,
+ 5.445s   D3 3E3E3E..E3r   elemlistc                     | j                  |      D ]  }	 |j                           | S # t        $ r& |j                         }||j	                  |       Y Fw xY w)zDelete selected elements.)iter	drop_treeAttributeError	getparentremove)r   r   elementparents       r+   
clean_htmlr      sd    99X& '	'	' K	  	'&&(F!g&	's   *,AAstringc                 @    dj                  | j                               S )z7Remove superfluous space and normalize remaining space.r   )joinsplit)r   s    r+   	trim_textr      s    88FLLN##r*   )9r%   loggingredataclassesr   r   typingr   urllib3cchardetr   r8   ImportErrorcharset_normalizerr   	lxml.htmlr   r	   r
   settingsr   	getLoggerr"   rH   r   setr(   r'   disable_warnings
exceptionsInsecureRequestWarningutilRetryRETRY_STRATEGYPoolManagerrX   r|   compileIrj   ro   r   r   r&   r1   r5   listrA   rL   rQ   r^   rb   ru   r}   r   r   r   r)   r*   r+   <module>r      sL    	 !   2 * 9 9 #			8	$$g.S .   ++BB C##
. $ 
  G7	uw4 bjj0"$$7bjj+RTT2 %   !1C%K+5< 1 1
 4 < <49 <,LUS[ LS L&-c -c -3 3: :#s #t #
3 3 3   t); 2%#+3 2d8J 2j[ DI + $c $c $O  Os   F F$#F$