
    AHjF                      	   d Z ddlZddlZddlmZ ddlmZ ddlmZ ddlm	Z
 ddlmZ ddlmZ d	d
lmZmZ d	dlmZmZ d	dlmZmZmZmZ  ej4                  e      Z eddddddgddddd      ZdZdZdZ  eedz         Z! ed      Z"dZ#dZ$dZ% ejL                  d      Z' ejL                  de% de$ de# d e# d!e$ d"      Z( ejL                  de% de$ d#e$ d$e% d%	      Z)d&Z* ejL                  d'e* d(e# d)e% d*e# d+e* d,e% d-jW                  d.d/      ejX                        Z- ejL                  d0e% d1e$ d1e# d2      Z. ejL                  d3e% d4e$ d4e# d-ejX                        Z/ ejL                  d5e% d4e$ d4e# d-ejX                        Z0 ejL                  d6e% d4e$ d4e# d7      Z1g d8Z2 e3e2d	9      D  ci c]  \  } }|D ]  }||   c}}} Z4 ejL                  d:      Z5 ejL                  d;      Z6 ejL                  d<ejX                        Z7 ejL                  d6e# d=e$ d=e% d-      Z8 ejL                  d6e# d>e$ d?e# d@e$ dA	      Z9 ejL                  d6e$ d=e% d-      Z: ejL                  dBe% d-      Z; ejL                  dCe% dDe% dE      Z< ejL                  dF      Z= ejL                  dG      Z> ejL                  dH      Z? ejL                  dI      Z@ ejL                  dJ      ZA ejL                  d6e% dK      ZB ejL                  dLe% d-      ZC ejL                  dM      ZD ejL                  d6e% dN      ZE ejL                  dO      ZF ejL                  dP      ZG ejL                  dQ      ZH ejL                  d6e% dR      ZI ejL                  dS      ZJ ejL                  d6e% dK      ZK ejL                  dTe% dE      ZLdUedVefdWZMdXeNdz  dYedVeNdz  fdZZOd[ePd\ePdVeQePePf   fd]ZRd^eNdVedz  fd_ZSd^eNd`eNdaedbedVeNdz  f
dcZTd^eNd`eNdVeNdz  fddZU eee      d^eNdz  d`eNdfeVdaedbedVeNdz  fdg       ZWdUedYedVeNdz  fdhZXdieNdjej                  eN   dYedVeNdz  fdkZZdUedYedVeNdz  fdlZ[dmeNdYedVeNdz  fdnZ\yc c}}} w )oz:
Custom parsers and XPath expressions for date extraction
    N)datetime)	lru_cache)DateDataParser)parse)XPath)HtmlElement   )
CACHE_SIZEMAX_SEGMENT_LEN)	Extractor	trim_text)convert_datecorrect_yearis_valid_datevalidate_and_convertTzcustom-formatszabsolute-timepastF)	NORMALIZEPARSERSPREFER_DATES_FROMPREFER_LOCALE_DATE_ORDERRETURN_AS_TIMEZONE_AWARESTRICT_PARSING)	languageslocalesregionsettingszr.//*[self::div or self::h2 or self::h3 or self::h4 or self::li or self::p or self::span or self::time or self::ul]z.//*a  
[
    contains(translate(@id|@class|@itemprop, "D", "d"), 'date') or
    contains(translate(@id|@class|@itemprop, "D", "d"), 'datum') or
    contains(translate(@id|@class, "M", "m"), 'meta') or
    contains(@id|@class, 'time') or
    contains(@id|@class, 'publish') or
    contains(@id|@class, 'footer') or
    contains(@class, 'info') or
    contains(@class, 'post_detail') or
    contains(@class, 'block-content') or
    contains(@class, 'byline') or
    contains(@class, 'subline') or
    contains(@class, 'posted') or
    contains(@class, 'submitted') or
    contains(@class, 'created-post') or
    contains(@class, 'publication') or
    contains(@class, 'author') or
    contains(@class, 'autor') or
    contains(@class, 'field-content') or
    contains(@class, 'fa-clock-o') or
    contains(@class, 'fa-calendar') or
    contains(@class, 'fecha') or
    contains(@class, 'parution') or
    contains(@id, 'footer-info-lastmod')
] |
.//footer | .//small
z/text()z).//div[@id="wm-ipp-base" or @id="wm-ipp"]z[0-3]?[0-9]z[0-1]?[0-9]z199[0-9]|20[0-3][0-9]z\b(\d{8})\bz(?:\D|^)(?:(?P<year>z)[\-/.](?P<month>z)[\-/.](?P<day>z)|(?P<day2>z)[\-/.](?P<month2>z")[\-/.](?P<year2>\d{2,4}))(?:\D|$)z)|(?P<month2>z)[\-/.](?P<year2>z
))(?:\D|$)u  
January?|February?|March|A[pv]ril|Ma[iy]|Jun[ei]|Jul[iy]|August|September|O[ck]tober|November|De[csz]ember|
Jan|Feb|M[aä]r|Apr|Jun|Jul|Aug|Sep|O[ck]t|Nov|De[cz]|
Januari|Februari|Maret|Mei|Agustus|
Jänner|Feber|März|
janvier|février|mars|juin|juillet|aout|septembre|octobre|novembre|décembre|
Ocak|Şubat|Mart|Nisan|Mayıs|Haziran|Temmuz|Ağustos|Eylül|Ekim|Kasım|Aralık|
Oca|Şub|Mar|Nis|Haz|Tem|Ağu|Eyl|Eki|Kas|Ara
z
(?P<month>z)\s
(?P<day>z)(?:st|nd|rd|th)?,? (?P<year>z)|
(?P<day2>z))(?:st|nd|rd|th|\.)? (?:of )?
(?P<month2>z)[,.]? (?P<year2>)
 z\D(z)[/_-](z	)(?:\D|$)z"dateModified": ?"(-z"datePublished": ?"((z).[0-9]{2}:[0-9]{2}:[0-9]{2}))janjanuaru   jännerjanuaryjanuarijanvierocakoca)febfebruarfeberfebruaryfebruariu   févrieru   şubatu   şub)maru   märu   märzmarchmaretmartmars)apraprilavrilnisannis)maymaimeiu   mayıs)junjunijunejuinhaziranhaz)juljulijulyjuillettemmuztem)augaugustagustusu   ağustosu   ağuaout)sep	september	septembreu   eylüleyl)octoktoberoctoberoctobreoktekimeki)novnovemberu   kasımkasnovembre)decdezdezemberdecemberdesemberu	   décembreu   aralıkara)startz[.:,_/ -]|^\d+$u   ^\d{2}:\d{2}(?: |:|$)|^\D*\d{4}\D*$|[$€¥Ұ£¢₽₱฿#₹]|[A-Z]{3}[^A-Z]|(?:^|\D)(?:\+\d{2}|\d{3}|\d{5})\D|ftps?|https?|sftp|\.(?:com|net|org|info|gov|edu|de|fr|io)\b|IBAN|[A-Z]{2}[0-9]{2}|®u  (?:date[^0-9"]{,20}|updated|last-modified|published|posted|on)(?:[ :])*?([0-9]{1,4})[./]([0-9]{1,2})[./]([0-9]{2,4})|(?:Datum|Stand|Veröffentlicht am):? ?([0-9]{1,2})\.([0-9]{1,2})\.([0-9]{2,4})|(?:güncellen?me|yayı(?:m|n)lan?ma) *?(?:tarihi)? *?:? *?([0-9]{1,2})[./]([0-9]{1,2})[./]([0-9]{2,4})|([0-9]{1,2})[./]([0-9]{1,2})[./]([0-9]{2,4}) *?(?:'de|'da|'te|'ta|’de|’da|’te|’ta|tarihinde) *(?:güncellendi|yayı(?:m|n)landı)z)[/.-](z)/(z)/([0-9]{2})|(z)[.-](z)[.-]([0-9]{2})z^\D?(u$   (?:©|\&copy;|Copyright|\(c\))\D*(?:z)?-?(z)\Dz"/([0-9]{4}/[0-9]{2}/[0-9]{2})[01/]z ([0-9]{4})/([0-9]{2})/([0-9]{2})z(\D([0-9]{4}[/.-][0-9]{2}[/.-][0-9]{2})\Dz(([0-9]{4})[/.-]([0-9]{2})[/.-]([0-9]{2})z-\D([0-3]?[0-9][/.-][01]?[0-9][/.-][0-9]{4})\Dz)\D?$z^(zE(\D19[0-9]{2}[01][0-9][0-3][0-9]\D|\D20[0-9]{2}[01][0-9][0-3][0-9]\D)z)([01][0-9])([0-3][0-9])zK\D([0-3]?[0-9]/[01]?[0-9]/[0129][0-9]|[0-3][0-9]\.[01][0-9]\.[0129][0-9])\Dz([0-9]{2})$z(\D([12][0-9]{3}[/.-](?:1[0-2]|0[1-9]))\Dz)[/.-](1[0-2]|0[1-9]|)z!\D([01]?[0-9][/.-][12][0-9]{3})\Dz(?<!w3.org)\D(treereturnc                 d    t        |       D ]!  }|j                         j                  |       # | S )z-Delete unwanted sections of an HTML document.)DISCARD_EXPRESSIONS	getparentremove)ra   subtrees     K/root/tools/cai/cai_env/lib/python3.12/site-packages/htmldate/extractors.pydiscard_unwantedri      s2    &t, ,""7+,K    testurloptionsc                    | t         j                  |       }|rxt        j                  d|d          	 t	        t        |d         t        |d         t        |d               }t        ||j                  |j                  |j                        S y# t        $ r$}t        j                  d|d   |       Y d}~yd}~ww xY w)	zEExtract the date out of an URL string complying with the Y-M-D formatNzfound date in URL: %sr   r	         earliestlatestzconversion error: %s %s)COMPLETE_URLsearchLOGGERdebugr   intr   formatminmax
ValueError)rk   rl   match
dateobjecterrs        rh   extract_url_dater      s    
 ##G,LL0%(;G%c%(mSq]CaMR
+W[[ 
   G6a#FFGs   AB 	C B;;C daymonthc                 &    |dkD  r	| dk  r|| fS | |fS )z/Swap day and month values if it seems feasible.    )r   r   s     rh   try_swap_valuesr      s!     2:#)E3<E#uErj   stringc                    t         j                  |       }|sy|j                  dk(  rdnd}	 t        |j	                  |d               t        t
        |j	                  |d         j                         j                  d               t        |j	                  |d               }}}t        |      }t        ||      \  }}t        |||      }t        j                  d	|       |S # t        $ r Y yw xY w)
zTry full-text parse for date elements using a series of regular expressions
    with particular emphasis on English, French, German and TurkishNyear)r   r   r   )day2month2year2r   r	   .rn   zmultilingual text found: %s)LONG_TEXT_PATTERNrt   	lastgrouprw   groupTEXT_MONTHSlowerstripr   r   r   r{   ru   rv   )r   r|   groupsr   r   r   r}   s          rh   regex_parser      s    
 $$V,E ??f$ 	!( 
F1I&'EKKq	288:@@EFGF1I&' U
 D!$S%0
UdE3/
 LL.
;  s   B"C' '	C32C3outputformatmin_datemax_datec           	         t         j                  d|        | dd j                         rd}| dd j                         r3	 t        t	        | dd       t	        | dd       t	        | dd             }n	 t        j                  |       }|6t        ||||      r't         j                  d|       |j                  |      S t        j                  |       }|rw	 t	        |d   dd       t	        |d   dd       t	        |d   dd       }}}t        |||      }t        |d||      r't         j                  d|       |j                  |      S t        j                  |       }|r
	 |j                  dk(  rOt	        |j!                  d            t	        |j!                  d            t	        |j!                  d            }}}nht	        |j!                  d            t	        |j!                  d            t	        |j!                  d            }}}t#        |      }t%        ||      \  }}t        |||      }t        |d||      r't         j                  d|       |j                  |      S t&        j                  |       }|r	 |j                  dk(  r>t        t	        |j!                  d            t	        |j!                  d            d      }n=t        t	        |j!                  d            t	        |j!                  d            d      }t        |d||      r't         j                  d|       |j                  |      S t)        |       }	t+        |	|||      S # t
        $ r t         j                  d| dd        Y w xY w# t
        $ rY t         j                  d|        	 t        | d	      }n0# t        t        t
        f$ r t         j                  d
|        Y nw xY wY dw xY w# t
        $ r t         j                  d|d          Y w xY w# t
        $ r t         j                  d|d          Y w xY w# t
        $ r t         j                  d|d          Y w xY w)z!Try to bypass the slow dateparserzcustom parse test: %sN         z8-digit error: %sznot an ISO date string: %sF)fuzzyzdateutil parsing error: %srp   zparsing result: %sr	   %Y-%m-%dzYYYYMMDD match: %szYYYYMMDD value error: %sr   r   r   r   r   r   r   zregex match: %szregex value error: %szY-M match: %szY-M value error: %s)ru   rv   isdigitr   rw   r{   fromisoformatdateutil_parseOverflowError	TypeErrorr   strftimeYMD_NO_SEP_PATTERNrt   YMD_PATTERNr   r   r   r   
YM_PATTERNr   r   )
r   r   r   r   	candidater|   r   r   r   r}   s
             rh   custom_parser     s    LL(&1 bqz	!A; >$r
OS!%5s6!A;7G	G$226:	  )\HXVLL-y9%%l33 %%f-E	8"58BQ<0#eAhqm2Dc%PQ(STUV-FX%D uc2I Y
XhW19= )),77 v&E	8%'F+,G,-E*+ !e F+,H-.G,- !U
 $D),S%8
U uc2I Y
XhW.	: )),77 f%E	8')$F+,c%++g2F.G	 %G,-s5;;x3H/I1	 Y
XhW_i8 )),77 V$JL8H a  >0&!*=>  G96BG .vU CI%y*= GLL!=vFG	G&  	?LL3U1X>	?6  	<LL0%(;	<&  	:LL.a9	:s    1M" 3N A O0 'CP B
Q ""NNO-+N98O-9*O&#O-%O&&O-,O-0"PP"P?>P?"Q('Q(c                     t         j                  d|        	 t        j                  |       d   }|r|j                  |      S dS # t        t
        f$ r#}d}t         j                  d| |       Y d}~Bd}~ww xY w)zEUse dateutil parser or dateparser module according to system settingszsend to external parser: %sdate_objNzexternal parser error: %s %s)ru   rv   EXTERNAL_PARSERget_date_datar   r{   errorr   )r   r   targetr~   s       rh   external_date_parserr   r  sv    
LL.7B ..v6zB -36??<(<<	 :& B3VSAABs   A A7A22A7)maxsizeextensive_searchc                 H   | syt        |       dt         } | r.dt        t        t        j
                  |             cxk  rdk  sy yt        j                  |       ryt        | |||      }||S |r2t        j                  |       rt        | |      }t        ||||      r|S y)zIUse a series of heuristics and rules to parse a potential date expressionNr      rp   )r   r   summapstrr   DISCARD_PATTERNSrt   r   TEXT_DATE_PATTERNr   r   )r   r   r   r   r   customresultdateparser_results          rh   try_date_exprr     s      v/0F c#ckk6":;ArA B v&  hIL -44V<0F|hx
 %$rj   c                 `    | j                  d      }|t        |j                  d      |      S y)zSkim through image elementsz'.//meta[@property="og:image"][@content]Ncontent)findr   get)ra   rl   elements      rh   
img_searchr     s:    
 iiABGKK	"
 	
 rj   textdate_patternc                     |j                  |       }|rZt        |d   d|j                  |j                        r4t        j                  d||d          t        |d   d|j                        S y)zILook for date expressions using a regular expression on a string of text.r	   r   rp   zregex found: %s %sr   N)rt   r   ry   rz   ru   rv   r   rx   )r   r   rl   r|   s       rh   pattern_searchr     sd     %Ea*w{{7;; 	)<qBE!Hj'..AArj   c                     |j                   rt        nt        }| j                  d      D ]:  }|j                  rd|j                  vrt        |j                  ||      }|8|c S  y)z8Look for JSON time patterns in JSON sections of the treezK.//script[@type="application/ld+json" or @type="application/settings+json"]z"dateN)originalJSON_PUBLISHEDJSON_MODIFIEDxpathr   r   )ra   rl   json_patternelemresults        rh   json_searchr     sg     &-%5%5>=L

U  yyG4994		<AM rj   
htmlstringc                 L   t         j                  |       }|rt        t        d|j	                                     }	 t        |d         dk(  r2t        t        |d         t        |d         t        |d               }nKt        t        |d         t        |d               \  }}t        t        |d               }t        |||      }t        ||j                  |j                  |j                        S y# t        t        f$ r t         j#                  d|d          Y yw xY w)z5Look for author-written dates throughout the web pageNr   r   r	   rn   rp   z!cannot process idiosyncrasies: %s)TEXT_PATTERNSrt   listfilterr   lenr   rw   r   r   r   rx   ry   rz   
IndexErrorr{   ru   rv   )r   rl   r|   partsr   r   r   r   s           rh   idiosyncrasies_searchr     s    
   ,EVD%,,.12	H58}!$Sq]CaM3uQx=Q	,Sq]CaMJ
U#CaM2$T5#6	'7>>GKK   J' 	HLL<eAhG	Hs   B:C8 8(D#"D#)]__doc__loggingrer   	functoolsr   
dateparserr   dateutil.parserr   r   
lxml.etreer   	lxml.htmlr   r   r
   r   utilsr   r   
validatorsr   r   r   r   	getLogger__name__ru   r   FAST_PREPENDSLOW_PREPENDDATE_EXPRESSIONSFREE_TEXT_EXPRESSIONSrd   DAY_REMONTH_REYEAR_REcompiler   r   r   REGEX_MONTHSreplaceIr   rs   r   r   TIMESTAMP_PATTERNMONTHS	enumerater   r   r   r   THREE_COMP_REGEX_ATHREE_COMP_REGEX_BTWO_COMP_REGEXYEAR_PATTERNCOPYRIGHT_PATTERNTHREE_PATTERNTHREE_CATCHTHREE_LOOSE_PATTERNTHREE_LOOSE_CATCHSELECT_YMD_PATTERNSELECT_YMD_YEARYMD_YEARDATESTRINGS_PATTERNDATESTRINGS_CATCHSLASHES_PATTERNSLASHES_YEARYYYYMM_PATTERNYYYYMM_CATCHMMYYYY_PATTERNMMYYYY_YEARSIMPLE_PATTERNri   r   r   rw   tupler   r   r   r   boolr   r   Patternr   r   r   )mnummlistr   s   000rh   <module>r     s    	   & 3  ! 2 ' W W			8	$ 
 $$($)
	$ D B lY67  GH 
 

!  RZZ/ bjjG9$5hZvh W*8*4XZ RZZG9$5hZ @*-gYjB

 BJJ	L> "		-gY 7

 >*7)18 9@b8IDD  rzzS	
'&ST

1'!H:QvhaPRTRVRVWG9AhZq:BDD BJJ	8*AfX%GH 

  $-V1#= D%5BGE4K	 BJJ12 2::

  

Q DD  RZZ1VHGH:WWIQ OP RZZH:-fXVH:EVW  q
''!<= rzzU7)1-.BJJ+G9E'#F  

@Abjj<= bjj!LM BJJJK RZZ PQ "**'%012::G9A&' bjjL  BJJ!G9,DEF "**R rzz.)GHrzzQwi'=>?@AbjjAgYe,-~gYc:;; ; 4Z 	4Z&F FS FU38_ F
 4 :aa"a.6aBJa4ZaH
= 
=C 
=C$J 
= :%$J%% % 	%
 % 	4Z% %P
 	4Z
**S/  	4Z	
 	4Z& 	4ZG
s   >R	