
    AHj5              	          d Z ddlZddlZddlZddlmZ ddlmZ ddlmZ ddl	m
Z
 ddlmZmZ  ej                  e      Zh d	Zh d
Zh dZ ej(                  dej*                        Z ej(                  dej*                        Z ej(                  d      Z ej(                  dej*                        Z ej(                  dej*                        Z ej(                  dej*                        Z ej(                  dej8                        Z ej(                  d      Z ej(                  dej8                        Z ej(                  d      Z dZ! ej(                  dej*                        Z" ej(                  dej*                        Z#de"fde#fgZ$ ej(                  dej8                        Z% ej(                  d      Z& ej(                  d      Z' ej(                  d      Z( ej(                  d       Z) ej(                  d!      Z* ej(                  d"ej8                        Z+ ej(                  d#      Z, ej(                  d$ej8                        Z- ej(                  d%ej\                        Z/d9d&e
d'ed(e0dz  d)e1fd*Z2d+ed&e
d)e
fd,Z3d-e4e   e5e0e0f   z  d&e
d)e
fd.Z6d/e0d0ee0   d)e0dz  fd1Z7d2e0d&e
d)e
fd3Z8d4e0d)e0fd5Z9d6e0dz  d7e0d)e0dz  fd8Z:y):z
Functions needed to scrape metadata from JSON-LD format.
For reference, here is the list of all JSON-LD types: https://schema.org/docs/full.html
    N)unescape)Pattern)Any   )Document)HTML_STRIP_TAGStrim>
   articleblogpostingnewsarticleliveblogpostingscholarlyarticleopinionnewsarticlesocialmediapostingreportagenewsarticlebackgroundnewsarticlemedicalscholarlyarticle>!   blogqapagereportr
   faqpagewebpagewebsiteitempage	aboutpage
jobpostingr   contactpager   profilepagetecharticlecheckoutpagecollectionpagemedicalwebpager   satiricalarticler   realestatelistingreviewnewsarticlesearchresultspager   r   analysisnewsarticleaskpublicnewsarticler   r   discussionforumpostingr   advertisercontentarticle>   r   r   organizationnewsmediaorganizationzM"author":[^}[]+?"name?\\?": ?\\?"([^"\\]+)|"author"[^}[]+?"names?".+?"([^"]+)z$"[Pp]erson"[^}]+?"names?".+?"([^"]+)z`,?(?:"\w+":?[:|,\[])?{?"@type":"(?:[Ii]mageObject|[Oo]rganization|[Ww]eb[Pp]age)",[^}[]+}[\]|}]?z,"publisher":[^}]+?"name?\\?": ?\\?"([^"\\]+)z"@type"\s*:\s*"([^"]*)"z"articleSection": ?"([^"\\]+)z"author":|"person":)flagsz<[^>]+>z^https?://schema\.orgz\\u([0-9a-fA-F]{4}))	givenNameadditionalName
familyNamez*"@type":"[Aa]rticle", ?"name": ?"([^"\\]+)z"headline": ?"([^"\\]+)z"name"z
"headline"uB   ^([a-zäöüß]+(ed|t))? ?(written by|words by|words|by|von|from) z\d.+?$z@[\w]+z[._+]u$   ["‘({\[’\'][^"]+?[‘’"\')\]}]u   [^\w]+$|[:()?*$#!%/<>{}~¿]u;   \b\s+(am|on|for|at|in|to|from|of|via|with|—|-|–)\s+(.*)z3\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\bz"/|;|,|\||&|(?:^|\W)[u|a]nd(?:$|\W)u>   [✀-➾😀-🙏☀-⛿🌀-🗿🤀-🧿🩰-🫿🚀-🛿]+metadata	candidatecontent_typereturnc                     |r|t        |t              rl| j                  r&t        | j                        t        |      k  r|dk7  ry| j                  r-| j                  j	                  d      r|j	                  d      syy)z6Determine if the candidate should be used as sitename.r   ThttpF)
isinstancestrsitenamelen
startswith)r1   r2   r3   s      Q/root/tools/cai/cai_env/lib/python3.12/site-packages/trafilatura/json_metadata.pyis_plausible_sitenamer=   m   si    Z	3/  S):):%;c)n%LQ]ajQj!2!2!=!=f!EiNbNbciNj    parentc                 d   t        d|       D ]  }|j                  d      }t        |t              rd|v r
|d   |_        d|vs|d   s=t        |d   t
              r|d   d   n|d   }|j                         }|t        v r|j                  st        |      |_        |t        v rM|j                  d      xs$ |j                  d      xs |j                  d      }t        |||      s||_        |dk(  rWt        |j                  d      t              s|d   j                  d	      r"t        |j                  |d         |_        B|t         v sLd
|v r|d
   }t        |t              r	 t#        j$                  |      }t        |t
              s|g}|D ]  t        t              rdidvs	d   dk(  s$d}dv r\j                  d      }t        |t
              r!dj)                  |      j+                  d      }n@t        |t              r0d|v r,|d   }n&dv r"dv rdj)                  fdt,        D              }t        |t              st        |j                  |      |_         |j.                  s@d|v r<t        |d   t              r|d   g|_        nt        t        d|d               |_        |j0                  rd|v r|dk(  r|d   |_        d|v s|d   |_         |S # t"        j&                  $ r t        |j                  |      |_        Y w xY w)z3Find and extract selected metadata from JSON parts.N	publishername@typer   	legalNamealternateNamepersonr6   authorPerson; r.   r0    c              3   2   K   | ]  }|v s|     y wN ).0xrG   s     r<   	<genexpr>z!process_parent.<locals>.<genexpr>   s     2bVW[aVa6!92bs   	
articleSectionr
   headline)filtergetr7   dictr9   listlowerJSON_OGTYPE_SCHEMApagetypenormalize_jsonJSON_PUBLISHER_SCHEMAr=   r8   r;   normalize_authorsrG   JSON_ARTICLE_SCHEMAjsonloadsJSONDecodeErrorjoinstripAUTHOR_ATTRS
categoriestitle)	r?   r1   contentrA   r3   r2   list_authorsauthor_namerG   s	           @r<   process_parentri   w   s   $' G9KK,	i&6Y+> )& 1H'!)9 /99I4.Pww'*V]^eVf#))+ --h6G6G .| <H00F+gw{{;/Gg7;;WfKgI$Xy,G$-!X%'++f-s3GFO<V<VW]<^"3HOOWV_"U007"&x0lC0['+zz,'?
 ",5$0>L* ^F!&#."(&!1f,w80K&*!V+*0**V*<K)+t<.2ii.D.J.J4.P!+K!>6[CX.9&.A(F2|v7M*-((2bl2b*bK%k37.?Q\.]HO!^& &&+;w+Fg&67=+23C+D*EH'*.vdGDT<U/V*WH' >>W$)B%,V_HN7*%,Z%8HNOG9P OM  // [*;HOO\*Z[s   5K==.L/.L/schemac                    t        | t              r| g} | D ]  }|j                  d      }|st        |t              s(t        j                  |      s>d|v rt        |d   t              r|d   n|d   g}n]d|v rKt        |d   t              r8d|d   j                         v r#d|v rt        |d   t              r|d   n|d   g}nt        | |      c S t        ||      } |S )z,Parse and extract metadata from JSON-LD dataz@contextz@graphrC   r   liveBlogUpdate)	r7   rU   rT   r8   JSON_SCHEMA_ORGmatchrV   rW   ri   )rj   r1   r?   contexts       r<   extract_jsonrp      s    &$ 8**Z(z'3/O4I4I'4R6!-7x8H$-O)V\]eVfUg6!vg4%)>)>)@@$.5?GW@XZ^5_ 01flm}f~e%fh77%fh7H!8$ Or>   elemtextregular_expressionc                     d}|j                  |       }|r_t        t        d|j                               d      }|rd|vrn4t	        ||      }|j                  d| d      } |j                  |       }|r_|xs dS )z.Crudely extract author names from JSON-LD dataNrJ    r   )count)searchnextrS   groupsr\   sub)rq   rr   authorsmymatchrB   s        r<   extract_json_authorr|      s    G ''1G
F4!12D9s$#GT2%))#xq)A$++H5  ?dr>   elemc                    t         j                  d|       }t        |t              xs t        |t              }|r||_        d| v rBt        j                  |       }|r+t        |d   j                               }|t        v r||_        d| v r?t        j                  |       }|r(d|d   vr!t        |d         }t        ||      r||_        d| v r+t        j                  |       }|rt        |d         g|_        t"        D ]@  \  }}|| v s|j$                  r|j                  |       }|s,t        |d         |_         |S  |S )z*Crudely extract metadata from JSON-LD datart   rC   r   z"publisher",z"articleSection")JSON_AUTHOR_REMOVEry   r|   JSON_AUTHOR_1JSON_AUTHOR_2rG   	JSON_TYPErv   rZ   rW   rX   rY   JSON_PUBLISHERr=   r9   JSON_CATEGORYrd   JSON_SEQre   )r}   r1   element_text_authorrG   r{   r2   keyregexs           r<   extract_json_parse_errorr      sL    -00T: !4mDH[\oq~HF  $""4(&wqz'7'7'9:I..$-!  ''-s'!*,&wqz2I$Xy9$-! T!&&t,#1'!*#=">H  
U$;x~~ll4(G!/
!;O Or>   stringc                     d| v rl| j                  dd      j                  dd      j                  dd      } t        j                  d |       } dj                  d | D              } t	        |       } t        t        j                  d|             S )z-Normalize unicode strings and trim the output\z\nrt   z\rz\tc                 2    t        t        | d   d            S )Nr      )chrint)rn   s    r<   <lambda>z normalize_json.<locals>.<lambda>  s    Ca"<M8N r>   c              3   Z   K   | ]#  }t        |      d k  st        |      dkD  s | % yw)i   i  N)ord)rN   cs     r<   rP   z!normalize_json.<locals>.<genexpr>  s$     Qqc!fvoQ&Qs   !++)replaceJSON_UNICODE_REPLACEry   ra   r   r	   JSON_REMOVE_HTML)r   s    r<   rZ   rZ     s~    v~r*225"=EEeRP%))*NPVWQFQQ&! $$R011r>   current_authorsauthor_stringc                    g }|j                         j                  d      st        j                  |      r| S | | j	                  d      }d|v r 	 |j                         j                  d      }d|v sd|v rt        |      }t        j                  d|      }t        j	                  |      D ]  }t        |      }t        j                  d|      }t         j                  d|      }t        t"        j                  d	|            }t$        j                  d|      }t&        j                  d|      }t(        j                  d|      }t*        j                  d|      }t,        j                  d|      }|rt/        |      d
k\  r	d	|vrd|vr|d   j1                         s|j3                         }||vs
|j5                  |        |D cg c]  t7        fd|D              r }}t/        |      dk(  r| S dj9                  |      j;                  d      S # t        $ r t        j                  d|       Y w xY wc c}w )z3Normalize author info to focus on author names onlyr6   rI   z\uunicode_escapez$invalid unicode escape in author: %sz&#z&amp;rt   rJ   2   -r   c              3   4   K   | ]  }|k7  xr |v   y wrL   rM   )rN   mns     r<   rP   z$normalize_authors.<locals>.<genexpr>K  s!     4\1Q!V5FQ5F4\s   )rW   r;   AUTHOR_EMAILrn   splitencodedecodeUnicodeDecodeErrorLOGGERdebugr   r   ry   AUTHOR_SPLITr	   AUTHOR_EMOJI_REMOVEAUTHOR_TWITTERAUTHOR_REPLACE_JOINAUTHOR_REMOVE_NICKNAMEAUTHOR_REMOVE_SPECIALAUTHOR_PREFIXAUTHOR_REMOVE_NUMBERSAUTHOR_REMOVE_PREPOSITIONr:   isupperre   appendanyra   rb   )r   r   new_authorsrG   r   s       `r<   r\   r\      s"   K''/<3E3Em3T"%++D1	P)00299:JKM }= 8 /#''M:M$$]3 'f$((V4##B/)--c6:;'++B7&**2v6""2v.&**2v6*..r6: #f++60AcQWFWay  "\\^F$v&-'0 *]4\P[4\1\1]K]
;199[!''--G " 	PLL?O	P@ ^s   H" I,I"IIrL   );__doc__r^   loggingrehtmlr   r   typingr   settingsr   utilsr   r	   	getLogger__name__r   r]   rX   r[   compileDOTALLr   r   r   r   r   r   
IGNORECASE
JSON_MATCHr   rm   r   rc   	JSON_NAMEJSON_HEADLINEr   r   r   r   r   r   r   r   r   r   UNICODEr   r8   boolr=   ri   rV   rU   rp   r|   r   rZ   r\   rM   r>   r<   <module>r      s+  
   	     (			8	$ " F X 

kmomvmvw

BBIINRZZg  KRYYWBJJ1299=	

;RYYGRZZ.bmmD
2::j) "**5R]]K!rzz"89 <BJJDbiiP	

5ryyAy!L-#@A

`hjhuhuv"

9- I& bjj* #$KL "

#AB &BJJ'emomzmz{ rzzPQrzz?r}}U bjj	 ** H  CRVJ bf J3 J( Jx JZcT#s(^3 x H 4# 73< CRVJ &3 &( &x &R23 23 2..sTz ..# ..#PT* ..r>   