
    %iG                     h    S SK r S SKrS SKrS SKrS SKrS SKrS SKrS SKJ	r	  S SK
Jr   " S S5      rg)    N)Welfo)Foldac                   n    \ rS rSrSS jrS rSS jrSS jrSS jrSS jr	S	 r
S
 rS rSS jrS rSrg)Dexis   c                 t    Xl         X l        0 U l        S U l        S U l        S U l        [        U5      U l        g N)e
do_verbosedata	issuedatedoc_file
issuefufisr   folda)selferimpr   s      !/home/ernad/ernad/python/dexis.py__init__Dexis.__init__   s8    $	5\
     c                     U R                   nUR                  R                  UR                  S   5      nX l        U H  nU R                  U5        M     g)z-> update for all issue filesissuesN)r
   ddates_uniquedirsr   update_issue)r   r
   r   r   s       r   update_all_issuesDexis.update_all_issues    sE    FFSS%%affX&67
$#Ii( $r   Nc                    XR                   :X  a  gXl         U R                  nUb  [        R                  " U5      nU R                  R                  S   S-   U-   S-   nU R
                  c.  UR                  R                  UR                  S   5      nXPl        Ub   U R                  (       a  [        SU-   5        OU R                  (       a  [        S5        XR
                  ;  a   Uc  [        S	U S
3[        R                  S9  gUc  U R
                  U   nO[        R                  " U5      n[        R                  " USS9nX`l        XR                  ;  a  [         R"                  R%                  U5      (       d"  Uc  [        SU-   S-   [        R                  S9  gU R                  (       a  [        SU-   5        [        R&                  " U5      U R                  U'   g)z3run this before working on a file, loads dexis dataTNdexis/.json.gzr   zdexis gets doc_fufi zdexis gets no doc_fufizdexis sees no issue fufi for .filerbmodez
dexis: no z, run dexis.zdexis loads )r   r
   filerget_issuedater   r   r   r   r   printsysstderrgzipopenr   r   ospathisfileload)r   r   doc_fufir
   
dexis_fufir   r   s          r   set_issuedateDexis.set_issuedate(   s   &"FF ++H5IVV[[)C/);jH
??"))!&&*:;J(O,x78./OO+0@1)A>zz#y1H++H5I99XD1 II%77>>*--#,3nD"zz+nz12#(::j#9DIIi r   c                     U R                   n[        U R                  U   R                  5       5      n[        R
                  " U5      nU R                  XAS9nU$ )N)with_ns)r   listr   keysrandomchoicefetch)r   r9   r   papidspapidoctetss         r   r<   Dexis.randomS   sJ    NN	dii	*//12f%E3r   c                    U R                   nX0R                  U   ;  aE  U R                  R                  S   S-   U-   S-   n[        R
                  " U5      U R                  U'   U R                  U   nUR                  5       nX;  a"  [        SU-   S-   U-   [        R                  S9  gXQ   nUS   nUS	   nU R                  R                  U5        U R                  R                  U5      n	U(       d  U	$ U R                  R                  S
   R                  S5      n
SU R                  R                  S   R                  S5      -   S-   nU	SS S:X  a  SU-   S-   U
-   S-   U	SS -   n	U	$ [        S5      e)z-get the doc papid, assumed it is in issuedater    r!   r"   z
dexis: no'z' in r$   Nslamfzutf-8s   xmlns:xsi="xsis   " r   
   s
   <text ref=s   <text s   xmlns="s   " ref="   zdexis has a namespace issue)r   r   r
   r   r)   r3   stripr+   r,   r-   r   seekreadnsencode	Exception)r   r@   r9   r   r5   
dexis_data
papid_datastartlengthrA   amf_nsxsi_nss               r   r>   Dexis.fetchZ   si   NN	IIi00W-3i?*LJ#(::j#9DIIi YYy)
",&09<zz#&
3C5!##F+M5!((1$&&))E"2"9"9'"BBUJ!B<=('*4v=#BC[)FM9::r   c                    Uc  U R                   U   nU R                  R                  S   S-   U-   S-   n[        R                  R                  U5      (       aA  [        R                  R                  U5      [        R                  R                  U5      :  a  g[        R                  " USS9n[        U R                  5      U l
        UR                  n[        R                  " USS9U l        [        R                  R                  R!                  5       U l        SU l        U R$                  U R                  l        U R(                  U R                  l        U R,                  U R                  l        0 U l        X@l        U R                  R5                  U5        [6        R8                  " U R0                  U5        U R0                  $ )z/updates an issue by issuedate, with mtime checkr    r!   r"   Fr&   r'   )r   r
   r   r0   r1   r2   getmtimer.   r/   r   welfoname
check_filexmlparsersexpatParserCreatedocendstart_elementStartElementHandlerend_elementEndElementHandlerend_doc_checkCharacterDataHandlerr   xml_file	ParseFiler)   dump)r   r   r4   out_fufirg   xml_fufis         r   r   Dexis.update_issuex   s>   y1H66;;w'#-	9JF77>>(##ww)BGG,<,<X,FF99XD1466]
==))H48[[&&335
)-););

&'+'7'7

$*.*<*<

'	 

X&

499h'yyr   c                     U R                  U5        US:w  a  gSU;  a  SU l        gSU l        US   U l        U R                  R                  U l        g)z"expat handler for start of elementtextNrefTF)re   skip_next_textro   r^   CurrentByteIndexrR   )r   rZ   attrss      r   ra   Dexis.start_element   sQ    4 6>"&D#<ZZ00
r   c                 t    U R                  U5        US:w  a  gU R                  (       a  SU l        gSU l        g)z expat handler for end of elementrn   NFT)re   rp   r`   r   rZ   s     r   rc   Dexis.end_element   s5    4 6>"'Dr   c                    U R                   (       d  gSU l         U R                  R                  U l        U R                  U R                  -
  U l        U R                  R                  U R                  U R                  U R
                  5        0 U R                  U R                  '   U R                  U R                  U R                     S'   U R
                  U R                  U R                     S'   g)z?expat handler to check if we have reached the end of a documentNFrD   rE   )r`   r^   rq   endrR   rS   rY   checkr[   r   ro   ru   s     r   re   Dexis.end_doc_check   s    {{::..hh+

$**dkkB 		$((#'::		$((C #';;		$((C r   c                 x   SSK nUR                  SS9nU R                  R                  S   nU R                  R                  USS9n0 nU Hl  nUb  Xq:w  a  M  [        R                  " XW   5      nU HA  n	XvS'   X   S	   US	'   X   S
   US
'   UR                  U	[        R                  " U5      5        MC     Mn     g)z,OBSOLETE -> feed issues into redis, OBSOLETEr   NTdecode_responsesr    r"   )extirD   rE   )redisRedisr
   r   r   dafusr)   r3   setjsondumps)
r   only_issuedater   confudir   inpr   r   r@   s
             r   ingest_via_rediesDexis.ingest_via_redies   s     	kk4k0vv{{7#

  : 6I)i.I::e./D$C;s+C;s+Ctzz#/	 	 r   c                     U R                   c  SSK nUR                  SS9U l         U R                   R                  U5      nU$ )OBSOLETENr   Tr|   )r   r   get)r   r@   r   outs       r   lookup_via_redisDexis.lookup_via_redis   s8    ::d;DJjjnnU#
r   )r[   r   r   r   r`   r
   rx   r^   r   r   r   rS   r   ro   rp   rR   rY   rg   )Fr	   )T)__name__
__module____qualname____firstlineno__r   r   r6   r<   r>   r   ra   rc   re   r   r   __static_attributes__ r   r   r   r      s:     ))V;<41/0(r   r   )r0   r.   r   r<   r,   xml.parsers.expatr\   r)   rY   r   r   r   r   r   r   r   <module>r      s,    	    
    B Br   