
    U|h1                     x    S SK r S SKrS SKrS SKrS SKrS SKrS SKJr  S SKJ	r	  S SK
Jr  S SKJr   " S S5      rg)    N)Dexis)Ishus)Trans)Saific                       \ rS rSrSS jrS rSS jrS rS rS r	S r
S	 rS
 rS rS rS rSS jrS rS rS rS rS rS rS rSrg)Clarf   c                    Xl         [        U R                   5      U l        [        U R                   5      U l        [        U R                   5      U l        [        U R                   5      U l        U R                   R                  S   U l
        [        US5      (       a  U R                   R                  U l        OX l        S U l        S U l        g )Nclarf
do_verbose)er   dexisr   ishusr   saifir   transdirsfolderhasattrr   dataout_fufi)selferimpr   s      !/home/ernad/ernad/python/clarf.py__init__Clarf.__init__   s    466]
466]
466]
466]
ffkk'*5,''"ff//DO(O	    c                    U R                   R                  5       R                  5       nU H\  nU R                  R                  S   S-   U-   S-   n[
        R                  R                  U5      (       d  MK  U R                  X25        M^     g)<- externalr   /.json.gzN)	r   dafuskeysr   r   ospathisfile
audit_fufi)r   
issuedates	issuedatefufis       r   	audit_allClarf.audit_all!   so    ZZ%%',,.
#I66;;w'#-	9JFD77>>$''OOD,	 $ 	r   c                    U R                   R                  nU R                   R                  nU R                  X#5      nUc	  SU SU S3$ U R                   R                  U   R
                  U   nU R                   R                  R                  U5      nX6;  a  SU SU SU S3$ Xc   nU R                   R                  R                  U5      U   nUn	Xx:w  a  SU SU SU S3$ [        R                  " U	5      n
U R                  R                  SU
5      n[        R                  " U5      nU H  nXM   X   :X  a  M    g	   g
)r   zI have to data for  .zI don't see  in  at zI see several weights_jsonFT)r   given_repcodegiven_issuedateread_archivereportr   dlastsfirstsfiler	parse_laxr   doc_to_stringjsonloads)r   stagerepcoder(   
clarf_data	stage_dirr7   last_stage_fufifirst_stage_fufi
stage_fufi	stage_docr1   weightspapids                 r   
head_checkClarf.head_check,   s=   &&&&FF**	&&w:
(	9+Q??FFMM'*//6	y)!!)DtG9AFF*6688??95i@%
.#I;d5'gYaHHOOJ/	zz//	J**\*E GN2  r   c                 l   [         R                  " U5      nU R                  R                  U5        [	        U R                  R
                  U   R                  5       5      nU HM  n[	        X5   5      nXd:X  a  M  [        US-   [        U5      -   SS9  [        SU-   S-   [        U5      -   5        MO     g )Nz has  )endz, )	r9   loadr   set_issuedatelenr   r"   printstr)r   r)   r(   r   issue_lengthr?   len_repcodes          r   r&   Clarf.audit_fufiG   s    zz$

  +4::??95::<=Gdm,K*)g%L(99rB$.7*S-==> r   c                 l    U R                  5         U R                   H  nU R                  U5        M     g)z<- main externalN	get_indatr'   do_issuedater   r(   s     r   updateClarf.updateR   (    Ii( )r   c                 l    U R                  5         U R                   H  nU R                  U5        M     g)z<- called by clarf -lNrV   rY   s     r   length_checkClarf.length_checkX   r\   r   c           
      &   [         R                  R                  U5      (       a%  Un[         R                  R                  U5      SS nOUnU R	                  U5      nX0R
                  R                  ;  a  U R
                  R                  U5        U R
                  R                  U   R                  5       n[         R                  R                  U5      (       d  [        SU 35        g[        R                  " U5      nSn0 nU H  n[        U5        U Hr  n	XU   ;  d  M  X;   a  M  [        SU	 SU SU 3[        R                  S	9  [        U5      n
[        XX   5      n[        S
U SU
 S3[        R                  S	9  SXx'   SnMt     M     U$ )-<- check whether the papid are those in dexisr   
   zclarf does not have FTzclarf misses r/   r0   filez
clarf has z entries of z papids   )r#   r$   r%   basenameget_out_fufir   r   rN   r"   rP   r9   rM   sysstderrrO   )r   ingest
clarf_fufir(   papidsr   retmissedr?   rG   
len_papids	len_clarfs               r   
check_keysClarf.check_keys^   s_    77>>&!!J((4Qr:II**95JJJOO+JJ$$Y/+002ww~~j))(56

:&G'Ng.( M%WIT)M"zz+!$VJ #EN 3IJykj\Q"zz+&'FOC    
r   c                 `    U R                   R                   H  nU R                  U5        M     g)ra   N)r   r'   rq   rY   s     r   check_keys_historicClarf.check_keys_historic   s"    **IOOI& +r   c                    XR                   R                  ;  a  [        S5        gU R                   R                  U   R                  S   nUS-   U-   S-   n[
        R                  R                  U5      (       d  [        SU 3[        R                  S9  gU R                  U5      nX R                  R                  ;  a  U R                  R                  U5        [        U R                  R                  U   R                  5       5      n[        U5      nXv:w  a*  SU S	U S
U S3nUS-  n[        U[        R                  S9  gU R!                  U5      n	0 U l        [
        R                  R                  U	5      (       a  ["        R$                  " U	5      U l        XR                  ;   aT  [        U5      n
[        U R                  U   5      nX:  a,  SU SU 3nUSU SU
 3-  n[        U[        R                  S9  gXPR                  U'   ["        R&                  " U R                  U	5        [        SU SU	 35        g)z)to call with release for special releasesz$clarf: I don't have repcode any moreFmoclar   z.sidezclarf: no so such file rc   clarf: r/   z but r-   zin issuezclarf will not shorten data on z from z to zclarf adds T)r   reportsrP   r5   r   r#   r$   r%   rh   ri   readr   r   rN   rO   r"   rg   r9   rM   dump)r   r?   r(   	mocla_dir	side_fufi	side_data
issue_size	side_sizemsgr   new_lenold_lens               r   singleClarf.single   s   &&..(89FFMM'*//8	Oi/'9	ww~~i((+I;7cjjIIIi(	JJOO+JJ$$Y/388:;
	N	"I;d9+U:,aHC:C#CJJ'$$Y/	77>>(##

8,DIii)nG$))G,-G 7y)MyWI66c

+&		'

499h'G9D
34r   c                 h    U R                   R                  S   S-   U-   S-   U l        U R                  $ )Nr   r   r    )r   r   r   rY   s     r   rg   Clarf.get_out_fufi   s/    G,s2Y>K}}r   c                 V   U R                  U5      U l        U R                  U5      n[        U5      S:X  a  gU R                  U    H4  nU R                  U   U   nU R                  U5      nXPR                  U'   M6     [        R                  " U R                  U R                  5        g )Nr   F)	rg   r   get_reports_that_need_renewalrO   r'   rz   r   r9   r{   )r   r(   reports_to_dor?   r)   r   s         r   rX   Clarf.do_issuedate   s    )))4::9E}"y1G??9-g6D99T?D!%IIg 2 	

499dmm,r   c                     [        US5      n0 nUR                  5        H&  nUR                  5       n[        US   5      X5S   '   M(     U$ )Nrr   re   )open	readlinessplitfloat)r   r)   rd   r   linepartss         r   rz   
Clarf.read   sH    D#NN$DJJLE"58_DqN % r   c                 b   U R                   n0 nUR                  U   R                  S   nUS-   S-   U-   n[        R                  " U5      n[	        U5      nUS:X  a'  UR                  5       (       d  [        SU SU 35        gU H  n	UR                  R                  U	5      n
U R                   R                  R                  S   R                  U
5      (       d'  [        S	U
 S
35        [        R                  " U	5        M|  X;   a   [        SU SU
 3[        R                  S9  M  XU
'   M     U$ )z!what should be 'model' or 'train'rw   r   z*.r   zmocla sees no z
 file for Nis_daterx   z is not a date.zclarf has several files at  and rc   )r   r5   r   globrO   
is_testingrP   fr(   Rrsmatchr#   removerh   ri   )r   r?   whatr   r!   r|   glob_stringfufiscount_fufisr)   dates              r   	get_dafusClarf.get_dafus   s   FFHHW%**73	#o,t3		+&%j!<<>>tfJwi@AD33==&D6688;;y)//55v_56 		$}3G9E$H::'$K  r   c                     0 U l         U R                  R                   HT  nU R                  U5      nUc  M  U H5  nX0R                   ;  a  0 U R                   U'   X#   U R                   U   U'   M7     MV     g N)r'   r   liver   )r   r?   r!   r   s       r   rW   Clarf.get_indat   sh    vv{{GNN7+E}.,.DOOD)16%g.  #r   c                 D   / n[         R                  R                  U R                  5      (       d  0 U l        U R
                  U   $ [        R                  " U R                  5      U l        [         R                  R                  U R                  5      nU R
                  U    HJ  n[         R                  R                  U R
                  U   U   5      nXS:  d  M9  UR                  U5        ML     U R                  (       a  [        S[        U5      -   S-   U-   5        U$ )Nzclarf needs r0   )r#   r$   r%   r   r   r'   r9   rM   getmtimeappendr   rP   rQ   )r   r(   ry   	file_timer?   	data_times         r   r   #Clarf.get_reports_that_need_renewal   s    ww~~dmm,,DI??9--JJt}}-	GG$$T]]3	y1G (()CG)LMI$w' 2 ??.3w</&89DEr   c                 R   U R                   S-   n/ n[        R                  " U5       Hq  n[        R                  R	                  U5      nUR                  S5      (       a  M:  UR                  S5      (       d  [        SU 35        M`  UR                  U5        Ms     [        USS9nU$ )Nz/*grundr    zclarf: foreign file T)reverse)
r   r   r#   r$   rf   
startswithendswithrP   r   sorted)r   r   r   r)   banas        r   
list_fufisClarf.list_fufis  s    kkD(IIk*D77##D)Dw''==,,,TF34LL + ud+r   c                     U R                   S-   U-   S-   n[        R                  R                  U5      (       d3  [	        SU S3[
        R                  S9  [        R                  " U0 5        [        R                  " U5      nUc  0 nX;  a}  U R                  X5      nU R                  c  [	        S5        g XR                  ;  a3  [	        SU SU 3[
        R                  S9  [
        R                  " 5         U R                  U   nU$ XA   nU$ )Nr   r    clarf does not see z creates emptyrc   zclarf: I have no datar/   )r   r#   r$   r%   rP   rh   ri   r9   r{   rM   r   r   exit)r   r?   r(   r)   r   outrepdatas          r   r4   Clarf.read_archive  s    {{S 9,z9ww~~d##'v^<3::NJJtR zz$<D++g1Cyy -.ii'+G9D?::'
ii(G  mGr   c                 <    U R                  X5      n[        U5        g r   )rz   rP   )r   r?   r(   r   s       r   show
Clarf.show+  s    ))G/gr   c                    U R                  X5      nSU SU 3nUc  SU-   nU$ U R                  R                  X5      nUc  SU-   nU$ U HJ  nXs;  a	  U S3nUs  $ X7   Xg   :X  a  M  US-   [        X7   5      -   nUS[        Xg   5      -   -  nSU-   nUs  $    U HO  nXv;  a  SU S	3U-   nUs  $ X7   Xg   :X  a  M!  [	        [        X7   5      5      S-   [        Xg   5      -     S
U-   nUs  $    g)Nz for r   zclarf has no archive z clarf: I don't see the side fufiz" in side file is not in clarf fufir-   zclarf: data is inconsistent 1 r   z in the side filezclarf: data is inconsistent 2T)r4   r   rz   rQ   rP   )	r   r?   r(   in_clarfrL   mesgin_saifirG   r   s	            r   is_clarf_equal_to_saifiClarf.is_clarf_equal_to_saifi0  s9   $$W8gYeI;/*S0DK::??765;DKE$ BC(/1#+HO 44CC#ho...C3c9DK  E$+E72CDsJ
(/1#ho&'#-HO0DD2S8DK  r   c                     [         R                  " 5       nU[         R                  " 5       :X  a  gU R                  U5      n[        R
                  R                  U5      (       a  g[        R                  " 0 U5        g)z'<- clarf, create issue for brown periodFT)	datertonighttodayrg   r#   r$   r%   r9   r{   )r   r   rk   s      r   create_in_brownClarf.create_in_brownO  sU    --/ekkm#&&w/
77>>*%%

2z"r   )
r   r   r   r   r   r   r'   r   r   r   N)F)source)side)__name__
__module____qualname____firstlineno__r   r*   rH   r&   rZ   r^   rq   rt   r   rg   rX   rz   r   rW   r   r   r4   r   r   r   __static_attributes__ r   r   r   r      sh     	6	?))"H'
%N	-8=$&.
>	r   r   )r   r#   rh   r   r9   r<   r   r   r   r   r   r   r   r   r   r   r   r   <module>r      s/     	 
       I Ir   