
    j܎g1                     x    d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZ d dlmZ  G d d      Zy)    N)Dexis)Ishus)Trans)Saific                       e Zd ZddZd ZddZd Zd Zd Zd Z	d Z
d	 Zd
 Zd Zd ZddZd Zd Zd Zd Zd Zd Zd Zy)Clarfc                    || _         t        | j                         | _        t        | j                         | _        t        | j                         | _        t        | j                         | _        | j                   j                  d   | _
        t        |d      r| j                   j                  | _        n|| _        d | _        d | _        y )Nclarf
do_verbose)er   dexisr   ishusr   saifir   transdirsfolderhasattrr   dataout_fufi)selferimpr   s      !/home/ernad/ernad/python/clarf.py__init__zClarf.__init__   s    466]
466]
466]
466]
ffkk'*5,'"ff//DO(DO	    c                 
   | j                   j                         j                         }|D ]V  }| j                  j                  d   dz   |z   dz   }t
        j                  j                  |      sE| j                  ||       X y)<- externalr
   /.json.gzN)	r   dafuskeysr   r   ospathisfile
audit_fufi)r   
issuedates	issuedatefufis       r   	audit_allzClarf.audit_all!   sn    ZZ%%',,.
#I66;;w'#-	9JFD77>>$'OOD),	 $ 	r   c                    | j                   j                  }| j                   j                  }| j                  ||      }|	d| d| dS | j                   j                  |   j
                  |   }| j                   j                  j                  |      }||vrd| d| d| dS ||   }| j                   j                  j                  |      |   }|}	||k7  rd| d| d| dS t        j                  |	      }
| j                  j                  d|
      }t        j                  |      }|D ]  }||   ||   k(  r y	 y
)r   zI have to data for  .zI don't see  in  at zI see several weights_jsonFT)r   given_repcodegiven_issuedateread_archivereportr   dlastsfirstsfiler	parse_laxr   doc_to_stringjsonloads)r   stagerepcoder&   
clarf_data	stage_dirr4   last_stage_fufifirst_stage_fufi
stage_fufi	stage_docr.   weightspapids                 r   
head_checkzClarf.head_check,   sJ   &&&&FF**	&&w	:
(	9+Q??FFMM'*//6	y)E!!)DtG9AFF	*6688??95i@%
..#I;d5'gYaHHOOJ/	zz//	J**\*E% GEN2  r   c                 l   t        j                  |      }| j                  j                  |       t	        | j                  j
                  |   j                               }|D ]O  }t	        ||         }||k(  rt        |dz   t        |      z   d       t        d|z   dz   t        |      z          Q y )Nz has  )endz, )	r6   loadr   set_issuedatelenr   r    printstr)r   r'   r&   r   issue_lengthr<   len_repcodes          r   r$   zClarf.audit_fufiG   s    zz$

  +4::??95::<=Gd7m,Kl*)g%L(99rB$.7*S-==> r   c                 h    | j                          | j                  D ]  }| j                  |        y)z<- main externalN	get_indatr%   do_issuedater   r&   s     r   updatezClarf.updateR   (    Ii( )r   c                 h    | j                          | j                  D ]  }| j                  |        y)z<- called by clarf -lNrQ   rT   s     r   length_checkzClarf.length_checkX   rV   r   c           
         t         j                  j                  |      r%|}t         j                  j                  |      dd }n|}| j	                  |      }|| j
                  j                  vr| j
                  j                  |       | j
                  j                  |   j                         }t         j                  j                  |      st        d|        yt        j                  |      }d}i }|D ]  }t        |       |D ]u  }	|	||   vs||v rt        d|	 d| d| t        j                  	       t        |      }
t        ||         }t        d
| d|
 dt        j                  	       d||<   d}w  |S )-<- check whether the papid are those in dexisr   
   zclarf does not have FTzclarf misses r,   r-   filez
clarf has z entries of z papids   )r!   r"   r#   basenameget_out_fufir   r   rJ   r    rL   r6   rI   sysstderrrK   )r   ingest
clarf_fufir&   papidsr
   retmissedr<   rD   
len_papids	len_clarfs               r   
check_keyszClarf.check_keys^   sc    77>>&!J((4Qr:II**95JDJJOO+JJ$$Y/+002ww~~j)(56

:&G'Ng.&( M%WIT)M"zz+!$VJ #E'N 3IJykj\Q"zz+&'F7OC    
r   c                 \    | j                   j                  D ]  }| j                  |        y)rZ   N)r   r%   rj   rT   s     r   check_keys_historiczClarf.check_keys_historic   s"    **IOOI& +r   c                 r   || j                   j                  vrt        d       y| j                   j                  |   j                  d   }|dz   |z   dz   }t
        j                  j                  |      st        dt        j                         y| j                  |      }|| j                  j                  vr| j                  j                  |       t        | j                  j                  |   j                               }t        |      }||k7  r-d| d	| d
| d}|dz  }t        |t        j                         y| j!                  |      }	i | _        t
        j                  j                  |	      rt#        j$                  |	      | _        || j                  v rWt        |      }
t        | j                  |         }||
kD  r/d| d| }|d| d|
 z  }t        |t        j                         y|| j                  |<   t#        j&                  | j                  |	       t        d| d|	        y)z)to call with release for special releasesz$clarf: I don't have repcode any moreFmoclar   z.sidezclarf: so such file {fufi}r\   clarf: r,   z but r*   zin issuezclarf will not shorten data on z from z to zclarf adds T)r   reportsrL   r2   r   r!   r"   r#   ra   rb   readr   r   rJ   rK   r    r`   r6   rI   dump)r   r<   r&   	mocla_dir	side_fufi	side_data
issue_size	side_sizemsgr   new_lenold_lens               r   singlezClarf.single   s   $&&..(89FFMM'*//8	Oi/'9	ww~~i(.SZZ@IIi(	DJJOO+JJ$$Y/388:;
	N	
"I;d9+U:,aHC:C#CJJ'$$Y/	77>>(#

8,DIdii)nG$))G,-G 7y)MyWI66c

+&		'

499h'G9D
34r   c                 h    | j                   j                  d   dz   |z   dz   | _        | j                  S )Nr
   r   r   )r   r   r   rT   s     r   r`   zClarf.get_out_fufi   s/    G,s2Y>K}}r   c                 P   | j                  |      | _        | j                  |      }t        |      dk(  ry| j                  |   D ]4  }| j                  |   |   }| j                  |      }|| j                  |<   6 t        j                  | j                  | j                         y )Nr   F)	r`   r   get_reports_that_need_renewalrK   r%   rq   r   r6   rr   )r   r&   reports_to_dor<   r'   r   s         r   rS   zClarf.do_issuedate   s    )))4::9E}"y1G??9-g6D99T?D!%DIIg 2 	

499dmm,r   c                     t        |d      }i }|j                         D ]&  }|j                         }t        |d         ||d   <   ( |S )Nrr   r^   )open	readlinessplitfloat)r   r'   r]   r   linepartss         r   rq   z
Clarf.read   sJ    D#NN$DJJLE"58_DqN % r   c                 F   | j                   }i }|j                  |   j                  d   }|dz   dz   |z   }t        j                  |      }t	        |      }|dk(  r"|j                         st        d| d|        y|D ]  }	|j                  j                  |	      }
| j                   j                  j                  d   j                  |
      s%t        d	|
 d
       t        j                  |	       u|
|v r"t        d| d|
 t        j                         |	||
<    |S )z!what should be 'model' or 'train'rn   r   z*.r   zmocla sees no z
 file for Nis_datero   z is not a date.zclarf has several files at  and r\   )r   r2   r   globrK   
is_testingrL   fr&   Rrsmatchr!   removera   rb   )r   r<   whatr   r   rs   glob_stringfufiscount_fufisr'   dates              r   	get_dafuszClarf.get_dafus   s   FFHHW%**73	#o,t3		+&%j!<<>tfJwi@AD33==&D6688;;y)//5v_56 		$u}3G9E$H::'E$K  r   c                     i | _         | j                  j                  D ]O  }| j                  |      }||D ]4  }|| j                   vri | j                   |<   ||   | j                   |   |<   6 Q y N)r%   r   liver   )r   r<   r   r   s       r   rR   zClarf.get_indat   sl    vv{{GNN7+E}t.,.DOOD)16t%g.  #r   c                 (   g }t         j                  j                  | j                        si | _        | j
                  |   S t        j                  | j                        | _        t         j                  j                  | j                        }| j
                  |   D ]H  }t         j                  j                  | j
                  |   |         }||kD  s8|j                  |       J | j                  rt        dt        |      z   dz   |z          |S )Nzclarf needs r-   )r!   r"   r#   r   r   r%   r6   rI   getmtimeappendr   rL   rM   )r   r&   rp   	file_timer<   	data_times         r   r~   z#Clarf.get_reports_that_need_renewal   s    ww~~dmm,DI??9--JJt}}-	GG$$T]]3	y1G (()CG)LMI9$w' 2 ??.3w</&89DEr   c                 :   | j                   dz   }g }t        j                  |      D ]d  }t        j                  j	                  |      }|j                  d      r4|j                  d      st        d|        T|j                  |       f t        |d      }|S )Nz/*grundr   zclarf: foreign file T)reverse)
r   r   r!   r"   r_   
startswithendswithrL   r   sorted)r   r   r   r'   banas        r   
list_fufiszClarf.list_fufis  s    kkD(IIk*D77##D)Dw'==,,TF34LL + ud+r   c                    | j                   dz   |z   dz   }t        j                  j                  |      st	        d| t
        j                         y t        j                  |      }|i }||vrf| j                  ||       || j                  vr5t	        d| d| t
        j                         t        j                          | j                  |   }|S ||   }|S )Nr   r   clarf does not see r\   r,   )r   r!   r"   r#   rL   ra   rb   r6   rI   r{   r   exit)r   r<   r&   r'   r   repdatas         r   r1   zClarf.read_archive  s    {{S 9,z9ww~~d#'v.SZZ@zz$<D$KK+dii'+G9D?::'
ii(G  7mGr   c                 >    | j                  ||      }t        |       y r   )rq   rL   )r   r<   r&   r   s       r   showz
Clarf.show(  s    ))GY/gr   c                    | j                  ||      }d| d| }|d|z   }|S | j                  j                  ||      }|d|z   }|S |D ]K  }||vr	| d}|c S ||   ||   k(  r|dz   t        ||         z   }|dt        ||         z   z  }d|z   }|c S  |D ]P  }||vrd| d	|z   }|c S ||   ||   k(  r t	        t        ||               dz   t        ||         z    d
|z   }|c S  y)Nz for r   zclarf has no archive for z clarf: I don't see the side fufiz" in side file is not in clarf fufir*   zclarf: data is inconsistent 1 r   z in the side filezclarf: data is inconsistent 2T)r1   r   rq   rM   rL   )	r   r<   r&   in_clarfrH   mesgin_saifirD   rx   s	            r   is_clarf_equal_to_saifizClarf.is_clarf_equal_to_saifi-  sQ   $$Wi8gYeI;/.4DK::??7I65;DKEH$ BC(5/1#+HUO 44CC#huo...C3c9DK  EH$+E72CDsJ
(5/1#huo&'#-HUO0DD2S8DK  r   c                     t        j                         }|t        j                         k(  ry| j                  |      }t        j
                  j                  |      ryt        j                  i |       y)z'<- clarf, create issue for brown periodFT)	datertonighttodayr`   r!   r"   r#   r6   rr   )r   r   rd   s      r   create_in_brownzClarf.create_in_brownL  sR    --/ekkm#&&w/
77>>*%

2z"r   N)F)source)side)__name__
__module____qualname__r   r(   rE   r$   rU   rX   rj   rl   r{   r`   rS   rq   r   rR   r~   r   r1   r   r   r    r   r   r   r      sh     	6	?))"H'
%N	-8=$&(
>	r   r   )r   r!   ra   r   r6   r9   r   r   r   r   r   r   r   r   r   r   r   r   <module>r      s/     	 
       F Fr   