
    h]                         S SK r S SKrS SKrS SKrS SKrS SKrS SKJr  S SKJ	r	  S SK
Jr  S SKJr  S SKJr  S SKJr  S SKJr  S S	KJr   " S
 S5      rg)    N)Ishus)Velop)Sigsi)Vemli)Svemd)Spana)Folda)Fitlic                       \ rS rSr  S&S jrS rS rS rS rS r	S	 r
S'S
 jrS rS rS(S jrS rS rS rS rS rS)S jrS rS rS rS rS rS rS rS rS rS rS rS r S  r!S! r"S" r#S# r$S$ r%S%r&g)*Grond   Nc                 J   Xl         SU R                   R                  ;  a  g XPl        X l        [	        U5      U l        [        U5      U l        [        XS9U l	        [        U5      U l        [        U5      U l        [        U5      U l        [!        U5      U l        [%        U5      U l        U R                   R(                  S   U l        SU l        Ub  X0l        OU R1                  5       U l        U R.                  c  [3        S5        g U R*                  S-   U R.                  -   S-   U l        U R                  R6                  nUc/  [3        S[8        R:                  S	9  [8        R<                  " S
5        g [>        R@                  RC                  U5      (       d2  [3        SU 3[8        R:                  S	9  [8        R<                  " S
5        g [D        RF                  " U5      U l$        S U l%        0 U l&        0 U l'        S U l(        [S        US5      (       a  UR                  U l        g )Nis_seedable)do_sigfifitliFzgrond has no sigfi/.grond.json.gzzgrond: I have no entsi fufifile   zgrond: I have no 
do_verbose)*econfr   do_tarsir	   foldar   ishusr   velopr   vemlir   spanar   sigsir   svemdr
   r   dirsfudiis_data_loadedsigfi
last_sigfiprintfufi
entsi_fufisysstderrexitospathisfilefilerloaddenom	issuedatedata
samsu_datadafushasattr)selferimpr   r   do_filer   r)   s          !/home/ernad/ernad/python/grond.py__init__Grond.__init__   s   +$ 5\
5\
54
5\
5\
5\
5\
5\
FFKK(	#!J*DJ::&'IIOdjj03CC	ZZ**
/cjjAHHQKww~~j))%j\2DHHQKZZ
+
	
5,''#..DO    c                     U R                   R                  U R                  SS9nUc  g[        R                  R                  U5      nUSS nU$ )z!last sigfi from the fufis we havez*.grond.json.gz)searchNi)r   last_by_mtimer#   r-   r.   basename)r8   	last_fufi	last_banar&   s       r;   r&   Grond.last_sigfiB   sS    JJ,,TYY4E - G	GG$$Y/	t_
r>   c                     0 nU R                   R                  SU R                  S9nU H@  n[        R                  " U5      nU R
                  R                  US5      (       d  M<  X1U'   MB     Xl        U$ )N
old_to_new)orderr   grospi)r   	out_fufisr%   r0   get_issuedater   is_it_inr6   )r8   r6   	fufi_listr(   r3   s        r;   	get_dafusGrond.get_dafusL   sq    JJ((|26** ) >	D++D1I::&&y(;;#)	 
 
r>   c                    U R                   b%  U R                   U R                  S'   U R                   $ SU R                  ;   a  U R                  S   b  U R                  S   $ U R                  R                  5       nXR                  S'   U$ )+the average size of the issue over the spanT)r   r4   r   average_sizer8   sizes     r;   get_sizeGrond.get_sizeX   sp    ==$!]]DIIcN== $))		# :99S>!zz&&(		#r>   c                 :    U R                   R                  5       nU$ )rQ   )r   rS   rT   s     r;   target_sizeGrond.target_sizec   s    zz&&(r>   c                    [         R                  " U R                  5      U l        Sn[	        U R                  S   5      nU H;  nX0R                  S   ;  d  M  [        SU S35        US-  nU R                  S   U	 M=     [        SU 35        [        SU R                   35        [         R                  " U R                  U R                  5        [        S	5        g )
Nr   vsgrond: z is lostr   zgrond lost grond writes zgrond finished clearing lost)r0   r1   r(   r4   listr'   dump)r8   
count_lostpapidspapids       r;   
clear_lostGrond.clear_losth   s    JJtyy)	
diin%EIIcN*wh/0a
IIcN5)	 
 	J<()dii[)*

499dii(,-r>   c                 l   U R                   R                  SS5      n[        R                  " XR                   /5      (       d  [        R                  " U5      nO5[        R                  " U R                   5      U l        U R                  5       n[        [        U5      5      nSnSnU H[  n[        X&   5      nXW-  n[        U5      R                  S5      nUS-   U-   n	US-  nUS-  S:  a  [        U	SS	9  MP  [        U	5        M]     [        S5        [        U S
U S35        U H5  nU R                  R                  US5      (       a  M&  [        SU S35        M7     U R                  R                    H<  nU R                  R                  US5      (       d  M&  Xc;  d  M-  [        SU S35        M>     g)z/show the samel, relating dates to member countsr   .grosa.json.gzr       r      z  endz issuedates, z recordsrI   r^   z in grond is not in grospiz in grospi is not in grondN)r(   replacer0   donerer1   r4   	get_samelsortedr`   lenstrrjustr'   r   rL   r   
issuedates)
r8   
samel_fufisamelsamel_datescount_datestotalr3   valuestringto_prints
             r;   show
Grond.showv   sy    YY&&'79IJ
||J44JJz*E

499-DINN$ET%[)$I()ENEZ%%a(F 3/H1KQ"hD)h % 	c
]5':;$I::&&y(;;	{*DEF % **I ::&&y(;;+	{*DEF +r>   c                 ,   Uc  [        U R                  R                  5      nU R                  c   [	        S5        [
        R                  " 5         U R                  5       nSnU H-  nX@l        U R                  U5      nU(       a  US-  nX1:  d  M-    g    g )Nz)grond does not know its fufi, run load().r   r   )
rr   r   ru   r(   r'   r*   r,   rN   r3   
feed_issue)r8   
max_issuesr6   
count_doner3   dones         r;   feed_allGrond.feed_all   s}    TVV../J99=>HHJ 
I
 'N??9-Da
' r>   c                     Sn[        [        U5      5      nU H/  nX!U   X   -  U R                  U   -  U R                  U   -  -  nM1     X R                  S   -  nU$ )z9simple square as the original value for an immature grondr   rR   )rs   rr   r2   r4   )r8   vempas	the_total
len_vempasfitnoaverages         r;   squareGrond.square   so    	V%
E6jj,-/3zz*/EF FI 
 iin,r>   c                     SnU R                   S    H   nU R                   S   U   nX1:  d  M  UnUnM"     [        SW SU 35        X@R                   S'   XR                   S'   U R                   S   $ )Nr   pzgrond: highest is z at HV)r4   r'   )r8   highestrd   r{   highest_papids        r;   set_highestGrond.set_highest   sv    YYs^EIIcN5)E %	 $
 	"=/gY?@&		# 		#yy~r>   c                     [        U R                  S   5       HK  nU R                  S   U   SS nU R                  R                  X15      (       a  M:  U R	                  U5        MM     g )Nr]   r   
   )r`   r4   r   rL   remove)r8   critrd   r3   s       r;   outdateGrond.outdate   sX    $))C.)E		#u-a3I zz""933KK *r>   c                    [        [        U5      5      nX0R                  ;  a  [        SU 3[        R
                  S9  g SnU R                  S    Hl  nXQ:X  a  M
  U R                  S   U   n[        [        U5      5      nU H6  nX;  a  M
  XBU   Xh   -  U R                  U   -  U R                  U   -  -  nM8     Mn     SU-  U R                  S   -  n	U	$ )Nz!grond does not see the denom for r   r   r\   i C)rs   rr   r2   r'   r*   r+   r4   )
r8   	the_papid
the_vempaslen_the_vempasr   rd   r   r   r   r   s
             r;   rechenGrond.rechen   s    S_-+5n5EFzz#	YYs^E!YYs^E*FS[)J*.>jj,-/3zz./IJ J	   $ i'$))C.8r>   c                 d    U R                   S   U	 U R                   S   U	 U R                   S   U	 g )Nr   r\   r]   )r4   )r8   rd   s     r;   r   Grond.remove   s2    IIcN5!IIcN5!IIcN5!r>   c                     g N )r8   rd   r{   s      r;   add	Grond.add   s    r>   c                 d    [        [        U R                  S   5      5      nXR                  S'   U$ )Nr   r   )rr   r`   r4   )r8   count_papidss     r;   
make_countGrond.make_count   s*    4		#/0%		#r>   c                 &   U R                  5       nX!::  a  [        SU SU 35        gX!-
  n[        [        U R                  S   R                  5       S SS95      nSnU H.  n[        S	U 35        U R                  U5        US
-  nXS:X  d  M.    g    g )Nzgrund: i have z, less than Fr   c                     U S   $ )Nr   r   )items    r;   <lambda>Grond.cut_to.<locals>.<lambda>  s    Qr>   T)keyreverser   zgrund: cut_to removes r   )r   r'   dictrq   r4   itemsr   )r8   limittotal_papidsgapby_valuer   rd   s          r;   cut_toGrond.cut_to   s    ( N<.UGDE"tyy~335#7'+- . E*5'23KKAL" r>   c           
      	   U R                   U   n[        R                  R                  U5      nUSS nXPR                  S   ;   a  U(       d  [        SU S35        g[        SU S3S	S
9  [        R                  " U5      n[        R                  " 5       U R                  S   US S '   U R                  5       n[        [        U R                  S   5      5      nXR                  S'   [        SU S35        US:X  a  [        S5        U Hu  n	Xi   n
U R                  R                  U
5      nXR                  S   U	'   U R                  U5      U R                  S   U	'   US S U R                  S   U	'   US-  nX:X  d  Mu    O   [        S5        [        SU R                    35        [        R"                  " U R                   U R                  5        O[        S5        U R%                  5         [        [        U R                  S   5      5      nXR                  S'   SnSnU GH*  n	XR                  S   ;   a  M  U R                  R                  Xi   5      nU R'                  X5      nUc  [        SU	 3[(        R*                  S9  Mf  US-  nUS-  S:X  a  [        SU S35        US:X  a  [        SU	 35        M  XR                  S   :  a4  U R,                  (       a!  [        SU	 S U S!U R                  S    35        M  U R                  S   U R                  S"   :  a"  U R                  S==   S-  ss'   [        SS	S
9  OI[        SU R                  S#    S$U R                  S    S3S	S
9  U R/                  U R                  S#   5        [        U	 S%U 35        XR                  S   U	'   XR                  S   U	'   US S U R                  S   U	'   U R%                  5         US-  nUS-  S:X  d  GM  [0        R2                  " 5         [        S&U R                    35        [        R"                  " U R                   U R                  5        GM-     US'S U R                  S   USS '   [        R                  " 5       U R                  S(   US S '   [        SU R                    35        [        R"                  " U R                   U R                  5        [        S)U 35        g )*Nr   r   izgrond skips z, issuedate seenFgrond loads     …  rl   air   r   r^   z papidszgrond: filling at startr\   r]   r   zgrond: initial fill done.r_   zgrond: initial fill not donez"grond can't calculate a value for r   i  zgrond has done z calcsu"   grond: lowest can’t be zero for r   zgrond z of z < rR   r   z leaves at z enters at zgrond writes new    ozgrond finished feeding )r6   r-   r.   rB   r4   r'   r0   r1   dater
pretty_nowrV   rr   r`   r   parser   r(   ra   r   r   r*   r+   r   r   gccollect)r8   r3   do_renewr(   banaissuedate_on_banavemlisrU   r   rd   r   parsedcount_changecount_calcsr   r{   s                   r;   r   Grond.feed_issue  sT    zz)$ww% 2J		#.xL&678TF%(b1D! %*$4$4$6		#tCRy!}}4		#/0%		#~W-.1+,  ))%0(.		#u%(,F(;		#u%(,Sb			#u%!'   -.M$))-.JJtyy$)),01 	4		#/0%		#E		#&ZZ%%fm4FKK.E}:5'B::'1KT!Q&}F;< z:5'BCyy~%??F5'eWC		#7GHIyy~		#.		#!#iR(		#/{499S>:J%PDIIcN+UG;ug./$)IIcN5!$*IIcN5!$("IDIIcN5!ALd"a'

)$))56

499dii0c f &*"R[		#tAbz"$)$4$4$6		#tCRy!dii[)*

499dii('	{34r>   c                 @   U R                   (       a  U R                  $ [        R                  R	                  U R
                  5      (       aZ  [        SU R
                  -   S-   SSS9  U R                  [        R                  " U R
                  5      5      U l        [        SSS9  O)U R                  0 5      U l        SU l         U R                  $ U R                  5       nXR                  S   :  a  [        S	5        U R                  U5        SU l         U R                  $ )
Nr   z ... Tr   )flushrm   r   )r   rR   zgrond needs to trim)r$   r4   r-   r.   r/   r(   r'   initr0   r1   rV   r   )r8   rY   s     r;   r1   
Grond.loadu  s    9977>>$))$$.499,w6dK		%**TYY"78DI&%		"DI"&D99mmo3''(KK$"yyr>   c                    U R                   R                  SS5      n[        R                  " XR                   /5      (       d  [        R                  " U5      nU$ 0 nU R	                  5         U R
                  S    H9  nU R
                  S   U   nUSS nXR;   a  X%   R                  U5        M4  U/X%'   M;     [        R                  " X5        U$ )z holds papids from all issuedatesr   rh   r]   r   r   )r(   rn   r0   ro   r1   r4   appendra   )r8   rv   rw   rd   sourcer3   s         r;   rp   Grond.get_samel  s    YY&&'79IJ
||J44JJz*EL		YYs^EYYs^E*FqI! ''.$)7  $ 	

:%r>   c                     SU;  a  0 US'   SU;  a  0 US'   U R                  5       US'   SUS'   SU;  a  0 US'   SU;  a  0 US'   SU;  a  0 US'   S	U;  a  0 US
'   Xl        U R                  $ )z0initial settings when we don't have a grond filer   r\   rR   r   r   r   r]   r   r   r   )rV   r4   )r8   r4   s     r;   r   
Grond.init  s     d?DId?DIMMOS	S	d?DId?DId?DId?DI
 	yyr>   c                    U R                   R                  5       nUc  [        S[        R                  S9  gXR
                  :X  a  [        SU 3[        R                  S9  gU R                  5         U R                  S-   U-   S-   n[        R                  R                  U5      (       a  [        SU 35        gU R                  5         [        SU R
                   S	U 35        U R                  5       nU H  nU R                  XX4   5        M     U R                  5         U R                  5         U R!                  5         U R                  5         U R                  S-   U-   S-   n[        R                  R                  U5      (       a  [        SU 3[        R                  S9  [        S
U 35        ["        R$                  " X R&                  5        g)Nz%grond: no ranfi for current issuedater   Fz
grond has r   r   zgrond will not overwrite zgrond transcribes z to zgrond saves T)r    get_last_ranfir'   r*   r+   r%   r1   r#   r-   r.   r/   r   rp   transcribe_for_issuedaterecalculater   r   r0   ra   r4   )r8   r%   r(   rw   r3   s        r;   
transcribeGrond.transcribe  st   

))+=9zz#JJJug&SZZ8		yy3&)9977>>$-dV45"4::,d5':; I))%E<LM yy3&)9977>>$-dV4zz#TF#$

4#r>   c                    U R                   nUR                  S   S-   U-   S-   U-   S-   n[        R                  R	                  U5      (       d  [        S5        [        SU S3SS	9  [        R                  " U5      n[        S
5        SnU Hl  nX;  a2  [        SU SU S3[        R                  S9  U R                  U5        M:  U R                  R                  Xh   5      U R                  S   U'   US-  nMn     [        SU S35        g )Nr   r   _.json.gzz#grond: I need the vemli file {fufi}zgrond reads r   r   rl   r   r   z
grond: no z in vemlis for .r   r\   r   zgrond transcribed z papids.)r   r"   r-   r.   r/   r'   r0   r1   r*   r+   r   r   r   r4   )	r8   r%   r3   rc   r   r(   r   r   rd   s	            r;   r   Grond.transcribe_for_issuedate  s    FFvvg$y036>Kww~~d##78TF%(b1D!fE"
5'1E::'E"$(JJ$4$4V]$CDIIcN5!AL  	"<.9:r>   c                    U R                  5         [        U R                  S   5      nU Hh  nU R                  X R                  S   U   5      n[	        [        U R                  S   U   5      S-   [        U5      -   5        X0R                  S   U'   Mj     [        R                  " U R                  U R                  5        g )Nr   r\   u    → )	r   r`   r4   r   r'   rs   r0   ra   r(   )r8   rc   rd   	new_values       r;   r   Grond.recalculate  s    diin%EE99S>%+@AI#diinU+,w6YGH$-IIcN5!  	

499dii(r>   c                 *   U R                  5       U l        XR                  ;  a.  U R                  (       a  [        SU S3[        R
                  S9  gSn[        [        U R                  5      5       H  nX1:X  a    U$ X R                  U   -  nM     U$ )z1used by spigr to find if there are data availabler^   z not in samelr   r   )get_samsu_cacher5   r   r'   r*   r+   rq   r`   )r8   r3   summeldates       r;   sum_samel_untilGrond.sum_samel_until  s    ..0OO+	{-8::'401D  ood++F 2 r>   c                    [        [        U R                  5      5      S:  a  U R                  $ U R                  S-   U R                  -   S-   n[
        R                  " XR                  /5      (       d'  [
        R                  " U5      U l        U R                  $ U R                  5       n0 nSnU H  nU[        X%   5      -  nXCU'   M     [
        R                  " X5        X0l        U$ )Nr   r   z.grosu.json.gz)rr   r`   r5   r#   r%   r0   ro   r(   r1   rp   ra   )r8   r(   rw   r5   totalsr3   s         r;   r   Grond.get_samsu_cache  s    tDOO$%)??"yy3+.>>||D99+..#jj.DO??" 
Ic%*++F$*y!  	

4$$r>   c                    U R                   R                  SS5      n[        R                  " U5      nU R                  R                  5       nU R                  R                  S   S-   U-   nU H  n0 nUS-   U-   S-   n[        R                  " Xq/5      (       d  [        SU 35        M<  U R                  5         X%    H6  nU R                  S   U   n	U R                  R                  XUS	5      n
XU'   M8     U R                  (       a  [        S
U 35        [        R                  " Xv5        M     U R                  R                  S   nU R                  R!                  USU R"                  5        [        S5        g)z/spread the grond into vemli files per issuedater   rh   grondr   r   r   zgrond does not renew r\   gr_   r   N)r(   rn   r0   r1   r   get_last_sigr   r"   ro   r'   r4   r!   
make_vemlir   ra   r   clear_by_prefixr%   )r8   rv   rw   sigbase_outr3   spreadspread_fufird   r   r   r#   s               r;   r   Grond.spread  s<   YY&&'79IJ


:&jj%%'66;;w'#-3IF"S.94zAK<<\::-k];<IIK)3.

--fYL %u	 * k]34JJ{+   vv{{7#

""4TZZ@fr>   c                 P    U R                   S-   U R                  -   S-   U-   S-   nU$ )Nr   r   r   r#   r%   )r8   kindr(   s      r;   get_fufiGrond.get_fufi4  s,    yy3+c1D8:Er>   c                    U R                  5       n[        R                  R                  U5      (       a  [	        SU S35        gU R                  S5      n[        R                  " U5      n[        R                  R                  U5      (       d  [	        SU 3[        R                  S9  g[        U R                  5       S5      nU Hd  nU R                  5         X5    HI  nU R                  S   U   nU R                  R                  XvUS	5      nUR                  S
U-   S-   5        MK     Mf     UR                  5         g)z2make a vemli class file for the grond, <- grond -lr^    is already doneFgrosagrond: I don't see r   wr\   r   0
T)set_class_fufir-   r.   r/   r'   r  r0   r1   r*   r+   openr4   r!   r   writeclose)	r8   
class_fufirv   rw   
class_filer3   rd   r   r   s	            r;   write_classGrond.write_class8  s   ((*
77>>*%%GJ<'789]]7+


:&ww~~j))'
|43::F$--/5
IIIK)3.

--fYL  ut!34	 *  	r>   c                 8   U R                  S5      n[        R                  R                  U5      (       a  [	        SU S35        gU R                  S5      n[
        R                  " U5      n[        R                  R                  U5      (       d  [	        SU 3[        R                  S9  0 nU HQ  nU R                  5         X5    H6  nU R                  S   U   nU R                  R                  XvUS	5      nXU'   M8     MS     [
        R                  " X5        g
)z)make a vemli file for the grond, not usedgrover^   r  Fr  r  r   r\   r   T)r  r-   r.   r/   r'   r0   r1   r*   r+   r4   r!   r   ra   )	r8   
vemli_fufirv   rw   r   r3   rd   r   r   s	            r;   write_vemlisGrond.write_vemlisN  s    ]]7+
77>>*%%GJ<'789]]7+


:&ww~~j))'
|43::FIIIK)3.

--fYL %u	 *  	

:&r>   c                 D    U R                   S-   U R                  -   S-   nU$ )Nr   z.grond.classr   r8   r(   s     r;   r  Grond.set_class_fufic  s"    yy3+n<r>   c                 H    [         R                  " U R                  S5      nU$ )Nz*.grond.class)foldermost_recentr#   r  s     r;   get_class_fufiGrond.get_class_fufig  s    !!$))_=r>   c                 >   U R                   R                  S   S-   U R                  -   nUS-   U-   S-   n[        R                  R                  U5      (       dD  U R                  R                  5       nU R                   R                  S   S-   U-   nUS-   U-   S-   nU$ )zthis is for readingr   r   r   r   )r   r"   r%   r-   r.   r/   r   	train_sig)r8   r3   r   r   r   s        r;   get_spread_fufiGrond.get_spread_fufik  s    66;;w'#-

:ny0:=ww~~k** **&&(Cvv{{7+c1C7H"S.94zAKr>   c                     g)zthis is for writingNr   )r8   r3   s     r;   set_spread_fufiGrond.set_spread_fufiw  s    r>   c                     U R                  U5      n[        R                  R                  U5      (       d  [	        SU 3[
        R                  S9  0 $ [        R                  " U5      nU$ )z5"this is from a spread fufi, i.e. spread by issuedatezgrond does not see spread file r   )	r"  r-   r.   r/   r'   r*   r+   r0   r1   )r8   r3   r   r   s       r;   
get_vemlisGrond.get_vemlis{  sV    **95ww~~k**3K=Azz#IK(r>   )r6   r4   r2   r   r   r   r   r   r#   r(   r$   r   r3   r5   r%   r    r   r!   r   r   )NNNFr   )rI   )F)'__name__
__module____qualname____firstlineno__r<   r&   rN   rV   rY   re   r~   r   r   r   r   r   r   r   r   r   r   r1   rp   r   r   r   r   r   r   r   r  r  r  r  r  r"  r%  r(  __static_attributes__r   r>   r;   r   r      s    6:*/*X
	
.#GJ*

0"
$gR&$> D;()$6,*
"r>   r   )r0   r-   r*   r   r   r  r   r   r   r   r    r   r   r   r!   r   r   r   r   r	   r   r
   r   r   r>   r;   <module>r/     s;     	 
 	          o	 o	r>   