
    4i(Y                         S SK r S SKrS SKrS SKrS SKrS SKrS SKJr  S SKJ	r	  S SK
Jr  S SKJr  S SKJr  S SKJr  S SKJr  S S	KJr  S S
KJr  S SKJr  S SKJr  S SKJr  S SKJr   " S S5      r g)    N)Alfit)Asemb)Caler)Krikt)Ranfi)Runer)Shrot)Svemd)Ursis)Tofde)Recon)Vemli)unquotec                       \ rS rSrSS jrS rS rS r  SS jrSS jr	SS	 jr
S
 rS rS rSS jrS rS rS rS r  SS jrSrg)Mocla   c                 
   Xl         [        U5      U l        [        U5      U l        [        U5      U l        [        U5      U l        [        U5      U l
        [        U5      U l        [        U5      U l        [        U5      U l        [#        U5      U l        0 U l        U R                   R(                  S   U l        [-        US5      (       a  U R                   R.                  U l        OX l        SU l        S U l        SU l        g )Nclass
do_verboseTF)er   asembr   calerr   kriktr   ranfir   vemlir
   svemdr	   shrotr   reconr   ursisall_sigs_useddirsfolderhasattrr   can_i_model
model_fufi	is_redone)selferimpr   s      !/home/ernad/ernad/python/mocla.py__init__Mocla.__init__   s    5\
5\
5\
5\
5\
5\
5\
5\
5\
ffkk'*5,''"ff//DO(O    c                     U R                   nUR                  U   R                  S   nUS-   U-   S-   n[        R                  " U5        g)z<fired up by clarf which ensures we have archived the presortmocla/z.*N)r   reportr!   filerclear_by_glob)r'   repcode	issuedater   	mocla_dirglobs         r)   clearMocla.clear4   sE    FFHHW%**73	3*T1D!r,   c                 B   U R                   nUR                  U   R                  S   nUS-   S-   U-   n[        R                  " U5      n[	        U5      nUS:X  a'  UR                  5       (       d  [        SU SU 35        gUS:  a  [        S	U S
U SU 35        US   $ )z4what should be 'model' or 'train', used by the otriar.   r/   z*.r   zmocla sees no z
 file for N   mocla sees  z files for )r   r0   r!   r6   len
is_testingprint)r'   r3   whatr   r5   glob_stringfufiscount_fufiss           r)   find
Mocla.find;   s    FFHHW%**73	#o,t3		+&%j!<<>>tfJwi@A?K}AdV;wiHI Qxr,   c                 F   U R                   nUR                  U   R                  S   nUR                  R	                  USS9nUc  [        S5        gX@l        UR                  R	                  USS9n[        R                  R                  U5      R                  SS5      nU$ )z3get model for a report, should not require a searchr.   z*.model)searchNz mocla: no last model, run start..model )r   r0   r!   dlast_by_mtimer?   r%   ospathbasenamereplace)r'   r3   r   r5   
last_modellast_model_fufisigs          r)   last_sigMocla.last_sigN   s    FFHHW%**73	SS&&y&C
45$##++Ii+Hgg/77"E
 
r,   Nc           
      	   SU l         U R                  R                  U   nU R                  R                  U   R                  S   n	UR	                  5       (       a  SU R                  R
                  ;   a  SU l         U R                  R                  U5      n
[        U
R                  5       5      nUS:X  a2  SnUSU 3-  n[        U5        U R                  R                  UUUS9nU$ US	:  a  [        S
[        R                  S9  g[        U
R                  5       5      S   nSU S3nUSU 3-  n[        U5        U R                  R!                  UUUUS9nU$ U(       d  U(       ay  U R                  R#                  XS9n[$        R&                  R)                  U5      (       a<  [        SU-   S-   5        U(       d  U(       a  U R*                  R-                  U5        U$ U R/                  U5      nU R0                  (       a  [        SU-   5        U R3                  X/UUS9nUc"  [        SU-   S-   U-   [        R                  S9  gU	S-   U-   S-   nSU-   S-   U-   nU R0                  (       a  [        SU-   5        [4        R6                  " U5        U R8                  c  [;        S5      e[4        R<                  " UUU R8                  /5      (       d  U R?                  UU5      (       a  U(       aP  SU-   S-   nUU R8                  S-   U-   -  n[A        U R                  S9n[        S U-   5        URC                  UUS!S"9  O[        S#U-   5        [$        R&                  R)                  U5      (       d   U R                   (       a  [        S$U-   5        g[        S%U-   5        U RE                  U5      U l#        U RI                  UU RF                  5      nUc  U R                   (       a  U RJ                  (       a0  [        S&[        R                  5        [        RL                  " S	5        [        S'[        R                  5        SU l%        U RO                  XUUXVSS(9nUc.  [        S)[        R                  S9  [        RL                  " S	5        [        S*U-   5        U(       d  U(       aS  U R                  R!                  XUS9nUc  [        S+[        R                  S9  gU R*                  R-                  U5        U$ U$ ),z<- bin/release Tr.   no_seedsFr   z#mocla: no model for empty, seedlessz and motherless )r3   r4   doc_fufir:   z%mocla: I can't handle several mothersfilezmocla: I use z for the first issue ofr<   )r3   r4   parentrW   )r3   r4   r;   z, no donere, assemble finished.mocla has issuedate r   Nmocla has no vemte for  at r/   .predipredict__zmocla: class fufi is zI need a model_fufi)/home/ernad/opt/usr/bin/svm-predict -b 1 r(   mocla runs iX  )timeoutmocla doneres mocla: I don't see mocla: predi is zmocla will not try another runzmocla tries another run)r   do_asembrW   do_cachedo_redoz&mocla can't continue with no side filezmocla: side is z#mocla: asemb returns None, returned)(r$   r   r0   r!   is_it_emptyconfr   get_mothersr=   keysr?   r   build_unsortedsysstderrlistbuildfufirL   rM   isfiler   cache_selection_screenrS   r   digr1   preparer%   	Exceptiondonerecompare_with_predir   run_with_lock
papid_file
papid_fufi	side_filer&   exitrun)r'   r3   r4   r   ri   rW   rj   rk   r0   r5   motherscount_mothersnote
asemb_fufimotherrR   
vemte_fufi
predi_fufiloggerrunnerruner	side_fufiouts                          r)   r   	Mocla.run_   su     w'FFMM'*//8	J$&&++$=$Djj,,W5G/M!<*7)44d!ZZ66wAJ@H 7 J
 "!q =::'',,.)!,F"6(*ABDayM!D$K))'4=173; * =J  wNJww~~j))mj078 9 w JJ55jA!!mmG$??(945 XXig)3  5
+g5>Jzz# _y08;
g%+i7??)J67j!??"122LLj$//%BCC++J
CC@:MPSSFdoo+j88F'E-&()<"Z/0ww~~j))d.>.>'*45 :-.//*5NN:t?	!1!1~~6jj"+**!DN((7*$,$,#'  )C {>::')+,w))'3; * =J!;::'JJ--j9r,   c                 6   Uc  [        S[        R                  S9  gU R                  S-   U-   S-   U-   S-   n[        R
                  R                  U5      (       a  [        S5        U$ U R                  S-   U-   S-   U-   S-   U R                  R                  -   n[        R
                  R                  U5      (       a  U R                  U5      n[        S	5        U$ U R                  U5      nUbE  [        S
U-   S-   5        [        R                  " Xv5        U R                  U5      n[        S5        U$ U R                  R                  US9nUc  [        SU-   5        gU R                  XS9n	U	c  [        SU-   5        gU(       a  [        SU	-   5        X R                  l        U R                  U R                  l        XR                  l        [        SU	-   5        U R                  R%                  XS9nUc  gU R                  U5      n[        S5        U$ )zAfind get material to work on a sig, if needed with only the kriktNzmocla has no model, run train.rX   r/   ra   z.txtzmocla: case 1z.vemlizmocla: case 2zmocla uses remote vemlis .zmocla: case3)rR   z+mocla can't continue to dig, no vemlis for r\   zmocla sees no alfit for zmocla needs to make vemlis for z mocla needs to build the vemlis zmocla: case 4)r?   rq   rr   r"   rL   rM   rv   r   extvemte_from_vemliget_from_trainshutilcopy2r   get	get_alfitrR   r
build_fufi)
r'   r4   rR   r3   r   r   
vemli_fufigot_from_trainr   
alfit_fufis
             r)   rx   	Mocla.dig   s    ;2D [[3&2S83>G
77>>*%%/"[[3&2S83>Ijjnn
77>>*%%..z:J/",,Z8%->DELL4..z:J.!

3'=?#EF ^^I^E
,y893j@A

 KK

 

0:=>ZZ**:*M
**:6
or,   c                    U R                   S-   U-   S-   n[        R                  R                  U5      (       a  U$ U R	                  U5        [        R                  R                  U5      (       a  U(       a  [        SU S35        U$ U(       a  [        SU-   S-   5        U R                   S-   U-   S-   n[        R                  R                  U5      (       d  U R	                  U5        [        R                  R                  U5      (       dL  [        SU-   5        U R                  R                  US	9  [        R                  R                  U5      (       d  g
U(       a  [        SU S35        [        U R                  S9U l
        U R                   U R                  l         U R                  R                  UUS9nUc  [        SU-   [        R                  S9  g
U(       a  [        SU 35        [        U R                  S9U l        U R                   U R                  l         U R                  R                  U5      n[        R                  R                  U5      (       a  U$ g
)z&This takes the approach from the kriktr/   z.alfit.json.gzz
mocla got z from training.zmocla needs to get alfits z from krikts.z.krikt.json.gzzmocla has no krikt for )do_only_issuedateNzmocla found the krikts r   rc   r\   zmocla can't get the tofde for rX   zmocla found the tofde )r"   rL   rM   rv   r   r?   r   renewr   r   tofdeupdate_for_krikt_fufirq   rr   r   alfit)r'   r4   r   r   
krikt_fufi
tofde_fufis         r)   r   Mocla.get_alfit  s   [[3&25EE
77>>*%%J'77>>*%%
:,o>?.;oMN[[3&25EE
ww~~j))
+77>>*--/);<

  9 =ww~~j11+J<q9:(
 KK

ZZ55jAK 6 M
2Z?zz#*:,78(
 KK

ZZ55jA
77>>*%%r,   c                     UnUSS nUR                  SU-   S5      nUR                  SSU-   S-   5      n[        R                  R                  U5      (       a  [        R
                  " X!5        U$ g )Niir   rI   /class/r/   )rO   rL   rM   rv   r   r   )r'   
local_fufiru   kinds       r)   r   Mocla.get_from_train9  sf    C|||C$J+||IsTzC'7877>>$LL*Kr,   c                 @   UR                  S5      (       a  UR                  SS5      nO2[        SU-   5        UR                  SS5      nUR                  SS5      n[        R                  R                  U5      (       a+  [        R                  " U5      (       d  [        SU-   5        U$ [        US5      n[        R                  " U5      nU H  nUR                  S	XE   -   S
-   5        M     UR                  5         [        SU-   5        U R                  U5        U$ )Nz.vemli.json.gz.classz!mocla: class case, vemli_fufi is z.json.gzz/vemli/r   mocla does not renew wz0 
mocla wrote )endswithrO   r?   rL   rM   rv   r1   is_emptyopenloadwritecloser~   )r'   r   
class_fufi
class_file
vemli_datapapids         r)   r   Mocla.vemte_from_vemliC  s    /00#++,<hGJ 5
BC#++JAJ#++IyAJ77>>*%%ennZ.H.H)J67*c*
ZZ
+
ETJ$55<=  nz)*
#r,   c                    [         R                  R                  U5      n[         R                  R                  U5      nUSS nUS-   U-   S-   n[        R
                  " XQ/5      (       d  [        SU-   5        U$ [        US5      n[        US5      n UR                  5       nW HQ  n
SU
;  a  [        SU
-   5        M  U
R                  S5      nUS
   n[        USS 5      S-   nUR!                  U5        MS     UR#                  5         [        SU-   5        U$ ! [         aO  n	[        [        U	5      5        [        SU-   [        R                  S	9  [        R                  " S
5         Sn	A	NSn	A	ff = f)z*generated from the vemte_fufi for an issuer   
   r/   .papidr   r   r   zmocla error reading rX   r:   N#zskip z # r   r   )rL   rM   dirnamerN   r1   r{   r?   r   	readlinesUnicodeDecodeErrorstrrq   rr   r   splitr   r   r   )r'   r   fudibanar4   r   r~   r   class_linesr   
class_linepartsr   s                r)   r~   Mocla.papid_fileZ  sW   wwz*ww
+2J	CZ)+h6
||J55)J67*c*
*c*
	$..0K
 &J*$g
*+$$U+E !HEE#2J'$.EU# & 	nz)*) " 	#a&M(:5CJJGHHQKK	s   D 
E4%AE//E4c                    [         R                  " U5      n[         R                  " U5      n[        U5      n[        U5      nUS-
  U:X  a  gU(       d  [        R                  " U5        g[        S[        R                  S9  [        [        R                  " 5       [        R                  S9  SU S3[        U5      -   S-   nUS	U S3[        U5      -   S
-   -  n[        U[        R                  S9  U R                  R                  U5        [        R                  R                  U5      (       a  [        R                  " U5        g)Nr:   TFzmocla prints tracebackrX   zmocla: z's z lines zdon't go with z + 1 )r1   read_as_linesr=   rL   remover?   rq   rr   	traceback
format_excr   r   makerM   rv   )	r'   r   r   do_warnpapid_linespredi_lines	len_predi	len_papiderrs	            r)   r|   Mocla.compare_with_predi}  s    ))*5))*5$	$	q=I%IIj!&SZZ8i""$3::6
|3'#i.89D
|3/#i.@7JJc

# 	


#77>>*%%IIj!r,   c                    UR                  SS5      n[        R                  " X1U/5      (       d  [        SU-   S-   5        U$ U R	                  X!SS9nU(       d  g[        R
                  " U5      nU R                  U5      nUS   nUS	   nS
n	[        US5      n
U	[        U5      :  aH  X   R                  S5      nX   nUS-   XY   -   nU
R                  U5        U	S-  n	U	[        U5      :  a  MH  U
R                  5         U$ )zmake the side filer_   z.sidezmocla finds that z needs no renewal.T)r   Nplus_poslinesr   r   r<   	r:   )rO   r1   r{   r?   r|   r   get_predi_linesr   r=   r   r   r   )r'   r   r   r   is_correct_sizer   predisr   r   countr   predi_partsprobalines                 r)   r   Mocla.side_file  s   &&x9	||IJ'?@@%	14HHI11*:> 2 @))*5%%j1*%WoC(	c+&&%,2237K)E 4<+"44D
 OOD!QJE c+&& 	r,   c                 ^    [         R                  R                  U5      nUS-   U-   nUS-  nU$ )Nr/   r_   )rL   rM   r   )r'   r%   r4   r5   r   s        r)   get_predi_fufiMocla.get_predi_fufi  s2    GGOOJ/	_y0
h
r,   c                 `   Uc  [        S5      eU R                  (       a  [        SU-   5        U R                  U5      nU R	                  X4UU R                  S9nUc"  [        SU-   S-   U-   [
        R                  S9  g SU-   S-   U-   nU R                  (       a  [        S	U-   5        U R                  X5      n[        R                  " U5        [        R                  " XuU R                  /5      (       d  U R                  XW5      (       dA  S
nX SU SU 3-  n[        U R                  S9n	[        SU-   5        U	R                  X5        O[        SU-   5        [         R"                  R%                  U5      (       d   U R&                  (       a  [        SU-   5        g [        SU-   5        U$ )Nz$this does not work with no issuedater[   r\   r]   r^   rX   r`   ra   zclass fufi is rb   r<   rc   rd   rf   rg   rh   )rz   r   r?   rS   rx   rq   rr   r   r1   ry   r{   r%   r|   r   r   r}   rL   rM   rv   r$   )
r'   r%   r3   r4   rR   r   r   r   r   r   s
             r)   class_by_modelMocla.class_by_model  s   BCC??(945mmG$XXig)-  :
+g5>Jzz# g%+i7??"Z/0((?
j!<<
$ABB&&z>>@FAj\:,??F'E-&()/"Z/0ww~~j))d.>.>'*45 :-.r,   c                    0 n[         R                  " U5      n US   nSnUS:X  a  SnOUS:X  a  SnOyUS:X  a\  [        S	U R                  -   [        R
                  S9  [        R                  " U R                  5        [        R                  " S5        O[        S
U-   S-   S-   U-   5      eUSS nXRS'   X2S'   U$ ! [         a    [        U[        R
                  S9   gf = f)zshould move to saifir   rX   Nzlabels 1 -1
r:   zlabels -1 1
   z	labels 1
z0mocla finds that the prediction failed, removes zmocla can't parse ''z in r   r   )r1   r   
IndexErrorr?   rq   rr   r%   rL   r   r   rz   )r'   r   r   r   	head_liner   s         r)   r   Mocla.get_predi_lines  s    ))*5	#AI
 'H/)H,&D//"(+

4IIdoo&HHQK1I=C"#%/0 1 1 "!"o"J"G
+  	+CJJ/	s   B; ;"C C c                 (   [         R                  R                  U5      (       d  [        SU 3[        R
                  S9  gUR                  SS5      nUR                  SS5      nUR                  SS5      nSU-   S	-   nXS	-   U-   -  n[        U R                  S
9n	[        SU-   S	-   U-   5        U	R                  X5        [         R                  R                  U5      (       d$  [        SU 35        [        R                  " S5        [         R                  R                  U5      (       d  U R                  R                  U5        U R                  Xg5      n
[        R                  " U
5      n0 nU H*  nUR!                  S5      n[#        US   5      XS   SS '   M,     U(       aB  [         R$                  " U5        [         R$                  " U5        [         R$                  " U5        U(       a!  [        S5        [         R$                  " U5        U$ )z?used by psigr. It should replace the invocations of svm-predictzmocla can't open rX   NrH   r   r_   r   rb   r<   rc   rd   zmocla has no r:   r   r   r   r   zmocla deletes {model_fufi})rL   rM   rv   r?   rq   rr   rO   r   r   r}   r   r   papid_fufi_from_train_fufir   r1   r   	partitionfloatr   )r'   r%   r   do_delete_resultdo_delete_modelr   r   r   r   r   r   
side_linesevals	side_liner   s                  r)   predictMocla.predict  s    ww~~j))%j\2zz#  ''(;
''(;
''(;
<zICOs"Z//DFF#mf$s*V34F+ww~~j))M*./HHQKww~~j))JJ11*=NN::	((3
#I''-E#(q?E(3B-  $ IIj!IIj!IIj!./IIj!r,   )r   r    r   r   r$   r   r   r"   r&   r   r%   r   r   r   r   r   r   r   r   )F)FTNTF)T)FF)__name__
__module____qualname____firstlineno__r*   r7   rD   rS   r   rx   r   r   r   r~   r|   r   r   r   r   r   __static_attributes__ r,   r)   r   r      sk    2"&" BF27rv6p%N.!F2>!F: <A %&r,   r   )!r6   rL   rq   r   r   r1   r   r   r   r   r   r   r   r   r   r   r   r   r   r	   r   r
   r   r   r   r   r   r   r   r   requests.utilsr   r   r  r,   r)   <module>r     sJ     	 
                "H Hr,   