
    g,Y                         d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m
Z
 d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ  G d d      Zy)    N)Alfit)Asemb)Krikt)Ranfi)Runer)Shrot)Tofde)Vemli)unquotec                       e Zd ZddZd Zd Zd Z	 	 ddZddZddZ	d	 Z
d
 Zd ZddZd Zd Zd Zd Zd Zd Z	 	 ddZy)Moclac                    || _         t        | j                         | _        t        | j                         | _        t        | j                         | _        t        | j                         | _        t        | j                         | _
        i | _        | j                   j                  d   | _        t        |d      r| j                   j                  | _        n|| _        d| _        d | _        d| _        y )Nclass
do_verboseTF)er   asembr   kriktr   ranfir
   vemlir   shrotall_sigs_useddirsfolderhasattrr   can_i_model
model_fufi	is_redone)selferimpr   s      !/home/ernad/ernad/python/mocla.py__init__zMocla.__init__   s    466]
466]
466]
466]
466]
ffkk'*5,'"ff//DO(DO    c                     | j                   }|j                  |   j                  d   }|dz   |z   dz   }t        j                  |       y)z<fired up by clarf which ensures we have archived the presortmocla/z.*N)r   reportr   filerclear_by_glob)r   repcode	issuedater   	mocla_dirglobs         r    clearzMocla.clear-   sE    FFHHW%**73	3*T1D!r"   c                 2   | j                   }|j                  |   j                  d   }|dz   dz   |z   }t        j                  |      }t	        |      }|dk(  r"|j                         st        d| d|        y|dkD  rt        d	| d
| d|        |d   S )z4what should be 'model' or 'train', used by the otriar$   r%   z*.r   zmocla sees no z
 file for N   mocla sees  z files for )r   r&   r   r,   len
is_testingprint)r   r)   whatr   r+   glob_stringfufiscount_fufiss           r    findz
Mocla.find4   s    FFHHW%**73	#o,t3		+&%j!<<>tfJwi@A?K}AdV;wiHI Qxr"   c                 R   | j                   }|j                  |   j                  d   }|j                  j	                  |d      }|t        d       y|| _        |j                  j	                  |d      }t        j                  j                  |      j                  dd      }|S )z3get model for a report, should not require a searchr$   z*.model)searchNz mocla: no last model, run start..model )r   r&   r   dlast_by_mtimer4   r   ospathbasenamereplace)r   r)   r   r+   
last_modellast_model_fufisigs          r    last_sigzMocla.last_sigF   s    FFHHW%**73	SS&&y&C
45$##++Ii+Hgg/77"E
 
r"   Nc           	         d| _         | j                  j                  |   }| j                  j                  |   j                  d   }	|j	                         rJd| j                  j
                  v r2d| _         t        d       | j                  j                  |||      }
|
S |s|rf| j                  j                  ||      }
t        j                  j                  |
      r*t        d|
z   d	z          |s|r| j                  |
||       |
S | j                  |      }| j                  rt        d
|z          | j!                  ||||      }|%t        d|z   dz   |z   t"        j$                         y|	dz   |z   dz   }d|z   dz   |z   }| j                  rt        d|z          t'        j(                  |       | j*                  t-        d      t'        j.                  ||| j*                  g      s| j1                  ||      r|rTd|z   dz   }|| j*                  dz   |z   z  }t3        | j                        }t        d|z          |j5                  ||       nt        d|z          t        j                  j                  |      s| j                   rt        d|z          yt        d|z          | j7                  |      | _        | j;                  || j8                        }|| j                   r| j<                  r/t        dt"        j$                         t#        j>                  d       t        dt"        j$                         d| _        | jA                  ||||||d       }|0t        d!t"        j$                         t#        j>                  d       t        d"|z          |s|rQ| j                  jC                  |||      }
|
t        d#t"        j$                         y| j                  |
||       |
S |S )$z <- bin/release Tr$   no_seedsFz2mocla: I can't model an empty report with no seeds)r)   r*   doc_fufi)r)   r*   r0   z, no donere, assemble finished.mocla has issuedate r   Nmocla has no vemte for  at filer%   .predipredict__class fufi is zI need a model_fufi)/home/ernad/opt/usr/bin/svm-predict -b 1 r1   r   mocla runs mocla doneres mocla: I don't see mocla: predi is zmocla will not try another runr/   zmocla tries another run)r   do_asembrJ   do_cachedo_redoz&mocla can't continue with no side filezmocla: side is z#mocla: asemb returns None, returned)"r   r   r&   r   is_it_emptyconfr4   r   build_unsortedfufir@   rA   isfilecache_selection_screenrG   r   digsysstderrr'   preparer   	Exceptiondonerecompare_with_predir   run_with_lock
papid_file
papid_fufi	side_filer   exitrunbuild)r   r)   r*   r   r[   rJ   r\   r]   r&   r+   
asemb_fufirF   
vemte_fufi
predi_fufiloggerrunnerruner	side_fufiouts                      r    rp   z	Mocla.runW   s     w'FFMM'*//8	TVV[[(#( JK!ZZ66wAJ@H 7 J
 "! wINJww~~j)mj078 9 w//
GYO!!mmG$??(945 XXig)3  5
+g5>Jzz# _y08;
g%+i7??"Z/0j!??"122LLj$//%BC''
J?G@:MPSSFdoo+j88F'E-&()/"Z/0ww~~j)d.>.>'*45 :-.//*5NN:t?	!1!1~~6jj"+**!DN((7I*$,$,x#'  )C {>::')+,w))'Y3; * =J!;::'''
GYGr"   c                 8   |t        dt        j                         y| j                  dz   |z   dz   |z   dz   }t        j
                  j                  |      rt        d       |S | j                  dz   |z   dz   |z   dz   | j                  j                  z   }t        j
                  j                  |      r| j                  |      }t        d	       |S | j                  |      }|Et        d
|z   dz          t        j                  ||       | j                  |      }t        d       |S | j                  j                  |      }|t        d|z          yd| }	| j                  ||      }
|
t        d|z          y|rt        d|
z          || j                  _        | j                  | j                  _        || j                  _        t        d|
z          | j                  j%                  |
|      }|y| j                  |      }t        d       |S )zB find get material to work on a sig, if needed with only the kriktNzmocla has no model, run train.rO   r%   rS   z.txtzmocla: case 1z.vemlizmocla: case 2zmocla uses remote vemlis .zmocla: case3)rF   z+mocla can't continue to dig, no vemlis for z$mocla needs to call get_alfit() for rL   zmocla sees no alfit for zmocla needs to make vemlis for z mocla needs to build the vemlis zmocla: case 4)r4   re   rf   r   r@   rA   rb   r   extvemte_from_vemliget_from_trainshutilcopy2r   get	get_alfitrF   r
build_fufi)r   r*   rF   r)   r   rs   
vemli_fufigot_from_trainr   note
alfit_fufis              r    rd   z	Mocla.dig   s    ;2D [[3&2S83>G
77>>*%/"[[3&2S83>Ijjnn
77>>*%..z:J/",,Z8%->DELL4..z:J.!

3'=?#EF 6gY?^^I*^E
,y893j@A

 KK

 

0:=>ZZ**:**M
**:6
or"   c                    | j                   dz   |z   dz   }t        j                  j                  |      r|S | j	                  |       t        j                  j                  |      r|rt        d| d       |S |rt        d|z   dz          | j                   dz   |z   dz   }t        j                  j                  |      sz| j	                  |       t        j                  j                  |      sJt        d|z          | j                  j                  |	       t        j                  j                  |      sy
|rt        d| d       t        | j                        | _
        | j                   | j                  _         | j                  j                  ||      }|t        d|z   t        j                         y
|rt        d|        t        | j                        | _        | j                   | j                  _         | j                  j                  |      }t        j                  j                  |      r|S y
)z&This takes the approach from the kriktr%   z.alfit.json.gzz
mocla got z from training.zmocla needs to get alfits z from krikts.z.krikt.json.gzzmocla has no krikt for )do_only_issuedateNzmocla found the krikts r{   rV   rL   zmocla can't get the tofde for rO   zmocla found the tofde )r   r@   rA   rb   r~   r4   r   renewr	   r   tofdeupdate_for_krikt_fufire   rf   r   alfit)r   r*   r   r   
krikt_fufi
tofde_fufis         r    r   zMocla.get_alfit   s   [[3&25EE
77>>*%J'77>>*%
:,o>?.;oMN[[3&25EE
ww~~j)
+77>>*-/);<

  9 =ww~~j1+J<q9:(
 KK

ZZ55jAK 6 M
2Z?zz#*:,78(
 KK

ZZ55jA
77>>*%r"   c                     |}|dd }|j                  d|z   d      }|j                  dd|z   dz         }t        j                  j                  |      rt	        j
                  ||       |S y )Niir{   r=   /class/r%   )rC   r@   rA   rb   r   r   )r   
local_fufira   kinds       r    r~   zMocla.get_from_train  se    C|||C$J+||IsTzC'7877>>$LLz*Kr"   c                    |j                  d      r|j                  dd      }n2t        d|z          |j                  dd      }|j                  dd      }t        j                  j                  |      r%t        j                  |      st        d|z          |S t        |d      }t        j                  |      }|D ]  }|j                  d	||   z   d
z           |j                          t        d|z          | j                  |       |S )Nz.vemli.json.gz.classz!mocla: class case, vemli_fufi is z.json.gzz/vemli/r   mocla does not renew w0 
mocla wrote )endswithrC   r4   r@   rA   rb   r'   is_emptyopenloadwritecloserl   )r   r   
class_fufi
class_file
vemli_datapapids         r    r}   zMocla.vemte_from_vemli%  s    /0#++,<hGJ 5
BC#++JAJ#++IyAJ77>>*%ennZ.H)J67*c*
ZZ
+
ETJu$55<=  nz)*
#r"   c                    t         j                  j                  |      }t         j                  j                  |      }|dd }|dz   |z   dz   }t	        j
                  ||g      st        d|z          |S t        |d      }t        |d      }	 |j                         }D ]M  }
d|
vrt        d|
z          |
j                  d      }|d   }t        |d
d       dz   }|j!                  |       O |j#                          t        d|z          |S # t        $ rF}	t        t        |	             t        d|z   t        j                  	       t                Y d
}	~	d
}	~	ww xY w)z*generated from the vemte_fufi for an issuer   
   r%   .papidr   r   r   zmocla error reading rO   N#zskip z # r/   r   r   )r@   rA   dirnamerB   r'   ri   r4   r   	readlinesUnicodeDecodeErrorstrre   rf   quitsplitr   r   r   )r   rs   fudibanar*   rm   rl   r   class_linesr   
class_linepartsr   s                r    rl   zMocla.papid_file<  sP   wwz*ww
+2J	CZ)+h6
||J5)J67*c*
*c*
	$..0K
 &J*$g
*+$$U+E !HEE#2J'$.EU# & 	nz)*) " 	#a&M(:5CJJGFF	s   D 	E <EE c                    t        j                  |      }t        j                  |      }t        |      }t        |      }|dz
  |k(  ry|st        j                  |       yt        dt        j                         t        t        j                         t        j                         d| dt        |      z   dz   }|d	| dt        |      z   d
z   z  }t        |t        j                         | j                  j                  |       t        j                  j                  |      rt        j                  |       y)Nr/   TFzmocla prints tracebackrO   zmocla: z's z lines zdon't go with z + 1 )r'   read_as_linesr2   r@   remover4   re   rf   	traceback
format_excr   r   makerA   rb   )	r   
input_fufirt   do_warninput_linespredi_lines	len_predi	len_inputerrs	            r    rj   zMocla.compare_with_predi_  s    ))*5))*5$	$	q=I%IIj!&SZZ8i""$3::6
|3'#i.89D
|3/#i.@7JJc

# 	


#77>>*%IIj!r"   c                    |j                  dd      }t        j                  |||g      st        d|z   dz          |S | j	                  ||d      }|syt        j
                  |      }| j                  |      }|d   }|d	   }d
}	t        |d      }
|	t        |      k  rI||	   j                  d      }||   }|dz   ||	   z   }|
j                  |       |	dz  }	|	t        |      k  rI|
j                          |S )zmake the side filerQ   z.sidezmocla finds that z needs no renewal.T)r   Nplus_poslinesr   r   r1   	r/   )rC   r'   ri   r4   rj   r   get_predi_linesr   r2   r   r   r   )r   rt   rm   rx   is_correct_sizepapid_linespredisr   r   countrn   predi_partsprobalines                 r    rn   zMocla.side_filex  s!   &&x9	||I
J'?@%	14HHI11*j:> 2 @))*5%%j1*%WoC(	c+&&%e,2237K)E 4<+e"44D
 OOD!QJE c+&& 	r"   c                    | j                   }ddlm}  || j                         }|j                  |   j                  d   }|dz   }t        |d      }t        |j                         t        j                  |j                        }|D ]  }	|j                  d||	   z   dz           |j                          t        d|z          |S )	Nr   )Grundr$   z/grund.classr   r   r   r   )r   grundr   r&   r   r   r4   r   r'   r   r   r   )
r   r)   r   r   r   r+   r   r   vemlisr   s
             r    grund_vemte_fufizMocla.grund_vemte_fufi  s    FFdffHHW%**73	/
*c*
eE,,-ETF5M1D89 nz)*r"   c                 x   | j                   }i }|j                  |   j                  d   }t        j                  j                  |      }t        j                  j                  |      }|dz  }| d| d}t        j                  ||g      st        d| d|        |S t        j                  |       d|d<   d	|d
<   t        j                  j                  |      st        j                  |       |j                  j                  d|||       t        d|z          |dz   }t        j                  ||       y)zshould move to ursisr$   z/cache/createdr%   z.htmlrX   z over z'source'	back_fromz	'created'stagepapers_screen)paramszmocla writes z
/peep.htmlN)r   r&   r   r@   rA   r   r'   ri   r4   rg   isdirmkdirt
via_system	link_fufi)	r   rr   r)   r*   r   r   out_dir
cache_fufi	peep_fufis	            r    rc   zMocla.cache_selection_screen  s   FF((7#((1''//'*''//'*##y)E2
||J5N:,fZLABj!({%www}}W%HHW	
JvNo
*+l*	
I.r"   c                 ^    t         j                  j                  |      }|dz   |z   }|dz  }|S )Nr%   rQ   )r@   rA   r   )r   r   r*   r+   rt   s        r    get_predi_fufizMocla.get_predi_fufi  s2    GGOOJ/	_y0
h
r"   c                 B   |t        d      | j                  rt        d|z          | j                  |      }| j	                  |||| j                        }|%t        d|z   dz   |z   t
        j                         y d|z   dz   |z   }| j                  rt        d	|z          | j                  ||      }t        j                  |       t        j                  ||| j                  g      s| j                  ||      sJd
|z   dz   }||dz   |z   z  }t        | j                        }	t        d|z          |	j                  ||       nt        d|z          t         j"                  j%                  |      s| j&                  rt        d|z          y t        d|z          |S )Nz$this does not work with no issuedaterK   rL   rM   rN   rO   rR   rS   rT   rU   r1   rV   rW   rX   rY   rZ   )rh   r   r4   rG   rd   re   rf   r   r'   rg   ri   r   rj   r   r   rk   r@   rA   rb   r   )
r   r   r)   r*   rF   rs   ru   rt   rv   rw   s
             r    class_by_modelzMocla.class_by_model  s   BCC??(945mmG$XXig)-  :
+g5>Jzz# g%+i7??"Z/0((Y?
j!<<
Z$AB&&z:>@:MPSSFj3&33F'E-&()/"Z/0ww~~j)d.>.>'*45 :-.r"   c                    i }t        j                  |      }	 |d   }d}|dk(  rd}nv|dk(  rd}nn|dk(  rRt        d	| j                  z   t        j
                         t        j                  | j                         t                nt        d
|z   dz   dz   |z         |dd }||d<   ||d<   |S # t        $ r t        |t        j
                         Y yw xY w)z should move to saifir   rO   Nzlabels 1 -1
r/   zlabels -1 1
   z	labels 1
z0mocla finds that the prediction failed, removes zmocla can't parse ''z in r   r   )r'   r   
IndexErrorr4   re   rf   r   r@   r   r   rh   )r   rt   ry   r   	head_liner   s         r    r   zMocla.get_predi_lines  s    ))*5	#AI
 'H/)H,&D//"(+

4IIdoo&F1I=C"#%/0 1 1 "!"o"J"G
+  	+CJJ/	s   B/ /$CCc                    t         j                  j                  |      st        d| t        j
                         y|j                  dd      }|j                  dd      }|j                  dd      }d|z   d	z   }||d	z   |z   z  }t        | j                  
      }	t        d|z   d	z   |z          |	j                  ||       t         j                  j                  |      st        d|        t                t         j                  j                  |      st        dt        j
                         | j                  ||      }
t        j                  |
      }i }|D ]*  }|j                  d      }t        |d         ||d   dd <   , |r?t        j                   |       t        j                   |       t        j                   |       |r t        d       t        j                   |       |S )z?used by psigr. It should replace the invocations of svm-predictzmocla can't open rO   Nr<   r   rQ   r   rU   r1   rV   rW   zmocla has no zmocla needs {papid_fufir   r   r   r   zmocla deletes {model_fufi})r@   rA   rb   r4   re   rf   rC   r   r   rk   r   rn   r'   r   	partitionfloatr   )r   r   ru   do_delete_resultdo_delete_modelrs   rt   rm   rv   rw   rx   
side_linesevals	side_liner   s                  r    predictzMocla.predict  s    ww~~j)%j\2zz#  ''(;
''(;
''(;
<zICO*s"Z//DFF#mf$s*V34FF+ww~~j)M*./Fww~~j)+#**=NN:z:	((3
#I''-E#(q?E%(3B-  $ IIj!IIj!IIj!./IIj!r"   )F)FTNTF)T)FF)__name__
__module____qualname__r!   r-   r9   rG   rp   rd   r   r~   r}   rl   rj   rn   r   rc   r   r   r   r    r"   r    r   r      su    *"$" BF27\J6p%N.!F2> /.!F: <A %$r"   r   )r,   r@   re   r   timer   r'   r   r   r   r   r   r   r   r   rw   r   r   r   r   r	   r   r
   requests.utilsr   r   r   r"   r    <module>r      sA     	 
             "R Rr"   