
    f-I                         d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ  G d
 d          ZdS )    N)Alfit)Asemb)Krikt)Ranfi)Runer)Tofde)Vemli)unquotec                   x    e Zd ZddZddZd Zd Zd Z	 	 dd	Zdd
Z	ddZ
d Zd Zd ZddZd Zd Zd ZdS )MoclaFc                    || _         t          | j                   | _        t          | j                   | _        t          | j                   | _        t          | j                   | _        | j        	                                | _	        i | _
        | j         j        d         | _        t          |d          r| j         j        | _        n|| _        d| _        d S )Nclass
do_verboseT)er   asembr   kriktr   ranfir	   vemlisigs_availableall_sigs_useddirsfolderhasattrr   can_i_model)selferimpr   s      !/home/ernad/ernad/python/mocla.py__init__zMocla.__init__   s    46]]
46]]
46]]
46]]
"j7799fk'*5,'' 	)"f/DOO(DOt    c                     | j         }|j        D ]}|                     |           |rt          | j                   | j        S )N)r   liveadd_all_sig_usedprintr   )r   r   r   repcodes       r   scan_sigs_usedzMocla.scan_sigs_used+   sS    Fv 	+ 	+G!!'**** 	&$$%%%!!r   c                 J   | j         }|j        |         j        d         }|dz   dz   |z   }t          j        |          }t	          |          }|dk    r+|                                st          d| d|            dS |dk    rt          d	| d
| d|            |d         S )z!what should be 'model' or 'train'mocla/z*.r   zmocla sees no z
 file for N   mocla sees  z files for )r   reportr   globlen
is_testingr#   )r   r$   whatr   	mocla_dirglob_stringfufiscount_fufiss           r   findz
Mocla.find3   s    FHW%*73	#o,t3	+&&%jj!<<>> B@t@@w@@AAA4??HHHdHHwHHIII Qxr   c                     |                      |d          }|dS t          j                            |          }|dd         }|| j        vr|g| j        |<   dS dS )z)this is for all sigs used, by any repcodemodelNr   iT)r5   ospathbasenamer   )r   r$   fufibanasigs        r   r"   zMocla.add_all_sig_usedE   sh    yy'**<4w%%1R4jd((('+fDs#4tr   c                 0   | j         }|j        |         j        d         }|j                            |d          }|dS || _        |j                            |d          }t          j                            |          	                    dd          }|S )z3get model for a report, should not require a searchr'   z*.model)searchNz.model )
r   r,   r   dlast_by_mtime
model_fufir8   r9   r:   replace)r   r$   r   r1   
last_modellast_model_fufir=   s          r   last_sigzMocla.last_sigS   s    FHW%*73	S&&y&CC
4$#++Ii+HHg//77"EE
r   TNc                    d| _         d| j        j        v r| j        j        |         j        d         }t          t          j        |dz                       }t          j        	                    |          r|dk    r5d| _         t          d           | j                            |||          }	|	S |rk| j                            ||	          }	t          j                            |	          r0t          d
|	z   dz              |r|                     |	||           |	S |                     |          }
| j        j        |         j        d         }|9| j        rt          d           |                     |          }|dz   }d|z   dz   }n| j        rt          d|z              |                     ||
||          }|&t          d|z   dz   |z   t(          j                   d S |dz   |z   dz   }d|z   dz   |z   }| j        rt          d|z              t-          j        |           t-          j        ||| j        g          s|                     ||          sVd|z   dz   }|| j        dz   |z   z  }t7          | j                  }t          d|z              |                    ||           nt          d|z              t          j                            |          s| j         rt          d|z              d S t          d |z              |                     |          | _        |                     || j                  }|8| j         r1t          d!t(          j                   t)          j         d"           d S t          d#|z              |r8| j        !                    |||          }	|	|S |                     |	||           |S )$NTno_seedssentz/*r   Fzmocla: I can't model)r$   	issuedatedoc_fufi)r$   rK   r*   z, no donere, assemble finished.r'   zmocla has no issuedatez/grund.predipredict__grundzmocla has issuedate r   zmocla has no vemte for z at filer(   .predi_zclass fufi is z/usr/bin/svm-predict -b 1 r+   r   zmocla runs mocla doneres zmocla: I don't see zmocla: predi is z&mocla can't continue with no side filer)   zmocla: side is )"r   r   confr,   r   r.   r-   r8   r9   isdirr#   r   build_unsortedr;   isfilecache_selection_screenrG   r   grund_vemte_fufidigsysstderrfilerpreparedonererC   compare_with_predir   run_with_lock
papid_file
papid_fufi	side_fileexitbuild)r   r$   rK   r   do_asembrL   do_cachesent_dir
total_sent
asemb_fufir=   r1   
vemte_fufi
predi_fufiloggerrunnerruner	side_fufis                     r   runz	Mocla.run_   s;   $$v}W-26:HTYx$7788JGMM(++ "
a#( ,---!Z66wAJ@H 7 J J
 "!  	"INNJw~~j)) "mj078 9 9 9  P//
GYOOO!!mmG$$FM'*/8	 0.///..w77J"^3J')H4FF  :,y8999)S'-7 " 9 9J!/'9FBYN:' ' ' ' t"S94x?J')C/);F 5&3444j!!!<
Z$ABB 	1&&z:>>	11J>DFdo+j88F'''E-&()))////"Z/000w~~j)) 	d.> 	'*45554 :-...//*55NN:t??	!1:z# # # #HQKKK4)+,,, 	H))'Y3; * = =J!  ''
GYGGGr   c                     |t          dt          j                   dS | j        dz   |z   dz   |z   dz   }t          j                            |          r|S | j        dz   |z   dz   |z   dz   | j        j        z   }t          j                            |          r| 	                    |          }|S | 
                    |          }|At          d|z   d	z              t          j        ||           | 	                    |          }|S | j                            |
          }|t          d|z              dS d| }	t          |	           |                     ||          }
|
t          d|z              dS |rt          d|
z              || j        _        | j        | j        _        || j        _        t          d|
z              | j                            |
|          }|dS | 	                    |          }|S )zB find get material to work on a sig, if needed with only the kriktNzmocla has no model, run train.rP   r(   rS   z.txtz.vemlizmocla uses remote vemlis .)r=   z+mocla can't continue to dig, no vemlis for z$mocla needs to call get_alfit() for rO   zmocla sees no alfit for zmocla needs to make vemlis for z mocla needs to build the vemlis )r#   r]   r^   r   r8   r9   rY   r   extvemte_from_vemliget_from_trainshutilcopy2r   get	get_alfitr=   r
build_fufi)r   rK   r=   r$   r   rn   
vemli_fufigot_from_trainr   note
alfit_fufis              r   r\   z	Mocla.dig   s+    ;2DDDD4 [3&2S83>G
7>>*%% 	[3&2S83>Ijn
7>>*%% 	..z::J,,Z88%->DEEEL444..z::J
3''=?#EFFF4 @g??d^^I*^EE
,y89994 	B3j@AAA
 K
 
0:=>>>Z**:**MM
4**:66
r   c                 ~   | j         dz   |z   dz   }t          j                            |          r|S |                     |           t          j                            |          r|rt          d| d           |S |rt          d|z   dz              | j         dz   |z   dz   }t          j                            |          s|                     |           t          j                            |          sNt          d|z              | j                            |	           t          j                            |          sd
S |rt          d| d           t          | j	                  | _
        | j         | j
        _         | j
                            ||          }| t          d|z   t          j                   d
S |rt          d|            t          | j	                  | _        | j         | j        _         | j                            |          }t          j                            |          r|S d
S )z&This takes the approach from the kriktr(   z.alfit.json.gzz
mocla got z from training.zmocla needs to get alfits z from krikts.z.krikt.json.gzzmocla has no krikt for )do_only_issuedateNzmocla found the krikts rv   rT   rO   zmocla can't get the tofde for rP   zmocla found the tofde )r   r8   r9   rY   ry   r#   r   renewr   r   tofdeupdate_for_krikt_fufir]   r^   r   alfit)r   rK   r   r   
krikt_fufi
tofde_fufis         r   r}   zMocla.get_alfit   sR   [3&25EE
7>>*%% 	J'''7>>*%% 	 @>:>>>??? 	O.;oMNNN[3&25EE
w~~j)) 	 
+++7>>*--  /);<<<
  9 ===w~~j11  4 	;9J999:::(((
 K
Z55jAK 6 M M
2Z?z# # # #4 	97:77888(((
 K
Z55jAA
7>>*%% 	tr   c                     |}|dd         }|                     d|z   d          }|                     dd|z   dz             }t          j                            |          rt	          j        ||           |S d S )Niirv   r@   /class/r(   )rD   r8   r9   rY   rz   r{   )r   
local_fufir;   kinds       r   ry   zMocla.get_from_train  sy    CF|||C$J++||IsTzC'7887>>$ 	Lz***Ktr   c                 r   |                     d          r|                    dd          }n>t          d|z              |                    dd          }|                    dd          }t          j                            |          r(t          j        |          st          d|z              |S t          |d          }t          j	        |          }|D ]#}|
                    d	||         z   d
z              $|                                 t          d|z              |                     |           |S )Nz.vemli.json.gzz.classz!mocla: class case, vemli_fufi is z.json.gzz/vemli/r   mocla does not renew w0 
mocla wrote )endswithrD   r#   r8   r9   rY   r_   is_emptyopenloadwritecloserd   )r   r   
class_fufi
class_file
vemli_datapapids         r   rx   zMocla.vemte_from_vemli$  sC   /00 	B#++,<hGGJJ 5
BCCC#++JAAJ#++IyAAJ7>>*%% 	enZ.H.H 	)J6777*c**
Z
++
 	> 	>ETJu$55<====nz)***
###r   c                 F   t           j                            |          }t           j                            |          }|dd         }|dz   |z   dz   }t	          j        ||g          st          d|z              |S t          |d          }t          |d          }	 |                                }n_# t          $ rR}	t          t          |	                     t          d|z   t          j        	           t                       Y d
}	~	nd
}	~	ww xY w|D ]e}
d|
vrt          d|
z              |
                    d          }|d         }t          |d
d                   dz   }|                    |           f|                                 t          d|z              |S )z*generated from the vemte_fufi for an issuer   
   r(   z.papidr   r   r~   zmocla error reading rP   N#zskip z # r)   r   r   )r8   r9   dirnamer:   r_   ra   r#   r   	readlinesUnicodeDecodeErrorstrr]   r^   quitsplitr
   r   r   )r   rn   fudir<   rK   re   rd   r   class_linesr   
class_linepartsr   s                r   rd   zMocla.papid_file;  s   wz**w
++2J	CZ)+h6
|J55 	)J6777*c**
*c**
	$..00KK! 	 	 	#a&&MMM(:5CJGGGGFFFFFFFF	 & 	$ 	$J*$$g
*+++$$U++E !HEE#2#J''$.EU####nz)***s   B4 4
D>ADDc                    t          j        |          }t          j        |          }t          |          }t          |          }|dz
  |k    rdS |sdS t          t	          j                    t          j                   d| dt          |          z   dz   }|d| dt          |          z   d	z   z  }t          |t          j                   dS )
Nr)   TFrP   zmocla: z's z lines zdon't go with z + 1 )	r_   read_as_linesr.   r#   	traceback
format_excr]   r^   r   )	r   
input_fufiro   do_warninput_linespredi_lines	len_predi	len_inputerrs	            r   rb   zMocla.compare_with_predi^  s    )*55)*55$$	$$	q=I%%4 	5i"$$3:6666'
'''#i..89D/
///#i..@7JJc
####ur   c                    |                     dd          }t          j        |||g          st          d|z   dz              |S |                     ||d          }|sd S t          j        |          }t          j        |          }	 |d         }n,# t          $ r t          |t          j                   Y d S w xY wd}t          |d	          }	d }
|d
k    rd}
nu|dk    rd}
nl|dk    rKt          d| j
        z   t          j                   t          j        | j
                   t                       nt          d|z   dz   dz   |z             |dd          }|t          |          k     r^||                             d          }||
         }|dz   ||         z   }|	                    |           |dz  }|t          |          k     ^|	                                 |S )NrR   z.sidezmocla finds that z needs no renewal.T)r   r   rP   r   zlabels 1 -1
r)   zlabels -1 1
   z	labels 1
z0mocla finds that the prediction failed, removes zmocla can't parse ''z in r+   	)rD   r_   ra   r#   rb   r   
IndexErrorr]   r^   r   rC   r8   remover   	Exceptionr.   r   r   r   )r   ro   re   rs   is_correct_sizer   papid_lines	head_linecountrf   plus_pospredi_partsprobalines                 r   rf   zMocla.side_filem  sI   &&x99	|I
J'?@@ 	%	14HHIII11*j:> 2 @ @ 	4)*55)*55	#AII 	 	 	+CJ////44	 C((	''HH/))HH,&&D/"(+
4 4 4 4Ido&&&FFFF1I=C"#%/0 1 1 1 "!""oc+&&&&%e,22377K)E 4<+e"44D
 OOD!!!QJE c+&&&& 	s   
B %B<;B<c                    | j         }ddlm}  || j                   }|j        |         j        d         }|dz   }t          |d          }t          |j                   t          j	        |j                  }|D ]#}	|
                    d||	         z   dz              $|                                 t          d|z              |S )	Nr   )Grundr'   z/grund.classr   r   r   r   )r   grundr   r,   r   r   r#   r   r_   r   r   r   )
r   r$   r   r   r   r1   r   r   vemlisr   s
             r   r[   zMocla.grund_vemte_fufi  s    FdfHW%*73	/
*c**
eE,-- 	: 	:ETF5M1D89999nz)***r   c                    | j         }i }|j        |         j        d         dz   }| d| d}t          j        ||g          st          d| d|            |S t          j        |           d|d<   d	|d
<   t          d|z              t          j        	                    |          st          j
        |           |j                            d|||           t          d|z              dS )zshould move to ursisr'   z/../../cache/createdr(   z.htmlrU   z over z'source'	back_fromz	'created'stagezmocla reads papers_screen)paramszmocla writes N)r   r,   r   r_   ra   r#   r`   r8   r9   rW   mkdirt
via_system)r   rm   r$   rK   r   r   out_dir
cache_fufis           r   rZ   zMocla.cache_selection_screen  s   F(7#(14JJ22)222
|J55 	A:AAZAABBBj!!!({%w 	nz)***
 w}}W%% 	HW	
JvNNNo
*+++++r   )F)FTNT)T)__name__
__module____qualname__r   r%   r5   r"   rG   rt   r\   r}   ry   rx   rd   rb   rf   r[   rZ    r   r   r   r      s&          "" " " "  $  
 
 
 BF$(R R R Rv7 7 7 7r% % % %N    .! ! !F   / / /b   , , , , ,r   r   )r-   r8   r]   rz   r   r_   r   r   r   r   r   r   r   r   rr   r   r   r   r   r	   requests.utilsr
   r   r   r   r   <module>r      s    				 



                                                 " " " " " "u, u, u, u, u, u, u, u, u, u,r   