
    ),iJ                         S SK r S SKrS SKrS SKrS SKJr  S SKJr  S SKJ	r	  S SK
Jr  S SKJr  S SKJr  S SKJr  S S	KJr  S S
KJr  S SKJr  S SKJr  S SKJr  S SKJr  S SKJr   " S S5      r g)    N)Caler)Fitli)Forpi)Learn)Ligro)Mocla)Plumi)Runer)Recon)Sigsi)Spigr)Usipi)Trumi)Vemlic                   X    \ rS rSrSS jrS r  SS jr  SS jrS rS r	S r
S	 rS
rg)Train   c                    Xl         [        U5      U l        [        U5      U l        [        U5      U l        [        U5      U l        [        U R                  5      U l
        [        U5      U l        [        U5      U l        [        U5      U l        [#        U5      U l        ['        U5      U l        [+        XS9U l        [/        U5      U l        [3        U5      U l        [7        U5      U l        U R                  R;                  5       U l        SU l        SU l         S U l!        S U l"        S U l#        S U l$        S U l%        / U l&        X l'        SU l(        g )N
do_verboseFd   n))er   calerr   fitlir   learnr	   plumir   forpir   moclar   vemlir
   runerr   ligror   reconr   spigrr   sigsir   trumir   usipi	train_sigsig	has_errormodel_min_size
train_fufi
train_file
model_fufireportrepcodeseen_papidsr   
is_derived)selferimpr   s      !/home/ernad/ernad/python/train.py__init__Train.__init__   s
   5\
5\
5\
5\
4::&
5\
5\
5\
5\
5\
58
5\
5\
5\

 ::'')!$      c                    U R                   nXR                  ;  a  [        SU-   [        R                  S9  gUR                  U   nX0l        Xl        U R                  R                  XR                  S9nUS   U l	        US   U l
        g)z9probably much of this is useless, used in spigr and otoclztrain does not see the repcode fileF)r)   trainmodelN)r   r/   printsysstderrr0   r   	get_fufisr)   r,   r.   )r3   r0   r   r/   fufiss        r5   setupTrain.setupC   sz    FF(("3g=zz#'" 

$$W(($;..r8   c                    U R                   nU R                  U5        U R                  nU R                   R                  U   R                  5       n[	        U5      nUS:  a  U(       a  [        SU SU S35        gUR                  U   R                  S   n	UR                  R                  U	5      n
UR                  S   nUR                  R                  USS	9nUb  [        R                  " US
SS9  Uc  g[        R                  R                  U5      U R                  :  a%  [        SU 35        [        R                   " U5        gU
c!  U R"                  (       a  [        SU S35        g[        R$                  " X/5      nU(       a'  U(       d   U R"                  (       a  [        SU 35        gU R&                  R)                  U5      nXR*                  :X  a  U R"                  (       a  [        S5        gg)zdo we need to train?r   train: z has z issues waiting, no trainFsentr   z*.model)search.modelr   Tz train: I delete the empty model ztrain sees no sent file for z, no trainingz,train skips training on newer than sent for z0train skips train building because sig is latest)r   rC   r/   waitinglenr>   dirsd	very_lastlast_by_mtimefilerclear_by_extensionospathgetsizer+   remover   donerer%   readr)   )r3   r0   skip_on_waitingskip_on_stalledno_feature_trainr   r/   waiting_issuesrJ   sent_dirlast_sent_file	mocla_dir
last_model
newer_sent	model_sigs                  r5   rV   Train.donereV   s    FF

7 w/779n%Q;?GG9E'2KLM 88G$))&1x0KK(	SS&&y&C
!$$ZeL77??:&)<)<<4ZLABIIj!!4WI]KL\\*.>?
 JDWINOJJOOJ/	 HIr8   c                    USL a  U R                   R                  U5      nU R                  nU(       + nU(       a  [        S5        U R	                  U5        U R
                  R                  U5      nU R                  nU R                  n	[        R                  " U5        U(       a:  [        R                  R                  U5      (       a  [        R                  " U5        [        R                  R                  U5      (       Ga0  [        R                  " U5      (       a}  [        SU-   5        [        R                  " U5        [        R                  R                  U R                  5      (       a*  [        SU	-   S-   U-   5        [        R                  " U	5        OU(       a  [        SU 35        U R!                  U5      (       a  U(       a  [        SU 35        O [        S5        ["        R$                  " 5         U R'                  UUS	9n
U
(       d   U R                  (       a  [        S
U 35        gU R                  (       a  [        SU 35        [)        US5      U l        U R*                  nU R
                  R-                  U5      n[/        U5      S:X  a  SU R0                  R2                  ;  a'  U R4                  R7                  U5        [        S5        gU R8                  R;                  XS9nUb  U R*                  R=                  U5        U R*                  R?                  5         [        SU 35        U RA                  5         U RC                  5         gU RD                  RG                  U5      S   nUnU RH                  RK                  X0 XOS9n[/        U5      n[        SU S35        U R0                  RL                  U   nU R0                  RN                  RQ                  U RL                  RR                  S   5      n[/        U5      S:X  a&  [        S5        U R4                  R7                  U5        U R
                  R-                  U5      nU RT                  RK                  XU5      nU RA                  5         U RC                  5         U RD                  RG                  U5      S   U-   nU RD                  RG                  U5      S   nSn[W        U R0                  RN                  RQ                  URR                  S   5      5      S   nU(       d(  U RX                  R[                  UUU5      (       d  US-  ngUR]                  SS5      nUR]                  SS 5      n	[)        US5      nU RX                  R_                  SU5      nUR=                  U5        UR?                  5         [        S!U 35        U S"3nU R`                  Rc                  U	U5        UR]                  SS#5      n[        R                  R                  U5      (       d  [        S$U S%3["        Rd                  S&9  U RX                  R_                  UU5      n[)        U R                  S'5      U l        U R*                  R=                  U5        U R*                  R?                  5         U RC                  5         gU RX                  Rg                  UUSUUS(9nUS:X  a  [        S)5        g[        R                  " U5        [        R                  " U5        [)        U R                  S'5      U l        U R*                  R=                  U5        U R*                  R?                  5         [        S*5        U RC                  5         g)+z=--> build the train file, only done if there is no train fileFztrain: start buildztrain deletes the empty ztrain deletes z
 based on ztrain checks train file ztrain: I checked ztrain ends without a train_fufi)rZ   ztrain: I don't renew ztrain: start build wr   is_seedablez)no plumis, train created forpi and stops.)no_pillsNztrain wrote Tpos)rf   	cut_pillsztrain added z seeds and pillsrG      ztrain runs forpi.create()neg z$ligro report no grond left, finishedz.trainz.classrI   ztrain wrote class file _lemmaz.sideztrain has no z, will use complete grondr:   a)cut_poscut_negztrain: no grond vemstztrain runs main svm)4r#   is_it_nopillsr   r>   rC   r   is_report_maturer,   r.   rP   preparerR   rS   isfilerU   is_emptycheck_train_fufir?   exitrV   openr-   train_dafusrK   r   confr   creater$   	get_vemstwriteclosefinish_buildrun_svmr   totalsr'   write_trainr/   rM   lastsrL   r&   listr"   do_i_have_grond_leftreplace
test_vemstr   predictr@   grond_vemst)r3   r0   allow_feature_trainingdo_testrf   r   rZ   is_it_maturer,   r.   need_newr-   plumi_dafusspigr_vemstrg   rh   r1   count_sidpilsr/   sent_issuedatesrj   msgfirst_issuedate
class_fufi
class_filer   logger	side_fufis                               r5   
build_fileTrain.build_file   s    uzz//8H__
55&'

7zz227;__
__

 	j!rww~~j11IIj!77>>*%%~~j))0:=>		*%77>>$//22 *Z7,F$% &IIj)4ZLAB((44! 1*>?;<HHJ;;w8H ' J 5j\BC ??'
|45z3/__
 jj,,W5
 {q DFFKK/

!!'*AB**..t.GK&%%k2OO!!#L-.LLN jj(/	jj,,Z"6> - UK(]O+;<=w'&&((..)9)9&)AB1$-.JJg&jj,,W5jj,,Z-8: jj(/-?jj(/" tvvxx~~fkk&.ABCAF::227O368 8==''(;
''(;
*c*
 jj++D+>%'
|459F#

:v.&&x9	ww~~i((M),EFzz# **//MK"4??C8DOOO!!+.OO!!#LLN jj,,[-658# - G ")*
		*
		)t4k*#$r8   c                     U R                   (       a  [        SU R                  -   5        U R                  R	                  5         g)Nztrain finishes building T)r   r>   r,   r-   r}   )r3   s    r5   r~   Train.finish_build:  s.    ??,t>?r8   c                    SnSU R                   R                  ;   a/  XR                   R                  S   S-   -  nXR                  S-   -  nXR                  -  nSU R                  -   S-   U R
                  -   nU R                  (       a  [        US-   U-   5        U R                  R                  XSS9nUc  [        SU R                  -   S	-   5        g U R                  (       a  [        S
5        [        R                  " U R                  U R                  S9(       aI  [        SU R                  -   S-   [        R                  S9  [        R                   " U R                  5        g g)Nz'/home/ernad/opt/usr/bin/svm-train -b 1 svm_train_flags train__T)do_skiprF   z
 is lockedzsvm-train done)min_sizeztrain finds that the model z is empty. It deletes it.r:   )r   ry   r,   r.   r0   r)   r   r>   r!   run_with_lockrP   rt   r+   r?   r@   rR   rU   )r3   runnerr   outs       r5   r   Train.run_svmg  s   :+ffkk"34s::Foo++F//!DLL(3.9??&3,'(jj&&vt&D;)doo-<=??"#>>$//D4G4GH/$//A-.47JJ@IIdoo&r8   c                 Z   [         R                  " U5      nSnSnU HY  nUR                  S5      (       a  US-  nM   UR                  S5      (       a  US-  nM=  [        SU S3[        R
                  S9    g   US:X  ae  [        S	U S
3[        R
                  S9  [        SU 35        [        R                  R                  U5      (       a  [        R                  " U5        gUS:X  aA  [        S	U S3[        R
                  S9  [        SU 35        [        R                  " U5        gg)Nr   z-1ri   z+1ztrain: bad train line ''r:   Fztrain: bad z without positive.ztrain removes z without negative.T)
rP   
read_lines
startswithr>   r?   r@   rR   rS   rs   rU   )r3   fufilines
count_plus
count_minulines         r5   ru   Train.check_train_fufi}  s     &

Dt$$a
t$$a
+D63zz#  ?Kv%78zz# N4&)*ww~~d##		$ ?Kv%78zz#
 N4&)*IIdO r8   c                    U R                   R                  U   nUR                  S   S-   n[        R                  " U5      n[	        U5      S:  a   [        SU 3[        R                  S9  US   $ [	        U5      S:X  a  g US   $ )Nr   z/*.trainri   z$train: more than one train file for r:   r   )r   r/   rL   globrK   r>   r?   r@   )r3   r0   r/   
train_globrB   s        r5   	last_fufiTrain.last_fufi  sz    w'[[)J6
		*%u:>8	Bzz#9u:?Qxr8   )r   r   r   r   r   r*   r2   r   r"   r   r.   r+   r   r#   r0   r/   r!   r1   r)   r%   r$   r-   r,   r&   r'   r    N)F)FTT)FFF)__name__
__module____qualname____firstlineno__r6   rC   rV   r   r~   r   ru   r   __static_attributes__ r8   r5   r   r      sB    %N)& /46:1f IN!obZ,'V
r8   r   )!r   rR   r?   rP   r   r   r   r   r   r   r   r   r"   r   r   r   r   r	   r!   r
   r#   r   r%   r   r$   r   r'   r   r&   r   r    r   r   r   r8   r5   <module>r      sG     	 
               x xr8   