Zf@ddlZddlZddlZddlZddlZddlZddlmZddlm Z ddl m Z ddl m Z GddZdS)N)Plumi)Vemli)Runer)Sidatc`eZdZddZddZdZddZdZdZdd Z dd Z d Z d Z d Z dZdS)TrainFc||_t|j|_t|j|_t |j|_t|j|_|jj |_ d|_ d|_ d|jj vr$t|jj d|_ ||_dS)NFtrain_limit_by_days)erplumirvemlirrunerrsidatsig has_error train_limitconfint do_verbose)selferimprs !/home/ernad/ernad/python/train.py__init__zTrain.__init__s46]] 46]] 46]] 46]] :> DFK / /"46;/D#EFFD $tc|j}||jvr td|ztjdS|j|}||_||_|si|_|j}|jddz|zdz|_ |jddz|zdz|_ |jddz|zdz|_ |jddz|zdz}tj |dS) Nztrain does not see the repcode fileFmocla/.train.modelz.sutra.json.gz)r reportprintsysstderrrepcodesutrardirs train_fufi model_fufi sutra_fufifilerprepare)rr& grund_boostr r"rr)s rsetupz Train.setup s F !( " " 3g=z # # # #5'"   DJh +g.4s:XE +g.4s:XE +g.4s:=MM[)C/#5@  j!!!!!rc|j}|||j}|j|jd}|j|}|dS|jd}|j|d}|dStj |dkrdStj ||gS) z4no renew if *any* model is newer than last sent filesentNFrz*.model)searchTd) r r/r"r(d very_last last_by_mtimeospathgetsizer,donere)rr&r r"sent_dirlast_sent_file mocla_dir last_models rr:z Train.donere5s F 78G$)&1x00  !5K( S&&y&CC  4 7??: & & , ,4|J(8999rNc8 |j}|rtd|||j}|j}i|_g|_tj|tj |rtj |rwtd|ztj |tj |jr,td|zdz|ztj |ndSt|d|_|j|}d}d|_|D]}||_i|j|<d|j|d<d|j|d <|jtt+j||jkrWd |_d |j|d<d |j|d <|std |zd }|jrtd |z|||_||d|jjvs|jr.td|jd|d Si|jd<d|jdd<d|jdd <|j|} || |j |} tj | r|j!"| } | td| zntj#| } | D]Z} |$| d|j%d| | zdz|jddxxdz cc<[ntd| z||&|tO|jjdtQ|jz }|dkrMtdtS|zdz|*|}|&|nK|jj+|j,ddz}t[j-|D]}tj ||d S)z=--> build the train file, only done if there is no train filezmocla: start buildtrain deletes the empty train deletes z based on Fwr+1-1NTztrain skips plumis older than ztrain skips plumis from is_seedabletrain: z is too old for seeds 0000-00-00z%train does not have sidat vemlis, no seednote+1  z(train does not see external seed krikts grund_sizezI add z from grund.rz/grund*).rr#r/r)r*r' seen_papidsr,r-r7r8isfileis_emptyremoveopen train_filer fufisam_i_older_than_train_limit issuedaterdaterago plumi_fufi feed_issuer rr& finish_buildr sidis_fufi add_sidis krikt_fufir build_fufiloadcheck_seen_papidswrite feed_grundrlenstrfind_grund_to_addr"r(glob)rr& grund_papidsrr)r* plumi_dafushave_i_printed_about_skiprWr]krikt_seed_fufiseed_vemli_fufi vemli_datapapid grund_gap grund_globfufis r build_filezTrain.build_fileHs_  ( & ' ' ' 7_ _   j!!! 7>>* % % ~j)) 0:=>>> *%%%7>>$/22**Z7,F$%&&&Ij)))uz3//j&&w// $)!+0($ ' 'I&DN$&DJy !*+DJy !$ '*+DJy !$ '+y##d&666370.2 9%d+.2 9%d+05:YFGGG04-?B4y@AAA))4DO OOI & & & &  + +  + , ?DL??? @ @ @      4#% < )* < &)* < & Z**733  z""" *//88 7>>/ * * P"j33ODDO&=OPPPP#Z88 '88E**5v*>>>O))%*U2C*Cd*JKKKJ|,T222a722228 <N O O O  # OOL ) ) )  L122S9I5J5JJ q== (S^^+n< = = =11)< ? ? ? trctj|s#ddl}|d|jzdt j|}|j}|D]}|j |||}||}|~td|j |j|j |||}|-d|d|}|d |fz }t|tj ||| |jd |zd z|jddxxdz cc<dS)zThis adds the interla seedrNzsidat T)shellz+train got a None sidis vemli, try to updatez train for z no z for sidis rrIrKrLrGrCrM)r7r8rP subprocessrunr&r,rar grepr#r update_reportr$r%rbrTrcr') rr]rypapisur&ror rWerrors rr^zTrain.add_sidissyw~~j)) @     NN8dl2$N ? ? ?J'', 0 0EJOOF5M599Eu I}CDDD ((666 u u======e==E5)5566E%cj1111  " "5y " 9 9 9 O ! !%%-$"6 7 7 7 J| $T * * *a / * * * *trc8||jvrv|*td|zdztjd|_dS|dkrtd|zdzdStd|zdz|ztjd|_dS|j|dS) N train sees zagain.rTFgrund in the grund.z in )rOr#r$r%rappend)rrorJs rrbzTrain.check_seen_papidss D$ $ $|me+h6SZHHHH!%uwme+.>>???ume+f4t;:''''!%u &&&trc||||j}tj|rRt j|r>td|zdztj|| |t j |jdtj|sk|j rtd|z| |tj|std|ztj |j rtd|zt j|jr>td |jztj tj|jd S|j}t j||gstd |d |dd S|jrCtdtd|ztj|t#d}d|jjvr ||jjddzz }||dzz }||z }d|zdz|jz}|j rt|dz|z|j||d}|td|zdzd S|j rtdt j|jdrAtd|jzdztj tj|jd St j |ddd S) z;--> run training, build train file only if no training filer.r@.r z train builds ztrain could not build the rz train uses ztrain has no data in Nztrain doneres z over Fzmocla: end because of errorzmocla: I delete z/usr/bin/svm-train -b 1 svm_train_flags train__T)do_skiprFz is lockedzsvm-train doneX)min_sizeztrain finds that the model z is empty. It deletes it.r!)r)r/r)r7r8rPr,rQr#rRrsclear_by_extensionrr$r%r*r:rquitr rrr run_with_lock)rr&r.r)r*runnerloggerouts rrzz Train.runsa 7 444_ 7>>* % % %%.*D*D % ,z9C? @ @ @ Ij ! ! ! OOG $ $ $  (;;;w~~j)) ' 4o 2333 OOG $ $ $7>>*-- '2Z?:'''' ? . -*, - - - >$/ * *  )DO;z # # # # Ido & & &4_ |J 55  B:BBZBBB C C C5 >  / 0 0 0 $z1 2 2 2 Ij ! ! ! FFF+  + + dfk"34s: :F j3& &F*G#c)DH4 ? ) &3,' ( ( (j&&vvt&DD ; )j(<7 8 8 84 ? $ " # # # >$/B 7 7 7  /$/A-.47J @ @ @ @ Ido & & &4  X$GGGGGGrc||j}|jrtd|zd|jzdz}|jddz|z|z}t j|s}td|zdztj td |j zd zt j |j td tj ttj|jd }|}t#j||_d |_|D]l}||dd|||m|j|jdS)zfeeds issue into train fileztrain: build_file adds rz.json.gzr rztrain does not see z' this should have done by post_process.rrArz train ends.rrIN)r rr#rr(r7r8rPr$r%r)rRrgziprSrZ readlinesr,ravstringrbdecodefeed_plumi_linerstriprTrc)rrWr end vemli_fufi plumi_file plumi_lineslines rr[zTrain.feed_issue)s F ? 9 +i7 8 8 8DHnz)VG_s*Y6< w~~j))  '*4;<z # # # # "T_4s: ; ; ; Ido & & & -cj 1 1 1 1 FFFYt44  **,, J''  9 9D  " "4":#4#4#6#6Y " G G G  !5!5!7!7 8 8 8 8 dk*****rcddlm}||j|_|jj}|Bt dt jtj|j t|j j j |vr'td|j j j zdz|ztj|jj}|D]*}||vr$t d|zd zd zt j+t#|j d |_|D]}||jvrt d |zd z!||ds9||vr>|jd||zdz|jddxxdz cc<dS)z'--> add only the papids in grund_papidsr)GrundNz9train can't continue without grund vemlis, try 'grund -l'rzsig inconsistency between z and ztrain: the requested z is not in the grund,z try to run grund -laztrain finds seen paper rrrIz-1 rLrGrDrMT)rrr rr#r$r%r7rRr)rrr r Exceptionr,rarSrTrOrbrcr')rrirgrund_vemli_fufi grund_vemlisros rrdzTrain.feed_grundAsU46]] :0  # Mz # # # # Ido & & & FFF :  '7 7 78"j.235<=./00 0z$*"788 ! A AEL((-58OO./58ZAAAAt44! 0 0E(((/%7:JJKKK))%g)>> L(( O ! !%,u*="="D E E E J| $T * * *a / * * * *trc|j}|dd}|dd}||vr(t|dz|jztjdS|j|j|xxdz cc<|xj|dz||zdzz c_dS) Nrrz is not in the vemli of rrMrrL)rr#rWr$r%r'r)rrrindicros rrzTrain.feed_plumi_linejs FQqS QRR >> %44t~Ez # # # # F 4>"5)))Q.))) us{QuX-44 rc|jtd||jdddlm}||j|_td|j|jdd|j }td |zdd l m }||j|_ |j | }d}t|jj d |z }g}|D]#} ||kr|dz }|| $t||krKtdt!t|zdzt!|zt"j|S)Nztrain runs for grundTrr)Mocla)rztrain runs grund classF)do_asembrr)Asemb)rrrNrMzcount problem rr)rTrvr#rzr&rrr rasembrparse_side_fufirrrrerfr$r%) rrpr side_fufirpapwescountstartriros rrgzTrain.find_grund_to_addxs  $%%% 4000U(((  &'''JNN4<.2o#??  mi'(((U((( +++;;DFK -..:  ' 'Eu}}     & & & & |   ) ) "S\):):%;%;;cA NN#z # # # #r)F)N)__name__ __module__ __qualname__rr/r:rsr\r^rbrzr[rdrrgrrrrs    """"*:::&qqqqf   0$3H3H3H3Hl+++0'''R 5 5 5rr)rhrr7r$r,rXr rr rrrrrrrrrrs BBBBBBBBBBr