
    V`                       h    d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlm	Z	  G d d          Z
dS )    N)Krikt)Natlic                   V    e Zd Zd ZddZddZddZdd	Zd
 ZddZ	ddZ
d Zd ZdS )Tofdec                 >   || _         t          | j                   | _        t          | j                   | _        | j                                        | _        | j        t          d          | j        j        | _        | j         j	        d         | _
        i | _        dS )zdetermine frases in S kritNz$tofde can't run without a nafli_fufitofde)er   kriktr   natlifufi
natli_fufi	Exceptionsigdirsfolderd)selferimps     !/home/ernad/ernad/python/tofde.py__init__zTofde.__init__   sz    46]]
46]]
*//++?"BCCC:> fk'*t    
new_to_oldc                     | j         dz   }t          j        |          }|dk    rt          j        |           |S t	          |d          }|S )z/-> list of tofde files, excludes reports tofdesz
/*.json.gzrandomT)reverse)r   globr   shufflesorted)r   orderglob_string	out_fufiss       r   r!   zTofde.out_fufis    sV    kL0Ik**	HN9%%%9d333	r   TFc                    | j                                         rt          d           d}| j        }| j        j        }| d| d| d}t          j                            |          r|S |dz   }| j         j	        
                    ||          }|s|S | j                            |          }	|                     |	          }
t          j                            |          r|S |
|
du st          d	|z              |S )
z&-> get the tofde fufi for an issuedatez#I set do_update to false on testingF/_.json.gzz
_*.json.gz)searchNztofde return on outdated )r	   
is_testingprintr   r   r   ospathisfiler   last_by_mtimer
   r   update_for_krikt_fufi)r   	issuedate	do_update
do_verbose	tofde_dirr   r   r&   	last_fufiin_fufiouts              r   r   z
Tofde.fufi*   s   6 	7888IK	jn77i77#7777>>$ 	K\)FH**9V*DD	 	*//),,((117>>$ 	Kse||-	9:::r   Nc                    | j                             d          }|rt          d| j        j        z              |D ]K}t
          j                            |          }|dd         }|||k    r4|                     ||           LdS )z -> makes update from krikt filesr   )r   ztofde uses r   
   Nr0   )	r
   r!   r(   r   r   r)   r*   basenamer-   )r   r0   only_issuedatekrikt_fufisr   banar.   s          r   updatezTofde.updateC   s    j***:: 	2-$*.0111 	D 	DD7##D))DQrT
I)..&&t
&CCCC	D 	Dr   c                 6   t          j        |          }| j        j        }|It          j                            |          }d| j        z   dz   }|dz   |z   }|rt          d|z              nQ| j	        
                    d          r| j	        dz   |z   dz   | j        z   dz   }n| j	        dz   |z   dz   | j        z   dz   }t          j        ||g          s|rt          d|z              |S t          j        |d	           |rt          d
|z              t          | j                  dk    r| j                                        | _        t          j        |          }i | _        |D ] }|                     ||         |d           !t          | j                  dk    r t          d|z   t*          j                   d S t          j        || j                   t          d|z              t          j        |           t          j        |           i | _        |S )Nztofde.r%   r#   ztofde: seed case to z/classr$   z.tofde.json.gzztofde doneres i  ztofde loads r   Fr7   ztofdes got nothing from )fileztofde wrote )filerget_issuedater	   r0   r)   r*   dirnamer   r(   r   endswithdonerelockerplacelenr   r   getloadf
feed_papidsysstderrdumpremoveclear_by_issuedate)	r   r   r0   r.   fudir;   out_fufir   papids	            r   r-   zTofde.update_for_krikt_fufiQ   s?   '--	V&
7??4((Ddh&3DczD(H 9,x7888 {##H-- *;,y83>H/0  ;,y83>H)*|Htf-- 	 3&1222OXt$$$ 	).4'((( tv;;!Z^^%%DFJt 	? 	?EOOAeHeO>>>>tv;;!,t3z# # # #4
8TV$$$nx'(((h ***r   c                    t          j        |          }t          j        |          }|                    d          }|dz   |d         z   dz   |d         z   }t	          j        |          D ]}||k    r	t          d|z              dS )z:clear old versions of the out file, after new one is saved.r#   r   z.*.   zI need to remove N)r)   r8   rA   splitr   r(   )r   r   r;   rP   partsr    
found_fufis          r   	clear_oldzTofde.clear_old   s    {4  z$

3Sj58+e3eAh>)K00 	4 	4JT!!%
23333	4 	4r   c                     i }d|vrd S |d         D ]D}|                      ||          }|D ](}||vr||         ||<   ||xx         ||         z  cc<   )E|| j        |<   dS )NSr7   T)scanrI   )r   r
   rR   r0   foundpartfrasesfrases           r   rJ   zTofde.feed_papid   s    e4#J 	. 	.DYYt
Y;;F . .%%#)%=E%Leu-	.
 utr   c                    | j                             |          }|                                }t          |          }i }| j        }d}|rt          d|z   dz              ||dz
  k     r|dz  }||         }||dz            }	||vr|rt          d|z   dz              >|	||         vr|rt          d|z   dz   |	z   dz              fd| _        |d	z   |	z   }
|rt          d
|
z              ||         |	         }|rt          dt          |          z              t          d |D                       rP|
|vrd||
<   n||
xx         dz  cc<   t          |          dk    r#|rt          dt          |          z              !| 	                    |          }|dz   }|r.t          dt          |          z              t          d           | 
                    |||d                    }|rt          d|z   dz              |D ]?}|||v r6|
d	z   |z   }||vrd||<   n||xx         dz  cc<   |rt          d|z              @||dz
  k     |S )Nr$   rU      z
no first ''z' no 'r    zbase:zsticks c              3      K   | ]}|d u V  	d S )N ).0xs     r   	<genexpr>zTofde.scan.<locals>.<genexpr>   s&      --19------r   zpass at zlongest_ahead 	remainingzremaining 'z	I append )r
   process_fieldrV   rF   r   r(   extentstranylongest_aheadfrase_ahead)r   r^   r0   termsmax_lenr_   r   count_startfirstseconbasesticksrp   ahead_startrq   stickto_adds                    r   r\   z
Tofde.scan   s   z''--

e**F 	$#*s"###GaK''1K+&E+/*EA~~ 6,.4555AeH$$ @#+0583>???DK3;&D &gn%%%uXe_F /i#f++-...--f----- 	v%%#$F4LL4LLLA%LLLv;;!##! 8j3v;;6777 ..v66M%/K #&]););;<<<k"""**=%:MNNK 9mk1C7888 
4 
4=K''!CZ%/FV++)*vv!+! 4kF2333a GaK''b r   c                     d}|D ];}||                                 }|t          |          k     rt          |          }<|S )z measures the longest frase aheadr   )rV   rF   )r   r_   rp   r`   wordss        r   rp   zTofde.longest_ahead   sN     	+ 	+E}KKMMEs5zz)) #E

r   c                 ~    t          |          }||z
  }|dk    r||z  }d                    |d|                   }|S )Nr   re   )rF   join)r   aheadr}   	len_wordsdiffr`   s         r   rq   zTofde.frase_ahead   sG    JJ	y !88TMEqw((r   )r   )TF)FN)F)__name__
__module____qualname__r   r!   r   r<   r-   rY   rJ   r\   rp   rq   rg   r   r   r   r      s                 2D D D D- - - -^	4 	4 	4   : : : :z	 	 	    r   r   )r   r)   rK   r   r?   rD   r
   r   r   r   r   rg   r   r   <module>r      s     				 



               \ \ \ \ \ \ \ \ \ \r   