#!/usr/bin/perl

use strict;
use warnings;

use Carp qw(confess);
use Data::Dumper;
use File::Basename;
use List::Util qw(shuffle);
use Storable;
use XML::LibXML;
use XML::LibXSLT;

use Ernad::Constant;
use Ernad::Common;
use Ernad::Dates;
use Ernad::Erimp;
use Ernad::Layer;
use Ernad::FullTextFetch;
use Krichel::File;

our $impna=$ARGV[0] // '';
if (not $impna) {
  print "I need an impna\n";
  exit;
}

my $what=$ARGV[1] // '';
if(not $what or ($what ne 'fields' and $what ne 'issues')) {
  print "I need a second argument 'fields' or 'issues'\n";
  exit;
}

my $limit=$ARGV[2] // 0;

our $e=Ernad::Erimp->new({'impna'=>$impna});
if(not $e) {
  print "I don't know about your impna '$impna'\n";
  exit;
}

my $amf_ns=$Ernad::Constant::c->{'amf_ns'} // die;

my $sent_dir=$e->{'dir'}->{'allport_sent'};
my $xslt_dir=$e->{'dir'}->{'xslt'};
my $style_dir=$e->{'dir'}->{'style'};
my $xslt_file=$e->{'dir'}->{'xslt'}.'/show_handles.xslt.xml';
my $archive_xslt_file=$e->{'dir'}->{'style'}.'/store_by_archive.xslt.xml';
my $issue_xslt_file=$e->{'dir'}->{'style'}.'/store_by_issue.xslt.xml';
my $archive_out_file=$e->{'dir'}->{'web_opt'}.'/store_by_archive.html';
my $issue_out_file=$e->{'dir'}->{'web_opt'}.'/store_by_issue.html';
my $fut_issues_dir=$e->{'dir'}->{'fut_issues'};
my $fut_fields_dir=$e->{'dir'}->{'fut_fields'};
my $fut_dir=$e->{'dir'}->{'fut'};
my $fut_archive_dir=$e->{'dir'}->{'fut_files'};
my $count_test=0;
my $today=&Ernad::Dates::today;

my $xslt = XML::LibXSLT->new();

my $r_nep={};
my $r_fields={};

## fills the var/fut/issues directory
if($what eq 'issues') {
  &build_files_per_issue($limit);
  $r_nep=&gather_data_from_issues();
  &create_issue_page();
  &create_archive_page();
}

if($what eq 'fields') {
  ### &build_files_per_fields($limit);
  $r_fields=gather_data_from_fields();
  #print Dumper $r_fields;
}

sub gather_data_from_issues {
  my $l_dir=length($fut_issues_dir);
  my $r={};
  foreach my $found_fufi (shuffle `/usr/bin/find $fut_issues_dir/* -type f`) {
    my $found_bana=basename($found_fufi);
    $found_bana=~m|^(\d{4}-\d{2}-\d{2})\.json$| or next;
    $r->{$1}=&Krichel::File::load($found_fufi);
  }
  return $r;
}

sub gather_data_from_fields {
  my $l_dir=length($fut_fields_dir);
  my $r={};
  ## summary
  my $sum;
  foreach my $found_fufi (`ls -S $fut_fields_dir/*`) {
    chomp $found_fufi;
    my $found_bana=basename($found_fufi);
    $found_bana=~m|^(\d{4}-\d{2}-\d{2})\.json$| or next;
    $sum=&process_field_data_to_html($found_fufi,$1,$sum);
  }
  my @sorted_sum;
  my $sum_sorted=[];
  foreach my $date (reverse sort keys %$sum) {
    my $s=$sum->{$date};
    $s->{'date'}=$date;
    push(@$sum_sorted,$s)
  }
  &field_cover_sheet($sum_sorted,'date');
  $sum_sorted=[];
  foreach my $date (sort {$sum->{$b}->{'mul'} <=> $sum->{$a}->{'mul'}} keys %$sum) {
    my $s=$sum->{$date};
    $s->{'date'}=$date;
    push(@$sum_sorted,$s)
  }
  &field_cover_sheet($sum_sorted,'multiples');
  $sum_sorted=[];
  foreach my $date (sort {$sum->{$b}->{'non'} cmp $sum->{$a}->{'non'}} keys %$sum) {
    my $s=$sum->{$date};
    $s->{'date'}=$date;
    push(@$sum_sorted,$s)
  }
  &field_cover_sheet($sum_sorted,'missing');
  return $r;
}

sub field_cover_sheet {
  my $in=shift; ## array reference
  my $criterion=shift;
  my $doc = XML::LibXML::Document->new('1.0','utf-8');
  my $fields_ele = $doc->createElement('fields');
  my $count=0;
  foreach my $date (@$in) {
    my $din=$in->[$count];
    my $date_ele = $doc->createElement('date');
    my $date=$din->{'date'};
    $date_ele->setAttribute('date',$date);
    $date_ele->setAttribute('pretty_date',&Ernad::Dates::pretty_date($date));
    $date_ele->setAttribute('one',$din->{'one'});
    $date_ele->setAttribute('non',$din->{'non'});
    $date_ele->setAttribute('mul',$din->{'mul'});
    $fields_ele->appendChild($date_ele);
    $count++;
  }
  $doc->setDocumentElement($fields_ele);
  my $html_doc=$e->{'t'}->transform($doc,'summarize_fields_in_store',
                             {'update' => "'$today'",
                              'criterion' => "'$criterion'"});
  my $crit_as_file=$criterion;
  $crit_as_file=~s| |_|g;
  my $xml_file=$e->{'dir'}->{'web'}."/field_in_store/$crit_as_file.xml";
  $doc->toFile($xml_file,1);
  my $out_file=$e->{'dir'}->{'web'}."/field_in_store/$crit_as_file.html";
  $html_doc->toFile($out_file,1);
}

##
sub process_field_data_to_html {
  my $fufi=shift // confess "I need a fufi here.";
  if(not -f $fufi) {
    confess "I can't open your file $fufi.";
  }
  ## the date
  my $date=shift;
  print "date is $date\n";
  ## summary structure, to be returned
  my $sum=shift;
  $sum->{$date}->{'one'}=0;
  $sum->{$date}->{'non'}=0;
  $sum->{$date}->{'mul'}=0;
  my $r=&Krichel::File::load($fufi);
  my $doc = XML::LibXML::Document->new('1.0','utf-8');
  my $col_ele = $doc->createElementNS($amf_ns,'collection');
  $col_ele->setAttribute('field_date',$date);
  my $pretty_date=&Ernad::Dates::pretty_date($date);
  $col_ele->setAttribute('field_pretty_date',$pretty_date);
  my $amf_ele = $doc->createElementNS($amf_ns,'amf');
  foreach my $id (keys %$r) {
    my $rid=$r->{$id};
    my $hp_ele = $doc->createElementNS($amf_ns,'haspart');
    my $text_ele = $doc->createElementNS($amf_ns,'text');
    $text_ele->setAttribute('ref',$id);
    foreach my $futli (@{$rid->{'futli'}}) {
      my $futli_ele = $doc->createElement('futli');
      $futli_ele->appendText($futli);
      $text_ele->appendChild($futli_ele);
    }
    if(ref $rid->{'fut'} eq 'ARRAY') {
      foreach my $fut (@{$rid->{'fut'}}) {
        my $fut_ele = $doc->createElement('fut');
        $fut_ele->appendText($fut);
        $text_ele->appendChild($fut_ele);
      }
      $sum->{$date}->{'mul'}++;
    }
    else {
      if(substr($rid->{'fut'},0,12) eq 'no directory') {
        $sum->{$date}->{'non'}++;
      }
      else {
        my $fut_ele = $doc->createElement('fut');
        if(substr($rid->{'fut'},0,3) eq 'no ') {
          $fut_ele->setAttribute('found','no');
        }
        $fut_ele->appendText($rid->{'fut'});
        $text_ele->appendChild($fut_ele);
      }
      $sum->{$date}->{'one'}++;
    }
    $hp_ele->appendChild($text_ele);
    $col_ele->appendChild($hp_ele);
  }
  $amf_ele->appendChild($col_ele);
  $doc->setDocumentElement($amf_ele);
  ## sort by reverse layer
  my $out_file=$e->{'dir'}->{'web'}."/field_in_store/$date.html";
  ## just for debugging
  my $xml_file=$e->{'dir'}->{'web'}."/field_in_store/$date.xml";
  #if(-f $out_file and -M $out_file < .2) {
  #  return $sum;
  #}
  $doc=&Ernad::Layer::layer($doc,$impna,'d');
  $doc->toFile($xml_file,1);
  my $html_doc=$e->{'t'}->transform($doc,'field_in_store');
  $html_doc->toFile($out_file,1);
  exit;
  return $sum;
}

sub create_issue_page {
  my $issue_doc = XML::LibXML::Document->new('1.0','utf-8');
  my $issues_element = $issue_doc->createElement('issues');
  foreach my $date (reverse sort keys %{$r_nep}) {
    my $d=$r_nep->{$date};
    my $count_total=0;
    my $count_found=0;
    foreach my $id (keys %$d) {
      my $found=$d->{$id};
      $count_total++;
      if (ref $found) {
        $count_found++;
      }
    }
    my $issue_element = $issue_doc->createElement('issue');
    my $percent=int(100*($count_found/$count_total)+.5);
    $issue_element->setAttribute('date',$date);
    my $pretty_date=$date;
    $pretty_date=~s|-|\x{2012}|g;
    $issue_element->setAttribute('pretty_date',$pretty_date);
    $issue_element->setAttribute('count_found',$count_found);
    $issue_element->setAttribute('count_total',$count_total);
    $issue_element->setAttribute('percent',$percent);
    $issues_element->appendChild($issue_element);
  }
  $issue_doc->setDocumentElement($issues_element);
  my $issue_style_doc = XML::LibXML->load_xml(location=>$issue_xslt_file,
                                              no_cdata=>1);
  my $issue_stylesheet = $xslt->parse_stylesheet($issue_style_doc);
  my $issue_results = $issue_stylesheet->transform($issue_doc);
  $issue_results->toFile($issue_out_file);
}

## print strange failures
my $strange='';
foreach my $date (sort keys %{$r_nep}) {
  my $d=$r_nep->{$date};
  print "I do $date\n";
  my $count_nodir=0;
  my $count_nofile=0;
  my $count_total=0;
  foreach my $id (keys %$d) {
    my $found=$d->{$id};
    if (ref $found) {
      my @files=@{$found};
      if (scalar @files == 1) {
        next;
      }
      $strange.="I have several files for $id\n";
      foreach my $file (@files) {
        $strange.= "$fut_dir/$file\n";
      }
      $strange.="\n";
    }
    if (substr($found,0,17) eq 'no suitable files') {
      my $dir=substr($found,21);
      $strange.="I found no suitable files in $dir\n";
      $strange.= `ls $dir`;
      $strange.="\n";
    }
  }
}

print $strange;

sub print_archive_page {
  ## print by archive
  my $count_found={};
  my $count_total={};
  foreach my $date (sort keys %{$r_nep}) {
    my $d=$r_nep->{$date};
    foreach my $id (keys %$d) {
      $id=~m|[^:]+:([^:]+):([^:]+):|;
      my $archive=$1;
      my $found=$d->{$id};
      if (not $count_found->{$archive}) {
        $count_found->{$archive}=0;
      }
      if (ref $found) {
        $count_found->{$archive}++;
      }
      $count_total->{$archive}++;
    }
  }
  my $archive_doc = XML::LibXML::Document->new('1.0','utf-8');
  my $archives_element = $archive_doc->createElement('archives');
  my @archives=keys %{$count_total};
  @archives=sort {$count_total->{$b} <=> $count_total->{$a}} @archives;
  foreach my $archive (@archives) {
    my $archive_element = $archive_doc->createElement('archive');
    my $percent=int(100*($count_found->{$archive}/$count_total->{$archive})+.5);
    $archive_element->setAttribute('id',$archive);
    $archive_element->setAttribute('count_found',$count_found->{$archive});
    $archive_element->setAttribute('count_total',$count_total->{$archive});
    $archive_element->setAttribute('percent',$percent);
    $archives_element->appendChild($archive_element);
  }
  $archive_doc->setDocumentElement($archives_element);
  my $archive_style_doc = XML::LibXML->load_xml(location=>$archive_xslt_file,
                                                no_cdata=>1);
  my $archive_stylesheet = $xslt->parse_stylesheet($archive_style_doc);
  my $archive_results = $archive_stylesheet->transform($archive_doc);
  $archive_results->toFile($archive_out_file);
}

sub build_files_per_issue {
  my $limit=shift // '';
  foreach my $file (shuffle `ls $sent_dir`) {
    my $ids='';
      chomp $file;
    $file=~m|(\d{4}-\d{2}-\d{2})| or next;
    my $r_nep={};
    my $date=$1;
    if($limit and &Ernad::Dates::diff_dates($date,$today) > $limit) {
      next;
    }
    my $dadi=&Ernad::Dates::date_to_dadi($date);
    my $data_file=$fut_issues_dir."/$dadi.json";
    &Krichel::File::prepare($data_file);
    my $fufi="$sent_dir/$file";
    $r_nep=&get_fut_from_amf_simple($fufi,$r_nep);
    &Krichel::File::save($data_file,$r_nep);
  }
}

sub build_files_per_fields {
  my $limit=shift // '';
  my $r_fields={};
  my $wa_dir=$e->{'conf'}->{'directory_to_watch'};
  if(not -d $wa_dir) {
    confess "I can't access your directory to watch $wa_dir.";
  }
  foreach my $file (shuffle `ls $wa_dir`) {
    chomp $file;
    my $ids='';
    $file=~m|(\d{4}-\d{2}-\d{2})| or next;
    my $date=$1;
    if($limit and &Ernad::Dates::diff_dates($date,$today) > $limit) {
      next;
    }
    ## skip the unfinished field
    if($date eq $today) {
      next;
    }
    my $data_file=$fut_fields_dir."/$date.json";
    &Krichel::File::prepare($data_file);
    my $fufi="$wa_dir/$file";
    $r_fields=&get_fut_from_amf($fufi);
    &Krichel::File::save($r_fields,$data_file);
  }
  return $r_fields;
}

sub get_fut_from_amf {
  my $fufi=shift // confess "I need a fufi here.";
  my $r=shift // {};
  my $doc=Krichel::File::load_xml($fufi);
  my $ids=$e->{'x'}->futlis($doc);
  foreach my $id (keys %$ids) {
    #print "id is $id\n";
    my $found=&{$Ernad::FullTextFetch::fff->{$impna}}($id,$fut_archive_dir);
    if (ref $found) {
      #my @founds=@{$found};
      #if(scalar @founds > 1) {
      #print "$id\n";
      #print Dumper $found,"\n";
      #}
    }
    $r->{$id}->{'futli'}=$ids->{$id}->{'futli'};
    $r->{$id}->{'fut'}=$found;
  }
  return $r;
}

sub get_fut_from_amf_simple {
  my $fufi=shift // confess "I need a fufi here.";
  my $r=shift // {};
  my $ids;
  if (not -f $fufi) {
    confess "I can't open your file $fufi";
  }
  if ($fufi=~m|\.amf\.xml\.gz$|) {
    my $cmd="zcat $fufi | xsltproc $xslt_file -";
    $ids=`zcat $fufi | xsltproc $xslt_file -`;
  } elsif ($fufi=~m|\.amf\.xml$|) {
    $ids=`xsltproc $xslt_file $fufi`;
  }
  foreach my $id (split(/\n/,$ids)) {
    my $found=&{$Ernad::FullTextFetch::fff->{'nep'}}($id,$fut_archive_dir);
    if (ref $found) {
      #my @founds=@{$found};
      #if(scalar @founds > 1) {
      #print "$id\n";
      #print Dumper $found,"\n";
      #}
    }
    $r->{$id}=$found;
  }
  return $r;
}


#print Dumper $r;



1;