// $Id: vargibbs.cpp 1367 2024-08-30 21:00:18Z ge $
/// \file var-gibbs.cpp
/// \brief program for searching parameters of Gibbs Free Energy Models
/// \author Gerald Weber <gweberbh@gmail.com>
/// $Revision: 1367 $

#include <valarray>
#include "Options.h"
#include "NeighbourSequence.h"
#include <unistd.h>
#include "ParameterSearch.h" 
#include "GibbsMinimization.h"
#include "OutStream.h"
#include "ContextBasePairModel.h" //Also includes IndividualNearestNeighbours.h

using namespace gbc;
using namespace std;

typedef double value_type;

typedef ParameterSearch<GibbsMinimization<IndividualNearestNeighbours> > min_function_type;
typedef ParameterSearch<GibbsMinimization<ContextBasePairModel> > cbp_min_function_type;
 
// We need to declare an object of type min_function_type to avoid compiler errors
// of inclomplete type
min_function_type place_holder;
template<> min_function_type *GSLMinimization<min_function_type>::Function=NULL;
cbp_min_function_type cbp_place_holder;
template<> cbp_min_function_type *GSLMinimization<cbp_min_function_type>::Function=NULL;

Option<std::string>    minimization_method, output_name, model, duplextype, salt_scheme, calculation, minimization_log_level, par_id, sequence, csequence, merit, print_flags, adjust_par;
Option<deque<string> > par_file, datafile, variable, debug, CG_equivalent, AT_equivalent, variable_random_increment, adjust_exclude;
Option<double>         temp_err, salt, org_tm, target_salt, multiply_sp_conc, limit, av_diff_limit, av_stdev_limit, dgtm_threshold, random_initial_values, svz;
Option<int>            random_seed, log_level_interval, max_random_rounds, max_eval, max_min;
Option<bool>           prediction_with_salt_correction, randomize_dataset, generate_n_files, keep_tmp_files, nn_parameters_only, log_compress,printusedpar;
Option<Range<double> > ee_range;
  
Option<deque<double> > total_strand_concentration;
  
Option<map<string,string> > dict;
Option<Range<double> > clength;
  
template<class _Minimization>
class NNSearch
  {
  public:
    typedef _Minimization minimization_type;
    typedef typename minimization_type::model_type model_type;
    typedef ParameterSearch<minimization_type> regress_type;
    
    regress_type pregress;
  
  NNSearch(void) {};
  
  void process(void)
    {
    if (minimization_method.provided()) pregress.minimization_method(minimization_method.Value);
    pregress.Avoid_negative_parameters=false;
  
    if (merit.provided())
      {
      if (merit.check("Tm")) pregress.Evaluation.merit_tm();
      if (merit.check("DeltaHS")) pregress.Evaluation.merit_deltahs();
      if (merit.check("TmDeltaHS")) pregress.Evaluation.merit_tmdeltahs();
      if (merit.check("RTmDeltaHS")) pregress.Evaluation.merit_rtmdeltahs();
      if (merit.check("DeltaGXTm")) 
        {
        pregress.Evaluation.merit_deltag_x_tm();
        if (dgtm_threshold.provided())
          {
          pregress.Evaluation.DeltaG_X_Tm_chi2_threshold=dgtm_threshold.Value;
          }
        }
      pregress.Evaluation.print_merit_text(std::cout,STRING_CODE_PRF(INFOPMERTN));
      }

    pregress.MaxEvaluations=max_eval;
  
    pregress.MaxMinimizations=max_min;

    pregress.Evaluation.Calculation.NN_parameters_only=nn_parameters_only.Value;
    
    if (max_random_rounds.provided()) pregress.MaxRandomMinimizations=max_random_rounds.Value;
  
    if (av_diff_limit.provided()) pregress.Limit_average_difference=av_diff_limit.Value;
    if (av_stdev_limit.provided()) pregress.Limit_standard_deviation_difference=av_stdev_limit.Value;
  
    if (minimization_log_level.check("full"))
      {
      pregress.set_print_log_no_improvement(true);
      pregress.set_print_log_progress(true);
      pregress.set_print_log_replace_worst(true);
      }
    
    if (minimization_log_level.check("progress"))
      {
      pregress.set_print_log_no_improvement(false);
      pregress.set_print_log_progress(true);
      pregress.set_print_log_replace_worst(false);
      }
    
    if (minimization_log_level.check("terse"))
      {
      pregress.set_print_log_no_improvement(false);
      pregress.set_print_log_progress(false);
      pregress.set_print_log_replace_worst(false);
      }
      
    pregress.SVDMinimize.ZeroLimit=svz.Value;
    
    if (log_level_interval.provided()) pregress.set_log_progress_interval(log_level_interval.Value);
    
    ModelDataSet<model_type> *fem = &pregress.Evaluation.Regression;
    model_type *inn = fem->model;

    if (CG_equivalent.provided())
      for (auto cg : CG_equivalent.Value)  fem->dataset.CG_equivalent_set.insert(cg);
    if (AT_equivalent.provided())
      for (auto at : AT_equivalent.Value)  fem->dataset.AT_equivalent_set.insert(at);
  
    if (par_file.provided()) fem->read_rules_and_parameters_from_list_of_files(par_file.Value);

    if (salt_scheme.provided()) 
      {
      fem->dataset.select_salt_correction(salt_scheme);
      fem->dataset.Target_salt_ct=target_salt.Value;
      if (fem->dataset.Target_salt_ct == value_type())
        COUT_INFO(INFOUSCDAT) << " -targetsalt=0, using salt concentration in dataset as target for salt correction" << std::endl; 
      fem->dataset.Prediction_with_salt_correction=prediction_with_salt_correction.Value;
    
      COUT_INFO(INFOASCS) << " applying salt correction scheme " << salt_scheme.Value;
      if (fem->dataset.Prediction_with_salt_correction) 
        COUT << " to predicted Tm" << std::endl;
      else 
        COUT << " to dataset" << std::endl;
      }
    
    if(print_flags.provided())
      {
      fem->dataset.PrintFlags=print_flags.Value;
      }
 
    if(datafile.provided())
      {
      if ( calculation.check("sprediction") )
        fem->dataset.set_list_of_data_files(datafile.Value);
      else
        fem->dataset.read_list_of_data_files(datafile.Value,total_strand_concentration.Value,multiply_sp_conc.Value);
      }
    else //if no datafile is provided get the sequence and add to dataset
      {
      std::string cseq;
      if (csequence.provided()) cseq=(std::string)csequence;
      else
        {
        COUT_INFO(INFOCSNPWC) << " complementary sequence not provided, working out from complementary nucleotides " << std::endl;
        NucleotideSequence<> seq(sequence.Value.length()),comp(sequence.Value.length());
        seq << sequence.Value;
        comp = seq;
        comp.complementary();
        cseq = (string)comp;
        }
      COUT_INFO(INFOCFSS) << " calculating for single sequence " << sequence.Value << "/" << cseq << std::endl;
      if (salt.provided()) COUT_INFO(INFOSSALTC) << " sequence salt concentration=" << salt.Value << " mM" << std::endl;
      fem->dataset.add_temperature(sequence.Value,cseq,salt.Value,org_tm.Value,total_strand_concentration.Value[0]);
      }

    OutStream outfile((string)output_name + "|dat");
    OutStream nncheckfile((string)output_name + "|nncheck"); 
    OutStream verfile((string)output_name + "|ver");
    OutStream used_par_file((string)output_name + "|usedpar");     
    
    if(calculation.check("nncheck"))
      {
      if (output_name.provided())
        {
        nncheckfile.stream_open();
        COUT_INFO(INFOWNNATF) << " writing nncheck analysis to file " << nncheckfile.formatted_name() << std::endl;
        fem->dataset.print_temperatures(nncheckfile,true);
        }
      else
        {
        fem->dataset.print_temperatures(std::cout,true);
        }
      }
     
    if(calculation.check("readjust"))
      {
      inn->NN_parameters_only=nn_parameters_only.Value;
      fem->dataset.ParameterReadjust=true;
      verfile.stream_open();
      verfile << fem->predict_all_adjusting(ee_range.Value,adjust_exclude.Value,adjust_par.Value) << " ";
      fem->dataset.print_temperatures(outfile);
      fem->dataset.print_verify(verfile);
      }

    if(calculation.check("prediction"))
      {
      inn->NN_parameters_only=nn_parameters_only.Value;
      fem->predict_all();
      outfile.stream_open();
      fem->dataset.print_temperatures(outfile);
      verfile.stream_open();
      fem->dataset.calculate_and_print_verify(verfile);
      }

    if(calculation.check("sprediction"))
      {
      inn->NN_parameters_only=nn_parameters_only.Value;
      outfile.stream_open();
      fem->predict_and_print_sequentially(outfile);
      fem->dataset.print_verify(verfile);
      }
      
    if(calculation.check("minlocal","minglobal"))
      {
      pregress.basefile(output_name);
      pregress.log_compress(log_compress);

      pregress.assign_variables(variable.Value);
      pregress.Generate_files_for_each_round=generate_n_files.Value;
      pregress.KeepTmpFiles=keep_tmp_files.Value;

      if(calculation.check("minlocal"))
        {
        pregress.minimization(par_id,salt,clength.Value,limit.Value);//salt only used for printing
        }

      if(calculation.check("minglobal"))
        {
        if (randomize_dataset.Value)
          {
          pregress.Randomise_dataset=randomize_dataset.Value;
          pregress.Evaluation.Regression.dataset.Temperature.error=temp_err;
          }
        if (random_initial_values.provided())
          {
          pregress.Randomise_initial_parameters=true;
          pregress.Initial_parameter_variation=random_initial_values.Value;
          }
        if (variable_random_increment.provided())
          {
          pregress.Randomise_initial_parameters=true;
          pregress.assign_variable_random_increment(variable_random_increment.Value);
          }
        pregress.minimization_random(random_seed,par_id,salt,clength.Value,limit.Value);//salt only used for printing
        }
      }

    if (printusedpar.check(true))
      {
      pregress.print_parameters_and_stats(used_par_file,"parameters");
      used_par_file.stream_close();
      }
    }
  
  };

#if !defined(SVNVERSION)
#define SVNVERSION "$Revision: 1367 $"
#endif
#if !defined(LINUXSYSTEM)
#define LINUXSYSTEM "unknown"
#endif 
#if !defined(BUILDTIME)
#define BUILDTIME __DATE__
#endif 
#ifndef COMPILERNAME
#ifdef __INTEL_COMPILER
#define COMPILERNAME "icpc"    
#else
#define COMPILERNAME "gcc"
#endif
#endif 

int main(int argc,char *argv[])
  {
  std::cout << "*** vargibbs version=" << std::string(SVNVERSION) << " system=" << std::string(LINUXSYSTEM) << " builddate=" << std::string(BUILDTIME) << " compiler=" << std::string(COMPILERNAME) << std::endl;
  
  OptionMap::environment_id("VARGIBBS");
  
  Nucleotide<>::dictionary.add('_','_');
  Nucleotide<>::dictionary.add('U','U');
  
  model.define("model","energy model",OptionMap::optional,"NN");
  model.add_restricted_option("NN","Nearest-neighbour model");
  model.add_restricted_option("CBP","Context Base-Pair model");
  
  output_name.define("o","output files basename, resulting files will be appended with .dat, .ver etc",OptionMap::necessary);
  par_file.define("par","list of parameter file",OptionMap::necessary);
  datafile.define("data","list of sequence data files",OptionMap::optional);
  OptionMap::add_synonymn("xml","data");//replace xml with data for compatibility with var-par

  duplextype.define("duplextype","type of duplex, e.g. DNA or RNA",OptionMap::optional,"DNA");
  temp_err.define("ee","experimental temperature error in celsius",OptionMap::optional,0.0);
  random_seed.define("rs","random_seed",OptionMap::optional,1);

  salt.define("seqsalt","salt concentration (needed for salt correction of isolated sequences)",OptionMap::optional);
  org_tm.define("tm","original Tm (needed for salt correction of isolated sequences)",OptionMap::optional);
  target_salt.define("targetsalt","target salt concentration (for converting Tm), if zero gets it from the data file",OptionMap::optional,0);

  salt_scheme.define("saltscheme","type of salt conversion scheme",OptionMap::optional,"nocorrection");
  salt_scheme.add_restricted_option("tm_ln","generic tm vs ln");
  salt_scheme.add_restricted_option("rec_tm_ln","generic 1/tm vs ln");
  salt_scheme.add_restricted_option("santalucia98","santalucia98 for DNA");
  salt_scheme.add_restricted_option("owczarzy04eq19","owczarzy04eq19 for DNA");
  salt_scheme.add_restricted_option("owczarzy04eq20","owczarzy04eq20 for DNA");
  salt_scheme.add_restricted_option("owczarzy04eq21","owczarzy04eq21 for DNA");
  salt_scheme.add_restricted_option("owczarzy04eq22","owczarzy04eq22 for DNA");
  salt_scheme.add_restricted_option("chen13eq19","chen13eq19 for RNA");
  salt_scheme.add_restricted_option("chen13eq20","chen13eq20 for RNA");
  salt_scheme.add_restricted_option("chen13eq21","chen13eq21 for RNA");
  salt_scheme.add_restricted_option("chen13eq22","chen13eq22 for RNA");
  salt_scheme.add_restricted_option("schildkraut65","old correction for DNA");
  salt_scheme.add_restricted_option("nakano99","nakano99 for DNA/RNA only for 1000 to 100 Na+");

  prediction_with_salt_correction.define("predsaltcorr","apply salt correction to predicted Tm (default: apply to dataset Tm)",OptionMap::optional,false);
  
  total_strand_concentration.define("ct","single or list of species concentration Ct (in 10^-6 M)",OptionMap::optional);
  multiply_sp_conc.define("mct","multiply species concentration Ct by a given factor",OptionMap::optional,1.0);
  calculation.define("calc","which type of calculation to perform",OptionMap::necessary,"prediction");
  calculation.add_restricted_option("nncheck","interprets the sequences in the -data file and exits without performing calculations");
  calculation.add_restricted_option("minlocal","runs local minimization");
  calculation.add_restricted_option("minglobal","runs global minimization, needs -mrr");
  calculation.add_restricted_option("prediction","performs simple Tm prediction");
  calculation.add_restricted_option("sprediction","same as prediction but reads/calculates sequentially from file using less memory");
  calculation.add_restricted_option("readjust","performs parameter readjustment");
  
  OptionMap::add_synonymn("minsimple","minlocal");//replace minsimple with minlocal
  OptionMap::add_synonymn("minmult","minglobal");//replace minmult with minglobal
  
  variable.define("var","list of variables to loop",OptionMap::optional);
  minimization_method.define("minmethod","select select minimization method",OptionMap::optional,"gsl");
  minimization_method.add_restricted_option("gsl","uses GNU scientific library minimization function");
  minimization_method.add_restricted_option("gbc","uses own minimization algorithm");
  minimization_method.add_restricted_option("svd","uses SVD algorithm");
  minimization_log_level.define("minloglevel","selects the amount of information for the minimization log", OptionMap::optional,"progress");
  minimization_log_level.add_restricted_option("progress","log minimization every time (see -loginterval) a better chi2 is achieved");
  minimization_log_level.add_restricted_option("full","log minimization for every evaluation, even if chi2 does not improve");
  minimization_log_level.add_restricted_option("terse","only log minimization for last chi of each round");
  log_level_interval.define("loginterval","in seconds, interval of -minloglevel=progress",0);
  
  randomize_dataset.define("randomizedataset","randomize dataset",OptionMap::optional,false);
  max_random_rounds.define("mrr","maximum number of local minimization rounds (option calc=minglobal)",OptionMap::optional,10);
  dict.define("dict","Additions to nucleotide dictionary",OptionMap::optional);
  limit.define("limit","minimization limit",OptionMap::optional,0.01);
  clength.define("cl","characteristic length",OptionMap::optional,10.0);
  max_eval.define("me","maximum number of evaluations",OptionMap::optional,std::numeric_limits<int>::max());
  max_min.define("mm","maximum number of minimizations",OptionMap::optional,2);
  av_diff_limit.define("adl","limit for total average difference",OptionMap::optional);
  av_stdev_limit.define("asl","limit for total standard deviation difference",OptionMap::optional);
  generate_n_files.define("gn","generate files for each round of minimization",OptionMap::optional,true);
  keep_tmp_files.define("keeptmp","do not delete .tmp files",OptionMap::optional,false);
  nn_parameters_only.define("nnpo","use NN parameters only, this ignores initialization, terminal and symmetry parameters",OptionMap::optional,false);
  log_compress.define("zlog","compress all log files in gzip format",OptionMap::optional,false);

  par_id.define("parid","parameter id",OptionMap::optional,"vargibbs");
  debug.define("debug","select debuging flags",OptionMap::optional);
  
  sequence.define("seq","sequence main strand 5'->3'",OptionMap::optional);
  csequence.define("cseq","sequence complementary strand 3'->5'",OptionMap::optional);
  
  svz.define("svz","singular value zero",OptionMap::optional,1e-06);
  
  merit.define("merit","merit function",OptionMap::optional,"Tm");
  merit.add_restricted_option("Tm","use Tm");
  merit.add_restricted_option("DeltaHS","use DeltaH and DeltaS");
  merit.add_restricted_option("TmDeltaHS","use Tm, DeltaH and DeltaS");
  merit.add_restricted_option("DeltaGXTm","use Tm together with DeltaG x Tm, see also -dgtmthreshold");
  
  dgtm_threshold.define("dgtmthreshold","threshold for chi2 of DeltaGXTm",OptionMap::optional,0.0);
  
  random_initial_values.define("randominitial","random initial values",0.0);
  variable_random_increment.define("varinitial","list of variables and their random initial increments",OptionMap::optional);
  
  //CG_equivalent and AT_equivalent will be compared to BP_reduced_duplex in sequencestructure/SequenceInfo.h
  CG_equivalent.define("cgeq","Additional base pairs equivalent to CG, used for calculating fCG",OptionMap::optional); 
  AT_equivalent.define("ateq","Additional base pairs equivalent to AT, used for Is_AT_only",OptionMap::optional); 

  printusedpar.define("printusedpar","print parameters that were used with file extension .usedpar",OptionMap::optional,false);

  print_flags.define("printflags","flags to control what is printed in .ver files",OptionMap::optional);
  
  
  //Special options for calc=readjust
  ee_range.define("eer","range of enthalpy or entropy for calc=readjust",OptionMap::optional);
  
  adjust_exclude.define("aexc","list of NNs or BPS to exclude from  calc=readjust",OptionMap::optional);
  
  adjust_par.define("apar","type of parameter to fix for calc=readjust",OptionMap::optional,"DeltaS");
  adjust_par.add_restricted_option("DeltaS","fix entropy Eq (15) martins24");
  adjust_par.add_restricted_option("DeltaH","fix enthalpy Eq (16) martins24");
  
  //Unused option, reserved for future use
//   Option<int>            seq_length("sl","fixed sequence length",OptionMap::optional,0);

  
  OptionMap::arguments(argc,argv);
  OptionMap::scan_arguments();
  if (calculation.check("minlocal","minglobal"))
    {
    variable.turn_necessary(true);
    datafile.turn_necessary(true);
    if (calculation.check("minglobal"))
      {
      temp_err.turn_necessary(randomize_dataset.Value==1);
      random_initial_values.turn_necessary(!randomize_dataset.Provided and !variable_random_increment.provided());
      variable_random_increment.turn_necessary(!randomize_dataset.Provided and !random_initial_values.provided());
      randomize_dataset.turn_necessary(!random_initial_values.Provided);
      }
    }
  if (calculation.check("prediction")  or calculation.check("sprediction") )
    {
    total_strand_concentration.turn_necessary(sequence.Provided);
    datafile.turn_optional(sequence.Provided);
    sequence.turn_necessary(!datafile.Provided);
    }
  if (calculation.check("nncheck"))
    {
    output_name.turn_optional(true);
    par_file.turn_optional(true);
    if (not sequence.provided()) datafile.turn_necessary(true); else datafile.turn_necessary(false);
    }
  if (target_salt.provided() or prediction_with_salt_correction.check(true) )
    {
    salt_scheme.turn_necessary(true);
    if (sequence.provided()) salt.turn_necessary(true); else salt.turn_necessary(false);
    }
  
  
  OptionMap::check_if_all_given();
  OptionMap::show_options();
  
  if (debug.provided()) 
    for(auto db : debug.Value) DebugControl::enable(db);
      
  if (dict.provided())
    {
    for(auto mit : dict.Value)
      {
      std::cout << "Adding new nucleotide to dictionary= "  << mit.first[0] << " and its complementary= " << mit.second[0] << std::endl;
      Nucleotide<>::dictionary.add(mit.first[0],mit.first[0],true);
      Nucleotide<>::dictionary.add(mit.second[0],mit.second[0],true);
      Nucleotide<>::dictionary.complementary_pair(mit.second[0],mit.first[0]);
      }
    }

  if (duplextype.check("RNA"))
    {
    Nucleotide<>::dictionary.complementary_pair('A','U');
    std::cout << Nucleotide<>::dictionary;
    }

  if (model.check("NN"))
    {
    NNSearch<GibbsMinimization<IndividualNearestNeighbours> > nn_regress;
    nn_regress.process();
    }
  if (model.check("CBP"))
    {
    NNSearch<GibbsMinimization<ContextBasePairModel> > cbp_regress;
    cbp_regress.process();
    }
  
  print_cpu();
  return 0;
}

