// $Id: ParameterSearch.h 1364 2024-06-13 20:31:00Z ge $
/// \file ParameterSearch.h
/// \brief contains the ParameterSearch function class
///
/// $Revision: 1364 $
/// \author Gerald Weber <gweberbh@gmail.com>
#ifndef GBC_PARAMETERSEARCH_H
#define GBC_PARAMETERSEARCH_H "$Id: ParameterSearch.h 1364 2024-06-13 20:31:00Z ge $"
#include "Minimization.h"
#include "GSLMinimization.h"
#include "LinearMinimization.h"
#include <cmath>
#include <boost/regex.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/iostreams/device/back_inserter.hpp>
#include <boost/iostreams/copy.hpp>
#include "CPUtime.h"
#include "ErrorCodes.h"
#include "OutStream.h"
#include "ParameterMap.h"

namespace gbc
{
template<class _value_type>
class VariableLimits
  {
  public:
    typedef _value_type value_type;
    typedef std::deque<std::string> associated_deque_type;
    value_type current;
    value_type start;
    value_type stop;
    bool limits_set;
    associated_deque_type Associated_variables;//Variables that should have the same value as the main one

  VariableLimits(void): current(), start(), stop(), limits_set(false) {};
  
  void set_limits(value_type st, value_type en)
    {
    start=st;
    stop=en;
    limits_set=true;
    }
    
  value_type new_random_value(const double &rnd, const bool &linear, const bool multiply=true)
    {
    double dice;
    if (limits_set)
      {
      dice=linear_ran();
      current = (start-stop)*dice + start;
      }
    else
      {
      if (linear) dice=linear_ran(2*rnd,0.5); //this will return a random value between -rnd and +rnd
      else        dice=gauss_ran(rnd);
      if (multiply) current *=(1.0+dice); //for linear this results in 1-rnd and 1+rnd
      else current += dice; //add or subtract 
      }
    return  current;
    }

    
  };
  

    class files_class 
      {
      public:
      OutStream Result;
      OutStream Parameter;
      OutStream ParAverage;
      OutStream Verification;
      OutStream Variation;
      OutStream Log;//for gz compression
      OutStream Regression;
      OutStream Evolution;
      OutStream Proceed;
      std::map<std::string,OutStream*> FileMap;
      std::string BaseName;
      std::string SuppExtension;
      int Version;
      bool Make_proceed_file;
      bool Make_files;
      bool Versioned;
      bool Log_compress;//for gz compression
      boost::iostreams::filtering_streambuf< boost::iostreams::output> log_filter;//for gz compression https://www.boost.org/doc/libs/1_46_1/libs/iostreams/doc/classes/filtering_streambuf.html
      
      files_class(void) : Result("|dat"), Parameter("|par"), ParAverage("|par|average"), Verification("|ver"), Variation("|var"), Log("|log"), Regression("|reg"), Evolution("|average_std_evolution"), Proceed("|proceed"),
      Version(0), Make_proceed_file(false), Make_files(false), Versioned(false), Log_compress(false)
        {
        FileMap["Result"]=&Result;
        FileMap["Parameter"]=&Parameter;
        FileMap["ParAverage"]=&ParAverage;
        FileMap["Verification"]=&Verification;
        FileMap["Variation"]=&Variation;
        FileMap["Log"]=&Log;
        FileMap["Regression"]=&Regression;
        FileMap["Evolution"]=&Evolution;
        FileMap["Proceed"]=&Proceed;
        };
        
      inline void next_version(void)
        {
        close_all();
        Version++;
        for(auto file: FileMap) if (file.first != std::string("Proceed")) file.second->Version=Version;
        }
        
      inline void overwrite(void)
        {
        close_all();
        }
        
      inline void set_basename(std::string bsname, std::string ext=std::string())
        {
        BaseName=bsname;
        SuppExtension=ext;
        FileMap["Log"]->Compression=Log_compress;
        for(auto file: FileMap) 
          {
          file.second->set_basename(BaseName);
          if (not SuppExtension.empty())file.second->set_basename("||" + SuppExtension);
          file.second->IgnoreOpen = not Make_files;
          }
        if (Make_proceed_file) 
          {
          Proceed << "1" << std::endl;
          Proceed.stream_close();
          }
        }
        
      inline void close_all(void)
        {
        if (not Make_files) return;//if we didn't open we do not close either
        for(auto file: FileMap) file.second->stream_close();
        }
        
      inline void remove_all(void)
        {
        for(auto file: FileMap) file.second->remove_all();
        }
        
      inline bool proceed(void)
        {
        std::ifstream ReadProceed;
        ReadProceed.open(Proceed.FormattedName.c_str());
        bool proc=true; //to avoid abort in case of problems with file reading
        ReadProceed >> proc;
        ReadProceed.close();
        return proc;
        }
      };
  
  
template<class _EvaluateTp>
class ParameterSearch
  {
  public:
    typedef _EvaluateTp evaluate_type;
    typedef typename evaluate_type::value_type value_type;
    typedef std::valarray<value_type> vector_type;
    typedef std::map<std::string,VariableLimits<value_type> >     map_type;
    typedef std::map<size_t,map_type>        all_map_type;
    typedef std::map<std::string,value_type> string_val_map_type;
    typedef std::set<std::string>            svd_variable_set_type;//To be used by IndividualNearestNeighbours
    typedef typename LinearMinimization::matrix_type matrix_type;

    enum minimization_engine_enum {GBC,GSL,SVD};
    minimization_engine_enum MinimizationEngine;
    /// The minimization class takes as template a class which performs the merit function evaluation
    Minimization<ParameterSearch<evaluate_type> > Minimize;
    GSLMinimization<ParameterSearch<evaluate_type> > GSLMinimize;
    LinearMinimization SVDMinimize;

    evaluate_type Evaluation; ///< This is the function that will run in minimization

    bool Debug;
    bool Random_search;
    bool Proceed;
    bool Avoid_negative_parameters;
    bool Randomise_initial_parameters;
    bool Initial_parameters_linear; //if true randomization is linear, if false gaussian
    bool Randomise_dataset;
    bool Generate_files_for_each_round;
    bool KeepTmpFiles;
    value_type Initial_parameter_variation;
    int MaxEvaluations;
    int MaxMinimizations;
    int MaxRandomMinimizations; ///< Maximum number of random regression minimization, 0 means go on forever
    long int Evaluations;
    value_type Total_average_difference;
    value_type Total_standard_deviation_difference;
    value_type Limit_average_difference;
    value_type Limit_standard_deviation_difference;
    map_type Variable_map;
    map_type Best_variable_map;
    map_type Average_variable_map;
    map_type Previous_average_variable_map;
    map_type Standard_deviation_variable_map;
    map_type Previous_standard_deviation_variable_map;
    all_map_type Complete_variable_map;
    vector_type Best_vector;
    value_type  Best_value;
    std::string BaseFileName;
    files_class File, PermanentFile, BestValueFile, TmpFile;
    Range<value_type> Parameter_range;
    value_type Salt;
    string_val_map_type VariableRandomIncrement; //Map containing regex increment values

    ParameterSearch(void)
      : MinimizationEngine(GBC), Minimize(), GSLMinimize(), SVDMinimize(), Evaluation(), Debug(false), Random_search(false), Proceed(true), Avoid_negative_parameters(true),
        Randomise_initial_parameters(false), Initial_parameters_linear(true), Randomise_dataset(true), Generate_files_for_each_round(true), KeepTmpFiles(false), Initial_parameter_variation(value_type()),
        MaxEvaluations(1000), MaxMinimizations(1), MaxRandomMinimizations(0) /*<==Has to be zero, do not change*/, Evaluations(), 
        Total_average_difference(), Total_standard_deviation_difference(), Limit_average_difference(), Limit_standard_deviation_difference(), Salt()
         {}
        
    void minimization_method(std::string option)
      {
      if (option == std::string("gbc")) MinimizationEngine=GBC;
      else 
        {
        if (option == std::string("gsl")) MinimizationEngine=GSL;
        else
          {
          if (option == std::string("svd")) MinimizationEngine=SVD;
          else
            {
            CERR_ERROR(ERRUNMINEN) <<  " Unknown minimization engine=" << option << ", options are: gbc or gsl" << std::endl;
            CERR_TERM
            }
          }
        }
      }
      
    // Inserts a new value p in Variable_map and in the calculation map
    inline void update_calculation_parameter_map(const std::string id, const value_type& p)
      {
      Evaluation.Calculation.parameter_map.set_parameters(id,p);//updates the calculation map
      Variable_map[id].current=p; //updates the variables map
      for(size_t i=0; i < Variable_map[id].Associated_variables.size(); i++)
        Evaluation.Calculation.parameter_map.set_parameters(Variable_map[id].Associated_variables[i],p);//updates all associated variables
      }
      

    /// \brief Assing list of parameters which will vary during the minimization
    ///
    /// This will assign the current definition of the parameter to its 
    /// corresponding position in Variable_map.
    /// The initial value of Variable_map is whatever was stored in ParameterMap initially.
    inline void assign_variables(const std::deque<std::string>& var) ///< List of parameters, usually given on command line
      {        
      const boost::regex regex_limits("^(.+)\\(([+-]?\\d+\\.?\\d*[eE]?[-+]?\\d*)\\|([+-]?\\d+\\.?\\d*[eE]?[-+]?\\d*)\\)");// Experimental/untested reserved for future implementation
      const boost::regex incomplete("^(.+)\\((.*)\\)");
      boost::smatch found;
      std::string variable;

      for(auto &var_list_it : var)
        {
        variable=var_list_it;
        if (variable == std::string("T")) 
          Variable_map[variable].current=Evaluation.Calculation.temperature(); // If parameter is simply a T, assing temperature to it.
        else 
          {
          if (boost::regex_search(variable,found,regex_limits)) // ATTENTION Experimental/untested reserved for future implementation
            {
            std::string v=found[1], l1=found[2], l2=found[3];
            Variable_map[v].set_limits(std::atof(l1.c_str()),std::atof(l2.c_str()));
            CERR_DEBUG(DPAS_AINTTO) << "Assigning interval to " << v << " (" << l1 << "|" << l2 << ")" << std::endl; 
            Evaluation.Calculation.parameter_map.get_parameter(v,Variable_map[v].current);
            }
          else
            {
            if (boost::regex_search(variable,found,incomplete)) // ATTENTION Experimental/untested reserved for future implementation
              {
              CERR_ERROR(ERRCLNOTP) << "could not parse " <<  variable << std::endl;
              CERR_TERM
              }
            else
              {
              //Variables may be concatenated by equal sign as in AT_AT:entropy=AT_TA:entropy=AT_CG:entropy=AT_GC:entropy
              //The following will split these, the first variable is the lead parameter, all others become
              //associated variables which are placed into the Associated_variables deque. Whenever the lead parameter changes
              //all associated variables take its value.
              std::vector<std::string> strs;
              boost::split(strs,variable,boost::is_any_of("="));
            
              variable=strs[0];//assign the lead parameter (the first before any =)
              Evaluation.Calculation.parameter_map.get_parameter(variable,Variable_map[variable].current);//assigns the lead parameter
              
              for(size_t i=1; i < strs.size(); i++) 
                Variable_map[variable].Associated_variables.push_back(strs[i]);
              }
            }
          }
          
        size_t varn = Variable_map.size();
        COUT_INFO(INFOVARPAR) << " Variable parameter [" << varn << "] = " << variable;
        if (Variable_map[variable].Associated_variables.size() > 0) 
          COUT << " associated variables are: " << boost::algorithm::join(Variable_map[variable].Associated_variables," ");
        COUT << std::endl;
        
        CERR_DDEBUG(DPAS_ASSIGNVAR) << " assign_variables " << variable << " " << Variable_map[variable].current << " = "
                                 << boost::algorithm::join(Variable_map[variable].Associated_variables," = ")
                                 << std::endl << std::endl;
          
      if (MinimizationEngine == SVD)
        {
        std::deque<std::string> vard;
        for(auto &vm : Variable_map) vard.push_back(vm.first);
      
        Evaluation.Regression.set_svd_variables(vard);//Regression=FreeEnergyModel
        }
      } 
    }

    /// \brief Replace values in ParameterMap with current values of Variable_map
    /// ATTENTION **** Apparently not in use
    inline void variable_map_reassign(void)
      {
      for(auto &it : Variable_map)
        {
        if (it.first == std::string("T")) 
          it.second.current=Evaluation.Calculation.temperature(); // If parameter is simply a T, assign temperature to it.
        else 
          update_calculation_parameter_map(it.first,it.second.current);
        CERR_DDEBUG(DPAS_VMAPREA) << " variable_map_reassign)" << it.first << " "<< Variable_map[it.first].current << std::endl << std::endl;
        }
      } 
      
    inline void assign_variable_random_increment(const std::deque<std::string> &var_list)
      {
      const boost::regex reg("^(.+):(.+)$");
      for(auto &it : var_list)
        {
        boost::smatch found;
        if (boost::regex_search(it,found,reg))
          {
          std::string r=found[1], inc=found[2];
          VariableRandomIncrement[r]=std::stod(inc);
          CERR_DEBUG(DPAS_AVRI) << " assigned " << r << " with increment " << inc << std::endl;
          }
        }
      }

    /// \brief Randomize the initial values of VariableMap from ParameterMap.
    inline void variable_map_randomise(double rnd)
      {
      for(auto &it : Variable_map)
        {
        if (it.first == std::string("T")) 
          it.second.current=Evaluation.Calculation.temperature(); // If parameter is simply a T, assing temperature to it.
        else
	  {
          if (not VariableRandomIncrement.empty())
            {
            for(auto &vri_it : VariableRandomIncrement)
              {
              const boost::regex reg(vri_it.first);
              boost::smatch found;
              if (boost::regex_search(it.first,found,reg))
                {
                it.second.new_random_value(vri_it.second,Initial_parameters_linear,false);
                }
              }
            }
          else it.second.new_random_value(rnd,Initial_parameters_linear);
          CERR_DDEBUG(DPAS_VMAPRAND) <<  " variable_map_randomise" << std::endl << it.first << "<--->" << it.second.current << std::endl << std::endl;
	  }
        CERR_DDEBUG(DPAS_VMAPRAND) <<  " variable_map_randomise" << std::endl << it.first << " "<< Variable_map[it.first].current << std::endl << std::endl;
        }
      } 

    inline void basefile(std::string bsname)
      {
      BaseFileName=bsname;
      }

    inline void log_compress(const bool cps)
      {
      File.Log_compress=PermanentFile.Log_compress=BestValueFile.Log_compress=cps;
      }
      
    /// \brief Assigns the vector from the Variable_map
    inline void matrix2vector(vector_type& vec)
      {
      if (vec.size() != Variable_map.size()) vec.resize(Variable_map.size());
      typename map_type::const_iterator it;
      size_t n=0;
      for(it=Variable_map.begin(), n=0; it != Variable_map.end(); ++it, ++n)
        vec[n]=it->second.current;
      }

    /// \brief Assigns the contents of a vector to the Variable_map
    ///
    /// The vector is assigned in the same order as the variables are stored.
    inline void vector2var(const vector_type& vec)
      {
      typename map_type::iterator it;
      size_t n=0;
      for(it=Variable_map.begin(), n=0; it != Variable_map.end(); ++it, ++n)
        {
        it->second.current=vec[n];
        }
      }

    inline vector_type randomise(value_type scale,const vector_type& vec)
      {
      vector_type nvec(vec.size());
      size_t n=0;
      for(n=0; n < vec.size(); ++n)
        {
        double dice=rand()/static_cast<double>(RAND_MAX)-0.5;
        nvec[n]=vec[n]+dice*scale*vec[n];
        }
      return nvec;
      }

    inline vector_type random_search(value_type scale, const vector_type& vec)
      {
      vector_type nvec=randomise(scale,vec);
      value_type nv=evaluate_from_vector(nvec);
      int evaluations=1;
      while ((nv > Best_value) && (evaluations < MaxEvaluations))
        {
        nvec=randomise(scale,vec);
        nv=evaluate_from_vector(nvec);
        evaluations++;
        }
      if (nv < Best_value) 
        {
        File.Log << "random_search: " << evaluations << " ";
        print_res(File.Log,nv);
        return nvec;
        }
      else return vec;
      }


    /// \brief Evaluates the function from the vector or parameters
    ///
    /// First we assign the new parameters to the parameter map of the HeterogenousTM class.
    /// At the same time we update the Variable_map.
    /// Then the HeterogenousTM issues a get_parameters() call which load the modified
    /// parameters into the ModelPotentials. Only then we do the Evaluation which
    /// is defined in the SearchAction class.
    /// \Attention This returns Evaluate*negative_parameter_penalty
    inline value_type evaluate_from_vector(const vector_type& vec)
     {
     value_type negative_parameter_penalty=1.0;
     typename map_type::iterator var_map_it;
     size_t n=0;
     for(var_map_it=Variable_map.begin(), n=0; var_map_it != Variable_map.end(); ++var_map_it, ++n)
       {
       if (var_map_it->first == std::string("T"))
         {
         Evaluation.Calculation.temperature(vec[n]);
         var_map_it->second.current=vec[n];
         }
       else
         {
         update_calculation_parameter_map(var_map_it->first,vec[n]);
         }
       
       if (Avoid_negative_parameters && (vec[n] < 0.0)) negative_parameter_penalty += 1.0;//this is a hack really
       }
     //if (Debug) std::cout << Calculation << std::endl;
     CERR_DDEBUG(DPAS_EVCALPM) << " Current parameters in map" << std::endl; 
     IFDDEBUG(DPAS_EVCALPM)  Evaluation.Calculation.parameter_map.print_parameters_and_stats(CERR); 
     CERR_DDEBUG_NH(DPAS_EVCALPM) << std::endl;

     Evaluation.Calculation.set_TM_parameters();
     Evaluations++;
     value_type evalresult=Evaluation.evaluate_from_variable_map();
     CERR_DDEBUG(DPAS_EVALRES) << " eval=" << evalresult << std::endl;
     return negative_parameter_penalty*evalresult;
     }

    /// \brief Evaluates the function from the Variable_map
    ///
    /// The Variable_map is saved into the ParameterMap of HeterogenousTM.
    /// Then the HeterogenousTM issues a set_TM_parameters() call which load the modified
    /// parameters into the ModelPotentials. Only then we do the Evaluation which
    /// is defined in the SearchAction class.
    inline value_type evaluate_from_variable_map(void)
     {
     for(auto var_map_it : Variable_map) update_calculation_parameter_map(var_map_it.first,var_map_it.second.current);

     CERR_DDEBUG(DPAS_EVCALPM) << " Current parameters in map" << std::endl; 
     IFDDEBUG(DPAS_EVCALPM) Evaluation.Calculation.parameter_map.print_parameters_and_stats(CERR); 
     CERR_DDEBUG_NH(DPAS_EVCALPM) << std::endl;

     Evaluation.Calculation.set_TM_parameters();
     return Evaluation.evaluate_from_variable_map();
     }

    /// \brief Scans the parameters for the Variable_map 
    ///
    /// We increment the parameters in Variable_map sequentially and run evaluate.
    /// \Attention This function can be very time consuming if the ranges have
    /// a large number of steps or if there are too many variables in Variable_map.
    inline void scan_parameters(Range<value_type>& prange)
      {
      value_type Best_sqrdiff=1e+12, sqrdiff;

      File.Make_files=true;
      File.set_basename(BaseFileName);
      typename std::vector<Range<value_type> > ranges;
      prange.reset();
      typename map_type::iterator it;
      for(it=Variable_map.begin(); it != Variable_map.end(); ++it)
        {
        ranges.push_back(prange);
        it->second.current=prange.begin();
        File.Log << it->first << " " << prange << " " << prange.steps() << " ";
        }
      File.Log << std::endl;

      for(prange.reset(); !prange.finished(); ++prange) File.Log << prange.current() << " ";
      File.Log << std::endl;

      it=Variable_map.begin(); ++it; File.Log << it->second.current << " ";
      sqrdiff=evaluate_from_variable_map();
      File.Log << sqrdiff << " ";
      if (sqrdiff < Best_sqrdiff)
        {
        Best_sqrdiff=sqrdiff;
        Best_variable_map=Variable_map;
        }

      typename std::vector<Range<value_type> >::iterator next=ranges.begin();
      ++next;
      typename std::vector<Range<value_type> >::iterator loop=ranges.begin();
      bool finished=false;
      typename map_type::iterator variable;
      variable=Variable_map.begin();
      ++variable;
      while (!finished)
        {
        if (loop->finished())
          {
          File.Log << std::endl;
          loop->reset(); Variable_map.begin()->second.current=loop->current();
          if (next != ranges.end())
            {
            if (next->finished())
              {
              ++next; ++variable; if (next == ranges.end()) break;
              }
            else
              {
              next->next_step(); variable->second.current=next->current();
              File.Log << variable->second.current << " ";
              }
            }
          else break;
          }
        else
          {
          loop->next_step(); Variable_map.begin()->second.current=loop->current();
          }

        if (!loop->finished() && !next->finished())
          {
          sqrdiff=evaluate_from_variable_map();
          File.Log << sqrdiff << " ";
          if (sqrdiff < Best_sqrdiff)
            {
            Best_sqrdiff=sqrdiff;
            Best_variable_map=Variable_map;
            }
          }

        }

      Variable_map=Best_variable_map;
      evaluate_from_variable_map();

      print_var(File.Variation,"");
      print_par(File.Parameter,"");
      print_best_value(File.Verification);
      print_verification(File.Verification);

      }

    /// \brief Scans the parameters for the Variable_map 
    ///
    /// We increment the parameters in Variable_map sequentially and run evaluate.
    /// \Attention This function can be very time consuming if the ranges have
    /// a large number of steps or if there are too many variables in Variable_map.
    inline void scan_parameters(Range<value_type>& prange1, Range<value_type>& prange2)
      {
      value_type Best_sqrdiff=1e+12, sqrdiff;

      File.Make_files=true;
      File.set_basename(BaseFileName);
      typename std::vector<Range<value_type> > ranges;
      prange1.reset();
      prange2.reset();
      typename map_type::iterator it=Variable_map.begin(); 
      ranges.push_back(prange1);
      it->second.current=prange1.begin();
      File.Log << it->first << " " << prange1 << " " << prange1.steps() << " ";
      ++it;
      ranges.push_back(prange2);
      it->second.current=prange2.begin();
      File.Log << it->first << " " << prange2 << " " << prange2.steps() << " ";
      File.Log << std::endl;

      for(prange1.reset(); !prange1.finished(); ++prange1) File.Log << prange1.current() << " ";
      File.Log << std::endl;

      it=Variable_map.begin(); ++it; File.Log << it->second.current << " ";
      sqrdiff=evaluate_from_variable_map();
      File.Log << sqrdiff << " ";
      if (sqrdiff < Best_sqrdiff)
        {
        Best_sqrdiff=sqrdiff;
        Best_variable_map=Variable_map;
        }

      typename std::vector<Range<value_type> >::iterator next=ranges.begin();
      ++next;
      typename std::vector<Range<value_type> >::iterator loop=ranges.begin();
      bool finished=false;
      typename map_type::iterator variable;
      variable=Variable_map.begin();
      ++variable;
      while (!finished)
        {
        if (loop->finished())
          {
          File.Log << std::endl;
          loop->reset(); Variable_map.begin()->second.current=loop->current();
          if (next != ranges.end())
            {
            if (next->finished())
              {
              ++next; ++variable; if (next == ranges.end()) break;
              }
            else
              {
              next->next_step(); variable->second.current=next->current();
              File.Log << variable->second.current << " ";
              }
            }
          else break;
          }
        else
          {
          loop->next_step(); Variable_map.begin()->second.current=loop->current();
          }

        if (!loop->finished() && !next->finished())
          {
          sqrdiff=evaluate_from_variable_map();
          File.Log << sqrdiff << " ";
          if (sqrdiff < Best_sqrdiff)
            {
            Best_sqrdiff=sqrdiff;
            Best_variable_map=Variable_map;
            }
          }

        }

      Variable_map=Best_variable_map;
      evaluate_from_variable_map();

      print_var(File.Variation,"");
      print_par(File.Parameter,"");
      print_best_value(File.Verification);
      print_verification(File.Verification);

      }
      
    inline void set_print_log_no_improvement(bool sp)
      {
      Minimize.PrintLogNoImprovement = GSLMinimize.PrintLogNoImprovement = sp;
      }

    inline void set_print_log_progress(bool sp)
      {
      Minimize.PrintLogProgress = GSLMinimize.PrintLogProgress = sp;
      }
      
    inline void set_log_progress_interval(int secs)
      {
      Minimize.ProgressInterval = GSLMinimize.ProgressInterval = secs;
      }

    inline void set_print_log_replace_worst(bool sp)
      {
      Minimize.PrintLogReplaceWorst = sp;
      }

    inline void minimize(Range<double>& clength,const double& limit)
      {
      //Evaluation.Debug=true;
      value_type cl=clength.current();//relates to option -cl
      vector_type  p; 
      matrix2vector(p);//here we place the numerical content of the variable map (option -var) into a vector
      Best_vector.resize(p.size());
      Best_value=std::numeric_limits<value_type>::max();
      value_type previous=Best_value, diff=std::numeric_limits<value_type>::max();
      int improove=0;
      evaluate_from_vector(p);
      //MaxMinimization relates to option -mm 
      //limit relates to option -limit
      
      if (MaxMinimizations == 0) 
        {
        MaxMinimizations=std::numeric_limits<int>::max();
        COUT_INFO(INFOMM0D) << " -mm=0 detected, setting -mm to maximum integer =" << MaxMinimizations << std::endl;
        }
      
      while((diff > limit) && (improove < MaxMinimizations) && Proceed)
        {
      File.Log << "*** Starting round " << improove << std::endl;
      switch (MinimizationEngine)
          {
          case GBC: Minimize.reset(*this,p,cl);
                    Minimize.MaxEvaluations=MaxEvaluations;
                    Minimize.run(limit);
                    if (Best_value > Minimize.best_value())
                      {            
                      Best_value=Minimize.best_value();
                      Best_vector=Minimize.best();
                      }
                    break;
          case GSL: GSLMinimize.reset(*this,p,cl);
                    GSLMinimize.MaxEvaluations=MaxEvaluations;
                    GSLMinimize.run(limit);
                    if (Best_value > GSLMinimize.best_value())
                      {            
                      Best_value=GSLMinimize.best_value();
                      Best_vector=GSLMinimize.best();
                      }
                    break;
          case SVD: matrix_type A, B;
                    Evaluation.Regression.set_svd_matrices(A,B);
                    CERR_DEBUG(DPAS_SHSVDR) << "A matrix" << A << std::endl;
                    CERR_DEBUG(DPAS_SHSVDR) << "B matrix" << B << std::endl;
                    SVDMinimize.set(A,B);
                    SVDMinimize.run();
                    CERR_DEBUG(DPAS_SHSVDR) << " Before unit adjustments:" << SVDMinimize.x << std::endl;
                    matrix_type X(A.Columns,1);
                    X=SVDMinimize.x;
                    CERR_DEBUG(DPAS_SHSVDR) << "X matrix" << X << std::endl;
                    matrix_type AX(B.Rows,B.Columns), Chi2(B.Rows,B.Columns);
                    AX=A*X;
                    Chi2=AX -B;
                    CERR_DEBUG(DPAS_SHSVDR) << "A*X-B" << Chi2 << std::endl;
                    Evaluation.Regression.adjust_svd_units(SVDMinimize.x);
                    CERR_DEBUG(DPAS_SHSVDR) << " After unit adjustments:"<< SVDMinimize.x << std::endl;
                    Best_vector=SVDMinimize.x;
                    Best_value=evaluate_from_vector(SVDMinimize.x);
                    Proceed=0;
                    break;
          };
        diff=fabs(Best_value-previous);
        previous=Best_value;
        if (Random_search) 
          {
          p=random_search(0.5,Best_vector);
          File.Log << "*** best vector randomized to 0.5" << std::endl;
          }
        else p=Best_vector;
        
        //Here we temporarily write the round results, this will later be overwritten by the final results
        //therefore this will only be visible while the program has not finished or it it is interrupted
        std::stringstream id;
        id << "round=" << improove << std::endl 
           << "#chi2=" << Best_value << std::endl 
           << "#ATTENTION this is temporary information, if your are seeing this then the program has not finished yet";
        TmpFile.set_basename(BaseFileName + "---mmround-" + std::to_string(improove) + "---");
        print_par(TmpFile.Parameter,id.str()); TmpFile.Parameter.stream_close();
        print_res(TmpFile.Result,Salt); TmpFile.Result.stream_close(); 
        
        improove++;
        }
        
      if (diff <= limit) 
        {
        File.Log << "*** Difference between the last two rounds " << diff << " below " << limit << ", stopping minimization altogether" << std::endl; 
        COUT_INFO(INFODBTLTR) << " Difference between the last two rounds " << diff << " below " << limit << ", stopping minimization altogether" << std::endl; 
        }
      File.Log << "*** Local minimization stopped at round " << improove << " (maximum rounds was "<< MaxMinimizations << ")" << std::endl;
      COUT_INFO(INFOLMSAR) << " Local minimization stopped at round " << improove << " (maximum rounds was "<< MaxMinimizations << ")" << std::endl;
      
      evaluate_from_vector(Best_vector);
      vector2var(Best_vector);
      
      }

   inline void minimization(std::string par_id, double salt, Range<double>& clength,const double& limit)
     {
     Salt=salt;
     File.Make_proceed_file=true;
     File.Make_files=true;
     File.set_basename(BaseFileName);
     
     TmpFile.Make_files=Generate_files_for_each_round;
     TmpFile.set_basename(BaseFileName,"tmp");//Files with supplementary extension .tmp will be deleted for each round after the 
     
     print_par(File.Verification,par_id);
     File.Verification << "Calculation:" << std::endl;
     print_calculation(File.Verification);
     print_parameters_and_stats(File.Verification,"Calculation.parameter_map:"); 
     File.Verification << std::endl;
     Minimize.LogFile=&File.Log;
     GSLMinimize.LogFile=&File.Log;
     print_verification(File.Verification);
     
     minimize(clength,limit);
     
     print_res(File.Result,salt);
     print_var(File.Variation,par_id);
     print_par(File.Parameter,par_id);
     print_best_value(File.Verification);
     print_verification(File.Verification);
     print_regression(File.Regression);
     File.close_all();
     
     if (not KeepTmpFiles) TmpFile.remove_all(); //Preliminary info no longer necessary
       
     }

   /// \brief Performs a minimization on randomized datasets
   inline void minimization_random(int random_seed,        ///< Randomization seed
                                   std::string par_id,     ///< parameter identification
                                   double salt,            ///< salt concentration, only used for printing out results, has otherwise no effect
                                   Range<double>& clength, ///< minimization parameter characteristic length
                                   const double& limit)    ///< minimization parameter limit below which we stop searching for better values
     {
     Salt=salt;
     ParameterMap<value_type> pm=Evaluation.Calculation.parameter_map; //Here we store the initial parameters
     map_type vm=Variable_map; //Here we store the initial Variable Map
     srand(random_seed);
     
     PermanentFile.Make_proceed_file=true;
     PermanentFile.Make_files=true;
     PermanentFile.set_basename(BaseFileName);
     PermanentFile.Log << "Random seed=" << random_seed << std::endl;

     //File is versioned, this is set on call to File.next_version()
     //Do not write to file before this call
     File.Make_files=Generate_files_for_each_round;
     File.set_basename(BaseFileName);
     
     BestValueFile.Make_files = true;
     BestValueFile.set_basename(BaseFileName,"best");
     
     TmpFile.Make_files=Generate_files_for_each_round;
     TmpFile.set_basename(BaseFileName,"tmp");//Files with supplementary extension .tmp will be deleted for each round after the minimization is finished
     
     Proceed=PermanentFile.proceed();
     
     print_verification(PermanentFile.Verification);
     PermanentFile.Verification 
       << Evaluation.Regression.dataset.Temperature.error << " "
       << Evaluation.Regression.dataset.Enthalpy.error << " "
       << Evaluation.Regression.dataset.Entropy.error << std::endl;
     value_type Global_best_value=1e+10;
     while(Proceed)
       {
       File.next_version(); TmpFile.next_version();
       if (Randomise_initial_parameters)
         {
         //Evaluation.Calculation.parameter_map.randomise(Initial_parameter_variation);
         //variable_map_reassign();
         variable_map_randomise(Initial_parameter_variation);
         COUT_INFO(INFOIVRMR) << "Initial variables randomised for minimization round " << File.Version << " (mrr)" << std::endl;
         print_cpu();//so that we can have an immediate access to how much memory was used
         }
       print_par(File.Verification,par_id);
       File.Verification << "Calculation:" << std::endl;
       print_calculation(File.Verification);
       print_parameters_and_stats(File.Verification,"Calculation.parameter_map:"); 
       File.Verification << std::endl;
       Minimize.LogFile=&File.Log;
       GSLMinimize.LogFile=&File.Log;
       if (Randomise_dataset)
         {
         Evaluation.Regression.dataset.randomise_dataset();
         File.Verification << Evaluation.Regression.dataset.Temperature.std_deviation << " "
                           << Evaluation.Regression.dataset.Enthalpy.std_deviation << " "
                           << Evaluation.Regression.dataset.Entropy.std_deviation << std::endl;
         PermanentFile.Verification << Evaluation.Regression.dataset.Temperature.std_deviation << " "
                           << Evaluation.Regression.dataset.Enthalpy.std_deviation << " "
                           << Evaluation.Regression.dataset.Entropy.std_deviation << std::endl;
	 }
       print_verification(File.Verification);
       minimize(clength,limit); //here is the part that takes time to run
       
       Complete_variable_map[File.Version]=Variable_map;

       print_res(File.Result,salt);
       print_res(PermanentFile.Result,salt);
       print_var(File.Variation,par_id);
       print_par(File.Parameter,par_id);
       print_best_value(File.Verification);
       print_verification(File.Verification);
       print_regression(File.Regression);
       
       if (Global_best_value > Best_value)
         {
         COUT_INFO(INFONBOPR) << "New best set of parameters at round " << File.Version << " with chi2 = " << Best_value << std::endl;
         Global_best_value = Best_value;
         BestValueFile.overwrite();
         print_res(BestValueFile.Result,salt);
         print_var(BestValueFile.Variation,par_id);
         print_par(BestValueFile.Parameter,par_id);
         print_best_value(BestValueFile.Verification);
         print_verification(BestValueFile.Verification);
         print_regression(BestValueFile.Regression);
         BestValueFile.close_all();
         }
       
       //From here we reset everything to start over
       Evaluation.Calculation.parameter_map=pm; // restore initial parameters
       Evaluation.Calculation.set_TM_parameters();
       Variable_map=vm; //restore initial variable map
       Proceed=PermanentFile.proceed();
       if (not Proceed) COUT_INFO(INFOCPFDEC) << "Change in proceed file " << PermanentFile.Proceed.FormattedName << " detected, ending calculation" << std::endl; 

       this->save_av_par_file(par_id,File.Version);
       
       //The next blocks stop the loop with a berak statement
       if (MaxRandomMinimizations and (File.Version >= MaxRandomMinimizations)) 
         {
         COUT_INFO(INFOMNRLMR) << " Maximum number of local minimization rounds (mrr) reached (max=" << File.Version << ")" << std::endl;
         break;
         }
       if ((Total_average_difference <  Limit_average_difference) and (File.Version >= 10)) 
         {
         COUT_INFO(INFOLTADR) << " Limit for total average difference reached " << Total_average_difference
         << " limit= " << Limit_average_difference << ", stoped at round " << File.Version << std::endl;
         break;
         }
      if ((Total_standard_deviation_difference <  Limit_standard_deviation_difference) and (File.Version >= 10)) 
         {
         COUT_INFO(INFOLTSDDR) << " Limit for total standard deviation difference reached " << Total_standard_deviation_difference
         << " limit= " << Limit_standard_deviation_difference << ", stoped at round " << File.Version << std::endl;
         break;
         }
          
    
       File.close_all(); TmpFile.close_all();
       }

     PermanentFile.close_all();
     if (not KeepTmpFiles) TmpFile.remove_all(); //Preliminary info no longer necessary

     }
     
   inline void make_averages(void)
     {
     Average_variable_map.clear();
     Standard_deviation_variable_map.clear();
     for(auto& mp: Complete_variable_map)
       {
       for(auto& var : mp.second) Average_variable_map[var.first].current += var.second.current;
       }
     for(auto& var : Average_variable_map) var.second.current /= Complete_variable_map.size();
     for(auto& mp : Complete_variable_map)
       {
       for(auto& var : mp.second)
         Standard_deviation_variable_map[var.first].current += pow(Average_variable_map[var.first].current - var.second.current,2);
       }
     for(auto& var : Standard_deviation_variable_map)
       var.second.current = sqrt(var.second.current/Complete_variable_map.size());  
     }

   //Prints out the variable map and its associated variables too
   inline void print_var(OutStream& out, std::string ident)
      {
      out.stream_open_if_not(); //may have been closed on previous read
      out << ident << std::endl;
      for(auto &it : Variable_map)
        {
        out << it.first << " " << it.second.current << std::endl;
        for(size_t i=0; i < it.second.Associated_variables.size(); i++) out << it.second.Associated_variables[i] << " " << it.second.current << std::endl;
        }
      }

   //same as the previous one, specific for std::ostream
   inline void print_var(std::ostream& out, std::string ident)
      {
      out << ident << std::endl;
      for(auto &it : Variable_map)
        {
        out << it.first << " " << it.second.current << std::endl;
        for(size_t i=0; i < it.second.Associated_variables.size(); i++) out << it.second.Associated_variables[i] << " " << it.second.current << std::endl;
        }
      }
      
      
    inline void print_par(OutStream& out, std::string ident)
      {
      out.stream_open_if_not(); //may have been closed on previous read
      Evaluation.Calculation.parameter_map.print_parameters_and_stats(out,ident);
      out << std::endl;
      }      

    //the same as print_par except for an additional std::endl
    inline void print_parameters_and_stats(OutStream& out, std::string cmt)
      {
      out.stream_open_if_not();       
      Evaluation.Calculation.parameter_map.print_parameters_and_stats(out,cmt);
      }

      
    inline void save_av_par_file(std::string indent, int version)
      {
      make_averages();
      PermanentFile.ParAverage << indent << "-average-" << version << std::endl;

      for(auto& var : Average_variable_map)
        {
        PermanentFile.ParAverage << var.first << " " << var.second.current << " +/- " << Standard_deviation_variable_map[var.first].current;
        if (Previous_average_variable_map[var.first].current != 0)
          {
          value_type average_difference=fabs((Previous_average_variable_map[var.first].current-var.second.current)/Previous_average_variable_map[var.first].current);
          PermanentFile.ParAverage << " [" << average_difference;
          Total_average_difference += average_difference;
          }
        if (Previous_standard_deviation_variable_map[var.first].current != 0)
          {
          value_type std_difference=fabs((Previous_standard_deviation_variable_map[var.first].current-Standard_deviation_variable_map[var.first].current)/Previous_standard_deviation_variable_map[var.first].current);
          PermanentFile.ParAverage << " " << std_difference << "]";
          Total_standard_deviation_difference += std_difference;
          }
        PermanentFile.ParAverage << std::endl;
        
        }
      PermanentFile.ParAverage.stream_close();//This file will be rewritten for every round
      
      Previous_average_variable_map=Average_variable_map;
      Previous_standard_deviation_variable_map=Standard_deviation_variable_map;
     
      Total_average_difference /= Average_variable_map.size();
      Total_standard_deviation_difference /= Average_variable_map.size();
      
      PermanentFile.Evolution << Complete_variable_map.size() << " " << Total_average_difference << " " << Total_standard_deviation_difference << std::endl;
      }

    inline void print_head(OutStream& out)
      {
      out << Best_value << "Best ";
      for(auto& it : Variable_map) out << it.first << " ";
      out << std::endl;
      }


    inline void print_res(OutStream& out, value_type sl)
      {
      if (sl != value_type()) out << sl << " ";
      out << Best_value << " ";
      for(auto it : Variable_map) out << it.second.current << " ";
      out << std::endl;
      }
      
    
    inline void print_verification(OutStream& out)
      {
      out.stream_open_if_not(); 
      Evaluation.print_verification(out);
      }
      
    inline void print_regression(OutStream& out)
      {
      out.stream_open_if_not(); 
      Evaluation.print_regression(out);
      }

    inline void print_calculation(OutStream& out)
      {
      out.stream_open_if_not();       
      Evaluation.Calculation.print_calculation(out);
      }
      
    inline void print_best_value(OutStream& out)
      {
      out << Best_value << std::endl;
      }

  };
}
#endif
