/*
    Copyright (C) 2012 Bartosz Broda, Jacek Bochenek, Łukasz Biłeńkij, 
    Damian Jaworski, Wojciech Mazur, Dominik Piasecki, Maciej Piasecki, 
    Radosław Ramocki
    
    This program is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by the Free
Software Foundation; either version 3 of the License, or (at your option)
any later version.

    This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE. 

    See the LICENSE and COPYING files for more details.
*/

#include "arch_headers.h"
#include "Config.h"

#include "Options.h"

#include <boost/date_time/posix_time/posix_time.hpp>

namespace me_architect
{

	Config* Config::m_pInstance = NULL;
	Config::Config()
			: m_sWorkdir("./wd/")
	{
		m_bFilterDisamb = true;
		m_bChceckDocuments = false;
		favorGers = false;
		m_OperatorParser = boost::shared_ptr<JOSKIPI::Parsers::CAntlrParser>(
				new JOSKIPI::Parsers::CAntlrParser()  );
		LexicalUnits::MEParser::ResourcesHolder::SetJOSKIPIParser(m_OperatorParser);
	}

	Config* Config::GetInstance()
	{
		return m_pInstance;
	}

	void Config::Delete()
	{
		if(m_pInstance)
		{
			delete m_pInstance;
			m_pInstance = NULL;
		}
	}

	int Config::GetOperatorsPerNode()
	{
		return operatorsPerNode;
	}

	void Config::init(int ac,char*av[])
	{
		if(m_pInstance != NULL )
			Delete();

		m_pInstance = new Config();
		Options options(ac, av, m_pInstance);

	}

	std::vector<Triplet_OperatorFlexLex*> Config::InitalizeForMasterNode(int ac, char *av[])
	{
		if(m_pInstance==NULL)
			init(ac,av);

		// load lexical units
		m_pInstance->LoadRowUnits();
		// load operators (& column LU)
		return m_pInstance->NumOfOperators();

	}//void Config::Initalize()



	Config* Config::Initalize(int ac, char *av[],int columnRangeMin,int columnRangeMax,int *shuffleTable)
	{

		if(m_pInstance==NULL)
			init(ac,av);

		// load lexical units
		m_pInstance->LoadRowUnits();
		// load operators (& column LU)
		m_pInstance->LoadOperators(columnRangeMin,columnRangeMax,shuffleTable);
		return m_pInstance;
	}//void Config::Initalize()

	void Config::LoadRowUnits()
	{
		LoadRowME();
		std::cout << "Loading row single word lexical units" << std::endl;


		for(size_t i = 0; i < m_vRowTermFiles.size(); i++)
		{
			std::vector<LexicalUnits::LexicalUnitPtr> words;
			std::vector<std::string> tokens;
			Corpus::Parser::Tokenize(m_vRowTermFiles[i], tokens, "|");

			if(tokens.size() != 2 )
				throw std::runtime_error("format for opertors: lex_file1;flex1,...,lex_fileN;flexN");

			LoadSimpleWords(tokens[0], tokens[1], words);
			std::string flex = tokens[1];
			boost::trim(flex);
			for(std::vector<LexicalUnits::LexicalUnitPtr>::iterator i = words.begin(); i != words.end(); i++)
			{
				int b = (*i)->GetBaseId();
				std::string base = Corpus::CWordDictionary::GetInstance()->GetAt(b);
				m_vLexicalUnits.push_back(*i);
				rowLU.push_back(m_vLexicalUnits.size() - 1);

				rowToFlex[ rowLU.size() -1 ] = flex;
			}
		}

		std::cout << "Loaded " << rowLU.size() << " row units..." << std::endl;

	}

	void Config::LoadRowME()
	{
		std::cout << "Loading row ME" << std::endl;
		using LexicalUnits::LexicalUnitPtr;
		try{
		for(size_t fileNo = 0; fileNo < m_vRowUnits.size(); fileNo++)
		{
			std::cout << "File no: " << fileNo <<"\n";
			LexicalUnits::MEParser::MEParser parser;
			std::cout << "Parse this: \n";
			std::cout << "$" << m_vRowUnits[fileNo].c_str() <<"$\n";
			parser.Parse(m_vRowUnits[fileNo].c_str());

			std::vector<LexicalUnitPtr> loaded = parser.GetUnits();
			std::cout << "Got units. \n";
			for(size_t i = 0; i < loaded.size(); i++)
			{
				m_vLexicalUnits.push_back(loaded[i]);
				rowLU.push_back(m_vLexicalUnits.size() - 1);
				rowToFlex[ rowLU.size() -1 ] = Corpus::CTag::GetFlexClassString( loaded[i]->GetFlexClass() );
			}
		}
		}
		catch (antlr::RecognitionException& e)
		{
			//std::cout<<"Mismatched token: "<<e.token.getText()<<"\n";
			std::cout<<"Recognition exception in line: "<<e.getLine()<<"\n";
			throw e;
		}
		catch (std::string& e)
		{
			std::cout<<"String exception: "<<e<<"\n";
			throw e;
		}

	}//void Config::LoadME()

	std::vector<Triplet_OperatorFlexLex*> Config::NumOfOperators()
	{
		using namespace boost::posix_time;
		std::vector<std::string> tokens;

		std::vector<Triplet_OperatorFlexLex*> results;
		std::cout << "Counting operators...\n";
		int counter=0;

		//Count ME operators
		size_t i = 0;
		for(; i < m_vOperatorsME.size(); i++)
		{
			tokens.clear();
			Corpus::Parser::Tokenize(m_vOperatorsME[i], tokens, "|");
			if(tokens.size() != 2 )
				throw std::runtime_error("format for opertors: operator_file1;lex_file1[;flex1],...,operator_fileN;lex_fileN[;flexN]");


			std::string line;
			std::ifstream operatorFile(tokens[0].c_str());
			std::getline(operatorFile,line);
			std::string flexClass = line.substr(2,std::string::npos);

			std::string operatorString = LoadSingleOperatorString(tokens[0]);

			std::vector<LexicalUnits::LexicalUnitPtr> words;
			LoadMEForColumns(tokens[1], words);

			Triplet_OperatorFlexLex* triplet;
			for(std::vector<LexicalUnits::LexicalUnitPtr>::iterator j = words.begin(); j != words.end(); j++)
			{

				//add LU
				m_vLexicalUnits.push_back(*j);
				//add column LU
				columnLU.push_back( m_vLexicalUnits.size() - 1 );

			}
			triplet = new Triplet_OperatorFlexLex;
			triplet->flex = flexClass;
			triplet->operatorIndex = i;
			triplet->nextIndex = words.size();
			results.push_back(triplet);
			counter+=words.size();
			operators_columns.push_back(std::pair<std::string,size_t>(tokens[0],counter));


		}

		//Dictionary...
		std::vector<std::pair<std::string, std::string> > wordsFromDictVector;
		std::cout<<"Loading words from dictionary...\n";
		std::ifstream inputDictFile(dictWithWords.c_str());

		if(!inputDictFile)
			throw std::runtime_error("File with dictionary does not exist\n");
		std::string line;
		while((std::getline(inputDictFile,line)))
		{
			Corpus::Parser::Tokenize(line,tokens,";");
			wordsFromDictVector.push_back(std::pair<std::string,std::string>(tokens[0],tokens[1]));
		}


		//Count simple words operators...
		//i continues from above loop
		for(; i < m_vOperatorsSimple.size(); i++)
		{

			std::vector<LexicalUnits::LexicalUnitPtr> words;


			std::ifstream operatorFile(m_vOperatorsSimple[i].c_str());
			std::getline(operatorFile,line);
			line = line.substr(2,std::string::npos);
			std::vector<std::string> flexClasses;

			Corpus::Parser::Tokenize(line,flexClasses,",");


			Triplet_OperatorFlexLex* triplet;

			int nextIndex;
			if(results.size()>0)
				nextIndex = results[results.size()-1]->nextIndex;
			else
				nextIndex=0;

			for(int k=0;k<(int)flexClasses.size();k++)
			{
				LoadSimpleWordsForColumns(wordsFromDictVector,flexClasses[k], words);

				triplet = new Triplet_OperatorFlexLex;
				triplet->flex = flexClasses[k];
				triplet->operatorIndex = i;

				triplet->nextIndex = nextIndex+words.size();

				results.push_back(triplet);
			}



			for(std::vector<LexicalUnits::LexicalUnitPtr>::iterator j = words.begin(); j != words.end(); j++)
			{



				//add LU
				m_vLexicalUnits.push_back(*j);
				//add column LU
				columnLU.push_back( m_vLexicalUnits.size() - 1 );

			}
			counter+=words.size();
			operators_columns.push_back(std::pair<std::string,size_t>(m_vOperatorsSimple[i],counter));


		}


		std::cout << "Counted column units..." << std::endl;

		return results;

	}//void Config::LoadOperators()
	std::vector<std::pair<std::string,size_t> > Config::GetOperators_Columns()
	{
		return operators_columns;
	}


	void Config::LoadOperators(int columnRangeMin,int columnRangeMax,int *shuffleTable)
	{
		using namespace boost::posix_time;
		std::vector<std::string> tokens;
		std::vector<std::pair<std::string, std::string> > wordsFromDictVector;
		std::cout<<"Loading words from dictionary...\n";
		std::ifstream inputDictFile(dictWithWords.c_str());
		std::vector<std::string> m_vOperatorDefines;
		if(!inputDictFile)
			throw std::runtime_error("File with dictionary does not exist\n");
		std::string line;
		//First line is just a header
		std::getline(inputDictFile,line);
		if(std::string("flex;lemma;frequency").compare(line)!= 0)
			throw std::runtime_error("Wrong header line in dictionary file!");
		while((std::getline(inputDictFile,line)))
		{
			Corpus::Parser::Tokenize(line,tokens,";");
			wordsFromDictVector.push_back(std::pair<std::string,std::string>(tokens[0],tokens[1]));
		}


		std::cout << "Loading column lexical units and operators operators...\n";
		std::vector<OperatorAndLex*> operatorAndLexVector;

		OperatorAndLex* opAndL;

		int i;
		//Loading ME operators
		for(i = 0; i < (int)m_vOperatorsME.size(); i++)
		{
			tokens.clear();
			Corpus::Parser::Tokenize(m_vOperatorsME[i], tokens, "|");
			if(tokens.size() != 2 )
				throw std::runtime_error("format for opertors: operator_file1;lex_file1[;flex1],...,operator_fileN;lex_fileN[;flexN]");


			m_vOperatorDefines.push_back(tokens[0]);
			std::vector<LexicalUnits::LexicalUnitPtr> words;

			LoadMEForColumns(tokens[1], words);


			for(std::vector<LexicalUnits::LexicalUnitPtr>::iterator j = words.begin(); j != words.end(); j++)
			{
				opAndL = new OperatorAndLex;

				opAndL->operatorIndex = i;
				opAndL->ptr= *j;

				operatorAndLexVector.push_back(opAndL);

			}
		}
		//Loading simple words operators
		for(; i < (int)m_vOperatorsSimple.size(); i++)   // i continues from above loop
		{
			tokens.clear();

			m_vOperatorDefines.push_back(m_vOperatorsSimple[i]);


			std::ifstream operatorFile(m_vOperatorsSimple[i].c_str());
			std::getline(operatorFile,line);
			line = line.substr(2,std::string::npos);
			std::vector<std::string> flexClasses;

			Corpus::Parser::Tokenize(line,flexClasses,",");

			std::vector<LexicalUnits::LexicalUnitPtr> words;

			for(int k=0;k<(int)flexClasses.size();k++)
				LoadSimpleWordsForColumns(wordsFromDictVector,flexClasses[k], words);




			for(std::vector<LexicalUnits::LexicalUnitPtr>::iterator j = words.begin(); j != words.end(); j++)
			{
				opAndL = new OperatorAndLex;

				opAndL->operatorIndex = i;
				opAndL->ptr= *j;

				operatorAndLexVector.push_back(opAndL);

			}
		}

		std::cout<<"Loaded vector with operators and lexical units. Length: "<<operatorAndLexVector.size()<<"\n";
		int previousOperator=-1;
		std::string operatorString;
		LexicalUnits::LexicalUnitPtr j;
		int counter = 0;
		int max = columnRangeMax-columnRangeMin;
		ptime start = microsec_clock::local_time();
		time_duration total_time;
		Corpus::CWordDictionary *dict = Corpus::CWordDictionary::GetInstance();

		int shuffledI;
		for(int i=columnRangeMin;i<columnRangeMax;i++)
		{
			shuffledI = shuffleTable[i];

			if(operatorAndLexVector[shuffledI]->operatorIndex!=previousOperator)
			{
				previousOperator=operatorAndLexVector[shuffledI]->operatorIndex;


				operatorString = LoadSingleOperatorString(m_vOperatorDefines[previousOperator]);


			}
			else if(i==columnRangeMin)
			{
				throw std::runtime_error("Didnt read operatorString during first iteration\n");
			}



			j = operatorAndLexVector[shuffledI]->ptr;

			const std::string &word = dict->GetAt((j)->GetBaseId());
			std::string copy = boost::algorithm::replace_all_copy(operatorString, "^Zmienna", word);
			JOSKIPI::Operators::CCondition* condition = m_OperatorParser->ParseCondition(copy);
			m_vConditions.push_back(CondPtr(condition));
			counter++;


			//add LU
			m_vLexicalUnits.push_back(j);
			//add column LU
			columnLU.push_back( m_vLexicalUnits.size() - 1 );


			columnToFlex[columnLU.size() -1 ] = Corpus::CTag::GetFlexClassString( (j)->GetFlexClass() );

			ptime end = microsec_clock::local_time();
			time_duration len = end - start;
			if( len.total_milliseconds() > 5000)
			{
				total_time += len;
				long total_time_ms = total_time.total_milliseconds();
				double op_per_sec = double(counter*1000) / double(max);
				long eta_ms = (total_time_ms * (max-counter) ) / counter;
				std::cout << "Operator nr " << counter << " / " << max
						<< " | operator/sec = " << op_per_sec << " ETA: " << to_simple_string( milliseconds(eta_ms) )
						<< std::endl;
				start = microsec_clock::local_time();
			}


		}//for(size_t i = 0; i < m_vOperatorDefines.size(); i++)

		std::cout << "Loaded " << columnLU.size() << " column units..." << std::endl;

	}//void Config::LoadOperators()


	void Config::LoadMEForColumns(const std::string &filename, std::vector<LexicalUnits::LexicalUnitPtr> &out)
	{
		using LexicalUnits::LexicalUnitPtr;

		LexicalUnits::MEParser::MEParser parser;
		parser.Parse(filename.c_str());

		std::vector<LexicalUnitPtr> loaded = parser.GetUnits();

		for(size_t i = 0; i < loaded.size(); i++)
		{
			out.push_back(loaded[i]);
		}
	}//void Config::LoadMEForColumns(
	void Config::LoadSimpleWordsForColumns(std::vector<std::pair<std::string,std::string> > dictFromFile, std::string& flex,
										   std::vector<LexicalUnits::LexicalUnitPtr>&out)
	{
		//get flex clas
		Corpus::TFlexClass flex_class = GetFlexClass(flex);
		//load words
		std::vector<std::string> words;
		for(int i=0;i<(int)dictFromFile.size();i++)
			if(dictFromFile[i].first.compare(flex)==0)
				words.push_back(dictFromFile[i].second);
		//conver to SimpleWord
		Corpus::CWordDictionary *dict = Corpus::CWordDictionary::GetInstance();
		for(size_t i = 0; i < words.size(); i++)
		{
			Corpus::CTag tag(flex_class);
			tag.Categories = Corpus::mask_All;
			std::string w = words[i];
			boost::trim(w);
			boost::shared_ptr<LexicalUnits::Word> word(new LexicalUnits::Word(tag, dict->Add( w) ));
			LexicalUnits::LexicalUnitPtr lu( new LexicalUnits::SimpleWord(word) );


			out.push_back(lu);
		}
	}
	void Config::LoadSimpleWords(const std::string &filename, std::string& flex,
								 std::vector<LexicalUnits::LexicalUnitPtr>&out)
	{
		//get flex clas
		Corpus::TFlexClass flex_class = GetFlexClass(flex);
		//load words
		std::vector<std::string> words;
		LoadStringVectror(filename, words);
		//conver to SimpleWord
		Corpus::CWordDictionary *dict = Corpus::CWordDictionary::GetInstance();
		std::set<std::string> wordsSet;
		for(size_t i = 0; i < words.size(); i++)
		{
			Corpus::CTag tag(flex_class);
			tag.Categories = Corpus::mask_All;
			std::string w = words[i];
			boost::trim(w);
			boost::shared_ptr<LexicalUnits::Word> word(new LexicalUnits::Word(tag, dict->Add( w) ));
			LexicalUnits::LexicalUnitPtr lu( new LexicalUnits::SimpleWord(word) );

			wordsSet.insert(w);
			out.push_back(lu);
		}
		if(wordsSet.size()!=out.size())
			std::cerr<<"Probable "<<out.size()-wordsSet.size()<<" duplicates in lexical units.\n";

	}//void Config::LoadSimpleWords(const std::string &filename, std::string& flex,

	Corpus::TFlexClass Config::GetFlexClass(const std::string& str)
	{
		Corpus::TFlexClass flex = 0;
		std::vector<std::string> flexes;
		boost::split(flexes, str, boost::is_any_of("|"));
		for( size_t i = 0;  i < flexes.size(); i++ )
		{
			std::string &f = flexes[i];
			boost::trim(f);
			flex |= Corpus::CTag::GetFlexClass(f);
		}

		return flex;
	}//Corpus::TFlexClass Config::GetFlexClass(const std::string& str)

	std::string Config::LoadSingleOperatorString(const std::string& filename)
	{
		std::ifstream file(filename.c_str());
		std::string line;
		std::stringstream str;

		if(!file)
			throw std::runtime_error("File not found: "+filename);

		while(file)
		{
			std::getline(file, line);
			str << line << std::endl;
		}

		return str.str();
	}//std::string Config::LoadSingleOperatorString(const std::string& filename)


	void Config::LoadStringVectror(const std::string &filename, std::vector<std::string> &vec)
	{
		std::ifstream in(filename.c_str());
		if(!in)
			throw std::runtime_error("Cannot open file " + filename);
		std::string line;
		while(std::getline(in,line))
		{
			if(line.size() )
				vec.push_back(line);
		}//while(in)
	}//void Config::LoadStringVectror(const std::string &filename, std::vector<std::string> &vec)
}

