/*
    Copyright (C) 2012 Bartosz Broda, Jacek Bochenek, Łukasz Biłeńkij, 
    Damian Jaworski, Wojciech Mazur, Dominik Piasecki, Maciej Piasecki, 
    Radosław Ramocki
    
    This program is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by the Free
Software Foundation; either version 3 of the License, or (at your option)
any later version.

    This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE. 

    See the LICENSE and COPYING files for more details.
*/

#include "arch_headers.h"
#include "Architect.h"
#include "Config.h"
#include "MatrixSaver.h"

#include <boost/date_time/posix_time/posix_time.hpp>
#include <poliqarp_reader/PoliqarpSentencer.h>
#include <poliqarp_reader/PoliqarpReader.h>


#ifdef JOSKIPI_OPERATOR_TEST
std::ofstream file("/mnt/data2/zdania_kipi_coord_puszczone_w_b4");
#endif// JOSKIPI_OPERATOR_TEST

namespace me_architect
{
	Corpus::CSentence* Architect::TransformSenetenceOnCurrentPosiotion(
			Corpus::CSentence* old, Triplets &t, size_t &removedBeforeCurrentPos,
			LexicalUnits::LexicalUnitPtr lu)
	{
		Corpus::CToken *substitute = old->GetToken(t.headId);
		std::string lu_base = Corpus::CWordDictionary::GetInstance()->GetAt(lu->GetBaseId() );
		
		removedBeforeCurrentPos = 0;
		old->EraseTokens(t.constituents);
		if( lu->GetConstituentsCount() > 1 )
			substitute->SetOrth( lu_base );
		for(size_t i = 0; i < substitute->GetLexCount(); i++)
		{
			Corpus::CLexem *lex = substitute->GetLexem(i);
			lex->SeBase( lu_base );
		}

		size_t position = ( size_t ) old->GetPosition();
		for (std::set<size_t>::iterator i = t.constituents.begin(); i
				!= t.constituents.end() && *i < position; i++)
			removedBeforeCurrentPos++;

		return old;
	}//Corpus::CSentence* Architect::TransformSenetenceOnCurrentPosiotion

	Architect::Architect()
	{
	}

	void Architect::Start()
	{
		Config *config = Config::GetInstance();
		std::cout << "Initalizing...\n";
		Initalize();
		std::cout << "Starting...\n";
		if(config->m_CorpusType == XML)
			ProcessDocuments();
		else
			ProcessBinaryCorpus();
#ifdef JOSKIPI_OPERATOR_TEST
		file.close();
		std::cout << "\nSentences retrieved..." << std::endl;
		exit(0);
#endif
		std::cout << "Saving data..." << std::endl;
		MatrixSaver ms;
		ms.SaveToSuperMatrix(m_pMatrix, m_stLexicalUnitCount);
		LexicalUnits::UnigramDictionary::GetInstance()->Save(config->m_sWorkdir + config->m_sMatrixName +"/udict.txt");
	}

	void Architect::ProcessSentence(Corpus::CSentence* sentence)
	{
		if( Config::GetInstance()->m_bFilterDisamb)
			sentence->FilterDisamb();
		m_stTokenCount += sentence->Size();
		// position in sentence -> number of row
		std::map<size_t, size_t> relevantRows;
		// 
		std::map<size_t, size_t> relevantColumns;

		CheckSentence(sentence, relevantRows, relevantColumns);
		m_stLexicalUnitCount+= (size_t)sentence->Size();//sentence is transformed
		CheckConditions(sentence, relevantRows, relevantColumns);

	}//void Architect::ProcessSentence(Corpus::CSentence* sentence)

	void Architect::CheckConditions(Corpus::CSentence * &sentence,
			std::map<size_t, size_t> &relevantRows,
			std::map<size_t, size_t> &relevantColumns)
	{
		Config *config = Config::GetInstance();
		for (std::map<size_t, size_t>::iterator inSentenceToRow =
				relevantRows.begin(); inSentenceToRow != relevantRows.end(); inSentenceToRow++)
		{
			//if( filter.isProperAt( inSentenceToRow->first ) )
			//	continue;

			sentence->Goto( (int) inSentenceToRow->first );
			for (std::map<size_t, size_t>::iterator column = relevantColumns.begin(); column
					!= relevantColumns.end(); column++)
			{
//				if (column->second == inSentenceToRow->second)
//					continue;

				//if( filter.isProperAt( column->first ) )
				//	continue;

				bool resultPositive = (config->m_vConditions)[column->second]->Evaluate(*sentence);
				if (resultPositive)
				{
#ifndef JOSKIPI_OPERATOR_TEST
					m_pMatrix->AddToCell(1.0, inSentenceToRow->second, column->second);
#else// JOSKIPI_OPERATOR_TEST
					Config* config = Config::GetInstance();
					// s1; pos1; s2; pos2; zdanie_orth_z_podkresleniem
					size_t pos1 = inSentenceToRow->first;
					LexicalUnits::LexicalUnitPtr pRowUnit = config->m_vLexicalUnits[ inSentenceToRow->second ];

					size_t pos2 = column->first;
					LexicalUnits::LexicalUnitPtr pColumnUnit = config->m_vLexicalUnits[ column->second ];
					// wypisz zdanie
					std::stringstream s;
					for(size_t sent_pos = 0; sent_pos < sentence->Size(); sent_pos++)
					{
						if(sent_pos == pos1)
							s << "*";
						if(sent_pos == pos2)
							s << "#";

						s << (*sentence)[sent_pos]->GetOrth();

						if(sent_pos == pos1)
							s << "*";
						if(sent_pos == pos2)
							s << "#";
						
						s << " ";
					}
					//std::cout << s.str() << std::endl;
					file << s.str() << std::endl;
#endif //JOSKIPI_OPERATOR_TEST

				}
			}
		}
	}//void CheckConditions(

	void Architect::CheckSentence(Corpus::CSentence * &sentence,
			std::map<size_t, size_t> &relevantRows,
			std::map<size_t, size_t> &relevantColumns)
	{
		std::set<size_t> relevantLUs;
		FindAllOccurrences(sentence, relevantLUs);
		// where have we finished after transformation
		int lastI = 0;
		// was the whole sentence checked
		bool checked = false;
	//	std::string sentence_str = sentence->ToOrthString();
	//	std::cout << sentence_str << std::endl;
		

		while ( !checked)
		{
			// current position in the sentence
			int i = lastI;
			// true if any lexical unit has been choosen
			bool choosen = false;

			for (; i < sentence->Size() && !choosen; i++)
			{
				std::vector< Triplets> foundLUs;

				sentence->Goto(i);
				FindAllPotentialForPosition(sentence, relevantLUs, foundLUs);
				// if unit was not found on currentt position, then go to next position in sentence	
				if (foundLUs.size() == 0)
					continue;
				LexicalUnits::LexicalUnitPtr lu;
				Triplets &t = foundLUs[0];
				choosen = ChooseLU(i, relevantRows, relevantColumns,
						t, foundLUs, lu);

				//transform sentence, set new as a current
				sentence->Goto(i);
				size_t removedBefore;
				//sentence transformation
				sentence = TransformSenetenceOnCurrentPosiotion(sentence, t,
						removedBefore, lu);
				if (removedBefore != 0)
					i -=(int) removedBefore;
				//the loop will always breka in here  (choosen must be true now), but we don't want to have i++ twice
				break;
			}//for ( int i = 0 ; i < sentence->Size() && !choosen; i++ )
			lastI = i+1;
			if (lastI >= sentence->Size() )
				checked = true;
		}//while !checked
	}//void CheckSentence

	bool Architect::ChooseLU(int pos, std::map<size_t, size_t> &relevantRows,
			std::map<size_t, size_t> &relevantColumns, Triplets &t,
			std::vector< Triplets> &foundLUs, LexicalUnits::LexicalUnitPtr &lu)
	{
		LexicalUnits::UnigramDictionary * dict = LexicalUnits::UnigramDictionary::GetInstance();
		Config *config = Config::GetInstance();
		bool choosen = false;
		bool choosenRow = false;
		bool choosenColumn = false;
		// choose first LU, but check rows first
		for (size_t luIndex = 0; luIndex < foundLUs.size() && !choosen; luIndex++)
		{
			if (!choosenRow && m_mUnit2Row.count(foundLUs[luIndex].lexId) )
			{
				choosenRow = true;
				if (choosenColumn)
					choosen = true;
				lu = (config->m_vLexicalUnits)[ foundLUs[luIndex].lexId ];
				t = foundLUs[luIndex];
				relevantRows.insert(std::pair<size_t, size_t> (pos, m_mUnit2Row[foundLUs[luIndex].lexId ]) );
				dict->AddLexicalUnit( lu );
			}
			if(!choosenColumn && m_mUnit2Column.count( foundLUs[luIndex].lexId ) )
			{
				choosenColumn = true;
				if(!choosenRow)
				{
					t = foundLUs[luIndex];
					lu = (config->m_vLexicalUnits)[ foundLUs[luIndex].lexId ];
				}
				else
					choosen = true;
				relevantColumns.insert( std::pair<size_t, size_t> ( pos, m_mUnit2Column[foundLUs[luIndex].lexId] ) );
				dict->AddLexicalUnit( (config->m_vLexicalUnits)[ foundLUs[luIndex].lexId ]);
			}
		}

		// if the choosen lexical unit was not in row's lexical unit, then choose first unit out 
		// of column's description
		if(!choosenRow && !choosenColumn)
		{
			throw std::runtime_error( "!choosenRow && ! choosenColumn" );
		}

		return choosen;
	}//ChooseLU(std::map<size_t, size_t>  &relevantRows,

	void Architect::FindAllPotentialForPosition(Corpus::CSentence *sentence, std::set<size_t> & relevantLUs,
			std::vector< Triplets> &foundLUs)
	{
		Config *config = Config::GetInstance();
		int maxConstituentsFound = -1;
		//find all possible LU for this position
		for(std::set<size_t>::iterator luIndexIter = relevantLUs.begin();
				luIndexIter != relevantLUs.end(); luIndexIter++)
		{

			LexicalUnits::LexicalUnit &lu= *(config->m_vLexicalUnits)[ *luIndexIter ];
			size_t headId;
			std::set<size_t> constituents;

			if( lu.CheckCurrentPosition(*sentence, headId, constituents) &&
					maxConstituentsFound <= (int) lu.GetConstituentsCount() )
			{
				if( (int)lu.GetConstituentsCount()> maxConstituentsFound )
				{
					maxConstituentsFound = (int) lu.GetConstituentsCount();
					foundLUs.clear();
				}
				Triplets t;
				t.lexId = *luIndexIter;
				t.constituents = constituents;
				t.headId = headId;
				foundLUs.push_back( t );
			}
		}
	}//FindAllPotentialForPosition(

	void Architect::FindAllOccurrences(Corpus::CSentence *sentence, std::set<size_t> &relevantLUs)
	{
		// preliminary check - check which lexical units can possibly occure in current sentence
		for(int i = 0; i < sentence->Size(); i++)
		{
			Corpus::CToken *t = sentence->operator [ ](i);
			for(size_t lex_no = 0; lex_no < t->GetLexCount(); lex_no++)
			{
				Corpus::CLexem *pDisamb = t->GetLexem(lex_no);
				if( pDisamb->GetDisamb() == false)
					continue;
				int base = pDisamb->GetBaseIdx();

				std::multimap<int, size_t>::iterator find = m_mBase2PotentialUnit.find(base);
				if(find == m_mBase2PotentialUnit.end() )
					continue;
				for(; find != m_mBase2PotentialUnit.end() && find->first == base; find++)
					relevantLUs.insert(find->second);
			}
		}//for each token in sentence
	}//void Architect::FindAllOccurrences(

	void Architect::Initalize()
	{
		Config* config = Config::GetInstance();

		//clear data
		m_mUnit2Row.clear();
		m_mUnit2Column.clear();
		m_mBase2PotentialUnit.clear();
		LexicalUnits::UnigramDictionary::GetInstance()->Clear();
		m_stTokenCount = 0;
		m_stLexicalUnitCount = 0;

		//create indices
		for ( size_t i = 0; i < config->rowLU.size(); i++ )
			m_mUnit2Row.insert ( std::pair<size_t, size_t> ( ( config->rowLU ) [i],i ) );

		for ( size_t i = 0; i < config->columnLU.size(); i++ )
			m_mUnit2Column.insert ( std::pair<size_t, size_t> ( ( config->columnLU ) [i],i ) );

		//Corpus::CWordDictionary * dict = Corpus::CWordDictionary::GetInstance();
		std::set<int> potential;
		for ( size_t i = 0; i < config->m_vLexicalUnits.size(); i++ )
		{
			( config->m_vLexicalUnits ) [i]->GetPotentialBases ( potential );
			for ( std::set<int>::iterator j = potential.begin(); j != potential.end();j++ )
				m_mBase2PotentialUnit.insert ( std::pair<int, size_t> ( *j,i ) );
		}

		//init matrix
		m_pMatrix = boost::shared_ptr<Matrices::CRSMatrix> ( new Matrices::CRSMatrix ( "data", config->rowLU.size(), config->columnLU.size()) );

	}//void Architect::InitalizeMatrix()

	void Architect::ProcessBinaryCorpus()
	{
		using namespace boost::posix_time;
		ptime start = microsec_clock::local_time();
		time_duration total_time;

		PoliqarpTools::CPoliqarpReader cpr(Config::GetInstance()->m_sCorpusFile, false, false);
		if(!cpr.CompileQuery("[]{0,10000} within s"))
			throw std::runtime_error("Failed to compile querry");
		size_t max = cpr.GetCorpusSize();
		PoliqarpTools::CPoliqarpSentencer sentencer(&cpr);
		Corpus::CSentence * sentence;
		int tokens = 0;
		std::cout << "Starting... " << std::endl << std::endl;

		while( (sentence = sentencer.GetNext()))
		{
			ProcessSentence ( sentence );
			tokens += sentence->Size();
			sentencer.Free(sentence);
			ptime end = microsec_clock::local_time();
			time_duration len = end - start;
			if( len.total_milliseconds()> 5000)
			{
				total_time += len;
				long total_time_ms = total_time.total_milliseconds();
				long eta_ms = (total_time_ms * (max-tokens) ) / tokens;
				std::cout << "Processed " << tokens << " / " << max 
					<< " Elapsed: " << milliseconds(total_time_ms) 	<< " ETA: " << to_simple_string( milliseconds(eta_ms) ) 	
					<< "            \r";
				std::cout.flush();
				start = microsec_clock::local_time();
			}//if len > 5000
		}
	}


	void Architect::ProcessDocuments()
	{
		using namespace boost::posix_time;
		std::vector<std::string> &documents = Config::GetInstance()->documents;
		size_t curr = 0;
		size_t max = documents.size();
		ptime start = microsec_clock::local_time();
		time_duration total_time;
		size_t tokens = 0;
		for ( std::vector<std::string>::iterator doc = documents.begin(); doc != documents.end(); doc++ )
		{
			curr++;
			std::ifstream file ( doc->c_str() );
			if ( !file )
			{
				std::cerr << "Cannot open " << *doc << ". Skipping...";
				continue;
			}

			Corpus::ExtSentencer::CCorpusSentenceReader sentencer(file, false, false);

			Corpus::CSentence *sentence = NULL;
			while ( ( sentence=sentencer.GetNext() ) != NULL )
			{
				ProcessSentence ( sentence );
				tokens += sentence->Size();
				Corpus::ExtSentencer::CSentenceReader::Free(sentence);
			}
			ptime end = microsec_clock::local_time();
			time_duration len = end - start;
			if( len.total_milliseconds()> 5000)
			{
				total_time += len;
				long total_time_ms = total_time.total_milliseconds();
				long eta_ms = (total_time_ms * (max-curr) ) / curr;
				std::cout << "Processed " << curr << " / " << max << " | tokens = " << tokens 
					<< " Elapsed: " << milliseconds(total_time_ms) 
					<< " ETA: "  << to_simple_string( milliseconds(eta_ms) )	<< "         \r";
				std::cout.flush();
				start = microsec_clock::local_time();
			}//if len > 5000
		}//for each document
	}//void Architect::ProcessDocuments()
}//namespace architect

