/*
    Copyright (C) 2012 Bartosz Broda, Jacek Bochenek, Łukasz Biłeńkij, 
    Damian Jaworski, Wojciech Mazur, Dominik Piasecki, Maciej Piasecki, 
    Radosław Ramocki
    
    This program is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by the Free
Software Foundation; either version 3 of the License, or (at your option)
any later version.

    This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE. 

    See the LICENSE and COPYING files for more details.
*/

#include "arch_headers.h"
#include <deque>
#include "boost/date_time/posix_time/posix_time.hpp"

#include "Architect.h"
#include "Config.h"
#include "UnigramDictionary.h"
#include "MatrixSaver.h"

#include <poliqarp_reader/PoliqarpSentencer.h>
#include <poliqarp_reader/PoliqarpReader.h>

namespace hal_architect
{
	Corpus::CSentence* Architect::TransformSenetenceOnCurrentPosiotion ( Corpus::CSentence* old,
									     Triplets &t, size_t &removedBeforeCurrentPos,
									     LexicalUnits::LexicalUnitPtr lu )
	{
		removedBeforeCurrentPos = 0;
		old->EraseTokens ( t.constituents );
		size_t position = ( size_t ) old->GetPosition();
		for ( std::set<size_t>::iterator i = t.constituents.begin();
		i != t.constituents.end() && *i < position ;i++ )
			removedBeforeCurrentPos++;
		return old;
	}//Corpus::CSentence* Architect::TransformSenetenceOnCurrentPosiotion

	Architect::Architect()
	{
		m_pDistFunct = DistanceOne;
	}//Architect::Architect()

	void Architect::Start()
	{
		Config *config = Config::GetInstance();
		std::cout << "Initalizing...\n";
		Initalize();
		std::cout << "Starting...\n";

		switch(config->m_ContextType)
		{
		case Window:
			std::cout << "Window mode ( " << config->m_stWindowLeft << ", " << config->m_stWindowRight << ")" << std::endl;
			std::cerr << "WARNING: matrix contains tokens count, NOT lexical units count!" << std::endl;
			break;
		case Sentence:
			std::cout << "Sentence mode..." << std::endl;
			break;
		case HAL:
			throw std::runtime_error("HAL mode is not yet supported!");
		}

		if(config->m_bBinaryCorpus)
			ProcessBinaryCorpus();
		else
			ProcessDocuments();
		std::cout << "Saving data..." << std::endl;
		MatrixSaver ms;
		ms.SaveToSuperMatrix( m_pMatrix, m_stLexicalUnitCount);
		LexicalUnits::UnigramDictionary::GetInstance()->Save(config->workdir + config->m_sMatrixName +"/udict.txt");
	}//void Architect::Start()
	
	void Architect::ProcessSentence ( Corpus::CSentence* sentence )
	{
		m_stTokenCount += sentence->Size();
		// position in sentence -> number of row
		std::map<size_t, size_t> relevantRows;
		std::map<size_t, size_t> relevantColumns;
		
		CheckSentence ( sentence, relevantRows, relevantColumns);
		CountCoocurrences(sentence, relevantRows, relevantColumns);

	}//void Architect::ProcessSentence(Corpus::CSentence* sentence)

	void Architect::CountCoocurrences ( Corpus::CSentence * &sentence, std::map<size_t, size_t>  &relevantRows,
					    std::map<size_t,size_t> &relevantColumns )
	{
		
		Config *config = Config::GetInstance();
		switch(config->m_ContextType)
		{
		case Window:
			CountCoocurrencesForWindowCenter(sentence, relevantRows, relevantColumns);
			break;
		case Sentence:
			CountCoocurrencesEveryWithEvery(sentence, relevantRows, relevantColumns);
			break;
		case HAL:
			CountCoocurrencesForHALWindow(sentence, relevantRows, relevantColumns);
			break;
		}
	}//void CountCoocurrences(

	void Architect::CountCoocurrencesEveryWithEvery( Corpus::CSentence * &sentence, std::map<size_t, size_t>  &relevantRows, 
							 std::map<size_t, size_t> &relevantColumns )
	{
		for(std::map<size_t, size_t>::iterator inSentenceToRow = relevantRows.begin();
		inSentenceToRow != relevantRows.end(); inSentenceToRow++)
		{
			sentence->Goto( (int) inSentenceToRow->first );

			for(std::map<size_t, size_t>::iterator column = relevantColumns.begin();
			column != relevantColumns.end(); column++)
			{
				if(column->second == inSentenceToRow->second)
					continue;
				m_pMatrix->AddToCell(1.0, inSentenceToRow->second, column->second);
			}
		}
	}//void CountCoocurrencesEveryWithEvery( Corpus::CSentence * &sentence, std::map<size_t, size_t>  &relevantRows, std::set<size_t> &relevantColumns )

	void Architect::CountCoocurrencesForWindowCenter( Corpus::CSentence * &sentence, std::map<size_t, size_t>  &relevantRows, 
							  std::map<size_t, size_t> &relevantColumns )
	{
		Config *config = Config::GetInstance();
		size_t left = config->m_stWindowLeft;
		size_t right = config->m_stWindowRight;
		size_t sentenceMiddle = size_t( (left+right) / 2 );

		std::set<size_t> columns_used;

		std::map<size_t, size_t>::iterator middle = relevantRows.find( sentenceMiddle );
		if(middle == relevantRows.end() )
			return;


		for(std::map<size_t, size_t>::iterator column = relevantColumns.begin();
		column != relevantColumns.end(); column++)
		{
			if(column->first == middle->first)
				continue;

			double weight_factor = 1.0;
			if(middle->first > column->second) // on the left
				weight_factor = (*m_pDistFunct)( middle->first - column->first );
			else
				weight_factor = (*m_pDistFunct) ( column->first - middle->first);

			m_pMatrix->AddToCell(weight_factor, middle->second, column->second);
		}


	}//void CountCoocurrencesForWindowCenter( Corpus::CSentence * &sentence, std::map<size_t, size_t>  &relevantRows, std::set<size_t> &relevantColumns )

	void Architect::CountCoocurrencesForHALWindow( Corpus::CSentence * &sentence, std::map<size_t, size_t>  &relevantRows, 
						       std::map<size_t, size_t> &relevantColumns )
	{
		throw std::runtime_error("HAL matrix not yet implemented");
	}//void CountCoocurrencesForHALWindow( Corpus::CSentence * &sentence, std::map<size_t, size_t>  &relevantRows, std::set<size_t> &relevantColumns )

	
	void Architect::CheckSentence ( Corpus::CSentence * &sentence, 
					std::map<size_t, size_t>  &relevantRows,
					std::map<size_t, size_t> &relevantColumns )
	{
		std::set<size_t> relevantLUs;
		FindAllOccurrences ( sentence, relevantLUs );
		// where have we finished after transformation
		int lastI = 0;
		// was the whole sentence checked
		bool checked = false;

		while ( !checked )
		{
			// current position in the sentence
			int i = lastI;
			// true if any lexical unit has been choosen
			bool choosen = false;			

			for (  ; i < sentence->Size() && !choosen; i++ )
			{
				std::vector< Triplets > foundLUs;

				sentence->Goto ( i );
				FindAllPotentialForPosition ( sentence, relevantLUs, foundLUs );
				// if unit was not found on currentt position, then go to next position in sentence	
				if ( foundLUs.size() == 0 )
					continue;
				LexicalUnits::LexicalUnitPtr lu;
				Triplets &t = foundLUs[0];
				choosen = ChooseLU(i,relevantRows, relevantColumns, t, foundLUs, lu); 
				
				//transform sentence, set new as a current
				sentence->Goto(i);
				size_t removedBefore;
				//sentence transformation
				sentence = TransformSenetenceOnCurrentPosiotion(sentence,t, removedBefore, lu);
				if(removedBefore != 0 )
					i -=(int) removedBefore;
				//the loop will always breka in here  (choosen must be true now), but we don't want to have i++ twice
				break;
			}//for ( int i = 0 ; i < sentence->Size() && !choosen; i++ )
			lastI = i+1;
			if(lastI >= sentence->Size() )
				checked = true;
		}//while !checked
	}//void CheckSentence
	
	bool Architect::ChooseLU(int pos, std::map<size_t, size_t>  &relevantRows,
				 std::map<size_t, size_t> &relevantColumns,
				 Triplets &t,
				 std::vector< Triplets >  &foundLUs, LexicalUnits::LexicalUnitPtr &lu)
	{
		LexicalUnits::UnigramDictionary * dict = LexicalUnits::UnigramDictionary::GetInstance();
		Config *config = Config::GetInstance();
		bool choosen = false;
		bool choosenRow = false;
		bool choosenColumn = false;
		// choose first LU, but check rows first
		for(size_t luIndex = 0; luIndex < foundLUs.size() && !choosen; luIndex++)
		{
			if(!choosenRow && m_mUnit2Row.count( foundLUs[luIndex].lexId ) )
			{
				choosenRow = true;
				if(choosenColumn)
					choosen = true;
				lu = (config->m_vLexicalUnits)[ foundLUs[luIndex].lexId  ];
				dict->AddLexicalUnit( lu );
				t = foundLUs[luIndex];
				relevantRows.insert(std::pair<size_t, size_t> (pos, m_mUnit2Row[foundLUs[luIndex].lexId  ]) );
			}
			if(!choosenColumn && m_mUnit2Column.count( foundLUs[luIndex].lexId ) )
			{
				choosenColumn = true;
				if(!choosenRow)
				{
					t = foundLUs[luIndex];
					lu = (config->m_vLexicalUnits)[ foundLUs[luIndex].lexId  ];
				}
				else
					choosen = true;
				relevantColumns.insert(std::pair<size_t, size_t> (pos, m_mUnit2Column[foundLUs[luIndex].lexId  ]) );
				dict->AddLexicalUnit(  (config->m_vLexicalUnits)[ foundLUs[luIndex].lexId  ]);
			}
		}

		// if the choosen lexical unit was not in row's lexical unit, then choose first unit out 
		// of column's description
		if(!choosenRow)
		{
			choosen = true;
			lu = (config->m_vLexicalUnits)[ foundLUs[0].lexId  ];
			relevantColumns.insert(std::pair<size_t, size_t>(pos, m_mUnit2Column[ foundLUs[0].lexId ]) ) ;
		}
		
		return choosen;
	}//ChooseLU(std::map<size_t, size_t>  &relevantRows,

	void Architect::FindAllPotentialForPosition(Corpus::CSentence *sentence, std::set<size_t> & relevantLUs,
						    std::vector< Triplets >  &foundLUs)
	{
		Config *config = Config::GetInstance();
		int maxConstituentsFound = -1;
		//find all possible LU for this position
		for(std::set<size_t>::iterator luIndexIter = relevantLUs.begin();
		luIndexIter != relevantLUs.end(); luIndexIter++)
		{

			LexicalUnits::LexicalUnit &lu= *(config->m_vLexicalUnits)[ *luIndexIter ];
			size_t headId;
			std::set<size_t> constituents;

			if( lu.CheckCurrentPosition(*sentence, headId, constituents) &&
			    maxConstituentsFound <= (int) lu.GetConstituentsCount() )
			{
				if( (int)lu.GetConstituentsCount() > maxConstituentsFound )
				{
					maxConstituentsFound = (int) lu.GetConstituentsCount();
					foundLUs.clear();
				}
				Triplets t;
				t.lexId = *luIndexIter;
				t.constituents = constituents;
				t.headId = headId;
				foundLUs.push_back( t );
			}
		}
	}//FindAllPotentialForPosition(
	
	void Architect::FindAllOccurrences(Corpus::CSentence *sentence, std::set<size_t> &relevantLUs)
	{	
		// preliminary check - check which lexical units can possibly occure in current sentence
		for(int i = 0; i < sentence->Size(); i++)
		{
			Corpus::CToken *t = sentence->operator [ ](i);			
			for(size_t lex_no = 0; lex_no < t->GetLexCount(); lex_no++)
			{
				Corpus::CLexem *pDisamb = t->GetLexem(lex_no);

				if( pDisamb->GetDisamb() == false)
					continue;
				int base = pDisamb->GetBaseIdx();

				std::multimap<int, size_t>::iterator find = m_mBase2PotentialUnit.find(base);
				if(find == m_mBase2PotentialUnit.end() )
					continue;
				for(; find != m_mBase2PotentialUnit.end() && find->first == base; find++)
					relevantLUs.insert(find->second);
			}
		}//for each token in sentence
	}//void Architect::FindAllOccurrences(

	void Architect::Initalize()
	{
		Config* config = Config::GetInstance();

		//clear data
		m_mUnit2Row.clear();
		m_mUnit2Column.clear();
		m_mBase2PotentialUnit.clear();
		LexicalUnits::UnigramDictionary::GetInstance()->Clear();
		m_stTokenCount = 0;
		m_stLexicalUnitCount = 0;

		//create indices
		for ( size_t i = 0; i < config->rowLU.size(); i++ )
			m_mUnit2Row.insert ( std::pair<size_t, size_t> ( ( config->rowLU ) [i],i ) );

		for ( size_t i = 0; i < config->columnLU.size(); i++ )
			m_mUnit2Column.insert ( std::pair<size_t, size_t> ( ( config->columnLU ) [i],i ) );

		//Corpus::CWordDictionary * dict = Corpus::CWordDictionary::GetInstance();
		std::set<int> potential;
		for ( size_t i = 0; i < config->m_vLexicalUnits.size(); i++ )
		{
			( config->m_vLexicalUnits ) [i]->GetPotentialBases ( potential );
			for ( std::set<int>::iterator j = potential.begin(); j != potential.end() ;j++ )
				m_mBase2PotentialUnit.insert ( std::pair<int, size_t> ( *j,i ) );
		}

		//init matrix
		m_pMatrix = boost::shared_ptr<Matrices::CRSMatrix> ( new Matrices::CRSMatrix ( "data", config->rowLU.size(), config->columnLU.size()) );

	}//void Architect::InitalizeMatrix()

	void Architect::ProcessBinaryCorpusByToken()
	{
		using namespace boost::posix_time;
		Config* config = Config::GetInstance();
		ptime start = microsec_clock::local_time();
		time_duration total_time;
		std::deque<Corpus::CToken*> tokens;
		size_t window_size = config->m_stWindowRight + config->m_stWindowRight + 1;

		PoliqarpTools::CPoliqarpReader cpr(Config::GetInstance()->m_sCorpusFile, false, false);
		if(!cpr.CompileQuery("[]"))
			throw std::runtime_error("Failed to compile querry");
		size_t max = cpr.GetCorpusSize();

		int tokens_processed = 0;
		std::cout << "Starting... " << std::endl << std::endl;

		Corpus::CToken *pToken = NULL;
		while( (pToken = cpr.GetNext()))
		{
			tokens_processed++;

			m_stLexicalUnitCount++;

			tokens.push_back(pToken);
			if(tokens.size() < window_size )
				continue;

			Corpus::CSentence sentence(tokens, false);
			if(Config::GetInstance()->m_bFilterDisamb)
				sentence.FilterDisamb();
			ProcessSentence( &sentence );
			pToken = tokens.front();
			cpr.Free(pToken);
			tokens.pop_front();

			ptime end = microsec_clock::local_time();
			time_duration len = end - start;
			if( len.total_milliseconds()> 5000)
			{
				total_time += len;
				long total_time_ms = total_time.total_milliseconds();
				long eta_ms = (total_time_ms * (max-tokens_processed) ) / tokens_processed;
				std::cout << "Processed " << tokens_processed << " / " << max << " Elapsed: " << milliseconds(total_time_ms) 
						<< " ETA: " << to_simple_string( milliseconds(eta_ms) ) 	<< "         \r";
				std::cout.flush();
				start = microsec_clock::local_time();
			}//if len > 5000
		}

		while(tokens.size() > 1 )
		{
			Corpus::CSentence sentence(tokens, false);
			if(Config::GetInstance()->m_bFilterDisamb)
				sentence.FilterDisamb();
			ProcessSentence( &sentence );
			pToken = tokens.front();
			cpr.Free(pToken);
			tokens.pop_front();
		}

		if(tokens.size())
		{
			pToken = tokens.front();
			cpr.Free(pToken);
			tokens.pop_front();
		}
		std::cout << std::endl;
	}
	void Architect::ProcessBinaryCorpusBySentence()
	{
		using namespace boost::posix_time;
		ptime start = microsec_clock::local_time();
		time_duration total_time;

		PoliqarpTools::CPoliqarpReader cpr(Config::GetInstance()->m_sCorpusFile, false, false);
		if(!cpr.CompileQuery("[]{0,10000} within s"))
			throw std::runtime_error("Failed to compile querry");
		size_t max = cpr.GetCorpusSize();
		PoliqarpTools::CPoliqarpSentencer sentencer(&cpr);
		Corpus::CSentence * sentence;
		int tokens = 0;
		std::cout << "Starting... " << std::endl << std::endl;

		while( (sentence = sentencer.GetNext()))
		{
			if(Config::GetInstance()->m_bFilterDisamb)
				sentence->FilterDisamb();

			ProcessSentence ( sentence );
			tokens += sentence->Size();
			m_stLexicalUnitCount+= sentence->Size();
			sentencer.Free(sentence);
			ptime end = microsec_clock::local_time();
			time_duration len = end - start;
			if( len.total_milliseconds()> 5000)
			{
				total_time += len;
				long total_time_ms = total_time.total_milliseconds();
				long eta_ms = (total_time_ms * (max-tokens) ) / tokens;
				std::cout << "Processed " << tokens << " / " << max << " Elapsed: " << milliseconds(total_time_ms) 
						<< " ETA: " << to_simple_string( milliseconds(eta_ms) ) 	<< "           \r";
				std::cout.flush();
				start = microsec_clock::local_time();
			}//if len > 5000
		}
		std::cout << std::endl;
	}
	void Architect::ProcessBinaryCorpus()
	{
		Config * cfg = Config::GetInstance();

		switch(cfg->m_ContextType)
		{
		case Window:
			ProcessBinaryCorpusByToken();
			break;
		case Sentence:
			ProcessBinaryCorpusBySentence();
			break;
		case HAL:
			throw std::runtime_error("Binary HAL not yet implemented");
			break;
		}

	}

	void Architect::ProcessDocuments()
	{
		using namespace boost::posix_time;
		std::vector<std::string> &documents = Config::GetInstance()->documents;
		size_t curr = 0;
		size_t max = documents.size();
		ptime start = microsec_clock::local_time();
		time_duration total_time;
		for ( std::vector<std::string>::iterator doc = documents.begin(); doc != documents.end(); doc++ )
		{
			curr++;
			std::ifstream file ( doc->c_str() );
			if ( !file )
			{
				std::cerr << "Cannot open " << *doc << ". Skipping...";
				continue;
			}
			Corpus::Tokenisation::CCorpusTokenReader reader ( file,true,false );
			ProcessSingleDocument( reader );
			ptime end = microsec_clock::local_time();
			time_duration len = end - start;
			if( len.total_milliseconds() > 5000)
			{
				total_time += len;
				long total_time_ms = total_time.total_milliseconds();
				double doc_per_sec = double(curr*1000) / double(total_time_ms);
				long eta_ms = (total_time_ms * (max-curr) ) / curr;
				std::cout << "Processed " << curr << " / " << max 
						<< " | doc/sec = " << doc_per_sec << " ETA: " << to_simple_string( milliseconds(eta_ms) )
						<< "           \r";
				std::cout.flush();
				start = microsec_clock::local_time();
			}//if len > 5000
		}//for each document
		std::cout << std::endl;
	}//void Architect::ProcessDocuments()

	void Architect::ProcessSingleDocument(Corpus::Tokenisation::CCorpusTokenReader &reader)
	{
		Config * cfg = Config::GetInstance();

		switch(cfg->m_ContextType)
		{
		case Window:
			ProcessSingleDocWindowContext(reader);
			break;
		case Sentence:
			ProcessSingleDocSentenceContext(reader);
			break;
		case HAL:
			ProcessSingleDocHalContext(reader);
			break;
		}
	}//void Architect::ProcessSingleDocument(Corpus::Tokenisation::CCorpusTokenReader &reader)

	void Architect::ProcessSingleDocSentenceContext(Corpus::Tokenisation::CCorpusTokenReader &reader)
	{
		Corpus::ExtSentencer::CSentencer sentencer ( reader );

		Corpus::CSentence *sentence = NULL;
		while ( ( sentence=sentencer.GetNext() ) != NULL )
		{
			ProcessSentence ( sentence );
			m_stLexicalUnitCount += sentence->Size();
			sentencer.Free ( sentence );
		}
	}//void Architect::ProcessSingleDocSentenceContext(Corpus::Tokenisation::CCorpusTokenReader &reader)

	void Architect::ProcessSingleDocHalContext(Corpus::Tokenisation::CCorpusTokenReader &reader)
	{
		//TODO ProcessSingleDocHalContext not implemented
		throw std::runtime_error("ProcessSingleDocHalContext not implemented!");
	}//void Architect::ProcessSingleDocHalContext(Corpus::Tokenisation::CCorpusTokenReader &reader)


	void Architect::ProcessSingleDocWindowContext(Corpus::Tokenisation::CCorpusTokenReader &reader)
	{
		Config *config = Config::GetInstance();
		std::deque<Corpus::CToken*> tokens;
		size_t window_size = config->m_stWindowRight + config->m_stWindowRight + 1;

		Corpus::CToken *pToken = NULL;

		while( (pToken = reader.GetNext()) != NULL )
		{
			tokens.push_back(pToken);
			m_stLexicalUnitCount++;
			if(tokens.size() < window_size )
				continue;

			Corpus::CSentence sentence(tokens, false);
			ProcessSentence( &sentence );
			pToken = tokens.front();
			reader.Free(pToken);
			tokens.pop_front();
		}			

		while(tokens.size() > 1 )
		{
			Corpus::CSentence sentence(tokens, false);
			ProcessSentence( &sentence );
			pToken = tokens.front();
			reader.Free(pToken);
			tokens.pop_front();
		}
		if(tokens.size())
		{
			pToken = tokens.front();
			reader.Free(pToken);
			tokens.pop_front();
		}
	}//void Architect::ProcessSingleDocWindowContext(Corpus::Tokenisation::CCorpusTokenReader &reader)


	double DistanceOne(size_t)
	{
		return 1;
	}

}//namespace hal_architect
