/*
    Copyright (C) 2012 Bartosz Broda, Jacek Bochenek, Łukasz Biłeńkij, 
    Damian Jaworski, Wojciech Mazur, Dominik Piasecki, Maciej Piasecki, 
    Radosław Ramocki
    
    This program is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by the Free
Software Foundation; either version 3 of the License, or (at your option)
any later version.

    This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE. 

    See the LICENSE and COPYING files for more details.
*/

#include "arch_headers.h"
#include "Architect.h"
#include "Config.h"

#include <stdlib.h>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <poliqarp_reader/PoliqarpSentencer.h>
#include <poliqarp_reader/PoliqarpReader.h>


#ifdef JOSKIPI_OPERATOR_TEST
std::ofstream file("/mnt/data2/zdania_kipi_coord_puszczone_w_b4");
#endif// JOSKIPI_OPERATOR_TEST

namespace me_architect
{
	Corpus::CSentence* Architect::TransformSentenceOnCurrentPosition(
			Corpus::CSentence* old, Triplets &t, size_t &removedBeforeCurrentPos,
			LexicalUnits::LexicalUnitPtr lu)
	{
		Corpus::CToken *substitute = old->GetToken(t.headId);
		std::string lu_base = Corpus::CWordDictionary::GetInstance()->GetAt(lu->GetBaseId() );

		removedBeforeCurrentPos = 0;
		old->EraseTokens(t.constituents);
		if( lu->GetConstituentsCount() > 1 )
			substitute->SetOrth( lu_base );
		for(size_t i = 0; i < substitute->GetLexCount(); i++)
		{
			Corpus::CLexem *lex = substitute->GetLexem(i);
			lex->SetBase( lu_base );
		}

		size_t position = ( size_t ) old->GetPosition();
		for (std::set<size_t>::iterator i = t.constituents.begin(); i
											!= t.constituents.end() && *i < position; i++)
			removedBeforeCurrentPos++;

		return old;
	}//Corpus::CSentence* Architect::TransformSentenceOnCurrentPosition

	Architect::Architect()
	{
	}

	Matrices::CCSMatrixPtr Architect::getMatrix()
	{
		return m_pMatrix;
	}
	void Architect::Start()
	{

		std::cout << "Initalizing...\n";
		Initalize();
		std::cout << "Starting...\n";
		ProcessBinaryCorpus();
#ifdef JOSKIPI_OPERATOR_TEST
		file.close();
		std::cout << "\nSentences retrieved..." << std::endl;
		exit(0);
#endif

		/*
				std::cout << "Saving data..." << std::endl;
				MatrixSaver ms;
				ms.SaveToSuperMatrix(m_pMatrix, m_stLexicalUnitCount);
				LexicalUnits::UnigramDictionary::GetInstance()->Save(config->m_sWorkdir + config->m_sMatrixName +"/udict.txt");
				*/
	}

	void Architect::ProcessSentence(Corpus::CSentence* sentence)
	{
		if( Config::GetInstance()->m_bFilterDisamb)
			sentence->FilterDisamb();
		m_stTokenCount += sentence->Size();
		// position in sentence -> number of row
		std::map<size_t, size_t> relevantRows;

		std::map<size_t, size_t> relevantColumns;

		CheckSentence(sentence, relevantRows, relevantColumns);
		m_stLexicalUnitCount+= (size_t)sentence->Size();//sentence is transformed
		CheckConditions(sentence, relevantRows, relevantColumns);

	}//void Architect::ProcessSentence(Corpus::CSentence* sentence)

	void Architect::CheckConditions(Corpus::CSentence * &sentence,
									std::map<size_t, size_t> &relevantRows,
									std::map<size_t, size_t> &relevantColumns)
	{
		Config *config = Config::GetInstance();
		for (std::map<size_t, size_t>::iterator inSentenceToRow =
			 relevantRows.begin(); inSentenceToRow != relevantRows.end(); inSentenceToRow++)
		{


			sentence->Goto( (int) inSentenceToRow->first );
			for (std::map<size_t, size_t>::iterator column = relevantColumns.begin(); column
															 != relevantColumns.end(); column++)
			{



				bool resultPositive = (config->m_vConditions)[column->second]->Evaluate(*sentence);
				if (resultPositive)
				{
					//
#ifndef JOSKIPI_OPERATOR_TEST

					m_pMatrix->AddToCell(1.0, inSentenceToRow->second, column->second);
#else// JOSKIPI_OPERATOR_TEST
					Config* config = Config::GetInstance();
					// s1; pos1; s2; pos2; zdanie_orth_z_podkresleniem
					size_t pos1 = inSentenceToRow->first;
					LexicalUnits::LexicalUnitPtr pRowUnit = config->m_vLexicalUnits[ inSentenceToRow->second ];

					size_t pos2 = column->first;
					LexicalUnits::LexicalUnitPtr pColumnUnit = config->m_vLexicalUnits[ column->second ];
					// wypisz zdanie
					std::stringstream s;
					for(size_t sent_pos = 0; sent_pos < sentence->Size(); sent_pos++)
					{
						if(sent_pos == pos1)
							s << "*";
						if(sent_pos == pos2)
							s << "#";

						s << (*sentence)[sent_pos]->GetOrth();

						if(sent_pos == pos1)
							s << "*";
						if(sent_pos == pos2)
							s << "#";

						s << " ";
					}
					//std::cout << s.str() << std::endl;
					file << s.str() << std::endl;
#endif //JOSKIPI_OPERATOR_TEST

				}
			}
		}
	}//void CheckConditions(

	void Architect::CheckSentence(Corpus::CSentence * &sentence,
								  std::map<size_t, size_t> &relevantRows,
								  std::map<size_t, size_t> &relevantColumns)
	{
		std::set<size_t> relevantLUs;
		FindAllOccurrences(sentence, relevantLUs);
		// where have we finished after transformation
		int lastI = 0;
		// was the whole sentence checked
		bool checked = false;


		while ( !checked)
		{
			// current position in the sentence
			int i = lastI;
			// true if any lexical unit has been choosen
			bool choosen = false;

			for (; i < sentence->Size() && !choosen; i++)
			{
				std::vector< Triplets> foundLUs;

				sentence->Goto(i);
				FindAllPotentialForPosition(sentence, relevantLUs, foundLUs);
				// if unit was not found on currentt position, then go to next position in sentence
				if (foundLUs.size() == 0)
					continue;
				LexicalUnits::LexicalUnitPtr lu;
				Triplets &t = foundLUs[0];
				choosen = ChooseLU(i, relevantRows, relevantColumns,
								   t, foundLUs, lu);

				//transform sentence, set new as a current
				sentence->Goto(i);
				size_t removedBefore = 0;
				//sentence transformation
				sentence = TransformSentenceOnCurrentPosition(sentence, t, removedBefore, lu);
				if (removedBefore != 0)
					i -=(int) removedBefore;
				//the loop will always breka in here  (choosen must be true now), but we don't want to have i++ twice
				break;
			}//for ( int i = 0 ; i < sentence->Size() && !choosen; i++ )
			lastI = i+1;
			if (lastI >= sentence->Size() )
				checked = true;
		}//while !checked
	}//void CheckSentence

	bool Architect::ChooseLU(int pos, std::map<size_t, size_t> &relevantRows,
							 std::map<size_t, size_t> &relevantColumns, Triplets &t,
							 std::vector< Triplets> &foundLUs, LexicalUnits::LexicalUnitPtr &lu)
	{
		LexicalUnits::UnigramDictionary * dict = LexicalUnits::UnigramDictionary::GetInstance();
		Config *config = Config::GetInstance();
		bool choosen = false;
		bool choosenRow = false;
		bool choosenColumn = false;
		// choose first LU, but check rows first
		for (size_t luIndex = 0; luIndex < foundLUs.size() && !choosen; luIndex++)
		{
			if (!choosenRow && m_mUnit2Row.count(foundLUs[luIndex].lexId) )
			{
				choosenRow = true;
				if (choosenColumn)
					choosen = true;
				lu = (config->m_vLexicalUnits)[ foundLUs[luIndex].lexId ];
				t = foundLUs[luIndex];
				relevantRows.insert(std::pair<size_t, size_t> (pos, m_mUnit2Row[foundLUs[luIndex].lexId ]) );
				dict->AddLexicalUnit( lu );
			}

			if(!choosenColumn && m_mUnit2Column.count( foundLUs[luIndex].lexId ) )
			{
				choosenColumn = true;
				if(!choosenRow)
				{
					t = foundLUs[luIndex];
					lu = (config->m_vLexicalUnits)[ foundLUs[luIndex].lexId ];
				}
				else
					choosen = true;
				relevantColumns.insert( std::pair<size_t, size_t> ( pos, m_mUnit2Column[foundLUs[luIndex].lexId] ) );
				dict->AddLexicalUnit( (config->m_vLexicalUnits)[ foundLUs[luIndex].lexId ]);
			}
		}

		// if the choosen lexical unit was not in row's lexical unit, then choose first unit out
		// of column's description
		if(!choosenRow && !choosenColumn)
		{
			throw std::runtime_error( "!choosenRow && ! choosenColumn" );
		}

		return choosen;
	}//ChooseLU(std::map<size_t, size_t>  &relevantRows,

	void Architect::FindAllPotentialForPosition(Corpus::CSentence *sentence, std::set<size_t> & relevantLUs,
												std::vector< Triplets> &foundLUs)
	{
		Config *config = Config::GetInstance();
		int maxConstituentsFound = -1;
		//find all possible LU for this position
		for(std::set<size_t>::iterator luIndexIter = relevantLUs.begin();
		luIndexIter != relevantLUs.end(); luIndexIter++)
		{

			LexicalUnits::LexicalUnit &lu= *(config->m_vLexicalUnits)[ *luIndexIter ];
			size_t headId;
			std::set<size_t> constituents;

			if( lu.CheckCurrentPosition(*sentence, headId, constituents) &&
				maxConstituentsFound <= (int) lu.GetConstituentsCount() )
			{
				if( (int)lu.GetConstituentsCount()> maxConstituentsFound )
				{
					maxConstituentsFound = (int) lu.GetConstituentsCount();
					foundLUs.clear();
				}
				Triplets t;
				t.lexId = *luIndexIter;
				t.constituents = constituents;
				t.headId = headId;
				foundLUs.push_back( t );
			}
		}
	}//FindAllPotentialForPosition(

	void Architect::FindAllOccurrences(Corpus::CSentence *sentence, std::set<size_t> &relevantLUs)
	{
		// preliminary check - check which lexical units can possibly occure in current sentence
		for(int i = 0; i < sentence->Size(); i++)
		{
			Corpus::CToken *t = sentence->operator [ ](i);
			for(size_t lex_no = 0; lex_no < t->GetLexCount(); lex_no++)
			{
				Corpus::CLexem *pDisamb = t->GetLexem(lex_no);
				if( pDisamb->GetDisamb() == false)
					continue;
				int base = pDisamb->GetBaseIdx();

				std::multimap<int, size_t>::iterator find = m_mBase2PotentialUnit.find(base);
				if(find == m_mBase2PotentialUnit.end() )
					continue;
				for(; find != m_mBase2PotentialUnit.end() && find->first == base; find++)
					relevantLUs.insert(find->second);
			}
		}//for each token in sentence
	}//void Architect::FindAllOccurrences(

	void Architect::Initalize()
	{
		Config* config = Config::GetInstance();

		//clear data
		m_mUnit2Row.clear();
		m_mUnit2Column.clear();
		m_mBase2PotentialUnit.clear();
		LexicalUnits::UnigramDictionary::GetInstance()->Clear();
		m_stTokenCount = 0;
		m_stLexicalUnitCount = 0;

		//create indices
		for ( size_t i = 0; i < config->rowLU.size(); i++ )
			m_mUnit2Row.insert ( std::pair<size_t, size_t> ( ( config->rowLU ) [i],i ) );

		for ( size_t i = 0; i < config->columnLU.size(); i++ )
			m_mUnit2Column.insert ( std::pair<size_t, size_t> ( ( config->columnLU ) [i],i ) );

		Corpus::CWordDictionary * dict = Corpus::CWordDictionary::GetInstance();
		int sie = dict->Add("się");
		std::set<int> potential;
		for ( size_t i = 0; i < config->m_vLexicalUnits.size(); i++ )
		{
			( config->m_vLexicalUnits ) [i]->GetPotentialBases ( potential );
			for ( std::set<int>::iterator j = potential.begin(); j != potential.end();j++ )
			{
				if( *j == sie )
					continue;
				m_mBase2PotentialUnit.insert ( std::pair<int, size_t> ( *j,i ) );
			}
		}

		//init matrix
		m_pMatrix = boost::shared_ptr<Matrices::CCSMatrix> ( new Matrices::CCSMatrix ( "data", config->rowLU.size(), config->columnLU.size()) );

	}//void Architect::InitalizeMatrix()

	int Architect::getCorpusSize()
	{
		return corpusSize;
	}

	void Architect::ProcessBinaryCorpus()
	{
		char processorName[MPI_MAX_PROCESSOR_NAME];
		int nameLen;
		int myRank;
		MPI_Get_processor_name( processorName, &nameLen );
		MPI_Comm_rank(MPI_COMM_WORLD,&myRank);
		boost::filesystem::create_directory(Config::GetInstance()->tempDictDir);
		std::stringstream ss;
		ss<<Config::GetInstance()->tempDictDir<<"globalDict_rank"<<myRank<<".txt";
		std::string globalDictFileName = ss.str();

		std::map<std::string, size_t> skippedTokens;
		std::map<std::string, size_t> skippedSentences;

		std::ofstream globalDictFile(globalDictFileName.c_str());
		Corpus::CWordDictionary::GetInstance()->Serialize(globalDictFile);
		globalDictFile.close();

		using namespace boost::posix_time;
		ptime start = microsec_clock::local_time();
		time_duration total_time;
		corpusSize = 0;
		size_t query_size = Config::GetInstance()->query_size;
		for(int i=0;i<(int)Config::GetInstance()->m_sCorpusFiles.size();i++)
		{
			start = microsec_clock::local_time();
			total_time = time_duration();
			std::string corpusName = Config::GetInstance()->m_sCorpusFiles[i];
			PoliqarpTools::CPoliqarpReader cpr(Config::GetInstance()->m_sCorpusFiles[i], false, false);
			std::cout<<"Corpus "<<corpusName<<" opened.\n";

			skippedTokens[corpusName] = 0;
			skippedSentences[corpusName] = 0;

			std::stringstream query;
			query << "[]{0," << query_size << "} within s";

			if(!cpr.CompileQuery(query.str().c_str() ) )
				throw std::runtime_error("Failed to compile query");
			size_t max = cpr.GetCorpusSize();
			corpusSize += cpr.GetCorpusSize();
			PoliqarpTools::CPoliqarpSentencer sentencer(&cpr);
			Corpus::CSentence * sentence;
			int tokens = 0;
			std::cout << "Starting... " << std::endl << std::endl;

			int tempTokenCounter=0;
			int ignoreSentences = 0;
			while( (sentence = sentencer.GetNext()))
			{
				if ((size_t)sentence->Size() <= query_size )
				{
					ProcessSentence ( sentence );
					tokens += sentence->Size();
					tempTokenCounter+= sentence->Size();

				}
				else
				{
					if (ignoreSentences>0)
					{
						ignoreSentences--;
					}
					else
					{
						ignoreSentences = ((size_t)sentence->Size()) / query_size;
						skippedTokens[corpusName] += (size_t)sentence->Size();
						skippedSentences[corpusName] ++;
					}
				}
				sentencer.Free(sentence);
				ptime end = microsec_clock::local_time();
				time_duration len = end - start;
				if( len.total_milliseconds()> 15000)
				{
					total_time += len;
					long total_time_ms = total_time.total_milliseconds();
					long eta_ms = (total_time_ms * (max-tokens) ) / tokens;
					FILE* popen_stream;
					popen_stream = popen("free -m | head -n 3 | tail -n 1 | awk -F ' '  '{print $4}'","r");
					char * ram = new char[30];
					fgets(ram,30, popen_stream);
					std::string ram_str(ram);
					boost::trim(ram_str);
					int val = boost::lexical_cast<int>(ram_str);
					std::cout << "Hostname(" << myRank << "): "<<  processorName<<" processed " << tokens << " / " << max
							<< " Elapsed: " << milliseconds(total_time_ms) 	<< " ETA: " << to_simple_string( milliseconds(eta_ms) )
							<< "\n";
					if (val<128)
					{
						std::cout<<"\033[31;1m!!!!!!WARNING!!!!!! ONLY "<< val << "MB OF FREE RAM LEFT ON HOST "<< processorName <<
								" (" << myRank << ")\033[0m\n";
					}
					else
					{
						if (val<512)
						{
							std::cout<<"Warning: only "<< val << " MB of free RAM left on host " << processorName <<
							" (" << myRank << ")\n";
						}
					}

					std::cout.flush();
					start = microsec_clock::local_time();
					pclose(popen_stream);
					delete [] ram;
				}//if len > 5000
				if(tempTokenCounter > 20000000)// Every 20M tokens flush dictionary
				{
					std::ifstream globalDictFile(globalDictFileName.c_str());
					Corpus::CWordDictionary::GetInstance()->Deserialize(globalDictFile);
					globalDictFile.close();
					tempTokenCounter = 0;

				}
			}
			//Every corpus flush dictionary
			std::ifstream globalDictFile(globalDictFileName.c_str());
			Corpus::CWordDictionary::GetInstance()->Deserialize(globalDictFile);
			globalDictFile.close();
			if (myRank == 1)
			{
				std::stringstream skipped_filename_stream;
				skipped_filename_stream<<Config::GetInstance()->tempDictDir<<"skipped_statistics"<<myRank<<".csv";
				std::string skipped_filename = skipped_filename_stream.str();

				std::ofstream skippedFile(skipped_filename.c_str());

				skippedFile << "Corpus;skipped sentences;skipped tokens\n";

				size_t sumSentences = 0;
				size_t sumTokens = 0;

				for (std::map<std::string, size_t>::iterator i = skippedTokens.begin(); i != skippedTokens.end(); ++i)
				{
					skippedFile << (*i).first << ";" << skippedSentences[(*i).first] << ";" << (*i).second << "\n";
					sumSentences += skippedSentences[(*i).first];
					sumTokens += (*i).second;
				}
				skippedFile << "\n";
				skippedFile << "TOTAL;" << sumSentences << ";" << sumTokens;

				skippedFile.close();
			}

		}
		boost::filesystem::remove(globalDictFileName);
	}



}//namespace architect

