/*
    Copyright (C) 2012 Bartosz Broda, Jacek Bochenek, Łukasz Biłeńkij, 
    Damian Jaworski, Wojciech Mazur, Dominik Piasecki, Maciej Piasecki, 
    Radosław Ramocki
    
    This program is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by the Free
Software Foundation; either version 3 of the License, or (at your option)
any later version.

    This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE. 

    See the LICENSE and COPYING files for more details.
*/

#include "doc_term_architect.h"
#include "Config.h"
#include "MatrixSaver.h"
#include <boost/thread/thread.hpp>
#include <boost/ref.hpp>

int main(int argc, char* argv[])
{
	Matrices::init_logging();
	std::cout << "Testujemy budowanie macierzy...\n";
	doc_term_architect::Config::Initalize(argc, argv);

	doc_term_architect::DocTermArchitect a;

	std::cout << "Start...\n";

	a.Start();

	return 0;
}


namespace doc_term_architect
{
	DocTermArchitect::DocTermArchitect()
	{
	}

	void DocTermArchitect::Start()
	{
		Config *c = Config::GetInstance();

		//split documents
		std::map<int, std::vector<size_t> > thread2docs;
		int thread_no = 0;
		for(size_t doc_no = 0; doc_no < c->m_vDocumentList.size(); doc_no++, thread_no++)
			thread2docs[ thread_no % c->m_iThreads ].push_back(doc_no);
		
		//spawn threads
		Matrices::CRSMatrixPtr pMatrix( new Matrices::CRSMatrix( c->m_sMatrixName, 
					c->m_vDocumentList.size(), c->m_vColumnTerms.size() ) );

		std::vector<std::pair<boost::thread*,DocTermMatrixBuilder*> > threads;
		for(int thread = 0; thread < c->m_iThreads; thread++)
		{
			DocTermMatrixBuilder *builder = new DocTermMatrixBuilder(thread2docs[thread],pMatrix);
			boost::reference_wrapper<DocTermMatrixBuilder> reference( *builder );
			threads.push_back( std::pair<boost::thread*, DocTermMatrixBuilder* > ( 
						new boost::thread( reference), builder) ) ;

		}

        for(int thread = 0; thread < c->m_iThreads; thread++)
			threads[thread].first->join();

		std::cout << "All threads finished...\n";

		size_t tokenCounter = 0;
		for(int thread = 0; thread < c->m_iThreads; thread++)
		{
			tokenCounter+= threads[thread].second->GetTokenCount();
			delete threads[thread].second;
			delete threads[thread].first;
		}

		// save dict
		if( c->m_iThreads < 2 && c->m_sDictionary.length() > 0)
		{
			Corpus::CWordDictionary *dict = Corpus::CWordDictionary::GetInstance();
			std::ofstream file(c->m_sDictionary.c_str() );
			if( ! file )
				throw std::runtime_error("Cannot open file for writing: " + c->m_sDictionary);
			dict->SerializeAsText(file);
			file.close();
		}

		//save matrix
		MatrixSaver saver;
		saver.SaveToSuperMatrix(pMatrix, tokenCounter);

	}

	//////////////////////////////////////////////////////////

	DocTermMatrixBuilder::DocTermMatrixBuilder(std::vector<size_t> documents, Matrices::CRSMatrixPtr pMatrix)
		: m_pMatrix(pMatrix),
		m_vMyDocuments(documents)
	{
		m_iThreadNo = thread_counter ++;
		m_stTokenCount = 0;
	}

	int DocTermMatrixBuilder::thread_counter = 1;
	boost::mutex DocTermMatrixBuilder::cout_mutex;

	size_t DocTermMatrixBuilder::GetTokenCount()
	{
		return m_stTokenCount;
	}

	void DocTermMatrixBuilder::operator() ()
	{
		{
			boost::mutex::scoped_lock lock(cout_mutex);
			std::cout << "[" << m_iThreadNo  << "] starting with " << m_vMyDocuments.size() << " documents." <<  std::endl;
		}
		Config *c = Config::GetInstance();
		//for each document
		for(size_t doc_no = 0; doc_no < m_vMyDocuments.size(); doc_no++)
		{
			std::string &doc = c->m_vDocumentList[ m_vMyDocuments[doc_no] ];
			std::ifstream file ( doc.c_str() );
			if ( !file )
			{
				std::cerr << "Cannot open " << doc << ". Skipping...";
				continue;
			}
			Corpus::Tokenisation::CCorpusTokenReader reader ( file,true );
			//for each term
			Corpus::CToken *token = NULL;
			Matrices::CompressedVectorPtr document_vector = m_pMatrix->GetCompressedRow( m_vMyDocuments[doc_no] );
			while( (token = reader.GetNext()) != NULL)
			{
				m_stTokenCount++;
				int baseId = token->DisambLexem().GetBaseIdx();
				Corpus::TFlexClass flex = token->DisambLexem().GetTag().GetFlex();
				std::map<Corpus::TFlexClass, std::map<int, size_t> >::iterator find = c->m_mTermIndex.find(flex);
				if( find == c->m_mTermIndex.end() )
				{
					Corpus::Tokenisation::CTokenReader::Free( token );
					continue;
				}
				std::map<int, size_t>::iterator inner_find = (find->second).find(baseId);
				if(inner_find == (find->second).end() )
				{
					Corpus::Tokenisation::CTokenReader::Free( token );
					continue;
				}
				size_t column_no = inner_find->second;
				document_vector->SetElement( column_no, (*document_vector)[column_no] + 1);
				Corpus::Tokenisation::CTokenReader::Free( token );

			}// for each term

			if( doc_no % 100 == 0 )
			{
				boost::mutex::scoped_lock lock(cout_mutex);
				std::cout << "[" << m_iThreadNo << "] Document " << doc_no << " / " << m_vMyDocuments.size()  << std::endl;
			}
		}//for each doc

		{
			boost::mutex::scoped_lock lock(cout_mutex);
			std::cout << "[" << m_iThreadNo  << "] finished" << std::endl;
		}
	}// operator()


}//namespace doc_term_architect
