/*
    Copyright (C) 2012 Bartosz Broda, Jacek Bochenek, Łukasz Biłeńkij, 
    Damian Jaworski, Wojciech Mazur, Dominik Piasecki, Maciej Piasecki, 
    Radosław Ramocki
    
    This program is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by the Free
Software Foundation; either version 3 of the License, or (at your option)
any later version.

    This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE. 

    See the LICENSE and COPYING files for more details.
*/

#include "Dictionary.h"


void onlyMasterDict()
{

	me_architect::Config *conf = me_architect::Config::GetInstance();
	boost::filesystem::create_directories( conf->m_sWorkdir);






	udict_map base_udict; // flex -> ( id -> count)
	for(std::set<Corpus::TFlexClass>::iterator iter = conf->flexes.begin(); iter != conf->flexes.end(); iter++)
		base_udict[*iter];// = std::map<int, size_t>();



	for(int i=0;i<(int)conf->m_sCorpusFiles.size();i++)
	{
		singleCorpusDict(i,base_udict);

	}//for each corpus


	std::cout << "\nWriting data..." << std::endl;



	write_bases(*conf, base_udict);


}

void singleCorpusDict(int corpNo,udict_map &base_udict)
{
	using namespace boost::posix_time;
	ptime start = microsec_clock::local_time();

	me_architect::Config *conf = me_architect::Config::GetInstance();


	Corpus::CSentence *pSentence = NULL;
	boost::shared_ptr<PoliqarpTools::CPoliqarpReader> pReader( new PoliqarpTools::CPoliqarpReader(conf->m_sCorpusFiles[corpNo]) );

	size_t query_size = conf->query_size;

	std::stringstream query;
	query << "[]{0," << query_size << "} within s";

	if(!pReader->CompileQuery(query.str().c_str() ) )
		throw std::runtime_error("Failed to compile query");


	size_t all_tokens = pReader->GetCorpusSize();
	boost::shared_ptr<PoliqarpTools::CPoliqarpSentencer> pSentencer( new PoliqarpTools::CPoliqarpSentencer(&*pReader) );

	std::cout << "Processing corpus... " << std::endl;
	while( (pSentence = pSentencer->GetNext()) )
	{
		if((size_t)pSentence->Size()<=query_size)

			for(int i = 0; i < pSentence->Size(); i++)
			{

			Corpus::CToken *pToken = (*pSentence)[i];


			add_base(pToken,base_udict,*conf);

			// time
			ptime end = microsec_clock::local_time();
			time_duration len = end - start;
			if( len.total_milliseconds()> 10000)
			{
				size_t pos = pReader->GetCorpusPos();
				std::cout <<"\n"<< pos / 1000000 << "M / " << all_tokens/1000000 << "M";
				std::cout.flush();
				start = microsec_clock::local_time();
			}

		} // for each token in a sentence
		pSentencer->Free(pSentence);
	}; // for each sentence

}
/*
std::multimap<size_t, std::string> prepare_bases(me_architect::Config &conf, std::map<Corpus::TFlexClass, std::map<int, size_t> > &base_udict)
{
	std::vector<std::string> result;
	std::set<std::string> stopList;
	std::string line;
	for(int i=0;i<(int)conf.stoplistFiles.size();i++)
	{
		std::ifstream stopListFile(conf.stoplistFiles[i].c_str());
		if(!stopListFile)
			throw std::runtime_error("Specified stoplist file not found!\n");
		while((std::getline(stopListFile,line)))
			stopList.insert(line);
	}

	std::multimap<size_t, std::string> count2flexlemma;
	Corpus::CWordDictionary *dict = Corpus::CWordDictionary::GetInstance();

	for(std::map<Corpus::TFlexClass, std::map<int, size_t> >::iterator iter = base_udict.begin(); iter != base_udict.end(); ++iter)
	{
		std::map<int, size_t> &inner = iter->second;
		std::string fc = Corpus::CTag::GetFlexClassString( iter->first );
		for(std::map<int, size_t>::iterator inner_iter = inner.begin(); inner_iter != inner.end(); ++inner_iter)
		{
			if((stopList.find(dict->GetAt(inner_iter->first))==stopList.end()) && inner_iter->second >=conf.min_tf)
			{
				std::string flexlemma = fc + ';' + dict->GetAt( inner_iter->first);
				count2flexlemma.insert( std::make_pair( inner_iter->second, flexlemma ) );
			}
		}
	}
	return count2flexlemma;

}
*/
std::vector<char> loadCharStoplist()
{
	me_architect::Config* conf = me_architect::Config::GetInstance();
	std::vector<char> charStoplist;
	std::string line;
	for(int i=0;i<(int)conf->charStoplistFiles.size();i++)
	{
		std::ifstream stopListFile(conf->charStoplistFiles[i].c_str());
		if(!stopListFile)
			throw std::runtime_error("Specified stoplist file not found!\n");
		while((std::getline(stopListFile,line)))
		{
			if(line.length()!=1)
				std::cerr<<"Wrong line in char stoplist file "<<conf->charStoplistFiles[i]<<std::endl;
			charStoplist.push_back(line[0]);
		}
		stopListFile.close();
	}
	return charStoplist;
}

void write_bases(me_architect::Config &conf, udict_map &base_udict)
{
	std::set<std::string> stopList;
	std::string line;
	for(int i=0;i<(int)conf.stoplistFiles.size();i++)
	{
		std::ifstream stopListFile(conf.stoplistFiles[i].c_str());
		if(!stopListFile)
			throw std::runtime_error("Specified stoplist file not found!\n");
		while((std::getline(stopListFile,line)))
			stopList.insert(line);
	}

	std::multimap<size_t, std::string> count2flexlemma;
	Corpus::CWordDictionary *dict = Corpus::CWordDictionary::GetInstance();

	for(udict_map::iterator iter = base_udict.begin(); iter != base_udict.end(); ++iter)
	{
		udict_inner_map &inner = iter->second;
		std::string fc = Corpus::CTag::GetFlexClassString( iter->first );
		for(udict_inner_map::iterator inner_iter = inner.begin(); inner_iter != inner.end(); ++inner_iter)
		{
			if((stopList.find(dict->GetAt(inner_iter->first))==stopList.end()))
			{
				std::string flexlemma = fc + ';' + dict->GetAt( inner_iter->first);
				count2flexlemma.insert( std::make_pair( inner_iter->second, flexlemma ) );
			}
		}
	}

	std::ofstream out(conf.dictWithWords.c_str());
	std::vector<char> charStoplist = loadCharStoplist();
	out << "flex;lemma;frequency\n";
	size_t added = 0;
	for(std::multimap<size_t, std::string>::reverse_iterator iter = count2flexlemma.rbegin();
	iter != count2flexlemma.rend(); ++iter)
	{
		if( iter->first >= conf.min_tf)
		{
			bool forbiddenCharFound=false;
			for(int i=0;i<(int)charStoplist.size();i++)
				if(iter->second.find_first_of(charStoplist[i],0)!=std::string::npos)
					forbiddenCharFound=true;
			if(!forbiddenCharFound)
			{
				added ++;

				out <<  iter->second << ';' << iter->first << std::endl;
			}
			else
				std::cout<<"Forbidden char found in "<<iter->second<<std::endl;
		}
	}

	std::cout << "Written down " << added << " / " << count2flexlemma.size() << " lexems." << std::endl;
}

void add_base(Corpus::CToken *pToken, udict_map &base_udict, me_architect::Config &conf)
{
	Corpus::CLexem *pDisamb = get_disamb(pToken, conf);
	if(!pDisamb)
		return;
	if(conf.flexes.find(pDisamb->GetTag().FlexClass) == conf.flexes.end() )
		return;
	udict_inner_map & id2count = base_udict[pDisamb->GetTag().FlexClass];

	int base_id = pDisamb->GetBaseIdx();

	udict_inner_map::iterator find = id2count.find( base_id );
	if(find == id2count.end() )
		id2count.insert( std::make_pair( base_id, size_t(1) ) );
	else
		find->second += 1;
}

Corpus::CLexem *get_disamb(Corpus::CToken *pToken,  me_architect::Config &conf)
{

	std::vector<Corpus::CLexem*> &lexems = pToken->GetLexems();
	std::vector<Corpus::CLexem*> disambs;
	for(size_t j = 0; j < lexems.size(); j++)
	{
		if(lexems[j]->GetDisamb() )
			disambs.push_back( lexems[j] );
	}

	Corpus::CLexem *pDisamb;
	int substid = -1;
	int gerid = -1;
	for(size_t j =0; j < disambs.size(); j++)
	{
		if( disambs[j]->GetTag().FlexClass == Corpus::cf_subst )
			substid = j;
		if( disambs[j]->GetTag().FlexClass == Corpus::cf_ger )
			gerid = j;
		if(conf.removeAmbInterp && disambs[j]->GetTag().FlexClass!=disambs[0]->GetTag().FlexClass)
			return NULL;
	}

	if(substid != -1 && gerid != -1)
	{
		if (!conf.favorGers)
			pDisamb = disambs[substid];
		else
			pDisamb = disambs[gerid];
	}
	else
		pDisamb = disambs[0];
	return pDisamb;
}

