/*
	Copyright (C) 2012 Bartosz Broda, Jacek Bochenek, Łukasz Biłeńkij,
	Damian Jaworski, Wojciech Mazur, Dominik Piasecki, Maciej Piasecki,
	Radosław Ramocki

	This program is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by the Free
Software Foundation; either version 3 of the License, or (at your option)
any later version.

	This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE.

	See the LICENSE and COPYING files for more details.
*/

#include <map>
#include <set>
#include <vector>
#include <fstream>

#include <boost/program_options.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/foreach.hpp>
#define foreach BOOST_FOREACH

#include <libcorpus2/tagsetmanager.h>
#include <libcorpus2/io/xcesreader.h>
#include <unicode/unistr.h>

#include <iostream>

using namespace std;

struct config
{
	std::string tagset_name;
	std::string reader_class;
	std::string corpus_list;
	std::string output_path;
	std::vector<std::string> corpus_paths;
	std::set<std::string> pos_filters;
	bool save_flex;
	bool disamb_only;
	bool lower;
	bool whole_info;
	size_t min_tf;

	static config process(int ac, char **av)
	{
		std::vector<std::string> pos_filters_vec;
		config cfg;

		namespace po = boost::program_options;

		po::options_description cmdline("Command line only options", 100);
		po::options_description basic("Basic options", 100);
		std::string configfile;

		cmdline.add_options()
			("help", "Prints this message")
			("cfg-file", po::value<std::string>(&configfile)->default_value(std::string("orth.cfg")), "Optional config file")
			;
		basic.add_options()
			("tagset", po::value<std::string>(&cfg.tagset_name)->default_value("kipi"),
				"Name of a tagset.")
			("reader", po::value<std::string>(&cfg.reader_class), "Name of a reader.")
			//("corpus", po::value<std::string>(&cfg.corpus_path), "Path to corpus.")
			("list-corpus", po::value<std::string>(&cfg.corpus_list), "Path to file with corpuses.")
			("corpus", po::value< std::vector<std::string> >(&cfg.corpus_paths), "Path to corpus")
			("posfilters", po::value<std::vector<std::string> >(&pos_filters_vec),
				"Filters for part of speech. If set then only specified POS will be stored. If not set then all POS will be stored.")
			("output", po::value<std::string>(&cfg.output_path),
				"Path for file with results.")
			("pos", po::value<bool>(&cfg.save_flex)->default_value(true),
				"If true then part-of-speech will be preserved.")
			("disamb-only", po::value<bool>(&cfg.disamb_only)->default_value(true),
				"If true then only disambiguated lemmas will be saved.")
			("to-lower", po::value<bool>(&cfg.lower)->default_value(true),
					"If true then orths will be converted to lowercase.")
			("mintf", po::value<size_t>(&cfg.min_tf)->default_value(0),
				"Min frequency of lemmas to store in the output file.")
			("whole-tag-info", po::value<bool>(&cfg.whole_info)->default_value(false),
				"Should whole tag info be keept (true) or only the POS (false) " )
			;

		po::options_description all("",80);
		all.add(cmdline).add(basic);

		po::variables_map vm;
		po::store(po::parse_command_line(ac, av, all), vm);
		po::notify(vm);

		if( vm.count("help" ) )
		{
			std::cout << "basedict by Bartosz Broda, bartosz.broda@gmail.com\n";
						std::cout << "Collects a unigram dictionary of orths\n";
			std::cout << all << std::endl;
			exit(0);
		}

		std::ifstream cfgfile( configfile.c_str() );

		if(cfgfile)
		{
			po::store(po::parse_config_file(cfgfile, basic), vm);
			po::notify(vm);
		}
		else if( vm.count("cfg-file") )
			std::cerr << "Cannot open config file " << configfile << ". Proceeding with commandline and defaults...\n";
		if( !vm.count("reader") )
		{
			std::cerr << "You need to specify reader." << std::endl;
			exit(1);
		}
		if( !vm.count("corpus") && !vm.count("list-corpus"))
		{
			std::cerr << "You need to specify corpus." << std::endl;
			exit(1);
		}
		if( !vm.count("output") )
		{
			std::cerr << "You need to specify output path." << std::endl;
			exit(1);
		}
		if(vm.count("posfilters"))
		{
			BOOST_FOREACH (std::string &f, pos_filters_vec)
				cfg.pos_filters.insert(f);
		}


		return cfg;
	}
};

struct vectorEl{
	int frequency;
	std::string str1;
	std::string str2;

	vectorEl(int frequency, std::string str1, std::string str2)
		:frequency(frequency), str1(str1), str2(str2){}
};

struct vectorEl_more{
	bool operator()(vectorEl const& a, vectorEl const& b) const{
		if(a.frequency > b.frequency) return true;
		else return false;
	}
};


int main(int ac, char**av)
{

	typedef std::map<std::string, size_t> inner_dictype; // orth -> count
	typedef std::map<std::string, inner_dictype> dictype; // base -> (orth -> count)

	config cfg = config::process(ac, av);
	dictype orthdict;




	string line;
	ifstream myfile (cfg.corpus_list.c_str());

	std::vector<std::string> list_of_corpuses;

	if (myfile.is_open())
	{
		while (! myfile.eof() )
		{
			getline (myfile,line);
			if(line.compare("") != 0)
			{
				list_of_corpuses.push_back(line);
			}
		}
	}
	for(int i=0; i<(int)(cfg.corpus_paths.size()); i++){
		list_of_corpuses.push_back(cfg.corpus_paths[i]);
	}


	if(list_of_corpuses.size()>0){
		for(int i=0; i<(int)list_of_corpuses.size(); i++){
			const Corpus2::Tagset& tagset = Corpus2::get_named_tagset(cfg.tagset_name);
			Corpus2::TokenReader::TokenReaderPtr pReader = Corpus2::TokenReader::create_path_reader(cfg.reader_class,
													 tagset,
													 list_of_corpuses[i]);

			Corpus2::Sentence::Ptr pSentence;
			cout << list_of_corpuses[i] << endl;
			std::cout << "Starting..." << std::endl;
			size_t sent_no = 0;

			while( (pSentence=pReader->get_next_sentence()))
			{
				BOOST_FOREACH (Corpus2::Token*token, pSentence->tokens())
				{
					std::string orth = token->orth_utf8();
					if(cfg.lower)
					{
						UnicodeString o(token->orth());
						o.toLower();
						std::string s;
						orth = o.toUTF8String(s);
					}
					BOOST_FOREACH (Corpus2::Lexeme& lexeme, token->lexemes())
					{
						if(cfg.disamb_only && !lexeme.is_disamb())
							continue;
						std::string base = lexeme.lemma_utf8();
						std::string pos = tagset.get_pos_name(lexeme.tag().get_pos());

						if(cfg.pos_filters.size() && cfg.pos_filters.find(pos) == cfg.pos_filters.end())
							continue;

						if(cfg.whole_info)
							pos = tagset.tag_to_string(lexeme.tag());

						if(cfg.save_flex)
							base = pos +";" +base;

						dictype::iterator find = orthdict.find(base);
						if(find == orthdict.end())
							orthdict.insert( std::make_pair(base,inner_dictype()));

						inner_dictype &inner = orthdict[base];
						inner_dictype::iterator find_orth = inner.find(orth);
						if(find_orth == inner.end())
							inner.insert( std::make_pair(orth, 1));
						else
							find_orth->second += 1;
					}
				}

				if(++sent_no % 100000 == 0){
					std::cout << "Sentence no: " << sent_no/1000 << "k " << std::endl;
				}

			}//while( (pSentence=pReader->get_next_sentence()))
		}
	}


	std::ofstream out(cfg.output_path.c_str());
	if(!out)
	{
		std::cerr << "Cannot open " << cfg.output_path << " for wrtiting!\n";
		exit(2);
	}

	std::cout << "Writing base dict to " << cfg.output_path << std::endl;

	if(cfg.save_flex)
		out << "flex;base;orth;frequency\n";
	else
		out << "base;base;orth;frequency\n";

	vector <vectorEl> frequencyVector;


	BOOST_FOREACH (dictype::value_type &base_iter, orthdict)
	{
		inner_dictype &inner = base_iter.second;
		BOOST_FOREACH (inner_dictype::value_type &orth_iter, inner)
			if(orth_iter.second >= cfg.min_tf){
				frequencyVector.push_back(vectorEl(orth_iter.second, base_iter.first, orth_iter.first));
			}
	}

	sort(frequencyVector.begin(), frequencyVector.end(), vectorEl_more());

	for(int i=0; i<(int)frequencyVector.size(); i++){
		out << frequencyVector[i].str1 << ";" << frequencyVector[i].str2 << ";" <<frequencyVector[i].frequency << std::endl;

	}


	std::cout << "All done..." << std::endl;
	return 0;
}















