/*
    Copyright (C) 2012 Bartosz Broda, Jacek Bochenek, Łukasz Biłeńkij, 
    Damian Jaworski, Wojciech Mazur, Dominik Piasecki, Maciej Piasecki, 
    Radosław Ramocki
    
    This program is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by the Free
Software Foundation; either version 3 of the License, or (at your option)
any later version.

    This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE. 

    See the LICENSE and COPYING files for more details.
*/

#include <map>
#include <set>
#include <vector>
#include <fstream>

#include <boost/program_options.hpp>
#include <boost/algorithm/string.hpp>
#include <boost/foreach.hpp>
#define foreach BOOST_FOREACH

#include <libcorpus2/tagsetmanager.h>
#include <libcorpus2/io/xcesreader.h>


struct config
{
	std::string tagset_name;
	std::string reader_class;
	std::vector<std::string>  corpus_paths;
	std::string output_path;
	std::set<std::string> pos_filters;
	bool save_flex;
	bool disamb_only;
	size_t min_tf;

	static config process(int ac, char **av)
	{
		std::vector<std::string> pos_filters_vec;
		config cfg;

		namespace po = boost::program_options;

		po::options_description cmdline("Command line only options", 100);
		po::options_description basic("Basic options", 100);
		std::string configfile;

		cmdline.add_options()
			("help", "Prints this message")
			("cfg-file", po::value<std::string>(&configfile)->default_value(std::string("basedict.cfg")), "Optional config file")
			;
		basic.add_options()
			("tagset", po::value<std::string>(&cfg.tagset_name)->default_value("kipi"),
				"Name of a tagset.")
			("reader", po::value<std::string>(&cfg.reader_class), "Name of a reader.")
			("corpus", po::value<std::vector<std::string> >(&cfg.corpus_paths), "Path(s) to corpus.")
			("posfilters", po::value<std::vector<std::string> >(&pos_filters_vec),
				"Filters for part of speech. If set then only specified POS will be stored. If not set then all POS will be stored.")
			("output", po::value<std::string>(&cfg.output_path),
				"Path for file with results.")
			("pos", po::value<bool>(&cfg.save_flex)->default_value(true),
				"If true then part-of-speech will be preserved.")
			("disamb-only", po::value<bool>(&cfg.disamb_only)->default_value(true),
				"If true then only disambiguated lemmas will be saved.")
			("mintf", po::value<size_t>(&cfg.min_tf)->default_value(0),
				"Min frequency of lemmas to store in the output file.")
			;

		po::options_description all("",80);
		all.add(cmdline).add(basic);

		po::variables_map vm;
		po::store(po::parse_command_line(ac, av, all), vm);
		po::notify(vm);

		if( vm.count("help" ) )
		{
			std::cout << "basedict by Bartosz Broda, bartosz.broda@gmail.com\n";
			std::cout << "Collects a unigram dictionary of lemmas (base forms)\n";
			std::cout << all << std::endl;
			exit(0);
		}

		std::ifstream cfgfile( configfile.c_str() );

		if(cfgfile)
		{
			po::store(po::parse_config_file(cfgfile, basic), vm);
			po::notify(vm);
		}
		else if( vm.count("cfg-file") )
			std::cout << "Cannot open config file " << configfile << ". Proceeding with commandline and defaults...\n";
		if( !vm.count("reader") )
		{
			std::cerr << "You need to specify reader." << std::endl;
			exit(1);
		}
		if( !vm.count("corpus") )
		{
			std::cerr << "You need to specify corpus." << std::endl;
			exit(1);
		}
		if( !vm.count("output") )
		{
			std::cerr << "You need to specify output path." << std::endl;
			exit(1);
		}
		if(vm.count("posfilters"))
		{
			BOOST_FOREACH (std::string &f, pos_filters_vec)
				cfg.pos_filters.insert(f);
		}

		return cfg;
	}
};


int main(int ac, char**av)
{
	typedef std::map<std::string, size_t> dictype;

	config cfg = config::process(ac, av);

	std::vector<std::string>::iterator iter;
	dictype basedict;
	Corpus2::Sentence::Ptr pSentence;

	std::cout << "Starting..." << std::endl;
	size_t sent_no = 0;
	size_t counter=0;
	for( iter=cfg.corpus_paths.begin(); iter!=cfg.corpus_paths.end(); iter++)
	{
		counter++;
		std::cout << "Creating dictionary for corpus: "<< *iter << std::endl;
		const Corpus2::Tagset& tagset = Corpus2::get_named_tagset(cfg.tagset_name);

		Corpus2::TokenReader::TokenReaderPtr pReader =
				Corpus2::TokenReader::create_path_reader(cfg.reader_class,
														 tagset,
														 *iter);



		while( (pSentence=pReader->get_next_sentence()))
		{
			BOOST_FOREACH (Corpus2::Token*token, pSentence->tokens())
			{
				BOOST_FOREACH (Corpus2::Lexeme& lexeme, token->lexemes())
				{
					if(cfg.disamb_only && !lexeme.is_disamb())
						continue;
					std::string base = lexeme.lemma_utf8();
					std::string pos = tagset.get_pos_name(lexeme.tag().get_pos());
					if(cfg.pos_filters.size() && cfg.pos_filters.find(pos) == cfg.pos_filters.end())
						continue;

					if(cfg.save_flex)
						base = pos +";" +base;

					dictype::iterator find = basedict.find(base);
					if(find == basedict.end())
						basedict.insert( std::make_pair(base, 1));
					else
						find->second += 1;
				}
			}

			if(++sent_no % 100000 == 0){
				std::cout << "Sentence no: " << sent_no/1000 << "k " << "\t Corpus:\t"<< counter << "/" << cfg.corpus_paths.size()<< std::endl;
			}
		}//while( (pSentence=pReader->get_next_sentence()))
	}


	std::ofstream out(cfg.output_path.c_str(), std::ios::trunc);
	if(!out)
	{
		std::cerr << "Cannot open " << cfg.output_path << " for wrtiting!\n";
		exit(2);
	}

	std::cout << "Writing base dict to " << cfg.output_path << std::endl;

	if(cfg.save_flex)
		out << "flex;lemma;frequency\n";
	else
		out << "lemma;frequency\n";
	BOOST_FOREACH (dictype::value_type &iter, basedict)
		if(iter.second >= cfg.min_tf)
			out << iter.first << ";" << iter.second << std::endl;

	std::cout << "All done..." << std::endl;

	return 0;
}
