/*
    Copyright (C) 2012 Bartosz Broda, Jacek Bochenek, Łukasz Biłeńkij, 
    Damian Jaworski, Wojciech Mazur, Dominik Piasecki, Maciej Piasecki, 
    Radosław Ramocki
    
    This program is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by the Free
Software Foundation; either version 3 of the License, or (at your option)
any later version.

    This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE. 

    See the LICENSE and COPYING files for more details.
*/

#include "doctermbuilder.h"

#include <set>
#include <vector>
#include <fstream>

#include <boost/program_options.hpp>
#include <boost/algorithm/string.hpp>


#include <boost/foreach.hpp>
#include <libcorpus2/tagsetmanager.h>
#include <libcorpus2/io/xcesreader.h>

#include "../../libnerreader/nerreader.h"

namespace doctermbuilder{

	DocTermBuilder::DocTermBuilder(const config &cfg)
		: config_(cfg)
	{
		// no op?
	}

	void DocTermBuilder::run()
	{
		using namespace std;
		using namespace boost;
		typedef pair<string,string> strpair;
		typedef map<string,size_t> str2st;
		typedef map<strpair,size_t> index_type;

		ofstream out(config_.outfile.c_str());
		if(!out)
			throw runtime_error("Cannot open file for writing: " + config_.outfile);


		const Corpus2::Tagset& tagset = Corpus2::get_named_tagset(config_.tagset);

		size_t doc_no = 0;
		std::cout << "Starting..." << std::endl;

		int start = 1;
		Corpus2::TokenReader::TokenReaderPtr pReader;
		boost::shared_ptr<Corpus2::NERReader> tmpReader;
		BOOST_FOREACH (std::string document, config_.corpus)
		{
			if(config_.ner_flag)
			{
				if(start)
				{
					tmpReader = boost::static_pointer_cast<Corpus2::NERReader> (Corpus2::TokenReader::create_path_reader(config_.reader_class,
																		tagset, document));
					start=0;
				}
				else
				{
					tmpReader->change_input_file(document);
				}
				pReader = tmpReader;
			}
			else
			{
				pReader =Corpus2::TokenReader::create_path_reader(config_.reader_class,
																 tagset, document);
			}

			Corpus2::Sentence::Ptr pSentence;



			while( (pSentence=pReader->get_next_sentence()))
			{
				index_type cols;

				// index sentence
				for(size_t i = 0; i < pSentence->size(); ++i)
				{
					Corpus2::Token* pToken = pSentence->at(i);
					Corpus2::Lexeme lex = pToken->get_preferred_lexeme(tagset);
					std::string pos_str = tagset.get_pos_name(lex.tag().get_pos());
					std::string lemma = lex.lemma_utf8();
					std::string ls = pos_str + ":" + lemma;

					if(config_.column_wordpos.size() == 0
					   || config_.column_wordpos.find(ls) != config_.column_wordpos.end())
					{
						if(config_.column_poses.size() == 0 ||
						   config_.column_poses.find(pos_str) != config_.column_poses.end())
						{
							cols.insert(make_pair(make_pair(pos_str, lemma), i));

							string entry = document + ";INDOC;" + ls;
							str2st::iterator find = data.find(entry);
							if(find == data.end())
								data.insert( make_pair(entry, 1) );
							else
								find->second  += 1;
						}
					}
				} // index sentence


			}// while sentence

			if(++doc_no % config_.dump_interval == 0)
			{
				// TODO: eta
				cerr << "Document no: " << human_readable(doc_no)
					 << "(" << document << "). Dumping data... " ;
				BOOST_FOREACH (str2st::value_type &entry2count, data)
					out << entry2count.first << ";" << entry2count.second
						<< std::endl;
				cerr << " Done." << std::endl;
				data.clear();
			}
		} // for each document

		cerr << "Document no: " << human_readable(doc_no)
			 << "(last document of corpus). Dumping data... " ;
		BOOST_FOREACH (str2st::value_type &entry2count, data)
			out << entry2count.first << ";" << entry2count.second
				<< std::endl;
		cerr << " Done." << std::endl;
		data.clear();


	} // run

	std::string DocTermBuilder::human_readable(double input, size_t level)
	{
		static const char* suffixes = " KMGT";
		if(input < 1000 || level >= 4 )
		{
			std::stringstream ss;
			ss.setf(std::ios::fixed,std::ios::floatfield);
			ss.precision(2);
			ss << input << suffixes[level];
			return ss.str();
		}
		return human_readable(input/1000, level+1);
	}

} // namespace windowbuilder
