/*
    Copyright (C) 2012 Bartosz Broda, Jacek Bochenek, Łukasz Biłeńkij, 
    Damian Jaworski, Wojciech Mazur, Dominik Piasecki, Maciej Piasecki, 
    Radosław Ramocki
    
    This program is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by the Free
Software Foundation; either version 3 of the License, or (at your option)
any later version.

    This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE. 

    See the LICENSE and COPYING files for more details.
*/

#include "questionbuilder.h"

#include <set>
#include <vector>
#include <fstream>

#include <boost/program_options.hpp>
#include <boost/algorithm/string.hpp>


#include <boost/foreach.hpp>
#include <libcorpus2/tagsetmanager.h>
#include <libcorpus2/io/xcesreader.h>

#include <libcorpus2/ann/annotatedsentence.h>
#include <libcorpus2/ann/channel.h>

#include "../../libnerreader/nerreader.h"

namespace questionbuilder{

	QuestionBuilder::QuestionBuilder(const config &cfg)
		: config_(cfg)
	{
		// no op?
	}

	std::vector<std::string> QuestionBuilder::index_sentence_words(
			Corpus2::Sentence::Ptr pSentence,
			const Corpus2::Tagset& tagset)
	{
		std::vector<std::string> indices;
		for(size_t i = 0; i < pSentence->size(); ++i)
		{
			Corpus2::Token* pToken = pSentence->at(i);
			Corpus2::Lexeme lex = pToken->get_preferred_lexeme(tagset);
			std::string pos_str = tagset.get_pos_name(lex.tag().get_pos());
			std::string lemma = lex.lemma_utf8();
			std::string ls = pos_str + ":" + lemma;

			if(config_.column_wordpos.size() == 0
			   || config_.column_wordpos.find(ls) != config_.column_wordpos.end())
			{
				if(config_.column_poses.size() == 0 ||
				   config_.column_poses.find(pos_str) != config_.column_poses.end())
				{
					std::string index = "INDOC;" + ls;
					indices.push_back(index);
				}
			}
		}
		return indices;
	}

	std::vector<std::string> QuestionBuilder::index_sentence_chunks(
			Corpus2::Sentence::Ptr pSentence,
			const Corpus2::Tagset& tagset)
	{
		std::vector<std::string> indices;
		boost::shared_ptr<Corpus2::AnnotatedSentence> pAsent =
				Corpus2::AnnotatedSentence::wrap_sentence(pSentence);
		// iterate over each chunk type as defined in config
		// e.g. chunk_np, chunk_vp
		BOOST_FOREACH (const std::string chan_name, config_.chunk_types)
		{
			if( pAsent->has_channel(chan_name) ) {
				// iterate over each chunk instance
				const Corpus2::AnnotationChannel chan =
						pAsent->get_channel(chan_name);
				std::vector<Corpus2::Annotation> ann_vec =
						chan.make_annotation_vector();
				BOOST_FOREACH (const Corpus2::Annotation ann, ann_vec) {
					// ann is one chunk instance of chunk_name type
					// generate string of lemmas
					bool first = true;
					std::string lemma_string = "";
					BOOST_FOREACH (int tok_idx, ann.indices) {
						Corpus2::Lexeme lex =
								pAsent->at(tok_idx)->get_preferred_lexeme(tagset);
						std::string lemma = lex.lemma_utf8();
						if( !first ) lemma_string += " ";
						lemma_string += lemma;
						first = false;
					}
					// append the string of lemmas to the output vector
					std::replace( lemma_string.begin(), lemma_string.end(), ';', '_');
					std::replace( lemma_string.begin(), lemma_string.end(), ':', '_');
					std::string index = "INDOC;" + chan_name + ":" + lemma_string;
					indices.push_back(index);
				}
			}
		}
		return indices;
	}

	void QuestionBuilder::run()
	{
		using namespace std;
		using namespace boost;
		typedef pair<string,string> strpair;
		typedef map<string,size_t> str2st;
		typedef map<strpair,size_t> index_type;

		ofstream out(config_.outfile.c_str());
		if(!out)
			throw runtime_error("Cannot open file for writing: " + config_.outfile);


		const Corpus2::Tagset& tagset = Corpus2::get_named_tagset(config_.tagset);

		size_t doc_no = 0;
		std::cout << "Starting..." << std::endl;

		int start = 1;
		Corpus2::TokenReader::TokenReaderPtr pReader;
		boost::shared_ptr<Corpus2::NERReader> tmpReader;

		BOOST_FOREACH (std::string document, config_.corpus)
		{
			if(config_.ner_flag)
			{
				if(start)
				{
					tmpReader = boost::static_pointer_cast<Corpus2::NERReader> (Corpus2::TokenReader::create_path_reader(config_.reader_class,
																		tagset, document));
					start=0;
				}
				else
				{
					tmpReader->change_input_file(document);
				}
				pReader = tmpReader;
			}
			else
			{
				pReader =Corpus2::TokenReader::create_path_reader(config_.reader_class,
																 tagset, document);
			}

			boost::shared_ptr<Corpus2::Chunk> pChunk;

			string answer="";
			string question="";
			while( (pChunk=pReader->get_next_chunk())){
				string ch_id = pChunk->get_attribute("id");

				vector<Corpus2::Sentence::Ptr> ss = pChunk->sentences();
				BOOST_FOREACH (Corpus2::Sentence::Ptr pSentence,ss){
					for(size_t i = 0; i < pSentence->size(); ++i)
					{
						Corpus2::Token* pToken = pSentence->at(i);
						std::string orth = pToken->orth_utf8();
						if (orth != ";"){
							if(ch_id[0]=='Q')
								question += orth+" ";
							if(ch_id[0]=='A')
								answer += orth +" ";
						}
					}
				}

			}
			if(config_.ner_flag)
			{
				tmpReader->change_input_file(document);
				pReader = tmpReader;
			}
			else
			{
				pReader =Corpus2::TokenReader::create_path_reader(config_.reader_class,
																 tagset, document);
			}
			while( (pChunk=pReader->get_next_chunk()))
			{
				string ch_id = pChunk->get_attribute("id");
				vector<Corpus2::Sentence::Ptr> ss = pChunk->sentences();
				// index_type cols;
				std::string prefix;
				std::string &text = question;
				if( ch_id[0]=='Q') { // a question
					prefix = "Q_A:";
					text = question;
				}
				else if( ch_id[0]=='A') { // an answer
					prefix = "A_A:";
					text = answer;
				}
				else {
					throw runtime_error("Each paragraph should be either"
										" a [Q]uestion or an [A]nswer.");
				}

				BOOST_FOREACH (Corpus2::Sentence::Ptr pSentence, ss){
					// index sentence
					std::vector<std::string> indices;
					if( config_.chunk_types.empty() ) {
						indices = index_sentence_words(pSentence, tagset);
					}
					else {
						indices = index_sentence_chunks(pSentence, tagset);
					}
					BOOST_FOREACH (std::string index, indices) {
						std::string entry = prefix + text + ";" + index;
						str2st::iterator find = data_.find(entry);
						if(find == data_.end())
							data_.insert( make_pair(entry, 1) );
						else
							find->second  += 1;
					}
				}


			}// while sentence

			if(++doc_no % config_.dump_interval == 0)
			{
				// TODO: eta
				cerr << "Document no: " << human_readable(doc_no)
					 << "(" << document << "). Dumping data_... " ;
				BOOST_FOREACH (str2st::value_type &entry2count, data_)
					out << entry2count.first << ";" << entry2count.second
						<< std::endl;
				cerr << " Done." << std::endl;
				data_.clear();
			}
		} // for each document

		cerr << "Document no: " << human_readable(doc_no)
			 << "(last document of corpus). Dumping data_... " ;
		BOOST_FOREACH (str2st::value_type &entry2count, data_)
			out << entry2count.first << ";" << entry2count.second
				<< std::endl;
		cerr << " Done." << std::endl;
		data_.clear();

	} // run

	std::string QuestionBuilder::human_readable(double input, size_t level)
	{
		static const char* suffixes = " KMGT";
		if(input < 1000 || level >= 4 )
		{
			std::stringstream ss;
			ss.setf(std::ios::fixed,std::ios::floatfield);
			ss.precision(2);
			ss << input << suffixes[level];
			return ss.str();
		}
		return human_readable(input/1000, level+1);
	}

} // namespace windowbuilder
