/*
    Copyright (C) 2012 Bartosz Broda, Jacek Bochenek, Łukasz Biłeńkij, 
    Damian Jaworski, Wojciech Mazur, Dominik Piasecki, Maciej Piasecki, 
    Radosław Ramocki
    
    This program is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by the Free
Software Foundation; either version 3 of the License, or (at your option)
any later version.

    This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE. 

    See the LICENSE and COPYING files for more details.
*/

#include "index.h"

#include <fstream>

#include <boost/algorithm/string.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/filesystem.hpp>
#include <boost/iterator.hpp>

#include <boost/foreach.hpp>
#define foreach BOOST_FOREACH


const char* FN_DICT = "dict.txt";
//const char* FN_ROWS = "rows.txt";
//const char* FN_COLUMNS = "columns.txt";
const char* FN_RELATIONS = "relations.txt";
const char* FN_TUPLES = "tuples.txt";

namespace tuplesproc{

std::ostream& operator<<(std::ostream& out, const tuple & t)
{
	out << (t.word_a_id) << ";" << (t.relation_id) << ";" << (t.word_b_id);

	return out;
}


Index::Index(size_t  report_interval)
	: report_interval_(report_interval)
{
}

void Index::row_filtering(const std::string &filepath)
{
	std::ifstream input(filepath.c_str());
	if(!input)
		throw std::runtime_error("Cannot open file: " + filepath);

	std::string line;
	while(std::getline(input, line))
	{
		row_filter.insert(line);
	}
}

void Index::col_filtering(const std::string &filepath)
{
	std::ifstream input(filepath.c_str());
	if(!input)
		throw std::runtime_error("Cannot open file: " + filepath);

	std::string line;
	while(std::getline(input, line))
	{
		col_filter.insert(line);
	}
}

void Index::relation_filtering(const std::string &filepath)
{
	std::ifstream input(filepath.c_str());
	if(!input)
		throw std::runtime_error("Cannot open file: " + filepath);

	std::string line;
	while(std::getline(input, line))
	{
		relation_filter.insert(line);
	}

}

void Index::relations_set_filtering(std::vector<std::string> &set)
{
	std::vector<std::string>::iterator iter;
	for(iter=set.begin(); iter!=set.end(); iter++)
		relation_filter.insert(*iter);
}

void Index::index_list(const std::string &path)
{
	std::ifstream input(path.c_str());
	if(!input)
		throw std::runtime_error("Cannot open file: " + path);

	std::string line;
	while( std::getline(input, line)){
		index_file(line, "");
	}
}

void Index::index_path(const std::string &path, const std::string &pattern)
{
	if(boost::filesystem::is_directory(path))
		index_dir(path, pattern);
	else
		index_file(path, pattern);
}

void Index::index_dir(const std::string &dirname, const std::string &pattern)
{
	if(!boost::filesystem::is_directory(dirname))
		return;

	boost::filesystem::path dir = boost::filesystem::system_complete(boost::filesystem::path(dirname));

	boost::filesystem::directory_iterator end_iter;
	for(boost::filesystem::directory_iterator diriter(dir);
		diriter != end_iter; ++diriter){
		if(boost::filesystem::is_directory(diriter->status()))
			index_dir(diriter->path().string(), pattern);
		else
			index_file(diriter->path().string(), pattern);
	}
}

void Index::index_file(const std::string &filename, const std::string &pattern)
{
	if(pattern != "" && pattern != boost::filesystem::path(filename).filename())
		return;

	std::ifstream input(filename.c_str());
	if(!input)
		throw std::runtime_error("Cannot open file: " + filename);
	std::string line;

	std::cout << "\033[34;1mProcessing " << filename << "\033[0m"<<std::endl;


	size_t line_no = 0;
	while( std::getline(input, line)){
		add_tuple(line);
		if(++line_no % report_interval_ == 0)
			std::cout << "Line " << human_readable(line_no) << std::endl;
	}
	std::cout << "Last line " << line_no << std::endl;

}

std::vector<std::string> Index::read_tuple(const std::string &line)
{
	std::vector<std::string> tokens;
	boost::algorithm::split(tokens, line, boost::is_any_of(";"));

	if(tokens.size() == 4)
		return tokens;
	else if(tokens.size() == 5) {
		tokens[2] = tokens[2] + ";";
		tokens[3] = tokens[4];
		return std::vector<std::string>(tokens.begin(), tokens.end()-1);
	} else
		return std::vector<std::string>();
}

bool Index::add_tuple(const std::string &line)
{
	std::vector<std::string> tokens = read_tuple(line);
	if(tokens.size() == 0) {
		std::cout << "\033[31;1mWrong line: " << line << "\033[0m"<< std::endl;
		return false;
	}

	tuple t;
	boost::algorithm::trim(tokens[0]);
	boost::algorithm::trim(tokens[1]);
	boost::algorithm::trim(tokens[2]);
	boost::algorithm::trim(tokens[3]);

	if(row_filter.size()>0)
	{
		std::set<std::string>::iterator find = row_filter.find(tokens[0]);
		if(find == row_filter.end())
			return false;
	}
	if(col_filter.size()>0)
	{
		std::set<std::string>::iterator find = col_filter.find((tokens[2]));
		if(find == col_filter.end())
			return false;
	}
	if(relation_filter.size()>0)
	{
		std::set<std::string>::iterator find = relation_filter.find(tokens[1]);
		if(find == relation_filter.end())
			return false;
	}


	t.word_a_id = get_str_id(tokens[0], words_dict_);
	t.word_b_id = get_str_id(tokens[2], words_dict_);
	t.relation_id = get_str_id(tokens[1], rel_dict_);
	add_tuple(t, boost::lexical_cast<double>(tokens[3]));
	words_a.insert(t.word_a_id);
	words_b_realtion.insert(word_rel(t.word_b_id,t.relation_id));

	return true;
}

void Index::add_tuple(const tuple &t, double count)
{
	tuplemap::iterator find = tuples_.find(t);
	if(find != tuples_.end())
		find->second += count;
	else
		tuples_.insert( std::make_pair(t, count) );
}

size_t Index::get_str_id(const std::string &word, Index::str2id &where)
{
	str2id::iterator found = where.find(word);
	if(found == where.end()){
		size_t size = where.size();
		where.insert( std::make_pair(word, size));
		return size;
	}

	return found->second;

}


void Index::add_row_id(size_t id)
{
	size_t size = word_to_row.size();
	word_to_row.insert( std::make_pair(id, size));
}

void Index::add_column(word_rel wr)
{
	size_t size = word_rel_to_column.size();
	word_rel_to_column.insert( std::make_pair(wr, size));

}

void Index::write(const std::string &dir)
{
	boost::filesystem::path path(dir);
	write_map((path / FN_DICT).string(), words_dict_ );
	write_map((path / FN_RELATIONS).string(), rel_dict_);

	// TODO: should this below be written to file (after fixing word->id to word+rel->id)
	//write_map((path / FN_ROWS).string(), word_a2id_);
	//write_map((path / FN_COLUMNS).string(), word_b2id_);

	write_map((path / FN_TUPLES).string(), tuples_) ;

}

template<class T>
void Index::write_map(const std::string &filename, const T& index) const
{
	std::ofstream out(filename.c_str());
	if(!out)
		throw std::runtime_error("Cannot open file for writing: " + filename);

	BOOST_FOREACH (const typename T::value_type & vt, index){
		out << vt.first << ";" << vt.second << std::endl;
	}
}

void Index::init_rows_columns()
{
	for(std::set<size_t>::iterator it=words_a.begin();it!=words_a.end();it++)
	{
		add_row_id(*it);
	}

	for(std::set<word_rel>::iterator it=words_b_realtion.begin();it!=words_b_realtion.end();it++)
	{
		add_column(*it);
	}
}

void Index::set_min_tf(long row_tf, long col_tf)
{
	row_min_tf=row_tf;
	col_min_tf=col_tf;
}

Matrices::SuperMatrixPtr Index::mkmatrix(const std::string &name,
										 size_t tokens,
										 const std::string &desc,
										 const std::map<std::string, size_t>  &tf_dict)
{

	init_rows_columns();
	size_t rows = word_to_row.size();
	size_t cols = word_rel_to_column.size();

	Matrices::CRSMatrixPtr crs(new Matrices::CRSMatrix("crs", rows, cols));
	Matrices::CCSMatrixPtr ccs(new Matrices::CCSMatrix("crs", rows, cols));

	BOOST_FOREACH (const tuplemap::value_type &vt, tuples_){
		const tuple &t = vt.first;
		double count = vt.second;

		id2id::const_iterator find = word_to_row.find(t.word_a_id);
		if(find == word_to_row.end())
			throw std::runtime_error( std::string("Wrong a id: ") +
				boost::lexical_cast<std::string>(t.word_a_id));
		size_t row_index = find->second;

		word_rel wr(t.word_b_id, t.relation_id); // stupid, todo: fix
		wordrel2id::const_iterator find2 = word_rel_to_column.find(wr);
		if(find2 == word_rel_to_column.end())
			throw std::runtime_error( std::string("Wrong b id: ") +
				boost::lexical_cast<std::string>(t.word_b_id));
		size_t column_index = find2->second;
		crs->SetCell(count, row_index, column_index);
		ccs->SetCell(count, row_index, column_index);

	}

	std::map<size_t, std::string> boundaries = prepare_boundaries(ccs, tf_dict);
	Matrices::FeatureVectorPtr row_features = prepare_row_featrures(crs, tf_dict);
	Matrices::FeatureVectorPtr column_features = prepare_column_featrures(ccs, tf_dict);
	size_t my_tokens = sum_tf(row_features);

	if(my_tokens > tokens && tokens != 0)
		std::cerr << "\033[31;1mTokens given in params (" << tokens << ") < tokens in matrix! ("<< my_tokens <<")\033[0m\n";

	if(excluded_cols.size()==0 && excluded_rows.size()==0)
		return Matrices::SuperMatrixPtr(
				new Matrices::SuperMatrix(name, ccs, crs, row_features,column_features, tokens , desc, boundaries));
	else
	{
		Matrices::SuperMatrixPtr matrix=Matrices::SuperMatrixPtr(
				new Matrices::SuperMatrix(name, ccs, crs, row_features,column_features, my_tokens , desc, boundaries));
		matrix=Matrices::FilterSuperMatrix(matrix, excluded_rows, excluded_cols, true, true, true);
		excluded_cols.clear();
		excluded_rows.clear();
		GetIndicesToRemove(excluded_rows, matrix->GetRowFeatures());
		GetIndicesToRemove(excluded_cols, matrix->GetColumnFeatures());
		matrix= Matrices::FilterSuperMatrix(matrix, excluded_rows, excluded_cols, true, true, true);
		matrix->SetNewName(name);
		return matrix;
	}

}

std::map<size_t, std::string> Index::prepare_boundaries(Matrices::CCSMatrixPtr &matrix,
	const std::map<std::string, size_t> &tf_dict)
{
	std::map<size_t, std::string> bounds;
	id2wordrel column_to_wordrel;
	reverse_index(word_rel_to_column, column_to_wordrel);

	id2str id_to_relation;
	reverse_index(rel_dict_, id_to_relation);
	
	std::string current_relation="";	
	size_t current_col = 0;
	BOOST_FOREACH (id2wordrel::value_type &vt, column_to_wordrel){
		word_rel wr = vt.second;
		if (id_to_relation[wr.relation]!= current_relation)
		{	
			if(current_relation=="")
				current_relation = id_to_relation[wr.relation];
			else
			{
				bounds.insert( std::pair<size_t, std::string>(current_col, current_relation) );
				current_relation = id_to_relation[wr.relation];
			}
		}
 		current_col++;

		if(current_col == matrix->NumberOfColumns())
			bounds.insert( std::pair<size_t, std::string>(current_col, current_relation) );
	}
	return bounds;

}

Matrices::FeatureVectorPtr Index::prepare_row_featrures(
	Matrices::CRSMatrixPtr &matrix,
	const std::map<std::string, size_t> &tf_dict)
{
	Matrices::FeatureVectorPtr features(new Matrices::FeatureVector);

	id2id row_to_word;
	reverse_index(word_to_row, row_to_word);

	id2str id_to_word;
	reverse_index(words_dict_, id_to_word);

	//////It seems that it works
	features->resize(row_to_word.size());
	//////////
	
	BOOST_FOREACH (id2id::value_type &vt, row_to_word){
		id2str::iterator find = id_to_word.find(vt.second);
		if(find == id_to_word.end())
			throw std::runtime_error(std::string("No word in index: ")
				+ boost::lexical_cast<std::string>(vt.second));

		size_t row_id = vt.first;
		const std::string &word = find->second;

		Matrices::CompressedVectorPtr vec = matrix->GetCompressedRow(row_id);
		size_t tf = size_t(Measures::CountVectorSum(vec)); // will be positive int


		size_t TF = tf;
		std::map<std::string, size_t>::const_iterator found = tf_dict.find(word);
		if(found != tf_dict.end())
			TF = found->second;

		double ent = Measures::CountVectorEntropy(vec);
		size_t nz = vec->GetValues()->size();
		if(tf<row_min_tf)
			excluded_rows.insert(row_id);
		Matrices::Feature f(word, nz, tf, TF, ent);
		//features->push_back(f);

		//////// It seems that it works
		(*features)[row_id] = f;
		////////////


	}

	return features;
}

size_t Index::sum_tf(Matrices::FeatureVectorPtr features) const
{
	size_t sum = 0;
	BOOST_FOREACH (const Matrices::Feature &f, *features)
		sum += f.GetTermFrequency();
	return sum;
}

Matrices::FeatureVectorPtr Index::prepare_column_featrures(
	Matrices::CCSMatrixPtr &matrix,
	const std::map<std::string, size_t> &tf_dict)
{
	Matrices::FeatureVectorPtr features(new Matrices::FeatureVector);

	id2wordrel column_to_wordrel;
	reverse_index(word_rel_to_column, column_to_wordrel);

	id2str id_to_word;
	reverse_index(words_dict_, id_to_word);

	id2str id_to_relation;
	reverse_index(rel_dict_, id_to_relation);
	
	std::string current_relation="";	

	////////// Test
	features->resize(column_to_wordrel.size());

	BOOST_FOREACH (id2wordrel::value_type &vt, column_to_wordrel){
		word_rel wr = vt.second;
		id2str::iterator find = id_to_word.find(wr.word);
		if(find == id_to_word.end())
			throw std::runtime_error(std::string("No word in index: ")
				+ boost::lexical_cast<std::string>(wr.word));

		size_t col_id = vt.first;
		std::string label =  id_to_relation[wr.relation] + ":" +find->second  ;



		Matrices::CompressedVectorPtr vec = matrix->GetCompressedColumn(col_id);

		size_t tf = size_t(Measures::CountVectorSum(vec)); // will be positive int


		size_t TF = tf;
		std::map<std::string, size_t>::const_iterator found = tf_dict.find(find->second);
		if(found != tf_dict.end())
			TF = found->second;
		double ent = Measures::CountVectorEntropy(vec);
		size_t nz = vec->GetValues()->size();
		if(tf<col_min_tf)
			excluded_cols.insert(col_id);
		Matrices::Feature f(label, nz, tf, TF, ent);
		//features->push_back(f);

		////// Test
		(*features)[col_id] = f;

	}
	return features;
}

void Index::GetIndicesToRemove(std::set<size_t> &indices, Matrices::FeatureVectorPtr features)
{
	for(size_t i = 0; i < features->size(); i++)
		if ( (*features)[i].GetNZ() == 0 )
			indices.insert( i );
}

template<class T, class R>
void Index::reverse_index(const T &index, R &new_index) const
{
	BOOST_FOREACH (const typename T::value_type &vt, index){
		typename R::iterator find = new_index.find( vt.second );
		if(find != new_index.end())
			throw std::runtime_error(std::string("Repeated index item: ")
				+ boost::lexical_cast<std::string>(vt.second));
		new_index.insert( std::make_pair(vt.second, vt.first));
	}
}

std::string Index::human_readable(double input, size_t level)
{
	static const char* suffixes = " KMGT";
	if(input < 1000 || level >= 4 )
	{
		std::stringstream ss;
		ss.setf(std::ios::fixed,std::ios::floatfield);
		ss.precision(2);
		ss << input << suffixes[level];
		return ss.str();
	}
	return human_readable(input/1000, level+1);
}

} // ns tuplesproc
