/*
    Copyright (C) 2012 Bartosz Broda, Jacek Bochenek, Łukasz Biłeńkij, 
    Damian Jaworski, Wojciech Mazur, Dominik Piasecki, Maciej Piasecki, 
    Radosław Ramocki
    
    This program is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by the Free
Software Foundation; either version 3 of the License, or (at your option)
any later version.

    This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE. 

    See the LICENSE and COPYING files for more details.
*/

/// @author Bartosz Broda, bartosz.broda@gmail.com
#ifndef TUPLESPROC_INDEX_H
#define TUPLESPROC_INDEX_H

#include <string>
#include <vector>
#include <map>
#include <iostream>
#include <boost/unordered_map.hpp>
#include "matrices_all.h"

namespace tuplesproc{

/**
  *	Connection between two words and relation
  */
struct tuple
{
	size_t word_a_id;
	size_t word_b_id;
	size_t relation_id;

	bool operator<(const tuple& t) const
	{
		if(word_a_id < t.word_a_id)
			return true;
		if(word_a_id > t.word_a_id)
			return false;
		if(word_b_id < t.word_b_id)
			return true;
		if(word_b_id > t.word_b_id)
			return false;

		return relation_id < t.relation_id;
	}
};

std::ostream& operator<<(std::ostream& out, const tuple & t);

/**
  * Connects word with the relation (by their id)
  */
struct word_rel
{
	size_t word;
	size_t relation;

	bool operator<(const word_rel &wr) const
	{
		if( relation < wr.relation )
			return true;
		if(relation > wr.relation)
			return false;
		return word < wr.word;
	}

	word_rel(size_t word, size_t relation)
	{
		this->word = word;
		this->relation = relation;
	}
};

std::ostream& operator<<(std::ostream& out, const tuple & t);

/**
  * Creates a SuperMatrix based on a tuples in csv file(s)
  */
class Index
{
public:
	Index(size_t report_interval = 1000000);

	typedef boost::unordered_map<std::string, size_t> str2id;
	typedef boost::unordered_map<size_t, std::string> id2str;

	typedef std::map<tuple, double> tuplemap;
	typedef std::map<word_rel, size_t> wordrel2id;
	typedef std::map<size_t, word_rel> id2wordrel;
	typedef boost::unordered_map<size_t, size_t> id2id;

	void row_filtering(const std::string &filepath);
	void col_filtering(const std::string &filepath);
	void relation_filtering(const std::string &filepath);
	void relations_set_filtering(std::vector<std::string> &set);

	/**
	  * Parsing files contained in file-list
	  */
	void index_list(const std::string &filepath);
	void index_path(const std::string &filename, const std::string &pattern);
	/**
	  * Parsing single file
	  */
	void index_file(const std::string &filename, const std::string &pattern);
	/**
	  * Parsing files contained in directory
	  */
	void index_dir(const std::string &dirname, const std::string &pattern);

	/**
	  * Creates tuple from single line
	  */
	bool add_tuple(const std::string &line);

	/**
	  * Writes dictionary of words, relations and tuples to files in selected directory
	  */
	void write(const std::string &dir);
	void set_min_tf(long row_tf, long col_tf);
	/**
	  * Creates SuperMatrix and returns pointer to it
	  * \param name name of the matrix
	  * \param tokens number of tokens in corpus
	  * \param desc description of corpus
	  * \param tf_dict dictionary with global Term Frequency
	  */
	Matrices::SuperMatrixPtr mkmatrix(const std::string &name,
									  size_t tokens,
									  const std::string &desc,
									  const std::map<std::string, size_t>  &tf_dict
									  );

private:

	Matrices::FeatureVectorPtr prepare_row_featrures(Matrices::CRSMatrixPtr &matrix,
													 const std::map<std::string, size_t> &tf_dict);
	Matrices::FeatureVectorPtr prepare_column_featrures(Matrices::CCSMatrixPtr &matrix,
														const std::map<std::string, size_t> &tf_dict);
	std::map<size_t, std::string> prepare_boundaries(Matrices::CCSMatrixPtr &matrix,
														const std::map<std::string, size_t> &tf_dict);
	
	/// Sum of term frequency in feature vector
	size_t sum_tf(Matrices::FeatureVectorPtr) const;
	/// Retrieves id of word in dictionary. If word dosn't exist in dictionary then it is insertet
	inline size_t get_str_id(const std::string &word, Index::str2id &where);

	std::vector<std::string> read_tuple(const std::string &line);
	inline void add_tuple(const tuple &t, double count);
	/// Sets the word of the id as a new row
	inline void add_row_id(size_t id);
	/// Sets the word and relation as a new column
	inline void add_column(word_rel wr);
	void init_rows_columns();
	void GetIndicesToRemove(std::set<size_t> &indices, Matrices::FeatureVectorPtr features);
	/// Writes map into file
	template<class T>
	void write_map(const std::string &filename, const T& index) const;

	/// 1000 -> 1k, 1000k -> 1M, 1000M -> 1G, 1000G -> 1T
	std::string human_readable(double input, size_t level=0);

	template<class T, class R>
	void reverse_index(const T &index, R &new_index) const;
	/// Set of first words in tuples
	std::set<size_t> words_a;
	/// Set of second words with relation in tuples
	std::set<word_rel> words_b_realtion;
	/// Set of acceptable columns in filtering
	std::set<std::string> col_filter;
	/// Set of acceptable words in filtering
	std::set<std::string> row_filter;
	/// Set of acceptable relations in filtering
	std::set<std::string> relation_filter;
	/// Dictionary of all words in tuples
	str2id words_dict_;
	/// Dictionary of all relations in tuples
	str2id rel_dict_;
	/// Describes which word is which row
	id2id word_to_row;// word -> row
	/// Describes which word and relation is which column
	wordrel2id word_rel_to_column;// word+word -> column
	/// map of all tuples
	tuplemap tuples_;
	double row_min_tf;
	double col_min_tf;
	std::set<size_t> excluded_rows;
	std::set<size_t> excluded_cols;

	size_t report_interval_;

};

} // ns tuplesproc

#endif //TUPLESPROC_INDEX_H
