/*
    Copyright (C) 2012 Bartosz Broda, Jacek Bochenek, Łukasz Biłeńkij, 
    Damian Jaworski, Wojciech Mazur, Dominik Piasecki, Maciej Piasecki, 
    Radosław Ramocki
    
    This program is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by the Free
Software Foundation; either version 3 of the License, or (at your option)
any later version.

    This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE. 

    See the LICENSE and COPYING files for more details.
*/

#include "../matrices_headers.h"
#include "SuperMatrixFilteredReader.h"
#include "defines.h"

static log4cxx::LoggerPtr logger(log4cxx::Logger::getLogger("SuperMatrix.matrices.io.SuperMatrixFilterReader"));

namespace Matrices
{
	namespace IO
	{
		SuperMatrixFilteredReader::SuperMatrixFilteredReader(const std::string &words_file, const std::string &dir,
				const std::string &name, bool load_by_columns, bool load_by_rows, bool filter_rows, bool filter_columns) :
			m_sDir(dir), m_sName(name), m_sWordsFile(words_file),
					m_bFilterRows(filter_rows), m_bFilterColumns(filter_columns), m_bLoadByColumns(load_by_columns),m_bLoadByRows(load_by_rows)
		{
			CheckSuperMatrixFiles(dir, name, load_by_columns, load_by_rows);
			LoadWords();
			LoadFeatures();
			if (m_bLoadByColumns)
				LoadByColumns();
			if (m_bLoadByRows)
				LoadByRows();
			LoadInfo();

			std::map<size_t, std::string> boundaries = TryLoadingBoundaries(dir, name);
			if(boundaries.size() == 0)
				boundaries.insert( std::make_pair( m_pvColumnFeatures->size() - 1, name) );

			m_pMatrix = Matrices::SuperMatrixPtr(new SuperMatrix(m_sName, m_pColumnMatrix, m_pRowMatrix, m_pvRowFeatures,
					m_pvColumnFeatures, m_stTokens, m_sCorpusInfo, boundaries));
		}

		Matrices::SuperMatrixPtr SuperMatrixFilteredReader::GetMatrix()
		{
			return m_pMatrix;
		}

		void SuperMatrixFilteredReader::LoadWords()
		{
			std::ifstream file(m_sWordsFile.c_str());
			std::string line;
			while (std::getline(file, line))
			{
				boost::trim(line);
				if (line.length())
					m_Words.insert(line);
			}
		}

		void SuperMatrixFilteredReader::LoadInfo()
		{
			boost::filesystem::path info_filename(m_sDir);
			info_filename /= m_sName;
			info_filename /= INFO_FILENAME;
			std::string token_count_str;

			std::ifstream file(info_filename.string().c_str());
			if (!file)
				throw filesystem_error("Cannot open file " + info_filename.string());

			std::getline(file, m_sCorpusInfo);
			std::getline(file, token_count_str);
			m_stTokens = boost::lexical_cast<size_t>(token_count_str);
		}

		void SuperMatrixFilteredReader::LoadFeatures()
		{
			boost::filesystem::path rowfeatures(m_sDir);
			rowfeatures /= m_sName;
			rowfeatures /= ROW_FEATURE_FILENAME;
			m_pvRowFeatures = FeatureVectorPtr(new FeatureVector());
			LoadFeatures(rowfeatures.string(), m_pvRowFeatures, m_RowIndexToRemove,m_mOldToNewRows, m_bFilterRows);

			boost::filesystem::path columnfeatures(m_sDir);
			columnfeatures /= m_sName;
			columnfeatures /= COLUMN_FEATURE_FILENAME;
			m_pvColumnFeatures = FeatureVectorPtr(new FeatureVector());
			LoadFeatures(columnfeatures.string(), m_pvColumnFeatures, m_ColumnIndexToRemove, m_mOldToNewColumns, m_bFilterColumns);
		}

		void SuperMatrixFilteredReader::LoadFeatures(const std::string &filename, FeatureVectorPtr features,
				std::set<size_t> &remove_indices, std::map<size_t, size_t> &old_to_new, bool filtered)
		{
			std::ifstream file(filename.c_str());
			std::string line;
			std::getline(file, line);
			boost::trim(line);

			if (line != "label;nz;tf;TF;entropy")
				throw std::runtime_error("File " + filename + " has wrong header: " + line);

			size_t lineNo = 0;
			size_t last_feature_index = 0;
			while (std::getline(file, line))
			{
				boost::trim(line);
				if (line.length() > 0)
				{
					std::vector<std::string> tokens;
					boost::split(tokens, line, boost::is_any_of(";"));
					if (tokens.size() != 5)
					{
						std::stringstream err;
						err << "File " << filename << " contains wrong line (" << lineNo << "): " << line;
						throw std::runtime_error(err.str());
					}

					std::string label = tokens[0];
					if (filtered && m_Words.find(label) == m_Words.end())
						remove_indices.insert(last_feature_index);
					else
					{
						size_t nz = boost::lexical_cast<size_t>(tokens[1]);
						size_t tf = boost::lexical_cast<size_t>(tokens[2]);
						size_t TF = boost::lexical_cast<size_t>(tokens[3]);
						double ent = boost::lexical_cast<double>(tokens[4]);
						Feature f(label, nz, tf, TF, ent);
						// features.size() - index of new feature
						// last_feature_index - index of old feature
						old_to_new[last_feature_index] = features->size();
						features->push_back(f);

					}

					last_feature_index++;
				}

				lineNo++;
			}
		}

		void SuperMatrixFilteredReader::LoadByRows()
		{
			// row_no -> new_no
			// col_no -> new_no
			using namespace boost::posix_time;
			boost::filesystem::path filename(m_sDir);
			filename /= m_sName;
			filename /= CRS_PART_NAME;
			LOG4CXX_INFO(logger, "Loading CRS from " << filename);
			std::ifstream file(filename.string().c_str());
			if (!file)
				throw filesystem_error("Cannot open file: " + filename.string());
			size_t rows, cols, nz;
			file >> rows >> cols >> nz;
			if (rows <= 0 || cols <= 0)
				throw std::runtime_error("Wrong dimensions of a matrix in file: " + filename.string());
			if (nz <= 0)
				LOG4CXX_WARN(logger,"WARNING: nz <= 0... Was it intentional?");
			// set new values for no of rows and columns
			if(m_bFilterRows)
				rows -= m_RowIndexToRemove.size();
			if(m_bFilterColumns)
				cols -= m_ColumnIndexToRemove.size();

			m_pRowMatrix = CRSMatrixPtr(new CRSMatrix(m_sName, rows, cols));
			std::string line;
			size_t row_no = -1;
			std::getline(file, line);//skip header line
			ptime start = microsec_clock::local_time();
			time_duration total_time;
			typedef boost::split_iterator<std::string::iterator> split_iterator;
			while (std::getline(file, line))
			{
				row_no++;
				size_t new_row_no = row_no;
				// skip filtered rows
				if (m_bFilterRows)
				{
					if(m_RowIndexToRemove.find(row_no) != m_RowIndexToRemove.end())
						continue;
					else
						new_row_no = m_mOldToNewRows[row_no];
				}
				// or read normally
				boost::trim(line);
				if (line.size() == 0)
					continue;

				split_iterator itr =
				   boost::algorithm::make_split_iterator(line,
					   boost::token_finder(boost::is_any_of( " \t" )));

				while(itr != split_iterator())
				{
					boost::iterator_range<std::string::iterator> r = *itr++;

					if (itr == split_iterator()) {
						std::stringstream err;
						err << "Wrong line " << row_no + 1 << " in file " << filename << ". Number of items is not even.";
						throw std::runtime_error( err.str() );
					}

					size_t colNo = 0;
					colNo = atoi(&r.front());

					if (m_bFilterColumns)
					{
						if( m_ColumnIndexToRemove.find(colNo) != m_ColumnIndexToRemove.end())
							continue;
						else
							colNo = m_mOldToNewColumns[colNo];
					}

					r = *itr++;
					double value = 0;
					value = atof(&r.front());

					if(colNo > cols)
					{
						std::stringstream err;
						err << "Wrong entry in "<< filename << ", line " << row_no+1 << ". Column number "
								<< colNo << " is grater then number of columns: " << cols ;
						throw std::runtime_error( err.str() );
					}
					if(value!=0)
						m_pRowMatrix->SetCell(value, new_row_no, colNo);
					else
						LOG4CXX_WARN(logger,"Warning, skipped 0 value read from file");
				}

				ptime end = microsec_clock::local_time();
				time_duration len = end - start;
				if (len.total_milliseconds() > 5000)
				{
					total_time += len;
					long total_ms = total_time.total_milliseconds();
					long eta_ms = (total_ms * (rows - row_no - 1)) / (row_no + 1);
					LOG4CXX_INFO(logger, "Reading CRS, row " << row_no << " / " << rows << " ETA: " << to_simple_string(milliseconds(eta_ms)));
					start = microsec_clock::local_time();
				}
			}
		}

		void SuperMatrixFilteredReader::LoadByColumns()
		{
			boost::filesystem::path filename(m_sDir);
			filename /= m_sName;
			filename /= CCS_PART_NAME;
			std::ifstream file(filename.string().c_str());
			if (!file)
				throw filesystem_error("Cannot open file: " + filename.string());
			size_t rows, cols, nz;
			file >> rows >> cols >> nz;
			if (rows <= 0 || cols <= 0)
				throw std::runtime_error("Wrong dimensions of a matrix in file: " + filename.string());
			if (nz <= 0)
				LOG4CXX_WARN(logger, "WARNING: nz <= 0... Was it intentional?");

			// set new values for no of rows and columns
			if(m_bFilterRows)
				rows -= m_RowIndexToRemove.size();
			if(m_bFilterColumns)
				cols -= m_ColumnIndexToRemove.size();

			m_pColumnMatrix = CCSMatrixPtr(new CCSMatrix(m_sName, rows, cols));
			std::string line;
			size_t column_no = -1;
			std::getline(file, line);//skip header line
			typedef boost::split_iterator<std::string::iterator> split_iterator;
			while (std::getline(file, line))
			{
				column_no++;
				size_t new_column_no = column_no;
				// skip filtered rows
				if (m_bFilterColumns)
				{
					if(m_ColumnIndexToRemove.find(column_no) != m_ColumnIndexToRemove.end()) {
						continue;
					}
					else
						new_column_no = m_mOldToNewColumns[column_no];
				}
				// or read normally
				boost::trim(line);
				if (line.size() == 0)
					continue;

				split_iterator itr =
				   boost::algorithm::make_split_iterator(line,
					   boost::token_finder(boost::is_any_of( " \t" )));

				while(itr != split_iterator())
				{
					boost::iterator_range<std::string::iterator> r = *itr++;

					if (itr == split_iterator()) {
						std::stringstream err;
						err << "Wrong line " << column_no + 1 << " in file " << filename << ". Number of items is not even.";
						throw std::runtime_error( err.str() );
					}

					size_t row_no = 0;
					row_no = atoi(&r.front());

					if (m_bFilterRows)
					{
						if(m_RowIndexToRemove.find(row_no) != m_RowIndexToRemove.end())
							continue;
						else
							row_no = m_mOldToNewRows[row_no];
					}

					r = *itr++;
					double value = 0;
					value = atof(&r.front());

					if(row_no > rows)
					{
						std::stringstream err;
						err << "Wrong entry in "<< filename << ", line " << column_no + 1 << ". Row number "
								<< row_no << " is grater then number of rows: " << rows ;
						throw std::runtime_error( err.str() );
					}
					if(value!=0)
						m_pColumnMatrix->SetCell(value, row_no, new_column_no);
					else
						LOG4CXX_WARN(logger,"Warning, skipped 0 value read from file");
				}
			}
		}

		Matrices::SuperMatrixPtr LoadSuperMatrixFiltered(const std::string &words_file, const std::string &dir,
				const std::string &name, bool load_by_columns, bool load_by_rows, bool filter_rows, bool filter_columns)
		{
			return SuperMatrixFilteredReader(words_file, dir, name, load_by_columns, load_by_rows, filter_rows, filter_columns).GetMatrix();
		}

	}//namespace IO
}
