/*
    Copyright (C) 2012 Bartosz Broda, Jacek Bochenek, Łukasz Biłeńkij, 
    Damian Jaworski, Wojciech Mazur, Dominik Piasecki, Maciej Piasecki, 
    Radosław Ramocki
    
    This program is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by the Free
Software Foundation; either version 3 of the License, or (at your option)
any later version.

    This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE. 

    See the LICENSE and COPYING files for more details.
*/

#include "matrixconv.h"


int main(int argc, char *argv[])
{
	Matrices::init_logging();
	if(argc != 4)
	{
		std::cout << "Matrix converter. Usage:\nmatrixconv dir_in dir_out corpus_info\n";
		return -1;
	}
	else
	{
		MatrixConverter conv;
		std::string in = argv[1];
		std::string out = argv[2];
		std::string info = argv[3];
		std::cout << "Converting matrix from: " << in << std::endl;
		std::cout << "Saving matrix to: " << out << std::endl;
		std::cout << "Corpus info: " << info << std::endl;
		conv.ConvertMatrix( in, out, info);
		std::cout << "\n\nRemeber to convert encoding of label and features files of output matrix to cp1250!" << std::endl;
	}
	return 0;
}

namespace fs = boost::filesystem;

MatrixConverter::MatrixConverter()
{
	token_count = 0;
}


void MatrixConverter::ConvertMatrix(const std::string &old_matrix_path, const std::string &new_matrix_path, 
		const std::string &corpus_info) 
{ 
	this->old_path = old_matrix_path;
	this->new_path = new_matrix_path;
	this->corpus_info = corpus_info;
	// check if everything is in place
	DoCheck();
	// preapre everything...
	PrepareOutputDir();
	// move data
	CopyData();
	// write info
	WriteInfo();
}

void MatrixConverter::WriteInfo()
{
	fs::path info(new_path);
	info /= matrix_name;
	info /= "info.txt";
	
	std::ofstream out( info.string().c_str() );
	out << corpus_info << std::endl;
	out << token_count << std::endl;
	out.close();

}
void MatrixConverter::CopyData()
{
	std::cout << "Convering...\n";
	// copy labels
	CopyLabels();
	// copy features
	CopyFeatures();
	// copy matrices
	CopyMatrices();
	std::cout << "Converted...\n";
}

void MatrixConverter::CopyMatrices()
{
	std::cout << "Copying matrices...\n";
	{
		std::cout << "Loading CCS part...\n";
		Matrices::CCSMatrixPtr m( new Matrices::CCSMatrix(old_matrix_ccs_name, 0,0) );
		Matrices::IO::CCSMatrixReader reader(old_path, old_matrix_ccs_name);
		reader.ReadMatrix( *m );
		std::cout << "Writing CCS part\n";
		fs::path out(new_path);
		out /= matrix_name;
		out /= CCS_PART_NAME;
		Matrices::IO::WriteToSingleFile( m, out.string() );
	}

	{
		std::cout << "Loading CRS part...\n";
		Matrices::CRSMatrixPtr m( new Matrices::CRSMatrix(old_matrix_crs_name, (size_t)0,0) );
		Matrices::IO::CRSMatrixReader reader(old_path, old_matrix_crs_name);
		reader.ReadMatrix( *m );
		std::cout << "Writing CRS part\n";
		fs::path out(new_path);
		out /= matrix_name;
		out /= CRS_PART_NAME;
		Matrices::IO::WriteToSingleFile(m, out.string());
	}

	std::cout << "Done copying matrices...\n";

}

void MatrixConverter::CopyFeatures()
{
	std::cout << "Copying row features\n";
	fs::path from(old_path);
	from /= "row_features.txt";
	fs::path to(new_path);
	to /= matrix_name;
	to /= "row_features.csv";
	ConvertFeatures( from.string().c_str(), to.string().c_str(), row_terms);

	std::cout << "Copying column features\n";
	from = fs::path(old_path);
	from /= "column_features.txt";
	to = fs::path(new_path);
	to /= matrix_name;
	to /= "column_features.csv";
	ConvertFeatures( from.string().c_str(), to.string().c_str(), column_terms);
}

void MatrixConverter::ConvertFeatures(const std::string &filename_in, const std::string filename_out, std::vector<std::string> &terms)
{
	std::ifstream in(filename_in.c_str() );
	if(!in)
		throw std::runtime_error( "Cannot open file for reading: " + filename_in);
	std::string line;
	bool found_header =false;
	size_t entropy_id = 0, TF_id = 0, nz_id = 0, tf_id = 0;
	bool found_entropy = false, found_TF = false, found_nz = false, found_tf = false;
	while( std::getline(in, line) && !found_header )
	{
		boost::trim(line);
		if( line == "#CSV Header:" )
		{
			std::getline(in, line);
			boost::trim(line);
			if(line.length() > 0)
			{
				found_header = true;
				std::vector<std::string> tokens;
				boost::split(tokens, line, boost::is_any_of(","));
				for(size_t i = 0;i < tokens.size(); i++)
				{
					//entropy,desc_id,global-tf,nz,tf
					if( tokens[i] == "entropy")
					{
						found_entropy = true;
						entropy_id = i;
					}
					if( tokens[i] == "tf")
					{
						found_tf = true;
						tf_id = i;
					}
					if( tokens[i] == "global-tf" )
					{
						found_TF = true;
						TF_id = i;
					}
					if( tokens[i] == "nz" )
					{
						found_nz = true;
						nz_id = i;
					}
				}
			}			
		}// search for header
	}// while getline

	if(!found_header || !found_entropy || !found_tf || !found_TF || !found_nz)
		throw std::runtime_error("Corupted file: "+ filename_in);

	std::ofstream out( filename_out.c_str() );
	out << "label;nz;tf;TF;entropy" << std::endl;		
	size_t term_no = 0;
	while( std::getline(in, line) )
	{
		boost::trim(line);
		std::vector<std::string> tokens;
		boost::split(tokens, line, boost::is_any_of(",") );
		size_t tf = boost::lexical_cast<size_t>( tokens[tf_id] );
		size_t TF = boost::lexical_cast<size_t>( tokens[TF_id] );
		size_t nz = boost::lexical_cast<size_t>( tokens[nz_id] );
		double ent= boost::lexical_cast<double>( tokens[entropy_id] );
		std::string &label = terms[ term_no ];
		out << label << ";" << nz << ";" << tf << ";" << TF << ";" << ent << std::endl;
		term_no++;
	}

	out.close();
	in.close();
}

void MatrixConverter::CopyLabels()
{
	fs::path from(old_path);
	from /= "row_lexems.txt";
	fs::path to(new_path);
	to /= matrix_name;
	to /= matrix_name + ".rlabel";
	fs::copy_file(from, to);
	LoadTerms( from.string(), row_terms);

	from = fs::path(old_path);
	from /= "column_lexems.txt";
	to = fs::path(new_path);
	to /= matrix_name;
	to /= matrix_name + ".clabel";
	fs::copy_file(from, to);
	LoadTerms( from.string(), column_terms );
}

void MatrixConverter::PrepareOutputDir()
{
	fs::path base(new_path);
	fs::path matrix(new_path);
	matrix /= matrix_name;
	if( ! fs::exists(base) )
		fs::create_directory( base );

	if( fs::exists(matrix) )
	{
		throw std::runtime_error( "Path for writing exists! Remove " + matrix.string() + " or change output dir." );
	}
	else fs::create_directory(matrix);
}

void MatrixConverter::DoCheck()
{
	std::cout << "Checking...\n";
	fs::path in_path( old_path );
	if( ! fs::exists (in_path) )
		throw std::runtime_error( "Input directory does not exists: " + in_path.string());
	Matrices::IO::CheckFile(in_path, "row_lexems.txt");
	Matrices::IO::CheckFile(in_path, "row_features.txt");
	Matrices::IO::CheckFile(in_path, "column_lexems.txt");
	Matrices::IO::CheckFile(in_path, "column_features.txt");
	Matrices::IO::CheckFile(in_path, "info.txt");

	LoadInfo();

	CheckCRSFiles();
	CheckCCSFiles();
}

void MatrixConverter::CheckCCSFiles()
{
	fs::path old( old_path);
	fs::path ccsbase(old_path);
	ccsbase /= old_matrix_ccs_name;
	Matrices::IO::CheckFile( old , old_matrix_ccs_name.c_str());
	Matrices::IO::CheckFile( ccsbase, (old_matrix_ccs_name + "_col_ccs").c_str());
	Matrices::IO::CheckFile( ccsbase, (old_matrix_ccs_name + "_row_ccs").c_str());
	Matrices::IO::CheckFile( ccsbase, (old_matrix_ccs_name + "_tf_nz_ccs").c_str());
	Matrices::IO::CheckFile( ccsbase, (old_matrix_ccs_name + "_dim").c_str());

}
void MatrixConverter::CheckCRSFiles()
{	
	fs::path old( old_path);
	fs::path crsbase(old_path);
	crsbase /= old_matrix_ccs_name;
	Matrices::IO::CheckFile( old, old_matrix_crs_name.c_str());
	Matrices::IO::CheckFile( crsbase, (old_matrix_crs_name + "_col_crs").c_str());
	Matrices::IO::CheckFile( crsbase, (old_matrix_crs_name + "_row_crs").c_str());
	Matrices::IO::CheckFile( crsbase, (old_matrix_crs_name + "_tf_nz_crs").c_str());
	Matrices::IO::CheckFile( crsbase, (old_matrix_crs_name + "_dim").c_str());
}

void MatrixConverter::LoadInfo()
{
	std::cout << "Loading info..\n";
	fs::path path(old_path);
	path /= "info.txt";
	std::ifstream info( path.string().c_str() );
	std::getline(info, matrix_name);
	boost::trim(matrix_name);
	std::getline(info, old_matrix_ccs_name);
	boost::trim(old_matrix_ccs_name);
	std::getline(info, old_matrix_crs_name);
	boost::trim(old_matrix_crs_name);
	std::string ctokens;
	std::getline(info, ctokens);
	boost::trim(ctokens);
	token_count = boost::lexical_cast<size_t>(ctokens);
	if( token_count == 0 )
		std::cerr << "Attention! Number of token is not set in the input matirx\n";

	std::cout << "Info loaded...\n";

}

void MatrixConverter::LoadTerms(const std::string &filename, std::vector<std::string> &v)
{
	std::ifstream file(filename.c_str());
	if( !file )
		throw std::runtime_error( "Cannot open file for reading: " + filename);

	v.clear();
	std::string line;
	while( std::getline( file, line) )
	{
		boost::trim(line);
		v.push_back(line);
	}
}

