/*
    Copyright (C) 2012 Bartosz Broda, Jacek Bochenek, Łukasz Biłeńkij, 
    Damian Jaworski, Wojciech Mazur, Dominik Piasecki, Maciej Piasecki, 
    Radosław Ramocki
    
    This program is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by the Free
Software Foundation; either version 3 of the License, or (at your option)
any later version.

    This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE. 

    See the LICENSE and COPYING files for more details.
*/

#include "filtrator.h"
#include "options.h"

int main(int ac, char *av[])
{
	try
	{
		Matrices::init_logging();
		filtrator2::Options options(ac,av);
		filtrator2::Filtrator ve(options.GetParams() );
		ve.Start();
	}
	catch(std::exception &e)
	{
		std::cerr << "Error: " << e.what() << std::endl;
		return -1;
	}
	return 0;
}

namespace filtrator2
{
	Filtrator::Filtrator(const Params &p) : m_Params(p)
	{		
		inMatrixDir = p.inMatrixDir;
		inMatrixName = p.inMatrixName;

		outMatrixDir = p.outMatrixDir;
		outMatrixName = p.outMatrixName;

		untilStable = p.untilStable;
		max_iter = p.max_iters;

		filterRowsOnly = p.rowsOnly;
		filterColumnsOnly = p.columnsOnly;

		mintf = p.min_tf;
		minTF = p.min_TF;
		minnz = p.min_nz;
		minentropy = p.min_entropy;

		maxtf = p.max_tf;
		maxTF = p.max_TF;
		maxnz = p.max_nz;
		maxentropy = p.max_entropy;

		m_setWords = std::set<std::string>();

	}

	void Filtrator::Start()
	{
		if (m_Params.m_sWordsFile != "")
		{
			LoadWords();
		}
		inMatrix = Matrices::IO::LoadSuperMatrix(inMatrixDir, inMatrixName, false, true);
		outMatrix = inMatrix;
		size_t it = 0;
		initialized = false;
		if (filterColumnsOnly)
		{
			outMatrix->SyncCRSToCCS();
		}
		do
		{
			std::cerr << "iteration " << it << "\n";
			FilterRows();
			FilterColumns();
			it++;
		}
		while (DimensionsChanged() && (it<max_iter || !untilStable));
		outMatrix->SetNewName(outMatrixName);
		Matrices::IO::WriteSuperMatrix(outMatrix, outMatrixDir);
	}

	bool Filtrator::DimensionsChanged()
	{
		return ! (inMatrix->NumberOfColumns() == outMatrix->NumberOfColumns() && inMatrix->NumberOfRows() == outMatrix->NumberOfRows());
	}

	bool Filtrator::FeatureCheck(const Matrices::Feature f) const
	{
		return f.GetGlobalTermFrequency() >= minTF &&
			f.GetTermFrequency() >= mintf &&
			f.GetNZ() >= minnz &&
			f.GetEntropy() >= minentropy &&
			f.GetGlobalTermFrequency() <= maxTF &&
			f.GetTermFrequency() <= maxtf &&
			f.GetNZ() <= maxnz && f.GetEntropy() <= maxentropy &&
			(m_Params.m_sWordsFile == "" || m_setWords.find(f.GetLabel()) != m_setWords.end())
			;
	}

	std::string Filtrator::getCurrentBoundary(Matrices::SuperMatrixPtr m, size_t colNo)
	{
		for (std::map<size_t, std::string>::iterator it = m->Boundaries().begin(); it != m->Boundaries().end(); ++it)
		{
			if (colNo <= (*it).first)
			{
				return (*it).second;
			}
		}
		throw std::runtime_error("Column index out of all boundaries!");
	}

	void Filtrator::FilterColumns()
	{
		if (filterRowsOnly)
		{
			return;
		}
		std::cerr<<"Filtering matrix columns... ";
		inMatrix = outMatrix;
		filteredColumns.clear();
		boost::shared_ptr<std::vector<Matrices::Feature> > columnFeaturesPtr = inMatrix->GetColumnFeatures();
		std::vector<Matrices::Feature> columnFeatures = (*columnFeaturesPtr);
		Matrices::Feature f;
		for (size_t i = 0; i<columnFeatures.size(); i++)
		{
			const Matrices::Feature & f = columnFeatures[i];
			if (FeatureCheck(f))
			{
				filteredColumns.push_back(i);
			}
		}
		std::string matrixName = inMatrix->GetName();
		std::cout<<"Column filtering creates matrix: "<<inMatrix->NumberOfRows()<<" x "<<filteredColumns.size()<<"\n";
		boost::shared_ptr<Matrices::CRSMatrix> outCRS(new Matrices::CRSMatrix(matrixName, inMatrix->NumberOfRows(), filteredColumns.size()));
		boost::shared_ptr<Matrices::CCSMatrix> outCCS(new Matrices::CCSMatrix(matrixName, inMatrix->NumberOfRows(), filteredColumns.size()));
		Matrices::FeatureVectorPtr columnF(new Matrices::FeatureVector());
		Matrices::FeatureVectorPtr rowF(new Matrices::FeatureVector());
		size_t tokens = inMatrix->GetTokenCount();
		std::string corpus_info = inMatrix->GetCorpusInfo();
		std::map<size_t, std::string> boundaries = inMatrix->Boundaries();
		std::map<size_t, std::string> newBoundaries;

		outMatrix = boost::shared_ptr<Matrices::SuperMatrix> (new Matrices::SuperMatrix(matrixName,outCCS,outCRS,rowF,columnF,tokens,corpus_info,boundaries));
		std::map<std::string, size_t> revBoundaries;
		std::map<std::string, size_t>::iterator it;
		std::string currentBoundary;

		for (size_t i = 0; i<filteredColumns.size(); i++)
		{
				currentBoundary = getCurrentBoundary(inMatrix, filteredColumns[i]);

				if (revBoundaries.count(currentBoundary) == 0)
					revBoundaries.insert(std::pair<std::string, size_t>(currentBoundary, i));
				else
					revBoundaries[currentBoundary]= i;

				if(i+1==outMatrix->NumberOfColumns())
					revBoundaries[currentBoundary]= i+1;

				outCCS->SetCompressedColumn(i,inMatrix->GetCCS()->GetCompressedColumn(filteredColumns[i]));
				(*columnF).push_back(columnFeatures[filteredColumns[i]]);
		}

		for(it=revBoundaries.begin(); it!=revBoundaries.end(); it++)
		{
			std::cout << (*it).first << " => " << (*it).second << std::endl;
			newBoundaries.insert(std::pair<size_t, std::string >((*it).second, (*it).first));
		}

		outMatrix->SyncCCSToCRS();
		recalculateRowFeatures();

		outMatrix->SetBoundaries(newBoundaries);

		std::cout<<"Done filtering columns.\n";
		std::cout<<filteredColumns.size()<<" columns retained\n";
	}//	void Filtrator::FilterColumns()

	void Filtrator::FilterRows()
	{
		if (filterColumnsOnly)
		{
			return;
		}
		std::cerr<<"Filtering matrix rows...";
		if (initialized)
		{
			inMatrix = outMatrix;
		}
		else
		{
			initialized = true;
		}
		filteredRows.clear();
		boost::shared_ptr<std::vector<Matrices::Feature> > rowFeaturesPtr = inMatrix->GetRowFeatures();
		std::vector<Matrices::Feature> rowFeatures = (*rowFeaturesPtr);
		Matrices::Feature f;
		for (size_t i = 0; i<rowFeatures.size(); i++)
		{
			const Matrices::Feature &f = rowFeatures[i];
			if (FeatureCheck(f))
				filteredRows.push_back(i);

		}
		std::string matrixName = inMatrix->GetName();
		std::cout<<"Row filtering creates matrix: "<<filteredRows.size()<<" x "<<inMatrix->NumberOfColumns()<<"\n";
		boost::shared_ptr<Matrices::CRSMatrix> outCRS(new Matrices::CRSMatrix(matrixName, filteredRows.size(), inMatrix->NumberOfColumns()));
		boost::shared_ptr<Matrices::CCSMatrix> outCCS(new Matrices::CCSMatrix(matrixName, filteredRows.size(), inMatrix->NumberOfColumns()));
		Matrices::FeatureVectorPtr columnF(new Matrices::FeatureVector());
		Matrices::FeatureVectorPtr rowF(new Matrices::FeatureVector());
		size_t tokens = inMatrix->GetTokenCount();
		std::string corpus_info = inMatrix->GetCorpusInfo();
		std::map<size_t, std::string> boundaries = inMatrix->Boundaries();
		outMatrix = boost::shared_ptr<Matrices::SuperMatrix> (new Matrices::SuperMatrix(matrixName,outCCS,outCRS,rowF,columnF,tokens,corpus_info,boundaries));
		for (size_t i = 0; i<filteredRows.size(); i++)
		{
			outCRS->SetCompressedRow(i,inMatrix->GetCRS()->GetCompressedRow(filteredRows[i]));
			(*rowF).push_back(rowFeatures[filteredRows[i]]);
		}
		outMatrix->SyncCRSToCCS();
		recalculateColumnFeatures();
		std::cout<<"Done filtering rows.\n";
		std::cout<<filteredRows.size()<<" rows retained\n";
	}//	void Filtrator::FilterRows()


	void Filtrator::WriteMatrixColumnBased()
	{

	}

	void Filtrator::recalculateRowFeatures()
	{
		std::cout<<"Recalculating row features... ";
		Matrices::FeatureVectorPtr rowF = outMatrix->GetRowFeatures();
		Matrices::FeatureVector inRowF = (*inMatrix->GetRowFeatures());
		for (size_t i = 0; i<outMatrix->NumberOfRows(); i++)
		{
			Matrices::CompressedVectorPtr rowContents = outMatrix->GetRow(i);
			std::string featurename = inRowF[i].GetLabel();
			size_t TF = inRowF[i].GetGlobalTermFrequency();
			size_t tf = (size_t) Measures::CountVectorSum(rowContents);
			rowF->push_back(Matrices::Feature(featurename,rowContents->GetValues()->size(),tf,TF,Measures::CountVectorEntropy(rowContents)));
		}
		std::cout<<"Done.\n";
	}

	void Filtrator::recalculateColumnFeatures()
	{
		std::cout<<"Recalculating column features... ";
		Matrices::FeatureVectorPtr colF = outMatrix->GetColumnFeatures();
		Matrices::FeatureVector inColf = (*inMatrix->GetColumnFeatures());
		for (size_t i = 0; i<outMatrix->NumberOfColumns(); i++)
		{
			Matrices::CompressedVectorPtr colContents = outMatrix->GetColumn(i);
			std::string featurename = inColf[i].GetLabel();
			size_t TF = inColf[i].GetGlobalTermFrequency();
			size_t tf = (size_t) Measures::CountVectorSum(colContents);
			colF->push_back(Matrices::Feature(featurename,colContents->GetValues()->size(),tf,TF,Measures::CountVectorEntropy(colContents)));
		}
		std::cout<<"Done.\n";
	}

	void Filtrator::PrepareFeaturesColumnBased(Matrices::CRSMatrixPtr pCRS)
	{

	}

	void Filtrator::LoadWords()
	{
		std::ifstream file( m_Params.m_sWordsFile.c_str() );
		if(!file)
			throw std::runtime_error( "Cannot open file: " + m_Params.m_sWordsFile );

		std::string line;
		while( std::getline(file,line) )
		{
			boost::trim(line);
			if(line.length() )
				m_setWords.insert(line);
		}

		std::cout << "Loaded " << m_setWords.size() << " unique words from " << m_Params.m_sWordsFile << std::endl;
	}

}//namespace filtrator
