/*
    Copyright (C) 2012 Bartosz Broda, Jacek Bochenek, Łukasz Biłeńkij, 
    Damian Jaworski, Wojciech Mazur, Dominik Piasecki, Maciej Piasecki, 
    Radosław Ramocki
    
    This program is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by the Free
Software Foundation; either version 3 of the License, or (at your option)
any later version.

    This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE. 

    See the LICENSE and COPYING files for more details.
*/

#include "matrices_all.h"
#include "comparator_all.h"
#include "vector_test.h"
#include "vector_definitions.h"

#include "log4cxx/propertyconfigurator.h"
#include "log4cxx/logger.h"
#include "log4cxx/basicconfigurator.h"



#include <ctime>

#include <boost/random.hpp>


using namespace boost::numeric::ublas;
typedef std::vector<std::string> STRVEC;

void load_words(std::string &filename, std::vector<std::string> &words);
void heurestic_weight(STRVEC words, Matrices::SuperMatrixPtr pMatrix, const std::string &outname);

void compare_weighting(std::string word, Matrices::SuperMatrixPtr orig, Matrices::SuperMatrixPtr weighted1, Matrices::SuperMatrixPtr weighted2, size_t no);
void weight(std::string cmpfun, Matrices::SuperMatrixPtr pMatrix);


int cosine_test();
void compare_weighting_main(std::vector<std::string> &words);
void print_pairs();
void rffTest();
void test_simfunction(std::string simstring, std::string mdir, std::string mname);

//log4cxx::LoggerPtr logger(log4cxx::Logger::getLogger("MyLogger"));


int main()
{
		//log4cxx::Logger::getRootLogger()->setLevel(log4cxx::Level::getWarn());
		log4cxx::BasicConfigurator::configure();
		//log4cxx::PropertyConfigurator::configure("/home/dominik/supermatrix/tools/tmp/log4j.properties");

		/*LOG4CXX_INFO(logger, "Info");
        LOG4CXX_ERROR(logger, "error");
		LOG4CXX_TRACE(logger, "trace");*/
  /*      LOG4CXX_DEBUG(logger, "debug");
        LOG4CXX_ERROR(logger, "error");
        LOG4CXX_FATAL(logger, "fatal");*/


		//Matrices::IO::LoadSimMatrix("aaa","bbb",20.5);
		std::string mdir = "/home/jawor/macierze/ang";
        std::string mname = "bnc_subj_summed";
        Matrices::SuperMatrixPtr pMatrix = Matrices::IO::LoadSuperMatrix(mdir,mname,false,true);
        std::string msr = "rwf_lin(rank_size=1000,uniform_rank=false)";
        smartcomparator::compare::CompareFunctionPtr pCmp = smartcomparator::compare::FunctionFactory::CreateFunction(msr,pMatrix);
        std::cout << "min value " << pCmp->MinValue() << std::endl;
		std::cout << "comp(pies,kot) =  " << pCmp->Compare("cat","dog") <<std::endl;

	return 0;
}

void test_simfunction(std::string simstring, std::string mdir, std::string mname)
{
	Matrices::SuperMatrixPtr m = Matrices::IO::LoadSuperMatrix(mdir, mname, false, true);
	smartcomparator::compare::FunctionFactory ff;

	smartcomparator::compare::CompareFunctionPtr pCmp = ff.CreateFunction(simstring, m);

	double val = pCmp->Compare("subst:pies", "subst:kot");

	if (val <=0 || val != val)
	{
		std::cerr << "ZOMG FAIL\n";
	}
	else
	{
		std::cerr << "PASS\n";
	}


}


void GetNZInd(Matrices::CompressedVectorPtr pVec, std::set<size_t> &nzs)
{
	std::map<size_t, double> *vals = pVec->GetValues();
	for(std::map<size_t, double>::iterator iter = vals->begin(); iter != vals->end(); ++iter)
		nzs.insert( iter->first);

}

void print_pairs()
{

	std::string mdir = "/mnt/workspace/macierze";
	std::string mname = "kgr31b_AICN_wlasne";
	std::string outname = "out.csv";

	Matrices::SuperMatrixPtr pMatrix = Matrices::IO::LoadSuperMatrix(mdir, mname, true, false);
	size_t columns = pMatrix->NumberOfColumns();

	boost::mt19937 rng;
	rng.seed(static_cast<unsigned int>(std::time(0)));
	boost::uniform_int<> myRange(0, columns-1);
	boost::variate_generator<boost::mt19937&, boost::uniform_int<> > myRand(rng, myRange);

	std::map< std::pair<size_t, size_t>, size_t > elems;

	size_t no = 10000000;
	size_t lf = 0;

	while( elems.size() < no )
	{
		size_t a = (size_t) myRand();
		size_t b = (size_t) myRand();
		std::pair<size_t, size_t> p(a,b);


		if( elems.find( p ) == elems.end() )
		{

			Matrices::CompressedVectorPtr pV1 = pMatrix->GetColumn(a);
			std::set<size_t> nz1, nz2, intersec;
			GetNZInd(pV1, nz1);

			Matrices::CompressedVectorPtr pV2 = pMatrix->GetColumn(b);
			GetNZInd(pV2, nz2);
			std::set_intersection( nz1.begin(), nz1.end(), nz2.begin(), nz2.end(), std::inserter(intersec, intersec.begin()));
			if (intersec.size())
				elems.insert( std::make_pair(p, intersec.size()));

		}

		size_t size = elems.size();
		if(size && size % 1000 == 0 && lf != size)
		{
			std::cout << "Fetched " << size << " elems." << std::endl;
			lf = size;
		}
	};

	std::ofstream out(outname.c_str());
	if(!out)
		throw std::runtime_error("Cannot open file: " + outname);
	for(std::map< std::pair<size_t, size_t>, size_t >::iterator iter = elems.begin(); iter != elems.end(); ++iter)
	{
		std::pair<size_t, size_t> ij = iter->first;
		size_t val = iter->second;
		out << ij.first << "; " << ij.second << "; "<< val << std::endl;
	}

}

int cosine_test()
{
	Matrices::CRSMatrixPtr crs(new Matrices::CRSMatrix("crs", 3, 3));
	Matrices::CCSMatrixPtr ccs(new Matrices::CCSMatrix("crs", 3, 3));
	Matrices::FeatureVectorPtr rowfeat(new Matrices::FeatureVector() );
	Matrices::FeatureVectorPtr colfeat(new Matrices::FeatureVector() );
	Matrices::CompressedVector v(3);
	v.SetElement(0,1);
	v.SetElement(1,1);
	v.SetElement(2,1);

	crs->SetCompressedRow(0,v);

	v.SetElement(0,1);
	v.SetElement(1,2);
	v.SetElement(2,3);
	crs->SetCompressedRow(1,v);

	std::string name("sm");
	std::string desc("sss");
	Matrices::SuperMatrixPtr matrix(new Matrices::SuperMatrix(name, ccs, crs, rowfeat, colfeat,0,desc) );

	std::set<size_t> indices;
	indices.insert(0);
	indices.insert(1);
	indices.insert(2);
	smartcomparator::simfunctions::CosineFunction cosine(matrix, false);
	std::cout << "Cosine(<1,1,1>,<1,2,3>) = " << cosine.CountSimilarity(* crs->GetCompressedRow(0),* crs->GetCompressedRow(1), indices, indices) << std::endl;
	return 0;
}

void compare_weighting_main(std::vector<std::string> &words)
{
	using namespace std;
	std::string mdir = "/mnt/data/workdir/macierze/supermatrix2/bnc/";
	std::string mname = "bnc_subj_summed";
	std::string cmpfun = "rwf_lin(rank_size=10000, uniform_rank = false, partial_order=true, div_factor=1)";
	std::string cmpfun2 = "lin()";
	Matrices::SuperMatrixPtr pMatrix = Matrices::IO::LoadSuperMatrix(mdir, mname, false, true);
	Matrices::SuperMatrixPtr w1 = Matrices::IO::LoadSuperMatrix(mdir, mname, true, true);
	Matrices::SuperMatrixPtr w2 = Matrices::IO::LoadSuperMatrix(mdir, mname, true, true);
	weight(cmpfun, w2);
	weight(cmpfun2, w1);
	//std::vector<std::string> words;
	//words.push_back("ibm");
	//words.push_back("duty");
	for(size_t i = 0; i < words.size(); i++)
		compare_weighting( words[i], pMatrix, w1, w2, 20);
}

std::vector< std::pair<double, std::string> > get_best_features(Matrices::SuperMatrixPtr matrix, std::string rowlable, size_t no)
{
	Matrices::CompressedVectorPtr row = matrix->GetRow(rowlable);
	std::multimap<double, std::string > vals;
	Matrices::FeatureVectorPtr rowfeatures = matrix->GetColumnFeatures();
	for(std::map<size_t, double>::iterator iter = row->GetValues()->begin(); iter != row->GetValues()->end(); iter++)
		vals.insert( std::pair<double, std::string>(iter->second, (*rowfeatures)[iter->first].GetLabel() ) );

	size_t printed = 0;
	std::vector< std::pair<double, std::string > >out;
	for(std::multimap<double, std::string>::reverse_iterator iter = vals.rbegin(); iter != vals.rend(); iter++)
	{
		printed+=1;
		out.push_back( std::pair<double, std::string>( iter->first , iter->second) );
		if (printed >= no)
			break;
	}

	return out;

}

void compare_weighting(std::string word, Matrices::SuperMatrixPtr orig, Matrices::SuperMatrixPtr weighted1, Matrices::SuperMatrixPtr weighted2, size_t no)
{
	std::cout << word << std::endl;
	using namespace std;
	std::vector< pair<double, string> > orig_rank, weighted1_rank, weighted2_rank;
	orig_rank = get_best_features(orig, word, no);
	weighted1_rank= get_best_features(weighted1, word, no);//lin
	weighted2_rank= get_best_features(weighted2, word, no);//rwf

	Matrices::CompressedVectorPtr row_w1 = weighted1->GetRow(word);
	Matrices::CompressedVectorPtr row_w2 = weighted2->GetRow(word);
	std::cout << "relation:word_POS; frequency; lin; grwf(lin)\n";
	for(size_t i = 0; i < orig_rank.size(); i++)
	{
		std::string column_label = orig_rank[i].second;
		double frequency = orig_rank[i].first;
		double w1 = (*row_w1->GetValues())[ weighted1->GetColumnIndex(column_label) ];
		double w2 = (*row_w2->GetValues())[ weighted2->GetColumnIndex(column_label) ];
		std::cout << column_label << "; " << frequency << "; " << w1 << "; " << w2 << std::endl;
	}

	std::cout << std::endl;
	std::cout << "no transformation; lin/grwf\n";
	for(size_t i = 0; i < orig_rank.size(); i++)
	{
		std::string c1 = orig_rank[i].second;
		std::string c3 = weighted2_rank[i].second;
		std::cout << c1 << "; " << c3 << std::endl;
	}
	std::cout << std::endl;
	std::cout << std::endl;

}

void weight(std::string cmpfun, Matrices::SuperMatrixPtr pMatrix)
{
	smartcomparator::compare::CompareFunctionPtr cmp = smartcomparator::compare::FunctionFactory::CreateFunction(cmpfun,pMatrix);
	smartcomparator::weights::WeightFeaturePtr weight = cmp->GetWeightFeature();
	std::cout << "Weighting..." << std::endl;
	smartcomparator::weights::WeightCRS( &(* pMatrix->GetCRS()), weight );
	std::cout << "Weighting... Done" << std::endl;
}


void heurestic_weight(STRVEC words, Matrices::SuperMatrixPtr pMatrix, const std::string &outname)
{
	// do tscore weighting
	boost::shared_ptr<smartcomparator::weights::TScoreWeight> pWeight( new smartcomparator::weights::TScoreWeight(pMatrix) );
	smartcomparator::weights::MatrixTransformer mt(pWeight);
	mt.Transform();
	size_t rows = pMatrix->NumberOfRows();

	Matrices::FeatureVectorPtr pColFeatures = pMatrix->GetColumnFeatures();

	std::ofstream out( outname.c_str() );
	out << "słowo;waga;nz" << std::endl;
	// for each word
	for(STRVEC::iterator wordIter = words.begin(); wordIter != words.end(); wordIter++)
	{
		std::string word = "subst:" + *wordIter;
		double sum = 0;
		size_t row_id = pMatrix->GetRowIndex(word);
		if( row_id >= rows )
		{
			std::cerr << "No word " << word << " in matrix. Skipping." << std::endl;
			//out << "No word " << word << " in matrix. Skipping." << std::endl;
			out << word << " NO DATA " << std::endl;
			continue;
		}
		// for each nz feature
		std::map<size_t, double> * pVals = pMatrix->GetRow(row_id)->GetValues();
		// sum
		for(std::map<size_t, double>::iterator valIter = pVals->begin(); valIter != pVals->end(); valIter++)
		{
			double entropy = (*pColFeatures)[valIter->first].GetEntropy();
			double tscore = valIter->second;
			if(entropy < 0.00001 || tscore < 3)
				continue;
			sum += log(tscore) / entropy;
		}

//		sum /= pVals->size();

		out << *wordIter << "; " << sum << ";" << pVals->size() << std::endl;
	}

	out.close();
	std::cout << "All done" << std::endl;

}

void load_words(std::string &filename, std::vector<std::string> &words)
{
	std::ifstream file(filename.c_str());
	if(!file)
		throw std::runtime_error( "Cannot open " + filename );

	std::string line;
	while( std::getline(file, line) )
	{
		boost::trim(line);
		if (line.length() > 0 )
			words.push_back(line);
	}
}

Matrices::CompressedVectorPtr create_centroid(Matrices::SuperMatrixPtr pMatrix, std::vector<std::string> &words)
{
	Matrices::CompressedVectorPtr pV( new Matrices::CompressedVector( *pMatrix->GetRow(words[0]) ) );
	smartcomparator::weights::RankBuilderPtr pRWF(new smartcomparator::weights::NormalRankBuilder(pMatrix, 10000, false) );
	smartcomparator::weights::RWFZScore weight(pMatrix, pRWF, 2.57);
	std::set<size_t> indices, to_erase;
	std::map<size_t, double> *centroid_vals = pV->GetValues();
	//weighting of first
	for(std::map<size_t, double>::iterator iter = centroid_vals->begin(); iter != centroid_vals->end(); iter ++ )
		indices.insert(iter->first);
	weight.Weight(pMatrix->GetRowIndex(words[0]), &*pV, indices );

	for(size_t i = 1; i < words.size(); i++)
	{
		std::cout << words[i] << std::endl;
		Matrices::CompressedVectorPtr pU = pMatrix->GetRow( words[i] );
		std::map<size_t, double> *vals = pU->GetValues();
		//weighting of others
		indices.clear();
		for(std::map<size_t, double>::iterator iter = vals->begin(); iter != vals->end(); iter ++ )
			indices.insert(iter->first);
		weight.Weight(pMatrix->GetRowIndex(words[i]), &*pU, indices);

		//summing intersecting features (ranks)
		for(std::map<size_t, double>::iterator iter = vals->begin(); iter != vals->end(); iter ++ )
		{
			std::map<size_t, double>::iterator find = centroid_vals->find( iter->first);
			if( (find != centroid_vals->end() ) && ( to_erase.find(iter->first) == to_erase.end()) )
				find->second = find->second + iter->second;
			//	find->second = find->second > iter->second? iter->second : find->second;
			else
				to_erase.insert(iter->first);
		}
	}

	//erasing not intersecing fetures
	for(std::set<size_t>::iterator iter = to_erase.begin(); iter != to_erase.end(); iter++)
		centroid_vals->erase(*iter);

	for(std::map<size_t, double>::iterator iter = centroid_vals->begin(); iter != centroid_vals->end(); iter ++ )
	{
		iter->second = iter->second / (double) words.size();
	}

	return pV;
}

void write_sims(Matrices::FeatureVectorPtr row_features, std::multimap<double, size_t> &sims, std::string filename, size_t k = 20)
{
	std::ofstream file(filename.c_str());
	size_t added = 0;
	for(std::multimap<double, size_t>::reverse_iterator riter = sims.rbegin(); riter != sims.rend() && added < k; riter++)
	{
		if(row_features->operator[](riter->second).GetGlobalTermFrequency() < 25000)
		{
			file << row_features->operator[](riter->second).GetLabel() << " " << riter->first << std::endl;
			added++;
		}
	}
}

int oldtest(int ac, char*av[])
{
	std::string matrix_dir = "/mnt/data/workdir/macierze/supermatrix2/noun/polaczone/";
	//std::string matrix_dir = "/mnt/data/workdir/macierze/supermatrix2/noun/";
	std::string matrix_name = "kipi_subst_AdjP_AmISubj_Coord_NGen";
	//std::string matrix_name = "kipi_substAdjP";
	std::string words_file = "./w.txt";
	std::vector<std::string> words;
	load_words(words_file, words);

	Matrices::SuperMatrixPtr pMatrix = Matrices::IO::LoadSuperMatrix(matrix_dir, matrix_name, false, true);
	// centroid wpolnych cech
	std::cout << "Creating centroid" << std::endl;
	Matrices::CompressedVectorPtr pCentroid = create_centroid(pMatrix, words);
	std::map<size_t, double> *centroid_vals = pCentroid->GetValues();
	std::cout << "Creating centroid...done" << std::endl;

	// porownanie po:
	// - wszystkich cechach
	std::multimap<double, size_t> sims;
	smartcomparator::weights::RankBuilderPtr pRWF(new smartcomparator::weights::NormalRankBuilder(pMatrix, 10000, false) );
	smartcomparator::weights::RWFZScore weight(pMatrix, pRWF, 2.57 );
	smartcomparator::simfunctions::LocalCosineSimilarity cosine(false, false);
	std::cout << "Counting all features" << std::endl;

	for(size_t row_no = 0; row_no < pMatrix->NumberOfRows(); row_no++)
	{
		Matrices::CompressedVector v( *pMatrix->GetRow(row_no) );
		std::set<size_t> indices;
		for(std::map<size_t, double>::iterator iter = v.GetValues()->begin(); iter != v.GetValues()->end(); iter ++ )
			indices.insert(iter->first);
		weight.Weight(row_no, &v, indices);
		double sim = cosine.CountSimilarity(*pCentroid, v);
		sims.insert(std::pair<double,size_t>(sim, row_no) );
	}

	write_sims(pMatrix->GetRowFeatures(), sims, "wszystkie_cechy.txt", 100);

	std::cout << "Counting features intersection" << std::endl;
	sims.clear();
	// porownanie po:
	// - wspolnych cechach
	for(size_t row_no = 0; row_no < pMatrix->NumberOfRows(); row_no++)
	{
		Matrices::CompressedVector v( *pMatrix->GetRow(row_no) );
		std::set<size_t> indices, erase;
		for(std::map<size_t, double>::iterator iter = v.GetValues()->begin(); iter != v.GetValues()->end(); iter ++ )
		{
			if(centroid_vals->find(iter->first) == centroid_vals->end() )
				erase.insert(iter->first);
			indices.insert(iter->first);
		}

		weight.Weight(row_no, &v, indices);

		for(std::set<size_t>::iterator iter = erase.begin(); iter != erase.end(); iter++)
			v.GetValues()->erase(*iter);

		double sim = cosine.CountSimilarity(*pCentroid, v);
		sims.insert(std::pair<double,size_t>(sim, row_no) );
	}
	write_sims(pMatrix->GetRowFeatures(), sims, "wspolne_cechy.txt", 100);

	return 0;
}
