/*
    Copyright (C) 2012 Bartosz Broda, Jacek Bochenek, Łukasz Biłeńkij, 
    Damian Jaworski, Wojciech Mazur, Dominik Piasecki, Maciej Piasecki, 
    Radosław Ramocki
    
    This program is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by the Free
Software Foundation; either version 3 of the License, or (at your option)
any later version.

    This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE. 

    See the LICENSE and COPYING files for more details.
*/

#include "clutowrapper.h"
#include "config.h"

#include <stdlib.h>
#include <boost/filesystem/operations.hpp>

namespace clustering
{
	namespace cbc
	{
		ClutoWrapper::ClutoWrapper(Matrices::SuperMatrixPtr pMatrix, smartcomparator::compare::CompareFunctionPtr pCmp, 
					KBestPtr pKBest, Matrices::SymmetricSimMatrixPtr pSimMatrix)
			: m_pMatrix(pMatrix),
			m_pSimMatrix(pSimMatrix),
			m_pCmp(pCmp),
			m_pKBest(pKBest)
		{
			Initalize();
		}

		void ClutoWrapper::Initalize()
		{
			m_sClutoExe = Config::cluto_exe+" ";
			m_sClMethod = Config::clmethod;
			m_sCrFun = Config::crfun;
		}

		void ClutoWrapper::StartClustering(const std::vector<std::string> & wordsToCluster, std::string outdir, size_t k)
		{
			PrepareCurrentWords(wordsToCluster);
			m_pOutClusters = boost::shared_ptr<Clusters>(new Clusters() );
			m_stK = k;
			m_sOutDir = outdir;
			if( m_sOutDir[ m_sOutDir.length() - 1 ] != '/' && m_sOutDir[ m_sOutDir.length() - 1 ] != '\\' )
				m_sOutDir += "/";
			boost::filesystem::create_directory(outdir);
			std::cout << "Outdir: " << m_sOutDir << std::endl;
			for(size_t word_no = 0; word_no < wordsToCluster.size(); word_no++)
			{
				const std::string &word = wordsToCluster[word_no];
				PrepareCurrentKBest(word);
				if(m_CurrentKBest.size() > 1)
					ClusterOne(word);
				if(word_no % 100 == 0 )
					std::cout << "Clusterign word " << word_no << " / " << wordsToCluster.size() << std::endl;
			}
		}

		void ClutoWrapper::PrepareCurrentWords(const std::vector<std::string> &words)
		{
			for(size_t i = 0; i < words.size(); i++)
				m_WordsToCluster.insert(words[i] );
		}

		void ClutoWrapper::PrepareCurrentKBest(const std::string &word)
		{
			std::vector<std::string>& kbest = m_pKBest->GetKBest(word);
			m_CurrentKBest.clear();
			for(size_t i = 0; i < kbest.size() && m_CurrentKBest.size() < m_stK; i++)
				if(m_WordsToCluster.find( kbest[i] ) != m_WordsToCluster.end() )
					m_CurrentKBest.push_back( kbest[i] );
		}

		void ClutoWrapper::ClusterOne(const std::string &word)
		{
			if (Config::cluto_verbose)
				std::cout << "Clustering: " << word << std::endl;

			// write (sub)matrix
			size_t size = WriteSubMatrix(word);
			if(size < 2)
				return;
			// execute cluto
			RunCLUTO(word, size);
			// read cluto output
			ReadCLUTOOutput(word);
			// score groups and choose the best one
			SelectBestOne( word );
		}

		size_t ClutoWrapper::WriteSubMatrix(const std::string &word)
		{
			if (Config::cluto_verbose)
				std::cout << "Writing submatrix for word: " << word << std::endl << "stK = " << m_stK << std::endl;
				//std::copy(kbest.begin(), kbest.end(), std::ostream_iterator<std::string>(std::cout, " ") );
				//std::cout << std::endl;

			std::vector< std::vector<std::pair<size_t, double> > > cache;
			size_t nz = 0;
			for(size_t word_no = 0; word_no < m_CurrentKBest.size() && word_no < m_stK; word_no++)
			{
				std::string &w1 = m_CurrentKBest[word_no];
				cache.push_back( std::vector<std::pair<size_t, double> >() );
				for(size_t word_col_no = 0; word_col_no < m_CurrentKBest.size() && word_col_no < m_stK; word_col_no++)
				{
					if(word_col_no == word_no)
					{
//						cache[word_no].push_back( std::pair<size_t, double> (word_col_no, 1) );
						continue;
					}
					std::string &w2 = m_CurrentKBest[word_col_no];
					double sim = m_pSimMatrix->GetSimilarity(w1,w2);
					if(sim < -0.00000001 || sim > 0.00000001)
					{
						cache[word_no].push_back( std::pair<size_t, double> (word_col_no, sim) );
						nz++;
					}
				}
			}
			std::ofstream file( (m_sOutDir+word+".mat").c_str() );
			size_t size = m_CurrentKBest.size() < m_stK ?  m_CurrentKBest.size() : m_stK;
			file << size << " "  << " " << nz << std::endl;
			size_t row_size = 0;
			for(size_t row_no = 0; row_no < cache.size(); row_no++)
			{
				bool row_added = false;
				for(size_t col_no = 0; col_no < cache[row_no].size(); col_no++)
				{
					if(!row_added)
					{
						row_added = true;
						row_size ++;
					}
					std::pair<size_t,double> &elem =  cache[row_no][col_no];
					file << elem.first+1 << " " << elem.second << " ";
				}
				file << std::endl;
			}
			file.close();

			if (Config::cluto_intermediate)
			{
				std::ofstream file_label( (m_sOutDir+word+".rlabel").c_str() );
				for(size_t wordno= 0; wordno< m_CurrentKBest.size() && wordno< m_stK; wordno++)
					file_label << m_CurrentKBest[wordno] << std::endl;
			}
			return row_size;
		}
		void ClutoWrapper::RunCLUTO(const std::string &word, size_t size)
		{
			std::stringstream command;
			command << m_sClutoExe << "-clmethod=" << m_sClMethod << " -crfun=" << m_sCrFun;

			if (Config::cluto_intermediate)
			{
				command << " -rlabelfile=\"" << m_sOutDir+word+".rlabel\"" << " -plotformat=ps";
				command << " -plottree=\"" << m_sOutDir+word+".ps\"";
			}
			command << " -treefile=\"" << m_sOutDir << word << ".tree\"" << " \"" << m_sOutDir << word << ".mat\"";
			command << " " << size / 2;// << " &> /dev/null";

			if (Config::cluto_verbose)
				std::cout << "Cluto command: " << command.str() << std::endl;

			int returnCode = system( command.str().c_str() );
			std::cout<<"Return from CLUTO: "<<returnCode<<std::endl;

			boost::filesystem::remove_all( m_sOutDir + word + ".mat" );
			boost::filesystem::remove_all( m_sOutDir + word + ".mat.clustering."+boost::lexical_cast<std::string>(size/2) );

			if (Config::cluto_intermediate)
				boost::filesystem::remove_all( m_sOutDir + word + ".rlabel" );
		}
		void ClutoWrapper::ReadCLUTOOutput(const std::string &word)
		{
			std::string filename = m_sOutDir+word+".tree";
			std::ifstream parrayfile( filename.c_str() );
			if(!parrayfile)
				throw std::runtime_error( "Cannot read " + filename);
			std::string line;
			std::vector<int> parray;
			while( std::getline(parrayfile, line) )
			{
				boost::trim(line);
				if(line.length() == 0)
					continue;
				std::vector<std::string> tokens;
				boost::split( tokens, line, boost::is_any_of(" \t") );
				parray.push_back( boost::lexical_cast<int>(tokens[0] ) );
			}

			if (Config::cluto_verbose)
			{
				std::copy( parray.begin(), parray.end(), std::ostream_iterator<int>(std::cout, " ") );
				std::cout << std::endl;
			}

			ClusterTree &clusters = m_CurrentCluster;
			clusters.clear();
			for(size_t i = 0; i < parray.size(); i++)
			{
				int pointingto = parray[i];
				clusters[pointingto].push_back(i);
			}

			if (Config::cluto_verbose)
			{
				for(ClusterTree::iterator i = clusters.begin(); i != clusters.end(); i++)
				{
					std::cout << i->first << ": ";
					std::copy( i->second.begin(), i->second.end(), std::ostream_iterator<int>(std::cout, " ") );
					std::cout << std::endl;
				}
				ClusterTree::iterator find_roots = clusters.find(-1);
				for(size_t i = 0; i < find_roots->second.size(); i++)
					PrintTree( find_roots->second[i], clusters,0, word);
			}			
		}//ReadClutioOutput

		void ClutoWrapper::PrintTree(size_t root_id, std::map<int, std::vector<int> >& clusters, size_t level, const std::string &word)
		{
			for(int i = 0; i < (int)level; i++)
				std::cout << "  ";

			if( clusters.find(root_id) == clusters.end() )
			{
				std::cout << m_CurrentKBest[root_id] << std::endl;
				return;
			}
			std::cout << root_id << std::endl;
			std::vector<int>& children = clusters[root_id];
			for(size_t i = 0; i < children.size(); i++)
				PrintTree(children[i], clusters, level+1, word);
		}//void PrintTree(size_t root_id, std::map<int, std::vector<int> >& clusters, size_t level = 0);


		void ClutoWrapper::GetChildren(int root, ClusterTree &clusters, std::vector<int> &leafs)
		{
			if( clusters.find(root) == clusters.end() )
				leafs.push_back(root);
			else
			{
				std::vector<int>& children = clusters[root];
				for(size_t i = 0; i < children.size(); i++)
					GetChildren( children[i], clusters, leafs);
			}
		}

		double ClutoWrapper::ScoreCluster(int cluster, ClusterTree &clusters, const std::string &word)
		{
			std::vector<int> elems;
			GetChildren(cluster, clusters, elems);
			double sum = 0;
			size_t n = 0;
			for(size_t i = 0; i < elems.size()-1; i++)
			{
				for(size_t j = i+1; j < elems.size(); j++)
				{
					sum += m_pSimMatrix->GetSimilarity( m_CurrentKBest[ elems[i] ], m_CurrentKBest[ elems[j] ] );
					n++;
				}
			}
			return (log( elems.size()) * sum)/ double(n);
		}//ScoreCluster

		void ClutoWrapper::SelectBestOne(const std::string &word)
		{
			std::multimap<double, int> clusters_score;
			ClusterTree &clusters = m_CurrentCluster;
			for(ClusterTree::iterator iter = clusters.begin(); iter != clusters.end(); iter++)
			{
				double score = ScoreCluster(iter->first, clusters, word);
				if (Config::cluto_verbose)
					std::cout << "Score for " << iter->first << " = " << score << std::endl;

				clusters_score.insert( std::pair<double, int> (score, iter->first) );
			}

			std::multimap<double,int>::reverse_iterator best = clusters_score.rbegin();
			if(best != clusters_score.rend() )
			{
				if (Config::cluto_verbose)
					std::cout << "Best: " << (best->second) << "( " << (best->first) << "): ";

				std::vector<int> elems;
				GetChildren(best->second, clusters, elems);
				std::set<std::string> c;
				for(size_t i = 0; i < elems.size(); i++)
					c.insert( m_CurrentKBest[elems[i] ] );
				m_pOutClusters->insert(ScoredClusterPair(best->first, c));
				if (Config::cluto_intermediate)
				{
					std::string filename = m_sOutDir + word + ".best_cluster";
					std::ofstream clusterfile(filename.c_str());
					clusterfile << best->first << "\n";
					for(size_t i = 0; i < elems.size(); i++)
						clusterfile << m_CurrentKBest[elems[i] ] << "\n";
					clusterfile.close();
				}

				if (Config::cluto_verbose)
				{
					for(size_t i = 0; i < elems.size(); i++)
						std::cout << m_CurrentKBest[elems[i] ] << " ";
					std::cout << std::endl;
				}
			}
		}//select best one

		ClustersPtr ClutoWrapper::GetClusters()
		{
			return m_pOutClusters;
		}

	}//namespace cbc
}//namespace clustering
