/*
    Copyright (C) 2012 Bartosz Broda, Jacek Bochenek, Łukasz Biłeńkij, 
    Damian Jaworski, Wojciech Mazur, Dominik Piasecki, Maciej Piasecki, 
    Radosław Ramocki
    
    This program is free software; you can redistribute it and/or modify it
under the terms of the GNU Lesser General Public License as published by the Free
Software Foundation; either version 3 of the License, or (at your option)
any later version.

    This program is distributed in the hope that it will be useful, but
WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
or FITNESS FOR A PARTICULAR PURPOSE. 

    See the LICENSE and COPYING files for more details.
*/

#include "HierarchicalSOM.h"

namespace clustering
{
	namespace som
	{

		HierarchicalSOM::HierarchicalSOM(boost::shared_ptr<Matrices::SuperMatrix> matrix,
			boost::shared_ptr<smartcomparator::simfunctions::LocalSimilarityFunction> simfun, double t1, double t2,
			int neurons , int epoch )
			:m_Matrix(matrix),
			m_pSimFunction(simfun), _t1(t1), _t2(t2)

		{
			CountMQE0();
			m_cEpoch = epoch;
			m_stInitialNeurons = neurons;
			m_cThreads = 1;
		}

		HierarchicalSOM::~HierarchicalSOM()
		{
		}

		double HierarchicalSOM::CountMQE0()
		{
			std::cout << "Counting MQE0...";
			m_dMQE0 = 0.;

			size_t number_of_rows = m_Matrix->NumberOfRows();
			std::cout << std::endl << number_of_rows << std::endl;
			Matrices::CompressedVector mean(number_of_rows, NULL);

			for(size_t i = 0; i < number_of_rows; i++)
				mean += *(m_Matrix->GetCRS()->GetCompressedRow(i));

			Measures::DevideByValue(mean, (double)number_of_rows);
			Measures::DevideBySum(mean);// normalization for cosine

			for(size_t i = 0; i < number_of_rows; i++)
			{
				double sim = m_pSimFunction -> CountSimilarity(mean, *m_Matrix->GetCRS()->GetCompressedRow(i));
				m_dMQE0 +=  sim;
			}

			m_dMQE0 /= (double)number_of_rows;
			std::cout << "MQE_o = " << m_dMQE0 << std::endl;
			return m_dMQE0;
		}

		double HierarchicalSOM::CountMapMeanMQE(std::vector< std::vector<double> > &map_desc)
		{
			double mean = 0;
			size_t dev = 0;
			for(std::vector< std::vector<double> >::iterator row = map_desc.begin();
				row != map_desc.end(); row++)
			{

				for(std::vector<double>::iterator mqe = row->begin(); mqe != row->end(); mqe++)
				{
					if( *mqe != 0) //WM: what is this condition for?
					{
						mean += *mqe;

					}
					dev++;
				}
			}
			return mean/(double)dev;
		}

		void HierarchicalSOM::Start()
		{
//			boost::shared_ptr<CSOMNet> net( new CSOMNet(m_stInitialNeurons, m_Matrix, m_pSimFunction,
//				m_dMQE0, m_cThreads));

			boost::shared_ptr<CSOMNet> net( new CSOMNet(m_stInitialNeurons, boost::shared_ptr<Matrices::SuperMatrix>(m_Matrix), m_pSimFunction,
				m_dMQE0));

			BuildHierarchy(net, 0, "");
			CreateClustersFromSOM();
			CreateLinksFromSOM();
		}

		void HierarchicalSOM::BuildHierarchy(boost::shared_ptr<CSOMNet> net, size_t layerNo, std::string name)
		{
			using std::vector;
			using boost::shared_ptr;

			std::cout << "Training layer " << layerNo << " map. Parent MMQE = " << net->GetMQE0() << std::endl;
			net->TrainNet((int)m_cEpoch);

			m_Maps.insert(std::pair<size_t, boost::shared_ptr<CSOMNet> >(layerNo, net) );

			double MQEm = CountMapMeanMQE(net->CountMQE());

			while( ExpandFurther(*net, MQEm) )
			{
				net->SetNumberOfNeurons(net->GetNumberOfNeurons() + 1 );
				std::cout << "Expanding network: " << net->GetId() << " layer: " << layerNo << " new size: " << net->GetNumberOfNeurons() << std::endl;
				net->TrainNet((int)m_cEpoch);
				MQEm = CountMapMeanMQE(net->CountMQE());
			}

			std::cout << "MMQE = " << CountMapMeanMQE(net->CountMQE() ) << std::endl;

			//WM: what is this for?
			std::pair<size_t, size_t> target = FindNeuron(net->GetParentId(), layerNo-1);

			std::string filename = std::string("layer_")+boost::lexical_cast<std::string>(layerNo)
				+ std::string("_") + name.substr(0, name.size()-1) + std::string(".som");
			//net->BinarySaveState(filename);
			std::cout << "Filename: " << filename << std::endl;
			vector<vector<shared_ptr<CSOMNeuron> > > neurons = net->GetNeurons();

			size_t rowNo = 0;
			for(vector<vector<shared_ptr<CSOMNeuron> > >::iterator row = neurons.begin(); row != neurons.end(); row++)
			{
				vector<shared_ptr<CSOMNeuron> > &neuronRow = *row;
				size_t colNo = 0;

				for(vector<shared_ptr<CSOMNeuron> >::iterator neuronInRow = neuronRow.begin();
					neuronInRow != neuronRow.end(); neuronInRow++)
				{
					shared_ptr<CSOMNeuron> neuron = *neuronInRow;

					if( GoDeeper(*net, *neuron, layerNo) )
					{
//						boost::shared_ptr<CSOMNet> net(
//							new CSOMNet(m_stInitialNeurons, m_Matrix, m_pSimFunction,
//									neuron->GetMQE(), m_cThreads,
//									neuron->GetDataIndices()));
						boost::shared_ptr<CSOMNet> net(
								new CSOMNet(m_stInitialNeurons, boost::shared_ptr<Matrices::SuperMatrix>(m_Matrix), m_pSimFunction,
									neuron->GetMQE(), neuron->GetDataIndices()));
						net->SetParentId(neuron->GetId());
						std::string sufixname = boost::lexical_cast<std::string>(rowNo) + "_x_"
							+ boost::lexical_cast<std::string>(colNo) + "_";
						BuildHierarchy(net, layerNo+1, name+sufixname );
					}
					colNo++;
				}
				rowNo++;
			}
		}

		bool HierarchicalSOM::ExpandFurther(CSOMNet &net, double MQEm)
		{
			//double t1 = 0.75; //expansion factor
			double mqe0 = 1-net.GetMQE0(); //1-mqe?

			if ((1-MQEm) > _t1*mqe0)
				return true;

			return false;
		}

		bool HierarchicalSOM::GoDeeper(CSOMNet &net, CSOMNeuron& neuron, size_t currlayer)
		{
			//double t2 = 0.05; //expansion factor

			//return false; //WM: Was uncommented
			if(currlayer > 10)
				return false;

			/*if( neuron.GetDataIndices()->size() > 20)
			return true;*/
			double mqe0 = 1-net.GetMQE0();

			//condition after && ??
			if( (1-neuron.GetMQE()) > _t2 * mqe0 && neuron.GetDataIndices()->size() > (m_stInitialNeurons*m_stInitialNeurons*2))
				return true;

			return false;
		}

		void HierarchicalSOM::TestSOMHierarchies()
		{
			size_t counter = 0;
			std::string prefix = "som_";
			std::string type = "tfidf_";

			for(size_t neurons = 2; neurons < 10; neurons++)
			{

				std::ofstream log_((type+std::string("log_") + boost::lexical_cast<std::string>(neurons) + std::string(".txt")).c_str());


				//CSOMNet net0(neurons,m_Matrix, m_pSimFunction, 0, 5);
				CSOMNet net0(neurons, boost::shared_ptr<Matrices::SuperMatrix>(m_Matrix), m_pSimFunction, 0);

				net0.TrainNet(5);

				std::vector< std::vector<double> > &mqs = net0.CountMQE();
				double mqe_layer_0 = CountMapMeanMQE(mqs);

				std::cout << "net_0 mqe = " << mqe_layer_0 << std::endl;
				log_ << "mqe_0 = " << m_dMQE0 << std::endl;
				log_ << "net_0 mqe = " << mqe_layer_0 << std::endl;


				std::cout << "Pozostale: " ;
				log_<< "Pozostale: " ;
				for(size_t i = 0; i < mqs.size(); i++)
				{
					for(size_t j = 0; j < mqs[i].size(); j++)
						if(mqs[i][j] > 0)
						{
							std::cout << mqs[i][j] << ", ";
							log_ << mqs[i][j] << ", ";
						}
				}
				std::cout << std::endl<< std::endl;
				log_ << std::endl;

				for(size_t row = 0; row < neurons; row++)
				{

					for(size_t column = 0; column < neurons; column++)
					{
						counter++;
						boost::shared_ptr<std::set<size_t> > set = net0.GetData(row, column);

						if(mqs[row].size() > column)
						{
							std::cout << "Trenuje podsiec[" << row << ", " << column << "] z mqe = "
								<< mqs[row][column] << std::endl;
							log_ << "Trenuje podsiec[" << row << ", " << column << "] ("<< counter <<")z mqe = "
								<< mqs[row][column] << std::endl;

						}
						else
						{
							std::cout << "Trenuje podsiec[" << row << ", " << column << "] z mqe = 0\n";
							log_ << "Trenuje podsiec[" << row << ", " << column << "] ("<< counter <<")z mqe = 0\n ";

						}

						//CSOMNet net(neurons,m_Matrix, m_pSimFunction, 0, 5, set);
						CSOMNet net(neurons, boost::shared_ptr<Matrices::SuperMatrix>(m_Matrix), m_pSimFunction, 0, set);

						net.TrainNet(5);

						std::vector< std::vector<double> > &MQE_M = net.CountMQE();

						std::cout << set->size() << " - net_" << counter <<" mqe = "
							<< CountMapMeanMQE(MQE_M) << std::endl;

						log_ << set->size() << " - net_" << counter <<" mqe = "
							<< CountMapMeanMQE(MQE_M) << std::endl << "\t";

						for(size_t i = 0; i < MQE_M.size(); i++)
						{
							for(size_t j = 0; j < MQE_M[i].size(); j++)
								if(MQE_M[i][j] > 0)
								{
									std::cout << MQE_M[i][j] << ", ";
									log_ << MQE_M[i][j] << ", ";
								}
						}

						std::cout << std::endl;
						log_ << std::endl<< std::endl;

						//net.BinarySaveState( prefix + boost::lexical_cast<std::string>(counter)+".som");
					}
				}
			}
		}

		std::pair<size_t, size_t> HierarchicalSOM::FindNeuron(size_t neuronId, int layerNo)
		{
			//XXX is it ok?
			//WM: size_t == -1 ? changed to int, not sure if it's ok now
			if(layerNo == -1)
				return std::pair<size_t,size_t>(0,0);

			std::multimap<size_t, boost::shared_ptr<CSOMNet> >::iterator layer = m_Maps.begin();

			for(; layer != m_Maps.end() ; layer++)
			{
				//if(layer->first == layerNo)
				//	continue;

				std::vector< std::vector<boost::shared_ptr<CSOMNeuron> > >& neurons = layer->second->GetNeurons();
				for(size_t row = 0; row < neurons.size(); row++)
				{
					std::vector<boost::shared_ptr<CSOMNeuron> > & neuronRow=  neurons[row];
					for(size_t column = 0; column < neuronRow.size(); column++)
					{
						boost::shared_ptr<CSOMNeuron> &thisNeuron = neuronRow[column];

						if(thisNeuron->GetId() == neuronId)
							return std::pair<size_t, size_t>(row,column);
					}
				}
			}

			return std::pair<size_t,size_t>(0,0);
		}

		void HierarchicalSOM::CreateClustersFromSOM()
		{
			using namespace ClusteringBase;
			m_pClusterSet = ClusterSetPtr(new ClusterSet());

			for(std::multimap<size_t, boost::shared_ptr<CSOMNet> >::iterator map = m_Maps.begin();
				map != m_Maps.end(); map++)
			{
				boost::shared_ptr<CSOMNet> &net = map->second;
				for(size_t row = 0; row < net->GetNumberOfNeurons(); row++)
				{
					for(size_t column = 0; column < net->GetNumberOfNeurons(); column++)
					{
						boost::shared_ptr<std::set<size_t> > data = net->GetData(row,column);
						boost::shared_ptr<CSOMNeuron> neuron = net->GetNeurons()[row][column];

						if(data->size() == 0)
							continue;

						ClusteringBase::ClusterPtr cluster(new ClusteringBase::Cluster(neuron->GetId()) );

						for(std::set<size_t>::iterator data_index = data->begin(); data_index != data->end(); data_index++)
							cluster->AddPoint(*data_index);

						m_pClusterSet->insert(cluster);
					}
				}
			}
		}

		void HierarchicalSOM::CreateLinksFromSOM()
		{
			m_pLinkSet = ClusteringBase::LinkSetPtr(new ClusteringBase::LinkSet());
			for(std::multimap<size_t, boost::shared_ptr<CSOMNet> >::iterator map = m_Maps.begin();
				map != m_Maps.end(); map++)
			{
				boost::shared_ptr<CSOMNet> &net = map->second;
				size_t id_parent = net->GetParentId();//bardziej ogolna grupa

				if(id_parent == 0)
					continue;

				if(net->GetParentId() != 0)
				{
					for(size_t row = 0; row < net->GetNumberOfNeurons(); row++)
					{
						for(size_t column = 0; column < net->GetNumberOfNeurons(); column++)
						{
							boost::shared_ptr<std::set<size_t> > data = net->GetData(row,column);
							boost::shared_ptr<CSOMNeuron> neuron = net->GetNeurons()[row][column];

							if(data->size() == 0)
								continue;

							size_t id_child = neuron->GetId();// grupa szczegolowa

							ClusteringBase::LinkPtr link( new ClusteringBase::Link(id_parent,id_child));
							m_pLinkSet->insert(link);

						}
					}
				}
			}
		}

		ClusteringBase::ClusterSetPtr HierarchicalSOM::GetClusters()
		{
			return m_pClusterSet;
		}

		ClusteringBase::LinkSetPtr HierarchicalSOM::GetLinks()
		{
			return m_pLinkSet;
		}

		ClusterVectorPtr HierarchicalSOM::GetClustersIndices(bool firstLayerOnly)
		{
			ClusterVectorPtr result = ClusterVectorPtr(new std::vector<size_t>(m_Matrix->NumberOfRows()));

			std::multimap<size_t, boost::shared_ptr<CSOMNet> >::iterator map = m_Maps.begin();
			for (; map != m_Maps.end(); ++map)
			{
				if (firstLayerOnly && map->first > 0) //skip layers other than 0
					continue;
				boost::shared_ptr<CSOMNet> &net = map->second;

				for(size_t row = 0; row < net->GetNumberOfNeurons(); row++)
				{
					for(size_t column = 0; column < net->GetNumberOfNeurons(); column++)
					{
						CSOMNeuron & neuron = *net->GetNeurons()[row][column];
						std::set<size_t> indices = *neuron.GetDataIndices(); //change to shared pointer?
						for (std::set<size_t>::iterator iter = indices.begin(); iter != indices.end(); ++iter)
						{
							(*result)[*iter] = neuron.GetId();
						}
					}
				}
			}
			return result;
		}

		std::multimap<size_t, boost::shared_ptr<CSOMNet> > HierarchicalSOM::GetMaps()
		{
			return m_Maps;
		}

	}//namespace SOM
}//namespace Clustering

