#!/usr/bin/python
from __future__ import print_function
import argparse as _argp
import csv as _csv

import numpy as _np
import time,glob,os,shutil,tempfile
from subprocess import call
from sklearn.externals import joblib

verbose = False

# Change names of features by replacing Polish letters
def remove_accents(text):
    text=unicode(text,"utf-8")
    trans = dict((ord(a), b+'_') for a, b in zip(u'ąćęłńóśżźĄĆĘŁŃÓŚŻŹ', u'acelnoszzACELNOSZZ'))
    
    return str(text.translate(trans))
	
	
# Reads data from set of csvs from fextor
# Creats matrix and normalise it (divides by tok_count)
def read_csvs(inputDir):	
	files=[];
	start_time = time.time()
	for filename in glob.glob(inputDir+'/*'):
		files.append(filename)
		
	rowlabels = tuple(files)
	rowvals = [None] * len(rowlabels)
	colset = set()
	tokcounts=[1] * len(rowlabels)
	tok_count='tok_count'
	for i, filename in enumerate(rowlabels):
		with open(filename, 'rb') as csv_ifs:
			cr = _csv.DictReader(csv_ifs, delimiter=';')
			try:
				valdict = next(cr)
			except StopIteration:
				raise ValueError('Malformed csv:', filename)

			colset.update(valdict)
			tokcounts[i]= int(valdict[tok_count])
			del valdict[tok_count]
			rowvals[i] = valdict
			
	if verbose:
		print("Read --- %s seconds ---" % (time.time() - start_time));start_time = time.time()
	
	# Any ordering must be enforced here, so convert to tuple
	colset.remove(tok_count)
	collabels = tuple(colset)
	col2idx = {col: idx for idx, col in enumerate(collabels)}

	arr = _np.zeros((len(rowlabels), len(collabels)), dtype=float)
	for i, cdict in enumerate(rowvals):
		for cnam, cval in cdict.iteritems():
			arr[i, col2idx[cnam]] = float(cval)/tokcounts[i];			
	if verbose:
		print("Merge --- %s seconds ---" % (time.time() - start_time));start_time = time.time()
			
	collabels = [remove_accents(colname) for colname in collabels]	
	rowlabels = [os.path.basename(rowname) for rowname in rowlabels]	
	
	return arr, rowlabels, collabels
	

def number_of_clusters(options,rowlabels):
	if 'no_clusters' in options:
		no_clusters=options['no_clusters']
		if not isinstance( no_clusters, int ):
			no_clusters=2
		if no_clusters<2:
			no_clusters=2
	else:
		no_clusters=2
	
	if int(no_clusters) > len(rowlabels):
		no_clusters = str(len(rowlabels))
		
	return no_clusters

def save_clutofiles(mat,rlabels,clabels,cluto_path,rlabel_path,clabel_path):
	# Save cluto file
	with open(cluto_path, 'w') as cluto_ofs:
		# Print header:
		# <num_rows> <num_cols> <num_nonzero>
		print(
				len(rlabels),
				len(clabels),
				_np.count_nonzero(mat),
				file=cluto_ofs,
				)

		for row in mat:
			buf = []

			for idx in row.nonzero()[0]:
				buf.append('{} {}'.format(idx+1, row[idx]))

			print(' '.join(buf), file=cluto_ofs)

	# Save label files
	with open(rlabel_path, 'w') as rlabel_ofs:
		for lab in rlabels:
			print(lab, file=rlabel_ofs)

	with open(clabel_path, 'w') as clabel_ofs:
		for lab in clabels:
			print(lab, file=clabel_ofs)

def run_cluto(options,no_clusters,cluto_input_file,rlabel_path,clabel_path,cl_out_file):
	clmethod="";
	if options['sim']=='dist' or  options['sim']=='jacc':
		clmethod='-clmethod=graph';
		if options['analysis_type']=='plotmatrix':
			options['analysis_type']='plotsmatrix'
		if options['analysis_type']=='plotclusters':
			options['analysis_type']='plotsclusters'
		if options['analysis_type']=='plottree':
			options['analysis_type']='plotsclusters'	
	#Call Cluto
	#cluto_path=self.config.get("options", "cluto_path")  
	cluto_path="/home/tomekw/3rdparty/cluto-2.1.2/Linux-x86_64/vcluster"
	#print cluto_input_file
	
	if clmethod!="":
		call([cluto_path, cluto_input_file, str(no_clusters),
			  '-rlabelfile', rlabel_path,
			  '-clabelfile', clabel_path,
			  '-plotformat','ps','-sim='+options['sim'],clmethod,
			  '-'+options['analysis_type']+'=' + cl_out_file,
			  '-colmodel=none'])
	else:
		call([cluto_path, cluto_input_file, str(no_clusters), '-fulltree',
			  '-rlabelfile', rlabel_path,
			  '-clabelfile', clabel_path,
			  '-plotformat','ps','-sim='+options['sim'],
			  '-'+options['analysis_type']+'=' + cl_out_file,
			  '-colmodel=none'])
		
		#print("fulltree")
				

def write_node(node_id, tree_dict):
    node_str = '{"id":"node_' + node_id + '", "name":"' + node_id + '", "data":{}, "children":['
    child_node_strings = []

    if node_id in tree_dict:
        for child in tree_dict[node_id]:
            child_node_strings.append(write_node(child, tree_dict))

    node_str += ', '.join(child_node_strings)
    node_str += ']}'
    return node_str
				
def run_convert2json(cl_out_file,out_file,labels):
    tree_dict = {}
    with open(cl_out_file, 'rb') as infile:
        for i, line in enumerate(infile.readlines()):
            if i < len(labels):
                child = labels[i]
            else:
                child = str(i)
            parent = line.split(' ')[0]
            if parent not in tree_dict:
                tree_dict[parent] = [child]
            else:
                tree_dict[parent].append(child)

    out_string = ''
    out_string += write_node(tree_dict['-1'][0], tree_dict)
    out_string += ''

    with open(out_file, 'wb') as outfile:
        outfile.write(out_string)

				
def run_convert(cl_out_file,out_file,options,rowlabels):
	density='150'
	if options['analysis_type']!='plottree':
		density='300'
	
	if len(rowlabels)<50:
		density='100'

	if len(rowlabels)<25:
		density='50'
	
		
	if options['analysis_type']=='plottree':
			resize='50%'
	else:
			resize='100%'
	
	#print density
	call(['convert','-density',density,cl_out_file,'png:'+out_file])
	
def run(inputFile, outputFile, options):
	
	#Fextorbis fomat, set of (CSVS)
	data,rowlabels, collabels=read_csvs(inputFile);
	
	no_clusters=number_of_clusters(options,rowlabels)
	
	'''
	if not 'weighting_method' in options or options['weighting_method']!='tf-idf':
	   options['weighting_method']='tf'	

	if options['weighting_method'] == 'tf-idf':
		data = tf_idf_weighting_method(data)  
	else:
		data = tf_weight_matrix(data)
	'''
	
	temp_folder = tempfile.mkdtemp()
	#temp_folder=outputFile
	if not os.path.exists(temp_folder):
			os.mkdir(temp_folder)
	
	cluto_path=os.path.join(temp_folder, 'matrix.txt');
	rlabel_path=os.path.join(temp_folder, 'documents_ids.txt');
	clabel_path=os.path.join(temp_folder, 'column_labels.txt');
	cluto_out_path=os.path.join(temp_folder, 'cluto.ps');

	save_clutofiles(data,rowlabels,collabels,cluto_path,rlabel_path,clabel_path)
	run_cluto(options,no_clusters,cluto_path,rlabel_path,clabel_path,cluto_out_path) 	
	os.mkdir(outputFile)
	run_convert2json(os.path.join(temp_folder, 'matrix.txt.tree'),os.path.join(outputFile,'result.json'),rowlabels)
	run_convert(cluto_out_path,os.path.join(outputFile,'result.png'),options,rowlabels);
	
	#for features
	shutil.copy2(os.path.join(temp_folder, 'matrix.txt.clustering.'+str(no_clusters)),os.path.join(outputFile,'result.clustering'))
	joblib.dump(collabels,outputFile+"/columns.pkl");
	joblib.dump(rowlabels,outputFile+"/rowlabels.pkl");
	joblib.dump(data,outputFile+"/data.pkl");
	
	#print temp_folder
	
	shutil.rmtree(temp_folder)	
	