import time,glob,os,json
import csv as _csv
import numpy as _np

verbose = True
# Reads data from set of csvs from fextor
# Creats matrix and normalise it (divides by tok_count)

translate={
 "lexclass_count:verb":"czasowniki",
 "lexclass_count:prep":"przyimki",
 "lexclass_count:all_adj":"przymiotniki",
 "lexclass_count:adv":"przysłówki",
 "lexclass_count:noun":"rzeczowniki",
}


def read_csvs(inputDir):	
	files=[];
	start_time = time.time()
	for filename in glob.glob(inputDir+'/*'):
		files.append(filename)
		
	rowlabels = tuple(files)
	rowvals = [None] * len(rowlabels)
	colset = set()
	tokcounts=[1] * len(rowlabels)
	tok_count='tok_count'
	for i, filename in enumerate(rowlabels):
		with open(filename, 'rb') as csv_ifs:
			cr = _csv.DictReader(csv_ifs, delimiter=';')
			try:
				valdict = next(cr)
			except StopIteration:
				raise ValueError('Malformed csv:', filename)

			colset.update(valdict)
			tokcounts[i]= int(valdict[tok_count])
			del valdict[tok_count]
			rowvals[i] = valdict
			
			
	if verbose:
		print("Read --- %s seconds ---" % (time.time() - start_time));start_time = time.time()
	
	# Any ordering must be enforced here, so convert to tuple
	colset.remove(tok_count)
	collabels = tuple(colset)
	col2idx = {col: idx for idx, col in enumerate(collabels)}

	arr = _np.zeros((len(rowlabels), len(collabels)), dtype=float)
	for i, cdict in enumerate(rowvals):
		for cnam, cval in cdict.iteritems():
			arr[i, col2idx[cnam]] = float(cval)/tokcounts[i];			
	if verbose:
		print("Merge --- %s seconds ---" % (time.time() - start_time));start_time = time.time()
			
	#collabels = [remove_accents(colname) for colname in collabels]	
	rowlabels = [os.path.basename(rowname) for rowname in rowlabels]	
	toks=dict();
	for indx,rowname in enumerate(rowlabels):
		toks[rowname]=tokcounts[indx];
	return arr, rowlabels, collabels,toks

	
def convert2csv(data,rowlabels, collabels,toks,output_file):
	with open(output_file, 'wb') as csv_ofs:
		wr = _csv.writer(csv_ofs, delimiter=';')
		list=list = [elem for elem in collabels]
		list.insert(0,"file");
		wr.writerow(list);
		for rowidx,rowname in enumerate(rowlabels):
			list=_np.rint(data[rowidx]*toks[rowname]).tolist();
			list = [int(float(num)) for num in list]
			list.insert(0,rowname);
			wr.writerow(list)	
		
def getgraphs(rowlabels):
	graphs=[];
	for colname in rowlabels:
		graph=dict();
		#graph["balloonText"]="Częstość [[category]] w "+ colname+ ": <b>[[value]]</b>",
		graph["balloonText"]="Czestość [[value]] dla [[category]]";
		graph["fillAlphas"]=0.9;
		graph["lineAlpha"]= 0.2;
		graph["title"]=getcolname(colname);
		graph["type"]= "column";
		graph["valueField"]=getcolname(colname);
		graphs.append(graph);
	return graphs;

def getcolname(name):	
	if name in translate:
		return translate[name]
	return name	
	
def getdata(data,rowlabels, collabels):
	results=[];
	for colidx,colname in enumerate(collabels):
		if not(colname in translate):
			continue;
		result=dict();
		result["feature"]= getcolname(colname);
		for rowidx,rowname in enumerate(rowlabels):			
			result[rowname]=data[rowidx][colidx]*100
		results.append(result);
	return results;
	
def convert2json(data,rowlabels, collabels,toks,out_file):
  
	result=dict();
	
	result["graphs"]=getgraphs(rowlabels);
	result["dataProvider"]=getdata(data,rowlabels, collabels);
	result["categoryField"]= "feature"
	
	result["valueAxes"]=[{"stackType": "3d","unit": "%","position": "left","title": "Częstość cech"}];
	result["toks"]=toks;

	with open(out_file, 'w') as fp:
		json.dump(result, fp)
	
	
def process(inputFile, outputFile, options):
	
	#Fextorbis fomat, set of (CSVS)
	data,rowlabels, collabels,toks=read_csvs(inputFile);
	if not os.path.exists(outputFile):
		os.mkdir(outputFile)
	convert2json(data,rowlabels, collabels,toks,os.path.join(outputFile,'result.json'))
	convert2csv(data,rowlabels, collabels,toks,os.path.join(outputFile,'result.csv'))
	
def main():
	   
	print "Start"
	
	
	process('test/p','result','{}')
	print "End"
	
if __name__ == '__main__':
	main()

