#!/usr/bin/python
# -*- coding: utf-8 -*-
import os, os.path, sys, argparse, codecs


def merge_tuples(tuples_set, output, remove_words, remove_flex):
	all_tuples = {}
	set_size = len(tuples_set)
	counter = 1
	for fname in tuples_set:
		file = codecs.open(fname, "r", 'utf-8')
		lines = file.readlines()
		print "Merging (%d/%d) %s..." % (counter, set_size, fname)
		counter+=1
		small_counter=1
		no_lines = len(lines)
		for line in lines:
			if (small_counter % 1000000)==0:
				print "%d/%d"%(small_counter, no_lines)
			small_counter+=1 
			pos = line.rfind(';')+1
			tuples = line[:pos]
			parts = line.split(';')
			quant = int(parts[3])
			if (parts[0] in remove_words) or (parts[2] in remove_words) or (parts[1] in remove_flex):
				continue
			if (tuples in all_tuples):
				all_tuples[tuples] = all_tuples[tuples]+quant
			else:
				all_tuples[tuples] = quant
		file.close()
	print "Writing file..."
	out = codecs.open(output, 'w', 'utf-8')
	for key, value in all_tuples.items():
		out.write(key+str(value)+'\n')
	out.close()
	print "Done"


def find_tuples(directory):
	if not os.path.exists(directory):
		print "Directory %s doesn't exist"%directory
		sys.exit(1)
	dirList=os.listdir(directory)
	tuples_list = []
	for fname in dirList:
		ext = os.path.splitext(fname)[1]
		if ext=='.tuples':
			tuples_list.append(directory+fname)
		if ext=='.csv':
			tuples_list.append(directory+fname)
	return tuples_list
		
def check_tuples(tuples_list):
	error = False
	for tuples in tuples_list:
		if not os.path.exists(tuples):
			print "%s doesn't exist" % tuples 
			error = True
	return not error

def get_remove_list(remove_path):
	lines = []
	result = []
	if remove_path==None:
		return lines
	if not os.path.exists(remove_path): 
		print "File %s doesn't exist" % remove_path
		sys.exit(1)
	file = codecs.open(remove_path, 'r', 'utf-8')
	lines = file.readlines();
	for line in lines:
		result.append(line[:len(line)-1])
	file.close()
	return result

def main():
	parser = argparse.ArgumentParser(description='Tuples Merger')
	parser.add_argument('--tuples', action='append', default=[],
		help='Path(s) to tuples file')
	parser.add_argument('--tuples-dir', action='store', help='Directory with tuples to merge')
	parser.add_argument('--output', action='store', help='Path to output file')
	parser.add_argument('--remove-words', action='store', help='Path to file with words to exclude')
	parser.add_argument('--remove-flex', action='store', help='Path to file with flex to exclude')
	
	args = parser.parse_args()
	if args.output==None:
		sys.exit("You need to set --output")
	if not check_tuples(args.tuples):
		sys.exit(1)
	if args.tuples_dir!=None:
		dir_set = set(find_tuples(args.tuples_dir))
		tuples_set = dir_set | set(args.tuples)
	else:
		if len(args.tuples)>=1:
			tuples_set = set(args.tuples)
		else:
			print "You need to specify --tuples or/and --tuples-dir"
			sys.exit("You need to specify --tuples or/and --tuples-dir")
	flex_list = get_remove_list(args.remove_flex)
	words_list = get_remove_list(args.remove_words)
	merge_tuples(tuples_set, args.output, words_list, flex_list)

if __name__ == "__main__":
	main()
