# -*- coding: utf-8 -*-
import codecs, pymatrices



def write_test(filename, test, enc='utf8'):
	file = codecs.open(filename, 'w', enc)
	file.write(u'pytanie;odpowiedź;termin 1;termin 2;termin 3;termin 4\n')
	for line in sorted(test):
		file.write(line + '\n')
	file.close()
	
	
def main(mdir, mname, test_file, out_dir, test_name, flex='subst'):
	print "Frequency splitter"
	matrix = pymatrices.LoadSuperMatrix(mdir, mname, False, False)
	
	rowfeatures = matrix.GetRowFeatures()
	rows = len(rowfeatures)

	tf10 = set([])
	tf10_words = set([])
	tf100 = set([])
	tf100_words = set([])
	tf500 = set([])
	tf500_words = set([])
	tf1000 = set([])
	tf1000_words = set([])

	tf_pon1000 = set([])
	tf_pon1000_words = set([])

	first = True
	brak = 0
	slowa_brakujace = set([])

		
	for line in codecs.open(test_file, 'r', 'utf8'):
		if first: # skip header
			first = False
			continue

		line = line.strip()
		if len(line) == 0: continue

		tokens = line.split(';')
		word = flex + ':' + tokens[0].strip()
		w = word.encode('utf8')
		row_id = matrix.GetRowIndex(w)
		if row_id > rows:
			print "No '%s' in matrix!" % word
			brak += 1
			slowa_brakujace.add(tokens[0])
			continue
		tf = rowfeatures[row_id].GetGlobalTermFrequency()

		if tf >= 10:
			tf10.add(line)
			tf10_words.add(tokens[0])
		if tf >= 100:
			tf100.add(line)
			tf100_words.add(tokens[0])
		if tf >= 500:
			tf500.add(line)
			tf500_words.add(tokens[0])
		if tf >= 1000:
			tf1000.add(line)
			tf1000_words.add(tokens[0])
		if tf <= 1000:
			tf_pon1000.add(line)
			tf_pon1000_words.add(tokens[0])


	print "Brakowalo %d pytan dla %d slow: %s" % (brak, len(slowa_brakujace), slowa_brakujace)

	print "Zbior tf>10 sklada sie z %d pytan (%d slow)" % (len(tf10), len(tf10_words))
	print "Zbior tf>100 sklada sie z %d pytan (%d slow)" % (len(tf100), len(tf100_words))
	print "Zbior tf>500 sklada sie z %d pytan (%d slow)" % (len(tf500), len(tf500_words))
	print "Zbior tf>1000 sklada sie z %d pytan (%d slow)" % (len(tf1000), len(tf1000_words))
	print "Zbior tf<1000 sklada sie z %d pytan (%d slow)" % (len(tf_pon1000), len(tf_pon1000_words))
	print "Writing test consisting of %s to file %s " % ( len(tf1000), out_dir + test_name + '.csv')
	write_test(out_dir + test_name + '.csv', tf1000)

	
if __name__ == "__main__":
	import sys, argparse
	parser = argparse.ArgumentParser(description='Frequency spliter')
	#positional arguments
	parser.add_argument('m_dir',  action='store', 
		help='Directory to the matrix')
	parser.add_argument('m_name',  action='store', 
		help='Matrix name')
	parser.add_argument('test_file',  action='store', 
		help='Path to the file with test')
	parser.add_argument('out_dir',  action='store', 
		help='Path to the output directory')
	parser.add_argument('test_name',  action='store', 
		help='Filename of new test (without extension)')
	parser.add_argument('-f', action='store', 
		dest='flex', default='subst',
		help='Type of flex (default=\'subst\')')
	
	args = parser.parse_args()
	
	main(args.m_dir, args.m_name, args.test_file, args.out_dir, args.test_name, args.flex)
	

