#!/usr/bin/python
# -*- coding: utf-8 -*-
import os, os.path, wordsutils, codecs, sys
from optparse import OptionParser


def select_questions(wbst_test, words, mintf, maxtf):
	new_test = set([])
	counter = 0;
	for qline in wbst_test:
		answers = qline.split(';')
		if counter == 0:
			counter = 1
			continue;
		for answer in answers:
			if not answer in words:
				counter += 1
				print "Question/answer %s not in dict. " % answer
				continue
			tf = words[answer]
			if tf >= mintf and tf < maxtf:
				new_test.add(qline)
	return new_test

def load_wbst(filename, encoding= 'utf-8'):
        first_line = True
        questions = set([])
        for line in codecs.open(filename, 'r', encoding):
                if first_line:
                        first_line = False
                        continue
                questions.add( line.strip() )

        return questions

def load_udict(filename, words, mintf = 5, encoding = 'utf-8'):
        """
        Load words from dictionary file, ie file of format:
        word := frequency

        Params:
        filename - file with dictionary
        encoding - file encoding
        """
        print "Reading words from", filename
        file = codecs.open(filename, "r", encoding)

        for line in file:
                line = line.strip()
                sp = line.split(":=")
                if len(sp) != 2:
                        continue
                word = sp[0].strip();
                count = int(sp[1])
                if count < mintf:
                        continue
                if words.has_key(word):
                        words[word] = words[word] + count
                else:
                        words[word] = count
        file.close()


def write_test(filename, test, encoding = 'utf-8'):
        header = u"pytanie;odpowiedź"
        first_q = test.pop()
        no_of_terms = len(first_q.split(';')) - 2
        test.add(first_q)
        for i in xrange(no_of_terms):
                header += u';termin %s' % (i+1)
        header += '\n'
        out = codecs.open(filename, 'w', encoding)
        out.write(header)
        for q in sorted(test):
                out.write(q + '\n')
        out.close()





def main():
	parser = OptionParser()
	parser.add_option("-w", "--words", action="store", dest="words_file", help="File with words")
	parser.add_option("-t", "--test", action="store",dest="test",  help="File with wbst test")
	parser.add_option("-i", "--mintf", action="store",dest="min_tf", type="int", default = 1, help="Minimum value of tf for question/answer")
	parser.add_option("-x", "--maxtf", action="store",dest="max_tf", type="int", default = 10000000000000, help="Maximum value of tf for question/answer")
	parser.add_option("-o", "--output", action="store",dest="output",  help="Path to output file")
	options = parser.parse_args()	
	
	if options[0].test==None or options[0].words_file==None or options[0].output==None:
		print 'You need to specify arguments --words, --test and --output'
	else:
		words = wordsutils.load_udict(options[0].words_file, [])
		wbst_test = load_wbst(options[0].test)
		test = select_questions(wbst_test, words, options[0].min_tf, options[0].max_tf)
		write_test(options[0].output, test)


#	words = wordsutils.load_udict('/home/lukasz/jacek/wiersze_nasze.csv', [])
#	wbst_test = load_wbst('/home/lukasz/jacek/hwbst_plwn_1_4_subst_all_words.csv')
#	test = select_questions(wbst_test, words, 1, 10000000000000000000)
#	write_test('/home/lukasz/jacek/kipirzepa4HWBST.csv', test)

if __name__ == "__main__":
	main()


	

