#!/usr/bin/python
# -*- coding: utf-8 -*-
import os, os.path

def save_questions( questions, filename, encoding = 'utf8' ):
	"""
	Writes collection of WBSTQuestions to file.
	"""
	import codecs

	if len(questions) == 0 :
		print "No questions to save..."
		return
	print "Writing hwbst test consisting of %s to file %s using encoding %s" % ( len(questions), filename, encoding)

	file = codecs.open(filename, "w", encoding )
	header = u"pytanie;odpowiedź;termin 1" 
	for i in xrange( len(questions[0].detractors) ):
		header += u";termin " + str(i+2)

	file.write(header + "\n")

	for q in questions:
		file.write( q.to_csv_string() + "\n")

	file.close()		
	
def load_words(filename, encoding = "utf8", udict = False ):
	"""
	Loads words from given filename in given encoding.
	If udict = True then it is assumed that file contains words and their 
	frequencies in format:
	words := count
	"""
	print "Loading words..."
	import codecs
	words = []
	f = codecs.open(filename, "r", encoding)
	lines = f.readlines()
	firstLine=True;
	
	
	if lines[0].strip() == 'flex;lemma;frequency':
		
		for line in lines: 
			if firstLine :
				firstLine=False;
				continue;
			line = line.strip()
			if len(line) > 0:
				tokens = line.split(';')
				words.append(tokens[1])
	elif lines[0].strip() == "label;nz;tf;TF;entropy":
		for line in lines: 
			if firstLine :
				firstLine=False;
				continue;
			line = line.strip()
			if len(line) > 0:
				tokens = line.split(';')
				parts = tokens[0].split(':')
				words.append(parts[1])
	else:		
		for line in lines: 
			line = line.strip()
			if len(line) > 0:
				words.append(line)

	return words

def load_udict(filename, flex, encoding = "utf8"):
	print "Loading udict from file %s for flex class %s, assuming encoding %s." \
			% (filename, flex, encoding)
	import codecs
	udict = {}
	f = codecs.open(filename, "r", encoding)
	lines = f.readlines()
	firstLine=True;
	print "Loading dictionary..."
	if lines[0].strip() == 'flex;lemma;frequency':
		for line in lines: 
			if firstLine :
				firstLine=False;
				continue;
			line = line.strip()
			if len(line) > 0:
				tokens = line.split(';')
				
				word = tokens[1].strip()
				word_flex = tokens[0].strip()
				
				if not word_flex in flex: continue
				
				tf = int(tokens[2].strip())
				udict[word] = tf  
	elif lines[0].strip() == "label;nz;tf;TF;entropy":
		for line in lines: 
			if firstLine :
				firstLine=False;
				continue;
			line = line.strip()
			if len(line) > 0:
				tokens = line.split(';')
				parts = tokens[0].split(':')
				tokens = line.split(';')
				
				word = parts[1].strip()
				word_flex = parts[0].strip()
				
				#if not word_flex in flex: continue
				
				tf = int(tokens[2].strip())
				udict[word] = tf
	else:
	
		for line in codecs.open(filename, "r", encoding):
			line = line.strip()
			first_split = line.split(":=")

			if len(first_split) != 2: continue
			flex_word = first_split[0].strip()
		
			second_split = flex_word.split(":")

			if len(second_split) != 2: continue

			word = second_split[1].strip()
			words_flex = second_split[0].strip()
			tf = int( first_split[1].strip() )

			if not words_flex in flex: continue

			udict[word] = tf

	print "Loaded %s words from udict" % len(udict)
	
	return udict

def filter_words(words, threshold, udict):
	frequent = []
	infrequent = []

	for word in words:
		if not word in udict:
			print "No TF information for '%s', will not use it in eighter frequent or infrequent words." % word
			continue
		tf = udict[word]
		if tf >= threshold:
			frequent.append(word)
		else:
			infrequent.append(word)
	
	return frequent, infrequent

def wordnet_filter_lower(words, db):
	all_lemmas = set( db.LexicalUnits.get_all_lemmas())
	all_lower = set([])
	for l in all_lemmas:
		all_lower.add(l.lower())

	filtered_words = []
	for w in words:
		if w in all_lower:
			filtered_words.append(w)
			
	return filtered_words		

def wordnet_filter(words, db):
	all_lemmas = set( db.LexicalUnits.get_all_lemmas())
	filtered_words = []
	for w in words:
		if w in all_lemmas:
			filtered_words.append(w)
			
	return filtered_words		
