#!/usr/bin/python
import codecs,os, sys

input_encoding = 'utf8'
output_encoding = 'utf8'
mintf = 10

def load_plain_words(filename, flexclass, dict):
	"""
	Loads words from 'plain' file, ie file that every line is one word.

	Params
	filename - file to load
	flexclass - flex class of words (not used here)
	dict - where to put words
	"""
	print "Reading words from", filename
	for line in codecs.open(filename, "r", input_encoding):
		line = line.strip()
		if len(line) > 0:			
			dict.add(line)

def load_dict_words(filename, flexclass, dict):
	"""
	Load words from dictionary file, ie file of format:
	word := frequency

	Params:
	filename - file with dictionary
	flexclass - flexclass for adding to dict
	dict - where to put words, according to flex class
	"""
	print "Reading words from", filename
	file = codecs.open(filename, "r", input_encoding)

	if dict.has_key(flexclass):
		words = dict[flexclass]
	else:
		words = {}
		dict[flexclass] = words

	for line in file:
		line = line.strip()
		sp = line.split(":=")
		if len(sp) != 2:
			continue
		word = sp[0].strip();
		count = int(sp[1])
		if count < mintf:
			continue
		if words.has_key(word):
			words[word] = words[word] + count
		else:
			words[word] = count

	file.close()

def read_dir(dir, fun, dict):
	"""
	Reads words from files from single directory
	Params:
	dir - directory containing files wiht naming convention: corpusname_flexclass,
	fun - function for loading single file, function shuld have three parameters: filename, flexclass and dictionary
	dict - where to put loaded words
	"""
	all = os.listdir(dir)
	print all
	for file in all:
		sp = file.split("_")
		if len(sp) < 2:
			continue
		if sp[-1] == 'stats':
			continue
		p = os.path.join(dir,file)
		print p
		fun(p, sp[-1], dict)

	return dict

def write(filename, dict, minlen = 0):
	"""
	Write simple dict (without flex classes) to file

	Params:
	filename - file for dictionary
	dict - dictionary 
	minlen - word lenght filter
	"""
	file = codecs.open(filename, 'w', output_encoding)
	for w in dict:
		if len(w) > minlen:
			file.write(w+"\n")
	file.close()

def write_udict(filename, dict, minlen):
	"""
	Writes unigram dictionary containing flex class information to file.

	Params:
	filename - file for writing
	dict - dictionary {flexclass: words_with_freq), where words_with_freq is any iterable container of pairs string,number
	minlen - minimum length of word to write to file
	"""
	print "Writing udict to file",filename
	file = codecs.open(filename, "w", "cp1250")
	for flexclass in dict:
		words = dict[flexclass]
		for word in words:
			if len(word) < minlen:
				continue
			word = u"%s:%s := %s\n" % (flexclass,word, words[word])
			file.write(word)

	file.close()

def join_plain_words(dirname, fileout, minlen):
	dict = set([])
	read_dir(dirname,load_plain_words,dict)
	write(fileout, dict, minlen)

def join_dict_words(dirname, fileout, minlen, min_tf):
	dict = {}
	mintf = min_tf
	dict = read_dir(dirname,load_dict_words, dict)
	write_udict(fileout, dict, minlen)


