# -*- coding: utf-8 -*-
import codecs, morfeusz, sys

def filter_flex(features, flex):
	bases = set([])
	for l, nz, tf, TF, ent in features:
		if ' ' in l: continue
		rel, w = l.split(':')
		analyse_results = morfeusz.analyse( w )
		for analysis in analyse_results:
			for single_analysis in analysis:
				base = single_analysis[1]
				ctag = single_analysis[2]
				if base == None or ctag == None:
					continue #we ommit igns
				sflex = ctag[:ctag.find(':')]
				if sflex == flex:
					bases.add(l)
	return bases

def save_words(filename, words):
	file = codecs.open(filename, 'w', 'utf8')
	for w in sorted(words):
		file.write(w+'\n')
	file.close()

def load_features(filename):
	features = set([])
	for line_no, line in enumerate(codecs.open(filename, 'r', 'utf8')):
		if line_no > 0:
			line = line.strip()
			if len(line):
				lemma, nz, tf, TF, entropy = line.split(';')
				features.add( (lemma, int(nz), int(tf), int(TF), float(entropy)  ) )
	return features
				
if __name__ == '__main__':
	features_file = '/home/boban/premorph_tagged/column_features.csv'
	features = load_features(features_file)
	print "Before filtering:", len(features)
	filtered = filter_flex(features, 'subst')
	print "After  filtering:", len(filtered)
	print "Writing words to file"
	save_words( 'out.txt', filtered)


