# -*- coding: utf-8 -*-
import codecs, morfeusz

def load_plain_list(filename, where):
	for line in codecs.open(filename, 'r', 'utf8'):
		line = line.strip()
		if len(line):
			where.add(line)


def filter_flex(words, flex):
	bases = set([])
	for w in words:
		if ' ' in w: continue
		analyse_results = morfeusz.analyse( w )
		for analysis in analyse_results:
			for single_analysis in analysis:
				base = single_analysis[1]
				ctag = single_analysis[2]
				if base == None or ctag == None:
					continue #we ommit igns
				sflex = ctag[:ctag.find(':')]
				if sflex == flex and base:
					bases.add(base)
	return bases

def save_words(filename, words):
	file = codecs.open(filename, 'w', 'utf8')
	for w in words:
		file.write(w+'\n')
	file.close()

if __name__ == '__main__':
	words = set([])
	stoplist = set([])
	load_plain_list('/mnt/data/dane/slowniki/angpol/wszystkie_tlumaczenia.txt', words)
	load_plain_list('/home/boban/tmp/words_wn.txt', words)
	load_plain_list('./adjs_frequent_to_morph.txt', words)
	bases = filter_flex(words, 'adj')
	load_plain_list('/mnt/data/dane/slowa/stoplisty/stoplista-Adj.utf', stoplist)
	save_words('adjs_wn_angpol_frequent_corpora_stoplist.txt', sorted(bases.difference(stoplist)) )


