#!/usr/bin/python
# -*- coding: utf-8 -*-

import codecs
import morfeusz


def word_has_analysis(word):
	try:
		analyses=morfeusz.analyse(word, False, False, False)
		a = analyses[0]
	except:
		return False

	if a[0][1] == None:
		return False
	else:
		return True

def main():
	outfile = '/home/jawor/korpusy/kgr7/kolumny_filtorwane_kgr7.txt'
	infile = '/home/jawor/korpusy/kgr7/dict_kgr7.txt'
	fin_prate_count = 0
	substs = 0
	rare_substs = 0
	removed_fin_praet = 0
	removed_subst = 0
	removed_adj_pact_ppas = 0
	adjs = 0
	skipped_junk = 0
	with codecs.open(outfile, 'w', 'utf8') as out:
		first_line = True
		line_no = 0
		for line in codecs.open(infile, 'r', 'utf8'):
			if line_no % 100000 == 0:
				print "Processing %d / 8336555" % line_no
			line_no += 1
			line = line.strip()
			if '/' in line or '\\' in line or '(' in line or ')' in line or '|' in line or ':' in line or ',' in line or '.' in line or '?' in line or '%' in line or '"' in line or "'" in line or '[' in line or ']' in line or '&' in line:
				skipped_junk += 1
				continue
			if first_line:
				out.write(line + '\n')
				first_line = False
				continue
			if len(line) == 0:
				continue
			try:
				flex, lemma, tfstr = line.split(';')
				if len(lemma) < 3:
					skipped_junk += 1
					continue
			except:
				print "Skipping %d line: %s" % (line_no, line)
				continue
			tf = int(tfstr)
			has_analysis = word_has_analysis(lemma)
			if flex in ['subst', 'depr', 'ger']:
				if (tf > 20 and tf < 200):
					if has_analysis:
						out.write(line + '\n')
						substs+=1
						rare_substs+=1
					else:
						removed_subst += 1
				elif tf > 200:
					out.write(line + '\n')
					substs+=1
			elif flex in ['fin', 'praet']:
				if  has_analysis:
					out.write(line + '\n')
					fin_prate_count += 1
				else:
					removed_fin_praet += 1
			elif flex in ['winien', 'bedzie']:
				out.write(line + '\n')
			elif flex in ['adj', 'pact', 'ppas']:
				if has_analysis or tf > 100:
					out.write(line + '\n')
					adjs+=1
				else:
					removed_adj_pact_ppas  += 1


	print "Fin and praet:", fin_prate_count
	print "Removed fin/preat:", removed_fin_praet
	print "Adj, pact and ppas:", adjs
	print "Removed Adj, pact and ppas:", removed_adj_pact_ppas
	print "Subst:", substs
	print "Rare subst:",rare_substs
	print "Removed subst:", removed_subst
	print "skipped junk:", skipped_junk



	
if __name__ == '__main__':
	main()
