import codecs
import argparse
from collections import defaultdict


argpar = argparse.ArgumentParser(description = "Script options",
		formatter_class = argparse.ArgumentDefaultsHelpFormatter)

argpar.add_argument("input", help = "input")
argpar.add_argument("frequency", help = "frequency file")
argpar.add_argument("prog", help = "prog czestosciowy poniżej którego odsiewamy")
argpar.add_argument("output", help = "output")
options = argpar.parse_args()



f_in = codecs.open(options.input,'r','utf8')
f_freq = codecs.open(options.frequency,'r','utf8')
f_out = codecs.open(options.output,'w','utf8')

freq = defaultdict(int)

i=0
for line in f_freq:
	if i!=0:
		#print line
		splited = line.split(";")
		howManyWords = 0
		word = ''
		for s in splited:
			if s.isdigit():
				break
			else:
				word += s+" "
				howManyWords+=1
		word = word[:-1]
		freq[word] = int(splited[howManyWords+1])
	i+=1
	if i%10000==0:
		print "Done: ",str(i)


czyWpisywac = 0
i =0
for line in f_in:
	if '\t' not in line:
		#print line.rstrip(),"   ->   ",freq[line.rstrip()]
		if freq[line.rstrip()] > int(options.prog):
			czyWpisywac = 1
		else:
			czyWpisywac = 0

	if czyWpisywac == 1:
		f_out.write(line)

	i+=1
	if i%10000==0:
		print "Done: ",str(i)


f_out.close()
