# -*- coding: utf-8 -*-
'''
Filtering by frequency of k-best lists.
Created on 2010-05-26

@author: Bartosz Broda
'''
import sys

import pymatrices
from wnp_eval import kbest

def main():
	filename = sys.argv[1]
	mdir = sys.argv[2]
	mname = sys.argv[3]
	TF = int(sys.argv[4])
	out = sys.argv[5]

	matrix = pymatrices.LoadSuperMatrix(mdir, mname, False, False)

	row_features = matrix.GetRowFeatures()

	print "Loading kbest file from", filename 
	klist = kbest.parse_kbest_txt_file(filename, 1000000)
	print "Loaded kbest lists for %d words" % len(klist)
	new_kbest = {}

	for head in klist:
		f = row_features[ matrix.GetRowIndex(head.encode('utf8')) ].GetTermFrequency()
		if f >= TF:
			new_kbest[head] = klist[head]
	print "After filtering: ", len(new_kbest)
	kbest.write_kbest( out, new_kbest )
	

if __name__ == '__main__':
	main()

