#!/usr/bin/python
# -*- coding: utf-8 -*-
import codecs, os

def load_file(filename, where):
	for line in codecs.open(filename, 'r', 'utf8'):
		line = line.strip()
		if len(line) == 0: continue
		where.add( line.split(':=')[0].strip() )

def load_dir(dirname, where):
	for filename in os.listdir(dirname):
		full_path = os.path.join(dirname, filename)
		if filename.startswith('.') or not os.path.isfile( full_path ):
			continue
		load_file(full_path, where)


if __name__ == '__main__':
	subst_file = '/home/boban/tmp/2/subst/'
	adj_verb_file = '/home/boban/tmp/2/verb_adj/'
	substs = set([])
	adjverb = set([])
	load_dir(adj_verb_file, adjverb)
	load_dir(subst_file, substs)

	subst_mwe_count = 2153
	print "Loaded %d subst and %d adj/verbs" % (len(substs), len(adjverb) )
	print "(subst+mwe) / (adj+verb) =", ( ( float(len(substs) + subst_mwe_count) ) / float(len(adjverb)) )
