# -*- coding: utf-8 -*-
"""
Builds a frequency dictionary out of some corpus.

Author: Bartosz Broda, bartosz.broda@gmail.com
"""

import codecs

class DictBuilder:
	"""
	Unigram dictionary builder.
	"""
	
	def __init__(self, minfreq = 5,  filter = lambda x : False):
		"""
		Arguments:
		- minfreq - a minimal frequency of token that will be written to file
		- filter - a function that returns true if a token (input argument) should be kept in dictionary
		"""
		self.data = {} # word -> count
		self.filter = filter
		self.minfreq = minfreq
		self.tok_count = 0
		
	def process_document(self, reader):
		"""
		Counts tokens in a given file using filters passed in __init__.
		"""
		for tokens in reader:
			for token in tokens:
				self.tok_count += 1
				if token:
					if not self.filter(token):
						if token in self.data:
							self.data[token] += 1
						else:
							self.data[token] = 1

	def dump(self, filename):
		"""
		Writes a dictionary to CSV file.
		First line of the file contains number of tokens in a corpus.
		Subsequent lines contains pairs of a form:
		word; frequency
		"""
		dumped = 0
		l = reversed(sorted( [(self.data[x], x) for x in self.data] ))
		with codecs.open(filename, 'w', 'utf8') as f:
			f.write('%d\n' % self.tok_count)
			for freq,w in l:
				if freq >= self.minfreq:
					f.write('%s; %d\n' % (w, freq) )
					dumped+=1
		print "Dumped %d of %d" % (dumped, len(self.data))

