# -*- coding: utf-8 -*-
'''
Created on Dec 10, 2010

@author: boban
'''
import os,sys
from architect2 import dictionary
from architect2 import filters
from architect2.readers import plaintext
def main():
	import argparse
	
	
	parser = argparse.ArgumentParser(description='Collects unigram dictionary.')
	parser.add_argument('-i', '--in', dest = 'indir', required=True, 
			help = 'Directory or file to process')
	parser.add_argument('-o', '--out', dest = 'out', required=True, 
			help = 'Output filename')
	parser.add_argument('-m', '--minf', dest = 'mintf', default = 100, type = int,
			help = 'Minimal frequency for words written to output, default: 100')
	parser.add_argument('-p', '--prefix', dest = 'prefix', default = 'xx', 
			help = 'Prefix for files that will be processed, default: xx')
	parser.add_argument('-t', '--token', help='Which token type to use',
					 choices=['simple', 'TreeTagger'])
	parser.add_argument('-r', '--taggerpath', 
					help='Path to TreeTagger (use when -t TreeTagger)', 
					)
	args = parser.parse_args()

	if args.token == 'TreeTagger':
		from architect2.utils import treetagger 
		processor = dictionary.DictBuilder(args.mintf, treetagger.eng_textmining() )
		reader_builder = lambda filename : plaintext.single_document_reader(filename, treetagger.TreeTaggerTokenizer)
	else: # simple
		processor = dictionary.DictBuilder(args.mintf, filters.eng_textmining() )
		reader_builder = plaintext.single_document_reader

	if os.path.isdir( args.indir ) :
		allfiles = []
		for x, ddd in sorted([(r,f) for (r,d,f) in os.walk(args.indir) if len(f)] ):
			allfiles.extend( [os.path.join(x, w) for w in ddd if w.startswith(args.prefix)] )

		allfiles = sorted(allfiles)
		print "Processing directory"
		for i,filename in enumerate(allfiles):
			if i % 100 == 0:
				print "Processing %d / %d (%s)" % (i, len(allfiles), filename)
			processor.process_document( reader_builder(filename)  )
	else:
		print "Processing file", args.indir
		processor.process_document( reader_builder(args.indir) )

	processor.dump(args.out)
	
if __name__ == '__main__':
	main()
