'''
Created on Dec 7, 2010

@author: boban
'''
import tempfile, shutil, codecs, os
from lxml import etree
import nltk.tokenize as tokenize
import subprocess
import sys

from architect2 import filters
from architect2.processors import  MemProcessor

def morph_to_tokens(filename):
	# grep 'disamb="1"' %s | cut -d'>' -f 3 | cut -d'<' -f 1
	# return [etree.tostring(x.xpath('base')[0], encoding='utf8', method='text').decode('utf8').lower().strip() for x in etree.parse(filename).xpath('//tok/lex[@disamb=1]')]
	
	output = subprocess.Popen(["grep 'disamb=\"1\"' %s | cut -d'>' -f 3 | cut -d'<' -f 1" % filename], stdout=subprocess.PIPE, shell=True).communicate()[0]
	#print [x.decode('utf8').lower().strip() for x in output.split() ]
	return [x.decode('utf8').lower().strip() for x in output.split() ]
	

def premorph_to_tokens(filename):
	root = etree.parse(filename)
	
	return [x.strip().lower() for x in tokenize.WordPunctTokenizer().tokenize(etree.tostring(root, method='text', encoding='utf8').decode('utf8') ) ]

def build_dict(filelist, filter):
	"""
	tutaj filtr filtruje, t.j. zwraca true jezeli slowo powinno byc odfiltrowane
	"""
	d = {} # word -> count
	tok_count = 0
	errors = 0
	for i,filename in enumerate(filelist):
		try:
			if i % 1000 == 0:
				print '%s (%d/%d)' % (filename, i, len(filelist))
			for token in morph_to_tokens(filename):
				tok_count += 1
				if not filter(token):
					if token in d:
						d[token] += 1
					else:
						d[token] = 1
		except:
			errors += 1
			print "Error in", filename
			#import sys
			#sys.exit()
	if errors > 0:
		print "Encountered %d errors while parsing %d files" % (errors, len(filelist))
	return d, tok_count

def triples_gen(filename, udict):
	for token in morph_to_tokens(filename):
		if token in udict:
			yield (filename, u'UNORDERED', token)

def main():
	inlist = [x.strip() for x in codecs.open('./pryzmaty_tag.txt', 'r', 'utf8').readlines()]
	outdir = '/home/boban/tmp/'
	outname = 'pryzmaty'
	tempdir = tempfile.mkdtemp( dir = './')
	print tempdir
	# dictionary creation
	print "Creation of dictionary"
	udict, tokcount = build_dict(inlist, filters.textmining() )
	tf2tok = list(reversed( sorted([(udict[k], k) for k in udict if udict[k] > 5] ) ))
	all_c = float(len(tf2tok))
	to_remove_freq = 100 # usuwamy 100 najczestszych
	to_remove_infreq = int(0.01 * all_c) # i procent infrequent
	
	cut_dict = tf2tok[to_remove_freq:-to_remove_infreq]
	
	filtered_udict = dict( (k,f) for (f,k) in cut_dict  )
	print "Dictionary after filtering has %d position" % (len(filtered_udict))
	del tf2tok
	del udict
	
	# matrix creation
	print "Creation of matrix"
	processor = MemProcessor()
	for i,doc in enumerate(inlist):
		if i % 1000 == 0:
			print '%s (%d/%d)' % (doc, i, len(inlist))
			fn = os.path.join(tempdir, 'triples_%d.txt' % i) 
			processor.dump( fn )
		processor.process( triples_gen(doc, filtered_udict) )
	processor.dump(os.path.join(tempdir, 'triples_last.txt'))
	
	# write dict
	udictname = os.path.join(tempdir, 'udict.txt')
	print udictname
	with codecs.open(udictname, 'w', 'utf8') as f:
		f.write('%d\n' % tokcount)
		for word in filtered_udict:
			f.write('%s; %d\n' % (word, filtered_udict[word]))
		for fn in inlist:
			f.write('%s; %d\n' % (fn, 1) )
	# write matrix
	
	cmd = 'python ../matrices/tri2mat.py -i %s -o %s -u %s -n %s -p triples' % (tempdir, outdir, udictname, outname)
	print cmd
	os.system(cmd)
	
	
	# cleanup
	
	shutil.rmtree(tempdir)
	
if __name__ == '__main__':
	main()
	
