'''
Created on Sep 11, 2010

@author: boban
'''
import nltk.data
import nltk.tokenize as tokenize
from nltk.stem.porter import PorterStemmer
import treetaggerwrapper
from  architect2.triples import triples

import nltk.corpus
import codecs

stoplist = set(nltk.corpus.stopwords.words('english'))

class TreeTaggerToken(object):
	
	def __init__(self, lemma, tag ):
		self.tag = tag
		self.lemma = lemma
		
	def __unicode__(self):
		return self.lemma + u':' + self.tag
	
	def __repr__(self):
		return self.lemma + ':' + self.tag
	
	def __hash__(self):
		return hash( unicode(self) )
	
	def __eq__(self, other):
		return unicode(self).__eq__(unicode(other) )
	
	def __gt__(self, other):
		return unicode(self).__gt__(unicode(other))
	
class TreeTaggerTokenizer:
	def __init__(self, treetagger = None):
		self.tagger = treetaggerwrapper.TreeTagger(TAGLANG='en', TAGDIR=treetagger)
		self.stemmer = PorterStemmer() 
	
	def __call__(self, text):
		text = '\n'.join( tokenize.WordPunctTokenizer().tokenize(text))
		tokens = []
		for (w,t,l) in [ tuple( x.split() )  for x in self.tagger.TagText(text) ]:
			if l == u'<unknown>':
				l = self.stemmer.stem(w).lower()
			tokens.append(TreeTaggerToken(l,t))
		return tokens

# filters
def eng_textmining():
	return lambda t: t.lemma.lower() not in stoplist and t.lemma.isalpha() and len(t.lemma) >= 3

def dict_eng_filter(dictfile):
	with codecs.open(dictfile, 'r', 'utf8') as f:
		dictionary = set([])
		for line in f.readlines()[1:]:
			token = line.split(';')[0]
			lemma, tag = token.split(':')
			dictionary.add( TreeTaggerToken(lemma, tag) )
	return lambda t: t.lemma.lower() not in stoplist and t.lemma.isalpha() and len(t.lemma) >= 3 and t in dictionary

def noun_selector():
	return lambda t: t.tag.startswith('N')

def noun_features():
	return lambda t: t.tag.startswith('V') or t.tag.startswith('J') or t.tag.startswith('N')

def tree_tagger_test():	
	t = TreeTaggerTokenizer('/home/boban/software/TreeTagger')
	text = codecs.open('outwiki/wiki_0/xx0000000', 'r', 'utf8').read().strip()
	punkt = nltk.data.load('tokenizers/punkt/english.pickle')
	for sentence in punkt.tokenize(text, realign_boundaries=True):
		tokens = t(sentence)
		
		print sentence		
		gen =  triples.gen_ordered_triples(winarch.gen_contexts(tokens, 3,3), noun_selector(), noun_features()  ) 
		for context in gen:
			print unicode(context[0]), context[1],unicode(context[2])
		print '\n---'

if __name__ == '__main__':
	tree_tagger_test()