# -*- coding: utf-8 -*-
"""
Some example token filters.

Author: Bartosz Broda, bartosz.broda@gmail.com

TODO: semantics of filters: true when word should be filtered, not the other way around
"""

import nltk.corpus
import codecs

en_stoplist = set(nltk.corpus.stopwords.words('english'))

def textmining():
	"""
	Very simple filter that removes preserves only alphabetical characters and words 
	with more then two letters
	
	>>> filter = textmining()
	>>> filter('dog')
	False
	>>> filter('d')
	True
	>>> filter('d3222')
	True
	>>> filter('http://www.google.pl/')
	True
	>>> filter('animal')
	False
	"""
	return lambda t: not t.isalpha() or len(t) < 3

def eng_textmining():
	"""
	Simple filter that removes English stop words, preserves only alphabetical characters and words 
	with more then two letters
	
	>>> filter = eng_textmining()
	>>> filter('dog')
	False
	>>> filter('d')
	True
	>>> filter('d3222')
	True
	>>> filter('http://www.google.pl/')
	True
	>>> filter('animal')
	False
	>>> filter('the')
	True
	"""
	return lambda t: t.lower() in en_stoplist or not t.isalpha() or len(t) < 3

def dict_eng_filter(dictfile):
	r"""
	A simple word filter for English. It removes stop words and words with less then 3 chars, 
	keeps only words that contains only alphabetical characters and every word tested has to be 
	in a given dictionary.

	Arguments:
	- dictifle a (unigram) dictionary; first line is skipped (i.e., it can be a header or a number of tokens), 
	  every subsequent line is in form word; TF

	Returns: a function that filters given tokens.
	
	>>> def mock_open(f, m, e):
	...     import tempfile
	...     tempfile = tempfile.TemporaryFile()
	...     tempfile.write(u'123\n')
	...     tempfile.write(u'dog;5\n')
	...     tempfile.write(u'animal;15\n')
	...     tempfile.write(u'cat;3\n')
	...     tempfile.seek(0)
	...     return tempfile
	>>> orig_open = codecs.open
	>>> codecs.open = mock_open
	>>> filter = dict_eng_filter('dummy')
	>>> filter('animal')
	False
	>>> filter('dog')
	False
	>>> filter('cat')
	False
	>>> filter('cat1')
	True
	>>> filter('car')
	True
	>>> filter('looks good')
	True
	>>> filter('an')
	True
	>>> codecs.open = orig_open
	"""
	with codecs.open(dictfile, 'r', 'utf8') as f:
		dictionary = dict( [(y[0], int(y[1])) for y in [ x.strip().split(';') for x in f.readlines()[1:]] ] ) 
	return lambda t: t.lower() in en_stoplist or not t.isalpha() or len(t) < 3 or t not in dictionary

def generic_dict_filter(dictfile, inner_filter):
	with codecs.open(dictfile, 'r', 'utf8') as f:
		dictionary = dict( [(y[0], int(y[1])) for y in [ x.strip().split(';') for x in f.readlines()[1:]] ] )
	return lambda t: inner_filter(t) or not t in dictionary	
	

if __name__ == '__main__':
	import doctest
	doctest.testmod()