'''
Created on Dec 12, 2010

@author: Bartosz Broda, bartosz.broda@gmail.com
'''
import codecs
from architect2.tokenizers import short

def single_document_reader(filename, tokenization_function = short.word_punct_tokenizer, encoding = 'utf8'):
	yield tokenization_function(codecs.open(filename, 'r', encoding).read().strip())


def multi_document_reader(filelist, tokenization_function = short.word_punct_tokenizer):
	for filename in filelist:
		single_doc_gen = single_document_reader(filename, tokenization_function)
		yield single_doc_gen.next()  