'''
TODO: unit testing of this module 

Created on Dec 12, 2010

@author: Bartosz Broda, bartosz.broda@gmail.com
'''
from architect2.tokenizers import short
from lxml import etree

def single_document_reader(filename, tokenization_function = short.word_punct_tokenizer):
	root = etree.parse(filename)
	yield tokenization_function(etree.tostring(root, method='text', encoding='utf8').decode('utf8').strip())
	

def multi_pre_morph_reader(filelist, tokenization_function = short.word_punct_tokenizer):
	for filename in filelist:
		single_doc_gen = single_document_reader(filename, tokenization_function)
		yield single_doc_gen.next()  