#encoding: utf8
import codecs
from architect2.processors import dumpingmemprocessor

def gen_inpar_triples(paragraph_list):
	for line in paragraph_list:
		id_, par = line.strip().split('\t')
		tokens = par.split()
		for tok in tokens:
			yield (tok, 'INPAR', id_)

def main(par_file, dump_file, dump_every=100000):
	proc = dumpingmemprocessor.DumpingMemProcessor(dump_file, dump_every)
	with codecs.open(par_file, 'r', 'utf8') as f:
		proc.process(gen_inpar_triples(f))
	proc.dump(dump_file)

if __name__ == '__main__':
	import sys
	main(sys.argv[1], sys.argv[2], int(sys.argv[3]) if len(sys.argv) >= 4 else 100000)
