import codecs

def load_gazetteer(filename, encoding = 'utf-8'):
	"""
	Loads gazetteer (format with |)
	"""
	file = codecs.open(filename, "r", encoding)

	words = set()
		
	for line in file:
		line = line.strip()
		if len(line) == 0: continue
		tokens = line.split('|')
		word = tokens[0]
		word = word.strip()
		words.add(word.lower())
	file.close()
	return words;

def load_words(filename, encoding='utf-8'):
	"""
	Loads words from plain file
	"""
	file = codecs.open(filename, "r", encoding)

	words = set()

	for line in file:
		line = line.strip()
		if len(line) == 0: continue
		words.add(line)
	file.close()
	return words

def filter(gazetter1_filename_in_utf8, gazetter2_filename_in_utf8, words_filename_in_utf8, to_remove_filename_utf8):
	gazetteer1 = load_gazetteer(gazetter1_filename_in_utf8)
	gazetteer2 = load_gazetteer(gazetter2_filename_in_utf8)
	words = load_words(words_filename_in_utf8)

	diff1 = words.intersection(gazetteer1)
	print diff1
	diff2 = words.intersection(gazetteer2)
	print diff2
	diff2.difference_update(diff1)
	print diff2

	out = codecs.open(to_remove_filename_utf8, "w", "utf-8")
	for w in diff1:
		out.write(w)
		out.write('\n')
	out.write('\n\n')

	for w in diff2:
		out.write(w)
		out.write('\n')

	out.close()

def stoplist(filename_stoplist, filename_words, out_filename, stop_encoding='utf-8', words_encoding='utf-8', out_enc='utf-8'):
	"""
	Filters words using stoplist.

	Params:
	filename_stoplist - file with stoplist
	filename_words - file with words
	out_filename - output file
	stop_encoding - stoplist file encoding
	words_encoding - words encoding
	out_enc - output encoding
	"""
	stoplist = load_words(filename_stoplist, stop_encoding)
	words = load_words(filename_words, words_encoding)

	words.difference_update(stoplist)
	
	out = codecs.open(out_filename, "w", out_enc)
	for w in sorted(words):
		out.write(w)
		out.write('\n')
	out.close()



if __name__ == "__main__":
	import sys
	if len(sys.argv) != 5 and len(sys.argv) != 4:
		print "Usage[1]: %s gazetter_1 gazetter_2 words_file outfile" % sys.argv[0]
		print "Usage[2]: %s stoplist words_file outfile" % sys.argv[0]
		sys.exit(-1)

	if len(sys.argv) == 5:
		filter(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4] )
	if len(sys.argv) == 4:
		stoplist(sys.argv[1], sys.argv[2], sys.argv[3])


