import sys, os, subprocess, codecs
import bash_colours, bnc_to_std, strip_special

def process_file(infile,outfile):
	print infile
	sed1 = subprocess.Popen(["sed", r's/<teiHeader.*<\/teiHeader>//',infile], stdout = subprocess.PIPE )
	sed2 = subprocess.Popen(["sed", 's/<[^<]*>//g'], stdin = sed1.stdout, stdout = subprocess.PIPE)
	sed3 = subprocess.Popen(["sed", '/^\s*$/d'], stdin = sed2.stdout, stdout = subprocess.PIPE) 
	out = sed3.communicate()[0]
	# convert special BNC characters
	out = bnc_to_std.handle_bnc_entities(out)
	#TODO: skonczone na /mnt/big/korpusy/bnc/Texts/A/AE/AE6, zacielo sie na lsqb:
	#file = codecs.open(outfile + "_bnc", 'w', 'utf')
	#file.write(out)
	#file.close()
	# and convert them to standard chars 
	out = strip_special.handle_html_entities(out)
	# write output	
	file = codecs.open(outfile, 'w', 'utf')
	file.write(out)
	file.close()

def process_dir(base_out, dirname, names):
	base_dir, out_dir = base_out 
	common = os.path.commonprefix([dirname, base_dir])
	out_path = os.path.join( out_dir, dirname.replace(common, ""))
	if not os.path.exists(out_path):
		os.makedirs(out_path)
	for filename in names:
		out_file  = os.path.join( out_path, filename)
		in_path = os.path.join(dirname, filename)
		if os.path.exists(in_path) and os.path.isfile(in_path):
			print "%s -> %s" % (dirname, out_file)
			process_file( in_path, out_file)
			#sys.exit(0)

def walk_corpus(in_dir, out_dir):
	os.path.walk(in_dir, process_dir, (in_dir,out_dir) )	

def print_usage(error_msg = None):
	if error_msg:
		bash_colours.msg_error(error_msg)
	print >> sys.stderr, 'Usage: python BNCConverter bnc_dir out_dir'
	sys.exit(-1)
	
if __name__ == '__main__':
	if len(sys.argv) != 3:
		print_usage()
	else:
		bnc_to_std.load_chars()
		in_dir = sys.argv[1]
		out_dir = sys.argv[2]
		if not os.path.isdir(in_dir):
			print_usage("%s is not a directory!" % in_dir)
		if os.path.exists(out_dir):
			print_usage("%s exists! Cannot write to existing directory." % out_dir)
		os.mkdir(out_dir)
		
		walk_corpus(in_dir, out_dir)