#!/usr/bin/python
# -*- coding: utf-8 -*-

wbst_file_encoding="cp1250"

class TestSet(object):
	"""
	WBST test set, with results devided on: good answers, wrong answers
	"""
	def __init__(self, filename):
		self.wrong = {}
		self.good = set([])
		self.load_file(filename)

	def load_file(self, filename):
		"""
		Reads results of WBST test from filename. 
		"""
		import codecs
		import re

		file = codecs.open(filename, "r", wbst_file_encoding)

		wrong_pattern = re.compile(r'\s*<wrong_answer\s+choosen="(.*)">(.*)</wrong_answer>\s*')
		good_pattern = re.compile(r'\s*<answer>(.*)</answer>\s*')
		correct_pattern = re.compile( r'\s*<correct>(.*)</correct>\s*')
		incorrect_pattern = re.compile( r'\s*<incorrect>(.*)</incorrect>\s*')

		correct = 0
		incorrect = 0
		
		print "Reading wbst results from %s. Skipping unanwserd." % filename
		for line in file:
			line.strip()
			match = correct_pattern.match(line)
			if match:
				correct =  int(match.group(1).strip() )
				continue
			
			match = incorrect_pattern.match(line)
			if match:
				incorrect = int(match.group(1).strip())
				continue
			
			match = wrong_pattern.match(line)
			if match:
				answer = match.group(1).strip()
				q_line = match.group(2).strip()
				self.wrong[q_line] = answer
				continue

			match = good_pattern.match(line)
			if match:
				q_line = match.group(1).strip()
				self.good.add(q_line)

		print "Correct in file: %s\nIncorrect in file: %s" % (correct, incorrect)
		if correct != len(self.good):
			print "Number of correct anwsers do not equal to those read from file!"

		if incorrect != len(self.wrong):
			print "Number of incorrect anwsers do not equal to those read from file!"

		print "Read %s good, %s bad, accuracy = %s " % (len(self.good), len(self.wrong),
													 len(self.good)/float( len(self.good)+len(self.wrong)) )
def test_mcnemar(ts1, ts2):
	"""
	Uses McNemar's test to test if difference in two tests is statisticaly significant
	"""
	import math
	wrong1 = set( ts1.wrong.keys() )
	wrong2 = set( ts2.wrong.keys() )
	# no of examples misclassified by ts1 and ts2
	n00 = len( wrong1.intersection(wrong2) )
	# no of examples misclassified by neither ts1 and ts2
	n11 = len( ts1.good.intersection( ts2.good ) )
	# no of examples misclassified by ts2 but not by ts1
	n10 = len( wrong2.difference( wrong1 ) )
	# no of examples misclassified by ts1 but not by ts2
	n01 = len( wrong1.difference( wrong2 ))
	print """
	n00 = %s (no of examples missclassified by ts1 and ts2)
	n01 = %s (no of examples missclassified by ts1 but not by ts2)
	n10 = %s (no of examples missclassified by by ts2 but not by ts1)
	n11 = %s (no of examples missclassified by neither ts1 nor ts2)
	""" % (n00,n01,n10,n11)
	chi2 = math.pow( math.fabs(n01-n10) - 1  , 2 ) /float(n01+n10)
	print "chi^2 value =", chi2
	if chi2 > 3.841459:
		print "Difference is statistically significant for ch^2_{1, 0.95}."
	else:
		print "Difference is not significant for ch^2_{1, 0.95}."

	if chi2 > 6.635:
		print "Difference is statistically significant for ch^2_{1, 0.99}."
	else:
		print "Difference is not significant for ch^2_{1, 0.99}."

	if chi2 > 7.879:
		print "Difference is statistically significant for ch^2_{1, 0.995}."
	else:
		print "Difference is not significant for ch^2_{1, 0.995}."


def perform_mcnemar_test(first_test_filename, second_test_filename):
		ts1 = TestSet(first_test_filename)
		ts2 = TestSet(second_test_filename)
		test_mcnemar(ts1, ts2)


if __name__ == "__main__":
	import sys
	if len(sys.argv) != 3:
		print "Usage %s first_test second_test" % sys.argv[0]
	else:
		perform_mcnemar_test(sys.argv[1], sys.argv[2])
