#!/usr/bin/python
# -*- coding: utf-8 -*-

import sys, types

from xml.sax import saxutils
from xml.sax.handler import EntityResolver

import string
import cStringIO

def normalize_whitespace(text):
    """Remove redundant whitespace from a string 
    taken from http://pyxml.sourceforge.net/topics/howto/node14.html """
    return ' '.join(text.split())

class KBestXmlDocumentHandler(saxutils.DefaultHandler, EntityResolver):
    
    def __init__(self):
        self.inSynset = False;
        self.currentLexicalUnit = ""
        self.inLU = False
        self.currentLU = ""
        self.synsetLU = []
        self.similarities = {}
    
    def resolveEntity(self, publicId, systemId):
        return cStringIO.StringIO()
    
    def startElement(self, name, attrs):
        if name == 'synset':
            self.inSynset = True
            unit = normalize_whitespace(attrs.get('lexicalunit')).encode("cp1250")
            self.currentLexicalUnit = unit
            self.synsetLU = []
        elif name == 'lu':
            self.inLU = True
            self.currentLU = ""    
    
    def characters(self, ch):
        if self.inLU:
            self.currentLU += ch
    
    def endElement(self, name):
        if name == 'synset':
            self.inSynset = False
            self.similarities[self.currentLexicalUnit] = self.synsetLU
                       
        elif name == 'lu':
            self.currentLU = normalize_whitespace(self.currentLU)
            self.synsetLU.append(self.currentLU.encode("cp1250"))
            self.inLU = False

def extract(simFile):
    from xml.sax import make_parser
    from xml.sax.handler import feature_namespaces
    parser = make_parser()
    parser.setFeature(feature_namespaces, 0)
    dh = KBestXmlDocumentHandler();
    parser.setContentHandler(dh)
    parser.setEntityResolver(dh)
    file = open(simFile, "r")
    parser.parse(file)
    file.close()
    
    return (dh.similarities)


def select_words_by_total_mi(matrix_name, matrix_dir, min_total_mi):
	import pymatrices, pycomparator
	print matrix_dir
	print matrix_name
	matrix = pymatrices.LoadSuperMatrix(matrix_dir, matrix_name, False, True)
	pycomparator.mi_weight_matrix(matrix)

	words = []
	removed = []

	no_of_rows = matrix.NumberOfRows()
	for row_no in xrange(no_of_rows):
		row = matrix.GetRow(row_no)
		row_sum = pymatrices.CountVectorSum(row)
		if row_sum > min_total_mi:
			word = matrix.GetRowFeatures()[row_no].GetLabel()
			words.append( word )
		else:
			removed.append( word )

	del matrix

	return (words, removed)
	
