#!/usr/bin/python
# -*- coding: utf-8 -*-

import pymatrices, math
import scipy.stats as st
import scipy.sparse


def write_reg_data(matrix, outfile):
	out = open(outfile, 'w')
	out.write("Zaleznosc miedzy TF a nnz\n")
	out.write("label;tf;nnz\n")
	rowfeatures = matrix.GetRowFeatures()
	for row_no in xrange( matrix.NumberOfRows() ):
		row = matrix.GetRow(row_no)
		feature = rowfeatures[row_no]

		tf = feature.GetGlobalTermFrequency()
		nnz = feature.GetNZ()
		label = feature.GetLabel()
		if nnz:
			out.write("%s;%d;%d\n" % (label,tf, nnz) )

	out.close()

def get_diff(matrix):
	diffs = []
	rowfeatures = matrix.GetRowFeatures()
	for row_no in xrange( matrix.NumberOfRows() ):
		feature = rowfeatures[row_no]

		tf = feature.GetGlobalTermFrequency()
		nnz = feature.GetNZ()
		label = feature.GetLabel()
		if nnz:
			y = 1.78 * math.pow(float(tf), 0.72)
			diff = math.fabs(float(nnz) - y)
			diffs.append(diff)
	
	return diffs

def get_outliers(matrix, moment, outfile):
	rowfeatures = matrix.GetRowFeatures()
	out = open(outfile, 'w')
	out.write("Label;TF;y - szacowana; nnz\n")
	for row_no in xrange( matrix.NumberOfRows() ):
		feature = rowfeatures[row_no]

		TF = feature.GetGlobalTermFrequency()
		tf = feature.GetTermFrequency()
		nnz = feature.GetNZ()
		label = feature.GetLabel()
		if nnz and nnz < 100:
			y = 1.78 * math.pow(float(TF), 0.72)
			if y - float(nnz) > moment:
				out.write("%s;%d;%f;%d\n" % (label, TF, y, nnz) )
	
def main():
	mdir = '/home/boban/workdir/macierze/supermatrix2/noun/polaczone/'
	mname = 'kipi_knigi_rzepa_substAdjP_AmIsubj_Coord_NGen_pq'
	matrix = pymatrices.LoadSuperMatrix(mdir, mname, False, True)

	outfile='do_wyrzucenia.csv'
	#write_reg_data(matrix, outfile)
	diffs = get_diff(matrix)
	mean = st.mean(diffs)
	get_outliers(matrix, mean *0.5, outfile)
	print "mean: %f" % mean


if __name__ == '__main__':
	main()
