#!/usr/bin/python
# -*- coding: utf-8 -*-

import pymatrices

mdir = '/home/boban/workdir/macierze/supermatrix2/noun/polaczone/'
mname = 'kipi_knigi_rzepa_substAdjP_AmIsubj_Coord_NGen_pq'

matrix = pymatrices.LoadSuperMatrix(mdir, mname, False, True)

rows = matrix.NumberOfRows()
row_features = matrix.GetRowFeatures()
all = 0
all_not = 0
in_rows = []
for row_no in xrange(rows):
	f = row_features[row_no]
	tf = f.GetTermFrequency()
	vals = matrix.GetRow(row_no).GetValues()
	in_curr_row = 0
	for col_no in vals.keys():
		all += 1
		val = vals[col_no]
		p = float(val) / float(tf)

		pr = float(tf) * p * (1 - p)
		if pr <= 5:
			all_not += 1
			in_curr_row += 1
	
	if len( vals.keys() ) > 0:
		in_rows.append( in_curr_row / float(len(vals.keys())) )

	if row_no % 100 == 0:
		print "%d/%d" % (row_no, rows)

print "NNZ:", all
print "np(1-p)<=5", all_not
print "np(1-p)<=5 in [%]", float(all_not) / float(all)

print "np(1-p)<=5 avg for row", sum(in_rows) / float(len(in_rows)) 


