# -*- coding: utf-8 -*-
import subprocess, codecs, os, sys

#subprocess.Popen(['./morpha.ix86_linux', ""], stdout=subprocess.PIPE, stdin=subprocess.PIPE).communicate('synthesizers_N ')[0]

class MiniPar:
	
	def __init__(self, morpha_path = '/home/boban/software/morph/morpha.ix86_linux', \
				morpha_verbstem =  '/home/boban/software/morph/verbstem.list', \
				minpar_data_path = '/home/boban/software/minipar/data', \
				minpar_pdemo_path = '/home/boban/software/minipar/pdemo/pdemo'):
		self.morpha_path = morpha_path
		self.minipar_data = minpar_data_path
		self.minipar_pdemo = minpar_pdemo_path
		self.morpha_verbstem = morpha_verbstem
		self.ommit_cats = set(['U','Det','Prep','VBE','Have', 'SentAdjunct', 'Q'])
		self.w1_categories = set(['N', 'NN'])
		self.TF = {}
		self.triplets = {}

	def count_tf(self, sentence):
		p = subprocess.Popen([self.minipar_pdemo,'-l' ,'-i', '-p', self.minipar_data], stdout=subprocess.PIPE, stdin = subprocess.PIPE)
		p.stdin.write(sentence.encode('utf8'))
		p.stdin.close()
			
		for line in p.stdout:
			tokens =  line.strip().split('\t')
			if len(tokens) > 1 and tokens[1] != '(()':
				word = tokens[1][1:].decode('utf8')
				root_cat = tokens[2].split()
				root = ( " ".join(root_cat[0:-1]) ).decode('utf8') 
				cat = root_cat[-1]
				if cat in self.ommit_cats: continue
				if root == '~' and ( cat == 'N' or cat == 'V' ):
					analysed = self.analyse(word, cat)
					root = analysed.decode('utf8')				
				if root == '~': 
					root = word
				p = (cat, root)
				if p in self.TF:
					self.TF[p] = self.TF[p] + 1
				else:
					self.TF[p] = 1
	
	def analyse(self, word, cat):
		to_analyse = word + u"_" + cat + u" "
		p = subprocess.Popen([self.morpha_path, '-f', self.morpha_verbstem], stdout=subprocess.PIPE, stdin=subprocess.PIPE)
		p.stdin.write(to_analyse.encode('utf') )
		p.stdin.close()
		return p.stdout.read()
	
	def count_coocurrances(self,sentence):
		p = subprocess.Popen([self.minipar_pdemo, '-t', '-i', '-p', self.minipar_data], stdout=subprocess.PIPE, stdin = subprocess.PIPE)
		p.stdin.write(sentence.encode('utf-8') )
		p.stdin.close()
		for line in p.stdout:
			tokens = line.strip().split()
			if len(tokens) == 3:
				w1 = tokens[0].decode('utf8')
				w2 = tokens[2].decode('utf8')
				if w1 == '~' or w2 == '~':
					continue
				#TODO: sprowadzanie słów do rdzeni
				reltokens = tokens[1].split(':')
				cat1 = reltokens[0]
				if not cat1 in self.w1_categories: continue 
				cat2 = reltokens[2]
				if cat1 in self.ommit_cats or cat2 in self.ommit_cats:
					continue
				relation = reltokens[1]
				
				triple = (w1, relation, w2)
				if triple in self.triplets:
					self.triplets[triple] = self.triplets[triple] +1
				else:
					self.triplets[triple] = 1  
				

def parse_file(minipar, filename):
	print "Processing", filename
	lines = int( subprocess.Popen(["wc", "-l", filename], stdout=subprocess.PIPE).communicate()[0].split()[0] )
	line_no = 0
	for line in codecs.open(filename, 'r', 'utf'):
		line_no += 1
		line = line.strip()
		if len(line) == 0: continue		
		minipar.count_tf(line)
		minipar.count_coocurrances(line)
		if line_no % 100 == 0:
			print "Sentence %s/%s, file: %s    \r"% (line_no, lines, filename)

def parse_single(minipar, dirname, names):
	for name in names:
		filename = os.path.join(dirname, name)
		parse_file(minipar, filename)

if __name__ == '__main__':
	tf_file = '2doc_tf_ana.csv'
	triplets = '2doc_triplets_ana.csv'
	file1 = '/mnt/big/korpusy/bnc_converted/A0/A00'
	file2 = '/mnt/big/korpusy/bnc_converted/A0/A02'
	dirname = '/mnt/big/korpusy/bnc_converted/A0/'
	sentence = "The subprocess module allows you to spawn new processes, connect to their input/output/error pipes, and obtain their return codes."
	sentence2 = 'Since I am also a (very amateur) musician, I also wanted to install one of my favorite software synthesizers, RTSynth, a physical modeling synthesizer which, alas, is free only as in beer because of concerns over patents on physical modeling.'
	minipar = MiniPar()
	
	#os.path.walk(dirname, parse_single, minipar)
	parse_file(minipar, file1)
	parse_file(minipar, file2)
	
	print "Saving TF..."
	file = codecs.open(tf_file, 'w', 'utf')
	for cat, word in sorted(minipar.TF):
		line = u"%s;%s;%s\n" % ( cat,word, minipar.TF[(cat,word)] )
		file.write( line )
	print "Saving dependency triplets"
	file.close()
	file = codecs.open(triplets, 'w', 'utf')	
	for w1,rel,w2 in sorted(minipar.triplets):
		line = u"%s;%s;%s;%s\n" % (w1,rel,w2, minipar.triplets[ (w1,rel,w2) ]  )
		file.write( line )
	print "Done"

	
	
