'''
Created on Feb 7, 2011

@author: boban
'''
import os,codecs,sys, shutil

def load_inlist(filename):
	one_list = []
	word = None
	kbest={}
	for line in codecs.open(filename, 'r', 'utf-8'):
		if not line.startswith('\t'):
			if word:
				kbest[word] = one_list
			one_list = []
			word = line.strip() 
		else:
			line = line.strip()
			w2, v = line.split(';')
			val = float(v)
			one_list.append((w2,val))
	return kbest
		
def sortl(l):
	lll = []
	
	for w in l:
		for (w2,v) in l[w]:
			lll.append( (v,w,w2) )
			
	return list(reversed( sorted(lll)) )

def remdup(l):
	for v,d1,d2 in l:
		print 'removing %s ( dup: %s sim:%.2f)' % (d1,d2,v)
		shutil.rmtree(os.path.dirname(d1)) 

def main():
	inlist = load_inlist('20best.txt')
	slist = sortl(inlist)
	clist = [x for x in slist if x[0] > 0.85]
	remdup(clist)
	
if __name__ == '__main__':
	main()
	