#from future_builtins import zip
import glob,os,time,zipfile,shutil,json,tempfile
from multiprocessing import Process,Queue
from threading import Lock
 


import collections as _coll
import csv as _csv

import corpus2 as _c2
import desifex as _dsf
import desifex.constants as _co

verbose=False;

class FextorLarge:
	
	def FextorLarge(self):
		self._active_processes = 0
		
	def process(self, inputFile, taskOptions, outputDir):
		start_time = time.time()
		taskOptions['features']+=' tok_count '
		fetcfg =  _FextorbisFeaturesConfig()
		for opt in taskOptions['features'].split():
			getattr(fetcfg, 'opt_' + opt)();
		
		
		
		
		#cerate output dir
		if not os.path.exists(outputDir):
			os.mkdir(outputDir)
		
		worker_number=10;
		queue=Queue();
		workers=[];
		for i in range(worker_number):
			worker=Worker(queue,fetcfg,i);
			worker.start()
			workers.append(worker)
		
		#unzipFiles
		
		temp_folder = tempfile.mkdtemp()
		if not os.path.exists(temp_folder):
			os.mkdir(temp_folder)
		tempf=tempfile.NamedTemporaryFile(delete=False);
		tempf.close();
		
		shutil.copy2(inputFile, tempf.name)	
		with zipfile.ZipFile(tempf.name) as zf:
			zf.extractall(temp_folder);
		print("--- %s seconds ---" % (time.time() - start_time));
		#iterate over all files
		for filename in glob.glob(temp_folder+'/*'):
			name=os.path.basename(filename);
			queue.put((filename,outputDir+'/'+name))
		for i in range(worker_number):
			queue.put(None)
		#waits for finishing
		for worker in workers:
			worker.join()
		
		#remove_tmp_files
		shutil.rmtree(temp_folder)		
		os.remove(tempf.name)
		print("--- %s seconds ---" % (time.time() - start_time));
		
_TAGSET = _c2.get_named_tagset('nkjp')		
		
class Worker(Process):
	def __init__(self,queue,fetcfg,id):
		super(Worker,self).__init__()
		self.queue=queue;
		self.fetcfg=fetcfg;
		self.id=id;
		
	def	run(self):
		if verbose:
			print("Starting worker #"+str(self.id));
		fetset = _dsf.FeatureSet(self.fetcfg.features, _TAGSET)
		docex = _dsf.DocExtractor(fetset, _TAGSET, 'ccl')
		
		for data in iter(self.queue.get,None):
			(input_file,output_file)=data;
			name=os.path.basename(input_file);
			if verbose:
				print("Starting: "+name);
				
			results = docex.extract_from_document(input_file)

			flat_results = {}

			for fet, res in results.iteritems():
				for resnam, resval in res.iteritems():
					flat_results[fet if len(res) == 1 else resnam] = resval
					# flat_results[':'.join((fet, resnam))] = resval

			with open(output_file, 'wb') as csv_ofs:
				self._write_csv(flat_results, csv_ofs)
			if verbose:
				print("Finished: "+name);	
		if verbose:
			print("Worker #"+str(self.id)+" finished...");		
	
	def _write_csv(self,dict_, outfile):
		wr = _csv.writer(outfile, delimiter=';')

		for row in zip(*dict_.iteritems()):
			wr.writerow([unicode(col).encode('utf8') for col in row])


class _FextorbisFeaturesConfig(object):

	def __init__(self):
		self.features = []
		self._args_lex_classes = None
		self._args_proper_names = None

	def opt_base(self):
		self.features.append({
				_co.CFG_FET_NAME: 'base',
				_co.CFG_FET_CLASS: 'lemmacount',
				_co.CFG_FET_SEC_ARGS: dict(
					limit_file='/home/tomekw/fextor/config/nkjp500',
					),
				})

	def opt_orth(self):
		self.features.append({
				_co.CFG_FET_NAME: 'orth',
				_co.CFG_FET_CLASS: 'orthcount',
				_co.CFG_FET_SEC_ARGS: dict(
					limit_file='/home/tomekw/fextor/config/wordlist100.txt',
					),
				})

	def opt_interp_signs(self):
		self.features.append({
				_co.CFG_FET_NAME: 'interp',
				_co.CFG_FET_CLASS: 'interpcount',
				_co.CFG_FET_SEC_ARGS: dict(
					grouping=dict(
						fullstop='.',
						comma=',',
						semicolon=';',
						question_mark='?',
						exclamation_mark='!',
						),
					limit_to_groups=True,
					),
				})

	def opt_verb_count(self):
		self._ensure_lex_classes()
		verbs = frozenset([
				'fin',
				'bedzie',
				'aglt',
				'praet',
				'impt',
				'imps',
				'inf',
				'pcon',
				'pant',
				'ger',
				'pact',
				'ppas',
				'winien',
				])
		self._args_lex_classes['limit_list'] |= verbs
		self._args_lex_classes['grouping']['verb'] = verbs

	def opt_noun_count(self):
		self._ensure_lex_classes()
		nouns = frozenset([
				'subst',
				'depr',
				'ppron12',
				'ppron3',
				])
		self._args_lex_classes['limit_list'] |= nouns
		self._args_lex_classes['grouping']['noun'] = nouns

	def opt_all_adj_count(self):
		self._ensure_lex_classes()
		adjs = frozenset([
				'adj',
				'adja',
				'adjp',
				'adjc',
				])
		self._args_lex_classes['limit_list'] |= adjs
		self._args_lex_classes['grouping']['all_adj'] = adjs

	def opt_subst_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('subst')

	def opt_depr_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('depr')

	def opt_num_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('num')

	def opt_numcol_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('numcol')

	def opt_adj_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('adj')

	def opt_adja_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('adja')

	def opt_adjp_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('adjp')

	def opt_adjc_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('adjc')

	def opt_adv_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('adv')

	def opt_ppron12_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('ppron12')

	def opt_ppron3_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('ppron3')

	def opt_siebie_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('siebie')

	def opt_winien_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('winien')

	def opt_pred_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('pred')

	def opt_prep_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('prep')

	def opt_conj_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('conj')

	def opt_comp_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('comp')

	def opt_interj_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('interj')

	def opt_burk_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('burk')

	def opt_qub_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('qub')

	def opt_brev_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('brev')

	def opt_fin_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('fin')

	def opt_bedzie_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('bedzie')

	def opt_aglt_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('aglt')

	def opt_praet_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('praet')

	def opt_impt_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('impt')

	def opt_imps_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('imps')

	def opt_inf_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('inf')

	def opt_pcon_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('pcon')

	def opt_pant_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('pant')

	def opt_ger_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('ger')

	def opt_pact_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('pact')

	def opt_ppas_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('ppas')

	def opt_interp_count(self):
		self._ensure_lex_classes()
		self._args_lex_classes['limit_list'].add('interp')

	def opt_nam_adj(self):
		self._ensure_proper_names()
		self._args_proper_names['grouping']['adjective_names'] = ['nam_adj']

	def opt_nam_eve(self):
		self._ensure_proper_names()
		self._args_proper_names['grouping']['event_names'] = ['nam_eve']
		
	def opt_nam_fac(self):
		self._ensure_proper_names()
		self._args_proper_names['grouping']['facility_names'] = ['nam_fac']	

	def opt_nam_liv(self):
		self._ensure_proper_names()
		self._args_proper_names['grouping']['living_names'] = ['nam_liv']
	
	def opt_nam_loc(self):
		self._ensure_proper_names()
		self._args_proper_names['grouping']['location_names'] = ['nam_loc']
		
	def opt_nam_num(self):
		self._ensure_proper_names()
		self._args_proper_names['grouping']['number_names'] = ['nam_num']	
	
	def opt_nam_org(self):
		self._ensure_proper_names()
		self._args_proper_names['grouping']['organization_names'] = ['nam_org']	
	
	def opt_nam_oth(self):
		self._ensure_proper_names()
		self._args_proper_names['grouping']['other_names'] = ['nam_oth']
		
	def opt_nam_pro(self):
		self._ensure_proper_names()
		self._args_proper_names['grouping']['product_names'] = ['nam_pro']
	

	def opt_verb12(self):
		self.features.append({
				_co.CFG_FET_NAME: 'verb12',
				_co.CFG_FET_CLASS: 'multilexclasscount',
				_co.CFG_FET_SEC_ARGS: dict(
					lex_class_conj=[
						frozenset([
							'praet',
							'fin',
							'impt',
							'bedzie',
							'pred',
							'aglt',
							]),
						frozenset([
							'pri',
							'sec',
							]),
						],
					),
				})

	def opt_bigrams(self):
		self.features.append({
				_co.CFG_FET_NAME: 'bigrams',
				_co.CFG_FET_CLASS: 'ngramcount',
				_co.CFG_FET_SEC_ARGS: dict(
					n=2,
					),
				})

	def opt_trigrams(self):
		self.features.append({
				_co.CFG_FET_NAME: 'trigrams',
				_co.CFG_FET_CLASS: 'ngramcount',
				_co.CFG_FET_SEC_ARGS: dict(
					n=3,
					),
				})

	def opt_tok_count(self):
		self.features.append({
				_co.CFG_FET_NAME: 'tok_count',
				_co.CFG_FET_CLASS: 'tokencount'
				})

	def _ensure_lex_classes(self):
		if self._args_lex_classes is not None:
			return

		self._args_lex_classes = dict(
				limit_list=set(),
				grouping={},
				also_count_ungrouped=True,
				)
		self.features.append({
				_co.CFG_FET_NAME: 'lex_classes',
				_co.CFG_FET_CLASS: 'lexclasscount',
				_co.CFG_FET_SEC_ARGS: self._args_lex_classes,
				})

	def _ensure_proper_names(self):
		if self._args_proper_names is not None:
			return

		self._args_proper_names = dict(
				grouping={},
				limit_to_groups=True,
				)
		self.features.append({
				_co.CFG_FET_NAME: 'proper_names',
				_co.CFG_FET_CLASS: 'propernamescount',
				_co.CFG_FET_SEC_ARGS: self._args_proper_names,
				})
		
def main():
	fextor=FextorLarge();
	options_fex='{"features":"base orth interp_signs interp_count verb_count noun_count adj_count adv_count prep_count subst_count depr_count num_count numcol_count adj_count adja_count adjp_count adjc_count ppron12_count ppron3_count siebie_count winien_count pred_count conj_count comp_count interj_count burk_count qub_count brev_count fin_count bedzie_count aglt_count praet_count impt_count imps_count inf_count pcon_count pant_count ger_count pact_count ppas_count verb12 bigrams trigrams  "}';
	
	inputFile="test/57.zip"
	outputFile="test/result"
	options_fex=json.loads(options_fex)
	
	start_time = time.time()
	fextor.process(inputFile,options_fex, outputFile);
	print("--- %s seconds ---" % (time.time() - start_time));

if __name__ == '__main__':
    main()

