/*
 * To change this license header, choose License Headers in Project Properties.
 * To change this template file, choose Tools | Templates
 * and open the template in the editor.
 */
package pl.edu.pwr.nlpengine;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.StringReader;
import java.io.StringWriter;
import java.net.URL;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.Semaphore;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
import javax.activation.DataHandler;
import javax.activation.DataSource;
import javax.activation.FileDataSource;
import javax.mail.BodyPart;
import javax.mail.Message;
import javax.mail.MessagingException;
import javax.mail.Multipart;
import javax.mail.PasswordAuthentication;
import javax.mail.Session;
import javax.mail.Transport;
import javax.mail.internet.InternetAddress;
import javax.mail.internet.MimeBodyPart;
import javax.mail.internet.MimeMessage;
import javax.mail.internet.MimeMultipart;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.parsers.ParserConfigurationException;
import javax.xml.transform.OutputKeys;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import org.apache.commons.io.FileUtils;
import org.json.JSONArray;
import org.json.JSONObject;
import org.w3c.dom.Element;
import org.w3c.dom.NodeList;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import pl.edu.pwr.nlptools.NLPTool;
import pl.edu.pwr.nlptools.NLPToolException;

/**
 *
 * @author www_zsk
 */
public class NLPEngine {

    private String TEST_PATH = "/test.xml";
    private org.w3c.dom.Document doc;
    private static int loglevel = 0;
    private NodeList events = null;
    private List<Source> sources = new ArrayList<>();
    ;
    final private List<Output> outputs = new ArrayList<>();
    final private List<Thread> threads = new ArrayList<>();
    private JSONObject resultJSON=new JSONObject();
    private boolean iserror;
    private String errors = "";
    private Date startDate;
    private Date finishedDate;
    
   private int stepsTotal=0;
   private int stepsFinished=1;
    private String sourceIP;

    public NLPEngine(String config,String ip) throws NLPEngineException {
        sourceIP=ip;
        init(config);

    }

    public Date getStartingTime() {
        return startDate;
    }
    
    public String getsourceIP() {
        return sourceIP;
    }

    public int getNumberOfThreads() {
        return threads.size();
    }

    public double processingTime() {
        double time;
        if (finishedDate == null) {
            time = (new Date().getTime() - startDate.getTime()) / 1000.0;
        } else {
            time = (finishedDate.getTime() - startDate.getTime()) / 1000.0;
        }
        return time;
    }

    public NLPEngine() throws NLPEngineException, IOException {
        URL res = this.getClass().getResource(TEST_PATH);
        if (res == null) {
            throw new NLPEngineException("No input file");
        }
        String path = res.getPath();
        byte[] bytes;

        bytes = Files.readAllBytes(Paths.get(path));
        init(new String(bytes, "UTF-8"));

    }

    final public void init(String xml) throws NLPEngineException {
        
        try {

            DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
            DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
            InputSource is = new InputSource();
            
            is.setCharacterStream(new StringReader(xml));
            doc = dBuilder.parse(is);

            //Deal with source attribute
            NodeList elems = doc.getDocumentElement().getChildNodes();
            //System.out.println(elems.getLength());
            for (int i = 0; i < elems.getLength(); i++) {
                if (elems.item(i) instanceof Element) {
                    Element el = (Element) elems.item(i);
                    if (el.getAttribute("source") != null) {
                        Element input = doc.createElement("input");
                        el.appendChild(input);
                        input.setAttribute("source", el.getAttribute("source"));
                        input.setAttribute("id", "1");
                    }
                }//
            }
            NodeList sourceNodes = doc.getElementsByTagName("source");
            if (sourceNodes.getLength() == 0) {
                throw new NLPEngineException("No <source> block in lpmn file");
            }
            for (int i = 0; i < sourceNodes.getLength(); i++) {
                Activity act = getDestination(((Element) sourceNodes.item(i)).getAttribute("id"));
                Source s = new Source((Element) sourceNodes.item(i), act);
                this.sources.add(s);

            }

            events = doc.getElementsByTagName("event");

        } catch (ParserConfigurationException ex) {

            Logger.getLogger(NLPEngine.class.getName()).log(Level.SEVERE, "Problems with parser", ex);
            throw new NLPEngineException("Internal error");
        } catch (SAXException ex) {
            System.out.println("File: "+xml);
            Logger.getLogger(NLPEngine.class.getName()).log(Level.WARNING,  ex.getMessage(),xml);
            throw new NLPEngineException("Input XML file error: " + ex.getMessage());
        } catch (IOException ex) {

            throw new NLPEngineException("Input XML file read error: " + ex.getMessage());
        }
    }

    public void start() throws NLPEngineException {
        startDate = new Date();
        Thread t = new Thread() {
                            public void run() {
                                        for (Source s : sources) { s.start();}
                            }
                        };
        t.start();
        addThread(t);
            


    }

    public String getXMLStatus() throws NLPEngineException {
        TransformerFactory tf = TransformerFactory.newInstance();
        try {
            javax.xml.transform.Transformer transformer = tf.newTransformer();
            transformer.setOutputProperty(OutputKeys.INDENT, "yes");
            //transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes");
            StringWriter writer = new StringWriter();
            synchronized (doc) {
                transformer.transform(new DOMSource(doc), new StreamResult(writer));
                return writer.getBuffer().toString();
            }
        } catch (TransformerException ex) {
            Logger.getLogger(NLPEngine.class.getName()).log(Level.SEVERE, null, ex);
            throw new NLPEngineException("Internal error");
        }
    }

    private List<Activity> getNextforGateway(Element actel) throws NLPEngineException {
        List<Activity> nexts = new ArrayList<>();
        NodeList outputs = actel.getElementsByTagName("output");
        for (int j = 0; j < outputs.getLength(); j++) {
            Element output = (Element) outputs.item(j);
            //if (loglevel>3) System.out.println("Gateway with output id="+output.getAttribute("id")+".");
            nexts.add(getDestination(output.getAttribute("id")));

        }
        return nexts;
    }

    private Activity getDestination(String id) throws NLPEngineException {
        Activity act = null;

        NodeList inputs = doc.getElementsByTagName("input");
        boolean found = false;
        for (int i = 0; i < inputs.getLength(); i++) {
            Element input = (Element) (inputs.item(i));
            if (input.getAttribute("source").equals(id)) {
                if (found) {
                    throw new NLPEngineException("At least two inputs with input id='" + id + "'");
                }
                found = true;
                Element actel = (Element) input.getParentNode();

                if (actel.getNodeName().equals("gateway")) {
                    List<Activity> nexts = getNextforGateway(actel);
                    act = new Gateway(actel, nexts);
                } else if (actel.getNodeName().equals("output")) {
                    act = new Output(actel);
                } else if (actel.getNodeName().equals("agregate")) {
                    Activity next = getDestination(actel.getAttribute("id"));
                    act = new Agregate(actel,next);
                }
                
                
                else {
                    Activity next = getDestination(actel.getAttribute("id"));
                    act = new Activity(actel, next);
                }
            }

        }
        if (!found) {
            throw new NLPEngineException("Warning. No block with <input source='" + id + "'> !");
        }

        return act;
    }

    public JSONObject getResult() {
        return resultJSON;
    }
    synchronized private void processingFinished() {
        finishedDate = new Date();

        if (events.getLength() == 0) {
            return;
        }
        Element event = (Element) events.item(0);
        String email = event.getAttribute("address");
        String text = event.getTextContent();
        try {
            Thread.sleep(200);
        } catch (InterruptedException ex) {
            Logger.getLogger(NLPEngine.class.getName()).log(Level.SEVERE, null, ex);
        }

    }

    public void killprocessing() {
        synchronized (threads) {
            for (Thread thread : threads) {
                thread.interrupt();
            }
        }

        finishedDate = new Date();

    }

    public void exception(Exception ex) {
        if (!iserror) {
            errors = errors + ex.getMessage();
            iserror = true;
            finishedDate = new Date();
            killprocessing();

        }
    }

    
    public boolean isError() {
        return iserror;
    }

    public String getError() {
        return errors;
    }

    private void removeThread(Thread t) {
        synchronized (threads) {
            this.threads.remove(t);
        }
    }

    private void addThread(Thread t) {
        synchronized (threads) {
            this.threads.add(t);
        }
    }

    public boolean isFinished() {
        if (iserror) {
            return true;
        }
        synchronized (outputs) {
            return outputs.isEmpty();
        }
    }

    public static void main(String[] args) throws NLPEngineException, InterruptedException, IOException {
        if (loglevel > 1) {
            System.out.println("Started");
        }
        NLPEngine engine = new NLPEngine();
        engine.start();

        Thread.sleep(4000);
        //engine.killprocessing();
        while (!engine.isFinished()) {
            Thread.sleep(1000);
            //System.out.println(engine.getXMLStatus());
        }
        if (engine.isError()) {
            System.out.println(engine.getError());
        }
        System.out.println("Finished");

    }

    public double getProgress() {
        double result=0;
        
                
        if (this.stepsTotal>0)
        synchronized (doc)
        {
            result=(double)this.stepsFinished/this.stepsTotal;
        }
        if (result>1) result=1.0;
        if (result<0) result=0;
                
        return result;
    }

    

    private class FileData {

        public String name;
        public String fileID;

        private FileData(String fileID, String file) {
            this.name = file;
            this.fileID = fileID;
        }

    }

    private class Output extends Activity {

        String resultPath;
        Element dspace;
        private List<String> plans;
        private List<FileData> finished;
        private Element mail;
        private JSONArray resultJSONArray=new JSONArray();

        public Output(Element el) throws NLPEngineException {
            super(el);
            if (loglevel > 3) {
                System.out.println("Output #" + id + " started !");
            }
            resultJSON.put(id, resultJSONArray);
            
            tool = new NLPTool();
            NodeList dess = el.getElementsByTagName("destination");

            if (dess.getLength() > 0) {
                resultPath = dess.item(0).getTextContent();
            }

            NodeList dspaces = el.getElementsByTagName("dspace");
            if (dspaces.getLength() > 0) {
                dspace = (Element) dspaces.item(0);
            }

            NodeList mails = el.getElementsByTagName("mail");
            if (mails.getLength() > 0) {
                mail = (Element) mails.item(0);
            }

            plans = new ArrayList<>();
            finished = new ArrayList<>();
            outputs.add(this);
            if (loglevel > 3) {
                System.out.println("Output #" + id + " created !");
            }
        }

        @Override
        public void plans(List<String> files) {
            for (String file : files) {
                plans.add(file);
            }
            if (loglevel > 5) {
                System.out.print("Output #" + id + " has plans for: ");
                for (String file : files) {
                    System.out.print(" " + file);
                }
                System.out.println();
            }
        }

        @Override
        public void process(String fileID, String file) throws NLPEngineException {
            if (resultPath != null) {
                process_dest(fileID, file);
            } else {
                process_new(fileID, file);
            }

        }

        public void process_new(String fileID, String file) throws NLPEngineException {
            Date startProcessDate = new Date();
            if (loglevel > 3) {
                System.out.println("Output #" + id + " processing " + file);
            }
            try {
                if (dspace != null) {
                    if (dspace.getAttribute("type").equals("nfs"))
                        tool.upload2dspacenfs(fileID, dspace.getAttribute("repo_id"), dspace.getAttribute("item_id"),file);
                    else        
                    tool.upload2dspace(fileID, dspace.getAttribute("repo_id"), dspace.getAttribute("item_id"), file, dspace.getTextContent());
                }
            } catch (NLPToolException ex) {
                Logger.getLogger(NLPEngine.class.getName()).log(Level.SEVERE, null, ex);
                throw new NLPEngineException(ex.getMessage());
            }

            synchronized (plans) {
                plans.remove(file);
                finished.add(new FileData(fileID, file));
            }
            synchronized (doc) {
                Element pro = doc.createElement("finished");
                element.appendChild(pro);
                pro.setAttribute("name", file);
                pro.setAttribute("fileID", tool.encrypt(fileID));
                JSONObject result=new JSONObject();
                resultJSONArray.put(result);
                result.put("name",file);
                result.put("fileID",tool.encrypt(fileID));
                
                pro.setAttribute("proceesingTime", "" + (new Date().getTime() - startProcessDate.getTime()) / 1000.0);

            }

            if (plans.isEmpty()) {
                if (loglevel > 1) {
                    System.out.println("Output #" + id + " finished processing of all files !");
                }

                if (mail != null) {
                    sendmail(mail.getAttribute("to"), mail.getTextContent(), mail.getAttribute("subject"), finished);
                     if (loglevel > 3) {
                    System.out.println("Output #" + id + " mail sent !");
                     }
                }

                synchronized (outputs) {
                    outputs.remove(this);
                    if (outputs.isEmpty()) {
                        Thread t = new Thread() {
                            @Override
                            public void run() {
                                processingFinished();
                            }
                        };
                        t.start();
                    }
                }
            }
        }

        public void process_dest(String fileID, String file) throws NLPEngineException {
            String filename = resultPath + "/" + file;
            if (Files.notExists(Paths.get(resultPath))) {
                new File(resultPath).mkdirs();
                if (loglevel > 4) {
                    System.out.println("Creating dir" + resultPath);
                }
            }
            try {
                Files.copy(Paths.get(fileID), Paths.get(filename), REPLACE_EXISTING);
            } catch (IOException ex) {
                throw new NLPEngineException("Output #" + id + " file error: " + ex.getMessage());
            }
            if (loglevel > 3) {
                System.out.println("Output #" + id + " finished copying file " + filename);
            }

            plans.remove(file);
            synchronized (doc) {
                Element pro = doc.createElement("finished");
                element.appendChild(pro);
                pro.setAttribute("name", file);
                pro.setAttribute("path", filename);

            }

            if (plans.isEmpty()) {
                if (loglevel > 1) {
                    System.out.println("Output #" + id + " finished processing of all files !");
                }
                synchronized (outputs) {
                    outputs.remove(this);
                    if (outputs.isEmpty()) {
                        Thread t = new Thread() {
                            public void run() {
                                processingFinished();
                            }
                        };
                        t.start();
                    }
                }
            }
        }
    }

    private class Agregate extends Activity {

        String resultPath;
        Element dspace;
        private List<String> plans;
        private List<FileData> finished;
        private Element mail;
        private String fileName;

        public Agregate(Element el, Activity next) throws NLPEngineException {
            super(el, next);
            if (loglevel > 3) {
                System.out.println("Agregate #" + id + " started !");
            }
            toolName = el.getAttribute("type");
            tool = new NLPTool();

            plans = new ArrayList<>();
            finished = new ArrayList<>();
            fileName = element.getAttribute("name");
            if (fileName.isEmpty()) {
                fileName = "input";
            }
            

        }

        @Override
        public void plans(List<String> files) {
            for (String file : files) {
                plans.add(file);
            }
            List<String> nexts = new ArrayList();
            
            nexts.add(fileName);
            super.plans(nexts);

        }

        @Override
        public void process(String fileID, String file) throws NLPEngineException {
             String resultfileID=null;
            synchronized (plans) {
                plans.remove(file);
                finished.add(new FileData(fileID, file));
            }
            Element processing;
            synchronized (doc) {
                processing = doc.createElement("processing");
                element.appendChild(processing);
                processing.setAttribute("name", file);
         
            }

            if (plans.isEmpty()) {
                Date startProcessDate = new Date();

                
                    String fileIDs[] = new String[finished.size()];
                    String fileNames[] = new String[finished.size()];
                    int i = 0;
                    for (FileData el : finished) {
                        fileIDs[i] = el.fileID;
                        fileNames[i] = el.name;
                        i++;
                    }
                    try {  if (toolName.equals("zip")) 
                            resultfileID=tool.zipFiles(fileIDs, fileNames);
                    else resultfileID=tool.dirFiles(fileIDs, fileNames);
                    } catch (NLPToolException ex) {
                        Logger.getLogger(NLPEngine.class.getName()).log(Level.SEVERE, "Problems with agregation", ex);
                        throw new NLPEngineException(ex.getMessage());
                    }
                
                
                    
                

                synchronized (doc) {
                    Element pro = doc.createElement("finished");
                    element.appendChild(pro);
                    pro.setAttribute("name", fileName);
                    pro.setAttribute("proceesingTime", "" + (new Date().getTime() - startProcessDate.getTime()) / 1000.0);

                }
                if (loglevel > 1) {
                    System.out.println("Agregate #" + id + " finished processing of all files !");
                }
                 if (resultfileID != null & nextActivity != null) {
                      nextActivity.process(resultfileID,fileName);
                     }
            }
        }

    }

    private class Activity {

        protected Activity nextActivity = null;
        protected NLPTool tool;
        protected String toolName;
        protected String id;
        protected Element element;
        protected String options;

        protected Activity(Element el) {
            element = el;
            id = el.getAttribute("id");
            if (loglevel > 3) {
                System.out.println(el.getNodeName() + " created.");
            }
        }

        public Activity(Element el, Activity next) {
            this(el);

            this.nextActivity = next;

            toolName = el.getAttribute("name");

            tool = new NLPTool(toolName);
            options=el.getAttribute("options");
        }

        public void plans(List<String> files) {
            nextActivity.plans(files);
            synchronized (doc)
             { stepsTotal+=files.size();
                        
              }
        }

        public void process(String fileID, String file) throws NLPEngineException {
            Date startProcessDate = new Date();
            Element processing;
            synchronized (doc) {
                processing = doc.createElement("processing");
                element.appendChild(processing);
                processing.setAttribute("name", file);

            }
            
            if (loglevel > 2) {
                System.out.println("Activity #" + id + " " + toolName + " started processsing file " + file + " " + fileID);
            }
            try {
                if (options.isEmpty())
                     fileID = tool.processFile(fileID);
                else
                    fileID = tool.processFile(fileID,options);
            } catch (NLPToolException ex) {
                synchronized (doc) {
                   Element pro = doc.createElement("error"); element.appendChild(pro);
                   pro.setAttribute("name", file);
                   pro.setAttribute("value", ex.getMessage());
                }
                throw new NLPEngineException("Activity #" + id + " error: " + ex.getMessage());
            }
            if (loglevel > 2) {
                System.out.println("Activity #" + id + " " + toolName + " finished processsing with new file " + file + " " + fileID);
            }

            synchronized (doc) {
                element.removeChild(processing);
                Element pro = doc.createElement("finished");
                element.appendChild(pro);
                pro.setAttribute("name", file);
                pro.setAttribute("fileID", tool.encrypt(fileID));
                pro.setAttribute("proceesingTime", "" + (new Date().getTime() - startProcessDate.getTime()) / 1000.0);
                stepsFinished++;
            }

            if (fileID != null & nextActivity != null) {
                nextActivity.process(fileID, file);
            }

        }
    }

    private class Gateway extends Activity {

        List<Activity> nextActivities;

        public Gateway(Element el, List<Activity> nexts) {
            super(el);

            toolName = "";
            tool = new NLPTool();
            nextActivities = nexts;
        }

        @Override
        public void plans(List<String> files) {
            for (Activity activity : nextActivities) {
                activity.plans(files);
            }
        }

        @Override
        public void process(String fileID, String file) throws NLPEngineException {
            //System.out.println("Not yet implemented");

            String[] fileIDs = new String[nextActivities.size()];

            for (int i = nextActivities.size() - 1; i > 0; i--) {
                try {
                    try {
                        Thread.sleep(100);
                    } catch (InterruptedException ex) {
                        new NLPEngineException("Internal error: sleep");
                    }
                    fileIDs[i] = tool.booking();
                    //Logger.getLogger(NLPEngine.class.getName()).log(Level.SEVERE,"Booking "+fileIDs[i]);
                } catch (NLPToolException ex) {
                    new NLPEngineException("Gateway error: " + ex.getMessage());
                }
                try {
                    File in=new File(fileID);
                    File out=new File(fileIDs[i]);
                    if (in.isDirectory())
                         FileUtils.copyDirectory(in,out);
                    else
                         FileUtils.copyFile(in, out);
                    
                    //Files.copy(Paths.get(fileID), Paths.get(fileIDs[i]), REPLACE_EXISTING);
                } catch (IOException ex) {
                    Logger.getLogger(NLPEngine.class.getName()).log(Level.SEVERE, null, ex);
                    new NLPEngineException("Internal error");
                }

            }
            fileIDs[0] = fileID;
            final String newfile = file;
            for (int i = nextActivities.size() - 1; i >= 0; i--) {
                if (i >= 1) {
                    final String newfileID = fileIDs[i];

                    final Activity next = nextActivities.get(i);

                    Thread t = new Thread() {
                        @Override
                        public void run() {
                            try {
                                next.process(newfileID, newfile);

                            } catch (NLPEngineException ex) {
                                exception(ex);
                            }
                            removeThread(this);
                        }
                    };

                    t.start();
                    addThread(t);
                } else {
                    nextActivities.get(i).process(fileIDs[0], file);
                }
            }

        }
    }

    
    
    
    
    private class Source {

        private final String path;
        ArrayList<FileSource> fileSources;
        
        private final Activity activity;
        private final String id;
        protected Element element;

        public Source(Element el, Activity act) throws NLPEngineException  {
            id = el.getAttribute("id");
            element=el;
            if (loglevel > 4) {
                System.out.println("Source #" + id + " started !");
            }
            if (el.getElementsByTagName("path").getLength() > 0) {
                path = el.getElementsByTagName("path").item(0).getTextContent();
                if (loglevel > 5) {
                    System.out.println("Source #" + id + " has path = " + path);
                }
                fileSources = null;
            } else {
                path = null;
                fileSources = new ArrayList<>();
                
                addfileSource(el,fileSources,"url",SourceType.URL);
                addfileSource(el,fileSources,"file",SourceType.FILEID);
                addfileSource(el,fileSources,"urlzip",SourceType.URLZIP);
                addfileSource(el,fileSources,"dSpace",SourceType.dSpace);
                addfileSource(el,fileSources,"dSpacezip",SourceType.dSpaceZIP);
                
               
                
                
            }
            if (fileSources.size()<1) throw new NLPEngineException("No source item in Source #"+id);            
            activity = act;

            if (loglevel > 3) {
                System.out.println("Source #" + id + " created !");
            }

        }

        public void start()  {
            if (path != null) {
                try {
                    startPath();
                } catch (NLPEngineException ex) {
                    exception(ex);
                }
            } else {
                startNew();
                
            }
            removeThread(Thread.currentThread());
        }

        public void startNew() {
            Date startProcessDate = new Date();
            try { dealwithURLZIPs();
            } catch (NLPEngineException ex) {
                exception(ex);
            }

            List<String> names = new ArrayList<>();
            for (int i = 0; i < fileSources.size(); i++) {
             if (fileSources.get(i).getType()!=SourceType.dSpaceZIP)
                names.add(fileSources.get(i).getName());
            }
            addNamesBasedOnDSPaceZIP(names);
            activity.plans(names);
            final Semaphore semStart=new Semaphore(5);
            final Semaphore semProcess=new Semaphore(30);
            for (int i = 0; i < fileSources.size(); i++) {
                if (fileSources.get(i).getType()==SourceType.dSpaceZIP) break;
                final int id = i;
                if (loglevel > 3)   System.out.println("Start processing of file " + names.get(i));
                try {
                    semProcess.acquire();
                    semStart.acquire();
                } catch (InterruptedException ex) {
                    Logger.getLogger(NLPEngine.class.getName()).log(Level.SEVERE, null, ex);
                }
                Thread t;
                t = new Thread() {
                    @Override
                    public void run() {
                        try {
                            String fileID = "aa";
                            Date startProcessDate = new Date();
                            NLPTool tool = new NLPTool();
                            FileSource fs = fileSources.get(id);
                            if (fs.getType() == SourceType.URL)  
                            { fileID = tool.saveURL(fs.getUrl());
                            synchronized (doc) {
                                Element pro = doc.createElement("finished");
                                element.appendChild(pro);
                                pro.setAttribute("name", fs.getName());
                                pro.setAttribute("name_url", fs.getUrl());
                                pro.setAttribute("proceesingTime", "" + (new Date().getTime() - startProcessDate.getTime()) / 1000.0);
                            }
                            }
                            else
                                if (fs.getType() == SourceType.dSpace)
                                {  String dSpacefile = tool.getDspaceZipPath(fs.getUrl());
                                   fileID=tool.booking();
                                   try {
                                    FileUtils.copyFile(new File(dSpacefile), new File(fileID));
                                    } catch (IOException ex) {
                                       Logger.getLogger(NLPEngine.class.getName()).log(Level.WARNING, null, ex);
                                       throw new NLPEngineException("The repository "+fs.getUrl()+" is not processed to CCL.");
                                       
                                     }
                                synchronized (doc) {
                                    Element pro = doc.createElement("finished");
                                    element.appendChild(pro);
                                    pro.setAttribute("name", fs.getName());
                                    pro.setAttribute("name_url", fs.getUrl());
                                    pro.setAttribute("proceesingTime", "" + (new Date().getTime() - startProcessDate.getTime()) / 1000.0);
                                }
                                }    
                                
                                else     fileID = tool.decrypt(fs.getUrl());
                            
                            semStart.release();

                            activity.process(fileID, fs.getName());
                            semProcess.release();
                        } catch (NLPEngineException | NLPToolException ex) {
                            exception(ex);
                        } 
                        removeThread(this);
                    }
                };

                t.start();
                addThread(t);
                

            }
            
            dealwithDSpaceZIP();
            synchronized (doc) {
                Element pro = doc.createElement("finishesource");
                element.appendChild(pro);
                pro.setAttribute("proceesingTime", "" + (new Date().getTime() - startProcessDate.getTime()) / 1000.0);
            }
        }
        
        private void addNamesBasedOnDSPaceZIP(List<String> names)
        { for (int i = 0; i < fileSources.size(); i++) {
              FileSource fs = fileSources.get(i);
              if (fs.getType()==SourceType.dSpaceZIP)
              {  NodeList el_items = fs.getElement().getElementsByTagName("ccl");
                        for (int j = 0; j < el_items.getLength(); j++) {
                           Element elem=(Element) el_items.item(j);
                           names.add(elem.getAttribute("name"));          
                        }
                }
            
            }
        }
        
        private void dealwithDSpaceZIP() 
        {   boolean found=false;
            for (int i = 0; i < fileSources.size(); i++) {
              FileSource fs = fileSources.get(i);
              if (fs.getType()==SourceType.dSpaceZIP) 
                try { 
                       //fileSources.remove(i);
                        found = true;
                        fileSources.remove(i);
                        String corpusID=fs.getElement().getAttribute("corpusid");
                        
                        NodeList el_items = fs.getElement().getElementsByTagName("ccl");
                        String cclNames[]=new String[el_items.getLength()];
                        String names[]=new String[el_items.getLength()];
                        
                        for (int j = 0; j < el_items.getLength(); j++) {
                           Element elem=(Element) el_items.item(j);
                           cclNames[j]=elem.getTextContent();
                           names[j]=elem.getAttribute("name");
                        }
                        
                        getCCLFromDspace(corpusID,cclNames,names);
                        
                    } catch (NLPToolException ex) {
                        exception(ex);
                        
                    }  
            }
             if (found) {
                    dealwithDSpaceZIP();
                }
        }
        
        //Czyta pliki z zip-a z  dspace_nfs i zapisuje go na nfs-ie (booking)
    public void getCCLFromDspace(String corpusid,String cclNames[],String names[]) throws NLPToolException
    {   Date startProcessDate1 = new Date();
        double sum=0;
         NLPTool tool=new NLPTool();
        
         String resultPath = tool.dspaceNSF + corpusid+"wlw_ccl.zip";
        if (Files.notExists(Paths.get(resultPath)))
          {  Logger.getLogger(NLPTool.class.getName()).log(Level.WARNING, "No "+resultPath+" file", "");
           throw new  NLPToolException("Corpus "+corpusid+ " is not processed in DSpace");
          }
          Semaphore sem=new Semaphore(30);
            try {  
                Charset charset = Charset.forName("CP437");
                ZipInputStream zipIn = new ZipInputStream(new FileInputStream(resultPath), charset);
                ZipEntry entry = zipIn.getNextEntry();
                String fileID_m = tool.booking();
                while (entry != null) 
                {   
                    String name=entry.getName();
                    for (int i=0;i<cclNames.length;i++)
                    {
                    if (name.equals(cclNames[i])) {
                     String fileID=fileID_m+i;
                     Date startProcessDate = new Date();
                     tool.extractFile(zipIn, tool.getNFSPath()+fileID);
                     sum+=(new Date().getTime() - startProcessDate.getTime()) / 1000.0;
                     
                     Thread t=new ProcessingThread(fileID,names[i],sem);
                     sem.acquire();
                     t.start();
                     addThread(t);
                     break;
                    }
                    }
                    zipIn.closeEntry();
                    entry = zipIn.getNextEntry();
                    
                }
                
                zipIn.close();
                System.out.println("Sum time"+(new Date().getTime() - startProcessDate1.getTime()) / 1000.0+ "  decrypt "+sum);
            } catch (Exception e) {
                Logger.getLogger(NLPTool.class.getName()).log(Level.SEVERE, "Problems in unziping:" + resultPath, e);
                throw new  NLPToolException("Problems in unziping corpus: "+corpusid+ " from DSpace");
            }
              
          
            
        
        
        
    }
    

        public void startPath() throws NLPEngineException {
            NLPTool tool = new NLPTool();
            List<File> files = new ArrayList<>();
            List<String> file_names = new ArrayList<>();

            File folder = new File(path);
            if (folder == null) {
                throw new NLPEngineException("The path " + path + " is wrong!");
            }
            if (folder.listFiles() == null) {
                throw new NLPEngineException("The path " + path + " is wrong!");
            }

            for (File file : folder.listFiles()) {
                if (file.isFile()) {
                    files.add(file);
                }
                file_names.add(file.getName());
            }

            activity.plans(file_names);
            for (int i = 0; i < files.size(); i++) {
                File file = files.get(i);

                try {
                    if (loglevel > 3) {
                        System.out.println("Start processing of file " + file.getName());
                    }

                    try {
                        Thread.sleep(200);
                    } catch (InterruptedException ex) {
                        new NLPEngineException("Internal error: sleep");
                    }
                    final String fileID = tool.booking();
                    // Logger.getLogger(NLPEngine.class.getName()).log(Level.SEVERE,"Booking "+fileID);
                    final String filename = file_names.get(i);
                    Files.copy(Paths.get(file.getPath()), Paths.get(fileID), REPLACE_EXISTING);
                    Thread t = new Thread() {
                        @Override
                        public void run() {
                            try {
                                activity.process(fileID, filename);

                            } catch (NLPEngineException ex) {
                                exception(ex);
                            }
                            removeThread(this);
                        }
                    };

                    t.start();
                    addThread(t);
                } catch (IOException ex) {
                    throw new NLPEngineException("Source #" + id + " error: " + ex.getMessage());
                } catch (NLPToolException ex) {
                    throw new NLPEngineException("Engine internal error");
                }
            }
        }

        private void addfileSource(Element el, ArrayList<FileSource> fileSources, String tagname, SourceType sourceType) {

            NodeList el_items = el.getElementsByTagName(tagname);
            FileSource fileSource;
            for (int i = 0; i < el_items.getLength(); i++) {
                if (sourceType == SourceType.dSpaceZIP) {
                    Element elem = (Element) el_items.item(i);
                    fileSource = new FileSource(elem, sourceType);
                } else {
                    fileSource = new FileSource(
                            el_items.item(i).getTextContent(),
                            ((Element) el_items.item(i)).getAttribute("name"),
                            sourceType);
                }
                fileSources.add(fileSource);


                }
        }

        private void dealwithURLZIPs() throws NLPEngineException {
            boolean found = false;
            for (int i = 0; i < fileSources.size(); i++) {
                FileSource fs = fileSources.get(i);
                if (fs.getType() == SourceType.URLZIP) {
                    try {
                        found = true;
                        fileSources.remove(i);
                        NLPTool tool = new NLPTool();
                        String fileID = tool.saveURL(fs.getUrl());
                        List<String> fileIDs = new ArrayList<>();
                        List<String> fileNames = new ArrayList<>();
                        tool.unzipFile(fileID, fileIDs, fileNames);
                        for (int j = 0; j < fileNames.size(); j++) {
                            fileSources.add(
                                    new FileSource(tool.encrypt(fileIDs.get(j)), fileNames.get(j), SourceType.FILEID));

                        }

                        break;
                    } catch (NLPToolException ex) {
                        throw new NLPEngineException(ex.getMessage());

                    }
                }

               

            }
             if (found) {
                    dealwithURLZIPs();
                }
        }

        private class ProcessingThread extends Thread {

            String name;
            String fileID="";
            Semaphore sem;

        

            public ProcessingThread(String fileID,String name,Semaphore sem) {
                this.name = name;
                this.fileID=fileID;
                this.sem=sem;
                
            }
            
            

            @Override
            public void run() {
                try {
                    activity.process(fileID, name);
                    sem.release();
                } catch (NLPEngineException ex) {
                    exception(ex);
                }
                removeThread(this);
            }
        }

    }

    private static void sendmail(String to, String text, String subject, List<FileData> files) {
          
        // Sender's email ID needs to be mentioned
        final String from = "webserwisy@clarin-pl.eu";

        final String password = "123qazxsw";

        // Assuming you are sending email from localhost
        String host = "mp2.pl";

        // Get system properties
        Properties properties = System.getProperties();

        // Setup mail server
        properties.setProperty("mail.smtp.host", host);
      //properties.setProperty("mail.user", "webserwisy");
        //properties.setProperty("mail.password", "webserwisy0101");

        properties.put("mail.smtp.auth", "true");
        properties.put("mail.smtp.starttls.enable", "true");
        properties.put("mail.smtp.port", "587");

      // Get the default Session object.
        //Session session = Session.getDefaultInstance(properties);
        Session session = Session.getInstance(properties,
                new javax.mail.Authenticator() {
                    protected PasswordAuthentication getPasswordAuthentication() {
                        return new PasswordAuthentication(from, password);
                    }
                });

        try {
            // Create a default MimeMessage object.
            MimeMessage message = new MimeMessage(session);

            // Set From: header field of the header.
            message.setFrom(new InternetAddress(from));

            // Set To: header field of the header.
            message.addRecipient(Message.RecipientType.TO,
                    new InternetAddress(to));

            // Set Subject: header field
            message.setSubject(subject);

            // Create the message part 
            BodyPart messageBodyPart = new MimeBodyPart();

            // Fill the message
            messageBodyPart.setText(text);

            // Create a multipar message
            Multipart multipart = new MimeMultipart();

            // Set text message part
            multipart.addBodyPart(messageBodyPart);

            for (FileData file : files) {
                // Part two is attachment
                messageBodyPart = new MimeBodyPart();

                DataSource source = new FileDataSource(file.fileID);
                messageBodyPart.setDataHandler(new DataHandler(source));
                messageBodyPart.setFileName(file.name);
                multipart.addBodyPart(messageBodyPart);
            }
            // Send the complete message parts
            message.setContent(multipart);

            // Send message
            Transport.send(message);
            System.out.println("Sent message successfully....");
        } catch (MessagingException mex) {
            mex.printStackTrace();
        }
    }
    
    
       


}
    
    
 