WIP

Mateusz Kopeć
1 parent cb490cab
Showing 44 changed files with 529 additions and 351 deletions
.gitignore
nicolas-cli/README.md
nicolas-cli/src/main/java/pl/waw/ipipan/zil/summ/nicolas/cli/Main.java
nicolas-common/pom.xml
nicolas-common/src/main/java/pl/waw/ipipan/zil/summ/nicolas/common/ThriftUtils.java
nicolas-common/src/main/java/pl/waw/ipipan/zil/summ/nicolas/common/Utils.java
nicolas-common/src/test/java/pl/waw/ipipan/zil/summ/nicolas/common/UtilsTest.java
nicolas-eval/pom.xml
nicolas-eval/src/main/java/pl/waw/ipipan/zil/summ/nicolas/eval/Constants.java
nicolas-eval/src/main/java/pl/waw/ipipan/zil/summ/nicolas/eval/Evaluate.java
nicolas-eval/src/main/java/pl/waw/ipipan/zil/summ/nicolas/eval/ExtractGoldSummaries.java
nicolas-eval/src/main/java/pl/waw/ipipan/zil/summ/nicolas/eval/SummarizeTestCorpus.java
nicolas-train/src/main/java/pl/waw/ipipan/zil/summ/nicolas/train/search/CrossvalidateCommon.java → nicolas-eval/src/main/java/pl/waw/ipipan/zil/summ/nicolas/eval/search/Crossvalidate.java
nicolas-train/src/main/resources/pl/waw/ipipan/zil/summ/nicolas/train/zero/test_ids.txt → nicolas-eval/src/main/resources/pl/waw/ipipan/zil/summ/nicolas/eval/test_text_ids.txt
nicolas-lib/pom.xml
nicolas-lib/src/main/java/pl/waw/ipipan/zil/summ/nicolas/ThriftUtils.java → nicolas-lib/src/main/java/pl/waw/ipipan/zil/summ/nicolas/InstanceUtils.java
nicolas-lib/src/main/java/pl/waw/ipipan/zil/summ/nicolas/Nicolas.java
nicolas-lib/src/main/java/pl/waw/ipipan/zil/summ/nicolas/NicolasException.java
nicolas-lib/src/main/java/pl/waw/ipipan/zil/summ/nicolas/apply/ApplyModel.java
nicolas-common/src/main/java/pl/waw/ipipan/zil/summ/nicolas/common/features/FeatureExtractor.java → nicolas-lib/src/main/java/pl/waw/ipipan/zil/summ/nicolas/features/FeatureExtractor.java
@@ -16,3 +16,5 @@ hs_err_pid*
 .idea
 *.iml
+
+/data
 \ No newline at end of file
@@ -3,6 +3,8 @@
 This module contains a sample command-line application, which uses Nicolas library to summarize chosen input text file.
 Summary is written to target output file. Additionally, user needs to specify desired number of tokens in the summary.
+Be aware that summarizer requires internet access and working Multiservice (multiservice.nlp.ipipan.waw.pl).
+
 ## Installation
     mvn clean install
@@ -3,10 +3,9 @@ package pl.waw.ipipan.zil.summ.nicolas.cli;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import pl.waw.ipipan.zil.summ.nicolas.Nicolas;
+import pl.waw.ipipan.zil.summ.nicolas.NicolasException;
 import pl.waw.ipipan.zil.summ.nicolas.multiservice.Preprocessor;
-import java.io.IOException;
-
 public class Main {
     private static final Logger LOG = LoggerFactory.getLogger(Main.class);
@@ -26,7 +25,7 @@ public class Main {
         try {
             nicolas = new Nicolas();
             preprocessor = new Preprocessor();
-        } catch (IOException | ClassNotFoundException e) {
+        } catch (NicolasException e) {
             LOG.error("Error loading Nicolas or Multiservice preprocessor! Will exit.");
             return;
         }
@@ -25,7 +25,7 @@
         <!-- third party -->
         <dependency>
             <groupId>nz.ac.waikato.cms.weka</groupId>
-            <artifactId>weka-dev</artifactId>
+            <artifactId>weka-stable</artifactId>
         </dependency>
         <dependency>
             <groupId>commons-io</groupId>
+package pl.waw.ipipan.zil.summ.nicolas.common;
+
+import com.google.common.base.Predicates;
+import com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import pl.waw.ipipan.zil.multiservice.thrift.types.TText;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Map;
+import java.util.function.Predicate;
+
+public class ThriftUtils {
+
+    private static final Logger LOG = LoggerFactory.getLogger(ThriftUtils.class);
+
+    private ThriftUtils() {
+    }
+
+    public static Map<String, TText> loadThriftTextsFromFolder(File folder, Predicate<String> idFilter) {
+        Map<String, TText> id2text = Maps.newHashMap();
+        File[] files = folder.listFiles();
+        if (files != null) {
+            for (File processedFullTextFile : files) {
+                String textId = processedFullTextFile.getName().split("\\.")[0];
+                if (!idFilter.test(textId))
+                    continue;
+                TText processedFullText = loadThriftTextFromFile(processedFullTextFile);
+                id2text.put(textId, processedFullText);
+            }
+        }
+        LOG.info("{} preprocessed texts found.", id2text.size());
+        return id2text;
+    }
+
+    public static Map<String, TText> loadThriftTextsFromFolder(File folder) {
+        return loadThriftTextsFromFolder(folder, Predicates.alwaysTrue());
+    }
+
+    public static TText loadThriftTextFromFile(File originalFile) {
+        try (FileInputStream inputStream = new FileInputStream(originalFile)) {
+            return loadThriftTextFromStream(inputStream);
+        } catch (IOException e) {
+            LOG.error("Error reading serialized Thrift file", e);
+            return null;
+        }
+    }
+
+    public static TText loadThriftTextFromStream(InputStream stream) {
+        try (VersionIgnoringObjectInputStream ois = new VersionIgnoringObjectInputStream(stream)) {
+            return (TText) ois.readObject();
+        } catch (ClassNotFoundException | IOException e) {
+            LOG.error("Error reading serialized Thrift stream", e);
+            return null;
+        }
+    }
+
+}
@@ -28,6 +28,12 @@ public class Utils {
     private Utils() {
     }
+    public static void writeStringToFile(String string, File file) throws IOException {
+        try (BufferedWriter bw = new BufferedWriter(new FileWriter(file))) {
+            bw.append(string);
+        }
+    }
+
     public static Classifier loadModelFromResource(String modelResourcePath) throws IOException {
         LOG.info("Loading classifier from path: {}...", modelResourcePath);
         try (InputStream stream = Utils.class.getResourceAsStream(modelResourcePath)) {
@@ -76,44 +82,15 @@ public class Utils {
         return instances;
     }
-    public static Classifier loadClassifier(String path) throws IOException, ClassNotFoundException {
+    public static Classifier loadClassifierFromResource(String resourcePath) throws IOException, ClassNotFoundException {
         LOG.info("Loading classifier...");
-        try (ObjectInputStream ois = new ObjectInputStream(new FileInputStream(path))) {
+        try (ObjectInputStream ois = new ObjectInputStream(Utils.class.getResourceAsStream(resourcePath))) {
             Classifier classifier = (Classifier) ois.readObject();
             LOG.info("Done. " + classifier.toString());
             return classifier;
         }
     }
-    public static Map<String, TText> loadPreprocessedTexts(String path) {
-        Map<String, TText> id2text = Maps.newHashMap();
-        for (File processedFullTextFile : new File(path).listFiles()) {
-            TText processedFullText = loadThrifted(processedFullTextFile);
-            id2text.put(processedFullTextFile.getName().split("\\.")[0], processedFullText);
-        }
-        LOG.info(id2text.size() + " preprocessed texts found.");
-        return id2text;
-    }
-
-
-    public static TText loadThrifted(File originalFile) {
-        try (FileInputStream inputStream = new FileInputStream(originalFile)) {
-            return loadThrifted(inputStream);
-        } catch (IOException e) {
-            LOG.error("Error reading serialized file: " + e);
-            return null;
-        }
-    }
-
-    public static TText loadThrifted(InputStream stream) {
-        try (VersionIgnoringObjectInputStream ois = new VersionIgnoringObjectInputStream(stream)) {
-            return (TText) ois.readObject();
-        } catch (ClassNotFoundException | IOException e) {
-            LOG.error("Error reading serialized file: " + e);
-            return null;
-        }
-    }
-
     public static List<String> tokenize(String text) {
         return Arrays.asList(text.split("[^\\p{L}0-9]+"));
     }
@@ -14,7 +14,7 @@ public class UtilsTest {
     @Test
     public void shouldDeserializeTextIgnoringClassVersionId() throws Exception {
         try (InputStream stream = UtilsTest.class.getResourceAsStream(SAMPLE_TEXT_PATH)) {
-            TText text = Utils.loadThrifted(stream);
+            TText text = ThriftUtils.loadThriftTextFromStream(stream);
             assertEquals(26, text.getParagraphs().size());
             assertEquals(2, text.getParagraphs().get(4).getSentences().size());
         }
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>nicolas-container</artifactId>
+        <groupId>pl.waw.ipipan.zil.summ</groupId>
+        <version>1.0-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>nicolas-eval</artifactId>
+
+    <dependencies>
+        <!-- project -->
+        <dependency>
+            <groupId>pl.waw.ipipan.zil.summ</groupId>
+            <artifactId>nicolas-lib</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>pl.waw.ipipan.zil.summ</groupId>
+            <artifactId>nicolas-common</artifactId>
+        </dependency>
+
+        <!-- internal -->
+        <dependency>
+            <groupId>pl.waw.ipipan.zil.summ</groupId>
+            <artifactId>eval</artifactId>
+        </dependency>
+
+        <!-- third party -->
+        <dependency>
+            <groupId>nz.ac.waikato.cms.weka</groupId>
+            <artifactId>weka-stable</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+
+        <!-- logging -->
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-simple</artifactId>
+        </dependency>
+
+    </dependencies>
+</project>
 \ No newline at end of file
+package pl.waw.ipipan.zil.summ.nicolas.eval;
+
+import org.apache.commons.io.IOUtils;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class Constants {
+
+    private static final String TEST_TEXT_IDS_RESOURCE_PATH = "/pl/waw/ipipan/zil/summ/nicolas/eval/test_text_ids.txt";
+
+    private Constants() {
+    }
+
+    public static Set<String> loadTestTextIds() throws IOException {
+        try (InputStream inputStream = SummarizeTestCorpus.class.getResourceAsStream(TEST_TEXT_IDS_RESOURCE_PATH)) {
+            List<String> testTextIds = IOUtils.readLines(inputStream, pl.waw.ipipan.zil.summ.nicolas.common.Constants.ENCODING);
+            return testTextIds.stream().map(String::trim).collect(Collectors.toSet());
+        }
+    }
+}
+package pl.waw.ipipan.zil.summ.nicolas.eval;
+
+import pl.waw.ipipan.zil.summ.eval.Main;
+
+public class Evaluate {
+
+    private Evaluate() {
+    }
+
+    public static void main(String[] args) {
+        String goldDirPath = "data/summaries-gold";
+        String systemDirPath = "data/summaries";
+        Main.main(new String[]{goldDirPath, systemDirPath});
+    }
+}
 \ No newline at end of file
+package pl.waw.ipipan.zil.summ.nicolas.eval;
+
+import pl.waw.ipipan.zil.summ.nicolas.common.Utils;
+import pl.waw.ipipan.zil.summ.pscapi.io.PSC_IO;
+import pl.waw.ipipan.zil.summ.pscapi.xml.Summary;
+import pl.waw.ipipan.zil.summ.pscapi.xml.Text;
+
+import javax.xml.bind.JAXBException;
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import static pl.waw.ipipan.zil.summ.nicolas.eval.Constants.loadTestTextIds;
+
+public class ExtractGoldSummaries {
+
+    private ExtractGoldSummaries() {
+    }
+
+    public static void main(String[] args) throws IOException, JAXBException {
+        File corpusDir = new File("data/corpus/PSC_1.0/data");
+        File targetDir = new File("data/summaries-gold");
+        targetDir.mkdir();
+
+        Set<String> testTextIds = loadTestTextIds();
+        File[] files = corpusDir.listFiles();
+        if (files != null) {
+            for (File file : files) {
+                Text text = PSC_IO.readText(file);
+                if (!testTextIds.contains(text.getId()))
+                    continue;
+
+                List<Summary> goldSummaries = text.getSummaries().getSummary().stream().filter(summary -> summary.getType().equals("abstract") && summary.getRatio().equals(20)).collect(Collectors.toList());
+
+                for (Summary summary : goldSummaries) {
+                    File targetFile = new File(targetDir, text.getId() + "_" + summary.getAuthor() + ".txt");
+                    Utils.writeStringToFile(summary.getBody(), targetFile);
+                }
+            }
+        }
+    }
+
+}
+package pl.waw.ipipan.zil.summ.nicolas.eval;
+
+import com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import pl.waw.ipipan.zil.multiservice.thrift.types.TSentence;
+import pl.waw.ipipan.zil.multiservice.thrift.types.TText;
+import pl.waw.ipipan.zil.summ.nicolas.Nicolas;
+import pl.waw.ipipan.zil.summ.nicolas.NicolasException;
+import pl.waw.ipipan.zil.summ.nicolas.common.ThriftUtils;
+import pl.waw.ipipan.zil.summ.nicolas.common.Utils;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static java.util.stream.Collectors.toList;
+import static pl.waw.ipipan.zil.summ.nicolas.eval.Constants.loadTestTextIds;
+
+public class SummarizeTestCorpus {
+
+    private static final Logger LOG = LoggerFactory.getLogger(SummarizeTestCorpus.class);
+
+
+    private static final String SUMMARY_FILE_SUFFIX = "_nicolas.txt";
+    private static final double SUMMARY_RATIO = 0.2;
+
+    private SummarizeTestCorpus() {
+    }
+
+    public static void main(String[] args) throws IOException, NicolasException {
+        File thriftedCorpusDir = new File("data/preprocessed");
+        File targetDir = new File("data/summaries");
+        targetDir.mkdir();
+
+        Set<String> testTextIds = loadTestTextIds();
+        Map<String, TText> id2preprocessedText = ThriftUtils.loadThriftTextsFromFolder(thriftedCorpusDir, testTextIds::contains);
+        LOG.info("{} test corpus texts in Thrift format loaded to be summarized.", id2preprocessedText.keySet().size());
+
+        Map<String, String> id2summary = summarizeTexts(id2preprocessedText);
+        LOG.info("Texts summarized.");
+
+        saveSummariesToFolder(id2summary, targetDir);
+        LOG.info("Texts saved to {} folder.", targetDir);
+    }
+
+    private static Map<String, String> summarizeTexts(Map<String, TText> id2preprocessedText) throws NicolasException {
+        Map<String, String> id2summary = Maps.newHashMap();
+        Nicolas nicolas = new Nicolas();
+        for (Map.Entry<String, TText> entry : id2preprocessedText.entrySet()) {
+            TText text = entry.getValue();
+            int targetSize = calculateTargetSize(text);
+            String summary = nicolas.summarizeThrift(text, targetSize);
+            id2summary.put(entry.getKey(), summary);
+        }
+        return id2summary;
+    }
+
+    private static int calculateTargetSize(TText text) {
+        List<TSentence> sentences = text.getParagraphs().stream().flatMap(p -> p.getSentences().stream()).collect(toList());
+        StringBuilder body = new StringBuilder();
+        for (TSentence sentence : sentences)
+            body.append(Utils.loadSentence2Orth(sentence)).append(" ");
+
+        int tokenCount = Utils.tokenizeOnWhitespace(body.toString().trim()).size();
+        return (int) (SUMMARY_RATIO * tokenCount);
+    }
+
+    private static void saveSummariesToFolder(Map<String, String> id2summary, File targetDir) throws IOException {
+        for (Map.Entry<String, String> entry : id2summary.entrySet()) {
+            String textId = entry.getKey();
+            String summary = entry.getValue();
+            String targetFileName = textId + SUMMARY_FILE_SUFFIX;
+            Utils.writeStringToFile(summary, new File(targetDir, targetFileName));
+        }
+    }
+
+}
-package pl.waw.ipipan.zil.summ.nicolas.train.search;
+package pl.waw.ipipan.zil.summ.nicolas.eval.search;
 import org.apache.commons.lang3.time.StopWatch;
 import org.apache.commons.lang3.tuple.Pair;
@@ -35,13 +35,13 @@ import java.util.Random;
 import java.util.logging.LogManager;
-class CrossvalidateCommon {
+class Crossvalidate {
-    private static final Logger LOG = LoggerFactory.getLogger(CrossvalidateCommon.class);
+    private static final Logger LOG = LoggerFactory.getLogger(Crossvalidate.class);
     private static final int NUM_FOLDS = 10;
-    private CrossvalidateCommon() {
+    private Crossvalidate() {
     }
     static void crossvalidateClassifiers(String datasetPath) throws IOException {
@@ -77,7 +77,7 @@ class CrossvalidateCommon {
                 new DecisionTable(), new JRip(), new PART(),
                 createAttributeSelectedClassifier()}).parallel().map(cls -> {
             String name = cls.getClass().getSimpleName();
-            double acc = 0;
+            double acc;
             Evaluation eval;
             try {
                 eval = new Evaluation(instances);
@@ -35,7 +35,7 @@
         <!-- third party -->
         <dependency>
             <groupId>nz.ac.waikato.cms.weka</groupId>
-            <artifactId>weka-dev</artifactId>
+            <artifactId>weka-stable</artifactId>
         </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>
@@ -18,18 +18,18 @@ import java.util.Set;
 import static java.util.stream.Collectors.toList;
-public class ThriftUtils {
+public class InstanceUtils {
-    private static final Logger LOG = LoggerFactory.getLogger(ThriftUtils.class);
+    private static final Logger LOG = LoggerFactory.getLogger(InstanceUtils.class);
-    private ThriftUtils() {
+    private InstanceUtils() {
     }
     public static Map<TMention, Instance> extractInstancesFromMentions(TText preprocessedText, MentionFeatureExtractor featureExtractor) {
         List<TSentence> sentences = preprocessedText.getParagraphs().stream().flatMap(p -> p.getSentences().stream()).collect(toList());
         Map<TMention, Map<Attribute, Double>> mention2features = featureExtractor.calculateFeatures(preprocessedText);
-        LOG.info("Extracting " + featureExtractor.getAttributesList().size() + " features of each mention.");
+        LOG.info("Extracting {} features of each mention.", featureExtractor.getAttributesList().size());
         Map<TMention, Instance> mention2instance = Maps.newHashMap();
         for (TMention tMention : sentences.stream().flatMap(s -> s.getMentions().stream()).collect(toList())) {
             Instance instance = new DenseInstance(featureExtractor.getAttributesList().size());
@@ -39,7 +39,7 @@ public class ThriftUtils {
             }
             mention2instance.put(tMention, instance);
         }
-        LOG.info("Extracted features of " + mention2instance.size() + " mentions.");
+        LOG.info("Extracted features of {} mentions.", mention2instance.size());
         return mention2instance;
     }
@@ -47,7 +47,7 @@ public class ThriftUtils {
         List<TSentence> sentences = preprocessedText.getParagraphs().stream().flatMap(p -> p.getSentences().stream()).collect(toList());
         Map<TSentence, Map<Attribute, Double>> sentence2features = featureExtractor.calculateFeatures(preprocessedText, goodMentions);
-        LOG.info("Extracting " + featureExtractor.getAttributesList().size() + " features of each sentence.");
+        LOG.info("Extracting {} features of each sentence.", featureExtractor.getAttributesList().size());
         Map<TSentence, Instance> sentence2instance = Maps.newHashMap();
         for (TSentence sentence : sentences) {
             Instance instance = new DenseInstance(featureExtractor.getAttributesList().size());
@@ -57,7 +57,7 @@ public class ThriftUtils {
             }
             sentence2instance.put(sentence, instance);
         }
-        LOG.info("Extracted features of " + sentence2instance.size() + " sentences.");
+        LOG.info("Extracted features of {} sentences.", sentence2instance.size());
         return sentence2instance;
     }
 }
@@ -29,14 +29,18 @@ public class Nicolas {
     private final SentenceFeatureExtractor sentenceFeatureExtractor;
     private final ZeroFeatureExtractor zeroFeatureExtractor;
-    public Nicolas() throws IOException, ClassNotFoundException {
-        mentionModel = Utils.loadModelFromResource(Constants.MENTION_MODEL_RESOURCE_PATH);
-        sentenceModel = Utils.loadModelFromResource(Constants.SENTENCE_MODEL_RESOURCE_PATH);
-        zeroModel = Utils.loadModelFromResource(Constants.ZERO_MODEL_RESOURCE_PATH);
-
-        mentionFeatureExtractor = new MentionFeatureExtractor();
-        sentenceFeatureExtractor = new SentenceFeatureExtractor();
-        zeroFeatureExtractor = new ZeroFeatureExtractor();
+    public Nicolas() throws NicolasException {
+        try {
+            mentionModel = Utils.loadModelFromResource(Constants.MENTION_MODEL_RESOURCE_PATH);
+            sentenceModel = Utils.loadModelFromResource(Constants.SENTENCE_MODEL_RESOURCE_PATH);
+            zeroModel = Utils.loadModelFromResource(Constants.ZERO_MODEL_RESOURCE_PATH);
+
+            mentionFeatureExtractor = new MentionFeatureExtractor();
+            sentenceFeatureExtractor = new SentenceFeatureExtractor();
+            zeroFeatureExtractor = new ZeroFeatureExtractor();
+        } catch (IOException e) {
+            throw new NicolasException(e);
+        }
     }
     public String summarizeThrift(TText text, int targetTokenCount) throws NicolasException {
@@ -59,17 +63,17 @@ public class Nicolas {
     }
     private List<TSentence> selectSummarySentences(TText thrifted, Set<TMention> goodMentions, int targetSize) throws Exception {
-        List<TSentence> sents = thrifted.getParagraphs().stream().flatMap(p -> p.getSentences().stream()).collect(toList());
+        List<TSentence> sentences = thrifted.getParagraphs().stream().flatMap(p -> p.getSentences().stream()).collect(toList());
         Map<TSentence, Double> sentence2score = SentenceModel.scoreSentences(thrifted, goodMentions, sentenceModel, sentenceFeatureExtractor);
-        List<TSentence> sortedSents = Lists.newArrayList(sents);
-        sortedSents.sort(Comparator.comparing(sentence2score::get).reversed());
+        List<TSentence> sortedSentences = Lists.newArrayList(sentences);
+        sortedSentences.sort(Comparator.comparing(sentence2score::get).reversed());
         int size = 0;
         Random r = new Random(1);
         Set<TSentence> summary = Sets.newHashSet();
-        for (TSentence sent : sortedSents) {
+        for (TSentence sent : sortedSentences) {
             size += Utils.tokenizeOnWhitespace(Utils.loadSentence2Orth(sent)).size();
             if (r.nextDouble() > 0.4 && size > targetSize)
                 break;
@@ -78,7 +82,7 @@ public class Nicolas {
                 break;
         }
         List<TSentence> selectedSentences = Lists.newArrayList();
-        for (TSentence sent : sents) {
+        for (TSentence sent : sentences) {
             if (summary.contains(sent))
                 selectedSentences.add(sent);
         }
+package pl.waw.ipipan.zil.summ.nicolas;
+
+public class NicolasException extends Exception {
+    public NicolasException(Exception e) {
+        super(e);
+    }
+}
@@ -8,8 +8,9 @@ import org.slf4j.LoggerFactory;
 import pl.waw.ipipan.zil.multiservice.thrift.types.TMention;
 import pl.waw.ipipan.zil.multiservice.thrift.types.TSentence;
 import pl.waw.ipipan.zil.multiservice.thrift.types.TText;
-import pl.waw.ipipan.zil.summ.nicolas.ThriftUtils;
+import pl.waw.ipipan.zil.summ.nicolas.InstanceUtils;
 import pl.waw.ipipan.zil.summ.nicolas.common.Constants;
+import pl.waw.ipipan.zil.summ.nicolas.common.ThriftUtils;
 import pl.waw.ipipan.zil.summ.nicolas.common.Utils;
 import pl.waw.ipipan.zil.summ.nicolas.mention.MentionFeatureExtractor;
 import pl.waw.ipipan.zil.summ.nicolas.mention.MentionModel;
@@ -34,15 +35,15 @@ public class ApplyModel {
     private static final String TARGET_DIR = "corpora/summaries";
     public static void main(String[] args) throws Exception {
-        Classifier mentionClassifier = Utils.loadClassifier(Constants.MENTION_MODEL_RESOURCE_PATH);
+        Classifier mentionClassifier = Utils.loadClassifierFromResource(Constants.MENTION_MODEL_RESOURCE_PATH);
         MentionFeatureExtractor featureExtractor = new MentionFeatureExtractor();
-        Classifier sentenceClassifier = Utils.loadClassifier(Constants.SENTENCE_MODEL_RESOURCE_PATH);
+        Classifier sentenceClassifier = Utils.loadClassifierFromResource(Constants.SENTENCE_MODEL_RESOURCE_PATH);
         SentenceFeatureExtractor sentenceFeatureExtractor = new SentenceFeatureExtractor();
         ZeroSubjectInjector zeroSubjectInjector = new ZeroSubjectInjector();
-        Map<String, TText> id2preprocessedText = Utils.loadPreprocessedTexts(TEST_PREPROCESSED_DATA_PATH);
+        Map<String, TText> id2preprocessedText = ThriftUtils.loadThriftTextsFromFolder(new File(TEST_PREPROCESSED_DATA_PATH));
         int i = 1;
         double avgSize = 0;
         for (Map.Entry<String, TText> entry : id2preprocessedText.entrySet()) {
@@ -91,7 +92,7 @@ public class ApplyModel {
         List<TSentence> sents = thrifted.getParagraphs().stream().flatMap(p -> p.getSentences().stream()).collect(toList());
         Instances instances = Utils.createNewInstances(sentenceFeatureExtractor.getAttributesList());
-        Map<TSentence, Instance> sentence2instance = ThriftUtils.extractInstancesFromSentences(thrifted, sentenceFeatureExtractor, goodMentions);
+        Map<TSentence, Instance> sentence2instance = InstanceUtils.extractInstancesFromSentences(thrifted, sentenceFeatureExtractor, goodMentions);
         Map<TSentence, Double> sentence2score = Maps.newHashMap();
         for (Map.Entry<TSentence, Instance> entry : sentence2instance.entrySet()) {
-package pl.waw.ipipan.zil.summ.nicolas.common.features;
+package pl.waw.ipipan.zil.summ.nicolas.features;
 import com.google.common.collect.*;
 import org.slf4j.Logger;
-package pl.waw.ipipan.zil.summ.nicolas.common.features;
+package pl.waw.ipipan.zil.summ.nicolas.features;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
-package pl.waw.ipipan.zil.summ.nicolas.common.features;
+package pl.waw.ipipan.zil.summ.nicolas.features;
 import pl.waw.ipipan.zil.multiservice.thrift.types.TInterpretation;
@@ -5,9 +5,9 @@ import com.google.common.collect.Maps;
 import pl.waw.ipipan.zil.multiservice.thrift.types.*;
 import pl.waw.ipipan.zil.summ.nicolas.common.Constants;
 import pl.waw.ipipan.zil.summ.nicolas.common.Utils;
-import pl.waw.ipipan.zil.summ.nicolas.common.features.FeatureExtractor;
-import pl.waw.ipipan.zil.summ.nicolas.common.features.FeatureHelper;
-import pl.waw.ipipan.zil.summ.nicolas.common.features.Interpretation;
+import pl.waw.ipipan.zil.summ.nicolas.features.FeatureExtractor;
+import pl.waw.ipipan.zil.summ.nicolas.features.FeatureHelper;
+import pl.waw.ipipan.zil.summ.nicolas.features.Interpretation;
 import weka.core.Attribute;
 import java.io.IOException;
@@ -5,7 +5,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import pl.waw.ipipan.zil.multiservice.thrift.types.TMention;
 import pl.waw.ipipan.zil.multiservice.thrift.types.TText;
-import pl.waw.ipipan.zil.summ.nicolas.ThriftUtils;
+import pl.waw.ipipan.zil.summ.nicolas.InstanceUtils;
 import pl.waw.ipipan.zil.summ.nicolas.common.Utils;
 import weka.classifiers.Classifier;
 import weka.core.Instance;
@@ -25,7 +25,7 @@ public class MentionModel {
         Set<TMention> goodMentions = Sets.newHashSet();
         Instances instances = Utils.createNewInstances(featureExtractor.getAttributesList());
-        Map<TMention, Instance> mention2instance = ThriftUtils.extractInstancesFromMentions(text, featureExtractor);
+        Map<TMention, Instance> mention2instance = InstanceUtils.extractInstancesFromMentions(text, featureExtractor);
         for (Map.Entry<TMention, Instance> entry : mention2instance.entrySet()) {
             Instance instance = entry.getValue();
             instance.setDataset(instances);
@@ -34,7 +34,7 @@ public class MentionModel {
             if (good)
                 goodMentions.add(entry.getKey());
         }
-        LOG.info("Classified " + goodMentions.size() + " mentions as good.");
+        LOG.info("Classified {} mentions as good.", goodMentions.size());
         return goodMentions;
     }
@@ -2,8 +2,8 @@ package pl.waw.ipipan.zil.summ.nicolas.sentence;
 import com.google.common.collect.Maps;
 import pl.waw.ipipan.zil.multiservice.thrift.types.*;
-import pl.waw.ipipan.zil.summ.nicolas.common.features.FeatureExtractor;
-import pl.waw.ipipan.zil.summ.nicolas.common.features.FeatureHelper;
+import pl.waw.ipipan.zil.summ.nicolas.features.FeatureExtractor;
+import pl.waw.ipipan.zil.summ.nicolas.features.FeatureHelper;
 import weka.core.Attribute;
 import java.util.List;
@@ -6,7 +6,7 @@ import org.slf4j.LoggerFactory;
 import pl.waw.ipipan.zil.multiservice.thrift.types.TMention;
 import pl.waw.ipipan.zil.multiservice.thrift.types.TSentence;
 import pl.waw.ipipan.zil.multiservice.thrift.types.TText;
-import pl.waw.ipipan.zil.summ.nicolas.ThriftUtils;
+import pl.waw.ipipan.zil.summ.nicolas.InstanceUtils;
 import pl.waw.ipipan.zil.summ.nicolas.common.Utils;
 import weka.classifiers.Classifier;
 import weka.core.Instance;
@@ -24,7 +24,7 @@ public class SentenceModel {
     public static Map<TSentence, Double> scoreSentences(TText thrifted, Set<TMention> goodMentions, Classifier sentenceClassifier, SentenceFeatureExtractor sentenceFeatureExtractor) throws Exception {
         Instances instances = Utils.createNewInstances(sentenceFeatureExtractor.getAttributesList());
-        Map<TSentence, Instance> sentence2instance = ThriftUtils.extractInstancesFromSentences(thrifted, sentenceFeatureExtractor, goodMentions);
+        Map<TSentence, Instance> sentence2instance = InstanceUtils.extractInstancesFromSentences(thrifted, sentenceFeatureExtractor, goodMentions);
         Map<TSentence, Double> sentence2score = Maps.newHashMap();
         for (Map.Entry<TSentence, Instance> entry : sentence2instance.entrySet()) {
@@ -8,8 +8,8 @@ import pl.waw.ipipan.zil.multiservice.thrift.types.TSentence;
 import pl.waw.ipipan.zil.multiservice.thrift.types.TText;
 import pl.waw.ipipan.zil.multiservice.thrift.types.TToken;
 import pl.waw.ipipan.zil.summ.nicolas.common.Constants;
-import pl.waw.ipipan.zil.summ.nicolas.common.features.FeatureExtractor;
-import pl.waw.ipipan.zil.summ.nicolas.common.features.FeatureHelper;
+import pl.waw.ipipan.zil.summ.nicolas.features.FeatureExtractor;
+import pl.waw.ipipan.zil.summ.nicolas.features.FeatureHelper;
 import weka.core.Attribute;
 import java.util.List;
@@ -5,8 +5,8 @@ import org.apache.commons.io.IOUtils;
 import org.junit.Test;
 import pl.waw.ipipan.zil.multiservice.thrift.types.TMention;
 import pl.waw.ipipan.zil.multiservice.thrift.types.TSentence;
-import pl.waw.ipipan.zil.summ.nicolas.common.Utils;
-import pl.waw.ipipan.zil.summ.nicolas.common.features.FeatureHelper;
+import pl.waw.ipipan.zil.summ.nicolas.common.ThriftUtils;
+import pl.waw.ipipan.zil.summ.nicolas.features.FeatureHelper;
 import java.io.IOException;
 import java.io.InputStream;
@@ -47,7 +47,7 @@ public class CandidateFinderTest {
     private FeatureHelper loadSampleTextHelper() throws IOException {
         try (InputStream stream = CandidateFinderTest.class.getResourceAsStream(SAMPLE_TEXT_PATH)) {
-            return new FeatureHelper(Utils.loadThrifted(stream));
+            return new FeatureHelper(ThriftUtils.loadThriftTextFromStream(stream));
         }
     }
 }
 \ No newline at end of file
@@ -25,6 +25,11 @@
             <groupId>pl.waw.ipipan.zil.summ</groupId>
             <artifactId>nicolas-multiservice</artifactId>
         </dependency>
+        <dependency>
+            <groupId>pl.waw.ipipan.zil.summ</groupId>
+            <artifactId>nicolas-model</artifactId>
+            <scope>runtime</scope>
+        </dependency>
         <!-- internal -->
         <dependency>
@@ -39,7 +44,7 @@
         <!-- third party -->
         <dependency>
             <groupId>nz.ac.waikato.cms.weka</groupId>
-            <artifactId>weka-dev</artifactId>
+            <artifactId>weka-stable</artifactId>
         </dependency>
         <dependency>
             <groupId>org.apache.commons</groupId>
@@ -5,11 +5,11 @@ import weka.classifiers.trees.RandomForest;
 public class ModelConstants {
-    public static final String MENTION_DATASET_PATH = "mentions_train.arff";
-    public static final String SENTENCE_DATASET_PATH = "sentences_train.arff";
-    public static final String ZERO_DATASET_PATH = "zeros_train.arff";
+    public static final String MENTION_DATASET_PATH = "data/arff/mentions_train.arff";
+    public static final String SENTENCE_DATASET_PATH = "data/arff/sentences_train.arff";
+    public static final String ZERO_DATASET_PATH = "data/arff/zeros_train.arff";
-    private static final int NUM_ITERATIONS = 16;
+    private static final int NUM_ITERATIONS = 250;
     private static final int NUM_EXECUTION_SLOTS = 8;
     private static final int SEED = 0;
@@ -26,17 +26,17 @@ public class ModelConstants {
     public static Classifier getSentenceClassifier() {
         RandomForest classifier = new RandomForest();
-        classifier.setNumIterations(16);
-        classifier.setSeed(0);
-        classifier.setNumExecutionSlots(8);
+        classifier.setNumIterations(NUM_ITERATIONS);
+        classifier.setSeed(SEED);
+        classifier.setNumExecutionSlots(NUM_EXECUTION_SLOTS);
         return classifier;
     }
     public static Classifier getZeroClassifier() {
         RandomForest classifier = new RandomForest();
-        classifier.setNumIterations(16);
-        classifier.setSeed(0);
-        classifier.setNumExecutionSlots(8);
+        classifier.setNumIterations(NUM_ITERATIONS);
+        classifier.setSeed(SEED);
+        classifier.setNumExecutionSlots(NUM_EXECUTION_SLOTS);
         return classifier;
     }
@@ -3,7 +3,6 @@ package pl.waw.ipipan.zil.summ.nicolas.train.model.common;
 import org.apache.commons.lang3.time.StopWatch;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import pl.waw.ipipan.zil.summ.nicolas.train.model.zero.TrainZeroModel;
 import weka.classifiers.Classifier;
 import weka.core.Instances;
 import weka.core.converters.ArffLoader;
@@ -16,7 +15,7 @@ import java.util.logging.LogManager;
 @SuppressWarnings("squid:S2118")
 public class TrainModelCommon {
-    private static final Logger LOG = LoggerFactory.getLogger(TrainZeroModel.class);
+    private static final Logger LOG = LoggerFactory.getLogger(TrainModelCommon.class);
     private static final String TARGET_MODEL_DIR = "nicolas-model/src/main/resources";
-package pl.waw.ipipan.zil.summ.nicolas.train.model.mention;
-
-import com.google.common.base.Charsets;
-import com.google.common.collect.Maps;
-import com.google.common.io.Files;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import pl.waw.ipipan.zil.multiservice.thrift.types.TMention;
-import pl.waw.ipipan.zil.multiservice.thrift.types.TText;
-import pl.waw.ipipan.zil.summ.nicolas.ThriftUtils;
-import pl.waw.ipipan.zil.summ.nicolas.common.Constants;
-import pl.waw.ipipan.zil.summ.nicolas.common.Utils;
-import pl.waw.ipipan.zil.summ.nicolas.mention.MentionFeatureExtractor;
-import pl.waw.ipipan.zil.summ.nicolas.train.model.common.ModelConstants;
-import weka.core.Instance;
-import weka.core.Instances;
-import weka.core.converters.ArffSaver;
-
-import java.io.File;
-import java.io.IOException;
-import java.util.Map;
-
-
-public class PrepareTrainingData {
-
-    private static final Logger LOG = LoggerFactory.getLogger(PrepareTrainingData.class);
-
-    private static final String PREPROCESSED_FULL_TEXTS_DIR_PATH = "src/main/resources/preprocessed_full_texts/dev";
-    private static final String OPTIMAL_SUMMARIES_DIR_PATH = "src/main/resources/optimal_summaries/dev";
-
-    private PrepareTrainingData() {
-    }
-
-    public static void main(String[] args) throws IOException {
-
-        Map<String, TText> id2preprocessedText = Utils.loadPreprocessedTexts(PREPROCESSED_FULL_TEXTS_DIR_PATH);
-        Map<String, String> id2optimalSummary = loadOptimalSummaries();
-
-        MentionScorer mentionScorer = new MentionScorer();
-        MentionFeatureExtractor featureExtractor = new MentionFeatureExtractor();
-
-        Instances instances = Utils.createNewInstances(featureExtractor.getAttributesList());
-
-        int i = 1;
-        for (Map.Entry<String, TText> entry : id2preprocessedText.entrySet()) {
-            LOG.info(i++ + "/" + id2preprocessedText.size());
-
-            String id = entry.getKey();
-            TText preprocessedText = entry.getValue();
-            String optimalSummary = id2optimalSummary.get(id);
-            if (optimalSummary == null)
-                continue;
-            Map<TMention, Double> mention2score = mentionScorer.calculateMentionScores(optimalSummary, preprocessedText);
-
-            Map<TMention, Instance> mention2instance = ThriftUtils.extractInstancesFromMentions(preprocessedText, featureExtractor);
-            for (Map.Entry<TMention, Instance> entry2 : mention2instance.entrySet()) {
-                TMention mention = entry2.getKey();
-                Instance instance = entry2.getValue();
-                instance.setDataset(instances);
-                instance.setClassValue(mention2score.get(mention));
-                instances.add(instance);
-            }
-        }
-        saveInstancesToFile(instances);
-    }
-
-    private static void saveInstancesToFile(Instances instances) throws IOException {
-        ArffSaver saver = new ArffSaver();
-        saver.setInstances(instances);
-        saver.setFile(new File(ModelConstants.MENTION_DATASET_PATH));
-        saver.writeBatch();
-    }
-
-    private static Map<String, String> loadOptimalSummaries() throws IOException {
-        Map<String, String> id2optimalSummary = Maps.newHashMap();
-        for (File optimalSummaryFile : new File(OPTIMAL_SUMMARIES_DIR_PATH).listFiles()) {
-            String optimalSummary = Files.toString(optimalSummaryFile, Charsets.UTF_8);
-            id2optimalSummary.put(optimalSummaryFile.getName().split("_")[0], optimalSummary);
-        }
-        LOG.info(id2optimalSummary.size() + " optimal summaries found.");
-        return id2optimalSummary;
-    }
-
-
-}
-package pl.waw.ipipan.zil.summ.nicolas.train.model.zero;
-
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-import org.apache.commons.io.IOUtils;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import pl.waw.ipipan.zil.multiservice.thrift.types.TText;
-import pl.waw.ipipan.zil.summ.nicolas.common.Utils;
-import pl.waw.ipipan.zil.summ.nicolas.common.features.FeatureHelper;
-import pl.waw.ipipan.zil.summ.nicolas.train.model.common.ModelConstants;
-import pl.waw.ipipan.zil.summ.nicolas.zero.CandidateFinder;
-import pl.waw.ipipan.zil.summ.nicolas.zero.InstanceCreator;
-import pl.waw.ipipan.zil.summ.nicolas.zero.ZeroFeatureExtractor;
-import pl.waw.ipipan.zil.summ.nicolas.zero.ZeroSubjectCandidate;
-import weka.core.Instance;
-import weka.core.Instances;
-import weka.core.converters.ArffSaver;
-
-import java.io.File;
-import java.io.FileReader;
-import java.io.IOException;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-public class PrepareTrainingData {
-
-    private static final Logger LOG = LoggerFactory.getLogger(PrepareTrainingData.class);
-
-    private static final String IDS_PATH = "corpora/summaries_dev";
-    private static final String THRIFTED_PATH = "corpora/preprocessed_full_texts/dev/";
-    private static final String GOLD_ZEROS_PATH = "/zeros.tsv";
-
-    private PrepareTrainingData() {
-    }
-
-    public static void main(String[] args) throws IOException {
-
-        Map<String, TText> id2preprocessedText = Utils.loadPreprocessedTexts(THRIFTED_PATH);
-        Map<String, Set<String>> id2sentIds = loadSentenceIds(IDS_PATH);
-
-        ZeroScorer zeroScorer = new ZeroScorer(GOLD_ZEROS_PATH);
-        ZeroFeatureExtractor featureExtractor = new ZeroFeatureExtractor();
-
-        Instances instances = Utils.createNewInstances(featureExtractor.getAttributesList());
-
-        int i = 1;
-        for (Map.Entry<String, TText> entry : id2preprocessedText.entrySet()) {
-            LOG.info(i++ + "/" + id2preprocessedText.size());
-
-            String textId = entry.getKey();
-
-            TText text = entry.getValue();
-            Set<String> sentenceIds = id2sentIds.get(textId);
-            FeatureHelper featureHelper = new FeatureHelper(text);
-
-            List<ZeroSubjectCandidate> zeroSubjectCandidates = CandidateFinder.findZeroSubjectCandidates(text, sentenceIds);
-            Map<ZeroSubjectCandidate, Instance> candidate2instance = InstanceCreator.extractInstancesFromZeroCandidates(zeroSubjectCandidates, text, featureExtractor);
-
-            for (Map.Entry<ZeroSubjectCandidate, Instance> entry2 : candidate2instance.entrySet()) {
-                boolean good = zeroScorer.isValidCandidate(entry2.getKey(), featureHelper);
-                Instance instance = entry2.getValue();
-                instance.setDataset(instances);
-                instance.setClassValue(good ? 1 : 0);
-                instances.add(instance);
-            }
-        }
-
-        saveInstancesToFile(instances);
-    }
-
-
-    private static void saveInstancesToFile(Instances instances) throws IOException {
-        ArffSaver saver = new ArffSaver();
-        saver.setInstances(instances);
-        saver.setFile(new File(ModelConstants.ZERO_DATASET_PATH));
-        saver.writeBatch();
-    }
-
-    private static Map<String, Set<String>> loadSentenceIds(String idsPath) throws IOException {
-        Map<String, Set<String>> result = Maps.newHashMap();
-        for (File f : new File(idsPath).listFiles()) {
-            String id = f.getName().split("_")[0];
-            List<String> sentenceIds = IOUtils.readLines(new FileReader(f));
-            result.put(id, Sets.newHashSet(sentenceIds));
-        }
-        return result;
-    }
-}
@@ -6,7 +6,7 @@ import org.apache.commons.csv.CSVParser;
 import org.apache.commons.csv.CSVRecord;
 import org.apache.commons.csv.QuoteMode;
 import pl.waw.ipipan.zil.summ.nicolas.common.Constants;
-import pl.waw.ipipan.zil.summ.nicolas.common.features.FeatureHelper;
+import pl.waw.ipipan.zil.summ.nicolas.features.FeatureHelper;
 import pl.waw.ipipan.zil.summ.nicolas.zero.ZeroSubjectCandidate;
 import java.io.IOException;
@@ -21,8 +21,8 @@ public class ZeroScorer {
     private final Map<String, Boolean> candidateEncoding2Decision = Maps.newHashMap();
-    public ZeroScorer(String goldZerosPath) throws IOException {
-        try (InputStream stream = ZeroScorer.class.getResourceAsStream(goldZerosPath);
+    public ZeroScorer(String goldZerosResourcePath) throws IOException {
+        try (InputStream stream = ZeroScorer.class.getResourceAsStream(goldZerosResourcePath);
              InputStreamReader reader = new InputStreamReader(stream, Constants.ENCODING);
              CSVParser parser = new CSVParser(reader, CSVFormat.DEFAULT.withDelimiter(DELIMITER).withEscape('|').withQuoteMode(QuoteMode.NONE).withQuote('~'))) {
             List<CSVRecord> records = parser.getRecords();
-package pl.waw.ipipan.zil.summ.nicolas.train;
+package pl.waw.ipipan.zil.summ.nicolas.train.pipeline;
 import net.lingala.zip4j.core.ZipFile;
 import org.apache.commons.io.FileUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
-import pl.waw.ipipan.zil.summ.nicolas.train.preprocess.Main;
 import java.io.File;
 import java.net.URL;
@@ -45,7 +44,7 @@ public class DownloadAndPreprocessCorpus {
         File preprocessed = new File(WORKING_DIR, "preprocessed");
         createFolder(preprocessed.getPath());
-        Main.main(new String[]{dataDir.getPath(), preprocessed.getPath()});
+        Preprocess.main(new String[]{dataDir.getPath(), preprocessed.getPath()});
     }
     private static File createFolder(String path) {
-package pl.waw.ipipan.zil.summ.nicolas.train.model.sentence;
+package pl.waw.ipipan.zil.summ.nicolas.train.pipeline;
 import com.google.common.base.Charsets;
 import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
 import com.google.common.io.Files;
+import org.apache.commons.io.IOUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import pl.waw.ipipan.zil.multiservice.thrift.types.TMention;
 import pl.waw.ipipan.zil.multiservice.thrift.types.TSentence;
 import pl.waw.ipipan.zil.multiservice.thrift.types.TText;
-import pl.waw.ipipan.zil.summ.nicolas.ThriftUtils;
+import pl.waw.ipipan.zil.summ.nicolas.InstanceUtils;
 import pl.waw.ipipan.zil.summ.nicolas.common.Constants;
+import pl.waw.ipipan.zil.summ.nicolas.common.ThriftUtils;
 import pl.waw.ipipan.zil.summ.nicolas.common.Utils;
+import pl.waw.ipipan.zil.summ.nicolas.features.FeatureHelper;
 import pl.waw.ipipan.zil.summ.nicolas.mention.MentionFeatureExtractor;
 import pl.waw.ipipan.zil.summ.nicolas.mention.MentionModel;
 import pl.waw.ipipan.zil.summ.nicolas.sentence.SentenceFeatureExtractor;
 import pl.waw.ipipan.zil.summ.nicolas.train.model.common.ModelConstants;
+import pl.waw.ipipan.zil.summ.nicolas.train.model.mention.MentionScorer;
+import pl.waw.ipipan.zil.summ.nicolas.train.model.sentence.SentenceScorer;
+import pl.waw.ipipan.zil.summ.nicolas.train.model.zero.ZeroScorer;
+import pl.waw.ipipan.zil.summ.nicolas.zero.CandidateFinder;
+import pl.waw.ipipan.zil.summ.nicolas.zero.InstanceCreator;
+import pl.waw.ipipan.zil.summ.nicolas.zero.ZeroFeatureExtractor;
+import pl.waw.ipipan.zil.summ.nicolas.zero.ZeroSubjectCandidate;
 import weka.classifiers.Classifier;
 import weka.core.Instance;
 import weka.core.Instances;
 import weka.core.converters.ArffSaver;
 import java.io.File;
+import java.io.FileReader;
 import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
 import java.util.Map;
 import java.util.Set;
-
+import java.util.function.Predicate;
+import java.util.stream.Collectors;
 public class PrepareTrainingData {
     private static final Logger LOG = LoggerFactory.getLogger(PrepareTrainingData.class);
-    private static final String PREPROCESSED_FULL_TEXTS_DIR_PATH = "src/main/resources/preprocessed_full_texts/dev";
-    private static final String OPTIMAL_SUMMARIES_DIR_PATH = "src/main/resources/optimal_summaries/dev";
+    private static final String THRIFT_TEXTS_PATH = "data/preprocessed";
+    private static final String OPTIMAL_SUMMARIES_DIR_PATH = "data/summaries-optimal";
+    private static final String SUMMARY_SENTENCE_IDS = "data/summaries-sentence-ids";
+
+    private static final String ZERO_TRAINING_DATA_RESOURCE_PATH = "/pl/waw/ipipan/zil/summ/nicolas/train/train_zero.tsv";
+    private static final String TRAIN_TEXT_IDS_RESOURCE_PATH = "/pl/waw/ipipan/zil/summ/nicolas/train/train_text_ids.txt";
     private PrepareTrainingData() {
     }
     public static void main(String[] args) throws Exception {
+        Set<String> trainTextIds = loadTrainTextIds();
+
+        Map<String, TText> id2preprocessedText = ThriftUtils.loadThriftTextsFromFolder(new File(THRIFT_TEXTS_PATH), trainTextIds::contains);
+        Map<String, String> id2optimalSummary = loadOptimalSummaries(trainTextIds::contains);
+
+        prepareMentionsDataset(id2preprocessedText, id2optimalSummary);
+        prepareSentencesDataset(id2preprocessedText, id2optimalSummary);
+        prepareZerosDataset(id2preprocessedText);
+    }
+
+    public static void prepareMentionsDataset(Map<String, TText> id2preprocessedText, Map<String, String> id2optimalSummary) throws IOException {
+        MentionScorer mentionScorer = new MentionScorer();
+        MentionFeatureExtractor featureExtractor = new MentionFeatureExtractor();
+
+        Instances instances = Utils.createNewInstances(featureExtractor.getAttributesList());
+
+        int i = 1;
+        for (Map.Entry<String, TText> entry : id2preprocessedText.entrySet()) {
+            LOG.info("{}/{}", i++, id2preprocessedText.size());
+
+            String id = entry.getKey();
+            TText preprocessedText = entry.getValue();
+            String optimalSummary = id2optimalSummary.get(id);
+            if (optimalSummary == null)
+                continue;
+            Map<TMention, Double> mention2score = mentionScorer.calculateMentionScores(optimalSummary, preprocessedText);
+
+            Map<TMention, Instance> mention2instance = InstanceUtils.extractInstancesFromMentions(preprocessedText, featureExtractor);
+            for (Map.Entry<TMention, Instance> entry2 : mention2instance.entrySet()) {
+                TMention mention = entry2.getKey();
+                Instance instance = entry2.getValue();
+                instance.setDataset(instances);
+                instance.setClassValue(mention2score.get(mention));
+                instances.add(instance);
+            }
+        }
+        saveInstancesToFile(instances, new File(ModelConstants.MENTION_DATASET_PATH));
+    }
+
+    private static Set<String> loadTrainTextIds() throws IOException {
+        try (InputStream inputStream = PrepareTrainingData.class.getResourceAsStream(TRAIN_TEXT_IDS_RESOURCE_PATH)) {
+            List<String> testTextIds = IOUtils.readLines(inputStream, Constants.ENCODING);
+            return testTextIds.stream().map(String::trim).collect(Collectors.toSet());
+        }
+    }
-        Map<String, TText> id2preprocessedText = Utils.loadPreprocessedTexts(PREPROCESSED_FULL_TEXTS_DIR_PATH);
-        Map<String, String> id2optimalSummary = loadOptimalSummaries();
+    public static void prepareSentencesDataset(Map<String, TText> id2preprocessedText, Map<String, String> id2optimalSummary) throws Exception {
         SentenceScorer sentenceScorer = new SentenceScorer();
         SentenceFeatureExtractor featureExtractor = new SentenceFeatureExtractor();
         Instances instances = Utils.createNewInstances(featureExtractor.getAttributesList());
-        Classifier classifier = Utils.loadClassifier(Constants.MENTION_MODEL_RESOURCE_PATH);
+        Classifier classifier = Utils.loadClassifierFromResource(Constants.MENTION_MODEL_RESOURCE_PATH);
         MentionFeatureExtractor mentionFeatureExtractor = new MentionFeatureExtractor();
         int i = 1;
         for (String textId : id2preprocessedText.keySet()) {
-            LOG.info(i++ + "/" + id2preprocessedText.size());
+            LOG.info("{}/{}", i++, id2preprocessedText.size());
             TText preprocessedText = id2preprocessedText.get(textId);
             String optimalSummary = id2optimalSummary.get(textId);
@@ -64,7 +127,7 @@ public class PrepareTrainingData {
 //            Set<TMention> goodMentions
 //                    = Utils.loadGoldGoodMentions(textId, preprocessedText, true);
-            Map<TSentence, Instance> sentence2instance = ThriftUtils.extractInstancesFromSentences(preprocessedText, featureExtractor, goodMentions);
+            Map<TSentence, Instance> sentence2instance = InstanceUtils.extractInstancesFromSentences(preprocessedText, featureExtractor, goodMentions);
             for (Map.Entry<TSentence, Instance> entry : sentence2instance.entrySet()) {
                 TSentence sentence = entry.getKey();
                 Instance instance = entry.getValue();
@@ -73,25 +136,74 @@ public class PrepareTrainingData {
                 instances.add(instance);
             }
         }
-        saveInstancesToFile(instances);
+        saveInstancesToFile(instances, new File(ModelConstants.SENTENCE_DATASET_PATH));
+    }
+
+    public static void prepareZerosDataset(Map<String, TText> id2preprocessedText) throws IOException {
+
+        Map<String, Set<String>> id2sentIds = loadSentenceIds(SUMMARY_SENTENCE_IDS);
+
+        ZeroScorer zeroScorer = new ZeroScorer(ZERO_TRAINING_DATA_RESOURCE_PATH);
+        ZeroFeatureExtractor featureExtractor = new ZeroFeatureExtractor();
+
+        Instances instances = Utils.createNewInstances(featureExtractor.getAttributesList());
+
+        int i = 1;
+        for (Map.Entry<String, TText> entry : id2preprocessedText.entrySet()) {
+            LOG.info(i++ + "/" + id2preprocessedText.size());
+
+            String textId = entry.getKey();
+
+            TText text = entry.getValue();
+            Set<String> sentenceIds = id2sentIds.get(textId);
+            FeatureHelper featureHelper = new FeatureHelper(text);
+
+            List<ZeroSubjectCandidate> zeroSubjectCandidates = CandidateFinder.findZeroSubjectCandidates(text, sentenceIds);
+            Map<ZeroSubjectCandidate, Instance> candidate2instance = InstanceCreator.extractInstancesFromZeroCandidates(zeroSubjectCandidates, text, featureExtractor);
+
+            for (Map.Entry<ZeroSubjectCandidate, Instance> entry2 : candidate2instance.entrySet()) {
+                boolean good = zeroScorer.isValidCandidate(entry2.getKey(), featureHelper);
+                Instance instance = entry2.getValue();
+                instance.setDataset(instances);
+                instance.setClassValue(good ? 1 : 0);
+                instances.add(instance);
+            }
+        }
+
+        saveInstancesToFile(instances, new File(ModelConstants.ZERO_DATASET_PATH));
+    }
+
+    private static Map<String, Set<String>> loadSentenceIds(String idsPath) throws IOException {
+        Map<String, Set<String>> result = Maps.newHashMap();
+        File[] files = new File(idsPath).listFiles();
+        if (files != null)
+            for (File f : files) {
+                String id = f.getName().split("_")[0];
+                List<String> sentenceIds = IOUtils.readLines(new FileReader(f));
+                result.put(id, Sets.newHashSet(sentenceIds));
+            }
+        return result;
     }
-    private static void saveInstancesToFile(Instances instances) throws IOException {
+    private static void saveInstancesToFile(Instances instances, File targetFile) throws IOException {
         ArffSaver saver = new ArffSaver();
         saver.setInstances(instances);
-        saver.setFile(new File(ModelConstants.SENTENCE_DATASET_PATH));
+        saver.setFile(targetFile);
         saver.writeBatch();
     }
-    private static Map<String, String> loadOptimalSummaries() throws IOException {
+    private static Map<String, String> loadOptimalSummaries(Predicate<String> idFilter) throws IOException {
         Map<String, String> id2optimalSummary = Maps.newHashMap();
-        for (File optimalSummaryFile : new File(OPTIMAL_SUMMARIES_DIR_PATH).listFiles()) {
-            String optimalSummary = Files.toString(optimalSummaryFile, Charsets.UTF_8);
-            id2optimalSummary.put(optimalSummaryFile.getName().split("_")[0], optimalSummary);
-        }
-        LOG.info(id2optimalSummary.size() + " optimal summaries found.");
+        File[] files = new File(OPTIMAL_SUMMARIES_DIR_PATH).listFiles();
+        if (files != null)
+            for (File optimalSummaryFile : files) {
+                String textId = optimalSummaryFile.getName().split("_")[0];
+                if (!idFilter.test(textId))
+                    continue;
+                String optimalSummary = Files.toString(optimalSummaryFile, Charsets.UTF_8);
+                id2optimalSummary.put(textId, optimalSummary);
+            }
+        LOG.info("{} optimal summaries found.", id2optimalSummary.size());
         return id2optimalSummary;
     }
-
-
 }
-package pl.waw.ipipan.zil.summ.nicolas.train.preprocess;
+package pl.waw.ipipan.zil.summ.nicolas.train.pipeline;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -9,19 +9,19 @@ import pl.waw.ipipan.zil.summ.pscapi.xml.Text;
 import java.io.File;
 import java.util.Arrays;
-public class Main {
+public class Preprocess {
-    private static final Logger LOG = LoggerFactory.getLogger(Main.class);
+    private static final Logger LOG = LoggerFactory.getLogger(Preprocess.class);
     private static final String CORPUS_FILE_SUFFIX = ".xml";
     private static final String OUTPUT_FILE_SUFFIX = ".thrift";
-    private Main() {
+    private Preprocess() {
     }
     public static void main(String[] args) {
         if (args.length != 2) {
-            LOG.error("Wrong usage! Try " + Main.class.getSimpleName() + " dirWithCorpusFiles targetDir");
+            LOG.error("Wrong usage! Try " + Preprocess.class.getSimpleName() + " dirWithCorpusFiles targetDir");
             return;
         }
         File corpusDir = new File(args[0]);
-package pl.waw.ipipan.zil.summ.nicolas.train;
+package pl.waw.ipipan.zil.summ.nicolas.train.pipeline;
 import pl.waw.ipipan.zil.summ.nicolas.train.model.mention.TrainMentionModel;
 import pl.waw.ipipan.zil.summ.nicolas.train.model.sentence.TrainSentenceModel;
-package pl.waw.ipipan.zil.summ.nicolas.train.search;
-
-import pl.waw.ipipan.zil.summ.nicolas.train.model.common.ModelConstants;
-
-
-public class CrossvalidateMention {
-
-    private CrossvalidateMention() {
-    }
-
-    public static void main(String[] args) throws Exception {
-        CrossvalidateCommon.crossvalidateClassifiers(ModelConstants.MENTION_DATASET_PATH);
-    }
-}
-package pl.waw.ipipan.zil.summ.nicolas.train.search;
-
-import pl.waw.ipipan.zil.summ.nicolas.train.model.common.ModelConstants;
-
-
-public class CrossvalidateSentence {
-
-    private CrossvalidateSentence() {
-    }
-
-    public static void main(String[] args) throws Exception {
-        CrossvalidateCommon.crossvalidateRegressors(ModelConstants.SENTENCE_DATASET_PATH);
-    }
-}
-package pl.waw.ipipan.zil.summ.nicolas.train.search;
-
-import pl.waw.ipipan.zil.summ.nicolas.train.model.common.ModelConstants;
-
-
-public class CrossvalidateZero {
-
-    private CrossvalidateZero() {
-    }
-
-    public static void main(String[] args) throws Exception {
-        CrossvalidateCommon.crossvalidateClassifiers(ModelConstants.ZERO_DATASET_PATH);
-    }
-}
@@ -18,6 +18,7 @@
         <module>nicolas-train</module>
         <module>nicolas-common</module>
         <module>nicolas-multiservice</module>
+        <module>nicolas-eval</module>
     </modules>
     <properties>
@@ -27,10 +28,11 @@
         <pscapi.version>1.0</pscapi.version>
         <utils.version>1.0</utils.version>
+        <eval.version>1.0</eval.version>
         <commons-csv.version>1.4</commons-csv.version>
         <guava.version>21.0</guava.version>
-        <weka-dev.version>3.9.1</weka-dev.version>
+        <weka-stable.version>3.8.1</weka-stable.version>
         <commons-lang3.version>3.5</commons-lang3.version>
         <commons-io.version>2.5</commons-io.version>
         <slf4j-api.version>1.7.22</slf4j-api.version>
@@ -98,6 +100,11 @@
                 <artifactId>utils</artifactId>
                 <version>${utils.version}</version>
             </dependency>
+            <dependency>
+                <groupId>pl.waw.ipipan.zil.summ</groupId>
+                <artifactId>eval</artifactId>
+                <version>${eval.version}</version>
+            </dependency>
             <!-- third party -->
             <dependency>
@@ -112,8 +119,8 @@
             </dependency>
             <dependency>
                 <groupId>nz.ac.waikato.cms.weka</groupId>
-                <artifactId>weka-dev</artifactId>
-                <version>${weka-dev.version}</version>
+                <artifactId>weka-stable</artifactId>
+                <version>${weka-stable.version}</version>
                 <exclusions>
                     <exclusion>
                         <groupId>org.slf4j</groupId>