Commit f72059fc9c0a8846ff28f003bb2642c4d95d645c
1 parent
c6d4dcbe
small improvements
Showing
4 changed files
with
69 additions
and
8 deletions
pom.xml
... | ... | @@ -57,6 +57,29 @@ |
57 | 57 | <target>${java.version.build}</target> |
58 | 58 | </configuration> |
59 | 59 | </plugin> |
60 | + <plugin> | |
61 | + <groupId>org.dstovall</groupId> | |
62 | + <artifactId>onejar-maven-plugin</artifactId> | |
63 | + <version>1.4.4</version> | |
64 | + <executions> | |
65 | + <execution> | |
66 | + <configuration> | |
67 | + <mainClass>pl.waw.ipipan.zil.summ.eval.Main</mainClass> | |
68 | + </configuration> | |
69 | + <goals> | |
70 | + <goal>one-jar</goal> | |
71 | + </goals> | |
72 | + </execution> | |
73 | + </executions> | |
74 | + </plugin> | |
60 | 75 | </plugins> |
61 | 76 | </build> |
77 | + | |
78 | + <pluginRepositories> | |
79 | + <pluginRepository> | |
80 | + <id>onejar-maven-plugin.googlecode.com</id> | |
81 | + <url>http://onejar-maven-plugin.googlecode.com/svn/mavenrepo</url> | |
82 | + </pluginRepository> | |
83 | + </pluginRepositories> | |
84 | + | |
62 | 85 | </project> |
63 | 86 | \ No newline at end of file |
... | ... |
src/main/java/pl/waw/ipipan/zil/summ/eval/Main.java
... | ... | @@ -19,11 +19,13 @@ import pl.waw.ipipan.zil.summ.eval.rouge.RougeN; |
19 | 19 | public class Main { |
20 | 20 | |
21 | 21 | private static final boolean LATEX = false; |
22 | + private static final File outputFile = null; | |
23 | + | |
22 | 24 | private static final double P_VALUE = 0.05; |
23 | 25 | |
24 | 26 | private static final Logger LOG = LogManager.getLogger(Main.class); |
25 | 27 | |
26 | - private static final int[] ROUGE_N = new int[] { 1, 2, 3 }; | |
28 | + public static final int[] ROUGE_N = new int[] { 1, 2, 3 }; | |
27 | 29 | |
28 | 30 | public static void main(String[] args) { |
29 | 31 | if (args.length != 2) { |
... | ... | @@ -58,6 +60,13 @@ public class Main { |
58 | 60 | OutputHelper.checkNormality(ra, P_VALUE); |
59 | 61 | OutputHelper.printResults(ra, LATEX); |
60 | 62 | OutputHelper.printSignificance(ra, P_VALUE, LATEX); |
63 | + | |
64 | + if (outputFile != null) | |
65 | + try { | |
66 | + OutputHelper.writeAllResults(ra, outputFile); | |
67 | + } catch (IOException e) { | |
68 | + LOG.error("Error writing to file:" + e); | |
69 | + } | |
61 | 70 | } |
62 | 71 | |
63 | 72 | private static ResultAccumulator countResults(Map<String, Set<File>> id2goldSummaryFiles, |
... | ... | @@ -100,7 +109,7 @@ public class Main { |
100 | 109 | return name.substring(name.lastIndexOf("_") + 1, name.lastIndexOf(".")); |
101 | 110 | } |
102 | 111 | |
103 | - private static String loadSummaryFromFile(File sysSummaryFile) { | |
112 | + public static String loadSummaryFromFile(File sysSummaryFile) { | |
104 | 113 | try { |
105 | 114 | return new String(Files.readAllBytes(Paths.get(sysSummaryFile.getPath()))); |
106 | 115 | } catch (IOException e) { |
... | ... | @@ -109,7 +118,7 @@ public class Main { |
109 | 118 | } |
110 | 119 | } |
111 | 120 | |
112 | - private static Map<String, Set<File>> loadFiles(File goldDir) { | |
121 | + public static Map<String, Set<File>> loadFiles(File goldDir) { | |
113 | 122 | Map<String, Set<File>> result = new HashMap<>(); |
114 | 123 | for (File goldFile : goldDir.listFiles()) { |
115 | 124 | String[] spl = goldFile.getName().split("_"); |
... | ... |
src/main/java/pl/waw/ipipan/zil/summ/eval/output/OutputHelper.java
1 | 1 | package pl.waw.ipipan.zil.summ.eval.output; |
2 | 2 | |
3 | +import java.io.BufferedWriter; | |
4 | +import java.io.File; | |
5 | +import java.io.FileWriter; | |
6 | +import java.io.IOException; | |
3 | 7 | import java.util.HashMap; |
4 | 8 | import java.util.List; |
5 | 9 | import java.util.Map; |
10 | +import java.util.Map.Entry; | |
6 | 11 | import java.util.Set; |
7 | 12 | import java.util.TreeSet; |
8 | 13 | import java.util.stream.Collectors; |
... | ... | @@ -60,9 +65,10 @@ public class OutputHelper { |
60 | 65 | System.out.print(systemId); |
61 | 66 | for (String metricId : ra.getMetricIds()) { |
62 | 67 | SummaryStatistics r = ra.getSystemSummaryStatistics(systemId, metricId); |
63 | - | |
64 | - System.out.print(String.format("\t" + (latex ? "&" : "") + " %.3f (%.3f)", r.getMean(), | |
65 | - r.getStandardDeviation())); | |
68 | + boolean best = latex && isBestSystem(systemId, metricId, ra); | |
69 | + System.out.print( | |
70 | + String.format("\t" + (latex ? "& " : " ") + (best ? "\\textbf{%.3f}" : "%.3f") + " (%.3f)", | |
71 | + r.getMean(), r.getStandardDeviation())); | |
66 | 72 | } |
67 | 73 | if (latex) |
68 | 74 | System.out.print("\\\\\\hline"); |
... | ... | @@ -72,6 +78,16 @@ public class OutputHelper { |
72 | 78 | System.out.println("\\end{tabular}"); |
73 | 79 | } |
74 | 80 | |
81 | + private static boolean isBestSystem(String systemId, String metricId, ResultAccumulator ra) { | |
82 | + double systemScore = ra.getSystemSummaryStatistics(systemId, metricId).getMean(); | |
83 | + for (String systemId2 : ra.getSystemIds()) { | |
84 | + double score2 = ra.getSystemSummaryStatistics(systemId2, metricId).getMean(); | |
85 | + if (score2 > systemScore) | |
86 | + return false; | |
87 | + } | |
88 | + return true; | |
89 | + } | |
90 | + | |
75 | 91 | public static void printSignificance(ResultAccumulator ra, double pValue, boolean latex) { |
76 | 92 | if (ra.getSystemResults(ra.getSystemIds().iterator().next(), ra.getMetricIds().iterator().next()).values() |
77 | 93 | .size() < 2) { |
... | ... | @@ -131,4 +147,17 @@ public class OutputHelper { |
131 | 147 | return signifWorseSystems; |
132 | 148 | } |
133 | 149 | |
150 | + public static void writeAllResults(ResultAccumulator ra, File file) throws IOException { | |
151 | + | |
152 | + try (BufferedWriter bw = new BufferedWriter(new FileWriter(file))) { | |
153 | + for (String metricId : ra.getMetricIds()) | |
154 | + for (String systemId : ra.getSystemIds()) | |
155 | + for (Entry<String, Double> e : ra.getSystemResults(systemId, metricId).entrySet()) { | |
156 | + String textId = e.getKey(); | |
157 | + Double score = e.getValue(); | |
158 | + bw.write(textId + "\t" + systemId + "\t" + metricId + "\t" + score + "\n"); | |
159 | + } | |
160 | + } | |
161 | + } | |
162 | + | |
134 | 163 | } |
... | ... |
src/main/java/pl/waw/ipipan/zil/summ/eval/rouge/RougeN.java
... | ... | @@ -59,7 +59,7 @@ public class RougeN { |
59 | 59 | return maxScore; |
60 | 60 | } |
61 | 61 | |
62 | - protected static void countNgrams(Multiset<List<String>> ngrams, int n, String text) { | |
62 | + public static void countNgrams(Multiset<List<String>> ngrams, int n, String text) { | |
63 | 63 | List<String> tokens = tokenize(text).stream().map(String::toLowerCase).collect(Collectors.toList()); |
64 | 64 | NgramQueue<String> ngram = new NgramQueue<>(n); |
65 | 65 | boolean ngramFull = false; |
... | ... | @@ -74,7 +74,7 @@ public class RougeN { |
74 | 74 | } |
75 | 75 | } |
76 | 76 | |
77 | - protected static List<String> tokenize(String text) { | |
77 | + public static List<String> tokenize(String text) { | |
78 | 78 | return Arrays.asList(text.split("[^\\p{L}0-9]+")); |
79 | 79 | } |
80 | 80 | |
... | ... |