calculating mean and sd scores

Mateusz Kopeć
1 parent 2692e607
Showing 5 changed files with 117 additions and 9 deletions
pom.xml
src/main/java/pl/waw/ipipan/zil/summ/eval/Main.java
src/main/java/pl/waw/ipipan/zil/summ/eval/Result.java
src/main/java/pl/waw/ipipan/zil/summ/eval/ResultAccumulator.java
src/main/resources/log4j2.xml
@@ -34,8 +34,13 @@
 			<artifactId>guava</artifactId>
 			<version>18.0</version>
 		</dependency>
+		<dependency>
+			<groupId>org.apache.commons</groupId>
+			<artifactId>commons-math3</artifactId>
+			<version>3.5</version>
+		</dependency>
 	</dependencies>
-	
+
 	<build>
 		<plugins>
 			<plugin>
@@ -16,8 +16,11 @@ import org.apache.logging.log4j.Logger;
 import pl.waw.ipipan.zil.summ.eval.rouge.RougeN;
  
 public class Main {
+
 	private static final Logger LOG = LogManager.getLogger(Main.class);
  
+	private static final int[] ROUGE_N = new int[] { 1, 2, 3 };
+
 	public static void main(String[] args) {
 		if (args.length != 2) {
 			LOG.error("Wrong number of arguments!");
@@ -35,31 +38,57 @@ public class Main {
 		Map<String, Set<File>> id2sysSummaryFiles = loadFiles(sysDir);
 		if (!id2goldSummaryFiles.keySet().equals(id2sysSummaryFiles.keySet())) {
 			LOG.error("Different text ids in gold and sys dirs!");
+			LOG.error(id2goldSummaryFiles.size() + " gold ids, " + id2sysSummaryFiles.size() + " sys ids.");
 			return;
 		}
 		LOG.info(id2goldSummaryFiles.size() + " gold and sys text(s) loaded.");
  
+		ResultAccumulator ra = new ResultAccumulator();
+		int totalGoldSummaries = 0;
+		int totalComparedTexts = 0;
+
 		for (String id : id2goldSummaryFiles.keySet()) {
 			Set<File> goldSummaryFiles = id2goldSummaryFiles.get(id);
 			Set<File> sysSummaryFiles = id2sysSummaryFiles.get(id);
  
+			Set<String> goldSummaries = goldSummaryFiles.stream().map(f -> loadSummaryFromFile(f))
+					.collect(Collectors.toSet());
+			if (goldSummaries.contains(null)) {
+				LOG.error("Empty gold summary for id: " + id);
+				return;
+			}
 			for (File sysSummaryFile : sysSummaryFiles) {
 				String systemSummary = loadSummaryFromFile(sysSummaryFile);
 				String systemId = getSystemIdFromFilename(sysSummaryFile);
-				Set<String> goldSummaries = goldSummaryFiles.stream().map(f -> loadSummaryFromFile(f))
-						.collect(Collectors.toSet());
-				if (goldSummaries.contains(null) || systemSummary == null) {
+				if (systemSummary == null) {
+					LOG.error("Empty system summary for id: " + id);
 					return;
 				}
-
-				for (int i = 1; i < 4; i++) {
+				for (int i : ROUGE_N) {
 					double score = RougeN.score(i, systemSummary, goldSummaries);
-					System.out.println(i + "\t" + systemId + "\t" + score);
-					System.out.println();
+					ra.logScore("ROUGE_" + i, systemId, score);
 				}
 			}
+
+			totalGoldSummaries += goldSummaries.size();
+			totalComparedTexts++;
 		}
  
+		LOG.info(totalComparedTexts + " - total compared texts.");
+		LOG.info(totalGoldSummaries * 1.0 / totalComparedTexts + " - average gold summaries per text.");
+
+		printResults(ra);
+	}
+
+	private static void printResults(ResultAccumulator ra) {
+		for (String systemId : ra.getSystemIds()) {
+			System.out.println("############## " + systemId);
+			for (String metricId : ra.getMetricIds()) {
+				Result r = ra.getSystemResult(systemId, metricId);
+				System.out.println(String.format("\t%s\t%.2f\t(+/- %.2f)", metricId, r.getMean(), r.getSD()));
+			}
+			System.out.println();
+		}
 	}
  
 	private static String getSystemIdFromFilename(File sysSummaryFile) {
+package pl.waw.ipipan.zil.summ.eval;
+
+public class Result {
+
+	private double sd;
+	private double mean;
+
+	public Result(double mean, double sd) {
+		this.mean = mean;
+		this.sd = sd;
+	}
+
+	public double getMean() {
+		return mean;
+	}
+
+	public double getSD() {
+		return sd;
+	}
+
+}
+package pl.waw.ipipan.zil.summ.eval;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.commons.math3.stat.descriptive.SummaryStatistics;
+
+public class ResultAccumulator {
+
+	private Map<String, CumulativeResult> systemId2result = new HashMap<>();
+	private Set<String> metricIds = new HashSet<>();
+
+	public void logScore(String metricId, String systemId, double score) {
+		systemId2result.putIfAbsent(systemId, new CumulativeResult());
+		CumulativeResult cr = systemId2result.get(systemId);
+		cr.addScore(metricId, score);
+		metricIds.add(metricId);
+	}
+
+	public List<String> getSystemIds() {
+		return systemId2result.keySet().stream().sorted().collect(Collectors.toList());
+	}
+
+	public List<String> getMetricIds() {
+		return metricIds.stream().sorted().collect(Collectors.toList());
+	}
+
+	public Result getSystemResult(String systemId, String metricId) {
+		return systemId2result.get(systemId).getResult(metricId);
+	}
+
+	private class CumulativeResult {
+		Map<String, SummaryStatistics> metricId2scores = new HashMap<>();
+
+		public void addScore(String metricId, double score) {
+			metricId2scores.putIfAbsent(metricId, new SummaryStatistics());
+			metricId2scores.get(metricId).addValue(score);
+		}
+
+		public Result getResult(String metricId) {
+			SummaryStatistics scores = metricId2scores.get(metricId);
+			return new Result(scores.getMean(), scores.getStandardDeviation());
+		}
+	}
+}
@@ -2,12 +2,16 @@
 <Configuration status="WARN">
 	<Appenders>
 		<Console name="Console" target="SYSTEM_OUT">
-			<PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{36} - %msg%n" />
+			<PatternLayout pattern="%d{HH:mm:ss.SSS} [%t] %-5level %logger{3} - %msg%n" />
 		</Console>
 	</Appenders>
 	<Loggers>
 		<Root level="error">
 			<AppenderRef ref="Console" />
 		</Root>
+		<Logger name="pl.waw.ipipan.zil.summ.eval" level="debug"
+			additivity="false">
+			<AppenderRef ref="Console" />
+		</Logger>
 	</Loggers>
 </Configuration>
 \ No newline at end of file