Evaluator.java
2.73 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
package is2.parser;
import is2.data.SentenceData09;
import is2.io.CONLLReader09;
public class Evaluator {
public static final String PUNCT = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~";
public static class Results {
public int total;
public int corr;
public float las;
public float ula;
}
public static Results evaluate(String act_file, String pred_file) throws Exception {
CONLLReader09 goldReader = new CONLLReader09(act_file, -1);
CONLLReader09 predictedReader = new CONLLReader09(pred_file, -1);
int total = 0, corr = 0, corrL = 0;
int numsent = 0, corrsent = 0, corrsentL = 0;
SentenceData09 goldInstance = goldReader.getNext();
SentenceData09 predInstance = predictedReader.getNext();
while (goldInstance != null) {
int instanceLength = goldInstance.length();
if (instanceLength != predInstance.length())
System.out.println("Lengths do not match on sentence " + numsent);
int[] goldHeads = goldInstance.heads;
String[] goldLabels = goldInstance.labels;
int[] predHeads = predInstance.heads;
String[] predLabels = predInstance.labels;
boolean whole = true;
boolean wholeL = true;
// NOTE: the first item is the root info added during
// nextInstance(), so we skip it.
int punc = 0;
for (int i = 1; i < instanceLength; i++) {
if (predHeads[i] == goldHeads[i]) {
corr++;
if (goldLabels[i].equals(predLabels[i]))
corrL++;
else {
// System.out.println(numsent+" error gold
// "+goldLabels[i]+" "+predLabels[i]+" head
// "+goldHeads[i]+" child "+i);
wholeL = false;
}
} else {
// System.out.println(numsent+"error gold "+goldLabels[i]+"
// "+predLabels[i]+" head "+goldHeads[i]+" child "+i);
whole = false;
wholeL = false;
}
}
total += ((instanceLength - 1) - punc); // Subtract one to not score
// fake root token
if (whole)
corrsent++;
if (wholeL)
corrsentL++;
numsent++;
goldInstance = goldReader.getNext();
predInstance = predictedReader.getNext();
}
Results r = new Results();
r.total = total;
r.corr = corr;
r.las = (float) Math.round(((double) corrL / total) * 100000) / 1000;
r.ula = (float) Math.round(((double) corr / total) * 100000) / 1000;
System.out.print("Total: " + total + " \tCorrect: " + corr + " ");
System.out.println("LAS: " + (double) Math.round(((double) corrL / total) * 100000) / 1000 + " \tTotal: "
+ (double) Math.round(((double) corrsentL / numsent) * 100000) / 1000 + " \tULA: "
+ (double) Math.round(((double) corr / total) * 100000) / 1000 + " \tTotal: "
+ (double) Math.round(((double) corrsent / numsent) * 100000) / 1000);
return r;
}
public static float round(double v) {
return Math.round(v * 10000F) / 10000F;
}
}