Blame view

tools/mate-tools/src/is2/parser/Evaluator.java 2.62 KB
Jan Lupa authored
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
package is2.parser;

import is2.data.SentenceData09;
import is2.io.CONLLReader09;


public class Evaluator {



	public static final String PUNCT ="!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~";

	public static class Results {

		public int total;
		public int corr;
		public float las;
		public float ula;

	}

	public static Results evaluate (String act_file,  String pred_file) throws Exception {

		CONLLReader09 goldReader = new CONLLReader09(act_file, -1);
		CONLLReader09 predictedReader = new CONLLReader09(pred_file, -1);

		int total = 0, corr = 0, corrL = 0;
		int numsent = 0, corrsent = 0, corrsentL = 0;
		SentenceData09 goldInstance = goldReader.getNext();
		SentenceData09 predInstance = predictedReader.getNext();

		while(goldInstance != null) {

			int instanceLength = goldInstance.length();

			if (instanceLength != predInstance.length())
				System.out.println("Lengths do not match on sentence "+numsent);

			int[] goldHeads = goldInstance.heads;
			String[] goldLabels = goldInstance.labels;
			int[] predHeads = predInstance.heads;
			String[] predLabels = predInstance.labels;

			boolean whole = true;
			boolean wholeL = true;

			// NOTE: the first item is the root info added during nextInstance(), so we skip it.

			int punc=0;
			for (int i = 1; i < instanceLength; i++) {
				if (predHeads[i] == goldHeads[i]) {
					corr++;

					if (goldLabels[i].equals(predLabels[i])) corrL++;
					else {
				//		System.out.println(numsent+" error gold "+goldLabels[i]+" "+predLabels[i]+" head "+goldHeads[i]+" child "+i);
						wholeL = false;
					}
				}
				else { 
			//		System.out.println(numsent+"error gold "+goldLabels[i]+" "+predLabels[i]+" head "+goldHeads[i]+" child "+i);
					whole = false; wholeL = false; 
				}
			}
			total += ((instanceLength - 1) - punc); // Subtract one to not score fake root token

			if(whole) corrsent++;
			if(wholeL) corrsentL++;
			numsent++;

			goldInstance = goldReader.getNext();
			predInstance = predictedReader.getNext();
		}

		Results r = new Results();

		r.total = total;
		r.corr = corr;
		r.las =(float)Math.round(((double)corrL/total)*100000)/1000;
		r.ula =(float)Math.round(((double)corr /total)*100000)/1000;
		System.out.print("Total: " + total+" \tCorrect: " + corr+" ");
		System.out.println("LAS: " + (double)Math.round(((double)corrL/total)*100000)/1000+" \tTotal: " + (double)Math.round(((double)corrsentL/numsent)*100000)/1000+
				" \tULA: " + (double)Math.round(((double)corr /total)*100000)/1000+" \tTotal: " + (double)Math.round(((double)corrsent /numsent)*100000)/1000);

		return r;
	}


	public static float round (double v){

		return Math.round(v*10000F)/10000F;
	}

}