-
Notifications
You must be signed in to change notification settings - Fork 12
Expand file tree
/
Copy pathRunDriver.java
More file actions
139 lines (131 loc) · 5.34 KB
/
Copy pathRunDriver.java
File metadata and controls
139 lines (131 loc) · 5.34 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
package run;
import java.util.*;
import structure.Problem;
import utils.Folds;
import utils.Params;
import edu.illinois.cs.cogcomp.core.datastructures.Pair;
import edu.illinois.cs.cogcomp.sl.core.AbstractFeatureGenerator;
import edu.illinois.cs.cogcomp.sl.core.SLModel;
import edu.illinois.cs.cogcomp.sl.core.SLParameters;
import edu.illinois.cs.cogcomp.sl.core.SLProblem;
import edu.illinois.cs.cogcomp.sl.learner.Learner;
import edu.illinois.cs.cogcomp.sl.learner.LearnerFactory;
import edu.illinois.cs.cogcomp.sl.util.Lexiconer;
public class RunDriver {
public static void crossVal(List<Problem> probs, List<List<Integer>> foldIndices)
throws Exception {
double acc1 = 0.0, acc2 = 0.0;
for(int i=0;i<foldIndices.size(); i++) {
List<Integer> train = new ArrayList<>();
List<Integer> test = new ArrayList<>();
for(int j=0; j<foldIndices.size(); ++j) {
if(i==j) test.addAll(foldIndices.get(j));
else train.addAll(foldIndices.get(j));
}
Pair<Double, Double> pair = doTrainTest(probs, train, test, i);
acc1 += pair.getFirst();
acc2 += pair.getSecond();
}
System.out.println("CV : " + (acc1/foldIndices.size()) + " " + (acc2/foldIndices.size()));
}
public static Pair<Double, Double> doTrainTest(List<Problem> probs, List<Integer> trainIndices,
List<Integer> testIndices, int id) throws Exception {
List<List<Problem>> split = Folds.getDataSplit(probs, trainIndices, testIndices, 0.0);
List<Problem> trainProbs = split.get(0);
List<Problem> testProbs = split.get(2);
SLProblem train = Annotations.getSP(trainProbs);
SLProblem test = Annotations.getSP(testProbs);
System.out.println("Train : "+train.instanceList.size()+" Test : "+test.instanceList.size());
trainModel(Params.modelDir+Params.runPrefix+id+Params.modelSuffix, train);
return testModel(Params.modelDir+Params.runPrefix+id+Params.modelSuffix, test);
}
public static Pair<Double, Double> testModel(String modelPath, SLProblem sp)
throws Exception {
SLModel model = SLModel.loadModel(modelPath);
Set<Integer> incorrect = new HashSet<>();
Set<Integer> total = new HashSet<>();
double acc = 0.0;
int countNoRel = 0;
Map<Pair<String, String>, Integer> counts = new HashMap<>();
for (int i = 0; i < sp.instanceList.size(); i++) {
RunX prob = (RunX) sp.instanceList.get(i);
RunY gold = (RunY) sp.goldStructureList.get(i);
RunY pred = (RunY) model.infSolver.getBestStructure(model.wv, prob);
if(pred.label.equalsIgnoreCase("NO_REL")) countNoRel++;
total.add(prob.problemId);
if(!counts.containsKey(new Pair<>(gold.label, pred.label))) {
counts.put(new Pair<>(gold.label, pred.label), 1);
} else {
counts.put(new Pair<>(gold.label, pred.label),
counts.get(new Pair<>(gold.label, pred.label))+1);
}
boolean correct = false;
if(RunY.getLoss(gold, pred) < 0.0001) {
acc += 1;
correct = true;
} else {
incorrect.add(prob.problemId);
}
if((correct && Params.printCorrect) ||
(!correct && Params.printMistakes)){
System.out.println(prob.problemId+" : "+prob.ta.getText());
System.out.println();
System.out.println("Schema : "+prob.schema);
System.out.println();
System.out.println("Quantities : "+prob.quantities);
System.out.println("Quant of Interest: "+prob.quantIndex1+" "+prob.quantIndex2);
System.out.println("Gold : "+gold);
System.out.println("Pred : "+pred);
System.out.println("Loss : "+RunY.getLoss(gold, pred));
System.out.println("Labels : "+Arrays.asList(getLabelsWithScores(prob, model)));
System.out.println();
}
}
System.out.println("Accuracy : = " + acc + " / " + sp.instanceList.size()
+ " = " + (acc/sp.instanceList.size()));
for(Pair<String, String> key : counts.keySet()) {
double tot = 0, count = 0;
for(Pair<String, String> key1 : counts.keySet()) {
if(key1.getFirst().equals(key.getFirst())) {
tot += counts.get(key1);
if(key1.getFirst().equals(key1.getSecond())) {
count += counts.get(key1);
}
}
}
System.out.println(key.getFirst()+" : "+count+" "+tot+" "+(count/tot));
}
System.out.println("Strict Accuracy : ="+ (1-1.0*incorrect.size()/total.size()));
System.out.println("NoRel : ="+ countNoRel);
return new Pair<>(acc/sp.instanceList.size(), 1-1.0*incorrect.size()/total.size());
}
public static void trainModel(String modelPath, SLProblem train)
throws Exception {
SLModel model = new SLModel();
Lexiconer lm = new Lexiconer();
lm.setAllowNewFeatures(true);
model.lm = lm;
AbstractFeatureGenerator fg = new RunFeatGen(lm);
model.featureGenerator = fg;
model.infSolver = new RunInfSolver(fg);
SLParameters para = new SLParameters();
para.loadConfigFile(Params.spConfigFile);
para.MAX_NUM_ITER = 5;
Learner learner = LearnerFactory.getLearner(model.infSolver, fg, para);
model.wv = learner.train(train);
lm.setAllowNewFeatures(false);
model.saveModel(modelPath);
}
public static Map<String, Double> getLabelsWithScores(RunX prob, SLModel model) {
List<String> labels = Arrays.asList("SAME_UNIT", "1_RATE_1", "1_RATE_2",
"2_RATE_1", "2_RATE_2", "NO_REL");
Map<String, Double> labelsWithScores = new HashMap<String, Double>();
for(String label : labels) {
labelsWithScores.put(prob.quantIndex1+"_"+prob.quantIndex2+"_"+label,
1.0*model.wv.dotProduct(model.featureGenerator.getFeatureVector(
prob,
new RunY(label))));
}
return labelsWithScores;
}
}