From a9aeb64d7019f7d45029f07623107823c7b5e744 Mon Sep 17 00:00:00 2001 From: Dan Date: Wed, 31 Aug 2016 19:42:32 +0800 Subject: [PATCH 1/3] Adding individual configuration run detailed output --- src/java/autoweka/ClassifierRunner.java | 24 +++++ src/java/autoweka/tools/ExperimentRunner.java | 1 + .../classifiers/meta/AutoWEKAClassifier.java | 98 ++++++++++++++++++- 3 files changed, 121 insertions(+), 2 deletions(-) diff --git a/src/java/autoweka/ClassifierRunner.java b/src/java/autoweka/ClassifierRunner.java index 10dc769e..2156fdc7 100644 --- a/src/java/autoweka/ClassifierRunner.java +++ b/src/java/autoweka/ClassifierRunner.java @@ -42,6 +42,7 @@ public class ClassifierRunner private boolean mDisableOutput = false; private java.io.PrintStream mSavedOutput = null; private String mPredictionsFileName = null; + private String mIndividualResultsFileName; /** * Prepares a runner with the specified properties. @@ -58,6 +59,7 @@ public ClassifierRunner(Properties props) mTestOnly = Boolean.valueOf(props.getProperty("onlyTest", "false")); mDisableOutput = Boolean.valueOf(props.getProperty("disableOutput", "false")); mPredictionsFileName = props.getProperty("predictionsFileName", null); + mIndividualResultsFileName = props.getProperty("individualResultsFile", "individual-results.tsv"); } /** @@ -313,6 +315,28 @@ private ClassifierResult _run(String instanceStr, String resultMetric, float tim attribEvalClassName, argMap.get("attributeeval"), attribSearchClassName, argMap.get("attributesearch"), instanceStr, res.getRawScore()); + + try { + FileWriter writer = new FileWriter(mIndividualResultsFileName, true); + StringBuilder builder = new StringBuilder(); + String delim = "\t"; + builder + .append(targetClassifierName).append(delim) + .append(Arrays.toString(argsArraySaved)).append(delim) + .append(attribEvalClassName).append(delim) + .append(argMap.get("attributeeval")).append(delim) + .append(attribSearchClassName).append(delim) + .append(argMap.get("attributesearch")).append(delim) + .append(instanceStr).append(delim) + .append(res.getRawScore()).append(delim) + .append("\n"); + writer.write(builder.toString()); + writer.flush(); + writer.close(); + } + catch(IOException e) { + log.error(e.toString()); + } log.debug("Num Training: {}, num testing: {}", training.numInstances(), testing.numInstances()); return res; diff --git a/src/java/autoweka/tools/ExperimentRunner.java b/src/java/autoweka/tools/ExperimentRunner.java index 49cf3653..ba6d357a 100644 --- a/src/java/autoweka/tools/ExperimentRunner.java +++ b/src/java/autoweka/tools/ExperimentRunner.java @@ -2,6 +2,7 @@ import java.io.File; import java.net.URLDecoder; +import java.util.Arrays; import autoweka.Experiment; import autoweka.TrajectoryParser; diff --git a/src/java/weka/classifiers/meta/AutoWEKAClassifier.java b/src/java/weka/classifiers/meta/AutoWEKAClassifier.java index db97cc40..9c63d137 100644 --- a/src/java/weka/classifiers/meta/AutoWEKAClassifier.java +++ b/src/java/weka/classifiers/meta/AutoWEKAClassifier.java @@ -42,13 +42,14 @@ import java.io.BufferedReader; import java.io.File; +import java.io.FileReader; import java.io.InputStreamReader; import java.io.Serializable; import java.nio.file.Files; - +import java.text.DecimalFormat; import java.net.URLDecoder; - +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Enumeration; @@ -101,6 +102,8 @@ public class AutoWEKAClassifier extends AbstractClassifier implements Additional static final int DEFAULT_MEM_LIMIT = 1024; /** Default */ static final int DEFAULT_N_BEST = 1; + static final int DEFAULT_SHOW_DETAILED_RESULTS = 0; + /** Internal evaluation method. */ static enum Resampling { CrossValidation, @@ -180,6 +183,10 @@ static enum Resampling { private transient weka.gui.Logger wLog; + private int nDetailedResults; + + private String detailedResultString; + /** * Main method for testing this class. * @@ -367,6 +374,42 @@ public void run() { log.info("classifier: {}, arguments: {}, attribute search: {}, attribute search arguments: {}, attribute evaluation: {}, attribute evaluation arguments: {}", classifierClass, classifierArgs, attributeSearchClass, attributeSearchArgs, attributeEvalClass, attributeEvalArgs); + if (nDetailedResults > 0) { + String indvResultsFile = "individual-results.tsv"; // TODO + BufferedReader reader = new BufferedReader(new FileReader(new File(msExperimentPath + expName, indvResultsFile))); + + Map totals = new HashMap<>(); + for (String line = reader.readLine(); line != null; line = reader.readLine()) { + String[] parts = line.split("\t"); + String id = parts[0] + ":" + parts[1] + ", "; + id += parts[2] + ": " + parts[3] + ", "; + id += parts[4] + ": " + parts[5]; + ConfigurationStats stats = totals.get(id); + if (stats == null) { + stats = new ConfigurationStats(id); + totals.put(id, stats); + } + stats.addValue(Double.parseDouble(parts[7])); + } + StringBuilder builder = new StringBuilder(); + builder.append("======= Detailed Results ======").append("\n"); + List sorted = new ArrayList<>(totals.values()); + Collections.sort(sorted); + int remaining = Math.min(nDetailedResults, sorted.size()); + for (ConfigurationStats stats : sorted) { + // TODO: Ignore config stats that do not have enough data to be meaningful + builder.append(stats.toString()).append("\n"); + remaining--; + if (remaining == 0) { + break; + } + } + if (remaining > 0) + builder.append(remaining + " other results truncated"); + builder.append("\n"); + detailedResultString = builder.toString(); + reader.close(); + } //Print log of best configurations if (nBestConfigs>1){ @@ -446,6 +489,9 @@ public Enumeration