Merge branch 'master' of https://github.com/biometrics/openbr

Scott Klum
2 parents 4936fcc1 ff04e9ed
Showing 21 changed files with 564 additions and 495 deletions
.gitignore
app/br/br.cpp
data/KTH/README.md
data/README.md
openbr/core/bee.cpp
openbr/core/classify.cpp
openbr/core/cluster.cpp
openbr/core/eval.cpp
openbr/core/classify.h → openbr/core/eval.h
openbr/core/plot.cpp
openbr/core/plot.h
openbr/openbr.cpp
openbr/openbr.h
openbr/openbr_plugin.cpp
openbr/openbr_plugin.h
openbr/plugins/eigen3.cpp
openbr/plugins/gallery.cpp
openbr/plugins/independent.cpp
openbr/plugins/output.cpp
openbr/plugins/svm.cpp
@@ -29,3 +29,6 @@ scripts/results
 ### OS X ###
 *.DS_Store
+
+### vim ###
+*.swp
@@ -140,13 +140,6 @@ public:
             } else if (!strcmp(fun, "evalRegression")) {
                 check(parc == 2, "Incorrect parameter count for 'evalRegression'.");
                 br_eval_regression(parv[0], parv[1]);
-            } else if (!strcmp(fun, "confusion")) {
-                check(parc == 2, "Incorrect parameter count for 'confusion'.");
-                int true_positives, false_positives, true_negatives, false_negatives;
-                br_confusion(parv[0], atof(parv[1]),
-                        &true_positives, &false_positives, &true_negatives, &false_negatives);
-                printf("True Positives = %d\nFalse Positives = %d\nTrue Negatives = %d\nFalseNegatives = %d\n",
-                       true_positives, false_positives, true_negatives, false_negatives);
             } else if (!strcmp(fun, "plotMetadata")) {
                 check(parc >= 2, "Incorrect parameter count for 'plotMetadata'.");
                 br_plot_metadata(parc-1, parv, parv[parc-1], true);
@@ -223,7 +216,6 @@ private:
                "-evalClustering <clusters> <gallery>\n"
                "-evalDetection <predicted_gallery> <truth_gallery>\n"
                "-evalRegression <predicted_gallery> <truth_gallery>\n"
-               "-confusion <file> <score>\n"
                "-plotMetadata <file> ... <file> <columns>\n"
                "-getHeader <matrix>\n"
                "-setHeader {<matrix>} <target_gallery> <query_gallery>\n"
+## KTH Human Action Database
+Grayscale human action videos. Six actions performed by 25 subjects in four scenarios, for a total of 600 160x120 videos.
+* [Website](http://www.nada.kth.se/cvap/actions/)
@@ -9,6 +9,7 @@
 * [MEDS](MEDS/README.md)
 * [MNIST](MNIST/README.md)
 * [PCSO](PCSO/README.md)
+* [KTH](KTH/README.md)
 For both practical and legal reasons we only include images for some of the datasets in this repository.
 Researchers should contact the respective owners of the other datasets in order to obtain a copy.
@@ -268,8 +268,8 @@ cv::Mat BEE::makeMask(const br::FileList &amp;targets, const br::FileList &amp;queries, 
 {
     // Would like to use indexProperty for this, but didn't make a version of that for Filelist yet
     // -cao
-    QList<QString> targetLabels = targets.get<QString>("Subject", "-1");
-    QList<QString> queryLabels = queries.get<QString>("Subject", "-1");
+    QList<QString> targetLabels = File::get<QString>(targets, "Subject", "-1");
+    QList<QString> queryLabels = File::get<QString>(queries, "Subject", "-1");
     QList<int> targetPartitions = targets.crossValidationPartitions();
     QList<int> queryPartitions = queries.crossValidationPartitions();
-/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
- * Copyright 2012 The MITRE Corporation                                      *
- *                                                                           *
- * Licensed under the Apache License, Version 2.0 (the "License");           *
- * you may not use this file except in compliance with the License.          *
- * You may obtain a copy of the License at                                   *
- *                                                                           *
- *     http://www.apache.org/licenses/LICENSE-2.0                            *
- *                                                                           *
- * Unless required by applicable law or agreed to in writing, software       *
- * distributed under the License is distributed on an "AS IS" BASIS,         *
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  *
- * See the License for the specific language governing permissions and       *
- * limitations under the License.                                            *
- * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-
-#include <openbr/openbr_plugin.h>
-
-#include "classify.h"
-#include "openbr/core/qtutils.h"
-
-// Helper struct for statistics accumulation
-struct Counter
-{
-    float truePositive, falsePositive, falseNegative;
-    Counter()
-    {
-        truePositive = 0;
-        falsePositive = 0;
-        falseNegative = 0;
-    }
-};
-
-void br::EvalClassification(const QString &predictedInput, const QString &truthInput)
-{
-    qDebug("Evaluating classification of %s against %s", qPrintable(predictedInput), qPrintable(truthInput));
-
-    TemplateList predicted(TemplateList::fromGallery(predictedInput));
-    TemplateList truth(TemplateList::fromGallery(truthInput));
-    if (predicted.size() != truth.size()) qFatal("Input size mismatch.");
-
-    QHash<QString, Counter> counters;
-    for (int i=0; i<predicted.size(); i++) {
-        if (predicted[i].file.name != truth[i].file.name)
-            qFatal("Input order mismatch.");
-
-        // Typically these lists will be of length one, but this generalization allows measuring multi-class labeling accuracy.
-        QString predictedSubject = predicted[i].file.get<QString>("Subject");
-        QString trueSubject = truth[i].file.get<QString>("Subject");
-
-        QStringList predictedSubjects(predictedSubject);
-        QStringList trueSubjects(trueSubject);
-
-        foreach (const QString &subject, trueSubjects.toVector() /* Hack to copy the list. */) {
-            if (predictedSubjects.contains(subject)) {
-                counters[subject].truePositive++;
-                trueSubjects.removeOne(subject);
-                predictedSubjects.removeOne(subject);
-            } else {
-                counters[subject].falseNegative++;
-            }
-        }
-
-        for (int i=0; i<trueSubjects.size(); i++)
-            foreach (const QString &subject, predictedSubjects)
-                counters[subject].falsePositive += 1.f / predictedSubjects.size();
-    }
-
-    const QStringList keys = counters.keys();
-    QSharedPointer<Output> output(Output::make("", FileList() << "Count" << "Precision" << "Recall" << "F-score", FileList(keys)));
-
-    int tpc = 0;
-    int fnc = 0;
-
-    for (int i=0; i<counters.size(); i++) {
-        const QString &subject = keys[i];
-        const Counter &counter = counters[subject];
-        tpc += counter.truePositive;
-        fnc += counter.falseNegative;
-        const int count = counter.truePositive + counter.falseNegative;
-        const float precision = counter.truePositive / (float)(counter.truePositive + counter.falsePositive);
-        const float recall = counter.truePositive / (float)(counter.truePositive + counter.falseNegative);
-        const float fscore = 2 * precision * recall / (precision + recall);
-        output->setRelative(count, i, 0);
-        output->setRelative(precision, i, 1);
-        output->setRelative(recall, i, 2);
-        output->setRelative(fscore, i, 3);
-    }
-
-    qDebug("Overall Accuracy = %f", (float)tpc / (float)(tpc + fnc));
-}
-
-void br::EvalDetection(const QString &predictedInput, const QString &truthInput)
-{
-    (void) predictedInput;
-    (void) truthInput;
-}
-
-void br::EvalRegression(const QString &predictedInput, const QString &truthInput)
-{
-    qDebug("Evaluating regression of %s against %s", qPrintable(predictedInput), qPrintable(truthInput));
-
-    const TemplateList predicted(TemplateList::fromGallery(predictedInput));
-    const TemplateList truth(TemplateList::fromGallery(truthInput));
-    if (predicted.size() != truth.size()) qFatal("Input size mismatch.");
-
-    float rmsError = 0;
-    QStringList truthValues, predictedValues;
-    for (int i=0; i<predicted.size(); i++) {
-        if (predicted[i].file.name != truth[i].file.name)
-            qFatal("Input order mismatch.");
-        rmsError += pow(predicted[i].file.get<float>("Subject")-truth[i].file.get<float>("Subject"), 2.f);
-        truthValues.append(QString::number(truth[i].file.get<float>("Subject")));
-        predictedValues.append(QString::number(predicted[i].file.get<float>("Subject")));
-    }
-
-    QStringList rSource;
-    rSource << "# Load libraries" << "library(ggplot2)" << "" << "# Set Data"
-            << "Actual <- c(" + truthValues.join(",") + ")"
-            << "Predicted <- c(" + predictedValues.join(",") + ")"
-            << "data <- data.frame(Actual, Predicted)"
-            << "" << "# Construct Plot" << "pdf(\"EvalRegression.pdf\")"
-            << "print(qplot(Actual, Predicted, data=data, geom=\"jitter\", alpha=I(2/3)) + geom_abline(intercept=0, slope=1, color=\"forestgreen\", size=I(1)) + geom_smooth(size=I(1), color=\"mediumblue\") + theme_bw())"
-            << "print(qplot(Actual, Predicted-Actual, data=data, geom=\"jitter\", alpha=I(2/3)) + geom_abline(intercept=0, slope=0, color=\"forestgreen\", size=I(1)) + geom_smooth(size=I(1), color=\"mediumblue\") + theme_bw())"
-            << "dev.off()";
-
-
-    QString rFile = "EvalRegression.R";
-    QtUtils::writeFile(rFile, rSource);
-    bool success = QtUtils::runRScript(rFile);
-    if (success) QtUtils::showFile("EvalRegression.pdf");
-
-    qDebug("RMS Error = %f", sqrt(rmsError/predicted.size()));
-}
@@ -280,7 +280,7 @@ void br::EvalClustering(const QString &amp;csv, const QString &amp;input)
     // We assume clustering algorithms store assigned cluster labels as integers (since the clusters are
     // not named).
-    QList<int> labels = TemplateList::fromGallery(input).files().get<int>("Subject");
+    QList<int> labels = File::get<int>(TemplateList::fromGallery(input), "Subject");
     QHash<int, int> labelToIndex;
     int nClusters = 0;
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright 2012 The MITRE Corporation                                      *
+ *                                                                           *
+ * Licensed under the Apache License, Version 2.0 (the "License");           *
+ * you may not use this file except in compliance with the License.          *
+ * You may obtain a copy of the License at                                   *
+ *                                                                           *
+ *     http://www.apache.org/licenses/LICENSE-2.0                            *
+ *                                                                           *
+ * Unless required by applicable law or agreed to in writing, software       *
+ * distributed under the License is distributed on an "AS IS" BASIS,         *
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  *
+ * See the License for the specific language governing permissions and       *
+ * limitations under the License.                                            *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+#include "bee.h"
+#include "eval.h"
+#include "openbr/core/qtutils.h"
+
+using namespace cv;
+
+namespace br
+{
+
+struct Comparison
+{
+    float score;
+    int target, query;
+    bool genuine;
+
+    Comparison() {}
+    Comparison(float _score, int _target, int _query, bool _genuine)
+        : score(_score), target(_target), query(_query), genuine(_genuine) {}
+    inline bool operator<(const Comparison &other) const { return score > other.score; }
+};
+
+#undef FAR // Windows preprecessor definition conflicts with variable name
+struct OperatingPoint
+{
+    float score, FAR, TAR;
+    OperatingPoint() {}
+    OperatingPoint(float _score, float _FAR, float _TAR)
+        : score(_score), FAR(_FAR), TAR(_TAR) {}
+};
+
+static float getTAR(const QList<OperatingPoint> &operatingPoints, float FAR)
+{
+    int index = 0;
+    while (operatingPoints[index].FAR < FAR) {
+        index++;
+        if (index == operatingPoints.size())
+            return 1;
+    }
+
+    const float x1 = (index == 0 ? 0 : operatingPoints[index-1].FAR);
+    const float y1 = (index == 0 ? 0 : operatingPoints[index-1].TAR);
+    const float x2 = operatingPoints[index].FAR;
+    const float y2 = operatingPoints[index].TAR;
+    const float m = (y2 - y1) / (x2 - x1);
+    const float b = y1 - m*x1;
+    return m * FAR + b;
+}
+
+float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const QString &csv, int partition)
+{
+    return Evaluate(scores, BEE::makeMask(target, query, partition), csv);
+}
+
+float Evaluate(const QString &simmat, const QString &mask, const QString &csv)
+{
+    qDebug("Evaluating %s%s%s",
+           qPrintable(simmat),
+           mask.isEmpty() ? "" : qPrintable(" with " + mask),
+           csv.isEmpty() ? "" : qPrintable(" to " + csv));
+
+    // Read similarity matrix
+    QString target, query;
+    const Mat scores = BEE::readSimmat(simmat, &target, &query);
+
+    // Read mask matrix
+    Mat truth;
+    if (mask.isEmpty()) {
+        // Use the galleries specified in the similarity matrix
+        truth = BEE::makeMask(TemplateList::fromGallery(target).files(),
+                              TemplateList::fromGallery(query).files());
+    } else {
+        File maskFile(mask);
+        maskFile.set("rows", scores.rows);
+        maskFile.set("columns", scores.cols);
+        truth = BEE::readMask(maskFile);
+    }
+
+    return Evaluate(scores, truth, csv);
+}
+
+float Evaluate(const Mat &simmat, const Mat &mask, const QString &csv)
+{
+    if (simmat.size() != mask.size())
+        qFatal("Similarity matrix (%ix%i) differs in size from mask matrix (%ix%i).",
+               simmat.rows, simmat.cols, mask.rows, mask.cols);
+
+    const int Max_Points = 500;
+    float result = -1;
+
+    // Make comparisons
+    QList<Comparison> comparisons; comparisons.reserve(simmat.rows*simmat.cols);
+    int genuineCount = 0, impostorCount = 0, numNaNs = 0;
+    for (int i=0; i<simmat.rows; i++) {
+        for (int j=0; j<simmat.cols; j++) {
+            const BEE::Mask_t mask_val = mask.at<BEE::Mask_t>(i,j);
+            const BEE::Simmat_t simmat_val = simmat.at<BEE::Simmat_t>(i,j);
+            if (mask_val == BEE::DontCare) continue;
+            if (simmat_val != simmat_val) { numNaNs++; continue; }
+            comparisons.append(Comparison(simmat_val, j, i, mask_val == BEE::Match));
+            if (comparisons.last().genuine) genuineCount++;
+            else                            impostorCount++;
+        }
+    }
+
+    if (numNaNs > 0) qWarning("Encountered %d NaN scores!", numNaNs);
+    if (genuineCount == 0) qFatal("No genuine scores!");
+    if (impostorCount == 0) qFatal("No impostor scores!");
+
+    // Sort comparisons by simmat_val (score)
+    std::sort(comparisons.begin(), comparisons.end());
+
+    QList<OperatingPoint> operatingPoints;
+    QList<float> genuines; genuines.reserve(sqrt((float)comparisons.size()));
+    QList<float> impostors; impostors.reserve(comparisons.size());
+    QVector<int> firstGenuineReturns(simmat.rows, 0);
+
+    int falsePositives = 0, previousFalsePositives = 0;
+    int truePositives = 0, previousTruePositives = 0;
+    int index = 0;
+    float minGenuineScore = std::numeric_limits<float>::max();
+    float minImpostorScore = std::numeric_limits<float>::max();
+
+    while (index < comparisons.size()) {
+        float thresh = comparisons[index].score;
+        // Compute genuine and imposter statistics at a threshold
+        while ((index < comparisons.size()) &&
+               (comparisons[index].score == thresh)) {
+            const Comparison &comparison = comparisons[index];
+            if (comparison.genuine) {
+                truePositives++;
+                genuines.append(comparison.score);
+                if (firstGenuineReturns[comparison.query] < 1)
+                    firstGenuineReturns[comparison.query] = abs(firstGenuineReturns[comparison.query]) + 1;
+                if ((comparison.score != -std::numeric_limits<float>::max()) &&
+                    (comparison.score < minGenuineScore))
+                    minGenuineScore = comparison.score;
+            } else {
+                falsePositives++;
+                impostors.append(comparison.score);
+                if (firstGenuineReturns[comparison.query] < 1)
+                    firstGenuineReturns[comparison.query]--;
+                if ((comparison.score != -std::numeric_limits<float>::max()) &&
+                    (comparison.score < minImpostorScore))
+                    minImpostorScore = comparison.score;
+            }
+            index++;
+        }
+
+        if ((falsePositives > previousFalsePositives) &&
+             (truePositives > previousTruePositives)) {
+            // Restrict the extreme ends of the curve
+            if ((truePositives >= 10) && (falsePositives < impostorCount/2))
+                operatingPoints.append(OperatingPoint(thresh, float(falsePositives)/impostorCount, float(truePositives)/genuineCount));
+            previousFalsePositives = falsePositives;
+            previousTruePositives = truePositives;
+        }
+    }
+
+    if (operatingPoints.size() == 0) operatingPoints.append(OperatingPoint(1, 1, 1));
+    if (operatingPoints.size() == 1) operatingPoints.prepend(OperatingPoint(0, 0, 0));
+    if (operatingPoints.size() > 2)  operatingPoints.takeLast(); // Remove point (1,1)
+
+    // Write Metadata table
+    QStringList lines;
+    lines.append("Plot,X,Y");
+    lines.append("Metadata,"+QString::number(simmat.cols)+",Gallery");
+    lines.append("Metadata,"+QString::number(simmat.rows)+",Probe");
+    lines.append("Metadata,"+QString::number(genuineCount)+",Genuine");
+    lines.append("Metadata,"+QString::number(impostorCount)+",Impostor");
+    lines.append("Metadata,"+QString::number(simmat.cols*simmat.rows-(genuineCount+impostorCount))+",Ignored");
+
+    // Write Detection Error Tradeoff (DET), PRE, REC
+    int points = qMin(operatingPoints.size(), Max_Points);
+    for (int i=0; i<points; i++) {
+        const OperatingPoint &operatingPoint = operatingPoints[double(i) / double(points-1) * double(operatingPoints.size()-1)];
+        lines.append(QString("DET,%1,%2").arg(QString::number(operatingPoint.FAR),
+                                              QString::number(1-operatingPoint.TAR)));
+        lines.append(QString("FAR,%1,%2").arg(QString::number(operatingPoint.score),
+                                              QString::number(operatingPoint.FAR)));
+        lines.append(QString("FRR,%1,%2").arg(QString::number(operatingPoint.score),
+                                              QString::number(1-operatingPoint.TAR)));
+    }
+
+    // Write FAR/TAR Bar Chart (BC)
+    lines.append(qPrintable(QString("BC,0.001,%1").arg(QString::number(getTAR(operatingPoints, 0.001), 'f', 3))));
+    lines.append(qPrintable(QString("BC,0.01,%1").arg(QString::number(result = getTAR(operatingPoints, 0.01), 'f', 3))));
+
+    // Write SD & KDE
+    points = qMin(qMin(Max_Points, genuines.size()), impostors.size());
+    QList<double> sampledGenuineScores; sampledGenuineScores.reserve(points);
+    QList<double> sampledImpostorScores; sampledImpostorScores.reserve(points);
+
+    if (points > 1) {
+        for (int i=0; i<points; i++) {
+            float genuineScore = genuines[double(i) / double(points-1) * double(genuines.size()-1)];
+            float impostorScore = impostors[double(i) / double(points-1) * double(impostors.size()-1)];
+            if (genuineScore == -std::numeric_limits<float>::max()) genuineScore = minGenuineScore;
+            if (impostorScore == -std::numeric_limits<float>::max()) impostorScore = minImpostorScore;
+            lines.append(QString("SD,%1,Genuine").arg(QString::number(genuineScore)));
+            lines.append(QString("SD,%1,Impostor").arg(QString::number(impostorScore)));
+            sampledGenuineScores.append(genuineScore);
+            sampledImpostorScores.append(impostorScore);
+        }
+    }
+
+    // Write Cumulative Match Characteristic (CMC) curve
+    const int Max_Retrieval = 200;
+    const int Report_Retrieval = 5;
+
+    float reportRetrievalRate = -1;
+    for (int i=1; i<=Max_Retrieval; i++) {
+        int realizedReturns = 0, possibleReturns = 0;
+        foreach (int firstGenuineReturn, firstGenuineReturns) {
+            if (firstGenuineReturn > 0) {
+                possibleReturns++;
+                if (firstGenuineReturn <= i) realizedReturns++;
+            }
+        }
+        const float retrievalRate = float(realizedReturns)/possibleReturns;
+        lines.append(qPrintable(QString("CMC,%1,%2").arg(QString::number(i), QString::number(retrievalRate))));
+        if (i == Report_Retrieval) reportRetrievalRate = retrievalRate;
+    }
+
+    if (!csv.isEmpty()) QtUtils::writeFile(csv, lines);
+    qDebug("TAR @ FAR = 0.01: %.3f\nRetrieval Rate @ Rank = %d: %.3f", result, Report_Retrieval, reportRetrievalRate);
+    return result;
+}
+
+// Helper struct for statistics accumulation
+struct Counter
+{
+    float truePositive, falsePositive, falseNegative;
+    Counter()
+    {
+        truePositive = 0;
+        falsePositive = 0;
+        falseNegative = 0;
+    }
+};
+
+void EvalClassification(const QString &predictedInput, const QString &truthInput)
+{
+    qDebug("Evaluating classification of %s against %s", qPrintable(predictedInput), qPrintable(truthInput));
+    TemplateList predicted(TemplateList::fromGallery(predictedInput));
+    TemplateList truth(TemplateList::fromGallery(truthInput));
+    if (predicted.size() != truth.size()) qFatal("Input size mismatch.");
+
+    QHash<QString, Counter> counters;
+    for (int i=0; i<predicted.size(); i++) {
+        if (predicted[i].file.name != truth[i].file.name)
+            qFatal("Input order mismatch.");
+
+        // Typically these lists will be of length one, but this generalization allows measuring multi-class labeling accuracy.
+        QString predictedSubject = predicted[i].file.get<QString>("Subject");
+        QString trueSubject = truth[i].file.get<QString>("Subject");
+
+        QStringList predictedSubjects(predictedSubject);
+        QStringList trueSubjects(trueSubject);
+
+        foreach (const QString &subject, trueSubjects.toVector() /* Hack to copy the list. */) {
+            if (predictedSubjects.contains(subject)) {
+                counters[subject].truePositive++;
+                trueSubjects.removeOne(subject);
+                predictedSubjects.removeOne(subject);
+            } else {
+                counters[subject].falseNegative++;
+            }
+        }
+
+        for (int i=0; i<trueSubjects.size(); i++)
+            foreach (const QString &subject, predictedSubjects)
+                counters[subject].falsePositive += 1.f / predictedSubjects.size();
+    }
+
+    const QStringList keys = counters.keys();
+    QSharedPointer<Output> output(Output::make("", FileList() << "Count" << "Precision" << "Recall" << "F-score", FileList(keys)));
+
+    int tpc = 0;
+    int fnc = 0;
+
+    for (int i=0; i<counters.size(); i++) {
+        const QString &subject = keys[i];
+        const Counter &counter = counters[subject];
+        tpc += counter.truePositive;
+        fnc += counter.falseNegative;
+        const int count = counter.truePositive + counter.falseNegative;
+        const float precision = counter.truePositive / (float)(counter.truePositive + counter.falsePositive);
+        const float recall = counter.truePositive / (float)(counter.truePositive + counter.falseNegative);
+        const float fscore = 2 * precision * recall / (precision + recall);
+        output->setRelative(count, i, 0);
+        output->setRelative(precision, i, 1);
+        output->setRelative(recall, i, 2);
+        output->setRelative(fscore, i, 3);
+    }
+
+    qDebug("Overall Accuracy = %f", (float)tpc / (float)(tpc + fnc));
+}
+
+struct Detection
+{
+    QRectF boundingBox;
+    float confidence;
+
+    Detection() {}
+    Detection(const QRectF &boundingBox_, float confidence_ = -1)
+        : boundingBox(boundingBox_), confidence(confidence_) {}
+
+    float area() const { return boundingBox.width() * boundingBox.height(); }
+    float overlap(const Detection &other) const
+    {
+        const Detection intersection(boundingBox.intersected(other.boundingBox));
+        return intersection.area() / (area() + other.area() - 2*intersection.area());
+    }
+};
+
+struct Detections
+{
+    QList<Detection> predicted, truth;
+};
+
+struct DetectionOperatingPoint
+{
+    float confidence, overlap;
+    DetectionOperatingPoint() : confidence(-1), overlap(-1) {}
+    DetectionOperatingPoint(float confidence_, float overlap_) : confidence(confidence_), overlap(overlap_) {}
+    inline bool operator<(const DetectionOperatingPoint &other) const { return confidence > other.confidence; }
+};
+
+float EvalDetection(const QString &predictedInput, const QString &truthInput, const QString &csv)
+{
+    qDebug("Evaluating detection of %s against %s", qPrintable(predictedInput), qPrintable(truthInput));
+    const TemplateList predicted(TemplateList::fromGallery(predictedInput));
+    const TemplateList truth(TemplateList::fromGallery(truthInput));
+
+    // Figure out which metadata field contains a bounding box
+    QString detectKey;
+    foreach (const QString &key, truth.first().file.localKeys())
+        if (!truth.first().file.get<QRectF>(key, QRectF()).isNull()) {
+            detectKey = key;
+            break;
+        }
+    if (detectKey.isNull()) qFatal("No suitable metadata key found.");
+    else                    qDebug("Using metadata key: %s", qPrintable(detectKey));
+
+    QHash<QString, Detections> allDetections; // Organized by file
+    foreach (const Template &t, predicted)
+        allDetections[t.file.baseName()].predicted.append(Detection(t.file.get<QRectF>(detectKey), t.file.get<float>("Confidence", -1)));
+    foreach (const Template &t, truth)
+        allDetections[t.file.baseName()].truth.append(Detection(t.file.get<QRectF>(detectKey)));
+
+    QList<DetectionOperatingPoint> points;
+    foreach (Detections detections, allDetections.values()) {
+        while (!detections.truth.isEmpty() && !detections.predicted.isEmpty()) {
+            Detection truth = detections.truth.takeFirst();
+            int bestIndex = -1;
+            float bestOverlap = -1;
+            for (int i=0; i<detections.predicted.size(); i++) {
+                const float overlap = truth.overlap(detections.predicted[i]);
+                if (overlap > bestOverlap) {
+                    bestOverlap = overlap;
+                    bestIndex = i;
+                }
+            }
+            Detection predicted = detections.predicted.takeAt(bestIndex);
+            points.append(DetectionOperatingPoint(predicted.confidence, bestOverlap));
+        }
+
+        foreach (const Detection &detection, detections.predicted)
+            points.append(DetectionOperatingPoint(detection.confidence, 0));
+        for (int i=0; i<detections.truth.size(); i++)
+            points.append(DetectionOperatingPoint(-std::numeric_limits<float>::max(), 0));
+    }
+
+    std::sort(points.begin(), points.end());
+
+    QStringList lines;
+    lines.append("Plot, X, Y");
+
+    // TODO: finish implementing
+
+    (void) csv;
+    return 0;
+}
+
+void EvalRegression(const QString &predictedInput, const QString &truthInput)
+{
+    qDebug("Evaluating regression of %s against %s", qPrintable(predictedInput), qPrintable(truthInput));
+    const TemplateList predicted(TemplateList::fromGallery(predictedInput));
+    const TemplateList truth(TemplateList::fromGallery(truthInput));
+    if (predicted.size() != truth.size()) qFatal("Input size mismatch.");
+
+    float rmsError = 0;
+    QStringList truthValues, predictedValues;
+    for (int i=0; i<predicted.size(); i++) {
+        if (predicted[i].file.name != truth[i].file.name)
+            qFatal("Input order mismatch.");
+        rmsError += pow(predicted[i].file.get<float>("Subject")-truth[i].file.get<float>("Subject"), 2.f);
+        truthValues.append(QString::number(truth[i].file.get<float>("Subject")));
+        predictedValues.append(QString::number(predicted[i].file.get<float>("Subject")));
+    }
+
+    QStringList rSource;
+    rSource << "# Load libraries" << "library(ggplot2)" << "" << "# Set Data"
+            << "Actual <- c(" + truthValues.join(",") + ")"
+            << "Predicted <- c(" + predictedValues.join(",") + ")"
+            << "data <- data.frame(Actual, Predicted)"
+            << "" << "# Construct Plot" << "pdf(\"EvalRegression.pdf\")"
+            << "print(qplot(Actual, Predicted, data=data, geom=\"jitter\", alpha=I(2/3)) + geom_abline(intercept=0, slope=1, color=\"forestgreen\", size=I(1)) + geom_smooth(size=I(1), color=\"mediumblue\") + theme_bw())"
+            << "print(qplot(Actual, Predicted-Actual, data=data, geom=\"jitter\", alpha=I(2/3)) + geom_abline(intercept=0, slope=0, color=\"forestgreen\", size=I(1)) + geom_smooth(size=I(1), color=\"mediumblue\") + theme_bw())"
+            << "dev.off()";
+
+
+    QString rFile = "EvalRegression.R";
+    QtUtils::writeFile(rFile, rSource);
+    bool success = QtUtils::runRScript(rFile);
+    if (success) QtUtils::showFile("EvalRegression.pdf");
+
+    qDebug("RMS Error = %f", sqrt(rmsError/predicted.size()));
+}
+
+} // namespace br
@@ -14,18 +14,22 @@
  * limitations under the License.                                            *
  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-#ifndef __CLASSIFY_H
-#define __CLASSIFY_H
+#ifndef __EVAL_H
+#define __EVAL_H
 #include <QList>
 #include <QString>
+#include "openbr/openbr_plugin.h"
 namespace br
 {
+    float Evaluate(const QString &simmat, const QString &mask = "", const QString &csv = ""); // Returns TAR @ FAR = 0.001
+    float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const QString &csv = "", int parition = 0);
+    float Evaluate(const cv::Mat &scores, const cv::Mat &masks, const QString &csv = "");
     void EvalClassification(const QString &predictedInput, const QString &truthInput);
-    void EvalDetection(const QString &predictedInput, const QString &truthInput);
+    float EvalDetection(const QString &predictedInput, const QString &truthInput, const QString &csv = ""); // Return average overlap
     void EvalRegression(const QString &predictedInput, const QString &truthInput);
 }
-#endif // __CLASSIFY_H
+#endif // __EVAL_H
@@ -14,57 +14,15 @@
  * limitations under the License.                                            *
  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-#include <QDebug>
-#include <QDir>
-#include <QFile>
-#include <QFileInfo>
-#include <QFuture>
-#include <QList>
-#include <QPair>
-#include <QPointF>
-#include <QRegExp>
-#include <QSet>
-#include <QStringList>
-#include <QVector>
-#include <QtAlgorithms>
-#include <opencv2/core/core.hpp>
-#include <assert.h>
-
 #include "plot.h"
 #include "version.h"
-#include "openbr/core/bee.h"
-#include "openbr/core/common.h"
-#include "openbr/core/opencvutils.h"
 #include "openbr/core/qtutils.h"
-#undef FAR // Windows preprecessor definition
-
 using namespace cv;
 namespace br
 {
-void Confusion(const QString &file, float score, int &true_positives, int &false_positives, int &true_negatives, int &false_negatives)
-{
-    qDebug("Computing confusion matrix of %s at %f", qPrintable(file), score);
-
-    QStringList lines = QtUtils::readLines(file);
-    true_positives = false_positives = true_negatives = false_negatives = 0;
-    foreach (const QString &line, lines) {
-        if (!line.startsWith("SD")) continue;
-        QStringList words = line.split(",");
-        bool ok;
-        float similarity = words[1].toFloat(&ok); assert(ok);
-        if (words[2] == "Genuine") {
-            if (similarity >= score) true_positives++;
-            else                     false_negatives++;
-        } else {
-            if (similarity >= score) false_positives++;
-            else                     true_negatives++;
-        }
-    }
-}
-
 static QStringList getPivots(const QString &file, bool headers)
 {
     QString str;
@@ -73,224 +31,6 @@ static QStringList getPivots(const QString &amp;file, bool headers)
     return str.split("_");
 }
-struct Comparison
-{
-    float score;
-    int target, query;
-    bool genuine;
-
-    Comparison() {}
-    Comparison(float _score, int _target, int _query, bool _genuine)
-        : score(_score), target(_target), query(_query), genuine(_genuine) {}
-    inline bool operator<(const Comparison &other) const { return score > other.score; }
-};
-
-struct OperatingPoint
-{
-    float score, FAR, TAR;
-    OperatingPoint() {}
-    OperatingPoint(float _score, float _FAR, float _TAR)
-        : score(_score), FAR(_FAR), TAR(_TAR) {}
-};
-
-static float getTAR(const QList<OperatingPoint> &operatingPoints, float FAR)
-{
-    int index = 0;
-    while (operatingPoints[index].FAR < FAR) {
-        index++;
-        if (index == operatingPoints.size())
-            return 1;
-    }
-
-    const float x1 = (index == 0 ? 0 : operatingPoints[index-1].FAR);
-    const float y1 = (index == 0 ? 0 : operatingPoints[index-1].TAR);
-    const float x2 = operatingPoints[index].FAR;
-    const float y2 = operatingPoints[index].TAR;
-    const float m = (y2 - y1) / (x2 - x1);
-    const float b = y1 - m*x1;
-    return m * FAR + b;
-}
-
-float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const QString &csv, int partition)
-{
-    return Evaluate(scores, BEE::makeMask(target, query, partition), csv);
-}
-
-float Evaluate(const QString &simmat, const QString &mask, const QString &csv)
-{
-    qDebug("Evaluating %s%s%s",
-           qPrintable(simmat),
-           mask.isEmpty() ? "" : qPrintable(" with " + mask),
-           csv.isEmpty() ? "" : qPrintable(" to " + csv));
-
-    // Read similarity matrix
-    QString target, query;
-    const Mat scores = BEE::readSimmat(simmat, &target, &query);
-
-    // Read mask matrix
-    Mat truth;
-    if (mask.isEmpty()) {
-        // Use the galleries specified in the similarity matrix
-        truth = BEE::makeMask(TemplateList::fromGallery(target).files(),
-                              TemplateList::fromGallery(query).files());
-    } else {
-        File maskFile(mask);
-        maskFile.set("rows", scores.rows);
-        maskFile.set("columns", scores.cols);
-        truth = BEE::readMask(maskFile);
-    }
-
-    return Evaluate(scores, truth, csv);
-}
-
-float Evaluate(const Mat &simmat, const Mat &mask, const QString &csv)
-{
-    if (simmat.size() != mask.size())
-        qFatal("Similarity matrix (%ix%i) differs in size from mask matrix (%ix%i).",
-               simmat.rows, simmat.cols, mask.rows, mask.cols);
-
-    const int Max_Points = 500;
-    float result = -1;
-
-    // Make comparisons
-    QList<Comparison> comparisons; comparisons.reserve(simmat.rows*simmat.cols);
-    int genuineCount = 0, impostorCount = 0, numNaNs = 0;
-    for (int i=0; i<simmat.rows; i++) {
-        for (int j=0; j<simmat.cols; j++) {
-            const BEE::Mask_t mask_val = mask.at<BEE::Mask_t>(i,j);
-            const BEE::Simmat_t simmat_val = simmat.at<BEE::Simmat_t>(i,j);
-            if (mask_val == BEE::DontCare) continue;
-            if (simmat_val != simmat_val) { numNaNs++; continue; }
-            comparisons.append(Comparison(simmat_val, j, i, mask_val == BEE::Match));
-            if (comparisons.last().genuine) genuineCount++;
-            else                            impostorCount++;
-        }
-    }
-
-    if (numNaNs > 0) qWarning("Encountered %d NaN scores!", numNaNs);
-    if (genuineCount == 0) qFatal("No genuine scores!");
-    if (impostorCount == 0) qFatal("No impostor scores!");
-
-    // Sort comparisons by simmat_val (score)
-    std::sort(comparisons.begin(), comparisons.end());
-
-    QList<OperatingPoint> operatingPoints;
-    QList<float> genuines; genuines.reserve(sqrt((float)comparisons.size()));
-    QList<float> impostors; impostors.reserve(comparisons.size());
-    QVector<int> firstGenuineReturns(simmat.rows, 0);
-
-    int falsePositives = 0, previousFalsePositives = 0;
-    int truePositives = 0, previousTruePositives = 0;
-    int index = 0;
-    float minGenuineScore = std::numeric_limits<float>::max();
-    float minImpostorScore = std::numeric_limits<float>::max();
-
-    while (index < comparisons.size()) {
-        float thresh = comparisons[index].score;
-        // Compute genuine and imposter statistics at a threshold
-        while ((index < comparisons.size()) &&
-               (comparisons[index].score == thresh)) {
-            const Comparison &comparison = comparisons[index];
-            if (comparison.genuine) {
-                truePositives++;
-                genuines.append(comparison.score);
-                if (firstGenuineReturns[comparison.query] < 1)
-                    firstGenuineReturns[comparison.query] = abs(firstGenuineReturns[comparison.query]) + 1;
-                if ((comparison.score != -std::numeric_limits<float>::max()) &&
-                    (comparison.score < minGenuineScore))
-                    minGenuineScore = comparison.score;
-            } else {
-                falsePositives++;
-                impostors.append(comparison.score);
-                if (firstGenuineReturns[comparison.query] < 1)
-                    firstGenuineReturns[comparison.query]--;
-                if ((comparison.score != -std::numeric_limits<float>::max()) &&
-                    (comparison.score < minImpostorScore))
-                    minImpostorScore = comparison.score;
-            }
-            index++;
-        }
-
-        if ((falsePositives > previousFalsePositives) &&
-             (truePositives > previousTruePositives)) {
-            // Restrict the extreme ends of the curve
-            if ((truePositives >= 10) && (falsePositives < impostorCount/2))
-                operatingPoints.append(OperatingPoint(thresh, float(falsePositives)/impostorCount, float(truePositives)/genuineCount));
-            previousFalsePositives = falsePositives;
-            previousTruePositives = truePositives;
-        }
-    }
-
-    if (operatingPoints.size() == 0) operatingPoints.append(OperatingPoint(1, 1, 1));
-    if (operatingPoints.size() == 1) operatingPoints.prepend(OperatingPoint(0, 0, 0));
-    if (operatingPoints.size() > 2)  operatingPoints.takeLast(); // Remove point (1,1)
-
-    // Write Metadata table
-    QStringList lines;
-    lines.append("Plot,X,Y");
-    lines.append("Metadata,"+QString::number(simmat.cols)+",Gallery");
-    lines.append("Metadata,"+QString::number(simmat.rows)+",Probe");
-    lines.append("Metadata,"+QString::number(genuineCount)+",Genuine");
-    lines.append("Metadata,"+QString::number(impostorCount)+",Impostor");
-    lines.append("Metadata,"+QString::number(simmat.cols*simmat.rows-(genuineCount+impostorCount))+",Ignored");
-
-    // Write Detection Error Tradeoff (DET), PRE, REC
-    int points = qMin(operatingPoints.size(), Max_Points);
-    for (int i=0; i<points; i++) {
-        const OperatingPoint &operatingPoint = operatingPoints[double(i) / double(points-1) * double(operatingPoints.size()-1)];
-        lines.append(QString("DET,%1,%2").arg(QString::number(operatingPoint.FAR),
-                                              QString::number(1-operatingPoint.TAR)));
-        lines.append(QString("FAR,%1,%2").arg(QString::number(operatingPoint.score),
-                                              QString::number(operatingPoint.FAR)));
-        lines.append(QString("FRR,%1,%2").arg(QString::number(operatingPoint.score),
-                                              QString::number(1-operatingPoint.TAR)));
-    }
-
-    // Write FAR/TAR Bar Chart (BC)
-    lines.append(qPrintable(QString("BC,0.001,%1").arg(QString::number(getTAR(operatingPoints, 0.001), 'f', 3))));
-    lines.append(qPrintable(QString("BC,0.01,%1").arg(QString::number(result = getTAR(operatingPoints, 0.01), 'f', 3))));
-
-    // Write SD & KDE
-    points = qMin(qMin(Max_Points, genuines.size()), impostors.size());
-    QList<double> sampledGenuineScores; sampledGenuineScores.reserve(points);
-    QList<double> sampledImpostorScores; sampledImpostorScores.reserve(points);
-
-    if (points > 1) {
-        for (int i=0; i<points; i++) {
-            float genuineScore = genuines[double(i) / double(points-1) * double(genuines.size()-1)];
-            float impostorScore = impostors[double(i) / double(points-1) * double(impostors.size()-1)];
-            if (genuineScore == -std::numeric_limits<float>::max()) genuineScore = minGenuineScore;
-            if (impostorScore == -std::numeric_limits<float>::max()) impostorScore = minImpostorScore;
-            lines.append(QString("SD,%1,Genuine").arg(QString::number(genuineScore)));
-            lines.append(QString("SD,%1,Impostor").arg(QString::number(impostorScore)));
-            sampledGenuineScores.append(genuineScore);
-            sampledImpostorScores.append(impostorScore);
-        }
-    }
-
-    // Write Cumulative Match Characteristic (CMC) curve
-    const int Max_Retrieval = 200;
-    const int Report_Retrieval = 5;
-
-    float reportRetrievalRate = -1;
-    for (int i=1; i<=Max_Retrieval; i++) {
-        int realizedReturns = 0, possibleReturns = 0;
-        foreach (int firstGenuineReturn, firstGenuineReturns) {
-            if (firstGenuineReturn > 0) {
-                possibleReturns++;
-                if (firstGenuineReturn <= i) realizedReturns++;
-            }
-        }
-        const float retrievalRate = float(realizedReturns)/possibleReturns;
-        lines.append(qPrintable(QString("CMC,%1,%2").arg(QString::number(i), QString::number(retrievalRate))));
-        if (i == Report_Retrieval) reportRetrievalRate = retrievalRate;
-    }
-
-    if (!csv.isEmpty()) QtUtils::writeFile(csv, lines);
-    qDebug("TAR @ FAR = 0.01: %.3f\nRetrieval Rate @ Rank = %d: %.3f", result, Report_Retrieval, reportRetrievalRate);
-    return result;
-}
-
 static QString getScale(const QString &mode, const QString &title, int vals)
 {
     if      (vals > 12) return " + scale_"+mode+"_discrete(\""+title+"\")";
@@ -474,7 +214,6 @@ struct RPlot
 };
 // Does not work if dataset folder starts with a number
-
 bool Plot(const QStringList &files, const br::File &destination, bool show)
 {
     qDebug("Plotting %d file(s) to %s", files.size(), qPrintable(destination));
@@ -24,14 +24,8 @@
 namespace br
 {
-
-void Confusion(const QString &file, float score, int &true_positives, int &false_positives, int &true_negatives, int &false_negatives);
-float Evaluate(const QString &simmat, const QString &mask = "", const QString &csv = ""); // Returns TAR @ FAR = 0.001
-float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const QString &csv = "", int parition = 0);
-float Evaluate(const cv::Mat &scores, const cv::Mat &masks, const QString &csv = "");
-bool Plot(const QStringList &files, const br::File &destination, bool show = false);
-bool PlotMetadata(const QStringList &files, const QString &destination, bool show = false);
-
+    bool Plot(const QStringList &files, const br::File &destination, bool show = false);
+    bool PlotMetadata(const QStringList &files, const QString &destination, bool show = false);
 }
 #endif // __PLOT_H
@@ -17,8 +17,8 @@
 #include <openbr/openbr_plugin.h>
 #include "core/bee.h"
-#include "core/classify.h"
 #include "core/cluster.h"
+#include "core/eval.h"
 #include "core/fuse.h"
 #include "core/plot.h"
 #include "core/qtutils.h"
@@ -51,11 +51,6 @@ void br_compare(const char *target_gallery, const char *query_gallery, const cha
     Compare(File(target_gallery), File(query_gallery), File(output));
 }
-void br_confusion(const char *file, float score, int *true_positives, int *false_positives, int *true_negatives, int *false_negatives)
-{
-    return Confusion(file, score, *true_positives, *false_positives, *true_negatives, *false_negatives);
-}
-
 void br_convert(const char *file_type, const char *input_file, const char *output_file)
 {
     Convert(File(file_type), File(input_file), File(output_file));
@@ -115,20 +115,6 @@ BR_EXPORT void br_combine_masks(int num_input_masks, const char *input_masks[], 
 BR_EXPORT void br_compare(const char *target_gallery, const char *query_gallery, const char *output = "");
 /*!
- * \brief Computes the confusion matrix for a dataset at a particular threshold.
- *
- * <a href="http://en.wikipedia.org/wiki/Confusion_matrix">Wikipedia Explanation</a>
- * \param file <tt>.csv</tt> file created using \ref br_eval.
- * \param score The similarity score to threshold at.
- * \param[out] true_positives The true positive count.
- * \param[out] false_positives The false positive count.
- * \param[out] true_negatives The true negative count.
- * \param[out] false_negatives The false negative count.
- */
-BR_EXPORT void br_confusion(const char *file, float score,
-                            int *true_positives, int *false_positives, int *true_negatives, int *false_negatives);
-
-/*!
  * \brief Wraps br::Convert()
  */
 BR_EXPORT void br_convert(const char *file_type, const char *input_file, const char *output_file);
@@ -436,7 +436,7 @@ TemplateList TemplateList::fromGallery(const br::File &amp;gallery)
 // stores the index values in "Label" of the output template list
 TemplateList TemplateList::relabel(const TemplateList &tl, const QString & propName)
 {
-    const QList<QString> originalLabels = tl.get<QString>(propName);
+    const QList<QString> originalLabels = File::get<QString>(tl, propName);
     QHash<QString,int> labelTable;
     foreach (const QString & label, originalLabels)
         if (!labelTable.contains(label))
@@ -464,7 +464,7 @@ QList&lt;int&gt; TemplateList::indexProperty(const QString &amp; propName, QHash&lt;QString, 
     valueMap.clear();
     reverseLookup.clear();
-    const QList<QVariant> originalLabels = values(propName);
+    const QList<QVariant> originalLabels = File::values(*this, propName);
     foreach (const QVariant & label, originalLabels) {
         QString labelString = label.toString();
         if (!valueMap.contains(labelString)) {
@@ -481,9 +481,9 @@ QList&lt;int&gt; TemplateList::indexProperty(const QString &amp; propName, QHash&lt;QString, 
 }
 // uses -1 for missing values
-QList<int> TemplateList::applyIndex(const QString & propName, const QHash<QString, int> & valueMap) const
+QList<int> TemplateList::applyIndex(const QString &propName, const QHash<QString, int> &valueMap) const
 {
-    const QList<QString> originalLabels = get<QString>(propName);
+    const QList<QString> originalLabels = File::get<QString>(*this, propName);
     QList<int> result;
     for (int i=0; i<originalLabels.size(); i++) {
@@ -228,7 +228,20 @@ struct BR_EXPORT File
         return variant.value<T>();
     }
-    /*!< \brief Returns a list of type T for the key, throwing an error if the key does not exist or if the value cannot be converted to the specified type. */
+    /*!< \brief Returns a value for the key, returning \em defaultValue if the key does not exist or can't be converted. */
+    template <typename T>
+    T get(const QString &key, const T &defaultValue) const
+    {
+        if (!contains(key)) return defaultValue;
+        QVariant variant = value(key);
+        if (!variant.canConvert<T>()) return defaultValue;
+        return variant.value<T>();
+    }
+
+    /*!< \brief Specialization for boolean type. */
+    bool getBool(const QString &key, bool defaultValue = false) const;
+
+    /*!< \brief Specialization for list type. Returns a list of type T for the key, throwing an error if the key does not exist or if the value cannot be converted to the specified type. */
     template <typename T>
     QList<T> getList(const QString &key) const
     {
@@ -241,17 +254,31 @@ struct BR_EXPORT File
         return list;
     }
-    /*!< \brief Specialization for boolean type. */
-    bool getBool(const QString &key, bool defaultValue = false) const;
+    /*!< \brief Returns the value for the specified key for every file in the list. */
+    template<class U>
+    static QList<QVariant> values(const QList<U> &fileList, const QString &key)
+    {
+        QList<QVariant> values; values.reserve(fileList.size());
+        foreach (const U &f, fileList) values.append(((const File&)f).value(key));
+        return values;
+    }
-    /*!< \brief Returns a value for the key, returning \em defaultValue if the key does not exist or can't be converted. */
-    template <typename T>
-    T get(const QString &key, const T &defaultValue) const
+    /*!< \brief Returns a value for the key for every file in the list, throwing an error if the key does not exist. */
+    template<class T, class U>
+    static QList<T> get(const QList<U> &fileList, const QString &key)
     {
-        if (!contains(key)) return defaultValue;
-        QVariant variant = value(key);
-        if (!variant.canConvert<T>()) return defaultValue;
-        return variant.value<T>();
+        QList<T> result; result.reserve(fileList.size());
+        foreach (const U &f, fileList) result.append(((const File&)f).get<T>(key));
+        return result;
+    }
+
+    /*!< \brief Returns a value for the key for every file in the list, returning \em defaultValue if the key does not exist or can't be converted. */
+    template<class T, class U>
+    static QList<T> get(const QList<U> &fileList, const QString &key, const T &defaultValue)
+    {
+        QList<T> result; result.reserve(fileList.size());
+        foreach (const U &f, fileList) result.append(static_cast<const File&>(f).get<T>(key, defaultValue));
+        return result;
     }
     inline bool failed() const { return getBool("FTE") || getBool("FTO"); } /*!< \brief Returns \c true if the file failed to open or enroll, \c false otherwise. */
@@ -297,23 +324,6 @@ struct BR_EXPORT FileList : public QList&lt;File&gt;
     QStringList flat() const; /*!< \brief Returns br::File::flat() for each file in the list. */
     QStringList names() const; /*!<  \brief Returns #br::File::name for each file in the list. */
     void sort(const QString& key); /*!<  \brief Sort the list based on metadata. */
-     /*!< \brief Returns values associated with the input propName for each file in the list. */
-    template<typename T>
-    QList<T> get(const QString & propName) const
-    {
-        QList<T> values; values.reserve(size());
-        foreach (const File &f, *this)
-            values.append(f.get<T>(propName));
-        return values;
-    }
-    template<typename T>
-    QList<T> get(const QString & propName, T defaultValue) const
-    {
-        QList<T> values; values.reserve(size());
-        foreach (const File &f, *this)
-            values.append(f.contains(propName) ? f.get<T>(propName) : defaultValue);
-        return values;
-    }
     QList<int> crossValidationPartitions() const; /*!< \brief Returns the cross-validation partition (default=0) for each file in the list. */
     int failures() const; /*!< \brief Returns the number of files with br::File::failed(). */
@@ -344,6 +354,7 @@ struct Template : public QList&lt;cv::Mat&gt;
     inline const cv::Mat &m() const { static const cv::Mat NullMatrix;
                                       return isEmpty() ? qFatal("Empty template."), NullMatrix : last(); } /*!< \brief Idiom to treat the template as a matrix. */
     inline cv::Mat &m() { return isEmpty() ? append(cv::Mat()), last() : last(); } /*!< \brief Idiom to treat the template as a matrix. */
+    inline const File &operator()() const { return file; }
     inline cv::Mat &operator=(const cv::Mat &other) { return m() = other; } /*!< \brief Idiom to treat the template as a matrix. */
     inline operator const cv::Mat&() const { return m(); } /*!< \brief Idiom to treat the template as a matrix. */
     inline operator cv::Mat&() { return m(); } /*!< \brief Idiom to treat the template as a matrix. */
@@ -406,7 +417,6 @@ struct TemplateList : public QList&lt;Template&gt;
     QList<int> indexProperty(const QString & propName, QHash<QString, int> & valueMap, QHash<int, QVariant> & reverseLookup) const;
     QList<int> applyIndex(const QString & propName, const QHash<QString, int> & valueMap) const;
-
     /*!
      * \brief Returns the total number of bytes in all the templates.
      */
@@ -477,23 +487,6 @@ struct TemplateList : public QList&lt;Template&gt;
     FileList operator()() const { return files(); }
     /*!
-     * \brief Returns br::Template::label() for each template in the list.
-     */
-    template<typename T>
-    QList<T> get(const QString & propName) const
-    {
-        QList<T> values; values.reserve(size());
-        foreach (const Template &t, *this) values.append(t.file.get<T>(propName));
-        return values;
-    }
-    QList<QVariant> values(const QString & propName) const
-    {
-        QList<QVariant> values; values.reserve(size());
-        foreach (const Template &t, *this) values.append(t.file.value(propName));
-        return values;
-    }
-
-    /*!
      * \brief Returns the number of occurences for each label in the list.
      */
     template<typename T>
@@ -814,7 +807,8 @@ struct Factory
     //! [Factory make]
     static T *make(const File &file)
     {
-        QString name = file.suffix();
+        QString name = file.get<QString>("plugin", "");
+        if (name.isEmpty()) name = file.suffix();
         if (!names().contains(name)) {
             if      (names().contains("Empty") && name.isEmpty()) name = "Empty";
             else if (names().contains("Default"))                 name = "Default";
@@ -1024,7 +1018,7 @@ public:
     virtual TemplateList readBlock(bool *done) = 0; /*!< \brief Retrieve a portion of the stored templates. */
     void writeBlock(const TemplateList &templates); /*!< \brief Serialize a template list. */
     virtual void write(const Template &t) = 0; /*!< \brief Serialize a template. */
-    static Gallery *make(const File &file); /*!< \brief Make a gallery from a file list. */
+    static Gallery *make(const File &file); /*!< \brief Make a gallery to/from a file on disk. */
 private:
     QSharedPointer<Gallery> next;
@@ -348,7 +348,7 @@ class LDATransform : public Transform
         // OpenBR ensures that class values range from 0 to numClasses-1.
         // Label exists because we created it earlier with relabel
-        QList<int> classes = trainingSet.get<int>("Label");
+        QList<int> classes = File::get<int>(trainingSet, "Label");
         QMap<int, int> classCounts = trainingSet.countValues<int>("Label");
         const int numClasses = classCounts.size();
@@ -881,6 +881,45 @@ class statGallery : public Gallery
 BR_REGISTER(Gallery, statGallery)
+/*!
+ * \ingroup galleries
+ * \brief Implements the FDDB detection format.
+ * \author Josh Klontz \cite jklontz
+ *
+ * http://vis-www.cs.umass.edu/fddb/README.txt
+ */
+class FDDBGallery : public Gallery
+{
+    Q_OBJECT
+
+    TemplateList readBlock(bool *done)
+    {
+        *done = true;
+        QStringList lines = QtUtils::readLines(file);
+        TemplateList templates;
+        while (!lines.empty()) {
+            const QString fileName = lines.takeFirst();
+            int numDetects = lines.takeFirst().toInt();
+            for (int i=0; i<numDetects; i++) {
+                const QStringList detect = lines.takeFirst().split(' ');
+                Template t(fileName);
+                t.file.set("Face", QRectF(detect[0].toFloat(), detect[1].toFloat(), detect[2].toFloat(), detect[3].toFloat()));
+                t.file.set("Confidence", detect[4].toFloat());
+                templates.append(t);
+            }
+        }
+        return templates;
+    }
+
+    void write(const Template &t)
+    {
+        (void) t;
+        qFatal("Not implemented.");
+    }
+};
+
+BR_REGISTER(Gallery, FDDBGallery)
+
 } // namespace br
 #include "gallery.moc"
@@ -20,7 +20,7 @@ static TemplateList Downsample(const TemplateList &amp;templates, const Transform *t
     const bool atLeast = transform->instances < 0;
     const int instances = abs(transform->instances);
-    QList<QString> allLabels = templates.get<QString>("Subject");
+    QList<QString> allLabels = File::get<QString>(templates, "Subject");
     QList<QString> uniqueLabels = allLabels.toSet().toList();
     qSort(uniqueLabels);
@@ -35,9 +35,9 @@
 #include "openbr_internal.h"
 #include "openbr/core/bee.h"
+#include "openbr/core/eval.h"
 #include "openbr/core/common.h"
 #include "openbr/core/opencvutils.h"
-#include "openbr/core/plot.h"
 #include "openbr/core/qtutils.h"
 namespace br
@@ -146,8 +146,8 @@ class meltOutput : public MatrixOutput
         QStringList lines;
         if (file.baseName() != "terminal") lines.append(QString("Query,Target,Mask,Similarity%1").arg(keys));
-        QList<QString> queryLabels = queryFiles.get<QString>("Subject");
-        QList<QString> targetLabels = targetFiles.get<QString>("Subject");
+        QList<QString> queryLabels = File::get<QString>(queryFiles, "Subject");
+        QList<QString> targetLabels = File::get<QString>(targetFiles, "Subject");
         for (int i=0; i<queryFiles.size(); i++) {
             for (int j=(selfSimilar ? i+1 : 0); j<targetFiles.size(); j++) {
@@ -130,7 +130,7 @@ private:
         Mat lab;
         // If we are doing regression, assume subject has float values
         if (type == EPS_SVR || type == NU_SVR) {
-            lab = OpenCVUtils::toMat(_data.get<float>("Subject"));
+            lab = OpenCVUtils::toMat(File::get<float>(_data, "Subject"));
         }
         // If we are doing classification, assume subject has discrete values, map them
         // and store the mapping data
@@ -48,3 +48,19 @@ if [ ! -d ../data/MEDS/img ]; then
   mv data/*/*.jpg ../data/MEDS/img
   rm -r data NIST_SD32_MEDS-II_face.zip
 fi
+
+# KTH
+if [ ! -d ../data/KTH/vid ]; then
+  echo "Downloading KTH..."
+  mkdir ../data/KTH/vid
+  for vidclass in {'boxing','handclapping','handwaving','jogging','running','walking'}; do
+    if hash curl 2>/dev/null; then
+      curl -OL http://www.nada.kth.se/cvap/actions/${vidclass}.zip
+    else
+      wget http://www.nada.kth.se/cvap/actions/${vidclass}.zip
+    fi
+    mkdir ../data/KTH/vid/${vidclass}
+    unzip ${vidclass}.zip -d ../data/KTH/vid/${vidclass}
+	rm ${vidclass}.zip
+  done
+fi