generalized evalClassification

Josh Klontz
1 parent 19a3de11
Showing 4 changed files with 25 additions and 17 deletions
openbr/core/classify.cpp
openbr/openbr_plugin.cpp
openbr/plugins/algorithms.cpp
openbr/plugins/cluster.cpp
@@ -14,8 +14,6 @@
  * limitations under the License.                                            *
  * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
-#include <QDebug>
-#include <QHash>
 #include <openbr/openbr_plugin.h>
 #include "classify.h"
@@ -24,7 +22,7 @@
 // Helper struct for statistics accumulation
 struct Counter
 {
-    int truePositive, falsePositive, falseNegative;
+    float truePositive, falsePositive, falseNegative;
     Counter()
     {
         truePositive = 0;
@@ -41,35 +39,44 @@ void br::EvalClassification(const QString &amp;predictedInput, const QString &amp;truthI
     TemplateList truth(TemplateList::fromGallery(truthInput));
     if (predicted.size() != truth.size()) qFatal("Input size mismatch.");
-    QHash<int, Counter> counters;
+    QHash<QString, Counter> counters;
     for (int i=0; i<predicted.size(); i++) {
         if (predicted[i].file.name != truth[i].file.name)
             qFatal("Input order mismatch.");
-        const int trueLabel = truth[i].file.label();
-        const int predictedLabel = predicted[i].file.label();
-        if (trueLabel == predictedLabel) {
-            counters[trueLabel].truePositive++;
-        } else {
-            counters[trueLabel].falseNegative++;
-            counters[predictedLabel].falsePositive++;
+        // Typically these lists will be of length one, but this generalization allows measuring multi-class labeling accuracy.
+        QStringList predictedSubjects = predicted[i].file.get<QStringList>("Subject");
+        QStringList trueSubjects = truth[i].file.get<QStringList>("Subject");
+        foreach (const QString &subject, trueSubjects.toVector() /* Hack to copy the list. */) {
+            if (predictedSubjects.contains(subject)) {
+                counters[subject].truePositive++;
+                trueSubjects.removeOne(subject);
+                predictedSubjects.removeOne(subject);
+            } else {
+                counters[subject].falseNegative++;
+            }
         }
+
+        for (int i=0; i<trueSubjects.size(); i++)
+            foreach (const QString &subject, predictedSubjects)
+                counters[subject].falsePositive += 1.f / predictedSubjects.size();
     }
     QSharedPointer<Output> output(Output::make("", FileList() << "Subject" << "Count" << "Precision" << "Recall" << "F-score", FileList(counters.size())));
     int tpc = 0;
     int fnc = 0;
+    const QStringList keys = counters.keys();
     for (int i=0; i<counters.size(); i++) {
-        int trueLabel = counters.keys()[i];
-        const Counter &counter = counters[trueLabel];
+        const QString &subject = keys[i];
+        const Counter &counter = counters[subject];
         tpc += counter.truePositive;
         fnc += counter.falseNegative;
         const int count = counter.truePositive + counter.falseNegative;
         const float precision = counter.truePositive / (float)(counter.truePositive + counter.falsePositive);
         const float recall = counter.truePositive / (float)(counter.truePositive + counter.falseNegative);
         const float fscore = 2 * precision * recall / (precision + recall);
-        output->setRelative(trueLabel, i, 0);
+        output->setRelative(File("", subject).label(), i, 0);
         output->setRelative(count, i, 1);
         output->setRelative(precision, i, 2);
         output->setRelative(recall, i, 3);
@@ -1025,7 +1025,7 @@ MatrixOutput *MatrixOutput::make(const FileList &amp;targetFiles, const FileList &amp;qu
 /* MatrixOutput - protected methods */
 QString MatrixOutput::toString(int row, int column) const
 {
-    if (targetFiles[column] == "Label") {
+    if (targetFiles[column] == "Subject") {
         const int label = data.at<float>(row,column);
         return Globals->subjects.key(label, QString::number(label));
     }
@@ -50,7 +50,7 @@ class AlgorithmsInitializer : public Initializer
         Globals->abbreviations.insert("SmallSIFT", "Open+LimitSize(512)+KeyPointDetector(SIFT)+KeyPointDescriptor(SIFT):KeyPointMatcher(BruteForce)");
         Globals->abbreviations.insert("SmallSURF", "Open+LimitSize(512)+KeyPointDetector(SURF)+KeyPointDescriptor(SURF):KeyPointMatcher(BruteForce)");
         Globals->abbreviations.insert("ColorHist", "Open+LimitSize(512)!EnsureChannels(3)+SplitChannels+Hist(256,0,8)+Cat+Normalize(L1):L2");
-        Globals->abbreviations.insert("ImageClassification", "Open+CropSquare+LimitSize(256)+Cvt(Gray)+Gradient+Bin(0,360,9,true)+Merge+Integral+RecursiveIntegralSampler(4,2,8,Singleton(KMeans(1024)[fraction=0.5]))+Cat+CvtFloat+Hist(1024)+KNS(5,Dist(L1))");
+        Globals->abbreviations.insert("ImageClassification", "Open+CropSquare+LimitSize(256)+Cvt(Gray)+Gradient+Bin(0,360,9,true)+Merge+Integral+RecursiveIntegralSampler(4,2,8,Singleton(KMeans(1024)[fraction=0.5]))+Cat+CvtFloat+Hist(1024)+KNS(5,Dist(L1))+Rename(KNS,Subject)");
         // Hash
         Globals->abbreviations.insert("FileName", "Name+Identity:Identical");
@@ -106,7 +106,8 @@ class KNSTransform : public Transform
             subjects.insert(gallery[sortedScores[i].second].file.subject());
             i++;
         }
-        dst.file.set("KNS", QStringList(subjects.toList()).join(", "));
+        const QStringList subjectList = subjects.toList();
+        dst.file.set("KNS", subjects.size() > 1 ? "[" + subjectList.join(",") + "]" : subjectList.first());
     }
     void store(QDataStream &stream) const