Add the ability to match object labels in the detection evaluation

Jordan Cheney
1 parent 34953f33
Showing 7 changed files with 68 additions and 21 deletions
app/br/br.cpp
openbr/core/eval.cpp
openbr/core/eval.h
openbr/core/evalutils.cpp
openbr/core/evalutils.h
openbr/openbr.cpp
openbr/openbr.h
@@ -160,8 +160,8 @@ public:
                 check((parc >= 2) && (parc <= 5), "Incorrect parameter count for 'evalClustering'.");
                 br_eval_clustering(parv[0], parv[1], parc > 2 ? parv[2] : "", parc > 3 ? atoi(parv[3]) : 1, parc > 4 ? parv[4] : "");
             } else if (!strcmp(fun, "evalDetection")) {
-                check((parc >= 2) && (parc <= 7), "Incorrect parameter count for 'evalDetection'.");
-                br_eval_detection(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? atoi(parv[3]) : 0, parc >= 5 ? atoi(parv[4]) : 0, parc >= 6 ? atoi(parv[5]) : 0, parc >= 7 ? atof(parv[6]) : 0);
+                check((parc >= 2) && (parc <= 8), "Incorrect parameter count for 'evalDetection'.");
+                br_eval_detection(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? atoi(parv[3]) : 0, parc >= 5 ? atoi(parv[4]) : 0, parc >= 6 ? atoi(parv[5]) : 0, parc >= 7 ? atof(parv[6]) : 0, parc >= 8 ? parv[7] : "");
             } else if (!strcmp(fun, "evalLandmarking")) {
                 check((parc >= 2) && (parc <= 7), "Incorrect parameter count for 'evalLandmarking'.");
                 br_eval_landmarking(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? atoi(parv[3]) : 0, parc >= 5 ? atoi(parv[4]) : 1,  parc >= 6 ? atoi(parv[5]) : 0, parc >= 7 ? atoi(parv[6]) : 5);
@@ -290,7 +290,7 @@ private:
                "-convert (Format|Gallery|Output) <input_file> {output_file}\n"
                "-evalClassification <predicted_gallery> <truth_gallery> <predicted property name> <ground truth proprty name>\n"
                "-evalClustering <clusters> <truth_gallery> [truth_property [cluster_csv [cluster_property]]]\n"
-               "-evalDetection <predicted_gallery> <truth_gallery> [{csv}] [{normalize}] [{minSize}] [{maxSize}]\n"
+               "-evalDetection <predicted_gallery> <truth_gallery> [{csv}] [{normalize}] [{minSize}] [{maxSize}] [{label_filter}]\n"
                "-evalLandmarking <predicted_gallery> <truth_gallery> [{csv} [<normalization_index_a> <normalization_index_b>] [sample_index] [total_examples]]\n"
                "-evalRegression <predicted_gallery> <truth_gallery> <predicted property name> <ground truth property name>\n"
                "-evalKNN <knn_graph> <knn_truth> [{csv}]\n"
@@ -769,7 +769,7 @@ void EvalClassification(const QString &amp;predictedGallery, const QString &amp;truthGal
     qDebug("Overall Accuracy = %f", (float)tpc / (float)(tpc + fnc));
 }
  
-float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv, bool normalize, int minSize, int maxSize, float relativeMinSize)
+float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv, bool normalize, int minSize, int maxSize, float relativeMinSize, const QString &label)
 {
     qDebug("Evaluating detection of %s against %s", qPrintable(predictedGallery), qPrintable(truthGallery));
     // Organized by file, QMap used to preserve order
@@ -789,6 +789,15 @@ float EvalDetection(const QString &amp;predictedGallery, const QString &amp;truthGallery
         allDetections = filterDetections(allDetections, maxSize, false);
     }
  
+    // Optionally, keep only detections with a specific label
+    if (!label.isEmpty()) {
+        if (Globals->verbose)
+            qDebug("Removing detections without label %s\n", qPrintable(label));
+        allDetections = filterLabels(allDetections, label);
+        if (allDetections.isEmpty())
+            qFatal("No detections left after filtering on label. Check your filter");
+    }
+
     QList<ResolvedDetection> resolvedDetections, falseNegativeDetections;
     QRectF normalizations(0, 0, 0, 0);
  
@@ -30,7 +30,7 @@ namespace br
     float InplaceEval(const QString & simmat, const QString & target, const QString & query, const QString & csv = "");
  
     void EvalClassification(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = "");
-    float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", bool normalize = false, int minSize = 0, int maxSize = 0, float relativeMinSize = 0); // Return average overlap
+    float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", bool normalize = false, int minSize = 0, int maxSize = 0, float relativeMinSize = 0, const QString &label = ""); // Return average overlap
     float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", int normalizationIndexA = 0, int normalizationIndexB = 1, int sampleIndex = 0, int totalExamples = 5); // Return average error
     void EvalRegression(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = "");
     void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &csv = "");
@@ -47,28 +47,34 @@ DetectionKey EvalUtils::getDetectKey(const FileList &amp;files)
 // return a list of detections independent of the detection key format
 QList<Detection> EvalUtils::getDetections(const DetectionKey &key, const File &f, bool isTruth)
 {
-    QString pose = f.get<QString>("Pose");
+    QString pose = f.get<QString>("Pose", "Frontal");
     if (pose.contains("Angle"))
         pose = "Frontal";
  
+    QString label = f.get<QString>("Label", "");
+
     const QString filePath = f.path() + "/" + f.fileName();
     QList<Detection> dets;
     if (key.type == DetectionKey::RectList) {
         QList<QRectF> rects = f.rects();
         QList<float> confidences = f.getList<float>("Confidences", QList<float>());
+        QList<QString> labels = f.getList<QString>("Labels", QList<QString>());
         if (!isTruth && rects.size() != confidences.size())
             qFatal("You don't have enough confidence. I mean, your detections don't all have confidence measures.");
+        if (!labels.empty() && rects.size() != labels.size())
+            qFatal("Some of your rects have labels but not all, it's all or nothing I'm afraid");
+
         for (int i=0; i<rects.size(); i++) {
             if (isTruth)
-                dets.append(Detection(rects[i], filePath));
+                dets.append(Detection(rects[i], filePath, -1, false, "Frontal", labels.empty() ? "" : labels[i]));
             else
-                dets.append(Detection(rects[i], filePath, confidences[i]));
+                dets.append(Detection(rects[i], filePath, confidences[i], false, "Frontal", labels.empty() ? "" : labels[i]));
         }
     } else if (key.type == DetectionKey::Rect) {
-        dets.append(Detection(f.get<QRectF>(key), filePath, isTruth ? -1 : f.get<float>("Confidence", -1), f.get<bool>("Ignore", false), pose));
+        dets.append(Detection(f.get<QRectF>(key), filePath, isTruth ? -1 : f.get<float>("Confidence", -1), f.get<bool>("Ignore", false), pose, label));
     } else if (key.type == DetectionKey::XYWidthHeight) {
         const QRectF rect(f.get<float>(key+"_X"), f.get<float>(key+"_Y"), f.get<float>(key+"_Width"), f.get<float>(key+"_Height"));
-        dets.append(Detection(rect, filePath, isTruth ? -1 : f.get<float>("Confidence", -1), f.get<bool>("Ignore", false), pose));
+        dets.append(Detection(rect, filePath, isTruth ? -1 : f.get<float>("Confidence", -1), f.get<bool>("Ignore", false), pose, label));
     }
     return dets;
 }
@@ -128,6 +134,27 @@ QMap&lt;QString, Detections&gt; EvalUtils::filterDetections(const QMap&lt;QString, Detect
         }
         if (!filteredDetections.truth.empty()) allFilteredDetections[key] = filteredDetections;
     }
+    
+    return allFilteredDetections;
+}
+
+QMap<QString, Detections> EvalUtils::filterLabels(const QMap<QString, Detections> &allDetections, const QString &label)
+{
+    QMap<QString, Detections> allFilteredDetections;
+    foreach (QString key, allDetections.keys()) {
+        Detections detections = allDetections[key];
+        Detections filteredDetections;
+        for (int i = 0; i < detections.predicted.size(); i++) {
+            if (detections.predicted[i].label == label)
+                filteredDetections.predicted.append(detections.predicted[i]);
+        }
+        for (int i = 0; i < detections.truth.size(); i++) {
+            if (detections.truth[i].label == label)
+                filteredDetections.truth.append(detections.truth[i]);
+        }
+        if (!filteredDetections.truth.empty()) allFilteredDetections[key] = filteredDetections;
+    }
+
     return allFilteredDetections;
 }
  
@@ -147,6 +174,10 @@ int EvalUtils::associateGroundTruthDetections(QList&lt;ResolvedDetection&gt; &amp;resolved
             for (int p = 0; p < detections.predicted.size(); p++) {
                 Detection predicted = detections.predicted[p];
  
+                // Only boxes of the same class can overlap
+                if (predicted.label != truth.label)
+                    continue;
+
                 float predictedWidth = predicted.boundingBox.width();
                 float x, y, width, height;
                 x = predicted.boundingBox.x() + offsets.x()*predictedWidth;
@@ -175,7 +206,7 @@ int EvalUtils::associateGroundTruthDetections(QList&lt;ResolvedDetection&gt; &amp;resolved
             const Detection predicted = detections.predicted[detection.predicted_idx];
  
             if (!truth.ignore)
-                resolved.append(ResolvedDetection(predicted.filePath, predicted.boundingBox, predicted.confidence, detection.overlap, truth.boundingBox, truth.pose == predicted.pose));
+                resolved.append(ResolvedDetection(predicted.filePath, predicted.boundingBox, predicted.confidence, detection.overlap, truth.boundingBox, truth.pose == predicted.pose, truth.label));
  
             removedTruth.append(detection.truth_idx);
             removedPredicted.append(detection.predicted_idx);
@@ -192,11 +223,11 @@ int EvalUtils::associateGroundTruthDetections(QList&lt;ResolvedDetection&gt; &amp;resolved
  
         // False positive
         for (int i = 0; i < detections.predicted.size(); i++)
-            if (!removedPredicted.contains(i)) resolved.append(ResolvedDetection(detections.predicted[i].filePath, detections.predicted[i].boundingBox, detections.predicted[i].confidence, 0, QRectF(), false));
+            if (!removedPredicted.contains(i)) resolved.append(ResolvedDetection(detections.predicted[i].filePath, detections.predicted[i].boundingBox, detections.predicted[i].confidence, 0, QRectF(), false, detections.predicted[i].label));
  
         // False negative
         for (int i = 0; i < detections.truth.size(); i++)
-            if (!removedTruth.contains(i) && !detections.truth[i].ignore) falseNegative.append(ResolvedDetection(detections.truth[i].filePath, detections.truth[i].boundingBox, -std::numeric_limits<float>::max(), 0, QRectF(), false));
+            if (!removedTruth.contains(i) && !detections.truth[i].ignore) falseNegative.append(ResolvedDetection(detections.truth[i].filePath, detections.truth[i].boundingBox, -std::numeric_limits<float>::max(), 0, QRectF(), false, detections.truth[i].label));
     }
  
     if (offsets.x() == 0) {
@@ -7,7 +7,8 @@
  
 namespace EvalUtils
 {
-    struct Detection
+    
+struct Detection
 {
     QRectF boundingBox;
     QString filePath;
@@ -18,14 +19,17 @@ namespace EvalUtils
     // true negative, or false negative, it will simply be ignored.
     bool ignore;
     QString pose;
+    // The label field can be used to distinguish between different object classes
+    QString label;
  
     Detection() {}
-    Detection(const QRectF &boundingBox, const QString &filePath = QString(), float confidence = -1, bool ignore = false, const QString &pose = "Frontal") :
+    Detection(const QRectF &boundingBox, const QString &filePath = QString(), float confidence = -1, bool ignore = false, const QString &pose = "Frontal", const QString &label = "") :
         boundingBox(boundingBox),
         filePath(filePath),
         confidence(confidence),
         ignore(ignore),
-        pose(pose)
+        pose(pose),
+        label(label)
     {}
  
     float overlap(const Detection &other) const
@@ -50,18 +54,20 @@ struct ResolvedDetection
     QRectF boundingBox, groundTruthBoundingBox;
     float confidence, overlap;
     bool poseMatch;
+    QString label;
     ResolvedDetection() :
     confidence(-1),
         overlap(-1)
         {}
  
-ResolvedDetection(const QString &filePath, const QRectF &boundingBox, float confidence, float overlap, const QRectF &groundTruthBoundingBox, bool poseMatch) :
+ResolvedDetection(const QString &filePath, const QRectF &boundingBox, float confidence, float overlap, const QRectF &groundTruthBoundingBox, bool poseMatch, const QString &label) :
     filePath(filePath),
     boundingBox(boundingBox),
     groundTruthBoundingBox(groundTruthBoundingBox),
     confidence(confidence),
     overlap(overlap),
-    poseMatch(poseMatch)
+    poseMatch(poseMatch),
+    label(label)
     {}
  
     inline bool operator<(const ResolvedDetection &other) const { return confidence > other.confidence; }
@@ -99,6 +105,7 @@ struct DetectionOperatingPoint
     QList<Detection> getDetections(const DetectionKey &key, const br::File &f, bool isTruth);
     QMap<QString, Detections> getDetections(const br::File &predictedGallery, const br::File &truthGallery);
     QMap<QString, Detections> filterDetections(const QMap<QString, Detections> &allDetections, int threshold, bool useMin = true, float relativeThreshold = 0);
+    QMap<QString, Detections> filterLabels(const QMap<QString, Detections> &allDetections, const QString &label);
     int associateGroundTruthDetections(QList<ResolvedDetection> &resolved, QList<ResolvedDetection> &falseNegative, QMap<QString, Detections> &all, QRectF &offsets);
     QStringList computeDetectionResults(const QList<ResolvedDetection> &detections, int totalTrueDetections, int numImages, bool discrete, QList<DetectionOperatingPoint> &points);
     inline int getNumberOfImages(const QMap<QString, Detections> detections)
@@ -126,9 +126,9 @@ void br_eval_clustering(const char *clusters, const char *truth_gallery, const c
     EvalClustering(clusters, truth_gallery, truth_property, cluster_csv, cluster_property);
 }
  
-float br_eval_detection(const char *predicted_gallery, const char *truth_gallery, const char *csv, bool normalize, int minSize, int maxSize, float relativeMinSize)
+float br_eval_detection(const char *predicted_gallery, const char *truth_gallery, const char *csv, bool normalize, int minSize, int maxSize, float relativeMinSize, const char* label)
 {
-    return EvalDetection(predicted_gallery, truth_gallery, csv, normalize, minSize, maxSize, relativeMinSize);
+    return EvalDetection(predicted_gallery, truth_gallery, csv, normalize, minSize, maxSize, relativeMinSize, label);
 }
  
 float br_eval_landmarking(const char *predicted_gallery, const char *truth_gallery, const char *csv, int normalization_index_a, int normalization_index_b, int sample_index, int total_examples)
@@ -58,7 +58,7 @@ BR_EXPORT void br_eval_classification(const char *predicted_gallery, const char 
  
 BR_EXPORT void br_eval_clustering(const char *clusters, const char *truth_gallery, const char *truth_property = "", bool cluster_csv = true, const char *cluster_property = "");
  
-BR_EXPORT float br_eval_detection(const char *predicted_gallery, const char *truth_gallery, const char *csv = "", bool normalize = false, int minSize = 0, int maxSize = 0, float relativeMinSize = 0);
+BR_EXPORT float br_eval_detection(const char *predicted_gallery, const char *truth_gallery, const char *csv = "", bool normalize = false, int minSize = 0, int maxSize = 0, float relativeMinSize = 0, const char* label = "");
  
 BR_EXPORT float br_eval_landmarking(const char *predicted_gallery, const char *truth_gallery, const char *csv = "", int normalization_index_a = 0, int normalization_index_b = 1, int sample_index = 0, int total_examples = 5);