Merge pull request #283 from biometrics/evaluation_updates

Updates to eval_detection

Merge pull request #283 from biometrics/evaluation_updates
Updates to eval_detection
Josh Klontz
2 parents 453d5f49 fa88ae6a
Showing 5 changed files with 110 additions and 56 deletions
app/br/br.cpp
openbr/core/eval.cpp
openbr/core/eval.h
openbr/openbr.cpp
openbr/openbr.h
@@ -160,8 +160,8 @@ public:
                 check((parc >= 2) && (parc <= 3), "Incorrect parameter count for 'evalClustering'.");
                 br_eval_clustering(parv[0], parv[1], parc == 3 ? parv[2] : "");
             } else if (!strcmp(fun, "evalDetection")) {
-                check((parc >= 2) && (parc <= 5), "Incorrect parameter count for 'evalDetection'.");
-                br_eval_detection(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? atoi(parv[3]) : 0, parc == 5 ? atoi(parv[4]) : 0);
+                check((parc >= 2) && (parc <= 6), "Incorrect parameter count for 'evalDetection'.");
+                br_eval_detection(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? atoi(parv[3]) : 0, parc >= 5 ? atoi(parv[4]) : 0, parc == 6 ? atoi(parv[5]) : 0);
             } else if (!strcmp(fun, "evalLandmarking")) {
                 check((parc >= 2) && (parc <= 5), "Incorrect parameter count for 'evalLandmarking'.");
                 br_eval_landmarking(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? atoi(parv[3]) : 0, parc >= 5 ? atoi(parv[4]) : 1);
@@ -666,10 +666,15 @@ struct Detection
 {
     QRectF boundingBox;
     float confidence;
+    // The ignore flag is useful when certain faces in an image should be ignored
+    // and should not effect detection performance. Predicted detections that overlap
+    // with an ignored truth detection will not count as a true positive, false positive,
+    // true negative, or false negative, it will simply be ignored.
+    bool ignore;
     Detection() {}
-    Detection(const QRectF &boundingBox_, float confidence_ = -1)
-        : boundingBox(boundingBox_), confidence(confidence_) {}
+    Detection(const QRectF &boundingBox_, float confidence_ = -1, bool ignore_ = false)
+        : boundingBox(boundingBox_), confidence(confidence_), ignore(ignore_) {}
     float area() const { return boundingBox.width() * boundingBox.height(); }
     float overlap(const Detection &other) const
@@ -679,6 +684,16 @@ struct Detection
     }
 };
+struct SortedDetection
+{
+    int truth_idx, predicted_idx;
+    float overlap;
+    SortedDetection() : truth_idx(-1), predicted_idx(-1), overlap(-1) {}
+    SortedDetection(int truth_idx_, int predicted_idx_, float overlap_)
+        : truth_idx(truth_idx_), predicted_idx(predicted_idx_), overlap(overlap_) {}
+    inline bool operator<(const SortedDetection &other) const { return overlap > other.overlap; }
+};
+
 struct Detections
 {
     QList<Detection> predicted, truth;
@@ -704,6 +719,7 @@ static QStringList computeDetectionResults(const QList&lt;ResolvedDetection&gt; &amp;detec
 {
     QList<DetectionOperatingPoint> points;
     float TP = 0, FP = 0, prevFP = -1;
+
     for (int i=0; i<detections.size(); i++) {
         const ResolvedDetection &detection = detections[i];
         if (discrete) {
@@ -716,6 +732,13 @@ static QStringList computeDetectionResults(const QList&lt;ResolvedDetection&gt; &amp;detec
         }
         if ((i == detections.size()-1) || (detection.confidence > detections[i+1].confidence)) {
             if (FP > prevFP || (i == detections.size()-1)) {
+                if (prevFP / numImages < 0.1 && FP / numImages > 0.1 && discrete) {
+                    qDebug("TAR @ FAR => %f : 0.1", TP / totalTrueDetections);
+                    qDebug("Confidence: %f", detection.confidence);
+                } else if (prevFP / numImages < 0.01 && FP / numImages > 0.01 && discrete) {
+                    qDebug("TAR @ FAR => %f : 0.01", TP / totalTrueDetections);
+                    qDebug("Confidence: %f", detection.confidence);
+                }
                 points.append(DetectionOperatingPoint(TP, FP, totalTrueDetections, numImages));
                 prevFP = FP;
             }
@@ -803,7 +826,7 @@ static QList&lt;Detection&gt; getDetections(const DetectionKey &amp;key, const File &amp;f, bo
         dets.append(Detection(f.get<QRectF>(key), isTruth ? -1 : f.get<float>("Confidence", -1)));
     } else if (key.type == DetectionKey::XYWidthHeight) {
         const QRectF rect(f.get<float>(key+"_X"), f.get<float>(key+"_Y"), f.get<float>(key+"_Width"), f.get<float>(key+"_Height"));
-        dets.append(Detection(rect, isTruth ? -1 : f.get<float>("Confidence", -1)));
+        dets.append(Detection(rect, isTruth ? -1 : f.get<float>("Confidence", -1), f.get<bool>("Ignore", false)));
     }
     return dets;
 }
@@ -827,21 +850,16 @@ static QMap&lt;QString, Detections&gt; getDetections(const File &amp;predictedGallery, con
            qPrintable(predictedDetectKey == truthDetectKey ? QString() : "/"+truthDetectKey));
     QMap<QString, Detections> allDetections;
-    foreach (const File &f, predicted)
-        allDetections[f.baseName()].predicted.append(getDetections(predictedDetectKey, f, false));
     foreach (const File &f, truth)
-        allDetections[f.baseName()].truth.append(getDetections(truthDetectKey, f, true));
+        allDetections[f.name].truth.append(getDetections(truthDetectKey, f, true));
+    foreach (const File &f, predicted)
+        if (allDetections.contains(f.name)) allDetections[f.name].predicted.append(getDetections(predictedDetectKey, f, false));
     return allDetections;
 }
-static int getNumberOfImages(const File &truthGallery)
-{
-    const FileList files = FileList::fromGallery(truthGallery);
-    
-    QSet<QString> names;
-    foreach(const File &file, files)
-        names.insert(file.fileName());
-    return names.size();
+static inline int getNumberOfImages(const QMap<QString, Detections> detections)
+{   
+    return detections.keys().size();
 }
 static int associateGroundTruthDetections(QList<ResolvedDetection> &resolved, QList<ResolvedDetection> &falseNegative, QMap<QString, Detections> &all, QRectF &offsets)
@@ -852,13 +870,13 @@ static int associateGroundTruthDetections(QList&lt;ResolvedDetection&gt; &amp;resolved, QL
     foreach (Detections detections, all.values()) {
         totalTrueDetections += detections.truth.size();
         // Try to associate ground truth detections with predicted detections
-        while (!detections.truth.isEmpty() && !detections.predicted.isEmpty()) {
-            const Detection truth = detections.truth.takeFirst(); // Take removes the detection
-            int bestIndex = -1;
-            float bestOverlap = -std::numeric_limits<float>::max();
-            // Find the nearest predicted detection to this ground truth detection
-            for (int i=0; i<detections.predicted.size(); i++) {
-                Detection predicted = detections.predicted[i];
+
+        QList<SortedDetection> sortedDetections; sortedDetections.reserve(detections.truth.size() * detections.predicted.size());
+        for (int t = 0; t < detections.truth.size(); t++) {
+            const Detection truth = detections.truth[t];
+            for (int p = 0; p < detections.predicted.size(); p++) {
+                Detection predicted = detections.predicted[p];
+
                 float predictedWidth = predicted.boundingBox.width();
                 float x, y, width, height;
                 x = predicted.boundingBox.x() + offsets.x()*predictedWidth;
@@ -868,35 +886,44 @@ static int associateGroundTruthDetections(QList&lt;ResolvedDetection&gt; &amp;resolved, QL
                 Detection newPredicted(QRectF(x, y, width, height), 0.0);
                 const float overlap = truth.overlap(newPredicted);
-                if (overlap > bestOverlap) {
-                    bestOverlap = overlap;
-                    bestIndex = i;
-                }
+                if (overlap > 0)
+                    sortedDetections.append(SortedDetection(t, p, overlap));
             }
-            // Removing the detection prevents us from considering it twice.
-            // We don't want to associate two ground truth detections with the
-            // same prediction, over vice versa.
-            const Detection predicted = detections.predicted.takeAt(bestIndex);
-            resolved.append(ResolvedDetection(predicted.confidence, bestOverlap));
-
-            if (offsets.x() == 0) {
-                // Add side differences to total only for pairs that meet the overlap threshold.
-                if (bestOverlap > 0.3) {
-                    count++;
-                    float width = predicted.boundingBox.width();
-                    dLeftTotal += (truth.boundingBox.left() - predicted.boundingBox.left()) / width;
-                    dRightTotal += (truth.boundingBox.right() - predicted.boundingBox.right()) / width;
-                    dTopTotal += (truth.boundingBox.top() - predicted.boundingBox.top()) / width;
-                    dBottomTotal += (truth.boundingBox.bottom() - predicted.boundingBox.bottom()) / width;
-                }
+        }
+
+        std::sort(sortedDetections.begin(), sortedDetections.end());
+
+        QList<int> removedTruth;
+        QList<int> removedPredicted;
+
+        foreach (const SortedDetection &detection, sortedDetections) {
+            if (removedTruth.contains(detection.truth_idx) || removedPredicted.contains(detection.predicted_idx))
+                continue;
+
+            const Detection truth = detections.truth[detection.truth_idx];
+            const Detection predicted = detections.predicted[detection.predicted_idx];
+
+            if (!truth.ignore) resolved.append(ResolvedDetection(predicted.confidence, detection.overlap));
+
+            removedTruth.append(detection.truth_idx);
+            removedPredicted.append(detection.predicted_idx);
+
+            if (offsets.x() == 0 && detection.overlap > 0.3) {
+                count++;
+                float width = predicted.boundingBox.width();
+                dLeftTotal += (truth.boundingBox.left() - predicted.boundingBox.left()) / width;
+                dRightTotal += (truth.boundingBox.right() - predicted.boundingBox.right()) / width;
+                dTopTotal += (truth.boundingBox.top() - predicted.boundingBox.top()) / width;
+                dBottomTotal += (truth.boundingBox.bottom() - predicted.boundingBox.bottom()) / width;
             }
         }
-        foreach (const Detection &detection, detections.predicted)
-            resolved.append(ResolvedDetection(detection.confidence, 0));
-        for (int i=0; i<detections.truth.size(); i++)
-            falseNegative.append(ResolvedDetection(-std::numeric_limits<float>::max(), 0));
+        for (int i = 0; i < detections.predicted.size(); i++)
+            if (!removedPredicted.contains(i)) resolved.append(ResolvedDetection(detections.predicted[i].confidence, 0));
+        for (int i = 0; i < detections.truth.size(); i++)
+            if (!removedTruth.contains(i) && !detections.truth[i].ignore) falseNegative.append(ResolvedDetection(-std::numeric_limits<float>::max(), 0));
     }
+
     if (offsets.x() == 0) {
         // Calculate average differences in each direction
         float dRight = dRightTotal / count;
@@ -914,7 +941,7 @@ static int associateGroundTruthDetections(QList&lt;ResolvedDetection&gt; &amp;resolved, QL
     return totalTrueDetections;
 }
-float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv, bool normalize, int minSize)
+float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv, bool normalize, int minSize, int maxSize)
 {
     qDebug("Evaluating detection of %s against %s", qPrintable(predictedGallery), qPrintable(truthGallery));
     // Organized by file, QMap used to preserve order
@@ -923,12 +950,13 @@ float EvalDetection(const QString &amp;predictedGallery, const QString &amp;truthGallery
     // Remove any bounding boxes with a side smaller than minSize
     if (minSize > 0) {
         qDebug("Removing boxes smaller than %d\n", minSize);
+        QMap<QString, Detections> allFilteredDetections;
         foreach (QString key, allDetections.keys()) {
             Detections detections = allDetections[key];
             Detections filteredDetections;
             for (int i = 0; i < detections.predicted.size(); i++) {
                 QRectF box = detections.predicted[i].boundingBox;
-                if (min(box.width(), box.height()) > minSize) {
+                if (min(box.width(), box.height()) > sqrt(0.5 * pow(minSize, 2))) {
                     filteredDetections.predicted.append(detections.predicted[i]);
                 }
             }
@@ -939,8 +967,34 @@ float EvalDetection(const QString &amp;predictedGallery, const QString &amp;truthGallery
                     filteredDetections.truth.append(detections.truth[i]);
                 }
             }
-            allDetections.insert(key, filteredDetections);
+            if (!filteredDetections.truth.empty()) allFilteredDetections[key] = filteredDetections;
+        }
+        allDetections = allFilteredDetections;
+    }
+
+    // Remove any bounding boxes with no side smaller than maxSize
+    if (maxSize > 0) {
+        qDebug("Removing boxes larger than %d\n", maxSize);
+        QMap<QString, Detections> allFilteredDetections;
+        foreach (QString key, allDetections.keys()) {
+            Detections detections = allDetections[key];
+            Detections filteredDetections;
+            for (int i = 0; i < detections.predicted.size(); i++) {
+                QRectF box = detections.predicted[i].boundingBox;
+                if (min(box.width(), box.height()) < sqrt(0.5 * pow(maxSize, 2))) {
+                    filteredDetections.predicted.append(detections.predicted[i]);
+                }
+            }
+
+            for (int i = 0; i < detections.truth.size(); i++) {
+                QRectF box = detections.truth[i].boundingBox;
+                if (min(box.width(), box.height()) < maxSize) {
+                    filteredDetections.truth.append(detections.truth[i]);
+                }
+            }
+            if (!filteredDetections.truth.empty()) allFilteredDetections[key] = filteredDetections;
         }
+        allDetections = allFilteredDetections;
     }
     QList<ResolvedDetection> resolvedDetections, falseNegativeDetections;
@@ -963,8 +1017,8 @@ float EvalDetection(const QString &amp;predictedGallery, const QString &amp;truthGallery
     std::sort(resolvedDetections.begin(), resolvedDetections.end());
     QStringList lines;
     lines.append("Plot, X, Y");
-    lines.append(computeDetectionResults(resolvedDetections, totalTrueDetections, getNumberOfImages(truthGallery), true));
-    lines.append(computeDetectionResults(resolvedDetections, totalTrueDetections, getNumberOfImages(truthGallery), false));
+    lines.append(computeDetectionResults(resolvedDetections, totalTrueDetections, getNumberOfImages(allDetections), true));
+    lines.append(computeDetectionResults(resolvedDetections, totalTrueDetections, getNumberOfImages(allDetections), false));
     float averageOverlap;
     { // Overlap Density
@@ -30,7 +30,7 @@ namespace br
     float InplaceEval(const QString & simmat, const QString & target, const QString & query, const QString & csv = "");
     void EvalClassification(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = "");
-    float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", bool normalize = false, int minSize = 0); // Return average overlap
+    float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", bool normalize = false, int minSize = 0, int maxSize = 0); // Return average overlap
     float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", int normalizationIndexA = 0, int normalizationIndexB = 1); // Return average error
     void EvalRegression(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = "");
 }
@@ -129,9 +129,9 @@ void br_eval_clustering(const char *csv, const char *gallery, const char *truth_
     EvalClustering(csv, gallery, truth_property);
 }
-float br_eval_detection(const char *predicted_gallery, const char *truth_gallery, const char *csv, bool normalize, int minSize)
+float br_eval_detection(const char *predicted_gallery, const char *truth_gallery, const char *csv, bool normalize, int minSize, int maxSize)
 {
-    return EvalDetection(predicted_gallery, truth_gallery, csv, normalize, minSize);
+    return EvalDetection(predicted_gallery, truth_gallery, csv, normalize, minSize, maxSize);
 }
 float br_eval_landmarking(const char *predicted_gallery, const char *truth_gallery, const char *csv, int normalization_index_a, int normalization_index_b)
@@ -205,7 +205,7 @@ BR_EXPORT void br_eval_clustering(const char *csv, const char *gallery, const ch
  * \param normalize Optional \c bool flag to normalize predicted bounding boxes for improved detection. 
  * \return Average detection bounding box overlap.
  */
-BR_EXPORT float br_eval_detection(const char *predicted_gallery, const char *truth_gallery, const char *csv = "", bool normalize = false, int minSize = 0);
+BR_EXPORT float br_eval_detection(const char *predicted_gallery, const char *truth_gallery, const char *csv = "", bool normalize = false, int minSize = 0, int maxSize = 0);
 /*!
  * \brief Evaluates and prints landmarking accuracy to terminal.