diff --git a/app/br/br.cpp b/app/br/br.cpp index dc5e2f4..9e79f37 100644 --- a/app/br/br.cpp +++ b/app/br/br.cpp @@ -160,8 +160,8 @@ public: check((parc >= 2) && (parc <= 3), "Incorrect parameter count for 'evalClustering'."); br_eval_clustering(parv[0], parv[1], parc == 3 ? parv[2] : ""); } else if (!strcmp(fun, "evalDetection")) { - check((parc >= 2) && (parc <= 5), "Incorrect parameter count for 'evalDetection'."); - br_eval_detection(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? atoi(parv[3]) : 0, parc == 5 ? atoi(parv[4]) : 0); + check((parc >= 2) && (parc <= 6), "Incorrect parameter count for 'evalDetection'."); + br_eval_detection(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? atoi(parv[3]) : 0, parc >= 5 ? atoi(parv[4]) : 0, parc == 6 ? atoi(parv[5]) : 0); } else if (!strcmp(fun, "evalLandmarking")) { check((parc >= 2) && (parc <= 5), "Incorrect parameter count for 'evalLandmarking'."); br_eval_landmarking(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? atoi(parv[3]) : 0, parc >= 5 ? atoi(parv[4]) : 1); diff --git a/openbr/core/eval.cpp b/openbr/core/eval.cpp index d0fafe6..92ca2b3 100755 --- a/openbr/core/eval.cpp +++ b/openbr/core/eval.cpp @@ -666,10 +666,15 @@ struct Detection { QRectF boundingBox; float confidence; + // The ignore flag is useful when certain faces in an image should be ignored + // and should not effect detection performance. Predicted detections that overlap + // with an ignored truth detection will not count as a true positive, false positive, + // true negative, or false negative, it will simply be ignored. + bool ignore; Detection() {} - Detection(const QRectF &boundingBox_, float confidence_ = -1) - : boundingBox(boundingBox_), confidence(confidence_) {} + Detection(const QRectF &boundingBox_, float confidence_ = -1, bool ignore_ = false) + : boundingBox(boundingBox_), confidence(confidence_), ignore(ignore_) {} float area() const { return boundingBox.width() * boundingBox.height(); } float overlap(const Detection &other) const @@ -679,6 +684,16 @@ struct Detection } }; +struct SortedDetection +{ + int truth_idx, predicted_idx; + float overlap; + SortedDetection() : truth_idx(-1), predicted_idx(-1), overlap(-1) {} + SortedDetection(int truth_idx_, int predicted_idx_, float overlap_) + : truth_idx(truth_idx_), predicted_idx(predicted_idx_), overlap(overlap_) {} + inline bool operator<(const SortedDetection &other) const { return overlap > other.overlap; } +}; + struct Detections { QList predicted, truth; @@ -704,6 +719,7 @@ static QStringList computeDetectionResults(const QList &detec { QList points; float TP = 0, FP = 0, prevFP = -1; + for (int i=0; i &detec } if ((i == detections.size()-1) || (detection.confidence > detections[i+1].confidence)) { if (FP > prevFP || (i == detections.size()-1)) { + if (prevFP / numImages < 0.1 && FP / numImages > 0.1 && discrete) { + qDebug("TAR @ FAR => %f : 0.1", TP / totalTrueDetections); + qDebug("Confidence: %f", detection.confidence); + } else if (prevFP / numImages < 0.01 && FP / numImages > 0.01 && discrete) { + qDebug("TAR @ FAR => %f : 0.01", TP / totalTrueDetections); + qDebug("Confidence: %f", detection.confidence); + } points.append(DetectionOperatingPoint(TP, FP, totalTrueDetections, numImages)); prevFP = FP; } @@ -803,7 +826,7 @@ static QList getDetections(const DetectionKey &key, const File &f, bo dets.append(Detection(f.get(key), isTruth ? -1 : f.get("Confidence", -1))); } else if (key.type == DetectionKey::XYWidthHeight) { const QRectF rect(f.get(key+"_X"), f.get(key+"_Y"), f.get(key+"_Width"), f.get(key+"_Height")); - dets.append(Detection(rect, isTruth ? -1 : f.get("Confidence", -1))); + dets.append(Detection(rect, isTruth ? -1 : f.get("Confidence", -1), f.get("Ignore", false))); } return dets; } @@ -827,21 +850,16 @@ static QMap getDetections(const File &predictedGallery, con qPrintable(predictedDetectKey == truthDetectKey ? QString() : "/"+truthDetectKey)); QMap allDetections; - foreach (const File &f, predicted) - allDetections[f.baseName()].predicted.append(getDetections(predictedDetectKey, f, false)); foreach (const File &f, truth) - allDetections[f.baseName()].truth.append(getDetections(truthDetectKey, f, true)); + allDetections[f.name].truth.append(getDetections(truthDetectKey, f, true)); + foreach (const File &f, predicted) + if (allDetections.contains(f.name)) allDetections[f.name].predicted.append(getDetections(predictedDetectKey, f, false)); return allDetections; } -static int getNumberOfImages(const File &truthGallery) -{ - const FileList files = FileList::fromGallery(truthGallery); - - QSet names; - foreach(const File &file, files) - names.insert(file.fileName()); - return names.size(); +static inline int getNumberOfImages(const QMap detections) +{ + return detections.keys().size(); } static int associateGroundTruthDetections(QList &resolved, QList &falseNegative, QMap &all, QRectF &offsets) @@ -852,13 +870,13 @@ static int associateGroundTruthDetections(QList &resolved, QL foreach (Detections detections, all.values()) { totalTrueDetections += detections.truth.size(); // Try to associate ground truth detections with predicted detections - while (!detections.truth.isEmpty() && !detections.predicted.isEmpty()) { - const Detection truth = detections.truth.takeFirst(); // Take removes the detection - int bestIndex = -1; - float bestOverlap = -std::numeric_limits::max(); - // Find the nearest predicted detection to this ground truth detection - for (int i=0; i sortedDetections; sortedDetections.reserve(detections.truth.size() * detections.predicted.size()); + for (int t = 0; t < detections.truth.size(); t++) { + const Detection truth = detections.truth[t]; + for (int p = 0; p < detections.predicted.size(); p++) { + Detection predicted = detections.predicted[p]; + float predictedWidth = predicted.boundingBox.width(); float x, y, width, height; x = predicted.boundingBox.x() + offsets.x()*predictedWidth; @@ -868,35 +886,44 @@ static int associateGroundTruthDetections(QList &resolved, QL Detection newPredicted(QRectF(x, y, width, height), 0.0); const float overlap = truth.overlap(newPredicted); - if (overlap > bestOverlap) { - bestOverlap = overlap; - bestIndex = i; - } + if (overlap > 0) + sortedDetections.append(SortedDetection(t, p, overlap)); } - // Removing the detection prevents us from considering it twice. - // We don't want to associate two ground truth detections with the - // same prediction, over vice versa. - const Detection predicted = detections.predicted.takeAt(bestIndex); - resolved.append(ResolvedDetection(predicted.confidence, bestOverlap)); - - if (offsets.x() == 0) { - // Add side differences to total only for pairs that meet the overlap threshold. - if (bestOverlap > 0.3) { - count++; - float width = predicted.boundingBox.width(); - dLeftTotal += (truth.boundingBox.left() - predicted.boundingBox.left()) / width; - dRightTotal += (truth.boundingBox.right() - predicted.boundingBox.right()) / width; - dTopTotal += (truth.boundingBox.top() - predicted.boundingBox.top()) / width; - dBottomTotal += (truth.boundingBox.bottom() - predicted.boundingBox.bottom()) / width; - } + } + + std::sort(sortedDetections.begin(), sortedDetections.end()); + + QList removedTruth; + QList removedPredicted; + + foreach (const SortedDetection &detection, sortedDetections) { + if (removedTruth.contains(detection.truth_idx) || removedPredicted.contains(detection.predicted_idx)) + continue; + + const Detection truth = detections.truth[detection.truth_idx]; + const Detection predicted = detections.predicted[detection.predicted_idx]; + + if (!truth.ignore) resolved.append(ResolvedDetection(predicted.confidence, detection.overlap)); + + removedTruth.append(detection.truth_idx); + removedPredicted.append(detection.predicted_idx); + + if (offsets.x() == 0 && detection.overlap > 0.3) { + count++; + float width = predicted.boundingBox.width(); + dLeftTotal += (truth.boundingBox.left() - predicted.boundingBox.left()) / width; + dRightTotal += (truth.boundingBox.right() - predicted.boundingBox.right()) / width; + dTopTotal += (truth.boundingBox.top() - predicted.boundingBox.top()) / width; + dBottomTotal += (truth.boundingBox.bottom() - predicted.boundingBox.bottom()) / width; } } - foreach (const Detection &detection, detections.predicted) - resolved.append(ResolvedDetection(detection.confidence, 0)); - for (int i=0; i::max(), 0)); + for (int i = 0; i < detections.predicted.size(); i++) + if (!removedPredicted.contains(i)) resolved.append(ResolvedDetection(detections.predicted[i].confidence, 0)); + for (int i = 0; i < detections.truth.size(); i++) + if (!removedTruth.contains(i) && !detections.truth[i].ignore) falseNegative.append(ResolvedDetection(-std::numeric_limits::max(), 0)); } + if (offsets.x() == 0) { // Calculate average differences in each direction float dRight = dRightTotal / count; @@ -914,7 +941,7 @@ static int associateGroundTruthDetections(QList &resolved, QL return totalTrueDetections; } -float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv, bool normalize, int minSize) +float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv, bool normalize, int minSize, int maxSize) { qDebug("Evaluating detection of %s against %s", qPrintable(predictedGallery), qPrintable(truthGallery)); // Organized by file, QMap used to preserve order @@ -923,12 +950,13 @@ float EvalDetection(const QString &predictedGallery, const QString &truthGallery // Remove any bounding boxes with a side smaller than minSize if (minSize > 0) { qDebug("Removing boxes smaller than %d\n", minSize); + QMap allFilteredDetections; foreach (QString key, allDetections.keys()) { Detections detections = allDetections[key]; Detections filteredDetections; for (int i = 0; i < detections.predicted.size(); i++) { QRectF box = detections.predicted[i].boundingBox; - if (min(box.width(), box.height()) > minSize) { + if (min(box.width(), box.height()) > sqrt(0.5 * pow(minSize, 2))) { filteredDetections.predicted.append(detections.predicted[i]); } } @@ -939,8 +967,34 @@ float EvalDetection(const QString &predictedGallery, const QString &truthGallery filteredDetections.truth.append(detections.truth[i]); } } - allDetections.insert(key, filteredDetections); + if (!filteredDetections.truth.empty()) allFilteredDetections[key] = filteredDetections; + } + allDetections = allFilteredDetections; + } + + // Remove any bounding boxes with no side smaller than maxSize + if (maxSize > 0) { + qDebug("Removing boxes larger than %d\n", maxSize); + QMap allFilteredDetections; + foreach (QString key, allDetections.keys()) { + Detections detections = allDetections[key]; + Detections filteredDetections; + for (int i = 0; i < detections.predicted.size(); i++) { + QRectF box = detections.predicted[i].boundingBox; + if (min(box.width(), box.height()) < sqrt(0.5 * pow(maxSize, 2))) { + filteredDetections.predicted.append(detections.predicted[i]); + } + } + + for (int i = 0; i < detections.truth.size(); i++) { + QRectF box = detections.truth[i].boundingBox; + if (min(box.width(), box.height()) < maxSize) { + filteredDetections.truth.append(detections.truth[i]); + } + } + if (!filteredDetections.truth.empty()) allFilteredDetections[key] = filteredDetections; } + allDetections = allFilteredDetections; } QList resolvedDetections, falseNegativeDetections; @@ -963,8 +1017,8 @@ float EvalDetection(const QString &predictedGallery, const QString &truthGallery std::sort(resolvedDetections.begin(), resolvedDetections.end()); QStringList lines; lines.append("Plot, X, Y"); - lines.append(computeDetectionResults(resolvedDetections, totalTrueDetections, getNumberOfImages(truthGallery), true)); - lines.append(computeDetectionResults(resolvedDetections, totalTrueDetections, getNumberOfImages(truthGallery), false)); + lines.append(computeDetectionResults(resolvedDetections, totalTrueDetections, getNumberOfImages(allDetections), true)); + lines.append(computeDetectionResults(resolvedDetections, totalTrueDetections, getNumberOfImages(allDetections), false)); float averageOverlap; { // Overlap Density diff --git a/openbr/core/eval.h b/openbr/core/eval.h index 183091d..bd83c98 100644 --- a/openbr/core/eval.h +++ b/openbr/core/eval.h @@ -30,7 +30,7 @@ namespace br float InplaceEval(const QString & simmat, const QString & target, const QString & query, const QString & csv = ""); void EvalClassification(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = ""); - float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", bool normalize = false, int minSize = 0); // Return average overlap + float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", bool normalize = false, int minSize = 0, int maxSize = 0); // Return average overlap float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", int normalizationIndexA = 0, int normalizationIndexB = 1); // Return average error void EvalRegression(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = ""); } diff --git a/openbr/openbr.cpp b/openbr/openbr.cpp index b18cfa7..b18f019 100644 --- a/openbr/openbr.cpp +++ b/openbr/openbr.cpp @@ -129,9 +129,9 @@ void br_eval_clustering(const char *csv, const char *gallery, const char *truth_ EvalClustering(csv, gallery, truth_property); } -float br_eval_detection(const char *predicted_gallery, const char *truth_gallery, const char *csv, bool normalize, int minSize) +float br_eval_detection(const char *predicted_gallery, const char *truth_gallery, const char *csv, bool normalize, int minSize, int maxSize) { - return EvalDetection(predicted_gallery, truth_gallery, csv, normalize, minSize); + return EvalDetection(predicted_gallery, truth_gallery, csv, normalize, minSize, maxSize); } float br_eval_landmarking(const char *predicted_gallery, const char *truth_gallery, const char *csv, int normalization_index_a, int normalization_index_b) diff --git a/openbr/openbr.h b/openbr/openbr.h index 2ff31fb..42bb578 100644 --- a/openbr/openbr.h +++ b/openbr/openbr.h @@ -205,7 +205,7 @@ BR_EXPORT void br_eval_clustering(const char *csv, const char *gallery, const ch * \param normalize Optional \c bool flag to normalize predicted bounding boxes for improved detection. * \return Average detection bounding box overlap. */ -BR_EXPORT float br_eval_detection(const char *predicted_gallery, const char *truth_gallery, const char *csv = "", bool normalize = false, int minSize = 0); +BR_EXPORT float br_eval_detection(const char *predicted_gallery, const char *truth_gallery, const char *csv = "", bool normalize = false, int minSize = 0, int maxSize = 0); /*! * \brief Evaluates and prints landmarking accuracy to terminal.