Peter M. Groen / openbr

Commit bf17972881c559482e56cb89ffc898df48eaeaab

Authored by Josh Klontz
2 parents 453d5f49 fa88ae6a

Merge pull request #283 from biometrics/evaluation_updates 

Updates to eval_detection
Inline Side-by-side
Showing 5 changed files with 110 additions and 56 deletions
app/br/br.cpp
  View file @bf17972
... ... @@ -160,8 +160,8 @@ public:
160 160 check((parc >= 2) && (parc <= 3), "Incorrect parameter count for 'evalClustering'.");
161 161 br_eval_clustering(parv[0], parv[1], parc == 3 ? parv[2] : "");
162 162 } else if (!strcmp(fun, "evalDetection")) {
163   - check((parc >= 2) && (parc <= 5), "Incorrect parameter count for 'evalDetection'.");
164   - br_eval_detection(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? atoi(parv[3]) : 0, parc == 5 ? atoi(parv[4]) : 0);
  163 + check((parc >= 2) && (parc <= 6), "Incorrect parameter count for 'evalDetection'.");
  164 + br_eval_detection(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? atoi(parv[3]) : 0, parc >= 5 ? atoi(parv[4]) : 0, parc == 6 ? atoi(parv[5]) : 0);
165 165 } else if (!strcmp(fun, "evalLandmarking")) {
166 166 check((parc >= 2) && (parc <= 5), "Incorrect parameter count for 'evalLandmarking'.");
167 167 br_eval_landmarking(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? atoi(parv[3]) : 0, parc >= 5 ? atoi(parv[4]) : 1);
... ...
openbr/core/eval.cpp
  View file @bf17972
... ... @@ -666,10 +666,15 @@ struct Detection
666 666 {
667 667 QRectF boundingBox;
668 668 float confidence;
  669 + // The ignore flag is useful when certain faces in an image should be ignored
  670 + // and should not effect detection performance. Predicted detections that overlap
  671 + // with an ignored truth detection will not count as a true positive, false positive,
  672 + // true negative, or false negative, it will simply be ignored.
  673 + bool ignore;
669 674  
670 675 Detection() {}
671   - Detection(const QRectF &boundingBox_, float confidence_ = -1)
672   - : boundingBox(boundingBox_), confidence(confidence_) {}
  676 + Detection(const QRectF &boundingBox_, float confidence_ = -1, bool ignore_ = false)
  677 + : boundingBox(boundingBox_), confidence(confidence_), ignore(ignore_) {}
673 678  
674 679 float area() const { return boundingBox.width() * boundingBox.height(); }
675 680 float overlap(const Detection &other) const
... ... @@ -679,6 +684,16 @@ struct Detection
679 684 }
680 685 };
681 686  
  687 +struct SortedDetection
  688 +{
  689 + int truth_idx, predicted_idx;
  690 + float overlap;
  691 + SortedDetection() : truth_idx(-1), predicted_idx(-1), overlap(-1) {}
  692 + SortedDetection(int truth_idx_, int predicted_idx_, float overlap_)
  693 + : truth_idx(truth_idx_), predicted_idx(predicted_idx_), overlap(overlap_) {}
  694 + inline bool operator<(const SortedDetection &other) const { return overlap > other.overlap; }
  695 +};
  696 +
682 697 struct Detections
683 698 {
684 699 QList<Detection> predicted, truth;
... ... @@ -704,6 +719,7 @@ static QStringList computeDetectionResults(const QList&lt;ResolvedDetection&gt; &amp;detec
704 719 {
705 720 QList<DetectionOperatingPoint> points;
706 721 float TP = 0, FP = 0, prevFP = -1;
  722 +
707 723 for (int i=0; i<detections.size(); i++) {
708 724 const ResolvedDetection &detection = detections[i];
709 725 if (discrete) {
... ... @@ -716,6 +732,13 @@ static QStringList computeDetectionResults(const QList&lt;ResolvedDetection&gt; &amp;detec
716 732 }
717 733 if ((i == detections.size()-1) || (detection.confidence > detections[i+1].confidence)) {
718 734 if (FP > prevFP || (i == detections.size()-1)) {
  735 + if (prevFP / numImages < 0.1 && FP / numImages > 0.1 && discrete) {
  736 + qDebug("TAR @ FAR => %f : 0.1", TP / totalTrueDetections);
  737 + qDebug("Confidence: %f", detection.confidence);
  738 + } else if (prevFP / numImages < 0.01 && FP / numImages > 0.01 && discrete) {
  739 + qDebug("TAR @ FAR => %f : 0.01", TP / totalTrueDetections);
  740 + qDebug("Confidence: %f", detection.confidence);
  741 + }
719 742 points.append(DetectionOperatingPoint(TP, FP, totalTrueDetections, numImages));
720 743 prevFP = FP;
721 744 }
... ... @@ -803,7 +826,7 @@ static QList&lt;Detection&gt; getDetections(const DetectionKey &amp;key, const File &amp;f, bo
803 826 dets.append(Detection(f.get<QRectF>(key), isTruth ? -1 : f.get<float>("Confidence", -1)));
804 827 } else if (key.type == DetectionKey::XYWidthHeight) {
805 828 const QRectF rect(f.get<float>(key+"_X"), f.get<float>(key+"_Y"), f.get<float>(key+"_Width"), f.get<float>(key+"_Height"));
806   - dets.append(Detection(rect, isTruth ? -1 : f.get<float>("Confidence", -1)));
  829 + dets.append(Detection(rect, isTruth ? -1 : f.get<float>("Confidence", -1), f.get<bool>("Ignore", false)));
807 830 }
808 831 return dets;
809 832 }
... ... @@ -827,21 +850,16 @@ static QMap&lt;QString, Detections&gt; getDetections(const File &amp;predictedGallery, con
827 850 qPrintable(predictedDetectKey == truthDetectKey ? QString() : "/"+truthDetectKey));
828 851  
829 852 QMap<QString, Detections> allDetections;
830   - foreach (const File &f, predicted)
831   - allDetections[f.baseName()].predicted.append(getDetections(predictedDetectKey, f, false));
832 853 foreach (const File &f, truth)
833   - allDetections[f.baseName()].truth.append(getDetections(truthDetectKey, f, true));
  854 + allDetections[f.name].truth.append(getDetections(truthDetectKey, f, true));
  855 + foreach (const File &f, predicted)
  856 + if (allDetections.contains(f.name)) allDetections[f.name].predicted.append(getDetections(predictedDetectKey, f, false));
834 857 return allDetections;
835 858 }
836 859  
837   -static int getNumberOfImages(const File &truthGallery)
838   -{
839   - const FileList files = FileList::fromGallery(truthGallery);
840   -
841   - QSet<QString> names;
842   - foreach(const File &file, files)
843   - names.insert(file.fileName());
844   - return names.size();
  860 +static inline int getNumberOfImages(const QMap<QString, Detections> detections)
  861 +{
  862 + return detections.keys().size();
845 863 }
846 864  
847 865 static int associateGroundTruthDetections(QList<ResolvedDetection> &resolved, QList<ResolvedDetection> &falseNegative, QMap<QString, Detections> &all, QRectF &offsets)
... ... @@ -852,13 +870,13 @@ static int associateGroundTruthDetections(QList&lt;ResolvedDetection&gt; &amp;resolved, QL
852 870 foreach (Detections detections, all.values()) {
853 871 totalTrueDetections += detections.truth.size();
854 872 // Try to associate ground truth detections with predicted detections
855   - while (!detections.truth.isEmpty() && !detections.predicted.isEmpty()) {
856   - const Detection truth = detections.truth.takeFirst(); // Take removes the detection
857   - int bestIndex = -1;
858   - float bestOverlap = -std::numeric_limits<float>::max();
859   - // Find the nearest predicted detection to this ground truth detection
860   - for (int i=0; i<detections.predicted.size(); i++) {
861   - Detection predicted = detections.predicted[i];
  873 +
  874 + QList<SortedDetection> sortedDetections; sortedDetections.reserve(detections.truth.size() * detections.predicted.size());
  875 + for (int t = 0; t < detections.truth.size(); t++) {
  876 + const Detection truth = detections.truth[t];
  877 + for (int p = 0; p < detections.predicted.size(); p++) {
  878 + Detection predicted = detections.predicted[p];
  879 +
862 880 float predictedWidth = predicted.boundingBox.width();
863 881 float x, y, width, height;
864 882 x = predicted.boundingBox.x() + offsets.x()*predictedWidth;
... ... @@ -868,35 +886,44 @@ static int associateGroundTruthDetections(QList&lt;ResolvedDetection&gt; &amp;resolved, QL
868 886 Detection newPredicted(QRectF(x, y, width, height), 0.0);
869 887  
870 888 const float overlap = truth.overlap(newPredicted);
871   - if (overlap > bestOverlap) {
872   - bestOverlap = overlap;
873   - bestIndex = i;
874   - }
  889 + if (overlap > 0)
  890 + sortedDetections.append(SortedDetection(t, p, overlap));
875 891 }
876   - // Removing the detection prevents us from considering it twice.
877   - // We don't want to associate two ground truth detections with the
878   - // same prediction, over vice versa.
879   - const Detection predicted = detections.predicted.takeAt(bestIndex);
880   - resolved.append(ResolvedDetection(predicted.confidence, bestOverlap));
881   -
882   - if (offsets.x() == 0) {
883   - // Add side differences to total only for pairs that meet the overlap threshold.
884   - if (bestOverlap > 0.3) {
885   - count++;
886   - float width = predicted.boundingBox.width();
887   - dLeftTotal += (truth.boundingBox.left() - predicted.boundingBox.left()) / width;
888   - dRightTotal += (truth.boundingBox.right() - predicted.boundingBox.right()) / width;
889   - dTopTotal += (truth.boundingBox.top() - predicted.boundingBox.top()) / width;
890   - dBottomTotal += (truth.boundingBox.bottom() - predicted.boundingBox.bottom()) / width;
891   - }
  892 + }
  893 +
  894 + std::sort(sortedDetections.begin(), sortedDetections.end());
  895 +
  896 + QList<int> removedTruth;
  897 + QList<int> removedPredicted;
  898 +
  899 + foreach (const SortedDetection &detection, sortedDetections) {
  900 + if (removedTruth.contains(detection.truth_idx) || removedPredicted.contains(detection.predicted_idx))
  901 + continue;
  902 +
  903 + const Detection truth = detections.truth[detection.truth_idx];
  904 + const Detection predicted = detections.predicted[detection.predicted_idx];
  905 +
  906 + if (!truth.ignore) resolved.append(ResolvedDetection(predicted.confidence, detection.overlap));
  907 +
  908 + removedTruth.append(detection.truth_idx);
  909 + removedPredicted.append(detection.predicted_idx);
  910 +
  911 + if (offsets.x() == 0 && detection.overlap > 0.3) {
  912 + count++;
  913 + float width = predicted.boundingBox.width();
  914 + dLeftTotal += (truth.boundingBox.left() - predicted.boundingBox.left()) / width;
  915 + dRightTotal += (truth.boundingBox.right() - predicted.boundingBox.right()) / width;
  916 + dTopTotal += (truth.boundingBox.top() - predicted.boundingBox.top()) / width;
  917 + dBottomTotal += (truth.boundingBox.bottom() - predicted.boundingBox.bottom()) / width;
892 918 }
893 919 }
894 920  
895   - foreach (const Detection &detection, detections.predicted)
896   - resolved.append(ResolvedDetection(detection.confidence, 0));
897   - for (int i=0; i<detections.truth.size(); i++)
898   - falseNegative.append(ResolvedDetection(-std::numeric_limits<float>::max(), 0));
  921 + for (int i = 0; i < detections.predicted.size(); i++)
  922 + if (!removedPredicted.contains(i)) resolved.append(ResolvedDetection(detections.predicted[i].confidence, 0));
  923 + for (int i = 0; i < detections.truth.size(); i++)
  924 + if (!removedTruth.contains(i) && !detections.truth[i].ignore) falseNegative.append(ResolvedDetection(-std::numeric_limits<float>::max(), 0));
899 925 }
  926 +
900 927 if (offsets.x() == 0) {
901 928 // Calculate average differences in each direction
902 929 float dRight = dRightTotal / count;
... ... @@ -914,7 +941,7 @@ static int associateGroundTruthDetections(QList&lt;ResolvedDetection&gt; &amp;resolved, QL
914 941 return totalTrueDetections;
915 942 }
916 943  
917   -float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv, bool normalize, int minSize)
  944 +float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv, bool normalize, int minSize, int maxSize)
918 945 {
919 946 qDebug("Evaluating detection of %s against %s", qPrintable(predictedGallery), qPrintable(truthGallery));
920 947 // Organized by file, QMap used to preserve order
... ... @@ -923,12 +950,13 @@ float EvalDetection(const QString &amp;predictedGallery, const QString &amp;truthGallery
923 950 // Remove any bounding boxes with a side smaller than minSize
924 951 if (minSize > 0) {
925 952 qDebug("Removing boxes smaller than %d\n", minSize);
  953 + QMap<QString, Detections> allFilteredDetections;
926 954 foreach (QString key, allDetections.keys()) {
927 955 Detections detections = allDetections[key];
928 956 Detections filteredDetections;
929 957 for (int i = 0; i < detections.predicted.size(); i++) {
930 958 QRectF box = detections.predicted[i].boundingBox;
931   - if (min(box.width(), box.height()) > minSize) {
  959 + if (min(box.width(), box.height()) > sqrt(0.5 * pow(minSize, 2))) {
932 960 filteredDetections.predicted.append(detections.predicted[i]);
933 961 }
934 962 }
... ... @@ -939,8 +967,34 @@ float EvalDetection(const QString &amp;predictedGallery, const QString &amp;truthGallery
939 967 filteredDetections.truth.append(detections.truth[i]);
940 968 }
941 969 }
942   - allDetections.insert(key, filteredDetections);
  970 + if (!filteredDetections.truth.empty()) allFilteredDetections[key] = filteredDetections;
  971 + }
  972 + allDetections = allFilteredDetections;
  973 + }
  974 +
  975 + // Remove any bounding boxes with no side smaller than maxSize
  976 + if (maxSize > 0) {
  977 + qDebug("Removing boxes larger than %d\n", maxSize);
  978 + QMap<QString, Detections> allFilteredDetections;
  979 + foreach (QString key, allDetections.keys()) {
  980 + Detections detections = allDetections[key];
  981 + Detections filteredDetections;
  982 + for (int i = 0; i < detections.predicted.size(); i++) {
  983 + QRectF box = detections.predicted[i].boundingBox;
  984 + if (min(box.width(), box.height()) < sqrt(0.5 * pow(maxSize, 2))) {
  985 + filteredDetections.predicted.append(detections.predicted[i]);
  986 + }
  987 + }
  988 +
  989 + for (int i = 0; i < detections.truth.size(); i++) {
  990 + QRectF box = detections.truth[i].boundingBox;
  991 + if (min(box.width(), box.height()) < maxSize) {
  992 + filteredDetections.truth.append(detections.truth[i]);
  993 + }
  994 + }
  995 + if (!filteredDetections.truth.empty()) allFilteredDetections[key] = filteredDetections;
943 996 }
  997 + allDetections = allFilteredDetections;
944 998 }
945 999  
946 1000 QList<ResolvedDetection> resolvedDetections, falseNegativeDetections;
... ... @@ -963,8 +1017,8 @@ float EvalDetection(const QString &amp;predictedGallery, const QString &amp;truthGallery
963 1017 std::sort(resolvedDetections.begin(), resolvedDetections.end());
964 1018 QStringList lines;
965 1019 lines.append("Plot, X, Y");
966   - lines.append(computeDetectionResults(resolvedDetections, totalTrueDetections, getNumberOfImages(truthGallery), true));
967   - lines.append(computeDetectionResults(resolvedDetections, totalTrueDetections, getNumberOfImages(truthGallery), false));
  1020 + lines.append(computeDetectionResults(resolvedDetections, totalTrueDetections, getNumberOfImages(allDetections), true));
  1021 + lines.append(computeDetectionResults(resolvedDetections, totalTrueDetections, getNumberOfImages(allDetections), false));
968 1022  
969 1023 float averageOverlap;
970 1024 { // Overlap Density
... ...
openbr/core/eval.h
  View file @bf17972
... ... @@ -30,7 +30,7 @@ namespace br
30 30 float InplaceEval(const QString & simmat, const QString & target, const QString & query, const QString & csv = "");
31 31  
32 32 void EvalClassification(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = "");
33   - float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", bool normalize = false, int minSize = 0); // Return average overlap
  33 + float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", bool normalize = false, int minSize = 0, int maxSize = 0); // Return average overlap
34 34 float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", int normalizationIndexA = 0, int normalizationIndexB = 1); // Return average error
35 35 void EvalRegression(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = "");
36 36 }
... ...
openbr/openbr.cpp
  View file @bf17972
... ... @@ -129,9 +129,9 @@ void br_eval_clustering(const char *csv, const char *gallery, const char *truth_
129 129 EvalClustering(csv, gallery, truth_property);
130 130 }
131 131  
132   -float br_eval_detection(const char *predicted_gallery, const char *truth_gallery, const char *csv, bool normalize, int minSize)
  132 +float br_eval_detection(const char *predicted_gallery, const char *truth_gallery, const char *csv, bool normalize, int minSize, int maxSize)
133 133 {
134   - return EvalDetection(predicted_gallery, truth_gallery, csv, normalize, minSize);
  134 + return EvalDetection(predicted_gallery, truth_gallery, csv, normalize, minSize, maxSize);
135 135 }
136 136  
137 137 float br_eval_landmarking(const char *predicted_gallery, const char *truth_gallery, const char *csv, int normalization_index_a, int normalization_index_b)
... ...
openbr/openbr.h
  View file @bf17972
... ... @@ -205,7 +205,7 @@ BR_EXPORT void br_eval_clustering(const char *csv, const char *gallery, const ch
205 205 * \param normalize Optional \c bool flag to normalize predicted bounding boxes for improved detection.
206 206 * \return Average detection bounding box overlap.
207 207 */
208   -BR_EXPORT float br_eval_detection(const char *predicted_gallery, const char *truth_gallery, const char *csv = "", bool normalize = false, int minSize = 0);
  208 +BR_EXPORT float br_eval_detection(const char *predicted_gallery, const char *truth_gallery, const char *csv = "", bool normalize = false, int minSize = 0, int maxSize = 0);
209 209  
210 210 /*!
211 211 * \brief Evaluates and prints landmarking accuracy to terminal.
... ...