Peter M. Groen / openbr

Commit 0721251695f01d27546f85ed299f29193f9a915e

Authored by Scott Klum
1 parent 8be00346

Crudely outputting some example detections in evalDetection, extra verbose output

Inline Side-by-side
Showing 1 changed file with 77 additions and 15 deletions
openbr/core/eval.cpp
  View file @0721251
... ... @@ -19,8 +19,10 @@
19 19 #include "openbr/core/common.h"
20 20 #include "openbr/core/qtutils.h"
21 21 #include "openbr/core/opencvutils.h"
  22 +#include "openbr/core/qtutils.h"
22 23 #include <QMapIterator>
23 24 #include <cmath>
  25 +#include <opencv2/highgui/highgui.hpp>
24 26  
25 27 using namespace cv;
26 28  
... ... @@ -755,6 +757,7 @@ void EvalClassification(const QString &amp;predictedGallery, const QString &amp;truthGal
755 757 struct Detection
756 758 {
757 759 QRectF boundingBox;
  760 + QString filePath;
758 761 float confidence;
759 762 // The ignore flag is useful when certain faces in an image should be ignored
760 763 // and should not effect detection performance. Predicted detections that overlap
... ... @@ -763,8 +766,8 @@ struct Detection
763 766 bool ignore;
764 767  
765 768 Detection() {}
766   - Detection(const QRectF &boundingBox_, float confidence_ = -1, bool ignore_ = false)
767   - : boundingBox(boundingBox_), confidence(confidence_), ignore(ignore_) {}
  769 + Detection(const QRectF &boundingBox_, const QString &filePath = QString(), float confidence_ = -1, bool ignore_ = false)
  770 + : boundingBox(boundingBox_), filePath(filePath), confidence(confidence_), ignore(ignore_) {}
768 771  
769 772 float area() const { return boundingBox.width() * boundingBox.height(); }
770 773 float overlap(const Detection &other) const
... ... @@ -791,12 +794,20 @@ struct Detections
791 794  
792 795 struct ResolvedDetection
793 796 {
  797 + QString filePath;
  798 + QRectF boundingBox;
794 799 float confidence, overlap;
795 800 ResolvedDetection() : confidence(-1), overlap(-1) {}
796   - ResolvedDetection(float confidence_, float overlap_) : confidence(confidence_), overlap(overlap_) {}
  801 + ResolvedDetection(const QString &filePath, const QRectF &boundingBox, float confidence_, float overlap_) :
  802 + filePath(filePath), boundingBox(boundingBox), confidence(confidence_), overlap(overlap_) {}
797 803 inline bool operator<(const ResolvedDetection &other) const { return confidence > other.confidence; }
798 804 };
799 805  
  806 +QDebug operator<<(QDebug dbg, const ResolvedDetection &d)
  807 +{
  808 + return dbg.nospace() << "(FilePath: " << d.filePath << " Bounding Box: " << d.boundingBox << ", Overlap: " << d.overlap << ", Confidence: " << d.confidence << ")";
  809 +}
  810 +
800 811 struct DetectionOperatingPoint
801 812 {
802 813 float Recall, FalsePositiveRate, Precision;
... ... @@ -808,8 +819,10 @@ struct DetectionOperatingPoint
808 819 static QStringList computeDetectionResults(const QList<ResolvedDetection> &detections, int totalTrueDetections, int numImages, bool discrete)
809 820 {
810 821 QList<DetectionOperatingPoint> points;
811   - float TP = 0, FP = 0, prevFP = -1;
  822 + float TP = 0, FP = 0, prevFP = -1, prevTP = -1;
812 823  
  824 + const int detectionsToKeep = 50;
  825 + QList<ResolvedDetection> topFalsePositives, bottomTruePositives;
813 826 for (int i=0; i<detections.size(); i++) {
814 827 const ResolvedDetection &detection = detections[i];
815 828 if (discrete) {
... ... @@ -822,29 +835,61 @@ static QStringList computeDetectionResults(const QList&lt;ResolvedDetection&gt; &amp;detec
822 835 }
823 836 if ((i == detections.size()-1) || (detection.confidence > detections[i+1].confidence)) {
824 837 if (FP > prevFP || (i == detections.size()-1)) {
825   - if (prevFP / numImages < 0.1 && FP / numImages > 0.1 && discrete) {
  838 + if (prevFP / numImages < 1 && FP / numImages >= 1 && discrete) {
  839 + qDebug("TAR @ FAR => %f : 1", TP / totalTrueDetections);
  840 + qDebug("Confidence: %f", detection.confidence);
  841 + qDebug("TP vs. FP: %f to %f", TP, FP);
  842 + } else if (prevFP / numImages < 0.1 && FP / numImages >= 0.1 && discrete) {
826 843 qDebug("TAR @ FAR => %f : 0.1", TP / totalTrueDetections);
827 844 qDebug("Confidence: %f", detection.confidence);
828 845 qDebug("TP vs. FP: %f to %f", TP, FP);
829   - } else if (prevFP / numImages < 0.01 && FP / numImages > 0.01 && discrete) {
  846 + } else if (prevFP / numImages < 0.02 && FP / numImages >= 0.02 && discrete) {
  847 + qDebug("TAR @ FAR => %f : 0.02", TP / totalTrueDetections);
  848 + qDebug("Confidence: %f", detection.confidence);
  849 + qDebug("TP vs. FP: %f to %f", TP, FP);
  850 + } else if (prevFP / numImages < 0.01 && FP / numImages >= 0.01 && discrete) {
830 851 qDebug("TAR @ FAR => %f : 0.01", TP / totalTrueDetections);
831 852 qDebug("Confidence: %f", detection.confidence);
832 853 qDebug("TP vs. FP: %f to %f", TP, FP);
833   - } else if (prevFP / numImages < 0.001 && FP / numImages > 0.001 && discrete) {
  854 + } else if (prevFP / numImages < 0.001 && FP / numImages >= 0.001 && discrete) {
834 855 qDebug("TAR @ FAR => %f : 0.001", TP / totalTrueDetections);
835 856 qDebug("Confidence: %f", detection.confidence);
836 857 qDebug("TP vs. FP: %f to %f", TP, FP);
837 858 }
838 859  
  860 + if (detection.overlap < 0.5 && topFalsePositives.size() < detectionsToKeep)
  861 + topFalsePositives.append(detection);
  862 +
839 863 points.append(DetectionOperatingPoint(TP, FP, totalTrueDetections, numImages));
840 864 prevFP = FP;
841 865 }
  866 +
  867 + if (TP > prevTP) {
  868 + bottomTruePositives.append(detection);
  869 + if (bottomTruePositives.size() > detectionsToKeep)
  870 + bottomTruePositives.removeFirst();
  871 + prevTP = TP;
  872 + }
842 873 }
843 874 }
844 875  
845 876 if (discrete) {
846 877 qDebug("Total TP vs. FP: %f to %f", TP, FP);
847 878 qDebug("Overall Recall (TP vs. possible TP): %f (%f vs. %d)", TP / totalTrueDetections, TP, totalTrueDetections);
  879 +
  880 + if (Globals->verbose) {
  881 + QtUtils::touchDir(QDir("./falsePos"));
  882 + qDebug("Highest Scoring False Positives:");
  883 + for (int i=0; i<detectionsToKeep; i++) {
  884 + Mat img = imread(qPrintable(Globals->path + "/" + topFalsePositives[i].filePath));
  885 + qDebug() << topFalsePositives[i];
  886 + const Scalar color(0,255,0);
  887 + rectangle(img, OpenCVUtils::toRect(topFalsePositives[i].boundingBox), color, 1);
  888 + imwrite(qPrintable(QString("./falsePos/falsePos%1.jpg").arg(QString::number(i))), img);
  889 + }
  890 + qDebug("Lowest Scoring True Positives:");
  891 + qDebug() << bottomTruePositives;
  892 + }
848 893 }
849 894  
850 895 const int keep = qMin(points.size(), Max_Points);
... ... @@ -912,6 +957,7 @@ static DetectionKey getDetectKey(const FileList &amp;files)
912 957 // return a list of detections independent of the detection key format
913 958 static QList<Detection> getDetections(const DetectionKey &key, const File &f, bool isTruth)
914 959 {
  960 + const QString filePath = f.path() + "/" + f.fileName();
915 961 QList<Detection> dets;
916 962 if (key.type == DetectionKey::RectList) {
917 963 QList<QRectF> rects = f.rects();
... ... @@ -920,15 +966,15 @@ static QList&lt;Detection&gt; getDetections(const DetectionKey &amp;key, const File &amp;f, bo
920 966 qFatal("You don't have enough confidence. I mean, your detections don't all have confidence measures.");
921 967 for (int i=0; i<rects.size(); i++) {
922 968 if (isTruth)
923   - dets.append(Detection(rects[i]));
  969 + dets.append(Detection(rects[i], filePath));
924 970 else
925   - dets.append(Detection(rects[i], confidences[i]));
  971 + dets.append(Detection(rects[i], filePath, confidences[i]));
926 972 }
927 973 } else if (key.type == DetectionKey::Rect) {
928   - dets.append(Detection(f.get<QRectF>(key), isTruth ? -1 : f.get<float>("Confidence", -1)));
  974 + dets.append(Detection(f.get<QRectF>(key), filePath, isTruth ? -1 : f.get<float>("Confidence", -1)));
929 975 } else if (key.type == DetectionKey::XYWidthHeight) {
930 976 const QRectF rect(f.get<float>(key+"_X"), f.get<float>(key+"_Y"), f.get<float>(key+"_Width"), f.get<float>(key+"_Height"));
931   - dets.append(Detection(rect, isTruth ? -1 : f.get<float>("Confidence", -1), f.get<bool>("Ignore", false)));
  977 + dets.append(Detection(rect, filePath, isTruth ? -1 : f.get<float>("Confidence", -1), f.get<bool>("Ignore", false)));
932 978 }
933 979 return dets;
934 980 }
... ... @@ -986,9 +1032,10 @@ static int associateGroundTruthDetections(QList&lt;ResolvedDetection&gt; &amp;resolved, QL
986 1032 y = predicted.boundingBox.y() + offsets.y()*predictedWidth;
987 1033 width = predicted.boundingBox.width() - offsets.width()*predictedWidth;
988 1034 height = predicted.boundingBox.height() - offsets.height()*predictedWidth;
989   - Detection newPredicted(QRectF(x, y, width, height), 0.0);
  1035 + Detection newPredicted(QRectF(x, y, width, height), predicted.filePath, 0.0);
990 1036  
991 1037 const float overlap = truth.overlap(newPredicted);
  1038 +
992 1039 if (overlap > 0)
993 1040 sortedDetections.append(SortedDetection(t, p, overlap));
994 1041 }
... ... @@ -1006,7 +1053,8 @@ static int associateGroundTruthDetections(QList&lt;ResolvedDetection&gt; &amp;resolved, QL
1006 1053 const Detection truth = detections.truth[detection.truth_idx];
1007 1054 const Detection predicted = detections.predicted[detection.predicted_idx];
1008 1055  
1009   - if (!truth.ignore) resolved.append(ResolvedDetection(predicted.confidence, detection.overlap));
  1056 + if (!truth.ignore)
  1057 + resolved.append(ResolvedDetection(predicted.filePath, predicted.boundingBox, predicted.confidence, detection.overlap));
1010 1058  
1011 1059 removedTruth.append(detection.truth_idx);
1012 1060 removedPredicted.append(detection.predicted_idx);
... ... @@ -1022,9 +1070,9 @@ static int associateGroundTruthDetections(QList&lt;ResolvedDetection&gt; &amp;resolved, QL
1022 1070 }
1023 1071  
1024 1072 for (int i = 0; i < detections.predicted.size(); i++)
1025   - if (!removedPredicted.contains(i)) resolved.append(ResolvedDetection(detections.predicted[i].confidence, 0));
  1073 + if (!removedPredicted.contains(i)) resolved.append(ResolvedDetection(detections.predicted[i].filePath, detections.predicted[i].boundingBox, detections.predicted[i].confidence, 0));
1026 1074 for (int i = 0; i < detections.truth.size(); i++)
1027   - if (!removedTruth.contains(i) && !detections.truth[i].ignore) falseNegative.append(ResolvedDetection(-std::numeric_limits<float>::max(), 0));
  1075 + if (!removedTruth.contains(i) && !detections.truth[i].ignore) falseNegative.append(ResolvedDetection(detections.truth[i].filePath, detections.truth[i].boundingBox, -std::numeric_limits<float>::max(), 0));
1028 1076 }
1029 1077  
1030 1078 if (offsets.x() == 0) {
... ... @@ -1117,6 +1165,20 @@ float EvalDetection(const QString &amp;predictedGallery, const QString &amp;truthGallery
1117 1165 falseNegativeDetections.clear();
1118 1166 totalTrueDetections = associateGroundTruthDetections(resolvedDetections, falseNegativeDetections, allDetections, normalizations);
1119 1167 }
  1168 +
  1169 + if (Globals->verbose) {
  1170 + qDebug("Total False negatives:");
  1171 + const int numFalseNegatives = 50;
  1172 + for (int i=0; i<numFalseNegatives; i++) {
  1173 + Mat img = imread(qPrintable(Globals->path + "/" + falseNegativeDetections[i].filePath));
  1174 + qDebug() << falseNegativeDetections[i];
  1175 + const Scalar color(0,255,0);
  1176 + rectangle(img, OpenCVUtils::toRect(falseNegativeDetections[i].boundingBox), color, 1);
  1177 + QtUtils::touchDir(QDir("./falseNegs"));
  1178 + imwrite(qPrintable(QString("./falseNegs/falseNeg%1.jpg").arg(QString::number(i))), img);
  1179 + }
  1180 + }
  1181 +
1120 1182 std::sort(resolvedDetections.begin(), resolvedDetections.end());
1121 1183 QStringList lines;
1122 1184 lines.append("Plot, X, Y");
... ...