Added output of sample landmarks, added index to control which image is used as sample

Scott Klum
1 parent 31008334
Showing 6 changed files with 114 additions and 12 deletions
app/br/br.cpp
openbr/core/eval.cpp
openbr/core/eval.h
openbr/core/plot.cpp
openbr/openbr.cpp
openbr/openbr.h
@@ -163,8 +163,8 @@ public:
                 check((parc >= 2) && (parc <= 6), "Incorrect parameter count for 'evalDetection'.");
                 br_eval_detection(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? atoi(parv[3]) : 0, parc >= 5 ? atoi(parv[4]) : 0, parc == 6 ? atoi(parv[5]) : 0);
             } else if (!strcmp(fun, "evalLandmarking")) {
-                check((parc >= 2) && (parc <= 5), "Incorrect parameter count for 'evalLandmarking'.");
-                br_eval_landmarking(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? atoi(parv[3]) : 0, parc >= 5 ? atoi(parv[4]) : 1);
+                check((parc >= 2) && (parc <= 6), "Incorrect parameter count for 'evalLandmarking'.");
+                br_eval_landmarking(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? atoi(parv[3]) : 0, parc >= 5 ? atoi(parv[4]) : 1,  parc >= 6 ? atoi(parv[5]) : 0);
             } else if (!strcmp(fun, "evalRegression")) {
                 check(parc >= 2 && parc <= 4, "Incorrect parameter count for 'evalRegression'.");
                 br_eval_regression(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? parv[3] : "");
@@ -1032,7 +1032,7 @@ float EvalDetection(const QString &amp;predictedGallery, const QString &amp;truthGallery
     return averageOverlap;
 }
  
-float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv, int normalizationIndexA, int normalizationIndexB)
+float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv, int normalizationIndexA, int normalizationIndexB, int sampleIndex)
 {
     qDebug("Evaluating landmarking of %s against %s", qPrintable(predictedGallery), qPrintable(truthGallery));
     const TemplateList predicted(TemplateList::fromGallery(predictedGallery));
@@ -1042,6 +1042,8 @@ float EvalLandmarking(const QString &amp;predictedGallery, const QString &amp;truthGalle
  
     int skipped = 0;
     QList< QList<float> > pointErrors;
+    QList<float> normalizedLengths;
+
     for (int i=0; i<predicted.size(); i++) {
         const QString &predictedName = predictedNames[i];
         const int truthIndex = truthNames.indexOf(predictedName);
@@ -1057,20 +1059,49 @@ float EvalLandmarking(const QString &amp;predictedGallery, const QString &amp;truthGalle
         if (normalizationIndexA >= truthPoints.size()) qFatal("Normalization index A is out of range.");
         if (normalizationIndexB >= truthPoints.size()) qFatal("Normalization index B is out of range.");
         const float normalizedLength = QtUtils::euclideanLength(truthPoints[normalizationIndexB] - truthPoints[normalizationIndexA]);
-        for (int j=0; j<predictedPoints.size(); j++)
+        normalizedLengths.append(normalizedLength);
+        for (int j=0; j<predictedPoints.size(); j++) {
             pointErrors[j].append(QtUtils::euclideanLength(predictedPoints[j] - truthPoints[j])/normalizedLength);
+        }
     }
-    qDebug() << "Skipped " << skipped << " files due to point size mismatch.";
+
+    qDebug() << "Skipped" << skipped << "files due to point size mismatch.";
  
     QList<float> averagePointErrors; averagePointErrors.reserve(pointErrors.size());
+    QList<float> medianPointErrors; medianPointErrors.reserve(pointErrors.size());
+    QList<float> stddevPointErrors; stddevPointErrors.reserve(pointErrors.size());
+
+    float normalizedErrorLimit = 1.5;
+
+    QSet<int> worstExamples;
     for (int i=0; i<pointErrors.size(); i++) {
+        if (skipped == 0) {
+            QList<QPair<float,int> > exampleIndices = Common::Sort(pointErrors[i],true);
+            for (int j=0; j<exampleIndices.size() && worstExamples.size()<(5*i+1); j++)
+                if (exampleIndices[j].first < normalizedErrorLimit)
+                    worstExamples.insert(exampleIndices[j].second);
+        }
         std::sort(pointErrors[i].begin(), pointErrors[i].end());
-        averagePointErrors.append(Common::Mean(pointErrors[i]));
+        double mean, stddev;
+        Common::MeanStdDev(pointErrors[i],&mean,&stddev);
+        averagePointErrors.append(mean);
+        stddevPointErrors.append(stddev);
+        medianPointErrors.append(Common::Median(pointErrors[i]));
     }
     const float averagePointError = Common::Mean(averagePointErrors);
+    const float medianPointError = Common::Mean(medianPointErrors);
+    const float stddevPointError = Common::Mean(stddevPointErrors);
  
     QStringList lines;
     lines.append("Plot,X,Y");
+
+    QFile exampleFile("landmarking_examples");
+    QtUtils::touchDir(exampleFile);
+    lines.append("EX,landmarking_examples/"+truth[sampleIndex].file.fileName()+","+QString::number(truth[sampleIndex].file.points().size()));
+
+    // Alternatively, can we just pass this through a predetermined transform and write?
+    Enroll(truth[sampleIndex],"landmarking_examples");
+
     for (int i=0; i<pointErrors.size(); i++) {
         const QList<float> &pointError = pointErrors[i];
         const int keep = qMin(Max_Points, pointError.size());
@@ -1081,7 +1112,18 @@ float EvalLandmarking(const QString &amp;predictedGallery, const QString &amp;truthGalle
     lines.append(QString("AvgError,0,%1").arg(averagePointError));
  
     QtUtils::writeFile(csv, lines);
-    qDebug("Average Error: %.3f", averagePointError);
+
+    for (int i=0; i<averagePointErrors.size(); i++) {
+        qDebug("Average Error for point %d: %.3f", i, averagePointErrors[i]);
+        qDebug("Median Error for point %d: %.3f", i, medianPointErrors[i]);
+        qDebug("Standard Deviation of Error for point %d: %.3f", i, stddevPointErrors[i]);
+    }
+
+    qDebug("Average Error for all Points: %.3f", averagePointError);
+    qDebug("Average Median Error for all Points: %.3f", medianPointError);
+    qDebug("Average Standard Deviation of Error for all Points: %.3f", stddevPointError);
+    qDebug("Average Normalization Length (pixels): %.3f", Common::Mean(normalizedLengths));
+
     return averagePointError;
 }
  
@@ -31,7 +31,7 @@ namespace br
  
     void EvalClassification(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = "");
     float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", bool normalize = false, int minSize = 0, int maxSize = 0); // Return average overlap
-    float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", int normalizationIndexA = 0, int normalizationIndexB = 1); // Return average error
+    float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", int normalizationIndexA = 0, int normalizationIndexB = 1, int sampleIndex = 0); // Return average error
     void EvalRegression(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = "");
 }
  
@@ -472,13 +472,72 @@ bool PlotLandmarking(const QStringList &amp;files, const File &amp;destination, bool sho
     p.file.write("# Split data into individual plots\n"
                  "plot_index = which(names(data)==\"Plot\")\n"
                  "Box <- data[grep(\"Box\",data$Plot),-c(1)]\n"
+                 "EX <- data[grep(\"EX\",data$Plot),-c(1)]\n"
+                 "EX$X <- as.character(EX$X)\n"
+                 "EX$Y <- as.character(EX$Y)\n"
                  "rm(data)\n"
                  "\n");
  
-    p.file.write(qPrintable(QString("ggplot(Box, aes(Y,%1%2))").arg(p.major.size > 1 ? QString(", colour=%1").arg(p.major.header) : QString(), p.minor.size > 1 ? QString(", linetype=%1").arg(p.minor.header) : QString()) +
+    // Load in the relevant libraries
+    p.file.write(qPrintable(QString("if (nrow(EX) != 0) { \
+                                    \n\tlibrary(jpeg) \
+                                    \n\tlibrary(png) \
+                                    \n\tlibrary(grid)\n\t") +
+
+                            QString("multiplot <- function(..., plotlist=NULL, cols) {") +
+                            QString("\n\t\t require(grid) \
+                                    \n\n\t\t# Make a list from the ... arguments and plotlist \
+                                    \n\t\t plots <- c(list(...), plotlist)\n") +
+
+                            QString("\t\tnumPlots = length(plots)\n\n\t\t \
+                                     # Make the panel \
+                                     \n\t\tplotCols = cols \
+                                     \n\t\tplotRows = ceiling(numPlots/plotCols) \
+                                     \n\n") +
+
+                            QString("\t\t# Set up the page \
+                                     \n\t\tgrid.newpage() \
+                                     \n\t\tpushViewport(viewport(layout = grid.layout(plotRows, plotCols))) \
+                                     \n\t\tvplayout <- function(x, y) \
+                                     \n\t\t\tviewport(layout.pos.row = x, layout.pos.col = y)\n\n") +
+
+                            QString("\t\t# Make each plot, in the correct location \
+                                     \n\t\tfor (i in 1:numPlots) { \
+                                     \n\t\t\tcurRow = ceiling(i/plotCols)\n\t\t\tcurCol = (i-1) %% plotCols + 1 \
+                                     \n\t\t\tprint(plots[[i]], vp = vplayout(curRow, curCol))\n\t\t}\n\t}\n\n")));
+
+    p.file.write(qPrintable(QString("\n\n\t# Print genuine matches below the EER \
+                                     \n\t \
+                                     for (i in 1:nrow(EX)) { \
+                                     \n\t\t path <- EX[i,1] \
+                                     \n\t\t points <- EX[i,2] \
+                                     \n\t\t file <- unlist(strsplit(path, \"[.]\"))[1] \
+                                     \n\t\t ext <- unlist(strsplit(path, \"[.]\"))[2]") +
+
+                            // These should be made into a function assuming we can return an image variable regardless of extension
+                            QString("\n\t\t\tif (ext == \"jpg\" || ext == \"JPEG\" || ext == \"jpeg\" || ext == \"JPG\") { \
+                                     \n\t\t\t img <- readJPEG(path)\n\t\t } \
+                                     else if (ext == \"PNG\" || ext == \"png\") { \
+                                     \n\t\t\t img <- readPNG(path)\n\t\t} \
+                                     else if (ext == \"TIFF\" || ext == \"tiff\" || ext == \"TIF\" || ext == \"tif\") { \
+                                     \n\t\t\t img <- readTIFF(path)\n\t\t} \
+                                     else {\n\t\t\tnext\n\t\t} ") +
+
+                            QString("\n\t\t name <- file \
+                                     \n\n\t\t g <- rasterGrob(img, interpolate=TRUE)\n\n\t\t") +
+
+                            QString("print(qplot(1:10, 1:10, geom=\"blank\") \
+                                    + annotation_custom(g, xmin=-Inf, xmax=Inf, ymin=-Inf, ymax=Inf) \
+                                    + theme(axis.line=element_blank(), axis.title.y=element_blank(), axis.text.x=element_blank(), axis.text.y=element_blank(), line=element_blank(), axis.ticks=element_blank(), panel.background=element_blank()) \
+                                    + labs(title=\"Sample Landmarks\") + xlab(sprintf(\"Total Landmarks: %s\",points)))\n\t\t}}\n")));
+
+    p.file.write(qPrintable(QString("ggplot(Box, aes(Y,%1%2))").arg(p.major.size > 1 ? QString(", colour=%1").arg(p.major.header) : QString(),
+                                                                    p.minor.size > 1 ? QString(", linetype=%1").arg(p.minor.header) : QString()) +
                             QString(" + annotation_logticks(sides=\"b\") + stat_ecdf() + scale_x_log10(\"Normalized Error\", breaks=c(0.001,0.01,0.1,1,10)) + scale_y_continuous(\"Cumulative Density\", label=percent) + theme_minimal()\n\n")));
+
     p.file.write(qPrintable(QString("ggplot(Box, aes(factor(X), Y%1%2))").arg(p.major.size > 1 ? QString(", colour=%1").arg(p.major.header) : QString(), p.minor.size > 1 ? QString(", linetype=%1").arg(p.minor.header) : QString()) +
                             QString("+ annotation_logticks(sides=\"l\") + geom_boxplot(alpha=0.5) + geom_jitter(size=1, alpha=0.5) + scale_x_discrete(\"Landmark\") + scale_y_log10(\"Normalized Error\", breaks=c(0.01,0.1,1,10)) + theme_minimal()\n\n")));
+
     p.file.write(qPrintable(QString("ggplot(Box, aes(factor(X), Y%1%2))").arg(p.major.size > 1 ? QString(", colour=%1").arg(p.major.header) : QString(), p.minor.size > 1 ? QString(", linetype=%1").arg(p.minor.header) : QString()) +
                             QString("+ annotation_logticks(sides=\"l\") + geom_violin(alpha=0.5) + scale_x_discrete(\"Landmark\") + scale_y_log10(\"Normalized Error\", breaks=c(0.001,0.01,0.1,1,10))\n\n")));
  
@@ -134,9 +134,9 @@ float br_eval_detection(const char *predicted_gallery, const char *truth_gallery
     return EvalDetection(predicted_gallery, truth_gallery, csv, normalize, minSize, maxSize);
 }
  
-float br_eval_landmarking(const char *predicted_gallery, const char *truth_gallery, const char *csv, int normalization_index_a, int normalization_index_b)
+float br_eval_landmarking(const char *predicted_gallery, const char *truth_gallery, const char *csv, int normalization_index_a, int normalization_index_b, int sample_index)
 {
-    return EvalLandmarking(predicted_gallery, truth_gallery, csv, normalization_index_a, normalization_index_b);
+    return EvalLandmarking(predicted_gallery, truth_gallery, csv, normalization_index_a, normalization_index_b, sample_index);
 }
  
 void br_eval_regression(const char *predicted_gallery, const char *truth_gallery, const char *predicted_property, const char *truth_property)
@@ -214,8 +214,9 @@ BR_EXPORT float br_eval_detection(const char *predicted_gallery, const char *tru
  * \param csv Optional \c .csv file to contain performance metrics.
  * \param normalization_index_a Optional first index in the list of points to use for normalization.
  * \param normalization_index_b Optional second index in the list of points to use for normalization.
+ * \param sample_index Optional index for sample landmark image in ground truth gallery.
  */
-BR_EXPORT float br_eval_landmarking(const char *predicted_gallery, const char *truth_gallery, const char *csv = "", int normalization_index_a = 0, int normalization_index_b = 1);
+BR_EXPORT float br_eval_landmarking(const char *predicted_gallery, const char *truth_gallery, const char *csv = "", int normalization_index_a = 0, int normalization_index_b = 1, int sample_index = 0);
  
 /*!
  * \brief Evaluates regression accuracy to disk.