From d6ae28496be88d981d27f70a2a326344a61c05e9 Mon Sep 17 00:00:00 2001 From: Scott Klum Date: Mon, 12 Jan 2015 15:24:22 -0500 Subject: [PATCH] Added output of sample landmarks, added index to control which image is used as sample --- app/br/br.cpp | 4 ++-- openbr/core/eval.cpp | 52 +++++++++++++++++++++++++++++++++++++++++++++++----- openbr/core/eval.h | 2 +- openbr/core/plot.cpp | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++- openbr/openbr.cpp | 4 ++-- openbr/openbr.h | 3 ++- 6 files changed, 114 insertions(+), 12 deletions(-) diff --git a/app/br/br.cpp b/app/br/br.cpp index 9e79f37..e135b93 100644 --- a/app/br/br.cpp +++ b/app/br/br.cpp @@ -163,8 +163,8 @@ public: check((parc >= 2) && (parc <= 6), "Incorrect parameter count for 'evalDetection'."); br_eval_detection(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? atoi(parv[3]) : 0, parc >= 5 ? atoi(parv[4]) : 0, parc == 6 ? atoi(parv[5]) : 0); } else if (!strcmp(fun, "evalLandmarking")) { - check((parc >= 2) && (parc <= 5), "Incorrect parameter count for 'evalLandmarking'."); - br_eval_landmarking(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? atoi(parv[3]) : 0, parc >= 5 ? atoi(parv[4]) : 1); + check((parc >= 2) && (parc <= 6), "Incorrect parameter count for 'evalLandmarking'."); + br_eval_landmarking(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? atoi(parv[3]) : 0, parc >= 5 ? atoi(parv[4]) : 1, parc >= 6 ? atoi(parv[5]) : 0); } else if (!strcmp(fun, "evalRegression")) { check(parc >= 2 && parc <= 4, "Incorrect parameter count for 'evalRegression'."); br_eval_regression(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? parv[3] : ""); diff --git a/openbr/core/eval.cpp b/openbr/core/eval.cpp index 10781ab..955deb4 100755 --- a/openbr/core/eval.cpp +++ b/openbr/core/eval.cpp @@ -1032,7 +1032,7 @@ float EvalDetection(const QString &predictedGallery, const QString &truthGallery return averageOverlap; } -float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv, int normalizationIndexA, int normalizationIndexB) +float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv, int normalizationIndexA, int normalizationIndexB, int sampleIndex) { qDebug("Evaluating landmarking of %s against %s", qPrintable(predictedGallery), qPrintable(truthGallery)); const TemplateList predicted(TemplateList::fromGallery(predictedGallery)); @@ -1042,6 +1042,8 @@ float EvalLandmarking(const QString &predictedGallery, const QString &truthGalle int skipped = 0; QList< QList > pointErrors; + QList normalizedLengths; + for (int i=0; i= truthPoints.size()) qFatal("Normalization index A is out of range."); if (normalizationIndexB >= truthPoints.size()) qFatal("Normalization index B is out of range."); const float normalizedLength = QtUtils::euclideanLength(truthPoints[normalizationIndexB] - truthPoints[normalizationIndexA]); - for (int j=0; j averagePointErrors; averagePointErrors.reserve(pointErrors.size()); + QList medianPointErrors; medianPointErrors.reserve(pointErrors.size()); + QList stddevPointErrors; stddevPointErrors.reserve(pointErrors.size()); + + float normalizedErrorLimit = 1.5; + + QSet worstExamples; for (int i=0; i > exampleIndices = Common::Sort(pointErrors[i],true); + for (int j=0; j &pointError = pointErrors[i]; const int keep = qMin(Max_Points, pointError.size()); @@ -1081,7 +1112,18 @@ float EvalLandmarking(const QString &predictedGallery, const QString &truthGalle lines.append(QString("AvgError,0,%1").arg(averagePointError)); QtUtils::writeFile(csv, lines); - qDebug("Average Error: %.3f", averagePointError); + + for (int i=0; i 1 ? QString(", colour=%1").arg(p.major.header) : QString(), p.minor.size > 1 ? QString(", linetype=%1").arg(p.minor.header) : QString()) + + // Load in the relevant libraries + p.file.write(qPrintable(QString("if (nrow(EX) != 0) { \ + \n\tlibrary(jpeg) \ + \n\tlibrary(png) \ + \n\tlibrary(grid)\n\t") + + + QString("multiplot <- function(..., plotlist=NULL, cols) {") + + QString("\n\t\t require(grid) \ + \n\n\t\t# Make a list from the ... arguments and plotlist \ + \n\t\t plots <- c(list(...), plotlist)\n") + + + QString("\t\tnumPlots = length(plots)\n\n\t\t \ + # Make the panel \ + \n\t\tplotCols = cols \ + \n\t\tplotRows = ceiling(numPlots/plotCols) \ + \n\n") + + + QString("\t\t# Set up the page \ + \n\t\tgrid.newpage() \ + \n\t\tpushViewport(viewport(layout = grid.layout(plotRows, plotCols))) \ + \n\t\tvplayout <- function(x, y) \ + \n\t\t\tviewport(layout.pos.row = x, layout.pos.col = y)\n\n") + + + QString("\t\t# Make each plot, in the correct location \ + \n\t\tfor (i in 1:numPlots) { \ + \n\t\t\tcurRow = ceiling(i/plotCols)\n\t\t\tcurCol = (i-1) %% plotCols + 1 \ + \n\t\t\tprint(plots[[i]], vp = vplayout(curRow, curCol))\n\t\t}\n\t}\n\n"))); + + p.file.write(qPrintable(QString("\n\n\t# Print genuine matches below the EER \ + \n\t \ + for (i in 1:nrow(EX)) { \ + \n\t\t path <- EX[i,1] \ + \n\t\t points <- EX[i,2] \ + \n\t\t file <- unlist(strsplit(path, \"[.]\"))[1] \ + \n\t\t ext <- unlist(strsplit(path, \"[.]\"))[2]") + + + // These should be made into a function assuming we can return an image variable regardless of extension + QString("\n\t\t\tif (ext == \"jpg\" || ext == \"JPEG\" || ext == \"jpeg\" || ext == \"JPG\") { \ + \n\t\t\t img <- readJPEG(path)\n\t\t } \ + else if (ext == \"PNG\" || ext == \"png\") { \ + \n\t\t\t img <- readPNG(path)\n\t\t} \ + else if (ext == \"TIFF\" || ext == \"tiff\" || ext == \"TIF\" || ext == \"tif\") { \ + \n\t\t\t img <- readTIFF(path)\n\t\t} \ + else {\n\t\t\tnext\n\t\t} ") + + + QString("\n\t\t name <- file \ + \n\n\t\t g <- rasterGrob(img, interpolate=TRUE)\n\n\t\t") + + + QString("print(qplot(1:10, 1:10, geom=\"blank\") \ + + annotation_custom(g, xmin=-Inf, xmax=Inf, ymin=-Inf, ymax=Inf) \ + + theme(axis.line=element_blank(), axis.title.y=element_blank(), axis.text.x=element_blank(), axis.text.y=element_blank(), line=element_blank(), axis.ticks=element_blank(), panel.background=element_blank()) \ + + labs(title=\"Sample Landmarks\") + xlab(sprintf(\"Total Landmarks: %s\",points)))\n\t\t}}\n"))); + + p.file.write(qPrintable(QString("ggplot(Box, aes(Y,%1%2))").arg(p.major.size > 1 ? QString(", colour=%1").arg(p.major.header) : QString(), + p.minor.size > 1 ? QString(", linetype=%1").arg(p.minor.header) : QString()) + QString(" + annotation_logticks(sides=\"b\") + stat_ecdf() + scale_x_log10(\"Normalized Error\", breaks=c(0.001,0.01,0.1,1,10)) + scale_y_continuous(\"Cumulative Density\", label=percent) + theme_minimal()\n\n"))); + p.file.write(qPrintable(QString("ggplot(Box, aes(factor(X), Y%1%2))").arg(p.major.size > 1 ? QString(", colour=%1").arg(p.major.header) : QString(), p.minor.size > 1 ? QString(", linetype=%1").arg(p.minor.header) : QString()) + QString("+ annotation_logticks(sides=\"l\") + geom_boxplot(alpha=0.5) + geom_jitter(size=1, alpha=0.5) + scale_x_discrete(\"Landmark\") + scale_y_log10(\"Normalized Error\", breaks=c(0.01,0.1,1,10)) + theme_minimal()\n\n"))); + p.file.write(qPrintable(QString("ggplot(Box, aes(factor(X), Y%1%2))").arg(p.major.size > 1 ? QString(", colour=%1").arg(p.major.header) : QString(), p.minor.size > 1 ? QString(", linetype=%1").arg(p.minor.header) : QString()) + QString("+ annotation_logticks(sides=\"l\") + geom_violin(alpha=0.5) + scale_x_discrete(\"Landmark\") + scale_y_log10(\"Normalized Error\", breaks=c(0.001,0.01,0.1,1,10))\n\n"))); diff --git a/openbr/openbr.cpp b/openbr/openbr.cpp index b18f019..96ec913 100644 --- a/openbr/openbr.cpp +++ b/openbr/openbr.cpp @@ -134,9 +134,9 @@ float br_eval_detection(const char *predicted_gallery, const char *truth_gallery return EvalDetection(predicted_gallery, truth_gallery, csv, normalize, minSize, maxSize); } -float br_eval_landmarking(const char *predicted_gallery, const char *truth_gallery, const char *csv, int normalization_index_a, int normalization_index_b) +float br_eval_landmarking(const char *predicted_gallery, const char *truth_gallery, const char *csv, int normalization_index_a, int normalization_index_b, int sample_index) { - return EvalLandmarking(predicted_gallery, truth_gallery, csv, normalization_index_a, normalization_index_b); + return EvalLandmarking(predicted_gallery, truth_gallery, csv, normalization_index_a, normalization_index_b, sample_index); } void br_eval_regression(const char *predicted_gallery, const char *truth_gallery, const char *predicted_property, const char *truth_property) diff --git a/openbr/openbr.h b/openbr/openbr.h index a21c93c..6dcf3e5 100644 --- a/openbr/openbr.h +++ b/openbr/openbr.h @@ -214,8 +214,9 @@ BR_EXPORT float br_eval_detection(const char *predicted_gallery, const char *tru * \param csv Optional \c .csv file to contain performance metrics. * \param normalization_index_a Optional first index in the list of points to use for normalization. * \param normalization_index_b Optional second index in the list of points to use for normalization. + * \param sample_index Optional index for sample landmark image in ground truth gallery. */ -BR_EXPORT float br_eval_landmarking(const char *predicted_gallery, const char *truth_gallery, const char *csv = "", int normalization_index_a = 0, int normalization_index_b = 1); +BR_EXPORT float br_eval_landmarking(const char *predicted_gallery, const char *truth_gallery, const char *csv = "", int normalization_index_a = 0, int normalization_index_b = 1, int sample_index = 0); /*! * \brief Evaluates regression accuracy to disk. -- libgit2 0.21.4