diff --git a/app/br/br.cpp b/app/br/br.cpp index 9e79f37..2e2293a 100644 --- a/app/br/br.cpp +++ b/app/br/br.cpp @@ -163,8 +163,8 @@ public: check((parc >= 2) && (parc <= 6), "Incorrect parameter count for 'evalDetection'."); br_eval_detection(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? atoi(parv[3]) : 0, parc >= 5 ? atoi(parv[4]) : 0, parc == 6 ? atoi(parv[5]) : 0); } else if (!strcmp(fun, "evalLandmarking")) { - check((parc >= 2) && (parc <= 5), "Incorrect parameter count for 'evalLandmarking'."); - br_eval_landmarking(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? atoi(parv[3]) : 0, parc >= 5 ? atoi(parv[4]) : 1); + check((parc >= 2) && (parc <= 7), "Incorrect parameter count for 'evalLandmarking'."); + br_eval_landmarking(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? atoi(parv[3]) : 0, parc >= 5 ? atoi(parv[4]) : 1, parc >= 6 ? atoi(parv[5]) : 0, parc >= 7 ? atoi(parv[6]) : 5); } else if (!strcmp(fun, "evalRegression")) { check(parc >= 2 && parc <= 4, "Incorrect parameter count for 'evalRegression'."); br_eval_regression(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? parv[3] : ""); @@ -264,7 +264,7 @@ private: "-evalClassification \n" "-evalClustering \n" "-evalDetection [{csv}] [{normalize}] [{minSize}]\n" - "-evalLandmarking [{csv} [ ]]\n" + "-evalLandmarking [{csv} [ ] [sample_index] [total_examples]]\n" "-evalRegression \n" "-assertEval \n" "-plotDetection ... {destination}\n" diff --git a/openbr/core/eval.cpp b/openbr/core/eval.cpp index ef5bc1c..006b322 100755 --- a/openbr/core/eval.cpp +++ b/openbr/core/eval.cpp @@ -18,6 +18,7 @@ #include "eval.h" #include "openbr/core/common.h" #include "openbr/core/qtutils.h" +#include "openbr/core/opencvutils.h" #include using namespace cv; @@ -1068,16 +1069,25 @@ float EvalDetection(const QString &predictedGallery, const QString &truthGallery return averageOverlap; } -float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv, int normalizationIndexA, int normalizationIndexB) +static void projectAndWrite(Transform *t, const Template &src, const QString &filePath) +{ + Template dst; + t->project(src,dst); + OpenCVUtils::saveImage(dst.m(),filePath); +} + +float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv, int normalizationIndexA, int normalizationIndexB, int sampleIndex, int totalExamples) { qDebug("Evaluating landmarking of %s against %s", qPrintable(predictedGallery), qPrintable(truthGallery)); - const TemplateList predicted(TemplateList::fromGallery(predictedGallery)); - const TemplateList truth(TemplateList::fromGallery(truthGallery)); - const QStringList predictedNames = File::get(predicted, "name"); - const QStringList truthNames = File::get(truth, "name"); + TemplateList predicted(TemplateList::fromGallery(predictedGallery)); + TemplateList truth(TemplateList::fromGallery(truthGallery)); + QStringList predictedNames = File::get(predicted, "name"); + QStringList truthNames = File::get(truth, "name"); int skipped = 0; QList< QList > pointErrors; + QList imageErrors; + QList normalizedLengths; for (int i=0; i predictedPoints = predicted[i].file.points(); const QList truthPoints = truth[truthIndex].file.points(); if (predictedPoints.size() != truthPoints.size() || truthPoints.contains(QPointF(-1,-1))) { - skipped++; + predicted.removeAt(i); + predictedNames.removeAt(i); + truth.removeAt(i); + truthNames.removeAt(i); + i--; skipped++; continue; } + while (pointErrors.size() < predictedPoints.size()) pointErrors.append(QList()); + + // Want to know error for every image. + if (normalizationIndexA >= truthPoints.size()) qFatal("Normalization index A is out of range."); if (normalizationIndexB >= truthPoints.size()) qFatal("Normalization index B is out of range."); const float normalizedLength = QtUtils::euclideanLength(truthPoints[normalizationIndexB] - truthPoints[normalizationIndexA]); - for (int j=0; j averagePointErrors; averagePointErrors.reserve(pointErrors.size()); - for (int i=0; i t(Transform::make("Open+Draw(verbose,rects=false,location=false)",NULL)); + + QString filePath = "landmarking_examples_truth/"+truth[sampleIndex].file.fileName(); + projectAndWrite(t.data(), truth[sampleIndex],filePath); + lines.append("Sample,"+filePath+","+QString::number(truth[sampleIndex].file.points().size())); + } + + // Get best and worst performing examples + QList< QPair > exampleIndices = Common::Sort(imageErrors,true); + + QScopedPointer t(Transform::make("Open+Draw(rects=false)",NULL)); + + for (int i=0; iexampleIndices.size()-totalExamples-1; i--) { + QString filePath = "landmarking_examples_truth/"+truth[exampleIndices[i].second].file.fileName(); + projectAndWrite(t.data(), truth[exampleIndices[i].second],filePath); + lines.append("EXT,"+filePath+","+QString::number(exampleIndices[i].first)); + + filePath = "landmarking_examples_predicted/"+predicted[exampleIndices[i].second].file.fileName(); + projectAndWrite(t.data(), predicted[exampleIndices[i].second],filePath); + lines.append("EXP,"+filePath+","+QString::number(exampleIndices[i].first)); + } + for (int i=0; i &pointError = pointErrors[i]; const int keep = qMin(Max_Points, pointError.size()); for (int j=0; j 1 ? QString(", colour=%1").arg(p.major.header) : QString(), p.minor.size > 1 ? QString(", linetype=%1").arg(p.minor.header) : QString()) + + p.file.write(qPrintable(QString("# Split data into individual plots\n" + "plot_index = which(names(data)==\"Plot\")\n" + "Box <- data[grep(\"Box\",data$Plot),-c(1)]\n" + "Box$X <- factor(Box$X, levels = Box$X, ordered = TRUE)\n" + "Sample <- data[grep(\"Sample\",data$Plot),-c(1)]\n" + "Sample$X <- as.character(Sample$X)\n" + "EXT <- data[grep(\"EXT\",data$Plot),-c(1)]\n" + "EXT$X <- as.character(EXT$X)\n" + "EXP <- data[grep(\"EXP\",data$Plot),-c(1)]\n" + "EXP$X <- as.character(EXP$X)\n" + "NormLength <- data[grep(\"NormLength\",data$Plot),-c(1)]\n" + "rm(data)\n" + "\n"))); + + p.file.write(qPrintable(QString("summarySE <- function(data=NULL, measurevar, groupvars=NULL, na.rm=FALSE, conf.interval=.95, .drop=TRUE) {\n\t" + "require(plyr)\n\n\tlength2 <- function (x, na.rm=FALSE) {\n\t\tif (na.rm) sum(!is.na(x))\n\t\telse length(x)" + "\n\t}\n\n\tdatac <- ddply(data, groupvars, .drop=.drop, .fun = function(xx, col) {\n\t\t" + "c(N=length2(xx[[col]], na.rm=na.rm), mean=mean(xx[[col]], na.rm=na.rm), sd=sd(xx[[col]], na.rm=na.rm))\n\t\t}," + "\n\t\tmeasurevar\n\t)\n\n\tdatac <- rename(datac, c(\"mean\" = measurevar))\n\tdatac$se <- datac$sd / sqrt(datac$N)" + "\n\tciMult <- qt(conf.interval/2 + .5, datac$N-1)\n\tdatac$ci <- datac$se * ciMult\n\n\treturn(datac)\n}\n"))); + + + p.file.write(qPrintable(QString("\nreadData <- function(data) {\n\texamples <- list()\n" + "\tfor (i in 1:nrow(data)) {\n" + "\t\tpath <- data[i,1]\n" + "\t\tvalue <- data[i,2]\n" + "\t\tfile <- unlist(strsplit(path, \"[.]\"))[1]\n" + "\t\text <- unlist(strsplit(path, \"[.]\"))[2]\n" + "\t\tif (ext == \"jpg\" || ext == \"JPEG\" || ext == \"jpeg\" || ext == \"JPG\") {\n" + "\t\t\timg <- readJPEG(path)\n" + "\t\t} else if (ext == \"PNG\" || ext == \"png\") {\n" + "\t\t\timg <- readPNG(path)\n" + "\t\t} else if (ext == \"TIFF\" || ext == \"tiff\" || ext == \"TIF\" || ext == \"tif\") { \n" + "\t\t\timg <- readTIFF(path)\n" + "}else {\n" + "\t\t\tnext\n" + "\t\t}\n" + "\t\texample <- list(file = file, value = value, image = img)\n" + "\t\texamples[[i]] <- example\n" + "\t}\n" + "\treturn(examples)\n" + "}\n"))); + + p.file.write(qPrintable(QString("\nlibrary(jpeg)\n" + "library(png)\n" + "library(grid)\n" + "multiplot <- function(..., plotlist=NULL, cols) {\n" + "\trequire(grid)\n" + "\t# Make a list from the ... arguments and plotlist\n" + "\tplots <- c(list(...), plotlist)\n" + "\tnumPlots = length(plots)\n" + "\t# Make the panel\n" + "\tplotCols = cols\n" + "\tplotRows = ceiling(numPlots/plotCols)\n" + "\t# Set up the page\n" + "\tgrid.newpage()\n" + "\tpushViewport(viewport(layout = grid.layout(plotRows, plotCols)))\n" + "\tvplayout <- function(x, y)\n" + "\tviewport(layout.pos.row = x, layout.pos.col = y)\n" + "\t# Make each plot, in the correct location\n" + "\tfor (i in 1:numPlots) {\n" + "\t\tcurRow = ceiling(i/plotCols)\n" + "\t\tcurCol = (i-1) %% plotCols + 1\n" + "\t\tprint(plots[[i]], vp = vplayout(curRow, curCol))\n" + "\t}\n" + "}\n"))); + + p.file.write(qPrintable(QString("\nplotImage <- function(image, title=NULL, label=NULL) { \n" + "\tp <- qplot(1:10, 1:10, geom=\"blank\") + annotation_custom(rasterGrob(image$image), xmin=-Inf, xmax=Inf, ymin=-Inf, ymax=Inf) + theme(axis.line=element_blank(), axis.title.y=element_blank(), axis.text.x=element_blank(), axis.text.y=element_blank(), line=element_blank(), axis.ticks=element_blank(), panel.background=element_blank()) + labs(title=title) + xlab(label)\n" + "\treturn(p)" + "}\n"))); + + p.file.write(qPrintable(QString("\nsample <- readData(Sample) \n" + "rows <- sample[[1]]$value\n" + "algs <- unique(Box$%1)\n" + "algs <- algs[!duplicated(algs)]\n" + "print(plotImage(sample[[1]],\"Sample Landmarks\",sprintf(\"Total Landmarks: %s\",sample[[1]]$value))) \n" + "if (nrow(EXT) != 0 && nrow(EXP)) {\n" + "\tfor (j in 1:length(algs)) {\n" + "\ttruthSample <- readData(EXT[EXT$. == algs[[j]],])\n" + "\tpredictedSample <- readData(EXP[EXP$. == algs[[j]],])\n" + "\t\tfor (i in 1:length(predictedSample)) {\n" + "\t\t\tmultiplot(plotImage(predictedSample[[i]],sprintf(\"%s\\nPredicted Landmarks\",algs[[j]]),sprintf(\"Average Landmark Error: %.3f\",predictedSample[[i]]$value)),plotImage(truthSample[[i]],\"Ground Truth\\nLandmarks\",\"\"),cols=2)\n" + "\t\t}\n" + "\t}\n" + "}\n").arg(p.major.size > 1 ? p.major.header : (p.minor.header.isEmpty() ? p.major.header : p.minor.header)))); + + p.file.write(qPrintable(QString("\n" + "# Code to format error table\n" + "StatBox <- summarySE(Box, measurevar=\"Y\", groupvars=c(\"%1\",\"X\"))\n" + "OverallStatBox <- summarySE(Box, measurevar=\"Y\", groupvars=c(\"%1\"))\n" + "mat <- matrix(paste(as.character(round(StatBox$Y, 3)), round(StatBox$ci, 3), sep=\" \\u00b1 \"),nrow=rows,ncol=length(algs),byrow=FALSE)\n" + "mat <- rbind(mat, paste(as.character(round(OverallStatBox$Y, 3)), round(OverallStatBox$ci, 3), sep=\" \\u00b1 \"))\n" + "mat <- rbind(mat, as.character(round(NormLength$Y, 3)))\n" + "colnames(mat) <- algs\n" + "rownames(mat) <- c(seq(0,rows-1),\"Aggregate\",\"Average IPD\")\n" + "ETable <- as.table(mat)\n").arg(p.major.size > 1 ? p.major.header : (p.minor.header.isEmpty() ? p.major.header : p.minor.header)))); + + p.file.write(qPrintable(QString("\n" + "print(textplot(ETable))\n" + "print(title(\"Landmarking Error Rates\"))\n"))); + + p.file.write(qPrintable(QString("ggplot(Box, aes(Y,%1%2))").arg(p.major.size > 1 ? QString(", colour=%1").arg(p.major.header) : QString(), + p.minor.size > 1 ? QString(", linetype=%1").arg(p.minor.header) : QString()) + QString(" + annotation_logticks(sides=\"b\") + stat_ecdf() + scale_x_log10(\"Normalized Error\", breaks=c(0.001,0.01,0.1,1,10)) + scale_y_continuous(\"Cumulative Density\", label=percent) + theme_minimal()\n\n"))); + p.file.write(qPrintable(QString("ggplot(Box, aes(factor(X), Y%1%2))").arg(p.major.size > 1 ? QString(", colour=%1").arg(p.major.header) : QString(), p.minor.size > 1 ? QString(", linetype=%1").arg(p.minor.header) : QString()) + - QString("+ annotation_logticks(sides=\"l\") + geom_boxplot(alpha=0.5) + geom_jitter(size=1, alpha=0.5) + scale_x_discrete(\"Landmark\") + scale_y_log10(\"Normalized Error\", breaks=c(0.01,0.1,1,10)) + theme_minimal()\n\n"))); + QString("+ annotation_logticks(sides=\"l\") + geom_boxplot(alpha=0.5) + geom_jitter(size=1, alpha=0.5) + scale_x_discrete(\"Landmark\") + scale_y_log10(\"Normalized Error\", breaks=c(0.001,0.01,0.1,1,10)) + theme_minimal()\n\n"))); + p.file.write(qPrintable(QString("ggplot(Box, aes(factor(X), Y%1%2))").arg(p.major.size > 1 ? QString(", colour=%1").arg(p.major.header) : QString(), p.minor.size > 1 ? QString(", linetype=%1").arg(p.minor.header) : QString()) + QString("+ annotation_logticks(sides=\"l\") + geom_violin(alpha=0.5) + scale_x_discrete(\"Landmark\") + scale_y_log10(\"Normalized Error\", breaks=c(0.001,0.01,0.1,1,10))\n\n"))); diff --git a/openbr/openbr.cpp b/openbr/openbr.cpp index b18f019..3162371 100644 --- a/openbr/openbr.cpp +++ b/openbr/openbr.cpp @@ -134,9 +134,9 @@ float br_eval_detection(const char *predicted_gallery, const char *truth_gallery return EvalDetection(predicted_gallery, truth_gallery, csv, normalize, minSize, maxSize); } -float br_eval_landmarking(const char *predicted_gallery, const char *truth_gallery, const char *csv, int normalization_index_a, int normalization_index_b) +float br_eval_landmarking(const char *predicted_gallery, const char *truth_gallery, const char *csv, int normalization_index_a, int normalization_index_b, int sample_index, int total_examples) { - return EvalLandmarking(predicted_gallery, truth_gallery, csv, normalization_index_a, normalization_index_b); + return EvalLandmarking(predicted_gallery, truth_gallery, csv, normalization_index_a, normalization_index_b, sample_index, total_examples); } void br_eval_regression(const char *predicted_gallery, const char *truth_gallery, const char *predicted_property, const char *truth_property) diff --git a/openbr/openbr.h b/openbr/openbr.h index a21c93c..4f32e86 100644 --- a/openbr/openbr.h +++ b/openbr/openbr.h @@ -214,8 +214,10 @@ BR_EXPORT float br_eval_detection(const char *predicted_gallery, const char *tru * \param csv Optional \c .csv file to contain performance metrics. * \param normalization_index_a Optional first index in the list of points to use for normalization. * \param normalization_index_b Optional second index in the list of points to use for normalization. + * \param sample_index Optional index for sample landmark image in ground truth gallery. + * \param total_examples Optional number of accurate and inaccurate examples to display. */ -BR_EXPORT float br_eval_landmarking(const char *predicted_gallery, const char *truth_gallery, const char *csv = "", int normalization_index_a = 0, int normalization_index_b = 1); +BR_EXPORT float br_eval_landmarking(const char *predicted_gallery, const char *truth_gallery, const char *csv = "", int normalization_index_a = 0, int normalization_index_b = 1, int sample_index = 0, int total_examples = 5); /*! * \brief Evaluates regression accuracy to disk. diff --git a/openbr/plugins/draw.cpp b/openbr/plugins/draw.cpp index 7c6a1a7..3dc6a9b 100644 --- a/openbr/plugins/draw.cpp +++ b/openbr/plugins/draw.cpp @@ -42,11 +42,15 @@ class DrawTransform : public UntrainableTransform Q_PROPERTY(bool rects READ get_rects WRITE set_rects RESET reset_rects STORED false) Q_PROPERTY(bool inPlace READ get_inPlace WRITE set_inPlace RESET reset_inPlace STORED false) Q_PROPERTY(int lineThickness READ get_lineThickness WRITE set_lineThickness RESET reset_lineThickness STORED false) + Q_PROPERTY(bool named READ get_named WRITE set_named RESET reset_named STORED false) + Q_PROPERTY(bool location READ get_location WRITE set_location RESET reset_location STORED false) BR_PROPERTY(bool, verbose, false) BR_PROPERTY(bool, points, true) BR_PROPERTY(bool, rects, true) BR_PROPERTY(bool, inPlace, false) BR_PROPERTY(int, lineThickness, 1) + BR_PROPERTY(bool, named, true) + BR_PROPERTY(bool, location, true) void project(const Template &src, Template &dst) const { @@ -55,11 +59,12 @@ class DrawTransform : public UntrainableTransform dst.m() = inPlace ? src.m() : src.m().clone(); if (points) { - const QList pointsList = OpenCVUtils::toPoints(src.file.namedPoints() + src.file.points()); + const QList pointsList = (named) ? OpenCVUtils::toPoints(src.file.points()+src.file.namedPoints()) : OpenCVUtils::toPoints(src.file.points()); for (int i=0; i