From 0a55e14fe472bf8dfd6eb384fdcc535fc36d1dc6 Mon Sep 17 00:00:00 2001 From: bhklein Date: Thu, 28 Aug 2014 16:35:24 -0400 Subject: [PATCH] Output top impostor matches and bottom genuine matches from eval, plot the matches in pdf. --- app/br/br.cpp | 25 +++++++++++++++++++------ openbr/core/eval.cpp | 59 +++++++++++++++++++++++++++++++++++++++++++++++++++-------- openbr/core/eval.h | 4 ++-- openbr/core/plot.cpp | 23 +++++++++++++++++++++++ openbr/openbr.cpp | 4 ++-- openbr/openbr.h | 3 ++- 6 files changed, 99 insertions(+), 19 deletions(-) diff --git a/app/br/br.cpp b/app/br/br.cpp index 088511a..20858b2 100644 --- a/app/br/br.cpp +++ b/app/br/br.cpp @@ -98,14 +98,27 @@ public: check((parc >= 2) && (parc <= 3), "Incorrect parameter count for 'pairwiseCompare'."); br_pairwise_compare(parv[0], parv[1], parc == 3 ? parv[2] : ""); } else if (!strcmp(fun, "eval")) { - check((parc >= 1) && (parc <= 3), "Incorrect parameter count for 'eval'."); + check((parc >= 1) && (parc <= 4), "Incorrect parameter count for 'eval'."); if (parc == 1) { - br_eval(parv[0], "", ""); + br_eval(parv[0], "", "", 0); } else if (parc == 2) { - if (br::File(parv[1]).suffix() == "csv") br_eval(parv[0], "", parv[1]); - else br_eval(parv[0], parv[1], ""); + if (br::File(parv[1]).suffix() == "csv") { + br_eval(parv[0], "", parv[1], 0); + } else if (br::File(parv[1]).suffix() == "mask") { + br_eval(parv[0], parv[1], "", 0); + } else { + br_eval(parv[0], "", "", atoi(parv[1])); + } + } else if (parc == 3) { + if (br::File(parv[2]).suffix() == "csv") { + br_eval(parv[0], parv[1], parv[2], 0); + } else if ( br::File(parv[1]).suffix() == "csv") { + br_eval(parv[0], "", parv[1], atoi(parv[2])); + } else { + br_eval(parv[0], parv[1], "", atoi(parv[2])); + } } else { - br_eval(parv[0], parv[1], parv[2]); + br_eval(parv[0], parv[1], parv[2], atoi(parv[3])); } } else if (!strcmp(fun, "inplaceEval")) { check((parc >= 3) && (parc <= 4), "Incorrect parameter count for 'inplaceEval'."); @@ -235,7 +248,7 @@ private: "-train ... [{model}]\n" "-enroll ... {output_gallery}\n" "-compare [{output}]\n" - "-eval [] [{csv}]\n" + "-eval [] [{csv}] [{matches}]\n" "-plot ... {destination}\n" "\n" "==== Other Commands ====\n" diff --git a/openbr/core/eval.cpp b/openbr/core/eval.cpp index 98ffe97..8ebcd48 100755 --- a/openbr/core/eval.cpp +++ b/openbr/core/eval.cpp @@ -100,10 +100,10 @@ static cv::Mat constructMatchingMask(const cv::Mat &scores, const FileList &targ float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const QString &csv, int partition) { - return Evaluate(scores, constructMatchingMask(scores, target, query, partition), csv); + return Evaluate(scores, constructMatchingMask(scores, target, query, partition), target, query, csv, true); } -float Evaluate(const QString &simmat, const QString &mask, const QString &csv) +float Evaluate(const QString &simmat, const QString &mask, const QString &csv, bool matches) { qDebug("Evaluating %s%s%s", qPrintable(simmat), @@ -137,10 +137,10 @@ float Evaluate(const QString &simmat, const QString &mask, const QString &csv) truth = format->read(); } - return Evaluate(scores, truth, csv); + return Evaluate(scores, truth, TemplateList::fromGallery(target).files(), TemplateList::fromGallery(query).files(), csv, matches); } -float Evaluate(const Mat &simmat, const Mat &mask, const QString &csv) +float Evaluate(const Mat &simmat, const Mat &mask, const FileList &target, const FileList &query, const QString &csv, bool matches) { if (simmat.size() != mask.size()) qFatal("Similarity matrix (%ix%i) differs in size from mask matrix (%ix%i).", @@ -154,6 +154,10 @@ float Evaluate(const Mat &simmat, const Mat &mask, const QString &csv) float result = -1; + // Lists of top impostors and worst genuine + QList topImpostors; topImpostors.reserve(10); + QList botGenuines; botGenuines.reserve(10); + // Make comparisons QList comparisons; comparisons.reserve(simmat.rows*simmat.cols); int genuineCount = 0, impostorCount = 0, numNaNs = 0; @@ -163,9 +167,30 @@ float Evaluate(const Mat &simmat, const Mat &mask, const QString &csv) const BEE::SimmatValue simmat_val = simmat.at(i,j); if (mask_val == BEE::DontCare) continue; if (simmat_val != simmat_val) { numNaNs++; continue; } - comparisons.append(Comparison(simmat_val, j, i, mask_val == BEE::Match)); - if (comparisons.last().genuine) genuineCount++; - else impostorCount++; + Comparison comparison(simmat_val, j, i, mask_val == BEE::Match); + comparisons.append(comparison); + if (comparison.genuine) { + genuineCount++; + if (botGenuines.size() < 10) { + botGenuines.append(comparison); + std::sort(botGenuines.begin(), botGenuines.end()); + } else if (comparison.score < botGenuines.first().score) { + botGenuines.removeFirst(); + botGenuines.append(comparison); + std::sort(botGenuines.begin(), botGenuines.end()); + } + } else { + impostorCount++; + if (topImpostors.size() < 10) { + topImpostors.append(comparison); + std::sort(topImpostors.begin(), topImpostors.end()); + } else if (topImpostors.last().score < comparison.score) { + topImpostors.removeLast(); + topImpostors.append(comparison); + std::sort(topImpostors.begin(), topImpostors.end()); + } + + } } } @@ -234,6 +259,18 @@ float Evaluate(const Mat &simmat, const Mat &mask, const QString &csv) lines.append("Metadata,"+QString::number(impostorCount)+",Impostor"); lines.append("Metadata,"+QString::number(simmat.cols*simmat.rows-(genuineCount+impostorCount))+",Ignored"); + QString filePath = Globals->path; + if (matches) { + for (int i=0; i("Label")+":" + +filePath+"/"+target[topImpostors[i].target].name+":"+query[topImpostors[i].query].get("Label")+":"+filePath+"/"+query[topImpostors[i].query].name); + } + for (int i=0; i("Label")+":" + +filePath+"/"+target[botGenuines[i].target].name+":"+query[botGenuines[i].query].get("Label")+":"+filePath+"/"+query[botGenuines[i].query].name); + } + } + // Write Detection Error Tradeoff (DET), PRE, REC int points = qMin(operatingPoints.size(), Max_Points); for (int i=0; i 1 ? QString(" + facet_wrap(~ %1, scales=\"free_x\")").arg(p.flip ? p.minor.header : p.major.header) : QString()) + QString(" + theme(aspect.ratio=1)\n\n"))); + p.file.write(qPrintable(QString("if (nrow(TI) != 0) {\n\tlibrary(jpeg)\n\tlibrary(png)\n\tlibrary(grid)\n\t") + + QString("multiplot <- function(..., plotlist=NULL, cols) {\n\t") + + QString("\trequire(grid)\n\n\t\t# Make a list from the ... arguments and plotlist\n\t\tplots <- c(list(...), plotlist)\n") + + QString("\t\tnumPlots = length(plots)\n\n\t\t# Make the panel\n\t\tplotCols = cols\n\t\tplotRows = ceiling(numPlots/plotCols)\n\n") + + QString("\t\t# Set up the page\n\t\tgrid.newpage()\n\t\tpushViewport(viewport(layout = grid.layout(plotRows, plotCols)))\n\t\tvplayout <- function(x, y)\n\t\t\tviewport(layout.pos.row = x, layout.pos.col = y)\n\n") + + QString("\t\t# Make each plot, in the correct location\n\t\tfor (i in 1:numPlots) {\n\t\t\tcurRow = ceiling(i/plotCols)\n\t\t\tcurCol = (i-1) %% plotCols + 1\n\t\t\tprint(plots[[i]], vp = vplayout(curRow, curCol))\n\t\t}\n\t}\n\n"))); + + p.file.write(qPrintable(QString("\t# Print top impostor matches\n\tfor (i in 1:nrow(TI)) {\n\t\tscore <- TI[i,1]\n\t\tfiles <- TI[i,2]\n\t\talg <- TI[i,3]\n\t\tfiles <- unlist(strsplit(files, \"[:]\"))\n\n\t\t") + + QString("name1 <- files[1]\n\t\timg1 <- readJPEG(files[2])\n\t\tname2 <- files[3]\n\t\timg2 <- readJPEG(files[4])\n\n\t\tg1 <- rasterGrob(img1, interpolate=TRUE)\n\t\tg2 <- rasterGrob(img2, interpolate=TRUE)\n\n\t\t") + + QString("plot1 <- qplot(1:10, 1:10, geom=\"blank\") + annotation_custom(g1, xmin=-Inf, xmax=Inf, ymin=-Inf, ymax=Inf) + theme(axis.line=element_blank(), axis.text.x=element_blank(), axis.text.y=element_blank(), axis.ticks=element_blank(), panel.background=element_blank()) + labs(title=alg) + ylab(files[2]) + xlab(name1)\n\t\t") + + QString("plot2 <- qplot(1:10, 1:10, geom=\"blank\") + annotation_custom(g2, xmin=-Inf, xmax=Inf, ymin=-Inf, ymax=Inf) + theme(axis.line=element_blank(), axis.text.x=element_blank(), axis.text.y=element_blank(), axis.ticks=element_blank(), panel.background=element_blank()) + labs(title=paste(\"Impostor score =\", score)) + ylab(files[4]) + xlab(name2)\n\n\t\t") + + QString("multiplot(plot1, plot2, cols=2)\n\t}"))); + + p.file.write(qPrintable(QString("\n\n\t# Print worst genuine matches\n\tfor (i in 1:nrow(BG)) {\n\t\tscore <- BG[i,1]\n\t\tfiles <- BG[i,2]\n\t\talg <- BG[i,3]\n\t\tfiles <- unlist(strsplit(files, \"[:]\"))\n\n\t\t") + + QString("name1 <- files[1]\n\t\timg1 <- readJPEG(files[2])\n\t\tname2 <- files[3]\n\t\timg2 <- readJPEG(files[4])\n\n\t\tg1 <- rasterGrob(img1, interpolate=TRUE)\n\t\tg2 <- rasterGrob(img2, interpolate=TRUE)\n\n\t\t") + + QString("plot1 <- qplot(1:10, 1:10, geom=\"blank\") + annotation_custom(g1, xmin=-Inf, xmax=Inf, ymin=-Inf, ymax=Inf) + theme(axis.line=element_blank(), axis.text.x=element_blank(), axis.text.y=element_blank(), axis.ticks=element_blank(), panel.background=element_blank()) + labs(title=alg) + ylab(files[2]) + xlab(name1)\n\t\t") + + QString("plot2 <- qplot(1:10, 1:10, geom=\"blank\") + annotation_custom(g2, xmin=-Inf, xmax=Inf, ymin=-Inf, ymax=Inf) + theme(axis.line=element_blank(), axis.text.x=element_blank(), axis.text.y=element_blank(), axis.ticks=element_blank(), panel.background=element_blank()) + labs(title=paste(\"Genuine score =\", score)) + ylab(files[4]) + xlab(name2)\n\n\t\t") + + QString("multiplot(plot1, plot2, cols=2)\n\t}\n}\n\n"))); + return p.finalize(show); } diff --git a/openbr/openbr.cpp b/openbr/openbr.cpp index ddac0a7..83e2eee 100644 --- a/openbr/openbr.cpp +++ b/openbr/openbr.cpp @@ -104,9 +104,9 @@ void br_project(const char *input, const char *gallery) Project(File(input), File(gallery)); } -float br_eval(const char *simmat, const char *mask, const char *csv) +float br_eval(const char *simmat, const char *mask, const char *csv, bool matches) { - return Evaluate(simmat, mask, csv); + return Evaluate(simmat, mask, csv, matches); } float br_inplace_eval(const char *simmat, const char *target, const char *query, const char *csv) diff --git a/openbr/openbr.h b/openbr/openbr.h index dd81b57..766602b 100644 --- a/openbr/openbr.h +++ b/openbr/openbr.h @@ -154,10 +154,11 @@ BR_EXPORT void br_project(const char *input, const char *output); * \param simmat The \ref simmat to use. * \param mask The \ref mask to use. * \param csv Optional \c .csv file to contain performance metrics. + * \param matches Optional bool flag to output top impostor matches and bottom genuine matches. * \return True accept rate at a false accept rate of one in one thousand. * \see br_plot */ -BR_EXPORT float br_eval(const char *simmat, const char *mask, const char *csv = ""); +BR_EXPORT float br_eval(const char *simmat, const char *mask, const char *csv = "", bool matches = false); /*! * \brief Creates a \c .csv file containing performance metrics from evaluating the similarity matrix using galleries containing ground truth labels -- libgit2 0.21.4