Peter M. Groen / openbr

Commit 0a55e14fe472bf8dfd6eb384fdcc535fc36d1dc6

Authored by bhklein
1 parent 33dd7d19

Output top impostor matches and bottom genuine matches from eval, plot the matches in pdf.

Inline Side-by-side
Showing 6 changed files with 99 additions and 19 deletions
app/br/br.cpp
  View file @0a55e14
... ... @@ -98,14 +98,27 @@ public:
98 98 check((parc >= 2) && (parc <= 3), "Incorrect parameter count for 'pairwiseCompare'.");
99 99 br_pairwise_compare(parv[0], parv[1], parc == 3 ? parv[2] : "");
100 100 } else if (!strcmp(fun, "eval")) {
101   - check((parc >= 1) && (parc <= 3), "Incorrect parameter count for 'eval'.");
  101 + check((parc >= 1) && (parc <= 4), "Incorrect parameter count for 'eval'.");
102 102 if (parc == 1) {
103   - br_eval(parv[0], "", "");
  103 + br_eval(parv[0], "", "", 0);
104 104 } else if (parc == 2) {
105   - if (br::File(parv[1]).suffix() == "csv") br_eval(parv[0], "", parv[1]);
106   - else br_eval(parv[0], parv[1], "");
  105 + if (br::File(parv[1]).suffix() == "csv") {
  106 + br_eval(parv[0], "", parv[1], 0);
  107 + } else if (br::File(parv[1]).suffix() == "mask") {
  108 + br_eval(parv[0], parv[1], "", 0);
  109 + } else {
  110 + br_eval(parv[0], "", "", atoi(parv[1]));
  111 + }
  112 + } else if (parc == 3) {
  113 + if (br::File(parv[2]).suffix() == "csv") {
  114 + br_eval(parv[0], parv[1], parv[2], 0);
  115 + } else if ( br::File(parv[1]).suffix() == "csv") {
  116 + br_eval(parv[0], "", parv[1], atoi(parv[2]));
  117 + } else {
  118 + br_eval(parv[0], parv[1], "", atoi(parv[2]));
  119 + }
107 120 } else {
108   - br_eval(parv[0], parv[1], parv[2]);
  121 + br_eval(parv[0], parv[1], parv[2], atoi(parv[3]));
109 122 }
110 123 } else if (!strcmp(fun, "inplaceEval")) {
111 124 check((parc >= 3) && (parc <= 4), "Incorrect parameter count for 'inplaceEval'.");
... ... @@ -235,7 +248,7 @@ private:
235 248 "-train <gallery> ... <gallery> [{model}]\n"
236 249 "-enroll <input_gallery> ... <input_gallery> {output_gallery}\n"
237 250 "-compare <target_gallery> <query_gallery> [{output}]\n"
238   - "-eval <simmat> [<mask>] [{csv}]\n"
  251 + "-eval <simmat> [<mask>] [{csv}] [{matches}]\n"
239 252 "-plot <file> ... <file> {destination}\n"
240 253 "\n"
241 254 "==== Other Commands ====\n"
... ...
openbr/core/eval.cpp
  View file @0a55e14
... ... @@ -100,10 +100,10 @@ static cv::Mat constructMatchingMask(const cv::Mat &amp;scores, const FileList &amp;targ
100 100  
101 101 float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const QString &csv, int partition)
102 102 {
103   - return Evaluate(scores, constructMatchingMask(scores, target, query, partition), csv);
  103 + return Evaluate(scores, constructMatchingMask(scores, target, query, partition), target, query, csv, true);
104 104 }
105 105  
106   -float Evaluate(const QString &simmat, const QString &mask, const QString &csv)
  106 +float Evaluate(const QString &simmat, const QString &mask, const QString &csv, bool matches)
107 107 {
108 108 qDebug("Evaluating %s%s%s",
109 109 qPrintable(simmat),
... ... @@ -137,10 +137,10 @@ float Evaluate(const QString &amp;simmat, const QString &amp;mask, const QString &amp;csv)
137 137 truth = format->read();
138 138 }
139 139  
140   - return Evaluate(scores, truth, csv);
  140 + return Evaluate(scores, truth, TemplateList::fromGallery(target).files(), TemplateList::fromGallery(query).files(), csv, matches);
141 141 }
142 142  
143   -float Evaluate(const Mat &simmat, const Mat &mask, const QString &csv)
  143 +float Evaluate(const Mat &simmat, const Mat &mask, const FileList &target, const FileList &query, const QString &csv, bool matches)
144 144 {
145 145 if (simmat.size() != mask.size())
146 146 qFatal("Similarity matrix (%ix%i) differs in size from mask matrix (%ix%i).",
... ... @@ -154,6 +154,10 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv)
154 154  
155 155 float result = -1;
156 156  
  157 + // Lists of top impostors and worst genuine
  158 + QList<Comparison> topImpostors; topImpostors.reserve(10);
  159 + QList<Comparison> botGenuines; botGenuines.reserve(10);
  160 +
157 161 // Make comparisons
158 162 QList<Comparison> comparisons; comparisons.reserve(simmat.rows*simmat.cols);
159 163 int genuineCount = 0, impostorCount = 0, numNaNs = 0;
... ... @@ -163,9 +167,30 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv)
163 167 const BEE::SimmatValue simmat_val = simmat.at<BEE::SimmatValue>(i,j);
164 168 if (mask_val == BEE::DontCare) continue;
165 169 if (simmat_val != simmat_val) { numNaNs++; continue; }
166   - comparisons.append(Comparison(simmat_val, j, i, mask_val == BEE::Match));
167   - if (comparisons.last().genuine) genuineCount++;
168   - else impostorCount++;
  170 + Comparison comparison(simmat_val, j, i, mask_val == BEE::Match);
  171 + comparisons.append(comparison);
  172 + if (comparison.genuine) {
  173 + genuineCount++;
  174 + if (botGenuines.size() < 10) {
  175 + botGenuines.append(comparison);
  176 + std::sort(botGenuines.begin(), botGenuines.end());
  177 + } else if (comparison.score < botGenuines.first().score) {
  178 + botGenuines.removeFirst();
  179 + botGenuines.append(comparison);
  180 + std::sort(botGenuines.begin(), botGenuines.end());
  181 + }
  182 + } else {
  183 + impostorCount++;
  184 + if (topImpostors.size() < 10) {
  185 + topImpostors.append(comparison);
  186 + std::sort(topImpostors.begin(), topImpostors.end());
  187 + } else if (topImpostors.last().score < comparison.score) {
  188 + topImpostors.removeLast();
  189 + topImpostors.append(comparison);
  190 + std::sort(topImpostors.begin(), topImpostors.end());
  191 + }
  192 +
  193 + }
169 194 }
170 195 }
171 196  
... ... @@ -234,6 +259,18 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv)
234 259 lines.append("Metadata,"+QString::number(impostorCount)+",Impostor");
235 260 lines.append("Metadata,"+QString::number(simmat.cols*simmat.rows-(genuineCount+impostorCount))+",Ignored");
236 261  
  262 + QString filePath = Globals->path;
  263 + if (matches) {
  264 + for (int i=0; i<topImpostors.size(); i++) {
  265 + lines.append("TI,"+QString::number(topImpostors[i].score)+","+target[topImpostors[i].target].get<QString>("Label")+":"
  266 + +filePath+"/"+target[topImpostors[i].target].name+":"+query[topImpostors[i].query].get<QString>("Label")+":"+filePath+"/"+query[topImpostors[i].query].name);
  267 + }
  268 + for (int i=0; i<botGenuines.size(); i++) {
  269 + lines.append("BG,"+QString::number(botGenuines[i].score)+","+target[botGenuines[i].target].get<QString>("Label")+":"
  270 + +filePath+"/"+target[botGenuines[i].target].name+":"+query[botGenuines[i].query].get<QString>("Label")+":"+filePath+"/"+query[botGenuines[i].query].name);
  271 + }
  272 + }
  273 +
237 274 // Write Detection Error Tradeoff (DET), PRE, REC
238 275 int points = qMin(operatingPoints.size(), Max_Points);
239 276 for (int i=0; i<points; i++) {
... ... @@ -292,7 +329,13 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv)
292 329 }
293 330  
294 331 QtUtils::writeFile(csv, lines);
295   - qDebug("TAR @ FAR = 0.01: %.3f\nRetrieval Rate @ Rank = %d: %.3f", result, Report_Retrieval, getCMC(firstGenuineReturns, Report_Retrieval));
  332 + qDebug("TAR @ FAR = 0.01: %.3f",getTAR(operatingPoints, 0.01));
  333 + qDebug("TAR @ FAR = 0.001: %.3f",getTAR(operatingPoints, 0.001));
  334 + qDebug("TAR @ FAR = 0.0001: %.3f",getTAR(operatingPoints, 0.0001));
  335 + qDebug("TAR @ FAR = 0.00001: %.3f",getTAR(operatingPoints, 0.00001));
  336 +
  337 + qDebug("\nRetrieval Rate @ Rank = %d: %.3f", Report_Retrieval, getCMC(firstGenuineReturns, Report_Retrieval));
  338 +
296 339 return result;
297 340 }
298 341  
... ...
openbr/core/eval.h
  View file @0a55e14
... ... @@ -23,9 +23,9 @@
23 23  
24 24 namespace br
25 25 {
26   - float Evaluate(const QString &simmat, const QString &mask = "", const QString &csv = ""); // Returns TAR @ FAR = 0.001
  26 + float Evaluate(const QString &simmat, const QString &mask = "", const QString &csv = "", bool matches = false); // Returns TAR @ FAR = 0.001
27 27 float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const QString &csv = "", int parition = 0);
28   - float Evaluate(const cv::Mat &scores, const cv::Mat &masks, const QString &csv = "");
  28 + float Evaluate(const cv::Mat &scores, const cv::Mat &masks, const FileList &target, const FileList &query, const QString &csv = "", bool matches = false);
29 29 float InplaceEval(const QString & simmat, const QString & target, const QString & query, const QString & csv = "");
30 30  
31 31 void EvalClassification(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = "");
... ...
openbr/core/plot.cpp
  View file @0a55e14
... ... @@ -148,6 +148,8 @@ struct RPlot
148 148 "# Split data into individual plots\n"
149 149 "plot_index = which(names(data)==\"Plot\")\n"
150 150 "Metadata <- data[grep(\"Metadata\",data$Plot),-c(1)]\n"
  151 + "TI <- data[grep(\"TI\",data$Plot),-c(1)]\n"
  152 + "BG <- data[grep(\"BG\",data$Plot),-c(1)]\n"
151 153 "DET <- data[grep(\"DET\",data$Plot),-c(1)]\n"
152 154 "FAR <- data[grep(\"FAR\",data$Plot),-c(1)]\n"
153 155 "FRR <- data[grep(\"FRR\",data$Plot),-c(1)]\n"
... ... @@ -163,6 +165,8 @@ struct RPlot
163 165 "\n"
164 166 "# Format data\n"
165 167 "Metadata$Y<-factor(Metadata$Y, levels=c(\"Genuine\",\"Impostor\",\"Ignored\",\"Gallery\",\"Probe\"))\n"
  168 + "TI$Y <- as.character(TI$Y)\n"
  169 + "BG$Y <- as.character(BG$Y)\n"
166 170 "DET$Y <- as.numeric(as.character(DET$Y))\n"
167 171 "ERR$Y <- as.numeric(as.character(ERR$Y))\n"
168 172 "SD$Y <- as.factor(unique(as.character(SD$Y)))\n"
... ... @@ -317,6 +321,25 @@ bool Plot(const QStringList &amp;files, const File &amp;destination, bool show)
317 321 ((p.flip ? p.minor.size : p.major.size) > 1 ? QString(" + facet_wrap(~ %1, scales=\"free_x\")").arg(p.flip ? p.minor.header : p.major.header) : QString()) +
318 322 QString(" + theme(aspect.ratio=1)\n\n")));
319 323  
  324 + p.file.write(qPrintable(QString("if (nrow(TI) != 0) {\n\tlibrary(jpeg)\n\tlibrary(png)\n\tlibrary(grid)\n\t") +
  325 + QString("multiplot <- function(..., plotlist=NULL, cols) {\n\t") +
  326 + QString("\trequire(grid)\n\n\t\t# Make a list from the ... arguments and plotlist\n\t\tplots <- c(list(...), plotlist)\n") +
  327 + QString("\t\tnumPlots = length(plots)\n\n\t\t# Make the panel\n\t\tplotCols = cols\n\t\tplotRows = ceiling(numPlots/plotCols)\n\n") +
  328 + QString("\t\t# Set up the page\n\t\tgrid.newpage()\n\t\tpushViewport(viewport(layout = grid.layout(plotRows, plotCols)))\n\t\tvplayout <- function(x, y)\n\t\t\tviewport(layout.pos.row = x, layout.pos.col = y)\n\n") +
  329 + QString("\t\t# Make each plot, in the correct location\n\t\tfor (i in 1:numPlots) {\n\t\t\tcurRow = ceiling(i/plotCols)\n\t\t\tcurCol = (i-1) %% plotCols + 1\n\t\t\tprint(plots[[i]], vp = vplayout(curRow, curCol))\n\t\t}\n\t}\n\n")));
  330 +
  331 + p.file.write(qPrintable(QString("\t# Print top impostor matches\n\tfor (i in 1:nrow(TI)) {\n\t\tscore <- TI[i,1]\n\t\tfiles <- TI[i,2]\n\t\talg <- TI[i,3]\n\t\tfiles <- unlist(strsplit(files, \"[:]\"))\n\n\t\t") +
  332 + QString("name1 <- files[1]\n\t\timg1 <- readJPEG(files[2])\n\t\tname2 <- files[3]\n\t\timg2 <- readJPEG(files[4])\n\n\t\tg1 <- rasterGrob(img1, interpolate=TRUE)\n\t\tg2 <- rasterGrob(img2, interpolate=TRUE)\n\n\t\t") +
  333 + QString("plot1 <- qplot(1:10, 1:10, geom=\"blank\") + annotation_custom(g1, xmin=-Inf, xmax=Inf, ymin=-Inf, ymax=Inf) + theme(axis.line=element_blank(), axis.text.x=element_blank(), axis.text.y=element_blank(), axis.ticks=element_blank(), panel.background=element_blank()) + labs(title=alg) + ylab(files[2]) + xlab(name1)\n\t\t") +
  334 + QString("plot2 <- qplot(1:10, 1:10, geom=\"blank\") + annotation_custom(g2, xmin=-Inf, xmax=Inf, ymin=-Inf, ymax=Inf) + theme(axis.line=element_blank(), axis.text.x=element_blank(), axis.text.y=element_blank(), axis.ticks=element_blank(), panel.background=element_blank()) + labs(title=paste(\"Impostor score =\", score)) + ylab(files[4]) + xlab(name2)\n\n\t\t") +
  335 + QString("multiplot(plot1, plot2, cols=2)\n\t}")));
  336 +
  337 + p.file.write(qPrintable(QString("\n\n\t# Print worst genuine matches\n\tfor (i in 1:nrow(BG)) {\n\t\tscore <- BG[i,1]\n\t\tfiles <- BG[i,2]\n\t\talg <- BG[i,3]\n\t\tfiles <- unlist(strsplit(files, \"[:]\"))\n\n\t\t") +
  338 + QString("name1 <- files[1]\n\t\timg1 <- readJPEG(files[2])\n\t\tname2 <- files[3]\n\t\timg2 <- readJPEG(files[4])\n\n\t\tg1 <- rasterGrob(img1, interpolate=TRUE)\n\t\tg2 <- rasterGrob(img2, interpolate=TRUE)\n\n\t\t") +
  339 + QString("plot1 <- qplot(1:10, 1:10, geom=\"blank\") + annotation_custom(g1, xmin=-Inf, xmax=Inf, ymin=-Inf, ymax=Inf) + theme(axis.line=element_blank(), axis.text.x=element_blank(), axis.text.y=element_blank(), axis.ticks=element_blank(), panel.background=element_blank()) + labs(title=alg) + ylab(files[2]) + xlab(name1)\n\t\t") +
  340 + QString("plot2 <- qplot(1:10, 1:10, geom=\"blank\") + annotation_custom(g2, xmin=-Inf, xmax=Inf, ymin=-Inf, ymax=Inf) + theme(axis.line=element_blank(), axis.text.x=element_blank(), axis.text.y=element_blank(), axis.ticks=element_blank(), panel.background=element_blank()) + labs(title=paste(\"Genuine score =\", score)) + ylab(files[4]) + xlab(name2)\n\n\t\t") +
  341 + QString("multiplot(plot1, plot2, cols=2)\n\t}\n}\n\n")));
  342 +
320 343 return p.finalize(show);
321 344 }
322 345  
... ...
openbr/openbr.cpp
  View file @0a55e14
... ... @@ -104,9 +104,9 @@ void br_project(const char *input, const char *gallery)
104 104 Project(File(input), File(gallery));
105 105 }
106 106  
107   -float br_eval(const char *simmat, const char *mask, const char *csv)
  107 +float br_eval(const char *simmat, const char *mask, const char *csv, bool matches)
108 108 {
109   - return Evaluate(simmat, mask, csv);
  109 + return Evaluate(simmat, mask, csv, matches);
110 110 }
111 111  
112 112 float br_inplace_eval(const char *simmat, const char *target, const char *query, const char *csv)
... ...
openbr/openbr.h
  View file @0a55e14
... ... @@ -154,10 +154,11 @@ BR_EXPORT void br_project(const char *input, const char *output);
154 154 * \param simmat The \ref simmat to use.
155 155 * \param mask The \ref mask to use.
156 156 * \param csv Optional \c .csv file to contain performance metrics.
  157 + * \param matches Optional bool flag to output top impostor matches and bottom genuine matches.
157 158 * \return True accept rate at a false accept rate of one in one thousand.
158 159 * \see br_plot
159 160 */
160   -BR_EXPORT float br_eval(const char *simmat, const char *mask, const char *csv = "");
  161 +BR_EXPORT float br_eval(const char *simmat, const char *mask, const char *csv = "", bool matches = false);
161 162  
162 163 /*!
163 164 * \brief Creates a \c .csv file containing performance metrics from evaluating the similarity matrix using galleries containing ground truth labels
... ...