Peter M. Groen / openbr

Commit d6832ad69f47fab2f05b1e715df805d37bfb0dc2

Authored by bhklein
Committed by Scott Klum
1 parent 608cb6b1

Allow user specification of evaluation range, sample FRR at fixed values

Inline Side-by-side
Showing 3 changed files with 30 additions and 32 deletions
openbr/core/eval.cpp
  View file @d6832ad
... ... @@ -134,17 +134,17 @@ static cv::Mat constructMatchingMask(const cv::Mat &scores, const FileList &targ
134 134 return cv::Mat();
135 135 }
136 136  
137   -float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const QString &csv, int partition)
  137 +float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const File &csv, int partition)
138 138 {
139 139 return Evaluate(scores, constructMatchingMask(scores, target, query, partition), csv, QString(), QString(), 0);
140 140 }
141 141  
142   -float Evaluate(const QString &simmat, const QString &mask, const QString &csv, unsigned int matches)
  142 +float Evaluate(const QString &simmat, const QString &mask, const File &csv, unsigned int matches)
143 143 {
144 144 qDebug("Evaluating %s%s%s",
145 145 qPrintable(simmat),
146 146 mask.isEmpty() ? "" : qPrintable(" with " + mask),
147   - csv.isEmpty() ? "" : qPrintable(" to " + csv));
  147 + csv.name.isEmpty() ? "" : qPrintable(" to " + csv));
148 148  
149 149 // Read similarity matrix
150 150 QString target, query;
... ... @@ -176,7 +176,7 @@ float Evaluate(const QString &simmat, const QString &mask, const QString &csv, u
176 176 return Evaluate(scores, truth, csv, target, query, matches);
177 177 }
178 178  
179   -float Evaluate(const Mat &simmat, const Mat &mask, const QString &csv, const QString &target, const QString &query, unsigned int matches)
  179 +float Evaluate(const Mat &simmat, const Mat &mask, const File &csv, const QString &target, const QString &query, unsigned int matches)
180 180 {
181 181 if (target.isEmpty() || query.isEmpty()) matches = 0;
182 182 if (simmat.size() != mask.size())
... ... @@ -232,7 +232,7 @@ float Evaluate(const Mat &simmat, const Mat &mask, const QString &csv, const QSt
232 232 if (numNaNs > 0) qWarning("Encountered %d NaN scores!", numNaNs);
233 233 if (genuineCount == 0) qFatal("No genuine scores!");
234 234 if (impostorCount == 0) qFatal("No impostor scores!");
235   - if (totalImpostorSearches == 0) totalImpostorSearches = 1;
  235 + if (totalImpostorSearches == 0) totalImpostorSearches = 10;
236 236  
237 237 // Sort comparisons by simmat_val (score)
238 238 std::stable_sort(comparisons.begin(), comparisons.end());
... ... @@ -346,33 +346,31 @@ float Evaluate(const Mat &simmat, const Mat &mask, const QString &csv, const QSt
346 346 }
347 347 }
348 348  
349   - // Write Detection Error Tradeoff (DET), PRE, REC
350   - float expFAR = ceil(log10(impostorCount));
  349 + // Write Detection Error Tradeoff (DET), PRE, REC, Identification Error Tradeoff (IET)
  350 + float expFAR = csv.get<float>("FAR", ceil(log10(impostorCount)));
  351 + float expFRR = csv.get<float>("FRR", ceil(log10(genuineCount)));
  352 + float expFPIR = csv.get<float>("FPIR", ceil(log10(totalImpostorSearches)));
  353 +
351 354 float FARstep = expFAR / (float)(Max_Points - 1);
352   - for (float i = -expFAR; i <= 0; i += FARstep) {
353   - float FAR = pow(10, i);
354   - OperatingPoint operatingPoint = getOperatingPointGivenFAR(operatingPoints, FAR);
  355 + float FRRstep = expFRR / (float)(Max_Points - 1);
  356 + float FPIRstep = expFPIR / (float)(Max_Points - 1);
  357 +
  358 + for (int i=0; i<Max_Points; i++) {
  359 + float FAR = pow(10, -expFAR + i*FARstep);
  360 + float FRR = pow(10, -expFRR + i*FRRstep);
  361 + float FPIR = pow(10, -expFPIR + i*FPIRstep);
  362 +
  363 + OperatingPoint operatingPointFAR = getOperatingPointGivenFAR(operatingPoints, FAR);
  364 + OperatingPoint operatingPointTAR = getOperatingPointGivenTAR(operatingPoints, 1-FRR);
  365 + OperatingPoint searchOperatingPoint = getOperatingPointGivenFAR(searchOperatingPoints, FPIR);
355 366 lines.append(QString("DET,%1,%2").arg(QString::number(FAR),
356   - QString::number(1-operatingPoint.TAR)));
357   - lines.append(QString("FAR,%1,%2").arg(QString::number(operatingPoint.score),
  367 + QString::number(1-operatingPointFAR.TAR)));
  368 + lines.append(QString("FAR,%1,%2").arg(QString::number(operatingPointFAR.score),
358 369 QString::number(FAR)));
359   - lines.append(QString("FRR,%1,%2").arg(QString::number(operatingPoint.score),
360   - QString::number(1-operatingPoint.TAR)));
361   - }
362   -
363   - // Write Identification Error Tradeoff (IET)
364   - float expFPIR = ceil(log10(totalImpostorSearches));
365   - if (expFPIR > 1) {
366   - float FPIRstep = expFPIR / (float)(Max_Points - 1);
367   - for (float i = -expFPIR; i <= 0; i += FPIRstep) {
368   - float FPIR = pow(10, i);
369   - OperatingPoint searchOperatingPoint = getOperatingPointGivenFAR(searchOperatingPoints, FPIR);
370   - lines.append(QString("IET,%1,%2").arg(QString::number(searchOperatingPoint.FAR),
  370 + lines.append(QString("FRR,%1,%2").arg(QString::number(operatingPointTAR.score),
  371 + QString::number(FRR)));
  372 + lines.append(QString("IET,%1,%2").arg(QString::number(searchOperatingPoint.FAR),
371 373 QString::number(1-searchOperatingPoint.TAR)));
372   - }
373   - } else {
374   - lines.append(QString("IET,%1,%2").arg(QString::number(1), QString::number(1)));
375   - lines.append(QString("IET,%1,%2").arg(QString::number(0), QString::number(0)));
376 374 }
377 375  
378 376 // Write TAR@FAR Table (TF)
... ...
openbr/core/eval.h
  View file @d6832ad
... ... @@ -23,9 +23,9 @@
23 23  
24 24 namespace br
25 25 {
26   - float Evaluate(const QString &simmat, const QString &mask = "", const QString &csv = "", unsigned int matches = 0); // Returns TAR @ FAR = 0.001
27   - float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const QString &csv = "", int parition = 0);
28   - float Evaluate(const cv::Mat &scores, const cv::Mat &masks, const QString &csv = "", const QString &target = "", const QString &query = "", unsigned int matches = 0);
  26 + float Evaluate(const QString &simmat, const QString &mask = "", const File &csv = "", unsigned int matches = 0); // Returns TAR @ FAR = 0.001
  27 + float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const File &csv = "", int parition = 0);
  28 + float Evaluate(const cv::Mat &scores, const cv::Mat &masks, const File &csv = "", const QString &target = "", const QString &query = "", unsigned int matches = 0);
29 29 void assertEval(const QString &simmat, const QString &mask, float accuracy); // Check to see if -eval achieves a given TAR @ FAR = 0.001
30 30 float InplaceEval(const QString & simmat, const QString & target, const QString & query, const QString & csv = "");
31 31  
... ...
openbr/core/plot.cpp
  View file @d6832ad
... ... @@ -381,7 +381,7 @@ bool Plot(const QStringList &amp;files, const File &amp;destination, bool show)
381 381 (p.minor.size > 1 ? QString(" + facet_grid(%2 ~ X)").arg(p.minor.header) : QString(" + facet_grid(. ~ X, labeller=far_labeller)")) +
382 382 QString(" + scale_y_continuous(labels=percent) + theme(legend.position=\"none\", axis.text.x=element_text(angle=-90, hjust=0))%1").arg((p.major.smooth || p.minor.smooth) ? "" : " + geom_text(data=BC, aes(label=Y, y=0.05))") + "\n\n"));
383 383  
384   - p.file.write(qPrintable(QString("qplot(X, Y, data=ERR%1, linetype=Error").arg((p.major.smooth || p.minor.smooth) ? ", geom=\"smooth\", method=loess, level=0.99" : ", geom=\"line\"") +
  384 + p.file.write(qPrintable(QString("qplot(X, Y, data=ERR, geom=\"line\", linetype=Error") +
385 385 ((p.flip ? p.major.size : p.minor.size) > 1 ? QString(", colour=factor(%1)").arg(p.flip ? p.major.header : p.minor.header) : QString()) +
386 386 QString(", xlab=\"Score\", ylab=\"Error Rate\") + theme_minimal()") +
387 387 ((p.flip ? p.major.size : p.minor.size) > 1 ? getScale("colour", p.flip ? p.major.header : p.minor.header, p.flip ? p.major.size : p.minor.size) : QString()) +
... ...