Allow user specification of evaluation range, sample FRR at fixed values

bhklein · Scott Klum
1 parent 608cb6b1
Showing 3 changed files with 30 additions and 32 deletions
openbr/core/eval.cpp
openbr/core/eval.h
openbr/core/plot.cpp
@@ -134,17 +134,17 @@ static cv::Mat constructMatchingMask(const cv::Mat &amp;scores, const FileList &amp;targ
     return cv::Mat();
 }
-float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const QString &csv, int partition)
+float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const File &csv, int partition)
 {
     return Evaluate(scores, constructMatchingMask(scores, target, query, partition), csv, QString(), QString(), 0);
 }
-float Evaluate(const QString &simmat, const QString &mask, const QString &csv, unsigned int matches)
+float Evaluate(const QString &simmat, const QString &mask, const File &csv, unsigned int matches)
 {
     qDebug("Evaluating %s%s%s",
            qPrintable(simmat),
            mask.isEmpty() ? "" : qPrintable(" with " + mask),
-           csv.isEmpty() ? "" : qPrintable(" to " + csv));
+           csv.name.isEmpty() ? "" : qPrintable(" to " + csv));
     // Read similarity matrix
     QString target, query;
@@ -176,7 +176,7 @@ float Evaluate(const QString &amp;simmat, const QString &amp;mask, const QString &amp;csv, u
     return Evaluate(scores, truth, csv, target, query, matches);
 }
-float Evaluate(const Mat &simmat, const Mat &mask, const QString &csv, const QString &target, const QString &query, unsigned int matches)
+float Evaluate(const Mat &simmat, const Mat &mask, const File &csv, const QString &target, const QString &query, unsigned int matches)
 {
     if (target.isEmpty() || query.isEmpty()) matches = 0;
     if (simmat.size() != mask.size())
@@ -232,7 +232,7 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv, const QSt
     if (numNaNs > 0) qWarning("Encountered %d NaN scores!", numNaNs);
     if (genuineCount == 0) qFatal("No genuine scores!");
     if (impostorCount == 0) qFatal("No impostor scores!");
-    if (totalImpostorSearches == 0) totalImpostorSearches = 1;
+    if (totalImpostorSearches == 0) totalImpostorSearches = 10;
     // Sort comparisons by simmat_val (score)
     std::stable_sort(comparisons.begin(), comparisons.end());
@@ -346,33 +346,31 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv, const QSt
         }
     }
-    // Write Detection Error Tradeoff (DET), PRE, REC
-    float expFAR = ceil(log10(impostorCount));
+    // Write Detection Error Tradeoff (DET), PRE, REC, Identification Error Tradeoff (IET)
+    float expFAR = csv.get<float>("FAR", ceil(log10(impostorCount)));
+    float expFRR = csv.get<float>("FRR", ceil(log10(genuineCount)));
+    float expFPIR = csv.get<float>("FPIR", ceil(log10(totalImpostorSearches)));
+
     float FARstep = expFAR / (float)(Max_Points - 1);
-    for (float i = -expFAR; i <= 0; i += FARstep) {
-        float FAR = pow(10, i);
-        OperatingPoint operatingPoint = getOperatingPointGivenFAR(operatingPoints, FAR);
+    float FRRstep = expFRR / (float)(Max_Points - 1);
+    float FPIRstep = expFPIR / (float)(Max_Points - 1);
+
+    for (int i=0; i<Max_Points; i++) {
+        float FAR = pow(10, -expFAR + i*FARstep);
+        float FRR = pow(10, -expFRR + i*FRRstep);
+        float FPIR = pow(10, -expFPIR + i*FPIRstep);
+
+        OperatingPoint operatingPointFAR = getOperatingPointGivenFAR(operatingPoints, FAR);
+        OperatingPoint operatingPointTAR = getOperatingPointGivenTAR(operatingPoints, 1-FRR);
+        OperatingPoint searchOperatingPoint = getOperatingPointGivenFAR(searchOperatingPoints, FPIR);
         lines.append(QString("DET,%1,%2").arg(QString::number(FAR),
-                                              QString::number(1-operatingPoint.TAR)));
-        lines.append(QString("FAR,%1,%2").arg(QString::number(operatingPoint.score),
+                                              QString::number(1-operatingPointFAR.TAR)));
+        lines.append(QString("FAR,%1,%2").arg(QString::number(operatingPointFAR.score),
                                               QString::number(FAR)));
-        lines.append(QString("FRR,%1,%2").arg(QString::number(operatingPoint.score),
-                                              QString::number(1-operatingPoint.TAR)));
-    }
-
-    // Write Identification Error Tradeoff (IET)
-    float expFPIR = ceil(log10(totalImpostorSearches));
-    if (expFPIR > 1) {
-        float FPIRstep = expFPIR / (float)(Max_Points - 1);
-        for (float i = -expFPIR; i <= 0; i += FPIRstep) {
-            float FPIR = pow(10, i);
-            OperatingPoint searchOperatingPoint = getOperatingPointGivenFAR(searchOperatingPoints, FPIR);
-            lines.append(QString("IET,%1,%2").arg(QString::number(searchOperatingPoint.FAR),
+        lines.append(QString("FRR,%1,%2").arg(QString::number(operatingPointTAR.score),
+                                              QString::number(FRR)));
+        lines.append(QString("IET,%1,%2").arg(QString::number(searchOperatingPoint.FAR),
                                               QString::number(1-searchOperatingPoint.TAR)));
-        }
-    } else {
-        lines.append(QString("IET,%1,%2").arg(QString::number(1), QString::number(1)));
-        lines.append(QString("IET,%1,%2").arg(QString::number(0), QString::number(0)));
     }
     // Write TAR@FAR Table (TF)
@@ -23,9 +23,9 @@
 namespace br
 {
-    float Evaluate(const QString &simmat, const QString &mask = "", const QString &csv = "", unsigned int matches = 0); // Returns TAR @ FAR = 0.001
-    float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const QString &csv = "", int parition = 0);
-    float Evaluate(const cv::Mat &scores, const cv::Mat &masks, const QString &csv = "", const QString &target = "", const QString &query = "", unsigned int matches = 0);
+    float Evaluate(const QString &simmat, const QString &mask = "", const File &csv = "", unsigned int matches = 0); // Returns TAR @ FAR = 0.001
+    float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const File &csv = "", int parition = 0);
+    float Evaluate(const cv::Mat &scores, const cv::Mat &masks, const File &csv = "", const QString &target = "", const QString &query = "", unsigned int matches = 0);
     void assertEval(const QString &simmat, const QString &mask, float accuracy); // Check to see if -eval achieves a given TAR @ FAR = 0.001
     float InplaceEval(const QString & simmat, const QString & target, const QString & query, const QString & csv = "");
@@ -381,7 +381,7 @@ bool Plot(const QStringList &amp;files, const File &amp;destination, bool show)
                             (p.minor.size > 1 ? QString(" + facet_grid(%2 ~ X)").arg(p.minor.header) : QString(" + facet_grid(. ~ X, labeller=far_labeller)")) +
                             QString(" + scale_y_continuous(labels=percent) + theme(legend.position=\"none\", axis.text.x=element_text(angle=-90, hjust=0))%1").arg((p.major.smooth || p.minor.smooth) ? "" : " + geom_text(data=BC, aes(label=Y, y=0.05))") + "\n\n"));
-    p.file.write(qPrintable(QString("qplot(X, Y, data=ERR%1, linetype=Error").arg((p.major.smooth || p.minor.smooth) ? ", geom=\"smooth\", method=loess, level=0.99" : ", geom=\"line\"") +
+    p.file.write(qPrintable(QString("qplot(X, Y, data=ERR, geom=\"line\", linetype=Error") +
                             ((p.flip ? p.major.size : p.minor.size) > 1 ? QString(", colour=factor(%1)").arg(p.flip ? p.major.header : p.minor.header) : QString()) +
                             QString(", xlab=\"Score\", ylab=\"Error Rate\") + theme_minimal()") +
                             ((p.flip ? p.major.size : p.minor.size) > 1 ? getScale("colour", p.flip ? p.major.header : p.minor.header, p.flip ? p.major.size : p.minor.size) : QString()) +