diff --git a/openbr/core/eval.cpp b/openbr/core/eval.cpp index 46b87d7..4310915 100755 --- a/openbr/core/eval.cpp +++ b/openbr/core/eval.cpp @@ -134,17 +134,17 @@ static cv::Mat constructMatchingMask(const cv::Mat &scores, const FileList &targ return cv::Mat(); } -float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const QString &csv, int partition) +float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const File &csv, int partition) { return Evaluate(scores, constructMatchingMask(scores, target, query, partition), csv, QString(), QString(), 0); } -float Evaluate(const QString &simmat, const QString &mask, const QString &csv, unsigned int matches) +float Evaluate(const QString &simmat, const QString &mask, const File &csv, unsigned int matches) { qDebug("Evaluating %s%s%s", qPrintable(simmat), mask.isEmpty() ? "" : qPrintable(" with " + mask), - csv.isEmpty() ? "" : qPrintable(" to " + csv)); + csv.name.isEmpty() ? "" : qPrintable(" to " + csv)); // Read similarity matrix QString target, query; @@ -176,7 +176,7 @@ float Evaluate(const QString &simmat, const QString &mask, const QString &csv, u return Evaluate(scores, truth, csv, target, query, matches); } -float Evaluate(const Mat &simmat, const Mat &mask, const QString &csv, const QString &target, const QString &query, unsigned int matches) +float Evaluate(const Mat &simmat, const Mat &mask, const File &csv, const QString &target, const QString &query, unsigned int matches) { if (target.isEmpty() || query.isEmpty()) matches = 0; if (simmat.size() != mask.size()) @@ -232,7 +232,7 @@ float Evaluate(const Mat &simmat, const Mat &mask, const QString &csv, const QSt if (numNaNs > 0) qWarning("Encountered %d NaN scores!", numNaNs); if (genuineCount == 0) qFatal("No genuine scores!"); if (impostorCount == 0) qFatal("No impostor scores!"); - if (totalImpostorSearches == 0) totalImpostorSearches = 1; + if (totalImpostorSearches == 0) totalImpostorSearches = 10; // Sort comparisons by simmat_val (score) std::stable_sort(comparisons.begin(), comparisons.end()); @@ -346,33 +346,31 @@ float Evaluate(const Mat &simmat, const Mat &mask, const QString &csv, const QSt } } - // Write Detection Error Tradeoff (DET), PRE, REC - float expFAR = ceil(log10(impostorCount)); + // Write Detection Error Tradeoff (DET), PRE, REC, Identification Error Tradeoff (IET) + float expFAR = csv.get("FAR", ceil(log10(impostorCount))); + float expFRR = csv.get("FRR", ceil(log10(genuineCount))); + float expFPIR = csv.get("FPIR", ceil(log10(totalImpostorSearches))); + float FARstep = expFAR / (float)(Max_Points - 1); - for (float i = -expFAR; i <= 0; i += FARstep) { - float FAR = pow(10, i); - OperatingPoint operatingPoint = getOperatingPointGivenFAR(operatingPoints, FAR); + float FRRstep = expFRR / (float)(Max_Points - 1); + float FPIRstep = expFPIR / (float)(Max_Points - 1); + + for (int i=0; i 1) { - float FPIRstep = expFPIR / (float)(Max_Points - 1); - for (float i = -expFPIR; i <= 0; i += FPIRstep) { - float FPIR = pow(10, i); - OperatingPoint searchOperatingPoint = getOperatingPointGivenFAR(searchOperatingPoints, FPIR); - lines.append(QString("IET,%1,%2").arg(QString::number(searchOperatingPoint.FAR), + lines.append(QString("FRR,%1,%2").arg(QString::number(operatingPointTAR.score), + QString::number(FRR))); + lines.append(QString("IET,%1,%2").arg(QString::number(searchOperatingPoint.FAR), QString::number(1-searchOperatingPoint.TAR))); - } - } else { - lines.append(QString("IET,%1,%2").arg(QString::number(1), QString::number(1))); - lines.append(QString("IET,%1,%2").arg(QString::number(0), QString::number(0))); } // Write TAR@FAR Table (TF) diff --git a/openbr/core/eval.h b/openbr/core/eval.h index a150e92..16ef375 100644 --- a/openbr/core/eval.h +++ b/openbr/core/eval.h @@ -23,9 +23,9 @@ namespace br { - float Evaluate(const QString &simmat, const QString &mask = "", const QString &csv = "", unsigned int matches = 0); // Returns TAR @ FAR = 0.001 - float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const QString &csv = "", int parition = 0); - float Evaluate(const cv::Mat &scores, const cv::Mat &masks, const QString &csv = "", const QString &target = "", const QString &query = "", unsigned int matches = 0); + float Evaluate(const QString &simmat, const QString &mask = "", const File &csv = "", unsigned int matches = 0); // Returns TAR @ FAR = 0.001 + float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const File &csv = "", int parition = 0); + float Evaluate(const cv::Mat &scores, const cv::Mat &masks, const File &csv = "", const QString &target = "", const QString &query = "", unsigned int matches = 0); void assertEval(const QString &simmat, const QString &mask, float accuracy); // Check to see if -eval achieves a given TAR @ FAR = 0.001 float InplaceEval(const QString & simmat, const QString & target, const QString & query, const QString & csv = ""); diff --git a/openbr/core/plot.cpp b/openbr/core/plot.cpp index 099da08..3bb1a10 100644 --- a/openbr/core/plot.cpp +++ b/openbr/core/plot.cpp @@ -381,7 +381,7 @@ bool Plot(const QStringList &files, const File &destination, bool show) (p.minor.size > 1 ? QString(" + facet_grid(%2 ~ X)").arg(p.minor.header) : QString(" + facet_grid(. ~ X, labeller=far_labeller)")) + QString(" + scale_y_continuous(labels=percent) + theme(legend.position=\"none\", axis.text.x=element_text(angle=-90, hjust=0))%1").arg((p.major.smooth || p.minor.smooth) ? "" : " + geom_text(data=BC, aes(label=Y, y=0.05))") + "\n\n")); - p.file.write(qPrintable(QString("qplot(X, Y, data=ERR%1, linetype=Error").arg((p.major.smooth || p.minor.smooth) ? ", geom=\"smooth\", method=loess, level=0.99" : ", geom=\"line\"") + + p.file.write(qPrintable(QString("qplot(X, Y, data=ERR, geom=\"line\", linetype=Error") + ((p.flip ? p.major.size : p.minor.size) > 1 ? QString(", colour=factor(%1)").arg(p.flip ? p.major.header : p.minor.header) : QString()) + QString(", xlab=\"Score\", ylab=\"Error Rate\") + theme_minimal()") + ((p.flip ? p.major.size : p.minor.size) > 1 ? getScale("colour", p.flip ? p.major.header : p.minor.header, p.flip ? p.major.size : p.minor.size) : QString()) +