added ROC to evalEER

bhklein
1 parent 28362462
Showing 5 changed files with 90 additions and 62 deletions
app/br/br.cpp
openbr/core/eval.cpp
openbr/core/eval.h
openbr/openbr.cpp
openbr/openbr.h
@@ -172,8 +172,8 @@ public:
                 check(parc >= 2 && parc <= 3, "Incorrect parameter count for 'evalKNN'.");
                 br_eval_knn(parv[0], parv[1], parc > 2 ? parv[2] : "");
             } else if (!strcmp(fun, "evalEER")) {
-                check(parc >=1 && parc <=3 , "Incorrect parameter count for 'evalEER'.");
-                br_eval_eer(parv[0], parc > 1 ? parv[1] : "", parc > 2 ? parv[2] : "");
+                check(parc >=1 && parc <=4 , "Incorrect parameter count for 'evalEER'.");
+                br_eval_eer(parv[0], parc > 1 ? parv[1] : "", parc > 2 ? parv[2] : "", parc > 3 ? parv[3] : "");
             } else if (!strcmp(fun, "pairwiseCompare")) {
                 check((parc >= 2) && (parc <= 3), "Incorrect parameter count for 'pairwiseCompare'.");
                 br_pairwise_compare(parv[0], parv[1], parc == 3 ? parv[2] : "");
@@ -1242,69 +1242,97 @@ void EvalKNN(const QString &amp;knnGraph, const QString &amp;knnTruth, const QString &amp;cs
     qDebug("FNIR @ FPIR = 0.01:  %.3f", 1-getOperatingPointGivenFAR(operatingPoints, 0.01).TAR);
 }
-void EvalEER(const QString &predictedXML, QString gt_property, QString distribution_property){
+void EvalEER(const QString &predictedXML, QString gt_property, QString distribution_property, const QString &pdf) {
     if (gt_property.isEmpty())
-            gt_property = "LivenessGT";
+        gt_property = "LivenessGT";
     if (distribution_property.isEmpty())
-            distribution_property = "LivenessDistribution";
-    double classOneTemplateCount = 0;
+        distribution_property = "LivenessClassScores";
+    int classOneTemplateCount = 0;
     const TemplateList templateList(TemplateList::fromGallery(predictedXML));
     QHash<QString, int> gtLabels;
     QHash<QString, QList<float> > scores;
-     for (double i=0; i<templateList.size(); i++) {
-	 if (!templateList[i].file.contains(distribution_property) || !templateList[i].file.contains(gt_property))
-             continue;
-         QString templateKey = templateList[i].file.path() + templateList[i].file.baseName();
-         int gtLabel = templateList[i].file.get<int>(gt_property);
-         if (gtLabel == 1)
-             classOneTemplateCount++;
-         QList<float> templateScores = templateList[i].file.getList<float>(distribution_property);
-         gtLabels[templateKey] = gtLabel;
-         scores[templateKey] = templateScores;
-     }
-
-     const int numPoints = 200;
-     const float stepSize = 100.0/numPoints;
-     const double numTemplates = scores.size();
-     float thres = 0.0; //Between [0,100]
-     float thresNorm = 0.0; //Between [0,1]
-     double FA = 0, FR = 0;
-     float minDiff = 100;
-     float EER = 100;
-     float EERThres = 0;
-
-     for(int i = 0; i <= numPoints; i++){
-         FA = 0, FR = 0;
-         thresNorm = thres/100.0;
-            foreach(const QString &key, scores.keys()){
-                int gtLabel = gtLabels[key];
-                //> thresNorm = class 0 (spoof) : < thresNorm = class 1 (genuine)
-                if (scores[key][0] >= thresNorm && gtLabel == 0)
-                    continue;
-                else if (scores[key][0] < thresNorm && gtLabel == 1)
-                    continue;
-                else if (scores[key][0] >= thresNorm && gtLabel == 1)
-                    FR +=1;
-                else if (scores[key][0] < thresNorm && gtLabel == 0)
-                    FA +=1;
-            }
-            float FAR = FA / fabs(numTemplates - classOneTemplateCount);
-            float FRR = FR / float(classOneTemplateCount);
-
-            float diff = std::abs(FAR-FRR);
-            if (diff < minDiff){
-                minDiff = diff;
-                EER = (FAR+FRR)/2.0;
-                EERThres = thresNorm;
-            }
-         thres += stepSize;
-     }
+    for (int i=0; i<templateList.size(); i++) {
+        if (!templateList[i].file.contains(distribution_property) || !templateList[i].file.contains(gt_property))
+            continue;
+        QString templateKey = templateList[i].file.path() + templateList[i].file.baseName();
+        const int gtLabel = templateList[i].file.get<int>(gt_property);
+        if (gtLabel == 1)
+            classOneTemplateCount++;
+        const QList<float> templateScores = templateList[i].file.getList<float>(distribution_property);
+        gtLabels[templateKey] = gtLabel;
+        scores[templateKey] = templateScores;
+    }
-     qDebug() <<"Class 0 Templates:" << fabs(numTemplates - classOneTemplateCount) << "Class 1 Templates:"
-             << classOneTemplateCount << "Total Templates:" << numTemplates;
-     qDebug("EER: %.3f @ Threshold %.3f", EER*100, EERThres);
-}
+    const int numPoints  = 200;
+    const float stepSize = 100.0/numPoints;
+    const int numTemplates = scores.size();
+    float thres     = 0.0; //Between [0,100]
+    float thresNorm = 0.0; //Between [0,1]
+    float minDiff = 100, EER = 100, EERThres = 0;
+    QList<OperatingPoint> operatingPoints;
+
+    for(int i = 0; i <= numPoints; i++) {
+        int FA = 0, FR = 0;
+        thresNorm = thres/100.0;
+        foreach(const QString &key, scores.keys()) {
+            int gtLabel = gtLabels[key];
+            //> thresNorm = class 0 (spoof) : < thresNorm = class 1 (genuine)
+            if (scores[key][0] >= thresNorm && gtLabel == 0)
+                continue;
+            else if (scores[key][0] < thresNorm && gtLabel == 1)
+                continue;
+            else if (scores[key][0] >= thresNorm && gtLabel == 1)
+                FR +=1;
+            else if (scores[key][0] < thresNorm && gtLabel == 0)
+                FA +=1;
+        }
+        const float FAR = FA / float(numTemplates - classOneTemplateCount);
+        const float FRR = FR / float(classOneTemplateCount);
+        operatingPoints.append(OperatingPoint(thresNorm, FAR, 1-FRR));
+
+        const float diff = std::abs(FAR-FRR);
+        if (diff < minDiff) {
+            minDiff = diff;
+            EER = (FAR+FRR)/2.0;
+            EERThres = thresNorm;
+        }
+        thres += stepSize;
+    }
+
+    printf("Class 0 Templates: %d\tClass 1 Templates: %d\tTotal Templates: %d\n",
+           numTemplates-classOneTemplateCount, classOneTemplateCount, numTemplates);
+    foreach (float FAR, QList<float>() << 0.1 << 0.01 << 0.001 << 0.0001) {
+        const OperatingPoint op = getOperatingPointGivenFAR(operatingPoints, FAR);
+        printf("TAR & Score @ FAR = %.0e: %.3f %.3f\n", FAR, op.TAR, op.score);
+    }
+    printf("EER: %.3f @ Threshold %.3f\n", EER*100, EERThres);
+
+    // Optionally write ROC curve
+    if (!pdf.isEmpty()) {
+        QStringList farValues, tarValues;
+        float expFAR  = std::max(ceil(log10(numTemplates - classOneTemplateCount)), 1.0);
+        float FARstep = expFAR / (float)(Max_Points - 1);
+        for (int i=0; i<Max_Points; i++) {
+            float FAR = pow(10, -expFAR + i*FARstep);
+            OperatingPoint op = getOperatingPointGivenFAR(operatingPoints, FAR);
+            farValues.append(QString::number(FAR));
+            tarValues.append(QString::number(op.TAR));
+        }
+        QStringList rSource;
+        rSource << "# Load libraries" << "library(ggplot2)" << "" << "# Set Data"
+                << "FAR <- c(" + farValues.join(",") + ")"
+                << "TAR <- c(" + tarValues.join(",") + ")"
+                << "data <- data.frame(FAR, TAR)"
+                << "" << "# Construct Plot" << "pdf(\"" + pdf + "\")"
+                << "print(qplot(FAR, TAR, data=data, geom=\"line\") + scale_x_log10() + theme_minimal())"
+                << "dev.off()";
+
+        QString rFile = "EvalEER.R";
+        QtUtils::writeFile(rFile, rSource);
+        QtUtils::runRScript(rFile);
+    }
+}
 } // namespace br
@@ -34,7 +34,7 @@ namespace br
     float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", int normalizationIndexA = 0, int normalizationIndexB = 1, int sampleIndex = 0, int totalExamples = 5); // Return average error
     void EvalRegression(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = "");
     void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &csv = "");
-    void EvalEER(const QString &predictedXML, const QString gt_property = "", const QString distribution_property = "");
+    void EvalEER(const QString &predictedXML, const QString gt_property = "", const QString distribution_property = "", const QString &pdf = "");
     struct Candidate
     {
         size_t index;
@@ -146,9 +146,9 @@ void br_eval_knn(const char *knnGraph, const char *knnTruth, const char *csv)
     EvalKNN(knnGraph, knnTruth, csv);
 }
-void br_eval_eer(const char *predicted_xml, const char *gt_property, const char *distribution_property )
+void br_eval_eer(const char *predicted_xml, const char *gt_property, const char *distribution_property, const char *pdf)
 {
-    EvalEER(predicted_xml, gt_property, distribution_property);
+    EvalEER(predicted_xml, gt_property, distribution_property, pdf);
 }
 void br_finalize()
@@ -66,7 +66,7 @@ BR_EXPORT void br_eval_regression(const char *predicted_gallery, const char *tru
 BR_EXPORT void br_eval_knn(const char *knnGraph, const char *knnTruth, const char *csv = "");
-BR_EXPORT void br_eval_eer(const char *predicted_xml, const char *gt_property = "", const char *distribution_property = "");
+BR_EXPORT void br_eval_eer(const char *predicted_xml, const char *gt_property = "", const char *distribution_property = "", const char *pdf = "");
 BR_EXPORT void br_finalize();