Peter M. Groen / openbr

Commit cbdb48ca080b09bf958b86aec1f29409b24efd37

Authored by Stephen Rawls
1 parent b5b7605d

Adding a new table to the OpenBR report show FAR values at various fixed TAR values

Inline Side-by-side
Showing 2 changed files with 66 additions and 17 deletions
openbr/core/eval.cpp
  View file @cbdb48c
@@ -53,7 +53,7 @@ struct OperatingPoint @@ -53,7 +53,7 @@ struct OperatingPoint
53 : score(_score), FAR(_FAR), TAR(_TAR) {} 53 : score(_score), FAR(_FAR), TAR(_TAR) {}
54 }; 54 };
55 55
56 -static OperatingPoint getOperatingPoint(const QList<OperatingPoint> &operatingPoints, float FAR) 56 +static OperatingPoint getOperatingPointGivenFAR(const QList<OperatingPoint> &operatingPoints, float FAR)
57 { 57 {
58 int index = 0; 58 int index = 0;
59 while (operatingPoints[index].FAR < FAR) { 59 while (operatingPoints[index].FAR < FAR) {
@@ -75,6 +75,32 @@ static OperatingPoint getOperatingPoint(const QList&lt;OperatingPoint&gt; &amp;operatingPo @@ -75,6 +75,32 @@ static OperatingPoint getOperatingPoint(const QList&lt;OperatingPoint&gt; &amp;operatingPo
75 return OperatingPoint(mScore * FAR + bScore,FAR, mTAR * FAR + bTAR); 75 return OperatingPoint(mScore * FAR + bScore,FAR, mTAR * FAR + bTAR);
76 } 76 }
77 77
  78 +static OperatingPoint getOperatingPointGivenTAR(const QList<OperatingPoint> &operatingPoints, float TAR)
  79 +{
  80 + int index = 0;
  81 + while (operatingPoints[index].TAR < TAR) {
  82 + index++;
  83 + if (index == operatingPoints.size())
  84 + return OperatingPoint(operatingPoints.last().score, operatingPoints.last().FAR, TAR);
  85 + }
  86 +
  87 +
  88 + const float FAR1 = (index == 0 ? 0 : operatingPoints[index-1].FAR);
  89 + const float TAR1 = (index == 0 ? 0 : operatingPoints[index-1].TAR);
  90 + const float score1 = (index == 0 ? operatingPoints[index].score : operatingPoints[index-1].score);
  91 + const float FAR2 = operatingPoints[index].FAR;
  92 + const float TAR2 = operatingPoints[index].TAR;
  93 + const float score2 = operatingPoints[index].score;
  94 + const float mTAR = (TAR2 - TAR1) / (FAR2 - FAR1);
  95 + const float bTAR = TAR1 - mTAR*FAR1;
  96 + const float mScore = (score2 - score1) / (FAR2 - FAR1);
  97 + const float bScore = score1 - mScore*FAR1;
  98 +
  99 + const float FAR = (TAR - bTAR) / mTAR;
  100 + return OperatingPoint(mScore * FAR + bScore,FAR, TAR);
  101 +}
  102 +
  103 +
78 static float getCMC(const QVector<int> &firstGenuineReturns, int rank) 104 static float getCMC(const QVector<int> &firstGenuineReturns, int rank)
79 { 105 {
80 int realizedReturns = 0, possibleReturns = 0; 106 int realizedReturns = 0, possibleReturns = 0;
@@ -231,6 +257,7 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv, const QSt @@ -231,6 +257,7 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv, const QSt
231 if ((falsePositives > previousFalsePositives) && 257 if ((falsePositives > previousFalsePositives) &&
232 (truePositives > previousTruePositives)) { 258 (truePositives > previousTruePositives)) {
233 operatingPoints.append(OperatingPoint(thresh, float(falsePositives)/impostorCount, float(truePositives)/genuineCount)); 259 operatingPoints.append(OperatingPoint(thresh, float(falsePositives)/impostorCount, float(truePositives)/genuineCount));
  260 +
234 if (EERIndex == 0) { 261 if (EERIndex == 0) {
235 if (floor(float(falsePositives)/impostorCount*100+0.5)/100 == floor((1-float(truePositives)/genuineCount)*100+0.5)/100) EERIndex = index-1; 262 if (floor(float(falsePositives)/impostorCount*100+0.5)/100 == floor((1-float(truePositives)/genuineCount)*100+0.5)/100) EERIndex = index-1;
236 } 263 }
@@ -277,7 +304,7 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv, const QSt @@ -277,7 +304,7 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv, const QSt
277 // Write Detection Error Tradeoff (DET), PRE, REC 304 // Write Detection Error Tradeoff (DET), PRE, REC
278 float FAR=0.000001; 305 float FAR=0.000001;
279 for (int i=0; i<Max_Points; i++) { 306 for (int i=0; i<Max_Points; i++) {
280 - OperatingPoint operatingPoint = getOperatingPoint(operatingPoints, FAR); 307 + OperatingPoint operatingPoint = getOperatingPointGivenFAR(operatingPoints, FAR);
281 lines.append(QString("DET,%1,%2").arg(QString::number(FAR), 308 lines.append(QString("DET,%1,%2").arg(QString::number(FAR),
282 QString::number(1-operatingPoint.TAR))); 309 QString::number(1-operatingPoint.TAR)));
283 lines.append(QString("FAR,%1,%2").arg(QString::number(operatingPoint.score), 310 lines.append(QString("FAR,%1,%2").arg(QString::number(operatingPoint.score),
@@ -288,13 +315,21 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv, const QSt @@ -288,13 +315,21 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv, const QSt
288 FAR *=1.02807; 315 FAR *=1.02807;
289 } 316 }
290 317
291 - // Write FAR/TAR Table (FT)  
292 - lines.append(qPrintable(QString("FT,0.000001,%1").arg(QString::number(getOperatingPoint(operatingPoints, 0.000001).TAR, 'f', 3))));  
293 - lines.append(qPrintable(QString("FT,0.00001,%1").arg(QString::number(getOperatingPoint(operatingPoints, 0.00001).TAR, 'f', 3))));  
294 - lines.append(qPrintable(QString("FT,0.0001,%1").arg(QString::number(getOperatingPoint(operatingPoints, 0.0001).TAR, 'f', 3))));  
295 - lines.append(qPrintable(QString("FT,0.001,%1").arg(QString::number(getOperatingPoint(operatingPoints, 0.001).TAR, 'f', 3))));  
296 - lines.append(qPrintable(QString("FT,0.01,%1").arg(QString::number(getOperatingPoint(operatingPoints, 0.01).TAR, 'f', 3))));  
297 - lines.append(qPrintable(QString("FT,0.1,%1").arg(QString::number(getOperatingPoint(operatingPoints, 0.1).TAR, 'f', 3)))); 318 + // Write TAR@FAR Table (FT)
  319 + lines.append(qPrintable(QString("FT,0.000001,%1").arg(QString::number(getOperatingPointGivenFAR(operatingPoints, 0.000001).TAR, 'f', 3))));
  320 + lines.append(qPrintable(QString("FT,0.00001,%1").arg(QString::number(getOperatingPointGivenFAR(operatingPoints, 0.00001).TAR, 'f', 3))));
  321 + lines.append(qPrintable(QString("FT,0.0001,%1").arg(QString::number(getOperatingPointGivenFAR(operatingPoints, 0.0001).TAR, 'f', 3))));
  322 + lines.append(qPrintable(QString("FT,0.001,%1").arg(QString::number(getOperatingPointGivenFAR(operatingPoints, 0.001).TAR, 'f', 3))));
  323 + lines.append(qPrintable(QString("FT,0.01,%1").arg(QString::number(getOperatingPointGivenFAR(operatingPoints, 0.01).TAR, 'f', 3))));
  324 + lines.append(qPrintable(QString("FT,0.1,%1").arg(QString::number(getOperatingPointGivenFAR(operatingPoints, 0.1).TAR, 'f', 3))));
  325 +
  326 + // Write FAR@TAR Table (FatT)
  327 + lines.append(qPrintable(QString("FatT,0.95,%1").arg(QString::number(getOperatingPointGivenTAR(operatingPoints, 0.95).FAR, 'f', 3))));
  328 + lines.append(qPrintable(QString("FatT,0.85,%1").arg(QString::number(getOperatingPointGivenTAR(operatingPoints, 0.85).FAR, 'f', 3))));
  329 + lines.append(qPrintable(QString("FatT,0.75,%1").arg(QString::number(getOperatingPointGivenTAR(operatingPoints, 0.75).FAR, 'f', 3))));
  330 + lines.append(qPrintable(QString("FatT,0.65,%1").arg(QString::number(getOperatingPointGivenTAR(operatingPoints, 0.65).FAR, 'f', 3))));
  331 + lines.append(qPrintable(QString("FatT,0.50,%1").arg(QString::number(getOperatingPointGivenTAR(operatingPoints, 0.50).FAR, 'f', 3))));
  332 + lines.append(qPrintable(QString("FatT,0.40,%1").arg(QString::number(getOperatingPointGivenTAR(operatingPoints, 0.40).FAR, 'f', 3))));
298 333
299 //Write CMC Table (CT) 334 //Write CMC Table (CT)
300 lines.append(qPrintable(QString("CT,1,%1").arg(QString::number(getCMC(firstGenuineReturns, 1), 'f', 3)))); 335 lines.append(qPrintable(QString("CT,1,%1").arg(QString::number(getCMC(firstGenuineReturns, 1), 'f', 3))));
@@ -305,8 +340,8 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv, const QSt @@ -305,8 +340,8 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv, const QSt
305 lines.append(qPrintable(QString("CT,100,%1").arg(QString::number(getCMC(firstGenuineReturns, 100), 'f', 3)))); 340 lines.append(qPrintable(QString("CT,100,%1").arg(QString::number(getCMC(firstGenuineReturns, 100), 'f', 3))));
306 341
307 // Write FAR/TAR Bar Chart (BC) 342 // Write FAR/TAR Bar Chart (BC)
308 - lines.append(qPrintable(QString("BC,0.001,%1").arg(QString::number(getOperatingPoint(operatingPoints, 0.001).TAR, 'f', 3))));  
309 - lines.append(qPrintable(QString("BC,0.01,%1").arg(QString::number(result = getOperatingPoint(operatingPoints, 0.01).TAR, 'f', 3)))); 343 + lines.append(qPrintable(QString("BC,0.001,%1").arg(QString::number(getOperatingPointGivenFAR(operatingPoints, 0.001).TAR, 'f', 3))));
  344 + lines.append(qPrintable(QString("BC,0.01,%1").arg(QString::number(result = getOperatingPointGivenFAR(operatingPoints, 0.01).TAR, 'f', 3))));
310 345
311 // Attempt to read template size from enrolled gallery and write to output CSV 346 // Attempt to read template size from enrolled gallery and write to output CSV
312 size_t maxSize(0); 347 size_t maxSize(0);
@@ -343,10 +378,10 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv, const QSt @@ -343,10 +378,10 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv, const QSt
343 378
344 QtUtils::writeFile(csv, lines); 379 QtUtils::writeFile(csv, lines);
345 if (maxSize > 0) qDebug("Template Size: %i bytes", (int)maxSize); 380 if (maxSize > 0) qDebug("Template Size: %i bytes", (int)maxSize);
346 - qDebug("TAR @ FAR = 0.01: %.3f",getOperatingPoint(operatingPoints, 0.01).TAR);  
347 - qDebug("TAR @ FAR = 0.001: %.3f",getOperatingPoint(operatingPoints, 0.001).TAR);  
348 - qDebug("TAR @ FAR = 0.0001: %.3f",getOperatingPoint(operatingPoints, 0.0001).TAR);  
349 - qDebug("TAR @ FAR = 0.00001: %.3f",getOperatingPoint(operatingPoints, 0.00001).TAR); 381 + qDebug("TAR @ FAR = 0.01: %.3f",getOperatingPointGivenFAR(operatingPoints, 0.01).TAR);
  382 + qDebug("TAR @ FAR = 0.001: %.3f",getOperatingPointGivenFAR(operatingPoints, 0.001).TAR);
  383 + qDebug("TAR @ FAR = 0.0001: %.3f",getOperatingPointGivenFAR(operatingPoints, 0.0001).TAR);
  384 + qDebug("TAR @ FAR = 0.00001: %.3f",getOperatingPointGivenFAR(operatingPoints, 0.00001).TAR);
350 385
351 qDebug("\nRetrieval Rate @ Rank = %d: %.3f", Report_Retrieval, getCMC(firstGenuineReturns, Report_Retrieval)); 386 qDebug("\nRetrieval Rate @ Rank = %d: %.3f", Report_Retrieval, getCMC(firstGenuineReturns, Report_Retrieval));
352 387
@@ -571,8 +606,8 @@ float InplaceEval(const QString &amp;simmat, const QString &amp;target, const QString &amp;q @@ -571,8 +606,8 @@ float InplaceEval(const QString &amp;simmat, const QString &amp;target, const QString &amp;q
571 606
572 float result; 607 float result;
573 // Write FAR/TAR Bar Chart (BC) 608 // Write FAR/TAR Bar Chart (BC)
574 - lines.append(qPrintable(QString("BC,0.001,%1").arg(QString::number(getOperatingPoint(operatingPoints, 0.001).TAR, 'f', 3))));  
575 - lines.append(qPrintable(QString("BC,0.01,%1").arg(QString::number(result = getOperatingPoint(operatingPoints, 0.01).TAR, 'f', 3)))); 609 + lines.append(qPrintable(QString("BC,0.001,%1").arg(QString::number(getOperatingPointGivenFAR(operatingPoints, 0.001).TAR, 'f', 3))));
  610 + lines.append(qPrintable(QString("BC,0.01,%1").arg(QString::number(result = getOperatingPointGivenFAR(operatingPoints, 0.01).TAR, 'f', 3))));
576 611
577 qDebug("TAR @ FAR = 0.01: %.3f", result); 612 qDebug("TAR @ FAR = 0.01: %.3f", result);
578 QtUtils::writeFile(csv, lines); 613 QtUtils::writeFile(csv, lines);
openbr/core/plot.cpp
  View file @cbdb48c
@@ -161,6 +161,7 @@ struct RPlot @@ -161,6 +161,7 @@ struct RPlot
161 "FRR <- data[grep(\"FRR\",data$Plot),-c(1)]\n" 161 "FRR <- data[grep(\"FRR\",data$Plot),-c(1)]\n"
162 "SD <- data[grep(\"SD\",data$Plot),-c(1)]\n" 162 "SD <- data[grep(\"SD\",data$Plot),-c(1)]\n"
163 "FT <- data[grep(\"FT\",data$Plot),-c(1)]\n" 163 "FT <- data[grep(\"FT\",data$Plot),-c(1)]\n"
  164 + "FatT <- data[grep(\"FatT\",data$Plot),-c(1)]\n"
164 "CT <- data[grep(\"CT\",data$Plot),-c(1)]\n" 165 "CT <- data[grep(\"CT\",data$Plot),-c(1)]\n"
165 "BC <- data[grep(\"BC\",data$Plot),-c(1)]\n" 166 "BC <- data[grep(\"BC\",data$Plot),-c(1)]\n"
166 "TS <- data[grep(\"TS\",data$Plot),-c(1)]\n" 167 "TS <- data[grep(\"TS\",data$Plot),-c(1)]\n"
@@ -178,6 +179,7 @@ struct RPlot @@ -178,6 +179,7 @@ struct RPlot
178 "ERR$Y <- as.numeric(as.character(ERR$Y))\n" 179 "ERR$Y <- as.numeric(as.character(ERR$Y))\n"
179 "SD$Y <- as.factor(unique(as.character(SD$Y)))\n" 180 "SD$Y <- as.factor(unique(as.character(SD$Y)))\n"
180 "FT$Y <- as.numeric(as.character(FT$Y))\n" 181 "FT$Y <- as.numeric(as.character(FT$Y))\n"
  182 + "FatT$Y <- as.numeric(as.character(FatT$Y))\n"
181 "CT$Y <- as.numeric(as.character(CT$Y))\n" 183 "CT$Y <- as.numeric(as.character(CT$Y))\n"
182 "BC$Y <- as.numeric(as.character(BC$Y))\n" 184 "BC$Y <- as.numeric(as.character(BC$Y))\n"
183 "TS$Y <- as.character(TS$Y)\n" 185 "TS$Y <- as.character(TS$Y)\n"
@@ -205,6 +207,16 @@ struct RPlot @@ -205,6 +207,16 @@ struct RPlot
205 "rownames(mat) <- c(\"FAR = 1e-06\", \"FAR = 1e-05\", \"FAR = 1e-04\", \"FAR = 1e-03\", \"FAR = 1e-02\", \"FAR = 1e-01\")\n" 207 "rownames(mat) <- c(\"FAR = 1e-06\", \"FAR = 1e-05\", \"FAR = 1e-04\", \"FAR = 1e-03\", \"FAR = 1e-02\", \"FAR = 1e-01\")\n"
206 "FTtable <- as.table(mat)\n" 208 "FTtable <- as.table(mat)\n"
207 "\n" 209 "\n"
  210 + "# Code to format TAR@FAR table\n"
  211 + "algs <- unique(FT$%2)\n"
  212 + "algs <- algs[!duplicated(algs)]\n"
  213 + "mat <- matrix(FatT$Y,nrow=6,ncol=length(algs),byrow=FALSE)\n"
  214 + "colnames(mat) <- algs \n"
  215 + "rownames(mat) <- c(\"TAR = 0.95\", \"TAR = 0.85\", \"TAR = 0.75\", \"TAR = 0.65\", \"TAR = 0.50\", \"TAR = 0.40\")\n"
  216 + "F_at_Ttable <- as.table(mat)\n"
  217 + "\n"
  218 +
  219 + "\n"
208 "# Code to format CMC Table\n" 220 "# Code to format CMC Table\n"
209 "mat <- matrix(%4,nrow=6,ncol=length(algs),byrow=FALSE)\n" 221 "mat <- matrix(%4,nrow=6,ncol=length(algs),byrow=FALSE)\n"
210 "colnames(mat) <- algs \n" 222 "colnames(mat) <- algs \n"
@@ -249,6 +261,8 @@ struct RPlot @@ -249,6 +261,8 @@ struct RPlot
249 "plot.new()\n" 261 "plot.new()\n"
250 "print(textplot(FTtable))\n" 262 "print(textplot(FTtable))\n"
251 "print(title(\"Table of True Accept Rates at various False Accept Rates\"))\n" 263 "print(title(\"Table of True Accept Rates at various False Accept Rates\"))\n"
  264 + "print(textplot(F_at_Ttable))\n"
  265 + "print(title(\"Table of False Accept Rates at various True Accept Rates\"))\n"
252 "print(textplot(CMCtable))\n" 266 "print(textplot(CMCtable))\n"
253 "print(title(\"Table of retrieval rate at various ranks\"))\n" 267 "print(title(\"Table of retrieval rate at various ranks\"))\n"
254 "if (nrow(TS) != 0) {\n\t" 268 "if (nrow(TS) != 0) {\n\t"