Peter M. Groen / openbr

Commit 9fdb4cedc875e0817c4979b5548440224e91a16b

Authored by Josh Klontz
2 parents f878e13c 788dee57

Merge pull request #313 from stephenrawls/fb-new-far-at-tar-table 

Adding a new table to the OpenBR report
Inline Side-by-side
Showing 2 changed files with 60 additions and 17 deletions
openbr/core/eval.cpp
  View file @9fdb4ce
@@ -53,7 +53,7 @@ struct OperatingPoint @@ -53,7 +53,7 @@ struct OperatingPoint
53 : score(_score), FAR(_FAR), TAR(_TAR) {} 53 : score(_score), FAR(_FAR), TAR(_TAR) {}
54 }; 54 };
55 55
56 -static OperatingPoint getOperatingPoint(const QList<OperatingPoint> &operatingPoints, float FAR) 56 +static OperatingPoint getOperatingPointGivenFAR(const QList<OperatingPoint> &operatingPoints, float FAR)
57 { 57 {
58 int index = 0; 58 int index = 0;
59 while (operatingPoints[index].FAR < FAR) { 59 while (operatingPoints[index].FAR < FAR) {
@@ -75,6 +75,32 @@ static OperatingPoint getOperatingPoint(const QList&lt;OperatingPoint&gt; &amp;operatingPo @@ -75,6 +75,32 @@ static OperatingPoint getOperatingPoint(const QList&lt;OperatingPoint&gt; &amp;operatingPo
75 return OperatingPoint(mScore * FAR + bScore,FAR, mTAR * FAR + bTAR); 75 return OperatingPoint(mScore * FAR + bScore,FAR, mTAR * FAR + bTAR);
76 } 76 }
77 77
  78 +static OperatingPoint getOperatingPointGivenTAR(const QList<OperatingPoint> &operatingPoints, float TAR)
  79 +{
  80 + int index = 0;
  81 + while (operatingPoints[index].TAR < TAR) {
  82 + index++;
  83 + if (index == operatingPoints.size())
  84 + return OperatingPoint(operatingPoints.last().score, operatingPoints.last().FAR, TAR);
  85 + }
  86 +
  87 +
  88 + const float FAR1 = (index == 0 ? 0 : operatingPoints[index-1].FAR);
  89 + const float TAR1 = (index == 0 ? 0 : operatingPoints[index-1].TAR);
  90 + const float score1 = (index == 0 ? operatingPoints[index].score : operatingPoints[index-1].score);
  91 + const float FAR2 = operatingPoints[index].FAR;
  92 + const float TAR2 = operatingPoints[index].TAR;
  93 + const float score2 = operatingPoints[index].score;
  94 + const float mTAR = (TAR2 - TAR1) / (FAR2 - FAR1);
  95 + const float bTAR = TAR1 - mTAR*FAR1;
  96 + const float mScore = (score2 - score1) / (FAR2 - FAR1);
  97 + const float bScore = score1 - mScore*FAR1;
  98 +
  99 + const float FAR = (TAR - bTAR) / mTAR;
  100 + return OperatingPoint(mScore * FAR + bScore,FAR, TAR);
  101 +}
  102 +
  103 +
78 static float getCMC(const QVector<int> &firstGenuineReturns, int rank) 104 static float getCMC(const QVector<int> &firstGenuineReturns, int rank)
79 { 105 {
80 int realizedReturns = 0, possibleReturns = 0; 106 int realizedReturns = 0, possibleReturns = 0;
@@ -231,6 +257,7 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv, const QSt @@ -231,6 +257,7 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv, const QSt
231 if ((falsePositives > previousFalsePositives) && 257 if ((falsePositives > previousFalsePositives) &&
232 (truePositives > previousTruePositives)) { 258 (truePositives > previousTruePositives)) {
233 operatingPoints.append(OperatingPoint(thresh, float(falsePositives)/impostorCount, float(truePositives)/genuineCount)); 259 operatingPoints.append(OperatingPoint(thresh, float(falsePositives)/impostorCount, float(truePositives)/genuineCount));
  260 +
234 if (EERIndex == 0) { 261 if (EERIndex == 0) {
235 if (floor(float(falsePositives)/impostorCount*100+0.5)/100 == floor((1-float(truePositives)/genuineCount)*100+0.5)/100) EERIndex = index-1; 262 if (floor(float(falsePositives)/impostorCount*100+0.5)/100 == floor((1-float(truePositives)/genuineCount)*100+0.5)/100) EERIndex = index-1;
236 } 263 }
@@ -277,7 +304,7 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv, const QSt @@ -277,7 +304,7 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv, const QSt
277 // Write Detection Error Tradeoff (DET), PRE, REC 304 // Write Detection Error Tradeoff (DET), PRE, REC
278 float FAR=0.000001; 305 float FAR=0.000001;
279 for (int i=0; i<Max_Points; i++) { 306 for (int i=0; i<Max_Points; i++) {
280 - OperatingPoint operatingPoint = getOperatingPoint(operatingPoints, FAR); 307 + OperatingPoint operatingPoint = getOperatingPointGivenFAR(operatingPoints, FAR);
281 lines.append(QString("DET,%1,%2").arg(QString::number(FAR), 308 lines.append(QString("DET,%1,%2").arg(QString::number(FAR),
282 QString::number(1-operatingPoint.TAR))); 309 QString::number(1-operatingPoint.TAR)));
283 lines.append(QString("FAR,%1,%2").arg(QString::number(operatingPoint.score), 310 lines.append(QString("FAR,%1,%2").arg(QString::number(operatingPoint.score),
@@ -288,13 +315,17 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv, const QSt @@ -288,13 +315,17 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv, const QSt
288 FAR *=1.02807; 315 FAR *=1.02807;
289 } 316 }
290 317
291 - // Write FAR/TAR Table (FT)  
292 - lines.append(qPrintable(QString("FT,0.000001,%1").arg(QString::number(getOperatingPoint(operatingPoints, 0.000001).TAR, 'f', 3))));  
293 - lines.append(qPrintable(QString("FT,0.00001,%1").arg(QString::number(getOperatingPoint(operatingPoints, 0.00001).TAR, 'f', 3))));  
294 - lines.append(qPrintable(QString("FT,0.0001,%1").arg(QString::number(getOperatingPoint(operatingPoints, 0.0001).TAR, 'f', 3))));  
295 - lines.append(qPrintable(QString("FT,0.001,%1").arg(QString::number(getOperatingPoint(operatingPoints, 0.001).TAR, 'f', 3))));  
296 - lines.append(qPrintable(QString("FT,0.01,%1").arg(QString::number(getOperatingPoint(operatingPoints, 0.01).TAR, 'f', 3))));  
297 - lines.append(qPrintable(QString("FT,0.1,%1").arg(QString::number(getOperatingPoint(operatingPoints, 0.1).TAR, 'f', 3)))); 318 + // Write TAR@FAR Table (FT)
  319 + foreach (float far, QList<float>() << 1e-6 << 1e-5 << 1e-4 << 1e-3 << 1e-2 << 1e-1)
  320 + lines.append(qPrintable(QString("FT,%1,%2").arg(
  321 + QString::number(far, 'f'),
  322 + QString::number(getOperatingPointGivenFAR(operatingPoints, far).TAR, 'f', 3))));
  323 +
  324 + // Write FAR@TAR Table (FatT)
  325 + foreach (float tar, QList<float>() << 0.95 << 0.85 << 0.75 << 0.65 << 0.5 << 0.4)
  326 + lines.append(qPrintable(QString("FatT,%1,%2").arg(
  327 + QString::number(tar, 'f', 2),
  328 + QString::number(getOperatingPointGivenTAR(operatingPoints, tar).FAR, 'f', 3))));
298 329
299 //Write CMC Table (CT) 330 //Write CMC Table (CT)
300 lines.append(qPrintable(QString("CT,1,%1").arg(QString::number(getCMC(firstGenuineReturns, 1), 'f', 3)))); 331 lines.append(qPrintable(QString("CT,1,%1").arg(QString::number(getCMC(firstGenuineReturns, 1), 'f', 3))));
@@ -305,8 +336,8 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv, const QSt @@ -305,8 +336,8 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv, const QSt
305 lines.append(qPrintable(QString("CT,100,%1").arg(QString::number(getCMC(firstGenuineReturns, 100), 'f', 3)))); 336 lines.append(qPrintable(QString("CT,100,%1").arg(QString::number(getCMC(firstGenuineReturns, 100), 'f', 3))));
306 337
307 // Write FAR/TAR Bar Chart (BC) 338 // Write FAR/TAR Bar Chart (BC)
308 - lines.append(qPrintable(QString("BC,0.001,%1").arg(QString::number(getOperatingPoint(operatingPoints, 0.001).TAR, 'f', 3))));  
309 - lines.append(qPrintable(QString("BC,0.01,%1").arg(QString::number(result = getOperatingPoint(operatingPoints, 0.01).TAR, 'f', 3)))); 339 + lines.append(qPrintable(QString("BC,0.001,%1").arg(QString::number(getOperatingPointGivenFAR(operatingPoints, 0.001).TAR, 'f', 3))));
  340 + lines.append(qPrintable(QString("BC,0.01,%1").arg(QString::number(result = getOperatingPointGivenFAR(operatingPoints, 0.01).TAR, 'f', 3))));
310 341
311 // Attempt to read template size from enrolled gallery and write to output CSV 342 // Attempt to read template size from enrolled gallery and write to output CSV
312 size_t maxSize(0); 343 size_t maxSize(0);
@@ -343,10 +374,10 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv, const QSt @@ -343,10 +374,10 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv, const QSt
343 374
344 QtUtils::writeFile(csv, lines); 375 QtUtils::writeFile(csv, lines);
345 if (maxSize > 0) qDebug("Template Size: %i bytes", (int)maxSize); 376 if (maxSize > 0) qDebug("Template Size: %i bytes", (int)maxSize);
346 - qDebug("TAR @ FAR = 0.01: %.3f",getOperatingPoint(operatingPoints, 0.01).TAR);  
347 - qDebug("TAR @ FAR = 0.001: %.3f",getOperatingPoint(operatingPoints, 0.001).TAR);  
348 - qDebug("TAR @ FAR = 0.0001: %.3f",getOperatingPoint(operatingPoints, 0.0001).TAR);  
349 - qDebug("TAR @ FAR = 0.00001: %.3f",getOperatingPoint(operatingPoints, 0.00001).TAR); 377 + qDebug("TAR @ FAR = 0.01: %.3f",getOperatingPointGivenFAR(operatingPoints, 0.01).TAR);
  378 + qDebug("TAR @ FAR = 0.001: %.3f",getOperatingPointGivenFAR(operatingPoints, 0.001).TAR);
  379 + qDebug("TAR @ FAR = 0.0001: %.3f",getOperatingPointGivenFAR(operatingPoints, 0.0001).TAR);
  380 + qDebug("TAR @ FAR = 0.00001: %.3f",getOperatingPointGivenFAR(operatingPoints, 0.00001).TAR);
350 381
351 qDebug("\nRetrieval Rate @ Rank = %d: %.3f", Report_Retrieval, getCMC(firstGenuineReturns, Report_Retrieval)); 382 qDebug("\nRetrieval Rate @ Rank = %d: %.3f", Report_Retrieval, getCMC(firstGenuineReturns, Report_Retrieval));
352 383
@@ -571,8 +602,8 @@ float InplaceEval(const QString &amp;simmat, const QString &amp;target, const QString &amp;q @@ -571,8 +602,8 @@ float InplaceEval(const QString &amp;simmat, const QString &amp;target, const QString &amp;q
571 602
572 float result; 603 float result;
573 // Write FAR/TAR Bar Chart (BC) 604 // Write FAR/TAR Bar Chart (BC)
574 - lines.append(qPrintable(QString("BC,0.001,%1").arg(QString::number(getOperatingPoint(operatingPoints, 0.001).TAR, 'f', 3))));  
575 - lines.append(qPrintable(QString("BC,0.01,%1").arg(QString::number(result = getOperatingPoint(operatingPoints, 0.01).TAR, 'f', 3)))); 605 + lines.append(qPrintable(QString("BC,0.001,%1").arg(QString::number(getOperatingPointGivenFAR(operatingPoints, 0.001).TAR, 'f', 3))));
  606 + lines.append(qPrintable(QString("BC,0.01,%1").arg(QString::number(result = getOperatingPointGivenFAR(operatingPoints, 0.01).TAR, 'f', 3))));
576 607
577 qDebug("TAR @ FAR = 0.01: %.3f", result); 608 qDebug("TAR @ FAR = 0.01: %.3f", result);
578 QtUtils::writeFile(csv, lines); 609 QtUtils::writeFile(csv, lines);
openbr/core/plot.cpp
  View file @9fdb4ce
@@ -161,6 +161,7 @@ struct RPlot @@ -161,6 +161,7 @@ struct RPlot
161 "FRR <- data[grep(\"FRR\",data$Plot),-c(1)]\n" 161 "FRR <- data[grep(\"FRR\",data$Plot),-c(1)]\n"
162 "SD <- data[grep(\"SD\",data$Plot),-c(1)]\n" 162 "SD <- data[grep(\"SD\",data$Plot),-c(1)]\n"
163 "FT <- data[grep(\"FT\",data$Plot),-c(1)]\n" 163 "FT <- data[grep(\"FT\",data$Plot),-c(1)]\n"
  164 + "FatT <- data[grep(\"FatT\",data$Plot),-c(1)]\n"
164 "CT <- data[grep(\"CT\",data$Plot),-c(1)]\n" 165 "CT <- data[grep(\"CT\",data$Plot),-c(1)]\n"
165 "BC <- data[grep(\"BC\",data$Plot),-c(1)]\n" 166 "BC <- data[grep(\"BC\",data$Plot),-c(1)]\n"
166 "TS <- data[grep(\"TS\",data$Plot),-c(1)]\n" 167 "TS <- data[grep(\"TS\",data$Plot),-c(1)]\n"
@@ -178,6 +179,7 @@ struct RPlot @@ -178,6 +179,7 @@ struct RPlot
178 "ERR$Y <- as.numeric(as.character(ERR$Y))\n" 179 "ERR$Y <- as.numeric(as.character(ERR$Y))\n"
179 "SD$Y <- as.factor(unique(as.character(SD$Y)))\n" 180 "SD$Y <- as.factor(unique(as.character(SD$Y)))\n"
180 "FT$Y <- as.numeric(as.character(FT$Y))\n" 181 "FT$Y <- as.numeric(as.character(FT$Y))\n"
  182 + "FatT$Y <- as.numeric(as.character(FatT$Y))\n"
181 "CT$Y <- as.numeric(as.character(CT$Y))\n" 183 "CT$Y <- as.numeric(as.character(CT$Y))\n"
182 "BC$Y <- as.numeric(as.character(BC$Y))\n" 184 "BC$Y <- as.numeric(as.character(BC$Y))\n"
183 "TS$Y <- as.character(TS$Y)\n" 185 "TS$Y <- as.character(TS$Y)\n"
@@ -205,6 +207,14 @@ struct RPlot @@ -205,6 +207,14 @@ struct RPlot
205 "rownames(mat) <- c(\"FAR = 1e-06\", \"FAR = 1e-05\", \"FAR = 1e-04\", \"FAR = 1e-03\", \"FAR = 1e-02\", \"FAR = 1e-01\")\n" 207 "rownames(mat) <- c(\"FAR = 1e-06\", \"FAR = 1e-05\", \"FAR = 1e-04\", \"FAR = 1e-03\", \"FAR = 1e-02\", \"FAR = 1e-01\")\n"
206 "FTtable <- as.table(mat)\n" 208 "FTtable <- as.table(mat)\n"
207 "\n" 209 "\n"
  210 + "# Code to format TAR@FAR table\n"
  211 + "algs <- unique(FT$%2)\n"
  212 + "algs <- algs[!duplicated(algs)]\n"
  213 + "mat <- matrix(FatT$Y,nrow=6,ncol=length(algs),byrow=FALSE)\n"
  214 + "colnames(mat) <- algs \n"
  215 + "rownames(mat) <- c(\"TAR = 0.95\", \"TAR = 0.85\", \"TAR = 0.75\", \"TAR = 0.65\", \"TAR = 0.50\", \"TAR = 0.40\")\n"
  216 + "F_at_Ttable <- as.table(mat)\n"
  217 + "\n"
208 "# Code to format CMC Table\n" 218 "# Code to format CMC Table\n"
209 "mat <- matrix(%4,nrow=6,ncol=length(algs),byrow=FALSE)\n" 219 "mat <- matrix(%4,nrow=6,ncol=length(algs),byrow=FALSE)\n"
210 "colnames(mat) <- algs \n" 220 "colnames(mat) <- algs \n"
@@ -249,6 +259,8 @@ struct RPlot @@ -249,6 +259,8 @@ struct RPlot
249 "plot.new()\n" 259 "plot.new()\n"
250 "print(textplot(FTtable))\n" 260 "print(textplot(FTtable))\n"
251 "print(title(\"Table of True Accept Rates at various False Accept Rates\"))\n" 261 "print(title(\"Table of True Accept Rates at various False Accept Rates\"))\n"
  262 + "print(textplot(F_at_Ttable))\n"
  263 + "print(title(\"Table of False Accept Rates at various True Accept Rates\"))\n"
252 "print(textplot(CMCtable))\n" 264 "print(textplot(CMCtable))\n"
253 "print(title(\"Table of retrieval rate at various ranks\"))\n" 265 "print(title(\"Table of retrieval rate at various ranks\"))\n"
254 "if (nrow(TS) != 0) {\n\t" 266 "if (nrow(TS) != 0) {\n\t"