Peter M. Groen / openbr

Commit cbdb48ca080b09bf958b86aec1f29409b24efd37

Authored by Stephen Rawls
1 parent b5b7605d

Adding a new table to the OpenBR report show FAR values at various fixed TAR values

Inline Side-by-side
Showing 2 changed files with 66 additions and 17 deletions
openbr/core/eval.cpp
  View file @cbdb48c
... ... @@ -53,7 +53,7 @@ struct OperatingPoint
53 53 : score(_score), FAR(_FAR), TAR(_TAR) {}
54 54 };
55 55  
56   -static OperatingPoint getOperatingPoint(const QList<OperatingPoint> &operatingPoints, float FAR)
  56 +static OperatingPoint getOperatingPointGivenFAR(const QList<OperatingPoint> &operatingPoints, float FAR)
57 57 {
58 58 int index = 0;
59 59 while (operatingPoints[index].FAR < FAR) {
... ... @@ -75,6 +75,32 @@ static OperatingPoint getOperatingPoint(const QList&lt;OperatingPoint&gt; &amp;operatingPo
75 75 return OperatingPoint(mScore * FAR + bScore,FAR, mTAR * FAR + bTAR);
76 76 }
77 77  
  78 +static OperatingPoint getOperatingPointGivenTAR(const QList<OperatingPoint> &operatingPoints, float TAR)
  79 +{
  80 + int index = 0;
  81 + while (operatingPoints[index].TAR < TAR) {
  82 + index++;
  83 + if (index == operatingPoints.size())
  84 + return OperatingPoint(operatingPoints.last().score, operatingPoints.last().FAR, TAR);
  85 + }
  86 +
  87 +
  88 + const float FAR1 = (index == 0 ? 0 : operatingPoints[index-1].FAR);
  89 + const float TAR1 = (index == 0 ? 0 : operatingPoints[index-1].TAR);
  90 + const float score1 = (index == 0 ? operatingPoints[index].score : operatingPoints[index-1].score);
  91 + const float FAR2 = operatingPoints[index].FAR;
  92 + const float TAR2 = operatingPoints[index].TAR;
  93 + const float score2 = operatingPoints[index].score;
  94 + const float mTAR = (TAR2 - TAR1) / (FAR2 - FAR1);
  95 + const float bTAR = TAR1 - mTAR*FAR1;
  96 + const float mScore = (score2 - score1) / (FAR2 - FAR1);
  97 + const float bScore = score1 - mScore*FAR1;
  98 +
  99 + const float FAR = (TAR - bTAR) / mTAR;
  100 + return OperatingPoint(mScore * FAR + bScore,FAR, TAR);
  101 +}
  102 +
  103 +
78 104 static float getCMC(const QVector<int> &firstGenuineReturns, int rank)
79 105 {
80 106 int realizedReturns = 0, possibleReturns = 0;
... ... @@ -231,6 +257,7 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv, const QSt
231 257 if ((falsePositives > previousFalsePositives) &&
232 258 (truePositives > previousTruePositives)) {
233 259 operatingPoints.append(OperatingPoint(thresh, float(falsePositives)/impostorCount, float(truePositives)/genuineCount));
  260 +
234 261 if (EERIndex == 0) {
235 262 if (floor(float(falsePositives)/impostorCount*100+0.5)/100 == floor((1-float(truePositives)/genuineCount)*100+0.5)/100) EERIndex = index-1;
236 263 }
... ... @@ -277,7 +304,7 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv, const QSt
277 304 // Write Detection Error Tradeoff (DET), PRE, REC
278 305 float FAR=0.000001;
279 306 for (int i=0; i<Max_Points; i++) {
280   - OperatingPoint operatingPoint = getOperatingPoint(operatingPoints, FAR);
  307 + OperatingPoint operatingPoint = getOperatingPointGivenFAR(operatingPoints, FAR);
281 308 lines.append(QString("DET,%1,%2").arg(QString::number(FAR),
282 309 QString::number(1-operatingPoint.TAR)));
283 310 lines.append(QString("FAR,%1,%2").arg(QString::number(operatingPoint.score),
... ... @@ -288,13 +315,21 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv, const QSt
288 315 FAR *=1.02807;
289 316 }
290 317  
291   - // Write FAR/TAR Table (FT)
292   - lines.append(qPrintable(QString("FT,0.000001,%1").arg(QString::number(getOperatingPoint(operatingPoints, 0.000001).TAR, 'f', 3))));
293   - lines.append(qPrintable(QString("FT,0.00001,%1").arg(QString::number(getOperatingPoint(operatingPoints, 0.00001).TAR, 'f', 3))));
294   - lines.append(qPrintable(QString("FT,0.0001,%1").arg(QString::number(getOperatingPoint(operatingPoints, 0.0001).TAR, 'f', 3))));
295   - lines.append(qPrintable(QString("FT,0.001,%1").arg(QString::number(getOperatingPoint(operatingPoints, 0.001).TAR, 'f', 3))));
296   - lines.append(qPrintable(QString("FT,0.01,%1").arg(QString::number(getOperatingPoint(operatingPoints, 0.01).TAR, 'f', 3))));
297   - lines.append(qPrintable(QString("FT,0.1,%1").arg(QString::number(getOperatingPoint(operatingPoints, 0.1).TAR, 'f', 3))));
  318 + // Write TAR@FAR Table (FT)
  319 + lines.append(qPrintable(QString("FT,0.000001,%1").arg(QString::number(getOperatingPointGivenFAR(operatingPoints, 0.000001).TAR, 'f', 3))));
  320 + lines.append(qPrintable(QString("FT,0.00001,%1").arg(QString::number(getOperatingPointGivenFAR(operatingPoints, 0.00001).TAR, 'f', 3))));
  321 + lines.append(qPrintable(QString("FT,0.0001,%1").arg(QString::number(getOperatingPointGivenFAR(operatingPoints, 0.0001).TAR, 'f', 3))));
  322 + lines.append(qPrintable(QString("FT,0.001,%1").arg(QString::number(getOperatingPointGivenFAR(operatingPoints, 0.001).TAR, 'f', 3))));
  323 + lines.append(qPrintable(QString("FT,0.01,%1").arg(QString::number(getOperatingPointGivenFAR(operatingPoints, 0.01).TAR, 'f', 3))));
  324 + lines.append(qPrintable(QString("FT,0.1,%1").arg(QString::number(getOperatingPointGivenFAR(operatingPoints, 0.1).TAR, 'f', 3))));
  325 +
  326 + // Write FAR@TAR Table (FatT)
  327 + lines.append(qPrintable(QString("FatT,0.95,%1").arg(QString::number(getOperatingPointGivenTAR(operatingPoints, 0.95).FAR, 'f', 3))));
  328 + lines.append(qPrintable(QString("FatT,0.85,%1").arg(QString::number(getOperatingPointGivenTAR(operatingPoints, 0.85).FAR, 'f', 3))));
  329 + lines.append(qPrintable(QString("FatT,0.75,%1").arg(QString::number(getOperatingPointGivenTAR(operatingPoints, 0.75).FAR, 'f', 3))));
  330 + lines.append(qPrintable(QString("FatT,0.65,%1").arg(QString::number(getOperatingPointGivenTAR(operatingPoints, 0.65).FAR, 'f', 3))));
  331 + lines.append(qPrintable(QString("FatT,0.50,%1").arg(QString::number(getOperatingPointGivenTAR(operatingPoints, 0.50).FAR, 'f', 3))));
  332 + lines.append(qPrintable(QString("FatT,0.40,%1").arg(QString::number(getOperatingPointGivenTAR(operatingPoints, 0.40).FAR, 'f', 3))));
298 333  
299 334 //Write CMC Table (CT)
300 335 lines.append(qPrintable(QString("CT,1,%1").arg(QString::number(getCMC(firstGenuineReturns, 1), 'f', 3))));
... ... @@ -305,8 +340,8 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv, const QSt
305 340 lines.append(qPrintable(QString("CT,100,%1").arg(QString::number(getCMC(firstGenuineReturns, 100), 'f', 3))));
306 341  
307 342 // Write FAR/TAR Bar Chart (BC)
308   - lines.append(qPrintable(QString("BC,0.001,%1").arg(QString::number(getOperatingPoint(operatingPoints, 0.001).TAR, 'f', 3))));
309   - lines.append(qPrintable(QString("BC,0.01,%1").arg(QString::number(result = getOperatingPoint(operatingPoints, 0.01).TAR, 'f', 3))));
  343 + lines.append(qPrintable(QString("BC,0.001,%1").arg(QString::number(getOperatingPointGivenFAR(operatingPoints, 0.001).TAR, 'f', 3))));
  344 + lines.append(qPrintable(QString("BC,0.01,%1").arg(QString::number(result = getOperatingPointGivenFAR(operatingPoints, 0.01).TAR, 'f', 3))));
310 345  
311 346 // Attempt to read template size from enrolled gallery and write to output CSV
312 347 size_t maxSize(0);
... ... @@ -343,10 +378,10 @@ float Evaluate(const Mat &amp;simmat, const Mat &amp;mask, const QString &amp;csv, const QSt
343 378  
344 379 QtUtils::writeFile(csv, lines);
345 380 if (maxSize > 0) qDebug("Template Size: %i bytes", (int)maxSize);
346   - qDebug("TAR @ FAR = 0.01: %.3f",getOperatingPoint(operatingPoints, 0.01).TAR);
347   - qDebug("TAR @ FAR = 0.001: %.3f",getOperatingPoint(operatingPoints, 0.001).TAR);
348   - qDebug("TAR @ FAR = 0.0001: %.3f",getOperatingPoint(operatingPoints, 0.0001).TAR);
349   - qDebug("TAR @ FAR = 0.00001: %.3f",getOperatingPoint(operatingPoints, 0.00001).TAR);
  381 + qDebug("TAR @ FAR = 0.01: %.3f",getOperatingPointGivenFAR(operatingPoints, 0.01).TAR);
  382 + qDebug("TAR @ FAR = 0.001: %.3f",getOperatingPointGivenFAR(operatingPoints, 0.001).TAR);
  383 + qDebug("TAR @ FAR = 0.0001: %.3f",getOperatingPointGivenFAR(operatingPoints, 0.0001).TAR);
  384 + qDebug("TAR @ FAR = 0.00001: %.3f",getOperatingPointGivenFAR(operatingPoints, 0.00001).TAR);
350 385  
351 386 qDebug("\nRetrieval Rate @ Rank = %d: %.3f", Report_Retrieval, getCMC(firstGenuineReturns, Report_Retrieval));
352 387  
... ... @@ -571,8 +606,8 @@ float InplaceEval(const QString &amp;simmat, const QString &amp;target, const QString &amp;q
571 606  
572 607 float result;
573 608 // Write FAR/TAR Bar Chart (BC)
574   - lines.append(qPrintable(QString("BC,0.001,%1").arg(QString::number(getOperatingPoint(operatingPoints, 0.001).TAR, 'f', 3))));
575   - lines.append(qPrintable(QString("BC,0.01,%1").arg(QString::number(result = getOperatingPoint(operatingPoints, 0.01).TAR, 'f', 3))));
  609 + lines.append(qPrintable(QString("BC,0.001,%1").arg(QString::number(getOperatingPointGivenFAR(operatingPoints, 0.001).TAR, 'f', 3))));
  610 + lines.append(qPrintable(QString("BC,0.01,%1").arg(QString::number(result = getOperatingPointGivenFAR(operatingPoints, 0.01).TAR, 'f', 3))));
576 611  
577 612 qDebug("TAR @ FAR = 0.01: %.3f", result);
578 613 QtUtils::writeFile(csv, lines);
... ...
openbr/core/plot.cpp
  View file @cbdb48c
... ... @@ -161,6 +161,7 @@ struct RPlot
161 161 "FRR <- data[grep(\"FRR\",data$Plot),-c(1)]\n"
162 162 "SD <- data[grep(\"SD\",data$Plot),-c(1)]\n"
163 163 "FT <- data[grep(\"FT\",data$Plot),-c(1)]\n"
  164 + "FatT <- data[grep(\"FatT\",data$Plot),-c(1)]\n"
164 165 "CT <- data[grep(\"CT\",data$Plot),-c(1)]\n"
165 166 "BC <- data[grep(\"BC\",data$Plot),-c(1)]\n"
166 167 "TS <- data[grep(\"TS\",data$Plot),-c(1)]\n"
... ... @@ -178,6 +179,7 @@ struct RPlot
178 179 "ERR$Y <- as.numeric(as.character(ERR$Y))\n"
179 180 "SD$Y <- as.factor(unique(as.character(SD$Y)))\n"
180 181 "FT$Y <- as.numeric(as.character(FT$Y))\n"
  182 + "FatT$Y <- as.numeric(as.character(FatT$Y))\n"
181 183 "CT$Y <- as.numeric(as.character(CT$Y))\n"
182 184 "BC$Y <- as.numeric(as.character(BC$Y))\n"
183 185 "TS$Y <- as.character(TS$Y)\n"
... ... @@ -205,6 +207,16 @@ struct RPlot
205 207 "rownames(mat) <- c(\"FAR = 1e-06\", \"FAR = 1e-05\", \"FAR = 1e-04\", \"FAR = 1e-03\", \"FAR = 1e-02\", \"FAR = 1e-01\")\n"
206 208 "FTtable <- as.table(mat)\n"
207 209 "\n"
  210 + "# Code to format TAR@FAR table\n"
  211 + "algs <- unique(FT$%2)\n"
  212 + "algs <- algs[!duplicated(algs)]\n"
  213 + "mat <- matrix(FatT$Y,nrow=6,ncol=length(algs),byrow=FALSE)\n"
  214 + "colnames(mat) <- algs \n"
  215 + "rownames(mat) <- c(\"TAR = 0.95\", \"TAR = 0.85\", \"TAR = 0.75\", \"TAR = 0.65\", \"TAR = 0.50\", \"TAR = 0.40\")\n"
  216 + "F_at_Ttable <- as.table(mat)\n"
  217 + "\n"
  218 +
  219 + "\n"
208 220 "# Code to format CMC Table\n"
209 221 "mat <- matrix(%4,nrow=6,ncol=length(algs),byrow=FALSE)\n"
210 222 "colnames(mat) <- algs \n"
... ... @@ -249,6 +261,8 @@ struct RPlot
249 261 "plot.new()\n"
250 262 "print(textplot(FTtable))\n"
251 263 "print(title(\"Table of True Accept Rates at various False Accept Rates\"))\n"
  264 + "print(textplot(F_at_Ttable))\n"
  265 + "print(title(\"Table of False Accept Rates at various True Accept Rates\"))\n"
252 266 "print(textplot(CMCtable))\n"
253 267 "print(title(\"Table of retrieval rate at various ranks\"))\n"
254 268 "if (nrow(TS) != 0) {\n\t"
... ...