Peter M. Groen / openbr

Commit 809523f2cd9e773a91c5f910fca16494c636db1b

Authored by bhklein
1 parent 3e6d1cff

add more cmc metrics

Inline Side-by-side
Showing 4 changed files with 30 additions and 19 deletions
app/br/br.cpp
  View file @809523f
... ... @@ -169,8 +169,8 @@ public:
169 169 check(parc >= 2 && parc <= 4, "Incorrect parameter count for 'evalRegression'.");
170 170 br_eval_regression(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? parv[3] : "");
171 171 } else if (!strcmp(fun, "evalKNN")) {
172   - check(parc >=2 && parc < 4, "Incorrect parameter count for 'evalKNN'.");
173   - br_eval_knn(parv[0], parv[1], parc >= 3 ? parv[2] : "");
  172 + check(parc == 3, "Incorrect parameter count for 'evalKNN'.");
  173 + br_eval_knn(parv[0], parv[1], parv[2]);
174 174 } else if (!strcmp(fun, "pairwiseCompare")) {
175 175 check((parc >= 2) && (parc <= 3), "Incorrect parameter count for 'pairwiseCompare'.");
176 176 br_pairwise_compare(parv[0], parv[1], parc == 3 ? parv[2] : "");
... ...
openbr/core/eval.cpp
  View file @809523f
... ... @@ -102,12 +102,14 @@ static OperatingPoint getOperatingPointGivenTAR(const QList&lt;OperatingPoint&gt; &amp;ope
102 102 }
103 103  
104 104  
105   -static float getCMC(const QVector<int> &firstGenuineReturns, int rank)
  105 +static float getCMC(const QVector<int> &firstGenuineReturns, int rank, size_t possibleReturns = 0)
106 106 {
107   - int realizedReturns = 0, possibleReturns = 0;
  107 + bool calcPossible = possibleReturns ? false : true;
  108 + int realizedReturns = 0;
108 109 foreach (int firstGenuineReturn, firstGenuineReturns) {
109 110 if (firstGenuineReturn > 0) {
110   - possibleReturns++;
  111 + if (calcPossible)
  112 + possibleReturns++;
111 113 if (firstGenuineReturn <= rank) realizedReturns++;
112 114 }
113 115 }
... ... @@ -1300,21 +1302,21 @@ void EvalRegression(const QString &amp;predictedGallery, const QString &amp;truthGallery
1300 1302 qDebug("MAE = %f", maeError/predicted.size());
1301 1303 }
1302 1304  
1303   -void readKNN(size_t &templateCount, size_t &k, QVector<Candidate> &neighbors, const QString &fileName)
  1305 +void readKNN(size_t &probeCount, size_t &k, QVector<Candidate> &neighbors, const QString &fileName)
1304 1306 {
1305 1307 QFile file(fileName);
1306 1308 if (!file.open(QFile::ReadOnly))
1307 1309 qFatal("Failed to open k-NN file for reading!");
1308   - file.read((char*) &templateCount, sizeof(size_t));
  1310 + file.read((char*) &probeCount, sizeof(size_t));
1309 1311 file.read((char*) &k, sizeof(size_t));
1310   - neighbors.resize(templateCount * k);
  1312 + neighbors.resize(probeCount * k);
1311 1313  
1312   - file.read((char*) neighbors.data(), templateCount * k * sizeof(Candidate));
  1314 + file.read((char*) neighbors.data(), probeCount * k * sizeof(Candidate));
1313 1315 }
1314 1316  
1315   -void readKNNTruth(size_t templateCount, QVector< QList<size_t> > &groundTruth, const QString &fileName)
  1317 +void readKNNTruth(size_t probeCount, QVector< QList<size_t> > &groundTruth, const QString &fileName)
1316 1318 {
1317   - groundTruth.reserve(templateCount);
  1319 + groundTruth.reserve(probeCount);
1318 1320 QFile truthFile(fileName);
1319 1321 if (!truthFile.open(QFile::ReadOnly | QFile::Text))
1320 1322 qFatal("Failed to open k-NN ground truth file for reading!");
... ... @@ -1330,7 +1332,7 @@ void readKNNTruth(size_t templateCount, QVector&lt; QList&lt;size_t&gt; &gt; &amp;groundTruth, c
1330 1332 }
1331 1333 i++;
1332 1334 }
1333   - if (i != templateCount)
  1335 + if (i != probeCount)
1334 1336 qFatal("Invalid ground truth file!");
1335 1337 }
1336 1338  
... ... @@ -1338,25 +1340,26 @@ void EvalKNN(const QString &amp;knnGraph, const QString &amp;knnTruth, const QString &amp;ie
1338 1340 {
1339 1341 qDebug("Evaluating k-NN of %s against %s", qPrintable(knnGraph), qPrintable(knnTruth));
1340 1342  
1341   - size_t templateCount;
  1343 + size_t probeCount;
1342 1344 size_t k;
1343 1345 QVector<Candidate> neighbors;
1344   - readKNN(templateCount, k, neighbors, knnGraph);
  1346 + readKNN(probeCount, k, neighbors, knnGraph);
1345 1347  
1346 1348 /*
1347 1349 * Read the ground truth from disk.
1348 1350 * Line i contains the template indicies of the mates for probe i.
1349 1351 * See the `gtGallery` implementation for details.
1350 1352 */
1351   - QVector< QList<size_t> > truth(templateCount);
1352   - readKNNTruth(templateCount, truth, knnTruth);
  1353 + QVector< QList<size_t> > truth(probeCount);
  1354 + readKNNTruth(probeCount, truth, knnTruth);
1353 1355  
1354 1356 /*
1355 1357 * For each probe, record the similarity of the highest mate (if one exists) and the highest non-mate.
1356 1358 */
  1359 + QVector<int> firstGenuineReturns(probeCount, 0);
1357 1360 QList<float> matedSimilarities, unmatedSimilarities;
1358 1361 size_t numMatedSearches = 0;
1359   - for (size_t i=0; i<templateCount; i++) {
  1362 + for (size_t i=0; i<probeCount; i++) {
1360 1363 const QList<size_t> &mates = truth[i];
1361 1364 if (!mates.empty())
1362 1365 numMatedSearches++;
... ... @@ -1372,12 +1375,14 @@ void EvalKNN(const QString &amp;knnGraph, const QString &amp;knnTruth, const QString &amp;ie
1372 1375 matedSimilarities.append(neighbor.similarity);
1373 1376 recordedHighestMatedSimilarity = true;
1374 1377 }
  1378 + if (firstGenuineReturns[i] < 1) firstGenuineReturns[i] = abs(firstGenuineReturns[i])+1;
1375 1379 } else {
1376 1380 // Found a non-mate
1377 1381 if (!recordedHighestUnmatedSimilarity) {
1378 1382 unmatedSimilarities.append(neighbor.similarity);
1379 1383 recordedHighestUnmatedSimilarity = true;
1380 1384 }
  1385 + if (firstGenuineReturns[i] < 1) firstGenuineReturns[i]--;
1381 1386 }
1382 1387  
1383 1388 if (recordedHighestMatedSimilarity && recordedHighestUnmatedSimilarity)
... ... @@ -1397,6 +1402,12 @@ void EvalKNN(const QString &amp;knnGraph, const QString &amp;knnTruth, const QString &amp;ie
1397 1402 if (numUnmatedSimilarities == 0)
1398 1403 qFatal("No unmated searches!");
1399 1404  
  1405 + printf("Rank-%i Return Rate: %g\n", 1, getCMC(firstGenuineReturns, 1, numMatedSearches));
  1406 + if (k >=5)
  1407 + printf("Rank-%i Return Rate: %g\n", 5, getCMC(firstGenuineReturns, 5, numMatedSearches));
  1408 + if (k >=10)
  1409 + printf("Rank-%i Return Rate: %g\n", 10, getCMC(firstGenuineReturns, 10, numMatedSearches));
  1410 +
1400 1411 printf("Rank-%zu Return Rate: %g\n", k, double(numMatedSimilarities) / double(numMatedSearches));
1401 1412  
1402 1413 // Open the output file
... ...
openbr/core/eval.h
  View file @809523f
... ... @@ -33,7 +33,7 @@ namespace br
33 33 float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", bool normalize = false, int minSize = 0, int maxSize = 0); // Return average overlap
34 34 float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", int normalizationIndexA = 0, int normalizationIndexB = 1, int sampleIndex = 0, int totalExamples = 5); // Return average error
35 35 void EvalRegression(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = "");
36   - void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &iet = "");
  36 + void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &iet);
37 37  
38 38 struct Candidate
39 39 {
... ...
openbr/openbr.h
  View file @809523f
... ... @@ -64,7 +64,7 @@ BR_EXPORT float br_eval_landmarking(const char *predicted_gallery, const char *t
64 64  
65 65 BR_EXPORT void br_eval_regression(const char *predicted_gallery, const char *truth_gallery, const char *predicted_property = "", const char *truth_property = "");
66 66  
67   -BR_EXPORT void br_eval_knn(const char *knnGraph, const char *knnTruth, const char *iet = "");
  67 +BR_EXPORT void br_eval_knn(const char *knnGraph, const char *knnTruth, const char *iet);
68 68  
69 69 BR_EXPORT void br_finalize();
70 70  
... ...