From 809523f2cd9e773a91c5f910fca16494c636db1b Mon Sep 17 00:00:00 2001 From: bhklein Date: Mon, 28 Sep 2015 23:52:50 -0400 Subject: [PATCH] add more cmc metrics --- app/br/br.cpp | 4 ++-- openbr/core/eval.cpp | 41 ++++++++++++++++++++++++++--------------- openbr/core/eval.h | 2 +- openbr/openbr.h | 2 +- 4 files changed, 30 insertions(+), 19 deletions(-) diff --git a/app/br/br.cpp b/app/br/br.cpp index c194e28..06a3589 100644 --- a/app/br/br.cpp +++ b/app/br/br.cpp @@ -169,8 +169,8 @@ public: check(parc >= 2 && parc <= 4, "Incorrect parameter count for 'evalRegression'."); br_eval_regression(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? parv[3] : ""); } else if (!strcmp(fun, "evalKNN")) { - check(parc >=2 && parc < 4, "Incorrect parameter count for 'evalKNN'."); - br_eval_knn(parv[0], parv[1], parc >= 3 ? parv[2] : ""); + check(parc == 3, "Incorrect parameter count for 'evalKNN'."); + br_eval_knn(parv[0], parv[1], parv[2]); } else if (!strcmp(fun, "pairwiseCompare")) { check((parc >= 2) && (parc <= 3), "Incorrect parameter count for 'pairwiseCompare'."); br_pairwise_compare(parv[0], parv[1], parc == 3 ? parv[2] : ""); diff --git a/openbr/core/eval.cpp b/openbr/core/eval.cpp index c9e3851..aba748f 100755 --- a/openbr/core/eval.cpp +++ b/openbr/core/eval.cpp @@ -102,12 +102,14 @@ static OperatingPoint getOperatingPointGivenTAR(const QList &ope } -static float getCMC(const QVector &firstGenuineReturns, int rank) +static float getCMC(const QVector &firstGenuineReturns, int rank, size_t possibleReturns = 0) { - int realizedReturns = 0, possibleReturns = 0; + bool calcPossible = possibleReturns ? false : true; + int realizedReturns = 0; foreach (int firstGenuineReturn, firstGenuineReturns) { if (firstGenuineReturn > 0) { - possibleReturns++; + if (calcPossible) + possibleReturns++; if (firstGenuineReturn <= rank) realizedReturns++; } } @@ -1300,21 +1302,21 @@ void EvalRegression(const QString &predictedGallery, const QString &truthGallery qDebug("MAE = %f", maeError/predicted.size()); } -void readKNN(size_t &templateCount, size_t &k, QVector &neighbors, const QString &fileName) +void readKNN(size_t &probeCount, size_t &k, QVector &neighbors, const QString &fileName) { QFile file(fileName); if (!file.open(QFile::ReadOnly)) qFatal("Failed to open k-NN file for reading!"); - file.read((char*) &templateCount, sizeof(size_t)); + file.read((char*) &probeCount, sizeof(size_t)); file.read((char*) &k, sizeof(size_t)); - neighbors.resize(templateCount * k); + neighbors.resize(probeCount * k); - file.read((char*) neighbors.data(), templateCount * k * sizeof(Candidate)); + file.read((char*) neighbors.data(), probeCount * k * sizeof(Candidate)); } -void readKNNTruth(size_t templateCount, QVector< QList > &groundTruth, const QString &fileName) +void readKNNTruth(size_t probeCount, QVector< QList > &groundTruth, const QString &fileName) { - groundTruth.reserve(templateCount); + groundTruth.reserve(probeCount); QFile truthFile(fileName); if (!truthFile.open(QFile::ReadOnly | QFile::Text)) qFatal("Failed to open k-NN ground truth file for reading!"); @@ -1330,7 +1332,7 @@ void readKNNTruth(size_t templateCount, QVector< QList > &groundTruth, c } i++; } - if (i != templateCount) + if (i != probeCount) qFatal("Invalid ground truth file!"); } @@ -1338,25 +1340,26 @@ void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &ie { qDebug("Evaluating k-NN of %s against %s", qPrintable(knnGraph), qPrintable(knnTruth)); - size_t templateCount; + size_t probeCount; size_t k; QVector neighbors; - readKNN(templateCount, k, neighbors, knnGraph); + readKNN(probeCount, k, neighbors, knnGraph); /* * Read the ground truth from disk. * Line i contains the template indicies of the mates for probe i. * See the `gtGallery` implementation for details. */ - QVector< QList > truth(templateCount); - readKNNTruth(templateCount, truth, knnTruth); + QVector< QList > truth(probeCount); + readKNNTruth(probeCount, truth, knnTruth); /* * For each probe, record the similarity of the highest mate (if one exists) and the highest non-mate. */ + QVector firstGenuineReturns(probeCount, 0); QList matedSimilarities, unmatedSimilarities; size_t numMatedSearches = 0; - for (size_t i=0; i &mates = truth[i]; if (!mates.empty()) numMatedSearches++; @@ -1372,12 +1375,14 @@ void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &ie matedSimilarities.append(neighbor.similarity); recordedHighestMatedSimilarity = true; } + if (firstGenuineReturns[i] < 1) firstGenuineReturns[i] = abs(firstGenuineReturns[i])+1; } else { // Found a non-mate if (!recordedHighestUnmatedSimilarity) { unmatedSimilarities.append(neighbor.similarity); recordedHighestUnmatedSimilarity = true; } + if (firstGenuineReturns[i] < 1) firstGenuineReturns[i]--; } if (recordedHighestMatedSimilarity && recordedHighestUnmatedSimilarity) @@ -1397,6 +1402,12 @@ void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &ie if (numUnmatedSimilarities == 0) qFatal("No unmated searches!"); + printf("Rank-%i Return Rate: %g\n", 1, getCMC(firstGenuineReturns, 1, numMatedSearches)); + if (k >=5) + printf("Rank-%i Return Rate: %g\n", 5, getCMC(firstGenuineReturns, 5, numMatedSearches)); + if (k >=10) + printf("Rank-%i Return Rate: %g\n", 10, getCMC(firstGenuineReturns, 10, numMatedSearches)); + printf("Rank-%zu Return Rate: %g\n", k, double(numMatedSimilarities) / double(numMatedSearches)); // Open the output file diff --git a/openbr/core/eval.h b/openbr/core/eval.h index 5e568a1..d09ce3d 100644 --- a/openbr/core/eval.h +++ b/openbr/core/eval.h @@ -33,7 +33,7 @@ namespace br float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", bool normalize = false, int minSize = 0, int maxSize = 0); // Return average overlap float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", int normalizationIndexA = 0, int normalizationIndexB = 1, int sampleIndex = 0, int totalExamples = 5); // Return average error void EvalRegression(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = ""); - void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &iet = ""); + void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &iet); struct Candidate { diff --git a/openbr/openbr.h b/openbr/openbr.h index 66ae7c9..dd06591 100644 --- a/openbr/openbr.h +++ b/openbr/openbr.h @@ -64,7 +64,7 @@ BR_EXPORT float br_eval_landmarking(const char *predicted_gallery, const char *t BR_EXPORT void br_eval_regression(const char *predicted_gallery, const char *truth_gallery, const char *predicted_property = "", const char *truth_property = ""); -BR_EXPORT void br_eval_knn(const char *knnGraph, const char *knnTruth, const char *iet = ""); +BR_EXPORT void br_eval_knn(const char *knnGraph, const char *knnTruth, const char *iet); BR_EXPORT void br_finalize(); -- libgit2 0.21.4