diff --git a/app/br/br.cpp b/app/br/br.cpp index fccc042..c194e28 100644 --- a/app/br/br.cpp +++ b/app/br/br.cpp @@ -168,6 +168,9 @@ public: } else if (!strcmp(fun, "evalRegression")) { check(parc >= 2 && parc <= 4, "Incorrect parameter count for 'evalRegression'."); br_eval_regression(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? parv[3] : ""); + } else if (!strcmp(fun, "evalKNN")) { + check(parc >=2 && parc < 4, "Incorrect parameter count for 'evalKNN'."); + br_eval_knn(parv[0], parv[1], parc >= 3 ? parv[2] : ""); } else if (!strcmp(fun, "pairwiseCompare")) { check((parc >= 2) && (parc <= 3), "Incorrect parameter count for 'pairwiseCompare'."); br_pairwise_compare(parv[0], parv[1], parc == 3 ? parv[2] : ""); @@ -276,6 +279,7 @@ private: "-evalDetection [{csv}] [{normalize}] [{minSize}] [{maxSize}]\n" "-evalLandmarking [{csv} [ ] [sample_index] [total_examples]]\n" "-evalRegression \n" + "-evalKNN [{iet_file}]\n" "-pairwiseCompare [{output}]\n" "-inplaceEval [{csv}]\n" "-assertEval \n" diff --git a/openbr/core/eval.cpp b/openbr/core/eval.cpp index 93cd3ac..c9e3851 100755 --- a/openbr/core/eval.cpp +++ b/openbr/core/eval.cpp @@ -1300,4 +1300,132 @@ void EvalRegression(const QString &predictedGallery, const QString &truthGallery qDebug("MAE = %f", maeError/predicted.size()); } +void readKNN(size_t &templateCount, size_t &k, QVector &neighbors, const QString &fileName) +{ + QFile file(fileName); + if (!file.open(QFile::ReadOnly)) + qFatal("Failed to open k-NN file for reading!"); + file.read((char*) &templateCount, sizeof(size_t)); + file.read((char*) &k, sizeof(size_t)); + neighbors.resize(templateCount * k); + + file.read((char*) neighbors.data(), templateCount * k * sizeof(Candidate)); +} + +void readKNNTruth(size_t templateCount, QVector< QList > &groundTruth, const QString &fileName) +{ + groundTruth.reserve(templateCount); + QFile truthFile(fileName); + if (!truthFile.open(QFile::ReadOnly | QFile::Text)) + qFatal("Failed to open k-NN ground truth file for reading!"); + size_t i=0; + while (!truthFile.atEnd()) { + const QString line = truthFile.readLine().trimmed(); + if (!line.isEmpty()) + foreach (const QString &index, line.split('\t')) { + bool ok; + groundTruth[i].append(index.toLong(&ok)); + if (!ok) + qFatal("Failed to parse long in k-NN ground truth!"); + } + i++; + } + if (i != templateCount) + qFatal("Invalid ground truth file!"); +} + +void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &iet) +{ + qDebug("Evaluating k-NN of %s against %s", qPrintable(knnGraph), qPrintable(knnTruth)); + + size_t templateCount; + size_t k; + QVector neighbors; + readKNN(templateCount, k, neighbors, knnGraph); + + /* + * Read the ground truth from disk. + * Line i contains the template indicies of the mates for probe i. + * See the `gtGallery` implementation for details. + */ + QVector< QList > truth(templateCount); + readKNNTruth(templateCount, truth, knnTruth); + + /* + * For each probe, record the similarity of the highest mate (if one exists) and the highest non-mate. + */ + QList matedSimilarities, unmatedSimilarities; + size_t numMatedSearches = 0; + for (size_t i=0; i &mates = truth[i]; + if (!mates.empty()) + numMatedSearches++; + + bool recordedHighestMatedSimilarity = false; + bool recordedHighestUnmatedSimilarity = false; + for (size_t j=0; j= threshold)) { + matedSimilarities.removeLast(); + matedCount++; + } + while (!unmatedSimilarities.empty() && (unmatedSimilarities.back() >= threshold)) { + unmatedSimilarities.removeLast(); + unmatedCount++; + } + ietFile.write(qPrintable(QString::number(threshold) + "," + + QString::number(double(unmatedCount) / double(numUnmatedSimilarities)) + "," + + QString::number(1.0 - double(matedCount) / double(numMatedSimilarities)) + "\n")); + } +} + } // namespace br diff --git a/openbr/core/eval.h b/openbr/core/eval.h index 16ef375..5e568a1 100644 --- a/openbr/core/eval.h +++ b/openbr/core/eval.h @@ -33,6 +33,17 @@ namespace br float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", bool normalize = false, int minSize = 0, int maxSize = 0); // Return average overlap float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", int normalizationIndexA = 0, int normalizationIndexB = 1, int sampleIndex = 0, int totalExamples = 5); // Return average error void EvalRegression(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = ""); + void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &iet = ""); + + struct Candidate + { + size_t index; + float similarity; + + Candidate() {} + Candidate(size_t index_, float similarity_) + : index(index_), similarity(similarity_) {} + }; } #endif // BR_EVAL_H diff --git a/openbr/openbr.cpp b/openbr/openbr.cpp index 7f15aae..cf69ecc 100644 --- a/openbr/openbr.cpp +++ b/openbr/openbr.cpp @@ -145,6 +145,11 @@ void br_eval_regression(const char *predicted_gallery, const char *truth_gallery EvalRegression(predicted_gallery, truth_gallery, predicted_property, truth_property); } +void br_eval_knn(const char *knnGraph, const char *knnTruth, const char *iet) +{ + EvalKNN(knnGraph, knnTruth, iet); +} + void br_finalize() { Context::finalize(); diff --git a/openbr/openbr.h b/openbr/openbr.h index b7a2fc9..66ae7c9 100644 --- a/openbr/openbr.h +++ b/openbr/openbr.h @@ -64,6 +64,8 @@ BR_EXPORT float br_eval_landmarking(const char *predicted_gallery, const char *t BR_EXPORT void br_eval_regression(const char *predicted_gallery, const char *truth_gallery, const char *predicted_property = "", const char *truth_property = ""); +BR_EXPORT void br_eval_knn(const char *knnGraph, const char *knnTruth, const char *iet = ""); + BR_EXPORT void br_finalize(); BR_EXPORT void br_fuse(int num_input_simmats, const char *input_simmats[], diff --git a/openbr/plugins/gallery/gt.cpp b/openbr/plugins/gallery/gt.cpp new file mode 100644 index 0000000..0048194 --- /dev/null +++ b/openbr/plugins/gallery/gt.cpp @@ -0,0 +1,49 @@ +#include "openbr/plugins/openbr_internal.h" +#include "openbr/core/qtutils.h" + +using namespace br; + +/*! + * \ingroup galleries + * \brief Ground truth format for evaluating kNN + * \author Josh Klontz \cite jklontz + */ +class gtGallery : public Gallery +{ + Q_OBJECT + + TemplateList templates; + + ~gtGallery() + { + const QList labels = File::get(TemplateList::relabel(templates, "Label", false), "Label"); + + QStringList lines; + for (int i=0; i +#include +#include +#include + +namespace br +{ + +/*! + * \ingroup outputs + * \brief Outputs the k-Nearest Neighbors from the gallery for each probe. + * \author Ben Klein \cite bhklein + */ +class knnOutput : public MatrixOutput +{ + Q_OBJECT + + ~knnOutput() + { + size_t num_probes = (size_t)queryFiles.size(); + if (targetFiles.isEmpty() || queryFiles.isEmpty()) return; + size_t k = file.get("k", 20); + + if ((size_t)targetFiles.size() < k) + qFatal("Gallery size %s is smaller than k = %s.", qPrintable(QString::number(targetFiles.size())), qPrintable(QString::number(k))); + + QFile f(file); + if (!f.open(QFile::WriteOnly)) + qFatal("Unable to open %s for writing.", qPrintable(file)); + f.write((const char*) &num_probes, sizeof(size_t)); + f.write((const char*) &k, sizeof(size_t)); + + QVector neighbors; neighbors.reserve(num_probes*k); + + for (size_t i=0; i Pair; + size_t rank = 0; + foreach (const Pair &pair, Common::Sort(OpenCVUtils::matrixToVector(data.row(i)), true)) { + if (QString(targetFiles[pair.second]) != QString(queryFiles[i])) { + Candidate candidate((size_t)pair.second, pair.first); + neighbors.push_back(candidate); + if (++rank >= k) break; + } + } + } + f.write((const char*) neighbors.data(), num_probes * k * sizeof(Candidate)); + f.close(); + } +}; + +BR_REGISTER(Output, knnOutput) + +} // namespace br + +#include "output/knn.moc"