add gtGallery, knnOutput and evalKNN

bhklein
1 parent 70a3f508
Showing 7 changed files with 254 additions and 0 deletions
app/br/br.cpp
openbr/core/eval.cpp
openbr/core/eval.h
openbr/openbr.cpp
openbr/openbr.h
openbr/plugins/gallery/gt.cpp
openbr/plugins/output/knn.cpp
@@ -168,6 +168,9 @@ public:
             } else if (!strcmp(fun, "evalRegression")) {
                 check(parc >= 2 && parc <= 4, "Incorrect parameter count for 'evalRegression'.");
                 br_eval_regression(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? parv[3] : "");
+            } else if (!strcmp(fun, "evalKNN")) {
+                check(parc >=2 && parc < 4, "Incorrect parameter count for 'evalKNN'.");
+                br_eval_knn(parv[0], parv[1], parc >= 3 ? parv[2] : "");
             } else if (!strcmp(fun, "pairwiseCompare")) {
                 check((parc >= 2) && (parc <= 3), "Incorrect parameter count for 'pairwiseCompare'.");
                 br_pairwise_compare(parv[0], parv[1], parc == 3 ? parv[2] : "");
@@ -276,6 +279,7 @@ private:
                "-evalDetection <predicted_gallery> <truth_gallery> [{csv}] [{normalize}] [{minSize}] [{maxSize}]\n"
                "-evalLandmarking <predicted_gallery> <truth_gallery> [{csv} [<normalization_index_a> <normalization_index_b>] [sample_index] [total_examples]]\n"
                "-evalRegression <predicted_gallery> <truth_gallery> <predicted property name> <ground truth property name>\n"
+               "-evalKNN <knn_graph> <knn_truth> [{iet_file}]\n"
                "-pairwiseCompare <target_gallery> <query_gallery> [{output}]\n"
                "-inplaceEval <simmat> <target> <query> [{csv}]\n"
                "-assertEval <simmat> <mask> <accuracy>\n"
@@ -1300,4 +1300,132 @@ void EvalRegression(const QString &amp;predictedGallery, const QString &amp;truthGallery
     qDebug("MAE = %f", maeError/predicted.size());
 }
  
+void readKNN(size_t &templateCount, size_t &k, QVector<Candidate> &neighbors, const QString &fileName)
+{
+    QFile file(fileName);
+    if (!file.open(QFile::ReadOnly))
+        qFatal("Failed to open k-NN file for reading!");
+    file.read((char*) &templateCount, sizeof(size_t));
+    file.read((char*) &k, sizeof(size_t));
+    neighbors.resize(templateCount * k);
+
+    file.read((char*) neighbors.data(), templateCount * k * sizeof(Candidate));
+}
+
+void readKNNTruth(size_t templateCount, QVector< QList<size_t> > &groundTruth, const QString &fileName)
+{
+    groundTruth.reserve(templateCount);
+    QFile truthFile(fileName);
+    if (!truthFile.open(QFile::ReadOnly | QFile::Text))
+        qFatal("Failed to open k-NN ground truth file for reading!");
+    size_t i=0;
+    while (!truthFile.atEnd()) {
+        const QString line = truthFile.readLine().trimmed();
+        if (!line.isEmpty())
+            foreach (const QString &index, line.split('\t')) {
+                bool ok;
+                groundTruth[i].append(index.toLong(&ok));
+                if (!ok)
+                    qFatal("Failed to parse long in k-NN ground truth!");
+            }
+        i++;
+    }
+    if (i != templateCount)
+        qFatal("Invalid ground truth file!");
+}
+
+void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &iet)
+{
+    qDebug("Evaluating k-NN of %s against %s", qPrintable(knnGraph), qPrintable(knnTruth));
+
+    size_t templateCount;
+    size_t k;
+    QVector<Candidate> neighbors;
+    readKNN(templateCount, k, neighbors, knnGraph);
+
+    /*
+     * Read the ground truth from disk.
+     * Line i contains the template indicies of the mates for probe i.
+     * See the `gtGallery` implementation for details.
+     */
+    QVector< QList<size_t> > truth(templateCount);
+    readKNNTruth(templateCount, truth, knnTruth);
+
+    /*
+     * For each probe, record the similarity of the highest mate (if one exists) and the highest non-mate.
+     */
+    QList<float> matedSimilarities, unmatedSimilarities;
+    size_t numMatedSearches = 0;
+    for (size_t i=0; i<templateCount; i++) {
+        const QList<size_t> &mates = truth[i];
+        if (!mates.empty())
+            numMatedSearches++;
+
+        bool recordedHighestMatedSimilarity = false;
+        bool recordedHighestUnmatedSimilarity = false;
+        for (size_t j=0; j<k; j++) {
+            const Candidate &neighbor = neighbors[i*k+j];
+
+            if (mates.contains(neighbor.index)) {
+                // Found a mate
+                if (!recordedHighestMatedSimilarity) {
+                    matedSimilarities.append(neighbor.similarity);
+                    recordedHighestMatedSimilarity = true;
+                }
+            } else {
+                // Found a non-mate
+                if (!recordedHighestUnmatedSimilarity) {
+                    unmatedSimilarities.append(neighbor.similarity);
+                    recordedHighestUnmatedSimilarity = true;
+                }
+            }
+
+            if (recordedHighestMatedSimilarity && recordedHighestUnmatedSimilarity)
+                break; // we can stop scanning the candidate list for this probe
+        }
+    }
+
+    // Sort the similarity scores lowest-to-highest
+    std::sort(matedSimilarities.begin(), matedSimilarities.end());
+    std::sort(unmatedSimilarities.begin(), unmatedSimilarities.end());
+    const size_t numMatedSimilarities = matedSimilarities.size();
+    const size_t numUnmatedSimilarities = unmatedSimilarities.size();
+
+    if (numMatedSimilarities == 0)
+        qFatal("No mated searches!");
+
+    if (numUnmatedSimilarities == 0)
+        qFatal("No unmated searches!");
+
+    printf("Rank-%zu Return Rate: %g\n", k, double(numMatedSimilarities) / double(numMatedSearches));
+
+    // Open the output file
+    QFile ietFile(iet);
+    if (!ietFile.open(QFile::WriteOnly | QFile::Text))
+        qFatal("Failed to open IET file for writing!");
+    ietFile.write("Threshold,FPIR,FNIR\n");
+
+    /*
+     * Iterate through the similarity scores highest-to-lowest,
+     * for each threshold count the number mated and unmated searches,
+     * record the corresponding FPIR and FNIR values for the threshold.
+     */
+    size_t matedCount = 0;
+    size_t unmatedCount = 0;
+    while (!matedSimilarities.empty()) {
+        const float threshold = matedSimilarities.back();
+        while (!matedSimilarities.empty() && (matedSimilarities.back() >= threshold)) {
+            matedSimilarities.removeLast();
+            matedCount++;
+        }
+        while (!unmatedSimilarities.empty() && (unmatedSimilarities.back() >= threshold)) {
+            unmatedSimilarities.removeLast();
+            unmatedCount++;
+        }
+        ietFile.write(qPrintable(QString::number(threshold) + "," +
+                                 QString::number(double(unmatedCount) / double(numUnmatedSimilarities)) + "," +
+                                 QString::number(1.0 - double(matedCount) / double(numMatedSimilarities)) + "\n"));
+    }
+}
+
 } // namespace br
@@ -33,6 +33,17 @@ namespace br
     float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", bool normalize = false, int minSize = 0, int maxSize = 0); // Return average overlap
     float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", int normalizationIndexA = 0, int normalizationIndexB = 1, int sampleIndex = 0, int totalExamples = 5); // Return average error
     void EvalRegression(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = "");
+    void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &iet = "");
+
+    struct Candidate
+    {
+        size_t index;
+        float similarity;
+
+        Candidate() {}
+        Candidate(size_t index_, float similarity_)
+            : index(index_), similarity(similarity_) {}
+    };
 }
  
 #endif // BR_EVAL_H
@@ -145,6 +145,11 @@ void br_eval_regression(const char *predicted_gallery, const char *truth_gallery
     EvalRegression(predicted_gallery, truth_gallery, predicted_property, truth_property);
 }
  
+void br_eval_knn(const char *knnGraph, const char *knnTruth, const char *iet)
+{
+    EvalKNN(knnGraph, knnTruth, iet);
+}
+
 void br_finalize()
 {
     Context::finalize();
@@ -64,6 +64,8 @@ BR_EXPORT float br_eval_landmarking(const char *predicted_gallery, const char *t
  
 BR_EXPORT void br_eval_regression(const char *predicted_gallery, const char *truth_gallery, const char *predicted_property = "", const char *truth_property = "");
  
+BR_EXPORT void br_eval_knn(const char *knnGraph, const char *knnTruth, const char *iet = "");
+
 BR_EXPORT void br_finalize();
  
 BR_EXPORT void br_fuse(int num_input_simmats, const char *input_simmats[],
+#include "openbr/plugins/openbr_internal.h"
+#include "openbr/core/qtutils.h"
+
+using namespace br;
+
+/*!
+ * \ingroup galleries
+ * \brief Ground truth format for evaluating kNN
+ * \author Josh Klontz \cite jklontz
+ */
+class gtGallery : public Gallery
+{
+    Q_OBJECT
+
+    TemplateList templates;
+
+    ~gtGallery()
+    {
+        const QList<int> labels = File::get<int>(TemplateList::relabel(templates, "Label", false), "Label");
+
+        QStringList lines;
+        for (int i=0; i<labels.size(); i++) {
+            QStringList words;
+            for (int j=0; j<labels.size(); j++) {
+                if ((labels[i] == labels[j]) && (i != j))
+                    words.append(QString::number(j));
+            }
+            lines.append(words.join("\t"));
+        }
+
+        QtUtils::writeFile(file.name, lines);
+    }
+
+    TemplateList readBlock(bool *done)
+    {
+        *done = true;
+        qFatal("Not supported!");
+        return TemplateList();
+    }
+
+    void write(const Template &t)
+    {
+        templates.append(t);
+    }
+};
+
+BR_REGISTER(Gallery, gtGallery)
+
+#include "gt.moc"
+#include <openbr/plugins/openbr_internal.h>
+#include <openbr/core/common.h>
+#include <openbr/core/opencvutils.h>
+#include <openbr/core/eval.h>
+
+namespace br
+{
+
+/*!
+ * \ingroup outputs
+ * \brief Outputs the k-Nearest Neighbors from the gallery for each probe.
+ * \author Ben Klein \cite bhklein
+ */
+class knnOutput : public MatrixOutput
+{
+    Q_OBJECT
+
+    ~knnOutput()
+    {
+        size_t num_probes = (size_t)queryFiles.size();
+        if (targetFiles.isEmpty() || queryFiles.isEmpty()) return;
+        size_t k = file.get<size_t>("k", 20);
+
+        if ((size_t)targetFiles.size() < k)
+            qFatal("Gallery size %s is smaller than k = %s.", qPrintable(QString::number(targetFiles.size())), qPrintable(QString::number(k)));
+
+        QFile f(file);
+        if (!f.open(QFile::WriteOnly))
+            qFatal("Unable to open %s for writing.", qPrintable(file));
+        f.write((const char*) &num_probes, sizeof(size_t));
+        f.write((const char*) &k, sizeof(size_t));
+
+        QVector<Candidate> neighbors; neighbors.reserve(num_probes*k);
+
+        for (size_t i=0; i<num_probes; i++) {
+            typedef QPair<float,int> Pair;
+            size_t rank = 0;
+            foreach (const Pair &pair, Common::Sort(OpenCVUtils::matrixToVector<float>(data.row(i)), true)) {
+                if (QString(targetFiles[pair.second]) != QString(queryFiles[i])) {
+                    Candidate candidate((size_t)pair.second, pair.first);
+                    neighbors.push_back(candidate);
+                    if (++rank >= k) break;
+                }
+            }
+        }
+        f.write((const char*) neighbors.data(), num_probes * k * sizeof(Candidate));
+        f.close();
+    }
+};
+
+BR_REGISTER(Output, knnOutput)
+
+} // namespace br
+
+#include "output/knn.moc"