Merge branch 'master' of https://github.com/biometrics/openbr

Scott Klum
2 parents 848a7e70 9b345841
Showing 18 changed files with 473 additions and 38 deletions
CMakeLists.txt
app/br/br.cpp
docs/docs/install.md
openbr/core/eval.cpp
openbr/core/eval.h
openbr/core/plot.cpp
openbr/core/plot.h
openbr/openbr.cpp
openbr/openbr.h
openbr/plugins/classification/dlib.cpp
openbr/plugins/core/algorithms.cpp
openbr/plugins/gallery/gt.cpp
openbr/plugins/gallery/keyframes.cpp
openbr/plugins/output/knn.cpp
scripts/face_cluster_viz.py → scripts/brpy/face_cluster_viz.py
scripts/brpy/html_viz.py
share/openbr/cmake/InstallDependencies.cmake
share/openbr/plotting/plot_utils.R
@@ -13,7 +13,7 @@ if(NOT DEFINED CPACK_PACKAGE_VERSION_MAJOR)
   set(CPACK_PACKAGE_VERSION_MAJOR 1)
 endif()
 if(NOT DEFINED CPACK_PACKAGE_VERSION_MINOR)
-  set(CPACK_PACKAGE_VERSION_MINOR 0)
+  set(CPACK_PACKAGE_VERSION_MINOR 1)
 endif()
 if(NOT DEFINED CPACK_PACKAGE_VERSION_PATCH)
   set(CPACK_PACKAGE_VERSION_PATCH 0)
@@ -158,8 +158,8 @@ endif()
  
 # Download the models
 ExternalProject_Add(models
-                    URL http://github.com/biometrics/openbr/releases/download/v1.0.0/models.tar.gz
-                    URL_MD5 0a7c79226d6629954aa32c835a1007b9
+                    URL http://github.com/biometrics/openbr/releases/download/v1.1.0/models.tar.gz
+                    URL_MD5 d06927071120d5a50eee9475fe8da280
                     SOURCE_DIR "${PROJECT_SOURCE_DIR}/share/openbr/models"
                     CONFIGURE_COMMAND ""
                     BUILD_COMMAND ""
@@ -168,6 +168,9 @@ public:
             } else if (!strcmp(fun, "evalRegression")) {
                 check(parc >= 2 && parc <= 4, "Incorrect parameter count for 'evalRegression'.");
                 br_eval_regression(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? parv[3] : "");
+            } else if (!strcmp(fun, "evalKNN")) {
+                check(parc >= 2 && parc <= 3, "Incorrect parameter count for 'evalKNN'.");
+                br_eval_knn(parv[0], parv[1], parc > 2 ? parv[2] : "");
             } else if (!strcmp(fun, "pairwiseCompare")) {
                 check((parc >= 2) && (parc <= 3), "Incorrect parameter count for 'pairwiseCompare'.");
                 br_pairwise_compare(parv[0], parv[1], parc == 3 ? parv[2] : "");
@@ -183,6 +186,9 @@ public:
             } else if (!strcmp(fun, "plotMetadata")) {
                 check(parc >= 2, "Incorrect parameter count for 'plotMetadata'.");
                 br_plot_metadata(parc-1, parv, parv[parc-1], true);
+            } else if (!strcmp(fun, "plotKNN")) {
+                check(parc >=2, "Incorrect parameter count for 'plotKNN'.");
+                br_plot_knn(parc-1, parv, parv[parc-1], true);
             } else if (!strcmp(fun, "project")) {
                 check(parc == 2, "Insufficient parameter count for 'project'.");
                 br_project(parv[0], parv[1]);
@@ -276,12 +282,14 @@ private:
                "-evalDetection <predicted_gallery> <truth_gallery> [{csv}] [{normalize}] [{minSize}] [{maxSize}]\n"
                "-evalLandmarking <predicted_gallery> <truth_gallery> [{csv} [<normalization_index_a> <normalization_index_b>] [sample_index] [total_examples]]\n"
                "-evalRegression <predicted_gallery> <truth_gallery> <predicted property name> <ground truth property name>\n"
+               "-evalKNN <knn_graph> <knn_truth> [{csv}]\n"
                "-pairwiseCompare <target_gallery> <query_gallery> [{output}]\n"
                "-inplaceEval <simmat> <target> <query> [{csv}]\n"
                "-assertEval <simmat> <mask> <accuracy>\n"
                "-plotDetection <file> ... <file> {destination}\n"
                "-plotLandmarking <file> ... <file> {destination}\n"
                "-plotMetadata <file> ... <file> <columns>\n"
+               "-plotKNN <file> ... <file> {destination}\n"
                "-project <input_gallery> {output_gallery}\n"
                "-deduplicate <input_gallery> <output_gallery> <threshold>\n"
                "-likely <input_type> <output_type> <output_likely_source>\n"
@@ -35,7 +35,7 @@ A hacker&#39;s guide to building, editing, and running OpenBR.
  
         $ git clone https://github.com/biometrics/openbr.git
         $ cd openbr
-        $ git checkout 1.0
+        $ git checkout v1.1.0
         $ git submodule init
         $ git submodule update
  
@@ -116,7 +116,7 @@ A hacker&#39;s guide to building, editing, and running OpenBR.
  
         $ git clone https://github.com/biometrics/openbr.git
         $ cd openbr
-        $ git checkout 1.0
+        $ git checkout v1.1.0
         $ git submodule init
         $ git submodule update
  
@@ -196,7 +196,7 @@ A hacker&#39;s guide to building, editing, and running OpenBR.
             $ cd /c
             $ git clone https://github.com/biometrics/openbr.git
             $ cd openbr
-            $ git checkout 1.0
+            $ git checkout v1.1.0
             $ git submodule init
             $ git submodule update
  
@@ -277,7 +277,7 @@ A hacker&#39;s guide to building, editing, and running OpenBR.
  
         $ git clone https://github.com/biometrics/openbr.git
         $ cd openbr
-        $ git checkout 1.0
+        $ git checkout v1.1.0
         $ git submodule init
         $ git submodule update
  
@@ -102,12 +102,14 @@ static OperatingPoint getOperatingPointGivenTAR(const QList&lt;OperatingPoint&gt; &amp;ope
 }
  
  
-static float getCMC(const QVector<int> &firstGenuineReturns, int rank)
+static float getCMC(const QVector<int> &firstGenuineReturns, int rank, size_t possibleReturns = 0)
 {
-    int realizedReturns = 0, possibleReturns = 0;
+    bool calcPossible = possibleReturns ? false : true;
+    int realizedReturns = 0;
     foreach (int firstGenuineReturn, firstGenuineReturns) {
         if (firstGenuineReturn > 0) {
-            possibleReturns++;
+            if (calcPossible)
+                possibleReturns++;
             if (firstGenuineReturn <= rank) realizedReturns++;
         }
     }
@@ -1300,4 +1302,168 @@ void EvalRegression(const QString &amp;predictedGallery, const QString &amp;truthGallery
     qDebug("MAE = %f", maeError/predicted.size());
 }
  
+void readKNN(size_t &probeCount, size_t &k, QVector<Candidate> &neighbors, const QString &fileName)
+{
+    QFile file(fileName);
+    if (!file.open(QFile::ReadOnly))
+        qFatal("Failed to open k-NN file for reading!");
+    file.read((char*) &probeCount, sizeof(size_t));
+    file.read((char*) &k, sizeof(size_t));
+    neighbors.resize(probeCount * k);
+
+    file.read((char*) neighbors.data(), probeCount * k * sizeof(Candidate));
+}
+
+void readKNNTruth(size_t probeCount, QVector< QList<size_t> > &groundTruth, const QString &fileName)
+{
+    groundTruth.reserve(probeCount);
+    QFile truthFile(fileName);
+    if (!truthFile.open(QFile::ReadOnly | QFile::Text))
+        qFatal("Failed to open k-NN ground truth file for reading!");
+    size_t i=0;
+    while (!truthFile.atEnd()) {
+        const QString line = truthFile.readLine().trimmed();
+        if (!line.isEmpty())
+            foreach (const QString &index, line.split('\t')) {
+                bool ok;
+                groundTruth[i].append(index.toLong(&ok));
+                if (!ok)
+                    qFatal("Failed to parse long in k-NN ground truth!");
+            }
+        i++;
+    }
+    if (i != probeCount)
+        qFatal("Invalid ground truth file!");
+}
+
+void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &csv)
+{
+    qDebug("Evaluating k-NN of %s against %s", qPrintable(knnGraph), qPrintable(knnTruth));
+
+    size_t probeCount;
+    size_t k;
+    QVector<Candidate> neighbors;
+    readKNN(probeCount, k, neighbors, knnGraph);
+
+    /*
+     * Read the ground truth from disk.
+     * Line i contains the template indicies of the mates for probe i.
+     * See the `gtGallery` implementation for details.
+     */
+    QVector< QList<size_t> > truth(probeCount);
+    readKNNTruth(probeCount, truth, knnTruth);
+
+    /*
+     * For each probe, record the similarity of the highest mate (if one exists) and the highest non-mate.
+     */
+    QVector<int> firstGenuineReturns(probeCount, 0);
+    QList<float> matedSimilarities, unmatedSimilarities;
+    size_t numMatedSearches = 0, numUnmatedSearches = 0;
+    for (size_t i=0; i<probeCount; i++) {
+        const QList<size_t> &mates = truth[i];
+        bool recordedHighestMatedSimilarity = false;
+        bool recordedHighestUnmatedSimilarity = false;
+        if (!mates.empty()) {
+            numMatedSearches++;
+            recordedHighestUnmatedSimilarity = true;
+        } else {
+            numUnmatedSearches++;
+            recordedHighestMatedSimilarity = true;
+        }
+
+        for (size_t j=0; j<k; j++) {
+            const Candidate &neighbor = neighbors[i*k+j];
+
+            if (mates.contains(neighbor.index)) {
+                // Found a mate
+                if (!recordedHighestMatedSimilarity) {
+                    matedSimilarities.append(neighbor.similarity);
+                    recordedHighestMatedSimilarity = true;
+                }
+                if (firstGenuineReturns[i] < 1) firstGenuineReturns[i] = abs(firstGenuineReturns[i])+1;
+            } else {
+                // Found a non-mate
+                if (!recordedHighestUnmatedSimilarity) {
+                    unmatedSimilarities.append(neighbor.similarity);
+                    recordedHighestUnmatedSimilarity = true;
+                }
+                if (firstGenuineReturns[i] < 1) firstGenuineReturns[i]--;
+            }
+
+            if (recordedHighestMatedSimilarity && recordedHighestUnmatedSimilarity)
+                break; // we can stop scanning the candidate list for this probe
+        }
+    }
+
+    // Sort the similarity scores lowest-to-highest
+    std::sort(matedSimilarities.begin(), matedSimilarities.end());
+    std::sort(unmatedSimilarities.begin(), unmatedSimilarities.end());
+    const size_t numMatedSimilarities = matedSimilarities.size();
+
+    if (numMatedSearches == 0)
+        qFatal("No mated searches!");
+
+    if (numUnmatedSearches == 0)
+        qFatal("No unmated searches!");
+
+
+    qDebug("Rank-%d Return Rate: %.3f", 1, getCMC(firstGenuineReturns, 1, numMatedSearches));
+    if (k >=5)
+        qDebug("Rank-%d Return Rate: %.3f", 5, getCMC(firstGenuineReturns, 5, numMatedSearches));
+    if (k >=10)
+        qDebug("Rank-%d Return Rate: %.3f", 10, getCMC(firstGenuineReturns, 10, numMatedSearches));
+
+    qDebug("Rank-%zu Return Rate: %.3f", k, double(numMatedSimilarities) / double(numMatedSearches));
+
+    /*
+     * Iterate through the similarity scores highest-to-lowest,
+     * for each threshold count the number mated and unmated searches,
+     * record the corresponding FPIR and FNIR values for the threshold.
+     */
+    QList<OperatingPoint> operatingPoints;
+    size_t matedCount = 0, previousMatedCount = 0;
+    size_t unmatedCount = 0, previousUnmatedCount = 0;
+    while (!matedSimilarities.empty()) {
+        const float threshold = matedSimilarities.back();
+        while (!matedSimilarities.empty() && (matedSimilarities.back() >= threshold)) {
+            matedSimilarities.removeLast();
+            matedCount++;
+        }
+        while (!unmatedSimilarities.empty() && (unmatedSimilarities.back() >= threshold)) {
+            unmatedSimilarities.removeLast();
+            unmatedCount++;
+        }
+        if ((unmatedCount > previousUnmatedCount) && (matedCount > previousMatedCount)) {
+            previousMatedCount = matedCount;
+            previousUnmatedCount = unmatedCount;
+            operatingPoints.append(OperatingPoint(threshold,
+                                                  double(unmatedCount) / double(numUnmatedSearches),
+                                                  1.0 - double(matedCount) / double(numMatedSearches)));
+        }
+    }
+
+    if (!csv.isEmpty()) {
+        // Open the output file
+        QFile ietFile(csv);
+        if (!ietFile.open(QFile::WriteOnly | QFile::Text))
+            qFatal("Failed to open IET file for writing!");
+        ietFile.write("Plot,X,Y,Z\n");
+        // Write CMC
+        const int Max_Retrieval = min(200, (int)k);
+        for (int i=1; i<=Max_Retrieval; i++) {
+            const float retrievalRate = getCMC(firstGenuineReturns, i, numMatedSearches);
+            ietFile.write(qPrintable(QString("CMC,%1,%2,0\n").arg(QString::number(i), QString::number(retrievalRate))));
+        }
+
+        foreach(const OperatingPoint &operatingPoint, operatingPoints)
+            ietFile.write(qPrintable("IET," +
+                                     QString::number(operatingPoint.FAR) + "," +
+                                     QString::number(operatingPoint.TAR) + "," +
+                                     QString::number(operatingPoint.score) + "\n"));
+    }
+
+    qDebug("FNIR @ FPIR = 0.1:   %.3f", 1-getOperatingPointGivenFAR(operatingPoints, 0.1).TAR);
+    qDebug("FNIR @ FPIR = 0.01:  %.3f", 1-getOperatingPointGivenFAR(operatingPoints, 0.01).TAR);
+}
+
 } // namespace br
@@ -33,6 +33,17 @@ namespace br
     float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", bool normalize = false, int minSize = 0, int maxSize = 0); // Return average overlap
     float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", int normalizationIndexA = 0, int normalizationIndexB = 1, int sampleIndex = 0, int totalExamples = 5); // Return average error
     void EvalRegression(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = "");
+    void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &csv = "");
+
+    struct Candidate
+    {
+        size_t index;
+        float similarity;
+
+        Candidate() {}
+        Candidate(size_t index_, float similarity_)
+            : index(index_), similarity(similarity_) {}
+    };
 }
  
 #endif // BR_EVAL_H
@@ -346,4 +346,32 @@ bool PlotMetadata(const QStringList &amp;files, const QString &amp;columns, bool show)
     return p.finalize(show);
 }
  
+bool PlotKNN(const QStringList &files, const File &destination, bool show)
+{
+    qDebug("Plotting %d k-NN file(s) to %s", files.size(), qPrintable(destination));
+    RPlot p(files, destination);
+    p.file.write("\nformatData(type=\"knn\")\n\n");
+
+    QMap<QString,File> optMap;
+    optMap.insert("rocOptions", File(QString("[xTitle=False Positive Identification Rate (FPIR),yTitle=True Positive Identification Rate (TPIR),xLog=true,yLog=false]")));
+    optMap.insert("ietOptions", File(QString("[xTitle=False Positive Identification Rate (FPIR),yTitle=False Negative Identification Rate (FNIR),xLog=true,yLog=true]")));
+    optMap.insert("cmcOptions", File(QString("[xTitle=Rank,yTitle=Retrieval Rate,xLog=true,yLog=false,size=1,xLabels=(1,5,10,50,100),xBreaks=(1,5,10,50,100)]")));
+
+    foreach (const QString &key, optMap.keys()) {
+        const QStringList options = destination.get<QStringList>(key, QStringList());
+        foreach (const QString &option, options) {
+            QStringList words = QtUtils::parse(option, '=');
+            QtUtils::checkArgsSize(words[0], words, 1, 2);
+            optMap[key].set(words[0], words[1]);
+        }
+    }
+
+    QString plot = "plot <- plotLine(lineData=%1, options=list(%2), flipY=%3)\nplot\n";
+    p.file.write(qPrintable(QString(plot).arg("IET", toRList(optMap["rocOptions"]), "TRUE")));
+    p.file.write(qPrintable(QString(plot).arg("IET", toRList(optMap["ietOptions"]), "FALSE")));
+    p.file.write(qPrintable(QString(plot).arg("CMC", toRList(optMap["cmcOptions"]), "FALSE")));
+
+    return p.finalize(show);
+}
+
 } // namespace br
@@ -28,6 +28,7 @@ namespace br
     bool PlotDetection(const QStringList &files, const File &destination, bool show = false);
     bool PlotLandmarking(const QStringList &files, const File &destination, bool show = false);
     bool PlotMetadata(const QStringList &files, const QString &destination, bool show = false);
+    bool PlotKNN(const QStringList &files, const File &destination, bool show = false);
 }
  
 #endif // BR_PLOT_H
@@ -145,6 +145,11 @@ void br_eval_regression(const char *predicted_gallery, const char *truth_gallery
     EvalRegression(predicted_gallery, truth_gallery, predicted_property, truth_property);
 }
  
+void br_eval_knn(const char *knnGraph, const char *knnTruth, const char *csv)
+{
+    EvalKNN(knnGraph, knnTruth, csv);
+}
+
 void br_finalize()
 {
     Context::finalize();
@@ -216,6 +221,11 @@ bool br_plot_metadata(int num_files, const char *files[], const char *columns, b
     return PlotMetadata(QtUtils::toStringList(num_files, files), columns, show);
 }
  
+bool br_plot_knn(int num_files, const char *files[], const char *destination, bool show)
+{
+    return PlotKNN(QtUtils::toStringList(num_files, files), destination, show);
+}
+
 float br_progress()
 {
     return Globals->progress();
@@ -64,6 +64,8 @@ BR_EXPORT float br_eval_landmarking(const char *predicted_gallery, const char *t
  
 BR_EXPORT void br_eval_regression(const char *predicted_gallery, const char *truth_gallery, const char *predicted_property = "", const char *truth_property = "");
  
+BR_EXPORT void br_eval_knn(const char *knnGraph, const char *knnTruth, const char *csv = "");
+
 BR_EXPORT void br_finalize();
  
 BR_EXPORT void br_fuse(int num_input_simmats, const char *input_simmats[],
@@ -91,6 +93,8 @@ BR_EXPORT bool br_plot_landmarking(int num_files, const char *files[], const cha
  
 BR_EXPORT bool br_plot_metadata(int num_files, const char *files[], const char *columns, bool show = false);
  
+BR_EXPORT bool br_plot_knn(int num_files, const char *files[], const char *destination, bool show = false);
+
 BR_EXPORT float br_progress();
  
 BR_EXPORT void br_read_pipe(const char *pipe, int *argc, char ***argv);
+#include <opencv2/imgproc/imgproc.hpp>
 #include <dlib/image_processing/frontal_face_detector.h>
 #include <dlib/opencv.h>
  
@@ -45,14 +46,18 @@ private:
     {
         dst = src;
  
-        shape_predictor *sp = shapeResource.acquire();
+        shape_predictor *const sp = shapeResource.acquire();
  
-        cv_image<unsigned char> cimg(src.m());
+        cv::Mat cvImage = src.m();
+        if (cvImage.channels() == 3)
+            cv::cvtColor(cvImage, cvImage, CV_BGR2GRAY);
+
+        cv_image<unsigned char> cimg(cvImage);
         array2d<unsigned char> image;
         assign_image(image,cimg);
  
         if (src.file.rects().isEmpty()) { //If the image has no rects assume the whole image is a face
-            rectangle r(0, 0, src.m().cols, src.m().rows);
+            rectangle r(0, 0, cvImage.cols, cvImage.rows);
             full_object_detection shape = (*sp)(image, r);
             for (size_t i = 0; i < shape.num_parts(); i++)
                 dst.file.appendPoint(QPointF(shape.part(i)(0),shape.part(i)(1)));
@@ -31,9 +31,9 @@ class AlgorithmsInitializer : public Initializer
     void initialize() const
     {
         // Face
-        Globals->abbreviations.insert("FaceRecognition", "FaceRecognition_1_0");
+        Globals->abbreviations.insert("FaceRecognition", "FaceRecognition_1_1");
  
-        Globals->abbreviations.insert("FaceRecognition_1_0", "FR_Detect+(FR_Eyes+FR_Represent)/(FR_Eyebrows+FR_Represent)/(FR_Mouth+FR_Represent)/(FR_Nose+FR_Represent)/(FR_Face+FR_Represent+ScaleMat(2.0))+Cat+LDA(768)+Normalize(L2):Dist(L2)");
+        Globals->abbreviations.insert("FaceRecognition_1_1", "FR_Detect+(FR_Eyes+FR_Represent)/(FR_Eyebrows+FR_Represent)/(FR_Mouth+FR_Represent)/(FR_Nose+FR_Represent)/(FR_Face+FR_Represent+ScaleMat(2.0))+Cat+LDA(768)+Normalize(L2):Unit(Dist(L2))");
         Globals->abbreviations.insert("FR_Eyes", "(CropFromLandmarks([30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47],paddingVertical=.8,paddingHorizontal=.2)+Resize(24,48))");
         Globals->abbreviations.insert("FR_Eyebrows", "(CropFromLandmarks([16,17,18,19,20,21,22,23,24,25,26,27],paddingVertical=.8,paddingHorizontal=.2)+Resize(24,48))");
         Globals->abbreviations.insert("FR_Mouth", "(CropFromLandmarks([59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76])+Resize(24,48))");
+#include "openbr/plugins/openbr_internal.h"
+#include "openbr/core/qtutils.h"
+
+using namespace br;
+
+/*!
+ * \ingroup galleries
+ * \brief Ground truth format for evaluating kNN
+ * \author Josh Klontz \cite jklontz
+ */
+class gtGallery : public Gallery
+{
+    Q_OBJECT
+
+    TemplateList templates;
+
+    ~gtGallery()
+    {
+        const QList<int> labels = File::get<int>(TemplateList::relabel(templates, "Label", false), "Label");
+
+        QStringList lines;
+        for (int i=0; i<labels.size(); i++) {
+            QStringList words;
+            for (int j=0; j<labels.size(); j++) {
+                if ((labels[i] == labels[j]) && (i != j))
+                    words.append(QString::number(j));
+            }
+            lines.append(words.join("\t"));
+        }
+
+        QtUtils::writeFile(file.name, lines);
+    }
+
+    TemplateList readBlock(bool *done)
+    {
+        *done = true;
+        qFatal("Not supported!");
+        return TemplateList();
+    }
+
+    void write(const Template &t)
+    {
+        templates.append(t);
+    }
+};
+
+BR_REGISTER(Gallery, gtGallery)
+
+#include "gt.moc"
@@ -59,6 +59,7 @@ public:
         fps = 0.f;
         time_base = 0.f;
         idx = 0;
+        idxOffset = -1;
     }
  
     ~keyframesGallery()
@@ -95,6 +96,9 @@ public:
         frame = av_frame_alloc();
         cvt_frame = av_frame_alloc();
  
+        av_init_packet(&packet);
+        packet.data = NULL;
+        packet.size = 0;
         // Get fps, stream time_base and allocate space for frame buffer with av_malloc.
         fps = (float)avFormatCtx->streams[streamID]->avg_frame_rate.num /
               (float)avFormatCtx->streams[streamID]->avg_frame_rate.den;
@@ -128,26 +132,50 @@ public:
         Template output;
         output.file = file;
  
-        AVPacket packet;
-        av_init_packet(&packet);
+
         int ret = 0;
         while (!ret) {
             if (av_read_frame(avFormatCtx, &packet) >= 0) {
                 if (packet.stream_index == streamID) {
                     avcodec_decode_video2(avCodecCtx, frame, &ret, &packet);
-                    idx = packet.dts; // decompression timestamp
+                    // Use presentation timestamp if available
+                    // Otherwise decode timestamp
+                    if (frame->pkt_pts != AV_NOPTS_VALUE)
+                        idx = frame->pkt_pts;
+                    else
+                        idx = frame->pkt_dts;
+
                     av_free_packet(&packet);
                 } else {
                     av_free_packet(&packet);
                 }
             } else {
-                av_free_packet(&packet);
-                release();
-                *done = true;
-                return TemplateList();
+                AVPacket empty_packet;
+                av_init_packet(&empty_packet);
+                empty_packet.data = NULL;
+                empty_packet.size = 0;
+
+                avcodec_decode_video2(avCodecCtx, frame, &ret, &empty_packet);
+                if (frame->pkt_pts != AV_NOPTS_VALUE)
+                    idx = frame->pkt_pts;
+                else if (frame->pkt_dts != AV_NOPTS_VALUE)
+                    idx = frame->pkt_dts;
+                else // invalid frame
+                    ret = 0;
+
+                if (!ret) {
+                    av_free_packet(&packet);
+                    av_free_packet(&empty_packet);
+                    release();
+                    *done = true;
+                    return TemplateList();
+                }
             }
         }
  
+        if (idxOffset < 0) {
+            idxOffset = idx;
+        }
         // Convert from native format
         sws_scale(avSwsCtx,
                   frame->data,
@@ -164,9 +192,9 @@ public:
             avcodec_flush_buffers(avCodecCtx);
  
             QString URL = file.get<QString>("URL", file.name);
-            output.file.set("URL", URL + "#t=" + QString::number((int)(idx * time_base)) + "s");
-            output.file.set("timestamp", QString::number((int)(idx * time_base * 1000)));
-            output.file.set("frame", QString::number(idx * time_base * fps));
+            output.file.set("URL", URL + "#t=" + QString::number((int)((idx-idxOffset) * time_base)) + "s");
+            output.file.set("timestamp", QString::number((int)((idx-idxOffset) * time_base * 1000)));
+            output.file.set("frame", QString::number((idx-idxOffset) * time_base * fps));
             TemplateList dst;
             dst.append(output);
             return dst;
@@ -204,7 +232,10 @@ protected:
     AVCodec *avCodec;
     AVFrame *frame;
     AVFrame *cvt_frame;
+    AVPacket packet;
     uint8_t *buffer;
+
+    int64_t idxOffset;
     bool opened;
     int streamID;
     float fps;
+#include <openbr/plugins/openbr_internal.h>
+#include <openbr/core/common.h>
+#include <openbr/core/opencvutils.h>
+#include <openbr/core/eval.h>
+
+namespace br
+{
+
+/*!
+ * \ingroup outputs
+ * \brief Outputs the k-Nearest Neighbors from the gallery for each probe.
+ * \author Ben Klein \cite bhklein
+ */
+class knnOutput : public MatrixOutput
+{
+    Q_OBJECT
+
+    ~knnOutput()
+    {
+        size_t num_probes = (size_t)queryFiles.size();
+        if (targetFiles.isEmpty() || queryFiles.isEmpty()) return;
+        size_t k = file.get<size_t>("k", 20);
+
+        if ((size_t)targetFiles.size() < k)
+            qFatal("Gallery size %s is smaller than k = %s.", qPrintable(QString::number(targetFiles.size())), qPrintable(QString::number(k)));
+
+        QFile f(file);
+        if (!f.open(QFile::WriteOnly))
+            qFatal("Unable to open %s for writing.", qPrintable(file));
+        f.write((const char*) &num_probes, sizeof(size_t));
+        f.write((const char*) &k, sizeof(size_t));
+
+        QVector<Candidate> neighbors; neighbors.reserve(num_probes*k);
+
+        for (size_t i=0; i<num_probes; i++) {
+            typedef QPair<float,int> Pair;
+            size_t rank = 0;
+            foreach (const Pair &pair, Common::Sort(OpenCVUtils::matrixToVector<float>(data.row(i)), true)) {
+                if (QString(targetFiles[pair.second]) != QString(queryFiles[i])) {
+                    Candidate candidate((size_t)pair.second, pair.first);
+                    neighbors.push_back(candidate);
+                    if (++rank >= k) break;
+                }
+            }
+        }
+        f.write((const char*) neighbors.data(), num_probes * k * sizeof(Candidate));
+        f.close();
+    }
+};
+
+BR_REGISTER(Output, knnOutput)
+
+} // namespace br
+
+#include "output/knn.moc"
 #!/usr/bin/env python
  
-from PIL import Image
 import csv, sys, json, argparse
+from brpy.html_viz import crop_to_bb
  
 parser = argparse.ArgumentParser(description='Visualize face cluster results in an HTML page.')
 parser.add_argument('input_file', type=str, help='Results from clustering (in csv format)')
@@ -16,24 +16,15 @@ clustmap = dict()
 with open(args.input_file) as f:
     for line in csv.DictReader(f):
         c = int(line[args.cluster_key])
+        if c not in clustmap:
+            clustmap[c] = []
         x,y,width,height = [ float(line[k]) for k in ('Face_X','Face_Y','Face_Width','Face_Height') ]
         imname = '%s/%s' % (args.img_loc, line['File'])
         try:
-            img = Image.open(imname)
-            imwidth, imheight = img.size
+            html = crop_to_bb(x,y,width,height,imname,maxheight=400)
         except IOError:
             print('problem with %s' % imname)
             continue
-        ratio = maxheight / height
-        # note for future me:
-        # image is cropped with div width/height + overflow:hidden,
-        # resized with img height,
-        # and positioned with img margin
-        html = '<div style="overflow:hidden;display:inline-block;width:%ipx;height:%ipx;">' % (width*ratio, maxheight)
-        html += '<img src="%s" style="height:%ipx;margin:-%ipx 0 0 -%ipx;"/>' % (imname, imheight*ratio, y*ratio, x*ratio)
-        html += '</div>'
-        if c not in clustmap:
-            clustmap[c] = []
         clustmap[c].append(html)
  
 # browsers crash for a DOM with this many img tags,
+'''
+Some funcs to generate HTML visualizations.
+Run from the folder you intend to save the HTML page so
+the relative paths in the HTML file are correct and PIL can find the images on disk.
+Requires local images, but should be pretty easy to set up an apache server (or whatev)
+and host them as long as the relative paths remain the same on ya serva.
+'''
+
+from PIL import Image
+
+def crop_to_bb(x, y, width, height, imname, maxheight=None):
+    '''
+    Generates an HTML string that crops to a given bounding box and resizes to maxheight pixels.
+    A maxheight of None will keep the original size (default).
+    When two crops are put next to each other, they will be inline. To make each crop its own line, wrap it in a div.
+    '''
+    img = Image.open(imname)
+    imwidth, imheight = img.size
+    if not maxheight:
+        maxheight = height
+    ratio = maxheight / height
+    # note for future me:
+    # image is cropped with div width/height + overflow:hidden,
+    # resized with img height,
+    # and positioned with img margin
+    html = '<div style="overflow:hidden; display:inline-block; width:%ipx; height:%ipx;">' % (width*ratio, maxheight)
+    html += '<img src="%s" style="height:%ipx; margin:-%ipx 0 0 -%ipx;"/>' % (imname, imheight*ratio, y*ratio, x*ratio)
+    html += '</div>'
+    return html
+
+def bbs_for_image(imname, bbs, maxheight=None, colors=None):
+    '''
+    Generates an HTML string for an image with bounding boxes.
+    bbs: iterable of (x,y,width,height) bounding box tuples
+    '''
+    img = Image.open(imname)
+    imwidth, imheight = img.size
+    if not maxheight:
+        maxheight = imheight
+    ratio = maxheight/imheight
+    html = [
+            '<div style="position:relative">',
+            '<img src="%s" style="height:%ipx" />' % (imname, maxheight)
+           ]
+    if not colors:
+        colors = ['green']*len(bbs)
+    html.extend([ bb(*box, ratio=ratio, color=color) for color,box in zip(colors,bbs) ])
+    html.append('</div>')
+    return '\n'.join(html)
+
+
+def bb(x, y, width, height, ratio=1.0, color='green'):
+    '''
+    Generates an HTML string bounding box.
+    '''
+    html = '<div style="position:absolute; border:2px solid %s; color:%s; left:%ipx; top:%ipx; width:%ipx; height:%ipx;"></div>'
+    return html % (color, color, x*ratio, y*ratio, width*ratio, height*ratio)
@@ -83,6 +83,19 @@ function(install_qt_imageformats)
   endif()
 endfunction()
  
+function(install_qt_platforms)
+  if(${BR_INSTALL_DEPENDENCIES})
+    if(CMAKE_HOST_WIN32)
+      #TODO
+    elseif(CMAKE_HOST_APPLE)
+      install(FILES ${_qt5Core_install_prefix}/plugins/platforms/libqcocoa.dylib
+              DESTINATION bin/platforms)
+    else()
+      #TODO
+    endif()
+  endif()
+endfunction()
+
 # Qt Other
 function(install_qt_misc)
   if(MSVC)
@@ -217,6 +217,12 @@ formatData &lt;- function(type=&quot;eval&quot;) {
         NormLength <<- data[grep("NormLength",data$Plot),-c(1)]
         sample <<- readImageData(Sample)
         rows <<- sample[[1]]$value
+    } else if (type == "knn") {
+        # Split data into individual plots
+        IET <<- data[grep("IET",data$Plot),-c(1)]
+        IET$Y <<- as.numeric(as.character(IET$Y))
+        CMC <<- data[grep("CMC",data$Plot),-c(1)]
+        CMC$Y <<- as.numeric(as.character(CMC$Y))
     }
 }