Merge branch 'master' into mosift

Austin Van Blanton
2 parents 848463f8 950ba90b
Showing 22 changed files with 411 additions and 177 deletions
CHANGELOG.md
openbr/core/bee.cpp
openbr/core/eval.cpp
openbr/core/plot.cpp
openbr/core/resource.h
openbr/openbr.h
openbr/openbr_plugin.cpp
openbr/plugins/algorithms.cpp
openbr/plugins/cascade.cpp
openbr/plugins/eyes.cpp
openbr/plugins/frames.cpp
openbr/plugins/gallery.cpp
openbr/plugins/gui.cpp
openbr/plugins/independent.cpp
openbr/plugins/landmarks.cpp
openbr/plugins/meta.cpp
openbr/plugins/output.cpp
openbr/plugins/regions.cpp
openbr/plugins/stream.cpp
openbr/plugins/validate.cpp
 0.4.0 - ??/??/??
 ================
+* Added -evalDetection and -plotDetection for evaluating and plotting object detection accuracy (#9)
 0.3.0 - 5/22/13
 ===============
@@ -102,7 +102,7 @@ void BEE::writeSigset(const QString &amp;sigset, const br::FileList &amp;files, bool ign
                 if ((key == "Index") || (key == "Subject")) continue;
                 metadata.append(key+"=\""+QtUtils::toString(file.value(key))+"\"");
             }
-        lines.append("\t<biometric-signature name=\"" + file.get<QString>("Subject") +"\">");
+        lines.append("\t<biometric-signature name=\"" + file.get<QString>("Subject",file.fileName()) +"\">");
         lines.append("\t\t<presentation file-name=\"" + file.name + "\" " + metadata.join(" ") + "/>");
         lines.append("\t</biometric-signature>");
     }
@@ -377,7 +377,7 @@ static QStringList computeDetectionResults(const QList&lt;ResolvedDetection&gt; &amp;detec
     for (int i=0; i<keep; i++) {
         const DetectionOperatingPoint &point = points[double(i) / double(keep-1) * double(points.size()-1)];
         lines.append(QString("%1ROC, %2, %3").arg(discrete ? "Discrete" : "Continuous", QString::number(point.FalsePositives), QString::number(point.Recall)));
-        lines.append(QString("%1PR, %2, %3").arg(discrete ? "Discrete" : "Continuous", QString::number(point.Precision), QString::number(point.Recall)));
+        lines.append(QString("%1PR, %2, %3").arg(discrete ? "Discrete" : "Continuous", QString::number(point.Recall), QString::number(point.Precision)));
     }
     return lines;
 }
@@ -238,7 +238,6 @@ bool Plot(const QStringList &amp;files, const File &amp;destination, bool show)
                             (p.minor.size > 1 ? QString(" + scale_linetype_discrete(\"%1\")").arg(p.minor.header) : QString()) +
                             QString(" + scale_x_log10(labels=percent) + scale_y_log10(labels=percent) + annotation_logticks()\n\n")));
-
     p.file.write(qPrintable(QString("qplot(X, data=SD, geom=\"histogram\", fill=Y, position=\"identity\", alpha=I(1/2)") +
                             QString(", xlab=\"Score%1\"").arg((p.flip ? p.major.size : p.minor.size) > 1 ? " / " + (p.flip ? p.major.header : p.minor.header) : QString()) +
                             QString(", ylab=\"Frequency%1\"").arg((p.flip ? p.minor.size : p.major.size) > 1 ? " / " + (p.flip ? p.minor.header : p.major.header) : QString()) +
@@ -278,6 +277,39 @@ bool PlotDetection(const QStringList &amp;files, const File &amp;destination, bool show)
     RPlot p(files, destination, false);
+    p.file.write("# Split data into individual plots\n"
+                 "plot_index = which(names(data)==\"Plot\")\n"
+                 "DiscreteROC <- data[grep(\"DiscreteROC\",data$Plot),-c(1)]\n"
+                 "ContinuousROC <- data[grep(\"ContinuousROC\",data$Plot),-c(1)]\n"
+                 "DiscretePR <- data[grep(\"DiscretePR\",data$Plot),-c(1)]\n"
+                 "ContinuousPR <- data[grep(\"ContinuousPR\",data$Plot),-c(1)]\n"
+                 "Overlap <- data[grep(\"Overlap\",data$Plot),-c(1)]\n"
+                 "rm(data)\n"
+                 "\n");
+
+    foreach (const QString &type, QStringList() << "Discrete" << "Continuous")
+        p.file.write(qPrintable(QString("qplot(X, Y, data=%1ROC%2").arg(type, (p.major.smooth || p.minor.smooth) ? ", geom=\"smooth\", method=loess, level=0.99" : ", geom=\"line\"") +
+                                (p.major.size > 1 ? QString(", colour=factor(%1)").arg(p.major.header) : QString()) +
+                                (p.minor.size > 1 ? QString(", linetype=factor(%1)").arg(p.minor.header) : QString()) +
+                                QString(", xlab=\"False Accepts\", ylab=\"True Accept Rate\") + theme_minimal()") +
+                                (p.major.size > 1 ? getScale("colour", p.major.header, p.major.size) : QString()) +
+                                (p.minor.size > 1 ? QString(" + scale_linetype_discrete(\"%1\")").arg(p.minor.header) : QString()) +
+                                QString(" + scale_x_log10() + scale_y_continuous(labels=percent) + annotation_logticks(sides=\"b\") + ggtitle(\"%1\")\n\n").arg(type)));
+
+    foreach (const QString &type, QStringList() << "Discrete" << "Continuous")
+        p.file.write(qPrintable(QString("qplot(X, Y, data=%1PR%2").arg(type, (p.major.smooth || p.minor.smooth) ? ", geom=\"smooth\", method=loess, level=0.99" : ", geom=\"line\"") +
+                                (p.major.size > 1 ? QString(", colour=factor(%1)").arg(p.major.header) : QString()) +
+                                (p.minor.size > 1 ? QString(", linetype=factor(%1)").arg(p.minor.header) : QString()) +
+                                QString(", xlab=\"Recall\", ylab=\"Precision\") + theme_minimal()") +
+                                (p.major.size > 1 ? getScale("colour", p.major.header, p.major.size) : QString()) +
+                                (p.minor.size > 1 ? QString(" + scale_linetype_discrete(\"%1\")").arg(p.minor.header) : QString()) +
+                                QString(" + scale_x_continuous(labels=percent) + scale_y_continuous(labels=percent) + ggtitle(\"%1\")\n\n").arg(type)));
+
+    p.file.write(qPrintable(QString("qplot(X, data=Overlap, geom=\"histogram\", position=\"identity\", xlab=\"Overlap\", ylab=\"Frequency\")") +
+                            QString(" + theme_minimal() + scale_x_continuous(minor_breaks=NULL) + scale_y_continuous(minor_breaks=NULL) + theme(axis.text.y=element_blank(), axis.ticks=element_blank(), axis.text.x=element_text(angle=-90, hjust=0))") +
+                            (p.major.size > 1 ? (p.minor.size > 1 ? QString(" + facet_grid(%2 ~ %1, scales=\"free\")").arg(p.minor.header, p.major.header) : QString(" + facet_wrap(~ %1, scales = \"free\")").arg(p.major.header)) : QString()) +
+                            QString(" + theme(aspect.ratio=1)\n\n")));
+
     return p.finalize(show);
 }
@@ -24,6 +24,7 @@
 #include <QSharedPointer>
 #include <QString>
 #include <QThread>
+#include <openbr/openbr_plugin.h>
 template <typename T>
 class ResourceMaker
@@ -52,7 +53,7 @@ public:
         : resourceMaker(rm)
         , availableResources(new QList<T*>())
         , lock(new QMutex())
-        , totalResources(new QSemaphore(QThread::idealThreadCount()))
+        , totalResources(new QSemaphore(br::Globals->parallelism))
     {}
     ~Resource()
@@ -268,6 +268,18 @@ BR_EXPORT bool br_plot(int num_files, const char *files[], const char *destinati
 /*!
  * \brief Renders detection performance figures for a set of <tt>.csv</tt> files created by \ref br_eval_detection.
+ *
+ * In order of their output, the figures are:
+ * -# Discrete Receiver Operating Characteristic (DiscreteROC)
+ * -# Continuous Receiver Operating Characteristic (ContinuousROC)
+ * -# Discrete Precision Recall (DiscretePR)
+ * -# Continuous Precision Recall (ContinuousPR)
+ * -# Bounding Box Overlap Histogram (Overlap)
+ *
+ * Detection accuracy is measured with <i>overlap fraction = bounding box intersection / union</i>.
+ * When computing <i>discrete</i> curves, an overlap >= 0.5 is considered a true positive, otherwise it is considered a false negative.
+ * When computing <i>continuous</i> curves, true positives and false negatives are measured fractionally as <i>overlap</i> and <i>1-overlap</i> respectively.
+ *
  * \see br_plot
  */
 BR_EXPORT bool br_plot_detection(int num_files, const char *files[], const char *destination, bool show = false);
@@ -812,7 +812,7 @@ float br::Context::progress() const
 void br::Context::setProperty(const QString &key, const QString &value)
 {
-    Object::setProperty(key, value);
+    Object::setProperty(key, value.isEmpty() ? QVariant() : value);
     qDebug("Set %s%s", qPrintable(key), value.isEmpty() ? "" : qPrintable(" to " + value));
     if (key == "parallelism") {
@@ -1166,7 +1166,8 @@ void Transform::project(const TemplateList &amp;src, TemplateList &amp;dst) const
         dst.append(Template());
     QFutureSynchronizer<void> futures;
     for (int i=0; i<dst.size(); i++)
-        futures.addFuture(QtConcurrent::run(_project, this, &src[i], &dst[i]));
+        if (Globals->parallelism > 1) futures.addFuture(QtConcurrent::run(_project, this, &src[i], &dst[i]));
+        else                          _project(this, &src[i], &dst[i]);
     futures.waitForFinished();
 }
@@ -46,7 +46,7 @@ class AlgorithmsInitializer : public Initializer
         Globals->abbreviations.insert("CropFace", "Open+Cvt(Gray)+Cascade(FrontalFace)+ASEFEyes+Affine(128,128,0.25,0.35)");
         // Video
-        Globals->abbreviations.insert("DisplayVideo", "Stream([Show(false,[FrameNumber])+Discard])");
+        Globals->abbreviations.insert("DisplayVideo", "Stream([FPSLimit(30)+Show(false,[FrameNumber])+Discard])");
         Globals->abbreviations.insert("PerFrameDetection", "Stream([SaveMat(original)+Cvt(Gray)+Cascade(FrontalFace)+ASEFEyes+RestoreMat(original)+Draw(inPlace=true),Show(false,[FrameNumber])+Discard])");
         Globals->abbreviations.insert("AgeGenderDemo", "Stream([SaveMat(original)+Cvt(Gray)+Cascade(FrontalFace)+Expand+<FaceClassificationRegistration>+<FaceClassificationExtraction>+(<AgeRegressor>+Rename(Subject,Age)+Discard)/(<GenderClassifier>+Rename(Subject,Gender)+Discard)+RestoreMat(original)+Draw(inPlace=true)+DrawPropertiesPoint([Age,Gender],Affine_0,inPlace=true)+SaveMat(original)+Discard+Contract,RestoreMat(original)+FPSCalc+Show(false,[AvgFPS,Age,Gender])+Discard])");
         Globals->abbreviations.insert("BoVW", "Flatten+CatRows+KMeans(500)+Hist(500)");
@@ -49,19 +49,20 @@ private:
     }
 };
-
 /*!
  * \ingroup transforms
  * \brief Wraps OpenCV cascade classifier
  * \author Josh Klontz \cite jklontz
  */
-class CascadeTransform : public UntrainableTransform
+class CascadeTransform : public UntrainableMetaTransform
 {
     Q_OBJECT
     Q_PROPERTY(QString model READ get_model WRITE set_model RESET reset_model STORED false)
     Q_PROPERTY(int minSize READ get_minSize WRITE set_minSize RESET reset_minSize STORED false)
+    Q_PROPERTY(bool ROCMode READ get_ROCMode WRITE set_ROCMode RESET reset_ROCMode STORED false)
     BR_PROPERTY(QString, model, "FrontalFace")
     BR_PROPERTY(int, minSize, 64)
+    BR_PROPERTY(bool, ROCMode, false)
     Resource<CascadeClassifier> cascadeResource;
@@ -72,18 +73,54 @@ class CascadeTransform : public UntrainableTransform
     void project(const Template &src, Template &dst) const
     {
+        TemplateList temp;
+        project(TemplateList() << src, temp);
+        if (!temp.isEmpty()) dst = temp.first();
+    }
+
+    void project(const TemplateList &src, TemplateList &dst) const
+    {
         CascadeClassifier *cascade = cascadeResource.acquire();
-        vector<Rect> rects;
-        cascade->detectMultiScale(src, rects, 1.2, 5, src.file.get<bool>("enrollAll", false) ? 0 : CV_HAAR_FIND_BIGGEST_OBJECT, Size(minSize, minSize));
+        foreach (const Template &t, src) {
+            const bool enrollAll = t.file.getBool("enrollAll");
+
+            for (int i=0; i<t.size(); i++) {
+                const Mat &m = t[i];
+                vector<Rect> rects;
+                vector<int> rejectLevels;
+                vector<double> levelWeights;
+                if (ROCMode) cascade->detectMultiScale(m, rects, rejectLevels, levelWeights, 1.2, 5, (enrollAll ? 0 : CV_HAAR_FIND_BIGGEST_OBJECT) | CV_HAAR_SCALE_IMAGE, Size(minSize, minSize), Size(), true);
+                else         cascade->detectMultiScale(m, rects, 1.2, 5, enrollAll ? 0 : CV_HAAR_FIND_BIGGEST_OBJECT, Size(minSize, minSize));
+
+                if (!enrollAll && rects.empty())
+                    rects.push_back(Rect(0, 0, m.cols, m.rows));
+
+                for (size_t j=0; j<rects.size(); j++) {
+                    Template u(t.file, m);
+                    if (rejectLevels.size() > j)
+                        u.file.set("Confidence", rejectLevels[j]*1000.0 + levelWeights[j]*1.0);
+                    const QRectF rect = OpenCVUtils::fromRect(rects[j]);
+                    u.file.appendRect(rect);
+                    u.file.set(model, rect);
+                    dst.append(u);
+                }
+            }
+        }
+
         cascadeResource.release(cascade);
+    }
-        if (!src.file.get<bool>("enrollAll", false) && rects.empty())
-            rects.push_back(Rect(0, 0, src.m().cols, src.m().rows));
+    // TODO: Remove this code when ready to break binary compatibility
+    void store(QDataStream &stream) const
+    {
+        int size = 1;
+        stream << size;
+    }
-        foreach (const Rect &rect, rects) {
-            dst += src;
-            dst.file.appendRect(OpenCVUtils::fromRect(rect));
-        }
+    void load(QDataStream &stream)
+    {
+        int size;
+        stream >> size;
     }
 };
@@ -186,7 +186,6 @@ private:
         dst.file.appendPoint(QPointF(second_eye_x, second_eye_y));
         dst.file.set("First_Eye", QPointF(first_eye_x, first_eye_y));
         dst.file.set("Second_Eye", QPointF(second_eye_x, second_eye_y));
-        dst.file.set("Face", QRect(roi.x, roi.y, roi.width, roi.height));
     }
 };
@@ -18,6 +18,10 @@ class AggregateFrames : public TimeVaryingTransform
     TemplateList buffer;
+public:
+    AggregateFrames() : TimeVaryingTransform(false) {}
+
+private:
     void train(const TemplateList &data)
     {
         (void) data;
@@ -165,10 +165,13 @@ class EmptyGallery : public Gallery
             templates.append(File(fileName, dir.dirName()));
         if (!regexp.isEmpty()) {
-            const QRegularExpression re(regexp);
-            for (int i=templates.size()-1; i>=0; i--)
-                if (!re.match(templates[i].file.suffix()).hasMatch())
+            QRegExp re(regexp);
+            re.setPatternSyntax(QRegExp::Wildcard);
+            for (int i=templates.size()-1; i>=0; i--) {
+                if (!re.exactMatch(templates[i].file.fileName())) {
                     templates.removeAt(i);
+                }
+            }
         }
         return templates;
@@ -212,8 +212,11 @@ public:
         foreach (const Template & t, src) {
             // build label
             QString newTitle;
+
             foreach (const QString & s, keys) {
-                if (t.file.contains(s)) {
+                if (s.compare("name", Qt::CaseInsensitive) == 0) {
+                    newTitle = newTitle + s + ": " + t.file.fileName() + " ";
+                } else if (t.file.contains(s)) {
                     QString out = t.file.get<QString>(s);
                     newTitle = newTitle + s + ": " + out + " ";
                 }
@@ -99,6 +99,8 @@ class IndependentTransform : public MetaTransform
         return independentTransform;
     }
+    bool timeVarying() const { return transform->timeVarying(); }
+
     static void _train(Transform *transform, const TemplateList *data)
     {
         transform->train(*data);
@@ -143,6 +145,27 @@ class IndependentTransform : public MetaTransform
         dst.append(mats);
     }
+    void projectUpdate(const Template &src, Template &dst)
+    {
+        dst.file = src.file;
+        QList<Mat> mats;
+        for (int i=0; i<src.size(); i++) {
+            transforms[i%transforms.size()]->projectUpdate(Template(src.file, src[i]), dst);
+            mats.append(dst);
+            dst.clear();
+        }
+        dst.append(mats);
+    }
+
+    void projectUpdate(const TemplateList &src, TemplateList &dst)
+    {
+        dst.reserve(src.size());
+        foreach (const Template &t, src) {
+            dst.append(Template());
+            projectUpdate(t, dst.last());
+        }
+    }
+
     void store(QDataStream &stream) const
     {
         const int size = transforms.size();
@@ -230,6 +230,8 @@ class DelaunayTransform : public UntrainableTransform
             QList<Point2f> mappedPoints;
+            dst.file = src.file;
+
             for (int i = 0; i < validTriangles.size(); i++) {
                 Eigen::MatrixXf srcMat(validTriangles[i].size(), 2);
@@ -272,8 +274,8 @@ class DelaunayTransform : public UntrainableTransform
                     bitwise_and(dst.m(),mask,overlap);
                     for (int j = 0; j < overlap.rows; j++) {
                         for (int k = 0; k < overlap.cols; k++) {
-                            if (overlap.at<uchar>(k,j) != 0) {
-                                mask.at<uchar>(k,j) = 0;
+                            if (overlap.at<uchar>(j,k) != 0) {
+                                mask.at<uchar>(j,k) = 0;
                             }
                         }
                     }
@@ -281,11 +283,13 @@ class DelaunayTransform : public UntrainableTransform
                 bitwise_and(buffer,mask,output);
+
                 dst.m() += output;
             }
+            // Overwrite any rects
             Rect boundingBox = boundingRect(mappedPoints.toVector().toStdVector());
-            dst.file.appendRect(OpenCVUtils::fromRect(boundingBox));
+            dst.file.setRects(QList<QRectF>() << OpenCVUtils::fromRect(boundingBox));
         }
     }
@@ -645,17 +645,28 @@ public:
         QList<TemplateList> input_buffer;
         input_buffer.reserve(src.size());
+        QFutureSynchronizer<void> futures;
+
         for (int i =0; i < src.size();i++) {
             input_buffer.append(TemplateList());
             output_buffer.append(TemplateList());
         }
-
-        QFutureSynchronizer<void> futures;
+        QList<QFuture<void> > temp;
+        temp.reserve(src.size());
         for (int i=0; i<src.size(); i++) {
             input_buffer[i].append(src[i]);
-            if (Globals->parallelism) futures.addFuture(QtConcurrent::run(_projectList, transform, &input_buffer[i], &output_buffer[i]));
-            else                                                          _projectList( transform, &input_buffer[i], &output_buffer[i]);
+
+            if (Globals->parallelism > 1) temp.append(QtConcurrent::run(_projectList, transform, &input_buffer[i], &output_buffer[i]));
+            else _projectList(transform, &input_buffer[i], &output_buffer[i]);
+        }
+        // We add the futures in reverse order, since in Qt 5.1 at least the
+        // waiting thread will wait on them in the order added (which for uniform priority
+        // threads is the order of execution), and we want the waiting thread to go in the opposite order
+        // so that it can steal runnables and do something besides wait.
+        for (int i = temp.size() - 1; i >= 0; i--) {
+            futures.addFuture(temp[i]);
         }
+
         futures.waitForFinished();
         for (int i=0; i<src.size(); i++) dst.append(output_buffer[i]);
@@ -98,7 +98,7 @@ class heatOutput : public MatrixOutput
 {
     Q_OBJECT
     Q_PROPERTY(int patches READ get_patches WRITE set_patches RESET reset_patches STORED false)
-    BR_PROPERTY(int, patches, -1);
+    BR_PROPERTY(int, patches, -1)
     ~heatOutput()
     {
@@ -426,20 +426,26 @@ class rankOutput : public MatrixOutput
             typedef QPair<float,int> Pair;
             int rank = 1;
             foreach (const Pair &pair, Common::Sort(OpenCVUtils::matrixToVector<float>(data.row(i)), true)) {
-                if (targetFiles[pair.second].get<QString>("Subject") == queryFiles[i].get<QString>("Subject")) {
-                    ranks.append(rank);
-                    positions.append(pair.second);
-                    scores.append(pair.first);
-                    break;
+                if (Globals->crossValidate > 0 ? (targetFiles[pair.second].get<int>("Partition",-1) == queryFiles[i].get<int>("Partition",-1)) : true) {
+                    if (QString(targetFiles[pair.second]) != QString(queryFiles[i])) {
+                        if (targetFiles[pair.second].get<QString>("Subject") == queryFiles[i].get<QString>("Subject")) {
+                            ranks.append(rank);
+                            positions.append(pair.second);
+                            scores.append(pair.first);
+                            break;
+                        }
+                        rank++;
+                    }
                 }
-                rank++;
             }
         }
         typedef QPair<int,int> RankPair;
         foreach (const RankPair &pair, Common::Sort(ranks, false))
+            // pair.first == rank retrieved, pair.second == original position
             lines.append(queryFiles[pair.second].name + " " + QString::number(pair.first) + " " + QString::number(scores[pair.second]) + " " + targetFiles[positions[pair.second]].name);
+
         QtUtils::writeFile(file, lines);
     }
 };
@@ -252,16 +252,12 @@ class CropRectTransform : public UntrainableTransform
     void project(const Template &src, Template &dst) const
     {
         dst = src;
-        QList<QRectF> rects = dst.file.rects();
+        QList<QRectF> rects = src.file.rects();
         for (int i=0;i < rects.size(); i++) {
-            QRectF rect = rects[i];
-
-            rect.setX(rect.x() + rect.width() * QtUtils::toPoint(widthCrop).x());
-            rect.setY(rect.y() + rect.height() * QtUtils::toPoint(heightCrop).x());
-            rect.setWidth(rect.width() * (1-QtUtils::toPoint(widthCrop).y()));
-            rect.setHeight(rect.height() * (1-QtUtils::toPoint(heightCrop).y()));
-
-            dst.m() = Mat(dst.m(), OpenCVUtils::toRect(rect));
+            rects[i].setX(rects[i].x() + rects[i].width() * QtUtils::toPoint(widthCrop).x());
+            rects[i].setY(rects[i].y() + rects[i].height() * QtUtils::toPoint(heightCrop).x());
+            rects[i].setWidth(rects[i].width() * (1-QtUtils::toPoint(widthCrop).y()));
+            rects[i].setHeight(rects[i].height() * (1-QtUtils::toPoint(heightCrop).y()));
         }
         dst.file.setRects(rects);
     }
@@ -160,9 +160,8 @@ class DataSource
 public:
     DataSource(int maxFrames=500)
     {
+        // The sequence number of the last frame
         final_frame = -1;
-        last_issued = -2;
-        last_received = -3;
         for (int i=0; i < maxFrames;i++)
         {
             allFrames.addItem(new FrameData());
@@ -181,51 +180,64 @@ public:
     }
     // non-blocking version of getFrame
-    FrameData * tryGetFrame()
+    // Returns a NULL FrameData if too many frames are out, or the
+    // data source is broken. Sets last_frame to true iff the FrameData
+    // returned is the last valid frame, and the data source is now broken.
+    FrameData * tryGetFrame(bool & last_frame)
     {
+        last_frame = false;
+
+        if (is_broken) {
+            return NULL;
+        }
+
+        // Try to get a FrameData from the pool, if we can't it means too many
+        // frames are already out, and we will return NULL to indicate failure
         FrameData * aFrame = allFrames.tryGetItem();
-        if (aFrame == NULL)
+        if (aFrame == NULL) {
             return NULL;
+        }
         aFrame->data.clear();
         aFrame->sequenceNumber = -1;
+        // Try to read a frame, if this returns false the data source is broken
         bool res = getNext(*aFrame);
-        // The datasource broke.
-        if (!res) {
+        // The datasource broke, update final_frame
+        if (!res)
+        {
+            QMutexLocker lock(&last_frame_update);
+            final_frame = lookAhead.back()->sequenceNumber;
             allFrames.addItem(aFrame);
+        }
+        else lookAhead.push_back(aFrame);
-            QMutexLocker lock(&last_frame_update);
-            // Did we already receive the last frame?
-            final_frame = last_issued;
+        FrameData * rVal = lookAhead.first();
+        lookAhead.pop_front();
-            // We got the last frame before the data source broke,
-            // better pulse lastReturned
-            if (final_frame == last_received) {
-                lastReturned.wakeAll();
-            }
-            else if (final_frame < last_received)
-                std::cout << "Bad last frame " << final_frame << " but received " << last_received << std::endl;
-            return NULL;
+        if (rVal->sequenceNumber == final_frame) {
+            last_frame = true;
+            is_broken = true;
         }
-        last_issued = aFrame->sequenceNumber;
-        return aFrame;
+
+        return rVal;
     }
-    // Returns true if the frame returned was the last
+    // Return a frame to the pool, returns true if the frame returned was the last
     // frame issued, false otherwise
     bool returnFrame(FrameData * inputFrame)
     {
+        int frameNumber = inputFrame->sequenceNumber;
+
         allFrames.addItem(inputFrame);
         bool rval = false;
         QMutexLocker lock(&last_frame_update);
-        last_received = inputFrame->sequenceNumber;
-        if (inputFrame->sequenceNumber == final_frame) {
+        if (frameNumber == final_frame) {
             // We just received the last frame, better pulse
             lastReturned.wakeAll();
             rval = true;
@@ -240,17 +252,57 @@ public:
         lastReturned.wait(&last_frame_update);
     }
-    virtual void close() = 0;
-    virtual bool open(Template & output, int start_index=0) = 0;
-    virtual bool isOpen() = 0;
+    bool open(Template & output, int start_index = 0)
+    {
+        is_broken = false;
+        // The last frame isn't initialized yet
+        final_frame = -1;
+        // Start our sequence numbers from the input index
+        next_sequence_number = start_index;
+
+        // Actually open the data source
+        bool open_res = concreteOpen(output);
+
+        // We couldn't open the data source
+        if (!open_res) {
+            is_broken = true;
+            return false;
+        }
+
+        // Try to get a frame from the global pool
+        FrameData * firstFrame = allFrames.tryGetItem();
+
+        // If this fails, things have gone pretty badly.
+        if (firstFrame == NULL) {
+            is_broken = true;
+            return false;
+        }
+
+        // Read a frame from the video source
+        bool res = getNext(*firstFrame);
+
+        // the data source broke already, we couldn't even get one frame
+        // from it.
+        if (!res) {
+            is_broken = true;
+            return false;
+        }
+        lookAhead.append(firstFrame);
+        return true;
+    }
+
+    virtual bool isOpen()=0;
+    virtual bool concreteOpen(Template & output) = 0;
     virtual bool getNext(FrameData & input) = 0;
+    virtual void close() = 0;
+    int next_sequence_number;
 protected:
     DoubleBuffer allFrames;
     int final_frame;
-    int last_issued;
-    int last_received;
+    bool is_broken;
+    QList<FrameData *> lookAhead;
     QWaitCondition lastReturned;
     QMutex last_frame_update;
@@ -262,13 +314,8 @@ class VideoDataSource : public DataSource
 public:
     VideoDataSource(int maxFrames) : DataSource(maxFrames) {}
-    bool open(Template &input, int start_index=0)
+    bool concreteOpen(Template &input)
     {
-        final_frame = -1;
-        last_issued = -2;
-        last_received = -3;
-
-        next_idx = start_index;
         basis = input;
         bool is_int = false;
         int anInt = input.file.name.toInt(&is_int);
@@ -306,25 +353,31 @@ private:
             return false;
         output.data.append(Template(basis.file));
-        output.data.last().append(cv::Mat());
+        output.data.last().m() = cv::Mat();
-        output.sequenceNumber = next_idx;
-        next_idx++;
+        output.sequenceNumber = next_sequence_number;
+        next_sequence_number++;
-        bool res = video.read(output.data.last().last());
-        output.data.last().last() = output.data.last().last().clone();
+        cv::Mat temp;
+        bool res = video.read(temp);
         if (!res) {
+            output.data.last().m() = cv::Mat();
             close();
             return false;
         }
+
+        // This clone is critical, if we don't do it then the matrix will
+        // be an alias of an internal buffer of the video source, leading
+        // to various problems later.
+        output.data.last().m() = temp.clone();
+
         output.data.last().file.set("FrameNumber", output.sequenceNumber);
         return true;
     }
     cv::VideoCapture video;
     Template basis;
-    int next_idx;
 };
 // Given a template as input, return its matrices one by one on subsequent calls
@@ -334,21 +387,16 @@ class TemplateDataSource : public DataSource
 public:
     TemplateDataSource(int maxFrames) : DataSource(maxFrames)
     {
-        current_idx = INT_MAX;
+        current_matrix_idx = INT_MAX;
         data_ok = false;
     }
-    bool data_ok;
-    bool open(Template &input, int start_index=0)
+    bool concreteOpen(Template &input)
     {
         basis = input;
-        current_idx = 0;
-        next_sequence = start_index;
-        final_frame = -1;
-        last_issued = -2;
-        last_received = -3;
+        current_matrix_idx = 0;
-        data_ok = current_idx < basis.size();
+        data_ok = current_matrix_idx < basis.size();
         return data_ok;
     }
@@ -358,39 +406,41 @@ public:
     void close()
     {
-        current_idx = INT_MAX;
+        current_matrix_idx = INT_MAX;
         basis.clear();
     }
 private:
     bool getNext(FrameData & output)
     {
-        data_ok = current_idx < basis.size();
+        data_ok = current_matrix_idx < basis.size();
         if (!data_ok)
             return false;
-        output.data.append(basis[current_idx]);
-        current_idx++;
+        output.data.append(basis[current_matrix_idx]);
+        current_matrix_idx++;
-        output.sequenceNumber = next_sequence;
-        next_sequence++;
+        output.sequenceNumber = next_sequence_number;
+        next_sequence_number++;
         output.data.last().file.set("FrameNumber", output.sequenceNumber);
         return true;
     }
     Template basis;
-    int current_idx;
-    int next_sequence;
+    // Index of the next matrix to output from the template
+    int current_matrix_idx;
+
+    // is current_matrix_idx in bounds?
+    bool data_ok;
 };
-// Given a template as input, create a VideoDataSource or a TemplateDataSource
-// depending on whether or not it looks like the input template has already
-// loaded frames into memory.
+// Given a templatelist as input, create appropriate data source for each
+// individual template
 class DataSourceManager : public DataSource
 {
 public:
-    DataSourceManager()
+    DataSourceManager() : DataSource(500)
     {
         actualSource = NULL;
     }
@@ -411,29 +461,25 @@ public:
     bool open(TemplateList & input)
     {
-        currentIdx = 0;
+        current_template_idx = 0;
         templates = input;
-        return open(templates[currentIdx]);
+        return DataSource::open(templates[current_template_idx]);
     }
-    bool open(Template & input, int start_index=0)
+    bool concreteOpen(Template & input)
     {
         close();
-        final_frame = -1;
-        last_issued = -2;
-        last_received = -3;
-        next_frame = start_index;
         // Input has no matrices? Its probably a video that hasn't been loaded yet
         if (input.empty()) {
             actualSource = new VideoDataSource(0);
-            actualSource->open(input, next_frame);
+            actualSource->concreteOpen(input);
         }
         else {
             // create frame dealer
             actualSource = new TemplateDataSource(0);
-            actualSource->open(input, next_frame);
+            actualSource->concreteOpen(input);
         }
         if (!isOpen()) {
             delete actualSource;
@@ -446,30 +492,47 @@ public:
     bool isOpen() { return !actualSource ? false : actualSource->isOpen(); }
 protected:
-    int currentIdx;
-    int next_frame;
+    // Index of the template in the templatelist we are currently reading from
+    int current_template_idx;
+
     TemplateList templates;
     DataSource * actualSource;
     bool getNext(FrameData & output)
     {
         bool res = actualSource->getNext(output);
+        output.sequenceNumber = next_sequence_number;
+
         if (res) {
-            next_frame = output.sequenceNumber+1;
+            output.data.last().file.set("FrameNumber", output.sequenceNumber);
+            next_sequence_number++;
+            if (output.data.last().last().empty())
+                qDebug("broken matrix");
             return true;
         }
+
         while(!res) {
-            currentIdx++;
+            output.data.clear();
+            current_template_idx++;
-            if (currentIdx >= templates.size())
+            // No more templates? We're done
+            if (current_template_idx >= templates.size())
                 return false;
-            bool open_res = open(templates[currentIdx], next_frame);
+
+            // open the next data source
+            bool open_res = concreteOpen(templates[current_template_idx]);
             if (!open_res)
                 return false;
+
+            // get a frame from it
             res = actualSource->getNext(output);
         }
+        output.sequenceNumber = next_sequence_number++;
+        output.data.last().file.set("FrameNumber", output.sequenceNumber);
+
+        if (output.data.last().last().empty())
+            qDebug("broken matrix");
-        next_frame = output.sequenceNumber+1;
         return res;
     }
@@ -477,9 +540,14 @@ protected:
 class ProcessingStage;
-class BasicLoop : public QRunnable
+class BasicLoop : public QRunnable, public QFutureInterface<void>
 {
 public:
+    BasicLoop()
+    {
+        this->reportStarted();
+    }
+
     void run();
     QList<ProcessingStage *> * stages;
@@ -505,13 +573,13 @@ public:
     int stage_id;
     virtual void reset()=0;
-
 protected:
     int thread_count;
     SharedBuffer * inputBuffer;
     ProcessingStage * nextStage;
     QList<ProcessingStage *> * stages;
+    QThreadPool * threads;
     Transform * transform;
 };
@@ -530,6 +598,7 @@ void BasicLoop::run()
         current_idx++;
         current_idx = current_idx % stages->size();
     }
+    this->reportFinished();
 }
 class MultiThreadStage : public ProcessingStage
@@ -564,7 +633,6 @@ public:
     }
 };
-
 class SingleThreadStage : public ProcessingStage
 {
 public:
@@ -627,18 +695,20 @@ public:
         lock.unlock();
         if (newItem)
-        {
-            BasicLoop * next = new BasicLoop();
-            next->stages = stages;
-            next->start_idx = this->stage_id;
-            next->startItem = newItem;
-
-            QThreadPool::globalInstance()->start(next, stages->size() - this->stage_id);
-        }
+            startThread(newItem);
         return input;
     }
+    void startThread(br::FrameData * newItem)
+    {
+        BasicLoop * next = new BasicLoop();
+        next->stages = stages;
+        next->start_idx = this->stage_id;
+        next->startItem = newItem;
+        this->threads->start(next, stages->size() - stage_id);
+    }
+
     // Calledfrom a different thread than run.
     bool tryAcquireNextStage(FrameData *& input)
@@ -674,7 +744,7 @@ public:
 };
 // No input buffer, instead we draw templates from some data source
-// Will be operated by the main thread for the stream
+// Will be operated by the main thread for the stream. starts threads
 class FirstStage : public SingleThreadStage
 {
 public:
@@ -684,44 +754,51 @@ public:
     FrameData * run(FrameData * input, bool & should_continue)
     {
-        // Is there anything on our input buffer? If so we should start a thread with that.
+        // Try to get a frame from the datasource
         QWriteLocker lock(&statusLock);
-        input = dataSource.tryGetFrame();
-        // Datasource broke?
-        if (!input)
+        bool last_frame = false;
+        input = dataSource.tryGetFrame(last_frame);
+
+        // Datasource broke, or is currently out of frames?
+        if (!input || last_frame)
         {
+            // We will just stop and not continue.
             currentStatus = STOPPING;
-            should_continue = false;
-            return NULL;
+            if (!input) {
+                should_continue = false;
+                return NULL;
+            }
         }
         lock.unlock();
-
+        // Can we enter the next stage?
         should_continue = nextStage->tryAcquireNextStage(input);
-        BasicLoop * next = new BasicLoop();
-        next->stages = stages;
-        next->start_idx = this->stage_id;
-        next->startItem = NULL;
-
-        QThreadPool::globalInstance()->start(next, stages->size() - this->stage_id);
+        // We are exiting leaving this stage, should we start another
+        // thread here? Normally we will always re-queue a thread on
+        // the first stage, but if we received the last frame there is
+        // no need to.
+        if (!last_frame) {
+            startThread(NULL);
+        }
         return input;
     }
-    // Calledfrom a different thread than run.
+    // The last stage, trying to access the first stage
     bool tryAcquireNextStage(FrameData *& input)
     {
+        // Return the frame, was it the last one?
         bool was_last = dataSource.returnFrame(input);
         input = NULL;
+
+        // OK we won't continue.
         if (was_last) {
             return false;
         }
-        if (!dataSource.isOpen())
-            return false;
-
         QReadLocker lock(&statusLock);
-        // Thread is already running, we should just return
+        // A thread is already in the first stage,
+        // we should just return
         if (currentStatus == STARTING)
         {
             return false;
@@ -744,6 +821,7 @@ public:
 };
+// starts threads
 class LastStage : public SingleThreadStage
 {
 public:
@@ -774,11 +852,14 @@ public:
         }
         next_target = input->sequenceNumber + 1;
+        // add the item to our output buffer
         collectedOutput.append(input->data);
+        // Can we enter the read stage?
         should_continue = nextStage->tryAcquireNextStage(input);
-        // Is there anything on our input buffer? If so we should start a thread with that.
+        // Is there anything on our input buffer? If so we should start a thread
+        // in this stage to process that frame.
         QWriteLocker lock(&statusLock);
         FrameData * newItem = inputBuffer->tryGetItem();
         if (!newItem)
@@ -788,23 +869,18 @@ public:
         lock.unlock();
         if (newItem)
-        {
-            BasicLoop * next = new BasicLoop();
-            next->stages = stages;
-            next->start_idx = this->stage_id;
-            next->startItem = newItem;
-
-            QThreadPool::globalInstance()->start(next, stages->size() - this->stage_id);
-        }
+            startThread(newItem);
         return input;
     }
 };
+
 class StreamTransform : public CompositeTransform
 {
     Q_OBJECT
 public:
+
     void train(const TemplateList & data)
     {
         foreach(Transform * transform, transforms) {
@@ -834,21 +910,17 @@ public:
         bool res = readStage->dataSource.open(dst);
         if (!res) return;
-        QThreadPool::globalInstance()->releaseThread();
+        // Start the first thread in the stream.
         readStage->currentStatus = SingleThreadStage::STARTING;
+        readStage->startThread(NULL);
-        BasicLoop loop;
-        loop.stages = &this->processingStages;
-        loop.start_idx = 0;
-        loop.startItem = NULL;
-        loop.setAutoDelete(false);
-
-        QThreadPool::globalInstance()->start(&loop, processingStages.size() - processingStages[0]->stage_id);
-
-        // Wait for the end.
+        // Wait for the stream to reach the last frame available from
+        // the data source.
         readStage->dataSource.waitLast();
-        QThreadPool::globalInstance()->reserveThread();
+        // Now that there are no more incoming frames, call finalize
+        // on each transform in turn to collect any last templates
+        // they wish to issue.
         TemplateList final_output;
         // Push finalize through the stages
@@ -864,7 +936,8 @@ public:
             final_output.append(output_set);
         }
-        // dst is set to all output received by the final stage
+        // dst is set to all output received by the final stage, along
+        // with anything output via the calls to finalize.
         dst = collectionStage->getOutput();
         dst.append(final_output);
@@ -876,7 +949,8 @@ public:
     virtual void finalize(TemplateList & output)
     {
         (void) output;
-        // Not handling this yet -cao
+        // Nothing in particular to do here, stream calls finalize
+        // on all child transforms as part of projectUpdate
     }
     // Create and link stages
@@ -884,6 +958,19 @@ public:
     {
         if (transforms.isEmpty()) return;
+        // We share a thread pool across streams attached to the same
+        // parent tranform, retrieve or create a thread pool based
+        // on our parent transform.
+        QMutexLocker poolLock(&poolsAccess);
+        QHash<QObject *, QThreadPool *>::Iterator it;
+        if (!pools.contains(this->parent())) {
+            it = pools.insert(this->parent(), new QThreadPool(this));
+            it.value()->setMaxThreadCount(Globals->parallelism);
+        }
+        else it = pools.find(this->parent());
+        threads = it.value();
+        poolLock.unlock();
+
         stage_variance.reserve(transforms.size());
         foreach (const br::Transform *transform, transforms) {
             stage_variance.append(transform->timeVarying());
@@ -894,6 +981,7 @@ public:
         processingStages.push_back(readStage);
         readStage->stage_id = 0;
         readStage->stages = &this->processingStages;
+        readStage->threads = this->threads;
         int next_stage_id = 1;
@@ -901,9 +989,7 @@ public:
         for (int i =0; i < transforms.size(); i++)
         {
             if (stage_variance[i])
-            {
                 processingStages.append(new SingleThreadStage(prev_stage_variance));
-            }
             else
                 processingStages.append(new MultiThreadStage(Globals->parallelism));
@@ -914,6 +1000,7 @@ public:
             processingStages[i]->nextStage = processingStages[i+1];
             processingStages.last()->stages = &this->processingStages;
+            processingStages.last()->threads = this->threads;
             processingStages.last()->transform = transforms[i];
             prev_stage_variance = stage_variance[i];
@@ -923,6 +1010,7 @@ public:
         processingStages.append(collectionStage);
         collectionStage->stage_id = next_stage_id;
         collectionStage->stages = &this->processingStages;
+        collectionStage->threads = this->threads;
         processingStages[processingStages.size() - 2]->nextStage = collectionStage;
@@ -945,6 +1033,10 @@ protected:
     QList<ProcessingStage *> processingStages;
+    static QHash<QObject *, QThreadPool *> pools;
+    static QMutex poolsAccess;
+    QThreadPool * threads;
+
     void _project(const Template &src, Template &dst) const
     {
         (void) src; (void) dst;
@@ -957,6 +1049,9 @@ protected:
     }
 };
+QHash<QObject *, QThreadPool *> StreamTransform::pools;
+QMutex StreamTransform::poolsAccess;
+
 BR_REGISTER(Transform, StreamTransform)
@@ -116,7 +116,8 @@ class FilterDistance : public Distance
         foreach (const QString &key, Globals->filters.keys()) {
             bool keep = false;
             const QString metadata = a.file.get<QString>(key, "");
-            if (metadata.isEmpty() || Globals->filters[key].isEmpty()) continue;
+            if (Globals->filters[key].isEmpty()) continue;
+            if (metadata.isEmpty()) return -std::numeric_limits<float>::max();
             foreach (const QString &value, Globals->filters[key]) {
                 if (metadata == value) {
                     keep = true;
@@ -64,5 +64,5 @@ if [ ! -d ../data/KTH/vid ]; then
 	rm ${vidclass}.zip
   done
   # this file is corrupted
-  rm ../data/KTH/vid/boxing/person01_boxing_d4_uncomp.avi
+  rm -f ../data/KTH/vid/boxing/person01_boxing_d4_uncomp.avi
 fi
@@ -34,6 +34,11 @@
 	Howpublished = {https://github.com/lbestrowden},
 	Title = {bestrow1 at msu.edu}}
+@misc{imaus10,
+  Author = {Austin Van Blanton},
+  Howpublished = {https://github.com/imaus10},
+  Title = {imaus10 at gmail.com}}
+
 % Software
 @misc{libface,
 	Howpublished = {http://libface.sourceforge.net/file/Home.html},