Update handling of FTE

If a transform sets FTE on a template, don't pass that on to subsequent transforms. This is done by placing checks after every project call by stream and pipe. Switch the FTE marker from metadata to a member variable of template.

Update handling of FTE
If a transform sets FTE on a template, don't pass that on to subsequent transforms. This is done by placing checks after every project call by stream and pipe. Switch the FTE marker from metadata to a member variable of template.
Charles Otto
1 parent fbd840a0
Showing 10 changed files with 130 additions and 39 deletions
openbr/core/core.cpp
openbr/openbr_plugin.cpp
openbr/openbr_plugin.h
openbr/plugins/draw.cpp
openbr/plugins/keypoint.cpp
openbr/plugins/meta.cpp
openbr/plugins/misc.cpp
openbr/plugins/pp5.cpp
openbr/plugins/stasm4.cpp
openbr/plugins/stream.cpp
@@ -195,13 +195,11 @@ struct AlgorithmCore
         if (fileExclusion)
             outputDesc = "FileExclusion(" + gallery.flat() + ")+";
         outputDesc.append("GalleryOutput("+gallery.flat()+")");
-        QScopedPointer<Transform> outputTform(Transform::make(outputDesc, NULL));
-        stages.append(outputTform.data());
         stages.append(progressCounter.data());
         QScopedPointer<Transform> pipeline(br::pipeTransforms(stages));
-        QScopedPointer<Transform> stream(br::wrapTransform(pipeline.data(), "Stream(readMode=StreamGallery, endPoint=DiscardTemplates)"));
+        QScopedPointer<Transform> stream(br::wrapTransform(pipeline.data(), "Stream(readMode=StreamGallery, endPoint="+outputDesc+"+DiscardTemplates)"));
         TemplateList data, output;
         data.append(input);
@@ -479,8 +477,6 @@ struct AlgorithmCore
         // output specification we were passed. Gallery metadata is necessary for some Outputs to function correctly.
         QString outputString = output.flat().isEmpty() ? "Empty" : output.flat();
         QString outputRegionDesc = "Output("+ outputString +"," + targetGallery.flat() +"," + queryGallery.flat() + ","+ QString::number(transposeMode ? 1 : 0) + ")";
-        QScopedPointer<Transform> outputTForm(Transform::make(outputRegionDesc,NULL));
-        compareOutput.append(outputTForm.data());
         // The ProgressCounter transform will simply provide a display about the number of rows completed.
         compareOutput.append(progressCounter.data());
@@ -491,7 +487,7 @@ struct AlgorithmCore
         // Now, we will give that base transform to a stream, which will incrementally read the row gallery
         // and pass the transforms it reads through the base algorithm.
-        QScopedPointer<Transform> streamWrapper(br::wrapTransform(pipeline, "Stream(readMode=StreamGallery, endPoint=DiscardTemplates)"));
+        QScopedPointer<Transform> streamWrapper(br::wrapTransform(pipeline, "Stream(readMode=StreamGallery, endPoint="+outputRegionDesc+"+DiscardTemplates)"));
         // We set up a template containing the rowGallery we want to compare. 
         TemplateList rowGalleryTemplate;
@@ -261,6 +261,7 @@ void File::appendRects(const QList&lt;cv::Rect&gt; &amp;rects)
 /* File - private methods */
 void File::init(const QString &file)
 {
+    fte = false;
     name = file;
     while (name.endsWith(']') || name.endsWith(')')) {
@@ -287,6 +288,11 @@ void File::init(const QString &amp;file)
         }
         name = name.left(index);
     }
+
+    if (contains("FTE")) {
+        fte = getBool("FTE");
+        remove("FTE");
+    }
 }
 /* File - global methods */
@@ -367,7 +373,7 @@ int FileList::failures() const
 {
     int failures = 0;
     foreach (const File &file, *this)
-        if (file.get<bool>("FTO", false) || file.get<bool>("FTE", false))
+        if (file.fte)
             failures++;
     return failures;
 }
@@ -1344,7 +1350,7 @@ static void _project(const Transform *transform, const Template *src, Template *
     } catch (...) {
         qWarning("Exception triggered when processing %s with transform %s", qPrintable(src->file.flat()), qPrintable(transform->objectName()));
         *dst = Template(src->file);
-        dst->file.set("FTE", true);
+        dst->file.fte = true;
     }
 }
@@ -1474,6 +1480,9 @@ float Distance::compare(const Template &amp;a, const Template &amp;b) const
 float Distance::compare(const cv::Mat &a, const cv::Mat &b) const
 {
+    if (a.empty() || b.empty() || a.rows != b.rows || a.cols != b.cols || a.elemSize() != b.elemSize())
+        return -std::numeric_limits<float>::max();
+
     return compare(a.data, b.data, a.rows * a.cols * a.elemSize());
 }
@@ -170,7 +170,7 @@ struct BR_EXPORT File
 {
     QString name; /*!< \brief Path to a file on disk. */
-    File() {}
+    File() { fte = false; }
     File(const QString &file) { init(file); } /*!< \brief Construct a file from a string. */
     File(const QString &file, const QVariant &label) { init(file); set("Label", label); } /*!< \brief Construct a file from a string and assign a label. */
     File(const char *file) { init(file); } /*!< \brief Construct a file from a c-style string. */
@@ -308,7 +308,7 @@ struct BR_EXPORT File
         return result;
     }
-    inline bool failed() const { return getBool("FTE") || getBool("FTO"); } /*!< \brief Returns \c true if the file failed to open or enroll, \c false otherwise. */
+    inline bool failed() const { return fte; } /*!< \brief Returns \c true if the file failed to open or enroll, \c false otherwise. */
     QList<QPointF> namedPoints() const; /*!< \brief Returns points convertible from metadata keys. */
     QList<QPointF> points() const; /*!< \brief Returns the file's points list. */
@@ -327,6 +327,7 @@ struct BR_EXPORT File
     inline void setRects(const QList<QRectF> &rects) { clearRects(); appendRects(rects); } /*!< \brief Overwrites the file's rect list. */
     inline void setRects(const QList<cv::Rect> &rects) { clearRects(); appendRects(rects); } /*!< \brief Overwrites the file's rect list. */
+    bool fte;
 private:
     QVariantMap m_metadata;
     BR_EXPORT friend QDataStream &operator<<(QDataStream &stream, const File &file);
@@ -1295,7 +1296,7 @@ public:
      * \brief Return a pointer to a simplified version of this transform (if possible). Transforms which are only active during training should remove
      * themselves by either returning their child transforms (where relevant) or returning NULL. Set newTransform to true if the transform returned is newly allocated.
      */
-    virtual Transform * simplify(bool & newTransform) { newTransform = false; return this; }
+    virtual Transform * simplify(bool &newTransform) { newTransform = false; return this; }
 protected:
     Transform(bool independent = true, bool trainable = true); /*!< \brief Construct a transform. */
@@ -607,7 +607,7 @@ class MeanImageTransform : public TimeVaryingTransform
         cnt++;
     }
-    virtual void finalize(TemplateList & output)
+    virtual void finalize(TemplateList &output)
     {
         average /= float(cnt);
         imwrite(QString("%1.%2").arg(imgname).arg(ext).toStdString(), average);
@@ -54,7 +54,7 @@ class KeyPointDetectorTransform : public UntrainableTransform
             featureDetector->detect(src, keyPoints);
         } catch (...) {
             qWarning("Key point detection failed for file %s", qPrintable(src.file.name));
-            dst.file.set("FTE", true);
+            dst.file.fte = true;
         }
         QList<Rect> rects;
@@ -76,8 +76,21 @@ class PipeTransform : public CompositeTransform
     void _projectPartial(TemplateList *srcdst, int startIndex, int stopIndex)
     {
-        for (int i=startIndex; i<stopIndex; i++)
-            *srcdst >> *transforms[i];
+        TemplateList dst = *srcdst;
+        TemplateList temp;
+        for (int i=startIndex; i<stopIndex; i++) {
+            TemplateList temp;
+            transforms[i]->project(dst, temp);
+
+            dst.clear();
+            foreach (const Template &t, *srcdst) {
+                if (!t.file.fte)
+                    dst.append(t);
+                else
+                    srcdst->append(t);
+            }
+        }
+        srcdst->append(dst);
     }
     void train(const QList<TemplateList> &data)
@@ -136,10 +149,13 @@ class PipeTransform : public CompositeTransform
         foreach (Transform *f, transforms) {
             try {
                 f->projectUpdate(dst);
+                if (dst.file.fte)
+                    break;
             } catch (...) {
                 qWarning("Exception triggered when processing %s with transform %s", qPrintable(src.file.flat()), qPrintable(f->objectName()));
                 dst = Template(src.file);
-                dst.file.set("FTE", true);
+                dst.file.fte = true;
+                break;
             }
         }
     }
@@ -149,8 +165,7 @@ class PipeTransform : public CompositeTransform
     void projectUpdate(const TemplateList &src, TemplateList &dst)
     {
         dst = src;
-        foreach (Transform *f, transforms)
-        {
+        foreach (Transform *f, transforms) {
             f->projectUpdate(dst);
         }
     }
@@ -188,9 +203,7 @@ class PipeTransform : public CompositeTransform
                 continue;
             }
             for (int j=0; j < probe->transforms.size(); j++)
-            {
                 flattened.append(probe->transforms[j]);
-            }
         }
         transforms = flattened;
@@ -202,10 +215,20 @@ protected:
     // or if parallelism is disabled, handle them sequentially
    void _project(const TemplateList &src, TemplateList &dst) const
     {
-        dst = src;
+        TemplateList temp, output;
+        temp = src;
         foreach (const Transform *f, transforms) {
-            dst >> *f;
+            dst.clear();
+            f->project(temp, dst);
+            temp.clear();
+            foreach(const Template &t, dst) {
+                if (!t.file.fte)
+                    temp.append(t);
+                else output.append(t);
+            }
         }
+        output.append(temp);
+        dst = output;
     }
    // Single template const project, pass the template through each sub-transform, one after the other
@@ -215,10 +238,12 @@ protected:
        foreach (const Transform *f, transforms) {
            try {
                dst >> *f;
+               if (dst.file.fte)
+                   break;
            } catch (...) {
                qWarning("Exception triggered when processing %s with transform %s", qPrintable(src.file.flat()), qPrintable(f->objectName()));
                dst = Template(src.file);
-               dst.file.set("FTE", true);
+               dst.file.fte = true;
            }
        }
    }
@@ -323,7 +348,8 @@ class ForkTransform : public CompositeTransform
             } catch (...) {
                 qWarning("Exception triggered when processing %s with transform %s", qPrintable(src.file.flat()), qPrintable(f->objectName()));
                 dst = Template(src.file);
-                dst.file.set("FTE", true);
+                dst.file.fte = true;
+                break;
             }
         }
     }
@@ -379,7 +405,8 @@ protected:
             } catch (...) {
                 qWarning("Exception triggered when processing %s with transform %s", qPrintable(src.file.flat()), qPrintable(f->objectName()));
                 dst = Template(src.file);
-                dst.file.set("FTE", true);
+                dst.file.fte = true;
+                break;
             }
         }
     }
@@ -52,7 +52,8 @@ class OpenTransform : public UntrainableMetaTransform
                 dst.append(t);
                 dst.file.append(t.file.localMetadata());
             }
-            dst.file.set("FTO", dst.isEmpty());
+            if (dst.isEmpty())
+                dst.file.fte = true;
         } else {
             // Propogate or decode existing matricies
             foreach (const Mat &m, src) {
@@ -610,6 +611,10 @@ class GalleryOutputTransform : public TimeVaryingTransform
         if (src.empty())
             return;
         dst = src;
+        for (int i=0; i < dst.size();i++) {
+            if (dst[i].file.getBool("FTE"))
+                dst[i].file.fte = true;
+        }
         writer->writeBlock(dst);
     }
@@ -725,8 +730,10 @@ class OutputTransform : public TimeVaryingTransform
         // we received a template, which is the next row/column in order
         foreach (const Template &t, dst) {
-            for (int i=0; i < t.m().cols; i++) {
-                output->setRelative(t.m().at<float>(0, i), currentRow, currentCol);
+            bool fte = t.file.getBool("FTE") || t.file.fte;
+
+            for (int i=0; i < scoresPerMat; i++) {
+                output->setRelative(fte ? -std::numeric_limits<float>::max() : t.m().at<float>(0, i), currentRow, currentCol);
                 // row-major input
                 if (!transposeMode)
@@ -794,10 +801,12 @@ class OutputTransform : public TimeVaryingTransform
             fragmentsPerRow = bufferedSize;
             // a single col contains comparisons to all query files
             fragmentsPerCol = queryFiles.size();
+            scoresPerMat = fragmentsPerCol;
         }
         else {
             // a single row contains comparisons to all target files
             fragmentsPerRow = targetFiles.size();
+            scoresPerMat = fragmentsPerRow;
             // we output rows one at a time
             fragmentsPerCol = 1;
         }
@@ -823,6 +832,8 @@ class OutputTransform : public TimeVaryingTransform
     int fragmentsPerRow;
     int fragmentsPerCol;
+    int scoresPerMat;
+
 public:
     OutputTransform() : TimeVaryingTransform(false,false) {}
 };
@@ -323,7 +323,7 @@ class PP5EnrollTransform : public UntrainableMetaTransform
             // No faces were detected when we were expecting one, output something with FTE set.
             if (!foundFace && !Globals->enrollAll) {
                 dstList.append(Template(src.file, detectOnly ? src.m() : cv::Mat()));
-                dstList.last().file.set("FTE", true);
+                dstList.last().file.fte = true;
             }
         }
@@ -144,7 +144,7 @@ class StasmTransform : public UntrainableTransform
         if (!foundFace) {
             if (Globals->verbose) qWarning("No face found in %s.", qPrintable(src.file.fileName()));
-            dst.file.set("FTE",true);
+            dst.file.fte = true;
         } else {
             QList<QPointF> points;
             for (int i = 0; i < nLandmarks; i++) {
@@ -907,7 +907,17 @@ public:
             qFatal("null input to multi-thread stage");
         }
-        input->data >> *transform;
+        TemplateList project;
+        TemplateList completed;
+        for (int i=0; i < input->data.size(); i++) {
+            if (input->data[i].file.fte)
+                completed.append(input->data[i]);
+            else
+                project.append(input->data[i]);
+        }
+        input->data.clear();
+        transform->project(project, input->data);
+        input->data.append(completed);
         should_continue = nextStage->tryAcquireNextStage(input, final);
@@ -927,7 +937,7 @@ public:
     {
         // nothing to do.
     }
-    void status(){
+    void status() {
         qDebug("multi thread stage %d, nothing to worry about", this->stage_id);
     }
 };
@@ -980,13 +990,22 @@ public:
             qFatal("NULL input to stage %d", this->stage_id);
         if (input->sequenceNumber != next_target)
-        {
             qFatal("out of order frames for stage %d, got %d expected %d", this->stage_id, input->sequenceNumber, this->next_target);
-        }
+
         next_target = input->sequenceNumber + 1;
+        TemplateList project;
+        TemplateList completed;
+        foreach (const Template &t, input->data) {
+            if (t.file.fte)
+                completed.append(t);
+            else
+                project.append(t);
+        }
+        input->data.clear();
         // Project the input we got
-        transform->projectUpdate(input->data);
+        transform->projectUpdate(project, input->data);
+        input->data.append(completed);
         should_continue = nextStage->tryAcquireNextStage(input,final);
@@ -1057,12 +1076,40 @@ public:
         return true;
     }
-    void status(){
+    void status() {
         qDebug("single thread stage %d, status starting? %d, next %d buffer size %d", this->stage_id, this->currentStatus == SingleThreadStage::STARTING, this->next_target, this->inputBuffer->size());
     }
 };
+class EndStage : public SingleThreadStage
+{
+public:
+    EndStage(bool input_variance) : SingleThreadStage(input_variance) {}
+
+    ~EndStage() {}
+
+    // Calledfrom a different thread than run.
+    bool tryAcquireNextStage(FrameData *& input, bool &final)
+    {
+        for (int i=0; i < input->data.size(); i++) {
+            if (input->data[i].file.fte) {
+                input->data[i].file.fte = false;
+                input->data[i].file.set("FTE",QVariant::fromValue(true));
+            }
+            else
+                input->data[i].file.set("FTE",QVariant::fromValue(false));
+        }
+        
+        return SingleThreadStage::tryAcquireNextStage(input, final);
+    }
+
+    void status() {
+        qDebug("end stage %d, status starting? %d, next %d buffer size %d", this->stage_id, this->currentStatus == SingleThreadStage::STARTING, this->next_target, this->inputBuffer->size());
+    }
+
+};
+
 // Semi-functional, doesn't do anything productive outside of stream::train
 class CollectSets : public TimeVaryingTransform
 {
@@ -1099,7 +1146,7 @@ public:
         data.append(src);
     }
-    void finalize(TemplateList & output)
+    void finalize(TemplateList &output)
     {
         output = data;
         data.clear();
@@ -1198,7 +1245,7 @@ public:
         return true;
     }
-    void status(){
+    void status() {
         qDebug("Read stage %d, status starting? %d, next frame %d buffer size %d", this->stage_id, this->currentStatus == SingleThreadStage::STARTING, this->next_target, this->dataSource.size());
     }
 };
@@ -1463,7 +1510,7 @@ public:
         // We also have the last stage, which just puts the output of the
         // previous stages on a template list.
-        collectionStage = new SingleThreadStage(prev_stage_variance);
+        collectionStage = new EndStage(prev_stage_variance);
         collectionStage->transform = this->endPoint;