Merge pull request #23 from biometrics/video_thing

Changes to support better video processing, round 1 "a lot of work to accomplish nothing"

Merge pull request #23 from biometrics/video_thing
Changes to support better video processing, round 1 "a lot of work to accomplish nothing"
jklontz
2 parents fe72ae12 9def2dd6
Showing 5 changed files with 366 additions and 121 deletions
CHANGELOG.md
sdk/openbr_plugin.cpp
sdk/openbr_plugin.h
sdk/plugins/meta.cpp
sdk/plugins/random.cpp
@@ -4,6 +4,7 @@
 * Enrolling files/folders are now sorted naturally instead of alpha numerically
 * YouTubeFacesDBTransform implements Dr. Wolf's experimental protocol
 * NEC3 refactored
+* Updated transform API to add support for time-varying transforms
 0.2.0 - 2/23/13
 ===============
@@ -494,7 +494,8 @@ TemplateList TemplateList::relabel(const TemplateList &amp;tl)
 QStringList Object::parameters() const
 {
     QStringList parameters;
-    for (int i=metaObject()->propertyOffset(); i<metaObject()->propertyCount(); i++) {
+
+    for (int i = firstAvailablePropertyIdx; i < metaObject()->propertyCount();i++) {
         QMetaProperty property = metaObject()->property(i);
         if (property.isStored(this)) continue;
         parameters.append(QString("%1 %2 = %3").arg(property.typeName(), property.name(), property.read(this).toString()));
@@ -713,18 +714,37 @@ void Object::init(const File &amp;file_)
     // Set name
     QString name = metaObject()->className();
     if (name.startsWith("br::")) name = name.right(name.size()-4);
-    const QMetaObject *superClass = metaObject()->superClass();
+
+    firstAvailablePropertyIdx = metaObject()->propertyCount();
+
+    const QMetaObject * baseClass = metaObject();
+    const QMetaObject * superClass = metaObject()->superClass();
+
     while (superClass != NULL) {
+        const QMetaObject * nextClass = superClass->superClass();
+
+        // baseClass <- something <- br::Object
+        // baseClass is the highest class whose properties we can set via positional arguments
+        if (nextClass && !strcmp(nextClass->className(),"br::Object")) {
+            firstAvailablePropertyIdx = baseClass->propertyOffset();
+        }
+
         QString superClassName = superClass->className();
+
+        // strip br:: prefix from superclass name
         if (superClassName.startsWith("br::"))
             superClassName = superClassName.right(superClassName.size()-4);
+
+        // Strip superclass name from base class name (e.g. PipeTransform -> Pipe)
         if (name.endsWith(superClassName))
             name = name.left(name.size() - superClassName.size());
+        baseClass = superClass;
         superClass = superClass->superClass();
+
     }
     setObjectName(name);
-    // Set properties
+    // Reset all properties
     for (int i=0; i<metaObject()->propertyCount(); i++) {
         QMetaProperty property = metaObject()->property(i);
         if (property.isResettable())
@@ -734,8 +754,17 @@ void Object::init(const File &amp;file_)
     foreach (QString key, file.localKeys()) {
         const QString value = file.value(key).toString();
-        if (key.startsWith("_Arg"))
-            key = metaObject()->property(metaObject()->propertyOffset()+key.mid(4).toInt()).name();
+
+        if (key.startsWith(("_Arg"))) {
+            int argument_number =  key.mid(4).toInt();
+            int target_idx = argument_number + firstAvailablePropertyIdx;
+
+            if (target_idx >= metaObject()->propertyCount()) {
+                qWarning("too many arguments for transform, ignoring %s\n", qPrintable(value));
+                continue;
+            }
+            key = metaObject()->property(target_idx).name();
+        }
         setProperty(key, value);
     }
@@ -421,6 +421,9 @@ class BR_EXPORT Object : public QObject
 {
     Q_OBJECT
+    // Index of the first property that can be set via command line arguments
+    int firstAvailablePropertyIdx;
+
 public:
     File file; /*!< \brief The file used to construct the plugin. */
@@ -989,6 +992,49 @@ public:
     virtual void backProject(const Template &dst, Template &src) const; /*!< \brief Invert the transform. */
     virtual void backProject(const TemplateList &dst, TemplateList &src) const; /*!< \brief Invert the transform. */
+    /*!< \brief Apply the transform, may update the transform's internal state */
+    virtual void projectUpdate(const Template &src, Template &dst)
+    {
+        project(src, dst);
+    }
+
+    /*!< \brief Apply the transform, may update the transform's internal state */
+    virtual void projectUpdate(const TemplateList &src, TemplateList &dst)
+    {
+        project(src,dst);
+    }
+
+    /*!< \brief inplace projectUpdate. */
+    void projectUpdate(Template &srcdst)
+    {
+        Template dst;
+        projectUpdate(srcdst, dst);
+        srcdst = dst;
+    }
+
+    /*!< \brief inplace projectUpdate. */
+    void projectUpdate(TemplateList &srcdst)
+    {
+        TemplateList dst;
+        projectUpdate(srcdst, dst);
+        srcdst = dst;
+    }
+
+    /*!
+     * Time-varying transforms may move away from a single input->single output model, and only emit
+     * templates under some conditions (e.g. a tracking thing may emit a template for each detected
+     * unique object), in this case finalize indicates that no further calls to project will be made
+     * and the transform can emit a final set if templates if it wants. Time-invariant transforms
+     * don't have to do anything.
+     */
+    virtual void finalize(TemplateList & output) { output = TemplateList(); }
+
+    /*!
+     * \brief Does the transform require the non-const version of project? Can vary for aggregation type transforms
+     * (if their children are time varying, they are also time varying, otherwise probably not)
+     */
+    virtual bool timeVarying() const { return false; }
+
     /*!
      * \brief Convenience function equivalent to project().
      */
@@ -1052,6 +1098,31 @@ inline QDataStream &amp;operator&gt;&gt;(QDataStream &amp;stream, Transform &amp;f)
 }
 /*!
+ * \brief A br::Transform for which the results of project may change due to prior calls to project
+ */
+class BR_EXPORT TimeVaryingTransform : public Transform
+{
+    Q_OBJECT
+
+    virtual bool timeVarying() const { return true; }
+
+    virtual void project(const Template &src, Template &dst) const
+    {
+        qFatal("No const project defined for time-varying transform");
+        (void) dst; (void) src;
+    }
+
+    virtual void project(const TemplateList &src, TemplateList &dst) const
+    {
+        qFatal("No const project defined for time-varying transform");
+        (void) dst; (void) src;
+    }
+
+protected:
+    TimeVaryingTransform(bool independent = true, bool trainable = true) : Transform(independent, trainable) {}
+};
+
+/*!
  * \brief A br::Transform expecting multiple matrices per template.
  */
 class BR_EXPORT MetaTransform : public Transform
@@ -1079,7 +1150,6 @@ private:
     void load(QDataStream &stream) { (void) stream; }
 };
-
 /*!
  * \brief A br::MetaTransform that does not require training data.
  */
@@ -86,6 +86,111 @@ static void incrementStep()
 }
 /*!
+ * \brief Use Expanded after basic calls that take a template list, used to implement ExpandTransform
+ */
+class ExpandDecorator : public Transform
+{
+    Q_OBJECT
+
+    Q_PROPERTY(br::Transform* transform READ get_transform WRITE set_transform RESET reset_transform)
+    BR_PROPERTY(br::Transform*, transform, NULL)
+
+public:
+    ExpandDecorator(Transform * input)
+    {
+        transform = input;
+        transform->setParent(this);
+        file = transform->file;
+        setObjectName(transform->objectName());
+    }
+
+    void train(const TemplateList &data)
+    {
+        transform->train(data);
+    }
+
+    void project(const Template &src, Template &dst) const
+    {
+        transform->project(src, dst);
+    }
+
+    void project(const TemplateList &src, TemplateList &dst) const
+    {
+        transform->project(src, dst);
+        dst = Expanded(dst);
+    }
+
+
+    void projectUpdate(const Template &src, Template &dst)
+    {
+        transform->projectUpdate(src, dst);
+    }
+
+    void projectUpdate(const TemplateList & src, TemplateList & dst)
+    {
+        transform->projectUpdate(src, dst);
+        dst = Expanded(dst);
+    }
+
+    bool timeVarying() const
+    {
+        return transform->timeVarying();
+    }
+
+    void finalize(TemplateList & output)
+    {
+        transform->finalize(output);
+        output = Expanded(output);
+    }
+
+};
+
+/*!
+ * \brief A MetaTransform that aggregates some sub-transforms
+ */
+class BR_EXPORT CompositeTransform : public TimeVaryingTransform
+{
+    Q_OBJECT
+
+public:
+    Q_PROPERTY(QList<br::Transform*> transforms READ get_transforms WRITE set_transforms RESET reset_transforms)
+    BR_PROPERTY(QList<br::Transform*>, transforms, QList<br::Transform*>())
+
+    virtual void project(const Template &src, Template &dst) const
+    {
+        if (timeVarying()) qFatal("No const project defined for time-varying transform");
+        _project(src, dst);
+    }
+
+    virtual void project(const TemplateList &src, TemplateList &dst) const
+    {
+        if (timeVarying()) qFatal("No const project defined for time-varying transform");
+        _project(src, dst);
+    }
+
+    bool timeVarying() const { return isTimeVarying; }
+
+    void init()
+    {
+        isTimeVarying = false;
+        foreach (const br::Transform *transform, transforms) {
+            if (transform->timeVarying()) {
+                isTimeVarying = true;
+                break;
+            }
+        }
+    }
+
+protected:
+    bool isTimeVarying;
+
+    virtual void _project(const Template & src, Template & dst) const = 0;
+    virtual void _project(const TemplateList & src, TemplateList & dst) const = 0;
+
+    CompositeTransform() : TimeVaryingTransform(false) {}
+};
+
+/*!
  * \ingroup Transforms
  * \brief Transforms in series.
  * \author Josh Klontz \cite jklontz
@@ -95,11 +200,9 @@ static void incrementStep()
  * \see ExpandTransform
  * \see ForkTransform
  */
-class PipeTransform : public MetaTransform
+class PipeTransform : public CompositeTransform
 {
     Q_OBJECT
-    Q_PROPERTY(QList<br::Transform*> transforms READ get_transforms WRITE set_transforms RESET reset_transforms)
-    BR_PROPERTY(QList<br::Transform*>, transforms, QList<br::Transform*>())
     void train(const TemplateList &data)
     {
@@ -115,12 +218,32 @@ class PipeTransform : public MetaTransform
         releaseStep();
     }
-    void project(const Template &src, Template &dst) const
+    void backProject(const Template &dst, Template &src) const
+    {
+        // Backprojecting a time-varying transform is probably not going to work.
+        if (timeVarying()) qFatal("No backProject defined for time-varying transform");
+
+        src = dst;
+        // Reverse order in which transforms are processed
+        int length = transforms.length();
+        for (int i=length-1; i>=0; i--) {
+            Transform *f = transforms.at(i);
+            try {
+                src >> *f;
+            } catch (...) {
+                qWarning("Exception triggered when processing %s with transform %s", qPrintable(dst.file.flat()), qPrintable(f->objectName()));
+                src = Template(src.file);
+                src.file.setBool("FTE");
+            }
+        }
+    }
+
+    void projectUpdate(const Template &src, Template &dst)
     {
         dst = src;
-        foreach (const Transform *f, transforms) {
+        foreach (Transform *f, transforms) {
             try {
-                dst >> *f;
+                f->projectUpdate(dst);
             } catch (...) {
                 qWarning("Exception triggered when processing %s with transform %s", qPrintable(src.file.flat()), qPrintable(f->objectName()));
                 dst = Template(src.file);
@@ -129,7 +252,44 @@ class PipeTransform : public MetaTransform
         }
     }
-    void project(const TemplateList &src, TemplateList &dst) const
+    // For time varying transforms, parallel execution over individual templates
+    // won't work.
+    void projectUpdate(const TemplateList & src, TemplateList & dst)
+    {
+        dst = src;
+        foreach (Transform *f, transforms)
+        {
+            f->projectUpdate(dst);
+        }
+    }
+
+    virtual void finalize(TemplateList & output)
+    {
+        output.clear();
+        // For each transform,
+        for (int i = 0; i < transforms.size(); i++)
+        {
+
+            // Collect any final templates
+            TemplateList last_set;
+            transforms[i]->finalize(last_set);
+            if (last_set.empty())
+                continue;
+            // Push any templates received through the remaining transforms in the sequence
+            for (int j = (i+1); j < transforms.size();j++)
+            {
+                transforms[j]->projectUpdate(last_set);
+            }
+            // append the result to the output set
+            output.append(last_set);
+        }
+    }
+
+
+protected:
+    // Template list project -- process templates in parallel through Transform::project
+    // or if parallelism is disabled, handle them sequentially
+   void _project(const TemplateList &src, TemplateList &dst) const
     {
         if (Globals->parallelism < 0) {
             dst = src;
@@ -140,22 +300,20 @@ class PipeTransform : public MetaTransform
         }
     }
-    void backProject(const Template &dst, Template &src) const
-    {
-        src = dst;
-        // Reverse order in which transforms are processed
-        int length = transforms.length();
-        for (int i=length-1; i>=0; i--) {
-            Transform *f = transforms.at(i);
-            try {
-                src >> *f;
-            } catch (...) {
-                qWarning("Exception triggered when processing %s with transform %s", qPrintable(dst.file.flat()), qPrintable(f->objectName()));
-                src = Template(src.file);
-                src.file.setBool("FTE");
-            }
-        }
-    }
+   // Single template const project, pass the template through each sub-transform, one after the other
+   virtual void _project(const Template & src, Template & dst) const
+   {
+       dst = src;
+       foreach (const Transform *f, transforms) {
+           try {
+               dst >> *f;
+           } catch (...) {
+               qWarning("Exception triggered when processing %s with transform %s", qPrintable(src.file.flat()), qPrintable(f->objectName()));
+               dst = Template(src.file);
+               dst.file.setBool("FTE");
+           }
+       }
+   }
 };
 BR_REGISTER(Transform, PipeTransform)
@@ -170,64 +328,31 @@ BR_REGISTER(Transform, PipeTransform)
  *
  * \see PipeTransform
  */
-class ExpandTransform : public MetaTransform
+class ExpandTransform : public PipeTransform
 {
     Q_OBJECT
-    Q_PROPERTY(QList<br::Transform*> transforms READ get_transforms WRITE set_transforms RESET reset_transforms)
-    BR_PROPERTY(QList<br::Transform*>, transforms, QList<br::Transform*>())
-    void train(const TemplateList &data)
+    void init()
     {
-        acquireStep();
-
-        TemplateList copy(data);
-        for (int i=0; i<transforms.size(); i++) {
-            transforms[i]->train(copy);
-            copy >> *transforms[i];
-            copy = Expanded(copy);
-            incrementStep();
+        for (int i = 0; i < transforms.size(); i++)
+        {
+            transforms[i] = new ExpandDecorator(transforms[i]);
         }
-
-        releaseStep();
+        // Need to call this to set up timevariance correctly, and it won't
+        // be called automatically
+        CompositeTransform::init();
     }
-    void project(const Template &src, Template &dst) const
-    {
-        dst = src;
-        foreach (const Transform *f, transforms) {
-            try {
-                dst >> *f;
-            } catch (...) {
-                qWarning("Exception triggered when processing %s with transform %s", qPrintable(src.file.flat()), qPrintable(f->objectName()));
-                dst = Template(src.file);
-                dst.file.setBool("FTE");
-            }
-        }
-    }
+protected:
-    void project(const TemplateList &src, TemplateList &dst) const
+    // Template list project -- project through transforms sequentially,
+    // then expand the results, can't use Transform::Project(templateList) since
+    // we need to expand between tranforms, so actually do need to overload this method
+    void _project(const TemplateList &src, TemplateList &dst) const
     {
         dst = src;
         for (int i=0; i<transforms.size(); i++) {
             dst >> *transforms[i];
-            dst = Expanded(dst);
-        }
-    }
-
-    void backProject(const Template &dst, Template &src) const
-    {
-        src = dst;
-        // Reverse order in which transforms are processed
-        int length = transforms.length();
-        for (int i=length-1; i>=0; i--) {
-            Transform *f = transforms.at(i);
-            try {
-                src >> *f;
-            } catch (...) {
-                qWarning("Exception triggered when processing %s with transform %s", qPrintable(dst.file.flat()), qPrintable(f->objectName()));
-                src = Template(src.file);
-                src.file.setBool("FTE");
-            }
         }
     }
 };
@@ -243,11 +368,9 @@ BR_REGISTER(Transform, ExpandTransform)
  *
  * \see PipeTransform
  */
-class ForkTransform : public MetaTransform
+class ForkTransform : public CompositeTransform
 {
     Q_OBJECT
-    Q_PROPERTY(QList<br::Transform*> transforms READ get_transforms WRITE set_transforms RESET reset_transforms)
-    BR_PROPERTY(QList<br::Transform*>, transforms, QList<br::Transform*>())
     void train(const TemplateList &data)
     {
@@ -260,7 +383,69 @@ class ForkTransform : public MetaTransform
         if (threaded) Globals->trackFutures(futures);
     }
-    void project(const Template &src, Template &dst) const
+    void backProject(const Template &dst, Template &src) const {Transform::backProject(dst, src);}
+
+    // same as _project, but calls projectUpdate on sub-transforms
+    void projectupdate(const Template & src, Template & dst)
+    {
+        foreach (Transform *f, transforms) {
+            try {
+                Template res;
+                f->projectUpdate(src, res);
+                dst.merge(res);
+            } catch (...) {
+                qWarning("Exception triggered when processing %s with transform %s", qPrintable(src.file.flat()), qPrintable(f->objectName()));
+                dst = Template(src.file);
+                dst.file.setBool("FTE");
+            }
+        }
+    }
+
+    void projectUpdate(const TemplateList & src, TemplateList & dst)
+    {
+        dst = src;
+        dst.reserve(src.size());
+        for (int i=0; i<src.size(); i++) dst.append(Template());
+        foreach (Transform *f, transforms) {
+            TemplateList m;
+            f->projectUpdate(src, m);
+            if (m.size() != dst.size()) qFatal("TemplateList is of an unexpected size.");
+            for (int i=0; i<src.size(); i++) dst[i].append(m[i]);
+        }
+    }
+
+    // this is probably going to go bad, fork transform probably won't work well in a variable
+    // input/output scenario
+    virtual void finalize(TemplateList & output)
+    {
+        output.clear();
+        // For each transform,
+        for (int i = 0; i < transforms.size(); i++)
+        {
+            // Collect any final templates
+            TemplateList last_set;
+            transforms[i]->finalize(last_set);
+            if (last_set.empty())
+                continue;
+
+            if (output.empty()) output = last_set;
+            else
+            {
+                // is the number of templates received from this transform consistent with the number
+                // received previously? If not we can't do anything coherent here.
+                if (last_set.size() != output.size())
+                    qFatal("mismatched template list sizes in ForkTransform");
+                for (int j = 0; j < output.size(); j++) {
+                    output[j].append(last_set[j]);
+                }
+            }
+        }
+    }
+
+protected:
+
+    // Apply each transform to src, concatenate the results
+    void _project(const Template &src, Template &dst) const
     {
         foreach (const Transform *f, transforms) {
             try {
@@ -273,7 +458,7 @@ class ForkTransform : public MetaTransform
         }
     }
-    void project(const TemplateList &src, TemplateList &dst) const
+    void _project(const TemplateList &src, TemplateList &dst) const
     {
         if (Globals->parallelism < 0) {
             dst.reserve(src.size());
@@ -288,6 +473,7 @@ class ForkTransform : public MetaTransform
             Transform::project(src, dst);
         }
     }
+
 };
 BR_REGISTER(Transform, ForkTransform)
@@ -27,47 +27,6 @@ namespace br
 /*!
  * \ingroup transforms
- * \brief Selects a random transform.
- * \author Josh Klontz \cite jklontz
- */
-class RndTransformTransform : public Transform
-{
-    Q_OBJECT
-    Q_PROPERTY(QList<br::Transform*> transforms READ get_transforms WRITE set_transforms RESET reset_transforms STORED false)
-    BR_PROPERTY(QList<br::Transform*>, transforms, QList<br::Transform*>())
-
-    int selectedIndex;
-    Transform *selectedTransform;
-
-    void train(const TemplateList &data)
-    {
-        selectedIndex = theRNG().uniform(0, transforms.size());
-        selectedTransform = transforms[selectedIndex]->clone();
-        selectedTransform->train(data);
-    }
-
-    void project(const Template &src, Template &dst) const
-    {
-        selectedTransform->project(src, dst);
-    }
-
-    void store(QDataStream &stream) const
-    {
-        stream << selectedIndex << *selectedTransform;
-    }
-
-    void load(QDataStream &stream)
-    {
-        stream >> selectedIndex;
-        selectedTransform = transforms[selectedIndex]->clone();
-        stream >> *selectedTransform;
-    }
-};
-
-BR_REGISTER(Transform, RndTransformTransform)
-
-/*!
- * \ingroup transforms
  * \brief Generates a random subspace.
  * \author Josh Klontz \cite jklontz
  */