API changes in support of adding a stream based processing model

Allow inherited properties to be set via a transforms argument list at the command line, but exclude properties of classes that directly inherit from br::Object (this excludes e.g. Transform's properties which should be set during training). Properties are set in the same order that is currently used (the order of their property declarations), but all superclass properties are set before base class properties. Changes to the Transform API: add a set of methods supporting non-const project operations. Since overloading with a non-const version of the same function name won't work, the methods are called projectUpdate, since they perform the expected project operation, but also allow the transform to update its internal state. This can be used to support tracking operations, as well as online learning methods. All current classes can implement projectUpdate by calling their const project method. To handle variable input/output situations (e.g. a tracking function may only wish to emit a single template per unique detected object), a "finalize" method is also added to Transform, to be called to indicate that no further calls to projectUpdate will be made, and allow the transform to emit any final templates This is only relevant for non-const project since a time-invariant transform has no way to manage an internal list of templates it is waiting to emit. Add a "timeVarying" method to Transform, which returns a bool indicate whether or not the transform is time-invariant (only projectUpdate should be called for a time-varying transform). Refactor Transforms that maintain a list of subclasses (pipe transform, expand transform, fork transform, random transform) to inherit from a common subclass (CompositeTransform). Time variance for composite transforms is set based on their child transforms -- a composite transform with any time-varying children is time-varying. Some code consolidation via the shared subclass.

API changes in support of adding a stream based processing model
Allow inherited properties to be set via a transforms argument list at the command line, but exclude properties of classes that directly inherit from br::Object (this excludes e.g. Transform's properties which should be set during training). Properties are set in the same order that is currently used (the order of their property declarations), but all superclass properties are set before base class properties. Changes to the Transform API: add a set of methods supporting non-const project operations. Since overloading with a non-const version of the same function name won't work, the methods are called projectUpdate, since they perform the expected project operation, but also allow the transform to update its internal state. This can be used to support tracking operations, as well as online learning methods. All current classes can implement projectUpdate by calling their const project method. To handle variable input/output situations (e.g. a tracking function may only wish to emit a single template per unique detected object), a "finalize" method is also added to Transform, to be called to indicate that no further calls to projectUpdate will be made, and allow the transform to emit any final templates This is only relevant for non-const project since a time-invariant transform has no way to manage an internal list of templates it is waiting to emit. Add a "timeVarying" method to Transform, which returns a bool indicate whether or not the transform is time-invariant (only projectUpdate should be called for a time-varying transform). Refactor Transforms that maintain a list of subclasses (pipe transform, expand transform, fork transform, random transform) to inherit from a common subclass (CompositeTransform). Time variance for composite transforms is set based on their child transforms -- a composite transform with any time-varying children is time-varying. Some code consolidation via the shared subclass.
Charles Otto
1 parent fe72ae12
Showing 4 changed files with 388 additions and 91 deletions
sdk/openbr_plugin.cpp
sdk/openbr_plugin.h
sdk/plugins/meta.cpp
sdk/plugins/random.cpp
@@ -494,7 +494,8 @@ TemplateList TemplateList::relabel(const TemplateList &amp;tl)
 QStringList Object::parameters() const
 {
     QStringList parameters;
-    for (int i=metaObject()->propertyOffset(); i<metaObject()->propertyCount(); i++) {
+
+    for (int i = first_available_property_idx; i < metaObject()->propertyCount();i++) {
         QMetaProperty property = metaObject()->property(i);
         if (property.isStored(this)) continue;
         parameters.append(QString("%1 %2 = %3").arg(property.typeName(), property.name(), property.read(this).toString()));
@@ -713,18 +714,37 @@ void Object::init(const File &amp;file_)
     // Set name
     QString name = metaObject()->className();
     if (name.startsWith("br::")) name = name.right(name.size()-4);
-    const QMetaObject *superClass = metaObject()->superClass();
+
+    first_available_property_idx = metaObject()->propertyCount();
+
+    const QMetaObject * baseClass = metaObject();
+    const QMetaObject * superClass = metaObject()->superClass();
+
     while (superClass != NULL) {
+        const QMetaObject * nextClass = superClass->superClass();
+
+        // baseClass <- something <- br::Object
+        // baseClass is the highest class whose properties we can set via positional arguments
+        if (nextClass && !strcmp(nextClass->className(),"br::Object")) {
+            first_available_property_idx = baseClass->propertyOffset();
+        }
+
         QString superClassName = superClass->className();
+
+        // strip br:: prefix from superclass name
         if (superClassName.startsWith("br::"))
             superClassName = superClassName.right(superClassName.size()-4);
+
+        // Strip superclass name from base class name (e.g. PipeTransform -> Pipe)
         if (name.endsWith(superClassName))
             name = name.left(name.size() - superClassName.size());
+        baseClass = superClass;
         superClass = superClass->superClass();
+
     }
     setObjectName(name);
-    // Set properties
+    // Reset all properties
     for (int i=0; i<metaObject()->propertyCount(); i++) {
         QMetaProperty property = metaObject()->property(i);
         if (property.isResettable())
@@ -734,8 +754,17 @@ void Object::init(const File &amp;file_)
     foreach (QString key, file.localKeys()) {
         const QString value = file.value(key).toString();
-        if (key.startsWith("_Arg"))
-            key = metaObject()->property(metaObject()->propertyOffset()+key.mid(4).toInt()).name();
+
+        if (key.startsWith(("_Arg"))) {
+            int argument_number =  key.mid(4).toInt();
+            int target_idx = argument_number + first_available_property_idx;
+
+            if (target_idx >= metaObject()->propertyCount()) {
+                qWarning("too many arguments for transform, ignoring %s\n", qPrintable(value));
+                continue;
+            }
+            key = metaObject()->property(target_idx).name();
+        }
         setProperty(key, value);
     }
@@ -421,6 +421,9 @@ class BR_EXPORT Object : public QObject
 {
     Q_OBJECT
+    // Index of the first property that can be set via command line arguments
+    int first_available_property_idx;
+
 public:
     File file; /*!< \brief The file used to construct the plugin. */
@@ -989,6 +992,48 @@ public:
     virtual void backProject(const Template &dst, Template &src) const; /*!< \brief Invert the transform. */
     virtual void backProject(const TemplateList &dst, TemplateList &src) const; /*!< \brief Invert the transform. */
+    /*!< \brief Apply the transform, may update the transform's internal state */
+    virtual void projectUpdate(const Template &src, Template &dst)
+    {
+        project(src, dst);
+    }
+
+    /*!< \brief Apply the transform, may update the transform's internal state */
+    virtual void projectUpdate(const TemplateList &src, TemplateList &dst)
+    {
+        project(src,dst);
+    }
+
+    // inplace project/update
+    void projectUpdate(Template & src)
+    {
+        Template dst = src;
+        projectUpdate(src, dst);
+        src = dst;
+    }
+    void projectUpdate(TemplateList & src)
+    {
+        TemplateList dst = src;
+        projectUpdate(src, dst);
+        src = dst;
+    }
+
+    // Time-varying transforms may move away from a single input->single output model, and only emit
+    // templates under some conditions (e.g. a tracking thing may emit a template for each detected
+    // unique object), in this case finalize indicates that no further calls to project will be made
+    // and the transform can emit a final set if templates if it wants. Time-invariant transforms
+    // don't have to do anything.
+    virtual void finalize(TemplateList & output)
+    {
+        output = TemplateList();
+    }
+
+    /*!
+     * \brief Does the transform require the non-const version of project? Can vary for aggregation type transforms
+     * (if their children are time varying, they are also time varying, otherwise probably not)
+     */
+    virtual bool timeVarying() const {return false;}
+
     /*!
      * \brief Convenience function equivalent to project().
      */
@@ -1051,6 +1096,33 @@ inline QDataStream &amp;operator&gt;&gt;(QDataStream &amp;stream, Transform &amp;f)
     return stream;
 }
+
+/*!
+ * \brief A br::Transform for which the results of project may change due to prior calls to project
+ */
+class BR_EXPORT TimeVaryingTransform : public Transform
+{
+    Q_OBJECT
+
+    virtual bool timeVarying() const {return true;}
+    virtual void project(const Template &src, Template &dst) const
+    {
+        qFatal("No const project defined for time-varying transform");
+        // shut up unused param warning? probably a better way to handle this
+        dst = src;
+    }
+    virtual void project(const TemplateList &src, TemplateList &dst) const
+    {
+        qFatal("No const project defined for time-varying transform");
+        dst = src;
+    }
+
+
+protected:
+    TimeVaryingTransform(bool independent = true, bool trainable = true) : Transform(independent, trainable) {}
+};
+
+
 /*!
  * \brief A br::Transform expecting multiple matrices per template.
  */
@@ -1063,6 +1135,137 @@ protected:
 };
 /*!
+ * \brief A MetaTransform that aggregates some sub-transforms
+ */
+class BR_EXPORT CompositeTransform : public TimeVaryingTransform
+{
+    Q_OBJECT
+public:
+    Q_PROPERTY(QList<br::Transform*> transforms READ get_transforms WRITE set_transforms RESET reset_transforms)
+    BR_PROPERTY(QList<br::Transform*>, transforms, QList<br::Transform*>())
+
+    virtual void train(const TemplateList &data);
+
+    virtual void project(const Template &src, Template &dst) const
+    {
+        if (timeVarying()) qFatal("No const project defined for time-varying transform");
+        _project(src, dst);
+    }
+
+    virtual void project(const TemplateList &src, TemplateList &dst) const
+    {
+        if (timeVarying()) qFatal("No const project defined for time-varying transform");
+        _project(src, dst);
+    }
+
+    void backProject(const Template &dst, Template &src) const
+    {
+        // Backprojecting a time-varying transform is probably not going to work.
+        if (timeVarying()) qFatal("No backProject defined for time-varying transform");
+
+        src = dst;
+        // Reverse order in which transforms are processed
+        int length = transforms.length();
+        for (int i=length-1; i>=0; i--) {
+            Transform *f = transforms.at(i);
+            try {
+                src >> *f;
+            } catch (...) {
+                qWarning("Exception triggered when processing %s with transform %s", qPrintable(dst.file.flat()), qPrintable(f->objectName()));
+                src = Template(src.file);
+                src.file.setBool("FTE");
+            }
+        }
+    }
+
+    void projectUpdate(const Template &src, Template &dst)
+    {
+        dst = src;
+        foreach (Transform *f, transforms) {
+            try {
+                f->projectUpdate(dst);
+            } catch (...) {
+                qWarning("Exception triggered when processing %s with transform %s", qPrintable(src.file.flat()), qPrintable(f->objectName()));
+                dst = Template(src.file);
+                dst.file.setBool("FTE");
+            }
+        }
+    }
+
+    // For time varying transforms, parallel execution over individual templates
+    // won't work.
+    void projectUpdate(const TemplateList & src, TemplateList & dst)
+    {
+        dst = src;
+        foreach (Transform *f, transforms)
+        {
+            f->projectUpdate(dst);
+        }
+    }
+
+    bool timeVarying() const
+    {
+        return time_varying;
+    }
+
+    void init()
+    {
+        time_varying = false;
+        foreach(const br::Transform * transform, transforms)
+        {
+            if (transform->timeVarying()) {
+                time_varying = true;
+                break;
+            }
+        }
+    }
+
+    virtual void finalize(TemplateList & output)
+    {
+        output.clear();
+        // For each transform,
+        for (int i = 0; i < transforms.size(); i++)
+        {
+
+            // Collect any final templates
+            TemplateList last_set;
+            transforms[i]->finalize(last_set);
+            if (last_set.empty())
+                continue;
+            // Push any templates received through the remaining transforms in the sequence
+            for (int j = (i+1); j < transforms.size();j++)
+            {
+                transforms[j]->projectUpdate(last_set);
+            }
+            // append the result to the output set
+            output.append(last_set);
+        }
+    }
+
+protected:
+    bool time_varying;
+    // Single template const project, default implementation for aggregate transforms--pass the template through each
+    // sub-transform, one after the other
+    virtual void _project(const Template & src, Template & dst) const
+    {
+        dst = src;
+        foreach (const Transform *f, transforms) {
+            try {
+                dst >> *f;
+            } catch (...) {
+                qWarning("Exception triggered when processing %s with transform %s", qPrintable(src.file.flat()), qPrintable(f->objectName()));
+                dst = Template(src.file);
+                dst.file.setBool("FTE");
+            }
+        }
+    }
+    virtual void _project(const TemplateList & src, TemplateList & dst) const = 0;
+
+    CompositeTransform() : TimeVaryingTransform(false) {}
+};
+
+
+/*!
  * \brief A br::Transform that does not require training data.
  */
 class BR_EXPORT UntrainableTransform : public Transform
@@ -85,6 +85,21 @@ static void incrementStep()
     Globals->currentStep += 1.0 / pow(10.0, double(depth));
 }
+// CompositeTransform::train placed here to pick up acquireStep and incrementStep
+void CompositeTransform::train(const TemplateList &data)
+{
+    acquireStep();
+
+    TemplateList copy(data);
+    for (int i=0; i<transforms.size(); i++) {
+        transforms[i]->train(copy);
+        copy >> *transforms[i];
+        incrementStep();
+    }
+
+    releaseStep();
+}
+
 /*!
  * \ingroup Transforms
  * \brief Transforms in series.
@@ -95,41 +110,15 @@ static void incrementStep()
  * \see ExpandTransform
  * \see ForkTransform
  */
-class PipeTransform : public MetaTransform
+class PipeTransform : public CompositeTransform
 {
     Q_OBJECT
-    Q_PROPERTY(QList<br::Transform*> transforms READ get_transforms WRITE set_transforms RESET reset_transforms)
-    BR_PROPERTY(QList<br::Transform*>, transforms, QList<br::Transform*>())
-    void train(const TemplateList &data)
-    {
-        acquireStep();
-        TemplateList copy(data);
-        for (int i=0; i<transforms.size(); i++) {
-            transforms[i]->train(copy);
-            copy >> *transforms[i];
-            incrementStep();
-        }
-
-        releaseStep();
-    }
-
-    void project(const Template &src, Template &dst) const
-    {
-        dst = src;
-        foreach (const Transform *f, transforms) {
-            try {
-                dst >> *f;
-            } catch (...) {
-                qWarning("Exception triggered when processing %s with transform %s", qPrintable(src.file.flat()), qPrintable(f->objectName()));
-                dst = Template(src.file);
-                dst.file.setBool("FTE");
-            }
-        }
-    }
-
-    void project(const TemplateList &src, TemplateList &dst) const
+protected:
+    // Template list project -- process templates in parallel through Transform::project
+    // or if parallelism is disabled, handle them sequentially
+   void _project(const TemplateList &src, TemplateList &dst) const
     {
         if (Globals->parallelism < 0) {
             dst = src;
@@ -139,23 +128,6 @@ class PipeTransform : public MetaTransform
             Transform::project(src, dst);
         }
     }
-
-    void backProject(const Template &dst, Template &src) const
-    {
-        src = dst;
-        // Reverse order in which transforms are processed
-        int length = transforms.length();
-        for (int i=length-1; i>=0; i--) {
-            Transform *f = transforms.at(i);
-            try {
-                src >> *f;
-            } catch (...) {
-                qWarning("Exception triggered when processing %s with transform %s", qPrintable(dst.file.flat()), qPrintable(f->objectName()));
-                src = Template(src.file);
-                src.file.setBool("FTE");
-            }
-        }
-    }
 };
 BR_REGISTER(Transform, PipeTransform)
@@ -170,11 +142,9 @@ BR_REGISTER(Transform, PipeTransform)
  *
  * \see PipeTransform
  */
-class ExpandTransform : public MetaTransform
+class ExpandTransform : public CompositeTransform
 {
     Q_OBJECT
-    Q_PROPERTY(QList<br::Transform*> transforms READ get_transforms WRITE set_transforms RESET reset_transforms)
-    BR_PROPERTY(QList<br::Transform*>, transforms, QList<br::Transform*>())
     void train(const TemplateList &data)
     {
@@ -191,21 +161,47 @@ class ExpandTransform : public MetaTransform
         releaseStep();
     }
-    void project(const Template &src, Template &dst) const
+    // same as _project, but calling projectUpdate on sub-transforms instead of using
+    // operator>>
+    void projectUpdate(const TemplateList &src, TemplateList &dst)
     {
         dst = src;
-        foreach (const Transform *f, transforms) {
-            try {
-                dst >> *f;
-            } catch (...) {
-                qWarning("Exception triggered when processing %s with transform %s", qPrintable(src.file.flat()), qPrintable(f->objectName()));
-                dst = Template(src.file);
-                dst.file.setBool("FTE");
+        for (int i=0; i<transforms.size(); i++) {
+            transforms[i]->projectUpdate(dst);
+            dst = Expanded(dst);
+        }
+    }
+
+    virtual void finalize(TemplateList & output)
+    {
+        output.clear();
+        // For each transform,
+        for (int i = 0; i < transforms.size(); i++)
+        {
+
+            // Collect any final templates
+            TemplateList last_set;
+            transforms[i]->finalize(last_set);
+            last_set = Expanded(last_set);
+            if (last_set.empty())
+                continue;
+            // Push any templates received through the remaining transforms in the sequence
+            for (int j = (i+1); j < transforms.size();j++)
+            {
+                transforms[j]->projectUpdate(last_set);
+                last_set = Expanded(last_set);
             }
+            // append the result to the output set
+            output.append(last_set);
         }
     }
-    void project(const TemplateList &src, TemplateList &dst) const
+protected:
+
+    // Template list project -- project through transforms sequentially,
+    // then expand the results, can't use Transform::Project(templateList) since
+    // we need to expand between tranforms
+    void _project(const TemplateList &src, TemplateList &dst) const
     {
         dst = src;
         for (int i=0; i<transforms.size(); i++) {
@@ -213,23 +209,6 @@ class ExpandTransform : public MetaTransform
             dst = Expanded(dst);
         }
     }
-
-    void backProject(const Template &dst, Template &src) const
-    {
-        src = dst;
-        // Reverse order in which transforms are processed
-        int length = transforms.length();
-        for (int i=length-1; i>=0; i--) {
-            Transform *f = transforms.at(i);
-            try {
-                src >> *f;
-            } catch (...) {
-                qWarning("Exception triggered when processing %s with transform %s", qPrintable(dst.file.flat()), qPrintable(f->objectName()));
-                src = Template(src.file);
-                src.file.setBool("FTE");
-            }
-        }
-    }
 };
 BR_REGISTER(Transform, ExpandTransform)
@@ -243,11 +222,9 @@ BR_REGISTER(Transform, ExpandTransform)
  *
  * \see PipeTransform
  */
-class ForkTransform : public MetaTransform
+class ForkTransform : public CompositeTransform
 {
     Q_OBJECT
-    Q_PROPERTY(QList<br::Transform*> transforms READ get_transforms WRITE set_transforms RESET reset_transforms)
-    BR_PROPERTY(QList<br::Transform*>, transforms, QList<br::Transform*>())
     void train(const TemplateList &data)
     {
@@ -259,8 +236,58 @@ class ForkTransform : public MetaTransform
         }
         if (threaded) Globals->trackFutures(futures);
     }
+    // The implementation of backProject in aggregate transform probably doesn't do anything useful here.
+    void backProject(const Template &dst, Template &src) const {Transform::backProject(dst, src);}
-    void project(const Template &src, Template &dst) const
+    // same as _project, but calls projectUpdate on sub-transforms
+    void projectupdate(const Template & src, Template & dst)
+    {
+        foreach (Transform *f, transforms) {
+            try {
+                Template res;
+                f->projectUpdate(src, res);
+                dst.merge(res);
+            } catch (...) {
+                qWarning("Exception triggered when processing %s with transform %s", qPrintable(src.file.flat()), qPrintable(f->objectName()));
+                dst = Template(src.file);
+                dst.file.setBool("FTE");
+            }
+        }
+    }
+
+
+    // this is probably going to go bad, fork transform probably won't work well in a variable
+    // input/output scenario
+    virtual void finalize(TemplateList & output)
+    {
+        output.clear();
+        // For each transform,
+        for (int i = 0; i < transforms.size(); i++)
+        {
+            // Collect any final templates
+            TemplateList last_set;
+            transforms[i]->finalize(last_set);
+            if (last_set.empty())
+                continue;
+
+            if (output.empty()) output = last_set;
+            else
+            {
+                // is the number of templates received from this transform consistent with the number
+                // received previously? If not we can't do anything coherent here.
+                if (last_set.size() != output.size())
+                    qFatal("mismatched template list sizes in ForkTransform");
+                for (int j = 0; j < output.size(); j++) {
+                    output[j].append(last_set[j]);
+                }
+            }
+        }
+    }
+
+protected:
+
+    // Apply each transform to src, concatenate the results
+    void _project(const Template &src, Template &dst) const
     {
         foreach (const Transform *f, transforms) {
             try {
@@ -273,7 +300,7 @@ class ForkTransform : public MetaTransform
         }
     }
-    void project(const TemplateList &src, TemplateList &dst) const
+    void _project(const TemplateList &src, TemplateList &dst) const
     {
         if (Globals->parallelism < 0) {
             dst.reserve(src.size());
@@ -288,6 +315,20 @@ class ForkTransform : public MetaTransform
             Transform::project(src, dst);
         }
     }
+
+    void projectUpdate(const TemplateList & src, TemplateList & dst)
+    {
+        dst = src;
+        dst.reserve(src.size());
+        for (int i=0; i<src.size(); i++) dst.append(Template());
+        foreach (Transform *f, transforms) {
+            TemplateList m;
+            f->projectUpdate(src, m);
+            if (m.size() != dst.size()) qFatal("TemplateList is of an unexpected size.");
+            for (int i=0; i<src.size(); i++) dst[i].append(m[i]);
+        }
+    }
+
 };
 BR_REGISTER(Transform, ForkTransform)
@@ -30,11 +30,9 @@ namespace br
  * \brief Selects a random transform.
  * \author Josh Klontz \cite jklontz
  */
-class RndTransformTransform : public Transform
+class RndTransformTransform : public CompositeTransform
 {
     Q_OBJECT
-    Q_PROPERTY(QList<br::Transform*> transforms READ get_transforms WRITE set_transforms RESET reset_transforms STORED false)
-    BR_PROPERTY(QList<br::Transform*>, transforms, QList<br::Transform*>())
     int selectedIndex;
     Transform *selectedTransform;
@@ -46,11 +44,37 @@ class RndTransformTransform : public Transform
         selectedTransform->train(data);
     }
-    void project(const Template &src, Template &dst) const
+    bool timeVarying() const
+    {
+        // calling on selectedTransform assumes train has already been called. -cao
+        return selectedTransform->timeVarying();
+    }
+
+    void _project(const Template &src, Template &dst) const
     {
         selectedTransform->project(src, dst);
     }
+    void _project(const TemplateList &src, TemplateList &dst) const
+    {
+        Transform::project(src, dst);
+    }
+
+    void projectUpdate(const Template &src, Template &dst)
+    {
+        selectedTransform->projectUpdate(src,dst);
+    }
+
+    void projectUpdate(const TemplateList & src, TemplateList & dst)
+    {
+        selectedTransform->projectUpdate(src, dst);
+    }
+
+    void finalize(TemplateList & output)
+    {
+        selectedTransform->finalize(output);
+    }
+
     void store(QDataStream &stream) const
     {
         stream << selectedIndex << *selectedTransform;