diff --git a/CHANGELOG.md b/CHANGELOG.md index f30a05a..8f3aa0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ * Enrolling files/folders are now sorted naturally instead of alpha numerically * YouTubeFacesDBTransform implements Dr. Wolf's experimental protocol * NEC3 refactored +* Updated transform API to add support for time-varying transforms 0.2.0 - 2/23/13 =============== diff --git a/sdk/openbr_plugin.cpp b/sdk/openbr_plugin.cpp index 9fffe4b..a767c72 100644 --- a/sdk/openbr_plugin.cpp +++ b/sdk/openbr_plugin.cpp @@ -494,7 +494,8 @@ TemplateList TemplateList::relabel(const TemplateList &tl) QStringList Object::parameters() const { QStringList parameters; - for (int i=metaObject()->propertyOffset(); ipropertyCount(); i++) { + + for (int i = firstAvailablePropertyIdx; i < metaObject()->propertyCount();i++) { QMetaProperty property = metaObject()->property(i); if (property.isStored(this)) continue; parameters.append(QString("%1 %2 = %3").arg(property.typeName(), property.name(), property.read(this).toString())); @@ -713,18 +714,37 @@ void Object::init(const File &file_) // Set name QString name = metaObject()->className(); if (name.startsWith("br::")) name = name.right(name.size()-4); - const QMetaObject *superClass = metaObject()->superClass(); + + firstAvailablePropertyIdx = metaObject()->propertyCount(); + + const QMetaObject * baseClass = metaObject(); + const QMetaObject * superClass = metaObject()->superClass(); + while (superClass != NULL) { + const QMetaObject * nextClass = superClass->superClass(); + + // baseClass <- something <- br::Object + // baseClass is the highest class whose properties we can set via positional arguments + if (nextClass && !strcmp(nextClass->className(),"br::Object")) { + firstAvailablePropertyIdx = baseClass->propertyOffset(); + } + QString superClassName = superClass->className(); + + // strip br:: prefix from superclass name if (superClassName.startsWith("br::")) superClassName = superClassName.right(superClassName.size()-4); + + // Strip superclass name from base class name (e.g. PipeTransform -> Pipe) if (name.endsWith(superClassName)) name = name.left(name.size() - superClassName.size()); + baseClass = superClass; superClass = superClass->superClass(); + } setObjectName(name); - // Set properties + // Reset all properties for (int i=0; ipropertyCount(); i++) { QMetaProperty property = metaObject()->property(i); if (property.isResettable()) @@ -734,8 +754,17 @@ void Object::init(const File &file_) foreach (QString key, file.localKeys()) { const QString value = file.value(key).toString(); - if (key.startsWith("_Arg")) - key = metaObject()->property(metaObject()->propertyOffset()+key.mid(4).toInt()).name(); + + if (key.startsWith(("_Arg"))) { + int argument_number = key.mid(4).toInt(); + int target_idx = argument_number + firstAvailablePropertyIdx; + + if (target_idx >= metaObject()->propertyCount()) { + qWarning("too many arguments for transform, ignoring %s\n", qPrintable(value)); + continue; + } + key = metaObject()->property(target_idx).name(); + } setProperty(key, value); } diff --git a/sdk/openbr_plugin.h b/sdk/openbr_plugin.h index 5b96809..aa116a1 100644 --- a/sdk/openbr_plugin.h +++ b/sdk/openbr_plugin.h @@ -421,6 +421,9 @@ class BR_EXPORT Object : public QObject { Q_OBJECT + // Index of the first property that can be set via command line arguments + int firstAvailablePropertyIdx; + public: File file; /*!< \brief The file used to construct the plugin. */ @@ -989,6 +992,49 @@ public: virtual void backProject(const Template &dst, Template &src) const; /*!< \brief Invert the transform. */ virtual void backProject(const TemplateList &dst, TemplateList &src) const; /*!< \brief Invert the transform. */ + /*!< \brief Apply the transform, may update the transform's internal state */ + virtual void projectUpdate(const Template &src, Template &dst) + { + project(src, dst); + } + + /*!< \brief Apply the transform, may update the transform's internal state */ + virtual void projectUpdate(const TemplateList &src, TemplateList &dst) + { + project(src,dst); + } + + /*!< \brief inplace projectUpdate. */ + void projectUpdate(Template &srcdst) + { + Template dst; + projectUpdate(srcdst, dst); + srcdst = dst; + } + + /*!< \brief inplace projectUpdate. */ + void projectUpdate(TemplateList &srcdst) + { + TemplateList dst; + projectUpdate(srcdst, dst); + srcdst = dst; + } + + /*! + * Time-varying transforms may move away from a single input->single output model, and only emit + * templates under some conditions (e.g. a tracking thing may emit a template for each detected + * unique object), in this case finalize indicates that no further calls to project will be made + * and the transform can emit a final set if templates if it wants. Time-invariant transforms + * don't have to do anything. + */ + virtual void finalize(TemplateList & output) { output = TemplateList(); } + + /*! + * \brief Does the transform require the non-const version of project? Can vary for aggregation type transforms + * (if their children are time varying, they are also time varying, otherwise probably not) + */ + virtual bool timeVarying() const { return false; } + /*! * \brief Convenience function equivalent to project(). */ @@ -1052,6 +1098,31 @@ inline QDataStream &operator>>(QDataStream &stream, Transform &f) } /*! + * \brief A br::Transform for which the results of project may change due to prior calls to project + */ +class BR_EXPORT TimeVaryingTransform : public Transform +{ + Q_OBJECT + + virtual bool timeVarying() const { return true; } + + virtual void project(const Template &src, Template &dst) const + { + qFatal("No const project defined for time-varying transform"); + (void) dst; (void) src; + } + + virtual void project(const TemplateList &src, TemplateList &dst) const + { + qFatal("No const project defined for time-varying transform"); + (void) dst; (void) src; + } + +protected: + TimeVaryingTransform(bool independent = true, bool trainable = true) : Transform(independent, trainable) {} +}; + +/*! * \brief A br::Transform expecting multiple matrices per template. */ class BR_EXPORT MetaTransform : public Transform @@ -1079,7 +1150,6 @@ private: void load(QDataStream &stream) { (void) stream; } }; - /*! * \brief A br::MetaTransform that does not require training data. */ diff --git a/sdk/plugins/meta.cpp b/sdk/plugins/meta.cpp index aa7561c..71257e0 100644 --- a/sdk/plugins/meta.cpp +++ b/sdk/plugins/meta.cpp @@ -86,6 +86,111 @@ static void incrementStep() } /*! + * \brief Use Expanded after basic calls that take a template list, used to implement ExpandTransform + */ +class ExpandDecorator : public Transform +{ + Q_OBJECT + + Q_PROPERTY(br::Transform* transform READ get_transform WRITE set_transform RESET reset_transform) + BR_PROPERTY(br::Transform*, transform, NULL) + +public: + ExpandDecorator(Transform * input) + { + transform = input; + transform->setParent(this); + file = transform->file; + setObjectName(transform->objectName()); + } + + void train(const TemplateList &data) + { + transform->train(data); + } + + void project(const Template &src, Template &dst) const + { + transform->project(src, dst); + } + + void project(const TemplateList &src, TemplateList &dst) const + { + transform->project(src, dst); + dst = Expanded(dst); + } + + + void projectUpdate(const Template &src, Template &dst) + { + transform->projectUpdate(src, dst); + } + + void projectUpdate(const TemplateList & src, TemplateList & dst) + { + transform->projectUpdate(src, dst); + dst = Expanded(dst); + } + + bool timeVarying() const + { + return transform->timeVarying(); + } + + void finalize(TemplateList & output) + { + transform->finalize(output); + output = Expanded(output); + } + +}; + +/*! + * \brief A MetaTransform that aggregates some sub-transforms + */ +class BR_EXPORT CompositeTransform : public TimeVaryingTransform +{ + Q_OBJECT + +public: + Q_PROPERTY(QList transforms READ get_transforms WRITE set_transforms RESET reset_transforms) + BR_PROPERTY(QList, transforms, QList()) + + virtual void project(const Template &src, Template &dst) const + { + if (timeVarying()) qFatal("No const project defined for time-varying transform"); + _project(src, dst); + } + + virtual void project(const TemplateList &src, TemplateList &dst) const + { + if (timeVarying()) qFatal("No const project defined for time-varying transform"); + _project(src, dst); + } + + bool timeVarying() const { return isTimeVarying; } + + void init() + { + isTimeVarying = false; + foreach (const br::Transform *transform, transforms) { + if (transform->timeVarying()) { + isTimeVarying = true; + break; + } + } + } + +protected: + bool isTimeVarying; + + virtual void _project(const Template & src, Template & dst) const = 0; + virtual void _project(const TemplateList & src, TemplateList & dst) const = 0; + + CompositeTransform() : TimeVaryingTransform(false) {} +}; + +/*! * \ingroup Transforms * \brief Transforms in series. * \author Josh Klontz \cite jklontz @@ -95,11 +200,9 @@ static void incrementStep() * \see ExpandTransform * \see ForkTransform */ -class PipeTransform : public MetaTransform +class PipeTransform : public CompositeTransform { Q_OBJECT - Q_PROPERTY(QList transforms READ get_transforms WRITE set_transforms RESET reset_transforms) - BR_PROPERTY(QList, transforms, QList()) void train(const TemplateList &data) { @@ -115,12 +218,32 @@ class PipeTransform : public MetaTransform releaseStep(); } - void project(const Template &src, Template &dst) const + void backProject(const Template &dst, Template &src) const + { + // Backprojecting a time-varying transform is probably not going to work. + if (timeVarying()) qFatal("No backProject defined for time-varying transform"); + + src = dst; + // Reverse order in which transforms are processed + int length = transforms.length(); + for (int i=length-1; i>=0; i--) { + Transform *f = transforms.at(i); + try { + src >> *f; + } catch (...) { + qWarning("Exception triggered when processing %s with transform %s", qPrintable(dst.file.flat()), qPrintable(f->objectName())); + src = Template(src.file); + src.file.setBool("FTE"); + } + } + } + + void projectUpdate(const Template &src, Template &dst) { dst = src; - foreach (const Transform *f, transforms) { + foreach (Transform *f, transforms) { try { - dst >> *f; + f->projectUpdate(dst); } catch (...) { qWarning("Exception triggered when processing %s with transform %s", qPrintable(src.file.flat()), qPrintable(f->objectName())); dst = Template(src.file); @@ -129,7 +252,44 @@ class PipeTransform : public MetaTransform } } - void project(const TemplateList &src, TemplateList &dst) const + // For time varying transforms, parallel execution over individual templates + // won't work. + void projectUpdate(const TemplateList & src, TemplateList & dst) + { + dst = src; + foreach (Transform *f, transforms) + { + f->projectUpdate(dst); + } + } + + virtual void finalize(TemplateList & output) + { + output.clear(); + // For each transform, + for (int i = 0; i < transforms.size(); i++) + { + + // Collect any final templates + TemplateList last_set; + transforms[i]->finalize(last_set); + if (last_set.empty()) + continue; + // Push any templates received through the remaining transforms in the sequence + for (int j = (i+1); j < transforms.size();j++) + { + transforms[j]->projectUpdate(last_set); + } + // append the result to the output set + output.append(last_set); + } + } + + +protected: + // Template list project -- process templates in parallel through Transform::project + // or if parallelism is disabled, handle them sequentially + void _project(const TemplateList &src, TemplateList &dst) const { if (Globals->parallelism < 0) { dst = src; @@ -140,22 +300,20 @@ class PipeTransform : public MetaTransform } } - void backProject(const Template &dst, Template &src) const - { - src = dst; - // Reverse order in which transforms are processed - int length = transforms.length(); - for (int i=length-1; i>=0; i--) { - Transform *f = transforms.at(i); - try { - src >> *f; - } catch (...) { - qWarning("Exception triggered when processing %s with transform %s", qPrintable(dst.file.flat()), qPrintable(f->objectName())); - src = Template(src.file); - src.file.setBool("FTE"); - } - } - } + // Single template const project, pass the template through each sub-transform, one after the other + virtual void _project(const Template & src, Template & dst) const + { + dst = src; + foreach (const Transform *f, transforms) { + try { + dst >> *f; + } catch (...) { + qWarning("Exception triggered when processing %s with transform %s", qPrintable(src.file.flat()), qPrintable(f->objectName())); + dst = Template(src.file); + dst.file.setBool("FTE"); + } + } + } }; BR_REGISTER(Transform, PipeTransform) @@ -170,64 +328,31 @@ BR_REGISTER(Transform, PipeTransform) * * \see PipeTransform */ -class ExpandTransform : public MetaTransform +class ExpandTransform : public PipeTransform { Q_OBJECT - Q_PROPERTY(QList transforms READ get_transforms WRITE set_transforms RESET reset_transforms) - BR_PROPERTY(QList, transforms, QList()) - void train(const TemplateList &data) + void init() { - acquireStep(); - - TemplateList copy(data); - for (int i=0; itrain(copy); - copy >> *transforms[i]; - copy = Expanded(copy); - incrementStep(); + for (int i = 0; i < transforms.size(); i++) + { + transforms[i] = new ExpandDecorator(transforms[i]); } - - releaseStep(); + // Need to call this to set up timevariance correctly, and it won't + // be called automatically + CompositeTransform::init(); } - void project(const Template &src, Template &dst) const - { - dst = src; - foreach (const Transform *f, transforms) { - try { - dst >> *f; - } catch (...) { - qWarning("Exception triggered when processing %s with transform %s", qPrintable(src.file.flat()), qPrintable(f->objectName())); - dst = Template(src.file); - dst.file.setBool("FTE"); - } - } - } +protected: - void project(const TemplateList &src, TemplateList &dst) const + // Template list project -- project through transforms sequentially, + // then expand the results, can't use Transform::Project(templateList) since + // we need to expand between tranforms, so actually do need to overload this method + void _project(const TemplateList &src, TemplateList &dst) const { dst = src; for (int i=0; i> *transforms[i]; - dst = Expanded(dst); - } - } - - void backProject(const Template &dst, Template &src) const - { - src = dst; - // Reverse order in which transforms are processed - int length = transforms.length(); - for (int i=length-1; i>=0; i--) { - Transform *f = transforms.at(i); - try { - src >> *f; - } catch (...) { - qWarning("Exception triggered when processing %s with transform %s", qPrintable(dst.file.flat()), qPrintable(f->objectName())); - src = Template(src.file); - src.file.setBool("FTE"); - } } } }; @@ -243,11 +368,9 @@ BR_REGISTER(Transform, ExpandTransform) * * \see PipeTransform */ -class ForkTransform : public MetaTransform +class ForkTransform : public CompositeTransform { Q_OBJECT - Q_PROPERTY(QList transforms READ get_transforms WRITE set_transforms RESET reset_transforms) - BR_PROPERTY(QList, transforms, QList()) void train(const TemplateList &data) { @@ -260,7 +383,69 @@ class ForkTransform : public MetaTransform if (threaded) Globals->trackFutures(futures); } - void project(const Template &src, Template &dst) const + void backProject(const Template &dst, Template &src) const {Transform::backProject(dst, src);} + + // same as _project, but calls projectUpdate on sub-transforms + void projectupdate(const Template & src, Template & dst) + { + foreach (Transform *f, transforms) { + try { + Template res; + f->projectUpdate(src, res); + dst.merge(res); + } catch (...) { + qWarning("Exception triggered when processing %s with transform %s", qPrintable(src.file.flat()), qPrintable(f->objectName())); + dst = Template(src.file); + dst.file.setBool("FTE"); + } + } + } + + void projectUpdate(const TemplateList & src, TemplateList & dst) + { + dst = src; + dst.reserve(src.size()); + for (int i=0; iprojectUpdate(src, m); + if (m.size() != dst.size()) qFatal("TemplateList is of an unexpected size."); + for (int i=0; ifinalize(last_set); + if (last_set.empty()) + continue; + + if (output.empty()) output = last_set; + else + { + // is the number of templates received from this transform consistent with the number + // received previously? If not we can't do anything coherent here. + if (last_set.size() != output.size()) + qFatal("mismatched template list sizes in ForkTransform"); + for (int j = 0; j < output.size(); j++) { + output[j].append(last_set[j]); + } + } + } + } + +protected: + + // Apply each transform to src, concatenate the results + void _project(const Template &src, Template &dst) const { foreach (const Transform *f, transforms) { try { @@ -273,7 +458,7 @@ class ForkTransform : public MetaTransform } } - void project(const TemplateList &src, TemplateList &dst) const + void _project(const TemplateList &src, TemplateList &dst) const { if (Globals->parallelism < 0) { dst.reserve(src.size()); @@ -288,6 +473,7 @@ class ForkTransform : public MetaTransform Transform::project(src, dst); } } + }; BR_REGISTER(Transform, ForkTransform) diff --git a/sdk/plugins/random.cpp b/sdk/plugins/random.cpp index a16eb81..a8b4d93 100644 --- a/sdk/plugins/random.cpp +++ b/sdk/plugins/random.cpp @@ -27,47 +27,6 @@ namespace br /*! * \ingroup transforms - * \brief Selects a random transform. - * \author Josh Klontz \cite jklontz - */ -class RndTransformTransform : public Transform -{ - Q_OBJECT - Q_PROPERTY(QList transforms READ get_transforms WRITE set_transforms RESET reset_transforms STORED false) - BR_PROPERTY(QList, transforms, QList()) - - int selectedIndex; - Transform *selectedTransform; - - void train(const TemplateList &data) - { - selectedIndex = theRNG().uniform(0, transforms.size()); - selectedTransform = transforms[selectedIndex]->clone(); - selectedTransform->train(data); - } - - void project(const Template &src, Template &dst) const - { - selectedTransform->project(src, dst); - } - - void store(QDataStream &stream) const - { - stream << selectedIndex << *selectedTransform; - } - - void load(QDataStream &stream) - { - stream >> selectedIndex; - selectedTransform = transforms[selectedIndex]->clone(); - stream >> *selectedTransform; - } -}; - -BR_REGISTER(Transform, RndTransformTransform) - -/*! - * \ingroup transforms * \brief Generates a random subspace. * \author Josh Klontz \cite jklontz */