Commit e2fb91d822245f82b00f49b21c6efa787e6036a6

Authored by jklontz
2 parents fe72ae12 9def2dd6

Merge pull request #23 from biometrics/video_thing

Changes to support better video processing, round 1 "a lot of work to accomplish nothing"
CHANGELOG.md
... ... @@ -4,6 +4,7 @@
4 4 * Enrolling files/folders are now sorted naturally instead of alpha numerically
5 5 * YouTubeFacesDBTransform implements Dr. Wolf's experimental protocol
6 6 * NEC3 refactored
  7 +* Updated transform API to add support for time-varying transforms
7 8  
8 9 0.2.0 - 2/23/13
9 10 ===============
... ...
sdk/openbr_plugin.cpp
... ... @@ -494,7 +494,8 @@ TemplateList TemplateList::relabel(const TemplateList &tl)
494 494 QStringList Object::parameters() const
495 495 {
496 496 QStringList parameters;
497   - for (int i=metaObject()->propertyOffset(); i<metaObject()->propertyCount(); i++) {
  497 +
  498 + for (int i = firstAvailablePropertyIdx; i < metaObject()->propertyCount();i++) {
498 499 QMetaProperty property = metaObject()->property(i);
499 500 if (property.isStored(this)) continue;
500 501 parameters.append(QString("%1 %2 = %3").arg(property.typeName(), property.name(), property.read(this).toString()));
... ... @@ -713,18 +714,37 @@ void Object::init(const File &amp;file_)
713 714 // Set name
714 715 QString name = metaObject()->className();
715 716 if (name.startsWith("br::")) name = name.right(name.size()-4);
716   - const QMetaObject *superClass = metaObject()->superClass();
  717 +
  718 + firstAvailablePropertyIdx = metaObject()->propertyCount();
  719 +
  720 + const QMetaObject * baseClass = metaObject();
  721 + const QMetaObject * superClass = metaObject()->superClass();
  722 +
717 723 while (superClass != NULL) {
  724 + const QMetaObject * nextClass = superClass->superClass();
  725 +
  726 + // baseClass <- something <- br::Object
  727 + // baseClass is the highest class whose properties we can set via positional arguments
  728 + if (nextClass && !strcmp(nextClass->className(),"br::Object")) {
  729 + firstAvailablePropertyIdx = baseClass->propertyOffset();
  730 + }
  731 +
718 732 QString superClassName = superClass->className();
  733 +
  734 + // strip br:: prefix from superclass name
719 735 if (superClassName.startsWith("br::"))
720 736 superClassName = superClassName.right(superClassName.size()-4);
  737 +
  738 + // Strip superclass name from base class name (e.g. PipeTransform -> Pipe)
721 739 if (name.endsWith(superClassName))
722 740 name = name.left(name.size() - superClassName.size());
  741 + baseClass = superClass;
723 742 superClass = superClass->superClass();
  743 +
724 744 }
725 745 setObjectName(name);
726 746  
727   - // Set properties
  747 + // Reset all properties
728 748 for (int i=0; i<metaObject()->propertyCount(); i++) {
729 749 QMetaProperty property = metaObject()->property(i);
730 750 if (property.isResettable())
... ... @@ -734,8 +754,17 @@ void Object::init(const File &amp;file_)
734 754  
735 755 foreach (QString key, file.localKeys()) {
736 756 const QString value = file.value(key).toString();
737   - if (key.startsWith("_Arg"))
738   - key = metaObject()->property(metaObject()->propertyOffset()+key.mid(4).toInt()).name();
  757 +
  758 + if (key.startsWith(("_Arg"))) {
  759 + int argument_number = key.mid(4).toInt();
  760 + int target_idx = argument_number + firstAvailablePropertyIdx;
  761 +
  762 + if (target_idx >= metaObject()->propertyCount()) {
  763 + qWarning("too many arguments for transform, ignoring %s\n", qPrintable(value));
  764 + continue;
  765 + }
  766 + key = metaObject()->property(target_idx).name();
  767 + }
739 768 setProperty(key, value);
740 769 }
741 770  
... ...
sdk/openbr_plugin.h
... ... @@ -421,6 +421,9 @@ class BR_EXPORT Object : public QObject
421 421 {
422 422 Q_OBJECT
423 423  
  424 + // Index of the first property that can be set via command line arguments
  425 + int firstAvailablePropertyIdx;
  426 +
424 427 public:
425 428 File file; /*!< \brief The file used to construct the plugin. */
426 429  
... ... @@ -989,6 +992,49 @@ public:
989 992 virtual void backProject(const Template &dst, Template &src) const; /*!< \brief Invert the transform. */
990 993 virtual void backProject(const TemplateList &dst, TemplateList &src) const; /*!< \brief Invert the transform. */
991 994  
  995 + /*!< \brief Apply the transform, may update the transform's internal state */
  996 + virtual void projectUpdate(const Template &src, Template &dst)
  997 + {
  998 + project(src, dst);
  999 + }
  1000 +
  1001 + /*!< \brief Apply the transform, may update the transform's internal state */
  1002 + virtual void projectUpdate(const TemplateList &src, TemplateList &dst)
  1003 + {
  1004 + project(src,dst);
  1005 + }
  1006 +
  1007 + /*!< \brief inplace projectUpdate. */
  1008 + void projectUpdate(Template &srcdst)
  1009 + {
  1010 + Template dst;
  1011 + projectUpdate(srcdst, dst);
  1012 + srcdst = dst;
  1013 + }
  1014 +
  1015 + /*!< \brief inplace projectUpdate. */
  1016 + void projectUpdate(TemplateList &srcdst)
  1017 + {
  1018 + TemplateList dst;
  1019 + projectUpdate(srcdst, dst);
  1020 + srcdst = dst;
  1021 + }
  1022 +
  1023 + /*!
  1024 + * Time-varying transforms may move away from a single input->single output model, and only emit
  1025 + * templates under some conditions (e.g. a tracking thing may emit a template for each detected
  1026 + * unique object), in this case finalize indicates that no further calls to project will be made
  1027 + * and the transform can emit a final set if templates if it wants. Time-invariant transforms
  1028 + * don't have to do anything.
  1029 + */
  1030 + virtual void finalize(TemplateList & output) { output = TemplateList(); }
  1031 +
  1032 + /*!
  1033 + * \brief Does the transform require the non-const version of project? Can vary for aggregation type transforms
  1034 + * (if their children are time varying, they are also time varying, otherwise probably not)
  1035 + */
  1036 + virtual bool timeVarying() const { return false; }
  1037 +
992 1038 /*!
993 1039 * \brief Convenience function equivalent to project().
994 1040 */
... ... @@ -1052,6 +1098,31 @@ inline QDataStream &amp;operator&gt;&gt;(QDataStream &amp;stream, Transform &amp;f)
1052 1098 }
1053 1099  
1054 1100 /*!
  1101 + * \brief A br::Transform for which the results of project may change due to prior calls to project
  1102 + */
  1103 +class BR_EXPORT TimeVaryingTransform : public Transform
  1104 +{
  1105 + Q_OBJECT
  1106 +
  1107 + virtual bool timeVarying() const { return true; }
  1108 +
  1109 + virtual void project(const Template &src, Template &dst) const
  1110 + {
  1111 + qFatal("No const project defined for time-varying transform");
  1112 + (void) dst; (void) src;
  1113 + }
  1114 +
  1115 + virtual void project(const TemplateList &src, TemplateList &dst) const
  1116 + {
  1117 + qFatal("No const project defined for time-varying transform");
  1118 + (void) dst; (void) src;
  1119 + }
  1120 +
  1121 +protected:
  1122 + TimeVaryingTransform(bool independent = true, bool trainable = true) : Transform(independent, trainable) {}
  1123 +};
  1124 +
  1125 +/*!
1055 1126 * \brief A br::Transform expecting multiple matrices per template.
1056 1127 */
1057 1128 class BR_EXPORT MetaTransform : public Transform
... ... @@ -1079,7 +1150,6 @@ private:
1079 1150 void load(QDataStream &stream) { (void) stream; }
1080 1151 };
1081 1152  
1082   -
1083 1153 /*!
1084 1154 * \brief A br::MetaTransform that does not require training data.
1085 1155 */
... ...
sdk/plugins/meta.cpp
... ... @@ -86,6 +86,111 @@ static void incrementStep()
86 86 }
87 87  
88 88 /*!
  89 + * \brief Use Expanded after basic calls that take a template list, used to implement ExpandTransform
  90 + */
  91 +class ExpandDecorator : public Transform
  92 +{
  93 + Q_OBJECT
  94 +
  95 + Q_PROPERTY(br::Transform* transform READ get_transform WRITE set_transform RESET reset_transform)
  96 + BR_PROPERTY(br::Transform*, transform, NULL)
  97 +
  98 +public:
  99 + ExpandDecorator(Transform * input)
  100 + {
  101 + transform = input;
  102 + transform->setParent(this);
  103 + file = transform->file;
  104 + setObjectName(transform->objectName());
  105 + }
  106 +
  107 + void train(const TemplateList &data)
  108 + {
  109 + transform->train(data);
  110 + }
  111 +
  112 + void project(const Template &src, Template &dst) const
  113 + {
  114 + transform->project(src, dst);
  115 + }
  116 +
  117 + void project(const TemplateList &src, TemplateList &dst) const
  118 + {
  119 + transform->project(src, dst);
  120 + dst = Expanded(dst);
  121 + }
  122 +
  123 +
  124 + void projectUpdate(const Template &src, Template &dst)
  125 + {
  126 + transform->projectUpdate(src, dst);
  127 + }
  128 +
  129 + void projectUpdate(const TemplateList & src, TemplateList & dst)
  130 + {
  131 + transform->projectUpdate(src, dst);
  132 + dst = Expanded(dst);
  133 + }
  134 +
  135 + bool timeVarying() const
  136 + {
  137 + return transform->timeVarying();
  138 + }
  139 +
  140 + void finalize(TemplateList & output)
  141 + {
  142 + transform->finalize(output);
  143 + output = Expanded(output);
  144 + }
  145 +
  146 +};
  147 +
  148 +/*!
  149 + * \brief A MetaTransform that aggregates some sub-transforms
  150 + */
  151 +class BR_EXPORT CompositeTransform : public TimeVaryingTransform
  152 +{
  153 + Q_OBJECT
  154 +
  155 +public:
  156 + Q_PROPERTY(QList<br::Transform*> transforms READ get_transforms WRITE set_transforms RESET reset_transforms)
  157 + BR_PROPERTY(QList<br::Transform*>, transforms, QList<br::Transform*>())
  158 +
  159 + virtual void project(const Template &src, Template &dst) const
  160 + {
  161 + if (timeVarying()) qFatal("No const project defined for time-varying transform");
  162 + _project(src, dst);
  163 + }
  164 +
  165 + virtual void project(const TemplateList &src, TemplateList &dst) const
  166 + {
  167 + if (timeVarying()) qFatal("No const project defined for time-varying transform");
  168 + _project(src, dst);
  169 + }
  170 +
  171 + bool timeVarying() const { return isTimeVarying; }
  172 +
  173 + void init()
  174 + {
  175 + isTimeVarying = false;
  176 + foreach (const br::Transform *transform, transforms) {
  177 + if (transform->timeVarying()) {
  178 + isTimeVarying = true;
  179 + break;
  180 + }
  181 + }
  182 + }
  183 +
  184 +protected:
  185 + bool isTimeVarying;
  186 +
  187 + virtual void _project(const Template & src, Template & dst) const = 0;
  188 + virtual void _project(const TemplateList & src, TemplateList & dst) const = 0;
  189 +
  190 + CompositeTransform() : TimeVaryingTransform(false) {}
  191 +};
  192 +
  193 +/*!
89 194 * \ingroup Transforms
90 195 * \brief Transforms in series.
91 196 * \author Josh Klontz \cite jklontz
... ... @@ -95,11 +200,9 @@ static void incrementStep()
95 200 * \see ExpandTransform
96 201 * \see ForkTransform
97 202 */
98   -class PipeTransform : public MetaTransform
  203 +class PipeTransform : public CompositeTransform
99 204 {
100 205 Q_OBJECT
101   - Q_PROPERTY(QList<br::Transform*> transforms READ get_transforms WRITE set_transforms RESET reset_transforms)
102   - BR_PROPERTY(QList<br::Transform*>, transforms, QList<br::Transform*>())
103 206  
104 207 void train(const TemplateList &data)
105 208 {
... ... @@ -115,12 +218,32 @@ class PipeTransform : public MetaTransform
115 218 releaseStep();
116 219 }
117 220  
118   - void project(const Template &src, Template &dst) const
  221 + void backProject(const Template &dst, Template &src) const
  222 + {
  223 + // Backprojecting a time-varying transform is probably not going to work.
  224 + if (timeVarying()) qFatal("No backProject defined for time-varying transform");
  225 +
  226 + src = dst;
  227 + // Reverse order in which transforms are processed
  228 + int length = transforms.length();
  229 + for (int i=length-1; i>=0; i--) {
  230 + Transform *f = transforms.at(i);
  231 + try {
  232 + src >> *f;
  233 + } catch (...) {
  234 + qWarning("Exception triggered when processing %s with transform %s", qPrintable(dst.file.flat()), qPrintable(f->objectName()));
  235 + src = Template(src.file);
  236 + src.file.setBool("FTE");
  237 + }
  238 + }
  239 + }
  240 +
  241 + void projectUpdate(const Template &src, Template &dst)
119 242 {
120 243 dst = src;
121   - foreach (const Transform *f, transforms) {
  244 + foreach (Transform *f, transforms) {
122 245 try {
123   - dst >> *f;
  246 + f->projectUpdate(dst);
124 247 } catch (...) {
125 248 qWarning("Exception triggered when processing %s with transform %s", qPrintable(src.file.flat()), qPrintable(f->objectName()));
126 249 dst = Template(src.file);
... ... @@ -129,7 +252,44 @@ class PipeTransform : public MetaTransform
129 252 }
130 253 }
131 254  
132   - void project(const TemplateList &src, TemplateList &dst) const
  255 + // For time varying transforms, parallel execution over individual templates
  256 + // won't work.
  257 + void projectUpdate(const TemplateList & src, TemplateList & dst)
  258 + {
  259 + dst = src;
  260 + foreach (Transform *f, transforms)
  261 + {
  262 + f->projectUpdate(dst);
  263 + }
  264 + }
  265 +
  266 + virtual void finalize(TemplateList & output)
  267 + {
  268 + output.clear();
  269 + // For each transform,
  270 + for (int i = 0; i < transforms.size(); i++)
  271 + {
  272 +
  273 + // Collect any final templates
  274 + TemplateList last_set;
  275 + transforms[i]->finalize(last_set);
  276 + if (last_set.empty())
  277 + continue;
  278 + // Push any templates received through the remaining transforms in the sequence
  279 + for (int j = (i+1); j < transforms.size();j++)
  280 + {
  281 + transforms[j]->projectUpdate(last_set);
  282 + }
  283 + // append the result to the output set
  284 + output.append(last_set);
  285 + }
  286 + }
  287 +
  288 +
  289 +protected:
  290 + // Template list project -- process templates in parallel through Transform::project
  291 + // or if parallelism is disabled, handle them sequentially
  292 + void _project(const TemplateList &src, TemplateList &dst) const
133 293 {
134 294 if (Globals->parallelism < 0) {
135 295 dst = src;
... ... @@ -140,22 +300,20 @@ class PipeTransform : public MetaTransform
140 300 }
141 301 }
142 302  
143   - void backProject(const Template &dst, Template &src) const
144   - {
145   - src = dst;
146   - // Reverse order in which transforms are processed
147   - int length = transforms.length();
148   - for (int i=length-1; i>=0; i--) {
149   - Transform *f = transforms.at(i);
150   - try {
151   - src >> *f;
152   - } catch (...) {
153   - qWarning("Exception triggered when processing %s with transform %s", qPrintable(dst.file.flat()), qPrintable(f->objectName()));
154   - src = Template(src.file);
155   - src.file.setBool("FTE");
156   - }
157   - }
158   - }
  303 + // Single template const project, pass the template through each sub-transform, one after the other
  304 + virtual void _project(const Template & src, Template & dst) const
  305 + {
  306 + dst = src;
  307 + foreach (const Transform *f, transforms) {
  308 + try {
  309 + dst >> *f;
  310 + } catch (...) {
  311 + qWarning("Exception triggered when processing %s with transform %s", qPrintable(src.file.flat()), qPrintable(f->objectName()));
  312 + dst = Template(src.file);
  313 + dst.file.setBool("FTE");
  314 + }
  315 + }
  316 + }
159 317 };
160 318  
161 319 BR_REGISTER(Transform, PipeTransform)
... ... @@ -170,64 +328,31 @@ BR_REGISTER(Transform, PipeTransform)
170 328 *
171 329 * \see PipeTransform
172 330 */
173   -class ExpandTransform : public MetaTransform
  331 +class ExpandTransform : public PipeTransform
174 332 {
175 333 Q_OBJECT
176   - Q_PROPERTY(QList<br::Transform*> transforms READ get_transforms WRITE set_transforms RESET reset_transforms)
177   - BR_PROPERTY(QList<br::Transform*>, transforms, QList<br::Transform*>())
178 334  
179   - void train(const TemplateList &data)
  335 + void init()
180 336 {
181   - acquireStep();
182   -
183   - TemplateList copy(data);
184   - for (int i=0; i<transforms.size(); i++) {
185   - transforms[i]->train(copy);
186   - copy >> *transforms[i];
187   - copy = Expanded(copy);
188   - incrementStep();
  337 + for (int i = 0; i < transforms.size(); i++)
  338 + {
  339 + transforms[i] = new ExpandDecorator(transforms[i]);
189 340 }
190   -
191   - releaseStep();
  341 + // Need to call this to set up timevariance correctly, and it won't
  342 + // be called automatically
  343 + CompositeTransform::init();
192 344 }
193 345  
194   - void project(const Template &src, Template &dst) const
195   - {
196   - dst = src;
197   - foreach (const Transform *f, transforms) {
198   - try {
199   - dst >> *f;
200   - } catch (...) {
201   - qWarning("Exception triggered when processing %s with transform %s", qPrintable(src.file.flat()), qPrintable(f->objectName()));
202   - dst = Template(src.file);
203   - dst.file.setBool("FTE");
204   - }
205   - }
206   - }
  346 +protected:
207 347  
208   - void project(const TemplateList &src, TemplateList &dst) const
  348 + // Template list project -- project through transforms sequentially,
  349 + // then expand the results, can't use Transform::Project(templateList) since
  350 + // we need to expand between tranforms, so actually do need to overload this method
  351 + void _project(const TemplateList &src, TemplateList &dst) const
209 352 {
210 353 dst = src;
211 354 for (int i=0; i<transforms.size(); i++) {
212 355 dst >> *transforms[i];
213   - dst = Expanded(dst);
214   - }
215   - }
216   -
217   - void backProject(const Template &dst, Template &src) const
218   - {
219   - src = dst;
220   - // Reverse order in which transforms are processed
221   - int length = transforms.length();
222   - for (int i=length-1; i>=0; i--) {
223   - Transform *f = transforms.at(i);
224   - try {
225   - src >> *f;
226   - } catch (...) {
227   - qWarning("Exception triggered when processing %s with transform %s", qPrintable(dst.file.flat()), qPrintable(f->objectName()));
228   - src = Template(src.file);
229   - src.file.setBool("FTE");
230   - }
231 356 }
232 357 }
233 358 };
... ... @@ -243,11 +368,9 @@ BR_REGISTER(Transform, ExpandTransform)
243 368 *
244 369 * \see PipeTransform
245 370 */
246   -class ForkTransform : public MetaTransform
  371 +class ForkTransform : public CompositeTransform
247 372 {
248 373 Q_OBJECT
249   - Q_PROPERTY(QList<br::Transform*> transforms READ get_transforms WRITE set_transforms RESET reset_transforms)
250   - BR_PROPERTY(QList<br::Transform*>, transforms, QList<br::Transform*>())
251 374  
252 375 void train(const TemplateList &data)
253 376 {
... ... @@ -260,7 +383,69 @@ class ForkTransform : public MetaTransform
260 383 if (threaded) Globals->trackFutures(futures);
261 384 }
262 385  
263   - void project(const Template &src, Template &dst) const
  386 + void backProject(const Template &dst, Template &src) const {Transform::backProject(dst, src);}
  387 +
  388 + // same as _project, but calls projectUpdate on sub-transforms
  389 + void projectupdate(const Template & src, Template & dst)
  390 + {
  391 + foreach (Transform *f, transforms) {
  392 + try {
  393 + Template res;
  394 + f->projectUpdate(src, res);
  395 + dst.merge(res);
  396 + } catch (...) {
  397 + qWarning("Exception triggered when processing %s with transform %s", qPrintable(src.file.flat()), qPrintable(f->objectName()));
  398 + dst = Template(src.file);
  399 + dst.file.setBool("FTE");
  400 + }
  401 + }
  402 + }
  403 +
  404 + void projectUpdate(const TemplateList & src, TemplateList & dst)
  405 + {
  406 + dst = src;
  407 + dst.reserve(src.size());
  408 + for (int i=0; i<src.size(); i++) dst.append(Template());
  409 + foreach (Transform *f, transforms) {
  410 + TemplateList m;
  411 + f->projectUpdate(src, m);
  412 + if (m.size() != dst.size()) qFatal("TemplateList is of an unexpected size.");
  413 + for (int i=0; i<src.size(); i++) dst[i].append(m[i]);
  414 + }
  415 + }
  416 +
  417 + // this is probably going to go bad, fork transform probably won't work well in a variable
  418 + // input/output scenario
  419 + virtual void finalize(TemplateList & output)
  420 + {
  421 + output.clear();
  422 + // For each transform,
  423 + for (int i = 0; i < transforms.size(); i++)
  424 + {
  425 + // Collect any final templates
  426 + TemplateList last_set;
  427 + transforms[i]->finalize(last_set);
  428 + if (last_set.empty())
  429 + continue;
  430 +
  431 + if (output.empty()) output = last_set;
  432 + else
  433 + {
  434 + // is the number of templates received from this transform consistent with the number
  435 + // received previously? If not we can't do anything coherent here.
  436 + if (last_set.size() != output.size())
  437 + qFatal("mismatched template list sizes in ForkTransform");
  438 + for (int j = 0; j < output.size(); j++) {
  439 + output[j].append(last_set[j]);
  440 + }
  441 + }
  442 + }
  443 + }
  444 +
  445 +protected:
  446 +
  447 + // Apply each transform to src, concatenate the results
  448 + void _project(const Template &src, Template &dst) const
264 449 {
265 450 foreach (const Transform *f, transforms) {
266 451 try {
... ... @@ -273,7 +458,7 @@ class ForkTransform : public MetaTransform
273 458 }
274 459 }
275 460  
276   - void project(const TemplateList &src, TemplateList &dst) const
  461 + void _project(const TemplateList &src, TemplateList &dst) const
277 462 {
278 463 if (Globals->parallelism < 0) {
279 464 dst.reserve(src.size());
... ... @@ -288,6 +473,7 @@ class ForkTransform : public MetaTransform
288 473 Transform::project(src, dst);
289 474 }
290 475 }
  476 +
291 477 };
292 478  
293 479 BR_REGISTER(Transform, ForkTransform)
... ...
sdk/plugins/random.cpp
... ... @@ -27,47 +27,6 @@ namespace br
27 27  
28 28 /*!
29 29 * \ingroup transforms
30   - * \brief Selects a random transform.
31   - * \author Josh Klontz \cite jklontz
32   - */
33   -class RndTransformTransform : public Transform
34   -{
35   - Q_OBJECT
36   - Q_PROPERTY(QList<br::Transform*> transforms READ get_transforms WRITE set_transforms RESET reset_transforms STORED false)
37   - BR_PROPERTY(QList<br::Transform*>, transforms, QList<br::Transform*>())
38   -
39   - int selectedIndex;
40   - Transform *selectedTransform;
41   -
42   - void train(const TemplateList &data)
43   - {
44   - selectedIndex = theRNG().uniform(0, transforms.size());
45   - selectedTransform = transforms[selectedIndex]->clone();
46   - selectedTransform->train(data);
47   - }
48   -
49   - void project(const Template &src, Template &dst) const
50   - {
51   - selectedTransform->project(src, dst);
52   - }
53   -
54   - void store(QDataStream &stream) const
55   - {
56   - stream << selectedIndex << *selectedTransform;
57   - }
58   -
59   - void load(QDataStream &stream)
60   - {
61   - stream >> selectedIndex;
62   - selectedTransform = transforms[selectedIndex]->clone();
63   - stream >> *selectedTransform;
64   - }
65   -};
66   -
67   -BR_REGISTER(Transform, RndTransformTransform)
68   -
69   -/*!
70   - * \ingroup transforms
71 30 * \brief Generates a random subspace.
72 31 * \author Josh Klontz \cite jklontz
73 32 */
... ...