Add a simplified interface for Stream

Now, Stream takes a single transform argument. The old interface remains, renamed to DirectStream. If that transform is a pipe, Stream will split its child transforms appropriately, and set up a DirectStream. Adjacent non-timeVarying transforms are grouped in pipes, time varying transforms all get separate stages. The default behavior isn't necessarily ideal if e.g. a lot of light time varying stages are present, in that case its possible the additional synchronization costs will outweight gains from paraellelism (due to having more separate single thread stages). In these cases DirectStream may be preferable, however current algorithms seem largely unaffected.

Add a simplified interface for Stream
Now, Stream takes a single transform argument. The old interface remains, renamed to DirectStream. If that transform is a pipe, Stream will split its child transforms appropriately, and set up a DirectStream. Adjacent non-timeVarying transforms are grouped in pipes, time varying transforms all get separate stages. The default behavior isn't necessarily ideal if e.g. a lot of light time varying stages are present, in that case its possible the additional synchronization costs will outweight gains from paraellelism (due to having more separate single thread stages). In these cases DirectStream may be preferable, however current algorithms seem largely unaffected.
Charles Otto
1 parent 4887d291
Showing 2 changed files with 157 additions and 11 deletions
openbr/plugins/algorithms.cpp
openbr/plugins/stream.cpp
@@ -46,12 +46,13 @@ class AlgorithmsInitializer : public Initializer
         Globals->abbreviations.insert("CropFace", "Open+Cvt(Gray)+Cascade(FrontalFace)+ASEFEyes+Affine(128,128,0.25,0.35)");
         // Video
-        Globals->abbreviations.insert("DisplayVideo", "Stream([FPSLimit(30)+Show(false,[FrameNumber])+Discard])");
-        Globals->abbreviations.insert("PerFrameDetection", "Stream([SaveMat(original)+Cvt(Gray)+Cascade(FrontalFace)+ASEFEyes+RestoreMat(original)+Draw(inPlace=true),Show(false,[FrameNumber])+Discard])");
-        Globals->abbreviations.insert("AgeGenderDemo", "Stream([SaveMat(original)+Cvt(Gray)+Cascade(FrontalFace)+Expand+<FaceClassificationRegistration>+<FaceClassificationExtraction>+<AgeRegressor>/<GenderClassifier>+Discard+RestoreMat(original)+Draw(inPlace=true)+DrawPropertiesPoint([Age,Gender],Affine_0,inPlace=true)+SaveMat(original)+Discard+Contract,RestoreMat(original)+FPSCalc+Show(false,[AvgFPS,Age,Gender])+Discard])");
-        Globals->abbreviations.insert("HOG", "Stream([DropFrames(5)+Cvt(Gray)+KeyPointDetector(SIFT)+ROI+Expand+Resize(32,32)+Gradient+RectRegions+Bin(0,360,8)+Hist(8)+Cat])+Contract+CatRows+KMeans(500)+Hist(500)+SVM");
-        Globals->abbreviations.insert("HOF", "Stream([DropFrames(5)+KeyPointDetector(SIFT),AggregateFrames(2),OpticalFlow+ROI+Expand+Resize(32,32)+Gradient+RectRegions+Bin(0,360,8)+Hist(8)+Cat])+Contract+CatRows+KMeans(500)+Hist(500)");
-        Globals->abbreviations.insert("HOGHOF", "Stream([DropFrames(5)+Cvt(Gray),KeyPointDetector(SIFT),AggregateFrames(2),(OpticalFlow+Gradient+Bin(0,360,8)+ROI+Hist(8))/(First+Gradient+Bin(0,360,8)+ROI+Hist(8)),CatCols])+Contract+CatRows+KMeans(500)+Hist(500)");
+        Globals->abbreviations.insert("DisplayVideo", "Stream(FPSLimit(30)+Show(false,[FrameNumber])+Discard)");
+        Globals->abbreviations.insert("PerFrameDetection", "Stream(SaveMat(original)+Cvt(Gray)+Cascade(FrontalFace)+ASEFEyes+RestoreMat(original)+Draw(inPlace=true)+Show(false,[FrameNumber])+Discard)");
+        Globals->abbreviations.insert("AgeGenderDemo", "Stream(SaveMat(original)+Cvt(Gray)+Cascade(FrontalFace)+Expand+<FaceClassificationRegistration>+<FaceClassificationExtraction>+<AgeRegressor>/<GenderClassifier>+Discard+RestoreMat(original)+Draw(inPlace=true)+DrawPropertiesPoint([Age,Gender],Affine_0,inPlace=true)+SaveMat(original)+Discard+Contract+RestoreMat(original)+FPSCalc+Show(false,[AvgFPS,Age,Gender])+Discard)");
+
+        Globals->abbreviations.insert("HOG", "Stream(DropFrames(5)+Cvt(Gray)+KeyPointDetector(SIFT)+ROI+Expand+Resize(32,32)+Gradient+RectRegions+Bin(0,360,8)+Hist(8)+Cat)+Contract+CatRows+KMeans(500)+Hist(500)+SVM");
+        Globals->abbreviations.insert("HOF", "Stream(DropFrames(5)+KeyPointDetector(SIFT)+AggregateFrames(2)+OpticalFlow+ROI+Expand+Resize(32,32)+Gradient+RectRegions+Bin(0,360,8)+Hist(8)+Cat)+Contract+CatRows+KMeans(500)+Hist(500)");
+        Globals->abbreviations.insert("HOGHOF", "Stream(DropFrames(5)+Cvt(Gray),KeyPointDetector(SIFT)+AggregateFrames(2)+(OpticalFlow+Gradient+Bin(0,360,8)+ROI+Hist(8))/(First+Gradient+Bin(0,360,8)+ROI+Hist(8))+CatCols)+Contract+CatRows+KMeans(500)+Hist(500)");
         // Generic Image Processing
         Globals->abbreviations.insert("SIFT", "Open+KeyPointDetector(SIFT)+KeyPointDescriptor(SIFT):KeyPointMatcher(BruteForce)");
@@ -649,7 +649,7 @@ public:
 class ProcessingStage
 {
 public:
-    friend class StreamTransform;
+    friend class DirectStreamTransform;
 public:
     ProcessingStage(int nThreads = 1)
     {
@@ -1015,13 +1015,16 @@ public:
 };
-class StreamTransform : public CompositeTransform
+
+class DirectStreamTransform : public CompositeTransform
 {
     Q_OBJECT
 public:
     Q_PROPERTY(int activeFrames READ get_activeFrames WRITE set_activeFrames RESET reset_activeFrames)
     BR_PROPERTY(int, activeFrames, 100)
+    friend class StreamTransfrom;
+
     void train(const TemplateList & data)
     {
         if (!trainable) {
@@ -1115,6 +1118,10 @@ public:
     {
         if (transforms.isEmpty()) return;
+        for (int i=0; i < processingStages.size();i++)
+            delete processingStages[i];
+        processingStages.clear();
+
         // call CompositeTransform::init so that trainable is set
         // correctly.
         CompositeTransform::init();
@@ -1190,12 +1197,13 @@ public:
         collectionStage->nextStage = readStage;
     }
-    ~StreamTransform()
+    ~DirectStreamTransform()
     {
         // Delete all the stages
         for (int i = 0; i < processingStages.size(); i++) {
             delete processingStages[i];
         }
+        processingStages.clear();
     }
 protected:
@@ -1237,12 +1245,149 @@ protected:
     }
 };
-QHash<QObject *, QThreadPool *> StreamTransform::pools;
-QMutex StreamTransform::poolsAccess;
+QHash<QObject *, QThreadPool *> DirectStreamTransform::pools;
+QMutex DirectStreamTransform::poolsAccess;
+
+BR_REGISTER(Transform, DirectStreamTransform)
+
+;
+
+class StreamTransform : public TimeVaryingTransform
+{
+    Q_OBJECT
+
+public:
+    StreamTransform() : TimeVaryingTransform(false)
+    {
+    }
+
+    Q_PROPERTY(br::Transform *transform READ get_transform WRITE set_transform STORED false)
+    Q_PROPERTY(int activeFrames READ get_activeFrames WRITE set_activeFrames RESET reset_activeFrames)
+    BR_PROPERTY(br::Transform *, transform, NULL)
+    BR_PROPERTY(int, activeFrames, 100)
+
+    bool timeVarying() const { return true; }
+
+    void project(const Template &src, Template &dst) const
+    {
+        basis.project(src,dst);
+    }
+
+    void projectUpdate(const Template &src, Template &dst)
+    {
+        basis.projectUpdate(src,dst);
+    }
+    void projectUpdate(const TemplateList & src, TemplateList & dst)
+    {
+        basis.projectUpdate(src,dst);
+    }
+
+
+    void train(const TemplateList & data)
+    {
+        basis.train(data);
+    }
+
+    virtual void finalize(TemplateList & output)
+    {
+        (void) output;
+        // Nothing in particular to do here, stream calls finalize
+        // on all child transforms as part of projectUpdate
+    }
+
+    // reinterpret transform, set up the actual stream. We can only reinterpret pipes
+    void init()
+    {
+        if (!transform)
+            return;
+
+        basis.setParent(this->parent());
+        basis.transforms.clear();
+        basis.activeFrames = this->activeFrames;
+
+        // We need at least a CompositeTransform * to acess transform's children.
+        CompositeTransform * downcast = dynamic_cast<CompositeTransform *> (transform);
+
+        // If this isn't even a composite transform, or it's not a pipe, just set up
+        // basis with 1 stage.
+        if (!downcast || QString(transform->metaObject()->className()) != "br::PipeTransform")
+        {
+            basis.transforms.append(transform);
+            basis.init();
+            return;
+        }
+        if (downcast->transforms.empty())
+        {
+            qWarning("Trying to set up empty stream");
+            basis.init();
+            return;
+        }
+
+        // OK now we will regroup downcast's children
+        QList<QList<Transform *> > sets;
+        sets.append(QList<Transform *> ());
+        sets.last().append(downcast->transforms[0]);
+        if (downcast->transforms[0]->timeVarying())
+            sets.append(QList<Transform *> ());
+
+        for (int i=1;i < downcast->transforms.size(); i++) {
+            // If this is time varying it becomse its own stage
+            if (downcast->transforms[i]->timeVarying()) {
+                // If a set was already active, we add another one
+                if (!sets.last().empty()) {
+                    sets.append(QList<Transform *>());
+                }
+                // add the item
+                sets.last().append(downcast->transforms[i]);
+                // Add another set to indicate separation.
+                sets.append(QList<Transform *>());
+            }
+            // otherwise, we can combine non time-varying stages
+            else {
+                sets.last().append(downcast->transforms[i]);
+            }
+
+        }
+        if (sets.last().empty())
+            sets.removeLast();
+
+        QList<Transform *> transform_set;
+        transform_set.reserve(sets.size());
+        for (int i=0; i < sets.size(); i++) {
+            // If this is a single transform set, we add that to the list
+            if (sets[i].size() == 1 ) {
+                transform_set.append(sets[i].at(0));
+            }
+            //otherwise we build a pipe
+            else {
+                CompositeTransform * pipe = dynamic_cast<CompositeTransform *>(Transform::make("Pipe([])", this));
+                pipe->transforms = sets[i];
+                pipe->init();
+                transform_set.append(pipe);
+            }
+        }
+
+        basis.transforms = transform_set;
+        basis.init();
+    }
+
+    Transform * smartCopy()
+    {
+        // We just want the DirectStream to begin with, so just return a copy of that.
+        DirectStreamTransform * res = (DirectStreamTransform *) basis.smartCopy();
+        res->activeFrames = this->activeFrames;
+        return res;
+    }
+
+
+private:
+    DirectStreamTransform basis;
+};
 BR_REGISTER(Transform, StreamTransform)
+
 } // namespace br
 #include "stream.moc"