Integrating OpenCV frontend

Jordan Cheney
1 parent dfea239c
Showing 5 changed files with 172 additions and 35 deletions
openbr/openbr_plugin.h
openbr/plugins/classification/boostedforest.cpp
openbr/plugins/classification/cascade.cpp
openbr/plugins/imgproc/slidingwindow.cpp
openbr/plugins/representation/mblbp.cpp
@@ -1411,9 +1411,6 @@ public:
     virtual cv::Size postWindowSize() const = 0; // window size after preprocessing
     virtual int numFeatures() const = 0;
     virtual int maxCatCount() const = 0;
-
-    // Temporary for OpenCV compatibility
-    virtual void write( cv::FileStorage &fs, const cv::Mat &featureMap ) { (void)fs; (void)featureMap; }
 };
 class BR_EXPORT Classifier : public Object
@@ -23,7 +23,6 @@ static void buildTreeRecursive(Node *node, const CvDTreeNode *cv_node, int maxCa
 {
     if (!cv_node->left) {
         node->value = cv_node->value;
-
         node->left = node->right = NULL;
     } else {
         if (maxCatCount > 0)
@@ -40,6 +39,29 @@ static void buildTreeRecursive(Node *node, const CvDTreeNode *cv_node, int maxCa
     }
 }
+static void readRecursive(const FileNode &fn, Node *node, int maxCatCount)
+{
+    bool hasChildren = (int)fn["hasChildren"];
+    if (!hasChildren) {
+        node->value = (float)fn["value"];
+        node->left = node->right = NULL;
+    } else {
+        if (maxCatCount > 0) {
+            FileNode subset_fn = fn["subset"];
+            for (FileNodeIterator subset_it = subset_fn.begin(); subset_it != subset_fn.end(); ++subset_it)
+                node->subset.append((int)*subset_it);
+        } else {
+            node->threshold = (float)fn["threshold"];
+        }
+
+        node->featureIdx = (int)fn["featureIdx"];
+
+        node->left = new Node; node->right = new Node;
+        readRecursive(fn["left"], node->left, maxCatCount);
+        readRecursive(fn["right"], node->right, maxCatCount);
+    }
+}
+
 static void writeRecursive(FileStorage &fs, const Node *node, int maxCatCount)
 {
     bool hasChildren = node->left ? true : false;
@@ -119,16 +141,17 @@ class BoostedForestClassifier : public Classifier
             while (node->left) {
                 if (representation->maxCatCount() > 1) {
                     int c = (int)representation->evaluate(image, node->featureIdx);
-                    node = (2*((node->subset[c >> 5] & (1 << (c & 31))) == 0) - 1) < 0 ? node->left : node->right;
+                    node = (node->subset[c >> 5] & (1 << (c & 31))) ? node->left : node->right;
                 } else {
                     double val = representation->evaluate(image, node->featureIdx);
                     node = val <= node->threshold ? node->left : node->right;
                 }
             }
+            qDebug("value: %f", node->value);
             sum += node->value;
         }
-        return sum < threshold - THRESHOLD_EPS ? 0.0f : 1.0f;
+        return sum < threshold - THRESHOLD_EPS ? -std::abs(sum) : std::abs(sum);
     }
     int numFeatures() const
@@ -146,6 +169,17 @@ class BoostedForestClassifier : public Classifier
         return representation->preWindowSize();
     }
+    void read(const FileNode &node)
+    {
+        threshold = (float)node["stageThreshold"];
+        FileNode weaks_fn = node["weakClassifiers"];
+        for (FileNodeIterator weaks_it = weaks_fn.begin(); weaks_it != weaks_fn.end(); ++weaks_it) {
+            Node *root = new Node;
+            readRecursive(*weaks_it, root, representation->maxCatCount());
+            classifiers.append(root);
+        }
+    }
+
     void write(FileStorage &fs) const
     {
         fs << "stageThreshold" << threshold;
@@ -154,10 +154,16 @@ class CascadeClassifier : public Classifier
     float classify(const Mat &image) const
     {
-        foreach (const Classifier *stage, stages)
-            if (stage->classify(image) == 0.0f)
-                return 0.0f;
-        return 1.0f;
+        if (stages.empty())
+             return 1.0f;
+
+        float val = 0.0f;
+        for (int i = 0; i < stages.size(); i++) {
+            val = stages[i]->classify(image);
+            if (val < 0.0f)
+                return stages.size() - i < 4 ? i * val : 0.0f;
+        }
+        return stages.size() * val;
     }
     int numFeatures() const
@@ -175,6 +181,16 @@ class CascadeClassifier : public Classifier
         return stages.first()->windowSize();
     }
+    void read(const FileNode &node)
+    {
+        FileNode stages_fn = node["stages"];
+        for (FileNodeIterator stages_it = stages_fn.begin(); stages_it != stages_fn.end(); ++stages_it) {
+            Classifier *nextStage = Classifier::make(stageDescription, NULL);
+            nextStage->read(*stages_it);
+            stages.append(nextStage);
+        }
+    }
+
     void write(FileStorage &fs) const
     {
         fs << "stages" << "[";
@@ -196,7 +212,7 @@ private:
             if (!imgHandler.getPos(pos))
                 qFatal("Cannot get another positive sample!");
-            if (classify(pos) == 1.0f) {
+            if (classify(pos) > 0.0f) {
                 printf("POS current samples: %d\r", images.size());
                 images.append(pos);
                 labels.append(1.0f);
@@ -212,7 +228,7 @@ private:
             if (!imgHandler.getNeg(neg))
                 qFatal("Cannot get another negative sample!");
-            if (classify(neg) == 1.0f) {
+            if (classify(neg) > 0.0f) {
                 printf("NEG current samples: %d\r", images.size() - posCount);
                 images.append(neg);
                 labels.append(0.0f);
@@ -19,8 +19,10 @@
 #include <openbr/plugins/openbr_internal.h>
 #include <openbr/core/opencvutils.h>
 #include <openbr/core/qtutils.h>
+#include <openbr/core/cascade.h>
 #include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
 using namespace cv;
@@ -38,8 +40,20 @@ class SlidingWindowTransform : public Transform
     Q_OBJECT
     Q_PROPERTY(br::Classifier *classifier READ get_classifier WRITE set_classifier RESET reset_classifier STORED false)
+    Q_PROPERTY(int minSize READ get_minSize WRITE set_minSize RESET reset_minSize STORED false)
+    Q_PROPERTY(int maxSize READ get_maxSize WRITE set_maxSize RESET reset_maxSize STORED false)
+    Q_PROPERTY(float scaleFactor READ get_scaleFactor WRITE set_scaleFactor RESET reset_scaleFactor STORED false)
+    Q_PROPERTY(int minNeighbors READ get_minNeighbors WRITE set_minNeighbors RESET reset_minNeighbors STORED false)
+    Q_PROPERTY(float eps READ get_eps WRITE set_eps RESET reset_eps STORED false)
+
     Q_PROPERTY(QString cascadeDir READ get_cascadeDir WRITE set_cascadeDir RESET reset_cascadeDir STORED false)
     BR_PROPERTY(br::Classifier *, classifier, NULL)
+    BR_PROPERTY(int, minSize, 20)
+    BR_PROPERTY(int, maxSize, -1)
+    BR_PROPERTY(float, scaleFactor, 1.2)
+    BR_PROPERTY(int, minNeighbors, 5)
+    BR_PROPERTY(float, eps, 0.2)
+
     BR_PROPERTY(QString, cascadeDir, "")
     void train(const TemplateList &data)
@@ -49,12 +63,104 @@ class SlidingWindowTransform : public Transform
     void project(const Template &src, Template &dst) const
     {
-        (void)src; (void)dst;
+        TemplateList temp;
+        project(TemplateList() << src, temp);
+        if (!temp.isEmpty()) dst = temp.first();
     }
+    void project(const TemplateList &src, TemplateList &dst) const
+    {
+        foreach (const Template &t, src) {
+            const bool enrollAll = t.file.getBool("enrollAll");
+
+            // Mirror the behavior of ExpandTransform in the special case
+            // of an empty template.
+            if (t.empty() && !enrollAll) {
+                dst.append(t);
+                continue;
+            }
+
+            for (int i = 0; i < t.size(); i++) {
+                Mat image;
+                OpenCVUtils::cvtUChar(t[i], image);
+
+                std::vector<Rect> rects;
+                std::vector<int> rejectLevels;
+                std::vector<double> levelWeights;
+
+                Size minObjectSize(minSize, minSize);
+                Size maxObjectSize(maxSize, maxSize);
+                if (maxObjectSize.height <= 0 || maxObjectSize.width <= 0)
+                    maxObjectSize = image.size();
+
+                Mat imageBuffer(image.rows + 1, image.cols + 1, CV_8U);
+
+                for (double factor = 1; ; factor *= scaleFactor) {
+                    Size originalWindowSize = classifier->windowSize();
+
+                    Size windowSize(cvRound(originalWindowSize.width*factor), cvRound(originalWindowSize.height*factor) );
+                    Size scaledImageSize(cvRound(image.cols/factor ), cvRound(image.rows/factor));
+                    Size processingRectSize(scaledImageSize.width - originalWindowSize.width, scaledImageSize.height - originalWindowSize.height);
+
+                    if (processingRectSize.width <= 0 || processingRectSize.height <= 0)
+                        break;
+                    if (windowSize.width > maxObjectSize.width || windowSize.height > maxObjectSize.height)
+                        break;
+                    if (windowSize.width < minObjectSize.width || windowSize.height < minObjectSize.height)
+                        continue;
+
+                    Mat scaledImage(scaledImageSize, CV_8U, imageBuffer.data);
+                    resize(image, scaledImage, scaledImageSize, 0, 0, CV_INTER_LINEAR);
+
+                    int yStep = factor > 2. ? 1 : 2;
+                    for (int y = 0; y < processingRectSize.height; y += yStep) {
+                        for (int x = 0; x < processingRectSize.width; x += yStep) {
+                            Mat window = scaledImage(Rect(Point(x, y), classifier->windowSize())).clone();
+
+                            float result = classifier->classify(window);
+                            qDebug("result: %f", result);
+                            if (result > 0) {
+                                rects.push_back(Rect(cvRound(x*factor), cvRound(y*factor), windowSize.width, windowSize.height));
+                                rejectLevels.push_back(1);
+                                levelWeights.push_back(result);
+                            }
+                            if (result == 0)
+                                x = yStep;
+                        }
+                    }
+                }
+
+                groupRectangles(rects, rejectLevels, levelWeights, minNeighbors, eps);
+
+                if (!enrollAll && rects.empty())
+                    rects.push_back(Rect(0, 0, image.cols, image.rows));
+
+                for (size_t j = 0; j < rects.size(); j++) {
+                    Template u(t.file, image);
+                    if (rejectLevels.size() > j)
+                        u.file.set("Confidence", rejectLevels[j]*levelWeights[j]);
+                    else
+                        u.file.set("Confidence", 1);
+                    const QRectF rect = OpenCVUtils::fromRect(rects[j]);
+                    u.file.appendRect(rect);
+                    u.file.set("Face", rect);
+                    dst.append(u);
+                }
+            }
+        }
+     }
+
     void load(QDataStream &stream)
     {
         (void) stream;
+
+        QString filename = Globals->sdkPath + "/share/openbr/models/openbrcascades/" + cascadeDir + "/cascade.xml";
+        FileStorage fs(filename.toStdString(), FileStorage::READ);
+        if (!fs.isOpened())
+            return;
+
+        classifier->read(fs.getFirstTopLevelNode());
+
         return;
     }
@@ -62,17 +168,17 @@ class SlidingWindowTransform : public Transform
     {
         (void) stream;
-	QString path = Globals->sdkPath + "/share/openbr/models/openbrcascades/" + cascadeDir;
-	QtUtils::touchDir(QDir(path));
+        QString path = Globals->sdkPath + "/share/openbr/models/openbrcascades/" + cascadeDir;
+        QtUtils::touchDir(QDir(path));
-	QString filename = path + "/cascade.xml";
+        QString filename = path + "/cascade.xml";
         FileStorage fs(filename.toStdString(), FileStorage::WRITE);
         if (!fs.isOpened()) {
-	    qWarning("Unable to open file: %s", qPrintable(filename));
+            qWarning("Unable to open file: %s", qPrintable(filename));
             return;
-	}
-	
+        }
+
         fs << FileStorage::getDefaultObjectName(filename.toStdString()) << "{";
         classifier->write(fs);
@@ -58,7 +58,6 @@ class MBLBPRepresentation : public Representation
         return result;
     }
-    void write(FileStorage &fs, const Mat &featureMap);
     int numFeatures() const { return features.size(); }
     Size preWindowSize() const { return Size(winWidth, winHeight); }
     Size postWindowSize() const { return Size(winWidth + 1, winHeight + 1); }
@@ -69,7 +68,6 @@ class MBLBPRepresentation : public Representation
         Feature() { rect = Rect(0, 0, 0, 0); }
         Feature( int offset, int x, int y, int _block_w, int _block_h  );
         uchar calc(const Mat &img) const;
-        void write( FileStorage &fs ) const { fs << "rect" << "[:" << rect.x << rect.y << rect.width << rect.height << "]"; }
         Rect rect;
         int p[16];
@@ -79,20 +77,6 @@ class MBLBPRepresentation : public Representation
 BR_REGISTER(Representation, MBLBPRepresentation)
-void MBLBPRepresentation::write(FileStorage &fs, const Mat &featureMap)
-{
-    fs << "features" << "[";
-    const Mat_<int>& featureMap_ = (const Mat_<int>&)featureMap;
-    for ( int fi = 0; fi < featureMap.cols; fi++ )
-        if ( featureMap_(0, fi) >= 0 )
-        {
-            fs << "{";
-            features[fi].write( fs );
-            fs << "}";
-        }
-    fs << "]";
-}
-
 MBLBPRepresentation::Feature::Feature( int offset, int x, int y, int _blockWidth, int _blockHeight )
 {
     Rect tr = rect = cvRect(x, y, _blockWidth, _blockHeight);