Merge branch 'master' of https://github.com/biometrics/openbr

Scott Klum
2 parents aca7accd 835f64e8
Showing 13 changed files with 120 additions and 132 deletions
CMakeLists.txt
openbr/core/boost.cpp
openbr/core/boost.h
openbr/openbr_plugin.h
openbr/plugins/classification/boostedforest.cpp
openbr/plugins/classification/cascade.cpp
openbr/plugins/classification/liblinear.cpp
openbr/plugins/imgproc/slidingwindow.cpp
openbr/plugins/metadata/cascade.cpp
openbr/plugins/representation/haar.cpp
openbr/plugins/representation/mblbp.cpp
openbr/plugins/representation/npd.cpp
openbr/plugins/representation/random.cpp
@@ -159,7 +159,7 @@ endif()
 # Download the models
 ExternalProject_Add(models
                     URL http://github.com/biometrics/openbr/releases/download/v1.1.0/models.tar.gz
-                    URL_MD5 22b158958907e08e611f11b1c62547c6
+                    URL_MD5 26cf71abd48cec8d7b2abf3e8f4ebfc4
                     SOURCE_DIR "${PROJECT_SOURCE_DIR}/share/openbr/models"
                     CONFIGURE_COMMAND ""
                     BUILD_COMMAND ""
@@ -131,23 +131,16 @@ static CvMat* cvPreprocessIndexArray( const CvMat* idx_arr, int data_arr_size, b
 //------------------------------------- FeatureEvaluator ---------------------------------------
-void FeatureEvaluator::init(Representation *_representation, int _maxSampleCount, int channels)
+void FeatureEvaluator::init(Representation *_representation, int _maxSampleCount)
 {
     representation = _representation;
-
-    int dx, dy;
-    Size windowSize = representation->windowSize(&dx, &dy);
-    data.create((int)_maxSampleCount, (windowSize.width + dx) * (windowSize.height + dy), CV_32SC(channels));
     cls.create( (int)_maxSampleCount, 1, CV_32FC1 );
 }
-void FeatureEvaluator::setImage(const Mat &img, uchar clsLabel, int idx)
+void FeatureEvaluator::setImage(const Template &src, uchar clsLabel, int idx)
 {
     cls.ptr<float>(idx)[0] = clsLabel;
-
-    Mat pp;
-    representation->preprocess(img, pp);
-    pp.reshape(0, 1).copyTo(data.row(idx));
+    data.append(representation->preprocess(src));
 }
 //----------------------------- CascadeBoostParams -------------------------------------------------
@@ -2,7 +2,6 @@
 #define _BOOST_H_
 #include "ml.h"
-#include <time.h>
 #include <openbr/openbr_plugin.h>
 #ifdef _WIN32
@@ -17,9 +16,9 @@ namespace br
 struct FeatureEvaluator
 {
     ~FeatureEvaluator() {}
-    void init(Representation *_representation, int _maxSampleCount, int channels);
-    void setImage(const cv::Mat& img, uchar clsLabel, int idx);
-    float operator()(int featureIdx, int sampleIdx) const { return representation->evaluate(data.row(sampleIdx), featureIdx); }
+    void init(Representation *_representation, int _maxSampleCount);
+    void setImage(const Template &src, uchar clsLabel, int idx);
+    float operator()(int featureIdx, int sampleIdx) const { return representation->evaluate(data[sampleIdx], featureIdx); }
     int getNumFeatures() const { return representation->numFeatures(); }
     int getMaxCatCount() const { return representation->maxCatCount(); }
@@ -27,7 +26,8 @@ struct FeatureEvaluator
     const cv::Mat& getCls() const { return cls; }
     float getCls(int si) const { return cls.at<float>(si, 0); }
-    cv::Mat data, cls;
+    cv::Mat cls;
+    TemplateList data;
     Representation *representation;
 };
@@ -882,13 +882,11 @@ public:
     static Representation *make(QString str, QObject *parent); /*!< \brief Make a representation from a string. */
-    virtual void preprocess(const cv::Mat &src, cv::Mat &dst) const { dst = src; }
-    virtual void train(const QList<cv::Mat> &images, const QList<float> &labels) { (void) images; (void)labels; }
-
-    virtual float evaluate(const cv::Mat &image, int idx) const = 0;
-    // By convention, an empty indices list will result in all feature responses being calculated
-    // and returned.
-    virtual cv::Mat evaluate(const cv::Mat &image, const QList<int> &indices = QList<int>()) const = 0;
+    virtual Template preprocess(const Template &src) const { return src; }
+    virtual void train(const TemplateList &data) { (void)data; }
+    virtual float evaluate(const Template &src, int idx) const = 0;
+    // By convention passing an empty list evaluates all features in the representation
+    virtual cv::Mat evaluate(const Template &src, const QList<int> &indices = QList<int>()) const = 0;
     virtual cv::Size windowSize(int *dx = NULL, int *dy = NULL) const = 0; // dx and dy should indicate the change to the original window size after preprocessing
     virtual int numChannels() const { return 1; }
@@ -905,13 +903,13 @@ public:
     static Classifier *make(QString str, QObject *parent);
-    virtual void train(const QList<cv::Mat> &images, const QList<float> &labels) = 0;
-    virtual float classify(const cv::Mat &image, bool process = true, float *confidence = NULL) const = 0;
+    virtual void train(const TemplateList &data) { (void)data; }
+    virtual float classify(const Template &src, bool process = true, float *confidence = NULL) const = 0;
     // Slots for representations
-    virtual cv::Mat preprocess(const cv::Mat &image) const = 0;
+    virtual Template preprocess(const Template &src) const { return src; }
     virtual cv::Size windowSize(int *dx = NULL, int *dy = NULL) const = 0;
-    virtual int numFeatures() const = 0;
+    virtual int numFeatures() const { return 0; }
 };
@@ -125,20 +125,20 @@ private:
     QList<Node*> classifiers;
     float threshold;
-    void train(const QList<Mat> &images, const QList<float> &labels)
+    void train(const TemplateList &data)
     {
-        representation->train(images, labels);
+        representation->train(data);
         CascadeBoostParams params(type, minTAR, maxFAR, trimRate, maxDepth, maxWeakCount);
         FeatureEvaluator featureEvaluator;
-        featureEvaluator.init(representation, images.size(), representation->numChannels());
+        featureEvaluator.init(representation, data.size());
-        for (int i = 0; i < images.size(); i++)
-            featureEvaluator.setImage(images[i], labels[i], i);
+        for (int i = 0; i < data.size(); i++)
+            featureEvaluator.setImage(data[i], data[i].file.get<float>("Label"), i);
         CascadeBoost boost;
-        boost.train(&featureEvaluator, images.size(), 1024, 1024, representation->numChannels(), params);
+        boost.train(&featureEvaluator, data.size(), 1024, 1024, representation->numChannels(), params);
         threshold = boost.getThreshold();
@@ -149,13 +149,9 @@ private:
         }
     }
-    float classify(const Mat &image, bool process, float *confidence) const
+    float classify(const Template &src, bool process, float *confidence) const
     {
-        Mat m;
-        if (process)
-            m = preprocess(image);
-        else
-            m = image;
+        Template t = process ? preprocess(src) : src;
         float sum = 0;
         for (int i = 0; i < classifiers.size(); i++) {
@@ -163,10 +159,10 @@ private:
             while (node->left) {
                 if (representation->maxCatCount() > 0) {
-                    int c = (int)representation->evaluate(m, node->featureIdx);
+                    int c = (int)representation->evaluate(t, node->featureIdx);
                     node = (node->subset[c >> 5] & (1 << (c & 31))) ? node->left : node->right;
                 } else {
-                    double val = representation->evaluate(m, node->featureIdx);
+                    double val = representation->evaluate(t, node->featureIdx);
                     node = val <= node->threshold ? node->left : node->right;
                 }
             }
@@ -184,11 +180,9 @@ private:
         return representation->numFeatures();
     }
-    Mat preprocess(const Mat &image) const
+    Template preprocess(const Template &src) const
     {
-        Mat dst;
-        representation->preprocess(image, dst);
-        return dst;
+        return representation->preprocess(src);
     }
     Size windowSize(int *dx, int *dy) const
 #include <opencv2/imgproc/imgproc.hpp>
-#include <opencv2/highgui/highgui.hpp>
+
 #include <openbr/plugins/openbr_internal.h>
 #include <openbr/core/common.h>
 #include "openbr/core/opencvutils.h"
@@ -101,8 +101,8 @@ class CascadeClassifier : public Classifier
     BR_PROPERTY(bool, requireAllStages, false)
     QList<Classifier *> stages;
-    QList<Mat> posImages, negImages;
-    QList<Mat> posSamples, negSamples;
+    TemplateList posImages, negImages;
+    TemplateList posSamples, negSamples;
     QList<int> indices;
     int negIndex, posIndex, samplingRound;
@@ -114,18 +114,17 @@ class CascadeClassifier : public Classifier
         negIndex = posIndex = samplingRound = 0;
     }
-    bool getPositive(Mat &img)
+    bool getPositive(Template &img)
     {
         if (posIndex >= posImages.size())
             return false;
-
-        posImages[indices[posIndex++]].copyTo(img);
+        img = posImages[indices[posIndex++]];
         return true;
     }
-    Mat getNegative(Point &offset)
+    Template getNegative(Point &offset)
     {
-        Mat negative;
+        Template negative;
         const Size size = windowSize();
         // Grab negative from list
@@ -137,9 +136,9 @@ class CascadeClassifier : public Classifier
             samplingRound = samplingRound % (size.width * size.height);
             negIndex %= count;
-            offset.x = qMin( (int)samplingRound % size.width, negative.cols - size.width);
-            offset.y = qMin( (int)samplingRound / size.width, negative.rows - size.height);
-            if (!negative.empty() && negative.type() == CV_8U
+            offset.x = qMin( (int)samplingRound % size.width, negative.m().cols - size.width);
+            offset.y = qMin( (int)samplingRound / size.width, negative.m().rows - size.height);
+            if (!negative.m().empty() && negative.m().type() == CV_8U
                     && offset.x >= 0 && offset.y >= 0)
                 break;
         }
@@ -151,16 +150,16 @@ class CascadeClassifier : public Classifier
     {
         uint64 passedNegatives = 0;
         forever {
-            Mat negative;
+            Template negative;
             Point offset;
             QMutexLocker samplingLocker(&samplingMutex);
             negative = getNegative(offset);
             samplingLocker.unlock();
-            Miner miner(negative, windowSize(), offset);
+            Miner miner(negative.m(), windowSize(), offset);
             forever {
                 bool newImg;
-                Mat sample = miner.mine(&newImg);
+                Template sample(negative.file, miner.mine(&newImg));
                 if (!newImg) {
                     if (negSamples.size() >= numNegs)
                         return passedNegatives;
@@ -182,16 +181,16 @@ class CascadeClassifier : public Classifier
         }
     }
-    void train(const QList<Mat> &images, const QList<float> &labels)
+    void train(const TemplateList &data)
     {
-        for (int i = 0; i < images.size(); i++)
-            labels[i] == 1 ? posImages.append(images[i]) : negImages.append(images[i]);
+        foreach (const Template &t, data)
+            t.file.get<float>("Label") == 1.0f ? posImages.append(t) : negImages.append(t);
-        qDebug() << "Total images:" << images.size()
+        qDebug() << "Total images:" << data.size()
                  << "\nTotal positive images:" << posImages.size()
                  << "\nTotal negative images:" << negImages.size();
-        indices = Common::RandSample(posImages.size(),posImages.size(),true);
+        indices = Common::RandSample(posImages.size(), posImages.size(), true);
         stages.reserve(numStages);
         for (int i = 0; i < numStages; i++) {
@@ -210,27 +209,17 @@ class CascadeClassifier : public Classifier
                 return;
             }
-            QList<float> posLabels;
-            posLabels.reserve(posSamples.size());
-            for (int j=0; j<posSamples.size(); j++)
-                posLabels.append(1);
-
-            QList<float> negLabels;
-            negLabels.reserve(negSamples.size());
-            for (int j=0; j<negSamples.size(); j++)
-                negLabels.append(0);
-
-            stages[i]->train(posSamples+negSamples, posLabels+negLabels);
+            stages[i]->train(posSamples + negSamples);
             qDebug() << "END>";
         }
     }
-    float classify(const Mat &image, bool process, float *confidence) const
+    float classify(const Template &src, bool process, float *confidence) const
     {
         float stageConf = 0.0f;
         foreach (const Classifier *stage, stages) {
-            float result = stage->classify(image, process, &stageConf);
+            float result = stage->classify(src, process, &stageConf);
             if (confidence)
                 *confidence += stageConf;
             if (result == 0.0f)
@@ -244,9 +233,9 @@ class CascadeClassifier : public Classifier
         return stages.first()->numFeatures();
     }
-    Mat preprocess(const Mat &image) const
+    Template preprocess(const Template &src) const
     {
-        return stages.first()->preprocess(image);
+        return stages.first()->preprocess(src);
     }
     Size windowSize(int *dx = NULL, int *dy = NULL) const
@@ -274,16 +263,13 @@ class CascadeClassifier : public Classifier
 private:
     float getSamples()
     {
-        posSamples.clear();
-        posSamples.reserve(numPos);
-        negSamples.clear();
-        negSamples.reserve(numNegs);
+        posSamples.clear(); posSamples.reserve(numPos);
+        negSamples.clear(); negSamples.reserve(numNegs);
         posIndex = 0;
         float confidence;
         while (posSamples.size() < numPos) {
-            Mat pos(windowSize(), CV_8U);
-
+            Template pos;
             if (!getPositive(pos))
                 qFatal("Cannot get another positive sample!");
@@ -138,7 +138,7 @@ private:
             param.weight = NULL;
         }
-        m = *train_svm(&prob, &param);
+        //m = *train_svm(&prob, &param);
         delete[] param.weight;
         delete[] param.weight_label;
@@ -17,7 +17,6 @@
 #include <openbr/plugins/openbr_internal.h>
 #include <openbr/core/opencvutils.h>
 #include <openbr/core/qtutils.h>
-#include <opencv2/highgui/highgui.hpp>
 #include <opencv2/imgproc/imgproc.hpp>
@@ -35,7 +34,6 @@ namespace br
  * \br_property int minSize The smallest sized object to detect in pixels
  * \br_property int maxSize The largest sized object to detect in pixels. A negative value will set maxSize == image size
  * \br_property float scaleFactor The factor to scale the image by during each resize.
- * \br_property int minNeighbors Parameter for non-maximum supression
  * \br_property float confidenceThreshold A threshold for positive detections. Positive detections returned by the classifier that have confidences below this threshold are considered negative detections.
  * \br_property float eps Parameter for non-maximum supression
  */
@@ -60,7 +58,7 @@ class SlidingWindowTransform : public MetaTransform
     void train(const TemplateList &data)
     {
-        classifier->train(data.data(), File::get<float>(data, "Label", -1));
+        classifier->train(data);
     }
     void project(const Template &src, Template &dst) const
@@ -123,15 +121,18 @@ class SlidingWindowTransform : public MetaTransform
                     Mat scaledImage(scaledImageSize, CV_8U, imageBuffer.data);
                     resize(m, scaledImage, scaledImageSize, 0, 0, CV_INTER_LINEAR);
-                    Mat repImage = classifier->preprocess(scaledImage);
+
+                    Template repImage(t.file, scaledImage);
+                    repImage = classifier->preprocess(repImage);
                     int step = factor > 2. ? 1 : 2;
                     for (int y = 0; y < processingRectSize.height; y += step) {
                         for (int x = 0; x < processingRectSize.width; x += step) {
-                            Mat window = repImage(Rect(Point(x, y), Size(originalWindowSize.width + dx, originalWindowSize.height + dy))).clone();
+                            Mat window = repImage.m()(Rect(Point(x, y), Size(originalWindowSize.width + dx, originalWindowSize.height + dy))).clone();
+                            Template t(window);
                             float confidence = 0;
-                            int result = classifier->classify(window, false, &confidence);
+                            int result = classifier->classify(t, false, &confidence);
                             if (result == 1) {
                                 rects.append(Rect(cvRound(x*factor), cvRound(y*factor), windowSize.width, windowSize.height));
@@ -413,6 +413,17 @@ class CascadeTransform : public MetaTransform
                 if (ROCMode) cascade->detectMultiScale(m, rects, rejectLevels, levelWeights, 1.2, minNeighbors, flags | CASCADE_SCALE_IMAGE, Size(minSize, minSize), Size(), true);
                 else         cascade->detectMultiScale(m, rects, 1.2, minNeighbors, flags, Size(minSize, minSize));
+                // It appears that flags is ignored for new model files:
+                // http://docs.opencv.org/modules/objdetect/doc/cascade_classification.html#cascadeclassifier-detectmultiscale
+                if ((flags == CASCADE_FIND_BIGGEST_OBJECT) && (rects.size() > 1)) {
+                    Rect biggest = rects[0];
+                    for (size_t j=0; j<rects.size(); j++)
+                        if (rects[j].area() > biggest.area())
+                            biggest = rects[j];
+                    rects.clear();
+                    rects.push_back(biggest);
+                }
+
                 bool empty = false;
                 if (!enrollAll && rects.empty()) {
                     empty = true;
@@ -80,23 +80,25 @@ class HaarRepresentation : public Representation
         }
     }
-    void preprocess(const Mat &src, Mat &dst) const
+    Template preprocess(const Template &src) const
     {
+        Template dst;
         integral(src, dst);
+        return dst;
     }
-    float evaluate(const Mat &image, int idx) const
+    float evaluate(const Template &src, int idx) const
     {
-        return (float)features[idx].calc(image);
+        return (float)features[idx].calc(src.m());
     }
-    Mat evaluate(const Mat &image, const QList<int> &indices) const
+    Mat evaluate(const Template &src, const QList<int> &indices) const
     {
         int size = indices.empty() ? numFeatures() : indices.size();
         Mat result(1, size, CV_32FC1);
         for (int i = 0; i < size; i++)
-            result.at<float>(i) = evaluate(image, indices.empty() ? i : indices[i]);
+            result.at<float>(i) = evaluate(src, indices.empty() ? i : indices[i]);
         return result;
     }
@@ -39,32 +39,36 @@ class MBLBPRepresentation : public Representation
     void init()
     {
-        int offset = winWidth + 1;
-        for (int x = 0; x < winWidth; x++ )
-            for (int y = 0; y < winHeight; y++ )
-                for (int w = 1; w <= winWidth / 3; w++ )
-                    for (int h = 1; h <= winHeight / 3; h++ )
-                        if ((x+3*w <= winWidth) && (y+3*h <= winHeight) )
-                            features.append(Feature(offset, x, y, w, h ) );
+        if (features.isEmpty()) {
+            int offset = winWidth + 1;
+            for (int x = 0; x < winWidth; x++ )
+                for (int y = 0; y < winHeight; y++ )
+                    for (int w = 1; w <= winWidth / 3; w++ )
+                        for (int h = 1; h <= winHeight / 3; h++ )
+                            if ((x+3*w <= winWidth) && (y+3*h <= winHeight) )
+                                features.append(Feature(offset, x, y, w, h ) );
+        }
     }
-    void preprocess(const Mat &src, Mat &dst) const
+    Template preprocess(const Template &src) const
     {
+        Template dst;
         integral(src, dst);
+        return dst;
     }
-    float evaluate(const Mat &image, int idx) const
+    float evaluate(const Template &src, int idx) const
     {
-        return (float)features[idx].calc(image);
+        return (float)features[idx].calc(src.m());
     }
-    Mat evaluate(const Mat &image, const QList<int> &indices) const
+    Mat evaluate(const Template &src, const QList<int> &indices) const
     {
         int size = indices.empty() ? numFeatures() : indices.size();
         Mat result(1, size, CV_32FC1);
         for (int i = 0; i < size; i++)
-            result.at<float>(i) = evaluate(image, indices.empty() ? i : indices[i]);
+            result.at<float>(i) = evaluate(src, indices.empty() ? i : indices[i]);
         return result;
     }
@@ -16,23 +16,25 @@ class NPDRepresentation : public Representation
     void init()
     {
-        for (int p1 = 0; p1 < (winWidth * winHeight); p1++)
-            for (int p2 = p1; p2 < (winWidth * winHeight); p2++)
-                features.append(Feature(p1, p2));
+        if (features.isEmpty()) {
+            for (int p1 = 0; p1 < (winWidth * winHeight); p1++)
+                for (int p2 = p1; p2 < (winWidth * winHeight); p2++)
+                    features.append(Feature(p1, p2));
+        }
     }
-    float evaluate(const Mat &image, int idx) const
+    float evaluate(const Template &src, int idx) const
     {
-        return features[idx].calc(image);
+        return features[idx].calc(src.m());
     }
-    Mat evaluate(const Mat &image, const QList<int> &indices) const
+    Mat evaluate(const Template &src, const QList<int> &indices) const
     {
         int size = indices.empty() ? numFeatures() : indices.size();
         Mat result(1, size, CV_32FC1);
         for (int i = 0; i < size; i++)
-            result.at<float>(i) = evaluate(image, indices.empty() ? i : indices[i]);
+            result.at<float>(i) = evaluate(src, indices.empty() ? i : indices[i]);
         return result;
     }
@@ -49,10 +51,10 @@ class NPDRepresentation : public Representation
     struct Feature
     {
         Feature() {}
-        Feature(int p1, int p2) { p[0] = p1; p[1] = p2; }
+        Feature(int p0, int p1) : p0(p0), p1(p1) {}
         float calc(const Mat &image) const;
-        int p[2];
+        int p0, p1;
     };
     QList<Feature> features;
 };
@@ -61,9 +63,9 @@ BR_REGISTER(Representation, NPDRepresentation)
 inline float NPDRepresentation::Feature::calc(const Mat &image) const
 {
-    const int *ptr = image.ptr<int>();
-    int v1 = ptr[p[0]], v2 = ptr[p[1]];
-    return v1 == 0 && v2 == 0 ? 0 : ((float)(v1 - v2)) / (v1 + v2);
+    const uchar *ptr = image.ptr();
+    int v1 = (int)ptr[p0], v2 = (int)ptr[p1];
+    return (v1 + v2) == 0 ? 0 : (1.0 * (v1 - v2)) / (v1 + v2);
 }
 } // namespace br
@@ -28,9 +28,9 @@ class RandomRepresentation : public Representation
     QList<int> features;
-    void train(const QList<Mat> &images, const QList<float> &labels)
+    void train(const TemplateList &data)
     {
-        representation->train(images, labels);
+        representation->train(data);
         const int nFeatures = representation->numFeatures();
@@ -40,17 +40,17 @@ class RandomRepresentation : public Representation
         features = Common::RandSample(count,nFeatures,0,true);
     }
-    void preprocess(const Mat &src, Mat &dst) const
+    Template preprocess(const Template &src) const
     {
-        representation->preprocess(src,dst);
+        return representation->preprocess(src);
     }
-    float evaluate(const Mat &image, int idx) const
+    float evaluate(const Template &src, int idx) const
     {
-        return representation->evaluate(image,features[idx]);
+        return representation->evaluate(src, features[idx]);
     }
-    Mat evaluate(const Mat &image, const QList<int> &indices) const
+    Mat evaluate(const Template &src, const QList<int> &indices) const
     {
         QList<int> newIndices;
         if (indices.empty())
@@ -59,7 +59,7 @@ class RandomRepresentation : public Representation
             for (int i = 0; i < indices.size(); i++)
                 newIndices.append(features[indices[i]]);
-        return representation->evaluate(image,newIndices);
+        return representation->evaluate(src, newIndices);
     }
     int numFeatures() const
@@ -108,6 +108,3 @@ BR_REGISTER(Representation, RandomRepresentation)
 } // namespace br
 #include "representation/random.moc"
-
-
-