Merge branch 'master' of https://github.com/biometrics/openbr

sklum
2 parents 3f40433a fe7a2c4d
Showing 5 changed files with 153 additions and 52 deletions
openbr/openbr_plugin.cpp
openbr/openbr_plugin.h
openbr/plugins/eigen3.cpp
openbr/plugins/slidingwindow.cpp
openbr/plugins/svm.cpp
@@ -234,6 +234,11 @@ void File::appendRect(const QRectF &amp;rect)
     m_metadata["Rects"] = newRects;
 }
+void File::appendRect(const Rect &rect)
+{
+    appendRect(OpenCVUtils::fromRect(rect));
+}
+
 void File::appendRects(const QList<QRectF> &rects)
 {
     QList<QVariant> newRects = m_metadata["Rects"].toList();
@@ -242,6 +247,11 @@ void File::appendRects(const QList&lt;QRectF&gt; &amp;rects)
     m_metadata["Rects"] = newRects;
 }
+void File::appendRects(const QList<Rect> &rects)
+{
+    appendRects(OpenCVUtils::fromRects(rects));
+}
+
 /* File - private methods */
 void File::init(const QString &file)
 {
@@ -41,8 +41,6 @@
 #include <QVector>
 #include <opencv2/core/core.hpp>
 #include <openbr/openbr.h>
-#include <openbr/core/qtutils.h>
-#include <openbr/core/opencvutils.h>
 /*!
  * \defgroup cpp_plugin_sdk C++ Plugin SDK
@@ -222,7 +220,11 @@ struct BR_EXPORT File
     template <typename T>
     void setList(const QString &key, const QList<T> &value)
     {
-        set(key, QtUtils::toVariantList(value));
+        QVariantList variantList;
+        variantList.reserve(value.size());
+        foreach (const T &item, value)
+            variantList << item;
+        set(key, variantList);
     }
     inline void remove(const QString &key) { m_metadata.remove(key); } /*!< \brief Remove the metadata key. */
@@ -315,9 +317,9 @@ struct BR_EXPORT File
     QList<QRectF> namedRects() const; /*!< \brief Returns rects convertible from metadata values. */
     QList<QRectF> rects() const; /*!< \brief Returns the file's rects list. */
     void appendRect(const QRectF &rect); /*!< \brief Adds a rect to the file's rect list. */
-    void appendRect(const cv::Rect &rect) { appendRect(OpenCVUtils::fromRect(rect)); } /*!< \brief Adds a rect to the file's rect list. */
+    void appendRect(const cv::Rect &rect); /*!< \brief Adds a rect to the file's rect list. */
     void appendRects(const QList<QRectF> &rects); /*!< \brief Adds rects to the file's rect list. */
-    void appendRects(const QList<cv::Rect> &rects) { appendRects(OpenCVUtils::fromRects(rects)); } /*!< \brief Adds rects to the file's rect list. */
+    void appendRects(const QList<cv::Rect> &rects); /*!< \brief Adds rects to the file's rect list. */
     inline void clearRects() { m_metadata["Rects"] = QList<QVariant>(); } /*!< \brief Clears the file's rect list. */
     inline void setRects(const QList<QRectF> &rects) { clearRects(); appendRects(rects); } /*!< \brief Overwrites the file's rect list. */
     inline void setRects(const QList<cv::Rect> &rects) { clearRects(); appendRects(rects); } /*!< \brief Overwrites the file's rect list. */
@@ -302,11 +302,13 @@ class LDATransform : public Transform
     Q_PROPERTY(int directLDA READ get_directLDA WRITE set_directLDA RESET reset_directLDA STORED false)
     Q_PROPERTY(float directDrop READ get_directDrop WRITE set_directDrop RESET reset_directDrop STORED false)
     Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false)
+    Q_PROPERTY(bool isBinary READ get_isBinary WRITE set_isBinary RESET reset_isBinary STORED false)
     BR_PROPERTY(float, pcaKeep, 0.98)
     BR_PROPERTY(bool, pcaWhiten, false)
     BR_PROPERTY(int, directLDA, 0)
     BR_PROPERTY(float, directDrop, 0.1)
     BR_PROPERTY(QString, inputVariable, "Label")
+    BR_PROPERTY(bool, isBinary, false)
     int dimsOut;
     Eigen::VectorXf mean;
@@ -316,7 +318,6 @@ class LDATransform : public Transform
     {
         // creates "Label"
         TemplateList trainingSet = TemplateList::relabel(_trainingSet, inputVariable);
-
         int instances = trainingSet.size();
         // Perform PCA dimensionality reduction
@@ -450,6 +451,34 @@ class LDATransform : public Transform
         // Compute final projection matrix
         projection = ((space2.eVecs.transpose() * space1.eVecs.transpose()) * pca.eVecs.transpose()).transpose();
         dimsOut = dim2;
+
+        if (isBinary) {
+            assert(dimsOut == 1);
+            TemplateList projected;
+            float posVal = 0;
+            float negVal = 0;
+            for (int i = 0; i < trainingSet.size(); i++) {
+                Template t;
+                project(trainingSet[i],t);
+                //Note: the positive class is assumed to be 0 b/c it will
+                // typically be the first gallery template in the TemplateList structure
+                if (classes[i] == 0)
+                    posVal += t.m().at<float>(0,0);
+                else if (classes[i] == 1)
+                    negVal += t.m().at<float>(0,0);
+                else
+                    qFatal("Binary mode only supports two class problems.");
+            }
+            posVal /= classCounts[0];
+            negVal /= classCounts[1];
+
+            if (posVal < negVal) {
+                //Ensure positive value is supposed to be > 0 after projection
+                Eigen::MatrixXf invert = Eigen::MatrixXf::Ones(dimsIn,1);
+                invert *= -1;
+                projection = invert.transpose() * projection;
+            }
+        }
     }
     void project(const Template &src, Template &dst) const
@@ -462,6 +491,10 @@ class LDATransform : public Transform
         // Do projection
         outMap = projection.transpose() * (inMap - mean);
+
+        if (isBinary) {
+            dst.file.set("conf",dst.m().at<float>(0,0));
+        }
     }
     void store(QDataStream &stream) const
@@ -8,11 +8,29 @@
 using namespace cv;
+namespace br
+{
+
 // Because MSVC doesn't provide a round() function in math.h
 static int round(float x) { return (floor(x + 0.5)); }
-
-namespace br
+// Find avg aspect ratio
+static float getAspectRatio(const TemplateList &data)
 {
+    double tempRatio = 0;
+    int ratioCnt = 0;
+
+    foreach (const Template &tmpl, data) {
+        QList<Rect> posRects = OpenCVUtils::toRects(tmpl.file.rects());
+        foreach (const Rect &posRect, posRects) {
+            if (posRect.x + posRect.width >= tmpl.m().cols || posRect.y + posRect.height >= tmpl.m().rows || posRect.x < 0 || posRect.y < 0) {
+                continue;
+            }
+            tempRatio += (float)posRect.width / (float)posRect.height;
+            ratioCnt += 1;
+        }
+    }
+    return tempRatio / (double)ratioCnt;
+}
 /*!
  * \ingroup transforms
@@ -25,23 +43,19 @@ class SlidingWindowTransform : public Transform
     Q_OBJECT
     Q_PROPERTY(br::Transform *transform READ get_transform WRITE set_transform RESET reset_transform STORED false)
     Q_PROPERTY(int minSize READ get_minSize WRITE set_minSize RESET reset_minSize STORED false)
-    Q_PROPERTY(double scaleFactor READ get_scaleFactor WRITE set_scaleFactor RESET reset_scaleFactor STORED false)
     Q_PROPERTY(int stepSize READ get_stepSize WRITE set_stepSize RESET reset_stepSize STORED false)
-    Q_PROPERTY(bool takeLargestScale READ get_takeLargestScale WRITE set_takeLargestScale RESET reset_takeLargestScale STORED false)
+    Q_PROPERTY(bool takeFirst READ get_takeFirst WRITE set_takeFirst RESET reset_takeFirst STORED false)
     Q_PROPERTY(bool negSamples READ get_negSamples WRITE set_negSamples RESET reset_negSamples STORED false)
     Q_PROPERTY(int negToPosRatio READ get_negToPosRatio WRITE set_negToPosRatio RESET reset_negToPosRatio STORED false)
     Q_PROPERTY(double maxOverlap READ get_maxOverlap WRITE set_maxOverlap RESET reset_maxOverlap STORED false)
-    Q_PROPERTY(float aspectRatio READ get_aspectRatio WRITE set_aspectRatio RESET reset_aspectRatio STORED true)
     Q_PROPERTY(int windowWidth READ get_windowWidth WRITE set_windowWidth RESET reset_windowWidth STORED false)
     BR_PROPERTY(br::Transform *, transform, NULL)
     BR_PROPERTY(int, minSize, 8)
-    BR_PROPERTY(double, scaleFactor, 0.75)
     BR_PROPERTY(int, stepSize, 1)
-    BR_PROPERTY(bool, takeLargestScale, true)
+    BR_PROPERTY(bool, takeFirst, true)
     BR_PROPERTY(bool, negSamples, true)
     BR_PROPERTY(int, negToPosRatio, 1)
     BR_PROPERTY(double, maxOverlap, 0)
-    BR_PROPERTY(float, aspectRatio, 1)
     BR_PROPERTY(int, windowWidth, 24)
 public:
@@ -50,24 +64,14 @@ private:
     void train(const TemplateList &data)
     {
+        // only calculate if the work hasn't been done
+        aspectRatio = data.first().file.get<float>("aspectRatio", -1);
+        if (aspectRatio == -1)
+            aspectRatio = getAspectRatio(data);
+
         if (transform->trainable) {
-            double tempRatio = 0;
-            int ratioCnt = 0;
             TemplateList full;
-            //First find avg aspect ratio
-            foreach (const Template &tmpl, data) {
-                QList<Rect> posRects = OpenCVUtils::toRects(tmpl.file.rects());
-                foreach (const Rect &posRect, posRects) {
-                    if (posRect.x + posRect.width >= tmpl.m().cols || posRect.y + posRect.height >= tmpl.m().rows || posRect.x < 0 || posRect.y < 0) {
-                        continue;
-                    }
-                    tempRatio += (float)posRect.width / (float)posRect.height;
-                    ratioCnt += 1;
-                }
-            }
-            aspectRatio = tempRatio / (double)ratioCnt;
-
             foreach (const Template &tmpl, data) {
                 QList<Rect> posRects = OpenCVUtils::toRects(tmpl.file.rects());
                 QList<Rect> negRects;
@@ -132,6 +136,72 @@ private:
         if (src.file.getBool("Train", false)) return;
         dst.file.clearRects();
+        int windowHeight = (int) round((float) windowWidth / aspectRatio);
+        int scale = src.file.get<float>("scale", 1);
+
+        for (double y = 0; y + windowHeight < src.m().rows; y += stepSize) {
+            for (double x = 0; x + windowWidth < src.m().cols; x += stepSize) {
+                Rect window(x, y, windowWidth, windowHeight);
+                Template windowMat(src.file, Mat(src, window));
+                Template detect;
+                transform->project(windowMat, detect);
+                float conf = detect.file.get<float>("conf");
+
+                // the result will be in the Label
+                if (conf > 0) {
+                    dst.file.appendRect(QRectF((float) x * scale, (float) y * scale, (float) windowWidth * scale, (float) windowHeight * scale));
+                    QList<float> confidences = dst.file.getList<float>("Confidences", QList<float>());
+                    confidences.append(conf);
+                    dst.file.setList<float>("Confidences", confidences);
+                    if (takeFirst)
+                        return;
+                }
+            }
+        }
+    }
+
+    float aspectRatio;
+};
+
+BR_REGISTER(Transform, SlidingWindowTransform)
+
+/*!
+ * \ingroup transforms
+ * \brief .
+ * \author Austin Blanton \cite imaus10
+ */
+class BuildScalesTransform : public Transform
+{
+    Q_OBJECT
+    Q_PROPERTY(br::Transform *transform READ get_transform WRITE set_transform RESET reset_transform STORED false)
+    Q_PROPERTY(double scaleFactor READ get_scaleFactor WRITE set_scaleFactor RESET reset_scaleFactor STORED false)
+    Q_PROPERTY(bool takeLargestScale READ get_takeLargestScale WRITE set_takeLargestScale RESET reset_takeLargestScale STORED false)
+    Q_PROPERTY(int windowWidth READ get_windowWidth WRITE set_windowWidth RESET reset_windowWidth STORED false)
+    BR_PROPERTY(br::Transform *, transform, NULL)
+    BR_PROPERTY(double, scaleFactor, 0.75)
+    BR_PROPERTY(bool, takeLargestScale, true)
+    BR_PROPERTY(int, windowWidth, 24)
+
+public:
+    BuildScalesTransform() : Transform(false, true) {}
+private:
+
+    void train(const TemplateList &data)
+    {
+        aspectRatio = getAspectRatio(data);
+        // have to make a copy b/c data is const
+        TemplateList cp = data;
+        cp.first().file.set("aspectRatio", aspectRatio);
+        if (transform->trainable)
+            transform->train(cp);
+    }
+
+    void project(const Template &src, Template &dst) const
+    {
+        dst = src;
+        // do not scale images during training
+        if (src.file.getBool("Train", false)) return;
+
         int rows = src.m().rows;
         int cols = src.m().cols;
         int windowHeight = (int) round((float) windowWidth / aspectRatio);
@@ -140,34 +210,20 @@ private:
             startScale = round((float) rows / (float) windowHeight);
         else
             startScale = round((float) cols / (float) windowWidth);
-
         for (float scale = startScale; scale >= 1.0; scale -= (1.0 - scaleFactor)) {
-            Mat scaleImg;
+            Template scaleImg(src.file, Mat());
+            scaleImg.file.set("scale", scale);
             resize(src, scaleImg, Size(round(cols / scale), round(rows / scale)));
-
-            for (double y = 0; y + windowHeight < scaleImg.rows; y += stepSize) {
-                for (double x = 0; x + windowWidth < scaleImg.cols; x += stepSize) {
-qDebug() << "x=" << x << "\ty=" << y;
-                    Rect window(x, y, windowWidth, windowHeight);
-                    Template windowMat(src.file, Mat(scaleImg, window));
-                    Template detect;
-                    transform->project(windowMat, detect);
-                    // the result will be in the Label
-                    if (detect.file.get<QString>("Label") == "pos") {
-                        dst.file.appendRect(QRectF((float) x * scale, (float) y * scale, (float) windowWidth * scale, (float) windowHeight * scale));
-                        float confidence = detect.file.get<float>("Dist");
-                        QList<float> confidences = dst.file.getList<float>("Confidences", QList<float>());
-                        confidences.append(confidence);
-                        dst.file.setList<float>("Confidences", confidences);
-                        if (takeLargestScale) return;
-                    }
-                }
-            }
+            transform->project(scaleImg, dst);
+            if (takeLargestScale && !dst.file.rects().empty())
+                return;
         }
     }
+
+    float aspectRatio;
 };
-BR_REGISTER(Transform, SlidingWindowTransform)
+BR_REGISTER(Transform, BuildScalesTransform)
 /*!
  * \ingroup transforms
@@ -157,7 +157,7 @@ private:
         dst = src;
         float prediction = svm.predict(src.m().reshape(1, 1), returnDFVal);
         if (returnDFVal) {
-            dst.file.set("Dist", prediction);
+            dst.file.set("conf", prediction);
             // positive values ==> first class
             // negative values ==> second class
             prediction = prediction > 0 ? 0 : 1;