Merge pull request #224 from biometrics/TextureMap

Texture map

Merge pull request #224 from biometrics/TextureMap
Texture map
Brendan K
2 parents a3093cc6 e6e03bfd
Showing 5 changed files with 120 additions and 189 deletions
openbr/core/eigenutils.cpp
openbr/core/eigenutils.h
openbr/plugins/draw.cpp
openbr/plugins/landmarks.cpp
openbr/plugins/stasm4.cpp
@@ -95,3 +95,12 @@ MatrixXf removeRowCol(MatrixXf X, int row, int col) {
     }
     return Y;
 }
+
+MatrixXf pointsToMatrix(QList<QPointF> points) {
+    MatrixXf P(points.size(), 2);
+    for (int i = 0; i < points.size(); i++) {
+        P(i, 0) = points[i].x();
+        P(i, 1) = points[i].y();
+    }
+    return P;
+}
@@ -29,9 +29,13 @@ void printEigen(Eigen::MatrixXd X);
 void printEigen(Eigen::MatrixXf X);
 void printSize(Eigen::MatrixXf X);
-//Remove row and column from the matrix
+//Remove row and column from the matrix:
 Eigen::MatrixXf removeRowCol(Eigen::MatrixXf X, int row, int col);
+//Convert a point list into a matrix:
+Eigen::MatrixXf pointsToMatrix(QList<QPointF> points);
+
+
 template<typename _Scalar, int _Rows, int _Cols, int _Options, int _MaxRows, int _MaxCols>
 inline QDataStream &operator<<(QDataStream &stream, const Eigen::Matrix< _Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols > &mat)
 {
@@ -121,4 +125,5 @@ Eigen::MatrixBase&lt;T&gt; eigStd(const Eigen::MatrixBase&lt;T&gt;&amp; x,int dim)
     }
     qFatal("A matrix can only have two dimensions");
 }
+
 #endif // EIGENUTILS_H
@@ -527,6 +527,101 @@ class DrawSegmentation : public UntrainableTransform
 };
 BR_REGISTER(Transform, DrawSegmentation)
+/*!
+ * \ingroup transforms
+ * \brief Write all mats to disk as images.
+ * \author Brendan Klare \bklare
+ */
+class WriteImageTransform : public TimeVaryingTransform
+{
+    Q_OBJECT
+    Q_PROPERTY(QString outputDirectory READ get_outputDirectory WRITE set_outputDirectory RESET reset_outputDirectory STORED false)
+    Q_PROPERTY(QString imageName READ get_imageName WRITE set_imageName RESET reset_imageName STORED false)
+    Q_PROPERTY(QString imgExtension READ get_imgExtension WRITE set_imgExtension RESET reset_imgExtension STORED false)
+    BR_PROPERTY(QString, outputDirectory, "Temp")
+    BR_PROPERTY(QString, imageName, "image")
+    BR_PROPERTY(QString, imgExtension, "jpg")
+
+    int cnt;
+
+    void init() {
+        cnt = 0;
+        if (! QDir(outputDirectory).exists())
+            QDir().mkdir(outputDirectory);
+    }
+
+    void projectUpdate(const Template &src, Template &dst)
+    {
+        dst = src;
+        OpenCVUtils::saveImage(dst.m(), QString("%1/%2_%3.%4").arg(outputDirectory).arg(imageName).arg(cnt++, 5, QChar('0')).arg(imgExtension));
+    }
+
+};
+BR_REGISTER(Transform, WriteImageTransform)
+
+
+/**
+ * @brief The MeanImageTransform class computes the average template/image
+ * and save the result as an encoded image.
+ */
+class MeanImageTransform : public TimeVaryingTransform
+{
+    Q_OBJECT
+
+    Q_PROPERTY(QString imgname READ get_imgname WRITE set_imgname RESET reset_imgname STORED false)
+    Q_PROPERTY(QString ext READ get_ext WRITE set_ext RESET reset_ext STORED false)
+
+    BR_PROPERTY(QString, imgname, "average")
+    BR_PROPERTY(QString, ext, "jpg")
+
+    Mat average;
+    int cnt;
+
+    void init()
+    {
+        cnt = 0;
+    }
+
+    void projectUpdate(const Template &src, Template &dst)
+    {
+        dst = src;
+        if (cnt == 0) {
+            if (src.m().channels() == 1)
+                average = Mat::zeros(dst.m().size(),CV_64FC1);
+            else if (src.m().channels() == 3)
+                average = Mat::zeros(dst.m().size(),CV_64FC3);
+            else
+                qFatal("Unsupported number of channels");
+        }
+
+        Mat temp;
+        if (src.m().channels() == 1) {
+            src.m().convertTo(temp, CV_64FC1);
+            average += temp;
+        } else if (src.m().channels() == 3) {
+            src.m().convertTo(temp, CV_64FC3);
+            average += temp;
+        } else
+            qFatal("Unsupported number of channels");
+
+        cnt++;
+    }
+
+    virtual void finalize(TemplateList & output)
+    {
+        average /= float(cnt);
+        imwrite(QString("%1.%2").arg(imgname).arg(ext).toStdString(), average);
+        output = TemplateList();
+    }
+
+
+public:
+    MeanImageTransform() : TimeVaryingTransform(false, false) {}
+};
+
+BR_REGISTER(Transform, MeanImageTransform)
+
+
 // TODO: re-implement EditTransform using Qt
 #if 0
 /*!
@@ -141,193 +141,10 @@ BR_REGISTER(Transform, ProcrustesTransform)
 /*!
  * \ingroup transforms
- * \brief Improved procrustes alignment of points, to include a post processing scaling of points
- * to faciliate subsequent texture mapping.
- * \author Brendan Klare \cite bklare
- */
-class ProcrustesAlignTransform : public Transform
-{
-    Q_OBJECT
-
-    Q_PROPERTY(float width READ get_width WRITE set_width RESET reset_width STORED false)
-    Q_PROPERTY(float padding READ get_padding WRITE set_padding RESET reset_padding STORED false)
-    BR_PROPERTY(float, width, 80)
-    BR_PROPERTY(float, padding, 8)
-
-    Eigen::MatrixXf referenceShape;
-    float minX;
-    float minY;
-    float maxX;
-    float maxY;
-    float aspectRatio;
-
-    void init() {
-        minX = FLT_MAX,
-        minY = FLT_MAX,
-        maxX = -FLT_MAX,
-        maxY = -FLT_MAX;
-        aspectRatio = 0;
-    }
-
-    static MatrixXf getRotation(MatrixXf ref, MatrixXf sample) {
-        MatrixXf R = ref.transpose() * sample;
-        JacobiSVD<MatrixXf> svd(R, ComputeFullU | ComputeFullV);
-        R = svd.matrixU() * svd.matrixV();
-        return R;
-    }
-
-    //Converts x y points in a single vector to two column matrix
-    static MatrixXf vectorToMatrix(MatrixXf vector) {
-        int n = vector.rows();
-        MatrixXf matrix(n / 2, 2);
-        for (int i = 0; i < n / 2; i++) {
-            for (int j = 0; j < 2; j++) {
-                matrix(i, j) = vector(i * 2 + j);
-            }
-        }
-        return matrix;
-    }
-
-    static MatrixXf matrixToVector(MatrixXf matrix) {
-        int n2 = matrix.rows();
-        MatrixXf vector(n2 * 2, 1);
-        for (int i = 0; i < n2; i++) {
-            for (int j = 0; j < 2; j++) {
-                vector(i * 2 + j) = matrix(i, j);
-            }
-        }
-        return vector;
-    }
-
-    void train(const TemplateList &data)
-    {
-        MatrixXf points(data[0].file.points().size() * 2, data.size());
-
-        // Normalize all sets of points
-        for (int j = 0; j < data.size(); j++) {
-            QList<QPointF> imagePoints = data[j].file.points();
-
-            float meanX = 0,
-                  meanY = 0;
-            for (int i = 0; i < imagePoints.size(); i++) {
-                points(i * 2, j) = imagePoints[i].x();
-                points(i * 2 + 1, j) = imagePoints[i].y();
-                meanX += imagePoints[i].x();
-                meanY += imagePoints[i].y();
-            }
-            meanX /= imagePoints.size();
-            meanY /= imagePoints.size();
-
-            for (int i = 0; i < imagePoints.size(); i++) {
-                points(i * 2, j) -= meanX;
-                points(i * 2 + 1, j) -= meanY;
-            }
-        }
-
-        //normalize scale
-        for (int i = 0; i < points.cols(); i++)
-            points.col(i) = points.col(i) / points.col(i).norm();
-
-        //Normalize rotation
-        MatrixXf refPrev;
-        referenceShape = vectorToMatrix(points.rowwise().sum() / points.cols());
-        float diff = FLT_MAX;
-        while (diff > 1e-5) {//iterate until reference shape is stable
-            refPrev = referenceShape;
-
-            for (int j = 0; j < points.cols(); j++) {
-                MatrixXf p = vectorToMatrix(points.col(j));
-                MatrixXf R = getRotation(referenceShape, p);
-                p = p * R.transpose();
-                points.col(j) = matrixToVector(p);
-            }
-            referenceShape = vectorToMatrix(points.rowwise().sum() / points.cols());
-            diff = (matrixToVector(referenceShape) - matrixToVector(refPrev)).norm();
-        }
-
-        referenceShape = vectorToMatrix(points.rowwise().sum() / points.cols());
-
-        //Choose crop boundaries and adjustments that captures all data
-        for (int i = 0; i < points.rows(); i++) {
-            for (int j = 0; j < points.cols(); j++) {
-                if (i % 2 == 0) {
-                if (points(i,j) > maxX)
-                    maxX = points(i, j);
-                if (points(i,j) < minX)
-                    minX = points(i, j);
-                } else {
-                if (points(i,j) > maxY)
-                    maxY = points(i, j);
-                if (points(i,j) < minY)
-                    minY = points(i, j);
-                }
-            }
-        }
-        aspectRatio = (maxX - minX) / (maxY - minY);
-    }
-
-    void project(const Template &src, Template &dst) const
-    {
-        QList<QPointF> imagePoints = src.file.points();
-        MatrixXf p(imagePoints.size() * 2, 1);
-        for (int i = 0; i < imagePoints.size(); i++) {
-            p(i * 2) = imagePoints[i].x();
-            p(i * 2 + 1) = imagePoints[i].y();
-        }
-        p = vectorToMatrix(p);
-
-        //Normalize translation
-        p.col(0) = p.col(0) - MatrixXf::Ones(p.rows(),1) * (p.col(0).sum() / p.rows());
-        p.col(1) = p.col(1) - MatrixXf::Ones(p.rows(),1) * (p.col(1).sum() / p.rows());
-
-        //Normalize scale
-        p /= matrixToVector(p).norm();
-
-        //Normalize rotation
-        MatrixXf R = getRotation(referenceShape, p);
-        p = p * R.transpose();
-
-        //Translate and scale into output space and store in output list
-        QList<QPointF> procrustesPoints;
-        for (int i = 0; i < p.rows(); i++)
-            procrustesPoints.append( QPointF(
-                (p(i, 0) - minX) / (maxX - minX) * (width - 1) + padding,
-                (p(i, 1) - minY) / (maxY - minY) * (qRound( width / aspectRatio) - 1) + padding));
-
-        dst = src;
-        dst.file.setList<QPointF>("ProcrustesPoints", procrustesPoints);
-        dst.file.set("ProcrustesBound", QRectF(0, 0, width + 2 * padding, (qRound(width / aspectRatio) + 2 * padding)));
-    }
-
-    void store(QDataStream &stream) const
-    {
-        stream << referenceShape;
-        stream << minX;
-        stream << minY;
-        stream << maxX;
-        stream << maxY;
-        stream << aspectRatio;
-    }
-
-    void load(QDataStream &stream)
-    {
-        stream >> referenceShape;
-        stream >> minX;
-        stream >> minY;
-        stream >> maxX;
-        stream >> maxY;
-        stream >> aspectRatio;
-    }
-};
-
-BR_REGISTER(Transform, ProcrustesAlignTransform)
-
-/*!
- * \ingroup transforms
  * \brief Creates a Delaunay triangulation based on a set of points
  * \author Scott Klum \cite sklum
  */
-class DelaunayTransform : public UntrainableTransform
+class DelaunayTransform : public Transform
 {
     Q_OBJECT
@@ -37,6 +37,8 @@ class StasmInitializer : public Initializer
     {
         Globals->abbreviations.insert("RectFromStasmEyes","RectFromPoints([28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47],0.3,5.3)");
         Globals->abbreviations.insert("RectFromStasmBrow","RectFromPoints([16,17,18,19,20,21,22,23,24,25,26,27],0.15,5)");
+        Globals->abbreviations.insert("RectFromStasmPeriocular","RectFromPoints([28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,16,17,18,19,20,21,22,23,24,25,26,27]],0.3,5.3)");
+        Globals->abbreviations.insert("RectFromStasmBrow","RectFromPoints([16,17,18,19,20,21,22,23,24,25,26,27],0.15,5)");
         Globals->abbreviations.insert("RectFromStasmNose","RectFromPoints([48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58],0.15,1.15)");
         Globals->abbreviations.insert("RectFromStasmNoseWithBridge", "RectFromPoints([21, 22, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58],0.15,.6)");
         Globals->abbreviations.insert("RectFromStasmMouth","RectFromPoints([59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76],0.3,2)");
@@ -73,8 +75,11 @@ class StasmTransform : public UntrainableTransform
     void project(const Template &src, Template &dst) const
     {
-        if (src.m().channels() != 1) qFatal("Stasm expects single channel matrices.");
-
+        Mat stasmSrc(src);
+        if (src.m().channels() == 3)
+            cvtColor(src, stasmSrc, CV_BGR2GRAY);
+        else if (src.m().channels() != 1)
+            qFatal("Stasm expects single channel matrices.");
         dst = src;
         StasmCascadeClassifier *stasmCascade = stasmCascadeResource.acquire();
@@ -115,14 +120,14 @@ class StasmTransform : public UntrainableTransform
                     else if (i == 39)  /*Stasm Left Eye*/ { eyes[2*i] = leftEye.x(); eyes[2*i+1] = leftEye.y(); }
                     else { eyes[2*i] = 0; eyes[2*i+1] = 0; }
                 }
-                stasm_search_pinned(landmarks, eyes, reinterpret_cast<const char*>(src.m().data), src.m().cols, src.m().rows, NULL);
+                stasm_search_pinned(landmarks, eyes, reinterpret_cast<const char*>(stasmSrc.data), stasmSrc.cols, stasmSrc.rows, NULL);
                 // The ASM in Stasm is guaranteed to converge in this case
                 foundFace = 1;
             }
         }
-        if (!foundFace) stasm_search_single(&foundFace, landmarks, reinterpret_cast<const char*>(src.m().data), src.m().cols, src.m().rows, *stasmCascade, NULL, NULL);
+        if (!foundFace) stasm_search_single(&foundFace, landmarks, reinterpret_cast<const char*>(stasmSrc.data), stasmSrc.cols, stasmSrc.rows, *stasmCascade, NULL, NULL);
         if (stasm3Format) {
             nLandmarks = 76;