Delauney/procrustes with pseudo-barycentric texture mapping

Scott Klum
1 parent 66e940bb
Showing 4 changed files with 171 additions and 36 deletions
openbr/plugins/algorithms.cpp
openbr/plugins/landmarks.cpp
openbr/plugins/regions.cpp
openbr/plugins/stasm4.cpp
@@ -46,7 +46,7 @@ class AlgorithmsInitializer : public Initializer
         Globals->abbreviations.insert("CropFace", "Open+Cvt(Gray)+Cascade(FrontalFace)+ASEFEyes+Affine(128,128,0.25,0.35)");
         // Video
-        Globals->abbreviations.insert("DisplayVideo", "Stream([Show(false)+Discard])");
+        Globals->abbreviations.insert("DisplayVideo", "Stream([Cvt(Gray)+Resize(400,640)+Stasm+Delauney(true)+Draw(inPlace=true)+Show(false)+Discard])");
         Globals->abbreviations.insert("PerFrameDetection", "Stream([SaveMat(original)+Cvt(Gray)+Cascade(FrontalFace)+ASEFEyes+RestoreMat(original)+Draw(inPlace=true),Show(false)+Discard])");
         Globals->abbreviations.insert("AgeGenderDemo", "Stream([SaveMat(original)+Cvt(Gray)+Cascade(FrontalFace)+Expand+<FaceClassificationRegistration>+<FaceClassificationExtraction>+(<AgeRegressor>+Rename(Subject,Age)+Discard)/(<GenderClassifier>+Rename(Subject,Gender)+Discard)+RestoreMat(original)+Draw(inPlace=true)+DrawPropertiesPoint([Age,Gender],Affine_0,inPlace=true)+SaveMat(original)+Discard+Contract,RestoreMat(original)+FPSCalc+Show(false,[AvgFPS,Age,Gender])+Discard])");
@@ -33,6 +33,15 @@ class ProcrustesTransform : public Transform
             if (points.empty()) continue;
+            QList<QRectF> rects = datum.file.rects();
+
+            if (rects.size() > 1) qWarning("More than one rect in training data templates.");
+
+            points.append(rects[0].topLeft());
+            points.append(rects[0].topRight());
+            points.append(rects[0].bottomLeft());
+            points.append(rects[0].bottomRight());
+
             cv::Scalar mean = cv::mean(OpenCVUtils::toPoints(points).toVector().toStdVector());
             for (int i = 0; i < points.size(); i++) points[i] -= QPointF(mean[0],mean[1]);
@@ -67,7 +76,17 @@ class ProcrustesTransform : public Transform
     void project(const Template &src, Template &dst) const
     {
+        dst.m() = src.m();
+
         QList<QPointF> points = src.file.points();
+        QList<QRectF> rects = src.file.rects();
+
+        if (rects.size() > 1) qWarning("More than one rect in training data templates.");
+
+        points.append(rects[0].topLeft());
+        points.append(rects[0].topRight());
+        points.append(rects[0].bottomLeft());
+        points.append(rects[0].bottomRight());
         cv::Scalar mean = cv::mean(OpenCVUtils::toPoints(points).toVector().toStdVector());
         for (int i = 0; i < points.size(); i++) points[i] -= QPointF(mean[0],mean[1]);
@@ -84,13 +103,13 @@ class ProcrustesTransform : public Transform
         Eigen::MatrixXf R = svd.matrixU()*svd.matrixV().transpose();
-        Eigen::MatrixXf dstPoints = srcPoints*R;
-
-        points.clear();
-
-        for (int i = 0; i < dstPoints.rows(); i++) points.append(QPointF(dstPoints(i,0),dstPoints(i,1)));
-
-        dst.file.appendPoints(points);
+        dst.file.set("Procrustes_0_0", R(0,0));
+        dst.file.set("Procrustes_1_0", R(1,0));
+        dst.file.set("Procrustes_1_1", R(1,1));
+        dst.file.set("Procrustes_0_1", R(0,1));
+        dst.file.set("Procrustes_mean_0", mean[0]);
+        dst.file.set("Procrustes_mean_1", mean[1]);
+        dst.file.set("Procrustes_norm", norm);
     }
     void store(QDataStream &stream) const
@@ -109,10 +128,10 @@ BR_REGISTER(Transform, ProcrustesTransform)
 /*!
  * \ingroup transforms
- * \brief Wraps STASM key point detector
+ * \brief Creates a delauney triangulation based on a set of points
  * \author Scott Klum \cite sklum
  */
-class DelauneyTransform : public UntrainableTransform
+class DelaunayTransform : public UntrainableTransform
 {
     Q_OBJECT
@@ -121,48 +140,161 @@ class DelauneyTransform : public UntrainableTransform
     void project(const Template &src, Template &dst) const
     {
-        dst = src;
-
         Subdiv2D subdiv(Rect(0,0,src.m().cols,src.m().rows));
-        foreach(const cv::Point2f& point, OpenCVUtils::toPoints(src.file.points())) subdiv.insert(point);
+        QList<cv::Point2f> points = OpenCVUtils::toPoints(src.file.points());
+        QList<QRectF> rects = src.file.rects();
+
+        if (rects.size() > 1) qWarning("More than one rect in training data templates.");
+
+        points.append(OpenCVUtils::toPoint(rects[0].topLeft()));
+        points.append(OpenCVUtils::toPoint(rects[0].topRight()));
+        points.append(OpenCVUtils::toPoint(rects[0].bottomLeft()));
+        points.append(OpenCVUtils::toPoint(rects[0].bottomRight()));
+
+        for (int i = 0; i < points.size(); i++) subdiv.insert(points[i]);
         vector<Vec6f> triangleList;
         subdiv.getTriangleList(triangleList);
-        vector<Point> pt(3);
-        Scalar delaunay_color(0, 0, 0);
+        QList< QList<Point> > validTriangles;
+        int count = 0;
+
+        for (size_t i = 0; i < triangleList.size(); ++i) {
+            vector<Point> pt(3);
+
+            Vec6f t = triangleList[i];
+
+            pt[0] = Point(cvRound(t[0]), cvRound(t[1]));
+            pt[1] = Point(cvRound(t[2]), cvRound(t[3]));
+            pt[2] = Point(cvRound(t[4]), cvRound(t[5]));
+
+            bool inside = true;
+            for (int j = 0; j < 3; j++) {
+                if (pt[j].x > src.m().cols || pt[j].y > src.m().rows || pt[j].x <= 0 || pt[j].y <= 0) inside = false;
+            }
+
+            if (inside) {
+                count++;
+                qDebug() << count << pt[0] << pt[1] << pt[2] << "Area" << contourArea(pt);
+
+                QList<Point> triangleBuffer;
+
+                triangleBuffer << pt[0] << pt[1] << pt[2];
+
+                validTriangles.append(triangleBuffer);
+            }
+        }
+
+        dst.m() = src.m().clone();
         if (draw) {
-                int count = 0;
-                for(size_t i = 0; i < triangleList.size(); ++i) {
-                Vec6f t = triangleList[i];
+            foreach(const QList<Point>& triangle, validTriangles) {
+                line(dst.m(), triangle[0], triangle[1], Scalar(0,0,0), 1);
+                line(dst.m(), triangle[1], triangle[2], Scalar(0,0,0), 1);
+                line(dst.m(), triangle[2], triangle[0], Scalar(0,0,0), 1);
+            }
+        }
-                pt[0] = Point(cvRound(t[0]), cvRound(t[1]));
-                pt[1] = Point(cvRound(t[2]), cvRound(t[3]));
-                pt[2] = Point(cvRound(t[4]), cvRound(t[5]));
+        bool warp = true;
-                bool inside = true;
-                for (int i = 0; i < 3; i++) {
-                    if(pt[i].x > dst.m().cols || pt[i].y > dst.m().rows || pt[i].x <= 0 || pt[i].y <= 0) {
-                        inside = false;
-                    }
+        if (warp) {
+            Eigen::MatrixXf R(2,2);
+            R(0,0) = src.file.get<float>("Procrustes_0_0");
+            R(1,0) = src.file.get<float>("Procrustes_1_0");
+            R(1,1) = src.file.get<float>("Procrustes_1_1");
+            R(0,1) = src.file.get<float>("Procrustes_0_1");
+            cv::Scalar mean(2);
+            mean[0] = src.file.get<float>("Procrustes_mean_0");
+            mean[1] = src.file.get<float>("Procrustes_mean_1");
+
+            qDebug() << mean[0] << mean[1];
+
+            float norm = src.file.get<float>("Procrustes_norm");
+
+            qDebug() << norm;
+
+            dst.m() = Mat::zeros(src.m().rows,src.m().cols,src.m().type());
+
+            foreach(const QList<Point> &triangle, validTriangles) {
+                Eigen::MatrixXf srcPoints(triangle.size(), 2);
+
+                for (int j = 0; j < triangle.size(); j++) {
+                    srcPoints(j,0) = (triangle[j].x-mean[0])/(norm/150.)+50;
+                    srcPoints(j,1) = (triangle[j].y-mean[1])/(norm/150.)+50;
                 }
-                if (inside) {
-                    count++;
-                    //qDebug() << count << pt[0] << pt[1] << pt[2] << "Area" << contourArea(pt);
-                    line(dst.m(), pt[0], pt[1], delaunay_color, 1);
-                    line(dst.m(), pt[1], pt[2], delaunay_color, 1);
-                    line(dst.m(), pt[2], pt[0], delaunay_color, 1);
-                }
+
+                Eigen::MatrixXf dstMat = srcPoints*R;
+
+                Point2f test[3];
+                test[0] = triangle[0];
+                test[1] = triangle[1];
+                test[2] = triangle[2];
+                Point2f dstPoints[3];
+                dstPoints[0] = Point2f(dstMat(0,0),dstMat(0,1));
+                dstPoints[1] = Point2f(dstMat(1,0),dstMat(1,1));
+                dstPoints[2] = Point2f(dstMat(2,0),dstMat(2,1));
+
+                Mat buffer(src.m().rows,src.m().cols,src.m().type());
+
+                warpAffine(src.m(), buffer, getAffineTransform(test, dstPoints), Size(src.m().cols,src.m().rows));
+
+                Mat mask = Mat::zeros(src.m().rows, src.m().cols, CV_8U);
+                Point maskPoints[1][3];
+                maskPoints[0][0] = dstPoints[0];
+                maskPoints[0][1] = dstPoints[1];
+                maskPoints[0][2] = dstPoints[2];
+                const Point* ppt = { maskPoints[0] };
+
+                fillConvexPoly(mask, ppt, 3, Scalar(255,255,255), 8);
+
+                Mat output(src.m().rows,src.m().cols,src.m().type());
+
+                bitwise_and(buffer,mask,output);
+
+                dst.m() += output;
             }
         }
     }
 };
-BR_REGISTER(Transform, DelauneyTransform)
+BR_REGISTER(Transform, DelaunayTransform)
+
+/*!
+ * \ingroup transforms
+ * \brief Wraps STASM key point detector
+ * \author Scott Klum \cite sklum
+ */
+class MeanTransform : public Transform
+{
+    Q_OBJECT
+
+    Mat mean;
+
+    void train(const TemplateList &data)
+    {
+        mean = Mat::zeros(data[0].m().rows,data[0].m().cols,CV_32F);
+
+        for (int i = 0; i < data.size()/2; i++) {
+            Mat converted;
+            data[i].m().convertTo(converted, CV_32F);
+            mean += converted;
+        }
+
+        mean /= data.size()/2;
+    }
+
+    void project(const Template &src, Template &dst) const
+    {
+        dst = src;
+        dst.m() = mean;
+    }
+
+};
+
+BR_REGISTER(Transform, MeanTransform)
 } // namespace br
@@ -243,7 +243,10 @@ class RectFromPointsTransform : public UntrainableTransform
         dst.file.setPoints(points);
         if (crop) dst.m() = src.m()(Rect(std::max(0.0, minX - deltaWidth/2.0), std::max(0.0, minY - deltaHeight/2.0), std::min((double)src.m().cols, width), std::min((double)src.m().rows, height)));
-        else dst.m() = src.m();
+        else {
+            dst.file.appendRect(QRectF(std::max(0.0, minX - deltaWidth/2.0), std::max(0.0, minY - deltaHeight/2.0), std::min((double)src.m().cols, width), std::min((double)src.m().rows, height)));
+            dst.m() = src.m();
+        }
     }
 };
@@ -38,7 +38,7 @@ class StasmInitializer : public Initializer
     {
         Globals->abbreviations.insert("RectFromStasmEyes","RectFromPoints([29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47],0.125,6.0)+Resize(44,164)");
         Globals->abbreviations.insert("RectFromStasmBrow","RectFromPoints([17, 18, 19, 20, 21, 22, 23, 24],0.15,6)+Resize(28,132)");
-        Globals->abbreviations.insert("RectFromStasmNose","RectFromPoints([48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58],0.15,1.25)+Resize(60,60)");
+        Globals->abbreviations.insert("RectFromStasmNose","RectFromPoints([48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58],0.15,1.25)");
         Globals->abbreviations.insert("RectFromStasmMouth","RectFromPoints([59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76],0.3,3.0)+Resize(28,68)");
     }
 };