Peter M. Groen / openbr

Commit 9e3e99a447540b0c82a71a3057ff271d3e3789bd

Authored by Scott Klum
1 parent 66e940bb

Delauney/procrustes with pseudo-barycentric texture mapping

Inline Side-by-side
Showing 4 changed files with 171 additions and 36 deletions
openbr/plugins/algorithms.cpp
  View file @9e3e99a
... ... @@ -46,7 +46,7 @@ class AlgorithmsInitializer : public Initializer
46 46 Globals->abbreviations.insert("CropFace", "Open+Cvt(Gray)+Cascade(FrontalFace)+ASEFEyes+Affine(128,128,0.25,0.35)");
47 47  
48 48 // Video
49   - Globals->abbreviations.insert("DisplayVideo", "Stream([Show(false)+Discard])");
  49 + Globals->abbreviations.insert("DisplayVideo", "Stream([Cvt(Gray)+Resize(400,640)+Stasm+Delauney(true)+Draw(inPlace=true)+Show(false)+Discard])");
50 50 Globals->abbreviations.insert("PerFrameDetection", "Stream([SaveMat(original)+Cvt(Gray)+Cascade(FrontalFace)+ASEFEyes+RestoreMat(original)+Draw(inPlace=true),Show(false)+Discard])");
51 51 Globals->abbreviations.insert("AgeGenderDemo", "Stream([SaveMat(original)+Cvt(Gray)+Cascade(FrontalFace)+Expand+<FaceClassificationRegistration>+<FaceClassificationExtraction>+(<AgeRegressor>+Rename(Subject,Age)+Discard)/(<GenderClassifier>+Rename(Subject,Gender)+Discard)+RestoreMat(original)+Draw(inPlace=true)+DrawPropertiesPoint([Age,Gender],Affine_0,inPlace=true)+SaveMat(original)+Discard+Contract,RestoreMat(original)+FPSCalc+Show(false,[AvgFPS,Age,Gender])+Discard])");
52 52  
... ...
openbr/plugins/landmarks.cpp
  View file @9e3e99a
... ... @@ -33,6 +33,15 @@ class ProcrustesTransform : public Transform
33 33  
34 34 if (points.empty()) continue;
35 35  
  36 + QList<QRectF> rects = datum.file.rects();
  37 +
  38 + if (rects.size() > 1) qWarning("More than one rect in training data templates.");
  39 +
  40 + points.append(rects[0].topLeft());
  41 + points.append(rects[0].topRight());
  42 + points.append(rects[0].bottomLeft());
  43 + points.append(rects[0].bottomRight());
  44 +
36 45 cv::Scalar mean = cv::mean(OpenCVUtils::toPoints(points).toVector().toStdVector());
37 46 for (int i = 0; i < points.size(); i++) points[i] -= QPointF(mean[0],mean[1]);
38 47  
... ... @@ -67,7 +76,17 @@ class ProcrustesTransform : public Transform
67 76  
68 77 void project(const Template &src, Template &dst) const
69 78 {
  79 + dst.m() = src.m();
  80 +
70 81 QList<QPointF> points = src.file.points();
  82 + QList<QRectF> rects = src.file.rects();
  83 +
  84 + if (rects.size() > 1) qWarning("More than one rect in training data templates.");
  85 +
  86 + points.append(rects[0].topLeft());
  87 + points.append(rects[0].topRight());
  88 + points.append(rects[0].bottomLeft());
  89 + points.append(rects[0].bottomRight());
71 90  
72 91 cv::Scalar mean = cv::mean(OpenCVUtils::toPoints(points).toVector().toStdVector());
73 92 for (int i = 0; i < points.size(); i++) points[i] -= QPointF(mean[0],mean[1]);
... ... @@ -84,13 +103,13 @@ class ProcrustesTransform : public Transform
84 103  
85 104 Eigen::MatrixXf R = svd.matrixU()*svd.matrixV().transpose();
86 105  
87   - Eigen::MatrixXf dstPoints = srcPoints*R;
88   -
89   - points.clear();
90   -
91   - for (int i = 0; i < dstPoints.rows(); i++) points.append(QPointF(dstPoints(i,0),dstPoints(i,1)));
92   -
93   - dst.file.appendPoints(points);
  106 + dst.file.set("Procrustes_0_0", R(0,0));
  107 + dst.file.set("Procrustes_1_0", R(1,0));
  108 + dst.file.set("Procrustes_1_1", R(1,1));
  109 + dst.file.set("Procrustes_0_1", R(0,1));
  110 + dst.file.set("Procrustes_mean_0", mean[0]);
  111 + dst.file.set("Procrustes_mean_1", mean[1]);
  112 + dst.file.set("Procrustes_norm", norm);
94 113 }
95 114  
96 115 void store(QDataStream &stream) const
... ... @@ -109,10 +128,10 @@ BR_REGISTER(Transform, ProcrustesTransform)
109 128  
110 129 /*!
111 130 * \ingroup transforms
112   - * \brief Wraps STASM key point detector
  131 + * \brief Creates a delauney triangulation based on a set of points
113 132 * \author Scott Klum \cite sklum
114 133 */
115   -class DelauneyTransform : public UntrainableTransform
  134 +class DelaunayTransform : public UntrainableTransform
116 135 {
117 136 Q_OBJECT
118 137  
... ... @@ -121,48 +140,161 @@ class DelauneyTransform : public UntrainableTransform
121 140  
122 141 void project(const Template &src, Template &dst) const
123 142 {
124   - dst = src;
125   -
126 143 Subdiv2D subdiv(Rect(0,0,src.m().cols,src.m().rows));
127 144  
128   - foreach(const cv::Point2f& point, OpenCVUtils::toPoints(src.file.points())) subdiv.insert(point);
  145 + QList<cv::Point2f> points = OpenCVUtils::toPoints(src.file.points());
  146 + QList<QRectF> rects = src.file.rects();
  147 +
  148 + if (rects.size() > 1) qWarning("More than one rect in training data templates.");
  149 +
  150 + points.append(OpenCVUtils::toPoint(rects[0].topLeft()));
  151 + points.append(OpenCVUtils::toPoint(rects[0].topRight()));
  152 + points.append(OpenCVUtils::toPoint(rects[0].bottomLeft()));
  153 + points.append(OpenCVUtils::toPoint(rects[0].bottomRight()));
  154 +
  155 + for (int i = 0; i < points.size(); i++) subdiv.insert(points[i]);
129 156  
130 157 vector<Vec6f> triangleList;
131 158 subdiv.getTriangleList(triangleList);
132   - vector<Point> pt(3);
133 159  
134   - Scalar delaunay_color(0, 0, 0);
  160 + QList< QList<Point> > validTriangles;
  161 + int count = 0;
  162 +
  163 + for (size_t i = 0; i < triangleList.size(); ++i) {
  164 + vector<Point> pt(3);
  165 +
  166 + Vec6f t = triangleList[i];
  167 +
  168 + pt[0] = Point(cvRound(t[0]), cvRound(t[1]));
  169 + pt[1] = Point(cvRound(t[2]), cvRound(t[3]));
  170 + pt[2] = Point(cvRound(t[4]), cvRound(t[5]));
  171 +
  172 + bool inside = true;
  173 + for (int j = 0; j < 3; j++) {
  174 + if (pt[j].x > src.m().cols || pt[j].y > src.m().rows || pt[j].x <= 0 || pt[j].y <= 0) inside = false;
  175 + }
  176 +
  177 + if (inside) {
  178 + count++;
  179 + qDebug() << count << pt[0] << pt[1] << pt[2] << "Area" << contourArea(pt);
  180 +
  181 + QList<Point> triangleBuffer;
  182 +
  183 + triangleBuffer << pt[0] << pt[1] << pt[2];
  184 +
  185 + validTriangles.append(triangleBuffer);
  186 + }
  187 + }
  188 +
  189 + dst.m() = src.m().clone();
135 190  
136 191 if (draw) {
137   - int count = 0;
138   - for(size_t i = 0; i < triangleList.size(); ++i) {
139   - Vec6f t = triangleList[i];
  192 + foreach(const QList<Point>& triangle, validTriangles) {
  193 + line(dst.m(), triangle[0], triangle[1], Scalar(0,0,0), 1);
  194 + line(dst.m(), triangle[1], triangle[2], Scalar(0,0,0), 1);
  195 + line(dst.m(), triangle[2], triangle[0], Scalar(0,0,0), 1);
  196 + }
  197 + }
140 198  
141   - pt[0] = Point(cvRound(t[0]), cvRound(t[1]));
142   - pt[1] = Point(cvRound(t[2]), cvRound(t[3]));
143   - pt[2] = Point(cvRound(t[4]), cvRound(t[5]));
  199 + bool warp = true;
144 200  
145   - bool inside = true;
146   - for (int i = 0; i < 3; i++) {
147   - if(pt[i].x > dst.m().cols || pt[i].y > dst.m().rows || pt[i].x <= 0 || pt[i].y <= 0) {
148   - inside = false;
149   - }
  201 + if (warp) {
  202 + Eigen::MatrixXf R(2,2);
  203 + R(0,0) = src.file.get<float>("Procrustes_0_0");
  204 + R(1,0) = src.file.get<float>("Procrustes_1_0");
  205 + R(1,1) = src.file.get<float>("Procrustes_1_1");
  206 + R(0,1) = src.file.get<float>("Procrustes_0_1");
150 207  
  208 + cv::Scalar mean(2);
  209 + mean[0] = src.file.get<float>("Procrustes_mean_0");
  210 + mean[1] = src.file.get<float>("Procrustes_mean_1");
  211 +
  212 + qDebug() << mean[0] << mean[1];
  213 +
  214 + float norm = src.file.get<float>("Procrustes_norm");
  215 +
  216 + qDebug() << norm;
  217 +
  218 + dst.m() = Mat::zeros(src.m().rows,src.m().cols,src.m().type());
  219 +
  220 + foreach(const QList<Point> &triangle, validTriangles) {
  221 + Eigen::MatrixXf srcPoints(triangle.size(), 2);
  222 +
  223 + for (int j = 0; j < triangle.size(); j++) {
  224 + srcPoints(j,0) = (triangle[j].x-mean[0])/(norm/150.)+50;
  225 + srcPoints(j,1) = (triangle[j].y-mean[1])/(norm/150.)+50;
151 226 }
152   - if (inside) {
153   - count++;
154   - //qDebug() << count << pt[0] << pt[1] << pt[2] << "Area" << contourArea(pt);
155   - line(dst.m(), pt[0], pt[1], delaunay_color, 1);
156   - line(dst.m(), pt[1], pt[2], delaunay_color, 1);
157   - line(dst.m(), pt[2], pt[0], delaunay_color, 1);
158   - }
  227 +
  228 + Eigen::MatrixXf dstMat = srcPoints*R;
  229 +
  230 + Point2f test[3];
  231 + test[0] = triangle[0];
  232 + test[1] = triangle[1];
  233 + test[2] = triangle[2];
  234 + Point2f dstPoints[3];
  235 + dstPoints[0] = Point2f(dstMat(0,0),dstMat(0,1));
  236 + dstPoints[1] = Point2f(dstMat(1,0),dstMat(1,1));
  237 + dstPoints[2] = Point2f(dstMat(2,0),dstMat(2,1));
  238 +
  239 + Mat buffer(src.m().rows,src.m().cols,src.m().type());
  240 +
  241 + warpAffine(src.m(), buffer, getAffineTransform(test, dstPoints), Size(src.m().cols,src.m().rows));
  242 +
  243 + Mat mask = Mat::zeros(src.m().rows, src.m().cols, CV_8U);
  244 + Point maskPoints[1][3];
  245 + maskPoints[0][0] = dstPoints[0];
  246 + maskPoints[0][1] = dstPoints[1];
  247 + maskPoints[0][2] = dstPoints[2];
  248 + const Point* ppt = { maskPoints[0] };
  249 +
  250 + fillConvexPoly(mask, ppt, 3, Scalar(255,255,255), 8);
  251 +
  252 + Mat output(src.m().rows,src.m().cols,src.m().type());
  253 +
  254 + bitwise_and(buffer,mask,output);
  255 +
  256 + dst.m() += output;
159 257 }
160 258 }
161 259 }
162 260  
163 261 };
164 262  
165   -BR_REGISTER(Transform, DelauneyTransform)
  263 +BR_REGISTER(Transform, DelaunayTransform)
  264 +
  265 +/*!
  266 + * \ingroup transforms
  267 + * \brief Wraps STASM key point detector
  268 + * \author Scott Klum \cite sklum
  269 + */
  270 +class MeanTransform : public Transform
  271 +{
  272 + Q_OBJECT
  273 +
  274 + Mat mean;
  275 +
  276 + void train(const TemplateList &data)
  277 + {
  278 + mean = Mat::zeros(data[0].m().rows,data[0].m().cols,CV_32F);
  279 +
  280 + for (int i = 0; i < data.size()/2; i++) {
  281 + Mat converted;
  282 + data[i].m().convertTo(converted, CV_32F);
  283 + mean += converted;
  284 + }
  285 +
  286 + mean /= data.size()/2;
  287 + }
  288 +
  289 + void project(const Template &src, Template &dst) const
  290 + {
  291 + dst = src;
  292 + dst.m() = mean;
  293 + }
  294 +
  295 +};
  296 +
  297 +BR_REGISTER(Transform, MeanTransform)
166 298  
167 299 } // namespace br
168 300  
... ...
openbr/plugins/regions.cpp
  View file @9e3e99a
... ... @@ -243,7 +243,10 @@ class RectFromPointsTransform : public UntrainableTransform
243 243 dst.file.setPoints(points);
244 244  
245 245 if (crop) dst.m() = src.m()(Rect(std::max(0.0, minX - deltaWidth/2.0), std::max(0.0, minY - deltaHeight/2.0), std::min((double)src.m().cols, width), std::min((double)src.m().rows, height)));
246   - else dst.m() = src.m();
  246 + else {
  247 + dst.file.appendRect(QRectF(std::max(0.0, minX - deltaWidth/2.0), std::max(0.0, minY - deltaHeight/2.0), std::min((double)src.m().cols, width), std::min((double)src.m().rows, height)));
  248 + dst.m() = src.m();
  249 + }
247 250 }
248 251 };
249 252  
... ...
openbr/plugins/stasm4.cpp
  View file @9e3e99a
... ... @@ -38,7 +38,7 @@ class StasmInitializer : public Initializer
38 38 {
39 39 Globals->abbreviations.insert("RectFromStasmEyes","RectFromPoints([29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47],0.125,6.0)+Resize(44,164)");
40 40 Globals->abbreviations.insert("RectFromStasmBrow","RectFromPoints([17, 18, 19, 20, 21, 22, 23, 24],0.15,6)+Resize(28,132)");
41   - Globals->abbreviations.insert("RectFromStasmNose","RectFromPoints([48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58],0.15,1.25)+Resize(60,60)");
  41 + Globals->abbreviations.insert("RectFromStasmNose","RectFromPoints([48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58],0.15,1.25)");
42 42 Globals->abbreviations.insert("RectFromStasmMouth","RectFromPoints([59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76],0.3,3.0)+Resize(28,68)");
43 43 }
44 44 };
... ...