diff --git a/openbr/core/eigenutils.cpp b/openbr/core/eigenutils.cpp index ad0dc48..2bae4cd 100644 --- a/openbr/core/eigenutils.cpp +++ b/openbr/core/eigenutils.cpp @@ -95,3 +95,12 @@ MatrixXf removeRowCol(MatrixXf X, int row, int col) { } return Y; } + +MatrixXf pointsToMatrix(QList points) { + MatrixXf P(points.size(), 2); + for (int i = 0; i < points.size(); i++) { + P(i, 0) = points[i].x(); + P(i, 1) = points[i].y(); + } + return P; +} diff --git a/openbr/core/eigenutils.h b/openbr/core/eigenutils.h index 8f5e0b7..50b4b5d 100644 --- a/openbr/core/eigenutils.h +++ b/openbr/core/eigenutils.h @@ -29,9 +29,13 @@ void printEigen(Eigen::MatrixXd X); void printEigen(Eigen::MatrixXf X); void printSize(Eigen::MatrixXf X); -//Remove row and column from the matrix +//Remove row and column from the matrix: Eigen::MatrixXf removeRowCol(Eigen::MatrixXf X, int row, int col); +//Convert a point list into a matrix: +Eigen::MatrixXf pointsToMatrix(QList points); + + template inline QDataStream &operator<<(QDataStream &stream, const Eigen::Matrix< _Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols > &mat) { @@ -121,4 +125,5 @@ Eigen::MatrixBase eigStd(const Eigen::MatrixBase& x,int dim) } qFatal("A matrix can only have two dimensions"); } + #endif // EIGENUTILS_H diff --git a/openbr/plugins/draw.cpp b/openbr/plugins/draw.cpp index 9a7437b..224f5b3 100644 --- a/openbr/plugins/draw.cpp +++ b/openbr/plugins/draw.cpp @@ -527,6 +527,101 @@ class DrawSegmentation : public UntrainableTransform }; BR_REGISTER(Transform, DrawSegmentation) +/*! + * \ingroup transforms + * \brief Write all mats to disk as images. + * \author Brendan Klare \bklare + */ +class WriteImageTransform : public TimeVaryingTransform +{ + Q_OBJECT + Q_PROPERTY(QString outputDirectory READ get_outputDirectory WRITE set_outputDirectory RESET reset_outputDirectory STORED false) + Q_PROPERTY(QString imageName READ get_imageName WRITE set_imageName RESET reset_imageName STORED false) + Q_PROPERTY(QString imgExtension READ get_imgExtension WRITE set_imgExtension RESET reset_imgExtension STORED false) + BR_PROPERTY(QString, outputDirectory, "Temp") + BR_PROPERTY(QString, imageName, "image") + BR_PROPERTY(QString, imgExtension, "jpg") + + int cnt; + + void init() { + cnt = 0; + if (! QDir(outputDirectory).exists()) + QDir().mkdir(outputDirectory); + } + + void projectUpdate(const Template &src, Template &dst) + { + dst = src; + OpenCVUtils::saveImage(dst.m(), QString("%1/%2_%3.%4").arg(outputDirectory).arg(imageName).arg(cnt++, 5, QChar('0')).arg(imgExtension)); + } + +}; +BR_REGISTER(Transform, WriteImageTransform) + + +/** + * @brief The MeanImageTransform class computes the average template/image + * and save the result as an encoded image. + */ +class MeanImageTransform : public TimeVaryingTransform +{ + Q_OBJECT + + Q_PROPERTY(QString imgname READ get_imgname WRITE set_imgname RESET reset_imgname STORED false) + Q_PROPERTY(QString ext READ get_ext WRITE set_ext RESET reset_ext STORED false) + + BR_PROPERTY(QString, imgname, "average") + BR_PROPERTY(QString, ext, "jpg") + + Mat average; + int cnt; + + void init() + { + cnt = 0; + } + + void projectUpdate(const Template &src, Template &dst) + { + dst = src; + if (cnt == 0) { + if (src.m().channels() == 1) + average = Mat::zeros(dst.m().size(),CV_64FC1); + else if (src.m().channels() == 3) + average = Mat::zeros(dst.m().size(),CV_64FC3); + else + qFatal("Unsupported number of channels"); + } + + Mat temp; + if (src.m().channels() == 1) { + src.m().convertTo(temp, CV_64FC1); + average += temp; + } else if (src.m().channels() == 3) { + src.m().convertTo(temp, CV_64FC3); + average += temp; + } else + qFatal("Unsupported number of channels"); + + cnt++; + } + + virtual void finalize(TemplateList & output) + { + average /= float(cnt); + imwrite(QString("%1.%2").arg(imgname).arg(ext).toStdString(), average); + output = TemplateList(); + } + + +public: + MeanImageTransform() : TimeVaryingTransform(false, false) {} +}; + +BR_REGISTER(Transform, MeanImageTransform) + + // TODO: re-implement EditTransform using Qt #if 0 /*! diff --git a/openbr/plugins/landmarks.cpp b/openbr/plugins/landmarks.cpp index 993312d..a64dba8 100644 --- a/openbr/plugins/landmarks.cpp +++ b/openbr/plugins/landmarks.cpp @@ -141,193 +141,10 @@ BR_REGISTER(Transform, ProcrustesTransform) /*! * \ingroup transforms - * \brief Improved procrustes alignment of points, to include a post processing scaling of points - * to faciliate subsequent texture mapping. - * \author Brendan Klare \cite bklare - */ -class ProcrustesAlignTransform : public Transform -{ - Q_OBJECT - - Q_PROPERTY(float width READ get_width WRITE set_width RESET reset_width STORED false) - Q_PROPERTY(float padding READ get_padding WRITE set_padding RESET reset_padding STORED false) - BR_PROPERTY(float, width, 80) - BR_PROPERTY(float, padding, 8) - - Eigen::MatrixXf referenceShape; - float minX; - float minY; - float maxX; - float maxY; - float aspectRatio; - - void init() { - minX = FLT_MAX, - minY = FLT_MAX, - maxX = -FLT_MAX, - maxY = -FLT_MAX; - aspectRatio = 0; - } - - static MatrixXf getRotation(MatrixXf ref, MatrixXf sample) { - MatrixXf R = ref.transpose() * sample; - JacobiSVD svd(R, ComputeFullU | ComputeFullV); - R = svd.matrixU() * svd.matrixV(); - return R; - } - - //Converts x y points in a single vector to two column matrix - static MatrixXf vectorToMatrix(MatrixXf vector) { - int n = vector.rows(); - MatrixXf matrix(n / 2, 2); - for (int i = 0; i < n / 2; i++) { - for (int j = 0; j < 2; j++) { - matrix(i, j) = vector(i * 2 + j); - } - } - return matrix; - } - - static MatrixXf matrixToVector(MatrixXf matrix) { - int n2 = matrix.rows(); - MatrixXf vector(n2 * 2, 1); - for (int i = 0; i < n2; i++) { - for (int j = 0; j < 2; j++) { - vector(i * 2 + j) = matrix(i, j); - } - } - return vector; - } - - void train(const TemplateList &data) - { - MatrixXf points(data[0].file.points().size() * 2, data.size()); - - // Normalize all sets of points - for (int j = 0; j < data.size(); j++) { - QList imagePoints = data[j].file.points(); - - float meanX = 0, - meanY = 0; - for (int i = 0; i < imagePoints.size(); i++) { - points(i * 2, j) = imagePoints[i].x(); - points(i * 2 + 1, j) = imagePoints[i].y(); - meanX += imagePoints[i].x(); - meanY += imagePoints[i].y(); - } - meanX /= imagePoints.size(); - meanY /= imagePoints.size(); - - for (int i = 0; i < imagePoints.size(); i++) { - points(i * 2, j) -= meanX; - points(i * 2 + 1, j) -= meanY; - } - } - - //normalize scale - for (int i = 0; i < points.cols(); i++) - points.col(i) = points.col(i) / points.col(i).norm(); - - //Normalize rotation - MatrixXf refPrev; - referenceShape = vectorToMatrix(points.rowwise().sum() / points.cols()); - float diff = FLT_MAX; - while (diff > 1e-5) {//iterate until reference shape is stable - refPrev = referenceShape; - - for (int j = 0; j < points.cols(); j++) { - MatrixXf p = vectorToMatrix(points.col(j)); - MatrixXf R = getRotation(referenceShape, p); - p = p * R.transpose(); - points.col(j) = matrixToVector(p); - } - referenceShape = vectorToMatrix(points.rowwise().sum() / points.cols()); - diff = (matrixToVector(referenceShape) - matrixToVector(refPrev)).norm(); - } - - referenceShape = vectorToMatrix(points.rowwise().sum() / points.cols()); - - //Choose crop boundaries and adjustments that captures all data - for (int i = 0; i < points.rows(); i++) { - for (int j = 0; j < points.cols(); j++) { - if (i % 2 == 0) { - if (points(i,j) > maxX) - maxX = points(i, j); - if (points(i,j) < minX) - minX = points(i, j); - } else { - if (points(i,j) > maxY) - maxY = points(i, j); - if (points(i,j) < minY) - minY = points(i, j); - } - } - } - aspectRatio = (maxX - minX) / (maxY - minY); - } - - void project(const Template &src, Template &dst) const - { - QList imagePoints = src.file.points(); - MatrixXf p(imagePoints.size() * 2, 1); - for (int i = 0; i < imagePoints.size(); i++) { - p(i * 2) = imagePoints[i].x(); - p(i * 2 + 1) = imagePoints[i].y(); - } - p = vectorToMatrix(p); - - //Normalize translation - p.col(0) = p.col(0) - MatrixXf::Ones(p.rows(),1) * (p.col(0).sum() / p.rows()); - p.col(1) = p.col(1) - MatrixXf::Ones(p.rows(),1) * (p.col(1).sum() / p.rows()); - - //Normalize scale - p /= matrixToVector(p).norm(); - - //Normalize rotation - MatrixXf R = getRotation(referenceShape, p); - p = p * R.transpose(); - - //Translate and scale into output space and store in output list - QList procrustesPoints; - for (int i = 0; i < p.rows(); i++) - procrustesPoints.append( QPointF( - (p(i, 0) - minX) / (maxX - minX) * (width - 1) + padding, - (p(i, 1) - minY) / (maxY - minY) * (qRound( width / aspectRatio) - 1) + padding)); - - dst = src; - dst.file.setList("ProcrustesPoints", procrustesPoints); - dst.file.set("ProcrustesBound", QRectF(0, 0, width + 2 * padding, (qRound(width / aspectRatio) + 2 * padding))); - } - - void store(QDataStream &stream) const - { - stream << referenceShape; - stream << minX; - stream << minY; - stream << maxX; - stream << maxY; - stream << aspectRatio; - } - - void load(QDataStream &stream) - { - stream >> referenceShape; - stream >> minX; - stream >> minY; - stream >> maxX; - stream >> maxY; - stream >> aspectRatio; - } -}; - -BR_REGISTER(Transform, ProcrustesAlignTransform) - -/*! - * \ingroup transforms * \brief Creates a Delaunay triangulation based on a set of points * \author Scott Klum \cite sklum */ -class DelaunayTransform : public UntrainableTransform +class DelaunayTransform : public Transform { Q_OBJECT diff --git a/openbr/plugins/stasm4.cpp b/openbr/plugins/stasm4.cpp index 6fa2db7..2d0a1fe 100644 --- a/openbr/plugins/stasm4.cpp +++ b/openbr/plugins/stasm4.cpp @@ -37,6 +37,8 @@ class StasmInitializer : public Initializer { Globals->abbreviations.insert("RectFromStasmEyes","RectFromPoints([28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47],0.3,5.3)"); Globals->abbreviations.insert("RectFromStasmBrow","RectFromPoints([16,17,18,19,20,21,22,23,24,25,26,27],0.15,5)"); + Globals->abbreviations.insert("RectFromStasmPeriocular","RectFromPoints([28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47,16,17,18,19,20,21,22,23,24,25,26,27]],0.3,5.3)"); + Globals->abbreviations.insert("RectFromStasmBrow","RectFromPoints([16,17,18,19,20,21,22,23,24,25,26,27],0.15,5)"); Globals->abbreviations.insert("RectFromStasmNose","RectFromPoints([48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58],0.15,1.15)"); Globals->abbreviations.insert("RectFromStasmNoseWithBridge", "RectFromPoints([21, 22, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58],0.15,.6)"); Globals->abbreviations.insert("RectFromStasmMouth","RectFromPoints([59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76],0.3,2)"); @@ -73,8 +75,11 @@ class StasmTransform : public UntrainableTransform void project(const Template &src, Template &dst) const { - if (src.m().channels() != 1) qFatal("Stasm expects single channel matrices."); - + Mat stasmSrc(src); + if (src.m().channels() == 3) + cvtColor(src, stasmSrc, CV_BGR2GRAY); + else if (src.m().channels() != 1) + qFatal("Stasm expects single channel matrices."); dst = src; StasmCascadeClassifier *stasmCascade = stasmCascadeResource.acquire(); @@ -115,14 +120,14 @@ class StasmTransform : public UntrainableTransform else if (i == 39) /*Stasm Left Eye*/ { eyes[2*i] = leftEye.x(); eyes[2*i+1] = leftEye.y(); } else { eyes[2*i] = 0; eyes[2*i+1] = 0; } } - stasm_search_pinned(landmarks, eyes, reinterpret_cast(src.m().data), src.m().cols, src.m().rows, NULL); + stasm_search_pinned(landmarks, eyes, reinterpret_cast(stasmSrc.data), stasmSrc.cols, stasmSrc.rows, NULL); // The ASM in Stasm is guaranteed to converge in this case foundFace = 1; } } - if (!foundFace) stasm_search_single(&foundFace, landmarks, reinterpret_cast(src.m().data), src.m().cols, src.m().rows, *stasmCascade, NULL, NULL); + if (!foundFace) stasm_search_single(&foundFace, landmarks, reinterpret_cast(stasmSrc.data), stasmSrc.cols, stasmSrc.rows, *stasmCascade, NULL, NULL); if (stasm3Format) { nLandmarks = 76;