Stasm properly uses internal detector to landmark all faces in image

Scott Klum
1 parent 7693fd5f
Showing 7 changed files with 116 additions and 282 deletions
3rdparty/stasm/stasm/CMakeLists.txt
3rdparty/stasm/stasm/stasm/src/print.cpp
3rdparty/stasm4.0.0/stasm/MOD_1/facedet.cpp
3rdparty/stasm4.0.0/stasm/MOD_1/facedet.h
3rdparty/stasm4.0.0/stasm/stasm_lib.cpp
3rdparty/stasm4.0.0/stasm/stasm_lib.h
openbr/plugins/metadata/stasm4.cpp
-# Stasm CMakeLists
-
-aux_source_directory(stasm/src STASM_SRC)
-include_directories(stasm/include)
-
-#aux_source_directory(tasm/src TASM_SRC)
-#include_directories(tasm/include)
-
-add_library(stasm SHARED ${STASM_SRC} ${TASM_SRC})
-qt5_use_modules(stasm ${QT_DEPENDENCIES})
-
-set_target_properties(stasm PROPERTIES
-                                   DEFINE_SYMBOL STASM_LIBRARY
-                                   VERSION ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH}
-                                   SOVERSION ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}
-                                   LINK_INTERFACE_LIBRARIES ""
-                                   AUTOMOC TRUE)
-
-target_link_libraries(stasm ${OpenCV_LIBS} ${Qt5Core_QTMAIN_LIBRARIES})
-
-file(GLOB STASM_HEADERS stasm/include/*.h)
-install(FILES ${STASM_HEADERS} DESTINATION include/stasm)
-file(GLOB CLASSIC_HEADERS stasm/include/classic/*.mh)
-install(FILES ${CLASSIC_HEADERS} DESTINATION include/stasm/classic)
-file(GLOB HAT_HEADERS stasm/include/hat/*.mh)
-install(FILES ${HAT_HEADERS} DESTINATION include/stasm/hat)
-#file(GLOB TASM_HEADERS tasm/include/*.h)
-#install(FILES ${TASM_HEADERS} DESTINATION include/tasm)
-
-install(TARGETS stasm RUNTIME DESTINATION bin LIBRARY DESTINATION lib ARCHIVE DESTINATION lib)
-// print.cpp: printing and logging utilities for the Stasm library
-//
-// Copyright (C) 2005-2013, Stephen Milborrow
-
-#include "print.h"
-#include "err.h"
-#include "misc.h"
-
-#include <stdio.h>
-#include <sys/stat.h>
-#include <stdarg.h>
-
-namespace stasm
-{
-bool print_g;   // true to allow output to stdout (but error msgs always printed)
-
-bool trace_g;   // true to trace Stasm internal operation
-
-static FILE* logfile_g;  // lprintfs go to this log file as well as stdout
-
-//-----------------------------------------------------------------------------
-
-// Open the log file. After this, when you call lprintf, you print to the log
-// file (as well as to stdout).  This inits the global variable logfile_g.
-
-void OpenLogFile(     // also inits the global variable logfile_g
-    const char* path) // in: log file path, default is "stasm.log"
-{
-    if (!logfile_g)
-    {
-        if (print_g)
-            printf("Generating %s\n", path);
-        logfile_g = fopen(path, "wb");
-        if (!logfile_g)
-            Err("Cannot open \"%s\"", path);
-        // check that we can write to the log file
-        if (fputs("log file\n", logfile_g) < 0)
-            Err("Cannot write to \"%s\"", path);
-        rewind(logfile_g); // rewind so above test msg is not in the log file
-    }
-}
-
-// Like printf but only prints if print_g flag is set.
-// Also prints to the log file if it is open (regardless of print_g).
-
-void lprintf(const char* format, ...)   // args like printf
-{
-    char s[SBIG];
-    va_list args;
-    va_start(args, format);
-    VSPRINTF(s, format, args);
-    va_end(args);
-    if (print_g)
-    {
-        printf("%s", s);
-        fflush(stdout); // flush so if there is a crash we can see what happened
-    }
-    if (logfile_g)
-    {
-        // we don't check fputs here, to prevent recursive calls and msgs
-        fputs(s, logfile_g);
-        fflush(logfile_g);
-    }
-}
-
-// Like printf but prints to the log file only (and not to stdout).
-// Used for detailed stuff that we don't usually want to see.
-
-void logprintf(const char* format, ...) // args like printf
-{
-    if (logfile_g)
-    {
-        char s[SBIG];
-        va_list args;
-        va_start(args, format);
-        VSPRINTF(s, format, args);
-        va_end(args);
-        // we don't check fputs here, to prevent recursive calls and msgs
-        fputs(s, logfile_g);
-        fflush(logfile_g);
-    }
-}
-
-// Like lprintf but always prints even if print_g is false.
-
-void lprintf_always(const char* format, ...)
-{
-    char s[SBIG];
-    va_list args;
-    va_start(args, format);
-    VSPRINTF(s, format, args);
-    va_end(args);
-    printf("%s", s);
-    fflush(stdout); // flush so if there is a crash we can see what happened
-    if (logfile_g)
-    {
-        // we don't check fputs here, to prevent recursive calls and msgs
-        fputs(s, logfile_g);
-        fflush(logfile_g);
-    }
-}
-
-// Like puts but prints to the log file as well if it is open,
-// and does not append a newline.
-
-void lputs(const char* s)
-{
-    printf("%s", s);
-    fflush(stdout);     // flush so if there is a crash we can see what happened
-    logprintf("%s", s);
-}
-
-// Print message only once on the screen, and only 100 times to the log file.
-// This is used when similar messages could be printed many times and it
-// suffices to let the user know just once.  By convention the message is
-// printed followed by "..." so the user knows that just the first message
-// was printed.  The user can look in the log file for further messages if
-// necessary (but we print only 100 times to the log file --- else all the
-// log prints make tasm slow).
-
-void PrintOnce(
-    int&        printed,     // io: zero=print, nonzero=no print
-    const char* format, ...) // in: args like printf
-{
-    char s[SBIG];
-    va_list args;
-    va_start(args, format);
-    VSPRINTF(s, format, args);
-    va_end(args);
-    if (printed == 0 && print_g)
-    {
-        printed = 1;
-        printf("%s", s);
-        fflush(stdout); // flush so if there is a crash we can see what happened
-    }
-    if (printed < 100 && logfile_g)
-    {
-        fputs(s, logfile_g);
-        fflush(logfile_g);
-        printed++;
-        if (printed == 100)
-            logprintf("no more prints of the above message (printed == 100)\n");
-    }
-}
-
-} // namespace stasm
@@ -47,28 +47,16 @@ void DetectFaces(          // all face rects into detpars
     int          minwidth,
     cv::CascadeClassifier cascade) // in: as percent of img width
 {
-    int leftborder = 0, topborder = 0; // border size in pixels
-    Image bordered_img(BORDER_FRAC == 0?
-                       img: EnborderImg(leftborder, topborder, img));
-
-    // Detection results are very slightly better with equalization
-    // (tested on the MUCT images, which are not pre-equalized), and
-    // it's quick enough to equalize (roughly 10ms on a 1.6 GHz laptop).
-
-    Image equalized_img; cv::equalizeHist(bordered_img, equalized_img);
-
     CV_Assert(minwidth >= 1 && minwidth <= 100);
-    int minpix = MAX(100, cvRound(img.cols * minwidth / 100.));
-
     // the params below are accurate but slow
-    static const double SCALE_FACTOR   = 1.1;
-    static const int    MIN_NEIGHBORS  = 3;
+    static const double SCALE_FACTOR   = 1.2;
+    static const int    MIN_NEIGHBORS  = 5;
     static const int    DETECTOR_FLAGS = 0;
     vec_Rect facerects = // all face rects in image
-        Detect(equalized_img, &cascade, NULL,
-               SCALE_FACTOR, MIN_NEIGHBORS, DETECTOR_FLAGS, minpix);
+        Detect(img, &cascade, NULL,
+               SCALE_FACTOR, MIN_NEIGHBORS, DETECTOR_FLAGS, 64);
     // copy face rects into the detpars vector
@@ -80,8 +68,6 @@ void DetectFaces(          // all face rects into detpars
         // detpar.x and detpar.y is the center of the face rectangle
         detpar.x = facerect->x + facerect->width / 2.;
         detpar.y = facerect->y + facerect->height / 2.;
-        detpar.x -= leftborder; // discount the border we added earlier
-        detpar.y -= topborder;
         detpar.width  = double(facerect->width);
         detpar.height = double(facerect->height);
         detpar.yaw = 0; // assume face has no yaw in this version of Stasm
@@ -187,10 +173,10 @@ void FaceDet::DetectFaces_(  // call once per image to find all the faces
     //CV_Assert(!facedet_g.empty()); // check that OpenFaceDetector_ was called
     DetectFaces(detpars_, img, minwidth, cascade);
-    DiscardMissizedFaces(detpars_);
+    //DiscardMissizedFaces(detpars_);
     if (multiface) // order faces on increasing distance from left margin
     {
-        sort(detpars_.begin(), detpars_.end(), IncreasingLeftMargin);
+        sort(detpars_.begin(), detpars_.end(), DecreasingWidth);
     }
     else
     {
@@ -39,9 +39,9 @@ public:
     FaceDet() {}                  // constructor
-private:
     vector<DetPar>  detpars_;     // all the valid faces in the current image
+private:
     int             iface_;       // index of current face for NextFace_
                                   // indexes into detpars_
@@ -160,7 +160,8 @@ int stasm_search_auto_ext( // extended version of stasm_search_auto
     const char* data,
     const int width,
     const int height,
-    StasmCascadeClassifier cascade)
+    StasmCascadeClassifier cascade,
+    FaceDet &detection)
 {
     int returnval = 1;     // assume success
     *foundface = 0;        // but assume no face found
@@ -176,15 +177,14 @@ int stasm_search_auto_ext( // extended version of stasm_search_auto
         // Allocate image
         Image img = Image(height, width,(unsigned char*)data);
-        FaceDet facedet;
-
         // call the face detector to detect the face rectangle(s)
-        facedet.DetectFaces_(img, NULL, false, 10, NULL, cascade.faceCascade);
+        if (detection.detpars_.empty())
+            detection.DetectFaces_(img, NULL, true, 10, NULL, cascade.faceCascade);
         // Get the start shape for the next face in the image, and the ROI around it.
         // The shape will be wrt the ROI frame.
         if (NextStartShapeAndRoi(shape, face_roi, detpar_roi, detpar,
-                                 img, mods_g, facedet, cascade))
+                                 img, mods_g, detection, cascade))
         {
             // now working with maybe flipped ROI and start shape in ROI frame
             *foundface = 1;
@@ -219,9 +219,10 @@ int stasm_search_auto(// call repeatedly to find all faces
     const char *data,
     const int width,
     const int height,
-    StasmCascadeClassifier cascade)
+    StasmCascadeClassifier cascade,
+    FaceDet &detection)
 {
-    return stasm_search_auto_ext(foundface, landmarks, NULL, data, width, height, cascade);
+    return stasm_search_auto_ext(foundface, landmarks, NULL, data, width, height, cascade, detection);
 }
 int stasm_search_single(   // wrapper for stasm_search_auto and friends
@@ -237,7 +238,8 @@ int stasm_search_single(   // wrapper for stasm_search_auto and friends
     (void) datadir;
     (void) imgpath;
-    return stasm_search_auto(foundface, landmarks, img, width, height, cascade);
+    FaceDet detection;
+    return stasm_search_auto(foundface, landmarks, img, width, height, cascade, detection);
 }
 int stasm_search_pinned(    // call after the user has pinned some points
@@ -64,6 +64,10 @@
 #include "stasmcascadeclassifier.h"
+namespace stasm {
+    class FaceDet;
+}
+
 static const int stasm_NLANDMARKS = 77; // number of landmarks
 extern const char* const stasm_VERSION;
@@ -89,7 +93,8 @@ int stasm_search_auto(       // call repeatedly to find all faces
     const char*  data,
     const int    width,
     const int    height,
-    StasmCascadeClassifier cascade);
+    StasmCascadeClassifier cascade,
+    stasm::FaceDet &detection);
 extern "C"
 int stasm_search_single(     // wrapper for stasm_search_auto and friends
 #include <QString>
-#include <stasm_lib.h>
 #include <stasmcascadeclassifier.h>
+#include <stasm_lib.h>
+#include <stasm.h>
 #include <opencv2/opencv.hpp>
 #include <openbr/plugins/openbr_internal.h>
@@ -56,14 +57,12 @@ BR_REGISTER(Initializer, StasmInitializer)
  * \brief Wraps STASM key point detector
  * \author Scott Klum \cite sklum
  */
-class StasmTransform : public UntrainableTransform
+class StasmTransform : public UntrainableMetaTransform
 {
     Q_OBJECT
     Q_PROPERTY(bool stasm3Format READ get_stasm3Format WRITE set_stasm3Format RESET reset_stasm3Format STORED false)
     BR_PROPERTY(bool, stasm3Format, false)
-    Q_PROPERTY(bool clearLandmarks READ get_clearLandmarks WRITE set_clearLandmarks RESET reset_clearLandmarks STORED false)
-    BR_PROPERTY(bool, clearLandmarks, false)
     Q_PROPERTY(QList<float> pinPoints READ get_pinPoints WRITE set_pinPoints RESET reset_pinPoints STORED false)
     BR_PROPERTY(QList<float>, pinPoints, QList<float>())
     Q_PROPERTY(QStringList pinLabels READ get_pinLabels WRITE set_pinLabels RESET reset_pinLabels STORED false)
@@ -77,86 +76,104 @@ class StasmTransform : public UntrainableTransform
         stasmCascadeResource.setResourceMaker(new StasmResourceMaker());
     }
-    void project(const Template &src, Template &dst) const
+    QList<QPointF> convertLandmarks(int nLandmarks, float *landmarks) const
     {
-        Mat stasmSrc(src);
-        if (src.m().channels() == 3)
-            cvtColor(src, stasmSrc, CV_BGR2GRAY);
-        else if (src.m().channels() != 1)
-            qFatal("Stasm expects single channel matrices.");
-
-        if (!stasmSrc.isContinuous())
-            qFatal("Stasm expects continuous matrix data.");
-        dst = src;
-
-        int foundFace = 0;
-        int nLandmarks = stasm_NLANDMARKS;
-        float landmarks[2 * stasm_NLANDMARKS];
-
-        bool searchPinned = false;
-
-        QPointF rightEye, leftEye;
-        /* Two use cases are accounted for:
-         * 1. Pin eyes without normalization: in this case the string list should contain the KEYS for right then left eyes, respectively.
-         * 2. Pin eyes with normalization: in this case the string list should contain the COORDINATES of the right then left eyes, respectively.
-         * Currently, we only support normalization with a transformation such that the src file contains Affine_0 and Affine_1.  Checking for
-         * these keys prevents us from pinning eyes on a face that wasn't actually transformed (see AffineTransform).
-         * If both cases fail, we default to stasm_search_single. */
-
-        if (!pinPoints.isEmpty() && src.file.contains("Affine_0") && src.file.contains("Affine_1")) {
-            rightEye = QPointF(pinPoints.at(0), pinPoints.at(1));
-            leftEye = QPointF(pinPoints.at(2), pinPoints.at(3));
-            searchPinned = true;
-        } else if (!pinLabels.isEmpty()) {
-            rightEye = src.file.get<QPointF>(pinLabels.at(0), QPointF());
-            leftEye = src.file.get<QPointF>(pinLabels.at(1), QPointF());
-            searchPinned = true;
-        }
-	
-        if (searchPinned) {
-            float pins[2 * stasm_NLANDMARKS];
-
-            for (int i = 0; i < nLandmarks; i++) {
-                if      (i == 38) /* Stasm Right Eye */ { pins[2*i] = rightEye.x(); pins[2*i+1] = rightEye.y(); }
-                else if (i == 39) /* Stasm Left Eye  */ { pins[2*i] = leftEye.x();  pins[2*i+1] = leftEye.y(); }
-                else { pins[2*i] = 0; pins[2*i+1] = 0; }
-            }
+        if (stasm3Format)
+            stasm_convert_shape(landmarks, 76);
-            stasm_search_pinned(landmarks, pins, reinterpret_cast<const char*>(stasmSrc.data), stasmSrc.cols, stasmSrc.rows, NULL);
-
-            // The ASM in Stasm is guaranteed to converge in this case
-            foundFace = 1;
+        QList<QPointF> points;
+        for (int i = 0; i < nLandmarks; i++) {
+            QPointF point(landmarks[2 * i], landmarks[2 * i + 1]);
+            points.append(point);
         }
-        if (!foundFace) {
-            StasmCascadeClassifier *stasmCascade = stasmCascadeResource.acquire();
-            stasm_search_single(&foundFace, landmarks, reinterpret_cast<const char*>(stasmSrc.data), stasmSrc.cols, stasmSrc.rows, *stasmCascade, NULL, NULL);
-            stasmCascadeResource.release(stasmCascade);
-        }
+        return points;
+    }
-        if (stasm3Format) {
-            nLandmarks = 76;
-            stasm_convert_shape(landmarks, nLandmarks);
-        }
+    void project(const Template &src, Template &dst) const
+    {
+        TemplateList temp;
+        project(TemplateList() << src, temp);
+        if (!temp.isEmpty()) dst = temp.first();
+    }
-        // For convenience, if these are the only points/rects we want to deal with as the algorithm progresses
-        if (clearLandmarks) {
-            dst.file.clearPoints();
-            dst.file.clearRects();
-        }
+    void project(const TemplateList &src, TemplateList &dst) const
+    {
+        foreach (const Template &t, src) {
+            Mat stasmSrc(t);
+            if (t.m().channels() == 3)
+                cvtColor(t, stasmSrc, CV_BGR2GRAY);
+            else if (t.m().channels() != 1)
+                qFatal("Stasm expects single channel matrices.");
+
+            if (!stasmSrc.isContinuous())
+                qFatal("Stasm expects continuous matrix data.");
+
+            int foundFace = 0;
+            int nLandmarks = stasm_NLANDMARKS;
+            float landmarks[2 * stasm_NLANDMARKS];
+
+            bool searchPinned = false;
+
+            QPointF rightEye, leftEye;
+            /* Two use cases are accounted for:
+             * 1. Pin eyes without normalization: in this case the string list should contain the KEYS for right then left eyes, respectively.
+             * 2. Pin eyes with normalization: in this case the string list should contain the COORDINATES of the right then left eyes, respectively.
+             * Currently, we only support normalization with a transformation such that the src file contains Affine_0 and Affine_1.  Checking for
+             * these keys prevents us from pinning eyes on a face that wasn't actually transformed (see AffineTransform).
+             * If both cases fail, we default to stasm_search_single. */
+
+            if (!pinPoints.isEmpty() && t.file.contains("Affine_0") && t.file.contains("Affine_1")) {
+                rightEye = QPointF(pinPoints.at(0), pinPoints.at(1));
+                leftEye = QPointF(pinPoints.at(2), pinPoints.at(3));
+                searchPinned = true;
+            } else if (!pinLabels.isEmpty()) {
+                rightEye = t.file.get<QPointF>(pinLabels.at(0), QPointF());
+                leftEye = t.file.get<QPointF>(pinLabels.at(1), QPointF());
+                searchPinned = true;
+            }
+
+            if (searchPinned) {
+                float pins[2 * stasm_NLANDMARKS];
+
+                for (int i = 0; i < nLandmarks; i++) {
+                    if      (i == 38) /* Stasm Right Eye */ { pins[2*i] = rightEye.x(); pins[2*i+1] = rightEye.y(); }
+                    else if (i == 39) /* Stasm Left Eye  */ { pins[2*i] = leftEye.x();  pins[2*i+1] = leftEye.y(); }
+                    else { pins[2*i] = 0; pins[2*i+1] = 0; }
+                }
+
+                stasm_search_pinned(landmarks, pins, reinterpret_cast<const char*>(stasmSrc.data), stasmSrc.cols, stasmSrc.rows, NULL);
+
+                // The ASM in Stasm is guaranteed to converge in this case
+                foundFace = 1;
+                Template u(t.file, t.m());
+                QList<QPointF> points = convertLandmarks(nLandmarks, landmarks);
+                u.file.set("StasmRightEye", points[38]);
+                u.file.set("StasmLeftEye", points[39]);
+                u.file.appendPoints(points);
+                dst.append(u);
+            }
-        if (!foundFace) {
-            if (Globals->verbose) qWarning("No face found in %s.", qPrintable(src.file.fileName()));
-            dst.file.fte = true;
-        } else {
-            QList<QPointF> points;
-            for (int i = 0; i < nLandmarks; i++) {
-                QPointF point(landmarks[2 * i], landmarks[2 * i + 1]);
-                points.append(point);
+            if (!foundFace) {
+                StasmCascadeClassifier *stasmCascade = stasmCascadeResource.acquire();
+                foundFace = 1;
+                stasm::FaceDet detection;
+                while (foundFace) {
+                    stasm_search_auto(&foundFace, landmarks, reinterpret_cast<const char*>(stasmSrc.data), stasmSrc.cols, stasmSrc.rows, *stasmCascade, detection);
+                    if (foundFace) {
+                        Template u(t.file, t.m());
+                        QList<QPointF> points = convertLandmarks(nLandmarks, landmarks);
+                        u.file.set("StasmRightEye", points[38]);
+                        u.file.set("StasmLeftEye", points[39]);
+                        u.file.appendPoints(points);
+                        dst.append(u);
+                    }
+
+                    if (!Globals->enrollAll)
+                        break;
+                }
+                stasmCascadeResource.release(stasmCascade);
             }
-            dst.file.set("StasmRightEye", points[38]);
-            dst.file.set("StasmLeftEye", points[39]);
-            dst.file.appendPoints(points);
         }
     }
 };