Merge branch 'master' of https://github.com/biometrics/openbr

Scott Klum
2 parents 5e6b30e9 9d711fd6
Showing 10 changed files with 199 additions and 92 deletions
openbr/CMakeLists.txt
openbr/core/bee.cpp
openbr/janus
openbr/janus.cpp
openbr/openbr.cpp
openbr/openbr_plugin.h
openbr/plugins/algorithms.cpp
openbr/plugins/draw.cpp
openbr/plugins/segmentation.cpp
openbr/plugins/stream.cpp
@@ -34,7 +34,7 @@ set(JANUS_BUILD_PP5_WRAPPER ${BR_WITH_PP5} CACHE BOOL &quot;Build Janus implementatio
 set(JANUS_BUILD_DOCS ${BR_BUILD_DOCUMENTATION} CACHE BOOL "Build Janus HTML Doxygen documentation")
 mark_as_advanced(JANUS_BUILD_PP5_WRAPPER)
 mark_as_advanced(JANUS_BUILD_DOCS)
-set(JANUS_TEST_IMPLEMENTATION openbr)
+set(JANUS_IMPLEMENTATION openbr)
 add_subdirectory(janus)
 # Install
@@ -172,6 +172,8 @@ Mat BEE::readMat(const br::File &amp;matrix, QString *targetSigset, QString *querySi
     qint64 read = file.read((char*)m.data, bytesExpected);
     if (read != bytesExpected)
         qFatal("Invalid matrix size.");
+    if (!file.atEnd())
+        qFatal("Expected matrix end of file.");
     file.close();
     Mat result;
-Subproject commit 0eaa00f19256fee2f24033ff68b526ea6320624d
+Subproject commit 02fd545b2dbb8ea2ba10dbf67e429da598545d75
@@ -3,140 +3,138 @@
 #endif
 #include "janus.h"
+#include "janus_io.h"
 #include "openbr_plugin.h"
-// Use the provided default implementation of some functions
-#include "janus/src/janus.cpp"
-
 using namespace br;
 static QSharedPointer<Transform> transform;
 static QSharedPointer<Distance> distance;
+size_t janus_max_template_size()
+{
+    return JANUS_MAX_TEMPLATE_SIZE_LIMIT;
+}
+
 janus_error janus_initialize(const char *sdk_path, const char *model_file)
 {
     int argc = 1;
     const char *argv[1] = { "janus" };
     Context::initialize(argc, (char**)argv, sdk_path);
     QString algorithm = model_file;
-    if (algorithm.isEmpty()) algorithm = "Cvt(Gray)+Affine(88,88,0.25,0.35)+<FaceRecognitionExtraction>+<FaceRecognitionEmbedding>+<FaceRecognitionQuantization>:ByteL1";
-    transform = Transform::fromAlgorithm(algorithm, false);
-    distance = Distance::fromAlgorithm(algorithm);
+    if (algorithm.isEmpty()) {
+        transform = Transform::fromAlgorithm("Cvt(Gray)+Affine(88,88,0.25,0.35)+<FaceRecognitionExtraction>+<FaceRecognitionEmbedding>+<FaceRecognitionQuantization>", false);
+        distance = Distance::fromAlgorithm("FaceRecognition");
+    } else if (algorithm == "PP5") {
+        transform.reset(Transform::make("PP5Enroll", NULL));
+        distance.reset(Distance::make("PP5Compare", NULL));
+    } else {
+        transform = Transform::fromAlgorithm(algorithm, false);
+        distance = Distance::fromAlgorithm(algorithm);
+    }
     return JANUS_SUCCESS;
 }
 janus_error janus_finalize()
 {
+    transform.reset();
+    distance.reset();
     Context::finalize();
     return JANUS_SUCCESS;
 }
-struct janus_incomplete_template_type
-{
-    QList<cv::Mat> data;
-};
+struct janus_template_type : public Template
+{};
-janus_error janus_initialize_template(janus_incomplete_template *incomplete_template)
+janus_error janus_initialize_template(janus_template *template_)
 {
-    *incomplete_template = new janus_incomplete_template_type();
+    *template_ = new janus_template_type();
     return JANUS_SUCCESS;
 }
-janus_error janus_add_image(const janus_image image, const janus_attribute_list attributes, janus_incomplete_template incomplete_template)
+janus_error janus_augment(const janus_image image, const janus_attribute_list attributes, janus_template template_)
 {
     Template t;
     t.append(cv::Mat(image.height,
                      image.width,
-                     image.color_space == JANUS_GRAY8 ? CV_8UC1 : CV_8UC1,
+                     image.color_space == JANUS_GRAY8 ? CV_8UC1 : CV_8UC3,
                      image.data));
     for (size_t i=0; i<attributes.size; i++)
         t.file.set(janus_attribute_to_string(attributes.attributes[i]), attributes.values[i]);
-    if (!t.file.contains("JANUS_RIGHT_EYE_X") ||
-        !t.file.contains("JANUS_RIGHT_EYE_Y") ||
-        !t.file.contains("JANUS_LEFT_EYE_X") ||
-        !t.file.contains("JANUS_LEFT_EYE_Y"))
+    if (!t.file.contains("RIGHT_EYE_X") ||
+        !t.file.contains("RIGHT_EYE_Y") ||
+        !t.file.contains("LEFT_EYE_X") ||
+        !t.file.contains("LEFT_EYE_Y"))
         return JANUS_SUCCESS;
-    t.file.set("Affine_0", QPointF(t.file.get<float>("JANUS_RIGHT_EYE_X"), t.file.get<float>("JANUS_RIGHT_EYE_Y")));
-    t.file.set("Affine_1", QPointF(t.file.get<float>("JANUS_LEFT_EYE_X"), t.file.get<float>("JANUS_LEFT_EYE_Y")));
+    t.file.set("Affine_0", QPointF(t.file.get<float>("RIGHT_EYE_X"), t.file.get<float>("RIGHT_EYE_Y")));
+    t.file.set("Affine_1", QPointF(t.file.get<float>("LEFT_EYE_X"), t.file.get<float>("LEFT_EYE_Y")));
     Template u;
     transform->project(t, u);
-    incomplete_template->data.append(u);
+    template_->append(u);
     return JANUS_SUCCESS;
 }
-janus_error janus_finalize_template(janus_incomplete_template incomplete_template, janus_template template_, size_t *bytes)
+janus_error janus_finalize_template(janus_template template_, janus_flat_template flat_template, size_t *bytes)
 {    
-    size_t templateBytes = 0;
-    size_t numTemplates = 0;
-    *bytes = sizeof(templateBytes) + sizeof(numTemplates);
-    janus_template pos = template_ + *bytes;
-
-    foreach (const cv::Mat &m, incomplete_template->data) {
+    foreach (const cv::Mat &m, *template_) {
         assert(m.isContinuous());
-        const size_t currentTemplateBytes = m.rows * m.cols * m.elemSize();
-        if (templateBytes == 0)
-            templateBytes = currentTemplateBytes;
-        if (templateBytes != currentTemplateBytes)
-            return JANUS_UNKNOWN_ERROR;
-        if (*bytes + templateBytes > janus_max_template_size())
+        const size_t templateBytes = m.rows * m.cols * m.elemSize();
+        if (*bytes + sizeof(size_t) + templateBytes > janus_max_template_size())
             break;
-        memcpy(pos, m.data, templateBytes);
-        *bytes += templateBytes;
-        pos = pos + templateBytes;
-        numTemplates++;
+        memcpy(flat_template, &templateBytes, sizeof(templateBytes));
+        flat_template += sizeof(templateBytes);
+        memcpy(flat_template, m.data, templateBytes);
+        flat_template += templateBytes;
+        *bytes += sizeof(size_t) + templateBytes;
     }
-    *(reinterpret_cast<size_t*>(template_)+0) = templateBytes;
-    *(reinterpret_cast<size_t*>(template_)+1) = numTemplates;
-    delete incomplete_template;
+    delete template_;
     return JANUS_SUCCESS;
 }
-janus_error janus_verify(const janus_template a, const size_t a_bytes, const janus_template b, const size_t b_bytes, double *similarity)
+janus_error janus_verify(const janus_flat_template a, const size_t a_bytes, const janus_flat_template b, const size_t b_bytes, double *similarity)
 {
-    (void) a_bytes;
-    (void) b_bytes;
-
-    size_t a_template_bytes, a_templates, b_template_bytes, b_templates;
-    a_template_bytes = *(reinterpret_cast<size_t*>(a)+0);
-    a_templates = *(reinterpret_cast<size_t*>(a)+1);
-    b_template_bytes = *(reinterpret_cast<size_t*>(b)+0);
-    b_templates = *(reinterpret_cast<size_t*>(b)+1);
-    if (a_template_bytes != b_template_bytes)
-        return JANUS_UNKNOWN_ERROR;
+    *similarity = 0;
-    float dist = 0;
-    for (size_t i=0; i<a_templates; i++)
-        for (size_t j=0; j<b_templates; j++)
-            dist += distance->compare(cv::Mat(1, a_template_bytes, CV_8UC1, a+2*sizeof(size_t)+i*a_template_bytes),
-                                      cv::Mat(1, b_template_bytes, CV_8UC1, b+2*sizeof(size_t)+i*b_template_bytes));
-    *similarity = a_templates * b_templates / dist;
-    return JANUS_SUCCESS;
-}
+    int comparisons = 0;
+    janus_flat_template a_template = a;
+    while (a_template < a + a_bytes) {
+        const size_t a_template_bytes = *reinterpret_cast<size_t*>(a_template);
+        a_template += sizeof(a_template_bytes);
-struct janus_incomplete_gallery_type
-{
-    QList< QPair<janus_template, janus_template_id> > templates;
-};
+        janus_flat_template b_template = b;
+        while (b_template < b + b_bytes) {
+                const size_t b_template_bytes = *reinterpret_cast<size_t*>(b_template);
+                b_template += sizeof(b_template_bytes);
-janus_error janus_initialize_gallery(janus_incomplete_gallery *incomplete_gallery)
-{
-    *incomplete_gallery = new janus_incomplete_gallery_type();
-    return JANUS_SUCCESS;
-}
+                *similarity += distance->compare(cv::Mat(1, a_template_bytes, CV_8UC1, a_template),
+                                                 cv::Mat(1, b_template_bytes, CV_8UC1, b_template));
+                comparisons++;
-janus_error janus_add_template(const janus_template template_, const size_t bytes, const janus_template_id template_id, janus_incomplete_gallery incomplete_gallery)
-{
-    (void) bytes;
-    incomplete_gallery->templates.append(QPair<janus_template, janus_template_id>(template_, template_id));
+                b_template += b_template_bytes;
+        }
+
+        a_template += a_template_bytes;
+    }
+
+    if (*similarity != *similarity) // True for NaN
+        return JANUS_UNKNOWN_ERROR;
+
+    *similarity /= comparisons;
     return JANUS_SUCCESS;
 }
-janus_error janus_finalize_gallery(janus_incomplete_gallery incomplete_gallery, const char *gallery_file)
+janus_error janus_enroll(const janus_template template_, const janus_template_id template_id, janus_gallery gallery)
 {
-    (void) incomplete_gallery;
-    (void) gallery_file;
+    template_->file.set("TEMPLATE_ID", template_id);
+    QFile file(gallery);
+    if (!file.open(QFile::WriteOnly | QFile::Append))
+        return JANUS_WRITE_ERROR;
+    QDataStream stream(&file);
+    stream << *template_;
+    file.close();
+    delete template_;
     return JANUS_SUCCESS;
 }
@@ -383,7 +383,9 @@ bool br_img_is_empty(br_template tmpl)
 const char* br_get_filename(br_template tmpl)
 {
     Template *t = reinterpret_cast<Template*>(tmpl);
-    return t->file.name.toStdString().c_str();
+		QByteArray s = t->file.name.toLocal8Bit();
+		char *buffer = s.data();
+		return buffer;
 }
 void br_set_filename(br_template tmpl, const char *filename)
@@ -1374,7 +1374,7 @@ BR_EXPORT void Cat(const QStringList &amp;inputGalleries, const QString &amp;outputGalle
 /*!
  * \brief Deduplicate a gallery.
- * \param inputGalleries Gallery to deduplicate.
+ * \param inputGallery Gallery to deduplicate.
  * \param outputGallery Gallery to store the deduplicated result.
  * \param threshold Match score threshold to determine duplicates.
  */
@@ -52,6 +52,7 @@ class AlgorithmsInitializer : public Initializer
         Globals->abbreviations.insert("AgeGenderDemo", "Stream(SaveMat(original)+Cvt(Gray)+Cascade(FrontalFace)+Expand+<FaceClassificationRegistration>+<FaceClassificationExtraction>+<AgeRegressor>/<GenderClassifier>+Discard+RestoreMat(original)+Draw(inPlace=true)+DrawPropertiesPoint([Age,Gender],Affine_0,inPlace=true)+SaveMat(original)+Discard+Contract+RestoreMat(original)+FPSCalc+Show(false,[AvgFPS,Age,Gender])+Discard)");
         Globals->abbreviations.insert("ShowOpticalFlowField", "Stream(SaveMat(original)+AggregateFrames(2)+OpticalFlow(useMagnitude=false)+Grid(100,100)+DrawOpticalFlow+FPSLimit(30)+Show(false)+Discard)");
         Globals->abbreviations.insert("ShowOpticalFlowMagnitude", "Stream(AggregateFrames(2)+OpticalFlow+Normalize(Range,false,0,255)+Cvt(Color)+Draw+FPSLimit(30)+Show(false)+Discard)");
+        Globals->abbreviations.insert("ShowMotionSegmentation", "Stream(DropFrames(5)+SaveMat(original)+AggregateFrames(2)+OpticalFlow+CvtUChar+Segmentation+DrawSegmentation+Draw+FPSLimit(30)+Show(false)+Discard)");
         Globals->abbreviations.insert("HOG", "Stream(DropFrames(5)+Cvt(Gray)+Grid(5,5)+ROIFromPts(32,24)+Expand+Resize(32,32)+Gradient+RectRegions+Bin(0,360,8)+Hist(8)+Cat)+Contract+CatRows+KMeans(500)+Hist(500)+SVM");
         Globals->abbreviations.insert("HOF", "Stream(DropFrames(5)+Grid(5,5)+AggregateFrames(2)+OpticalFlow+ROIFromPts(32,24)+Expand+Resize(32,32)+Gradient+RectRegions+Bin(0,360,8)+Hist(8)+Cat)+Contract+CatRows+KMeans(500)+Hist(500)");
@@ -78,7 +79,7 @@ class AlgorithmsInitializer : public Initializer
         Globals->abbreviations.insert("ColoredLBP", "Open+Affine(128,128,0.37,0.45)+Cvt(Gray)+Blur(1.1)+Gamma(0.2)+DoG(1,2)+ContrastEq(0.1,10)+LBP(1,2)+ColoredU2");
         // Transforms
-        Globals->abbreviations.insert("FaceDetection", "(Open+Cvt(Gray)+Cascade(FrontalFace))");
+        Globals->abbreviations.insert("FaceDetection", "Open+Cvt(Gray)+Cascade(FrontalFace)");
         Globals->abbreviations.insert("DenseLBP", "(Blur(1.1)+Gamma(0.2)+DoG(1,2)+ContrastEq(0.1,10)+LBP(1,2)+RectRegions(8,8,6,6)+Hist(59))");
         Globals->abbreviations.insert("DenseSIFT", "(Grid(10,10)+SIFTDescriptor(12)+ByRow)");
         Globals->abbreviations.insert("FaceRecognitionRegistration", "(ASEFEyes+Affine(88,88,0.25,0.35)+DownsampleTraining(FTE(DFFS),instances=1))");
@@ -374,6 +374,56 @@ class DrawOpticalFlow : public UntrainableTransform
 };
 BR_REGISTER(Transform, DrawOpticalFlow)
+/*!
+ * \ingroup transforms
+ * \brief Fill in the segmentations or draw a line between intersecting segments.
+ * \author Austin Blanton \cite imaus10
+ */
+class DrawSegmentation : public UntrainableTransform
+{
+    Q_OBJECT
+    Q_PROPERTY(bool fillSegment READ get_fillSegment WRITE set_fillSegment RESET reset_fillSegment STORED false)
+    BR_PROPERTY(bool, fillSegment, true)
+    Q_PROPERTY(QString original READ get_original WRITE set_original RESET reset_original STORED false)
+    BR_PROPERTY(QString, original, "original")
+
+    void project(const Template &src, Template &dst) const
+    {
+        if (!src.file.contains("SegmentsMask") || !src.file.contains("NumSegments")) qFatal("Must supply a Contours object in the metadata to drawContours.");
+        Mat segments = src.file.get<Mat>("SegmentsMask");
+        int numSegments = src.file.get<int>("NumSegments");
+
+        dst.file = src.file;
+        Mat drawn;
+        if (fillSegment) {
+            drawn = Mat(segments.size(), CV_8UC3, Scalar::all(0));
+        } else {
+            if (!dst.file.contains(original)) qFatal("You must store the original image in the metadata with SaveMat.");
+            drawn = dst.file.get<Mat>(original);
+            dst.file.remove(original);
+        }
+
+        for (int i=1; i<numSegments+1; i++) {
+            Mat mask = segments == i;
+            if (fillSegment) { // color the whole segment
+                // set to a random color - get ready for a craaaazy acid trip
+                int b = theRNG().uniform(0, 255);
+                int g = theRNG().uniform(0, 255);
+                int r = theRNG().uniform(0, 255);
+                drawn.setTo(Scalar(r,g,b), mask);
+            } else { // draw lines where there's a color change
+                vector<vector<Point> > contours;
+                Scalar color(0,255,0);
+                findContours(mask, contours, CV_RETR_LIST, CV_CHAIN_APPROX_NONE);
+                drawContours(drawn, contours, -1, color);
+            }
+        }
+
+        dst.m() = drawn;
+    }
+};
+BR_REGISTER(Transform, DrawSegmentation)
+
 // TODO: re-implement EditTransform using Qt
 #if 0
 /*!
+#include <opencv2/imgproc/imgproc.hpp>
+#include "openbr_internal.h"
+#include "openbr/core/opencvutils.h"
+
+using namespace cv;
+
+namespace br
+{
+
+/*!
+ * \ingroup transforms
+ * \brief Applies watershed segmentation.
+ * \author Austin Blanton \cite imaus10
+ */
+class WatershedSegmentationTransform : public UntrainableTransform
+{
+    Q_OBJECT
+    void project(const Template &src, Template &dst) const
+    {
+        dst = src;
+
+        Mat mod;
+//        adaptiveThreshold(src.m(), src.m(), 255, ADAPTIVE_THRESH_GAUSSIAN_C, THRESH_BINARY, 33, 5);
+        threshold(src.m(), mod, 0, 255, THRESH_BINARY+THRESH_OTSU);
+
+        // findContours requires an 8-bit 1-channel image
+        // and modifies its source image
+        if (mod.depth() != CV_8U) OpenCVUtils::cvtUChar(mod, mod);
+        if (mod.channels() != 1) OpenCVUtils::cvtGray(mod, mod);
+        vector<vector<Point> > contours;
+        vector<Vec4i> hierarchy;
+        findContours(mod, contours, hierarchy, CV_RETR_CCOMP, CV_CHAIN_APPROX_SIMPLE);
+
+        // draw the contour delineations as 1,2,3... for input to watershed
+        Mat markers = Mat::zeros(mod.size(), CV_32S);
+        int compCount=0;
+        for (int idx=0; idx>=0; idx=hierarchy[idx][0], compCount++) {
+            drawContours(markers, contours, idx, Scalar::all(compCount+1), -1, 8, hierarchy, INT_MAX);
+        }
+
+        Mat orig = src.m();
+        // watershed requires a 3-channel 8-bit image
+        if (orig.channels() == 1) cvtColor(orig, orig, CV_GRAY2BGR);
+        watershed(orig, markers);
+        dst.file.set("SegmentsMask", QVariant::fromValue(markers));
+        dst.file.set("NumSegments", compCount);
+    }
+};
+BR_REGISTER(Transform, WatershedSegmentationTransform)
+
+} // namespace br
+
+#include "segmentation.moc"
@@ -347,10 +347,11 @@ public:
         }
         // first 4 bytes store 0xEDFE, next 24 store 'Norpix seq  '
-        char *firstFour = new char[4], *nextTwentyFour;
+        char firstFour[4];
         seqFile.seekg(0, ios::beg);
         seqFile.read(firstFour, 4);
-        nextTwentyFour = readText(24);
+        char nextTwentyFour[24];
+        readText(24, nextTwentyFour);
         if (firstFour[0] != (char)0xED || firstFour[1] != (char)0xFE || strncmp(nextTwentyFour, "Norpix seq", 10) != 0) {
             qDebug("Invalid header in seq file");
             return false;
@@ -363,7 +364,8 @@ public:
             qDebug("Invalid header size");
             return false;
         }
-        char *desc = readText(512);
+        char desc[512];
+        readText(512, desc);
         basis.file.set("Description", QString(desc));
         width = readInt();
@@ -414,9 +416,9 @@ public:
                 // but there might be 16 extra bytes instead of 8...
                 if (i == 1) {
                     seqFile.seekg(s, ios::beg);
-                    char *zero = new char[1];
-                    seqFile.read(zero, 1);
-                    if (zero[0] == 0) {
+                    char zero;
+                    seqFile.read(&zero, 1);
+                    if (zero == 0) {
                         s += 8;
                         extra += 8;
                     }
@@ -498,14 +500,13 @@ private:
     // apparently the text in seq files is 16 bit characters (UTF-16?)
     // since we don't really need the last byte, snad since it gets interpreted as
     // a terminating char, let's just grab the first byte for storage
-    char* readText(int bytes)
+    void readText(int bytes, char * buffer)
     {
-        char *text = new char[bytes], *ret = new char[bytes/2];
-        seqFile.read(text, bytes);
+        seqFile.read(buffer, bytes);
         for (int i=0; i<bytes; i+=2) {
-            ret[i/2] = text[i];
+            buffer[i/2] = buffer[i];
         }
-        return ret;
+        buffer[bytes/2] = '\0';
     }
 protected: