Merge pull request #213 from biometrics/inline_pp5

Refactor PP5 comparison handling

Merge pull request #213 from biometrics/inline_pp5
Refactor PP5 comparison handling
caotto
2 parents 7c9ace46 a438d3fa
Showing 5 changed files with 167 additions and 77 deletions
openbr/core/core.cpp
openbr/openbr_plugin.cpp
openbr/plugins/gallery.cpp
openbr/plugins/misc.cpp
openbr/plugins/pp5.cpp
@@ -27,6 +27,7 @@ struct AlgorithmCore
 {
     QSharedPointer<Transform> transform;
     QSharedPointer<Distance> distance;
+    QString galleryCompareString;
     QString transformString;
     QString distanceString;
@@ -147,9 +148,6 @@ struct AlgorithmCore
         Gallery *temp = Gallery::make(input);
         qint64 total = temp->totalSize();
-        Globals->currentStep = 0;
-        Globals->totalSteps = total;
-
         QScopedPointer<Transform> basePipe;
         QString pipeDesc = "GalleryOutput("+gallery.flat()+")+ProgressCounter("+QString::number(total)+")+Discard";
@@ -189,6 +187,8 @@ struct AlgorithmCore
         wrapper->init();
         Globals->startTime.start();
+        Globals->currentStep = 0;
+        Globals->totalSteps = total;
         TemplateList data, output;
         data.append(input);
@@ -335,7 +335,7 @@ struct AlgorithmCore
                output.isNull() ? "" : qPrintable(" to " + output.flat()));
         // Escape hatch for distances that need to operate directly on the gallery files
-        if (distance->compare(targetGallery, queryGallery, output))
+        if (distance && distance->compare(targetGallery, queryGallery, output))
             return;
         // Are we comparing the same gallery against itself?
@@ -442,8 +442,11 @@ struct AlgorithmCore
         // The actual comparison step is done by a GalleryCompare transform, which has a Distance, and a gallery as data.
         // Incoming templates are compared against the templates in the gallery, and the output is the resulting score
         // vector.
-        QString compareRegionDesc = "Pipe([GalleryCompare("+Globals->algorithm + "," + colEnrolledGallery.flat() + ")])";
-
+        QString compareRegionDesc;
+        if (this->galleryCompareString.isEmpty() )
+            compareRegionDesc = "Pipe([GalleryCompare("+Globals->algorithm + "," + colEnrolledGallery.flat() + ")])";
+        else
+            compareRegionDesc = "Pipe(["+this->galleryCompareString+"("+Globals->algorithm + "," + colEnrolledGallery.flat() + ")])";
         QScopedPointer<Transform> compareRegion;
         // If we need to enroll the row set, we add the current algorithm's enrollment transform before the
@@ -519,6 +522,7 @@ struct AlgorithmCore
         // Set up progress counting variables
         Globals->currentStep = 0;
+        Globals->currentProgress = 0;
         Globals->totalSteps = rowSize;
         Globals->startTime.start();
@@ -553,19 +557,27 @@ private:
         if (Globals->abbreviations.contains(description))
             return init(Globals->abbreviations[description]);
-        //! [Parsing the algorithm description]
-        QStringList words = QtUtils::parse(description, ':');
+        const bool compareTransform = description.contains('!');
+        QStringList words = QtUtils::parse(description, compareTransform ? '!' : ':');
+
         if ((words.size() < 1) || (words.size() > 2)) qFatal("Invalid algorithm format.");
-        //! [Parsing the algorithm description]
+        //! [Parsing the algorithm description]
         transformString = words[0];
-
         //! [Creating the template generation and comparison methods]
         transform = QSharedPointer<Transform>(Transform::make(words[0], NULL));
         if (words.size() > 1) {
-            distance = QSharedPointer<Distance>(Distance::make(words[1], NULL));
-            distanceString = words[1];
+            if (!compareTransform) {
+                distance = QSharedPointer<Distance>(Distance::make(words[1], NULL));
+                distanceString = words[1];
+                galleryCompareString.clear();
+            }
+            else {
+                galleryCompareString = words[1];
+                distanceString.clear();
+            }
+            
         }
         //! [Creating the template generation and comparison methods]
     }
@@ -875,7 +875,7 @@ void br::Context::printStatus()
     const float p = progress();
     if (p < 1) {
         int s = timeRemaining();
-        fprintf(stderr,"%05.2f%%  ELAPSED=%s  REMAINING=%s  COUNT=%g  \r", p*100, QtUtils::toTime(Globals->startTime.elapsed()/1000.0f).toStdString().c_str(), QtUtils::toTime(s).toStdString().c_str(), Globals->currentStep);
+        fprintf(stderr,"\r%05.2f%%  ELAPSED=%s  REMAINING=%s  COUNT=%g", p*100, QtUtils::toTime(Globals->startTime.elapsed()/1000.0f).toStdString().c_str(), QtUtils::toTime(s).toStdString().c_str(), Globals->currentStep);
         fflush(stderr);
     }
 }
@@ -129,7 +129,7 @@ class BinaryGallery : public Gallery
             if (t.isEmpty() && t.file.isNull())
                 continue;
             templates.append(t);
-            templates.last().file.set("progress", totalSize());
+            templates.last().file.set("progress", position());
             // Special case for pipes where we want to process data as soon as it is available
             if (gallery.isSequential())
@@ -625,11 +625,18 @@ class ProgressCounterTransform : public TimeVaryingTransform
         dst = src;
         qint64 elapsed = timer.elapsed();
-
+        int last_frame = -2;
         if (!dst.empty()) {
-            Globals->currentProgress = dst.last().file.get<qint64>("progress",0);
-            dst.last().file.remove("progress");
-            Globals->currentStep++;
+            for (int i=0;i < dst.size();i++) {
+                int frame = dst[i].file.get<int>("FrameNumber", -1);
+                if (frame == last_frame)
+                    continue;
+
+                Globals->currentProgress = dst[i].file.get<qint64>("progress",0);
+                dst[i].file.remove("progress");
+                Globals->currentStep++;
+                last_frame = frame;
+            }
         }
         // updated every second
@@ -650,13 +657,17 @@ class ProgressCounterTransform : public TimeVaryingTransform
     {
         (void) data;
         float p = br_progress();
-        qDebug("%05.2f%%  ELAPSED=%s  REMAINING=%s  COUNT=%g  \r", p*100, QtUtils::toTime(Globals->startTime.elapsed()/1000.0f).toStdString().c_str(), QtUtils::toTime(0).toStdString().c_str(), Globals->currentStep);
+        qDebug("\r%05.2f%%  ELAPSED=%s  REMAINING=%s  COUNT=%g", p*100, QtUtils::toTime(Globals->startTime.elapsed()/1000.0f).toStdString().c_str(), QtUtils::toTime(0).toStdString().c_str(), Globals->currentStep);
+        Globals->currentStep = 0;
+        Globals->currentProgress = 0;
+        Globals->totalSteps = 0;
     }
     void init()
     {
         timer.start();
         Globals->currentStep = 0;
+        Globals->currentProgress = 0;
     }
 public:
@@ -713,26 +724,26 @@ class OutputTransform : public TimeVaryingTransform
                 currentCol++;
                 currentRow = 0;
             }
-        }
-        bool blockDone = false;
-        // In direct mode, we don't buffer rows
-        if (!transposeMode) {
-            currentBlockRow++;
-            blockDone = true;
-        }
-        // in transpose mode, we buffer 100 cols before writing the block
-        else if (currentCol == bufferedSize) {
-            currentBlockCol++;
-            blockDone = true;
-        }
-        else return;
+            bool blockDone = false;
+            // In direct mode, we don't buffer rows
+            if (!transposeMode) {
+                currentBlockRow++;
+                blockDone = true;
+            }
+            // in transpose mode, we buffer 100 cols before writing the block
+            else if (currentCol == bufferedSize) {
+                currentBlockCol++;
+                blockDone = true;
+            }
+            else return;
-        if (blockDone) {
-            // set the next block, only necessary if we haven't buffered the current item
-            output->setBlock(currentBlockRow, currentBlockCol);
-            currentRow = 0;
-            currentCol = 0;
+            if (blockDone) {
+                // set the next block, only necessary if we haven't buffered the current item
+                output->setBlock(currentBlockRow, currentBlockCol);
+                currentRow = 0;
+                currentCol = 0;
+            }
         }
     }
@@ -45,7 +45,7 @@ class PP5Initializer : public Initializer
     void initialize() const
     {
         TRY(ppr_initialize_sdk(qPrintable(Globals->sdkPath + "/share/openbr/models/pp5/"), my_license_id, my_license_key))
-        Globals->abbreviations.insert("PP5","Open+Expand+PP5Enroll:PP5Compare");
+        Globals->abbreviations.insert("PP5","Open+Expand+PP5Enroll!PP5Gallery");
         Globals->abbreviations.insert("PP5Register", "Open+PP5Enroll(true)+RenameFirst([eyeL,PP5_Landmark0_Right_Eye],Affine_0)+RenameFirst([eyeR,PP5_Landmark1_Left_Eye],Affine_1)");
         Globals->abbreviations.insert("PP5CropFace", "Open+PP5Enroll(true)+RenameFirst([eyeL,PP5_Landmark0_Right_Eye],Affine_0)+RenameFirst([eyeR,PP5_Landmark1_Left_Eye],Affine_1)+Affine(128,128,0.25,0.35)+Cvt(Gray)");
     }
@@ -85,7 +85,7 @@ struct PP5Context
         default_settings.recognition.automatically_extract_templates = 1;
         default_settings.recognition.enable_comparison = 1;
         default_settings.recognition.enable_extraction = 1;
-        default_settings.recognition.num_comparison_threads = QThreadPool::globalInstance()->maxThreadCount();
+        default_settings.recognition.num_comparison_threads = 1;
         default_settings.recognition.recognizer = PPR_RECOGNIZER_MULTI_POSE;
         TRY(ppr_initialize_context(default_settings, &context))
     }
@@ -205,6 +205,45 @@ struct PP5Context
         return metadata;
     }
+
+    void compareNative(ppr_gallery_type target, const QList<int> &targetIDs, ppr_gallery_type query, const QList<int> &queryIDs, Output *output) const
+    {
+        ppr_similarity_matrix_type simmat;
+        TRY(ppr_compare_galleries(context, query, target, &simmat))
+        for (int i=0; i<queryIDs.size(); i++) {
+            int query_subject_id = queryIDs[i];
+            for (int j=0; j<targetIDs.size(); j++) {
+                int target_subject_id = targetIDs[j];
+                float score = -std::numeric_limits<float>::max();
+                if ((query_subject_id != -1) && (target_subject_id != -1)) {
+                    TRY(ppr_get_subject_similarity_score(context, simmat, query_subject_id, target_subject_id, &score))
+                }
+                output->setRelative(score, i, j);
+            }
+        }
+        ppr_free_similarity_matrix(simmat);
+    }
+
+    void enroll(const TemplateList &templates, ppr_gallery_type *gallery, QList<int> &subject_ids) const
+    {
+        int subject_id = 0, face_id = 0;
+        foreach (const Template &src, templates) {
+            if (!src.empty() && src.m().data) {
+                foreach (const cv::Mat &m, src) {
+                    ppr_face_type face;
+                    createFace(m, &face);
+                    TRY(ppr_add_face(context, gallery, face, subject_id, face_id))
+                    face_id++;
+                    ppr_free_face(face);
+                }
+                subject_ids.append(subject_id);
+                subject_id++;
+            } else {
+                subject_ids.append(-1);
+            }
+        }
+    }
+
 };
 /*!
@@ -294,6 +333,7 @@ class PP5EnrollTransform : public UntrainableMetaTransform
 BR_REGISTER(Transform, PP5EnrollTransform)
+
 /*!
  * \ingroup distances
  * \brief Compare templates with PP5
@@ -351,44 +391,6 @@ class PP5CompareDistance : public Distance
         ppr_free_gallery(query_gallery);
     }
-    void compareNative(ppr_gallery_type target, const QList<int> &targetIDs, ppr_gallery_type query, const QList<int> &queryIDs, Output *output) const
-    {
-        ppr_similarity_matrix_type simmat;
-        TRY(ppr_compare_galleries(context, query, target, &simmat))
-        for (int i=0; i<queryIDs.size(); i++) {
-            int query_subject_id = queryIDs[i];
-            for (int j=0; j<targetIDs.size(); j++) {
-                int target_subject_id = targetIDs[j];
-                float score = -std::numeric_limits<float>::max();
-                if ((query_subject_id != -1) && (target_subject_id != -1)) {
-                    TRY(ppr_get_subject_similarity_score(context, simmat, query_subject_id, target_subject_id, &score))
-                }
-                output->setRelative(score, i, j);
-            }
-        }
-        ppr_free_similarity_matrix(simmat);
-    }
-
-    void enroll(const TemplateList &templates, ppr_gallery_type *gallery, QList<int> &subject_ids) const
-    {
-        int subject_id = 0, face_id = 0;
-        foreach (const Template &src, templates) {
-            if (!src.empty() && src.m().data) {
-                foreach (const cv::Mat &m, src) {
-                    ppr_face_type face;
-                    createFace(m, &face);
-                    TRY(ppr_add_face(context, gallery, face, subject_id, face_id))
-                    face_id++;
-                    ppr_free_face(face);
-                }
-                subject_ids.append(subject_id);
-                subject_id++;
-            } else {
-                subject_ids.append(-1);
-            }
-        }
-    }
-
     NativeGallery cacheRetain(const File &gallery) const
     {
         QMutexLocker locker(&cacheLock);
@@ -439,4 +441,69 @@ class PP5CompareDistance : public Distance
 BR_REGISTER(Distance, PP5CompareDistance)
+class PP5GalleryTransform: public UntrainableMetaTransform
+                         , public PP5Context
+{
+    Q_OBJECT
+    Q_PROPERTY(QString junk READ get_junk WRITE set_junk RESET reset_junk STORED false)
+    Q_PROPERTY(QString galleryName READ get_galleryName WRITE set_galleryName RESET reset_galleryName STORED false)
+    BR_PROPERTY(QString, junk, "")
+    BR_PROPERTY(QString, galleryName, "")
+
+    ppr_gallery_type target;
+    QList<int> targetIDs;
+
+    void project(const Template & src, Template & dst) const
+    {
+        TemplateList temp, output;
+        temp.append(src);
+        project(temp, output);
+        if (!output.empty())
+           dst = output[0];
+    }
+
+    void project(const TemplateList & src, TemplateList & dst) const
+    {
+        dst.clear();
+        QList<int> queryIDs;
+
+        ppr_gallery_type query;
+        ppr_create_gallery(context, &query);
+        enroll(src,&query, queryIDs);
+        
+        ppr_similarity_matrix_type simmat;
+        
+        TRY(ppr_compare_galleries(context, query, target, &simmat))
+
+        for (int i=0; i<queryIDs.size(); i++) {
+            dst.append(Template());
+            dst[i].file = src[i].file;
+            dst[i].m() = cv::Mat(1,targetIDs.size(), CV_32FC1);
+
+            int query_subject_id = queryIDs[i];
+            for (int j=0; j<targetIDs.size(); j++) {
+                int target_subject_id = targetIDs[j];
+                float score = -std::numeric_limits<float>::max();
+                if ((query_subject_id != -1) && (target_subject_id != -1)) {
+                    TRY(ppr_get_subject_similarity_score(context, simmat, query_subject_id, target_subject_id, &score))
+                }
+                dst[i].m().at<float>(0,j) = score;
+            }
+        }
+
+        ppr_free_similarity_matrix(simmat);
+        ppr_free_gallery(query);
+    }
+
+    void init()
+    {
+        // set up the gallery
+        ppr_create_gallery(context, &target);
+        TemplateList templates = TemplateList::fromGallery(galleryName);
+        enroll(templates,&target, targetIDs);
+    }
+};
+
+BR_REGISTER(Transform, PP5GalleryTransform)
+
 #include "plugins/pp5.moc"