Refactor PP5 comparison handling

Introduce PP5GalleryTransform, a transform that compares incoming templates against a fixed gallery (thereby avoiding the cost of repeatedly setting up pp5 galleries for the same templates). Modify AlgorithmCore to support supplying Transforms to the right of the algorithm string, in this case ! should be used instead of :.

Refactor PP5 comparison handling
Introduce PP5GalleryTransform, a transform that compares incoming templates against a fixed gallery (thereby avoiding the cost of repeatedly setting up pp5 galleries for the same templates). Modify AlgorithmCore to support supplying Transforms to the right of the algorithm string, in this case ! should be used instead of :.
Charles Otto
1 parent 7c9ace46
Showing 5 changed files with 167 additions and 77 deletions
openbr/core/core.cpp
openbr/openbr_plugin.cpp
openbr/plugins/gallery.cpp
openbr/plugins/misc.cpp
openbr/plugins/pp5.cpp
@@ -27,6 +27,7 @@ struct AlgorithmCore
 {
     QSharedPointer<Transform> transform;
     QSharedPointer<Distance> distance;
+    QString galleryCompareString;
     QString transformString;
     QString distanceString;
@@ -147,9 +148,6 @@ struct AlgorithmCore
         Gallery *temp = Gallery::make(input);
         qint64 total = temp->totalSize();
-        Globals->currentStep = 0;
-        Globals->totalSteps = total;
-
         QScopedPointer<Transform> basePipe;
         QString pipeDesc = "GalleryOutput("+gallery.flat()+")+ProgressCounter("+QString::number(total)+")+Discard";
@@ -189,6 +187,8 @@ struct AlgorithmCore
         wrapper->init();
         Globals->startTime.start();
+        Globals->currentStep = 0;
+        Globals->totalSteps = total;
         TemplateList data, output;
         data.append(input);
@@ -335,7 +335,7 @@ struct AlgorithmCore
                output.isNull() ? "" : qPrintable(" to " + output.flat()));
         // Escape hatch for distances that need to operate directly on the gallery files
-        if (distance->compare(targetGallery, queryGallery, output))
+        if (distance && distance->compare(targetGallery, queryGallery, output))
             return;
         // Are we comparing the same gallery against itself?
@@ -442,8 +442,11 @@ struct AlgorithmCore
         // The actual comparison step is done by a GalleryCompare transform, which has a Distance, and a gallery as data.
         // Incoming templates are compared against the templates in the gallery, and the output is the resulting score
         // vector.
-        QString compareRegionDesc = "Pipe([GalleryCompare("+Globals->algorithm + "," + colEnrolledGallery.flat() + ")])";
-
+        QString compareRegionDesc;
+        if (this->galleryCompareString.isEmpty() )
+            compareRegionDesc = "Pipe([GalleryCompare("+Globals->algorithm + "," + colEnrolledGallery.flat() + ")])";
+        else
+            compareRegionDesc = "Pipe(["+this->galleryCompareString+"("+Globals->algorithm + "," + colEnrolledGallery.flat() + ")])";
         QScopedPointer<Transform> compareRegion;
         // If we need to enroll the row set, we add the current algorithm's enrollment transform before the
@@ -519,6 +522,7 @@ struct AlgorithmCore
         // Set up progress counting variables
         Globals->currentStep = 0;
+        Globals->currentProgress = 0;
         Globals->totalSteps = rowSize;
         Globals->startTime.start();
@@ -553,19 +557,27 @@ private:
         if (Globals->abbreviations.contains(description))
             return init(Globals->abbreviations[description]);
-        //! [Parsing the algorithm description]
-        QStringList words = QtUtils::parse(description, ':');
+        const bool compareTransform = description.contains('!');
+        QStringList words = QtUtils::parse(description, compareTransform ? '!' : ':');
+
         if ((words.size() < 1) || (words.size() > 2)) qFatal("Invalid algorithm format.");
-        //! [Parsing the algorithm description]
+        //! [Parsing the algorithm description]
         transformString = words[0];
-
         //! [Creating the template generation and comparison methods]
         transform = QSharedPointer<Transform>(Transform::make(words[0], NULL));
         if (words.size() > 1) {
-            distance = QSharedPointer<Distance>(Distance::make(words[1], NULL));
-            distanceString = words[1];
+            if (!compareTransform) {
+                distance = QSharedPointer<Distance>(Distance::make(words[1], NULL));
+                distanceString = words[1];
+                galleryCompareString.clear();
+            }
+            else {
+                galleryCompareString = words[1];
+                distanceString.clear();
+            }
+            
         }
         //! [Creating the template generation and comparison methods]
     }
@@ -875,7 +875,7 @@ void br::Context::printStatus()
     const float p = progress();
     if (p < 1) {
         int s = timeRemaining();
-        fprintf(stderr,"%05.2f%%  ELAPSED=%s  REMAINING=%s  COUNT=%g  \r", p*100, QtUtils::toTime(Globals->startTime.elapsed()/1000.0f).toStdString().c_str(), QtUtils::toTime(s).toStdString().c_str(), Globals->currentStep);
+        fprintf(stderr,"\r%05.2f%%  ELAPSED=%s  REMAINING=%s  COUNT=%g", p*100, QtUtils::toTime(Globals->startTime.elapsed()/1000.0f).toStdString().c_str(), QtUtils::toTime(s).toStdString().c_str(), Globals->currentStep);
         fflush(stderr);
     }
 }
@@ -129,7 +129,7 @@ class BinaryGallery : public Gallery
             if (t.isEmpty() && t.file.isNull())
                 continue;
             templates.append(t);
-            templates.last().file.set("progress", totalSize());
+            templates.last().file.set("progress", position());
             // Special case for pipes where we want to process data as soon as it is available
             if (gallery.isSequential())
@@ -625,11 +625,18 @@ class ProgressCounterTransform : public TimeVaryingTransform
         dst = src;
         qint64 elapsed = timer.elapsed();
-
+        int last_frame = -2;
         if (!dst.empty()) {
-            Globals->currentProgress = dst.last().file.get<qint64>("progress",0);
-            dst.last().file.remove("progress");
-            Globals->currentStep++;
+            for (int i=0;i < dst.size();i++) {
+                int frame = dst[i].file.get<int>("FrameNumber", -1);
+                if (frame == last_frame)
+                    continue;
+
+                Globals->currentProgress = dst[i].file.get<qint64>("progress",0);
+                dst[i].file.remove("progress");
+                Globals->currentStep++;
+                last_frame = frame;
+            }
         }
         // updated every second
@@ -650,13 +657,17 @@ class ProgressCounterTransform : public TimeVaryingTransform
     {
         (void) data;
         float p = br_progress();
-        qDebug("%05.2f%%  ELAPSED=%s  REMAINING=%s  COUNT=%g  \r", p*100, QtUtils::toTime(Globals->startTime.elapsed()/1000.0f).toStdString().c_str(), QtUtils::toTime(0).toStdString().c_str(), Globals->currentStep);
+        qDebug("\r%05.2f%%  ELAPSED=%s  REMAINING=%s  COUNT=%g", p*100, QtUtils::toTime(Globals->startTime.elapsed()/1000.0f).toStdString().c_str(), QtUtils::toTime(0).toStdString().c_str(), Globals->currentStep);
+        Globals->currentStep = 0;
+        Globals->currentProgress = 0;
+        Globals->totalSteps = 0;
     }
     void init()
     {
         timer.start();
         Globals->currentStep = 0;
+        Globals->currentProgress = 0;
     }
 public:
@@ -713,26 +724,26 @@ class OutputTransform : public TimeVaryingTransform
                 currentCol++;
                 currentRow = 0;
             }
-        }
-        bool blockDone = false;
-        // In direct mode, we don't buffer rows
-        if (!transposeMode) {
-            currentBlockRow++;
-            blockDone = true;
-        }
-        // in transpose mode, we buffer 100 cols before writing the block
-        else if (currentCol == bufferedSize) {
-            currentBlockCol++;
-            blockDone = true;
-        }
-        else return;
+            bool blockDone = false;
+            // In direct mode, we don't buffer rows
+            if (!transposeMode) {
+                currentBlockRow++;
+                blockDone = true;
+            }
+            // in transpose mode, we buffer 100 cols before writing the block
+            else if (currentCol == bufferedSize) {
+                currentBlockCol++;
+                blockDone = true;
+            }
+            else return;
-        if (blockDone) {
-            // set the next block, only necessary if we haven't buffered the current item
-            output->setBlock(currentBlockRow, currentBlockCol);
-            currentRow = 0;
-            currentCol = 0;
+            if (blockDone) {
+                // set the next block, only necessary if we haven't buffered the current item
+                output->setBlock(currentBlockRow, currentBlockCol);
+                currentRow = 0;
+                currentCol = 0;
+            }
         }
     }
@@ -45,7 +45,7 @@ class PP5Initializer : public Initializer
     void initialize() const
     {
         TRY(ppr_initialize_sdk(qPrintable(Globals->sdkPath + "/share/openbr/models/pp5/"), my_license_id, my_license_key))
-        Globals->abbreviations.insert("PP5","Open+Expand+PP5Enroll:PP5Compare");
+        Globals->abbreviations.insert("PP5","Open+Expand+PP5Enroll!PP5Gallery");
         Globals->abbreviations.insert("PP5Register", "Open+PP5Enroll(true)+RenameFirst([eyeL,PP5_Landmark0_Right_Eye],Affine_0)+RenameFirst([eyeR,PP5_Landmark1_Left_Eye],Affine_1)");
         Globals->abbreviations.insert("PP5CropFace", "Open+PP5Enroll(true)+RenameFirst([eyeL,PP5_Landmark0_Right_Eye],Affine_0)+RenameFirst([eyeR,PP5_Landmark1_Left_Eye],Affine_1)+Affine(128,128,0.25,0.35)+Cvt(Gray)");
     }
@@ -85,7 +85,7 @@ struct PP5Context
         default_settings.recognition.automatically_extract_templates = 1;
         default_settings.recognition.enable_comparison = 1;
         default_settings.recognition.enable_extraction = 1;
-        default_settings.recognition.num_comparison_threads = QThreadPool::globalInstance()->maxThreadCount();
+        default_settings.recognition.num_comparison_threads = 1;
         default_settings.recognition.recognizer = PPR_RECOGNIZER_MULTI_POSE;
         TRY(ppr_initialize_context(default_settings, &context))
     }
@@ -205,6 +205,45 @@ struct PP5Context
         return metadata;
     }
+
+    void compareNative(ppr_gallery_type target, const QList<int> &targetIDs, ppr_gallery_type query, const QList<int> &queryIDs, Output *output) const
+    {
+        ppr_similarity_matrix_type simmat;
+        TRY(ppr_compare_galleries(context, query, target, &simmat))
+        for (int i=0; i<queryIDs.size(); i++) {
+            int query_subject_id = queryIDs[i];
+            for (int j=0; j<targetIDs.size(); j++) {
+                int target_subject_id = targetIDs[j];
+                float score = -std::numeric_limits<float>::max();
+                if ((query_subject_id != -1) && (target_subject_id != -1)) {
+                    TRY(ppr_get_subject_similarity_score(context, simmat, query_subject_id, target_subject_id, &score))
+                }
+                output->setRelative(score, i, j);
+            }
+        }
+        ppr_free_similarity_matrix(simmat);
+    }
+
+    void enroll(const TemplateList &templates, ppr_gallery_type *gallery, QList<int> &subject_ids) const
+    {
+        int subject_id = 0, face_id = 0;
+        foreach (const Template &src, templates) {
+            if (!src.empty() && src.m().data) {
+                foreach (const cv::Mat &m, src) {
+                    ppr_face_type face;
+                    createFace(m, &face);
+                    TRY(ppr_add_face(context, gallery, face, subject_id, face_id))
+                    face_id++;
+                    ppr_free_face(face);
+                }
+                subject_ids.append(subject_id);
+                subject_id++;
+            } else {
+                subject_ids.append(-1);
+            }
+        }
+    }
+
 };
 /*!
@@ -294,6 +333,7 @@ class PP5EnrollTransform : public UntrainableMetaTransform
 BR_REGISTER(Transform, PP5EnrollTransform)
+
 /*!
  * \ingroup distances
  * \brief Compare templates with PP5
@@ -351,44 +391,6 @@ class PP5CompareDistance : public Distance
         ppr_free_gallery(query_gallery);
     }
-    void compareNative(ppr_gallery_type target, const QList<int> &targetIDs, ppr_gallery_type query, const QList<int> &queryIDs, Output *output) const
-    {
-        ppr_similarity_matrix_type simmat;
-        TRY(ppr_compare_galleries(context, query, target, &simmat))
-        for (int i=0; i<queryIDs.size(); i++) {
-            int query_subject_id = queryIDs[i];
-            for (int j=0; j<targetIDs.size(); j++) {
-                int target_subject_id = targetIDs[j];
-                float score = -std::numeric_limits<float>::max();
-                if ((query_subject_id != -1) && (target_subject_id != -1)) {
-                    TRY(ppr_get_subject_similarity_score(context, simmat, query_subject_id, target_subject_id, &score))
-                }
-                output->setRelative(score, i, j);
-            }
-        }
-        ppr_free_similarity_matrix(simmat);
-    }
-
-    void enroll(const TemplateList &templates, ppr_gallery_type *gallery, QList<int> &subject_ids) const
-    {
-        int subject_id = 0, face_id = 0;
-        foreach (const Template &src, templates) {
-            if (!src.empty() && src.m().data) {
-                foreach (const cv::Mat &m, src) {
-                    ppr_face_type face;
-                    createFace(m, &face);
-                    TRY(ppr_add_face(context, gallery, face, subject_id, face_id))
-                    face_id++;
-                    ppr_free_face(face);
-                }
-                subject_ids.append(subject_id);
-                subject_id++;
-            } else {
-                subject_ids.append(-1);
-            }
-        }
-    }
-
     NativeGallery cacheRetain(const File &gallery) const
     {
         QMutexLocker locker(&cacheLock);
@@ -439,4 +441,69 @@ class PP5CompareDistance : public Distance
 BR_REGISTER(Distance, PP5CompareDistance)
+class PP5GalleryTransform: public UntrainableMetaTransform
+                         , public PP5Context
+{
+    Q_OBJECT
+    Q_PROPERTY(QString junk READ get_junk WRITE set_junk RESET reset_junk STORED false)
+    Q_PROPERTY(QString galleryName READ get_galleryName WRITE set_galleryName RESET reset_galleryName STORED false)
+    BR_PROPERTY(QString, junk, "")
+    BR_PROPERTY(QString, galleryName, "")
+
+    ppr_gallery_type target;
+    QList<int> targetIDs;
+
+    void project(const Template & src, Template & dst) const
+    {
+        TemplateList temp, output;
+        temp.append(src);
+        project(temp, output);
+        if (!output.empty())
+           dst = output[0];
+    }
+
+    void project(const TemplateList & src, TemplateList & dst) const
+    {
+        dst.clear();
+        QList<int> queryIDs;
+
+        ppr_gallery_type query;
+        ppr_create_gallery(context, &query);
+        enroll(src,&query, queryIDs);
+        
+        ppr_similarity_matrix_type simmat;
+        
+        TRY(ppr_compare_galleries(context, query, target, &simmat))
+
+        for (int i=0; i<queryIDs.size(); i++) {
+            dst.append(Template());
+            dst[i].file = src[i].file;
+            dst[i].m() = cv::Mat(1,targetIDs.size(), CV_32FC1);
+
+            int query_subject_id = queryIDs[i];
+            for (int j=0; j<targetIDs.size(); j++) {
+                int target_subject_id = targetIDs[j];
+                float score = -std::numeric_limits<float>::max();
+                if ((query_subject_id != -1) && (target_subject_id != -1)) {
+                    TRY(ppr_get_subject_similarity_score(context, simmat, query_subject_id, target_subject_id, &score))
+                }
+                dst[i].m().at<float>(0,j) = score;
+            }
+        }
+
+        ppr_free_similarity_matrix(simmat);
+        ppr_free_gallery(query);
+    }
+
+    void init()
+    {
+        // set up the gallery
+        ppr_create_gallery(context, &target);
+        TemplateList templates = TemplateList::fromGallery(galleryName);
+        enroll(templates,&target, targetIDs);
+    }
+};
+
+BR_REGISTER(Transform, PP5GalleryTransform)
+
 #include "plugins/pp5.moc"