Merge pull request #191 from biometrics/streaming_gallery

Preliminary work on not loading complete galleries during enrollment

Merge pull request #191 from biometrics/streaming_gallery
Preliminary work on not loading complete galleries during enrollment
Josh Klontz
2 parents 6dc7d8cd e2a5bc7e
Showing 7 changed files with 292 additions and 90 deletions
openbr/core/core.cpp
openbr/openbr_plugin.cpp
openbr/openbr_plugin.h
openbr/plugins/gallery.cpp
openbr/plugins/misc.cpp
openbr/plugins/openbr_internal.h
openbr/plugins/process.cpp
@@ -134,55 +134,51 @@ struct AlgorithmCore
             if (input.name.isEmpty()) return FileList();
             else                      gallery = getMemoryGallery(input);
         }
-        TemplateList data(TemplateList::fromGallery(input));
         bool multiProcess = Globals->file.getBool("multiProcess", false);
+        bool fileExclusion = false;
-        if (gallery.contains("append"))
-        {
-            // Remove any templates which are already in the gallery
-            QScopedPointer<Gallery> g(Gallery::make(gallery));
-            files = g->files();
-            QSet<QString> nameSet = QSet<QString>::fromList(files.names());
-            for (int i = data.size() - 1; i>=0; i--) {
-                if (nameSet.contains(data[i].file.name))
-                {
-                    data.removeAt(i);
-                }
-            }
+        // In append mode, we will exclude any templates with filenames already present in the output gallery
+        if (gallery.contains("append") && gallery.exists() ) {
+            FileList::fromGallery(gallery,true);
+            fileExclusion = true;
         }
-        if (data.empty())
-            return files;
+        Gallery * temp = Gallery::make(input);
+        qint64 total = temp->totalSize();
-        // Store steps for ProgressCounter
         Globals->currentStep = 0;
-        Globals->totalSteps = data.length();
+        Globals->totalSteps = total;
         QScopedPointer<Transform> basePipe;
-        if (!multiProcess)
-        {
-            QString pipeDesc = "GalleryOutput("+gallery.flat()+")+ProgressCounter("+QString::number(data.length())+")+Discard";
+        QString pipeDesc = "GalleryOutput("+gallery.flat()+")+ProgressCounter("+QString::number(total)+")+Discard";
+
+        if (!multiProcess) {
             basePipe.reset(Transform::make(pipeDesc,NULL));
             CompositeTransform * downcast = dynamic_cast<CompositeTransform *>(basePipe.data());
-            if (downcast == NULL)
-                qFatal("downcast failed?");
-            // replace that placeholder with the current algorithm
+            if (downcast == NULL) qFatal("downcast failed?");
+
             downcast->transforms.prepend(this->transform.data());
+            if (fileExclusion) {
+                Transform * temp = Transform::make("FileExclusion(" + gallery.flat() + ")", downcast);
+                downcast->transforms.prepend(temp);
+            }
             // call init on the pipe to collapse the algorithm (if its top level is a pipe)
             downcast->init();
         }
-        else
-        {
-            QString pipeDesc = "ProcessWrapper("+transformString+")"+"+GalleryOutput("+gallery.flat()+")+ProgressCounter("+QString::number(data.length())+")+Discard";
+        else {
+            pipeDesc = "ProcessWrapper("+transformString+")"+pipeDesc;
+            if (fileExclusion)
+                pipeDesc = "FileExclusion(" + gallery.flat() +")" + pipeDesc;
+
             basePipe.reset(Transform::make(pipeDesc,NULL));
         }
         // Next, we make a Stream (with placeholder transform)
-        QString streamDesc = "Stream(readMode=DistributeFrames)";
+        QString streamDesc = "Stream(readMode=StreamGallery)";
         QScopedPointer<Transform> baseStream(Transform::make(streamDesc, NULL));
         WrapperTransform * wrapper = dynamic_cast<WrapperTransform *> (baseStream.data());
@@ -194,9 +190,11 @@ struct AlgorithmCore
         Globals->startTime.start();
-        wrapper->projectUpdate(data,data);
+        TemplateList data, output;
+        data.append(input);
+        wrapper->projectUpdate(data, output);
-        files.append(data.files());
+        files.append(output.files());
         return files;
     }
@@ -337,13 +335,9 @@ struct AlgorithmCore
         // comparison against the smaller gallery (which will be enrolled, and stored in memory).
         bool needEnrollRows = false;
-
-
-
         if (output.exists() && output.get<bool>("cache", false)) return;
         if (queryGallery == ".") queryGallery = targetGallery;
-
         // To decide which gallery is larger, we need to read both, but at this point we just want the
         // metadata, and don't need the enrolled matrices.
         FileList targetMetadata;
@@ -359,15 +353,21 @@ struct AlgorithmCore
         File rowGallery = queryGallery;
         File colGallery = targetGallery;
-        int rowSize = queryMetadata.size();
+        qint64 rowSize;
+        Gallery * temp;
         if (transposeMode)
         {
             rowGallery = targetGallery;
             colGallery = queryGallery;
-            rowSize = targetMetadata.size();
+            temp = Gallery::make(targetGallery);
         }
-
+        else
+        {
+            temp = Gallery::make(queryGallery);
+        }
+        rowSize = temp->totalSize();
+        delete temp;
         // Is the column gallery already enrolled? We keep the enrolled column gallery in memory, and in multi-process
         // mode, every worker process retains a copy of this gallery in memory. When not in multi-process mode, we can
@@ -423,8 +423,6 @@ struct AlgorithmCore
         // progress counting step.
         // After the base algorithm is built, the whole thing will be run in a stream, so that I/O can be handled sequentially.
-
-
         // The actual comparison step is done by a GalleryCompare transform, which has a Distance, and a gallery as data.
         // Incoming templates are compared against the templates in the gallery, and the output is the resulting score
         // vector.
@@ -865,14 +865,14 @@ void br::Context::printStatus()
     const float p = progress();
     if (p < 1) {
         int s = timeRemaining();
-        fprintf(stderr, "%05.2f%%  ELAPSED=%s  REMAINING=%s  COUNT=%g/%g  \r", p*100, QtUtils::toTime(Globals->startTime.elapsed()/1000.0f).toStdString().c_str(), QtUtils::toTime(s).toStdString().c_str(), Globals->currentStep, Globals->totalSteps);
+        fprintf(stderr,"%05.2f%%  ELAPSED=%s  REMAINING=%s  COUNT=%g  \r", p*100, QtUtils::toTime(Globals->startTime.elapsed()/1000.0f).toStdString().c_str(), QtUtils::toTime(s).toStdString().c_str(), Globals->currentStep);
     }
 }
 float br::Context::progress() const
 {
     if (totalSteps == 0) return -1;
-    return currentStep / totalSteps;
+    return currentProgress / totalSteps;
 }
 void br::Context::setProperty(const QString &key, const QString &value)
@@ -687,6 +687,10 @@ public:
     Q_PROPERTY(double currentStep READ get_currentStep WRITE set_currentStep RESET reset_currentStep)
     BR_PROPERTY(double, currentStep, 0)
+    Q_PROPERTY(double currentProgress READ get_currentProgress WRITE set_currentProgress RESET reset_currentProgress)
+    BR_PROPERTY(double, currentProgress, 0)
+
+
     /*!
      * \brief Used internally to compute progress() and timeRemaining().
      */
@@ -1095,6 +1099,9 @@ public:
     static Gallery *make(const File &file); /*!< \brief Make a gallery to/from a file on disk. */
     void init();
+    virtual qint64 totalSize() { return std::numeric_limits<qint64>::max(); }
+    virtual qint64 position() { return 0; }
+
 private:
     QSharedPointer<Gallery> next;
 };
@@ -23,6 +23,7 @@
 #include <QSqlError>
 #include <QSqlQuery>
 #include <QSqlRecord>
+#include <QXmlStreamReader>
 #endif // BR_EMBEDDED
 #include <opencv2/highgui/highgui.hpp>
 #include "openbr_internal.h"
@@ -127,6 +128,7 @@ class galGallery : public Gallery
             Template m;
             stream >> m;
             templates.append(m);
+            templates.last().file.set("progress", totalSize());
         }
         *done = stream.atEnd();
@@ -140,6 +142,17 @@ class galGallery : public Gallery
         stream << t;
     }
+
+    qint64 totalSize()
+    {
+        return gallery.size();
+    }
+
+    qint64 position()
+    {
+        return gallery.pos();
+    }
+
 };
 BR_REGISTER(Gallery, galGallery)
@@ -328,6 +341,7 @@ class memGallery : public Gallery
 {
     Q_OBJECT
     int block;
+    qint64 gallerySize;
     void init()
     {
@@ -338,6 +352,7 @@ class memGallery : public Gallery
             MemoryGalleries::galleries[file] = gallery->read();
             align(MemoryGalleries::galleries[file]);
             MemoryGalleries::aligned[file] = true;
+            gallerySize = MemoryGalleries::galleries[file].size();
         }
     }
@@ -349,6 +364,10 @@ class memGallery : public Gallery
         }
         TemplateList templates = MemoryGalleries::galleries[file].mid(block*readBlockSize, readBlockSize);
+        for (qint64 i = 0; i < templates.size();i++) {
+            templates[i].file.set("progress", i + block * readBlockSize);
+        }
+
         *done = (templates.size() < readBlockSize);
         block = *done ? 0 : block+1;
         return templates;
@@ -389,6 +408,16 @@ class memGallery : public Gallery
         templates.alignedData = alignedData;
     }
+    qint64 totalSize()
+    {
+        return gallerySize;
+    }
+
+    qint64 position()
+    {
+        return block * readBlockSize;
+    }
+
 };
 BR_REGISTER(Gallery, memGallery)
@@ -449,16 +478,19 @@ FileList FileList::fromGallery(const File &amp; file, bool cache)
  *
  * \see txtGallery
  */
-class csvGallery : public Gallery
+class csvGallery : public FileGallery
 {
     Q_OBJECT
     Q_PROPERTY(int fileIndex READ get_fileIndex WRITE set_fileIndex RESET reset_fileIndex)
     BR_PROPERTY(int, fileIndex, 0)
     FileList files;
+    QStringList headers;
     ~csvGallery()
     {
+        f.close();
+
         if (files.isEmpty()) return;
         QMap<QString,QVariant> samples;
@@ -496,25 +528,37 @@ class csvGallery : public Gallery
     TemplateList readBlock(bool *done)
     {
-        *done = true;
+        *done = false;
         TemplateList templates;
-        if (!file.exists()) return templates;
-
-        QStringList lines = QtUtils::readLines(file);
+        if (!file.exists()) {
+            *done = true;
+            return templates;
+        }
         QRegExp regexp("\\s*,\\s*");
-        QStringList headers;
-        if (!lines.isEmpty()) headers = lines.takeFirst().split(regexp);
-        foreach (const QString &line, lines) {
+        if (f.pos() == 0)
+        {
+            // read a line
+            QByteArray lineBytes = f.readLine();
+            QString line = QString::fromLocal8Bit(lineBytes).trimmed();
+            headers = line.split(regexp);
+        }
+
+        for (qint64 i = 0; i < this->readBlockSize && !f.atEnd(); i++){
+            QByteArray lineBytes = f.readLine();
+            QString line = QString::fromLocal8Bit(lineBytes).trimmed();
+
             QStringList words = line.split(regexp);
             if (words.size() != headers.size()) continue;
-            File f;
-            for (int i=0; i<words.size(); i++) {
-                if (i == 0) f.name = words[i];
-                else        f.set(headers[i], words[i]);
+            File fi;
+            for (int j=0; j<words.size(); j++) {
+                if (j == 0) fi.name = words[j];
+                else        fi.set(headers[j], words[j]);
             }
-            templates.append(f);
+            templates.append(fi);
+            templates.last().file.set("progress", f.pos());
         }
+        *done = f.atEnd();
         return templates;
     }
@@ -568,18 +612,12 @@ BR_REGISTER(Gallery, csvGallery)
 \endverbatim
  * \see csvGallery
  */
-class txtGallery : public Gallery
+class txtGallery : public FileGallery
 {
     Q_OBJECT
     Q_PROPERTY(QString label READ get_label WRITE set_label RESET reset_label STORED false)
     BR_PROPERTY(QString, label, "")
-    QFile f;
-    ~txtGallery()
-    {
-        f.close();
-    }
-
     TemplateList readBlock(bool *done)
     {
         *done = false;
@@ -597,6 +635,7 @@ class txtGallery : public Gallery
                 int splitIndex = line.lastIndexOf(' ');
                 if (splitIndex == -1) templates.append(File(line));
                 else                  templates.append(File(line.mid(0, splitIndex), line.mid(splitIndex+1)));
+                templates.last().file.set("progress", this->position());
             }
             if (f.atEnd()) {
@@ -608,14 +647,6 @@ class txtGallery : public Gallery
         return templates;
     }
-    void init()
-    {
-        f.setFileName(file);
-        QtUtils::touchDir(f);
-        if (!f.open(QFile::ReadWrite))
-            qFatal("Failed to open %s for read/write.", qPrintable(file));
-    }
-
     void write(const Template &t)
     {
         QString line = t.file.name;
@@ -627,21 +658,16 @@ class txtGallery : public Gallery
 };
 BR_REGISTER(Gallery, txtGallery)
+
 /*!
  * \ingroup galleries
  * \brief Treats each line as a call to File::flat()
  * \author Josh Klontz \cite jklontz
  */
-class flatGallery : public Gallery
+class flatGallery : public FileGallery
 {
     Q_OBJECT
-    QFile f;
-    ~flatGallery()
-    {
-        f.close();
-    }
-
     TemplateList readBlock(bool *done)
     {
         *done = false;
@@ -654,8 +680,10 @@ class flatGallery : public Gallery
         {
             QByteArray line = f.readLine();
-            if (!line.isEmpty())
+            if (!line.isEmpty()) {
                 templates.append(File(QString::fromLocal8Bit(line).trimmed()));
+                templates.last().file.set("progress", this->position());
+            }
             if (f.atEnd()) {
                 *done=true;
@@ -666,15 +694,6 @@ class flatGallery : public Gallery
         return templates;
     }
-    void init()
-    {
-        f.setFileName(file);
-        QtUtils::touchDir(f);
-        if (!f.open(QFile::ReadWrite))
-            qFatal("Failed to open %s for read/write.", qPrintable(file));
-
-    }
-
     void write(const Template &t)
     {
         f.write((t.file.flat()+"\n").toLocal8Bit() );
@@ -688,29 +707,140 @@ BR_REGISTER(Gallery, flatGallery)
  * \brief A \ref sigset input.
  * \author Josh Klontz \cite jklontz
  */
-class xmlGallery : public Gallery
+class xmlGallery : public FileGallery
 {
     Q_OBJECT
     Q_PROPERTY(bool ignoreMetadata READ get_ignoreMetadata WRITE set_ignoreMetadata RESET reset_ignoreMetadata STORED false)
     BR_PROPERTY(bool, ignoreMetadata, false)
     FileList files;
+    QXmlStreamReader reader;
+
+    QString currentSignatureName;
+    bool signatureActive;
+
     ~xmlGallery()
     {
+        f.close();
         if (!files.isEmpty())
             BEE::writeSigset(file, files, ignoreMetadata);
     }
     TemplateList readBlock(bool *done)
     {
+        if (reader.atEnd())
+            f.seek(0);
+
+        TemplateList templates;
+        qint64 count = 0;
+        while (!reader.atEnd())
+        {
+            // if an identity is active we try to read presentations
+            if (signatureActive)
+            {
+                while (signatureActive)
+                {
+                    QXmlStreamReader::TokenType signatureToken = reader.readNext();
+
+                    // did the signature end?
+                    if (signatureToken == QXmlStreamReader::EndElement && reader.name() == "biometric-signature") {
+                        signatureActive = false;
+                        break;
+                    }
+                    // did we reach the end of the document? Theoretically this shoudln't happen without reaching the end of
+                    if (signatureToken == QXmlStreamReader::EndDocument)
+                        break;
+
+                    // a presentation!
+                    if (signatureToken == QXmlStreamReader::StartElement && reader.name() == "presentation") {
+                        templates.append(Template(File("",currentSignatureName)));
+                        foreach (const QXmlStreamAttribute & attribute, reader.attributes()) {
+                            // file-name is stored directly on file, not as a key/value pair
+                            if (attribute.name() == "file-name")
+                                templates.last().file.name = attribute.value().toString();
+                            // other values are directly set as metadata
+                            else if (!ignoreMetadata) templates.last().file.set(attribute.name().toString(), attribute.value().toString());
+                        }
+
+                        // a presentation can have bounding boxes as child elements
+                        bool signatureActive = true;
+                        QList<QRectF> rects = templates.last().file.rects();
+                        while (signatureActive)
+                        {
+                            QXmlStreamReader::TokenType pToken = reader.readNext();
+                            if (pToken == QXmlStreamReader::EndElement && reader.name() == "presentation")
+                                break;
+
+                            if (pToken == QXmlStreamReader::StartElement)
+                            {
+                                // get boudning box properties as attributes, just going to assume this all works
+                                qreal x = reader.attributes().value("x").toDouble();
+                                qreal y = reader.attributes().value("y").toDouble();
+                                qreal width =  reader.attributes().value("width").toDouble();
+                                qreal height = reader.attributes().value("height").toDouble();
+                                rects += QRectF(x, y, width, height);
+                            }
+                        }
+                        templates.last().file.setRects(rects);
+                        templates.last().file.set("progress", f.pos());
+
+                        count++;
+                        if (count >= this->readBlockSize) {
+                            *done = false;
+                            return templates;
+                        }
+                    }
+                }
+            }
+            // otherwise, keep reading elements until the next identity is reacehed
+            else
+            {
+                QXmlStreamReader::TokenType token = reader.readNext();
+
+                // end of file?
+                if (token == QXmlStreamReader::EndDocument)
+                    break;
+
+                // we are only interested in new elements
+                if (token != QXmlStreamReader::StartElement)
+                    continue;
+
+                QStringRef elName = reader.name();
+
+                // biometric-signature-set is the root element
+                if (elName == "biometric-signature-set")
+                    continue;
+
+                // biometric-signature -- an identity
+                if (elName == "biometric-signature")
+                {
+                    // read the name associated with the current signature
+                    if (!reader.attributes().hasAttribute("name"))
+                    {
+                        qDebug() << "Biometric signature missing name";
+                        continue;
+                    }
+                    currentSignatureName = reader.attributes().value("name").toString();
+                    signatureActive = true;
+                }
+            }
+
+        }
         *done = true;
-        return TemplateList(BEE::readSigset(file, ignoreMetadata));
+
+        return templates;
     }
     void write(const Template &t)
     {
         files.append(t.file);
     }
+
+    void init()
+    {
+        FileGallery::init();
+        reader.setDevice(&f);
+    }
 };
 BR_REGISTER(Gallery, xmlGallery)
@@ -1178,6 +1308,17 @@ BR_REGISTER(Gallery, vbbGallery)
 #endif
+void FileGallery::init()
+{
+    f.setFileName(file);
+    QtUtils::touchDir(f);
+    if (!f.open(QFile::ReadWrite))
+        qFatal("Failed to open %s for read/write.", qPrintable(file));
+    fileSize = f.size();
+
+    Gallery::init();
+}
+
 } // namespace br
 #include "gallery.moc"
@@ -517,14 +517,18 @@ class ProgressCounterTransform : public TimeVaryingTransform
         qint64 elapsed = timer.elapsed();
+        if (!dst.empty()) {
+            Globals->currentProgress = dst.last().file.get<qint64>("progress",0);
+            dst.last().file.remove("progress");
+            Globals->currentStep++;
+        }
+
         // updated every second
         if (elapsed > 1000) {
             Globals->printStatus();
             timer.start();
         }
-        Globals->currentStep++;
-
         return;
     }
@@ -537,12 +541,13 @@ class ProgressCounterTransform : public TimeVaryingTransform
     {
         (void) data;
         float p = br_progress();
-        qDebug("%05.2f%%  ELAPSED=%s  REMAINING=%s  COUNT=%g/%g  \r", p*100, QtUtils::toTime(Globals->startTime.elapsed()/1000.0f).toStdString().c_str(), QtUtils::toTime(0).toStdString().c_str(), Globals->currentStep, Globals->totalSteps);
+        qDebug("%05.2f%%  ELAPSED=%s  REMAINING=%s  COUNT=%g  \r", p*100, QtUtils::toTime(Globals->startTime.elapsed()/1000.0f).toStdString().c_str(), QtUtils::toTime(0).toStdString().c_str(), Globals->currentStep);
     }
     void init()
     {
         timer.start();
+        Globals->currentStep = 0;
     }
 public:
@@ -689,6 +694,40 @@ public:
 BR_REGISTER(Transform, OutputTransform)
+class FileExclusionTransform : public UntrainableMetaTransform
+{
+    Q_OBJECT
+
+    Q_PROPERTY(QString exclusionGallery READ get_exclusionGallery WRITE set_exclusionGallery RESET reset_exclusionGallery STORED false)
+    BR_PROPERTY(QString, exclusionGallery, "")
+
+    QSet<QString> excluded;
+
+    void project(const Template & src, Template & dst) const
+    {
+        qFatal("FileExclusion can't do anything here");
+    }
+
+    void project(const TemplateList &src, TemplateList &dst) const
+    {
+        foreach(const Template & srcTemp, src)
+        {
+            if (!excluded.contains(srcTemp.file))
+                dst.append(srcTemp);
+        }
+    }
+
+    void init()
+    {
+        if (exclusionGallery.isEmpty())
+            return;
+        FileList temp = FileList::fromGallery(exclusionGallery);
+        excluded = QSet<QString>::fromList(temp.names());
+    }
+};
+
+BR_REGISTER(Transform, FileExclusionTransform)
+
 }
 #include "misc.moc"
@@ -345,6 +345,21 @@ protected:
     UntrainableMetadataTransform() : MetadataTransform(false) {}
 };
+class FileGallery : public Gallery
+{
+    Q_OBJECT
+public:
+    QFile f;
+    qint64 fileSize;
+
+    virtual ~FileGallery() { f.close(); }
+
+    void init();
+
+    qint64 totalSize() { return fileSize; }
+    qint64 position() { return f.pos(); }
+};
+
 }
 #endif // OPENBR_INTERNAL_H
@@ -497,6 +497,8 @@ class ProcessWrapperTransform : public TimeVaryingTransform
     void projectUpdate(const TemplateList &src, TemplateList &dst)
     {
+        if (src.empty())
+            return;
         if (!processActive)
         {