Preliminary work on not loading complete galleries during enrollment

The basic idea is to read galleries incrementally, but there are some complications especially related to progress counting--if we don't read a gallery we don't know how many templates are stored in it since gallery formats aren't nice enough to provide headers with that information. One solution to the progress counting problem is to measure progress based on the position of a file pointer in the gallery file (i.e. measure the current position in the gallery file, divide by the total size of the gallery file). This is supported by expanding the Gallery API to include a totalSize method indicating the total size of the gallery file (or total number of templates if that is known), and then as templates are read, their position is stored in metadata (using the "p" key). Several galleries are updated to respect readBlockSize, and also to store position data in read templates. Support for filtering out already enrolled templates in read-mode was maintained by making the filtering an online process (part of the enrollment pipeline) rather than a batch process done before enrollment-proper starts.

Preliminary work on not loading complete galleries during enrollment
The basic idea is to read galleries incrementally, but there are some complications especially related to progress counting--if we don't read a gallery we don't know how many templates are stored in it since gallery formats aren't nice enough to provide headers with that information. One solution to the progress counting problem is to measure progress based on the position of a file pointer in the gallery file (i.e. measure the current position in the gallery file, divide by the total size of the gallery file). This is supported by expanding the Gallery API to include a totalSize method indicating the total size of the gallery file (or total number of templates if that is known), and then as templates are read, their position is stored in metadata (using the "p" key). Several galleries are updated to respect readBlockSize, and also to store position data in read templates. Support for filtering out already enrolled templates in read-mode was maintained by making the filtering an online process (part of the enrollment pipeline) rather than a batch process done before enrollment-proper starts.
Charles Otto
1 parent 8a207670
Showing 7 changed files with 291 additions and 90 deletions
openbr/core/core.cpp
openbr/openbr_plugin.cpp
openbr/openbr_plugin.h
openbr/plugins/gallery.cpp
openbr/plugins/misc.cpp
openbr/plugins/openbr_internal.h
openbr/plugins/process.cpp
@@ -134,55 +134,51 @@ struct AlgorithmCore
             if (input.name.isEmpty()) return FileList();
             else                      gallery = getMemoryGallery(input);
         }
-        TemplateList data(TemplateList::fromGallery(input));
         bool multiProcess = Globals->file.getBool("multiProcess", false);
+        bool fileExclusion = false;
-        if (gallery.contains("append"))
-        {
-            // Remove any templates which are already in the gallery
-            QScopedPointer<Gallery> g(Gallery::make(gallery));
-            files = g->files();
-            QSet<QString> nameSet = QSet<QString>::fromList(files.names());
-            for (int i = data.size() - 1; i>=0; i--) {
-                if (nameSet.contains(data[i].file.name))
-                {
-                    data.removeAt(i);
-                }
-            }
+        // In append mode, we will exclude any templates with filenames already present in the output gallery
+        if (gallery.contains("append") && gallery.exists() ) {
+            FileList::fromGallery(gallery,true);
+            fileExclusion = true;
         }
-        if (data.empty())
-            return files;
+        Gallery * temp = Gallery::make(input);
+        qint64 total = temp->totalSize();
-        // Store steps for ProgressCounter
         Globals->currentStep = 0;
-        Globals->totalSteps = data.length();
+        Globals->totalSteps = total;
         QScopedPointer<Transform> basePipe;
-        if (!multiProcess)
-        {
-            QString pipeDesc = "GalleryOutput("+gallery.flat()+")+ProgressCounter("+QString::number(data.length())+")+Discard";
+        QString pipeDesc = "GalleryOutput("+gallery.flat()+")+ProgressCounter("+QString::number(total)+")+Discard";
+
+        if (!multiProcess) {
             basePipe.reset(Transform::make(pipeDesc,NULL));
             CompositeTransform * downcast = dynamic_cast<CompositeTransform *>(basePipe.data());
-            if (downcast == NULL)
-                qFatal("downcast failed?");
-            // replace that placeholder with the current algorithm
+            if (downcast == NULL) qFatal("downcast failed?");
+
             downcast->transforms.prepend(this->transform.data());
+            if (fileExclusion) {
+                Transform * temp = Transform::make("FileExclusion(" + gallery.flat() + ")", downcast);
+                downcast->transforms.prepend(temp);
+            }
             // call init on the pipe to collapse the algorithm (if its top level is a pipe)
             downcast->init();
         }
-        else
-        {
-            QString pipeDesc = "ProcessWrapper("+transformString+")"+"+GalleryOutput("+gallery.flat()+")+ProgressCounter("+QString::number(data.length())+")+Discard";
+        else {
+            pipeDesc = "ProcessWrapper("+transformString+")"+pipeDesc;
+            if (fileExclusion)
+                pipeDesc = "FileExclusion(" + gallery.flat() +")" + pipeDesc;
+
             basePipe.reset(Transform::make(pipeDesc,NULL));
         }
         // Next, we make a Stream (with placeholder transform)
-        QString streamDesc = "Stream(readMode=DistributeFrames)";
+        QString streamDesc = "Stream(readMode=StreamGallery)";
         QScopedPointer<Transform> baseStream(Transform::make(streamDesc, NULL));
         WrapperTransform * wrapper = dynamic_cast<WrapperTransform *> (baseStream.data());
@@ -194,9 +190,11 @@ struct AlgorithmCore
         Globals->startTime.start();
-        wrapper->projectUpdate(data,data);
+        TemplateList data, output;
+        data.append(input);
+        wrapper->projectUpdate(data, output);
-        files.append(data.files());
+        files.append(output.files());
         return files;
     }
@@ -337,13 +335,9 @@ struct AlgorithmCore
         // comparison against the smaller gallery (which will be enrolled, and stored in memory).
         bool needEnrollRows = false;
-
-
-
         if (output.exists() && output.get<bool>("cache", false)) return;
         if (queryGallery == ".") queryGallery = targetGallery;
-
         // To decide which gallery is larger, we need to read both, but at this point we just want the
         // metadata, and don't need the enrolled matrices.
         FileList targetMetadata;
@@ -359,15 +353,21 @@ struct AlgorithmCore
         File rowGallery = queryGallery;
         File colGallery = targetGallery;
-        int rowSize = queryMetadata.size();
+        qint64 rowSize;
+        Gallery * temp;
         if (transposeMode)
         {
             rowGallery = targetGallery;
             colGallery = queryGallery;
-            rowSize = targetMetadata.size();
+            temp = Gallery::make(targetGallery);
         }
-
+        else
+        {
+            temp = Gallery::make(queryGallery);
+        }
+        rowSize = temp->totalSize();
+        delete temp;
         // Is the column gallery already enrolled? We keep the enrolled column gallery in memory, and in multi-process
         // mode, every worker process retains a copy of this gallery in memory. When not in multi-process mode, we can
@@ -423,8 +423,6 @@ struct AlgorithmCore
         // progress counting step.
         // After the base algorithm is built, the whole thing will be run in a stream, so that I/O can be handled sequentially.
-
-
         // The actual comparison step is done by a GalleryCompare transform, which has a Distance, and a gallery as data.
         // Incoming templates are compared against the templates in the gallery, and the output is the resulting score
         // vector.
@@ -865,14 +865,14 @@ void br::Context::printStatus()
     const float p = progress();
     if (p < 1) {
         int s = timeRemaining();
-        fprintf(stderr, "%05.2f%%  ELAPSED=%s  REMAINING=%s  COUNT=%g/%g  \r", p*100, QtUtils::toTime(Globals->startTime.elapsed()/1000.0f).toStdString().c_str(), QtUtils::toTime(s).toStdString().c_str(), Globals->currentStep, Globals->totalSteps);
+        fprintf(stderr,"%05.2f%%  ELAPSED=%s  REMAINING=%s  COUNT=%g  \r", p*100, QtUtils::toTime(Globals->startTime.elapsed()/1000.0f).toStdString().c_str(), QtUtils::toTime(s).toStdString().c_str(), Globals->currentStep);
     }
 }
 float br::Context::progress() const
 {
     if (totalSteps == 0) return -1;
-    return currentStep / totalSteps;
+    return currentProgress / totalSteps;
 }
 void br::Context::setProperty(const QString &key, const QString &value)
@@ -687,6 +687,10 @@ public:
     Q_PROPERTY(double currentStep READ get_currentStep WRITE set_currentStep RESET reset_currentStep)
     BR_PROPERTY(double, currentStep, 0)
+    Q_PROPERTY(double currentProgress READ get_currentProgress WRITE set_currentProgress RESET reset_currentProgress)
+    BR_PROPERTY(double, currentProgress, 0)
+
+
     /*!
      * \brief Used internally to compute progress() and timeRemaining().
      */
@@ -1095,6 +1099,9 @@ public:
     static Gallery *make(const File &file); /*!< \brief Make a gallery to/from a file on disk. */
     void init();
+    virtual qint64 totalSize() { return std::numeric_limits<qint64>::max(); }
+    virtual qint64 position() { return 0; }
+
 private:
     QSharedPointer<Gallery> next;
 };
@@ -23,6 +23,7 @@
 #include <QSqlError>
 #include <QSqlQuery>
 #include <QSqlRecord>
+#include <QXmlStreamReader>
 #endif // BR_EMBEDDED
 #include <opencv2/highgui/highgui.hpp>
 #include "openbr_internal.h"
@@ -127,6 +128,7 @@ class galGallery : public Gallery
             Template m;
             stream >> m;
             templates.append(m);
+            templates.last().file.set("p", totalSize());
         }
         *done = stream.atEnd();
@@ -140,6 +142,17 @@ class galGallery : public Gallery
         stream << t;
     }
+
+    qint64 totalSize()
+    {
+        return gallery.size();
+    }
+
+    qint64 position()
+    {
+        return gallery.pos();
+    }
+
 };
 BR_REGISTER(Gallery, galGallery)
@@ -328,6 +341,7 @@ class memGallery : public Gallery
 {
     Q_OBJECT
     int block;
+    qint64 gallerySize;
     void init()
     {
@@ -338,6 +352,7 @@ class memGallery : public Gallery
             MemoryGalleries::galleries[file] = gallery->read();
             align(MemoryGalleries::galleries[file]);
             MemoryGalleries::aligned[file] = true;
+            gallerySize = MemoryGalleries::galleries[file].size();
         }
     }
@@ -349,6 +364,10 @@ class memGallery : public Gallery
         }
         TemplateList templates = MemoryGalleries::galleries[file].mid(block*readBlockSize, readBlockSize);
+        for (qint64 i = 0; i < templates.size();i++) {
+            templates[i].file.set("p", i + block * readBlockSize);
+        }
+
         *done = (templates.size() < readBlockSize);
         block = *done ? 0 : block+1;
         return templates;
@@ -389,6 +408,16 @@ class memGallery : public Gallery
         templates.alignedData = alignedData;
     }
+    qint64 totalSize()
+    {
+        return gallerySize;
+    }
+
+    qint64 position()
+    {
+        return block * readBlockSize;
+    }
+
 };
 BR_REGISTER(Gallery, memGallery)
@@ -449,16 +478,19 @@ FileList FileList::fromGallery(const File &amp; file, bool cache)
  *
  * \see txtGallery
  */
-class csvGallery : public Gallery
+class csvGallery : public FileGallery
 {
     Q_OBJECT
     Q_PROPERTY(int fileIndex READ get_fileIndex WRITE set_fileIndex RESET reset_fileIndex)
     BR_PROPERTY(int, fileIndex, 0)
     FileList files;
+    QStringList headers;
     ~csvGallery()
     {
+        f.close();
+
         if (files.isEmpty()) return;
         QMap<QString,QVariant> samples;
@@ -496,25 +528,37 @@ class csvGallery : public Gallery
     TemplateList readBlock(bool *done)
     {
-        *done = true;
+        *done = false;
         TemplateList templates;
-        if (!file.exists()) return templates;
-
-        QStringList lines = QtUtils::readLines(file);
+        if (!file.exists()) {
+            *done = true;
+            return templates;
+        }
         QRegExp regexp("\\s*,\\s*");
-        QStringList headers;
-        if (!lines.isEmpty()) headers = lines.takeFirst().split(regexp);
-        foreach (const QString &line, lines) {
+        if (f.pos() == 0)
+        {
+            // read a line
+            QByteArray lineBytes = f.readLine();
+            QString line = QString::fromLocal8Bit(lineBytes).trimmed();
+            headers = line.split(regexp);
+        }
+
+        for (qint64 i = 0; i < this->readBlockSize && !f.atEnd(); i++){
+            QByteArray lineBytes = f.readLine();
+            QString line = QString::fromLocal8Bit(lineBytes).trimmed();
+
             QStringList words = line.split(regexp);
             if (words.size() != headers.size()) continue;
-            File f;
-            for (int i=0; i<words.size(); i++) {
-                if (i == 0) f.name = words[i];
-                else        f.set(headers[i], words[i]);
+            File fi;
+            for (int j=0; j<words.size(); j++) {
+                if (j == 0) fi.name = words[j];
+                else        fi.set(headers[j], words[j]);
             }
-            templates.append(f);
+            templates.append(fi);
+            templates.last().file.set("p", f.pos());
         }
+        *done = f.atEnd();
         return templates;
     }
@@ -568,18 +612,12 @@ BR_REGISTER(Gallery, csvGallery)
 \endverbatim
  * \see csvGallery
  */
-class txtGallery : public Gallery
+class txtGallery : public FileGallery
 {
     Q_OBJECT
     Q_PROPERTY(QString label READ get_label WRITE set_label RESET reset_label STORED false)
     BR_PROPERTY(QString, label, "")
-    QFile f;
-    ~txtGallery()
-    {
-        f.close();
-    }
-
     TemplateList readBlock(bool *done)
     {
         *done = false;
@@ -597,6 +635,7 @@ class txtGallery : public Gallery
                 int splitIndex = line.lastIndexOf(' ');
                 if (splitIndex == -1) templates.append(File(line));
                 else                  templates.append(File(line.mid(0, splitIndex), line.mid(splitIndex+1)));
+                templates.last().file.set("p", this->position());
             }
             if (f.atEnd()) {
@@ -608,14 +647,6 @@ class txtGallery : public Gallery
         return templates;
     }
-    void init()
-    {
-        f.setFileName(file);
-        QtUtils::touchDir(f);
-        if (!f.open(QFile::ReadWrite))
-            qFatal("Failed to open %s for read/write.", qPrintable(file));
-    }
-
     void write(const Template &t)
     {
         QString line = t.file.name;
@@ -627,21 +658,16 @@ class txtGallery : public Gallery
 };
 BR_REGISTER(Gallery, txtGallery)
+
 /*!
  * \ingroup galleries
  * \brief Treats each line as a call to File::flat()
  * \author Josh Klontz \cite jklontz
  */
-class flatGallery : public Gallery
+class flatGallery : public FileGallery
 {
     Q_OBJECT
-    QFile f;
-    ~flatGallery()
-    {
-        f.close();
-    }
-
     TemplateList readBlock(bool *done)
     {
         *done = false;
@@ -654,8 +680,10 @@ class flatGallery : public Gallery
         {
             QByteArray line = f.readLine();
-            if (!line.isEmpty())
+            if (!line.isEmpty()) {
                 templates.append(File(QString::fromLocal8Bit(line).trimmed()));
+                templates.last().file.set("p", this->position());
+            }
             if (f.atEnd()) {
                 *done=true;
@@ -666,15 +694,6 @@ class flatGallery : public Gallery
         return templates;
     }
-    void init()
-    {
-        f.setFileName(file);
-        QtUtils::touchDir(f);
-        if (!f.open(QFile::ReadWrite))
-            qFatal("Failed to open %s for read/write.", qPrintable(file));
-
-    }
-
     void write(const Template &t)
     {
         f.write((t.file.flat()+"\n").toLocal8Bit() );
@@ -688,29 +707,140 @@ BR_REGISTER(Gallery, flatGallery)
  * \brief A \ref sigset input.
  * \author Josh Klontz \cite jklontz
  */
-class xmlGallery : public Gallery
+class xmlGallery : public FileGallery
 {
     Q_OBJECT
     Q_PROPERTY(bool ignoreMetadata READ get_ignoreMetadata WRITE set_ignoreMetadata RESET reset_ignoreMetadata STORED false)
     BR_PROPERTY(bool, ignoreMetadata, false)
     FileList files;
+    QXmlStreamReader reader;
+
+    QString currentSignatureName;
+    bool signatureActive;
+
     ~xmlGallery()
     {
+        f.close();
         if (!files.isEmpty())
             BEE::writeSigset(file, files, ignoreMetadata);
     }
     TemplateList readBlock(bool *done)
     {
+        if (reader.atEnd())
+            f.seek(0);
+
+        TemplateList templates;
+        qint64 count = 0;
+        while (!reader.atEnd())
+        {
+            // if an identity is active we try to read presentations
+            if (signatureActive)
+            {
+                while (signatureActive)
+                {
+                    QXmlStreamReader::TokenType signatureToken = reader.readNext();
+
+                    // did the signature end?
+                    if (signatureToken == QXmlStreamReader::EndElement && reader.name() == "biometric-signature") {
+                        signatureActive = false;
+                        break;
+                    }
+                    // did we reach the end of the document? Theoretically this shoudln't happen without reaching the end of
+                    if (signatureToken == QXmlStreamReader::EndDocument)
+                        break;
+
+                    // a presentation!
+                    if (signatureToken == QXmlStreamReader::StartElement && reader.name() == "presentation") {
+                        templates.append(Template(File("",currentSignatureName)));
+                        foreach (const QXmlStreamAttribute & attribute, reader.attributes()) {
+                            // file-name is stored directly on file, not as a key/value pair
+                            if (attribute.name() == "file-name")
+                                templates.last().file.name = attribute.value().toString();
+                            // other values are directly set as metadata
+                            else if (!ignoreMetadata) templates.last().file.set(attribute.name().toString(), attribute.value().toString());
+                        }
+
+                        // a presentation can have bounding boxes as child elements
+                        bool signatureActive = true;
+                        QList<QRectF> rects = templates.last().file.rects();
+                        while (signatureActive)
+                        {
+                            QXmlStreamReader::TokenType pToken = reader.readNext();
+                            if (pToken == QXmlStreamReader::EndElement && reader.name() == "presentation")
+                                break;
+
+                            if (pToken == QXmlStreamReader::StartElement)
+                            {
+                                // get boudning box properties as attributes, just going to assume this all works
+                                qreal x = reader.attributes().value("x").toDouble();
+                                qreal y = reader.attributes().value("y").toDouble();
+                                qreal width =  reader.attributes().value("width").toDouble();
+                                qreal height = reader.attributes().value("height").toDouble();
+                                rects += QRectF(x, y, width, height);
+                            }
+                        }
+                        templates.last().file.setRects(rects);
+                        templates.last().file.set("p", f.pos());
+
+                        count++;
+                        if (count >= this->readBlockSize) {
+                            *done = false;
+                            return templates;
+                        }
+                    }
+                }
+            }
+            // otherwise, keep reading elements until the next identity is reacehed
+            else
+            {
+                QXmlStreamReader::TokenType token = reader.readNext();
+
+                // end of file?
+                if (token == QXmlStreamReader::EndDocument)
+                    break;
+
+                // we are only interested in new elements
+                if (token != QXmlStreamReader::StartElement)
+                    continue;
+
+                QStringRef elName = reader.name();
+
+                // biometric-signature-set is the root element
+                if (elName == "biometric-signature-set")
+                    continue;
+
+                // biometric-signature -- an identity
+                if (elName == "biometric-signature")
+                {
+                    // read the name associated with the current signature
+                    if (!reader.attributes().hasAttribute("name"))
+                    {
+                        qDebug() << "Biometric signature missing name";
+                        continue;
+                    }
+                    currentSignatureName = reader.attributes().value("name").toString();
+                    signatureActive = true;
+                }
+            }
+
+        }
         *done = true;
-        return TemplateList(BEE::readSigset(file, ignoreMetadata));
+
+        return templates;
     }
     void write(const Template &t)
     {
         files.append(t.file);
     }
+
+    void init()
+    {
+        FileGallery::init();
+        reader.setDevice(&f);
+    }
 };
 BR_REGISTER(Gallery, xmlGallery)
@@ -1178,6 +1308,17 @@ BR_REGISTER(Gallery, vbbGallery)
 #endif
+void FileGallery::init()
+{
+    f.setFileName(file);
+    QtUtils::touchDir(f);
+    if (!f.open(QFile::ReadWrite))
+        qFatal("Failed to open %s for read/write.", qPrintable(file));
+    fileSize = f.size();
+
+    Gallery::init();
+}
+
 } // namespace br
 #include "gallery.moc"
@@ -517,14 +517,17 @@ class ProgressCounterTransform : public TimeVaryingTransform
         qint64 elapsed = timer.elapsed();
+        if (!dst.empty()) {
+            Globals->currentProgress = dst.last().file.get<qint64>("p",0);
+            Globals->currentStep++;
+        }
+
         // updated every second
         if (elapsed > 1000) {
             Globals->printStatus();
             timer.start();
         }
-        Globals->currentStep++;
-
         return;
     }
@@ -537,12 +540,13 @@ class ProgressCounterTransform : public TimeVaryingTransform
     {
         (void) data;
         float p = br_progress();
-        qDebug("%05.2f%%  ELAPSED=%s  REMAINING=%s  COUNT=%g/%g  \r", p*100, QtUtils::toTime(Globals->startTime.elapsed()/1000.0f).toStdString().c_str(), QtUtils::toTime(0).toStdString().c_str(), Globals->currentStep, Globals->totalSteps);
+        qDebug("%05.2f%%  ELAPSED=%s  REMAINING=%s  COUNT=%g  \r", p*100, QtUtils::toTime(Globals->startTime.elapsed()/1000.0f).toStdString().c_str(), QtUtils::toTime(0).toStdString().c_str(), Globals->currentStep);
     }
     void init()
     {
         timer.start();
+        Globals->currentStep = 0;
     }
 public:
@@ -689,6 +693,40 @@ public:
 BR_REGISTER(Transform, OutputTransform)
+class FileExclusionTransform : public UntrainableMetaTransform
+{
+    Q_OBJECT
+
+    Q_PROPERTY(QString exclusionGallery READ get_exclusionGallery WRITE set_exclusionGallery RESET reset_exclusionGallery STORED false)
+    BR_PROPERTY(QString, exclusionGallery, "")
+
+    QSet<QString> excluded;
+
+    void project(const Template & src, Template & dst) const
+    {
+        qFatal("FileExclusion can't do anything here");
+    }
+
+    void project(const TemplateList &src, TemplateList &dst) const
+    {
+        foreach(const Template & srcTemp, src)
+        {
+            if (!excluded.contains(srcTemp.file))
+                dst.append(srcTemp);
+        }
+    }
+
+    void init()
+    {
+        if (exclusionGallery.isEmpty())
+            return;
+        FileList temp = FileList::fromGallery(exclusionGallery);
+        excluded = QSet<QString>::fromList(temp.names());
+    }
+};
+
+BR_REGISTER(Transform, FileExclusionTransform)
+
 }
 #include "misc.moc"
@@ -345,6 +345,21 @@ protected:
     UntrainableMetadataTransform() : MetadataTransform(false) {}
 };
+class FileGallery : public Gallery
+{
+    Q_OBJECT
+public:
+    QFile f;
+    qint64 fileSize;
+
+    virtual ~FileGallery() { f.close(); }
+
+    void init();
+
+    qint64 totalSize() { return fileSize; }
+    qint64 position() { return f.pos(); }
+};
+
 }
 #endif // OPENBR_INTERNAL_H
@@ -497,6 +497,8 @@ class ProcessWrapperTransform : public TimeVaryingTransform
     void projectUpdate(const TemplateList &src, TemplateList &dst)
     {
+        if (src.empty())
+            return;
         if (!processActive)
         {