Merge pull request #190 from biometrics/large_galleries

Better support for dealing with large galleries

Merge pull request #190 from biometrics/large_galleries
Better support for dealing with large galleries
Josh Klontz
2 parents 063496f6 29a644f2
Showing 4 changed files with 109 additions and 36 deletions
openbr/core/bee.cpp
openbr/core/qtutils.cpp
openbr/plugins/format.cpp
openbr/plugins/gallery.cpp
@@ -164,23 +164,30 @@ Mat BEE::readMat(const br::File &amp;matrix, QString *targetSigset, QString *querySi
     int typeSize = isMask ? sizeof(BEE::Mask_t) : sizeof(BEE::Simmat_t);
     // Get matrix data
-    qint64 bytesExpected = (qint64)rows*(qint64)cols*(qint64)typeSize;
     Mat m;
     if (isMask)
         m.create(rows, cols, OpenCVType<BEE::Mask_t,1>::make());
     else
         m.create(rows, cols, OpenCVType<BEE::Simmat_t,1>::make());
-    qint64 read = file.read((char*)m.data, bytesExpected);
-    if (read != bytesExpected)
-        qFatal("Invalid matrix size.");
+    qint64 bytesPerRow = m.cols * typeSize;
+
+    for (int i=0; i < m.rows;i++)
+    {
+        cv::Mat aRow = m.row(i);
+        qint64 bytesRead = file.read((char *)aRow.data, bytesPerRow);
+        if (bytesRead != bytesPerRow)
+        {
+            qFatal("Didn't read complete row!");
+        }
+    }
     if (!file.atEnd())
         qFatal("Expected matrix end of file.");
     file.close();
-    Mat result;
+    Mat result = m;
     if (isDistance ^ matrix.get<bool>("negate", false)) m.convertTo(result, -1, -1);
-    else                                                result = m.clone();
+
     return result;
 }
@@ -99,7 +99,16 @@ void readFile(const QString &amp;file, QByteArray &amp;data, bool uncompress)
 void writeFile(const QString &file, const QStringList &lines)
 {
-    writeFile(file, lines.join("\n"));
+    QFile f(file);
+    touchDir(f);
+
+    if (!f.open(QFile::WriteOnly))
+        qFatal("Failed to open %s for writing.", qPrintable(file));
+
+    foreach(const QString & line, lines) {
+        f.write((line+"\n").toLocal8Bit() );
+    }
+    f.close();
 }
 void writeFile(const QString &file, const QString &data)
@@ -20,6 +20,9 @@
 #ifndef BR_EMBEDDED
 #include <QtXml>
 #endif // BR_EMBEDDED
+#include <QFile>
+#include <QFileInfo>
+
 #include <opencv2/highgui/highgui.hpp>
 #include <opencv2/highgui/highgui_c.h>
 #include "openbr_internal.h"
@@ -117,22 +120,30 @@ class binFormat : public Format
     void write(const Template &t) const
     {
-        QByteArray data;
-        QDataStream stream(&data, QFile::WriteOnly);
-        if (raw) {
-            const Mat &m = t;
-            stream.writeRawData((const char*)m.data, m.total()*m.elemSize());
-        } else {
-            Mat m;
-            t.m().convertTo(m, CV_32F);
+        QFile f(file);
+        QtUtils::touchDir(f);
+        if (!f.open(QFile::WriteOnly))
+            qFatal("Failed to open %s for writing.", qPrintable(file));
+
+        Mat m;
+        if (!raw) {
+            if (t.m().type() != CV_32FC1)
+                t.m().convertTo(m, CV_32F);
+            else m = t.m();
+
             if (m.channels() != 1) qFatal("Only supports single channel matrices.");
-            stream.writeRawData((const char*)&m.rows, 4);
-            stream.writeRawData((const char*)&m.cols, 4);
-            stream.writeRawData((const char*)m.data, 4*m.rows*m.cols);
+            f.write((const char *) &m.rows, 4);
+            f.write((const char *) &m.cols, 4);
         }
+        else m =  t.m();
-        QtUtils::writeFile(file, data);
+        qint64 rowSize = m.cols * sizeof(float);
+        for (int i=0; i < m.rows; i++)
+        {
+            f.write((const char *) m.row(i).data, rowSize);
+        }
+        f.close();
     }
 };
@@ -574,32 +574,55 @@ class txtGallery : public Gallery
     Q_PROPERTY(QString label READ get_label WRITE set_label RESET reset_label STORED false)
     BR_PROPERTY(QString, label, "")
-    QStringList lines;
-
+    QFile f;
     ~txtGallery()
     {
-        if (!lines.isEmpty())
-            QtUtils::writeFile(file.name, lines);
+        f.close();
     }
     TemplateList readBlock(bool *done)
     {
+        *done = false;
+        if (f.atEnd())
+            f.seek(0);
+
         TemplateList templates;
-        foreach (const QString &line, QtUtils::readLines(file)) {
-            int splitIndex = line.lastIndexOf(' ');
-            if (splitIndex == -1) templates.append(File(line));
-            else                  templates.append(File(line.mid(0, splitIndex), line.mid(splitIndex+1)));
+
+        for (qint64 i = 0; i < readBlockSize; i++)
+        {
+            QByteArray lineBytes = f.readLine();
+            QString line(lineBytes);
+
+            if (!line.isEmpty()){
+                int splitIndex = line.lastIndexOf(' ');
+                if (splitIndex == -1) templates.append(File(line));
+                else                  templates.append(File(line.mid(0, splitIndex), line.mid(splitIndex+1)));
+            }
+
+            if (f.atEnd()) {
+                *done=true;
+                break;
+            }
         }
-        *done = true;
+
         return templates;
     }
+    void init()
+    {
+        f.setFileName(file);
+        QtUtils::touchDir(f);
+        if (!f.open(QFile::ReadWrite))
+            qFatal("Failed to open %s for read/write.", qPrintable(file));
+    }
+
     void write(const Template &t)
     {
         QString line = t.file.name;
         if (!label.isEmpty())
             line += " " + t.file.get<QString>(label);
-        lines.append(line);
+
+        f.write((line+"\n").toLocal8Bit() );
     }
 };
@@ -612,26 +635,49 @@ BR_REGISTER(Gallery, txtGallery)
 class flatGallery : public Gallery
 {
     Q_OBJECT
-    QStringList lines;
+    QFile f;
     ~flatGallery()
     {
-        if (!lines.isEmpty())
-            QtUtils::writeFile(file.name, lines);
+        f.close();
     }
     TemplateList readBlock(bool *done)
     {
+        *done = false;
+        if (f.atEnd())
+            f.seek(0);
+
         TemplateList templates;
-        foreach (const QString &line, QtUtils::readLines(file))
-            templates.append(File(line));
-        *done = true;
+
+        for (qint64 i = 0; i < readBlockSize; i++)
+        {
+            QByteArray line = f.readLine();
+
+            if (!line.isEmpty())
+                templates.append(File(QString(line)));
+
+            if (f.atEnd()) {
+                *done=true;
+                break;
+            }
+        }
+
         return templates;
     }
+    void init()
+    {
+        f.setFileName(file);
+        QtUtils::touchDir(f);
+        if (!f.open(QFile::ReadWrite))
+            qFatal("Failed to open %s for read/write.", qPrintable(file));
+
+    }
+
     void write(const Template &t)
     {
-        lines.append(t.file.flat());
+        f.write((t.file.flat()+"\n").toLocal8Bit() );
     }
 };