Merge pull request #369 from biometrics/bootStrap

Random test/train split in TemplateList::Partition

Merge pull request #369 from biometrics/bootStrap
Random test/train split in TemplateList::Partition
Ben Klein
2 parents 9d766e1e cb24123a
Showing 7 changed files with 39 additions and 16 deletions
app/br/br.cpp
openbr/core/bee.cpp
openbr/core/core.cpp
openbr/openbr_plugin.cpp
openbr/openbr_plugin.h
openbr/plugins/core/crossvalidate.cpp
openbr/plugins/output/eval.cpp
@@ -69,6 +69,7 @@ public:
         bool daemon = false;
         const char *daemon_pipe = NULL;
+        bool isInt = false;
         while (daemon || (argc > 0)) {
             const char *fun;
             int parc;
@@ -78,7 +79,12 @@ public:
             fun = argv[0];
             if (fun[0] == '-') fun++;
-            parc = 0; while ((parc+1 < argc) && (argv[parc+1][0] != '-')) parc++;
+            parc = 0;
+            QString(argv[parc+1]).toInt(&isInt);
+            while ((parc+1 < argc) && ((argv[parc+1][0] != '-') || isInt)) {
+                parc++;
+                QString(argv[parc+1]).toInt(&isInt);
+            }
             parv = (const char **)&argv[1];
             argc = argc - (parc+1);
             argv = &argv[parc+1];
@@ -230,7 +230,7 @@ void makeMask(const QString &amp;targetInput, const QString &amp;queryInput, const QStri
     const FileList targets = TemplateList::fromGallery(targetInput).files();
     const FileList queries = (queryInput == ".") ? targets : TemplateList::fromGallery(queryInput).files();
     const int partitions = targets.first().get<int>("crossValidate");
-    if (partitions == 0) {
+    if (partitions <= 0) {
         writeMatrix(makeMask(targets, queries), mask, targetInput, queryInput);
     } else {
         if (!mask.contains("%1")) qFatal("Mask file name missing partition number place marker (%%1)");
@@ -246,7 +246,7 @@ void makePairwiseMask(const QString &amp;targetInput, const QString &amp;queryInput, con
     const FileList targets = TemplateList::fromGallery(targetInput).files();
     const FileList queries = (queryInput == ".") ? targets : TemplateList::fromGallery(queryInput).files();
     const int partitions = targets.first().get<int>("crossValidate");
-    if (partitions == 0) {
+    if (partitions <= 0) {
         writeMatrix(makePairwiseMask(targets, queries), mask, targetInput, queryInput);
     } else {
         if (!mask.contains("%1")) qFatal("Mask file name missing partition number place marker (%%1)");
@@ -69,7 +69,7 @@ struct AlgorithmCore
         QScopedPointer<Transform> trainingWrapper(br::wrapTransform(transform.data(), "Stream(readMode=DistributeFrames)"));
         TemplateList data(TemplateList::fromGallery(input));
-        if (Globals->crossValidate > 1)
+        if (abs(Globals->crossValidate) > 1)
             for (int i=data.size()-1; i>=0; i--)
                 if (data[i].file.get<bool>("allPartitions",false) || data[i].file.get<bool>("duplicatePartitions",false))
                     data.removeAt(i);
@@ -28,6 +28,7 @@
 #include <QtConcurrentRun>
 #include <algorithm>
 #include <iostream>
+#include <RandomLib/Random.hpp>
 #ifndef BR_EMBEDDED
 #include <QApplication>
@@ -433,7 +434,7 @@ TemplateList TemplateList::fromGallery(const br::File &amp;gallery)
         if (gallery.getBool("reduce"))
             newTemplates = newTemplates.reduced();
-        if (Globals->crossValidate > 1)
+        if (abs(Globals->crossValidate) > 1)
             newTemplates = newTemplates.partition("Label");
         if (!templates.isEmpty() && gallery.get<bool>("merge", false)) {
@@ -528,12 +529,16 @@ QList&lt;int&gt; TemplateList::applyIndex(const QString &amp;propName, const QHash&lt;QString
     return result;
 }
-TemplateList TemplateList::partition(const QString &inputVariable) const
+TemplateList TemplateList::partition(const QString &inputVariable, unsigned int randomSeed) const
 {
-    const int crossValidate = Globals->crossValidate;
+    const int crossValidate = std::abs(Globals->crossValidate);
     if (crossValidate < 2)
         return *this;
+    // Use separate RNG from Common::randN() to avoid re-seeding the global RNG.
+    // rng is seeded with the inputVariable hash in order to maintain partition across br runs, given the same randomSeed seed.
+    RandomLib::Random rng;
+
     TemplateList partitioned = *this;
     for (int i=partitioned.size()-1; i>=0; i--) {
         // See CrossValidateTransform for description of these variables
@@ -546,8 +551,12 @@ TemplateList TemplateList::partition(const QString &amp;inputVariable) const
         } else if (partitioned[i].file.getBool("allPartitions")) {
             partitioned[i].file.set("Partition", -1);
         } else {
-            if (!partitioned[i].file.contains(("Partition"))) {
-                const QByteArray md5 = QCryptographicHash::hash(partitioned[i].file.get<QString>(inputVariable).toLatin1(), QCryptographicHash::Md5);
+            const QByteArray md5 = QCryptographicHash::hash(partitioned[i].file.get<QString>(inputVariable).toLatin1(), QCryptographicHash::Md5);
+            if (randomSeed) {
+                quint64 labelSeed = md5.toHex().right(8).toULongLong(0, 16) + randomSeed;
+                rng.Reseed(labelSeed);
+                partitioned[i].file.set("Partition", rng.Integer() % crossValidate);
+            } else if (!partitioned[i].file.contains(("Partition"))) {
                 // Select the right 8 hex characters so that it can be represented as a 64 bit integer without overflow
                 partitioned[i].file.set("Partition", md5.toHex().right(8).toULongLong(0, 16) % crossValidate);
             }
@@ -459,7 +459,7 @@ struct TemplateList : public QList&lt;Template&gt;
     BR_EXPORT static TemplateList relabel(const TemplateList &tl, const QString &propName, bool preserveIntegers);
     /*!< \brief Assign templates to folds partitions. */
-    BR_EXPORT TemplateList partition(const QString &inputVariable) const;
+    BR_EXPORT TemplateList partition(const QString &inputVariable, unsigned int randomSeed = 0) const;
     BR_EXPORT QList<int> indexProperty(const QString &propName, QHash<QString, int> * valueMap=NULL,QHash<int, QVariant> * reverseLookup = NULL) const;
     BR_EXPORT QList<int> indexProperty(const QString &propName, QHash<QString, int> &valueMap, QHash<int, QVariant> &reverseLookup) const;
@@ -56,8 +56,10 @@ class CrossValidateTransform : public MetaTransform
     Q_OBJECT
     Q_PROPERTY(QString description READ get_description WRITE set_description RESET reset_description STORED false)
     Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false)
+    Q_PROPERTY(unsigned int randomSeed READ get_randomSeed WRITE set_randomSeed RESET reset_randomSeed STORED false)
     BR_PROPERTY(QString, description, "Identity")
     BR_PROPERTY(QString, inputVariable, "Label")
+    BR_PROPERTY(unsigned int, randomSeed, 0)
     // numPartitions copies of transform specified by description.
     QList<br::Transform*> transforms;
@@ -67,13 +69,14 @@ class CrossValidateTransform : public MetaTransform
     // is generally incorrect behavior.
     void train(const TemplateList &data)
     {
-        QList<int> partitions = data.partition(inputVariable).files().crossValidationPartitions();
-        const int numPartitions = Common::Max(partitions)+1;
-
+        QList<int> partitions = data.partition(inputVariable, randomSeed).files().crossValidationPartitions();
+        const int crossValidate = Globals->crossValidate;
+        // Only train once based on the 0th partition if crossValidate is negative.
+        const int numPartitions = (crossValidate < 0) ? 1 : Common::Max(partitions)+1;
         while (transforms.size() < numPartitions)
             transforms.append(make(description));
-        if (numPartitions < 2) {
+        if (std::abs(crossValidate) < 2) {
             transforms.first()->train(data);
             return;
         }
@@ -101,8 +104,13 @@ class CrossValidateTransform : public MetaTransform
     void project(const TemplateList &src, TemplateList &dst) const
     {
-        TemplateList partitioned = src.partition(inputVariable);
+        TemplateList partitioned = src.partition(inputVariable, randomSeed);
+        const int crossValidate = Globals->crossValidate;
+        if (crossValidate < 0) {
+            transforms[0]->project(partitioned, dst);
+            return;
+        }
         for (int i=0; i<partitioned.size(); i++) {
             int partition = partitioned[i].file.get<int>("Partition", 0);
             transforms[partition]->project(partitioned, dst);
@@ -45,7 +45,7 @@ class evalOutput : public MatrixOutput
         if (data.data) {
             const QString csv = QString(file.name).replace(".eval", ".csv");
-            if ((Globals->crossValidate == 0) || (!crossValidate)) {
+            if ((Globals->crossValidate <= 0) || (!crossValidate)) {
                 Evaluate(data, targetFiles, queryFiles, csv);
             } else {
                 QFutureSynchronizer<float> futures;