Merge pull request #148 from biometrics/score_level_fusion

Score level fusion support

Merge pull request #148 from biometrics/score_level_fusion
Score level fusion support
sklum
2 parents 9deea8e7 f95f259e
Showing 5 changed files with 107 additions and 20 deletions
openbr/core/core.cpp
openbr/core/fuse.cpp
openbr/openbr_plugin.h
openbr/plugins/distance.cpp
openbr/plugins/quality.cpp
@@ -56,11 +56,6 @@ struct AlgorithmCore
         TemplateList data(TemplateList::fromGallery(input));
-        // set the Train bool metadata, in case a Transform's project
-        // needs to know if it's called during train or enroll
-        for (int i=0; i<data.size(); i++)
-            data[i].file.set("Train", true);
-
         if (transform.isNull()) qFatal("Null transform.");
         qDebug("%d Training Files", data.size());
@@ -463,7 +458,7 @@ void br::Convert(const File &amp;fileType, const File &amp;inputFile, const File &amp;output
         if ((targetFiles.size() != m.cols || queryFiles.size() != m.rows)
             && (m.cols != 1 || targetFiles.size() != m.rows || queryFiles.size() != m.rows))
-            qFatal("Similarity matrix and file size mismatch.");
+            qFatal("Similarity matrix (%d, %d) and header (%d, %d) size mismatch.", m.rows, m.cols, queryFiles.size(), targetFiles.size());
         QSharedPointer<Output> o(Factory<Output>::make(outputFile));
         o->initialize(targetFiles, queryFiles);
@@ -30,6 +30,9 @@ using namespace cv;
 static void normalizeMatrix(Mat &matrix, const Mat &mask, const QString &method)
 {
+    if (matrix.rows != mask.rows && matrix.cols != mask.cols)
+        qFatal("Similarity matrix (%d, %d) and mask (%d, %d) size mismatch.", matrix.rows, matrix.cols, mask.rows, mask.cols);
+
     if (method == "None") return;
     QList<float> vals; vals.reserve(matrix.rows*matrix.cols);
@@ -483,7 +483,7 @@ struct TemplateList : public QList&lt;Template&gt;
     }
     /*!
-     * \brief Returns a #br::TemplateList containing templates with one matrix at the specified index \em index.
+     * \brief Returns a list of #br::TemplateList with each #br::Template in a given #br::TemplateList containing the number of matrices specified by \em partitionSizes.
      */
     QList<TemplateList> partition(const QList<int> &partitionSizes) const
     {
@@ -179,58 +179,88 @@ BR_REGISTER(Distance, PipeDistance)
 /*!
  * \ingroup distances
- * \brief Computes an operation on distances across multiple matrices of compared templates
+ * \brief Fuses similarity scores across multiple matrices of compared templates
  * \author Scott Klum \cite sklum
  * \note Operation: Mean, sum, min, max are supported.
  */
-class OperationDistance : public Distance
+class FuseDistance : public Distance
 {
     Q_OBJECT
     Q_ENUMS(Operation)
-    Q_PROPERTY(br::Distance* distance READ get_distance WRITE set_distance RESET reset_distance STORED false)
+    Q_PROPERTY(QString description READ get_description WRITE set_description RESET reset_description STORED false)
     Q_PROPERTY(Operation operation READ get_operation WRITE set_operation RESET reset_operation STORED false)
+    QList<br::Distance*> distances;
+
 public:
     /*!< */
     enum Operation {Mean, Sum, Max, Min};
 private:
-    BR_PROPERTY(br::Distance*, distance, make("Dist(L2)"))
+    BR_PROPERTY(QString, description, "IdenticalDistance")
     BR_PROPERTY(Operation, operation, Mean)
     void train(const TemplateList &src)
     {
-        distance->train(src);
+        // Partition the templates by matrix
+        QList<int> split;
+        for (int i=0; i<src.at(0).size(); i++) split.append(1);
+
+        QList<TemplateList> partitionedSrc = src.partition(split);
+
+        while (distances.size() < partitionedSrc.size())
+            distances.append(make(description));
+
+        // Train on each of the partitions
+        for (int i=0; i<distances.size(); i++)
+            distances[i]->train(partitionedSrc[i]);
     }
     float compare(const Template &a, const Template &b) const
     {
         if (a.size() != b.size()) qFatal("Comparison size mismatch");
-        QList<float> distances;
-        for (int i = 0; i < a.size(); i++)
-            distances.append(distance->compare(a[i],b[i]));
+        QList<float> scores;
+        for (int i=0; i<distances.size(); i++)
+            scores.append(distances[i]->compare(Template(a.file, a[i]),Template(b.file, b[i])));
         switch (operation) {
           case Mean:
-            return std::accumulate(distances.begin(),distances.end(),0.0)/(float)distances.size();
+            return std::accumulate(scores.begin(),scores.end(),0.0)/(float)scores.size();
             break;
           case Sum:
-            return std::accumulate(distances.begin(),distances.end(),0.0);
+            return std::accumulate(scores.begin(),scores.end(),0.0);
             break;
           case Min:
-            return *std::min_element(distances.begin(),distances.end());
+            return *std::min_element(scores.begin(),scores.end());
             break;
           case Max:
-            return *std::max_element(distances.begin(),distances.end());
+            return *std::max_element(scores.begin(),scores.end());
             break;
           default:
             qFatal("Invalid operation.");
         }
     }
+
+    void store(QDataStream &stream) const
+    {
+        stream << distances.size();
+        foreach (Distance *distance, distances)
+            distance->store(stream);
+    }
+
+    void load(QDataStream &stream)
+    {
+        int numDistances;
+        stream >> numDistances;
+        while (distances.size() < numDistances)
+            distances.append(make(description));
+        foreach (Distance *distance, distances)
+            distance->load(stream);
+    }
 };
-BR_REGISTER(Distance, OperationDistance)
+BR_REGISTER(Distance, FuseDistance)
 /*!
  * \ingroup distances
@@ -211,6 +211,65 @@ protected:
 BR_REGISTER(Distance, MatchProbabilityDistance)
+class ZScoreDistance : public Distance
+{
+    Q_OBJECT
+    Q_PROPERTY(br::Distance* distance READ get_distance WRITE set_distance RESET reset_distance STORED false)
+    Q_PROPERTY(bool crossModality READ get_crossModality WRITE set_crossModality RESET reset_crossModality STORED false)
+    BR_PROPERTY(br::Distance*, distance, make("Dist(L2)"))
+    BR_PROPERTY(bool, crossModality, false)
+
+    float min, max;
+    double mean, stddev;
+
+    void train(const TemplateList &src)
+    {
+        distance->train(src);
+
+        QScopedPointer<MatrixOutput> matrixOutput(MatrixOutput::make(FileList(src.size()), FileList(src.size())));
+        distance->compare(src, src, matrixOutput.data());
+
+        QList<float> scores;
+        scores.reserve(src.size()*src.size());
+        for (int i=0; i<src.size(); i++) {
+            for (int j=0; j<i; j++) {
+                const float score = matrixOutput.data()->data.at<float>(i, j);
+                if (score == -std::numeric_limits<float>::max()) continue;
+                if (crossModality && src[i].file.get<QString>("MODALITY") == src[j].file.get<QString>("MODALITY")) continue;
+                scores.append(score);
+            }
+        }
+
+        Common::MinMax(scores, &min, &max);
+        Common::MeanStdDev(scores, &mean, &stddev);
+
+        if (stddev == 0) qFatal("Stddev is 0.");
+    }
+
+    float compare(const Template &target, const Template &query) const
+    {
+        float score = distance->compare(target,query);
+        if      (score == -std::numeric_limits<float>::max()) score = (min - mean) / stddev;
+        else if (score ==  std::numeric_limits<float>::max()) score = (max - mean) / stddev;
+        else                                                  score = (score - mean) / stddev;
+        return score;
+    }
+
+    void store(QDataStream &stream) const
+    {
+        distance->store(stream);
+        stream << min << max << mean << stddev;
+    }
+
+    void load(QDataStream &stream)
+    {
+        distance->load(stream);
+        stream >> min >> max >> mean >> stddev;
+    }
+};
+
+BR_REGISTER(Distance, ZScoreDistance)
+
 /*!
  * \ingroup distances
  * \brief Match Probability modification for heat maps \cite klare12