introduced evalOutput

Josh Klontz
1 parent edbb16c9
Showing 5 changed files with 56 additions and 26 deletions
openbr/core/bee.cpp
openbr/core/bee.h
openbr/core/plot.cpp
openbr/core/plot.h
openbr/plugins/output.cpp
@@ -234,22 +234,26 @@ void BEE::writeMask(const Mat &amp;m, const QString &amp;mask, const QString &amp;targetSigs
 void BEE::makeMask(const QString &targetInput, const QString &queryInput, const QString &mask)
 {
     qDebug("Making mask from %s and %s to %s", qPrintable(targetInput), qPrintable(queryInput), qPrintable(mask));
+    FileList targes = TemplateList::fromGallery(targetInput).files();
+    FileList queries = (queryInput == ".") ? targes : TemplateList::fromGallery(queryInput).files();
+    writeMask(makeMask(targes, queries), mask, targetInput, queryInput);
+}
-    FileList targetFiles = TemplateList::fromGallery(targetInput).files();
-    FileList queryFiles = (queryInput == ".") ? targetFiles : TemplateList::fromGallery(queryInput).files();
-    QList<float> targetLabels = targetFiles.labels();
-    QList<float> queryLabels = queryFiles.labels();
-    QList<int> targetPartitions = targetFiles.crossValidationPartitions();
-    QList<int> queryPartitions = queryFiles.crossValidationPartitions();
-
-    Mat vals(queryFiles.size(), targetFiles.size(), CV_8UC1);
-    for (int i=0; i<queryFiles.size(); i++) {
-        const QString &fileA = queryFiles[i];
+cv::Mat BEE::makeMask(const br::FileList &targets, const br::FileList &queries)
+{
+    QList<float> targetLabels = targets.labels();
+    QList<float> queryLabels = queries.labels();
+    QList<int> targetPartitions = targets.crossValidationPartitions();
+    QList<int> queryPartitions = queries.crossValidationPartitions();
+
+    Mat mask(queries.size(), targets.size(), CV_8UC1);
+    for (int i=0; i<queries.size(); i++) {
+        const QString &fileA = queries[i];
         const int labelA = queryLabels[i];
         const int partitionA = queryPartitions[i];
-        for (int j=0; j<targetFiles.size(); j++) {
-            const QString &fileB = targetFiles[j];
+        for (int j=0; j<targets.size(); j++) {
+            const QString &fileB = targets[j];
             const int labelB = targetLabels[j];
             const int partitionB = targetPartitions[j];
@@ -260,10 +264,11 @@ void BEE::makeMask(const QString &amp;targetInput, const QString &amp;queryInput, const 
             else if (partitionA != partitionB) val = DontCare;
             else if (labelA == labelB)         val = Match;
             else                               val = NonMatch;
-            vals.at<Mask_t>(i,j) = val;
+            mask.at<Mask_t>(i,j) = val;
         }
     }
-    writeMask(vals, mask, targetInput, queryInput);
+
+    return mask;
 }
 void BEE::combineMasks(const QStringList &inputMasks, const QString &outputMask, const QString &method)
@@ -46,6 +46,7 @@ namespace BEE
     // Write BEE files
     void makeMask(const QString &targetInput, const QString &queryInput, const QString &mask);
+    cv::Mat makeMask(const br::FileList &targets, const br::FileList &queries);
     void combineMasks(const QStringList &inputMasks, const QString &outputMask, const QString &method);
 }
@@ -113,9 +113,6 @@ float Evaluate(const QString &amp;simmat, const QString &amp;mask, const QString &amp;csv)
 {
     qDebug("Evaluating %s with %s", qPrintable(simmat), qPrintable(mask));
-    const int Max_Points = 500;
-    float result = -1;
-
     // Read files
     const Mat scores = BEE::readSimmat(simmat);
     File maskFile(mask);
@@ -124,13 +121,21 @@ float Evaluate(const QString &amp;simmat, const QString &amp;mask, const QString &amp;csv)
     const Mat masks = BEE::readMask(maskFile);
     if (scores.size() != masks.size()) qFatal("Simmat/Mask size mismatch.");
+    return Evaluate(scores, masks, csv);
+}
+
+float Evaluate(const Mat &simmat, const Mat &mask, const QString &csv)
+{
+    const int Max_Points = 500;
+    float result = -1;
+
     // Make comparisons
-    QList<Comparison> comparisons; comparisons.reserve(scores.rows*scores.cols);
+    QList<Comparison> comparisons; comparisons.reserve(simmat.rows*simmat.cols);
     int genuineCount = 0, impostorCount = 0, numNaNs = 0;
-    for (int i=0; i<scores.rows; i++) {
-        for (int j=0; j<scores.cols; j++) {
-            const BEE::Mask_t mask_val = masks.at<BEE::Mask_t>(i,j);
-            const BEE::Simmat_t simmat_val = scores.at<BEE::Simmat_t>(i,j);
+    for (int i=0; i<simmat.rows; i++) {
+        for (int j=0; j<simmat.cols; j++) {
+            const BEE::Mask_t mask_val = mask.at<BEE::Mask_t>(i,j);
+            const BEE::Simmat_t simmat_val = simmat.at<BEE::Simmat_t>(i,j);
             if (mask_val == BEE::DontCare) continue;
             if (simmat_val != simmat_val) { numNaNs++; continue; }
             comparisons.append(Comparison(simmat_val, j, i, mask_val == BEE::Match));
@@ -149,7 +154,7 @@ float Evaluate(const QString &amp;simmat, const QString &amp;mask, const QString &amp;csv)
     double genuineSum = 0, impostorSum = 0;
     QList<OperatingPoint> operatingPoints;
     QList<float> genuines, impostors;
-    QVector<int> firstGenuineReturns(scores.rows, 0);
+    QVector<int> firstGenuineReturns(simmat.rows, 0);
     int falsePositives = 0, previousFalsePositives = 0;
     int truePositives = 0, previousTruePositives = 0;
@@ -202,11 +207,11 @@ float Evaluate(const QString &amp;simmat, const QString &amp;mask, const QString &amp;csv)
     // Write Metadata table
     QStringList lines;
     lines.append("Plot,X,Y");
-    lines.append("Metadata,"+QString::number(scores.cols)+",Gallery");
-    lines.append("Metadata,"+QString::number(scores.rows)+",Probe");
+    lines.append("Metadata,"+QString::number(simmat.cols)+",Gallery");
+    lines.append("Metadata,"+QString::number(simmat.rows)+",Probe");
     lines.append("Metadata,"+QString::number(genuineCount)+",Genuine");
     lines.append("Metadata,"+QString::number(impostorCount)+",Impostor");
-    lines.append("Metadata,"+QString::number(scores.cols*scores.rows-(genuineCount+impostorCount))+",Ignored");
+    lines.append("Metadata,"+QString::number(simmat.cols*simmat.rows-(genuineCount+impostorCount))+",Ignored");
     // Write Detection Error Tradeoff (DET), PRE, REC
     int points = qMin(operatingPoints.size(), Max_Points);
@@ -27,6 +27,7 @@ namespace br
 void Confusion(const QString &file, float score, int &true_positives, int &false_positives, int &true_negatives, int &false_negatives);
 float Evaluate(const QString &simmat, const QString &mask, const QString &csv = ""); // Returns TAR @ FAR = 0.01
+float Evaluate(const cv::Mat &scores, const cv::Mat &masks, const QString &csv = "");
 bool Plot(const QStringList &files, const br::File &destination, bool show = false);
 bool PlotMetadata(const QStringList &files, const QString &destination, bool show = false);
@@ -40,6 +40,7 @@
 #include "openbr/core/bee.h"
 #include "openbr/core/common.h"
 #include "openbr/core/opencvutils.h"
+#include "openbr/core/plot.h"
 #include "openbr/core/qtutils.h"
 namespace br
@@ -259,6 +260,23 @@ BR_REGISTER(Output, EmptyOutput)
 /*!
  * \ingroup outputs
+ * \brief Evaluate the output matrix.
+ * \author Josh Klontz \cite jklontz
+ */
+class evalOutput : public MatrixOutput
+{
+    Q_OBJECT
+
+    ~evalOutput()
+    {
+        Evaluate(data, BEE::makeMask(targetFiles, queryFiles), "");
+    }
+};
+
+BR_REGISTER(Output, evalOutput)
+
+/*!
+ * \ingroup outputs
  * \brief Outputs highest ranked matches with scores.
  * \author Scott Klum \cite sklum
  */