add -plotKNN

bhklein
1 parent 9633db66
Showing 8 changed files with 81 additions and 22 deletions
app/br/br.cpp
openbr/core/eval.cpp
openbr/core/eval.h
openbr/core/plot.cpp
openbr/core/plot.h
openbr/openbr.cpp
openbr/openbr.h
share/openbr/plotting/plot_utils.R
@@ -169,8 +169,8 @@ public:
                 check(parc >= 2 && parc <= 4, "Incorrect parameter count for 'evalRegression'.");
                 br_eval_regression(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? parv[3] : "");
             } else if (!strcmp(fun, "evalKNN")) {
-                check(parc == 3, "Incorrect parameter count for 'evalKNN'.");
-                br_eval_knn(parv[0], parv[1], parv[2]);
+                check(parc >= 2 && parc <= 3, "Incorrect parameter count for 'evalKNN'.");
+                br_eval_knn(parv[0], parv[1], parc > 2 ? parv[2] : "");
             } else if (!strcmp(fun, "pairwiseCompare")) {
                 check((parc >= 2) && (parc <= 3), "Incorrect parameter count for 'pairwiseCompare'.");
                 br_pairwise_compare(parv[0], parv[1], parc == 3 ? parv[2] : "");
@@ -186,6 +186,9 @@ public:
             } else if (!strcmp(fun, "plotMetadata")) {
                 check(parc >= 2, "Incorrect parameter count for 'plotMetadata'.");
                 br_plot_metadata(parc-1, parv, parv[parc-1], true);
+            } else if (!strcmp(fun, "plotKNN")) {
+                check(parc >=2, "Incorrect parameter count for 'plotKNN'.");
+                br_plot_knn(parc-1, parv, parv[parc-1], true);
             } else if (!strcmp(fun, "project")) {
                 check(parc == 2, "Insufficient parameter count for 'project'.");
                 br_project(parv[0], parv[1]);
@@ -279,13 +282,14 @@ private:
                "-evalDetection <predicted_gallery> <truth_gallery> [{csv}] [{normalize}] [{minSize}] [{maxSize}]\n"
                "-evalLandmarking <predicted_gallery> <truth_gallery> [{csv} [<normalization_index_a> <normalization_index_b>] [sample_index] [total_examples]]\n"
                "-evalRegression <predicted_gallery> <truth_gallery> <predicted property name> <ground truth property name>\n"
-               "-evalKNN <knn_graph> <knn_truth> [{iet_file}]\n"
+               "-evalKNN <knn_graph> <knn_truth> [{csv}]\n"
                "-pairwiseCompare <target_gallery> <query_gallery> [{output}]\n"
                "-inplaceEval <simmat> <target> <query> [{csv}]\n"
                "-assertEval <simmat> <mask> <accuracy>\n"
                "-plotDetection <file> ... <file> {destination}\n"
                "-plotLandmarking <file> ... <file> {destination}\n"
                "-plotMetadata <file> ... <file> <columns>\n"
+               "-plotKNN <file> ... <file> {destination}\n"
                "-project <input_gallery> {output_gallery}\n"
                "-deduplicate <input_gallery> <output_gallery> <threshold>\n"
                "-likely <input_type> <output_type> <output_likely_source>\n"
@@ -1336,7 +1336,7 @@ void readKNNTruth(size_t probeCount, QVector&lt; QList&lt;size_t&gt; &gt; &amp;groundTruth, cons
         qFatal("Invalid ground truth file!");
 }
  
-void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &iet)
+void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &csv)
 {
     qDebug("Evaluating k-NN of %s against %s", qPrintable(knnGraph), qPrintable(knnTruth));
  
@@ -1406,19 +1406,14 @@ void EvalKNN(const QString &amp;knnGraph, const QString &amp;knnTruth, const QString &amp;ie
     if (numUnmatedSearches == 0)
         qFatal("No unmated searches!");
  
-    printf("Rank-%i Return Rate: %g\n", 1, getCMC(firstGenuineReturns, 1, numMatedSearches));
+
+    qDebug("Rank-%d Return Rate: %.3f", 1, getCMC(firstGenuineReturns, 1, numMatedSearches));
     if (k >=5)
-        printf("Rank-%i Return Rate: %g\n", 5, getCMC(firstGenuineReturns, 5, numMatedSearches));
+        qDebug("Rank-%d Return Rate: %.3f", 5, getCMC(firstGenuineReturns, 5, numMatedSearches));
     if (k >=10)
-        printf("Rank-%i Return Rate: %g\n", 10, getCMC(firstGenuineReturns, 10, numMatedSearches));
-
-    printf("Rank-%zu Return Rate: %g\n", k, double(numMatedSimilarities) / double(numMatedSearches));
+        qDebug("Rank-%d Return Rate: %.3f", 10, getCMC(firstGenuineReturns, 10, numMatedSearches));
  
-    // Open the output file
-    QFile ietFile(iet);
-    if (!ietFile.open(QFile::WriteOnly | QFile::Text))
-        qFatal("Failed to open IET file for writing!");
-    ietFile.write("Threshold,FPIR,FNIR\n");
+    qDebug("Rank-%zu Return Rate: %.3f", k, double(numMatedSimilarities) / double(numMatedSearches));
  
     /*
      * Iterate through the similarity scores highest-to-lowest,
@@ -1447,10 +1442,28 @@ void EvalKNN(const QString &amp;knnGraph, const QString &amp;knnTruth, const QString &amp;ie
         }
     }
  
-    foreach(const OperatingPoint &operatingPoint, operatingPoints)
-        ietFile.write(qPrintable(QString::number(operatingPoint.score) + "," +
-                                 QString::number(operatingPoint.FAR) + "," +
-                                 QString::number(operatingPoint.TAR) + "\n"));
+    if (!csv.isEmpty()) {
+        // Open the output file
+        QFile ietFile(csv);
+        if (!ietFile.open(QFile::WriteOnly | QFile::Text))
+            qFatal("Failed to open IET file for writing!");
+        ietFile.write("Plot,X,Y,Z\n");
+        // Write CMC
+        const int Max_Retrieval = min(200, (int)k);
+        for (int i=1; i<=Max_Retrieval; i++) {
+            const float retrievalRate = getCMC(firstGenuineReturns, i, numMatedSearches);
+            ietFile.write(qPrintable(QString("CMC,%1,%2,0\n").arg(QString::number(i), QString::number(retrievalRate))));
+        }
+
+        foreach(const OperatingPoint &operatingPoint, operatingPoints)
+            ietFile.write(qPrintable("IET," +
+                                     QString::number(operatingPoint.FAR) + "," +
+                                     QString::number(operatingPoint.TAR) + "," +
+                                     QString::number(operatingPoint.score) + "\n"));
+    }
+
+    qDebug("FNIR @ FPIR = 0.1:   %.3f", 1-getOperatingPointGivenFAR(operatingPoints, 0.1).TAR);
+    qDebug("FNIR @ FPIR = 0.01:  %.3f", 1-getOperatingPointGivenFAR(operatingPoints, 0.01).TAR);
 }
  
 } // namespace br
@@ -33,7 +33,7 @@ namespace br
     float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", bool normalize = false, int minSize = 0, int maxSize = 0); // Return average overlap
     float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", int normalizationIndexA = 0, int normalizationIndexB = 1, int sampleIndex = 0, int totalExamples = 5); // Return average error
     void EvalRegression(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = "");
-    void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &iet);
+    void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &csv = "");
  
     struct Candidate
     {
@@ -346,4 +346,32 @@ bool PlotMetadata(const QStringList &amp;files, const QString &amp;columns, bool show)
     return p.finalize(show);
 }
  
+bool PlotKNN(const QStringList &files, const File &destination, bool show)
+{
+    qDebug("Plotting %d k-NN file(s) to %s", files.size(), qPrintable(destination));
+    RPlot p(files, destination);
+    p.file.write("\nformatData(type=\"knn\")\n\n");
+
+    QMap<QString,File> optMap;
+    optMap.insert("rocOptions", File(QString("[xTitle=False Positive Identification Rate (FPIR),yTitle=True Positive Identification Rate (TPIR),xLog=true,yLog=false]")));
+    optMap.insert("ietOptions", File(QString("[xTitle=False Positive Identification Rate (FPIR),yTitle=False Negative Identification Rate (FNIR),xLog=true,yLog=true]")));
+    optMap.insert("cmcOptions", File(QString("[xTitle=Rank,yTitle=Retrieval Rate,xLog=true,yLog=false,size=1,xLabels=(1,5,10,50,100),xBreaks=(1,5,10,50,100)]")));
+
+    foreach (const QString &key, optMap.keys()) {
+        const QStringList options = destination.get<QStringList>(key, QStringList());
+        foreach (const QString &option, options) {
+            QStringList words = QtUtils::parse(option, '=');
+            QtUtils::checkArgsSize(words[0], words, 1, 2);
+            optMap[key].set(words[0], words[1]);
+        }
+    }
+
+    QString plot = "plot <- plotLine(lineData=%1, options=list(%2), flipY=%3)\nplot\n";
+    p.file.write(qPrintable(QString(plot).arg("IET", toRList(optMap["rocOptions"]), "TRUE")));
+    p.file.write(qPrintable(QString(plot).arg("IET", toRList(optMap["ietOptions"]), "FALSE")));
+    p.file.write(qPrintable(QString(plot).arg("CMC", toRList(optMap["cmcOptions"]), "FALSE")));
+
+    return p.finalize(show);
+}
+
 } // namespace br
@@ -28,6 +28,7 @@ namespace br
     bool PlotDetection(const QStringList &files, const File &destination, bool show = false);
     bool PlotLandmarking(const QStringList &files, const File &destination, bool show = false);
     bool PlotMetadata(const QStringList &files, const QString &destination, bool show = false);
+    bool PlotKNN(const QStringList &files, const File &destination, bool show = false);
 }
  
 #endif // BR_PLOT_H
@@ -145,9 +145,9 @@ void br_eval_regression(const char *predicted_gallery, const char *truth_gallery
     EvalRegression(predicted_gallery, truth_gallery, predicted_property, truth_property);
 }
  
-void br_eval_knn(const char *knnGraph, const char *knnTruth, const char *iet)
+void br_eval_knn(const char *knnGraph, const char *knnTruth, const char *csv)
 {
-    EvalKNN(knnGraph, knnTruth, iet);
+    EvalKNN(knnGraph, knnTruth, csv);
 }
  
 void br_finalize()
@@ -221,6 +221,11 @@ bool br_plot_metadata(int num_files, const char *files[], const char *columns, b
     return PlotMetadata(QtUtils::toStringList(num_files, files), columns, show);
 }
  
+bool br_plot_knn(int num_files, const char *files[], const char *destination, bool show)
+{
+    return PlotKNN(QtUtils::toStringList(num_files, files), destination, show);
+}
+
 float br_progress()
 {
     return Globals->progress();
@@ -64,7 +64,7 @@ BR_EXPORT float br_eval_landmarking(const char *predicted_gallery, const char *t
  
 BR_EXPORT void br_eval_regression(const char *predicted_gallery, const char *truth_gallery, const char *predicted_property = "", const char *truth_property = "");
  
-BR_EXPORT void br_eval_knn(const char *knnGraph, const char *knnTruth, const char *iet);
+BR_EXPORT void br_eval_knn(const char *knnGraph, const char *knnTruth, const char *csv = "");
  
 BR_EXPORT void br_finalize();
  
@@ -93,6 +93,8 @@ BR_EXPORT bool br_plot_landmarking(int num_files, const char *files[], const cha
  
 BR_EXPORT bool br_plot_metadata(int num_files, const char *files[], const char *columns, bool show = false);
  
+BR_EXPORT bool br_plot_knn(int num_files, const char *files[], const char *destination, bool show = false);
+
 BR_EXPORT float br_progress();
  
 BR_EXPORT void br_read_pipe(const char *pipe, int *argc, char ***argv);
@@ -217,6 +217,12 @@ formatData &lt;- function(type=&quot;eval&quot;) {
         NormLength <<- data[grep("NormLength",data$Plot),-c(1)]
         sample <<- readImageData(Sample)
         rows <<- sample[[1]]$value
+    } else if (type == "knn") {
+        # Split data into individual plots
+        IET <<- data[grep("IET",data$Plot),-c(1)]
+        IET$Y <<- as.numeric(as.character(IET$Y))
+        CMC <<- data[grep("CMC",data$Plot),-c(1)]
+        CMC$Y <<- as.numeric(as.character(CMC$Y))
     }
 }