diff --git a/app/br/br.cpp b/app/br/br.cpp index 06a3589..89b96be 100644 --- a/app/br/br.cpp +++ b/app/br/br.cpp @@ -169,8 +169,8 @@ public: check(parc >= 2 && parc <= 4, "Incorrect parameter count for 'evalRegression'."); br_eval_regression(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? parv[3] : ""); } else if (!strcmp(fun, "evalKNN")) { - check(parc == 3, "Incorrect parameter count for 'evalKNN'."); - br_eval_knn(parv[0], parv[1], parv[2]); + check(parc >= 2 && parc <= 3, "Incorrect parameter count for 'evalKNN'."); + br_eval_knn(parv[0], parv[1], parc > 2 ? parv[2] : ""); } else if (!strcmp(fun, "pairwiseCompare")) { check((parc >= 2) && (parc <= 3), "Incorrect parameter count for 'pairwiseCompare'."); br_pairwise_compare(parv[0], parv[1], parc == 3 ? parv[2] : ""); @@ -186,6 +186,9 @@ public: } else if (!strcmp(fun, "plotMetadata")) { check(parc >= 2, "Incorrect parameter count for 'plotMetadata'."); br_plot_metadata(parc-1, parv, parv[parc-1], true); + } else if (!strcmp(fun, "plotKNN")) { + check(parc >=2, "Incorrect parameter count for 'plotKNN'."); + br_plot_knn(parc-1, parv, parv[parc-1], true); } else if (!strcmp(fun, "project")) { check(parc == 2, "Insufficient parameter count for 'project'."); br_project(parv[0], parv[1]); @@ -279,13 +282,14 @@ private: "-evalDetection [{csv}] [{normalize}] [{minSize}] [{maxSize}]\n" "-evalLandmarking [{csv} [ ] [sample_index] [total_examples]]\n" "-evalRegression \n" - "-evalKNN [{iet_file}]\n" + "-evalKNN [{csv}]\n" "-pairwiseCompare [{output}]\n" "-inplaceEval [{csv}]\n" "-assertEval \n" "-plotDetection ... {destination}\n" "-plotLandmarking ... {destination}\n" "-plotMetadata ... \n" + "-plotKNN ... {destination}\n" "-project {output_gallery}\n" "-deduplicate \n" "-likely \n" diff --git a/openbr/core/eval.cpp b/openbr/core/eval.cpp index e940aff..3b19938 100755 --- a/openbr/core/eval.cpp +++ b/openbr/core/eval.cpp @@ -1336,7 +1336,7 @@ void readKNNTruth(size_t probeCount, QVector< QList > &groundTruth, cons qFatal("Invalid ground truth file!"); } -void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &iet) +void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &csv) { qDebug("Evaluating k-NN of %s against %s", qPrintable(knnGraph), qPrintable(knnTruth)); @@ -1406,19 +1406,14 @@ void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &ie if (numUnmatedSearches == 0) qFatal("No unmated searches!"); - printf("Rank-%i Return Rate: %g\n", 1, getCMC(firstGenuineReturns, 1, numMatedSearches)); + + qDebug("Rank-%d Return Rate: %.3f", 1, getCMC(firstGenuineReturns, 1, numMatedSearches)); if (k >=5) - printf("Rank-%i Return Rate: %g\n", 5, getCMC(firstGenuineReturns, 5, numMatedSearches)); + qDebug("Rank-%d Return Rate: %.3f", 5, getCMC(firstGenuineReturns, 5, numMatedSearches)); if (k >=10) - printf("Rank-%i Return Rate: %g\n", 10, getCMC(firstGenuineReturns, 10, numMatedSearches)); - - printf("Rank-%zu Return Rate: %g\n", k, double(numMatedSimilarities) / double(numMatedSearches)); + qDebug("Rank-%d Return Rate: %.3f", 10, getCMC(firstGenuineReturns, 10, numMatedSearches)); - // Open the output file - QFile ietFile(iet); - if (!ietFile.open(QFile::WriteOnly | QFile::Text)) - qFatal("Failed to open IET file for writing!"); - ietFile.write("Threshold,FPIR,FNIR\n"); + qDebug("Rank-%zu Return Rate: %.3f", k, double(numMatedSimilarities) / double(numMatedSearches)); /* * Iterate through the similarity scores highest-to-lowest, @@ -1447,10 +1442,28 @@ void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &ie } } - foreach(const OperatingPoint &operatingPoint, operatingPoints) - ietFile.write(qPrintable(QString::number(operatingPoint.score) + "," + - QString::number(operatingPoint.FAR) + "," + - QString::number(operatingPoint.TAR) + "\n")); + if (!csv.isEmpty()) { + // Open the output file + QFile ietFile(csv); + if (!ietFile.open(QFile::WriteOnly | QFile::Text)) + qFatal("Failed to open IET file for writing!"); + ietFile.write("Plot,X,Y,Z\n"); + // Write CMC + const int Max_Retrieval = min(200, (int)k); + for (int i=1; i<=Max_Retrieval; i++) { + const float retrievalRate = getCMC(firstGenuineReturns, i, numMatedSearches); + ietFile.write(qPrintable(QString("CMC,%1,%2,0\n").arg(QString::number(i), QString::number(retrievalRate)))); + } + + foreach(const OperatingPoint &operatingPoint, operatingPoints) + ietFile.write(qPrintable("IET," + + QString::number(operatingPoint.FAR) + "," + + QString::number(operatingPoint.TAR) + "," + + QString::number(operatingPoint.score) + "\n")); + } + + qDebug("FNIR @ FPIR = 0.1: %.3f", 1-getOperatingPointGivenFAR(operatingPoints, 0.1).TAR); + qDebug("FNIR @ FPIR = 0.01: %.3f", 1-getOperatingPointGivenFAR(operatingPoints, 0.01).TAR); } } // namespace br diff --git a/openbr/core/eval.h b/openbr/core/eval.h index d09ce3d..0edfd25 100644 --- a/openbr/core/eval.h +++ b/openbr/core/eval.h @@ -33,7 +33,7 @@ namespace br float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", bool normalize = false, int minSize = 0, int maxSize = 0); // Return average overlap float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", int normalizationIndexA = 0, int normalizationIndexB = 1, int sampleIndex = 0, int totalExamples = 5); // Return average error void EvalRegression(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = ""); - void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &iet); + void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &csv = ""); struct Candidate { diff --git a/openbr/core/plot.cpp b/openbr/core/plot.cpp index 94f84dd..bff51d1 100644 --- a/openbr/core/plot.cpp +++ b/openbr/core/plot.cpp @@ -346,4 +346,32 @@ bool PlotMetadata(const QStringList &files, const QString &columns, bool show) return p.finalize(show); } +bool PlotKNN(const QStringList &files, const File &destination, bool show) +{ + qDebug("Plotting %d k-NN file(s) to %s", files.size(), qPrintable(destination)); + RPlot p(files, destination); + p.file.write("\nformatData(type=\"knn\")\n\n"); + + QMap optMap; + optMap.insert("rocOptions", File(QString("[xTitle=False Positive Identification Rate (FPIR),yTitle=True Positive Identification Rate (TPIR),xLog=true,yLog=false]"))); + optMap.insert("ietOptions", File(QString("[xTitle=False Positive Identification Rate (FPIR),yTitle=False Negative Identification Rate (FNIR),xLog=true,yLog=true]"))); + optMap.insert("cmcOptions", File(QString("[xTitle=Rank,yTitle=Retrieval Rate,xLog=true,yLog=false,size=1,xLabels=(1,5,10,50,100),xBreaks=(1,5,10,50,100)]"))); + + foreach (const QString &key, optMap.keys()) { + const QStringList options = destination.get(key, QStringList()); + foreach (const QString &option, options) { + QStringList words = QtUtils::parse(option, '='); + QtUtils::checkArgsSize(words[0], words, 1, 2); + optMap[key].set(words[0], words[1]); + } + } + + QString plot = "plot <- plotLine(lineData=%1, options=list(%2), flipY=%3)\nplot\n"; + p.file.write(qPrintable(QString(plot).arg("IET", toRList(optMap["rocOptions"]), "TRUE"))); + p.file.write(qPrintable(QString(plot).arg("IET", toRList(optMap["ietOptions"]), "FALSE"))); + p.file.write(qPrintable(QString(plot).arg("CMC", toRList(optMap["cmcOptions"]), "FALSE"))); + + return p.finalize(show); +} + } // namespace br diff --git a/openbr/core/plot.h b/openbr/core/plot.h index 223f746..26db428 100644 --- a/openbr/core/plot.h +++ b/openbr/core/plot.h @@ -28,6 +28,7 @@ namespace br bool PlotDetection(const QStringList &files, const File &destination, bool show = false); bool PlotLandmarking(const QStringList &files, const File &destination, bool show = false); bool PlotMetadata(const QStringList &files, const QString &destination, bool show = false); + bool PlotKNN(const QStringList &files, const File &destination, bool show = false); } #endif // BR_PLOT_H diff --git a/openbr/openbr.cpp b/openbr/openbr.cpp index cf69ecc..e11acf4 100644 --- a/openbr/openbr.cpp +++ b/openbr/openbr.cpp @@ -145,9 +145,9 @@ void br_eval_regression(const char *predicted_gallery, const char *truth_gallery EvalRegression(predicted_gallery, truth_gallery, predicted_property, truth_property); } -void br_eval_knn(const char *knnGraph, const char *knnTruth, const char *iet) +void br_eval_knn(const char *knnGraph, const char *knnTruth, const char *csv) { - EvalKNN(knnGraph, knnTruth, iet); + EvalKNN(knnGraph, knnTruth, csv); } void br_finalize() @@ -221,6 +221,11 @@ bool br_plot_metadata(int num_files, const char *files[], const char *columns, b return PlotMetadata(QtUtils::toStringList(num_files, files), columns, show); } +bool br_plot_knn(int num_files, const char *files[], const char *destination, bool show) +{ + return PlotKNN(QtUtils::toStringList(num_files, files), destination, show); +} + float br_progress() { return Globals->progress(); diff --git a/openbr/openbr.h b/openbr/openbr.h index dd06591..12a769a 100644 --- a/openbr/openbr.h +++ b/openbr/openbr.h @@ -64,7 +64,7 @@ BR_EXPORT float br_eval_landmarking(const char *predicted_gallery, const char *t BR_EXPORT void br_eval_regression(const char *predicted_gallery, const char *truth_gallery, const char *predicted_property = "", const char *truth_property = ""); -BR_EXPORT void br_eval_knn(const char *knnGraph, const char *knnTruth, const char *iet); +BR_EXPORT void br_eval_knn(const char *knnGraph, const char *knnTruth, const char *csv = ""); BR_EXPORT void br_finalize(); @@ -93,6 +93,8 @@ BR_EXPORT bool br_plot_landmarking(int num_files, const char *files[], const cha BR_EXPORT bool br_plot_metadata(int num_files, const char *files[], const char *columns, bool show = false); +BR_EXPORT bool br_plot_knn(int num_files, const char *files[], const char *destination, bool show = false); + BR_EXPORT float br_progress(); BR_EXPORT void br_read_pipe(const char *pipe, int *argc, char ***argv); diff --git a/share/openbr/plotting/plot_utils.R b/share/openbr/plotting/plot_utils.R index ef49963..82a7f0c 100644 --- a/share/openbr/plotting/plot_utils.R +++ b/share/openbr/plotting/plot_utils.R @@ -217,6 +217,12 @@ formatData <- function(type="eval") { NormLength <<- data[grep("NormLength",data$Plot),-c(1)] sample <<- readImageData(Sample) rows <<- sample[[1]]$value + } else if (type == "knn") { + # Split data into individual plots + IET <<- data[grep("IET",data$Plot),-c(1)] + IET$Y <<- as.numeric(as.character(IET$Y)) + CMC <<- data[grep("CMC",data$Plot),-c(1)] + CMC$Y <<- as.numeric(as.character(CMC$Y)) } }