diff --git a/app/br/br.cpp b/app/br/br.cpp index 938a035..651e0c6 100644 --- a/app/br/br.cpp +++ b/app/br/br.cpp @@ -20,7 +20,7 @@ #include #include #include -#include +#include /*! * \defgroup cli Command Line Interface @@ -95,8 +95,15 @@ public: check((parc >= 2) && (parc <= 3), "Incorrect parameter count for 'compare'."); br_compare(parv[0], parv[1], parc == 3 ? parv[2] : ""); } else if (!strcmp(fun, "eval")) { - check((parc >= 2) && (parc <= 3), "Incorrect parameter count for 'eval'."); - br_eval(parv[0], parv[1], parc == 3 ? parv[2] : ""); + check((parc >= 1) && (parc <= 3), "Incorrect parameter count for 'eval'."); + if (parc == 1) { + br_eval(parv[0], "", ""); + } else if (parc == 2) { + if (br::File(parv[1]).suffix() == "csv") br_eval(parv[0], "", parv[1]); + else br_eval(parv[0], parv[1], ""); + } else { + br_eval(parv[0], parv[1], parv[2]); + } } else if (!strcmp(fun, "plot")) { check(parc >= 2, "Incorrect parameter count for 'plot'."); br_plot(parc-1, parv, parv[parc-1], true); @@ -194,7 +201,7 @@ private: "-train ... [{model}]\n" "-enroll ... {output_gallery}\n" "-compare [{output}]\n" - "-eval [{csv}]\n" + "-eval [] [{csv}]\n" "-plot ... {destination}\n" "\n" "==== Other Commands ====\n" diff --git a/openbr/core/bee.cpp b/openbr/core/bee.cpp index f24a6a2..443c323 100644 --- a/openbr/core/bee.cpp +++ b/openbr/core/bee.cpp @@ -105,7 +105,7 @@ void BEE::writeSigset(const QString &sigset, const br::FileList &files, bool ign } template -Mat readMatrix(const br::File &matrix) +Mat readMatrix(const br::File &matrix, QString *targetSigset = NULL, QString *querySigset = NULL) { // Special case matrix construction if (matrix == "Identity") { @@ -150,9 +150,11 @@ Mat readMatrix(const br::File &matrix) bool isDistance = (format[0] == 'D'); if (format[1] != '2') qFatal("Invalid matrix header."); - // Skip sigset lines - file.readLine(); - file.readLine(); + // Read sigsets + if (targetSigset != NULL) *targetSigset = file.readLine().simplified(); + else file.readLine(); + if (querySigset != NULL) *querySigset = file.readLine().simplified(); + else file.readLine(); // Get matrix size QStringList words = QString(file.readLine()).split(" "); @@ -172,9 +174,9 @@ Mat readMatrix(const br::File &matrix) return result; } -Mat BEE::readSimmat(const br::File &simmat) +Mat BEE::readSimmat(const br::File &simmat, QString *targetSigset, QString *querySigset) { - return readMatrix(simmat); + return readMatrix(simmat, targetSigset, querySigset); } Mat BEE::readMask(const br::File &mask) @@ -198,9 +200,9 @@ void writeMatrix(const Mat &m, const QString &matrix, const QString &targetSigse QtUtils::touchDir(file); bool success = file.open(QFile::WriteOnly); if (!success) qFatal("Unable to open %s for writing.", qPrintable(matrix)); file.write("S2\n"); - file.write(qPrintable(QFileInfo(targetSigset).fileName())); + file.write(qPrintable(targetSigset)); file.write("\n"); - file.write(qPrintable(QFileInfo(querySigset).fileName())); + file.write(qPrintable(querySigset)); file.write("\n"); file.write("M"); file.write(qPrintable(matrixType)); diff --git a/openbr/core/bee.h b/openbr/core/bee.h index c015d35..1ab4708 100644 --- a/openbr/core/bee.h +++ b/openbr/core/bee.h @@ -39,7 +39,7 @@ namespace BEE void writeSigset(const QString &sigset, const br::FileList &files, bool ignoreMetadata = false); // Matrix IO - cv::Mat readSimmat(const br::File &simmat); + cv::Mat readSimmat(const br::File &simmat, QString *targetSigset = NULL, QString *querySigset = NULL); cv::Mat readMask(const br::File &mask); void writeSimmat(const cv::Mat &m, const QString &simmat, const QString &targetSigset = "Unknown_Target", const QString &querySigset = "Unknown_Query"); void writeMask(const cv::Mat &m, const QString &mask, const QString &targetSigset = "Unknown_Target", const QString &querySigset = "Unknown_Query"); diff --git a/openbr/core/plot.cpp b/openbr/core/plot.cpp index b89706a..425b1bf 100644 --- a/openbr/core/plot.cpp +++ b/openbr/core/plot.cpp @@ -112,21 +112,37 @@ static float getTAR(const QList &operatingPoints, float FAR) float Evaluate(const QString &simmat, const QString &mask, const QString &csv) { - qDebug("Evaluating %s with %s", qPrintable(simmat), qPrintable(mask)); - - // Read files - const Mat scores = BEE::readSimmat(simmat); - File maskFile(mask); - maskFile.set("rows", scores.rows); - maskFile.set("columns", scores.cols); - const Mat masks = BEE::readMask(maskFile); - if (scores.size() != masks.size()) qFatal("Simmat (%i,%i) / Mask (%i,%i) size mismatch.", scores.rows, scores.cols, masks.rows, masks.cols); + qDebug("Evaluating %s%s%s", + qPrintable(simmat), + mask.isEmpty() ? "" : qPrintable(" with " + mask), + csv.isEmpty() ? "" : qPrintable(" to " + csv)); + + // Read similarity matrix + QString target, query; + const Mat scores = BEE::readSimmat(simmat, &target, &query); + + // Read mask matrix + Mat truth; + if (mask.isEmpty()) { + // Use the galleries specified in the similarity matrix + truth = BEE::makeMask(TemplateList::fromGallery(target).files(), + TemplateList::fromGallery(query).files()); + } else { + File maskFile(mask); + maskFile.set("rows", scores.rows); + maskFile.set("columns", scores.cols); + truth = BEE::readMask(maskFile); + } - return Evaluate(scores, masks, csv); + return Evaluate(scores, truth, csv); } float Evaluate(const Mat &simmat, const Mat &mask, const QString &csv) { + if (simmat.size() != mask.size()) + qFatal("Similarity matrix (%ix%i) differs in size from mask matrix (%ix%i).", + simmat.rows, simmat.cols, mask.rows, mask.cols); + const int Max_Points = 500; float result = -1; diff --git a/openbr/core/plot.h b/openbr/core/plot.h index 20486e7..efabc0b 100644 --- a/openbr/core/plot.h +++ b/openbr/core/plot.h @@ -26,7 +26,7 @@ namespace br { void Confusion(const QString &file, float score, int &true_positives, int &false_positives, int &true_negatives, int &false_negatives); -float Evaluate(const QString &simmat, const QString &mask, const QString &csv = ""); // Returns TAR @ FAR = 0.01 +float Evaluate(const QString &simmat, const QString &mask = "", const QString &csv = ""); // Returns TAR @ FAR = 0.01 float Evaluate(const cv::Mat &scores, const cv::Mat &masks, const QString &csv = ""); bool Plot(const QStringList &files, const br::File &destination, bool show = false); bool PlotMetadata(const QStringList &files, const QString &destination, bool show = false); diff --git a/openbr/openbr_plugin.cpp b/openbr/openbr_plugin.cpp index 47ecc4b..a65dd24 100644 --- a/openbr/openbr_plugin.cpp +++ b/openbr/openbr_plugin.cpp @@ -411,6 +411,7 @@ TemplateList TemplateList::fromGallery(const br::File &gallery) newTemplates[i].file.append(gallery.localMetadata()); newTemplates[i].file.append(file.localMetadata()); newTemplates[i].file.set("Index", i+templates.size()); + newTemplates[i].file.set("Gallery", gallery.name); if (newTemplates[i].file.getBool("allPartitions")) { if (crossValidate > 0) { // Set template to the first parition diff --git a/openbr/plugins/output.cpp b/openbr/plugins/output.cpp index e07a78b..b824090 100644 --- a/openbr/plugins/output.cpp +++ b/openbr/plugins/output.cpp @@ -148,7 +148,10 @@ class mtxOutput : public MatrixOutput ~mtxOutput() { if (file.isNull() || targetFiles.isEmpty() || queryFiles.isEmpty()) return; - BEE::writeSimmat(data, file.name); + BEE::writeSimmat(data, + file.name, + targetFiles.first().get("Gallery", "Unknown_Target"), + queryFiles.first().get("Gallery", "Unknown_Query")); } }; diff --git a/scripts/helloWorld.sh b/scripts/helloWorld.sh index f4cab0c..3cec377 100755 --- a/scripts/helloWorld.sh +++ b/scripts/helloWorld.sh @@ -40,3 +40,4 @@ br -algorithm Eigenfaces -path MEDS/img -compare MEDS/sigset/MEDS_frontal_target # br -algorithm Eigenfaces -path MEDS/img -enroll MEDS/sigset/MEDS_frontal_target.xml target.gal -enroll MEDS/sigset/MEDS_frontal_query.xml query.gal -compare target.gal query.gal scores.mtx # Evaluate Eigenfaces accuracy +br -eval scores.mtx results.csv -plot results.csv results.pdf