diff --git a/.gitignore b/.gitignore index de489d6..25abe9f 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ scripts/results ### QtCreator ### *CMakeLists.txt.user* +*.autosave ### R ### *.RData diff --git a/README.md b/README.md index 834cf02..262d664 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,13 @@ -www.openbiometrics.org +**www.openbiometrics.org** $ git clone https://github.com/biometrics/openbr.git $ cd openbr $ git submodule init $ git submodule update + +To optionally check out a particular [tagged release](https://github.com/biometrics/openbr/releases): -[Build Instructions](http://openbiometrics.org/doxygen/latest/installation.html) + $ git checkout + $ git submodule update + +**[Build Instructions](http://openbiometrics.org/doxygen/latest/installation.html)** diff --git a/app/br/br.cpp b/app/br/br.cpp index 8615274..c7ecdc1 100644 --- a/app/br/br.cpp +++ b/app/br/br.cpp @@ -129,17 +129,20 @@ public: check(parc == 3, "Incorrect parameter count for 'convert'."); br_convert(parv[0], parv[1], parv[2]); } else if (!strcmp(fun, "evalClassification")) { - check(parc == 2, "Incorrect parameter count for 'evalClassification'."); - br_eval_classification(parv[0], parv[1]); + check(parc >= 2 && parc <= 4, "Incorrect parameter count for 'evalClassification'."); + br_eval_classification(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? parv[3] : ""); } else if (!strcmp(fun, "evalClustering")) { check(parc == 2, "Incorrect parameter count for 'evalClustering'."); br_eval_clustering(parv[0], parv[1]); } else if (!strcmp(fun, "evalDetection")) { check((parc >= 2) && (parc <= 3), "Incorrect parameter count for 'evalDetection'."); br_eval_detection(parv[0], parv[1], parc == 3 ? parv[2] : ""); + } else if (!strcmp(fun, "evalLandmarking")) { + check((parc >= 2) && (parc <= 5), "Incorrect parameter count for 'evalLandmarking'."); + br_eval_landmarking(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? atoi(parv[3]) : 0, parc >= 5 ? atoi(parv[4]) : 1); } else if (!strcmp(fun, "evalRegression")) { - check(parc == 2, "Incorrect parameter count for 'evalRegression'."); - br_eval_regression(parv[0], parv[1]); + check(parc >= 2 && parc <= 4, "Incorrect parameter count for 'evalRegression'."); + br_eval_regression(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? parv[3] : ""); } else if (!strcmp(fun, "plotDetection")) { check(parc >= 2, "Incorrect parameter count for 'plotDetection'."); br_plot_detection(parc-1, parv, parv[parc-1], true); @@ -215,10 +218,11 @@ private: "-combineMasks ... {mask} (And|Or)\n" "-cat ... {gallery}\n" "-convert (Format|Gallery|Output) {output_file}\n" - "-evalClassification \n" + "-evalClassification \n" "-evalClustering \n" "-evalDetection [{csv}]\n" - "-evalRegression \n" + "-evalLandmarking [{csv} [ ]]\n" + "-evalRegression \n" "-plotDetection ... {destination}\n" "-plotMetadata ... \n" "-getHeader \n" diff --git a/app/examples/age_estimation.cpp b/app/examples/age_estimation.cpp index c2a90e5..2958b71 100644 --- a/app/examples/age_estimation.cpp +++ b/app/examples/age_estimation.cpp @@ -29,7 +29,7 @@ static void printTemplate(const br::Template &t) { - printf("%s age: %d\n", qPrintable(t.file.fileName()), int(t.file.get("Subject"))); + printf("%s age: %d\n", qPrintable(t.file.fileName()), int(t.file.get("Age"))); } int main(int argc, char *argv[]) diff --git a/app/examples/face_recognition_evaluation.cpp b/app/examples/face_recognition_evaluation.cpp index ebf6462..745f522 100644 --- a/app/examples/face_recognition_evaluation.cpp +++ b/app/examples/face_recognition_evaluation.cpp @@ -25,8 +25,7 @@ * -compare target.gal query.gal scores.mtx \ * -makeMask ../data/MEDS/sigset/MEDS_frontal_target.xml ../data/MEDS/sigset/MEDS_frontal_query.xml MEDS.mask \ * -eval scores.mtx MEDS.mask Algorithm_Dataset/FaceRecognition_MEDS.csv \ - * -eval ../data/MEDS/simmat/COTS_MEDS.mtx MEDS.mask Algorithm_Dataset/COTS_MEDS.csv \ - * -plot Algorithm_Dataset/FaceRecognition_MEDS.csv Algorithm_Dataset/COTS_MEDS.csv MEDS + * -plot Algorithm_Dataset/FaceRecognition_MEDS.csv MEDS * \endcode */ diff --git a/app/examples/gender_estimation.cpp b/app/examples/gender_estimation.cpp index dd4ebd2..d2a824d 100644 --- a/app/examples/gender_estimation.cpp +++ b/app/examples/gender_estimation.cpp @@ -29,7 +29,7 @@ static void printTemplate(const br::Template &t) { - printf("%s gender: %s\n", qPrintable(t.file.fileName()), qPrintable(t.file.get("Subject"))); + printf("%s gender: %s\n", qPrintable(t.file.fileName()), qPrintable(t.file.get("Gender"))); } int main(int argc, char *argv[]) diff --git a/data/KTH/test_9ppl.xml b/data/KTH/test_9ppl.xml new file mode 100644 index 0000000..c211650 --- /dev/null +++ b/data/KTH/test_9ppl.xml @@ -0,0 +1,651 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/data/KTH/train_16ppl.xml b/data/KTH/train_16ppl.xml new file mode 100644 index 0000000..c4bdc71 --- /dev/null +++ b/data/KTH/train_16ppl.xml @@ -0,0 +1,1149 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/openbr/core/bee.cpp b/openbr/core/bee.cpp index c6cb254..9639657 100644 --- a/openbr/core/bee.cpp +++ b/openbr/core/bee.cpp @@ -99,10 +99,10 @@ void BEE::writeSigset(const QString &sigset, const br::FileList &files, bool ign QStringList metadata; if (!ignoreMetadata) foreach (const QString &key, file.localKeys()) { - if ((key == "Index") || (key == "Subject")) continue; + if ((key == "Index") || (key == "Label")) continue; metadata.append(key+"=\""+QtUtils::toString(file.value(key))+"\""); } - lines.append("\t("Subject",file.fileName()) +"\">"); + lines.append("\t("Label",file.fileName()) +"\">"); lines.append("\t\t"); lines.append("\t"); } @@ -266,10 +266,11 @@ void BEE::makeMask(const QString &targetInput, const QString &queryInput, const cv::Mat BEE::makeMask(const br::FileList &targets, const br::FileList &queries, int partition) { - // Would like to use indexProperty for this, but didn't make a version of that for Filelist yet - // -cao - QList targetLabels = File::get(targets, "Subject", "-1"); - QList queryLabels = File::get(queries, "Subject", "-1"); + // Direct use of "Label" isn't general, also would prefer to use indexProperty, rather than + // doing string comparisons (but that isn't implemented yet for FileList) -cao + QList targetLabels = File::get(targets, "Label", "-1"); + QList queryLabels = File::get(queries, "Label", "-1"); + QList targetPartitions = targets.crossValidationPartitions(); QList queryPartitions = queries.crossValidationPartitions(); diff --git a/openbr/core/cluster.cpp b/openbr/core/cluster.cpp index 6a6b954..8c880d9 100644 --- a/openbr/core/cluster.cpp +++ b/openbr/core/cluster.cpp @@ -279,8 +279,8 @@ void br::EvalClustering(const QString &csv, const QString &input) qDebug("Evaluating %s against %s", qPrintable(csv), qPrintable(input)); // We assume clustering algorithms store assigned cluster labels as integers (since the clusters are - // not named). - QList labels = File::get(TemplateList::fromGallery(input), "Subject"); + // not named). Direct use of ClusterID is not general -cao + QList labels = File::get(TemplateList::fromGallery(input), "ClusterID"); QHash labelToIndex; int nClusters = 0; diff --git a/openbr/core/common.cpp b/openbr/core/common.cpp index 17584ec..55cbd3e 100644 --- a/openbr/core/common.cpp +++ b/openbr/core/common.cpp @@ -15,11 +15,15 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "common.h" +#include using namespace std; /**** GLOBAL ****/ void Common::seedRNG() { + static QMutex seedControl; + QMutexLocker lock(&seedControl); + static bool seeded = false; if (!seeded) { srand(0); // We seed with 0 instead of time(NULL) to have reproducible randomness @@ -29,8 +33,6 @@ void Common::seedRNG() { QList Common::RandSample(int n, int max, int min, bool unique) { - seedRNG(); - QList samples; samples.reserve(n); int range = max-min; if (range <= 0) qFatal("Non-positive range."); @@ -50,8 +52,6 @@ QList Common::RandSample(int n, int max, int min, bool unique) QList Common::RandSample(int n, const QSet &values, bool unique) { - seedRNG(); - QList valueList = values.toList(); if (unique && (values.size() <= n)) return valueList; diff --git a/openbr/core/eval.cpp b/openbr/core/eval.cpp index 9e813d8..05ce85d 100644 --- a/openbr/core/eval.cpp +++ b/openbr/core/eval.cpp @@ -255,11 +255,22 @@ struct Counter } }; -void EvalClassification(const QString &predictedInput, const QString &truthInput) +void EvalClassification(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty, QString truthProperty) { - qDebug("Evaluating classification of %s against %s", qPrintable(predictedInput), qPrintable(truthInput)); - TemplateList predicted(TemplateList::fromGallery(predictedInput)); - TemplateList truth(TemplateList::fromGallery(truthInput)); + qDebug("Evaluating classification of %s against %s", qPrintable(predictedGallery), qPrintable(truthGallery)); + + if (predictedProperty.isEmpty()) + predictedProperty = "Label"; + // If predictedProperty is specified, but truthProperty isn't, copy over the value from + // predicted property + else if (truthProperty.isEmpty()) + truthProperty = predictedProperty; + + if (truthProperty.isEmpty()) + truthProperty = "Label"; + + TemplateList predicted(TemplateList::fromGallery(predictedGallery)); + TemplateList truth(TemplateList::fromGallery(truthGallery)); if (predicted.size() != truth.size()) qFatal("Input size mismatch."); QHash counters; @@ -267,9 +278,8 @@ void EvalClassification(const QString &predictedInput, const QString &truthInput if (predicted[i].file.name != truth[i].file.name) qFatal("Input order mismatch."); - // Typically these lists will be of length one, but this generalization allows measuring multi-class labeling accuracy. - QString predictedSubject = predicted[i].file.get("Subject"); - QString trueSubject = truth[i].file.get("Subject"); + QString predictedSubject = predicted[i].file.get(predictedProperty); + QString trueSubject = truth[i].file.get(truthProperty); QStringList predictedSubjects(predictedSubject); QStringList trueSubjects(trueSubject); @@ -393,11 +403,11 @@ QString getDetectKey(const TemplateList &templates) return ""; } -float EvalDetection(const QString &predictedInput, const QString &truthInput, const QString &csv) +float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv) { - qDebug("Evaluating detection of %s against %s", qPrintable(predictedInput), qPrintable(truthInput)); - const TemplateList predicted(TemplateList::fromGallery(predictedInput)); - const TemplateList truth(TemplateList::fromGallery(truthInput)); + qDebug("Evaluating detection of %s against %s", qPrintable(predictedGallery), qPrintable(truthGallery)); + const TemplateList predicted(TemplateList::fromGallery(predictedGallery)); + const TemplateList truth(TemplateList::fromGallery(truthGallery)); // Figure out which metadata field contains a bounding box QString truthDetectKey = getDetectKey(truth); @@ -466,21 +476,48 @@ float EvalDetection(const QString &predictedInput, const QString &truthInput, co return averageOverlap; } -void EvalRegression(const QString &predictedInput, const QString &truthInput) +float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv, int normalizationIndexA, int normalizationIndexB) +{ + (void) predictedGallery; + (void) truthGallery; + (void) csv; + (void) normalizationIndexA; + (void) normalizationIndexB; + + return 0; +} + +void EvalRegression(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty, QString truthProperty) { - qDebug("Evaluating regression of %s against %s", qPrintable(predictedInput), qPrintable(truthInput)); - const TemplateList predicted(TemplateList::fromGallery(predictedInput)); - const TemplateList truth(TemplateList::fromGallery(truthInput)); + qDebug("Evaluating regression of %s against %s", qPrintable(predictedGallery), qPrintable(truthGallery)); + + if (predictedProperty.isEmpty()) + predictedProperty = "Regressor"; + // If predictedProperty is specified, but truthProperty isn't, copy the value over + // rather than using the default for truthProperty + else if (truthProperty.isEmpty()) + truthProperty = predictedProperty; + + if (truthProperty.isEmpty()) + predictedProperty = "Regressand"; + + const TemplateList predicted(TemplateList::fromGallery(predictedGallery)); + const TemplateList truth(TemplateList::fromGallery(truthGallery)); if (predicted.size() != truth.size()) qFatal("Input size mismatch."); float rmsError = 0; + float maeError = 0; QStringList truthValues, predictedValues; for (int i=0; i("Subject")-truth[i].file.get("Subject"), 2.f); - truthValues.append(QString::number(truth[i].file.get("Subject"))); - predictedValues.append(QString::number(predicted[i].file.get("Subject"))); + + float difference = predicted[i].file.get(predictedProperty) - truth[i].file.get(truthProperty); + + rmsError += pow(difference, 2.f); + maeError += fabsf(difference); + truthValues.append(QString::number(truth[i].file.get(truthProperty))); + predictedValues.append(QString::number(predicted[i].file.get(predictedProperty))); } QStringList rSource; @@ -500,6 +537,7 @@ void EvalRegression(const QString &predictedInput, const QString &truthInput) if (success) QtUtils::showFile("EvalRegression.pdf"); qDebug("RMS Error = %f", sqrt(rmsError/predicted.size())); + qDebug("MAE = %f", maeError/predicted.size()); } } // namespace br diff --git a/openbr/core/eval.h b/openbr/core/eval.h index d90d1a5..c58b60a 100644 --- a/openbr/core/eval.h +++ b/openbr/core/eval.h @@ -26,9 +26,10 @@ namespace br float Evaluate(const QString &simmat, const QString &mask = "", const QString &csv = ""); // Returns TAR @ FAR = 0.001 float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const QString &csv = "", int parition = 0); float Evaluate(const cv::Mat &scores, const cv::Mat &masks, const QString &csv = ""); - void EvalClassification(const QString &predictedInput, const QString &truthInput); - float EvalDetection(const QString &predictedInput, const QString &truthInput, const QString &csv = ""); // Return average overlap - void EvalRegression(const QString &predictedInput, const QString &truthInput); + void EvalClassification(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = ""); + float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv = ""); // Return average overlap + float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", int normalizationIndexA = 0, int normalizationIndexB = 1); // Return average error + void EvalRegression(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = ""); } #endif // __EVAL_H diff --git a/openbr/frvt2012.cpp b/openbr/frvt2012.cpp index ac47ac8..b0c2d29 100644 --- a/openbr/frvt2012.cpp +++ b/openbr/frvt2012.cpp @@ -132,7 +132,7 @@ int32_t SdkEstimator::estimate_age(const ONEFACE &input_face, int32_t &age) TemplateList templates; templates.append(templateFromONEFACE(input_face)); templates >> *frvt2012_age_transform.data(); - age = templates.first().file.get("Subject"); + age = templates.first().file.get("Age"); return templates.first().file.failed() ? 4 : 0; } @@ -141,6 +141,6 @@ int32_t SdkEstimator::estimate_gender(const ONEFACE &input_face, int8_t &gender, TemplateList templates; templates.append(templateFromONEFACE(input_face)); templates >> *frvt2012_gender_transform.data(); - mf = gender = templates.first().file.get("Subject") == "Male" ? 0 : 1; + mf = gender = templates.first().file.get("Gender") == "Male" ? 0 : 1; return templates.first().file.failed() ? 4 : 0; } diff --git a/openbr/gui/classifier.cpp b/openbr/gui/classifier.cpp index 2df9f5e..71a3102 100644 --- a/openbr/gui/classifier.cpp +++ b/openbr/gui/classifier.cpp @@ -39,19 +39,23 @@ void Classifier::_classify(File file) { QString key, value; foreach (const File &f, Enroll(file.flat(), File("[algorithm=" + algorithm + "]"))) { - if (!f.contains("Label")) - continue; if (algorithm == "GenderClassification") { key = "Gender"; - value = (f.get("Subject")); } else if (algorithm == "AgeRegression") { key = "Age"; - value = QString::number(int(f.get("Subject")+0.5)) + " Years"; } else { key = algorithm; - value = f.get("Subject"); } + + if (!f.contains(key)) + continue; + + if (algorithm == "AgeRegression") + value = QString::number(int(f.get(key)+0.5)) + " Years"; + else + value = f.get(key); + break; } diff --git a/openbr/openbr.cpp b/openbr/openbr.cpp index c269366..9092d6b 100644 --- a/openbr/openbr.cpp +++ b/openbr/openbr.cpp @@ -72,9 +72,9 @@ float br_eval(const char *simmat, const char *mask, const char *csv) return Evaluate(simmat, mask, csv); } -void br_eval_classification(const char *predicted_gallery, const char *truth_gallery) +void br_eval_classification(const char *predicted_gallery, const char *truth_gallery, const char *predicted_property, const char * truth_property) { - EvalClassification(predicted_gallery, truth_gallery); + EvalClassification(predicted_gallery, truth_gallery, predicted_property, truth_property); } void br_eval_clustering(const char *csv, const char *gallery) @@ -87,9 +87,14 @@ float br_eval_detection(const char *predicted_gallery, const char *truth_gallery return EvalDetection(predicted_gallery, truth_gallery, csv); } -void br_eval_regression(const char *predicted_gallery, const char *truth_gallery) +float br_eval_landmarking(const char *predicted_gallery, const char *truth_gallery, const char *csv, int normalization_index_a, int normalization_index_b) { - EvalRegression(predicted_gallery, truth_gallery); + return EvalLandmarking(predicted_gallery, truth_gallery, csv, normalization_index_a, normalization_index_b); +} + +void br_eval_regression(const char *predicted_gallery, const char *truth_gallery, const char * predicted_property, const char * truth_property) +{ + EvalRegression(predicted_gallery, truth_gallery, predicted_property, truth_property); } void br_finalize() diff --git a/openbr/openbr.h b/openbr/openbr.h index 6d96a28..9165932 100644 --- a/openbr/openbr.h +++ b/openbr/openbr.h @@ -149,7 +149,7 @@ BR_EXPORT float br_eval(const char *simmat, const char *mask, const char *csv = * \param predicted_gallery The predicted br::Gallery. * \param truth_gallery The ground truth br::Gallery. */ -BR_EXPORT void br_eval_classification(const char *predicted_gallery, const char *truth_gallery); +BR_EXPORT void br_eval_classification(const char *predicted_gallery, const char *truth_gallery, const char * predicted_property="", const char * truth_property=""); /*! * \brief Evaluates and prints clustering accuracy to the terminal. @@ -169,11 +169,21 @@ BR_EXPORT void br_eval_clustering(const char *csv, const char *gallery); BR_EXPORT float br_eval_detection(const char *predicted_gallery, const char *truth_gallery, const char *csv = ""); /*! + * \brief Evaluates and prints landmarking accuracy to terminal. + * \param predicted_gallery The predicted br::Gallery. + * \param truth_gallery The ground truth br::Gallery. + * \param csv Optional \c .csv file to contain performance metrics. + * \param normalization_index_a Optional first index in the list of points to use for normalization. + * \param normalization_index_b Optional second index in the list of points to use for normalization. + */ +BR_EXPORT float br_eval_landmarking(const char *predicted_gallery, const char *truth_gallery, const char *csv = "", int normalization_index_a = 0, int normalization_index_b = 1); + +/*! * \brief Evaluates regression accuracy to disk. * \param predicted_gallery The predicted br::Gallery. * \param truth_gallery The ground truth br::Gallery. */ -BR_EXPORT void br_eval_regression(const char *predicted_gallery, const char *truth_gallery); +BR_EXPORT void br_eval_regression(const char *predicted_gallery, const char *truth_gallery, const char * predicted_property="", const char * truth_property=""); /*! * \brief Wraps br::Context::finalize() diff --git a/openbr/openbr_export.cpp b/openbr/openbr_export.cpp index 0eccdb5..dac8526 100644 --- a/openbr/openbr_export.cpp +++ b/openbr/openbr_export.cpp @@ -69,7 +69,6 @@ $ cd bin $ export LD_LIBRARY_PATH=../lib:${LD_LIBRARY_PATH} $ sudo ldconfig -$ sudo cp ../share/openbr/70-yubikey.rules /etc/udev/rules.d # Only needed if you were given a license dongle. \endverbatim * \par OS X \verbatim @@ -80,10 +79,6 @@ $ export DYLD_FRAMEWORK_PATH=../lib:${DYLD_FRAMEWORK_PATH} * \par Windows * No configuration is necessary! * - * \section installation_license_dongle License Dongle - * In the unlikely event that you were given a USB License Dongle, then dongle must be in the computer in order to use the SDK. - * No configuration of the dongle is needed. - * * \section installation_done Start Working * To test for successful installation: \verbatim diff --git a/openbr/openbr_plugin.cpp b/openbr/openbr_plugin.cpp index 1becbbf..38aeb80 100644 --- a/openbr/openbr_plugin.cpp +++ b/openbr/openbr_plugin.cpp @@ -412,7 +412,8 @@ TemplateList TemplateList::fromGallery(const br::File &gallery) // of target images to every partition newTemplates[i].file.set("Partition", -1); } else { - const QByteArray md5 = QCryptographicHash::hash(newTemplates[i].file.get("Subject").toLatin1(), QCryptographicHash::Md5); + // Direct use of "Label" is not general -cao + const QByteArray md5 = QCryptographicHash::hash(newTemplates[i].file.get("Label").toLatin1(), QCryptographicHash::Md5); // Select the right 8 hex characters so that it can be represented as a 64 bit integer without overflow newTemplates[i].file.set("Partition", md5.toHex().right(8).toULongLong(0, 16) % crossValidate); } @@ -890,6 +891,8 @@ void br::Context::initialize(int &argc, char *argv[], QString sdkPath, bool use_ qInstallMessageHandler(messageHandler); + Common::seedRNG(); + // Search for SDK if (sdkPath.isEmpty()) { QStringList checkPaths; checkPaths << QDir::currentPath() << QCoreApplication::applicationDirPath(); @@ -1082,9 +1085,6 @@ Transform::Transform(bool _independent, bool _trainable) { independent = _independent; trainable = _trainable; - classes = std::numeric_limits::max(); - instances = std::numeric_limits::max(); - fraction = 1; } Transform *Transform::make(QString str, QObject *parent) @@ -1140,9 +1140,6 @@ Transform *Transform::make(QString str, QObject *parent) Transform *Transform::clone() const { Transform *clone = Factory::make(file.flat()); - clone->classes = classes; - clone->instances = instances; - clone->fraction = fraction; return clone; } diff --git a/openbr/openbr_plugin.h b/openbr/openbr_plugin.h index a5171b2..0f0d800 100644 --- a/openbr/openbr_plugin.h +++ b/openbr/openbr_plugin.h @@ -130,13 +130,6 @@ void reset_##NAME() { NAME = DEFAULT; } * -# If the value is convertable to a floating point number then it is represented with \c float. * -# Otherwise, it is represented with \c QString. * - * The metadata keys \c Subject and \c Label have special significance in the system. - * \c Subject is a string specifying a unique identifier used to determine ground truth match/non-match. - * \c Label is a floating point value used for supervised learning. - * When the system needs labels for training, but only subjects are provided in the file metadata, the rule for generating labels is as follows. - * If the subject value can be converted to a float then do so and consider that the label. - * Otherwise, generate a unique integer ID for the string starting from zero and incrementing by one everytime another ID is needed. - * * Metadata keys fall into one of two categories: * - \c camelCaseKeys are inputs that specify how to process the file. * - \c Capitalized_Underscored_Keys are outputs computed from processing the file. @@ -147,8 +140,6 @@ void reset_##NAME() { NAME = DEFAULT; } * --- | ---- | ----------- * separator | QString | Seperate #name into multiple files * Index | int | Index of a template in a template list - * Subject | QString | Class name - * Label | float | Class value * Confidence | float | Classification/Regression quality * FTE | bool | Failure to enroll * FTO | bool | Failure to open @@ -157,13 +148,15 @@ void reset_##NAME() { NAME = DEFAULT; } * *_Width | float | Size * *_Height | float | Size * *_Radius | float | Size + * Label | QString | Class label * Theta | float | Pose * Roll | float | Pose * Pitch | float | Pose * Yaw | float | Pose * Points | QList | List of unnamed points * Rects | QList | List of unnamed rects - * Age | QString | Age used for demographic filtering + * Age | float | Age used for demographic filtering + * Gender | QString | Subject gender * _* | * | Reserved for internal use */ struct BR_EXPORT File @@ -172,7 +165,7 @@ struct BR_EXPORT File File() {} File(const QString &file) { init(file); } /*!< \brief Construct a file from a string. */ - File(const QString &file, const QVariant &subject) { init(file); set("Subject", subject); } /*!< \brief Construct a file from a string and assign a label. */ + File(const QString &file, const QVariant &label) { init(file); set("Label", label); } /*!< \brief Construct a file from a string and assign a label. */ File(const char *file) { init(file); } /*!< \brief Construct a file from a c-style string. */ inline operator QString() const { return name; } /*!< \brief Returns #name. */ QString flat() const; /*!< \brief A stringified version of the file with metadata. */ @@ -1058,12 +1051,6 @@ class BR_EXPORT Transform : public Object Q_OBJECT public: - Q_PROPERTY(int classes READ get_classes WRITE set_classes RESET reset_classes STORED false) - Q_PROPERTY(int instances READ get_instances WRITE set_instances RESET reset_instances STORED false) - Q_PROPERTY(float fraction READ get_fraction WRITE set_fraction RESET reset_fraction STORED false) - BR_PROPERTY(int, classes, std::numeric_limits::max()) - BR_PROPERTY(int, instances, std::numeric_limits::max()) - BR_PROPERTY(float, fraction, 1) bool independent, trainable; virtual ~Transform() {} diff --git a/openbr/plugins/algorithms.cpp b/openbr/plugins/algorithms.cpp index fa91da6..b8aea8a 100644 --- a/openbr/plugins/algorithms.cpp +++ b/openbr/plugins/algorithms.cpp @@ -48,7 +48,10 @@ class AlgorithmsInitializer : public Initializer // Video Globals->abbreviations.insert("DisplayVideo", "Stream([FPSLimit(30)+Show(false,[FrameNumber])+Discard])"); Globals->abbreviations.insert("PerFrameDetection", "Stream([SaveMat(original)+Cvt(Gray)+Cascade(FrontalFace)+ASEFEyes+RestoreMat(original)+Draw(inPlace=true),Show(false,[FrameNumber])+Discard])"); - Globals->abbreviations.insert("AgeGenderDemo", "Stream([SaveMat(original)+Cvt(Gray)+Cascade(FrontalFace)+Expand+++(+Rename(Subject,Age)+Discard)/(+Rename(Subject,Gender)+Discard)+RestoreMat(original)+Draw(inPlace=true)+DrawPropertiesPoint([Age,Gender],Affine_0,inPlace=true)+SaveMat(original)+Discard+Contract,RestoreMat(original)+FPSCalc+Show(false,[AvgFPS,Age,Gender])+Discard])"); + Globals->abbreviations.insert("AgeGenderDemo", "Stream([SaveMat(original)+Cvt(Gray)+Cascade(FrontalFace)+Expand+++/+Discard+RestoreMat(original)+Draw(inPlace=true)+DrawPropertiesPoint([Age,Gender],Affine_0,inPlace=true)+SaveMat(original)+Discard+Contract,RestoreMat(original)+FPSCalc+Show(false,[AvgFPS,Age,Gender])+Discard])"); + Globals->abbreviations.insert("HOG", "Stream([DropFrames(5)+Cvt(Gray)+KeyPointDetector(SIFT)+ROI+Expand+Resize(32,32)+Gradient+RectRegions+Bin(0,360,8)+Hist(8)+Cat])+Contract+CatRows+KMeans(500)+Hist(500)+SVM"); + Globals->abbreviations.insert("HOF", "Stream([DropFrames(5)+KeyPointDetector(SIFT),AggregateFrames(2),OpticalFlow+ROI+Expand+Resize(32,32)+Gradient+RectRegions+Bin(0,360,8)+Hist(8)+Cat])+Contract+CatRows+KMeans(500)+Hist(500)"); + Globals->abbreviations.insert("HOGHOF", "Stream([DropFrames(5)+Cvt(Gray),KeyPointDetector(SIFT),AggregateFrames(2),(OpticalFlow+Gradient+Bin(0,360,8)+ROI+Hist(8))/(First+Gradient+Bin(0,360,8)+ROI+Hist(8)),CatCols])+Contract+CatRows+KMeans(500)+Hist(500)"); // Generic Image Processing Globals->abbreviations.insert("SIFT", "Open+KeyPointDetector(SIFT)+KeyPointDescriptor(SIFT):KeyPointMatcher(BruteForce)"); @@ -74,14 +77,14 @@ class AlgorithmsInitializer : public Initializer Globals->abbreviations.insert("FaceDetection", "(Open+Cvt(Gray)+Cascade(FrontalFace))"); Globals->abbreviations.insert("DenseLBP", "(Blur(1.1)+Gamma(0.2)+DoG(1,2)+ContrastEq(0.1,10)+LBP(1,2)+RectRegions(8,8,6,6)+Hist(59))"); Globals->abbreviations.insert("DenseSIFT", "(Grid(10,10)+SIFTDescriptor(12)+ByRow)"); - Globals->abbreviations.insert("FaceRecognitionRegistration", "(ASEFEyes+Affine(88,88,0.25,0.35)+FTE(DFFS,instances=1))"); - Globals->abbreviations.insert("FaceRecognitionExtraction", "(Mask+DenseSIFT/DenseLBP+PCA(0.95,instances=1)+Normalize(L2)+Cat)"); - Globals->abbreviations.insert("FaceRecognitionEmbedding", "(Dup(12)+RndSubspace(0.05,1)+LDA(0.98,instances=-2)+Cat+PCA(768,instances=1))"); + Globals->abbreviations.insert("FaceRecognitionRegistration", "(ASEFEyes+Affine(88,88,0.25,0.35)+DownsampleTraining(FTE(DFFS),instances=1))"); + Globals->abbreviations.insert("FaceRecognitionExtraction", "(Mask+DenseSIFT/DenseLBP+DownsampleTraining(PCA(0.95),instances=1)+Normalize(L2)+Cat)"); + Globals->abbreviations.insert("FaceRecognitionEmbedding", "(Dup(12)+RndSubspace(0.05,1)+DownsampleTraining(LDA(0.98),instances=-2)+Cat+DownsampleTraining(PCA(768),instances=1))"); Globals->abbreviations.insert("FaceRecognitionQuantization", "(Normalize(L1)+Quantize)"); Globals->abbreviations.insert("FaceClassificationRegistration", "(ASEFEyes+Affine(56,72,0.33,0.45)+FTE(DFFS))"); - Globals->abbreviations.insert("FaceClassificationExtraction", "((Grid(7,7)+SIFTDescriptor(8)+ByRow)/DenseLBP+PCA(0.95,instances=-1)+Cat)"); - Globals->abbreviations.insert("AgeRegressor", "Center(Range,instances=-1)+SVM(RBF,EPS_SVR,instances=100)"); - Globals->abbreviations.insert("GenderClassifier", "Center(Range,instances=-1)+SVM(RBF,C_SVC,instances=4000)"); + Globals->abbreviations.insert("FaceClassificationExtraction", "((Grid(7,7)+SIFTDescriptor(8)+ByRow)/DenseLBP+DownsampleTraining(PCA(0.95),instances=-1, inputVariable=Gender)+Cat)"); + Globals->abbreviations.insert("AgeRegressor", "DownsampleTraining(Center(Range),instances=-1, inputVariable=Age)+DownsampleTraining(SVM(RBF,EPS_SVR,inputVariable=Age),instances=100, inputVariable=Age)"); + Globals->abbreviations.insert("GenderClassifier", "DownsampleTraining(Center(Range),instances=-1, inputVariable=Gender)+DownsampleTraining(SVM(RBF,C_SVC,inputVariable=Gender),instances=4000, inputVariable=Gender)"); Globals->abbreviations.insert("UCharL1", "Unit(ByteL1)"); } }; diff --git a/openbr/plugins/cluster.cpp b/openbr/plugins/cluster.cpp index e6ff87c..3a9b577 100644 --- a/openbr/plugins/cluster.cpp +++ b/openbr/plugins/cluster.cpp @@ -89,10 +89,14 @@ class KNNTransform : public Transform Q_PROPERTY(br::Distance *distance READ get_distance WRITE set_distance RESET reset_distance STORED false) Q_PROPERTY(bool weighted READ get_weighted WRITE set_weighted RESET reset_weighted STORED false) Q_PROPERTY(int numSubjects READ get_numSubjects WRITE set_numSubjects RESET reset_numSubjects STORED false) + Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) + Q_PROPERTY(QString outputVariable READ get_outputVariable WRITE set_outputVariable RESET reset_outputVariable STORED false) BR_PROPERTY(int, k, 1) BR_PROPERTY(br::Distance*, distance, NULL) BR_PROPERTY(bool, weighted, false) BR_PROPERTY(int, numSubjects, 1) + BR_PROPERTY(QString, inputVariable, "Label") + BR_PROPERTY(QString, outputVariable, "KNN") TemplateList gallery; @@ -111,17 +115,17 @@ class KNNTransform : public Transform QHash votes; const int max = (k < 1) ? sortedScores.size() : std::min(k, sortedScores.size()); for (int j=0; j("Subject")] += (weighted ? sortedScores[j].first : 1); + votes[gallery[sortedScores[j].second].file.get(inputVariable)] += (weighted ? sortedScores[j].first : 1); subjects.append(votes.keys()[votes.values().indexOf(Common::Max(votes.values()))]); // Remove subject from consideration if (subjects.size() < numSubjects) for (int j=sortedScores.size()-1; j>=0; j--) - if (gallery[sortedScores[j].second].file.get("Subject") == subjects.last()) + if (gallery[sortedScores[j].second].file.get(inputVariable) == subjects.last()) sortedScores.removeAt(j); } - dst.file.set("KNN", subjects.size() > 1 ? "[" + subjects.join(",") + "]" : subjects.first()); + dst.file.set(outputVariable, subjects.size() > 1 ? "[" + subjects.join(",") + "]" : subjects.first()); } void store(QDataStream &stream) const diff --git a/openbr/plugins/eigen3.cpp b/openbr/plugins/eigen3.cpp index 89765db..4a904c2 100644 --- a/openbr/plugins/eigen3.cpp +++ b/openbr/plugins/eigen3.cpp @@ -303,10 +303,12 @@ class LDATransform : public Transform Q_PROPERTY(bool pcaWhiten READ get_pcaWhiten WRITE set_pcaWhiten RESET reset_pcaWhiten STORED false) Q_PROPERTY(int directLDA READ get_directLDA WRITE set_directLDA RESET reset_directLDA STORED false) Q_PROPERTY(float directDrop READ get_directDrop WRITE set_directDrop RESET reset_directDrop STORED false) + Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) BR_PROPERTY(float, pcaKeep, 0.98) BR_PROPERTY(bool, pcaWhiten, false) BR_PROPERTY(int, directLDA, 0) BR_PROPERTY(float, directDrop, 0.1) + BR_PROPERTY(QString, inputVariable, "Label") int dimsOut; Eigen::VectorXf mean; @@ -315,7 +317,7 @@ class LDATransform : public Transform void train(const TemplateList &_trainingSet) { // creates "Label" - TemplateList trainingSet = TemplateList::relabel(_trainingSet, "Subject"); + TemplateList trainingSet = TemplateList::relabel(_trainingSet, inputVariable); int instances = trainingSet.size(); diff --git a/openbr/plugins/frames.cpp b/openbr/plugins/frames.cpp index 74aa978..f5f8609 100644 --- a/openbr/plugins/frames.cpp +++ b/openbr/plugins/frames.cpp @@ -19,7 +19,7 @@ class AggregateFrames : public TimeVaryingTransform TemplateList buffer; public: - AggregateFrames() : TimeVaryingTransform(false) {} + AggregateFrames() : TimeVaryingTransform(false, false) {} private: void train(const TemplateList &data) @@ -35,6 +35,12 @@ private: dst.file = buffer.takeFirst().file; } + void finalize(TemplateList & output) + { + (void) output; + buffer.clear(); + } + void store(QDataStream &stream) const { (void) stream; @@ -48,6 +54,33 @@ private: BR_REGISTER(Transform, AggregateFrames) +/*! + * \ingroup transforms + * \brief Only use one frame every n frames. + * \author Austin Blanton \cite imaus10 + * + * For a video with m frames, DropFrames will pass on m/n frames. + */ +class DropFrames : public UntrainableMetaTransform +{ + Q_OBJECT + Q_PROPERTY(int n READ get_n WRITE set_n RESET reset_n STORED false) + BR_PROPERTY(int, n, 1) + + void project(const TemplateList &src, TemplateList &dst) const + { + if (src.first().file.get("FrameNumber") % n != 0) return; + dst = src; + } + + void project(const Template &src, Template &dst) const + { + (void) src; (void) dst; qFatal("shouldn't be here"); + } +}; + +BR_REGISTER(Transform, DropFrames) + } // namespace br #include "frames.moc" diff --git a/openbr/plugins/gallery.cpp b/openbr/plugins/gallery.cpp index 18f04e0..6c53efd 100644 --- a/openbr/plugins/gallery.cpp +++ b/openbr/plugins/gallery.cpp @@ -71,7 +71,7 @@ class arffGallery : public Gallery } arffFile.write(qPrintable(OpenCVUtils::matrixToStringList(t).join(','))); - arffFile.write(qPrintable(",'" + t.file.get("Subject") + "'\n")); + arffFile.write(qPrintable(",'" + t.file.get("Label") + "'\n")); } }; @@ -643,11 +643,16 @@ class dbGallery : public Gallery query = query.mid(1, query.size()-2); if (!q.exec(query)) qFatal("%s.", qPrintable(q.lastError().text())); + if ((q.record().count() == 0) || (q.record().count() > 3)) qFatal("Query record expected one to three fields, got %d.", q.record().count()); const bool hasMetadata = (q.record().count() >= 2); const bool hasFilter = (q.record().count() >= 3); + QString labelName = "Label"; + if (q.record().count() >= 2) + labelName = q.record().fieldName(1); + // subset = seed:subjectMaxSize:numSubjects:subjectMinSize or // subset = seed:{Metadata,...,Metadata}:numSubjects int seed = 0, subjectMaxSize = std::numeric_limits::max(), numSubjects = std::numeric_limits::max(), subjectMinSize = 0; @@ -673,6 +678,7 @@ class dbGallery : public Gallery QHash > entries; // QHash > while (q.next()) { if (hasFilter && (seed >= 0) && (qHash(q.value(2).toString()) % 2 != (uint)seed % 2)) continue; // Ensures training and testing filters don't overlap + if (metadataFields.isEmpty()) entries[hasMetadata ? q.value(1).toString() : ""].append(QPair(q.value(0).toString(), hasFilter ? q.value(2).toString() : "")); else @@ -707,8 +713,10 @@ class dbGallery : public Gallery if (entryList.size() > subjectMaxSize) std::random_shuffle(entryList.begin(), entryList.end()); - foreach (const Entry &entry, entryList.mid(0, subjectMaxSize)) - templates.append(File(entry.first, label)); + foreach (const Entry &entry, entryList.mid(0, subjectMaxSize)) { + templates.append(File(entry.first)); + templates.last().file.set(labelName, label); + } numSubjects--; } } @@ -816,7 +824,7 @@ class statGallery : public Gallery void write(const Template &t) { - subjects.insert(t.file.get("Subject")); + subjects.insert(t.file.get("Label")); bytes.append(t.bytes()); } }; @@ -862,6 +870,50 @@ class FDDBGallery : public Gallery BR_REGISTER(Gallery, FDDBGallery) +/*! + * \ingroup galleries + * \brief Text format for associating anonymous landmarks with images. + * \author Josh Klontz \cite jklontz + * + * \code + * file_name:x1,y1,x2,y2,...,xn,yn + * file_name:x1,y1,x2,y2,...,xn,yn + * ... + * file_name:x1,y1,x2,y2,...,xn,yn + * \endcode + */ +class landmarksGallery : public Gallery +{ + Q_OBJECT + + TemplateList readBlock(bool *done) + { + *done = true; + TemplateList templates; + foreach (const QString &line, QtUtils::readLines(file)) { + const QStringList words = line.split(':'); + if (words.size() != 2) qFatal("Expected exactly one ':' in: %s.", qPrintable(line)); + File file(words[0]); + const QList vals = QtUtils::toFloats(words[1].split(',')); + if (vals.size() % 2 != 0) qFatal("Expected an even number of comma-separated values."); + QList points; points.reserve(vals.size()/2); + for (int i=0; iinstallEventFilter(this); + } - GUIProxy() +public slots: + void showImage(const QPixmap & input) { - window = NULL; + pixmap = input; + + show(); + setPixmap(pixmap); + setFixedSize(input.size()); } + bool eventFilter(QObject * obj, QEvent * event) { if (event->type() == QEvent::KeyPress) @@ -92,41 +99,10 @@ public: return QList(); } - -public slots: - - void showImage(const QPixmap & input) - { - pixmap = input; - - window->show(); - window->setPixmap(pixmap); - window->setFixedSize(input.size()); - } - - void createWindow() - { - delete window; - QApplication::instance()->removeEventFilter(this); - - window = new QLabel(); - window->setVisible(true); - - QApplication::instance()->installEventFilter(this); - Qt::WindowFlags flags = window->windowFlags(); - - flags = flags & ~Qt::WindowCloseButtonHint; - window->setWindowFlags(flags); - } - }; -class LandmarkProxy : public GUIProxy +class PointMarkingWindow : public DisplayWindow { - Q_OBJECT - -public: - bool eventFilter(QObject *obj, QEvent *event) { if (event->type() == QEvent::MouseButtonPress) @@ -144,11 +120,11 @@ public: painter.setBrush(Qt::red); foreach(const QPointF &point, points) painter.drawEllipse(point, 4, 4); - window->setPixmap(pixmapBuffer); + setPixmap(pixmapBuffer); return true; } else { - return GUIProxy::eventFilter(obj, event); + return DisplayWindow::eventFilter(obj, event); } } @@ -156,14 +132,130 @@ public: { points.clear(); - GUIProxy::waitForKey(); + DisplayWindow::waitForKey(); return points; } private: - QList points; + + +}; + +class PromptWindow : public DisplayWindow +{ + bool eventFilter(QObject * obj, QEvent * event) + { + if (event->type() == QEvent::KeyPress) + { + event->accept(); + + QKeyEvent * key_event = dynamic_cast (event); + if (key_event == NULL) { + qDebug("failed to donwcast key event"); + return true; + } + + QString text = key_event->text(); + + text =text.toLower(); + if (text == "y" || text == "n") + { + gotString = key_event->text(); + wait.wakeAll(); + } + else qDebug("Please answer y/n"); + + return true; + } else { + return QObject::eventFilter(obj, event); + } + } + +public: + QString waitForKeyPress() + { + QMutexLocker locker(&lock); + wait.wait(&lock); + + return gotString; + } + +private: + QString gotString; + + +}; + + +// I want a template class that doesn't look like a template class +class NominalCreation +{ +public: + virtual ~NominalCreation() {} + virtual void creation()=0; +}; + +// Putting the template on a subclass means we can maintain a pointer that +// doesn't include T in its type. +template +class ActualCreation : public NominalCreation +{ +public: + T * basis; + + void creation() + { + basis = new T(); + } +}; + +// We want to create a QLabel subclass on the main thread, but are running in another thread. +// We cannot move QWidget subclasses to a different thread (obviously that would be crazy), but +// we can create one of these, and move it to the main thread, and then use it to create the object +// we want. +// Additional fact: QObject subclasses cannot be template classes. +class MainThreadCreator : public QObject +{ + Q_OBJECT +public: + + MainThreadCreator() + { + this->moveToThread(QApplication::instance()->thread()); + + connect(this, SIGNAL(needCreation()), this, SLOT(createThing()), Qt::BlockingQueuedConnection); + } + + // While this cannot be a template class, it can still have a template method. + template + T * getItem() + { + if (QThread::currentThread() == QApplication::instance()->thread()) + return new T(); + + ActualCreation * actualWorker; + actualWorker = new ActualCreation (); + worker = actualWorker; + + emit needCreation(); + + T * output = actualWorker->basis; + delete actualWorker; + return output; + } + + NominalCreation * worker; + +signals: + void needCreation(); + +public slots: + void createThing() + { + worker->creation(); + } }; /*! @@ -184,14 +276,14 @@ public: ShowTransform() : TimeVaryingTransform(false, false) { - gui = NULL; displayBuffer = NULL; + window = NULL; } ~ShowTransform() { - delete gui; delete displayBuffer; + delete window; } void train(const TemplateList &data) { (void) data; } @@ -235,7 +327,7 @@ public: // Blocking wait for a key-press if (this->waitInput) - gui->waitForKey(); + window->waitForKey(); } } @@ -249,33 +341,36 @@ public: void init() { + initActual(); + } + + template + void initActual() + { if (!Globals->useGui) return; + if (displayBuffer) + delete displayBuffer; displayBuffer = new QPixmap(); - // Create our GUI proxy - gui = new GUIProxy(); - // Move it to the main thread, this means signals we send to it will - // be run in the main thread, which is hopefully in an event loop - gui->moveToThread(QApplication::instance()->thread()); + if (window) + delete window; - // Connect our signals to the proxy's slots - connect(this, SIGNAL(needWindow()), gui, SLOT(createWindow()), Qt::BlockingQueuedConnection); - connect(this, SIGNAL(updateImage(QPixmap)), gui,SLOT(showImage(QPixmap))); - - emit needWindow(); - connect(this, SIGNAL(changeTitle(QString)), gui->window, SLOT(setWindowTitle(QString))); - connect(this, SIGNAL(hideWindow()), gui->window, SLOT(hide())); + window = creator.getItem(); + // Connect our signals to the window's slots + connect(this, SIGNAL(updateImage(QPixmap)), window,SLOT(showImage(QPixmap))); + connect(this, SIGNAL(changeTitle(QString)), window, SLOT(setWindowTitle(QString))); + connect(this, SIGNAL(hideWindow()), window, SLOT(hide())); } protected: - GUIProxy * gui; + MainThreadCreator creator; + DisplayWindow * window; QImage qImageBuffer; QPixmap * displayBuffer; signals: - void needWindow(); void updateImage(const QPixmap & input); void changeTitle(const QString & input); void hideWindow(); @@ -312,7 +407,7 @@ public: // Blocking wait for a key-press if (this->waitInput) { - QList points = gui->waitForKey(); + QList points = window->waitForKey(); if (keys.isEmpty()) dst[i].file.appendPoints(points); else { if (keys.size() == points.size()) @@ -326,28 +421,73 @@ public: void init() { - if (!Globals->useGui) + initActual(); + } +}; + +BR_REGISTER(Transform, ManualTransform) + + +/*! + * \ingroup transforms + * \brief Display an image, and asks a yes/no question about it + * \author Charles Otto \cite caotto + */ +class SurveyTransform : public ShowTransform +{ + Q_OBJECT + +public: + Q_PROPERTY(QString question READ get_question WRITE set_question RESET reset_question STORED false) + BR_PROPERTY(QString, question, "Yes/No") + + Q_PROPERTY(QString propertyName READ get_propertyName WRITE set_propertyName RESET reset_propertyName STORED false) + BR_PROPERTY(QString, propertyName, "answer") + + + void projectUpdate(const TemplateList &src, TemplateList &dst) + { + if (Globals->parallelism > 1) + qFatal("SurveyTransform cannot execute in parallel."); + + dst = src; + + if (src.empty()) return; - displayBuffer = new QPixmap(); + for (int i = 0; i < dst.size(); i++) { + foreach(const cv::Mat &m, dst[i]) { + qImageBuffer = toQImage(m); + displayBuffer->convertFromImage(qImageBuffer); - // Create our GUI proxy - gui = new LandmarkProxy(); + emit updateImage(displayBuffer->copy(displayBuffer->rect())); - // Move it to the main thread, this means signals we send to it will - // be run in the main thread, which is hopefully in an event loop - gui->moveToThread(QApplication::instance()->thread()); + // Blocking wait for a key-press + if (this->waitInput) { + QString answer = p_window->waitForKeyPress(); - // Connect our signals to the proxy's slots - connect(this, SIGNAL(needWindow()), gui, SLOT(createWindow()), Qt::BlockingQueuedConnection); - connect(this, SIGNAL(updateImage(QPixmap)), gui,SLOT(showImage(QPixmap))); + dst[i].file.set(this->propertyName, answer); + } + } + } + } + PromptWindow * p_window; - emit needWindow(); - connect(this, SIGNAL(hideWindow()), gui->window, SLOT(hide())); + + void init() + { + if (!Globals->useGui) + return; + + initActual(); + p_window = (PromptWindow *) window; + + emit changeTitle(this->question); } }; -BR_REGISTER(Transform, ManualTransform) +BR_REGISTER(Transform, SurveyTransform) + class FPSLimit : public TimeVaryingTransform { diff --git a/openbr/plugins/hist.cpp b/openbr/plugins/hist.cpp index b3efd8b..1896ae0 100644 --- a/openbr/plugins/hist.cpp +++ b/openbr/plugins/hist.cpp @@ -52,8 +52,11 @@ class HistTransform : public UntrainableTransform int histSize[] = {dims}; float range[] = {min, max}; const float* ranges[] = {range}; - Mat hist; - calcHist(&mv[i], 1, channels, Mat(), hist, 1, histSize, ranges); + Mat hist, chan = mv[i]; + // calcHist requires F or U, might as well convert just in case + if (mv[i].depth() != CV_8U || mv[i].depth() == CV_32F) + mv[i].convertTo(chan, CV_32F); + calcHist(&chan, 1, channels, Mat(), hist, 1, histSize, ranges); memcpy(m.ptr(i), hist.ptr(), dims * sizeof(float)); } diff --git a/openbr/plugins/independent.cpp b/openbr/plugins/independent.cpp index 123c979..4a1ec03 100644 --- a/openbr/plugins/independent.cpp +++ b/openbr/plugins/independent.cpp @@ -9,37 +9,36 @@ using namespace cv; namespace br { -static TemplateList Downsample(const TemplateList &templates, const Transform *transform) +static TemplateList Downsample(const TemplateList &templates, int classes, int instances, float fraction, const QString & inputVariable) { // Return early when no downsampling is required - if ((transform->classes == std::numeric_limits::max()) && - (transform->instances == std::numeric_limits::max()) && - (transform->fraction >= 1)) + if ((classes == std::numeric_limits::max()) && + (instances == std::numeric_limits::max()) && + (fraction >= 1)) return templates; - const bool atLeast = transform->instances < 0; - const int instances = abs(transform->instances); + const bool atLeast = instances < 0; + instances = abs(instances); - QList allLabels = File::get(templates, "Subject"); + QList allLabels = File::get(templates, inputVariable); QList uniqueLabels = allLabels.toSet().toList(); qSort(uniqueLabels); - QMap counts = templates.countValues("Subject", instances != std::numeric_limits::max()); + QMap counts = templates.countValues(inputVariable, instances != std::numeric_limits::max()); - if ((instances != std::numeric_limits::max()) && (transform->classes != std::numeric_limits::max())) + if ((instances != std::numeric_limits::max()) && (classes != std::numeric_limits::max())) foreach (const QString & label, counts.keys()) if (counts[label] < instances) counts.remove(label); uniqueLabels = counts.keys(); - if ((transform->classes != std::numeric_limits::max()) && (uniqueLabels.size() < transform->classes)) - qWarning("Downsample requested %d classes but only %d are available.", transform->classes, uniqueLabels.size()); + if ((classes != std::numeric_limits::max()) && (uniqueLabels.size() < classes)) + qWarning("Downsample requested %d classes but only %d are available.", classes, uniqueLabels.size()); - Common::seedRNG(); QList selectedLabels = uniqueLabels; - if (transform->classes < uniqueLabels.size()) { + if (classes < uniqueLabels.size()) { std::random_shuffle(selectedLabels.begin(), selectedLabels.end()); - selectedLabels = selectedLabels.mid(0, transform->classes); + selectedLabels = selectedLabels.mid(0, classes); } TemplateList downsample; @@ -56,14 +55,45 @@ static TemplateList Downsample(const TemplateList &templates, const Transform *t downsample.append(templates.value(indices[j])); } - if (transform->fraction < 1) { + if (fraction < 1) { std::random_shuffle(downsample.begin(), downsample.end()); - downsample = downsample.mid(0, downsample.size()*transform->fraction); + downsample = downsample.mid(0, downsample.size()*fraction); } return downsample; } +class DownsampleTrainingTransform : public Transform +{ + Q_OBJECT + Q_PROPERTY(br::Transform* transform READ get_transform WRITE set_transform RESET reset_transform STORED true) + Q_PROPERTY(int classes READ get_classes WRITE set_classes RESET reset_classes STORED false) + Q_PROPERTY(int instances READ get_instances WRITE set_instances RESET reset_instances STORED false) + Q_PROPERTY(float fraction READ get_fraction WRITE set_fraction RESET reset_fraction STORED false) + Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) + BR_PROPERTY(br::Transform*, transform, NULL) + BR_PROPERTY(int, classes, std::numeric_limits::max()) + BR_PROPERTY(int, instances, std::numeric_limits::max()) + BR_PROPERTY(float, fraction, 1) + BR_PROPERTY(QString, inputVariable, "Label") + + void project(const Template & src, Template & dst) const + { + transform->project(src,dst); + } + + + void train(const TemplateList &data) + { + if (!transform || !transform->trainable) + return; + + TemplateList downsampled = Downsample(data, classes, instances, fraction, inputVariable); + transform->train(downsampled); + } +}; +BR_REGISTER(Transform, DownsampleTrainingTransform) + /*! * \ingroup transforms * \brief Clones the transform so that it can be applied independently. @@ -124,13 +154,10 @@ class IndependentTransform : public MetaTransform while (transforms.size() < templatesList.size()) transforms.append(transform->clone()); - for (int i=0; i futures; for (int i=0; i +#include #include #include "openbr_internal.h" #include "openbr/core/opencvutils.h" @@ -178,6 +179,33 @@ BR_REGISTER(Transform, SIFTDescriptorTransform) /*! * \ingroup transforms + * \brief OpenCV HOGDescriptor wrapper + * \author Austin Blanton \cite imaus10 + */ +class HoGDescriptorTransform : public UntrainableTransform +{ + Q_OBJECT + + HOGDescriptor hog; + + void project(const Template &src, Template &dst) const + { + std::vector descriptorVals; + std::vector locations; + Size winStride = Size(0,0); + Size padding = Size(0,0); + foreach (const Mat &rect, src) { + hog.compute(rect, descriptorVals, winStride, padding, locations); + Mat HoGFeats(descriptorVals, true); + dst += HoGFeats; + } + } +}; + +BR_REGISTER(Transform, HoGDescriptorTransform) + +/*! + * \ingroup transforms * \brief Add landmarks to the template in a grid layout * \author Josh Klontz \cite jklontz */ diff --git a/openbr/plugins/mask.cpp b/openbr/plugins/mask.cpp index 8b86221..81fa8fb 100644 --- a/openbr/plugins/mask.cpp +++ b/openbr/plugins/mask.cpp @@ -158,6 +158,9 @@ class LargestConvexAreaTransform : public UntrainableTransform { Q_OBJECT + Q_PROPERTY(QString outputVariable READ get_outputVariable WRITE set_outputVariable RESET reset_outputVariable STORED false) + BR_PROPERTY(QString, outputVariable, "Label") + void project(const Template &src, Template &dst) const { std::vector< std::vector > contours; @@ -171,7 +174,7 @@ class LargestConvexAreaTransform : public UntrainableTransform if (area / hullArea > 0.98) maxArea = std::max(maxArea, area); } - dst.file.set("Label", maxArea); + dst.file.set(outputVariable, maxArea); } }; diff --git a/openbr/plugins/meta.cpp b/openbr/plugins/meta.cpp index b5e14cb..b4cc865 100644 --- a/openbr/plugins/meta.cpp +++ b/openbr/plugins/meta.cpp @@ -105,6 +105,24 @@ class PipeTransform : public CompositeTransform transforms[i]->train(copy); } + // if the transform is time varying, we can't project it in parallel + if (transforms[i]->timeVarying()) { + fprintf(stderr, "\n%s projecting...", qPrintable(transforms[i]->objectName())); + for (int j=0; j < singleItemLists.size();j++) + transforms[i]->projectUpdate(singleItemLists[j], singleItemLists[j]); + + // advance i since we already projected for this stage. + i++; + + // set up copy again + copy.clear(); + for (int j=0; j < singleItemLists.size(); j++) + copy.append(singleItemLists[j]); + + // the next stage might be trainable, so continue to evaluate it. + continue; + } + // We project through any subsequent untrainable transforms at once // as a memory optimization in case any of these intermediate // transforms allocate a lot of memory (like OpenTransform) @@ -112,7 +130,8 @@ class PipeTransform : public CompositeTransform // by that transform at once if we can avoid it. int nextTrainableTransform = i+1; while ((nextTrainableTransform < transforms.size()) && - !transforms[nextTrainableTransform]->trainable) + !transforms[nextTrainableTransform]->trainable && + !transforms[nextTrainableTransform]->timeVarying()) nextTrainableTransform++; fprintf(stderr, " projecting..."); diff --git a/openbr/plugins/misc.cpp b/openbr/plugins/misc.cpp index 567fa0d..a799968 100644 --- a/openbr/plugins/misc.cpp +++ b/openbr/plugins/misc.cpp @@ -179,6 +179,10 @@ class FirstTransform : public UntrainableMetaTransform void project(const Template &src, Template &dst) const { + // AggregateFrames will leave the Template empty + // if it hasn't filled up the buffer + // so we gotta anticipate an empty Template + if (src.empty()) return; dst.file = src.file; dst = src.m(); } diff --git a/openbr/plugins/normalize.cpp b/openbr/plugins/normalize.cpp index 5c85415..842e952 100644 --- a/openbr/plugins/normalize.cpp +++ b/openbr/plugins/normalize.cpp @@ -97,6 +97,7 @@ class CenterTransform : public Transform Q_OBJECT Q_ENUMS(Method) Q_PROPERTY(Method method READ get_method WRITE set_method RESET reset_method STORED false) + Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) public: /*!< */ @@ -107,6 +108,7 @@ public: private: BR_PROPERTY(Method, method, Mean) + BR_PROPERTY(QString, inputVariable, "Label") Mat a, b; // dst = (src - b) / a @@ -127,7 +129,7 @@ private: Mat m; OpenCVUtils::toMat(data.data()).convertTo(m, CV_64F); - const QList labels = data.indexProperty("Subject"); + const QList labels = data.indexProperty(inputVariable); const int dims = m.cols; vector mv, av, bv; diff --git a/openbr/plugins/openbr_internal.h b/openbr/plugins/openbr_internal.h index fb7a1b5..27136da 100644 --- a/openbr/plugins/openbr_internal.h +++ b/openbr/plugins/openbr_internal.h @@ -170,14 +170,8 @@ public: virtual void project(const TemplateList &src, TemplateList &dst) const { if (timeVarying()) { - if (!this->timeInvariantAlias) { - QMutexLocker lock(&aliasLock); - CompositeTransform * non_const = const_cast(this); - non_const->timeInvariantAlias = non_const->smartCopy(); - non_const->timeInvariantAlias->setParent(non_const); - lock.unlock(); - } - timeInvariantAlias->projectUpdate(src,dst); + CompositeTransform * non_const = const_cast(this); + non_const->projectUpdate(src,dst); return; } _project(src, dst); @@ -225,10 +219,6 @@ public: } output->file = this->file; - output->classes = classes; - output->instances = instances; - output->fraction = fraction; - output->init(); return output; @@ -237,13 +227,10 @@ public: protected: bool isTimeVarying; - mutable QMutex aliasLock; - Transform * timeInvariantAlias; - virtual void _project(const Template & src, Template & dst) const = 0; virtual void _project(const TemplateList & src, TemplateList & dst) const = 0; - CompositeTransform() : TimeVaryingTransform(false) { timeInvariantAlias = NULL; } + CompositeTransform() : TimeVaryingTransform(false) {} }; } diff --git a/openbr/plugins/opticalflow.cpp b/openbr/plugins/opticalflow.cpp new file mode 100644 index 0000000..5246ad5 --- /dev/null +++ b/openbr/plugins/opticalflow.cpp @@ -0,0 +1,57 @@ +#include +#include "openbr_internal.h" +#include "openbr/core/opencvutils.h" + +using namespace cv; + +namespace br +{ + +/*! + * \ingroup transforms + * \brief Gets a one-channel dense optical flow from two images + * \author Austin Blanton \cite imaus10 + */ +class OpticalFlowTransform : public UntrainableMetaTransform +{ + Q_OBJECT + Q_PROPERTY(double pyr_scale READ get_pyr_scale WRITE set_pyr_scale RESET reset_pyr_scale STORED false) + Q_PROPERTY(int levels READ get_levels WRITE set_levels RESET reset_levels STORED false) + Q_PROPERTY(int winsize READ get_winsize WRITE set_winsize RESET reset_winsize STORED false) + Q_PROPERTY(int iterations READ get_iterations WRITE set_iterations RESET reset_iterations STORED false) + Q_PROPERTY(int poly_n READ get_poly_n WRITE set_poly_n RESET reset_poly_n STORED false) + Q_PROPERTY(double poly_sigma READ get_poly_sigma WRITE set_poly_sigma RESET reset_poly_sigma STORED false) + Q_PROPERTY(int flags READ get_flags WRITE set_flags RESET reset_flags STORED false) + // these defaults are optimized for KTH + BR_PROPERTY(double, pyr_scale, 0.1) + BR_PROPERTY(int, levels, 1) + BR_PROPERTY(int, winsize, 5) + BR_PROPERTY(int, iterations, 10) + BR_PROPERTY(int, poly_n, 7) + BR_PROPERTY(double, poly_sigma, 1.1) + BR_PROPERTY(int, flags, 0) + + void project(const Template &src, Template &dst) const + { + // get the two images put there by AggregateFrames + if (src.size() < 2) return; + Mat prevImg = src[0], nextImg = src[1], flow, flowOneCh; + if (src[0].channels() != 1) OpenCVUtils::cvtGray(src[0], prevImg); + if (src[1].channels() != 1) OpenCVUtils::cvtGray(src[1], nextImg); + calcOpticalFlowFarneback(prevImg, nextImg, flow, pyr_scale, levels, winsize, iterations, poly_n, poly_sigma, flags); + + // the result is two channels + std::vector channels(2); + split(flow, channels); + magnitude(channels[0], channels[1], flowOneCh); + + dst += flowOneCh; + dst.file = src.file; + } +}; + +BR_REGISTER(Transform, OpticalFlowTransform) + +} // namespace br + +#include "opticalflow.moc" diff --git a/openbr/plugins/output.cpp b/openbr/plugins/output.cpp index 2b75582..e71adc1 100644 --- a/openbr/plugins/output.cpp +++ b/openbr/plugins/output.cpp @@ -146,8 +146,8 @@ class meltOutput : public MatrixOutput QStringList lines; if (file.baseName() != "terminal") lines.append(QString("Query,Target,Mask,Similarity%1").arg(keys)); - QList queryLabels = File::get(queryFiles, "Subject"); - QList targetLabels = File::get(targetFiles, "Subject"); + QList queryLabels = File::get(queryFiles, "Label"); + QList targetLabels = File::get(targetFiles, "Label"); for (int i=0; i("Subject")); + lines.append(file.name + " " + file.get("Label")); QtUtils::writeFile(file, lines); } }; @@ -428,7 +428,7 @@ class rankOutput : public MatrixOutput foreach (const Pair &pair, Common::Sort(OpenCVUtils::matrixToVector(data.row(i)), true)) { if (Globals->crossValidate > 0 ? (targetFiles[pair.second].get("Partition",-1) == queryFiles[i].get("Partition",-1)) : true) { if (QString(targetFiles[pair.second]) != QString(queryFiles[i])) { - if (targetFiles[pair.second].get("Subject") == queryFiles[i].get("Subject")) { + if (targetFiles[pair.second].get("Label") == queryFiles[i].get("Label")) { ranks.append(rank); positions.append(pair.second); scores.append(pair.first); diff --git a/openbr/plugins/quality.cpp b/openbr/plugins/quality.cpp index fd3209c..e372460 100644 --- a/openbr/plugins/quality.cpp +++ b/openbr/plugins/quality.cpp @@ -19,17 +19,20 @@ class ImpostorUniquenessMeasureTransform : public Transform Q_PROPERTY(br::Distance* distance READ get_distance WRITE set_distance RESET reset_distance STORED false) Q_PROPERTY(double mean READ get_mean WRITE set_mean RESET reset_mean) Q_PROPERTY(double stddev READ get_stddev WRITE set_stddev RESET reset_stddev) + Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) BR_PROPERTY(br::Distance*, distance, Distance::make("Dist(L2)", this)) BR_PROPERTY(double, mean, 0) BR_PROPERTY(double, stddev, 1) + BR_PROPERTY(QString, inputVariable, "Label") + TemplateList impostors; float calculateIUM(const Template &probe, const TemplateList &gallery) const { - const QString probeLabel = probe.file.get("Subject"); + const QString probeLabel = probe.file.get(inputVariable); TemplateList subset = gallery; for (int j=subset.size()-1; j>=0; j--) - if (subset[j].file.get("Subject") == probeLabel) + if (subset[j].file.get(inputVariable) == probeLabel) subset.removeAt(j); QList scores = distance->compare(subset, probe); @@ -151,6 +154,7 @@ class MatchProbabilityDistance : public Distance Q_PROPERTY(br::Distance* distance READ get_distance WRITE set_distance RESET reset_distance STORED false) Q_PROPERTY(bool gaussian READ get_gaussian WRITE set_gaussian RESET reset_gaussian STORED false) Q_PROPERTY(bool crossModality READ get_crossModality WRITE set_crossModality RESET reset_crossModality STORED false) + Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) MP mp; @@ -158,7 +162,7 @@ class MatchProbabilityDistance : public Distance { distance->train(src); - const QList labels = src.indexProperty("Subject"); + const QList labels = src.indexProperty(inputVariable); QScopedPointer matrixOutput(MatrixOutput::make(FileList(src.size()), FileList(src.size()))); distance->compare(src, src, matrixOutput.data()); @@ -201,6 +205,7 @@ protected: BR_PROPERTY(br::Distance*, distance, make("Dist(L2)")) BR_PROPERTY(bool, gaussian, true) BR_PROPERTY(bool, crossModality, false) + BR_PROPERTY(QString, inputVariable, "Label") }; BR_REGISTER(Distance, MatchProbabilityDistance) @@ -217,10 +222,12 @@ class HeatMapDistance : public Distance Q_PROPERTY(bool gaussian READ get_gaussian WRITE set_gaussian RESET reset_gaussian STORED false) Q_PROPERTY(bool crossModality READ get_crossModality WRITE set_crossModality RESET reset_crossModality STORED false) Q_PROPERTY(int step READ get_step WRITE set_step RESET reset_step STORED false) + Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) BR_PROPERTY(br::Distance*, distance, make("Dist(L2)")) BR_PROPERTY(bool, gaussian, true) BR_PROPERTY(bool, crossModality, false) BR_PROPERTY(int, step, 1) + BR_PROPERTY(QString, inputVariable, "Label") QList mp; @@ -228,7 +235,7 @@ class HeatMapDistance : public Distance { distance->train(src); - const QList labels = src.indexProperty("Subject"); + const QList labels = src.indexProperty(inputVariable); QList patches; @@ -307,14 +314,16 @@ class UnitDistance : public Distance Q_PROPERTY(br::Distance *distance READ get_distance WRITE set_distance RESET reset_distance) Q_PROPERTY(float a READ get_a WRITE set_a RESET reset_a) Q_PROPERTY(float b READ get_b WRITE set_b RESET reset_b) + Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) BR_PROPERTY(br::Distance*, distance, make("Dist(L2)")) BR_PROPERTY(float, a, 1) BR_PROPERTY(float, b, 0) + BR_PROPERTY(QString, inputVariable, "Label") void train(const TemplateList &templates) { const TemplateList samples = templates.mid(0, 2000); - const QList sampleLabels = samples.indexProperty("Subject"); + const QList sampleLabels = samples.indexProperty(inputVariable); QScopedPointer matrixOutput(MatrixOutput::make(FileList(samples.size()), FileList(samples.size()))); Distance::compare(samples, samples, matrixOutput.data()); diff --git a/openbr/plugins/quantize.cpp b/openbr/plugins/quantize.cpp index 94840f6..e74513f 100644 --- a/openbr/plugins/quantize.cpp +++ b/openbr/plugins/quantize.cpp @@ -120,6 +120,10 @@ BR_REGISTER(Transform, HistEqQuantizationTransform) class BayesianQuantizationDistance : public Distance { Q_OBJECT + + Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) + BR_PROPERTY(QString, inputVariable, "Label") + QVector loglikelihoods; static void computeLogLikelihood(const Mat &data, const QList &labels, float *loglikelihood) @@ -150,7 +154,7 @@ class BayesianQuantizationDistance : public Distance qFatal("Expected sigle matrix templates of type CV_8UC1!"); const Mat data = OpenCVUtils::toMat(src.data()); - const QList templateLabels = src.indexProperty("Subject"); + const QList templateLabels = src.indexProperty(inputVariable); loglikelihoods = QVector(data.cols*256, 0); QFutureSynchronizer futures; @@ -343,9 +347,11 @@ class ProductQuantizationTransform : public Transform Q_PROPERTY(int n READ get_n WRITE set_n RESET reset_n STORED false) Q_PROPERTY(br::Distance *distance READ get_distance WRITE set_distance RESET reset_distance STORED false) Q_PROPERTY(bool bayesian READ get_bayesian WRITE set_bayesian RESET reset_bayesian STORED false) + Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) BR_PROPERTY(int, n, 2) BR_PROPERTY(br::Distance*, distance, Distance::make("L2", this)) BR_PROPERTY(bool, bayesian, false) + BR_PROPERTY(QString, inputVariable, "Label") quint16 index; QList centers; @@ -474,7 +480,7 @@ private: Mat data = OpenCVUtils::toMat(src.data()); const int step = getStep(data.cols); - const QList labels = src.indexProperty("Subject"); + const QList labels = src.indexProperty(inputVariable); Mat &lut = ProductQuantizationLUTs[index]; lut = Mat(getDims(data.cols), 256*(256+1)/2, CV_32FC1); diff --git a/openbr/plugins/quantize2.cpp b/openbr/plugins/quantize2.cpp index 63ea769..8d05523 100644 --- a/openbr/plugins/quantize2.cpp +++ b/openbr/plugins/quantize2.cpp @@ -19,6 +19,10 @@ namespace br class BayesianQuantizationTransform : public Transform { Q_OBJECT + + Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) + BR_PROPERTY(QString, inputVariable, "Label") + QVector thresholds; static void computeThresholdsRecursive(const QVector &cumulativeGenuines, const QVector &cumulativeImpostors, @@ -77,7 +81,7 @@ class BayesianQuantizationTransform : public Transform void train(const TemplateList &src) { const Mat data = OpenCVUtils::toMat(src.data()); - const QList labels = src.indexProperty("Subject"); + const QList labels = src.indexProperty(inputVariable); thresholds = QVector(256*data.cols); diff --git a/openbr/plugins/regions.cpp b/openbr/plugins/regions.cpp index 4197b95..e6c30fa 100644 --- a/openbr/plugins/regions.cpp +++ b/openbr/plugins/regions.cpp @@ -114,7 +114,7 @@ BR_REGISTER(Transform, CatTransform) /*! * \ingroup transforms * \brief Concatenates all input matrices by row into a single matrix. - * All matricies must have the same row counts. + * All matricies must have the same column counts. * \author Josh Klontz \cite jklontz */ class CatRowsTransform : public UntrainableMetaTransform @@ -131,6 +131,31 @@ BR_REGISTER(Transform, CatRowsTransform) /*! * \ingroup transforms + * \brief Concatenates all input matrices by column into a single matrix. + * Use after a fork to concatenate two feature matrices by column. + * \author Austin Blanton \cite imaus10 + */ +class CatColsTransform : public UntrainableMetaTransform +{ + Q_OBJECT + + void project(const Template &src, Template &dst) const + { + if (src.empty()) return; + dst.file = src.file; + Mat m = OpenCVUtils::toMatByRow(src); + // right now this just splits src in half and joins them horizontally + // TODO: add partitions parameter for more than a single split + Mat first = m.rowRange(Range(0, m.rows/2)); + Mat second = m.rowRange(Range(m.rows/2, m.rows)); + hconcat(first, second, dst); + } +}; + +BR_REGISTER(Transform, CatColsTransform) + +/*! + * \ingroup transforms * \brief Reshape the each matrix to the specified number of rows. * \author Josh Klontz \cite jklontz */ diff --git a/openbr/plugins/stream.cpp b/openbr/plugins/stream.cpp index e02092c..553393f 100644 --- a/openbr/plugins/stream.cpp +++ b/openbr/plugins/stream.cpp @@ -31,8 +31,10 @@ public: virtual ~SharedBuffer() {} virtual void addItem(FrameData * input)=0; + virtual void reset()=0; virtual FrameData * tryGetItem()=0; + virtual int size()=0; }; // for n - 1 boundaries, multiple threads call addItem, the frames are @@ -74,6 +76,21 @@ public: return output; } + virtual int size() + { + QMutexLocker lock(&bufferGuard); + return buffer.size(); + } + virtual void reset() + { + if (size() != 0) + qDebug("Sequencing buffer has non-zero size during reset!"); + + QMutexLocker lock(&bufferGuard); + next_target = 0; + } + + private: QMutex bufferGuard; int next_target; @@ -95,6 +112,11 @@ public: outputBuffer = &buffer2; } + int size() + { + QReadLocker readLock(&bufferGuard); + return inputBuffer->size() + outputBuffer->size(); + } // called from the producer thread void addItem(FrameData * input) @@ -133,6 +155,13 @@ public: return output; } + virtual void reset() + { + if (this->size() != 0) + qDebug("Shared buffer has non-zero size during reset!"); + } + + private: // The read-write lock. The thread adding to this buffer can add // to the current input buffer if it has a read lock. The thread @@ -194,14 +223,10 @@ public: // Try to get a FrameData from the pool, if we can't it means too many // frames are already out, and we will return NULL to indicate failure FrameData * aFrame = allFrames.tryGetItem(); - if (aFrame == NULL) { + if (aFrame == NULL) return NULL; - } - aFrame->data.clear(); - aFrame->sequenceNumber = -1; - - // Try to read a frame, if this returns false the data source is broken + // Try to actually read a frame, if this returns false the data source is broken bool res = getNext(*aFrame); // The datasource broke, update final_frame @@ -211,12 +236,15 @@ public: final_frame = lookAhead.back()->sequenceNumber; allFrames.addItem(aFrame); } - else lookAhead.push_back(aFrame); + else { + lookAhead.push_back(aFrame); + } + // we will return the first frame on the lookAhead buffer FrameData * rVal = lookAhead.first(); lookAhead.pop_front(); - + // If this is the last frame, say so if (rVal->sequenceNumber == final_frame) { last_frame = true; is_broken = true; @@ -231,6 +259,8 @@ public: { int frameNumber = inputFrame->sequenceNumber; + inputFrame->data.clear(); + inputFrame->sequenceNumber = -1; allFrames.addItem(inputFrame); bool rval = false; @@ -239,6 +269,7 @@ public: if (frameNumber == final_frame) { // We just received the last frame, better pulse + allReturned = true; lastReturned.wakeAll(); rval = true; } @@ -246,15 +277,24 @@ public: return rval; } - void waitLast() + bool waitLast() { QMutexLocker lock(&last_frame_update); - lastReturned.wait(&last_frame_update); + + while (!allReturned) + { + // This would be a safer wait if we used a timeout, but + // theoretically that should never matter. + lastReturned.wait(&last_frame_update); + } + return true; } bool open(Template & output, int start_index = 0) { is_broken = false; + allReturned = false; + // The last frame isn't initialized yet final_frame = -1; // Start our sequence numbers from the input index @@ -282,19 +322,34 @@ public: bool res = getNext(*firstFrame); // the data source broke already, we couldn't even get one frame - // from it. + // from it even though it claimed to have opened successfully. if (!res) { is_broken = true; return false; } + // We read one frame ahead of the last one returned, this allows + // us to know which frame is the final frame when we return it. lookAhead.append(firstFrame); return true; } + /* + * Pure virtual methods + */ + + // isOpen doesn't appear to particularly work when used on opencv + // VideoCaptures, so we don't use it for anything important. virtual bool isOpen()=0; + // Called from open, open the data source specified by the input + // template, don't worry about setting any of the state variables + // set in open. virtual bool concreteOpen(Template & output) = 0; + // Get the next frame from the data source, store the results in + // FrameData (including the actual frame and appropriate sequence + // number). virtual bool getNext(FrameData & input) = 0; + // close the currently open data source. virtual void close() = 0; int next_sequence_number; @@ -302,6 +357,7 @@ protected: DoubleBuffer allFrames; int final_frame; bool is_broken; + bool allReturned; QList lookAhead; QWaitCondition lastReturned; @@ -317,6 +373,11 @@ public: bool concreteOpen(Template &input) { basis = input; + + // We can open either files (well actually this includes addresses of ip cameras + // through ffmpeg), or webcams. Webcam VideoCaptures are created through a separate + // overload of open that takes an integer, not a string. + // So, does this look like an integer? bool is_int = false; int anInt = input.file.name.toInt(&is_int); if (is_int) @@ -334,7 +395,8 @@ public: } else { // Yes, we should specify absolute path: // http://stackoverflow.com/questions/9396459/loading-a-video-in-opencv-in-python - video.open(QFileInfo(input.file.name).absoluteFilePath().toStdString()); + QString fileName = (Globals->path.isEmpty() ? "" : Globals->path + "/") + input.file.name; + video.open(QFileInfo(fileName).absoluteFilePath().toStdString()); } return video.isOpened(); @@ -349,8 +411,10 @@ public: private: bool getNext(FrameData & output) { - if (!isOpen()) + if (!isOpen()) { + qDebug("video source is not open"); return false; + } output.data.append(Template(basis.file)); output.data.last().m() = cv::Mat(); @@ -362,6 +426,7 @@ private: bool res = video.read(temp); if (!res) { + // The video capture broke, return false. output.data.last().m() = cv::Mat(); close(); return false; @@ -391,6 +456,8 @@ public: data_ok = false; } + // To "open" it we just set appropriate indices, we assume that if this + // is an image, it is already loaded into memory. bool concreteOpen(Template &input) { basis = input; @@ -440,7 +507,7 @@ private: class DataSourceManager : public DataSource { public: - DataSourceManager() : DataSource(500) + DataSourceManager(int activeFrames=100) : DataSource(activeFrames) { actualSource = NULL; } @@ -450,6 +517,11 @@ public: close(); } + int size() + { + return this->allFrames.size(); + } + void close() { if (actualSource) { @@ -459,29 +531,40 @@ public: } } + // We are used through a call to open(TemplateList) bool open(TemplateList & input) { + // Set up variables specific to us current_template_idx = 0; templates = input; + // Call datasourece::open on the first template to set up + // state variables return DataSource::open(templates[current_template_idx]); } + // Create an actual data source of appropriate type for this template + // (initially called via the call to DataSource::open, called later + // as we run out of frames on our templates). bool concreteOpen(Template & input) { close(); + bool open_res = false; // Input has no matrices? Its probably a video that hasn't been loaded yet if (input.empty()) { actualSource = new VideoDataSource(0); - actualSource->concreteOpen(input); + open_res = actualSource->concreteOpen(input); } + // If the input is not empty, we assume it is a set of frames already + // in memory. else { - // create frame dealer actualSource = new TemplateDataSource(0); - actualSource->concreteOpen(input); + open_res = actualSource->concreteOpen(input); } - if (!isOpen()) { + + // The data source failed to open + if (!open_res) { delete actualSource; actualSource = NULL; return false; @@ -497,12 +580,16 @@ protected: TemplateList templates; DataSource * actualSource; + // Get the next frame, if we run out of frames on the current template + // move on to the next one. bool getNext(FrameData & output) { bool res = actualSource->getNext(output); output.sequenceNumber = next_sequence_number; + // OK we got a frame if (res) { + // Override the sequence number set by actualSource output.data.last().file.set("FrameNumber", output.sequenceNumber); next_sequence_number++; if (output.data.last().last().empty()) @@ -510,7 +597,7 @@ protected: return true; } - + // We didn't get a frame, try to move on to the next template. while(!res) { output.data.clear(); current_template_idx++; @@ -521,12 +608,16 @@ protected: // open the next data source bool open_res = concreteOpen(templates[current_template_idx]); + // We couldn't open it, give up? We could maybe continue here + // but don't currently. if (!open_res) return false; - // get a frame from it + // get a frame from the newly opened data source, if that fails + // we continue to open the next one. res = actualSource->getNext(output); } + // Finally, set the sequence number for the frame we actually return. output.sequenceNumber = next_sequence_number++; output.data.last().file.set("FrameNumber", output.sequenceNumber); @@ -573,6 +664,9 @@ public: int stage_id; virtual void reset()=0; + + virtual void status()=0; + protected: int thread_count; @@ -606,7 +700,8 @@ class MultiThreadStage : public ProcessingStage public: MultiThreadStage(int _input) : ProcessingStage(_input) {} - + // Not much to worry about here, we will project the input + // and try to continue to the next stage. FrameData * run(FrameData * input, bool & should_continue) { if (input == NULL) { @@ -620,7 +715,8 @@ public: return input; } - // Called from a different thread than run + // Called from a different thread than run. Nothing to worry about + // we offer no restrictions on when loops may enter this stage. virtual bool tryAcquireNextStage(FrameData *& input) { (void) input; @@ -631,6 +727,9 @@ public: { // nothing to do. } + void status(){ + qDebug("multi thread stage %d, nothing to worry about", this->stage_id); + } }; class SingleThreadStage : public ProcessingStage @@ -640,13 +739,18 @@ public: { currentStatus = STOPPING; next_target = 0; + // If the previous stage is single-threaded, queued inputs + // are stored in a double buffer if (input_variance) { this->inputBuffer = new DoubleBuffer(); } + // If it's multi-threaded we need to put the inputs back in order + // before we can use them, so we use a sequencing buffer. else { this->inputBuffer = new SequencingBuffer(); } } + ~SingleThreadStage() { delete inputBuffer; @@ -657,6 +761,7 @@ public: QWriteLocker writeLock(&statusLock); currentStatus = STOPPING; next_target = 0; + inputBuffer->reset(); } @@ -706,7 +811,13 @@ public: next->stages = stages; next->start_idx = this->stage_id; next->startItem = newItem; - this->threads->start(next, stages->size() - stage_id); + + // We start threads with priority equal to their stage id + // This is intended to ensure progression, we do queued late stage + // jobs before queued early stage jobs, and so tend to finish frames + // rather than go stage by stage. In Qt 5.1, priorities are priorities + // so we use the stage_id directly. + this->threads->start(next, stage_id); } @@ -741,45 +852,50 @@ public: return true; } + + void status(){ + qDebug("single thread stage %d, status starting? %d, next %d buffer size %d", this->stage_id, this->currentStatus == SingleThreadStage::STARTING, this->next_target, this->inputBuffer->size()); + } + }; -// No input buffer, instead we draw templates from some data source -// Will be operated by the main thread for the stream. starts threads +// This stage reads new frames from the data source. class FirstStage : public SingleThreadStage { public: - FirstStage() : SingleThreadStage(true) {} + FirstStage(int activeFrames = 100) : SingleThreadStage(true), dataSource(activeFrames){ } DataSourceManager dataSource; + void reset() + { + dataSource.close(); + SingleThreadStage::reset(); + } + FrameData * run(FrameData * input, bool & should_continue) { - // Try to get a frame from the datasource + if (input == NULL) + qFatal("NULL frame in input stage"); + + // Can we enter the next stage? + should_continue = nextStage->tryAcquireNextStage(input); + + // Try to get a frame from the datasource, we keep working on + // the frame we have, but we will queue another job for the next + // frame if a frame is currently available. QWriteLocker lock(&statusLock); bool last_frame = false; - input = dataSource.tryGetFrame(last_frame); + FrameData * newFrame = dataSource.tryGetFrame(last_frame); - // Datasource broke, or is currently out of frames? - if (!input || last_frame) - { - // We will just stop and not continue. + // Were we able to get a frame? + if (newFrame) startThread(newFrame); + // If not this stage will enter a stopped state. + else { currentStatus = STOPPING; - if (!input) { - should_continue = false; - return NULL; - } } - lock.unlock(); - // Can we enter the next stage? - should_continue = nextStage->tryAcquireNextStage(input); - // We are exiting leaving this stage, should we start another - // thread here? Normally we will always re-queue a thread on - // the first stage, but if we received the last frame there is - // no need to. - if (!last_frame) { - startThread(NULL); - } + lock.unlock(); return input; } @@ -797,31 +913,48 @@ public: } QReadLocker lock(&statusLock); - // A thread is already in the first stage, - // we should just return + // If the first stage is already active we will just end. if (currentStatus == STARTING) { return false; } - // Have to change to a write lock to modify currentStatus + + // Otherwise we will try to continue, but to do so we have to + // escalate the lock, and sadly there is no way to do so without + // releasing the read-mode lock, and getting a new write-mode lock. lock.unlock(); QWriteLocker writeLock(&statusLock); - // But someone else might have started a thread in the meantime + // currentStatus might have changed in the gap between releasing the read + // lock and getting the write lock. if (currentStatus == STARTING) { return false; } - // Ok we'll start a thread + + bool last_frame = false; + // Try to get a frame from the data source, if we get one we will + // continue to the first stage. + input = dataSource.tryGetFrame(last_frame); + + if (!input) { + return false; + } + currentStatus = STARTING; - // We always start a readstage thread with null input, so nothing to do here return true; } + void status(){ + qDebug("Read stage %d, status starting? %d, next frame %d buffer size %d", this->stage_id, this->currentStatus == SingleThreadStage::STARTING, this->next_target, this->dataSource.size()); + } + + }; -// starts threads +// Appened to the end of a Stream's transform sequence. Collects the output +// from each frame on a single templatelist class LastStage : public SingleThreadStage { public: @@ -834,6 +967,7 @@ public: private: TemplateList collectedOutput; public: + void reset() { collectedOutput.clear(); @@ -873,6 +1007,11 @@ public: return input; } + + void status(){ + qDebug("Collection stage %d, status starting? %d, next %d buffer size %d", this->stage_id, this->currentStatus == SingleThreadStage::STARTING, this->next_target, this->inputBuffer->size()); + } + }; @@ -880,9 +1019,12 @@ class StreamTransform : public CompositeTransform { Q_OBJECT public: + Q_PROPERTY(int activeFrames READ get_activeFrames WRITE set_activeFrames RESET reset_activeFrames) + BR_PROPERTY(int, activeFrames, 100) void train(const TemplateList & data) { + qFatal("Stream train is currently not implemented."); foreach(Transform * transform, transforms) { transform->train(data); } @@ -902,7 +1044,8 @@ public: qFatal("whatever"); } - // start processing + // start processing, consider all templates in src a continuous + // 'video' void projectUpdate(const TemplateList & src, TemplateList & dst) { dst = src; @@ -911,12 +1054,22 @@ public: if (!res) return; // Start the first thread in the stream. + QWriteLocker lock(&readStage->statusLock); readStage->currentStatus = SingleThreadStage::STARTING; - readStage->startThread(NULL); - // Wait for the stream to reach the last frame available from + // We have to get a frame before starting the thread + bool last_frame = false; + FrameData * firstFrame = readStage->dataSource.tryGetFrame(last_frame); + if (firstFrame == NULL) + qFatal("Failed to read first frame of video"); + + readStage->startThread(firstFrame); + lock.unlock(); + + // Wait for the stream to process the last frame available from // the data source. - readStage->dataSource.waitLast(); + bool wait_res = false; + wait_res = readStage->dataSource.waitLast(); // Now that there are no more incoming frames, call finalize // on each transform in turn to collect any last templates @@ -958,6 +1111,10 @@ public: { if (transforms.isEmpty()) return; + // call CompositeTransform::init so that trainable is set + // correctly. + CompositeTransform::init(); + // We share a thread pool across streams attached to the same // parent tranform, retrieve or create a thread pool based // on our parent transform. @@ -971,24 +1128,31 @@ public: threads = it.value(); poolLock.unlock(); + // Are our children time varying or not? This decides whether + // we run them in single threaded or multi threaded stages stage_variance.reserve(transforms.size()); foreach (const br::Transform *transform, transforms) { stage_variance.append(transform->timeVarying()); } - readStage = new FirstStage(); + // Additionally, we have a separate stage responsible for reading + // frames from the data source + readStage = new FirstStage(activeFrames); processingStages.push_back(readStage); readStage->stage_id = 0; readStage->stages = &this->processingStages; readStage->threads = this->threads; + // Initialize and link a processing stage for each of our child + // transforms. int next_stage_id = 1; - bool prev_stage_variance = true; for (int i =0; i < transforms.size(); i++) { if (stage_variance[i]) + // Whether or not the previous stage is multi-threaded controls + // the type of input buffer we need in a single threaded stage. processingStages.append(new SingleThreadStage(prev_stage_variance)); else processingStages.append(new MultiThreadStage(Globals->parallelism)); @@ -1006,20 +1170,25 @@ public: prev_stage_variance = stage_variance[i]; } + // We also have the last stage, which just puts the output of the + // previous stages on a template list. collectionStage = new LastStage(prev_stage_variance); processingStages.append(collectionStage); collectionStage->stage_id = next_stage_id; collectionStage->stages = &this->processingStages; collectionStage->threads = this->threads; + // the last transform stage points to collection stage processingStages[processingStages.size() - 2]->nextStage = collectionStage; - // It's a ring buffer, get it? + // And the collection stage points to the read stage, because this is + // a ring buffer. collectionStage->nextStage = readStage; } ~StreamTransform() { + // Delete all the stages for (int i = 0; i < processingStages.size(); i++) { delete processingStages[i]; } @@ -1033,6 +1202,21 @@ protected: QList processingStages; + // This is a map from parent transforms (of Streams) to thread pools. Rather + // than starting threads on the global thread pool, Stream uses separate thread pools + // keyed on their parent transform. This is necessary because stream's project starts + // threads, then enters an indefinite wait for them to finish. Since we are starting + // threads using thread pools, threads themselves are a limited resource. Therefore, + // the type of hold and wait done by stream project can lead to deadlock unless + // resources are ordered in such a way that a circular wait will not occur. The points + // of this hash is to introduce a resource ordering (on threads) that mirrors the structure + // of the algorithm. So, as long as the structure of the algorithm is a DAG, the wait done + // by stream project will not be circular, since every thread in stream project is waiting + // for threads at a lower level to do the work. + // This issue doesn't come up in distribute, since a thread waiting on a QFutureSynchronizer + // will steal work from those jobs, so in that sense distribute isn't doing a hold and wait. + // Waiting for a QFutureSynchronzier isn't really possible here since stream runs an indeteriminate + // number of jobs. static QHash pools; static QMutex poolsAccess; QThreadPool * threads; diff --git a/openbr/plugins/svm.cpp b/openbr/plugins/svm.cpp index 39efa4d..d4e2aad 100644 --- a/openbr/plugins/svm.cpp +++ b/openbr/plugins/svm.cpp @@ -101,6 +101,8 @@ class SVMTransform : public Transform Q_PROPERTY(Type type READ get_type WRITE set_type RESET reset_type STORED false) Q_PROPERTY(float C READ get_C WRITE set_C RESET reset_C STORED false) Q_PROPERTY(float gamma READ get_gamma WRITE set_gamma RESET reset_gamma STORED false) + Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) + Q_PROPERTY(QString outputVariable READ get_outputVariable WRITE set_outputVariable RESET reset_outputVariable STORED false) public: enum Kernel { Linear = CvSVM::LINEAR, @@ -119,6 +121,9 @@ private: BR_PROPERTY(Type, type, C_SVC) BR_PROPERTY(float, C, -1) BR_PROPERTY(float, gamma, -1) + BR_PROPERTY(QString, inputVariable, "") + BR_PROPERTY(QString, outputVariable, "") + SVM svm; QHash labelMap; @@ -128,14 +133,15 @@ private: { Mat data = OpenCVUtils::toMat(_data.data()); Mat lab; - // If we are doing regression, assume subject has float values + // If we are doing regression, the input variable should have float + // values if (type == EPS_SVR || type == NU_SVR) { - lab = OpenCVUtils::toMat(File::get(_data, "Subject")); + lab = OpenCVUtils::toMat(File::get(_data, inputVariable)); } - // If we are doing classification, assume subject has discrete values, map them - // and store the mapping data + // If we are doing classification, we should be dealing with discrete + // values. Map them and store the mapping data else { - QList dataLabels = _data.indexProperty("Subject", labelMap, reverseLookup); + QList dataLabels = _data.indexProperty(inputVariable, labelMap, reverseLookup); lab = OpenCVUtils::toMat(dataLabels); } trainSVM(svm, data, lab, kernel, type, C, gamma); @@ -146,9 +152,9 @@ private: dst = src; float prediction = svm.predict(src.m().reshape(1, 1)); if (type == EPS_SVR || type == NU_SVR) - dst.file.set("Subject", prediction); + dst.file.set(outputVariable, prediction); else - dst.file.set("Subject", reverseLookup[prediction]); + dst.file.set(outputVariable, reverseLookup[prediction]); } void store(QDataStream &stream) const @@ -162,6 +168,24 @@ private: loadSVM(svm, stream); stream >> labelMap >> reverseLookup; } + + void init() + { + // Since SVM can do regression or classification, we have to check the problem type before + // specifying target variable names + if (inputVariable.isEmpty()) + { + if (type == EPS_SVR || type == NU_SVR) { + inputVariable = "Regressor"; + if (outputVariable.isEmpty()) + outputVariable = "Regressand"; + } + else + inputVariable = "Label"; + } + if (outputVariable.isEmpty()) + outputVariable = inputVariable; + } }; BR_REGISTER(Transform, SVMTransform) @@ -178,6 +202,8 @@ class SVMDistance : public Distance Q_ENUMS(Type) Q_PROPERTY(Kernel kernel READ get_kernel WRITE set_kernel RESET reset_kernel STORED false) Q_PROPERTY(Type type READ get_type WRITE set_type RESET reset_type STORED false) + Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) + public: enum Kernel { Linear = CvSVM::LINEAR, @@ -194,13 +220,14 @@ public: private: BR_PROPERTY(Kernel, kernel, Linear) BR_PROPERTY(Type, type, EPS_SVR) + BR_PROPERTY(QString, inputVariable, "Label") SVM svm; void train(const TemplateList &src) { const Mat data = OpenCVUtils::toMat(src.data()); - const QList lab = src.indexProperty("Subject"); + const QList lab = src.indexProperty(inputVariable); const int instances = data.rows * (data.rows+1) / 2; Mat deltaData(instances, data.cols, data.type()); diff --git a/openbr/plugins/validate.cpp b/openbr/plugins/validate.cpp index 8b3a174..f2f7b96 100644 --- a/openbr/plugins/validate.cpp +++ b/openbr/plugins/validate.cpp @@ -147,26 +147,29 @@ class MetadataDistance : public Distance float compare(const Template &a, const Template &b) const { foreach (const QString &key, filters) { + QString aValue = a.file.get(key, QString()); + QString bValue = b.file.get(key, QString()); - const QString aValue = a.file.get(key, ""); - const QString bValue = b.file.get(key, ""); + // The query value may be a range. Let's check. + if (bValue.isEmpty()) bValue = QtUtils::toString(b.file.get(key, QPointF())); if (aValue.isEmpty() || bValue.isEmpty()) continue; bool keep = false; + bool ok; - if (aValue[0] == '(') /* Range */ { - QStringList values = aValue.split(','); + QPointF range = QtUtils::toPoint(bValue,&ok); - int value = values[0].mid(1).toInt(); - values[1].chop(1); - int upperBound = values[1].toInt(); + if (ok) /* Range */ { + int value = range.x(); + int upperBound = range.y(); while (value <= upperBound) { - if (aValue == bValue) { + if (aValue == QString::number(value)) { keep = true; break; } + value++; } } else if (aValue == bValue) keep = true; diff --git a/scripts/evalAgeRegression-PCSO.sh b/scripts/evalAgeRegression-PCSO.sh index 8d0f286..9e294fe 100755 --- a/scripts/evalAgeRegression-PCSO.sh +++ b/scripts/evalAgeRegression-PCSO.sh @@ -4,8 +4,12 @@ if [ ! -f evalAgeRegression-PCSO.sh ]; then exit fi +export BR="../build/app/br/br -useGui 0" +export PCSO_DIR=/user/pripshare/Databases/FaceDatabases/PCSO/PCSO/ +export ageAlg=AgeRegression + # Create a file list by querying the database -br -quiet -algorithm Identity -enroll "../data/PCSO/PCSO.db[query='SELECT File,Age,PersonID FROM PCSO WHERE Age >= 15 AND AGE <= 75', subset=1:200]" terminal.txt > Input.txt +$BR -quiet -algorithm Identity -enroll "$PCSO_DIR/PCSO.db[query='SELECT File,Age,PersonID FROM PCSO WHERE Age >= 17 AND AGE <= 68', subset=1:200]" terminal.txt > Input.txt # Enroll the file list and evaluate performance -br -algorithm AgeRegression -path ../data/PCSO/img -enroll Input.txt Output.txt -evalRegression Output.txt Input.txt +$BR -algorithm $ageAlg -path $PCSO_DIR/Images -enroll Input.txt Output.txt -evalRegression Output.txt Input.txt Age diff --git a/scripts/evalFaceRecognition-MEDS.sh b/scripts/evalFaceRecognition-MEDS.sh index 231370f..673f674 100755 --- a/scripts/evalFaceRecognition-MEDS.sh +++ b/scripts/evalFaceRecognition-MEDS.sh @@ -20,11 +20,11 @@ if [ ! -e Algorithm_Dataset ]; then fi if [ ! -e MEDS.mask ]; then - br -makeMask ../data/MEDS/sigset/MEDS_frontal_target.xml ../data/MEDS/sigset/MEDS_frontal_query.xml MEDS.mask + br -useGui 0 -makeMask ../data/MEDS/sigset/MEDS_frontal_target.xml ../data/MEDS/sigset/MEDS_frontal_query.xml MEDS.mask fi # Run Algorithm on MEDS -br -algorithm ${ALGORITHM} -path ../data/MEDS/img -compare ../data/MEDS/sigset/MEDS_frontal_target.xml ../data/MEDS/sigset/MEDS_frontal_query.xml ${ALGORITHM}_MEDS.mtx -eval ${ALGORITHM}_MEDS.mtx MEDS.mask Algorithm_Dataset/${ALGORITHM}_MEDS.csv +br -useGui 0 -algorithm ${ALGORITHM} -path ../data/MEDS/img -compare ../data/MEDS/sigset/MEDS_frontal_target.xml ../data/MEDS/sigset/MEDS_frontal_query.xml ${ALGORITHM}_MEDS.mtx -eval ${ALGORITHM}_MEDS.mtx MEDS.mask Algorithm_Dataset/${ALGORITHM}_MEDS.csv # Plot results -br -plot Algorithm_Dataset/*_MEDS.csv MEDS +br -useGui 0 -plot Algorithm_Dataset/*_MEDS.csv MEDS diff --git a/scripts/evalGenderClassification-PCSO.sh b/scripts/evalGenderClassification-PCSO.sh index 881214e..9373645 100755 --- a/scripts/evalGenderClassification-PCSO.sh +++ b/scripts/evalGenderClassification-PCSO.sh @@ -4,8 +4,13 @@ if [ ! -f evalGenderClassification-PCSO.sh ]; then exit fi +export BR=../build/app/br/br +export genderAlg=GenderClassification + +export PCSO_DIR=/user/pripshare/Databases/FaceDatabases/PCSO/PCSO/ + # Create a file list by querying the database -br -quiet -algorithm Identity -enroll "../data/PCSO/PCSO.db[query='SELECT File,Gender,PersonID FROM PCSO', subset=1:8000]" terminal.txt > Input.txt +$BR -useGui 0 -quiet -algorithm Identity -enroll "$PCSO_DIR/PCSO.db[query='SELECT File,Gender,PersonID FROM PCSO', subset=1:8000]" terminal.txt > Input.txt # Enroll the file list and evaluate performance -br -algorithm GenderClassification -path ../data/PCSO/img -enroll Input.txt Output.txt -evalClassification Output.txt Input.txt +$BR -useGui 0 -algorithm $genderAlg -path $PCSO_DIR/Images -enroll Input.txt Output.txt -evalClassification Output.txt Input.txt Gender \ No newline at end of file diff --git a/scripts/trainAgeRegression-PCSO.sh b/scripts/trainAgeRegression-PCSO.sh index c063055..c1f3ef1 100755 --- a/scripts/trainAgeRegression-PCSO.sh +++ b/scripts/trainAgeRegression-PCSO.sh @@ -6,6 +6,11 @@ fi #rm -f ../share/openbr/models/features/FaceClassificationRegistration #rm -f ../share/openbr/models/features/FaceClassificationExtraction -rm -f ../share/openbr/models/algorithms/AgeRegression +#rm -f ../share/openbr/models/algorithms/AgeRegression -br -algorithm AgeRegression -path ../data/PCSO/Images -train "../data/PCSO/PCSO.db[query='SELECT File,Age,PersonID FROM PCSO WHERE Age >= 15 AND AGE <= 75', subset=0:200]" ../share/openbr/models/algorithms/AgeRegression +export BR=../build/app/br/br +export ageAlg=AgeRegression + +export PCSO_DIR=/user/pripshare/Databases/FaceDatabases/PCSO/PCSO/ + +$BR -useGui 0 -algorithm $ageAlg -path $PCSO_DIR/Images -train "$PCSO_DIR/PCSO.db[query='SELECT File,Age,PersonID FROM PCSO WHERE Age >= 17 AND AGE <= 68', subset=0:200]" ../share/openbr/models/algorithms/AgeRegression diff --git a/scripts/trainFaceRecognition-PCSO.sh b/scripts/trainFaceRecognition-PCSO.sh index e67b3aa..ebeb056 100755 --- a/scripts/trainFaceRecognition-PCSO.sh +++ b/scripts/trainFaceRecognition-PCSO.sh @@ -8,6 +8,13 @@ fi #rm -f ../share/openbr/models/features/FaceRecognitionExtraction #rm -f ../share/openbr/models/features/FaceRecognitionEmbedding #rm -f ../share/openbr/models/features/FaceRecognitionQuantization -rm -f ../share/openbr/models/algorithms/FaceRecognition +#rm -f ../share/openbr/models/algorithms/FaceRecognition + +export BR=../build/app/br/br + +export PCSO_DIR=/user/pripshare/Databases/FaceDatabases/PCSO/PCSO/ + + + +$BR -useGui 0 -algorithm FaceRecognition -path "$PCSO_DIR/Images/" -train "$PCSO_DIR/PCSO.db[query='SELECT File,PersonID as Label,PersonID FROM PCSO', subset=0:5:6000]" ../share/openbr/models/algorithms/FaceRecognition -br -algorithm FaceRecognition -path ../data/PCSO/img -train "../data/PCSO/PCSO.db[query='SELECT File,'S'||PersonID,PersonID FROM PCSO', subset=0:5:6000]" ../share/openbr/models/algorithms/FaceRecognition diff --git a/scripts/trainGenderClassification-PCSO.sh b/scripts/trainGenderClassification-PCSO.sh index 91fae04..acb8146 100755 --- a/scripts/trainGenderClassification-PCSO.sh +++ b/scripts/trainGenderClassification-PCSO.sh @@ -6,6 +6,11 @@ fi #rm -f ../share/openbr/models/features/FaceClassificationRegistration #rm -f ../share/openbr/models/features/FaceClassificationExtraction -rm -f ../share/openbr/models/algorithms/GenderClassification +#rm -f ../share/openbr/models/algorithms/GenderClassification -br -algorithm GenderClassification -path ../data/PCSO/Images -train "../data/PCSO/PCSO.db[query='SELECT File,Gender,PersonID FROM PCSO', subset=0:8000]" ../share/openbr/models/algorithms/GenderClassification +export BR=../build/app/br/br +export genderAlg=GenderClassification + +export PCSO_DIR=/user/pripshare/Databases/FaceDatabases/PCSO/PCSO/ + +$BR -useGui 0 -algorithm $genderAlg -path $PCSO_DIR/Images -train "$PCSO_DIR/PCSO.db[query='SELECT File,Gender,PersonID FROM PCSO', subset=0:8000]" ../share/openbr/models/algorithms/GenderClassification