diff --git a/.gitignore b/.gitignore index de489d6..25abe9f 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ scripts/results ### QtCreator ### *CMakeLists.txt.user* +*.autosave ### R ### *.RData diff --git a/CHANGELOG.md b/CHANGELOG.md index 21efc83..c56439e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ 0.4.0 - ??/??/?? ================ * Added -evalDetection and -plotDetection for evaluating and plotting object detection accuracy (#9) +* Deprecated Transform::backProject 0.3.0 - 5/22/13 =============== diff --git a/README.md b/README.md index 834cf02..262d664 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,13 @@ -www.openbiometrics.org +**www.openbiometrics.org** $ git clone https://github.com/biometrics/openbr.git $ cd openbr $ git submodule init $ git submodule update + +To optionally check out a particular [tagged release](https://github.com/biometrics/openbr/releases): -[Build Instructions](http://openbiometrics.org/doxygen/latest/installation.html) + $ git checkout + $ git submodule update + +**[Build Instructions](http://openbiometrics.org/doxygen/latest/installation.html)** diff --git a/app/br/br.cpp b/app/br/br.cpp index 8615274..f3e1208 100644 --- a/app/br/br.cpp +++ b/app/br/br.cpp @@ -129,17 +129,20 @@ public: check(parc == 3, "Incorrect parameter count for 'convert'."); br_convert(parv[0], parv[1], parv[2]); } else if (!strcmp(fun, "evalClassification")) { - check(parc == 2, "Incorrect parameter count for 'evalClassification'."); - br_eval_classification(parv[0], parv[1]); + check(parc >= 2 && parc <= 4, "Incorrect parameter count for 'evalClassification'."); + br_eval_classification(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? parv[3] : ""); } else if (!strcmp(fun, "evalClustering")) { check(parc == 2, "Incorrect parameter count for 'evalClustering'."); br_eval_clustering(parv[0], parv[1]); } else if (!strcmp(fun, "evalDetection")) { check((parc >= 2) && (parc <= 3), "Incorrect parameter count for 'evalDetection'."); br_eval_detection(parv[0], parv[1], parc == 3 ? parv[2] : ""); + } else if (!strcmp(fun, "evalLandmarking")) { + check((parc >= 2) && (parc <= 3), "Incorrect parameter count for 'evalLandmarking'."); + br_eval_detection(parv[0], parv[1], parc == 3 ? parv[2] : ""); } else if (!strcmp(fun, "evalRegression")) { - check(parc == 2, "Incorrect parameter count for 'evalRegression'."); - br_eval_regression(parv[0], parv[1]); + check(parc >= 2 && parc <= 4, "Incorrect parameter count for 'evalRegression'."); + br_eval_regression(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? parv[3] : ""); } else if (!strcmp(fun, "plotDetection")) { check(parc >= 2, "Incorrect parameter count for 'plotDetection'."); br_plot_detection(parc-1, parv, parv[parc-1], true); @@ -215,10 +218,11 @@ private: "-combineMasks ... {mask} (And|Or)\n" "-cat ... {gallery}\n" "-convert (Format|Gallery|Output) {output_file}\n" - "-evalClassification \n" + "-evalClassification \n" "-evalClustering \n" "-evalDetection [{csv}]\n" - "-evalRegression \n" + "-evalLandmarking [{csv}]\n" + "-evalRegression \n" "-plotDetection ... {destination}\n" "-plotMetadata ... \n" "-getHeader \n" diff --git a/app/examples/age_estimation.cpp b/app/examples/age_estimation.cpp index c2a90e5..2958b71 100644 --- a/app/examples/age_estimation.cpp +++ b/app/examples/age_estimation.cpp @@ -29,7 +29,7 @@ static void printTemplate(const br::Template &t) { - printf("%s age: %d\n", qPrintable(t.file.fileName()), int(t.file.get("Subject"))); + printf("%s age: %d\n", qPrintable(t.file.fileName()), int(t.file.get("Age"))); } int main(int argc, char *argv[]) diff --git a/app/examples/face_recognition_evaluation.cpp b/app/examples/face_recognition_evaluation.cpp index ebf6462..745f522 100644 --- a/app/examples/face_recognition_evaluation.cpp +++ b/app/examples/face_recognition_evaluation.cpp @@ -25,8 +25,7 @@ * -compare target.gal query.gal scores.mtx \ * -makeMask ../data/MEDS/sigset/MEDS_frontal_target.xml ../data/MEDS/sigset/MEDS_frontal_query.xml MEDS.mask \ * -eval scores.mtx MEDS.mask Algorithm_Dataset/FaceRecognition_MEDS.csv \ - * -eval ../data/MEDS/simmat/COTS_MEDS.mtx MEDS.mask Algorithm_Dataset/COTS_MEDS.csv \ - * -plot Algorithm_Dataset/FaceRecognition_MEDS.csv Algorithm_Dataset/COTS_MEDS.csv MEDS + * -plot Algorithm_Dataset/FaceRecognition_MEDS.csv MEDS * \endcode */ diff --git a/app/examples/gender_estimation.cpp b/app/examples/gender_estimation.cpp index dd4ebd2..d2a824d 100644 --- a/app/examples/gender_estimation.cpp +++ b/app/examples/gender_estimation.cpp @@ -29,7 +29,7 @@ static void printTemplate(const br::Template &t) { - printf("%s gender: %s\n", qPrintable(t.file.fileName()), qPrintable(t.file.get("Subject"))); + printf("%s gender: %s\n", qPrintable(t.file.fileName()), qPrintable(t.file.get("Gender"))); } int main(int argc, char *argv[]) diff --git a/openbr/core/bee.cpp b/openbr/core/bee.cpp index c6cb254..9639657 100644 --- a/openbr/core/bee.cpp +++ b/openbr/core/bee.cpp @@ -99,10 +99,10 @@ void BEE::writeSigset(const QString &sigset, const br::FileList &files, bool ign QStringList metadata; if (!ignoreMetadata) foreach (const QString &key, file.localKeys()) { - if ((key == "Index") || (key == "Subject")) continue; + if ((key == "Index") || (key == "Label")) continue; metadata.append(key+"=\""+QtUtils::toString(file.value(key))+"\""); } - lines.append("\t("Subject",file.fileName()) +"\">"); + lines.append("\t("Label",file.fileName()) +"\">"); lines.append("\t\t"); lines.append("\t"); } @@ -266,10 +266,11 @@ void BEE::makeMask(const QString &targetInput, const QString &queryInput, const cv::Mat BEE::makeMask(const br::FileList &targets, const br::FileList &queries, int partition) { - // Would like to use indexProperty for this, but didn't make a version of that for Filelist yet - // -cao - QList targetLabels = File::get(targets, "Subject", "-1"); - QList queryLabels = File::get(queries, "Subject", "-1"); + // Direct use of "Label" isn't general, also would prefer to use indexProperty, rather than + // doing string comparisons (but that isn't implemented yet for FileList) -cao + QList targetLabels = File::get(targets, "Label", "-1"); + QList queryLabels = File::get(queries, "Label", "-1"); + QList targetPartitions = targets.crossValidationPartitions(); QList queryPartitions = queries.crossValidationPartitions(); diff --git a/openbr/core/cluster.cpp b/openbr/core/cluster.cpp index 6a6b954..8c880d9 100644 --- a/openbr/core/cluster.cpp +++ b/openbr/core/cluster.cpp @@ -279,8 +279,8 @@ void br::EvalClustering(const QString &csv, const QString &input) qDebug("Evaluating %s against %s", qPrintable(csv), qPrintable(input)); // We assume clustering algorithms store assigned cluster labels as integers (since the clusters are - // not named). - QList labels = File::get(TemplateList::fromGallery(input), "Subject"); + // not named). Direct use of ClusterID is not general -cao + QList labels = File::get(TemplateList::fromGallery(input), "ClusterID"); QHash labelToIndex; int nClusters = 0; diff --git a/openbr/core/common.cpp b/openbr/core/common.cpp index 17584ec..55cbd3e 100644 --- a/openbr/core/common.cpp +++ b/openbr/core/common.cpp @@ -15,11 +15,15 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include "common.h" +#include using namespace std; /**** GLOBAL ****/ void Common::seedRNG() { + static QMutex seedControl; + QMutexLocker lock(&seedControl); + static bool seeded = false; if (!seeded) { srand(0); // We seed with 0 instead of time(NULL) to have reproducible randomness @@ -29,8 +33,6 @@ void Common::seedRNG() { QList Common::RandSample(int n, int max, int min, bool unique) { - seedRNG(); - QList samples; samples.reserve(n); int range = max-min; if (range <= 0) qFatal("Non-positive range."); @@ -50,8 +52,6 @@ QList Common::RandSample(int n, int max, int min, bool unique) QList Common::RandSample(int n, const QSet &values, bool unique) { - seedRNG(); - QList valueList = values.toList(); if (unique && (values.size() <= n)) return valueList; diff --git a/openbr/core/core.cpp b/openbr/core/core.cpp index b781c87..ecdc3b6 100644 --- a/openbr/core/core.cpp +++ b/openbr/core/core.cpp @@ -147,13 +147,7 @@ struct AlgorithmCore data.removeAt(i); const int numFiles = data.size(); - if (Globals->backProject) { - TemplateList backProjectedData; - transform->backProject(data, backProjectedData); - data = backProjectedData; - } else { - data >> *transform; - } + data >> *transform; g->writeBlock(data); const FileList newFiles = data.files(); @@ -394,7 +388,6 @@ void br::Convert(const File &fileType, const File &inputFile, const File &output QSharedPointer o(Factory::make(outputFile)); o->initialize(targetFiles, queryFiles); - qDebug() << m.rows << m.cols << targetFiles.size() << queryFiles.size(); for (int i=0; isetRelative(m.at(i,j), i, j); diff --git a/openbr/core/eval.cpp b/openbr/core/eval.cpp index 13f761a..567c99d 100644 --- a/openbr/core/eval.cpp +++ b/openbr/core/eval.cpp @@ -255,9 +255,20 @@ struct Counter } }; -void EvalClassification(const QString &predictedInput, const QString &truthInput) +void EvalClassification(const QString &predictedInput, const QString &truthInput, QString predictedProperty, QString truthProperty) { qDebug("Evaluating classification of %s against %s", qPrintable(predictedInput), qPrintable(truthInput)); + + if (predictedProperty.isEmpty()) + predictedProperty = "Label"; + // If predictedProperty is specified, but truthProperty isn't, copy over the value from + // predicted property + else if (truthProperty.isEmpty()) + truthProperty = predictedProperty; + + if (truthProperty.isEmpty()) + truthProperty = "Label"; + TemplateList predicted(TemplateList::fromGallery(predictedInput)); TemplateList truth(TemplateList::fromGallery(truthInput)); if (predicted.size() != truth.size()) qFatal("Input size mismatch."); @@ -267,9 +278,8 @@ void EvalClassification(const QString &predictedInput, const QString &truthInput if (predicted[i].file.name != truth[i].file.name) qFatal("Input order mismatch."); - // Typically these lists will be of length one, but this generalization allows measuring multi-class labeling accuracy. - QString predictedSubject = predicted[i].file.get("Subject"); - QString trueSubject = truth[i].file.get("Subject"); + QString predictedSubject = predicted[i].file.get(predictedProperty); + QString trueSubject = truth[i].file.get(truthProperty); QStringList predictedSubjects(predictedSubject); QStringList trueSubjects(trueSubject); @@ -373,6 +383,8 @@ static QStringList computeDetectionResults(const QList &detec } const int keep = qMin(points.size(), Max_Points); + if (keep < 2) qFatal("Insufficient points."); + QStringList lines; lines.reserve(keep); for (int i=0; i &detec return lines; } +QString getDetectKey(const TemplateList &templates) +{ + const File &f = templates.first().file; + foreach (const QString &key, f.localKeys()) + if (!f.get(key, QRectF()).isNull()) + return key; + return ""; +} + float EvalDetection(const QString &predictedInput, const QString &truthInput, const QString &csv) { qDebug("Evaluating detection of %s against %s", qPrintable(predictedInput), qPrintable(truthInput)); @@ -389,20 +410,22 @@ float EvalDetection(const QString &predictedInput, const QString &truthInput, co const TemplateList truth(TemplateList::fromGallery(truthInput)); // Figure out which metadata field contains a bounding box - QString detectKey; - foreach (const QString &key, truth.first().file.localKeys()) - if (!truth.first().file.get(key, QRectF()).isNull()) { - detectKey = key; - break; - } - if (detectKey.isNull()) qFatal("No suitable metadata key found."); - else qDebug("Using metadata key: %s", qPrintable(detectKey)); + QString truthDetectKey = getDetectKey(truth); + if (truthDetectKey.isEmpty()) qFatal("No suitable ground truth metadata key found."); + QString predictedDetectKey = truthDetectKey; + if (predicted.first().file.get(predictedDetectKey, QRectF()).isNull()) + predictedDetectKey = getDetectKey(predicted); + if (predictedDetectKey.isEmpty()) qFatal("No suitable predicted metadata key found."); + + qDebug("Using metadata key: %s%s", + qPrintable(predictedDetectKey), + qPrintable(predictedDetectKey == truthDetectKey ? QString() : "/"+truthDetectKey)); QMap allDetections; // Organized by file, QMap used to preserve order foreach (const Template &t, predicted) - allDetections[t.file.baseName()].predicted.append(Detection(t.file.get(detectKey), t.file.get("Confidence", -1))); + allDetections[t.file.baseName()].predicted.append(Detection(t.file.get(predictedDetectKey), t.file.get("Confidence", -1))); foreach (const Template &t, truth) - allDetections[t.file.baseName()].truth.append(Detection(t.file.get(detectKey))); + allDetections[t.file.baseName()].truth.append(Detection(t.file.get(truthDetectKey))); QList resolvedDetections, falseNegativeDetections; foreach (Detections detections, allDetections.values()) { @@ -453,21 +476,44 @@ float EvalDetection(const QString &predictedInput, const QString &truthInput, co return averageOverlap; } -void EvalRegression(const QString &predictedInput, const QString &truthInput) +void EvalLandmarking(const QString &predictedInput, const QString &truthInput, const QString &csv) +{ + (void) predictedInput; + (void) truthInput; + (void) csv; +} + +void EvalRegression(const QString &predictedInput, const QString &truthInput, QString predictedProperty, QString truthProperty) { qDebug("Evaluating regression of %s against %s", qPrintable(predictedInput), qPrintable(truthInput)); + + if (predictedProperty.isEmpty()) + predictedProperty = "Regressor"; + // If predictedProperty is specified, but truthProperty isn't, copy the value over + // rather than using the default for truthProperty + else if (truthProperty.isEmpty()) + truthProperty = predictedProperty; + + if (truthProperty.isEmpty()) + predictedProperty = "Regressand"; + const TemplateList predicted(TemplateList::fromGallery(predictedInput)); const TemplateList truth(TemplateList::fromGallery(truthInput)); if (predicted.size() != truth.size()) qFatal("Input size mismatch."); float rmsError = 0; + float maeError = 0; QStringList truthValues, predictedValues; for (int i=0; i("Subject")-truth[i].file.get("Subject"), 2.f); - truthValues.append(QString::number(truth[i].file.get("Subject"))); - predictedValues.append(QString::number(predicted[i].file.get("Subject"))); + + float difference = predicted[i].file.get(predictedProperty) - truth[i].file.get(truthProperty); + + rmsError += pow(difference, 2.f); + maeError += fabsf(difference); + truthValues.append(QString::number(truth[i].file.get(truthProperty))); + predictedValues.append(QString::number(predicted[i].file.get(predictedProperty))); } QStringList rSource; @@ -487,6 +533,7 @@ void EvalRegression(const QString &predictedInput, const QString &truthInput) if (success) QtUtils::showFile("EvalRegression.pdf"); qDebug("RMS Error = %f", sqrt(rmsError/predicted.size())); + qDebug("MAE = %f", maeError/predicted.size()); } } // namespace br diff --git a/openbr/core/eval.h b/openbr/core/eval.h index d90d1a5..a3d8266 100644 --- a/openbr/core/eval.h +++ b/openbr/core/eval.h @@ -26,9 +26,10 @@ namespace br float Evaluate(const QString &simmat, const QString &mask = "", const QString &csv = ""); // Returns TAR @ FAR = 0.001 float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const QString &csv = "", int parition = 0); float Evaluate(const cv::Mat &scores, const cv::Mat &masks, const QString &csv = ""); - void EvalClassification(const QString &predictedInput, const QString &truthInput); + void EvalClassification(const QString &predictedInput, const QString &truthInput, QString predictedProperty="", QString truthProperty=""); float EvalDetection(const QString &predictedInput, const QString &truthInput, const QString &csv = ""); // Return average overlap - void EvalRegression(const QString &predictedInput, const QString &truthInput); + void EvalLandmarking(const QString &predictedInput, const QString &truthInput, const QString &csv = ""); + void EvalRegression(const QString &predictedInput, const QString &truthInput, QString predictedProperty="", QString truthProperty=""); } #endif // __EVAL_H diff --git a/openbr/core/qtutils.cpp b/openbr/core/qtutils.cpp index 0fa103e..fe295cf 100644 --- a/openbr/core/qtutils.cpp +++ b/openbr/core/qtutils.cpp @@ -334,8 +334,8 @@ QRectF QtUtils::toRect(const QString &string, bool *ok) bool okX, okY, okWidth, okHeight; x = words[0].toFloat(&okX); y = words[1].toFloat(&okY); - width = words[0].toFloat(&okWidth); - height = words[1].toFloat(&okHeight); + width = words[2].toFloat(&okWidth); + height = words[3].toFloat(&okHeight); if (okX && okY && okWidth && okHeight) { if (ok) *ok = true; return QRectF(x, y, width, height); diff --git a/openbr/core/resource.h b/openbr/core/resource.h index ae18d36..a620969 100644 --- a/openbr/core/resource.h +++ b/openbr/core/resource.h @@ -24,6 +24,7 @@ #include #include #include +#include template class ResourceMaker @@ -52,7 +53,7 @@ public: : resourceMaker(rm) , availableResources(new QList()) , lock(new QMutex()) - , totalResources(new QSemaphore(QThread::idealThreadCount())) + , totalResources(new QSemaphore(br::Globals->parallelism)) {} ~Resource() diff --git a/openbr/frvt2012.cpp b/openbr/frvt2012.cpp index ac47ac8..b0c2d29 100644 --- a/openbr/frvt2012.cpp +++ b/openbr/frvt2012.cpp @@ -132,7 +132,7 @@ int32_t SdkEstimator::estimate_age(const ONEFACE &input_face, int32_t &age) TemplateList templates; templates.append(templateFromONEFACE(input_face)); templates >> *frvt2012_age_transform.data(); - age = templates.first().file.get("Subject"); + age = templates.first().file.get("Age"); return templates.first().file.failed() ? 4 : 0; } @@ -141,6 +141,6 @@ int32_t SdkEstimator::estimate_gender(const ONEFACE &input_face, int8_t &gender, TemplateList templates; templates.append(templateFromONEFACE(input_face)); templates >> *frvt2012_gender_transform.data(); - mf = gender = templates.first().file.get("Subject") == "Male" ? 0 : 1; + mf = gender = templates.first().file.get("Gender") == "Male" ? 0 : 1; return templates.first().file.failed() ? 4 : 0; } diff --git a/openbr/gui/classifier.cpp b/openbr/gui/classifier.cpp index 2df9f5e..71a3102 100644 --- a/openbr/gui/classifier.cpp +++ b/openbr/gui/classifier.cpp @@ -39,19 +39,23 @@ void Classifier::_classify(File file) { QString key, value; foreach (const File &f, Enroll(file.flat(), File("[algorithm=" + algorithm + "]"))) { - if (!f.contains("Label")) - continue; if (algorithm == "GenderClassification") { key = "Gender"; - value = (f.get("Subject")); } else if (algorithm == "AgeRegression") { key = "Age"; - value = QString::number(int(f.get("Subject")+0.5)) + " Years"; } else { key = algorithm; - value = f.get("Subject"); } + + if (!f.contains(key)) + continue; + + if (algorithm == "AgeRegression") + value = QString::number(int(f.get(key)+0.5)) + " Years"; + else + value = f.get(key); + break; } diff --git a/openbr/openbr.cpp b/openbr/openbr.cpp index c269366..491b91d 100644 --- a/openbr/openbr.cpp +++ b/openbr/openbr.cpp @@ -72,9 +72,9 @@ float br_eval(const char *simmat, const char *mask, const char *csv) return Evaluate(simmat, mask, csv); } -void br_eval_classification(const char *predicted_gallery, const char *truth_gallery) +void br_eval_classification(const char *predicted_gallery, const char *truth_gallery, const char *predicted_property, const char * truth_property) { - EvalClassification(predicted_gallery, truth_gallery); + EvalClassification(predicted_gallery, truth_gallery, predicted_property, truth_property); } void br_eval_clustering(const char *csv, const char *gallery) @@ -87,9 +87,14 @@ float br_eval_detection(const char *predicted_gallery, const char *truth_gallery return EvalDetection(predicted_gallery, truth_gallery, csv); } -void br_eval_regression(const char *predicted_gallery, const char *truth_gallery) +void br_eval_landmarking(const char *predicted_gallery, const char *truth_gallery, const char *csv) { - EvalRegression(predicted_gallery, truth_gallery); + return EvalLandmarking(predicted_gallery, truth_gallery, csv); +} + +void br_eval_regression(const char *predicted_gallery, const char *truth_gallery, const char * predicted_property, const char * truth_property) +{ + EvalRegression(predicted_gallery, truth_gallery, predicted_property, truth_property); } void br_finalize() diff --git a/openbr/openbr.h b/openbr/openbr.h index 6d96a28..2fc5c2b 100644 --- a/openbr/openbr.h +++ b/openbr/openbr.h @@ -149,7 +149,7 @@ BR_EXPORT float br_eval(const char *simmat, const char *mask, const char *csv = * \param predicted_gallery The predicted br::Gallery. * \param truth_gallery The ground truth br::Gallery. */ -BR_EXPORT void br_eval_classification(const char *predicted_gallery, const char *truth_gallery); +BR_EXPORT void br_eval_classification(const char *predicted_gallery, const char *truth_gallery, const char * predicted_property="", const char * truth_property=""); /*! * \brief Evaluates and prints clustering accuracy to the terminal. @@ -169,11 +169,19 @@ BR_EXPORT void br_eval_clustering(const char *csv, const char *gallery); BR_EXPORT float br_eval_detection(const char *predicted_gallery, const char *truth_gallery, const char *csv = ""); /*! + * \brief Evaluates and prints landmarking accuracy to terminal. + * \param predicted_gallery The predicted br::Gallery. + * \param truth_gallery The ground truth br::Gallery. + * \param csv Optional \c .csv file to contain performance metrics. + */ +BR_EXPORT void br_eval_landmarking(const char *predicted_gallery, const char *truth_gallery, const char *csv = ""); + +/*! * \brief Evaluates regression accuracy to disk. * \param predicted_gallery The predicted br::Gallery. * \param truth_gallery The ground truth br::Gallery. */ -BR_EXPORT void br_eval_regression(const char *predicted_gallery, const char *truth_gallery); +BR_EXPORT void br_eval_regression(const char *predicted_gallery, const char *truth_gallery, const char * predicted_property="", const char * truth_property=""); /*! * \brief Wraps br::Context::finalize() diff --git a/openbr/openbr_export.cpp b/openbr/openbr_export.cpp index 0eccdb5..dac8526 100644 --- a/openbr/openbr_export.cpp +++ b/openbr/openbr_export.cpp @@ -69,7 +69,6 @@ $ cd bin $ export LD_LIBRARY_PATH=../lib:${LD_LIBRARY_PATH} $ sudo ldconfig -$ sudo cp ../share/openbr/70-yubikey.rules /etc/udev/rules.d # Only needed if you were given a license dongle. \endverbatim * \par OS X \verbatim @@ -80,10 +79,6 @@ $ export DYLD_FRAMEWORK_PATH=../lib:${DYLD_FRAMEWORK_PATH} * \par Windows * No configuration is necessary! * - * \section installation_license_dongle License Dongle - * In the unlikely event that you were given a USB License Dongle, then dongle must be in the computer in order to use the SDK. - * No configuration of the dongle is needed. - * * \section installation_done Start Working * To test for successful installation: \verbatim diff --git a/openbr/openbr_plugin.cpp b/openbr/openbr_plugin.cpp index 9985bc8..e21e373 100644 --- a/openbr/openbr_plugin.cpp +++ b/openbr/openbr_plugin.cpp @@ -437,6 +437,18 @@ TemplateList TemplateList::fromGallery(const br::File &gallery) // Select the right 8 hex characters so that it can be represented as a 64 bit integer without overflow newTemplates[i].file.set("Partition", md5.toHex().right(8).toULongLong(0, 16) % crossValidate); } +<<<<<<< HEAD +======= + } else if (newTemplates[i].file.getBool("allPartitions")) { + // The allPartitions flag is used to add an extended set + // of target images to every partition + newTemplates[i].file.set("Partition", -1); + } else { + // Direct use of "Label" is not general -cao + const QByteArray md5 = QCryptographicHash::hash(newTemplates[i].file.get("Label").toLatin1(), QCryptographicHash::Md5); + // Select the right 8 hex characters so that it can be represented as a 64 bit integer without overflow + newTemplates[i].file.set("Partition", md5.toHex().right(8).toULongLong(0, 16) % crossValidate); +>>>>>>> c2b1835e05d3b229db72d8829fb9ebf7e3cf31d8 } } } @@ -834,7 +846,7 @@ float br::Context::progress() const void br::Context::setProperty(const QString &key, const QString &value) { - Object::setProperty(key, value); + Object::setProperty(key, value.isEmpty() ? QVariant() : value); qDebug("Set %s%s", qPrintable(key), value.isEmpty() ? "" : qPrintable(" to " + value)); if (key == "parallelism") { @@ -912,6 +924,8 @@ void br::Context::initialize(int &argc, char *argv[], QString sdkPath, bool use_ qInstallMessageHandler(messageHandler); + Common::seedRNG(); + // Search for SDK if (sdkPath.isEmpty()) { QStringList checkPaths; checkPaths << QDir::currentPath() << QCoreApplication::applicationDirPath(); @@ -1104,9 +1118,6 @@ Transform::Transform(bool _independent, bool _trainable) { independent = _independent; trainable = _trainable; - classes = std::numeric_limits::max(); - instances = std::numeric_limits::max(); - fraction = 1; } Transform *Transform::make(QString str, QObject *parent) @@ -1162,9 +1173,6 @@ Transform *Transform::make(QString str, QObject *parent) Transform *Transform::clone() const { Transform *clone = Factory::make(file.flat()); - clone->classes = classes; - clone->instances = instances; - clone->fraction = fraction; return clone; } @@ -1193,28 +1201,6 @@ void Transform::project(const TemplateList &src, TemplateList &dst) const futures.waitForFinished(); } -static void _backProject(const Transform *transform, const Template *dst, Template *src) -{ - try { - transform->backProject(*dst, *src); - } catch (...) { - qWarning("Exception triggered when processing %s with transform %s", qPrintable(src->file.flat()), qPrintable(transform->objectName())); - *src = Template(dst->file); - src->file.set("FTE", true); - } -} - -void Transform::backProject(const TemplateList &dst, TemplateList &src) const -{ - src.reserve(dst.size()); - for (int i=0; i futures; - for (int i=0; i Transform::getChildren() const { QList output; diff --git a/openbr/openbr_plugin.h b/openbr/openbr_plugin.h index dbf69b4..c50be94 100644 --- a/openbr/openbr_plugin.h +++ b/openbr/openbr_plugin.h @@ -130,13 +130,6 @@ void reset_##NAME() { NAME = DEFAULT; } * -# If the value is convertable to a floating point number then it is represented with \c float. * -# Otherwise, it is represented with \c QString. * - * The metadata keys \c Subject and \c Label have special significance in the system. - * \c Subject is a string specifying a unique identifier used to determine ground truth match/non-match. - * \c Label is a floating point value used for supervised learning. - * When the system needs labels for training, but only subjects are provided in the file metadata, the rule for generating labels is as follows. - * If the subject value can be converted to a float then do so and consider that the label. - * Otherwise, generate a unique integer ID for the string starting from zero and incrementing by one everytime another ID is needed. - * * Metadata keys fall into one of two categories: * - \c camelCaseKeys are inputs that specify how to process the file. * - \c Capitalized_Underscored_Keys are outputs computed from processing the file. @@ -147,8 +140,6 @@ void reset_##NAME() { NAME = DEFAULT; } * --- | ---- | ----------- * separator | QString | Seperate #name into multiple files * Index | int | Index of a template in a template list - * Subject | QString | Class name - * Label | float | Class value * Confidence | float | Classification/Regression quality * FTE | bool | Failure to enroll * FTO | bool | Failure to open @@ -157,13 +148,15 @@ void reset_##NAME() { NAME = DEFAULT; } * *_Width | float | Size * *_Height | float | Size * *_Radius | float | Size + * Label | QString | Class label * Theta | float | Pose * Roll | float | Pose * Pitch | float | Pose * Yaw | float | Pose * Points | QList | List of unnamed points * Rects | QList | List of unnamed rects - * Age | QString | Age used for demographic filtering + * Age | float | Age used for demographic filtering + * Gender | QString | Subject gender * _* | * | Reserved for internal use */ struct BR_EXPORT File @@ -172,7 +165,7 @@ struct BR_EXPORT File File() {} File(const QString &file) { init(file); } /*!< \brief Construct a file from a string. */ - File(const QString &file, const QVariant &subject) { init(file); set("Subject", subject); } /*!< \brief Construct a file from a string and assign a label. */ + File(const QString &file, const QVariant &label) { init(file); set("Label", label); } /*!< \brief Construct a file from a string and assign a label. */ File(const char *file) { init(file); } /*!< \brief Construct a file from a c-style string. */ inline operator QString() const { return name; } /*!< \brief Returns #name. */ QString flat() const; /*!< \brief A stringified version of the file with metadata. */ @@ -616,13 +609,6 @@ public: BR_PROPERTY(int, blockSize, parallelism * ((sizeof(void*) == 4) ? 128 : 1024)) /*! - * \brief true if backProject should be used instead of project (the algorithm should be inverted) - */ - Q_PROPERTY(bool backProject READ get_backProject WRITE set_backProject RESET reset_backProject) - BR_PROPERTY(bool, backProject, false) - - - /*! * \brief If \c true no messages will be sent to the terminal, \c false by default. */ Q_PROPERTY(bool quiet READ get_quiet WRITE set_quiet RESET reset_quiet) @@ -1048,6 +1034,10 @@ private: * @{ */ +/*! + * \brief For asynchronous events during template projection. + * \see #Transform::getEvent + */ class TemplateEvent : public QObject { Q_OBJECT @@ -1062,7 +1052,6 @@ signals: void theSignal(const Template & output) const; }; - /*! * \brief Plugin base class for processing a template. * @@ -1076,12 +1065,6 @@ class BR_EXPORT Transform : public Object Q_OBJECT public: - Q_PROPERTY(int classes READ get_classes WRITE set_classes RESET reset_classes STORED false) - Q_PROPERTY(int instances READ get_instances WRITE set_instances RESET reset_instances STORED false) - Q_PROPERTY(float fraction READ get_fraction WRITE set_fraction RESET reset_fraction STORED false) - BR_PROPERTY(int, classes, std::numeric_limits::max()) - BR_PROPERTY(int, instances, std::numeric_limits::max()) - BR_PROPERTY(float, fraction, 1) bool independent, trainable; virtual ~Transform() {} @@ -1092,8 +1075,6 @@ public: virtual void train(const TemplateList &data) = 0; /*!< \brief Train the transform. */ virtual void project(const Template &src, Template &dst) const = 0; /*!< \brief Apply the transform. */ virtual void project(const TemplateList &src, TemplateList &dst) const; /*!< \brief Apply the transform. */ - virtual void backProject(const Template &dst, Template &src) const { src = dst; } /*!< \brief Invert the transform. */ - virtual void backProject(const TemplateList &dst, TemplateList &src) const; /*!< \brief Invert the transform. */ /*!< \brief Apply the transform, may update the transform's internal state */ virtual void projectUpdate(const Template &src, Template &dst) diff --git a/openbr/plugins/algorithms.cpp b/openbr/plugins/algorithms.cpp index 12754a7..523b234 100644 --- a/openbr/plugins/algorithms.cpp +++ b/openbr/plugins/algorithms.cpp @@ -46,9 +46,12 @@ class AlgorithmsInitializer : public Initializer Globals->abbreviations.insert("CropFace", "Open+Cvt(Gray)+Cascade(FrontalFace)+ASEFEyes+Affine(128,128,0.25,0.35)"); // Video - Globals->abbreviations.insert("DisplayVideo", "Stream([Show(false,[FrameNumber])+Discard])"); + Globals->abbreviations.insert("DisplayVideo", "Stream([FPSLimit(30)+Show(false,[FrameNumber])+Discard])"); Globals->abbreviations.insert("PerFrameDetection", "Stream([SaveMat(original)+Cvt(Gray)+Cascade(FrontalFace)+ASEFEyes+RestoreMat(original)+Draw(inPlace=true),Show(false,[FrameNumber])+Discard])"); - Globals->abbreviations.insert("AgeGenderDemo", "Stream([SaveMat(original)+Cvt(Gray)+Cascade(FrontalFace)+Expand+++(+Rename(Subject,Age)+Discard)/(+Rename(Subject,Gender)+Discard)+RestoreMat(original)+Draw(inPlace=true)+DrawPropertiesPoint([Age,Gender],Affine_0,inPlace=true)+SaveMat(original)+Discard+Contract,RestoreMat(original)+FPSCalc+Show(false,[AvgFPS,Age,Gender])+Discard])"); + Globals->abbreviations.insert("AgeGenderDemo", "Stream([SaveMat(original)+Cvt(Gray)+Cascade(FrontalFace)+Expand+++/+Discard+RestoreMat(original)+Draw(inPlace=true)+DrawPropertiesPoint([Age,Gender],Affine_0,inPlace=true)+SaveMat(original)+Discard+Contract,RestoreMat(original)+FPSCalc+Show(false,[AvgFPS,Age,Gender])+Discard])"); + Globals->abbreviations.insert("HOG", "Stream([KeyPointDetector(SIFT)+ROI+Expand+Resize(32,32)+Gradient+RectRegions+Bin(0,360,8)+Hist(8)+Cat])+Contract+CatRows+KMeans(500)+Hist(500)+SVM"); + Globals->abbreviations.insert("HOF", "Stream([KeyPointDetector(SIFT),AggregateFrames(2)+OpticalFlow+Gradient+Bin(0,360,8)+ROI+Hist(8)])+Contract+CatRows+KMeans(500)+Hist(500)"); + Globals->abbreviations.insert("HOGHOF", "Stream([Cvt(Gray),KeyPointDetector(SIFT),AggregateFrames(2),(OpticalFlow+Gradient+Bin(0,360,8)+ROI+Hist(8))/(First+Gradient+Bin(0,360,8)+ROI+Hist(8)),CatCols])+Contract+CatRows+KMeans(500)+Hist(500)"); // Generic Image Processing Globals->abbreviations.insert("SIFT", "Open+KeyPointDetector(SIFT)+KeyPointDescriptor(SIFT):KeyPointMatcher(BruteForce)"); @@ -74,14 +77,14 @@ class AlgorithmsInitializer : public Initializer Globals->abbreviations.insert("FaceDetection", "(Open+Cvt(Gray)+Cascade(FrontalFace))"); Globals->abbreviations.insert("DenseLBP", "(Blur(1.1)+Gamma(0.2)+DoG(1,2)+ContrastEq(0.1,10)+LBP(1,2)+RectRegions(8,8,6,6)+Hist(59))"); Globals->abbreviations.insert("DenseSIFT", "(Grid(10,10)+SIFTDescriptor(12)+ByRow)"); - Globals->abbreviations.insert("FaceRecognitionRegistration", "(ASEFEyes+Affine(88,88,0.25,0.35)+FTE(DFFS,instances=1))"); - Globals->abbreviations.insert("FaceRecognitionExtraction", "(Mask+DenseSIFT/DenseLBP+PCA(0.95,instances=1)+Normalize(L2)+Cat)"); - Globals->abbreviations.insert("FaceRecognitionEmbedding", "(Dup(12)+RndSubspace(0.05,1)+LDA(0.98,instances=-2)+Cat+PCA(768,instances=1))"); + Globals->abbreviations.insert("FaceRecognitionRegistration", "(ASEFEyes+Affine(88,88,0.25,0.35)+DownsampleTraining(FTE(DFFS),instances=1))"); + Globals->abbreviations.insert("FaceRecognitionExtraction", "(Mask+DenseSIFT/DenseLBP+DownsampleTraining(PCA(0.95),instances=1)+Normalize(L2)+Cat)"); + Globals->abbreviations.insert("FaceRecognitionEmbedding", "(Dup(12)+RndSubspace(0.05,1)+DownsampleTraining(LDA(0.98),instances=-2)+Cat+DownsampleTraining(PCA(768),instances=1))"); Globals->abbreviations.insert("FaceRecognitionQuantization", "(Normalize(L1)+Quantize)"); Globals->abbreviations.insert("FaceClassificationRegistration", "(ASEFEyes+Affine(56,72,0.33,0.45)+FTE(DFFS))"); - Globals->abbreviations.insert("FaceClassificationExtraction", "((Grid(7,7)+SIFTDescriptor(8)+ByRow)/DenseLBP+PCA(0.95,instances=-1)+Cat)"); - Globals->abbreviations.insert("AgeRegressor", "Center(Range,instances=-1)+SVM(RBF,EPS_SVR,instances=100)"); - Globals->abbreviations.insert("GenderClassifier", "Center(Range,instances=-1)+SVM(RBF,C_SVC,instances=4000)"); + Globals->abbreviations.insert("FaceClassificationExtraction", "((Grid(7,7)+SIFTDescriptor(8)+ByRow)/DenseLBP+DownsampleTraining(PCA(0.95),instances=-1, inputVariable=Gender)+Cat)"); + Globals->abbreviations.insert("AgeRegressor", "DownsampleTraining(Center(Range),instances=-1, inputVariable=Age)+DownsampleTraining(SVM(RBF,EPS_SVR,inputVariable=Age),instances=100, inputVariable=Age)"); + Globals->abbreviations.insert("GenderClassifier", "DownsampleTraining(Center(Range),instances=-1, inputVariable=Gender)+DownsampleTraining(SVM(RBF,C_SVC,inputVariable=Gender),instances=4000, inputVariable=Gender)"); Globals->abbreviations.insert("UCharL1", "Unit(ByteL1)"); } }; diff --git a/openbr/plugins/cascade.cpp b/openbr/plugins/cascade.cpp index 0c00523..e05d9d2 100644 --- a/openbr/plugins/cascade.cpp +++ b/openbr/plugins/cascade.cpp @@ -49,19 +49,20 @@ private: } }; - /*! * \ingroup transforms * \brief Wraps OpenCV cascade classifier * \author Josh Klontz \cite jklontz */ -class CascadeTransform : public UntrainableTransform +class CascadeTransform : public UntrainableMetaTransform { Q_OBJECT Q_PROPERTY(QString model READ get_model WRITE set_model RESET reset_model STORED false) Q_PROPERTY(int minSize READ get_minSize WRITE set_minSize RESET reset_minSize STORED false) + Q_PROPERTY(bool ROCMode READ get_ROCMode WRITE set_ROCMode RESET reset_ROCMode STORED false) BR_PROPERTY(QString, model, "FrontalFace") BR_PROPERTY(int, minSize, 64) + BR_PROPERTY(bool, ROCMode, false) Resource cascadeResource; @@ -72,18 +73,54 @@ class CascadeTransform : public UntrainableTransform void project(const Template &src, Template &dst) const { + TemplateList temp; + project(TemplateList() << src, temp); + if (!temp.isEmpty()) dst = temp.first(); + } + + void project(const TemplateList &src, TemplateList &dst) const + { CascadeClassifier *cascade = cascadeResource.acquire(); - vector rects; - cascade->detectMultiScale(src, rects, 1.2, 5, src.file.get("enrollAll", false) ? 0 : CV_HAAR_FIND_BIGGEST_OBJECT, Size(minSize, minSize)); + foreach (const Template &t, src) { + const bool enrollAll = t.file.getBool("enrollAll"); + + for (int i=0; i rects; + vector rejectLevels; + vector levelWeights; + if (ROCMode) cascade->detectMultiScale(m, rects, rejectLevels, levelWeights, 1.2, 5, (enrollAll ? 0 : CV_HAAR_FIND_BIGGEST_OBJECT) | CV_HAAR_SCALE_IMAGE, Size(minSize, minSize), Size(), true); + else cascade->detectMultiScale(m, rects, 1.2, 5, enrollAll ? 0 : CV_HAAR_FIND_BIGGEST_OBJECT, Size(minSize, minSize)); + + if (!enrollAll && rects.empty()) + rects.push_back(Rect(0, 0, m.cols, m.rows)); + + for (size_t j=0; j j) + u.file.set("Confidence", rejectLevels[j]*levelWeights[j]); + const QRectF rect = OpenCVUtils::fromRect(rects[j]); + u.file.appendRect(rect); + u.file.set(model, rect); + dst.append(u); + } + } + } + cascadeResource.release(cascade); + } - if (!src.file.get("enrollAll", false) && rects.empty()) - rects.push_back(Rect(0, 0, src.m().cols, src.m().rows)); + // TODO: Remove this code when ready to break binary compatibility + void store(QDataStream &stream) const + { + int size = 1; + stream << size; + } - foreach (const Rect &rect, rects) { - dst += src; - dst.file.appendRect(OpenCVUtils::fromRect(rect)); - } + void load(QDataStream &stream) + { + int size; + stream >> size; } }; diff --git a/openbr/plugins/cluster.cpp b/openbr/plugins/cluster.cpp index e6ff87c..3a9b577 100644 --- a/openbr/plugins/cluster.cpp +++ b/openbr/plugins/cluster.cpp @@ -89,10 +89,14 @@ class KNNTransform : public Transform Q_PROPERTY(br::Distance *distance READ get_distance WRITE set_distance RESET reset_distance STORED false) Q_PROPERTY(bool weighted READ get_weighted WRITE set_weighted RESET reset_weighted STORED false) Q_PROPERTY(int numSubjects READ get_numSubjects WRITE set_numSubjects RESET reset_numSubjects STORED false) + Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) + Q_PROPERTY(QString outputVariable READ get_outputVariable WRITE set_outputVariable RESET reset_outputVariable STORED false) BR_PROPERTY(int, k, 1) BR_PROPERTY(br::Distance*, distance, NULL) BR_PROPERTY(bool, weighted, false) BR_PROPERTY(int, numSubjects, 1) + BR_PROPERTY(QString, inputVariable, "Label") + BR_PROPERTY(QString, outputVariable, "KNN") TemplateList gallery; @@ -111,17 +115,17 @@ class KNNTransform : public Transform QHash votes; const int max = (k < 1) ? sortedScores.size() : std::min(k, sortedScores.size()); for (int j=0; j("Subject")] += (weighted ? sortedScores[j].first : 1); + votes[gallery[sortedScores[j].second].file.get(inputVariable)] += (weighted ? sortedScores[j].first : 1); subjects.append(votes.keys()[votes.values().indexOf(Common::Max(votes.values()))]); // Remove subject from consideration if (subjects.size() < numSubjects) for (int j=sortedScores.size()-1; j>=0; j--) - if (gallery[sortedScores[j].second].file.get("Subject") == subjects.last()) + if (gallery[sortedScores[j].second].file.get(inputVariable) == subjects.last()) sortedScores.removeAt(j); } - dst.file.set("KNN", subjects.size() > 1 ? "[" + subjects.join(",") + "]" : subjects.first()); + dst.file.set(outputVariable, subjects.size() > 1 ? "[" + subjects.join(",") + "]" : subjects.first()); } void store(QDataStream &stream) const diff --git a/openbr/plugins/eigen3.cpp b/openbr/plugins/eigen3.cpp index d1b877b..4a904c2 100644 --- a/openbr/plugins/eigen3.cpp +++ b/openbr/plugins/eigen3.cpp @@ -59,21 +59,6 @@ public: PCATransform() : keep(0.95), drop(0), whiten(false) {} private: - /* - void backProject(const Template &src, Template &dst) const - { - const cv::Mat &m = src; - dst = cv::Mat(originalRows, m.rows*m.cols/originalRows, CV_32FC1); - - // Map Eigen into OpenCV - Eigen::Map inMap(m.ptr(), keep, 1); - Eigen::Map outMap(dst.m().ptr(), m.rows*m.cols, 1); - - // Do projection - outMap = (eVecs * inMap) + mean; - } - */ - double residualReconstructionError(const Template &src) const { Template proj; @@ -318,10 +303,12 @@ class LDATransform : public Transform Q_PROPERTY(bool pcaWhiten READ get_pcaWhiten WRITE set_pcaWhiten RESET reset_pcaWhiten STORED false) Q_PROPERTY(int directLDA READ get_directLDA WRITE set_directLDA RESET reset_directLDA STORED false) Q_PROPERTY(float directDrop READ get_directDrop WRITE set_directDrop RESET reset_directDrop STORED false) + Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) BR_PROPERTY(float, pcaKeep, 0.98) BR_PROPERTY(bool, pcaWhiten, false) BR_PROPERTY(int, directLDA, 0) BR_PROPERTY(float, directDrop, 0.1) + BR_PROPERTY(QString, inputVariable, "Label") int dimsOut; Eigen::VectorXf mean; @@ -330,7 +317,7 @@ class LDATransform : public Transform void train(const TemplateList &_trainingSet) { // creates "Label" - TemplateList trainingSet = TemplateList::relabel(_trainingSet, "Subject"); + TemplateList trainingSet = TemplateList::relabel(_trainingSet, inputVariable); int instances = trainingSet.size(); diff --git a/openbr/plugins/eyes.cpp b/openbr/plugins/eyes.cpp index dd56767..5316705 100644 --- a/openbr/plugins/eyes.cpp +++ b/openbr/plugins/eyes.cpp @@ -186,7 +186,6 @@ private: //dst.file.appendPoint(QPointF(second_eye_x, second_eye_y)); dst.file.set("First_Eye", QPointF(first_eye_x, first_eye_y)); dst.file.set("Second_Eye", QPointF(second_eye_x, second_eye_y)); - dst.file.set("Face", QRect(roi.x, roi.y, roi.width, roi.height)); } }; diff --git a/openbr/plugins/gallery.cpp b/openbr/plugins/gallery.cpp index 18f04e0..85a307b 100644 --- a/openbr/plugins/gallery.cpp +++ b/openbr/plugins/gallery.cpp @@ -71,7 +71,7 @@ class arffGallery : public Gallery } arffFile.write(qPrintable(OpenCVUtils::matrixToStringList(t).join(','))); - arffFile.write(qPrintable(",'" + t.file.get("Subject") + "'\n")); + arffFile.write(qPrintable(",'" + t.file.get("Label") + "'\n")); } }; @@ -643,11 +643,16 @@ class dbGallery : public Gallery query = query.mid(1, query.size()-2); if (!q.exec(query)) qFatal("%s.", qPrintable(q.lastError().text())); + if ((q.record().count() == 0) || (q.record().count() > 3)) qFatal("Query record expected one to three fields, got %d.", q.record().count()); const bool hasMetadata = (q.record().count() >= 2); const bool hasFilter = (q.record().count() >= 3); + QString labelName = "Label"; + if (q.record().count() >= 2) + labelName = q.record().fieldName(1); + // subset = seed:subjectMaxSize:numSubjects:subjectMinSize or // subset = seed:{Metadata,...,Metadata}:numSubjects int seed = 0, subjectMaxSize = std::numeric_limits::max(), numSubjects = std::numeric_limits::max(), subjectMinSize = 0; @@ -673,6 +678,7 @@ class dbGallery : public Gallery QHash > entries; // QHash > while (q.next()) { if (hasFilter && (seed >= 0) && (qHash(q.value(2).toString()) % 2 != (uint)seed % 2)) continue; // Ensures training and testing filters don't overlap + if (metadataFields.isEmpty()) entries[hasMetadata ? q.value(1).toString() : ""].append(QPair(q.value(0).toString(), hasFilter ? q.value(2).toString() : "")); else @@ -707,8 +713,10 @@ class dbGallery : public Gallery if (entryList.size() > subjectMaxSize) std::random_shuffle(entryList.begin(), entryList.end()); - foreach (const Entry &entry, entryList.mid(0, subjectMaxSize)) - templates.append(File(entry.first, label)); + foreach (const Entry &entry, entryList.mid(0, subjectMaxSize)) { + templates.append(File(entry.first)); + templates.last().file.set(labelName, label); + } numSubjects--; } } @@ -816,7 +824,7 @@ class statGallery : public Gallery void write(const Template &t) { - subjects.insert(t.file.get("Subject")); + subjects.insert(t.file.get("Label")); bytes.append(t.bytes()); } }; diff --git a/openbr/plugins/gui.cpp b/openbr/plugins/gui.cpp index e7357c9..b63adae 100644 --- a/openbr/plugins/gui.cpp +++ b/openbr/plugins/gui.cpp @@ -52,26 +52,33 @@ QImage toQImage(const Mat &mat) return QImage(mat8uc3.data, mat8uc3.cols, mat8uc3.rows, 3*mat8uc3.cols, QImage::Format_RGB888).copy(); } - -// Provides slots for manipulating a QLabel, but does not inherit from QWidget. -// Therefore, it can be moved to the main thread if not created there initially -// since god forbid you create a QWidget subclass in not the main thread. -class GUIProxy : public QObject +class DisplayWindow : public QLabel { Q_OBJECT + +protected: QMutex lock; QWaitCondition wait; + QPixmap pixmap; public: - QLabel *window; - QPixmap pixmap; + DisplayWindow(QWidget * parent = NULL) : QLabel(parent) + { + QApplication::instance()->installEventFilter(this); + } - GUIProxy() +public slots: + void showImage(const QPixmap & input) { - window = NULL; + pixmap = input; + + show(); + setPixmap(pixmap); + setFixedSize(input.size()); } + bool eventFilter(QObject * obj, QEvent * event) { if (event->type() == QEvent::KeyPress) @@ -92,41 +99,10 @@ public: return QList(); } - -public slots: - - void showImage(const QPixmap & input) - { - pixmap = input; - - window->show(); - window->setPixmap(pixmap); - window->setFixedSize(input.size()); - } - - void createWindow() - { - delete window; - QApplication::instance()->removeEventFilter(this); - - window = new QLabel(); - window->setVisible(true); - - QApplication::instance()->installEventFilter(this); - Qt::WindowFlags flags = window->windowFlags(); - - flags = flags & ~Qt::WindowCloseButtonHint; - window->setWindowFlags(flags); - } - }; -class LandmarkProxy : public GUIProxy +class PointMarkingWindow : public DisplayWindow { - Q_OBJECT - -public: - bool eventFilter(QObject *obj, QEvent *event) { if (event->type() == QEvent::MouseButtonPress) @@ -144,11 +120,11 @@ public: painter.setBrush(Qt::red); foreach(const QPointF &point, points) painter.drawEllipse(point, 4, 4); - window->setPixmap(pixmapBuffer); + setPixmap(pixmapBuffer); return true; } else { - return GUIProxy::eventFilter(obj, event); + return DisplayWindow::eventFilter(obj, event); } } @@ -156,14 +132,85 @@ public: { points.clear(); - GUIProxy::waitForKey(); + DisplayWindow::waitForKey(); return points; } private: - QList points; + + +}; + + +// I want a template class that doesn't look like a template class +class NominalCreation +{ +public: + virtual ~NominalCreation() {} + virtual void creation()=0; +}; + +// Putting the template on a subclass means we can maintain a pointer that +// doesn't include T in its type. +template +class ActualCreation : public NominalCreation +{ +public: + T * basis; + + void creation() + { + basis = new T(); + } +}; + +// We want to create a QLabel subclass on the main thread, but are running in another thread. +// We cannot move QWidget subclasses to a different thread (obviously that would be crazy), but +// we can create one of these, and move it to the main thread, and then use it to create the object +// we want. +// Additional fact: QObject subclasses cannot be template classes. +class MainThreadCreator : public QObject +{ + Q_OBJECT +public: + + MainThreadCreator() + { + this->moveToThread(QApplication::instance()->thread()); + + connect(this, SIGNAL(needCreation()), this, SLOT(createThing()), Qt::BlockingQueuedConnection); + } + + // While this cannot be a template class, it can still have a template method. + template + T * getItem() + { + if (QThread::currentThread() == QApplication::instance()->thread()) + return new T(); + + ActualCreation * actualWorker; + actualWorker = new ActualCreation (); + worker = actualWorker; + + emit needCreation(); + + T * output = actualWorker->basis; + delete actualWorker; + return output; + } + + NominalCreation * worker; + +signals: + void needCreation(); + +public slots: + void createThing() + { + worker->creation(); + } }; /*! @@ -184,14 +231,14 @@ public: ShowTransform() : TimeVaryingTransform(false, false) { - gui = NULL; displayBuffer = NULL; + window = NULL; } ~ShowTransform() { - delete gui; delete displayBuffer; + delete window; } void train(const TemplateList &data) { (void) data; } @@ -235,7 +282,7 @@ public: // Blocking wait for a key-press if (this->waitInput) - gui->waitForKey(); + window->waitForKey(); } } @@ -249,33 +296,36 @@ public: void init() { + initActual(); + } + + template + void initActual() + { if (!Globals->useGui) return; + if (displayBuffer) + delete displayBuffer; displayBuffer = new QPixmap(); - // Create our GUI proxy - gui = new GUIProxy(); - // Move it to the main thread, this means signals we send to it will - // be run in the main thread, which is hopefully in an event loop - gui->moveToThread(QApplication::instance()->thread()); - - // Connect our signals to the proxy's slots - connect(this, SIGNAL(needWindow()), gui, SLOT(createWindow()), Qt::BlockingQueuedConnection); - connect(this, SIGNAL(updateImage(QPixmap)), gui,SLOT(showImage(QPixmap))); + if (window) + delete window; - emit needWindow(); - connect(this, SIGNAL(changeTitle(QString)), gui->window, SLOT(setWindowTitle(QString))); - connect(this, SIGNAL(hideWindow()), gui->window, SLOT(hide())); + window = creator.getItem(); + // Connect our signals to the window's slots + connect(this, SIGNAL(updateImage(QPixmap)), window,SLOT(showImage(QPixmap))); + connect(this, SIGNAL(changeTitle(QString)), window, SLOT(setWindowTitle(QString))); + connect(this, SIGNAL(hideWindow()), window, SLOT(hide())); } protected: - GUIProxy * gui; + MainThreadCreator creator; + DisplayWindow * window; QImage qImageBuffer; QPixmap * displayBuffer; signals: - void needWindow(); void updateImage(const QPixmap & input); void changeTitle(const QString & input); void hideWindow(); @@ -312,7 +362,7 @@ public: // Blocking wait for a key-press if (this->waitInput) { - QList points = gui->waitForKey(); + QList points = window->waitForKey(); if (keys.isEmpty()) dst[i].file.appendPoints(points); else { if (keys.size() == points.size()) @@ -326,24 +376,7 @@ public: void init() { - if (!Globals->useGui) - return; - - displayBuffer = new QPixmap(); - - // Create our GUI proxy - gui = new LandmarkProxy(); - - // Move it to the main thread, this means signals we send to it will - // be run in the main thread, which is hopefully in an event loop - gui->moveToThread(QApplication::instance()->thread()); - - // Connect our signals to the proxy's slots - connect(this, SIGNAL(needWindow()), gui, SLOT(createWindow()), Qt::BlockingQueuedConnection); - connect(this, SIGNAL(updateImage(QPixmap)), gui,SLOT(showImage(QPixmap))); - - emit needWindow(); - connect(this, SIGNAL(hideWindow()), gui->window, SLOT(hide())); + initActual(); } }; diff --git a/openbr/plugins/hist.cpp b/openbr/plugins/hist.cpp index b3efd8b..1896ae0 100644 --- a/openbr/plugins/hist.cpp +++ b/openbr/plugins/hist.cpp @@ -52,8 +52,11 @@ class HistTransform : public UntrainableTransform int histSize[] = {dims}; float range[] = {min, max}; const float* ranges[] = {range}; - Mat hist; - calcHist(&mv[i], 1, channels, Mat(), hist, 1, histSize, ranges); + Mat hist, chan = mv[i]; + // calcHist requires F or U, might as well convert just in case + if (mv[i].depth() != CV_8U || mv[i].depth() == CV_32F) + mv[i].convertTo(chan, CV_32F); + calcHist(&chan, 1, channels, Mat(), hist, 1, histSize, ranges); memcpy(m.ptr(i), hist.ptr(), dims * sizeof(float)); } diff --git a/openbr/plugins/independent.cpp b/openbr/plugins/independent.cpp index 123c979..4a1ec03 100644 --- a/openbr/plugins/independent.cpp +++ b/openbr/plugins/independent.cpp @@ -9,37 +9,36 @@ using namespace cv; namespace br { -static TemplateList Downsample(const TemplateList &templates, const Transform *transform) +static TemplateList Downsample(const TemplateList &templates, int classes, int instances, float fraction, const QString & inputVariable) { // Return early when no downsampling is required - if ((transform->classes == std::numeric_limits::max()) && - (transform->instances == std::numeric_limits::max()) && - (transform->fraction >= 1)) + if ((classes == std::numeric_limits::max()) && + (instances == std::numeric_limits::max()) && + (fraction >= 1)) return templates; - const bool atLeast = transform->instances < 0; - const int instances = abs(transform->instances); + const bool atLeast = instances < 0; + instances = abs(instances); - QList allLabels = File::get(templates, "Subject"); + QList allLabels = File::get(templates, inputVariable); QList uniqueLabels = allLabels.toSet().toList(); qSort(uniqueLabels); - QMap counts = templates.countValues("Subject", instances != std::numeric_limits::max()); + QMap counts = templates.countValues(inputVariable, instances != std::numeric_limits::max()); - if ((instances != std::numeric_limits::max()) && (transform->classes != std::numeric_limits::max())) + if ((instances != std::numeric_limits::max()) && (classes != std::numeric_limits::max())) foreach (const QString & label, counts.keys()) if (counts[label] < instances) counts.remove(label); uniqueLabels = counts.keys(); - if ((transform->classes != std::numeric_limits::max()) && (uniqueLabels.size() < transform->classes)) - qWarning("Downsample requested %d classes but only %d are available.", transform->classes, uniqueLabels.size()); + if ((classes != std::numeric_limits::max()) && (uniqueLabels.size() < classes)) + qWarning("Downsample requested %d classes but only %d are available.", classes, uniqueLabels.size()); - Common::seedRNG(); QList selectedLabels = uniqueLabels; - if (transform->classes < uniqueLabels.size()) { + if (classes < uniqueLabels.size()) { std::random_shuffle(selectedLabels.begin(), selectedLabels.end()); - selectedLabels = selectedLabels.mid(0, transform->classes); + selectedLabels = selectedLabels.mid(0, classes); } TemplateList downsample; @@ -56,14 +55,45 @@ static TemplateList Downsample(const TemplateList &templates, const Transform *t downsample.append(templates.value(indices[j])); } - if (transform->fraction < 1) { + if (fraction < 1) { std::random_shuffle(downsample.begin(), downsample.end()); - downsample = downsample.mid(0, downsample.size()*transform->fraction); + downsample = downsample.mid(0, downsample.size()*fraction); } return downsample; } +class DownsampleTrainingTransform : public Transform +{ + Q_OBJECT + Q_PROPERTY(br::Transform* transform READ get_transform WRITE set_transform RESET reset_transform STORED true) + Q_PROPERTY(int classes READ get_classes WRITE set_classes RESET reset_classes STORED false) + Q_PROPERTY(int instances READ get_instances WRITE set_instances RESET reset_instances STORED false) + Q_PROPERTY(float fraction READ get_fraction WRITE set_fraction RESET reset_fraction STORED false) + Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) + BR_PROPERTY(br::Transform*, transform, NULL) + BR_PROPERTY(int, classes, std::numeric_limits::max()) + BR_PROPERTY(int, instances, std::numeric_limits::max()) + BR_PROPERTY(float, fraction, 1) + BR_PROPERTY(QString, inputVariable, "Label") + + void project(const Template & src, Template & dst) const + { + transform->project(src,dst); + } + + + void train(const TemplateList &data) + { + if (!transform || !transform->trainable) + return; + + TemplateList downsampled = Downsample(data, classes, instances, fraction, inputVariable); + transform->train(downsampled); + } +}; +BR_REGISTER(Transform, DownsampleTrainingTransform) + /*! * \ingroup transforms * \brief Clones the transform so that it can be applied independently. @@ -124,13 +154,10 @@ class IndependentTransform : public MetaTransform while (transforms.size() < templatesList.size()) transforms.append(transform->clone()); - for (int i=0; i futures; for (int i=0; i +#include #include #include "openbr_internal.h" #include "openbr/core/opencvutils.h" @@ -178,6 +179,33 @@ BR_REGISTER(Transform, SIFTDescriptorTransform) /*! * \ingroup transforms + * \brief OpenCV HOGDescriptor wrapper + * \author Austin Blanton \cite imaus10 + */ +class HoGDescriptorTransform : public UntrainableTransform +{ + Q_OBJECT + + HOGDescriptor hog; + + void project(const Template &src, Template &dst) const + { + std::vector descriptorVals; + std::vector locations; + Size winStride = Size(0,0); + Size padding = Size(0,0); + foreach (const Mat &rect, src) { + hog.compute(rect, descriptorVals, winStride, padding, locations); + Mat HoGFeats(descriptorVals, true); + dst += HoGFeats; + } + } +}; + +BR_REGISTER(Transform, HoGDescriptorTransform) + +/*! + * \ingroup transforms * \brief Add landmarks to the template in a grid layout * \author Josh Klontz \cite jklontz */ diff --git a/openbr/plugins/mask.cpp b/openbr/plugins/mask.cpp index 8b86221..81fa8fb 100644 --- a/openbr/plugins/mask.cpp +++ b/openbr/plugins/mask.cpp @@ -158,6 +158,9 @@ class LargestConvexAreaTransform : public UntrainableTransform { Q_OBJECT + Q_PROPERTY(QString outputVariable READ get_outputVariable WRITE set_outputVariable RESET reset_outputVariable STORED false) + BR_PROPERTY(QString, outputVariable, "Label") + void project(const Template &src, Template &dst) const { std::vector< std::vector > contours; @@ -171,7 +174,7 @@ class LargestConvexAreaTransform : public UntrainableTransform if (area / hullArea > 0.98) maxArea = std::max(maxArea, area); } - dst.file.set("Label", maxArea); + dst.file.set(outputVariable, maxArea); } }; diff --git a/openbr/plugins/meta.cpp b/openbr/plugins/meta.cpp index f00676c..b4cc865 100644 --- a/openbr/plugins/meta.cpp +++ b/openbr/plugins/meta.cpp @@ -105,6 +105,24 @@ class PipeTransform : public CompositeTransform transforms[i]->train(copy); } + // if the transform is time varying, we can't project it in parallel + if (transforms[i]->timeVarying()) { + fprintf(stderr, "\n%s projecting...", qPrintable(transforms[i]->objectName())); + for (int j=0; j < singleItemLists.size();j++) + transforms[i]->projectUpdate(singleItemLists[j], singleItemLists[j]); + + // advance i since we already projected for this stage. + i++; + + // set up copy again + copy.clear(); + for (int j=0; j < singleItemLists.size(); j++) + copy.append(singleItemLists[j]); + + // the next stage might be trainable, so continue to evaluate it. + continue; + } + // We project through any subsequent untrainable transforms at once // as a memory optimization in case any of these intermediate // transforms allocate a lot of memory (like OpenTransform) @@ -112,7 +130,8 @@ class PipeTransform : public CompositeTransform // by that transform at once if we can avoid it. int nextTrainableTransform = i+1; while ((nextTrainableTransform < transforms.size()) && - !transforms[nextTrainableTransform]->trainable) + !transforms[nextTrainableTransform]->trainable && + !transforms[nextTrainableTransform]->timeVarying()) nextTrainableTransform++; fprintf(stderr, " projecting..."); @@ -129,26 +148,6 @@ class PipeTransform : public CompositeTransform } } - void backProject(const Template &dst, Template &src) const - { - // Backprojecting a time-varying transform is probably not going to work. - if (timeVarying()) qFatal("No backProject defined for time-varying transform"); - - src = dst; - // Reverse order in which transforms are processed - int length = transforms.length(); - for (int i=length-1; i>=0; i--) { - Transform *f = transforms.at(i); - try { - src >> *f; - } catch (...) { - qWarning("Exception triggered when processing %s with transform %s", qPrintable(dst.file.flat()), qPrintable(f->objectName())); - src = Template(src.file); - src.file.set("FTE", true); - } - } - } - void projectUpdate(const Template &src, Template &dst) { dst = src; @@ -306,8 +305,6 @@ class ForkTransform : public CompositeTransform futures.waitForFinished(); } - void backProject(const Template &dst, Template &src) const {Transform::backProject(dst, src);} - // same as _project, but calls projectUpdate on sub-transforms void projectupdate(const Template & src, Template & dst) { @@ -645,17 +642,28 @@ public: QList input_buffer; input_buffer.reserve(src.size()); + QFutureSynchronizer futures; + for (int i =0; i < src.size();i++) { input_buffer.append(TemplateList()); output_buffer.append(TemplateList()); } - - QFutureSynchronizer futures; + QList > temp; + temp.reserve(src.size()); for (int i=0; iparallelism) futures.addFuture(QtConcurrent::run(_projectList, transform, &input_buffer[i], &output_buffer[i])); - else _projectList( transform, &input_buffer[i], &output_buffer[i]); + + if (Globals->parallelism > 1) temp.append(QtConcurrent::run(_projectList, transform, &input_buffer[i], &output_buffer[i])); + else _projectList(transform, &input_buffer[i], &output_buffer[i]); } + // We add the futures in reverse order, since in Qt 5.1 at least the + // waiting thread will wait on them in the order added (which for uniform priority + // threads is the order of execution), and we want the waiting thread to go in the opposite order + // so that it can steal runnables and do something besides wait. + for (int i = temp.size() - 1; i >= 0; i--) { + futures.addFuture(temp[i]); + } + futures.waitForFinished(); for (int i=0; i labels = data.indexProperty("Subject"); + const QList labels = data.indexProperty(inputVariable); const int dims = m.cols; vector mv, av, bv; diff --git a/openbr/plugins/openbr_internal.h b/openbr/plugins/openbr_internal.h index fb7a1b5..27136da 100644 --- a/openbr/plugins/openbr_internal.h +++ b/openbr/plugins/openbr_internal.h @@ -170,14 +170,8 @@ public: virtual void project(const TemplateList &src, TemplateList &dst) const { if (timeVarying()) { - if (!this->timeInvariantAlias) { - QMutexLocker lock(&aliasLock); - CompositeTransform * non_const = const_cast(this); - non_const->timeInvariantAlias = non_const->smartCopy(); - non_const->timeInvariantAlias->setParent(non_const); - lock.unlock(); - } - timeInvariantAlias->projectUpdate(src,dst); + CompositeTransform * non_const = const_cast(this); + non_const->projectUpdate(src,dst); return; } _project(src, dst); @@ -225,10 +219,6 @@ public: } output->file = this->file; - output->classes = classes; - output->instances = instances; - output->fraction = fraction; - output->init(); return output; @@ -237,13 +227,10 @@ public: protected: bool isTimeVarying; - mutable QMutex aliasLock; - Transform * timeInvariantAlias; - virtual void _project(const Template & src, Template & dst) const = 0; virtual void _project(const TemplateList & src, TemplateList & dst) const = 0; - CompositeTransform() : TimeVaryingTransform(false) { timeInvariantAlias = NULL; } + CompositeTransform() : TimeVaryingTransform(false) {} }; } diff --git a/openbr/plugins/opticalflow.cpp b/openbr/plugins/opticalflow.cpp new file mode 100644 index 0000000..5246ad5 --- /dev/null +++ b/openbr/plugins/opticalflow.cpp @@ -0,0 +1,57 @@ +#include +#include "openbr_internal.h" +#include "openbr/core/opencvutils.h" + +using namespace cv; + +namespace br +{ + +/*! + * \ingroup transforms + * \brief Gets a one-channel dense optical flow from two images + * \author Austin Blanton \cite imaus10 + */ +class OpticalFlowTransform : public UntrainableMetaTransform +{ + Q_OBJECT + Q_PROPERTY(double pyr_scale READ get_pyr_scale WRITE set_pyr_scale RESET reset_pyr_scale STORED false) + Q_PROPERTY(int levels READ get_levels WRITE set_levels RESET reset_levels STORED false) + Q_PROPERTY(int winsize READ get_winsize WRITE set_winsize RESET reset_winsize STORED false) + Q_PROPERTY(int iterations READ get_iterations WRITE set_iterations RESET reset_iterations STORED false) + Q_PROPERTY(int poly_n READ get_poly_n WRITE set_poly_n RESET reset_poly_n STORED false) + Q_PROPERTY(double poly_sigma READ get_poly_sigma WRITE set_poly_sigma RESET reset_poly_sigma STORED false) + Q_PROPERTY(int flags READ get_flags WRITE set_flags RESET reset_flags STORED false) + // these defaults are optimized for KTH + BR_PROPERTY(double, pyr_scale, 0.1) + BR_PROPERTY(int, levels, 1) + BR_PROPERTY(int, winsize, 5) + BR_PROPERTY(int, iterations, 10) + BR_PROPERTY(int, poly_n, 7) + BR_PROPERTY(double, poly_sigma, 1.1) + BR_PROPERTY(int, flags, 0) + + void project(const Template &src, Template &dst) const + { + // get the two images put there by AggregateFrames + if (src.size() < 2) return; + Mat prevImg = src[0], nextImg = src[1], flow, flowOneCh; + if (src[0].channels() != 1) OpenCVUtils::cvtGray(src[0], prevImg); + if (src[1].channels() != 1) OpenCVUtils::cvtGray(src[1], nextImg); + calcOpticalFlowFarneback(prevImg, nextImg, flow, pyr_scale, levels, winsize, iterations, poly_n, poly_sigma, flags); + + // the result is two channels + std::vector channels(2); + split(flow, channels); + magnitude(channels[0], channels[1], flowOneCh); + + dst += flowOneCh; + dst.file = src.file; + } +}; + +BR_REGISTER(Transform, OpticalFlowTransform) + +} // namespace br + +#include "opticalflow.moc" diff --git a/openbr/plugins/output.cpp b/openbr/plugins/output.cpp index 6353c56..d87c518 100644 --- a/openbr/plugins/output.cpp +++ b/openbr/plugins/output.cpp @@ -146,8 +146,8 @@ class meltOutput : public MatrixOutput QStringList lines; if (file.baseName() != "terminal") lines.append(QString("Query,Target,Mask,Similarity%1").arg(keys)); - QList queryLabels = File::get(queryFiles, "Subject"); - QList targetLabels = File::get(targetFiles, "Subject"); + QList queryLabels = File::get(queryFiles, "Label"); + QList targetLabels = File::get(targetFiles, "Label"); for (int i=0; i("Subject")); + lines.append(file.name + " " + file.get("Label")); QtUtils::writeFile(file, lines); } }; @@ -433,7 +433,7 @@ class rankOutput : public MatrixOutput foreach (const Pair &pair, Common::Sort(OpenCVUtils::matrixToVector(data.row(i)), true)) { if (Globals->crossValidate > 0 ? (targetFiles[pair.second].get("Partition",-1) == queryFiles[i].get("Partition",-1)) : true) { if (QString(targetFiles[pair.second]) != QString(queryFiles[i])) { - if (targetFiles[pair.second].get("Subject") == queryFiles[i].get("Subject")) { + if (targetFiles[pair.second].get("Label") == queryFiles[i].get("Label")) { ranks.append(rank); positions.append(pair.second); scores.append(pair.first); diff --git a/openbr/plugins/quality.cpp b/openbr/plugins/quality.cpp index fd3209c..e372460 100644 --- a/openbr/plugins/quality.cpp +++ b/openbr/plugins/quality.cpp @@ -19,17 +19,20 @@ class ImpostorUniquenessMeasureTransform : public Transform Q_PROPERTY(br::Distance* distance READ get_distance WRITE set_distance RESET reset_distance STORED false) Q_PROPERTY(double mean READ get_mean WRITE set_mean RESET reset_mean) Q_PROPERTY(double stddev READ get_stddev WRITE set_stddev RESET reset_stddev) + Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) BR_PROPERTY(br::Distance*, distance, Distance::make("Dist(L2)", this)) BR_PROPERTY(double, mean, 0) BR_PROPERTY(double, stddev, 1) + BR_PROPERTY(QString, inputVariable, "Label") + TemplateList impostors; float calculateIUM(const Template &probe, const TemplateList &gallery) const { - const QString probeLabel = probe.file.get("Subject"); + const QString probeLabel = probe.file.get(inputVariable); TemplateList subset = gallery; for (int j=subset.size()-1; j>=0; j--) - if (subset[j].file.get("Subject") == probeLabel) + if (subset[j].file.get(inputVariable) == probeLabel) subset.removeAt(j); QList scores = distance->compare(subset, probe); @@ -151,6 +154,7 @@ class MatchProbabilityDistance : public Distance Q_PROPERTY(br::Distance* distance READ get_distance WRITE set_distance RESET reset_distance STORED false) Q_PROPERTY(bool gaussian READ get_gaussian WRITE set_gaussian RESET reset_gaussian STORED false) Q_PROPERTY(bool crossModality READ get_crossModality WRITE set_crossModality RESET reset_crossModality STORED false) + Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) MP mp; @@ -158,7 +162,7 @@ class MatchProbabilityDistance : public Distance { distance->train(src); - const QList labels = src.indexProperty("Subject"); + const QList labels = src.indexProperty(inputVariable); QScopedPointer matrixOutput(MatrixOutput::make(FileList(src.size()), FileList(src.size()))); distance->compare(src, src, matrixOutput.data()); @@ -201,6 +205,7 @@ protected: BR_PROPERTY(br::Distance*, distance, make("Dist(L2)")) BR_PROPERTY(bool, gaussian, true) BR_PROPERTY(bool, crossModality, false) + BR_PROPERTY(QString, inputVariable, "Label") }; BR_REGISTER(Distance, MatchProbabilityDistance) @@ -217,10 +222,12 @@ class HeatMapDistance : public Distance Q_PROPERTY(bool gaussian READ get_gaussian WRITE set_gaussian RESET reset_gaussian STORED false) Q_PROPERTY(bool crossModality READ get_crossModality WRITE set_crossModality RESET reset_crossModality STORED false) Q_PROPERTY(int step READ get_step WRITE set_step RESET reset_step STORED false) + Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) BR_PROPERTY(br::Distance*, distance, make("Dist(L2)")) BR_PROPERTY(bool, gaussian, true) BR_PROPERTY(bool, crossModality, false) BR_PROPERTY(int, step, 1) + BR_PROPERTY(QString, inputVariable, "Label") QList mp; @@ -228,7 +235,7 @@ class HeatMapDistance : public Distance { distance->train(src); - const QList labels = src.indexProperty("Subject"); + const QList labels = src.indexProperty(inputVariable); QList patches; @@ -307,14 +314,16 @@ class UnitDistance : public Distance Q_PROPERTY(br::Distance *distance READ get_distance WRITE set_distance RESET reset_distance) Q_PROPERTY(float a READ get_a WRITE set_a RESET reset_a) Q_PROPERTY(float b READ get_b WRITE set_b RESET reset_b) + Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) BR_PROPERTY(br::Distance*, distance, make("Dist(L2)")) BR_PROPERTY(float, a, 1) BR_PROPERTY(float, b, 0) + BR_PROPERTY(QString, inputVariable, "Label") void train(const TemplateList &templates) { const TemplateList samples = templates.mid(0, 2000); - const QList sampleLabels = samples.indexProperty("Subject"); + const QList sampleLabels = samples.indexProperty(inputVariable); QScopedPointer matrixOutput(MatrixOutput::make(FileList(samples.size()), FileList(samples.size()))); Distance::compare(samples, samples, matrixOutput.data()); diff --git a/openbr/plugins/quantize.cpp b/openbr/plugins/quantize.cpp index 94840f6..e74513f 100644 --- a/openbr/plugins/quantize.cpp +++ b/openbr/plugins/quantize.cpp @@ -120,6 +120,10 @@ BR_REGISTER(Transform, HistEqQuantizationTransform) class BayesianQuantizationDistance : public Distance { Q_OBJECT + + Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) + BR_PROPERTY(QString, inputVariable, "Label") + QVector loglikelihoods; static void computeLogLikelihood(const Mat &data, const QList &labels, float *loglikelihood) @@ -150,7 +154,7 @@ class BayesianQuantizationDistance : public Distance qFatal("Expected sigle matrix templates of type CV_8UC1!"); const Mat data = OpenCVUtils::toMat(src.data()); - const QList templateLabels = src.indexProperty("Subject"); + const QList templateLabels = src.indexProperty(inputVariable); loglikelihoods = QVector(data.cols*256, 0); QFutureSynchronizer futures; @@ -343,9 +347,11 @@ class ProductQuantizationTransform : public Transform Q_PROPERTY(int n READ get_n WRITE set_n RESET reset_n STORED false) Q_PROPERTY(br::Distance *distance READ get_distance WRITE set_distance RESET reset_distance STORED false) Q_PROPERTY(bool bayesian READ get_bayesian WRITE set_bayesian RESET reset_bayesian STORED false) + Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) BR_PROPERTY(int, n, 2) BR_PROPERTY(br::Distance*, distance, Distance::make("L2", this)) BR_PROPERTY(bool, bayesian, false) + BR_PROPERTY(QString, inputVariable, "Label") quint16 index; QList centers; @@ -474,7 +480,7 @@ private: Mat data = OpenCVUtils::toMat(src.data()); const int step = getStep(data.cols); - const QList labels = src.indexProperty("Subject"); + const QList labels = src.indexProperty(inputVariable); Mat &lut = ProductQuantizationLUTs[index]; lut = Mat(getDims(data.cols), 256*(256+1)/2, CV_32FC1); diff --git a/openbr/plugins/quantize2.cpp b/openbr/plugins/quantize2.cpp index 63ea769..8d05523 100644 --- a/openbr/plugins/quantize2.cpp +++ b/openbr/plugins/quantize2.cpp @@ -19,6 +19,10 @@ namespace br class BayesianQuantizationTransform : public Transform { Q_OBJECT + + Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) + BR_PROPERTY(QString, inputVariable, "Label") + QVector thresholds; static void computeThresholdsRecursive(const QVector &cumulativeGenuines, const QVector &cumulativeImpostors, @@ -77,7 +81,7 @@ class BayesianQuantizationTransform : public Transform void train(const TemplateList &src) { const Mat data = OpenCVUtils::toMat(src.data()); - const QList labels = src.indexProperty("Subject"); + const QList labels = src.indexProperty(inputVariable); thresholds = QVector(256*data.cols); diff --git a/openbr/plugins/regions.cpp b/openbr/plugins/regions.cpp index 4197b95..e6c30fa 100644 --- a/openbr/plugins/regions.cpp +++ b/openbr/plugins/regions.cpp @@ -114,7 +114,7 @@ BR_REGISTER(Transform, CatTransform) /*! * \ingroup transforms * \brief Concatenates all input matrices by row into a single matrix. - * All matricies must have the same row counts. + * All matricies must have the same column counts. * \author Josh Klontz \cite jklontz */ class CatRowsTransform : public UntrainableMetaTransform @@ -131,6 +131,31 @@ BR_REGISTER(Transform, CatRowsTransform) /*! * \ingroup transforms + * \brief Concatenates all input matrices by column into a single matrix. + * Use after a fork to concatenate two feature matrices by column. + * \author Austin Blanton \cite imaus10 + */ +class CatColsTransform : public UntrainableMetaTransform +{ + Q_OBJECT + + void project(const Template &src, Template &dst) const + { + if (src.empty()) return; + dst.file = src.file; + Mat m = OpenCVUtils::toMatByRow(src); + // right now this just splits src in half and joins them horizontally + // TODO: add partitions parameter for more than a single split + Mat first = m.rowRange(Range(0, m.rows/2)); + Mat second = m.rowRange(Range(m.rows/2, m.rows)); + hconcat(first, second, dst); + } +}; + +BR_REGISTER(Transform, CatColsTransform) + +/*! + * \ingroup transforms * \brief Reshape the each matrix to the specified number of rows. * \author Josh Klontz \cite jklontz */ diff --git a/openbr/plugins/stream.cpp b/openbr/plugins/stream.cpp index 884176a..298b822 100644 --- a/openbr/plugins/stream.cpp +++ b/openbr/plugins/stream.cpp @@ -31,8 +31,10 @@ public: virtual ~SharedBuffer() {} virtual void addItem(FrameData * input)=0; + virtual void reset()=0; virtual FrameData * tryGetItem()=0; + virtual int size()=0; }; // for n - 1 boundaries, multiple threads call addItem, the frames are @@ -74,6 +76,21 @@ public: return output; } + virtual int size() + { + QMutexLocker lock(&bufferGuard); + return buffer.size(); + } + virtual void reset() + { + if (size() != 0) + qDebug("Sequencing buffer has non-zero size during reset!"); + + QMutexLocker lock(&bufferGuard); + next_target = 0; + } + + private: QMutex bufferGuard; int next_target; @@ -95,6 +112,11 @@ public: outputBuffer = &buffer2; } + int size() + { + QReadLocker readLock(&bufferGuard); + return inputBuffer->size() + outputBuffer->size(); + } // called from the producer thread void addItem(FrameData * input) @@ -133,6 +155,13 @@ public: return output; } + virtual void reset() + { + if (this->size() != 0) + qDebug("Shared buffer has non-zero size during reset!"); + } + + private: // The read-write lock. The thread adding to this buffer can add // to the current input buffer if it has a read lock. The thread @@ -160,9 +189,8 @@ class DataSource public: DataSource(int maxFrames=500) { + // The sequence number of the last frame final_frame = -1; - last_issued = -2; - last_received = -3; for (int i=0; i < maxFrames;i++) { allFrames.addItem(new FrameData()); @@ -181,52 +209,67 @@ public: } // non-blocking version of getFrame - FrameData * tryGetFrame() + // Returns a NULL FrameData if too many frames are out, or the + // data source is broken. Sets last_frame to true iff the FrameData + // returned is the last valid frame, and the data source is now broken. + FrameData * tryGetFrame(bool & last_frame) { + last_frame = false; + + if (is_broken) { + return NULL; + } + + // Try to get a FrameData from the pool, if we can't it means too many + // frames are already out, and we will return NULL to indicate failure FrameData * aFrame = allFrames.tryGetItem(); if (aFrame == NULL) return NULL; - aFrame->data.clear(); - aFrame->sequenceNumber = -1; - + // Try to actually read a frame, if this returns false the data source is broken bool res = getNext(*aFrame); - // The datasource broke. - if (!res) { - allFrames.addItem(aFrame); - + // The datasource broke, update final_frame + if (!res) + { QMutexLocker lock(&last_frame_update); - // Did we already receive the last frame? - final_frame = last_issued; + final_frame = lookAhead.back()->sequenceNumber; + allFrames.addItem(aFrame); + } + else { + lookAhead.push_back(aFrame); + } - // We got the last frame before the data source broke, - // better pulse lastReturned - if (final_frame == last_received) { - lastReturned.wakeAll(); - } - else if (final_frame < last_received) - std::cout << "Bad last frame " << final_frame << " but received " << last_received << std::endl; + // we will return the first frame on the lookAhead buffer + FrameData * rVal = lookAhead.first(); + lookAhead.pop_front(); - return NULL; + // If this is the last frame, say so + if (rVal->sequenceNumber == final_frame) { + last_frame = true; + is_broken = true; } - last_issued = aFrame->sequenceNumber; - return aFrame; + + return rVal; } - // Returns true if the frame returned was the last + // Return a frame to the pool, returns true if the frame returned was the last // frame issued, false otherwise bool returnFrame(FrameData * inputFrame) { + int frameNumber = inputFrame->sequenceNumber; + + inputFrame->data.clear(); + inputFrame->sequenceNumber = -1; allFrames.addItem(inputFrame); bool rval = false; QMutexLocker lock(&last_frame_update); - last_received = inputFrame->sequenceNumber; - if (inputFrame->sequenceNumber == final_frame) { + if (frameNumber == final_frame) { // We just received the last frame, better pulse + allReturned = true; lastReturned.wakeAll(); rval = true; } @@ -234,23 +277,88 @@ public: return rval; } - void waitLast() + bool waitLast() { QMutexLocker lock(&last_frame_update); - lastReturned.wait(&last_frame_update); + + while (!allReturned) + { + // This would be a safer wait if we used a timeout, but + // theoretically that should never matter. + lastReturned.wait(&last_frame_update); + } + return true; } - virtual void close() = 0; - virtual bool open(Template & output, int start_index=0) = 0; - virtual bool isOpen() = 0; + bool open(Template & output, int start_index = 0) + { + is_broken = false; + allReturned = false; + + // The last frame isn't initialized yet + final_frame = -1; + // Start our sequence numbers from the input index + next_sequence_number = start_index; + + // Actually open the data source + bool open_res = concreteOpen(output); + + // We couldn't open the data source + if (!open_res) { + is_broken = true; + return false; + } + + // Try to get a frame from the global pool + FrameData * firstFrame = allFrames.tryGetItem(); + + // If this fails, things have gone pretty badly. + if (firstFrame == NULL) { + is_broken = true; + return false; + } + + // Read a frame from the video source + bool res = getNext(*firstFrame); + // the data source broke already, we couldn't even get one frame + // from it even though it claimed to have opened successfully. + if (!res) { + is_broken = true; + return false; + } + + // We read one frame ahead of the last one returned, this allows + // us to know which frame is the final frame when we return it. + lookAhead.append(firstFrame); + return true; + } + + /* + * Pure virtual methods + */ + + // isOpen doesn't appear to particularly work when used on opencv + // VideoCaptures, so we don't use it for anything important. + virtual bool isOpen()=0; + // Called from open, open the data source specified by the input + // template, don't worry about setting any of the state variables + // set in open. + virtual bool concreteOpen(Template & output) = 0; + // Get the next frame from the data source, store the results in + // FrameData (including the actual frame and appropriate sequence + // number). virtual bool getNext(FrameData & input) = 0; + // close the currently open data source. + virtual void close() = 0; + int next_sequence_number; protected: DoubleBuffer allFrames; int final_frame; - int last_issued; - int last_received; + bool is_broken; + bool allReturned; + QList lookAhead; QWaitCondition lastReturned; QMutex last_frame_update; @@ -262,14 +370,14 @@ class VideoDataSource : public DataSource public: VideoDataSource(int maxFrames) : DataSource(maxFrames) {} - bool open(Template &input, int start_index=0) + bool concreteOpen(Template &input) { - final_frame = -1; - last_issued = -2; - last_received = -3; - - next_idx = start_index; basis = input; + + // We can open either files (well actually this includes addresses of ip cameras + // through ffmpeg), or webcams. Webcam VideoCaptures are created through a separate + // overload of open that takes an integer, not a string. + // So, does this look like an integer? bool is_int = false; int anInt = input.file.name.toInt(&is_int); if (is_int) @@ -287,7 +395,8 @@ public: } else { // Yes, we should specify absolute path: // http://stackoverflow.com/questions/9396459/loading-a-video-in-opencv-in-python - video.open(QFileInfo(input.file.name).absoluteFilePath().toStdString()); + QString fileName = (Globals->path.isEmpty() ? "" : Globals->path + "/") + input.file.name; + video.open(QFileInfo(fileName).absoluteFilePath().toStdString()); } return video.isOpened(); @@ -302,29 +411,38 @@ public: private: bool getNext(FrameData & output) { - if (!isOpen()) + if (!isOpen()) { + qDebug("video source is not open"); return false; + } output.data.append(Template(basis.file)); - output.data.last().append(cv::Mat()); + output.data.last().m() = cv::Mat(); - output.sequenceNumber = next_idx; - next_idx++; + output.sequenceNumber = next_sequence_number; + next_sequence_number++; - bool res = video.read(output.data.last().last()); - output.data.last().last() = output.data.last().last().clone(); + cv::Mat temp; + bool res = video.read(temp); if (!res) { + // The video capture broke, return false. + output.data.last().m() = cv::Mat(); close(); return false; } + + // This clone is critical, if we don't do it then the matrix will + // be an alias of an internal buffer of the video source, leading + // to various problems later. + output.data.last().m() = temp.clone(); + output.data.last().file.set("FrameNumber", output.sequenceNumber); return true; } cv::VideoCapture video; Template basis; - int next_idx; }; // Given a template as input, return its matrices one by one on subsequent calls @@ -334,21 +452,18 @@ class TemplateDataSource : public DataSource public: TemplateDataSource(int maxFrames) : DataSource(maxFrames) { - current_idx = INT_MAX; + current_matrix_idx = INT_MAX; data_ok = false; } - bool data_ok; - bool open(Template &input, int start_index=0) + // To "open" it we just set appropriate indices, we assume that if this + // is an image, it is already loaded into memory. + bool concreteOpen(Template &input) { basis = input; - current_idx = 0; - next_sequence = start_index; - final_frame = -1; - last_issued = -2; - last_received = -3; + current_matrix_idx = 0; - data_ok = current_idx < basis.size(); + data_ok = current_matrix_idx < basis.size(); return data_ok; } @@ -358,39 +473,41 @@ public: void close() { - current_idx = INT_MAX; + current_matrix_idx = INT_MAX; basis.clear(); } private: bool getNext(FrameData & output) { - data_ok = current_idx < basis.size(); + data_ok = current_matrix_idx < basis.size(); if (!data_ok) return false; - output.data.append(basis[current_idx]); - current_idx++; + output.data.append(basis[current_matrix_idx]); + current_matrix_idx++; - output.sequenceNumber = next_sequence; - next_sequence++; + output.sequenceNumber = next_sequence_number; + next_sequence_number++; output.data.last().file.set("FrameNumber", output.sequenceNumber); return true; } Template basis; - int current_idx; - int next_sequence; + // Index of the next matrix to output from the template + int current_matrix_idx; + + // is current_matrix_idx in bounds? + bool data_ok; }; -// Given a template as input, create a VideoDataSource or a TemplateDataSource -// depending on whether or not it looks like the input template has already -// loaded frames into memory. +// Given a templatelist as input, create appropriate data source for each +// individual template class DataSourceManager : public DataSource { public: - DataSourceManager() + DataSourceManager(int activeFrames=100) : DataSource(activeFrames) { actualSource = NULL; } @@ -400,6 +517,11 @@ public: close(); } + int size() + { + return this->allFrames.size(); + } + void close() { if (actualSource) { @@ -409,33 +531,40 @@ public: } } + // We are used through a call to open(TemplateList) bool open(TemplateList & input) { - currentIdx = 0; + // Set up variables specific to us + current_template_idx = 0; templates = input; - return open(templates[currentIdx]); + // Call datasourece::open on the first template to set up + // state variables + return DataSource::open(templates[current_template_idx]); } - bool open(Template & input, int start_index=0) + // Create an actual data source of appropriate type for this template + // (initially called via the call to DataSource::open, called later + // as we run out of frames on our templates). + bool concreteOpen(Template & input) { close(); - final_frame = -1; - last_issued = -2; - last_received = -3; - next_frame = start_index; + bool open_res = false; // Input has no matrices? Its probably a video that hasn't been loaded yet if (input.empty()) { actualSource = new VideoDataSource(0); - actualSource->open(input, next_frame); + open_res = actualSource->concreteOpen(input); } + // If the input is not empty, we assume it is a set of frames already + // in memory. else { - // create frame dealer actualSource = new TemplateDataSource(0); - actualSource->open(input, next_frame); + open_res = actualSource->concreteOpen(input); } - if (!isOpen()) { + + // The data source failed to open + if (!open_res) { delete actualSource; actualSource = NULL; return false; @@ -446,30 +575,55 @@ public: bool isOpen() { return !actualSource ? false : actualSource->isOpen(); } protected: - int currentIdx; - int next_frame; + // Index of the template in the templatelist we are currently reading from + int current_template_idx; + TemplateList templates; DataSource * actualSource; + // Get the next frame, if we run out of frames on the current template + // move on to the next one. bool getNext(FrameData & output) { bool res = actualSource->getNext(output); + output.sequenceNumber = next_sequence_number; + + // OK we got a frame if (res) { - next_frame = output.sequenceNumber+1; + // Override the sequence number set by actualSource + output.data.last().file.set("FrameNumber", output.sequenceNumber); + next_sequence_number++; + if (output.data.last().last().empty()) + qDebug("broken matrix"); return true; } + // We didn't get a frame, try to move on to the next template. while(!res) { - currentIdx++; + output.data.clear(); + current_template_idx++; - if (currentIdx >= templates.size()) + // No more templates? We're done + if (current_template_idx >= templates.size()) return false; - bool open_res = open(templates[currentIdx], next_frame); + + // open the next data source + bool open_res = concreteOpen(templates[current_template_idx]); + // We couldn't open it, give up? We could maybe continue here + // but don't currently. if (!open_res) return false; + + // get a frame from the newly opened data source, if that fails + // we continue to open the next one. res = actualSource->getNext(output); } + // Finally, set the sequence number for the frame we actually return. + output.sequenceNumber = next_sequence_number++; + output.data.last().file.set("FrameNumber", output.sequenceNumber); + + if (output.data.last().last().empty()) + qDebug("broken matrix"); - next_frame = output.sequenceNumber+1; return res; } @@ -477,9 +631,14 @@ protected: class ProcessingStage; -class BasicLoop : public QRunnable +class BasicLoop : public QRunnable, public QFutureInterface { public: + BasicLoop() + { + this->reportStarted(); + } + void run(); QList * stages; @@ -506,12 +665,15 @@ public: virtual void reset()=0; + virtual void status()=0; + protected: int thread_count; SharedBuffer * inputBuffer; ProcessingStage * nextStage; QList * stages; + QThreadPool * threads; Transform * transform; }; @@ -530,6 +692,7 @@ void BasicLoop::run() current_idx++; current_idx = current_idx % stages->size(); } + this->reportFinished(); } class MultiThreadStage : public ProcessingStage @@ -537,7 +700,8 @@ class MultiThreadStage : public ProcessingStage public: MultiThreadStage(int _input) : ProcessingStage(_input) {} - + // Not much to worry about here, we will project the input + // and try to continue to the next stage. FrameData * run(FrameData * input, bool & should_continue) { if (input == NULL) { @@ -551,7 +715,8 @@ public: return input; } - // Called from a different thread than run + // Called from a different thread than run. Nothing to worry about + // we offer no restrictions on when loops may enter this stage. virtual bool tryAcquireNextStage(FrameData *& input) { (void) input; @@ -562,9 +727,11 @@ public: { // nothing to do. } + void status(){ + qDebug("multi thread stage %d, nothing to worry about", this->stage_id); + } }; - class SingleThreadStage : public ProcessingStage { public: @@ -572,13 +739,18 @@ public: { currentStatus = STOPPING; next_target = 0; + // If the previous stage is single-threaded, queued inputs + // are stored in a double buffer if (input_variance) { this->inputBuffer = new DoubleBuffer(); } + // If it's multi-threaded we need to put the inputs back in order + // before we can use them, so we use a sequencing buffer. else { this->inputBuffer = new SequencingBuffer(); } } + ~SingleThreadStage() { delete inputBuffer; @@ -589,6 +761,7 @@ public: QWriteLocker writeLock(&statusLock); currentStatus = STOPPING; next_target = 0; + inputBuffer->reset(); } @@ -627,18 +800,26 @@ public: lock.unlock(); if (newItem) - { - BasicLoop * next = new BasicLoop(); - next->stages = stages; - next->start_idx = this->stage_id; - next->startItem = newItem; - - QThreadPool::globalInstance()->start(next, stages->size() - this->stage_id); - } + startThread(newItem); return input; } + void startThread(br::FrameData * newItem) + { + BasicLoop * next = new BasicLoop(); + next->stages = stages; + next->start_idx = this->stage_id; + next->startItem = newItem; + + // We start threads with priority equal to their stage id + // This is intended to ensure progression, we do queued late stage + // jobs before queued early stage jobs, and so tend to finish frames + // rather than go stage by stage. In Qt 5.1, priorities are priorities + // so we use the stage_id directly. + this->threads->start(next, stage_id); + } + // Calledfrom a different thread than run. bool tryAcquireNextStage(FrameData *& input) @@ -671,79 +852,109 @@ public: return true; } + + void status(){ + qDebug("single thread stage %d, status starting? %d, next %d buffer size %d", this->stage_id, this->currentStatus == SingleThreadStage::STARTING, this->next_target, this->inputBuffer->size()); + } + }; -// No input buffer, instead we draw templates from some data source -// Will be operated by the main thread for the stream +// This stage reads new frames from the data source. class FirstStage : public SingleThreadStage { public: - FirstStage() : SingleThreadStage(true) {} + FirstStage(int activeFrames = 100) : SingleThreadStage(true), dataSource(activeFrames){ } DataSourceManager dataSource; + void reset() + { + dataSource.close(); + SingleThreadStage::reset(); + } + FrameData * run(FrameData * input, bool & should_continue) { - // Is there anything on our input buffer? If so we should start a thread with that. - QWriteLocker lock(&statusLock); - input = dataSource.tryGetFrame(); - // Datasource broke? - if (!input) - { - currentStatus = STOPPING; - should_continue = false; - return NULL; - } - lock.unlock(); + if (input == NULL) + qFatal("NULL frame in input stage"); + // Can we enter the next stage? should_continue = nextStage->tryAcquireNextStage(input); - BasicLoop * next = new BasicLoop(); - next->stages = stages; - next->start_idx = this->stage_id; - next->startItem = NULL; + // Try to get a frame from the datasource, we keep working on + // the frame we have, but we will queue another job for the next + // frame if a frame is currently available. + QWriteLocker lock(&statusLock); + bool last_frame = false; + FrameData * newFrame = dataSource.tryGetFrame(last_frame); - QThreadPool::globalInstance()->start(next, stages->size() - this->stage_id); + // Were we able to get a frame? + if (newFrame) startThread(newFrame); + // If not this stage will enter a stopped state. + else { + currentStatus = STOPPING; + } + + lock.unlock(); return input; } - // Calledfrom a different thread than run. + // The last stage, trying to access the first stage bool tryAcquireNextStage(FrameData *& input) { + // Return the frame, was it the last one? bool was_last = dataSource.returnFrame(input); input = NULL; + + // OK we won't continue. if (was_last) { return false; } - if (!dataSource.isOpen()) - return false; - QReadLocker lock(&statusLock); - // Thread is already running, we should just return + // If the first stage is already active we will just end. if (currentStatus == STARTING) { return false; } - // Have to change to a write lock to modify currentStatus + + // Otherwise we will try to continue, but to do so we have to + // escalate the lock, and sadly there is no way to do so without + // releasing the read-mode lock, and getting a new write-mode lock. lock.unlock(); QWriteLocker writeLock(&statusLock); - // But someone else might have started a thread in the meantime + // currentStatus might have changed in the gap between releasing the read + // lock and getting the write lock. if (currentStatus == STARTING) { return false; } - // Ok we'll start a thread + + bool last_frame = false; + // Try to get a frame from the data source, if we get one we will + // continue to the first stage. + input = dataSource.tryGetFrame(last_frame); + + if (!input) { + return false; + } + currentStatus = STARTING; - // We always start a readstage thread with null input, so nothing to do here return true; } + void status(){ + qDebug("Read stage %d, status starting? %d, next frame %d buffer size %d", this->stage_id, this->currentStatus == SingleThreadStage::STARTING, this->next_target, this->dataSource.size()); + } + + }; +// Appened to the end of a Stream's transform sequence. Collects the output +// from each frame on a single templatelist class LastStage : public SingleThreadStage { public: @@ -756,6 +967,7 @@ public: private: TemplateList collectedOutput; public: + void reset() { collectedOutput.clear(); @@ -774,11 +986,14 @@ public: } next_target = input->sequenceNumber + 1; + // add the item to our output buffer collectedOutput.append(input->data); + // Can we enter the read stage? should_continue = nextStage->tryAcquireNextStage(input); - // Is there anything on our input buffer? If so we should start a thread with that. + // Is there anything on our input buffer? If so we should start a thread + // in this stage to process that frame. QWriteLocker lock(&statusLock); FrameData * newItem = inputBuffer->tryGetItem(); if (!newItem) @@ -788,23 +1003,25 @@ public: lock.unlock(); if (newItem) - { - BasicLoop * next = new BasicLoop(); - next->stages = stages; - next->start_idx = this->stage_id; - next->startItem = newItem; - - QThreadPool::globalInstance()->start(next, stages->size() - this->stage_id); - } + startThread(newItem); return input; } + + void status(){ + qDebug("Collection stage %d, status starting? %d, next %d buffer size %d", this->stage_id, this->currentStatus == SingleThreadStage::STARTING, this->next_target, this->inputBuffer->size()); + } + }; + class StreamTransform : public CompositeTransform { Q_OBJECT public: + Q_PROPERTY(int activeFrames READ get_activeFrames WRITE set_activeFrames RESET reset_activeFrames) + BR_PROPERTY(int, activeFrames, 100) + void train(const TemplateList & data) { foreach(Transform * transform, transforms) { @@ -826,7 +1043,8 @@ public: qFatal("whatever"); } - // start processing + // start processing, consider all templates in src a continuous + // 'video' void projectUpdate(const TemplateList & src, TemplateList & dst) { dst = src; @@ -834,21 +1052,27 @@ public: bool res = readStage->dataSource.open(dst); if (!res) return; - QThreadPool::globalInstance()->releaseThread(); + // Start the first thread in the stream. + QWriteLocker lock(&readStage->statusLock); readStage->currentStatus = SingleThreadStage::STARTING; - BasicLoop loop; - loop.stages = &this->processingStages; - loop.start_idx = 0; - loop.startItem = NULL; - loop.setAutoDelete(false); + // We have to get a frame before starting the thread + bool last_frame = false; + FrameData * firstFrame = readStage->dataSource.tryGetFrame(last_frame); + if (firstFrame == NULL) + qFatal("Failed to read first frame of video"); - QThreadPool::globalInstance()->start(&loop, processingStages.size() - processingStages[0]->stage_id); + readStage->startThread(firstFrame); + lock.unlock(); - // Wait for the end. - readStage->dataSource.waitLast(); - QThreadPool::globalInstance()->reserveThread(); + // Wait for the stream to process the last frame available from + // the data source. + bool wait_res = false; + wait_res = readStage->dataSource.waitLast(); + // Now that there are no more incoming frames, call finalize + // on each transform in turn to collect any last templates + // they wish to issue. TemplateList final_output; // Push finalize through the stages @@ -864,7 +1088,8 @@ public: final_output.append(output_set); } - // dst is set to all output received by the final stage + // dst is set to all output received by the final stage, along + // with anything output via the calls to finalize. dst = collectionStage->getOutput(); dst.append(final_output); @@ -876,7 +1101,8 @@ public: virtual void finalize(TemplateList & output) { (void) output; - // Not handling this yet -cao + // Nothing in particular to do here, stream calls finalize + // on all child transforms as part of projectUpdate } // Create and link stages @@ -884,26 +1110,49 @@ public: { if (transforms.isEmpty()) return; + // call CompositeTransform::init so that trainable is set + // correctly. + CompositeTransform::init(); + + // We share a thread pool across streams attached to the same + // parent tranform, retrieve or create a thread pool based + // on our parent transform. + QMutexLocker poolLock(&poolsAccess); + QHash::Iterator it; + if (!pools.contains(this->parent())) { + it = pools.insert(this->parent(), new QThreadPool(this)); + it.value()->setMaxThreadCount(Globals->parallelism); + } + else it = pools.find(this->parent()); + threads = it.value(); + poolLock.unlock(); + + // Are our children time varying or not? This decides whether + // we run them in single threaded or multi threaded stages stage_variance.reserve(transforms.size()); foreach (const br::Transform *transform, transforms) { stage_variance.append(transform->timeVarying()); } - readStage = new FirstStage(); + // Additionally, we have a separate stage responsible for reading + // frames from the data source + readStage = new FirstStage(activeFrames); processingStages.push_back(readStage); readStage->stage_id = 0; readStage->stages = &this->processingStages; + readStage->threads = this->threads; + // Initialize and link a processing stage for each of our child + // transforms. int next_stage_id = 1; - bool prev_stage_variance = true; for (int i =0; i < transforms.size(); i++) { if (stage_variance[i]) - { + // Whether or not the previous stage is multi-threaded controls + // the type of input buffer we need in a single threaded stage. processingStages.append(new SingleThreadStage(prev_stage_variance)); - } else processingStages.append(new MultiThreadStage(Globals->parallelism)); @@ -914,24 +1163,31 @@ public: processingStages[i]->nextStage = processingStages[i+1]; processingStages.last()->stages = &this->processingStages; + processingStages.last()->threads = this->threads; processingStages.last()->transform = transforms[i]; prev_stage_variance = stage_variance[i]; } + // We also have the last stage, which just puts the output of the + // previous stages on a template list. collectionStage = new LastStage(prev_stage_variance); processingStages.append(collectionStage); collectionStage->stage_id = next_stage_id; collectionStage->stages = &this->processingStages; + collectionStage->threads = this->threads; + // the last transform stage points to collection stage processingStages[processingStages.size() - 2]->nextStage = collectionStage; - // It's a ring buffer, get it? + // And the collection stage points to the read stage, because this is + // a ring buffer. collectionStage->nextStage = readStage; } ~StreamTransform() { + // Delete all the stages for (int i = 0; i < processingStages.size(); i++) { delete processingStages[i]; } @@ -945,6 +1201,25 @@ protected: QList processingStages; + // This is a map from parent transforms (of Streams) to thread pools. Rather + // than starting threads on the global thread pool, Stream uses separate thread pools + // keyed on their parent transform. This is necessary because stream's project starts + // threads, then enters an indefinite wait for them to finish. Since we are starting + // threads using thread pools, threads themselves are a limited resource. Therefore, + // the type of hold and wait done by stream project can lead to deadlock unless + // resources are ordered in such a way that a circular wait will not occur. The points + // of this hash is to introduce a resource ordering (on threads) that mirrors the structure + // of the algorithm. So, as long as the structure of the algorithm is a DAG, the wait done + // by stream project will not be circular, since every thread in stream project is waiting + // for threads at a lower level to do the work. + // This issue doesn't come up in distribute, since a thread waiting on a QFutureSynchronizer + // will steal work from those jobs, so in that sense distribute isn't doing a hold and wait. + // Waiting for a QFutureSynchronzier isn't really possible here since stream runs an indeteriminate + // number of jobs. + static QHash pools; + static QMutex poolsAccess; + QThreadPool * threads; + void _project(const Template &src, Template &dst) const { (void) src; (void) dst; @@ -957,6 +1232,9 @@ protected: } }; +QHash StreamTransform::pools; +QMutex StreamTransform::poolsAccess; + BR_REGISTER(Transform, StreamTransform) diff --git a/openbr/plugins/svm.cpp b/openbr/plugins/svm.cpp index 39efa4d..d4e2aad 100644 --- a/openbr/plugins/svm.cpp +++ b/openbr/plugins/svm.cpp @@ -101,6 +101,8 @@ class SVMTransform : public Transform Q_PROPERTY(Type type READ get_type WRITE set_type RESET reset_type STORED false) Q_PROPERTY(float C READ get_C WRITE set_C RESET reset_C STORED false) Q_PROPERTY(float gamma READ get_gamma WRITE set_gamma RESET reset_gamma STORED false) + Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) + Q_PROPERTY(QString outputVariable READ get_outputVariable WRITE set_outputVariable RESET reset_outputVariable STORED false) public: enum Kernel { Linear = CvSVM::LINEAR, @@ -119,6 +121,9 @@ private: BR_PROPERTY(Type, type, C_SVC) BR_PROPERTY(float, C, -1) BR_PROPERTY(float, gamma, -1) + BR_PROPERTY(QString, inputVariable, "") + BR_PROPERTY(QString, outputVariable, "") + SVM svm; QHash labelMap; @@ -128,14 +133,15 @@ private: { Mat data = OpenCVUtils::toMat(_data.data()); Mat lab; - // If we are doing regression, assume subject has float values + // If we are doing regression, the input variable should have float + // values if (type == EPS_SVR || type == NU_SVR) { - lab = OpenCVUtils::toMat(File::get(_data, "Subject")); + lab = OpenCVUtils::toMat(File::get(_data, inputVariable)); } - // If we are doing classification, assume subject has discrete values, map them - // and store the mapping data + // If we are doing classification, we should be dealing with discrete + // values. Map them and store the mapping data else { - QList dataLabels = _data.indexProperty("Subject", labelMap, reverseLookup); + QList dataLabels = _data.indexProperty(inputVariable, labelMap, reverseLookup); lab = OpenCVUtils::toMat(dataLabels); } trainSVM(svm, data, lab, kernel, type, C, gamma); @@ -146,9 +152,9 @@ private: dst = src; float prediction = svm.predict(src.m().reshape(1, 1)); if (type == EPS_SVR || type == NU_SVR) - dst.file.set("Subject", prediction); + dst.file.set(outputVariable, prediction); else - dst.file.set("Subject", reverseLookup[prediction]); + dst.file.set(outputVariable, reverseLookup[prediction]); } void store(QDataStream &stream) const @@ -162,6 +168,24 @@ private: loadSVM(svm, stream); stream >> labelMap >> reverseLookup; } + + void init() + { + // Since SVM can do regression or classification, we have to check the problem type before + // specifying target variable names + if (inputVariable.isEmpty()) + { + if (type == EPS_SVR || type == NU_SVR) { + inputVariable = "Regressor"; + if (outputVariable.isEmpty()) + outputVariable = "Regressand"; + } + else + inputVariable = "Label"; + } + if (outputVariable.isEmpty()) + outputVariable = inputVariable; + } }; BR_REGISTER(Transform, SVMTransform) @@ -178,6 +202,8 @@ class SVMDistance : public Distance Q_ENUMS(Type) Q_PROPERTY(Kernel kernel READ get_kernel WRITE set_kernel RESET reset_kernel STORED false) Q_PROPERTY(Type type READ get_type WRITE set_type RESET reset_type STORED false) + Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) + public: enum Kernel { Linear = CvSVM::LINEAR, @@ -194,13 +220,14 @@ public: private: BR_PROPERTY(Kernel, kernel, Linear) BR_PROPERTY(Type, type, EPS_SVR) + BR_PROPERTY(QString, inputVariable, "Label") SVM svm; void train(const TemplateList &src) { const Mat data = OpenCVUtils::toMat(src.data()); - const QList lab = src.indexProperty("Subject"); + const QList lab = src.indexProperty(inputVariable); const int instances = data.rows * (data.rows+1) / 2; Mat deltaData(instances, data.cols, data.type()); diff --git a/openbr/plugins/validate.cpp b/openbr/plugins/validate.cpp index 8f84fcf..a776574 100644 --- a/openbr/plugins/validate.cpp +++ b/openbr/plugins/validate.cpp @@ -159,26 +159,29 @@ class MetadataDistance : public Distance float compare(const Template &a, const Template &b) const { foreach (const QString &key, filters) { + QString aValue = a.file.get(key, QString()); + QString bValue = b.file.get(key, QString()); - const QString aValue = a.file.get(key, ""); - const QString bValue = b.file.get(key, ""); + // The query value may be a range. Let's check. + if (bValue.isEmpty()) bValue = QtUtils::toString(b.file.get(key, QPointF())); if (aValue.isEmpty() || bValue.isEmpty()) continue; bool keep = false; + bool ok; - if (aValue[0] == '(') /* Range */ { - QStringList values = aValue.split(','); + QPointF range = QtUtils::toPoint(bValue,&ok); - int value = values[0].mid(1).toInt(); - values[1].chop(1); - int upperBound = values[1].toInt(); + if (ok) /* Range */ { + int value = range.x(); + int upperBound = range.y(); while (value <= upperBound) { - if (aValue == bValue) { + if (aValue == QString::number(value)) { keep = true; break; } + value++; } } else if (aValue == bValue) keep = true; diff --git a/scripts/downloadDatasets.sh b/scripts/downloadDatasets.sh index 7aa9e39..35130ae 100755 --- a/scripts/downloadDatasets.sh +++ b/scripts/downloadDatasets.sh @@ -64,6 +64,5 @@ if [ ! -d ../data/KTH/vid ]; then rm ${vidclass}.zip done # this file is corrupted - chmod +w ../data/KTH/vid/boxing/person01_boxing_d4_uncomp.avi - rm ../data/KTH/vid/boxing/person01_boxing_d4_uncomp.avi + rm -f ../data/KTH/vid/boxing/person01_boxing_d4_uncomp.avi fi diff --git a/scripts/evalAgeRegression-PCSO.sh b/scripts/evalAgeRegression-PCSO.sh index 8d0f286..9e294fe 100755 --- a/scripts/evalAgeRegression-PCSO.sh +++ b/scripts/evalAgeRegression-PCSO.sh @@ -4,8 +4,12 @@ if [ ! -f evalAgeRegression-PCSO.sh ]; then exit fi +export BR="../build/app/br/br -useGui 0" +export PCSO_DIR=/user/pripshare/Databases/FaceDatabases/PCSO/PCSO/ +export ageAlg=AgeRegression + # Create a file list by querying the database -br -quiet -algorithm Identity -enroll "../data/PCSO/PCSO.db[query='SELECT File,Age,PersonID FROM PCSO WHERE Age >= 15 AND AGE <= 75', subset=1:200]" terminal.txt > Input.txt +$BR -quiet -algorithm Identity -enroll "$PCSO_DIR/PCSO.db[query='SELECT File,Age,PersonID FROM PCSO WHERE Age >= 17 AND AGE <= 68', subset=1:200]" terminal.txt > Input.txt # Enroll the file list and evaluate performance -br -algorithm AgeRegression -path ../data/PCSO/img -enroll Input.txt Output.txt -evalRegression Output.txt Input.txt +$BR -algorithm $ageAlg -path $PCSO_DIR/Images -enroll Input.txt Output.txt -evalRegression Output.txt Input.txt Age diff --git a/scripts/evalFaceRecognition-MEDS.sh b/scripts/evalFaceRecognition-MEDS.sh index 231370f..673f674 100755 --- a/scripts/evalFaceRecognition-MEDS.sh +++ b/scripts/evalFaceRecognition-MEDS.sh @@ -20,11 +20,11 @@ if [ ! -e Algorithm_Dataset ]; then fi if [ ! -e MEDS.mask ]; then - br -makeMask ../data/MEDS/sigset/MEDS_frontal_target.xml ../data/MEDS/sigset/MEDS_frontal_query.xml MEDS.mask + br -useGui 0 -makeMask ../data/MEDS/sigset/MEDS_frontal_target.xml ../data/MEDS/sigset/MEDS_frontal_query.xml MEDS.mask fi # Run Algorithm on MEDS -br -algorithm ${ALGORITHM} -path ../data/MEDS/img -compare ../data/MEDS/sigset/MEDS_frontal_target.xml ../data/MEDS/sigset/MEDS_frontal_query.xml ${ALGORITHM}_MEDS.mtx -eval ${ALGORITHM}_MEDS.mtx MEDS.mask Algorithm_Dataset/${ALGORITHM}_MEDS.csv +br -useGui 0 -algorithm ${ALGORITHM} -path ../data/MEDS/img -compare ../data/MEDS/sigset/MEDS_frontal_target.xml ../data/MEDS/sigset/MEDS_frontal_query.xml ${ALGORITHM}_MEDS.mtx -eval ${ALGORITHM}_MEDS.mtx MEDS.mask Algorithm_Dataset/${ALGORITHM}_MEDS.csv # Plot results -br -plot Algorithm_Dataset/*_MEDS.csv MEDS +br -useGui 0 -plot Algorithm_Dataset/*_MEDS.csv MEDS diff --git a/scripts/evalGenderClassification-PCSO.sh b/scripts/evalGenderClassification-PCSO.sh index 881214e..9373645 100755 --- a/scripts/evalGenderClassification-PCSO.sh +++ b/scripts/evalGenderClassification-PCSO.sh @@ -4,8 +4,13 @@ if [ ! -f evalGenderClassification-PCSO.sh ]; then exit fi +export BR=../build/app/br/br +export genderAlg=GenderClassification + +export PCSO_DIR=/user/pripshare/Databases/FaceDatabases/PCSO/PCSO/ + # Create a file list by querying the database -br -quiet -algorithm Identity -enroll "../data/PCSO/PCSO.db[query='SELECT File,Gender,PersonID FROM PCSO', subset=1:8000]" terminal.txt > Input.txt +$BR -useGui 0 -quiet -algorithm Identity -enroll "$PCSO_DIR/PCSO.db[query='SELECT File,Gender,PersonID FROM PCSO', subset=1:8000]" terminal.txt > Input.txt # Enroll the file list and evaluate performance -br -algorithm GenderClassification -path ../data/PCSO/img -enroll Input.txt Output.txt -evalClassification Output.txt Input.txt +$BR -useGui 0 -algorithm $genderAlg -path $PCSO_DIR/Images -enroll Input.txt Output.txt -evalClassification Output.txt Input.txt Gender \ No newline at end of file diff --git a/scripts/trainAgeRegression-PCSO.sh b/scripts/trainAgeRegression-PCSO.sh index c063055..c1f3ef1 100755 --- a/scripts/trainAgeRegression-PCSO.sh +++ b/scripts/trainAgeRegression-PCSO.sh @@ -6,6 +6,11 @@ fi #rm -f ../share/openbr/models/features/FaceClassificationRegistration #rm -f ../share/openbr/models/features/FaceClassificationExtraction -rm -f ../share/openbr/models/algorithms/AgeRegression +#rm -f ../share/openbr/models/algorithms/AgeRegression -br -algorithm AgeRegression -path ../data/PCSO/Images -train "../data/PCSO/PCSO.db[query='SELECT File,Age,PersonID FROM PCSO WHERE Age >= 15 AND AGE <= 75', subset=0:200]" ../share/openbr/models/algorithms/AgeRegression +export BR=../build/app/br/br +export ageAlg=AgeRegression + +export PCSO_DIR=/user/pripshare/Databases/FaceDatabases/PCSO/PCSO/ + +$BR -useGui 0 -algorithm $ageAlg -path $PCSO_DIR/Images -train "$PCSO_DIR/PCSO.db[query='SELECT File,Age,PersonID FROM PCSO WHERE Age >= 17 AND AGE <= 68', subset=0:200]" ../share/openbr/models/algorithms/AgeRegression diff --git a/scripts/trainFaceRecognition-PCSO.sh b/scripts/trainFaceRecognition-PCSO.sh index e67b3aa..ebeb056 100755 --- a/scripts/trainFaceRecognition-PCSO.sh +++ b/scripts/trainFaceRecognition-PCSO.sh @@ -8,6 +8,13 @@ fi #rm -f ../share/openbr/models/features/FaceRecognitionExtraction #rm -f ../share/openbr/models/features/FaceRecognitionEmbedding #rm -f ../share/openbr/models/features/FaceRecognitionQuantization -rm -f ../share/openbr/models/algorithms/FaceRecognition +#rm -f ../share/openbr/models/algorithms/FaceRecognition + +export BR=../build/app/br/br + +export PCSO_DIR=/user/pripshare/Databases/FaceDatabases/PCSO/PCSO/ + + + +$BR -useGui 0 -algorithm FaceRecognition -path "$PCSO_DIR/Images/" -train "$PCSO_DIR/PCSO.db[query='SELECT File,PersonID as Label,PersonID FROM PCSO', subset=0:5:6000]" ../share/openbr/models/algorithms/FaceRecognition -br -algorithm FaceRecognition -path ../data/PCSO/img -train "../data/PCSO/PCSO.db[query='SELECT File,'S'||PersonID,PersonID FROM PCSO', subset=0:5:6000]" ../share/openbr/models/algorithms/FaceRecognition diff --git a/scripts/trainGenderClassification-PCSO.sh b/scripts/trainGenderClassification-PCSO.sh index 91fae04..acb8146 100755 --- a/scripts/trainGenderClassification-PCSO.sh +++ b/scripts/trainGenderClassification-PCSO.sh @@ -6,6 +6,11 @@ fi #rm -f ../share/openbr/models/features/FaceClassificationRegistration #rm -f ../share/openbr/models/features/FaceClassificationExtraction -rm -f ../share/openbr/models/algorithms/GenderClassification +#rm -f ../share/openbr/models/algorithms/GenderClassification -br -algorithm GenderClassification -path ../data/PCSO/Images -train "../data/PCSO/PCSO.db[query='SELECT File,Gender,PersonID FROM PCSO', subset=0:8000]" ../share/openbr/models/algorithms/GenderClassification +export BR=../build/app/br/br +export genderAlg=GenderClassification + +export PCSO_DIR=/user/pripshare/Databases/FaceDatabases/PCSO/PCSO/ + +$BR -useGui 0 -algorithm $genderAlg -path $PCSO_DIR/Images -train "$PCSO_DIR/PCSO.db[query='SELECT File,Gender,PersonID FROM PCSO', subset=0:8000]" ../share/openbr/models/algorithms/GenderClassification diff --git a/share/openbr/openbr.bib b/share/openbr/openbr.bib index b07abd7..111cd1b 100644 --- a/share/openbr/openbr.bib +++ b/share/openbr/openbr.bib @@ -34,6 +34,11 @@ Howpublished = {https://github.com/lbestrowden}, Title = {bestrow1 at msu.edu}} +@misc{imaus10, + Author = {Austin Van Blanton}, + Howpublished = {https://github.com/imaus10}, + Title = {imaus10 at gmail.com}} + % Software @misc{libface, Howpublished = {http://libface.sourceforge.net/file/Home.html},