diff --git a/.gitignore b/.gitignore index 8c7c3c0..102657e 100644 --- a/.gitignore +++ b/.gitignore @@ -41,4 +41,9 @@ scripts/results data/INRIAPerson/sigset data/KTH/sigset data/CaltechPedestrians/annotations +data/CaltechPedestrians/*.xml +### Sublime ### +*.check_cache +*.sublime-project +*.sublime-workspace diff --git a/README.md b/README.md index e5f8c42..95b43b3 100644 --- a/README.md +++ b/README.md @@ -14,4 +14,3 @@ To optionally check out a particular [tagged release](https://github.com/biometr [![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/biometrics/openbr/trend.png)](https://bitdeli.com/free "Bitdeli Badge") - diff --git a/app/CMakeLists.txt b/app/CMakeLists.txt index 7b198e4..075f48c 100644 --- a/app/CMakeLists.txt +++ b/app/CMakeLists.txt @@ -5,4 +5,6 @@ add_subdirectory(br) add_subdirectory(examples) # Build OpenBR GUI application -add_subdirectory(br-gui) +if(NOT ${BR_EMBEDDED}) + add_subdirectory(br-gui) +endif() diff --git a/app/br/br.cpp b/app/br/br.cpp index d7ca6be..e78c71d 100644 --- a/app/br/br.cpp +++ b/app/br/br.cpp @@ -158,15 +158,24 @@ public: } else if (!strcmp(fun, "plotMetadata")) { check(parc >= 2, "Incorrect parameter count for 'plotMetadata'."); br_plot_metadata(parc-1, parv, parv[parc-1], true); + } else if (!strcmp(fun, "deduplicate")) { + check(parc == 3, "Incorrect parameter count for 'deduplicate'."); + br_deduplicate(parv[0], parv[1], parv[2]); } // Miscellaneous else if (!strcmp(fun, "help")) { check(parc == 0, "No parameters expected for 'help'."); help(); + } else if (!strcmp(fun, "gui")) { + // Do nothing because we checked for this flag prior to initialization } else if (!strcmp(fun, "objects")) { check(parc <= 2, "Incorrect parameter count for 'objects'."); - printf("%s\n", br_objects(parc >= 1 ? parv[0] : ".*", parc >= 2 ? parv[1] : ".*")); + int size = br_objects(NULL, 0, parc >= 1 ? parv[0] : ".*", parc >= 2 ? parv[1] : ".*"); + char * temp = new char[size]; + br_objects(temp, size, parc >= 1 ? parv[0] : ".*", parc >= 2 ? parv[1] : ".*"); + printf("%s\n", temp); + delete [] temp; } else if (!strcmp(fun, "about")) { check(parc == 0, "No parameters expected for 'about'."); printf("%s\n", br_about()); @@ -177,11 +186,10 @@ public: check(parc == 1, "Incorrect parameter count for 'daemon'."); daemon = true; daemon_pipe = parv[0]; - } else if (!strcmp(fun,"slave")) { + } else if (!strcmp(fun, "slave")) { check(parc == 1, "Incorrect parameter count for 'slave'"); br_slave_process(parv[0]); - } - else if (!strcmp(fun, "exit")) { + } else if (!strcmp(fun, "exit")) { check(parc == 0, "No parameters expected for 'exit'."); daemon = false; } else if (!strcmp(fun, "getHeader")) { @@ -245,6 +253,7 @@ private: "\n" "==== Miscellaneous ====\n" "-help\n" + "-gui\n" "-objects [abstraction [implementation]]\n" "-about\n" "-version\n" @@ -255,7 +264,7 @@ private: int main(int argc, char *argv[]) { - br_initialize(argc, argv); + br_initialize(argc, argv, "", argc >= 2 && !strcmp(argv[1], "-gui")); FakeMain *fakeMain = new FakeMain(argc, argv); QThreadPool::globalInstance()->start(fakeMain); diff --git a/openbr/CMakeLists.txt b/openbr/CMakeLists.txt index b38cdfd..c4bf747 100644 --- a/openbr/CMakeLists.txt +++ b/openbr/CMakeLists.txt @@ -34,7 +34,7 @@ set(JANUS_BUILD_PP5_WRAPPER ${BR_WITH_PP5} CACHE BOOL "Build Janus implementatio set(JANUS_BUILD_DOCS ${BR_BUILD_DOCUMENTATION} CACHE BOOL "Build Janus HTML Doxygen documentation") mark_as_advanced(JANUS_BUILD_PP5_WRAPPER) mark_as_advanced(JANUS_BUILD_DOCS) -set(JANUS_TEST_IMPLEMENTATION openbr) +set(JANUS_IMPLEMENTATION openbr) add_subdirectory(janus) # Install diff --git a/openbr/core/bee.cpp b/openbr/core/bee.cpp index 940ddf2..a429f92 100644 --- a/openbr/core/bee.cpp +++ b/openbr/core/bee.cpp @@ -172,6 +172,8 @@ Mat BEE::readMat(const br::File &matrix, QString *targetSigset, QString *querySi qint64 read = file.read((char*)m.data, bytesExpected); if (read != bytesExpected) qFatal("Invalid matrix size."); + if (!file.atEnd()) + qFatal("Expected matrix end of file."); file.close(); Mat result; @@ -303,11 +305,14 @@ cv::Mat BEE::makeMask(const br::FileList &targets, const br::FileList &queries, QList targetPartitions = targets.crossValidationPartitions(); QList queryPartitions = queries.crossValidationPartitions(); + QList targetsOnly = File::get(queries, "targetOnly", false); + Mat mask(queries.size(), targets.size(), CV_8UC1); for (int i=0; i #include #include +#include #include "openbr/core/bee.h" #include "openbr/core/cluster.h" diff --git a/openbr/core/core.cpp b/openbr/core/core.cpp index 2d6e5c9..962d5b6 100644 --- a/openbr/core/core.cpp +++ b/openbr/core/core.cpp @@ -65,6 +65,9 @@ struct AlgorithmCore downcast->train(data); if (!distance.isNull()) { + if (Globals->crossValidate > 0) + for (int i=data.size()-1; i>=0; i--) if (data[i].file.get("allPartitions",false)) data.removeAt(i); + qDebug("Projecting Enrollment"); downcast->projectUpdate(data,data); @@ -251,6 +254,56 @@ struct AlgorithmCore Globals->blockSize = old_block_size; } + void deduplicate(const File &inputGallery, const File &outputGallery, const float threshold) + { + qDebug("Deduplicating %s to %s with a score threshold of %f", qPrintable(inputGallery.flat()), qPrintable(outputGallery.flat()), threshold); + + if (distance.isNull()) qFatal("Null distance."); + + QScopedPointer i; + FileList inputFiles; + retrieveOrEnroll(inputGallery, i, inputFiles); + + TemplateList t = i->read(); + + Output *o = Output::make(QString("buffer.tail[selfSimilar,threshold=%1,atLeast=0]").arg(QString::number(threshold)),inputFiles,inputFiles); + + // Compare to global tail output + distance->compare(t,t,o); + + delete o; + + QString buffer(Globals->buffer); + + QStringList tail = buffer.split("\n"); + + // Remove header + tail.removeFirst(); + + QStringList toRemove; + foreach(const QString &s, tail) + toRemove.append(s.split(',').at(1)); + + QSet duplicates = QSet::fromList(toRemove); + + QStringList fileNames = inputFiles.names(); + + QList indices; + foreach(const QString &d, duplicates) + indices.append(fileNames.indexOf(d)); + + std::sort(indices.begin(),indices.end(),std::greater()); + + qDebug("\n%d duplicates removed.", indices.size()); + + for (int i=0; i og(Gallery::make(outputGallery)); + + og->writeBlock(inputFiles); + } + void compare(File targetGallery, File queryGallery, File output) { qDebug("Comparing %s and %s%s", qPrintable(targetGallery.flat()), @@ -496,6 +549,14 @@ void br::Cat(const QStringList &inputGalleries, const QString &outputGallery) } } +void br::Deduplicate(const File &inputGallery, const File &outputGallery, const QString &threshold) +{ + bool ok; + float thresh = threshold.toFloat(&ok); + if (ok) AlgorithmManager::getAlgorithm(inputGallery.get("algorithm"))->deduplicate(inputGallery, outputGallery, thresh); + else qFatal("Unable to convert deduplication threshold to float."); +} + QSharedPointer br::Transform::fromAlgorithm(const QString &algorithm, bool preprocess) { if (!preprocess) diff --git a/openbr/core/opencvutils.cpp b/openbr/core/opencvutils.cpp index ebef768..ae59267 100644 --- a/openbr/core/opencvutils.cpp +++ b/openbr/core/opencvutils.cpp @@ -15,7 +15,9 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include +#include #include +#include #include #include "opencvutils.h" diff --git a/openbr/core/opencvutils.h b/openbr/core/opencvutils.h index 2d7cd68..35b16c9 100644 --- a/openbr/core/opencvutils.h +++ b/openbr/core/opencvutils.h @@ -22,6 +22,7 @@ #include #include #include +#include namespace OpenCVUtils { diff --git a/openbr/gui/algorithm.cpp b/openbr/gui/algorithm.cpp index 39fe742..dbce8dd 100644 --- a/openbr/gui/algorithm.cpp +++ b/openbr/gui/algorithm.cpp @@ -1,5 +1,6 @@ #include #include +#include #include "algorithm.h" @@ -18,7 +19,7 @@ bool br::Algorithm::addAlgorithm(const QString &algorithm, const QString &displa { static QStringList availableAlgorithms; if (availableAlgorithms.isEmpty()) - availableAlgorithms = QString(br_objects("Abbreviation", ".*", false)).split("\n"); + availableAlgorithms = br::Context::objects("Abbreviation", ".*", false); if (!availableAlgorithms.contains(algorithm)) return false; diff --git a/openbr/gui/gallerytoolbar.cpp b/openbr/gui/gallerytoolbar.cpp index 7b6dde4..673e60f 100644 --- a/openbr/gui/gallerytoolbar.cpp +++ b/openbr/gui/gallerytoolbar.cpp @@ -84,7 +84,7 @@ void br::GalleryToolBar::_enroll(const br::File &input) galleryLock.lock(); this->input = input; if (input.suffix() == "gal") gallery = input.name + ".mem"; - else gallery = QString("%1/galleries/%2.gal[cache]").arg(br_scratch_path(), qPrintable(input.baseName()+input.hash())); + else gallery = QString("%1/galleries/%2.gal[cache]").arg(br::Globals->scratchPath(), qPrintable(input.baseName()+input.hash())); files = br::Enroll(input.flat(), gallery.flat()); galleryLock.unlock(); } @@ -148,7 +148,7 @@ void br::GalleryToolBar::home() void br::GalleryToolBar::mean() { - const QString file = QString("%1/mean/%2.png").arg(br_scratch_path(), input.baseName()+input.hash()); + const QString file = QString("%1/mean/%2.png").arg(br::Globals->scratchPath(), input.baseName()+input.hash()); br_set_property("CENTER_TRAIN_B", qPrintable(file)); br::File trainingFile = input; br_train(qPrintable(trainingFile.flat()), "[algorithm=MedianFace]"); diff --git a/openbr/gui/progress.cpp b/openbr/gui/progress.cpp index c162b5d..482367b 100644 --- a/openbr/gui/progress.cpp +++ b/openbr/gui/progress.cpp @@ -1,4 +1,5 @@ #include +#include #include "progress.h" @@ -29,7 +30,7 @@ void br::Progress::checkProgress() const bool visible = progress >= 0 && progress < 100; if (visible) { - showMessage(br_most_recent_message()); + showMessage(Globals->mostRecentMessage); pbProgress.setValue(progress); if (progress > 100) pbProgress.setMaximum(0); else pbProgress.setMaximum(100); diff --git a/openbr/gui/tail.cpp b/openbr/gui/tail.cpp index 13afff2..e544dfe 100644 --- a/openbr/gui/tail.cpp +++ b/openbr/gui/tail.cpp @@ -1,6 +1,7 @@ #include #include "tail.h" +#include using namespace br; diff --git a/openbr/gui/templateviewer.cpp b/openbr/gui/templateviewer.cpp index a811cb2..56ab028 100644 --- a/openbr/gui/templateviewer.cpp +++ b/openbr/gui/templateviewer.cpp @@ -72,7 +72,7 @@ void TemplateViewer::refreshImage() if (file.isNull() || (format == "Photo")) { setImage(file, true); } else { - const QString path = QString(br_scratch_path()) + "/thumbnails"; + const QString path = QString(br::Globals->scratchPath()) + "/thumbnails"; const QString hash = file.hash()+format; const QString processedFile = path+"/"+file.baseName()+hash+".png"; if (!QFileInfo(processedFile).exists()) { diff --git a/openbr/gui/transformeditor.cpp b/openbr/gui/transformeditor.cpp index 4f2752a..c679e4e 100644 --- a/openbr/gui/transformeditor.cpp +++ b/openbr/gui/transformeditor.cpp @@ -24,7 +24,7 @@ using namespace br; br::TransformEditor::TransformEditor(Transform *transform, QWidget *parent) : QWidget(parent) { - name.addItems(QString(br_objects("Transform", ".*", false)).split('\n')); + name.addItems(br::Context::objects("Transform", ".*", false)); layout.addWidget(&name); setLayout(&layout); diff --git a/openbr/gui/utility.cpp b/openbr/gui/utility.cpp index 77d6d25..013ece0 100644 --- a/openbr/gui/utility.cpp +++ b/openbr/gui/utility.cpp @@ -1,6 +1,8 @@ #include #include +#include #include +#include #include "utility.h" using namespace cv; diff --git a/openbr/janus b/openbr/janus index 0eaa00f..f8d9c86 160000 --- a/openbr/janus +++ b/openbr/janus @@ -1 +1 @@ -Subproject commit 0eaa00f19256fee2f24033ff68b526ea6320624d +Subproject commit f8d9c869c821cc032d092ab1274da8f3cabf5eb7 diff --git a/openbr/janus.cpp b/openbr/janus.cpp index 3f6f75f..3a567fb 100644 --- a/openbr/janus.cpp +++ b/openbr/janus.cpp @@ -3,140 +3,138 @@ #endif #include "janus.h" +#include "janus_io.h" #include "openbr_plugin.h" -// Use the provided default implementation of some functions -#include "janus/src/janus.cpp" - using namespace br; static QSharedPointer transform; static QSharedPointer distance; +size_t janus_max_template_size() +{ + return 33554432; // 32 MB +} + janus_error janus_initialize(const char *sdk_path, const char *model_file) { int argc = 1; const char *argv[1] = { "janus" }; Context::initialize(argc, (char**)argv, sdk_path); QString algorithm = model_file; - if (algorithm.isEmpty()) algorithm = "Cvt(Gray)+Affine(88,88,0.25,0.35)+++:ByteL1"; - transform = Transform::fromAlgorithm(algorithm, false); - distance = Distance::fromAlgorithm(algorithm); + if (algorithm.isEmpty()) { + transform = Transform::fromAlgorithm("Cvt(Gray)+Affine(88,88,0.25,0.35)+++", false); + distance = Distance::fromAlgorithm("FaceRecognition"); + } else if (algorithm == "PP5") { + transform.reset(Transform::make("PP5Enroll", NULL)); + distance.reset(Distance::make("PP5Compare", NULL)); + } else { + transform = Transform::fromAlgorithm(algorithm, false); + distance = Distance::fromAlgorithm(algorithm); + } return JANUS_SUCCESS; } janus_error janus_finalize() { + transform.reset(); + distance.reset(); Context::finalize(); return JANUS_SUCCESS; } -struct janus_incomplete_template_type -{ - QList data; -}; +struct janus_template_type : public Template +{}; -janus_error janus_initialize_template(janus_incomplete_template *incomplete_template) +janus_error janus_initialize_template(janus_template *template_) { - *incomplete_template = new janus_incomplete_template_type(); + *template_ = new janus_template_type(); return JANUS_SUCCESS; } -janus_error janus_add_image(const janus_image image, const janus_attribute_list attributes, janus_incomplete_template incomplete_template) +janus_error janus_augment(const janus_image image, const janus_attribute_list attributes, janus_template template_) { Template t; t.append(cv::Mat(image.height, image.width, - image.color_space == JANUS_GRAY8 ? CV_8UC1 : CV_8UC1, + image.color_space == JANUS_GRAY8 ? CV_8UC1 : CV_8UC3, image.data)); for (size_t i=0; i("JANUS_RIGHT_EYE_X"), t.file.get("JANUS_RIGHT_EYE_Y"))); - t.file.set("Affine_1", QPointF(t.file.get("JANUS_LEFT_EYE_X"), t.file.get("JANUS_LEFT_EYE_Y"))); + t.file.set("Affine_0", QPointF(t.file.get("RIGHT_EYE_X"), t.file.get("RIGHT_EYE_Y"))); + t.file.set("Affine_1", QPointF(t.file.get("LEFT_EYE_X"), t.file.get("LEFT_EYE_Y"))); Template u; transform->project(t, u); - incomplete_template->data.append(u); + template_->append(u); return JANUS_SUCCESS; } -janus_error janus_finalize_template(janus_incomplete_template incomplete_template, janus_template template_, size_t *bytes) +janus_error janus_finalize_template(janus_template template_, janus_flat_template flat_template, size_t *bytes) { - size_t templateBytes = 0; - size_t numTemplates = 0; - *bytes = sizeof(templateBytes) + sizeof(numTemplates); - janus_template pos = template_ + *bytes; - - foreach (const cv::Mat &m, incomplete_template->data) { + foreach (const cv::Mat &m, *template_) { assert(m.isContinuous()); - const size_t currentTemplateBytes = m.rows * m.cols * m.elemSize(); - if (templateBytes == 0) - templateBytes = currentTemplateBytes; - if (templateBytes != currentTemplateBytes) - return JANUS_UNKNOWN_ERROR; - if (*bytes + templateBytes > janus_max_template_size()) + const size_t templateBytes = m.rows * m.cols * m.elemSize(); + if (*bytes + sizeof(size_t) + templateBytes > janus_max_template_size()) break; - memcpy(pos, m.data, templateBytes); - *bytes += templateBytes; - pos = pos + templateBytes; - numTemplates++; + memcpy(flat_template, &templateBytes, sizeof(templateBytes)); + flat_template += sizeof(templateBytes); + memcpy(flat_template, m.data, templateBytes); + flat_template += templateBytes; + *bytes += sizeof(size_t) + templateBytes; } - *(reinterpret_cast(template_)+0) = templateBytes; - *(reinterpret_cast(template_)+1) = numTemplates; - delete incomplete_template; + delete template_; return JANUS_SUCCESS; } -janus_error janus_verify(const janus_template a, const size_t a_bytes, const janus_template b, const size_t b_bytes, double *similarity) +janus_error janus_verify(const janus_flat_template a, const size_t a_bytes, const janus_flat_template b, const size_t b_bytes, double *similarity) { - (void) a_bytes; - (void) b_bytes; - - size_t a_template_bytes, a_templates, b_template_bytes, b_templates; - a_template_bytes = *(reinterpret_cast(a)+0); - a_templates = *(reinterpret_cast(a)+1); - b_template_bytes = *(reinterpret_cast(b)+0); - b_templates = *(reinterpret_cast(b)+1); - if (a_template_bytes != b_template_bytes) - return JANUS_UNKNOWN_ERROR; + *similarity = 0; - float dist = 0; - for (size_t i=0; icompare(cv::Mat(1, a_template_bytes, CV_8UC1, a+2*sizeof(size_t)+i*a_template_bytes), - cv::Mat(1, b_template_bytes, CV_8UC1, b+2*sizeof(size_t)+i*b_template_bytes)); - *similarity = a_templates * b_templates / dist; - return JANUS_SUCCESS; -} + int comparisons = 0; + janus_flat_template a_template = a; + while (a_template < a + a_bytes) { + const size_t a_template_bytes = *reinterpret_cast(a_template); + a_template += sizeof(a_template_bytes); -struct janus_incomplete_gallery_type -{ - QList< QPair > templates; -}; + janus_flat_template b_template = b; + while (b_template < b + b_bytes) { + const size_t b_template_bytes = *reinterpret_cast(b_template); + b_template += sizeof(b_template_bytes); -janus_error janus_initialize_gallery(janus_incomplete_gallery *incomplete_gallery) -{ - *incomplete_gallery = new janus_incomplete_gallery_type(); - return JANUS_SUCCESS; -} + *similarity += distance->compare(cv::Mat(1, a_template_bytes, CV_8UC1, a_template), + cv::Mat(1, b_template_bytes, CV_8UC1, b_template)); + comparisons++; -janus_error janus_add_template(const janus_template template_, const size_t bytes, const janus_template_id template_id, janus_incomplete_gallery incomplete_gallery) -{ - (void) bytes; - incomplete_gallery->templates.append(QPair(template_, template_id)); + b_template += b_template_bytes; + } + + a_template += a_template_bytes; + } + + if (*similarity != *similarity) // True for NaN + return JANUS_UNKNOWN_ERROR; + + *similarity /= comparisons; return JANUS_SUCCESS; } -janus_error janus_finalize_gallery(janus_incomplete_gallery incomplete_gallery, const char *gallery_file) +janus_error janus_enroll(const janus_template template_, const janus_template_id template_id, janus_gallery gallery) { - (void) incomplete_gallery; - (void) gallery_file; + template_->file.set("TEMPLATE_ID", template_id); + QFile file(gallery); + if (!file.open(QFile::WriteOnly | QFile::Append)) + return JANUS_WRITE_ERROR; + QDataStream stream(&file); + stream << *template_; + file.close(); + delete template_; return JANUS_SUCCESS; } diff --git a/openbr/openbr.cpp b/openbr/openbr.cpp index 1addb9a..e1d67fe 100644 --- a/openbr/openbr.cpp +++ b/openbr/openbr.cpp @@ -24,12 +24,31 @@ #include "core/qtutils.h" #include "plugins/openbr_internal.h" #include +#include using namespace br; +static int partialCopy(const QString & string, char * buffer, int buffer_length) +{ + + QByteArray byteArray = string.toLocal8Bit(); + + int copyLength = std::min(buffer_length-1, byteArray.size()); + if (copyLength < 0) + return byteArray.size() + 1; + + memcpy(buffer, byteArray.data(), copyLength); + buffer[copyLength] = '\0'; + + return byteArray.size() + 1; +} + const char *br_about() { + static QMutex aboutLock; + QMutexLocker lock(&aboutLock); static QByteArray about = Context::about().toLocal8Bit(); + return about.data(); } @@ -121,9 +140,9 @@ void br_fuse(int num_input_simmats, const char *input_simmats[], Fuse(QtUtils::toStringList(num_input_simmats, input_simmats), normalization, fusion, output_simmat); } -void br_initialize(int &argc, char *argv[], const char *sdk_path) +void br_initialize(int &argc, char *argv[], const char *sdk_path, bool use_gui) { - Context::initialize(argc, argv, sdk_path); + Context::initialize(argc, argv, sdk_path, use_gui); } void br_initialize_default() @@ -149,53 +168,14 @@ void br_make_pairwise_mask(const char *target_input, const char *query_input, co BEE::makePairwiseMask(target_input, query_input, mask); } -const char *br_most_recent_message() +int br_most_recent_message(char * buffer, int buffer_length) { - static QByteArray byteArray; - byteArray = Globals->mostRecentMessage.toLocal8Bit(); - return byteArray.data(); + return partialCopy(Globals->mostRecentMessage, buffer, buffer_length); } -const char *br_objects(const char *abstractions, const char *implementations, bool parameters) +int br_objects(char * buffer, int buffer_length, const char *abstractions, const char *implementations, bool parameters) { - static QByteArray objects; - - QStringList objectList; - QRegExp abstractionsRegExp(abstractions); - QRegExp implementationsRegExp(implementations); - - if (abstractionsRegExp.exactMatch("Abbreviation")) - foreach (const QString &name, Globals->abbreviations.keys()) - if (implementationsRegExp.exactMatch(name)) - objectList.append(name + (parameters ? "\t" + Globals->abbreviations[name] : "")); - - if (abstractionsRegExp.exactMatch("Distance")) - foreach (const QString &name, Factory::names()) - if (implementationsRegExp.exactMatch(name)) - objectList.append(name + (parameters ? "\t" + Factory::parameters(name) : "")); - - if (abstractionsRegExp.exactMatch("Format")) - foreach (const QString &name, Factory::names()) - if (implementationsRegExp.exactMatch(name)) - objectList.append(name + (parameters ? "\t" + Factory::parameters(name) : "")); - - if (abstractionsRegExp.exactMatch("Initializer")) - foreach (const QString &name, Factory::names()) - if (implementationsRegExp.exactMatch(name)) - objectList.append(name + (parameters ? "\t" + Factory::parameters(name) : "")); - - if (abstractionsRegExp.exactMatch("Output")) - foreach (const QString &name, Factory::names()) - if (implementationsRegExp.exactMatch(name)) - objectList.append(name + (parameters ? "\t" + Factory::parameters(name) : "")); - - if (abstractionsRegExp.exactMatch("Transform")) - foreach (const QString &name, Factory::names()) - if (implementationsRegExp.exactMatch(name)) - objectList.append(name + (parameters ? "\t" + Factory::parameters(name) : "")); - - objects = objectList.join("\n").toLocal8Bit(); - return objects.data(); + return partialCopy(br::Context::objects(abstractions, implementations, parameters).join('\n'), buffer, buffer_length); } bool br_plot(int num_files, const char *files[], const char *destination, bool show) @@ -250,15 +230,15 @@ void br_read_pipe(const char *pipe, int *argc, char ***argv) *argv = rawCharArrayList.data(); } -const char *br_scratch_path() +int br_scratch_path(char * buffer, int buffer_length) { - static QByteArray byteArray; - byteArray = Context::scratchPath().toLocal8Bit(); - return byteArray.data(); + return partialCopy(Context::scratchPath(), buffer, buffer_length); } const char *br_sdk_path() { + static QMutex sdkLock; + QMutexLocker lock(&sdkLock); static QByteArray sdkPath = QDir(Globals->sdkPath).absolutePath().toLocal8Bit(); return sdkPath.data(); } @@ -302,6 +282,8 @@ void br_train_n(int num_inputs, const char *inputs[], const char *model) const char *br_version() { + static QMutex versionLock; + QMutexLocker lock(&versionLock); static QByteArray version = Context::version().toLocal8Bit(); return version.data(); } @@ -379,10 +361,9 @@ bool br_img_is_empty(br_template tmpl) return t->m().empty(); } -const char* br_get_filename(br_template tmpl) +int br_get_filename(char * buffer, int buffer_length, br_template tmpl) { - Template *t = reinterpret_cast(tmpl); - return t->file.name.toStdString().c_str(); + return partialCopy(reinterpret_cast(tmpl)->file.name, buffer, buffer_length); } void br_set_filename(br_template tmpl, const char *filename) @@ -391,15 +372,11 @@ void br_set_filename(br_template tmpl, const char *filename) t->file.name = filename; } -const char* br_get_metadata_string(br_template tmpl, const char *key) +int br_get_metadata_string(char * buffer, int buffer_length, br_template tmpl, const char *key) { Template *t = reinterpret_cast(tmpl); - // need an object outside of this scope - // so the char pointer is valid - static QByteArray result; QVariant qvar = t->file.value(key); - result = QtUtils::toString(qvar).toUtf8(); - return result.data(); + return partialCopy(QtUtils::toString(qvar), buffer, buffer_length); } br_template_list br_enroll_template(br_template tmpl) @@ -470,3 +447,8 @@ void br_close_gallery(br_gallery gallery) Gallery *gal = reinterpret_cast(gallery); delete gal; } + +void br_deduplicate(const char *input_gallery, const char *output_gallery, const char *threshold) +{ + br::Deduplicate(input_gallery, output_gallery, threshold); +} diff --git a/openbr/openbr.h b/openbr/openbr.h index 4fc9a7c..f2cce16 100644 --- a/openbr/openbr.h +++ b/openbr/openbr.h @@ -41,6 +41,10 @@ extern "C" { * \section managed_return_value Managed Return Value * Memory for const char* return values is managed internally and guaranteed until the next call to the function. * + * \section input_string_buffer Input String Buffer + * Users should input a char * buffer and the size of that buffer. String data will be copied into the buffer, if the buffer is too + * small, only part of the string will be copied. Returns the buffer size required to contain the complete string. + * * \section examples Examples * - \ref c_face_recognition_evaluation * @@ -56,7 +60,6 @@ extern "C" { /*! * \brief Wraps br::Context::about() - * \note \ref managed_return_value * \see br_version */ BR_EXPORT const char *br_about(); @@ -67,6 +70,17 @@ BR_EXPORT const char *br_about(); BR_EXPORT void br_cat(int num_input_galleries, const char *input_galleries[], const char *output_gallery); /*! + * \brief Removes duplicate templates in a gallery. + * \param input_gallery Gallery to be deduplicated. + * \param output_gallery Deduplicated gallery. + * \param threshold Comparisons with a match score >= this value are designated to be duplicates. + * \note If a gallery contains n duplicates, the first n-1 duplicates in the gallery will be removed and the nth will be kept. + * \note Users are encouraged to use binary gallery formats as the entire gallery is read into memory in one call to Gallery::read. + */ + +BR_EXPORT void br_deduplicate(const char *input_gallery, const char *output_gallery, const char *threshold); + +/*! * \brief Clusters one or more similarity matrices into a list of subjects. * * A similarity matrix is a type of br::Output. The current clustering algorithm is a simplified implementation of \cite zhu11. @@ -213,7 +227,7 @@ BR_EXPORT void br_fuse(int num_input_simmats, const char *input_simmats[], * \brief Wraps br::Context::initialize() * \see br_finalize */ -BR_EXPORT void br_initialize(int &argc, char *argv[], const char *sdk_path = ""); +BR_EXPORT void br_initialize(int &argc, char *argv[], const char *sdk_path = "", bool use_gui = false); /*! * \brief Wraps br::Context::initialize() with default arguments. * \see br_finalize @@ -245,10 +259,10 @@ BR_EXPORT void br_make_pairwise_mask(const char *target_input, const char *query /*! * \brief Returns the most recent line sent to stderr. - * \note \ref managed_return_value + * \note \ref input_string_buffer * \see br_progress br_time_remaining */ -BR_EXPORT const char *br_most_recent_message(); +BR_EXPORT int br_most_recent_message(char * buffer, int buffer_length); /*! * \brief Returns names and parameters for the requested objects. @@ -257,10 +271,10 @@ BR_EXPORT const char *br_most_recent_message(); * \param abstractions Regular expression of the abstractions to search. * \param implementations Regular expression of the implementations to search. * \param parameters Include parameters after object name. - * \note \ref managed_return_value + * \note \ref input_string_buffer * \note This function uses Qt's QRegExp syntax. */ -BR_EXPORT const char *br_objects(const char *abstractions = ".*", const char *implementations = ".*", bool parameters = true); +BR_EXPORT int br_objects(char * buffer, int buffer_length, const char *abstractions = ".*", const char *implementations = ".*", bool parameters = true); /*! * \brief Renders recognition performance figures for a set of .csv files created by \ref br_eval. @@ -365,14 +379,14 @@ BR_EXPORT void br_read_pipe(const char *pipe, int *argc, char ***argv); /*! * \brief Wraps br::Context::scratchPath() - * \note \ref managed_return_value + * \note \ref input_string_buffer * \see br_version */ -BR_EXPORT const char *br_scratch_path(); +BR_EXPORT int br_scratch_path(char * buffer, int buffer_length); + /*! * \brief Returns the full path to the root of the SDK. - * \note \ref managed_return_value * \see br_initialize */ BR_EXPORT const char *br_sdk_path(); @@ -425,7 +439,6 @@ BR_EXPORT void br_train_n(int num_inputs, const char *inputs[], const char *mode /*! * \brief Wraps br::Context::version() - * \note \ref managed_return_value * \see br_about br_scratch_path */ BR_EXPORT const char *br_version(); @@ -497,16 +510,18 @@ BR_EXPORT int br_img_channels(br_template tmpl); BR_EXPORT bool br_img_is_empty(br_template tmpl); /*! * \brief Get the filename for a br::Template + * \note \ref input_string_buffer */ -BR_EXPORT const char* br_get_filename(br_template tmpl); +BR_EXPORT int br_get_filename(char * buffer, int buffer_length, br_template tmpl); /*! * \brief Set the filename for a br::Template. */ BR_EXPORT void br_set_filename(br_template tmpl, const char *filename); /*! * \brief Get metadata as a string for the given key in the given template. + * \note \ref input_string_buffer */ -BR_EXPORT const char* br_get_metadata_string(br_template, const char *key); +BR_EXPORT int br_get_metadata_string(char * buffer, int buffer_length, br_template tmpl, const char *key); /*! * \brief Enroll a br::Template from the C API! Returns a pointer to a br::TemplateList * \param tmpl Pointer to a br::Template. diff --git a/openbr/openbr_plugin.cpp b/openbr/openbr_plugin.cpp index 80f0559..1740bde 100644 --- a/openbr/openbr_plugin.cpp +++ b/openbr/openbr_plugin.cpp @@ -203,6 +203,13 @@ QList File::namedRects() const const QVariant &variant = m_metadata[key]; if (variant.canConvert()) rects.append(variant.value()); + else if(variant.canConvert >()) { + QList list = variant.value >(); + for (int i=0;i < list.size();i++) + { + rects.append(list[i]); + } + } } return rects; } @@ -392,7 +399,12 @@ TemplateList TemplateList::fromGallery(const br::File &gallery) const int crossValidate = gallery.get("crossValidate"); - if (gallery.getBool("leaveOneImageOut")) { + // The leaveOneImageOut flag is used when we want to train on n-1 of a subject's images + // Thus, we find all the images for a particular subject, and set their partitions based on + // the crossValidate parameter + // Note that when the number of images per subject varies from subject to subject + // the number of subjects will decrease as the partition increases + if (gallery.getBool("leaveOneImageOut") && crossValidate > 0) { QStringList labels; for (int i=newTemplates.size()-1; i>=0; i--) { newTemplates[i].file.set("Index", i+templates.size()); @@ -414,7 +426,7 @@ TemplateList TemplateList::fromGallery(const br::File &gallery) Template leaveOneImageOutTemplate = newTemplates[labelIndices[j]]; if (k!=leaveOneImageOutTemplate.file.get("Partition")) { leaveOneImageOutTemplate.file.set("Partition", k); - leaveOneImageOutTemplate.file.set("testOnly", true); + leaveOneImageOutTemplate.file.set("targetOnly", true); newTemplates.insert(i+1,leaveOneImageOutTemplate); } } @@ -899,8 +911,12 @@ void br::Context::initialize(int &argc, char *argv[], QString sdkPath, bool useG { qInstallMessageHandler(messageHandler); + QString sep; #ifndef _WIN32 useGui = useGui && (getenv("DISPLAY") != NULL); + sep = ":"; +#else + sep = ";"; #endif // not _WIN32 // We take in argc as a reference due to: @@ -944,6 +960,7 @@ void br::Context::initialize(int &argc, char *argv[], QString sdkPath, bool useG // Search for SDK if (sdkPath.isEmpty()) { QStringList checkPaths; checkPaths << QDir::currentPath() << QCoreApplication::applicationDirPath(); + checkPaths << QString(getenv("PATH")).split(sep, QString::SkipEmptyParts); bool foundSDK = false; foreach (const QString &path, checkPaths) { @@ -999,6 +1016,47 @@ QString br::Context::scratchPath() return QString("%1/%2-%3.%4").arg(QDir::homePath(), PRODUCT_NAME, QString::number(PRODUCT_VERSION_MAJOR), QString::number(PRODUCT_VERSION_MINOR)); } + +QStringList br::Context::objects(const char *abstractions, const char *implementations, bool parameters) +{ + QStringList objectList; + QRegExp abstractionsRegExp(abstractions); + QRegExp implementationsRegExp(implementations); + + if (abstractionsRegExp.exactMatch("Abbreviation")) + foreach (const QString &name, Globals->abbreviations.keys()) + if (implementationsRegExp.exactMatch(name)) + objectList.append(name + (parameters ? "\t" + Globals->abbreviations[name] : "")); + + if (abstractionsRegExp.exactMatch("Distance")) + foreach (const QString &name, Factory::names()) + if (implementationsRegExp.exactMatch(name)) + objectList.append(name + (parameters ? "\t" + Factory::parameters(name) : "")); + + if (abstractionsRegExp.exactMatch("Format")) + foreach (const QString &name, Factory::names()) + if (implementationsRegExp.exactMatch(name)) + objectList.append(name + (parameters ? "\t" + Factory::parameters(name) : "")); + + if (abstractionsRegExp.exactMatch("Initializer")) + foreach (const QString &name, Factory::names()) + if (implementationsRegExp.exactMatch(name)) + objectList.append(name + (parameters ? "\t" + Factory::parameters(name) : "")); + + if (abstractionsRegExp.exactMatch("Output")) + foreach (const QString &name, Factory::names()) + if (implementationsRegExp.exactMatch(name)) + objectList.append(name + (parameters ? "\t" + Factory::parameters(name) : "")); + + if (abstractionsRegExp.exactMatch("Transform")) + foreach (const QString &name, Factory::names()) + if (implementationsRegExp.exactMatch(name)) + objectList.append(name + (parameters ? "\t" + Factory::parameters(name) : "")); + + + return objectList; +} + void br::Context::messageHandler(QtMsgType type, const QMessageLogContext &context, const QString &msg) { // Something about this method is not thread safe, and will lead to crashes if qDebug diff --git a/openbr/openbr_plugin.h b/openbr/openbr_plugin.h index d50a183..3f84d7e 100644 --- a/openbr/openbr_plugin.h +++ b/openbr/openbr_plugin.h @@ -41,6 +41,7 @@ #include #include #include +#include /*! * \defgroup cpp_plugin_sdk C++ Plugin SDK @@ -829,6 +830,18 @@ public: */ static QString scratchPath(); + /*! + * \brief Returns names and parameters for the requested objects. + * + * Each object is \c \\n seperated. Arguments are seperated from the object name with a \c \\t. + * \param abstractions Regular expression of the abstractions to search. + * \param implementations Regular expression of the implementations to search. + * \param parameters Include parameters after object name. + * \note \ref managed_return_value + * \note This function uses Qt's QRegExp syntax. + */ + static QStringList objects(const char *abstractions = ".*", const char *implementations = ".*", bool parameters = true); + private: static void messageHandler(QtMsgType type, const QMessageLogContext &context, const QString &msg); }; @@ -1371,6 +1384,14 @@ BR_EXPORT void Convert(const File &fileType, const File &inputFile, const File & */ BR_EXPORT void Cat(const QStringList &inputGalleries, const QString &outputGallery); +/*! + * \brief Deduplicate a gallery. + * \param inputGallery Gallery to deduplicate. + * \param outputGallery Gallery to store the deduplicated result. + * \param threshold Match score threshold to determine duplicates. + */ +BR_EXPORT void Deduplicate(const File &inputGallery, const File &outputGallery, const QString &threshold); + /*! @}*/ } // namespace br diff --git a/openbr/plugins/algorithms.cpp b/openbr/plugins/algorithms.cpp index f6900c4..06e3275 100644 --- a/openbr/plugins/algorithms.cpp +++ b/openbr/plugins/algorithms.cpp @@ -50,6 +50,9 @@ class AlgorithmsInitializer : public Initializer Globals->abbreviations.insert("DisplayVideo", "Stream(FPSLimit(30)+Show(false,[FrameNumber])+Discard)"); Globals->abbreviations.insert("PerFrameDetection", "Stream(SaveMat(original)+Cvt(Gray)+Cascade(FrontalFace)+ASEFEyes+RestoreMat(original)+Draw(inPlace=true)+Show(false,[FrameNumber])+Discard)"); Globals->abbreviations.insert("AgeGenderDemo", "Stream(SaveMat(original)+Cvt(Gray)+Cascade(FrontalFace)+Expand+++/+Discard+RestoreMat(original)+Draw(inPlace=true)+DrawPropertiesPoint([Age,Gender],Affine_0,inPlace=true)+SaveMat(original)+Discard+Contract+RestoreMat(original)+FPSCalc+Show(false,[AvgFPS,Age,Gender])+Discard)"); + Globals->abbreviations.insert("ShowOpticalFlowField", "Stream(SaveMat(original)+AggregateFrames(2)+OpticalFlow(useMagnitude=false)+Grid(100,100)+DrawOpticalFlow+FPSLimit(30)+Show(false)+Discard)"); + Globals->abbreviations.insert("ShowOpticalFlowMagnitude", "Stream(AggregateFrames(2)+OpticalFlow+Normalize(Range,false,0,255)+Cvt(Color)+Draw+FPSLimit(30)+Show(false)+Discard)"); + Globals->abbreviations.insert("ShowMotionSegmentation", "Stream(DropFrames(5)+AggregateFrames(2)+OpticalFlow+CvtUChar+WatershedSegmentation+DrawSegmentation+Draw+FPSLimit(30)+Show(false)+Discard)"); Globals->abbreviations.insert("HOG", "Stream(DropFrames(5)+Cvt(Gray)+Grid(5,5)+ROIFromPts(32,24)+Expand+Resize(32,32)+Gradient+RectRegions+Bin(0,360,8)+Hist(8)+Cat)+Contract+CatRows+KMeans(500)+Hist(500)+SVM"); Globals->abbreviations.insert("HOF", "Stream(DropFrames(5)+Grid(5,5)+AggregateFrames(2)+OpticalFlow+ROIFromPts(32,24)+Expand+Resize(32,32)+Gradient+RectRegions+Bin(0,360,8)+Hist(8)+Cat)+Contract+CatRows+KMeans(500)+Hist(500)"); @@ -76,7 +79,7 @@ class AlgorithmsInitializer : public Initializer Globals->abbreviations.insert("ColoredLBP", "Open+Affine(128,128,0.37,0.45)+Cvt(Gray)+Blur(1.1)+Gamma(0.2)+DoG(1,2)+ContrastEq(0.1,10)+LBP(1,2)+ColoredU2"); // Transforms - Globals->abbreviations.insert("FaceDetection", "(Open+Cvt(Gray)+Cascade(FrontalFace))"); + Globals->abbreviations.insert("FaceDetection", "Open+Cvt(Gray)+Cascade(FrontalFace)"); Globals->abbreviations.insert("DenseLBP", "(Blur(1.1)+Gamma(0.2)+DoG(1,2)+ContrastEq(0.1,10)+LBP(1,2)+RectRegions(8,8,6,6)+Hist(59))"); Globals->abbreviations.insert("DenseSIFT", "(Grid(10,10)+SIFTDescriptor(12)+ByRow)"); Globals->abbreviations.insert("FaceRecognitionRegistration", "(ASEFEyes+Affine(88,88,0.25,0.35)+DownsampleTraining(FTE(DFFS),instances=1))"); diff --git a/openbr/plugins/cascade.cpp b/openbr/plugins/cascade.cpp index 2abb3c6..cc928b5 100644 --- a/openbr/plugins/cascade.cpp +++ b/openbr/plugins/cascade.cpp @@ -15,6 +15,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include +//#include #include "openbr_internal.h" #include "openbr/core/opencvutils.h" #include "openbr/core/resource.h" @@ -86,11 +87,11 @@ class CascadeTransform : public UntrainableMetaTransform for (int i=0; i rects; - vector rejectLevels; - vector levelWeights; - if (ROCMode) cascade->detectMultiScale(m, rects, rejectLevels, levelWeights, 1.2, 5, (enrollAll ? 0 : CV_HAAR_FIND_BIGGEST_OBJECT) | CV_HAAR_SCALE_IMAGE, Size(minSize, minSize), Size(), true); - else cascade->detectMultiScale(m, rects, 1.2, 5, enrollAll ? 0 : CV_HAAR_FIND_BIGGEST_OBJECT, Size(minSize, minSize)); + std::vector rects; + std::vector rejectLevels; + std::vector levelWeights; + if (ROCMode) cascade->detectMultiScale(m, rects, rejectLevels, levelWeights, 1.2, 5, (enrollAll ? 0 : CASCADE_FIND_BIGGEST_OBJECT) | CASCADE_SCALE_IMAGE, Size(minSize, minSize), Size(), true); + else cascade->detectMultiScale(m, rects, 1.2, 5, enrollAll ? 0 : CASCADE_FIND_BIGGEST_OBJECT, Size(minSize, minSize)); if (!enrollAll && rects.empty()) rects.push_back(Rect(0, 0, m.cols, m.rows)); diff --git a/openbr/plugins/cvt.cpp b/openbr/plugins/cvt.cpp index a0b12e2..fc8d9fb 100644 --- a/openbr/plugins/cvt.cpp +++ b/openbr/plugins/cvt.cpp @@ -14,6 +14,7 @@ * limitations under the License. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +#include #include #include "openbr_internal.h" #include "openbr/core/opencvutils.h" @@ -44,7 +45,8 @@ public: Luv = CV_BGR2Luv, RGB = CV_BGR2RGB, XYZ = CV_BGR2XYZ, - YCrCb = CV_BGR2YCrCb }; + YCrCb = CV_BGR2YCrCb, + Color = CV_GRAY2BGR }; private: BR_PROPERTY(ColorSpace, colorSpace, Gray) @@ -52,8 +54,8 @@ private: void project(const Template &src, Template &dst) const { - if (src.m().channels() > 1) cvtColor(src, dst, colorSpace); - else dst = src; + if (src.m().channels() > 1 || colorSpace == CV_GRAY2BGR) cvtColor(src, dst, colorSpace); + else dst = src; if (channel != -1) { std::vector mv; diff --git a/openbr/plugins/distance.cpp b/openbr/plugins/distance.cpp index b2106c8..b3c9d70 100644 --- a/openbr/plugins/distance.cpp +++ b/openbr/plugins/distance.cpp @@ -18,10 +18,12 @@ #include #include #include +#include #include "openbr_internal.h" #include "openbr/core/distance_sse.h" #include "openbr/core/qtutils.h" +#include "openbr/core/opencvutils.h" using namespace cv; @@ -61,6 +63,7 @@ private: (a.m().type() != b.m().type())) return -std::numeric_limits::max(); +// TODO: this max value is never returned based on the switch / default float result = std::numeric_limits::max(); switch (metric) { case Correlation: @@ -386,5 +389,103 @@ class OnlineDistance : public Distance BR_REGISTER(Distance, OnlineDistance) +/*! + * \ingroup distances + * \brief Attenuation function based distance from attributes + * \author Scott Klum \cite sklum + */ +class AttributeDistance : public Distance +{ + Q_OBJECT + Q_PROPERTY(QString attribute READ get_attribute WRITE set_attribute RESET reset_attribute STORED false) + BR_PROPERTY(QString, attribute, QString()) + + float compare(const Template &target, const Template &query) const + { + float queryValue = query.file.get(attribute); + float targetValue = target.file.get(attribute); + + // TODO: Set this magic number to something meaningful + float stddev = 1; + + if (queryValue == targetValue) return 1; + else return 1/(stddev*sqrt(2*CV_PI))*exp(-0.5*pow((targetValue-queryValue)/stddev, 2)); + } +}; + +BR_REGISTER(Distance, AttributeDistance) + +/*! + * \ingroup distances + * \brief Sum match scores across multiple distances + * \author Scott Klum \cite sklum + */ +class SumDistance : public Distance +{ + Q_OBJECT + Q_PROPERTY(QList distances READ get_distances WRITE set_distances RESET reset_distances) + BR_PROPERTY(QList, distances, QList()) + + void train(const TemplateList &data) + { + QFutureSynchronizer futures; + foreach (br::Distance *distance, distances) + futures.addFuture(QtConcurrent::run(distance, &Distance::train, data)); + futures.waitForFinished(); + } + + float compare(const Template &target, const Template &query) const + { + float result = 0; + + foreach (br::Distance *distance, distances) { + result += distance->compare(target, query); + + if (result == -std::numeric_limits::max()) + return result; + } + + return result; + } +}; + +BR_REGISTER(Distance, SumDistance) + +/*! + * \ingroup transforms + * \brief Compare each template to a fixed gallery (with name = galleryName), using the specified distance. + * dst will contain a 1 by n vector of scores. + * \author Charles Otto \cite caotto + */ +class GalleryCompareTransform : public Transform +{ + Q_OBJECT + Q_PROPERTY(br::Distance *distance READ get_distance WRITE set_distance RESET reset_distance STORED false) + Q_PROPERTY(QString galleryName READ get_galleryName WRITE set_galleryName RESET reset_galleryName STORED false) + BR_PROPERTY(br::Distance*, distance, NULL) + BR_PROPERTY(QString, galleryName, "") + + TemplateList gallery; + + void project(const Template &src, Template &dst) const + { + dst = src; + if (gallery.isEmpty()) + return; + + QList line = distance->compare(gallery, src); + dst.m() = OpenCVUtils::toMat(line, 1); + } + + void init() + { + if (!galleryName.isEmpty()) + gallery = TemplateList::fromGallery(galleryName); + } +}; + +BR_REGISTER(Transform, GalleryCompareTransform) + + } // namespace br #include "distance.moc" diff --git a/openbr/plugins/draw.cpp b/openbr/plugins/draw.cpp index 0a63e8c..00f9a8e 100644 --- a/openbr/plugins/draw.cpp +++ b/openbr/plugins/draw.cpp @@ -15,6 +15,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include +#include #include #include #include "openbr_internal.h" @@ -345,6 +346,75 @@ class AdjacentOverlayTransform : public Transform BR_REGISTER(Transform, AdjacentOverlayTransform) +/*! + * \ingroup transforms + * \brief Draw a line representing the direction and magnitude of optical flow at the specified points. + * \author Austin Blanton \cite imaus10 + */ +class DrawOpticalFlow : public UntrainableTransform +{ + Q_OBJECT + Q_PROPERTY(QString original READ get_original WRITE set_original RESET reset_original STORED false) + BR_PROPERTY(QString, original, "original") + + void project(const Template &src, Template &dst) const + { + const Scalar color(0,255,0); + Mat flow = src.m(); + dst = src; + if (!dst.file.contains(original)) qFatal("The original img must be saved in the metadata with SaveMat."); + dst.m() = dst.file.get(original); + dst.file.remove(original); + foreach (const Point2f &pt, OpenCVUtils::toPoints(dst.file.points())) { + Point2f dxy = flow.at(pt.y, pt.x); + Point2f newPt(pt.x+dxy.x, pt.y+dxy.y); + line(dst, pt, newPt, color); + } + } +}; +BR_REGISTER(Transform, DrawOpticalFlow) + +/*! + * \ingroup transforms + * \brief Fill in the segmentations or draw a line between intersecting segments. + * \author Austin Blanton \cite imaus10 + */ +class DrawSegmentation : public UntrainableTransform +{ + Q_OBJECT + Q_PROPERTY(bool fillSegment READ get_fillSegment WRITE set_fillSegment RESET reset_fillSegment STORED false) + BR_PROPERTY(bool, fillSegment, true) + + void project(const Template &src, Template &dst) const + { + if (!src.file.contains("SegmentsMask") || !src.file.contains("NumSegments")) qFatal("Must supply a Contours object in the metadata to drawContours."); + Mat segments = src.file.get("SegmentsMask"); + int numSegments = src.file.get("NumSegments"); + + dst.file = src.file; + Mat drawn = fillSegment ? Mat(segments.size(), CV_8UC3, Scalar::all(0)) : src.m(); + + for (int i=1; i > contours; + Scalar color(0,255,0); + findContours(mask, contours, CV_RETR_LIST, CV_CHAIN_APPROX_NONE); + drawContours(drawn, contours, -1, color); + } + } + + dst.m() = drawn; + } +}; +BR_REGISTER(Transform, DrawSegmentation) + // TODO: re-implement EditTransform using Qt #if 0 /*! diff --git a/openbr/plugins/eyes.cpp b/openbr/plugins/eyes.cpp index cdef8df..5d3929d 100644 --- a/openbr/plugins/eyes.cpp +++ b/openbr/plugins/eyes.cpp @@ -34,6 +34,7 @@ */ #include +#include #include "openbr_internal.h" #include "openbr/core/opencvutils.h" diff --git a/openbr/plugins/format.cpp b/openbr/plugins/format.cpp index aad33e8..14d2b97 100644 --- a/openbr/plugins/format.cpp +++ b/openbr/plugins/format.cpp @@ -16,10 +16,12 @@ #include #include +#include #ifndef BR_EMBEDDED #include #endif // BR_EMBEDDED #include +#include #include "openbr_internal.h" #include "openbr/core/bee.h" @@ -33,7 +35,7 @@ namespace br /*! * \ingroup formats - * \brief Read all frames of a video using OpenCV + * \brief Read all frames of a video using OpenCV * \author Charles Otto \cite caotto */ class videoFormat : public Format @@ -45,11 +47,11 @@ public: { if (!file.exists() ) return Template(); - + VideoCapture videoSource(file.name.toStdString()); videoSource.open(file.name.toStdString() ); - - + + Template frames; if (!videoSource.isOpened()) { qWarning("video file open failed"); @@ -71,7 +73,7 @@ public: void write(const Template &t) const { - int fourcc = OpenCVUtils::getFourcc(); + int fourcc = OpenCVUtils::getFourcc(); VideoWriter videoSink(file.name.toStdString(), fourcc, 30, t.begin()->size()); // Did we successfully open the output file? @@ -719,7 +721,7 @@ BR_REGISTER(Format, xmlFormat) /*! * \ingroup formats * \brief Reads in scores or ground truth from a text table. - * \author Josh Klontz + * \author Josh Klontz \cite jklontz * * Example of the format: * \code @@ -768,6 +770,191 @@ class scoresFormat : public Format BR_REGISTER(Format, scoresFormat) +/*! + * \ingroup formats + * \brief Reads FBI EBTS transactions. + * \author Scott Klum \cite sklum + * https://www.fbibiospecs.org/ebts.html + */ +class ebtsFormat : public Format +{ + Q_OBJECT + + struct Field { + int type; + QList data; + }; + + struct Record { + int type; + quint32 bytes; + int position; // Starting position of record + + QHash > fields; + }; + + quint32 recordBytes(const QByteArray &byteArray, const float recordType, int from) const + { + bool ok; + quint32 size; + + if (recordType == 4 || recordType == 7) { + // read first four bytes + ok = true; + size = qFromBigEndian((const uchar*)byteArray.mid(from,4).constData()); + } else { + int index = byteArray.indexOf(QChar(0x1D), from); + size = byteArray.mid(from, index-from).split(':').last().toInt(&ok); + } + + return ok ? size : -1; + } + + void parseRecord(const QByteArray &byteArray, Record &record) const + { + if (record.type == 4 || record.type == 7) { + // Just a binary blob + // Read everything after the first four bytes + // Not current supported + } else { + // Continue reading fields until we get all the data + unsigned int position = record.position; + while (position < record.position + record.bytes) { + int index = byteArray.indexOf(QChar(0x1D), position); + Field field = parseField(byteArray.mid(position, index-position),QChar(0x1F)); + if (field.type == 999 ) { + // Data begin after the field identifier and the colon + int dataBegin = byteArray.indexOf(':', position)+1; + field.data.clear(); + field.data.append(byteArray.mid(dataBegin, record.bytes-(dataBegin-record.position))); + + // Data fields are always last in the record + record.fields.insert(field.type,field.data); + break; + } + // Advance the position accounting for the separator + position += index-position+1; + record.fields.insert(field.type,field.data); + } + } + } + + Field parseField(const QByteArray &byteArray, const QChar &sep) const + { + bool ok; + Field f; + + QList data = byteArray.split(':'); + + f.type = data.first().split('.').last().toInt(&ok); + f.data = data.last().split(sep.toLatin1()); + + return f; + } + + Template read() const + { + QByteArray byteArray; + QtUtils::readFile(file, byteArray); + + Template t; + + Mat m; + + QList records; + + // Read the type one record (every EBTS file will have one of these) + Record r1; + r1.type = 1; + r1.position = 0; + r1.bytes = recordBytes(byteArray,r1.type,r1.position); + + // The fields in a type 1 record are strictly defined + QList data = byteArray.mid(r1.position,r1.bytes).split(QChar(0x1D).toLatin1()); + foreach (const QByteArray &datum, data) { + Field f = parseField(datum,QChar(0x1F)); + r1.fields.insert(f.type,f.data); + } + + records.append(r1); + + // Read the type two record (every EBTS file will have one of these) + Record r2; + r2.type = 2; + r2.position = r1.bytes; + r2.bytes = recordBytes(byteArray,r2.type,r2.position); + + // The fields in a type 2 record are strictly defined + data = byteArray.mid(r2.position,r2.bytes).split(QChar(0x1D).toLatin1()); + foreach (const QByteArray &datum, data) { + Field f = parseField(datum,QChar(0x1F)); + r2.fields.insert(f.type,f.data); + } + + // Demographics + if (r2.fields.contains(18)) { + QString name = r2.fields.value(18).first(); + QStringList names = name.split(','); + t.file.set("FIRSTNAME", names.at(1)); + t.file.set("LASTNAME", names.at(0)); + } + + if (r2.fields.contains(22)) t.file.set("DOB", r2.fields.value(22).first().toInt()); + if (r2.fields.contains(24)) t.file.set("GENDER", QString(r2.fields.value(24).first())); + if (r2.fields.contains(25)) t.file.set("RACE", QString(r2.fields.value(25).first())); + + if (t.file.contains("DOB")) { + const QDate dob = QDate::fromString(t.file.get("DOB"), "yyyyMMdd"); + const QDate current = QDate::currentDate(); + int age = current.year() - dob.year(); + if (current.month() < dob.month()) age--; + t.file.set("Age", age); + } + + records.append(r2); + + // The third field of the first record contains informations about all the remaining records in the transaction + // We don't care about the first two and the final items + QList recordTypes = r1.fields.value(3); + for (int i=2; i frontalIdxs; + int position = r1.bytes + r2.bytes; + for (int i=2; ipath.isEmpty() ? "" : Globals->path + "/") + file.name; bool ok = matio.open(filename.toStdString(), "r"); if (!ok) qFatal("Couldn't open the vbb file"); diff --git a/openbr/plugins/gui.cpp b/openbr/plugins/gui.cpp index 6d82687..557ac9c 100644 --- a/openbr/plugins/gui.cpp +++ b/openbr/plugins/gui.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -12,6 +13,7 @@ #include #include +#include #include "openbr_internal.h" #include "openbr/gui/utility.h" @@ -181,6 +183,149 @@ public slots: } }; +class RectMarkingWindow : public DisplayWindow +{ +public: + RectMarkingWindow() : DisplayWindow() + { + drawingRect = false; + } + + bool drawingRect; + QVector rects; + QList rectLabels; + + QPointF rectOrigin; + QPointF currentEnd; + QRectF currentRect; + bool disableAccept; + + bool eventFilter(QObject *obj, QEvent *event) + { + if (disableAccept) + return QObject::eventFilter(obj, event); + + + if (event->type() == QEvent::MouseButtonPress || event->type() == QEvent::MouseMove) + { + event->accept(); + + QMouseEvent *mouseEvent = (QMouseEvent*)event; + + if (event->type() == QEvent::MouseButtonPress) + { + + if (mouseEvent->button() == Qt::LeftButton) { + if (!drawingRect) + { + drawingRect = true; + rectOrigin = mouseEvent->pos(); + return true; + } + else + { + drawingRect = false; + + rects.append(QRectF(rectOrigin, mouseEvent->pos())); + rects.last() = rects.last().normalized(); + // If no labels were provided, we store everything as anonymous rectangles + if (promptKeys.empty()) + rectLabels.append("rects"); + // otherwise, prompt the user to select a label + else + { + // get a label from the user + bool ok = false; + + // Don't intercept events while the sub-dialog is up (if we take the events, then it will not work correctly) + disableAccept = true; + QString res = QInputDialog::getItem(this, "Select a label", "", promptKeys, next_idx, false, &ok); + + disableAccept = false; + if (ok) { + rectLabels.append(res); + for (int i=0; i < promptKeys.size(); i++) + { + if (res == promptKeys[i]) { + next_idx = (i + 1) % promptKeys.size(); + break; + } + } + } + else { + rects.remove(rects.size()-1); + } + } + } + } + // rclick -- reset state if drawing, remove last rect if done + else if (mouseEvent->button() == Qt::RightButton && (!rects.isEmpty() || drawingRect)) + { + if(drawingRect) + drawingRect = false; + else + { + rects.remove(rects.size()-1); + rectLabels.removeLast(); + } + } + } + else + currentEnd = mouseEvent->pos(); + QPixmap pixmapBuffer = pixmap; + + QPainter painter(&pixmapBuffer); + painter.setPen(Qt::red); + + painter.drawRects(rects); + + if (drawingRect) + { + currentRect = QRectF(rectOrigin, currentEnd); + painter.setPen(Qt::green); + painter.drawRect(currentRect); + } + + setPixmap(pixmapBuffer); + + return true; + } else { + if (event->type() == QEvent::KeyPress) + { + QKeyEvent * kevent = (QKeyEvent *) event; + if (kevent->key() == Qt::Key_Enter || kevent->key() == Qt::Key_Return) { + event->accept(); + return true; + } + } + return DisplayWindow::eventFilter(obj, event); + } + } + + + QList waitForKey() + { + + rects.clear(); + drawingRect = false; + disableAccept = false; + next_idx = 0; + DisplayWindow::waitForKey(); + + return QList(); + } + + void setKeys(const QStringList & keys) + { + promptKeys = keys; + } + + int next_idx; +private: + QStringList promptKeys; + +}; + class PointMarkingWindow : public DisplayWindow { bool eventFilter(QObject *obj, QEvent *event) @@ -555,6 +700,91 @@ BR_REGISTER(Transform, ManualTransform) /*! * \ingroup transforms + * \brief Manual select rectangular regions on an image. + * Stores marked rectangles as anonymous rectangles, or if a set of labels is provided, prompt the user + * to select one of those labels after drawing each rectangle. + * \author Charles Otto \cite caotto + */ +class ManualRectsTransform : public ShowTransform +{ + Q_OBJECT + +public: + + Q_PROPERTY(QStringList labels READ get_labels WRITE set_labels RESET reset_labels STORED false) + BR_PROPERTY(QStringList, labels, QStringList()) + + void projectUpdate(const TemplateList &src, TemplateList &dst) + { + + dst = src; + + if (!Globals->useGui) + return; + if (src.empty()) + return; + + for (int i = 0; i < dst.size(); i++) { + foreach(const cv::Mat &m, dst[i]) { + qImageBuffer = toQImage(m); + displayBuffer->convertFromImage(qImageBuffer); + + emit updateImage(displayBuffer->copy(displayBuffer->rect())); + + // Blocking wait for a key-press + if (this->waitInput) { + window->waitForKey(); + QVector rectSet = trueWindow->rects; + QList labelSet= trueWindow->rectLabels; + + for (int idx = 0; idx < rectSet.size(); idx++) + { + if (dst[i].file.contains(labelSet[idx])) + { + QVariant currentProp = dst[i].file.value(labelSet[idx]); + QList currentPropList; + + if (currentProp.canConvert >() ) + { + currentPropList = currentProp.toList(); + } + else if (currentProp.canConvert()) + { + currentPropList.append(currentProp); + } + else + { + qFatal("Unknown type of property"); + } + + currentPropList.append(rectSet[idx]); + dst[i].file.set(labelSet[idx], QVariant::fromValue(currentPropList)); + } + else + { + dst[i].file.set(labelSet[idx], rectSet[idx]); + } + } + } + + } + } + } + RectMarkingWindow * trueWindow; + void init() + { + if (!Globals->useGui) + return; + initActual(); + trueWindow = dynamic_cast (this->window); + trueWindow->setKeys(this->keys); + } +}; + +BR_REGISTER(Transform, ManualRectsTransform) + +/*! + * \ingroup transforms * \brief Elicits metadata for templates in a pretty GUI * \author Scott Klum \cite sklum */ diff --git a/openbr/plugins/hist.cpp b/openbr/plugins/hist.cpp index 6fc3bb9..d9f0a05 100644 --- a/openbr/plugins/hist.cpp +++ b/openbr/plugins/hist.cpp @@ -94,7 +94,7 @@ class BinTransform : public UntrainableTransform } else if (channels == 2) { // If there are two channels, the first is channel is assumed to be a weight vector // and the second channel contains the vectors we would like to bin. - vector mv; + std::vector mv; cv::split(src, mv); weights = mv[0]; weights.convertTo(weights, CV_32F); diff --git a/openbr/plugins/integral.cpp b/openbr/plugins/integral.cpp index bb42b4a..f6117a0 100644 --- a/openbr/plugins/integral.cpp +++ b/openbr/plugins/integral.cpp @@ -1,4 +1,5 @@ #include +#include #include #include "openbr_internal.h" @@ -293,7 +294,7 @@ private: Sobel(src, dx, CV_32F, 1, 0, CV_SCHARR); Sobel(src, dy, CV_32F, 0, 1, CV_SCHARR); cartToPolar(dx, dy, magnitude, angle, true); - vector mv; + std::vector mv; if ((channel == Magnitude) || (channel == MagnitudeAndAngle)) { const float theoreticalMaxMagnitude = sqrt(2*pow(float(2*(3+10+3)*255), 2.f)); mv.push_back(magnitude / theoreticalMaxMagnitude); diff --git a/openbr/plugins/landmarks.cpp b/openbr/plugins/landmarks.cpp index 1cb4445..c3b924b 100644 --- a/openbr/plugins/landmarks.cpp +++ b/openbr/plugins/landmarks.cpp @@ -290,16 +290,18 @@ class DrawDelaunayTransform : public UntrainableTransform void project(const Template &src, Template &dst) const { - QList validTriangles = OpenCVUtils::toPoints(src.file.getList("DelaunayTriangles")); + dst = src; - // Clone the matrix do draw on it - dst.m() = src.m().clone(); + if (src.file.contains("DelaunayTriangles")) { + QList validTriangles = OpenCVUtils::toPoints(src.file.getList("DelaunayTriangles")); - for (int i = 0; i < validTriangles.size(); i+=3) { - line(dst.m(), validTriangles[i], validTriangles[i+1], Scalar(0,0,0), 1); - line(dst.m(), validTriangles[i+1], validTriangles[i+2], Scalar(0,0,0), 1); - line(dst.m(), validTriangles[i+2], validTriangles[i], Scalar(0,0,0), 1); - } + // Clone the matrix do draw on it + for (int i = 0; i < validTriangles.size(); i+=3) { + line(dst, validTriangles[i], validTriangles[i+1], Scalar(0,0,0), 1); + line(dst, validTriangles[i+1], validTriangles[i+2], Scalar(0,0,0), 1); + line(dst, validTriangles[i+2], validTriangles[i], Scalar(0,0,0), 1); + } + } else qWarning("Template does not contain Delaunay triangulation."); } }; diff --git a/openbr/plugins/lbp.cpp b/openbr/plugins/lbp.cpp index 40e972e..b295286 100644 --- a/openbr/plugins/lbp.cpp +++ b/openbr/plugins/lbp.cpp @@ -15,7 +15,9 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include +#include #include +#include #include #include "openbr_internal.h" diff --git a/openbr/plugins/ltp.cpp b/openbr/plugins/ltp.cpp index 984987f..df34070 100644 --- a/openbr/plugins/ltp.cpp +++ b/openbr/plugins/ltp.cpp @@ -15,6 +15,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include +#include #include #include "openbr_internal.h" diff --git a/openbr/plugins/mask.cpp b/openbr/plugins/mask.cpp index 3019934..6898c70 100644 --- a/openbr/plugins/mask.cpp +++ b/openbr/plugins/mask.cpp @@ -15,6 +15,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include +#include #include "openbr_internal.h" using namespace cv; @@ -164,7 +165,7 @@ class LargestConvexAreaTransform : public UntrainableTransform void project(const Template &src, Template &dst) const { std::vector< std::vector > contours; - findContours(src.clone(), contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE); + findContours(src.m().clone(), contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE); double maxArea = 0; foreach (const std::vector &contour, contours) { std::vector hull; diff --git a/openbr/plugins/misc.cpp b/openbr/plugins/misc.cpp index 2aee23f..16c8c39 100644 --- a/openbr/plugins/misc.cpp +++ b/openbr/plugins/misc.cpp @@ -537,7 +537,7 @@ class ProgressCounterTransform : public TimeVaryingTransform { (void) data; float p = br_progress(); - qDebug("%05.2f%% ELAPSED=%s REMAINING=%s COUNT=%g/%g \r", p*100., QtUtils::toTime(Globals->startTime.elapsed()/1000.0f).toStdString().c_str(), QtUtils::toTime(0).toStdString().c_str(), Globals->currentStep, Globals->totalSteps); + qDebug("%05.2f%% ELAPSED=%s REMAINING=%s COUNT=%g/%g \r", p*100, QtUtils::toTime(Globals->startTime.elapsed()/1000.0f).toStdString().c_str(), QtUtils::toTime(0).toStdString().c_str(), Globals->currentStep, Globals->totalSteps); } void init() diff --git a/openbr/plugins/motion.cpp b/openbr/plugins/motion.cpp index d746527..4511336 100644 --- a/openbr/plugins/motion.cpp +++ b/openbr/plugins/motion.cpp @@ -23,6 +23,7 @@ class OpticalFlowTransform : public UntrainableMetaTransform Q_PROPERTY(int poly_n READ get_poly_n WRITE set_poly_n RESET reset_poly_n STORED false) Q_PROPERTY(double poly_sigma READ get_poly_sigma WRITE set_poly_sigma RESET reset_poly_sigma STORED false) Q_PROPERTY(int flags READ get_flags WRITE set_flags RESET reset_flags STORED false) + Q_PROPERTY(bool useMagnitude READ get_useMagnitude WRITE set_useMagnitude RESET reset_useMagnitude STORED false) // these defaults are optimized for KTH BR_PROPERTY(double, pyr_scale, 0.1) BR_PROPERTY(int, levels, 1) @@ -31,22 +32,27 @@ class OpticalFlowTransform : public UntrainableMetaTransform BR_PROPERTY(int, poly_n, 7) BR_PROPERTY(double, poly_sigma, 1.1) BR_PROPERTY(int, flags, 0) + BR_PROPERTY(bool, useMagnitude, true) void project(const Template &src, Template &dst) const { // get the two images put there by AggregateFrames if (src.size() != 2) qFatal("Optical Flow requires two images."); - Mat prevImg = src[0], nextImg = src[1], flow, flowOneCh; + Mat prevImg = src[0], nextImg = src[1], flow; if (src[0].channels() != 1) OpenCVUtils::cvtGray(src[0], prevImg); if (src[1].channels() != 1) OpenCVUtils::cvtGray(src[1], nextImg); calcOpticalFlowFarneback(prevImg, nextImg, flow, pyr_scale, levels, winsize, iterations, poly_n, poly_sigma, flags); - // the result is two channels - std::vector channels(2); - split(flow, channels); - magnitude(channels[0], channels[1], flowOneCh); - - dst += flowOneCh; + if (useMagnitude) { + // the result is two channels + Mat flowOneCh; + std::vector channels(2); + split(flow, channels); + magnitude(channels[0], channels[1], flowOneCh); + dst += flowOneCh; + } else { + dst += flow; + } dst.file = src.file; } }; @@ -62,7 +68,8 @@ class SubtractBackgroundTransform : public TimeVaryingTransform { Q_OBJECT - BackgroundSubtractorMOG2 mog; + // TODO: This is broken. + // BackgroundSubtractorMOG2 mog; public: SubtractBackgroundTransform() : TimeVaryingTransform(false, false) {} @@ -72,7 +79,8 @@ private: { dst = src; Mat mask; - mog(src, mask); + // TODO: broken + // mog(src, mask); erode(mask, mask, Mat()); dilate(mask, mask, Mat()); dst.file.set("Mask", QVariant::fromValue(mask)); @@ -86,7 +94,8 @@ private: void finalize(TemplateList &output) { (void) output; - mog = BackgroundSubtractorMOG2(); + // TODO: Broken + // mog = BackgroundSubtractorMOG2(); } }; diff --git a/openbr/plugins/normalize.cpp b/openbr/plugins/normalize.cpp index 842e952..449ca61 100644 --- a/openbr/plugins/normalize.cpp +++ b/openbr/plugins/normalize.cpp @@ -60,24 +60,29 @@ class NormalizeTransform : public UntrainableTransform Q_PROPERTY(bool ByRow READ get_ByRow WRITE set_ByRow RESET reset_ByRow STORED false) BR_PROPERTY(bool, ByRow, false) + Q_PROPERTY(int alpha READ get_alpha WRITE set_alpha RESET reset_alpha STORED false) + BR_PROPERTY(int, alpha, 1) + Q_PROPERTY(int beta READ get_beta WRITE set_beta RESET reset_beta STORED false) + BR_PROPERTY(int, beta, 0) public: /*!< */ enum NormType { Inf = NORM_INF, L1 = NORM_L1, - L2 = NORM_L2 }; + L2 = NORM_L2, + Range = NORM_MINMAX }; private: BR_PROPERTY(NormType, normType, L2) void project(const Template &src, Template &dst) const { - if (!ByRow) normalize(src, dst, 1, 0, normType, CV_32F); + if (!ByRow) normalize(src, dst, alpha, beta, normType, CV_32F); else { dst = src; for (int i=0; i labels = data.indexProperty(inputVariable); const int dims = m.cols; - vector mv, av, bv; + std::vector mv, av, bv; split(m, mv); for (size_t c = 0; c < mv.size(); c++) { av.push_back(Mat(1, dims, CV_64FC1)); diff --git a/openbr/plugins/output.cpp b/openbr/plugins/output.cpp index 1ad0995..b9a290a 100644 --- a/openbr/plugins/output.cpp +++ b/openbr/plugins/output.cpp @@ -268,7 +268,7 @@ class rrOutput : public MatrixOutput for (int i=0; i Pair; foreach (const Pair &pair, Common::Sort(OpenCVUtils::matrixToVector(data.row(i)), true, limit)) { @@ -276,7 +276,7 @@ class rrOutput : public MatrixOutput if (pair.first < threshold) break; File target = targetFiles[pair.second]; target.set("Score", QString::number(pair.first)); - if (simple) files.append(target.baseName() + " " + QString::number(pair.first)); + if (simple) files.append(target.fileName() + " " + QString::number(pair.first)); else files.append(target.flat()); } } @@ -528,7 +528,7 @@ class tailOutput : public Output } else { // General case for (int k=0; k atLeast) && (comparisons.last().value < threshold)) comparisons.removeLast(); + lastValue = comparisons.last().value; comparisonsLock.unlock(); } diff --git a/openbr/plugins/segmentation.cpp b/openbr/plugins/segmentation.cpp new file mode 100644 index 0000000..f11532b --- /dev/null +++ b/openbr/plugins/segmentation.cpp @@ -0,0 +1,53 @@ +#include +#include "openbr_internal.h" +#include "openbr/core/opencvutils.h" + +using namespace cv; + +namespace br +{ + +/*! + * \ingroup transforms + * \brief Applies watershed segmentation. + * \author Austin Blanton \cite imaus10 + */ +class WatershedSegmentationTransform : public UntrainableTransform +{ + Q_OBJECT + void project(const Template &src, Template &dst) const + { + dst = src; + + Mat mod; +// adaptiveThreshold(src.m(), src.m(), 255, ADAPTIVE_THRESH_GAUSSIAN_C, THRESH_BINARY, 33, 5); + threshold(src.m(), mod, 0, 255, THRESH_BINARY+THRESH_OTSU); + + // findContours requires an 8-bit 1-channel image + // and modifies its source image + if (mod.depth() != CV_8U) OpenCVUtils::cvtUChar(mod, mod); + if (mod.channels() != 1) OpenCVUtils::cvtGray(mod, mod); + vector > contours; + vector hierarchy; + findContours(mod, contours, hierarchy, CV_RETR_CCOMP, CV_CHAIN_APPROX_SIMPLE); + + // draw the contour delineations as 1,2,3... for input to watershed + Mat markers = Mat::zeros(mod.size(), CV_32S); + int compCount=0; + for (int idx=0; idx>=0; idx=hierarchy[idx][0], compCount++) { + drawContours(markers, contours, idx, Scalar::all(compCount+1), -1, 8, hierarchy, INT_MAX); + } + + Mat orig = src.m(); + // watershed requires a 3-channel 8-bit image + if (orig.channels() == 1) cvtColor(orig, orig, CV_GRAY2BGR); + watershed(orig, markers); + dst.file.set("SegmentsMask", QVariant::fromValue(markers)); + dst.file.set("NumSegments", compCount); + } +}; +BR_REGISTER(Transform, WatershedSegmentationTransform) + +} // namespace br + +#include "segmentation.moc" diff --git a/openbr/plugins/stasm4.cpp b/openbr/plugins/stasm4.cpp index c32bde3..6fa2db7 100644 --- a/openbr/plugins/stasm4.cpp +++ b/openbr/plugins/stasm4.cpp @@ -38,8 +38,10 @@ class StasmInitializer : public Initializer Globals->abbreviations.insert("RectFromStasmEyes","RectFromPoints([28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47],0.3,5.3)"); Globals->abbreviations.insert("RectFromStasmBrow","RectFromPoints([16,17,18,19,20,21,22,23,24,25,26,27],0.15,5)"); Globals->abbreviations.insert("RectFromStasmNose","RectFromPoints([48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58],0.15,1.15)"); + Globals->abbreviations.insert("RectFromStasmNoseWithBridge", "RectFromPoints([21, 22, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58],0.15,.6)"); Globals->abbreviations.insert("RectFromStasmMouth","RectFromPoints([59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76],0.3,2)"); Globals->abbreviations.insert("RectFromStasmHair", "RectFromPoints([13,14,15],1.75,1.5)"); + Globals->abbreviations.insert("RectFromStasmJaw", "RectFromPoints([2,3,4,5,6,7,8,9,10],.25,1.6)"); } }; diff --git a/openbr/plugins/stream.cpp b/openbr/plugins/stream.cpp index 229a418..749c44a 100644 --- a/openbr/plugins/stream.cpp +++ b/openbr/plugins/stream.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include "openbr_internal.h" #include "openbr/core/common.h" #include "openbr/core/opencvutils.h" @@ -346,10 +347,11 @@ public: } // first 4 bytes store 0xEDFE, next 24 store 'Norpix seq ' - char *firstFour = new char[4], *nextTwentyFour; + char firstFour[4]; seqFile.seekg(0, ios::beg); seqFile.read(firstFour, 4); - nextTwentyFour = readText(24); + char nextTwentyFour[24]; + readText(24, nextTwentyFour); if (firstFour[0] != (char)0xED || firstFour[1] != (char)0xFE || strncmp(nextTwentyFour, "Norpix seq", 10) != 0) { qDebug("Invalid header in seq file"); return false; @@ -362,7 +364,8 @@ public: qDebug("Invalid header size"); return false; } - char *desc = readText(512); + char desc[512]; + readText(512, desc); basis.file.set("Description", QString(desc)); width = readInt(); @@ -413,9 +416,9 @@ public: // but there might be 16 extra bytes instead of 8... if (i == 1) { seqFile.seekg(s, ios::beg); - char *zero = new char[1]; - seqFile.read(zero, 1); - if (zero[0] == 0) { + char zero; + seqFile.read(&zero, 1); + if (zero == 0) { s += 8; extra += 8; } @@ -430,10 +433,12 @@ public: } #ifdef CVMATIO - QString f = basis.file.name; - QString vbb = f.replace(f.lastIndexOf("."), 4, ".vbb"); - vbb.replace(vbb.lastIndexOf("vid"), 3, "annotations"); - annotations = TemplateList::fromGallery(File(vbb)); + if (basis.file.contains("vbb")) { + QString vbb = basis.file.get("vbb"); + annotations = TemplateList::fromGallery(File(vbb)); + } +#else + qWarning("cvmatio not installed, bounding boxes will not be available. Add -DBR_WITH_CVMATIO cmake flag to install."); #endif return true; @@ -497,14 +502,13 @@ private: // apparently the text in seq files is 16 bit characters (UTF-16?) // since we don't really need the last byte, snad since it gets interpreted as // a terminating char, let's just grab the first byte for storage - char* readText(int bytes) + void readText(int bytes, char * buffer) { - char *text = new char[bytes], *ret = new char[bytes/2]; - seqFile.read(text, bytes); + seqFile.read(buffer, bytes); for (int i=0; iopen(curr); if (!open_res) { @@ -1420,6 +1423,7 @@ public: { // Delete all the stages for (int i = 0; i < processingStages.size(); i++) { +// TODO: Are we releasing memory which is already freed? delete processingStages[i]; } processingStages.clear(); diff --git a/openbr/plugins/validate.cpp b/openbr/plugins/validate.cpp index d1f1dc0..75bca83 100644 --- a/openbr/plugins/validate.cpp +++ b/openbr/plugins/validate.cpp @@ -65,9 +65,9 @@ class CrossValidateTransform : public MetaTransform const QString label = partitionedData.at(j).file.get("Label"); QList subjectIndices = partitionedData.find("Label",label); QList removed; - // Remove test only data + // Remove target only data for (int k=subjectIndices.size()-1; k>=0; k--) - if (partitionedData[subjectIndices[k]].file.getBool("testOnly")) { + if (partitionedData[subjectIndices[k]].file.getBool("targetOnly")) { removed.append(subjectIndices[k]); subjectIndices.removeAt(k); } @@ -108,6 +108,7 @@ class CrossValidateTransform : public MetaTransform // If we want to duplicate templates but use the same training data // for all partitions (i.e. transforms.size() == 1), we need to // restrict the partition + int partition = src.file.get("Partition", 0); partition = (partition >= transforms.size()) ? 0 : partition; transforms[partition]->project(src, dst); diff --git a/scripts/brpy/__init__.py b/scripts/brpy/__init__.py index bd9a37d..4060fbc 100644 --- a/scripts/brpy/__init__.py +++ b/scripts/brpy/__init__.py @@ -12,6 +12,14 @@ def _var_string_args(n): s.extend(_string_args(n)) return s +def _handle_string_func(func): + def call_func(*args): + howlong = func('', 0, *args) + msg = 'x'*(howlong-1) + func(msg, howlong, *args) + return msg + return call_func + def init_brpy(br_loc='/usr/local/lib'): """Takes the ctypes lib object for br and initializes all function inputs and outputs""" br_loc += '/libopenbr.%s' @@ -48,9 +56,14 @@ def init_brpy(br_loc='/usr/local/lib'): br.br_is_classifier.restype = c_bool br.br_make_mask.argtypes = _string_args(3) br.br_make_pairwise_mask.argtypes = _string_args(3) - br.br_most_recent_message.restype = c_char_p - br.br_objects.argtypes = _string_args(2) + [c_bool] - br.br_objects.restype = c_char_p + br.br_most_recent_message.argtypes = [c_char_p, c_int] + br.br_most_recent_message.restype = c_int + func = br.br_most_recent_message.__call__ + br.br_most_recent_message = _handle_string_func(func) + br.br_objects.argtypes = [c_char_p, c_int] + _string_args(2) + [c_bool] + br.br_objects.restype = c_int + func2 = br.br_objects.__call__ + br.br_objects = _handle_string_func(func2) br.br_plot.argtypes = plot_args br.br_plot.restype = c_bool br.br_plot_detection.argtypes = plot_args @@ -61,7 +74,10 @@ def init_brpy(br_loc='/usr/local/lib'): br.br_plot_metadata.restype = c_bool br.br_progress.restype = c_float br.br_read_pipe.argtypes = [c_char_p, POINTER(c_int), POINTER(POINTER(c_char_p))] - br.br_scratch_path.restype = c_char_p + br.br_scratch_path.argtypes = [c_char_p, c_int] + br.br_scratch_path.restype = c_int + func3 = br.br_scratch_path.__call__ + br.br_scratch_path = _handle_string_func(func3) br.br_sdk_path.restype = c_char_p br.br_get_header.argtypes = [c_char_p, POINTER(c_char_p), POINTER(c_char_p)] br.br_set_header.argtypes = _string_args(3) @@ -88,11 +104,15 @@ def init_brpy(br_loc='/usr/local/lib'): br.br_img_channels.restype = c_int br.br_img_is_empty.argtypes = [c_void_p] br.br_img_is_empty.restype = c_bool - br.br_get_filename.argtypes = [c_void_p] - br.br_get_filename.restype = c_char_p + br.br_get_filename.argtypes = [c_char_p, c_int, c_void_p] + br.br_get_filename.restype = c_int + func4 = br.br_get_filename.__call__ + br.br_get_filename = _handle_string_func(func4) br.br_set_filename.argtypes = [c_void_p, c_char_p] - br.br_get_metadata_string.argtypes = [c_void_p, c_char_p] - br.br_get_metadata_string.restype = c_char_p + br.br_get_metadata_string.argtypes = [c_char_p, c_int, c_void_p, c_char_p] + br.br_get_metadata_string.restype = c_int + func5 = br.br_get_metadata_string.__call__ + br.br_get_metadata_string = _handle_string_func(func5) br.br_enroll_template.argtypes = [c_void_p] br.br_enroll_template.restype = c_void_p br.br_enroll_template_list.argtypes = [c_void_p] diff --git a/scripts/downloadDatasets.sh b/scripts/downloadDatasets.sh index 7011117..0cb08f6 100755 --- a/scripts/downloadDatasets.sh +++ b/scripts/downloadDatasets.sh @@ -51,6 +51,8 @@ if [ ! -d ../data/CaltechPedestrians/vid ]; then tar -xf $fname done rm *.tar + ./writeCaltechPedestrianSigset.sh 0 5 train > ../data/CaltechPedestrians/train.xml + ./writeCaltechPedestrianSigset.sh 6 10 test > ../data/CaltechPedestrians/test.xml mv set* ../data/CaltechPedestrians/vid if hash curl 2>/dev/null; then curl -OL "$prefix/annotations.zip" diff --git a/scripts/writeCaltechPedestrianSigset.sh b/scripts/writeCaltechPedestrianSigset.sh new file mode 100755 index 0000000..be8387e --- /dev/null +++ b/scripts/writeCaltechPedestrianSigset.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +echo '' +echo '' +for ((set=$1; set <= $2; set++)); do + echo -e "\t" + for vid in `printf "set%02d/*.seq" $set`; do + if [ $3 == "train" ]; then + vbb=`echo "vbb=\"annotations/$vid\"" | sed s/seq/vbb/` + fi + printf "\t\t\n" + done + echo -e "\t" +done +echo '' diff --git a/scripts/writeKTHSigset.sh b/scripts/writeKTHSigset.sh old mode 100644 new mode 100755 index 5c2abd0..5c2abd0 --- a/scripts/writeKTHSigset.sh +++ b/scripts/writeKTHSigset.sh diff --git a/share/openbr/cmake/Findcvmatio.cmake b/share/openbr/cmake/Findcvmatio.cmake index dc8f172..a4f68ae 100644 --- a/share/openbr/cmake/Findcvmatio.cmake +++ b/share/openbr/cmake/Findcvmatio.cmake @@ -1,7 +1,7 @@ set(CVMATIO_DIR "${BR_THIRDPARTY_DIR}/cvmatio") if(NOT EXISTS ${CVMATIO_DIR}) # download source from github - execute_process(COMMAND "git" "clone" "https://github.com/biometrics/cvmatio.git" WORKING_DIRECTORY ${BR_THIRDPARTY_DIR}) + execute_process(COMMAND "git" "clone" "https://github.com/hbristow/cvmatio.git" WORKING_DIRECTORY ${BR_THIRDPARTY_DIR}) else() # update the source execute_process(COMMAND "git" "pull" WORKING_DIRECTORY ${CVMATIO_DIR})