diff --git a/.gitignore b/.gitignore index 4116733..102657e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ ### Repository Specific ### 3rdparty/LatentSDK* 3rdparty/pittpatt* +3rdparty/cvmatio data/*/img data/*/vid data/PCSO/* @@ -39,4 +40,10 @@ scripts/results ### autogenerated sigsets ### data/INRIAPerson/sigset data/KTH/sigset +data/CaltechPedestrians/annotations +data/CaltechPedestrians/*.xml +### Sublime ### +*.check_cache +*.sublime-project +*.sublime-workspace diff --git a/CMakeLists.txt b/CMakeLists.txt index 039c678..d878599 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,6 +4,7 @@ cmake_minimum_required(VERSION 2.8.9) # Global settings set(BR_SHARE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/share/openbr") set(BR_SCRIPTS_DIR "${CMAKE_CURRENT_SOURCE_DIR}/scripts") +set(BR_THIRDPARTY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/3rdparty") set(CMAKE_AUTOMOC ON) set(CPACK_PACKAGE_NAME "OpenBR") set(CPACK_PACKAGE_VENDOR "OpenBiometrics") @@ -90,6 +91,18 @@ set(BR_THIRDPARTY_LIBS ${BR_THIRDPARTY_LIBS} ${OpenCV_LIBS}) # Find Alphanum find_package(Alphanum REQUIRED) +# Find cvmatio (if using it) +set(BR_WITH_CVMATIO OFF CACHE BOOL "Build with cvmatio library to read Matlab data files (required to use Caltech Pedestrians dataset)") +if(${BR_WITH_CVMATIO}) + find_package(cvmatio REQUIRED) + add_definitions(-DCVMATIO) + add_subdirectory(${CVMATIO_DIR}) + include_directories(${CVMATIO_DIR}/include) + set(BR_THIRDPARTY_SRC ${BR_THIRDPARTY_SRC} ${CVMATIO_DIR}/src/MatlabIO.cpp) + link_directories(${CVMATIO_LIB_DIR}) + set(BR_THIRDPARTY_LIBS ${BR_THIRDPARTY_LIBS} cvmatio) +endif() + # Compiler flags if(UNIX) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wextra -Wno-strict-overflow -fvisibility=hidden -fno-omit-frame-pointer") diff --git a/README.md b/README.md index e5f8c42..95b43b3 100644 --- a/README.md +++ b/README.md @@ -14,4 +14,3 @@ To optionally check out a particular [tagged release](https://github.com/biometr [![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/biometrics/openbr/trend.png)](https://bitdeli.com/free "Bitdeli Badge") - diff --git a/app/CMakeLists.txt b/app/CMakeLists.txt index 7b198e4..075f48c 100644 --- a/app/CMakeLists.txt +++ b/app/CMakeLists.txt @@ -5,4 +5,6 @@ add_subdirectory(br) add_subdirectory(examples) # Build OpenBR GUI application -add_subdirectory(br-gui) +if(NOT ${BR_EMBEDDED}) + add_subdirectory(br-gui) +endif() diff --git a/app/br/br.cpp b/app/br/br.cpp index d7ca6be..abeeab1 100644 --- a/app/br/br.cpp +++ b/app/br/br.cpp @@ -158,12 +158,17 @@ public: } else if (!strcmp(fun, "plotMetadata")) { check(parc >= 2, "Incorrect parameter count for 'plotMetadata'."); br_plot_metadata(parc-1, parv, parv[parc-1], true); + } else if (!strcmp(fun, "deduplicate")) { + check(parc == 3, "Incorrect parameter count for 'deduplicate'."); + br_deduplicate(parv[0], parv[1], parv[2]); } // Miscellaneous else if (!strcmp(fun, "help")) { check(parc == 0, "No parameters expected for 'help'."); help(); + } else if (!strcmp(fun, "gui")) { + // Do nothing because we checked for this flag prior to initialization } else if (!strcmp(fun, "objects")) { check(parc <= 2, "Incorrect parameter count for 'objects'."); printf("%s\n", br_objects(parc >= 1 ? parv[0] : ".*", parc >= 2 ? parv[1] : ".*")); @@ -177,11 +182,10 @@ public: check(parc == 1, "Incorrect parameter count for 'daemon'."); daemon = true; daemon_pipe = parv[0]; - } else if (!strcmp(fun,"slave")) { + } else if (!strcmp(fun, "slave")) { check(parc == 1, "Incorrect parameter count for 'slave'"); br_slave_process(parv[0]); - } - else if (!strcmp(fun, "exit")) { + } else if (!strcmp(fun, "exit")) { check(parc == 0, "No parameters expected for 'exit'."); daemon = false; } else if (!strcmp(fun, "getHeader")) { @@ -245,6 +249,7 @@ private: "\n" "==== Miscellaneous ====\n" "-help\n" + "-gui\n" "-objects [abstraction [implementation]]\n" "-about\n" "-version\n" @@ -255,7 +260,7 @@ private: int main(int argc, char *argv[]) { - br_initialize(argc, argv); + br_initialize(argc, argv, "", argc >= 2 && !strcmp(argv[1], "-gui")); FakeMain *fakeMain = new FakeMain(argc, argv); QThreadPool::globalInstance()->start(fakeMain); diff --git a/data/CaltechPedestrians/README.md b/data/CaltechPedestrians/README.md new file mode 100644 index 0000000..8074010 --- /dev/null +++ b/data/CaltechPedestrians/README.md @@ -0,0 +1,3 @@ +## Caltech Pedestrians Dataset +The Caltech Pedestrian Dataset consists of approximately 10 hours of 640x480 30Hz video taken from a vehicle driving through regular traffic in an urban environment. About 250,000 frames (in 137 approximately minute long segments) with a total of 350,000 bounding boxes and 2300 unique pedestrians were annotated. The annotation includes temporal correspondence between bounding boxes and detailed occlusion labels. +* [Website](http://www.vision.caltech.edu/Image_Datasets/CaltechPedestrians/) diff --git a/openbr/CMakeLists.txt b/openbr/CMakeLists.txt index b38cdfd..c4bf747 100644 --- a/openbr/CMakeLists.txt +++ b/openbr/CMakeLists.txt @@ -34,7 +34,7 @@ set(JANUS_BUILD_PP5_WRAPPER ${BR_WITH_PP5} CACHE BOOL "Build Janus implementatio set(JANUS_BUILD_DOCS ${BR_BUILD_DOCUMENTATION} CACHE BOOL "Build Janus HTML Doxygen documentation") mark_as_advanced(JANUS_BUILD_PP5_WRAPPER) mark_as_advanced(JANUS_BUILD_DOCS) -set(JANUS_TEST_IMPLEMENTATION openbr) +set(JANUS_IMPLEMENTATION openbr) add_subdirectory(janus) # Install diff --git a/openbr/core/bee.cpp b/openbr/core/bee.cpp index 940ddf2..db67479 100644 --- a/openbr/core/bee.cpp +++ b/openbr/core/bee.cpp @@ -172,6 +172,8 @@ Mat BEE::readMat(const br::File &matrix, QString *targetSigset, QString *querySi qint64 read = file.read((char*)m.data, bytesExpected); if (read != bytesExpected) qFatal("Invalid matrix size."); + if (!file.atEnd()) + qFatal("Expected matrix end of file."); file.close(); Mat result; diff --git a/openbr/core/cluster.cpp b/openbr/core/cluster.cpp index 9657a70..9b3beda 100644 --- a/openbr/core/cluster.cpp +++ b/openbr/core/cluster.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include "openbr/core/bee.h" #include "openbr/core/cluster.h" diff --git a/openbr/core/common.cpp b/openbr/core/common.cpp index 55cbd3e..bfd38fb 100644 --- a/openbr/core/common.cpp +++ b/openbr/core/common.cpp @@ -63,3 +63,16 @@ QList Common::RandSample(int n, const QSet &values, bool unique) } return samples; } + +QList Common::linspace(float start, float stop, int n) { + float delta = (stop - start) / (n - 1); + float curValue = start; + QList spaced; + spaced.reserve(n); + spaced.append(start); + for (int i = 1; i < (n - 1); i++) { + spaced.append(curValue += delta); + } + spaced.append(stop); + return spaced; +} diff --git a/openbr/core/common.h b/openbr/core/common.h index bd0b603..15516d0 100644 --- a/openbr/core/common.h +++ b/openbr/core/common.h @@ -253,6 +253,11 @@ QList RandSample(int n, const QList &weights, bool unique = false) } /*! + * \brief See Matlab function linspace() for documentation. + */ +QList linspace(float start, float stop, int n); + +/*! * \brief See Matlab function unique() for documentation. */ template diff --git a/openbr/core/core.cpp b/openbr/core/core.cpp index 2f041a6..962d5b6 100644 --- a/openbr/core/core.cpp +++ b/openbr/core/core.cpp @@ -254,6 +254,56 @@ struct AlgorithmCore Globals->blockSize = old_block_size; } + void deduplicate(const File &inputGallery, const File &outputGallery, const float threshold) + { + qDebug("Deduplicating %s to %s with a score threshold of %f", qPrintable(inputGallery.flat()), qPrintable(outputGallery.flat()), threshold); + + if (distance.isNull()) qFatal("Null distance."); + + QScopedPointer i; + FileList inputFiles; + retrieveOrEnroll(inputGallery, i, inputFiles); + + TemplateList t = i->read(); + + Output *o = Output::make(QString("buffer.tail[selfSimilar,threshold=%1,atLeast=0]").arg(QString::number(threshold)),inputFiles,inputFiles); + + // Compare to global tail output + distance->compare(t,t,o); + + delete o; + + QString buffer(Globals->buffer); + + QStringList tail = buffer.split("\n"); + + // Remove header + tail.removeFirst(); + + QStringList toRemove; + foreach(const QString &s, tail) + toRemove.append(s.split(',').at(1)); + + QSet duplicates = QSet::fromList(toRemove); + + QStringList fileNames = inputFiles.names(); + + QList indices; + foreach(const QString &d, duplicates) + indices.append(fileNames.indexOf(d)); + + std::sort(indices.begin(),indices.end(),std::greater()); + + qDebug("\n%d duplicates removed.", indices.size()); + + for (int i=0; i og(Gallery::make(outputGallery)); + + og->writeBlock(inputFiles); + } + void compare(File targetGallery, File queryGallery, File output) { qDebug("Comparing %s and %s%s", qPrintable(targetGallery.flat()), @@ -499,6 +549,14 @@ void br::Cat(const QStringList &inputGalleries, const QString &outputGallery) } } +void br::Deduplicate(const File &inputGallery, const File &outputGallery, const QString &threshold) +{ + bool ok; + float thresh = threshold.toFloat(&ok); + if (ok) AlgorithmManager::getAlgorithm(inputGallery.get("algorithm"))->deduplicate(inputGallery, outputGallery, thresh); + else qFatal("Unable to convert deduplication threshold to float."); +} + QSharedPointer br::Transform::fromAlgorithm(const QString &algorithm, bool preprocess) { if (!preprocess) diff --git a/openbr/core/eigenutils.cpp b/openbr/core/eigenutils.cpp index df21a86..c054155 100644 --- a/openbr/core/eigenutils.cpp +++ b/openbr/core/eigenutils.cpp @@ -54,3 +54,16 @@ void printEigen(Eigen::MatrixXd X) { qDebug() << str; } } +void printEigen(Eigen::MatrixXf X) { + for (int i = 0; i < X.rows(); i++) { + QString str; + for (int j = 0; j < X.cols(); j++) { + str.append(QString::number(X(i,j)) + " "); + } + qDebug() << str; + } +} + +void printSize(Eigen::MatrixXf X) { + qDebug() << "Rows=" << X.rows() << "\tCols=" << X.cols(); +} diff --git a/openbr/core/eigenutils.h b/openbr/core/eigenutils.h index 89e36c0..1ca8fa7 100644 --- a/openbr/core/eigenutils.h +++ b/openbr/core/eigenutils.h @@ -26,6 +26,8 @@ void writeEigen(Eigen::MatrixXd X, QString filename); void writeEigen(Eigen::VectorXd X, QString filename); void writeEigen(Eigen::VectorXf X, QString filename); void printEigen(Eigen::MatrixXd X); +void printEigen(Eigen::MatrixXf X); +void printSize(Eigen::MatrixXf X); template inline QDataStream &operator<<(QDataStream &stream, const Eigen::Matrix< _Scalar, _Rows, _Cols, _Options, _MaxRows, _MaxCols > &mat) diff --git a/openbr/core/opencvutils.cpp b/openbr/core/opencvutils.cpp index ebef768..ae59267 100644 --- a/openbr/core/opencvutils.cpp +++ b/openbr/core/opencvutils.cpp @@ -15,7 +15,9 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include +#include #include +#include #include #include "opencvutils.h" diff --git a/openbr/core/opencvutils.h b/openbr/core/opencvutils.h index 2d7cd68..35b16c9 100644 --- a/openbr/core/opencvutils.h +++ b/openbr/core/opencvutils.h @@ -22,6 +22,7 @@ #include #include #include +#include namespace OpenCVUtils { diff --git a/openbr/gui/tail.cpp b/openbr/gui/tail.cpp index 13afff2..e544dfe 100644 --- a/openbr/gui/tail.cpp +++ b/openbr/gui/tail.cpp @@ -1,6 +1,7 @@ #include #include "tail.h" +#include using namespace br; diff --git a/openbr/gui/utility.cpp b/openbr/gui/utility.cpp index 77d6d25..013ece0 100644 --- a/openbr/gui/utility.cpp +++ b/openbr/gui/utility.cpp @@ -1,6 +1,8 @@ #include #include +#include #include +#include #include "utility.h" using namespace cv; diff --git a/openbr/janus b/openbr/janus index 0eaa00f..f8d9c86 160000 --- a/openbr/janus +++ b/openbr/janus @@ -1 +1 @@ -Subproject commit 0eaa00f19256fee2f24033ff68b526ea6320624d +Subproject commit f8d9c869c821cc032d092ab1274da8f3cabf5eb7 diff --git a/openbr/janus.cpp b/openbr/janus.cpp index 3f6f75f..3a567fb 100644 --- a/openbr/janus.cpp +++ b/openbr/janus.cpp @@ -3,140 +3,138 @@ #endif #include "janus.h" +#include "janus_io.h" #include "openbr_plugin.h" -// Use the provided default implementation of some functions -#include "janus/src/janus.cpp" - using namespace br; static QSharedPointer transform; static QSharedPointer distance; +size_t janus_max_template_size() +{ + return 33554432; // 32 MB +} + janus_error janus_initialize(const char *sdk_path, const char *model_file) { int argc = 1; const char *argv[1] = { "janus" }; Context::initialize(argc, (char**)argv, sdk_path); QString algorithm = model_file; - if (algorithm.isEmpty()) algorithm = "Cvt(Gray)+Affine(88,88,0.25,0.35)+++:ByteL1"; - transform = Transform::fromAlgorithm(algorithm, false); - distance = Distance::fromAlgorithm(algorithm); + if (algorithm.isEmpty()) { + transform = Transform::fromAlgorithm("Cvt(Gray)+Affine(88,88,0.25,0.35)+++", false); + distance = Distance::fromAlgorithm("FaceRecognition"); + } else if (algorithm == "PP5") { + transform.reset(Transform::make("PP5Enroll", NULL)); + distance.reset(Distance::make("PP5Compare", NULL)); + } else { + transform = Transform::fromAlgorithm(algorithm, false); + distance = Distance::fromAlgorithm(algorithm); + } return JANUS_SUCCESS; } janus_error janus_finalize() { + transform.reset(); + distance.reset(); Context::finalize(); return JANUS_SUCCESS; } -struct janus_incomplete_template_type -{ - QList data; -}; +struct janus_template_type : public Template +{}; -janus_error janus_initialize_template(janus_incomplete_template *incomplete_template) +janus_error janus_initialize_template(janus_template *template_) { - *incomplete_template = new janus_incomplete_template_type(); + *template_ = new janus_template_type(); return JANUS_SUCCESS; } -janus_error janus_add_image(const janus_image image, const janus_attribute_list attributes, janus_incomplete_template incomplete_template) +janus_error janus_augment(const janus_image image, const janus_attribute_list attributes, janus_template template_) { Template t; t.append(cv::Mat(image.height, image.width, - image.color_space == JANUS_GRAY8 ? CV_8UC1 : CV_8UC1, + image.color_space == JANUS_GRAY8 ? CV_8UC1 : CV_8UC3, image.data)); for (size_t i=0; i("JANUS_RIGHT_EYE_X"), t.file.get("JANUS_RIGHT_EYE_Y"))); - t.file.set("Affine_1", QPointF(t.file.get("JANUS_LEFT_EYE_X"), t.file.get("JANUS_LEFT_EYE_Y"))); + t.file.set("Affine_0", QPointF(t.file.get("RIGHT_EYE_X"), t.file.get("RIGHT_EYE_Y"))); + t.file.set("Affine_1", QPointF(t.file.get("LEFT_EYE_X"), t.file.get("LEFT_EYE_Y"))); Template u; transform->project(t, u); - incomplete_template->data.append(u); + template_->append(u); return JANUS_SUCCESS; } -janus_error janus_finalize_template(janus_incomplete_template incomplete_template, janus_template template_, size_t *bytes) +janus_error janus_finalize_template(janus_template template_, janus_flat_template flat_template, size_t *bytes) { - size_t templateBytes = 0; - size_t numTemplates = 0; - *bytes = sizeof(templateBytes) + sizeof(numTemplates); - janus_template pos = template_ + *bytes; - - foreach (const cv::Mat &m, incomplete_template->data) { + foreach (const cv::Mat &m, *template_) { assert(m.isContinuous()); - const size_t currentTemplateBytes = m.rows * m.cols * m.elemSize(); - if (templateBytes == 0) - templateBytes = currentTemplateBytes; - if (templateBytes != currentTemplateBytes) - return JANUS_UNKNOWN_ERROR; - if (*bytes + templateBytes > janus_max_template_size()) + const size_t templateBytes = m.rows * m.cols * m.elemSize(); + if (*bytes + sizeof(size_t) + templateBytes > janus_max_template_size()) break; - memcpy(pos, m.data, templateBytes); - *bytes += templateBytes; - pos = pos + templateBytes; - numTemplates++; + memcpy(flat_template, &templateBytes, sizeof(templateBytes)); + flat_template += sizeof(templateBytes); + memcpy(flat_template, m.data, templateBytes); + flat_template += templateBytes; + *bytes += sizeof(size_t) + templateBytes; } - *(reinterpret_cast(template_)+0) = templateBytes; - *(reinterpret_cast(template_)+1) = numTemplates; - delete incomplete_template; + delete template_; return JANUS_SUCCESS; } -janus_error janus_verify(const janus_template a, const size_t a_bytes, const janus_template b, const size_t b_bytes, double *similarity) +janus_error janus_verify(const janus_flat_template a, const size_t a_bytes, const janus_flat_template b, const size_t b_bytes, double *similarity) { - (void) a_bytes; - (void) b_bytes; - - size_t a_template_bytes, a_templates, b_template_bytes, b_templates; - a_template_bytes = *(reinterpret_cast(a)+0); - a_templates = *(reinterpret_cast(a)+1); - b_template_bytes = *(reinterpret_cast(b)+0); - b_templates = *(reinterpret_cast(b)+1); - if (a_template_bytes != b_template_bytes) - return JANUS_UNKNOWN_ERROR; + *similarity = 0; - float dist = 0; - for (size_t i=0; icompare(cv::Mat(1, a_template_bytes, CV_8UC1, a+2*sizeof(size_t)+i*a_template_bytes), - cv::Mat(1, b_template_bytes, CV_8UC1, b+2*sizeof(size_t)+i*b_template_bytes)); - *similarity = a_templates * b_templates / dist; - return JANUS_SUCCESS; -} + int comparisons = 0; + janus_flat_template a_template = a; + while (a_template < a + a_bytes) { + const size_t a_template_bytes = *reinterpret_cast(a_template); + a_template += sizeof(a_template_bytes); -struct janus_incomplete_gallery_type -{ - QList< QPair > templates; -}; + janus_flat_template b_template = b; + while (b_template < b + b_bytes) { + const size_t b_template_bytes = *reinterpret_cast(b_template); + b_template += sizeof(b_template_bytes); -janus_error janus_initialize_gallery(janus_incomplete_gallery *incomplete_gallery) -{ - *incomplete_gallery = new janus_incomplete_gallery_type(); - return JANUS_SUCCESS; -} + *similarity += distance->compare(cv::Mat(1, a_template_bytes, CV_8UC1, a_template), + cv::Mat(1, b_template_bytes, CV_8UC1, b_template)); + comparisons++; -janus_error janus_add_template(const janus_template template_, const size_t bytes, const janus_template_id template_id, janus_incomplete_gallery incomplete_gallery) -{ - (void) bytes; - incomplete_gallery->templates.append(QPair(template_, template_id)); + b_template += b_template_bytes; + } + + a_template += a_template_bytes; + } + + if (*similarity != *similarity) // True for NaN + return JANUS_UNKNOWN_ERROR; + + *similarity /= comparisons; return JANUS_SUCCESS; } -janus_error janus_finalize_gallery(janus_incomplete_gallery incomplete_gallery, const char *gallery_file) +janus_error janus_enroll(const janus_template template_, const janus_template_id template_id, janus_gallery gallery) { - (void) incomplete_gallery; - (void) gallery_file; + template_->file.set("TEMPLATE_ID", template_id); + QFile file(gallery); + if (!file.open(QFile::WriteOnly | QFile::Append)) + return JANUS_WRITE_ERROR; + QDataStream stream(&file); + stream << *template_; + file.close(); + delete template_; return JANUS_SUCCESS; } diff --git a/openbr/openbr.cpp b/openbr/openbr.cpp index 1addb9a..5582f8d 100644 --- a/openbr/openbr.cpp +++ b/openbr/openbr.cpp @@ -24,6 +24,7 @@ #include "core/qtutils.h" #include "plugins/openbr_internal.h" #include +#include using namespace br; @@ -121,9 +122,9 @@ void br_fuse(int num_input_simmats, const char *input_simmats[], Fuse(QtUtils::toStringList(num_input_simmats, input_simmats), normalization, fusion, output_simmat); } -void br_initialize(int &argc, char *argv[], const char *sdk_path) +void br_initialize(int &argc, char *argv[], const char *sdk_path, bool use_gui) { - Context::initialize(argc, argv, sdk_path); + Context::initialize(argc, argv, sdk_path, use_gui); } void br_initialize_default() @@ -381,8 +382,9 @@ bool br_img_is_empty(br_template tmpl) const char* br_get_filename(br_template tmpl) { - Template *t = reinterpret_cast(tmpl); - return t->file.name.toStdString().c_str(); + static QByteArray buffer; + buffer = reinterpret_cast(tmpl)->file.name.toLocal8Bit(); + return buffer.data(); } void br_set_filename(br_template tmpl, const char *filename) @@ -470,3 +472,8 @@ void br_close_gallery(br_gallery gallery) Gallery *gal = reinterpret_cast(gallery); delete gal; } + +void br_deduplicate(const char *input_gallery, const char *output_gallery, const char *threshold) +{ + br::Deduplicate(input_gallery, output_gallery, threshold); +} diff --git a/openbr/openbr.h b/openbr/openbr.h index 4fc9a7c..d1bd2c2 100644 --- a/openbr/openbr.h +++ b/openbr/openbr.h @@ -67,6 +67,17 @@ BR_EXPORT const char *br_about(); BR_EXPORT void br_cat(int num_input_galleries, const char *input_galleries[], const char *output_gallery); /*! + * \brief Removes duplicate templates in a gallery. + * \param input_gallery Gallery to be deduplicated. + * \param output_gallery Deduplicated gallery. + * \param threshold Comparisons with a match score >= this value are designated to be duplicates. + * \note If a gallery contains n duplicates, the first n-1 duplicates in the gallery will be removed and the nth will be kept. + * \note Users are encouraged to use binary gallery formats as the entire gallery is read into memory in one call to Gallery::read. + */ + +BR_EXPORT void br_deduplicate(const char *input_gallery, const char *output_gallery, const char *threshold); + +/*! * \brief Clusters one or more similarity matrices into a list of subjects. * * A similarity matrix is a type of br::Output. The current clustering algorithm is a simplified implementation of \cite zhu11. @@ -213,7 +224,7 @@ BR_EXPORT void br_fuse(int num_input_simmats, const char *input_simmats[], * \brief Wraps br::Context::initialize() * \see br_finalize */ -BR_EXPORT void br_initialize(int &argc, char *argv[], const char *sdk_path = ""); +BR_EXPORT void br_initialize(int &argc, char *argv[], const char *sdk_path = "", bool use_gui = false); /*! * \brief Wraps br::Context::initialize() with default arguments. * \see br_finalize diff --git a/openbr/openbr_plugin.cpp b/openbr/openbr_plugin.cpp index d4df6d1..91c4d70 100644 --- a/openbr/openbr_plugin.cpp +++ b/openbr/openbr_plugin.cpp @@ -44,12 +44,6 @@ using namespace cv; Q_DECLARE_METATYPE(QLocalSocket::LocalSocketState) -// Some globals used to transfer data to Context::messageHandler so that -// we can restart the process if we try and fail to create a QApplication. -static bool creating_qapp = false; -static int * argc_ptr = NULL; -static char ** argv_ptr = NULL; - /* File - public methods */ // Note that the convention for displaying metadata is as follows: // [] for lists in which argument order does not matter (e.g. [FTO=false, Index=0]), @@ -848,13 +842,7 @@ int br::Context::blocks(int size) const bool br::Context::contains(const QString &name) { - QByteArray bytes = name.toLocal8Bit(); - const char * c_name = bytes.constData(); - - for (int i=0; ipropertyCount(); i++) - if (!strcmp(c_name, metaObject()->property(i).name())) - return true; - return false; + return property(qPrintable(name)).isValid(); } void br::Context::printStatus() @@ -907,42 +895,30 @@ bool br::Context::checkSDKPath(const QString &sdkPath) // We create our own when the user hasn't static QCoreApplication *application = NULL; -void br::Context::initialize(int &argc, char *argv[], QString sdkPath, bool use_gui) +void br::Context::initialize(int &argc, char *argv[], QString sdkPath, bool useGui) { - for (int i=0; i < argc; i ++) - { - if (strcmp("-useGui", argv[i]) == 0) { - const char * val = i+1 < argc ? argv[i+1] : ""; - if (strcmp(val, "false") ==0 || strcmp(val, "0") == 0) - use_gui = false; - break; - } - } - qInstallMessageHandler(messageHandler); + QString sep; +#ifndef _WIN32 + useGui = useGui && (getenv("DISPLAY") != NULL); + sep = ":"; +#else + sep = ";"; +#endif // not _WIN32 + // We take in argc as a reference due to: // https://bugreports.qt-project.org/browse/QTBUG-5637 // QApplication should be initialized before anything else. // Since we can't ensure that it gets deleted last, we never delete it. if (QCoreApplication::instance() == NULL) { #ifndef BR_EMBEDDED - if (use_gui) { - // Set up variables to be used in the message handler if this fails. - // Just so you know, we - creating_qapp = true; - argc_ptr = &argc; - argv_ptr = argv; - - application = new QApplication(argc, argv); - creating_qapp = false; - } - else { - application = new QCoreApplication(argc, argv); - } -#else + if (useGui) application = new QApplication(argc, argv); + else application = new QCoreApplication(argc, argv); +#else // not BR_EMBEDDED + useGui = false; application = new QCoreApplication(argc, argv); -#endif +#endif // BR_EMBEDDED } QCoreApplication::setOrganizationName(COMPANY_NAME); @@ -965,14 +941,14 @@ void br::Context::initialize(int &argc, char *argv[], QString sdkPath, bool use_ Globals = new Context(); Globals->init(File()); - Globals->useGui = use_gui; - + Globals->useGui = useGui; Common::seedRNG(); // Search for SDK if (sdkPath.isEmpty()) { QStringList checkPaths; checkPaths << QDir::currentPath() << QCoreApplication::applicationDirPath(); + checkPaths << QString(getenv("PATH")).split(sep, QString::SkipEmptyParts); bool foundSDK = false; foreach (const QString &path, checkPaths) { @@ -1035,26 +1011,6 @@ void br::Context::messageHandler(QtMsgType type, const QMessageLogContext &conte static QMutex generalLock; QMutexLocker locker(&generalLock); - // If we are trying to create a QApplication, and get a fatal, then restart the process - // with useGui set to 0. - if (creating_qapp && type == QtFatalMsg) - { - // re-launch process with useGui = 0 - std::cout << "Failed to initialize gui, restarting with -useGui 0" << std::endl; - QStringList arguments; - arguments.append("-useGui"); - arguments.append("0"); - for (int i=1; i < *argc_ptr; i++) - { - arguments.append(argv_ptr[i]); - } - // QProcess::execute blocks until the other process completes. - QProcess::execute(argv_ptr[0], arguments); - // have to unlock this for some reason - locker.unlock(); - std::exit(0); - } - QString txt; if (type == QtDebugMsg) { if (Globals->quiet) return; @@ -1076,8 +1032,13 @@ void br::Context::messageHandler(QtMsgType type, const QMessageLogContext &conte Globals->logFile.flush(); } - if (type == QtFatalMsg) + if (type == QtFatalMsg) { +#ifdef _WIN32 + QCoreApplication::quit(); // abort() hangs the console on Windows for some reason related to the event loop not being exited +#else // not _WIN32 abort(); // We abort so we can get a stack trace back to the code that triggered the message. +#endif // _WIN32 + } } Context *br::Globals = NULL; diff --git a/openbr/openbr_plugin.h b/openbr/openbr_plugin.h index d74d87e..e6f2850 100644 --- a/openbr/openbr_plugin.h +++ b/openbr/openbr_plugin.h @@ -41,6 +41,7 @@ #include #include #include +#include /*! * \defgroup cpp_plugin_sdk C++ Plugin SDK @@ -794,12 +795,12 @@ public: * By default share/openbr/openbr.bib will be searched for relative to: * -# The working directory * -# The executable's location - * \param use_gui Create a QApplication instead of a QCoreApplication. + * \param useGui Create a QApplication instead of a QCoreApplication. * \note Tiggers \em abort() on failure to locate share/openbr/openbr.bib. * \note Qt users should instead call this after initializing QApplication. * \see finalize */ - static void initialize(int &argc, char *argv[], QString sdkPath = "", bool use_gui = true); + static void initialize(int &argc, char *argv[], QString sdkPath = "", bool useGui = true); /*! * \brief Call \em once at the end of the application to deallocate global variables. @@ -1371,6 +1372,14 @@ BR_EXPORT void Convert(const File &fileType, const File &inputFile, const File & */ BR_EXPORT void Cat(const QStringList &inputGalleries, const QString &outputGallery); +/*! + * \brief Deduplicate a gallery. + * \param inputGallery Gallery to deduplicate. + * \param outputGallery Gallery to store the deduplicated result. + * \param threshold Match score threshold to determine duplicates. + */ +BR_EXPORT void Deduplicate(const File &inputGallery, const File &outputGallery, const QString &threshold); + /*! @}*/ } // namespace br diff --git a/openbr/plugins/algorithms.cpp b/openbr/plugins/algorithms.cpp index f6900c4..06e3275 100644 --- a/openbr/plugins/algorithms.cpp +++ b/openbr/plugins/algorithms.cpp @@ -50,6 +50,9 @@ class AlgorithmsInitializer : public Initializer Globals->abbreviations.insert("DisplayVideo", "Stream(FPSLimit(30)+Show(false,[FrameNumber])+Discard)"); Globals->abbreviations.insert("PerFrameDetection", "Stream(SaveMat(original)+Cvt(Gray)+Cascade(FrontalFace)+ASEFEyes+RestoreMat(original)+Draw(inPlace=true)+Show(false,[FrameNumber])+Discard)"); Globals->abbreviations.insert("AgeGenderDemo", "Stream(SaveMat(original)+Cvt(Gray)+Cascade(FrontalFace)+Expand+++/+Discard+RestoreMat(original)+Draw(inPlace=true)+DrawPropertiesPoint([Age,Gender],Affine_0,inPlace=true)+SaveMat(original)+Discard+Contract+RestoreMat(original)+FPSCalc+Show(false,[AvgFPS,Age,Gender])+Discard)"); + Globals->abbreviations.insert("ShowOpticalFlowField", "Stream(SaveMat(original)+AggregateFrames(2)+OpticalFlow(useMagnitude=false)+Grid(100,100)+DrawOpticalFlow+FPSLimit(30)+Show(false)+Discard)"); + Globals->abbreviations.insert("ShowOpticalFlowMagnitude", "Stream(AggregateFrames(2)+OpticalFlow+Normalize(Range,false,0,255)+Cvt(Color)+Draw+FPSLimit(30)+Show(false)+Discard)"); + Globals->abbreviations.insert("ShowMotionSegmentation", "Stream(DropFrames(5)+AggregateFrames(2)+OpticalFlow+CvtUChar+WatershedSegmentation+DrawSegmentation+Draw+FPSLimit(30)+Show(false)+Discard)"); Globals->abbreviations.insert("HOG", "Stream(DropFrames(5)+Cvt(Gray)+Grid(5,5)+ROIFromPts(32,24)+Expand+Resize(32,32)+Gradient+RectRegions+Bin(0,360,8)+Hist(8)+Cat)+Contract+CatRows+KMeans(500)+Hist(500)+SVM"); Globals->abbreviations.insert("HOF", "Stream(DropFrames(5)+Grid(5,5)+AggregateFrames(2)+OpticalFlow+ROIFromPts(32,24)+Expand+Resize(32,32)+Gradient+RectRegions+Bin(0,360,8)+Hist(8)+Cat)+Contract+CatRows+KMeans(500)+Hist(500)"); @@ -76,7 +79,7 @@ class AlgorithmsInitializer : public Initializer Globals->abbreviations.insert("ColoredLBP", "Open+Affine(128,128,0.37,0.45)+Cvt(Gray)+Blur(1.1)+Gamma(0.2)+DoG(1,2)+ContrastEq(0.1,10)+LBP(1,2)+ColoredU2"); // Transforms - Globals->abbreviations.insert("FaceDetection", "(Open+Cvt(Gray)+Cascade(FrontalFace))"); + Globals->abbreviations.insert("FaceDetection", "Open+Cvt(Gray)+Cascade(FrontalFace)"); Globals->abbreviations.insert("DenseLBP", "(Blur(1.1)+Gamma(0.2)+DoG(1,2)+ContrastEq(0.1,10)+LBP(1,2)+RectRegions(8,8,6,6)+Hist(59))"); Globals->abbreviations.insert("DenseSIFT", "(Grid(10,10)+SIFTDescriptor(12)+ByRow)"); Globals->abbreviations.insert("FaceRecognitionRegistration", "(ASEFEyes+Affine(88,88,0.25,0.35)+DownsampleTraining(FTE(DFFS),instances=1))"); diff --git a/openbr/plugins/cascade.cpp b/openbr/plugins/cascade.cpp index 2abb3c6..cc928b5 100644 --- a/openbr/plugins/cascade.cpp +++ b/openbr/plugins/cascade.cpp @@ -15,6 +15,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include +//#include #include "openbr_internal.h" #include "openbr/core/opencvutils.h" #include "openbr/core/resource.h" @@ -86,11 +87,11 @@ class CascadeTransform : public UntrainableMetaTransform for (int i=0; i rects; - vector rejectLevels; - vector levelWeights; - if (ROCMode) cascade->detectMultiScale(m, rects, rejectLevels, levelWeights, 1.2, 5, (enrollAll ? 0 : CV_HAAR_FIND_BIGGEST_OBJECT) | CV_HAAR_SCALE_IMAGE, Size(minSize, minSize), Size(), true); - else cascade->detectMultiScale(m, rects, 1.2, 5, enrollAll ? 0 : CV_HAAR_FIND_BIGGEST_OBJECT, Size(minSize, minSize)); + std::vector rects; + std::vector rejectLevels; + std::vector levelWeights; + if (ROCMode) cascade->detectMultiScale(m, rects, rejectLevels, levelWeights, 1.2, 5, (enrollAll ? 0 : CASCADE_FIND_BIGGEST_OBJECT) | CASCADE_SCALE_IMAGE, Size(minSize, minSize), Size(), true); + else cascade->detectMultiScale(m, rects, 1.2, 5, enrollAll ? 0 : CASCADE_FIND_BIGGEST_OBJECT, Size(minSize, minSize)); if (!enrollAll && rects.empty()) rects.push_back(Rect(0, 0, m.cols, m.rows)); diff --git a/openbr/plugins/cvt.cpp b/openbr/plugins/cvt.cpp index a0b12e2..fc8d9fb 100644 --- a/openbr/plugins/cvt.cpp +++ b/openbr/plugins/cvt.cpp @@ -14,6 +14,7 @@ * limitations under the License. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +#include #include #include "openbr_internal.h" #include "openbr/core/opencvutils.h" @@ -44,7 +45,8 @@ public: Luv = CV_BGR2Luv, RGB = CV_BGR2RGB, XYZ = CV_BGR2XYZ, - YCrCb = CV_BGR2YCrCb }; + YCrCb = CV_BGR2YCrCb, + Color = CV_GRAY2BGR }; private: BR_PROPERTY(ColorSpace, colorSpace, Gray) @@ -52,8 +54,8 @@ private: void project(const Template &src, Template &dst) const { - if (src.m().channels() > 1) cvtColor(src, dst, colorSpace); - else dst = src; + if (src.m().channels() > 1 || colorSpace == CV_GRAY2BGR) cvtColor(src, dst, colorSpace); + else dst = src; if (channel != -1) { std::vector mv; diff --git a/openbr/plugins/distance.cpp b/openbr/plugins/distance.cpp index b2106c8..0e5bb07 100644 --- a/openbr/plugins/distance.cpp +++ b/openbr/plugins/distance.cpp @@ -18,6 +18,7 @@ #include #include #include +#include #include "openbr_internal.h" #include "openbr/core/distance_sse.h" @@ -61,6 +62,7 @@ private: (a.m().type() != b.m().type())) return -std::numeric_limits::max(); +// TODO: this max value is never returned based on the switch / default float result = std::numeric_limits::max(); switch (metric) { case Correlation: @@ -386,5 +388,67 @@ class OnlineDistance : public Distance BR_REGISTER(Distance, OnlineDistance) +/*! + * \ingroup distances + * \brief Attenuation function based distance from attributes + * \author Scott Klum \cite sklum + */ +class AttributeDistance : public Distance +{ + Q_OBJECT + Q_PROPERTY(QString attribute READ get_attribute WRITE set_attribute RESET reset_attribute STORED false) + BR_PROPERTY(QString, attribute, QString()) + + float compare(const Template &target, const Template &query) const + { + float queryValue = query.file.get(attribute); + float targetValue = target.file.get(attribute); + + // TODO: Set this magic number to something meaningful + float stddev = 1; + + if (queryValue == targetValue) return 1; + else return 1/(stddev*sqrt(2*CV_PI))*exp(-0.5*pow((targetValue-queryValue)/stddev, 2)); + } +}; + +BR_REGISTER(Distance, AttributeDistance) + +/*! + * \ingroup distances + * \brief Sum match scores across multiple distances + * \author Scott Klum \cite sklum + */ +class SumDistance : public Distance +{ + Q_OBJECT + Q_PROPERTY(QList distances READ get_distances WRITE set_distances RESET reset_distances) + BR_PROPERTY(QList, distances, QList()) + + void train(const TemplateList &data) + { + QFutureSynchronizer futures; + foreach (br::Distance *distance, distances) + futures.addFuture(QtConcurrent::run(distance, &Distance::train, data)); + futures.waitForFinished(); + } + + float compare(const Template &target, const Template &query) const + { + float result = 0; + + foreach (br::Distance *distance, distances) { + result += distance->compare(target, query); + + if (result == -std::numeric_limits::max()) + return result; + } + + return result; + } +}; + +BR_REGISTER(Distance, SumDistance) + } // namespace br #include "distance.moc" diff --git a/openbr/plugins/draw.cpp b/openbr/plugins/draw.cpp index 3057b8a..00f9a8e 100644 --- a/openbr/plugins/draw.cpp +++ b/openbr/plugins/draw.cpp @@ -15,6 +15,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include +#include #include #include #include "openbr_internal.h" @@ -40,10 +41,12 @@ class DrawTransform : public UntrainableTransform Q_PROPERTY(bool points READ get_points WRITE set_points RESET reset_points STORED false) Q_PROPERTY(bool rects READ get_rects WRITE set_rects RESET reset_rects STORED false) Q_PROPERTY(bool inPlace READ get_inPlace WRITE set_inPlace RESET reset_inPlace STORED false) + Q_PROPERTY(int lineThickness READ get_lineThickness WRITE set_lineThickness RESET reset_lineThickness STORED false) BR_PROPERTY(bool, verbose, false) BR_PROPERTY(bool, points, true) BR_PROPERTY(bool, rects, true) BR_PROPERTY(bool, inPlace, false) + BR_PROPERTY(int, lineThickness, 1) void project(const Template &src, Template &dst) const { @@ -61,7 +64,7 @@ class DrawTransform : public UntrainableTransform } if (rects) { foreach (const Rect &rect, OpenCVUtils::toRects(src.file.namedRects() + src.file.rects())) - rectangle(dst, rect, color); + rectangle(dst, rect, color, lineThickness); } } }; @@ -343,6 +346,75 @@ class AdjacentOverlayTransform : public Transform BR_REGISTER(Transform, AdjacentOverlayTransform) +/*! + * \ingroup transforms + * \brief Draw a line representing the direction and magnitude of optical flow at the specified points. + * \author Austin Blanton \cite imaus10 + */ +class DrawOpticalFlow : public UntrainableTransform +{ + Q_OBJECT + Q_PROPERTY(QString original READ get_original WRITE set_original RESET reset_original STORED false) + BR_PROPERTY(QString, original, "original") + + void project(const Template &src, Template &dst) const + { + const Scalar color(0,255,0); + Mat flow = src.m(); + dst = src; + if (!dst.file.contains(original)) qFatal("The original img must be saved in the metadata with SaveMat."); + dst.m() = dst.file.get(original); + dst.file.remove(original); + foreach (const Point2f &pt, OpenCVUtils::toPoints(dst.file.points())) { + Point2f dxy = flow.at(pt.y, pt.x); + Point2f newPt(pt.x+dxy.x, pt.y+dxy.y); + line(dst, pt, newPt, color); + } + } +}; +BR_REGISTER(Transform, DrawOpticalFlow) + +/*! + * \ingroup transforms + * \brief Fill in the segmentations or draw a line between intersecting segments. + * \author Austin Blanton \cite imaus10 + */ +class DrawSegmentation : public UntrainableTransform +{ + Q_OBJECT + Q_PROPERTY(bool fillSegment READ get_fillSegment WRITE set_fillSegment RESET reset_fillSegment STORED false) + BR_PROPERTY(bool, fillSegment, true) + + void project(const Template &src, Template &dst) const + { + if (!src.file.contains("SegmentsMask") || !src.file.contains("NumSegments")) qFatal("Must supply a Contours object in the metadata to drawContours."); + Mat segments = src.file.get("SegmentsMask"); + int numSegments = src.file.get("NumSegments"); + + dst.file = src.file; + Mat drawn = fillSegment ? Mat(segments.size(), CV_8UC3, Scalar::all(0)) : src.m(); + + for (int i=1; i > contours; + Scalar color(0,255,0); + findContours(mask, contours, CV_RETR_LIST, CV_CHAIN_APPROX_NONE); + drawContours(drawn, contours, -1, color); + } + } + + dst.m() = drawn; + } +}; +BR_REGISTER(Transform, DrawSegmentation) + // TODO: re-implement EditTransform using Qt #if 0 /*! diff --git a/openbr/plugins/eigen3.cpp b/openbr/plugins/eigen3.cpp index d7ed624..710570e 100644 --- a/openbr/plugins/eigen3.cpp +++ b/openbr/plugins/eigen3.cpp @@ -302,8 +302,8 @@ class LDATransform : public Transform Q_PROPERTY(int directLDA READ get_directLDA WRITE set_directLDA RESET reset_directLDA STORED false) Q_PROPERTY(float directDrop READ get_directDrop WRITE set_directDrop RESET reset_directDrop STORED false) Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) - Q_PROPERTY(bool isBinary READ get_isBinary WRITE set_isBinary RESET reset_isBinary STORED true) - Q_PROPERTY(bool normalize READ get_normalize WRITE set_normalize RESET reset_normalize STORED true) + Q_PROPERTY(bool isBinary READ get_isBinary WRITE set_isBinary RESET reset_isBinary STORED false) + Q_PROPERTY(bool normalize READ get_normalize WRITE set_normalize RESET reset_normalize STORED false) BR_PROPERTY(float, pcaKeep, 0.98) BR_PROPERTY(bool, pcaWhiten, false) BR_PROPERTY(int, directLDA, 0) @@ -316,12 +316,6 @@ class LDATransform : public Transform Eigen::VectorXf mean; Eigen::MatrixXf projection; float stdDev; - bool trained; - - void init() - { - trained = false; - } void train(const TemplateList &_trainingSet) { @@ -461,9 +455,9 @@ class LDATransform : public Transform projection = ((space2.eVecs.transpose() * space1.eVecs.transpose()) * pca.eVecs.transpose()).transpose(); dimsOut = dim2; + stdDev = 1; // default initialize if (isBinary) { assert(dimsOut == 1); - TemplateList projected; float posVal = 0; float negVal = 0; Eigen::MatrixXf results(trainingSet.size(),1); @@ -493,8 +487,6 @@ class LDATransform : public Transform if (normalize) stdDev = sqrt(results.array().square().sum() / trainingSet.size()); } - - trained = true; } void project(const Template &src, Template &dst) const @@ -508,18 +500,22 @@ class LDATransform : public Transform // Do projection outMap = projection.transpose() * (inMap - mean); - if (normalize && isBinary && trained) + if (normalize && isBinary) dst.m().at(0,0) = dst.m().at(0,0) / stdDev; } void store(QDataStream &stream) const { - stream << pcaKeep << directLDA << directDrop << dimsOut << mean << projection << stdDev << normalize << isBinary << trained; + stream << pcaKeep << directLDA << directDrop << dimsOut << mean << projection; + if (normalize && isBinary) + stream << stdDev; } void load(QDataStream &stream) { - stream >> pcaKeep >> directLDA >> directDrop >> dimsOut >> mean >> projection >> stdDev >> normalize >> isBinary >> trained; + stream >> pcaKeep >> directLDA >> directDrop >> dimsOut >> mean >> projection; + if (normalize && isBinary) + stream >> stdDev; } }; diff --git a/openbr/plugins/eyes.cpp b/openbr/plugins/eyes.cpp index cdef8df..5d3929d 100644 --- a/openbr/plugins/eyes.cpp +++ b/openbr/plugins/eyes.cpp @@ -34,6 +34,7 @@ */ #include +#include #include "openbr_internal.h" #include "openbr/core/opencvutils.h" diff --git a/openbr/plugins/format.cpp b/openbr/plugins/format.cpp index aad33e8..de96360 100644 --- a/openbr/plugins/format.cpp +++ b/openbr/plugins/format.cpp @@ -16,10 +16,12 @@ #include #include +#include #ifndef BR_EMBEDDED #include #endif // BR_EMBEDDED #include +#include #include "openbr_internal.h" #include "openbr/core/bee.h" @@ -719,7 +721,7 @@ BR_REGISTER(Format, xmlFormat) /*! * \ingroup formats * \brief Reads in scores or ground truth from a text table. - * \author Josh Klontz + * \author Josh Klontz \cite jklontz * * Example of the format: * \code @@ -768,6 +770,90 @@ class scoresFormat : public Format BR_REGISTER(Format, scoresFormat) +/*! + * \ingroup formats + * \brief Reads FBI EBTS transactions. + * \author Scott Klum \cite sklum + * https://www.fbibiospecs.org/ebts.html + * \note This will fail if a binary blob contains any of the fields attempt to locate within the file + */ +class ebtsFormat : public Format +{ + Q_OBJECT + + QString textFieldValue(const QByteArray &byteArray, const QString &fieldNumber, int from = 0) const + { + // Find the field, skip the number bytes, and account for the semicolon + int fieldPosition = byteArray.indexOf(fieldNumber, from) + fieldNumber.size() + 1; + int sepPosition = byteArray.indexOf(QChar(0x1D),fieldPosition); + + return byteArray.mid(fieldPosition,sepPosition-fieldPosition); + } + + Template read() const + { + QByteArray byteArray; + QtUtils::readFile(file, byteArray); + + Template t; + + Mat m; + + // Demographics + { + QString name = textFieldValue(byteArray, "2.018"); + QStringList names = name.split(','); + t.file.set("FIRSTNAME", names.at(1)); + t.file.set("LASTNAME", names.at(0)); + t.file.set("DOB", textFieldValue(byteArray, "2.022").toInt()); + t.file.set("GENDER", textFieldValue(byteArray, "2.024")); + t.file.set("RACE", textFieldValue(byteArray, "2.025")); + } + + // Mugshot (first in file used) + // Todo: Check for face designation + { + const QString imageRecord = "10.001:"; + const QString imageDataRecord = "10.999:"; + + int fieldPosition = byteArray.indexOf(imageRecord); + + if (fieldPosition != -1) { + int sepPosition = byteArray.indexOf(QChar(0x1D),fieldPosition); + int dataPosition = byteArray.indexOf(imageDataRecord,sepPosition); + + int recordBytes = byteArray.mid(fieldPosition,sepPosition-fieldPosition).toInt(); + int headerBytes = byteArray.mid(fieldPosition,dataPosition-fieldPosition).size() + imageDataRecord.size(); + + QByteArray data = byteArray.mid(dataPosition+imageRecord.size(),recordBytes-headerBytes); + + m = imdecode(Mat(3, data.size(), CV_8UC3, data.data()), CV_LOAD_IMAGE_COLOR); + if (!m.data) qWarning("ebtsFormat::read failed to decode image data."); + t.m() = m; + } else qWarning("ebtsFormat::cannot find image data within file."); + + } + + if (t.file.contains("DOB")) { + const QDate dob = QDate::fromString(t.file.get("DOB"), "yyyyMMdd"); + const QDate current = QDate::currentDate(); + int age = current.year() - dob.year(); + if (current.month() < dob.month()) age--; + t.file.set("Age", age); + } + + return t; + } + + void write(const Template &t) const + { + (void) t; + qFatal("Writing EBTS files is not supported."); + } +}; + +BR_REGISTER(Format, ebtsFormat) + } // namespace br #include "format.moc" diff --git a/openbr/plugins/gallery.cpp b/openbr/plugins/gallery.cpp index bb1b4f2..fbc4950 100644 --- a/openbr/plugins/gallery.cpp +++ b/openbr/plugins/gallery.cpp @@ -32,6 +32,11 @@ #include "openbr/core/opencvutils.h" #include "openbr/core/qtutils.h" +#ifdef CVMATIO +#include "MatlabIO.hpp" +#include "MatlabIOContainer.hpp" +#endif + namespace br { @@ -969,6 +974,74 @@ class landmarksGallery : public Gallery BR_REGISTER(Gallery, landmarksGallery) +#ifdef CVMATIO + +using namespace cv; + +class vbbGallery : public Gallery +{ + Q_OBJECT + + void init() + { + MatlabIO matio; + QString filename = (Globals->path.isEmpty() ? "" : Globals->path + "/") + file.name; + bool ok = matio.open(filename.toStdString(), "r"); + if (!ok) qFatal("Couldn't open the vbb file"); + + vector variables; + variables = matio.read(); + matio.close(); + + double vers = variables[1].data().at(0,0); + if (vers != 1.4) qFatal("This is an old vbb version, we don't mess with that."); + + A = variables[0].data > >().at(0); + objLists = A.at(1).data >(); + + // start at the first frame (duh!) + currFrame = 0; + } + + TemplateList readBlock(bool *done) + { + *done = false; + Template rects(file); + if (objLists[currFrame].typeEquals > >()) { + vector > bbs = objLists[currFrame].data > >(); + for (unsigned int i=0; i bb = bbs[i]; + Mat pos = bb[1].data(); + double left = pos.at(0,0); + double top = pos.at(0,1); + double width = pos.at(0,2); + double height = pos.at(0,3); + rects.file.appendRect(QRectF(left, top, width, height)); + } + } + TemplateList tl; + tl.append(rects); + if (++currFrame == (int)objLists.size()) *done = true; + return tl; + } + + void write(const Template &t) + { + (void)t; qFatal("Not implemented"); + } + +private: + // this holds a bunch of stuff, maybe we'll use it all later + vector A; + // this, a field in A, holds bounding boxes for each frame + vector objLists; + int currFrame; +}; + +BR_REGISTER(Gallery, vbbGallery) + +#endif + } // namespace br #include "gallery.moc" diff --git a/openbr/plugins/gui.cpp b/openbr/plugins/gui.cpp index 6d82687..a508d26 100644 --- a/openbr/plugins/gui.cpp +++ b/openbr/plugins/gui.cpp @@ -12,6 +12,7 @@ #include #include +#include #include "openbr_internal.h" #include "openbr/gui/utility.h" diff --git a/openbr/plugins/hist.cpp b/openbr/plugins/hist.cpp index 6fc3bb9..d9f0a05 100644 --- a/openbr/plugins/hist.cpp +++ b/openbr/plugins/hist.cpp @@ -94,7 +94,7 @@ class BinTransform : public UntrainableTransform } else if (channels == 2) { // If there are two channels, the first is channel is assumed to be a weight vector // and the second channel contains the vectors we would like to bin. - vector mv; + std::vector mv; cv::split(src, mv); weights = mv[0]; weights.convertTo(weights, CV_32F); diff --git a/openbr/plugins/integral.cpp b/openbr/plugins/integral.cpp index bb42b4a..f6117a0 100644 --- a/openbr/plugins/integral.cpp +++ b/openbr/plugins/integral.cpp @@ -1,4 +1,5 @@ #include +#include #include #include "openbr_internal.h" @@ -293,7 +294,7 @@ private: Sobel(src, dx, CV_32F, 1, 0, CV_SCHARR); Sobel(src, dy, CV_32F, 0, 1, CV_SCHARR); cartToPolar(dx, dy, magnitude, angle, true); - vector mv; + std::vector mv; if ((channel == Magnitude) || (channel == MagnitudeAndAngle)) { const float theoreticalMaxMagnitude = sqrt(2*pow(float(2*(3+10+3)*255), 2.f)); mv.push_back(magnitude / theoreticalMaxMagnitude); diff --git a/openbr/plugins/landmarks.cpp b/openbr/plugins/landmarks.cpp index 1cb4445..c3b924b 100644 --- a/openbr/plugins/landmarks.cpp +++ b/openbr/plugins/landmarks.cpp @@ -290,16 +290,18 @@ class DrawDelaunayTransform : public UntrainableTransform void project(const Template &src, Template &dst) const { - QList validTriangles = OpenCVUtils::toPoints(src.file.getList("DelaunayTriangles")); + dst = src; - // Clone the matrix do draw on it - dst.m() = src.m().clone(); + if (src.file.contains("DelaunayTriangles")) { + QList validTriangles = OpenCVUtils::toPoints(src.file.getList("DelaunayTriangles")); - for (int i = 0; i < validTriangles.size(); i+=3) { - line(dst.m(), validTriangles[i], validTriangles[i+1], Scalar(0,0,0), 1); - line(dst.m(), validTriangles[i+1], validTriangles[i+2], Scalar(0,0,0), 1); - line(dst.m(), validTriangles[i+2], validTriangles[i], Scalar(0,0,0), 1); - } + // Clone the matrix do draw on it + for (int i = 0; i < validTriangles.size(); i+=3) { + line(dst, validTriangles[i], validTriangles[i+1], Scalar(0,0,0), 1); + line(dst, validTriangles[i+1], validTriangles[i+2], Scalar(0,0,0), 1); + line(dst, validTriangles[i+2], validTriangles[i], Scalar(0,0,0), 1); + } + } else qWarning("Template does not contain Delaunay triangulation."); } }; diff --git a/openbr/plugins/lbp.cpp b/openbr/plugins/lbp.cpp index 40e972e..b295286 100644 --- a/openbr/plugins/lbp.cpp +++ b/openbr/plugins/lbp.cpp @@ -15,7 +15,9 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include +#include #include +#include #include #include "openbr_internal.h" diff --git a/openbr/plugins/ltp.cpp b/openbr/plugins/ltp.cpp index 984987f..df34070 100644 --- a/openbr/plugins/ltp.cpp +++ b/openbr/plugins/ltp.cpp @@ -15,6 +15,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include +#include #include #include "openbr_internal.h" diff --git a/openbr/plugins/mask.cpp b/openbr/plugins/mask.cpp index 3019934..6898c70 100644 --- a/openbr/plugins/mask.cpp +++ b/openbr/plugins/mask.cpp @@ -15,6 +15,7 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include +#include #include "openbr_internal.h" using namespace cv; @@ -164,7 +165,7 @@ class LargestConvexAreaTransform : public UntrainableTransform void project(const Template &src, Template &dst) const { std::vector< std::vector > contours; - findContours(src.clone(), contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE); + findContours(src.m().clone(), contours, CV_RETR_EXTERNAL, CV_CHAIN_APPROX_SIMPLE); double maxArea = 0; foreach (const std::vector &contour, contours) { std::vector hull; diff --git a/openbr/plugins/misc.cpp b/openbr/plugins/misc.cpp index 2aee23f..16c8c39 100644 --- a/openbr/plugins/misc.cpp +++ b/openbr/plugins/misc.cpp @@ -537,7 +537,7 @@ class ProgressCounterTransform : public TimeVaryingTransform { (void) data; float p = br_progress(); - qDebug("%05.2f%% ELAPSED=%s REMAINING=%s COUNT=%g/%g \r", p*100., QtUtils::toTime(Globals->startTime.elapsed()/1000.0f).toStdString().c_str(), QtUtils::toTime(0).toStdString().c_str(), Globals->currentStep, Globals->totalSteps); + qDebug("%05.2f%% ELAPSED=%s REMAINING=%s COUNT=%g/%g \r", p*100, QtUtils::toTime(Globals->startTime.elapsed()/1000.0f).toStdString().c_str(), QtUtils::toTime(0).toStdString().c_str(), Globals->currentStep, Globals->totalSteps); } void init() diff --git a/openbr/plugins/motion.cpp b/openbr/plugins/motion.cpp index d746527..4511336 100644 --- a/openbr/plugins/motion.cpp +++ b/openbr/plugins/motion.cpp @@ -23,6 +23,7 @@ class OpticalFlowTransform : public UntrainableMetaTransform Q_PROPERTY(int poly_n READ get_poly_n WRITE set_poly_n RESET reset_poly_n STORED false) Q_PROPERTY(double poly_sigma READ get_poly_sigma WRITE set_poly_sigma RESET reset_poly_sigma STORED false) Q_PROPERTY(int flags READ get_flags WRITE set_flags RESET reset_flags STORED false) + Q_PROPERTY(bool useMagnitude READ get_useMagnitude WRITE set_useMagnitude RESET reset_useMagnitude STORED false) // these defaults are optimized for KTH BR_PROPERTY(double, pyr_scale, 0.1) BR_PROPERTY(int, levels, 1) @@ -31,22 +32,27 @@ class OpticalFlowTransform : public UntrainableMetaTransform BR_PROPERTY(int, poly_n, 7) BR_PROPERTY(double, poly_sigma, 1.1) BR_PROPERTY(int, flags, 0) + BR_PROPERTY(bool, useMagnitude, true) void project(const Template &src, Template &dst) const { // get the two images put there by AggregateFrames if (src.size() != 2) qFatal("Optical Flow requires two images."); - Mat prevImg = src[0], nextImg = src[1], flow, flowOneCh; + Mat prevImg = src[0], nextImg = src[1], flow; if (src[0].channels() != 1) OpenCVUtils::cvtGray(src[0], prevImg); if (src[1].channels() != 1) OpenCVUtils::cvtGray(src[1], nextImg); calcOpticalFlowFarneback(prevImg, nextImg, flow, pyr_scale, levels, winsize, iterations, poly_n, poly_sigma, flags); - // the result is two channels - std::vector channels(2); - split(flow, channels); - magnitude(channels[0], channels[1], flowOneCh); - - dst += flowOneCh; + if (useMagnitude) { + // the result is two channels + Mat flowOneCh; + std::vector channels(2); + split(flow, channels); + magnitude(channels[0], channels[1], flowOneCh); + dst += flowOneCh; + } else { + dst += flow; + } dst.file = src.file; } }; @@ -62,7 +68,8 @@ class SubtractBackgroundTransform : public TimeVaryingTransform { Q_OBJECT - BackgroundSubtractorMOG2 mog; + // TODO: This is broken. + // BackgroundSubtractorMOG2 mog; public: SubtractBackgroundTransform() : TimeVaryingTransform(false, false) {} @@ -72,7 +79,8 @@ private: { dst = src; Mat mask; - mog(src, mask); + // TODO: broken + // mog(src, mask); erode(mask, mask, Mat()); dilate(mask, mask, Mat()); dst.file.set("Mask", QVariant::fromValue(mask)); @@ -86,7 +94,8 @@ private: void finalize(TemplateList &output) { (void) output; - mog = BackgroundSubtractorMOG2(); + // TODO: Broken + // mog = BackgroundSubtractorMOG2(); } }; diff --git a/openbr/plugins/normalize.cpp b/openbr/plugins/normalize.cpp index 842e952..449ca61 100644 --- a/openbr/plugins/normalize.cpp +++ b/openbr/plugins/normalize.cpp @@ -60,24 +60,29 @@ class NormalizeTransform : public UntrainableTransform Q_PROPERTY(bool ByRow READ get_ByRow WRITE set_ByRow RESET reset_ByRow STORED false) BR_PROPERTY(bool, ByRow, false) + Q_PROPERTY(int alpha READ get_alpha WRITE set_alpha RESET reset_alpha STORED false) + BR_PROPERTY(int, alpha, 1) + Q_PROPERTY(int beta READ get_beta WRITE set_beta RESET reset_beta STORED false) + BR_PROPERTY(int, beta, 0) public: /*!< */ enum NormType { Inf = NORM_INF, L1 = NORM_L1, - L2 = NORM_L2 }; + L2 = NORM_L2, + Range = NORM_MINMAX }; private: BR_PROPERTY(NormType, normType, L2) void project(const Template &src, Template &dst) const { - if (!ByRow) normalize(src, dst, 1, 0, normType, CV_32F); + if (!ByRow) normalize(src, dst, alpha, beta, normType, CV_32F); else { dst = src; for (int i=0; i labels = data.indexProperty(inputVariable); const int dims = m.cols; - vector mv, av, bv; + std::vector mv, av, bv; split(m, mv); for (size_t c = 0; c < mv.size(); c++) { av.push_back(Mat(1, dims, CV_64FC1)); diff --git a/openbr/plugins/output.cpp b/openbr/plugins/output.cpp index 1ad0995..b9a290a 100644 --- a/openbr/plugins/output.cpp +++ b/openbr/plugins/output.cpp @@ -268,7 +268,7 @@ class rrOutput : public MatrixOutput for (int i=0; i Pair; foreach (const Pair &pair, Common::Sort(OpenCVUtils::matrixToVector(data.row(i)), true, limit)) { @@ -276,7 +276,7 @@ class rrOutput : public MatrixOutput if (pair.first < threshold) break; File target = targetFiles[pair.second]; target.set("Score", QString::number(pair.first)); - if (simple) files.append(target.baseName() + " " + QString::number(pair.first)); + if (simple) files.append(target.fileName() + " " + QString::number(pair.first)); else files.append(target.flat()); } } @@ -528,7 +528,7 @@ class tailOutput : public Output } else { // General case for (int k=0; k atLeast) && (comparisons.last().value < threshold)) comparisons.removeLast(); + lastValue = comparisons.last().value; comparisonsLock.unlock(); } diff --git a/openbr/plugins/process.cpp b/openbr/plugins/process.cpp index d5c687b..cde5d87 100644 --- a/openbr/plugins/process.cpp +++ b/openbr/plugins/process.cpp @@ -531,8 +531,6 @@ class ProcessWrapperTransform : public TimeVaryingTransform baseKey = id.toString(); QStringList argumentList; - argumentList.append("-useGui"); - argumentList.append("0"); argumentList.append("-algorithm"); argumentList.append(transform); if (!Globals->path.isEmpty()) { diff --git a/openbr/plugins/quality.cpp b/openbr/plugins/quality.cpp index dfa0a21..0abc8a5 100644 --- a/openbr/plugins/quality.cpp +++ b/openbr/plugins/quality.cpp @@ -272,92 +272,75 @@ BR_REGISTER(Distance, ZScoreDistance) /*! * \ingroup distances - * \brief Match Probability modification for heat maps \cite klare12 + * \brief 1v1 heat map comparison * \author Scott Klum \cite sklum */ class HeatMapDistance : public Distance { Q_OBJECT - Q_PROPERTY(br::Distance* distance READ get_distance WRITE set_distance RESET reset_distance STORED false) - Q_PROPERTY(bool gaussian READ get_gaussian WRITE set_gaussian RESET reset_gaussian STORED false) - Q_PROPERTY(bool crossModality READ get_crossModality WRITE set_crossModality RESET reset_crossModality STORED false) + Q_PROPERTY(QString description READ get_description WRITE set_description RESET reset_description STORED false) Q_PROPERTY(int step READ get_step WRITE set_step RESET reset_step STORED false) Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) - BR_PROPERTY(br::Distance*, distance, make("Dist(L2)")) - BR_PROPERTY(bool, gaussian, true) - BR_PROPERTY(bool, crossModality, false) + BR_PROPERTY(QString, description, "IdenticalDistance") BR_PROPERTY(int, step, 1) BR_PROPERTY(QString, inputVariable, "Label") - QList mp; + QList distances; void train(const TemplateList &src) { - distance->train(src); - - const QList labels = src.indexProperty(inputVariable); - QList patches; // Split src into list of TemplateLists of corresponding patches across all Templates for (int i=0; i matrixOutput(MatrixOutput::make(FileList(patches[0].size()), FileList(patches[0].size()))); + while (distances.size() < patches.size()) + distances.append(make(description)); - for (int i=0; icompare(patches[i], patches[i], matrixOutput.data()); - QList genuineScores, impostorScores; - genuineScores.reserve(step); - impostorScores.reserve(step); - for (int j=0; jdata.rows; j++) { - for (int k=0; kdata.at(j, k); - if (score == -std::numeric_limits::max()) continue; - if (crossModality) if(src[j*step].file.get("MODALITY") == src[k*step].file.get("MODALITY")) continue; - if (labels[j*step] == labels[k*step]) genuineScores.append(score); - else impostorScores.append(score); - } - } - - mp.append(MP(genuineScores, impostorScores)); - } + // Train on a distance for each patch + for (int i=0; itrain(patches[i]); } float compare(const Template &target, const Template &query) const { (void) target; (void) query; - qFatal("You did this wrong"); + qFatal("Heatmap Distance not compatible with Template to Template comparison."); return 0; } void compare(const TemplateList &target, const TemplateList &query, Output *output) const { - for (int i=0; icompare(target[i],query[i]); - if (rawScore == -std::numeric_limits::max()) output->setRelative(rawScore, i, 0); - else output->setRelative(mp[i](rawScore, gaussian), i, 0); + for (int i=0; isetRelative(distances[j]->compare(target[i][j],query[i][j]), j, 0); } } void store(QDataStream &stream) const { - distance->store(stream); - stream << mp; + stream << distances.size(); + foreach (Distance *distance, distances) + distance->store(stream); } void load(QDataStream &stream) { - distance->load(stream); - stream >> mp; + int numDistances; + stream >> numDistances; + while (distances.size() < numDistances) + distances.append(make(description)); + foreach (Distance *distance, distances) + distance->load(stream); } }; diff --git a/openbr/plugins/segmentation.cpp b/openbr/plugins/segmentation.cpp new file mode 100644 index 0000000..f11532b --- /dev/null +++ b/openbr/plugins/segmentation.cpp @@ -0,0 +1,53 @@ +#include +#include "openbr_internal.h" +#include "openbr/core/opencvutils.h" + +using namespace cv; + +namespace br +{ + +/*! + * \ingroup transforms + * \brief Applies watershed segmentation. + * \author Austin Blanton \cite imaus10 + */ +class WatershedSegmentationTransform : public UntrainableTransform +{ + Q_OBJECT + void project(const Template &src, Template &dst) const + { + dst = src; + + Mat mod; +// adaptiveThreshold(src.m(), src.m(), 255, ADAPTIVE_THRESH_GAUSSIAN_C, THRESH_BINARY, 33, 5); + threshold(src.m(), mod, 0, 255, THRESH_BINARY+THRESH_OTSU); + + // findContours requires an 8-bit 1-channel image + // and modifies its source image + if (mod.depth() != CV_8U) OpenCVUtils::cvtUChar(mod, mod); + if (mod.channels() != 1) OpenCVUtils::cvtGray(mod, mod); + vector > contours; + vector hierarchy; + findContours(mod, contours, hierarchy, CV_RETR_CCOMP, CV_CHAIN_APPROX_SIMPLE); + + // draw the contour delineations as 1,2,3... for input to watershed + Mat markers = Mat::zeros(mod.size(), CV_32S); + int compCount=0; + for (int idx=0; idx>=0; idx=hierarchy[idx][0], compCount++) { + drawContours(markers, contours, idx, Scalar::all(compCount+1), -1, 8, hierarchy, INT_MAX); + } + + Mat orig = src.m(); + // watershed requires a 3-channel 8-bit image + if (orig.channels() == 1) cvtColor(orig, orig, CV_GRAY2BGR); + watershed(orig, markers); + dst.file.set("SegmentsMask", QVariant::fromValue(markers)); + dst.file.set("NumSegments", compCount); + } +}; +BR_REGISTER(Transform, WatershedSegmentationTransform) + +} // namespace br + +#include "segmentation.moc" diff --git a/openbr/plugins/stasm4.cpp b/openbr/plugins/stasm4.cpp index 10aaef1..c32bde3 100644 --- a/openbr/plugins/stasm4.cpp +++ b/openbr/plugins/stasm4.cpp @@ -35,7 +35,7 @@ class StasmInitializer : public Initializer void initialize() const { - Globals->abbreviations.insert("RectFromStasmEyes","RectFromPoints([28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47],0.15,5.3)"); + Globals->abbreviations.insert("RectFromStasmEyes","RectFromPoints([28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47],0.3,5.3)"); Globals->abbreviations.insert("RectFromStasmBrow","RectFromPoints([16,17,18,19,20,21,22,23,24,25,26,27],0.15,5)"); Globals->abbreviations.insert("RectFromStasmNose","RectFromPoints([48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58],0.15,1.15)"); Globals->abbreviations.insert("RectFromStasmMouth","RectFromPoints([59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76],0.3,2)"); diff --git a/openbr/plugins/stream.cpp b/openbr/plugins/stream.cpp index 440ab1c..749c44a 100644 --- a/openbr/plugins/stream.cpp +++ b/openbr/plugins/stream.cpp @@ -1,17 +1,20 @@ +#include #include #include #include #include #include -#include +#include #include +#include +#include #include "openbr_internal.h" - #include "openbr/core/common.h" #include "openbr/core/opencvutils.h" #include "openbr/core/qtutils.h" using namespace cv; +using namespace std; namespace br { @@ -203,6 +206,13 @@ public: virtual bool getNextTemplate(Template & output)=0; protected: Template basis; + string getAbsolutePath(QString filename) + { + // Yes, we should specify absolute path: + // http://stackoverflow.com/questions/9396459/loading-a-video-in-opencv-in-python + QString fileName = (Globals->path.isEmpty() ? "" : Globals->path + "/") + filename; + return QFileInfo(fileName).absoluteFilePath().toStdString(); + } }; static QMutex openLock; @@ -236,12 +246,9 @@ public: qDebug("Video not open!"); } } else { - // Yes, we should specify absolute path: - // http://stackoverflow.com/questions/9396459/loading-a-video-in-opencv-in-python - QString fileName = (Globals->path.isEmpty() ? "" : Globals->path + "/") + input.file.name; // On windows, this appears to not be thread-safe QMutexLocker lock(&openLock); - video.open(QFileInfo(fileName).absoluteFilePath().toStdString()); + video.open(getAbsolutePath(input.file.name)); } return video.isOpened(); @@ -319,6 +326,198 @@ protected: bool data_ok; }; +class SeqReader : public TemplateProcessor +{ +public: + SeqReader() {} + + bool open(Template &input) + { + basis = input; + + seqFile.open(getAbsolutePath(input.file.name).c_str(), ios::in | ios::binary | ios::ate); + if (!isOpen()) return false; + + int headSize = 1024; + // start at end of file to get full size + int fileSize = seqFile.tellg(); + if (fileSize < headSize) { + qDebug("No header in seq file"); + return false; + } + + // first 4 bytes store 0xEDFE, next 24 store 'Norpix seq ' + char firstFour[4]; + seqFile.seekg(0, ios::beg); + seqFile.read(firstFour, 4); + char nextTwentyFour[24]; + readText(24, nextTwentyFour); + if (firstFour[0] != (char)0xED || firstFour[1] != (char)0xFE || strncmp(nextTwentyFour, "Norpix seq", 10) != 0) { + qDebug("Invalid header in seq file"); + return false; + } + + // next 8 bytes for version (skipped below) and header size (1024), then 512 for descr + seqFile.seekg(4, ios::cur); + int hSize = readInt(); + if (hSize != headSize) { + qDebug("Invalid header size"); + return false; + } + char desc[512]; + readText(512, desc); + basis.file.set("Description", QString(desc)); + + width = readInt(); + height = readInt(); + // get # channels from bit depth + numChan = readInt()/8; + int imageBitDepthReal = readInt(); + if (imageBitDepthReal != 8) { + qDebug("Invalid bit depth"); + return false; + } + // the size of just the image part of a raw img + imgSizeBytes = readInt(); + + int imgFormatInt = readInt(); + if (imgFormatInt == 100 || imgFormatInt == 200 || imgFormatInt == 101) { + imgFormat = "raw"; + } else if (imgFormatInt == 102 || imgFormatInt == 201 || imgFormatInt == 103 || + imgFormatInt == 1 || imgFormatInt == 2) { + imgFormat = "compressed"; + } else { + qFatal("unsupported image format"); + } + + numFrames = readInt(); + // skip empty int + seqFile.seekg(4, ios::cur); + // the size of a full raw file, with extra crap after img data + trueImgSizeBytes = readInt(); + + // gather all the frame positions in an array + seekPos.reserve(numFrames); + // start at end of header + seekPos.append(headSize); + // extra 8 bytes at end of img + int extra = 8; + for (int i=1; i("vbb"); + annotations = TemplateList::fromGallery(File(vbb)); + } +#else + qWarning("cvmatio not installed, bounding boxes will not be available. Add -DBR_WITH_CVMATIO cmake flag to install."); +#endif + + return true; + } + + bool isOpen() + { + return seqFile.is_open(); + } + + void close() + { + seqFile.close(); + } + + bool getNextTemplate(Template &output) + { + if (!isOpen()) { + qDebug("Seq not open"); + return false; + } + // if we've reached the last frame, we're done + if (seekPos.size() == 0) return false; + + seqFile.seekg(seekPos.dequeue(), ios::beg); + + Mat temp; + // let imdecode do all the work to decode the compressed img + if (imgFormat == "compressed") { + int imgSize = readInt() - 4; + vector imgBuf(imgSize); + seqFile.read(&imgBuf[0], imgSize); + // flags < 0 means load image as-is (keep color info if available) + imdecode(imgBuf, -1, &temp); + } + // raw images can be loaded straight into a Mat + else { + char *imgBuf = new char[imgSizeBytes]; + seqFile.read(imgBuf, imgSizeBytes); + int type = (numChan == 1 ? CV_8UC1 : CV_8UC3); + temp = Mat(height, width, type, imgBuf); + } + + output.file = basis.file; + if (!annotations.empty()) { + output.file.setRects(annotations.first().file.rects()); + annotations.removeFirst(); + } + output.m() = temp; + + return true; + } +private: + int readInt() + { + int num; + seqFile.read((char*)&num, 4); + return num; + } + + // apparently the text in seq files is 16 bit characters (UTF-16?) + // since we don't really need the last byte, snad since it gets interpreted as + // a terminating char, let's just grab the first byte for storage + void readText(int bytes, char * buffer) + { + seqFile.read(buffer, bytes); + for (int i=0; i seekPos; + int width, height, numChan, imgSizeBytes, trueImgSizeBytes, numFrames; + QString imgFormat; + TemplateList annotations; +}; // Interface for sequentially getting data from some data source. // Given a TemplateList, return single template frames sequentially by applying a TemplateProcessor @@ -514,11 +713,16 @@ protected: if (frameSource) frameSource->close(); + Template curr = this->templates[current_template_idx]; if (mode == br::Idiocy::Auto) { delete frameSource; - if (this->templates[this->current_template_idx].empty()) - frameSource = new VideoReader(); + if (curr.empty()) { + if (curr.file.name.right(3) == "seq") + frameSource = new SeqReader(); + else + frameSource = new VideoReader(); + } else frameSource = new DirectReturn(); } @@ -529,11 +733,14 @@ protected: } else if (mode == br::Idiocy::StreamVideo) { - if (!frameSource) - frameSource = new VideoReader(); + if (!frameSource) { + if (curr.file.name.right(3) == "seq") + frameSource = new SeqReader(); + else + frameSource = new VideoReader(); + } } - - open_res = frameSource->open(this->templates[current_template_idx]); + open_res = frameSource->open(curr); if (!open_res) { current_template_idx++; @@ -1079,8 +1286,7 @@ public: // Wait for the stream to process the last frame available from // the data source. - bool wait_res = false; - wait_res = readStage->dataSource.waitLast(); + readStage->dataSource.waitLast(); // Now that there are no more incoming frames, call finalize // on each transform in turn to collect any last templates @@ -1217,6 +1423,7 @@ public: { // Delete all the stages for (int i = 0; i < processingStages.size(); i++) { +// TODO: Are we releasing memory which is already freed? delete processingStages[i]; } processingStages.clear(); diff --git a/scripts/downloadDatasets.sh b/scripts/downloadDatasets.sh index 22fb149..0cb08f6 100755 --- a/scripts/downloadDatasets.sh +++ b/scripts/downloadDatasets.sh @@ -35,6 +35,35 @@ if [ ! -d ../data/BioID/img ]; then rm *.eye description.txt BioID-FaceDatabase-V1.2.zip fi +# Caltech Pedestrian +if [ ! -d ../data/CaltechPedestrians/vid ]; then + mkdir ../data/CaltechPedestrians/vid + echo "Downloading Caltech Pedestrians dataset..." + prefix="http://www.vision.caltech.edu/Image_Datasets/CaltechPedestrians/datasets/USA" + for seq in {0..10}; do + fname=`printf "set%02d.tar" $seq` + dlpath="$prefix/$fname" + if hash curl 2>/dev/null; then + curl -OL $dlpath + else + wget $dlpath + fi + tar -xf $fname + done + rm *.tar + ./writeCaltechPedestrianSigset.sh 0 5 train > ../data/CaltechPedestrians/train.xml + ./writeCaltechPedestrianSigset.sh 6 10 test > ../data/CaltechPedestrians/test.xml + mv set* ../data/CaltechPedestrians/vid + if hash curl 2>/dev/null; then + curl -OL "$prefix/annotations.zip" + else + wget "$prefix/annotations.zip" + fi + unzip annotations.zip + rm annotations.zip + mv annotations ../data/CaltechPedestrians +fi + # INRIA person if [ ! -d ../data/INRIAPerson/img ]; then echo "Downloading INRIA person dataset..." @@ -66,7 +95,7 @@ if [ ! -d ../data/KTH/vid ]; then fi mkdir ../data/KTH/vid/${vidclass} unzip ${vidclass}.zip -d ../data/KTH/vid/${vidclass} - rm ${vidclass}.zip + rm ${vidclass}.zip done # this file is corrupted rm -f ../data/KTH/vid/boxing/person01_boxing_d4_uncomp.avi diff --git a/scripts/evalAgeRegression-PCSO.sh b/scripts/evalAgeRegression-PCSO.sh index 9e294fe..2f06c67 100755 --- a/scripts/evalAgeRegression-PCSO.sh +++ b/scripts/evalAgeRegression-PCSO.sh @@ -4,7 +4,7 @@ if [ ! -f evalAgeRegression-PCSO.sh ]; then exit fi -export BR="../build/app/br/br -useGui 0" +export BR=../build/app/br/br export PCSO_DIR=/user/pripshare/Databases/FaceDatabases/PCSO/PCSO/ export ageAlg=AgeRegression diff --git a/scripts/evalFaceRecognition-LFW.sh b/scripts/evalFaceRecognition-LFW.sh index 4f0e6c4..53e5636 100755 --- a/scripts/evalFaceRecognition-LFW.sh +++ b/scripts/evalFaceRecognition-LFW.sh @@ -20,7 +20,7 @@ if [ ! -e Algorithm_Dataset ]; then fi # Run the LFW test protocol -br -useGui 0 -algorithm $ALGORITHM -path ../data/LFW/img/ -crossValidate 10 -pairwiseCompare ../data/LFW/sigset/test_image_restricted_target.xml ../data/LFW/sigset/test_image_restricted_query.xml ${ALGORITHM}_LFW.mtx -convert Output ${ALGORITHM}_lfw.mtx Algorithm_Dataset/${ALGORITHM}_LFW%1.eval +br -algorithm $ALGORITHM -path ../data/LFW/img/ -crossValidate 10 -pairwiseCompare ../data/LFW/sigset/test_image_restricted_target.xml ../data/LFW/sigset/test_image_restricted_query.xml ${ALGORITHM}_LFW.mtx -convert Output ${ALGORITHM}_lfw.mtx Algorithm_Dataset/${ALGORITHM}_LFW%1.eval # Plot results -br -useGui 0 -plot Algorithm_Dataset/* 'lfw_results.pdf[smooth=Dataset,rocOptions[yLimits=(0,1)]]' +br -plot Algorithm_Dataset/* 'lfw_results.pdf[smooth=Dataset,rocOptions[yLimits=(0,1)]]' diff --git a/scripts/evalFaceRecognition-MEDS.sh b/scripts/evalFaceRecognition-MEDS.sh index 673f674..231370f 100755 --- a/scripts/evalFaceRecognition-MEDS.sh +++ b/scripts/evalFaceRecognition-MEDS.sh @@ -20,11 +20,11 @@ if [ ! -e Algorithm_Dataset ]; then fi if [ ! -e MEDS.mask ]; then - br -useGui 0 -makeMask ../data/MEDS/sigset/MEDS_frontal_target.xml ../data/MEDS/sigset/MEDS_frontal_query.xml MEDS.mask + br -makeMask ../data/MEDS/sigset/MEDS_frontal_target.xml ../data/MEDS/sigset/MEDS_frontal_query.xml MEDS.mask fi # Run Algorithm on MEDS -br -useGui 0 -algorithm ${ALGORITHM} -path ../data/MEDS/img -compare ../data/MEDS/sigset/MEDS_frontal_target.xml ../data/MEDS/sigset/MEDS_frontal_query.xml ${ALGORITHM}_MEDS.mtx -eval ${ALGORITHM}_MEDS.mtx MEDS.mask Algorithm_Dataset/${ALGORITHM}_MEDS.csv +br -algorithm ${ALGORITHM} -path ../data/MEDS/img -compare ../data/MEDS/sigset/MEDS_frontal_target.xml ../data/MEDS/sigset/MEDS_frontal_query.xml ${ALGORITHM}_MEDS.mtx -eval ${ALGORITHM}_MEDS.mtx MEDS.mask Algorithm_Dataset/${ALGORITHM}_MEDS.csv # Plot results -br -useGui 0 -plot Algorithm_Dataset/*_MEDS.csv MEDS +br -plot Algorithm_Dataset/*_MEDS.csv MEDS diff --git a/scripts/evalGenderClassification-PCSO.sh b/scripts/evalGenderClassification-PCSO.sh index d8afbbb..b68315e 100755 --- a/scripts/evalGenderClassification-PCSO.sh +++ b/scripts/evalGenderClassification-PCSO.sh @@ -9,7 +9,7 @@ export ALGORITHM=GenderClassification export PCSO_DIR=../data/PCSO/img # Create a file list by querying the database -$BR -useGui 0 -quiet -algorithm Identity -enroll "$PCSO_DIR/PCSO.db[query='SELECT File,Gender,PersonID FROM PCSO', subset=1:8000]" terminal.txt > Input.txt +$BR -quiet -algorithm Identity -enroll "$PCSO_DIR/PCSO.db[query='SELECT File,Gender,PersonID FROM PCSO', subset=1:8000]" terminal.txt > Input.txt # Enroll the file list and evaluate performance -$BR -useGui 0 -algorithm $ALGORITHM -path $PCSO_DIR -enroll Input.txt Output.txt -evalClassification Output.txt Input.txt Gender \ No newline at end of file +$BR -algorithm $ALGORITHM -path $PCSO_DIR -enroll Input.txt Output.txt -evalClassification Output.txt Input.txt Gender \ No newline at end of file diff --git a/scripts/pedestrianBaselineLBP.sh b/scripts/pedestrianBaselineLBP.sh index 1ecbffe..2e02790 100755 --- a/scripts/pedestrianBaselineLBP.sh +++ b/scripts/pedestrianBaselineLBP.sh @@ -27,8 +27,7 @@ else TEST=testSmall.xml fi -br -useGui 0 \ - -algorithm "${ALG}" \ +br -algorithm "${ALG}" \ -path $INRIA_PATH/img \ -train $INRIA_PATH/sigset/train.xml pedModel \ -enroll $INRIA_PATH/sigset/$TEST pedResults.xml diff --git a/scripts/trainAgeRegression-PCSO.sh b/scripts/trainAgeRegression-PCSO.sh index c1f3ef1..4c30f10 100755 --- a/scripts/trainAgeRegression-PCSO.sh +++ b/scripts/trainAgeRegression-PCSO.sh @@ -13,4 +13,4 @@ export ageAlg=AgeRegression export PCSO_DIR=/user/pripshare/Databases/FaceDatabases/PCSO/PCSO/ -$BR -useGui 0 -algorithm $ageAlg -path $PCSO_DIR/Images -train "$PCSO_DIR/PCSO.db[query='SELECT File,Age,PersonID FROM PCSO WHERE Age >= 17 AND AGE <= 68', subset=0:200]" ../share/openbr/models/algorithms/AgeRegression +$BR -algorithm $ageAlg -path $PCSO_DIR/Images -train "$PCSO_DIR/PCSO.db[query='SELECT File,Age,PersonID FROM PCSO WHERE Age >= 17 AND AGE <= 68', subset=0:200]" ../share/openbr/models/algorithms/AgeRegression diff --git a/scripts/trainFaceRecognition-PCSO.sh b/scripts/trainFaceRecognition-PCSO.sh index ebeb056..1ef1153 100755 --- a/scripts/trainFaceRecognition-PCSO.sh +++ b/scripts/trainFaceRecognition-PCSO.sh @@ -14,7 +14,5 @@ export BR=../build/app/br/br export PCSO_DIR=/user/pripshare/Databases/FaceDatabases/PCSO/PCSO/ - - -$BR -useGui 0 -algorithm FaceRecognition -path "$PCSO_DIR/Images/" -train "$PCSO_DIR/PCSO.db[query='SELECT File,PersonID as Label,PersonID FROM PCSO', subset=0:5:6000]" ../share/openbr/models/algorithms/FaceRecognition +$BR -algorithm FaceRecognition -path "$PCSO_DIR/Images/" -train "$PCSO_DIR/PCSO.db[query='SELECT File,PersonID as Label,PersonID FROM PCSO', subset=0:5:6000]" ../share/openbr/models/algorithms/FaceRecognition diff --git a/scripts/trainGenderClassification-PCSO.sh b/scripts/trainGenderClassification-PCSO.sh index acb8146..8875fc9 100755 --- a/scripts/trainGenderClassification-PCSO.sh +++ b/scripts/trainGenderClassification-PCSO.sh @@ -13,4 +13,4 @@ export genderAlg=GenderClassification export PCSO_DIR=/user/pripshare/Databases/FaceDatabases/PCSO/PCSO/ -$BR -useGui 0 -algorithm $genderAlg -path $PCSO_DIR/Images -train "$PCSO_DIR/PCSO.db[query='SELECT File,Gender,PersonID FROM PCSO', subset=0:8000]" ../share/openbr/models/algorithms/GenderClassification +$BR -algorithm $genderAlg -path $PCSO_DIR/Images -train "$PCSO_DIR/PCSO.db[query='SELECT File,Gender,PersonID FROM PCSO', subset=0:8000]" ../share/openbr/models/algorithms/GenderClassification diff --git a/scripts/writeCaltechPedestrianSigset.sh b/scripts/writeCaltechPedestrianSigset.sh new file mode 100755 index 0000000..be8387e --- /dev/null +++ b/scripts/writeCaltechPedestrianSigset.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +echo '' +echo '' +for ((set=$1; set <= $2; set++)); do + echo -e "\t" + for vid in `printf "set%02d/*.seq" $set`; do + if [ $3 == "train" ]; then + vbb=`echo "vbb=\"annotations/$vid\"" | sed s/seq/vbb/` + fi + printf "\t\t\n" + done + echo -e "\t" +done +echo '' diff --git a/scripts/writeKTHSigset.sh b/scripts/writeKTHSigset.sh old mode 100644 new mode 100755 index 5c2abd0..5c2abd0 --- a/scripts/writeKTHSigset.sh +++ b/scripts/writeKTHSigset.sh diff --git a/share/openbr/cmake/Findcvmatio.cmake b/share/openbr/cmake/Findcvmatio.cmake new file mode 100644 index 0000000..a4f68ae --- /dev/null +++ b/share/openbr/cmake/Findcvmatio.cmake @@ -0,0 +1,9 @@ +set(CVMATIO_DIR "${BR_THIRDPARTY_DIR}/cvmatio") +if(NOT EXISTS ${CVMATIO_DIR}) + # download source from github + execute_process(COMMAND "git" "clone" "https://github.com/hbristow/cvmatio.git" WORKING_DIRECTORY ${BR_THIRDPARTY_DIR}) +else() + # update the source + execute_process(COMMAND "git" "pull" WORKING_DIRECTORY ${CVMATIO_DIR}) +endif() +set(CVMATIO_LIB_DIR ${CMAKE_BINARY_DIR}/3rdparty/cvmatio/src) diff --git a/share/openbr/cmake/OpenBRConfig.cmake b/share/openbr/cmake/OpenBRConfig.cmake index f844cfe..0952de3 100644 --- a/share/openbr/cmake/OpenBRConfig.cmake +++ b/share/openbr/cmake/OpenBRConfig.cmake @@ -8,6 +8,7 @@ # target_link_libraries(MY_TARGET_NAME ${OPENBR_LIBS}) # ================================================================ +find_path(OPENBR_DIR include/openbr/openbr.h) include_directories(${OPENBR_DIR}/include) link_directories(${OPENBR_DIR}/lib) set(OPENBR_LIBS "openbr")