Peter M. Groen / openbr

Commit 02f50481a426be971bcd172209cf99f81785cd9e

Authored by Cheney
2 parents 57118ef4 94f189ad

Merge branch 'pp4' of https://github.com/biometrics/openbr

Inline Side-by-side
Showing 3 changed files with 396 additions and 0 deletions
openbr/plugins/pp4.cmake 0 → 100644
  View file @02f5048
  1 +set(BR_WITH_PP4 ON CACHE BOOL "Build with PittPatt 4")
  2 +
  3 +if(${BR_WITH_PP4})
  4 + find_package(PP4 REQUIRED)
  5 + set(BR_THIRDPARTY_SRC ${BR_THIRDPARTY_SRC} plugins/pp4.cpp)
  6 + set(BR_THIRDPARTY_LIBS ${BR_THIRDPARTY_LIBS} ${PP4_LIBS})
  7 + install(DIRECTORY ${PP4_DIR}/lib/ DESTINATION lib)
  8 + install(DIRECTORY ${PP4_DIR}/models/ DESTINATION models/pp4)
  9 +endif()
openbr/plugins/pp4.cpp 0 → 100644
  View file @02f5048
  1 +#include <QThreadPool>
  2 +#include <QMap>
  3 +#include <QVariant>
  4 +#include <pittpatt_errors.h>
  5 +#include <pittpatt_nc_sdk.h>
  6 +#include <pittpatt_raw_image_io.h>
  7 +#include <pittpatt_license.h>
  8 +#include <openbr/openbr_plugin.h>
  9 +#include "openbr_internal.h"
  10 +#include "openbr/core/resource.h"
  11 +
  12 +#define TRY(CC) \
  13 +{ \
  14 + if ((CC) != PPR_SUCCESS) qFatal("%d error (%s, %d): %s.", CC, __FILE__, __LINE__, ppr_error_message(CC)); \
  15 +}
  16 +
  17 +#define TRY_VIDEO(CC) \
  18 +{ \
  19 + if ((CC) != PPR_VIDEO_IO_SUCCESS) qFatal("%d error (%s, %d): %s.", CC, __FILE__, __LINE__, ppr_video_io_error_message(CC)); \
  20 +}
  21 +
  22 +#define TRY_RAW_IMAGE(CC) \
  23 +{ \
  24 + if ((CC) != PPR_RAW_IMAGE_SUCCESS) qFatal("%d error (%s, %d): %s.", CC, __FILE__, __LINE__, ppr_raw_image_error_message(CC)); \
  25 +}
  26 +
  27 +using namespace br;
  28 +
  29 +/*!
  30 + * \ingroup initializers
  31 + * \brief Initialize PittPatt 4
  32 + * \author Josh Klontz \cite jklontz
  33 + * \warning Needs a maintainer.
  34 + */
  35 +class PP4Initializer : public Initializer
  36 +{
  37 + Q_OBJECT
  38 +
  39 + void initialize() const
  40 + {
  41 + Globals->abbreviations.insert("PP4", "Open+PP4Enroll:PP4Compare");
  42 + }
  43 +
  44 + void finalize() const
  45 + {
  46 + ppr_finalize_sdk();
  47 + }
  48 +};
  49 +
  50 +BR_REGISTER(Initializer, PP4Initializer)
  51 +
  52 +/*!
  53 + * \brief PittPatt 4 context
  54 + * \author Josh Klontz \cite jklontz
  55 + * \warning Needs a maintainer.
  56 + */
  57 +struct PP4Context
  58 +{
  59 + ppr_context_type context;
  60 +
  61 + PP4Context()
  62 + {
  63 + context = ppr_get_context();
  64 + TRY(ppr_enable_recognition(context))
  65 + TRY(ppr_set_license(context, my_license_id, my_license_key))
  66 + TRY(ppr_set_models_path(context, qPrintable(Globals->sdkPath + "/models/pp4")))
  67 + TRY(ppr_set_num_recognition_threads(context, QThreadPool::globalInstance()->maxThreadCount()))
  68 + TRY(ppr_set_num_detection_threads(context, 1))
  69 + TRY(ppr_set_detection_precision(context, PPR_FINE_PRECISION))
  70 + TRY(ppr_set_landmark_detector_type(context, PPR_DUAL_MULTI_POSE_LANDMARK_DETECTOR, PPR_AUTOMATIC_LANDMARKS))
  71 + TRY(ppr_set_min_size(context, 4))
  72 + TRY(ppr_set_frontal_yaw_constraint(context, PPR_FRONTAL_YAW_CONSTRAINT_PERMISSIVE))
  73 + TRY(ppr_set_template_extraction_type(context, PPR_EXTRACT_DOUBLE))
  74 + TRY(ppr_initialize_context(context))
  75 + }
  76 +
  77 + ~PP4Context()
  78 + {
  79 + TRY(ppr_release_context(context))
  80 + }
  81 +
  82 + static void createRawImage(const cv::Mat &src, ppr_raw_image_type &dst)
  83 + {
  84 + ppr_raw_image_create(&dst, src.cols, src.rows, PPR_RAW_IMAGE_BGR24);
  85 + assert((src.type() == CV_8UC3) && src.isContinuous());
  86 + memcpy(dst.data, src.data, 3*src.rows*src.cols);
  87 + }
  88 +
  89 + void createMat(const ppr_template_type &src, cv::Mat &dst) const
  90 + {
  91 + ppr_flat_template_type flat_template;
  92 + TRY(ppr_flatten_template(context,src,&flat_template))
  93 + dst = cv::Mat(1, flat_template.num_bytes, CV_8UC1, flat_template.data).clone();
  94 + ppr_free_flat_template(flat_template);
  95 + }
  96 +
  97 + void createTemplate(const cv::Mat &src, ppr_template_type *dst) const
  98 + {
  99 + ppr_flat_template_type flat_template;
  100 + flat_template.num_bytes = src.cols;
  101 + flat_template.data = src.data;
  102 + TRY(ppr_unflatten_template(context, flat_template, dst))
  103 + }
  104 +
  105 + static QString toString(const ppr_landmark_category_type &category)
  106 + {
  107 + switch (category) {
  108 + case PPR_LANDMARK_LEFT_EYE:
  109 + return "Left_Eye";
  110 + case PPR_LANDMARK_RIGHT_EYE:
  111 + return "Right_Eye";
  112 + case PPR_LANDMARK_NOSE_BASE:
  113 + return "Nose_Base";
  114 + case PPR_LANDMARK_NOSE_BRIDGE:
  115 + return "Nose_Bridge";
  116 + case PPR_LANDMARK_NOSE_TIP:
  117 + return "Nose_Tip";
  118 + case PPR_LANDMARK_NOSE_TOP:
  119 + return "Nose_Top";
  120 + case PPR_LANDMARK_EYE_NOSE:
  121 + return "Eye_Nose";
  122 + case PPR_LANDMARK_MOUTH:
  123 + return "Mouth";
  124 + }
  125 +
  126 + return "Unknown";
  127 + }
  128 +
  129 + static QMap<QString,QVariant> toMetadata(const ppr_object_type &object)
  130 + {
  131 + QMap<QString,QVariant> metadata;
  132 +
  133 + metadata.insert("FrontalFace", QRectF(object.position.x - object.dimensions.width/2,
  134 + object.position.y - object.dimensions.height/2,
  135 + object.dimensions.width,
  136 + object.dimensions.height));
  137 + metadata.insert("Confidence", object.confidence);
  138 + metadata.insert("PP4_Object_X", object.position.x - object.dimensions.width/2);
  139 + metadata.insert("PP4_Object_Y", object.position.y - object.dimensions.height/2);
  140 + metadata.insert("PP4_Object_Width", object.dimensions.width);
  141 + metadata.insert("PP4_Object_Height", object.dimensions.height);
  142 + metadata.insert("PP4_Object_Roll", object.rotation.roll);
  143 + metadata.insert("PP4_Object_Pitch", object.rotation.pitch);
  144 + metadata.insert("PP4_Object_Yaw", object.rotation.yaw);
  145 + metadata.insert("PP4_Object_Precision", object.rotation.precision);
  146 + metadata.insert("PP4_Object_ModelID", object.model_id);
  147 + metadata.insert("PP4_Object_NumLandmarks", object.num_landmarks);
  148 + metadata.insert("PP4_Object_Size", object.size);
  149 +
  150 + QList<ppr_landmark_category_type> categories;
  151 + categories << PPR_LANDMARK_RIGHT_EYE
  152 + << PPR_LANDMARK_LEFT_EYE
  153 + << PPR_LANDMARK_NOSE_BASE
  154 + << PPR_LANDMARK_NOSE_BRIDGE
  155 + << PPR_LANDMARK_NOSE_TIP
  156 + << PPR_LANDMARK_NOSE_TOP
  157 + << PPR_LANDMARK_EYE_NOSE
  158 + << PPR_LANDMARK_MOUTH;
  159 +
  160 + for (int i=0; i<categories.size(); i++) {
  161 + ppr_landmark_category_type category = categories[i];
  162 + QString metadataString = QString("PP4_Landmark%1_%2").arg(QString::number(i), toString(category));
  163 +
  164 + bool found = false;
  165 + for (int j=0; j<object.num_landmarks; j++) {
  166 + ppr_landmark_type &landmark = object.landmarks[j];
  167 + if (landmark.category != category) continue;
  168 +
  169 + metadata.insert(metadataString+"_X", landmark.position.x);
  170 + metadata.insert(metadataString+"_Y", landmark.position.y);
  171 + metadata.insert(metadataString+"_Category", landmark.category);
  172 + metadata.insert(metadataString+"_ModelID", landmark.model_id);
  173 + metadata.insert(metadataString+"_Index", j);
  174 + found = true;
  175 + break;
  176 + }
  177 +
  178 + if (!found) {
  179 + metadata.insert(metadataString+"_X", -1);
  180 + metadata.insert(metadataString+"_Y", -1);
  181 + metadata.insert(metadataString+"_Category", -1);
  182 + metadata.insert(metadataString+"_ModelID", -1);
  183 + metadata.insert(metadataString+"_Index", -1);
  184 + }
  185 + }
  186 +
  187 + return metadata;
  188 + }
  189 +
  190 + static void freeObject(ppr_object_type &object)
  191 + {
  192 + delete[] object.landmarks;
  193 + object.landmarks = NULL;
  194 + object.num_landmarks = 0;
  195 + }
  196 +};
  197 +
  198 +/*!
  199 + * \ingroup transforms
  200 + * \brief Enroll faces in PittPatt 4
  201 + * \author Josh Klontz \cite jklontz
  202 + * \warning Needs a maintainer.
  203 + */
  204 +class PP4EnrollTransform : public UntrainableMetaTransform
  205 +{
  206 + Q_OBJECT
  207 + Q_PROPERTY(bool detectOnly READ get_detectOnly WRITE set_detectOnly RESET reset_detectOnly STORED false)
  208 + BR_PROPERTY(bool, detectOnly, false)
  209 + Resource<PP4Context> contexts;
  210 +
  211 + void project(const Template &src, Template &dst) const
  212 + {
  213 + if (Globals->enrollAll)
  214 + qFatal("single template project doesn't support enrollAll");
  215 +
  216 + TemplateList srcList;
  217 + srcList.append(src);
  218 + TemplateList dstList;
  219 + project(srcList, dstList);
  220 + dst = dstList.first();
  221 + }
  222 +
  223 + void project(const TemplateList &srcList, TemplateList &dstList) const
  224 + {
  225 + if (srcList.empty())
  226 + return;
  227 +
  228 + PP4Context *context = contexts.acquire();
  229 +
  230 + foreach(const Template &src, srcList) {
  231 + if (!src.isEmpty()) {
  232 + ppr_raw_image_type raw_image;
  233 + PP4Context::createRawImage(src, raw_image);
  234 + ppr_image_type image;
  235 + TRY(ppr_create_image(raw_image, &image))
  236 + ppr_object_list_type object_list;
  237 + TRY(ppr_detect_objects(context->context, image, &object_list))
  238 +
  239 + QList<ppr_object_type> objects;
  240 + if (Globals->enrollAll) objects = getAllObjects(object_list);
  241 + else objects = getBestObject(context, object_list);
  242 +
  243 + foreach (const ppr_object_type &object, objects) {
  244 + ppr_object_suitability_type suitability;
  245 + TRY(ppr_is_object_suitable_for_recognition(context->context, object, &suitability))
  246 + if (suitability != PPR_OBJECT_SUITABLE_FOR_RECOGNITION && !detectOnly) continue;
  247 +
  248 + cv::Mat m;
  249 + if (detectOnly)
  250 + m = src;
  251 + else {
  252 + ppr_template_type curr_template;
  253 + TRY(ppr_extract_template_from_object(context->context, image, object, &curr_template))
  254 + context->createMat(curr_template, m);
  255 + }
  256 +
  257 + Template dst;
  258 + dst.file = src.file;
  259 +
  260 + dst.file.append(PP4Context::toMetadata(object));
  261 + dst += m;
  262 + dstList.append(dst);
  263 +
  264 + if (!Globals->enrollAll)
  265 + break;
  266 + }
  267 +
  268 + ppr_free_object_list(object_list);
  269 + ppr_free_image(image);
  270 + ppr_raw_image_free(raw_image);
  271 + }
  272 +
  273 + if (!Globals->enrollAll && dstList.empty()) {
  274 + dstList.append(Template(src.file, detectOnly ? src.m() : cv::Mat()));
  275 + dstList.last().file.fte = true;
  276 + }
  277 + }
  278 +
  279 + contexts.release(context);
  280 + }
  281 +
  282 +private:
  283 + QList<ppr_object_type> getBestObject(PP4Context *context, ppr_object_list_type object_list) const
  284 + {
  285 + int best_index = -1;
  286 + float best_confidence = 0;
  287 + for (int i=0; i<object_list.num_objects; i++) {
  288 + ppr_object_type object = object_list.objects[i];
  289 + ppr_object_suitability_type suitability;
  290 + TRY(ppr_is_object_suitable_for_recognition(context->context, object, &suitability))
  291 + if (suitability != PPR_OBJECT_SUITABLE_FOR_RECOGNITION) continue;
  292 + if ((object.confidence > best_confidence) ||
  293 + (best_index == -1)) {
  294 + best_confidence = object.confidence;
  295 + best_index = i;
  296 + }
  297 + }
  298 +
  299 + QList<ppr_object_type> objects;
  300 + if (best_index != -1) objects.append(object_list.objects[best_index]);
  301 + return objects;
  302 + }
  303 +
  304 + QList<ppr_object_type> getAllObjects(ppr_object_list_type object_list) const
  305 + {
  306 + QList<ppr_object_type> objects;
  307 + for (int i=0; i<object_list.num_objects; i++)
  308 + objects.append(object_list.objects[i]);
  309 + return objects;
  310 + }
  311 +};
  312 +
  313 +BR_REGISTER(Transform, PP4EnrollTransform)
  314 +
  315 +
  316 +class PP4Compare : public Distance,
  317 + public PP4Context
  318 +{
  319 + Q_OBJECT
  320 +
  321 + void compare(const TemplateList &target, const TemplateList &query, Output *output) const
  322 + {
  323 + ppr_gallery_type target_gallery, query_gallery;
  324 + ppr_create_gallery(context, &target_gallery);
  325 + ppr_create_gallery(context, &query_gallery);
  326 + QList<int> target_template_ids, query_template_ids;
  327 + enroll(target, &target_gallery, target_template_ids);
  328 + enroll(query, &query_gallery, query_template_ids);
  329 +
  330 + ppr_similarity_matrix_type similarity_matrix;
  331 + TRY(ppr_compare_galleries(context, query_gallery, target_gallery, &similarity_matrix))
  332 +
  333 + for (int i=0; i<query_template_ids.size(); i++) {
  334 + int query_template_id = query_template_ids[i];
  335 + for (int j=0; j<target_template_ids.size(); j++) {
  336 + int target_template_id = target_template_ids[j];
  337 + float score = -std::numeric_limits<float>::max();
  338 + if ((query_template_id != -1) && (target_template_id != -1)) {
  339 + TRY(ppr_get_similarity_matrix_element(context, similarity_matrix, query_template_id, target_template_id, &score))
  340 + }
  341 + output->setRelative(score, i, j);
  342 + }
  343 + }
  344 +
  345 + ppr_free_similarity_matrix(similarity_matrix);
  346 + ppr_free_gallery(target_gallery);
  347 + ppr_free_gallery(query_gallery);
  348 + }
  349 +
  350 + void enroll(const TemplateList &templates, ppr_gallery_type *gallery, QList<int> &template_ids) const
  351 + {
  352 + foreach (const Template &t, templates) {
  353 + if (t.m().data) {
  354 + ppr_template_type u;
  355 + createTemplate(t.m(), &u);
  356 + int template_id;
  357 + TRY(ppr_copy_template_to_gallery(context, gallery, u, &template_id))
  358 + template_ids.append(template_id);
  359 + ppr_free_template(u);
  360 + } else {
  361 + template_ids.append(-1);
  362 + }
  363 + }
  364 + }
  365 +};
  366 +
  367 +BR_REGISTER(Distance, PP4Compare)
  368 +
  369 +#include "plugins/pp4.moc"
share/openbr/cmake/FindPP4.cmake 0 → 100644
  View file @02f5048
  1 +# ================================================================
  2 +# The PP4 CMake configuration file
  3 +#
  4 +# Usage from an external project:
  5 +# In your CMakeLists.txt, add these lines:
  6 +#
  7 +# find_package(PP4 REQUIRED)
  8 +# target_link_libraries(MY_TARGET ${PP4_LIBS})
  9 +# ================================================================
  10 +
  11 +find_path(PP4_DIR include/pittpatt_nc_sdk.h ${CMAKE_SOURCE_DIR}/3rdparty/*)
  12 +include_directories(${PP4_DIR}/include)
  13 +link_directories(${PP4_DIR}/lib)
  14 +set(PP4_LIBS pittpatt_nc_sdk
  15 + pittpatt_raw_image
  16 + pittpatt_raw_image_io
  17 + pittpatt_recognition_core
  18 + pittpatt_video_io)