diff --git a/openbr/plugins/pp4.cmake b/openbr/plugins/pp4.cmake new file mode 100644 index 0000000..a285e86 --- /dev/null +++ b/openbr/plugins/pp4.cmake @@ -0,0 +1,9 @@ +set(BR_WITH_PP4 ON CACHE BOOL "Build with PittPatt 4") + +if(${BR_WITH_PP4}) + find_package(PP4 REQUIRED) + set(BR_THIRDPARTY_SRC ${BR_THIRDPARTY_SRC} plugins/pp4.cpp) + set(BR_THIRDPARTY_LIBS ${BR_THIRDPARTY_LIBS} ${PP4_LIBS}) + install(DIRECTORY ${PP4_DIR}/lib/ DESTINATION lib) + install(DIRECTORY ${PP4_DIR}/models/ DESTINATION models/pp4) +endif() diff --git a/openbr/plugins/pp4.cpp b/openbr/plugins/pp4.cpp new file mode 100644 index 0000000..69bac4b --- /dev/null +++ b/openbr/plugins/pp4.cpp @@ -0,0 +1,369 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include "openbr_internal.h" +#include "openbr/core/resource.h" + +#define TRY(CC) \ +{ \ + if ((CC) != PPR_SUCCESS) qFatal("%d error (%s, %d): %s.", CC, __FILE__, __LINE__, ppr_error_message(CC)); \ +} + +#define TRY_VIDEO(CC) \ +{ \ + if ((CC) != PPR_VIDEO_IO_SUCCESS) qFatal("%d error (%s, %d): %s.", CC, __FILE__, __LINE__, ppr_video_io_error_message(CC)); \ +} + +#define TRY_RAW_IMAGE(CC) \ +{ \ + if ((CC) != PPR_RAW_IMAGE_SUCCESS) qFatal("%d error (%s, %d): %s.", CC, __FILE__, __LINE__, ppr_raw_image_error_message(CC)); \ +} + +using namespace br; + +/*! + * \ingroup initializers + * \brief Initialize PittPatt 4 + * \author Josh Klontz \cite jklontz + * \warning Needs a maintainer. + */ +class PP4Initializer : public Initializer +{ + Q_OBJECT + + void initialize() const + { + Globals->abbreviations.insert("PP4", "Open+PP4Enroll:PP4Compare"); + } + + void finalize() const + { + ppr_finalize_sdk(); + } +}; + +BR_REGISTER(Initializer, PP4Initializer) + +/*! + * \brief PittPatt 4 context + * \author Josh Klontz \cite jklontz + * \warning Needs a maintainer. + */ +struct PP4Context +{ + ppr_context_type context; + + PP4Context() + { + context = ppr_get_context(); + TRY(ppr_enable_recognition(context)) + TRY(ppr_set_license(context, my_license_id, my_license_key)) + TRY(ppr_set_models_path(context, qPrintable(Globals->sdkPath + "/models/pp4"))) + TRY(ppr_set_num_recognition_threads(context, QThreadPool::globalInstance()->maxThreadCount())) + TRY(ppr_set_num_detection_threads(context, 1)) + TRY(ppr_set_detection_precision(context, PPR_FINE_PRECISION)) + TRY(ppr_set_landmark_detector_type(context, PPR_DUAL_MULTI_POSE_LANDMARK_DETECTOR, PPR_AUTOMATIC_LANDMARKS)) + TRY(ppr_set_min_size(context, 4)) + TRY(ppr_set_frontal_yaw_constraint(context, PPR_FRONTAL_YAW_CONSTRAINT_PERMISSIVE)) + TRY(ppr_set_template_extraction_type(context, PPR_EXTRACT_DOUBLE)) + TRY(ppr_initialize_context(context)) + } + + ~PP4Context() + { + TRY(ppr_release_context(context)) + } + + static void createRawImage(const cv::Mat &src, ppr_raw_image_type &dst) + { + ppr_raw_image_create(&dst, src.cols, src.rows, PPR_RAW_IMAGE_BGR24); + assert((src.type() == CV_8UC3) && src.isContinuous()); + memcpy(dst.data, src.data, 3*src.rows*src.cols); + } + + void createMat(const ppr_template_type &src, cv::Mat &dst) const + { + ppr_flat_template_type flat_template; + TRY(ppr_flatten_template(context,src,&flat_template)) + dst = cv::Mat(1, flat_template.num_bytes, CV_8UC1, flat_template.data).clone(); + ppr_free_flat_template(flat_template); + } + + void createTemplate(const cv::Mat &src, ppr_template_type *dst) const + { + ppr_flat_template_type flat_template; + flat_template.num_bytes = src.cols; + flat_template.data = src.data; + TRY(ppr_unflatten_template(context, flat_template, dst)) + } + + static QString toString(const ppr_landmark_category_type &category) + { + switch (category) { + case PPR_LANDMARK_LEFT_EYE: + return "Left_Eye"; + case PPR_LANDMARK_RIGHT_EYE: + return "Right_Eye"; + case PPR_LANDMARK_NOSE_BASE: + return "Nose_Base"; + case PPR_LANDMARK_NOSE_BRIDGE: + return "Nose_Bridge"; + case PPR_LANDMARK_NOSE_TIP: + return "Nose_Tip"; + case PPR_LANDMARK_NOSE_TOP: + return "Nose_Top"; + case PPR_LANDMARK_EYE_NOSE: + return "Eye_Nose"; + case PPR_LANDMARK_MOUTH: + return "Mouth"; + } + + return "Unknown"; + } + + static QMap toMetadata(const ppr_object_type &object) + { + QMap metadata; + + metadata.insert("FrontalFace", QRectF(object.position.x - object.dimensions.width/2, + object.position.y - object.dimensions.height/2, + object.dimensions.width, + object.dimensions.height)); + metadata.insert("Confidence", object.confidence); + metadata.insert("PP4_Object_X", object.position.x - object.dimensions.width/2); + metadata.insert("PP4_Object_Y", object.position.y - object.dimensions.height/2); + metadata.insert("PP4_Object_Width", object.dimensions.width); + metadata.insert("PP4_Object_Height", object.dimensions.height); + metadata.insert("PP4_Object_Roll", object.rotation.roll); + metadata.insert("PP4_Object_Pitch", object.rotation.pitch); + metadata.insert("PP4_Object_Yaw", object.rotation.yaw); + metadata.insert("PP4_Object_Precision", object.rotation.precision); + metadata.insert("PP4_Object_ModelID", object.model_id); + metadata.insert("PP4_Object_NumLandmarks", object.num_landmarks); + metadata.insert("PP4_Object_Size", object.size); + + QList categories; + categories << PPR_LANDMARK_RIGHT_EYE + << PPR_LANDMARK_LEFT_EYE + << PPR_LANDMARK_NOSE_BASE + << PPR_LANDMARK_NOSE_BRIDGE + << PPR_LANDMARK_NOSE_TIP + << PPR_LANDMARK_NOSE_TOP + << PPR_LANDMARK_EYE_NOSE + << PPR_LANDMARK_MOUTH; + + for (int i=0; i contexts; + + void project(const Template &src, Template &dst) const + { + if (Globals->enrollAll) + qFatal("single template project doesn't support enrollAll"); + + TemplateList srcList; + srcList.append(src); + TemplateList dstList; + project(srcList, dstList); + dst = dstList.first(); + } + + void project(const TemplateList &srcList, TemplateList &dstList) const + { + if (srcList.empty()) + return; + + PP4Context *context = contexts.acquire(); + + foreach(const Template &src, srcList) { + if (!src.isEmpty()) { + ppr_raw_image_type raw_image; + PP4Context::createRawImage(src, raw_image); + ppr_image_type image; + TRY(ppr_create_image(raw_image, &image)) + ppr_object_list_type object_list; + TRY(ppr_detect_objects(context->context, image, &object_list)) + + QList objects; + if (Globals->enrollAll) objects = getAllObjects(object_list); + else objects = getBestObject(context, object_list); + + foreach (const ppr_object_type &object, objects) { + ppr_object_suitability_type suitability; + TRY(ppr_is_object_suitable_for_recognition(context->context, object, &suitability)) + if (suitability != PPR_OBJECT_SUITABLE_FOR_RECOGNITION && !detectOnly) continue; + + cv::Mat m; + if (detectOnly) + m = src; + else { + ppr_template_type curr_template; + TRY(ppr_extract_template_from_object(context->context, image, object, &curr_template)) + context->createMat(curr_template, m); + } + + Template dst; + dst.file = src.file; + + dst.file.append(PP4Context::toMetadata(object)); + dst += m; + dstList.append(dst); + + if (!Globals->enrollAll) + break; + } + + ppr_free_object_list(object_list); + ppr_free_image(image); + ppr_raw_image_free(raw_image); + } + + if (!Globals->enrollAll && dstList.empty()) { + dstList.append(Template(src.file, detectOnly ? src.m() : cv::Mat())); + dstList.last().file.fte = true; + } + } + + contexts.release(context); + } + +private: + QList getBestObject(PP4Context *context, ppr_object_list_type object_list) const + { + int best_index = -1; + float best_confidence = 0; + for (int i=0; icontext, object, &suitability)) + if (suitability != PPR_OBJECT_SUITABLE_FOR_RECOGNITION) continue; + if ((object.confidence > best_confidence) || + (best_index == -1)) { + best_confidence = object.confidence; + best_index = i; + } + } + + QList objects; + if (best_index != -1) objects.append(object_list.objects[best_index]); + return objects; + } + + QList getAllObjects(ppr_object_list_type object_list) const + { + QList objects; + for (int i=0; i target_template_ids, query_template_ids; + enroll(target, &target_gallery, target_template_ids); + enroll(query, &query_gallery, query_template_ids); + + ppr_similarity_matrix_type similarity_matrix; + TRY(ppr_compare_galleries(context, query_gallery, target_gallery, &similarity_matrix)) + + for (int i=0; i::max(); + if ((query_template_id != -1) && (target_template_id != -1)) { + TRY(ppr_get_similarity_matrix_element(context, similarity_matrix, query_template_id, target_template_id, &score)) + } + output->setRelative(score, i, j); + } + } + + ppr_free_similarity_matrix(similarity_matrix); + ppr_free_gallery(target_gallery); + ppr_free_gallery(query_gallery); + } + + void enroll(const TemplateList &templates, ppr_gallery_type *gallery, QList &template_ids) const + { + foreach (const Template &t, templates) { + if (t.m().data) { + ppr_template_type u; + createTemplate(t.m(), &u); + int template_id; + TRY(ppr_copy_template_to_gallery(context, gallery, u, &template_id)) + template_ids.append(template_id); + ppr_free_template(u); + } else { + template_ids.append(-1); + } + } + } +}; + +BR_REGISTER(Distance, PP4Compare) + +#include "plugins/pp4.moc" diff --git a/share/openbr/cmake/FindPP4.cmake b/share/openbr/cmake/FindPP4.cmake new file mode 100644 index 0000000..bea3d42 --- /dev/null +++ b/share/openbr/cmake/FindPP4.cmake @@ -0,0 +1,18 @@ +# ================================================================ +# The PP4 CMake configuration file +# +# Usage from an external project: +# In your CMakeLists.txt, add these lines: +# +# find_package(PP4 REQUIRED) +# target_link_libraries(MY_TARGET ${PP4_LIBS}) +# ================================================================ + +find_path(PP4_DIR include/pittpatt_nc_sdk.h ${CMAKE_SOURCE_DIR}/3rdparty/*) +include_directories(${PP4_DIR}/include) +link_directories(${PP4_DIR}/lib) +set(PP4_LIBS pittpatt_nc_sdk + pittpatt_raw_image + pittpatt_raw_image_io + pittpatt_recognition_core + pittpatt_video_io)