diff --git a/3rdparty/stasm/stasm/CMakeLists.txt b/3rdparty/stasm/stasm/CMakeLists.txt deleted file mode 100644 index 352c5c9..0000000 --- a/3rdparty/stasm/stasm/CMakeLists.txt +++ /dev/null @@ -1,30 +0,0 @@ -# Stasm CMakeLists - -aux_source_directory(stasm/src STASM_SRC) -include_directories(stasm/include) - -#aux_source_directory(tasm/src TASM_SRC) -#include_directories(tasm/include) - -add_library(stasm SHARED ${STASM_SRC} ${TASM_SRC}) -qt5_use_modules(stasm ${QT_DEPENDENCIES}) - -set_target_properties(stasm PROPERTIES - DEFINE_SYMBOL STASM_LIBRARY - VERSION ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR}.${CPACK_PACKAGE_VERSION_PATCH} - SOVERSION ${CPACK_PACKAGE_VERSION_MAJOR}.${CPACK_PACKAGE_VERSION_MINOR} - LINK_INTERFACE_LIBRARIES "" - AUTOMOC TRUE) - -target_link_libraries(stasm ${OpenCV_LIBS} ${Qt5Core_QTMAIN_LIBRARIES}) - -file(GLOB STASM_HEADERS stasm/include/*.h) -install(FILES ${STASM_HEADERS} DESTINATION include/stasm) -file(GLOB CLASSIC_HEADERS stasm/include/classic/*.mh) -install(FILES ${CLASSIC_HEADERS} DESTINATION include/stasm/classic) -file(GLOB HAT_HEADERS stasm/include/hat/*.mh) -install(FILES ${HAT_HEADERS} DESTINATION include/stasm/hat) -#file(GLOB TASM_HEADERS tasm/include/*.h) -#install(FILES ${TASM_HEADERS} DESTINATION include/tasm) - -install(TARGETS stasm RUNTIME DESTINATION bin LIBRARY DESTINATION lib ARCHIVE DESTINATION lib) diff --git a/3rdparty/stasm/stasm/stasm/src/print.cpp b/3rdparty/stasm/stasm/stasm/src/print.cpp deleted file mode 100755 index 7525d6b..0000000 --- a/3rdparty/stasm/stasm/stasm/src/print.cpp +++ /dev/null @@ -1,146 +0,0 @@ -// print.cpp: printing and logging utilities for the Stasm library -// -// Copyright (C) 2005-2013, Stephen Milborrow - -#include "print.h" -#include "err.h" -#include "misc.h" - -#include -#include -#include - -namespace stasm -{ -bool print_g; // true to allow output to stdout (but error msgs always printed) - -bool trace_g; // true to trace Stasm internal operation - -static FILE* logfile_g; // lprintfs go to this log file as well as stdout - -//----------------------------------------------------------------------------- - -// Open the log file. After this, when you call lprintf, you print to the log -// file (as well as to stdout). This inits the global variable logfile_g. - -void OpenLogFile( // also inits the global variable logfile_g - const char* path) // in: log file path, default is "stasm.log" -{ - if (!logfile_g) - { - if (print_g) - printf("Generating %s\n", path); - logfile_g = fopen(path, "wb"); - if (!logfile_g) - Err("Cannot open \"%s\"", path); - // check that we can write to the log file - if (fputs("log file\n", logfile_g) < 0) - Err("Cannot write to \"%s\"", path); - rewind(logfile_g); // rewind so above test msg is not in the log file - } -} - -// Like printf but only prints if print_g flag is set. -// Also prints to the log file if it is open (regardless of print_g). - -void lprintf(const char* format, ...) // args like printf -{ - char s[SBIG]; - va_list args; - va_start(args, format); - VSPRINTF(s, format, args); - va_end(args); - if (print_g) - { - printf("%s", s); - fflush(stdout); // flush so if there is a crash we can see what happened - } - if (logfile_g) - { - // we don't check fputs here, to prevent recursive calls and msgs - fputs(s, logfile_g); - fflush(logfile_g); - } -} - -// Like printf but prints to the log file only (and not to stdout). -// Used for detailed stuff that we don't usually want to see. - -void logprintf(const char* format, ...) // args like printf -{ - if (logfile_g) - { - char s[SBIG]; - va_list args; - va_start(args, format); - VSPRINTF(s, format, args); - va_end(args); - // we don't check fputs here, to prevent recursive calls and msgs - fputs(s, logfile_g); - fflush(logfile_g); - } -} - -// Like lprintf but always prints even if print_g is false. - -void lprintf_always(const char* format, ...) -{ - char s[SBIG]; - va_list args; - va_start(args, format); - VSPRINTF(s, format, args); - va_end(args); - printf("%s", s); - fflush(stdout); // flush so if there is a crash we can see what happened - if (logfile_g) - { - // we don't check fputs here, to prevent recursive calls and msgs - fputs(s, logfile_g); - fflush(logfile_g); - } -} - -// Like puts but prints to the log file as well if it is open, -// and does not append a newline. - -void lputs(const char* s) -{ - printf("%s", s); - fflush(stdout); // flush so if there is a crash we can see what happened - logprintf("%s", s); -} - -// Print message only once on the screen, and only 100 times to the log file. -// This is used when similar messages could be printed many times and it -// suffices to let the user know just once. By convention the message is -// printed followed by "..." so the user knows that just the first message -// was printed. The user can look in the log file for further messages if -// necessary (but we print only 100 times to the log file --- else all the -// log prints make tasm slow). - -void PrintOnce( - int& printed, // io: zero=print, nonzero=no print - const char* format, ...) // in: args like printf -{ - char s[SBIG]; - va_list args; - va_start(args, format); - VSPRINTF(s, format, args); - va_end(args); - if (printed == 0 && print_g) - { - printed = 1; - printf("%s", s); - fflush(stdout); // flush so if there is a crash we can see what happened - } - if (printed < 100 && logfile_g) - { - fputs(s, logfile_g); - fflush(logfile_g); - printed++; - if (printed == 100) - logprintf("no more prints of the above message (printed == 100)\n"); - } -} - -} // namespace stasm diff --git a/3rdparty/stasm4.0.0/stasm/MOD_1/facedet.cpp b/3rdparty/stasm4.0.0/stasm/MOD_1/facedet.cpp index a2e730a..ce92c57 100755 --- a/3rdparty/stasm4.0.0/stasm/MOD_1/facedet.cpp +++ b/3rdparty/stasm4.0.0/stasm/MOD_1/facedet.cpp @@ -47,28 +47,16 @@ void DetectFaces( // all face rects into detpars int minwidth, cv::CascadeClassifier cascade) // in: as percent of img width { - int leftborder = 0, topborder = 0; // border size in pixels - Image bordered_img(BORDER_FRAC == 0? - img: EnborderImg(leftborder, topborder, img)); - - // Detection results are very slightly better with equalization - // (tested on the MUCT images, which are not pre-equalized), and - // it's quick enough to equalize (roughly 10ms on a 1.6 GHz laptop). - - Image equalized_img; cv::equalizeHist(bordered_img, equalized_img); - CV_Assert(minwidth >= 1 && minwidth <= 100); - int minpix = MAX(100, cvRound(img.cols * minwidth / 100.)); - // the params below are accurate but slow - static const double SCALE_FACTOR = 1.1; - static const int MIN_NEIGHBORS = 3; + static const double SCALE_FACTOR = 1.2; + static const int MIN_NEIGHBORS = 5; static const int DETECTOR_FLAGS = 0; vec_Rect facerects = // all face rects in image - Detect(equalized_img, &cascade, NULL, - SCALE_FACTOR, MIN_NEIGHBORS, DETECTOR_FLAGS, minpix); + Detect(img, &cascade, NULL, + SCALE_FACTOR, MIN_NEIGHBORS, DETECTOR_FLAGS, 64); // copy face rects into the detpars vector @@ -80,8 +68,6 @@ void DetectFaces( // all face rects into detpars // detpar.x and detpar.y is the center of the face rectangle detpar.x = facerect->x + facerect->width / 2.; detpar.y = facerect->y + facerect->height / 2.; - detpar.x -= leftborder; // discount the border we added earlier - detpar.y -= topborder; detpar.width = double(facerect->width); detpar.height = double(facerect->height); detpar.yaw = 0; // assume face has no yaw in this version of Stasm @@ -187,10 +173,10 @@ void FaceDet::DetectFaces_( // call once per image to find all the faces //CV_Assert(!facedet_g.empty()); // check that OpenFaceDetector_ was called DetectFaces(detpars_, img, minwidth, cascade); - DiscardMissizedFaces(detpars_); + //DiscardMissizedFaces(detpars_); if (multiface) // order faces on increasing distance from left margin { - sort(detpars_.begin(), detpars_.end(), IncreasingLeftMargin); + sort(detpars_.begin(), detpars_.end(), DecreasingWidth); } else { diff --git a/3rdparty/stasm4.0.0/stasm/MOD_1/facedet.h b/3rdparty/stasm4.0.0/stasm/MOD_1/facedet.h index 101e250..487226a 100755 --- a/3rdparty/stasm4.0.0/stasm/MOD_1/facedet.h +++ b/3rdparty/stasm4.0.0/stasm/MOD_1/facedet.h @@ -38,13 +38,11 @@ public: FaceDet() {} // constructor - -private: - vector detpars_; // all the valid faces in the current image - int iface_; // index of current face for NextFace_ // indexes into detpars_ + vector detpars_; // all the valid faces in the current image +private: DISALLOW_COPY_AND_ASSIGN(FaceDet); }; // end class FaceDet diff --git a/3rdparty/stasm4.0.0/stasm/stasm_lib.cpp b/3rdparty/stasm4.0.0/stasm/stasm_lib.cpp index bfea3f9..891e30c 100755 --- a/3rdparty/stasm4.0.0/stasm/stasm_lib.cpp +++ b/3rdparty/stasm4.0.0/stasm/stasm_lib.cpp @@ -160,7 +160,8 @@ int stasm_search_auto_ext( // extended version of stasm_search_auto const char* data, const int width, const int height, - StasmCascadeClassifier cascade) + StasmCascadeClassifier cascade, + FaceDet &detection) { int returnval = 1; // assume success *foundface = 0; // but assume no face found @@ -176,15 +177,14 @@ int stasm_search_auto_ext( // extended version of stasm_search_auto // Allocate image Image img = Image(height, width,(unsigned char*)data); - FaceDet facedet; - // call the face detector to detect the face rectangle(s) - facedet.DetectFaces_(img, NULL, false, 10, NULL, cascade.faceCascade); + if (detection.detpars_.empty()) + detection.DetectFaces_(img, NULL, true, 10, NULL, cascade.faceCascade); // Get the start shape for the next face in the image, and the ROI around it. // The shape will be wrt the ROI frame. if (NextStartShapeAndRoi(shape, face_roi, detpar_roi, detpar, - img, mods_g, facedet, cascade)) + img, mods_g, detection, cascade)) { // now working with maybe flipped ROI and start shape in ROI frame *foundface = 1; @@ -219,9 +219,10 @@ int stasm_search_auto(// call repeatedly to find all faces const char *data, const int width, const int height, - StasmCascadeClassifier cascade) + StasmCascadeClassifier cascade, + FaceDet &detection) { - return stasm_search_auto_ext(foundface, landmarks, NULL, data, width, height, cascade); + return stasm_search_auto_ext(foundface, landmarks, NULL, data, width, height, cascade, detection); } int stasm_search_single( // wrapper for stasm_search_auto and friends @@ -237,7 +238,8 @@ int stasm_search_single( // wrapper for stasm_search_auto and friends (void) datadir; (void) imgpath; - return stasm_search_auto(foundface, landmarks, img, width, height, cascade); + FaceDet detection; + return stasm_search_auto(foundface, landmarks, img, width, height, cascade, detection); } int stasm_search_pinned( // call after the user has pinned some points diff --git a/3rdparty/stasm4.0.0/stasm/stasm_lib.h b/3rdparty/stasm4.0.0/stasm/stasm_lib.h index ba56b55..10b0912 100755 --- a/3rdparty/stasm4.0.0/stasm/stasm_lib.h +++ b/3rdparty/stasm4.0.0/stasm/stasm_lib.h @@ -64,6 +64,10 @@ #include "stasmcascadeclassifier.h" +namespace stasm { + class FaceDet; +} + static const int stasm_NLANDMARKS = 77; // number of landmarks extern const char* const stasm_VERSION; @@ -89,7 +93,8 @@ int stasm_search_auto( // call repeatedly to find all faces const char* data, const int width, const int height, - StasmCascadeClassifier cascade); + StasmCascadeClassifier cascade, + stasm::FaceDet &detection); extern "C" int stasm_search_single( // wrapper for stasm_search_auto and friends diff --git a/CMakeLists.txt b/CMakeLists.txt index 080fdc1..ae24144 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,7 +13,7 @@ if(NOT DEFINED CPACK_PACKAGE_VERSION_MAJOR) set(CPACK_PACKAGE_VERSION_MAJOR 1) endif() if(NOT DEFINED CPACK_PACKAGE_VERSION_MINOR) - set(CPACK_PACKAGE_VERSION_MINOR 0) + set(CPACK_PACKAGE_VERSION_MINOR 1) endif() if(NOT DEFINED CPACK_PACKAGE_VERSION_PATCH) set(CPACK_PACKAGE_VERSION_PATCH 0) @@ -158,8 +158,8 @@ endif() # Download the models ExternalProject_Add(models - URL http://github.com/biometrics/openbr/releases/download/v1.0.0/models.tar.gz - URL_MD5 0a7c79226d6629954aa32c835a1007b9 + URL http://github.com/biometrics/openbr/releases/download/v1.1.0/models.tar.gz + URL_MD5 26cf71abd48cec8d7b2abf3e8f4ebfc4 SOURCE_DIR "${PROJECT_SOURCE_DIR}/share/openbr/models" CONFIGURE_COMMAND "" BUILD_COMMAND "" diff --git a/app/br/br.cpp b/app/br/br.cpp index 06a3589..89b96be 100644 --- a/app/br/br.cpp +++ b/app/br/br.cpp @@ -169,8 +169,8 @@ public: check(parc >= 2 && parc <= 4, "Incorrect parameter count for 'evalRegression'."); br_eval_regression(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? parv[3] : ""); } else if (!strcmp(fun, "evalKNN")) { - check(parc == 3, "Incorrect parameter count for 'evalKNN'."); - br_eval_knn(parv[0], parv[1], parv[2]); + check(parc >= 2 && parc <= 3, "Incorrect parameter count for 'evalKNN'."); + br_eval_knn(parv[0], parv[1], parc > 2 ? parv[2] : ""); } else if (!strcmp(fun, "pairwiseCompare")) { check((parc >= 2) && (parc <= 3), "Incorrect parameter count for 'pairwiseCompare'."); br_pairwise_compare(parv[0], parv[1], parc == 3 ? parv[2] : ""); @@ -186,6 +186,9 @@ public: } else if (!strcmp(fun, "plotMetadata")) { check(parc >= 2, "Incorrect parameter count for 'plotMetadata'."); br_plot_metadata(parc-1, parv, parv[parc-1], true); + } else if (!strcmp(fun, "plotKNN")) { + check(parc >=2, "Incorrect parameter count for 'plotKNN'."); + br_plot_knn(parc-1, parv, parv[parc-1], true); } else if (!strcmp(fun, "project")) { check(parc == 2, "Insufficient parameter count for 'project'."); br_project(parv[0], parv[1]); @@ -279,13 +282,14 @@ private: "-evalDetection [{csv}] [{normalize}] [{minSize}] [{maxSize}]\n" "-evalLandmarking [{csv} [ ] [sample_index] [total_examples]]\n" "-evalRegression \n" - "-evalKNN [{iet_file}]\n" + "-evalKNN [{csv}]\n" "-pairwiseCompare [{output}]\n" "-inplaceEval [{csv}]\n" "-assertEval \n" "-plotDetection ... {destination}\n" "-plotLandmarking ... {destination}\n" "-plotMetadata ... \n" + "-plotKNN ... {destination}\n" "-project {output_gallery}\n" "-deduplicate \n" "-likely \n" diff --git a/docs/docs/api_docs/cl_api.md b/docs/docs/api_docs/cl_api.md index 9245eaa..cd49b0a 100644 --- a/docs/docs/api_docs/cl_api.md +++ b/docs/docs/api_docs/cl_api.md @@ -59,6 +59,17 @@ DOCUMENT ME * **wraps:** [br_pairwise_compare](c_api/functions.md#br_pairwise_compare) +### -crossValidate {: #crossvalidate } + +Either performs n fold cross validation (if nFolds > 0), or a single iteration of a train / test split with the first of the n folds as the test partition (nFolds < 0). + +* **arguments:** + + -crossValidate -algorithm "CrossValidate(description=$ALG,randomSeed=5):CrossValidate+Dist" -train + + -crossValidate -algorithm -enroll input.gal -compare input.gal . out.eval + + ### -eval {: #eval } Evaluate a similarity matrix diff --git a/docs/docs/install.md b/docs/docs/install.md index bc90a4a..505be33 100644 --- a/docs/docs/install.md +++ b/docs/docs/install.md @@ -35,7 +35,7 @@ A hacker's guide to building, editing, and running OpenBR. $ git clone https://github.com/biometrics/openbr.git $ cd openbr - $ git checkout 1.0 + $ git checkout v1.1.0 $ git submodule init $ git submodule update @@ -116,7 +116,7 @@ A hacker's guide to building, editing, and running OpenBR. $ git clone https://github.com/biometrics/openbr.git $ cd openbr - $ git checkout 1.0 + $ git checkout v1.1.0 $ git submodule init $ git submodule update @@ -196,7 +196,7 @@ A hacker's guide to building, editing, and running OpenBR. $ cd /c $ git clone https://github.com/biometrics/openbr.git $ cd openbr - $ git checkout 1.0 + $ git checkout v1.1.0 $ git submodule init $ git submodule update @@ -277,7 +277,7 @@ A hacker's guide to building, editing, and running OpenBR. $ git clone https://github.com/biometrics/openbr.git $ cd openbr - $ git checkout 1.0 + $ git checkout v1.1.0 $ git submodule init $ git submodule update diff --git a/openbr/core/boost.cpp b/openbr/core/boost.cpp index 6a75156..0f4e7fc 100644 --- a/openbr/core/boost.cpp +++ b/openbr/core/boost.cpp @@ -131,23 +131,16 @@ static CvMat* cvPreprocessIndexArray( const CvMat* idx_arr, int data_arr_size, b //------------------------------------- FeatureEvaluator --------------------------------------- -void FeatureEvaluator::init(Representation *_representation, int _maxSampleCount, int channels) +void FeatureEvaluator::init(Representation *_representation, int _maxSampleCount) { representation = _representation; - - int dx, dy; - Size windowSize = representation->windowSize(&dx, &dy); - data.create((int)_maxSampleCount, (windowSize.width + dx) * (windowSize.height + dy), CV_32SC(channels)); cls.create( (int)_maxSampleCount, 1, CV_32FC1 ); } -void FeatureEvaluator::setImage(const Mat &img, uchar clsLabel, int idx) +void FeatureEvaluator::setImage(const Template &src, uchar clsLabel, int idx) { cls.ptr(idx)[0] = clsLabel; - - Mat pp; - representation->preprocess(img, pp); - pp.reshape(0, 1).copyTo(data.row(idx)); + data.append(representation->preprocess(src)); } //----------------------------- CascadeBoostParams ------------------------------------------------- diff --git a/openbr/core/boost.h b/openbr/core/boost.h index 31c4a63..7a462a6 100644 --- a/openbr/core/boost.h +++ b/openbr/core/boost.h @@ -2,7 +2,6 @@ #define _BOOST_H_ #include "ml.h" -#include #include #ifdef _WIN32 @@ -17,9 +16,9 @@ namespace br struct FeatureEvaluator { ~FeatureEvaluator() {} - void init(Representation *_representation, int _maxSampleCount, int channels); - void setImage(const cv::Mat& img, uchar clsLabel, int idx); - float operator()(int featureIdx, int sampleIdx) const { return representation->evaluate(data.row(sampleIdx), featureIdx); } + void init(Representation *_representation, int _maxSampleCount); + void setImage(const Template &src, uchar clsLabel, int idx); + float operator()(int featureIdx, int sampleIdx) const { return representation->evaluate(data[sampleIdx], featureIdx); } int getNumFeatures() const { return representation->numFeatures(); } int getMaxCatCount() const { return representation->maxCatCount(); } @@ -27,7 +26,8 @@ struct FeatureEvaluator const cv::Mat& getCls() const { return cls; } float getCls(int si) const { return cls.at(si, 0); } - cv::Mat data, cls; + cv::Mat cls; + TemplateList data; Representation *representation; }; diff --git a/openbr/core/eval.cpp b/openbr/core/eval.cpp index e940aff..c7dcf66 100755 --- a/openbr/core/eval.cpp +++ b/openbr/core/eval.cpp @@ -825,16 +825,25 @@ static QStringList computeDetectionResults(const QList &detec if (prevFP / numImages < 0.1 && FP / numImages > 0.1 && discrete) { qDebug("TAR @ FAR => %f : 0.1", TP / totalTrueDetections); qDebug("Confidence: %f", detection.confidence); + qDebug("TP vs. FP: %f to %f", TP, FP); } else if (prevFP / numImages < 0.01 && FP / numImages > 0.01 && discrete) { qDebug("TAR @ FAR => %f : 0.01", TP / totalTrueDetections); qDebug("Confidence: %f", detection.confidence); + qDebug("TP vs. FP: %f to %f", TP, FP); + } else if (prevFP / numImages < 0.001 && FP / numImages > 0.001 && discrete) { + qDebug("TAR @ FAR => %f : 0.001", TP / totalTrueDetections); + qDebug("Confidence: %f", detection.confidence); + qDebug("TP vs. FP: %f to %f", TP, FP); } + points.append(DetectionOperatingPoint(TP, FP, totalTrueDetections, numImages)); prevFP = FP; } } } + if (discrete) qDebug("Total TP vs. FP: %f to %f", TP, FP); + const int keep = qMin(points.size(), Max_Points); if (keep < 1) qFatal("Insufficient points."); @@ -1336,7 +1345,7 @@ void readKNNTruth(size_t probeCount, QVector< QList > &groundTruth, cons qFatal("Invalid ground truth file!"); } -void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &iet) +void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &csv) { qDebug("Evaluating k-NN of %s against %s", qPrintable(knnGraph), qPrintable(knnTruth)); @@ -1406,19 +1415,14 @@ void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &ie if (numUnmatedSearches == 0) qFatal("No unmated searches!"); - printf("Rank-%i Return Rate: %g\n", 1, getCMC(firstGenuineReturns, 1, numMatedSearches)); + + qDebug("Rank-%d Return Rate: %.3f", 1, getCMC(firstGenuineReturns, 1, numMatedSearches)); if (k >=5) - printf("Rank-%i Return Rate: %g\n", 5, getCMC(firstGenuineReturns, 5, numMatedSearches)); + qDebug("Rank-%d Return Rate: %.3f", 5, getCMC(firstGenuineReturns, 5, numMatedSearches)); if (k >=10) - printf("Rank-%i Return Rate: %g\n", 10, getCMC(firstGenuineReturns, 10, numMatedSearches)); - - printf("Rank-%zu Return Rate: %g\n", k, double(numMatedSimilarities) / double(numMatedSearches)); + qDebug("Rank-%d Return Rate: %.3f", 10, getCMC(firstGenuineReturns, 10, numMatedSearches)); - // Open the output file - QFile ietFile(iet); - if (!ietFile.open(QFile::WriteOnly | QFile::Text)) - qFatal("Failed to open IET file for writing!"); - ietFile.write("Threshold,FPIR,FNIR\n"); + qDebug("Rank-%zu Return Rate: %.3f", k, double(numMatedSimilarities) / double(numMatedSearches)); /* * Iterate through the similarity scores highest-to-lowest, @@ -1447,10 +1451,28 @@ void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &ie } } - foreach(const OperatingPoint &operatingPoint, operatingPoints) - ietFile.write(qPrintable(QString::number(operatingPoint.score) + "," + - QString::number(operatingPoint.FAR) + "," + - QString::number(operatingPoint.TAR) + "\n")); + if (!csv.isEmpty()) { + // Open the output file + QFile ietFile(csv); + if (!ietFile.open(QFile::WriteOnly | QFile::Text)) + qFatal("Failed to open IET file for writing!"); + ietFile.write("Plot,X,Y,Z\n"); + // Write CMC + const int Max_Retrieval = min(200, (int)k); + for (int i=1; i<=Max_Retrieval; i++) { + const float retrievalRate = getCMC(firstGenuineReturns, i, numMatedSearches); + ietFile.write(qPrintable(QString("CMC,%1,%2,0\n").arg(QString::number(i), QString::number(retrievalRate)))); + } + + foreach(const OperatingPoint &operatingPoint, operatingPoints) + ietFile.write(qPrintable("IET," + + QString::number(operatingPoint.FAR) + "," + + QString::number(operatingPoint.TAR) + "," + + QString::number(operatingPoint.score) + "\n")); + } + + qDebug("FNIR @ FPIR = 0.1: %.3f", 1-getOperatingPointGivenFAR(operatingPoints, 0.1).TAR); + qDebug("FNIR @ FPIR = 0.01: %.3f", 1-getOperatingPointGivenFAR(operatingPoints, 0.01).TAR); } } // namespace br diff --git a/openbr/core/eval.h b/openbr/core/eval.h index d09ce3d..0edfd25 100644 --- a/openbr/core/eval.h +++ b/openbr/core/eval.h @@ -33,7 +33,7 @@ namespace br float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", bool normalize = false, int minSize = 0, int maxSize = 0); // Return average overlap float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", int normalizationIndexA = 0, int normalizationIndexB = 1, int sampleIndex = 0, int totalExamples = 5); // Return average error void EvalRegression(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = ""); - void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &iet); + void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &csv = ""); struct Candidate { diff --git a/openbr/core/opencvutils.cpp b/openbr/core/opencvutils.cpp index cb20741..1583188 100644 --- a/openbr/core/opencvutils.cpp +++ b/openbr/core/opencvutils.cpp @@ -436,7 +436,7 @@ public: }; // TODO: Make sure case where no confidences are inputted works. -void OpenCVUtils::group(QList &rects, QList &confidences, float confidenceThreshold, float epsilon) +void OpenCVUtils::group(QList &rects, QList &confidences, float confidenceThreshold, int minNeighbors, float epsilon) { if (rects.isEmpty()) return; @@ -450,7 +450,7 @@ void OpenCVUtils::group(QList &rects, QList &confidences, float con vector rrects(nClasses); // Total number of rects in each class - vector rweights(nClasses, 0); + vector neighbors(nClasses, 0); vector rejectWeights(nClasses, -std::numeric_limits::max()); for (size_t i = 0; i < labels.size(); i++) @@ -460,7 +460,7 @@ void OpenCVUtils::group(QList &rects, QList &confidences, float con rrects[cls].y += rects[i].y; rrects[cls].width += rects[i].width; rrects[cls].height += rects[i].height; - rweights[cls]++; + neighbors[cls]++; } if (useConfidences) @@ -478,7 +478,7 @@ void OpenCVUtils::group(QList &rects, QList &confidences, float con for (int i = 0; i < nClasses; i++) { Rect r = rrects[i]; - float s = 1.f/rweights[i]; + float s = 1.f/neighbors[i]; rrects[i] = Rect(saturate_cast(r.x*s), saturate_cast(r.y*s), saturate_cast(r.width*s), @@ -488,7 +488,7 @@ void OpenCVUtils::group(QList &rects, QList &confidences, float con rects.clear(); confidences.clear(); - // Aggregate by comparing average rectangles against other average rectangels + // Aggregate by comparing average rectangles against other average rectangles for (int i = 0; i < nClasses; i++) { // Average rectangle @@ -496,19 +496,22 @@ void OpenCVUtils::group(QList &rects, QList &confidences, float con // Used to eliminate rectangles with few neighbors in the case of no weights // int n1 = levelWeights ? rejectLevels[i] : rweights[i]; - float w1 = rejectWeights[i]; + const float w1 = rejectWeights[i]; // Eliminate rectangle if it doesn't meet confidence criteria if (w1 <= confidenceThreshold) continue; + const int n1 = neighbors[i]; + if (n1 < minNeighbors) + continue; + // filter out small face rectangles inside large rectangles int j; for (j = 0; j < nClasses; j++) { - float w2 = rejectWeights[j]; - - if (j == i) + const int n2 = neighbors[j]; + if (j == i || n2 < minNeighbors) continue; Rect r2 = rrects[j]; @@ -516,13 +519,11 @@ void OpenCVUtils::group(QList &rects, QList &confidences, float con int dx = saturate_cast(r2.width * epsilon); int dy = saturate_cast(r2.height * epsilon); + float w2 = rejectWeights[j]; + // If, r1 is within the r2 AND - // the second rectangle reaches a later stage than the first - // where both the first and the second must have a stage greater than three OR - // the first doens't reach the third stage. - // Changeto: second rectangle has a higher confidence than the first OR - // the first has a low confidence. - // Then, eliminate the first rectangle. + // r2 has a higher confidence than r1 + // then, eliminate the r1 if(r1.x >= r2.x - dx && r1.y >= r2.y - dy && r1.x + r1.width <= r2.x + r2.width + dx && @@ -531,7 +532,6 @@ void OpenCVUtils::group(QList &rects, QList &confidences, float con break; } - // Need to return rects and confidences if( j == nClasses ) { rects.append(r1); diff --git a/openbr/core/opencvutils.h b/openbr/core/opencvutils.h index 9754817..ae2e6a9 100644 --- a/openbr/core/opencvutils.h +++ b/openbr/core/opencvutils.h @@ -102,7 +102,7 @@ namespace OpenCVUtils float overlap(const QRectF &rect1, const QRectF &rect2); // Misc - void group(QList &rects, QList &confidences, float confidenceThreshold, float epsilon); + void group(QList &rects, QList &confidences, float confidenceThreshold, int minNeighbors, float epsilon); void flip(const br::Template &src, br::Template &dst, int axis, bool flipMat=true, bool flipPoints=true, bool flipRects=true); void flip(const br::TemplateList &src, br::TemplateList &dst, int axis, bool flipMat=true, bool flipPoints=true, bool flipRects=true); diff --git a/openbr/core/plot.cpp b/openbr/core/plot.cpp index 94f84dd..bff51d1 100644 --- a/openbr/core/plot.cpp +++ b/openbr/core/plot.cpp @@ -346,4 +346,32 @@ bool PlotMetadata(const QStringList &files, const QString &columns, bool show) return p.finalize(show); } +bool PlotKNN(const QStringList &files, const File &destination, bool show) +{ + qDebug("Plotting %d k-NN file(s) to %s", files.size(), qPrintable(destination)); + RPlot p(files, destination); + p.file.write("\nformatData(type=\"knn\")\n\n"); + + QMap optMap; + optMap.insert("rocOptions", File(QString("[xTitle=False Positive Identification Rate (FPIR),yTitle=True Positive Identification Rate (TPIR),xLog=true,yLog=false]"))); + optMap.insert("ietOptions", File(QString("[xTitle=False Positive Identification Rate (FPIR),yTitle=False Negative Identification Rate (FNIR),xLog=true,yLog=true]"))); + optMap.insert("cmcOptions", File(QString("[xTitle=Rank,yTitle=Retrieval Rate,xLog=true,yLog=false,size=1,xLabels=(1,5,10,50,100),xBreaks=(1,5,10,50,100)]"))); + + foreach (const QString &key, optMap.keys()) { + const QStringList options = destination.get(key, QStringList()); + foreach (const QString &option, options) { + QStringList words = QtUtils::parse(option, '='); + QtUtils::checkArgsSize(words[0], words, 1, 2); + optMap[key].set(words[0], words[1]); + } + } + + QString plot = "plot <- plotLine(lineData=%1, options=list(%2), flipY=%3)\nplot\n"; + p.file.write(qPrintable(QString(plot).arg("IET", toRList(optMap["rocOptions"]), "TRUE"))); + p.file.write(qPrintable(QString(plot).arg("IET", toRList(optMap["ietOptions"]), "FALSE"))); + p.file.write(qPrintable(QString(plot).arg("CMC", toRList(optMap["cmcOptions"]), "FALSE"))); + + return p.finalize(show); +} + } // namespace br diff --git a/openbr/core/plot.h b/openbr/core/plot.h index 223f746..26db428 100644 --- a/openbr/core/plot.h +++ b/openbr/core/plot.h @@ -28,6 +28,7 @@ namespace br bool PlotDetection(const QStringList &files, const File &destination, bool show = false); bool PlotLandmarking(const QStringList &files, const File &destination, bool show = false); bool PlotMetadata(const QStringList &files, const QString &destination, bool show = false); + bool PlotKNN(const QStringList &files, const File &destination, bool show = false); } #endif // BR_PLOT_H diff --git a/openbr/openbr.cpp b/openbr/openbr.cpp index cf69ecc..e11acf4 100644 --- a/openbr/openbr.cpp +++ b/openbr/openbr.cpp @@ -145,9 +145,9 @@ void br_eval_regression(const char *predicted_gallery, const char *truth_gallery EvalRegression(predicted_gallery, truth_gallery, predicted_property, truth_property); } -void br_eval_knn(const char *knnGraph, const char *knnTruth, const char *iet) +void br_eval_knn(const char *knnGraph, const char *knnTruth, const char *csv) { - EvalKNN(knnGraph, knnTruth, iet); + EvalKNN(knnGraph, knnTruth, csv); } void br_finalize() @@ -221,6 +221,11 @@ bool br_plot_metadata(int num_files, const char *files[], const char *columns, b return PlotMetadata(QtUtils::toStringList(num_files, files), columns, show); } +bool br_plot_knn(int num_files, const char *files[], const char *destination, bool show) +{ + return PlotKNN(QtUtils::toStringList(num_files, files), destination, show); +} + float br_progress() { return Globals->progress(); diff --git a/openbr/openbr.h b/openbr/openbr.h index dd06591..12a769a 100644 --- a/openbr/openbr.h +++ b/openbr/openbr.h @@ -64,7 +64,7 @@ BR_EXPORT float br_eval_landmarking(const char *predicted_gallery, const char *t BR_EXPORT void br_eval_regression(const char *predicted_gallery, const char *truth_gallery, const char *predicted_property = "", const char *truth_property = ""); -BR_EXPORT void br_eval_knn(const char *knnGraph, const char *knnTruth, const char *iet); +BR_EXPORT void br_eval_knn(const char *knnGraph, const char *knnTruth, const char *csv = ""); BR_EXPORT void br_finalize(); @@ -93,6 +93,8 @@ BR_EXPORT bool br_plot_landmarking(int num_files, const char *files[], const cha BR_EXPORT bool br_plot_metadata(int num_files, const char *files[], const char *columns, bool show = false); +BR_EXPORT bool br_plot_knn(int num_files, const char *files[], const char *destination, bool show = false); + BR_EXPORT float br_progress(); BR_EXPORT void br_read_pipe(const char *pipe, int *argc, char ***argv); diff --git a/openbr/openbr_plugin.h b/openbr/openbr_plugin.h index 5ae5069..783cb5d 100644 --- a/openbr/openbr_plugin.h +++ b/openbr/openbr_plugin.h @@ -882,13 +882,11 @@ public: static Representation *make(QString str, QObject *parent); /*!< \brief Make a representation from a string. */ - virtual void preprocess(const cv::Mat &src, cv::Mat &dst) const { dst = src; } - virtual void train(const QList &images, const QList &labels) { (void) images; (void)labels; } - - virtual float evaluate(const cv::Mat &image, int idx) const = 0; - // By convention, an empty indices list will result in all feature responses being calculated - // and returned. - virtual cv::Mat evaluate(const cv::Mat &image, const QList &indices = QList()) const = 0; + virtual Template preprocess(const Template &src) const { return src; } + virtual void train(const TemplateList &data) { (void)data; } + virtual float evaluate(const Template &src, int idx) const = 0; + // By convention passing an empty list evaluates all features in the representation + virtual cv::Mat evaluate(const Template &src, const QList &indices = QList()) const = 0; virtual cv::Size windowSize(int *dx = NULL, int *dy = NULL) const = 0; // dx and dy should indicate the change to the original window size after preprocessing virtual int numChannels() const { return 1; } @@ -905,13 +903,13 @@ public: static Classifier *make(QString str, QObject *parent); - virtual void train(const QList &images, const QList &labels) = 0; - virtual float classify(const cv::Mat &image, bool process = true, float *confidence = NULL) const = 0; + virtual void train(const TemplateList &data) { (void)data; } + virtual float classify(const Template &src, bool process = true, float *confidence = NULL) const = 0; // Slots for representations - virtual cv::Mat preprocess(const cv::Mat &image) const = 0; + virtual Template preprocess(const Template &src) const { return src; } virtual cv::Size windowSize(int *dx = NULL, int *dy = NULL) const = 0; - virtual int numFeatures() const = 0; + virtual int numFeatures() const { return 0; } }; diff --git a/openbr/plugins/classification/boostedforest.cpp b/openbr/plugins/classification/boostedforest.cpp index 809b5a9..3cf941e 100644 --- a/openbr/plugins/classification/boostedforest.cpp +++ b/openbr/plugins/classification/boostedforest.cpp @@ -125,20 +125,20 @@ private: QList classifiers; float threshold; - void train(const QList &images, const QList &labels) + void train(const TemplateList &data) { - representation->train(images, labels); + representation->train(data); CascadeBoostParams params(type, minTAR, maxFAR, trimRate, maxDepth, maxWeakCount); FeatureEvaluator featureEvaluator; - featureEvaluator.init(representation, images.size(), representation->numChannels()); + featureEvaluator.init(representation, data.size()); - for (int i = 0; i < images.size(); i++) - featureEvaluator.setImage(images[i], labels[i], i); + for (int i = 0; i < data.size(); i++) + featureEvaluator.setImage(data[i], data[i].file.get("Label"), i); CascadeBoost boost; - boost.train(&featureEvaluator, images.size(), 1024, 1024, representation->numChannels(), params); + boost.train(&featureEvaluator, data.size(), 1024, 1024, representation->numChannels(), params); threshold = boost.getThreshold(); @@ -149,13 +149,9 @@ private: } } - float classify(const Mat &image, bool process, float *confidence) const + float classify(const Template &src, bool process, float *confidence) const { - Mat m; - if (process) - m = preprocess(image); - else - m = image; + Template t = process ? preprocess(src) : src; float sum = 0; for (int i = 0; i < classifiers.size(); i++) { @@ -163,10 +159,10 @@ private: while (node->left) { if (representation->maxCatCount() > 0) { - int c = (int)representation->evaluate(m, node->featureIdx); + int c = (int)representation->evaluate(t, node->featureIdx); node = (node->subset[c >> 5] & (1 << (c & 31))) ? node->left : node->right; } else { - double val = representation->evaluate(m, node->featureIdx); + double val = representation->evaluate(t, node->featureIdx); node = val <= node->threshold ? node->left : node->right; } } @@ -184,11 +180,9 @@ private: return representation->numFeatures(); } - Mat preprocess(const Mat &image) const + Template preprocess(const Template &src) const { - Mat dst; - representation->preprocess(image, dst); - return dst; + return representation->preprocess(src); } Size windowSize(int *dx, int *dy) const diff --git a/openbr/plugins/classification/cascade.cpp b/openbr/plugins/classification/cascade.cpp index 292e405..e37160d 100644 --- a/openbr/plugins/classification/cascade.cpp +++ b/openbr/plugins/classification/cascade.cpp @@ -1,7 +1,8 @@ #include -#include + #include #include +#include "openbr/core/opencvutils.h" #include @@ -37,7 +38,7 @@ struct Miner Mat mine(bool *newImg) { // Copy region of winSize region of img into m - Mat window(windowSize.height, windowSize.width, CV_8UC1, + Mat window(windowSize.height, windowSize.width, CV_8U, (void*)(scaledSrc.data + point.y * scaledSrc.step + point.x * scaledSrc.elemSize()), scaledSrc.step); @@ -100,31 +101,30 @@ class CascadeClassifier : public Classifier BR_PROPERTY(bool, requireAllStages, false) QList stages; - QList posImages, negImages; - QList posSamples, negSamples; + TemplateList posImages, negImages; + TemplateList posSamples, negSamples; QList indices; int negIndex, posIndex, samplingRound; - QMutex samplingMutex, miningMutex, passedMutex; + QMutex samplingMutex, miningMutex; void init() { negIndex = posIndex = samplingRound = 0; } - bool getPositive(Mat &img) + bool getPositive(Template &img) { if (posIndex >= posImages.size()) return false; - - posImages[indices[posIndex++]].copyTo(img); + img = posImages[indices[posIndex++]]; return true; } - Mat getNegative(Point &offset) + Template getNegative(Point &offset) { - Mat negative; + Template negative; const Size size = windowSize(); // Grab negative from list @@ -136,9 +136,9 @@ class CascadeClassifier : public Classifier samplingRound = samplingRound % (size.width * size.height); negIndex %= count; - offset.x = qMin( (int)samplingRound % size.width, negative.cols - size.width); - offset.y = qMin( (int)samplingRound / size.width, negative.rows - size.height); - if (!negative.empty() && negative.type() == CV_8UC1 + offset.x = qMin( (int)samplingRound % size.width, negative.m().cols - size.width); + offset.y = qMin( (int)samplingRound / size.width, negative.m().rows - size.height); + if (!negative.m().empty() && negative.m().type() == CV_8U && offset.x >= 0 && offset.y >= 0) break; } @@ -146,35 +146,32 @@ class CascadeClassifier : public Classifier return negative; } - int mine() + uint64 mine() { - int passedNegatives = 0; + uint64 passedNegatives = 0; forever { - Mat negative; + Template negative; Point offset; - samplingMutex.lock(); + QMutexLocker samplingLocker(&samplingMutex); negative = getNegative(offset); - samplingMutex.unlock(); + samplingLocker.unlock(); - Miner miner(negative, windowSize(), offset); + Miner miner(negative.m(), windowSize(), offset); forever { bool newImg; - Mat sample = miner.mine(&newImg); + Template sample(negative.file, miner.mine(&newImg)); if (!newImg) { if (negSamples.size() >= numNegs) return passedNegatives; float confidence; if (classify(sample, true, &confidence) != 0) { - miningMutex.lock(); - if (negSamples.size() >= numNegs) { - miningMutex.unlock(); + QMutexLocker miningLocker(&miningMutex); + if (negSamples.size() >= numNegs) return passedNegatives; - } negSamples.append(sample); printf("Negative samples: %d\r", negSamples.size()); - miningMutex.unlock(); } passedNegatives++; @@ -184,16 +181,16 @@ class CascadeClassifier : public Classifier } } - void train(const QList &images, const QList &labels) + void train(const TemplateList &data) { - for (int i = 0; i < images.size(); i++) - labels[i] == 1 ? posImages.append(images[i]) : negImages.append(images[i]); + foreach (const Template &t, data) + t.file.get("Label") == 1.0f ? posImages.append(t) : negImages.append(t); - qDebug() << "Total images:" << images.size() + qDebug() << "Total images:" << data.size() << "\nTotal positive images:" << posImages.size() << "\nTotal negative images:" << negImages.size(); - indices = Common::RandSample(posImages.size(),posImages.size(),true); + indices = Common::RandSample(posImages.size(), posImages.size(), true); stages.reserve(numStages); for (int i = 0; i < numStages; i++) { @@ -212,27 +209,17 @@ class CascadeClassifier : public Classifier return; } - QList posLabels; - posLabels.reserve(posSamples.size()); - for (int j=0; j negLabels; - negLabels.reserve(negSamples.size()); - for (int j=0; jtrain(posSamples+negSamples, posLabels+negLabels); + stages[i]->train(posSamples + negSamples); qDebug() << "END>"; } } - float classify(const Mat &image, bool process, float *confidence) const + float classify(const Template &src, bool process, float *confidence) const { float stageConf = 0.0f; foreach (const Classifier *stage, stages) { - float result = stage->classify(image, process, &stageConf); + float result = stage->classify(src, process, &stageConf); if (confidence) *confidence += stageConf; if (result == 0.0f) @@ -246,9 +233,9 @@ class CascadeClassifier : public Classifier return stages.first()->numFeatures(); } - Mat preprocess(const Mat &image) const + Template preprocess(const Template &src) const { - return stages.first()->preprocess(image); + return stages.first()->preprocess(src); } Size windowSize(int *dx = NULL, int *dy = NULL) const @@ -276,16 +263,13 @@ class CascadeClassifier : public Classifier private: float getSamples() { - posSamples.clear(); - posSamples.reserve(numPos); - negSamples.clear(); - negSamples.reserve(numNegs); + posSamples.clear(); posSamples.reserve(numPos); + negSamples.clear(); negSamples.reserve(numNegs); posIndex = 0; float confidence; while (posSamples.size() < numPos) { - Mat pos(windowSize(), CV_8UC1); - + Template pos; if (!getPositive(pos)) qFatal("Cannot get another positive sample!"); @@ -297,13 +281,13 @@ private: qDebug() << "POS count : consumed " << posSamples.size() << ":" << posIndex; - QFutureSynchronizer futures; + QFutureSynchronizer futures; for (int i=0; i > results = futures.futures(); + uint64 passedNegs = 0; + QList > results = futures.futures(); for (int i=0; i #include #include @@ -45,14 +46,18 @@ private: { dst = src; - shape_predictor *sp = shapeResource.acquire(); + shape_predictor *const sp = shapeResource.acquire(); - cv_image cimg(src.m()); + cv::Mat cvImage = src.m(); + if (cvImage.channels() == 3) + cv::cvtColor(cvImage, cvImage, CV_BGR2GRAY); + + cv_image cimg(cvImage); array2d image; assign_image(image,cimg); if (src.file.rects().isEmpty()) { //If the image has no rects assume the whole image is a face - rectangle r(0, 0, src.m().cols, src.m().rows); + rectangle r(0, 0, cvImage.cols, cvImage.rows); full_object_detection shape = (*sp)(image, r); for (size_t i = 0; i < shape.num_parts(); i++) dst.file.appendPoint(QPointF(shape.part(i)(0),shape.part(i)(1))); diff --git a/openbr/plugins/classification/liblinear.cpp b/openbr/plugins/classification/liblinear.cpp index 2bebbc4..b3d4035 100644 --- a/openbr/plugins/classification/liblinear.cpp +++ b/openbr/plugins/classification/liblinear.cpp @@ -138,7 +138,7 @@ private: param.weight = NULL; } - m = *train_svm(&prob, ¶m); + //m = *train_svm(&prob, ¶m); delete[] param.weight; delete[] param.weight_label; diff --git a/openbr/plugins/core/algorithms.cpp b/openbr/plugins/core/algorithms.cpp index e34e6bb..17e93a0 100644 --- a/openbr/plugins/core/algorithms.cpp +++ b/openbr/plugins/core/algorithms.cpp @@ -39,7 +39,7 @@ class AlgorithmsInitializer : public Initializer Globals->abbreviations.insert("FR_Mouth", "(CropFromLandmarks([59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76])+Resize(24,48))"); Globals->abbreviations.insert("FR_Nose", "(CropFromLandmarks([16,17,18,19,20,21,22,23,24,25,26,27],padding=3)+Resize(36,36))"); Globals->abbreviations.insert("FR_Face", "(Crop(24,24,88,88)+Resize(44,44))"); - Globals->abbreviations.insert("FR_Detect", "(FaceDetection+Stasm+Rename(StasmLeftEye,Affine_1,true)+Rename(StasmRightEye,Affine_0,true)+Affine(136,136,0.35,0.35,warpPoints=true))"); + Globals->abbreviations.insert("FR_Detect", "(Open+Cvt(Gray)+Cascade+Stasm+Rename(StasmLeftEye,Affine_1,true)+Rename(StasmRightEye,Affine_0,true)+Affine(136,136,0.35,0.35,warpPoints=true))"); Globals->abbreviations.insert("FR_Represent", "((DenseHOG/DenseLBP)+Cat+LDA(.98)+Normalize(L2))"); Globals->abbreviations.insert("GenderClassification", "FaceDetection+Expand+FaceClassificationRegistration+Expand+++Discard"); diff --git a/openbr/plugins/distance/unit.cpp b/openbr/plugins/distance/unit.cpp index 491ffad..e2d60ed 100644 --- a/openbr/plugins/distance/unit.cpp +++ b/openbr/plugins/distance/unit.cpp @@ -76,12 +76,18 @@ class UnitDistance : public Distance float compare(const Template &target, const Template &query) const { - return a * (distance->compare(target, query) - b); + return normalize(distance->compare(target, query)); } float compare(const cv::Mat &target, const cv::Mat &query) const { - return a * (distance->compare(target, query) - b); + return normalize(distance->compare(target, query)); + } + + float normalize(float score) const + { + if (!Globals->scoreNormalization) return score; + return a * (score - b); } }; diff --git a/openbr/plugins/gallery/keyframes.cpp b/openbr/plugins/gallery/keyframes.cpp index 73ad7ee..add0e08 100644 --- a/openbr/plugins/gallery/keyframes.cpp +++ b/openbr/plugins/gallery/keyframes.cpp @@ -26,6 +26,12 @@ extern "C" #include } +// http://stackoverflow.com/questions/24057248/ffmpeg-undefined-references-to-av-frame-alloc +#if LIBAVCODEC_VERSION_INT < AV_VERSION_INT(55,28,1) +#define av_frame_alloc avcodec_alloc_frame +#define av_frame_free avcodec_free_frame +#endif + using namespace cv; namespace br diff --git a/openbr/plugins/imgproc/equalizehist.cpp b/openbr/plugins/imgproc/equalizehist.cpp index 50b4894..9c9d614 100644 --- a/openbr/plugins/imgproc/equalizehist.cpp +++ b/openbr/plugins/imgproc/equalizehist.cpp @@ -18,6 +18,8 @@ #include +using namespace cv; + namespace br { @@ -32,7 +34,20 @@ class EqualizeHistTransform : public UntrainableTransform void project(const Template &src, Template &dst) const { - cv::equalizeHist(src, dst); + if (src.m().channels() == 1) { + equalizeHist(src, dst); + } else if (src.m().channels() == 3) { + // http://stackoverflow.com/questions/15007304/histogram-equalization-not-working-on-color-image-opencv + Mat ycrcb; + cvtColor(src, ycrcb, CV_BGR2YCrCb); + vector channels; + split(ycrcb, channels); + equalizeHist(channels[0], channels[0]); + merge(channels, ycrcb); + cvtColor(ycrcb, dst, CV_YCrCb2BGR); + } else { + qFatal("Invalid channel count!"); + } } }; diff --git a/openbr/plugins/imgproc/pad.cpp b/openbr/plugins/imgproc/pad.cpp index 2e4de15..3c40221 100644 --- a/openbr/plugins/imgproc/pad.cpp +++ b/openbr/plugins/imgproc/pad.cpp @@ -1,3 +1,4 @@ +#include #include using namespace cv; @@ -5,24 +6,36 @@ using namespace cv; namespace br { +/*! + * \ingroup transforms + * \brief Pads an image. + * \author Scott Klum \cite sklum + */ class PadTransform : public UntrainableTransform { Q_OBJECT - - Q_PROPERTY(int padSize READ get_padSize WRITE set_padSize RESET reset_padSize STORED false) - Q_PROPERTY(int padValue READ get_padValue WRITE set_padValue RESET reset_padValue STORED false) - BR_PROPERTY(int, padSize, 0) - BR_PROPERTY(int, padValue, 0) + Q_ENUMS(Method) + +public: + /*!< */ + enum Border { Replicate = BORDER_REPLICATE, + Reflect = BORDER_REFLECT_101, + Constant = BORDER_CONSTANT}; + +private: + Q_PROPERTY(Border border READ get_border WRITE set_border RESET reset_border STORED false) + Q_PROPERTY(float percent READ get_percent WRITE set_percent RESET reset_percent STORED false) + Q_PROPERTY(int value READ get_value WRITE set_value RESET reset_value STORED false) + BR_PROPERTY(Border, border, Replicate) + BR_PROPERTY(float, percent, .1) + BR_PROPERTY(float, value, 0) void project(const Template &src, Template &dst) const { - dst.file = src.file; - - foreach (const Mat &m, src) { - Mat padded = padValue * Mat::ones(m.rows + 2*padSize, m.cols + 2*padSize, m.type()); - padded(Rect(padSize, padSize, padded.cols - padSize, padded.rows - padSize)) = m; - dst += padded; - } + int top, bottom, left, right; + top = percent*src.m().rows; bottom = percent*src.m().rows; + left = percent*src.m().cols; right = percent*src.m().cols; + copyMakeBorder(src, dst, top, bottom, left, right, border, Scalar(value)); } }; diff --git a/openbr/plugins/imgproc/resize.cpp b/openbr/plugins/imgproc/resize.cpp index edbdea5..9a5310f 100644 --- a/openbr/plugins/imgproc/resize.cpp +++ b/openbr/plugins/imgproc/resize.cpp @@ -55,9 +55,15 @@ private: void project(const Template &src, Template &dst) const { - if (!preserveAspect) + if (!preserveAspect) { resize(src, dst, Size((columns == -1) ? src.m().cols*rows/src.m().rows : columns, rows), 0, 0, method); - else { + const float rowScaleFactor = (float)rows/src.m().rows; + const float colScaleFactor = (float)columns/src.m().cols; + QList points = src.file.points(); + for (int i=0; i #include +#include using namespace cv; @@ -32,12 +33,14 @@ class RndRotateTransform : public UntrainableTransform Q_OBJECT Q_PROPERTY(QList range READ get_range WRITE set_range RESET reset_range STORED false) + Q_PROPERTY(int center READ get_center WRITE set_center RESET reset_center STORED false) BR_PROPERTY(QList, range, QList() << -15 << 15) + BR_PROPERTY(int, center, -1) void project(const Template &src, Template &dst) const { int span = range.first() - range.last(); - int angle = (rand() % span) + range.first(); - Mat rotMatrix = getRotationMatrix2D(Point2f(src.m().rows/2,src.m().cols/2),angle,1.0); + int angle = span == 0 ? range.first() : (rand() % span) + range.first(); + Mat rotMatrix = getRotationMatrix2D(center == -1 ? Point2f(src.m().rows/2,src.m().cols/2) : OpenCVUtils::toPoint(src.file.points()[center]),angle,1.0); warpAffine(src,dst,rotMatrix,Size(src.m().cols,src.m().rows),INTER_LINEAR,BORDER_REFLECT_101); QList points = src.file.points(); diff --git a/openbr/plugins/imgproc/roi.cpp b/openbr/plugins/imgproc/roi.cpp index f0339bd..d1e492d 100644 --- a/openbr/plugins/imgproc/roi.cpp +++ b/openbr/plugins/imgproc/roi.cpp @@ -34,8 +34,10 @@ class ROITransform : public UntrainableTransform Q_OBJECT Q_PROPERTY(QString propName READ get_propName WRITE set_propName RESET reset_propName STORED false) Q_PROPERTY(bool copyOnCrop READ get_copyOnCrop WRITE set_copyOnCrop RESET reset_copyOnCrop STORED false) + Q_PROPERTY(int shiftPoints READ get_shiftPoints WRITE set_shiftPoints RESET reset_shiftPoints STORED false) BR_PROPERTY(QString, propName, "") BR_PROPERTY(bool, copyOnCrop, false) + BR_PROPERTY(int, shiftPoints, -1) void project(const Template &src, Template &dst) const { @@ -55,6 +57,16 @@ class ROITransform : public UntrainableTransform if (Globals->verbose) qWarning("No rects present in file."); } + + if (shiftPoints != -1) { + // Shift the points to the rect with the index provided + QRectF rect = src.file.rects()[shiftPoints]; + QList points = src.file.points(); + for (int i=0; i #include #include -#include #include @@ -35,7 +34,6 @@ namespace br * \br_property int minSize The smallest sized object to detect in pixels * \br_property int maxSize The largest sized object to detect in pixels. A negative value will set maxSize == image size * \br_property float scaleFactor The factor to scale the image by during each resize. - * \br_property int minNeighbors Parameter for non-maximum supression * \br_property float confidenceThreshold A threshold for positive detections. Positive detections returned by the classifier that have confidences below this threshold are considered negative detections. * \br_property float eps Parameter for non-maximum supression */ @@ -50,6 +48,7 @@ class SlidingWindowTransform : public MetaTransform Q_PROPERTY(float scaleFactor READ get_scaleFactor WRITE set_scaleFactor RESET reset_scaleFactor STORED false) Q_PROPERTY(float confidenceThreshold READ get_confidenceThreshold WRITE set_confidenceThreshold RESET reset_confidenceThreshold STORED false) Q_PROPERTY(float eps READ get_eps WRITE set_eps RESET reset_eps STORED false) + Q_PROPERTY(float minNeighbors READ get_minNeighbors WRITE set_minNeighbors RESET reset_minNeighbors STORED false) BR_PROPERTY(br::Classifier*, classifier, NULL) BR_PROPERTY(int, minSize, 20) @@ -57,10 +56,11 @@ class SlidingWindowTransform : public MetaTransform BR_PROPERTY(float, scaleFactor, 1.2) BR_PROPERTY(float, confidenceThreshold, 10) BR_PROPERTY(float, eps, 0.2) + BR_PROPERTY(int, minNeighbors, 3) void train(const TemplateList &data) { - classifier->train(data.data(), File::get(data, "Label", -1)); + classifier->train(data); } void project(const Template &src, Template &dst) const @@ -123,15 +123,18 @@ class SlidingWindowTransform : public MetaTransform Mat scaledImage(scaledImageSize, CV_8U, imageBuffer.data); resize(m, scaledImage, scaledImageSize, 0, 0, CV_INTER_LINEAR); - Mat repImage = classifier->preprocess(scaledImage); + + Template repImage(t.file, scaledImage); + repImage = classifier->preprocess(repImage); int step = factor > 2. ? 1 : 2; for (int y = 0; y < processingRectSize.height; y += step) { for (int x = 0; x < processingRectSize.width; x += step) { - Mat window = repImage(Rect(Point(x, y), Size(originalWindowSize.width + dx, originalWindowSize.height + dy))).clone(); + Mat window = repImage.m()(Rect(Point(x, y), Size(originalWindowSize.width + dx, originalWindowSize.height + dy))).clone(); + Template t(window); float confidence = 0; - int result = classifier->classify(window, false, &confidence); + int result = classifier->classify(t, false, &confidence); if (result == 1) { rects.append(Rect(cvRound(x*factor), cvRound(y*factor), windowSize.width, windowSize.height)); @@ -146,7 +149,7 @@ class SlidingWindowTransform : public MetaTransform } } - OpenCVUtils::group(rects, confidences, confidenceThreshold, eps); + OpenCVUtils::group(rects, confidences, confidenceThreshold, minNeighbors, eps); if (!enrollAll && rects.empty()) { rects.append(Rect(0, 0, m.cols, m.rows)); diff --git a/openbr/plugins/metadata/averagepoints.cpp b/openbr/plugins/metadata/averagepoints.cpp index 704356d..09d2285 100644 --- a/openbr/plugins/metadata/averagepoints.cpp +++ b/openbr/plugins/metadata/averagepoints.cpp @@ -14,23 +14,14 @@ class AveragePointsTransform : public UntrainableMetadataTransform Q_PROPERTY(QList indices READ get_indices WRITE set_indices RESET reset_indices STORED false) Q_PROPERTY(QString metaName READ get_metaName WRITE set_metaName RESET reset_metaName STORED true) Q_PROPERTY(bool append READ get_append WRITE set_append RESET reset_append STORED true) - Q_PROPERTY(int nLandmarks READ get_nLandmarks WRITE set_nLandmarks RESET reset_nLandmarks STORED true) BR_PROPERTY(QList, indices, QList()) BR_PROPERTY(QString, metaName, "") BR_PROPERTY(bool, append, false) - BR_PROPERTY(int, nLandmarks, 51) void projectMetadata(const File &src, File &dst) const { dst = src; - if (src.points().size() != nLandmarks) { - if (Globals->verbose) - qDebug() << "Warning: Face has " << src.points().size() << "points; should be " << nLandmarks; - dst.fte = true; - return; - } - int x1 = 0, - y1 = 0; + float x1 = 0, y1 = 0; for (int i = 0; i < indices.size(); i++) { x1 += src.points()[indices[i]].x(); diff --git a/openbr/plugins/metadata/cascade.cpp b/openbr/plugins/metadata/cascade.cpp index cd983b9..8dd9a54 100644 --- a/openbr/plugins/metadata/cascade.cpp +++ b/openbr/plugins/metadata/cascade.cpp @@ -202,7 +202,8 @@ class CascadeTransform : public MetaTransform Q_PROPERTY(int minSize READ get_minSize WRITE set_minSize RESET reset_minSize STORED false) Q_PROPERTY(int minNeighbors READ get_minNeighbors WRITE set_minNeighbors RESET reset_minNeighbors STORED false) Q_PROPERTY(bool ROCMode READ get_ROCMode WRITE set_ROCMode RESET reset_ROCMode STORED false) - + Q_PROPERTY(float scaleFactor READ get_scaleFactor WRITE set_scaleFactor RESET reset_scaleFactor STORED false) + // Training parameters Q_PROPERTY(int numStages READ get_numStages WRITE set_numStages RESET reset_numStages STORED false) Q_PROPERTY(int w READ get_w WRITE set_w RESET reset_w STORED false) @@ -227,7 +228,8 @@ class CascadeTransform : public MetaTransform BR_PROPERTY(int, minSize, 64) BR_PROPERTY(int, minNeighbors, 5) BR_PROPERTY(bool, ROCMode, false) - + BR_PROPERTY(float, scaleFactor, 1.2) + // Training parameters - Default values provided trigger OpenCV defaults BR_PROPERTY(int, numStages, -1) BR_PROPERTY(int, w, -1) @@ -410,8 +412,19 @@ class CascadeTransform : public MetaTransform std::vector rects; std::vector rejectLevels; std::vector levelWeights; - if (ROCMode) cascade->detectMultiScale(m, rects, rejectLevels, levelWeights, 1.2, minNeighbors, flags | CASCADE_SCALE_IMAGE, Size(minSize, minSize), Size(), true); - else cascade->detectMultiScale(m, rects, 1.2, minNeighbors, flags, Size(minSize, minSize)); + if (ROCMode) cascade->detectMultiScale(m, rects, rejectLevels, levelWeights, scaleFactor, minNeighbors, flags | CASCADE_SCALE_IMAGE, Size(minSize, minSize), Size(), true); + else cascade->detectMultiScale(m, rects, scaleFactor, minNeighbors, flags, Size(minSize, minSize)); + + // It appears that flags is ignored for new model files: + // http://docs.opencv.org/modules/objdetect/doc/cascade_classification.html#cascadeclassifier-detectmultiscale + if ((flags == CASCADE_FIND_BIGGEST_OBJECT) && (rects.size() > 1)) { + Rect biggest = rects[0]; + for (size_t j=0; j biggest.area()) + biggest = rects[j]; + rects.clear(); + rects.push_back(biggest); + } bool empty = false; if (!enrollAll && rects.empty()) { @@ -426,7 +439,7 @@ class CascadeTransform : public MetaTransform u.file.set("Confidence",-std::numeric_limits::max()); } else if (rejectLevels.size() > j) u.file.set("Confidence", rejectLevels[j]*levelWeights[j]); - else + else u.file.set("Confidence", rects[j].area()); const QRectF rect = OpenCVUtils::fromRect(rects[j]); u.file.appendRect(rect); diff --git a/openbr/plugins/metadata/keytorect.cpp b/openbr/plugins/metadata/keytolandmark.cpp index c36ae4d..44556a7 100644 --- a/openbr/plugins/metadata/keytorect.cpp +++ b/openbr/plugins/metadata/keytolandmark.cpp @@ -24,28 +24,35 @@ namespace br * \brief Convert values of key_X, key_Y, key_Width, key_Height to a rect. * \author Jordan Cheney \cite JordanCheney */ -class KeyToRectTransform : public UntrainableMetadataTransform +class KeyToLandmarkTransform : public UntrainableMetadataTransform { Q_OBJECT Q_PROPERTY(QString key READ get_key WRITE set_key RESET reset_key STORED false) + Q_PROPERTY(bool point READ get_point WRITE set_point RESET reset_point STORED false) BR_PROPERTY(QString, key, "") + BR_PROPERTY(bool, point, false) void projectMetadata(const File &src, File &dst) const { dst = src; - if (src.contains(QStringList() << key + "_X" << key + "_Y" << key + "_Width" << key + "_Height")) - dst.appendRect(QRectF(src.get(key + "_X"), - src.get(key + "_Y"), - src.get(key + "_Width"), - src.get(key + "_Height"))); + if (point) { + if (src.contains(QStringList() << key + "_X" << key + "_Y")) + dst.appendPoint(QPointF(src.get(key + "_X"), + src.get(key + "_Y"))); + } else { + if (src.contains(QStringList() << key + "_X" << key + "_Y" << key + "_Width" << key + "_Height")) + dst.appendRect(QRectF(src.get(key + "_X"), + src.get(key + "_Y"), + src.get(key + "_Width"), + src.get(key + "_Height"))); + } } - }; -BR_REGISTER(Transform, KeyToRectTransform) +BR_REGISTER(Transform, KeyToLandmarkTransform) } // namespace br -#include "metadata/keytorect.moc" +#include "metadata/keytolandmark.moc" diff --git a/openbr/plugins/metadata/removetemplates.cpp b/openbr/plugins/metadata/removetemplates.cpp index ec8f76d..ac704f3 100644 --- a/openbr/plugins/metadata/removetemplates.cpp +++ b/openbr/plugins/metadata/removetemplates.cpp @@ -31,15 +31,22 @@ class RemoveTemplatesTransform : public UntrainableMetaTransform Q_OBJECT Q_PROPERTY(QString regexp READ get_regexp WRITE set_regexp RESET reset_regexp STORED false) Q_PROPERTY(QString key READ get_key WRITE set_key RESET reset_key STORED false) + Q_PROPERTY(bool keep READ get_keep WRITE set_keep RESET reset_keep STORED false) BR_PROPERTY(QString, regexp, "") BR_PROPERTY(QString, key, "") + BR_PROPERTY(bool, keep, false) void project(const Template &src, Template &dst) const { - const QRegularExpression re(regexp); - const QRegularExpressionMatch match = re.match(key.isEmpty() ? src.file.suffix() : src.file.get(key)); - if (match.hasMatch()) dst = Template(); - else dst = src; + dst = src; + QRegExp re(regexp); + re.setPatternSyntax(QRegExp::Wildcard); + bool match = re.exactMatch(key.isEmpty() ? src.file.suffix() : src.file.get(key)); + + if (keep && !match) + dst.file.fte = true; + else if (!keep && match) + dst.file.fte = true; } }; diff --git a/openbr/plugins/metadata/stasm4.cpp b/openbr/plugins/metadata/stasm4.cpp index 6cc0c06..4305156 100644 --- a/openbr/plugins/metadata/stasm4.cpp +++ b/openbr/plugins/metadata/stasm4.cpp @@ -1,6 +1,7 @@ #include -#include #include +#include +#include #include #include @@ -56,18 +57,18 @@ BR_REGISTER(Initializer, StasmInitializer) * \brief Wraps STASM key point detector * \author Scott Klum \cite sklum */ -class StasmTransform : public UntrainableTransform +class StasmTransform : public UntrainableMetaTransform { Q_OBJECT Q_PROPERTY(bool stasm3Format READ get_stasm3Format WRITE set_stasm3Format RESET reset_stasm3Format STORED false) BR_PROPERTY(bool, stasm3Format, false) - Q_PROPERTY(bool clearLandmarks READ get_clearLandmarks WRITE set_clearLandmarks RESET reset_clearLandmarks STORED false) - BR_PROPERTY(bool, clearLandmarks, false) Q_PROPERTY(QList pinPoints READ get_pinPoints WRITE set_pinPoints RESET reset_pinPoints STORED false) BR_PROPERTY(QList, pinPoints, QList()) Q_PROPERTY(QStringList pinLabels READ get_pinLabels WRITE set_pinLabels RESET reset_pinLabels STORED false) BR_PROPERTY(QStringList, pinLabels, QStringList()) + Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) + BR_PROPERTY(QString, inputVariable, "FrontalFace") Resource stasmCascadeResource; @@ -77,86 +78,119 @@ class StasmTransform : public UntrainableTransform stasmCascadeResource.setResourceMaker(new StasmResourceMaker()); } - void project(const Template &src, Template &dst) const + QList convertLandmarks(int nLandmarks, float *landmarks) const { - Mat stasmSrc(src); - if (src.m().channels() == 3) - cvtColor(src, stasmSrc, CV_BGR2GRAY); - else if (src.m().channels() != 1) - qFatal("Stasm expects single channel matrices."); - - if (!stasmSrc.isContinuous()) - qFatal("Stasm expects continuous matrix data."); - dst = src; - - int foundFace = 0; - int nLandmarks = stasm_NLANDMARKS; - float landmarks[2 * stasm_NLANDMARKS]; - - bool searchPinned = false; - - QPointF rightEye, leftEye; - /* Two use cases are accounted for: - * 1. Pin eyes without normalization: in this case the string list should contain the KEYS for right then left eyes, respectively. - * 2. Pin eyes with normalization: in this case the string list should contain the COORDINATES of the right then left eyes, respectively. - * Currently, we only support normalization with a transformation such that the src file contains Affine_0 and Affine_1. Checking for - * these keys prevents us from pinning eyes on a face that wasn't actually transformed (see AffineTransform). - * If both cases fail, we default to stasm_search_single. */ - - if (!pinPoints.isEmpty() && src.file.contains("Affine_0") && src.file.contains("Affine_1")) { - rightEye = QPointF(pinPoints.at(0), pinPoints.at(1)); - leftEye = QPointF(pinPoints.at(2), pinPoints.at(3)); - searchPinned = true; - } else if (!pinLabels.isEmpty()) { - rightEye = src.file.get(pinLabels.at(0), QPointF()); - leftEye = src.file.get(pinLabels.at(1), QPointF()); - searchPinned = true; - } - - if (searchPinned) { - float pins[2 * stasm_NLANDMARKS]; - - for (int i = 0; i < nLandmarks; i++) { - if (i == 38) /* Stasm Right Eye */ { pins[2*i] = rightEye.x(); pins[2*i+1] = rightEye.y(); } - else if (i == 39) /* Stasm Left Eye */ { pins[2*i] = leftEye.x(); pins[2*i+1] = leftEye.y(); } - else { pins[2*i] = 0; pins[2*i+1] = 0; } - } + if (stasm3Format) + stasm_convert_shape(landmarks, 76); - stasm_search_pinned(landmarks, pins, reinterpret_cast(stasmSrc.data), stasmSrc.cols, stasmSrc.rows, NULL); - - // The ASM in Stasm is guaranteed to converge in this case - foundFace = 1; + QList points; + for (int i = 0; i < nLandmarks; i++) { + QPointF point(landmarks[2 * i], landmarks[2 * i + 1]); + points.append(point); } - if (!foundFace) { - StasmCascadeClassifier *stasmCascade = stasmCascadeResource.acquire(); - stasm_search_single(&foundFace, landmarks, reinterpret_cast(stasmSrc.data), stasmSrc.cols, stasmSrc.rows, *stasmCascade, NULL, NULL); - stasmCascadeResource.release(stasmCascade); - } + return points; + } - if (stasm3Format) { - nLandmarks = 76; - stasm_convert_shape(landmarks, nLandmarks); - } + void project(const Template &src, Template &dst) const + { + TemplateList temp; + project(TemplateList() << src, temp); + if (!temp.isEmpty()) dst = temp.first(); + } - // For convenience, if these are the only points/rects we want to deal with as the algorithm progresses - if (clearLandmarks) { - dst.file.clearPoints(); - dst.file.clearRects(); - } + void project(const TemplateList &src, TemplateList &dst) const + { + foreach (const Template &t, src) { + Mat stasmSrc(t); + if (t.m().channels() == 3) + cvtColor(t, stasmSrc, CV_BGR2GRAY); + else if (t.m().channels() != 1) + qFatal("Stasm expects single channel matrices."); + + if (!stasmSrc.isContinuous()) + qFatal("Stasm expects continuous matrix data."); + + int foundFace = 0; + int nLandmarks = stasm_NLANDMARKS; + float landmarks[2 * stasm_NLANDMARKS]; + + bool searchPinned = false; + + QPointF rightEye, leftEye; + /* Two use cases are accounted for: + * 1. Pin eyes without normalization: in this case the string list should contain the KEYS for right then left eyes, respectively. + * 2. Pin eyes with normalization: in this case the string list should contain the COORDINATES of the right then left eyes, respectively. + * Currently, we only support normalization with a transformation such that the src file contains Affine_0 and Affine_1. Checking for + * these keys prevents us from pinning eyes on a face that wasn't actually transformed (see AffineTransform). + * If both cases fail, we default to stasm_search_single. */ + + if (!pinPoints.isEmpty() && t.file.contains("Affine_0") && t.file.contains("Affine_1")) { + rightEye = QPointF(pinPoints.at(0), pinPoints.at(1)); + leftEye = QPointF(pinPoints.at(2), pinPoints.at(3)); + searchPinned = true; + } else if (!pinLabels.isEmpty()) { + rightEye = t.file.get(pinLabels.at(0), QPointF()); + leftEye = t.file.get(pinLabels.at(1), QPointF()); + searchPinned = true; + } + + if (searchPinned) { + float pins[2 * stasm_NLANDMARKS]; + + for (int i = 0; i < nLandmarks; i++) { + if (i == 38) /* Stasm Right Eye */ { pins[2*i] = rightEye.x(); pins[2*i+1] = rightEye.y(); } + else if (i == 39) /* Stasm Left Eye */ { pins[2*i] = leftEye.x(); pins[2*i+1] = leftEye.y(); } + else { pins[2*i] = 0; pins[2*i+1] = 0; } + } + + stasm_search_pinned(landmarks, pins, reinterpret_cast(stasmSrc.data), stasmSrc.cols, stasmSrc.rows, NULL); + + // The ASM in Stasm is guaranteed to converge in this case + foundFace = 1; + Template u(t.file, t.m()); + QList points = convertLandmarks(nLandmarks, landmarks); + u.file.set("StasmRightEye", points[38]); + u.file.set("StasmLeftEye", points[39]); + u.file.appendPoints(points); + dst.append(u); + } - if (!foundFace) { - if (Globals->verbose) qWarning("No face found in %s.", qPrintable(src.file.fileName())); - dst.file.fte = true; - } else { - QList points; - for (int i = 0; i < nLandmarks; i++) { - QPointF point(landmarks[2 * i], landmarks[2 * i + 1]); - points.append(point); + if (!foundFace) { + StasmCascadeClassifier *stasmCascade = stasmCascadeResource.acquire(); + foundFace = 1; + stasm::FaceDet detection; + + QList rects = t.file.contains(inputVariable) ? QList() << t.file.get(inputVariable) : t.file.rects(); + if (!rects.isEmpty()) detection.iface_ = 0; + for (int i=0; i(stasmSrc.data), stasmSrc.cols, stasmSrc.rows, *stasmCascade, detection); + if (foundFace) { + Template u(t.file, t.m()); + QList points = convertLandmarks(nLandmarks, landmarks); + u.file.set("StasmRightEye", points[38]); + u.file.set("StasmLeftEye", points[39]); + u.file.appendPoints(points); + dst.append(u); + } + + if (!Globals->enrollAll) + break; + } + stasmCascadeResource.release(stasmCascade); } - dst.file.set("StasmRightEye", points[38]); - dst.file.set("StasmLeftEye", points[39]); - dst.file.appendPoints(points); } } }; diff --git a/openbr/plugins/output/knn.cpp b/openbr/plugins/output/knn.cpp index 0160ecb..bedd989 100644 --- a/openbr/plugins/output/knn.cpp +++ b/openbr/plugins/output/knn.cpp @@ -6,44 +6,80 @@ namespace br { +typedef QPair Pair; + /*! * \ingroup outputs * \brief Outputs the k-Nearest Neighbors from the gallery for each probe. * \author Ben Klein \cite bhklein */ -class knnOutput : public MatrixOutput +class knnOutput : public Output { Q_OBJECT + int rowBlock, columnBlock; + size_t headerSize, k; + cv::Mat blockScores; + ~knnOutput() { - size_t num_probes = (size_t)queryFiles.size(); - if (targetFiles.isEmpty() || queryFiles.isEmpty()) return; - size_t k = file.get("k", 20); + writeBlock(); + } - if ((size_t)targetFiles.size() < k) - qFatal("Gallery size %s is smaller than k = %s.", qPrintable(QString::number(targetFiles.size())), qPrintable(QString::number(k))); + void setBlock(int rowBlock, int columnBlock) + { + if ((rowBlock == 0) && (columnBlock == 0)) { + k = file.get("k", 20); + QFile f(file); + if (!f.open(QFile::WriteOnly)) + qFatal("Unable to open %s for writing.", qPrintable(file)); + size_t querySize = (size_t)queryFiles.size(); + f.write((const char*) &querySize, sizeof(size_t)); + f.write((const char*) &k, sizeof(size_t)); + headerSize = 2 * sizeof(size_t); + } else { + writeBlock(); + } - QFile f(file); - if (!f.open(QFile::WriteOnly)) - qFatal("Unable to open %s for writing.", qPrintable(file)); - f.write((const char*) &num_probes, sizeof(size_t)); - f.write((const char*) &k, sizeof(size_t)); + this->rowBlock = rowBlock; + this->columnBlock = columnBlock; + + int matrixRows = std::min(queryFiles.size()-rowBlock*this->blockRows, blockRows); + int matrixCols = std::min(targetFiles.size()-columnBlock*this->blockCols, blockCols); + + blockScores = cv::Mat(matrixRows, matrixCols, CV_32FC1); + } - QVector neighbors; neighbors.reserve(num_probes*k); + void setRelative(float value, int i, int j) + { + blockScores.at(i,j) = value; + } + + void set(float value, int i, int j) + { + (void) value; (void) i; (void) j; + qFatal("Logic error."); + } + + void writeBlock() + { + QFile f(file); + if (!f.open(QFile::ReadWrite)) + qFatal("Unable to open %s for modifying.", qPrintable(file)); + QVector neighbors; neighbors.reserve(k * blockScores.rows); - for (size_t i=0; i Pair; + for (int i=0; i(data.row(i)), true)) { - if (QString(targetFiles[pair.second]) != QString(queryFiles[i])) { + foreach (const Pair &pair, Common::Sort(OpenCVUtils::matrixToVector(blockScores.row(i)), true)) { + if (QString(targetFiles[pair.second]) != QString(queryFiles[rowBlock*this->blockRows+i])) { Candidate candidate((size_t)pair.second, pair.first); neighbors.push_back(candidate); if (++rank >= k) break; } } } - f.write((const char*) neighbors.data(), num_probes * k * sizeof(Candidate)); + f.seek(headerSize + sizeof(Candidate)*quint64(rowBlock*this->blockRows)*k); + f.write((const char*) neighbors.data(), blockScores.rows * k * sizeof(Candidate)); f.close(); } }; diff --git a/openbr/plugins/representation/haar.cpp b/openbr/plugins/representation/haar.cpp index c803aa3..fcf94d1 100644 --- a/openbr/plugins/representation/haar.cpp +++ b/openbr/plugins/representation/haar.cpp @@ -80,23 +80,25 @@ class HaarRepresentation : public Representation } } - void preprocess(const Mat &src, Mat &dst) const + Template preprocess(const Template &src) const { + Template dst; integral(src, dst); + return dst; } - float evaluate(const Mat &image, int idx) const + float evaluate(const Template &src, int idx) const { - return (float)features[idx].calc(image); + return (float)features[idx].calc(src.m()); } - Mat evaluate(const Mat &image, const QList &indices) const + Mat evaluate(const Template &src, const QList &indices) const { int size = indices.empty() ? numFeatures() : indices.size(); Mat result(1, size, CV_32FC1); for (int i = 0; i < size; i++) - result.at(i) = evaluate(image, indices.empty() ? i : indices[i]); + result.at(i) = evaluate(src, indices.empty() ? i : indices[i]); return result; } diff --git a/openbr/plugins/representation/mblbp.cpp b/openbr/plugins/representation/mblbp.cpp index 2a312f6..7f815ba 100644 --- a/openbr/plugins/representation/mblbp.cpp +++ b/openbr/plugins/representation/mblbp.cpp @@ -39,32 +39,36 @@ class MBLBPRepresentation : public Representation void init() { - int offset = winWidth + 1; - for (int x = 0; x < winWidth; x++ ) - for (int y = 0; y < winHeight; y++ ) - for (int w = 1; w <= winWidth / 3; w++ ) - for (int h = 1; h <= winHeight / 3; h++ ) - if ((x+3*w <= winWidth) && (y+3*h <= winHeight) ) - features.append(Feature(offset, x, y, w, h ) ); + if (features.isEmpty()) { + int offset = winWidth + 1; + for (int x = 0; x < winWidth; x++ ) + for (int y = 0; y < winHeight; y++ ) + for (int w = 1; w <= winWidth / 3; w++ ) + for (int h = 1; h <= winHeight / 3; h++ ) + if ((x+3*w <= winWidth) && (y+3*h <= winHeight) ) + features.append(Feature(offset, x, y, w, h ) ); + } } - void preprocess(const Mat &src, Mat &dst) const + Template preprocess(const Template &src) const { + Template dst; integral(src, dst); + return dst; } - float evaluate(const Mat &image, int idx) const + float evaluate(const Template &src, int idx) const { - return (float)features[idx].calc(image); + return (float)features[idx].calc(src.m()); } - Mat evaluate(const Mat &image, const QList &indices) const + Mat evaluate(const Template &src, const QList &indices) const { int size = indices.empty() ? numFeatures() : indices.size(); Mat result(1, size, CV_32FC1); for (int i = 0; i < size; i++) - result.at(i) = evaluate(image, indices.empty() ? i : indices[i]); + result.at(i) = evaluate(src, indices.empty() ? i : indices[i]); return result; } diff --git a/openbr/plugins/representation/npd.cpp b/openbr/plugins/representation/npd.cpp index 30f1c5e..90655f8 100644 --- a/openbr/plugins/representation/npd.cpp +++ b/openbr/plugins/representation/npd.cpp @@ -16,23 +16,25 @@ class NPDRepresentation : public Representation void init() { - for (int p1 = 0; p1 < (winWidth * winHeight); p1++) - for (int p2 = p1; p2 < (winWidth * winHeight); p2++) - features.append(Feature(p1, p2)); + if (features.isEmpty()) { + for (int p1 = 0; p1 < (winWidth * winHeight); p1++) + for (int p2 = p1; p2 < (winWidth * winHeight); p2++) + features.append(Feature(p1, p2)); + } } - float evaluate(const Mat &image, int idx) const + float evaluate(const Template &src, int idx) const { - return features[idx].calc(image); + return features[idx].calc(src.m()); } - Mat evaluate(const Mat &image, const QList &indices) const + Mat evaluate(const Template &src, const QList &indices) const { int size = indices.empty() ? numFeatures() : indices.size(); Mat result(1, size, CV_32FC1); for (int i = 0; i < size; i++) - result.at(i) = evaluate(image, indices.empty() ? i : indices[i]); + result.at(i) = evaluate(src, indices.empty() ? i : indices[i]); return result; } @@ -49,10 +51,10 @@ class NPDRepresentation : public Representation struct Feature { Feature() {} - Feature(int p1, int p2) { p[0] = p1; p[1] = p2; } + Feature(int p0, int p1) : p0(p0), p1(p1) {} float calc(const Mat &image) const; - int p[2]; + int p0, p1; }; QList features; }; @@ -61,9 +63,9 @@ BR_REGISTER(Representation, NPDRepresentation) inline float NPDRepresentation::Feature::calc(const Mat &image) const { - const int *ptr = image.ptr(); - int v1 = ptr[p[0]], v2 = ptr[p[1]]; - return v1 == 0 && v2 == 0 ? 0 : ((float)(v1 - v2)) / (v1 + v2); + const uchar *ptr = image.ptr(); + int v1 = (int)ptr[p0], v2 = (int)ptr[p1]; + return (v1 + v2) == 0 ? 0 : (1.0 * (v1 - v2)) / (v1 + v2); } } // namespace br diff --git a/openbr/plugins/representation/random.cpp b/openbr/plugins/representation/random.cpp index 6ac9fa2..1fc56bd 100644 --- a/openbr/plugins/representation/random.cpp +++ b/openbr/plugins/representation/random.cpp @@ -28,9 +28,9 @@ class RandomRepresentation : public Representation QList features; - void train(const QList &images, const QList &labels) + void train(const TemplateList &data) { - representation->train(images, labels); + representation->train(data); const int nFeatures = representation->numFeatures(); @@ -40,17 +40,17 @@ class RandomRepresentation : public Representation features = Common::RandSample(count,nFeatures,0,true); } - void preprocess(const Mat &src, Mat &dst) const + Template preprocess(const Template &src) const { - representation->preprocess(src,dst); + return representation->preprocess(src); } - float evaluate(const Mat &image, int idx) const + float evaluate(const Template &src, int idx) const { - return representation->evaluate(image,features[idx]); + return representation->evaluate(src, features[idx]); } - Mat evaluate(const Mat &image, const QList &indices) const + Mat evaluate(const Template &src, const QList &indices) const { QList newIndices; if (indices.empty()) @@ -59,7 +59,7 @@ class RandomRepresentation : public Representation for (int i = 0; i < indices.size(); i++) newIndices.append(features[indices[i]]); - return representation->evaluate(image,newIndices); + return representation->evaluate(src, newIndices); } int numFeatures() const @@ -108,6 +108,3 @@ BR_REGISTER(Representation, RandomRepresentation) } // namespace br #include "representation/random.moc" - - - diff --git a/scripts/face_cluster_viz.py b/scripts/brpy/face_cluster_viz.py index 7752c34..09b23de 100755 --- a/scripts/face_cluster_viz.py +++ b/scripts/brpy/face_cluster_viz.py @@ -1,7 +1,7 @@ #!/usr/bin/env python -from PIL import Image import csv, sys, json, argparse +from brpy.html_viz import crop_to_bb parser = argparse.ArgumentParser(description='Visualize face cluster results in an HTML page.') parser.add_argument('input_file', type=str, help='Results from clustering (in csv format)') @@ -16,24 +16,15 @@ clustmap = dict() with open(args.input_file) as f: for line in csv.DictReader(f): c = int(line[args.cluster_key]) + if c not in clustmap: + clustmap[c] = [] x,y,width,height = [ float(line[k]) for k in ('Face_X','Face_Y','Face_Width','Face_Height') ] imname = '%s/%s' % (args.img_loc, line['File']) try: - img = Image.open(imname) - imwidth, imheight = img.size + html = crop_to_bb(x,y,width,height,imname,maxheight=400) except IOError: print('problem with %s' % imname) continue - ratio = maxheight / height - # note for future me: - # image is cropped with div width/height + overflow:hidden, - # resized with img height, - # and positioned with img margin - html = '
' % (width*ratio, maxheight) - html += '' % (imname, imheight*ratio, y*ratio, x*ratio) - html += '
' - if c not in clustmap: - clustmap[c] = [] clustmap[c].append(html) # browsers crash for a DOM with this many img tags, diff --git a/scripts/brpy/html_viz.py b/scripts/brpy/html_viz.py new file mode 100644 index 0000000..665afcd --- /dev/null +++ b/scripts/brpy/html_viz.py @@ -0,0 +1,57 @@ +''' +Some funcs to generate HTML visualizations. +Run from the folder you intend to save the HTML page so +the relative paths in the HTML file are correct and PIL can find the images on disk. +Requires local images, but should be pretty easy to set up an apache server (or whatev) +and host them as long as the relative paths remain the same on ya serva. +''' + +from PIL import Image + +def crop_to_bb(x, y, width, height, imname, maxheight=None): + ''' + Generates an HTML string that crops to a given bounding box and resizes to maxheight pixels. + A maxheight of None will keep the original size (default). + When two crops are put next to each other, they will be inline. To make each crop its own line, wrap it in a div. + ''' + img = Image.open(imname) + imwidth, imheight = img.size + if not maxheight: + maxheight = height + ratio = maxheight / height + # note for future me: + # image is cropped with div width/height + overflow:hidden, + # resized with img height, + # and positioned with img margin + html = '
' % (width*ratio, maxheight) + html += '' % (imname, imheight*ratio, y*ratio, x*ratio) + html += '
' + return html + +def bbs_for_image(imname, bbs, maxheight=None, colors=None): + ''' + Generates an HTML string for an image with bounding boxes. + bbs: iterable of (x,y,width,height) bounding box tuples + ''' + img = Image.open(imname) + imwidth, imheight = img.size + if not maxheight: + maxheight = imheight + ratio = maxheight/imheight + html = [ + '
', + '' % (imname, maxheight) + ] + if not colors: + colors = ['green']*len(bbs) + html.extend([ bb(*box, ratio=ratio, color=color) for color,box in zip(colors,bbs) ]) + html.append('
') + return '\n'.join(html) + + +def bb(x, y, width, height, ratio=1.0, color='green'): + ''' + Generates an HTML string bounding box. + ''' + html = '
' + return html % (color, color, x*ratio, y*ratio, width*ratio, height*ratio) diff --git a/share/openbr/cmake/InstallDependencies.cmake b/share/openbr/cmake/InstallDependencies.cmake index 117f5e1..8828b16 100644 --- a/share/openbr/cmake/InstallDependencies.cmake +++ b/share/openbr/cmake/InstallDependencies.cmake @@ -86,12 +86,14 @@ endfunction() function(install_qt_platforms) if(${BR_INSTALL_DEPENDENCIES}) if(CMAKE_HOST_WIN32) - #TODO + install(FILES ${_qt5Core_install_prefix}/plugins/platforms/qwindows.dll + DESTINATION bin/platforms) elseif(CMAKE_HOST_APPLE) install(FILES ${_qt5Core_install_prefix}/plugins/platforms/libqcocoa.dylib DESTINATION bin/platforms) else() - #TODO + install(FILES ${_qt5Core_install_prefix}/plugins/platforms/libqlinuxfb.so + DESTINATION bin/platforms) endif() endif() endfunction() diff --git a/share/openbr/plotting/plot_utils.R b/share/openbr/plotting/plot_utils.R index ef49963..82a7f0c 100644 --- a/share/openbr/plotting/plot_utils.R +++ b/share/openbr/plotting/plot_utils.R @@ -217,6 +217,12 @@ formatData <- function(type="eval") { NormLength <<- data[grep("NormLength",data$Plot),-c(1)] sample <<- readImageData(Sample) rows <<- sample[[1]]$value + } else if (type == "knn") { + # Split data into individual plots + IET <<- data[grep("IET",data$Plot),-c(1)] + IET$Y <<- as.numeric(as.character(IET$Y)) + CMC <<- data[grep("CMC",data$Plot),-c(1)] + CMC$Y <<- as.numeric(as.character(CMC$Y)) } }