diff --git a/.gitignore b/.gitignore index f178bea..d265940 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,9 @@ share/openbr/models *.RData *.Rhistory +### Python ### +*.pyc + ### Subversion ### *.svn* diff --git a/CMakeLists.txt b/CMakeLists.txt index 11d905e..b3a1e15 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -86,7 +86,7 @@ set(BR_THIRDPARTY_LIBS ${BR_THIRDPARTY_LIBS} ${Qt5Core_QTMAIN_LIBRARIES}) # Find OpenCV find_package(OpenCV 2.4.5 REQUIRED) -set(OPENCV_DEPENDENCIES opencv_core opencv_highgui opencv_imgproc opencv_ml opencv_objdetect) +set(OPENCV_DEPENDENCIES opencv_core opencv_highgui opencv_imgproc opencv_ml) set(BR_THIRDPARTY_LIBS ${BR_THIRDPARTY_LIBS} ${OPENCV_DEPENDENCIES}) # Find Alphanum diff --git a/docs/docs/api_docs/python_api.md b/docs/docs/api_docs/python_api.md index 3cd7957..03059a7 100644 --- a/docs/docs/api_docs/python_api.md +++ b/docs/docs/api_docs/python_api.md @@ -46,6 +46,14 @@ The Python API is a light wrapper of the C API. It creates an object that has [a br.br_free_template_list(query) br.br_finalize() +Some functions were made more pythonic and thus differ slightly from the C API. In particular, the C functions that populate a string buffer have that logic abstracted away. For example, `br_get_filename(char * buffer, int buffer_length, br_template tmpl)` in C becomes `filename = br.br_get_filename(tmpl)` in Python. Here are the functions that differ from the C API: + +- br_most_recent_message +- br_objects +- br_scratch_path +- br_get_filename +- br_get_metadata_string + To enable the module, add `-DBR_INSTALL_BRPY=ON` to your cmake command (or use the ccmake GUI - highly recommended). Currently only OS X and Linux are supported. diff --git a/openbr/CMakeLists.txt b/openbr/CMakeLists.txt index bd07993..fa7b500 100644 --- a/openbr/CMakeLists.txt +++ b/openbr/CMakeLists.txt @@ -34,10 +34,7 @@ if(NOT BR_EMBEDDED) install(FILES ${HEADERS} DESTINATION include/openbr/gui) endif() -# normal BR library declaration - added openbr-cuda library -message(STATUS "BR_THIRDPARTY_SRC") -message(STATUS ${BR_THIRDPARTY_SRC}) -cuda_add_library(openbr SHARED ${SRC} ${BR_CORE} ${BR_JANUS} ${BR_GUI} ${BR_ICONS} ${BR_THIRDPARTY_SRC} ${BR_RESOURCES} ${NATURALSTRINGCOMPARE_SRC}) +cuda_add_library(openbr SHARED ${SRC} ${BR_CORE} ${BR_JANUS} ${BR_GUI} ${BR_ICONS} ${BR_THIRDPARTY_SRC} ${BR_RESOURCES} ${NATURALSTRINGCOMPARE_SRC} ${THIRDPARTY_RESOURCES}) qt5_use_modules(openbr ${QT_DEPENDENCIES}) set_target_properties(openbr PROPERTIES DEFINE_SYMBOL BR_LIBRARY diff --git a/openbr/core/bee.h b/openbr/core/bee.h index e4c7fa9..e6aab7a 100644 --- a/openbr/core/bee.h +++ b/openbr/core/bee.h @@ -34,21 +34,21 @@ namespace BEE const MaskValue DontCare(0x00); // Sigset - br::FileList readSigset(const br::File &sigset, bool ignoreMetadata = false); - void writeSigset(const QString &sigset, const br::FileList &files, bool ignoreMetadata = false); + BR_EXPORT br::FileList readSigset(const br::File &sigset, bool ignoreMetadata = false); + BR_EXPORT void writeSigset(const QString &sigset, const br::FileList &files, bool ignoreMetadata = false); // Matrix - cv::Mat readMatrix(const br::File &mat, QString *targetSigset = NULL, QString *querySigset = NULL); - void writeMatrix(const cv::Mat &m, const QString &fileName, const QString &targetSigset = "Unknown_Target", const QString &querySigset = "Unknown_Query"); - void readMatrixHeader(const QString &matrix, QString *targetSigset, QString *querySigset); - void writeMatrixHeader(const QString &matrix, const QString &targetSigset, const QString &querySigset); + BR_EXPORT cv::Mat readMatrix(const br::File &mat, QString *targetSigset = NULL, QString *querySigset = NULL); + BR_EXPORT void writeMatrix(const cv::Mat &m, const QString &fileName, const QString &targetSigset = "Unknown_Target", const QString &querySigset = "Unknown_Query"); + BR_EXPORT void readMatrixHeader(const QString &matrix, QString *targetSigset, QString *querySigset); + BR_EXPORT void writeMatrixHeader(const QString &matrix, const QString &targetSigset, const QString &querySigset); // Mask - void makeMask(const QString &targetInput, const QString &queryInput, const QString &mask); - cv::Mat makeMask(const br::FileList &targets, const br::FileList &queries, int partition = 0); - void makePairwiseMask(const QString &targetInput, const QString &queryInput, const QString &mask); - cv::Mat makePairwiseMask(const br::FileList &targets, const br::FileList &queries, int partition = 0); - void combineMasks(const QStringList &inputMasks, const QString &outputMask, const QString &method); + BR_EXPORT void makeMask(const QString &targetInput, const QString &queryInput, const QString &mask); + BR_EXPORT cv::Mat makeMask(const br::FileList &targets, const br::FileList &queries, int partition = 0); + BR_EXPORT void makePairwiseMask(const QString &targetInput, const QString &queryInput, const QString &mask); + BR_EXPORT cv::Mat makePairwiseMask(const br::FileList &targets, const br::FileList &queries, int partition = 0); + BR_EXPORT void combineMasks(const QStringList &inputMasks, const QString &outputMask, const QString &method); } #endif // BEE_BEE_H diff --git a/openbr/core/boost.cpp b/openbr/core/boost.cpp index 0f4e7fc..874f974 100644 --- a/openbr/core/boost.cpp +++ b/openbr/core/boost.cpp @@ -788,14 +788,17 @@ void CascadeBoostTrainData::precalculate() { int minNum = MIN( numPrecalcVal, numPrecalcIdx); - double proctime = -TIME( 0 ); + QTime time; + time.start(); + parallel_for_( Range(numPrecalcVal, numPrecalcIdx), FeatureIdxOnlyPrecalc(featureEvaluator, buf, sample_count, is_buf_16u!=0) ); parallel_for_( Range(0, minNum), FeatureValAndIdxPrecalc(featureEvaluator, buf, &valCache, sample_count, is_buf_16u!=0) ); parallel_for_( Range(minNum, numPrecalcVal), - FeatureValOnlyPrecalc(featureEvaluator, &valCache, sample_count) ); - cout << "Precalculation time: " << (proctime + TIME( 0 )) << endl; + FeatureValOnlyPrecalc(featureEvaluator, &valCache, sample_count) ); + + cout << "Precalculation time (ms): " << time.elapsed() << endl; } //-------------------------------- CascadeBoostTree ---------------------------------------- diff --git a/openbr/core/boost.h b/openbr/core/boost.h index 7a462a6..1b54e3a 100644 --- a/openbr/core/boost.h +++ b/openbr/core/boost.h @@ -4,12 +4,6 @@ #include "ml.h" #include -#ifdef _WIN32 -#define TIME( arg ) (((double) clock()) / CLOCKS_PER_SEC) -#else -#define TIME( arg ) (time( arg )) -#endif - namespace br { diff --git a/openbr/core/eval.cpp b/openbr/core/eval.cpp index a7d7edf..4cfcd5c 100755 --- a/openbr/core/eval.cpp +++ b/openbr/core/eval.cpp @@ -19,10 +19,13 @@ #include "openbr/core/common.h" #include "openbr/core/qtutils.h" #include "openbr/core/opencvutils.h" +#include "openbr/core/evalutils.h" #include #include +#include using namespace cv; +using namespace EvalUtils; namespace br { @@ -752,298 +755,6 @@ void EvalClassification(const QString &predictedGallery, const QString &truthGal qDebug("Overall Accuracy = %f", (float)tpc / (float)(tpc + fnc)); } -struct Detection -{ - QRectF boundingBox; - float confidence; - // The ignore flag is useful when certain faces in an image should be ignored - // and should not effect detection performance. Predicted detections that overlap - // with an ignored truth detection will not count as a true positive, false positive, - // true negative, or false negative, it will simply be ignored. - bool ignore; - - Detection() {} - Detection(const QRectF &boundingBox_, float confidence_ = -1, bool ignore_ = false) - : boundingBox(boundingBox_), confidence(confidence_), ignore(ignore_) {} - - float area() const { return boundingBox.width() * boundingBox.height(); } - float overlap(const Detection &other) const - { - const Detection intersection(boundingBox.intersected(other.boundingBox)); - return intersection.area() / (area() + other.area() - intersection.area()); - } -}; - -struct SortedDetection -{ - int truth_idx, predicted_idx; - float overlap; - SortedDetection() : truth_idx(-1), predicted_idx(-1), overlap(-1) {} - SortedDetection(int truth_idx_, int predicted_idx_, float overlap_) - : truth_idx(truth_idx_), predicted_idx(predicted_idx_), overlap(overlap_) {} - inline bool operator<(const SortedDetection &other) const { return overlap > other.overlap; } -}; - -struct Detections -{ - QList predicted, truth; -}; - -struct ResolvedDetection -{ - float confidence, overlap; - ResolvedDetection() : confidence(-1), overlap(-1) {} - ResolvedDetection(float confidence_, float overlap_) : confidence(confidence_), overlap(overlap_) {} - inline bool operator<(const ResolvedDetection &other) const { return confidence > other.confidence; } -}; - -struct DetectionOperatingPoint -{ - float Recall, FalsePositiveRate, Precision; - DetectionOperatingPoint() : Recall(-1), FalsePositiveRate(-1), Precision(-1) {} - DetectionOperatingPoint(float TP, float FP, float totalPositives, float numImages) - : Recall(TP/totalPositives), FalsePositiveRate(FP/numImages), Precision(TP/(TP+FP)) {} -}; - -static QStringList computeDetectionResults(const QList &detections, int totalTrueDetections, int numImages, bool discrete) -{ - QList points; - float TP = 0, FP = 0, prevFP = -1; - - for (int i=0; i= 0.5) TP++; - else FP++; - } else { - TP += detection.overlap; - FP += 1 - detection.overlap; - } - if ((i == detections.size()-1) || (detection.confidence > detections[i+1].confidence)) { - if (FP > prevFP || (i == detections.size()-1)) { - if (prevFP / numImages < 0.1 && FP / numImages > 0.1 && discrete) { - qDebug("TAR @ FAR => %f : 0.1", TP / totalTrueDetections); - qDebug("Confidence: %f", detection.confidence); - qDebug("TP vs. FP: %f to %f", TP, FP); - } else if (prevFP / numImages < 0.01 && FP / numImages > 0.01 && discrete) { - qDebug("TAR @ FAR => %f : 0.01", TP / totalTrueDetections); - qDebug("Confidence: %f", detection.confidence); - qDebug("TP vs. FP: %f to %f", TP, FP); - } else if (prevFP / numImages < 0.001 && FP / numImages > 0.001 && discrete) { - qDebug("TAR @ FAR => %f : 0.001", TP / totalTrueDetections); - qDebug("Confidence: %f", detection.confidence); - qDebug("TP vs. FP: %f to %f", TP, FP); - } - - points.append(DetectionOperatingPoint(TP, FP, totalTrueDetections, numImages)); - prevFP = FP; - } - } - } - - if (discrete) { - qDebug("Total TP vs. FP: %f to %f", TP, FP); - qDebug("Overall Recall (TP vs. possible TP): %f (%f vs. %d)", TP / totalTrueDetections, TP, totalTrueDetections); - } - - const int keep = qMin(points.size(), Max_Points); - if (keep < 1) qFatal("Insufficient points."); - - QStringList lines; lines.reserve(keep); - if (keep == 1) { - const DetectionOperatingPoint &point = points[0]; - lines.append(QString("%1ROC, %2, %3").arg(discrete ? "Discrete" : "Continuous", QString::number(point.FalsePositiveRate), QString::number(point.Recall))); - lines.append(QString("%1PR, %2, %3").arg(discrete ? "Discrete" : "Continuous", QString::number(point.Recall), QString::number(point.Precision))); - } else { - for (int i=0; i(key, QRectF()).isNull()) - return DetectionKey(key, DetectionKey::Rect); - - // and then multiple - if (!f.rects().empty()) - return DetectionKey("Rects", DetectionKey::RectList); - - // check for _X, _Y, _Width, _Height - foreach (const QString &localKey, localKeys) { - if (!localKey.endsWith("_X")) - continue; - const QString key = localKey.mid(0, localKey.size()-2); - if (localKeys.contains(key+"_Y") && - localKeys.contains(key+"_Width") && - localKeys.contains(key+"_Height")) - return DetectionKey(key, DetectionKey::XYWidthHeight); - } - - return DetectionKey(); -} - -// return a list of detections independent of the detection key format -static QList getDetections(const DetectionKey &key, const File &f, bool isTruth) -{ - QList dets; - if (key.type == DetectionKey::RectList) { - QList rects = f.rects(); - QList confidences = f.getList("Confidences", QList()); - if (!isTruth && rects.size() != confidences.size()) - qFatal("You don't have enough confidence. I mean, your detections don't all have confidence measures."); - for (int i=0; i(key), isTruth ? -1 : f.get("Confidence", -1))); - } else if (key.type == DetectionKey::XYWidthHeight) { - const QRectF rect(f.get(key+"_X"), f.get(key+"_Y"), f.get(key+"_Width"), f.get(key+"_Height")); - dets.append(Detection(rect, isTruth ? -1 : f.get("Confidence", -1), f.get("Ignore", false))); - } - return dets; -} - -static QMap getDetections(const File &predictedGallery, const File &truthGallery) -{ - const FileList predicted = TemplateList::fromGallery(predictedGallery).files(); - const FileList truth = TemplateList::fromGallery(truthGallery).files(); - - // Figure out which metadata field contains a bounding box - DetectionKey truthDetectKey = getDetectKey(truth); - if (truthDetectKey.isEmpty()) - qFatal("No suitable ground truth metadata key found."); - - DetectionKey predictedDetectKey = getDetectKey(predicted); - if (predictedDetectKey.isEmpty()) - qFatal("No suitable predicted metadata key found."); - - qDebug("Using metadata key: %s%s", - qPrintable(predictedDetectKey), - qPrintable(predictedDetectKey == truthDetectKey ? QString() : "/"+truthDetectKey)); - - QMap allDetections; - foreach (const File &f, truth) - allDetections[f.name].truth.append(getDetections(truthDetectKey, f, true)); - foreach (const File &f, predicted) - if (allDetections.contains(f.name)) allDetections[f.name].predicted.append(getDetections(predictedDetectKey, f, false)); - return allDetections; -} - -static inline int getNumberOfImages(const QMap detections) -{ - return detections.keys().size(); -} - -static int associateGroundTruthDetections(QList &resolved, QList &falseNegative, QMap &all, QRectF &offsets) -{ - float dLeftTotal = 0.0, dRightTotal = 0.0, dTopTotal = 0.0, dBottomTotal = 0.0; - int count = 0, totalTrueDetections = 0; - - foreach (Detections detections, all.values()) { - for (int i=0; i sortedDetections; sortedDetections.reserve(detections.truth.size() * detections.predicted.size()); - for (int t = 0; t < detections.truth.size(); t++) { - const Detection truth = detections.truth[t]; - for (int p = 0; p < detections.predicted.size(); p++) { - Detection predicted = detections.predicted[p]; - - float predictedWidth = predicted.boundingBox.width(); - float x, y, width, height; - x = predicted.boundingBox.x() + offsets.x()*predictedWidth; - y = predicted.boundingBox.y() + offsets.y()*predictedWidth; - width = predicted.boundingBox.width() - offsets.width()*predictedWidth; - height = predicted.boundingBox.height() - offsets.height()*predictedWidth; - Detection newPredicted(QRectF(x, y, width, height), 0.0); - - const float overlap = truth.overlap(newPredicted); - if (overlap > 0) - sortedDetections.append(SortedDetection(t, p, overlap)); - } - } - - std::sort(sortedDetections.begin(), sortedDetections.end()); - - QList removedTruth; - QList removedPredicted; - - foreach (const SortedDetection &detection, sortedDetections) { - if (removedTruth.contains(detection.truth_idx) || removedPredicted.contains(detection.predicted_idx)) - continue; - - const Detection truth = detections.truth[detection.truth_idx]; - const Detection predicted = detections.predicted[detection.predicted_idx]; - - if (!truth.ignore) resolved.append(ResolvedDetection(predicted.confidence, detection.overlap)); - - removedTruth.append(detection.truth_idx); - removedPredicted.append(detection.predicted_idx); - - if (offsets.x() == 0 && detection.overlap > 0.3) { - count++; - float width = predicted.boundingBox.width(); - dLeftTotal += (truth.boundingBox.left() - predicted.boundingBox.left()) / width; - dRightTotal += (truth.boundingBox.right() - predicted.boundingBox.right()) / width; - dTopTotal += (truth.boundingBox.top() - predicted.boundingBox.top()) / width; - dBottomTotal += (truth.boundingBox.bottom() - predicted.boundingBox.bottom()) / width; - } - } - - for (int i = 0; i < detections.predicted.size(); i++) - if (!removedPredicted.contains(i)) resolved.append(ResolvedDetection(detections.predicted[i].confidence, 0)); - for (int i = 0; i < detections.truth.size(); i++) - if (!removedTruth.contains(i) && !detections.truth[i].ignore) falseNegative.append(ResolvedDetection(-std::numeric_limits::max(), 0)); - } - - if (offsets.x() == 0) { - // Calculate average differences in each direction - float dRight = dRightTotal / count; - float dBottom = dBottomTotal / count; - float dX = dLeftTotal / count; - float dY = dTopTotal / count; - float dWidth = dX - dRight; - float dHeight = dY - dBottom; - - offsets.setX(dX); - offsets.setY(dY); - offsets.setWidth(dWidth); - offsets.setHeight(dHeight); - } - return totalTrueDetections; -} - float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv, bool normalize, int minSize, int maxSize) { qDebug("Evaluating detection of %s against %s", qPrintable(predictedGallery), qPrintable(truthGallery)); @@ -1053,51 +764,13 @@ float EvalDetection(const QString &predictedGallery, const QString &truthGallery // Remove any bounding boxes with a side smaller than minSize if (minSize > 0) { qDebug("Removing boxes smaller than %d\n", minSize); - QMap allFilteredDetections; - foreach (QString key, allDetections.keys()) { - Detections detections = allDetections[key]; - Detections filteredDetections; - for (int i = 0; i < detections.predicted.size(); i++) { - QRectF box = detections.predicted[i].boundingBox; - if (min(box.width(), box.height()) > sqrt(0.5 * pow(minSize, 2))) { - filteredDetections.predicted.append(detections.predicted[i]); - } - } - - for (int i = 0; i < detections.truth.size(); i++) { - QRectF box = detections.truth[i].boundingBox; - if (min(box.width(), box.height()) < minSize) - detections.truth[i].ignore = true; - filteredDetections.truth.append(detections.truth[i]); - } - if (!filteredDetections.truth.empty()) allFilteredDetections[key] = filteredDetections; - } - allDetections = allFilteredDetections; + allDetections = filterDetections(allDetections,minSize); } // Remove any bounding boxes with no side smaller than maxSize if (maxSize > 0) { qDebug("Removing boxes larger than %d\n", maxSize); - QMap allFilteredDetections; - foreach (QString key, allDetections.keys()) { - Detections detections = allDetections[key]; - Detections filteredDetections; - for (int i = 0; i < detections.predicted.size(); i++) { - QRectF box = detections.predicted[i].boundingBox; - if (min(box.width(), box.height()) < sqrt(0.5 * pow(maxSize, 2))) { - filteredDetections.predicted.append(detections.predicted[i]); - } - } - - for (int i = 0; i < detections.truth.size(); i++) { - QRectF box = detections.truth[i].boundingBox; - if (min(box.width(), box.height()) > maxSize) - detections.truth[i].ignore = true; - filteredDetections.truth.append(detections.truth[i]); - } - if (!filteredDetections.truth.empty()) allFilteredDetections[key] = filteredDetections; - } - allDetections = allFilteredDetections; + allDetections = filterDetections(allDetections,maxSize,false); } QList resolvedDetections, falseNegativeDetections; @@ -1117,11 +790,27 @@ float EvalDetection(const QString &predictedGallery, const QString &truthGallery falseNegativeDetections.clear(); totalTrueDetections = associateGroundTruthDetections(resolvedDetections, falseNegativeDetections, allDetections, normalizations); } + + if (Globals->verbose) { + qDebug("Total False negatives:"); + const int numFalseNegatives = 50; + for (int i=0; ipath + "/" + falseNegativeDetections[i].filePath)); + qDebug() << falseNegativeDetections[i]; + const Scalar color(0,255,0); + rectangle(img, OpenCVUtils::toRect(falseNegativeDetections[i].boundingBox), color, 1); + QtUtils::touchDir(QDir("./falseNegs")); + imwrite(qPrintable(QString("./falseNegs/falseNeg%1.jpg").arg(QString::number(i))), img); + } + } + std::sort(resolvedDetections.begin(), resolvedDetections.end()); QStringList lines; lines.append("Plot, X, Y"); - lines.append(computeDetectionResults(resolvedDetections, totalTrueDetections, getNumberOfImages(allDetections), true)); - lines.append(computeDetectionResults(resolvedDetections, totalTrueDetections, getNumberOfImages(allDetections), false)); + QList points; + lines.append(computeDetectionResults(resolvedDetections, totalTrueDetections, getNumberOfImages(allDetections), true, points)); + points.clear(); + lines.append(computeDetectionResults(resolvedDetections, totalTrueDetections, getNumberOfImages(allDetections), false, points)); float averageOverlap; { // Overlap Density @@ -1209,7 +898,7 @@ float EvalLandmarking(const QString &predictedGallery, const QString &truthGalle { QScopedPointer t(Transform::make("Open+Draw(verbose,rects=false,location=false)",NULL)); - QString filePath = "landmarking_examples_truth/"+truth[sampleIndex].file.fileName(); + QString filePath = "landmarking_examples_truth/sample.jpg"; projectAndWrite(t.data(), truth[sampleIndex],filePath); lines.append("Sample,"+filePath+","+QString::number(truth[sampleIndex].file.points().size())); } @@ -1217,26 +906,26 @@ float EvalLandmarking(const QString &predictedGallery, const QString &truthGalle // Get best and worst performing examples QList< QPair > exampleIndices = Common::Sort(imageErrors,true); - QScopedPointer t(Transform::make("Open+Draw(rects=false)",NULL)); + QScopedPointer t(Transform::make("Open+CropFromLandmarks(paddingHorizontal=.3,paddingVertical=.3,shiftPoints=true)+Resize(128,method=Area)+Draw(rects=false,pointRadius=2)",NULL)); for (int i=0; iexampleIndices.size()-totalExamples-1; i--) { QString filePath = "landmarking_examples_truth/"+truth[exampleIndices[i].second].file.fileName(); projectAndWrite(t.data(), truth[exampleIndices[i].second],filePath); - lines.append("EXT,"+filePath+","+QString::number(exampleIndices[i].first)); + lines.append("EXT,"+filePath+":"+truth[exampleIndices[i].second].file.name+","+QString::number(exampleIndices[i].first)); filePath = "landmarking_examples_predicted/"+predicted[exampleIndices[i].second].file.fileName(); projectAndWrite(t.data(), predicted[exampleIndices[i].second],filePath); - lines.append("EXP,"+filePath+","+QString::number(exampleIndices[i].first)); + lines.append("EXP,"+filePath+":"+predicted[exampleIndices[i].second].file.name+","+QString::number(exampleIndices[i].first)); } for (int i=0; i + +using namespace std; +using namespace br; +using namespace cv; + +static const int Max_Points = 500; // Maximum number of points to render on plots + +DetectionKey EvalUtils::getDetectKey(const FileList &files) +{ + if (files.empty()) + return DetectionKey(); + + const File &f = files.first(); + const QStringList localKeys = f.localKeys(); + + // first check for single detections + foreach (const QString &key, localKeys) + if (!f.get(key, QRectF()).isNull()) + return DetectionKey(key, DetectionKey::Rect); + + // and then multiple + if (!f.rects().empty()) + return DetectionKey("Rects", DetectionKey::RectList); + + // check for _X, _Y, _Width, _Height + foreach (const QString &localKey, localKeys) { + if (!localKey.endsWith("_X")) + continue; + const QString key = localKey.mid(0, localKey.size()-2); + if (localKeys.contains(key+"_Y") && + localKeys.contains(key+"_Width") && + localKeys.contains(key+"_Height")) + return DetectionKey(key, DetectionKey::XYWidthHeight); + } + + return DetectionKey(); +} + +// return a list of detections independent of the detection key format +QList EvalUtils::getDetections(const DetectionKey &key, const File &f, bool isTruth) +{ + const QString filePath = f.path() + "/" + f.fileName(); + QList dets; + if (key.type == DetectionKey::RectList) { + QList rects = f.rects(); + QList confidences = f.getList("Confidences", QList()); + if (!isTruth && rects.size() != confidences.size()) + qFatal("You don't have enough confidence. I mean, your detections don't all have confidence measures."); + for (int i=0; i(key), filePath, isTruth ? -1 : f.get("Confidence", -1))); + } else if (key.type == DetectionKey::XYWidthHeight) { + const QRectF rect(f.get(key+"_X"), f.get(key+"_Y"), f.get(key+"_Width"), f.get(key+"_Height")); + dets.append(Detection(rect, filePath, isTruth ? -1 : f.get("Confidence", -1), f.get("Ignore", false))); + } + return dets; +} + +QMap EvalUtils::getDetections(const File &predictedGallery, const File &truthGallery) +{ + const FileList predicted = TemplateList::fromGallery(predictedGallery).files(); + const FileList truth = TemplateList::fromGallery(truthGallery).files(); + + // Figure out which metadata field contains a bounding box + DetectionKey truthDetectKey = getDetectKey(truth); + if (truthDetectKey.isEmpty()) + qFatal("No suitable ground truth metadata key found."); + + DetectionKey predictedDetectKey = getDetectKey(predicted); + if (predictedDetectKey.isEmpty()) + qFatal("No suitable predicted metadata key found."); + + qDebug("Using metadata key: %s%s", + qPrintable(predictedDetectKey), + qPrintable(predictedDetectKey == truthDetectKey ? QString() : "/"+truthDetectKey)); + + QMap allDetections; + foreach (const File &f, truth) + allDetections[f.name].truth.append(getDetections(truthDetectKey, f, true)); + foreach (const File &f, predicted) + if (allDetections.contains(f.name)) allDetections[f.name].predicted.append(getDetections(predictedDetectKey, f, false)); + return allDetections; +} + +QMap EvalUtils::filterDetections(const QMap &allDetections, int threshold, bool useMin) +{ + QMap allFilteredDetections; + foreach (QString key, allDetections.keys()) { + Detections detections = allDetections[key]; + Detections filteredDetections; + for (int i = 0; i < detections.predicted.size(); i++) { + const QRectF box = detections.predicted[i].boundingBox; + const qreal minBoxDim = min(box.width(), box.height()); + const double t = sqrt(0.5 * pow(threshold, 2)); + if (useMin ? minBoxDim > t : minBoxDim < t) + filteredDetections.predicted.append(detections.predicted[i]); + } + + for (int i = 0; i < detections.truth.size(); i++) { + const QRectF box = detections.truth[i].boundingBox; + const qreal minBoxDim = min(box.width(), box.height()); + if (useMin ? minBoxDim < threshold : minBoxDim > threshold) + detections.truth[i].ignore = true; + filteredDetections.truth.append(detections.truth[i]); + } + if (!filteredDetections.truth.empty()) allFilteredDetections[key] = filteredDetections; + } + return allFilteredDetections; +} + +int EvalUtils::associateGroundTruthDetections(QList &resolved, QList &falseNegative, QMap &all, QRectF &offsets) +{ + float dLeftTotal = 0.0, dRightTotal = 0.0, dTopTotal = 0.0, dBottomTotal = 0.0; + int count = 0, totalTrueDetections = 0; + + foreach (Detections detections, all.values()) { + for (int i=0; i sortedDetections; sortedDetections.reserve(detections.truth.size() * detections.predicted.size()); + for (int t = 0; t < detections.truth.size(); t++) { + const Detection truth = detections.truth[t]; + for (int p = 0; p < detections.predicted.size(); p++) { + Detection predicted = detections.predicted[p]; + + float predictedWidth = predicted.boundingBox.width(); + float x, y, width, height; + x = predicted.boundingBox.x() + offsets.x()*predictedWidth; + y = predicted.boundingBox.y() + offsets.y()*predictedWidth; + width = predicted.boundingBox.width() - offsets.width()*predictedWidth; + height = predicted.boundingBox.height() - offsets.height()*predictedWidth; + Detection newPredicted(QRectF(x, y, width, height), predicted.filePath, 0.0); + + const float overlap = truth.overlap(newPredicted); + + if (overlap > 0) + sortedDetections.append(SortedDetection(t, p, overlap)); + } + } + + std::sort(sortedDetections.begin(), sortedDetections.end()); + + QList removedTruth; + QList removedPredicted; + + foreach (const SortedDetection &detection, sortedDetections) { + if (removedTruth.contains(detection.truth_idx) || removedPredicted.contains(detection.predicted_idx)) + continue; + + const Detection truth = detections.truth[detection.truth_idx]; + const Detection predicted = detections.predicted[detection.predicted_idx]; + + if (!truth.ignore) + resolved.append(ResolvedDetection(predicted.filePath, predicted.boundingBox, predicted.confidence, detection.overlap)); + + removedTruth.append(detection.truth_idx); + removedPredicted.append(detection.predicted_idx); + + if (offsets.x() == 0 && detection.overlap > 0.3) { + count++; + float width = predicted.boundingBox.width(); + dLeftTotal += (truth.boundingBox.left() - predicted.boundingBox.left()) / width; + dRightTotal += (truth.boundingBox.right() - predicted.boundingBox.right()) / width; + dTopTotal += (truth.boundingBox.top() - predicted.boundingBox.top()) / width; + dBottomTotal += (truth.boundingBox.bottom() - predicted.boundingBox.bottom()) / width; + } + } + + for (int i = 0; i < detections.predicted.size(); i++) + if (!removedPredicted.contains(i)) resolved.append(ResolvedDetection(detections.predicted[i].filePath, detections.predicted[i].boundingBox, detections.predicted[i].confidence, 0)); + for (int i = 0; i < detections.truth.size(); i++) + if (!removedTruth.contains(i) && !detections.truth[i].ignore) falseNegative.append(ResolvedDetection(detections.truth[i].filePath, detections.truth[i].boundingBox, -std::numeric_limits::max(), 0)); + } + + if (offsets.x() == 0) { + // Calculate average differences in each direction + float dRight = dRightTotal / count; + float dBottom = dBottomTotal / count; + float dX = dLeftTotal / count; + float dY = dTopTotal / count; + float dWidth = dX - dRight; + float dHeight = dY - dBottom; + + offsets.setX(dX); + offsets.setY(dY); + offsets.setWidth(dWidth); + offsets.setHeight(dHeight); + } + return totalTrueDetections; +} + +QStringList EvalUtils::computeDetectionResults(const QList &detections, int totalTrueDetections, int numImages, bool discrete, QList &points) +{ + float TP = 0, FP = 0, prevFP = -1, prevTP = -1; + + const int detectionsToKeep = 50; + QList topFalsePositives, bottomTruePositives; + for (int i=0; i= 0.5) TP++; + else FP++; + } else { + TP += detection.overlap; + FP += 1 - detection.overlap; + } + if ((i == detections.size()-1) || (detection.confidence > detections[i+1].confidence)) { + if (FP > prevFP || (i == detections.size()-1)) { + if (prevFP / numImages < 1 && FP / numImages >= 1 && discrete) { + qDebug("TAR @ FAR => %f : 1", TP / totalTrueDetections); + qDebug("Confidence: %f", detection.confidence); + qDebug("TP vs. FP: %f to %f", TP, FP); + } else if (prevFP / numImages < 0.5 && FP / numImages >= 0.5 && discrete) { + qDebug("TAR @ FAR => %f : 0.5", TP / totalTrueDetections); + qDebug("Confidence: %f", detection.confidence); + qDebug("TP vs. FP: %f to %f", TP, FP); + } else if (prevFP / numImages < 0.2 && FP / numImages >= 0.2 && discrete) { + qDebug("TAR @ FAR => %f : 0.2", TP / totalTrueDetections); + qDebug("Confidence: %f", detection.confidence); + qDebug("TP vs. FP: %f to %f", TP, FP); + } else if (prevFP / numImages < 0.1 && FP / numImages >= 0.1 && discrete) { + qDebug("TAR @ FAR => %f : 0.1", TP / totalTrueDetections); + qDebug("Confidence: %f", detection.confidence); + qDebug("TP vs. FP: %f to %f", TP, FP); + } else if (prevFP / numImages < 0.02 && FP / numImages >= 0.02 && discrete) { + qDebug("TAR @ FAR => %f : 0.02", TP / totalTrueDetections); + qDebug("Confidence: %f", detection.confidence); + qDebug("TP vs. FP: %f to %f", TP, FP); + } else if (prevFP / numImages < 0.01 && FP / numImages >= 0.01 && discrete) { + qDebug("TAR @ FAR => %f : 0.01", TP / totalTrueDetections); + qDebug("Confidence: %f", detection.confidence); + qDebug("TP vs. FP: %f to %f", TP, FP); + } else if (prevFP / numImages < 0.001 && FP / numImages >= 0.001 && discrete) { + qDebug("TAR @ FAR => %f : 0.001", TP / totalTrueDetections); + qDebug("Confidence: %f", detection.confidence); + qDebug("TP vs. FP: %f to %f", TP, FP); + } + + if (detection.overlap < 0.5 && topFalsePositives.size() < detectionsToKeep) + topFalsePositives.append(detection); + + points.append(DetectionOperatingPoint(TP, FP, totalTrueDetections, numImages, detection.confidence)); + prevFP = FP; + } + + if (TP > prevTP) { + bottomTruePositives.append(detection); + if (bottomTruePositives.size() > detectionsToKeep) + bottomTruePositives.removeFirst(); + prevTP = TP; + } + } + } + + if (discrete) { + qDebug("Total TP vs. FP: %f to %f", TP, FP); + qDebug("Overall Recall (TP vs. possible TP): %f (%f vs. %d)", TP / totalTrueDetections, TP, totalTrueDetections); + + if (Globals->verbose) { + QtUtils::touchDir(QDir("./falsePos")); + qDebug("Highest Scoring False Positives:"); + for (int i=0; ipath + "/" + topFalsePositives[i].filePath)); + qDebug() << topFalsePositives[i]; + const Scalar color(0,255,0); + rectangle(img, OpenCVUtils::toRect(topFalsePositives[i].boundingBox), color, 1); + imwrite(qPrintable(QString("./falsePos/falsePos%1.jpg").arg(QString::number(i))), img); + } + qDebug("Lowest Scoring True Positives:"); + qDebug() << bottomTruePositives; + } + } + + const int keep = qMin(points.size(), Max_Points); + if (keep < 1) qFatal("Insufficient points."); + + QStringList lines; lines.reserve(keep); + if (keep == 1) { + const DetectionOperatingPoint &point = points[0]; + lines.append(QString("%1ROC, %2, %3").arg(discrete ? "Discrete" : "Continuous", QString::number(point.FalsePositiveRate), QString::number(point.Recall))); + lines.append(QString("%1PR, %2, %3").arg(discrete ? "Discrete" : "Continuous", QString::number(point.Recall), QString::number(point.Precision))); + } else { + for (int i=0; i + +struct Detection +{ + QRectF boundingBox; + QString filePath; + float confidence; + // The ignore flag is useful when certain faces in an image should be ignored + // and should not effect detection performance. Predicted detections that overlap + // with an ignored truth detection will not count as a true positive, false positive, + // true negative, or false negative, it will simply be ignored. + bool ignore; + + Detection() {} + Detection(const QRectF &boundingBox_, const QString &filePath = QString(), float confidence_ = -1, bool ignore_ = false) + : boundingBox(boundingBox_), filePath(filePath), confidence(confidence_), ignore(ignore_) {} + + float area() const { return boundingBox.width() * boundingBox.height(); } + float overlap(const Detection &other) const + { + const Detection intersection(boundingBox.intersected(other.boundingBox)); + return intersection.area() / (area() + other.area() - intersection.area()); + } +}; + +struct SortedDetection +{ + int truth_idx, predicted_idx; + float overlap; + SortedDetection() : truth_idx(-1), predicted_idx(-1), overlap(-1) {} + SortedDetection(int truth_idx_, int predicted_idx_, float overlap_) + : truth_idx(truth_idx_), predicted_idx(predicted_idx_), overlap(overlap_) {} + inline bool operator<(const SortedDetection &other) const { return overlap > other.overlap; } +}; + +struct ResolvedDetection +{ + QString filePath; + QRectF boundingBox; + float confidence, overlap; + ResolvedDetection() : confidence(-1), overlap(-1) {} + ResolvedDetection(const QString &filePath, const QRectF &boundingBox, float confidence_, float overlap_) : + filePath(filePath), boundingBox(boundingBox), confidence(confidence_), overlap(overlap_) {} + inline bool operator<(const ResolvedDetection &other) const { return confidence > other.confidence; } +}; + +struct Detections +{ + QList predicted, truth; +}; + +struct DetectionKey : public QString +{ + enum Type { + Invalid, + Rect, + RectList, + XYWidthHeight + } type; + + DetectionKey(const QString &key = "", Type type = Invalid) + : QString(key), type(type) {} +}; + +struct DetectionOperatingPoint +{ + float Recall, FalsePositiveRate, Precision, Confidence; + DetectionOperatingPoint() : Recall(-1), FalsePositiveRate(-1), Precision(-1) {} + DetectionOperatingPoint(float TP, float FP, float totalPositives, float numImages, float confidence) + : Recall(TP/totalPositives), FalsePositiveRate(FP/numImages), Precision(TP/(TP+FP)), Confidence(confidence) {} +}; + +namespace EvalUtils +{ + // Detection + DetectionKey getDetectKey(const br::FileList &files); + QList getDetections(const DetectionKey &key, const br::File &f, bool isTruth); + QMap getDetections(const br::File &predictedGallery, const br::File &truthGallery); + QMap filterDetections(const QMap &allDetections, int threshold, bool useMin=true); + int associateGroundTruthDetections(QList &resolved, QList &falseNegative, QMap &all, QRectF &offsets); + QStringList computeDetectionResults(const QList &detections, int totalTrueDetections, int numImages, bool discrete, QList &points); + inline int getNumberOfImages(const QMap detections) + { + return detections.keys().size(); + } +} + +QDebug operator<<(QDebug dbg, const ResolvedDetection &d); + +#endif // EVALUTILS_EVALUTILS_H diff --git a/openbr/core/opencvutils.cpp b/openbr/core/opencvutils.cpp index d1a0981..630e512 100644 --- a/openbr/core/opencvutils.cpp +++ b/openbr/core/opencvutils.cpp @@ -436,7 +436,7 @@ public: }; // TODO: Make sure case where no confidences are inputted works. -void OpenCVUtils::group(QList &rects, QList &confidences, float confidenceThreshold, int minNeighbors, float epsilon) +void OpenCVUtils::group(QList &rects, QList &confidences, float confidenceThreshold, int minNeighbors, float epsilon, bool useMax, QList *maxIndices) { if (rects.isEmpty()) return; @@ -448,8 +448,9 @@ void OpenCVUtils::group(QList &rects, QList &confidences, float con vector rrects(nClasses); // Total number of rects in each class - vector neighbors(nClasses, 0); - vector classConfidence(nClasses, 0); + vector neighbors(nClasses, -1); + vector classConfidence(nClasses, useMax ? -std::numeric_limits::max() : 0); + vector classMax(nClasses, 0); for (size_t i = 0; i < labels.size(); i++) { @@ -459,18 +460,26 @@ void OpenCVUtils::group(QList &rects, QList &confidences, float con rrects[cls].width += rects[i].width; rrects[cls].height += rects[i].height; neighbors[cls]++; - classConfidence[cls] += confidences[i]; + if (useMax) { + if (confidences[i] > classConfidence[cls]) { + classConfidence[cls] = confidences[i]; + classMax[cls] = i; + } + } else + classConfidence[cls] += confidences[i]; } // Find average rectangle for all classes for (int i = 0; i < nClasses; i++) { - Rect r = rrects[i]; - float s = 1.f/neighbors[i]; - rrects[i] = Rect(saturate_cast(r.x*s), - saturate_cast(r.y*s), - saturate_cast(r.width*s), - saturate_cast(r.height*s)); + if (neighbors[i] > 0) { + Rect r = rrects[i]; + float s = 1.f/(neighbors[i]+1); + rrects[i] = Rect(saturate_cast(r.x*s), + saturate_cast(r.y*s), + saturate_cast(r.width*s), + saturate_cast(r.height*s)); + } } rects.clear(); @@ -480,7 +489,7 @@ void OpenCVUtils::group(QList &rects, QList &confidences, float con for (int i = 0; i < nClasses; i++) { // Average rectangle - Rect r1 = rrects[i]; + const Rect r1 = rrects[i]; // Used to eliminate rectangles with few neighbors in the case of no weights const float w1 = classConfidence[i]; @@ -501,21 +510,19 @@ void OpenCVUtils::group(QList &rects, QList &confidences, float con if (j == i || n2 < minNeighbors) continue; - Rect r2 = rrects[j]; + const Rect r2 = rrects[j]; - int dx = saturate_cast(r2.width * epsilon); - int dy = saturate_cast(r2.height * epsilon); + const int dx = saturate_cast(r2.width * epsilon); + const int dy = saturate_cast(r2.height * epsilon); - float w2 = classConfidence[j]; + const float w2 = classConfidence[j]; - // If, r1 is within the r2 AND - // r2 has a higher confidence than r1 - // then, eliminate the r1 if(r1.x >= r2.x - dx && r1.y >= r2.y - dy && r1.x + r1.width <= r2.x + r2.width + dx && r1.y + r1.height <= r2.y + r2.height + dy && - (w2 > std::max(confidenceThreshold, w1))) + (w2 > w1) && + (n2 > n1)) break; } @@ -523,11 +530,49 @@ void OpenCVUtils::group(QList &rects, QList &confidences, float con { rects.append(r1); confidences.append(w1); + if (maxIndices) + maxIndices->append(classMax[i]); } } } -void OpenCVUtils::rotate(const br::Template &src, br::Template &dst, int degrees, bool rotateMat, bool rotatePoints, bool rotateRects) +void OpenCVUtils::pad(const br::Template &src, br::Template &dst, bool padMat, const QList &padding, bool padPoints, bool padRects, int border, int value) +{ + // Padding is expected to be top, bottom, left, right + if (padMat) + copyMakeBorder(src, dst, padding[0], padding[1], padding[2], padding[3], border, Scalar(value)); + else + dst = src; + + if (padPoints) { + QList points = src.file.points(); + QList paddedPoints; + for (int i=0; i rects = src.file.rects(); + QList paddedRects; + for (int i=0; i &padding, bool padPoints, bool padRects, int border, int value) +{ + for (int i=0; i &rects, QList &confidences, float confidenceThreshold, int minNeighbors, float epsilon); - void rotate(const br::Template &src, br::Template &dst, int degrees, bool rotateMat=true, bool rotatePoints=true, bool rotateRects=true); - void rotate(const br::TemplateList &src, br::TemplateList &dst, int degrees, bool rotateMat=true, bool rotatePoint=true, bool rotateRects=true); + void group(QList &rects, QList &confidences, float confidenceThreshold, int minNeighbors, float epsilon, bool useMax=false, QList *maxIndices=NULL); + void pad(const br::Template &src, br::Template &dst, bool padMat, const QList &padding, bool padPoints, bool padRects, int border=0, int value=0); + void pad(const br::TemplateList &src, br::TemplateList &dst, bool padMat, const QList &padding, bool padPoints, bool padRects, int border=0, int value=0); + void rotate(const br::Template &src, br::Template &dst, float degrees, bool rotateMat=true, bool rotatePoints=true, bool rotateRects=true); + void rotate(const br::TemplateList &src, br::TemplateList &dst, float degrees, bool rotateMat=true, bool rotatePoint=true, bool rotateRects=true); void flip(const br::Template &src, br::Template &dst, int axis, bool flipMat=true, bool flipPoints=true, bool flipRects=true); void flip(const br::TemplateList &src, br::TemplateList &dst, int axis, bool flipMat=true, bool flipPoints=true, bool flipRects=true); diff --git a/openbr/core/plot.cpp b/openbr/core/plot.cpp index 889911e..ef56e14 100644 --- a/openbr/core/plot.cpp +++ b/openbr/core/plot.cpp @@ -318,9 +318,9 @@ bool PlotLandmarking(const QStringList &files, const File &destination, bool sho qDebug("Plotting %d landmarking file(s) to %s", files.size(), qPrintable(destination)); RPlot p(files, destination); p.file.write("\nformatData(type=\"landmarking\")\n\n"); - p.file.write(qPrintable(QString("algs <- uniqueBox$%1)\n").arg(p.major.size > 1 ? p.major.header : (p.minor.header.isEmpty() ? p.major.header : p.minor.header)))); + p.file.write(qPrintable(QString("algs <- unique(Box$%1)\n").arg(p.major.size > 1 ? p.major.header : (p.minor.header.isEmpty() ? p.major.header : p.minor.header)))); p.file.write("algs <- algs[!duplicated(algs)]\n"); - p.file.write("plotLandmarkSamples(samples=sample, expData=EXP, extData=EXT)\n"); + p.file.write("plotLandmarkSamples(displaySample=displaySample, expData=EXP, extData=EXT)\n"); p.file.write("plotLandmarkTables(tableData=Box)\n"); p.file.write(qPrintable(QString("ggplot(Box, aes(Y,%1%2))").arg(p.major.size > 1 ? QString(", colour=%1").arg(p.major.header) : QString(), @@ -330,9 +330,6 @@ bool PlotLandmarking(const QStringList &files, const File &destination, bool sho p.file.write(qPrintable(QString("ggplot(Box, aes(factor(X), Y%1%2))").arg(p.major.size > 1 ? QString(", colour=%1").arg(p.major.header) : QString(), p.minor.size > 1 ? QString(", linetype=%1").arg(p.minor.header) : QString()) + QString("+ annotation_logticks(sides=\"l\") + geom_boxplot(alpha=0.5) + geom_jitter(size=1, alpha=0.5) + scale_x_discrete(\"Landmark\") + scale_y_log10(\"Normalized Error\", breaks=c(0.001,0.01,0.1,1,10)) + theme_minimal()\n\n"))); - p.file.write(qPrintable(QString("ggplot(Box, aes(factor(X), Y%1%2))").arg(p.major.size > 1 ? QString(", colour=%1").arg(p.major.header) : QString(), p.minor.size > 1 ? QString(", linetype=%1").arg(p.minor.header) : QString()) + - QString("+ annotation_logticks(sides=\"l\") + geom_violin(alpha=0.5) + scale_x_discrete(\"Landmark\") + scale_y_log10(\"Normalized Error\", breaks=c(0.001,0.01,0.1,1,10))\n\n"))); - return p.finalize(show); } diff --git a/openbr/openbr_plugin.cpp b/openbr/openbr_plugin.cpp index 5fa7765..3f1adc8 100644 --- a/openbr/openbr_plugin.cpp +++ b/openbr/openbr_plugin.cpp @@ -1210,7 +1210,8 @@ bool br::Context::checkSDKPath(const QString &sdkPath) return QFileInfo(sdkPath + "/share/openbr/openbr.bib").exists(); } -// We create our own when the user hasn't +// We create our own when the user hasn't. +// Since we can't ensure that it gets deleted last, we never delete it. static QCoreApplication *application = NULL; void br::Context::initialize(int &argc, char *argv[], QString sdkPath, bool useGui) @@ -1230,7 +1231,6 @@ void br::Context::initialize(int &argc, char *argv[], QString sdkPath, bool useG // We take in argc as a reference due to: // https://bugreports.qt-project.org/browse/QTBUG-5637 // QApplication should be initialized before anything else. - // Since we can't ensure that it gets deleted last, we never delete it. if (QCoreApplication::instance() == NULL) { #ifndef BR_EMBEDDED if (useGui) application = new QApplication(argc, argv); @@ -1245,6 +1245,7 @@ void br::Context::initialize(int &argc, char *argv[], QString sdkPath, bool useG if (sdkPath.isEmpty()) { QStringList checkPaths; checkPaths << QCoreApplication::applicationDirPath() << QDir::currentPath(); checkPaths << QString(getenv("PATH")).split(sep, QString::SkipEmptyParts); + QSet checkedPaths; // Avoid infinite loops from symlinks bool foundSDK = false; foreach (const QString &path, checkPaths) { @@ -1252,6 +1253,8 @@ void br::Context::initialize(int &argc, char *argv[], QString sdkPath, bool useG QDir dir(path); do { sdkPath = dir.absolutePath(); + if (checkedPaths.contains(sdkPath)) break; + else checkedPaths.insert(sdkPath); foundSDK = checkSDKPath(sdkPath); dir.cdUp(); } while (!foundSDK && !dir.isRoot()); @@ -1316,9 +1319,6 @@ void br::Context::finalize() delete Globals; Globals = NULL; - - delete application; - application = NULL; } QString br::Context::about() @@ -1602,7 +1602,8 @@ Transform *Transform::make(QString str, QObject *parent) // Base name not found? Try constructing it via LoadStore if (!Factory::names().contains(parsed.suffix()) && (QFileInfo(parsed.suffix()).exists() - || QFileInfo(Globals->sdkPath + "/share/openbr/models/transforms/"+parsed.suffix()).exists())) { + || QFileInfo(Globals->sdkPath + "/share/openbr/models/transforms/"+parsed.suffix()).exists() + || QFileInfo(Globals->sdkPath + "/../share/openbr/models/transforms/"+parsed.suffix()).exists())) { Transform *tform = make("<"+parsed.suffix()+">", parent); applyAdditionalProperties(parsed, tform); return tform; diff --git a/openbr/plugins/classification/cascade.cpp b/openbr/plugins/classification/cascade_classifier.cpp index 5620c3b..fd471d9 100644 --- a/openbr/plugins/classification/cascade.cpp +++ b/openbr/plugins/classification/cascade_classifier.cpp @@ -323,4 +323,4 @@ BR_REGISTER(Classifier, CascadeClassifier) } // namespace br -#include "classification/cascade.moc" +#include "classification/cascade_classifier.moc" diff --git a/openbr/plugins/classification/dlib.cpp b/openbr/plugins/classification/dlib.cpp index 0881429..6dbbb88 100644 --- a/openbr/plugins/classification/dlib.cpp +++ b/openbr/plugins/classification/dlib.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include "openbr/plugins/openbr_internal.h" @@ -34,29 +35,29 @@ class DLandmarkerTransform : public UntrainableTransform Q_OBJECT private: - Resource shapeResource; void init() { shapeResource.setResourceMaker(new DLibShapeResourceMaker()); + shapeResource.release(shapeResource.acquire()); // Pre-load one instance of the model } QPointF averagePoints(const QList &points, int rangeBegin, int rangeEnd) const { - QPointF point; + QPointF point; for (int i=rangeBegin; i points = dst.file.points(); - dst.file.set("LeftEye",averagePoints(points,36,42)); - dst.file.set("RightEye",averagePoints(points,42,48)); - dst.file.set("Chin",points[8]); + dst.file.set("RightEye", averagePoints(points, 36, 42)); + dst.file.set("LeftEye" , averagePoints(points, 42, 48)); + dst.file.set("Chin", points[8]); } void project(const Template &src, Template &dst) const @@ -73,25 +74,21 @@ private: array2d image; assign_image(image,cimg); - if (src.file.rects().isEmpty()) { //If the image has no rects assume the whole image is a face - rectangle r(0, 0, cvImage.cols, cvImage.rows); - full_object_detection shape = (*sp)(image, r); - for (size_t i = 0; i < shape.num_parts(); i++) - dst.file.appendPoint(QPointF(shape.part(i)(0),shape.part(i)(1))); - setFacePoints(dst); - } - else { // Crop the image on the rects - for (int j=0; j points; + for (size_t i=0; itrainable; + } else { trainable = false; + } } bool timeVarying() const diff --git a/openbr/plugins/format/video.cpp b/openbr/plugins/format/videoFormats.cpp index fdf659d..0498e40 100644 --- a/openbr/plugins/format/video.cpp +++ b/openbr/plugins/format/videoFormats.cpp @@ -160,4 +160,4 @@ BR_REGISTER(Format, DefaultFormat) } // namespace br -#include "format/video.moc" +#include "format/videoFormats.moc" diff --git a/openbr/plugins/gallery/lmdb.cpp b/openbr/plugins/gallery/lmdb.cpp index 84a1176..6bef3ff 100644 --- a/openbr/plugins/gallery/lmdb.cpp +++ b/openbr/plugins/gallery/lmdb.cpp @@ -112,7 +112,19 @@ class lmdbGallery : public Gallery foreach(const Template &t, working) { // add current image to transaction caffe::Datum datum; - caffe::CVMatToDatum(t.m(), &datum); + + const cv::Mat &m = t.m(); + if (m.depth() == CV_32F) { + datum.set_channels(m.channels()); + datum.set_height(m.rows); + datum.set_width(m.cols); + for (int i=0; i(j)[k*m.channels()+i]); + } else { + caffe::CVMatToDatum(m, &datum); + } QVariant base_label = t.file.value("Label"); QString label_str = base_label.toString(); diff --git a/openbr/plugins/gallery/txt.cpp b/openbr/plugins/gallery/txt.cpp index d62106f..3070471 100644 --- a/openbr/plugins/gallery/txt.cpp +++ b/openbr/plugins/gallery/txt.cpp @@ -61,7 +61,7 @@ class txtGallery : public FileGallery if (!line.isEmpty()){ int splitIndex = line.lastIndexOf(' '); - if (splitIndex == -1) templates.append(File(line)); + if (splitIndex == -1) templates.append(File(line, QFileInfo(line).dir().dirName())); else templates.append(File(line.mid(0, splitIndex), line.mid(splitIndex+1))); templates.last().file.set("progress", this->position()); } diff --git a/openbr/plugins/gui/draw.cpp b/openbr/plugins/gui/draw.cpp index 69177ec..5f5fe76 100644 --- a/openbr/plugins/gui/draw.cpp +++ b/openbr/plugins/gui/draw.cpp @@ -38,6 +38,7 @@ class DrawTransform : public UntrainableTransform Q_PROPERTY(bool rects READ get_rects WRITE set_rects RESET reset_rects STORED false) Q_PROPERTY(bool inPlace READ get_inPlace WRITE set_inPlace RESET reset_inPlace STORED false) Q_PROPERTY(int lineThickness READ get_lineThickness WRITE set_lineThickness RESET reset_lineThickness STORED false) + Q_PROPERTY(int pointRadius READ get_pointRadius WRITE set_pointRadius RESET reset_pointRadius STORED false) Q_PROPERTY(bool named READ get_named WRITE set_named RESET reset_named STORED false) Q_PROPERTY(bool location READ get_location WRITE set_location RESET reset_location STORED false) BR_PROPERTY(bool, verbose, false) @@ -45,6 +46,7 @@ class DrawTransform : public UntrainableTransform BR_PROPERTY(bool, rects, true) BR_PROPERTY(bool, inPlace, false) BR_PROPERTY(int, lineThickness, 1) + BR_PROPERTY(int, pointRadius, 3) BR_PROPERTY(bool, named, true) BR_PROPERTY(bool, location, true) @@ -58,7 +60,7 @@ class DrawTransform : public UntrainableTransform const QList pointsList = (named) ? OpenCVUtils::toPoints(src.file.points()+src.file.namedPoints()) : OpenCVUtils::toPoints(src.file.points()); for (int i=0; i points = src.file.points(); diff --git a/openbr/plugins/imgproc/cropfromlandmarks.cpp b/openbr/plugins/imgproc/cropfromlandmarks.cpp index 8842ccb..13d8cbd 100644 --- a/openbr/plugins/imgproc/cropfromlandmarks.cpp +++ b/openbr/plugins/imgproc/cropfromlandmarks.cpp @@ -19,26 +19,33 @@ class CropFromLandmarksTransform : public UntrainableTransform Q_PROPERTY(QList indices READ get_indices WRITE set_indices RESET reset_indices STORED false) Q_PROPERTY(float paddingHorizontal READ get_paddingHorizontal WRITE set_paddingHorizontal RESET reset_paddingHorizontal STORED false) Q_PROPERTY(float paddingVertical READ get_paddingVertical WRITE set_paddingVertical RESET reset_paddingVertical STORED false) + Q_PROPERTY(bool shiftPoints READ get_shiftPoints WRITE set_shiftPoints RESET reset_shiftPoints STORED false) BR_PROPERTY(QList, indices, QList()) BR_PROPERTY(float, paddingHorizontal, .1) BR_PROPERTY(float, paddingVertical, .1) + BR_PROPERTY(bool, shiftPoints, false) void project(const Template &src, Template &dst) const { + QList cropIndices = indices; + if (cropIndices.isEmpty() && !src.file.points().isEmpty()) + for (int i=0; i src.file.points()[indices[i]].x()) - minX = src.file.points()[indices[i]].x(); - if (minY > src.file.points()[indices[i]].y()) - minY = src.file.points()[indices[i]].y(); - if (maxX < src.file.points()[indices[i]].x()) - maxX = src.file.points()[indices[i]].x(); - if (maxY < src.file.points()[indices[i]].y()) - maxY = src.file.points()[indices[i]].y(); + for (int i = 0; i src.file.points()[cropIndices[i]].x()) + minX = src.file.points()[cropIndices[i]].x(); + if (minY > src.file.points()[cropIndices[i]].y()) + minY = src.file.points()[cropIndices[i]].y(); + if (maxX < src.file.points()[cropIndices[i]].x()) + maxX = src.file.points()[cropIndices[i]].x(); + if (maxY < src.file.points()[cropIndices[i]].y()) + maxY = src.file.points()[cropIndices[i]].y(); } int padW = qRound((maxX - minX) * (paddingHorizontal / 2)); @@ -50,6 +57,13 @@ class CropFromLandmarksTransform : public UntrainableTransform if (rect.x() + rect.width() > src.m().cols) rect.setWidth(src.m().cols - rect.x()); if (rect.y() + rect.width() > src.m().rows) rect.setHeight(src.m().rows - rect.y()); + if (shiftPoints) { + QList points = src.file.points(); + for (int i=0; i(key))) + ifTrue.append(t); + else + ifFalse.append(t); + } + + transform->project(ifTrue,dst); + dst.append(ifFalse); + } + }; BR_REGISTER(Transform, IfTransform) diff --git a/openbr/plugins/imgproc/pack.cpp b/openbr/plugins/imgproc/pack.cpp deleted file mode 100644 index 9954af3..0000000 --- a/openbr/plugins/imgproc/pack.cpp +++ /dev/null @@ -1,52 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Copyright 2012 The MITRE Corporation * - * * - * Licensed under the Apache License, Version 2.0 (the "License"); * - * you may not use this file except in compliance with the License. * - * You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, software * - * distributed under the License is distributed on an "AS IS" BASIS, * - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * - * See the License for the specific language governing permissions and * - * limitations under the License. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#include - -using namespace cv; - -namespace br -{ - -/*! - * \ingroup transforms - * \brief Compress two uchar into one uchar. - * \author Josh Klontz \cite jklontz - */ -class PackTransform : public UntrainableTransform -{ - Q_OBJECT - - void project(const Template &src, Template &dst) const - { - const Mat &m = src; - if ((m.cols % 2 != 0) || (m.type() != CV_8UC1)) - qFatal("Invalid template format."); - - Mat n(m.rows, m.cols/2, CV_8UC1); - for (int i=0; i(i,j) = ((m.at(i,2*j+0) >> 4) << 4) + - ((m.at(i,2*j+1) >> 4) << 0); - dst = n; - } -}; - -BR_REGISTER(Transform, PackTransform) - -} // namespace br - -#include "imgproc/pack.moc" diff --git a/openbr/plugins/imgproc/pad.cpp b/openbr/plugins/imgproc/pad.cpp index 3c40221..297efd5 100644 --- a/openbr/plugins/imgproc/pad.cpp +++ b/openbr/plugins/imgproc/pad.cpp @@ -1,5 +1,6 @@ #include #include +#include using namespace cv; @@ -35,7 +36,7 @@ private: int top, bottom, left, right; top = percent*src.m().rows; bottom = percent*src.m().rows; left = percent*src.m().cols; right = percent*src.m().cols; - copyMakeBorder(src, dst, top, bottom, left, right, border, Scalar(value)); + OpenCVUtils::pad(src,dst,true,QList() << top << bottom << left << right,true,true,border,value); } }; diff --git a/openbr/plugins/imgproc/quantize.cpp b/openbr/plugins/imgproc/quantize.cpp index 3bd911c..2f3d941 100644 --- a/openbr/plugins/imgproc/quantize.cpp +++ b/openbr/plugins/imgproc/quantize.cpp @@ -30,8 +30,10 @@ namespace br class QuantizeTransform : public Transform { Q_OBJECT + Q_PROPERTY(float c READ get_c WRITE set_c RESET reset_c STORED false) Q_PROPERTY(float a READ get_a WRITE set_a RESET reset_a) Q_PROPERTY(float b READ get_b WRITE set_b RESET reset_b) + BR_PROPERTY(float, c, 1) BR_PROPERTY(float, a, 1) BR_PROPERTY(float, b, 0) @@ -46,7 +48,7 @@ class QuantizeTransform : public Transform void project(const Template &src, Template &dst) const { - src.m().convertTo(dst, CV_8U, a, b); + src.m().convertTo(dst, CV_8U, a / c, b / c); } QByteArray likely(const QByteArray &indentation) const diff --git a/openbr/plugins/imgproc/resize.cpp b/openbr/plugins/imgproc/resize.cpp index 9a5310f..9f16d3c 100644 --- a/openbr/plugins/imgproc/resize.cpp +++ b/openbr/plugins/imgproc/resize.cpp @@ -48,21 +48,35 @@ private: Q_PROPERTY(int columns READ get_columns WRITE set_columns RESET reset_columns STORED false) Q_PROPERTY(Method method READ get_method WRITE set_method RESET reset_method STORED false) Q_PROPERTY(bool preserveAspect READ get_preserveAspect WRITE set_preserveAspect RESET reset_preserveAspect STORED false) + Q_PROPERTY(bool pad READ get_pad WRITE set_pad RESET reset_pad STORED false) BR_PROPERTY(int, rows, -1) BR_PROPERTY(int, columns, -1) BR_PROPERTY(Method, method, Bilin) BR_PROPERTY(bool, preserveAspect, false) + BR_PROPERTY(bool, pad, true) void project(const Template &src, Template &dst) const { + if ((rows == -1) && (columns == -1)) { + dst = src; + return; + } + if (!preserveAspect) { resize(src, dst, Size((columns == -1) ? src.m().cols*rows/src.m().rows : columns, rows), 0, 0, method); const float rowScaleFactor = (float)rows/src.m().rows; - const float colScaleFactor = (float)columns/src.m().cols; + const float colScaleFactor = (columns == -1) ? rowScaleFactor : (float)columns/src.m().cols; QList points = src.file.points(); for (int i=0; i src.m().cols) + resize(src, dst, Size(size/ratio, size), 0, 0, method); + else + resize(src, dst, Size(size, size*ratio), 0, 0, method); } else { float inRatio = (float) src.m().rows / src.m().cols; float outRatio = (float) rows / columns; diff --git a/openbr/plugins/imgproc/rndtranslate.cpp b/openbr/plugins/imgproc/rndtranslate.cpp index 721872e..c57e7cf 100644 --- a/openbr/plugins/imgproc/rndtranslate.cpp +++ b/openbr/plugins/imgproc/rndtranslate.cpp @@ -38,7 +38,7 @@ class RndTranslateTransform : public UntrainableMetaTransform Q_PROPERTY(int nStages READ get_nStages WRITE set_nStages RESET reset_nStages STORED false) BR_PROPERTY(int, nStages, 3) - void project(const Template &src, Template &dst) const + void project(const Template &, Template &) const { qFatal("Shoult not be here (RndTranslate)"); } diff --git a/openbr/plugins/imgproc/slidingwindow.cpp b/openbr/plugins/imgproc/slidingwindow.cpp index cb1124c..7e2df3a 100644 --- a/openbr/plugins/imgproc/slidingwindow.cpp +++ b/openbr/plugins/imgproc/slidingwindow.cpp @@ -35,7 +35,7 @@ namespace br * \br_property int minSize The smallest sized object to detect in pixels * \br_property int maxSize The largest sized object to detect in pixels. A negative value will set maxSize == image size * \br_property float scaleFactor The factor to scale the image by during each resize. - * \br_property float confidenceThreshold A threshold for positive detections. Positive detections returned by the classifier that have confidences below this threshold are considered negative detections. + * \br_property float minGroupingConfidence A threshold for positive detections. Positive detections returned by the classifier that have confidences below this threshold are considered negative detections. * \br_property float eps Parameter for non-maximum supression * \br_property int minNeighbors Parameter for non-maximum supression * \br_property bool group If false, non-maxima supression will not be performed @@ -51,23 +51,28 @@ class SlidingWindowTransform : public MetaTransform Q_PROPERTY(int minSize READ get_minSize WRITE set_minSize RESET reset_minSize STORED false) Q_PROPERTY(int maxSize READ get_maxSize WRITE set_maxSize RESET reset_maxSize STORED false) Q_PROPERTY(float scaleFactor READ get_scaleFactor WRITE set_scaleFactor RESET reset_scaleFactor STORED false) - Q_PROPERTY(float confidenceThreshold READ get_confidenceThreshold WRITE set_confidenceThreshold RESET reset_confidenceThreshold STORED false) + Q_PROPERTY(float minGroupingConfidence READ get_minGroupingConfidence WRITE set_minGroupingConfidence RESET reset_minGroupingConfidence STORED false) Q_PROPERTY(float eps READ get_eps WRITE set_eps RESET reset_eps STORED false) Q_PROPERTY(float minNeighbors READ get_minNeighbors WRITE set_minNeighbors RESET reset_minNeighbors STORED false) Q_PROPERTY(bool group READ get_group WRITE set_group RESET reset_group STORED false) Q_PROPERTY(int shrinkingFactor READ get_shrinkingFactor WRITE set_shrinkingFactor RESET reset_shrinkingFactor STORED false) Q_PROPERTY(bool clone READ get_clone WRITE set_clone RESET reset_clone STORED false) - + Q_PROPERTY(float minConfidence READ get_minConfidence WRITE set_minConfidence RESET reset_minConfidence STORED false) + Q_PROPERTY(bool ROCMode READ get_ROCMode WRITE set_ROCMode RESET reset_ROCMode STORED false) + Q_PROPERTY(QString outputVariable READ get_outputVariable WRITE set_outputVariable RESET reset_outputVariable STORED false) BR_PROPERTY(br::Classifier*, classifier, NULL) BR_PROPERTY(int, minSize, 20) BR_PROPERTY(int, maxSize, -1) BR_PROPERTY(float, scaleFactor, 1.2) - BR_PROPERTY(float, confidenceThreshold, 10) + BR_PROPERTY(float, minGroupingConfidence, -std::numeric_limits::max()) BR_PROPERTY(float, eps, 0.2) BR_PROPERTY(int, minNeighbors, 3) BR_PROPERTY(bool, group, true) BR_PROPERTY(int, shrinkingFactor, 1) BR_PROPERTY(bool, clone, true) + BR_PROPERTY(float, minConfidence, 0) + BR_PROPERTY(bool, ROCMode, false) + BR_PROPERTY(QString, outputVariable, "Face") void train(const TemplateList &data) { @@ -85,9 +90,9 @@ class SlidingWindowTransform : public MetaTransform { foreach (const Template &t, src) { // As a special case, skip detection if the appropriate metadata already exists - if (t.file.contains("Face")) { + if (t.file.contains(outputVariable)) { Template u = t; - u.file.setRects(QList() << t.file.get("Face")); + u.file.setRects(QList() << t.file.get(outputVariable)); u.file.set("Confidence", t.file.get("Confidence", 1)); dst.append(u); continue; @@ -106,6 +111,9 @@ class SlidingWindowTransform : public MetaTransform // different channels of the same image! const Size imageSize = t.m().size(); const int minSize = t.file.get("MinSize", this->minSize); + const int maxDetections = t.file.get("MaxDetections", std::numeric_limits::max()); + const bool findMostConfident = (enrollAll && (maxDetections != 1)) ? false : true; + QList rects; QList confidences; @@ -163,19 +171,43 @@ class SlidingWindowTransform : public MetaTransform } if (group) - OpenCVUtils::group(rects, confidences, confidenceThreshold, minNeighbors, eps); + OpenCVUtils::group(rects, confidences, minGroupingConfidence, minNeighbors, eps); + + if (!ROCMode && findMostConfident && !rects.isEmpty()) { + Rect rect = rects.first(); + float maxConfidence = confidences.first(); + for (int i=0; i maxConfidence) { + rect = rects[i]; + maxConfidence = confidences[i]; + } + rects.clear(); + confidences.clear(); + rects.append(rect); + confidences.append(maxConfidence); + } - if (!enrollAll && rects.empty()) { - rects.append(Rect(0, 0, imageSize.width, imageSize.height)); - confidences.append(-std::numeric_limits::max()); + const float minConfidence = t.file.get("MinConfidence", this->minConfidence); + QList rectsAboveMinConfidence; + QList confidencesAboveMinConfidence; + for (int i=0; i= minConfidence) { + rectsAboveMinConfidence.append(OpenCVUtils::fromRect(rects[i])); + confidencesAboveMinConfidence.append(confidences[i]); + } } - for (int j=0; j::max()); + } + + for (int i=0; i(subDir, "Temp"); QString path = QString("%1/%2/").arg(outputDirectory, dir); int value = numImages.value(dir, 0); - path += QString("%1_%2.%3").arg(src.file.get(subDir, "Image")).arg(value, padding, 10, QChar('0')).arg(imgExtension); + path += preserveFilename ? QString("%1.%2").arg(src.file.baseName().split('.')[0], imgExtension) + : QString("%1_%2.%3").arg(src.file.get(subDir, "Image")).arg(value, padding, 10, QChar('0')).arg(imgExtension); + if ((QDir::currentPath() + "/" + path) == src.file.name) + qFatal("Attempted to overwrite image!"); + numImages[dir] = ++value; OpenCVUtils::saveImage(dst.m(), path); } else { - QString path = QString("%1/image%2%3.%4").arg(outputDirectory).arg(cnt++, padding, 10, QChar('0')).arg(underscore.isEmpty() ? "" : "_" + underscore).arg(imgExtension); + QString path = preserveFilename ? QString("%1/%2.%3").arg(outputDirectory, src.file.baseName().split('.')[0], imgExtension) + : QString("%1/image%2%3.%4").arg(outputDirectory).arg(cnt++, padding, 10, QChar('0')).arg(underscore.isEmpty() ? "" : "_" + underscore).arg(imgExtension); + if ((QDir::currentPath() + "/" + path) == src.file.name) + qFatal("Attempted to overwrite image!"); + OpenCVUtils::saveImage(dst.m(), path); } } diff --git a/openbr/plugins/distance/halfbyteL1.cpp b/openbr/plugins/metadata/clearrects.cpp index d70c0cf..1f34fd2 100644 --- a/openbr/plugins/distance/halfbyteL1.cpp +++ b/openbr/plugins/metadata/clearrects.cpp @@ -1,5 +1,5 @@ /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Copyright 2012 The MITRE Corporation * + * Copyright 2016 Rank One Computing Corporation. * * * Licensed under the Apache License, Version 2.0 (the "License"); * * you may not use this file except in compliance with the License. * @@ -15,31 +15,28 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include -#include - -using namespace cv; namespace br { /*! - * \ingroup distances - * \brief Fast 4-bit L1 distance - * \author Josh Klontz \cite jklontz + * \ingroup transforms + * \brief Clears the rects from a Template + * \author Brendan Klare \cite bklare */ -class HalfByteL1Distance : public UntrainableDistance +class ClearRectsTransform : public UntrainableMetadataTransform { Q_OBJECT - float compare(const Mat &a, const Mat &b) const + void projectMetadata(const File &src, File &dst) const { - return packed_l1(a.data, b.data, a.total()); + dst = src; + dst.clearRects(); } }; -BR_REGISTER(Distance, HalfByteL1Distance) - +BR_REGISTER(Transform, ClearRectsTransform) } // namespace br -#include "distance/halfbyteL1.moc" +#include "metadata/clearrects.moc" diff --git a/openbr/plugins/metadata/ifmetadata.cpp b/openbr/plugins/metadata/ifmetadata.cpp index 798c02c..931c15d 100644 --- a/openbr/plugins/metadata/ifmetadata.cpp +++ b/openbr/plugins/metadata/ifmetadata.cpp @@ -21,7 +21,7 @@ namespace br /*! * \ingroup transforms - * \brief Clear Templates without the required metadata. + * \brief Remove templates without the required metadata. * \author Josh Klontz \cite jklontz */ class IfMetadataTransform : public UntrainableMetadataTransform @@ -34,8 +34,10 @@ class IfMetadataTransform : public UntrainableMetadataTransform void projectMetadata(const File &src, File &dst) const { - if (src.get(key, "") == value) - dst = src; + if (key == "_basename") + dst.fte = src.baseName() != value; + else + dst.fte = src.get(key, "") != value; } }; diff --git a/openbr/plugins/metadata/randomrects.cpp b/openbr/plugins/metadata/randomrects.cpp index 4f44de1..9f86088 100644 --- a/openbr/plugins/metadata/randomrects.cpp +++ b/openbr/plugins/metadata/randomrects.cpp @@ -21,7 +21,7 @@ class RandomRectsTransform : public UntrainableMetaTransform BR_PROPERTY(int, numRects, 135) BR_PROPERTY(int, minSize, 24) - void project(const Template &src, Template &dst) const + void project(const Template &, Template &) const { qFatal("NOT SUPPORTED"); } diff --git a/openbr/plugins/metadata/randomtemplates.cpp b/openbr/plugins/metadata/randomtemplates.cpp index 466a3a4..7830e52 100644 --- a/openbr/plugins/metadata/randomtemplates.cpp +++ b/openbr/plugins/metadata/randomtemplates.cpp @@ -14,7 +14,7 @@ class RandomTemplatesTransform : public UntrainableMetaTransform Q_PROPERTY(float percent READ get_percent WRITE set_percent RESET reset_percent) BR_PROPERTY(float, percent, .01) - void project(const Template &src, Template &dst) const { + void project(const Template &, Template &) const { qFatal("Not supported in RandomTemplates."); } diff --git a/openbr/plugins/metadata/removefte.cpp b/openbr/plugins/metadata/removefte.cpp index 17b3ceb..6032d3b 100644 --- a/openbr/plugins/metadata/removefte.cpp +++ b/openbr/plugins/metadata/removefte.cpp @@ -6,13 +6,14 @@ namespace br /*! * \ingroup transforms * \author Brendan Klare \cite bklare - * \brief Remove any templates that failed to enroll (FTE) + * \brief Remove any templates that failed to enroll (FTE). + * Important note: this will not work without the global enrollAll being true */ class RemoveFTETransform : public UntrainableMetaTransform { Q_OBJECT - void project(const Template &src, Template &dst) const + void project(const Template &, Template &) const { qFatal("Not supported in RemoveFTE."); } diff --git a/openbr/plugins/metadata/savemat.cpp b/openbr/plugins/metadata/savemat.cpp index 03048f1..0099bd8 100644 --- a/openbr/plugins/metadata/savemat.cpp +++ b/openbr/plugins/metadata/savemat.cpp @@ -59,7 +59,11 @@ class JustTransform : public UntrainableMetaTransform Template tmp; transform->project(src, tmp); foreach (const QString &key, keys) - dst.file.set(key, tmp.file.value(key)); + if (key == "_Points") { + dst.file.setPoints(tmp.file.points()); + } else { + dst.file.set(key, tmp.file.value(key)); + } } }; diff --git a/openbr/plugins/representation/haar.cpp b/openbr/plugins/representation/haar.cpp index fcf94d1..d711731 100644 --- a/openbr/plugins/representation/haar.cpp +++ b/openbr/plugins/representation/haar.cpp @@ -40,37 +40,51 @@ class HaarRepresentation : public Representation void init() { if (features.isEmpty()) { - int offset = winWidth + 1; + // Pre-determine the size of features to avoid reallocations + int numFeatures = 0; + for (int x = 0; x < winWidth; x++) + for (int y = 0; y < winHeight; y++) + for (int dx = 1; dx <= winWidth; dx++) + for (int dy = 1; dy <= winHeight; dy++) + numFeatures += ((x+dx*2 <= winWidth) && (y+dy <= winHeight)) + + ((x+dx <= winWidth) && (y+dy*2 <= winHeight)) + + ((x+dx*3 <= winWidth) && (y+dy <= winHeight)) + + ((x+dx <= winWidth) && (y+dy*3 <= winHeight)) + + ((x+dx*2 <= winWidth) && (y+dy*2 <= winHeight)); + features.reserve(numFeatures); + + const int offset = winWidth + 1; + int index = 0; for (int x = 0; x < winWidth; x++) { for (int y = 0; y < winHeight; y++) { for (int dx = 1; dx <= winWidth; dx++) { for (int dy = 1; dy <= winHeight; dy++) { // haar_x2 if ((x+dx*2 <= winWidth) && (y+dy <= winHeight)) - features.append(Feature(offset, + features[index++] = Feature(offset, x, y, dx*2, dy, -1, - x+dx, y, dx , dy, +2)); + x+dx, y, dx , dy, +2); // haar_y2 if ((x+dx <= winWidth) && (y+dy*2 <= winHeight)) - features.append(Feature(offset, + features[index++] = Feature(offset, x, y, dx, dy*2, -1, - x, y+dy, dx, dy, +2)); + x, y+dy, dx, dy, +2); // haar_x3 if ((x+dx*3 <= winWidth) && (y+dy <= winHeight)) - features.append(Feature(offset, + features[index++] = Feature(offset, x, y, dx*3, dy, -1, - x+dx, y, dx , dy, +3)); + x+dx, y, dx , dy, +3); // haar_y3 if ((x+dx <= winWidth) && (y+dy*3 <= winHeight)) - features.append(Feature(offset, + features[index++] = Feature(offset, x, y, dx, dy*3, -1, - x, y+dy, dx, dy, +3)); + x, y+dy, dx, dy, +3); // x2_y2 if ((x+dx*2 <= winWidth) && (y+dy*2 <= winHeight)) - features.append(Feature(offset, + features[index++] = Feature(offset, x, y, dx*2, dy*2, -1, x, y, dx, dy, +2, - x+dx, y+dy, dx, dy, +2)); + x+dx, y+dy, dx, dy, +2); } @@ -89,7 +103,7 @@ class HaarRepresentation : public Representation float evaluate(const Template &src, int idx) const { - return (float)features[idx].calc(src.m()); + return features[idx].calc(src.m()); } Mat evaluate(const Template &src, const QList &indices) const @@ -126,24 +140,23 @@ class HaarRepresentation : public Representation float calc(const Mat &img) const; struct { - Rect r; float weight; - } rect[3]; - - struct { int p0, p1, p2, p3; } fastRect[3]; }; - QList features; + QVector features; }; BR_REGISTER(Representation, HaarRepresentation) HaarRepresentation::Feature::Feature() { - rect[0].r = rect[1].r = rect[2].r = Rect(0,0,0,0); - rect[0].weight = rect[1].weight = rect[2].weight = 0; + fastRect[0].p0 = fastRect[1].p0 = fastRect[2].p0 = 0; + fastRect[0].p1 = fastRect[1].p1 = fastRect[2].p1 = 0; + fastRect[0].p2 = fastRect[1].p2 = fastRect[2].p2 = 0; + fastRect[0].p3 = fastRect[1].p3 = fastRect[2].p3 = 0; + fastRect[0].weight = fastRect[1].weight = fastRect[2].weight = 0; } HaarRepresentation::Feature::Feature(int offset, @@ -151,38 +164,21 @@ HaarRepresentation::Feature::Feature(int offset, int x1, int y1, int w1, int h1, float wt1, int x2, int y2, int w2, int h2, float wt2) { - rect[0].r.x = x0; - rect[0].r.y = y0; - rect[0].r.width = w0; - rect[0].r.height = h0; - rect[0].weight = wt0; - - rect[1].r.x = x1; - rect[1].r.y = y1; - rect[1].r.width = w1; - rect[1].r.height = h1; - rect[1].weight = wt1; - - rect[2].r.x = x2; - rect[2].r.y = y2; - rect[2].r.width = w2; - rect[2].r.height = h2; - rect[2].weight = wt2; - - for (int j = 0; j < 3; j++) { - if( rect[j].weight == 0.0F ) - break; - CV_SUM_OFFSETS(fastRect[j].p0, fastRect[j].p1, fastRect[j].p2, fastRect[j].p3, rect[j].r, offset) - } + CV_SUM_OFFSETS(fastRect[0].p0, fastRect[0].p1, fastRect[0].p2, fastRect[0].p3, Rect(x0, y0, w0, h0), offset) + CV_SUM_OFFSETS(fastRect[1].p0, fastRect[1].p1, fastRect[1].p2, fastRect[1].p3, Rect(x1, y1, w1, h1), offset) + CV_SUM_OFFSETS(fastRect[2].p0, fastRect[2].p1, fastRect[2].p2, fastRect[2].p3, Rect(x2, y2, w2, h2), offset) + fastRect[0].weight = wt0; + fastRect[1].weight = wt1; + fastRect[2].weight = wt2; } inline float HaarRepresentation::Feature::calc(const Mat &img) const { const int* ptr = img.ptr(); - float ret = rect[0].weight * (ptr[fastRect[0].p0] - ptr[fastRect[0].p1] - ptr[fastRect[0].p2] + ptr[fastRect[0].p3]) + - rect[1].weight * (ptr[fastRect[1].p0] - ptr[fastRect[1].p1] - ptr[fastRect[1].p2] + ptr[fastRect[1].p3]); - if (rect[2].weight != 0.0f) - ret += rect[2].weight * (ptr[fastRect[2].p0] - ptr[fastRect[2].p1] - ptr[fastRect[2].p2] + ptr[fastRect[2].p3]); + float ret = fastRect[0].weight * (ptr[fastRect[0].p0] - ptr[fastRect[0].p1] - ptr[fastRect[0].p2] + ptr[fastRect[0].p3]) + + fastRect[1].weight * (ptr[fastRect[1].p0] - ptr[fastRect[1].p1] - ptr[fastRect[1].p2] + ptr[fastRect[1].p3]); + if (fastRect[2].weight != 0.0f) + ret += fastRect[2].weight * (ptr[fastRect[2].p0] - ptr[fastRect[2].p1] - ptr[fastRect[2].p2] + ptr[fastRect[2].p3]); return ret; } diff --git a/scripts/brpy/__init__.py b/scripts/brpy/__init__.py index 6cfd9f3..066824c 100644 --- a/scripts/brpy/__init__.py +++ b/scripts/brpy/__init__.py @@ -53,6 +53,10 @@ def _handle_string_func(func, *moretypes): return msg return call_func +# TODO: small wrapper around any objects that need br_free_* +# so when python object dies, user doesn't need to call br_free_* +# TODO: iterator for templatelist object +# (or mimic C++ obj API via C? overkill?) def init_brpy(br_loc='/usr/local/lib'): """Initializes all function inputs and outputs for the br ctypes lib object""" diff --git a/scripts/brpy/html_viz.py b/scripts/brpy/html_viz.py index 665afcd..5b2f90d 100644 --- a/scripts/brpy/html_viz.py +++ b/scripts/brpy/html_viz.py @@ -8,24 +8,57 @@ and host them as long as the relative paths remain the same on ya serva. from PIL import Image +LMSIZE = 8 + +def landmarks_on_crop(landmarks, x, y, width, height, imname, maxheight=None, color='green'): + ''' + Generates an HTML string that shows landmarks within a given cropped image. + When two landmarked crops are put next to each other, they will be inline. To make each crop its own line, wrap it in a div. + ''' + html = _bbcrop(x, y, width, height, imname, maxheight) + if not maxheight: + maxheight = height + ratio = float(maxheight) / height + print(ratio) + if len(landmarks) > 0 and len(landmarks[0]) != 3: + landmarks = [ lm + (color,) for lm in landmarks ] + for lmx,lmy,col in landmarks: + html += landmark((lmx-x)*ratio - (LMSIZE/2), (lmy-y)*ratio - (LMSIZE/2), col) + html += '' + return html + +def landmark_img(x, y, img, color='green'): + html = '
' + html += '' % img + html += landmark(x,y,color) + html += '
' + return html + +def landmark(x, y, color='green'): + return '
'.format(y,x,color,LMSIZE) + def crop_to_bb(x, y, width, height, imname, maxheight=None): ''' Generates an HTML string that crops to a given bounding box and resizes to maxheight pixels. A maxheight of None will keep the original size (default). When two crops are put next to each other, they will be inline. To make each crop its own line, wrap it in a div. ''' + html = _bbcrop(x, y, width, height, imname, maxheight) + html += '' + return html + +def _bbcrop(x, y, width, height, imname, maxheight): img = Image.open(imname) imwidth, imheight = img.size if not maxheight: maxheight = height - ratio = maxheight / height + ratio = float(maxheight) / height # note for future me: # image is cropped with div width/height + overflow:hidden, # resized with img height, # and positioned with img margin - html = '
' % (width*ratio, maxheight) + html = '
' % (width*ratio, maxheight) html += '' % (imname, imheight*ratio, y*ratio, x*ratio) - html += '
' return html def bbs_for_image(imname, bbs, maxheight=None, colors=None): @@ -37,7 +70,7 @@ def bbs_for_image(imname, bbs, maxheight=None, colors=None): imwidth, imheight = img.size if not maxheight: maxheight = imheight - ratio = maxheight/imheight + ratio = float(maxheight)/imheight html = [ '
', '' % (imname, maxheight) diff --git a/share/openbr/cmake/InstallDependencies.cmake b/share/openbr/cmake/InstallDependencies.cmake index e6c0d8b..2e62207 100644 --- a/share/openbr/cmake/InstallDependencies.cmake +++ b/share/openbr/cmake/InstallDependencies.cmake @@ -2,10 +2,8 @@ set(BR_INSTALL_DEPENDENCIES OFF CACHE BOOL "Install runtime dependencies.") # OpenCV Libs function(install_opencv_library lib) - if(${BR_INSTALL_DEPENDENCIES}) - if(ANDROID) - # Do nothing assuming we are using OpenCV static libs - elseif(CMAKE_HOST_WIN32) + if(${BR_INSTALL_DEPENDENCIES} AND ${OpenCV_SHARED}) + if(CMAKE_HOST_WIN32) if(${CMAKE_BUILD_TYPE} MATCHES Debug) set(BR_INSTALL_DEPENDENCIES_SUFFIX "d") endif() @@ -65,7 +63,10 @@ endfunction() function(install_qt_imageformats) if(${BR_INSTALL_DEPENDENCIES}) set(IMAGE_FORMATS_DIR "${_qt5Core_install_prefix}/plugins/imageformats") - if(CMAKE_HOST_WIN32) + if(ANDROID) + set(INSTALL_DEPENDENCIES_PREFIX "lib") + set(INSTALL_DEPENDENCIES_EXTENSION ".so") + elseif(CMAKE_HOST_WIN32) set(INSTALL_DEPENDENCIES_PREFIX "") set(INSTALL_DEPENDENCIES_EXTENSION ".dll") elseif(CMAKE_HOST_APPLE) @@ -118,6 +119,11 @@ function(install_qt_misc) file(GLOB d3dcomp ${_qt5Core_install_prefix}/bin/d3dcompiler_*.dll) install(FILES ${d3dcomp} DESTINATION bin) install(FILES ${_qt5Core_install_prefix}/plugins/platforms/qwindows${BR_INSTALL_DEPENDENCIES_SUFFIX}.dll DESTINATION bin/platforms) + elseif(ANDROID) + install(FILES ${__libstl} DESTINATION lib) + elseif(UNIX AND NOT APPLE) + file(GLOB icudlls ${_qt5Core_install_prefix}/lib/libicu*.so*) + install(FILES ${icudlls} DESTINATION lib) endif() endfunction() diff --git a/share/openbr/plotting/plot_utils.R b/share/openbr/plotting/plot_utils.R index ec7a7c3..ac97ef4 100644 --- a/share/openbr/plotting/plot_utils.R +++ b/share/openbr/plotting/plot_utils.R @@ -216,13 +216,13 @@ formatData <- function(type="eval") { Box$X <<- factor(Box$X, levels = Box$X, ordered = TRUE) Sample <<- data[grep("Sample",data$Plot),-c(1)] Sample$X <<- as.character(Sample$X) + displaySample <<- readImageData(Sample) + rows <<- displaySample[[1]]$value EXT <<- data[grep("EXT",data$Plot),-c(1)] EXT$X <<- as.character(EXT$X) EXP <<- data[grep("EXP",data$Plot),-c(1)] EXP$X <<- as.character(EXP$X) NormLength <<- data[grep("NormLength",data$Plot),-c(1)] - sample <<- readImageData(Sample) - rows <<- sample[[1]]$value } else if (type == "knn") { # Split data into individual plots IET <<- data[grep("IET",data$Plot),-c(1)] @@ -325,14 +325,15 @@ plotEERSamples <- function(imData=NULL, gmData=NULL) { printImages(gmData, "Genuine") } -plotLandmarkSamples <- function(samples=NULL, expData=NULL, extData=NULL) { - print(plotImage(samples[[1]], "Sample Landmarks", sprintf("Total Landmarks: %s", samples[[1]]$value))) +plotLandmarkSamples <- function(displaySample=NULL, expData=NULL, extData=NULL) { + print(plotImage(displaySample[[1]], "Sample Landmarks", sprintf("Total Landmarks: %s", displaySample[[1]]$value))) + column <- if(majorSize > 1) majorHeader else "File" if (nrow(EXT) != 0 && nrow(EXP)) { for (j in 1:length(algs)) { - truthSample <- readData(EXT[EXT$. == algs[[j]],]) - predictedSample <- readData(EXP[EXP$. == algs[[j]],]) + truthSample <- readImageData(EXT[EXT[,column] == algs[[j]],]) + predictedSample <- readImageData(EXP[EXP[,column] == algs[[j]],]) for (i in 1:length(predictedSample)) { - multiplot(plotImage(predictedSample[[i]], sprintf("%s\nPredicted Landmarks", algs[[j]]), sprintf("Average Landmark Error: %.3f", predictedSample[[i]]$value)), plotImage(truthSample[[i]], "Ground Truth\nLandmarks", ""), cols=2) + multiplot(plotImage(predictedSample[[i]], sprintf("%s\nPredicted Landmarks", algs[[j]]), sprintf("Average Landmark Error: %.3f", predictedSample[[i]]$value)), plotImage(truthSample[[i]], "Ground Truth\nLandmarks", truthSample[[i]]$path), cols=2) } } } @@ -341,20 +342,20 @@ plotLandmarkSamples <- function(samples=NULL, expData=NULL, extData=NULL) { readImageData <- function(data) { examples <- list() for (i in 1:nrow(data)) { - path <- data[i,1] + examplePath <- unlist(strsplit(data[i,1], "[:]"))[1] + path <- unlist(strsplit(data[i,1], "[:]"))[2] value <- data[i,2] - file <- unlist(strsplit(path, "[.]"))[1] - ext <- unlist(strsplit(path, "[.]"))[2] + ext <- unlist(strsplit(examplePath, "[.]"))[2] if (ext == "jpg" || ext == "JPEG" || ext == "jpeg" || ext == "JPG") { - img <- readJPEG(path) + img <- readJPEG(examplePath) } else if (ext == "PNG" || ext == "png") { - img <- readPNG(path) + img <- readPNG(examplePath) } else if (ext == "TIFF" || ext == "tiff" || ext == "TIF" || ext == "tif") { - img <- readTIFF(path) + img <- readTIFF(examplePath) }else { next } - example <- list(file = file, value = value, image = img) + example <- list(path = path, value = value, image = img) examples[[i]] <- example } return(examples)