diff --git a/.gitignore b/.gitignore index 25abe9f..4727cfa 100644 --- a/.gitignore +++ b/.gitignore @@ -34,3 +34,6 @@ scripts/results ### vim ### *.swp + +### autogenerated sigsets ### +data/INRIAPerson/sigset diff --git a/3rdparty/stasm4.0.0/stasm/faceroi.cpp b/3rdparty/stasm4.0.0/stasm/faceroi.cpp index b7f9120..e999cfd 100755 --- a/3rdparty/stasm4.0.0/stasm/faceroi.cpp +++ b/3rdparty/stasm4.0.0/stasm/faceroi.cpp @@ -3,6 +3,7 @@ // Copyright (C) 2005-2013, Stephen Milborrow #include "stasm.h" +#include namespace stasm { diff --git a/3rdparty/stasm4.0.0/stasm/pinstart.cpp b/3rdparty/stasm4.0.0/stasm/pinstart.cpp index c22e122..55c14ec 100755 --- a/3rdparty/stasm4.0.0/stasm/pinstart.cpp +++ b/3rdparty/stasm4.0.0/stasm/pinstart.cpp @@ -3,6 +3,7 @@ // Copyright (C) 2005-2013, Stephen Milborrow #include "stasm.h" +#include namespace stasm { @@ -267,8 +268,8 @@ void PinnedStartShapeAndRoi( // use the pinned landmarks to init the start sha const Shape& pinned) // in: manually pinned landmarks { double rot, yaw; - EstRotAndYawFrom5PointShape(rot, yaw, - As5PointShape(pinned, mods[0]->MeanShape_())); + EstRotAndYawFrom5PointShape(rot, yaw, As5PointShape(pinned, mods[0]->MeanShape_())); + const EYAW eyaw = DegreesAsEyaw(yaw, NSIZE(mods)); const int imod = EyawAsModIndex(eyaw, mods); // select ASM model based on yaw if (trace_g) diff --git a/3rdparty/stasm4.0.0/stasm/stasm_lib.cpp b/3rdparty/stasm4.0.0/stasm/stasm_lib.cpp index fa4b26c..6295fa8 100755 --- a/3rdparty/stasm4.0.0/stasm/stasm_lib.cpp +++ b/3rdparty/stasm4.0.0/stasm/stasm_lib.cpp @@ -244,24 +244,21 @@ int stasm_search_single( // wrapper for stasm_search_auto and friends int stasm_search_pinned( // call after the user has pinned some points float* landmarks, // out: x0, y0, x1, y1, ..., caller must allocate const float* pinned, // in: pinned landmarks (0,0 points not pinned) - const char* img, // in: gray image data, top left corner at 0,0 + const char* data, // in: gray image data, top left corner at 0,0 int width, // in: image width int height, // in: image height const char* imgpath) // in: image path, used only for err msgs and debug { - (void) img; (void) width; (void) height; (void) imgpath; int returnval = 1; // assume success - CatchOpenCvErrs(); try { - CV_Assert(imgpath && STRNLEN(imgpath, SLEN) < SLEN); CheckStasmInit(); - //img_g = Image(height, width, (unsigned char*)img); + Image img = Image(height, width,(unsigned char*)data); const Shape pinnedshape(LandmarksAsShape(pinned)); @@ -271,8 +268,7 @@ int stasm_search_pinned( // call after the user has pinned some points DetPar detpar_roi; // detpar translated to ROI frame DetPar detpar; // params returned by pseudo face det, in img frame - /*PinnedStartShapeAndRoi(shape, face_roi, detpar_roi, detpar, pinned_roi, - img_g, mods_g, pinnedshape);*/ + PinnedStartShapeAndRoi(shape, face_roi, detpar_roi, detpar, pinned_roi, img, mods_g, pinnedshape); // now working with maybe flipped ROI and start shape in ROI frame const int imod = ABS(EyawAsModIndex(detpar.eyaw, mods_g)); @@ -284,14 +280,11 @@ int stasm_search_pinned( // call after the user has pinned some points RoundMat(shape); ForcePinnedPoints(shape, pinnedshape); // undo above RoundMat on pinned points ShapeToLandmarks(landmarks, shape); - if (trace_g) - lprintf("\n"); } catch(...) { returnval = 0; // a call was made to Err or a CV_Assert failed } - UncatchOpenCvErrs(); return returnval; } diff --git a/app/br/br.cpp b/app/br/br.cpp index 13b9258..53040f8 100644 --- a/app/br/br.cpp +++ b/app/br/br.cpp @@ -111,8 +111,8 @@ public: // Secondary Tasks else if (!strcmp(fun, "fuse")) { - check(parc >= 5, "Insufficient parameter count for 'fuse'."); - br_fuse(parc-4, parv, parv[parc-4], parv[parc-3], parv[parc-2], parv[parc-1]); + check(parc >= 4, "Insufficient parameter count for 'fuse'."); + br_fuse(parc-3, parv, parv[parc-3], parv[parc-2], parv[parc-1]); } else if (!strcmp(fun, "cluster")) { check(parc >= 3, "Insufficient parameter count for 'cluster'."); br_cluster(parc-2, parv, atof(parv[parc-2]), parv[parc-1]); @@ -215,7 +215,7 @@ private: "-plot ... {destination}\n" "\n" "==== Other Commands ====\n" - "-fuse ... (None|MinMax|ZScore|WScore) (Min|Max|Sum[W1:W2:...:Wn]|Replace|Difference|None) {simmat}\n" + "-fuse ... (None|MinMax|ZScore|WScore) (Min|Max|Sum[W1:W2:...:Wn]|Replace|Difference|None) {simmat}\n" "-cluster ... {csv}\n" "-makeMask {mask}\n" "-combineMasks ... {mask} (And|Or)\n" diff --git a/app/examples/face_recognition.cpp b/app/examples/face_recognition.cpp index d68635a..36a4447 100644 --- a/app/examples/face_recognition.cpp +++ b/app/examples/face_recognition.cpp @@ -45,7 +45,7 @@ int main(int argc, char *argv[]) // Initialize templates br::Template queryA("../data/MEDS/img/S354-01-t10_01.jpg"); - br::Template queryB("../data/MEDS/img/S386-04-t10_01.jpg"); + br::Template queryB("../data/MEDS/img/S382-08-t10_01.jpg"); br::Template target("../data/MEDS/img/S354-02-t10_01.jpg"); // Enroll templates diff --git a/data/INRIAPerson/README.md b/data/INRIAPerson/README.md new file mode 100644 index 0000000..61d79f6 --- /dev/null +++ b/data/INRIAPerson/README.md @@ -0,0 +1,3 @@ +## INRIA Person Database +Dataset for human detection in several formats: original positive and negative images with bounding box annotations and normalized positive images (just the bounding box). +* [Website](http://pascal.inrialpes.fr/data/human/) diff --git a/openbr/core/bee.cpp b/openbr/core/bee.cpp index f137dd3..bd7a92f 100644 --- a/openbr/core/bee.cpp +++ b/openbr/core/bee.cpp @@ -75,6 +75,21 @@ FileList BEE::readSigset(const File &sigset, bool ignoreMetadata) else if (!ignoreMetadata) file.set(key, value); } + // add bounding boxes, if they exist (will be child elements of ) + if (fileNode.hasChildNodes()) { + QList rects; + QDomNodeList bboxes = fileNode.childNodes(); + for (int i=0; i("Label",file.fileName()) +"\">"); + lines.append("\t("Label",file.baseName()) +"\">"); lines.append("\t\t"); lines.append("\t"); } @@ -302,7 +317,6 @@ cv::Mat BEE::makeMask(const br::FileList &targets, const br::FileList &queries, else if (partitionA != partition) val = DontCare; else if (partitionB == -1) val = NonMatch; else if (partitionB != partition) val = DontCare; - else if (partitionA != partitionB) val = DontCare; else if (labelA == labelB) val = Match; else val = NonMatch; mask.at(i,j) = val; diff --git a/openbr/core/core.cpp b/openbr/core/core.cpp index 1378302..71e3d51 100644 --- a/openbr/core/core.cpp +++ b/openbr/core/core.cpp @@ -43,6 +43,11 @@ struct AlgorithmCore { TemplateList data(TemplateList::fromGallery(input)); + // set the Train bool metadata, in case a Transform's project + // needs to know if it's called during train or enroll + for (int i=0; i #include +#include "openbr/core/opencvutils.h" #include #include #include @@ -36,8 +37,8 @@ static void normalizeMatrix(Mat &matrix, const Mat &mask, const QString &method) for (int j=0; j(i,j); if ((mask.at(i,j) == BEE::DontCare) || - (val == -std::numeric_limits::infinity()) || - (val == std::numeric_limits::infinity())) + (val == -std::numeric_limits::max()) || + (val == std::numeric_limits::max())) continue; vals.append(val); } @@ -53,20 +54,20 @@ static void normalizeMatrix(Mat &matrix, const Mat &mask, const QString &method) for (int j=0; j(i,j) == BEE::DontCare) continue; float &val = matrix.at(i,j); - if (val == -std::numeric_limits::infinity()) val = 0; - else if (val == std::numeric_limits::infinity()) val = 1; + if (val == -std::numeric_limits::max()) val = 0; + else if (val == std::numeric_limits::max()) val = 1; else val = (val - min) / (max - min); } } } else if (method == "ZScore") { if (stddev == 0) qFatal("Stddev is 0."); - for (int i=0; i(i,j) == BEE::DontCare) continue; float &val = matrix.at(i,j); - if (val == -std::numeric_limits::infinity()) val = (min - mean) / stddev; - else if (val == std::numeric_limits::infinity()) val = (max - mean) / stddev; - else val = (val - mean) / stddev; + if (val == -std::numeric_limits::max()) val = (min - mean) / stddev; + else if (val == std::numeric_limits::max()) val = (max - mean) / stddev; + else val = (val - mean) / stddev; } } } else { @@ -74,63 +75,90 @@ static void normalizeMatrix(Mat &matrix, const Mat &mask, const QString &method) } } -void br::Fuse(const QStringList &inputSimmats, File mask, const QString &normalization, const QString &fusion, const QString &outputSimmat) +void br::Fuse(const QStringList &inputSimmats, const QString &normalization, const QString &fusion, const QString &outputSimmat) { qDebug("Fusing %d to %s", inputSimmats.size(), qPrintable(outputSimmat)); - QList matrices; - foreach (const QString &simmat, inputSimmats) - matrices.append(BEE::readSimmat(simmat)); - if ((matrices.size() < 2) && (fusion != "None")) qFatal("Expected at least two similarity matrices."); - if ((matrices.size() > 1) && (fusion == "None")) qFatal("Expected exactly one similarity matrix."); - - mask.set("rows", matrices.first().rows); - mask.set("columns", matrices.first().cols); - Mat matrix_mask = BEE::readMask(mask); - - for (int i=0; i weights; - QStringList words = fusion.right(fusion.size()-3).split(":", QString::SkipEmptyParts); - if (words.size() == 0) { - for (int k=0; k originalMatrices; + foreach (const QString &simmat, inputSimmats) { + originalMatrices.append(BEE::readSimmat(simmat,&target,&query)); + // Make we're fusing score matrices for the same set of targets and querys + if (!previousTarget.isEmpty() && !previousQuery.isEmpty() && (previousTarget != target || previousQuery != query)) + qFatal("Target or query files are not the same across fused matrices."); + previousTarget = target; previousQuery = query; + } + + if ((originalMatrices.size() < 2) && (fusion != "None")) qFatal("Expected at least two similarity matrices."); + if ((originalMatrices.size() > 1) && (fusion == "None")) qFatal("Expected exactly one similarity matrix."); + + const FileList targetFiles = TemplateList::fromGallery(target).files(); + const FileList queryFiles = TemplateList::fromGallery(query).files(); + + int partition = 0; + int crossValidate = Globals->crossValidate; + Mat buffer = Mat::zeros(originalMatrices.last().size(),originalMatrices.last().type()); + + do { + QList matrices; + foreach (const Mat& matrix, originalMatrices) + matrices.append(matrix.clone()); + + Mat matrix_mask = BEE::makeMask(targetFiles,queryFiles,partition); + for (int i=0; i weights; + QStringList words = fusion.right(fusion.size()-3).split(":", QString::SkipEmptyParts); + if (words.size() == 0) { + for (int k=0; k.pro project. + * - \ref examples - Source code illustrating common use cases. + * - \ref help - Talk to the experts. * * \section learn_more Learn More * - \ref algorithm_grammar - How algorithms are constructed from string descriptions. @@ -37,6 +38,7 @@ * - \ref c_sdk - High-level API for running algorithms and evaluating results. * - \ref cpp_plugin_sdk - Plugin API for extending OpenBR functionality. * - \ref bee - A NIST standard for evaluating biometric algorithms. + * - \ref qmake_integration - Add OpenBR to your Qt .pro project. */ /*! @@ -397,6 +399,12 @@ $ br -help * \endcode */ + /*! + * \page help Help + * - Developer mailing list: openbr-dev@googlegroups.com + * - IRC Channel: irc.freenode.net#openbr + */ + /*! * \page qmake_integration QMake Integration * \brief Add OpenBR to your Qt .pro project. diff --git a/openbr/openbr_plugin.cpp b/openbr/openbr_plugin.cpp index bd0e28b..ccfae5c 100644 --- a/openbr/openbr_plugin.cpp +++ b/openbr/openbr_plugin.cpp @@ -441,10 +441,10 @@ TemplateList TemplateList::fromGallery(const br::File &gallery) // of target images to every partition newTemplates[i].file.set("Partition", -1); } else { - // Direct use of "Label" is not general -cao - const QByteArray md5 = QCryptographicHash::hash(newTemplates[i].file.get("Label").toLatin1(), QCryptographicHash::Md5); - // Select the right 8 hex characters so that it can be represented as a 64 bit integer without overflow - newTemplates[i].file.set("Partition", md5.toHex().right(8).toULongLong(0, 16) % crossValidate); + // Direct use of "Label" is not general -cao + const QByteArray md5 = QCryptographicHash::hash(newTemplates[i].file.get("Label").toLatin1(), QCryptographicHash::Md5); + // Select the right 8 hex characters so that it can be represented as a 64 bit integer without overflow + newTemplates[i].file.set("Partition", md5.toHex().right(8).toULongLong(0, 16) % crossValidate); } } } @@ -996,9 +996,9 @@ void br::Context::messageHandler(QtMsgType type, const QMessageLogContext &conte switch (type) { case QtWarningMsg: txt = QString("Warning: %1\n" ).arg(msg); break; case QtCriticalMsg: txt = QString("Critical: %1\n").arg(msg); break; - default: txt = QString("Fatal: %1\n" ).arg(msg); + default: txt = QString("Fatal: %1\n" ).arg(msg); break; } - txt += " File: " + QString(context.file) + "\n Function: " + QString(context.function) + "\n Line: " + QString::number(context.line) + "\n"; + txt += " SDK Path: " + Globals->sdkPath + "\n File: " + QString(context.file) + "\n Function: " + QString(context.function) + "\n Line: " + QString::number(context.line) + "\n"; } std::cerr << txt.toStdString(); diff --git a/openbr/openbr_plugin.h b/openbr/openbr_plugin.h index f33d00c..638cb3c 100644 --- a/openbr/openbr_plugin.h +++ b/openbr/openbr_plugin.h @@ -162,6 +162,7 @@ void reset_##NAME() { NAME = DEFAULT; } * Rects | QList | List of unnamed rects * Age | float | Age used for demographic filtering * Gender | QString | Subject gender + * Train | bool | The data is for training, as opposed to enrollment * _* | * | Reserved for internal use */ struct BR_EXPORT File diff --git a/openbr/plugins/crop.cpp b/openbr/plugins/crop.cpp index 30a75ea..d742fd4 100644 --- a/openbr/plugins/crop.cpp +++ b/openbr/plugins/crop.cpp @@ -60,8 +60,13 @@ class ROITransform : public UntrainableTransform void project(const Template &src, Template &dst) const { - foreach (const QRectF &rect, src.file.rects()) - dst += src.m()(OpenCVUtils::toRect(rect)); + if (src.file.rects().empty()) { + dst = src; + qWarning("No rects present in file."); + } + else + foreach (const QRectF &rect, src.file.rects()) + dst += src.m()(OpenCVUtils::toRect(rect)); } }; diff --git a/openbr/plugins/distance.cpp b/openbr/plugins/distance.cpp index 7f0cf71..d4a3808 100644 --- a/openbr/plugins/distance.cpp +++ b/openbr/plugins/distance.cpp @@ -310,7 +310,7 @@ class OnlineDistance : public Distance Q_PROPERTY(br::Distance* distance READ get_distance WRITE set_distance RESET reset_distance STORED false) Q_PROPERTY(float alpha READ get_alpha WRITE set_alpha RESET reset_alpha STORED false) BR_PROPERTY(br::Distance*, distance, NULL) - BR_PROPERTY(float, alpha, 0.1f); + BR_PROPERTY(float, alpha, 0.1f) mutable QHash scoreHash; mutable QMutex mutex; diff --git a/openbr/plugins/filter.cpp b/openbr/plugins/filter.cpp index 016f94e..ef9265f 100644 --- a/openbr/plugins/filter.cpp +++ b/openbr/plugins/filter.cpp @@ -139,10 +139,8 @@ class CSDNTransform : public UntrainableTransform const int surround = s/2; - for ( int i = 0; i < nRows; i++ ) - { - for ( int j = 0; j < nCols; j++ ) - { + for ( int i = 0; i < nRows; i++ ) { + for ( int j = 0; j < nCols; j++ ) { int width = min( j+surround, nCols ) - max( 0, j-surround ); int height = min( i+surround, nRows ) - max( 0, i-surround ); @@ -154,7 +152,7 @@ class CSDNTransform : public UntrainableTransform } } - dst = m; + dst = m; } }; diff --git a/openbr/plugins/gallery.cpp b/openbr/plugins/gallery.cpp index 7a1f59d..43a52b2 100644 --- a/openbr/plugins/gallery.cpp +++ b/openbr/plugins/gallery.cpp @@ -119,6 +119,7 @@ class galGallery : public Gallery { if (t.isEmpty() && t.file.isNull()) return; + stream << t; } }; @@ -856,8 +857,18 @@ class FDDBGallery : public Gallery for (int i=0; icompute(src, keyPoints, dst); } }; diff --git a/openbr/plugins/landmarks.cpp b/openbr/plugins/landmarks.cpp index 020dec1..a87a161 100644 --- a/openbr/plugins/landmarks.cpp +++ b/openbr/plugins/landmarks.cpp @@ -269,22 +269,14 @@ class DelaunayTransform : public UntrainableTransform Mat output(src.m().rows,src.m().cols,src.m().type()); - // Optimization needed if (i > 0) { Mat overlap; bitwise_and(dst.m(),mask,overlap); - for (int j = 0; j < overlap.rows; j++) { - for (int k = 0; k < overlap.cols; k++) { - if (overlap.at(j,k) != 0) { - mask.at(j,k) = 0; - } - } - } + mask.setTo(0, overlap!=0); } bitwise_and(buffer,mask,output); - dst.m() += output; } @@ -297,53 +289,6 @@ class DelaunayTransform : public UntrainableTransform BR_REGISTER(Transform, DelaunayTransform) -/*! - * \ingroup transforms - * \brief Loads a set of fiduciary points from a .dat file - * \author Scott Klum \cite sklum - */ -class LoadLandmarksTransform : public UntrainableTransform -{ - Q_OBJECT - - Q_PROPERTY(QString filePath READ get_filePath WRITE set_filePath RESET reset_filePath STORED false) - BR_PROPERTY(QString, filePath, QString()) - - void project(const Template &src, Template &dst) const - { - dst = src; - - // Assume the fiduciary file has the same basename as src - QString path = filePath + "/" + src.file.baseName() + ".dat"; - - QFile f(path); - if (!f.open(QIODevice::ReadOnly)) qFatal("Unable to open %s for reading.", qPrintable(path)); - - QList landmarks; - while(!f.atEnd()) { - QByteArray line = f.readLine(); - QString pointSet(line); - pointSet = pointSet.simplified(); - if (!pointSet.isEmpty()) { - QStringList points = pointSet.split(" "); - landmarks.append(QPointF(points[0].toFloat(),points[1].toFloat())); - } - } - - if (landmarks.size() < 35) qFatal("Unrecognized landmark set format."); - - dst.file.set("rightEye", landmarks[16]); - dst.file.set("leftEye", landmarks[18]); - - landmarks.removeAt(18); - landmarks.removeAt(16); - - dst.file.appendPoints(landmarks); - } -}; - -BR_REGISTER(Transform, LoadLandmarksTransform) - } // namespace br #include "landmarks.moc" diff --git a/openbr/plugins/misc.cpp b/openbr/plugins/misc.cpp index 915d391..9b66378 100644 --- a/openbr/plugins/misc.cpp +++ b/openbr/plugins/misc.cpp @@ -382,53 +382,6 @@ BR_REGISTER(Transform, RegexPropertyTransform) /*! * \ingroup transforms - * \brief Remove templates with the specified file extension or metadata value. - * \author Josh Klontz \cite jklontz - */ -class RemoveTemplatesTransform : public UntrainableMetaTransform -{ - Q_OBJECT - Q_PROPERTY(QString regexp READ get_regexp WRITE set_regexp RESET reset_regexp STORED false) - Q_PROPERTY(QString key READ get_key WRITE set_key RESET reset_key STORED false) - BR_PROPERTY(QString, regexp, "") - BR_PROPERTY(QString, key, "") - - void project(const Template &src, Template &dst) const - { - const QRegularExpression re(regexp); - const QRegularExpressionMatch match = re.match(key.isEmpty() ? src.file.suffix() : src.file.get(key)); - if (match.hasMatch()) dst = Template(); - else dst = src; - } -}; - -BR_REGISTER(Transform, RemoveTemplatesTransform) - -/*! - * \ingroup transforms - * \brief Remove template metadata with the specified key(s). - * \author Josh Klontz \cite jklontz - */ -class RemoveMetadataTransform : public UntrainableMetaTransform -{ - Q_OBJECT - Q_PROPERTY(QString regexp READ get_regexp WRITE set_regexp RESET reset_regexp STORED false) - BR_PROPERTY(QString, regexp, "") - - void project(const Template &src, Template &dst) const - { - dst = src; - const QRegularExpression re(regexp); - foreach (const QString &key, dst.file.localKeys()) - if (re.match(key).hasMatch()) - dst.file.remove(key); - } -}; - -BR_REGISTER(Transform, RemoveMetadataTransform) - -/*! - * \ingroup transforms * \brief Store the last matrix of the input template as a metadata key with input property name. * \author Charles Otto \cite caotto */ diff --git a/openbr/plugins/output.cpp b/openbr/plugins/output.cpp index f2a3be4..7b643a6 100644 --- a/openbr/plugins/output.cpp +++ b/openbr/plugins/output.cpp @@ -249,7 +249,8 @@ BR_REGISTER(Output, mtxOutput) /*! * \ingroup outputs * \brief Rank retrieval output. - * \author Josh Klontz \cite jklontz Scott Klum \cite sklum + * \author Josh Klontz \cite jklontz + * \author Scott Klum \cite sklum */ class rrOutput : public MatrixOutput { @@ -267,15 +268,17 @@ class rrOutput : public MatrixOutput for (int i=0; i Pair; foreach (const Pair &pair, Common::Sort(OpenCVUtils::matrixToVector(data.row(i)), true, limit)) { - if (pair.first < threshold) break; - File target = targetFiles[pair.second]; - target.set("Score", QString::number(pair.first)); - if (simple) files.append(target.baseName() + " " + QString::number(pair.first)); - else files.append(target.flat()); + if (Globals->crossValidate > 0 ? (targetFiles[pair.second].get("Partition",-1) == -1 || targetFiles[pair.second].get("Partition",-1) == queryFiles[i].get("Partition",-1)) : true) { + if (pair.first < threshold) break; + File target = targetFiles[pair.second]; + target.set("Score", QString::number(pair.first)); + if (simple) files.append(target.baseName() + " " + QString::number(pair.first)); + else files.append(target.flat()); + } } lines.append(files.join(byLine ? "\n" : ",")); } @@ -428,7 +431,7 @@ class rankOutput : public MatrixOutput typedef QPair Pair; int rank = 1; foreach (const Pair &pair, Common::Sort(OpenCVUtils::matrixToVector(data.row(i)), true)) { - if (Globals->crossValidate > 0 ? (targetFiles[pair.second].get("Partition",-1) == queryFiles[i].get("Partition",-1)) : true) { + if (Globals->crossValidate > 0 ? (targetFiles[pair.second].get("Partition",-1) == -1 || targetFiles[pair.second].get("Partition",-1) == queryFiles[i].get("Partition",-1)) : true) { if (QString(targetFiles[pair.second]) != QString(queryFiles[i])) { if (targetFiles[pair.second].get("Label") == queryFiles[i].get("Label")) { ranks.append(rank); diff --git a/openbr/plugins/pp5.cpp b/openbr/plugins/pp5.cpp index 0c934fb..6587603 100644 --- a/openbr/plugins/pp5.cpp +++ b/openbr/plugins/pp5.cpp @@ -154,10 +154,10 @@ struct PP5Context ppr_face_attributes_type face_attributes; ppr_get_face_attributes(face, &face_attributes); - metadata.insert("Face", QRectF(face_attributes.position.x - face_attributes.dimensions.width/2, - face_attributes.position.y - face_attributes.dimensions.height/2, - face_attributes.dimensions.width, - face_attributes.dimensions.height)); + metadata.insert("FrontalFace", QRectF(face_attributes.position.x - face_attributes.dimensions.width/2, + face_attributes.position.y - face_attributes.dimensions.height/2, + face_attributes.dimensions.width, + face_attributes.dimensions.height)); metadata.insert("PP5_Face_Confidence", face_attributes.confidence); metadata.insert("PP5_Face_Roll", face_attributes.rotation.roll); metadata.insert("PP5_Face_Pitch", face_attributes.rotation.pitch); @@ -240,7 +240,8 @@ class PP5EnrollTransform : public UntrainableMetaTransform PP5Context *context = contexts.acquire(); - foreach(const Template & src, srcList) { + foreach (const Template &src, srcList) { + bool foundFace = false; if (!src.isEmpty()) { ppr_raw_image_type raw_image; PP5Context::createRawImage(src, raw_image); @@ -254,6 +255,7 @@ class PP5EnrollTransform : public UntrainableMetaTransform int extractable; TRY(ppr_is_template_extractable(context->context, face, &extractable)) if (!extractable && !detectOnly) continue; + else foundFace = true; cv::Mat m; if (detectOnly) { @@ -272,20 +274,18 @@ class PP5EnrollTransform : public UntrainableMetaTransform // Found a face, nothing else to do (if we aren't trying to find multiple faces). if (!Globals->enrollAll) break; - } + } ppr_free_face_list(face_list); ppr_free_image(image); ppr_raw_image_free(raw_image); } - } - // No faces were detected, output something with FTE set. - if (dstList.empty()) { - dstList.append(srcList.first()); - dstList.first().file.set("FTE",true); - if (!detectOnly) - dstList.first().m() = cv::Mat(); + // No faces were detected when we were expecting one, output something with FTE set. + if (!foundFace && !Globals->enrollAll) { + dstList.append(Template(src.file, detectOnly ? src.m() : cv::Mat())); + dstList.last().file.set("FTE", true); + } } contexts.release(context); diff --git a/openbr/plugins/slidingwindow.cpp b/openbr/plugins/slidingwindow.cpp new file mode 100644 index 0000000..27a37a2 --- /dev/null +++ b/openbr/plugins/slidingwindow.cpp @@ -0,0 +1,105 @@ +#include "openbr_internal.h" +#include "openbr/core/opencvutils.h" +#include "openbr/core/common.h" + +using namespace cv; + +namespace br +{ + +/*! + * \ingroup transforms + * \brief Applies a transform to a sliding window. + * Discards negative detections. + * \author Austin Blanton \cite imaus10 + */ +class SlidingWindowTransform : public Transform +{ + Q_OBJECT + Q_PROPERTY(br::Transform *transform READ get_transform WRITE set_transform RESET reset_transform STORED false) + Q_PROPERTY(int minSize READ get_minSize WRITE set_minSize RESET reset_minSize STORED false) + Q_PROPERTY(double scaleFactor READ get_scaleFactor WRITE set_scaleFactor RESET reset_scaleFactor STORED false) + Q_PROPERTY(double stepSize READ get_stepSize WRITE set_stepSize RESET reset_stepSize STORED false) + Q_PROPERTY(bool takeLargestScale READ get_takeLargestScale WRITE set_takeLargestScale RESET reset_takeLargestScale STORED false) + Q_PROPERTY(bool negSamples READ get_negSamples WRITE set_negSamples RESET reset_negSamples STORED false) + Q_PROPERTY(int negToPosRatio READ get_negToPosRatio WRITE set_negToPosRatio RESET reset_negToPosRatio STORED false) + Q_PROPERTY(double maxOverlap READ get_maxOverlap WRITE set_maxOverlap RESET reset_maxOverlap STORED false) + BR_PROPERTY(br::Transform *, transform, NULL) + BR_PROPERTY(int, minSize, 8) + BR_PROPERTY(double, scaleFactor, 0.75) + BR_PROPERTY(double, stepSize, 1) + BR_PROPERTY(bool, takeLargestScale, true) + BR_PROPERTY(bool, negSamples, true) + BR_PROPERTY(int, negToPosRatio, 1) + BR_PROPERTY(double, maxOverlap, 0) + +public: + SlidingWindowTransform() : Transform(false, true) {} + +private: + void train(const TemplateList &data) + { + if (transform->trainable) { + TemplateList full; + foreach (const Template &tmpl, data) { + foreach (const Rect &rect, OpenCVUtils::toRects(tmpl.file.rects())) { + Template pos(tmpl.file, Mat(tmpl, rect)); + full += pos; + + // add random negative samples + if (negSamples) { + Mat m = tmpl.m(); + int sample = 0; + while (sample < negToPosRatio) { + int x = Common::RandSample(1, m.cols)[0]; + int y = Common::RandSample(1, m.rows)[0]; + int maxWidth = m.cols - x, maxHeight = m.rows - y; + int maxSize = std::min(maxWidth, maxHeight); + int size = (maxSize <= minSize ? maxSize : Common::RandSample(1, maxSize, minSize)[0]); + Rect negRect(x, y, size, size); + Rect intersect = negRect & rect; + if (intersect.area() > maxOverlap*rect.area()) + continue; + Template neg(tmpl.file, Mat(tmpl, negRect)); + neg.file.set("Label", QString("neg")); + full += neg; + sample++; + } + } + } + } + transform->train(full); + } + } + + void project(const Template &src, Template &dst) const + { + dst = src; + // no need to slide a window over ground truth data + if (src.file.getBool("Train", false)) return; + + dst.file.clearRects(); + int rows = src.m().rows, cols = src.m().cols; + for (double size=std::min(rows, cols); size>=minSize; size*=scaleFactor) { + for (double y=0; y+sizeproject(windowMat, detect); + // the result will be in the Label + if (detect.file.get(QString("Label")) == "pos") { + dst.file.appendRect(OpenCVUtils::fromRect(window)); + if (takeLargestScale) return; + } + } + } + } + } +}; + +BR_REGISTER(Transform, SlidingWindowTransform) + +} // namespace br + +#include "slidingwindow.moc" diff --git a/openbr/plugins/stasm4.cpp b/openbr/plugins/stasm4.cpp index c8195b1..0e9f667 100644 --- a/openbr/plugins/stasm4.cpp +++ b/openbr/plugins/stasm4.cpp @@ -56,6 +56,10 @@ class StasmTransform : public UntrainableTransform Q_PROPERTY(bool stasm3Format READ get_stasm3Format WRITE set_stasm3Format RESET reset_stasm3Format STORED false) BR_PROPERTY(bool, stasm3Format, false) + Q_PROPERTY(bool clearLandmarks READ get_clearLandmarks WRITE set_clearLandmarks RESET reset_clearLandmarks STORED false) + BR_PROPERTY(bool, clearLandmarks, false) + Q_PROPERTY(QStringList pinEyes READ get_pinEyes WRITE set_pinEyes RESET reset_pinEyes STORED false) + BR_PROPERTY(QStringList, pinEyes, QStringList()) Resource stasmCascadeResource; @@ -69,12 +73,49 @@ class StasmTransform : public UntrainableTransform { if (src.m().channels() != 1) qFatal("Stasm expects single channel matrices."); + dst = src; + StasmCascadeClassifier *stasmCascade = stasmCascadeResource.acquire(); int foundface; int nLandmarks = stasm_NLANDMARKS; float landmarks[2 * stasm_NLANDMARKS]; - stasm_search_single(&foundface, landmarks, reinterpret_cast(src.m().data), src.m().cols, src.m().rows, *stasmCascade, NULL, NULL); + + if (!pinEyes.isEmpty()) { + // Two use cases are accounted for: + // 1. Pin eyes without normalization: in this case the string list should contain the KEYS for right then left eyes, respectively. + // 2. Pin eyes with normalization: in this case the string list should contain the COORDINATES of the right then left eyes, respectively. + // Note that for case 2, if Affine_0 and Affine_1 are not present (indicating no normalization has taken place), we default to stasm_search_single. + + bool ok = false; + QPointF rightEye; + QPointF leftEye; + + if (src.file.contains("Affine_0") && src.file.contains("Affine_1")) { + rightEye = QtUtils::toPoint(pinEyes.at(0),&ok); + leftEye = QtUtils::toPoint(pinEyes.at(1),&ok); + } + + if (!ok) { + rightEye = QtUtils::toPoint(src.file.get(pinEyes.at(0), QString()),&ok); + leftEye = QtUtils::toPoint(src.file.get(pinEyes.at(1), QString()),&ok); + } + + float eyes[2 * stasm_NLANDMARKS]; + + if (ok) { + for (int i = 0; i < nLandmarks; i++) { + if (i == 38) /*Stasm Right Eye*/ { eyes[2*i] = rightEye.x(); eyes[2*i+1] = rightEye.y(); } + else if (i == 39) /*Stasm Left Eye*/ { eyes[2*i] = leftEye.x(); eyes[2*i+1] = leftEye.y(); } + else { eyes[2*i] = 0; eyes[2*i+1] = 0; } + } + } else qFatal("Unable to interpret pinned eyes."); + + stasm_search_pinned(landmarks, eyes, reinterpret_cast(src.m().data), src.m().cols, src.m().rows, NULL); + + // The ASM in Stasm is guaranteed to converge in this case + foundface = 1; + } stasm_search_single(&foundface, landmarks, reinterpret_cast(src.m().data), src.m().cols, src.m().rows, *stasmCascade, NULL, NULL); if (stasm3Format) { nLandmarks = 76; @@ -83,13 +124,22 @@ class StasmTransform : public UntrainableTransform stasmCascadeResource.release(stasmCascade); - if (!foundface) qWarning("No face found in %s", qPrintable(src.file.fileName())); - else { - for (int i = 0; i < nLandmarks; i++) - dst.file.appendPoint(QPointF(landmarks[2 * i], landmarks[2 * i + 1])); + // For convenience, if these are the only points/rects we want to deal with as the algorithm progresses + if (clearLandmarks) { + dst.file.clearPoints(); + dst.file.clearRects(); } - dst.m() = src.m(); + if (!foundface) { + qWarning("No face found in %s", qPrintable(src.file.fileName())); + } else { + for (int i = 0; i < nLandmarks; i++) { + QPointF point(landmarks[2 * i], landmarks[2 * i + 1]); + dst.file.appendPoint(point); + if (i == 38) dst.file.set("StasmRightEye",point); + else if (i == 39) dst.file.set("StasmLeftEye", point); + } + } } }; diff --git a/openbr/plugins/template.cpp b/openbr/plugins/template.cpp index dd5f88d..cd68a32 100644 --- a/openbr/plugins/template.cpp +++ b/openbr/plugins/template.cpp @@ -1,4 +1,5 @@ #include "openbr_internal.h" +#include namespace br { @@ -25,6 +26,30 @@ class RetainTransform : public UntrainableTransform BR_REGISTER(Transform, RetainTransform) +/*! + * \ingroup transforms + * \brief Remove templates with the specified file extension or metadata value. + * \author Josh Klontz \cite jklontz + */ +class RemoveTemplatesTransform : public UntrainableMetaTransform +{ + Q_OBJECT + Q_PROPERTY(QString regexp READ get_regexp WRITE set_regexp RESET reset_regexp STORED false) + Q_PROPERTY(QString key READ get_key WRITE set_key RESET reset_key STORED false) + BR_PROPERTY(QString, regexp, "") + BR_PROPERTY(QString, key, "") + + void project(const Template &src, Template &dst) const + { + const QRegularExpression re(regexp); + const QRegularExpressionMatch match = re.match(key.isEmpty() ? src.file.suffix() : src.file.get(key)); + if (match.hasMatch()) dst = Template(); + else dst = src; + } +}; + +BR_REGISTER(Transform, RemoveTemplatesTransform) + } // namespace br #include "template.moc" diff --git a/openbr/plugins/validate.cpp b/openbr/plugins/validate.cpp index a40bc9c..49b2c03 100644 --- a/openbr/plugins/validate.cpp +++ b/openbr/plugins/validate.cpp @@ -75,6 +75,7 @@ class CrossValidateTransform : public MetaTransform if (subjectIndices.size() > 1 && subjectIndices.size() <= i) { removed.append(subjectIndices[i%subjectIndices.size()]); } + // For the time being, we don't support addition training data added to every fold in the case of leaveOneImageOut else if (partitionsBuffer[j] == i) { removed.append(j); } @@ -87,7 +88,12 @@ class CrossValidateTransform : public MetaTransform } else { j--; } + } else if (partitions[j] == -1) { + // Keep data for training, but modify the partition so we project into the correct space + partitionedData[j].file.set("Partition",i); + j--; } else if (partitions[j] == i) { + // Remove data, it's designated for testing partitionedData.removeAt(j); j--; } else j--; @@ -99,7 +105,8 @@ class CrossValidateTransform : public MetaTransform } void project(const Template &src, Template &dst) const - { + { + qDebug() << src.file.get("Partition", 0); transforms[src.file.get("Partition", 0)]->project(src, dst); } diff --git a/scripts/downloadDatasets.sh b/scripts/downloadDatasets.sh index 37daff3..734573c 100755 --- a/scripts/downloadDatasets.sh +++ b/scripts/downloadDatasets.sh @@ -35,6 +35,24 @@ if [ ! -d ../data/BioID/img ]; then rm *.eye description.txt BioID-FaceDatabase-V1.2.zip fi +# INRIA person +if [ ! -d ../data/INRIAPerson/img ]; then + echo "Downloading INRIA person dataset..." + if hash curl 2>/dev/null; then + curl -OL http://pascal.inrialpes.fr/data/human/INRIAPerson.tar + else + wget http://pascal.inrialpes.fr/data/human/INRIAPerson.tar + fi + tar -xf INRIAPerson.tar + mkdir ../data/INRIAPerson/img ../data/INRIAPerson/sigset + ./writeINRIAPersonSigset.sh Train > ../data/INRIAPerson/sigset/train.xml + ./writeINRIAPersonSigset.sh Test > ../data/INRIAPerson/sigset/test.xml + ./writeINRIAPersonSigset.sh train_64x128_H96 > ../data/INRIAPerson/sigset/train_normalized.xml + ./writeINRIAPersonSigset.sh test_64x128_H96 > ../data/INRIAPerson/sigset/test_normalized.xml + mv INRIAPerson/* ../data/INRIAPerson/img + rm -r INRIAPerson* +fi + # KTH if [ ! -d ../data/KTH/vid ]; then echo "Downloading KTH..." diff --git a/scripts/writeINRIAPersonSigset.sh b/scripts/writeINRIAPersonSigset.sh new file mode 100755 index 0000000..b3aac66 --- /dev/null +++ b/scripts/writeINRIAPersonSigset.sh @@ -0,0 +1,47 @@ +#!/bin/bash + +# prints out a element from rectangle coordinates +# (from the ViPER standard: http://viper-toolkit.sourceforge.net/docs/file/) +printBBox() +{ + width=$(($3-$1)) + height=$(($4-$2)) + echo -e "\t\t\t" +} +# export printBBox so xargs can call it using bash -c below +export -f printBBox +SEDREGEX='s/.*(\([0-9]*\), \([0-9]*\)) - (\([0-9]*\), \([0-9]*\))/printBBox \1 \2 \3 \4/' + +echo '' +echo '' + +# print out the positive image sigs +for fullpath in INRIAPerson/$1/pos/*; do + # get just the filename, minus the path + filename=$(basename "$fullpath") + echo -e "\t" + + # if this folder has annotations, add bounding boxes + echo -en "\t\t" + annotation="INRIAPerson/$1/annotations/${filename%.*}.txt" + grep 'Bounding box' $annotation | sed "$SEDREGEX" | xargs -n 5 bash -c 'printBBox $@' + echo -e "\t\t" + # otherwise, just end the presentation + else + echo " />" + fi + + echo -e '\t' +done + +# print out the negative image sigs +for fullpath in INRIAPerson/$1/neg/*; do + filename=$(basename "$fullpath") + echo -e "\t" + echo -e "\t\t" + echo -e '\t' +done + +echo ''