diff --git a/openbr/openbr_plugin.cpp b/openbr/openbr_plugin.cpp index 4f61df9..4963867 100644 --- a/openbr/openbr_plugin.cpp +++ b/openbr/openbr_plugin.cpp @@ -234,6 +234,11 @@ void File::appendRect(const QRectF &rect) m_metadata["Rects"] = newRects; } +void File::appendRect(const Rect &rect) +{ + appendRect(OpenCVUtils::fromRect(rect)); +} + void File::appendRects(const QList &rects) { QList newRects = m_metadata["Rects"].toList(); @@ -242,6 +247,11 @@ void File::appendRects(const QList &rects) m_metadata["Rects"] = newRects; } +void File::appendRects(const QList &rects) +{ + appendRects(OpenCVUtils::fromRects(rects)); +} + /* File - private methods */ void File::init(const QString &file) { diff --git a/openbr/openbr_plugin.h b/openbr/openbr_plugin.h index d551bb0..eb563c9 100644 --- a/openbr/openbr_plugin.h +++ b/openbr/openbr_plugin.h @@ -41,8 +41,6 @@ #include #include #include -#include -#include /*! * \defgroup cpp_plugin_sdk C++ Plugin SDK @@ -222,7 +220,11 @@ struct BR_EXPORT File template void setList(const QString &key, const QList &value) { - set(key, QtUtils::toVariantList(value)); + QVariantList variantList; + variantList.reserve(value.size()); + foreach (const T &item, value) + variantList << item; + set(key, variantList); } inline void remove(const QString &key) { m_metadata.remove(key); } /*!< \brief Remove the metadata key. */ @@ -315,9 +317,9 @@ struct BR_EXPORT File QList namedRects() const; /*!< \brief Returns rects convertible from metadata values. */ QList rects() const; /*!< \brief Returns the file's rects list. */ void appendRect(const QRectF &rect); /*!< \brief Adds a rect to the file's rect list. */ - void appendRect(const cv::Rect &rect) { appendRect(OpenCVUtils::fromRect(rect)); } /*!< \brief Adds a rect to the file's rect list. */ + void appendRect(const cv::Rect &rect); /*!< \brief Adds a rect to the file's rect list. */ void appendRects(const QList &rects); /*!< \brief Adds rects to the file's rect list. */ - void appendRects(const QList &rects) { appendRects(OpenCVUtils::fromRects(rects)); } /*!< \brief Adds rects to the file's rect list. */ + void appendRects(const QList &rects); /*!< \brief Adds rects to the file's rect list. */ inline void clearRects() { m_metadata["Rects"] = QList(); } /*!< \brief Clears the file's rect list. */ inline void setRects(const QList &rects) { clearRects(); appendRects(rects); } /*!< \brief Overwrites the file's rect list. */ inline void setRects(const QList &rects) { clearRects(); appendRects(rects); } /*!< \brief Overwrites the file's rect list. */ diff --git a/openbr/plugins/eigen3.cpp b/openbr/plugins/eigen3.cpp index accea75..1b71955 100644 --- a/openbr/plugins/eigen3.cpp +++ b/openbr/plugins/eigen3.cpp @@ -302,11 +302,13 @@ class LDATransform : public Transform Q_PROPERTY(int directLDA READ get_directLDA WRITE set_directLDA RESET reset_directLDA STORED false) Q_PROPERTY(float directDrop READ get_directDrop WRITE set_directDrop RESET reset_directDrop STORED false) Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false) + Q_PROPERTY(bool isBinary READ get_isBinary WRITE set_isBinary RESET reset_isBinary STORED false) BR_PROPERTY(float, pcaKeep, 0.98) BR_PROPERTY(bool, pcaWhiten, false) BR_PROPERTY(int, directLDA, 0) BR_PROPERTY(float, directDrop, 0.1) BR_PROPERTY(QString, inputVariable, "Label") + BR_PROPERTY(bool, isBinary, false) int dimsOut; Eigen::VectorXf mean; @@ -316,7 +318,6 @@ class LDATransform : public Transform { // creates "Label" TemplateList trainingSet = TemplateList::relabel(_trainingSet, inputVariable); - int instances = trainingSet.size(); // Perform PCA dimensionality reduction @@ -450,6 +451,34 @@ class LDATransform : public Transform // Compute final projection matrix projection = ((space2.eVecs.transpose() * space1.eVecs.transpose()) * pca.eVecs.transpose()).transpose(); dimsOut = dim2; + + if (isBinary) { + assert(dimsOut == 1); + TemplateList projected; + float posVal = 0; + float negVal = 0; + for (int i = 0; i < trainingSet.size(); i++) { + Template t; + project(trainingSet[i],t); + //Note: the positive class is assumed to be 0 b/c it will + // typically be the first gallery template in the TemplateList structure + if (classes[i] == 0) + posVal += t.m().at(0,0); + else if (classes[i] == 1) + negVal += t.m().at(0,0); + else + qFatal("Binary mode only supports two class problems."); + } + posVal /= classCounts[0]; + negVal /= classCounts[1]; + + if (posVal < negVal) { + //Ensure positive value is supposed to be > 0 after projection + Eigen::MatrixXf invert = Eigen::MatrixXf::Ones(dimsIn,1); + invert *= -1; + projection = invert.transpose() * projection; + } + } } void project(const Template &src, Template &dst) const @@ -462,6 +491,10 @@ class LDATransform : public Transform // Do projection outMap = projection.transpose() * (inMap - mean); + + if (isBinary) { + dst.file.set("conf",dst.m().at(0,0)); + } } void store(QDataStream &stream) const diff --git a/openbr/plugins/slidingwindow.cpp b/openbr/plugins/slidingwindow.cpp index 84c93a2..6e0156f 100644 --- a/openbr/plugins/slidingwindow.cpp +++ b/openbr/plugins/slidingwindow.cpp @@ -8,11 +8,29 @@ using namespace cv; +namespace br +{ + // Because MSVC doesn't provide a round() function in math.h static int round(float x) { return (floor(x + 0.5)); } - -namespace br +// Find avg aspect ratio +static float getAspectRatio(const TemplateList &data) { + double tempRatio = 0; + int ratioCnt = 0; + + foreach (const Template &tmpl, data) { + QList posRects = OpenCVUtils::toRects(tmpl.file.rects()); + foreach (const Rect &posRect, posRects) { + if (posRect.x + posRect.width >= tmpl.m().cols || posRect.y + posRect.height >= tmpl.m().rows || posRect.x < 0 || posRect.y < 0) { + continue; + } + tempRatio += (float)posRect.width / (float)posRect.height; + ratioCnt += 1; + } + } + return tempRatio / (double)ratioCnt; +} /*! * \ingroup transforms @@ -25,23 +43,19 @@ class SlidingWindowTransform : public Transform Q_OBJECT Q_PROPERTY(br::Transform *transform READ get_transform WRITE set_transform RESET reset_transform STORED false) Q_PROPERTY(int minSize READ get_minSize WRITE set_minSize RESET reset_minSize STORED false) - Q_PROPERTY(double scaleFactor READ get_scaleFactor WRITE set_scaleFactor RESET reset_scaleFactor STORED false) Q_PROPERTY(int stepSize READ get_stepSize WRITE set_stepSize RESET reset_stepSize STORED false) - Q_PROPERTY(bool takeLargestScale READ get_takeLargestScale WRITE set_takeLargestScale RESET reset_takeLargestScale STORED false) + Q_PROPERTY(bool takeFirst READ get_takeFirst WRITE set_takeFirst RESET reset_takeFirst STORED false) Q_PROPERTY(bool negSamples READ get_negSamples WRITE set_negSamples RESET reset_negSamples STORED false) Q_PROPERTY(int negToPosRatio READ get_negToPosRatio WRITE set_negToPosRatio RESET reset_negToPosRatio STORED false) Q_PROPERTY(double maxOverlap READ get_maxOverlap WRITE set_maxOverlap RESET reset_maxOverlap STORED false) - Q_PROPERTY(float aspectRatio READ get_aspectRatio WRITE set_aspectRatio RESET reset_aspectRatio STORED true) Q_PROPERTY(int windowWidth READ get_windowWidth WRITE set_windowWidth RESET reset_windowWidth STORED false) BR_PROPERTY(br::Transform *, transform, NULL) BR_PROPERTY(int, minSize, 8) - BR_PROPERTY(double, scaleFactor, 0.75) BR_PROPERTY(int, stepSize, 1) - BR_PROPERTY(bool, takeLargestScale, true) + BR_PROPERTY(bool, takeFirst, true) BR_PROPERTY(bool, negSamples, true) BR_PROPERTY(int, negToPosRatio, 1) BR_PROPERTY(double, maxOverlap, 0) - BR_PROPERTY(float, aspectRatio, 1) BR_PROPERTY(int, windowWidth, 24) public: @@ -50,24 +64,14 @@ private: void train(const TemplateList &data) { + // only calculate if the work hasn't been done + aspectRatio = data.first().file.get("aspectRatio", -1); + if (aspectRatio == -1) + aspectRatio = getAspectRatio(data); + if (transform->trainable) { - double tempRatio = 0; - int ratioCnt = 0; TemplateList full; - //First find avg aspect ratio - foreach (const Template &tmpl, data) { - QList posRects = OpenCVUtils::toRects(tmpl.file.rects()); - foreach (const Rect &posRect, posRects) { - if (posRect.x + posRect.width >= tmpl.m().cols || posRect.y + posRect.height >= tmpl.m().rows || posRect.x < 0 || posRect.y < 0) { - continue; - } - tempRatio += (float)posRect.width / (float)posRect.height; - ratioCnt += 1; - } - } - aspectRatio = tempRatio / (double)ratioCnt; - foreach (const Template &tmpl, data) { QList posRects = OpenCVUtils::toRects(tmpl.file.rects()); QList negRects; @@ -132,6 +136,72 @@ private: if (src.file.getBool("Train", false)) return; dst.file.clearRects(); + int windowHeight = (int) round((float) windowWidth / aspectRatio); + int scale = src.file.get("scale", 1); + + for (double y = 0; y + windowHeight < src.m().rows; y += stepSize) { + for (double x = 0; x + windowWidth < src.m().cols; x += stepSize) { + Rect window(x, y, windowWidth, windowHeight); + Template windowMat(src.file, Mat(src, window)); + Template detect; + transform->project(windowMat, detect); + float conf = detect.file.get("conf"); + + // the result will be in the Label + if (conf > 0) { + dst.file.appendRect(QRectF((float) x * scale, (float) y * scale, (float) windowWidth * scale, (float) windowHeight * scale)); + QList confidences = dst.file.getList("Confidences", QList()); + confidences.append(conf); + dst.file.setList("Confidences", confidences); + if (takeFirst) + return; + } + } + } + } + + float aspectRatio; +}; + +BR_REGISTER(Transform, SlidingWindowTransform) + +/*! + * \ingroup transforms + * \brief . + * \author Austin Blanton \cite imaus10 + */ +class BuildScalesTransform : public Transform +{ + Q_OBJECT + Q_PROPERTY(br::Transform *transform READ get_transform WRITE set_transform RESET reset_transform STORED false) + Q_PROPERTY(double scaleFactor READ get_scaleFactor WRITE set_scaleFactor RESET reset_scaleFactor STORED false) + Q_PROPERTY(bool takeLargestScale READ get_takeLargestScale WRITE set_takeLargestScale RESET reset_takeLargestScale STORED false) + Q_PROPERTY(int windowWidth READ get_windowWidth WRITE set_windowWidth RESET reset_windowWidth STORED false) + BR_PROPERTY(br::Transform *, transform, NULL) + BR_PROPERTY(double, scaleFactor, 0.75) + BR_PROPERTY(bool, takeLargestScale, true) + BR_PROPERTY(int, windowWidth, 24) + +public: + BuildScalesTransform() : Transform(false, true) {} +private: + + void train(const TemplateList &data) + { + aspectRatio = getAspectRatio(data); + // have to make a copy b/c data is const + TemplateList cp = data; + cp.first().file.set("aspectRatio", aspectRatio); + if (transform->trainable) + transform->train(cp); + } + + void project(const Template &src, Template &dst) const + { + dst = src; + // do not scale images during training + if (src.file.getBool("Train", false)) return; + int rows = src.m().rows; int cols = src.m().cols; int windowHeight = (int) round((float) windowWidth / aspectRatio); @@ -140,34 +210,20 @@ private: startScale = round((float) rows / (float) windowHeight); else startScale = round((float) cols / (float) windowWidth); - for (float scale = startScale; scale >= 1.0; scale -= (1.0 - scaleFactor)) { - Mat scaleImg; + Template scaleImg(src.file, Mat()); + scaleImg.file.set("scale", scale); resize(src, scaleImg, Size(round(cols / scale), round(rows / scale))); - - for (double y = 0; y + windowHeight < scaleImg.rows; y += stepSize) { - for (double x = 0; x + windowWidth < scaleImg.cols; x += stepSize) { -qDebug() << "x=" << x << "\ty=" << y; - Rect window(x, y, windowWidth, windowHeight); - Template windowMat(src.file, Mat(scaleImg, window)); - Template detect; - transform->project(windowMat, detect); - // the result will be in the Label - if (detect.file.get("Label") == "pos") { - dst.file.appendRect(QRectF((float) x * scale, (float) y * scale, (float) windowWidth * scale, (float) windowHeight * scale)); - float confidence = detect.file.get("Dist"); - QList confidences = dst.file.getList("Confidences", QList()); - confidences.append(confidence); - dst.file.setList("Confidences", confidences); - if (takeLargestScale) return; - } - } - } + transform->project(scaleImg, dst); + if (takeLargestScale && !dst.file.rects().empty()) + return; } } + + float aspectRatio; }; -BR_REGISTER(Transform, SlidingWindowTransform) +BR_REGISTER(Transform, BuildScalesTransform) /*! * \ingroup transforms diff --git a/openbr/plugins/svm.cpp b/openbr/plugins/svm.cpp index a30fe9b..776426e 100644 --- a/openbr/plugins/svm.cpp +++ b/openbr/plugins/svm.cpp @@ -157,7 +157,7 @@ private: dst = src; float prediction = svm.predict(src.m().reshape(1, 1), returnDFVal); if (returnDFVal) { - dst.file.set("Dist", prediction); + dst.file.set("conf", prediction); // positive values ==> first class // negative values ==> second class prediction = prediction > 0 ? 0 : 1;