diff --git a/openbr/openbr_plugin.cpp b/openbr/openbr_plugin.cpp index 9322f5d..97eaa3d 100644 --- a/openbr/openbr_plugin.cpp +++ b/openbr/openbr_plugin.cpp @@ -476,13 +476,18 @@ TemplateList TemplateList::fromGallery(const br::File &gallery) // indexes some property, assigns an integer id to each unique value of propName // stores the index values in "Label" of the output template list -TemplateList TemplateList::relabel(const TemplateList &tl, const QString & propName) +TemplateList TemplateList::relabel(const TemplateList &tl, const QString &propName, bool preserveIntegers) { const QList originalLabels = File::get(tl, propName); QHash labelTable; - foreach (const QString & label, originalLabels) - if (!labelTable.contains(label)) - labelTable.insert(label, labelTable.size()); + foreach (const QString &label, originalLabels) + if (!labelTable.contains(label)) { + int value; bool ok; + value = label.toInt(&ok); + // If the label is already an integer value we don't want to change it. + if (ok && preserveIntegers) labelTable.insert(label, value); + else labelTable.insert(label, labelTable.size()); + } TemplateList result = tl; for (int i=0; i BR_EXPORT static TemplateList fromGallery(const File &gallery); /*!< \brief Create a template list from a br::Gallery. */ /*!< \brief Ensure labels are in the range [0,numClasses-1]. */ - BR_EXPORT static TemplateList relabel(const TemplateList & tl, const QString & propName); + BR_EXPORT static TemplateList relabel(const TemplateList &tl, const QString &propName, bool preserveIntegers); QList indexProperty(const QString & propName, QHash * valueMap=NULL,QHash * reverseLookup = NULL) const; QList indexProperty(const QString & propName, QHash & valueMap, QHash & reverseLookup) const; diff --git a/openbr/plugins/eigen3.cpp b/openbr/plugins/eigen3.cpp index 8341b22..28a805f 100644 --- a/openbr/plugins/eigen3.cpp +++ b/openbr/plugins/eigen3.cpp @@ -317,7 +317,7 @@ class LDATransform : public Transform void train(const TemplateList &_trainingSet) { // creates "Label" - TemplateList trainingSet = TemplateList::relabel(_trainingSet, inputVariable); + TemplateList trainingSet = TemplateList::relabel(_trainingSet, inputVariable, isBinary); int instances = trainingSet.size(); // Perform PCA dimensionality reduction diff --git a/openbr/plugins/gui.cpp b/openbr/plugins/gui.cpp index 526a480..d7bdd74 100644 --- a/openbr/plugins/gui.cpp +++ b/openbr/plugins/gui.cpp @@ -447,7 +447,8 @@ public: } emit this->changeTitle(newTitle); - foreach(const cv::Mat & m, t) { + foreach (const cv::Mat &m, t) { + if (!m.data) continue; qImageBuffer = toQImage(m); displayBuffer->convertFromImage(qImageBuffer); diff --git a/openbr/plugins/slidingwindow.cpp b/openbr/plugins/slidingwindow.cpp index 8063f46..0bb5103 100644 --- a/openbr/plugins/slidingwindow.cpp +++ b/openbr/plugins/slidingwindow.cpp @@ -42,15 +42,15 @@ class SlidingWindowTransform : public MetaTransform { Q_OBJECT Q_PROPERTY(br::Transform *transform READ get_transform WRITE set_transform RESET reset_transform STORED false) - Q_PROPERTY(int stepSize READ get_stepSize WRITE set_stepSize RESET reset_stepSize STORED false) - Q_PROPERTY(bool takeFirst READ get_takeFirst WRITE set_takeFirst RESET reset_takeFirst STORED false) Q_PROPERTY(int windowWidth READ get_windowWidth WRITE set_windowWidth RESET reset_windowWidth STORED false) + Q_PROPERTY(bool takeFirst READ get_takeFirst WRITE set_takeFirst RESET reset_takeFirst STORED false) Q_PROPERTY(float threshold READ get_threshold WRITE set_threshold RESET reset_threshold STORED false) + Q_PROPERTY(float stepFraction READ get_stepFraction WRITE set_stepFraction RESET reset_stepFraction STORED false) BR_PROPERTY(br::Transform *, transform, NULL) - BR_PROPERTY(int, stepSize, 1) - BR_PROPERTY(bool, takeFirst, false) BR_PROPERTY(int, windowWidth, 24) + BR_PROPERTY(bool, takeFirst, false) BR_PROPERTY(float, threshold, 0) + BR_PROPERTY(float, stepFraction, 0.25) private: int windowHeight; @@ -66,36 +66,6 @@ private: } } - void project(const Template &src, Template &dst) const - { - dst = src; - // no need to slide a window over ground truth data - if (src.file.getBool("Train", false)) return; - - dst.file.clearRects(); - float scale = src.file.get("scale", 1); - Template windowTemplate(src.file, src); - QList confidences = dst.file.getList("Confidences", QList()); - for (double y = 0; y + windowHeight < src.m().rows; y += stepSize) { - for (double x = 0; x + windowWidth < src.m().cols; x += stepSize) { - Mat windowMat(src, Rect(x, y, windowWidth, windowHeight)); - windowTemplate.replace(0,windowMat); - Template detect; - transform->project(windowTemplate, detect); - float conf = detect.m().at(0); - - // the result will be in the Label - if (conf > threshold) { - dst.file.appendRect(QRectF((float) x * scale, (float) y * scale, (float) windowWidth * scale, (float) windowHeight * scale)); - confidences.append(conf); - if (takeFirst) - return; - } - } - } - dst.file.setList("Confidences", confidences); - } - void store(QDataStream &stream) const { transform->store(stream); @@ -107,11 +77,71 @@ private: transform->load(stream); stream >> windowHeight; } + + void project(const Template &src, Template &dst) const + { + (void)src;(void)dst;qFatal("don't do that"); + } + + void project(const TemplateList &src, TemplateList &dst) const + { + float scale = src.first().file.get("scale", 1); + projectHelp(src, dst, windowWidth, windowHeight, scale); + } + +protected: + void projectHelp(const TemplateList &src, TemplateList &dst, int windowWidth, int windowHeight, float scale = 1) const + { + // no need to slide a window over ground truth data + if (src.first().file.getBool("Train", false)) { + dst = src; + return; + } + + foreach (const Template &t, src) { + for (float y = 0; y + windowHeight < t.m().rows; y += windowHeight*stepFraction) { + for (float x = 0; x + windowWidth < t.m().cols; x += windowWidth*stepFraction) { + Mat windowMat(t.m(), Rect(x, y, windowWidth, windowHeight)); + Template detect; + transform->project(Template(t.file, windowMat), detect); + + // the result will be the only value in the Mat + float conf = detect.m().at(0); + if (conf > threshold) { + detect.file.set("Detection", QRectF(x*scale, y*scale, windowWidth*scale, windowHeight*scale)); + detect.file.set("Confidence", conf); + dst.append(detect); + if (takeFirst) + return; + } + } + } + } + } }; BR_REGISTER(Transform, SlidingWindowTransform) -static TemplateList cropTrainingSamples(const TemplateList &data, const float aspectRatio, const int minSize = 0, const float maxOverlap = 0.5, const int negToPosRatio = 1) +/*! + * \ingroup transforms + * \brief Overloads SlidingWindowTransform for integral images that should be + * sampled at multiple scales. + * \author Josh Klontz \cite jklontz + */ +class IntegralSlidingWindowTransform : public SlidingWindowTransform +{ + Q_OBJECT + + void project(const TemplateList &src, TemplateList &dst) const + { + // TODO: call SlidingWindowTransform::project on multiple scales + SlidingWindowTransform::projectHelp(src, dst, 24, 24); + } +}; + +BR_REGISTER(Transform, IntegralSlidingWindowTransform) + +static TemplateList cropTrainingSamples(const TemplateList &data, const float aspectRatio, const int minSize = 32, const float maxOverlap = 0.5, const int negToPosRatio = 1) { TemplateList result; foreach (const Template &tmpl, data) { @@ -121,8 +151,8 @@ static TemplateList cropTrainingSamples(const TemplateList &data, const float as Rect &posRect = posRects[i]; // Adjust for training samples that have different aspect ratios - const int diff = posRect.width - int(posRect.height * aspectRatio); - posRect.x += diff / 2; + const int diff = int(posRect.height * aspectRatio) - posRect.width; + posRect.x -= diff / 2; posRect.width += diff; // Ignore samples larger than the image @@ -174,7 +204,7 @@ static TemplateList cropTrainingSamples(const TemplateList &data, const float as /*! * \ingroup transforms - * \brief . + * \brief Pass along images at different scales. * \author Austin Blanton \cite imaus10 */ class BuildScalesTransform : public MetaTransform @@ -219,25 +249,38 @@ private: void project(const Template &src, Template &dst) const { - dst = src; + (void)src;(void)dst;qFatal("please don't"); + } + + void project(const TemplateList &src, TemplateList &dst) const + { // do not scale images during training - if (src.file.getBool("Train", false)) return; - - int rows = src.m().rows; - int cols = src.m().cols; - int windowHeight = (int) qRound((float) windowWidth / aspectRatio); - float startScale; - if ((cols / rows) > aspectRatio) - startScale = qRound((float) rows / (float) windowHeight); - else - startScale = qRound((float) cols / (float) windowWidth); - for (float scale = startScale; scale >= minScale; scale -= (1.0 - scaleFactor)) { - Template scaleImg(src.file, Mat()); - scaleImg.file.set("scale", scale); - resize(src, scaleImg, Size(qRound(cols / scale), qRound(rows / scale))); - transform->project(scaleImg, dst); - if (takeLargestScale && !dst.file.rects().empty()) - return; + if (src.first().file.getBool("Train", false)) { + dst = src; + return; + } + + foreach(const Template &t, src) { + int rows = t.m().rows; + int cols = t.m().cols; + int windowHeight = (int) qRound((float) windowWidth / aspectRatio); + float startScale; + if ((cols / rows) > aspectRatio) + startScale = qRound((float) rows / (float) windowHeight); + else + startScale = qRound((float) cols / (float) windowWidth); + for (float scale = startScale; scale >= minScale; scale -= (1.0 - scaleFactor)) { + Template scaleImg(t.file, Mat()); + scaleImg.file.set("scale", scale); + resize(t.m(), scaleImg.m(), Size(qRound(cols / scale), qRound(rows / scale))); + TemplateList results; + TemplateList input; + input.append(scaleImg); + transform->project(input, results); + dst.append(results); + if (takeLargestScale && !dst.empty()) + return; + } } } @@ -270,6 +313,7 @@ class Detector : public Transform { const float aspectRatio = getAspectRatio(data); TemplateList cropped = cropTrainingSamples(data, aspectRatio); + qDebug("Detector using: %d training samples.", cropped.size()); cropped.first().file.set("aspectRatio", aspectRatio); transform->train(cropped); } @@ -327,12 +371,19 @@ private: void project(const Template &src, Template &dst) const { - dst = src; - if (!dst.file.contains("Confidences")) - return; + (void)src;(void)dst;qFatal("nope"); + } - //Compute overlap between rectangles and create discrete Laplacian matrix - QList rects = OpenCVUtils::toRects(src.file.rects()); + void project(const TemplateList &src, TemplateList &dst) const + { + QList rects; + QList confidences; + foreach (const Template &t, src) { + rects.append(OpenCVUtils::toRect(t.file.get("Detection"))); + confidences.append(t.file.get("Confidence")); + } + + // Compute overlap between rectangles and create discrete Laplacian matrix int n = rects.size(); if (n == 0) return; @@ -379,12 +430,12 @@ private: // each input dimension. Each input dimension corresponds to // one of the input rect region. Thus, each eigenvector represents // a set of overlaping regions. - float midX[nRegions]; - float midY[nRegions]; - float avgWidth[nRegions]; - float avgHeight[nRegions]; - float confs[nRegions]; - int cnts[nRegions]; + float * midX = new float[nRegions]; + float * midY = new float[nRegions]; + float * avgWidth = new float[nRegions]; + float *avgHeight = new float[nRegions]; + float *confs = new float[nRegions]; + int *cnts = new int[nRegions]; int mx; int mxIdx; for (int i = 0 ; i < nRegions; i++) { @@ -396,7 +447,6 @@ private: cnts[i] = 0; } - QList confidences = dst.file.getList("Confidences"); for (int i = 0; i < n; i++) { mx = 0.0; mxIdx = -1; @@ -431,8 +481,20 @@ private: } } - dst.file.setRects(consolidatedRects); - dst.file.setList("Confidences", consolidatedConfidences); + for (int i=0; itemplates.size(); } - bool open(TemplateList & input, br::Idiocy::StreamModes _mode) + bool open(const TemplateList & input, br::Idiocy::StreamModes _mode) { // Set up variables specific to us current_template_idx = 0; @@ -1030,8 +1030,10 @@ public: void projectUpdate(const TemplateList & src, TemplateList & dst) { dst = src; + if (src.empty()) + return; - bool res = readStage->dataSource.open(dst,readMode); + bool res = readStage->dataSource.open(src,readMode); if (!res) { qDebug("stream failed to open %s", qPrintable(dst[0].file.name)); return; @@ -1082,6 +1084,8 @@ public: // with anything output via the calls to finalize. //dst = collectionStage->getOutput(); + // dst is set to all output received by the final stage, along + // with anything output via the calls to finalize. foreach(const TemplateList & list, collector->sets) { dst.append(list); } diff --git a/scripts/downloadDatasets.sh b/scripts/downloadDatasets.sh index 95f9a74..22fb149 100755 --- a/scripts/downloadDatasets.sh +++ b/scripts/downloadDatasets.sh @@ -102,3 +102,21 @@ if [ ! -d ../data/MEDS/img ]; then mv data/*/*.jpg ../data/MEDS/img rm -r data NIST_SD32_MEDS-II_face.zip fi + +#LFPW +if [ ! -d ../data/lfpw/trainset ]; then + echo "Downloading LFPW..." + if hash curl 2>/dev/null; then + curl -OL http://ibug.doc.ic.ac.uk/media/uploads/competitions/lfpw.zip + else + wget http://ibug.doc.ic.ac.uk/media/uploads/competitions/lfpw.zip + fi + + unzip lfpw.zip + mv lfpw ../data + cd ../data/lfpw + python ../../scripts/lfpwToSigset.py + cd ../../scripts + rm lfpw.zip +fi + diff --git a/scripts/lfpwToSigset.py b/scripts/lfpwToSigset.py new file mode 100644 index 0000000..39e6fe0 --- /dev/null +++ b/scripts/lfpwToSigset.py @@ -0,0 +1,43 @@ +#!/usr/bin/python + +# This scripts converts the LFPW .pts files into xml sigsets that can be readily +# used within openbr. + +from xml.dom.minidom import Document +import glob +import os + +for lfpwType in ['train','test']: + files = glob.glob('%sset/*.pts' % lfpwType) + files.sort() + cnt = 0 + + xmlDoc = Document() + xmlRoot = xmlDoc.createElement('biometric-signature-set') + for ptsFile in files: + cnt += 1 + ini = open(ptsFile) + lines = ini.readlines() + ini.close() + + pntStrList = [] + n = int(lines[1].split()[1]) + for i in range(3,3+n): + pntStrList.append('(%s)' % (','.join(lines[i].split()))) + pntStr = '[%s]' % ','.join(pntStrList) + + xmlSubj = xmlDoc.createElement('biometric-signature') + xmlSubj.setAttribute('name','subj_%05d' % cnt) + xmlPres = xmlDoc.createElement('presentation') + xmlPres.setAttribute('file-name','%sset/%s.png' % (lfpwType,os.path.splitext(ptsFile)[0])) + xmlPres.setAttribute('Points',pntStr) + xmlSubj.appendChild(xmlPres) + xmlRoot.appendChild(xmlSubj) + + if not os.path.exists('sigset'): + os.mkdir('sigset') + out = open('sigset/%s.xml' % lfpwType,'w') + print >> out, xmlRoot.toprettyxml() + out.close() + + diff --git a/scripts/pedestrianBaselineLBP.sh b/scripts/pedestrianBaselineLBP.sh index e5e70e6..ea6af8f 100755 --- a/scripts/pedestrianBaselineLBP.sh +++ b/scripts/pedestrianBaselineLBP.sh @@ -13,7 +13,7 @@ fi ALG="Open+Cvt(Gray)+Rename(neg,0)+BuildScales(Blur(2)+LBP(1,2)+SlidingWindow(Hist(59)+Cat+LDA(isBinary=true),windowWidth=10,takeLargestScale=false,threshold=2),windowWidth=10,takeLargestScale=false,minScale=4)+ConsolidateDetections+Discard" # Josh's new algorithm (in progress) -# ALG2="Open+Cvt(Gray)+Detector(Gradient+Bin(0,360,9,true)+Merge+Integral+SlidingWindow(Identity))" +# ALG="Open+Cvt(Gray)+Detector(Gradient+Bin(0,360,9,true)+Merge+Integral+IntegralSlidingWindow(RecursiveIntegralSampler(2,2,0,PCA(0.95))+Cat+LDA(0.95,isBinary=true)))" br -useGui 0 \ -algorithm "${ALG}" \