Looks like caotto beat me to it

Scott Klum
2 parents 7c7a4a39 5020073a
Showing 9 changed files with 213 additions and 80 deletions
openbr/openbr_plugin.cpp
openbr/openbr_plugin.h
openbr/plugins/eigen3.cpp
openbr/plugins/gui.cpp
openbr/plugins/slidingwindow.cpp
openbr/plugins/stream.cpp
scripts/downloadDatasets.sh
scripts/lfpwToSigset.py
scripts/pedestrianBaselineLBP.sh
@@ -476,13 +476,18 @@ TemplateList TemplateList::fromGallery(const br::File &amp;gallery)
 // indexes some property, assigns an integer id to each unique value of propName
 // stores the index values in "Label" of the output template list
-TemplateList TemplateList::relabel(const TemplateList &tl, const QString & propName)
+TemplateList TemplateList::relabel(const TemplateList &tl, const QString &propName, bool preserveIntegers)
 {
     const QList<QString> originalLabels = File::get<QString>(tl, propName);
     QHash<QString,int> labelTable;
-    foreach (const QString & label, originalLabels)
-        if (!labelTable.contains(label))
-            labelTable.insert(label, labelTable.size());
+    foreach (const QString &label, originalLabels)
+        if (!labelTable.contains(label)) {
+            int value; bool ok;
+            value = label.toInt(&ok);
+            // If the label is already an integer value we don't want to change it.
+            if (ok && preserveIntegers) labelTable.insert(label, value);
+            else                        labelTable.insert(label, labelTable.size());
+        }
     TemplateList result = tl;
     for (int i=0; i<result.size(); i++)
@@ -438,7 +438,7 @@ struct TemplateList : public QList&lt;Template&gt;
     BR_EXPORT static TemplateList fromGallery(const File &gallery); /*!< \brief Create a template list from a br::Gallery. */
     /*!< \brief Ensure labels are in the range [0,numClasses-1]. */
-    BR_EXPORT static TemplateList relabel(const TemplateList & tl, const QString & propName);
+    BR_EXPORT static TemplateList relabel(const TemplateList &tl, const QString &propName, bool preserveIntegers);
     QList<int> indexProperty(const QString & propName, QHash<QString, int> * valueMap=NULL,QHash<int, QVariant> * reverseLookup = NULL) const;
     QList<int> indexProperty(const QString & propName, QHash<QString, int> & valueMap, QHash<int, QVariant> & reverseLookup) const;
@@ -317,7 +317,7 @@ class LDATransform : public Transform
     void train(const TemplateList &_trainingSet)
     {
         // creates "Label"
-        TemplateList trainingSet = TemplateList::relabel(_trainingSet, inputVariable);
+        TemplateList trainingSet = TemplateList::relabel(_trainingSet, inputVariable, isBinary);
         int instances = trainingSet.size();
         // Perform PCA dimensionality reduction
@@ -447,7 +447,8 @@ public:
             }
             emit this->changeTitle(newTitle);
-            foreach(const cv::Mat & m, t) {
+            foreach (const cv::Mat &m, t) {
+                if (!m.data) continue;
                 qImageBuffer = toQImage(m);
                 displayBuffer->convertFromImage(qImageBuffer);
@@ -42,15 +42,15 @@ class SlidingWindowTransform : public MetaTransform
 {
     Q_OBJECT
     Q_PROPERTY(br::Transform *transform READ get_transform WRITE set_transform RESET reset_transform STORED false)
-    Q_PROPERTY(int stepSize READ get_stepSize WRITE set_stepSize RESET reset_stepSize STORED false)
-    Q_PROPERTY(bool takeFirst READ get_takeFirst WRITE set_takeFirst RESET reset_takeFirst STORED false)
     Q_PROPERTY(int windowWidth READ get_windowWidth WRITE set_windowWidth RESET reset_windowWidth STORED false)
+    Q_PROPERTY(bool takeFirst READ get_takeFirst WRITE set_takeFirst RESET reset_takeFirst STORED false)
     Q_PROPERTY(float threshold READ get_threshold WRITE set_threshold RESET reset_threshold STORED false)
+    Q_PROPERTY(float stepFraction READ get_stepFraction WRITE set_stepFraction RESET reset_stepFraction STORED false)
     BR_PROPERTY(br::Transform *, transform, NULL)
-    BR_PROPERTY(int, stepSize, 1)
-    BR_PROPERTY(bool, takeFirst, false)
     BR_PROPERTY(int, windowWidth, 24)
+    BR_PROPERTY(bool, takeFirst, false)
     BR_PROPERTY(float, threshold, 0)
+    BR_PROPERTY(float, stepFraction, 0.25)
 private:
     int windowHeight;
@@ -66,36 +66,6 @@ private:
         }
     }
-    void project(const Template &src, Template &dst) const
-    {
-        dst = src;
-        // no need to slide a window over ground truth data
-        if (src.file.getBool("Train", false)) return;
-
-        dst.file.clearRects();
-        float scale = src.file.get<float>("scale", 1);
-        Template windowTemplate(src.file, src);
-        QList<float> confidences = dst.file.getList<float>("Confidences", QList<float>());
-        for (double y = 0; y + windowHeight < src.m().rows; y += stepSize) {
-            for (double x = 0; x + windowWidth < src.m().cols; x += stepSize) {
-                Mat windowMat(src, Rect(x, y, windowWidth, windowHeight));
-                windowTemplate.replace(0,windowMat);
-                Template detect;
-                transform->project(windowTemplate, detect);
-                float conf = detect.m().at<float>(0);
-
-                // the result will be in the Label
-                if (conf > threshold) {
-                    dst.file.appendRect(QRectF((float) x * scale, (float) y * scale, (float) windowWidth * scale, (float) windowHeight * scale));
-                    confidences.append(conf);
-                    if (takeFirst)
-                        return;
-                }
-            }
-        }
-        dst.file.setList<float>("Confidences", confidences);
-    }
-
     void store(QDataStream &stream) const
     {
         transform->store(stream);
@@ -107,11 +77,71 @@ private:
         transform->load(stream);
         stream >> windowHeight;
     }
+
+    void project(const Template &src, Template &dst) const
+    {
+        (void)src;(void)dst;qFatal("don't do that");
+    }
+
+    void project(const TemplateList &src, TemplateList &dst) const
+    {
+        float scale = src.first().file.get<float>("scale", 1);
+        projectHelp(src, dst, windowWidth, windowHeight, scale);
+    }
+
+protected:
+    void projectHelp(const TemplateList &src, TemplateList &dst, int windowWidth, int windowHeight, float scale = 1) const
+    {
+        // no need to slide a window over ground truth data
+        if (src.first().file.getBool("Train", false)) {
+            dst = src;
+            return;
+        }
+
+        foreach (const Template &t, src) {
+            for (float y = 0; y + windowHeight < t.m().rows; y += windowHeight*stepFraction) {
+                for (float x = 0; x + windowWidth < t.m().cols; x += windowWidth*stepFraction) {
+                    Mat windowMat(t.m(), Rect(x, y, windowWidth, windowHeight));
+                    Template detect;
+                    transform->project(Template(t.file, windowMat), detect);
+
+                    // the result will be the only value in the Mat
+                    float conf = detect.m().at<float>(0);
+                    if (conf > threshold) {
+                        detect.file.set("Detection", QRectF(x*scale, y*scale, windowWidth*scale, windowHeight*scale));
+                        detect.file.set("Confidence", conf);
+                        dst.append(detect);
+                        if (takeFirst)
+                            return;
+                    }
+                }
+            }
+        }
+    }
 };
 BR_REGISTER(Transform, SlidingWindowTransform)
-static TemplateList cropTrainingSamples(const TemplateList &data, const float aspectRatio, const int minSize = 0, const float maxOverlap = 0.5, const int negToPosRatio = 1)
+/*!
+ * \ingroup transforms
+ * \brief Overloads SlidingWindowTransform for integral images that should be
+ *        sampled at multiple scales.
+ * \author Josh Klontz \cite jklontz
+ */
+class IntegralSlidingWindowTransform : public SlidingWindowTransform
+{
+    Q_OBJECT
+
+    void project(const TemplateList &src, TemplateList &dst) const
+    {
+        // TODO: call SlidingWindowTransform::project on multiple scales
+        SlidingWindowTransform::projectHelp(src, dst, 24, 24);
+    }
+};
+
+BR_REGISTER(Transform, IntegralSlidingWindowTransform)
+
+static TemplateList cropTrainingSamples(const TemplateList &data, const float aspectRatio, const int minSize = 32, const float maxOverlap = 0.5, const int negToPosRatio = 1)
 {
     TemplateList result;
     foreach (const Template &tmpl, data) {
@@ -121,8 +151,8 @@ static TemplateList cropTrainingSamples(const TemplateList &amp;data, const float as
             Rect &posRect = posRects[i];
             // Adjust for training samples that have different aspect ratios
-            const int diff = posRect.width - int(posRect.height * aspectRatio);
-            posRect.x += diff / 2;
+            const int diff = int(posRect.height * aspectRatio) - posRect.width;
+            posRect.x -= diff / 2;
             posRect.width += diff;
             // Ignore samples larger than the image
@@ -174,7 +204,7 @@ static TemplateList cropTrainingSamples(const TemplateList &amp;data, const float as
 /*!
  * \ingroup transforms
- * \brief .
+ * \brief Pass along images at different scales.
  * \author Austin Blanton \cite imaus10
  */
 class BuildScalesTransform : public MetaTransform
@@ -219,25 +249,38 @@ private:
     void project(const Template &src, Template &dst) const
     {
-        dst = src;
+        (void)src;(void)dst;qFatal("please don't");
+    }
+
+    void project(const TemplateList &src, TemplateList &dst) const
+    {
         // do not scale images during training
-        if (src.file.getBool("Train", false)) return;
-
-        int rows = src.m().rows;
-        int cols = src.m().cols;
-        int windowHeight = (int) qRound((float) windowWidth / aspectRatio);
-        float startScale;
-        if ((cols / rows) > aspectRatio)
-            startScale = qRound((float) rows / (float) windowHeight);
-        else
-            startScale = qRound((float) cols / (float) windowWidth);
-        for (float scale = startScale; scale >= minScale; scale -= (1.0 - scaleFactor)) {
-            Template scaleImg(src.file, Mat());
-            scaleImg.file.set("scale", scale);
-            resize(src, scaleImg, Size(qRound(cols / scale), qRound(rows / scale)));
-            transform->project(scaleImg, dst);
-            if (takeLargestScale && !dst.file.rects().empty())
-                return;
+        if (src.first().file.getBool("Train", false)) {
+            dst = src;
+            return;
+        }
+
+        foreach(const Template &t, src) {
+            int rows = t.m().rows;
+            int cols = t.m().cols;
+            int windowHeight = (int) qRound((float) windowWidth / aspectRatio);
+            float startScale;
+            if ((cols / rows) > aspectRatio)
+                startScale = qRound((float) rows / (float) windowHeight);
+            else
+                startScale = qRound((float) cols / (float) windowWidth);
+            for (float scale = startScale; scale >= minScale; scale -= (1.0 - scaleFactor)) {
+                Template scaleImg(t.file, Mat());
+                scaleImg.file.set("scale", scale);
+                resize(t.m(), scaleImg.m(), Size(qRound(cols / scale), qRound(rows / scale)));
+                TemplateList results;
+                TemplateList input;
+                input.append(scaleImg);
+                transform->project(input, results);
+                dst.append(results);
+                if (takeLargestScale && !dst.empty())
+                    return;
+            }
         }
     }
@@ -270,6 +313,7 @@ class Detector : public Transform
     {
         const float aspectRatio = getAspectRatio(data);
         TemplateList cropped = cropTrainingSamples(data, aspectRatio);
+        qDebug("Detector using: %d training samples.", cropped.size());
         cropped.first().file.set("aspectRatio", aspectRatio);
         transform->train(cropped);
     }
@@ -327,12 +371,19 @@ private:
     void project(const Template &src, Template &dst) const
     {
-        dst = src;
-        if (!dst.file.contains("Confidences"))
-            return;
+        (void)src;(void)dst;qFatal("nope");
+    }
-        //Compute overlap between rectangles and create discrete Laplacian matrix
-        QList<Rect> rects = OpenCVUtils::toRects(src.file.rects());
+    void project(const TemplateList &src, TemplateList &dst) const
+    {
+        QList<Rect> rects;
+        QList<float> confidences;
+        foreach (const Template &t, src) {
+            rects.append(OpenCVUtils::toRect(t.file.get<QRectF>("Detection")));
+            confidences.append(t.file.get<float>("Confidence"));
+        }
+
+        // Compute overlap between rectangles and create discrete Laplacian matrix
         int n = rects.size();
         if (n == 0)
             return;
@@ -379,12 +430,12 @@ private:
         // each input dimension. Each input dimension corresponds to
         // one of the input rect region. Thus, each eigenvector represents
         // a set of overlaping regions.
-        float midX[nRegions];
-        float midY[nRegions];
-        float avgWidth[nRegions];
-        float avgHeight[nRegions];
-        float confs[nRegions];
-        int cnts[nRegions];
+        float  * midX = new float[nRegions];
+        float * midY = new float[nRegions];
+        float * avgWidth = new float[nRegions];
+        float *avgHeight = new float[nRegions];
+        float *confs = new float[nRegions];
+        int *cnts = new int[nRegions];
         int mx;
         int mxIdx;
         for (int i = 0 ; i < nRegions; i++) {
@@ -396,7 +447,6 @@ private:
             cnts[i] = 0;
         }
-        QList<float> confidences = dst.file.getList<float>("Confidences");
         for (int i = 0; i < n; i++) {
             mx = 0.0;
             mxIdx = -1;
@@ -431,8 +481,20 @@ private:
             }
         }
-        dst.file.setRects(consolidatedRects);
-        dst.file.setList<float>("Confidences", consolidatedConfidences);
+        for (int i=0; i<consolidatedRects.size(); i++) {
+            Template t(src.first().file);
+            t.file.set("Detection", OpenCVUtils::fromRect(consolidatedRects.at(i)));
+            t.file.set("Confidence", consolidatedConfidences.at(i));
+            dst.append(t);
+        }
+
+        delete [] midX;
+        delete [] midY;
+        delete [] avgWidth;
+        delete [] avgHeight;
+        delete [] confs;
+        delete [] cnts;
+
     }
 };
@@ -363,7 +363,7 @@ public:
         return this->templates.size();
     }
-    bool open(TemplateList & input, br::Idiocy::StreamModes _mode)
+    bool open(const TemplateList & input, br::Idiocy::StreamModes _mode)
     {
         // Set up variables specific to us
         current_template_idx = 0;
@@ -1030,8 +1030,10 @@ public:
     void projectUpdate(const TemplateList & src, TemplateList & dst)
     {
         dst = src;
+        if (src.empty())
+            return;
-        bool res = readStage->dataSource.open(dst,readMode);
+        bool res = readStage->dataSource.open(src,readMode);
         if (!res) {
             qDebug("stream failed to open %s", qPrintable(dst[0].file.name));
             return;
@@ -1082,6 +1084,8 @@ public:
         // with anything output via the calls to finalize.
         //dst = collectionStage->getOutput();
+        // dst is set to all output received by the final stage, along
+        // with anything output via the calls to finalize.
         foreach(const TemplateList & list, collector->sets) {
             dst.append(list);
         }
@@ -102,3 +102,21 @@ if [ ! -d ../data/MEDS/img ]; then
   mv data/*/*.jpg ../data/MEDS/img
   rm -r data NIST_SD32_MEDS-II_face.zip
 fi
+
+#LFPW
+if [ ! -d ../data/lfpw/trainset ]; then
+  echo "Downloading LFPW..."
+  if hash curl 2>/dev/null; then
+    curl -OL http://ibug.doc.ic.ac.uk/media/uploads/competitions/lfpw.zip
+  else
+    wget  http://ibug.doc.ic.ac.uk/media/uploads/competitions/lfpw.zip
+  fi
+  
+  unzip lfpw.zip
+  mv lfpw ../data
+  cd ../data/lfpw
+  python ../../scripts/lfpwToSigset.py
+  cd ../../scripts
+  rm lfpw.zip
+fi
+
+#!/usr/bin/python
+
+# This scripts converts the LFPW .pts files into xml sigsets that can be readily 
+#   used within openbr. 
+
+from xml.dom.minidom import Document
+import glob
+import os
+
+for lfpwType in ['train','test']:
+    files = glob.glob('%sset/*.pts' % lfpwType)
+    files.sort()
+    cnt = 0
+    
+    xmlDoc = Document()
+    xmlRoot = xmlDoc.createElement('biometric-signature-set')
+    for ptsFile in files:
+        cnt += 1
+        ini = open(ptsFile)
+        lines = ini.readlines()
+        ini.close()
+
+        pntStrList = []
+        n = int(lines[1].split()[1])
+        for i in range(3,3+n):
+            pntStrList.append('(%s)' % (','.join(lines[i].split())))
+        pntStr = '[%s]' % ','.join(pntStrList)
+
+        xmlSubj = xmlDoc.createElement('biometric-signature')
+        xmlSubj.setAttribute('name','subj_%05d' % cnt) 
+        xmlPres = xmlDoc.createElement('presentation')
+        xmlPres.setAttribute('file-name','%sset/%s.png' % (lfpwType,os.path.splitext(ptsFile)[0]))
+        xmlPres.setAttribute('Points',pntStr)
+        xmlSubj.appendChild(xmlPres)
+        xmlRoot.appendChild(xmlSubj)
+    
+    if not os.path.exists('sigset'):
+        os.mkdir('sigset')
+    out = open('sigset/%s.xml' % lfpwType,'w')
+    print >> out, xmlRoot.toprettyxml()
+    out.close()
+
+
@@ -13,7 +13,7 @@ fi
 ALG="Open+Cvt(Gray)+Rename(neg,0)+BuildScales(Blur(2)+LBP(1,2)+SlidingWindow(Hist(59)+Cat+LDA(isBinary=true),windowWidth=10,takeLargestScale=false,threshold=2),windowWidth=10,takeLargestScale=false,minScale=4)+ConsolidateDetections+Discard"
 # Josh's new algorithm (in progress)
-# ALG2="Open+Cvt(Gray)+Detector(Gradient+Bin(0,360,9,true)+Merge+Integral+SlidingWindow(Identity))"
+# ALG="Open+Cvt(Gray)+Detector(Gradient+Bin(0,360,9,true)+Merge+Integral+IntegralSlidingWindow(RecursiveIntegralSampler(2,2,0,PCA(0.95))+Cat+LDA(0.95,isBinary=true)))"
 br -useGui 0 \
    -algorithm "${ALG}" \