Merge pull request #262 from biometrics/neural_networks

Neural networks

Merge pull request #262 from biometrics/neural_networks
Neural networks
caotto
2 parents 4fe4b46a 4e6f3c59
Showing 6 changed files with 295 additions and 18 deletions
openbr/core/eval.cpp
openbr/core/opencvutils.cpp
openbr/core/opencvutils.h
openbr/plugins/landmarks.cpp
openbr/plugins/misc.cpp
openbr/plugins/nn.cpp
@@ -1044,12 +1044,14 @@ void EvalRegression(const QString &amp;predictedGallery, const QString &amp;truthGallery
         if (predicted[i].file.name != truth[i].file.name)
             qFatal("Input order mismatch.");
-        float difference = predicted[i].file.get<float>(predictedProperty) - truth[i].file.get<float>(truthProperty);
+        if (predicted[i].file.contains(predictedProperty) && truth[i].file.contains(truthProperty)) {
+            float difference = predicted[i].file.get<float>(predictedProperty) - truth[i].file.get<float>(truthProperty);
-        rmsError += pow(difference, 2.f);
-        maeError += fabsf(difference);
-        truthValues.append(QString::number(truth[i].file.get<float>(truthProperty)));
-        predictedValues.append(QString::number(predicted[i].file.get<float>(predictedProperty)));
+            rmsError += pow(difference, 2.f);
+            maeError += fabsf(difference);
+            truthValues.append(QString::number(truth[i].file.get<float>(truthProperty)));
+            predictedValues.append(QString::number(predicted[i].file.get<float>(predictedProperty)));
+        }
     }
     QStringList rSource;
@@ -121,6 +121,17 @@ Mat OpenCVUtils::toMat(const QList&lt;float&gt; &amp;src, int rows)
     return dst;
 }
+Mat OpenCVUtils::pointsToMatrix(const QList<QPointF> &qPoints)
+{
+    QList<float> points;
+    foreach(const QPointF &point, qPoints) {
+        points.append(point.x());
+        points.append(point.y());
+    }
+
+    return toMat(points);
+}
+
 Mat OpenCVUtils::toMat(const QList<QList<float> > &srcs, int rows)
 {
     QList<float> flat;
@@ -84,6 +84,7 @@ namespace OpenCVUtils
     QPointF fromPoint(const cv::Point2f &cvPoint);
     QList<cv::Point2f> toPoints(const QList<QPointF> &qPoints);
     QList<QPointF> fromPoints(const QList<cv::Point2f> &cvPoints);
+    cv::Mat pointsToMatrix(const QList<QPointF> &qPoints);
     cv::Rect toRect(const QRectF &qRect);
     QRectF fromRect(const cv::Rect &cvRect);
     QList<cv::Rect> toRects(const QList<QRectF> &qRects);
@@ -369,12 +369,14 @@ BR_REGISTER(Transform, ReadLandmarksTransform)
 /*!
  * \ingroup transforms
- * \brief Name a point
+ * \brief Name a point/rect
  * \author Scott Klum \cite sklum
  */
-class NamePointsTransform : public UntrainableMetadataTransform
+class NameLandmarksTransform : public UntrainableMetadataTransform
 {
     Q_OBJECT
+    Q_PROPERTY(bool point READ get_point WRITE set_point RESET reset_point STORED false)
+    BR_PROPERTY(bool, point, true)
     Q_PROPERTY(QList<int> indices READ get_indices WRITE set_indices RESET reset_indices STORED false)
     Q_PROPERTY(QStringList names READ get_names WRITE set_names RESET reset_names STORED false)
     BR_PROPERTY(QList<int>, indices, QList<int>())
@@ -382,27 +384,36 @@ class NamePointsTransform : public UntrainableMetadataTransform
     void projectMetadata(const File &src, File &dst) const
     {
-        if (indices.size() != names.size()) qFatal("Point/name size mismatch");
+        if (indices.size() != names.size()) qFatal("Index/name size mismatch");
         dst = src;
-        QList<QPointF> points = src.points();
+        if (point) {
+            QList<QPointF> points = src.points();
-        for (int i=0; i<indices.size(); i++) {
-            if (indices[i] < points.size()) dst.set(names[i], points[indices[i]]);
-            else qFatal("Index out of range.");
+            for (int i=0; i<indices.size(); i++) {
+                if (indices[i] < points.size()) dst.set(names[i], points[indices[i]]);
+                else qFatal("Index out of range.");
+            }
+        } else {
+            QList<QRectF> rects = src.rects();
+
+            for (int i=0; i<indices.size(); i++) {
+                if (indices[i] < rects.size()) dst.set(names[i], rects[indices[i]]);
+                else qFatal("Index out of range.");
+            }
         }
     }
 };
-BR_REGISTER(Transform, NamePointsTransform)
+BR_REGISTER(Transform, NameLandmarksTransform)
 /*!
  * \ingroup transforms
- * \brief Remove a name from a point
+ * \brief Remove a name from a point/rect
  * \author Scott Klum \cite sklum
  */
-class AnonymizePointsTransform : public UntrainableMetadataTransform
+class AnonymizeLandmarksTransform : public UntrainableMetadataTransform
 {
     Q_OBJECT
     Q_PROPERTY(QStringList names READ get_names WRITE set_names RESET reset_names STORED false)
@@ -412,12 +423,112 @@ class AnonymizePointsTransform : public UntrainableMetadataTransform
     {
         dst = src;
-        foreach (const QString &name, names)
-            if (src.contains(name)) dst.appendPoint(src.get<QPointF>(name));
+        foreach (const QString &name, names) {
+            if (src.contains(name)) {
+                QVariant variant = src.value(name);
+                if (variant.canConvert(QMetaType::QPointF)) {
+                    dst.appendPoint(variant.toPointF());
+                } else if (variant.canConvert(QMetaType::QRectF)) {
+                    dst.appendRect(variant.toRectF());
+                } else {
+                    qFatal("Cannot convert landmark to point or rect.");
+                }
+            }
+        }
+    }
+};
+
+BR_REGISTER(Transform, AnonymizeLandmarksTransform)
+
+/*!
+ * \ingroup transforms
+ * \brief Converts either the file::points() list or a QList<QPointF> metadata item to be the template's matrix
+ * \author Scott Klum \cite sklum
+ */
+class PointsToMatrixTransform : public UntrainableTransform
+{
+    Q_OBJECT
+
+    Q_PROPERTY(QString inputVariable READ get_inputVariable WRITE set_inputVariable RESET reset_inputVariable STORED false)
+    BR_PROPERTY(QString, inputVariable, QString())
+
+    void project(const Template &src, Template &dst) const
+    {
+        dst = src;
+
+        if (inputVariable.isEmpty()) {
+            dst.m() = OpenCVUtils::pointsToMatrix(dst.file.points());
+        } else {
+            if (src.file.contains(inputVariable))
+                dst.m() = OpenCVUtils::pointsToMatrix(dst.file.get<QList<QPointF> >(inputVariable));
+        }
+    }
+};
+
+BR_REGISTER(Transform, PointsToMatrixTransform)
+
+/*!
+ * \ingroup transforms
+ * \brief Normalize points to be relative to a single point
+ * \author Scott Klum \cite sklum
+ */
+class NormalizePointsTransform : public UntrainableTransform
+{
+    Q_OBJECT
+
+    Q_PROPERTY(int index READ get_index WRITE set_index RESET reset_index STORED false)
+    BR_PROPERTY(int, index, 0)
+
+    void project(const Template &src, Template &dst) const
+    {
+        dst = src;
+
+        QList<QPointF> points = dst.file.points();
+        QPointF normPoint = points.at(index);
+
+        QList<QPointF> normalizedPoints;
+
+        for (int i=0; i<points.size(); i++)
+            if (i!=index)
+                normalizedPoints.append(normPoint-points[i]);
+
+        dst.file.setPoints(normalizedPoints);
+    }
+};
+
+BR_REGISTER(Transform, NormalizePointsTransform)
+
+/*!
+ * \ingroup transforms
+ * \brief Normalize points to be relative to a single point
+ * \author Scott Klum \cite sklum
+ */
+class PointDisplacementTransform : public UntrainableTransform
+{
+    Q_OBJECT
+
+    void project(const Template &src, Template &dst) const
+    {
+        dst = src;
+
+        QList<QPointF> points = dst.file.points();
+        QList<QPointF> normalizedPoints;
+
+        for (int i=0; i<points.size(); i++)
+            for (int j=0; j<points.size(); j++)
+                // There is redundant information here
+                if (j!=i) {
+                    QPointF normalizedPoint = points[i]-points[j];
+                    normalizedPoint.setX(pow(normalizedPoint.x(),2));
+                    normalizedPoint.setY(pow(normalizedPoint.y(),2));
+                    normalizedPoints.append(normalizedPoint);
+                }
+
+        dst.file.setPoints(normalizedPoints);
     }
 };
-BR_REGISTER(Transform, AnonymizePointsTransform)
+BR_REGISTER(Transform, PointDisplacementTransform)
 } // namespace br
@@ -937,6 +937,18 @@ class FileExclusionTransform : public UntrainableMetaTransform
 BR_REGISTER(Transform, FileExclusionTransform)
+class TransposeTransform : public UntrainableTransform
+{
+    Q_OBJECT
+
+    void project(const Template &src, Template &dst) const
+    {
+        dst.m() = src.m().t();
+    }
+};
+
+BR_REGISTER(Transform, TransposeTransform)
+
 }
 #include "misc.moc"
+#include <opencv2/ml/ml.hpp>
+
+#include "openbr_internal.h"
+#include "openbr/core/qtutils.h"
+#include "openbr/core/opencvutils.h"
+#include "openbr/core/eigenutils.h"
+#include <QString>
+#include <QTemporaryFile>
+
+using namespace std;
+using namespace cv;
+
+namespace br
+{
+
+static void storeMLP(const CvANN_MLP &mlp, QDataStream &stream)
+{
+    // Create local file
+    QTemporaryFile tempFile;
+    tempFile.open();
+    tempFile.close();
+
+    // Save MLP to local file
+    mlp.save(qPrintable(tempFile.fileName()));
+
+    // Copy local file contents to stream
+    tempFile.open();
+    QByteArray data = tempFile.readAll();
+    tempFile.close();
+    stream << data;
+}
+
+static void loadMLP(CvANN_MLP &mlp, QDataStream &stream)
+{
+    // Copy local file contents from stream
+    QByteArray data;
+    stream >> data;
+
+    // Create local file
+    QTemporaryFile tempFile(QDir::tempPath()+"/MLP");
+    tempFile.open();
+    tempFile.write(data);
+    tempFile.close();
+
+    // Load MLP from local file
+    mlp.load(qPrintable(tempFile.fileName()));
+}
+
+/*!
+ * \ingroup transforms
+ * \brief Wraps OpenCV's multi-layer perceptron framework
+ * \author Scott Klum \cite sklum
+ * \brief http://docs.opencv.org/modules/ml/doc/neural_networks.html
+ */
+class MLPTransform : public MetaTransform
+{
+    Q_OBJECT
+
+    Q_ENUMS(Kernel)
+    Q_PROPERTY(Kernel kernel READ get_kernel WRITE set_kernel RESET reset_kernel STORED false)
+    Q_PROPERTY(float alpha READ get_alpha WRITE set_alpha RESET reset_alpha STORED false)
+    Q_PROPERTY(float beta READ get_beta WRITE set_beta RESET reset_beta STORED false)
+    Q_PROPERTY(QStringList inputVariables READ get_inputVariables WRITE set_inputVariables RESET reset_inputVariables STORED false)
+    Q_PROPERTY(QStringList outputVariables READ get_outputVariables WRITE set_outputVariables RESET reset_outputVariables STORED false)
+    Q_PROPERTY(QList<int> neuronsPerLayer READ get_neuronsPerLayer WRITE set_neuronsPerLayer RESET reset_neuronsPerLayer STORED false)
+
+public:
+
+    enum Kernel { Identity = CvANN_MLP::IDENTITY,
+                  Sigmoid = CvANN_MLP::SIGMOID_SYM,
+                  Gaussian = CvANN_MLP::GAUSSIAN};
+
+private:
+    BR_PROPERTY(Kernel, kernel, Sigmoid)
+    BR_PROPERTY(float, alpha, 1)
+    BR_PROPERTY(float, beta, 1)
+    BR_PROPERTY(QStringList, inputVariables, QStringList())
+    BR_PROPERTY(QStringList, outputVariables, QStringList())
+    BR_PROPERTY(QList<int>, neuronsPerLayer, QList<int>() << 1 << 1)
+
+    CvANN_MLP mlp;
+
+    void init()
+    {
+        if (kernel == Gaussian)
+            qWarning("The OpenCV documentation warns that the Gaussian kernel, \"is not completely supported at the moment\"");
+
+        Mat layers = Mat(neuronsPerLayer.size(), 1, CV_32SC1);
+        for (int i=0; i<neuronsPerLayer.size(); i++)
+            layers.row(i) = Scalar(neuronsPerLayer.at(i));
+
+        mlp.create(layers,kernel, alpha, beta);
+    }
+
+    void train(const TemplateList &data)
+    {
+        Mat _data = OpenCVUtils::toMat(data.data());
+
+        // Assuming data has n templates
+        // _data needs to be n x size of input layer
+        // Labels needs to be a n x outputs matrix
+        // For the time being we're going to assume a single output
+        Mat labels = Mat::zeros(data.size(),inputVariables.size(),CV_32F);
+        for (int i=0; i<inputVariables.size(); i++)
+            labels.col(i) += OpenCVUtils::toMat(File::get<float>(data, inputVariables.at(i)));
+
+        mlp.train(_data,labels,Mat());
+
+        if (Globals->verbose)
+            for (int i=0; i<neuronsPerLayer.size(); i++) qDebug() << *mlp.get_weights(i);
+    }
+
+    void project(const Template &src, Template &dst) const
+    {
+        dst = src;
+
+        // See above for response dimensionality
+        Mat response(outputVariables.size(), 1, CV_32FC1);
+        mlp.predict(src.m().reshape(1,1),response);
+
+        // Apparently mlp.predict reshapes the response matrix?
+        for (int i=0; i<outputVariables.size(); i++) dst.file.set(outputVariables.at(i),response.at<float>(0,i));
+    }
+
+    void load(QDataStream &stream)
+    {
+        loadMLP(mlp,stream);
+    }
+
+    void store(QDataStream &stream) const
+    {
+        storeMLP(mlp,stream);
+    }
+};
+
+BR_REGISTER(Transform, MLPTransform)
+
+} // namespace br
+
+#include "nn.moc"