Update to Caffe framework

Jordan Cheney
1 parent 5c945c17
Showing 4 changed files with 76 additions and 55 deletions
openbr/plugins/classification/caffe.cpp
openbr/plugins/cmake/lmdbGallery.cmake
openbr/plugins/imgproc/pad.cpp
openbr/plugins/imgproc/roi.cpp
 #include <openbr/plugins/openbr_internal.h>
+#include <openbr/core/opencvutils.h>
+#include <openbr/core/qtutils.h>
+#include <opencv2/imgproc/imgproc.hpp>
 #include <caffe/caffe.hpp>
 using caffe::Caffe;
-using caffe::Solver;
 using caffe::Net;
+using caffe::MemoryDataLayer;
 using caffe::Blob;
 using caffe::shared_ptr;
-using caffe::vector;
+
+using namespace cv;
 namespace br
 {
 /*!
- * \brief A transform that wraps the Caffe Deep learning library
+ * \brief A transform that wraps the Caffe deep learning library. This transform expects the input to a given Caffe model to be a MemoryDataLayer.
+ * The output of the Caffe network is treated as a feature vector and is stored in dst. Batch processing is possible. For a given batch size set in
+ * the memory data layer, src is expected to have an equal number of mats. Dst will always have the same size (number of mats) as src and the ordering
+ * will be preserved, so dst[1] is the output of src[1] after it passes through the
  * \author Jordan Cheney \cite jcheney
- * \br_property QString modelFile path to prototxt model file
- * \br_property QString solverFile path to prototxt solver file
- * \br_property QString weightsFile path to caffemodel file
+ * \br_property QString model path to prototxt model file
+ * \br_property QString weights path to caffemodel file
  * \br_property int gpuDevice ID of GPU to use. gpuDevice < 0 runs on the CPU only.
  * \br_link Caffe Integration Tutorial ../tutorials.md#caffe
  * \br_link Caffe website http://caffe.berkeleyvision.org
  */
-class CaffeTransform : public Transform
+class CaffeFVTransform : public UntrainableTransform
 {
     Q_OBJECT
-    Q_PROPERTY(QString modelFile READ get_modelFile WRITE set_modelFile RESET reset_modelFile STORED false)
-    Q_PROPERTY(QString solverFile READ get_solverFile WRITE set_solverFile RESET reset_solverFile STORED false)
-    Q_PROPERTY(QString weightsFile READ get_weightsFile WRITE set_weightsFile RESET reset_weightsFile STORED false)
+    Q_PROPERTY(QString model READ get_model WRITE set_model RESET reset_model STORED false)
+    Q_PROPERTY(QString weights READ get_weights WRITE set_weights RESET reset_weights STORED false)
     Q_PROPERTY(int gpuDevice READ get_gpuDevice WRITE set_gpuDevice RESET reset_gpuDevice STORED false)
-    BR_PROPERTY(QString, modelFile, "")
-    BR_PROPERTY(QString, solverFile, "")
-    BR_PROPERTY(QString, weightsFile, "")
+    BR_PROPERTY(QString, model, "")
+    BR_PROPERTY(QString, weights, "")
     BR_PROPERTY(int, gpuDevice, -1)
+    QSharedPointer<Net<float> > net;
+
     void init()
     {
         if (gpuDevice >= 0) {
@@ -43,47 +49,31 @@ class CaffeTransform : public Transform
         } else {
             Caffe::set_mode(Caffe::CPU);
         }
+
+        net.reset(new Net<float>(model.toStdString(), caffe::TEST));
+        net->CopyTrainedLayersFrom(weights.toStdString());
     }
-    void train(const TemplateList &data)
+    void project(const Template &src, Template &dst) const
     {
-        (void) data;
+        MemoryDataLayer<float> *data_layer = static_cast<MemoryDataLayer<float> *>(net->layers()[0].get());
-        caffe::SolverParameter solver_param;
-        caffe::ReadProtoFromTextFileOrDie(solverFile.toStdString(), &solver_param);
+        if (src.size() != data_layer->batch_size())
+            qFatal("src should have %d (batch size) mats. It has %d mats.", data_layer->batch_size(), src.size());
-        shared_ptr<Solver<float> > solver(caffe::GetSolver<float>(solver_param));
-        solver->Solve();
-    }
+        dst.file = src.file;
-    void project(const Template &src, Template &dst) const
-    {
-        (void)src; (void)dst;
-        Net<float> net(modelFile.toStdString(), caffe::TEST);
-        net.CopyTrainedLayersFrom(weightsFile.toStdString());
-
-        vector<Blob<float> *> bottom_vec; // perhaps src data should go here?
-        vector<int> test_score_output_id;
-        vector<float> test_score;
-
-        float loss;
-        const vector<Blob<float> *> &result = net.Forward(bottom_vec, &loss);
-
-        int idx = 0;
-        for (int i = 0; i < (int)result.size(); i++) {
-            const float *result_data = result[i]->cpu_data();
-            for (int j = 0; j < result[i]->count(); j++, idx++) {
-                test_score.push_back(result_data[j]);
-                test_score_output_id.push_back(i);
-
-                if (Globals->verbose)
-                    qDebug("%s = %f", net.blob_names()[net.output_blob_indices()[i]].c_str(), result_data[j]);
-            }
-        }
+        data_layer->AddMatVector(src.toVector().toStdVector(), std::vector<int>(src.size(), 0));
+
+        Blob<float> *output = net->ForwardPrefilled()[1]; // index 0 is the labels from the data layer (in this case the 0 array we passed in above).
+                                                          // index 1 is the ouput of the final layer, which is what we want
+        int dim_features = output->count() / data_layer->batch_size();
+        for (int n = 0; n < data_layer->batch_size(); n++)
+            dst += Mat(1, dim_features, CV_32FC1, output->mutable_cpu_data() + output->offset(n));
     }
 };
-BR_REGISTER(Transform, CaffeTransform)
+BR_REGISTER(Transform, CaffeFVTransform)
 } // namespace br
-set(BR_WITH_CAFFE OFF CACHE BOOL "Build with Caffe")
-
-if (${BR_WITH_CAFFE})
-   find_package(Caffe)
-   set(BR_THIRDPARTY_LIBS ${BR_THIRDPARTY_LIBS} ${Caffe_LIBRARIES})
-   include_directories(${Caffe_INCLUDE_DIRS})
-else()
-   set(BR_EXCLUDED_PLUGINS ${BR_EXCLUDED_PLUGINS} plugins/gallery/lmdbGallery.cpp)
-endif()
-
+#include <openbr/plugins/openbr_internal.h>
+
+using namespace cv;
+
+namespace br
+{
+
+class PadTransform : public UntrainableTransform
+{
+    Q_OBJECT
+
+    Q_PROPERTY(int padSize READ get_padSize WRITE set_padSize RESET reset_padSize STORED false)
+    Q_PROPERTY(int padValue READ get_padValue WRITE set_padValue RESET reset_padValue STORED false)
+    BR_PROPERTY(int, padSize, 0)
+    BR_PROPERTY(int, padValue, 0)
+
+    void project(const Template &src, Template &dst) const
+    {
+        dst.file = src.file;
+
+        foreach (const Mat &m, src) {
+            Mat padded = padValue * Mat::ones(m.rows + 2*padSize, m.cols + 2*padSize, m.type());
+            padded(Rect(padSize, padSize, padded.cols - padSize, padded.rows - padSize)) = m;
+            dst += padded;
+        }
+    }
+};
+
+BR_REGISTER(Transform, PadTransform)
+
+} // namespace br
+
+#include "imgproc/pad.moc"
@@ -26,12 +26,16 @@ namespace br
  * \ingroup transforms
  * \brief Crops the rectangular regions of interest.
  * \author Josh Klontz \cite jklontz
+ * \br_property QString propName Optional property name for a rectangle in metadata. If no propName is given the transform will use rects stored in the file.rects field or build a rectangle using "X", "Y", "Width", and "Height" fields if they exist.
+ * \br_property bool copyOnCrop If true make a clone of each crop before appending the crop to dst. This guarantees that the crops will be continuous in memory, which is an occasionally useful property. Default is false.
  */
 class ROITransform : public UntrainableTransform
 {
     Q_OBJECT
     Q_PROPERTY(QString propName READ get_propName WRITE set_propName RESET reset_propName STORED false)
+    Q_PROPERTY(bool copyOnCrop READ get_copyOnCrop WRITE set_copyOnCrop RESET reset_copyOnCrop STORED false)
     BR_PROPERTY(QString, propName, "")
+    BR_PROPERTY(bool, copyOnCrop, false)
     void project(const Template &src, Template &dst) const
     {
@@ -52,6 +56,10 @@ class ROITransform : public UntrainableTransform
                 qWarning("No rects present in file.");
         }
         dst.file.clearRects();
+
+        if (copyOnCrop)
+            for (int i = 0; i < dst.size(); i++)
+                dst.replace(i, dst[i].clone());
     }
 };