Peter M. Groen / openbr

Commit df6b3ee439e2e2836ca4f15a1cbd7dcfd52eba95

Authored by JordanCheney
2 parents 6ad9e3fa 77224652

Merge pull request #407 from biometrics/caffe_classifier 

Caffe classifier
Inline Side-by-side
Showing 1 changed file with 85 additions and 18 deletions
openbr/plugins/classification/caffe.cpp
  View file @df6b3ee
1 1 #include <openbr/plugins/openbr_internal.h>
2 2 #include <openbr/core/opencvutils.h>
3   -#include <openbr/core/qtutils.h>
4 3  
5   -#include <opencv2/imgproc/imgproc.hpp>
6 4 #include <caffe/caffe.hpp>
7 5  
8 6 using caffe::Caffe;
... ... @@ -52,21 +50,21 @@ private:
52 50 };
53 51  
54 52 /*!
55   - * \brief A transform that wraps the Caffe deep learning library. This transform expects the input to a given Caffe model to be a MemoryDataLayer.
56   - * The output of the Caffe network is treated as a feature vector and is stored in dst. Batch processing is possible. For a given batch size set in
57   - * the memory data layer, src is expected to have an equal number of mats. Dst will always have the same size (number of mats) as src and the ordering
58   - * will be preserved, so dst[1] is the output of src[1] after it passes through the neural net.
59   - * \author Jordan Cheney \cite jcheney
  53 + * \brief The base transform for wrapping the Caffe deep learning library. This transform expects the input to a given Caffe model to be a MemoryDataLayer.
  54 + * The output of the forward pass of the Caffe network is stored in dst as a list of matrices, the size of which is equal to the batch_size of the network.
  55 + * Children of this transform should process dst to acheieve specifc use cases.
  56 + * \author Jordan Cheney \cite JordanCheney
60 57 * \br_property QString model path to prototxt model file
61 58 * \br_property QString weights path to caffemodel file
62 59 * \br_property int gpuDevice ID of GPU to use. gpuDevice < 0 runs on the CPU only.
63 60 * \br_link Caffe Integration Tutorial ../tutorials.md#caffe
64 61 * \br_link Caffe website http://caffe.berkeleyvision.org
65 62 */
66   -class CaffeFVTransform : public UntrainableTransform
  63 +class CaffeBaseTransform : public UntrainableMetaTransform
67 64 {
68 65 Q_OBJECT
69 66  
  67 +public:
70 68 Q_PROPERTY(QString model READ get_model WRITE set_model RESET reset_model STORED false)
71 69 Q_PROPERTY(QString weights READ get_weights WRITE set_weights RESET reset_weights STORED false)
72 70 Q_PROPERTY(int gpuDevice READ get_gpuDevice WRITE set_gpuDevice RESET reset_gpuDevice STORED false)
... ... @@ -76,6 +74,7 @@ class CaffeFVTransform : public UntrainableTransform
76 74  
77 75 Resource<CaffeNet> caffeResource;
78 76  
  77 +protected:
79 78 void init()
80 79 {
81 80 caffeResource.setResourceMaker(new CaffeResourceMaker(model, weights, gpuDevice));
... ... @@ -90,27 +89,95 @@ class CaffeFVTransform : public UntrainableTransform
90 89 {
91 90 CaffeNet *net = caffeResource.acquire();
92 91  
93   - MemoryDataLayer<float> *data_layer = static_cast<MemoryDataLayer<float> *>(net->layers()[0].get());
  92 + if (net->layers()[0]->layer_param().type() != "MemoryData")
  93 + qFatal("OpenBR requires the first layer in the network to be a MemoryDataLayer");
94 94  
95   - if (src.size() != data_layer->batch_size())
96   - qFatal("src should have %d (batch size) mats. It has %d mats.", data_layer->batch_size(), src.size());
  95 + MemoryDataLayer<float> *dataLayer = static_cast<MemoryDataLayer<float> *>(net->layers()[0].get());
97 96  
98   - dst.file = src.file;
  97 + if (src.size() != dataLayer->batch_size())
  98 + qFatal("src should have %d (batch size) mats. It has %d mats.", dataLayer->batch_size(), src.size());
99 99  
100   - data_layer->AddMatVector(src.toVector().toStdVector(), std::vector<int>(src.size(), 0));
  100 + dataLayer->AddMatVector(src.toVector().toStdVector(), std::vector<int>(src.size(), 0));
101 101  
102   - Blob<float> *output = net->ForwardPrefilled()[1]; // index 0 is the labels from the data layer (in this case the 0 array we passed in above).
103   - // index 1 is the ouput of the final layer, which is what we want
104   - int dim_features = output->count() / data_layer->batch_size();
105   - for (int n = 0; n < data_layer->batch_size(); n++)
106   - dst += Mat(1, dim_features, CV_32FC1, output->mutable_cpu_data() + output->offset(n));
  102 + net->ForwardPrefilled();
  103 + Blob<float> *output = net->blobs().back().get();
  104 +
  105 + int dimFeatures = output->count() / dataLayer->batch_size();
  106 + for (int n = 0; n < dataLayer->batch_size(); n++)
  107 + dst += Mat(1, dimFeatures, CV_32FC1, output->mutable_cpu_data() + output->offset(n));
107 108  
108 109 caffeResource.release(net);
109 110 }
110 111 };
111 112  
  113 +/*!
  114 + * \brief This transform treats the output of the network as a feature vector and appends it unchanged to dst. Dst will have
  115 + * length equal to the batch size of the network.
  116 + * \author Jordan Cheney \cite JordanCheney
  117 + * \br_property QString model path to prototxt model file
  118 + * \br_property QString weights path to caffemodel file
  119 + * \br_property int gpuDevice ID of GPU to use. gpuDevice < 0 runs on the CPU only.
  120 + */
  121 +class CaffeFVTransform : public CaffeBaseTransform
  122 +{
  123 + Q_OBJECT
  124 +
  125 + void project(const Template &src, Template &dst) const
  126 + {
  127 + Template caffeOutput;
  128 + CaffeBaseTransform::project(src, caffeOutput);
  129 +
  130 + dst.file = src.file;
  131 + dst.append(caffeOutput);
  132 + }
  133 +};
  134 +
112 135 BR_REGISTER(Transform, CaffeFVTransform)
113 136  
  137 +/*!
  138 + * \brief This transform treats the output of the network as a score distribution for an arbitrary number of classes.
  139 + * The maximum score and location for each input image is determined and stored in the template metadata. The template
  140 + * matrix is not changed. If the network batch size is > 1, the results are stored as lists in the dst template's metadata
  141 + * using the keys "Labels" and "Confidences" respectively. The length of these lists is equivalent to the provided batch size.
  142 + * If batch size == 1, the results are stored as a float and int using the keys "Label", and "Confidence" respectively.
  143 + * \author Jordan Cheney \cite jcheney
  144 + * \br_property QString model path to prototxt model file
  145 + * \br_property QString weights path to caffemodel file
  146 + * \br_property int gpuDevice ID of GPU to use. gpuDevice < 0 runs on the CPU only.
  147 + */
  148 +class CaffeClassifierTransform : public CaffeBaseTransform
  149 +{
  150 + Q_OBJECT
  151 +
  152 + void project(const Template &src, Template &dst) const
  153 + {
  154 + Template caffeOutput;
  155 + CaffeBaseTransform::project(src, caffeOutput);
  156 +
  157 + dst = src;
  158 +
  159 + QList<int> labels; QList<float> confidences;
  160 +
  161 + foreach (const Mat &m, caffeOutput) {
  162 + double maxVal; int maxLoc;
  163 + minMaxIdx(m, NULL, &maxVal, NULL, &maxLoc);
  164 +
  165 + labels.append(maxLoc);
  166 + confidences.append(maxVal);
  167 + }
  168 +
  169 + if (labels.size() == 1) {
  170 + dst.file.set("Label", labels[0]);
  171 + dst.file.set("Confidence", confidences[0]);
  172 + } else {
  173 + dst.file.setList<int>("Labels", labels);
  174 + dst.file.setList<float>("Confidences", confidences);
  175 + }
  176 + }
  177 +};
  178 +
  179 +BR_REGISTER(Transform, CaffeClassifierTransform)
  180 +
114 181 } // namespace br
115 182  
116 183 #include "classification/caffe.moc"
... ...