diff --git a/openbr/plugins/classification/caffe.cpp b/openbr/plugins/classification/caffe.cpp index cd2cf2c..3f393ce 100644 --- a/openbr/plugins/classification/caffe.cpp +++ b/openbr/plugins/classification/caffe.cpp @@ -1,40 +1,46 @@ #include +#include +#include +#include #include using caffe::Caffe; -using caffe::Solver; using caffe::Net; +using caffe::MemoryDataLayer; using caffe::Blob; using caffe::shared_ptr; -using caffe::vector; + +using namespace cv; namespace br { /*! - * \brief A transform that wraps the Caffe Deep learning library + * \brief A transform that wraps the Caffe deep learning library. This transform expects the input to a given Caffe model to be a MemoryDataLayer. + * The output of the Caffe network is treated as a feature vector and is stored in dst. Batch processing is possible. For a given batch size set in + * the memory data layer, src is expected to have an equal number of mats. Dst will always have the same size (number of mats) as src and the ordering + * will be preserved, so dst[1] is the output of src[1] after it passes through the * \author Jordan Cheney \cite jcheney - * \br_property QString modelFile path to prototxt model file - * \br_property QString solverFile path to prototxt solver file - * \br_property QString weightsFile path to caffemodel file + * \br_property QString model path to prototxt model file + * \br_property QString weights path to caffemodel file * \br_property int gpuDevice ID of GPU to use. gpuDevice < 0 runs on the CPU only. * \br_link Caffe Integration Tutorial ../tutorials.md#caffe * \br_link Caffe website http://caffe.berkeleyvision.org */ -class CaffeTransform : public Transform +class CaffeFVTransform : public UntrainableTransform { Q_OBJECT - Q_PROPERTY(QString modelFile READ get_modelFile WRITE set_modelFile RESET reset_modelFile STORED false) - Q_PROPERTY(QString solverFile READ get_solverFile WRITE set_solverFile RESET reset_solverFile STORED false) - Q_PROPERTY(QString weightsFile READ get_weightsFile WRITE set_weightsFile RESET reset_weightsFile STORED false) + Q_PROPERTY(QString model READ get_model WRITE set_model RESET reset_model STORED false) + Q_PROPERTY(QString weights READ get_weights WRITE set_weights RESET reset_weights STORED false) Q_PROPERTY(int gpuDevice READ get_gpuDevice WRITE set_gpuDevice RESET reset_gpuDevice STORED false) - BR_PROPERTY(QString, modelFile, "") - BR_PROPERTY(QString, solverFile, "") - BR_PROPERTY(QString, weightsFile, "") + BR_PROPERTY(QString, model, "") + BR_PROPERTY(QString, weights, "") BR_PROPERTY(int, gpuDevice, -1) + QSharedPointer > net; + void init() { if (gpuDevice >= 0) { @@ -43,47 +49,31 @@ class CaffeTransform : public Transform } else { Caffe::set_mode(Caffe::CPU); } + + net.reset(new Net(model.toStdString(), caffe::TEST)); + net->CopyTrainedLayersFrom(weights.toStdString()); } - void train(const TemplateList &data) + void project(const Template &src, Template &dst) const { - (void) data; + MemoryDataLayer *data_layer = static_cast *>(net->layers()[0].get()); - caffe::SolverParameter solver_param; - caffe::ReadProtoFromTextFileOrDie(solverFile.toStdString(), &solver_param); + if (src.size() != data_layer->batch_size()) + qFatal("src should have %d (batch size) mats. It has %d mats.", data_layer->batch_size(), src.size()); - shared_ptr > solver(caffe::GetSolver(solver_param)); - solver->Solve(); - } + dst.file = src.file; - void project(const Template &src, Template &dst) const - { - (void)src; (void)dst; - Net net(modelFile.toStdString(), caffe::TEST); - net.CopyTrainedLayersFrom(weightsFile.toStdString()); - - vector *> bottom_vec; // perhaps src data should go here? - vector test_score_output_id; - vector test_score; - - float loss; - const vector *> &result = net.Forward(bottom_vec, &loss); - - int idx = 0; - for (int i = 0; i < (int)result.size(); i++) { - const float *result_data = result[i]->cpu_data(); - for (int j = 0; j < result[i]->count(); j++, idx++) { - test_score.push_back(result_data[j]); - test_score_output_id.push_back(i); - - if (Globals->verbose) - qDebug("%s = %f", net.blob_names()[net.output_blob_indices()[i]].c_str(), result_data[j]); - } - } + data_layer->AddMatVector(src.toVector().toStdVector(), std::vector(src.size(), 0)); + + Blob *output = net->ForwardPrefilled()[1]; // index 0 is the labels from the data layer (in this case the 0 array we passed in above). + // index 1 is the ouput of the final layer, which is what we want + int dim_features = output->count() / data_layer->batch_size(); + for (int n = 0; n < data_layer->batch_size(); n++) + dst += Mat(1, dim_features, CV_32FC1, output->mutable_cpu_data() + output->offset(n)); } }; -BR_REGISTER(Transform, CaffeTransform) +BR_REGISTER(Transform, CaffeFVTransform) } // namespace br diff --git a/openbr/plugins/cmake/lmdbGallery.cmake b/openbr/plugins/cmake/lmdbGallery.cmake deleted file mode 100644 index 28dc467..0000000 --- a/openbr/plugins/cmake/lmdbGallery.cmake +++ /dev/null @@ -1,10 +0,0 @@ -set(BR_WITH_CAFFE OFF CACHE BOOL "Build with Caffe") - -if (${BR_WITH_CAFFE}) - find_package(Caffe) - set(BR_THIRDPARTY_LIBS ${BR_THIRDPARTY_LIBS} ${Caffe_LIBRARIES}) - include_directories(${Caffe_INCLUDE_DIRS}) -else() - set(BR_EXCLUDED_PLUGINS ${BR_EXCLUDED_PLUGINS} plugins/gallery/lmdbGallery.cpp) -endif() - diff --git a/openbr/plugins/imgproc/pad.cpp b/openbr/plugins/imgproc/pad.cpp new file mode 100644 index 0000000..2e4de15 --- /dev/null +++ b/openbr/plugins/imgproc/pad.cpp @@ -0,0 +1,33 @@ +#include + +using namespace cv; + +namespace br +{ + +class PadTransform : public UntrainableTransform +{ + Q_OBJECT + + Q_PROPERTY(int padSize READ get_padSize WRITE set_padSize RESET reset_padSize STORED false) + Q_PROPERTY(int padValue READ get_padValue WRITE set_padValue RESET reset_padValue STORED false) + BR_PROPERTY(int, padSize, 0) + BR_PROPERTY(int, padValue, 0) + + void project(const Template &src, Template &dst) const + { + dst.file = src.file; + + foreach (const Mat &m, src) { + Mat padded = padValue * Mat::ones(m.rows + 2*padSize, m.cols + 2*padSize, m.type()); + padded(Rect(padSize, padSize, padded.cols - padSize, padded.rows - padSize)) = m; + dst += padded; + } + } +}; + +BR_REGISTER(Transform, PadTransform) + +} // namespace br + +#include "imgproc/pad.moc" diff --git a/openbr/plugins/imgproc/roi.cpp b/openbr/plugins/imgproc/roi.cpp index 0da96c8..f0339bd 100644 --- a/openbr/plugins/imgproc/roi.cpp +++ b/openbr/plugins/imgproc/roi.cpp @@ -26,12 +26,16 @@ namespace br * \ingroup transforms * \brief Crops the rectangular regions of interest. * \author Josh Klontz \cite jklontz + * \br_property QString propName Optional property name for a rectangle in metadata. If no propName is given the transform will use rects stored in the file.rects field or build a rectangle using "X", "Y", "Width", and "Height" fields if they exist. + * \br_property bool copyOnCrop If true make a clone of each crop before appending the crop to dst. This guarantees that the crops will be continuous in memory, which is an occasionally useful property. Default is false. */ class ROITransform : public UntrainableTransform { Q_OBJECT Q_PROPERTY(QString propName READ get_propName WRITE set_propName RESET reset_propName STORED false) + Q_PROPERTY(bool copyOnCrop READ get_copyOnCrop WRITE set_copyOnCrop RESET reset_copyOnCrop STORED false) BR_PROPERTY(QString, propName, "") + BR_PROPERTY(bool, copyOnCrop, false) void project(const Template &src, Template &dst) const { @@ -52,6 +56,10 @@ class ROITransform : public UntrainableTransform qWarning("No rects present in file."); } dst.file.clearRects(); + + if (copyOnCrop) + for (int i = 0; i < dst.size(); i++) + dst.replace(i, dst[i].clone()); } };