Peter M. Groen / openbr

Commit 18e65e137bbe766377bfc6baf66bd7c81590191f

Authored by Jordan Cheney
1 parent 8eead12b

Merge of LMDB gallery into existing cmake structure

Inline Side-by-side
Showing 3 changed files with 108 additions and 152 deletions
openbr/plugins/cmake/caffe.cmake
  View file @18e65e1
... ... @@ -7,4 +7,5 @@ if(${BR_WITH_CAFFE})
7 7 set(BR_THIRDPARTY_LIBS ${BR_THIRDPARTY_LIBS} ${Caffe_LIBRARIES})
8 8 else()
9 9 set(BR_EXCLUDED_PLUGINS ${BR_EXCLUDED_PLUGINS} plugins/classification/caffe.cpp)
  10 + set(BR_EXCLUDED_PLUGINS ${BR_EXCLUDED_PLUGINS} plugins/gallery/lmdb.cpp)
10 11 endif()
... ...
openbr/plugins/gallery/lmdbGallery.cpp renamed to openbr/plugins/gallery/lmdb.cpp
  View file @18e65e1
1 1 #include <openbr/openbr_plugin.h>
2 2 #include "openbr/plugins/openbr_internal.h"
  3 +#include <openbr/core/qtutils.h>
  4 +
3 5 #include <QFutureSynchronizer>
4 6 #include <QtConcurrentRun>
5 7 #include <QMutexLocker>
... ... @@ -13,57 +15,58 @@ using namespace br;
13 15 class lmdbGallery : public Gallery
14 16 {
15 17 Q_OBJECT
  18 +
16 19 TemplateList readBlock(bool *done)
17 20 {
18   - *done = false;
19   - if (!initialized) {
20   - db = QSharedPointer<caffe::db::DB>(caffe::db::GetDB("lmdb"));
21   - db->Open(file.name.toStdString(),caffe::db::READ);
22   - cursor = QSharedPointer<caffe::db::Cursor>(db->NewCursor());
23   - initialized = true;
24   - }
25   -
26   - caffe::Datum datum;
27   - datum.ParseFromString(cursor->value());
28   -
29   - cv::Mat img;
30   - if (datum.encoded()) {
31   - img = caffe::DecodeDatumToCVMatNative(datum);
32   - }
33   - else {
34   - // create output image of appropriate size
35   - img.create(datum.height(), datum.width(), CV_MAKETYPE(CV_8U, datum.channels()));
36   - // copy matrix data from datum.
37   - for (int h = 0; h < datum.height(); ++h) {
38   - uchar* ptr = img.ptr<uchar>(h);
39   - int img_index = 0;
40   - for (int w = 0; w < datum.width(); ++w) {
41   - for (int c = 0; c < datum.channels(); ++c) {
42   - int datum_index = (c * datum.height() + h) * datum.width() + w;
43   - ptr[img_index++] = (unsigned char) datum.data()[datum_index];
44   - }
45   - }
46   - }
47   - }
48   -
49   - // We acquired the image data, now decode filename from db key
50   - QString baseKey = cursor->key().c_str();
51   -
52   - int idx = baseKey.indexOf("_");
53   - if (idx != -1)
54   - baseKey = baseKey.right(baseKey.size() - idx - 1);
55   -
56   - TemplateList output;
57   - output.append(Template(img));
58   - output.last().file.name = baseKey;
59   - output.last().file.set("Label", datum.label());
60   -
61   - cursor->Next();
62   -
63   - if (!cursor->valid())
64   - *done = true;
65   -
66   - return output;
  21 + *done = false;
  22 + if (!initialized) {
  23 + db = QSharedPointer<caffe::db::DB>(caffe::db::GetDB("lmdb"));
  24 + db->Open(file.name.toStdString(),caffe::db::READ);
  25 + cursor = QSharedPointer<caffe::db::Cursor>(db->NewCursor());
  26 + initialized = true;
  27 + }
  28 +
  29 + caffe::Datum datum;
  30 + datum.ParseFromString(cursor->value());
  31 +
  32 + cv::Mat img;
  33 + if (datum.encoded()) {
  34 + img = caffe::DecodeDatumToCVMatNative(datum);
  35 + }
  36 + else {
  37 + // create output image of appropriate size
  38 + img.create(datum.height(), datum.width(), CV_MAKETYPE(CV_8U, datum.channels()));
  39 + // copy matrix data from datum.
  40 + for (int h = 0; h < datum.height(); ++h) {
  41 + uchar* ptr = img.ptr<uchar>(h);
  42 + int img_index = 0;
  43 + for (int w = 0; w < datum.width(); ++w) {
  44 + for (int c = 0; c < datum.channels(); ++c) {
  45 + int datum_index = (c * datum.height() + h) * datum.width() + w;
  46 + ptr[img_index++] = (unsigned char)datum.data()[datum_index];
  47 + }
  48 + }
  49 + }
  50 + }
  51 +
  52 + // We acquired the image data, now decode filename from db key
  53 + QString baseKey = cursor->key().c_str();
  54 +
  55 + int idx = baseKey.indexOf("_");
  56 + if (idx != -1)
  57 + baseKey = baseKey.right(baseKey.size() - idx - 1);
  58 +
  59 + TemplateList output;
  60 + output.append(Template(img));
  61 + output.last().file.name = baseKey;
  62 + output.last().file.set("Label", datum.label());
  63 +
  64 + cursor->Next();
  65 +
  66 + if (!cursor->valid())
  67 + *done = true;
  68 +
  69 + return output;
67 70 }
68 71  
69 72 bool initialized;
... ... @@ -77,113 +80,93 @@ class lmdbGallery : public Gallery
77 80 bool should_end;
78 81 TemplateList data;
79 82  
80   -
81   - QHash<QString,int> observedLabels;
82   -
83 83 static void commitLoop(lmdbGallery * base)
84 84 {
85   - QSharedPointer<caffe::db::Transaction> txn;
  85 + QSharedPointer<caffe::db::Transaction> txn;
86 86  
87   - int total_count = 0;
  87 + int total_count = 0;
88 88  
89   - // Acquire the lock
90   - QMutexLocker lock(&base->dataLock);
  89 + // Acquire the lock
  90 + QMutexLocker lock(&base->dataLock);
91 91  
92   - while (true)
93   - {
94   - // wait for data, or end signal
95   - while(base->data.isEmpty() && !base->should_end)
96   - base->dataWait.wait(&base->dataLock);
  92 + while (true) {
  93 + // wait for data, or end signal
  94 + while(base->data.isEmpty() && !base->should_end)
  95 + base->dataWait.wait(&base->dataLock);
97 96  
98   - // If should_end, but there is still data, we need another commit
99   - // round
100   - if (base->should_end && base->data.isEmpty())
101   - break;
  97 + // If should_end, but there is still data, we need another commit
  98 + // round
  99 + if (base->should_end && base->data.isEmpty())
  100 + break;
102 101  
103   - txn = QSharedPointer<caffe::db::Transaction>(base->db->NewTransaction());
  102 + txn = QSharedPointer<caffe::db::Transaction>(base->db->NewTransaction());
104 103  
105   - TemplateList working = base->data;
106   - base->data.clear();
  104 + TemplateList working = base->data;
  105 + base->data.clear();
107 106  
108   - // no longer blocking dataLock
109   - lock.unlock();
  107 + // no longer blocking dataLock
  108 + lock.unlock();
110 109  
111   - foreach(const Template & t, working) {
112   - qDebug() << "Converting t to datum";
113   - // add current image to transaction
114   - caffe::Datum datum;
115   - caffe::CVMatToDatum(t.m(), &datum);
  110 + foreach(const Template &t, working) {
  111 + // add current image to transaction
  112 + caffe::Datum datum;
  113 + caffe::CVMatToDatum(t.m(), &datum);
  114 + datum.set_label(t.file.get<int>("Label"));
116 115  
117   - qDebug() << "Did conversion";
118   - QVariant base_label = t.file.value("Label");
119   - QString label_str = base_label.toString();
  116 + std::string out;
  117 + datum.SerializeToString(&out);
120 118  
121   -
122   - if (!base->observedLabels.contains(label_str) )
123   - base->observedLabels[label_str] = base->observedLabels.size();
  119 + char key_cstr[256];
  120 + int len = snprintf(key_cstr, 256, "%08d_%s", total_count, qPrintable(t.file.name));
  121 + txn->Put(std::string(key_cstr, len), out);
124 122  
125   - datum.set_label(base->observedLabels[label_str]);
126   - qDebug() << "Did labels";
127   - std::string out;
128   - datum.SerializeToString(&out);
129   - qDebug() << "Serialized";
130   -
131   - char key_cstr[256];
132   - int len = snprintf(key_cstr, 256, "%08d_%s", total_count, qPrintable(t.file.name));
133   - txn->Put(std::string(key_cstr, len), out);
134   - qDebug() << "added to txn";
135   - total_count++;
136   -
137   - }
138   - qDebug() << "committing!";
139   - txn->Commit();
140   - qDebug() << "Finished commit";
141   - lock.relock();
142   - }
  123 + total_count++;
  124 + }
143 125  
  126 + txn->Commit();
  127 + lock.relock();
  128 + }
144 129 }
145 130  
146 131 void write(const Template &t)
147 132 {
148   - if (!initialized) {
149   - db = QSharedPointer<caffe::db::DB> (caffe::db::GetDB("lmdb"));
150   - db->Open(file.name.toStdString(), caffe::db::NEW);
151   - observedLabels.clear();
152   - initialized = true;
153   - should_end = false;
154   - // fire thread
155   - aThread.clearFutures();
156   - aThread.addFuture(QtConcurrent::run(lmdbGallery::commitLoop, this));
157   - }
158   -
159   - QMutexLocker lock(&dataLock);
160   - data.append(t);
161   - dataWait.wakeAll();
162   -
  133 + if (!initialized) {
  134 + db = QSharedPointer<caffe::db::DB> (caffe::db::GetDB("lmdb"));
  135 + db->Open(file.name.toStdString(), caffe::db::NEW);
  136 + initialized = true;
  137 + should_end = false;
  138 + // fire thread
  139 + aThread.clearFutures();
  140 + aThread.addFuture(QtConcurrent::run(lmdbGallery::commitLoop, this));
  141 + }
  142 +
  143 + QMutexLocker lock(&dataLock);
  144 + data.append(t);
  145 + dataWait.wakeAll();
163 146 }
164 147  
165 148 ~lmdbGallery()
166 149 {
167   - if (initialized) {
168   - QMutexLocker lock(&dataLock);
169   - should_end = true;
170   - dataWait.wakeAll();
171   - lock.unlock();
172   -
173   - aThread.waitForFinished();
174   - }
  150 + if (initialized) {
  151 + QMutexLocker lock(&dataLock);
  152 + should_end = true;
  153 + dataWait.wakeAll();
  154 + lock.unlock();
  155 +
  156 + aThread.waitForFinished();
  157 + }
175 158 }
176 159  
177 160  
178 161 void init()
179 162 {
180 163 initialized = false;
181   - should_end = false;
  164 + should_end = false;
182 165 }
183 166 };
184 167  
185 168 BR_REGISTER(Gallery, lmdbGallery)
186 169  
187 170  
188   -#include "lmdbGallery.moc"
  171 +#include "gallery/lmdb.moc"
189 172  
... ...
share/openbr/cmake/FindLMDB.cmake deleted
View file @8eead12
1   -# Try to find the LMBD libraries and headers
2   -# LMDB_FOUND - system has LMDB lib
3   -# LMDB_INCLUDE_DIR - the LMDB include directory
4   -# LMDB_LIBRARIES - Libraries needed to use LMDB
5   -
6   -# FindCWD based on FindGMP by:
7   -# Copyright (c) 2006, Laurent Montel, <montel@kde.org>
8   -#
9   -# Redistribution and use is allowed according to the terms of the BSD license.
10   -
11   -# Adapted from FindCWD by:
12   -# Copyright 2013 Conrad Steenberg <conrad.steenberg@gmail.com>
13   -# Aug 31, 2013
14   -
15   -find_path(LMDB_INCLUDE_DIR NAMES lmdb.h PATHS "$ENV{LMDB_DIR}/include")
16   -find_library(LMDB_LIBRARIES NAMES lmdb PATHS "$ENV{LMDB_DIR}/lib" )
17   -
18   -include(FindPackageHandleStandardArgs)
19   -find_package_handle_standard_args(LMDB DEFAULT_MSG LMDB_INCLUDE_DIR LMDB_LIBRARIES)
20   -
21   -if(LMDB_FOUND)
22   - message(STATUS "Found lmdb (include: ${LMDB_INCLUDE_DIR}, library: ${LMDB_LIBRARIES})")
23   - mark_as_advanced(LMDB_INCLUDE_DIR LMDB_LIBRARIES)
24   -
25   - caffe_parse_header(${LMDB_INCLUDE_DIR}/lmdb.h
26   - LMDB_VERSION_LINES MDB_VERSION_MAJOR MDB_VERSION_MINOR MDB_VERSION_PATCH)
27   - set(LMDB_VERSION "${MDB_VERSION_MAJOR}.${MDB_VERSION_MINOR}.${MDB_VERSION_PATCH}")
28   -endif()