diff --git a/openbr/core/core.cpp b/openbr/core/core.cpp index 2069395..fbb52b6 100644 --- a/openbr/core/core.cpp +++ b/openbr/core/core.cpp @@ -19,6 +19,7 @@ #include "bee.h" #include "common.h" #include "qtutils.h" +#include "../plugins/openbr_internal.h" using namespace br; @@ -44,6 +45,14 @@ struct AlgorithmCore qDebug("Training on %s%s", qPrintable(input.flat()), model.isEmpty() ? "" : qPrintable(" to " + model)); + QScopedPointer trainingWrapper(Transform::make("DirectStream([Identity])", NULL)); + CompositeTransform * downcast = dynamic_cast(trainingWrapper.data()); + if (downcast == NULL) + qFatal("downcast failed?"); + downcast->transforms[0] = this->transform.data(); + + downcast->init(); + TemplateList data(TemplateList::fromGallery(input)); // set the Train bool metadata, in case a Transform's project @@ -56,7 +65,7 @@ struct AlgorithmCore QTime time; time.start(); qDebug("Training Enrollment"); - transform->train(data); + downcast->train(data); if (!distance.isNull()) { qDebug("Projecting Enrollment"); @@ -114,74 +123,64 @@ struct AlgorithmCore FileList enroll(File input, File gallery = File()) { + FileList files; + qDebug("Enrolling %s%s", qPrintable(input.flat()), gallery.isNull() ? "" : qPrintable(" to " + gallery.flat())); - FileList fileList; if (gallery.name.isEmpty()) { if (input.name.isEmpty()) return FileList(); else gallery = getMemoryGallery(input); } + TemplateList data(TemplateList::fromGallery(input)); - QScopedPointer g(Gallery::make(gallery)); - if (g.isNull()) qFatal("Null gallery!"); - - do { - fileList.clear(); - - if (gallery.contains("read") || gallery.contains("cache")) - fileList = g->files(); - - if (!fileList.isEmpty() && gallery.contains("cache")) - return fileList; - - const TemplateList i(TemplateList::fromGallery(input)); - if (i.isEmpty()) return fileList; // Nothing to enroll - - if (transform.isNull()) qFatal("Null transform."); - const int blocks = Globals->blocks(i.size()); - Globals->currentStep = 0; - Globals->totalSteps = i.size(); - Globals->startTime.start(); - - const bool noDuplicates = gallery.contains("noDuplicates"); - QStringList fileNames = noDuplicates ? fileList.names() : QStringList(); - const int subBlockSize = 4*std::max(1, Globals->parallelism); - const int numSubBlocks = ceil(1.0*Globals->blockSize/subBlockSize); - int totalCount = 0, failureCount = 0; - double totalBytes = 0; - for (int block=0; blockblockSize + subBlock*subBlockSize, subBlockSize); - if (data.isEmpty()) break; - if (noDuplicates) - for (int i=data.size()-1; i>=0; i--) - if (fileNames.contains(data[i].file.name)) - data.removeAt(i); - const int numFiles = data.size(); - - data >> *transform; - - g->writeBlock(data); - const FileList newFiles = data.files(); - fileList.append(newFiles); - - totalCount += newFiles.size(); - failureCount += newFiles.failures(); - totalBytes += data.bytes(); - Globals->currentStep += numFiles; - Globals->printStatus(); + if (gallery.contains("append")) + { + // Remove any templates which are already in the gallery + QScopedPointer g(Gallery::make(gallery)); + files = g->files(); + QSet nameSet = QSet::fromList(files.names()); + for (int i = data.size() - 1; i>=0; i--) { + if (nameSet.contains(data[i].file.name)) + { + data.removeAt(i); } } + } + + if (data.empty()) + return files; + + // Trust me, this makes complete sense. + // We're just going to make a pipe with a placeholder first transform + QString pipeDesc = "Identity+GalleryOutput("+gallery.flat()+")+ProgressCounter("+QString::number(data.length())+")+Discard"; + QScopedPointer basePipe(Transform::make(pipeDesc,NULL)); + + CompositeTransform * downcast = dynamic_cast(basePipe.data()); + if (downcast == NULL) + qFatal("downcast failed?"); + + // replace that placeholder with the current algorithm + downcast->transforms[0] = this->transform.data(); + + // call init on the pipe to collapse the algorithm (if its top level is a pipe) + downcast->init(); + + // Next, we make a Stream (with placeholder transform) + QString streamDesc = "Stream(Identity, readMode=DistributeFrames)"; + QScopedPointer baseStream(Transform::make(streamDesc, NULL)); + WrapperTransform * wrapper = dynamic_cast (baseStream.data()); + + // replace that placeholder with the pipe we built + wrapper->transform = downcast; + + // and get the final stream's stages by reinterpreting the pipe. Perfectly straightforward. + wrapper->init(); - const float speed = 1000 * Globals->totalSteps / Globals->startTime.elapsed() / std::max(1, abs(Globals->parallelism)); - if (!Globals->quiet && (Globals->totalSteps > 1)) - fprintf(stderr, "\rTIME ELAPSED (MINS) %f SPEED=%.1e SIZE=%.4g FAILURES=%d/%d \n", - Globals->startTime.elapsed()/1000./60.,speed, totalBytes/totalCount, failureCount, totalCount); - Globals->totalSteps = 0; - } while (input.getBool("infinite")); + wrapper->projectUpdate(data,data); - return fileList; + files.append(data.files()); + return files; } void enroll(TemplateList &data) @@ -312,8 +311,6 @@ private: if ((words.size() < 1) || (words.size() > 2)) qFatal("Invalid algorithm format."); //! [Parsing the algorithm description] - if (description.getBool("distribute", true)) - words[0] = "DistributeTemplate(" + words[0] + ")"; //! [Creating the template generation and comparison methods] transform = QSharedPointer(Transform::make(words[0], NULL)); diff --git a/openbr/core/qtutils.cpp b/openbr/core/qtutils.cpp index c18907b..0d5b511 100644 --- a/openbr/core/qtutils.cpp +++ b/openbr/core/qtutils.cpp @@ -427,6 +427,17 @@ QString toString(const QVariantList &variantList) return QString(); } +QString toTime(int s) +{ + int h = s / (60*60); + int m = (s - h*60*60) / 60; + s = (s - h*60*60 - m*60); + + const QChar fillChar = QLatin1Char('0'); + + return QString("%1:%2:%3").arg(h,2,10,fillChar).arg(m,2,10,fillChar).arg(s,2,10,fillChar); +} + float euclideanLength(const QPointF &point) { return sqrt(pow(point.x(), 2) + pow(point.y(), 2)); diff --git a/openbr/core/qtutils.h b/openbr/core/qtutils.h index 3435b80..256b968 100644 --- a/openbr/core/qtutils.h +++ b/openbr/core/qtutils.h @@ -65,6 +65,7 @@ namespace QtUtils QPointF toPoint(const QString &string, bool *ok = NULL); QRectF toRect(const QString &string, bool *ok = NULL); QStringList naturalSort(const QStringList &strings); + QString toTime(int s); /**** Process Utilities ****/ bool runRScript(const QString &file); diff --git a/openbr/openbr_plugin.cpp b/openbr/openbr_plugin.cpp index 5ae2006..9322f5d 100644 --- a/openbr/openbr_plugin.cpp +++ b/openbr/openbr_plugin.cpp @@ -839,10 +839,7 @@ void br::Context::printStatus() const float p = progress(); if (p < 1) { int s = timeRemaining(); - int h = s / (60*60); - int m = (s - h*60*60) / 60; - s = (s - h*60*60 - m*60); - fprintf(stderr, "%05.2f%% REMAINING=%02d:%02d:%02d COUNT=%g \r", 100 * p, h, m, s, totalSteps); + fprintf(stderr, "%05.2f%% REMAINING=%s COUNT=%g \r", 100 * p, QtUtils::toTime(s/1000.0f).toStdString().c_str(), totalSteps); } } @@ -1156,6 +1153,17 @@ Gallery *Gallery::make(const File &file) return gallery; } +// Default init -- if the file contains "append", read the existing +// data and immediately write it +void Gallery::init() +{ + if (file.exists() && file.contains("append")) + { + TemplateList data = this->read(); + this->writeBlock(data); + } +} + /* Transform - public methods */ Transform::Transform(bool _independent, bool _trainable) { diff --git a/openbr/openbr_plugin.h b/openbr/openbr_plugin.h index 6472337..d96c71e 100644 --- a/openbr/openbr_plugin.h +++ b/openbr/openbr_plugin.h @@ -1060,6 +1060,7 @@ public: void writeBlock(const TemplateList &templates); /*!< \brief Serialize a template list. */ virtual void write(const Template &t) = 0; /*!< \brief Serialize a template. */ static Gallery *make(const File &file); /*!< \brief Make a gallery to/from a file on disk. */ + void init(); private: QSharedPointer next; diff --git a/openbr/plugins/algorithms.cpp b/openbr/plugins/algorithms.cpp index 2bf0895..1b4e0f4 100644 --- a/openbr/plugins/algorithms.cpp +++ b/openbr/plugins/algorithms.cpp @@ -38,7 +38,7 @@ class AlgorithmsInitializer : public Initializer Globals->abbreviations.insert("MedianFace", "Open!Cascade(FrontalFace)+ASEFEyes+Affine(256,256,0.37,0.45)+Center(Median)"); Globals->abbreviations.insert("BlurredFaceDetection", "Open+LimitSize(1024)+SkinMask/(Cvt(Gray)+GradientMask)+And+Morph(Erode,16)+LargestConvexArea"); Globals->abbreviations.insert("DrawFaceDetection", "Open+Cascade(FrontalFace)!ASEFEyes+Draw"); - Globals->abbreviations.insert("ShowFaceDetection", "DrawFaceDetection!Show[distribute=false]"); + Globals->abbreviations.insert("ShowFaceDetection", "DrawFaceDetection!Show"); Globals->abbreviations.insert("OpenBR", "FaceRecognition"); Globals->abbreviations.insert("GenderEstimation", "GenderClassification"); Globals->abbreviations.insert("AgeEstimation", "AgeRegression"); diff --git a/openbr/plugins/gallery.cpp b/openbr/plugins/gallery.cpp index 1549bcd..4cc2221 100644 --- a/openbr/plugins/gallery.cpp +++ b/openbr/plugins/gallery.cpp @@ -73,6 +73,11 @@ class arffGallery : public Gallery arffFile.write(qPrintable(OpenCVUtils::matrixToStringList(t).join(','))); arffFile.write(qPrintable(",'" + t.file.get("Label") + "'\n")); } + + void init() + { + // + } }; BR_REGISTER(Gallery, arffGallery) @@ -94,7 +99,12 @@ class galGallery : public Gallery if (file.get("remove", false)) gallery.remove(); QtUtils::touchDir(gallery); - if (!gallery.open(QFile::ReadWrite | QFile::Append)) + QFile::OpenMode mode = QFile::ReadWrite; + + if (file.contains("append")) + mode |= QFile::Append; + + if (!gallery.open(mode)) qFatal("Can't open gallery: %s", qPrintable(gallery.fileName())); stream.setDevice(&gallery); } @@ -579,6 +589,11 @@ class templateGallery : public Gallery (void) t; qFatal("No supported."); } + + void init() + { + // + } }; BR_REGISTER(Gallery, templateGallery) @@ -737,6 +752,11 @@ class dbGallery : public Gallery (void) t; qFatal("Not supported."); } + + void init() + { + // + } }; BR_REGISTER(Gallery, dbGallery) @@ -790,6 +810,11 @@ class googleGallery : public Gallery (void) t; qFatal("Not supported."); } + + void init() + { + // + } }; BR_REGISTER(Gallery, googleGallery) @@ -883,6 +908,11 @@ class FDDBGallery : public Gallery (void) t; qFatal("Not implemented."); } + + void init() + { + // + } }; BR_REGISTER(Gallery, FDDBGallery) @@ -927,6 +957,11 @@ class landmarksGallery : public Gallery (void) t; qFatal("Not implemented."); } + + void init() + { + // + } }; BR_REGISTER(Gallery, landmarksGallery) diff --git a/openbr/plugins/misc.cpp b/openbr/plugins/misc.cpp index 9b66378..3dd8c4b 100644 --- a/openbr/plugins/misc.cpp +++ b/openbr/plugins/misc.cpp @@ -14,10 +14,12 @@ * limitations under the License. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +#include #include #include #include "openbr_internal.h" #include "openbr/core/opencvutils.h" +#include "openbr/core/qtutils.h" using namespace cv; @@ -517,6 +519,106 @@ class EventTransform : public UntrainableMetaTransform }; BR_REGISTER(Transform, EventTransform) + +class GalleryOutputTransform : public TimeVaryingTransform +{ + Q_OBJECT + + Q_PROPERTY(QString outputString READ get_outputString WRITE set_outputString RESET reset_outputString STORED false) + BR_PROPERTY(QString, outputString, "") + + void projectUpdate(const TemplateList &src, TemplateList &dst) + { + if (src.empty()) + return; + dst = src; + writer->writeBlock(dst); + } + + void train(const TemplateList& data) + { + (void) data; + } + ; + void init() + { + writer = QSharedPointer(Gallery::make(outputString)); + } + + QSharedPointer writer; +public: + GalleryOutputTransform() : TimeVaryingTransform(false,false) {} +}; + +BR_REGISTER(Transform, GalleryOutputTransform) + +class ProgressCounterTransform : public TimeVaryingTransform +{ + Q_OBJECT + + Q_PROPERTY(int totalTemplates READ get_totalTemplates WRITE set_totalTemplates RESET reset_totalTemplates STORED false) + BR_PROPERTY(int, totalTemplates, 1) + + void projectUpdate(const TemplateList &src, TemplateList &dst) + { + dst = src; + qint64 elapsed = timer.elapsed(); + calls++; + set_calls++; + // updated every 10 seconds + if (elapsed > 5 * 1000) { + float f_elapsed = elapsed / 1000.0f; + // remaining calls (according to our input variable) + int remaining = totalTemplates - calls; + // calls / second + float speed = set_calls / f_elapsed; + + float p = 100 * float(calls) / totalTemplates; + + // seconds remaining + int s = float(remaining) / speed; + + fprintf(stderr, "%05.2f%% ELAPSED=%s REMAINING=%s COUNT=%g \r", p, QtUtils::toTime(Globals->startTime.elapsed()/1000.0f).toStdString().c_str(), QtUtils::toTime(s).toStdString().c_str(), float(calls)); + + timer.start(); + set_calls = 0; + } + + + return; + } + + void train(const TemplateList& data) + { + (void) data; + } + + void finalize(TemplateList & data) + { + (void) data; + float p = 100 * float(calls) / totalTemplates; + qDebug("%05.2f%% ELAPSED=%s REMAINING=%s COUNT=%g \r", p, QtUtils::toTime(Globals->startTime.elapsed()/1000.0f).toStdString().c_str(), QtUtils::toTime(0).toStdString().c_str(), float(calls)); + } + + void init() + { + calls = 0; + set_calls = 0; + timer.start(); + Globals->startTime.start(); + } + +public: + ProgressCounterTransform() : TimeVaryingTransform(false,false) {} + bool initialized; + QElapsedTimer timer; + qint64 calls; + qint64 set_calls; + +}; + +BR_REGISTER(Transform, ProgressCounterTransform) + } #include "misc.moc" diff --git a/openbr/plugins/openbr_internal.h b/openbr/plugins/openbr_internal.h index e40a770..b56daad 100644 --- a/openbr/plugins/openbr_internal.h +++ b/openbr/plugins/openbr_internal.h @@ -154,6 +154,53 @@ protected: } }; +/*! + * \brief Interface for transforms that act as decorators of another transform + */ +class BR_EXPORT WrapperTransform : public TimeVaryingTransform +{ + Q_OBJECT +public: + WrapperTransform(bool independent = true) : TimeVaryingTransform(independent) + { + } + + Q_PROPERTY(br::Transform *transform READ get_transform WRITE set_transform RESET reset_transform STORED false) + BR_PROPERTY(br::Transform *, transform, NULL) + + bool timeVarying() const { return transform->timeVarying(); } + + void project(const Template &src, Template &dst) const + { + transform->project(src,dst); + } + + void projectUpdate(const Template &src, Template &dst) + { + transform->projectUpdate(src,dst); + } + void projectUpdate(const TemplateList & src, TemplateList & dst) + { + transform->projectUpdate(src,dst); + } + + void train(const QList & data) + { + transform->train(data); + } + + virtual void finalize(TemplateList & output) + { + transform->finalize(output); + } + + void init() + { + if (transform) + this->trainable = transform->trainable; + } + +}; /*! * \brief A MetaTransform that aggregates some sub-transforms diff --git a/openbr/plugins/stream.cpp b/openbr/plugins/stream.cpp index 14e3ab5..0679746 100644 --- a/openbr/plugins/stream.cpp +++ b/openbr/plugins/stream.cpp @@ -16,6 +16,17 @@ using namespace cv; namespace br { +class Idiocy : public QObject +{ + Q_OBJECT +public: + enum StreamModes { StreamVideo, + DistributeFrames, + Auto}; + + Q_ENUMS(StreamModes) +}; + class FrameData { public: @@ -180,198 +191,29 @@ private: QList buffer2; }; - -// Interface for sequentially getting data from some data source. -// Initialized off of a template, can represent a video file (stored in the template's filename) -// or a set of images already loaded into memory stored as multiple matrices in an input template. -class DataSource +// Given a template as input, return N templates as output, one at a time on subsequent +// calls to getNext +class TemplateProcessor { public: - DataSource(int maxFrames=500) - { - // The sequence number of the last frame - final_frame = -1; - for (int i=0; i < maxFrames;i++) - { - allFrames.addItem(new FrameData()); - } - } - - virtual ~DataSource() - { - while (true) - { - FrameData * frame = allFrames.tryGetItem(); - if (frame == NULL) - break; - delete frame; - } - } - - // non-blocking version of getFrame - // Returns a NULL FrameData if too many frames are out, or the - // data source is broken. Sets last_frame to true iff the FrameData - // returned is the last valid frame, and the data source is now broken. - FrameData * tryGetFrame(bool & last_frame) - { - last_frame = false; - - if (is_broken) { - return NULL; - } - - // Try to get a FrameData from the pool, if we can't it means too many - // frames are already out, and we will return NULL to indicate failure - FrameData * aFrame = allFrames.tryGetItem(); - if (aFrame == NULL) - return NULL; - - // Try to actually read a frame, if this returns false the data source is broken - bool res = getNext(*aFrame); - - // The datasource broke, update final_frame - if (!res) - { - QMutexLocker lock(&last_frame_update); - final_frame = lookAhead.back()->sequenceNumber; - allFrames.addItem(aFrame); - } - else { - lookAhead.push_back(aFrame); - } - - // we will return the first frame on the lookAhead buffer - FrameData * rVal = lookAhead.first(); - lookAhead.pop_front(); - - // If this is the last frame, say so - if (rVal->sequenceNumber == final_frame) { - last_frame = true; - is_broken = true; - } - - return rVal; - } - - // Return a frame to the pool, returns true if the frame returned was the last - // frame issued, false otherwise - bool returnFrame(FrameData * inputFrame) - { - int frameNumber = inputFrame->sequenceNumber; - - inputFrame->data.clear(); - inputFrame->sequenceNumber = -1; - allFrames.addItem(inputFrame); - - bool rval = false; - - QMutexLocker lock(&last_frame_update); - - if (frameNumber == final_frame) { - // We just received the last frame, better pulse - allReturned = true; - lastReturned.wakeAll(); - rval = true; - } - - return rval; - } - - bool waitLast() - { - QMutexLocker lock(&last_frame_update); - - while (!allReturned) - { - // This would be a safer wait if we used a timeout, but - // theoretically that should never matter. - lastReturned.wait(&last_frame_update); - } - return true; - } - - bool open(Template & output, int start_index = 0) - { - is_broken = false; - allReturned = false; - - // The last frame isn't initialized yet - final_frame = -1; - // Start our sequence numbers from the input index - next_sequence_number = start_index; - - // Actually open the data source - bool open_res = concreteOpen(output); - - // We couldn't open the data source - if (!open_res) { - is_broken = true; - return false; - } - - // Try to get a frame from the global pool - FrameData * firstFrame = allFrames.tryGetItem(); - - // If this fails, things have gone pretty badly. - if (firstFrame == NULL) { - is_broken = true; - return false; - } - - // Read a frame from the video source - bool res = getNext(*firstFrame); - - // the data source broke already, we couldn't even get one frame - // from it even though it claimed to have opened successfully. - if (!res) { - is_broken = true; - return false; - } - - // We read one frame ahead of the last one returned, this allows - // us to know which frame is the final frame when we return it. - lookAhead.append(firstFrame); - return true; - } - - /* - * Pure virtual methods - */ - - // isOpen doesn't appear to particularly work when used on opencv - // VideoCaptures, so we don't use it for anything important. + virtual ~TemplateProcessor() {} + virtual bool open(Template & input)=0; virtual bool isOpen()=0; - // Called from open, open the data source specified by the input - // template, don't worry about setting any of the state variables - // set in open. - virtual bool concreteOpen(Template & output) = 0; - // Get the next frame from the data source, store the results in - // FrameData (including the actual frame and appropriate sequence - // number). - virtual bool getNext(FrameData & input) = 0; - // close the currently open data source. - virtual void close() = 0; - - int next_sequence_number; + virtual void close()=0; + virtual bool getNextTemplate(Template & output)=0; protected: - DoubleBuffer allFrames; - int final_frame; - bool is_broken; - bool allReturned; - QList lookAhead; - - QWaitCondition lastReturned; - QMutex last_frame_update; + Template basis; }; static QMutex openLock; + // Read a video frame by frame using cv::VideoCapture -class VideoDataSource : public DataSource +class VideoReader : public TemplateProcessor { public: - VideoDataSource(int maxFrames) : DataSource(maxFrames) {} + VideoReader() {} - bool concreteOpen(Template &input) + bool open(Template &input) { basis = input; @@ -407,30 +249,23 @@ public: bool isOpen() { return video.isOpened(); } - void close() { - video.release(); - } + void close() { video.release(); } -private: - bool getNext(FrameData & output) + bool getNextTemplate(Template & output) { if (!isOpen()) { qDebug("video source is not open"); return false; } - - output.data.append(Template(basis.file)); - output.data.last().m() = cv::Mat(); - - output.sequenceNumber = next_sequence_number; - next_sequence_number++; + output.file = basis.file; + output.m() = cv::Mat(); cv::Mat temp; bool res = video.read(temp); if (!res) { // The video capture broke, return false. - output.data.last().m() = cv::Mat(); + output.m() = cv::Mat(); close(); return false; } @@ -438,198 +273,330 @@ private: // This clone is critical, if we don't do it then the matrix will // be an alias of an internal buffer of the video source, leading // to various problems later. - output.data.last().m() = temp.clone(); - - output.data.last().file.set("FrameNumber", output.sequenceNumber); + output.m() = temp.clone(); return true; } - +protected: cv::VideoCapture video; - Template basis; }; -// Given a template as input, return its matrices one by one on subsequent calls -// to getNext -class TemplateDataSource : public DataSource + +class DirectReturn : public TemplateProcessor { public: - TemplateDataSource(int maxFrames) : DataSource(maxFrames) + DirectReturn() { - current_matrix_idx = INT_MAX; data_ok = false; } - // To "open" it we just set appropriate indices, we assume that if this - // is an image, it is already loaded into memory. - bool concreteOpen(Template &input) + // We don't do anything, just prepare to return input when getNext is called. + bool open(Template &input) { basis = input; - current_matrix_idx = 0; - - data_ok = current_matrix_idx < basis.size(); + data_ok =true; return data_ok; } - bool isOpen() { - return data_ok; - } + bool isOpen() { return data_ok; } void close() { - current_matrix_idx = INT_MAX; + data_ok = false; basis.clear(); } -private: - bool getNext(FrameData & output) + bool getNextTemplate(Template & output) { - data_ok = current_matrix_idx < basis.size(); if (!data_ok) return false; - - output.data.append(basis[current_matrix_idx]); - current_matrix_idx++; - - output.sequenceNumber = next_sequence_number; - next_sequence_number++; - - output.data.last().file.set("FrameNumber", output.sequenceNumber); + output = basis; + data_ok = false; return true; } - Template basis; - // Index of the next matrix to output from the template - int current_matrix_idx; - - // is current_matrix_idx in bounds? +protected: + // Have we sent our template yet? bool data_ok; }; -// Given a templatelist as input, create appropriate data source for each -// individual template -class DataSourceManager : public DataSource + +// Interface for sequentially getting data from some data source. +// Given a TemplateList, return single template frames sequentially by applying a TemplateProcessor +// to each individual template. +class DataSource { public: - DataSourceManager(int activeFrames=100) : DataSource(activeFrames) + DataSource(int maxFrames=500) { - actualSource = NULL; + // The sequence number of the last frame + final_frame = -1; + for (int i=0; i < maxFrames;i++) + { + allFrames.addItem(new FrameData()); + } + frameSource = NULL; } - ~DataSourceManager() + virtual ~DataSource() { - close(); + while (true) + { + FrameData * frame = allFrames.tryGetItem(); + if (frame == NULL) + break; + delete frame; + } } - int size() + void close() { - return this->allFrames.size(); + if (this->frameSource) + { + frameSource->close(); + delete frameSource; + frameSource = NULL; + } } - void close() + int size() { - if (actualSource) { - actualSource->close(); - delete actualSource; - actualSource = NULL; - } + return this->templates.size(); } - // We are used through a call to open(TemplateList) - bool open(TemplateList & input) + bool open(TemplateList & input, br::Idiocy::StreamModes _mode) { // Set up variables specific to us current_template_idx = 0; templates = input; + mode = _mode; + + is_broken = false; + allReturned = false; - // Call datasourece::open on the first template to set up - // state variables - return DataSource::open(templates[current_template_idx]); + // The last frame isn't initialized yet + final_frame = -1; + // Start our sequence numbers from the input index + next_sequence_number = 0; + + // Actually open the data source + bool open_res = openNextTemplate(); + + // We couldn't open the data source + if (!open_res) { + is_broken = true; + return false; + } + + // Try to get a frame from the global pool + FrameData * firstFrame = allFrames.tryGetItem(); + + // If this fails, things have gone pretty badly. + if (firstFrame == NULL) { + is_broken = true; + return false; + } + + // Read a frame from the video source + bool res = getNextFrame(*firstFrame); + + // the data source broke already, we couldn't even get one frame + // from it even though it claimed to have opened successfully. + if (!res) { + is_broken = true; + return false; + } + + // We read one frame ahead of the last one returned, this allows + // us to know which frame is the final frame when we return it. + lookAhead.append(firstFrame); + return true; } - // Create an actual data source of appropriate type for this template - // (initially called via the call to DataSource::open, called later - // as we run out of frames on our templates). - bool concreteOpen(Template & input) + + // non-blocking version of getFrame + // Returns a NULL FrameData if too many frames are out, or the + // data source is broken. Sets last_frame to true iff the FrameData + // returned is the last valid frame, and the data source is now broken. + FrameData * tryGetFrame(bool & last_frame) { - close(); + last_frame = false; - bool open_res = false; - // Input has no matrices? Its probably a video that hasn't been loaded yet - if (input.empty()) { - actualSource = new VideoDataSource(0); - open_res = actualSource->concreteOpen(input); + if (is_broken) { + return NULL; + } + + // Try to get a FrameData from the pool, if we can't it means too many + // frames are already out, and we will return NULL to indicate failure + FrameData * aFrame = allFrames.tryGetItem(); + if (aFrame == NULL) + return NULL; + + // Try to actually read a frame, if this returns false the data source is broken + bool res = getNextFrame(*aFrame); + + // The datasource broke, update final_frame + if (!res) + { + QMutexLocker lock(&last_frame_update); + final_frame = lookAhead.back()->sequenceNumber; + allFrames.addItem(aFrame); } - // If the input is not empty, we assume it is a set of frames already - // in memory. else { - actualSource = new TemplateDataSource(0); - open_res = actualSource->concreteOpen(input); + lookAhead.push_back(aFrame); } - // The data source failed to open - if (!open_res) { - delete actualSource; - actualSource = NULL; - return false; + // we will return the first frame on the lookAhead buffer + FrameData * rVal = lookAhead.first(); + lookAhead.pop_front(); + if (rVal->data.empty()) + qDebug("returning empty frame from look ahead!"); + + // If this is the last frame, say so + if (rVal->sequenceNumber == final_frame) { + last_frame = true; + is_broken = true; } - return true; + + return rVal; } - bool isOpen() { return !actualSource ? false : actualSource->isOpen(); } + // Return a frame to the pool, returns true if the frame returned was the last + // frame issued, false otherwise + bool returnFrame(FrameData * inputFrame) + { + int frameNumber = inputFrame->sequenceNumber; -protected: - // Index of the template in the templatelist we are currently reading from - int current_template_idx; + inputFrame->data.clear(); + inputFrame->sequenceNumber = -1; + allFrames.addItem(inputFrame); - TemplateList templates; - DataSource * actualSource; - // Get the next frame, if we run out of frames on the current template - // move on to the next one. - bool getNext(FrameData & output) + bool rval = false; + + QMutexLocker lock(&last_frame_update); + + if (frameNumber == final_frame) { + // We just received the last frame, better pulse + allReturned = true; + lastReturned.wakeAll(); + rval = true; + } + + return rval; + } + + bool waitLast() { - bool res = actualSource->getNext(output); - output.sequenceNumber = next_sequence_number; - - // OK we got a frame - if (res) { - // Override the sequence number set by actualSource - output.data.last().file.set("FrameNumber", output.sequenceNumber); - next_sequence_number++; - if (output.data.last().last().empty()) - qDebug("broken matrix"); - return true; + QMutexLocker lock(&last_frame_update); + + while (!allReturned) + { + // This would be a safer wait if we used a timeout, but + // theoretically that should never matter. + lastReturned.wait(&last_frame_update); } + return true; + } - // We didn't get a frame, try to move on to the next template. - while(!res) { - output.data.clear(); - current_template_idx++; +protected: - // No more templates? We're done - if (current_template_idx >= templates.size()) - return false; + bool openNextTemplate() + { + if (this->current_template_idx >= this->templates.size()) + return false; - // open the next data source - bool open_res = concreteOpen(templates[current_template_idx]); - // We couldn't open it, give up? We could maybe continue here - // but don't currently. - if (!open_res) - return false; + bool open_res = false; + while (!open_res) + { + if (frameSource) + frameSource->close(); - // get a frame from the newly opened data source, if that fails - // we continue to open the next one. - res = actualSource->getNext(output); + if (mode == br::Idiocy::Auto) + { + delete frameSource; + if (this->templates[this->current_template_idx].empty()) + frameSource = new VideoReader(); + else + frameSource = new DirectReturn(); + } + else if (mode == br::Idiocy::DistributeFrames) + { + if (!frameSource) + frameSource = new DirectReturn(); + } + else if (mode == br::Idiocy::StreamVideo) + { + if (!frameSource) + frameSource = new VideoReader(); + } + + open_res = frameSource->open(this->templates[current_template_idx]); + if (!open_res) + { + current_template_idx++; + if (current_template_idx >= this->templates.size()) + return false; + } } - // Finally, set the sequence number for the frame we actually return. - output.sequenceNumber = next_sequence_number++; - output.data.last().file.set("FrameNumber", output.sequenceNumber); + return true; + } - if (output.data.last().last().empty()) - qDebug("broken matrix"); + bool getNextFrame(FrameData & output) + { + bool got_frame = false; - return res; + Template aTemplate; + + while (!got_frame) + { + got_frame = frameSource->getNextTemplate(aTemplate); + + // OK we got a frame + if (got_frame) { + // set the sequence number and tempalte of this frame + output.sequenceNumber = next_sequence_number; + output.data.append(aTemplate); + // set the frame number in the template's metadata + output.data.last().file.set("FrameNumber", output.sequenceNumber); + next_sequence_number++; + return true; + } + + // advance to the next tempalte in our list + this->current_template_idx++; + bool open_res = this->openNextTemplate(); + + // couldn't get the next template? nothing to do, otherwise we try to read + // a frame at the top of this loop. + if (!open_res) { + return false; + } + } + + return false; } + // Index of the template in the templatelist we are currently reading from + int current_template_idx; + + // What do we do to each template + br::Idiocy::StreamModes mode; + + // list of templates we are workign from + TemplateList templates; + + // processor for the current template + TemplateProcessor * frameSource; + + int next_sequence_number; + int final_frame; + bool is_broken; + bool allReturned; + + DoubleBuffer allFrames; + QList lookAhead; + + QWaitCondition lastReturned; + QMutex last_frame_update; }; class ProcessingStage; @@ -710,6 +677,7 @@ public: if (input == NULL) { qFatal("null input to multi-thread stage"); } + input->data >> *transform; should_continue = nextStage->tryAcquireNextStage(input); @@ -861,13 +829,35 @@ public: }; +// Semi-functional, doesn't do anything productive outside of stream::train +class CollectSets : public TimeVaryingTransform +{ + Q_OBJECT +public: + CollectSets() : TimeVaryingTransform(false, false) {} + + QList sets; + + void projectUpdate(const TemplateList &src, TemplateList &dst) + { + (void) dst; + sets.append(src); + } + + void train(const TemplateList & data) + { + (void) data; + } + +}; + // This stage reads new frames from the data source. -class FirstStage : public SingleThreadStage +class ReadStage : public SingleThreadStage { public: - FirstStage(int activeFrames = 100) : SingleThreadStage(true), dataSource(activeFrames){ } + ReadStage(int activeFrames = 100) : SingleThreadStage(true), dataSource(activeFrames){ } - DataSourceManager dataSource; + DataSource dataSource; void reset() { @@ -951,99 +941,17 @@ public: void status(){ qDebug("Read stage %d, status starting? %d, next frame %d buffer size %d", this->stage_id, this->currentStatus == SingleThreadStage::STARTING, this->next_target, this->dataSource.size()); } - - -}; - -// Appened to the end of a Stream's transform sequence. Collects the output -// from each frame on a single templatelist -class LastStage : public SingleThreadStage -{ -public: - LastStage(bool _prev_stage_variance) : SingleThreadStage(_prev_stage_variance) {} - TemplateList getOutput() - { - return collectedOutput; - } - -private: - TemplateList collectedOutput; -public: - - void reset() - { - collectedOutput.clear(); - SingleThreadStage::reset(); - } - - FrameData * run(FrameData * input, bool & should_continue) - { - if (input == NULL) { - qFatal("NULL input to stage %d", this->stage_id); - } - - if (input->sequenceNumber != next_target) - { - qFatal("out of order frames for stage %d, got %d expected %d", this->stage_id, input->sequenceNumber, this->next_target); - } - next_target = input->sequenceNumber + 1; - - // add the item to our output buffer - collectedOutput.append(input->data); - - // Can we enter the read stage? - should_continue = nextStage->tryAcquireNextStage(input); - - // Is there anything on our input buffer? If so we should start a thread - // in this stage to process that frame. - QWriteLocker lock(&statusLock); - FrameData * newItem = inputBuffer->tryGetItem(); - if (!newItem) - { - this->currentStatus = STOPPING; - } - lock.unlock(); - - if (newItem) - startThread(newItem); - - return input; - } - - void status(){ - qDebug("Collection stage %d, status starting? %d, next %d buffer size %d", this->stage_id, this->currentStatus == SingleThreadStage::STARTING, this->next_target, this->inputBuffer->size()); - } - -}; - -// Semi-functional, doesn't do anything productive outside of stream::train -class CollectSets : public TimeVaryingTransform -{ - Q_OBJECT -public: - CollectSets() : TimeVaryingTransform(false, false) {} - - QList sets; - - void projectUpdate(const TemplateList &src, TemplateList &dst) - { - (void) dst; - sets.append(src); - } - - void train(const TemplateList & data) - { - (void) data; - } - }; class DirectStreamTransform : public CompositeTransform { Q_OBJECT public: + Q_PROPERTY(int activeFrames READ get_activeFrames WRITE set_activeFrames RESET reset_activeFrames) + Q_PROPERTY(br::Idiocy::StreamModes readMode READ get_readMode WRITE set_readMode RESET reset_readMode) BR_PROPERTY(int, activeFrames, 100) + BR_PROPERTY(br::Idiocy::StreamModes, readMode, br::Idiocy::Auto) friend class StreamTransfrom; @@ -1109,13 +1017,21 @@ public: qFatal("whatever"); } + + virtual void finalize(TemplateList & output) + { + (void) output; + // Nothing in particular to do here, stream calls finalize + // on all child transforms as part of projectUpdate + } + // start processing, consider all templates in src a continuous // 'video' void projectUpdate(const TemplateList & src, TemplateList & dst) { dst = src; - bool res = readStage->dataSource.open(dst); + bool res = readStage->dataSource.open(dst,readMode); if (!res) { qDebug("stream failed to open %s", qPrintable(dst[0].file.name)); return; @@ -1149,6 +1065,8 @@ public: { TemplateList output_set; transforms[i]->finalize(output_set); + if (output_set.empty()) + continue; for (int j=i+1; j < transforms.size();j++) { @@ -1159,7 +1077,12 @@ public: // dst is set to all output received by the final stage, along // with anything output via the calls to finalize. - dst = collectionStage->getOutput(); + //dst = collectionStage->getOutput(); + foreach(const TemplateList & list, collector->sets) { + dst.append(list); + } + collector->sets.clear(); + dst.append(final_output); foreach(ProcessingStage * stage, processingStages) { @@ -1167,12 +1090,6 @@ public: } } - virtual void finalize(TemplateList & output) - { - (void) output; - // Nothing in particular to do here, stream calls finalize - // on all child transforms as part of projectUpdate - } // Create and link stages void init() @@ -1193,7 +1110,7 @@ public: QMutexLocker poolLock(&poolsAccess); QHash::Iterator it; if (!pools.contains(this->parent())) { - it = pools.insert(this->parent(), new QThreadPool(this)); + it = pools.insert(this->parent(), new QThreadPool(this->parent())); it.value()->setMaxThreadCount(Globals->parallelism); } else it = pools.find(this->parent()); @@ -1202,6 +1119,7 @@ public: // Are our children time varying or not? This decides whether // we run them in single threaded or multi threaded stages + stage_variance.clear(); stage_variance.reserve(transforms.size()); foreach (const br::Transform *transform, transforms) { stage_variance.append(transform->timeVarying()); @@ -1209,7 +1127,7 @@ public: // Additionally, we have a separate stage responsible for reading // frames from the data source - readStage = new FirstStage(activeFrames); + readStage = new ReadStage(activeFrames); processingStages.push_back(readStage); readStage->stage_id = 0; @@ -1244,7 +1162,10 @@ public: // We also have the last stage, which just puts the output of the // previous stages on a template list. - collectionStage = new LastStage(prev_stage_variance); + collectionStage = new SingleThreadStage(prev_stage_variance); + collectionStage->transform = this->collector.data(); + + processingStages.append(collectionStage); collectionStage->stage_id = next_stage_id; collectionStage->stages = &this->processingStages; @@ -1258,6 +1179,11 @@ public: collectionStage->nextStage = readStage; } + DirectStreamTransform() + { + this->collector = QSharedPointer(new CollectSets()); + } + ~DirectStreamTransform() { // Delete all the stages @@ -1270,8 +1196,9 @@ public: protected: QList stage_variance; - FirstStage * readStage; - LastStage * collectionStage; + ReadStage * readStage; + SingleThreadStage * collectionStage; + QSharedPointer collector; QList processingStages; @@ -1311,21 +1238,20 @@ QMutex DirectStreamTransform::poolsAccess; BR_REGISTER(Transform, DirectStreamTransform) -; - -class StreamTransform : public TimeVaryingTransform +class StreamTransform : public WrapperTransform { Q_OBJECT public: - StreamTransform() : TimeVaryingTransform(false) + StreamTransform() : WrapperTransform(false) { } - Q_PROPERTY(br::Transform *transform READ get_transform WRITE set_transform RESET reset_transform STORED false) Q_PROPERTY(int activeFrames READ get_activeFrames WRITE set_activeFrames RESET reset_activeFrames) - BR_PROPERTY(br::Transform *, transform, NULL) + Q_PROPERTY(br::Idiocy::StreamModes readMode READ get_readMode WRITE set_readMode RESET reset_readMode) + BR_PROPERTY(int, activeFrames, 100) + BR_PROPERTY(br::Idiocy::StreamModes, readMode, br::Idiocy::Auto) bool timeVarying() const { return true; } @@ -1366,6 +1292,7 @@ public: basis.setParent(this->parent()); basis.transforms.clear(); basis.activeFrames = this->activeFrames; + basis.readMode = this->readMode; // We need at least a CompositeTransform * to acess transform's children. CompositeTransform * downcast = dynamic_cast (transform); @@ -1448,8 +1375,6 @@ private: BR_REGISTER(Transform, StreamTransform) - - } // namespace br #include "stream.moc"