diff --git a/openbr/core/core.cpp b/openbr/core/core.cpp index 962d5b6..838c1f1 100644 --- a/openbr/core/core.cpp +++ b/openbr/core/core.cpp @@ -29,6 +29,9 @@ struct AlgorithmCore QSharedPointer transform; QSharedPointer distance; + QString transformString; + QString distanceString; + AlgorithmCore(const QString &name) { this->name = name; @@ -134,6 +137,8 @@ struct AlgorithmCore } TemplateList data(TemplateList::fromGallery(input)); + bool multiProcess = Globals->file.getBool("multiProcess", false); + if (gallery.contains("append")) { // Remove any templates which are already in the gallery @@ -155,20 +160,27 @@ struct AlgorithmCore Globals->currentStep = 0; Globals->totalSteps = data.length(); - // Trust me, this makes complete sense. - // We're just going to make a pipe with a placeholder first transform - QString pipeDesc = "Identity+GalleryOutput("+gallery.flat()+")+ProgressCounter("+QString::number(data.length())+")+Discard"; - QScopedPointer basePipe(Transform::make(pipeDesc,NULL)); + QScopedPointer basePipe; - CompositeTransform * downcast = dynamic_cast(basePipe.data()); - if (downcast == NULL) - qFatal("downcast failed?"); + if (!multiProcess) + { + QString pipeDesc = "Identity+GalleryOutput("+gallery.flat()+")+ProgressCounter("+QString::number(data.length())+")+Discard"; + basePipe.reset(Transform::make(pipeDesc,NULL)); + CompositeTransform * downcast = dynamic_cast(basePipe.data()); + if (downcast == NULL) + qFatal("downcast failed?"); - // replace that placeholder with the current algorithm - downcast->transforms[0] = this->transform.data(); + // replace that placeholder with the current algorithm + downcast->transforms[0] = this->transform.data(); - // call init on the pipe to collapse the algorithm (if its top level is a pipe) - downcast->init(); + // call init on the pipe to collapse the algorithm (if its top level is a pipe) + downcast->init(); + } + else + { + QString pipeDesc = "ProcessWrapper("+transformString+")"+"+GalleryOutput("+gallery.flat()+")+ProgressCounter("+QString::number(data.length())+")+Discard"; + basePipe.reset(Transform::make(pipeDesc,NULL)); + } // Next, we make a Stream (with placeholder transform) QString streamDesc = "Stream(Identity, readMode=DistributeFrames)"; @@ -176,7 +188,7 @@ struct AlgorithmCore WrapperTransform * wrapper = dynamic_cast (baseStream.data()); // replace that placeholder with the pipe we built - wrapper->transform = downcast; + wrapper->transform = basePipe.data(); // and get the final stream's stages by reinterpreting the pipe. Perfectly straightforward. wrapper->init(); @@ -196,6 +208,38 @@ struct AlgorithmCore data >> *transform; } + // Read metadata for all templates stored in the specified gallery, return the read + // TeamplateList. If the gallery contains matrices, they are dropped. + void emptyRead(const File & file, TemplateList & templates) + { + // Is this a gallery type containing matrices? + if ((QStringList() << "gal" << "mem" << "template").contains(file.suffix())) { + // Retrieve it block by block, dropping matrices from read templates. + QScopedPointer gallery(Gallery::make(file)); + gallery->set_readBlockSize(10); + bool done = false; + while (!done) + { + TemplateList tList = gallery->readBlock(&done); + for (int i=0; i < tList.size();i++) + { + tList[i].clear(); + templates.append(tList[i]); + } + } + } + else { + // The file may have already been enrolled to a memory gallery + emptyRead(getMemoryGallery(file), templates); + if (!templates.empty()) + return; + + // Nope, just retrieve the metadata + QScopedPointer gallery(Gallery::make(file)); + templates = gallery->read(); + } + } + void retrieveOrEnroll(const File &file, QScopedPointer &gallery, FileList &galleryFiles) { if (!file.getBool("enroll") && (QStringList() << "gal" << "mem" << "template").contains(file.suffix())) { @@ -241,17 +285,14 @@ struct AlgorithmCore dummyTarget.append(targets[0]); QScopedPointer realOutput(Output::make(output, dummyTarget, queryFiles)); - // Some outputs assume Globals->blockSize is a real thing, of course we have no interest in it. - int old_block_size = Globals->blockSize; - Globals->blockSize = INT_MAX; + realOutput->set_blockRows(INT_MAX); + realOutput->set_blockCols(INT_MAX); realOutput->setBlock(0,0); for (int i=0; i < queries.length(); i++) { float res = distance->compare(queries[i], targets[i]); realOutput->setRelative(res, 0,i); } - - Globals->blockSize = old_block_size; } void deduplicate(const File &inputGallery, const File &outputGallery, const float threshold) @@ -310,71 +351,154 @@ struct AlgorithmCore qPrintable(queryGallery.flat()), output.isNull() ? "" : qPrintable(" to " + output.flat())); + bool multiProcess = Globals->file.getBool("multiProcess", false); + if (output.exists() && output.get("cache", false)) return; if (queryGallery == ".") queryGallery = targetGallery; - QScopedPointer t, q; - FileList targetFiles, queryFiles; - retrieveOrEnroll(targetGallery, t, targetFiles); - retrieveOrEnroll(queryGallery, q, queryFiles); + // Read metadata for the target and query sets, the resulting + // TemplateLists do not contain matrices + TemplateList targetMetadata; + TemplateList queryMetadata; - QList partitionSizes; - QList outputFiles; - if (output.contains("split")) { - if (!output.fileName().contains("%1")) qFatal("Output file name missing split number place marker (%%1)"); - partitionSizes = output.getList("split"); - for (int i=0; i outputs; - foreach (const File &outputFile, outputFiles) outputs.append(Output::make(outputFile, targetFiles, queryFiles)); + // Enroll the metadata we read to memory galleries + File targetMetaMem = targetGallery; + targetMetaMem.name = targetMetaMem.baseName() + "_meta.mem"; + File queryMetaMem = queryGallery; + queryMetaMem.name = queryMetaMem.baseName() + "_meta.mem"; - if (distance.isNull()) qFatal("Null distance."); - Globals->currentStep = 0; - Globals->totalSteps = double(targetFiles.size()) * double(queryFiles.size()); - Globals->startTime.start(); + // Store the metadata in memory galleries. + QScopedPointer targetMeta(Gallery::make(targetMetaMem)); + QScopedPointer queryMeta(Gallery::make(queryMetaMem)); - int queryBlock = -1; - bool queryDone = false; - while (!queryDone) { - queryBlock++; - TemplateList queries = q->readBlock(&queryDone); + targetMeta->writeBlock(targetMetadata); + queryMeta->writeBlock(queryMetadata); - QList queryPartitions; - if (!partitionSizes.empty()) queryPartitions = queries.partition(partitionSizes); - else queryPartitions.append(queries); - for (int i=0; i queryMetadata.size(); - TemplateList targets = t->readBlock(&targetDone); + File rowGallery = queryGallery; + File colGallery = targetGallery; + int rowSize = queryMetadata.size(); - QList targetPartitions; - if (!partitionSizes.empty()) targetPartitions = targets.partition(partitionSizes); - else targetPartitions.append(targets); + if (transposeCompare) + { + rowGallery = targetGallery; + colGallery = queryGallery; + rowSize = targetMetadata.size(); + } - outputs[i]->setBlock(queryBlock, targetBlock); + // Do we need to enroll the row set? If so we will do it inline with the comparisons + bool needEnrollRows = false; + if (!(QStringList() << "gal" << "mem" << "template").contains(rowGallery.suffix())) + { + needEnrollRows = true; + } - distance->compare(targetPartitions[i], queryPartitions[i], outputs[i]); + // Do we need to enroll the column set? We want it to be in a memory gallery, unless we + // are in multi-process mode + File colEnrolledGallery = colGallery; + QString targetExtension = multiProcess ? "gal" : "mem"; + if (colGallery.suffix() != targetExtension) + { + if (multiProcess) { + colEnrolledGallery = colGallery.baseName() + colGallery.hash() + ".gal"; + } + else { + colEnrolledGallery = colGallery.baseName() + colGallery.hash() + ".mem"; + } - Globals->currentStep += double(targets.size()) * double(queries.size()); - Globals->printStatus(); - } + // We have to do actual enrollment if the gallery just specified metadata + if (!(QStringList() << "gal" << "template" << "mem").contains(colGallery.suffix())) + { + enroll(colGallery, colEnrolledGallery); + } + // If it did specify templates, but wasn't the write type, we still need to convert + // to the correct gallery type. + else + { + QScopedPointer readColGallery(Gallery::make(colGallery)); + TemplateList templates = readColGallery->read(); + QScopedPointer enrolledColOutput(Gallery::make(colEnrolledGallery)); + enrolledColOutput->writeBlock(templates); } } - qDeleteAll(outputs); + // Describe a GalleryCompare transform, using the data we enrolled + QString compareRegionDesc = "GalleryCompare("+Globals->algorithm + "," + colEnrolledGallery.flat() + ")"; + + QScopedPointer compareRegion; + + // If we need to enroll th row set, add the current transform to the aglorithm + if (needEnrollRows) + { + if (!multiProcess) + { + compareRegionDesc = "Identity+" + compareRegionDesc; + compareRegion.reset(Transform::make(compareRegionDesc,NULL)); + CompositeTransform * downcast = dynamic_cast (compareRegion.data()); + if (downcast == NULL) + qFatal("Pipe downcast failed in compare"); + + downcast->transforms[0] = this->transform.data(); + downcast->init(); + } + else + { + compareRegionDesc = "ProcessWrapper(" + this->transformString + "+" + compareRegionDesc + ")"; + compareRegion.reset(Transform::make(compareRegionDesc, NULL)); + } + } + else { + compareRegion.reset(Transform::make(compareRegionDesc,NULL)); + } - const float speed = 1000 * Globals->totalSteps / Globals->startTime.elapsed() / std::max(1, abs(Globals->parallelism)); - if (!Globals->quiet && (Globals->totalSteps > 1)) fprintf(stderr, "\rSPEED=%.1e \n", speed); - Globals->totalSteps = 0; + compareRegion->init(); + + // We also need to add Output and progress counting to the algorithm we are building + QString joinDesc = "Identity+Identity"; + QScopedPointer join(Transform::make(joinDesc, NULL)); + + // The output transform takes the metadata memGalleries we set up previously as input, along with the + // output specification we were passed + QString outputRegionDesc = "Output("+ output.flat() +"," + targetMetaMem.flat() +"," + queryMetaMem.flat() + ","+ QString::number(transposeCompare ? 1 : 0) + ")"; + outputRegionDesc += "+ProgressCounter("+QString::number(rowSize)+")+Discard"; + QScopedPointer outputTform(Transform::make(outputRegionDesc, NULL)); + + CompositeTransform * downcast = dynamic_cast (join.data()); + downcast->transforms[0] = compareRegion.data(); + downcast->transforms[1] = outputTform.data(); + + // With this, we have set up a transform which (optionally) enrolls templates, compares them + // against a gallery, and outputs them. + join->init(); + + + // Now, we will give that base algorithm to a stream, operating in StreamGallery mode + QString streamDesc = "Stream(Identity, readMode=StreamGallery)"; + QScopedPointer streamBase(Transform::make(streamDesc, NULL)); + WrapperTransform * streamWrapper = dynamic_cast (streamBase.data()); + streamWrapper->transform = join.data(); + + streamWrapper->init(); + + // We set up a template containing the file iwth the row gallery we + // want to compare + TemplateList rowGalleryTemplate; + rowGalleryTemplate.append(Template(rowGallery)); + TemplateList outputGallery; + + // for prgress counting + Globals->currentStep = 0; + Globals->totalSteps = rowSize; + Globals->startTime.start(); + + // Do the actual comparisons + streamWrapper->projectUpdate(rowGalleryTemplate, outputGallery); } private: @@ -407,10 +531,15 @@ private: if ((words.size() < 1) || (words.size() > 2)) qFatal("Invalid algorithm format."); //! [Parsing the algorithm description] + transformString = words[0]; + //! [Creating the template generation and comparison methods] transform = QSharedPointer(Transform::make(words[0], NULL)); - if (words.size() > 1) distance = QSharedPointer(Distance::make(words[1], NULL)); + if (words.size() > 1) { + distance = QSharedPointer(Distance::make(words[1], NULL)); + distanceString = words[1]; + } //! [Creating the template generation and comparison methods] } }; diff --git a/openbr/openbr_plugin.cpp b/openbr/openbr_plugin.cpp index f073913..47791fe 100644 --- a/openbr/openbr_plugin.cpp +++ b/openbr/openbr_plugin.cpp @@ -1101,13 +1101,19 @@ void Output::initialize(const FileList &targetFiles, const FileList &queryFiles) { this->targetFiles = targetFiles; this->queryFiles = queryFiles; + if (this->blockRows == -1) + blockRows = Globals->blockSize; + + if (this->blockCols == -1) + blockCols = Globals->blockSize; + selfSimilar = (queryFiles == targetFiles) && (targetFiles.size() > 1) && (queryFiles.size() > 1); } void Output::setBlock(int rowBlock, int columnBlock) { - offset = QPoint((columnBlock == -1) ? 0 : Globals->blockSize*columnBlock, - (rowBlock == -1) ? 0 : Globals->blockSize*rowBlock); + offset = QPoint((columnBlock == -1) ? 0 : blockCols*columnBlock, + (rowBlock == -1) ? 0 : blockRows*rowBlock); if (!next.isNull()) next->setBlock(rowBlock, columnBlock); } diff --git a/openbr/openbr_plugin.h b/openbr/openbr_plugin.h index 3f84d7e..01de310 100644 --- a/openbr/openbr_plugin.h +++ b/openbr/openbr_plugin.h @@ -996,6 +996,11 @@ class BR_EXPORT Output : public Object Q_OBJECT public: + Q_PROPERTY(int blockRows READ get_blockRows WRITE set_blockRows RESET reset_blockRows STORED false) + Q_PROPERTY(int blockCols READ get_blockCols WRITE set_blockCols RESET reset_blockCols STORED false) + BR_PROPERTY(int, blockRows, -1) + BR_PROPERTY(int, blockCols, -1) + FileList targetFiles; /*!< \brief List of files representing the gallery templates. */ FileList queryFiles; /*!< \brief List of files representing the probe templates. */ bool selfSimilar; /*!< \brief \c true if the \em targetFiles == \em queryFiles, \c false otherwise. */ @@ -1074,9 +1079,11 @@ public: */ class BR_EXPORT Gallery : public Object { - Q_OBJECT - + Q_OBJECT public: + Q_PROPERTY(int readBlockSize READ get_readBlockSize WRITE set_readBlockSize RESET reset_readBlockSize STORED false) + BR_PROPERTY(int, readBlockSize, Globals->blockSize) + virtual ~Gallery() {} TemplateList read(); /*!< \brief Retrieve all the stored templates. */ FileList files(); /*!< \brief Retrieve all the stored template files. */ diff --git a/openbr/plugins/distance.cpp b/openbr/plugins/distance.cpp index b3c9d70..057e218 100644 --- a/openbr/plugins/distance.cpp +++ b/openbr/plugins/distance.cpp @@ -460,12 +460,13 @@ BR_REGISTER(Distance, SumDistance) class GalleryCompareTransform : public Transform { Q_OBJECT - Q_PROPERTY(br::Distance *distance READ get_distance WRITE set_distance RESET reset_distance STORED false) + Q_PROPERTY(QString distanceAlgorithm READ get_distanceAlgorithm WRITE set_distanceAlgorithm RESET reset_distanceAlgorithm STORED false) Q_PROPERTY(QString galleryName READ get_galleryName WRITE set_galleryName RESET reset_galleryName STORED false) - BR_PROPERTY(br::Distance*, distance, NULL) + BR_PROPERTY(QString, distanceAlgorithm, "") BR_PROPERTY(QString, galleryName, "") TemplateList gallery; + QSharedPointer distance; void project(const Template &src, Template &dst) const { @@ -479,8 +480,13 @@ class GalleryCompareTransform : public Transform void init() { - if (!galleryName.isEmpty()) + if (!galleryName.isEmpty()) { gallery = TemplateList::fromGallery(galleryName); + } + if (!distanceAlgorithm.isEmpty()) + { + distance = Distance::fromAlgorithm(distanceAlgorithm); + } } }; diff --git a/openbr/plugins/gallery.cpp b/openbr/plugins/gallery.cpp index fbc4950..a57aae2 100644 --- a/openbr/plugins/gallery.cpp +++ b/openbr/plugins/gallery.cpp @@ -123,7 +123,7 @@ class galGallery : public Gallery gallery.seek(0); TemplateList templates; - while ((templates.size() < Globals->blockSize) && !stream.atEnd()) { + while ((templates.size() < readBlockSize) && !stream.atEnd()) { Template m; stream >> m; templates.append(m); @@ -348,8 +348,8 @@ class memGallery : public Gallery MemoryGalleries::aligned[file] = true; } - TemplateList templates = MemoryGalleries::galleries[file].mid(block*Globals->blockSize, Globals->blockSize); - *done = (templates.size() < Globals->blockSize); + TemplateList templates = MemoryGalleries::galleries[file].mid(block*readBlockSize, readBlockSize); + *done = (templates.size() < readBlockSize); block = *done ? 0 : block+1; return templates; } diff --git a/openbr/plugins/misc.cpp b/openbr/plugins/misc.cpp index 16c8c39..eee78ca 100644 --- a/openbr/plugins/misc.cpp +++ b/openbr/plugins/misc.cpp @@ -552,6 +552,146 @@ public: BR_REGISTER(Transform, ProgressCounterTransform) + +class OutputTransform : public TimeVaryingTransform +{ + Q_OBJECT + + Q_PROPERTY(QString outputString READ get_outputString WRITE set_outputString RESET reset_outputString STORED false) + // names of mem galleries containing filelists we need. + Q_PROPERTY(QString targetName READ get_targetName WRITE set_targetName RESET reset_targetName STORED false) + Q_PROPERTY(QString queryName READ get_queryName WRITE set_queryName RESET reset_queryName STORED false) + Q_PROPERTY(bool transposeMode READ get_transposeMode WRITE set_transposeMode RESET reset_transposeMode STORED false) + + BR_PROPERTY(QString, outputString, "") + BR_PROPERTY(QString, targetName, "") + BR_PROPERTY(QString, queryName, "") + + BR_PROPERTY(bool,transposeMode, false) + ; + + void projectUpdate(const TemplateList &src, TemplateList &dst) + { + dst = src; + + if (src.empty()) + return; + + // we received a template, which is the next row/column in order + foreach(const Template & t, dst) { + for (int i=0; i < t.m().cols; i++) + { + output->setRelative(t.m().at(0, i), currentRow, currentCol); + + // row-major input + if (!transposeMode) + currentCol++; + // col-major input + else + currentRow++; + } + // filled in a row, advance to the next, reset column position + if (!transposeMode) { + currentRow++; + currentCol = 0; + } + // filled in a column, advance, reset row + else { + currentCol++; + currentRow = 0; + } + } + + bool blockDone = false; + // In direct mode, we don't buffer rows + if (!transposeMode) + { + currentBlockRow++; + blockDone = true; + } + // in transpose mode, we buffer 100 cols before writing the block + else if (currentCol == bufferedSize) + { + currentBlockCol++; + blockDone = true; + } + else return; + + if (blockDone) + { + // set the next block, only necessary if we haven't buffered the current item + output->setBlock(currentBlockRow, currentBlockCol); + currentRow = 0; + currentCol = 0; + } + } + + void train(const TemplateList& data) + { + (void) data; + } + + void init() + { + if (targetName.isEmpty() || queryName.isEmpty() || outputString.isEmpty()) + return; + + QScopedPointer tGallery(Gallery::make(targetName)); + QScopedPointer qGallery(Gallery::make(queryName)); + + FileList targetFiles = tGallery->files(); + FileList queryFiles = qGallery->files(); + + currentBlockRow = 0; + currentBlockCol = 0; + + currentRow = 0; + currentCol = 0; + + bufferedSize = 100; + + if (transposeMode) + { + // buffer 100 cols at a time + fragmentsPerRow = bufferedSize; + // a single col contains comparisons to all query files + fragmentsPerCol = queryFiles.size(); + } + else + { + // a single row contains comparisons to all target files + fragmentsPerRow = targetFiles.size(); + // we output rows one at a time + fragmentsPerCol = 1; + } + + output = QSharedPointer(Output::make(outputString, targetFiles, queryFiles)); + output->blockRows = fragmentsPerCol; + output->blockCols = fragmentsPerRow; + output->initialize(targetFiles, queryFiles); + + output->setBlock(currentBlockRow, currentBlockCol); + } + + QSharedPointer output; + + int bufferedSize; + + int currentRow; + int currentCol; + + int currentBlockRow; + int currentBlockCol; + + int fragmentsPerRow; + int fragmentsPerCol; + +public: + OutputTransform() : TimeVaryingTransform(false,false) {} +}; + +BR_REGISTER(Transform, OutputTransform) + } #include "misc.moc" diff --git a/openbr/plugins/output.cpp b/openbr/plugins/output.cpp index b9a290a..b58bf49 100644 --- a/openbr/plugins/output.cpp +++ b/openbr/plugins/output.cpp @@ -215,9 +215,11 @@ class mtxOutput : public Output this->rowBlock = rowBlock; this->columnBlock = columnBlock; - blockScores = cv::Mat(std::min(queryFiles.size()-rowBlock*Globals->blockSize, Globals->blockSize), - std::min(targetFiles.size()-columnBlock*Globals->blockSize, Globals->blockSize), - CV_32FC1); + + int matrixRows = std::min(queryFiles.size()-rowBlock*this->blockRows, blockRows); + int matrixCols = std::min(targetFiles.size()-columnBlock*this->blockCols, blockCols); + + blockScores = cv::Mat(matrixRows, matrixCols, CV_32FC1); } void setRelative(float value, int i, int j) @@ -237,7 +239,7 @@ class mtxOutput : public Output if (!f.open(QFile::ReadWrite)) qFatal("Unable to open %s for modifying.", qPrintable(file)); for (int i=0; iblockSize+i)*targetFiles.size()+(columnBlock*Globals->blockSize))); + f.seek(headerSize + sizeof(float)*(quint64(rowBlock*this->blockRows+i)*targetFiles.size()+(columnBlock*this->blockCols))); f.write((const char*)blockScores.row(i).data, sizeof(float)*blockScores.cols); } f.close(); diff --git a/openbr/plugins/stream.cpp b/openbr/plugins/stream.cpp index 749c44a..6d106e3 100644 --- a/openbr/plugins/stream.cpp +++ b/openbr/plugins/stream.cpp @@ -25,6 +25,7 @@ class Idiocy : public QObject public: enum StreamModes { StreamVideo, DistributeFrames, + StreamGallery, Auto}; Q_ENUMS(StreamModes) @@ -288,6 +289,75 @@ protected: }; +class StreamGallery : public TemplateProcessor +{ +public: + StreamGallery() + { + + } + + bool open(Template &input) + { + // Create a gallery + gallery = QSharedPointer(Gallery::make(input.file)); + // Failed ot open the gallery? + if (gallery.isNull()) { + qDebug()<<"Failed to create gallery!"; + galleryOk = false; + return false; + } + + // Set up state variables for future reads + galleryOk = true; + gallery->set_readBlockSize(100); + nextIdx = 0; + lastBlock = false; + return galleryOk; + } + + bool isOpen() { return galleryOk; } + + void close() + { + galleryOk = false; + currentData.clear(); + nextIdx = 0; + lastBlock = true; + } + + bool getNextTemplate(Template & output) + { + // If we still have data available, we return one of those + if (nextIdx >= currentData.size()) + { + // Otherwise, read another block + if (!lastBlock) { + currentData = gallery->readBlock(&lastBlock); + nextIdx = 0; + } + else + { + galleryOk = false; + return false; + } + } + // Return the indicated template, and advance the index + output = currentData[nextIdx++]; + return true; + } + +protected: + + QSharedPointer gallery; + bool galleryOk; + bool lastBlock; + + TemplateList currentData; + int nextIdx; + +}; + class DirectReturn : public TemplateProcessor { public: @@ -731,6 +801,11 @@ protected: if (!frameSource) frameSource = new DirectReturn(); } + else if (mode == br::Idiocy::StreamGallery) + { + if (!frameSource) + frameSource = new StreamGallery(); + } else if (mode == br::Idiocy::StreamVideo) { if (!frameSource) {