diff --git a/CHANGELOG.md b/CHANGELOG.md index bfdce1f..21efc83 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ 0.4.0 - ??/??/?? ================ +* Added -evalDetection and -plotDetection for evaluating and plotting object detection accuracy (#9) 0.3.0 - 5/22/13 =============== diff --git a/app/br/br.cpp b/app/br/br.cpp index 0258c32..795e6af 100644 --- a/app/br/br.cpp +++ b/app/br/br.cpp @@ -140,6 +140,9 @@ public: } else if (!strcmp(fun, "evalRegression")) { check(parc >= 2 && parc <= 4, "Incorrect parameter count for 'evalRegression'."); br_eval_regression(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? parv[3] : ""); + } else if (!strcmp(fun, "plotDetection")) { + check(parc >= 2, "Incorrect parameter count for 'plotDetection'."); + br_plot_detection(parc-1, parv, parv[parc-1], true); } else if (!strcmp(fun, "plotMetadata")) { check(parc >= 2, "Incorrect parameter count for 'plotMetadata'."); br_plot_metadata(parc-1, parv, parv[parc-1], true); @@ -216,6 +219,7 @@ private: "-evalClustering \n" "-evalDetection [{csv}]\n" "-evalRegression \n" + "-plotDetection ... {destination}\n" "-plotMetadata ... \n" "-getHeader \n" "-setHeader {} \n" diff --git a/openbr/core/bee.cpp b/openbr/core/bee.cpp index 58673d4..9639657 100644 --- a/openbr/core/bee.cpp +++ b/openbr/core/bee.cpp @@ -102,7 +102,7 @@ void BEE::writeSigset(const QString &sigset, const br::FileList &files, bool ign if ((key == "Index") || (key == "Label")) continue; metadata.append(key+"=\""+QtUtils::toString(file.value(key))+"\""); } - lines.append("\t("Label") +"\">"); + lines.append("\t("Label",file.fileName()) +"\">"); lines.append("\t\t"); lines.append("\t"); } diff --git a/openbr/core/eval.cpp b/openbr/core/eval.cpp index 180cc8a..9ad79b7 100644 --- a/openbr/core/eval.cpp +++ b/openbr/core/eval.cpp @@ -387,7 +387,7 @@ static QStringList computeDetectionResults(const QList &detec for (int i=0; i 1 ? QString(" + scale_linetype_discrete(\"%1\")").arg(p.minor.header) : QString()) + QString(" + scale_x_log10(labels=percent) + scale_y_log10(labels=percent) + annotation_logticks()\n\n"))); - p.file.write(qPrintable(QString("qplot(X, data=SD, geom=\"histogram\", fill=Y, position=\"identity\", alpha=I(1/2)") + QString(", xlab=\"Score%1\"").arg((p.flip ? p.major.size : p.minor.size) > 1 ? " / " + (p.flip ? p.major.header : p.minor.header) : QString()) + QString(", ylab=\"Frequency%1\"").arg((p.flip ? p.minor.size : p.major.size) > 1 ? " / " + (p.flip ? p.minor.header : p.major.header) : QString()) + @@ -272,6 +271,48 @@ bool Plot(const QStringList &files, const br::File &destination, bool show) return p.finalize(show); } +bool PlotDetection(const QStringList &files, const File &destination, bool show) +{ + qDebug("Plotting %d detection file(s) to %s", files.size(), qPrintable(destination)); + + RPlot p(files, destination, false); + + p.file.write("# Split data into individual plots\n" + "plot_index = which(names(data)==\"Plot\")\n" + "DiscreteROC <- data[grep(\"DiscreteROC\",data$Plot),-c(1)]\n" + "ContinuousROC <- data[grep(\"ContinuousROC\",data$Plot),-c(1)]\n" + "DiscretePR <- data[grep(\"DiscretePR\",data$Plot),-c(1)]\n" + "ContinuousPR <- data[grep(\"ContinuousPR\",data$Plot),-c(1)]\n" + "Overlap <- data[grep(\"Overlap\",data$Plot),-c(1)]\n" + "rm(data)\n" + "\n"); + + foreach (const QString &type, QStringList() << "Discrete" << "Continuous") + p.file.write(qPrintable(QString("qplot(X, Y, data=%1ROC%2").arg(type, (p.major.smooth || p.minor.smooth) ? ", geom=\"smooth\", method=loess, level=0.99" : ", geom=\"line\"") + + (p.major.size > 1 ? QString(", colour=factor(%1)").arg(p.major.header) : QString()) + + (p.minor.size > 1 ? QString(", linetype=factor(%1)").arg(p.minor.header) : QString()) + + QString(", xlab=\"False Accepts\", ylab=\"True Accept Rate\") + theme_minimal()") + + (p.major.size > 1 ? getScale("colour", p.major.header, p.major.size) : QString()) + + (p.minor.size > 1 ? QString(" + scale_linetype_discrete(\"%1\")").arg(p.minor.header) : QString()) + + QString(" + scale_x_log10() + scale_y_continuous(labels=percent) + annotation_logticks(sides=\"b\") + ggtitle(\"%1\")\n\n").arg(type))); + + foreach (const QString &type, QStringList() << "Discrete" << "Continuous") + p.file.write(qPrintable(QString("qplot(X, Y, data=%1PR%2").arg(type, (p.major.smooth || p.minor.smooth) ? ", geom=\"smooth\", method=loess, level=0.99" : ", geom=\"line\"") + + (p.major.size > 1 ? QString(", colour=factor(%1)").arg(p.major.header) : QString()) + + (p.minor.size > 1 ? QString(", linetype=factor(%1)").arg(p.minor.header) : QString()) + + QString(", xlab=\"Recall\", ylab=\"Precision\") + theme_minimal()") + + (p.major.size > 1 ? getScale("colour", p.major.header, p.major.size) : QString()) + + (p.minor.size > 1 ? QString(" + scale_linetype_discrete(\"%1\")").arg(p.minor.header) : QString()) + + QString(" + scale_x_continuous(labels=percent) + scale_y_continuous(labels=percent) + ggtitle(\"%1\")\n\n").arg(type))); + + p.file.write(qPrintable(QString("qplot(X, data=Overlap, geom=\"histogram\", position=\"identity\", xlab=\"Overlap\", ylab=\"Frequency\")") + + QString(" + theme_minimal() + scale_x_continuous(minor_breaks=NULL) + scale_y_continuous(minor_breaks=NULL) + theme(axis.text.y=element_blank(), axis.ticks=element_blank(), axis.text.x=element_text(angle=-90, hjust=0))") + + (p.major.size > 1 ? (p.minor.size > 1 ? QString(" + facet_grid(%2 ~ %1, scales=\"free\")").arg(p.minor.header, p.major.header) : QString(" + facet_wrap(~ %1, scales = \"free\")").arg(p.major.header)) : QString()) + + QString(" + theme(aspect.ratio=1)\n\n"))); + + return p.finalize(show); +} + bool PlotMetadata(const QStringList &files, const QString &columns, bool show) { qDebug("Plotting %d metadata file(s) for columns %s", files.size(), qPrintable(columns)); diff --git a/openbr/core/plot.h b/openbr/core/plot.h index b015b24..7aa364c 100644 --- a/openbr/core/plot.h +++ b/openbr/core/plot.h @@ -24,7 +24,8 @@ namespace br { - bool Plot(const QStringList &files, const br::File &destination, bool show = false); + bool Plot(const QStringList &files, const File &destination, bool show = false); + bool PlotDetection(const QStringList &files, const File &destination, bool show = false); bool PlotMetadata(const QStringList &files, const QString &destination, bool show = false); } diff --git a/openbr/core/resource.h b/openbr/core/resource.h index ae18d36..a620969 100644 --- a/openbr/core/resource.h +++ b/openbr/core/resource.h @@ -24,6 +24,7 @@ #include #include #include +#include template class ResourceMaker @@ -52,7 +53,7 @@ public: : resourceMaker(rm) , availableResources(new QList()) , lock(new QMutex()) - , totalResources(new QSemaphore(QThread::idealThreadCount())) + , totalResources(new QSemaphore(br::Globals->parallelism)) {} ~Resource() diff --git a/openbr/openbr.cpp b/openbr/openbr.cpp index 0f27041..09ca043 100644 --- a/openbr/openbr.cpp +++ b/openbr/openbr.cpp @@ -172,6 +172,11 @@ bool br_plot(int num_files, const char *files[], const char *destination, bool s return Plot(QtUtils::toStringList(num_files, files), destination, show); } +bool br_plot_detection(int num_files, const char *files[], const char *destination, bool show) +{ + return PlotDetection(QtUtils::toStringList(num_files, files), destination, show); +} + bool br_plot_metadata(int num_files, const char *files[], const char *columns, bool show) { return PlotMetadata(QtUtils::toStringList(num_files, files), columns, show); diff --git a/openbr/openbr.h b/openbr/openbr.h index f4e84f0..6a26a77 100644 --- a/openbr/openbr.h +++ b/openbr/openbr.h @@ -240,7 +240,7 @@ BR_EXPORT const char *br_most_recent_message(); BR_EXPORT const char *br_objects(const char *abstractions = ".*", const char *implementations = ".*", bool parameters = true); /*! - * \brief Renders performance figures for a set of .csv files. + * \brief Renders recognition performance figures for a set of .csv files created by \ref br_eval. * * In order of their output, the figures are: * -# Metadata table @@ -262,11 +262,29 @@ BR_EXPORT const char *br_objects(const char *abstractions = ".*", const char *im * \return Returns \c true on success. Returns false on a failure to compile the figures due to a missing, out of date, or incomplete \c R installation. * \note This function requires a current R installation with the following packages: * \code install.packages(c("ggplot2", "gplots", "reshape", "scales")) \endcode - * \see br_plot_metadata + * \see br_eval */ BR_EXPORT bool br_plot(int num_files, const char *files[], const char *destination, bool show = false); /*! + * \brief Renders detection performance figures for a set of .csv files created by \ref br_eval_detection. + * + * In order of their output, the figures are: + * -# Discrete Receiver Operating Characteristic (DiscreteROC) + * -# Continuous Receiver Operating Characteristic (ContinuousROC) + * -# Discrete Precision Recall (DiscretePR) + * -# Continuous Precision Recall (ContinuousPR) + * -# Bounding Box Overlap Histogram (Overlap) + * + * Detection accuracy is measured with overlap fraction = bounding box intersection / union. + * When computing discrete curves, an overlap >= 0.5 is considered a true positive, otherwise it is considered a false negative. + * When computing continuous curves, true positives and false negatives are measured fractionally as overlap and 1-overlap respectively. + * + * \see br_plot + */ +BR_EXPORT bool br_plot_detection(int num_files, const char *files[], const char *destination, bool show = false); + +/*! * \brief Renders metadata figures for a set of .csv files with specified columns. * * Several files will be created: diff --git a/openbr/openbr_plugin.cpp b/openbr/openbr_plugin.cpp index f3c30e6..a27b0f9 100644 --- a/openbr/openbr_plugin.cpp +++ b/openbr/openbr_plugin.cpp @@ -813,7 +813,7 @@ float br::Context::progress() const void br::Context::setProperty(const QString &key, const QString &value) { - Object::setProperty(key, value); + Object::setProperty(key, value.isEmpty() ? QVariant() : value); qDebug("Set %s%s", qPrintable(key), value.isEmpty() ? "" : qPrintable(" to " + value)); if (key == "parallelism") { @@ -1163,7 +1163,8 @@ void Transform::project(const TemplateList &src, TemplateList &dst) const dst.append(Template()); QFutureSynchronizer futures; for (int i=0; iparallelism > 1) futures.addFuture(QtConcurrent::run(_project, this, &src[i], &dst[i])); + else _project(this, &src[i], &dst[i]); futures.waitForFinished(); } diff --git a/openbr/plugins/cascade.cpp b/openbr/plugins/cascade.cpp index 0c00523..f54e240 100644 --- a/openbr/plugins/cascade.cpp +++ b/openbr/plugins/cascade.cpp @@ -49,19 +49,20 @@ private: } }; - /*! * \ingroup transforms * \brief Wraps OpenCV cascade classifier * \author Josh Klontz \cite jklontz */ -class CascadeTransform : public UntrainableTransform +class CascadeTransform : public UntrainableMetaTransform { Q_OBJECT Q_PROPERTY(QString model READ get_model WRITE set_model RESET reset_model STORED false) Q_PROPERTY(int minSize READ get_minSize WRITE set_minSize RESET reset_minSize STORED false) + Q_PROPERTY(bool ROCMode READ get_ROCMode WRITE set_ROCMode RESET reset_ROCMode STORED false) BR_PROPERTY(QString, model, "FrontalFace") BR_PROPERTY(int, minSize, 64) + BR_PROPERTY(bool, ROCMode, false) Resource cascadeResource; @@ -72,18 +73,54 @@ class CascadeTransform : public UntrainableTransform void project(const Template &src, Template &dst) const { + TemplateList temp; + project(TemplateList() << src, temp); + if (!temp.isEmpty()) dst = temp.first(); + } + + void project(const TemplateList &src, TemplateList &dst) const + { CascadeClassifier *cascade = cascadeResource.acquire(); - vector rects; - cascade->detectMultiScale(src, rects, 1.2, 5, src.file.get("enrollAll", false) ? 0 : CV_HAAR_FIND_BIGGEST_OBJECT, Size(minSize, minSize)); + foreach (const Template &t, src) { + const bool enrollAll = t.file.getBool("enrollAll"); + + for (int i=0; i rects; + vector rejectLevels; + vector levelWeights; + if (ROCMode) cascade->detectMultiScale(m, rects, rejectLevels, levelWeights, 1.2, 5, (enrollAll ? 0 : CV_HAAR_FIND_BIGGEST_OBJECT) | CV_HAAR_SCALE_IMAGE, Size(minSize, minSize), Size(), true); + else cascade->detectMultiScale(m, rects, 1.2, 5, enrollAll ? 0 : CV_HAAR_FIND_BIGGEST_OBJECT, Size(minSize, minSize)); + + if (!enrollAll && rects.empty()) + rects.push_back(Rect(0, 0, m.cols, m.rows)); + + for (size_t j=0; j j) + u.file.set("Confidence", rejectLevels[j]*1000.0 + levelWeights[j]*1.0); + const QRectF rect = OpenCVUtils::fromRect(rects[j]); + u.file.appendRect(rect); + u.file.set(model, rect); + dst.append(u); + } + } + } + cascadeResource.release(cascade); + } - if (!src.file.get("enrollAll", false) && rects.empty()) - rects.push_back(Rect(0, 0, src.m().cols, src.m().rows)); + // TODO: Remove this code when ready to break binary compatibility + void store(QDataStream &stream) const + { + int size = 1; + stream << size; + } - foreach (const Rect &rect, rects) { - dst += src; - dst.file.appendRect(OpenCVUtils::fromRect(rect)); - } + void load(QDataStream &stream) + { + int size; + stream >> size; } }; diff --git a/openbr/plugins/eyes.cpp b/openbr/plugins/eyes.cpp index e8e2ae9..cdef8df 100644 --- a/openbr/plugins/eyes.cpp +++ b/openbr/plugins/eyes.cpp @@ -186,7 +186,6 @@ private: dst.file.appendPoint(QPointF(second_eye_x, second_eye_y)); dst.file.set("First_Eye", QPointF(first_eye_x, first_eye_y)); dst.file.set("Second_Eye", QPointF(second_eye_x, second_eye_y)); - dst.file.set("Face", QRect(roi.x, roi.y, roi.width, roi.height)); } }; diff --git a/openbr/plugins/frames.cpp b/openbr/plugins/frames.cpp new file mode 100644 index 0000000..74aa978 --- /dev/null +++ b/openbr/plugins/frames.cpp @@ -0,0 +1,53 @@ +#include "openbr_internal.h" + +namespace br +{ + +/*! + * \ingroup transforms + * \brief Passes along n sequential frames to the next transform. + * \author Josh Klontz \cite jklontz + * + * For a video with m frames, AggregateFrames would create a total of m-n+1 sequences ([0,n] ... [m-n+1, m]). + */ +class AggregateFrames : public TimeVaryingTransform +{ + Q_OBJECT + Q_PROPERTY(int n READ get_n WRITE set_n RESET reset_n STORED false) + BR_PROPERTY(int, n, 1) + + TemplateList buffer; + +public: + AggregateFrames() : TimeVaryingTransform(false) {} + +private: + void train(const TemplateList &data) + { + (void) data; + } + + void projectUpdate(const Template &src, Template &dst) + { + buffer.append(src); + if (buffer.size() < n) return; + foreach (const Template &t, buffer) dst.append(t); + dst.file = buffer.takeFirst().file; + } + + void store(QDataStream &stream) const + { + (void) stream; + } + + void load(QDataStream &stream) + { + (void) stream; + } +}; + +BR_REGISTER(Transform, AggregateFrames) + +} // namespace br + +#include "frames.moc" diff --git a/openbr/plugins/gallery.cpp b/openbr/plugins/gallery.cpp index 6f29cb7..85a307b 100644 --- a/openbr/plugins/gallery.cpp +++ b/openbr/plugins/gallery.cpp @@ -165,10 +165,13 @@ class EmptyGallery : public Gallery templates.append(File(fileName, dir.dirName())); if (!regexp.isEmpty()) { - const QRegularExpression re(regexp); - for (int i=templates.size()-1; i>=0; i--) - if (!re.match(templates[i].file.suffix()).hasMatch()) + QRegExp re(regexp); + re.setPatternSyntax(QRegExp::Wildcard); + for (int i=templates.size()-1; i>=0; i--) { + if (!re.exactMatch(templates[i].file.fileName())) { templates.removeAt(i); + } + } } return templates; @@ -269,67 +272,6 @@ class matrixGallery : public Gallery BR_REGISTER(Gallery, matrixGallery) /*! - * \ingroup galleries - * \brief Treat a video as a gallery, making a single template from each frame - * \author Charles Otto \cite caotto - */ -class aviGallery : public Gallery -{ - Q_OBJECT - - TemplateList output_set; - QScopedPointer videoOut; - - ~aviGallery() - { - if (videoOut && videoOut->isOpened()) videoOut->release(); - } - - TemplateList readBlock(bool * done) - { - std::string fname = file.name.toStdString(); - *done = true; - - TemplateList output; - if (!file.exists()) - return output; - - cv::VideoCapture videoReader(file.name.toStdString()); - - bool open = videoReader.isOpened(); - - while (open) { - cv::Mat frame; - - open = videoReader.read(frame); - if (!open) break; - output.append(Template()); - output.back() = frame.clone(); - } - - return TemplateList(); - } - - void write(const Template & t) - { - if (videoOut.isNull() || !videoOut->isOpened()) { - int fourcc = OpenCVUtils::getFourcc(); - videoOut.reset(new cv::VideoWriter(qPrintable(file.name), fourcc, 30, t.m().size())); - } - - if (!videoOut->isOpened()) { - qWarning("Failed to open %s for writing\n", qPrintable(file.name)); - return; - } - - foreach(const cv::Mat & m, t) { - videoOut->write(m); - } - } -}; -BR_REGISTER(Gallery, aviGallery) - -/*! * \ingroup initializers * \brief Initialization support for memGallery. * \author Josh Klontz \cite jklontz diff --git a/openbr/plugins/gui.cpp b/openbr/plugins/gui.cpp index 11df368..e7357c9 100644 --- a/openbr/plugins/gui.cpp +++ b/openbr/plugins/gui.cpp @@ -212,8 +212,11 @@ public: foreach (const Template & t, src) { // build label QString newTitle; + foreach (const QString & s, keys) { - if (t.file.contains(s)) { + if (s.compare("name", Qt::CaseInsensitive) == 0) { + newTitle = newTitle + s + ": " + t.file.fileName() + " "; + } else if (t.file.contains(s)) { QString out = t.file.get(s); newTitle = newTitle + s + ": " + out + " "; } diff --git a/openbr/plugins/independent.cpp b/openbr/plugins/independent.cpp index 637f1fe..4a1ec03 100644 --- a/openbr/plugins/independent.cpp +++ b/openbr/plugins/independent.cpp @@ -129,6 +129,8 @@ class IndependentTransform : public MetaTransform return independentTransform; } + bool timeVarying() const { return transform->timeVarying(); } + static void _train(Transform *transform, const TemplateList *data) { transform->train(*data); @@ -170,6 +172,27 @@ class IndependentTransform : public MetaTransform dst.append(mats); } + void projectUpdate(const Template &src, Template &dst) + { + dst.file = src.file; + QList mats; + for (int i=0; iprojectUpdate(Template(src.file, src[i]), dst); + mats.append(dst); + dst.clear(); + } + dst.append(mats); + } + + void projectUpdate(const TemplateList &src, TemplateList &dst) + { + dst.reserve(src.size()); + foreach (const Template &t, src) { + dst.append(Template()); + projectUpdate(t, dst.last()); + } + } + void store(QDataStream &stream) const { const int size = transforms.size(); diff --git a/openbr/plugins/landmarks.cpp b/openbr/plugins/landmarks.cpp index 4f25593..9f06f7f 100644 --- a/openbr/plugins/landmarks.cpp +++ b/openbr/plugins/landmarks.cpp @@ -230,6 +230,8 @@ class DelaunayTransform : public UntrainableTransform QList mappedPoints; + dst.file = src.file; + for (int i = 0; i < validTriangles.size(); i++) { Eigen::MatrixXf srcMat(validTriangles[i].size(), 2); @@ -272,8 +274,8 @@ class DelaunayTransform : public UntrainableTransform bitwise_and(dst.m(),mask,overlap); for (int j = 0; j < overlap.rows; j++) { for (int k = 0; k < overlap.cols; k++) { - if (overlap.at(k,j) != 0) { - mask.at(k,j) = 0; + if (overlap.at(j,k) != 0) { + mask.at(j,k) = 0; } } } @@ -281,11 +283,13 @@ class DelaunayTransform : public UntrainableTransform bitwise_and(buffer,mask,output); + dst.m() += output; } + // Overwrite any rects Rect boundingBox = boundingRect(mappedPoints.toVector().toStdVector()); - dst.file.appendRect(OpenCVUtils::fromRect(boundingBox)); + dst.file.setRects(QList() << OpenCVUtils::fromRect(boundingBox)); } } diff --git a/openbr/plugins/meta.cpp b/openbr/plugins/meta.cpp index f00676c..7f827dc 100644 --- a/openbr/plugins/meta.cpp +++ b/openbr/plugins/meta.cpp @@ -645,17 +645,28 @@ public: QList input_buffer; input_buffer.reserve(src.size()); + QFutureSynchronizer futures; + for (int i =0; i < src.size();i++) { input_buffer.append(TemplateList()); output_buffer.append(TemplateList()); } - - QFutureSynchronizer futures; + QList > temp; + temp.reserve(src.size()); for (int i=0; iparallelism) futures.addFuture(QtConcurrent::run(_projectList, transform, &input_buffer[i], &output_buffer[i])); - else _projectList( transform, &input_buffer[i], &output_buffer[i]); + + if (Globals->parallelism > 1) temp.append(QtConcurrent::run(_projectList, transform, &input_buffer[i], &output_buffer[i])); + else _projectList(transform, &input_buffer[i], &output_buffer[i]); + } + // We add the futures in reverse order, since in Qt 5.1 at least the + // waiting thread will wait on them in the order added (which for uniform priority + // threads is the order of execution), and we want the waiting thread to go in the opposite order + // so that it can steal runnables and do something besides wait. + for (int i = temp.size() - 1; i > 0; i--) { + futures.addFuture(temp[i]); } + futures.waitForFinished(); for (int i=0; i Pair; int rank = 1; foreach (const Pair &pair, Common::Sort(OpenCVUtils::matrixToVector(data.row(i)), true)) { - if (targetFiles[pair.second].get("Label") == queryFiles[i].get("Label")) { - ranks.append(rank); - positions.append(pair.second); - scores.append(pair.first); - break; + if (Globals->crossValidate > 0 ? (targetFiles[pair.second].get("Partition",-1) == queryFiles[i].get("Partition",-1)) : true) { + if (QString(targetFiles[pair.second]) != QString(queryFiles[i])) { + if (targetFiles[pair.second].get("Label") == queryFiles[i].get("Label")) { + ranks.append(rank); + positions.append(pair.second); + scores.append(pair.first); + break; + } + rank++; + } } - rank++; } } typedef QPair RankPair; foreach (const RankPair &pair, Common::Sort(ranks, false)) + // pair.first == rank retrieved, pair.second == original position lines.append(queryFiles[pair.second].name + " " + QString::number(pair.first) + " " + QString::number(scores[pair.second]) + " " + targetFiles[positions[pair.second]].name); + QtUtils::writeFile(file, lines); } }; diff --git a/openbr/plugins/regions.cpp b/openbr/plugins/regions.cpp index 57cbf6f..4197b95 100644 --- a/openbr/plugins/regions.cpp +++ b/openbr/plugins/regions.cpp @@ -252,16 +252,12 @@ class CropRectTransform : public UntrainableTransform void project(const Template &src, Template &dst) const { dst = src; - QList rects = dst.file.rects(); + QList rects = src.file.rects(); for (int i=0;i < rects.size(); i++) { - QRectF rect = rects[i]; - - rect.setX(rect.x() + rect.width() * QtUtils::toPoint(widthCrop).x()); - rect.setY(rect.y() + rect.height() * QtUtils::toPoint(heightCrop).x()); - rect.setWidth(rect.width() * (1-QtUtils::toPoint(widthCrop).y())); - rect.setHeight(rect.height() * (1-QtUtils::toPoint(heightCrop).y())); - - dst.m() = Mat(dst.m(), OpenCVUtils::toRect(rect)); + rects[i].setX(rects[i].x() + rects[i].width() * QtUtils::toPoint(widthCrop).x()); + rects[i].setY(rects[i].y() + rects[i].height() * QtUtils::toPoint(heightCrop).x()); + rects[i].setWidth(rects[i].width() * (1-QtUtils::toPoint(widthCrop).y())); + rects[i].setHeight(rects[i].height() * (1-QtUtils::toPoint(heightCrop).y())); } dst.file.setRects(rects); } diff --git a/openbr/plugins/stream.cpp b/openbr/plugins/stream.cpp index 5470f51..e02092c 100644 --- a/openbr/plugins/stream.cpp +++ b/openbr/plugins/stream.cpp @@ -160,9 +160,8 @@ class DataSource public: DataSource(int maxFrames=500) { + // The sequence number of the last frame final_frame = -1; - last_issued = -2; - last_received = -3; for (int i=0; i < maxFrames;i++) { allFrames.addItem(new FrameData()); @@ -181,51 +180,64 @@ public: } // non-blocking version of getFrame - FrameData * tryGetFrame() + // Returns a NULL FrameData if too many frames are out, or the + // data source is broken. Sets last_frame to true iff the FrameData + // returned is the last valid frame, and the data source is now broken. + FrameData * tryGetFrame(bool & last_frame) { + last_frame = false; + + if (is_broken) { + return NULL; + } + + // Try to get a FrameData from the pool, if we can't it means too many + // frames are already out, and we will return NULL to indicate failure FrameData * aFrame = allFrames.tryGetItem(); - if (aFrame == NULL) + if (aFrame == NULL) { return NULL; + } aFrame->data.clear(); aFrame->sequenceNumber = -1; + // Try to read a frame, if this returns false the data source is broken bool res = getNext(*aFrame); - // The datasource broke. - if (!res) { + // The datasource broke, update final_frame + if (!res) + { + QMutexLocker lock(&last_frame_update); + final_frame = lookAhead.back()->sequenceNumber; allFrames.addItem(aFrame); + } + else lookAhead.push_back(aFrame); - QMutexLocker lock(&last_frame_update); - // Did we already receive the last frame? - final_frame = last_issued; + FrameData * rVal = lookAhead.first(); + lookAhead.pop_front(); - // We got the last frame before the data source broke, - // better pulse lastReturned - if (final_frame == last_received) { - lastReturned.wakeAll(); - } - else if (final_frame < last_received) - std::cout << "Bad last frame " << final_frame << " but received " << last_received << std::endl; - return NULL; + if (rVal->sequenceNumber == final_frame) { + last_frame = true; + is_broken = true; } - last_issued = aFrame->sequenceNumber; - return aFrame; + + return rVal; } - // Returns true if the frame returned was the last + // Return a frame to the pool, returns true if the frame returned was the last // frame issued, false otherwise bool returnFrame(FrameData * inputFrame) { + int frameNumber = inputFrame->sequenceNumber; + allFrames.addItem(inputFrame); bool rval = false; QMutexLocker lock(&last_frame_update); - last_received = inputFrame->sequenceNumber; - if (inputFrame->sequenceNumber == final_frame) { + if (frameNumber == final_frame) { // We just received the last frame, better pulse lastReturned.wakeAll(); rval = true; @@ -240,17 +252,57 @@ public: lastReturned.wait(&last_frame_update); } - virtual void close() = 0; - virtual bool open(Template & output, int start_index=0) = 0; - virtual bool isOpen() = 0; + bool open(Template & output, int start_index = 0) + { + is_broken = false; + // The last frame isn't initialized yet + final_frame = -1; + // Start our sequence numbers from the input index + next_sequence_number = start_index; + + // Actually open the data source + bool open_res = concreteOpen(output); + + // We couldn't open the data source + if (!open_res) { + is_broken = true; + return false; + } + + // Try to get a frame from the global pool + FrameData * firstFrame = allFrames.tryGetItem(); + + // If this fails, things have gone pretty badly. + if (firstFrame == NULL) { + is_broken = true; + return false; + } + + // Read a frame from the video source + bool res = getNext(*firstFrame); + + // the data source broke already, we couldn't even get one frame + // from it. + if (!res) { + is_broken = true; + return false; + } + lookAhead.append(firstFrame); + return true; + } + + virtual bool isOpen()=0; + virtual bool concreteOpen(Template & output) = 0; virtual bool getNext(FrameData & input) = 0; + virtual void close() = 0; + int next_sequence_number; protected: DoubleBuffer allFrames; int final_frame; - int last_issued; - int last_received; + bool is_broken; + QList lookAhead; QWaitCondition lastReturned; QMutex last_frame_update; @@ -262,13 +314,8 @@ class VideoDataSource : public DataSource public: VideoDataSource(int maxFrames) : DataSource(maxFrames) {} - bool open(Template &input, int start_index=0) + bool concreteOpen(Template &input) { - final_frame = -1; - last_issued = -2; - last_received = -3; - - next_idx = start_index; basis = input; bool is_int = false; int anInt = input.file.name.toInt(&is_int); @@ -284,8 +331,11 @@ public: { qDebug("Video not open!"); } + } else { + // Yes, we should specify absolute path: + // http://stackoverflow.com/questions/9396459/loading-a-video-in-opencv-in-python + video.open(QFileInfo(input.file.name).absoluteFilePath().toStdString()); } - else video.open(input.file.name.toStdString()); return video.isOpened(); } @@ -303,25 +353,31 @@ private: return false; output.data.append(Template(basis.file)); - output.data.last().append(cv::Mat()); + output.data.last().m() = cv::Mat(); - output.sequenceNumber = next_idx; - next_idx++; + output.sequenceNumber = next_sequence_number; + next_sequence_number++; - bool res = video.read(output.data.last().last()); - output.data.last().last() = output.data.last().last().clone(); + cv::Mat temp; + bool res = video.read(temp); if (!res) { + output.data.last().m() = cv::Mat(); close(); return false; } + + // This clone is critical, if we don't do it then the matrix will + // be an alias of an internal buffer of the video source, leading + // to various problems later. + output.data.last().m() = temp.clone(); + output.data.last().file.set("FrameNumber", output.sequenceNumber); return true; } cv::VideoCapture video; Template basis; - int next_idx; }; // Given a template as input, return its matrices one by one on subsequent calls @@ -331,21 +387,16 @@ class TemplateDataSource : public DataSource public: TemplateDataSource(int maxFrames) : DataSource(maxFrames) { - current_idx = INT_MAX; + current_matrix_idx = INT_MAX; data_ok = false; } - bool data_ok; - bool open(Template &input, int start_index=0) + bool concreteOpen(Template &input) { basis = input; - current_idx = 0; - next_sequence = start_index; - final_frame = -1; - last_issued = -2; - last_received = -3; + current_matrix_idx = 0; - data_ok = current_idx < basis.size(); + data_ok = current_matrix_idx < basis.size(); return data_ok; } @@ -355,39 +406,41 @@ public: void close() { - current_idx = INT_MAX; + current_matrix_idx = INT_MAX; basis.clear(); } private: bool getNext(FrameData & output) { - data_ok = current_idx < basis.size(); + data_ok = current_matrix_idx < basis.size(); if (!data_ok) return false; - output.data.append(basis[current_idx]); - current_idx++; + output.data.append(basis[current_matrix_idx]); + current_matrix_idx++; - output.sequenceNumber = next_sequence; - next_sequence++; + output.sequenceNumber = next_sequence_number; + next_sequence_number++; output.data.last().file.set("FrameNumber", output.sequenceNumber); return true; } Template basis; - int current_idx; - int next_sequence; + // Index of the next matrix to output from the template + int current_matrix_idx; + + // is current_matrix_idx in bounds? + bool data_ok; }; -// Given a template as input, create a VideoDataSource or a TemplateDataSource -// depending on whether or not it looks like the input template has already -// loaded frames into memory. +// Given a templatelist as input, create appropriate data source for each +// individual template class DataSourceManager : public DataSource { public: - DataSourceManager() + DataSourceManager() : DataSource(500) { actualSource = NULL; } @@ -408,29 +461,25 @@ public: bool open(TemplateList & input) { - currentIdx = 0; + current_template_idx = 0; templates = input; - return open(templates[currentIdx]); + return DataSource::open(templates[current_template_idx]); } - bool open(Template & input, int start_index=0) + bool concreteOpen(Template & input) { close(); - final_frame = -1; - last_issued = -2; - last_received = -3; - next_frame = start_index; // Input has no matrices? Its probably a video that hasn't been loaded yet if (input.empty()) { actualSource = new VideoDataSource(0); - actualSource->open(input, next_frame); + actualSource->concreteOpen(input); } else { // create frame dealer actualSource = new TemplateDataSource(0); - actualSource->open(input, next_frame); + actualSource->concreteOpen(input); } if (!isOpen()) { delete actualSource; @@ -443,30 +492,47 @@ public: bool isOpen() { return !actualSource ? false : actualSource->isOpen(); } protected: - int currentIdx; - int next_frame; + // Index of the template in the templatelist we are currently reading from + int current_template_idx; + TemplateList templates; DataSource * actualSource; bool getNext(FrameData & output) { bool res = actualSource->getNext(output); + output.sequenceNumber = next_sequence_number; + if (res) { - next_frame = output.sequenceNumber+1; + output.data.last().file.set("FrameNumber", output.sequenceNumber); + next_sequence_number++; + if (output.data.last().last().empty()) + qDebug("broken matrix"); return true; } + while(!res) { - currentIdx++; + output.data.clear(); + current_template_idx++; - if (currentIdx >= templates.size()) + // No more templates? We're done + if (current_template_idx >= templates.size()) return false; - bool open_res = open(templates[currentIdx], next_frame); + + // open the next data source + bool open_res = concreteOpen(templates[current_template_idx]); if (!open_res) return false; + + // get a frame from it res = actualSource->getNext(output); } + output.sequenceNumber = next_sequence_number++; + output.data.last().file.set("FrameNumber", output.sequenceNumber); + + if (output.data.last().last().empty()) + qDebug("broken matrix"); - next_frame = output.sequenceNumber+1; return res; } @@ -474,9 +540,14 @@ protected: class ProcessingStage; -class BasicLoop : public QRunnable +class BasicLoop : public QRunnable, public QFutureInterface { public: + BasicLoop() + { + this->reportStarted(); + } + void run(); QList * stages; @@ -502,13 +573,13 @@ public: int stage_id; virtual void reset()=0; - protected: int thread_count; SharedBuffer * inputBuffer; ProcessingStage * nextStage; QList * stages; + QThreadPool * threads; Transform * transform; }; @@ -527,6 +598,7 @@ void BasicLoop::run() current_idx++; current_idx = current_idx % stages->size(); } + this->reportFinished(); } class MultiThreadStage : public ProcessingStage @@ -561,7 +633,6 @@ public: } }; - class SingleThreadStage : public ProcessingStage { public: @@ -624,18 +695,20 @@ public: lock.unlock(); if (newItem) - { - BasicLoop * next = new BasicLoop(); - next->stages = stages; - next->start_idx = this->stage_id; - next->startItem = newItem; - - QThreadPool::globalInstance()->start(next, stages->size() - this->stage_id); - } + startThread(newItem); return input; } + void startThread(br::FrameData * newItem) + { + BasicLoop * next = new BasicLoop(); + next->stages = stages; + next->start_idx = this->stage_id; + next->startItem = newItem; + this->threads->start(next, stages->size() - stage_id); + } + // Calledfrom a different thread than run. bool tryAcquireNextStage(FrameData *& input) @@ -671,7 +744,7 @@ public: }; // No input buffer, instead we draw templates from some data source -// Will be operated by the main thread for the stream +// Will be operated by the main thread for the stream. starts threads class FirstStage : public SingleThreadStage { public: @@ -681,44 +754,51 @@ public: FrameData * run(FrameData * input, bool & should_continue) { - // Is there anything on our input buffer? If so we should start a thread with that. + // Try to get a frame from the datasource QWriteLocker lock(&statusLock); - input = dataSource.tryGetFrame(); - // Datasource broke? - if (!input) + bool last_frame = false; + input = dataSource.tryGetFrame(last_frame); + + // Datasource broke, or is currently out of frames? + if (!input || last_frame) { + // We will just stop and not continue. currentStatus = STOPPING; - should_continue = false; - return NULL; + if (!input) { + should_continue = false; + return NULL; + } } lock.unlock(); - + // Can we enter the next stage? should_continue = nextStage->tryAcquireNextStage(input); - BasicLoop * next = new BasicLoop(); - next->stages = stages; - next->start_idx = this->stage_id; - next->startItem = NULL; - - QThreadPool::globalInstance()->start(next, stages->size() - this->stage_id); + // We are exiting leaving this stage, should we start another + // thread here? Normally we will always re-queue a thread on + // the first stage, but if we received the last frame there is + // no need to. + if (!last_frame) { + startThread(NULL); + } return input; } - // Calledfrom a different thread than run. + // The last stage, trying to access the first stage bool tryAcquireNextStage(FrameData *& input) { + // Return the frame, was it the last one? bool was_last = dataSource.returnFrame(input); input = NULL; + + // OK we won't continue. if (was_last) { return false; } - if (!dataSource.isOpen()) - return false; - QReadLocker lock(&statusLock); - // Thread is already running, we should just return + // A thread is already in the first stage, + // we should just return if (currentStatus == STARTING) { return false; @@ -741,6 +821,7 @@ public: }; +// starts threads class LastStage : public SingleThreadStage { public: @@ -771,11 +852,14 @@ public: } next_target = input->sequenceNumber + 1; + // add the item to our output buffer collectedOutput.append(input->data); + // Can we enter the read stage? should_continue = nextStage->tryAcquireNextStage(input); - // Is there anything on our input buffer? If so we should start a thread with that. + // Is there anything on our input buffer? If so we should start a thread + // in this stage to process that frame. QWriteLocker lock(&statusLock); FrameData * newItem = inputBuffer->tryGetItem(); if (!newItem) @@ -785,23 +869,18 @@ public: lock.unlock(); if (newItem) - { - BasicLoop * next = new BasicLoop(); - next->stages = stages; - next->start_idx = this->stage_id; - next->startItem = newItem; - - QThreadPool::globalInstance()->start(next, stages->size() - this->stage_id); - } + startThread(newItem); return input; } }; + class StreamTransform : public CompositeTransform { Q_OBJECT public: + void train(const TemplateList & data) { foreach(Transform * transform, transforms) { @@ -831,21 +910,17 @@ public: bool res = readStage->dataSource.open(dst); if (!res) return; - QThreadPool::globalInstance()->releaseThread(); + // Start the first thread in the stream. readStage->currentStatus = SingleThreadStage::STARTING; + readStage->startThread(NULL); - BasicLoop loop; - loop.stages = &this->processingStages; - loop.start_idx = 0; - loop.startItem = NULL; - loop.setAutoDelete(false); - - QThreadPool::globalInstance()->start(&loop, processingStages.size() - processingStages[0]->stage_id); - - // Wait for the end. + // Wait for the stream to reach the last frame available from + // the data source. readStage->dataSource.waitLast(); - QThreadPool::globalInstance()->reserveThread(); + // Now that there are no more incoming frames, call finalize + // on each transform in turn to collect any last templates + // they wish to issue. TemplateList final_output; // Push finalize through the stages @@ -861,7 +936,8 @@ public: final_output.append(output_set); } - // dst is set to all output received by the final stage + // dst is set to all output received by the final stage, along + // with anything output via the calls to finalize. dst = collectionStage->getOutput(); dst.append(final_output); @@ -873,7 +949,8 @@ public: virtual void finalize(TemplateList & output) { (void) output; - // Not handling this yet -cao + // Nothing in particular to do here, stream calls finalize + // on all child transforms as part of projectUpdate } // Create and link stages @@ -881,6 +958,19 @@ public: { if (transforms.isEmpty()) return; + // We share a thread pool across streams attached to the same + // parent tranform, retrieve or create a thread pool based + // on our parent transform. + QMutexLocker poolLock(&poolsAccess); + QHash::Iterator it; + if (!pools.contains(this->parent())) { + it = pools.insert(this->parent(), new QThreadPool(this)); + it.value()->setMaxThreadCount(Globals->parallelism); + } + else it = pools.find(this->parent()); + threads = it.value(); + poolLock.unlock(); + stage_variance.reserve(transforms.size()); foreach (const br::Transform *transform, transforms) { stage_variance.append(transform->timeVarying()); @@ -891,6 +981,7 @@ public: processingStages.push_back(readStage); readStage->stage_id = 0; readStage->stages = &this->processingStages; + readStage->threads = this->threads; int next_stage_id = 1; @@ -898,9 +989,7 @@ public: for (int i =0; i < transforms.size(); i++) { if (stage_variance[i]) - { processingStages.append(new SingleThreadStage(prev_stage_variance)); - } else processingStages.append(new MultiThreadStage(Globals->parallelism)); @@ -911,6 +1000,7 @@ public: processingStages[i]->nextStage = processingStages[i+1]; processingStages.last()->stages = &this->processingStages; + processingStages.last()->threads = this->threads; processingStages.last()->transform = transforms[i]; prev_stage_variance = stage_variance[i]; @@ -920,6 +1010,7 @@ public: processingStages.append(collectionStage); collectionStage->stage_id = next_stage_id; collectionStage->stages = &this->processingStages; + collectionStage->threads = this->threads; processingStages[processingStages.size() - 2]->nextStage = collectionStage; @@ -942,6 +1033,10 @@ protected: QList processingStages; + static QHash pools; + static QMutex poolsAccess; + QThreadPool * threads; + void _project(const Template &src, Template &dst) const { (void) src; (void) dst; @@ -954,6 +1049,9 @@ protected: } }; +QHash StreamTransform::pools; +QMutex StreamTransform::poolsAccess; + BR_REGISTER(Transform, StreamTransform) diff --git a/openbr/plugins/validate.cpp b/openbr/plugins/validate.cpp index 68c603e..8b3a174 100644 --- a/openbr/plugins/validate.cpp +++ b/openbr/plugins/validate.cpp @@ -116,7 +116,8 @@ class FilterDistance : public Distance foreach (const QString &key, Globals->filters.keys()) { bool keep = false; const QString metadata = a.file.get(key, ""); - if (metadata.isEmpty() || Globals->filters[key].isEmpty()) continue; + if (Globals->filters[key].isEmpty()) continue; + if (metadata.isEmpty()) return -std::numeric_limits::max(); foreach (const QString &value, Globals->filters[key]) { if (metadata == value) { keep = true; diff --git a/scripts/downloadDatasets.sh b/scripts/downloadDatasets.sh index 62020ab..35130ae 100755 --- a/scripts/downloadDatasets.sh +++ b/scripts/downloadDatasets.sh @@ -63,4 +63,6 @@ if [ ! -d ../data/KTH/vid ]; then unzip ${vidclass}.zip -d ../data/KTH/vid/${vidclass} rm ${vidclass}.zip done + # this file is corrupted + rm -f ../data/KTH/vid/boxing/person01_boxing_d4_uncomp.avi fi diff --git a/share/openbr/openbr.bib b/share/openbr/openbr.bib index b07abd7..111cd1b 100644 --- a/share/openbr/openbr.bib +++ b/share/openbr/openbr.bib @@ -34,6 +34,11 @@ Howpublished = {https://github.com/lbestrowden}, Title = {bestrow1 at msu.edu}} +@misc{imaus10, + Author = {Austin Van Blanton}, + Howpublished = {https://github.com/imaus10}, + Title = {imaus10 at gmail.com}} + % Software @misc{libface, Howpublished = {http://libface.sourceforge.net/file/Home.html},