diff --git a/LICENSE.txt b/LICENSE.txt index c27dfaf..661310f 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright 2012-2014 The MITRE Corporation +Copyright 2012 The MITRE Corporation Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/app/br/br.cpp b/app/br/br.cpp index e78c71d..49f9a95 100644 --- a/app/br/br.cpp +++ b/app/br/br.cpp @@ -138,8 +138,8 @@ public: check(parc >= 2 && parc <= 4, "Incorrect parameter count for 'evalClassification'."); br_eval_classification(parv[0], parv[1], parc >= 3 ? parv[2] : "", parc >= 4 ? parv[3] : ""); } else if (!strcmp(fun, "evalClustering")) { - check(parc == 2, "Incorrect parameter count for 'evalClustering'."); - br_eval_clustering(parv[0], parv[1]); + check((parc >= 2) && (parc <= 3), "Incorrect parameter count for 'evalClustering'."); + br_eval_clustering(parv[0], parv[1], parc == 3 ? parv[2] : ""); } else if (!strcmp(fun, "evalDetection")) { check((parc >= 2) && (parc <= 3), "Incorrect parameter count for 'evalDetection'."); br_eval_detection(parv[0], parv[1], parc == 3 ? parv[2] : ""); diff --git a/openbr/core/cluster.cpp b/openbr/core/cluster.cpp index 9b3beda..9e87b44 100644 --- a/openbr/core/cluster.cpp +++ b/openbr/core/cluster.cpp @@ -100,7 +100,9 @@ Neighborhood getNeighborhood(const QStringList &simmats) int currentRows = -1; int columnOffset = 0; for (int j=0; j format(br::Factory::make(simmats[i*numGalleries+j])); + br::Template t = format->read(); + cv::Mat m = t.m(); if (j==0) { currentRows = m.rows; allNeighbors.resize(currentRows); @@ -115,8 +117,9 @@ Neighborhood getNeighborhood(const QStringList &simmats) float val = m.at(k,l); if ((i==j) && (k==l)) continue; // Skips self-similarity scores - if ((val != -std::numeric_limits::infinity()) && - (val != std::numeric_limits::infinity())) { + if (val != -std::numeric_limits::max() + && val != -std::numeric_limits::infinity() + && val != std::numeric_limits::infinity()) { globalMax = std::max(globalMax, val); globalMin = std::min(globalMin, val); } @@ -157,7 +160,7 @@ Neighborhood getNeighborhood(const QStringList &simmats) // Zhu et al. "A Rank-Order Distance based Clustering Algorithm for Face Tagging", CVPR 2011 br::Clusters br::ClusterGallery(const QStringList &simmats, float aggressiveness, const QString &csv) { - qDebug("Clustering %d simmat(s)", simmats.size()); + qDebug("Clustering %d simmat(s), aggressiveness %f", simmats.size(), aggressiveness); // Read in gallery parts, keeping top neighbors of each template Neighborhood neighborhood = getNeighborhood(simmats); @@ -275,13 +278,14 @@ float jaccardIndex(const QVector &indicesA, const QVector &indicesB) // Evaluates clustering algorithms based on metrics described in // Santo Fortunato "Community detection in graphs", Physics Reports 486 (2010) -void br::EvalClustering(const QString &csv, const QString &input) +void br::EvalClustering(const QString &csv, const QString &input, QString truth_property) { + if (truth_property.isEmpty()) + truth_property = "Label"; qDebug("Evaluating %s against %s", qPrintable(csv), qPrintable(input)); - // We assume clustering algorithms store assigned cluster labels as integers (since the clusters are - // not named). Direct use of ClusterID is not general -cao - QList labels = File::get(TemplateList::fromGallery(input), "ClusterID"); + TemplateList tList = TemplateList::fromGallery(input); + QList labels = tList.indexProperty(truth_property); QHash labelToIndex; int nClusters = 0; diff --git a/openbr/core/cluster.h b/openbr/core/cluster.h index 9d5ce7b..84bb9ba 100644 --- a/openbr/core/cluster.h +++ b/openbr/core/cluster.h @@ -28,7 +28,7 @@ namespace br typedef QVector Clusters; Clusters ClusterGallery(const QStringList &simmats, float aggressiveness, const QString &csv); - void EvalClustering(const QString &csv, const QString &input); + void EvalClustering(const QString &csv, const QString &input, QString truth_property); Clusters ReadClusters(const QString &csv); void WriteClusters(const Clusters &clusters, const QString &csv); diff --git a/openbr/core/common.cpp b/openbr/core/common.cpp index bfd38fb..d6c445a 100644 --- a/openbr/core/common.cpp +++ b/openbr/core/common.cpp @@ -76,3 +76,11 @@ QList Common::linspace(float start, float stop, int n) { spaced.append(stop); return spaced; } + +QList Common::ind2sub(int dims, int nPerDim, int idx) { + QList subIndices; + for (int j = 0; j < dims; j++) { + subIndices.append(((int)floor( idx / pow((float)nPerDim, j))) % nPerDim); + } + return subIndices; +} diff --git a/openbr/core/common.h b/openbr/core/common.h index 15516d0..09c171e 100644 --- a/openbr/core/common.h +++ b/openbr/core/common.h @@ -335,6 +335,10 @@ V Downsample(V vals, int k) return newVals; } +/*! \brief Converts index into subdimensions. +*/ +QList ind2sub(int dims, int nPerDim, int idx); + } #endif // COMMON_COMMON_H diff --git a/openbr/core/eigenutils.cpp b/openbr/core/eigenutils.cpp index c054155..f927b33 100644 --- a/openbr/core/eigenutils.cpp +++ b/openbr/core/eigenutils.cpp @@ -67,3 +67,12 @@ void printEigen(Eigen::MatrixXf X) { void printSize(Eigen::MatrixXf X) { qDebug() << "Rows=" << X.rows() << "\tCols=" << X.cols(); } + +float eigMean(const Eigen::MatrixXf& x) { + return x.array().sum() / (x.rows() * x.cols()); +} + +float eigStd(const Eigen::MatrixXf& x) { + float mean = eigMean(x); + return sqrt((x.array() - mean).pow(2).sum() / (x.cols() * x.rows())); +} diff --git a/openbr/core/eigenutils.h b/openbr/core/eigenutils.h index 1ca8fa7..fa9a075 100644 --- a/openbr/core/eigenutils.h +++ b/openbr/core/eigenutils.h @@ -67,4 +67,55 @@ inline QDataStream &operator>>(QDataStream &stream, Eigen::Matrix< _Scalar, _Row return stream; } +/*Compute the mean of the each column (dim == 1) or row (dim == 2) + of the matrix*/ +template +Eigen::MatrixBase eigMean(const Eigen::MatrixBase& x,int dim) +{ + if (dim == 1) { + Eigen::MatrixBase y(1,x.cols()); + for (int i = 0; i < x.cols(); i++) + y(i) = x.col(i).sum() / x.rows(); + return y; + } else if (dim == 2) { + Eigen::MatrixBase y(x.rows(),1); + for (int i = 0; i < x.rows(); i++) + y(i) = x.row(i).sum() / x.cols(); + return y; + } + qFatal("A matrix can only have two dimensions"); +} + +/*Compute the element-wise mean*/ +float eigMean(const Eigen::MatrixXf& x); +/*Compute the element-wise mean*/ +float eigStd(const Eigen::MatrixXf& x); + +/*Compute the std dev of the each column (dim == 1) or row (dim == 2) + of the matrix*/ +template +Eigen::MatrixBase eigStd(const Eigen::MatrixBase& x,int dim) +{ + Eigen::MatrixBase mean = eigMean(x, dim); + if (dim == 1) { + Eigen::MatrixBase y(1,x.cols()); + for (int i = 0; i < x.cols(); i++) { + T value = 0; + for (int j = 0; j < x.rows(); j++) + value += pow(y(j, i) - mean(i), 2); + y(i) = sqrt(value / (x.rows() - 1)); + } + return y; + } else if (dim == 2) { + Eigen::MatrixBase y(x.rows(),1); + for (int i = 0; i < x.rows(); i++) { + T value = 0; + for (int j = 0; j < x.cols(); j++) + value += pow(y(i, j) - mean(j), 2); + y(i) = sqrt(value / (x.cols() - 1)); + } + return y; + } + qFatal("A matrix can only have two dimensions"); +} #endif // EIGENUTILS_H diff --git a/openbr/core/eval.cpp b/openbr/core/eval.cpp index 97df2e5..d2f2f67 100644 --- a/openbr/core/eval.cpp +++ b/openbr/core/eval.cpp @@ -572,6 +572,7 @@ float EvalLandmarking(const QString &predictedGallery, const QString &truthGalle const QStringList predictedNames = File::get(predicted, "name"); const QStringList truthNames = File::get(truth, "name"); + int skipped = 0; QList< QList > pointErrors; for (int i=0; i predictedPoints = predicted[i].file.points(); const QList truthPoints = truth[truthIndex].file.points(); - if (predictedPoints.size() != truthPoints.size()) qFatal("Points size mismatch for file: %s", qPrintable(predictedName)); + if (predictedPoints.size() != truthPoints.size()) { + skipped++; + continue; + } while (pointErrors.size() < predictedPoints.size()) pointErrors.append(QList()); if (normalizationIndexA >= truthPoints.size()) qFatal("Normalization index A is out of range."); @@ -588,6 +592,7 @@ float EvalLandmarking(const QString &predictedGallery, const QString &truthGalle for (int j=0; j averagePointErrors; averagePointErrors.reserve(pointErrors.size()); for (int i=0; i #include #include +#include #include #include #include @@ -81,7 +82,7 @@ void readFile(const QString &file, QStringList &lines) { QByteArray data; readFile(file, data); - lines = QString(data).split('\n', QString::SkipEmptyParts); + lines = QString(data).split(QRegularExpression("[\n|\r\n|\r]"), QString::SkipEmptyParts); for (int i=0; i(gallery); + Template *t = reinterpret_cast(tmpl); + gal->write(*t); +} + +void br_add_template_list_to_gallery(br_gallery gallery, br_template_list tl) { Gallery *gal = reinterpret_cast(gallery); TemplateList *realTL = reinterpret_cast(tl); diff --git a/openbr/openbr.h b/openbr/openbr.h index f2cce16..0c3a22f 100644 --- a/openbr/openbr.h +++ b/openbr/openbr.h @@ -167,9 +167,10 @@ BR_EXPORT void br_eval_classification(const char *predicted_gallery, const char * \brief Evaluates and prints clustering accuracy to the terminal. * \param csv The cluster results file. * \param gallery The br::Gallery used to generate the \ref simmat that was clustered. + * \param truth_property (Optional) which metadata key to use from gallery, defaults to Label * \see br_cluster */ -BR_EXPORT void br_eval_clustering(const char *csv, const char *gallery); +BR_EXPORT void br_eval_clustering(const char *csv, const char *gallery, const char * truth_property); /*! * \brief Evaluates and prints detection accuracy to terminal. @@ -562,9 +563,13 @@ BR_EXPORT br_gallery br_make_gallery(const char *gallery); */ BR_EXPORT br_template_list br_load_from_gallery(br_gallery gallery); /*! + * \brief Write a br::Template to the br::Gallery on disk. + */ +BR_EXPORT void br_add_template_to_gallery(br_gallery gallery, br_template tmpl); +/*! * \brief Write a br::TemplateList to the br::Gallery on disk. */ -BR_EXPORT void br_add_to_gallery(br_gallery gallery, br_template_list tl); +BR_EXPORT void br_add_template_list_to_gallery(br_gallery gallery, br_template_list tl); /*! * \brief Close the br::Gallery. */ diff --git a/openbr/openbr_plugin.cpp b/openbr/openbr_plugin.cpp index 47791fe..94fc737 100644 --- a/openbr/openbr_plugin.cpp +++ b/openbr/openbr_plugin.cpp @@ -137,6 +137,8 @@ QVariant File::parse(const QString &value) if (ok) return point; const QRectF rect = QtUtils::toRect(value, &ok); if (ok) return rect; + const int i = value.toInt(&ok); + if (ok) return i; const float f = value.toFloat(&ok); if (ok) return f; return value; diff --git a/openbr/plugins/algorithms.cpp b/openbr/plugins/algorithms.cpp index 06e3275..1c09600 100644 --- a/openbr/plugins/algorithms.cpp +++ b/openbr/plugins/algorithms.cpp @@ -44,7 +44,7 @@ class AlgorithmsInitializer : public Initializer Globals->abbreviations.insert("AgeEstimation", "AgeRegression"); Globals->abbreviations.insert("FaceRecognition2", "{PP5Register+Affine(128,128,0.25,0.35)+Cvt(Gray)}+(Gradient+Bin(0,360,9,true))/(Blur(1)+Gamma(0.2)+DoG(1,2)+ContrastEq(0.1,10)+LBP(1,2,true)+Bin(0,10,10,true))+Merge+Integral+RecursiveIntegralSampler(4,2,8,LDA(.98)+Normalize(L1))+Cat+PCA(768)+Normalize(L1)+Quantize:UCharL1"); Globals->abbreviations.insert("CropFace", "Open+Cvt(Gray)+Cascade(FrontalFace)+ASEFEyes+Affine(128,128,0.25,0.35)"); - Globals->abbreviations.insert("4SF", "Open+Cvt(Gray)+Cascade(FrontalFace)+ASEFEyes+Affine(128,128,0.33,0.45)+(Grid(10,10)+SIFTDescriptor(12)+ByRow)/(Blur(1.1)+Gamma(0.2)+DoG(1,2)+ContrastEq(0.1,10)+LBP(1,2)+RectRegions(8,8,6,6)+Hist(59))+PCA(0.95)+Normalize(L2)+Dup(12)+RndSubspace(0.05,1)+LDA(0.98)+Cat+PCA(0.95)+Normalize(L1)+Quantize:NegativeLogPlusOne(ByteL1)"); + Globals->abbreviations.insert("4SF", "Open+Cvt(Gray)+Cascade(FrontalFace)+ASEFEyes+Affine(128,128,0.33,0.45)+(Grid(10,10)+SIFTDescriptor(12)+ByRow)/(Blur(1.1)+Gamma(0.2)+DoG(1,2)+ContrastEq(0.1,10)+LBP(1,2)+RectRegions(8,8,6,6)+Hist(59))+PCA(0.95)+Cat+Normalize(L2)+Dup(12)+RndSubspace(0.05,1)+LDA(0.98)+Cat+PCA(0.95)+Normalize(L1)+Quantize:NegativeLogPlusOne(ByteL1)"); // Video Globals->abbreviations.insert("DisplayVideo", "Stream(FPSLimit(30)+Show(false,[FrameNumber])+Discard)"); @@ -64,6 +64,7 @@ class AlgorithmsInitializer : public Initializer Globals->abbreviations.insert("SmallSIFT", "Open+LimitSize(512)+KeyPointDetector(SIFT)+KeyPointDescriptor(SIFT):KeyPointMatcher(BruteForce)"); Globals->abbreviations.insert("SmallSURF", "Open+LimitSize(512)+KeyPointDetector(SURF)+KeyPointDescriptor(SURF):KeyPointMatcher(BruteForce)"); Globals->abbreviations.insert("ColorHist", "Open+LimitSize(512)+Expand+EnsureChannels(3)+SplitChannels+Hist(256,0,8)+Cat+Normalize(L1):L2"); + Globals->abbreviations.insert("ImageSimilarity", "Open+EnsureChannels(3)+Resize(256,256)+SplitChannels+RectRegions(64,64,64,64)+Hist(256,0,8)+Cat:NegativeLogPlusOne(L2)"); Globals->abbreviations.insert("ImageClassification", "Open+CropSquare+LimitSize(256)+Cvt(Gray)+Gradient+Bin(0,360,9,true)+Merge+Integral+RecursiveIntegralSampler(4,2,8,Singleton(KMeans(256)))+Cat+CvtFloat+Hist(256)+KNN(5,Dist(L1),false,5)+Rename(KNN,Subject)"); Globals->abbreviations.insert("TanTriggs", "Blur(1.1)+Gamma(0.2)+DoG(1,2)+ContrastEq(0.1,10)"); diff --git a/openbr/plugins/eigen3.cpp b/openbr/plugins/eigen3.cpp index 710570e..4965d2f 100644 --- a/openbr/plugins/eigen3.cpp +++ b/openbr/plugins/eigen3.cpp @@ -15,15 +15,35 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ #include + #include "openbr_internal.h" #include "openbr/core/common.h" #include "openbr/core/eigenutils.h" +#include "openbr/core/opencvutils.h" namespace br { /*! + * \ingroup initializers + * \brief Initialize Eigen + * http://eigen.tuxfamily.org/dox/TopicMultiThreading.html + * \author Scott Klum \cite sklum + */ +class EigenInitializer : public Initializer +{ + Q_OBJECT + + void initialize() const + { + Eigen::initParallel(); + } +}; + +BR_REGISTER(Initializer, EigenInitializer) + +/*! * \ingroup transforms * \brief Projects input into learned Principal Component Analysis subspace. * \author Brendan Klare \cite bklare @@ -296,6 +316,8 @@ BR_REGISTER(Transform, DFFSTransform) */ class LDATransform : public Transform { + friend class SparseLDATransform; + Q_OBJECT Q_PROPERTY(float pcaKeep READ get_pcaKeep WRITE set_pcaKeep RESET reset_pcaKeep STORED false) Q_PROPERTY(bool pcaWhiten READ get_pcaWhiten WRITE set_pcaWhiten RESET reset_pcaWhiten STORED false) @@ -499,21 +521,30 @@ class LDATransform : public Transform // Do projection outMap = projection.transpose() * (inMap - mean); - if (normalize && isBinary) dst.m().at(0,0) = dst.m().at(0,0) / stdDev; } void store(QDataStream &stream) const { - stream << pcaKeep << directLDA << directDrop << dimsOut << mean << projection; + stream << pcaKeep; + stream << directLDA; + stream << directDrop; + stream << dimsOut; + stream << mean; + stream << projection; if (normalize && isBinary) stream << stdDev; } void load(QDataStream &stream) { - stream >> pcaKeep >> directLDA >> directDrop >> dimsOut >> mean >> projection; + stream >> pcaKeep; + stream >> directLDA; + stream >> directDrop; + stream >> dimsOut; + stream >> mean; + stream >> projection; if (normalize && isBinary) stream >> stdDev; } @@ -522,6 +553,104 @@ class LDATransform : public Transform BR_REGISTER(Transform, LDATransform) /*! + * \ingroup transforms + * \brief Projects input into learned Linear Discriminant Analysis subspace + * learned on a sparse subset of features with the highest weight + * in the original LDA algorithm. + * \author Brendan Klare \cite bklare + */ +class SparseLDATransform : public Transform +{ + Q_OBJECT + Q_PROPERTY(float varThreshold READ get_varThreshold WRITE set_varThreshold RESET reset_varThreshold STORED false) + Q_PROPERTY(float pcaKeep READ get_pcaKeep WRITE set_pcaKeep RESET reset_pcaKeep STORED false) + Q_PROPERTY(bool normalize READ get_normalize WRITE set_normalize RESET reset_normalize STORED false) + BR_PROPERTY(float, varThreshold, 1.5) + BR_PROPERTY(float, pcaKeep, 0.98) + BR_PROPERTY(bool, normalize, true) + + LDATransform ldaSparse; + int dimsOut; + QList selections; + + Eigen::VectorXf mean; + + void init() + { + ldaSparse.init(); + ldaSparse.pcaKeep = pcaKeep; + ldaSparse.inputVariable = "Label"; + ldaSparse.isBinary = true; + ldaSparse.normalize = true; + } + + void train(const TemplateList &_trainingSet) + { + + LDATransform ldaOrig; + ldaOrig.init(); + ldaOrig.inputVariable = "Label"; + ldaOrig.pcaKeep = pcaKeep; + ldaOrig.isBinary = true; + ldaOrig.normalize = true; + + ldaOrig.train(_trainingSet); + + //Only works on binary class problems for now + assert(ldaOrig.projection.cols() == 1); + float ldaStd = eigStd(ldaOrig.projection); + for (int i = 0; i < ldaOrig.projection.rows(); i++) + if (abs(ldaOrig.projection(i)) > varThreshold * ldaStd) + selections.append(i); + + TemplateList newSet; + for (int i = 0; i < _trainingSet.size(); i++) { + cv::Mat x(_trainingSet[i]); + cv::Mat y = cv::Mat(selections.size(), 1, CV_32FC1); + int idx = 0; + int cnt = 0; + for (int j = 0; j < x.rows; j++) + for (int k = 0; k < x.cols; k++, cnt++) + if (selections.contains(cnt)) + y.at(idx++,0) = x.at(j, k); + newSet.append(Template(_trainingSet[i].file, y)); + } + ldaSparse.train(newSet); + dimsOut = ldaSparse.dimsOut; + } + + void project(const Template &src, Template &dst) const + { + Eigen::Map inMap((float*)src.m().ptr(), src.m().rows*src.m().cols, 1); + Eigen::Map outMap(dst.m().ptr(), dimsOut, 1); + + int d = selections.size(); + cv::Mat inSelect(d,1,CV_32F); + for (int i = 0; i < d; i++) + inSelect.at(i) = src.m().at(selections[i]); + ldaSparse.project(Template(src.file, inSelect), dst); + } + + void store(QDataStream &stream) const + { + stream << pcaKeep; + stream << ldaSparse; + stream << dimsOut; + stream << selections; + } + + void load(QDataStream &stream) + { + stream >> pcaKeep; + stream >> ldaSparse; + stream >> dimsOut; + stream >> selections; + } +}; + +BR_REGISTER(Transform, SparseLDATransform) + +/*! * \ingroup distances * \brief L1 distance computed using eigen. * \author Josh Klontz \cite jklontz diff --git a/openbr/plugins/format.cpp b/openbr/plugins/format.cpp index 14d2b97..ca8b372 100644 --- a/openbr/plugins/format.cpp +++ b/openbr/plugins/format.cpp @@ -151,7 +151,7 @@ class csvFormat : public Format { QFile f(file.name); f.open(QFile::ReadOnly); - QStringList lines(QString(f.readAll()).split('\n')); + QStringList lines(QString(f.readAll()).split(QRegularExpression("[\n|\r\n|\r]"), QString::SkipEmptyParts)); f.close(); bool isUChar = true; diff --git a/openbr/plugins/gallery.cpp b/openbr/plugins/gallery.cpp index 8463e77..d3187a8 100644 --- a/openbr/plugins/gallery.cpp +++ b/openbr/plugins/gallery.cpp @@ -506,29 +506,78 @@ BR_REGISTER(Gallery, csvGallery) * \brief Treats each line as a file. * \author Josh Klontz \cite jklontz * - * The entire line is treated as the file path. + * The entire line is treated as the file path. An optional label may be specified using a space ' ' separator: * +\verbatim + + +... + +\endverbatim + * or +\verbatim +