Merge turk

Brendan Klare
2 parents 0ecab0f9 f60a7d57
Showing 4 changed files with 62 additions and 156 deletions
openbr/core/plot.cpp
openbr/plugins/misc.cpp
openbr/plugins/output.cpp
openbr/plugins/turk.cpp
@@ -219,6 +219,7 @@ bool Plot(const QStringList &amp;files, const File &amp;destination, bool show)
     qDebug("Plotting %d file(s) to %s", files.size(), qPrintable(destination));
     const bool minimalist = destination.getBool("minimalist");
+    const bool uncertainty = destination.get<bool>("uncertainty");
     // Use a br::file for simple storage of plot options
     File cmcOpts;
@@ -268,7 +269,7 @@ bool Plot(const QStringList &amp;files, const File &amp;destination, bool show)
                             QString(" + theme(aspect.ratio=1)\n\n")));
     p.file.write(qPrintable(QString("ggplot(CMC, aes(x=X, y=Y%1%2)) + ggtitle(\"%3\") + xlab(\"Rank\") + ylab(\"Retrieval Rate\")").arg(p.major.size > 1 ? QString(" ,colour=factor(%1)").arg(p.major.header) : QString(), p.minor.size > 1 ? QString(", linetype=factor(%1)").arg(p.minor.header) : QString(), cmcOpts.get<QString>("title",QString())) +
-                            QString(((p.major.smooth || p.minor.smooth) ? (minimalist ? " + stat_summary(geom=\"line\", fun.y=mean, size=%1)" : " + stat_summary(geom=\"line\", fun.y=min, aes(linetype=\"Min/Max\"), size=%1) + stat_summary(geom=\"line\", "
+                            QString(((p.major.smooth || p.minor.smooth) ? (!uncertainty ? " + stat_summary(geom=\"line\", fun.y=mean, size=%1)" : " + stat_summary(geom=\"line\", fun.y=min, aes(linetype=\"Min/Max\"), size=%1) + stat_summary(geom=\"line\", "
                             "fun.y=max, aes(linetype=\"Min/Max\"), size=%1) + stat_summary(geom=\"line\", fun.y=mean, aes(linetype=\"Mean\"), size=%1) + scale_linetype_manual(\"Legend\", values=c(\"Mean\"=1, \"Min/Max\"=2))") : " + geom_line(size=%1)")).arg(QString::number(cmcOpts.get<float>("thickness",1))) +
                             (minimalist ? "" : " + scale_x_log10(labels=c(1,5,10,50,100), breaks=c(1,5,10,50,100)) + annotation_logticks(sides=\"b\")") +
                             (p.major.size > 1 ? getScale("colour", p.major.header, p.major.size) : QString()) +
@@ -283,7 +284,7 @@ bool Plot(const QStringList &amp;files, const File &amp;destination, bool show)
                             QString(", ylab=\"True Accept Rate%1\") + theme_minimal()").arg(p.minor.size > 1 ? " / " + p.minor.header : QString()) +
                             (p.major.size > 1 ? getScale("fill", p.major.header, p.major.size) : QString()) +
                             (p.minor.size > 1 ? QString(" + facet_grid(%2 ~ X)").arg(p.minor.header) : QString(" + facet_grid(. ~ X, labeller=far_labeller)")) +
-                            QString(" + scale_y_continuous(labels=percent) + theme(legend.position=\"none\", axis.text.x=element_text(angle=-90, hjust=0))%1").arg((p.major.smooth || p.minor.smooth) ? "" : " + geom_text(data=BC, aes(label=Y, y=0.05))\n\n")));
+                            QString(" + scale_y_continuous(labels=percent) + theme(legend.position=\"none\", axis.text.x=element_text(angle=-90, hjust=0))%1").arg((p.major.smooth || p.minor.smooth) ? "" : " + geom_text(data=BC, aes(label=Y, y=0.05))") + "\n\n"));
     p.file.write(qPrintable(QString("qplot(X, Y, data=ERR%1, linetype=Error").arg((p.major.smooth || p.minor.smooth) ? ", geom=\"smooth\", method=loess, level=0.99" : ", geom=\"line\"") +
                             ((p.flip ? p.major.size : p.minor.size) > 1 ? QString(", colour=factor(%1)").arg(p.flip ? p.major.header : p.minor.header) : QString()) +
@@ -366,6 +366,29 @@ BR_REGISTER(Transform, RegexPropertyTransform)
 /*!
  * \ingroup transforms
+ * \brief Create matrix from metadata values.
+ * \author Josh Klontz \cite jklontz
+ */
+class ExtractMetadataTransform : public UntrainableMetaTransform
+{
+    Q_OBJECT
+    Q_PROPERTY(QStringList keys READ get_keys WRITE set_keys RESET reset_keys STORED false)
+    BR_PROPERTY(QStringList, keys, QStringList())
+
+    void project(const Template &src, Template &dst) const
+    {
+        dst.file = src.file;
+        QList<float> values;
+        foreach (const QString &key, keys)
+            values.append(src.file.get<float>(key));
+        dst.append(OpenCVUtils::toMat(values, 1));
+    }
+};
+
+BR_REGISTER(Transform, ExtractMetadataTransform)
+
+/*!
+ * \ingroup transforms
  * \brief Store the last matrix of the input template as a metadata key with input property name.
  * \author Charles Otto \cite caotto
  */
@@ -365,15 +365,22 @@ BR_REGISTER(Output, EmptyOutput)
 class evalOutput : public MatrixOutput
 {
     Q_OBJECT
+    Q_PROPERTY(QString target READ get_target WRITE set_target RESET reset_target STORED false)
+    Q_PROPERTY(QString query READ get_query WRITE set_query RESET reset_query STORED false)
     Q_PROPERTY(bool crossValidate READ get_crossValidate WRITE set_crossValidate RESET reset_crossValidate STORED false)
     BR_PROPERTY(bool, crossValidate, true)
+    BR_PROPERTY(QString, target, QString())
+    BR_PROPERTY(QString, query, QString())
     ~evalOutput()
     {
+        if (!target.isEmpty()) targetFiles = TemplateList::fromGallery(target).files();
+        if (!query.isEmpty())  queryFiles  = TemplateList::fromGallery(query).files();
+
         if (data.data) {
             const QString csv = QString(file.name).replace(".eval", ".csv");
             if ((Globals->crossValidate == 0) || (!crossValidate)) {
-                Evaluate(data,targetFiles, queryFiles, csv);
+                Evaluate(data, targetFiles, queryFiles, csv);
             } else {
                 QFutureSynchronizer<float> futures;
                 for (int i=0; i<Globals->crossValidate; i++)
@@ -13,10 +13,6 @@ namespace br
 class turkGallery : public Gallery
 {
     Q_OBJECT
-    Q_PROPERTY(bool flat READ get_flat WRITE set_flat RESET reset_flat STORED false)
-    Q_PROPERTY(bool normalize READ get_normalize WRITE set_normalize RESET reset_normalize STORED false)
-    BR_PROPERTY(bool, flat, false)
-    BR_PROPERTY(bool, normalize, false)
     struct Attribute : public QStringList
     {
@@ -40,7 +36,8 @@ class turkGallery : public Gallery
             }
             Attribute normal(name);
-            const float sum = Common::Sum(values);
+            float sum = Common::Sum(values);
+            if (sum == 0) sum = 1;
             for (int i=0; i<values.size(); i++)
                 normal.append(QString::number(values[i] / sum));
             return normal;
@@ -50,38 +47,28 @@ class turkGallery : public Gallery
     TemplateList readBlock(bool *done)
     {
         *done = true;
-        TemplateList templates;
         QStringList lines = QtUtils::readLines(file);
-        if (lines.empty())
-            qFatal(".turk Gallery missing header.");
-        QList<Attribute> types;
-        foreach (const QString &header, parse(lines.takeFirst()))
-            types.append(header);
+        QList<Attribute> headers;
+        if (!lines.isEmpty())
+            foreach (const QString &header, parse(lines.takeFirst()))
+                headers.append(header);
+        TemplateList templates;
         foreach (const QString &line, lines) {
-            const QStringList words = parse(line);
-            if (words.size() != types.size())
-                qFatal(".turk Gallery incorrect column count.");
-
-            File f(words[0], words[0].mid(0,5));
-            for (int i=1; i<words.size(); i++) {
-                Attribute &type = types[i];
-                Attribute rating(words[i]);
-                if (type.size() != rating.size())
-                    qFatal(".turk Gallery incorrect ratings count.");
+            QStringList words = parse(line);
+            if (words.size() != headers.size())
+                qFatal("turkGallery invalid column count");
-                if (normalize)
-                    rating = rating.normalized();
+            File f;
+            f.name = words[0];
+            f.set("Label", words[0].mid(0,5));
-                if (flat) {
-                    for (int j=0; j<type.size(); j++)
-                        f.set(type.name + "_" + type[j], rating[j]);
-                } else {
-                    QMap<QString,QVariant> categoryMap;
-                    for (int j=0; j<type.size(); j++)
-                        categoryMap.insert(type[j], rating[j]);
-                    f.set(type.name, categoryMap);
-                }
+            for (int i=1; i<words.size(); i++) {
+                Attribute ratings = Attribute(words[i]).normalized();
+                if (headers[i].size() != ratings.size())
+                    qFatal("turkGallery invalid attribute count");
+                for (int j=0; j<ratings.size(); j++)
+                    f.set(headers[i].name + "_" + headers[i][j], ratings[j]);
             }
             templates.append(f);
         }
@@ -97,58 +84,6 @@ class turkGallery : public Gallery
 BR_REGISTER(Gallery, turkGallery)
-static Template unmap(const Template &t, const QString& variable, const float maxVotes, const float maxRange, const float minRange, const bool classify, const bool consensusOnly) {
-    // Create a new template matching the one containing the votes in the map structure
-    // but remove the map structure
-    Template expandedT = t;
-    expandedT.file.remove(variable);
-
-    QMap<QString,QVariant> map = t.file.get<QMap<QString,QVariant> >(variable);
-    QMapIterator<QString, QVariant> i(map);
-    bool ok;
-
-    while (i.hasNext()) {
-        i.next();
-        // Normalize to [minRange,maxRange]
-        float value = i.value().toFloat(&ok)*(maxRange-minRange)/maxVotes - minRange;
-        if (!ok) qFatal("Failed to expand Turk votes for %s", qPrintable(variable));
-        if (classify) (value > maxRange-((maxRange-minRange)/2)) ? value = maxRange : value = minRange;
-        else if (consensusOnly && (value != maxRange && value != minRange)) continue;
-        expandedT.file.set(i.key(),value);
-    }
-
-    return expandedT;
-}
-
-/*!
- * \ingroup transforms
- * \brief Converts Amazon MTurk labels to a non-map format for use in a transform
- * \author Scott Klum \cite sklum
- */
-class TurkTransform : public UntrainableTransform
-{
-    Q_OBJECT
-    Q_PROPERTY(QString HIT READ get_HIT WRITE set_HIT RESET reset_HIT STORED false)
-    Q_PROPERTY(float maxVotes READ get_maxVotes WRITE set_maxVotes RESET reset_maxVotes STORED false)
-    Q_PROPERTY(float maxRange READ get_maxRange WRITE set_maxRange RESET reset_maxRange STORED false)
-    Q_PROPERTY(float minRange READ get_minRange WRITE set_minRange RESET reset_minRange STORED false)
-    Q_PROPERTY(bool classify READ get_classify WRITE set_classify RESET reset_classify STORED false)
-    Q_PROPERTY(bool consensusOnly READ get_consensusOnly WRITE set_consensusOnly RESET reset_consensusOnly STORED false)
-    BR_PROPERTY(QString, HIT, QString())
-    BR_PROPERTY(float, maxVotes, 1)
-    BR_PROPERTY(float, maxRange, 1)
-    BR_PROPERTY(float, minRange, 0)
-    BR_PROPERTY(bool, classify, false)
-    BR_PROPERTY(bool, consensusOnly, false)
-
-    void project(const Template &src, Template &dst) const
-    {
-        dst = unmap(src, HIT, maxVotes, maxRange, minRange, classify, consensusOnly);
-    }
-};
-
-BR_REGISTER(Transform, TurkTransform)
-
 /*!
  * \ingroup transforms
  * \brief Convenience class for training turk attribute regressors
@@ -159,27 +94,21 @@ class TurkClassifierTransform : public Transform
     Q_OBJECT
     Q_PROPERTY(QString key READ get_key WRITE set_key RESET reset_key STORED false)
     Q_PROPERTY(QStringList values READ get_values WRITE set_values RESET reset_values STORED false)
-    Q_PROPERTY(float maxVotes READ get_maxVotes WRITE set_maxVotes RESET reset_maxVotes STORED false)
     Q_PROPERTY(bool isMeta READ get_isMeta WRITE set_isMeta RESET reset_isMeta STORED false)
     BR_PROPERTY(QString, key, QString())
     BR_PROPERTY(QStringList, values, QStringList())
-    BR_PROPERTY(float, maxVotes, 1)
     BR_PROPERTY(bool, isMeta, false)
     Transform *child;
     void init()
     {
-        QString algorithm = QString("Turk(%1, %2)+").arg(key, QString::number(maxVotes));
         QStringList classifiers;
         foreach (const QString &value, values)
-            classifiers.append(QString("SVM(RBF,EPS_SVR,returnDFVal=true,inputVariable=%1,outputVariable=predicted_%1)").arg(value));
-        algorithm += classifiers.join("/");
-        if (values.size() > 1)
-            algorithm += "+Cat";
+            classifiers.append(QString("SVM(RBF,EPS_SVR,returnDFVal=true,inputVariable=%1,outputVariable=predicted_%1)").arg(key + "_" + value));
+        child = Transform::make(classifiers.join("/") + (classifiers.size() > 1 ? "+Cat" : ""));
         if (isMeta)
             algorithm += QsString("+Average+SaveMat(predicted_%1)").arg(value);
-        child = Transform::make(algorithm);
     }
     void train(const QList<TemplateList> &data)
@@ -206,44 +135,6 @@ class TurkClassifierTransform : public Transform
 BR_REGISTER(Transform, TurkClassifierTransform)
 /*!
- * \ingroup transforms
- * \brief Converts metadata into a map structure
- * \author Scott Klum \cite sklum
- */
-class MapTransform : public UntrainableTransform
-{
-    Q_OBJECT
-    Q_PROPERTY(QStringList inputVariables READ get_inputVariables WRITE set_inputVariables RESET reset_inputVariables STORED false)
-    Q_PROPERTY(QString outputVariable READ get_outputVariable WRITE set_outputVariable RESET reset_outputVariable STORED false)
-    BR_PROPERTY(QStringList, inputVariables, QStringList())
-    BR_PROPERTY(QString, outputVariable, QString())
-
-    void project(const Template &src, Template &dst) const
-    {
-        dst = map(src);
-    }
-
-    Template map(const Template &t) const {
-        Template mappedT = t;
-        QMap<QString,QVariant> map;
-
-        foreach(const QString &s, inputVariables) {
-            if (t.file.contains(s)) {
-                map.insert(s,t.file.value(s));
-                mappedT.file.remove(s);
-            }
-        }
-
-        if (!map.isEmpty()) mappedT.file.set(outputVariable,map);
-
-        return mappedT;
-    }
-};
-
-BR_REGISTER(Transform, MapTransform)
-
-
-/*!
  * \ingroup distances
  * \brief Unmaps Turk HITs to be compared against query mats
  * \author Scott Klum \cite sklum
@@ -251,33 +142,17 @@ BR_REGISTER(Transform, MapTransform)
 class TurkDistance : public Distance
 {
     Q_OBJECT
-    Q_PROPERTY(QString HIT READ get_HIT WRITE set_HIT RESET reset_HIT)
-    Q_PROPERTY(QStringList keys READ get_keys WRITE set_keys RESET reset_keys STORED false)
-    Q_PROPERTY(float maxVotes READ get_maxVotes WRITE set_maxVotes RESET reset_maxVotes STORED false)
-    Q_PROPERTY(float maxRange READ get_maxRange WRITE set_maxRange RESET reset_maxRange STORED false)
-    Q_PROPERTY(float minRange READ get_minRange WRITE set_minRange RESET reset_minRange STORED false)
-    Q_PROPERTY(bool classify READ get_classify WRITE set_classify RESET reset_classify STORED false)
-    Q_PROPERTY(bool consensusOnly READ get_consensusOnly WRITE set_consensusOnly RESET reset_consensusOnly STORED false)
-    BR_PROPERTY(QString, HIT, QString())
-    BR_PROPERTY(QStringList, keys, QStringList())
-    BR_PROPERTY(float, maxVotes, 1)
-    BR_PROPERTY(float, maxRange, 1)
-    BR_PROPERTY(float, minRange, 0)
-    BR_PROPERTY(bool, classify, false)
-    BR_PROPERTY(bool, consensusOnly, false)
+    Q_PROPERTY(QString key READ get_key WRITE set_key RESET reset_key)
+    Q_PROPERTY(QStringList values READ get_values WRITE set_values RESET reset_values STORED false)
+    BR_PROPERTY(QString, key, QString())
+    BR_PROPERTY(QStringList, values, QStringList())
     float compare(const Template &target, const Template &query) const
     {
-        Template t = unmap(target, HIT, maxVotes, maxRange, minRange, classify, consensusOnly);
-
-        QList<float> targetValues;
-        foreach(const QString &s, keys) targetValues.append(t.file.get<float>(s));
-
-        float stddev = .75;
-
+        const float stddev = .75;
         float score = 0;
-        for (int i=0; i<targetValues.size(); i++) score += 1/(stddev*sqrt(2*CV_PI))*exp(-0.5*pow((query.m().at<float>(0,i)-targetValues[i])/stddev, 2));
-
+        for (int i=0; i<values.size(); i++)
+            score += 1 / (stddev*sqrt(2*CV_PI)) * exp(-0.5*pow((query.m().at<float>(0,i)-target.file.get<float>(key + "_" + values[i]))/stddev, 2));
         return score;
     }
 };