Further changes to label/subject

Remove global label/subject lookup table Consistently use "Subject" rather than "Label", subject is assumed to be convertable to QString. When desirable, map discrete subject values to ints. For classifiers such as svm that require numeric labels, generate a string->int mapping for the training data, and store it (local to the transform). Utility functions for collecting all values of a given property (on a template list), and mapping discrete property values to 0-based integers Some outstanding issues include use of label/subject in matrix output

Further changes to label/subject
Remove global label/subject lookup table Consistently use "Subject" rather than "Label", subject is assumed to be convertable to QString. When desirable, map discrete subject values to ints. For classifiers such as svm that require numeric labels, generate a string->int mapping for the training data, and store it (local to the transform). Utility functions for collecting all values of a given property (on a template list), and mapping discrete property values to 0-based integers Some outstanding issues include use of label/subject in matrix output
Charles Otto
1 parent 16039af1
Showing 22 changed files with 153 additions and 48 deletions
app/examples/age_estimation.cpp
openbr/core/bee.cpp
openbr/core/classify.cpp
openbr/core/cluster.cpp
openbr/core/core.cpp
openbr/core/opencvutils.cpp
openbr/core/opencvutils.h
openbr/frvt2012.cpp
openbr/gui/classifier.cpp
openbr/openbr_plugin.cpp
openbr/openbr_plugin.h
openbr/plugins/eigen3.cpp
openbr/plugins/gallery.cpp
openbr/plugins/independent.cpp
openbr/plugins/meta.cpp
openbr/plugins/normalize.cpp
openbr/plugins/output.cpp
openbr/plugins/pixel.cpp
openbr/plugins/quality.cpp
openbr/plugins/quantize.cpp
@@ -29,7 +29,7 @@
 static void printTemplate(const br::Template &t)
 {
-    printf("%s age: %d\n", qPrintable(t.file.fileName()), int(t.file.get<float>("Label")));
+    printf("%s age: %d\n", qPrintable(t.file.fileName()), int(t.file.get<float>("Subject")));
 }
 int main(int argc, char *argv[])
@@ -260,26 +260,28 @@ void BEE::makeMask(const QString &amp;targetInput, const QString &amp;queryInput, const 
 cv::Mat BEE::makeMask(const br::FileList &targets, const br::FileList &queries, int partition)
 {
-    QList<float> targetLabels = targets.collectValues<float>("Label");
-    QList<float> queryLabels = queries.collectValues<float>("Label");
+    // Would like to use indexProperty for this, but didn't make a version of that for Filelist yet
+    // -cao
+    QList<QString> targetLabels = targets.collectValues<QString>("Subject", "-1");
+    QList<QString> queryLabels = queries.collectValues<QString>("Subject", "-1");
     QList<int> targetPartitions = targets.crossValidationPartitions();
     QList<int> queryPartitions = queries.crossValidationPartitions();
     Mat mask(queries.size(), targets.size(), CV_8UC1);
     for (int i=0; i<queries.size(); i++) {
         const QString &fileA = queries[i];
-        const int labelA = queryLabels[i];
+        const QString labelA = queryLabels[i];
         const int partitionA = queryPartitions[i];
         for (int j=0; j<targets.size(); j++) {
             const QString &fileB = targets[j];
-            const int labelB = targetLabels[j];
+            const QString labelB = targetLabels[j];
             const int partitionB = targetPartitions[j];
             Mask_t val;
             if      (fileA == fileB)           val = DontCare;
-            else if (labelA == -1)             val = DontCare;
-            else if (labelB == -1)             val = DontCare;
+            else if (labelA == "-1")             val = DontCare;
+            else if (labelB == "-1")             val = DontCare;
             else if (partitionA != partition)  val = DontCare;
             else if (partitionB == -1)         val = NonMatch;
             else if (partitionB != partition)  val = DontCare;
@@ -104,9 +104,9 @@ void br::EvalRegression(const QString &amp;predictedInput, const QString &amp;truthInput
     for (int i=0; i<predicted.size(); i++) {
         if (predicted[i].file.name != truth[i].file.name)
             qFatal("Input order mismatch.");
-        rmsError += pow(predicted[i].file.get<float>("Label")-truth[i].file.get<float>("Label"), 2.f);
-        truthValues.append(QString::number(truth[i].file.get<float>("Label")));
-        predictedValues.append(QString::number(predicted[i].file.get<float>("Label")));
+        rmsError += pow(predicted[i].file.get<float>("Subject")-truth[i].file.get<float>("Subject"), 2.f);
+        truthValues.append(QString::number(truth[i].file.get<float>("Subject")));
+        predictedValues.append(QString::number(predicted[i].file.get<float>("Subject")));
     }
     QStringList rSource;
@@ -278,7 +278,9 @@ void br::EvalClustering(const QString &amp;csv, const QString &amp;input)
 {
     qDebug("Evaluating %s against %s", qPrintable(csv), qPrintable(input));
-    QList<float> labels = TemplateList::fromGallery(input).files().collectValues<float>("Label");
+    // We assume clustering algorithms store assigned cluster labels as integers (since the clusters are
+    // not named).
+    QList<int> labels = TemplateList::fromGallery(input).files().collectValues<int>("Subject");
     QHash<int, int> labelToIndex;
     int nClusters = 0;
@@ -78,7 +78,6 @@ struct AlgorithmCore
         const bool hasComparer = !distance.isNull();
         out << hasComparer;
         if (hasComparer) distance->store(out);
-        out << Globals->subjects;
         // Compress and save to file
         QtUtils::writeFile(model, data, -1);
@@ -98,7 +97,6 @@ struct AlgorithmCore
         transform->load(in);
         bool hasDistance; in >> hasDistance;
         if (hasDistance) distance->load(in);
-        in >> Globals->subjects;
     }
     File getMemoryGallery(const File &file) const
@@ -119,6 +119,17 @@ Mat OpenCVUtils::toMat(const QList&lt;float&gt; &amp;src, int rows)
     return dst;
 }
+Mat OpenCVUtils::toMat(const QList<int> &src, int rows)
+{
+    if (rows == -1) rows = src.size();
+    int columns = src.isEmpty() ? 0 : src.size() / rows;
+    if (rows*columns != src.size()) qFatal("Invalid matrix size.");
+    Mat dst(rows, columns, CV_32FC1);
+    for (int i=0; i<src.size(); i++)
+        dst.at<float>(i/columns,i%columns) = src[i];
+    return dst;
+}
+
 Mat OpenCVUtils::toMat(const QList<Mat> &src)
 {
     if (src.isEmpty()) return Mat();
@@ -35,6 +35,8 @@ namespace OpenCVUtils
     // To image
     cv::Mat toMat(const QList<float> &src, int rows = -1);
+    cv::Mat toMat(const QList<int> &src, int rows = -1);
+
     cv::Mat toMat(const QList<cv::Mat> &src);      // Data organized one matrix per row
     cv::Mat toMatByRow(const QList<cv::Mat> &src); // Data organized one row per row
@@ -132,7 +132,7 @@ int32_t SdkEstimator::estimate_age(const ONEFACE &amp;input_face, int32_t &amp;age)
     TemplateList templates;
     templates.append(templateFromONEFACE(input_face));
     templates >> *frvt2012_age_transform.data();
-    age = templates.first().file.get<float>("Label");
+    age = templates.first().file.get<float>("Subject");
     return templates.first().file.failed() ? 4 : 0;
 }
@@ -141,6 +141,7 @@ int32_t SdkEstimator::estimate_gender(const ONEFACE &amp;input_face, int8_t &amp;gender,
     TemplateList templates;
     templates.append(templateFromONEFACE(input_face));
     templates >> *frvt2012_gender_transform.data();
-    mf = gender = templates.first().file.get<float>("Label");
+    // TODO: lookup gender strings/expected int outputs -cao
+    mf = gender = templates.first().file.get<QString>("Subject") == "Male" ? 0 : 1;
     return templates.first().file.failed() ? 4 : 0;
 }
@@ -42,16 +42,15 @@ void Classifier::_classify(File file)
         if (!f.contains("Label"))
             continue;
-        // What's with the special casing -cao
         if (algorithm == "GenderClassification") {
             key = "Gender";
             value = (f.get<QString>("Subject"));
         } else if (algorithm == "AgeRegression") {
             key = "Age";
-            value = QString::number(int(f.get<float>("Label")+0.5)) + " Years";
+            value = QString::number(int(f.get<float>("Subject")+0.5)) + " Years";
         } else {
             key = algorithm;
-            value = QString::number(f.get<float>("Label"));
+            value = f.get<QString>("Subject");
         }
         break;
     }
@@ -435,11 +435,13 @@ TemplateList TemplateList::fromGallery(const br::File &amp;gallery)
     return templates;
 }
+// indexes some property, assigns an integer id to each unique value of propName
+// stores the index values in "Label" of the output template list -cao
 TemplateList TemplateList::relabel(const TemplateList &tl, const QString & propName)
 {
-    const QList<int> originalLabels = tl.collectValues<int>(propName);
-    QHash<int,int> labelTable;
-    foreach (int label, originalLabels)
+    const QList<QString> originalLabels = tl.collectValues<QString>(propName);
+    QHash<QString,int> labelTable;
+    foreach (const QString & label, originalLabels)
         if (!labelTable.contains(label))
             labelTable.insert(label, labelTable.size());
@@ -449,6 +451,52 @@ TemplateList TemplateList::relabel(const TemplateList &amp;tl, const QString &amp; propN
     return result;
 }
+QList<int> TemplateList::indexProperty(const QString & propName, QHash<QString, int> * valueMap,QHash<int, QVariant> * reverseLookup) const
+{
+    QHash<QString, int> dummyForwards;
+    QHash<int, QVariant> dummyBackwards;
+
+    if (!valueMap) valueMap = &dummyForwards;
+    if (!reverseLookup) reverseLookup = &dummyBackwards;
+
+    return indexProperty(propName, *valueMap, *reverseLookup);
+}
+
+QList<int> TemplateList::indexProperty(const QString & propName, QHash<QString, int> & valueMap, QHash<int, QVariant> & reverseLookup) const
+{
+    valueMap.clear();
+    reverseLookup.clear();
+
+    const QList<QVariant> originalLabels = collectValues<QVariant>(propName);
+    foreach (const QVariant & label, originalLabels) {
+        QString labelString = label.toString();
+        if (!valueMap.contains(labelString)) {
+            reverseLookup.insert(valueMap.size(), label);
+            valueMap.insert(labelString, valueMap.size());
+        }
+    }
+
+    QList<int> result;
+    for (int i=0; i<originalLabels.size(); i++)
+        result.append(valueMap[originalLabels[i].toString()]);
+
+    return result;
+}
+
+// uses -1 for missing values
+QList<int> TemplateList::applyIndex(const QString & propName, const QHash<QString, int> & valueMap) const
+{
+    const QList<QString> originalLabels = collectValues<QString>(propName);
+
+    QList<int> result;
+    for (int i=0; i<originalLabels.size(); i++) {
+        if (!valueMap.contains(originalLabels[i])) result.append(-1);
+        else result.append(valueMap[originalLabels[i]]);
+    }
+
+    return result;
+}
+
 /* Object - public methods */
 QStringList Object::parameters() const
 {
@@ -989,7 +1037,8 @@ QString MatrixOutput::toString(int row, int column) const
 {
     if (targetFiles[column] == "Subject") {
         const int label = data.at<float>(row,column);
-        return Globals->subjects.key(label, QString::number(label));
+        // problem -cao
+        return QString::number(label);
     }
     return QString::number(data.at<float>(row,column));
 }
@@ -306,6 +306,15 @@ struct BR_EXPORT FileList : public QList&lt;File&gt;
             values.append(f.get<T>(propName));
         return values;
     }
+    template<typename T>
+    QList<T> collectValues(const QString & propName, T defaultValue) const
+    {
+        QList<T> values; values.reserve(size());
+        foreach (const File &f, *this)
+            values.append(f.contains(propName) ? f.get<T>(propName) : defaultValue);
+        return values;
+    }
+
     QList<int> crossValidationPartitions() const; /*!< \brief Returns the cross-validation partition (default=0) for each file in the list. */
     int failures() const; /*!< \brief Returns the number of files with br::File::failed(). */
 };
@@ -393,6 +402,11 @@ struct TemplateList : public QList&lt;Template&gt;
     /*!< \brief Ensure labels are in the range [0,numClasses-1]. */
     BR_EXPORT static TemplateList relabel(const TemplateList & tl, const QString & propName);
+    QList<int> indexProperty(const QString & propName, QHash<QString, int> * valueMap=NULL,QHash<int, QVariant> * reverseLookup = NULL) const;
+    QList<int> indexProperty(const QString & propName, QHash<QString, int> & valueMap, QHash<int, QVariant> & reverseLookup) const;
+    QList<int> applyIndex(const QString & propName, const QHash<QString, int> & valueMap) const;
+
+
     /*!
      * \brief Returns the total number of bytes in all the templates.
      */
@@ -658,7 +672,6 @@ public:
     BR_PROPERTY(int, crossValidate, 0)
     QHash<QString,QString> abbreviations; /*!< \brief Used by br::Transform::make() to expand abbreviated algorithms into their complete definitions. */
-    QHash<QString,int> subjects; /*!< \brief Used by classifiers to associate text class labels with unique integers IDs. */
     QTime startTime; /*!< \brief Used to estimate timeRemaining(). */
     /*!
@@ -329,7 +329,8 @@ class LDATransform : public Transform
     void train(const TemplateList &_trainingSet)
     {
-        TemplateList trainingSet = TemplateList::relabel(_trainingSet, "Label");
+        // creates "Label" -cao
+        TemplateList trainingSet = TemplateList::relabel(_trainingSet, "Subject");
         int instances = trainingSet.size();
@@ -342,12 +343,13 @@ class LDATransform : public Transform
         TemplateList ldaTrainingSet;
         static_cast<Transform*>(&pca)->project(trainingSet, ldaTrainingSet);
+        // Reindex label, is this still necessary? -cao
         ldaTrainingSet = TemplateList::relabel(ldaTrainingSet, "Label");
         int dimsIn = ldaTrainingSet.first().m().rows * ldaTrainingSet.first().m().cols;
         // OpenBR ensures that class values range from 0 to numClasses-1.
-        // Assumed label is stored as float or int? -cao
+        // Label exists because we created it earlier with relabel
         QList<int> classes = trainingSet.collectValues<int>("Label");
         QMap<int, int> classCounts = trainingSet.countValues<int>("Label");
         const int numClasses = classCounts.size();
@@ -65,7 +65,7 @@ class arffGallery : public Gallery
             const int dimensions = t.m().rows * t.m().cols;
             for (int i=0; i<dimensions; i++)
                 arffFile.write(qPrintable("@ATTRIBUTE v" + QString::number(i) + " REAL\n"));
-            arffFile.write(qPrintable("@ATTRIBUTE class {" + QStringList(Globals->subjects.keys()).join(',') + "}\n"));
+            arffFile.write(qPrintable("@ATTRIBUTE class string\n"));
             arffFile.write("\n@DATA\n");
         }
@@ -518,9 +518,8 @@ class csvGallery : public Gallery
     static QString getCSVElement(const QString &key, const QVariant &value, bool header)
     {
         if ((key == "Label") && !header) {
-            QString stringLabel = Globals->subjects.key(value.value<int>());
-            if (stringLabel.isEmpty()) return value.value<QString>();
-            else                       return stringLabel;
+            // problem -cao
+            return value.value<QString>();
         } else if (value.canConvert<QString>()) {
             if (header) return key;
             else        return value.value<QString>();
@@ -20,21 +20,23 @@ static TemplateList Downsample(const TemplateList &amp;templates, const Transform *t
     const bool atLeast = transform->instances < 0;
     const int instances = abs(transform->instances);
-    QList<int> allLabels = templates.collectValues<int>("Label");
-    QList<int> uniqueLabels = allLabels.toSet().toList();
+    QList<QString> allLabels = templates.collectValues<QString>("Subject");
+    QList<QString> uniqueLabels = allLabels.toSet().toList();
     qSort(uniqueLabels);
-    QMap<int,int> counts = templates.countValues<int>("Label", instances != std::numeric_limits<int>::max());
+    QMap<QString,int> counts = templates.countValues<QString>("Subject", instances != std::numeric_limits<int>::max());
+
     if ((instances != std::numeric_limits<int>::max()) && (transform->classes != std::numeric_limits<int>::max()))
-        foreach (int label, counts.keys())
+        foreach (const QString & label, counts.keys())
             if (counts[label] < instances)
                 counts.remove(label);
+
     uniqueLabels = counts.keys();
     if ((transform->classes != std::numeric_limits<int>::max()) && (uniqueLabels.size() < transform->classes))
         qWarning("Downsample requested %d classes but only %d are available.", transform->classes, uniqueLabels.size());
     Common::seedRNG();
-    QList<int> selectedLabels = uniqueLabels;
+    QList<QString> selectedLabels = uniqueLabels;
     if (transform->classes < uniqueLabels.size()) {
         std::random_shuffle(selectedLabels.begin(), selectedLabels.end());
         selectedLabels = selectedLabels.mid(0, transform->classes);
@@ -42,7 +44,7 @@ static TemplateList Downsample(const TemplateList &amp;templates, const Transform *t
     TemplateList downsample;
     for (int i=0; i<selectedLabels.size(); i++) {
-        const int selectedLabel = selectedLabels[i];
+        const QString selectedLabel = selectedLabels[i];
         QList<int> indices;
         for (int j=0; j<allLabels.size(); j++)
             if ((allLabels[j] == selectedLabel) && (!templates.value(j).file.get<bool>("FTE", false)))
@@ -453,7 +453,8 @@ private:
         const QString &file = src.file;
         if (cache.contains(file)) {
             dst = cache[file];
-            dst.file.set("Label", src.file.value("Label"));
+            // don't get this -cao
+//            dst.file.set("Label", src.file.value("Label"));
         } else {
             transform->project(src, dst);
             cacheLock.lock();
@@ -126,7 +126,8 @@ private:
     {
         Mat m;
         OpenCVUtils::toMat(data.data()).convertTo(m, CV_64F);
-        const QList<int> labels = data.collectValues<int>("Label");
+
+        const QList<int> labels = data.indexProperty("Subject");
         const int dims = m.cols;
         vector<Mat> mv, av, bv;
@@ -153,8 +153,10 @@ class meltOutput : public MatrixOutput
         QStringList lines;
         if (file.baseName() != "terminal") lines.append(QString("Query,Target,Mask,Similarity%1").arg(keys));
-        QList<float> queryLabels = queryFiles.collectValues<float>("Label");
-        QList<float> targetLabels = targetFiles.collectValues<float>("Label");
+
+        QList<QString> queryLabels = queryFiles.collectValues<QString>("Subject");
+        QList<QString> targetLabels = targetFiles.collectValues<QString>("Subject");
+
         for (int i=0; i<queryFiles.size(); i++) {
             for (int j=(selfSimilar ? i+1 : 0); j<targetFiles.size(); j++) {
                 const bool genuine = queryLabels[i] == targetLabels[j];
@@ -28,6 +28,7 @@ namespace br
  */
 class PerPixelClassifierTransform : public MetaTransform
 {
+    // problematic -cao
     Q_OBJECT
     Q_PROPERTY(br::Transform* transform READ get_transform WRITE set_transform RESET reset_transform)
     Q_PROPERTY(int pixels READ get_pixels WRITE set_pixels RESET reset_pixels STORED false)
@@ -273,6 +274,7 @@ BR_REGISTER(Transform, ToBinaryVectorTransform)
  * \author E. Taborsky \cite mmtaborsky
  */
+// What does this do? -cao
 class ToMetadataTransform : public UntrainableMetaTransform
 {
     Q_OBJECT
@@ -26,10 +26,10 @@ class ImpostorUniquenessMeasureTransform : public Transform
     float calculateIUM(const Template &probe, const TemplateList &gallery) const
     {
-        const int probeLabel = probe.file.get<float>("Label");
+        const QString probeLabel = probe.file.get<QString>("Subject");
         TemplateList subset = gallery;
         for (int j=subset.size()-1; j>=0; j--)
-            if (subset[j].file.get<int>("Label") == probeLabel)
+            if (subset[j].file.get<QString>("Subject") == probeLabel)
                 subset.removeAt(j);
         QList<float> scores = distance->compare(subset, probe);
@@ -159,7 +159,7 @@ class MatchProbabilityDistance : public Distance
     {
         distance->train(src);
-        const QList<int> labels = src.collectValues<int>("Label");
+        const QList<int> labels = src.indexProperty("Subject");
         QScopedPointer<MatrixOutput> matrixOutput(MatrixOutput::make(FileList(src.size()), FileList(src.size())));
         distance->compare(src, src, matrixOutput.data());
@@ -219,7 +219,7 @@ class UnitDistance : public Distance
     void train(const TemplateList &templates)
     {
         const TemplateList samples = templates.mid(0, 2000);
-        const QList<float> sampleLabels = samples.collectValues<float>("Label");
+        const QList<int> sampleLabels = samples.indexProperty("Subject");
         QScopedPointer<MatrixOutput> matrixOutput(MatrixOutput::make(FileList(samples.size()), FileList(samples.size())));
         Distance::compare(samples, samples, matrixOutput.data());
@@ -150,7 +150,7 @@ class BayesianQuantizationDistance : public Distance
             qFatal("Expected sigle matrix templates of type CV_8UC1!");
         const Mat data = OpenCVUtils::toMat(src.data());
-        const QList<int> templateLabels = src.collectValues<int>("Label");
+        const QList<int> templateLabels = src.indexProperty("Subject");
         loglikelihoods = QVector<float>(data.cols*256, 0);
         QFutureSynchronizer<void> futures;
@@ -473,7 +473,8 @@ private:
     {
         Mat data = OpenCVUtils::toMat(src.data());
         const int step = getStep(data.cols);
-        const QList<int> labels = src.collectValues<int>("Label");
+
+        const QList<int> labels = src.indexProperty("Subject");
         Mat &lut = ProductQuantizationLUTs[index];
         lut = Mat(getDims(data.cols), 256*(256+1)/2, CV_32FC1);
@@ -77,7 +77,7 @@ class BayesianQuantizationTransform : public Transform
     void train(const TemplateList &src)
     {
         const Mat data = OpenCVUtils::toMat(src.data());
-        const QList<int> labels = src.collectValues<int>("Label");
+        const QList<int> labels = src.indexProperty("Subject");
         thresholds = QVector<float>(256*data.cols);
@@ -121,28 +121,46 @@ private:
     BR_PROPERTY(float, gamma, -1)
     SVM svm;
+    QHash<QString, int> labelMap;
+    QHash<int, QVariant> reverseLookup;
     void train(const TemplateList &_data)
     {
         Mat data = OpenCVUtils::toMat(_data.data());
-        Mat lab = OpenCVUtils::toMat(_data.collectValues<float>("Label"));
+        Mat lab;
+        // If we are doing regression, assume subject has float values
+        if (type == EPS_SVR || type == NU_SVR) {
+            lab = OpenCVUtils::toMat(_data.collectValues<float>("Subject"));
+        }
+        // If we are doing classification, assume subject has discrete values, map them
+        // and store the mapping data
+        else {
+            QList<int> dataLabels = _data.indexProperty("Subject", labelMap, reverseLookup);
+            lab = OpenCVUtils::toMat(dataLabels);
+        }
         trainSVM(svm, data, lab, kernel, type, C, gamma);
     }
     void project(const Template &src, Template &dst) const
     {
         dst = src;
-        dst.file.set("Label", svm.predict(src.m().reshape(1, 1)));
+        float prediction = svm.predict(src.m().reshape(1, 1));
+        if (type == EPS_SVR || type == NU_SVR)
+            dst.file.set("Subject", prediction);
+        else
+            dst.file.set("Subject", reverseLookup[prediction]);
     }
     void store(QDataStream &stream) const
     {
         storeSVM(svm, stream);
+        stream << labelMap << reverseLookup;
     }
     void load(QDataStream &stream)
     {
         loadSVM(svm, stream);
+        stream >> labelMap >> reverseLookup;
     }
 };