Merge branch 'csv'

Scott Klum
2 parents 4b927765 92de65c4
Showing 5 changed files with 194 additions and 72 deletions
openbr/core/evalutils.cpp
openbr/core/qtutils.cpp
openbr/core/qtutils.h
openbr/openbr_plugin.cpp
openbr/plugins/gallery/csv.cpp
@@ -263,7 +263,7 @@ QStringList EvalUtils::computeDetectionResults(const QList&lt;ResolvedDetection&gt; &amp;d
                         debug << QString("|") << QString::number(FAR, 'f', 4).leftJustified(10, ' ');
                         debug << QString("|") << QString::number(TP / totalTrueDetections, 'f', 4).leftJustified(10, ' ');
                         debug << QString("|") << QString::number(detection.confidence, 'f', 4).leftJustified(10, ' ');
-                        debug << QString("|") << QString::number(poseMatch / TP, 'f', 4).leftJustified(10, ' ');
+                        debug << QString("|") << QString::number(TP ? poseMatch / TP : 0., 'f', 4).leftJustified(10, ' ');
                         debug << QString("|") << endl;
                         break;
                     }
@@ -172,12 +172,20 @@ QString find(const QString &amp;file, const QString &amp;alt)
     return "";
 }
-bool toBool(const QString &string)
+bool toBool(const QString &string, bool *ok)
 {
-    bool ok;
-    bool result = (string.toFloat(&ok) != 0.f);
-    if (ok) return result;
-    else    return (string != "FALSE") && (string != "false") && (string != "F") && (string != "f");
+    bool floatOk;
+    bool result = (string.toFloat(&floatOk) != 0.f);
+    if (floatOk) {
+        if (ok) *ok = true;
+        return result;
+    } else {
+        if (ok) *ok = (string.compare("false", Qt::CaseInsensitive) == 0 ||
+                       string.compare("true", Qt::CaseInsensitive)  == 0 ||
+                       string.compare("f", Qt::CaseInsensitive)     == 0 ||
+                       string.compare("t", Qt::CaseInsensitive)     == 0);
+        return (string.compare("false", Qt::CaseInsensitive) != 0 && string.compare("f", Qt::CaseInsensitive) != 0);
+    }
 }
 int toInt(const QString &string)
@@ -245,10 +253,10 @@ QStringList parse(QString args, char split, bool *ok)
     QStack<QChar> subexpressions;
     for (int i=0; i<args.size(); i++) {
         if (inQuote) {
-            if (args[i] == '\'')
+            if (args[i] == '\'' || args[i] == '\"')
                 inQuote = false;
         } else {
-            if (args[i] == '\'') {
+            if (args[i] == '\'' || args[i] == '\"') {
                 inQuote = true;
             } else if ((args[i] == '(') || (args[i] == '[') || (args[i] == '<') || (args[i] == '{')) {
                 subexpressions.push(args[i]);
@@ -277,7 +285,10 @@ QStringList parse(QString args, char split, bool *ok)
                     return words;
                 }
             } else if (subexpressions.isEmpty() && (args[i] == split)) {
-                words.append(args.mid(start, i-start).trimmed());
+                QString word = args.mid(start, i-start).trimmed();
+                if (word.contains('\'') || word.contains('\"'))
+                    word = word.mid(1, word.size()-2);
+                words.append(word);
                 start = i+1;
             }
         }
@@ -512,6 +523,13 @@ QVariantMap fromJsonObject(const QJsonObject &amp;object)
 QVariant fromString(const QString &value)
 {
+    if (value.startsWith('[') && value.endsWith(']')) {
+        QVariantList variants;
+        foreach (const QString &value, QtUtils::parse(value.mid(1, value.size()-2)))
+            variants.append(fromString(value));
+        return variants;
+    }
+
     bool ok = false;
     const QPointF point = QtUtils::toPoint(value, &ok);
     if (ok) return point;
@@ -523,6 +541,8 @@ QVariant fromString(const QString &amp;value)
     if (ok) return i;
     const float f = value.toFloat(&ok);
     if (ok) return f;
+    const bool b = QtUtils::toBool(value, &ok);
+    if (ok) return b;
     return value;
 }
@@ -56,7 +56,7 @@ namespace QtUtils
     QString getAbsolutePath(const QString &filename);
     /**** String Utilities ****/
-    bool toBool(const QString &string);
+    bool toBool(const QString &string, bool *ok = NULL);
     int toInt(const QString &string);
     float toFloat(const QString &string);
     QList<float> toFloats(const QStringList &strings);
@@ -156,30 +156,12 @@ QVariant File::value(const QString &amp;key) const
 QVariant File::parse(const QString &value)
 {
-    bool ok = false;
-    const QPointF point = QtUtils::toPoint(value, &ok);
-    if (ok) return point;
-    const QRectF rect = QtUtils::toRect(value, &ok);
-    if (ok) return rect;
-    const cv::RotatedRect rotatedRect = OpenCVUtils::rotateRectFromString(value, &ok);
-    if (ok) return QVariant::fromValue(rotatedRect);
-    const int i = value.toInt(&ok);
-    if (ok) return i;
-    const float f = value.toFloat(&ok);
-    if (ok) return f;
-    return value;
+    return QtUtils::fromString(value);
 }
 void File::set(const QString &key, const QString &value)
 {
-    if (value.startsWith('[') && value.endsWith(']')) {
-        QVariantList variants;
-        foreach (const QString &value, QtUtils::parse(value.mid(1, value.size()-2)))
-            variants.append(parse(value));
-        set(key, variants);
-    } else {
-        set(key, QVariant(parse(value)));
-    }
+    set(key, QtUtils::fromString(value));
 }
 bool File::getBool(const QString &key, bool defaultValue) const
@@ -16,10 +16,101 @@
 #include <openbr/plugins/openbr_internal.h>
 #include <openbr/core/qtutils.h>
+#include <openbr/core/common.h>
 namespace br
 {
+struct CSVHeader
+{
+  QList<int> indices;
+
+  CSVHeader()
+  {}
+
+  CSVHeader(const QString &key)
+    : key(key)
+  {}
+  
+  QString key;
+  QStringList subKeys;  
+};
+  
+class CSVHeaderList : public QList<CSVHeader>
+{
+public:
+  CSVHeaderList()
+  {}
+
+  CSVHeaderList(const QList<CSVHeader> &headers)
+  {
+    foreach (const CSVHeader &header, headers)
+      append(header);
+  }
+
+  CSVHeaderList(const QStringList &keys)
+  {
+    foreach (const QString &key, keys)
+      append(CSVHeader(key));
+  }
+  
+  void sort()
+  {
+    typedef QPair<QString, int> IndexPair;
+    QList<IndexPair> sortedKeys = Common::Sort(keys());
+
+    CSVHeaderList sortedList;
+    foreach (const IndexPair sortedKey, sortedKeys)
+      sortedList.append((*this)[sortedKey.second]);
+    *this = sortedList;
+  }
+
+  QStringList keys() const
+  {
+    QStringList keys;
+    for (int i=0; i<this->size(); i++)
+      keys.append((*this)[i].key);
+    return keys;
+  }
+
+    static CSVHeaderList fromHeaders(const QStringList &headers)
+  {
+    CSVHeaderList csvHeaders;
+    QStringList processedKeys;
+    
+    for (int i=0; i<headers.size(); i++) {
+      CSVHeader header;
+      if (headers[i].contains("_")) {
+	const QStringList subKeys = headers[i].split("_");
+	header.key = subKeys.first();
+	
+	if (processedKeys.contains(header.key))
+	  continue;
+	else
+	  processedKeys.append(header.key);
+	
+	header.subKeys.append(subKeys.last());
+	header.indices.append(i);
+	
+	// Look for other subheaders with the same key
+	for (int j=i+1; j<headers.size(); j++)
+	  if (headers[j].contains("_")) {
+	    const QStringList subKeys = headers[j].split("_");
+	    if (subKeys.first() == header.key && !header.subKeys.contains(subKeys.last()) /* Check for ill-formed csvs */) {
+	      header.indices.append(j);
+	      header.subKeys.append(subKeys.last());
+	    }
+	  }
+      } else {
+	header.key = headers[i];
+	header.indices.append(i);
+      }
+      csvHeaders.append(header);
+    }
+    return csvHeaders;
+  }
+};
+  
 /*!
  * \ingroup galleries
  * \brief Treats each line as a file.
@@ -35,10 +126,12 @@ class csvGallery : public FileGallery
     Q_OBJECT
     Q_PROPERTY(bool inPlace READ get_inPlace WRITE set_inPlace RESET reset_inPlace STORED false)
     BR_PROPERTY(bool, inPlace, false)
+    Q_PROPERTY(bool combineFiles READ get_combineFiles WRITE set_combineFiles RESET reset_combineFiles STORED false)
+    BR_PROPERTY(bool, combineFiles, false)
     FileList files;
-    QStringList headers;
-
+    CSVHeaderList headers;
+  
     ~csvGallery()
     {
         f.close();
@@ -53,9 +146,9 @@ class csvGallery : public FileGallery
         lines.reserve(files.size()+1);
         // Make header
-        headers = samples.values();
+        headers = CSVHeaderList(samples.values());
         headers.sort();
-        lines.append(QStringList(QStringList("File") + headers).join(","));
+        lines.append(QStringList(QStringList("File") + headers.keys()).join(","));
         // Make table
         foreach (const File &file, files)
@@ -64,6 +157,37 @@ class csvGallery : public FileGallery
         QtUtils::writeFile(file, lines);
     }
+    void setValuesFromHeaders(File &f, const CSVHeaderList &headers, const QVariantList &values)
+    {
+        foreach (const CSVHeader &header, headers) {
+            if (header.indices.size() == 1) {
+                if (header.key == "Rects")
+                    foreach(const QVariant &rect, values[header.indices.first()].toList())
+                        f.appendRect(rect.toRectF());
+                else if (header.key == "Points")
+                    foreach(const QVariant &point, values[header.indices.first()].toList())
+                        f.appendPoint(point.toPointF());
+                else {
+                    const QVariant value = values[header.indices.first()];
+                    if (!value.canConvert<QString>() || !value.toString().isEmpty())
+                        f.set(header.key, values[header.indices.first()]);
+                }
+            } else if (header.indices.size() == 2) { // QPointF
+                const QPointF point(values[header.indices[header.subKeys.indexOf("X")]].toFloat(),
+                                    values[header.indices[header.subKeys.indexOf("Y")]].toFloat());
+                f.set(header.key, point);
+                f.appendPoint(point);
+            } else if (header.indices.size() == 4) { // QRectF
+                const QRectF rect(values[header.indices[header.subKeys.indexOf("X")]].toFloat(),
+                                  values[header.indices[header.subKeys.indexOf("Y")]].toFloat(),
+                                  values[header.indices[header.subKeys.indexOf("Width")]].toFloat(),
+                                  values[header.indices[header.subKeys.indexOf("Height")]].toFloat());
+                f.set(header.key, rect);
+                f.appendRect(rect);
+            }
+        }
+    }
+  
     TemplateList readBlock(bool *done)
     {
         readOpen();
@@ -79,22 +203,41 @@ class csvGallery : public FileGallery
             QByteArray lineBytes = f.readLine();
             QString line = QString::fromLocal8Bit(lineBytes).trimmed();
             QRegExp regexp("\\s*,\\s*");
-            headers = line.split(regexp);
+            headers = CSVHeaderList::fromHeaders(line.split(regexp).mid(1));
         }
-        for (qint64 i = 0; i < this->readBlockSize && !f.atEnd(); i++) {
-            const QVariantList values = parseLine(f.readLine());
-            if (values.size() != headers.size()) continue;
+	if (combineFiles) {
+  	    *done = true;
+ 	    QMap<QString, File> combinedFiles;
+	  
+  	    while (!f.atEnd()) {
+                QVariantList values;
+                foreach (const QString &value, QtUtils::parse(f.readLine(), ','))
+                    values.append(QtUtils::fromString(value));
-            File in;
-            for (int j=0; j<values.size(); j++) {
-                if (j == 0) in.name = values[j].toString();
-                else        in.set(headers[j], values[j].toString());
-            }
-            in.set("progress", f.pos());
-            templates.append(in);
+	       const QString name = values.first().toString();
+	       File &in = combinedFiles[name];
+	       in.name = name;
+	       setValuesFromHeaders(in, headers, values.mid(1));
+	    }
+
+	    foreach (const File &in, combinedFiles.values())
+	      templates.append(in);
+	} else {
+  	    for (qint64 i = 0; i < this->readBlockSize && !f.atEnd(); i++) {
+                QVariantList values;
+                foreach (const QString &value, QtUtils::parse(f.readLine(), ','))
+                    values.append(QtUtils::fromString(value));
+
+               File in;
+	       in.name = values.first().toString();
+	       setValuesFromHeaders(in, headers, values.mid(1));
+	       in.set("progress", f.pos());
+	       templates.append(in);
+	    }
+	    *done = f.atEnd();
         }
-        *done = f.atEnd();
+       
         return templates;
     }
@@ -104,10 +247,10 @@ class csvGallery : public FileGallery
             writeOpen();
             if (headers.isEmpty()) {
                 foreach (const QString &key, t.file.localKeys())
-                    headers.append(key);
+		  headers.append(CSVHeader(key));
                 headers.sort();
-                const QString header = QString(QStringList(QStringList("File") + headers).join(",") + "\n");
+                const QString header = QString(QStringList(QStringList("File") + headers.keys()).join(",") + "\n");
                 f.write(header.toLocal8Bit());
             }
             f.write(QString(lineFromFile(t.file) + "\n").toLocal8Bit());
@@ -119,7 +262,7 @@ class csvGallery : public FileGallery
     {
         QStringList words;
         words.append(file.name);
-        foreach (const QString &key, headers) {
+        foreach (const QString &key, headers.keys()) {
             QString value = QtUtils::toString(file.value(key));
             if (value.contains(","))
                 value = '"' + value + '"';
@@ -127,29 +270,6 @@ class csvGallery : public FileGallery
         }
         return words.join(",");
     }
-
-    static QVariantList parseLine(const QByteArray bytes)
-    {
-        bool inQuote(false);
-        QVariantList values;
-        QString value = QString();
-        const QString line = QString::fromLocal8Bit(bytes).trimmed();
-        for (int i=0; i<line.size(); i++) {
-            const QChar c = line[i];
-            if (c == '"') {
-                inQuote = !inQuote;
-                continue;
-            } else if (c == ',' && !inQuote) {
-                values.append(QVariant(value));
-                value = QString();
-            } else
-                value.append(c);
-        }
-        if (!value.isEmpty())
-            values.append(QVariant(value));
-
-        return values;
-    }
 };
 BR_REGISTER(Gallery, csvGallery)