Using R for statistics computations, finished error table

Scott Klum
1 parent 259aa3f0
Showing 2 changed files with 22 additions and 32 deletions
openbr/core/eval.cpp
openbr/core/plot.cpp
@@ -1056,6 +1056,7 @@ float EvalLandmarking(const QString &amp;predictedGallery, const QString &amp;truthGalle
         }
         while (pointErrors.size() < predictedPoints.size())
             pointErrors.append(QList<float>());
+
         if (normalizationIndexA >= truthPoints.size()) qFatal("Normalization index A is out of range.");
         if (normalizationIndexB >= truthPoints.size()) qFatal("Normalization index B is out of range.");
         const float normalizedLength = QtUtils::euclideanLength(truthPoints[normalizationIndexB] - truthPoints[normalizationIndexA]);
@@ -1068,8 +1069,6 @@ float EvalLandmarking(const QString &amp;predictedGallery, const QString &amp;truthGalle
     qDebug() << "Skipped" << skipped << "files due to point size mismatch.";
     QList<float> averagePointErrors; averagePointErrors.reserve(pointErrors.size());
-    QList<float> medianPointErrors; medianPointErrors.reserve(pointErrors.size());
-    QList<float> stddevPointErrors; stddevPointErrors.reserve(pointErrors.size());
     float normalizedErrorLimit = 1.5;
@@ -1082,15 +1081,9 @@ float EvalLandmarking(const QString &amp;predictedGallery, const QString &amp;truthGalle
                     worstExamples.insert(exampleIndices[j].second);
         }
         std::sort(pointErrors[i].begin(), pointErrors[i].end());
-        double mean, stddev;
-        Common::MeanStdDev(pointErrors[i],&mean,&stddev);
-        averagePointErrors.append(mean);
-        stddevPointErrors.append(stddev);
-        medianPointErrors.append(Common::Median(pointErrors[i]));
+        averagePointErrors.append(Common::Mean(pointErrors[i]));
     }
     const float averagePointError = Common::Mean(averagePointErrors);
-    const float medianPointError = Common::Mean(medianPointErrors);
-    const float stddevPointError = Common::Mean(stddevPointErrors);
     QStringList lines;
     lines.append("Plot,X,Y");
@@ -1103,17 +1096,6 @@ float EvalLandmarking(const QString &amp;predictedGallery, const QString &amp;truthGalle
     // Alternatively, can we just pass this through a predetermined transform and write?
     Enroll(truth[sampleIndex],"landmarking_examples");
-    // Error table
-    for (int i=0; i<averagePointErrors.size(); i++) {
-        lines.append(QString("AE,%1,%2").arg(QString::number(i),QString::number(averagePointErrors[i], 'f', 3)));
-        lines.append(QString("ME,%1,%2").arg(QString::number(i),QString::number(medianPointErrors[i], 'f', 3)));
-        lines.append(QString("SE,%1,%2").arg(QString::number(i),QString::number(stddevPointErrors[i], 'f', 3)));
-    }
-
-    lines.append(QString("AE,%1,%2").arg(QString::number(averagePointErrors.size()),QString::number(averagePointError, 'f', 3)));
-    lines.append(QString("ME,%1,%2").arg(QString::number(averagePointErrors.size()),QString::number(medianPointError, 'f', 3)));
-    lines.append(QString("SE,%1,%2").arg(QString::number(averagePointErrors.size()),QString::number(stddevPointError, 'f', 3)));
-
     for (int i=0; i<pointErrors.size(); i++) {
         const QList<float> &pointError = pointErrors[i];
         const int keep = qMin(Max_Points, pointError.size());
@@ -1122,13 +1104,11 @@ float EvalLandmarking(const QString &amp;predictedGallery, const QString &amp;truthGalle
     }
     lines.append(QString("AvgError,0,%1").arg(averagePointError));
+    lines.append(QString("NormLength,0,%1").arg(Common::Mean(normalizedLengths)));
     QtUtils::writeFile(csv, lines);
     qDebug("Average Error for all Points: %.3f", averagePointError);
-    qDebug("Average Median Error for all Points: %.3f", medianPointError);
-    qDebug("Average Standard Deviation of Error for all Points: %.3f", stddevPointError);
-    qDebug("Average Normalization Length (pixels): %.3f", Common::Mean(normalizedLengths));
     return averagePointError;
 }
@@ -469,15 +469,23 @@ bool PlotLandmarking(const QStringList &amp;files, const File &amp;destination, bool sho
     qDebug("Plotting %d landmarking file(s) to %s", files.size(), qPrintable(destination));
     RPlot p(files, destination, false);
+    qDebug() << p.major.header << p.minor.header;
+
     p.file.write(qPrintable(QString("# Split data into individual plots\n"
                  "plot_index = which(names(data)==\"Plot\")\n"
                  "Box <- data[grep(\"Box\",data$Plot),-c(1)]\n"
+                 "Box$X <- factor(Box$X, levels = Box$X, ordered = TRUE)\n"
                  "EX <- data[grep(\"EX\",data$Plot),-c(1)]\n"
-                 "AE <- data[grep(\"AE\",data$Plot),-c(1)]\n"
-                 "ME <- data[grep(\"ME\",data$Plot),-c(1)]\n"
-                 "SE <- data[grep(\"SE\",data$Plot),-c(1)]\n"
+                 "NormLength <- data[grep(\"NormLength\",data$Plot),-c(1)]\n"
                  "EX$X <- as.character(EX$X)\n"
                  "EX$Y <- as.character(EX$Y)\n"
+                                    "\n"
+                                    "\n\tsummarySE <- function(data=NULL, measurevar, groupvars=NULL, na.rm=FALSE, conf.interval=.95, .drop=TRUE) {\n\t\t"
+                                    "require(plyr)\n\n\t\tlength2 <- function (x, na.rm=FALSE) {\n\t\t\tif (na.rm) sum(!is.na(x))\n\t\t\telse       length(x)"
+                                    "\n\t\t}\n\n\t\tdatac <- ddply(data, groupvars, .drop=.drop, .fun = function(xx, col) {\n\t\t\t"
+                                    "c(N=length2(xx[[col]], na.rm=na.rm), mean=mean(xx[[col]], na.rm=na.rm), sd=sd(xx[[col]], na.rm=na.rm))\n\t\t\t},"
+                                    "\n\t\t\tmeasurevar\n\t\t)\n\n\t\tdatac <- rename(datac, c(\"mean\" = measurevar))\n\t\tdatac$se <- datac$sd / sqrt(datac$N)"
+                                    "\n\t\tciMult <- qt(conf.interval/2 + .5, datac$N-1)\n\t\tdatac$ci <- datac$se * ciMult\n\n\t\treturn(datac)\n\t}\n\t"
                  "rm(data)\n"
                  "\n")));
@@ -535,22 +543,24 @@ bool PlotLandmarking(const QStringList &amp;files, const File &amp;destination, bool sho
     p.file.write(qPrintable(QString("\n"
                  "# Code to format error table\n"
-                 "l <- list(AE$Y,ME$Y,SE$Y)\n"
-                 "mat <- matrix(do.call(rbind, l),nrow=nrow(AE),ncol=3,byrow=TRUE)\n"
-                 "colnames(mat) <- c(\"Mean\",\"Median\",\"Std. Dev.\") \n"
-                 "rownames(mat) <- c(seq(0,nrow(AE)-2),\"Average\")\n"
+                 "StatBox <- summarySE(Box, measurevar=\"Y\", groupvars=c(\"X\"))\n\t"
+                 "OverallStatBox <- summarySE(Box, measurevar=\"Y\")\n"
+                 "mat <- matrix(paste(as.character(round(StatBox$Y, 3)), round(StatBox$ci, 3), sep=\" \\u00b1 \"),nrow=nrow(StatBox),ncol=1,byrow=TRUE)\n"
+                 "mat <- rbind(mat, paste(as.character(round(OverallStatBox$Y, 3)), round(OverallStatBox$ci, 3), sep=\" \\u00b1 \"))\n"
+                 "colnames(mat) <- c(\"Error Rate\")\n"
+                 "rownames(mat) <- c(seq(0,nrow(StatBox)-1),\"Aggregate\")\n"
                  "ETable <- as.table(mat)\n")));
     p.file.write(qPrintable(QString("\n"
                        "print(textplot(ETable))\n"
-                       "print(title(expression(atop(\"Landmark Error Rates\", atop(italic(\"Average Normalization Distance: 91.419 (pixels)\"))))))\n")));
+                       "print(title(sprintf(\"Landmark Error Rates\\nAverage Normalization Distance: %.3f (pixels)\",NormLength$Y)))\n")));
     p.file.write(qPrintable(QString("ggplot(Box, aes(Y,%1%2))").arg(p.major.size > 1 ? QString(", colour=%1").arg(p.major.header) : QString(),
                                                                     p.minor.size > 1 ? QString(", linetype=%1").arg(p.minor.header) : QString()) +
                             QString(" + annotation_logticks(sides=\"b\") + stat_ecdf() + scale_x_log10(\"Normalized Error\", breaks=c(0.001,0.01,0.1,1,10)) + scale_y_continuous(\"Cumulative Density\", label=percent) + theme_minimal()\n\n")));
     p.file.write(qPrintable(QString("ggplot(Box, aes(factor(X), Y%1%2))").arg(p.major.size > 1 ? QString(", colour=%1").arg(p.major.header) : QString(), p.minor.size > 1 ? QString(", linetype=%1").arg(p.minor.header) : QString()) +
-                            QString("+ annotation_logticks(sides=\"l\") + geom_boxplot(alpha=0.5) + geom_jitter(size=1, alpha=0.5) + scale_x_discrete(\"Landmark\") + scale_y_log10(\"Normalized Error\", breaks=c(0.01,0.1,1,10)) + theme_minimal()\n\n")));
+                            QString("+ annotation_logticks(sides=\"l\") + geom_boxplot(alpha=0.5) + geom_jitter(size=1, alpha=0.5) + scale_x_discrete(\"Landmark\") + scale_y_log10(\"Normalized Error\", breaks=c(0.001,0.01,0.1,1,10)) + theme_minimal()\n\n")));
     p.file.write(qPrintable(QString("ggplot(Box, aes(factor(X), Y%1%2))").arg(p.major.size > 1 ? QString(", colour=%1").arg(p.major.header) : QString(), p.minor.size > 1 ? QString(", linetype=%1").arg(p.minor.header) : QString()) +
                             QString("+ annotation_logticks(sides=\"l\") + geom_violin(alpha=0.5) + scale_x_discrete(\"Landmark\") + scale_y_log10(\"Normalized Error\", breaks=c(0.001,0.01,0.1,1,10))\n\n")));