Peter M. Groen / openbr

Commit 5b53a8c471a6ceae9427502b261270228c110db7

Authored by Scott Klum
1 parent 259aa3f0

Using R for statistics computations, finished error table

Inline Side-by-side
Showing 2 changed files with 22 additions and 32 deletions
openbr/core/eval.cpp
  View file @5b53a8c
... ... @@ -1056,6 +1056,7 @@ float EvalLandmarking(const QString &predictedGallery, const QString &truthGalle
1056 1056 }
1057 1057 while (pointErrors.size() < predictedPoints.size())
1058 1058 pointErrors.append(QList<float>());
  1059 +
1059 1060 if (normalizationIndexA >= truthPoints.size()) qFatal("Normalization index A is out of range.");
1060 1061 if (normalizationIndexB >= truthPoints.size()) qFatal("Normalization index B is out of range.");
1061 1062 const float normalizedLength = QtUtils::euclideanLength(truthPoints[normalizationIndexB] - truthPoints[normalizationIndexA]);
... ... @@ -1068,8 +1069,6 @@ float EvalLandmarking(const QString &amp;predictedGallery, const QString &amp;truthGalle
1068 1069 qDebug() << "Skipped" << skipped << "files due to point size mismatch.";
1069 1070  
1070 1071 QList<float> averagePointErrors; averagePointErrors.reserve(pointErrors.size());
1071   - QList<float> medianPointErrors; medianPointErrors.reserve(pointErrors.size());
1072   - QList<float> stddevPointErrors; stddevPointErrors.reserve(pointErrors.size());
1073 1072  
1074 1073 float normalizedErrorLimit = 1.5;
1075 1074  
... ... @@ -1082,15 +1081,9 @@ float EvalLandmarking(const QString &amp;predictedGallery, const QString &amp;truthGalle
1082 1081 worstExamples.insert(exampleIndices[j].second);
1083 1082 }
1084 1083 std::sort(pointErrors[i].begin(), pointErrors[i].end());
1085   - double mean, stddev;
1086   - Common::MeanStdDev(pointErrors[i],&mean,&stddev);
1087   - averagePointErrors.append(mean);
1088   - stddevPointErrors.append(stddev);
1089   - medianPointErrors.append(Common::Median(pointErrors[i]));
  1084 + averagePointErrors.append(Common::Mean(pointErrors[i]));
1090 1085 }
1091 1086 const float averagePointError = Common::Mean(averagePointErrors);
1092   - const float medianPointError = Common::Mean(medianPointErrors);
1093   - const float stddevPointError = Common::Mean(stddevPointErrors);
1094 1087  
1095 1088 QStringList lines;
1096 1089 lines.append("Plot,X,Y");
... ... @@ -1103,17 +1096,6 @@ float EvalLandmarking(const QString &amp;predictedGallery, const QString &amp;truthGalle
1103 1096 // Alternatively, can we just pass this through a predetermined transform and write?
1104 1097 Enroll(truth[sampleIndex],"landmarking_examples");
1105 1098  
1106   - // Error table
1107   - for (int i=0; i<averagePointErrors.size(); i++) {
1108   - lines.append(QString("AE,%1,%2").arg(QString::number(i),QString::number(averagePointErrors[i], 'f', 3)));
1109   - lines.append(QString("ME,%1,%2").arg(QString::number(i),QString::number(medianPointErrors[i], 'f', 3)));
1110   - lines.append(QString("SE,%1,%2").arg(QString::number(i),QString::number(stddevPointErrors[i], 'f', 3)));
1111   - }
1112   -
1113   - lines.append(QString("AE,%1,%2").arg(QString::number(averagePointErrors.size()),QString::number(averagePointError, 'f', 3)));
1114   - lines.append(QString("ME,%1,%2").arg(QString::number(averagePointErrors.size()),QString::number(medianPointError, 'f', 3)));
1115   - lines.append(QString("SE,%1,%2").arg(QString::number(averagePointErrors.size()),QString::number(stddevPointError, 'f', 3)));
1116   -
1117 1099 for (int i=0; i<pointErrors.size(); i++) {
1118 1100 const QList<float> &pointError = pointErrors[i];
1119 1101 const int keep = qMin(Max_Points, pointError.size());
... ... @@ -1122,13 +1104,11 @@ float EvalLandmarking(const QString &amp;predictedGallery, const QString &amp;truthGalle
1122 1104 }
1123 1105  
1124 1106 lines.append(QString("AvgError,0,%1").arg(averagePointError));
  1107 + lines.append(QString("NormLength,0,%1").arg(Common::Mean(normalizedLengths)));
1125 1108  
1126 1109 QtUtils::writeFile(csv, lines);
1127 1110  
1128 1111 qDebug("Average Error for all Points: %.3f", averagePointError);
1129   - qDebug("Average Median Error for all Points: %.3f", medianPointError);
1130   - qDebug("Average Standard Deviation of Error for all Points: %.3f", stddevPointError);
1131   - qDebug("Average Normalization Length (pixels): %.3f", Common::Mean(normalizedLengths));
1132 1112  
1133 1113 return averagePointError;
1134 1114 }
... ...
openbr/core/plot.cpp
  View file @5b53a8c
... ... @@ -469,15 +469,23 @@ bool PlotLandmarking(const QStringList &amp;files, const File &amp;destination, bool sho
469 469 qDebug("Plotting %d landmarking file(s) to %s", files.size(), qPrintable(destination));
470 470 RPlot p(files, destination, false);
471 471  
  472 + qDebug() << p.major.header << p.minor.header;
  473 +
472 474 p.file.write(qPrintable(QString("# Split data into individual plots\n"
473 475 "plot_index = which(names(data)==\"Plot\")\n"
474 476 "Box <- data[grep(\"Box\",data$Plot),-c(1)]\n"
  477 + "Box$X <- factor(Box$X, levels = Box$X, ordered = TRUE)\n"
475 478 "EX <- data[grep(\"EX\",data$Plot),-c(1)]\n"
476   - "AE <- data[grep(\"AE\",data$Plot),-c(1)]\n"
477   - "ME <- data[grep(\"ME\",data$Plot),-c(1)]\n"
478   - "SE <- data[grep(\"SE\",data$Plot),-c(1)]\n"
  479 + "NormLength <- data[grep(\"NormLength\",data$Plot),-c(1)]\n"
479 480 "EX$X <- as.character(EX$X)\n"
480 481 "EX$Y <- as.character(EX$Y)\n"
  482 + "\n"
  483 + "\n\tsummarySE <- function(data=NULL, measurevar, groupvars=NULL, na.rm=FALSE, conf.interval=.95, .drop=TRUE) {\n\t\t"
  484 + "require(plyr)\n\n\t\tlength2 <- function (x, na.rm=FALSE) {\n\t\t\tif (na.rm) sum(!is.na(x))\n\t\t\telse length(x)"
  485 + "\n\t\t}\n\n\t\tdatac <- ddply(data, groupvars, .drop=.drop, .fun = function(xx, col) {\n\t\t\t"
  486 + "c(N=length2(xx[[col]], na.rm=na.rm), mean=mean(xx[[col]], na.rm=na.rm), sd=sd(xx[[col]], na.rm=na.rm))\n\t\t\t},"
  487 + "\n\t\t\tmeasurevar\n\t\t)\n\n\t\tdatac <- rename(datac, c(\"mean\" = measurevar))\n\t\tdatac$se <- datac$sd / sqrt(datac$N)"
  488 + "\n\t\tciMult <- qt(conf.interval/2 + .5, datac$N-1)\n\t\tdatac$ci <- datac$se * ciMult\n\n\t\treturn(datac)\n\t}\n\t"
481 489 "rm(data)\n"
482 490 "\n")));
483 491  
... ... @@ -535,22 +543,24 @@ bool PlotLandmarking(const QStringList &amp;files, const File &amp;destination, bool sho
535 543  
536 544 p.file.write(qPrintable(QString("\n"
537 545 "# Code to format error table\n"
538   - "l <- list(AE$Y,ME$Y,SE$Y)\n"
539   - "mat <- matrix(do.call(rbind, l),nrow=nrow(AE),ncol=3,byrow=TRUE)\n"
540   - "colnames(mat) <- c(\"Mean\",\"Median\",\"Std. Dev.\") \n"
541   - "rownames(mat) <- c(seq(0,nrow(AE)-2),\"Average\")\n"
  546 + "StatBox <- summarySE(Box, measurevar=\"Y\", groupvars=c(\"X\"))\n\t"
  547 + "OverallStatBox <- summarySE(Box, measurevar=\"Y\")\n"
  548 + "mat <- matrix(paste(as.character(round(StatBox$Y, 3)), round(StatBox$ci, 3), sep=\" \\u00b1 \"),nrow=nrow(StatBox),ncol=1,byrow=TRUE)\n"
  549 + "mat <- rbind(mat, paste(as.character(round(OverallStatBox$Y, 3)), round(OverallStatBox$ci, 3), sep=\" \\u00b1 \"))\n"
  550 + "colnames(mat) <- c(\"Error Rate\")\n"
  551 + "rownames(mat) <- c(seq(0,nrow(StatBox)-1),\"Aggregate\")\n"
542 552 "ETable <- as.table(mat)\n")));
543 553  
544 554 p.file.write(qPrintable(QString("\n"
545 555 "print(textplot(ETable))\n"
546   - "print(title(expression(atop(\"Landmark Error Rates\", atop(italic(\"Average Normalization Distance: 91.419 (pixels)\"))))))\n")));
  556 + "print(title(sprintf(\"Landmark Error Rates\\nAverage Normalization Distance: %.3f (pixels)\",NormLength$Y)))\n")));
547 557  
548 558 p.file.write(qPrintable(QString("ggplot(Box, aes(Y,%1%2))").arg(p.major.size > 1 ? QString(", colour=%1").arg(p.major.header) : QString(),
549 559 p.minor.size > 1 ? QString(", linetype=%1").arg(p.minor.header) : QString()) +
550 560 QString(" + annotation_logticks(sides=\"b\") + stat_ecdf() + scale_x_log10(\"Normalized Error\", breaks=c(0.001,0.01,0.1,1,10)) + scale_y_continuous(\"Cumulative Density\", label=percent) + theme_minimal()\n\n")));
551 561  
552 562 p.file.write(qPrintable(QString("ggplot(Box, aes(factor(X), Y%1%2))").arg(p.major.size > 1 ? QString(", colour=%1").arg(p.major.header) : QString(), p.minor.size > 1 ? QString(", linetype=%1").arg(p.minor.header) : QString()) +
553   - QString("+ annotation_logticks(sides=\"l\") + geom_boxplot(alpha=0.5) + geom_jitter(size=1, alpha=0.5) + scale_x_discrete(\"Landmark\") + scale_y_log10(\"Normalized Error\", breaks=c(0.01,0.1,1,10)) + theme_minimal()\n\n")));
  563 + QString("+ annotation_logticks(sides=\"l\") + geom_boxplot(alpha=0.5) + geom_jitter(size=1, alpha=0.5) + scale_x_discrete(\"Landmark\") + scale_y_log10(\"Normalized Error\", breaks=c(0.001,0.01,0.1,1,10)) + theme_minimal()\n\n")));
554 564  
555 565 p.file.write(qPrintable(QString("ggplot(Box, aes(factor(X), Y%1%2))").arg(p.major.size > 1 ? QString(", colour=%1").arg(p.major.header) : QString(), p.minor.size > 1 ? QString(", linetype=%1").arg(p.minor.header) : QString()) +
556 566 QString("+ annotation_logticks(sides=\"l\") + geom_violin(alpha=0.5) + scale_x_discrete(\"Landmark\") + scale_y_log10(\"Normalized Error\", breaks=c(0.001,0.01,0.1,1,10))\n\n")));
... ...