Peter M. Groen / openbr

Commit 06a58254a6f517b7c2e444cffc03e061e2bdc742

Authored by Scott Klum
1 parent 5b53a8c4

Added good and bad examples, lots of cleanup

Inline Side-by-side
Showing 2 changed files with 123 additions and 97 deletions
openbr/core/eval.cpp
  View file @06a5825
@@ -1035,15 +1035,15 @@ float EvalDetection(const QString &predictedGallery, const QString &truthGallery @@ -1035,15 +1035,15 @@ float EvalDetection(const QString &predictedGallery, const QString &truthGallery
1035 float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv, int normalizationIndexA, int normalizationIndexB, int sampleIndex) 1035 float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv, int normalizationIndexA, int normalizationIndexB, int sampleIndex)
1036 { 1036 {
1037 qDebug("Evaluating landmarking of %s against %s", qPrintable(predictedGallery), qPrintable(truthGallery)); 1037 qDebug("Evaluating landmarking of %s against %s", qPrintable(predictedGallery), qPrintable(truthGallery));
1038 - const TemplateList predicted(TemplateList::fromGallery(predictedGallery));  
1039 - const TemplateList truth(TemplateList::fromGallery(truthGallery));  
1040 - const QStringList predictedNames = File::get<QString>(predicted, "name");  
1041 - const QStringList truthNames = File::get<QString>(truth, "name"); 1038 + TemplateList predicted(TemplateList::fromGallery(predictedGallery));
  1039 + TemplateList truth(TemplateList::fromGallery(truthGallery));
  1040 + QStringList predictedNames = File::get<QString>(predicted, "name");
  1041 + QStringList truthNames = File::get<QString>(truth, "name");
1042 1042
1043 int skipped = 0; 1043 int skipped = 0;
1044 QList< QList<float> > pointErrors; 1044 QList< QList<float> > pointErrors;
  1045 + QList<float> imageErrors;
1045 QList<float> normalizedLengths; 1046 QList<float> normalizedLengths;
1046 -  
1047 for (int i=0; i<predicted.size(); i++) { 1047 for (int i=0; i<predicted.size(); i++) {
1048 const QString &predictedName = predictedNames[i]; 1048 const QString &predictedName = predictedNames[i];
1049 const int truthIndex = truthNames.indexOf(predictedName); 1049 const int truthIndex = truthNames.indexOf(predictedName);
@@ -1051,58 +1051,76 @@ float EvalLandmarking(const QString &amp;predictedGallery, const QString &amp;truthGalle @@ -1051,58 +1051,76 @@ float EvalLandmarking(const QString &amp;predictedGallery, const QString &amp;truthGalle
1051 const QList<QPointF> predictedPoints = predicted[i].file.points(); 1051 const QList<QPointF> predictedPoints = predicted[i].file.points();
1052 const QList<QPointF> truthPoints = truth[truthIndex].file.points(); 1052 const QList<QPointF> truthPoints = truth[truthIndex].file.points();
1053 if (predictedPoints.size() != truthPoints.size()) { 1053 if (predictedPoints.size() != truthPoints.size()) {
1054 - skipped++; 1054 + predicted.removeAt(i);
  1055 + predictedNames.removeAt(i);
  1056 + truth.removeAt(i);
  1057 + truthNames.removeAt(i);
  1058 + i--; skipped++;
1055 continue; 1059 continue;
1056 } 1060 }
  1061 +
1057 while (pointErrors.size() < predictedPoints.size()) 1062 while (pointErrors.size() < predictedPoints.size())
1058 pointErrors.append(QList<float>()); 1063 pointErrors.append(QList<float>());
1059 1064
  1065 + // Want to know error for every image.
  1066 +
1060 if (normalizationIndexA >= truthPoints.size()) qFatal("Normalization index A is out of range."); 1067 if (normalizationIndexA >= truthPoints.size()) qFatal("Normalization index A is out of range.");
1061 if (normalizationIndexB >= truthPoints.size()) qFatal("Normalization index B is out of range."); 1068 if (normalizationIndexB >= truthPoints.size()) qFatal("Normalization index B is out of range.");
1062 const float normalizedLength = QtUtils::euclideanLength(truthPoints[normalizationIndexB] - truthPoints[normalizationIndexA]); 1069 const float normalizedLength = QtUtils::euclideanLength(truthPoints[normalizationIndexB] - truthPoints[normalizationIndexA]);
1063 normalizedLengths.append(normalizedLength); 1070 normalizedLengths.append(normalizedLength);
  1071 + float totalError = 0;
1064 for (int j=0; j<predictedPoints.size(); j++) { 1072 for (int j=0; j<predictedPoints.size(); j++) {
1065 - pointErrors[j].append(QtUtils::euclideanLength(predictedPoints[j] - truthPoints[j])/normalizedLength); 1073 + float error = QtUtils::euclideanLength(predictedPoints[j] - truthPoints[j])/normalizedLength;
  1074 + totalError += error;
  1075 + pointErrors[j].append(error);
1066 } 1076 }
  1077 + imageErrors.append(totalError/predictedPoints.size());
1067 } 1078 }
1068 1079
1069 qDebug() << "Skipped" << skipped << "files due to point size mismatch."; 1080 qDebug() << "Skipped" << skipped << "files due to point size mismatch.";
1070 1081
1071 QList<float> averagePointErrors; averagePointErrors.reserve(pointErrors.size()); 1082 QList<float> averagePointErrors; averagePointErrors.reserve(pointErrors.size());
1072 1083
1073 - float normalizedErrorLimit = 1.5;  
1074 -  
1075 - QSet<int> worstExamples;  
1076 - for (int i=0; i<pointErrors.size(); i++) {  
1077 - if (skipped == 0) {  
1078 - QList<QPair<float,int> > exampleIndices = Common::Sort(pointErrors[i],true);  
1079 - for (int j=0; j<exampleIndices.size() && worstExamples.size()<(5*i+1); j++)  
1080 - if (exampleIndices[j].first < normalizedErrorLimit)  
1081 - worstExamples.insert(exampleIndices[j].second);  
1082 - }  
1083 - std::sort(pointErrors[i].begin(), pointErrors[i].end());  
1084 - averagePointErrors.append(Common::Mean(pointErrors[i]));  
1085 - }  
1086 - const float averagePointError = Common::Mean(averagePointErrors);  
1087 -  
1088 QStringList lines; 1084 QStringList lines;
1089 lines.append("Plot,X,Y"); 1085 lines.append("Plot,X,Y");
1090 1086
1091 - // Sample  
1092 - QFile exampleFile("landmarking_examples");  
1093 - QtUtils::touchDir(exampleFile);  
1094 - lines.append("EX,landmarking_examples/"+truth[sampleIndex].file.fileName()+","+QString::number(truth[sampleIndex].file.points().size())); 1087 + // Example
  1088 + lines.append("Sample,landmarking_examples_truth/"+truth[sampleIndex].file.fileName()+","+QString::number(truth[sampleIndex].file.points().size()));
1095 1089
1096 // Alternatively, can we just pass this through a predetermined transform and write? 1090 // Alternatively, can we just pass this through a predetermined transform and write?
1097 - Enroll(truth[sampleIndex],"landmarking_examples"); 1091 + Enroll(truth[sampleIndex],"landmarking_examples_truth");
  1092 +
  1093 + // Get best and worst performing examples
  1094 + QList< QPair<float,int> > exampleIndices = Common::Sort(imageErrors,true);
  1095 +
  1096 + const int totalExamples = 10;
  1097 + for (int i=0; i<totalExamples; i++) {
  1098 + Enroll(truth[exampleIndices[i].second],"landmarking_examples_truth");
  1099 + lines.append("EXT,landmarking_examples_truth/"+truth[exampleIndices[i].second].file.fileName()+","+QString::number(exampleIndices[i].first));
  1100 + Enroll(predicted[exampleIndices[i].second],"landmarking_examples_prediced");
  1101 + lines.append("EXP,landmarking_examples_prediced/"+predicted[exampleIndices[i].second].file.fileName()+","+QString::number(exampleIndices[i].first));
  1102 +
  1103 + }
  1104 +
  1105 + for (int i=exampleIndices.size()-1; i>exampleIndices.size()-totalExamples-1; i--) {
  1106 + Enroll(truth[exampleIndices[i].second],"landmarking_examples_truth");
  1107 + lines.append("EXT,landmarking_examples_truth/"+truth[exampleIndices[i].second].file.fileName()+","+QString::number(exampleIndices[i].first));
  1108 + Enroll(predicted[exampleIndices[i].second],"landmarking_examples_prediced");
  1109 + lines.append("EXP,landmarking_examples_prediced/"+predicted[exampleIndices[i].second].file.fileName()+","+QString::number(exampleIndices[i].first));
  1110 +
  1111 + }
1098 1112
1099 for (int i=0; i<pointErrors.size(); i++) { 1113 for (int i=0; i<pointErrors.size(); i++) {
  1114 + std::sort(pointErrors[i].begin(), pointErrors[i].end());
  1115 + averagePointErrors.append(Common::Mean(pointErrors[i]));
1100 const QList<float> &pointError = pointErrors[i]; 1116 const QList<float> &pointError = pointErrors[i];
1101 const int keep = qMin(Max_Points, pointError.size()); 1117 const int keep = qMin(Max_Points, pointError.size());
1102 for (int j=0; j<keep; j++) 1118 for (int j=0; j<keep; j++)
1103 lines.append(QString("Box,%1,%2").arg(QString::number(i), QString::number(pointError[j*(pointError.size()-1)/(keep-1)]))); 1119 lines.append(QString("Box,%1,%2").arg(QString::number(i), QString::number(pointError[j*(pointError.size()-1)/(keep-1)])));
1104 } 1120 }
1105 1121
  1122 + const float averagePointError = Common::Mean(averagePointErrors);
  1123 +
1106 lines.append(QString("AvgError,0,%1").arg(averagePointError)); 1124 lines.append(QString("AvgError,0,%1").arg(averagePointError));
1107 lines.append(QString("NormLength,0,%1").arg(Common::Mean(normalizedLengths))); 1125 lines.append(QString("NormLength,0,%1").arg(Common::Mean(normalizedLengths)));
1108 1126
openbr/core/plot.cpp
  View file @06a5825
@@ -469,77 +469,85 @@ bool PlotLandmarking(const QStringList &amp;files, const File &amp;destination, bool sho @@ -469,77 +469,85 @@ bool PlotLandmarking(const QStringList &amp;files, const File &amp;destination, bool sho
469 qDebug("Plotting %d landmarking file(s) to %s", files.size(), qPrintable(destination)); 469 qDebug("Plotting %d landmarking file(s) to %s", files.size(), qPrintable(destination));
470 RPlot p(files, destination, false); 470 RPlot p(files, destination, false);
471 471
472 - qDebug() << p.major.header << p.minor.header;  
473 -  
474 p.file.write(qPrintable(QString("# Split data into individual plots\n" 472 p.file.write(qPrintable(QString("# Split data into individual plots\n"
475 - "plot_index = which(names(data)==\"Plot\")\n"  
476 - "Box <- data[grep(\"Box\",data$Plot),-c(1)]\n"  
477 - "Box$X <- factor(Box$X, levels = Box$X, ordered = TRUE)\n"  
478 - "EX <- data[grep(\"EX\",data$Plot),-c(1)]\n"  
479 - "NormLength <- data[grep(\"NormLength\",data$Plot),-c(1)]\n"  
480 - "EX$X <- as.character(EX$X)\n"  
481 - "EX$Y <- as.character(EX$Y)\n"  
482 - "\n"  
483 - "\n\tsummarySE <- function(data=NULL, measurevar, groupvars=NULL, na.rm=FALSE, conf.interval=.95, .drop=TRUE) {\n\t\t"  
484 - "require(plyr)\n\n\t\tlength2 <- function (x, na.rm=FALSE) {\n\t\t\tif (na.rm) sum(!is.na(x))\n\t\t\telse length(x)"  
485 - "\n\t\t}\n\n\t\tdatac <- ddply(data, groupvars, .drop=.drop, .fun = function(xx, col) {\n\t\t\t"  
486 - "c(N=length2(xx[[col]], na.rm=na.rm), mean=mean(xx[[col]], na.rm=na.rm), sd=sd(xx[[col]], na.rm=na.rm))\n\t\t\t},"  
487 - "\n\t\t\tmeasurevar\n\t\t)\n\n\t\tdatac <- rename(datac, c(\"mean\" = measurevar))\n\t\tdatac$se <- datac$sd / sqrt(datac$N)"  
488 - "\n\t\tciMult <- qt(conf.interval/2 + .5, datac$N-1)\n\t\tdatac$ci <- datac$se * ciMult\n\n\t\treturn(datac)\n\t}\n\t"  
489 - "rm(data)\n"  
490 - "\n")));  
491 -  
492 - p.file.write(qPrintable(QString("if (nrow(EX) != 0) { \  
493 - \n\tlibrary(jpeg) \  
494 - \n\tlibrary(png) \  
495 - \n\tlibrary(grid)\n\t") +  
496 -  
497 - QString("multiplot <- function(..., plotlist=NULL, cols) {") +  
498 - QString("\n\t\t require(grid) \  
499 - \n\n\t\t# Make a list from the ... arguments and plotlist \  
500 - \n\t\t plots <- c(list(...), plotlist)\n") +  
501 -  
502 - QString("\t\tnumPlots = length(plots)\n\n\t\t \  
503 - # Make the panel \  
504 - \n\t\tplotCols = cols \  
505 - \n\t\tplotRows = ceiling(numPlots/plotCols) \  
506 - \n\n") +  
507 -  
508 - QString("\t\t# Set up the page \  
509 - \n\t\tgrid.newpage() \  
510 - \n\t\tpushViewport(viewport(layout = grid.layout(plotRows, plotCols))) \  
511 - \n\t\tvplayout <- function(x, y) \  
512 - \n\t\t\tviewport(layout.pos.row = x, layout.pos.col = y)\n\n") +  
513 -  
514 - QString("\t\t# Make each plot, in the correct location \  
515 - \n\t\tfor (i in 1:numPlots) { \  
516 - \n\t\t\tcurRow = ceiling(i/plotCols)\n\t\t\tcurCol = (i-1) %% plotCols + 1 \  
517 - \n\t\t\tprint(plots[[i]], vp = vplayout(curRow, curCol))\n\t\t}\n\t}\n\n")));  
518 -  
519 - p.file.write(qPrintable(QString("\n\n\t# Print genuine matches below the EER \  
520 - \n\t \  
521 - for (i in 1:nrow(EX)) { \  
522 - \n\t\t path <- EX[i,1] \  
523 - \n\t\t points <- EX[i,2] \  
524 - \n\t\t file <- unlist(strsplit(path, \"[.]\"))[1] \  
525 - \n\t\t ext <- unlist(strsplit(path, \"[.]\"))[2]") +  
526 -  
527 - // These should be made into a function assuming we can return an image variable regardless of extension  
528 - QString("\n\t\t\tif (ext == \"jpg\" || ext == \"JPEG\" || ext == \"jpeg\" || ext == \"JPG\") { \  
529 - \n\t\t\t img <- readJPEG(path)\n\t\t } \  
530 - else if (ext == \"PNG\" || ext == \"png\") { \  
531 - \n\t\t\t img <- readPNG(path)\n\t\t} \  
532 - else if (ext == \"TIFF\" || ext == \"tiff\" || ext == \"TIF\" || ext == \"tif\") { \  
533 - \n\t\t\t img <- readTIFF(path)\n\t\t} \  
534 - else {\n\t\t\tnext\n\t\t} ") +  
535 -  
536 - QString("\n\t\t name <- file \  
537 - \n\n\t\t g <- rasterGrob(img, interpolate=TRUE)\n\n\t\t") +  
538 -  
539 - QString("print(qplot(1:10, 1:10, geom=\"blank\") \  
540 - + annotation_custom(g, xmin=-Inf, xmax=Inf, ymin=-Inf, ymax=Inf) \  
541 - + theme(axis.line=element_blank(), axis.title.y=element_blank(), axis.text.x=element_blank(), axis.text.y=element_blank(), line=element_blank(), axis.ticks=element_blank(), panel.background=element_blank()) \  
542 - + labs(title=\"Sample Landmarks\") + xlab(sprintf(\"Total Landmarks: %s\",points)))\n\t\t}}\n"))); 473 + "plot_index = which(names(data)==\"Plot\")\n"
  474 + "Box <- data[grep(\"Box\",data$Plot),-c(1)]\n"
  475 + "Box$X <- factor(Box$X, levels = Box$X, ordered = TRUE)\n"
  476 + "Sample <- data[grep(\"Sample\",data$Plot),-c(1)]\n"
  477 + "Sample$X <- as.character(Sample$X)\n"
  478 + "EXT <- data[grep(\"EXT\",data$Plot),-c(1)]\n"
  479 + "EXT$X <- as.character(EXT$X)\n"
  480 + "EXP <- data[grep(\"EXP\",data$Plot),-c(1)]\n"
  481 + "EXP$X <- as.character(EXP$X)\n"
  482 + "NormLength <- data[grep(\"NormLength\",data$Plot),-c(1)]\n"
  483 + "rm(data)\n"
  484 + "\n")));
  485 +
  486 + p.file.write(qPrintable(QString("summarySE <- function(data=NULL, measurevar, groupvars=NULL, na.rm=FALSE, conf.interval=.95, .drop=TRUE) {\n\t"
  487 + "require(plyr)\n\n\tlength2 <- function (x, na.rm=FALSE) {\n\t\tif (na.rm) sum(!is.na(x))\n\t\telse length(x)"
  488 + "\n\t}\n\n\tdatac <- ddply(data, groupvars, .drop=.drop, .fun = function(xx, col) {\n\t\t"
  489 + "c(N=length2(xx[[col]], na.rm=na.rm), mean=mean(xx[[col]], na.rm=na.rm), sd=sd(xx[[col]], na.rm=na.rm))\n\t\t},"
  490 + "\n\t\tmeasurevar\n\t)\n\n\tdatac <- rename(datac, c(\"mean\" = measurevar))\n\tdatac$se <- datac$sd / sqrt(datac$N)"
  491 + "\n\tciMult <- qt(conf.interval/2 + .5, datac$N-1)\n\tdatac$ci <- datac$se * ciMult\n\n\treturn(datac)\n}\n")));
  492 +
  493 +
  494 + p.file.write(qPrintable(QString("\nreadData <- function(data) {\n\texamples <- list()\n"
  495 + "\tfor (i in 1:nrow(data)) {\n"
  496 + "\t\tpath <- data[i,1]\n"
  497 + "\t\tvalue <- data[i,2]\n"
  498 + "\t\tfile <- unlist(strsplit(path, \"[.]\"))[1]\n"
  499 + "\t\text <- unlist(strsplit(path, \"[.]\"))[2]\n"
  500 + "\t\tif (ext == \"jpg\" || ext == \"JPEG\" || ext == \"jpeg\" || ext == \"JPG\") {\n"
  501 + "\t\t\timg <- readJPEG(path)\n"
  502 + "\t\t} else if (ext == \"PNG\" || ext == \"png\") {\n"
  503 + "\t\t\timg <- readPNG(path)\n"
  504 + "\t\t} else if (ext == \"TIFF\" || ext == \"tiff\" || ext == \"TIF\" || ext == \"tif\") { \n"
  505 + "\t\t\timg <- readTIFF(path)\n"
  506 + "}else {\n"
  507 + "\t\t\tnext\n"
  508 + "\t\t}\n"
  509 + "\t\texample <- list(file = file, value = value, image = img)\n"
  510 + "\t\texamples[[i]] <- example\n"
  511 + "\t}\n"
  512 + "\treturn(examples)\n"
  513 + "}\n")));
  514 +
  515 + p.file.write(qPrintable(QString("\nlibrary(jpeg)\n"
  516 + "library(png)\n"
  517 + "library(grid)\n"
  518 + "multiplot <- function(..., plotlist=NULL, cols) {\n"
  519 + "\trequire(grid)\n"
  520 + "\t# Make a list from the ... arguments and plotlist\n"
  521 + "\tplots <- c(list(...), plotlist)\n"
  522 + "\tnumPlots = length(plots)\n"
  523 + "\t# Make the panel\n"
  524 + "\tplotCols = cols\n"
  525 + "\tplotRows = ceiling(numPlots/plotCols)\n"
  526 + "\t# Set up the page\n"
  527 + "\tgrid.newpage()\n"
  528 + "\tpushViewport(viewport(layout = grid.layout(plotRows, plotCols)))\n"
  529 + "\tvplayout <- function(x, y)\n"
  530 + "\tviewport(layout.pos.row = x, layout.pos.col = y)\n"
  531 + "\t# Make each plot, in the correct location\n"
  532 + "\tfor (i in 1:numPlots) {\n"
  533 + "\t\tcurRow = ceiling(i/plotCols)\n"
  534 + "\t\tcurCol = (i-1) %% plotCols + 1\n"
  535 + "\t\tprint(plots[[i]], vp = vplayout(curRow, curCol))\n"
  536 + "\t}\n"
  537 + "}\n")));
  538 +
  539 + p.file.write(qPrintable(QString("\nplotImage <- function(image, title=NULL, label=NULL) { \n"
  540 + "\tp <- qplot(1:10, 1:10, geom=\"blank\") + annotation_custom(rasterGrob(image$image), xmin=-Inf, xmax=Inf, ymin=-Inf, ymax=Inf) + theme(axis.line=element_blank(), axis.title.y=element_blank(), axis.text.x=element_blank(), axis.text.y=element_blank(), line=element_blank(), axis.ticks=element_blank(), panel.background=element_blank()) + labs(title=title) + xlab(label)\n"
  541 + "\treturn(p)"
  542 + "}\n")));
  543 +
  544 + p.file.write(qPrintable(QString("\nsample <- readData(Sample) \n"
  545 + "print(plotImage(sample[[1]],\"Sample Landmarks\",sprintf(\"Total Landmarks: %s\",sample[[1]]$value))) \n"
  546 + "truthSample <- readData(EXT)\n"
  547 + "predictedSample <- readData(EXP)\n"
  548 + "for (i in 1:length(predictedSample)) {\n"
  549 + "\tmultiplot(plotImage(predictedSample[[i]],\"Predicted Landmarks\",sprintf(\"Average Landmark Error: %.3f\",predictedSample[[i]]$value)),plotImage(truthSample[[i]],\"Ground Truth Landmarks\",\"\"),cols=2)\n"
  550 + "}\n")));
543 551
544 p.file.write(qPrintable(QString("\n" 552 p.file.write(qPrintable(QString("\n"
545 "# Code to format error table\n" 553 "# Code to format error table\n"