Peter M. Groen / openbr

Commit 06a58254a6f517b7c2e444cffc03e061e2bdc742

Authored by Scott Klum
1 parent 5b53a8c4

Added good and bad examples, lots of cleanup

Inline Side-by-side
Showing 2 changed files with 123 additions and 97 deletions
openbr/core/eval.cpp
  View file @06a5825
... ... @@ -1035,15 +1035,15 @@ float EvalDetection(const QString &predictedGallery, const QString &truthGallery
1035 1035 float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv, int normalizationIndexA, int normalizationIndexB, int sampleIndex)
1036 1036 {
1037 1037 qDebug("Evaluating landmarking of %s against %s", qPrintable(predictedGallery), qPrintable(truthGallery));
1038   - const TemplateList predicted(TemplateList::fromGallery(predictedGallery));
1039   - const TemplateList truth(TemplateList::fromGallery(truthGallery));
1040   - const QStringList predictedNames = File::get<QString>(predicted, "name");
1041   - const QStringList truthNames = File::get<QString>(truth, "name");
  1038 + TemplateList predicted(TemplateList::fromGallery(predictedGallery));
  1039 + TemplateList truth(TemplateList::fromGallery(truthGallery));
  1040 + QStringList predictedNames = File::get<QString>(predicted, "name");
  1041 + QStringList truthNames = File::get<QString>(truth, "name");
1042 1042  
1043 1043 int skipped = 0;
1044 1044 QList< QList<float> > pointErrors;
  1045 + QList<float> imageErrors;
1045 1046 QList<float> normalizedLengths;
1046   -
1047 1047 for (int i=0; i<predicted.size(); i++) {
1048 1048 const QString &predictedName = predictedNames[i];
1049 1049 const int truthIndex = truthNames.indexOf(predictedName);
... ... @@ -1051,58 +1051,76 @@ float EvalLandmarking(const QString &amp;predictedGallery, const QString &amp;truthGalle
1051 1051 const QList<QPointF> predictedPoints = predicted[i].file.points();
1052 1052 const QList<QPointF> truthPoints = truth[truthIndex].file.points();
1053 1053 if (predictedPoints.size() != truthPoints.size()) {
1054   - skipped++;
  1054 + predicted.removeAt(i);
  1055 + predictedNames.removeAt(i);
  1056 + truth.removeAt(i);
  1057 + truthNames.removeAt(i);
  1058 + i--; skipped++;
1055 1059 continue;
1056 1060 }
  1061 +
1057 1062 while (pointErrors.size() < predictedPoints.size())
1058 1063 pointErrors.append(QList<float>());
1059 1064  
  1065 + // Want to know error for every image.
  1066 +
1060 1067 if (normalizationIndexA >= truthPoints.size()) qFatal("Normalization index A is out of range.");
1061 1068 if (normalizationIndexB >= truthPoints.size()) qFatal("Normalization index B is out of range.");
1062 1069 const float normalizedLength = QtUtils::euclideanLength(truthPoints[normalizationIndexB] - truthPoints[normalizationIndexA]);
1063 1070 normalizedLengths.append(normalizedLength);
  1071 + float totalError = 0;
1064 1072 for (int j=0; j<predictedPoints.size(); j++) {
1065   - pointErrors[j].append(QtUtils::euclideanLength(predictedPoints[j] - truthPoints[j])/normalizedLength);
  1073 + float error = QtUtils::euclideanLength(predictedPoints[j] - truthPoints[j])/normalizedLength;
  1074 + totalError += error;
  1075 + pointErrors[j].append(error);
1066 1076 }
  1077 + imageErrors.append(totalError/predictedPoints.size());
1067 1078 }
1068 1079  
1069 1080 qDebug() << "Skipped" << skipped << "files due to point size mismatch.";
1070 1081  
1071 1082 QList<float> averagePointErrors; averagePointErrors.reserve(pointErrors.size());
1072 1083  
1073   - float normalizedErrorLimit = 1.5;
1074   -
1075   - QSet<int> worstExamples;
1076   - for (int i=0; i<pointErrors.size(); i++) {
1077   - if (skipped == 0) {
1078   - QList<QPair<float,int> > exampleIndices = Common::Sort(pointErrors[i],true);
1079   - for (int j=0; j<exampleIndices.size() && worstExamples.size()<(5*i+1); j++)
1080   - if (exampleIndices[j].first < normalizedErrorLimit)
1081   - worstExamples.insert(exampleIndices[j].second);
1082   - }
1083   - std::sort(pointErrors[i].begin(), pointErrors[i].end());
1084   - averagePointErrors.append(Common::Mean(pointErrors[i]));
1085   - }
1086   - const float averagePointError = Common::Mean(averagePointErrors);
1087   -
1088 1084 QStringList lines;
1089 1085 lines.append("Plot,X,Y");
1090 1086  
1091   - // Sample
1092   - QFile exampleFile("landmarking_examples");
1093   - QtUtils::touchDir(exampleFile);
1094   - lines.append("EX,landmarking_examples/"+truth[sampleIndex].file.fileName()+","+QString::number(truth[sampleIndex].file.points().size()));
  1087 + // Example
  1088 + lines.append("Sample,landmarking_examples_truth/"+truth[sampleIndex].file.fileName()+","+QString::number(truth[sampleIndex].file.points().size()));
1095 1089  
1096 1090 // Alternatively, can we just pass this through a predetermined transform and write?
1097   - Enroll(truth[sampleIndex],"landmarking_examples");
  1091 + Enroll(truth[sampleIndex],"landmarking_examples_truth");
  1092 +
  1093 + // Get best and worst performing examples
  1094 + QList< QPair<float,int> > exampleIndices = Common::Sort(imageErrors,true);
  1095 +
  1096 + const int totalExamples = 10;
  1097 + for (int i=0; i<totalExamples; i++) {
  1098 + Enroll(truth[exampleIndices[i].second],"landmarking_examples_truth");
  1099 + lines.append("EXT,landmarking_examples_truth/"+truth[exampleIndices[i].second].file.fileName()+","+QString::number(exampleIndices[i].first));
  1100 + Enroll(predicted[exampleIndices[i].second],"landmarking_examples_prediced");
  1101 + lines.append("EXP,landmarking_examples_prediced/"+predicted[exampleIndices[i].second].file.fileName()+","+QString::number(exampleIndices[i].first));
  1102 +
  1103 + }
  1104 +
  1105 + for (int i=exampleIndices.size()-1; i>exampleIndices.size()-totalExamples-1; i--) {
  1106 + Enroll(truth[exampleIndices[i].second],"landmarking_examples_truth");
  1107 + lines.append("EXT,landmarking_examples_truth/"+truth[exampleIndices[i].second].file.fileName()+","+QString::number(exampleIndices[i].first));
  1108 + Enroll(predicted[exampleIndices[i].second],"landmarking_examples_prediced");
  1109 + lines.append("EXP,landmarking_examples_prediced/"+predicted[exampleIndices[i].second].file.fileName()+","+QString::number(exampleIndices[i].first));
  1110 +
  1111 + }
1098 1112  
1099 1113 for (int i=0; i<pointErrors.size(); i++) {
  1114 + std::sort(pointErrors[i].begin(), pointErrors[i].end());
  1115 + averagePointErrors.append(Common::Mean(pointErrors[i]));
1100 1116 const QList<float> &pointError = pointErrors[i];
1101 1117 const int keep = qMin(Max_Points, pointError.size());
1102 1118 for (int j=0; j<keep; j++)
1103 1119 lines.append(QString("Box,%1,%2").arg(QString::number(i), QString::number(pointError[j*(pointError.size()-1)/(keep-1)])));
1104 1120 }
1105 1121  
  1122 + const float averagePointError = Common::Mean(averagePointErrors);
  1123 +
1106 1124 lines.append(QString("AvgError,0,%1").arg(averagePointError));
1107 1125 lines.append(QString("NormLength,0,%1").arg(Common::Mean(normalizedLengths)));
1108 1126  
... ...
openbr/core/plot.cpp
  View file @06a5825
... ... @@ -469,77 +469,85 @@ bool PlotLandmarking(const QStringList &amp;files, const File &amp;destination, bool sho
469 469 qDebug("Plotting %d landmarking file(s) to %s", files.size(), qPrintable(destination));
470 470 RPlot p(files, destination, false);
471 471  
472   - qDebug() << p.major.header << p.minor.header;
473   -
474 472 p.file.write(qPrintable(QString("# Split data into individual plots\n"
475   - "plot_index = which(names(data)==\"Plot\")\n"
476   - "Box <- data[grep(\"Box\",data$Plot),-c(1)]\n"
477   - "Box$X <- factor(Box$X, levels = Box$X, ordered = TRUE)\n"
478   - "EX <- data[grep(\"EX\",data$Plot),-c(1)]\n"
479   - "NormLength <- data[grep(\"NormLength\",data$Plot),-c(1)]\n"
480   - "EX$X <- as.character(EX$X)\n"
481   - "EX$Y <- as.character(EX$Y)\n"
482   - "\n"
483   - "\n\tsummarySE <- function(data=NULL, measurevar, groupvars=NULL, na.rm=FALSE, conf.interval=.95, .drop=TRUE) {\n\t\t"
484   - "require(plyr)\n\n\t\tlength2 <- function (x, na.rm=FALSE) {\n\t\t\tif (na.rm) sum(!is.na(x))\n\t\t\telse length(x)"
485   - "\n\t\t}\n\n\t\tdatac <- ddply(data, groupvars, .drop=.drop, .fun = function(xx, col) {\n\t\t\t"
486   - "c(N=length2(xx[[col]], na.rm=na.rm), mean=mean(xx[[col]], na.rm=na.rm), sd=sd(xx[[col]], na.rm=na.rm))\n\t\t\t},"
487   - "\n\t\t\tmeasurevar\n\t\t)\n\n\t\tdatac <- rename(datac, c(\"mean\" = measurevar))\n\t\tdatac$se <- datac$sd / sqrt(datac$N)"
488   - "\n\t\tciMult <- qt(conf.interval/2 + .5, datac$N-1)\n\t\tdatac$ci <- datac$se * ciMult\n\n\t\treturn(datac)\n\t}\n\t"
489   - "rm(data)\n"
490   - "\n")));
491   -
492   - p.file.write(qPrintable(QString("if (nrow(EX) != 0) { \
493   - \n\tlibrary(jpeg) \
494   - \n\tlibrary(png) \
495   - \n\tlibrary(grid)\n\t") +
496   -
497   - QString("multiplot <- function(..., plotlist=NULL, cols) {") +
498   - QString("\n\t\t require(grid) \
499   - \n\n\t\t# Make a list from the ... arguments and plotlist \
500   - \n\t\t plots <- c(list(...), plotlist)\n") +
501   -
502   - QString("\t\tnumPlots = length(plots)\n\n\t\t \
503   - # Make the panel \
504   - \n\t\tplotCols = cols \
505   - \n\t\tplotRows = ceiling(numPlots/plotCols) \
506   - \n\n") +
507   -
508   - QString("\t\t# Set up the page \
509   - \n\t\tgrid.newpage() \
510   - \n\t\tpushViewport(viewport(layout = grid.layout(plotRows, plotCols))) \
511   - \n\t\tvplayout <- function(x, y) \
512   - \n\t\t\tviewport(layout.pos.row = x, layout.pos.col = y)\n\n") +
513   -
514   - QString("\t\t# Make each plot, in the correct location \
515   - \n\t\tfor (i in 1:numPlots) { \
516   - \n\t\t\tcurRow = ceiling(i/plotCols)\n\t\t\tcurCol = (i-1) %% plotCols + 1 \
517   - \n\t\t\tprint(plots[[i]], vp = vplayout(curRow, curCol))\n\t\t}\n\t}\n\n")));
518   -
519   - p.file.write(qPrintable(QString("\n\n\t# Print genuine matches below the EER \
520   - \n\t \
521   - for (i in 1:nrow(EX)) { \
522   - \n\t\t path <- EX[i,1] \
523   - \n\t\t points <- EX[i,2] \
524   - \n\t\t file <- unlist(strsplit(path, \"[.]\"))[1] \
525   - \n\t\t ext <- unlist(strsplit(path, \"[.]\"))[2]") +
526   -
527   - // These should be made into a function assuming we can return an image variable regardless of extension
528   - QString("\n\t\t\tif (ext == \"jpg\" || ext == \"JPEG\" || ext == \"jpeg\" || ext == \"JPG\") { \
529   - \n\t\t\t img <- readJPEG(path)\n\t\t } \
530   - else if (ext == \"PNG\" || ext == \"png\") { \
531   - \n\t\t\t img <- readPNG(path)\n\t\t} \
532   - else if (ext == \"TIFF\" || ext == \"tiff\" || ext == \"TIF\" || ext == \"tif\") { \
533   - \n\t\t\t img <- readTIFF(path)\n\t\t} \
534   - else {\n\t\t\tnext\n\t\t} ") +
535   -
536   - QString("\n\t\t name <- file \
537   - \n\n\t\t g <- rasterGrob(img, interpolate=TRUE)\n\n\t\t") +
538   -
539   - QString("print(qplot(1:10, 1:10, geom=\"blank\") \
540   - + annotation_custom(g, xmin=-Inf, xmax=Inf, ymin=-Inf, ymax=Inf) \
541   - + theme(axis.line=element_blank(), axis.title.y=element_blank(), axis.text.x=element_blank(), axis.text.y=element_blank(), line=element_blank(), axis.ticks=element_blank(), panel.background=element_blank()) \
542   - + labs(title=\"Sample Landmarks\") + xlab(sprintf(\"Total Landmarks: %s\",points)))\n\t\t}}\n")));
  473 + "plot_index = which(names(data)==\"Plot\")\n"
  474 + "Box <- data[grep(\"Box\",data$Plot),-c(1)]\n"
  475 + "Box$X <- factor(Box$X, levels = Box$X, ordered = TRUE)\n"
  476 + "Sample <- data[grep(\"Sample\",data$Plot),-c(1)]\n"
  477 + "Sample$X <- as.character(Sample$X)\n"
  478 + "EXT <- data[grep(\"EXT\",data$Plot),-c(1)]\n"
  479 + "EXT$X <- as.character(EXT$X)\n"
  480 + "EXP <- data[grep(\"EXP\",data$Plot),-c(1)]\n"
  481 + "EXP$X <- as.character(EXP$X)\n"
  482 + "NormLength <- data[grep(\"NormLength\",data$Plot),-c(1)]\n"
  483 + "rm(data)\n"
  484 + "\n")));
  485 +
  486 + p.file.write(qPrintable(QString("summarySE <- function(data=NULL, measurevar, groupvars=NULL, na.rm=FALSE, conf.interval=.95, .drop=TRUE) {\n\t"
  487 + "require(plyr)\n\n\tlength2 <- function (x, na.rm=FALSE) {\n\t\tif (na.rm) sum(!is.na(x))\n\t\telse length(x)"
  488 + "\n\t}\n\n\tdatac <- ddply(data, groupvars, .drop=.drop, .fun = function(xx, col) {\n\t\t"
  489 + "c(N=length2(xx[[col]], na.rm=na.rm), mean=mean(xx[[col]], na.rm=na.rm), sd=sd(xx[[col]], na.rm=na.rm))\n\t\t},"
  490 + "\n\t\tmeasurevar\n\t)\n\n\tdatac <- rename(datac, c(\"mean\" = measurevar))\n\tdatac$se <- datac$sd / sqrt(datac$N)"
  491 + "\n\tciMult <- qt(conf.interval/2 + .5, datac$N-1)\n\tdatac$ci <- datac$se * ciMult\n\n\treturn(datac)\n}\n")));
  492 +
  493 +
  494 + p.file.write(qPrintable(QString("\nreadData <- function(data) {\n\texamples <- list()\n"
  495 + "\tfor (i in 1:nrow(data)) {\n"
  496 + "\t\tpath <- data[i,1]\n"
  497 + "\t\tvalue <- data[i,2]\n"
  498 + "\t\tfile <- unlist(strsplit(path, \"[.]\"))[1]\n"
  499 + "\t\text <- unlist(strsplit(path, \"[.]\"))[2]\n"
  500 + "\t\tif (ext == \"jpg\" || ext == \"JPEG\" || ext == \"jpeg\" || ext == \"JPG\") {\n"
  501 + "\t\t\timg <- readJPEG(path)\n"
  502 + "\t\t} else if (ext == \"PNG\" || ext == \"png\") {\n"
  503 + "\t\t\timg <- readPNG(path)\n"
  504 + "\t\t} else if (ext == \"TIFF\" || ext == \"tiff\" || ext == \"TIF\" || ext == \"tif\") { \n"
  505 + "\t\t\timg <- readTIFF(path)\n"
  506 + "}else {\n"
  507 + "\t\t\tnext\n"
  508 + "\t\t}\n"
  509 + "\t\texample <- list(file = file, value = value, image = img)\n"
  510 + "\t\texamples[[i]] <- example\n"
  511 + "\t}\n"
  512 + "\treturn(examples)\n"
  513 + "}\n")));
  514 +
  515 + p.file.write(qPrintable(QString("\nlibrary(jpeg)\n"
  516 + "library(png)\n"
  517 + "library(grid)\n"
  518 + "multiplot <- function(..., plotlist=NULL, cols) {\n"
  519 + "\trequire(grid)\n"
  520 + "\t# Make a list from the ... arguments and plotlist\n"
  521 + "\tplots <- c(list(...), plotlist)\n"
  522 + "\tnumPlots = length(plots)\n"
  523 + "\t# Make the panel\n"
  524 + "\tplotCols = cols\n"
  525 + "\tplotRows = ceiling(numPlots/plotCols)\n"
  526 + "\t# Set up the page\n"
  527 + "\tgrid.newpage()\n"
  528 + "\tpushViewport(viewport(layout = grid.layout(plotRows, plotCols)))\n"
  529 + "\tvplayout <- function(x, y)\n"
  530 + "\tviewport(layout.pos.row = x, layout.pos.col = y)\n"
  531 + "\t# Make each plot, in the correct location\n"
  532 + "\tfor (i in 1:numPlots) {\n"
  533 + "\t\tcurRow = ceiling(i/plotCols)\n"
  534 + "\t\tcurCol = (i-1) %% plotCols + 1\n"
  535 + "\t\tprint(plots[[i]], vp = vplayout(curRow, curCol))\n"
  536 + "\t}\n"
  537 + "}\n")));
  538 +
  539 + p.file.write(qPrintable(QString("\nplotImage <- function(image, title=NULL, label=NULL) { \n"
  540 + "\tp <- qplot(1:10, 1:10, geom=\"blank\") + annotation_custom(rasterGrob(image$image), xmin=-Inf, xmax=Inf, ymin=-Inf, ymax=Inf) + theme(axis.line=element_blank(), axis.title.y=element_blank(), axis.text.x=element_blank(), axis.text.y=element_blank(), line=element_blank(), axis.ticks=element_blank(), panel.background=element_blank()) + labs(title=title) + xlab(label)\n"
  541 + "\treturn(p)"
  542 + "}\n")));
  543 +
  544 + p.file.write(qPrintable(QString("\nsample <- readData(Sample) \n"
  545 + "print(plotImage(sample[[1]],\"Sample Landmarks\",sprintf(\"Total Landmarks: %s\",sample[[1]]$value))) \n"
  546 + "truthSample <- readData(EXT)\n"
  547 + "predictedSample <- readData(EXP)\n"
  548 + "for (i in 1:length(predictedSample)) {\n"
  549 + "\tmultiplot(plotImage(predictedSample[[i]],\"Predicted Landmarks\",sprintf(\"Average Landmark Error: %.3f\",predictedSample[[i]]$value)),plotImage(truthSample[[i]],\"Ground Truth Landmarks\",\"\"),cols=2)\n"
  550 + "}\n")));
543 551  
544 552 p.file.write(qPrintable(QString("\n"
545 553 "# Code to format error table\n"
... ...