Updated deduplicate to use tailOutput

Scott Klum
1 parent d5486fb5
Showing 4 changed files with 40 additions and 25 deletions
openbr/core/core.cpp
openbr/openbr.cpp
openbr/openbr.h
openbr/plugins/output.cpp
@@ -263,33 +263,42 @@ struct AlgorithmCore
  
         TemplateList t = i->read();
  
-        QList<int> duplicates;
+        Output *o = Output::make(QString("buffer.tail[selfSimilar,threshold=%1,atLeast=0]").arg(QString::number(threshold)),inputFiles,inputFiles);
  
-        Globals->totalSteps = inputFiles.size();
-        Globals->currentStep = 0;
+        // Compare to global tail output
+        distance->compare(t,t,o);
  
-        for (int i=0; i<t.size(); i++) {
-            for (int j=0; j<i; j++) {
-                float score = distance->compare(t[i], t[j]);
-                if (score >= threshold) {
-                    duplicates.append(i);
-                    break;
-                }
-            }
-            Globals->currentStep++;
-            Globals->printStatus();
-        }
+        delete o;
+
+        QString buffer(Globals->buffer);
+
+        QStringList tail = buffer.split("\n");
+
+        // Remove header
+        tail.removeFirst();
+
+        QStringList toRemove;
+        foreach(const QString &s, tail)
+            toRemove.append(s.split(',').at(1));
+
+        QSet<QString> duplicates = QSet<QString>::fromList(toRemove);
+
+        QStringList fileNames = inputFiles.names();
+
+        QList<int> indices;
+        foreach(const QString &d, duplicates)
+            indices.append(fileNames.indexOf(d));
  
-        std::sort(duplicates.begin(),duplicates.end(),std::greater<float>());
+        std::sort(indices.begin(),indices.end(),std::greater<float>());
  
-        qDebug("\n%d duplicates removed.", duplicates.size());
+        qDebug("\n%d duplicates removed.", indices.size());
  
-        for (int i=0; i<duplicates.size(); i++)
-            inputFiles.removeAt(duplicates[i]);
+        for (int i=0; i<indices.size(); i++)
+            inputFiles.removeAt(indices[i]);
  
-        QScopedPointer<Gallery> o(Gallery::make(outputGallery));
+        QScopedPointer<Gallery> og(Gallery::make(outputGallery));
  
-        o->writeBlock(inputFiles);
+        og->writeBlock(inputFiles);
     }
  
     void compare(File targetGallery, File queryGallery, File output)
@@ -471,7 +471,7 @@ void br_close_gallery(br_gallery gallery)
     delete gal;
 }
  
-void br_deduplicate(const char *inputGallery, const char *outputGallery, const char *threshold)
+void br_deduplicate(const char *input_gallery, const char *output_gallery, const char *threshold)
 {
-    br::Deduplicate(inputGallery, outputGallery, threshold);
+    br::Deduplicate(input_gallery, output_gallery, threshold);
 }
@@ -67,10 +67,15 @@ BR_EXPORT const char *br_about();
 BR_EXPORT void br_cat(int num_input_galleries, const char *input_galleries[], const char *output_gallery);
  
 /*!
- * \brief Wraps br::Deduplicate()
+ * \brief Removes duplicate templates in a gallery.
+ * \param input_gallery Gallery to be deduplicated.
+ * \param output_gallery Deduplicated gallery.
+ * \param threshold Comparisons with a match score >= this value are designated to be duplicates.
+ * \note If a gallery contains n duplicates, the first n-1 duplicates in the gallery will be removed and the nth will be kept.
+ * \note Users are encouraged to use binary gallery formats as the entire gallery is read into memory in one call to Gallery::read.
  */
  
-BR_EXPORT void br_deduplicate(const char *inputGallery, const char *outputGallery, const char *threshold);
+BR_EXPORT void br_deduplicate(const char *input_gallery, const char *output_gallery, const char *threshold);
  
 /*!
  * \brief Clusters one or more similarity matrices into a list of subjects.
@@ -528,7 +528,7 @@ class tailOutput : public Output
         } else {
             // General case
             for (int k=0; k<comparisons.size(); k++) {
-                if (comparisons[k].value < value) {
+                if (comparisons[k].value <= value) {
                     comparisons.insert(k, Comparison(queryFiles[i], targetFiles[j], value));
                     break;
                 }
@@ -539,6 +539,7 @@ class tailOutput : public Output
             comparisons.removeLast();
         while ((comparisons.size() > atLeast) && (comparisons.last().value < threshold))
             comparisons.removeLast();
+
         lastValue = comparisons.last().value;
         comparisonsLock.unlock();
     }