Reimplement name and number tree object helpers

Create a computationally and memory efficient implementation of name and number trees that does binary searches as intended by the data structure rather than loading into a map, which can use a great deal of memory and can be very slow.

Reimplement name and number tree object helpers
Create a computationally and memory efficient implementation of name and number trees that does binary searches as intended by the data structure rather than loading into a map, which can use a great deal of memory and can be very slow.
Jay Berkenbilt
1 parent 9ad6cfd4
Showing 11 changed files with 786 additions and 142 deletions
include/qpdf/QPDFNameTreeObjectHelper.hh
include/qpdf/QPDFNumberTreeObjectHelper.hh
libqpdf/NNTree.cc
libqpdf/QPDFNameTreeObjectHelper.cc
libqpdf/QPDFNumberTreeObjectHelper.cc
libqpdf/build.mk
libqpdf/qpdf/NNTree.hh
libtests/build.mk
libtests/nntree.cc
libtests/qtest/nntree.test
qpdf/qtest/qpdf/name-tree.pdf
@@ -25,17 +25,16 @@
 #include <qpdf/QPDFObjectHelper.hh>
 #include <qpdf/QPDFObjGen.hh>
 #include <map>
+#include <memory>
 #include <qpdf/DLL.h>
 // This is an object helper for name trees. See section 7.9.6 in the
-// PDF spec (ISO 32000) for a description of name trees. This
-// implementation disregards stated limits and sequencing and simply
-// builds a map from string object. If the array of values does not
-// contain a string where expected, this implementation silently skips
-// forward until it finds a string. When looking up items in the name
-// tree, use UTF-8 strings. All names are normalized for lookup
-// purposes.
+// PDF spec (ISO 32000) for a description of name trees. When looking
+// up items in the name tree, use UTF-8 strings. All names are
+// normalized for lookup purposes.
+
+class NNTreeImpl;
 class QPDFNameTreeObjectHelper: public QPDFObjectHelper
 {
@@ -55,6 +54,9 @@ class QPDFNameTreeObjectHelper: public QPDFObjectHelper
     QPDF_DLL
     bool findObject(std::string const& utf8, QPDFObjectHandle& oh);
+    // Return the contents of the name tree as a map. Note that name
+    // trees may be very large, so this may use a lot of RAM. It is
+    // more efficient to use QPDFNameTreeObjectHelper's iterator.
     QPDF_DLL
     std::map<std::string, QPDFObjectHandle> getAsMap() const;
@@ -68,15 +70,12 @@ class QPDFNameTreeObjectHelper: public QPDFObjectHelper
         ~Members();
       private:
-        Members();
-        Members(Members const&);
+        Members(QPDFObjectHandle& oh);
+        Members(Members const&) = delete;
-        std::map<std::string, QPDFObjectHandle> entries;
-        std::set<QPDFObjGen> seen;
+        std::shared_ptr<NNTreeImpl> impl;
     };
-    void updateMap(QPDFObjectHandle oh);
-
     PointerHolder<Members> m;
 };
@@ -24,17 +24,15 @@
 #include <qpdf/QPDFObjectHelper.hh>
 #include <qpdf/QPDFObjGen.hh>
-#include <functional>
 #include <map>
+#include <memory>
 #include <qpdf/DLL.h>
 // This is an object helper for number trees. See section 7.9.7 in the
-// PDF spec (ISO 32000) for a description of number trees. This
-// implementation disregards stated limits and sequencing and simply
-// builds a map from numerical index to object. If the array of
-// numbers does not contain a numerical value where expected, this
-// implementation silently skips forward until it finds a number.
+// PDF spec (ISO 32000) for a description of number trees.
+
+class NNTreeImpl;
 class QPDFNumberTreeObjectHelper: public QPDFObjectHelper
 {
@@ -75,6 +73,10 @@ class QPDFNumberTreeObjectHelper: public QPDFObjectHelper
     bool findObjectAtOrBelow(numtree_number idx, QPDFObjectHandle& oh,
                              numtree_number& offset);
+    // Return the contents of the number tree as a map. Note that
+    // number trees may be very large, so this may use a lot of RAM.
+    // It is more efficient to use QPDFNumberTreeObjectHelper's
+    // iterator.
     typedef std::map<numtree_number, QPDFObjectHandle> idx_map;
     QPDF_DLL
     idx_map getAsMap() const;
@@ -90,23 +92,11 @@ class QPDFNumberTreeObjectHelper: public QPDFObjectHelper
         ~Members();
       private:
-        Members();
-        Members(Members const&);
-
-        // Use a reverse sorted map so we can use the lower_bound
-        // method for searching. lower_bound returns smallest entry
-        // not before the searched entry, meaning that the searched
-        // entry is the lower bound. There's also an upper_bound
-        // method, but it does not do what you'd think it should.
-        // lower_bound implements >=, and upper_bound implements >.
-        typedef std::map<numtree_number,
-                         QPDFObjectHandle,
-                         std::greater<numtree_number> > idx_map;
-        idx_map entries;
-        std::set<QPDFObjGen> seen;
-    };
+        Members(QPDFObjectHandle& oh);
+        Members(Members const&) = delete;
-    void updateMap(QPDFObjectHandle oh);
+        std::shared_ptr<NNTreeImpl> impl;
+    };
     PointerHolder<Members> m;
 };
+#include <qpdf/NNTree.hh>
+#include <qpdf/QUtil.hh>
+
+#include <exception>
+
+NNTreeIterator::PathElement::PathElement(
+    QPDFObjectHandle const& node, int kid_number) :
+    node(node),
+    kid_number(kid_number)
+{
+}
+
+QPDFObjectHandle
+NNTreeIterator::PathElement::getNextKid(bool backward)
+{
+    kid_number += backward ? -1 : 1;
+    auto kids = node.getKey("/Kids");
+    QPDFObjectHandle result;
+    if ((kid_number >= 0) && (kid_number < kids.getArrayNItems()))
+    {
+        result = kids.getArrayItem(kid_number);
+    }
+    else
+    {
+        result = QPDFObjectHandle::newNull();
+    }
+    return result;
+}
+
+bool
+NNTreeIterator::valid() const
+{
+    return this->item_number >= 0;
+}
+
+void
+NNTreeIterator::increment(bool backward)
+{
+    if (this->item_number < 0)
+    {
+        throw std::logic_error(
+            "attempt made to increment or decrement an invalid"
+            " name/number tree iterator");
+    }
+    this->item_number += backward ? -2 : 2;
+    auto items = this->node.getKey(details.itemsKey());
+    if ((this->item_number < 0) ||
+        (this->item_number >= items.getArrayNItems()))
+    {
+        bool found = false;
+        setItemNumber(QPDFObjectHandle(), -1);
+        while (! (found || this->path.empty()))
+        {
+            auto& element = this->path.back();
+            auto node = element.getNextKid(backward);
+            if (node.isNull())
+            {
+                this->path.pop_back();
+            }
+            else
+            {
+                deepen(node, ! backward);
+                found = true;
+            }
+        }
+    }
+}
+
+NNTreeIterator&
+NNTreeIterator::operator++()
+{
+    increment(false);
+    return *this;
+}
+
+NNTreeIterator&
+NNTreeIterator::operator--()
+{
+    increment(true);
+    return *this;
+}
+
+NNTreeIterator::reference
+NNTreeIterator::operator*()
+{
+    if (this->item_number < 0)
+    {
+        throw std::logic_error(
+            "attempt made to dereference an invalid"
+            " name/number tree iterator");
+    }
+    auto items = this->node.getKey(details.itemsKey());
+    return std::make_pair(items.getArrayItem(this->item_number),
+                          items.getArrayItem(1+this->item_number));
+}
+
+bool
+NNTreeIterator::operator==(NNTreeIterator const& other) const
+{
+    if ((this->item_number == -1) && (other.item_number == -1))
+    {
+        return true;
+    }
+    if (this->path.size() != other.path.size())
+    {
+        return false;
+    }
+    auto tpi = this->path.begin();
+    auto opi = other.path.begin();
+    while (tpi != this->path.end())
+    {
+        if ((*tpi).kid_number != (*opi).kid_number)
+        {
+            return false;
+        }
+        ++tpi;
+        ++opi;
+    }
+    if (this->item_number != other.item_number)
+    {
+        return false;
+    }
+    return true;
+}
+
+void
+NNTreeIterator::setItemNumber(QPDFObjectHandle const& node, int n)
+{
+    this->node = node;
+    this->item_number = n;
+}
+
+void
+NNTreeIterator::addPathElement(QPDFObjectHandle const& node,
+                               int kid_number)
+{
+    this->path.push_back(PathElement(node, kid_number));
+}
+
+void
+NNTreeIterator::deepen(QPDFObjectHandle node, bool first)
+{
+    std::set<QPDFObjGen> seen;
+    while (true)
+    {
+        if (node.isIndirect())
+        {
+            auto og = node.getObjGen();
+            if (seen.count(og))
+            {
+                throw std::runtime_error("loop detected");
+            }
+            seen.insert(og);
+        }
+        auto kids = node.getKey("/Kids");
+        int nkids = kids.isArray() ? kids.getArrayNItems() : 0;
+        auto items = node.getKey(details.itemsKey());
+        int nitems = items.isArray() ? items.getArrayNItems() : 0;
+        if (nitems > 0)
+        {
+            setItemNumber(node, first ? 0 : nitems - 2);
+            break;
+        }
+        else if (nkids > 0)
+        {
+            int kid_number = first ? 0 : nkids - 1;
+            addPathElement(node, kid_number);
+            node = kids.getArrayItem(kid_number);
+        }
+        else
+        {
+            throw std::runtime_error("node has neither /Kids nor /Names");
+        }
+    }
+}
+
+NNTreeImpl::NNTreeImpl(NNTreeDetails const& details,
+                       QPDF* qpdf,
+                       QPDFObjectHandle& oh,
+                       bool auto_repair) :
+    details(details),
+    oh(oh)
+{
+}
+
+NNTreeImpl::iterator
+NNTreeImpl::begin()
+{
+    iterator result(details);
+    result.deepen(this->oh, true);
+    return result;
+}
+
+NNTreeImpl::iterator
+NNTreeImpl::end()
+{
+    return iterator(details);
+}
+
+NNTreeImpl::iterator
+NNTreeImpl::last()
+{
+    iterator result(details);
+    result.deepen(this->oh, false);
+    return result;
+}
+
+int
+NNTreeImpl::withinLimits(QPDFObjectHandle key, QPDFObjectHandle node)
+{
+    int result = 0;
+    auto limits = node.getKey("/Limits");
+    if (limits.isArray() && (limits.getArrayNItems() >= 2) &&
+        details.keyValid(limits.getArrayItem(0)) &&
+        details.keyValid(limits.getArrayItem(1)))
+    {
+        if (details.compareKeys(key, limits.getArrayItem(0)) < 0)
+        {
+            result = -1;
+        }
+        else if (details.compareKeys(key, limits.getArrayItem(1)) > 0)
+        {
+            result = 1;
+        }
+    }
+    else
+    {
+        // The root node has no limits, so consider the item to be in
+        // here if there are no limits. This will cause checking lower
+        // items.
+    }
+    return result;
+}
+
+int
+NNTreeImpl::binarySearch(
+    QPDFObjectHandle key, QPDFObjectHandle items,
+    int num_items, bool return_prev_if_not_found,
+    int (NNTreeImpl::*compare)(QPDFObjectHandle& key,
+                             QPDFObjectHandle& node,
+                             int item))
+{
+    int max_idx = 1;
+    while (max_idx < num_items)
+    {
+        max_idx <<= 1;
+    }
+
+    int step = max_idx / 2;
+    int checks = max_idx;
+    int idx = step;
+    int found_idx = -1;
+    bool found = false;
+    bool found_leq = false;
+    int status = 0;
+
+    while ((! found) && (checks > 0))
+    {
+        if (idx < num_items)
+        {
+            status = (this->*compare)(key, items, idx);
+            if (status >= 0)
+            {
+                found_leq = true;
+                found_idx = idx;
+            }
+        }
+        else
+        {
+            // consider item to be below anything after the top
+            status = -1;
+        }
+
+        if (status == 0)
+        {
+            found = true;
+        }
+        else
+        {
+            checks >>= 1;
+            if (checks > 0)
+            {
+                step >>= 1;
+                if (step == 0)
+                {
+                    step = 1;
+                }
+
+                if (status < 0)
+                {
+                    idx -= step;
+                }
+                else
+                {
+                    idx += step;
+                }
+            }
+        }
+    }
+
+    if (found || (found_leq && return_prev_if_not_found))
+    {
+        return found_idx;
+    }
+    else
+    {
+        return -1;
+    }
+}
+
+int
+NNTreeImpl::compareKeyItem(
+    QPDFObjectHandle& key, QPDFObjectHandle& items, int idx)
+{
+    if (! ((items.isArray() && (items.getArrayNItems() > (2 * idx)) &&
+            details.keyValid(items.getArrayItem(2 * idx)))))
+    {
+        throw std::runtime_error("item at index " +
+                                 QUtil::int_to_string(2 * idx) +
+                                 " is not the right type");
+    }
+    return details.compareKeys(key, items.getArrayItem(2 * idx));
+}
+
+int
+NNTreeImpl::compareKeyKid(QPDFObjectHandle& key, QPDFObjectHandle& kids, int idx)
+{
+    if (! (kids.isArray() && (idx < kids.getArrayNItems()) &&
+           kids.getArrayItem(idx).isDictionary()))
+    {
+        throw std::runtime_error("invalid kid at index " +
+                                 QUtil::int_to_string(idx));
+    }
+    return withinLimits(key, kids.getArrayItem(idx));
+}
+
+
+NNTreeImpl::iterator
+NNTreeImpl::find(QPDFObjectHandle key, bool return_prev_if_not_found)
+{
+    auto first_item = begin();
+    auto last_item = end();
+    if (first_item.valid() &&
+        details.keyValid((*first_item).first) &&
+        details.compareKeys(key, (*first_item).first) < 0)
+    {
+        // Before the first key
+        return end();
+    }
+    else if (last_item.valid() &&
+             details.keyValid((*last_item).first) &&
+             details.compareKeys(key, (*last_item).first) > 0)
+    {
+        // After the last key
+        if (return_prev_if_not_found)
+        {
+            return last_item;
+        }
+        else
+        {
+            return end();
+        }
+    }
+
+    std::set<QPDFObjGen> seen;
+    auto node = this->oh;
+    iterator result(details);
+
+    while (true)
+    {
+        auto og = node.getObjGen();
+        if (seen.count(og))
+        {
+            throw std::runtime_error("loop detected in find");
+        }
+        seen.insert(og);
+
+        auto kids = node.getKey("/Kids");
+        int nkids = kids.isArray() ? kids.getArrayNItems() : 0;
+        auto items = node.getKey(details.itemsKey());
+        int nitems = items.isArray() ? items.getArrayNItems() : 0;
+        if (nitems > 0)
+        {
+            int idx = binarySearch(
+                key, items, nitems / 2, return_prev_if_not_found,
+                &NNTreeImpl::compareKeyItem);
+            if (idx >= 0)
+            {
+                result.setItemNumber(node, 2 * idx);
+            }
+            break;
+        }
+        else if (nkids > 0)
+        {
+            int idx = binarySearch(
+                key, kids, nkids, true,
+                &NNTreeImpl::compareKeyKid);
+            if (idx == -1)
+            {
+                throw std::runtime_error(
+                    "unexpected -1 from binary search of kids;"
+                    " tree may not be sorted");
+            }
+            result.addPathElement(node, idx);
+            node = kids.getArrayItem(idx);
+        }
+        else
+        {
+            throw std::runtime_error("bad node during find");
+        }
+    }
+
+    return result;
+}
 #include <qpdf/QPDFNameTreeObjectHelper.hh>
+#include <qpdf/NNTree.hh>
+
+class NameTreeDetails: public NNTreeDetails
+{
+  public:
+    virtual std::string const& itemsKey() const override
+    {
+        static std::string k("/Names");
+        return k;
+    }
+    virtual bool keyValid(QPDFObjectHandle oh) const override
+    {
+        return oh.isString();
+    }
+    virtual int compareKeys(
+        QPDFObjectHandle a, QPDFObjectHandle b) const override
+    {
+        if (! (keyValid(a) && keyValid(b)))
+        {
+            // We don't call this without calling keyValid first
+            throw std::logic_error("comparing invalid keys");
+        }
+        auto as = a.getUTF8Value();
+        auto bs = b.getUTF8Value();
+        return ((as < bs) ? -1 : (as > bs) ? 1 : 0);
+    }
+};
+
+static NameTreeDetails name_tree_details;
 QPDFNameTreeObjectHelper::Members::~Members()
 {
 }
-QPDFNameTreeObjectHelper::Members::Members()
+QPDFNameTreeObjectHelper::Members::Members(QPDFObjectHandle& oh) :
+    impl(std::make_shared<NNTreeImpl>(name_tree_details, nullptr, oh, false))
 {
 }
 QPDFNameTreeObjectHelper::QPDFNameTreeObjectHelper(QPDFObjectHandle oh) :
     QPDFObjectHelper(oh),
-    m(new Members())
+    m(new Members(oh))
 {
-    updateMap(oh);
 }
 QPDFNameTreeObjectHelper::~QPDFNameTreeObjectHelper()
 {
 }
-void
-QPDFNameTreeObjectHelper::updateMap(QPDFObjectHandle oh)
-{
-    if (this->m->seen.count(oh.getObjGen()))
-    {
-        return;
-    }
-    this->m->seen.insert(oh.getObjGen());
-    QPDFObjectHandle names = oh.getKey("/Names");
-    if (names.isArray())
-    {
-        int nitems = names.getArrayNItems();
-        int i = 0;
-        while (i < nitems - 1)
-        {
-            QPDFObjectHandle name = names.getArrayItem(i);
-            if (name.isString())
-            {
-                ++i;
-                QPDFObjectHandle obj = names.getArrayItem(i);
-                this->m->entries[name.getUTF8Value()] = obj;
-            }
-            ++i;
-        }
-    }
-    QPDFObjectHandle kids = oh.getKey("/Kids");
-    if (kids.isArray())
-    {
-        int nitems = kids.getArrayNItems();
-        for (int i = 0; i < nitems; ++i)
-        {
-            updateMap(kids.getArrayItem(i));
-        }
-    }
-}
-
 bool
 QPDFNameTreeObjectHelper::hasName(std::string const& name)
 {
-    return this->m->entries.count(name) != 0;
+    auto i = this->m->impl->find(QPDFObjectHandle::newUnicodeString(name));
+    return (i != this->m->impl->end());
 }
 bool
 QPDFNameTreeObjectHelper::findObject(
     std::string const& name, QPDFObjectHandle& oh)
 {
-    std::map<std::string, QPDFObjectHandle>::iterator i =
-        this->m->entries.find(name);
-    if (i == this->m->entries.end())
+    auto i = this->m->impl->find(QPDFObjectHandle::newUnicodeString(name));
+    if (i == this->m->impl->end())
     {
         return false;
     }
@@ -78,5 +71,12 @@ QPDFNameTreeObjectHelper::findObject(
 std::map<std::string, QPDFObjectHandle>
 QPDFNameTreeObjectHelper::getAsMap() const
 {
-    return this->m->entries;
+    std::map<std::string, QPDFObjectHandle> result;
+    for (auto i: *(this->m->impl))
+    {
+        result.insert(
+            std::make_pair(i.first.getUTF8Value(),
+                           i.second));
+    }
+    return result;
 }
 #include <qpdf/QPDFNumberTreeObjectHelper.hh>
+#include <qpdf/NNTree.hh>
-QPDFNumberTreeObjectHelper::Members::~Members()
-{
-}
-
-QPDFNumberTreeObjectHelper::Members::Members()
+class NumberTreeDetails: public NNTreeDetails
 {
-}
-
-QPDFNumberTreeObjectHelper::QPDFNumberTreeObjectHelper(QPDFObjectHandle oh) :
-    QPDFObjectHelper(oh),
-    m(new Members())
-{
-    updateMap(oh);
-}
-
-void
-QPDFNumberTreeObjectHelper::updateMap(QPDFObjectHandle oh)
-{
-    if (this->m->seen.count(oh.getObjGen()))
+  public:
+    virtual std::string const& itemsKey() const override
     {
-        return;
+        static std::string k("/Nums");
+        return k;
     }
-    this->m->seen.insert(oh.getObjGen());
-    QPDFObjectHandle nums = oh.getKey("/Nums");
-    if (nums.isArray())
+    virtual bool keyValid(QPDFObjectHandle oh) const override
     {
-        int nitems = nums.getArrayNItems();
-        int i = 0;
-        while (i < nitems - 1)
-        {
-            QPDFObjectHandle num = nums.getArrayItem(i);
-            if (num.isInteger())
-            {
-                ++i;
-                QPDFObjectHandle obj = nums.getArrayItem(i);
-                this->m->entries[num.getIntValue()] = obj;
-            }
-            ++i;
-        }
+        return oh.isInteger();
     }
-    QPDFObjectHandle kids = oh.getKey("/Kids");
-    if (kids.isArray())
+    virtual int compareKeys(
+        QPDFObjectHandle a, QPDFObjectHandle b) const override
     {
-        int nitems = kids.getArrayNItems();
-        for (int i = 0; i < nitems; ++i)
+        if (! (keyValid(a) && keyValid(b)))
         {
-            updateMap(kids.getArrayItem(i));
+            // We don't call this without calling keyValid first
+            throw std::logic_error("comparing invalid keys");
         }
+        auto as = a.getIntValue();
+        auto bs = b.getIntValue();
+        return ((as < bs) ? -1 : (as > bs) ? 1 : 0);
     }
+};
+
+static NumberTreeDetails number_tree_details;
+
+QPDFNumberTreeObjectHelper::Members::~Members()
+{
+}
+
+QPDFNumberTreeObjectHelper::Members::Members(QPDFObjectHandle& oh) :
+    impl(std::make_shared<NNTreeImpl>(number_tree_details, nullptr, oh, false))
+{
 }
+QPDFNumberTreeObjectHelper::QPDFNumberTreeObjectHelper(QPDFObjectHandle oh) :
+    QPDFObjectHelper(oh),
+    m(new Members(oh))
+{
+}
 QPDFNumberTreeObjectHelper::numtree_number
 QPDFNumberTreeObjectHelper::getMin()
 {
-    if (this->m->entries.empty())
+    auto i = this->m->impl->begin();
+    if (i == this->m->impl->end())
     {
         return 0;
     }
-    // Our map is sorted in reverse.
-    return this->m->entries.rbegin()->first;
+    return (*i).first.getIntValue();
 }
 QPDFNumberTreeObjectHelper::numtree_number
 QPDFNumberTreeObjectHelper::getMax()
 {
-    if (this->m->entries.empty())
+    auto i = this->m->impl->last();
+    if (i == this->m->impl->end())
     {
         return 0;
     }
-    // Our map is sorted in reverse.
-    return this->m->entries.begin()->first;
+    return (*i).first.getIntValue();
 }
 bool
 QPDFNumberTreeObjectHelper::hasIndex(numtree_number idx)
 {
-    return this->m->entries.count(idx) != 0;
+    auto i = this->m->impl->find(QPDFObjectHandle::newInteger(idx));
+    return (i != this->m->impl->end());
 }
 bool
 QPDFNumberTreeObjectHelper::findObject(
     numtree_number idx, QPDFObjectHandle& oh)
 {
-    Members::idx_map::iterator i = this->m->entries.find(idx);
-    if (i == this->m->entries.end())
+    auto i = this->m->impl->find(QPDFObjectHandle::newInteger(idx));
+    if (i == this->m->impl->end())
     {
         return false;
     }
@@ -98,13 +91,13 @@ QPDFNumberTreeObjectHelper::findObjectAtOrBelow(
     numtree_number idx, QPDFObjectHandle& oh,
     numtree_number& offset)
 {
-    Members::idx_map::iterator i = this->m->entries.lower_bound(idx);
-    if (i == this->m->entries.end())
+    auto i = this->m->impl->find(QPDFObjectHandle::newInteger(idx), true);
+    if (i == this->m->impl->end())
     {
         return false;
     }
     oh = (*i).second;
-    offset = idx - (*i).first;
+    offset = idx - (*i).first.getIntValue();
     return true;
 }
@@ -112,10 +105,11 @@ std::map&lt;QPDFNumberTreeObjectHelper::numtree_number, QPDFObjectHandle&gt;
 QPDFNumberTreeObjectHelper::getAsMap() const
 {
     std::map<numtree_number, QPDFObjectHandle> result;
-    for (Members::idx_map::const_iterator iter = this->m->entries.begin();
-         iter != this->m->entries.end(); ++iter)
+    for (auto i: *(this->m->impl))
     {
-        result[(*iter).first] = (*iter).second;
+        result.insert(
+            std::make_pair(i.first.getIntValue(),
+                           i.second));
     }
     return result;
 }
@@ -33,6 +33,7 @@ SRCS_libqpdf = \
 	libqpdf/InsecureRandomDataProvider.cc \
 	libqpdf/JSON.cc \
 	libqpdf/MD5.cc \
+	libqpdf/NNTree.cc \
 	libqpdf/OffsetInputSource.cc \
 	libqpdf/Pipeline.cc \
 	libqpdf/Pl_AES_PDF.cc \
+#ifndef NNTREE_HH
+#define NNTREE_HH
+
+#include <qpdf/QPDF.hh>
+#include <qpdf/QPDFObjectHandle.hh>
+
+#include <iterator>
+#include <list>
+
+class NNTreeDetails
+{
+  public:
+    virtual std::string const& itemsKey() const = 0;
+    virtual bool keyValid(QPDFObjectHandle) const = 0;
+    virtual int compareKeys(QPDFObjectHandle, QPDFObjectHandle) const = 0;
+};
+
+class NNTreeIterator: public std::iterator<
+    std::bidirectional_iterator_tag,
+    std::pair<QPDFObjectHandle, QPDFObjectHandle>,
+    void,
+    std::pair<QPDFObjectHandle, QPDFObjectHandle>*,
+    std::pair<QPDFObjectHandle, QPDFObjectHandle>>
+{
+    friend class NNTreeImpl;
+  public:
+    bool valid() const;
+    NNTreeIterator& operator++();
+    NNTreeIterator operator++(int)
+    {
+        NNTreeIterator t = *this;
+        ++(*this);
+        return t;
+    }
+    NNTreeIterator& operator--();
+    NNTreeIterator operator--(int)
+    {
+        NNTreeIterator t = *this;
+        --(*this);
+        return t;
+    }
+    reference operator*();
+    bool operator==(NNTreeIterator const& other) const;
+    bool operator!=(NNTreeIterator const& other) const
+    {
+        return ! operator==(other);
+    }
+
+  private:
+    class PathElement
+    {
+      public:
+        PathElement(QPDFObjectHandle const& node, int kid_number);
+        QPDFObjectHandle getNextKid(bool backward);
+
+        QPDFObjectHandle node;
+        int kid_number;
+    };
+
+    NNTreeIterator(NNTreeDetails const& details) :
+        details(details),
+        item_number(-1)
+    {
+    }
+    void deepen(QPDFObjectHandle node, bool first);
+    void setItemNumber(QPDFObjectHandle const& node, int);
+    void addPathElement(QPDFObjectHandle const& node, int kid_number);
+    void increment(bool backward);
+
+    NNTreeDetails const& details;
+    std::list<PathElement> path;
+    QPDFObjectHandle node;
+    int item_number;
+};
+
+class NNTreeImpl
+{
+  public:
+    typedef NNTreeIterator iterator;
+
+    NNTreeImpl(NNTreeDetails const&, QPDF*, QPDFObjectHandle&,
+               bool auto_repair = true);
+    iterator begin();
+    iterator end();
+    iterator last();
+    iterator find(QPDFObjectHandle key, bool return_prev_if_not_found = false);
+
+  private:
+    int withinLimits(QPDFObjectHandle key, QPDFObjectHandle node);
+    int binarySearch(
+        QPDFObjectHandle key, QPDFObjectHandle items,
+        int num_items, bool return_prev_if_not_found,
+        int (NNTreeImpl::*compare)(QPDFObjectHandle& key,
+                                 QPDFObjectHandle& node,
+                                 int item));
+    int compareKeyItem(
+        QPDFObjectHandle& key, QPDFObjectHandle& items, int idx);
+    int compareKeyKid(
+        QPDFObjectHandle& key, QPDFObjectHandle& items, int idx);
+
+    NNTreeDetails const& details;
+    QPDFObjectHandle oh;
+};
+
+#endif // NNTREE_HH
@@ -16,6 +16,7 @@ BINS_libtests = \
 	main_from_wmain \
 	matrix \
 	md5 \
+	nntree \
 	numrange \
 	pointer_holder \
 	predictors \
+#include <qpdf/QPDFNumberTreeObjectHelper.hh>
+#include <qpdf/QPDF.hh>
+#include <qpdf/QUtil.hh>
+#include <iostream>
+
+bool report(QPDFObjectHandle oh, long long item, long long exp_item)
+{
+    QPDFNumberTreeObjectHelper nh(oh);
+    QPDFObjectHandle o1;
+    long long offset = 0;
+    bool f1 = nh.findObjectAtOrBelow(item, o1, offset);
+    QPDFObjectHandle o2;
+    bool f2 = nh.findObject(item, o2);
+
+    bool failed = false;
+    auto show = [&failed, &oh, &item] () {
+        if (! failed)
+        {
+            failed = true;
+            std::cout << "key = " << item << ", oh = "
+                      << oh.unparseResolved() << std::endl;
+        }
+    };
+
+    auto mk_wanted = [](long long i) {
+        return ((i == -1)
+                ? "end"
+                : (QUtil::int_to_string(i) +
+                   "/(-" + QUtil::int_to_string(i) + "-)"));
+    };
+    std::string i1_wanted = mk_wanted(exp_item);
+    std::string i2_wanted = mk_wanted(item == exp_item ? item : -1);
+    auto mk_actual = [](bool found, long long v, QPDFObjectHandle& o) {
+        return (found
+                ? QUtil::int_to_string(v) + "/" + o.unparse()
+                : "end");
+    };
+    std::string i1_actual = mk_actual(f1, item - offset, o1);
+    std::string i2_actual = mk_actual(f2, item, o2);
+
+    if (i1_wanted != i1_actual)
+    {
+        show();
+        std::cout << "i1: wanted " << i1_wanted
+                  << ", got " << i1_actual
+                  << std::endl;
+    }
+    if (i2_wanted != i2_actual)
+    {
+        show();
+        std::cout << "i2: wanted " << i2_wanted
+                  << ", got " << i2_actual
+                  << std::endl;
+    }
+
+    return failed;
+}
+
+int main()
+{
+    QPDF q;
+    q.emptyPDF();
+
+    auto mk = [&q] (std::vector<int> const& v) {
+        auto nums = QPDFObjectHandle::newArray();
+        for (auto i: v)
+        {
+            nums.appendItem(QPDFObjectHandle::newInteger(i));
+            nums.appendItem(QPDFObjectHandle::newString(
+                                "-" + QUtil::int_to_string(i) + "-"));
+        }
+        auto limits = QPDFObjectHandle::newArray();
+        limits.appendItem(QPDFObjectHandle::newInteger(v.at(0)));
+        limits.appendItem(QPDFObjectHandle::newInteger(v.at(v.size() - 1)));
+        auto node = q.makeIndirectObject(QPDFObjectHandle::newDictionary());
+        node.replaceKey("/Nums", nums);
+        node.replaceKey("/Limits", limits);
+        return node;
+    };
+
+    bool any_failures = false;
+    auto r = [&any_failures](QPDFObjectHandle& oh, int item, int exp) {
+        if (report(oh, item, exp))
+        {
+            any_failures = true;
+        }
+    };
+
+    auto a = mk({2, 3, 5, 9, 11, 12, 14, 18});
+    r(a, 1, -1);
+    r(a, 2, 2);
+    r(a, 9, 9);
+    r(a, 11, 11);
+    r(a, 10, 9);
+    r(a, 7, 5);
+    r(a, 18, 18);
+    r(a, 19, 18);
+
+    auto b = mk({2, 4});
+    r(b, 1, -1);
+    r(b, 2, 2);
+    r(b, 3, 2);
+    r(b, 4, 4);
+    r(b, 5, 4);
+
+    auto c = mk({3});
+    r(c, 1, -1);
+    r(c, 3, 3);
+    r(c, 5, 3);
+
+    auto d = mk({2, 3, 5, 9, 10, 12, 14, 18, 19, 20});
+    r(d, 1, -1);
+    r(d, 2, 2);
+    r(d, 18, 18);
+    r(d, 14, 14);
+    r(d, 19, 19);
+    r(d, 20, 20);
+    r(d, 25, 20);
+
+    if (! any_failures)
+    {
+        std::cout << "all tests passed" << std::endl;
+    }
+}
+#!/usr/bin/env perl
+require 5.008;
+use warnings;
+use strict;
+
+require TestDriver;
+
+my $td = new TestDriver('nntree');
+
+$td->runtest("nntree",
+	     {$td->COMMAND => "nntree"},
+	     {$td->STRING => "all tests passed\n",
+	      $td->EXIT_STATUS => 0},
+	     $td->NORMALIZE_NEWLINES);
+
+$td->report(1);
@@ -91,8 +91,8 @@ endobj
     12 0 R
   ]
   /Limits [
-    0
-    19
+    (01 one)
+    (15 fifteen)
   ]
 >>
 endobj
@@ -100,8 +100,8 @@ endobj
 10 0 obj
 <<
   /Limits [
-    20
-    29
+    (20 twenty)
+    (29 twenty-nine)
   ]
   /Names [
     (20 twenty) (twenty.)
@@ -152,9 +152,9 @@ xref
 0000000612 00000 n 
 0000000647 00000 n 
 0000000704 00000 n 
-0000000791 00000 n 
-0000000955 00000 n 
-0000001151 00000 n 
+0000000808 00000 n 
+0000000995 00000 n 
+0000001191 00000 n 
 trailer <<
   /Root 1 0 R
   /QTest 8 0 R
@@ -162,5 +162,5 @@ trailer &lt;&lt;
   /ID [<2c3b7a6ec7fc61db8a5db4eebf57f540><2c3b7a6ec7fc61db8a5db4eebf57f540>]
 >>
 startxref
-1325
+1365
 %%EOF