Merge pull request #1397 from m-holger/ostream

Refactor QPDF::resolveObjectsInStream

Merge pull request #1397 from m-holger/ostream
Refactor QPDF::resolveObjectsInStream
m-holger · GitHub
2 parents 49043112 626d5061
Showing 9 changed files with 125 additions and 70 deletions
include/qpdf/QPDF.hh
libqpdf/QPDFObject.cc
libqpdf/QPDFParser.cc
libqpdf/QPDF_objects.cc
libqpdf/qpdf/QPDFObjectHandle_private.hh
libqpdf/qpdf/QPDFObject_private.hh
libqpdf/qpdf/QPDFParser.hh
libqpdf/qpdf/QPDF_private.hh
manual/release-notes.rst
@@ -48,6 +48,7 @@
 class QPDF_Stream;
 class BitStream;
 class BitWriter;
+class BufferInputSource;
 class QPDFLogger;
 class QPDFParser;
@@ -784,7 +785,7 @@ class QPDF
     QPDFObjectHandle readObject(std::string const& description, QPDFObjGen og);
     void readStream(QPDFObjectHandle& object, QPDFObjGen og, qpdf_offset_t offset);
     void validateStreamLineEnd(QPDFObjectHandle& object, QPDFObjGen og, qpdf_offset_t offset);
-    QPDFObjectHandle readObjectInStream(std::shared_ptr<InputSource>& input, int obj);
+    QPDFObjectHandle readObjectInStream(BufferInputSource& input, int stream_id, int obj_id);
     size_t recoverStreamLength(
         std::shared_ptr<InputSource> input, QPDFObjGen og, qpdf_offset_t stream_offset);
     QPDFTokenizer::Token readToken(InputSource&, size_t max_len = 0);
@@ -3,6 +3,10 @@
 std::string
 QPDFObject::getDescription()
 {
+    qpdf_offset_t shift = (getTypeCode() == ::ot_dictionary) ? 2
+        : (getTypeCode() == ::ot_array)                      ? 1
+                                                             : 0;
+
     if (object_description) {
         switch (object_description->index()) {
         case 0:
@@ -14,10 +18,6 @@ QPDFObject::getDescription()
                     description.replace(pos, 3, og.unparse(' '));
                 }
                 if (auto pos = description.find("$PO"); pos != std::string::npos) {
-                    qpdf_offset_t shift = (getTypeCode() == ::ot_dictionary) ? 2
-                        : (getTypeCode() == ::ot_array)                      ? 1
-                                                                             : 0;
-
                     description.replace(pos, 3, std::to_string(parsed_offset + shift));
                 }
                 return description;
@@ -44,7 +44,14 @@ QPDFObject::getDescription()
                 }
                 return result;
             }
+        case 3:
+            auto [stream_id, obj_id] = std::get<3>(*object_description);
+            std::string result = qpdf ? qpdf->getFilename() : "";
+            result += " object stream " + std::to_string(stream_id) + ", object " +
+                std::to_string(obj_id) + " 0 at offset " + std::to_string(parsed_offset + shift);
+            return result;
         }
+
     } else if (og.isIndirect()) {
         return "object " + og.unparse(' ');
     }
@@ -10,6 +10,8 @@
 #include <memory>
+using namespace std::literals;
+
 using ObjectPtr = std::shared_ptr<QPDFObject>;
 QPDFObjectHandle
@@ -524,7 +526,13 @@ QPDFParser::warnDuplicateKey()
 void
 QPDFParser::warn(qpdf_offset_t offset, std::string const& msg) const
 {
-    warn(QPDFExc(qpdf_e_damaged_pdf, input.getName(), object_description, offset, msg));
+    if (stream_id) {
+        std::string descr = "object "s + std::to_string(obj_id) + " 0";
+        std::string name = context->getFilename() + " object stream " + std::to_string(stream_id);
+        warn(QPDFExc(qpdf_e_damaged_pdf, name, descr, offset, msg));
+    } else {
+        warn(QPDFExc(qpdf_e_damaged_pdf, input.getName(), object_description, offset, msg));
+    }
 }
 void
@@ -1292,19 +1292,22 @@ QPDF::validateStreamLineEnd(QPDFObjectHandle&amp; object, QPDFObjGen og, qpdf_offset
 }
 QPDFObjectHandle
-QPDF::readObjectInStream(std::shared_ptr<InputSource>& input, int obj)
+QPDF::readObjectInStream(BufferInputSource& input, int stream_id, int obj_id)
 {
-    m->last_object_description.erase(7); // last_object_description starts with "object "
-    m->last_object_description += std::to_string(obj);
-    m->last_object_description += " 0";
-
     bool empty = false;
-    auto object = QPDFParser(*input, m->last_object_description, m->tokenizer, nullptr, this, true)
-                      .parse(empty, false);
+    auto object =
+        QPDFParser(input, stream_id, obj_id, m->last_object_description, m->tokenizer, this)
+            .parse(empty, false);
     if (empty) {
         // Nothing in the PDF spec appears to allow empty objects, but they have been encountered in
         // actual PDF files and Adobe Reader appears to ignore them.
-        warn(damagedPDF(*input, input->getLastOffset(), "empty object treated as null"));
+        warn(QPDFExc(
+            qpdf_e_damaged_pdf,
+            m->file->getName() + " object stream " + std::to_string(stream_id),
+            +"object " + std::to_string(obj_id) + " 0, offset " +
+                std::to_string(input.getLastOffset()),
+            0,
+            "empty object treated as null"));
     }
     return object;
 }
@@ -1605,13 +1608,23 @@ QPDF::resolve(QPDFObjGen og)
 void
 QPDF::resolveObjectsInStream(int obj_stream_number)
 {
+    auto damaged =
+        [this, obj_stream_number](int id, qpdf_offset_t offset, std::string const& msg) -> QPDFExc {
+        return {
+            qpdf_e_damaged_pdf,
+            m->file->getName() + " object stream " + std::to_string(obj_stream_number),
+            +"object " + std::to_string(id) + " 0",
+            offset,
+            msg};
+    };
+
     if (m->resolved_object_streams.count(obj_stream_number)) {
         return;
     }
     m->resolved_object_streams.insert(obj_stream_number);
     // Force resolution of object stream
-    QPDFObjectHandle obj_stream = getObjectByID(obj_stream_number, 0);
-    if (!obj_stream.isStream()) {
+    auto obj_stream = getObject(obj_stream_number, 0).as_stream();
+    if (!obj_stream) {
         throw damagedPDF(
             "object " + std::to_string(obj_stream_number) + " 0",
             "supposed object stream " + std::to_string(obj_stream_number) + " is not a stream");
@@ -1631,34 +1644,25 @@ QPDF::resolveObjectsInStream(int obj_stream_number)
             "supposed object stream " + std::to_string(obj_stream_number) + " has wrong type"));
     }
-    if (!(dict.getKey("/N").isInteger() && dict.getKey("/First").isInteger())) {
+    unsigned int n{0};
+    int first{0};
+    if (!(dict.getKey("/N").getValueAsUInt(n) && dict.getKey("/First").getValueAsInt(first))) {
         throw damagedPDF(
             "object " + std::to_string(obj_stream_number) + " 0",
             "object stream " + std::to_string(obj_stream_number) + " has incorrect keys");
     }
-    int n = dict.getKey("/N").getIntValueAsInt();
-    int first = dict.getKey("/First").getIntValueAsInt();
+    std::vector<std::pair<int, long long>> offsets;
-    std::map<int, int> offsets;
-
-    std::shared_ptr<Buffer> bp = obj_stream.getStreamData(qpdf_dl_specialized);
-    auto input = std::shared_ptr<InputSource>(
-        // line-break
-        new BufferInputSource(
-            (m->file->getName() + " object stream " + std::to_string(obj_stream_number)),
-            bp.get()));
+    auto bp = obj_stream.getStreamData(qpdf_dl_specialized);
+    BufferInputSource input("", bp.get());
     long long last_offset = -1;
-    for (int i = 0; i < n; ++i) {
-        QPDFTokenizer::Token tnum = readToken(*input);
-        QPDFTokenizer::Token toffset = readToken(*input);
+    for (unsigned int i = 0; i < n; ++i) {
+        auto tnum = readToken(input);
+        auto toffset = readToken(input);
         if (!(tnum.isInteger() && toffset.isInteger())) {
-            throw damagedPDF(
-                *input,
-                "object " + std::to_string(obj_stream_number) + " 0",
-                input->getLastOffset(),
-                "expected integer in object stream header");
+            throw damaged(0, input.getLastOffset(), "expected integer in object stream header");
         }
         int num = QUtil::string_to_int(tnum.getValue().c_str());
@@ -1666,29 +1670,20 @@ QPDF::resolveObjectsInStream(int obj_stream_number)
         if (num == obj_stream_number) {
             QTC::TC("qpdf", "QPDF ignore self-referential object stream");
-            warn(damagedPDF(
-                *input,
-                "object " + std::to_string(obj_stream_number) + " 0",
-                input->getLastOffset(),
-                "object stream claims to contain itself"));
+            warn(damaged(num, input.getLastOffset(), "object stream claims to contain itself"));
             continue;
         }
         if (num < 1) {
             QTC::TC("qpdf", "QPDF object stream contains id < 1");
-            warn(damagedPDF(
-                *input,
-                "object " + std::to_string(num) + " 0",
-                input->getLastOffset(),
-                "object id is invalid"s));
+            warn(damaged(num, input.getLastOffset(), "object id is invalid"s));
             continue;
         }
         if (offset <= last_offset) {
             QTC::TC("qpdf", "QPDF object stream offsets not increasing");
-            warn(damagedPDF(
-                *input,
-                "object " + std::to_string(num) + " 0",
+            warn(damaged(
+                num,
                 offset,
                 "offset is invalid (must be larger than previous offset " +
                     std::to_string(last_offset) + ")"));
@@ -1700,23 +1695,20 @@ QPDF::resolveObjectsInStream(int obj_stream_number)
             continue;
         }
-        offsets[num] = toI(offset + first);
+        offsets.emplace_back(num, offset + first);
     }
     // To avoid having to read the object stream multiple times, store all objects that would be
     // found here in the cache.  Remember that some objects stored here might have been overridden
     // by new objects appended to the file, so it is necessary to recheck the xref table and only
     // cache what would actually be resolved here.
-    m->last_object_description.clear();
-    m->last_object_description += "object ";
-    for (auto const& iter: offsets) {
-        QPDFObjGen og(iter.first, 0);
+    for (auto const& [id, offset]: offsets) {
+        QPDFObjGen og(id, 0);
         auto entry = m->xref_table.find(og);
         if (entry != m->xref_table.end() && entry->second.getType() == 2 &&
             entry->second.getObjStreamNumber() == obj_stream_number) {
-            int offset = iter.second;
-            input->seek(offset, SEEK_SET);
-            QPDFObjectHandle oh = readObjectInStream(input, iter.first);
+            input.seek(offset, SEEK_SET);
+            QPDFObjectHandle oh = readObjectInStream(input, obj_stream_number, id);
             updateCache(og, oh.getObj(), end_before_space, end_after_space);
         } else {
             QTC::TC("qpdf", "QPDF not caching overridden objstm object");
@@ -4,6 +4,7 @@
 #include <qpdf/QPDFObjectHandle.hh>
 #include <qpdf/QPDFObject_private.hh>
+#include <qpdf/QPDF_private.hh>
 #include <qpdf/QUtil.hh>
 namespace qpdf
@@ -428,6 +429,18 @@ QPDFObject::create(Args&amp;&amp;... args)
     return std::make_shared<QPDFObject>(std::forward<T>(T(std::forward<Args>(args)...)));
 }
+inline qpdf_object_type_e
+QPDFObject::getResolvedTypeCode() const
+{
+    if (getTypeCode() == ::ot_unresolved) {
+        return QPDF::Resolver::resolved(qpdf, og)->getTypeCode();
+    }
+    if (getTypeCode() == ::ot_reference) {
+        return std::get<QPDF_Reference>(value).obj->getTypeCode();
+    }
+    return getTypeCode();
+}
+
 inline qpdf::Array
 QPDFObjectHandle::as_array(qpdf::typed options) const
 {
@@ -7,8 +7,8 @@
 #include <qpdf/Constants.h>
 #include <qpdf/JSON.hh>
 #include <qpdf/JSON_writer.hh>
+#include <qpdf/QPDF.hh>
 #include <qpdf/QPDFObjGen.hh>
-#include <qpdf/QPDF_private.hh>
 #include <qpdf/Types.h>
 #include <map>
@@ -301,17 +301,8 @@ class QPDFObject
     std::string getStringValue() const;
     // Return a unique type code for the resolved object
-    qpdf_object_type_e
-    getResolvedTypeCode() const
-    {
-        if (getTypeCode() == ::ot_unresolved) {
-            return QPDF::Resolver::resolved(qpdf, og)->getTypeCode();
-        }
-        if (getTypeCode() == ::ot_reference) {
-            return std::get<QPDF_Reference>(value).obj->getTypeCode();
-        }
-        return getTypeCode();
-    }
+    inline qpdf_object_type_e getResolvedTypeCode() const;
+
     // Return a unique type code for the object
     qpdf_object_type_e
     getTypeCode() const
@@ -390,7 +381,17 @@ class QPDFObject
         std::string var_descr;
     };
-    using Description = std::variant<std::string, JSON_Descr, ChildDescr>;
+    struct ObjStreamDescr
+    {
+        ObjStreamDescr(int stream_id, int obj_id) :
+            stream_id(stream_id),
+            obj_id(obj_id) {};
+
+        int stream_id;
+        int obj_id;
+    };
+
+    using Description = std::variant<std::string, JSON_Descr, ChildDescr, ObjStreamDescr>;
     void
     setDescription(
@@ -62,9 +62,32 @@ class QPDFParser
         decrypter(nullptr),
         context(context),
         description(std::move(sp_description)),
-        parse_pdf(false)
+        parse_pdf(true)
     {
     }
+
+    // Used by readObjectInStream only
+    QPDFParser(
+        InputSource& input,
+        int stream_id,
+        int obj_id,
+        std::string const& object_description,
+        qpdf::Tokenizer& tokenizer,
+        QPDF* context) :
+        input(input),
+        object_description(object_description),
+        tokenizer(tokenizer),
+        decrypter(nullptr),
+        context(context),
+        description(
+            std::make_shared<QPDFObject::Description>(
+                QPDFObject::ObjStreamDescr(stream_id, obj_id))),
+        parse_pdf(true),
+        stream_id(stream_id),
+        obj_id(obj_id)
+    {
+    }
+
     ~QPDFParser() = default;
     QPDFObjectHandle parse(bool& empty, bool content_stream);
@@ -124,6 +147,8 @@ class QPDFParser
     QPDF* context;
     std::shared_ptr<QPDFObject::Description> description;
     bool parse_pdf;
+    int stream_id{0};
+    int obj_id{0};
     std::vector<StackFrame> stack;
     StackFrame* frame{nullptr};
@@ -3,6 +3,7 @@
 #include <qpdf/QPDF.hh>
+#include <qpdf/QPDFObject_private.hh>
 #include <qpdf/QPDFTokenizer_private.hh>
 // Writer class is restricted to QPDFWriter so that only it can call certain methods.
@@ -457,6 +458,7 @@ class QPDF::Members
     qpdf::Tokenizer tokenizer;
     std::shared_ptr<InputSource> file;
     std::string last_object_description;
+    std::shared_ptr<QPDFObject::Description> last_ostream_description;
     bool provided_password_is_hex_key{false};
     bool ignore_xref_streams{false};
     bool suppress_warnings{false};
@@ -29,6 +29,12 @@ more detail.
     - There have been further enhancements to how files with damaged xref
       tables are recovered.
+  - Other changes
+
+    - The parsing of object streams including the creation of error/warning
+      messages and object descriptions has been refactored with some
+      improvement both in runtime and memory usage.
+
     - There has been some refactoring of how object streams are written with
       some performance improvement.