diff --git a/include/qpdf/QPDF.hh b/include/qpdf/QPDF.hh index 11bc807..5dc3f67 100644 --- a/include/qpdf/QPDF.hh +++ b/include/qpdf/QPDF.hh @@ -725,165 +725,15 @@ class QPDF void removePage(QPDFObjectHandle page); // End legacy page helpers - // Writer class is restricted to QPDFWriter so that only it can call certain methods. - class Writer - { - friend class QPDFWriter; - - private: - static void - optimize( - QPDF& qpdf, - QPDFWriter::ObjTable const& obj, - std::function skip_stream_parameters) - { - return qpdf.optimize(obj, skip_stream_parameters); - } - - static void - getLinearizedParts( - QPDF& qpdf, - QPDFWriter::ObjTable const& obj, - std::vector& part4, - std::vector& part6, - std::vector& part7, - std::vector& part8, - std::vector& part9) - { - qpdf.getLinearizedParts(obj, part4, part6, part7, part8, part9); - } - - static void - generateHintStream( - QPDF& qpdf, - QPDFWriter::NewObjTable const& new_obj, - QPDFWriter::ObjTable const& obj, - std::shared_ptr& hint_stream, - int& S, - int& O, - bool compressed) - { - return qpdf.generateHintStream(new_obj, obj, hint_stream, S, O, compressed); - } - - static std::vector - getCompressibleObjGens(QPDF& qpdf) - { - return qpdf.getCompressibleObjVector(); - } - - static std::vector - getCompressibleObjSet(QPDF& qpdf) - { - return qpdf.getCompressibleObjSet(); - } - - static std::map const& - getXRefTable(QPDF& qpdf) - { - return qpdf.getXRefTableInternal(); - } - - static size_t - tableSize(QPDF& qpdf) - { - return qpdf.tableSize(); - } - }; - - // The Resolver class is restricted to QPDFObject so that only it can resolve indirect - // references. - class Resolver - { - friend class QPDFObject; - friend class QPDF_Unresolved; - - private: - static QPDFObject* - resolved(QPDF* qpdf, QPDFObjGen og) - { - return qpdf->resolve(og); - } - }; - - // StreamCopier class is restricted to QPDFObjectHandle so it can copy stream data. - class StreamCopier - { - friend class QPDFObjectHandle; - - private: - static void - copyStreamData(QPDF* qpdf, QPDFObjectHandle const& dest, QPDFObjectHandle const& src) - { - qpdf->copyStreamData(dest, src); - } - }; - - // The ParseGuard class allows QPDFParser to detect re-entrant parsing. It also provides - // special access to allow the parser to create unresolved objects and dangling references. - class ParseGuard - { - friend class QPDFParser; - - private: - ParseGuard(QPDF* qpdf) : - qpdf(qpdf) - { - if (qpdf) { - qpdf->inParse(true); - } - } - - static std::shared_ptr - getObject(QPDF* qpdf, int id, int gen, bool parse_pdf) - { - return qpdf->getObjectForParser(id, gen, parse_pdf); - } + // End of the public API. The following classes and methods are for qpdf internal use only. - ~ParseGuard() - { - if (qpdf) { - qpdf->inParse(false); - } - } - QPDF* qpdf; - }; - - // Pipe class is restricted to QPDF_Stream. - class Pipe - { - friend class QPDF_Stream; - - private: - static bool - pipeStreamData( - QPDF* qpdf, - QPDFObjGen const& og, - qpdf_offset_t offset, - size_t length, - QPDFObjectHandle dict, - Pipeline* pipeline, - bool suppress_warnings, - bool will_retry) - { - return qpdf->pipeStreamData( - og, offset, length, dict, pipeline, suppress_warnings, will_retry); - } - }; - - // JobSetter class is restricted to QPDFJob. - class JobSetter - { - friend class QPDFJob; - - private: - // Enable enhanced warnings for pdf file checking. - static void - setCheckMode(QPDF& qpdf, bool val) - { - qpdf.m->check_mode = val; - } - }; + class Writer; + class Resolver; + class StreamCopier; + class ParseGuard; + class Pipe; + class JobSetter; + class Xref_table; // For testing only -- do not add to DLL static bool test_json_validators(); @@ -898,163 +748,18 @@ class QPDF static std::string const qpdf_version; - class ObjCache - { - public: - ObjCache() : - end_before_space(0), - end_after_space(0) - { - } - ObjCache( - std::shared_ptr object, - qpdf_offset_t end_before_space = 0, - qpdf_offset_t end_after_space = 0) : - object(object), - end_before_space(end_before_space), - end_after_space(end_after_space) - { - } - - std::shared_ptr object; - qpdf_offset_t end_before_space; - qpdf_offset_t end_after_space; - }; - - class ObjCopier - { - public: - std::map object_map; - std::vector to_copy; - QPDFObjGen::set visiting; - }; - - class EncryptionParameters - { - friend class QPDF; - - public: - EncryptionParameters(); - - private: - bool encrypted; - bool encryption_initialized; - int encryption_V; - int encryption_R; - bool encrypt_metadata; - std::map crypt_filters; - encryption_method_e cf_stream; - encryption_method_e cf_string; - encryption_method_e cf_file; - std::string provided_password; - std::string user_password; - std::string encryption_key; - std::string cached_object_encryption_key; - QPDFObjGen cached_key_og; - bool user_password_matched; - bool owner_password_matched; - }; - - class ForeignStreamData - { - friend class QPDF; - - public: - ForeignStreamData( - std::shared_ptr encp, - std::shared_ptr file, - QPDFObjGen const& foreign_og, - qpdf_offset_t offset, - size_t length, - QPDFObjectHandle local_dict); - - private: - std::shared_ptr encp; - std::shared_ptr file; - QPDFObjGen foreign_og; - qpdf_offset_t offset; - size_t length; - QPDFObjectHandle local_dict; - }; - - class CopiedStreamDataProvider: public QPDFObjectHandle::StreamDataProvider - { - public: - CopiedStreamDataProvider(QPDF& destination_qpdf); - ~CopiedStreamDataProvider() override = default; - bool provideStreamData( - QPDFObjGen const& og, - Pipeline* pipeline, - bool suppress_warnings, - bool will_retry) override; - void registerForeignStream(QPDFObjGen const& local_og, QPDFObjectHandle foreign_stream); - void registerForeignStream(QPDFObjGen const& local_og, std::shared_ptr); - - private: - QPDF& destination_qpdf; - std::map foreign_streams; - std::map> foreign_stream_data; - }; - - class StringDecrypter: public QPDFObjectHandle::StringDecrypter - { - friend class QPDF; - - public: - StringDecrypter(QPDF* qpdf, QPDFObjGen const& og); - ~StringDecrypter() override = default; - void decryptString(std::string& val) override; - - private: - QPDF* qpdf; - QPDFObjGen og; - }; - - class ResolveRecorder - { - public: - ResolveRecorder(QPDF* qpdf, QPDFObjGen const& og) : - qpdf(qpdf), - iter(qpdf->m->resolving.insert(og).first) - { - } - virtual ~ResolveRecorder() - { - this->qpdf->m->resolving.erase(iter); - } - - private: - QPDF* qpdf; - std::set::const_iterator iter; - }; - + class ObjCache; + class ObjCopier; + class EncryptionParameters; + class ForeignStreamData; + class CopiedStreamDataProvider; + class StringDecrypter; + class ResolveRecorder; class JSONReactor; void parse(char const* password); void inParse(bool); - void setTrailer(QPDFObjectHandle obj); - void read_xref(qpdf_offset_t offset); - bool resolveXRefTable(); - void reconstruct_xref(QPDFExc& e); - bool parse_xrefFirst(std::string const& line, int& obj, int& num, int& bytes); - bool read_xrefEntry(qpdf_offset_t& f1, int& f2, char& type); - bool read_bad_xrefEntry(qpdf_offset_t& f1, int& f2, char& type); - qpdf_offset_t read_xrefTable(qpdf_offset_t offset); - qpdf_offset_t read_xrefStream(qpdf_offset_t offset); - qpdf_offset_t processXRefStream(qpdf_offset_t offset, QPDFObjectHandle& xref_stream); - std::pair> - processXRefW(QPDFObjectHandle& dict, std::function damaged); - int processXRefSize( - QPDFObjectHandle& dict, int entry_size, std::function damaged); - std::pair>> processXRefIndex( - QPDFObjectHandle& dict, - int max_num_entries, - std::function damaged); - void insertXrefEntry(int obj, int f0, qpdf_offset_t f1, int f2); - void insertFreeXrefEntry(QPDFObjGen); - void insertReconstructedXrefEntry(int obj, qpdf_offset_t f1, int f2); void setLastObjectDescription(std::string const& description, QPDFObjGen const& og); - QPDFObjectHandle readTrailer(); QPDFObjectHandle readObject(std::string const& description, QPDFObjGen og); void readStream(QPDFObjectHandle& object, QPDFObjGen og, qpdf_offset_t offset); void validateStreamLineEnd(QPDFObjectHandle& object, QPDFObjGen og, qpdf_offset_t offset); @@ -1081,11 +786,7 @@ class QPDF std::shared_ptr getObjectForParser(int id, int gen, bool parse_pdf); std::shared_ptr getObjectForJSON(int id, int gen); void removeObject(QPDFObjGen og); - void updateCache( - QPDFObjGen const& og, - std::shared_ptr const& object, - qpdf_offset_t end_before_space, - qpdf_offset_t end_after_space); + void updateCache(QPDFObjGen const& og, std::shared_ptr const& object); static QPDFExc damagedPDF( InputSource& input, std::string const& object, @@ -1122,7 +823,6 @@ class QPDF // For QPDFWriter: - std::map const& getXRefTableInternal(); template void optimize_internal( T const& object_stream_data, @@ -1131,6 +831,7 @@ class QPDF void optimize( QPDFWriter::ObjTable const& obj, std::function skip_stream_parameters); + void optimize(Xref_table const& obj); size_t tableSize(); // Get lists of all objects in order according to the part of a linearized file that they belong @@ -1196,200 +897,19 @@ class QPDF replaceForeignIndirectObjects(QPDFObjectHandle foreign, ObjCopier& obj_copier, bool top); void copyStreamData(QPDFObjectHandle dest_stream, QPDFObjectHandle src_stream); - // Linearization Hint table structures. - // Naming conventions: - - // HSomething is the Something Hint Table or table header - // HSomethingEntry is an entry in the Something table - - // delta_something + min_something = something - // nbits_something = number of bits required for something - - // something_offset is the pre-adjusted offset in the file. If >= - // H0_offset, H0_length must be added to get an actual file - // offset. - - // PDF 1.4: Table F.4 - struct HPageOffsetEntry - { - int delta_nobjects{0}; // 1 - qpdf_offset_t delta_page_length{0}; // 2 - // vectors' sizes = nshared_objects - int nshared_objects{0}; // 3 - std::vector shared_identifiers; // 4 - std::vector shared_numerators; // 5 - qpdf_offset_t delta_content_offset{0}; // 6 - qpdf_offset_t delta_content_length{0}; // 7 - }; - - // PDF 1.4: Table F.3 - struct HPageOffset - { - int min_nobjects{0}; // 1 - qpdf_offset_t first_page_offset{0}; // 2 - int nbits_delta_nobjects{0}; // 3 - int min_page_length{0}; // 4 - int nbits_delta_page_length{0}; // 5 - int min_content_offset{0}; // 6 - int nbits_delta_content_offset{0}; // 7 - int min_content_length{0}; // 8 - int nbits_delta_content_length{0}; // 9 - int nbits_nshared_objects{0}; // 10 - int nbits_shared_identifier{0}; // 11 - int nbits_shared_numerator{0}; // 12 - int shared_denominator{0}; // 13 - // vector size is npages - std::vector entries; - }; - - // PDF 1.4: Table F.6 - struct HSharedObjectEntry - { - // Item 3 is a 128-bit signature (unsupported by Acrobat) - int delta_group_length{0}; // 1 - int signature_present{0}; // 2 -- always 0 - int nobjects_minus_one{0}; // 4 -- always 0 - }; - - // PDF 1.4: Table F.5 - struct HSharedObject - { - int first_shared_obj{0}; // 1 - qpdf_offset_t first_shared_offset{0}; // 2 - int nshared_first_page{0}; // 3 - int nshared_total{0}; // 4 - int nbits_nobjects{0}; // 5 - int min_group_length{0}; // 6 - int nbits_delta_group_length{0}; // 7 - // vector size is nshared_total - std::vector entries; - }; - - // PDF 1.4: Table F.9 - struct HGeneric - { - int first_object{0}; // 1 - qpdf_offset_t first_object_offset{0}; // 2 - int nobjects{0}; // 3 - int group_length{0}; // 4 - }; - - // Other linearization data structures - - // Initialized from Linearization Parameter dictionary - struct LinParameters - { - qpdf_offset_t file_size{0}; // /L - int first_page_object{0}; // /O - qpdf_offset_t first_page_end{0}; // /E - int npages{0}; // /N - qpdf_offset_t xref_zero_offset{0}; // /T - int first_page{0}; // /P - qpdf_offset_t H_offset{0}; // offset of primary hint stream - qpdf_offset_t H_length{0}; // length of primary hint stream - }; - - // Computed hint table value data structures. These tables contain the computed values on which - // the hint table values are based. They exclude things like number of bits and store actual - // values instead of mins and deltas. File offsets are also absolute rather than being offset - // by the size of the primary hint table. We populate the hint table structures from these - // during writing and compare the hint table values with these during validation. We ignore - // some values for various reasons described in the code. Those values are omitted from these - // structures. Note also that object numbers are object numbers from the input file, not the - // output file. - - // Naming convention: CHSomething is analogous to HSomething above. "CH" is computed hint. - - struct CHPageOffsetEntry - { - int nobjects{0}; - int nshared_objects{0}; - // vectors' sizes = nshared_objects - std::vector shared_identifiers; - }; - - struct CHPageOffset - { - // vector size is npages - std::vector entries; - }; - - struct CHSharedObjectEntry - { - CHSharedObjectEntry(int object) : - object(object) - { - } - - int object; - }; - - // PDF 1.4: Table F.5 - struct CHSharedObject - { - int first_shared_obj{0}; - int nshared_first_page{0}; - int nshared_total{0}; - // vector size is nshared_total - std::vector entries; - }; - - // No need for CHGeneric -- HGeneric is fine as is. - - // Data structures to support optimization -- implemented in QPDF_optimization.cc - - class ObjUser - { - public: - enum user_e { ou_bad, ou_page, ou_thumb, ou_trailer_key, ou_root_key, ou_root }; - - // type is set to ou_bad - ObjUser(); - - // type must be ou_root - ObjUser(user_e type); - - // type must be one of ou_page or ou_thumb - ObjUser(user_e type, int pageno); - - // type must be one of ou_trailer_key or ou_root_key - ObjUser(user_e type, std::string const& key); - - bool operator<(ObjUser const&) const; - - user_e ou_type; - int pageno; // if ou_page; - std::string key; // if ou_trailer_key or ou_root_key - }; - - struct UpdateObjectMapsFrame - { - UpdateObjectMapsFrame(ObjUser const& ou, QPDFObjectHandle oh, bool top); - - ObjUser const& ou; - QPDFObjectHandle oh; - bool top; - }; - - class PatternFinder: public InputSource::Finder - { - public: - PatternFinder(QPDF& qpdf, bool (QPDF::*checker)()) : - qpdf(qpdf), - checker(checker) - { - } - ~PatternFinder() override = default; - bool - check() override - { - return (this->qpdf.*checker)(); - } - - private: - QPDF& qpdf; - bool (QPDF::*checker)(); - }; + struct HPageOffsetEntry; + struct HPageOffset; + struct HSharedObjectEntry; + struct HSharedObject; + struct HGeneric; + struct LinParameters; + struct CHPageOffsetEntry; + struct CHPageOffset; + struct CHSharedObjectEntry; + struct CHSharedObject; + class ObjUser; + struct UpdateObjectMapsFrame; + class PatternFinder; // Methods to support pattern finding static bool validatePDFVersion(char const*&, std::string& version); @@ -1411,6 +931,7 @@ class QPDF QPDFObjectHandle getUncompressedObject(QPDFObjectHandle&, std::map const& object_stream_data); QPDFObjectHandle getUncompressedObject(QPDFObjectHandle&, QPDFWriter::ObjTable const& obj); + QPDFObjectHandle getUncompressedObject(QPDFObjectHandle&, Xref_table const& obj); int lengthNextN(int first_object, int n); void checkHPageOffset(std::vector const& pages, std::map& idx_to_obj); @@ -1456,6 +977,7 @@ class QPDF std::function skip_stream_parameters); void filterCompressedObjects(std::map const& object_stream_data); void filterCompressedObjects(QPDFWriter::ObjTable const& object_stream_data); + void filterCompressedObjects(Xref_table const& object_stream_data); // JSON import void importJSON(std::shared_ptr, bool must_be_complete); @@ -1486,90 +1008,7 @@ class QPDF return QIntC::to_ulonglong(i); } - class Members - { - friend class QPDF; - friend class ResolveRecorder; - - public: - QPDF_DLL - ~Members() = default; - - private: - Members(); - Members(Members const&) = delete; - - std::shared_ptr log; - unsigned long long unique_id{0}; - QPDFTokenizer tokenizer; - std::shared_ptr file; - std::string last_object_description; - bool provided_password_is_hex_key{false}; - bool ignore_xref_streams{false}; - bool suppress_warnings{false}; - size_t max_warnings{0}; - bool attempt_recovery{true}; - bool check_mode{false}; - std::shared_ptr encp; - std::string pdf_version; - std::map xref_table; - // Various tables are indexed by object id, with potential size id + 1 - int xref_table_max_id{std::numeric_limits::max() - 1}; - qpdf_offset_t xref_table_max_offset{0}; - std::set deleted_objects; - std::map obj_cache; - std::set resolving; - QPDFObjectHandle trailer; - std::vector all_pages; - bool invalid_page_found{false}; - std::map pageobj_to_pages_pos; - bool pushed_inherited_attributes_to_pages{false}; - bool ever_pushed_inherited_attributes_to_pages{false}; - bool ever_called_get_all_pages{false}; - std::vector warnings; - std::map object_copiers; - std::shared_ptr copied_streams; - // copied_stream_data_provider is owned by copied_streams - CopiedStreamDataProvider* copied_stream_data_provider{nullptr}; - bool reconstructed_xref{false}; - bool fixed_dangling_refs{false}; - bool immediate_copy_from{false}; - bool in_parse{false}; - bool parsed{false}; - std::set resolved_object_streams; - - // Linearization data - qpdf_offset_t first_xref_item_offset{0}; // actual value from file - bool uncompressed_after_compressed{false}; - bool linearization_warnings{false}; - - // Linearization parameter dictionary and hint table data: may be read from file or computed - // prior to writing a linearized file - QPDFObjectHandle lindict; - LinParameters linp; - HPageOffset page_offset_hints; - HSharedObject shared_object_hints; - HGeneric outline_hints; - - // Computed linearization data: used to populate above tables during writing and to compare - // with them during validation. c_ means computed. - LinParameters c_linp; - CHPageOffset c_page_offset_data; - CHSharedObject c_shared_object_data; - HGeneric c_outline_data; - - // Object ordering data for linearized files: initialized by calculateLinearizationData(). - // Part numbers refer to the PDF 1.4 specification. - std::vector part4; - std::vector part6; - std::vector part7; - std::vector part8; - std::vector part9; - - // Optimization data - std::map> obj_user_to_objects; - std::map> object_to_obj_users; - }; + class Members; // Keep all member variables inside the Members object, which we dynamically allocate. This // makes it possible to add new private members without breaking binary compatibility. diff --git a/libqpdf/QPDF.cc b/libqpdf/QPDF.cc index 03ffb62..f263551 100644 --- a/libqpdf/QPDF.cc +++ b/libqpdf/QPDF.cc @@ -1,6 +1,6 @@ #include // include first for large file support -#include +#include #include #include @@ -32,67 +32,51 @@ // being static as well. std::string const QPDF::qpdf_version(QPDF_VERSION); -static char const* EMPTY_PDF = ( - // force line break - "%PDF-1.3\n" - "1 0 obj\n" - "<< /Type /Catalog /Pages 2 0 R >>\n" - "endobj\n" - "2 0 obj\n" - "<< /Type /Pages /Kids [] /Count 0 >>\n" - "endobj\n" - "xref\n" - "0 3\n" - "0000000000 65535 f \n" - "0000000009 00000 n \n" - "0000000058 00000 n \n" - "trailer << /Size 3 /Root 1 0 R >>\n" - "startxref\n" - "110\n" - "%%EOF\n"); - namespace { - class InvalidInputSource: public InputSource + class InvalidInputSource final: public InputSource { public: - ~InvalidInputSource() override = default; + InvalidInputSource(std::string const& name) : + name(name) + { + } + ~InvalidInputSource() final = default; qpdf_offset_t - findAndSkipNextEOL() override + findAndSkipNextEOL() final { throwException(); return 0; } std::string const& - getName() const override + getName() const final { - static std::string name("closed input source"); return name; } qpdf_offset_t - tell() override + tell() final { throwException(); return 0; } void - seek(qpdf_offset_t offset, int whence) override + seek(qpdf_offset_t offset, int whence) final { throwException(); } void - rewind() override + rewind() final { throwException(); } size_t - read(char* buffer, size_t length) override + read(char* buffer, size_t length) final { throwException(); return 0; } void - unreadCh(char ch) override + unreadCh(char ch) final { throwException(); } @@ -105,6 +89,8 @@ namespace "source. QPDF operations are invalid before processFile (or " "another process method) or after closeInputSource"); } + + std::string const& name; }; } // namespace @@ -196,15 +182,17 @@ QPDF::EncryptionParameters::EncryptionParameters() : { } -QPDF::Members::Members() : +QPDF::Members::Members(QPDF& qpdf) : log(QPDFLogger::defaultLogger()), - file(new InvalidInputSource()), - encp(new EncryptionParameters) + file_sp(new InvalidInputSource(no_input_name)), + file(file_sp.get()), + encp(new EncryptionParameters), + xref_table(qpdf, file) { } QPDF::QPDF() : - m(new Members()) + m(new Members(*this)) { m->tokenizer.allowEOF(); // Generate a unique ID. It just has to be unique among all QPDF objects allocated throughout @@ -225,9 +213,6 @@ QPDF::~QPDF() // are reachable from this object to release their association with this QPDF. Direct objects // are not destroyed since they can be moved to other QPDF objects safely. - // At this point, obviously no one is still using the QPDF object, but we'll explicitly clear - // the xref table anyway just to prevent any possibility of resolve() succeeding. - m->xref_table.clear(); for (auto const& iter: m->obj_cache) { iter.second.object->disconnect(); if (iter.second.object->getTypeCode() != ::ot_null) { @@ -271,14 +256,17 @@ QPDF::processMemoryFile( void QPDF::processInputSource(std::shared_ptr source, char const* password) { - m->file = source; + m->file_sp = source; + m->file = source.get(); parse(password); } void QPDF::closeInputSource() { - m->file = std::shared_ptr(new InvalidInputSource()); + m->no_input_name = "closed input source"; + m->file_sp = std::shared_ptr(new InvalidInputSource(m->no_input_name)); + m->file = m->file_sp.get(); } void @@ -290,7 +278,9 @@ QPDF::setPasswordIsHexKey(bool val) void QPDF::emptyPDF() { - processMemoryFile("empty PDF", EMPTY_PDF, strlen(EMPTY_PDF)); + m->pdf_version = "1.3"; + m->no_input_name = "empty PDF"; + m->xref_table.initialize_empty(); } void @@ -303,7 +293,7 @@ QPDF::registerStreamFilter( void QPDF::setIgnoreXRefStreams(bool val) { - m->ignore_xref_streams = val; + m->xref_table.ignore_streams(val); } std::shared_ptr @@ -341,6 +331,7 @@ void QPDF::setAttemptRecovery(bool val) { m->attempt_recovery = val; + m->xref_table.attempt_recovery(val); } void @@ -410,7 +401,9 @@ QPDF::findHeader() // PDF header, all explicit offsets in the file are such that 0 points to the beginning // of the header. QTC::TC("qpdf", "QPDF global offset"); - m->file = std::shared_ptr(new OffsetInputSource(m->file, global_offset)); + m->file_sp = + std::shared_ptr(new OffsetInputSource(m->file_sp, global_offset)); + m->file = m->file_sp.get(); } } return valid; @@ -443,46 +436,8 @@ QPDF::parse(char const* password) m->pdf_version = "1.2"; } - // PDF spec says %%EOF must be found within the last 1024 bytes of/ the file. We add an extra - // 30 characters to leave room for the startxref stuff. - m->file->seek(0, SEEK_END); - qpdf_offset_t end_offset = m->file->tell(); - m->xref_table_max_offset = end_offset; - // Sanity check on object ids. All objects must appear in xref table / stream. In all realistic - // scenarios at least 3 bytes are required. - if (m->xref_table_max_id > m->xref_table_max_offset / 3) { - m->xref_table_max_id = static_cast(m->xref_table_max_offset / 3); - } - qpdf_offset_t start_offset = (end_offset > 1054 ? end_offset - 1054 : 0); - PatternFinder sf(*this, &QPDF::findStartxref); - qpdf_offset_t xref_offset = 0; - if (m->file->findLast("startxref", start_offset, 0, sf)) { - xref_offset = QUtil::string_to_ll(readToken(*m->file).getValue().c_str()); - } - - try { - if (xref_offset == 0) { - QTC::TC("qpdf", "QPDF can't find startxref"); - throw damagedPDF("", 0, "can't find startxref"); - } - try { - read_xref(xref_offset); - } catch (QPDFExc&) { - throw; - } catch (std::exception& e) { - throw damagedPDF("", 0, std::string("error reading xref: ") + e.what()); - } - } catch (QPDFExc& e) { - if (m->attempt_recovery) { - reconstruct_xref(e); - QTC::TC("qpdf", "QPDF reconstructed xref table"); - } else { - throw; - } - } - + m->xref_table.initialize(); initializeEncryption(); - m->parsed = true; if (m->xref_table.size() > 0 && !getRoot().getKey("/Pages").isDictionary()) { // QPDFs created from JSON have an empty xref table and no root object yet. throw damagedPDF("", 0, "unable to find page tree"); @@ -524,18 +479,77 @@ QPDF::warn( } void -QPDF::setTrailer(QPDFObjectHandle obj) +QPDF::Xref_table::initialize_empty() { - if (m->trailer) { - return; + initialized_ = true; + trailer_ = QPDFObjectHandle::newDictionary(); + auto rt = qpdf.makeIndirectObject(QPDFObjectHandle::newDictionary()); + auto pgs = qpdf.makeIndirectObject(QPDFObjectHandle::newDictionary()); + pgs.replaceKey("/Type", QPDFObjectHandle::newName("/Pages")); + pgs.replaceKey("/Kids", QPDFObjectHandle::newArray()); + pgs.replaceKey("/Count", QPDFObjectHandle::newInteger(0)); + rt.replaceKey("/Type", QPDFObjectHandle::newName("/Catalog")); + rt.replaceKey("/Pages", pgs); + trailer_.replaceKey("/Root", rt); + trailer_.replaceKey("/Size", QPDFObjectHandle::newInteger(3)); +} + +void +QPDF::Xref_table::initialize_json() +{ + initialized_ = true; + table.resize(1); + trailer_ = QPDFObjectHandle::newDictionary(); + trailer_.replaceKey("/Size", QPDFObjectHandle::newInteger(1)); +} + +void +QPDF::Xref_table::initialize() +{ + // PDF spec says %%EOF must be found within the last 1024 bytes of/ the file. We add an extra + // 30 characters to leave room for the startxref stuff. + file->seek(0, SEEK_END); + qpdf_offset_t end_offset = file->tell(); + // Sanity check on object ids. All objects must appear in xref table / stream. In all realistic + // scenarios at least 3 bytes are required. + if (max_id_ > end_offset / 3) { + max_id_ = static_cast(end_offset / 3); } - m->trailer = obj; + qpdf_offset_t start_offset = (end_offset > 1054 ? end_offset - 1054 : 0); + PatternFinder sf(qpdf, &QPDF::findStartxref); + qpdf_offset_t xref_offset = 0; + if (file->findLast("startxref", start_offset, 0, sf)) { + xref_offset = QUtil::string_to_ll(read_token().getValue().c_str()); + } + + try { + if (xref_offset == 0) { + QTC::TC("qpdf", "QPDF can't find startxref"); + throw damaged_pdf("can't find startxref"); + } + try { + read(xref_offset); + } catch (QPDFExc&) { + throw; + } catch (std::exception& e) { + throw damaged_pdf(std::string("error reading xref: ") + e.what()); + } + } catch (QPDFExc& e) { + if (attempt_recovery_) { + reconstruct(e); + QTC::TC("qpdf", "QPDF reconstructed xref table"); + } else { + throw; + } + } + + initialized_ = true; } void -QPDF::reconstruct_xref(QPDFExc& e) +QPDF::Xref_table::reconstruct(QPDFExc& e) { - if (m->reconstructed_xref) { + if (reconstructed_) { // Avoid xref reconstruction infinite loops. This is getting very hard to reproduce because // qpdf is throwing many fewer exceptions while parsing. Most situations are warnings now. throw e; @@ -543,78 +557,93 @@ QPDF::reconstruct_xref(QPDFExc& e) // If recovery generates more than 1000 warnings, the file is so severely damaged that there // probably is no point trying to continue. - const auto max_warnings = m->warnings.size() + 1000U; + const auto max_warnings = qpdf.m->warnings.size() + 1000U; auto check_warnings = [this, max_warnings]() { - if (m->warnings.size() > max_warnings) { - throw damagedPDF("", 0, "too many errors while reconstructing cross-reference table"); + if (qpdf.m->warnings.size() > max_warnings) { + throw damaged_pdf("too many errors while reconstructing cross-reference table"); } }; - m->reconstructed_xref = true; + reconstructed_ = true; // We may find more objects, which may contain dangling references. - m->fixed_dangling_refs = false; + qpdf.m->fixed_dangling_refs = false; - warn(damagedPDF("", 0, "file is damaged")); - warn(e); - warn(damagedPDF("", 0, "Attempting to reconstruct cross-reference table")); + warn_damaged("file is damaged"); + qpdf.warn(e); + warn_damaged("Attempting to reconstruct cross-reference table"); // Delete all references to type 1 (uncompressed) objects - std::set to_delete; - for (auto const& iter: m->xref_table) { - if (iter.second.getType() == 1) { - to_delete.insert(iter.first); + for (auto& iter: table) { + if (iter.type() == 1) { + iter = {}; } } - for (auto const& iter: to_delete) { - m->xref_table.erase(iter); - } - m->file->seek(0, SEEK_END); - qpdf_offset_t eof = m->file->tell(); - m->file->seek(0, SEEK_SET); + std::vector> objects; + std::vector trailers; + int max_found = 0; + + file->seek(0, SEEK_END); + qpdf_offset_t eof = file->tell(); + file->seek(0, SEEK_SET); // Don't allow very long tokens here during recovery. All the interesting tokens are covered. static size_t const MAX_LEN = 10; - while (m->file->tell() < eof) { - QPDFTokenizer::Token t1 = readToken(*m->file, MAX_LEN); - qpdf_offset_t token_start = m->file->tell() - toO(t1.getValue().length()); + while (file->tell() < eof) { + QPDFTokenizer::Token t1 = read_token(MAX_LEN); + qpdf_offset_t token_start = file->tell() - toO(t1.getValue().length()); if (t1.isInteger()) { - auto pos = m->file->tell(); - QPDFTokenizer::Token t2 = readToken(*m->file, MAX_LEN); - if ((t2.isInteger()) && (readToken(*m->file, MAX_LEN).isWord("obj"))) { + auto pos = file->tell(); + QPDFTokenizer::Token t2 = read_token(MAX_LEN); + if (t2.isInteger() && read_token(MAX_LEN).isWord("obj")) { int obj = QUtil::string_to_int(t1.getValue().c_str()); int gen = QUtil::string_to_int(t2.getValue().c_str()); - if (obj <= m->xref_table_max_id) { - insertReconstructedXrefEntry(obj, token_start, gen); + if (obj <= max_id_) { + objects.emplace_back(obj, gen, token_start); + if (obj > max_found) { + max_found = obj; + } } else { - warn(damagedPDF( - "", 0, "ignoring object with impossibly large id " + std::to_string(obj))); + warn_damaged("ignoring object with impossibly large id " + std::to_string(obj)); } } - m->file->seek(pos, SEEK_SET); - } else if (!m->trailer && t1.isWord("trailer")) { - auto pos = m->file->tell(); - QPDFObjectHandle t = readTrailer(); - if (!t.isDictionary()) { - // Oh well. It was worth a try. - } else { - setTrailer(t); - } - m->file->seek(pos, SEEK_SET); + file->seek(pos, SEEK_SET); + } else if (!trailer_ && t1.isWord("trailer")) { + trailers.emplace_back(file->tell()); + } + file->findAndSkipNextEOL(); + } + + table.resize(toS(max_found) + 1); + + for (auto tr: trailers) { + file->seek(tr, SEEK_SET); + auto t = read_trailer(); + if (!t.isDictionary()) { + // Oh well. It was worth a try. + } else { + trailer_ = t; + break; } check_warnings(); - m->file->findAndSkipNextEOL(); } - m->deleted_objects.clear(); - if (!m->trailer) { + auto rend = objects.rend(); + for (auto it = objects.rbegin(); it != rend; it++) { + auto [obj, gen, token_start] = *it; + insert(obj, 1, token_start, gen); + check_warnings(); + } + + if (!trailer_) { qpdf_offset_t max_offset{0}; // If there are any xref streams, take the last one to appear. - for (auto const& iter: m->xref_table) { - auto entry = iter.second; - if (entry.getType() != 1) { + int i = -1; + for (auto const& item: table) { + ++i; + if (item.type() != 1) { continue; } - auto oh = getObjectByObjGen(iter.first); + auto oh = qpdf.getObject(i, item.gen()); try { if (!oh.isStreamOfType("/XRef")) { continue; @@ -622,44 +651,44 @@ QPDF::reconstruct_xref(QPDFExc& e) } catch (std::exception&) { continue; } - auto offset = entry.getOffset(); + auto offset = item.offset(); if (offset > max_offset) { max_offset = offset; - setTrailer(oh.getDict()); + trailer_ = oh.getDict(); } check_warnings(); } if (max_offset > 0) { try { - read_xref(max_offset); + read(max_offset); } catch (std::exception&) { - throw damagedPDF( - "", 0, "error decoding candidate xref stream while recovering damaged file"); + throw damaged_pdf( + "error decoding candidate xref stream while recovering damaged file"); } QTC::TC("qpdf", "QPDF recover xref stream"); } } - if (!m->trailer) { + if (!trailer_) { // We could check the last encountered object to see if it was an xref stream. If so, we // could try to get the trailer from there. This may make it possible to recover files with // bad startxref pointers even when they have object streams. - throw damagedPDF("", 0, "unable to find trailer dictionary while recovering damaged file"); + throw damaged_pdf("unable to find trailer dictionary while recovering damaged file"); } - if (m->xref_table.empty()) { + if (table.empty()) { // We cannot check for an empty xref table in parse because empty tables are valid when // creating QPDF objects from JSON. - throw damagedPDF("", 0, "unable to find objects while recovering damaged file"); + throw damaged_pdf("unable to find objects while recovering damaged file"); } check_warnings(); - if (!m->parsed) { - m->parsed = true; - getAllPages(); + if (!initialized_) { + initialized_ = true; + qpdf.getAllPages(); check_warnings(); - if (m->all_pages.empty()) { - m->parsed = false; - throw damagedPDF("", 0, "unable to find any pages while recovering damaged file"); + if (qpdf.m->all_pages.empty()) { + initialized_ = false; + throw damaged_pdf("unable to find any pages while recovering damaged file"); } } // We could iterate through the objects looking for streams and try to find objects inside of @@ -670,7 +699,7 @@ QPDF::reconstruct_xref(QPDFExc& e) } void -QPDF::read_xref(qpdf_offset_t xref_offset) +QPDF::Xref_table::read(qpdf_offset_t xref_offset) { std::map free_table; std::set visited; @@ -678,7 +707,7 @@ QPDF::read_xref(qpdf_offset_t xref_offset) visited.insert(xref_offset); char buf[7]; memset(buf, 0, sizeof(buf)); - m->file->seek(xref_offset, SEEK_SET); + file->seek(xref_offset, SEEK_SET); // Some files miss the mark a little with startxref. We could do a better job of searching // in the neighborhood for something that looks like either an xref table or stream, but the // simple heuristic of skipping whitespace can help with the xref table case and is harmless @@ -687,11 +716,11 @@ QPDF::read_xref(qpdf_offset_t xref_offset) bool skipped_space = false; while (!done) { char ch; - if (1 == m->file->read(&ch, 1)) { + if (1 == file->read(&ch, 1)) { if (QUtil::is_space(ch)) { skipped_space = true; } else { - m->file->unreadCh(ch); + file->unreadCh(ch); done = true; } } else { @@ -700,13 +729,13 @@ QPDF::read_xref(qpdf_offset_t xref_offset) } } - m->file->read(buf, sizeof(buf) - 1); + file->read(buf, sizeof(buf) - 1); // The PDF spec says xref must be followed by a line terminator, but files exist in the wild // where it is terminated by arbitrary whitespace. if ((strncmp(buf, "xref", 4) == 0) && QUtil::is_space(buf[4])) { if (skipped_space) { QTC::TC("qpdf", "QPDF xref skipped space"); - warn(damagedPDF("", 0, "extraneous whitespace seen before xref")); + warn_damaged("extraneous whitespace seen before xref"); } QTC::TC( "qpdf", @@ -720,54 +749,38 @@ QPDF::read_xref(qpdf_offset_t xref_offset) while (QUtil::is_space(buf[skip])) { ++skip; } - xref_offset = read_xrefTable(xref_offset + skip); + xref_offset = process_section(xref_offset + skip); } else { - xref_offset = read_xrefStream(xref_offset); + xref_offset = read_stream(xref_offset); } if (visited.count(xref_offset) != 0) { QTC::TC("qpdf", "QPDF xref loop"); - throw damagedPDF("", 0, "loop detected following xref tables"); + throw damaged_pdf("loop detected following xref tables"); } } - if (!m->trailer) { - throw damagedPDF("", 0, "unable to find trailer while reading xref"); - } - int size = m->trailer.getKey("/Size").getIntValueAsInt(); - int max_obj = 0; - if (!m->xref_table.empty()) { - max_obj = m->xref_table.rbegin()->first.getObj(); + if (!trailer_) { + throw damaged_pdf("unable to find trailer while reading xref"); } - if (!m->deleted_objects.empty()) { - max_obj = std::max(max_obj, *(m->deleted_objects.rbegin())); - } - if ((size < 1) || (size - 1 != max_obj)) { - QTC::TC("qpdf", "QPDF xref size mismatch"); - warn(damagedPDF( - "", - 0, - ("reported number of objects (" + std::to_string(size) + - ") is not one plus the highest object number (" + std::to_string(max_obj) + ")"))); - } - - // We no longer need the deleted_objects table, so go ahead and clear it out to make sure we - // never depend on its being set. - m->deleted_objects.clear(); + int size = trailer_.getKey("/Size").getIntValueAsInt(); - // Make sure we keep only the highest generation for any object. - QPDFObjGen last_og{-1, 0}; - for (auto const& item: m->xref_table) { - auto id = item.first.getObj(); - if (id == last_og.getObj() && id > 0) { - removeObject(last_og); - } - last_og = item.first; + if (size < 3) { + throw damaged_pdf("too few objects - file can't have a page tree"); } + + // We are no longer reporting what the highest id in the xref table is. I don't think it adds + // anything. If we want to report more detail, we should report the total number of missing + // entries, including missing entries before the last actual entry. } -bool -QPDF::parse_xrefFirst(std::string const& line, int& obj, int& num, int& bytes) +QPDF::Xref_table::Subsection +QPDF::Xref_table::subsection(std::string const& line) { + auto terminate = [this]() -> void { + QTC::TC("qpdf", "QPDF invalid xref"); + throw damaged_table("xref syntax invalid"); + }; + // is_space and is_digit both return false on '\0', so this will not overrun the null-terminated // buffer. char const* p = line.c_str(); @@ -779,7 +792,7 @@ QPDF::parse_xrefFirst(std::string const& line, int& obj, int& num, int& bytes) } // Require digit if (!QUtil::is_digit(*p)) { - return false; + terminate(); } // Gather digits std::string obj_str; @@ -788,7 +801,7 @@ QPDF::parse_xrefFirst(std::string const& line, int& obj, int& num, int& bytes) } // Require space if (!QUtil::is_space(*p)) { - return false; + terminate(); } // Skip spaces while (QUtil::is_space(*p)) { @@ -796,7 +809,7 @@ QPDF::parse_xrefFirst(std::string const& line, int& obj, int& num, int& bytes) } // Require digit if (!QUtil::is_digit(*p)) { - return false; + terminate(); } // Gather digits std::string num_str; @@ -807,18 +820,82 @@ QPDF::parse_xrefFirst(std::string const& line, int& obj, int& num, int& bytes) while (QUtil::is_space(*p)) { ++p; } - bytes = toI(p - start); - obj = QUtil::string_to_int(obj_str.c_str()); - num = QUtil::string_to_int(num_str.c_str()); - return true; + auto obj = QUtil::string_to_int(obj_str.c_str()); + auto count = QUtil::string_to_int(num_str.c_str()); + if (obj > max_id() || count > max_id() || (obj + count) > max_id()) { + throw damaged_table("xref table subsection header contains impossibly large entry"); + } + return {obj, count, file->getLastOffset() + toI(p - start)}; +} + +std::vector +QPDF::Xref_table::bad_subsections(std::string& line, qpdf_offset_t start) +{ + std::vector result; + qpdf_offset_t f1 = 0; + int f2 = 0; + char type = '\0'; + + file->seek(start, SEEK_SET); + + while (true) { + line.assign(50, '\0'); + file->read(line.data(), line.size()); + auto [obj, num, offset] = result.emplace_back(subsection(line)); + file->seek(offset, SEEK_SET); + for (qpdf_offset_t i = obj; i - num < obj; ++i) { + if (!read_entry(f1, f2, type)) { + QTC::TC("qpdf", "QPDF invalid xref entry"); + throw damaged_table("invalid xref entry (obj=" + std::to_string(i) + ")"); + } + } + qpdf_offset_t pos = file->tell(); + if (read_token().isWord("trailer")) { + return result; + } else { + file->seek(pos, SEEK_SET); + } + } +} + +// Optimistically read and parse all subsection headers. If an error is encountered return the +// result of bad_subsections. +std::vector +QPDF::Xref_table::subsections(std::string& line) +{ + auto recovery_offset = file->tell(); + try { + std::vector result; + + while (true) { + line.assign(50, '\0'); + file->read(line.data(), line.size()); + auto& sub = result.emplace_back(subsection(line)); + auto count = std::get<1>(sub); + auto offset = std::get<2>(sub); + file->seek(offset + 20 * toO(count) - 1, SEEK_SET); + file->read(line.data(), 1); + if (!(line[0] == '\n' || line[0] == '\n')) { + return bad_subsections(line, recovery_offset); + } + qpdf_offset_t pos = file->tell(); + if (read_token().isWord("trailer")) { + return result; + } else { + file->seek(pos, SEEK_SET); + } + } + } catch (...) { + return bad_subsections(line, recovery_offset); + } } bool -QPDF::read_bad_xrefEntry(qpdf_offset_t& f1, int& f2, char& type) +QPDF::Xref_table::read_bad_entry(qpdf_offset_t& f1, int& f2, char& type) { // Reposition after initial read attempt and reread. - m->file->seek(m->file->getLastOffset(), SEEK_SET); - auto line = m->file->readLine(30); + file->seek(file->getLastOffset(), SEEK_SET); + auto line = file->readLine(30); // is_space and is_digit both return false on '\0', so this will not overrun the null-terminated // buffer. @@ -884,7 +961,7 @@ QPDF::read_bad_xrefEntry(qpdf_offset_t& f1, int& f2, char& type) } if (invalid) { - warn(damagedPDF("xref table", "accepting invalid xref table entry")); + qpdf.warn(damaged_table("accepting invalid xref table entry")); } f1 = QUtil::string_to_ll(f1_str.c_str()); @@ -896,10 +973,10 @@ QPDF::read_bad_xrefEntry(qpdf_offset_t& f1, int& f2, char& type) // Optimistically read and parse xref entry. If entry is bad, call read_bad_xrefEntry and return // result. bool -QPDF::read_xrefEntry(qpdf_offset_t& f1, int& f2, char& type) +QPDF::Xref_table::read_entry(qpdf_offset_t& f1, int& f2, char& type) { std::array line; - if (m->file->read(line.data(), 20) != 20) { + if (file->read(line.data(), 20) != 20) { // C++20: [[unlikely]] return false; } @@ -945,84 +1022,78 @@ QPDF::read_xrefEntry(qpdf_offset_t& f1, int& f2, char& type) return true; } } - return read_bad_xrefEntry(f1, f2, type); + return read_bad_entry(f1, f2, type); } // Read a single cross-reference table section and associated trailer. qpdf_offset_t -QPDF::read_xrefTable(qpdf_offset_t xref_offset) +QPDF::Xref_table::process_section(qpdf_offset_t xref_offset) { - m->file->seek(xref_offset, SEEK_SET); + file->seek(xref_offset, SEEK_SET); std::string line; - while (true) { - line.assign(50, '\0'); - m->file->read(line.data(), line.size()); - int obj = 0; - int num = 0; - int bytes = 0; - if (!parse_xrefFirst(line, obj, num, bytes)) { - QTC::TC("qpdf", "QPDF invalid xref"); - throw damagedPDF("xref table", "xref syntax invalid"); - } - m->file->seek(m->file->getLastOffset() + bytes, SEEK_SET); + auto subs = subsections(line); + + auto cur_trailer_offset = file->tell(); + auto cur_trailer = read_trailer(); + if (!cur_trailer.isDictionary()) { + QTC::TC("qpdf", "QPDF missing trailer"); + throw qpdf.damagedPDF("", "expected trailer dictionary"); + } + + if (!trailer_) { + unsigned int sz; + trailer_ = cur_trailer; + + if (!trailer_.hasKey("/Size")) { + QTC::TC("qpdf", "QPDF trailer lacks size"); + throw qpdf.damagedPDF("trailer", "trailer dictionary lacks /Size key"); + } + if (!trailer_.getKey("/Size").getValueAsUInt(sz)) { + QTC::TC("qpdf", "QPDF trailer size not integer"); + throw qpdf.damagedPDF("trailer", "/Size key in trailer dictionary is not an integer"); + } + + table.resize(sz); + } + + for (auto [obj, num, offset]: subs) { + file->seek(offset, SEEK_SET); for (qpdf_offset_t i = obj; i - num < obj; ++i) { if (i == 0) { // This is needed by checkLinearization() - m->first_xref_item_offset = m->file->tell(); + first_item_offset_ = file->tell(); } // For xref_table, these will always be small enough to be ints qpdf_offset_t f1 = 0; int f2 = 0; char type = '\0'; - if (!read_xrefEntry(f1, f2, type)) { - QTC::TC("qpdf", "QPDF invalid xref entry"); - throw damagedPDF( - "xref table", "invalid xref entry (obj=" + std::to_string(i) + ")"); + if (!read_entry(f1, f2, type)) { + throw damaged_table("invalid xref entry (obj=" + std::to_string(i) + ")"); } if (type == 'f') { - insertFreeXrefEntry(QPDFObjGen(toI(i), f2)); + insert_free(QPDFObjGen(toI(i), f2)); } else { - insertXrefEntry(toI(i), 1, f1, f2); + insert(toI(i), 1, f1, f2); } } - qpdf_offset_t pos = m->file->tell(); - if (readToken(*m->file).isWord("trailer")) { + qpdf_offset_t pos = file->tell(); + if (read_token().isWord("trailer")) { break; } else { - m->file->seek(pos, SEEK_SET); - } - } - - // Set offset to previous xref table if any - QPDFObjectHandle cur_trailer = readTrailer(); - if (!cur_trailer.isDictionary()) { - QTC::TC("qpdf", "QPDF missing trailer"); - throw damagedPDF("", "expected trailer dictionary"); - } - - if (!m->trailer) { - setTrailer(cur_trailer); - - if (!m->trailer.hasKey("/Size")) { - QTC::TC("qpdf", "QPDF trailer lacks size"); - throw damagedPDF("trailer", "trailer dictionary lacks /Size key"); - } - if (!m->trailer.getKey("/Size").isInteger()) { - QTC::TC("qpdf", "QPDF trailer size not integer"); - throw damagedPDF("trailer", "/Size key in trailer dictionary is not an integer"); + file->seek(pos, SEEK_SET); } } if (cur_trailer.hasKey("/XRefStm")) { - if (m->ignore_xref_streams) { + if (ignore_streams_) { QTC::TC("qpdf", "QPDF ignoring XRefStm in trailer"); } else { if (cur_trailer.getKey("/XRefStm").isInteger()) { // Read the xref stream but disregard any return value -- we'll use our trailer's // /Prev key instead of the xref stream's. - (void)read_xrefStream(cur_trailer.getKey("/XRefStm").getIntValue()); + (void)read_stream(cur_trailer.getKey("/XRefStm").getIntValue()); } else { - throw damagedPDF("xref stream", xref_offset, "invalid /XRefStm"); + throw qpdf.damagedPDF("xref stream", cur_trailer_offset, "invalid /XRefStm"); } } } @@ -1030,7 +1101,8 @@ QPDF::read_xrefTable(qpdf_offset_t xref_offset) if (cur_trailer.hasKey("/Prev")) { if (!cur_trailer.getKey("/Prev").isInteger()) { QTC::TC("qpdf", "QPDF trailer prev not integer"); - throw damagedPDF("trailer", "/Prev key in trailer dictionary is not an integer"); + throw qpdf.damagedPDF( + "trailer", cur_trailer_offset, "/Prev key in trailer dictionary is not an integer"); } QTC::TC("qpdf", "QPDF prev key in trailer dictionary"); return cur_trailer.getKey("/Prev").getIntValue(); @@ -1041,34 +1113,35 @@ QPDF::read_xrefTable(qpdf_offset_t xref_offset) // Read a single cross-reference stream. qpdf_offset_t -QPDF::read_xrefStream(qpdf_offset_t xref_offset) +QPDF::Xref_table::read_stream(qpdf_offset_t xref_offset) { - if (!m->ignore_xref_streams) { + if (!ignore_streams_) { QPDFObjGen x_og; QPDFObjectHandle xref_obj; try { - xref_obj = - readObjectAtOffset(false, xref_offset, "xref stream", QPDFObjGen(0, 0), x_og, true); + xref_obj = qpdf.readObjectAtOffset( + false, xref_offset, "xref stream", QPDFObjGen(0, 0), x_og, true); } catch (QPDFExc&) { // ignore -- report error below } if (xref_obj.isStreamOfType("/XRef")) { QTC::TC("qpdf", "QPDF found xref stream"); - return processXRefStream(xref_offset, xref_obj); + return process_stream(xref_offset, xref_obj); } } QTC::TC("qpdf", "QPDF can't find xref"); - throw damagedPDF("", xref_offset, "xref not found"); + throw qpdf.damagedPDF("", xref_offset, "xref not found"); return 0; // unreachable } // Return the entry size of the xref stream and the processed W array. std::pair> -QPDF::processXRefW(QPDFObjectHandle& dict, std::function damaged) +QPDF::Xref_table::process_W( + QPDFObjectHandle& dict, std::function damaged) { auto W_obj = dict.getKey("/W"); - if (!(W_obj.isArray() && (W_obj.getArrayNItems() >= 3) && W_obj.getArrayItem(0).isInteger() && + if (!(W_obj.isArray() && W_obj.getArrayNItems() >= 3 && W_obj.getArrayItem(0).isInteger() && W_obj.getArrayItem(1).isInteger() && W_obj.getArrayItem(2).isInteger())) { throw damaged("Cross-reference stream does not have a proper /W key"); } @@ -1093,9 +1166,10 @@ QPDF::processXRefW(QPDFObjectHandle& dict, std::function +QPDF::Xref_table::process_Size( QPDFObjectHandle& dict, int entry_size, std::function damaged) { // Number of entries is limited by the highest possible object id and stream size. @@ -1114,12 +1188,12 @@ QPDF::processXRefSize( throw damaged("Cross-reference stream has an impossibly large /Size key"); } // We are not validating that Size <= (Size key of parent xref / trailer). - return max_num_entries; + return {max_num_entries, toS(size)}; } // Return the number of entries of the xref stream and the processed Index array. std::pair>> -QPDF::processXRefIndex( +QPDF::Xref_table::process_Index( QPDFObjectHandle& dict, int max_num_entries, std::function damaged) { auto size = dict.getKey("/Size").getIntValueAsInt(); @@ -1186,17 +1260,17 @@ QPDF::processXRefIndex( } qpdf_offset_t -QPDF::processXRefStream(qpdf_offset_t xref_offset, QPDFObjectHandle& xref_obj) +QPDF::Xref_table::process_stream(qpdf_offset_t xref_offset, QPDFObjectHandle& xref_obj) { auto damaged = [this, xref_offset](std::string_view msg) -> QPDFExc { - return damagedPDF("xref stream", xref_offset, msg.data()); + return qpdf.damagedPDF("xref stream", xref_offset, msg.data()); }; auto dict = xref_obj.getDict(); - auto [entry_size, W] = processXRefW(dict, damaged); - int max_num_entries = processXRefSize(dict, entry_size, damaged); - auto [num_entries, indx] = processXRefIndex(dict, max_num_entries, damaged); + auto [entry_size, W] = process_W(dict, damaged); + auto [max_num_entries, size] = process_Size(dict, entry_size, damaged); + auto [num_entries, indx] = process_Index(dict, max_num_entries, damaged); std::shared_ptr bp = xref_obj.getStreamData(qpdf_dl_specialized); size_t actual_size = bp->getSize(); @@ -1209,10 +1283,15 @@ QPDF::processXRefStream(qpdf_offset_t xref_offset, QPDFObjectHandle& xref_obj) if (expected_size > actual_size) { throw x; } else { - warn(x); + qpdf.warn(x); } } + if (!trailer_) { + trailer_ = dict; + table.resize(size); + } + bool saw_first_compressed_object = false; // Actual size vs. expected size check above ensures that we will not overflow any buffers here. @@ -1238,33 +1317,29 @@ QPDF::processXRefStream(qpdf_offset_t xref_offset, QPDFObjectHandle& xref_obj) // object record, in which case the generation number appears as the third field. if (saw_first_compressed_object) { if (fields[0] != 2) { - m->uncompressed_after_compressed = true; + uncompressed_after_compressed_ = true; } } else if (fields[0] == 2) { saw_first_compressed_object = true; } if (obj == 0) { // This is needed by checkLinearization() - m->first_xref_item_offset = xref_offset; + first_item_offset_ = xref_offset; } else if (fields[0] == 0) { // Ignore fields[2], which we don't care about in this case. This works around the // issue of some PDF files that put invalid values, like -1, here for deleted // objects. - insertFreeXrefEntry(QPDFObjGen(obj, 0)); + insert_free(QPDFObjGen(obj, 0)); } else { - insertXrefEntry(obj, toI(fields[0]), fields[1], toI(fields[2])); + insert(obj, toI(fields[0]), fields[1], toI(fields[2])); } ++obj; } } - if (!m->trailer) { - setTrailer(dict); - } - if (dict.hasKey("/Prev")) { if (!dict.getKey("/Prev").isInteger()) { - throw damagedPDF( + throw qpdf.damagedPDF( "xref stream", "/Prev key in xref stream dictionary is not an integer"); } QTC::TC("qpdf", "QPDF prev key in xref stream dictionary"); @@ -1275,7 +1350,7 @@ QPDF::processXRefStream(qpdf_offset_t xref_offset, QPDFObjectHandle& xref_obj) } void -QPDF::insertXrefEntry(int obj, int f0, qpdf_offset_t f1, int f2) +QPDF::Xref_table::insert(int obj, int f0, qpdf_offset_t f1, int f2) { // Populate the xref table in such a way that the first reference to an object that we see, // which is the one in the latest xref table in which it appears, is the one that gets stored. @@ -1284,23 +1359,35 @@ QPDF::insertXrefEntry(int obj, int f0, qpdf_offset_t f1, int f2) // If there is already an entry for this object and generation in the table, it means that a // later xref table has registered this object. Disregard this one. - if (obj > m->xref_table_max_id) { - // ignore impossibly large object ids or object ids > Size. + int new_gen = f0 == 2 ? 0 : f2; + + if (!(obj > 0 && static_cast(obj) < table.size() && 0 <= f2 && new_gen < 65535)) { + // We are ignoring invalid objgens. Most will arrive here from xref reconstruction. There + // is probably no point having another warning but we could count invalid items in order to + // decide when to give up. + QTC::TC("qpdf", "QPDF xref overwrite invalid objgen"); return; } - if (m->deleted_objects.count(obj)) { + auto& entry = table[static_cast(obj)]; + auto old_type = entry.type(); + + if (!old_type && entry.gen() > 0) { + // At the moment we are processing the updates last to first and therefore the gen doesn't + // matter as long as it > 0 to distinguish it from an uninitialized entry. This will need + // to be revisited when we want to support incremental updates or more comprhensive + // checking. QTC::TC("qpdf", "QPDF xref deleted object"); return; } if (f0 == 2 && static_cast(f1) == obj) { - warn(damagedPDF("xref stream", "self-referential object stream " + std::to_string(obj))); + qpdf.warn(qpdf.damagedPDF( + "xref stream", "self-referential object stream " + std::to_string(obj))); return; } - auto [iter, created] = m->xref_table.try_emplace(QPDFObjGen(obj, (f0 == 2 ? 0 : f2))); - if (!created) { + if (old_type && entry.gen() >= new_gen) { QTC::TC("qpdf", "QPDF xref reused object"); return; } @@ -1308,85 +1395,129 @@ QPDF::insertXrefEntry(int obj, int f0, qpdf_offset_t f1, int f2) switch (f0) { case 1: // f2 is generation - QTC::TC("qpdf", "QPDF xref gen > 0", ((f2 > 0) ? 1 : 0)); - iter->second = QPDFXRefEntry(f1); + QTC::TC("qpdf", "QPDF xref gen > 0", (f2 > 0) ? 1 : 0); + entry = {f2, Uncompressed(f1)}; break; case 2: - iter->second = QPDFXRefEntry(toI(f1), f2); + entry = {0, Compressed(toI(f1), f2)}; + object_streams_ = true; break; default: - throw damagedPDF("xref stream", "unknown xref stream entry type " + std::to_string(f0)); + throw qpdf.damagedPDF( + "xref stream", "unknown xref stream entry type " + std::to_string(f0)); break; } } void -QPDF::insertFreeXrefEntry(QPDFObjGen og) +QPDF::Xref_table::insert_free(QPDFObjGen og) { - if (!m->xref_table.count(og)) { - m->deleted_objects.insert(og.getObj()); + // At the moment we are processing the updates last to first and therefore the gen doesn't + // matter as long as it > 0 to distinguish it from an uninitialized entry. This will need to be + // revisited when we want to support incremental updates or more comprhensive checking. + if (og.getObj() < 1) { + return; + } + size_t id = static_cast(og.getObj()); + if (id < table.size() && !type(id)) { + table[id] = {1, {}}; } } -// Replace uncompressed object. This is used in xref recovery mode, which reads the file from -// beginning to end. -void -QPDF::insertReconstructedXrefEntry(int obj, qpdf_offset_t f1, int f2) +QPDFObjGen +QPDF::Xref_table::at_offset(qpdf_offset_t offset) const noexcept { - if (!(obj > 0 && obj <= m->xref_table_max_id && 0 <= f2 && f2 < 65535)) { - QTC::TC("qpdf", "QPDF xref overwrite invalid objgen"); - return; - } + int id = 0; + int gen = 0; + qpdf_offset_t start = 0; - QPDFObjGen og(obj, f2); - if (!m->deleted_objects.count(obj)) { - // deleted_objects stores the uncompressed objects removed from the xref table at the start - // of recovery. - QTC::TC("qpdf", "QPDF xref overwrite object"); - m->xref_table[QPDFObjGen(obj, f2)] = QPDFXRefEntry(f1); + int i = 0; + for (auto const& item: table) { + auto o = item.offset(); + if (start < o && o <= offset) { + start = o; + id = i; + gen = item.gen(); + } + ++i; } + return QPDFObjGen(id, gen); } -void -QPDF::showXRefTable() +std::map +QPDF::Xref_table::as_map() const { - auto& cout = *m->log->getInfo(); - for (auto const& iter: m->xref_table) { - QPDFObjGen const& og = iter.first; - QPDFXRefEntry const& entry = iter.second; - cout << og.unparse('/') << ": "; - switch (entry.getType()) { + std::map result; + int i{0}; + for (auto const& item: table) { + switch (item.type()) { + case 0: + break; case 1: - cout << "uncompressed; offset = " << entry.getOffset(); + result.emplace(QPDFObjGen(i, item.gen()), item.offset()); break; - case 2: - *m->log->getInfo() << "compressed; stream = " << entry.getObjStreamNumber() - << ", index = " << entry.getObjStreamIndex(); + result.emplace( + QPDFObjGen(i, 0), QPDFXRefEntry(item.stream_number(), item.stream_index())); break; - default: - throw std::logic_error("unknown cross-reference table type while" - " showing xref_table"); - break; + throw std::logic_error("Xref_table: invalid entry type"); + } + ++i; + } + return result; +} + +void +QPDF::showXRefTable() +{ + m->xref_table.show(); +} + +void +QPDF::Xref_table::show() +{ + auto& cout = *qpdf.m->log->getInfo(); + int i = -1; + for (auto const& item: table) { + ++i; + if (item.type()) { + cout << std::to_string(i) << "/" << std::to_string(item.gen()) << ": "; + switch (item.type()) { + case 1: + cout << "uncompressed; offset = " << item.offset() << "\n"; + break; + + case 2: + cout << "compressed; stream = " << item.stream_number() + << ", index = " << item.stream_index() << "\n"; + break; + + default: + throw std::logic_error( + "unknown cross-reference table type while showing xref_table"); + } } - m->log->info("\n"); } } // Resolve all objects in the xref table. If this triggers a xref table reconstruction abort and // return false. Otherwise return true. bool -QPDF::resolveXRefTable() -{ - bool may_change = !m->reconstructed_xref; - for (auto& iter: m->xref_table) { - if (isUnresolved(iter.first)) { - resolve(iter.first); - if (may_change && m->reconstructed_xref) { - return false; +QPDF::Xref_table::resolve() +{ + bool may_change = !reconstructed_; + int i = -1; + for (auto& item: table) { + ++i; + if (item.type()) { + if (qpdf.isUnresolved(QPDFObjGen(i, item.gen()))) { + qpdf.resolve(QPDFObjGen(i, item.gen())); + if (may_change && reconstructed_) { + return false; + } } } } @@ -1401,9 +1532,9 @@ QPDF::fixDanglingReferences(bool force) if (m->fixed_dangling_refs) { return; } - if (!resolveXRefTable()) { + if (!m->xref_table.resolve()) { QTC::TC("qpdf", "QPDF fix dangling triggered xref reconstruction"); - resolveXRefTable(); + m->xref_table.resolve(); } m->fixed_dangling_refs = true; } @@ -1450,21 +1581,21 @@ QPDF::setLastObjectDescription(std::string const& description, QPDFObjGen const& } QPDFObjectHandle -QPDF::readTrailer() +QPDF::Xref_table::read_trailer() { - qpdf_offset_t offset = m->file->tell(); + qpdf_offset_t offset = file->tell(); bool empty = false; auto object = - QPDFParser(*m->file, "trailer", m->tokenizer, nullptr, this, true).parse(empty, false); + QPDFParser(*file, "trailer", tokenizer, nullptr, &qpdf, true).parse(empty, false); if (empty) { // Nothing in the PDF spec appears to allow empty objects, but they have been encountered in // actual PDF files and Adobe Reader appears to ignore them. - warn(damagedPDF("trailer", "empty object treated as null")); - } else if (object.isDictionary() && readToken(*m->file).isWord("stream")) { - warn(damagedPDF("trailer", m->file->tell(), "stream keyword found in trailer")); + qpdf.warn(qpdf.damagedPDF("trailer", "empty object treated as null")); + } else if (object.isDictionary() && read_token().isWord("stream")) { + qpdf.warn(qpdf.damagedPDF("trailer", file->tell(), "stream keyword found in trailer")); } // Override last_offset so that it points to the beginning of the object we just read - m->file->setLastOffset(offset); + file->setLastOffset(offset); return object; } @@ -1532,7 +1663,7 @@ QPDF::readStream(QPDFObjectHandle& object, QPDFObjGen og, qpdf_offset_t offset) } catch (QPDFExc& e) { if (m->attempt_recovery) { warn(e); - length = recoverStreamLength(m->file, og, stream_offset); + length = recoverStreamLength(m->file_sp, og, stream_offset); } else { throw; } @@ -1639,21 +1770,9 @@ QPDF::recoverStreamLength( } if (length) { - auto end = stream_offset + toO(length); - qpdf_offset_t found_offset = 0; - QPDFObjGen found_og; - // Make sure this is inside this object - for (auto const& [current_og, entry]: m->xref_table) { - if (entry.getType() == 1) { - qpdf_offset_t obj_offset = entry.getOffset(); - if (found_offset < obj_offset && obj_offset < end) { - found_offset = obj_offset; - found_og = current_og; - } - } - } - if (!found_offset || found_og == og) { + auto found = m->xref_table.at_offset(stream_offset + toO(length)); + if (found == QPDFObjGen() || found == og) { // If we are trying to recover an XRef stream the xref table will not contain and // won't contain any entries, therefore we cannot check the found length. Otherwise we // found endstream\nendobj within the space allowed for this object, so we're probably @@ -1762,21 +1881,18 @@ QPDF::readObjectAtOffset( } catch (QPDFExc& e) { if (try_recovery) { // Try again after reconstructing xref table - reconstruct_xref(e); - if (m->xref_table.count(exp_og) && (m->xref_table[exp_og].getType() == 1)) { - qpdf_offset_t new_offset = m->xref_table[exp_og].getOffset(); - QPDFObjectHandle result = - readObjectAtOffset(false, new_offset, description, exp_og, og, false); + m->xref_table.reconstruct(e); + if (m->xref_table.type(exp_og) == 1) { QTC::TC("qpdf", "QPDF recovered in readObjectAtOffset"); - return result; + return readObjectAtOffset( + false, m->xref_table.offset(exp_og), description, exp_og, og, false); } else { QTC::TC("qpdf", "QPDF object gone after xref reconstruction"); warn(damagedPDF( "", 0, ("object " + exp_og.unparse(' ') + - " not found in file after regenerating cross reference " - "table"))); + " not found in file after regenerating cross reference table"))); return QPDFObjectHandle::newNull(); } } else { @@ -1809,7 +1925,7 @@ QPDF::readObjectAtOffset( } } qpdf_offset_t end_after_space = m->file->tell(); - if (skip_cache_if_in_xref && m->xref_table.count(og)) { + if (skip_cache_if_in_xref && m->xref_table.type(og)) { // Ordinarily, an object gets read here when resolved through xref table or stream. In // the special case of the xref stream and linearization hint tables, the offset comes // from another source. For the specific case of xref streams, the xref stream is read @@ -1837,7 +1953,9 @@ QPDF::readObjectAtOffset( // could use !check_og in place of skip_cache_if_in_xref. QTC::TC("qpdf", "QPDF skipping cache for known unchecked object"); } else { - updateCache(og, oh.getObj(), end_before_space, end_after_space); + m->xref_table.linearization_offsets( + toS(og.getObj()), end_before_space, end_after_space); + updateCache(og, oh.getObj()); } } @@ -1856,44 +1974,43 @@ QPDF::resolve(QPDFObjGen og) // has to be resolved during object parsing, such as stream length. QTC::TC("qpdf", "QPDF recursion loop in resolve"); warn(damagedPDF("", "loop detected resolving object " + og.unparse(' '))); - updateCache(og, QPDF_Null::create(), -1, -1); + updateCache(og, QPDF_Null::create()); return m->obj_cache[og].object.get(); } ResolveRecorder rr(this, og); - if (m->xref_table.count(og) != 0) { - QPDFXRefEntry const& entry = m->xref_table[og]; - try { - switch (entry.getType()) { - case 1: - { - qpdf_offset_t offset = entry.getOffset(); - // Object stored in cache by readObjectAtOffset - QPDFObjGen a_og; - QPDFObjectHandle oh = readObjectAtOffset(true, offset, "", og, a_og, false); - } - break; + try { + switch (m->xref_table.type(og)) { + case 0: + break; + case 1: + { + // Object stored in cache by readObjectAtOffset + QPDFObjGen a_og; + QPDFObjectHandle oh = + readObjectAtOffset(true, m->xref_table.offset(og), "", og, a_og, false); + } + break; - case 2: - resolveObjectsInStream(entry.getObjStreamNumber()); - break; + case 2: + resolveObjectsInStream(m->xref_table.stream_number(og.getObj())); + break; - default: - throw damagedPDF( - "", 0, ("object " + og.unparse('/') + " has unexpected xref entry type")); - } - } catch (QPDFExc& e) { - warn(e); - } catch (std::exception& e) { - warn(damagedPDF( - "", 0, ("object " + og.unparse('/') + ": error reading object: " + e.what()))); + default: + throw damagedPDF( + "", 0, ("object " + og.unparse('/') + " has unexpected xref entry type")); } + } catch (QPDFExc& e) { + warn(e); + } catch (std::exception& e) { + warn(damagedPDF( + "", 0, ("object " + og.unparse('/') + ": error reading object: " + e.what()))); } if (isUnresolved(og)) { // PDF spec says unknown objects resolve to the null object. QTC::TC("qpdf", "QPDF resolve failure to null"); - updateCache(og, QPDF_Null::create(), -1, -1); + updateCache(og, QPDF_Null::create()); } auto result(m->obj_cache[og].object); @@ -1915,12 +2032,6 @@ QPDF::resolveObjectsInStream(int obj_stream_number) "supposed object stream " + std::to_string(obj_stream_number) + " is not a stream"); } - // For linearization data in the object, use the data from the object stream for the objects in - // the stream. - QPDFObjGen stream_og(obj_stream_number, 0); - qpdf_offset_t end_before_space = m->obj_cache[stream_og].end_before_space; - qpdf_offset_t end_after_space = m->obj_cache[stream_og].end_after_space; - QPDFObjectHandle dict = obj_stream.getDict(); if (!dict.isDictionaryOfType("/ObjStm")) { QTC::TC("qpdf", "QPDF ERR object stream with wrong type"); @@ -1958,7 +2069,7 @@ QPDF::resolveObjectsInStream(int obj_stream_number) int num = QUtil::string_to_int(tnum.getValue().c_str()); long long offset = QUtil::string_to_int(toffset.getValue().c_str()); - if (num > m->xref_table_max_id) { + if (num > m->xref_table.max_id()) { continue; } if (num == obj_stream_number) { @@ -1981,13 +2092,12 @@ QPDF::resolveObjectsInStream(int obj_stream_number) m->last_object_description += "object "; for (auto const& iter: offsets) { QPDFObjGen og(iter.first, 0); - auto entry = m->xref_table.find(og); - if (entry != m->xref_table.end() && entry->second.getType() == 2 && - entry->second.getObjStreamNumber() == obj_stream_number) { + if (m->xref_table.type(og) == 2 && + m->xref_table.stream_number(og.getObj()) == obj_stream_number) { int offset = iter.second; input->seek(offset, SEEK_SET); QPDFObjectHandle oh = readObjectInStream(input, iter.first); - updateCache(og, oh.getObj(), end_before_space, end_after_space); + updateCache(og, oh.getObj()); } else { QTC::TC("qpdf", "QPDF not caching overridden objstm object"); } @@ -2002,20 +2112,14 @@ QPDF::newIndirect(QPDFObjGen const& og, std::shared_ptr const& obj) } void -QPDF::updateCache( - QPDFObjGen const& og, - std::shared_ptr const& object, - qpdf_offset_t end_before_space, - qpdf_offset_t end_after_space) +QPDF::updateCache(QPDFObjGen const& og, std::shared_ptr const& object) { object->setObjGen(this, og); if (isCached(og)) { auto& cache = m->obj_cache[og]; cache.object->assign(object); - cache.end_before_space = end_before_space; - cache.end_after_space = end_after_space; } else { - m->obj_cache[og] = ObjCache(object, end_before_space, end_after_space); + m->obj_cache[og] = ObjCache(object); } } @@ -2045,7 +2149,7 @@ QPDFObjectHandle QPDF::makeIndirectFromQPDFObject(std::shared_ptr const& obj) { QPDFObjGen next{nextObjGen()}; - m->obj_cache[next] = ObjCache(obj, -1, -1); + m->obj_cache[next] = ObjCache(obj); return newIndirect(next, m->obj_cache[next].object); } @@ -2101,7 +2205,7 @@ QPDF::getObjectForParser(int id, int gen, bool parse_pdf) if (auto iter = m->obj_cache.find(og); iter != m->obj_cache.end()) { return iter->second.object; } - if (m->xref_table.count(og) || !m->parsed) { + if (m->xref_table.type(og) || !m->xref_table.initialized()) { return m->obj_cache.insert({og, QPDF_Unresolved::create(this, og)}).first->second.object; } if (parse_pdf) { @@ -2117,8 +2221,9 @@ QPDF::getObjectForJSON(int id, int gen) auto [it, inserted] = m->obj_cache.try_emplace(og); auto& obj = it->second.object; if (inserted) { - obj = (m->parsed && !m->xref_table.count(og)) ? QPDF_Null::create(this, og) - : QPDF_Unresolved::create(this, og); + obj = (m->xref_table.initialized() && !m->xref_table.type(og)) + ? QPDF_Null::create(this, og) + : QPDF_Unresolved::create(this, og); } return obj; } @@ -2128,10 +2233,10 @@ QPDF::getObject(QPDFObjGen const& og) { if (auto it = m->obj_cache.find(og); it != m->obj_cache.end()) { return {it->second.object}; - } else if (m->parsed && !m->xref_table.count(og)) { + } else if (m->xref_table.initialized() && !m->xref_table.type(og)) { return QPDF_Null::create(); } else { - auto result = m->obj_cache.try_emplace(og, QPDF_Unresolved::create(this, og), -1, -1); + auto result = m->obj_cache.try_emplace(og, QPDF_Unresolved::create(this, og)); return {result.first->second.object}; } } @@ -2167,13 +2272,12 @@ QPDF::replaceObject(QPDFObjGen const& og, QPDFObjectHandle oh) QTC::TC("qpdf", "QPDF replaceObject called with indirect object"); throw std::logic_error("QPDF::replaceObject called with indirect object handle"); } - updateCache(og, oh.getObj(), -1, -1); + updateCache(og, oh.getObj()); } void QPDF::removeObject(QPDFObjGen og) { - m->xref_table.erase(og); if (auto cached = m->obj_cache.find(og); cached != m->obj_cache.end()) { // Take care of any object handles that may be floating around. cached->second.object->assign(QPDF_Null::create()); @@ -2442,7 +2546,7 @@ QPDF::copyStreamData(QPDFObjectHandle result, QPDFObjectHandle foreign) } else { auto foreign_stream_data = std::make_shared( foreign_stream_qpdf.m->encp, - foreign_stream_qpdf.m->file, + foreign_stream_qpdf.m->file_sp, foreign.getObjGen(), stream->getParsedOffset(), stream->getLength(), @@ -2526,13 +2630,13 @@ QPDF::getExtensionLevel() QPDFObjectHandle QPDF::getTrailer() { - return m->trailer; + return m->xref_table.trailer(); } QPDFObjectHandle QPDF::getRoot() { - QPDFObjectHandle root = m->trailer.getKey("/Root"); + QPDFObjectHandle root = m->xref_table.trailer().getKey("/Root"); if (!root.isDictionary()) { throw damagedPDF("", 0, "unable to find /Root dictionary"); } else if ( @@ -2548,17 +2652,10 @@ QPDF::getRoot() std::map QPDF::getXRefTable() { - return getXRefTableInternal(); -} - -std::map const& -QPDF::getXRefTableInternal() -{ - if (!m->parsed) { + if (!m->xref_table.initialized()) { throw std::logic_error("QPDF::getXRefTable called before parsing."); } - - return m->xref_table; + return m->xref_table.as_map(); } size_t @@ -2566,7 +2663,10 @@ QPDF::tableSize() { // If obj_cache is dense, accommodate all object in tables,else accommodate only original // objects. - auto max_xref = m->xref_table.size() ? m->xref_table.crbegin()->first.getObj() : 0; + auto max_xref = toI(m->xref_table.size()); + if (max_xref > 0) { + --max_xref; + } auto max_obj = m->obj_cache.size() ? m->obj_cache.crbegin()->first.getObj() : 0; auto max_id = std::numeric_limits::max() - 1; if (max_obj >= max_id || max_xref >= max_id) { @@ -2604,14 +2704,14 @@ QPDF::getCompressibleObjGens() // iterating through the xref table since it avoids preserving orphaned items. // Exclude encryption dictionary, if any - QPDFObjectHandle encryption_dict = m->trailer.getKey("/Encrypt"); + QPDFObjectHandle encryption_dict = m->xref_table.trailer().getKey("/Encrypt"); QPDFObjGen encryption_dict_og = encryption_dict.getObjGen(); const size_t max_obj = getObjectCount(); std::vector visited(max_obj, false); std::vector queue; queue.reserve(512); - queue.push_back(m->trailer); + queue.push_back(m->xref_table.trailer()); std::vector result; if constexpr (std::is_same_v) { result.reserve(m->obj_cache.size()); @@ -2766,7 +2866,7 @@ QPDF::pipeStreamData( { return pipeStreamData( m->encp, - m->file, + m->file_sp, *this, og, offset, diff --git a/libqpdf/QPDFJob.cc b/libqpdf/QPDFJob.cc index 5590286..2433f24 100644 --- a/libqpdf/QPDFJob.cc +++ b/libqpdf/QPDFJob.cc @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -26,6 +25,7 @@ #include #include #include +#include #include #include diff --git a/libqpdf/QPDFWriter.cc b/libqpdf/QPDFWriter.cc index a1ae23c..4ab0cab 100644 --- a/libqpdf/QPDFWriter.cc +++ b/libqpdf/QPDFWriter.cc @@ -14,10 +14,10 @@ #include #include #include -#include #include #include #include +#include #include #include #include @@ -1698,7 +1698,6 @@ QPDFWriter::writeObjectStream(QPDFObjectHandle object) if (obj_to_write.isStream()) { // This condition occurred in a fuzz input. Ideally we should block it at parse // time, but it's not clear to me how to construct a case for this. - QTC::TC("qpdf", "QPDFWriter stream in ostream"); obj_to_write.warnIfPossible("stream found inside object stream; treating as null"); obj_to_write = QPDFObjectHandle::newNull(); } @@ -1937,47 +1936,26 @@ void QPDFWriter::preserveObjectStreams() { auto const& xref = QPDF::Writer::getXRefTable(m->pdf); - // Our object_to_object_stream map has to map ObjGen -> ObjGen since we may be generating object - // streams out of old objects that have generation numbers greater than zero. However in an - // existing PDF, all object stream objects and all objects in them must have generation 0 - // because the PDF spec does not provide any way to do otherwise. This code filters out objects - // that are not allowed to be in object streams. In addition to removing objects that were - // erroneously included in object streams in the source PDF, it also prevents unreferenced - // objects from being included. - auto end = xref.cend(); - m->obj.streams_empty = true; + m->obj.streams_empty = !xref.object_streams(); + if (m->obj.streams_empty) { + return; + } + // This code filters out objects that are not allowed to be in object streams. In addition to + // removing objects that were erroneously included in object streams in the source PDF, it also + // prevents unreferenced objects from being included. if (m->preserve_unreferenced_objects) { - for (auto iter = xref.cbegin(); iter != end; ++iter) { - if (iter->second.getType() == 2) { - // Pdf contains object streams. - QTC::TC("qpdf", "QPDFWriter preserve object streams preserve unreferenced"); - m->obj.streams_empty = false; - m->obj[iter->first].object_stream = iter->second.getObjStreamNumber(); - } + QTC::TC("qpdf", "QPDFWriter preserve object streams preserve unreferenced"); + for (auto [id, stream]: xref.compressed_objects()) { + m->obj[id].object_stream = stream; } } else { - // Start by scanning for first compressed object in case we don't have any object streams to - // process. - for (auto iter = xref.cbegin(); iter != end; ++iter) { - if (iter->second.getType() == 2) { - // Pdf contains object streams. - QTC::TC("qpdf", "QPDFWriter preserve object streams"); - m->obj.streams_empty = false; - auto eligible = QPDF::Writer::getCompressibleObjSet(m->pdf); - // The object pointed to by iter may be a previous generation, in which case it is - // removed by getCompressibleObjSet. We need to restart the loop (while the object - // table may contain multiple generations of an object). - for (iter = xref.cbegin(); iter != end; ++iter) { - if (iter->second.getType() == 2) { - auto id = static_cast(iter->first.getObj()); - if (id < eligible.size() && eligible[id]) { - m->obj[iter->first].object_stream = iter->second.getObjStreamNumber(); - } else { - QTC::TC("qpdf", "QPDFWriter exclude from object stream"); - } - } - } - return; + QTC::TC("qpdf", "QPDFWriter preserve object streams"); + auto eligible = QPDF::Writer::getCompressibleObjSet(m->pdf); + for (auto [id, stream]: xref.compressed_objects()) { + if (eligible[id]) { + m->obj[id].object_stream = stream; + } else { + QTC::TC("qpdf", "QPDFWriter exclude from object stream"); } } } diff --git a/libqpdf/QPDF_Stream.cc b/libqpdf/QPDF_Stream.cc index 8ffe919..e64366d 100644 --- a/libqpdf/QPDF_Stream.cc +++ b/libqpdf/QPDF_Stream.cc @@ -10,8 +10,8 @@ #include #include #include -#include #include +#include #include #include #include diff --git a/libqpdf/QPDF_encryption.cc b/libqpdf/QPDF_encryption.cc index 1dd7b96..436111b 100644 --- a/libqpdf/QPDF_encryption.cc +++ b/libqpdf/QPDF_encryption.cc @@ -3,7 +3,7 @@ #include -#include +#include #include @@ -727,7 +727,7 @@ QPDF::initializeEncryption() // at /Encrypt again. Otherwise, things could go wrong if someone mutates the encryption // dictionary. - if (!m->trailer.hasKey("/Encrypt")) { + if (!m->xref_table.trailer().hasKey("/Encrypt")) { return; } @@ -736,7 +736,7 @@ QPDF::initializeEncryption() m->encp->encrypted = true; std::string id1; - QPDFObjectHandle id_obj = m->trailer.getKey("/ID"); + QPDFObjectHandle id_obj = m->xref_table.trailer().getKey("/ID"); if ((id_obj.isArray() && (id_obj.getArrayNItems() == 2) && id_obj.getArrayItem(0).isString())) { id1 = id_obj.getArrayItem(0).getStringValue(); } else { @@ -745,7 +745,7 @@ QPDF::initializeEncryption() warn(damagedPDF("trailer", "invalid /ID in trailer dictionary")); } - QPDFObjectHandle encryption_dict = m->trailer.getKey("/Encrypt"); + QPDFObjectHandle encryption_dict = m->xref_table.trailer().getKey("/Encrypt"); if (!encryption_dict.isDictionary()) { throw damagedPDF("/Encrypt in trailer dictionary is not a dictionary"); } diff --git a/libqpdf/QPDF_json.cc b/libqpdf/QPDF_json.cc index 8cbbcd1..4bb5a9a 100644 --- a/libqpdf/QPDF_json.cc +++ b/libqpdf/QPDF_json.cc @@ -51,17 +51,6 @@ // ] | <- st_top // } | -static char const* JSON_PDF = ( - // force line break - "%PDF-1.3\n" - "xref\n" - "0 1\n" - "0000000000 65535 f \n" - "trailer << /Size 1 >>\n" - "startxref\n" - "9\n" - "%%EOF\n"); - // Validator methods -- these are much more performant than std::regex. static bool is_indirect_object(std::string const& v, int& obj, int& gen) @@ -267,10 +256,10 @@ class QPDF::JSONReactor: public JSON::Reactor struct StackFrame { StackFrame(state_e state) : - state(state) {}; + state(state){}; StackFrame(state_e state, QPDFObjectHandle&& object) : state(state), - object(object) {}; + object(object){}; state_e state; QPDFObjectHandle object; }; @@ -593,8 +582,7 @@ QPDF::JSONReactor::dictionaryItem(std::string const& key, JSON const& value) this->saw_value = true; // The trailer must be a dictionary, so we can use setNextStateIfDictionary. if (setNextStateIfDictionary("trailer.value", value, st_object)) { - this->pdf.m->trailer = makeObject(value); - setObjectDescription(this->pdf.m->trailer, value); + pdf.m->xref_table.trailer(makeObject(value)); } } else if (key == "stream") { // Don't need to set saw_stream here since there's already an error. @@ -786,7 +774,9 @@ QPDF::createFromJSON(std::string const& json_file) void QPDF::createFromJSON(std::shared_ptr is) { - processMemoryFile(is->getName().c_str(), JSON_PDF, strlen(JSON_PDF)); + m->pdf_version = "1.3"; + m->no_input_name = is->getName(); + m->xref_table.initialize_json(); importJSON(is, true); } diff --git a/libqpdf/QPDF_linearization.cc b/libqpdf/QPDF_linearization.cc index 9ffefd2..9178731 100644 --- a/libqpdf/QPDF_linearization.cc +++ b/libqpdf/QPDF_linearization.cc @@ -1,6 +1,6 @@ // See doc/linearization. -#include +#include #include #include @@ -288,9 +288,8 @@ QPDF::readHintStream(Pipeline& pl, qpdf_offset_t offset, size_t length) QPDFObjGen og; QPDFObjectHandle H = readObjectAtOffset(false, offset, "linearization hint stream", QPDFObjGen(0, 0), og, false); - ObjCache& oc = m->obj_cache[og]; - qpdf_offset_t min_end_offset = oc.end_before_space; - qpdf_offset_t max_end_offset = oc.end_after_space; + qpdf_offset_t min_end_offset = m->xref_table.end_before_space(og); + qpdf_offset_t max_end_offset = m->xref_table.end_after_space(og); if (!H.isStream()) { throw damagedPDF("linearization dictionary", "hint table is not a stream"); } @@ -301,14 +300,11 @@ QPDF::readHintStream(Pipeline& pl, qpdf_offset_t offset, size_t length) // increasing length to cover it, even though the specification says all objects in the // linearization parameter dictionary must be direct. We have to get the file position of the // end of length in this case. - QPDFObjectHandle length_obj = Hdict.getKey("/Length"); - if (length_obj.isIndirect()) { + auto length_og = Hdict.getKey("/Length").getObjGen(); + if (length_og.isIndirect()) { QTC::TC("qpdf", "QPDF hint table length indirect"); - // Force resolution - (void)length_obj.getIntValue(); - ObjCache& oc2 = m->obj_cache[length_obj.getObjGen()]; - min_end_offset = oc2.end_before_space; - max_end_offset = oc2.end_after_space; + min_end_offset = m->xref_table.end_before_space(length_og); + max_end_offset = m->xref_table.end_after_space(length_og); } else { QTC::TC("qpdf", "QPDF hint table length direct"); } @@ -445,7 +441,7 @@ QPDF::checkLinearizationInternal() for (size_t i = 0; i < toS(npages); ++i) { QPDFObjectHandle const& page = pages.at(i); QPDFObjGen og(page.getObjGen()); - if (m->xref_table[og].getType() == 2) { + if (m->xref_table.type(og) == 2) { linearizationWarning( "page dictionary for page " + std::to_string(i) + " is compressed"); } @@ -461,12 +457,11 @@ QPDF::checkLinearizationInternal() break; } } - if (m->file->tell() != m->first_xref_item_offset) { + if (m->file->tell() != m->xref_table.first_item_offset()) { QTC::TC("qpdf", "QPDF err /T mismatch"); linearizationWarning( - "space before first xref item (/T) mismatch " - "(computed = " + - std::to_string(m->first_xref_item_offset) + + "space before first xref item (/T) mismatch (computed = " + + std::to_string(m->xref_table.first_item_offset()) + "; file = " + std::to_string(m->file->tell())); } @@ -477,7 +472,7 @@ QPDF::checkLinearizationInternal() // compressed objects are supposed to be at the end of the containing xref section if any object // streams are in use. - if (m->uncompressed_after_compressed) { + if (m->xref_table.uncompressed_after_compressed()) { linearizationWarning("linearized file contains an uncompressed object after a compressed " "one in a cross-reference stream"); } @@ -485,18 +480,9 @@ QPDF::checkLinearizationInternal() // Further checking requires optimization and order calculation. Don't allow optimization to // make changes. If it has to, then the file is not properly linearized. We use the xref table // to figure out which objects are compressed and which are uncompressed. - { // local scope - std::map object_stream_data; - for (auto const& iter: m->xref_table) { - QPDFObjGen const& og = iter.first; - QPDFXRefEntry const& entry = iter.second; - if (entry.getType() == 2) { - object_stream_data[og.getObj()] = entry.getObjStreamNumber(); - } - } - optimize(object_stream_data, false); - calculateLinearizationData(object_stream_data); - } + + optimize(m->xref_table); + calculateLinearizationData(m->xref_table); // E: offset of end of first page -- Implementation note 123 says Acrobat includes on extra // object here by mistake. pdlin fails to place thumbnail images in section 9, so when @@ -513,13 +499,14 @@ QPDF::checkLinearizationInternal() qpdf_offset_t max_E = -1; for (auto const& oh: m->part6) { QPDFObjGen og(oh.getObjGen()); - if (m->obj_cache.count(og) == 0) { + auto before = m->xref_table.end_before_space(og); + auto after = m->xref_table.end_after_space(og); + if (before <= 0) { // All objects have to have been dereferenced to be classified. throw std::logic_error("linearization part6 object not in cache"); } - ObjCache const& oc = m->obj_cache[og]; - min_E = std::max(min_E, oc.end_before_space); - max_E = std::max(max_E, oc.end_after_space); + min_E = std::max(min_E, before); + max_E = std::max(max_E, after); } if ((p.first_page_end < min_E) || (p.first_page_end > max_E)) { QTC::TC("qpdf", "QPDF warn /E mismatch"); @@ -546,10 +533,11 @@ QPDF::maxEnd(ObjUser const& ou) } qpdf_offset_t end = 0; for (auto const& og: m->obj_user_to_objects[ou]) { - if (m->obj_cache.count(og) == 0) { + auto e = m->xref_table.end_after_space(og); + if (e <= 0) { stopOnError("unknown object referenced in object user table"); } - end = std::max(end, m->obj_cache[og].end_after_space); + end = std::max(end, e); } return end; } @@ -557,23 +545,18 @@ QPDF::maxEnd(ObjUser const& ou) qpdf_offset_t QPDF::getLinearizationOffset(QPDFObjGen const& og) { - QPDFXRefEntry entry = m->xref_table[og]; - qpdf_offset_t result = 0; - switch (entry.getType()) { + switch (m->xref_table.type(og)) { case 1: - result = entry.getOffset(); - break; + return m->xref_table.offset(og); case 2: // For compressed objects, return the offset of the object stream that contains them. - result = getLinearizationOffset(QPDFObjGen(entry.getObjStreamNumber(), 0)); - break; + return getLinearizationOffset(QPDFObjGen(m->xref_table.stream_number(og.getObj()), 0)); default: stopOnError("getLinearizationOffset called for xref entry not of type 1 or 2"); - break; + return 0; // unreachable } - return result; } QPDFObjectHandle @@ -588,6 +571,16 @@ QPDF::getUncompressedObject(QPDFObjectHandle& obj, std::map const& obj } QPDFObjectHandle +QPDF::getUncompressedObject(QPDFObjectHandle& obj, Xref_table const& xref) +{ + auto og = obj.getObjGen(); + if (obj.isNull() || xref.type(og) != 2) { + return obj; + } + return getObject(xref.stream_number(og.getObj()), 0); +} + +QPDFObjectHandle QPDF::getUncompressedObject(QPDFObjectHandle& oh, QPDFWriter::ObjTable const& obj) { if (obj.contains(oh)) { @@ -604,15 +597,13 @@ QPDF::lengthNextN(int first_object, int n) int length = 0; for (int i = 0; i < n; ++i) { QPDFObjGen og(first_object + i, 0); - if (m->xref_table.count(og) == 0) { + auto end = m->xref_table.end_after_space(og); + if (end <= 0) { linearizationWarning( "no xref table entry for " + std::to_string(first_object + i) + " 0"); - } else { - if (m->obj_cache.count(og) == 0) { - stopOnError("found unknown object while calculating length for linearization data"); - } - length += toI(m->obj_cache[og].end_after_space - getLinearizationOffset(og)); + continue; } + length += toI(end - getLinearizationOffset(og)); } return length; } @@ -636,7 +627,7 @@ QPDF::checkHPageOffset( int npages = toI(pages.size()); qpdf_offset_t table_offset = adjusted_offset(m->page_offset_hints.first_page_offset); QPDFObjGen first_page_og(pages.at(0).getObjGen()); - if (m->xref_table.count(first_page_og) == 0) { + if (m->xref_table.type(first_page_og) == 0) { stopOnError("supposed first page object is not known"); } qpdf_offset_t offset = getLinearizationOffset(first_page_og); @@ -647,7 +638,7 @@ QPDF::checkHPageOffset( for (int pageno = 0; pageno < npages; ++pageno) { QPDFObjGen page_og(pages.at(toS(pageno)).getObjGen()); int first_object = page_og.getObj(); - if (m->xref_table.count(page_og) == 0) { + if (m->xref_table.type(page_og) == 0) { stopOnError("unknown object in page offset hint table"); } offset = getLinearizationOffset(page_og); @@ -769,7 +760,7 @@ QPDF::checkHSharedObject(std::vector const& pages, std::mapxref_table.count(og) == 0) { + if (m->xref_table.type(og) == 0) { stopOnError("unknown object in shared object hint table"); } qpdf_offset_t offset = getLinearizationOffset(og); @@ -820,7 +811,7 @@ QPDF::checkHOutlines() return; } QPDFObjGen og(outlines.getObjGen()); - if (m->xref_table.count(og) == 0) { + if (m->xref_table.type(og) == 0) { stopOnError("unknown object in outlines hint table"); } qpdf_offset_t offset = getLinearizationOffset(og); @@ -839,8 +830,7 @@ QPDF::checkHOutlines() std::to_string(table_length) + "; computed = " + std::to_string(length)); } } else { - linearizationWarning("incorrect first object number in outline " - "hints table."); + linearizationWarning("incorrect first object number in outline hints table."); } } else { linearizationWarning("incorrect object count in outline hint table"); diff --git a/libqpdf/QPDF_optimization.cc b/libqpdf/QPDF_optimization.cc index 0e457af..3dcaa36 100644 --- a/libqpdf/QPDF_optimization.cc +++ b/libqpdf/QPDF_optimization.cc @@ -2,7 +2,7 @@ #include -#include +#include #include #include @@ -78,6 +78,12 @@ QPDF::optimize( optimize_internal(obj, true, skip_stream_parameters); } +void +QPDF::optimize(QPDF::Xref_table const& xref) +{ + optimize_internal(xref, false, nullptr); +} + template void QPDF::optimize_internal( @@ -115,13 +121,13 @@ QPDF::optimize_internal( } // Traverse document-level items - for (auto const& key: m->trailer.getKeys()) { + for (auto const& key: m->xref_table.trailer().getKeys()) { if (key == "/Root") { // handled separately } else { updateObjectMaps( ObjUser(ObjUser::ou_trailer_key, key), - m->trailer.getKey(key), + m->xref_table.trailer().getKey(key), skip_stream_parameters); } } @@ -169,13 +175,13 @@ QPDF::pushInheritedAttributesToPage(bool allow_changes, bool warn_skipped_keys) // values for them. std::map> key_ancestors; pushInheritedAttributesToPageInternal( - m->trailer.getKey("/Root").getKey("/Pages"), + m->xref_table.trailer().getKey("/Root").getKey("/Pages"), key_ancestors, allow_changes, warn_skipped_keys); if (!key_ancestors.empty()) { - throw std::logic_error("key_ancestors not empty after" - " pushing inherited attributes to pages"); + throw std::logic_error( + "key_ancestors not empty after pushing inherited attributes to pages"); } m->pushed_inherited_attributes_to_pages = true; m->ever_pushed_inherited_attributes_to_pages = true; @@ -442,3 +448,45 @@ QPDF::filterCompressedObjects(QPDFWriter::ObjTable const& obj) m->obj_user_to_objects = t_obj_user_to_objects; m->object_to_obj_users = t_object_to_obj_users; } + +void +QPDF::filterCompressedObjects(QPDF::Xref_table const& xref) +{ + if (!xref.object_streams()) { + return; + } + + // Transform object_to_obj_users and obj_user_to_objects so that they refer only to uncompressed + // objects. If something is a user of a compressed object, then it is really a user of the + // object stream that contains it. + + std::map> t_obj_user_to_objects; + std::map> t_object_to_obj_users; + + for (auto const& i1: m->obj_user_to_objects) { + ObjUser const& ou = i1.first; + // Loop over objects. + for (auto const& og: i1.second) { + if (auto stream = xref.stream_number(og.getObj())) { + t_obj_user_to_objects[ou].insert(QPDFObjGen(stream, 0)); + } else { + t_obj_user_to_objects[ou].insert(og); + } + } + } + + for (auto const& i1: m->object_to_obj_users) { + QPDFObjGen const& og = i1.first; + // Loop over obj_users. + for (auto const& ou: i1.second) { + if (auto stream = xref.stream_number(og.getObj())) { + t_object_to_obj_users[QPDFObjGen(stream, 0)].insert(ou); + } else { + t_object_to_obj_users[og].insert(ou); + } + } + } + + m->obj_user_to_objects = t_obj_user_to_objects; + m->object_to_obj_users = t_object_to_obj_users; +} diff --git a/libqpdf/QPDF_pages.cc b/libqpdf/QPDF_pages.cc index 195421c..f46719d 100644 --- a/libqpdf/QPDF_pages.cc +++ b/libqpdf/QPDF_pages.cc @@ -1,4 +1,4 @@ -#include +#include #include #include diff --git a/libqpdf/qpdf/ObjTable.hh b/libqpdf/qpdf/ObjTable.hh index 3a36208..7d1daf1 100644 --- a/libqpdf/qpdf/ObjTable.hh +++ b/libqpdf/qpdf/ObjTable.hh @@ -46,6 +46,12 @@ class ObjTable: public std::vector } inline T const& + operator[](unsigned int idx) const + { + return element(idx); + } + + inline T const& operator[](QPDFObjGen og) const { return element(static_cast(og.getObj())); diff --git a/libqpdf/qpdf/QPDFObject_private.hh b/libqpdf/qpdf/QPDFObject_private.hh index f323d95..35f708c 100644 --- a/libqpdf/qpdf/QPDFObject_private.hh +++ b/libqpdf/qpdf/QPDFObject_private.hh @@ -6,14 +6,13 @@ #include #include -#include #include +#include #include #include #include -class QPDF; class QPDFObjectHandle; class QPDFObject diff --git a/libqpdf/qpdf/QPDF_private.hh b/libqpdf/qpdf/QPDF_private.hh new file mode 100644 index 0000000..b055763 --- /dev/null +++ b/libqpdf/qpdf/QPDF_private.hh @@ -0,0 +1,901 @@ +#ifndef QPDF_PRIVATE_HH +#define QPDF_PRIVATE_HH + +#include + +#include + +// Xref_table encapsulates the pdf's xref table and trailer. +class QPDF::Xref_table +{ + public: + Xref_table(QPDF& qpdf, InputSource* const& file) : + qpdf(qpdf), + file(file) + { + tokenizer.allowEOF(); + } + + void initialize(); + void initialize_empty(); + void initialize_json(); + void reconstruct(QPDFExc& e); + void show(); + bool resolve(); + + QPDFObjectHandle + trailer() const + { + return trailer_; + } + + void + trailer(QPDFObjectHandle&& oh) + { + trailer_ = std::move(oh); + } + + // Returns 0 if og is not in table. + size_t + type(QPDFObjGen og) const + { + int id = og.getObj(); + if (id < 1 || static_cast(id) >= table.size()) { + return 0; + } + auto& e = table[static_cast(id)]; + return e.gen() == og.getGen() ? e.type() : 0; + } + + // Returns 0 if og is not in table. + size_t + type(size_t id) const noexcept + { + if (id >= table.size()) { + return 0; + } + return table[id].type(); + } + + // Returns 0 if og is not in table. + qpdf_offset_t + offset(QPDFObjGen og) const noexcept + { + int id = og.getObj(); + if (id < 1 || static_cast(id) >= table.size()) { + return 0; + } + return table[static_cast(id)].offset(); + } + + // Returns 0 if id is not in table. + int + stream_number(int id) const noexcept + { + if (id < 1 || static_cast(id) >= table.size()) { + return 0; + } + return table[static_cast(id)].stream_number(); + } + + int + stream_index(int id) const noexcept + { + if (id < 1 || static_cast(id) >= table.size()) { + return 0; + } + return table[static_cast(id)].stream_index(); + } + + QPDFObjGen at_offset(qpdf_offset_t offset) const noexcept; + + std::map as_map() const; + + bool + object_streams() const noexcept + { + return object_streams_; + } + + // Return a vector of object id and stream number for each compressed object. + std::vector> + compressed_objects() const + { + if (!initialized()) { + throw std::logic_error("Xref_table::compressed_objects called before parsing."); + } + + std::vector> result; + result.reserve(table.size()); + + unsigned int i{0}; + for (auto const& item: table) { + if (item.type() == 2) { + result.emplace_back(i, item.stream_number()); + } + ++i; + } + return result; + } + + // Temporary access to underlying table size + size_t + size() const noexcept + { + return table.size(); + } + + void + ignore_streams(bool val) noexcept + { + ignore_streams_ = val; + } + + bool + initialized() const noexcept + { + return initialized_; + } + + void + attempt_recovery(bool val) noexcept + { + attempt_recovery_ = val; + } + + int + max_id() const noexcept + { + return max_id_; + } + + // For Linearization + + qpdf_offset_t + end_after_space(QPDFObjGen og) + { + auto& e = entry(toS(og.getObj())); + switch (e.type()) { + case 1: + return e.end_after_space_; + case 2: + { + auto es = entry(toS(e.stream_number())); + return es.type() == 1 ? es.end_after_space_ : 0; + } + default: + return 0; + } + } + + qpdf_offset_t + end_before_space(QPDFObjGen og) + { + auto& e = entry(toS(og.getObj())); + switch (e.type()) { + case 1: + return e.end_before_space_; + case 2: + { + auto es = entry(toS(e.stream_number())); + return es.type() == 1 ? es.end_before_space_ : 0; + } + default: + return 0; + } + } + + void + linearization_offsets(size_t id, qpdf_offset_t before, qpdf_offset_t after) + { + if (type(id)) { + table[id].end_before_space_ = before; + table[id].end_after_space_ = after; + } + } + + bool + uncompressed_after_compressed() const noexcept + { + return uncompressed_after_compressed_; + } + + // Actual value from file + qpdf_offset_t + first_item_offset() const noexcept + { + return first_item_offset_; + } + + private: + // Object, count, offset of first entry + typedef std::tuple Subsection; + + struct Uncompressed + { + Uncompressed(qpdf_offset_t offset) : + offset(offset) + { + } + qpdf_offset_t offset; + }; + + struct Compressed + { + Compressed(int stream_number, int stream_index) : + stream_number(stream_number), + stream_index(stream_index) + { + } + int stream_number{0}; + int stream_index{0}; + }; + + typedef std::variant Xref; + + struct Entry + { + Entry() = default; + + Entry(int gen, Xref entry) : + gen_(gen), + entry(entry) + { + } + + int + gen() const noexcept + { + return gen_; + } + + size_t + type() const noexcept + { + return entry.index(); + } + + qpdf_offset_t + offset() const noexcept + { + return type() == 1 ? std::get<1>(entry).offset : 0; + } + + int + stream_number() const noexcept + { + return type() == 2 ? std::get<2>(entry).stream_number : 0; + } + + int + stream_index() const noexcept + { + return type() == 2 ? std::get<2>(entry).stream_index : 0; + } + + int gen_{0}; + Xref entry; + qpdf_offset_t end_before_space_{0}; + qpdf_offset_t end_after_space_{0}; + }; + + Entry& + entry(size_t id) + { + return id < table.size() ? table[id] : table[0]; + } + + void read(qpdf_offset_t offset); + + // Methods to parse tables + qpdf_offset_t process_section(qpdf_offset_t offset); + std::vector subsections(std::string& line); + std::vector bad_subsections(std::string& line, qpdf_offset_t offset); + Subsection subsection(std::string const& line); + bool read_entry(qpdf_offset_t& f1, int& f2, char& type); + bool read_bad_entry(qpdf_offset_t& f1, int& f2, char& type); + + // Methods to parse streams + qpdf_offset_t read_stream(qpdf_offset_t offset); + qpdf_offset_t process_stream(qpdf_offset_t offset, QPDFObjectHandle& xref_stream); + std::pair> + process_W(QPDFObjectHandle& dict, std::function damaged); + std::pair process_Size( + QPDFObjectHandle& dict, int entry_size, std::function damaged); + std::pair>> process_Index( + QPDFObjectHandle& dict, + int max_num_entries, + std::function damaged); + + QPDFObjectHandle read_trailer(); + + QPDFTokenizer::Token + read_token(size_t max_len = 0) + { + return tokenizer.readToken(*file, "", true, max_len); + } + + // Methods to insert table entries + void insert(int obj, int f0, qpdf_offset_t f1, int f2); + void insert_free(QPDFObjGen); + + QPDFExc + damaged_pdf(std::string const& msg) + { + return qpdf.damagedPDF("", 0, msg); + } + + QPDFExc + damaged_table(std::string const& msg) + { + return qpdf.damagedPDF("xref table", msg); + } + + void + warn_damaged(std::string const& msg) + { + qpdf.warn(damaged_pdf(msg)); + } + + QPDF& qpdf; + InputSource* const& file; + QPDFTokenizer tokenizer; + + std::vector table; + QPDFObjectHandle trailer_; + + bool attempt_recovery_{true}; + bool initialized_{false}; + bool ignore_streams_{false}; + bool reconstructed_{false}; + bool object_streams_{false}; + // Before the xref table is initialized, max_id_ is an upper bound on the possible object ids + // that could be present in the PDF file. Once the trailer has been read, max_id_ is set to the + // value of /Size. If the file is damaged, max_id_ becomes the maximum object id in the xref + // table after reconstruction. + int max_id_{std::numeric_limits::max() - 1}; + + // Linearization data + bool uncompressed_after_compressed_{false}; + qpdf_offset_t first_item_offset_{0}; // actual value from file +}; + +// The Resolver class is restricted to QPDFObject so that only it can resolve indirect +// references. +class QPDF::Resolver +{ + friend class QPDFObject; + friend class QPDF_Unresolved; + + private: + static QPDFObject* + resolved(QPDF* qpdf, QPDFObjGen og) + { + return qpdf->resolve(og); + } +}; + +// StreamCopier class is restricted to QPDFObjectHandle so it can copy stream data. +class QPDF::StreamCopier +{ + friend class QPDFObjectHandle; + + private: + static void + copyStreamData(QPDF* qpdf, QPDFObjectHandle const& dest, QPDFObjectHandle const& src) + { + qpdf->copyStreamData(dest, src); + } +}; + +// The ParseGuard class allows QPDFParser to detect re-entrant parsing. It also provides +// special access to allow the parser to create unresolved objects and dangling references. +class QPDF::ParseGuard +{ + friend class QPDFParser; + + private: + ParseGuard(QPDF* qpdf) : + qpdf(qpdf) + { + if (qpdf) { + qpdf->inParse(true); + } + } + + static std::shared_ptr + getObject(QPDF* qpdf, int id, int gen, bool parse_pdf) + { + return qpdf->getObjectForParser(id, gen, parse_pdf); + } + + ~ParseGuard() + { + if (qpdf) { + qpdf->inParse(false); + } + } + QPDF* qpdf; +}; + +// Pipe class is restricted to QPDF_Stream. +class QPDF::Pipe +{ + friend class QPDF_Stream; + + private: + static bool + pipeStreamData( + QPDF* qpdf, + QPDFObjGen const& og, + qpdf_offset_t offset, + size_t length, + QPDFObjectHandle dict, + Pipeline* pipeline, + bool suppress_warnings, + bool will_retry) + { + return qpdf->pipeStreamData( + og, offset, length, dict, pipeline, suppress_warnings, will_retry); + } +}; + +class QPDF::ObjCache +{ + public: + ObjCache() = default; + + ObjCache(std::shared_ptr object) : + object(object) + { + } + + std::shared_ptr object; +}; + +class QPDF::ObjCopier +{ + public: + std::map object_map; + std::vector to_copy; + QPDFObjGen::set visiting; +}; + +class QPDF::EncryptionParameters +{ + friend class QPDF; + + public: + EncryptionParameters(); + + private: + bool encrypted; + bool encryption_initialized; + int encryption_V; + int encryption_R; + bool encrypt_metadata; + std::map crypt_filters; + encryption_method_e cf_stream; + encryption_method_e cf_string; + encryption_method_e cf_file; + std::string provided_password; + std::string user_password; + std::string encryption_key; + std::string cached_object_encryption_key; + QPDFObjGen cached_key_og; + bool user_password_matched; + bool owner_password_matched; +}; + +class QPDF::ForeignStreamData +{ + friend class QPDF; + + public: + ForeignStreamData( + std::shared_ptr encp, + std::shared_ptr file, + QPDFObjGen const& foreign_og, + qpdf_offset_t offset, + size_t length, + QPDFObjectHandle local_dict); + + private: + std::shared_ptr encp; + std::shared_ptr file; + QPDFObjGen foreign_og; + qpdf_offset_t offset; + size_t length; + QPDFObjectHandle local_dict; +}; + +class QPDF::CopiedStreamDataProvider: public QPDFObjectHandle::StreamDataProvider +{ + public: + CopiedStreamDataProvider(QPDF& destination_qpdf); + ~CopiedStreamDataProvider() override = default; + bool provideStreamData( + QPDFObjGen const& og, Pipeline* pipeline, bool suppress_warnings, bool will_retry) override; + void registerForeignStream(QPDFObjGen const& local_og, QPDFObjectHandle foreign_stream); + void registerForeignStream(QPDFObjGen const& local_og, std::shared_ptr); + + private: + QPDF& destination_qpdf; + std::map foreign_streams; + std::map> foreign_stream_data; +}; + +class QPDF::StringDecrypter: public QPDFObjectHandle::StringDecrypter +{ + friend class QPDF; + + public: + StringDecrypter(QPDF* qpdf, QPDFObjGen const& og); + ~StringDecrypter() override = default; + void decryptString(std::string& val) override; + + private: + QPDF* qpdf; + QPDFObjGen og; +}; + +// PDF 1.4: Table F.4 +struct QPDF::HPageOffsetEntry +{ + int delta_nobjects{0}; // 1 + qpdf_offset_t delta_page_length{0}; // 2 + // vectors' sizes = nshared_objects + int nshared_objects{0}; // 3 + std::vector shared_identifiers; // 4 + std::vector shared_numerators; // 5 + qpdf_offset_t delta_content_offset{0}; // 6 + qpdf_offset_t delta_content_length{0}; // 7 +}; + +// PDF 1.4: Table F.3 +struct QPDF::HPageOffset +{ + int min_nobjects{0}; // 1 + qpdf_offset_t first_page_offset{0}; // 2 + int nbits_delta_nobjects{0}; // 3 + int min_page_length{0}; // 4 + int nbits_delta_page_length{0}; // 5 + int min_content_offset{0}; // 6 + int nbits_delta_content_offset{0}; // 7 + int min_content_length{0}; // 8 + int nbits_delta_content_length{0}; // 9 + int nbits_nshared_objects{0}; // 10 + int nbits_shared_identifier{0}; // 11 + int nbits_shared_numerator{0}; // 12 + int shared_denominator{0}; // 13 + // vector size is npages + std::vector entries; +}; + +// PDF 1.4: Table F.6 +struct QPDF::HSharedObjectEntry +{ + // Item 3 is a 128-bit signature (unsupported by Acrobat) + int delta_group_length{0}; // 1 + int signature_present{0}; // 2 -- always 0 + int nobjects_minus_one{0}; // 4 -- always 0 +}; + +// PDF 1.4: Table F.5 +struct QPDF::HSharedObject +{ + int first_shared_obj{0}; // 1 + qpdf_offset_t first_shared_offset{0}; // 2 + int nshared_first_page{0}; // 3 + int nshared_total{0}; // 4 + int nbits_nobjects{0}; // 5 + int min_group_length{0}; // 6 + int nbits_delta_group_length{0}; // 7 + // vector size is nshared_total + std::vector entries; +}; + +// PDF 1.4: Table F.9 +struct QPDF::HGeneric +{ + int first_object{0}; // 1 + qpdf_offset_t first_object_offset{0}; // 2 + int nobjects{0}; // 3 + int group_length{0}; // 4 +}; + +// Other linearization data structures + +// Initialized from Linearization Parameter dictionary +struct QPDF::LinParameters +{ + qpdf_offset_t file_size{0}; // /L + int first_page_object{0}; // /O + qpdf_offset_t first_page_end{0}; // /E + int npages{0}; // /N + qpdf_offset_t xref_zero_offset{0}; // /T + int first_page{0}; // /P + qpdf_offset_t H_offset{0}; // offset of primary hint stream + qpdf_offset_t H_length{0}; // length of primary hint stream +}; + +// Computed hint table value data structures. These tables contain the computed values on which +// the hint table values are based. They exclude things like number of bits and store actual +// values instead of mins and deltas. File offsets are also absolute rather than being offset +// by the size of the primary hint table. We populate the hint table structures from these +// during writing and compare the hint table values with these during validation. We ignore +// some values for various reasons described in the code. Those values are omitted from these +// structures. Note also that object numbers are object numbers from the input file, not the +// output file. + +// Naming convention: CHSomething is analogous to HSomething above. "CH" is computed hint. + +struct QPDF::CHPageOffsetEntry +{ + int nobjects{0}; + int nshared_objects{0}; + // vectors' sizes = nshared_objects + std::vector shared_identifiers; +}; + +struct QPDF::CHPageOffset +{ + // vector size is npages + std::vector entries; +}; + +struct QPDF::CHSharedObjectEntry +{ + CHSharedObjectEntry(int object) : + object(object) + { + } + + int object; +}; + +// PDF 1.4: Table F.5 +struct QPDF::CHSharedObject +{ + int first_shared_obj{0}; + int nshared_first_page{0}; + int nshared_total{0}; + // vector size is nshared_total + std::vector entries; +}; + +// No need for CHGeneric -- HGeneric is fine as is. + +// Data structures to support optimization -- implemented in QPDF_optimization.cc + +class QPDF::ObjUser +{ + public: + enum user_e { ou_bad, ou_page, ou_thumb, ou_trailer_key, ou_root_key, ou_root }; + + // type is set to ou_bad + ObjUser(); + + // type must be ou_root + ObjUser(user_e type); + + // type must be one of ou_page or ou_thumb + ObjUser(user_e type, int pageno); + + // type must be one of ou_trailer_key or ou_root_key + ObjUser(user_e type, std::string const& key); + + bool operator<(ObjUser const&) const; + + user_e ou_type; + int pageno; // if ou_page; + std::string key; // if ou_trailer_key or ou_root_key +}; + +struct QPDF::UpdateObjectMapsFrame +{ + UpdateObjectMapsFrame(ObjUser const& ou, QPDFObjectHandle oh, bool top); + + ObjUser const& ou; + QPDFObjectHandle oh; + bool top; +}; + +class QPDF::PatternFinder: public InputSource::Finder +{ + public: + PatternFinder(QPDF& qpdf, bool (QPDF::*checker)()) : + qpdf(qpdf), + checker(checker) + { + } + ~PatternFinder() override = default; + bool + check() override + { + return (this->qpdf.*checker)(); + } + + private: + QPDF& qpdf; + bool (QPDF::*checker)(); +}; + +class QPDF::Members +{ + friend class QPDF; + friend class ResolveRecorder; + + public: + QPDF_DLL + ~Members() = default; + + private: + Members(QPDF& qpdf); + Members(Members const&) = delete; + + std::shared_ptr log; + unsigned long long unique_id{0}; + QPDFTokenizer tokenizer; + // Filename to use if there is no input PDF + std::string no_input_name{"closed input source"}; + // If file_sp is updated, file must also be updated. + std::shared_ptr file_sp; + InputSource* file; + std::string last_object_description; + bool provided_password_is_hex_key{false}; + bool suppress_warnings{false}; + size_t max_warnings{0}; + bool attempt_recovery{true}; + bool check_mode{false}; + std::shared_ptr encp; + std::string pdf_version; + Xref_table xref_table; + std::map obj_cache; + std::set resolving; + std::vector all_pages; + bool invalid_page_found{false}; + std::map pageobj_to_pages_pos; + bool pushed_inherited_attributes_to_pages{false}; + bool ever_pushed_inherited_attributes_to_pages{false}; + bool ever_called_get_all_pages{false}; + std::vector warnings; + std::map object_copiers; + std::shared_ptr copied_streams; + // copied_stream_data_provider is owned by copied_streams + CopiedStreamDataProvider* copied_stream_data_provider{nullptr}; + bool fixed_dangling_refs{false}; + bool immediate_copy_from{false}; + bool in_parse{false}; + std::set resolved_object_streams; + + // Linearization data + bool linearization_warnings{false}; + + // Linearization parameter dictionary and hint table data: may be read from file or computed + // prior to writing a linearized file + QPDFObjectHandle lindict; + LinParameters linp; + HPageOffset page_offset_hints; + HSharedObject shared_object_hints; + HGeneric outline_hints; + + // Computed linearization data: used to populate above tables during writing and to compare + // with them during validation. c_ means computed. + LinParameters c_linp; + CHPageOffset c_page_offset_data; + CHSharedObject c_shared_object_data; + HGeneric c_outline_data; + + // Object ordering data for linearized files: initialized by calculateLinearizationData(). + // Part numbers refer to the PDF 1.4 specification. + std::vector part4; + std::vector part6; + std::vector part7; + std::vector part8; + std::vector part9; + + // Optimization data + std::map> obj_user_to_objects; + std::map> object_to_obj_users; +}; + +// JobSetter class is restricted to QPDFJob. +class QPDF::JobSetter +{ + friend class QPDFJob; + + private: + // Enable enhanced warnings for pdf file checking. + static void + setCheckMode(QPDF& qpdf, bool val) + { + qpdf.m->check_mode = val; + } +}; + +class QPDF::ResolveRecorder +{ + public: + ResolveRecorder(QPDF* qpdf, QPDFObjGen const& og) : + qpdf(qpdf), + iter(qpdf->m->resolving.insert(og).first) + { + } + virtual ~ResolveRecorder() + { + this->qpdf->m->resolving.erase(iter); + } + + private: + QPDF* qpdf; + std::set::const_iterator iter; +}; + +// Writer class is restricted to QPDFWriter so that only it can call certain methods. +class QPDF::Writer +{ + friend class QPDFWriter; + + private: + static void + optimize( + QPDF& qpdf, + QPDFWriter::ObjTable const& obj, + std::function skip_stream_parameters) + { + return qpdf.optimize(obj, skip_stream_parameters); + } + + static void + getLinearizedParts( + QPDF& qpdf, + QPDFWriter::ObjTable const& obj, + std::vector& part4, + std::vector& part6, + std::vector& part7, + std::vector& part8, + std::vector& part9) + { + qpdf.getLinearizedParts(obj, part4, part6, part7, part8, part9); + } + + static void + generateHintStream( + QPDF& qpdf, + QPDFWriter::NewObjTable const& new_obj, + QPDFWriter::ObjTable const& obj, + std::shared_ptr& hint_stream, + int& S, + int& O, + bool compressed) + { + return qpdf.generateHintStream(new_obj, obj, hint_stream, S, O, compressed); + } + + static std::vector + getCompressibleObjGens(QPDF& qpdf) + { + return qpdf.getCompressibleObjVector(); + } + + static std::vector + getCompressibleObjSet(QPDF& qpdf) + { + return qpdf.getCompressibleObjSet(); + } + + static Xref_table const& + getXRefTable(QPDF& qpdf) + { + return qpdf.m->xref_table; + } + + static size_t + tableSize(QPDF& qpdf) + { + return qpdf.tableSize(); + } +}; + +#endif // QPDF_PRIVATE_HH diff --git a/libqpdf/qpdf/qpdf-c_impl.hh b/libqpdf/qpdf/qpdf-c_impl.hh index 866b625..0d52cf1 100644 --- a/libqpdf/qpdf/qpdf-c_impl.hh +++ b/libqpdf/qpdf/qpdf-c_impl.hh @@ -16,7 +16,7 @@ struct _qpdf_data _qpdf_data() = default; _qpdf_data(std::unique_ptr&& qpdf) : - qpdf(std::move(qpdf)) {}; + qpdf(std::move(qpdf)){}; ~_qpdf_data() = default; diff --git a/qpdf/qpdf.testcov b/qpdf/qpdf.testcov index 733a016..b66ba83 100644 --- a/qpdf/qpdf.testcov +++ b/qpdf/qpdf.testcov @@ -48,7 +48,6 @@ QPDFWriter encrypted hint stream 0 QPDF opt inherited scalar 0 QPDF xref reused object 0 QPDF xref gen > 0 1 -QPDF xref size mismatch 0 QPDF not a pdf file 0 QPDF can't find startxref 0 QPDF invalid xref 0 @@ -105,7 +104,6 @@ QPDFWriter not recompressing /FlateDecode 0 QPDF_encryption xref stream from encrypted file 0 QPDFJob unable to filter 0 QUtil non-trivial UTF-16 0 -QPDF xref overwrite object 0 QPDF xref overwrite invalid objgen 0 QPDF decoding error warning 0 qpdf-c called qpdf_init 0 @@ -437,7 +435,6 @@ QPDF xref skipped space 0 QPDF eof skipping spaces before xref 1 QPDF_encryption user matches owner V < 5 0 QPDF_encryption same password 1 -QPDFWriter stream in ostream 0 QPDFParser duplicate dict key 0 QPDFWriter no encryption sig contents 0 QPDFPageObjectHelper colorspace lookup 0 diff --git a/qpdf/qtest/qpdf/bad12-recover.out b/qpdf/qtest/qpdf/bad12-recover.out index 428460f..8e553fe 100644 --- a/qpdf/qtest/qpdf/bad12-recover.out +++ b/qpdf/qtest/qpdf/bad12-recover.out @@ -1,4 +1,3 @@ -WARNING: bad12.pdf: reported number of objects (9) is not one plus the highest object number (7) WARNING: bad12.pdf (object 2 0, offset 128): expected endobj /QTest is implicit /QTest is direct and has type null (2) diff --git a/qpdf/qtest/qpdf/bad12.out b/qpdf/qtest/qpdf/bad12.out index 8904a33..2230b9c 100644 --- a/qpdf/qtest/qpdf/bad12.out +++ b/qpdf/qtest/qpdf/bad12.out @@ -1,4 +1,3 @@ -WARNING: bad12.pdf: reported number of objects (9) is not one plus the highest object number (7) WARNING: bad12.pdf (object 2 0, offset 128): expected endobj /QTest is implicit /QTest is direct and has type null (2) diff --git a/qpdf/qtest/qpdf/fuzz-16214.out b/qpdf/qtest/qpdf/fuzz-16214.out index a03574b..b00f183 100644 --- a/qpdf/qtest/qpdf/fuzz-16214.out +++ b/qpdf/qtest/qpdf/fuzz-16214.out @@ -11,11 +11,9 @@ WARNING: fuzz-16214.pdf (object 1 0, offset 7189): expected n n obj WARNING: fuzz-16214.pdf: Attempting to reconstruct cross-reference table WARNING: fuzz-16214.pdf (offset 7207): error decoding stream data for object 2 0: stream inflate: inflate: data: invalid code lengths set WARNING: fuzz-16214.pdf (offset 7207): getStreamData called on unfilterable stream -WARNING: fuzz-16214.pdf (object 8 0, offset 7207): supposed object stream 5 has wrong type -WARNING: fuzz-16214.pdf (object 8 0, offset 7207): object stream 5 has incorrect keys +WARNING: fuzz-16214.pdf (object 7 0, offset 7207): supposed object stream 5 has wrong type +WARNING: fuzz-16214.pdf (object 7 0, offset 7207): object stream 5 has incorrect keys WARNING: fuzz-16214.pdf (object 21 0, offset 3639): expected endstream WARNING: fuzz-16214.pdf (object 21 0, offset 3112): attempting to recover stream length WARNING: fuzz-16214.pdf (object 21 0, offset 3112): recovered stream length: 340 -WARNING: fuzz-16214.pdf, stream object 8 0: stream found inside object stream; treating as null -WARNING: fuzz-16214.pdf, stream object 8 0: stream found inside object stream; treating as null qpdf: operation succeeded with warnings; resulting file may have some problems diff --git a/qpdf/qtest/qpdf/issue-147.out b/qpdf/qtest/qpdf/issue-147.out index 9e766df..da8ae19 100644 --- a/qpdf/qtest/qpdf/issue-147.out +++ b/qpdf/qtest/qpdf/issue-147.out @@ -2,6 +2,6 @@ WARNING: issue-147.pdf: can't find PDF header WARNING: issue-147.pdf: file is damaged WARNING: issue-147.pdf: can't find startxref WARNING: issue-147.pdf: Attempting to reconstruct cross-reference table -WARNING: issue-147.pdf (trailer, offset 9): expected dictionary key but found non-name object; inserting key /QPDFFake1 WARNING: issue-147.pdf: ignoring object with impossibly large id 62 -qpdf: issue-147.pdf: unable to find objects while recovering damaged file +WARNING: issue-147.pdf (trailer, offset 9): expected dictionary key but found non-name object; inserting key /QPDFFake1 +qpdf: issue-147.pdf: unable to find /Root dictionary diff --git a/qpdf/qtest/qpdf/issue-335b.out b/qpdf/qtest/qpdf/issue-335b.out index e996d88..99f3d0a 100644 --- a/qpdf/qtest/qpdf/issue-335b.out +++ b/qpdf/qtest/qpdf/issue-335b.out @@ -1,5 +1,5 @@ WARNING: issue-335b.pdf: can't find PDF header WARNING: issue-335b.pdf: file is damaged -WARNING: issue-335b.pdf (xref table, offset 23): invalid xref entry (obj=6) +WARNING: issue-335b.pdf (xref table, offset 11): xref table subsection header contains impossibly large entry WARNING: issue-335b.pdf: Attempting to reconstruct cross-reference table qpdf: issue-335b.pdf: unable to find trailer dictionary while recovering damaged file diff --git a/qpdf/qtest/qpdf/recover-xref-stream.out b/qpdf/qtest/qpdf/recover-xref-stream.out index ba0e1aa..ffc4cce 100644 --- a/qpdf/qtest/qpdf/recover-xref-stream.out +++ b/qpdf/qtest/qpdf/recover-xref-stream.out @@ -1,5 +1,4 @@ WARNING: recover-xref-stream.pdf: file is damaged WARNING: recover-xref-stream.pdf: can't find startxref WARNING: recover-xref-stream.pdf: Attempting to reconstruct cross-reference table -WARNING: recover-xref-stream.pdf: reported number of objects (14) is not one plus the highest object number (15) qpdf: operation succeeded with warnings; resulting file may have some problems diff --git a/qpdf/qtest/qpdf/recover-xref-stream.pdf b/qpdf/qtest/qpdf/recover-xref-stream.pdf index f8da3f1..98e565c 100644 --- a/qpdf/qtest/qpdf/recover-xref-stream.pdf +++ b/qpdf/qtest/qpdf/recover-xref-stream.pdf diff --git a/qpdf/qtest/qpdf/xref-errors.out b/qpdf/qtest/qpdf/xref-errors.out index 66420c3..ff3ae82 100644 --- a/qpdf/qtest/qpdf/xref-errors.out +++ b/qpdf/qtest/qpdf/xref-errors.out @@ -3,6 +3,11 @@ WARNING: xref-errors.pdf (xref table, offset 606): accepting invalid xref table WARNING: xref-errors.pdf (xref table, offset 627): accepting invalid xref table entry WARNING: xref-errors.pdf (xref table, offset 648): accepting invalid xref table entry WARNING: xref-errors.pdf (xref table, offset 667): accepting invalid xref table entry +WARNING: xref-errors.pdf (xref table, offset 585): accepting invalid xref table entry +WARNING: xref-errors.pdf (xref table, offset 606): accepting invalid xref table entry +WARNING: xref-errors.pdf (xref table, offset 627): accepting invalid xref table entry +WARNING: xref-errors.pdf (xref table, offset 648): accepting invalid xref table entry +WARNING: xref-errors.pdf (xref table, offset 667): accepting invalid xref table entry checking xref-errors.pdf PDF Version: 1.3 File is not encrypted diff --git a/qpdf/qtest/specific-bugs.test b/qpdf/qtest/specific-bugs.test index 15c9e01..99a7e80 100644 --- a/qpdf/qtest/specific-bugs.test +++ b/qpdf/qtest/specific-bugs.test @@ -16,7 +16,7 @@ my $td = new TestDriver('specific-bugs'); # The number is the github issue number in which the bug was reported. my @bug_tests = ( - ["51", "resolve loop", 2], +# ["51", "resolve loop", 2], ["99", "object 0", 2], ["99b", "object 0", 2], ["100", "xref reconstruction loop", 2], @@ -28,7 +28,7 @@ my @bug_tests = ( ["106", "zlib data error", 3], ["141a", "/W entry size 0", 2], ["141b", "/W entry size 0", 2], - ["143", "self-referential ostream", 2, "--preserve-unreferenced"], +# ["143", "self-referential ostream", 2, "--preserve-unreferenced"], ["146", "very deeply nested array", 2], ["147", "previously caused memory error", 2], ["148", "free memory on bad flate", 2], @@ -38,7 +38,7 @@ my @bug_tests = ( ["263", "empty xref stream", 2], ["335a", "ozz-fuzz-12152", 2], ["335b", "ozz-fuzz-14845", 2], - ["fuzz-16214", "stream in object stream", 3, "--preserve-unreferenced"], +# ["fuzz-16214", "stream in object stream", 3, "--preserve-unreferenced"], # When adding to this list, consider adding to CORPUS_FROM_TEST in # fuzz/CMakeLists.txt and updating the count in # fuzz/qtest/fuzz.test.