diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml index 4c77ebf..cc5fe12 100644 --- a/.idea/codeStyles/Project.xml +++ b/.idea/codeStyles/Project.xml @@ -3,7 +3,7 @@ - + diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 848f7a2..8d00138 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -2,7 +2,6 @@ # Read the Docs configuration file # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details -# Required version: 2 build: @@ -11,13 +10,11 @@ build: python: "3.11" sphinx: - configuration: manual/conf.py + configuration: manual/conf.py + fail_on_warning: true formats: all -sphinx: - fail_on_warning: true - python: install: - requirements: manual/requirements.txt diff --git a/fuzz/CMakeLists.txt b/fuzz/CMakeLists.txt index 6b3cccf..029ed05 100644 --- a/fuzz/CMakeLists.txt +++ b/fuzz/CMakeLists.txt @@ -142,9 +142,6 @@ set(CORPUS_OTHER 70306b.fuzz 71624.fuzz 71689.fuzz - 99999a.fuzz - 99999b.fuzz - 99999c.fuzz 99999d.fuzz 99999e.fuzz 369662293.fuzz diff --git a/fuzz/qpdf_extra/99999a.fuzz b/fuzz/qpdf_extra/99999a.fuzz deleted file mode 100644 index 026c742..0000000 --- a/fuzz/qpdf_extra/99999a.fuzz +++ /dev/null @@ -1,63 +0,0 @@ -%PDF-1.5 -%€€€€ -1 0 obj -<< - /Type /Catalog - /Pages 2 0 R ->> -endobj -2 0 obj -<< - /Count 6 Ri - 0K/ds [3 0 R] - /Type /Pages ->> -endobj -3 0 obj -<< - /Resources << - /Font << - /F1 5 0 R - >> - >> - /MediaBox [0 0 795 842] - /Parent 2 0 R - /Contents 4 0 R - /Type /Page -=> -endobj -4 0 obj -<<444444444444444444444444 1 Tr /F1 30 Tf 350 750 Td (foobar) Tj ET -endstream -endobj -5 0 obj -<< - /Name /F1 - /BaseFont /Helvetica - /Type /Font - /Subtype /Type1 ->> -e„dobj -6 0 obj -<< /Length 6 0 R >> -stre444444444444444444444444444444<<>> -endobj -xref -0 8 -0000000000 65535 f -0000000015 00000 n -0000000066 00000 n -0000000130 00000 n -0000000269 00000 n -0000000362 00000 n -000000ÎËËÉßÏÏÏ00 n -0000000500 00000 n -trailer -<< - /Size 713115528178535 - /Root 1 0 R - /Info 7 0 R ->> -startxref -520 -%%EOF \ No newline at end of file diff --git a/fuzz/qpdf_extra/99999b.fuzz b/fuzz/qpdf_extra/99999b.fuzz deleted file mode 100644 index 288a6b5..0000000 --- a/fuzz/qpdf_extra/99999b.fuzz +++ /dev/null diff --git a/fuzz/qpdf_extra/99999c.fuzz b/fuzz/qpdf_extra/99999c.fuzz deleted file mode 100644 index c856648..0000000 --- a/fuzz/qpdf_extra/99999c.fuzz +++ /dev/null diff --git a/fuzz/qtest/fuzz.test b/fuzz/qtest/fuzz.test index efffdc6..cf38a63 100644 --- a/fuzz/qtest/fuzz.test +++ b/fuzz/qtest/fuzz.test @@ -11,7 +11,7 @@ my $td = new TestDriver('fuzz'); my $qpdf_corpus = $ENV{'QPDF_FUZZ_CORPUS'} || die "must set QPDF_FUZZ_CORPUS"; -my $n_qpdf_files = 87; # increment when adding new files +my $n_qpdf_files = 84; # increment when adding new files my @fuzzers = ( ['ascii85' => 1], diff --git a/include/qpdf/QPDF.hh b/include/qpdf/QPDF.hh index 756b22d..11bc807 100644 --- a/include/qpdf/QPDF.hh +++ b/include/qpdf/QPDF.hh @@ -391,7 +391,7 @@ class QPDF void replaceObject(int objid, int generation, QPDFObjectHandle); // Swap two objects given by ID. Prior to qpdf 10.2.1, existing QPDFObjectHandle instances that - // reference the objects did not notice the swap, but this was fixed in 10.2.1. + // reference them objects not notice the swap, but this was fixed in 10.2.1. QPDF_DLL void swapObjects(QPDFObjGen const& og1, QPDFObjGen const& og2); QPDF_DLL @@ -645,7 +645,7 @@ class QPDF QPDF_DLL void fixDanglingReferences(bool force = false); - // Return the approximate number of indirect objects. It is approximate because not all objects + // Return the approximate number of indirect objects. It is/ approximate because not all objects // in the file are preserved in all cases, and gaps in object numbering are not preserved. QPDF_DLL size_t getObjectCount(); @@ -725,15 +725,165 @@ class QPDF void removePage(QPDFObjectHandle page); // End legacy page helpers - // End of the public API. The following classes and methods are for qpdf internal use only. + // Writer class is restricted to QPDFWriter so that only it can call certain methods. + class Writer + { + friend class QPDFWriter; + + private: + static void + optimize( + QPDF& qpdf, + QPDFWriter::ObjTable const& obj, + std::function skip_stream_parameters) + { + return qpdf.optimize(obj, skip_stream_parameters); + } + + static void + getLinearizedParts( + QPDF& qpdf, + QPDFWriter::ObjTable const& obj, + std::vector& part4, + std::vector& part6, + std::vector& part7, + std::vector& part8, + std::vector& part9) + { + qpdf.getLinearizedParts(obj, part4, part6, part7, part8, part9); + } + + static void + generateHintStream( + QPDF& qpdf, + QPDFWriter::NewObjTable const& new_obj, + QPDFWriter::ObjTable const& obj, + std::shared_ptr& hint_stream, + int& S, + int& O, + bool compressed) + { + return qpdf.generateHintStream(new_obj, obj, hint_stream, S, O, compressed); + } + + static std::vector + getCompressibleObjGens(QPDF& qpdf) + { + return qpdf.getCompressibleObjVector(); + } + + static std::vector + getCompressibleObjSet(QPDF& qpdf) + { + return qpdf.getCompressibleObjSet(); + } + + static std::map const& + getXRefTable(QPDF& qpdf) + { + return qpdf.getXRefTableInternal(); + } + + static size_t + tableSize(QPDF& qpdf) + { + return qpdf.tableSize(); + } + }; - class Writer; - class Resolver; - class StreamCopier; - class Objects; - class ParseGuard; - class Pipe; - class JobSetter; + // The Resolver class is restricted to QPDFObject so that only it can resolve indirect + // references. + class Resolver + { + friend class QPDFObject; + friend class QPDF_Unresolved; + + private: + static QPDFObject* + resolved(QPDF* qpdf, QPDFObjGen og) + { + return qpdf->resolve(og); + } + }; + + // StreamCopier class is restricted to QPDFObjectHandle so it can copy stream data. + class StreamCopier + { + friend class QPDFObjectHandle; + + private: + static void + copyStreamData(QPDF* qpdf, QPDFObjectHandle const& dest, QPDFObjectHandle const& src) + { + qpdf->copyStreamData(dest, src); + } + }; + + // The ParseGuard class allows QPDFParser to detect re-entrant parsing. It also provides + // special access to allow the parser to create unresolved objects and dangling references. + class ParseGuard + { + friend class QPDFParser; + + private: + ParseGuard(QPDF* qpdf) : + qpdf(qpdf) + { + if (qpdf) { + qpdf->inParse(true); + } + } + + static std::shared_ptr + getObject(QPDF* qpdf, int id, int gen, bool parse_pdf) + { + return qpdf->getObjectForParser(id, gen, parse_pdf); + } + + ~ParseGuard() + { + if (qpdf) { + qpdf->inParse(false); + } + } + QPDF* qpdf; + }; + + // Pipe class is restricted to QPDF_Stream. + class Pipe + { + friend class QPDF_Stream; + + private: + static bool + pipeStreamData( + QPDF* qpdf, + QPDFObjGen const& og, + qpdf_offset_t offset, + size_t length, + QPDFObjectHandle dict, + Pipeline* pipeline, + bool suppress_warnings, + bool will_retry) + { + return qpdf->pipeStreamData( + og, offset, length, dict, pipeline, suppress_warnings, will_retry); + } + }; + + // JobSetter class is restricted to QPDFJob. + class JobSetter + { + friend class QPDFJob; + + private: + // Enable enhanced warnings for pdf file checking. + static void + setCheckMode(QPDF& qpdf, bool val) + { + qpdf.m->check_mode = val; + } + }; // For testing only -- do not add to DLL static bool test_json_validators(); @@ -748,23 +898,194 @@ class QPDF static std::string const qpdf_version; - class ObjCopier; - class EncryptionParameters; - class ForeignStreamData; - class CopiedStreamDataProvider; - class StringDecrypter; - class ResolveRecorder; + class ObjCache + { + public: + ObjCache() : + end_before_space(0), + end_after_space(0) + { + } + ObjCache( + std::shared_ptr object, + qpdf_offset_t end_before_space = 0, + qpdf_offset_t end_after_space = 0) : + object(object), + end_before_space(end_before_space), + end_after_space(end_after_space) + { + } + + std::shared_ptr object; + qpdf_offset_t end_before_space; + qpdf_offset_t end_after_space; + }; + + class ObjCopier + { + public: + std::map object_map; + std::vector to_copy; + QPDFObjGen::set visiting; + }; + + class EncryptionParameters + { + friend class QPDF; + + public: + EncryptionParameters(); + + private: + bool encrypted; + bool encryption_initialized; + int encryption_V; + int encryption_R; + bool encrypt_metadata; + std::map crypt_filters; + encryption_method_e cf_stream; + encryption_method_e cf_string; + encryption_method_e cf_file; + std::string provided_password; + std::string user_password; + std::string encryption_key; + std::string cached_object_encryption_key; + QPDFObjGen cached_key_og; + bool user_password_matched; + bool owner_password_matched; + }; + + class ForeignStreamData + { + friend class QPDF; + + public: + ForeignStreamData( + std::shared_ptr encp, + std::shared_ptr file, + QPDFObjGen const& foreign_og, + qpdf_offset_t offset, + size_t length, + QPDFObjectHandle local_dict); + + private: + std::shared_ptr encp; + std::shared_ptr file; + QPDFObjGen foreign_og; + qpdf_offset_t offset; + size_t length; + QPDFObjectHandle local_dict; + }; + + class CopiedStreamDataProvider: public QPDFObjectHandle::StreamDataProvider + { + public: + CopiedStreamDataProvider(QPDF& destination_qpdf); + ~CopiedStreamDataProvider() override = default; + bool provideStreamData( + QPDFObjGen const& og, + Pipeline* pipeline, + bool suppress_warnings, + bool will_retry) override; + void registerForeignStream(QPDFObjGen const& local_og, QPDFObjectHandle foreign_stream); + void registerForeignStream(QPDFObjGen const& local_og, std::shared_ptr); + + private: + QPDF& destination_qpdf; + std::map foreign_streams; + std::map> foreign_stream_data; + }; + + class StringDecrypter: public QPDFObjectHandle::StringDecrypter + { + friend class QPDF; + + public: + StringDecrypter(QPDF* qpdf, QPDFObjGen const& og); + ~StringDecrypter() override = default; + void decryptString(std::string& val) override; + + private: + QPDF* qpdf; + QPDFObjGen og; + }; + + class ResolveRecorder + { + public: + ResolveRecorder(QPDF* qpdf, QPDFObjGen const& og) : + qpdf(qpdf), + iter(qpdf->m->resolving.insert(og).first) + { + } + virtual ~ResolveRecorder() + { + this->qpdf->m->resolving.erase(iter); + } + + private: + QPDF* qpdf; + std::set::const_iterator iter; + }; + class JSONReactor; - inline Objects& objects() noexcept; - inline Objects const& objects() const noexcept; void parse(char const* password); void inParse(bool); + void setTrailer(QPDFObjectHandle obj); + void read_xref(qpdf_offset_t offset); + bool resolveXRefTable(); + void reconstruct_xref(QPDFExc& e); + bool parse_xrefFirst(std::string const& line, int& obj, int& num, int& bytes); + bool read_xrefEntry(qpdf_offset_t& f1, int& f2, char& type); + bool read_bad_xrefEntry(qpdf_offset_t& f1, int& f2, char& type); + qpdf_offset_t read_xrefTable(qpdf_offset_t offset); + qpdf_offset_t read_xrefStream(qpdf_offset_t offset); + qpdf_offset_t processXRefStream(qpdf_offset_t offset, QPDFObjectHandle& xref_stream); + std::pair> + processXRefW(QPDFObjectHandle& dict, std::function damaged); + int processXRefSize( + QPDFObjectHandle& dict, int entry_size, std::function damaged); + std::pair>> processXRefIndex( + QPDFObjectHandle& dict, + int max_num_entries, + std::function damaged); + void insertXrefEntry(int obj, int f0, qpdf_offset_t f1, int f2); + void insertFreeXrefEntry(QPDFObjGen); + void insertReconstructedXrefEntry(int obj, qpdf_offset_t f1, int f2); void setLastObjectDescription(std::string const& description, QPDFObjGen const& og); + QPDFObjectHandle readTrailer(); + QPDFObjectHandle readObject(std::string const& description, QPDFObjGen og); + void readStream(QPDFObjectHandle& object, QPDFObjGen og, qpdf_offset_t offset); + void validateStreamLineEnd(QPDFObjectHandle& object, QPDFObjGen og, qpdf_offset_t offset); + QPDFObjectHandle readObjectInStream(std::shared_ptr& input, int obj); + size_t recoverStreamLength( + std::shared_ptr input, QPDFObjGen const& og, qpdf_offset_t stream_offset); QPDFTokenizer::Token readToken(InputSource&, size_t max_len = 0); + QPDFObjectHandle readObjectAtOffset( + bool attempt_recovery, + qpdf_offset_t offset, + std::string const& description, + QPDFObjGen exp_og, + QPDFObjGen& og, + bool skip_cache_if_in_xref); + QPDFObject* resolve(QPDFObjGen og); + void resolveObjectsInStream(int obj_stream_number); void stopOnError(std::string const& message); + QPDFObjGen nextObjGen(); QPDFObjectHandle newIndirect(QPDFObjGen const&, std::shared_ptr const&); + QPDFObjectHandle makeIndirectFromQPDFObject(std::shared_ptr const& obj); + bool isCached(QPDFObjGen const& og); + bool isUnresolved(QPDFObjGen const& og); + std::shared_ptr getObjectForParser(int id, int gen, bool parse_pdf); + std::shared_ptr getObjectForJSON(int id, int gen); + void removeObject(QPDFObjGen og); + void updateCache( + QPDFObjGen const& og, + std::shared_ptr const& object, + qpdf_offset_t end_before_space, + qpdf_offset_t end_after_space); static QPDFExc damagedPDF( InputSource& input, std::string const& object, @@ -801,6 +1122,7 @@ class QPDF // For QPDFWriter: + std::map const& getXRefTableInternal(); template void optimize_internal( T const& object_stream_data, @@ -809,7 +1131,7 @@ class QPDF void optimize( QPDFWriter::ObjTable const& obj, std::function skip_stream_parameters); - void optimize(Objects const& obj); + size_t tableSize(); // Get lists of all objects in order according to the part of a linearized file that they belong // to. @@ -829,6 +1151,12 @@ class QPDF int& O, bool compressed); + // Get a list of objects that would be permitted in an object stream. + template + std::vector getCompressibleObjGens(); + std::vector getCompressibleObjVector(); + std::vector getCompressibleObjSet(); + // methods to support page handling void getAllPagesInternal( @@ -868,19 +1196,200 @@ class QPDF replaceForeignIndirectObjects(QPDFObjectHandle foreign, ObjCopier& obj_copier, bool top); void copyStreamData(QPDFObjectHandle dest_stream, QPDFObjectHandle src_stream); - struct HPageOffsetEntry; - struct HPageOffset; - struct HSharedObjectEntry; - struct HSharedObject; - struct HGeneric; - struct LinParameters; - struct CHPageOffsetEntry; - struct CHPageOffset; - struct CHSharedObjectEntry; - struct CHSharedObject; - class ObjUser; - struct UpdateObjectMapsFrame; - class PatternFinder; + // Linearization Hint table structures. + // Naming conventions: + + // HSomething is the Something Hint Table or table header + // HSomethingEntry is an entry in the Something table + + // delta_something + min_something = something + // nbits_something = number of bits required for something + + // something_offset is the pre-adjusted offset in the file. If >= + // H0_offset, H0_length must be added to get an actual file + // offset. + + // PDF 1.4: Table F.4 + struct HPageOffsetEntry + { + int delta_nobjects{0}; // 1 + qpdf_offset_t delta_page_length{0}; // 2 + // vectors' sizes = nshared_objects + int nshared_objects{0}; // 3 + std::vector shared_identifiers; // 4 + std::vector shared_numerators; // 5 + qpdf_offset_t delta_content_offset{0}; // 6 + qpdf_offset_t delta_content_length{0}; // 7 + }; + + // PDF 1.4: Table F.3 + struct HPageOffset + { + int min_nobjects{0}; // 1 + qpdf_offset_t first_page_offset{0}; // 2 + int nbits_delta_nobjects{0}; // 3 + int min_page_length{0}; // 4 + int nbits_delta_page_length{0}; // 5 + int min_content_offset{0}; // 6 + int nbits_delta_content_offset{0}; // 7 + int min_content_length{0}; // 8 + int nbits_delta_content_length{0}; // 9 + int nbits_nshared_objects{0}; // 10 + int nbits_shared_identifier{0}; // 11 + int nbits_shared_numerator{0}; // 12 + int shared_denominator{0}; // 13 + // vector size is npages + std::vector entries; + }; + + // PDF 1.4: Table F.6 + struct HSharedObjectEntry + { + // Item 3 is a 128-bit signature (unsupported by Acrobat) + int delta_group_length{0}; // 1 + int signature_present{0}; // 2 -- always 0 + int nobjects_minus_one{0}; // 4 -- always 0 + }; + + // PDF 1.4: Table F.5 + struct HSharedObject + { + int first_shared_obj{0}; // 1 + qpdf_offset_t first_shared_offset{0}; // 2 + int nshared_first_page{0}; // 3 + int nshared_total{0}; // 4 + int nbits_nobjects{0}; // 5 + int min_group_length{0}; // 6 + int nbits_delta_group_length{0}; // 7 + // vector size is nshared_total + std::vector entries; + }; + + // PDF 1.4: Table F.9 + struct HGeneric + { + int first_object{0}; // 1 + qpdf_offset_t first_object_offset{0}; // 2 + int nobjects{0}; // 3 + int group_length{0}; // 4 + }; + + // Other linearization data structures + + // Initialized from Linearization Parameter dictionary + struct LinParameters + { + qpdf_offset_t file_size{0}; // /L + int first_page_object{0}; // /O + qpdf_offset_t first_page_end{0}; // /E + int npages{0}; // /N + qpdf_offset_t xref_zero_offset{0}; // /T + int first_page{0}; // /P + qpdf_offset_t H_offset{0}; // offset of primary hint stream + qpdf_offset_t H_length{0}; // length of primary hint stream + }; + + // Computed hint table value data structures. These tables contain the computed values on which + // the hint table values are based. They exclude things like number of bits and store actual + // values instead of mins and deltas. File offsets are also absolute rather than being offset + // by the size of the primary hint table. We populate the hint table structures from these + // during writing and compare the hint table values with these during validation. We ignore + // some values for various reasons described in the code. Those values are omitted from these + // structures. Note also that object numbers are object numbers from the input file, not the + // output file. + + // Naming convention: CHSomething is analogous to HSomething above. "CH" is computed hint. + + struct CHPageOffsetEntry + { + int nobjects{0}; + int nshared_objects{0}; + // vectors' sizes = nshared_objects + std::vector shared_identifiers; + }; + + struct CHPageOffset + { + // vector size is npages + std::vector entries; + }; + + struct CHSharedObjectEntry + { + CHSharedObjectEntry(int object) : + object(object) + { + } + + int object; + }; + + // PDF 1.4: Table F.5 + struct CHSharedObject + { + int first_shared_obj{0}; + int nshared_first_page{0}; + int nshared_total{0}; + // vector size is nshared_total + std::vector entries; + }; + + // No need for CHGeneric -- HGeneric is fine as is. + + // Data structures to support optimization -- implemented in QPDF_optimization.cc + + class ObjUser + { + public: + enum user_e { ou_bad, ou_page, ou_thumb, ou_trailer_key, ou_root_key, ou_root }; + + // type is set to ou_bad + ObjUser(); + + // type must be ou_root + ObjUser(user_e type); + + // type must be one of ou_page or ou_thumb + ObjUser(user_e type, int pageno); + + // type must be one of ou_trailer_key or ou_root_key + ObjUser(user_e type, std::string const& key); + + bool operator<(ObjUser const&) const; + + user_e ou_type; + int pageno; // if ou_page; + std::string key; // if ou_trailer_key or ou_root_key + }; + + struct UpdateObjectMapsFrame + { + UpdateObjectMapsFrame(ObjUser const& ou, QPDFObjectHandle oh, bool top); + + ObjUser const& ou; + QPDFObjectHandle oh; + bool top; + }; + + class PatternFinder: public InputSource::Finder + { + public: + PatternFinder(QPDF& qpdf, bool (QPDF::*checker)()) : + qpdf(qpdf), + checker(checker) + { + } + ~PatternFinder() override = default; + bool + check() override + { + return (this->qpdf.*checker)(); + } + + private: + QPDF& qpdf; + bool (QPDF::*checker)(); + }; // Methods to support pattern finding static bool validatePDFVersion(char const*&, std::string& version); @@ -902,7 +1411,6 @@ class QPDF QPDFObjectHandle getUncompressedObject(QPDFObjectHandle&, std::map const& object_stream_data); QPDFObjectHandle getUncompressedObject(QPDFObjectHandle&, QPDFWriter::ObjTable const& obj); - QPDFObjectHandle getUncompressedObject(QPDFObjectHandle&, Objects const& obj); int lengthNextN(int first_object, int n); void checkHPageOffset(std::vector const& pages, std::map& idx_to_obj); @@ -948,7 +1456,6 @@ class QPDF std::function skip_stream_parameters); void filterCompressedObjects(std::map const& object_stream_data); void filterCompressedObjects(QPDFWriter::ObjTable const& object_stream_data); - void filterCompressedObjects(Objects const& object_stream_data); // JSON import void importJSON(std::shared_ptr, bool must_be_complete); @@ -979,7 +1486,90 @@ class QPDF return QIntC::to_ulonglong(i); } - class Members; + class Members + { + friend class QPDF; + friend class ResolveRecorder; + + public: + QPDF_DLL + ~Members() = default; + + private: + Members(); + Members(Members const&) = delete; + + std::shared_ptr log; + unsigned long long unique_id{0}; + QPDFTokenizer tokenizer; + std::shared_ptr file; + std::string last_object_description; + bool provided_password_is_hex_key{false}; + bool ignore_xref_streams{false}; + bool suppress_warnings{false}; + size_t max_warnings{0}; + bool attempt_recovery{true}; + bool check_mode{false}; + std::shared_ptr encp; + std::string pdf_version; + std::map xref_table; + // Various tables are indexed by object id, with potential size id + 1 + int xref_table_max_id{std::numeric_limits::max() - 1}; + qpdf_offset_t xref_table_max_offset{0}; + std::set deleted_objects; + std::map obj_cache; + std::set resolving; + QPDFObjectHandle trailer; + std::vector all_pages; + bool invalid_page_found{false}; + std::map pageobj_to_pages_pos; + bool pushed_inherited_attributes_to_pages{false}; + bool ever_pushed_inherited_attributes_to_pages{false}; + bool ever_called_get_all_pages{false}; + std::vector warnings; + std::map object_copiers; + std::shared_ptr copied_streams; + // copied_stream_data_provider is owned by copied_streams + CopiedStreamDataProvider* copied_stream_data_provider{nullptr}; + bool reconstructed_xref{false}; + bool fixed_dangling_refs{false}; + bool immediate_copy_from{false}; + bool in_parse{false}; + bool parsed{false}; + std::set resolved_object_streams; + + // Linearization data + qpdf_offset_t first_xref_item_offset{0}; // actual value from file + bool uncompressed_after_compressed{false}; + bool linearization_warnings{false}; + + // Linearization parameter dictionary and hint table data: may be read from file or computed + // prior to writing a linearized file + QPDFObjectHandle lindict; + LinParameters linp; + HPageOffset page_offset_hints; + HSharedObject shared_object_hints; + HGeneric outline_hints; + + // Computed linearization data: used to populate above tables during writing and to compare + // with them during validation. c_ means computed. + LinParameters c_linp; + CHPageOffset c_page_offset_data; + CHSharedObject c_shared_object_data; + HGeneric c_outline_data; + + // Object ordering data for linearized files: initialized by calculateLinearizationData(). + // Part numbers refer to the PDF 1.4 specification. + std::vector part4; + std::vector part6; + std::vector part7; + std::vector part8; + std::vector part9; + + // Optimization data + std::map> obj_user_to_objects; + std::map> object_to_obj_users; + }; // Keep all member variables inside the Members object, which we dynamically allocate. This // makes it possible to add new private members without breaking binary compatibility. diff --git a/libqpdf/CMakeLists.txt b/libqpdf/CMakeLists.txt index 189e506..eb30b62 100644 --- a/libqpdf/CMakeLists.txt +++ b/libqpdf/CMakeLists.txt @@ -107,7 +107,6 @@ set(libqpdf_SOURCES QPDF_encryption.cc QPDF_json.cc QPDF_linearization.cc - QPDF_objects.cc QPDF_optimization.cc QPDF_pages.cc QTC.cc diff --git a/libqpdf/QPDF.cc b/libqpdf/QPDF.cc index cc54db2..03ffb62 100644 --- a/libqpdf/QPDF.cc +++ b/libqpdf/QPDF.cc @@ -1,9 +1,11 @@ #include // include first for large file support -#include +#include +#include #include #include +#include #include #include #include @@ -30,51 +32,67 @@ // being static as well. std::string const QPDF::qpdf_version(QPDF_VERSION); +static char const* EMPTY_PDF = ( + // force line break + "%PDF-1.3\n" + "1 0 obj\n" + "<< /Type /Catalog /Pages 2 0 R >>\n" + "endobj\n" + "2 0 obj\n" + "<< /Type /Pages /Kids [] /Count 0 >>\n" + "endobj\n" + "xref\n" + "0 3\n" + "0000000000 65535 f \n" + "0000000009 00000 n \n" + "0000000058 00000 n \n" + "trailer << /Size 3 /Root 1 0 R >>\n" + "startxref\n" + "110\n" + "%%EOF\n"); + namespace { - class InvalidInputSource final: public InputSource + class InvalidInputSource: public InputSource { public: - InvalidInputSource(std::string const& name) : - name(name) - { - } - ~InvalidInputSource() final = default; + ~InvalidInputSource() override = default; qpdf_offset_t - findAndSkipNextEOL() final + findAndSkipNextEOL() override { throwException(); return 0; } std::string const& - getName() const final + getName() const override { + static std::string name("closed input source"); return name; } qpdf_offset_t - tell() final + tell() override { throwException(); return 0; } void - seek(qpdf_offset_t offset, int whence) final + seek(qpdf_offset_t offset, int whence) override { throwException(); } void - rewind() final + rewind() override { throwException(); } size_t - read(char* buffer, size_t length) final + read(char* buffer, size_t length) override { throwException(); return 0; } void - unreadCh(char ch) final + unreadCh(char ch) override { throwException(); } @@ -87,8 +105,6 @@ namespace "source. QPDF operations are invalid before processFile (or " "another process method) or after closeInputSource"); } - - std::string const& name; }; } // namespace @@ -180,17 +196,15 @@ QPDF::EncryptionParameters::EncryptionParameters() : { } -QPDF::Members::Members(QPDF& qpdf) : +QPDF::Members::Members() : log(QPDFLogger::defaultLogger()), - file_sp(new InvalidInputSource(no_input_name)), - file(file_sp.get()), - encp(new EncryptionParameters), - objects(qpdf, this, file) + file(new InvalidInputSource()), + encp(new EncryptionParameters) { } QPDF::QPDF() : - m(new Members(*this)) + m(new Members()) { m->tokenizer.allowEOF(); // Generate a unique ID. It just has to be unique among all QPDF objects allocated throughout @@ -199,7 +213,28 @@ QPDF::QPDF() : m->unique_id = unique_id.fetch_add(1ULL); } -QPDF::~QPDF() = default; +QPDF::~QPDF() +{ + // If two objects are mutually referential (through each object having an array or dictionary + // that contains an indirect reference to the other), the circular references in the + // std::shared_ptr objects will prevent the objects from being deleted. Walk through all objects + // in the object cache, which is those objects that we read from the file, and break all + // resolved indirect references by replacing them with an internal object type representing that + // they have been destroyed. Note that we can't break references like this at any time when the + // QPDF object is active. The call to reset also causes all direct QPDFObjectHandle objects that + // are reachable from this object to release their association with this QPDF. Direct objects + // are not destroyed since they can be moved to other QPDF objects safely. + + // At this point, obviously no one is still using the QPDF object, but we'll explicitly clear + // the xref table anyway just to prevent any possibility of resolve() succeeding. + m->xref_table.clear(); + for (auto const& iter: m->obj_cache) { + iter.second.object->disconnect(); + if (iter.second.object->getTypeCode() != ::ot_null) { + iter.second.object->destroy(); + } + } +} std::shared_ptr QPDF::create() @@ -236,17 +271,14 @@ QPDF::processMemoryFile( void QPDF::processInputSource(std::shared_ptr source, char const* password) { - m->file_sp = source; - m->file = source.get(); + m->file = source; parse(password); } void QPDF::closeInputSource() { - m->no_input_name = "closed input source"; - m->file_sp = std::shared_ptr(new InvalidInputSource(m->no_input_name)); - m->file = m->file_sp.get(); + m->file = std::shared_ptr(new InvalidInputSource()); } void @@ -258,9 +290,7 @@ QPDF::setPasswordIsHexKey(bool val) void QPDF::emptyPDF() { - m->pdf_version = "1.3"; - m->no_input_name = "empty PDF"; - m->objects.xref_table().initialize_empty(); + processMemoryFile("empty PDF", EMPTY_PDF, strlen(EMPTY_PDF)); } void @@ -273,7 +303,7 @@ QPDF::registerStreamFilter( void QPDF::setIgnoreXRefStreams(bool val) { - m->objects.xref_table().ignore_streams(val); + m->ignore_xref_streams = val; } std::shared_ptr @@ -311,7 +341,6 @@ void QPDF::setAttemptRecovery(bool val) { m->attempt_recovery = val; - m->objects.xref_table().attempt_recovery(val); } void @@ -381,14 +410,23 @@ QPDF::findHeader() // PDF header, all explicit offsets in the file are such that 0 points to the beginning // of the header. QTC::TC("qpdf", "QPDF global offset"); - m->file_sp = - std::shared_ptr(new OffsetInputSource(m->file_sp, global_offset)); - m->file = m->file_sp.get(); + m->file = std::shared_ptr(new OffsetInputSource(m->file, global_offset)); } } return valid; } +bool +QPDF::findStartxref() +{ + if (readToken(*m->file).isWord("startxref") && readToken(*m->file).isInteger()) { + // Position in front of offset token + m->file->seek(m->file->getLastOffset(), SEEK_SET); + return true; + } + return false; +} + void QPDF::parse(char const* password) { @@ -405,9 +443,47 @@ QPDF::parse(char const* password) m->pdf_version = "1.2"; } - m->objects.xref_table().initialize(); + // PDF spec says %%EOF must be found within the last 1024 bytes of/ the file. We add an extra + // 30 characters to leave room for the startxref stuff. + m->file->seek(0, SEEK_END); + qpdf_offset_t end_offset = m->file->tell(); + m->xref_table_max_offset = end_offset; + // Sanity check on object ids. All objects must appear in xref table / stream. In all realistic + // scenarios at least 3 bytes are required. + if (m->xref_table_max_id > m->xref_table_max_offset / 3) { + m->xref_table_max_id = static_cast(m->xref_table_max_offset / 3); + } + qpdf_offset_t start_offset = (end_offset > 1054 ? end_offset - 1054 : 0); + PatternFinder sf(*this, &QPDF::findStartxref); + qpdf_offset_t xref_offset = 0; + if (m->file->findLast("startxref", start_offset, 0, sf)) { + xref_offset = QUtil::string_to_ll(readToken(*m->file).getValue().c_str()); + } + + try { + if (xref_offset == 0) { + QTC::TC("qpdf", "QPDF can't find startxref"); + throw damagedPDF("", 0, "can't find startxref"); + } + try { + read_xref(xref_offset); + } catch (QPDFExc&) { + throw; + } catch (std::exception& e) { + throw damagedPDF("", 0, std::string("error reading xref: ") + e.what()); + } + } catch (QPDFExc& e) { + if (m->attempt_recovery) { + reconstruct_xref(e); + QTC::TC("qpdf", "QPDF reconstructed xref table"); + } else { + throw; + } + } + initializeEncryption(); - if (m->objects.xref_table().size() > 0 && !getRoot().getKey("/Pages").isDictionary()) { + m->parsed = true; + if (m->xref_table.size() > 0 && !getRoot().getKey("/Pages").isDictionary()) { // QPDFs created from JSON have an empty xref table and no root object yet. throw damagedPDF("", 0, "unable to find page tree"); } @@ -448,9 +524,873 @@ QPDF::warn( } void +QPDF::setTrailer(QPDFObjectHandle obj) +{ + if (m->trailer) { + return; + } + m->trailer = obj; +} + +void +QPDF::reconstruct_xref(QPDFExc& e) +{ + if (m->reconstructed_xref) { + // Avoid xref reconstruction infinite loops. This is getting very hard to reproduce because + // qpdf is throwing many fewer exceptions while parsing. Most situations are warnings now. + throw e; + } + + // If recovery generates more than 1000 warnings, the file is so severely damaged that there + // probably is no point trying to continue. + const auto max_warnings = m->warnings.size() + 1000U; + auto check_warnings = [this, max_warnings]() { + if (m->warnings.size() > max_warnings) { + throw damagedPDF("", 0, "too many errors while reconstructing cross-reference table"); + } + }; + + m->reconstructed_xref = true; + // We may find more objects, which may contain dangling references. + m->fixed_dangling_refs = false; + + warn(damagedPDF("", 0, "file is damaged")); + warn(e); + warn(damagedPDF("", 0, "Attempting to reconstruct cross-reference table")); + + // Delete all references to type 1 (uncompressed) objects + std::set to_delete; + for (auto const& iter: m->xref_table) { + if (iter.second.getType() == 1) { + to_delete.insert(iter.first); + } + } + for (auto const& iter: to_delete) { + m->xref_table.erase(iter); + } + + m->file->seek(0, SEEK_END); + qpdf_offset_t eof = m->file->tell(); + m->file->seek(0, SEEK_SET); + // Don't allow very long tokens here during recovery. All the interesting tokens are covered. + static size_t const MAX_LEN = 10; + while (m->file->tell() < eof) { + QPDFTokenizer::Token t1 = readToken(*m->file, MAX_LEN); + qpdf_offset_t token_start = m->file->tell() - toO(t1.getValue().length()); + if (t1.isInteger()) { + auto pos = m->file->tell(); + QPDFTokenizer::Token t2 = readToken(*m->file, MAX_LEN); + if ((t2.isInteger()) && (readToken(*m->file, MAX_LEN).isWord("obj"))) { + int obj = QUtil::string_to_int(t1.getValue().c_str()); + int gen = QUtil::string_to_int(t2.getValue().c_str()); + if (obj <= m->xref_table_max_id) { + insertReconstructedXrefEntry(obj, token_start, gen); + } else { + warn(damagedPDF( + "", 0, "ignoring object with impossibly large id " + std::to_string(obj))); + } + } + m->file->seek(pos, SEEK_SET); + } else if (!m->trailer && t1.isWord("trailer")) { + auto pos = m->file->tell(); + QPDFObjectHandle t = readTrailer(); + if (!t.isDictionary()) { + // Oh well. It was worth a try. + } else { + setTrailer(t); + } + m->file->seek(pos, SEEK_SET); + } + check_warnings(); + m->file->findAndSkipNextEOL(); + } + m->deleted_objects.clear(); + + if (!m->trailer) { + qpdf_offset_t max_offset{0}; + // If there are any xref streams, take the last one to appear. + for (auto const& iter: m->xref_table) { + auto entry = iter.second; + if (entry.getType() != 1) { + continue; + } + auto oh = getObjectByObjGen(iter.first); + try { + if (!oh.isStreamOfType("/XRef")) { + continue; + } + } catch (std::exception&) { + continue; + } + auto offset = entry.getOffset(); + if (offset > max_offset) { + max_offset = offset; + setTrailer(oh.getDict()); + } + check_warnings(); + } + if (max_offset > 0) { + try { + read_xref(max_offset); + } catch (std::exception&) { + throw damagedPDF( + "", 0, "error decoding candidate xref stream while recovering damaged file"); + } + QTC::TC("qpdf", "QPDF recover xref stream"); + } + } + + if (!m->trailer) { + // We could check the last encountered object to see if it was an xref stream. If so, we + // could try to get the trailer from there. This may make it possible to recover files with + // bad startxref pointers even when they have object streams. + + throw damagedPDF("", 0, "unable to find trailer dictionary while recovering damaged file"); + } + if (m->xref_table.empty()) { + // We cannot check for an empty xref table in parse because empty tables are valid when + // creating QPDF objects from JSON. + throw damagedPDF("", 0, "unable to find objects while recovering damaged file"); + } + check_warnings(); + if (!m->parsed) { + m->parsed = true; + getAllPages(); + check_warnings(); + if (m->all_pages.empty()) { + m->parsed = false; + throw damagedPDF("", 0, "unable to find any pages while recovering damaged file"); + } + } + // We could iterate through the objects looking for streams and try to find objects inside of + // them, but it's probably not worth the trouble. Acrobat can't recover files with any errors + // in an xref stream, and this would be a real long shot anyway. If we wanted to do anything + // that involved looking at stream contents, we'd also have to call initializeEncryption() here. + // It's safe to call it more than once. +} + +void +QPDF::read_xref(qpdf_offset_t xref_offset) +{ + std::map free_table; + std::set visited; + while (xref_offset) { + visited.insert(xref_offset); + char buf[7]; + memset(buf, 0, sizeof(buf)); + m->file->seek(xref_offset, SEEK_SET); + // Some files miss the mark a little with startxref. We could do a better job of searching + // in the neighborhood for something that looks like either an xref table or stream, but the + // simple heuristic of skipping whitespace can help with the xref table case and is harmless + // with the stream case. + bool done = false; + bool skipped_space = false; + while (!done) { + char ch; + if (1 == m->file->read(&ch, 1)) { + if (QUtil::is_space(ch)) { + skipped_space = true; + } else { + m->file->unreadCh(ch); + done = true; + } + } else { + QTC::TC("qpdf", "QPDF eof skipping spaces before xref", skipped_space ? 0 : 1); + done = true; + } + } + + m->file->read(buf, sizeof(buf) - 1); + // The PDF spec says xref must be followed by a line terminator, but files exist in the wild + // where it is terminated by arbitrary whitespace. + if ((strncmp(buf, "xref", 4) == 0) && QUtil::is_space(buf[4])) { + if (skipped_space) { + QTC::TC("qpdf", "QPDF xref skipped space"); + warn(damagedPDF("", 0, "extraneous whitespace seen before xref")); + } + QTC::TC( + "qpdf", + "QPDF xref space", + ((buf[4] == '\n') ? 0 + : (buf[4] == '\r') ? 1 + : (buf[4] == ' ') ? 2 + : 9999)); + int skip = 4; + // buf is null-terminated, and QUtil::is_space('\0') is false, so this won't overrun. + while (QUtil::is_space(buf[skip])) { + ++skip; + } + xref_offset = read_xrefTable(xref_offset + skip); + } else { + xref_offset = read_xrefStream(xref_offset); + } + if (visited.count(xref_offset) != 0) { + QTC::TC("qpdf", "QPDF xref loop"); + throw damagedPDF("", 0, "loop detected following xref tables"); + } + } + + if (!m->trailer) { + throw damagedPDF("", 0, "unable to find trailer while reading xref"); + } + int size = m->trailer.getKey("/Size").getIntValueAsInt(); + int max_obj = 0; + if (!m->xref_table.empty()) { + max_obj = m->xref_table.rbegin()->first.getObj(); + } + if (!m->deleted_objects.empty()) { + max_obj = std::max(max_obj, *(m->deleted_objects.rbegin())); + } + if ((size < 1) || (size - 1 != max_obj)) { + QTC::TC("qpdf", "QPDF xref size mismatch"); + warn(damagedPDF( + "", + 0, + ("reported number of objects (" + std::to_string(size) + + ") is not one plus the highest object number (" + std::to_string(max_obj) + ")"))); + } + + // We no longer need the deleted_objects table, so go ahead and clear it out to make sure we + // never depend on its being set. + m->deleted_objects.clear(); + + // Make sure we keep only the highest generation for any object. + QPDFObjGen last_og{-1, 0}; + for (auto const& item: m->xref_table) { + auto id = item.first.getObj(); + if (id == last_og.getObj() && id > 0) { + removeObject(last_og); + } + last_og = item.first; + } +} + +bool +QPDF::parse_xrefFirst(std::string const& line, int& obj, int& num, int& bytes) +{ + // is_space and is_digit both return false on '\0', so this will not overrun the null-terminated + // buffer. + char const* p = line.c_str(); + char const* start = line.c_str(); + + // Skip zero or more spaces + while (QUtil::is_space(*p)) { + ++p; + } + // Require digit + if (!QUtil::is_digit(*p)) { + return false; + } + // Gather digits + std::string obj_str; + while (QUtil::is_digit(*p)) { + obj_str.append(1, *p++); + } + // Require space + if (!QUtil::is_space(*p)) { + return false; + } + // Skip spaces + while (QUtil::is_space(*p)) { + ++p; + } + // Require digit + if (!QUtil::is_digit(*p)) { + return false; + } + // Gather digits + std::string num_str; + while (QUtil::is_digit(*p)) { + num_str.append(1, *p++); + } + // Skip any space including line terminators + while (QUtil::is_space(*p)) { + ++p; + } + bytes = toI(p - start); + obj = QUtil::string_to_int(obj_str.c_str()); + num = QUtil::string_to_int(num_str.c_str()); + return true; +} + +bool +QPDF::read_bad_xrefEntry(qpdf_offset_t& f1, int& f2, char& type) +{ + // Reposition after initial read attempt and reread. + m->file->seek(m->file->getLastOffset(), SEEK_SET); + auto line = m->file->readLine(30); + + // is_space and is_digit both return false on '\0', so this will not overrun the null-terminated + // buffer. + char const* p = line.data(); + + // Skip zero or more spaces. There aren't supposed to be any. + bool invalid = false; + while (QUtil::is_space(*p)) { + ++p; + QTC::TC("qpdf", "QPDF ignore first space in xref entry"); + invalid = true; + } + // Require digit + if (!QUtil::is_digit(*p)) { + return false; + } + // Gather digits + std::string f1_str; + while (QUtil::is_digit(*p)) { + f1_str.append(1, *p++); + } + // Require space + if (!QUtil::is_space(*p)) { + return false; + } + if (QUtil::is_space(*(p + 1))) { + QTC::TC("qpdf", "QPDF ignore first extra space in xref entry"); + invalid = true; + } + // Skip spaces + while (QUtil::is_space(*p)) { + ++p; + } + // Require digit + if (!QUtil::is_digit(*p)) { + return false; + } + // Gather digits + std::string f2_str; + while (QUtil::is_digit(*p)) { + f2_str.append(1, *p++); + } + // Require space + if (!QUtil::is_space(*p)) { + return false; + } + if (QUtil::is_space(*(p + 1))) { + QTC::TC("qpdf", "QPDF ignore second extra space in xref entry"); + invalid = true; + } + // Skip spaces + while (QUtil::is_space(*p)) { + ++p; + } + if ((*p == 'f') || (*p == 'n')) { + type = *p; + } else { + return false; + } + if ((f1_str.length() != 10) || (f2_str.length() != 5)) { + QTC::TC("qpdf", "QPDF ignore length error xref entry"); + invalid = true; + } + + if (invalid) { + warn(damagedPDF("xref table", "accepting invalid xref table entry")); + } + + f1 = QUtil::string_to_ll(f1_str.c_str()); + f2 = QUtil::string_to_int(f2_str.c_str()); + + return true; +} + +// Optimistically read and parse xref entry. If entry is bad, call read_bad_xrefEntry and return +// result. +bool +QPDF::read_xrefEntry(qpdf_offset_t& f1, int& f2, char& type) +{ + std::array line; + if (m->file->read(line.data(), 20) != 20) { + // C++20: [[unlikely]] + return false; + } + line[20] = '\0'; + char const* p = line.data(); + + int f1_len = 0; + int f2_len = 0; + + // is_space and is_digit both return false on '\0', so this will not overrun the null-terminated + // buffer. + + // Gather f1 digits. NB No risk of overflow as 9'999'999'999 < max long long. + while (*p == '0') { + ++f1_len; + ++p; + } + while (QUtil::is_digit(*p) && f1_len++ < 10) { + f1 *= 10; + f1 += *p++ - '0'; + } + // Require space + if (!QUtil::is_space(*p++)) { + // Entry doesn't start with space or digit. + // C++20: [[unlikely]] + return false; + } + // Gather digits. NB No risk of overflow as 99'999 < max int. + while (*p == '0') { + ++f2_len; + ++p; + } + while (QUtil::is_digit(*p) && f2_len++ < 5) { + f2 *= 10; + f2 += static_cast(*p++ - '0'); + } + if (QUtil::is_space(*p++) && (*p == 'f' || *p == 'n')) { + // C++20: [[likely]] + type = *p; + // No test for valid line[19]. + if (*(++p) && *(++p) && (*p == '\n' || *p == '\r') && f1_len == 10 && f2_len == 5) { + // C++20: [[likely]] + return true; + } + } + return read_bad_xrefEntry(f1, f2, type); +} + +// Read a single cross-reference table section and associated trailer. +qpdf_offset_t +QPDF::read_xrefTable(qpdf_offset_t xref_offset) +{ + m->file->seek(xref_offset, SEEK_SET); + std::string line; + while (true) { + line.assign(50, '\0'); + m->file->read(line.data(), line.size()); + int obj = 0; + int num = 0; + int bytes = 0; + if (!parse_xrefFirst(line, obj, num, bytes)) { + QTC::TC("qpdf", "QPDF invalid xref"); + throw damagedPDF("xref table", "xref syntax invalid"); + } + m->file->seek(m->file->getLastOffset() + bytes, SEEK_SET); + for (qpdf_offset_t i = obj; i - num < obj; ++i) { + if (i == 0) { + // This is needed by checkLinearization() + m->first_xref_item_offset = m->file->tell(); + } + // For xref_table, these will always be small enough to be ints + qpdf_offset_t f1 = 0; + int f2 = 0; + char type = '\0'; + if (!read_xrefEntry(f1, f2, type)) { + QTC::TC("qpdf", "QPDF invalid xref entry"); + throw damagedPDF( + "xref table", "invalid xref entry (obj=" + std::to_string(i) + ")"); + } + if (type == 'f') { + insertFreeXrefEntry(QPDFObjGen(toI(i), f2)); + } else { + insertXrefEntry(toI(i), 1, f1, f2); + } + } + qpdf_offset_t pos = m->file->tell(); + if (readToken(*m->file).isWord("trailer")) { + break; + } else { + m->file->seek(pos, SEEK_SET); + } + } + + // Set offset to previous xref table if any + QPDFObjectHandle cur_trailer = readTrailer(); + if (!cur_trailer.isDictionary()) { + QTC::TC("qpdf", "QPDF missing trailer"); + throw damagedPDF("", "expected trailer dictionary"); + } + + if (!m->trailer) { + setTrailer(cur_trailer); + + if (!m->trailer.hasKey("/Size")) { + QTC::TC("qpdf", "QPDF trailer lacks size"); + throw damagedPDF("trailer", "trailer dictionary lacks /Size key"); + } + if (!m->trailer.getKey("/Size").isInteger()) { + QTC::TC("qpdf", "QPDF trailer size not integer"); + throw damagedPDF("trailer", "/Size key in trailer dictionary is not an integer"); + } + } + + if (cur_trailer.hasKey("/XRefStm")) { + if (m->ignore_xref_streams) { + QTC::TC("qpdf", "QPDF ignoring XRefStm in trailer"); + } else { + if (cur_trailer.getKey("/XRefStm").isInteger()) { + // Read the xref stream but disregard any return value -- we'll use our trailer's + // /Prev key instead of the xref stream's. + (void)read_xrefStream(cur_trailer.getKey("/XRefStm").getIntValue()); + } else { + throw damagedPDF("xref stream", xref_offset, "invalid /XRefStm"); + } + } + } + + if (cur_trailer.hasKey("/Prev")) { + if (!cur_trailer.getKey("/Prev").isInteger()) { + QTC::TC("qpdf", "QPDF trailer prev not integer"); + throw damagedPDF("trailer", "/Prev key in trailer dictionary is not an integer"); + } + QTC::TC("qpdf", "QPDF prev key in trailer dictionary"); + return cur_trailer.getKey("/Prev").getIntValue(); + } + + return 0; +} + +// Read a single cross-reference stream. +qpdf_offset_t +QPDF::read_xrefStream(qpdf_offset_t xref_offset) +{ + if (!m->ignore_xref_streams) { + QPDFObjGen x_og; + QPDFObjectHandle xref_obj; + try { + xref_obj = + readObjectAtOffset(false, xref_offset, "xref stream", QPDFObjGen(0, 0), x_og, true); + } catch (QPDFExc&) { + // ignore -- report error below + } + if (xref_obj.isStreamOfType("/XRef")) { + QTC::TC("qpdf", "QPDF found xref stream"); + return processXRefStream(xref_offset, xref_obj); + } + } + + QTC::TC("qpdf", "QPDF can't find xref"); + throw damagedPDF("", xref_offset, "xref not found"); + return 0; // unreachable +} + +// Return the entry size of the xref stream and the processed W array. +std::pair> +QPDF::processXRefW(QPDFObjectHandle& dict, std::function damaged) +{ + auto W_obj = dict.getKey("/W"); + if (!(W_obj.isArray() && (W_obj.getArrayNItems() >= 3) && W_obj.getArrayItem(0).isInteger() && + W_obj.getArrayItem(1).isInteger() && W_obj.getArrayItem(2).isInteger())) { + throw damaged("Cross-reference stream does not have a proper /W key"); + } + + std::array W; + int entry_size = 0; + auto w_vector = W_obj.getArrayAsVector(); + int max_bytes = sizeof(qpdf_offset_t); + for (size_t i = 0; i < 3; ++i) { + W[i] = w_vector[i].getIntValueAsInt(); + if (W[i] > max_bytes) { + throw damaged("Cross-reference stream's /W contains impossibly large values"); + } + if (W[i] < 0) { + throw damaged("Cross-reference stream's /W contains negative values"); + } + entry_size += W[i]; + } + if (entry_size == 0) { + throw damaged("Cross-reference stream's /W indicates entry size of 0"); + } + return {entry_size, W}; +} + +// Validate Size key and return the maximum number of entries that the xref stream can contain. +int +QPDF::processXRefSize( + QPDFObjectHandle& dict, int entry_size, std::function damaged) +{ + // Number of entries is limited by the highest possible object id and stream size. + auto max_num_entries = std::numeric_limits::max(); + if (max_num_entries > (std::numeric_limits::max() / entry_size)) { + max_num_entries = toI(std::numeric_limits::max() / entry_size); + } + + auto Size_obj = dict.getKey("/Size"); + long long size; + if (!dict.getKey("/Size").getValueAsInt(size)) { + throw damaged("Cross-reference stream does not have a proper /Size key"); + } else if (size < 0) { + throw damaged("Cross-reference stream has a negative /Size key"); + } else if (size >= max_num_entries) { + throw damaged("Cross-reference stream has an impossibly large /Size key"); + } + // We are not validating that Size <= (Size key of parent xref / trailer). + return max_num_entries; +} + +// Return the number of entries of the xref stream and the processed Index array. +std::pair>> +QPDF::processXRefIndex( + QPDFObjectHandle& dict, int max_num_entries, std::function damaged) +{ + auto size = dict.getKey("/Size").getIntValueAsInt(); + auto Index_obj = dict.getKey("/Index"); + + if (Index_obj.isArray()) { + std::vector> indx; + int num_entries = 0; + auto index_vec = Index_obj.getArrayAsVector(); + if ((index_vec.size() % 2) || index_vec.size() < 2) { + throw damaged("Cross-reference stream's /Index has an invalid number of values"); + } + + int i = 0; + long long first = 0; + for (auto& val: index_vec) { + if (val.isInteger()) { + if (i % 2) { + auto count = val.getIntValue(); + if (count <= 0) { + throw damaged( + "Cross-reference stream section claims to contain " + + std::to_string(count) + " entries"); + } + // We are guarding against the possibility of num_entries * entry_size + // overflowing. We are not checking that entries are in ascending order as + // required by the spec, which probably should generate a warning. We are also + // not checking that for each subsection first object number + number of entries + // <= /Size. The spec requires us to ignore object number > /Size. + if (first > (max_num_entries - count) || + count > (max_num_entries - num_entries)) { + throw damaged( + "Cross-reference stream claims to contain too many entries: " + + std::to_string(first) + " " + std::to_string(max_num_entries) + " " + + std::to_string(num_entries)); + } + indx.emplace_back(static_cast(first), static_cast(count)); + num_entries += static_cast(count); + } else { + first = val.getIntValue(); + if (first < 0) { + throw damaged( + "Cross-reference stream's /Index contains a negative object id"); + } else if (first > max_num_entries) { + throw damaged("Cross-reference stream's /Index contains an impossibly " + "large object id"); + } + } + } else { + throw damaged( + "Cross-reference stream's /Index's item " + std::to_string(i) + + " is not an integer"); + } + i++; + } + QTC::TC("qpdf", "QPDF xref /Index is array", index_vec.size() == 2 ? 0 : 1); + return {num_entries, indx}; + } else if (Index_obj.isNull()) { + QTC::TC("qpdf", "QPDF xref /Index is null"); + return {size, {{0, size}}}; + } else { + throw damaged("Cross-reference stream does not have a proper /Index key"); + } +} + +qpdf_offset_t +QPDF::processXRefStream(qpdf_offset_t xref_offset, QPDFObjectHandle& xref_obj) +{ + auto damaged = [this, xref_offset](std::string_view msg) -> QPDFExc { + return damagedPDF("xref stream", xref_offset, msg.data()); + }; + + auto dict = xref_obj.getDict(); + + auto [entry_size, W] = processXRefW(dict, damaged); + int max_num_entries = processXRefSize(dict, entry_size, damaged); + auto [num_entries, indx] = processXRefIndex(dict, max_num_entries, damaged); + + std::shared_ptr bp = xref_obj.getStreamData(qpdf_dl_specialized); + size_t actual_size = bp->getSize(); + auto expected_size = toS(entry_size) * toS(num_entries); + + if (expected_size != actual_size) { + QPDFExc x = damaged( + "Cross-reference stream data has the wrong size; expected = " + + std::to_string(expected_size) + "; actual = " + std::to_string(actual_size)); + if (expected_size > actual_size) { + throw x; + } else { + warn(x); + } + } + + bool saw_first_compressed_object = false; + + // Actual size vs. expected size check above ensures that we will not overflow any buffers here. + // We know that entry_size * num_entries is less or equal to the size of the buffer. + auto p = bp->getBuffer(); + for (auto [obj, sec_entries]: indx) { + // Process a subsection. + for (int i = 0; i < sec_entries; ++i) { + // Read this entry + std::array fields{}; + if (W[0] == 0) { + QTC::TC("qpdf", "QPDF default for xref stream field 0"); + fields[0] = 1; + } + for (size_t j = 0; j < 3; ++j) { + for (int k = 0; k < W[j]; ++k) { + fields[j] <<= 8; + fields[j] |= *p++; + } + } + + // Get the generation number. The generation number is 0 unless this is an uncompressed + // object record, in which case the generation number appears as the third field. + if (saw_first_compressed_object) { + if (fields[0] != 2) { + m->uncompressed_after_compressed = true; + } + } else if (fields[0] == 2) { + saw_first_compressed_object = true; + } + if (obj == 0) { + // This is needed by checkLinearization() + m->first_xref_item_offset = xref_offset; + } else if (fields[0] == 0) { + // Ignore fields[2], which we don't care about in this case. This works around the + // issue of some PDF files that put invalid values, like -1, here for deleted + // objects. + insertFreeXrefEntry(QPDFObjGen(obj, 0)); + } else { + insertXrefEntry(obj, toI(fields[0]), fields[1], toI(fields[2])); + } + ++obj; + } + } + + if (!m->trailer) { + setTrailer(dict); + } + + if (dict.hasKey("/Prev")) { + if (!dict.getKey("/Prev").isInteger()) { + throw damagedPDF( + "xref stream", "/Prev key in xref stream dictionary is not an integer"); + } + QTC::TC("qpdf", "QPDF prev key in xref stream dictionary"); + return dict.getKey("/Prev").getIntValue(); + } else { + return 0; + } +} + +void +QPDF::insertXrefEntry(int obj, int f0, qpdf_offset_t f1, int f2) +{ + // Populate the xref table in such a way that the first reference to an object that we see, + // which is the one in the latest xref table in which it appears, is the one that gets stored. + // This works because we are reading more recent appends before older ones. + + // If there is already an entry for this object and generation in the table, it means that a + // later xref table has registered this object. Disregard this one. + + if (obj > m->xref_table_max_id) { + // ignore impossibly large object ids or object ids > Size. + return; + } + + if (m->deleted_objects.count(obj)) { + QTC::TC("qpdf", "QPDF xref deleted object"); + return; + } + + if (f0 == 2 && static_cast(f1) == obj) { + warn(damagedPDF("xref stream", "self-referential object stream " + std::to_string(obj))); + return; + } + + auto [iter, created] = m->xref_table.try_emplace(QPDFObjGen(obj, (f0 == 2 ? 0 : f2))); + if (!created) { + QTC::TC("qpdf", "QPDF xref reused object"); + return; + } + + switch (f0) { + case 1: + // f2 is generation + QTC::TC("qpdf", "QPDF xref gen > 0", ((f2 > 0) ? 1 : 0)); + iter->second = QPDFXRefEntry(f1); + break; + + case 2: + iter->second = QPDFXRefEntry(toI(f1), f2); + break; + + default: + throw damagedPDF("xref stream", "unknown xref stream entry type " + std::to_string(f0)); + break; + } +} + +void +QPDF::insertFreeXrefEntry(QPDFObjGen og) +{ + if (!m->xref_table.count(og)) { + m->deleted_objects.insert(og.getObj()); + } +} + +// Replace uncompressed object. This is used in xref recovery mode, which reads the file from +// beginning to end. +void +QPDF::insertReconstructedXrefEntry(int obj, qpdf_offset_t f1, int f2) +{ + if (!(obj > 0 && obj <= m->xref_table_max_id && 0 <= f2 && f2 < 65535)) { + QTC::TC("qpdf", "QPDF xref overwrite invalid objgen"); + return; + } + + QPDFObjGen og(obj, f2); + if (!m->deleted_objects.count(obj)) { + // deleted_objects stores the uncompressed objects removed from the xref table at the start + // of recovery. + QTC::TC("qpdf", "QPDF xref overwrite object"); + m->xref_table[QPDFObjGen(obj, f2)] = QPDFXRefEntry(f1); + } +} + +void QPDF::showXRefTable() { - m->objects.xref_table().show(); + auto& cout = *m->log->getInfo(); + for (auto const& iter: m->xref_table) { + QPDFObjGen const& og = iter.first; + QPDFXRefEntry const& entry = iter.second; + cout << og.unparse('/') << ": "; + switch (entry.getType()) { + case 1: + cout << "uncompressed; offset = " << entry.getOffset(); + break; + + case 2: + *m->log->getInfo() << "compressed; stream = " << entry.getObjStreamNumber() + << ", index = " << entry.getObjStreamIndex(); + break; + + default: + throw std::logic_error("unknown cross-reference table type while" + " showing xref_table"); + break; + } + m->log->info("\n"); + } +} + +// Resolve all objects in the xref table. If this triggers a xref table reconstruction abort and +// return false. Otherwise return true. +bool +QPDF::resolveXRefTable() +{ + bool may_change = !m->reconstructed_xref; + for (auto& iter: m->xref_table) { + if (isUnresolved(iter.first)) { + resolve(iter.first); + if (may_change && m->reconstructed_xref) { + return false; + } + } + } + return true; } // Ensure all objects in the pdf file, including those in indirect references, appear in the object @@ -461,9 +1401,9 @@ QPDF::fixDanglingReferences(bool force) if (m->fixed_dangling_refs) { return; } - if (!m->objects.xref_table().resolve()) { + if (!resolveXRefTable()) { QTC::TC("qpdf", "QPDF fix dangling triggered xref reconstruction"); - m->objects.xref_table().resolve(); + resolveXRefTable(); } m->fixed_dangling_refs = true; } @@ -474,13 +1414,24 @@ QPDF::getObjectCount() // This method returns the next available indirect object number. makeIndirectObject uses it for // this purpose. After fixDanglingReferences is called, all objects in the xref table will also // be in obj_cache. - return toS(m->objects.next_id().getObj() - 1); + fixDanglingReferences(); + QPDFObjGen og; + if (!m->obj_cache.empty()) { + og = (*(m->obj_cache.rbegin())).first; + } + return toS(og.getObj()); } std::vector QPDF::getAllObjects() { - return m->objects.all(); + // After fixDanglingReferences is called, all objects are in the object cache. + fixDanglingReferences(); + std::vector result; + for (auto const& iter: m->obj_cache) { + result.push_back(newIndirect(iter.first, iter.second.object)); + } + return result; } void @@ -498,6 +1449,233 @@ QPDF::setLastObjectDescription(std::string const& description, QPDFObjGen const& } } +QPDFObjectHandle +QPDF::readTrailer() +{ + qpdf_offset_t offset = m->file->tell(); + bool empty = false; + auto object = + QPDFParser(*m->file, "trailer", m->tokenizer, nullptr, this, true).parse(empty, false); + if (empty) { + // Nothing in the PDF spec appears to allow empty objects, but they have been encountered in + // actual PDF files and Adobe Reader appears to ignore them. + warn(damagedPDF("trailer", "empty object treated as null")); + } else if (object.isDictionary() && readToken(*m->file).isWord("stream")) { + warn(damagedPDF("trailer", m->file->tell(), "stream keyword found in trailer")); + } + // Override last_offset so that it points to the beginning of the object we just read + m->file->setLastOffset(offset); + return object; +} + +QPDFObjectHandle +QPDF::readObject(std::string const& description, QPDFObjGen og) +{ + setLastObjectDescription(description, og); + qpdf_offset_t offset = m->file->tell(); + bool empty = false; + + StringDecrypter decrypter{this, og}; + StringDecrypter* decrypter_ptr = m->encp->encrypted ? &decrypter : nullptr; + auto object = + QPDFParser(*m->file, m->last_object_description, m->tokenizer, decrypter_ptr, this, true) + .parse(empty, false); + if (empty) { + // Nothing in the PDF spec appears to allow empty objects, but they have been encountered in + // actual PDF files and Adobe Reader appears to ignore them. + warn(damagedPDF(*m->file, m->file->getLastOffset(), "empty object treated as null")); + return object; + } + auto token = readToken(*m->file); + if (object.isDictionary() && token.isWord("stream")) { + readStream(object, og, offset); + token = readToken(*m->file); + } + if (!token.isWord("endobj")) { + QTC::TC("qpdf", "QPDF err expected endobj"); + warn(damagedPDF("expected endobj")); + } + return object; +} + +// After reading stream dictionary and stream keyword, read rest of stream. +void +QPDF::readStream(QPDFObjectHandle& object, QPDFObjGen og, qpdf_offset_t offset) +{ + validateStreamLineEnd(object, og, offset); + + // Must get offset before accessing any additional objects since resolving a previously + // unresolved indirect object will change file position. + qpdf_offset_t stream_offset = m->file->tell(); + size_t length = 0; + + try { + auto length_obj = object.getKey("/Length"); + + if (!length_obj.isInteger()) { + if (length_obj.isNull()) { + QTC::TC("qpdf", "QPDF stream without length"); + throw damagedPDF(offset, "stream dictionary lacks /Length key"); + } + QTC::TC("qpdf", "QPDF stream length not integer"); + throw damagedPDF(offset, "/Length key in stream dictionary is not an integer"); + } + + length = toS(length_obj.getUIntValue()); + // Seek in two steps to avoid potential integer overflow + m->file->seek(stream_offset, SEEK_SET); + m->file->seek(toO(length), SEEK_CUR); + if (!readToken(*m->file).isWord("endstream")) { + QTC::TC("qpdf", "QPDF missing endstream"); + throw damagedPDF("expected endstream"); + } + } catch (QPDFExc& e) { + if (m->attempt_recovery) { + warn(e); + length = recoverStreamLength(m->file, og, stream_offset); + } else { + throw; + } + } + object = {QPDF_Stream::create(this, og, object, stream_offset, length)}; +} + +void +QPDF::validateStreamLineEnd(QPDFObjectHandle& object, QPDFObjGen og, qpdf_offset_t offset) +{ + // The PDF specification states that the word "stream" should be followed by either a carriage + // return and a newline or by a newline alone. It specifically disallowed following it by a + // carriage return alone since, in that case, there would be no way to tell whether the NL in a + // CR NL sequence was part of the stream data. However, some readers, including Adobe reader, + // accept a carriage return by itself when followed by a non-newline character, so that's what + // we do here. We have also seen files that have extraneous whitespace between the stream + // keyword and the newline. + while (true) { + char ch; + if (m->file->read(&ch, 1) == 0) { + // A premature EOF here will result in some other problem that will get reported at + // another time. + return; + } + if (ch == '\n') { + // ready to read stream data + QTC::TC("qpdf", "QPDF stream with NL only"); + return; + } + if (ch == '\r') { + // Read another character + if (m->file->read(&ch, 1) != 0) { + if (ch == '\n') { + // Ready to read stream data + QTC::TC("qpdf", "QPDF stream with CRNL"); + } else { + // Treat the \r by itself as the whitespace after endstream and start reading + // stream data in spite of not having seen a newline. + QTC::TC("qpdf", "QPDF stream with CR only"); + m->file->unreadCh(ch); + warn(damagedPDF( + m->file->tell(), "stream keyword followed by carriage return only")); + } + } + return; + } + if (!QUtil::is_space(ch)) { + QTC::TC("qpdf", "QPDF stream without newline"); + m->file->unreadCh(ch); + warn(damagedPDF( + m->file->tell(), "stream keyword not followed by proper line terminator")); + return; + } + warn(damagedPDF(m->file->tell(), "stream keyword followed by extraneous whitespace")); + } +} + +QPDFObjectHandle +QPDF::readObjectInStream(std::shared_ptr& input, int obj) +{ + m->last_object_description.erase(7); // last_object_description starts with "object " + m->last_object_description += std::to_string(obj); + m->last_object_description += " 0"; + + bool empty = false; + auto object = QPDFParser(*input, m->last_object_description, m->tokenizer, nullptr, this, true) + .parse(empty, false); + if (empty) { + // Nothing in the PDF spec appears to allow empty objects, but they have been encountered in + // actual PDF files and Adobe Reader appears to ignore them. + warn(damagedPDF(*input, input->getLastOffset(), "empty object treated as null")); + } + return object; +} + +bool +QPDF::findEndstream() +{ + // Find endstream or endobj. Position the input at that token. + auto t = readToken(*m->file, 20); + if (t.isWord("endobj") || t.isWord("endstream")) { + m->file->seek(m->file->getLastOffset(), SEEK_SET); + return true; + } + return false; +} + +size_t +QPDF::recoverStreamLength( + std::shared_ptr input, QPDFObjGen const& og, qpdf_offset_t stream_offset) +{ + // Try to reconstruct stream length by looking for endstream or endobj + warn(damagedPDF(*input, stream_offset, "attempting to recover stream length")); + + PatternFinder ef(*this, &QPDF::findEndstream); + size_t length = 0; + if (m->file->findFirst("end", stream_offset, 0, ef)) { + length = toS(m->file->tell() - stream_offset); + // Reread endstream but, if it was endobj, don't skip that. + QPDFTokenizer::Token t = readToken(*m->file); + if (t.getValue() == "endobj") { + m->file->seek(m->file->getLastOffset(), SEEK_SET); + } + } + + if (length) { + auto end = stream_offset + toO(length); + qpdf_offset_t found_offset = 0; + QPDFObjGen found_og; + + // Make sure this is inside this object + for (auto const& [current_og, entry]: m->xref_table) { + if (entry.getType() == 1) { + qpdf_offset_t obj_offset = entry.getOffset(); + if (found_offset < obj_offset && obj_offset < end) { + found_offset = obj_offset; + found_og = current_og; + } + } + } + if (!found_offset || found_og == og) { + // If we are trying to recover an XRef stream the xref table will not contain and + // won't contain any entries, therefore we cannot check the found length. Otherwise we + // found endstream\nendobj within the space allowed for this object, so we're probably + // in good shape. + } else { + QTC::TC("qpdf", "QPDF found wrong endstream in recovery"); + length = 0; + } + } + + if (length == 0) { + warn(damagedPDF( + *input, stream_offset, "unable to recover stream data; treating stream as empty")); + } else { + warn(damagedPDF( + *input, stream_offset, "recovered stream length: " + std::to_string(length))); + } + + QTC::TC("qpdf", "QPDF recovered stream length"); + return length; +} + QPDFTokenizer::Token QPDF::readToken(InputSource& input, size_t max_len) { @@ -505,38 +1683,398 @@ QPDF::readToken(InputSource& input, size_t max_len) } QPDFObjectHandle +QPDF::readObjectAtOffset( + bool try_recovery, + qpdf_offset_t offset, + std::string const& description, + QPDFObjGen exp_og, + QPDFObjGen& og, + bool skip_cache_if_in_xref) +{ + bool check_og = true; + if (exp_og.getObj() == 0) { + // This method uses an expect object ID of 0 to indicate that we don't know or don't care + // what the actual object ID is at this offset. This is true when we read the xref stream + // and linearization hint streams. In this case, we don't verify the expect object + // ID/generation against what was read from the file. There is also no reason to attempt + // xref recovery if we get a failure in this case since the read attempt was not triggered + // by an xref lookup. + check_og = false; + try_recovery = false; + } + setLastObjectDescription(description, exp_og); + + if (!m->attempt_recovery) { + try_recovery = false; + } + + // Special case: if offset is 0, just return null. Some PDF writers, in particular + // "Mac OS X 10.7.5 Quartz PDFContext", may store deleted objects in the xref table as + // "0000000000 00000 n", which is not correct, but it won't hurt anything for us to ignore + // these. + if (offset == 0) { + QTC::TC("qpdf", "QPDF bogus 0 offset", 0); + warn(damagedPDF(0, "object has offset 0")); + return QPDFObjectHandle::newNull(); + } + + m->file->seek(offset, SEEK_SET); + try { + QPDFTokenizer::Token tobjid = readToken(*m->file); + bool objidok = tobjid.isInteger(); + QTC::TC("qpdf", "QPDF check objid", objidok ? 1 : 0); + if (!objidok) { + QTC::TC("qpdf", "QPDF expected n n obj"); + throw damagedPDF(offset, "expected n n obj"); + } + QPDFTokenizer::Token tgen = readToken(*m->file); + bool genok = tgen.isInteger(); + QTC::TC("qpdf", "QPDF check generation", genok ? 1 : 0); + if (!genok) { + throw damagedPDF(offset, "expected n n obj"); + } + QPDFTokenizer::Token tobj = readToken(*m->file); + + bool objok = tobj.isWord("obj"); + QTC::TC("qpdf", "QPDF check obj", objok ? 1 : 0); + + if (!objok) { + throw damagedPDF(offset, "expected n n obj"); + } + int objid = QUtil::string_to_int(tobjid.getValue().c_str()); + int generation = QUtil::string_to_int(tgen.getValue().c_str()); + og = QPDFObjGen(objid, generation); + if (objid == 0) { + QTC::TC("qpdf", "QPDF object id 0"); + throw damagedPDF(offset, "object with ID 0"); + } + if (check_og && (exp_og != og)) { + QTC::TC("qpdf", "QPDF err wrong objid/generation"); + QPDFExc e = damagedPDF(offset, "expected " + exp_og.unparse(' ') + " obj"); + if (try_recovery) { + // Will be retried below + throw e; + } else { + // We can try reading the object anyway even if the ID doesn't match. + warn(e); + } + } + } catch (QPDFExc& e) { + if (try_recovery) { + // Try again after reconstructing xref table + reconstruct_xref(e); + if (m->xref_table.count(exp_og) && (m->xref_table[exp_og].getType() == 1)) { + qpdf_offset_t new_offset = m->xref_table[exp_og].getOffset(); + QPDFObjectHandle result = + readObjectAtOffset(false, new_offset, description, exp_og, og, false); + QTC::TC("qpdf", "QPDF recovered in readObjectAtOffset"); + return result; + } else { + QTC::TC("qpdf", "QPDF object gone after xref reconstruction"); + warn(damagedPDF( + "", + 0, + ("object " + exp_og.unparse(' ') + + " not found in file after regenerating cross reference " + "table"))); + return QPDFObjectHandle::newNull(); + } + } else { + throw; + } + } + + QPDFObjectHandle oh = readObject(description, og); + + if (isUnresolved(og)) { + // Store the object in the cache here so it gets cached whether we first know the offset or + // whether we first know the object ID and generation (in which we case we would get here + // through resolve). + + // Determine the end offset of this object before and after white space. We use these + // numbers to validate linearization hint tables. Offsets and lengths of objects may imply + // the end of an object to be anywhere between these values. + qpdf_offset_t end_before_space = m->file->tell(); + + // skip over spaces + while (true) { + char ch; + if (m->file->read(&ch, 1)) { + if (!isspace(static_cast(ch))) { + m->file->seek(-1, SEEK_CUR); + break; + } + } else { + throw damagedPDF(m->file->tell(), "EOF after endobj"); + } + } + qpdf_offset_t end_after_space = m->file->tell(); + if (skip_cache_if_in_xref && m->xref_table.count(og)) { + // Ordinarily, an object gets read here when resolved through xref table or stream. In + // the special case of the xref stream and linearization hint tables, the offset comes + // from another source. For the specific case of xref streams, the xref stream is read + // and loaded into the object cache very early in parsing. Ordinarily, when a file is + // updated by appending, items inserted into the xref table in later updates take + // precedence over earlier items. In the special case of reusing the object number + // previously used as the xref stream, we have the following order of events: + // + // * reused object gets loaded into the xref table + // * old object is read here while reading xref streams + // * original xref entry is ignored (since already in xref table) + // + // It is the second step that causes a problem. Even though the xref table is correct in + // this case, the old object is already in the cache and so effectively prevails over + // the reused object. To work around this issue, we have a special case for the xref + // stream (via the skip_cache_if_in_xref): if the object is already in the xref stream, + // don't cache what we read here. + // + // It is likely that the same bug may exist for linearization hint tables, but the + // existing code uses end_before_space and end_after_space from the cache, so fixing + // that would require more significant rework. The chances of a linearization hint + // stream being reused seems smaller because the xref stream is probably the highest + // object in the file and the linearization hint stream would be some random place in + // the middle, so I'm leaving that bug unfixed for now. If the bug were to be fixed, we + // could use !check_og in place of skip_cache_if_in_xref. + QTC::TC("qpdf", "QPDF skipping cache for known unchecked object"); + } else { + updateCache(og, oh.getObj(), end_before_space, end_after_space); + } + } + + return oh; +} + +QPDFObject* +QPDF::resolve(QPDFObjGen og) +{ + if (!isUnresolved(og)) { + return m->obj_cache[og].object.get(); + } + + if (m->resolving.count(og)) { + // This can happen if an object references itself directly or indirectly in some key that + // has to be resolved during object parsing, such as stream length. + QTC::TC("qpdf", "QPDF recursion loop in resolve"); + warn(damagedPDF("", "loop detected resolving object " + og.unparse(' '))); + updateCache(og, QPDF_Null::create(), -1, -1); + return m->obj_cache[og].object.get(); + } + ResolveRecorder rr(this, og); + + if (m->xref_table.count(og) != 0) { + QPDFXRefEntry const& entry = m->xref_table[og]; + try { + switch (entry.getType()) { + case 1: + { + qpdf_offset_t offset = entry.getOffset(); + // Object stored in cache by readObjectAtOffset + QPDFObjGen a_og; + QPDFObjectHandle oh = readObjectAtOffset(true, offset, "", og, a_og, false); + } + break; + + case 2: + resolveObjectsInStream(entry.getObjStreamNumber()); + break; + + default: + throw damagedPDF( + "", 0, ("object " + og.unparse('/') + " has unexpected xref entry type")); + } + } catch (QPDFExc& e) { + warn(e); + } catch (std::exception& e) { + warn(damagedPDF( + "", 0, ("object " + og.unparse('/') + ": error reading object: " + e.what()))); + } + } + + if (isUnresolved(og)) { + // PDF spec says unknown objects resolve to the null object. + QTC::TC("qpdf", "QPDF resolve failure to null"); + updateCache(og, QPDF_Null::create(), -1, -1); + } + + auto result(m->obj_cache[og].object); + result->setDefaultDescription(this, og); + return result.get(); +} + +void +QPDF::resolveObjectsInStream(int obj_stream_number) +{ + if (m->resolved_object_streams.count(obj_stream_number)) { + return; + } + m->resolved_object_streams.insert(obj_stream_number); + // Force resolution of object stream + QPDFObjectHandle obj_stream = getObjectByID(obj_stream_number, 0); + if (!obj_stream.isStream()) { + throw damagedPDF( + "supposed object stream " + std::to_string(obj_stream_number) + " is not a stream"); + } + + // For linearization data in the object, use the data from the object stream for the objects in + // the stream. + QPDFObjGen stream_og(obj_stream_number, 0); + qpdf_offset_t end_before_space = m->obj_cache[stream_og].end_before_space; + qpdf_offset_t end_after_space = m->obj_cache[stream_og].end_after_space; + + QPDFObjectHandle dict = obj_stream.getDict(); + if (!dict.isDictionaryOfType("/ObjStm")) { + QTC::TC("qpdf", "QPDF ERR object stream with wrong type"); + warn(damagedPDF( + "supposed object stream " + std::to_string(obj_stream_number) + " has wrong type")); + } + + if (!(dict.getKey("/N").isInteger() && dict.getKey("/First").isInteger())) { + throw damagedPDF( + ("object stream " + std::to_string(obj_stream_number) + " has incorrect keys")); + } + + int n = dict.getKey("/N").getIntValueAsInt(); + int first = dict.getKey("/First").getIntValueAsInt(); + + std::map offsets; + + std::shared_ptr bp = obj_stream.getStreamData(qpdf_dl_specialized); + auto input = std::shared_ptr( + // line-break + new BufferInputSource( + (m->file->getName() + " object stream " + std::to_string(obj_stream_number)), + bp.get())); + + for (int i = 0; i < n; ++i) { + QPDFTokenizer::Token tnum = readToken(*input); + QPDFTokenizer::Token toffset = readToken(*input); + if (!(tnum.isInteger() && toffset.isInteger())) { + throw damagedPDF( + *input, + m->last_object_description, + input->getLastOffset(), + "expected integer in object stream header"); + } + + int num = QUtil::string_to_int(tnum.getValue().c_str()); + long long offset = QUtil::string_to_int(toffset.getValue().c_str()); + if (num > m->xref_table_max_id) { + continue; + } + if (num == obj_stream_number) { + QTC::TC("qpdf", "QPDF ignore self-referential object stream"); + warn(damagedPDF( + *input, + m->last_object_description, + input->getLastOffset(), + "object stream claims to contain itself")); + continue; + } + offsets[num] = toI(offset + first); + } + + // To avoid having to read the object stream multiple times, store all objects that would be + // found here in the cache. Remember that some objects stored here might have been overridden + // by new objects appended to the file, so it is necessary to recheck the xref table and only + // cache what would actually be resolved here. + m->last_object_description.clear(); + m->last_object_description += "object "; + for (auto const& iter: offsets) { + QPDFObjGen og(iter.first, 0); + auto entry = m->xref_table.find(og); + if (entry != m->xref_table.end() && entry->second.getType() == 2 && + entry->second.getObjStreamNumber() == obj_stream_number) { + int offset = iter.second; + input->seek(offset, SEEK_SET); + QPDFObjectHandle oh = readObjectInStream(input, iter.first); + updateCache(og, oh.getObj(), end_before_space, end_after_space); + } else { + QTC::TC("qpdf", "QPDF not caching overridden objstm object"); + } + } +} + +QPDFObjectHandle QPDF::newIndirect(QPDFObjGen const& og, std::shared_ptr const& obj) { obj->setDefaultDescription(this, og); return {obj}; } +void +QPDF::updateCache( + QPDFObjGen const& og, + std::shared_ptr const& object, + qpdf_offset_t end_before_space, + qpdf_offset_t end_after_space) +{ + object->setObjGen(this, og); + if (isCached(og)) { + auto& cache = m->obj_cache[og]; + cache.object->assign(object); + cache.end_before_space = end_before_space; + cache.end_after_space = end_after_space; + } else { + m->obj_cache[og] = ObjCache(object, end_before_space, end_after_space); + } +} + +bool +QPDF::isCached(QPDFObjGen const& og) +{ + return m->obj_cache.count(og) != 0; +} + +bool +QPDF::isUnresolved(QPDFObjGen const& og) +{ + return !isCached(og) || m->obj_cache[og].object->isUnresolved(); +} + +QPDFObjGen +QPDF::nextObjGen() +{ + int max_objid = toI(getObjectCount()); + if (max_objid == std::numeric_limits::max()) { + throw std::range_error("max object id is too high to create new objects"); + } + return QPDFObjGen(max_objid + 1, 0); +} + +QPDFObjectHandle +QPDF::makeIndirectFromQPDFObject(std::shared_ptr const& obj) +{ + QPDFObjGen next{nextObjGen()}; + m->obj_cache[next] = ObjCache(obj, -1, -1); + return newIndirect(next, m->obj_cache[next].object); +} + QPDFObjectHandle QPDF::makeIndirectObject(QPDFObjectHandle oh) { if (!oh) { throw std::logic_error("attempted to make an uninitialized QPDFObjectHandle indirect"); } - return m->objects.make_indirect(oh.getObj()); + return makeIndirectFromQPDFObject(oh.getObj()); } QPDFObjectHandle QPDF::newReserved() { - return m->objects.make_indirect(QPDF_Reserved::create()); + return makeIndirectFromQPDFObject(QPDF_Reserved::create()); } QPDFObjectHandle QPDF::newIndirectNull() { - return m->objects.make_indirect(QPDF_Null::create()); + return makeIndirectFromQPDFObject(QPDF_Null::create()); } QPDFObjectHandle QPDF::newStream() { - return m->objects.make_indirect( - QPDF_Stream::create(this, m->objects.next_id(), QPDFObjectHandle::newDictionary(), 0, 0)); + return makeIndirectFromQPDFObject( + QPDF_Stream::create(this, nextObjGen(), QPDFObjectHandle::newDictionary(), 0, 0)); } QPDFObjectHandle @@ -555,40 +2093,93 @@ QPDF::newStream(std::string const& data) return result; } +std::shared_ptr +QPDF::getObjectForParser(int id, int gen, bool parse_pdf) +{ + // This method is called by the parser and therefore must not resolve any objects. + auto og = QPDFObjGen(id, gen); + if (auto iter = m->obj_cache.find(og); iter != m->obj_cache.end()) { + return iter->second.object; + } + if (m->xref_table.count(og) || !m->parsed) { + return m->obj_cache.insert({og, QPDF_Unresolved::create(this, og)}).first->second.object; + } + if (parse_pdf) { + return QPDF_Null::create(); + } + return m->obj_cache.insert({og, QPDF_Null::create(this, og)}).first->second.object; +} + +std::shared_ptr +QPDF::getObjectForJSON(int id, int gen) +{ + auto og = QPDFObjGen(id, gen); + auto [it, inserted] = m->obj_cache.try_emplace(og); + auto& obj = it->second.object; + if (inserted) { + obj = (m->parsed && !m->xref_table.count(og)) ? QPDF_Null::create(this, og) + : QPDF_Unresolved::create(this, og); + } + return obj; +} + QPDFObjectHandle QPDF::getObject(QPDFObjGen const& og) { - return m->objects.get(og); + if (auto it = m->obj_cache.find(og); it != m->obj_cache.end()) { + return {it->second.object}; + } else if (m->parsed && !m->xref_table.count(og)) { + return QPDF_Null::create(); + } else { + auto result = m->obj_cache.try_emplace(og, QPDF_Unresolved::create(this, og), -1, -1); + return {result.first->second.object}; + } } QPDFObjectHandle -QPDF::getObject(int id, int gen) +QPDF::getObject(int objid, int generation) { - return m->objects.get(id, gen); + return getObject(QPDFObjGen(objid, generation)); } QPDFObjectHandle QPDF::getObjectByObjGen(QPDFObjGen const& og) { - return m->objects.get(og); + return getObject(og); } QPDFObjectHandle -QPDF::getObjectByID(int id, int gen) +QPDF::getObjectByID(int objid, int generation) { - return m->objects.get(id, gen); + return getObject(QPDFObjGen(objid, generation)); } void -QPDF::replaceObject(int id, int gen, QPDFObjectHandle replacement) +QPDF::replaceObject(int objid, int generation, QPDFObjectHandle oh) { - m->objects.replace(QPDFObjGen(id, gen), replacement); + replaceObject(QPDFObjGen(objid, generation), oh); } void -QPDF::replaceObject(QPDFObjGen const& og, QPDFObjectHandle replacement) +QPDF::replaceObject(QPDFObjGen const& og, QPDFObjectHandle oh) { - m->objects.replace(og, replacement); + if (!oh || (oh.isIndirect() && !(oh.isStream() && oh.getObjGen() == og))) { + QTC::TC("qpdf", "QPDF replaceObject called with indirect object"); + throw std::logic_error("QPDF::replaceObject called with indirect object handle"); + } + updateCache(og, oh.getObj(), -1, -1); +} + +void +QPDF::removeObject(QPDFObjGen og) +{ + m->xref_table.erase(og); + if (auto cached = m->obj_cache.find(og); cached != m->obj_cache.end()) { + // Take care of any object handles that may be floating around. + cached->second.object->assign(QPDF_Null::create()); + cached->second.object->setObjGen(nullptr, QPDFObjGen()); + m->obj_cache.erase(cached); + } } void @@ -599,7 +2190,7 @@ QPDF::replaceReserved(QPDFObjectHandle reserved, QPDFObjectHandle replacement) if (!(tc == ::ot_reserved || tc == ::ot_null)) { throw std::logic_error("replaceReserved called with non-reserved object"); } - m->objects.replace(reserved.getObjGen(), replacement); + replaceObject(reserved.getObjGen(), replacement); } QPDFObjectHandle @@ -851,7 +2442,7 @@ QPDF::copyStreamData(QPDFObjectHandle result, QPDFObjectHandle foreign) } else { auto foreign_stream_data = std::make_shared( foreign_stream_qpdf.m->encp, - foreign_stream_qpdf.m->file_sp, + foreign_stream_qpdf.m->file, foreign.getObjGen(), stream->getParsedOffset(), stream->getLength(), @@ -865,13 +2456,16 @@ QPDF::copyStreamData(QPDFObjectHandle result, QPDFObjectHandle foreign) void QPDF::swapObjects(int objid1, int generation1, int objid2, int generation2) { - m->objects.swap(QPDFObjGen(objid1, generation1), QPDFObjGen(objid2, generation2)); + swapObjects(QPDFObjGen(objid1, generation1), QPDFObjGen(objid2, generation2)); } void QPDF::swapObjects(QPDFObjGen const& og1, QPDFObjGen const& og2) { - m->objects.swap(og1, og2); + // Force objects to be read from the input source if needed, then swap them in the cache. + resolve(og1); + resolve(og2); + m->obj_cache[og1].object->swapWith(m->obj_cache[og2].object); } unsigned long long @@ -932,13 +2526,13 @@ QPDF::getExtensionLevel() QPDFObjectHandle QPDF::getTrailer() { - return m->objects.trailer(); + return m->trailer; } QPDFObjectHandle QPDF::getRoot() { - auto root = m->objects.trailer().getKey("/Root"); + QPDFObjectHandle root = m->trailer.getKey("/Root"); if (!root.isDictionary()) { throw damagedPDF("", 0, "unable to find /Root dictionary"); } else if ( @@ -954,10 +2548,145 @@ QPDF::getRoot() std::map QPDF::getXRefTable() { - if (!m->objects.xref_table().initialized()) { + return getXRefTableInternal(); +} + +std::map const& +QPDF::getXRefTableInternal() +{ + if (!m->parsed) { throw std::logic_error("QPDF::getXRefTable called before parsing."); } - return m->objects.xref_table().as_map(); + + return m->xref_table; +} + +size_t +QPDF::tableSize() +{ + // If obj_cache is dense, accommodate all object in tables,else accommodate only original + // objects. + auto max_xref = m->xref_table.size() ? m->xref_table.crbegin()->first.getObj() : 0; + auto max_obj = m->obj_cache.size() ? m->obj_cache.crbegin()->first.getObj() : 0; + auto max_id = std::numeric_limits::max() - 1; + if (max_obj >= max_id || max_xref >= max_id) { + // Temporary fix. Long-term solution is + // - QPDFObjGen to enforce objgens are valid and sensible + // - xref table and obj cache to protect against insertion of impossibly large obj ids + stopOnError("Impossibly large object id encountered."); + } + if (max_obj < 1.1 * std::max(toI(m->obj_cache.size()), max_xref)) { + return toS(++max_obj); + } + return toS(++max_xref); +} + +std::vector +QPDF::getCompressibleObjVector() +{ + return getCompressibleObjGens(); +} + +std::vector +QPDF::getCompressibleObjSet() +{ + return getCompressibleObjGens(); +} + +template +std::vector +QPDF::getCompressibleObjGens() +{ + // Return a list of objects that are allowed to be in object streams. Walk through the objects + // by traversing the document from the root, including a traversal of the pages tree. This + // makes that objects that are on the same page are more likely to be in the same object stream, + // which is slightly more efficient, particularly with linearized files. This is better than + // iterating through the xref table since it avoids preserving orphaned items. + + // Exclude encryption dictionary, if any + QPDFObjectHandle encryption_dict = m->trailer.getKey("/Encrypt"); + QPDFObjGen encryption_dict_og = encryption_dict.getObjGen(); + + const size_t max_obj = getObjectCount(); + std::vector visited(max_obj, false); + std::vector queue; + queue.reserve(512); + queue.push_back(m->trailer); + std::vector result; + if constexpr (std::is_same_v) { + result.reserve(m->obj_cache.size()); + } else if constexpr (std::is_same_v) { + result.resize(max_obj + 1U, false); + } else { + throw std::logic_error("Unsupported type in QPDF::getCompressibleObjGens"); + } + while (!queue.empty()) { + auto obj = queue.back(); + queue.pop_back(); + if (obj.getObjectID() > 0) { + QPDFObjGen og = obj.getObjGen(); + const size_t id = toS(og.getObj() - 1); + if (id >= max_obj) { + throw std::logic_error( + "unexpected object id encountered in getCompressibleObjGens"); + } + if (visited[id]) { + QTC::TC("qpdf", "QPDF loop detected traversing objects"); + continue; + } + + // Check whether this is the current object. If not, remove it (which changes it into a + // direct null and therefore stops us from revisiting it) and move on to the next object + // in the queue. + auto upper = m->obj_cache.upper_bound(og); + if (upper != m->obj_cache.end() && upper->first.getObj() == og.getObj()) { + removeObject(og); + continue; + } + + visited[id] = true; + + if (og == encryption_dict_og) { + QTC::TC("qpdf", "QPDF exclude encryption dictionary"); + } else if (!(obj.isStream() || + (obj.isDictionaryOfType("/Sig") && obj.hasKey("/ByteRange") && + obj.hasKey("/Contents")))) { + if constexpr (std::is_same_v) { + result.push_back(og); + } else if constexpr (std::is_same_v) { + result[id + 1U] = true; + } + } + } + if (obj.isStream()) { + QPDFObjectHandle dict = obj.getDict(); + std::set keys = dict.getKeys(); + for (auto iter = keys.rbegin(); iter != keys.rend(); ++iter) { + std::string const& key = *iter; + QPDFObjectHandle value = dict.getKey(key); + if (key == "/Length") { + // omit stream lengths + if (value.isIndirect()) { + QTC::TC("qpdf", "QPDF exclude indirect length"); + } + } else { + queue.push_back(value); + } + } + } else if (obj.isDictionary()) { + std::set keys = obj.getKeys(); + for (auto iter = keys.rbegin(); iter != keys.rend(); ++iter) { + queue.push_back(obj.getKey(*iter)); + } + } else if (obj.isArray()) { + int n = obj.getArrayNItems(); + for (int i = 1; i <= n; ++i) { + queue.push_back(obj.getArrayItem(n - i)); + } + } + } + + return result; } bool @@ -1037,7 +2766,7 @@ QPDF::pipeStreamData( { return pipeStreamData( m->encp, - m->file_sp, + m->file, *this, og, offset, diff --git a/libqpdf/QPDFJob.cc b/libqpdf/QPDFJob.cc index ab57dd1..9ed1685 100644 --- a/libqpdf/QPDFJob.cc +++ b/libqpdf/QPDFJob.cc @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -25,7 +26,6 @@ #include #include #include -#include #include #include diff --git a/libqpdf/QPDFWriter.cc b/libqpdf/QPDFWriter.cc index 4ab0cab..a1ae23c 100644 --- a/libqpdf/QPDFWriter.cc +++ b/libqpdf/QPDFWriter.cc @@ -14,10 +14,10 @@ #include #include #include +#include #include #include #include -#include #include #include #include @@ -1698,6 +1698,7 @@ QPDFWriter::writeObjectStream(QPDFObjectHandle object) if (obj_to_write.isStream()) { // This condition occurred in a fuzz input. Ideally we should block it at parse // time, but it's not clear to me how to construct a case for this. + QTC::TC("qpdf", "QPDFWriter stream in ostream"); obj_to_write.warnIfPossible("stream found inside object stream; treating as null"); obj_to_write = QPDFObjectHandle::newNull(); } @@ -1936,26 +1937,47 @@ void QPDFWriter::preserveObjectStreams() { auto const& xref = QPDF::Writer::getXRefTable(m->pdf); - m->obj.streams_empty = !xref.object_streams(); - if (m->obj.streams_empty) { - return; - } - // This code filters out objects that are not allowed to be in object streams. In addition to - // removing objects that were erroneously included in object streams in the source PDF, it also - // prevents unreferenced objects from being included. + // Our object_to_object_stream map has to map ObjGen -> ObjGen since we may be generating object + // streams out of old objects that have generation numbers greater than zero. However in an + // existing PDF, all object stream objects and all objects in them must have generation 0 + // because the PDF spec does not provide any way to do otherwise. This code filters out objects + // that are not allowed to be in object streams. In addition to removing objects that were + // erroneously included in object streams in the source PDF, it also prevents unreferenced + // objects from being included. + auto end = xref.cend(); + m->obj.streams_empty = true; if (m->preserve_unreferenced_objects) { - QTC::TC("qpdf", "QPDFWriter preserve object streams preserve unreferenced"); - for (auto [id, stream]: xref.compressed_objects()) { - m->obj[id].object_stream = stream; + for (auto iter = xref.cbegin(); iter != end; ++iter) { + if (iter->second.getType() == 2) { + // Pdf contains object streams. + QTC::TC("qpdf", "QPDFWriter preserve object streams preserve unreferenced"); + m->obj.streams_empty = false; + m->obj[iter->first].object_stream = iter->second.getObjStreamNumber(); + } } } else { - QTC::TC("qpdf", "QPDFWriter preserve object streams"); - auto eligible = QPDF::Writer::getCompressibleObjSet(m->pdf); - for (auto [id, stream]: xref.compressed_objects()) { - if (eligible[id]) { - m->obj[id].object_stream = stream; - } else { - QTC::TC("qpdf", "QPDFWriter exclude from object stream"); + // Start by scanning for first compressed object in case we don't have any object streams to + // process. + for (auto iter = xref.cbegin(); iter != end; ++iter) { + if (iter->second.getType() == 2) { + // Pdf contains object streams. + QTC::TC("qpdf", "QPDFWriter preserve object streams"); + m->obj.streams_empty = false; + auto eligible = QPDF::Writer::getCompressibleObjSet(m->pdf); + // The object pointed to by iter may be a previous generation, in which case it is + // removed by getCompressibleObjSet. We need to restart the loop (while the object + // table may contain multiple generations of an object). + for (iter = xref.cbegin(); iter != end; ++iter) { + if (iter->second.getType() == 2) { + auto id = static_cast(iter->first.getObj()); + if (id < eligible.size() && eligible[id]) { + m->obj[iter->first].object_stream = iter->second.getObjStreamNumber(); + } else { + QTC::TC("qpdf", "QPDFWriter exclude from object stream"); + } + } + } + return; } } } diff --git a/libqpdf/QPDF_Stream.cc b/libqpdf/QPDF_Stream.cc index e64366d..8ffe919 100644 --- a/libqpdf/QPDF_Stream.cc +++ b/libqpdf/QPDF_Stream.cc @@ -10,8 +10,8 @@ #include #include #include +#include #include -#include #include #include #include diff --git a/libqpdf/QPDF_encryption.cc b/libqpdf/QPDF_encryption.cc index eacbd6c..1dd7b96 100644 --- a/libqpdf/QPDF_encryption.cc +++ b/libqpdf/QPDF_encryption.cc @@ -3,7 +3,7 @@ #include -#include +#include #include @@ -727,7 +727,7 @@ QPDF::initializeEncryption() // at /Encrypt again. Otherwise, things could go wrong if someone mutates the encryption // dictionary. - if (!m->objects.trailer().hasKey("/Encrypt")) { + if (!m->trailer.hasKey("/Encrypt")) { return; } @@ -736,7 +736,7 @@ QPDF::initializeEncryption() m->encp->encrypted = true; std::string id1; - QPDFObjectHandle id_obj = m->objects.trailer().getKey("/ID"); + QPDFObjectHandle id_obj = m->trailer.getKey("/ID"); if ((id_obj.isArray() && (id_obj.getArrayNItems() == 2) && id_obj.getArrayItem(0).isString())) { id1 = id_obj.getArrayItem(0).getStringValue(); } else { @@ -745,7 +745,7 @@ QPDF::initializeEncryption() warn(damagedPDF("trailer", "invalid /ID in trailer dictionary")); } - QPDFObjectHandle encryption_dict = m->objects.trailer().getKey("/Encrypt"); + QPDFObjectHandle encryption_dict = m->trailer.getKey("/Encrypt"); if (!encryption_dict.isDictionary()) { throw damagedPDF("/Encrypt in trailer dictionary is not a dictionary"); } diff --git a/libqpdf/QPDF_json.cc b/libqpdf/QPDF_json.cc index 13276cc..8cbbcd1 100644 --- a/libqpdf/QPDF_json.cc +++ b/libqpdf/QPDF_json.cc @@ -51,6 +51,17 @@ // ] | <- st_top // } | +static char const* JSON_PDF = ( + // force line break + "%PDF-1.3\n" + "xref\n" + "0 1\n" + "0000000000 65535 f \n" + "trailer << /Size 1 >>\n" + "startxref\n" + "9\n" + "%%EOF\n"); + // Validator methods -- these are much more performant than std::regex. static bool is_indirect_object(std::string const& v, int& obj, int& gen) @@ -256,10 +267,10 @@ class QPDF::JSONReactor: public JSON::Reactor struct StackFrame { StackFrame(state_e state) : - state(state){}; + state(state) {}; StackFrame(state_e state, QPDFObjectHandle&& object) : state(state), - object(object){}; + object(object) {}; state_e state; QPDFObjectHandle object; }; @@ -536,7 +547,7 @@ QPDF::JSONReactor::dictionaryItem(std::string const& key, JSON const& value) } else if (is_obj_key(key, obj, gen)) { this->cur_object = key; if (setNextStateIfDictionary(key, value, st_object_top)) { - next_obj = pdf.objects().get_for_json(obj, gen); + next_obj = pdf.getObjectForJSON(obj, gen); } } else { QTC::TC("qpdf", "QPDF_json bad object key"); @@ -582,7 +593,8 @@ QPDF::JSONReactor::dictionaryItem(std::string const& key, JSON const& value) this->saw_value = true; // The trailer must be a dictionary, so we can use setNextStateIfDictionary. if (setNextStateIfDictionary("trailer.value", value, st_object)) { - pdf.m->objects.xref_table().trailer(makeObject(value)); + this->pdf.m->trailer = makeObject(value); + setObjectDescription(this->pdf.m->trailer, value); } } else if (key == "stream") { // Don't need to set saw_stream here since there's already an error. @@ -740,7 +752,7 @@ QPDF::JSONReactor::makeObject(JSON const& value) int gen = 0; std::string str; if (is_indirect_object(str_v, obj, gen)) { - result = pdf.objects().get_for_json(obj, gen); + result = pdf.getObjectForJSON(obj, gen); } else if (is_unicode_string(str_v, str)) { result = QPDFObjectHandle::newUnicodeString(str); } else if (is_binary_string(str_v, str)) { @@ -774,9 +786,7 @@ QPDF::createFromJSON(std::string const& json_file) void QPDF::createFromJSON(std::shared_ptr is) { - m->pdf_version = "1.3"; - m->no_input_name = is->getName(); - m->objects.xref_table().initialize_json(); + processMemoryFile(is->getName().c_str(), JSON_PDF, strlen(JSON_PDF)); importJSON(is, true); } diff --git a/libqpdf/QPDF_linearization.cc b/libqpdf/QPDF_linearization.cc index 538d2df..9ffefd2 100644 --- a/libqpdf/QPDF_linearization.cc +++ b/libqpdf/QPDF_linearization.cc @@ -1,6 +1,6 @@ // See doc/linearization. -#include +#include #include #include @@ -130,7 +130,7 @@ QPDF::isLinearized() return false; } - auto candidate = m->objects.get(lindict_obj, 0); + auto candidate = getObjectByID(lindict_obj, 0); if (!candidate.isDictionary()) { return false; } @@ -287,9 +287,10 @@ QPDF::readHintStream(Pipeline& pl, qpdf_offset_t offset, size_t length) { QPDFObjGen og; QPDFObjectHandle H = - objects().read(false, offset, "linearization hint stream", QPDFObjGen(0, 0), og, false); - qpdf_offset_t min_end_offset = m->objects.xref_table().end_before_space(og); - qpdf_offset_t max_end_offset = m->objects.xref_table().end_after_space(og); + readObjectAtOffset(false, offset, "linearization hint stream", QPDFObjGen(0, 0), og, false); + ObjCache& oc = m->obj_cache[og]; + qpdf_offset_t min_end_offset = oc.end_before_space; + qpdf_offset_t max_end_offset = oc.end_after_space; if (!H.isStream()) { throw damagedPDF("linearization dictionary", "hint table is not a stream"); } @@ -300,11 +301,14 @@ QPDF::readHintStream(Pipeline& pl, qpdf_offset_t offset, size_t length) // increasing length to cover it, even though the specification says all objects in the // linearization parameter dictionary must be direct. We have to get the file position of the // end of length in this case. - auto length_og = Hdict.getKey("/Length").getObjGen(); - if (length_og.isIndirect()) { + QPDFObjectHandle length_obj = Hdict.getKey("/Length"); + if (length_obj.isIndirect()) { QTC::TC("qpdf", "QPDF hint table length indirect"); - min_end_offset = m->objects.xref_table().end_before_space(length_og); - max_end_offset = m->objects.xref_table().end_after_space(length_og); + // Force resolution + (void)length_obj.getIntValue(); + ObjCache& oc2 = m->obj_cache[length_obj.getObjGen()]; + min_end_offset = oc2.end_before_space; + max_end_offset = oc2.end_after_space; } else { QTC::TC("qpdf", "QPDF hint table length direct"); } @@ -441,7 +445,7 @@ QPDF::checkLinearizationInternal() for (size_t i = 0; i < toS(npages); ++i) { QPDFObjectHandle const& page = pages.at(i); QPDFObjGen og(page.getObjGen()); - if (m->objects.xref_table().type(og) == 2) { + if (m->xref_table[og].getType() == 2) { linearizationWarning( "page dictionary for page " + std::to_string(i) + " is compressed"); } @@ -457,11 +461,12 @@ QPDF::checkLinearizationInternal() break; } } - if (m->file->tell() != m->objects.xref_table().first_item_offset()) { + if (m->file->tell() != m->first_xref_item_offset) { QTC::TC("qpdf", "QPDF err /T mismatch"); linearizationWarning( - "space before first xref item (/T) mismatch (computed = " + - std::to_string(m->objects.xref_table().first_item_offset()) + + "space before first xref item (/T) mismatch " + "(computed = " + + std::to_string(m->first_xref_item_offset) + "; file = " + std::to_string(m->file->tell())); } @@ -472,7 +477,7 @@ QPDF::checkLinearizationInternal() // compressed objects are supposed to be at the end of the containing xref section if any object // streams are in use. - if (m->objects.xref_table().uncompressed_after_compressed()) { + if (m->uncompressed_after_compressed) { linearizationWarning("linearized file contains an uncompressed object after a compressed " "one in a cross-reference stream"); } @@ -480,9 +485,18 @@ QPDF::checkLinearizationInternal() // Further checking requires optimization and order calculation. Don't allow optimization to // make changes. If it has to, then the file is not properly linearized. We use the xref table // to figure out which objects are compressed and which are uncompressed. - - optimize(m->objects); - calculateLinearizationData(m->objects); + { // local scope + std::map object_stream_data; + for (auto const& iter: m->xref_table) { + QPDFObjGen const& og = iter.first; + QPDFXRefEntry const& entry = iter.second; + if (entry.getType() == 2) { + object_stream_data[og.getObj()] = entry.getObjStreamNumber(); + } + } + optimize(object_stream_data, false); + calculateLinearizationData(object_stream_data); + } // E: offset of end of first page -- Implementation note 123 says Acrobat includes on extra // object here by mistake. pdlin fails to place thumbnail images in section 9, so when @@ -499,14 +513,13 @@ QPDF::checkLinearizationInternal() qpdf_offset_t max_E = -1; for (auto const& oh: m->part6) { QPDFObjGen og(oh.getObjGen()); - auto before = m->objects.xref_table().end_before_space(og); - auto after = m->objects.xref_table().end_after_space(og); - if (before <= 0) { + if (m->obj_cache.count(og) == 0) { // All objects have to have been dereferenced to be classified. throw std::logic_error("linearization part6 object not in cache"); } - min_E = std::max(min_E, before); - max_E = std::max(max_E, after); + ObjCache const& oc = m->obj_cache[og]; + min_E = std::max(min_E, oc.end_before_space); + max_E = std::max(max_E, oc.end_after_space); } if ((p.first_page_end < min_E) || (p.first_page_end > max_E)) { QTC::TC("qpdf", "QPDF warn /E mismatch"); @@ -533,11 +546,10 @@ QPDF::maxEnd(ObjUser const& ou) } qpdf_offset_t end = 0; for (auto const& og: m->obj_user_to_objects[ou]) { - auto e = m->objects.xref_table().end_after_space(og); - if (e <= 0) { + if (m->obj_cache.count(og) == 0) { stopOnError("unknown object referenced in object user table"); } - end = std::max(end, e); + end = std::max(end, m->obj_cache[og].end_after_space); } return end; } @@ -545,40 +557,34 @@ QPDF::maxEnd(ObjUser const& ou) qpdf_offset_t QPDF::getLinearizationOffset(QPDFObjGen const& og) { - switch (m->objects.xref_table().type(og)) { + QPDFXRefEntry entry = m->xref_table[og]; + qpdf_offset_t result = 0; + switch (entry.getType()) { case 1: - return m->objects.xref_table().offset(og); + result = entry.getOffset(); + break; case 2: // For compressed objects, return the offset of the object stream that contains them. - return getLinearizationOffset( - QPDFObjGen(m->objects.xref_table().stream_number(og.getObj()), 0)); + result = getLinearizationOffset(QPDFObjGen(entry.getObjStreamNumber(), 0)); + break; default: stopOnError("getLinearizationOffset called for xref entry not of type 1 or 2"); - return 0; // unreachable + break; } + return result; } QPDFObjectHandle QPDF::getUncompressedObject(QPDFObjectHandle& obj, std::map const& object_stream_data) { - if (obj.isNull() || !object_stream_data.count(obj.getObjectID())) { + if (obj.isNull() || (object_stream_data.count(obj.getObjectID()) == 0)) { return obj; } else { int repl = (*(object_stream_data.find(obj.getObjectID()))).second; - return m->objects.get(repl, 0); - } -} - -QPDFObjectHandle -QPDF::getUncompressedObject(QPDFObjectHandle& obj, Objects const& objects) -{ - auto og = obj.getObjGen(); - if (obj.isNull() || objects.xref_table().type(og) != 2) { - return obj; + return getObject(repl, 0); } - return m->objects.get(objects.xref_table().stream_number(og.getObj()), 0); } QPDFObjectHandle @@ -586,7 +592,7 @@ QPDF::getUncompressedObject(QPDFObjectHandle& oh, QPDFWriter::ObjTable const& ob { if (obj.contains(oh)) { if (auto id = obj[oh].object_stream; id > 0) { - return oh.isNull() ? oh : m->objects.get(id, 0); + return oh.isNull() ? oh : getObject(id, 0); } } return oh; @@ -598,13 +604,15 @@ QPDF::lengthNextN(int first_object, int n) int length = 0; for (int i = 0; i < n; ++i) { QPDFObjGen og(first_object + i, 0); - auto end = m->objects.xref_table().end_after_space(og); - if (end <= 0) { + if (m->xref_table.count(og) == 0) { linearizationWarning( "no xref table entry for " + std::to_string(first_object + i) + " 0"); - continue; + } else { + if (m->obj_cache.count(og) == 0) { + stopOnError("found unknown object while calculating length for linearization data"); + } + length += toI(m->obj_cache[og].end_after_space - getLinearizationOffset(og)); } - length += toI(end - getLinearizationOffset(og)); } return length; } @@ -628,7 +636,7 @@ QPDF::checkHPageOffset( int npages = toI(pages.size()); qpdf_offset_t table_offset = adjusted_offset(m->page_offset_hints.first_page_offset); QPDFObjGen first_page_og(pages.at(0).getObjGen()); - if (m->objects.xref_table().type(first_page_og) == 0) { + if (m->xref_table.count(first_page_og) == 0) { stopOnError("supposed first page object is not known"); } qpdf_offset_t offset = getLinearizationOffset(first_page_og); @@ -639,7 +647,7 @@ QPDF::checkHPageOffset( for (int pageno = 0; pageno < npages; ++pageno) { QPDFObjGen page_og(pages.at(toS(pageno)).getObjGen()); int first_object = page_og.getObj(); - if (m->objects.xref_table().type(page_og) == 0) { + if (m->xref_table.count(page_og) == 0) { stopOnError("unknown object in page offset hint table"); } offset = getLinearizationOffset(page_og); @@ -761,7 +769,7 @@ QPDF::checkHSharedObject(std::vector const& pages, std::mapobjects.xref_table().type(og) == 0) { + if (m->xref_table.count(og) == 0) { stopOnError("unknown object in shared object hint table"); } qpdf_offset_t offset = getLinearizationOffset(og); @@ -812,7 +820,7 @@ QPDF::checkHOutlines() return; } QPDFObjGen og(outlines.getObjGen()); - if (m->objects.xref_table().type(og) == 0) { + if (m->xref_table.count(og) == 0) { stopOnError("unknown object in outlines hint table"); } qpdf_offset_t offset = getLinearizationOffset(og); @@ -831,7 +839,8 @@ QPDF::checkHOutlines() std::to_string(table_length) + "; computed = " + std::to_string(length)); } } else { - linearizationWarning("incorrect first object number in outline hints table."); + linearizationWarning("incorrect first object number in outline " + "hints table."); } } else { linearizationWarning("incorrect object count in outline hint table"); @@ -1159,7 +1168,7 @@ QPDF::calculateLinearizationData(T const& object_stream_data) // Map all page objects to the containing object stream. This should be a no-op in a // properly linearized file. for (auto oh: getAllPages()) { - pages.emplace_back(getUncompressedObject(oh, object_stream_data)); + pages.push_back(getUncompressedObject(oh, object_stream_data)); } } int npages = toI(pages.size()); @@ -1430,9 +1439,9 @@ QPDF::pushOutlinesToPart( m->c_outline_data.first_object = outlines_og.getObj(); m->c_outline_data.nobjects = 1; lc_outlines.erase(outlines_og); - part.emplace_back(outlines); + part.push_back(outlines); for (auto const& og: lc_outlines) { - part.emplace_back(m->objects.get(og)); + part.push_back(getObject(og)); ++m->c_outline_data.nobjects; } } diff --git a/libqpdf/QPDF_objects.cc b/libqpdf/QPDF_objects.cc deleted file mode 100644 index 9e82df9..0000000 --- a/libqpdf/QPDF_objects.cc +++ /dev/null @@ -1,1944 +0,0 @@ -#include // include first for large file support - -#include - -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using Objects = QPDF::Objects; -using Xref_table = Objects::Xref_table; - -namespace -{ - class InvalidInputSource final: public InputSource - { - public: - InvalidInputSource(std::string const& name) : - name(name) - { - } - ~InvalidInputSource() final = default; - qpdf_offset_t - findAndSkipNextEOL() final - { - throwException(); - return 0; - } - std::string const& - getName() const final - { - return name; - } - qpdf_offset_t - tell() final - { - throwException(); - return 0; - } - void - seek(qpdf_offset_t offset, int whence) final - { - throwException(); - } - void - rewind() final - { - throwException(); - } - size_t - read(char* buffer, size_t length) final - { - throwException(); - return 0; - } - void - unreadCh(char ch) final - { - throwException(); - } - - private: - void - throwException() - { - throw std::logic_error("QPDF operation attempted on a QPDF object with no input " - "source. QPDF operations are invalid before processFile (or " - "another process method) or after closeInputSource"); - } - - std::string const& name; - }; -} // namespace - -bool -QPDF::findStartxref() -{ - if (readToken(*m->file).isWord("startxref") && readToken(*m->file).isInteger()) { - // Position in front of offset token - m->file->seek(m->file->getLastOffset(), SEEK_SET); - return true; - } - return false; -} - -void -Xref_table::initialize_empty() -{ - initialized_ = true; - trailer_ = QPDFObjectHandle::newDictionary(); - auto rt = qpdf.makeIndirectObject(QPDFObjectHandle::newDictionary()); - auto pgs = qpdf.makeIndirectObject(QPDFObjectHandle::newDictionary()); - pgs.replaceKey("/Type", QPDFObjectHandle::newName("/Pages")); - pgs.replaceKey("/Kids", QPDFObjectHandle::newArray()); - pgs.replaceKey("/Count", QPDFObjectHandle::newInteger(0)); - rt.replaceKey("/Type", QPDFObjectHandle::newName("/Catalog")); - rt.replaceKey("/Pages", pgs); - trailer_.replaceKey("/Root", rt); - trailer_.replaceKey("/Size", QPDFObjectHandle::newInteger(3)); -} - -void -Xref_table::initialize_json() -{ - initialized_ = true; - table.resize(1); - trailer_ = QPDFObjectHandle::newDictionary(); - trailer_.replaceKey("/Size", QPDFObjectHandle::newInteger(1)); -} - -void -Xref_table::initialize() -{ - // PDF spec says %%EOF must be found within the last 1024 bytes of/ the file. We add an extra - // 30 characters to leave room for the startxref stuff. - file->seek(0, SEEK_END); - qpdf_offset_t end_offset = file->tell(); - // Sanity check on object ids. All objects must appear in xref table / stream. In all realistic - // scenarios at least 3 bytes are required. - if (max_id_ > end_offset / 3) { - max_id_ = static_cast(end_offset / 3); - } - qpdf_offset_t start_offset = (end_offset > 1054 ? end_offset - 1054 : 0); - PatternFinder sf(qpdf, &QPDF::findStartxref); - qpdf_offset_t xref_offset = 0; - if (file->findLast("startxref", start_offset, 0, sf)) { - xref_offset = QUtil::string_to_ll(read_token().getValue().c_str()); - } - - try { - if (xref_offset == 0) { - QTC::TC("qpdf", "QPDF can't find startxref"); - throw damaged_pdf("can't find startxref"); - } - try { - read(xref_offset); - } catch (QPDFExc&) { - throw; - } catch (std::exception& e) { - throw damaged_pdf(std::string("error reading xref: ") + e.what()); - } - } catch (QPDFExc& e) { - if (attempt_recovery_) { - reconstruct(e); - QTC::TC("qpdf", "QPDF reconstructed xref table"); - } else { - throw; - } - } - - initialized_ = true; -} - -void -Xref_table::reconstruct(QPDFExc& e) -{ - if (reconstructed_) { - // Avoid xref reconstruction infinite loops. This is getting very hard to reproduce because - // qpdf is throwing many fewer exceptions while parsing. Most situations are warnings now. - throw e; - } - - // If recovery generates more than 1000 warnings, the file is so severely damaged that there - // probably is no point trying to continue. - const auto max_warnings = qpdf.m->warnings.size() + 1000U; - auto check_warnings = [this, max_warnings]() { - if (qpdf.m->warnings.size() > max_warnings) { - throw damaged_pdf("too many errors while reconstructing cross-reference table"); - } - }; - - reconstructed_ = true; - // We may find more objects, which may contain dangling references. - qpdf.m->fixed_dangling_refs = false; - - warn_damaged("file is damaged"); - qpdf.warn(e); - warn_damaged("Attempting to reconstruct cross-reference table"); - - // Delete all references to type 1 (uncompressed) objects - for (auto& iter: table) { - if (iter.type() == 1) { - iter = {}; - } - } - - std::vector> found_objects; - std::vector trailers; - int max_found = 0; - - file->seek(0, SEEK_END); - qpdf_offset_t eof = file->tell(); - file->seek(0, SEEK_SET); - // Don't allow very long tokens here during recovery. All the interesting tokens are covered. - static size_t const MAX_LEN = 10; - while (file->tell() < eof) { - QPDFTokenizer::Token t1 = read_token(MAX_LEN); - qpdf_offset_t token_start = file->tell() - toO(t1.getValue().length()); - if (t1.isInteger()) { - auto pos = file->tell(); - QPDFTokenizer::Token t2 = read_token(MAX_LEN); - if (t2.isInteger() && read_token(MAX_LEN).isWord("obj")) { - int obj = QUtil::string_to_int(t1.getValue().c_str()); - int gen = QUtil::string_to_int(t2.getValue().c_str()); - if (obj <= max_id_) { - found_objects.emplace_back(obj, gen, token_start); - if (obj > max_found) { - max_found = obj; - } - } else { - warn_damaged("ignoring object with impossibly large id " + std::to_string(obj)); - } - } - file->seek(pos, SEEK_SET); - } else if (!trailer_ && t1.isWord("trailer")) { - trailers.emplace_back(file->tell()); - } - file->findAndSkipNextEOL(); - } - - table.resize(toS(max_found) + 1); - - for (auto tr: trailers) { - file->seek(tr, SEEK_SET); - auto t = read_trailer(); - if (!t.isDictionary()) { - // Oh well. It was worth a try. - } else { - trailer_ = t; - break; - } - check_warnings(); - } - - auto rend = found_objects.rend(); - for (auto it = found_objects.rbegin(); it != rend; it++) { - auto [obj, gen, token_start] = *it; - insert(obj, 1, token_start, gen); - check_warnings(); - } - - if (!trailer_) { - qpdf_offset_t max_offset{0}; - // If there are any xref streams, take the last one to appear. - int i = -1; - for (auto const& item: table) { - ++i; - if (item.type() != 1) { - continue; - } - auto oh = objects.get(i, item.gen()); - try { - if (!oh.isStreamOfType("/XRef")) { - continue; - } - } catch (std::exception&) { - continue; - } - auto offset = item.offset(); - if (offset > max_offset) { - max_offset = offset; - trailer_ = oh.getDict(); - } - check_warnings(); - } - if (max_offset > 0) { - try { - read(max_offset); - } catch (std::exception&) { - throw damaged_pdf( - "error decoding candidate xref stream while recovering damaged file"); - } - QTC::TC("qpdf", "QPDF recover xref stream"); - } - } - - if (!trailer_) { - // We could check the last encountered object to see if it was an xref stream. If so, we - // could try to get the trailer from there. This may make it possible to recover files with - // bad startxref pointers even when they have object streams. - - throw damaged_pdf("unable to find trailer dictionary while recovering damaged file"); - } - if (table.empty()) { - // We cannot check for an empty xref table in parse because empty tables are valid when - // creating QPDF objects from JSON. - throw damaged_pdf("unable to find objects while recovering damaged file"); - } - check_warnings(); - if (!initialized_) { - initialized_ = true; - qpdf.getAllPages(); - check_warnings(); - if (qpdf.m->all_pages.empty()) { - initialized_ = false; - throw damaged_pdf("unable to find any pages while recovering damaged file"); - } - } - // We could iterate through the objects looking for streams and try to find objects inside of - // them, but it's probably not worth the trouble. Acrobat can't recover files with any errors - // in an xref stream, and this would be a real long shot anyway. If we wanted to do anything - // that involved looking at stream contents, we'd also have to call initializeEncryption() here. - // It's safe to call it more than once. -} - -void -Xref_table::read(qpdf_offset_t xref_offset) -{ - std::map free_table; - std::set visited; - while (xref_offset) { - visited.insert(xref_offset); - char buf[7]; - memset(buf, 0, sizeof(buf)); - file->seek(xref_offset, SEEK_SET); - // Some files miss the mark a little with startxref. We could do a better job of searching - // in the neighborhood for something that looks like either an xref table or stream, but the - // simple heuristic of skipping whitespace can help with the xref table case and is harmless - // with the stream case. - bool done = false; - bool skipped_space = false; - while (!done) { - char ch; - if (1 == file->read(&ch, 1)) { - if (QUtil::is_space(ch)) { - skipped_space = true; - } else { - file->unreadCh(ch); - done = true; - } - } else { - QTC::TC("qpdf", "QPDF eof skipping spaces before xref", skipped_space ? 0 : 1); - done = true; - } - } - - file->read(buf, sizeof(buf) - 1); - // The PDF spec says xref must be followed by a line terminator, but files exist in the wild - // where it is terminated by arbitrary whitespace. - if ((strncmp(buf, "xref", 4) == 0) && QUtil::is_space(buf[4])) { - if (skipped_space) { - QTC::TC("qpdf", "QPDF xref skipped space"); - warn_damaged("extraneous whitespace seen before xref"); - } - QTC::TC( - "qpdf", - "QPDF xref space", - ((buf[4] == '\n') ? 0 - : (buf[4] == '\r') ? 1 - : (buf[4] == ' ') ? 2 - : 9999)); - int skip = 4; - // buf is null-terminated, and QUtil::is_space('\0') is false, so this won't overrun. - while (QUtil::is_space(buf[skip])) { - ++skip; - } - xref_offset = process_section(xref_offset + skip); - } else { - xref_offset = read_stream(xref_offset); - } - if (visited.count(xref_offset) != 0) { - QTC::TC("qpdf", "QPDF xref loop"); - throw damaged_pdf("loop detected following xref tables"); - } - } - - if (!trailer_) { - throw damaged_pdf("unable to find trailer while reading xref"); - } - int size = trailer_.getKey("/Size").getIntValueAsInt(); - - if (size < 3) { - throw damaged_pdf("too few objects - file can't have a page tree"); - } - - // We are no longer reporting what the highest id in the xref table is. I don't think it adds - // anything. If we want to report more detail, we should report the total number of missing - // entries, including missing entries before the last actual entry. -} - -Xref_table::Subsection -Xref_table::subsection(std::string const& line) -{ - auto terminate = [this]() -> void { - QTC::TC("qpdf", "QPDF invalid xref"); - throw damaged_table("xref syntax invalid"); - }; - - // is_space and is_digit both return false on '\0', so this will not overrun the null-terminated - // buffer. - char const* p = line.c_str(); - char const* start = line.c_str(); - - // Skip zero or more spaces - while (QUtil::is_space(*p)) { - ++p; - } - // Require digit - if (!QUtil::is_digit(*p)) { - terminate(); - } - // Gather digits - std::string obj_str; - while (QUtil::is_digit(*p)) { - obj_str.append(1, *p++); - } - // Require space - if (!QUtil::is_space(*p)) { - terminate(); - } - // Skip spaces - while (QUtil::is_space(*p)) { - ++p; - } - // Require digit - if (!QUtil::is_digit(*p)) { - terminate(); - } - // Gather digits - std::string num_str; - while (QUtil::is_digit(*p)) { - num_str.append(1, *p++); - } - // Skip any space including line terminators - while (QUtil::is_space(*p)) { - ++p; - } - auto obj = QUtil::string_to_int(obj_str.c_str()); - auto count = QUtil::string_to_int(num_str.c_str()); - if (obj > max_id() || count > max_id() || (obj + count) > max_id()) { - throw damaged_table("xref table subsection header contains impossibly large entry"); - } - return {obj, count, file->getLastOffset() + toI(p - start)}; -} - -std::vector -Xref_table::bad_subsections(std::string& line, qpdf_offset_t start) -{ - std::vector result; - file->seek(start, SEEK_SET); - - while (true) { - line.assign(50, '\0'); - file->read(line.data(), line.size()); - auto [obj, num, offset] = result.emplace_back(subsection(line)); - file->seek(offset, SEEK_SET); - for (qpdf_offset_t i = obj; i - num < obj; ++i) { - if (!std::get<0>(read_entry())) { - QTC::TC("qpdf", "QPDF invalid xref entry"); - throw damaged_table("invalid xref entry (obj=" + std::to_string(i) + ")"); - } - } - qpdf_offset_t pos = file->tell(); - if (read_token().isWord("trailer")) { - return result; - } else { - file->seek(pos, SEEK_SET); - } - } -} - -// Optimistically read and parse all subsection headers. If an error is encountered return the -// result of bad_subsections. -std::vector -Xref_table::subsections(std::string& line) -{ - auto recovery_offset = file->tell(); - try { - std::vector result; - - while (true) { - line.assign(50, '\0'); - file->read(line.data(), line.size()); - auto& sub = result.emplace_back(subsection(line)); - auto count = std::get<1>(sub); - auto offset = std::get<2>(sub); - file->seek(offset + 20 * toO(count) - 1, SEEK_SET); - file->read(line.data(), 1); - if (!(line[0] == '\n' || line[0] == '\r')) { - return bad_subsections(line, recovery_offset); - } - qpdf_offset_t pos = file->tell(); - if (read_token().isWord("trailer")) { - return result; - } else { - file->seek(pos, SEEK_SET); - } - } - } catch (...) { - return bad_subsections(line, recovery_offset); - } -} - -// Returns (success, f1, f2, type). -std::tuple -Xref_table::read_bad_entry() -{ - qpdf_offset_t f1{0}; - int f2{0}; - char type{'\0'}; - // Reposition after initial read attempt and reread. - file->seek(file->getLastOffset(), SEEK_SET); - auto line = file->readLine(30); - - // is_space and is_digit both return false on '\0', so this will not overrun the null-terminated - // buffer. - char const* p = line.data(); - - // Skip zero or more spaces. There aren't supposed to be any. - bool invalid = false; - while (QUtil::is_space(*p)) { - ++p; - QTC::TC("qpdf", "QPDF ignore first space in xref entry"); - invalid = true; - } - // Require digit - if (!QUtil::is_digit(*p)) { - return {false, 0, 0, '\0'}; - } - // Gather digits - std::string f1_str; - while (QUtil::is_digit(*p)) { - f1_str.append(1, *p++); - } - // Require space - if (!QUtil::is_space(*p)) { - return {false, 0, 0, '\0'}; - } - if (QUtil::is_space(*(p + 1))) { - QTC::TC("qpdf", "QPDF ignore first extra space in xref entry"); - invalid = true; - } - // Skip spaces - while (QUtil::is_space(*p)) { - ++p; - } - // Require digit - if (!QUtil::is_digit(*p)) { - return {false, 0, 0, '\0'}; - } - // Gather digits - std::string f2_str; - while (QUtil::is_digit(*p)) { - f2_str.append(1, *p++); - } - // Require space - if (!QUtil::is_space(*p)) { - return {false, 0, 0, '\0'}; - } - if (QUtil::is_space(*(p + 1))) { - QTC::TC("qpdf", "QPDF ignore second extra space in xref entry"); - invalid = true; - } - // Skip spaces - while (QUtil::is_space(*p)) { - ++p; - } - if ((*p == 'f') || (*p == 'n')) { - type = *p; - } else { - return {false, 0, 0, '\0'}; - } - if ((f1_str.length() != 10) || (f2_str.length() != 5)) { - QTC::TC("qpdf", "QPDF ignore length error xref entry"); - invalid = true; - } - - if (invalid) { - qpdf.warn(damaged_table("accepting invalid xref table entry")); - } - - f1 = QUtil::string_to_ll(f1_str.c_str()); - f2 = QUtil::string_to_int(f2_str.c_str()); - - return {true, f1, f2, type}; -} - -// Optimistically read and parse xref entry. If entry is bad, call read_bad_xrefEntry and return -// result. Returns (success, f1, f2, type). -std::tuple -Xref_table::read_entry() -{ - qpdf_offset_t f1{0}; - int f2{0}; - char type{'\0'}; - std::array line; - f1 = 0; - f2 = 0; - if (file->read(line.data(), 20) != 20) { - // C++20: [[unlikely]] - return {false, 0, 0, '\0'}; - } - line[20] = '\0'; - char const* p = line.data(); - - int f1_len = 0; - int f2_len = 0; - - // is_space and is_digit both return false on '\0', so this will not overrun the null-terminated - // buffer. - - // Gather f1 digits. NB No risk of overflow as 9'999'999'999 < max long long. - while (*p == '0') { - ++f1_len; - ++p; - } - while (QUtil::is_digit(*p) && f1_len++ < 10) { - f1 *= 10; - f1 += *p++ - '0'; - } - // Require space - if (!QUtil::is_space(*p++)) { - // Entry doesn't start with space or digit. - // C++20: [[unlikely]] - return {false, 0, 0, '\0'}; - } - // Gather digits. NB No risk of overflow as 99'999 < max int. - while (*p == '0') { - ++f2_len; - ++p; - } - while (QUtil::is_digit(*p) && f2_len++ < 5) { - f2 *= 10; - f2 += static_cast(*p++ - '0'); - } - if (QUtil::is_space(*p++) && (*p == 'f' || *p == 'n')) { - // C++20: [[likely]] - type = *p; - // No test for valid line[19]. - if (*(++p) && *(++p) && (*p == '\n' || *p == '\r') && f1_len == 10 && f2_len == 5) { - // C++20: [[likely]] - return {true, f1, f2, type}; - } - } - return read_bad_entry(); -} - -// Read a single cross-reference table section and associated trailer. -qpdf_offset_t -Xref_table::process_section(qpdf_offset_t xref_offset) -{ - file->seek(xref_offset, SEEK_SET); - std::string line; - auto subs = subsections(line); - - auto cur_trailer_offset = file->tell(); - auto cur_trailer = read_trailer(); - if (!cur_trailer.isDictionary()) { - QTC::TC("qpdf", "QPDF missing trailer"); - throw qpdf.damagedPDF("", "expected trailer dictionary"); - } - - if (!trailer_) { - unsigned int sz; - trailer_ = cur_trailer; - - if (!trailer_.hasKey("/Size")) { - QTC::TC("qpdf", "QPDF trailer lacks size"); - throw qpdf.damagedPDF("trailer", "trailer dictionary lacks /Size key"); - } - if (!trailer_.getKey("/Size").getValueAsUInt(sz)) { - QTC::TC("qpdf", "QPDF trailer size not integer"); - throw qpdf.damagedPDF("trailer", "/Size key in trailer dictionary is not an integer"); - } - if (sz >= static_cast(max_id_)) { - QTC::TC("qpdf", "QPDF trailer size impossibly large"); - throw qpdf.damagedPDF("trailer", "/Size key in trailer dictionary is impossibly large"); - } - table.resize(sz); - } - - for (auto [obj, num, offset]: subs) { - file->seek(offset, SEEK_SET); - for (qpdf_offset_t i = obj; i - num < obj; ++i) { - if (i == 0) { - // This is needed by checkLinearization() - first_item_offset_ = file->tell(); - } - // For xref_table, these will always be small enough to be ints - auto [success, f1, f2, type] = read_entry(); - if (!success) { - throw damaged_table("invalid xref entry (obj=" + std::to_string(i) + ")"); - } - if (type == 'f') { - insert_free(QPDFObjGen(toI(i), f2)); - } else { - insert(toI(i), 1, f1, f2); - } - } - qpdf_offset_t pos = file->tell(); - if (read_token().isWord("trailer")) { - break; - } else { - file->seek(pos, SEEK_SET); - } - } - - if (cur_trailer.hasKey("/XRefStm")) { - if (ignore_streams_) { - QTC::TC("qpdf", "QPDF ignoring XRefStm in trailer"); - } else { - if (cur_trailer.getKey("/XRefStm").isInteger()) { - // Read the xref stream but disregard any return value -- we'll use our trailer's - // /Prev key instead of the xref stream's. - (void)read_stream(cur_trailer.getKey("/XRefStm").getIntValue()); - } else { - throw qpdf.damagedPDF("xref stream", cur_trailer_offset, "invalid /XRefStm"); - } - } - } - - if (cur_trailer.hasKey("/Prev")) { - if (!cur_trailer.getKey("/Prev").isInteger()) { - QTC::TC("qpdf", "QPDF trailer prev not integer"); - throw qpdf.damagedPDF( - "trailer", cur_trailer_offset, "/Prev key in trailer dictionary is not an integer"); - } - QTC::TC("qpdf", "QPDF prev key in trailer dictionary"); - return cur_trailer.getKey("/Prev").getIntValue(); - } - - return 0; -} - -// Read a single cross-reference stream. -qpdf_offset_t -Xref_table::read_stream(qpdf_offset_t xref_offset) -{ - if (!ignore_streams_) { - QPDFObjGen x_og; - QPDFObjectHandle xref_obj; - try { - xref_obj = - objects.read(false, xref_offset, "xref stream", QPDFObjGen(0, 0), x_og, true); - } catch (QPDFExc&) { - // ignore -- report error below - } - if (xref_obj.isStreamOfType("/XRef")) { - QTC::TC("qpdf", "QPDF found xref stream"); - return process_stream(xref_offset, xref_obj); - } - } - - QTC::TC("qpdf", "QPDF can't find xref"); - throw qpdf.damagedPDF("", xref_offset, "xref not found"); - return 0; // unreachable -} - -// Return the entry size of the xref stream and the processed W array. -std::pair> -Xref_table::process_W(QPDFObjectHandle& dict, std::function damaged) -{ - auto W_obj = dict.getKey("/W"); - if (!(W_obj.isArray() && W_obj.getArrayNItems() >= 3 && W_obj.getArrayItem(0).isInteger() && - W_obj.getArrayItem(1).isInteger() && W_obj.getArrayItem(2).isInteger())) { - throw damaged("Cross-reference stream does not have a proper /W key"); - } - - std::array W; - int entry_size = 0; - auto w_vector = W_obj.getArrayAsVector(); - int max_bytes = sizeof(qpdf_offset_t); - for (size_t i = 0; i < 3; ++i) { - W[i] = w_vector[i].getIntValueAsInt(); - if (W[i] > max_bytes) { - throw damaged("Cross-reference stream's /W contains impossibly large values"); - } - if (W[i] < 0) { - throw damaged("Cross-reference stream's /W contains negative values"); - } - entry_size += W[i]; - } - if (entry_size == 0) { - throw damaged("Cross-reference stream's /W indicates entry size of 0"); - } - return {entry_size, W}; -} - -// Validate Size entry and return the maximum number of entries that the xref stream can contain and -// the value of the Size entry. -std::pair -Xref_table::process_Size( - QPDFObjectHandle& dict, int entry_size, std::function damaged) -{ - // Number of entries is limited by the highest possible object id and stream size. - auto max_num_entries = std::numeric_limits::max(); - if (max_num_entries > (std::numeric_limits::max() / entry_size)) { - max_num_entries = toI(std::numeric_limits::max() / entry_size); - } - - auto Size_obj = dict.getKey("/Size"); - long long size; - if (!dict.getKey("/Size").getValueAsInt(size)) { - throw damaged("Cross-reference stream does not have a proper /Size key"); - } else if (size < 0) { - throw damaged("Cross-reference stream has a negative /Size key"); - } else if (size >= max_num_entries) { - throw damaged("Cross-reference stream has an impossibly large /Size key"); - } - // We are not validating that Size <= (Size key of parent xref / trailer). - return {max_num_entries, toS(size)}; -} - -// Return the number of entries of the xref stream and the processed Index array. -std::pair>> -Xref_table::process_Index( - QPDFObjectHandle& dict, int max_num_entries, std::function damaged) -{ - auto size = dict.getKey("/Size").getIntValueAsInt(); - auto Index_obj = dict.getKey("/Index"); - - if (Index_obj.isArray()) { - std::vector> indx; - int num_entries = 0; - auto index_vec = Index_obj.getArrayAsVector(); - if ((index_vec.size() % 2) || index_vec.size() < 2) { - throw damaged("Cross-reference stream's /Index has an invalid number of values"); - } - - int i = 0; - long long first = 0; - for (auto& val: index_vec) { - if (val.isInteger()) { - if (i % 2) { - auto count = val.getIntValue(); - if (count <= 0) { - throw damaged( - "Cross-reference stream section claims to contain " + - std::to_string(count) + " entries"); - } - // We are guarding against the possibility of num_entries * entry_size - // overflowing. We are not checking that entries are in ascending order as - // required by the spec, which probably should generate a warning. We are also - // not checking that for each subsection first object number + number of entries - // <= /Size. The spec requires us to ignore object number > /Size. - if (first > (max_num_entries - count) || - count > (max_num_entries - num_entries)) { - throw damaged( - "Cross-reference stream claims to contain too many entries: " + - std::to_string(first) + " " + std::to_string(max_num_entries) + " " + - std::to_string(num_entries)); - } - indx.emplace_back(static_cast(first), static_cast(count)); - num_entries += static_cast(count); - } else { - first = val.getIntValue(); - if (first < 0) { - throw damaged( - "Cross-reference stream's /Index contains a negative object id"); - } else if (first > max_num_entries) { - throw damaged("Cross-reference stream's /Index contains an impossibly " - "large object id"); - } - } - } else { - throw damaged( - "Cross-reference stream's /Index's item " + std::to_string(i) + - " is not an integer"); - } - i++; - } - QTC::TC("qpdf", "QPDF xref /Index is array", index_vec.size() == 2 ? 0 : 1); - return {num_entries, indx}; - } else if (Index_obj.isNull()) { - QTC::TC("qpdf", "QPDF xref /Index is null"); - return {size, {{0, size}}}; - } else { - throw damaged("Cross-reference stream does not have a proper /Index key"); - } -} - -qpdf_offset_t -Xref_table::process_stream(qpdf_offset_t xref_offset, QPDFObjectHandle& xref_obj) -{ - auto damaged = [this, xref_offset](std::string_view msg) -> QPDFExc { - return qpdf.damagedPDF("xref stream", xref_offset, msg.data()); - }; - - auto dict = xref_obj.getDict(); - - auto [entry_size, W] = process_W(dict, damaged); - auto [max_num_entries, size] = process_Size(dict, entry_size, damaged); - auto [num_entries, indx] = process_Index(dict, max_num_entries, damaged); - - std::shared_ptr bp = xref_obj.getStreamData(qpdf_dl_specialized); - size_t actual_size = bp->getSize(); - auto expected_size = toS(entry_size) * toS(num_entries); - - if (expected_size != actual_size) { - QPDFExc x = damaged( - "Cross-reference stream data has the wrong size; expected = " + - std::to_string(expected_size) + "; actual = " + std::to_string(actual_size)); - if (expected_size > actual_size) { - throw x; - } else { - qpdf.warn(x); - } - } - - if (!trailer_) { - trailer_ = dict; - if (size > toS(max_id_)) { - throw damaged("Cross-reference stream /Size entry is impossibly large"); - } - table.resize(size); - } - - bool saw_first_compressed_object = false; - - // Actual size vs. expected size check above ensures that we will not overflow any buffers here. - // We know that entry_size * num_entries is less or equal to the size of the buffer. - auto p = bp->getBuffer(); - for (auto [obj, sec_entries]: indx) { - // Process a subsection. - for (int i = 0; i < sec_entries; ++i) { - // Read this entry - std::array fields{}; - if (W[0] == 0) { - QTC::TC("qpdf", "QPDF default for xref stream field 0"); - fields[0] = 1; - } - for (size_t j = 0; j < 3; ++j) { - for (int k = 0; k < W[j]; ++k) { - fields[j] <<= 8; - fields[j] |= *p++; - } - } - - // Get the generation number. The generation number is 0 unless this is an uncompressed - // object record, in which case the generation number appears as the third field. - if (saw_first_compressed_object) { - if (fields[0] != 2) { - uncompressed_after_compressed_ = true; - } - } else if (fields[0] == 2) { - saw_first_compressed_object = true; - } - if (obj == 0) { - // This is needed by checkLinearization() - first_item_offset_ = xref_offset; - } else if (fields[0] == 0) { - // Ignore fields[2], which we don't care about in this case. This works around the - // issue of some PDF files that put invalid values, like -1, here for deleted - // objects. - insert_free(QPDFObjGen(obj, 0)); - } else { - insert(obj, toI(fields[0]), fields[1], toI(fields[2])); - } - ++obj; - } - } - - if (dict.hasKey("/Prev")) { - if (!dict.getKey("/Prev").isInteger()) { - throw qpdf.damagedPDF( - "xref stream", "/Prev key in xref stream dictionary is not an integer"); - } - QTC::TC("qpdf", "QPDF prev key in xref stream dictionary"); - return dict.getKey("/Prev").getIntValue(); - } else { - return 0; - } -} - -void -Xref_table::insert(int obj, int f0, qpdf_offset_t f1, int f2) -{ - // Populate the xref table in such a way that the first reference to an object that we see, - // which is the one in the latest xref table in which it appears, is the one that gets stored. - // This works because we are reading more recent appends before older ones. - - // If there is already an entry for this object and generation in the table, it means that a - // later xref table has registered this object. Disregard this one. - - int new_gen = f0 == 2 ? 0 : f2; - - if (!(obj > 0 && static_cast(obj) < table.size() && 0 <= f2 && new_gen < 65535)) { - // We are ignoring invalid objgens. Most will arrive here from xref reconstruction. There - // is probably no point having another warning but we could count invalid items in order to - // decide when to give up. - QTC::TC("qpdf", "QPDF xref overwrite invalid objgen"); - return; - } - - auto& entry = table[static_cast(obj)]; - auto old_type = entry.type(); - - if (!old_type && entry.gen() > 0) { - // At the moment we are processing the updates last to first and therefore the gen doesn't - // matter as long as it > 0 to distinguish it from an uninitialized entry. This will need - // to be revisited when we want to support incremental updates or more comprehensive - // checking. - QTC::TC("qpdf", "QPDF xref deleted object"); - return; - } - - if (f0 == 2 && static_cast(f1) == obj) { - qpdf.warn(qpdf.damagedPDF( - "xref stream", "self-referential object stream " + std::to_string(obj))); - return; - } - - if (old_type && entry.gen() >= new_gen) { - QTC::TC("qpdf", "QPDF xref reused object"); - return; - } - - switch (f0) { - case 1: - // f2 is generation - QTC::TC("qpdf", "QPDF xref gen > 0", (f2 > 0) ? 1 : 0); - entry = {f2, Uncompressed(f1)}; - break; - - case 2: - entry = {0, Compressed(toI(f1), f2)}; - object_streams_ = true; - break; - - default: - throw qpdf.damagedPDF( - "xref stream", "unknown xref stream entry type " + std::to_string(f0)); - break; - } -} - -void -Xref_table::insert_free(QPDFObjGen og) -{ - // At the moment we are processing the updates last to first and therefore the gen doesn't - // matter as long as it > 0 to distinguish it from an uninitialized entry. This will need to be - // revisited when we want to support incremental updates or more comprehensive checking. - if (og.getObj() < 1) { - return; - } - size_t id = static_cast(og.getObj()); - if (id < table.size() && !type(id)) { - table[id] = {1, {}}; - } -} - -QPDFObjGen -Xref_table::at_offset(qpdf_offset_t offset) const noexcept -{ - int id = 0; - int gen = 0; - qpdf_offset_t start = 0; - - int i = 0; - for (auto const& item: table) { - auto o = item.offset(); - if (start < o && o <= offset) { - start = o; - id = i; - gen = item.gen(); - } - ++i; - } - return QPDFObjGen(id, gen); -} - -std::map -Xref_table::as_map() const -{ - std::map result; - int i{0}; - for (auto const& item: table) { - switch (item.type()) { - case 0: - break; - case 1: - result.emplace(QPDFObjGen(i, item.gen()), item.offset()); - break; - case 2: - result.emplace( - QPDFObjGen(i, 0), QPDFXRefEntry(item.stream_number(), item.stream_index())); - break; - default: - throw std::logic_error("Xref_table: invalid entry type"); - } - ++i; - } - return result; -} - -void -Xref_table::show() -{ - auto& cout = *qpdf.m->log->getInfo(); - int i = -1; - for (auto const& item: table) { - ++i; - if (item.type()) { - cout << std::to_string(i) << "/" << std::to_string(item.gen()) << ": "; - switch (item.type()) { - case 1: - cout << "uncompressed; offset = " << item.offset() << "\n"; - break; - - case 2: - cout << "compressed; stream = " << item.stream_number() - << ", index = " << item.stream_index() << "\n"; - break; - - default: - throw std::logic_error( - "unknown cross-reference table type while showing xref_table"); - } - } - } -} - -// Resolve all objects in the xref table. If this triggers a xref table reconstruction abort and -// return false. Otherwise return true. -bool -Xref_table::resolve() -{ - bool may_change = !reconstructed_; - int i = -1; - for (auto& item: table) { - ++i; - if (item.type()) { - if (objects.unresolved(QPDFObjGen(i, item.gen()))) { - objects.resolve(QPDFObjGen(i, item.gen())); - if (may_change && reconstructed_) { - return false; - } - } - } - } - return true; -} - -std::vector -Objects ::all() -{ - // After fixDanglingReferences is called, all objects are in the object cache. - qpdf.fixDanglingReferences(); - std::vector result; - for (auto const& iter: table) { - result.emplace_back(iter.second.object); - } - return result; -} - -QPDFObjectHandle -Xref_table::read_trailer() -{ - qpdf_offset_t offset = file->tell(); - bool empty = false; - auto object = QPDFParser(*file, "trailer", tokenizer, nullptr, &qpdf, true).parse(empty, false); - if (empty) { - // Nothing in the PDF spec appears to allow empty objects, but they have been encountered in - // actual PDF files and Adobe Reader appears to ignore them. - qpdf.warn(qpdf.damagedPDF("trailer", "empty object treated as null")); - } else if (object.isDictionary() && read_token().isWord("stream")) { - qpdf.warn(qpdf.damagedPDF("trailer", file->tell(), "stream keyword found in trailer")); - } - // Override last_offset so that it points to the beginning of the object we just read - file->setLastOffset(offset); - return object; -} - -QPDFObjectHandle -Objects::read_object(std::string const& description, QPDFObjGen og) -{ - qpdf.setLastObjectDescription(description, og); - qpdf_offset_t offset = m->file->tell(); - bool empty = false; - - StringDecrypter decrypter{&qpdf, og}; - StringDecrypter* decrypter_ptr = m->encp->encrypted ? &decrypter : nullptr; - auto object = - QPDFParser(*m->file, m->last_object_description, m->tokenizer, decrypter_ptr, &qpdf, true) - .parse(empty, false); - if (empty) { - // Nothing in the PDF spec appears to allow empty objects, but they have been encountered in - // actual PDF files and Adobe Reader appears to ignore them. - qpdf.warn( - qpdf.damagedPDF(*m->file, m->file->getLastOffset(), "empty object treated as null")); - return object; - } - auto token = qpdf.readToken(*m->file); - if (object.isDictionary() && token.isWord("stream")) { - read_stream(object, og, offset); - token = qpdf.readToken(*m->file); - } - if (!token.isWord("endobj")) { - QTC::TC("qpdf", "QPDF err expected endobj"); - qpdf.warn(qpdf.damagedPDF("expected endobj")); - } - return object; -} - -// After reading stream dictionary and stream keyword, read rest of stream. -void -Objects::read_stream(QPDFObjectHandle& object, QPDFObjGen og, qpdf_offset_t offset) -{ - validate_stream_line_end(object, og, offset); - - // Must get offset before accessing any additional objects since resolving a previously - // unresolved indirect object will change file position. - qpdf_offset_t stream_offset = m->file->tell(); - size_t length = 0; - - try { - auto length_obj = object.getKey("/Length"); - - if (!length_obj.isInteger()) { - if (length_obj.isNull()) { - QTC::TC("qpdf", "QPDF stream without length"); - throw qpdf.damagedPDF(offset, "stream dictionary lacks /Length key"); - } - QTC::TC("qpdf", "QPDF stream length not integer"); - throw qpdf.damagedPDF(offset, "/Length key in stream dictionary is not an integer"); - } - - length = toS(length_obj.getUIntValue()); - // Seek in two steps to avoid potential integer overflow - m->file->seek(stream_offset, SEEK_SET); - m->file->seek(toO(length), SEEK_CUR); - if (!qpdf.readToken(*m->file).isWord("endstream")) { - QTC::TC("qpdf", "QPDF missing endstream"); - throw qpdf.damagedPDF("expected endstream"); - } - } catch (QPDFExc& e) { - if (m->attempt_recovery) { - qpdf.warn(e); - length = recover_stream_length(m->file_sp, og, stream_offset); - } else { - throw; - } - } - object = {QPDF_Stream::create(&qpdf, og, object, stream_offset, length)}; -} - -void -Objects::validate_stream_line_end(QPDFObjectHandle& object, QPDFObjGen og, qpdf_offset_t offset) -{ - // The PDF specification states that the word "stream" should be followed by either a carriage - // return and a newline or by a newline alone. It specifically disallowed following it by a - // carriage return alone since, in that case, there would be no way to tell whether the NL in a - // CR NL sequence was part of the stream data. However, some readers, including Adobe reader, - // accept a carriage return by itself when followed by a non-newline character, so that's what - // we do here. We have also seen files that have extraneous whitespace between the stream - // keyword and the newline. - while (true) { - char ch; - if (m->file->read(&ch, 1) == 0) { - // A premature EOF here will result in some other problem that will get reported at - // another time. - return; - } - if (ch == '\n') { - // ready to read stream data - QTC::TC("qpdf", "QPDF stream with NL only"); - return; - } - if (ch == '\r') { - // Read another character - if (m->file->read(&ch, 1) != 0) { - if (ch == '\n') { - // Ready to read stream data - QTC::TC("qpdf", "QPDF stream with CRNL"); - } else { - // Treat the \r by itself as the whitespace after endstream and start reading - // stream data in spite of not having seen a newline. - QTC::TC("qpdf", "QPDF stream with CR only"); - m->file->unreadCh(ch); - qpdf.warn(qpdf.damagedPDF( - m->file->tell(), "stream keyword followed by carriage return only")); - } - } - return; - } - if (!QUtil::is_space(ch)) { - QTC::TC("qpdf", "QPDF stream without newline"); - m->file->unreadCh(ch); - qpdf.warn(qpdf.damagedPDF( - m->file->tell(), "stream keyword not followed by proper line terminator")); - return; - } - qpdf.warn( - qpdf.damagedPDF(m->file->tell(), "stream keyword followed by extraneous whitespace")); - } -} - -QPDFObjectHandle -Objects::readObjectInStream(std::shared_ptr& input, int obj) -{ - m->last_object_description.erase(7); // last_object_description starts with "object " - m->last_object_description += std::to_string(obj); - m->last_object_description += " 0"; - - bool empty = false; - auto object = QPDFParser(*input, m->last_object_description, m->tokenizer, nullptr, &qpdf, true) - .parse(empty, false); - if (empty) { - // Nothing in the PDF spec appears to allow empty objects, but they have been encountered in - // actual PDF files and Adobe Reader appears to ignore them. - qpdf.warn(qpdf.damagedPDF(*input, input->getLastOffset(), "empty object treated as null")); - } - return object; -} - -bool -QPDF::findEndstream() -{ - // Find endstream or endobj. Position the input at that token. - auto t = readToken(*m->file, 20); - if (t.isWord("endobj") || t.isWord("endstream")) { - m->file->seek(m->file->getLastOffset(), SEEK_SET); - return true; - } - return false; -} - -size_t -Objects::recover_stream_length( - std::shared_ptr input, QPDFObjGen og, qpdf_offset_t stream_offset) -{ - // Try to reconstruct stream length by looking for endstream or endobj - qpdf.warn(qpdf.damagedPDF(*input, stream_offset, "attempting to recover stream length")); - - PatternFinder ef(qpdf, &QPDF::findEndstream); - size_t length = 0; - if (m->file->findFirst("end", stream_offset, 0, ef)) { - length = toS(m->file->tell() - stream_offset); - // Reread endstream but, if it was endobj, don't skip that. - QPDFTokenizer::Token t = qpdf.readToken(*m->file); - if (t.getValue() == "endobj") { - m->file->seek(m->file->getLastOffset(), SEEK_SET); - } - } - - if (length) { - // Make sure this is inside this object - auto found = xref.at_offset(stream_offset + toO(length)); - if (found == QPDFObjGen() || found == og) { - // If we are trying to recover an XRef stream the xref table will not contain and - // won't contain any entries, therefore we cannot check the found length. Otherwise we - // found endstream\endobj within the space allowed for this object, so we're probably - // in good shape. - } else { - QTC::TC("qpdf", "QPDF found wrong endstream in recovery"); - length = 0; - } - } - - if (length == 0) { - qpdf.warn(qpdf.damagedPDF( - *input, stream_offset, "unable to recover stream data; treating stream as empty")); - } else { - qpdf.warn(qpdf.damagedPDF( - *input, stream_offset, "recovered stream length: " + std::to_string(length))); - } - - QTC::TC("qpdf", "QPDF recovered stream length"); - return length; -} - -QPDFObjectHandle -Objects::read( - bool try_recovery, - qpdf_offset_t offset, - std::string const& description, - QPDFObjGen exp_og, - QPDFObjGen& og, - bool skip_cache_if_in_xref) -{ - bool check_og = true; - if (exp_og.getObj() == 0) { - // This method uses an expect object ID of 0 to indicate that we don't know or don't care - // what the actual object ID is at this offset. This is true when we read the xref stream - // and linearization hint streams. In this case, we don't verify the expect object - // ID/generation against what was read from the file. There is also no reason to attempt - // xref recovery if we get a failure in this case since the read attempt was not triggered - // by an xref lookup. - check_og = false; - try_recovery = false; - } - qpdf.setLastObjectDescription(description, exp_og); - - if (!m->attempt_recovery) { - try_recovery = false; - } - - // Special case: if offset is 0, just return null. Some PDF writers, in particular - // "Mac OS X 10.7.5 Quartz PDFContext", may store deleted objects in the xref table as - // "0000000000 00000 n", which is not correct, but it won't hurt anything for us to ignore - // these. - if (offset == 0) { - QTC::TC("qpdf", "QPDF bogus 0 offset", 0); - qpdf.warn(qpdf.damagedPDF(0, "object has offset 0")); - return QPDFObjectHandle::newNull(); - } - - m->file->seek(offset, SEEK_SET); - try { - QPDFTokenizer::Token tobjid = qpdf.readToken(*m->file); - bool objidok = tobjid.isInteger(); - QTC::TC("qpdf", "QPDF check objid", objidok ? 1 : 0); - if (!objidok) { - QTC::TC("qpdf", "QPDF expected n n obj"); - throw qpdf.damagedPDF(offset, "expected n n obj"); - } - QPDFTokenizer::Token tgen = qpdf.readToken(*m->file); - bool genok = tgen.isInteger(); - QTC::TC("qpdf", "QPDF check generation", genok ? 1 : 0); - if (!genok) { - throw qpdf.damagedPDF(offset, "expected n n obj"); - } - QPDFTokenizer::Token tobj = qpdf.readToken(*m->file); - - bool objok = tobj.isWord("obj"); - QTC::TC("qpdf", "QPDF check obj", objok ? 1 : 0); - - if (!objok) { - throw qpdf.damagedPDF(offset, "expected n n obj"); - } - int objid = QUtil::string_to_int(tobjid.getValue().c_str()); - int generation = QUtil::string_to_int(tgen.getValue().c_str()); - og = QPDFObjGen(objid, generation); - if (objid == 0) { - QTC::TC("qpdf", "QPDF object id 0"); - throw qpdf.damagedPDF(offset, "object with ID 0"); - } - if (check_og && (exp_og != og)) { - QTC::TC("qpdf", "QPDF err wrong objid/generation"); - QPDFExc e = qpdf.damagedPDF(offset, "expected " + exp_og.unparse(' ') + " obj"); - if (try_recovery) { - // Will be retried below - throw e; - } else { - // We can try reading the object anyway even if the ID doesn't match. - qpdf.warn(e); - } - } - } catch (QPDFExc& e) { - if (try_recovery) { - // Try again after reconstructing xref table - xref.reconstruct(e); - if (xref.type(exp_og) == 1) { - QTC::TC("qpdf", "QPDF recovered in readObjectAtOffset"); - return read(false, xref.offset(exp_og), description, exp_og, og, false); - } else { - QTC::TC("qpdf", "QPDF object gone after xref reconstruction"); - qpdf.warn(qpdf.damagedPDF( - "", - 0, - ("object " + exp_og.unparse(' ') + - " not found in file after regenerating cross reference table"))); - return QPDFObjectHandle::newNull(); - } - } else { - throw; - } - } - - QPDFObjectHandle oh = read_object(description, og); - - if (unresolved(og)) { - // Store the object in the cache here so it gets cached whether we first know the offset or - // whether we first know the object ID and generation (in which we case we would get here - // through resolve). - - // Determine the end offset of this object before and after white space. We use these - // numbers to validate linearization hint tables. Offsets and lengths of objects may imply - // the end of an object to be anywhere between these values. - qpdf_offset_t end_before_space = m->file->tell(); - - // skip over spaces - while (true) { - char ch; - if (m->file->read(&ch, 1)) { - if (!isspace(static_cast(ch))) { - m->file->seek(-1, SEEK_CUR); - break; - } - } else { - throw qpdf.damagedPDF(m->file->tell(), "EOF after endobj"); - } - } - qpdf_offset_t end_after_space = m->file->tell(); - if (skip_cache_if_in_xref && xref.type(og)) { - // Ordinarily, an object gets read here when resolved through xref table or stream. In - // the special case of the xref stream and linearization hint tables, the offset comes - // from another source. For the specific case of xref streams, the xref stream is read - // and loaded into the object cache very early in parsing. Ordinarily, when a file is - // updated by appending, items inserted into the xref table in later updates take - // precedence over earlier items. In the special case of reusing the object number - // previously used as the xref stream, we have the following order of events: - // - // * reused object gets loaded into the xref table - // * old object is read here while reading xref streams - // * original xref entry is ignored (since already in xref table) - // - // It is the second step that causes a problem. Even though the xref table is correct in - // this case, the old object is already in the cache and so effectively prevails over - // the reused object. To work around this issue, we have a special case for the xref - // stream (via the skip_cache_if_in_xref): if the object is already in the xref stream, - // don't cache what we read here. - // - // It is likely that the same bug may exist for linearization hint tables, but the - // existing code uses end_before_space and end_after_space from the cache, so fixing - // that would require more significant rework. The chances of a linearization hint - // stream being reused seems smaller because the xref stream is probably the highest - // object in the file and the linearization hint stream would be some random place in - // the middle, so I'm leaving that bug unfixed for now. If the bug were to be fixed, we - // could use !check_og in place of skip_cache_if_in_xref. - QTC::TC("qpdf", "QPDF skipping cache for known unchecked object"); - } else { - xref.linearization_offsets(toS(og.getObj()), end_before_space, end_after_space); - update_table(og, oh.getObj()); - } - } - - return oh; -} - -QPDFObject* -Objects::resolve(QPDFObjGen og) -{ - if (!unresolved(og)) { - return table[og].object.get(); - } - - if (m->resolving.count(og)) { - // This can happen if an object references itself directly or indirectly in some key that - // has to be resolved during object parsing, such as stream length. - QTC::TC("qpdf", "QPDF recursion loop in resolve"); - qpdf.warn(qpdf.damagedPDF("", "loop detected resolving object " + og.unparse(' '))); - update_table(og, QPDF_Null::create()); - return table[og].object.get(); - } - ResolveRecorder rr(&qpdf, og); - - try { - switch (xref.type(og)) { - case 0: - break; - case 1: - { - // Object stored in cache by readObjectAtOffset - QPDFObjGen a_og; - QPDFObjectHandle oh = read(true, xref.offset(og), "", og, a_og, false); - } - break; - - case 2: - resolveObjectsInStream(xref.stream_number(og.getObj())); - break; - - default: - throw qpdf.damagedPDF( - "", 0, ("object " + og.unparse('/') + " has unexpected xref entry type")); - } - } catch (QPDFExc& e) { - qpdf.warn(e); - } catch (std::exception& e) { - qpdf.warn(qpdf.damagedPDF( - "", 0, ("object " + og.unparse('/') + ": error reading object: " + e.what()))); - } - - if (unresolved(og)) { - // PDF spec says unknown objects resolve to the null object. - QTC::TC("qpdf", "QPDF resolve failure to null"); - update_table(og, QPDF_Null::create()); - } - - auto result(table[og].object); - result->setDefaultDescription(&qpdf, og); - return result.get(); -} - -void -Objects::resolveObjectsInStream(int obj_stream_number) -{ - if (m->resolved_object_streams.count(obj_stream_number)) { - return; - } - m->resolved_object_streams.insert(obj_stream_number); - // Force resolution of object stream - QPDFObjectHandle obj_stream = get(obj_stream_number, 0); - if (!obj_stream.isStream()) { - throw qpdf.damagedPDF( - "supposed object stream " + std::to_string(obj_stream_number) + " is not a stream"); - } - - QPDFObjectHandle dict = obj_stream.getDict(); - if (!dict.isDictionaryOfType("/ObjStm")) { - QTC::TC("qpdf", "QPDF ERR object stream with wrong type"); - qpdf.warn(qpdf.damagedPDF( - "supposed object stream " + std::to_string(obj_stream_number) + " has wrong type")); - } - - if (!(dict.getKey("/N").isInteger() && dict.getKey("/First").isInteger())) { - throw qpdf.damagedPDF( - ("object stream " + std::to_string(obj_stream_number) + " has incorrect keys")); - } - - int n = dict.getKey("/N").getIntValueAsInt(); - int first = dict.getKey("/First").getIntValueAsInt(); - - std::map offsets; - - std::shared_ptr bp = obj_stream.getStreamData(qpdf_dl_specialized); - auto input = std::shared_ptr( - // line-break - new BufferInputSource( - (m->file->getName() + " object stream " + std::to_string(obj_stream_number)), - bp.get())); - - qpdf_offset_t last_offset = -1; - for (int i = 0; i < n; ++i) { - QPDFTokenizer::Token tnum = qpdf.readToken(*input); - QPDFTokenizer::Token toffset = qpdf.readToken(*input); - if (!(tnum.isInteger() && toffset.isInteger())) { - throw damagedPDF( - *input, - m->last_object_description, - input->getLastOffset(), - "expected integer in object stream header"); - } - - int num = QUtil::string_to_int(tnum.getValue().c_str()); - long long offset = QUtil::string_to_int(toffset.getValue().c_str()); - if (num > xref.max_id()) { - continue; - } - if (num == obj_stream_number) { - QTC::TC("qpdf", "QPDF ignore self-referential object stream"); - qpdf.warn(damagedPDF( - *input, - m->last_object_description, - input->getLastOffset(), - "object stream claims to contain itself")); - continue; - } - if (offset <= last_offset) { - throw damagedPDF( - *input, - m->last_object_description, - input->getLastOffset(), - "expected offsets in object stream to be increasing"); - } - last_offset = offset; - - offsets[num] = toI(offset + first); - } - - // To avoid having to read the object stream multiple times, store all objects that would be - // found here in the cache. Remember that some objects stored here might have been overridden - // by new objects appended to the file, so it is necessary to recheck the xref table and only - // cache what would actually be resolved here. - m->last_object_description.clear(); - m->last_object_description += "object "; - for (auto const& iter: offsets) { - QPDFObjGen og(iter.first, 0); - if (xref.type(og) == 2 && xref.stream_number(og.getObj()) == obj_stream_number) { - int offset = iter.second; - input->seek(offset, SEEK_SET); - QPDFObjectHandle oh = readObjectInStream(input, iter.first); - update_table(og, oh.getObj()); - } else { - QTC::TC("qpdf", "QPDF not caching overridden objstm object"); - } - } -} - -Objects::~Objects() -{ - // If two objects are mutually referential (through each object having an array or dictionary - // that contains an indirect reference to the other), the circular references in the - // std::shared_ptr objects will prevent the objects from being deleted. Walk through all objects - // in the object cache, which is those objects that we read from the file, and break all - // resolved indirect references by replacing them with an internal object type representing that - // they have been destroyed. Note that we can't break references like this at any time when the - // QPDF object is active. The call to reset also causes all direct QPDFObjectHandle objects that - // are reachable from this object to release their association with this QPDF. Direct objects - // are not destroyed since they can be moved to other QPDF objects safely. - - for (auto const& iter: table) { - iter.second.object->disconnect(); - if (iter.second.object->getTypeCode() != ::ot_null) { - iter.second.object->destroy(); - } - } -} - -void -Objects::update_table(QPDFObjGen og, const std::shared_ptr& object) -{ - object->setObjGen(&qpdf, og); - if (cached(og)) { - auto& cache = table[og]; - cache.object->assign(object); - } else { - table[og] = Entry(object); - } -} - -bool -Objects::cached(QPDFObjGen og) -{ - return table.count(og) != 0; -} - -bool -Objects::unresolved(QPDFObjGen og) -{ - return !cached(og) || table[og].object->isUnresolved(); -} - -QPDFObjGen -Objects::next_id() -{ - qpdf.fixDanglingReferences(); - QPDFObjGen og; - if (!table.empty()) { - og = (*(m->objects.table.rbegin())).first; - } - int max_objid = og.getObj(); - if (max_objid == std::numeric_limits::max()) { - throw std::range_error("max object id is too high to create new objects"); - } - return QPDFObjGen(max_objid + 1, 0); -} - -QPDFObjectHandle -Objects::make_indirect(std::shared_ptr const& obj) -{ - QPDFObjGen next{next_id()}; - table[next] = Entry(obj); - return qpdf.newIndirect(next, table[next].object); -} - -std::shared_ptr -Objects::get_for_parser(int id, int gen, bool parse_pdf) -{ - // This method is called by the parser and therefore must not resolve any objects. - auto og = QPDFObjGen(id, gen); - if (auto iter = table.find(og); iter != table.end()) { - return iter->second.object; - } - if (xref.type(og) || !xref.initialized()) { - return table.insert({og, QPDF_Unresolved::create(&qpdf, og)}).first->second.object; - } - if (parse_pdf) { - return QPDF_Null::create(); - } - return table.insert({og, QPDF_Null::create(&qpdf, og)}).first->second.object; -} - -std::shared_ptr -Objects::get_for_json(int id, int gen) -{ - auto og = QPDFObjGen(id, gen); - auto [it, inserted] = table.try_emplace(og); - auto& obj = it->second.object; - if (inserted) { - obj = (xref.initialized() && !xref.type(og)) ? QPDF_Null::create(&qpdf, og) - : QPDF_Unresolved::create(&qpdf, og); - } - return obj; -} - -void -Objects::replace(QPDFObjGen og, QPDFObjectHandle oh) -{ - if (!oh || (oh.isIndirect() && !(oh.isStream() && oh.getObjGen() == og))) { - QTC::TC("qpdf", "QPDF replaceObject called with indirect object"); - throw std::logic_error("QPDF::replaceObject called with indirect object handle"); - } - update_table(og, oh.getObj()); -} - -void -Objects::erase(QPDFObjGen og) -{ - if (auto cached = table.find(og); cached != table.end()) { - // Take care of any object handles that may be floating around. - cached->second.object->assign(QPDF_Null::create()); - cached->second.object->setObjGen(nullptr, QPDFObjGen()); - table.erase(cached); - } -} - -void -Objects::swap(QPDFObjGen og1, QPDFObjGen og2) -{ - // Force objects to be read from the input source if needed, then swap them in the cache. - resolve(og1); - resolve(og2); - table[og1].object->swapWith(table[og2].object); -} - -size_t -Objects::table_size() -{ - // If table is dense, accommodate all object in tables,else accommodate only original - // objects. - auto max_xref = toI(xref.size()); - if (max_xref > 0) { - --max_xref; - } - auto max_obj = table.size() ? table.crbegin()->first.getObj() : 0; - auto max_id = std::numeric_limits::max() - 1; - if (max_obj >= max_id || max_xref >= max_id) { - // Temporary fix. Long-term solution is - // - QPDFObjGen to enforce objgens are valid and sensible - // - xref table and obj cache to protect against insertion of impossibly large obj ids - qpdf.stopOnError("Impossibly large object id encountered."); - } - if (max_obj < 1.1 * std::max(toI(table.size()), max_xref)) { - return toS(++max_obj); - } - return toS(++max_xref); -} - -std::vector -Objects::compressible_vector() -{ - return compressible(); -} - -std::vector -Objects::compressible_set() -{ - return compressible(); -} - -template -std::vector -Objects::compressible() -{ - // Return a list of objects that are allowed to be in object streams. Walk through the objects - // by traversing the document from the root, including a traversal of the pages tree. This - // makes that objects that are on the same page are more likely to be in the same object stream, - // which is slightly more efficient, particularly with linearized files. This is better than - // iterating through the xref table since it avoids preserving orphaned items. - - // Exclude encryption dictionary, if any - QPDFObjectHandle encryption_dict = trailer().getKey("/Encrypt"); - QPDFObjGen encryption_dict_og = encryption_dict.getObjGen(); - - const size_t max_obj = qpdf.getObjectCount(); - std::vector visited(max_obj, false); - std::vector queue; - queue.reserve(512); - queue.emplace_back(trailer()); - std::vector result; - if constexpr (std::is_same_v) { - result.reserve(table.size()); - } else if constexpr (std::is_same_v) { - result.resize(max_obj + 1U, false); - } else { - throw std::logic_error("Unsupported type in QPDF::getCompressibleObjGens"); - } - while (!queue.empty()) { - auto obj = queue.back(); - queue.pop_back(); - if (obj.getObjectID() > 0) { - QPDFObjGen og = obj.getObjGen(); - const size_t id = toS(og.getObj() - 1); - if (id >= max_obj) { - throw std::logic_error( - "unexpected object id encountered in getCompressibleObjGens"); - } - if (visited[id]) { - QTC::TC("qpdf", "QPDF loop detected traversing objects"); - continue; - } - - // Check whether this is the current object. If not, remove it (which changes it into a - // direct null and therefore stops us from revisiting it) and move on to the next object - // in the queue. - auto upper = table.upper_bound(og); - if (upper != table.end() && upper->first.getObj() == og.getObj()) { - erase(og); - continue; - } - - visited[id] = true; - - if (og == encryption_dict_og) { - QTC::TC("qpdf", "QPDF exclude encryption dictionary"); - } else if (!(obj.isStream() || - (obj.isDictionaryOfType("/Sig") && obj.hasKey("/ByteRange") && - obj.hasKey("/Contents")))) { - if constexpr (std::is_same_v) { - result.push_back(og); - } else if constexpr (std::is_same_v) { - result[id + 1U] = true; - } - } - } - if (obj.isStream()) { - QPDFObjectHandle dict = obj.getDict(); - std::set keys = dict.getKeys(); - for (auto iter = keys.rbegin(); iter != keys.rend(); ++iter) { - std::string const& key = *iter; - QPDFObjectHandle value = dict.getKey(key); - if (key == "/Length") { - // omit stream lengths - if (value.isIndirect()) { - QTC::TC("qpdf", "QPDF exclude indirect length"); - } - } else { - queue.push_back(value); - } - } - } else if (obj.isDictionary()) { - std::set keys = obj.getKeys(); - for (auto iter = keys.rbegin(); iter != keys.rend(); ++iter) { - queue.push_back(obj.getKey(*iter)); - } - } else if (obj.isArray()) { - int n = obj.getArrayNItems(); - for (int i = 1; i <= n; ++i) { - queue.push_back(obj.getArrayItem(n - i)); - } - } - } - - return result; -} diff --git a/libqpdf/QPDF_optimization.cc b/libqpdf/QPDF_optimization.cc index 585b707..0e457af 100644 --- a/libqpdf/QPDF_optimization.cc +++ b/libqpdf/QPDF_optimization.cc @@ -2,7 +2,7 @@ #include -#include +#include #include #include @@ -78,12 +78,6 @@ QPDF::optimize( optimize_internal(obj, true, skip_stream_parameters); } -void -QPDF::optimize(QPDF::Objects const& objects) -{ - optimize_internal(objects, false, nullptr); -} - template void QPDF::optimize_internal( @@ -121,13 +115,13 @@ QPDF::optimize_internal( } // Traverse document-level items - for (auto const& key: m->objects.trailer().getKeys()) { + for (auto const& key: m->trailer.getKeys()) { if (key == "/Root") { // handled separately } else { updateObjectMaps( ObjUser(ObjUser::ou_trailer_key, key), - m->objects.trailer().getKey(key), + m->trailer.getKey(key), skip_stream_parameters); } } @@ -175,13 +169,13 @@ QPDF::pushInheritedAttributesToPage(bool allow_changes, bool warn_skipped_keys) // values for them. std::map> key_ancestors; pushInheritedAttributesToPageInternal( - m->objects.trailer().getKey("/Root").getKey("/Pages"), + m->trailer.getKey("/Root").getKey("/Pages"), key_ancestors, allow_changes, warn_skipped_keys); if (!key_ancestors.empty()) { - throw std::logic_error( - "key_ancestors not empty after pushing inherited attributes to pages"); + throw std::logic_error("key_ancestors not empty after" + " pushing inherited attributes to pages"); } m->pushed_inherited_attributes_to_pages = true; m->ever_pushed_inherited_attributes_to_pages = true; @@ -448,46 +442,3 @@ QPDF::filterCompressedObjects(QPDFWriter::ObjTable const& obj) m->obj_user_to_objects = t_obj_user_to_objects; m->object_to_obj_users = t_object_to_obj_users; } - -void -QPDF::filterCompressedObjects(QPDF::Objects const& objects) -{ - auto const& xref = objects.xref_table(); - if (!xref.object_streams()) { - return; - } - - // Transform object_to_obj_users and obj_user_to_objects so that they refer only to uncompressed - // objects. If something is a user of a compressed object, then it is really a user of the - // object stream that contains it. - - std::map> t_obj_user_to_objects; - std::map> t_object_to_obj_users; - - for (auto const& i1: m->obj_user_to_objects) { - ObjUser const& ou = i1.first; - // Loop over objects. - for (auto const& og: i1.second) { - if (auto stream = xref.stream_number(og.getObj())) { - t_obj_user_to_objects[ou].insert(QPDFObjGen(stream, 0)); - } else { - t_obj_user_to_objects[ou].insert(og); - } - } - } - - for (auto const& i1: m->object_to_obj_users) { - QPDFObjGen const& og = i1.first; - // Loop over obj_users. - for (auto const& ou: i1.second) { - if (auto stream = xref.stream_number(og.getObj())) { - t_object_to_obj_users[QPDFObjGen(stream, 0)].insert(ou); - } else { - t_object_to_obj_users[og].insert(ou); - } - } - } - - m->obj_user_to_objects = t_obj_user_to_objects; - m->object_to_obj_users = t_object_to_obj_users; -} diff --git a/libqpdf/QPDF_pages.cc b/libqpdf/QPDF_pages.cc index f46719d..195421c 100644 --- a/libqpdf/QPDF_pages.cc +++ b/libqpdf/QPDF_pages.cc @@ -1,4 +1,4 @@ -#include +#include #include #include diff --git a/libqpdf/qpdf-c.cc b/libqpdf/qpdf-c.cc index 0fb582a..a491218 100644 --- a/libqpdf/qpdf-c.cc +++ b/libqpdf/qpdf-c.cc @@ -905,7 +905,7 @@ qpdf_oh qpdf_get_object_by_id(qpdf_data qpdf, int objid, int generation) { QTC::TC("qpdf", "qpdf-c called qpdf_get_object_by_id"); - return new_object(qpdf, qpdf->qpdf->getObject(objid, generation)); + return new_object(qpdf, qpdf->qpdf->getObjectByID(objid, generation)); } template diff --git a/libqpdf/qpdf/ObjTable.hh b/libqpdf/qpdf/ObjTable.hh index 7d1daf1..3a36208 100644 --- a/libqpdf/qpdf/ObjTable.hh +++ b/libqpdf/qpdf/ObjTable.hh @@ -46,12 +46,6 @@ class ObjTable: public std::vector } inline T const& - operator[](unsigned int idx) const - { - return element(idx); - } - - inline T const& operator[](QPDFObjGen og) const { return element(static_cast(og.getObj())); diff --git a/libqpdf/qpdf/QPDFObject_private.hh b/libqpdf/qpdf/QPDFObject_private.hh index 35f708c..f323d95 100644 --- a/libqpdf/qpdf/QPDFObject_private.hh +++ b/libqpdf/qpdf/QPDFObject_private.hh @@ -6,13 +6,14 @@ #include #include +#include #include -#include #include #include #include +class QPDF; class QPDFObjectHandle; class QPDFObject diff --git a/libqpdf/qpdf/QPDF_objects.hh b/libqpdf/qpdf/QPDF_objects.hh deleted file mode 100644 index 83b7480..0000000 --- a/libqpdf/qpdf/QPDF_objects.hh +++ /dev/null @@ -1,497 +0,0 @@ -#ifndef QPDF_OBJECTS_HH -#define QPDF_OBJECTS_HH - -#include - -#include -#include - -#include - -// The Objects class is responsible for keeping track of all objects belonging to a QPDF instance, -// including loading it from an input source when required. -class QPDF::Objects -{ - public: - // Xref_table encapsulates the pdf's xref table and trailer. - class Xref_table - { - public: - Xref_table(Objects& objects) : - qpdf(objects.qpdf), - objects(objects), - file(objects.file) - { - tokenizer.allowEOF(); - } - - void initialize(); - void initialize_empty(); - void initialize_json(); - void reconstruct(QPDFExc& e); - void show(); - bool resolve(); - - QPDFObjectHandle - trailer() noexcept - { - return trailer_; - } - - QPDFObjectHandle const& - trailer() const noexcept - { - return trailer_; - } - - void - trailer(QPDFObjectHandle&& oh) - { - trailer_ = std::move(oh); - } - - // Returns 0 if og is not in table. - size_t - type(QPDFObjGen og) const - { - int id = og.getObj(); - if (id < 1 || static_cast(id) >= table.size()) { - return 0; - } - auto& e = table[static_cast(id)]; - return e.gen() == og.getGen() ? e.type() : 0; - } - - // Returns 0 if og is not in table. - size_t - type(size_t id) const noexcept - { - if (id >= table.size()) { - return 0; - } - return table[id].type(); - } - - // Returns 0 if og is not in table. - qpdf_offset_t - offset(QPDFObjGen og) const noexcept - { - int id = og.getObj(); - if (id < 1 || static_cast(id) >= table.size()) { - return 0; - } - return table[static_cast(id)].offset(); - } - - // Returns 0 if id is not in table. - int - stream_number(int id) const noexcept - { - if (id < 1 || static_cast(id) >= table.size()) { - return 0; - } - return table[static_cast(id)].stream_number(); - } - - int - stream_index(int id) const noexcept - { - if (id < 1 || static_cast(id) >= table.size()) { - return 0; - } - return table[static_cast(id)].stream_index(); - } - - QPDFObjGen at_offset(qpdf_offset_t offset) const noexcept; - - std::map as_map() const; - - bool - object_streams() const noexcept - { - return object_streams_; - } - - // Return a vector of object id and stream number for each compressed object. - std::vector> - compressed_objects() const - { - if (!initialized()) { - throw std::logic_error("Xref_table::compressed_objects called before parsing."); - } - - std::vector> result; - result.reserve(table.size()); - - unsigned int i{0}; - for (auto const& item: table) { - if (item.type() == 2) { - result.emplace_back(i, item.stream_number()); - } - ++i; - } - return result; - } - - // Temporary access to underlying table size - size_t - size() const noexcept - { - return table.size(); - } - - void - ignore_streams(bool val) noexcept - { - ignore_streams_ = val; - } - - bool - initialized() const noexcept - { - return initialized_; - } - - void - attempt_recovery(bool val) noexcept - { - attempt_recovery_ = val; - } - - int - max_id() const noexcept - { - return max_id_; - } - - // For Linearization - - qpdf_offset_t - end_after_space(QPDFObjGen og) - { - auto& e = entry(toS(og.getObj())); - switch (e.type()) { - case 1: - return e.end_after_space_; - case 2: - { - auto es = entry(toS(e.stream_number())); - return es.type() == 1 ? es.end_after_space_ : 0; - } - default: - return 0; - } - } - - qpdf_offset_t - end_before_space(QPDFObjGen og) - { - auto& e = entry(toS(og.getObj())); - switch (e.type()) { - case 1: - return e.end_before_space_; - case 2: - { - auto es = entry(toS(e.stream_number())); - return es.type() == 1 ? es.end_before_space_ : 0; - } - default: - return 0; - } - } - - void - linearization_offsets(size_t id, qpdf_offset_t before, qpdf_offset_t after) - { - if (type(id)) { - table[id].end_before_space_ = before; - table[id].end_after_space_ = after; - } - } - - bool - uncompressed_after_compressed() const noexcept - { - return uncompressed_after_compressed_; - } - - // Actual value from file - qpdf_offset_t - first_item_offset() const noexcept - { - return first_item_offset_; - } - - private: - // Object, count, offset of first entry - typedef std::tuple Subsection; - - struct Uncompressed - { - Uncompressed(qpdf_offset_t offset) : - offset(offset) - { - } - qpdf_offset_t offset; - }; - - struct Compressed - { - Compressed(int stream_number, int stream_index) : - stream_number(stream_number), - stream_index(stream_index) - { - } - int stream_number{0}; - int stream_index{0}; - }; - - typedef std::variant Xref; - - struct Entry - { - Entry() = default; - - Entry(int gen, Xref entry) : - gen_(gen), - entry(entry) - { - } - - int - gen() const noexcept - { - return gen_; - } - - size_t - type() const noexcept - { - return entry.index(); - } - - qpdf_offset_t - offset() const noexcept - { - return type() == 1 ? std::get<1>(entry).offset : 0; - } - - int - stream_number() const noexcept - { - return type() == 2 ? std::get<2>(entry).stream_number : 0; - } - - int - stream_index() const noexcept - { - return type() == 2 ? std::get<2>(entry).stream_index : 0; - } - - int gen_{0}; - Xref entry; - qpdf_offset_t end_before_space_{0}; - qpdf_offset_t end_after_space_{0}; - }; - - Entry& - entry(size_t id) - { - return id < table.size() ? table[id] : table[0]; - } - - void read(qpdf_offset_t offset); - - // Methods to parse tables - qpdf_offset_t process_section(qpdf_offset_t offset); - std::vector subsections(std::string& line); - std::vector bad_subsections(std::string& line, qpdf_offset_t offset); - Subsection subsection(std::string const& line); - std::tuple read_entry(); - std::tuple read_bad_entry(); - - // Methods to parse streams - qpdf_offset_t read_stream(qpdf_offset_t offset); - qpdf_offset_t process_stream(qpdf_offset_t offset, QPDFObjectHandle& xref_stream); - std::pair> - process_W(QPDFObjectHandle& dict, std::function damaged); - std::pair process_Size( - QPDFObjectHandle& dict, - int entry_size, - std::function damaged); - std::pair>> process_Index( - QPDFObjectHandle& dict, - int max_num_entries, - std::function damaged); - - QPDFObjectHandle read_trailer(); - - QPDFTokenizer::Token - read_token(size_t max_len = 0) - { - return tokenizer.readToken(*file, "", true, max_len); - } - - // Methods to insert table entries - void insert(int obj, int f0, qpdf_offset_t f1, int f2); - void insert_free(QPDFObjGen); - - QPDFExc - damaged_pdf(std::string const& msg) - { - return qpdf.damagedPDF("", 0, msg); - } - - QPDFExc - damaged_table(std::string const& msg) - { - return qpdf.damagedPDF("xref table", msg); - } - - void - warn_damaged(std::string const& msg) - { - qpdf.warn(damaged_pdf(msg)); - } - - QPDF& qpdf; - QPDF::Objects& objects; - InputSource* const& file; - QPDFTokenizer tokenizer; - - std::vector table; - QPDFObjectHandle trailer_; - - bool attempt_recovery_{true}; - bool initialized_{false}; - bool ignore_streams_{false}; - bool reconstructed_{false}; - bool object_streams_{false}; - // Before the xref table is initialized, max_id_ is an upper bound on the possible object - // ids that could be present in the PDF file. Once the trailer has been read, max_id_ is set - // to the value of /Size. If the file is damaged, max_id_ becomes the maximum object id in - // the xref table after reconstruction. - int max_id_{std::numeric_limits::max() - 1}; - - // Linearization data - bool uncompressed_after_compressed_{false}; - qpdf_offset_t first_item_offset_{0}; // actual value from file - }; // Xref_table; - - ~Objects(); - - Objects(QPDF& qpdf, QPDF::Members* m, InputSource* const& file) : - qpdf(qpdf), - file(file), - m(m), - xref(*this) - { - } - - Xref_table& - xref_table() noexcept - { - return xref; - } - - Xref_table const& - xref_table() const noexcept - { - return xref; - } - - QPDFObjectHandle - trailer() noexcept - { - return xref.trailer(); - } - - QPDFObjectHandle const& - trailer() const noexcept - { - return xref.trailer(); - } - - QPDFObjectHandle - get(QPDFObjGen og) - { - if (auto it = table.find(og); it != table.end()) { - return {it->second.object}; - } else if (xref.initialized() && !xref.type(og)) { - return QPDF_Null::create(); - } else { - auto result = table.try_emplace(og, QPDF_Unresolved::create(&qpdf, og)); - return {result.first->second.object}; - } - } - - QPDFObjectHandle - get(int id, int gen) - { - return get(QPDFObjGen(id, gen)); - } - - std::vector all(); - - void erase(QPDFObjGen og); - - void replace(QPDFObjGen og, QPDFObjectHandle oh); - - void swap(QPDFObjGen og1, QPDFObjGen og2); - - QPDFObjectHandle read( - bool attempt_recovery, - qpdf_offset_t offset, - std::string const& description, - QPDFObjGen exp_og, - QPDFObjGen& og, - bool skip_cache_if_in_xref); - QPDFObject* resolve(QPDFObjGen og); - void update_table(QPDFObjGen og, std::shared_ptr const& object); - QPDFObjGen next_id(); - QPDFObjectHandle make_indirect(std::shared_ptr const& obj); - std::shared_ptr get_for_parser(int id, int gen, bool parse_pdf); - std::shared_ptr get_for_json(int id, int gen); - - // Get a list of objects that would be permitted in an object stream. - template - std::vector compressible(); - std::vector compressible_vector(); - std::vector compressible_set(); - - // Used by QPDFWriter to determine the vector part of its object tables. - size_t table_size(); - - private: - struct Entry - { - Entry() = default; - - Entry(std::shared_ptr object) : - object(object) - { - } - - std::shared_ptr object; - }; - - bool cached(QPDFObjGen og); - bool unresolved(QPDFObjGen og); - - QPDFObjectHandle readObjectInStream(std::shared_ptr& input, int obj); - void resolveObjectsInStream(int obj_stream_number); - QPDFObjectHandle read_object(std::string const& description, QPDFObjGen og); - void read_stream(QPDFObjectHandle& object, QPDFObjGen og, qpdf_offset_t offset); - void validate_stream_line_end(QPDFObjectHandle& object, QPDFObjGen og, qpdf_offset_t offset); - size_t recover_stream_length( - std::shared_ptr input, QPDFObjGen og, qpdf_offset_t stream_offset); - - QPDF& qpdf; - InputSource* const& file; - QPDF::Members* m; - Xref_table xref; - - std::map table; -}; // Objects - -#endif // QPDF_OBJECTS_HH diff --git a/libqpdf/qpdf/QPDF_private.hh b/libqpdf/qpdf/QPDF_private.hh deleted file mode 100644 index e628e05..0000000 --- a/libqpdf/qpdf/QPDF_private.hh +++ /dev/null @@ -1,546 +0,0 @@ -#ifndef QPDF_PRIVATE_HH -#define QPDF_PRIVATE_HH - -#include - -#include - -#include - -// StreamCopier class is restricted to QPDFObjectHandle so it can copy stream data. -class QPDF::StreamCopier -{ - friend class QPDFObjectHandle; - - private: - static void - copyStreamData(QPDF* qpdf, QPDFObjectHandle const& dest, QPDFObjectHandle const& src) - { - qpdf->copyStreamData(dest, src); - } -}; - -// The ParseGuard class allows QPDFParser to detect re-entrant parsing. It also provides -// special access to allow the parser to create unresolved objects and dangling references. -class QPDF::ParseGuard -{ - friend class QPDFParser; - - private: - ParseGuard(QPDF* qpdf) : - qpdf(qpdf) - { - if (qpdf) { - qpdf->inParse(true); - } - } - - static std::shared_ptr - getObject(QPDF* qpdf, int id, int gen, bool parse_pdf) - { - return qpdf->objects().get_for_parser(id, gen, parse_pdf); - } - - ~ParseGuard() - { - if (qpdf) { - qpdf->inParse(false); - } - } - QPDF* qpdf; -}; - -// Pipe class is restricted to QPDF_Stream. -class QPDF::Pipe -{ - friend class QPDF_Stream; - - private: - static bool - pipeStreamData( - QPDF* qpdf, - QPDFObjGen const& og, - qpdf_offset_t offset, - size_t length, - QPDFObjectHandle dict, - Pipeline* pipeline, - bool suppress_warnings, - bool will_retry) - { - return qpdf->pipeStreamData( - og, offset, length, dict, pipeline, suppress_warnings, will_retry); - } -}; - -class QPDF::ObjCopier -{ - public: - std::map object_map; - std::vector to_copy; - QPDFObjGen::set visiting; -}; - -class QPDF::EncryptionParameters -{ - friend class QPDF; - - public: - EncryptionParameters(); - - private: - bool encrypted; - bool encryption_initialized; - int encryption_V; - int encryption_R; - bool encrypt_metadata; - std::map crypt_filters; - encryption_method_e cf_stream; - encryption_method_e cf_string; - encryption_method_e cf_file; - std::string provided_password; - std::string user_password; - std::string encryption_key; - std::string cached_object_encryption_key; - QPDFObjGen cached_key_og; - bool user_password_matched; - bool owner_password_matched; -}; - -class QPDF::ForeignStreamData -{ - friend class QPDF; - - public: - ForeignStreamData( - std::shared_ptr encp, - std::shared_ptr file, - QPDFObjGen const& foreign_og, - qpdf_offset_t offset, - size_t length, - QPDFObjectHandle local_dict); - - private: - std::shared_ptr encp; - std::shared_ptr file; - QPDFObjGen foreign_og; - qpdf_offset_t offset; - size_t length; - QPDFObjectHandle local_dict; -}; - -class QPDF::CopiedStreamDataProvider: public QPDFObjectHandle::StreamDataProvider -{ - public: - CopiedStreamDataProvider(QPDF& destination_qpdf); - ~CopiedStreamDataProvider() override = default; - bool provideStreamData( - QPDFObjGen const& og, Pipeline* pipeline, bool suppress_warnings, bool will_retry) override; - void registerForeignStream(QPDFObjGen const& local_og, QPDFObjectHandle foreign_stream); - void registerForeignStream(QPDFObjGen const& local_og, std::shared_ptr); - - private: - QPDF& destination_qpdf; - std::map foreign_streams; - std::map> foreign_stream_data; -}; - -class QPDF::StringDecrypter: public QPDFObjectHandle::StringDecrypter -{ - friend class QPDF; - - public: - StringDecrypter(QPDF* qpdf, QPDFObjGen const& og); - ~StringDecrypter() override = default; - void decryptString(std::string& val) override; - - private: - QPDF* qpdf; - QPDFObjGen og; -}; - -// PDF 1.4: Table F.4 -struct QPDF::HPageOffsetEntry -{ - int delta_nobjects{0}; // 1 - qpdf_offset_t delta_page_length{0}; // 2 - // vectors' sizes = nshared_objects - int nshared_objects{0}; // 3 - std::vector shared_identifiers; // 4 - std::vector shared_numerators; // 5 - qpdf_offset_t delta_content_offset{0}; // 6 - qpdf_offset_t delta_content_length{0}; // 7 -}; - -// PDF 1.4: Table F.3 -struct QPDF::HPageOffset -{ - int min_nobjects{0}; // 1 - qpdf_offset_t first_page_offset{0}; // 2 - int nbits_delta_nobjects{0}; // 3 - int min_page_length{0}; // 4 - int nbits_delta_page_length{0}; // 5 - int min_content_offset{0}; // 6 - int nbits_delta_content_offset{0}; // 7 - int min_content_length{0}; // 8 - int nbits_delta_content_length{0}; // 9 - int nbits_nshared_objects{0}; // 10 - int nbits_shared_identifier{0}; // 11 - int nbits_shared_numerator{0}; // 12 - int shared_denominator{0}; // 13 - // vector size is npages - std::vector entries; -}; - -// PDF 1.4: Table F.6 -struct QPDF::HSharedObjectEntry -{ - // Item 3 is a 128-bit signature (unsupported by Acrobat) - int delta_group_length{0}; // 1 - int signature_present{0}; // 2 -- always 0 - int nobjects_minus_one{0}; // 4 -- always 0 -}; - -// PDF 1.4: Table F.5 -struct QPDF::HSharedObject -{ - int first_shared_obj{0}; // 1 - qpdf_offset_t first_shared_offset{0}; // 2 - int nshared_first_page{0}; // 3 - int nshared_total{0}; // 4 - int nbits_nobjects{0}; // 5 - int min_group_length{0}; // 6 - int nbits_delta_group_length{0}; // 7 - // vector size is nshared_total - std::vector entries; -}; - -// PDF 1.4: Table F.9 -struct QPDF::HGeneric -{ - int first_object{0}; // 1 - qpdf_offset_t first_object_offset{0}; // 2 - int nobjects{0}; // 3 - int group_length{0}; // 4 -}; - -// Other linearization data structures - -// Initialized from Linearization Parameter dictionary -struct QPDF::LinParameters -{ - qpdf_offset_t file_size{0}; // /L - int first_page_object{0}; // /O - qpdf_offset_t first_page_end{0}; // /E - int npages{0}; // /N - qpdf_offset_t xref_zero_offset{0}; // /T - int first_page{0}; // /P - qpdf_offset_t H_offset{0}; // offset of primary hint stream - qpdf_offset_t H_length{0}; // length of primary hint stream -}; - -// Computed hint table value data structures. These tables contain the computed values on which -// the hint table values are based. They exclude things like number of bits and store actual -// values instead of mins and deltas. File offsets are also absolute rather than being offset -// by the size of the primary hint table. We populate the hint table structures from these -// during writing and compare the hint table values with these during validation. We ignore -// some values for various reasons described in the code. Those values are omitted from these -// structures. Note also that object numbers are object numbers from the input file, not the -// output file. - -// Naming convention: CHSomething is analogous to HSomething above. "CH" is computed hint. - -struct QPDF::CHPageOffsetEntry -{ - int nobjects{0}; - int nshared_objects{0}; - // vectors' sizes = nshared_objects - std::vector shared_identifiers; -}; - -struct QPDF::CHPageOffset -{ - // vector size is npages - std::vector entries; -}; - -struct QPDF::CHSharedObjectEntry -{ - CHSharedObjectEntry(int object) : - object(object) - { - } - - int object; -}; - -// PDF 1.4: Table F.5 -struct QPDF::CHSharedObject -{ - int first_shared_obj{0}; - int nshared_first_page{0}; - int nshared_total{0}; - // vector size is nshared_total - std::vector entries; -}; - -// No need for CHGeneric -- HGeneric is fine as is. - -// Data structures to support optimization -- implemented in QPDF_optimization.cc - -class QPDF::ObjUser -{ - public: - enum user_e { ou_bad, ou_page, ou_thumb, ou_trailer_key, ou_root_key, ou_root }; - - // type is set to ou_bad - ObjUser(); - - // type must be ou_root - ObjUser(user_e type); - - // type must be one of ou_page or ou_thumb - ObjUser(user_e type, int pageno); - - // type must be one of ou_trailer_key or ou_root_key - ObjUser(user_e type, std::string const& key); - - bool operator<(ObjUser const&) const; - - user_e ou_type; - int pageno; // if ou_page; - std::string key; // if ou_trailer_key or ou_root_key -}; - -struct QPDF::UpdateObjectMapsFrame -{ - UpdateObjectMapsFrame(ObjUser const& ou, QPDFObjectHandle oh, bool top); - - ObjUser const& ou; - QPDFObjectHandle oh; - bool top; -}; - -class QPDF::PatternFinder: public InputSource::Finder -{ - public: - PatternFinder(QPDF& qpdf, bool (QPDF::*checker)()) : - qpdf(qpdf), - checker(checker) - { - } - ~PatternFinder() override = default; - bool - check() override - { - return (this->qpdf.*checker)(); - } - - private: - QPDF& qpdf; - bool (QPDF::*checker)(); -}; - -class QPDF::Members -{ - friend class QPDF; - friend class ResolveRecorder; - - public: - QPDF_DLL - ~Members() = default; - - private: - Members(QPDF& qpdf); - Members(Members const&) = delete; - - std::shared_ptr log; - unsigned long long unique_id{0}; - QPDFTokenizer tokenizer; - // Filename to use if there is no input PDF - std::string no_input_name{"closed input source"}; - // If file_sp is updated, file must also be updated. - std::shared_ptr file_sp; - InputSource* file; - std::string last_object_description; - bool provided_password_is_hex_key{false}; - bool suppress_warnings{false}; - size_t max_warnings{0}; - bool attempt_recovery{true}; - bool check_mode{false}; - std::shared_ptr encp; - std::string pdf_version; - Objects objects; - std::set resolving; - std::vector all_pages; - bool invalid_page_found{false}; - std::map pageobj_to_pages_pos; - bool pushed_inherited_attributes_to_pages{false}; - bool ever_pushed_inherited_attributes_to_pages{false}; - bool ever_called_get_all_pages{false}; - std::vector warnings; - std::map object_copiers; - std::shared_ptr copied_streams; - // copied_stream_data_provider is owned by copied_streams - CopiedStreamDataProvider* copied_stream_data_provider{nullptr}; - bool fixed_dangling_refs{false}; - bool immediate_copy_from{false}; - bool in_parse{false}; - std::set resolved_object_streams; - - // Linearization data - bool linearization_warnings{false}; - - // Linearization parameter dictionary and hint table data: may be read from file or computed - // prior to writing a linearized file - QPDFObjectHandle lindict; - LinParameters linp; - HPageOffset page_offset_hints; - HSharedObject shared_object_hints; - HGeneric outline_hints; - - // Computed linearization data: used to populate above tables during writing and to compare - // with them during validation. c_ means computed. - LinParameters c_linp; - CHPageOffset c_page_offset_data; - CHSharedObject c_shared_object_data; - HGeneric c_outline_data; - - // Object ordering data for linearized files: initialized by calculateLinearizationData(). - // Part numbers refer to the PDF 1.4 specification. - std::vector part4; - std::vector part6; - std::vector part7; - std::vector part8; - std::vector part9; - - // Optimization data - std::map> obj_user_to_objects; - std::map> object_to_obj_users; -}; - -inline QPDF::Objects& -QPDF::objects() noexcept -{ - return m->objects; -} - -inline QPDF::Objects const& -QPDF::objects() const noexcept -{ - return m->objects; -} - -// The Resolver class is restricted to QPDFObject so that only it can resolve indirect -// references. -class QPDF::Resolver -{ - friend class QPDFObject; - friend class QPDF_Unresolved; - - private: - static QPDFObject* - resolved(QPDF* qpdf, QPDFObjGen og) - { - return qpdf->m->objects.resolve(og); - } -}; - -// JobSetter class is restricted to QPDFJob. -class QPDF::JobSetter -{ - friend class QPDFJob; - - private: - // Enable enhanced warnings for pdf file checking. - static void - setCheckMode(QPDF& qpdf, bool val) - { - qpdf.m->check_mode = val; - } -}; - -class QPDF::ResolveRecorder -{ - public: - ResolveRecorder(QPDF* qpdf, QPDFObjGen const& og) : - qpdf(qpdf), - iter(qpdf->m->resolving.insert(og).first) - { - } - virtual ~ResolveRecorder() - { - this->qpdf->m->resolving.erase(iter); - } - - private: - QPDF* qpdf; - std::set::const_iterator iter; -}; - -// Writer class is restricted to QPDFWriter so that only it can call certain methods. -class QPDF::Writer -{ - friend class QPDFWriter; - - private: - static void - optimize( - QPDF& qpdf, - QPDFWriter::ObjTable const& obj, - std::function skip_stream_parameters) - { - return qpdf.optimize(obj, skip_stream_parameters); - } - - static void - getLinearizedParts( - QPDF& qpdf, - QPDFWriter::ObjTable const& obj, - std::vector& part4, - std::vector& part6, - std::vector& part7, - std::vector& part8, - std::vector& part9) - { - qpdf.getLinearizedParts(obj, part4, part6, part7, part8, part9); - } - - static void - generateHintStream( - QPDF& qpdf, - QPDFWriter::NewObjTable const& new_obj, - QPDFWriter::ObjTable const& obj, - std::shared_ptr& hint_stream, - int& S, - int& O, - bool compressed) - { - return qpdf.generateHintStream(new_obj, obj, hint_stream, S, O, compressed); - } - - static std::vector - getCompressibleObjGens(QPDF& qpdf) - { - return qpdf.objects().compressible_vector(); - } - - static std::vector - getCompressibleObjSet(QPDF& qpdf) - { - return qpdf.objects().compressible_set(); - } - - static Objects::Xref_table const& - getXRefTable(QPDF& qpdf) - { - return qpdf.objects().xref_table(); - } - - static size_t - tableSize(QPDF& qpdf) - { - return qpdf.objects().table_size(); - } -}; - -#endif // QPDF_PRIVATE_HH diff --git a/libqpdf/qpdf/qpdf-c_impl.hh b/libqpdf/qpdf/qpdf-c_impl.hh index 0d52cf1..866b625 100644 --- a/libqpdf/qpdf/qpdf-c_impl.hh +++ b/libqpdf/qpdf/qpdf-c_impl.hh @@ -16,7 +16,7 @@ struct _qpdf_data _qpdf_data() = default; _qpdf_data(std::unique_ptr&& qpdf) : - qpdf(std::move(qpdf)){}; + qpdf(std::move(qpdf)) {}; ~_qpdf_data() = default; diff --git a/qpdf/qpdf.testcov b/qpdf/qpdf.testcov index 6f8d556..6c0450a 100644 --- a/qpdf/qpdf.testcov +++ b/qpdf/qpdf.testcov @@ -48,6 +48,7 @@ QPDFWriter encrypted hint stream 0 QPDF opt inherited scalar 0 QPDF xref reused object 0 QPDF xref gen > 0 1 +QPDF xref size mismatch 0 QPDF not a pdf file 0 QPDF can't find startxref 0 QPDF invalid xref 0 @@ -55,7 +56,6 @@ QPDF invalid xref entry 0 QPDF missing trailer 0 QPDF trailer lacks size 0 QPDF trailer size not integer 0 -QPDF trailer size impossibly large 0 QPDF trailer prev not integer 0 QPDFParser bad brace 0 QPDFParser bad brace in parseRemainder 0 @@ -105,6 +105,7 @@ QPDFWriter not recompressing /FlateDecode 0 QPDF_encryption xref stream from encrypted file 0 QPDFJob unable to filter 0 QUtil non-trivial UTF-16 0 +QPDF xref overwrite object 0 QPDF xref overwrite invalid objgen 0 QPDF decoding error warning 0 qpdf-c called qpdf_init 0 @@ -436,6 +437,7 @@ QPDF xref skipped space 0 QPDF eof skipping spaces before xref 1 QPDF_encryption user matches owner V < 5 0 QPDF_encryption same password 1 +QPDFWriter stream in ostream 0 QPDFParser duplicate dict key 0 QPDFWriter no encryption sig contents 0 QPDFPageObjectHelper colorspace lookup 0 diff --git a/qpdf/qtest/qpdf/bad12-recover.out b/qpdf/qtest/qpdf/bad12-recover.out index 8e553fe..428460f 100644 --- a/qpdf/qtest/qpdf/bad12-recover.out +++ b/qpdf/qtest/qpdf/bad12-recover.out @@ -1,3 +1,4 @@ +WARNING: bad12.pdf: reported number of objects (9) is not one plus the highest object number (7) WARNING: bad12.pdf (object 2 0, offset 128): expected endobj /QTest is implicit /QTest is direct and has type null (2) diff --git a/qpdf/qtest/qpdf/bad12.out b/qpdf/qtest/qpdf/bad12.out index 2230b9c..8904a33 100644 --- a/qpdf/qtest/qpdf/bad12.out +++ b/qpdf/qtest/qpdf/bad12.out @@ -1,3 +1,4 @@ +WARNING: bad12.pdf: reported number of objects (9) is not one plus the highest object number (7) WARNING: bad12.pdf (object 2 0, offset 128): expected endobj /QTest is implicit /QTest is direct and has type null (2) diff --git a/qpdf/qtest/qpdf/fuzz-16214.out b/qpdf/qtest/qpdf/fuzz-16214.out index b00f183..a03574b 100644 --- a/qpdf/qtest/qpdf/fuzz-16214.out +++ b/qpdf/qtest/qpdf/fuzz-16214.out @@ -11,9 +11,11 @@ WARNING: fuzz-16214.pdf (object 1 0, offset 7189): expected n n obj WARNING: fuzz-16214.pdf: Attempting to reconstruct cross-reference table WARNING: fuzz-16214.pdf (offset 7207): error decoding stream data for object 2 0: stream inflate: inflate: data: invalid code lengths set WARNING: fuzz-16214.pdf (offset 7207): getStreamData called on unfilterable stream -WARNING: fuzz-16214.pdf (object 7 0, offset 7207): supposed object stream 5 has wrong type -WARNING: fuzz-16214.pdf (object 7 0, offset 7207): object stream 5 has incorrect keys +WARNING: fuzz-16214.pdf (object 8 0, offset 7207): supposed object stream 5 has wrong type +WARNING: fuzz-16214.pdf (object 8 0, offset 7207): object stream 5 has incorrect keys WARNING: fuzz-16214.pdf (object 21 0, offset 3639): expected endstream WARNING: fuzz-16214.pdf (object 21 0, offset 3112): attempting to recover stream length WARNING: fuzz-16214.pdf (object 21 0, offset 3112): recovered stream length: 340 +WARNING: fuzz-16214.pdf, stream object 8 0: stream found inside object stream; treating as null +WARNING: fuzz-16214.pdf, stream object 8 0: stream found inside object stream; treating as null qpdf: operation succeeded with warnings; resulting file may have some problems diff --git a/qpdf/qtest/qpdf/issue-147.out b/qpdf/qtest/qpdf/issue-147.out index da8ae19..9e766df 100644 --- a/qpdf/qtest/qpdf/issue-147.out +++ b/qpdf/qtest/qpdf/issue-147.out @@ -2,6 +2,6 @@ WARNING: issue-147.pdf: can't find PDF header WARNING: issue-147.pdf: file is damaged WARNING: issue-147.pdf: can't find startxref WARNING: issue-147.pdf: Attempting to reconstruct cross-reference table -WARNING: issue-147.pdf: ignoring object with impossibly large id 62 WARNING: issue-147.pdf (trailer, offset 9): expected dictionary key but found non-name object; inserting key /QPDFFake1 -qpdf: issue-147.pdf: unable to find /Root dictionary +WARNING: issue-147.pdf: ignoring object with impossibly large id 62 +qpdf: issue-147.pdf: unable to find objects while recovering damaged file diff --git a/qpdf/qtest/qpdf/issue-335b.out b/qpdf/qtest/qpdf/issue-335b.out index 99f3d0a..e996d88 100644 --- a/qpdf/qtest/qpdf/issue-335b.out +++ b/qpdf/qtest/qpdf/issue-335b.out @@ -1,5 +1,5 @@ WARNING: issue-335b.pdf: can't find PDF header WARNING: issue-335b.pdf: file is damaged -WARNING: issue-335b.pdf (xref table, offset 11): xref table subsection header contains impossibly large entry +WARNING: issue-335b.pdf (xref table, offset 23): invalid xref entry (obj=6) WARNING: issue-335b.pdf: Attempting to reconstruct cross-reference table qpdf: issue-335b.pdf: unable to find trailer dictionary while recovering damaged file diff --git a/qpdf/qtest/qpdf/issue-fuzz.out b/qpdf/qtest/qpdf/issue-fuzz.out deleted file mode 100644 index 456485b..0000000 --- a/qpdf/qtest/qpdf/issue-fuzz.out +++ /dev/null @@ -1,19 +0,0 @@ -WARNING: issue-fuzz.pdf: can't find PDF header -WARNING: issue-fuzz.pdf (xref table, offset 19): accepting invalid xref table entry -WARNING: issue-fuzz.pdf (trailer, offset 36): unknown token while reading object; treating as string -WARNING: issue-fuzz.pdf (trailer, offset 53): unexpected > -WARNING: issue-fuzz.pdf (trailer, offset 54): unknown token while reading object; treating as string -WARNING: issue-fuzz.pdf (trailer, offset 58): unknown token while reading object; treating as string -WARNING: issue-fuzz.pdf (trailer, offset 72): unknown token while reading object; treating as string -WARNING: issue-fuzz.pdf (trailer, offset 36): dictionary ended prematurely; using null as value for last key -WARNING: issue-fuzz.pdf (trailer, offset 36): expected dictionary key but found non-name object; inserting key /QPDFFake1 -WARNING: issue-fuzz.pdf (trailer, offset 36): expected dictionary key but found non-name object; inserting key /QPDFFake2 -WARNING: issue-fuzz.pdf (trailer, offset 36): expected dictionary key but found non-name object; inserting key /QPDFFake3 -WARNING: issue-fuzz.pdf (trailer, offset 36): expected dictionary key but found non-name object; inserting key /QPDFFake4 -WARNING: issue-fuzz.pdf (trailer, offset 36): expected dictionary key but found non-name object; inserting key /QPDFFake5 -WARNING: issue-fuzz.pdf (trailer, offset 36): expected dictionary key but found non-name object; inserting key /QPDFFake6 -WARNING: issue-fuzz.pdf (trailer, offset 36): expected dictionary key but found non-name object; inserting key /QPDFFake7 -WARNING: issue-fuzz.pdf: file is damaged -WARNING: issue-fuzz.pdf (trailer, offset 32): /Size key in trailer dictionary is impossibly large -WARNING: issue-fuzz.pdf: Attempting to reconstruct cross-reference table -qpdf: issue-fuzz.pdf: unable to find /Root dictionary diff --git a/qpdf/qtest/qpdf/issue-fuzz.pdf b/qpdf/qtest/qpdf/issue-fuzz.pdf deleted file mode 100644 index 288a6b5..0000000 --- a/qpdf/qtest/qpdf/issue-fuzz.pdf +++ /dev/null diff --git a/qpdf/qtest/qpdf/recover-xref-stream.out b/qpdf/qtest/qpdf/recover-xref-stream.out index ffc4cce..ba0e1aa 100644 --- a/qpdf/qtest/qpdf/recover-xref-stream.out +++ b/qpdf/qtest/qpdf/recover-xref-stream.out @@ -1,4 +1,5 @@ WARNING: recover-xref-stream.pdf: file is damaged WARNING: recover-xref-stream.pdf: can't find startxref WARNING: recover-xref-stream.pdf: Attempting to reconstruct cross-reference table +WARNING: recover-xref-stream.pdf: reported number of objects (14) is not one plus the highest object number (15) qpdf: operation succeeded with warnings; resulting file may have some problems diff --git a/qpdf/qtest/qpdf/recover-xref-stream.pdf b/qpdf/qtest/qpdf/recover-xref-stream.pdf index 98e565c..f8da3f1 100644 --- a/qpdf/qtest/qpdf/recover-xref-stream.pdf +++ b/qpdf/qtest/qpdf/recover-xref-stream.pdf diff --git a/qpdf/qtest/qpdf/xref-errors.out b/qpdf/qtest/qpdf/xref-errors.out index ff3ae82..66420c3 100644 --- a/qpdf/qtest/qpdf/xref-errors.out +++ b/qpdf/qtest/qpdf/xref-errors.out @@ -3,11 +3,6 @@ WARNING: xref-errors.pdf (xref table, offset 606): accepting invalid xref table WARNING: xref-errors.pdf (xref table, offset 627): accepting invalid xref table entry WARNING: xref-errors.pdf (xref table, offset 648): accepting invalid xref table entry WARNING: xref-errors.pdf (xref table, offset 667): accepting invalid xref table entry -WARNING: xref-errors.pdf (xref table, offset 585): accepting invalid xref table entry -WARNING: xref-errors.pdf (xref table, offset 606): accepting invalid xref table entry -WARNING: xref-errors.pdf (xref table, offset 627): accepting invalid xref table entry -WARNING: xref-errors.pdf (xref table, offset 648): accepting invalid xref table entry -WARNING: xref-errors.pdf (xref table, offset 667): accepting invalid xref table entry checking xref-errors.pdf PDF Version: 1.3 File is not encrypted diff --git a/qpdf/qtest/specific-bugs.test b/qpdf/qtest/specific-bugs.test index 428471b..15c9e01 100644 --- a/qpdf/qtest/specific-bugs.test +++ b/qpdf/qtest/specific-bugs.test @@ -16,7 +16,7 @@ my $td = new TestDriver('specific-bugs'); # The number is the github issue number in which the bug was reported. my @bug_tests = ( -# ["51", "resolve loop", 2], + ["51", "resolve loop", 2], ["99", "object 0", 2], ["99b", "object 0", 2], ["100", "xref reconstruction loop", 2], @@ -28,7 +28,7 @@ my @bug_tests = ( ["106", "zlib data error", 3], ["141a", "/W entry size 0", 2], ["141b", "/W entry size 0", 2], -# ["143", "self-referential ostream", 2, "--preserve-unreferenced"], + ["143", "self-referential ostream", 2, "--preserve-unreferenced"], ["146", "very deeply nested array", 2], ["147", "previously caused memory error", 2], ["148", "free memory on bad flate", 2], @@ -38,8 +38,7 @@ my @bug_tests = ( ["263", "empty xref stream", 2], ["335a", "ozz-fuzz-12152", 2], ["335b", "ozz-fuzz-14845", 2], - ["fuzz", "impossibly large trailer /Size"], -# ["fuzz-16214", "stream in object stream", 3, "--preserve-unreferenced"], + ["fuzz-16214", "stream in object stream", 3, "--preserve-unreferenced"], # When adding to this list, consider adding to CORPUS_FROM_TEST in # fuzz/CMakeLists.txt and updating the count in # fuzz/qtest/fuzz.test.