Merge pull request #1568 from m-holger/inspect

Document new `inspection_mode` in design and release notes

Merge pull request #1568 from m-holger/inspect
Document new `inspection_mode` in design and release notes
m-holger · GitHub
2 parents e848395d 675777bb
Showing 13 changed files with 227 additions and 24 deletions
include/qpdf/Constants.h
include/qpdf/global.hh
libqpdf/QPDF_encryption.cc
libqpdf/QPDF_objects.cc
libqpdf/global.cc
libqpdf/qpdf/QPDF_private.hh
libqpdf/qpdf/global_private.hh
manual/design.rst
manual/release-notes.rst
qpdf/qtest/inspection-mode.test
qpdf/qtest/qpdf/inspect.pdf
qpdf/qtest/qpdf/inspection-mode.out
qpdf/test_driver.cc
@@ -277,10 +277,11 @@ enum qpdf_param_e {
     qpdf_p_limit_errors = 0x10020,
     /* global options */
+    qpdf_p_inspection_mode = 0x11000,
     qpdf_p_default_limits = 0x11100,
     /* global limits */
-    /* object - parser limits */
+    /* parser limits */
     qpdf_p_parser_max_nesting = 0x13000,
     qpdf_p_parser_max_errors,
     qpdf_p_parser_max_container_size,
@@ -68,6 +68,30 @@ namespace qpdf::global
     namespace options
     {
+        /// @brief  Retrieves whether inspection mode is set.
+        ///
+        /// @return True if inspection mode is set.
+        ///
+        /// @since 12.3
+        bool inline inspection_mode()
+        {
+            return get_uint32(qpdf_p_inspection_mode) != 0;
+        }
+
+        /// @brief  Set inspection mode if `true` is passed.
+        ///
+        /// This function enables restrictive inspection mode if `true` is passed. Inspection mode
+        /// must be enabled before a QPDF object is created. By default inspection mode is off.
+        /// Calling `inspection_mode(false)` is not supported and currently is a no-op.
+        ///
+        /// @param value A boolean indicating whether to enable (true) inspection mode.
+        ///
+        /// @since 12.3
+        void inline inspection_mode(bool value)
+        {
+            set_uint32(qpdf_p_inspection_mode, value ? QPDF_TRUE : QPDF_FALSE);
+        }
+
         /// @brief  Retrieves whether default limits are enabled.
         ///
         /// @return True if default limits are enabled.
@@ -758,7 +758,7 @@ QPDF::EncryptionParameters::initialize(QPDF&amp; qpdf)
     // at /Encrypt again.  Otherwise, things could go wrong if someone mutates the encryption
     // dictionary.
-    if (!trailer.hasKey("/Encrypt")) {
+    if (!trailer.contains("/Encrypt")) {
         return;
     }
@@ -257,11 +257,21 @@ Objects::parse(char const* password)
             throw damagedPDF("", -1, std::string("error reading xref: ") + e.what());
         }
     } catch (QPDFExc& e) {
-        if (!cf.surpress_recovery()) {
-            reconstruct_xref(e, xref_offset > 0);
-        } else {
+        if (global::Options::inspection_mode()) {
+            try {
+                reconstruct_xref(e, xref_offset > 0);
+            } catch (std::exception& er) {
+                warn(damagedPDF("", -1, "error reconstructing xref: "s + er.what()));
+            }
+            if (!m->trailer) {
+                m->trailer = Dictionary::empty();
+            }
+            return;
+        }
+        if (cf.surpress_recovery()) {
             throw;
         }
+        reconstruct_xref(e, xref_offset > 0);
     }
     m->encp->initialize(qpdf);
@@ -35,6 +35,9 @@ qpdf_global_get_uint32(qpdf_param_e param, uint32_t* value)
 {
     qpdf_expect(value);
     switch (param) {
+    case qpdf_p_inspection_mode:
+        *value = Options::inspection_mode();
+        return qpdf_r_ok;
     case qpdf_p_default_limits:
         *value = Options::default_limits();
         return qpdf_r_ok;
@@ -62,6 +65,9 @@ qpdf_result_e
 qpdf_global_set_uint32(qpdf_param_e param, uint32_t value)
 {
     switch (param) {
+    case qpdf_p_inspection_mode:
+        Options::inspection_mode(value);
+        return qpdf_r_ok;
     case qpdf_p_default_limits:
         Options::default_limits(value);
         return qpdf_r_ok;
@@ -12,6 +12,9 @@
 #include <qpdf/QPDFPageDocumentHelper.hh>
 #include <qpdf/QPDFPageLabelDocumentHelper.hh>
 #include <qpdf/QPDFTokenizer_private.hh>
+#include <qpdf/global_private.hh>
+
+#include <exception>
 using namespace qpdf;
@@ -374,6 +377,7 @@ class QPDF::Doc
     acroform()
     {
         if (!acroform_) {
+            no_inspection();
             acroform_ = std::make_unique<QPDFAcroFormDocumentHelper>(qpdf);
         }
         return *acroform_;
@@ -383,6 +387,7 @@ class QPDF::Doc
     embedded_files()
     {
         if (!embedded_files_) {
+            no_inspection();
             embedded_files_ = std::make_unique<QPDFEmbeddedFileDocumentHelper>(qpdf);
         }
         return *embedded_files_;
@@ -392,6 +397,7 @@ class QPDF::Doc
     outlines()
     {
         if (!outlines_) {
+            no_inspection();
             outlines_ = std::make_unique<QPDFOutlineDocumentHelper>(qpdf);
         }
         return *outlines_;
@@ -401,6 +407,7 @@ class QPDF::Doc
     page_dh()
     {
         if (!page_dh_) {
+            no_inspection();
             page_dh_ = std::make_unique<QPDFPageDocumentHelper>(qpdf);
         }
         return *page_dh_;
@@ -410,12 +417,21 @@ class QPDF::Doc
     page_labels()
     {
         if (!page_labels_) {
+            no_inspection();
             page_labels_ = std::make_unique<QPDFPageLabelDocumentHelper>(qpdf);
         }
         return *page_labels_;
     }
   protected:
+    void
+    no_inspection()
+    {
+        if (global::Options::inspection_mode()) {
+            throw std::logic_error("Attempted unsupported operation in inspection mode");
+        }
+    }
+
     QPDF& qpdf;
     QPDF::Members* m;
@@ -85,6 +85,20 @@ namespace qpdf::global
     {
       public:
         static bool
+        inspection_mode()
+        {
+            return static_cast<bool>(o.inspection_mode_);
+        }
+
+        static void
+        inspection_mode(bool value)
+        {
+            if (value) {
+                o.inspection_mode_ = true;
+            }
+        }
+
+        static bool
         default_limits()
         {
             return static_cast<bool>(o.default_limits_);
@@ -102,6 +116,7 @@ namespace qpdf::global
       private:
         static Options o;
+        bool inspection_mode_{false};
         bool default_limits_{true};
     };
 } // namespace qpdf::global
@@ -35,6 +35,38 @@ behind the initial creation of qpdf. That said, qpdf is not a strict
 PDF checker. There are many ways in which a PDF file can be out of
 conformance to the spec that qpdf doesn't notice or report.
+.. _inspection-mode:
+
+Inspection Mode
+---------------
+
+The approach as described above has shifted somewhat over time for
+pragmatic reasons. A large number of essential repairs that can be
+carried out safely are now happening even with ``setAttemptRecovery(false)``.
+At the same time a number of minor infractions of the PDF standards are quietly
+dealt with to avoid creating distracting noise. This has been helpful to users
+who use qpdf to perform content-preserving transformations and who usually
+want qpdf to work reliably and to produce correct PDF output files that comply
+with the PDF standards even if the input files are somewhat less than perfect.
+
+However, there is another stated purpose of qpdf - to provide a tool for the
+study and analysis of PDF files. When used in this way, repairing the faults
+in input files or preventing the creation of unusable output files, often with
+unacceptable resource usage, is counter-productive rather than useful.
+
+To accommodate the needs of those who use qpdf as a tool for inspecting and
+investigating PDF files, qpdf version 12.3 introduced a special
+``ìnspection mode`` which is enabled using the
+``qpdf::global::options::inspection_mode`` function. In inspection mode, only a
+very limited set of basic operations is supported and a number of automatic
+repairs are disabled. Transformations of the input files such as linearizing files,
+creating object streams or encrypting files are not supported, as is the use of
+document and object helpers.
+
+Inspection mode is intended for manual investigations and repairs. As such,
+stability is less of a concern than for other uses of qpdf. The exact effect
+of specifying inspection mode will evolve from version to version.
+
 .. _design-goals:
 Design Goals
@@ -91,6 +91,14 @@ more detail.
   - Other enhancements
+    - Add new ``inspection mode`` to help with the inspection and manual repair
+      of damaged PDF files. In this mode some of the exceptions thrown if a PDF
+      file is damaged and unrepairable are replaced with warnings and some automatic
+      repairs are suppressed. Only a very limited range of operations are supported.
+      Inspection mode is selected with the new function
+      ``qpdf::global::options::inspection_mode``. For more detail see
+      :ref:`inspection-mode`.
+
     - ``QPDFWriter`` will no longer add filters when writing empty streams.
     - More sanity checks have been added when files with damaged xref tables
@@ -123,7 +131,7 @@ more detail.
       is the intended behaviour.
     - There has been some refactoring of stream filtering. These are optimized
-      for the common case where no user provided stream filters  are
+      for the common case where no user provided stream filters are
       registered by calling ``QPDF::registerStreamFilter``. If you are
       providing your own stream filters please open a ticket_.
+#!/usr/bin/env perl
+require 5.008;
+use warnings;
+use strict;
+
+unshift(@INC, '.');
+require qpdf_test_helpers;
+
+chdir("qpdf") or die "chdir testdir failed: $!\n";
+
+require TestDriver;
+
+cleanup();
+
+my $td = new TestDriver('inspection-mode');
+
+my $n_tests = 1;
+
+$td->runtest("inspection mode",
+             {$td->COMMAND => "test_driver 101 - -"},
+             {$td->FILE => "inspection-mode.out", $td->EXIT_STATUS => 0},
+             $td->NORMALIZE_NEWLINES);
+
+cleanup();
+$td->report($n_tests);
+WARNING: inspect.pdf: can't find PDF header
+WARNING: inspect.pdf: file is damaged
+WARNING: inspect.pdf: can't find startxref
+WARNING: inspect.pdf: Attempting to reconstruct cross-reference table
+WARNING: inspect.pdf (trailer, offset 38): unknown token while reading object; treating as null
+WARNING: inspect.pdf (trailer, offset 60): unknown token while reading object; treating as null
+WARNING: inspect.pdf (trailer, offset 82): treating bad indirect reference (0 0 R) as null
+WARNING: inspect.pdf (trailer, offset 90): unknown token while reading object; treating as null
+WARNING: inspect.pdf (trailer, offset 100): unexpected >
+WARNING: inspect.pdf (trailer, offset 71): expected dictionary keys but found non-name objects; ignoring
+WARNING: inspect.pdf (trailer, offset 202): unexpected 'endobj' or 'endstream' while reading object; giving up on reading object
+WARNING: inspect.pdf: error reconstructing xref: inspect.pdf: unable to find trailer dictionary while recovering damaged file
+5 0
+null
+20 0
+<< /Fields [ 21 0 R ] >>
+21 0
+<< /Kids [ 22 0 R 23 0 R 24 0 R 25 0 R ] /Rect [ 100 100 500 500 ] /Subtype /Widget /T (MyFie\224d) /Type /Annot >>
+22 0
+null
+23 0
+<< /FT /Tx /Rect [ 401 401 421 421 ] /Subtype /Widget /T (Sub_RightTop) /Type /Annot >>
+24 0
+<< /FT /Tx /Rect [ 201 400 221 420 ] /Subtype /Widget /T (Sub_LeftTop) /Type /Annot >>
+25 0
+<< /FT /Tx /Rect [ 400 201 420 221 ] /Subtype /Widget /T (Sub_RightBottom) /Type /Annot >>
+test 101 done
@@ -3550,6 +3550,41 @@ test_100(QPDF&amp; pdf, char const* arg2)
     }
 }
+static void
+test_101(QPDF& pdf, char const* arg2)
+{
+    // Test inspection mode
+    QPDF qpdf;
+    assert(!qpdf::global::options::inspection_mode());
+    qpdf::global::options::inspection_mode(true);
+    assert(qpdf::global::options::inspection_mode());
+    qpdf::global::options::inspection_mode(false);
+    // Setting inspection mode is irreversible
+    assert(qpdf::global::options::inspection_mode());
+    qpdf.processFile("inspect.pdf");
+    for (auto& oh: qpdf.getAllObjects()) {
+        std::cout << oh.getObjGen().unparse(' ') << '\n';
+        std::cout << oh.unparseResolved() << '\n';
+    }
+
+
+    auto test_helper_throws = [&qpdf](auto helper_func) {
+        bool thrown = false;
+        try {
+            helper_func(qpdf);
+        } catch (std::logic_error&) {
+            thrown = true;
+        }
+        assert(thrown);
+    };
+
+    test_helper_throws([](QPDF& q) { (void)QPDFAcroFormDocumentHelper::get(q); });
+    test_helper_throws([](QPDF& q) { (void)QPDFEmbeddedFileDocumentHelper::get(q); });
+    test_helper_throws([](QPDF& q) { (void)QPDFOutlineDocumentHelper::get(q); });
+    test_helper_throws([](QPDF& q) { (void)QPDFPageDocumentHelper::get(q); });
+    test_helper_throws([](QPDF& q) { (void)QPDFPageLabelDocumentHelper::get(q); });
+}
+
 void
 runtest(int n, char const* filename1, char const* arg2)
 {
@@ -3557,7 +3592,7 @@ runtest(int n, char const* filename1, char const* arg2)
     // the test suite to see how the test is invoked to find the file
     // that the test is supposed to operate on.
-    std::set<int> ignore_filename = {61, 62, 81, 83, 84, 85, 86, 87, 92, 95, 96};
+    std::set<int> ignore_filename = {61, 62, 81, 83, 84, 85, 86, 87, 92, 95, 96, 101};
     if (n == 0) {
         // Throw in some random test cases that don't fit anywhere
@@ -3631,23 +3666,27 @@ runtest(int n, char const* filename1, char const* arg2)
     }
     std::map<int, void (*)(QPDF&, char const*)> test_functions = {
-        {0, test_0_1}, {1, test_0_1}, {2, test_2},   {3, test_3},   {4, test_4},    {5, test_5},
-        {6, test_6},   {7, test_7},   {8, test_8},   {9, test_9},   {10, test_10},  {11, test_11},
-        {12, test_12}, {13, test_13}, {14, test_14}, {15, test_15}, {16, test_16},  {17, test_17},
-        {18, test_18}, {19, test_19}, {20, test_20}, {21, test_21}, {22, test_22},  {23, test_23},
-        {24, test_24}, {25, test_25}, {26, test_26}, {27, test_27}, {28, test_28},  {29, test_29},
-        {30, test_30}, {31, test_31}, {32, test_32}, {33, test_33}, {34, test_34},  {35, test_35},
-        {36, test_36}, {37, test_37}, {38, test_38}, {39, test_39}, {40, test_40},  {41, test_41},
-        {42, test_42}, {43, test_43}, {44, test_44}, {45, test_45}, {46, test_46},  {47, test_47},
-        {48, test_48}, {49, test_49}, {50, test_50}, {51, test_51}, {52, test_52},  {53, test_53},
-        {54, test_54}, {55, test_55}, {56, test_56}, {57, test_57}, {58, test_58},  {59, test_59},
-        {60, test_60}, {61, test_61}, {62, test_62}, {63, test_63}, {64, test_64},  {65, test_65},
-        {66, test_66}, {67, test_67}, {68, test_68}, {69, test_69}, {70, test_70},  {71, test_71},
-        {72, test_72}, {73, test_73}, {74, test_74}, {75, test_75}, {76, test_76},  {77, test_77},
-        {78, test_78}, {79, test_79}, {80, test_80}, {81, test_81}, {82, test_82},  {83, test_83},
-        {84, test_84}, {85, test_85}, {86, test_86}, {87, test_87}, {88, test_88},  {89, test_89},
-        {90, test_90}, {91, test_91}, {92, test_92}, {93, test_93}, {94, test_94},  {95, test_95},
-        {96, test_96}, {97, test_97}, {98, test_98}, {99, test_99}, {100, test_100}};
+        {0, test_0_1},   {1, test_0_1},  {2, test_2},   {3, test_3},   {4, test_4},
+        {5, test_5},     {6, test_6},    {7, test_7},   {8, test_8},   {9, test_9},
+        {10, test_10},   {11, test_11},  {12, test_12}, {13, test_13}, {14, test_14},
+        {15, test_15},   {16, test_16},  {17, test_17}, {18, test_18}, {19, test_19},
+        {20, test_20},   {21, test_21},  {22, test_22}, {23, test_23}, {24, test_24},
+        {25, test_25},   {26, test_26},  {27, test_27}, {28, test_28}, {29, test_29},
+        {30, test_30},   {31, test_31},  {32, test_32}, {33, test_33}, {34, test_34},
+        {35, test_35},   {36, test_36},  {37, test_37}, {38, test_38}, {39, test_39},
+        {40, test_40},   {41, test_41},  {42, test_42}, {43, test_43}, {44, test_44},
+        {45, test_45},   {46, test_46},  {47, test_47}, {48, test_48}, {49, test_49},
+        {50, test_50},   {51, test_51},  {52, test_52}, {53, test_53}, {54, test_54},
+        {55, test_55},   {56, test_56},  {57, test_57}, {58, test_58}, {59, test_59},
+        {60, test_60},   {61, test_61},  {62, test_62}, {63, test_63}, {64, test_64},
+        {65, test_65},   {66, test_66},  {67, test_67}, {68, test_68}, {69, test_69},
+        {70, test_70},   {71, test_71},  {72, test_72}, {73, test_73}, {74, test_74},
+        {75, test_75},   {76, test_76},  {77, test_77}, {78, test_78}, {79, test_79},
+        {80, test_80},   {81, test_81},  {82, test_82}, {83, test_83}, {84, test_84},
+        {85, test_85},   {86, test_86},  {87, test_87}, {88, test_88}, {89, test_89},
+        {90, test_90},   {91, test_91},  {92, test_92}, {93, test_93}, {94, test_94},
+        {95, test_95},   {96, test_96},  {97, test_97}, {98, test_98}, {99, test_99},
+        {100, test_100}, {101, test_101}};
     auto fn = test_functions.find(n);
     if (fn == test_functions.end()) {