Merge pull request #1340 from m-holger/i1286

Change QPDFWriter stream_decode_level default to qpdf_dl_generalized (fixes #1286)

Merge pull request #1340 from m-holger/i1286
Change QPDFWriter stream_decode_level default to qpdf_dl_generalized (fixes #1286)
m-holger · GitHub
2 parents d9e230bf e7dcbe93
Showing 12 changed files with 52 additions and 23 deletions
ChangeLog
libqpdf/QPDFWriter.cc
libqpdf/qpdf/QPDFWriter_private.hh
qpdf/qtest/encryption.test
qpdf/qtest/inline-images.test
qpdf/qtest/qpdf/bad-encryption-length.out
qpdf/qtest/qpdf/damaged-inline-image.out
qpdf/qtest/qpdf/fuzz-16214.out
qpdf/qtest/qpdf/invalid-id-xref.out
qpdf/qtest/qpdf/job-json-input-file-password.pdf
qpdf/qtest/qpdf/qjson-bad-data.out
qpdf/qtest/qpdf/qjson-bad-datafile.out
@@ -9,6 +9,11 @@
 	* Add C API function qpdf_oh_free_buffer to release memory allocated
 	by stream data functions.
+2024-09-19  M Holger  <m.holger@qpdf.org>
+
+	* Bug fix: QPDFWriter stream DecodeLevel incorrectly defaulted to
+	none instead of generalied. Fixes #1286.
+
 2024-08-25  M Holger  <m.holger@qpdf.org>
 	* Add new command-line arguments --remove-metadata and --remove-info
@@ -26,6 +26,8 @@
 #include <cstdlib>
 #include <stdexcept>
+using namespace std::literals;
+
 QPDFWriter::ProgressReporter::~ProgressReporter() // NOLINT (modernize-use-equals-default)
 {
     // Must be explicit and not inline -- see QPDF_DLL_CLASS in README-maintainer
@@ -1249,7 +1251,7 @@ QPDFWriter::willFilterStream(
     if (stream_dict.isDictionaryOfType("/Metadata")) {
         is_metadata = true;
     }
-    bool filter = (stream.isDataModified() || m->compress_streams || m->stream_decode_level);
+    bool filter = stream.isDataModified() || m->compress_streams || m->stream_decode_level;
     bool filter_on_write = stream.getFilterOnWrite();
     if (!filter_on_write) {
         QTC::TC("qpdf", "QPDFWriter getFilterOnWrite false");
@@ -1261,15 +1263,15 @@ QPDFWriter::willFilterStream(
         // CPU cycles uncompressing and recompressing stuff. This can be overridden with
         // setRecompressFlate(true).
         QPDFObjectHandle filter_obj = stream_dict.getKey("/Filter");
-        if ((!m->recompress_flate) && (!stream.isDataModified()) && filter_obj.isName() &&
-            ((filter_obj.getName() == "/FlateDecode") || (filter_obj.getName() == "/Fl"))) {
+        if (!m->recompress_flate && !stream.isDataModified() && filter_obj.isName() &&
+            (filter_obj.getName() == "/FlateDecode" || filter_obj.getName() == "/Fl")) {
             QTC::TC("qpdf", "QPDFWriter not recompressing /FlateDecode");
             filter = false;
         }
     }
     bool normalize = false;
     bool uncompress = false;
-    if (filter_on_write && is_metadata && ((!m->encrypted) || (m->encrypt_metadata == false))) {
+    if (filter_on_write && is_metadata && (!m->encrypted || !m->encrypt_metadata)) {
         QTC::TC("qpdf", "QPDFWriter not compressing metadata");
         filter = true;
         compress_stream = false;
@@ -1283,29 +1285,37 @@ QPDFWriter::willFilterStream(
     }
     bool filtered = false;
-    for (int attempt = 1; attempt <= 2; ++attempt) {
+    for (bool first_attempt: {true, false}) {
         pushPipeline(new Pl_Buffer("stream data"));
         PipelinePopper pp_stream_data(this, stream_data);
         activatePipelineStack(pp_stream_data);
         try {
             filtered = stream.pipeStreamData(
                 m->pipeline,
-                (((filter && normalize) ? qpdf_ef_normalize : 0) |
-                 ((filter && compress_stream) ? qpdf_ef_compress : 0)),
-                (filter ? (uncompress ? qpdf_dl_all : m->stream_decode_level) : qpdf_dl_none),
+                !filter ? 0
+                        : ((normalize ? qpdf_ef_normalize : 0) |
+                           (compress_stream ? qpdf_ef_compress : 0)),
+                !filter ? qpdf_dl_none : (uncompress ? qpdf_dl_all : m->stream_decode_level),
                 false,
-                (attempt == 1));
+                first_attempt);
+            if (filter && !filtered) {
+                // Try again
+                filter = false;
+                stream.setFilterOnWrite(false);
+            } else {
+                break;
+            }
         } catch (std::runtime_error& e) {
+            if (filter && first_attempt) {
+                stream.warnIfPossible("error while getting stream data: "s + e.what());
+                stream.warnIfPossible("qpdf will attempt to write the damaged stream unchanged");
+                filter = false;
+                stream.setFilterOnWrite(false);
+                continue;
+            }
             throw std::runtime_error(
                 "error while getting stream data for " + stream.unparse() + ": " + e.what());
         }
-        if (filter && !filtered) {
-            // Try again
-            filter = false;
-            stream.setFilterOnWrite(false);
-        } else {
-            break;
-        }
     }
     if (!filtered) {
         compress_stream = false;
@@ -1866,7 +1876,7 @@ QPDFWriter::generateID()
         if (m->deterministic_id) {
             if (m->deterministic_id_data.empty()) {
                 QTC::TC("qpdf", "QPDFWriter deterministic with no data");
-                throw std::logic_error("INTERNAL ERROR: QPDFWriter::generateID has no data for "
+                throw std::runtime_error("INTERNAL ERROR: QPDFWriter::generateID has no data for "
                                        "deterministic ID.  This may happen if deterministic ID and "
                                        "file encryption are requested together.");
             }
@@ -2116,7 +2126,7 @@ QPDFWriter::doWriteSetup()
     if (m->encrypted) {
         // Encryption has been explicitly set
         m->preserve_encryption = false;
-    } else if (m->normalize_content || m->stream_decode_level || m->pclm || m->qdf_mode) {
+    } else if (m->normalize_content || !m->compress_streams || m->pclm || m->qdf_mode) {
         // Encryption makes looking at contents pretty useless.  If the user explicitly encrypted
         // though, we still obey that.
         m->preserve_encryption = false;
@@ -65,7 +65,7 @@ class QPDFWriter::Members
     bool normalize_content{false};
     bool compress_streams{true};
     bool compress_streams_set{false};
-    qpdf_stream_decode_level_e stream_decode_level{qpdf_dl_none};
+    qpdf_stream_decode_level_e stream_decode_level{qpdf_dl_generalized};
     bool stream_decode_level_set{false};
     bool recompress_flate{false};
     bool qdf_mode{false};
@@ -723,7 +723,7 @@ $td-&gt;runtest(&quot;check file&#39;s validity&quot;,
 $td->runtest("handle missing/invalid Length",
              {$td->COMMAND => "qpdf --check bad-encryption-length.pdf"},
              {$td->FILE => "bad-encryption-length.out",
-                  $td->EXIT_STATUS => 0},
+                  $td->EXIT_STATUS => 3},
              $td->NORMALIZE_NEWLINES);
 cleanup();
@@ -56,7 +56,7 @@ $td-&gt;runtest(&quot;externalize damaged image&quot;,
                   "qpdf --externalize-inline-images" .
                   " --compress-streams=n --static-id" .
                   " damaged-inline-image.pdf a.pdf"},
-             {$td->STRING => "", $td->EXIT_STATUS => 0},
+             {$td->FILE => "damaged-inline-image.out", $td->EXIT_STATUS => 3},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("check output",
              {$td->FILE => "a.pdf"},
@@ -14,5 +14,5 @@ modify annotations: allowed
 modify other: allowed
 modify anything: allowed
 File is not linearized
-No syntax or stream encoding errors found; the file may still contain
-errors that qpdf cannot detect
+WARNING: bad-encryption-length.pdf, object 7 0 at offset 531 -> dictionary key /Length: operation for integer attempted on object of type null: returning 0
+qpdf: operation succeeded with warnings
+WARNING: damaged-inline-image.pdf, stream object 8 0: error while getting stream data: stream inflate: inflate: data: invalid distance too far back
+WARNING: damaged-inline-image.pdf, stream object 8 0: qpdf will attempt to write the damaged stream unchanged
+qpdf: operation succeeded with warnings; resulting file may have some problems
@@ -6,6 +6,11 @@ WARNING: fuzz-16214.pdf (xref stream, offset 116): Cross-reference stream data h
 WARNING: fuzz-16214.pdf: reported number of objects (6) is not one plus the highest object number (35)
 WARNING: fuzz-16214.pdf (object 14 0, offset 652): expected dictionary key but found non-name object; inserting key /QPDFFake1
 WARNING: fuzz-16214.pdf (object 14 0, offset 734): expected endobj
+WARNING: fuzz-16214.pdf (object 15 0, offset 869): unknown token while reading object; treating as string
+WARNING: fuzz-16214.pdf (object 15 0, offset 745): expected dictionary key but found non-name object; inserting key /QPDFFake1
+WARNING: fuzz-16214.pdf (object 15 0, offset 894): expected endobj
+WARNING: fuzz-16214.pdf, object 15 0 at offset 745: kid 0 (from 0) MediaBox is undefined; setting to letter / ANSI A
+WARNING: fuzz-16214.pdf, object 15 0 at offset 745: /Type key should be /Page but is not; overriding
 WARNING: fuzz-16214.pdf: file is damaged
 WARNING: fuzz-16214.pdf (object 1 0, offset 7189): expected n n obj
 WARNING: fuzz-16214.pdf: Attempting to reconstruct cross-reference table
@@ -15,4 +15,6 @@ modify annotations: allowed
 modify other: not allowed
 modify anything: not allowed
 File is not linearized
+WARNING: invalid-id-xref.pdf, trailer at offset 910 -> dictionary key /ID: operation for array attempted on object of type null: returning null
+WARNING: invalid-id-xref.pdf, trailer at offset 910 -> dictionary key /ID -> null returned from invalid array access: operation for string attempted on object of type null: returning empty string
 qpdf: operation succeeded with warnings
 Binary files a/qpdf/qtest/qpdf/job-json-input-file-password.pdf and b/qpdf/qtest/qpdf/job-json-input-file-password.pdf differ
+WARNING: qjson-bad-data.json, obj:4 0 R at offset 0: error while getting stream data: base64-decode: base64 decode: invalid input
+WARNING: qjson-bad-data.json, obj:4 0 R at offset 0: qpdf will attempt to write the damaged stream unchanged
 qpdf: error while getting stream data for 4 0 R: base64-decode: base64 decode: invalid input
+WARNING: qjson-bad-datafile.json, obj:4 0 R at offset 0: error while getting stream data: open file does not exist: No such file or directory
+WARNING: qjson-bad-datafile.json, obj:4 0 R at offset 0: qpdf will attempt to write the damaged stream unchanged
 qpdf: error while getting stream data for 4 0 R: open file does not exist: No such file or directory