Limit memory used by Pl_PNGFilter and Pl_TIFFPredictor during fuzzing

m-holger
1 parent fe1fffe8
Showing 5 changed files with 41 additions and 0 deletions
fuzz/qpdf_fuzzer.cc
libqpdf/Pl_PNGFilter.cc
libqpdf/Pl_TIFFPredictor.cc
libqpdf/qpdf/Pl_PNGFilter.hh
libqpdf/qpdf/Pl_TIFFPredictor.hh
@@ -2,6 +2,8 @@
 #include <qpdf/BufferInputSource.hh>
 #include <qpdf/Pl_DCT.hh>
 #include <qpdf/Pl_Discard.hh>
+#include <qpdf/Pl_PNGFilter.hh>
+#include <qpdf/Pl_TIFFPredictor.hh>
 #include <qpdf/QPDF.hh>
 #include <qpdf/QPDFAcroFormDocumentHelper.hh>
 #include <qpdf/QPDFOutlineDocumentHelper.hh>
@@ -179,6 +181,9 @@ FuzzHelper::doChecks()
     // occur legitimately and therefore must be allowed during normal operations.
     Pl_DCT::setMemoryLimit(1'000'000'000);
+    Pl_PNGFilter::setMemoryLimit(1'000'000'000);
+    Pl_TIFFPredictor::setMemoryLimit(1'000'000'000);
+
     // Do not decompress corrupt data. This may cause extended runtime within jpeglib without
     // exercising additional code paths in qpdf, and potentially causing counterproductive timeouts.
     Pl_DCT::setThrowOnCorruptData(true);
@@ -7,6 +7,11 @@
 #include <cstring>
 #include <stdexcept>
+namespace
+{
+    unsigned long long memory_limit{0};
+} // namespace
+
 static int
 abs_diff(int a, int b)
 {
@@ -41,6 +46,9 @@ Pl_PNGFilter::Pl_PNGFilter(
     if ((bpr == 0) || (bpr > (UINT_MAX - 1))) {
         throw std::runtime_error("PNGFilter created with invalid columns value");
     }
+    if (memory_limit > 0 && bpr > (memory_limit / 2U)) {
+        throw std::runtime_error("PNGFilter memory limit exceeded");
+    }
     this->bytes_per_row = bpr & UINT_MAX;
     this->buf1 = QUtil::make_shared_array<unsigned char>(this->bytes_per_row + 1);
     this->buf2 = QUtil::make_shared_array<unsigned char>(this->bytes_per_row + 1);
@@ -54,6 +62,12 @@ Pl_PNGFilter::Pl_PNGFilter(
 }
 void
+Pl_PNGFilter::setMemoryLimit(unsigned long long limit)
+{
+    memory_limit = limit;
+}
+
+void
 Pl_PNGFilter::write(unsigned char const* data, size_t len)
 {
     size_t left = this->incoming - this->pos;
@@ -7,6 +7,11 @@
 #include <climits>
 #include <stdexcept>
+namespace
+{
+    unsigned long long memory_limit{0};
+} // namespace
+
 Pl_TIFFPredictor::Pl_TIFFPredictor(
     char const* identifier,
     Pipeline* next,
@@ -31,10 +36,19 @@ Pl_TIFFPredictor::Pl_TIFFPredictor(
     if ((bpr == 0) || (bpr > (UINT_MAX - 1))) {
         throw std::runtime_error("TIFFPredictor created with invalid columns value");
     }
+    if (memory_limit > 0 && bpr > (memory_limit / 2U)) {
+        throw std::runtime_error("TIFFPredictor memory limit exceeded");
+    }
     this->bytes_per_row = bpr & UINT_MAX;
 }
 void
+Pl_TIFFPredictor::setMemoryLimit(unsigned long long limit)
+{
+    memory_limit = limit;
+}
+
+void
 Pl_TIFFPredictor::write(unsigned char const* data, size_t len)
 {
     auto end = data + len;
@@ -24,6 +24,10 @@ class Pl_PNGFilter: public Pipeline
         unsigned int bits_per_sample = 8);
     ~Pl_PNGFilter() override = default;
+    // Limit the memory used.
+    // NB This is a static option affecting all Pl_PNGFilter instances.
+    static void setMemoryLimit(unsigned long long limit);
+
     void write(unsigned char const* data, size_t len) override;
     void finish() override;
@@ -22,6 +22,10 @@ class Pl_TIFFPredictor: public Pipeline
         unsigned int bits_per_sample = 8);
     ~Pl_TIFFPredictor() override = default;
+    // Limit the memory used.
+    // NB This is a static option affecting all Pl_TIFFPredictor instances.
+    static void setMemoryLimit(unsigned long long limit);
+
     void write(unsigned char const* data, size_t len) override;
     void finish() override;