Bug fix: honor encryption key length with R=3 (fixes #212)

Jay Berkenbilt
1 parent c543c1e4
Showing 4 changed files with 82 additions and 5 deletions
libqpdf/QPDF_encryption.cc
qpdf/qtest/qpdf.test
qpdf/qtest/qpdf/encrypted-40-bit-R3.out
qpdf/qtest/qpdf/encrypted-40-bit-R3.pdf
@@ -183,7 +183,7 @@ truncate_password_V5(std::string const&amp; password)
 }
 static void
-iterate_md5_digest(MD5& md5, MD5::Digest& digest, int iterations)
+iterate_md5_digest(MD5& md5, MD5::Digest& digest, int iterations, int key_len)
 {
     md5.digest(digest);
@@ -191,7 +191,7 @@ iterate_md5_digest(MD5&amp; md5, MD5::Digest&amp; digest, int iterations)
     {
 	MD5 m;
 	m.encodeDataIncrementally(reinterpret_cast<char*>(digest),
-                                  sizeof(digest));
+                                  key_len);
 	m.digest(digest);
     }
 }
@@ -437,7 +437,8 @@ QPDF::compute_encryption_key_from_password(
 	md5.encodeDataIncrementally(bytes, 4);
     }
     MD5::Digest digest;
-    iterate_md5_digest(md5, digest, ((data.getR() >= 3) ? 50 : 0));
+    iterate_md5_digest(md5, digest, ((data.getR() >= 3) ? 50 : 0),
+                       data.getLengthBytes());
     return std::string(reinterpret_cast<char*>(digest),
                        std::min(static_cast<int>(sizeof(digest)),
                                 data.getLengthBytes()));
@@ -463,7 +464,8 @@ compute_O_rc4_key(std::string const&amp; user_password,
     md5.encodeDataIncrementally(
 	pad_or_truncate_password_V4(password).c_str(), key_bytes);
     MD5::Digest digest;
-    iterate_md5_digest(md5, digest, ((data.getR() >= 3) ? 50 : 0));
+    iterate_md5_digest(md5, digest, ((data.getR() >= 3) ? 50 : 0),
+                       data.getLengthBytes());
     memcpy(key, digest, OU_key_bytes_V4);
 }
@@ -933,6 +935,11 @@ QPDF::initializeEncryption()
     if (encryption_dict.getKey("/Length").isInteger())
     {
 	Length = encryption_dict.getKey("/Length").getIntValue();
+        if (R < 3)
+        {
+            // Force Length to 40 regardless of what the file says.
+            Length = 40;
+        }
 	if ((Length % 8) || (Length < 40) || (Length > 256))
 	{
 	    throw QPDFExc(qpdf_e_damaged_pdf, this->m->file->getName(),
@@ -453,7 +453,7 @@ $td-&gt;runtest(&quot;check output&quot;,
 show_ntests();
 # ----------
 $td->notify("--- ID and Encryption Parameter Issues ---");
-$n_tests += 12;
+$n_tests += 13;
 # Encrypt files whose /ID strings are other than 32 bytes long (bug
 # 2991412). Also linearize these files, which was reported in a
@@ -509,6 +509,19 @@ $td-&gt;runtest(&quot;short /O or /U&quot;,
               $td->EXIT_STATUS => 0},
              $td->NORMALIZE_NEWLINES);
+# A file was sent to me privately as part of issue 212. This file was
+# encrypted and had /R=3 and /V=1 and was using a 40-bit key. qpdf was
+# failing to work properly on files with /R=3 and 40-bit keys. The
+# test file is not this private file, but the encryption parameters
+# were copied from it. Like the bug file, qpdf < 8.1 can't decrypt it.
+$td->runtest("/R 3 with 40-bit key",
+             {$td->COMMAND =>
+                  "qpdf --password=623 --check --show-encryption-key" .
+                  " encrypted-40-bit-R3.pdf"},
+             {$td->FILE => "encrypted-40-bit-R3.out",
+              $td->EXIT_STATUS => 0},
+             $td->NORMALIZE_NEWLINES);
+
 show_ntests();
 # ----------
 $td->notify("--- Min/force version ---");
+checking encrypted-40-bit-R3.pdf
+PDF Version: 1.4
+R = 3
+P = -12
+User password = 623
+Encryption key = e390e220da
+extract for accessibility: allowed
+extract for any purpose: allowed
+print low resolution: allowed
+print high resolution: allowed
+modify document assembly: allowed
+modify forms: allowed
+modify annotations: allowed
+modify other: not allowed
+modify anything: not allowed
+File is not linearized
+No syntax or stream encoding errors found; the file may still contain
+errors that qpdf cannot detect