AES_PDF: move CBC logic from pipeline to AES_PDF implementation

Jay Berkenbilt
1 parent c8cda4f9
Showing 7 changed files with 50 additions and 29 deletions
include/qpdf/QPDFCryptoImpl.hh
libqpdf/AES_PDF_native.cc
libqpdf/Pl_AES_PDF.cc
libqpdf/QPDFCrypto_native.cc
libqpdf/qpdf/AES_PDF_native.hh
libqpdf/qpdf/Pl_AES_PDF.hh
libqpdf/qpdf/QPDFCrypto_native.hh
@@ -74,9 +74,11 @@ class QPDF_DLL_CLASS QPDFCryptoImpl
     QPDF_DLL
     virtual void RC4_finalize() = 0;
+    static size_t constexpr rijndael_buf_size = 16;
     QPDF_DLL
     virtual void rijndael_init(
-        bool encrypt, unsigned char const* key_data, size_t key_len) = 0;
+        bool encrypt, unsigned char const* key_data, size_t key_len,
+        bool cbc_mode, unsigned char* cbc_block) = 0;
     QPDF_DLL
     virtual void rijndael_process(
         unsigned char* in_data, unsigned char* out_data) = 0;
@@ -7,10 +7,14 @@
 #include <qpdf/QIntC.hh>
 #include <string>
 #include <stdlib.h>
+#include <qpdf/QPDFCryptoImpl.hh>
 AES_PDF_native::AES_PDF_native(bool encrypt, unsigned char const* key,
-                               size_t key_bytes) :
+                               size_t key_bytes, bool cbc_mode,
+                               unsigned char* cbc_block) :
     encrypt(encrypt),
+    cbc_mode(cbc_mode),
+    cbc_block(cbc_block),
     nrounds(0)
 {
     size_t keybits = 8 * key_bytes;
@@ -44,12 +48,33 @@ AES_PDF_native::update(unsigned char* in_data, unsigned char* out_data)
 {
     if (this->encrypt)
     {
+	if (this->cbc_mode)
+	{
+	    for (size_t i = 0; i < QPDFCryptoImpl::rijndael_buf_size; ++i)
+	    {
+		in_data[i] ^= this->cbc_block[i];
+	    }
+	}
 	rijndaelEncrypt(this->rk.get(),
                         this->nrounds, in_data, out_data);
+	if (this->cbc_mode)
+	{
+	    memcpy(this->cbc_block, out_data,
+                   QPDFCryptoImpl::rijndael_buf_size);
+	}
     }
     else
     {
 	rijndaelDecrypt(this->rk.get(),
                         this->nrounds, in_data, out_data);
+	if (this->cbc_mode)
+	{
+	    for (size_t i = 0; i < QPDFCryptoImpl::rijndael_buf_size; ++i)
+	    {
+		out_data[i] ^= this->cbc_block[i];
+	    }
+	    memcpy(this->cbc_block, in_data,
+                   QPDFCryptoImpl::rijndael_buf_size);
+	}
     }
 }
@@ -19,6 +19,7 @@ Pl_AES_PDF::Pl_AES_PDF(char const* identifier, Pipeline* next,
     cbc_mode(true),
     first(true),
     offset(0),
+    key_bytes(key_bytes),
     use_zero_iv(false),
     use_specified_iv(false),
     disable_padding(false)
@@ -30,7 +31,6 @@ Pl_AES_PDF::Pl_AES_PDF(char const* identifier, Pipeline* next,
     std::memset(this->inbuf, 0, this->buf_size);
     std::memset(this->outbuf, 0, this->buf_size);
     std::memset(this->cbc_block, 0, this->buf_size);
-    this->crypto->rijndael_init(encrypt, this->key.get(), key_bytes);
 }
 Pl_AES_PDF::~Pl_AES_PDF()
@@ -172,6 +172,7 @@ Pl_AES_PDF::flush(bool strip_padding)
     if (first)
     {
 	first = false;
+        bool return_after_init = false;
 	if (this->cbc_mode)
 	{
 	    if (encrypt)
@@ -196,37 +197,25 @@ Pl_AES_PDF::flush(bool strip_padding)
 		// vector.  There's nothing to write at this time.
 		memcpy(this->cbc_block, this->inbuf, this->buf_size);
 		this->offset = 0;
-		return;
+                return_after_init = true;
 	    }
 	}
+        this->crypto->rijndael_init(
+            encrypt, this->key.get(), key_bytes,
+            this->cbc_mode, this->cbc_block);
+        if (return_after_init)
+        {
+            return;
+        }
     }
     if (this->encrypt)
     {
-	if (this->cbc_mode)
-	{
-	    for (unsigned int i = 0; i < this->buf_size; ++i)
-	    {
-		this->inbuf[i] ^= this->cbc_block[i];
-	    }
-	}
 	this->crypto->rijndael_process(this->inbuf, this->outbuf);
-	if (this->cbc_mode)
-	{
-	    memcpy(this->cbc_block, this->outbuf, this->buf_size);
-	}
     }
     else
     {
 	this->crypto->rijndael_process(this->inbuf, this->outbuf);
-	if (this->cbc_mode)
-	{
-	    for (unsigned int i = 0; i < this->buf_size; ++i)
-	    {
-		this->outbuf[i] ^= this->cbc_block[i];
-	    }
-	    memcpy(this->cbc_block, this->inbuf, this->buf_size);
-	}
     }
     unsigned int bytes = this->buf_size;
     if (strip_padding)
@@ -69,10 +69,12 @@ QPDFCrypto_native::SHA2_digest()
 void
 QPDFCrypto_native::rijndael_init(
-    bool encrypt, unsigned char const* key_data, size_t key_len)
+    bool encrypt, unsigned char const* key_data, size_t key_len,
+    bool cbc_mode, unsigned char* cbc_block)
+
 {
     this->aes_pdf = std::make_shared<AES_PDF_native>(
-        encrypt, key_data, key_len);
+        encrypt, key_data, key_len, cbc_mode, cbc_block);
 }
 void
@@ -10,13 +10,15 @@ class AES_PDF_native
   public:
     // key should be a pointer to key_bytes bytes of data
     AES_PDF_native(bool encrypt, unsigned char const* key,
-                   size_t key_bytes);
+                   size_t key_bytes, bool cbc_mode, unsigned char* cbc_block);
     ~AES_PDF_native();
     void update(unsigned char* in_data, unsigned char* out_data);
   private:
     bool encrypt;
+    bool cbc_mode;
+    unsigned char* cbc_block;
     std::unique_ptr<unsigned char[]> key;
     std::unique_ptr<uint32_t[]> rk;
     unsigned int nrounds;
@@ -2,7 +2,6 @@
 #define PL_AES_PDF_HH
 #include <qpdf/Pipeline.hh>
-#include <qpdf/qpdf-config.h>
 #include <qpdf/QPDFCryptoImpl.hh>
 #include <memory>
@@ -47,7 +46,7 @@ class Pl_AES_PDF: public Pipeline
     void flush(bool discard_padding);
     void initializeVector();
-    static unsigned int const buf_size = 16;
+    static unsigned int const buf_size = QPDFCryptoImpl::rijndael_buf_size;
     static bool use_static_iv;
     std::shared_ptr<QPDFCryptoImpl> crypto;
@@ -56,6 +55,7 @@ class Pl_AES_PDF: public Pipeline
     bool first;
     size_t offset;              // offset into memory buffer
     std::unique_ptr<unsigned char[]> key;
+    size_t key_bytes;
     unsigned char inbuf[buf_size];
     unsigned char outbuf[buf_size];
     unsigned char cbc_block[buf_size];
@@ -33,7 +33,8 @@ class QPDFCrypto_native: public QPDFCryptoImpl
     virtual std::string SHA2_digest();
     virtual void rijndael_init(
-        bool encrypt, unsigned char const* key_data, size_t key_len);
+        bool encrypt, unsigned char const* key_data, size_t key_len,
+        bool cbc_mode, unsigned char* cbc_block);
     virtual void rijndael_process(
         unsigned char* in_data, unsigned char* out_data);
     virtual void rijndael_finalize();