Fix major performance bug with openssl crypto (fixes #798)

Lazily load MD5 and RC4 once in the life of the program. Only load the legacy provider if RC4 is actually being used.

Fix major performance bug with openssl crypto (fixes #798)
Lazily load MD5 and RC4 once in the life of the program. Only load the legacy provider if RC4 is actually being used.
Jay Berkenbilt
1 parent b7459209
Showing 4 changed files with 73 additions and 36 deletions
ChangeLog
libqpdf/QPDFCrypto_openssl.cc
libqpdf/qpdf/QPDFCrypto_openssl.hh
manual/release-notes.rst
 2022-10-08  Jay Berkenbilt  <ejb@ql.org>
+        * Fix major performance bug with the openssl crypto provider when
+        using OpenSSL 3. The legacy loader and rc4 algorithm was being
+        loaded with every call to the crypto provider instead of once in
+        the life of the program. Fixes #798.
+
         * performance_check: add --test option to limit which tests are
         run.
 #include <qpdf/QPDFCrypto_openssl.hh>
 #include <cstring>
+#include <memory>
 #include <stdexcept>
 #include <string>
@@ -18,6 +19,60 @@
 #include <qpdf/QIntC.hh>
+#ifndef QPDF_OPENSSL_1
+namespace
+{
+    class RC4Loader
+    {
+      public:
+        static EVP_CIPHER const* getRC4();
+        ~RC4Loader();
+
+      private:
+        RC4Loader();
+        OSSL_PROVIDER* legacy;
+        OSSL_LIB_CTX* libctx;
+        EVP_CIPHER* rc4;
+    };
+} // namespace
+
+EVP_CIPHER const*
+RC4Loader::getRC4()
+{
+    static auto loader = std::shared_ptr<RC4Loader>(new RC4Loader());
+    return loader->rc4;
+}
+
+RC4Loader::RC4Loader()
+{
+    libctx = OSSL_LIB_CTX_new();
+    if (libctx == nullptr) {
+        throw std::runtime_error("unable to create openssl library context");
+        return;
+    }
+    legacy = OSSL_PROVIDER_load(libctx, "legacy");
+    if (legacy == nullptr) {
+        OSSL_LIB_CTX_free(libctx);
+        throw std::runtime_error("unable to load openssl legacy provider");
+        return;
+    }
+    rc4 = EVP_CIPHER_fetch(libctx, "RC4", nullptr);
+    if (rc4 == nullptr) {
+        OSSL_PROVIDER_unload(legacy);
+        OSSL_LIB_CTX_free(libctx);
+        throw std::runtime_error("unable to load openssl rc4 algorithm");
+        return;
+    }
+}
+
+RC4Loader::~RC4Loader()
+{
+    EVP_CIPHER_free(rc4);
+    OSSL_PROVIDER_unload(legacy);
+    OSSL_LIB_CTX_free(libctx);
+}
+#endif // not QPDF_OPENSSL_1
+
 static void
 bad_bits(int bits)
 {
@@ -41,32 +96,9 @@ check_openssl(int status)
 }
 QPDFCrypto_openssl::QPDFCrypto_openssl() :
-#ifdef QPDF_OPENSSL_1
-    rc4(EVP_rc4()),
-#endif
     md_ctx(EVP_MD_CTX_new()),
     cipher_ctx(EVP_CIPHER_CTX_new())
 {
-#ifndef QPDF_OPENSSL_1
-    libctx = OSSL_LIB_CTX_new();
-    if (libctx == nullptr) {
-        throw std::runtime_error("unable to create openssl library context");
-        return;
-    }
-    legacy = OSSL_PROVIDER_load(libctx, "legacy");
-    if (legacy == nullptr) {
-        OSSL_LIB_CTX_free(libctx);
-        throw std::runtime_error("unable to load openssl legacy provider");
-        return;
-    }
-    rc4 = EVP_CIPHER_fetch(libctx, "RC4", nullptr);
-    if (rc4 == nullptr) {
-        OSSL_PROVIDER_unload(legacy);
-        OSSL_LIB_CTX_free(libctx);
-        throw std::runtime_error("unable to load openssl rc4 algorithm");
-        return;
-    }
-#endif
     memset(md_out, 0, sizeof(md_out));
     EVP_MD_CTX_init(md_ctx);
     EVP_CIPHER_CTX_init(cipher_ctx);
@@ -77,11 +109,6 @@ QPDFCrypto_openssl::~QPDFCrypto_openssl()
     EVP_MD_CTX_reset(md_ctx);
     EVP_CIPHER_CTX_reset(cipher_ctx);
     EVP_CIPHER_CTX_free(cipher_ctx);
-#ifndef QPDF_OPENSSL_1
-    EVP_CIPHER_free(rc4);
-    OSSL_PROVIDER_unload(legacy);
-    OSSL_LIB_CTX_free(libctx);
-#endif
     EVP_MD_CTX_free(md_ctx);
 }
@@ -101,7 +128,7 @@ QPDFCrypto_openssl::MD5_init()
 void
 QPDFCrypto_openssl::SHA2_init(int bits)
 {
-    const EVP_MD* md = EVP_sha512();
+    static const EVP_MD* md = EVP_sha512();
     switch (bits) {
     case 256:
         md = EVP_sha256();
@@ -174,6 +201,11 @@ QPDFCrypto_openssl::SHA2_digest()
 void
 QPDFCrypto_openssl::RC4_init(unsigned char const* key_data, int key_len)
 {
+#ifdef QPDF_OPENSSL_1
+    static auto const rc4 = EVP_rc4();
+#else
+    static auto const rc4 = RC4Loader::getRC4();
+#endif
     check_openssl(EVP_CIPHER_CTX_reset(cipher_ctx));
     if (key_len == -1) {
         key_len =
@@ -58,13 +58,6 @@ class QPDFCrypto_openssl: public QPDFCryptoImpl
     void rijndael_finalize() override;
   private:
-#ifdef QPDF_OPENSSL_1
-    EVP_CIPHER const* rc4;
-#else
-    OSSL_LIB_CTX* libctx;
-    OSSL_PROVIDER* legacy;
-    EVP_CIPHER* rc4;
-#endif
     EVP_MD_CTX* const md_ctx;
     EVP_CIPHER_CTX* const cipher_ctx;
     uint8_t md_out[EVP_MAX_MD_SIZE];
@@ -13,6 +13,13 @@ For a detailed list of changes, please see the file
     - A C++-17 compiler is now required.
+  - Bug fixes
+
+    - Fix major performance bug with the OpenSSL crypto provider. This
+      bug was causing a 6x to 12x slowdown for encrypted files when
+      OpenSSL 3 was in use. This includes the default Windows builds
+      distributed with the qpdf release.
+
 11.1.1: October 1, 2022
   - Bug fixes