Add QUtil::get_max_memory_usage for testing

Jay Berkenbilt
1 parent 0adfd74f
Showing 7 changed files with 127 additions and 1 deletions
cSpell.json
include/qpdf/QUtil.hh
libqpdf/CMakeLists.txt
libqpdf/QUtil.cc
libqpdf/qpdf/qpdf-config.h.in
libtests/qtest/qutil/qutil.out
libtests/qutil.cc
@@ -131,6 +131,7 @@
     "esize",
     "eval",
     "extlibdir",
+    "fclose",
     "fdict",
     "ffield",
     "fghij",
@@ -268,6 +269,7 @@
     "maxdepth",
     "maxobjectid",
     "mdash",
+    "memstream",
     "mindepth",
     "mkdir",
     "mkinstalldirs",
@@ -525,7 +525,17 @@ namespace QUtil
         wchar_t const* const argv[],
         std::function<int(int, char const* const[])> realmain);
 #endif // QPDF_NO_WCHAR_T
-};     // namespace QUtil
+
+    // Try to return the maximum amount of memory allocated by the
+    // current process and its threads. Return 0 if unable to
+    // determine. This is Linux-specific and not implemented to be
+    // completely reliable. It is used during development for
+    // performance testing to detect changes that may significantly
+    // change memory usage. It is not recommended for use for other
+    // purposes.
+    QPDF_DLL
+    size_t get_max_memory_usage();
+}; // namespace QUtil
  
 inline bool
 QUtil::is_hex_digit(char ch)
@@ -375,6 +375,29 @@ int main(int argc, char* argv[]) {
   endif()
 endfunction()
  
+check_c_source_compiles(
+"#include <malloc.h>
+#include <stdio.h>
+int main(int argc, char* argv[]) {
+    malloc_info(0, stdout);
+    return 0;
+}"
+    HAVE_MALLOC_INFO)
+
+check_c_source_compiles(
+"#include <stdio.h>
+#include <stdlib.h>
+int main(int argc, char* argv[]) {
+    char* buf;
+    size_t size;
+    FILE* f;
+    f = open_memstream(&buf, &size);
+    fclose(f);
+    free(buf);
+    return 0;
+}"
+    HAVE_OPEN_MEMSTREAM)
+
 qpdf_check_ll_fmt("%lld" fmt_lld)
 qpdf_check_ll_fmt("%I64d" fmt_i64d)
 qpdf_check_ll_fmt("%I64lld" fmt_i64lld)
@@ -37,6 +37,9 @@
 # include <sys/stat.h>
 # include <unistd.h>
 #endif
+#ifdef HAVE_MALLOC_INFO
+# include <malloc.h>
+#endif
  
 // First element is 24
 static unsigned short pdf_doc_low_to_unicode[] = {
@@ -1968,3 +1971,73 @@ QUtil::call_main_from_wmain(
 }
  
 #endif // QPDF_NO_WCHAR_T
+
+size_t
+QUtil::get_max_memory_usage()
+{
+#if defined(HAVE_MALLOC_INFO) && defined(HAVE_OPEN_MEMSTREAM)
+    static std::regex tag_re("<(/?\\w+)([^>]*?)>");
+    static std::regex attr_re("(\\w+)=\"(.*?)\"");
+
+    char* buf;
+    size_t size;
+    FILE* f = open_memstream(&buf, &size);
+    if (f == nullptr) {
+        return 0;
+    }
+    malloc_info(0, f);
+    fclose(f);
+    if (QUtil::get_env("QPDF_DEBUG_MEM_USAGE")) {
+        fprintf(stderr, "%s", buf);
+    }
+
+    // Warning: this code uses regular expression to extract data from
+    // an XML string. This is generally a bad idea, but we're going to
+    // do it anyway because QUtil.hh warns against using this function
+    // for other than development/testing, and if this function fails
+    // to generate reasonable output during performance testing, it
+    // will be noticed.
+
+    // This is my best guess at how to interpret malloc_info. Anyway
+    // it seems to provide useful information for detecting code
+    // changes that drastically change memory usage.
+    size_t result = 0;
+    try {
+        std::cregex_iterator m_begin(buf, buf + size, tag_re);
+        std::cregex_iterator cr_end;
+        std::sregex_iterator sr_end;
+
+        int in_heap = 0;
+        for (auto m = m_begin; m != cr_end; ++m) {
+            std::string tag(m->str(1));
+            if (tag == "heap") {
+                ++in_heap;
+            } else if (tag == "/heap") {
+                --in_heap;
+            } else if (in_heap == 0) {
+                std::string rest = m->str(2);
+                std::map<std::string, std::string> attrs;
+                std::sregex_iterator a_begin(rest.begin(), rest.end(), attr_re);
+                for (auto m2 = a_begin; m2 != sr_end; ++m2) {
+                    attrs[m2->str(1)] = m2->str(2);
+                }
+                if (tag == "total") {
+                    if (attrs.count("size") > 0) {
+                        result += QIntC::to_size(
+                            QUtil::string_to_ull(attrs["size"].c_str()));
+                    }
+                } else if (tag == "system" && attrs["type"] == "max") {
+                    result += QIntC::to_size(
+                        QUtil::string_to_ull(attrs["size"].c_str()));
+                }
+            }
+        }
+    } catch (...) {
+        // ignore -- just return 0
+    }
+    free(buf);
+    return result;
+#else
+    return 0;
+#endif
+}
@@ -21,6 +21,8 @@
 #cmakedefine HAVE_LOCALTIME_R 1
 #cmakedefine HAVE_RANDOM 1
 #cmakedefine HAVE_TM_GMTOFF 1
+#cmakedefine HAVE_MALLOC_INFO 1
+#cmakedefine HAVE_OPEN_MEMSTREAM 1
  
 /* printf format for long long */
 #cmakedefine LL_FMT "${LL_FMT}"
@@ -134,3 +134,5 @@ D:20210209191925Z
 2021-02-09T19:19:25Z
 ---- is_long_long
 done
+---- memory usage
+memory usage okay
@@ -703,6 +703,18 @@ is_long_long_test()
     std::cout << "done" << std::endl;
 }
  
+void
+memory_usage_test()
+{
+    auto u1 = QUtil::get_max_memory_usage();
+    if (u1 > 0) {
+        auto x = QUtil::make_shared_array<int>(10 << 20);
+        auto u2 = QUtil::get_max_memory_usage();
+        assert(u2 > u1);
+    }
+    std::cout << "memory usage okay" << std::endl;
+}
+
 int
 main(int argc, char* argv[])
 {
@@ -739,6 +751,8 @@ main(int argc, char* argv[])
         timestamp_test();
         std::cout << "---- is_long_long" << std::endl;
         is_long_long_test();
+        std::cout << "---- memory usage" << std::endl;
+        memory_usage_test();
     } catch (std::exception& e) {
         std::cout << "unexpected exception: " << e.what() << std::endl;
     }