Add --keep-files-open-threshold (fixes #288)

Jay Berkenbilt
1 parent 011695df
Showing 6 changed files with 57 additions and 327 deletions
ChangeLog
manual/qpdf-manual.xml
qpdf/qpdf.cc
qpdf/qtest/qpdf.test
qpdf/qtest/qpdf/disable-kfo.out
qpdf/qtest/qpdf/enable-kfo.out
 2019-04-20  Jay Berkenbilt  <ejb@ql.org>
  
+	* Add parameter --keep-files-open-threshold to override the
+	maximum number of files that qpdf will allow to be kept open at
+	once. Fixes #288.
+
 	* Handle Unicode characters in filenames. The changes to support
 	Unicode on the CLI in Windows broke Unicode filenames. Fixes #298.
  
@@ -652,6 +652,19 @@ make
         more than the OS limit's number of files from a script and are
         not worried about a few seconds additional processing time,
         you may want to specify <option>--keep-files-open=n</option>.
+        The threshold for switching may be changed from the default
+        200 with the <option>--keep-files-open-threshold</option>
+        option.
+       </para>
+      </listitem>
+     </varlistentry>
+     <varlistentry>
+      <term><option>--keep-files-open-threshold=<replaceable>count</replaceable></option></term>
+      <listitem>
+       <para>
+        If specified, overrides the default value of 200 used as the
+        threshold for qpdf deciding whether or not to keep files open.
+        See <option>--keep-files-open</option> for details.
        </para>
       </listitem>
      </varlistentry>
@@ -134,6 +134,7 @@ struct Options
         preserve_unreferenced_page_resources(false),
         keep_files_open(true),
         keep_files_open_set(false),
+        keep_files_open_threshold(200), // default known in help and docs
         newline_before_endstream(false),
         coalesce_contents(false),
         flatten_annotations(false),
@@ -225,6 +226,7 @@ struct Options
     bool preserve_unreferenced_page_resources;
     bool keep_files_open;
     bool keep_files_open_set;
+    size_t keep_files_open_threshold;
     bool newline_before_endstream;
     std::string linearize_pass1;
     bool coalesce_contents;
@@ -634,6 +636,7 @@ class ArgParser
     void argPreserveUnreferenced();
     void argPreserveUnreferencedResources();
     void argKeepFilesOpen(char* parameter);
+    void argKeepFilesOpenThreshold(char* parameter);
     void argNewlineBeforeEndstream();
     void argLinearizePass1(char* parameter);
     void argCoalesceContents();
@@ -857,6 +860,8 @@ ArgParser::initOptionTable()
         &ArgParser::argPreserveUnreferencedResources);
     (*t)["keep-files-open"] = oe_requiredChoices(
         &ArgParser::argKeepFilesOpen, yn);
+    (*t)["keep-files-open-threshold"] = oe_requiredParameter(
+        &ArgParser::argKeepFilesOpenThreshold, "count");
     (*t)["newline-before-endstream"] = oe_bare(
         &ArgParser::argNewlineBeforeEndstream);
     (*t)["linearize-pass1"] = oe_requiredParameter(
@@ -1209,6 +1214,7 @@ ArgParser::argHelp()
         << "starting point, but its pages are replaced with pages as specified.\n"
         << "\n"
         << "--keep-files-open=[yn]\n"
+        << "--keep-files-threshold=count\n"
         << "--pages file [ --password=password ] [ page-range ] ... --\n"
         << "\n"
         << "For each file that pages should be taken from, specify the file, a\n"
@@ -1227,7 +1233,8 @@ ArgParser::argHelp()
         << "files will be kept open at the same time. This behavior can be overridden\n"
         << "by specifying --keep-files-open=[yn]. Closing and opening files can have\n"
         << "very high overhead on certain file systems, especially networked file\n"
-        << "systems.\n"
+        << "systems. The threshold of 200 can be modified with\n"
+        << "--keep-files-open-threshold\n"
         << "\n"
         << "The page range is a set of numbers separated by commas, ranges of\n"
         << "numbers separated dashes, or combinations of those.  The character\n"
@@ -1782,6 +1789,13 @@ ArgParser::argKeepFilesOpen(char* parameter)
 }
  
 void
+ArgParser::argKeepFilesOpenThreshold(char* parameter)
+{
+    o.keep_files_open_threshold =
+        static_cast<size_t>(QUtil::string_to_int(parameter));
+}
+
+void
 ArgParser::argNewlineBeforeEndstream()
 {
     o.newline_before_endstream = true;
@@ -4352,9 +4366,7 @@ static void handle_page_specs(QPDF&amp; pdf, Options&amp; o)
             PageSpec& page_spec = *iter;
             filenames.insert(page_spec.filename);
         }
-        // NOTE: The number 200 for this threshold is in the help
-        // message and manual and is baked into the test suite.
-        if (filenames.size() > 200)
+        if (filenames.size() > o.keep_files_open_threshold)
         {
             QTC::TC("qpdf", "qpdf disable keep files open");
             if (o.verbose)
@@ -1685,7 +1685,7 @@ $n_tests += 4;
     binmode F;
     my $content = <F>;
     close(F);
-    for (my $i = 1; $i <= 201; ++$i)
+    for (my $i = 1; $i <= 51; ++$i)
     {
         open(F, sprintf(">%03d-kfo.pdf", $i)) or die;
         binmode F;
@@ -1693,16 +1693,17 @@ $n_tests += 4;
         close(F);
     }
 }
-$td->runtest("disable keep files open",
+$td->runtest("automatic disable keep files open",
              {$td->COMMAND =>
                   "qpdf --verbose --static-id --empty" .
+                  " --keep-files-open-threshold=50" .
                   " --pages *kfo.pdf -- a.pdf"},
              {$td->FILE => "disable-kfo.out", $td->EXIT_STATUS => 0},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("don't disable keep files open",
              {$td->COMMAND =>
                   "qpdf --verbose --static-id --empty" .
-                  " --pages 11*kfo.pdf -- a.pdf"},
+                  " --pages 01*kfo.pdf -- a.pdf"},
              {$td->FILE => "enable-kfo.out", $td->EXIT_STATUS => 0},
              $td->NORMALIZE_NEWLINES);
 $td->runtest("explict keep files open",
@@ -50,156 +50,6 @@ qpdf: processing 048-kfo.pdf
 qpdf: processing 049-kfo.pdf
 qpdf: processing 050-kfo.pdf
 qpdf: processing 051-kfo.pdf
-qpdf: processing 052-kfo.pdf
-qpdf: processing 053-kfo.pdf
-qpdf: processing 054-kfo.pdf
-qpdf: processing 055-kfo.pdf
-qpdf: processing 056-kfo.pdf
-qpdf: processing 057-kfo.pdf
-qpdf: processing 058-kfo.pdf
-qpdf: processing 059-kfo.pdf
-qpdf: processing 060-kfo.pdf
-qpdf: processing 061-kfo.pdf
-qpdf: processing 062-kfo.pdf
-qpdf: processing 063-kfo.pdf
-qpdf: processing 064-kfo.pdf
-qpdf: processing 065-kfo.pdf
-qpdf: processing 066-kfo.pdf
-qpdf: processing 067-kfo.pdf
-qpdf: processing 068-kfo.pdf
-qpdf: processing 069-kfo.pdf
-qpdf: processing 070-kfo.pdf
-qpdf: processing 071-kfo.pdf
-qpdf: processing 072-kfo.pdf
-qpdf: processing 073-kfo.pdf
-qpdf: processing 074-kfo.pdf
-qpdf: processing 075-kfo.pdf
-qpdf: processing 076-kfo.pdf
-qpdf: processing 077-kfo.pdf
-qpdf: processing 078-kfo.pdf
-qpdf: processing 079-kfo.pdf
-qpdf: processing 080-kfo.pdf
-qpdf: processing 081-kfo.pdf
-qpdf: processing 082-kfo.pdf
-qpdf: processing 083-kfo.pdf
-qpdf: processing 084-kfo.pdf
-qpdf: processing 085-kfo.pdf
-qpdf: processing 086-kfo.pdf
-qpdf: processing 087-kfo.pdf
-qpdf: processing 088-kfo.pdf
-qpdf: processing 089-kfo.pdf
-qpdf: processing 090-kfo.pdf
-qpdf: processing 091-kfo.pdf
-qpdf: processing 092-kfo.pdf
-qpdf: processing 093-kfo.pdf
-qpdf: processing 094-kfo.pdf
-qpdf: processing 095-kfo.pdf
-qpdf: processing 096-kfo.pdf
-qpdf: processing 097-kfo.pdf
-qpdf: processing 098-kfo.pdf
-qpdf: processing 099-kfo.pdf
-qpdf: processing 100-kfo.pdf
-qpdf: processing 101-kfo.pdf
-qpdf: processing 102-kfo.pdf
-qpdf: processing 103-kfo.pdf
-qpdf: processing 104-kfo.pdf
-qpdf: processing 105-kfo.pdf
-qpdf: processing 106-kfo.pdf
-qpdf: processing 107-kfo.pdf
-qpdf: processing 108-kfo.pdf
-qpdf: processing 109-kfo.pdf
-qpdf: processing 110-kfo.pdf
-qpdf: processing 111-kfo.pdf
-qpdf: processing 112-kfo.pdf
-qpdf: processing 113-kfo.pdf
-qpdf: processing 114-kfo.pdf
-qpdf: processing 115-kfo.pdf
-qpdf: processing 116-kfo.pdf
-qpdf: processing 117-kfo.pdf
-qpdf: processing 118-kfo.pdf
-qpdf: processing 119-kfo.pdf
-qpdf: processing 120-kfo.pdf
-qpdf: processing 121-kfo.pdf
-qpdf: processing 122-kfo.pdf
-qpdf: processing 123-kfo.pdf
-qpdf: processing 124-kfo.pdf
-qpdf: processing 125-kfo.pdf
-qpdf: processing 126-kfo.pdf
-qpdf: processing 127-kfo.pdf
-qpdf: processing 128-kfo.pdf
-qpdf: processing 129-kfo.pdf
-qpdf: processing 130-kfo.pdf
-qpdf: processing 131-kfo.pdf
-qpdf: processing 132-kfo.pdf
-qpdf: processing 133-kfo.pdf
-qpdf: processing 134-kfo.pdf
-qpdf: processing 135-kfo.pdf
-qpdf: processing 136-kfo.pdf
-qpdf: processing 137-kfo.pdf
-qpdf: processing 138-kfo.pdf
-qpdf: processing 139-kfo.pdf
-qpdf: processing 140-kfo.pdf
-qpdf: processing 141-kfo.pdf
-qpdf: processing 142-kfo.pdf
-qpdf: processing 143-kfo.pdf
-qpdf: processing 144-kfo.pdf
-qpdf: processing 145-kfo.pdf
-qpdf: processing 146-kfo.pdf
-qpdf: processing 147-kfo.pdf
-qpdf: processing 148-kfo.pdf
-qpdf: processing 149-kfo.pdf
-qpdf: processing 150-kfo.pdf
-qpdf: processing 151-kfo.pdf
-qpdf: processing 152-kfo.pdf
-qpdf: processing 153-kfo.pdf
-qpdf: processing 154-kfo.pdf
-qpdf: processing 155-kfo.pdf
-qpdf: processing 156-kfo.pdf
-qpdf: processing 157-kfo.pdf
-qpdf: processing 158-kfo.pdf
-qpdf: processing 159-kfo.pdf
-qpdf: processing 160-kfo.pdf
-qpdf: processing 161-kfo.pdf
-qpdf: processing 162-kfo.pdf
-qpdf: processing 163-kfo.pdf
-qpdf: processing 164-kfo.pdf
-qpdf: processing 165-kfo.pdf
-qpdf: processing 166-kfo.pdf
-qpdf: processing 167-kfo.pdf
-qpdf: processing 168-kfo.pdf
-qpdf: processing 169-kfo.pdf
-qpdf: processing 170-kfo.pdf
-qpdf: processing 171-kfo.pdf
-qpdf: processing 172-kfo.pdf
-qpdf: processing 173-kfo.pdf
-qpdf: processing 174-kfo.pdf
-qpdf: processing 175-kfo.pdf
-qpdf: processing 176-kfo.pdf
-qpdf: processing 177-kfo.pdf
-qpdf: processing 178-kfo.pdf
-qpdf: processing 179-kfo.pdf
-qpdf: processing 180-kfo.pdf
-qpdf: processing 181-kfo.pdf
-qpdf: processing 182-kfo.pdf
-qpdf: processing 183-kfo.pdf
-qpdf: processing 184-kfo.pdf
-qpdf: processing 185-kfo.pdf
-qpdf: processing 186-kfo.pdf
-qpdf: processing 187-kfo.pdf
-qpdf: processing 188-kfo.pdf
-qpdf: processing 189-kfo.pdf
-qpdf: processing 190-kfo.pdf
-qpdf: processing 191-kfo.pdf
-qpdf: processing 192-kfo.pdf
-qpdf: processing 193-kfo.pdf
-qpdf: processing 194-kfo.pdf
-qpdf: processing 195-kfo.pdf
-qpdf: processing 196-kfo.pdf
-qpdf: processing 197-kfo.pdf
-qpdf: processing 198-kfo.pdf
-qpdf: processing 199-kfo.pdf
-qpdf: processing 200-kfo.pdf
-qpdf: processing 201-kfo.pdf
 qpdf: removing unreferenced pages from primary input
 qpdf: adding pages from 001-kfo.pdf
 qpdf: adding pages from 002-kfo.pdf
@@ -252,154 +102,4 @@ qpdf: adding pages from 048-kfo.pdf
 qpdf: adding pages from 049-kfo.pdf
 qpdf: adding pages from 050-kfo.pdf
 qpdf: adding pages from 051-kfo.pdf
-qpdf: adding pages from 052-kfo.pdf
-qpdf: adding pages from 053-kfo.pdf
-qpdf: adding pages from 054-kfo.pdf
-qpdf: adding pages from 055-kfo.pdf
-qpdf: adding pages from 056-kfo.pdf
-qpdf: adding pages from 057-kfo.pdf
-qpdf: adding pages from 058-kfo.pdf
-qpdf: adding pages from 059-kfo.pdf
-qpdf: adding pages from 060-kfo.pdf
-qpdf: adding pages from 061-kfo.pdf
-qpdf: adding pages from 062-kfo.pdf
-qpdf: adding pages from 063-kfo.pdf
-qpdf: adding pages from 064-kfo.pdf
-qpdf: adding pages from 065-kfo.pdf
-qpdf: adding pages from 066-kfo.pdf
-qpdf: adding pages from 067-kfo.pdf
-qpdf: adding pages from 068-kfo.pdf
-qpdf: adding pages from 069-kfo.pdf
-qpdf: adding pages from 070-kfo.pdf
-qpdf: adding pages from 071-kfo.pdf
-qpdf: adding pages from 072-kfo.pdf
-qpdf: adding pages from 073-kfo.pdf
-qpdf: adding pages from 074-kfo.pdf
-qpdf: adding pages from 075-kfo.pdf
-qpdf: adding pages from 076-kfo.pdf
-qpdf: adding pages from 077-kfo.pdf
-qpdf: adding pages from 078-kfo.pdf
-qpdf: adding pages from 079-kfo.pdf
-qpdf: adding pages from 080-kfo.pdf
-qpdf: adding pages from 081-kfo.pdf
-qpdf: adding pages from 082-kfo.pdf
-qpdf: adding pages from 083-kfo.pdf
-qpdf: adding pages from 084-kfo.pdf
-qpdf: adding pages from 085-kfo.pdf
-qpdf: adding pages from 086-kfo.pdf
-qpdf: adding pages from 087-kfo.pdf
-qpdf: adding pages from 088-kfo.pdf
-qpdf: adding pages from 089-kfo.pdf
-qpdf: adding pages from 090-kfo.pdf
-qpdf: adding pages from 091-kfo.pdf
-qpdf: adding pages from 092-kfo.pdf
-qpdf: adding pages from 093-kfo.pdf
-qpdf: adding pages from 094-kfo.pdf
-qpdf: adding pages from 095-kfo.pdf
-qpdf: adding pages from 096-kfo.pdf
-qpdf: adding pages from 097-kfo.pdf
-qpdf: adding pages from 098-kfo.pdf
-qpdf: adding pages from 099-kfo.pdf
-qpdf: adding pages from 100-kfo.pdf
-qpdf: adding pages from 101-kfo.pdf
-qpdf: adding pages from 102-kfo.pdf
-qpdf: adding pages from 103-kfo.pdf
-qpdf: adding pages from 104-kfo.pdf
-qpdf: adding pages from 105-kfo.pdf
-qpdf: adding pages from 106-kfo.pdf
-qpdf: adding pages from 107-kfo.pdf
-qpdf: adding pages from 108-kfo.pdf
-qpdf: adding pages from 109-kfo.pdf
-qpdf: adding pages from 110-kfo.pdf
-qpdf: adding pages from 111-kfo.pdf
-qpdf: adding pages from 112-kfo.pdf
-qpdf: adding pages from 113-kfo.pdf
-qpdf: adding pages from 114-kfo.pdf
-qpdf: adding pages from 115-kfo.pdf
-qpdf: adding pages from 116-kfo.pdf
-qpdf: adding pages from 117-kfo.pdf
-qpdf: adding pages from 118-kfo.pdf
-qpdf: adding pages from 119-kfo.pdf
-qpdf: adding pages from 120-kfo.pdf
-qpdf: adding pages from 121-kfo.pdf
-qpdf: adding pages from 122-kfo.pdf
-qpdf: adding pages from 123-kfo.pdf
-qpdf: adding pages from 124-kfo.pdf
-qpdf: adding pages from 125-kfo.pdf
-qpdf: adding pages from 126-kfo.pdf
-qpdf: adding pages from 127-kfo.pdf
-qpdf: adding pages from 128-kfo.pdf
-qpdf: adding pages from 129-kfo.pdf
-qpdf: adding pages from 130-kfo.pdf
-qpdf: adding pages from 131-kfo.pdf
-qpdf: adding pages from 132-kfo.pdf
-qpdf: adding pages from 133-kfo.pdf
-qpdf: adding pages from 134-kfo.pdf
-qpdf: adding pages from 135-kfo.pdf
-qpdf: adding pages from 136-kfo.pdf
-qpdf: adding pages from 137-kfo.pdf
-qpdf: adding pages from 138-kfo.pdf
-qpdf: adding pages from 139-kfo.pdf
-qpdf: adding pages from 140-kfo.pdf
-qpdf: adding pages from 141-kfo.pdf
-qpdf: adding pages from 142-kfo.pdf
-qpdf: adding pages from 143-kfo.pdf
-qpdf: adding pages from 144-kfo.pdf
-qpdf: adding pages from 145-kfo.pdf
-qpdf: adding pages from 146-kfo.pdf
-qpdf: adding pages from 147-kfo.pdf
-qpdf: adding pages from 148-kfo.pdf
-qpdf: adding pages from 149-kfo.pdf
-qpdf: adding pages from 150-kfo.pdf
-qpdf: adding pages from 151-kfo.pdf
-qpdf: adding pages from 152-kfo.pdf
-qpdf: adding pages from 153-kfo.pdf
-qpdf: adding pages from 154-kfo.pdf
-qpdf: adding pages from 155-kfo.pdf
-qpdf: adding pages from 156-kfo.pdf
-qpdf: adding pages from 157-kfo.pdf
-qpdf: adding pages from 158-kfo.pdf
-qpdf: adding pages from 159-kfo.pdf
-qpdf: adding pages from 160-kfo.pdf
-qpdf: adding pages from 161-kfo.pdf
-qpdf: adding pages from 162-kfo.pdf
-qpdf: adding pages from 163-kfo.pdf
-qpdf: adding pages from 164-kfo.pdf
-qpdf: adding pages from 165-kfo.pdf
-qpdf: adding pages from 166-kfo.pdf
-qpdf: adding pages from 167-kfo.pdf
-qpdf: adding pages from 168-kfo.pdf
-qpdf: adding pages from 169-kfo.pdf
-qpdf: adding pages from 170-kfo.pdf
-qpdf: adding pages from 171-kfo.pdf
-qpdf: adding pages from 172-kfo.pdf
-qpdf: adding pages from 173-kfo.pdf
-qpdf: adding pages from 174-kfo.pdf
-qpdf: adding pages from 175-kfo.pdf
-qpdf: adding pages from 176-kfo.pdf
-qpdf: adding pages from 177-kfo.pdf
-qpdf: adding pages from 178-kfo.pdf
-qpdf: adding pages from 179-kfo.pdf
-qpdf: adding pages from 180-kfo.pdf
-qpdf: adding pages from 181-kfo.pdf
-qpdf: adding pages from 182-kfo.pdf
-qpdf: adding pages from 183-kfo.pdf
-qpdf: adding pages from 184-kfo.pdf
-qpdf: adding pages from 185-kfo.pdf
-qpdf: adding pages from 186-kfo.pdf
-qpdf: adding pages from 187-kfo.pdf
-qpdf: adding pages from 188-kfo.pdf
-qpdf: adding pages from 189-kfo.pdf
-qpdf: adding pages from 190-kfo.pdf
-qpdf: adding pages from 191-kfo.pdf
-qpdf: adding pages from 192-kfo.pdf
-qpdf: adding pages from 193-kfo.pdf
-qpdf: adding pages from 194-kfo.pdf
-qpdf: adding pages from 195-kfo.pdf
-qpdf: adding pages from 196-kfo.pdf
-qpdf: adding pages from 197-kfo.pdf
-qpdf: adding pages from 198-kfo.pdf
-qpdf: adding pages from 199-kfo.pdf
-qpdf: adding pages from 200-kfo.pdf
-qpdf: adding pages from 201-kfo.pdf
 qpdf: wrote file a.pdf
 qpdf: selecting --keep-open-files=y
-qpdf: processing 110-kfo.pdf
-qpdf: processing 111-kfo.pdf
-qpdf: processing 112-kfo.pdf
-qpdf: processing 113-kfo.pdf
-qpdf: processing 114-kfo.pdf
-qpdf: processing 115-kfo.pdf
-qpdf: processing 116-kfo.pdf
-qpdf: processing 117-kfo.pdf
-qpdf: processing 118-kfo.pdf
-qpdf: processing 119-kfo.pdf
+qpdf: processing 010-kfo.pdf
+qpdf: processing 011-kfo.pdf
+qpdf: processing 012-kfo.pdf
+qpdf: processing 013-kfo.pdf
+qpdf: processing 014-kfo.pdf
+qpdf: processing 015-kfo.pdf
+qpdf: processing 016-kfo.pdf
+qpdf: processing 017-kfo.pdf
+qpdf: processing 018-kfo.pdf
+qpdf: processing 019-kfo.pdf
 qpdf: removing unreferenced pages from primary input
-qpdf: adding pages from 110-kfo.pdf
-qpdf: adding pages from 111-kfo.pdf
-qpdf: adding pages from 112-kfo.pdf
-qpdf: adding pages from 113-kfo.pdf
-qpdf: adding pages from 114-kfo.pdf
-qpdf: adding pages from 115-kfo.pdf
-qpdf: adding pages from 116-kfo.pdf
-qpdf: adding pages from 117-kfo.pdf
-qpdf: adding pages from 118-kfo.pdf
-qpdf: adding pages from 119-kfo.pdf
+qpdf: adding pages from 010-kfo.pdf
+qpdf: adding pages from 011-kfo.pdf
+qpdf: adding pages from 012-kfo.pdf
+qpdf: adding pages from 013-kfo.pdf
+qpdf: adding pages from 014-kfo.pdf
+qpdf: adding pages from 015-kfo.pdf
+qpdf: adding pages from 016-kfo.pdf
+qpdf: adding pages from 017-kfo.pdf
+qpdf: adding pages from 018-kfo.pdf
+qpdf: adding pages from 019-kfo.pdf
 qpdf: wrote file a.pdf