Add basic appearance stream generation

Jay Berkenbilt
1 parent 02281632
Showing 20 changed files with 31076 additions and 394 deletions
ChangeLog
include/qpdf/QPDFAcroFormDocumentHelper.hh
include/qpdf/QPDFFormFieldObjectHelper.hh
libqpdf/QPDFAcroFormDocumentHelper.cc
libqpdf/QPDFFormFieldObjectHelper.cc
qpdf/qpdf.cc
qpdf/qpdf.testcov
qpdf/qtest/qpdf.test
qpdf/qtest/qpdf/appearances-1.pdf
qpdf/qtest/qpdf/appearances-11.pdf
qpdf/qtest/qpdf/appearances-12.pdf
qpdf/qtest/qpdf/appearances-2.pdf
qpdf/qtest/qpdf/appearances-a.pdf
qpdf/qtest/qpdf/appearances-b.pdf
qpdf/qtest/qpdf/appearances-quack.pdf
qpdf/qtest/qpdf/json-need-appearances-acroform.out
qpdf/qtest/qpdf/more-choices.pdf
qpdf/qtest/qpdf/need-appearances-out.pdf
qpdf/qtest/qpdf/need-appearances.pdf
qpdf/test_driver.cc
 2019-01-03  Jay Berkenbilt  <ejb@ql.org>
  
+	* Fix behavior of form field value setting to handle the following
+	cases:
+	  - Strings are always written as UTF-16
+	  - Check boxes and radio buttons are handled properly with
+	    synchronization of values and appearance states
+
+	* Define constants in qpdf/Constants.h for interpretation of
+	annotation and form field flags
+
+	* Add QPDFAnnotationObjectHelper::getFlags
+
+	* Add many new methods to QPDFFormFieldObjectHelper for querying
+	flags and field types
+
+	* Add new methods for appearance stream generation. See comments
+	in QPDFFormFieldObjectHelper.hh for generateAppearance() for a
+	description of limitations.
+	  - QPDFAcroFormDocumentHelper::generateAppearancesIfNeeded
+	  - QPDFFormFieldObjectHelper::generateAppearance
+
+	* Bug fix: when writing form field values, always write string
+	values encoded as UTF-16.
+
 	* Add method QUtil::utf8_to_ascii, which returns an ASCII string
 	for a UTF-8 string, replacing out-of-range characters with a
 	specified substitute.
@@ -157,6 +157,16 @@ class QPDFAcroFormDocumentHelper: public QPDFDocumentHelper
     QPDF_DLL
     void setNeedAppearances(bool);
  
+    // If /NeedAppearances is false, do nothing. Otherwise generate
+    // appearance streams for all widget annotations that need them.
+    // See comments in QPDFFormFieldObjectHelper.hh for
+    // generateAppearance for limitations. For checkbox and radio
+    // button fields, this code ensures that appearance state is
+    // consistent with the field's value and uses any pre-existing
+    // appearance streams.
+    QPDF_DLL
+    void generateAppearancesIfNeeded();
+
   private:
     void analyze();
     void traverseField(QPDFObjectHandle field,
@@ -31,6 +31,8 @@
 #include <qpdf/DLL.h>
 #include <vector>
  
+class QPDFAnnotationObjectHelper;
+
 class QPDFFormFieldObjectHelper: public QPDFObjectHelper
 {
   public:
@@ -179,9 +181,21 @@ class QPDFFormFieldObjectHelper: public QPDFObjectHelper
     QPDF_DLL
     void setV(std::string const& utf8_value, bool need_appearances = true);
  
+    // Update the appearance stream for this field. Note that qpdf's
+    // abilitiy to generate appearance streams is limited. We only
+    // generate appearance streams for streams of type text or choice.
+    // The appearance uses the default parameters provided in the
+    // file, and it only supports ASCII characters. Quadding is
+    // currently ignored. While this functionality is limited, it
+    // should do a decent job on properly constructed PDF files when
+    // field values are restricted to ASCII characters.
+    QPDF_DLL
+    void generateAppearance(QPDFAnnotationObjectHelper&);
+
   private:
     void setRadioButtonValue(QPDFObjectHandle name);
     void setCheckBoxValue(bool value);
+    void generateTextAppearance(QPDFAnnotationObjectHelper&);
  
     class Members
     {
@@ -285,3 +285,48 @@ QPDFAcroFormDocumentHelper::setNeedAppearances(bool val)
         acroform.removeKey("/NeedAppearances");
     }
 }
+
+void
+QPDFAcroFormDocumentHelper::generateAppearancesIfNeeded()
+{
+    if (! getNeedAppearances())
+    {
+        return;
+    }
+
+    QPDFPageDocumentHelper pdh(this->qpdf);
+    std::vector<QPDFPageObjectHelper> pages = pdh.getAllPages();
+    for (std::vector<QPDFPageObjectHelper>::iterator page_iter =
+             pages.begin();
+         page_iter != pages.end(); ++page_iter)
+    {
+        std::vector<QPDFAnnotationObjectHelper> annotations =
+            getWidgetAnnotationsForPage(*page_iter);
+        for (std::vector<QPDFAnnotationObjectHelper>::iterator annot_iter =
+                 annotations.begin();
+             annot_iter != annotations.end(); ++annot_iter)
+        {
+            QPDFAnnotationObjectHelper& aoh = *annot_iter;
+            QPDFFormFieldObjectHelper ffh =
+                getFieldForAnnotation(aoh);
+            if (ffh.getFieldType() == "/Btn")
+            {
+                // Rather than generating appearances for button
+                // fields, rely on what's already there. Just make
+                // sure /AS is consistent with /V, which we can do by
+                // resetting the value of the field back to itself.
+                // This code is referenced in a comment in
+                // QPDFFormFieldObjectHelper::generateAppearance.
+                if (ffh.isRadioButton() || ffh.isCheckbox())
+                {
+                    ffh.setV(ffh.getValue());
+                }
+            }
+            else
+            {
+                ffh.generateAppearance(aoh);
+            }
+        }
+    }
+    setNeedAppearances(false);
+}
 #include <qpdf/QPDFFormFieldObjectHelper.hh>
 #include <qpdf/QTC.hh>
 #include <qpdf/QPDFAcroFormDocumentHelper.hh>
+#include <qpdf/QPDFAnnotationObjectHelper.hh>
+#include <qpdf/QUtil.hh>
+#include <qpdf/Pl_QPDFTokenizer.hh>
+#include <stdlib.h>
  
 QPDFFormFieldObjectHelper::Members::~Members()
 {
@@ -313,7 +317,15 @@ QPDFFormFieldObjectHelper::setV(
         }
         return;
     }
-    setFieldAttribute("/V", value);
+    if (value.isString())
+    {
+        setFieldAttribute(
+            "/V", QPDFObjectHandle::newUnicodeString(value.getUTF8Value()));
+    }
+    else
+    {
+        setFieldAttribute("/V", value);
+    }
     if (need_appearances)
     {
         QPDF* qpdf = this->oh.getOwningQPDF();
@@ -470,3 +482,290 @@ QPDFFormFieldObjectHelper::setCheckBoxValue(bool value)
     QTC::TC("qpdf", "QPDFFormFieldObjectHelper set checkbox AS");
     annot.replaceKey("/AS", name);
 }
+
+void
+QPDFFormFieldObjectHelper::generateAppearance(QPDFAnnotationObjectHelper& aoh)
+{
+    std::string ft = getFieldType();
+    // Ignore field types we don't know how to generate appearances
+    // for. Button fields don't really need them -- see code in
+    // QPDFAcroFormDocumentHelper::generateAppearancesIfNeeded.
+    if ((ft == "/Tx") || (ft == "/Ch"))
+    {
+        generateTextAppearance(aoh);
+    }
+}
+
+class ValueSetter: public QPDFObjectHandle::TokenFilter
+{
+  public:
+    ValueSetter(std::string const& DA, std::string const& V,
+                std::vector<std::string> const& opt, double tf,
+                QPDFObjectHandle::Rectangle const& bbox);
+    virtual ~ValueSetter()
+    {
+    }
+    virtual void handleToken(QPDFTokenizer::Token const&);
+    void writeAppearance();
+
+  private:
+    std::string DA;
+    std::string V;
+    std::vector<std::string> opt;
+    double tf;
+    QPDFObjectHandle::Rectangle bbox;
+    enum { st_top, st_bmc, st_emc, st_end } state;
+};
+
+ValueSetter::ValueSetter(std::string const& DA, std::string const& V,
+                         std::vector<std::string> const& opt, double tf,
+                         QPDFObjectHandle::Rectangle const& bbox) :
+    DA(DA),
+    V(V),
+    opt(opt),
+    tf(tf),
+    bbox(bbox),
+    state(st_top)
+{
+}
+
+void
+ValueSetter::handleToken(QPDFTokenizer::Token const& token)
+{
+    QPDFTokenizer::token_type_e ttype = token.getType();
+    std::string value = token.getValue();
+    bool do_replace = false;
+    switch (state)
+    {
+        case st_top:
+          writeToken(token);
+          if ((ttype == QPDFTokenizer::tt_word) && (value == "BMC"))
+          {
+              state = st_bmc;
+          }
+          break;
+
+      case st_bmc:
+        if ((ttype == QPDFTokenizer::tt_space) ||
+            (ttype == QPDFTokenizer::tt_comment))
+        {
+            writeToken(token);
+        }
+        else
+        {
+            state = st_emc;
+        }
+        // fall through to emc
+
+      case st_emc:
+        if ((ttype == QPDFTokenizer::tt_word) && (value == "EMC"))
+        {
+            do_replace = true;
+            state = st_end;
+        }
+        break;
+
+      case st_end:
+        writeToken(token);
+        break;
+    }
+    if (do_replace)
+    {
+        writeAppearance();
+    }
+}
+
+void ValueSetter::writeAppearance()
+{
+    // This code does not take quadding into consideration because
+    // doing so requires font metric information, which we don't
+    // have in many cases.
+
+    double tfh = 1.2 * tf;
+    int dx = 1;
+
+    // Write one or more lines, centered vertically, possibly with
+    // one row highlighted.
+
+    size_t max_rows = static_cast<size_t>((bbox.ury - bbox.lly) / tfh);
+    bool highlight = false;
+    size_t highlight_idx = 0;
+
+    std::vector<std::string> lines;
+    if (opt.empty() || (max_rows < 2))
+    {
+        lines.push_back(V);
+    }
+    else
+    {
+        // Figure out what rows to write
+        size_t nopt = opt.size();
+        size_t found_idx = 0;
+        bool found = false;
+        for (found_idx = 0; found_idx < nopt; ++found_idx)
+        {
+            if (opt.at(found_idx) == V)
+            {
+                found = true;
+                break;
+            }
+        }
+        if (found)
+        {
+            // Try to make the found item the second one, but
+            // adjust for under/overflow.
+            int wanted_first = found_idx - 1;
+            int wanted_last = found_idx + max_rows - 2;
+            QTC::TC("qpdf", "QPDFFormFieldObjectHelper list found");
+            while (wanted_first < 0)
+            {
+                QTC::TC("qpdf", "QPDFFormFieldObjectHelper list first too low");
+                ++wanted_first;
+                ++wanted_last;
+            }
+            while (wanted_last >= static_cast<int>(nopt))
+            {
+                QTC::TC("qpdf", "QPDFFormFieldObjectHelper list last too high");
+                if (wanted_first > 0)
+                {
+                    --wanted_first;
+                }
+                --wanted_last;
+            }
+            highlight = true;
+            highlight_idx = found_idx - wanted_first;
+            for (int i = wanted_first; i <= wanted_last; ++i)
+            {
+                lines.push_back(opt.at(i));
+            }
+        }
+        else
+        {
+            QTC::TC("qpdf", "QPDFFormFieldObjectHelper list not found");
+            // include our value and the first n-1 rows
+            highlight_idx = 0;
+            highlight = true;
+            lines.push_back(V);
+            for (size_t i = 0; ((i < nopt) && (i < (max_rows - 1))); ++i)
+            {
+                lines.push_back(opt.at(i));
+            }
+        }
+    }
+
+    // Write the lines centered vertically, highlighting if needed
+    size_t nlines = lines.size();
+    double dy = bbox.ury - ((bbox.ury - bbox.lly - (nlines * tfh)) / 2.0);
+    write(DA + "\nq\n");
+    if (highlight)
+    {
+        write("q\n0.85 0.85 0.85 rg\n" +
+              QUtil::int_to_string(bbox.llx) + " " +
+              QUtil::double_to_string(bbox.lly + dy -
+                                      (tfh * (highlight_idx + 1))) + " " +
+              QUtil::int_to_string(bbox.urx - bbox.llx) + " " +
+              QUtil::double_to_string(tfh) +
+              " re f\nQ\n");
+    }
+    dy += 0.2 * tf;
+    for (size_t i = 0; i < nlines; ++i)
+    {
+        dy -= tfh;
+        write("BT\n" +
+              QUtil::int_to_string(bbox.llx + dx) + " " +
+              QUtil::double_to_string(bbox.lly + dy) + " Td\n" +
+              QPDFObjectHandle::newString(lines.at(i)).unparse() +
+              " Tj\nET\n");
+    }
+    write("Q\nEMC");
+}
+
+class TfFinder: public QPDFObjectHandle::TokenFilter
+{
+  public:
+    TfFinder();
+    virtual ~TfFinder()
+    {
+    }
+    virtual void handleToken(QPDFTokenizer::Token const&);
+    double getTf();
+
+  private:
+    double tf;
+    double last_num;
+};
+
+TfFinder::TfFinder() :
+    tf(11.0),
+    last_num(0.0)
+{
+}
+
+void
+TfFinder::handleToken(QPDFTokenizer::Token const& token)
+{
+    QPDFTokenizer::token_type_e ttype = token.getType();
+    std::string value = token.getValue();
+    switch (ttype)
+    {
+      case QPDFTokenizer::tt_integer:
+      case QPDFTokenizer::tt_real:
+        last_num = strtod(value.c_str(), 0);
+        break;
+
+      case QPDFTokenizer::tt_word:
+        if ((value == "Tf") &&
+            (last_num > 1.0) &&
+            (last_num < 1000.0))
+        {
+            // These ranges are arbitrary but keep us from doing
+            // insane things or suffering from over/underflow
+            tf = last_num;
+        }
+        break;
+
+      default:
+        break;
+    }
+}
+
+double
+TfFinder::getTf()
+{
+    return this->tf;
+}
+
+void
+QPDFFormFieldObjectHelper::generateTextAppearance(
+    QPDFAnnotationObjectHelper& aoh)
+{
+    QPDFObjectHandle AS = aoh.getAppearanceStream("/N");
+    if (! AS.isStream())
+    {
+        aoh.getObjectHandle().warnIfPossible(
+            "unable to get normal appearance stream for update");
+        return;
+    }
+    QPDFObjectHandle bbox_obj = AS.getDict().getKey("/BBox");
+    if (! bbox_obj.isRectangle())
+    {
+        aoh.getObjectHandle().warnIfPossible(
+            "unable to get appearance stream bounding box");
+        return;
+    }
+    QPDFObjectHandle::Rectangle bbox = bbox_obj.getArrayAsRectangle();
+    std::string DA = getDefaultAppearance();
+    std::string V = QUtil::utf8_to_ascii(getValueAsString());
+
+    TfFinder tff;
+    Pl_QPDFTokenizer tok("tf", &tff);
+    tok.write(QUtil::unsigned_char_pointer(DA.c_str()), DA.length());
+    tok.finish();
+    double tf = tff.getTf();
+    std::vector<std::string> opt;
+    if (isChoice() && ((getFlags() & ff_ch_combo) == 0))
+    {
+        opt = getChoices();
+    }
+    AS.addTokenFilter(new ValueSetter(DA, V, opt, tf, bbox));
+}
@@ -106,6 +106,7 @@ struct Options
         flatten_annotations(false),
         flatten_annotations_required(0),
         flatten_annotations_forbidden(an_invisible | an_hidden),
+        generate_appearances(false),
         show_npages(false),
         deterministic_id(false),
         static_id(false),
@@ -181,6 +182,7 @@ struct Options
     bool flatten_annotations;
     int flatten_annotations_required;
     int flatten_annotations_forbidden;
+    bool generate_appearances;
     std::string min_version;
     std::string force_version;
     bool show_npages;
@@ -570,6 +572,7 @@ class ArgParser
     void argLinearizePass1(char* parameter);
     void argCoalesceContents();
     void argFlattenAnnotations(char* parameter);
+    void argGenerateAppearances();
     void argMinVersion(char* parameter);
     void argForceVersion(char* parameter);
     void argSplitPages(char* parameter);
@@ -771,6 +774,8 @@ ArgParser::initOptionTable()
     char const* flatten_choices[] = {"all", "print", "screen", 0};
     (*t)["flatten-annotations"] = oe_requiredChoices(
         &ArgParser::argFlattenAnnotations, flatten_choices);
+    (*t)["generate-appearances"] =
+        oe_bare(&ArgParser::argGenerateAppearances);
     (*t)["min-version"] = oe_requiredParameter(
         &ArgParser::argMinVersion, "version");
     (*t)["force-version"] = oe_requiredParameter(
@@ -1234,6 +1239,12 @@ ArgParser::argFlattenAnnotations(char* parameter)
 }
  
 void
+ArgParser::argGenerateAppearances()
+{
+    o.generate_appearances = true;
+}
+
+void
 ArgParser::argMinVersion(char* parameter)
 {
     o.min_version = parameter;
@@ -1877,18 +1888,28 @@ familiar with the PDF file format or who are PDF developers.\n\
 --flatten-annotations=option\n\
                           incorporate rendering of annotations into page\n\
                           contents including those for interactive form\n\
-                          fields\n\
+                          fields; may also want --generate-appearances\n\
+--generate-appearances    generate appearance streams for form fields\n\
 --qdf                     turns on \"QDF mode\" (below)\n\
 --linearize-pass1=file    write intermediate pass of linearized file\n\
                           for debugging\n\
 --min-version=version     sets the minimum PDF version of the output file\n\
 --force-version=version   forces this to be the PDF version of the output file\n\
 \n\
-Options for --flatten-annotations are all, print, or screen. If the\n\
-option is print, only annotations marked as print are included. If the\n\
-option is screen, options marked as \"no view\" are excluded.\n\
-Otherwise, annotations are flattened regardless of the presence of\n\
-print or NoView flags.\n\
+Options for --flatten-annotations are all, print, or screen. If the option\n\
+is print, only annotations marked as print are included. If the option is\n\
+screen, options marked as \"no view\" are excluded. Otherwise, annotations\n\
+are flattened regardless of the presence of print or NoView flags. It is\n\
+common for PDF files to have a flag set that appearance streams need to be\n\
+regenerated. This happens when someone changes a form value with software\n\
+that does not know how to render the new value. qpdf will not flatten form\n\
+fields in files like this. If you get this warning, you have two choices:\n\
+either use qpdf's --generate-appearances flag to tell qpdf to go ahead and\n\
+regenerate appearances, or use some other tool to generate the appearances.\n\
+qpdf does a pretty good job with most forms when only ASCII characters are\n\
+used in form field values, but if your form fields contain other\n\
+characters, rich text, or are other than left justified, you will get\n\
+better results first saving with other software.\n\
 \n\
 Version numbers may be expressed as major.minor.extension-level, so 1.7.3\n\
 means PDF version 1.7 at extension level 3.\n\
@@ -3356,6 +3377,11 @@ static void do_inspection(QPDF&amp; pdf, Options&amp; o)
 static void handle_transformations(QPDF& pdf, Options& o)
 {
     QPDFPageDocumentHelper dh(pdf);
+    if (o.generate_appearances)
+    {
+        QPDFAcroFormDocumentHelper afdh(pdf);
+        afdh.generateAppearancesIfNeeded();
+    }
     if (o.flatten_annotations)
     {
         dh.flattenAnnotations(o.flatten_annotations_required,
@@ -398,3 +398,7 @@ QPDFFormFieldObjectHelper checkbox kid widget 0
 QPDFObjectHandle broken radio button 0
 QPDFFormFieldObjectHelper set checkbox AS 0
 QPDFObjectHandle broken checkbox 0
+QPDFFormFieldObjectHelper list not found 0
+QPDFFormFieldObjectHelper list found 0
+QPDFFormFieldObjectHelper list first too low 0
+QPDFFormFieldObjectHelper list last too high 0
@@ -234,6 +234,57 @@ $td-&gt;runtest(&quot;compare files&quot;,
  
 show_ntests();
 # ----------
+$td->notify("--- Appearance Streams ---");
+$n_tests += 4;
+
+$td->runtest("generate appearances and flatten",
+             {$td->COMMAND =>
+                  "qpdf --qdf --no-original-object-ids --static-id" .
+                  " --generate-appearances --flatten-annotations=all" .
+                  " need-appearances.pdf a.pdf"},
+             {$td->STRING => "", $td->EXIT_STATUS => 0},
+             $td->NORMALIZE_NEWLINES);
+$td->runtest("compare files",
+             {$td->FILE => "a.pdf"},
+             {$td->FILE => "appearances-a.pdf"});
+
+$td->runtest("more choices",
+             {$td->COMMAND =>
+                  "qpdf --qdf --no-original-object-ids --static-id" .
+                  " --generate-appearances" .
+                  " more-choices.pdf b.pdf"},
+             {$td->STRING => "", $td->EXIT_STATUS => 0},
+             $td->NORMALIZE_NEWLINES);
+# b.pdf still has forms
+$td->runtest("compare files",
+             {$td->FILE => "b.pdf"},
+             {$td->FILE => "appearances-b.pdf"});
+
+my @choice_values = qw(1 2 11 12 quack);
+$n_tests += 3 * scalar(@choice_values);
+foreach my $i (@choice_values)
+{
+    # b.pdf was generated by qpdf and needs appearances
+    # test_driver 52 writes a.pdf
+    $td->runtest("set value to $i",
+                 {$td->COMMAND => "test_driver 52 b.pdf $i"},
+                 {$td->STRING => "setting list1 value\ntest 52 done\n",
+                      $td->EXIT_STATUS => 0},
+                 $td->NORMALIZE_NEWLINES);
+    $td->runtest("regenerate appearances",
+                 {$td->COMMAND =>
+                      "qpdf --qdf --no-original-object-ids --static-id" .
+                      " --generate-appearances" .
+                      " a.pdf b.pdf"},
+                 {$td->STRING => "", $td->EXIT_STATUS => 0},
+                 $td->NORMALIZE_NEWLINES);
+    $td->runtest("compare files",
+                 {$td->FILE => "b.pdf"},
+                 {$td->FILE => "appearances-$i.pdf"});
+}
+
+show_ntests();
+# ----------
 $td->notify("--- Stream Replacement Tests ---");
 $n_tests += 8;
  
@@ -321,7 +321,7 @@
         "parent": null,
         "partialname": "list1",
         "quadding": 0,
-        "value": "five"
+        "value": "six"
       },
       {
         "alternativename": "drop1",
@@ -1820,6 +1820,32 @@ void runtest(int n, char const* filename1, char const* arg2)
         w.setStaticID(true);
         w.write();
     }
+    else if (n == 52)
+    {
+        // This test just sets a field value for appearance stream
+        // generating testing.
+        QPDFObjectHandle acroform = pdf.getRoot().getKey("/AcroForm");
+        QPDFObjectHandle fields = acroform.getKey("/Fields");
+        int n = fields.getArrayNItems();
+        for (int i = 0; i < n; ++i)
+        {
+            QPDFObjectHandle field = fields.getArrayItem(i);
+            QPDFObjectHandle T = field.getKey("/T");
+            if (! T.isString())
+            {
+                continue;
+            }
+            std::string Tval = T.getUTF8Value();
+            if (Tval == "list1")
+            {
+                std::cout << "setting list1 value\n";
+                QPDFFormFieldObjectHelper foh(field);
+                foh.setV(QPDFObjectHandle::newString(arg2));
+            }
+        }
+        QPDFWriter w(pdf, "a.pdf");
+        w.write();
+    }
     else
     {
 	throw std::runtime_error(std::string("invalid test ") +