Honor other base encodings when generating appearances

Jay Berkenbilt
1 parent b341d742
Showing 17 changed files with 1994 additions and 1921 deletions
ChangeLog
include/qpdf/QPDFFormFieldObjectHelper.hh
libqpdf/QPDFFormFieldObjectHelper.cc
manual/qpdf-manual.xml
qpdf/qpdf.cc
qpdf/qpdf.testcov
qpdf/qtest/qpdf/appearances-1.pdf
qpdf/qtest/qpdf/appearances-11.pdf
qpdf/qtest/qpdf/appearances-12.pdf
qpdf/qtest/qpdf/appearances-2.pdf
qpdf/qtest/qpdf/appearances-a.pdf
qpdf/qtest/qpdf/appearances-b.pdf
qpdf/qtest/qpdf/appearances-quack.pdf
qpdf/qtest/qpdf/json-need-appearances-acroform.out
qpdf/qtest/qpdf/more-choices.pdf
qpdf/qtest/qpdf/need-appearances-out.pdf
qpdf/qtest/qpdf/need-appearances.pdf
 2019-01-05  Jay Berkenbilt  <ejb@ql.org>
+	* When generating appearances, if the font uses one of the
+	standard, built-in encodings, restrict the character set to that
+	rather than just to ASCII. This will allow most appearances to
+	contain characters from the ISO-Latin-1 range plus a few
+	additional characters.
+
 	* Add methods QUtil::utf8_to_win_ansi and
 	QUtil::utf8_to_mac_roman.
@@ -196,6 +196,8 @@ class QPDFFormFieldObjectHelper: public QPDFObjectHelper
     void setRadioButtonValue(QPDFObjectHandle name);
     void setCheckBoxValue(bool value);
     void generateTextAppearance(QPDFAnnotationObjectHelper&);
+    QPDFObjectHandle getFontFromResource(
+        QPDFObjectHandle resources, std::string const& font_name);
     class Members
     {
@@ -689,10 +689,13 @@ class TfFinder: public QPDFObjectHandle::TokenFilter
     }
     virtual void handleToken(QPDFTokenizer::Token const&);
     double getTf();
+    std::string getFontName();
   private:
     double tf;
+    std::string font_name;
     double last_num;
+    std::string last_name;
 };
 TfFinder::TfFinder() :
@@ -713,6 +716,10 @@ TfFinder::handleToken(QPDFTokenizer::Token const&amp; token)
         last_num = strtod(value.c_str(), 0);
         break;
+      case QPDFTokenizer::tt_name:
+        last_name = value;
+        break;
+
       case QPDFTokenizer::tt_word:
         if ((value == "Tf") &&
             (last_num > 1.0) &&
@@ -722,6 +729,7 @@ TfFinder::handleToken(QPDFTokenizer::Token const&amp; token)
             // insane things or suffering from over/underflow
             tf = last_num;
         }
+        font_name = last_name;
         break;
       default:
@@ -735,6 +743,26 @@ TfFinder::getTf()
     return this->tf;
 }
+std::string
+TfFinder::getFontName()
+{
+    return this->font_name;
+}
+
+QPDFObjectHandle
+QPDFFormFieldObjectHelper::getFontFromResource(
+    QPDFObjectHandle resources, std::string const& name)
+{
+    QPDFObjectHandle result;
+    if (resources.isDictionary() &&
+        resources.getKey("/Font").isDictionary() &&
+        resources.getKey("/Font").hasKey(name))
+    {
+        result = resources.getKey("/Font").getKey(name);
+    }
+    return result;
+}
+
 void
 QPDFFormFieldObjectHelper::generateTextAppearance(
     QPDFAnnotationObjectHelper& aoh)
@@ -755,17 +783,52 @@ QPDFFormFieldObjectHelper::generateTextAppearance(
     }
     QPDFObjectHandle::Rectangle bbox = bbox_obj.getArrayAsRectangle();
     std::string DA = getDefaultAppearance();
-    std::string V = QUtil::utf8_to_ascii(getValueAsString());
+    std::string V = getValueAsString();
+    std::vector<std::string> opt;
+    if (isChoice() && ((getFlags() & ff_ch_combo) == 0))
+    {
+        opt = getChoices();
+    }
     TfFinder tff;
     Pl_QPDFTokenizer tok("tf", &tff);
     tok.write(QUtil::unsigned_char_pointer(DA.c_str()), DA.length());
     tok.finish();
     double tf = tff.getTf();
-    std::vector<std::string> opt;
-    if (isChoice() && ((getFlags() & ff_ch_combo) == 0))
+
+    std::string (*encoder)(std::string const&, char) = &QUtil::utf8_to_ascii;
+    std::string font_name = tff.getFontName();
+    if (! font_name.empty())
     {
-        opt = getChoices();
+        // See if the font is encoded with something we know about.
+        QPDFObjectHandle resources = AS.getDict().getKey("/Resources");
+        QPDFObjectHandle font = getFontFromResource(resources, font_name);
+        if (! font.isInitialized())
+        {
+            QPDFObjectHandle dr = getInheritableFieldValue("/DR");
+            font = getFontFromResource(dr, font_name);
+        }
+        if (font.isDictionary() &&
+            font.getKey("/Encoding").isName())
+        {
+            std::string encoding = font.getKey("/Encoding").getName();
+            if (encoding == "/WinAnsiEncoding")
+            {
+                QTC::TC("qpdf", "QPDFFormFieldObjectHelper WinAnsi");
+                encoder = &QUtil::utf8_to_win_ansi;
+            }
+            else if (encoding == "/MacRomanEncoding")
+            {
+                encoder = &QUtil::utf8_to_mac_roman;
+            }
+        }
     }
+
+    V = (*encoder)(V, '?');
+    for (size_t i = 0; i < opt.size(); ++i)
+    {
+        opt.at(i) = (*encoder)(opt.at(i), '?');
+    }
+
     AS.addTokenFilter(new ValueSetter(DA, V, opt, tf, bbox));
 }
@@ -1422,8 +1422,9 @@ outfile.pdf&lt;/option&gt;
          <listitem>
           <para>
            For text fields and list boxes, any characters that fall
-           outside of US-ASCII will be replaced by the
-           <literal>?</literal> character.
+           outside of US-ASCII or, if detected, &ldquo;Windows
+           ANSI&rdquo; or &ldquo;Mac Roman&rdquo; encoding, will be
+           replaced by the <literal>?</literal> character.
           </para>
          </listitem>
         </itemizedlist>
@@ -1184,10 +1184,10 @@ ArgParser::argHelp()
         << "fields in files like this. If you get this warning, you have two choices:\n"
         << "either use qpdf's --generate-appearances flag to tell qpdf to go ahead and\n"
         << "regenerate appearances, or use some other tool to generate the appearances.\n"
-        << "qpdf does a pretty good job with most forms when only ASCII characters are\n"
-        << "used in form field values, but if your form fields contain other\n"
-        << "characters, rich text, or are other than left justified, you will get\n"
-        << "better results first saving with other software.\n"
+        << "qpdf does a pretty good job with most forms when only ASCII and \"Windows\n"
+        << "ANSI\" characters are used in form field values, but if your form fields\n"
+        << "contain other characters, rich text, or are other than left justified, you\n"
+        << "will get better results first saving with other software.\n"
         << "\n"
         << "Version numbers may be expressed as major.minor.extension-level, so 1.7.3\n"
         << "means PDF version 1.7 at extension level 3.\n"
@@ -405,3 +405,4 @@ QPDF detected dangling ref 0
 qpdf image optimize no pipeline 0
 qpdf image optimize no shrink 0
 qpdf image optimize too small 0
+QPDFFormFieldObjectHelper WinAnsi 0
@@ -263,7 +263,7 @@
         "parent": null,
         "partialname": "text2",
         "quadding": 0,
-        "value": "salad πʬ"
+        "value": "salad ÷πʬ"
       },
       {
         "alternativename": "combolist1",