Merge pull request #1575 from m-holger/string

Add new private-API class String

Merge pull request #1575 from m-holger/string
Add new private-API class String
m-holger · GitHub
2 parents 08d49887 a367a367
Showing 18 changed files with 4006 additions and 115 deletions
include/qpdf/ObjectHandle.hh
include/qpdf/QPDFObjectHandle.hh
libqpdf/QPDFFormFieldObjectHelper.cc
libqpdf/QPDFObjectHandle.cc
libqpdf/QPDFParser.cc
libqpdf/QPDF_String.cc
libqpdf/QPDF_json.cc
libqpdf/QPDF_objects.cc
libqpdf/QUtil.cc
libqpdf/qpdf/QPDFObjectHandle_private.hh
libqpdf/qpdf/QPDFObject_private.hh
libqpdf/qpdf/Util.hh
libtests/qutil.cc
qpdf/qpdf.testcov
qpdf/qtest/json.test
qpdf/qtest/qpdf/json-need-appearances-acroform-v1.out → qpdf/qtest/qpdf/json-need-appearances-utf8-acroform-v1.out
qpdf/qtest/qpdf/json-need-appearances-acroform-v2.out → qpdf/qtest/qpdf/json-need-appearances-utf8-acroform-v2.out
qpdf/qtest/qpdf/need-appearances-utf8.pdf
@@ -105,6 +105,9 @@ namespace qpdf
         void warn(QPDFExc&&) const;
         void warn(std::string const& warning) const;
+        inline std::shared_ptr<QPDFObject> const& obj_sp() const;
+        inline QPDFObjectHandle oh() const;
+
       protected:
         BaseHandle() = default;
         BaseHandle(std::shared_ptr<QPDFObject> const& obj) :
@@ -1321,21 +1321,6 @@ class QPDFObjectHandle: public qpdf::BaseHandle
     {
         return obj;
     }
-    std::shared_ptr<QPDFObject>
-    getObj() const
-    {
-        return obj;
-    }
-    QPDFObject*
-    getObjectPtr()
-    {
-        return obj.get();
-    }
-    QPDFObject* const
-    getObjectPtr() const
-    {
-        return obj.get();
-    }
     void writeJSON(int json_version, JSON::Writer& p, bool dereference_indirect = false) const;
@@ -765,7 +765,7 @@ QPDFFormFieldObjectHelper::generateTextAppearance(QPDFAnnotationObjectHelper&amp; ao
         return;
     }
-    if (AS.getObj().use_count() > 4) {
+    if (AS.obj_sp().use_count() > 3) {
         aoh.warn("unable to generate text appearance from shared appearance stream for update");
         return;
     }
@@ -318,7 +318,7 @@ BaseHandle::copy(bool shallow) const
         throw std::logic_error("attempted to shallow copy QPDFObjectHandle from destroyed QPDF");
         return {}; // does not return
     case ::ot_reference:
-        return obj->qpdf->getObject(obj->og).getObj();
+        return obj->qpdf->getObject(obj->og).obj_sp();
     }
     return {}; // unreachable
 }
@@ -470,7 +470,7 @@ BaseHandle::write_json(int json_version, JSON::Writer&amp; p) const
                         p.writeNext() << "null";
                     }
                     p.writeNext();
-                    auto item_og = value.getObj()->getObjGen();
+                    auto item_og = value.id_gen();
                     if (item_og.isIndirect()) {
                         p << "\"" << item_og.unparse(' ') << " R\"";
                     } else {
@@ -999,50 +999,110 @@ QPDFObjectHandle::getValueAsName(std::string&amp; value) const
     return true;
 }
-// String accessors
+// String methods
+
+QPDFObjectHandle
+QPDFObjectHandle::newString(std::string const& str)
+{
+    return {QPDFObject::create<QPDF_String>(str)};
+}
+
+QPDFObjectHandle
+QPDFObjectHandle::newUnicodeString(std::string const& utf8_str)
+{
+    return {String::utf16(utf8_str).obj_sp()};
+}
+
+String::String(std::string const& str) :
+    BaseHandle(QPDFObject::create<QPDF_String>(str))
+{
+}
+
+String::String(std::string&& str) :
+    BaseHandle(QPDFObject::create<QPDF_String>(std::move(str)))
+{
+}
+
+String
+String::utf16(std::string const& utf8_str)
+{
+    std::string result;
+    if (QUtil::utf8_to_pdf_doc(utf8_str, result, '?')) {
+        return String(result);
+    }
+    return String(QUtil::utf8_to_utf16(utf8_str));
+}
+
+std::string const&
+String::value() const
+{
+    auto* s = as<QPDF_String>();
+    if (!s) {
+        throw invalid_error("String");
+    }
+    return s->val;
+}
+
+std::string
+String::utf8_value() const
+{
+    auto* s = as<QPDF_String>();
+    if (!s) {
+        throw invalid_error("String");
+    }
+    if (util::is_utf16(s->val)) {
+        return QUtil::utf16_to_utf8(s->val);
+    }
+    if (util::is_explicit_utf8(s->val)) {
+        // PDF 2.0 allows UTF-8 strings when explicitly prefixed with the three-byte representation
+        // of U+FEFF.
+        return s->val.substr(3);
+    }
+    return QUtil::pdf_doc_to_utf8(s->val);
+}
 std::string
 QPDFObjectHandle::getStringValue() const
 {
-    if (isString()) {
-        return obj->getStringValue();
-    } else {
+    try {
+        return String(obj).value();
+    } catch (std::invalid_argument&) {
         typeWarning("string", "returning empty string");
-        QTC::TC("qpdf", "QPDFObjectHandle string returning empty string");
-        return "";
+        return {};
     }
 }
 bool
 QPDFObjectHandle::getValueAsString(std::string& value) const
 {
-    if (!isString()) {
+    try {
+        value = String(obj).value();
+        return true;
+    } catch (std::invalid_argument&) {
         return false;
     }
-    value = obj->getStringValue();
-    return true;
 }
 std::string
 QPDFObjectHandle::getUTF8Value() const
 {
-    if (auto str = as<QPDF_String>()) {
-        return str->getUTF8Val();
-    } else {
+    try {
+        return String(obj).utf8_value();
+    } catch (std::invalid_argument&) {
         typeWarning("string", "returning empty string");
-        QTC::TC("qpdf", "QPDFObjectHandle string returning empty utf8");
-        return "";
+        return {};
     }
 }
 bool
 QPDFObjectHandle::getValueAsUTF8(std::string& value) const
 {
-    if (auto str = as<QPDF_String>()) {
-        value = str->getUTF8Val();
+    try {
+        value = String(obj).utf8_value();
         return true;
+    } catch (std::invalid_argument&) {
+        return false;
     }
-    return false;
 }
 // Operator and Inline Image accessors
@@ -1718,18 +1778,6 @@ QPDFObjectHandle::newReal(double value, int decimal_places, bool trim_trailing_z
 }
 QPDFObjectHandle
-QPDFObjectHandle::newString(std::string const& str)
-{
-    return {QPDFObject::create<QPDF_String>(str)};
-}
-
-QPDFObjectHandle
-QPDFObjectHandle::newUnicodeString(std::string const& utf8_str)
-{
-    return {QPDF_String::create_utf16(utf8_str)};
-}
-
-QPDFObjectHandle
 QPDFObjectHandle::newOperator(std::string const& value)
 {
     return {QPDFObject::create<QPDF_Operator>(value)};
@@ -626,8 +626,8 @@ QPDFParser::fixMissingKeys()
 {
     std::set<std::string> names;
     for (auto& obj: frame->olist) {
-        if (obj.getObj()->getTypeCode() == ::ot_name) {
-            names.insert(obj.getObj()->getStringValue());
+        if (obj.raw_type_code() == ::ot_name) {
+            names.insert(obj.obj_sp()->getStringValue());
         }
     }
     int next_fake_key = 1;
@@ -2,6 +2,7 @@
 #include <qpdf/QPDFObjectHandle_private.hh>
 #include <qpdf/QUtil.hh>
+#include <qpdf/Util.hh>
 // DO NOT USE ctype -- it is locale dependent for some things, and it's not worth the risk of
 // including it in case it may accidentally be used.
@@ -9,40 +10,46 @@
 static bool
 is_iso_latin1_printable(char ch)
 {
-    return (((ch >= 32) && (ch <= 126)) || (static_cast<unsigned char>(ch) >= 160));
-}
-
-std::shared_ptr<QPDFObject>
-QPDF_String::create_utf16(std::string const& utf8_val)
-{
-    std::string result;
-    if (!QUtil::utf8_to_pdf_doc(utf8_val, result, '?')) {
-        result = QUtil::utf8_to_utf16(utf8_val);
-    }
-    return QPDFObject::create<QPDF_String>(result);
+    return (ch >= 32 && ch <= 126) || static_cast<unsigned char>(ch) >= 160;
 }
 void
 QPDF_String::writeJSON(int json_version, JSON::Writer& p)
 {
-    auto candidate = getUTF8Val();
     if (json_version == 1) {
-        p << "\"" << JSON::Writer::encode_string(candidate) << "\"";
-    } else {
-        // See if we can unambiguously represent as Unicode.
-        if (QUtil::is_utf16(val) || QUtil::is_explicit_utf8(val)) {
+        if (util::is_utf16(val)) {
+            p << "\"" << JSON::Writer::encode_string(QUtil::utf16_to_utf8(val)) << "\"";
+            return;
+        }
+        if (util::is_explicit_utf8(val)) {
+            // PDF 2.0 allows UTF-8 strings when explicitly prefixed with the three-byte
+            // representation of U+FEFF.
+            p << "\"" << JSON::Writer::encode_string(val.substr(3)) << "\"";
+            return;
+        }
+        p << "\"" << JSON::Writer::encode_string(QUtil::pdf_doc_to_utf8(val)) << "\"";
+        return;
+    }
+    // See if we can unambiguously represent as Unicode.
+    if (util::is_utf16(val)) {
+        p << "\"u:" << JSON::Writer::encode_string(QUtil::utf16_to_utf8(val)) << "\"";
+        return;
+    }
+    // See if we can unambiguously represent as Unicode.
+    if (util::is_explicit_utf8(val)) {
+        p << "\"u:" << JSON::Writer::encode_string(val.substr(3)) << "\"";
+        return;
+    }
+    if (!useHexString()) {
+        auto candidate = QUtil::pdf_doc_to_utf8(val);
+        std::string test;
+        if (QUtil::utf8_to_pdf_doc(candidate, test, '?') && test == val) {
+            // This is a PDF-doc string that can be losslessly encoded as Unicode.
             p << "\"u:" << JSON::Writer::encode_string(candidate) << "\"";
             return;
-        } else if (!useHexString()) {
-            std::string test;
-            if (QUtil::utf8_to_pdf_doc(candidate, test, '?') && (test == val)) {
-                // This is a PDF-doc string that can be losslessly encoded as Unicode.
-                p << "\"u:" << JSON::Writer::encode_string(candidate) << "\"";
-                return;
-            }
         }
-        p << "\"b:" << QUtil::hex_encode(val) << "\"";
     }
+    p << "\"b:" << QUtil::hex_encode(val) << "\"";
 }
 bool
@@ -133,17 +140,3 @@ QPDF_String::unparse(bool force_binary)
     return result;
 }
-
-std::string
-QPDF_String::getUTF8Val() const
-{
-    if (QUtil::is_utf16(val)) {
-        return QUtil::utf16_to_utf8(val);
-    } else if (QUtil::is_explicit_utf8(val)) {
-        // PDF 2.0 allows UTF-8 strings when explicitly prefixed with the three-byte representation
-        // of U+FEFF.
-        return val.substr(3);
-    } else {
-        return QUtil::pdf_doc_to_utf8(val);
-    }
-}
@@ -693,7 +693,7 @@ QPDF::JSONReactor::setObjectDescription(QPDFObjectHandle&amp; oh, JSON const&amp; value)
             QPDFObject::JSON_Descr(j_descr.input, cur_object));
     }
-    oh.getObjectPtr()->setDescription(&pdf, descr, value.getStart());
+    oh.obj_sp()->setDescription(&pdf, descr, value.getStart());
 }
 QPDFObjectHandle
@@ -1554,7 +1554,7 @@ Objects::readObjectAtOffset(
             break;
         }
     }
-    m->objects.updateCache(og, oh.getObj(), end_before_space, m->file->tell());
+    m->objects.updateCache(og, oh.obj_sp(), end_before_space, m->file->tell());
 }
 QPDFObjectHandle
@@ -1613,7 +1613,7 @@ Objects::readObjectAtOffset(
             break;
         }
     }
-    m->objects.updateCache(og, oh.getObj(), end_before_space, m->file->tell());
+    m->objects.updateCache(og, oh.obj_sp(), end_before_space, m->file->tell());
     return oh;
 }
@@ -1805,7 +1805,7 @@ Objects::resolveObjectsInStream(int obj_stream_number)
             entry->second.getObjStreamNumber() == obj_stream_number) {
             is::OffsetBuffer in("", {b_start + obj_offset, obj_size}, obj_offset);
             auto oh = readObjectInStream(in, obj_stream_number, obj_id);
-            updateCache(og, oh.getObj(), end_before_space, end_after_space);
+            updateCache(og, oh.obj_sp(), end_before_space, end_after_space);
         } else {
             QTC::TC("qpdf", "QPDF not caching overridden objstm object");
         }
@@ -1874,7 +1874,7 @@ QPDF::makeIndirectObject(QPDFObjectHandle oh)
     if (!oh) {
         throw std::logic_error("attempted to make an uninitialized QPDFObjectHandle indirect");
     }
-    return m->objects.makeIndirectFromQPDFObject(oh.getObj());
+    return m->objects.makeIndirectFromQPDFObject(oh.obj_sp());
 }
 std::shared_ptr<QPDFObject>
@@ -1935,7 +1935,7 @@ QPDF::replaceObject(QPDFObjGen og, QPDFObjectHandle oh)
     if (!oh || (oh.isIndirect() && !(oh.isStream() && oh.getObjGen() == og))) {
         throw std::logic_error("QPDF::replaceObject called with indirect object handle");
     }
-    m->objects.updateCache(og, oh.getObj(), -1, -1, false);
+    m->objects.updateCache(og, oh.obj_sp(), -1, -1, false);
 }
 void
@@ -1688,19 +1688,13 @@ QUtil::utf8_to_pdf_doc(std::string const&amp; utf8, std::string&amp; pdfdoc, char unknow
 bool
 QUtil::is_utf16(std::string const& val)
 {
-    return (
-        (val.length() >= 2) &&
-        (((val.at(0) == '\xfe') && (val.at(1) == '\xff')) ||
-         ((val.at(0) == '\xff') && (val.at(1) == '\xfe'))));
+    return util::is_utf16(val);
 }
 bool
 QUtil::is_explicit_utf8(std::string const& val)
 {
-    // QPDF_String.cc knows that this is a 3-byte sequence.
-    return (
-        (val.length() >= 3) && (val.at(0) == '\xef') && (val.at(1) == '\xbb') &&
-        (val.at(2) == '\xbf'));
+    return util::is_explicit_utf8(val);
 }
 std::string
@@ -412,15 +412,15 @@ namespace qpdf
         {
         }
-        // Return the name value. If the object is not a valid Name, throw a
-        // std::invalid_argument exception.
+        // Return the name value. If the object is not a valid Name, throw a std::invalid_argument
+        // exception.
         operator std::string() const&
         {
             return value();
         }
-        // Return the integer value. If the object is not a valid integer, throw a
-        // std::invalid_argument exception.
+        // Return the name value. If the object is not a valid name, throw a std::invalid_argument
+        // exception.
         std::string const& value() const;
         // Return true if object value is equal to the 'rhs' value. Return false if the object is
@@ -589,7 +589,56 @@ namespace qpdf
         void warn(std::string const& message);
         static std::map<std::string, std::string> filter_abbreviations;
-    };
+    }; // class Stream
+
+    class String final: public BaseHandle
+    {
+      public:
+        String() = default;
+        String(String const&) = default;
+        String(String&&) = default;
+        String& operator=(String const&) = default;
+        String& operator=(String&&) = default;
+        ~String() = default;
+
+        explicit String(std::string const&);
+        explicit String(std::string&&);
+
+        String(QPDFObjectHandle const& oh) :
+            BaseHandle(oh.type_code() == ::ot_string ? oh : QPDFObjectHandle())
+        {
+        }
+
+        String(QPDFObjectHandle&& oh) :
+            BaseHandle(oh.type_code() == ::ot_string ? std::move(oh) : QPDFObjectHandle())
+        {
+        }
+
+        static String utf16(std::string const&);
+
+        // Return the string value. If the object is not a valid string, throw a
+        // std::invalid_argument exception.
+        operator std::string() const&
+        {
+            return value();
+        }
+
+        // Return the string value. If the object is not a valid string, throw a
+        // std::invalid_argument exception.
+        std::string const& value() const;
+
+        // Return the string value. If the object is not a valid string, throw a
+        // std::invalid_argument exception.
+        std::string utf8_value() const;
+
+        // Return true if object value is equal to the 'rhs' value. Return false if the object is
+        // not a valid String.
+        friend bool
+        operator==(String const& lhs, std::string_view rhs)
+        {
+            return lhs && lhs.value() == rhs;
+        }
+    }; // class String
     template <typename T>
     T*
@@ -621,6 +670,18 @@ namespace qpdf
     {
     }
+    inline std::shared_ptr<QPDFObject> const&
+    BaseHandle::obj_sp() const
+    {
+        return obj;
+    }
+
+    inline QPDFObjectHandle
+    BaseHandle::oh() const
+    {
+        return {obj};
+    }
+
     inline void
     BaseHandle::assign(qpdf_object_type_e required, BaseHandle const& other)
     {
@@ -30,6 +30,7 @@ namespace qpdf
     class Integer;
     class Name;
     class Stream;
+    class String;
     namespace impl
     {
@@ -261,20 +262,24 @@ class QPDF_String final
 {
     friend class QPDFObject;
     friend class qpdf::BaseHandle;
+    friend class qpdf::String;
     friend class qpdf::impl::Writer;
   public:
-    static std::shared_ptr<QPDFObject> create_utf16(std::string const& utf8_val);
     std::string unparse(bool force_binary = false);
     void writeJSON(int json_version, JSON::Writer& p);
-    std::string getUTF8Val() const;
   private:
-    QPDF_String(std::string val) :
+    QPDF_String(std::string const& val) :
+        val(val)
+    {
+    }
+    QPDF_String(std::string&& val) :
         val(std::move(val))
     {
     }
     bool useHexString() const;
+
     std::string val;
 };
@@ -74,6 +74,19 @@ namespace qpdf::util
         s.insert(0, 1, '1');
     }
+    inline bool
+    is_utf16(std::string const& str)
+    {
+        return str.starts_with("\xfe\xff") || str.starts_with("\xff\xfe");
+    }
+
+    inline bool
+    is_explicit_utf8(std::string const& str)
+    {
+        // QPDF_String.cc knows that this is a 3-byte sequence.
+        return str.starts_with("\xef\xbb\xbf");
+    }
+
     std::string random_string(size_t len);
 } // namespace qpdf::util
@@ -367,6 +367,16 @@ check_analyze(std::string const&amp; str, bool has8bit, bool utf8, bool utf16)
 }
 void
+explicit_utf8_test()
+{
+    assert(QUtil::is_explicit_utf8("\xef\xbb\xbfnot empty"));
+    assert(QUtil::is_explicit_utf8("\xef\xbb\xbf"));
+    assert(!QUtil::is_explicit_utf8("\xef\xbb\xbenot explicit"));
+    assert(!QUtil::is_explicit_utf8("\xef\xbe\xbfnot explicit"));
+    assert(!QUtil::is_explicit_utf8("\xee\xbb\xbfnot explicit"));
+}
+
+void
 print_alternatives(std::string const& str)
 {
     std::vector<std::string> result = QUtil::possible_repaired_encodings(str);
@@ -432,7 +442,7 @@ transcoding_test()
     std::string other_to_utf8;
     assert(!QUtil::utf8_to_pdf_doc(other_utf8, other_to_utf8));
     std::cout << other_to_utf8 << '\n';
-    std::cout << "done other characters" << '\n';
+    std::cout << "done other characters\n";
     // These valid UTF8 strings when converted to PDFDoc would end up
     // with a byte sequence that would be recognized as UTF-8 or
     // UTF-16 rather than PDFDoc. A special case is required to store
@@ -747,6 +757,7 @@ main(int argc, char* argv[])
         getenv_test();
         std::cout << "---- utf8" << '\n';
         to_utf8_test();
+        explicit_utf8_test();
         std::cout << "---- utf16" << '\n';
         to_utf16_test();
         std::cout << "---- utf8_to_ascii" << '\n';
@@ -174,8 +174,6 @@ QPDFParser eof in parse 0
 QPDFParser eof in parseRemainder 0
 QPDFObjectHandle boolean returning false 0
 QPDFObjectHandle real returning 0.0 0
-QPDFObjectHandle string returning empty string 0
-QPDFObjectHandle string returning empty utf8 0
 QPDFObjectHandle operator returning fake value 0
 QPDFObjectHandle inlineimage returning empty data 0
 QPDFObjectHandle array treating as empty vector 0
@@ -38,7 +38,7 @@ my @json_files = (
     ['page-labels-and-outlines',
      ['--json-key=objects', '--json-object=trailer', '--json-object=2 0 R']],
     ['field-types', ['--json-key=acroform']],
-    ['need-appearances', ['--json-key=acroform']],
+    ['need-appearances-utf8', ['--json-key=acroform']],
     ['V4-aes', ['--json-key=encrypt']],
     ['V4-aes', ['--json-key=encrypt', '--show-encryption-key']],
 );
@@ -27,7 +27,7 @@
         "parent": null,
         "partialname": "text",
         "quadding": 0,
-        "value": "abc"
+        "value": "abcde"
       },
       {
         "alternativename": "r1",
@@ -27,7 +27,7 @@
         "parent": null,
         "partialname": "text",
         "quadding": 0,
-        "value": "u:abc"
+        "value": "u:abcde"
       },
       {
         "alternativename": "r1",