Fix some PDF test files (#350)

Author: andiwand

CMakeLists.txt

@@ -141,6 +141,7 @@ set(ODR_SOURCE_FILES
         "src/odr/internal/pdf/pdf_document_element.cpp"
         "src/odr/internal/pdf/pdf_document_parser.cpp"
         "src/odr/internal/pdf/pdf_file.cpp"
+        "src/odr/internal/pdf/pdf_file_object.cpp"
         "src/odr/internal/pdf/pdf_file_parser.cpp"
         "src/odr/internal/pdf/pdf_graphics_operator.cpp"
         "src/odr/internal/pdf/pdf_graphics_operator_parser.cpp"

src/odr/internal/html/pdf_file.cpp

@@ -71,12 +71,17 @@ Html html::translate_pdf_file(const PdfFile &pdf_file,
           o << "height:" << page_box[3].as_real() / 72.0 << "in;";
         }));
 
-    pdf::IndirectObject page_contents_object =
-        parser.read_object(page->contents_reference);
-    std::string stream = parser.read_object_stream(page_contents_object);
-    std::string page_content = crypto::util::zlib_inflate(stream);
+    std::string stream;
+    for (const auto &content_reference : page->contents_reference) {
+      pdf::IndirectObject page_contents_object =
+          parser.read_object(content_reference);
+      std::string page_content =
+          parser.read_object_stream(page_contents_object);
+      page_content = crypto::util::zlib_inflate(page_content);
+      stream += page_content;
+    }
 
-    std::istringstream ss(page_content);
+    std::istringstream ss(stream);
     pdf::GraphicsOperatorParser parser2(ss);
     pdf::GraphicsState state;
     while (!ss.eof()) {

src/odr/internal/pdf/README.md

@@ -41,7 +41,7 @@ Apart from that PDF can also modify the spacing between the characters for examp
   - Document Structure https://www.oreilly.com/library/view/pdf-explained/9781449321581/ch04.html
   - Cross-reference stream https://www.verypdf.com/document/pdf-format-reference/pg_0106.htm
 - Graphics Operators
-  - https://gendignoux.com/blog/images/pdf-graphics/cheat-sheet-by-nc-sa.png 
+  - https://github.com/gendx/pdf-cheat-sheets/blob/master/pdf-graphics.clean.pdf
   - https://pdfa.org/wp-content/uploads/2023/08/PDF-Operators-CheatSheet.pdf
   - General https://gendignoux.com/blog/2017/01/05/pdf-graphics.html
   - Text https://www.syncfusion.com/succinctly-free-ebooks/pdf/text-operators

src/odr/internal/pdf/pdf_cmap_parser.cpp

@@ -37,15 +37,14 @@ std::variant<Object, std::string> CMapParser::read_token() const {
 
   std::string token;
   while (true) {
-    int_type c = sb().sgetc();
+    int_type c = m_parser.geti();
     if (c == eof) {
-      in().setstate(std::ios::eofbit);
       return token;
     }
     if (ObjectParser::is_whitespace(c)) {
       return token;
     }
-    sb().sbumpc();
+    m_parser.bumpc();
     token += (char_type)c;
   }
 }
@@ -76,10 +75,10 @@ void CMapParser::read_bfchar(std::uint32_t n, CMap &cmap) const {
         reinterpret_cast<const char16_t *>(unicode.data()), unicode.size() / 2);
 
     if (glyph.length() != 1) {
-      throw std::runtime_error("unexpected glyph length");
+      std::cerr << "unexpected glyph length" << std::endl;
     }
     if (unicode16.length() != 1) {
-      throw std::runtime_error("unexpected unicode length");
+      std::cerr << "unexpected unicode length" << std::endl;
     }
 
     cmap.map_bfchar(glyph[0], unicode16[0]);

src/odr/internal/pdf/pdf_document_element.hpp

@@ -49,7 +49,7 @@ struct Page : Element {
   std::vector<Annotation *> annotations;
 
   // TODO remove
-  ObjectReference contents_reference;
+  std::vector<ObjectReference> contents_reference;
 };
 
 struct Annotation : Element {};

src/odr/internal/pdf/pdf_document_parser.cpp

@@ -6,6 +6,8 @@
 #include <odr/internal/pdf/pdf_document_element.hpp>
 #include <odr/internal/pdf/pdf_file_parser.hpp>
 
+#include <functional>
+#include <iostream>
 #include <sstream>
 
 namespace odr::internal::pdf {
@@ -27,7 +29,7 @@ pdf::Font *parse_font(DocumentParser &parser, const ObjectReference &reference,
   font->object = dictionary;
 
   if (dictionary.has_key("ToUnicode")) {
-    auto to_unicode_obj =
+    IndirectObject to_unicode_obj =
         parser.read_object(dictionary["ToUnicode"].as_reference());
     std::string stream = parser.read_object_stream(to_unicode_obj);
     std::string inflate = crypto::util::zlib_inflate(stream);
@@ -39,26 +41,21 @@ pdf::Font *parse_font(DocumentParser &parser, const ObjectReference &reference,
   return font;
 }
 
-pdf::Resources *parse_resources(DocumentParser &parser,
-                                const ObjectReference &reference,
+pdf::Resources *parse_resources(DocumentParser &parser, const Object &object,
                                 Document &document) {
   Resources *resources = document.create_element<Resources>();
 
-  IndirectObject object = parser.read_object(reference);
-  const Dictionary &dictionary = object.object.as_dictionary();
+  Dictionary dictionary = parser.resolve_object_copy(object).as_dictionary();
 
   resources->type = Type::resources;
-  resources->object_reference = reference;
   resources->object = dictionary;
 
-  if (dictionary["Font"].is_reference()) {
-    Dictionary table = parser.read_object(dictionary["Font"].as_reference())
-                           .object.as_dictionary();
-    for (const auto &[key, value] : table) {
+  if (!dictionary["Font"].is_null()) {
+    Dictionary font_table =
+        parser.resolve_object_copy(dictionary["Font"]).as_dictionary();
+    for (const auto &[key, value] : font_table) {
       resources->font[key] = parse_font(parser, value.as_reference(), document);
     }
-  } else {
-    throw std::runtime_error("problem");
   }
 
   return resources;
@@ -90,12 +87,21 @@ pdf::Page *parse_page(DocumentParser &parser, const ObjectReference &reference,
   page->object_reference = reference;
   page->object = dictionary;
   page->parent = dynamic_cast<Pages *>(parent);
-  page->contents_reference = dictionary["Contents"].as_reference();
-  page->resources =
-      parse_resources(parser, dictionary["Resources"].as_reference(), document);
+  page->resources = parse_resources(parser, dictionary["Resources"], document);
+
+  if (dictionary["Contents"].is_reference()) {
+    page->contents_reference = {dictionary["Contents"].as_reference()};
+  } else {
+    for (const Object &e : dictionary["Contents"].as_array()) {
+      page->contents_reference.push_back(e.as_reference());
+    }
+  }
 
   if (dictionary.has_key("Annots")) {
-    for (Object annotation : dictionary["Annots"].as_array()) {
+    // TODO why rvalue not working?
+    Array annotations =
+        parser.resolve_object_copy(dictionary["Annots"]).as_array();
+    for (const Object &annotation : annotations) {
       page->annotations.push_back(
           parse_annotation(parser, annotation.as_reference(), document));
     }
@@ -116,7 +122,7 @@ pdf::Pages *parse_pages(DocumentParser &parser,
   pages->object = dictionary;
   pages->count = dictionary["Count"].as_integer();
 
-  for (const auto &kid : dictionary["Kids"].as_array()) {
+  for (const Object &kid : dictionary["Kids"].as_array()) {
     pages->kids.push_back(
         parse_page_or_pages(parser, kid.as_reference(), document, pages));
   }
@@ -169,10 +175,17 @@ const FileParser &DocumentParser::parser() const { return m_parser; }
 
 const Xref &DocumentParser::xref() const { return m_xref; }
 
-IndirectObject DocumentParser::read_object(const ObjectReference &reference) {
-  std::uint32_t position = m_xref.table[reference.id].position;
+const IndirectObject &
+DocumentParser::read_object(const ObjectReference &reference) {
+  if (auto it = m_objects.find(reference); it != std::end(m_objects)) {
+    return it->second;
+  }
+
+  std::uint32_t position = m_xref.table.at(reference).position;
   in().seekg(position);
-  return parser().read_indirect_object();
+  IndirectObject object = parser().read_indirect_object();
+
+  return m_objects.emplace(reference, std::move(object)).first->second;
 }
 
 std::string
@@ -198,15 +211,64 @@ std::string DocumentParser::read_object_stream(const IndirectObject &object) {
 std::unique_ptr<Document> DocumentParser::parse_document() {
   parser().seek_start_xref();
   StartXref start_xref = parser().read_start_xref();
-  in().seekg(start_xref.start);
 
-  m_xref = parser().read_xref();
-  parser().parser().skip_whitespace();
-  Trailer trailer = parser().read_trailer();
+  std::uint32_t xref_position = start_xref.start;
+  std::optional<Trailer> trailer;
+
+  while (true) {
+    in().seekg(xref_position);
+
+    m_xref.append(parser().read_xref());
+    parser().parser().skip_whitespace();
+    Trailer new_trailer = parser().read_trailer();
+    if (!trailer) {
+      trailer = new_trailer;
+    }
+
+    if (new_trailer.dictionary.has_key("Prev")) {
+      xref_position = new_trailer.dictionary["Prev"].as_integer();
+      continue;
+    }
+
+    break;
+  }
 
   auto document = std::make_unique<Document>();
-  document->catalog = parse_catalog(*this, trailer.root_reference, *document);
+  document->catalog =
+      parse_catalog(*this, trailer->root_reference(), *document);
   return document;
 }
 
+void DocumentParser::resolve_object(Object &object) {
+  if (object.is_reference()) {
+    object = read_object(object.as_reference()).object;
+  }
+}
+
+void DocumentParser::deep_resolve_object(Object &object) {
+  if (object.is_reference()) {
+    object = read_object(object.as_reference()).object;
+  } else if (object.is_array()) {
+    for (Object &e : object.as_array()) {
+      deep_resolve_object(e);
+    }
+  } else if (object.is_dictionary()) {
+    for (auto &[k, v] : object.as_dictionary()) {
+      deep_resolve_object(v);
+    }
+  }
+}
+
+Object DocumentParser::resolve_object_copy(const Object &object) {
+  Object result = object;
+  resolve_object(result);
+  return result;
+}
+
+Object DocumentParser::deep_resolve_object_copy(const Object &object) {
+  Object result = object;
+  deep_resolve_object(result);
+  return result;
+}
+
 } // namespace odr::internal::pdf

src/odr/internal/pdf/pdf_document_parser.hpp

@@ -5,6 +5,7 @@
 #include <odr/internal/pdf/pdf_file_parser.hpp>
 
 #include <iosfwd>
+#include <map>
 #include <memory>
 
 namespace odr::internal::pdf {
@@ -19,15 +20,22 @@ class DocumentParser {
   const FileParser &parser() const;
   const Xref &xref() const;
 
-  IndirectObject read_object(const ObjectReference &reference);
+  const IndirectObject &read_object(const ObjectReference &reference);
   std::string read_object_stream(const ObjectReference &reference);
   std::string read_object_stream(const IndirectObject &object);
 
+  void resolve_object(Object &object);
+  void deep_resolve_object(Object &object);
+
+  Object resolve_object_copy(const Object &object);
+  Object deep_resolve_object_copy(const Object &object);
+
   std::unique_ptr<Document> parse_document();
 
 private:
   FileParser m_parser;
   Xref m_xref;
+  std::map<ObjectReference, IndirectObject> m_objects;
 };
 
 } // namespace odr::internal::pdf

src/odr/internal/pdf/pdf_file_object.cpp

@@ -1,3 +1,13 @@
 #include <odr/internal/pdf/pdf_file_object.hpp>
 
-namespace odr::internal::pdf {}
+namespace odr::internal::pdf {
+
+const ObjectReference &Trailer::root_reference() const {
+  return dictionary["Root"].as_reference();
+}
+
+void Xref::append(const Xref &xref) {
+  table.insert(std::begin(xref.table), std::end(xref.table));
+}
+
+} // namespace odr::internal::pdf

src/odr/internal/pdf/pdf_file_object.hpp

@@ -19,21 +19,22 @@ struct IndirectObject {
 
 struct Trailer {
   std::uint32_t size;
-  ObjectReference root_reference;
-  ObjectReference info_reference;
 
   Dictionary dictionary;
+
+  const ObjectReference &root_reference() const;
 };
 
 struct Xref {
   struct Entry {
     std::uint32_t position{};
-    std::uint32_t generation{};
     bool in_use{};
   };
-  using Table = std::map<std::uint32_t, Entry>;
+  using Table = std::map<ObjectReference, Entry>;
 
   Table table;
+
+  void append(const Xref &xref);
 };
 
 struct StartXref {

src/odr/internal/pdf/pdf_file_parser.cpp

@@ -29,7 +29,7 @@ IndirectObject FileParser::read_indirect_object() const {
   }
 
   result.object = m_parser.read_object();
-  m_parser.skip_line();
+  m_parser.skip_whitespace();
 
   auto next = m_parser.read_line();
 
@@ -49,16 +49,13 @@ IndirectObject FileParser::read_indirect_object() const {
 }
 
 Trailer FileParser::read_trailer() const {
-  if (std::string line = m_parser.read_line(); line != "trailer") {
-    throw std::runtime_error("expected trailer");
-  }
+  m_parser.expect_characters("trailer");
+  m_parser.skip_whitespace();
 
   Trailer result;
 
   result.dictionary = m_parser.read_dictionary();
   result.size = result.dictionary["Size"].as_integer();
-  result.root_reference = result.dictionary["Root"].as_reference();
-  result.info_reference = result.dictionary["Info"].as_reference();
 
   m_parser.skip_line();
   m_parser.skip_whitespace();
@@ -89,11 +86,11 @@ Xref FileParser::read_xref() const {
 
       entry.position = m_parser.read_unsigned_integer();
       m_parser.skip_whitespace();
-      entry.generation = m_parser.read_unsigned_integer();
+      std::uint64_t generation = m_parser.read_unsigned_integer();
       m_parser.skip_whitespace();
       entry.in_use = m_parser.read_line().at(0) == 'n';
 
-      result.table.emplace(first_id + i, std::move(entry));
+      result.table.emplace(ObjectReference(first_id + i, generation), entry);
     }
   }
 }