First PDF parsing (#345)

Author: andiwand

CMakeLists.txt

@@ -146,6 +146,17 @@ add_library(odr
         "src/odr/internal/ooxml/ooxml_meta.cpp"
         "src/odr/internal/ooxml/ooxml_util.cpp"
 
+        "src/odr/internal/pdf/pdf_document.cpp"
+        "src/odr/internal/pdf/pdf_document_element.cpp"
+        "src/odr/internal/pdf/pdf_document_parser.cpp"
+        "src/odr/internal/pdf/pdf_object.cpp"
+        "src/odr/internal/pdf/pdf_file_parser.cpp"
+        "src/odr/internal/pdf/pdf_graphics_operator.cpp"
+        "src/odr/internal/pdf/pdf_graphics_operator_parser.cpp"
+        "src/odr/internal/pdf/pdf_graphics_state.cpp"
+        "src/odr/internal/pdf/pdf_object.cpp"
+        "src/odr/internal/pdf/pdf_object_parser.cpp"
+
         "src/odr/internal/svm/svm_file.cpp"
         "src/odr/internal/svm/svm_format.cpp"
         "src/odr/internal/svm/svm_to_svg.cpp"

src/odr/internal/crypto/crypto_util.cpp

@@ -11,6 +11,7 @@
 #include <cryptopp/pwdbased.h>
 #include <cryptopp/sha.h>
 #include <cryptopp/zinflate.h>
+#include <cryptopp/zlib.h>
 
 namespace odr::internal::crypto {
 
@@ -146,4 +147,12 @@ std::size_t util::padding(const std::string &input) {
   return inflator.GetPadding();
 }
 
+std::string util::zlib_inflate(const std::string &input) {
+  std::string result;
+  CryptoPP::ZlibDecompressor inflator(new CryptoPP::StringSink(result));
+  inflator.Put(reinterpret_cast<const byte *>(input.data()), input.size());
+  inflator.MessageEnd();
+  return result;
+}
+
 } // namespace odr::internal::crypto

src/odr/internal/crypto/crypto_util.hpp

@@ -29,6 +29,8 @@ std::string decrypt_Blowfish(const std::string &key, const std::string &iv,
 std::string inflate(const std::string &input);
 std::size_t padding(const std::string &input);
 
+std::string zlib_inflate(const std::string &input);
+
 } // namespace util
 
 } // namespace odr::internal::crypto

src/odr/internal/html/text_file.cpp

@@ -56,7 +56,7 @@ Html html::translate_text_file(const TextFile &text_file,
     out.write_element_begin("td");
 
     std::ostringstream ss_out;
-    util::stream::getline(*in, ss_out);
+    util::stream::pipe_line(*in, ss_out);
     out.out() << escape_text(ss_out.str());
 
     out.write_element_end("td");

src/odr/internal/pdf/README.md

@@ -0,0 +1,53 @@
+# PDF
+
+This is in a very early state and not tested thoroughly.
+
+It includes:
+- Reading PDF data types from stream.
+- Opening a PDF file and extracting file objects.
+  - Everything is done via streams but some parts require seeking in the file.
+  - **What is missing:**
+    - Support for linearized files
+    - Support for cross-reference stream
+    - Updated files?
+- Extracting information from the PDF file objects.
+  - We can construct a page tree and link to annotations and resources.
+  - **What is missing:**
+    - Properly parse annotation information
+    - Properly parse font information
+
+## Discussion: HTML translation
+
+The primer problem of translating PDF to HTML will be to map glyph IDs to Unicode characters.
+There can be mapping information in the PDF and in the font but both is not guaranteed.
+HTML and SVG do not support drawing characters based on glyph IDs, so we are forced to use Unicode mappings.
+If there is no Unicode mapping in the font we would have to transform the font and make up mappings.
+This would require the ability of reading and writing font files which may not be trivial and there are a couple of different formats with different versions.
+
+Ultimately it also depends on how often this really happens that we do not have any Unicode mappings at all.
+
+Another problem might be the font support from the browser. Googling suggests that not all formats are supported by popular browsers.
+
+After all this is solved the questions is if we can place text the same way in HTML as it is done in PDF.
+PDF has the ability to continue text where it left of or move down a line which could be tricky in HTML.
+Apart from that PDF can also modify the spacing between the characters for example which we might be able to translate to CSS positioning.
+
+## References
+- https://en.wikipedia.org/wiki/PDF
+- File Format
+  - Objects https://www.oreilly.com/library/view/developing-with-pdf/9781449327903/ch01.html
+  - https://resources.infosecinstitute.com/topics/hacking/pdf-file-format-basic-structure/
+- Structure
+  - Document Structure https://www.oreilly.com/library/view/pdf-explained/9781449321581/ch04.html
+  - Cross-reference stream https://www.verypdf.com/document/pdf-format-reference/pg_0106.htm
+- Graphics Operators
+  - https://gendignoux.com/blog/images/pdf-graphics/cheat-sheet-by-nc-sa.png 
+  - https://pdfa.org/wp-content/uploads/2023/08/PDF-Operators-CheatSheet.pdf
+  - General https://gendignoux.com/blog/2017/01/05/pdf-graphics.html
+  - Text https://www.syncfusion.com/succinctly-free-ebooks/pdf/text-operators
+  - Drawing https://www.syncfusion.com/succinctly-free-ebooks/pdf/graphics-operators
+- CMap https://blog.idrsolutions.com/how-are-embedded-cmap-tables-in-pdf-file/
+- Related projects
+  - https://github.com/qpdf/qpdf
+  - https://github.com/caradoc-org/caradoc
+- Nasty test files https://github.com/angea/PDF101

src/odr/internal/pdf/pdf_document.cpp

@@ -0,0 +1,3 @@
+#include <odr/internal/pdf/pdf_document.hpp>
+
+namespace odr::internal::pdf {}

src/odr/internal/pdf/pdf_document.hpp

@@ -0,0 +1,26 @@
+#ifndef ODR_INTERNAL_PDF_DOCUMENT_HPP
+#define ODR_INTERNAL_PDF_DOCUMENT_HPP
+
+#include <memory>
+#include <vector>
+
+namespace odr::internal::pdf {
+
+struct Catalog;
+struct Element;
+
+struct Document {
+  Catalog *catalog;
+  std::vector<std::unique_ptr<Element>> elements;
+
+  template <typename T, typename... Args> T *create_element(Args &&...args) {
+    auto unique = std::make_unique<T>(std::forward<Args>(args)...);
+    auto pointer = unique.get();
+    elements.push_back(std::move(unique));
+    return pointer;
+  }
+};
+
+} // namespace odr::internal::pdf
+
+#endif // ODR_INTERNAL_PDF_DOCUMENT_HPP

src/odr/internal/pdf/pdf_document_element.cpp

@@ -0,0 +1,3 @@
+#include <odr/internal/pdf/pdf_document_element.hpp>
+
+namespace odr::internal::pdf {}

src/odr/internal/pdf/pdf_document_element.hpp

@@ -0,0 +1,64 @@
+#ifndef ODR_INTERNAL_PDF_DOCUMENT_ELEMENT_HPP
+#define ODR_INTERNAL_PDF_DOCUMENT_ELEMENT_HPP
+
+#include <odr/internal/pdf/pdf_object.hpp>
+
+#include <unordered_map>
+#include <vector>
+
+namespace odr::internal::pdf {
+
+struct Pages;
+struct Page;
+struct Annotation;
+struct Resources;
+struct Font;
+
+enum class Type {
+  unknown,
+  catalog,
+  pages,
+  page,
+  annotation,
+  resources,
+  font,
+};
+
+struct Element {
+  virtual ~Element() = default;
+
+  Type type{Type::unknown};
+  ObjectReference object_reference;
+  Object object;
+};
+
+struct Catalog : Element {
+  Pages *pages{nullptr};
+};
+
+struct Pages : Element {
+  std::vector<Element *> kids;
+  std::uint32_t count{0};
+};
+
+struct Page : Element {
+  Pages *parent{nullptr};
+
+  Resources *resources;
+  std::vector<Annotation *> annotations;
+
+  // TODO remove
+  ObjectReference contents_reference;
+};
+
+struct Annotation : Element {};
+
+struct Resources : Element {
+  std::unordered_map<std::string, Font *> font;
+};
+
+struct Font : Element {};
+
+} // namespace odr::internal::pdf
+
+#endif // ODR_INTERNAL_PDF_DOCUMENT_ELEMENT_HPP