fix: respect ByteStream encoding in HTMLToDocument and MarkdownToDocument

haystack/components/converters/html.py

@@ -35,7 +35,9 @@ class HTMLToDocument:
     ```
     """
 
-    def __init__(self, extraction_kwargs: dict[str, Any] | None = None, store_full_path: bool = False) -> None:
+    def __init__(
+        self, extraction_kwargs: dict[str, Any] | None = None, store_full_path: bool = False, encoding: str = "utf-8"
+    ) -> None:
         """
         Create an HTMLToDocument component.
 
@@ -45,11 +47,15 @@ def __init__(self, extraction_kwargs: dict[str, Any] | None = None, store_full_p
         :param store_full_path:
         If True, the full path of the file is stored in the metadata of the document.
         If False, only the file name is stored.
+        :param encoding:
+            The default encoding to use when converting HTML files. If the encoding is specified in the metadata of a
+            source ByteStream, it overrides this value.
         """
         trafilatura_import.check()
 
         self.extraction_kwargs = extraction_kwargs or {}
         self.store_full_path = store_full_path
+        self.encoding = encoding
 
     def to_dict(self) -> dict[str, Any]:
         """
@@ -58,7 +64,9 @@ def to_dict(self) -> dict[str, Any]:
         :returns:
             Dictionary with serialized data.
         """
-        return default_to_dict(self, extraction_kwargs=self.extraction_kwargs, store_full_path=self.store_full_path)
+        return default_to_dict(
+            self, extraction_kwargs=self.extraction_kwargs, store_full_path=self.store_full_path, encoding=self.encoding
+        )
 
     @classmethod
     def from_dict(cls, data: dict[str, Any]) -> "HTMLToDocument":
@@ -116,7 +124,8 @@ def run(
                 continue
 
             try:
-                text = extract(bytestream.data.decode("utf-8"), **merged_extraction_kwargs)
+                encoding = bytestream.meta.get("encoding", self.encoding)
+                text = extract(bytestream.data.decode(encoding), **merged_extraction_kwargs)
             except Exception as conversion_e:
                 logger.warning(
                     "Failed to extract text from {source}. Skipping it. Error: {error}",

haystack/components/converters/markdown.py

@@ -52,6 +52,7 @@ def __init__(
         table_to_single_line: bool = False,
         progress_bar: bool = True,
         store_full_path: bool = False,
+        encoding: str = "utf-8",
         *,
         extract_frontmatter: bool = False,
     ) -> None:
@@ -65,6 +66,9 @@ def __init__(
         :param store_full_path:
             If True, the full path of the file is stored in the metadata of the document.
             If False, only the file name is stored.
+        :param encoding:
+            The default encoding to use when converting Markdown files. If the encoding is specified in the metadata
+            of a source ByteStream, it overrides this value.
         :param extract_frontmatter:
             If True, YAML frontmatter at the beginning of the Markdown file is
             removed from the document content and added to the document metadata.
@@ -74,6 +78,7 @@ def __init__(
         self.table_to_single_line = table_to_single_line
         self.progress_bar = progress_bar
         self.store_full_path = store_full_path
+        self.encoding = encoding
         self.extract_frontmatter = extract_frontmatter
 
     @component.output_types(documents=list[Document])
@@ -116,7 +121,8 @@ def run(
                 logger.warning("Could not read {source}. Skipping it. Error: {error}", source=source, error=e)
                 continue
             try:
-                file_content = bytestream.data.decode("utf-8")
+                encoding = bytestream.meta.get("encoding", self.encoding)
+                file_content = bytestream.data.decode(encoding)
                 file_content, frontmatter = self._extract_frontmatter(file_content, source)
                 text = parser.render(file_content)
             except Exception as conversion_e:

releasenotes/notes/html-md-converter-encoding-87047137c613d143.yaml

@@ -0,0 +1,8 @@
+---
+fixes:
+  - |
+    ``HTMLToDocument`` and ``MarkdownToDocument`` now accept an ``encoding`` parameter (default
+    ``"utf-8"``) and honour ``ByteStream.meta["encoding"]`` at run time, mirroring the
+    behaviour of ``TextFileToDocument``. Previously both converters hardcoded
+    ``decode("utf-8")``, causing ``UnicodeDecodeError`` for non-UTF-8 sources whose
+    encoding was supplied via ``ByteStream`` metadata.

test/components/converters/test_html_to_document.py

@@ -192,14 +192,44 @@ def test_mixed_sources_run(self, test_files_path):
         for doc in docs:
             assert "Haystack" in doc.content
 
+    def test_bytestream_encoding_from_meta(self):
+        """
+        Test that a non-UTF-8 ByteStream is decoded using the encoding specified in its meta.
+        """
+        # "caf\xe9" is "café" in latin-1; decoding as utf-8 would raise UnicodeDecodeError.
+        latin1_html = b"<html><body><p>caf\xe9</p></body></html>"
+        bytestream = ByteStream(data=latin1_html, meta={"encoding": "latin-1"})
+
+        converter = HTMLToDocument()
+        results = converter.run(sources=[bytestream])
+        docs = results["documents"]
+
+        assert len(docs) == 1
+        assert "café" in docs[0].content
+
+    def test_bytestream_encoding_from_init(self):
+        """
+        Test that the encoding passed to __init__ is used as a fallback when not set in ByteStream meta.
+        """
+        latin1_html = b"<html><body><p>caf\xe9</p></body></html>"
+        bytestream = ByteStream(data=latin1_html)
+
+        converter = HTMLToDocument(encoding="latin-1")
+        results = converter.run(sources=[bytestream])
+        docs = results["documents"]
+
+        assert len(docs) == 1
+        assert "café" in docs[0].content
+
     def test_serde(self):
         """
         Test if the component runs correctly gets serialized and deserialized.
         """
-        converter = HTMLToDocument()
+        converter = HTMLToDocument(encoding="latin-1")
         serde_data = converter.to_dict()
         new_converter = HTMLToDocument.from_dict(serde_data)
         assert new_converter.extraction_kwargs == converter.extraction_kwargs
+        assert new_converter.encoding == converter.encoding
 
     def test_run_difficult_html(self, test_files_path):
         converter = HTMLToDocument()

test/components/converters/test_markdown_to_document.py

@@ -16,6 +16,7 @@ def test_init_params_default(self):
         converter = MarkdownToDocument()
         assert converter.table_to_single_line is False
         assert converter.progress_bar is True
+        assert converter.encoding == "utf-8"
         assert converter.extract_frontmatter is False
 
     def test_init_params_custom(self):
@@ -201,3 +202,32 @@ def test_mixed_sources_run(self, test_files_path):
         for doc in docs:
             assert "What to build with Haystack" in doc.content
             assert "# git clone https://github.com/deepset-ai/haystack.git" in doc.content
+
+    def test_bytestream_encoding_from_meta(self):
+        """
+        Test that a non-UTF-8 ByteStream is decoded using the encoding specified in its meta.
+        """
+        # "caf\xe9" is "café" in latin-1; decoding as utf-8 would raise UnicodeDecodeError.
+        latin1_md = "# caf\xe9".encode("latin-1")
+        bytestream = ByteStream(data=latin1_md, meta={"encoding": "latin-1"})
+
+        converter = MarkdownToDocument(progress_bar=False)
+        output = converter.run(sources=[bytestream])
+        docs = output["documents"]
+
+        assert len(docs) == 1
+        assert "café" in docs[0].content
+
+    def test_bytestream_encoding_from_init(self):
+        """
+        Test that the encoding passed to __init__ is used as a fallback when not set in ByteStream meta.
+        """
+        latin1_md = "# caf\xe9".encode("latin-1")
+        bytestream = ByteStream(data=latin1_md)
+
+        converter = MarkdownToDocument(encoding="latin-1", progress_bar=False)
+        output = converter.run(sources=[bytestream])
+        docs = output["documents"]
+
+        assert len(docs) == 1
+        assert "café" in docs[0].content