diff --git a/haystack/components/converters/html.py b/haystack/components/converters/html.py
index f4770ba0c36..35a702f4c04 100644
--- a/haystack/components/converters/html.py
+++ b/haystack/components/converters/html.py
@@ -35,7 +35,9 @@ class HTMLToDocument:
```
"""
- def __init__(self, extraction_kwargs: dict[str, Any] | None = None, store_full_path: bool = False) -> None:
+ def __init__(
+ self, extraction_kwargs: dict[str, Any] | None = None, store_full_path: bool = False, encoding: str = "utf-8"
+ ) -> None:
"""
Create an HTMLToDocument component.
@@ -45,11 +47,15 @@ def __init__(self, extraction_kwargs: dict[str, Any] | None = None, store_full_p
:param store_full_path:
If True, the full path of the file is stored in the metadata of the document.
If False, only the file name is stored.
+ :param encoding:
+ The default encoding to use when converting HTML files. If the encoding is specified in the metadata of a
+ source ByteStream, it overrides this value.
"""
trafilatura_import.check()
self.extraction_kwargs = extraction_kwargs or {}
self.store_full_path = store_full_path
+ self.encoding = encoding
def to_dict(self) -> dict[str, Any]:
"""
@@ -58,7 +64,9 @@ def to_dict(self) -> dict[str, Any]:
:returns:
Dictionary with serialized data.
"""
- return default_to_dict(self, extraction_kwargs=self.extraction_kwargs, store_full_path=self.store_full_path)
+ return default_to_dict(
+ self, extraction_kwargs=self.extraction_kwargs, store_full_path=self.store_full_path, encoding=self.encoding
+ )
@classmethod
def from_dict(cls, data: dict[str, Any]) -> "HTMLToDocument":
@@ -116,7 +124,8 @@ def run(
continue
try:
- text = extract(bytestream.data.decode("utf-8"), **merged_extraction_kwargs)
+ encoding = bytestream.meta.get("encoding", self.encoding)
+ text = extract(bytestream.data.decode(encoding), **merged_extraction_kwargs)
except Exception as conversion_e:
logger.warning(
"Failed to extract text from {source}. Skipping it. Error: {error}",
diff --git a/haystack/components/converters/markdown.py b/haystack/components/converters/markdown.py
index d7f15bbc5c3..ffcd5f78fba 100644
--- a/haystack/components/converters/markdown.py
+++ b/haystack/components/converters/markdown.py
@@ -52,6 +52,7 @@ def __init__(
table_to_single_line: bool = False,
progress_bar: bool = True,
store_full_path: bool = False,
+ encoding: str = "utf-8",
*,
extract_frontmatter: bool = False,
) -> None:
@@ -65,6 +66,9 @@ def __init__(
:param store_full_path:
If True, the full path of the file is stored in the metadata of the document.
If False, only the file name is stored.
+ :param encoding:
+ The default encoding to use when converting Markdown files. If the encoding is specified in the metadata
+ of a source ByteStream, it overrides this value.
:param extract_frontmatter:
If True, YAML frontmatter at the beginning of the Markdown file is
removed from the document content and added to the document metadata.
@@ -74,6 +78,7 @@ def __init__(
self.table_to_single_line = table_to_single_line
self.progress_bar = progress_bar
self.store_full_path = store_full_path
+ self.encoding = encoding
self.extract_frontmatter = extract_frontmatter
@component.output_types(documents=list[Document])
@@ -116,7 +121,8 @@ def run(
logger.warning("Could not read {source}. Skipping it. Error: {error}", source=source, error=e)
continue
try:
- file_content = bytestream.data.decode("utf-8")
+ encoding = bytestream.meta.get("encoding", self.encoding)
+ file_content = bytestream.data.decode(encoding)
file_content, frontmatter = self._extract_frontmatter(file_content, source)
text = parser.render(file_content)
except Exception as conversion_e:
diff --git a/releasenotes/notes/html-md-converter-encoding-87047137c613d143.yaml b/releasenotes/notes/html-md-converter-encoding-87047137c613d143.yaml
new file mode 100644
index 00000000000..2e655bc6a5c
--- /dev/null
+++ b/releasenotes/notes/html-md-converter-encoding-87047137c613d143.yaml
@@ -0,0 +1,8 @@
+---
+fixes:
+ - |
+ ``HTMLToDocument`` and ``MarkdownToDocument`` now accept an ``encoding`` parameter (default
+ ``"utf-8"``) and honour ``ByteStream.meta["encoding"]`` at run time, mirroring the
+ behaviour of ``TextFileToDocument``. Previously both converters hardcoded
+ ``decode("utf-8")``, causing ``UnicodeDecodeError`` for non-UTF-8 sources whose
+ encoding was supplied via ``ByteStream`` metadata.
diff --git a/test/components/converters/test_html_to_document.py b/test/components/converters/test_html_to_document.py
index b32509ce4b4..53ca7a1b106 100644
--- a/test/components/converters/test_html_to_document.py
+++ b/test/components/converters/test_html_to_document.py
@@ -192,14 +192,44 @@ def test_mixed_sources_run(self, test_files_path):
for doc in docs:
assert "Haystack" in doc.content
+ def test_bytestream_encoding_from_meta(self):
+ """
+ Test that a non-UTF-8 ByteStream is decoded using the encoding specified in its meta.
+ """
+ # "caf\xe9" is "café" in latin-1; decoding as utf-8 would raise UnicodeDecodeError.
+ latin1_html = b"caf\xe9
"
+ bytestream = ByteStream(data=latin1_html, meta={"encoding": "latin-1"})
+
+ converter = HTMLToDocument()
+ results = converter.run(sources=[bytestream])
+ docs = results["documents"]
+
+ assert len(docs) == 1
+ assert "café" in docs[0].content
+
+ def test_bytestream_encoding_from_init(self):
+ """
+ Test that the encoding passed to __init__ is used as a fallback when not set in ByteStream meta.
+ """
+ latin1_html = b"caf\xe9
"
+ bytestream = ByteStream(data=latin1_html)
+
+ converter = HTMLToDocument(encoding="latin-1")
+ results = converter.run(sources=[bytestream])
+ docs = results["documents"]
+
+ assert len(docs) == 1
+ assert "café" in docs[0].content
+
def test_serde(self):
"""
Test if the component runs correctly gets serialized and deserialized.
"""
- converter = HTMLToDocument()
+ converter = HTMLToDocument(encoding="latin-1")
serde_data = converter.to_dict()
new_converter = HTMLToDocument.from_dict(serde_data)
assert new_converter.extraction_kwargs == converter.extraction_kwargs
+ assert new_converter.encoding == converter.encoding
def test_run_difficult_html(self, test_files_path):
converter = HTMLToDocument()
diff --git a/test/components/converters/test_markdown_to_document.py b/test/components/converters/test_markdown_to_document.py
index a427d7c6550..19a86e8d58b 100644
--- a/test/components/converters/test_markdown_to_document.py
+++ b/test/components/converters/test_markdown_to_document.py
@@ -16,6 +16,7 @@ def test_init_params_default(self):
converter = MarkdownToDocument()
assert converter.table_to_single_line is False
assert converter.progress_bar is True
+ assert converter.encoding == "utf-8"
assert converter.extract_frontmatter is False
def test_init_params_custom(self):
@@ -201,3 +202,32 @@ def test_mixed_sources_run(self, test_files_path):
for doc in docs:
assert "What to build with Haystack" in doc.content
assert "# git clone https://github.com/deepset-ai/haystack.git" in doc.content
+
+ def test_bytestream_encoding_from_meta(self):
+ """
+ Test that a non-UTF-8 ByteStream is decoded using the encoding specified in its meta.
+ """
+ # "caf\xe9" is "café" in latin-1; decoding as utf-8 would raise UnicodeDecodeError.
+ latin1_md = "# caf\xe9".encode("latin-1")
+ bytestream = ByteStream(data=latin1_md, meta={"encoding": "latin-1"})
+
+ converter = MarkdownToDocument(progress_bar=False)
+ output = converter.run(sources=[bytestream])
+ docs = output["documents"]
+
+ assert len(docs) == 1
+ assert "café" in docs[0].content
+
+ def test_bytestream_encoding_from_init(self):
+ """
+ Test that the encoding passed to __init__ is used as a fallback when not set in ByteStream meta.
+ """
+ latin1_md = "# caf\xe9".encode("latin-1")
+ bytestream = ByteStream(data=latin1_md)
+
+ converter = MarkdownToDocument(encoding="latin-1", progress_bar=False)
+ output = converter.run(sources=[bytestream])
+ docs = output["documents"]
+
+ assert len(docs) == 1
+ assert "café" in docs[0].content