Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 1 addition & 5 deletions haystack/components/converters/file_to_file_content.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,12 +82,8 @@ def run(
continue

base64_data = base64.b64encode(bytestream.data).decode("utf-8")
# ``normalize_metadata`` returns the same dict object for every source when ``extra`` is a
# single dict (or ``None``), so give each FileContent its own copy. Otherwise mutating one
# file's ``extra`` downstream would leak into all the others. The other converters avoid this
# implicitly by merging ``extra`` into a fresh ``{**bytestream.meta, ...}`` dict.
file_content = FileContent(
base64_data=base64_data, mime_type=bytestream.mime_type, filename=filename, extra=dict(extra_dict)
base64_data=base64_data, mime_type=bytestream.mime_type, filename=filename, extra=extra_dict
)
file_contents.append(file_content)

Expand Down
9 changes: 7 additions & 2 deletions haystack/components/converters/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0

from copy import deepcopy
from pathlib import Path
from typing import Any

Expand Down Expand Up @@ -39,11 +40,15 @@ def normalize_metadata(meta: dict[str, Any] | list[dict[str, Any]] | None, sourc
:param meta: the meta input of the converter, as-is
:param sources_count: the number of sources the converter received
:returns: a list of dictionaries of the make length as the sources list

Each source always gets its own independent dictionary. When ``meta`` is ``None`` or a single
dictionary, a separate copy is returned for every source so that mutating one source's metadata
downstream does not leak into the others.
"""
if meta is None:
return [{}] * sources_count
return [{} for _ in range(sources_count)]
if isinstance(meta, dict):
return [meta] * sources_count
return [deepcopy(meta) for _ in range(sources_count)]
if isinstance(meta, list):
if sources_count != len(meta):
raise ValueError("The length of the metadata list must match the number of sources.")
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
---
fixes:
- |
Fixed ``normalize_metadata`` (used by all file converters) returning the same dictionary object
for every source when ``meta`` is ``None`` or a single dictionary. Each source now receives an
independent copy, so mutating one source's metadata downstream no longer leaks into the others.
15 changes: 15 additions & 0 deletions test/components/converters/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,21 @@ def test_normalize_metadata_single_dict():
assert normalize_metadata({"a": 1}, sources_count=3) == [{"a": 1}, {"a": 1}, {"a": 1}]


def test_normalize_metadata_none_returns_independent_dicts():
result = normalize_metadata(None, sources_count=3)
result[0]["file_path"] = "a.txt"
assert result == [{"file_path": "a.txt"}, {}, {}]


def test_normalize_metadata_single_dict_returns_independent_copies():
meta = {"a": 1}
result = normalize_metadata(meta, sources_count=3)
result[0]["b"] = 2
# Mutating one source's metadata must not leak into the others or the original input.
assert result == [{"a": 1, "b": 2}, {"a": 1}, {"a": 1}]
assert meta == {"a": 1}


def test_normalize_metadata_list_of_right_size():
assert normalize_metadata([{"a": 1}], sources_count=1) == [{"a": 1}]
assert normalize_metadata([{"a": 1}, {"b": 2}, {"c": 3}], sources_count=3) == [{"a": 1}, {"b": 2}, {"c": 3}]
Expand Down