Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions haystack/components/converters/json.py
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ def _get_content_and_meta(self, source: ByteStream) -> list[tuple[str, dict[str,
except UnicodeError as exc:
logger.warning(
"Failed to extract text from {source}. Skipping it. Error: {error}",
source=source.meta["file_path"],
source=source.meta.get("file_path", "unknown"),
error=exc,
)
return []
Expand All @@ -204,7 +204,7 @@ def _get_content_and_meta(self, source: ByteStream) -> list[tuple[str, dict[str,
except Exception as exc:
logger.warning(
"Failed to extract text from {source}. Skipping it. Error: {error}",
source=source.meta["file_path"],
source=source.meta.get("file_path", "unknown"),
error=exc,
)
return []
Expand All @@ -216,7 +216,7 @@ def _get_content_and_meta(self, source: ByteStream) -> list[tuple[str, dict[str,
except json.JSONDecodeError as exc:
logger.warning(
"Failed to extract text from {source}. Skipping it. Error: {error}",
source=source.meta["file_path"],
source=source.meta.get("file_path", "unknown"),
error=exc,
)
return []
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
---
fixes:
- |
Fixed ``JSONConverter`` raising a ``KeyError`` instead of logging its
intended "Failed to extract text, skipping it" warning when a source is a
``ByteStream`` without a ``file_path`` in its ``meta`` (for example
``ByteStream.from_string(...)``, the exact usage shown in the component's
own docstring examples). Affected error paths: invalid UTF-8 content, a
``jq_schema`` filter that fails to apply, and malformed JSON content.
45 changes: 45 additions & 0 deletions test/components/converters/test_json.py
Original file line number Diff line number Diff line change
Expand Up @@ -275,6 +275,51 @@ def test_run_with_malformed_json_and_content_key(tmpdir, caplog):
assert result == {"documents": []}


def test_run_with_bad_filter_and_bytestream_without_file_path(caplog):
"""A bare ByteStream source (no 'file_path' in its meta) must not raise a KeyError."""
source = ByteStream(data=json.dumps(test_data[0]).encode("utf-8"))
converter = JSONConverter(".laureates | .motivation")

caplog.clear()
with caplog.at_level(logging.WARNING):
result = converter.run(sources=[source])

records = caplog.records
assert len(records) == 1
assert records[0].msg.startswith("Failed to extract text from unknown. Skipping it. Error:")
assert result == {"documents": []}


def test_run_with_bad_encoding_and_bytestream_without_file_path(caplog):
"""A bare ByteStream source (no 'file_path' in its meta) must not raise a KeyError."""
source = ByteStream(data=json.dumps(test_data[0]).encode("utf-16"))
converter = JSONConverter(".laureates")

caplog.clear()
with caplog.at_level(logging.WARNING):
result = converter.run(sources=[source])

records = caplog.records
assert len(records) == 1
assert records[0].msg.startswith("Failed to extract text from unknown. Skipping it. Error:")
assert result == {"documents": []}


def test_run_with_malformed_json_and_bytestream_without_file_path(caplog):
"""A bare ByteStream source (no 'file_path' in its meta) must not raise a KeyError."""
source = ByteStream(data=b"This is not valid JSON.")
converter = JSONConverter(content_key="motivation")

caplog.clear()
with caplog.at_level(logging.WARNING):
result = converter.run(sources=[source])

records = caplog.records
assert len(records) == 1
assert records[0].msg.startswith("Failed to extract text from unknown. Skipping it. Error:")
assert result == {"documents": []}


def test_run_with_single_meta(tmpdir):
first_test_file = Path(tmpdir / "first_test_file.json")
second_test_file = Path(tmpdir / "second_test_file.json")
Expand Down