From 4d9082be92d11d11e82c14030073ee70c64fdb53 Mon Sep 17 00:00:00 2001 From: Praveen Mittal Date: Fri, 31 Jul 2026 09:52:48 +0200 Subject: [PATCH] fix: JSONConverter raises KeyError on ByteStream sources without file_path All three exception handlers in _get_content_and_meta() (invalid UTF-8, a failing jq_schema filter, malformed JSON) log a warning using source.meta["file_path"] -- an unconditional dict-key access. A bare ByteStream (e.g. ByteStream.from_string(...), the exact pattern shown in this component's own docstring examples) has no such key, so the error-handling code itself raised KeyError, masking the real error and crashing run() instead of skipping that one source as intended. Fixed by using source.meta.get("file_path", "unknown") at all three sites. --- haystack/components/converters/json.py | 6 +-- ...-bytestream-keyerror-9c55c4ccd0ccca4c.yaml | 9 ++++ test/components/converters/test_json.py | 45 +++++++++++++++++++ 3 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 releasenotes/notes/fix-json-converter-bytestream-keyerror-9c55c4ccd0ccca4c.yaml diff --git a/haystack/components/converters/json.py b/haystack/components/converters/json.py index 9dd79bd3314..102137d1dc5 100644 --- a/haystack/components/converters/json.py +++ b/haystack/components/converters/json.py @@ -191,7 +191,7 @@ def _get_content_and_meta(self, source: ByteStream) -> list[tuple[str, dict[str, except UnicodeError as exc: logger.warning( "Failed to extract text from {source}. Skipping it. Error: {error}", - source=source.meta["file_path"], + source=source.meta.get("file_path", "unknown"), error=exc, ) return [] @@ -204,7 +204,7 @@ def _get_content_and_meta(self, source: ByteStream) -> list[tuple[str, dict[str, except Exception as exc: logger.warning( "Failed to extract text from {source}. Skipping it. Error: {error}", - source=source.meta["file_path"], + source=source.meta.get("file_path", "unknown"), error=exc, ) return [] @@ -216,7 +216,7 @@ def _get_content_and_meta(self, source: ByteStream) -> list[tuple[str, dict[str, except json.JSONDecodeError as exc: logger.warning( "Failed to extract text from {source}. Skipping it. Error: {error}", - source=source.meta["file_path"], + source=source.meta.get("file_path", "unknown"), error=exc, ) return [] diff --git a/releasenotes/notes/fix-json-converter-bytestream-keyerror-9c55c4ccd0ccca4c.yaml b/releasenotes/notes/fix-json-converter-bytestream-keyerror-9c55c4ccd0ccca4c.yaml new file mode 100644 index 00000000000..573302b37be --- /dev/null +++ b/releasenotes/notes/fix-json-converter-bytestream-keyerror-9c55c4ccd0ccca4c.yaml @@ -0,0 +1,9 @@ +--- +fixes: + - | + Fixed ``JSONConverter`` raising a ``KeyError`` instead of logging its + intended "Failed to extract text, skipping it" warning when a source is a + ``ByteStream`` without a ``file_path`` in its ``meta`` (for example + ``ByteStream.from_string(...)``, the exact usage shown in the component's + own docstring examples). Affected error paths: invalid UTF-8 content, a + ``jq_schema`` filter that fails to apply, and malformed JSON content. diff --git a/test/components/converters/test_json.py b/test/components/converters/test_json.py index 3d55a89eb14..a2c538ec9cc 100644 --- a/test/components/converters/test_json.py +++ b/test/components/converters/test_json.py @@ -275,6 +275,51 @@ def test_run_with_malformed_json_and_content_key(tmpdir, caplog): assert result == {"documents": []} +def test_run_with_bad_filter_and_bytestream_without_file_path(caplog): + """A bare ByteStream source (no 'file_path' in its meta) must not raise a KeyError.""" + source = ByteStream(data=json.dumps(test_data[0]).encode("utf-8")) + converter = JSONConverter(".laureates | .motivation") + + caplog.clear() + with caplog.at_level(logging.WARNING): + result = converter.run(sources=[source]) + + records = caplog.records + assert len(records) == 1 + assert records[0].msg.startswith("Failed to extract text from unknown. Skipping it. Error:") + assert result == {"documents": []} + + +def test_run_with_bad_encoding_and_bytestream_without_file_path(caplog): + """A bare ByteStream source (no 'file_path' in its meta) must not raise a KeyError.""" + source = ByteStream(data=json.dumps(test_data[0]).encode("utf-16")) + converter = JSONConverter(".laureates") + + caplog.clear() + with caplog.at_level(logging.WARNING): + result = converter.run(sources=[source]) + + records = caplog.records + assert len(records) == 1 + assert records[0].msg.startswith("Failed to extract text from unknown. Skipping it. Error:") + assert result == {"documents": []} + + +def test_run_with_malformed_json_and_bytestream_without_file_path(caplog): + """A bare ByteStream source (no 'file_path' in its meta) must not raise a KeyError.""" + source = ByteStream(data=b"This is not valid JSON.") + converter = JSONConverter(content_key="motivation") + + caplog.clear() + with caplog.at_level(logging.WARNING): + result = converter.run(sources=[source]) + + records = caplog.records + assert len(records) == 1 + assert records[0].msg.startswith("Failed to extract text from unknown. Skipping it. Error:") + assert result == {"documents": []} + + def test_run_with_single_meta(tmpdir): first_test_file = Path(tmpdir / "first_test_file.json") second_test_file = Path(tmpdir / "second_test_file.json")