From 78166807b18bc13895565aabdb5bd0a9c12ac058 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-S=C3=A9bastien=20=7B=7BEbaAaZ=7D=7D?= Date: Mon, 18 May 2026 00:04:20 -0400 Subject: [PATCH 1/3] feat: expose LLM evaluator row statuses --- .../components/evaluators/llm_evaluator.py | 8 +++++-- .../test_context_relevance_evaluator.py | 3 +++ .../evaluators/test_faithfulness_evaluator.py | 3 +++ .../evaluators/test_llm_evaluator.py | 24 ++++++++++++++++++- 4 files changed, 35 insertions(+), 3 deletions(-) diff --git a/haystack/components/evaluators/llm_evaluator.py b/haystack/components/evaluators/llm_evaluator.py index cd4bebedf12..e0de7658a07 100644 --- a/haystack/components/evaluators/llm_evaluator.py +++ b/haystack/components/evaluators/llm_evaluator.py @@ -175,7 +175,7 @@ def validate_init_parameters( ) raise ValueError(msg) - @component.output_types(results=list[dict[str, Any]]) + @component.output_types(results=list[dict[str, Any]], evaluation_statuses=list[str]) def run(self, **inputs: Any) -> dict[str, Any]: """ Run the LLM evaluator. @@ -204,6 +204,7 @@ def run(self, **inputs: Any) -> dict[str, Any]: list_of_input_names_to_values = [dict(zip(input_names, v, strict=True)) for v in values] results: list[dict[str, Any] | None] = [] + evaluation_statuses: list[str] = [] metadata = [] errors = 0 for input_names_to_values in tqdm(list_of_input_names_to_values, disable=not self.progress_bar): @@ -216,6 +217,7 @@ def run(self, **inputs: Any) -> dict[str, Any]: raise ValueError(f"Error while generating response for prompt: {prompt}. Error: {e}") from e logger.warning("Error while generating response for prompt: {prompt}. Error: {e}", prompt=prompt, e=e) results.append(None) + evaluation_statuses.append("error") errors += 1 continue @@ -224,9 +226,11 @@ def run(self, **inputs: Any) -> dict[str, Any]: ) if parsed_result is None: results.append(None) + evaluation_statuses.append("error") errors += 1 else: results.append(parsed_result) + evaluation_statuses.append("evaluated") if result["replies"][0].meta: metadata.append(result["replies"][0].meta) @@ -238,7 +242,7 @@ def run(self, **inputs: Any) -> dict[str, Any]: len=len(list_of_input_names_to_values), ) - return {"results": results, "meta": metadata or None} + return {"results": results, "meta": metadata or None, "evaluation_statuses": evaluation_statuses} def prepare_template(self) -> str: """ diff --git a/test/components/evaluators/test_context_relevance_evaluator.py b/test/components/evaluators/test_context_relevance_evaluator.py index 9eca6988795..a37748a8ed4 100644 --- a/test/components/evaluators/test_context_relevance_evaluator.py +++ b/test/components/evaluators/test_context_relevance_evaluator.py @@ -169,6 +169,7 @@ def chat_generator_run(self, *args, **kwargs): "score": 0.5, "meta": None, "individual_scores": [1, 0], + "evaluation_statuses": ["evaluated", "evaluated"], } def test_run_no_statements_extracted(self, monkeypatch): @@ -198,6 +199,7 @@ def chat_generator_run(self, *args, **kwargs): "score": 0.5, "meta": None, "individual_scores": [1, 0], + "evaluation_statuses": ["evaluated", "evaluated"], } def test_run_missing_parameters(self, monkeypatch): @@ -237,6 +239,7 @@ def chat_generator_run(self, *args, **kwargs): assert results["results"][0] == {"relevant_statements": ["c", "d"], "score": 1} assert results["results"][1]["relevant_statements"] == [] assert math.isnan(results["results"][1]["score"]) + assert results["evaluation_statuses"] == ["evaluated", "error"] @pytest.mark.skipif( not os.environ.get("OPENAI_API_KEY", None), diff --git a/test/components/evaluators/test_faithfulness_evaluator.py b/test/components/evaluators/test_faithfulness_evaluator.py index 64d113462a8..77934384af3 100644 --- a/test/components/evaluators/test_faithfulness_evaluator.py +++ b/test/components/evaluators/test_faithfulness_evaluator.py @@ -207,6 +207,7 @@ def chat_generator_run(self, *args, **kwargs): ], "score": 0.75, "meta": None, + "evaluation_statuses": ["evaluated", "evaluated"], } def test_run_no_statements_extracted(self, monkeypatch): @@ -245,6 +246,7 @@ def chat_generator_run(self, *args, **kwargs): ], "score": 0.25, "meta": None, + "evaluation_statuses": ["evaluated", "evaluated"], } def test_run_missing_parameters(self, monkeypatch): @@ -294,6 +296,7 @@ def chat_generator_run(self, *args, **kwargs): assert results["results"][1]["statements"] == [] assert results["results"][1]["statement_scores"] == [] assert math.isnan(results["results"][1]["score"]) + assert results["evaluation_statuses"] == ["evaluated", "error"] @pytest.mark.skipif( not os.environ.get("OPENAI_API_KEY", None), diff --git a/test/components/evaluators/test_llm_evaluator.py b/test/components/evaluators/test_llm_evaluator.py index a9573277cd8..2084ff5174b 100644 --- a/test/components/evaluators/test_llm_evaluator.py +++ b/test/components/evaluators/test_llm_evaluator.py @@ -332,7 +332,7 @@ def chat_generator_run(self, *args, **kwargs): monkeypatch.setattr("haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run", chat_generator_run) results = component.run(questions=["What is the capital of Germany?"], predicted_answers=["Berlin"]) - assert results == {"results": [{"score": 0.5}], "meta": None} + assert results == {"results": [{"score": 0.5}], "meta": None, "evaluation_statuses": ["evaluated"]} def test_prepare_template(self, monkeypatch): monkeypatch.setenv("OPENAI_API_KEY", "test-api-key") @@ -433,6 +433,28 @@ def chat_generator_run(self, *args, **kwargs): result = component.run(predicted_answers=["answer"]) assert result["results"] == [None] + assert result["evaluation_statuses"] == ["error"] + + def test_run_returns_error_status_raise_on_failure_false(self, monkeypatch): + monkeypatch.setenv("OPENAI_API_KEY", "test-api-key") + component = LLMEvaluator( + instructions="test-instruction", + inputs=[("predicted_answers", list[str])], + outputs=["score"], + examples=[ + {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}} + ], + raise_on_failure=False, + ) + + def chat_generator_run(self, *args, **kwargs): + raise Exception("OpenAI API request failed.") + + monkeypatch.setattr("haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run", chat_generator_run) + + result = component.run(predicted_answers=["answer"]) + assert result["results"] == [None] + assert result["evaluation_statuses"] == ["error"] def test_output_invalid_json_raise_on_failure_true(self, monkeypatch): monkeypatch.setenv("OPENAI_API_KEY", "test-api-key") From b281672ec8228415e8c410cf7a84dd7d1f033703 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-S=C3=A9bastien=20=7B=7BEbaAaZ=7D=7D?= Date: Mon, 18 May 2026 00:10:09 -0400 Subject: [PATCH 2/3] chore: add evaluator status release note --- .../add-llm-evaluator-row-statuses-78166807b18bc138.yaml | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 releasenotes/notes/add-llm-evaluator-row-statuses-78166807b18bc138.yaml diff --git a/releasenotes/notes/add-llm-evaluator-row-statuses-78166807b18bc138.yaml b/releasenotes/notes/add-llm-evaluator-row-statuses-78166807b18bc138.yaml new file mode 100644 index 00000000000..e6a8bde70e5 --- /dev/null +++ b/releasenotes/notes/add-llm-evaluator-row-statuses-78166807b18bc138.yaml @@ -0,0 +1,4 @@ +--- +enhancements: + - | + Adds an ``evaluation_statuses`` output to ``LLMEvaluator`` results so each evaluated row can report whether it was successfully evaluated or failed during generation/parsing when ``raise_on_failure=False`` is used. From 64efaa7a86760075503707b5a9b60cd00be5caf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-S=C3=A9bastien=20=7B=7BEbaAaZ=7D=7D?= Date: Mon, 18 May 2026 00:26:48 -0400 Subject: [PATCH 3/3] feat: add optional BM25 retrieval confidence metadata --- .../retrievers/in_memory/bm25_retriever.py | 31 ++++++++++++++ ...-retrieval-confidence-metadata-phase2.yaml | 8 ++++ .../test_in_memory_bm25_retriever.py | 42 ++++++++++++++++++- 3 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/add-bm25-retrieval-confidence-metadata-phase2.yaml diff --git a/haystack/components/retrievers/in_memory/bm25_retriever.py b/haystack/components/retrievers/in_memory/bm25_retriever.py index 05618f7d810..1e22c9f664f 100644 --- a/haystack/components/retrievers/in_memory/bm25_retriever.py +++ b/haystack/components/retrievers/in_memory/bm25_retriever.py @@ -44,6 +44,7 @@ def __init__( filters: dict[str, Any] | None = None, top_k: int = 10, scale_score: bool = False, + include_confidence: bool = False, filter_policy: FilterPolicy = FilterPolicy.REPLACE, ) -> None: """ @@ -58,6 +59,10 @@ def __init__( :param scale_score: When `True`, scales the score of retrieved documents to a range of 0 to 1, where 1 means extremely relevant. When `False`, uses raw similarity scores. + :param include_confidence: + When `True`, adds optional retrieval confidence metadata to returned documents when `scale_score` is also + `True`. The metadata is exposed via `Document.meta["retrieval_confidence"]` and + `Document.meta["retrieval_confidence_source"]`. :param filter_policy: The filter policy to apply during retrieval. Filter policy determines how filters are applied when retrieving documents. You can choose: - `REPLACE` (default): Overrides the initialization filters with the filters specified at runtime. @@ -78,6 +83,7 @@ def __init__( self.filters = filters self.top_k = top_k self.scale_score = scale_score + self.include_confidence = include_confidence self.filter_policy = filter_policy def _get_telemetry_data(self) -> dict[str, Any]: @@ -99,6 +105,7 @@ def to_dict(self) -> dict[str, Any]: filters=self.filters, top_k=self.top_k, scale_score=self.scale_score, + include_confidence=self.include_confidence, filter_policy=self.filter_policy.value, ) @@ -124,6 +131,7 @@ def run( filters: dict[str, Any] | None = None, top_k: int | None = None, scale_score: bool | None = None, + include_confidence: bool | None = None, ) -> dict[str, list[Document]]: """ Run the InMemoryBM25Retriever on the given input data. @@ -137,6 +145,9 @@ def run( :param scale_score: When `True`, scales the score of retrieved documents to a range of 0 to 1, where 1 means extremely relevant. When `False`, uses raw similarity scores. + :param include_confidence: + When `True`, adds optional retrieval confidence metadata to returned documents when `scale_score` is also + `True`. When `False`, no retrieval confidence metadata is added. :returns: The retrieved documents. @@ -151,8 +162,12 @@ def run( top_k = self.top_k if scale_score is None: scale_score = self.scale_score + if include_confidence is None: + include_confidence = self.include_confidence docs = self.document_store.bm25_retrieval(query=query, filters=filters, top_k=top_k, scale_score=scale_score) + if include_confidence and scale_score: + self._add_confidence_metadata(docs) return {"documents": docs} @component.output_types(documents=list[Document]) @@ -162,6 +177,7 @@ async def run_async( filters: dict[str, Any] | None = None, top_k: int | None = None, scale_score: bool | None = None, + include_confidence: bool | None = None, ) -> dict[str, list[Document]]: """ Run the InMemoryBM25Retriever on the given input data. @@ -175,6 +191,9 @@ async def run_async( :param scale_score: When `True`, scales the score of retrieved documents to a range of 0 to 1, where 1 means extremely relevant. When `False`, uses raw similarity scores. + :param include_confidence: + When `True`, adds optional retrieval confidence metadata to returned documents when `scale_score` is also + `True`. When `False`, no retrieval confidence metadata is added. :returns: The retrieved documents. @@ -189,8 +208,20 @@ async def run_async( top_k = self.top_k if scale_score is None: scale_score = self.scale_score + if include_confidence is None: + include_confidence = self.include_confidence docs = await self.document_store.bm25_retrieval_async( query=query, filters=filters, top_k=top_k, scale_score=scale_score ) + if include_confidence and scale_score: + self._add_confidence_metadata(docs) return {"documents": docs} + + @staticmethod + def _add_confidence_metadata(documents: list[Document]) -> None: + for document in documents: + if document.score is None: + continue + document.meta["retrieval_confidence"] = document.score + document.meta["retrieval_confidence_source"] = "bm25_scaled_score" diff --git a/releasenotes/notes/add-bm25-retrieval-confidence-metadata-phase2.yaml b/releasenotes/notes/add-bm25-retrieval-confidence-metadata-phase2.yaml new file mode 100644 index 00000000000..456c65e9391 --- /dev/null +++ b/releasenotes/notes/add-bm25-retrieval-confidence-metadata-phase2.yaml @@ -0,0 +1,8 @@ +--- +enhancements: + - | + Adds an optional `include_confidence` parameter to `InMemoryBM25Retriever`. + When enabled together with `scale_score=True`, returned documents now expose + BM25 retrieval confidence metadata via `Document.meta["retrieval_confidence"]` + and `Document.meta["retrieval_confidence_source"]` without changing + `Document.score` semantics. diff --git a/test/components/retrievers/test_in_memory_bm25_retriever.py b/test/components/retrievers/test_in_memory_bm25_retriever.py index 5946ce9b7c8..6adc86ea3b3 100644 --- a/test/components/retrievers/test_in_memory_bm25_retriever.py +++ b/test/components/retrievers/test_in_memory_bm25_retriever.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 +import asyncio from typing import Any import pytest @@ -31,12 +32,16 @@ def test_init_default(self, in_memory_doc_store): assert retriever.filters is None assert retriever.top_k == 10 assert retriever.scale_score is False + assert retriever.include_confidence is False def test_init_with_parameters(self, in_memory_doc_store): - retriever = InMemoryBM25Retriever(in_memory_doc_store, filters={"name": "test.txt"}, top_k=5, scale_score=True) + retriever = InMemoryBM25Retriever( + in_memory_doc_store, filters={"name": "test.txt"}, top_k=5, scale_score=True, include_confidence=True + ) assert retriever.filters == {"name": "test.txt"} assert retriever.top_k == 5 assert retriever.scale_score + assert retriever.include_confidence is True def test_init_with_invalid_top_k_parameter(self, in_memory_doc_store): with pytest.raises(ValueError): @@ -56,6 +61,7 @@ def test_to_dict(self): "filters": None, "top_k": 10, "scale_score": False, + "include_confidence": False, "filter_policy": "replace", }, } @@ -78,6 +84,7 @@ def test_to_dict_with_custom_init_parameters(self): "filters": {"name": "test.txt"}, "top_k": 5, "scale_score": True, + "include_confidence": False, "filter_policy": "replace", }, } @@ -99,6 +106,7 @@ def test_from_dict(self): assert component.filters == {"name": "test.txt"} assert component.top_k == 5 assert component.scale_score is False + assert component.include_confidence is False assert component.filter_policy == FilterPolicy.REPLACE def test_from_dict_without_docstore(self): @@ -140,6 +148,38 @@ def test_invalid_run_wrong_store_type(self): with pytest.raises(TypeError, match="document_store must be an instance of InMemoryDocumentStore"): InMemoryBM25Retriever(SomeOtherDocumentStore()) + def test_run_with_include_confidence_adds_metadata_when_scaled(self, in_memory_doc_store, mock_docs): + in_memory_doc_store.write_documents(mock_docs) + + retriever = InMemoryBM25Retriever(in_memory_doc_store, top_k=3, scale_score=True, include_confidence=True) + result = retriever.run(query="PHP") + + first_document = result["documents"][0] + assert first_document.score is not None + assert first_document.meta["retrieval_confidence"] == first_document.score + assert first_document.meta["retrieval_confidence_source"] == "bm25_scaled_score" + + def test_run_with_include_confidence_does_not_add_metadata_when_not_scaled(self, in_memory_doc_store, mock_docs): + in_memory_doc_store.write_documents(mock_docs) + + retriever = InMemoryBM25Retriever(in_memory_doc_store, top_k=3, include_confidence=True) + result = retriever.run(query="PHP") + + first_document = result["documents"][0] + assert "retrieval_confidence" not in first_document.meta + assert "retrieval_confidence_source" not in first_document.meta + + def test_run_async_with_include_confidence_matches_sync_behavior(self, in_memory_doc_store, mock_docs): + in_memory_doc_store.write_documents(mock_docs) + + retriever = InMemoryBM25Retriever(in_memory_doc_store, top_k=3, scale_score=True, include_confidence=True) + result = asyncio.run(retriever.run_async(query="PHP")) + + first_document = result["documents"][0] + assert first_document.score is not None + assert first_document.meta["retrieval_confidence"] == first_document.score + assert first_document.meta["retrieval_confidence_source"] == "bm25_scaled_score" + @pytest.mark.integration @pytest.mark.parametrize( "query, query_result",