feat: add optional BM25 retrieval confidence metadata

haystack/components/evaluators/llm_evaluator.py

@@ -175,7 +175,7 @@ def validate_init_parameters(
                 )
                 raise ValueError(msg)
 
-    @component.output_types(results=list[dict[str, Any]])
+    @component.output_types(results=list[dict[str, Any]], evaluation_statuses=list[str])
     def run(self, **inputs: Any) -> dict[str, Any]:
         """
         Run the LLM evaluator.
@@ -204,6 +204,7 @@ def run(self, **inputs: Any) -> dict[str, Any]:
         list_of_input_names_to_values = [dict(zip(input_names, v, strict=True)) for v in values]
 
         results: list[dict[str, Any] | None] = []
+        evaluation_statuses: list[str] = []
         metadata = []
         errors = 0
         for input_names_to_values in tqdm(list_of_input_names_to_values, disable=not self.progress_bar):
@@ -216,6 +217,7 @@ def run(self, **inputs: Any) -> dict[str, Any]:
                     raise ValueError(f"Error while generating response for prompt: {prompt}. Error: {e}") from e
                 logger.warning("Error while generating response for prompt: {prompt}. Error: {e}", prompt=prompt, e=e)
                 results.append(None)
+                evaluation_statuses.append("error")
                 errors += 1
                 continue
 
@@ -224,9 +226,11 @@ def run(self, **inputs: Any) -> dict[str, Any]:
             )
             if parsed_result is None:
                 results.append(None)
+                evaluation_statuses.append("error")
                 errors += 1
             else:
                 results.append(parsed_result)
+                evaluation_statuses.append("evaluated")
 
             if result["replies"][0].meta:
                 metadata.append(result["replies"][0].meta)
@@ -238,7 +242,7 @@ def run(self, **inputs: Any) -> dict[str, Any]:
                 len=len(list_of_input_names_to_values),
             )
 
-        return {"results": results, "meta": metadata or None}
+        return {"results": results, "meta": metadata or None, "evaluation_statuses": evaluation_statuses}
 
     def prepare_template(self) -> str:
         """

haystack/components/retrievers/in_memory/bm25_retriever.py

@@ -44,6 +44,7 @@ def __init__(
         filters: dict[str, Any] | None = None,
         top_k: int = 10,
         scale_score: bool = False,
+        include_confidence: bool = False,
         filter_policy: FilterPolicy = FilterPolicy.REPLACE,
     ) -> None:
         """
@@ -58,6 +59,10 @@ def __init__(
         :param scale_score:
             When `True`, scales the score of retrieved documents to a range of 0 to 1, where 1 means extremely relevant.
             When `False`, uses raw similarity scores.
+        :param include_confidence:
+            When `True`, adds optional retrieval confidence metadata to returned documents when `scale_score` is also
+            `True`. The metadata is exposed via `Document.meta["retrieval_confidence"]` and
+            `Document.meta["retrieval_confidence_source"]`.
         :param filter_policy: The filter policy to apply during retrieval.
         Filter policy determines how filters are applied when retrieving documents. You can choose:
         - `REPLACE` (default): Overrides the initialization filters with the filters specified at runtime.
@@ -78,6 +83,7 @@ def __init__(
         self.filters = filters
         self.top_k = top_k
         self.scale_score = scale_score
+        self.include_confidence = include_confidence
         self.filter_policy = filter_policy
 
     def _get_telemetry_data(self) -> dict[str, Any]:
@@ -99,6 +105,7 @@ def to_dict(self) -> dict[str, Any]:
             filters=self.filters,
             top_k=self.top_k,
             scale_score=self.scale_score,
+            include_confidence=self.include_confidence,
             filter_policy=self.filter_policy.value,
         )
 
@@ -124,6 +131,7 @@ def run(
         filters: dict[str, Any] | None = None,
         top_k: int | None = None,
         scale_score: bool | None = None,
+        include_confidence: bool | None = None,
     ) -> dict[str, list[Document]]:
         """
         Run the InMemoryBM25Retriever on the given input data.
@@ -137,6 +145,9 @@ def run(
         :param scale_score:
             When `True`, scales the score of retrieved documents to a range of 0 to 1, where 1 means extremely relevant.
             When `False`, uses raw similarity scores.
+        :param include_confidence:
+            When `True`, adds optional retrieval confidence metadata to returned documents when `scale_score` is also
+            `True`. When `False`, no retrieval confidence metadata is added.
         :returns:
             The retrieved documents.
 
@@ -151,8 +162,12 @@ def run(
             top_k = self.top_k
         if scale_score is None:
             scale_score = self.scale_score
+        if include_confidence is None:
+            include_confidence = self.include_confidence
 
         docs = self.document_store.bm25_retrieval(query=query, filters=filters, top_k=top_k, scale_score=scale_score)
+        if include_confidence and scale_score:
+            self._add_confidence_metadata(docs)
         return {"documents": docs}
 
     @component.output_types(documents=list[Document])
@@ -162,6 +177,7 @@ async def run_async(
         filters: dict[str, Any] | None = None,
         top_k: int | None = None,
         scale_score: bool | None = None,
+        include_confidence: bool | None = None,
     ) -> dict[str, list[Document]]:
         """
         Run the InMemoryBM25Retriever on the given input data.
@@ -175,6 +191,9 @@ async def run_async(
         :param scale_score:
             When `True`, scales the score of retrieved documents to a range of 0 to 1, where 1 means extremely relevant.
             When `False`, uses raw similarity scores.
+        :param include_confidence:
+            When `True`, adds optional retrieval confidence metadata to returned documents when `scale_score` is also
+            `True`. When `False`, no retrieval confidence metadata is added.
         :returns:
             The retrieved documents.
 
@@ -189,8 +208,20 @@ async def run_async(
             top_k = self.top_k
         if scale_score is None:
             scale_score = self.scale_score
+        if include_confidence is None:
+            include_confidence = self.include_confidence
 
         docs = await self.document_store.bm25_retrieval_async(
             query=query, filters=filters, top_k=top_k, scale_score=scale_score
         )
+        if include_confidence and scale_score:
+            self._add_confidence_metadata(docs)
         return {"documents": docs}
+
+    @staticmethod
+    def _add_confidence_metadata(documents: list[Document]) -> None:
+        for document in documents:
+            if document.score is None:
+                continue
+            document.meta["retrieval_confidence"] = document.score
+            document.meta["retrieval_confidence_source"] = "bm25_scaled_score"

releasenotes/notes/add-bm25-retrieval-confidence-metadata-phase2.yaml

@@ -0,0 +1,8 @@
+---
+enhancements:
+  - |
+    Adds an optional `include_confidence` parameter to `InMemoryBM25Retriever`.
+    When enabled together with `scale_score=True`, returned documents now expose
+    BM25 retrieval confidence metadata via `Document.meta["retrieval_confidence"]`
+    and `Document.meta["retrieval_confidence_source"]` without changing
+    `Document.score` semantics.

releasenotes/notes/add-llm-evaluator-row-statuses-78166807b18bc138.yaml

@@ -0,0 +1,4 @@
+---
+enhancements:
+  - |
+    Adds an ``evaluation_statuses`` output to ``LLMEvaluator`` results so each evaluated row can report whether it was successfully evaluated or failed during generation/parsing when ``raise_on_failure=False`` is used.

test/components/evaluators/test_context_relevance_evaluator.py

@@ -169,6 +169,7 @@ def chat_generator_run(self, *args, **kwargs):
             "score": 0.5,
             "meta": None,
             "individual_scores": [1, 0],
+            "evaluation_statuses": ["evaluated", "evaluated"],
         }
 
     def test_run_no_statements_extracted(self, monkeypatch):
@@ -198,6 +199,7 @@ def chat_generator_run(self, *args, **kwargs):
             "score": 0.5,
             "meta": None,
             "individual_scores": [1, 0],
+            "evaluation_statuses": ["evaluated", "evaluated"],
         }
 
     def test_run_missing_parameters(self, monkeypatch):
@@ -237,6 +239,7 @@ def chat_generator_run(self, *args, **kwargs):
         assert results["results"][0] == {"relevant_statements": ["c", "d"], "score": 1}
         assert results["results"][1]["relevant_statements"] == []
         assert math.isnan(results["results"][1]["score"])
+        assert results["evaluation_statuses"] == ["evaluated", "error"]
 
     @pytest.mark.skipif(
         not os.environ.get("OPENAI_API_KEY", None),

test/components/evaluators/test_faithfulness_evaluator.py

@@ -207,6 +207,7 @@ def chat_generator_run(self, *args, **kwargs):
             ],
             "score": 0.75,
             "meta": None,
+            "evaluation_statuses": ["evaluated", "evaluated"],
         }
 
     def test_run_no_statements_extracted(self, monkeypatch):
@@ -245,6 +246,7 @@ def chat_generator_run(self, *args, **kwargs):
             ],
             "score": 0.25,
             "meta": None,
+            "evaluation_statuses": ["evaluated", "evaluated"],
         }
 
     def test_run_missing_parameters(self, monkeypatch):
@@ -294,6 +296,7 @@ def chat_generator_run(self, *args, **kwargs):
         assert results["results"][1]["statements"] == []
         assert results["results"][1]["statement_scores"] == []
         assert math.isnan(results["results"][1]["score"])
+        assert results["evaluation_statuses"] == ["evaluated", "error"]
 
     @pytest.mark.skipif(
         not os.environ.get("OPENAI_API_KEY", None),

test/components/evaluators/test_llm_evaluator.py

@@ -332,7 +332,7 @@ def chat_generator_run(self, *args, **kwargs):
         monkeypatch.setattr("haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run", chat_generator_run)
 
         results = component.run(questions=["What is the capital of Germany?"], predicted_answers=["Berlin"])
-        assert results == {"results": [{"score": 0.5}], "meta": None}
+        assert results == {"results": [{"score": 0.5}], "meta": None, "evaluation_statuses": ["evaluated"]}
 
     def test_prepare_template(self, monkeypatch):
         monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
@@ -433,6 +433,28 @@ def chat_generator_run(self, *args, **kwargs):
 
         result = component.run(predicted_answers=["answer"])
         assert result["results"] == [None]
+        assert result["evaluation_statuses"] == ["error"]
+
+    def test_run_returns_error_status_raise_on_failure_false(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = LLMEvaluator(
+            instructions="test-instruction",
+            inputs=[("predicted_answers", list[str])],
+            outputs=["score"],
+            examples=[
+                {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
+            ],
+            raise_on_failure=False,
+        )
+
+        def chat_generator_run(self, *args, **kwargs):
+            raise Exception("OpenAI API request failed.")
+
+        monkeypatch.setattr("haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run", chat_generator_run)
+
+        result = component.run(predicted_answers=["answer"])
+        assert result["results"] == [None]
+        assert result["evaluation_statuses"] == ["error"]
 
     def test_output_invalid_json_raise_on_failure_true(self, monkeypatch):
         monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")

test/components/retrievers/test_in_memory_bm25_retriever.py

@@ -2,6 +2,7 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+import asyncio
 from typing import Any
 
 import pytest
@@ -31,12 +32,16 @@ def test_init_default(self, in_memory_doc_store):
         assert retriever.filters is None
         assert retriever.top_k == 10
         assert retriever.scale_score is False
+        assert retriever.include_confidence is False
 
     def test_init_with_parameters(self, in_memory_doc_store):
-        retriever = InMemoryBM25Retriever(in_memory_doc_store, filters={"name": "test.txt"}, top_k=5, scale_score=True)
+        retriever = InMemoryBM25Retriever(
+            in_memory_doc_store, filters={"name": "test.txt"}, top_k=5, scale_score=True, include_confidence=True
+        )
         assert retriever.filters == {"name": "test.txt"}
         assert retriever.top_k == 5
         assert retriever.scale_score
+        assert retriever.include_confidence is True
 
     def test_init_with_invalid_top_k_parameter(self, in_memory_doc_store):
         with pytest.raises(ValueError):
@@ -56,6 +61,7 @@ def test_to_dict(self):
                 "filters": None,
                 "top_k": 10,
                 "scale_score": False,
+                "include_confidence": False,
                 "filter_policy": "replace",
             },
         }
@@ -78,6 +84,7 @@ def test_to_dict_with_custom_init_parameters(self):
                 "filters": {"name": "test.txt"},
                 "top_k": 5,
                 "scale_score": True,
+                "include_confidence": False,
                 "filter_policy": "replace",
             },
         }
@@ -99,6 +106,7 @@ def test_from_dict(self):
         assert component.filters == {"name": "test.txt"}
         assert component.top_k == 5
         assert component.scale_score is False
+        assert component.include_confidence is False
         assert component.filter_policy == FilterPolicy.REPLACE
 
     def test_from_dict_without_docstore(self):
@@ -140,6 +148,38 @@ def test_invalid_run_wrong_store_type(self):
         with pytest.raises(TypeError, match="document_store must be an instance of InMemoryDocumentStore"):
             InMemoryBM25Retriever(SomeOtherDocumentStore())
 
+    def test_run_with_include_confidence_adds_metadata_when_scaled(self, in_memory_doc_store, mock_docs):
+        in_memory_doc_store.write_documents(mock_docs)
+
+        retriever = InMemoryBM25Retriever(in_memory_doc_store, top_k=3, scale_score=True, include_confidence=True)
+        result = retriever.run(query="PHP")
+
+        first_document = result["documents"][0]
+        assert first_document.score is not None
+        assert first_document.meta["retrieval_confidence"] == first_document.score
+        assert first_document.meta["retrieval_confidence_source"] == "bm25_scaled_score"
+
+    def test_run_with_include_confidence_does_not_add_metadata_when_not_scaled(self, in_memory_doc_store, mock_docs):
+        in_memory_doc_store.write_documents(mock_docs)
+
+        retriever = InMemoryBM25Retriever(in_memory_doc_store, top_k=3, include_confidence=True)
+        result = retriever.run(query="PHP")
+
+        first_document = result["documents"][0]
+        assert "retrieval_confidence" not in first_document.meta
+        assert "retrieval_confidence_source" not in first_document.meta
+
+    def test_run_async_with_include_confidence_matches_sync_behavior(self, in_memory_doc_store, mock_docs):
+        in_memory_doc_store.write_documents(mock_docs)
+
+        retriever = InMemoryBM25Retriever(in_memory_doc_store, top_k=3, scale_score=True, include_confidence=True)
+        result = asyncio.run(retriever.run_async(query="PHP"))
+
+        first_document = result["documents"][0]
+        assert first_document.score is not None
+        assert first_document.meta["retrieval_confidence"] == first_document.score
+        assert first_document.meta["retrieval_confidence_source"] == "bm25_scaled_score"
+
     @pytest.mark.integration
     @pytest.mark.parametrize(
         "query, query_result",