fix: preserve evaluator result status

Author: SeCuReDmE-main-dev

haystack/components/evaluators/context_relevance.py

@@ -85,17 +85,20 @@ class ContextRelevanceEvaluator(LLMEvaluator):
     print(result["results"])
     # [{
     #   'relevant_statements': ['Python, created by Guido van Rossum in the late 1980s.'],
-    #    'score': 1.0
+    #   'score': 1.0,
+    #   'status': 'evaluated'
     #  },
     #  {
     #   'relevant_statements': ['The JVM has two primary functions: to allow Java programs to run on any device or
     #                           operating system (known as the "write once, run anywhere" principle), and to manage and
     #                           optimize program memory'],
-    #   'score': 1.0
+    #   'score': 1.0,
+    #   'status': 'evaluated'
     #  },
     #  {
     #   'relevant_statements': [],
-    #   'score': 0.0
+    #   'score': 0.0,
+    #   'status': 'evaluated'
     #  }]
     ```
     """
@@ -171,7 +174,8 @@ def run(self, **inputs: Any) -> dict[str, Any]:
         :returns:
             A dictionary with the following outputs:
                 - `score`: Mean context relevance score over all the provided input questions.
-                - `results`: A list of dictionaries with `relevant_statements` and `score` for each input context.
+                - `results`: A list of dictionaries with `relevant_statements`, `score`, and `status` for each input
+                  context. `status` is `evaluated` for valid results and `error` for failed evaluations.
         """
         result = super(ContextRelevanceEvaluator, self).run(**inputs)  # noqa: UP008
         # Post-process the raw results to calculate relevance metrics and scores
@@ -189,7 +193,8 @@ async def run_async(self, **inputs: Any) -> dict[str, Any]:
         :returns:
             A dictionary with the following outputs:
                 - `score`: Mean context relevance score over all the provided input questions.
-                - `results`: A list of dictionaries with `relevant_statements` and `score` for each input context.
+                - `results`: A list of dictionaries with `relevant_statements`, `score`, and `status` for each input
+                  context. `status` is `evaluated` for valid results and `error` for failed evaluations.
         """
         result = await super(ContextRelevanceEvaluator, self).run_async(**inputs)  # noqa: UP008
         # Post-process the raw results to calculate relevance metrics and scores
@@ -209,8 +214,9 @@ def _postprocess_results(self, result: dict[str, Any]) -> dict[str, Any]:
         """
         for idx, res in enumerate(result["results"]):
             if res is None:
-                result["results"][idx] = {"relevant_statements": [], "score": float("nan")}
+                result["results"][idx] = {"relevant_statements": [], "score": float("nan"), "status": "error"}
                 continue
+            res["status"] = "evaluated"
             if len(res["relevant_statements"]) > 0:
                 res["score"] = 1
             else:

haystack/components/evaluators/faithfulness.py

@@ -83,7 +83,8 @@ class FaithfulnessEvaluator(LLMEvaluator):
     # 0.5
     print(result["results"])
     # [{'statements': ['Python is a high-level general-purpose programming language.',
-    # 'Python was created by George Lucas.'], 'statement_scores': [1, 0], 'score': 0.5}]
+    # 'Python was created by George Lucas.'], 'statement_scores': [1, 0], 'score': 0.5,
+    # 'status': 'evaluated'}]
     ```
     """
 
@@ -164,7 +165,8 @@ def run(self, **inputs: Any) -> dict[str, Any]:
             A dictionary with the following outputs:
                 - `score`: Mean faithfulness score over all the provided input answers.
                 - `individual_scores`: A list of faithfulness scores for each input answer.
-                - `results`: A list of dictionaries with `statements` and `statement_scores` for each input answer.
+                - `results`: A list of dictionaries with `statements`, `statement_scores`, `score`, and `status` for
+                  each input answer. `status` is `evaluated` for valid results and `error` for failed evaluations.
         """
         result = super(FaithfulnessEvaluator, self).run(**inputs)  # noqa: UP008
         # Post-process the raw results to calculate relevance metrics and scores
@@ -185,7 +187,8 @@ async def run_async(self, **inputs: Any) -> dict[str, Any]:
             A dictionary with the following outputs:
                 - `score`: Mean faithfulness score over all the provided input answers.
                 - `individual_scores`: A list of faithfulness scores for each input answer.
-                - `results`: A list of dictionaries with `statements` and `statement_scores` for each input answer.
+                - `results`: A list of dictionaries with `statements`, `statement_scores`, `score`, and `status` for
+                  each input answer. `status` is `evaluated` for valid results and `error` for failed evaluations.
         """
         result = await super(FaithfulnessEvaluator, self).run_async(**inputs)  # noqa: UP008
         # Post-process the raw results to calculate relevance metrics and scores
@@ -207,8 +210,14 @@ def _postprocess_results(self, result: dict[str, Any]) -> dict[str, Any]:
         # calculate average statement faithfulness score per query
         for idx, res in enumerate(result["results"]):
             if res is None:
-                result["results"][idx] = {"statements": [], "statement_scores": [], "score": float("nan")}
+                result["results"][idx] = {
+                    "statements": [],
+                    "statement_scores": [],
+                    "score": float("nan"),
+                    "status": "error",
+                }
                 continue
+            res["status"] = "evaluated"
             if not res["statements"]:
                 res["score"] = 0
             else:

haystack/components/evaluators/llm_evaluator.py

@@ -198,7 +198,7 @@ def validate_init_parameters(
                 )
                 raise ValueError(msg)
 
-    @component.output_types(results=list[dict[str, Any]], evaluation_statuses=list[str])
+    @component.output_types(results=list[dict[str, Any]])
     def run(self, **inputs: Any) -> dict[str, Any]:
         """
         Run the LLM evaluator.
@@ -226,7 +226,6 @@ def run(self, **inputs: Any) -> dict[str, Any]:
         list_of_input_names_to_values = [dict(zip(input_names, v, strict=True)) for v in values]
 
         results: list[dict[str, Any] | None] = []
-        evaluation_statuses: list[str] = []
         metadata = []
         errors = 0
         for input_names_to_values in tqdm(list_of_input_names_to_values, disable=not self.progress_bar):
@@ -239,7 +238,6 @@ def run(self, **inputs: Any) -> dict[str, Any]:
                     raise ValueError(f"Error while generating response for prompt: {prompt}. Error: {e}") from e
                 logger.warning("Error while generating response for prompt: {prompt}. Error: {e}", prompt=prompt, e=e)
                 results.append(None)
-                evaluation_statuses.append("error")
                 errors += 1
                 continue
 
@@ -248,11 +246,9 @@ def run(self, **inputs: Any) -> dict[str, Any]:
             )
             if parsed_result is None:
                 results.append(None)
-                evaluation_statuses.append("error")
                 errors += 1
             else:
                 results.append(parsed_result)
-                evaluation_statuses.append("evaluated")
 
             if result["replies"][0].meta:
                 metadata.append(result["replies"][0].meta)
@@ -264,7 +260,7 @@ def run(self, **inputs: Any) -> dict[str, Any]:
                 len=len(list_of_input_names_to_values),
             )
 
-        return {"results": results, "meta": metadata or None, "evaluation_statuses": evaluation_statuses}
+        return {"results": results, "meta": metadata or None}
 
     @component.output_types(results=list[dict[str, Any]])
     async def run_async(self, **inputs: Any) -> dict[str, Any]:

releasenotes/notes/add-llm-evaluator-row-statuses-78166807b18bc138.yaml

@@ -1,4 +1,6 @@
 ---
 enhancements:
   - |
-    Adds an ``evaluation_statuses`` output to ``LLMEvaluator`` results so each evaluated row can report whether it was successfully evaluated or failed during generation/parsing when ``raise_on_failure=False`` is used.
+    Adds a ``status`` field to each result produced by ``ContextRelevanceEvaluator`` and ``FaithfulnessEvaluator``.
+    The field is ``evaluated`` for valid results and ``error`` for generation or parsing failures continued with
+    ``raise_on_failure=False``.

test/components/evaluators/test_context_relevance_evaluator.py

@@ -166,11 +166,13 @@ def chat_generator_run(self, *args, **kwargs):
         results = component.run(questions=questions, contexts=contexts)
 
         assert results == {
-            "results": [{"score": 1, "relevant_statements": ["a", "b"]}, {"score": 0, "relevant_statements": []}],
+            "results": [
+                {"score": 1, "relevant_statements": ["a", "b"], "status": "evaluated"},
+                {"score": 0, "relevant_statements": [], "status": "evaluated"},
+            ],
             "score": 0.5,
             "meta": None,
             "individual_scores": [1, 0],
-            "evaluation_statuses": ["evaluated", "evaluated"],
         }
 
     def test_run_no_statements_extracted(self, monkeypatch):
@@ -196,11 +198,13 @@ def chat_generator_run(self, *args, **kwargs):
         ]
         results = component.run(questions=questions, contexts=contexts)
         assert results == {
-            "results": [{"score": 1, "relevant_statements": ["a", "b"]}, {"score": 0, "relevant_statements": []}],
+            "results": [
+                {"score": 1, "relevant_statements": ["a", "b"], "status": "evaluated"},
+                {"score": 0, "relevant_statements": [], "status": "evaluated"},
+            ],
             "score": 0.5,
             "meta": None,
             "individual_scores": [1, 0],
-            "evaluation_statuses": ["evaluated", "evaluated"],
         }
 
     def test_run_missing_parameters(self, monkeypatch):
@@ -209,13 +213,16 @@ def test_run_missing_parameters(self, monkeypatch):
         with pytest.raises(ValueError, match="LLM evaluator expected input parameter"):
             component.run()
 
-    def test_run_returns_nan_raise_on_failure_false(self, monkeypatch, caplog):
+    @pytest.mark.parametrize("failure_mode", ["generation", "parsing"])
+    def test_run_returns_nan_raise_on_failure_false(self, monkeypatch, caplog, failure_mode):
         monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
         component = ContextRelevanceEvaluator(raise_on_failure=False)
 
         def chat_generator_run(self, *args, **kwargs):
             if "Python" in kwargs["messages"][0].text:
-                raise Exception("OpenAI API request failed.")
+                if failure_mode == "generation":
+                    raise Exception("OpenAI API request failed.")
+                return {"replies": [ChatMessage.from_assistant("not valid JSON")]}
             return {"replies": [ChatMessage.from_assistant('{"relevant_statements": ["c", "d"], "score": 1}')]}
 
         monkeypatch.setattr("haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run", chat_generator_run)
@@ -238,13 +245,28 @@ def chat_generator_run(self, *args, **kwargs):
             results = component.run(questions=questions, contexts=contexts)
 
         assert results["score"] == 1
-        assert results["results"][0] == {"relevant_statements": ["c", "d"], "score": 1}
+        assert results["results"][0] == {"relevant_statements": ["c", "d"], "score": 1, "status": "evaluated"}
         assert results["results"][1]["relevant_statements"] == []
         assert math.isnan(results["results"][1]["score"])
-        assert results["evaluation_statuses"] == ["evaluated", "error"]
+        assert results["results"][1]["status"] == "error"
 
         assert "1 query(s) failed and were excluded from the score." in caplog.text
 
+    def test_run_all_failures_returns_nan_and_error_statuses(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = ContextRelevanceEvaluator(raise_on_failure=False)
+
+        def chat_generator_run(self, *args, **kwargs):
+            raise Exception("OpenAI API request failed.")
+
+        monkeypatch.setattr("haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run", chat_generator_run)
+
+        results = component.run(questions=["q1", "q2"], contexts=[["c1"], ["c2"]])
+
+        assert math.isnan(results["score"])
+        assert all(math.isnan(score) for score in results["individual_scores"])
+        assert [result["status"] for result in results["results"]] == ["error", "error"]
+
     @pytest.mark.skipif(
         not os.environ.get("OPENAI_API_KEY", None),
         reason="Export an env var called OPENAI_API_KEY containing the OpenAI API key to run this test.",
@@ -259,7 +281,7 @@ def test_live_run(self):
 
         required_fields = {"results"}
         assert all(field in result for field in required_fields)
-        nested_required_fields = {"score", "relevant_statements"}
+        nested_required_fields = {"score", "relevant_statements", "status"}
         assert all(field in result["results"][0] for field in nested_required_fields)
 
         assert "meta" in result
@@ -292,20 +314,26 @@ async def chat_generator_run_async(self, *args, **kwargs):
         results = await component.run_async(questions=questions, contexts=contexts)
 
         assert results == {
-            "results": [{"score": 1, "relevant_statements": ["a", "b"]}, {"score": 0, "relevant_statements": []}],
+            "results": [
+                {"score": 1, "relevant_statements": ["a", "b"], "status": "evaluated"},
+                {"score": 0, "relevant_statements": [], "status": "evaluated"},
+            ],
             "score": 0.5,
             "meta": None,
             "individual_scores": [1, 0],
         }
 
     @pytest.mark.asyncio
-    async def test_run_async_returns_nan_raise_on_failure_false(self, monkeypatch, caplog):
+    @pytest.mark.parametrize("failure_mode", ["generation", "parsing"])
+    async def test_run_async_returns_nan_raise_on_failure_false(self, monkeypatch, caplog, failure_mode):
         monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
         component = ContextRelevanceEvaluator(raise_on_failure=False)
 
         async def chat_generator_run_async(self, *args, **kwargs):
             if "Python" in kwargs["messages"][0].text:
-                raise Exception("OpenAI API request failed.")
+                if failure_mode == "generation":
+                    raise Exception("OpenAI API request failed.")
+                return {"replies": [ChatMessage.from_assistant("not valid JSON")]}
             return {"replies": [ChatMessage.from_assistant('{"relevant_statements": ["c", "d"], "score": 1}')]}
 
         monkeypatch.setattr(
@@ -319,9 +347,10 @@ async def chat_generator_run_async(self, *args, **kwargs):
             results = await component.run_async(questions=questions, contexts=contexts)
 
         assert results["score"] == 1
-        assert results["results"][0] == {"relevant_statements": ["c", "d"], "score": 1}
+        assert results["results"][0] == {"relevant_statements": ["c", "d"], "score": 1, "status": "evaluated"}
         assert results["results"][1]["relevant_statements"] == []
         assert math.isnan(results["results"][1]["score"])
+        assert results["results"][1]["status"] == "error"
 
         assert "1 query(s) failed and were excluded from the score." in caplog.text
 
@@ -340,7 +369,7 @@ async def test_live_run_async(self):
 
         required_fields = {"results"}
         assert all(field in result for field in required_fields)
-        nested_required_fields = {"score", "relevant_statements"}
+        nested_required_fields = {"score", "relevant_statements", "status"}
         assert all(field in result["results"][0] for field in nested_required_fields)
 
         assert "meta" in result