fix: preserve evaluator result status

Author: SeCuReDmE-main-dev

haystack/components/evaluators/context_relevance.py

@@ -85,17 +85,20 @@ class ContextRelevanceEvaluator(LLMEvaluator):
     print(result["results"])
     # [{
     #   'relevant_statements': ['Python, created by Guido van Rossum in the late 1980s.'],
-    #    'score': 1.0
+    #   'score': 1.0,
+    #   'status': 'evaluated'
     #  },
     #  {
     #   'relevant_statements': ['The JVM has two primary functions: to allow Java programs to run on any device or
     #                           operating system (known as the "write once, run anywhere" principle), and to manage and
     #                           optimize program memory'],
-    #   'score': 1.0
+    #   'score': 1.0,
+    #   'status': 'evaluated'
     #  },
     #  {
     #   'relevant_statements': [],
-    #   'score': 0.0
+    #   'score': 0.0,
+    #   'status': 'evaluated'
     #  }]
     ```
     """
@@ -171,7 +174,8 @@ def run(self, **inputs: Any) -> dict[str, Any]:
         :returns:
             A dictionary with the following outputs:
                 - `score`: Mean context relevance score over all the provided input questions.
-                - `results`: A list of dictionaries with `relevant_statements` and `score` for each input context.
+                - `results`: A list of dictionaries with `relevant_statements`, `score`, and `status` for each input
+                  context. `status` is `evaluated` for valid results and `error` for failed evaluations.
         """
         result = super(ContextRelevanceEvaluator, self).run(**inputs)  # noqa: UP008
         # Post-process the raw results to calculate relevance metrics and scores
@@ -189,7 +193,8 @@ async def run_async(self, **inputs: Any) -> dict[str, Any]:
         :returns:
             A dictionary with the following outputs:
                 - `score`: Mean context relevance score over all the provided input questions.
-                - `results`: A list of dictionaries with `relevant_statements` and `score` for each input context.
+                - `results`: A list of dictionaries with `relevant_statements`, `score`, and `status` for each input
+                  context. `status` is `evaluated` for valid results and `error` for failed evaluations.
         """
         result = await super(ContextRelevanceEvaluator, self).run_async(**inputs)  # noqa: UP008
         # Post-process the raw results to calculate relevance metrics and scores
@@ -209,8 +214,9 @@ def _postprocess_results(self, result: dict[str, Any]) -> dict[str, Any]:
         """
         for idx, res in enumerate(result["results"]):
             if res is None:
-                result["results"][idx] = {"relevant_statements": [], "score": float("nan")}
+                result["results"][idx] = {"relevant_statements": [], "score": float("nan"), "status": "error"}
                 continue
+            res["status"] = "evaluated"
             if len(res["relevant_statements"]) > 0:
                 res["score"] = 1
             else:

haystack/components/evaluators/faithfulness.py

@@ -83,7 +83,8 @@ class FaithfulnessEvaluator(LLMEvaluator):
     # 0.5
     print(result["results"])
     # [{'statements': ['Python is a high-level general-purpose programming language.',
-    # 'Python was created by George Lucas.'], 'statement_scores': [1, 0], 'score': 0.5}]
+    # 'Python was created by George Lucas.'], 'statement_scores': [1, 0], 'score': 0.5,
+    # 'status': 'evaluated'}]
     ```
     """
 
@@ -164,7 +165,8 @@ def run(self, **inputs: Any) -> dict[str, Any]:
             A dictionary with the following outputs:
                 - `score`: Mean faithfulness score over all the provided input answers.
                 - `individual_scores`: A list of faithfulness scores for each input answer.
-                - `results`: A list of dictionaries with `statements` and `statement_scores` for each input answer.
+                - `results`: A list of dictionaries with `statements`, `statement_scores`, `score`, and `status` for
+                  each input answer. `status` is `evaluated` for valid results and `error` for failed evaluations.
         """
         result = super(FaithfulnessEvaluator, self).run(**inputs)  # noqa: UP008
         # Post-process the raw results to calculate relevance metrics and scores
@@ -185,7 +187,8 @@ async def run_async(self, **inputs: Any) -> dict[str, Any]:
             A dictionary with the following outputs:
                 - `score`: Mean faithfulness score over all the provided input answers.
                 - `individual_scores`: A list of faithfulness scores for each input answer.
-                - `results`: A list of dictionaries with `statements` and `statement_scores` for each input answer.
+                - `results`: A list of dictionaries with `statements`, `statement_scores`, `score`, and `status` for
+                  each input answer. `status` is `evaluated` for valid results and `error` for failed evaluations.
         """
         result = await super(FaithfulnessEvaluator, self).run_async(**inputs)  # noqa: UP008
         # Post-process the raw results to calculate relevance metrics and scores
@@ -207,8 +210,14 @@ def _postprocess_results(self, result: dict[str, Any]) -> dict[str, Any]:
         # calculate average statement faithfulness score per query
         for idx, res in enumerate(result["results"]):
             if res is None:
-                result["results"][idx] = {"statements": [], "statement_scores": [], "score": float("nan")}
+                result["results"][idx] = {
+                    "statements": [],
+                    "statement_scores": [],
+                    "score": float("nan"),
+                    "status": "error",
+                }
                 continue
+            res["status"] = "evaluated"
             if not res["statements"]:
                 res["score"] = 0
             else:

releasenotes/notes/add-llm-evaluator-row-statuses-78166807b18bc138.yaml

@@ -0,0 +1,6 @@
+---
+enhancements:
+  - |
+    Adds a ``status`` field to each result produced by ``ContextRelevanceEvaluator`` and ``FaithfulnessEvaluator``.
+    The field is ``evaluated`` for valid results and ``error`` for generation or parsing failures continued with
+    ``raise_on_failure=False``.

test/components/evaluators/test_context_relevance_evaluator.py

@@ -166,7 +166,10 @@ def chat_generator_run(self, *args, **kwargs):
         results = component.run(questions=questions, contexts=contexts)
 
         assert results == {
-            "results": [{"score": 1, "relevant_statements": ["a", "b"]}, {"score": 0, "relevant_statements": []}],
+            "results": [
+                {"score": 1, "relevant_statements": ["a", "b"], "status": "evaluated"},
+                {"score": 0, "relevant_statements": [], "status": "evaluated"},
+            ],
             "score": 0.5,
             "meta": None,
             "individual_scores": [1, 0],
@@ -195,7 +198,10 @@ def chat_generator_run(self, *args, **kwargs):
         ]
         results = component.run(questions=questions, contexts=contexts)
         assert results == {
-            "results": [{"score": 1, "relevant_statements": ["a", "b"]}, {"score": 0, "relevant_statements": []}],
+            "results": [
+                {"score": 1, "relevant_statements": ["a", "b"], "status": "evaluated"},
+                {"score": 0, "relevant_statements": [], "status": "evaluated"},
+            ],
             "score": 0.5,
             "meta": None,
             "individual_scores": [1, 0],
@@ -207,13 +213,16 @@ def test_run_missing_parameters(self, monkeypatch):
         with pytest.raises(ValueError, match="LLM evaluator expected input parameter"):
             component.run()
 
-    def test_run_returns_nan_raise_on_failure_false(self, monkeypatch, caplog):
+    @pytest.mark.parametrize("failure_mode", ["generation", "parsing"])
+    def test_run_returns_nan_raise_on_failure_false(self, monkeypatch, caplog, failure_mode):
         monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
         component = ContextRelevanceEvaluator(raise_on_failure=False)
 
         def chat_generator_run(self, *args, **kwargs):
             if "Python" in kwargs["messages"][0].text:
-                raise Exception("OpenAI API request failed.")
+                if failure_mode == "generation":
+                    raise Exception("OpenAI API request failed.")
+                return {"replies": [ChatMessage.from_assistant("not valid JSON")]}
             return {"replies": [ChatMessage.from_assistant('{"relevant_statements": ["c", "d"], "score": 1}')]}
 
         monkeypatch.setattr("haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run", chat_generator_run)
@@ -236,12 +245,28 @@ def chat_generator_run(self, *args, **kwargs):
             results = component.run(questions=questions, contexts=contexts)
 
         assert results["score"] == 1
-        assert results["results"][0] == {"relevant_statements": ["c", "d"], "score": 1}
+        assert results["results"][0] == {"relevant_statements": ["c", "d"], "score": 1, "status": "evaluated"}
         assert results["results"][1]["relevant_statements"] == []
         assert math.isnan(results["results"][1]["score"])
+        assert results["results"][1]["status"] == "error"
 
         assert "1 query(s) failed and were excluded from the score." in caplog.text
 
+    def test_run_all_failures_returns_nan_and_error_statuses(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = ContextRelevanceEvaluator(raise_on_failure=False)
+
+        def chat_generator_run(self, *args, **kwargs):
+            raise Exception("OpenAI API request failed.")
+
+        monkeypatch.setattr("haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run", chat_generator_run)
+
+        results = component.run(questions=["q1", "q2"], contexts=[["c1"], ["c2"]])
+
+        assert math.isnan(results["score"])
+        assert all(math.isnan(score) for score in results["individual_scores"])
+        assert [result["status"] for result in results["results"]] == ["error", "error"]
+
     @pytest.mark.skipif(
         not os.environ.get("OPENAI_API_KEY", None),
         reason="Export an env var called OPENAI_API_KEY containing the OpenAI API key to run this test.",
@@ -256,7 +281,7 @@ def test_live_run(self):
 
         required_fields = {"results"}
         assert all(field in result for field in required_fields)
-        nested_required_fields = {"score", "relevant_statements"}
+        nested_required_fields = {"score", "relevant_statements", "status"}
         assert all(field in result["results"][0] for field in nested_required_fields)
 
         assert "meta" in result
@@ -289,20 +314,26 @@ async def chat_generator_run_async(self, *args, **kwargs):
         results = await component.run_async(questions=questions, contexts=contexts)
 
         assert results == {
-            "results": [{"score": 1, "relevant_statements": ["a", "b"]}, {"score": 0, "relevant_statements": []}],
+            "results": [
+                {"score": 1, "relevant_statements": ["a", "b"], "status": "evaluated"},
+                {"score": 0, "relevant_statements": [], "status": "evaluated"},
+            ],
             "score": 0.5,
             "meta": None,
             "individual_scores": [1, 0],
         }
 
     @pytest.mark.asyncio
-    async def test_run_async_returns_nan_raise_on_failure_false(self, monkeypatch, caplog):
+    @pytest.mark.parametrize("failure_mode", ["generation", "parsing"])
+    async def test_run_async_returns_nan_raise_on_failure_false(self, monkeypatch, caplog, failure_mode):
         monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
         component = ContextRelevanceEvaluator(raise_on_failure=False)
 
         async def chat_generator_run_async(self, *args, **kwargs):
             if "Python" in kwargs["messages"][0].text:
-                raise Exception("OpenAI API request failed.")
+                if failure_mode == "generation":
+                    raise Exception("OpenAI API request failed.")
+                return {"replies": [ChatMessage.from_assistant("not valid JSON")]}
             return {"replies": [ChatMessage.from_assistant('{"relevant_statements": ["c", "d"], "score": 1}')]}
 
         monkeypatch.setattr(
@@ -316,9 +347,10 @@ async def chat_generator_run_async(self, *args, **kwargs):
             results = await component.run_async(questions=questions, contexts=contexts)
 
         assert results["score"] == 1
-        assert results["results"][0] == {"relevant_statements": ["c", "d"], "score": 1}
+        assert results["results"][0] == {"relevant_statements": ["c", "d"], "score": 1, "status": "evaluated"}
         assert results["results"][1]["relevant_statements"] == []
         assert math.isnan(results["results"][1]["score"])
+        assert results["results"][1]["status"] == "error"
 
         assert "1 query(s) failed and were excluded from the score." in caplog.text
 
@@ -337,7 +369,7 @@ async def test_live_run_async(self):
 
         required_fields = {"results"}
         assert all(field in result for field in required_fields)
-        nested_required_fields = {"score", "relevant_statements"}
+        nested_required_fields = {"score", "relevant_statements", "status"}
         assert all(field in result["results"][0] for field in nested_required_fields)
 
         assert "meta" in result