feat: expose LLM evaluator row statuses

Author: SeCuReDmE-main-dev

haystack/components/evaluators/llm_evaluator.py

@@ -175,7 +175,7 @@ def validate_init_parameters(
                 )
                 raise ValueError(msg)
 
-    @component.output_types(results=list[dict[str, Any]])
+    @component.output_types(results=list[dict[str, Any]], evaluation_statuses=list[str])
     def run(self, **inputs: Any) -> dict[str, Any]:
         """
         Run the LLM evaluator.
@@ -204,6 +204,7 @@ def run(self, **inputs: Any) -> dict[str, Any]:
         list_of_input_names_to_values = [dict(zip(input_names, v, strict=True)) for v in values]
 
         results: list[dict[str, Any] | None] = []
+        evaluation_statuses: list[str] = []
         metadata = []
         errors = 0
         for input_names_to_values in tqdm(list_of_input_names_to_values, disable=not self.progress_bar):
@@ -216,6 +217,7 @@ def run(self, **inputs: Any) -> dict[str, Any]:
                     raise ValueError(f"Error while generating response for prompt: {prompt}. Error: {e}") from e
                 logger.warning("Error while generating response for prompt: {prompt}. Error: {e}", prompt=prompt, e=e)
                 results.append(None)
+                evaluation_statuses.append("error")
                 errors += 1
                 continue
 
@@ -224,9 +226,11 @@ def run(self, **inputs: Any) -> dict[str, Any]:
             )
             if parsed_result is None:
                 results.append(None)
+                evaluation_statuses.append("error")
                 errors += 1
             else:
                 results.append(parsed_result)
+                evaluation_statuses.append("evaluated")
 
             if result["replies"][0].meta:
                 metadata.append(result["replies"][0].meta)
@@ -238,7 +242,7 @@ def run(self, **inputs: Any) -> dict[str, Any]:
                 len=len(list_of_input_names_to_values),
             )
 
-        return {"results": results, "meta": metadata or None}
+        return {"results": results, "meta": metadata or None, "evaluation_statuses": evaluation_statuses}
 
     def prepare_template(self) -> str:
         """

test/components/evaluators/test_context_relevance_evaluator.py

@@ -169,6 +169,7 @@ def chat_generator_run(self, *args, **kwargs):
             "score": 0.5,
             "meta": None,
             "individual_scores": [1, 0],
+            "evaluation_statuses": ["evaluated", "evaluated"],
         }
 
     def test_run_no_statements_extracted(self, monkeypatch):
@@ -198,6 +199,7 @@ def chat_generator_run(self, *args, **kwargs):
             "score": 0.5,
             "meta": None,
             "individual_scores": [1, 0],
+            "evaluation_statuses": ["evaluated", "evaluated"],
         }
 
     def test_run_missing_parameters(self, monkeypatch):
@@ -237,6 +239,7 @@ def chat_generator_run(self, *args, **kwargs):
         assert results["results"][0] == {"relevant_statements": ["c", "d"], "score": 1}
         assert results["results"][1]["relevant_statements"] == []
         assert math.isnan(results["results"][1]["score"])
+        assert results["evaluation_statuses"] == ["evaluated", "error"]
 
     @pytest.mark.skipif(
         not os.environ.get("OPENAI_API_KEY", None),

test/components/evaluators/test_faithfulness_evaluator.py

@@ -207,6 +207,7 @@ def chat_generator_run(self, *args, **kwargs):
             ],
             "score": 0.75,
             "meta": None,
+            "evaluation_statuses": ["evaluated", "evaluated"],
         }
 
     def test_run_no_statements_extracted(self, monkeypatch):
@@ -245,6 +246,7 @@ def chat_generator_run(self, *args, **kwargs):
             ],
             "score": 0.25,
             "meta": None,
+            "evaluation_statuses": ["evaluated", "evaluated"],
         }
 
     def test_run_missing_parameters(self, monkeypatch):
@@ -294,6 +296,7 @@ def chat_generator_run(self, *args, **kwargs):
         assert results["results"][1]["statements"] == []
         assert results["results"][1]["statement_scores"] == []
         assert math.isnan(results["results"][1]["score"])
+        assert results["evaluation_statuses"] == ["evaluated", "error"]
 
     @pytest.mark.skipif(
         not os.environ.get("OPENAI_API_KEY", None),

test/components/evaluators/test_llm_evaluator.py

@@ -332,7 +332,7 @@ def chat_generator_run(self, *args, **kwargs):
         monkeypatch.setattr("haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run", chat_generator_run)
 
         results = component.run(questions=["What is the capital of Germany?"], predicted_answers=["Berlin"])
-        assert results == {"results": [{"score": 0.5}], "meta": None}
+        assert results == {"results": [{"score": 0.5}], "meta": None, "evaluation_statuses": ["evaluated"]}
 
     def test_prepare_template(self, monkeypatch):
         monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
@@ -433,6 +433,28 @@ def chat_generator_run(self, *args, **kwargs):
 
         result = component.run(predicted_answers=["answer"])
         assert result["results"] == [None]
+        assert result["evaluation_statuses"] == ["error"]
+
+    def test_run_returns_error_status_raise_on_failure_false(self, monkeypatch):
+        monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")
+        component = LLMEvaluator(
+            instructions="test-instruction",
+            inputs=[("predicted_answers", list[str])],
+            outputs=["score"],
+            examples=[
+                {"inputs": {"predicted_answers": "Football is the most popular sport."}, "outputs": {"score": 0}}
+            ],
+            raise_on_failure=False,
+        )
+
+        def chat_generator_run(self, *args, **kwargs):
+            raise Exception("OpenAI API request failed.")
+
+        monkeypatch.setattr("haystack.components.evaluators.llm_evaluator.OpenAIChatGenerator.run", chat_generator_run)
+
+        result = component.run(predicted_answers=["answer"])
+        assert result["results"] == [None]
+        assert result["evaluation_statuses"] == ["error"]
 
     def test_output_invalid_json_raise_on_failure_true(self, monkeypatch):
         monkeypatch.setenv("OPENAI_API_KEY", "test-api-key")