fix: add missing output sockes to LLM evaluators (#12049)

Author: anakin87

haystack/components/evaluators/context_relevance.py

@@ -162,7 +162,9 @@ def __init__(
             progress_bar=progress_bar,
         )
 
-    @component.output_types(score=float, results=list[dict[str, Any]])
+    @component.output_types(
+        score=float, individual_scores=list[float], results=list[dict[str, Any]], meta=list[dict[str, Any]] | None
+    )
     def run(self, **inputs: Any) -> dict[str, Any]:
         """
         Run the LLM evaluator.
@@ -174,14 +176,17 @@ def run(self, **inputs: Any) -> dict[str, Any]:
         :returns:
             A dictionary with the following outputs:
                 - `score`: Mean context relevance score over all the provided input questions.
+                - `individual_scores`: A list of context relevance scores for each input question.
                 - `results`: A list of dictionaries with `relevant_statements`, `score`, and `status` for each input
                   context. `status` is `evaluated` for valid results and `error` for failed evaluations.
         """
         result = super(ContextRelevanceEvaluator, self).run(**inputs)  # noqa: UP008
         # Post-process the raw results to calculate relevance metrics and scores
         return self._postprocess_results(result)
 
-    @component.output_types(score=float, results=list[dict[str, Any]])
+    @component.output_types(
+        score=float, individual_scores=list[float], results=list[dict[str, Any]], meta=list[dict[str, Any]] | None
+    )
     async def run_async(self, **inputs: Any) -> dict[str, Any]:
         """
         Run the LLM evaluator asynchronously.
@@ -193,6 +198,7 @@ async def run_async(self, **inputs: Any) -> dict[str, Any]:
         :returns:
             A dictionary with the following outputs:
                 - `score`: Mean context relevance score over all the provided input questions.
+                - `individual_scores`: A list of context relevance scores for each input question.
                 - `results`: A list of dictionaries with `relevant_statements`, `score`, and `status` for each input
                   context. `status` is `evaluated` for valid results and `error` for failed evaluations.
         """

haystack/components/evaluators/faithfulness.py

@@ -150,7 +150,9 @@ def __init__(
             progress_bar=progress_bar,
         )
 
-    @component.output_types(individual_scores=list[float], score=float, results=list[dict[str, Any]])
+    @component.output_types(
+        individual_scores=list[float], score=float, results=list[dict[str, Any]], meta=list[dict[str, Any]] | None
+    )
     def run(self, **inputs: Any) -> dict[str, Any]:
         """
         Run the LLM evaluator.
@@ -172,7 +174,9 @@ def run(self, **inputs: Any) -> dict[str, Any]:
         # Post-process the raw results to calculate relevance metrics and scores
         return self._postprocess_results(result)
 
-    @component.output_types(individual_scores=list[float], score=float, results=list[dict[str, Any]])
+    @component.output_types(
+        individual_scores=list[float], score=float, results=list[dict[str, Any]], meta=list[dict[str, Any]] | None
+    )
     async def run_async(self, **inputs: Any) -> dict[str, Any]:
         """
         Run the LLM evaluator asynchronously.

haystack/components/evaluators/llm_evaluator.py

@@ -198,7 +198,7 @@ def validate_init_parameters(
                 )
                 raise ValueError(msg)
 
-    @component.output_types(results=list[dict[str, Any]])
+    @component.output_types(results=list[dict[str, Any]], meta=list[dict[str, Any]] | None)
     def run(self, **inputs: Any) -> dict[str, Any]:
         """
         Run the LLM evaluator.
@@ -262,7 +262,7 @@ def run(self, **inputs: Any) -> dict[str, Any]:
 
         return {"results": results, "meta": metadata or None}
 
-    @component.output_types(results=list[dict[str, Any]])
+    @component.output_types(results=list[dict[str, Any]], meta=list[dict[str, Any]] | None)
     async def run_async(self, **inputs: Any) -> dict[str, Any]:
         """
         Run the LLM evaluator asynchronously

releasenotes/notes/declare-evaluator-output-sockets-3a1f5c2e4b7d9018.yaml

@@ -0,0 +1,6 @@
+---
+fixes:
+  - |
+    Declare the ``meta`` output of ``LLMEvaluator``, ``ContextRelevanceEvaluator``, and ``FaithfulnessEvaluator``, and
+    the ``individual_scores`` output of ``ContextRelevanceEvaluator``. These outputs were already returned but not
+    declared, so they could not be connected to other components in a Pipeline.

test/components/evaluators/test_context_relevance_evaluator.py

@@ -55,6 +55,8 @@ def test_init_default(self, monkeypatch):
         assert component._chat_generator.api_key.resolve_value() == "test-api-key"
         assert component._chat_generator.generation_kwargs == {"response_format": {"type": "json_object"}, "seed": 42}
 
+        assert set(component.__haystack_output__._sockets_dict) == {"score", "individual_scores", "results", "meta"}
+
     def test_key_resolved_at_warm_up_not_init(self, monkeypatch):
         monkeypatch.delenv("OPENAI_API_KEY", raising=False)
         component = ContextRelevanceEvaluator()

test/components/evaluators/test_faithfulness_evaluator.py

@@ -70,6 +70,8 @@ def test_init_default(self, monkeypatch):
         assert component._chat_generator.api_key.resolve_value() == "test-api-key"
         assert component._chat_generator.generation_kwargs == {"response_format": {"type": "json_object"}, "seed": 42}
 
+        assert set(component.__haystack_output__._sockets_dict) == {"score", "individual_scores", "results", "meta"}
+
     def test_key_resolved_at_warm_up_not_init(self, monkeypatch):
         monkeypatch.delenv("OPENAI_API_KEY", raising=False)
         component = FaithfulnessEvaluator()

test/components/evaluators/test_llm_evaluator.py

@@ -33,6 +33,8 @@ def test_init_default(self, monkeypatch):
         assert isinstance(component._chat_generator, OpenAIChatGenerator)
         assert component._chat_generator.generation_kwargs == {"response_format": {"type": "json_object"}, "seed": 42}
 
+        assert set(component.__haystack_output__._sockets_dict) == {"results", "meta"}
+
     def test_key_resolved_at_warm_up_not_init(self, monkeypatch):
         monkeypatch.delenv("OPENAI_API_KEY", raising=False)
         component = LLMEvaluator(