diff --git a/haystack/components/evaluators/context_relevance.py b/haystack/components/evaluators/context_relevance.py index 993a9353bc..e300752629 100644 --- a/haystack/components/evaluators/context_relevance.py +++ b/haystack/components/evaluators/context_relevance.py @@ -162,7 +162,9 @@ def __init__( progress_bar=progress_bar, ) - @component.output_types(score=float, results=list[dict[str, Any]]) + @component.output_types( + score=float, individual_scores=list[float], results=list[dict[str, Any]], meta=list[dict[str, Any]] | None + ) def run(self, **inputs: Any) -> dict[str, Any]: """ Run the LLM evaluator. @@ -174,6 +176,7 @@ def run(self, **inputs: Any) -> dict[str, Any]: :returns: A dictionary with the following outputs: - `score`: Mean context relevance score over all the provided input questions. + - `individual_scores`: A list of context relevance scores for each input question. - `results`: A list of dictionaries with `relevant_statements`, `score`, and `status` for each input context. `status` is `evaluated` for valid results and `error` for failed evaluations. """ @@ -181,7 +184,9 @@ def run(self, **inputs: Any) -> dict[str, Any]: # Post-process the raw results to calculate relevance metrics and scores return self._postprocess_results(result) - @component.output_types(score=float, results=list[dict[str, Any]]) + @component.output_types( + score=float, individual_scores=list[float], results=list[dict[str, Any]], meta=list[dict[str, Any]] | None + ) async def run_async(self, **inputs: Any) -> dict[str, Any]: """ Run the LLM evaluator asynchronously. @@ -193,6 +198,7 @@ async def run_async(self, **inputs: Any) -> dict[str, Any]: :returns: A dictionary with the following outputs: - `score`: Mean context relevance score over all the provided input questions. + - `individual_scores`: A list of context relevance scores for each input question. - `results`: A list of dictionaries with `relevant_statements`, `score`, and `status` for each input context. `status` is `evaluated` for valid results and `error` for failed evaluations. """ diff --git a/haystack/components/evaluators/faithfulness.py b/haystack/components/evaluators/faithfulness.py index 2a0542ab0e..eb7bd2f53e 100644 --- a/haystack/components/evaluators/faithfulness.py +++ b/haystack/components/evaluators/faithfulness.py @@ -150,7 +150,9 @@ def __init__( progress_bar=progress_bar, ) - @component.output_types(individual_scores=list[float], score=float, results=list[dict[str, Any]]) + @component.output_types( + individual_scores=list[float], score=float, results=list[dict[str, Any]], meta=list[dict[str, Any]] | None + ) def run(self, **inputs: Any) -> dict[str, Any]: """ Run the LLM evaluator. @@ -172,7 +174,9 @@ def run(self, **inputs: Any) -> dict[str, Any]: # Post-process the raw results to calculate relevance metrics and scores return self._postprocess_results(result) - @component.output_types(individual_scores=list[float], score=float, results=list[dict[str, Any]]) + @component.output_types( + individual_scores=list[float], score=float, results=list[dict[str, Any]], meta=list[dict[str, Any]] | None + ) async def run_async(self, **inputs: Any) -> dict[str, Any]: """ Run the LLM evaluator asynchronously. diff --git a/haystack/components/evaluators/llm_evaluator.py b/haystack/components/evaluators/llm_evaluator.py index 42e5c7b255..505571fb72 100644 --- a/haystack/components/evaluators/llm_evaluator.py +++ b/haystack/components/evaluators/llm_evaluator.py @@ -198,7 +198,7 @@ def validate_init_parameters( ) raise ValueError(msg) - @component.output_types(results=list[dict[str, Any]]) + @component.output_types(results=list[dict[str, Any]], meta=list[dict[str, Any]] | None) def run(self, **inputs: Any) -> dict[str, Any]: """ Run the LLM evaluator. @@ -262,7 +262,7 @@ def run(self, **inputs: Any) -> dict[str, Any]: return {"results": results, "meta": metadata or None} - @component.output_types(results=list[dict[str, Any]]) + @component.output_types(results=list[dict[str, Any]], meta=list[dict[str, Any]] | None) async def run_async(self, **inputs: Any) -> dict[str, Any]: """ Run the LLM evaluator asynchronously diff --git a/releasenotes/notes/declare-evaluator-output-sockets-3a1f5c2e4b7d9018.yaml b/releasenotes/notes/declare-evaluator-output-sockets-3a1f5c2e4b7d9018.yaml new file mode 100644 index 0000000000..39abd97970 --- /dev/null +++ b/releasenotes/notes/declare-evaluator-output-sockets-3a1f5c2e4b7d9018.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Declare the ``meta`` output of ``LLMEvaluator``, ``ContextRelevanceEvaluator``, and ``FaithfulnessEvaluator``, and + the ``individual_scores`` output of ``ContextRelevanceEvaluator``. These outputs were already returned but not + declared, so they could not be connected to other components in a Pipeline. diff --git a/test/components/evaluators/test_context_relevance_evaluator.py b/test/components/evaluators/test_context_relevance_evaluator.py index b357d8b64c..647a994dec 100644 --- a/test/components/evaluators/test_context_relevance_evaluator.py +++ b/test/components/evaluators/test_context_relevance_evaluator.py @@ -55,6 +55,8 @@ def test_init_default(self, monkeypatch): assert component._chat_generator.api_key.resolve_value() == "test-api-key" assert component._chat_generator.generation_kwargs == {"response_format": {"type": "json_object"}, "seed": 42} + assert set(component.__haystack_output__._sockets_dict) == {"score", "individual_scores", "results", "meta"} + def test_key_resolved_at_warm_up_not_init(self, monkeypatch): monkeypatch.delenv("OPENAI_API_KEY", raising=False) component = ContextRelevanceEvaluator() diff --git a/test/components/evaluators/test_faithfulness_evaluator.py b/test/components/evaluators/test_faithfulness_evaluator.py index 49b3fc22cb..7cf54b0450 100644 --- a/test/components/evaluators/test_faithfulness_evaluator.py +++ b/test/components/evaluators/test_faithfulness_evaluator.py @@ -70,6 +70,8 @@ def test_init_default(self, monkeypatch): assert component._chat_generator.api_key.resolve_value() == "test-api-key" assert component._chat_generator.generation_kwargs == {"response_format": {"type": "json_object"}, "seed": 42} + assert set(component.__haystack_output__._sockets_dict) == {"score", "individual_scores", "results", "meta"} + def test_key_resolved_at_warm_up_not_init(self, monkeypatch): monkeypatch.delenv("OPENAI_API_KEY", raising=False) component = FaithfulnessEvaluator() diff --git a/test/components/evaluators/test_llm_evaluator.py b/test/components/evaluators/test_llm_evaluator.py index 71724ae99c..ee2c512f85 100644 --- a/test/components/evaluators/test_llm_evaluator.py +++ b/test/components/evaluators/test_llm_evaluator.py @@ -33,6 +33,8 @@ def test_init_default(self, monkeypatch): assert isinstance(component._chat_generator, OpenAIChatGenerator) assert component._chat_generator.generation_kwargs == {"response_format": {"type": "json_object"}, "seed": 42} + assert set(component.__haystack_output__._sockets_dict) == {"results", "meta"} + def test_key_resolved_at_warm_up_not_init(self, monkeypatch): monkeypatch.delenv("OPENAI_API_KEY", raising=False) component = LLMEvaluator(