fix(eval): include intermediate text in final response match (#5698)

Author: he-yufeng

src/google/adk/evaluation/final_response_match_v2.py

@@ -159,13 +159,22 @@ def format_auto_rater_prompt(
     if expected_invocation is None:
       raise ValueError("expected_invocation is required for this metric.")
 
-    reference = get_text_from_content(expected_invocation.final_response)
-    response = get_text_from_content(actual_invocation.final_response)
+    include_intermediate = (
+        self._criterion.include_intermediate_responses_in_final
+    )
+    reference = get_text_from_content(
+        expected_invocation,
+        include_intermediate_responses_in_final=include_intermediate,
+    )
+    response = get_text_from_content(
+        actual_invocation,
+        include_intermediate_responses_in_final=include_intermediate,
+    )
     user_prompt = get_text_from_content(expected_invocation.user_content)
     return self._auto_rater_prompt_template.format(
         prompt=user_prompt,
-        response=response,
-        golden_response=reference,
+        response=response or "",
+        golden_response=reference or "",
     )
 
   @override

src/google/adk/evaluation/llm_as_judge_utils.py

@@ -25,6 +25,7 @@
 from .app_details import AppDetails
 from .common import EvalBaseModel
 from .eval_case import get_all_tool_calls_with_responses
+from .eval_case import IntermediateData
 from .eval_case import IntermediateDataType
 from .eval_case import Invocation
 from .eval_case import InvocationEvents
@@ -71,6 +72,12 @@ def get_text_from_content(
         text = get_text_from_content(event.content)
         if text:
           parts.append(text)
+    elif isinstance(content.intermediate_data, IntermediateData):
+      for _, response_parts in content.intermediate_data.intermediate_responses:
+        text = get_text_from_content(genai_types.Content(parts=response_parts))
+        if text:
+          parts.append(text)
+
     # Then fetch the final response text and append it to the end.
     final_text = get_text_from_content(content.final_response)
     if final_text:

tests/unittests/evaluation/test_final_response_match_v2.py

@@ -15,6 +15,8 @@
 from __future__ import annotations
 
 from google.adk.evaluation.eval_case import Invocation
+from google.adk.evaluation.eval_case import InvocationEvent
+from google.adk.evaluation.eval_case import InvocationEvents
 from google.adk.evaluation.eval_metrics import BaseCriterion
 from google.adk.evaluation.eval_metrics import EvalMetric
 from google.adk.evaluation.eval_metrics import EvalStatus
@@ -127,13 +129,18 @@ def create_test_template() -> str:
 
 def _create_test_evaluator_gemini(
     threshold: float,
+    *,
+    include_intermediate_responses_in_final: bool = False,
 ) -> FinalResponseMatchV2Evaluator:
   evaluator = FinalResponseMatchV2Evaluator(
       EvalMetric(
           metric_name="final_response_match_v2",
           threshold=threshold,
           criterion=BaseCriterion(
               threshold=0.5,
+              include_intermediate_responses_in_final=(
+                  include_intermediate_responses_in_final
+              ),
           ),
       ),
   )
@@ -168,6 +175,21 @@ def _create_test_invocations(
   return actual_invocation, expected_invocation
 
 
+def _add_intermediate_text(invocation: Invocation, text: str) -> Invocation:
+  invocation.intermediate_data = InvocationEvents(
+      invocation_events=[
+          InvocationEvent(
+              author="agent",
+              content=genai_types.Content(
+                  parts=[genai_types.Part(text=text)],
+                  role="model",
+              ),
+          ),
+      ]
+  )
+  return invocation
+
+
 def test_format_auto_rater_prompt():
   evaluator = _create_test_evaluator_gemini(threshold=0.8)
   actual_invocation, expected_invocation = _create_test_invocations(
@@ -193,6 +215,59 @@ def test_format_auto_rater_prompt():
 """
 
 
+def test_format_auto_rater_prompt_uses_empty_text_for_missing_final_response():
+  evaluator = _create_test_evaluator_gemini(threshold=0.8)
+  actual_invocation, expected_invocation = _create_test_invocations(
+      "candidate text", "reference text"
+  )
+  actual_invocation.final_response = None
+  expected_invocation.final_response = None
+
+  prompt = evaluator.format_auto_rater_prompt(
+      actual_invocation, expected_invocation
+  )
+
+  assert "None" not in prompt
+  assert '"Agent response": ,' in prompt
+  assert '"Reference response": ,' in prompt
+
+
+def test_format_auto_rater_prompt_ignores_intermediate_by_default():
+  evaluator = _create_test_evaluator_gemini(threshold=0.8)
+  actual_invocation, expected_invocation = _create_test_invocations(
+      "candidate final", "reference final"
+  )
+  _add_intermediate_text(actual_invocation, "candidate intro")
+  _add_intermediate_text(expected_invocation, "reference intro")
+
+  prompt = evaluator.format_auto_rater_prompt(
+      actual_invocation, expected_invocation
+  )
+
+  assert "candidate final" in prompt
+  assert "reference final" in prompt
+  assert "candidate intro" not in prompt
+  assert "reference intro" not in prompt
+
+
+def test_format_auto_rater_prompt_includes_intermediate_when_enabled():
+  evaluator = _create_test_evaluator_gemini(
+      threshold=0.8, include_intermediate_responses_in_final=True
+  )
+  actual_invocation, expected_invocation = _create_test_invocations(
+      "candidate final", "reference final"
+  )
+  _add_intermediate_text(actual_invocation, "candidate intro")
+  _add_intermediate_text(expected_invocation, "reference intro")
+
+  prompt = evaluator.format_auto_rater_prompt(
+      actual_invocation, expected_invocation
+  )
+
+  assert "candidate intro\ncandidate final" in prompt
+  assert "reference intro\nreference final" in prompt
+
+
 def test_convert_auto_rater_response_to_score_valid():
   evaluator = _create_test_evaluator_gemini(threshold=0.8)
   auto_rater_response = """```json

tests/unittests/evaluation/test_llm_as_judge_utils.py

@@ -132,6 +132,36 @@ def test_get_text_from_content_with_invocation_include_intermediate_responses_in
   )
 
 
+def test_get_text_from_content_with_intermediate_data_full_response():
+  invocation = Invocation(
+      user_content=genai_types.Content(parts=[genai_types.Part(text="user")]),
+      intermediate_data=IntermediateData(
+          intermediate_responses=[
+              ("agent", [genai_types.Part(text="legacy intro")]),
+              (
+                  "tool",
+                  [
+                      genai_types.Part(
+                          function_call=genai_types.FunctionCall(name="lookup")
+                      )
+                  ],
+              ),
+          ]
+      ),
+      final_response=genai_types.Content(
+          parts=[genai_types.Part(text="final answer")]
+      ),
+  )
+
+  assert get_text_from_content(invocation) == "final answer"
+  assert (
+      get_text_from_content(
+          invocation, include_intermediate_responses_in_final=True
+      )
+      == "legacy intro\nfinal answer"
+  )
+
+
 def test_get_eval_status_with_none_score():
   """Tests get_eval_status returns NOT_EVALUATED for a None score."""
   assert get_eval_status(score=None, threshold=0.5) == EvalStatus.NOT_EVALUATED