fix: reject incomplete evaluation inputs

Several evaluators paired actual and expected invocations with a plain zip, so extra turns were silently dropped and a truncated run could still pass. The rubric parser likewise discarded incomplete fields, and empty multi-turn inputs could crash. This validates invocation counts and pairs strictly, rejects partially parsed rubric responses, and returns a not-evaluated result for empty multi-turn inputs.

Co-authored-by: George Weale <gweale@google.com>
PiperOrigin-RevId: 949253981

Author: GWeale

src/google/adk/evaluation/evaluator.py

@@ -26,6 +26,21 @@
 from .eval_rubrics import RubricScore
 
 
+def _validate_invocation_lengths(
+    actual_invocations: list[Invocation],
+    expected_invocations: Optional[list[Invocation]],
+) -> None:
+  """Rejects invocation lists that cannot be paired without truncation."""
+  if expected_invocations is not None and len(actual_invocations) != len(
+      expected_invocations
+  ):
+    raise ValueError(
+        "actual_invocations and expected_invocations must have the same"
+        f" length; got {len(actual_invocations)} and"
+        f" {len(expected_invocations)}."
+    )
+
+
 class PerInvocationResult(BaseModel):
   """Metric evaluation score per invocation."""
 

src/google/adk/evaluation/final_response_match_v1.py

@@ -23,6 +23,7 @@
 from .eval_case import ConversationScenario
 from .eval_case import Invocation
 from .eval_metrics import EvalMetric
+from .evaluator import _validate_invocation_lengths
 from .evaluator import EvalStatus
 from .evaluator import EvaluationResult
 from .evaluator import Evaluator
@@ -47,12 +48,15 @@ def evaluate_invocations(
   ) -> EvaluationResult:
     if expected_invocations is None:
       raise ValueError("expected_invocations is required for this metric.")
+    _validate_invocation_lengths(actual_invocations, expected_invocations)
     del conversation_scenario  # not used by this metric.
 
     total_score = 0.0
     num_invocations = 0
     per_invocation_results = []
-    for actual, expected in zip(actual_invocations, expected_invocations):
+    for actual, expected in zip(
+        actual_invocations, expected_invocations, strict=True
+    ):
       reference = _get_text_from_content(expected.final_response)
       response = _get_text_from_content(actual.final_response)
       rouge_1_scores = _calculate_rouge_1_scores(response, reference)

src/google/adk/evaluation/final_response_match_v2.py

@@ -102,7 +102,7 @@ def _parse_critique(response: str) -> Label:
   )
   # Remove any trailing whitespace, commas, or end-brackets from the label.
   if label_match_is_response_valid:
-    label = label_match_is_response_valid.group(1).strip(r"\s,\}")
+    label = label_match_is_response_valid.group(1).strip().rstrip(",}")
     if label in [
         Label.INVALID.value,
         Label.ALMOST.value,
@@ -115,7 +115,7 @@ def _parse_critique(response: str) -> Label:
     else:
       label = Label.NOT_FOUND
   elif label_match_is_response_invalid:
-    label = label_match_is_response_invalid.group(1).strip(r"\s,\}")
+    label = label_match_is_response_invalid.group(1).strip().rstrip(",}")
     label = (
         Label.INVALID
         if label in [Label.TRUE.value, Label.INVALID.value]

src/google/adk/evaluation/hallucinations_v1.py

@@ -39,6 +39,7 @@
 from .eval_case import InvocationEvents
 from .eval_metrics import EvalMetric
 from .eval_metrics import HallucinationsCriterion
+from .evaluator import _validate_invocation_lengths
 from .evaluator import EvalStatus
 from .evaluator import EvaluationResult
 from .evaluator import Evaluator
@@ -704,6 +705,7 @@ async def evaluate_invocations(
       conversation_scenario: Optional[ConversationScenario] = None,
   ) -> EvaluationResult:
     del conversation_scenario  # not used by this metric.
+    _validate_invocation_lengths(actual_invocations, expected_invocations)
 
     # expected_invocations are not required by the metric and if they are not
     # supplied, we provide a list of None to rest of the code.
@@ -714,7 +716,9 @@ async def evaluate_invocations(
     )
 
     per_invocation_results = []
-    for actual, expected in zip(actual_invocations, expected_invocations):
+    for actual, expected in zip(
+        actual_invocations, expected_invocations, strict=True
+    ):
       step_evaluations = self._get_steps_to_evaluate(actual)
 
       if not step_evaluations:

src/google/adk/evaluation/llm_as_judge.py

@@ -34,6 +34,7 @@
 from .eval_metrics import BaseCriterion
 from .eval_metrics import EvalMetric
 from .eval_metrics import RubricScore
+from .evaluator import _validate_invocation_lengths
 from .evaluator import EvaluationResult
 from .evaluator import Evaluator
 from .evaluator import PerInvocationResult
@@ -123,6 +124,7 @@ async def evaluate_invocations(
   ) -> EvaluationResult:
     if self._expected_invocations_required and expected_invocations is None:
       raise ValueError("expected_invocations is needed by this metric.")
+    _validate_invocation_lengths(actual_invocations, expected_invocations)
     del conversation_scenario  # not supported for per-invocation evaluation.
 
     # If expected_invocation are not required by the metric and if they are not
@@ -134,7 +136,9 @@ async def evaluate_invocations(
     )
 
     per_invocation_results = []
-    for actual, expected in zip(actual_invocations, expected_invocations):
+    for actual, expected in zip(
+        actual_invocations, expected_invocations, strict=True
+    ):
       auto_rater_prompt = self.format_auto_rater_prompt(actual, expected)
       llm_request = LlmRequest(
           model=self._judge_model_options.judge_model,

src/google/adk/evaluation/rubric_based_evaluator.py

@@ -80,8 +80,12 @@ def parse(self, auto_rater_response: str) -> list[RubricResponse]:
 
       scores.append(score)
 
+    # A partial parse can silently omit a failed rubric and inflate the score.
+    if not len(properties) == len(rationales) == len(scores):
+      return []
+
     rubric_responses = []
-    for p, r, s in zip(properties, rationales, scores):
+    for p, r, s in zip(properties, rationales, scores, strict=True):
       rubric_responses.append(
           RubricResponse(property_text=p.strip(), rationale=r.strip(), score=s)
       )

src/google/adk/evaluation/rubric_based_multi_turn_trajectory_evaluator.py

@@ -27,6 +27,7 @@
 from .eval_metrics import EvalMetric
 from .eval_metrics import EvalStatus
 from .eval_metrics import RubricsBasedCriterion
+from .evaluator import _validate_invocation_lengths
 from .evaluator import EvaluationResult
 from .evaluator import PerInvocationResult
 from .rubric_based_evaluator import RubricBasedEvaluator
@@ -278,6 +279,9 @@ async def evaluate_invocations(
         "Local evaluator start (invocations: %d)",
         len(actual_invocations),
     )
+    _validate_invocation_lengths(actual_invocations, expected_invocations)
+    if not actual_invocations:
+      return EvaluationResult()
     self._assemble_dialogue_history(actual_invocations)
 
     # If expected_invocations are not supplied, provide a list of None.
@@ -290,7 +294,7 @@ async def evaluate_invocations(
     # Mark the first N-1 turns as NOT_EVALUATED.
     per_invocation_results = []
     for actual, expected in zip(
-        actual_invocations[:-1], expected_invocations[:-1]
+        actual_invocations[:-1], expected_invocations[:-1], strict=True
     ):
       per_invocation_results.append(
           PerInvocationResult(

src/google/adk/evaluation/trajectory_evaluator.py

@@ -27,6 +27,7 @@
 from .eval_case import Invocation
 from .eval_metrics import EvalMetric
 from .eval_metrics import ToolTrajectoryCriterion
+from .evaluator import _validate_invocation_lengths
 from .evaluator import EvalStatus
 from .evaluator import EvaluationResult
 from .evaluator import Evaluator
@@ -105,13 +106,16 @@ def evaluate_invocations(
     """Returns EvaluationResult after performing evaluations using actual and expected invocations."""
     if expected_invocations is None:
       raise ValueError("expected_invocations is needed by this metric.")
+    _validate_invocation_lengths(actual_invocations, expected_invocations)
     del conversation_scenario  # not supported for per-invocation evaluation.
 
     total_tool_use_accuracy = 0.0
     num_invocations = 0
     per_invocation_results = []
 
-    for actual, expected in zip(actual_invocations, expected_invocations):
+    for actual, expected in zip(
+        actual_invocations, expected_invocations, strict=True
+    ):
       tool_use_accuracy = self._calculate_tool_use_accuracy(actual, expected)
       per_invocation_results.append(
           PerInvocationResult(
@@ -259,7 +263,9 @@ def _are_tool_calls_exact_match(
     if len(actual_tool_calls) != len(expected_tool_calls):
       return False
 
-    for actual, expected in zip(actual_tool_calls, expected_tool_calls):
+    for actual, expected in zip(
+        actual_tool_calls, expected_tool_calls, strict=True
+    ):
       if actual.name != expected.name or actual.args != expected.args:
         return False
 

src/google/adk/evaluation/vertex_ai_eval_facade.py

@@ -30,6 +30,7 @@
 from .eval_case import ConversationScenario
 from .eval_case import Invocation
 from .eval_case import InvocationEvent
+from .evaluator import _validate_invocation_lengths
 from .evaluator import EvalStatus
 from .evaluator import EvaluationResult
 from .evaluator import Evaluator
@@ -157,6 +158,7 @@ def evaluate_invocations(
   ) -> EvaluationResult:
     if self._expected_invocations_required and expected_invocations is None:
       raise ValueError("expected_invocations is needed by this metric.")
+    _validate_invocation_lengths(actual_invocations, expected_invocations)
     del conversation_scenario  # not supported for per-invocation evaluation.
 
     # If expected_invocation are not required by the metric and if they are not
@@ -170,7 +172,9 @@ def evaluate_invocations(
     total_score = 0.0
     num_invocations = 0
     per_invocation_results = []
-    for actual, expected in zip(actual_invocations, expected_invocations):
+    for actual, expected in zip(
+        actual_invocations, expected_invocations, strict=True
+    ):
       prompt = self._get_text(actual.user_content)
       reference = self._get_text(expected.final_response) if expected else None
       response = self._get_text(actual.final_response)
@@ -224,6 +228,9 @@ def evaluate_invocations(
       conversation_scenario: Optional[ConversationScenario] = None,
   ) -> EvaluationResult:
     del conversation_scenario
+    _validate_invocation_lengths(actual_invocations, expected_invocations)
+    if not actual_invocations:
+      return EvaluationResult()
 
     per_invocation_results = []
     # If expected_invocation are not required by the metric and if they are not
@@ -236,7 +243,7 @@ def evaluate_invocations(
 
     # We mark all the n-1 turns as NOT-EVALUATED for these metrics.
     for actual, expected in zip(
-        actual_invocations[:-1], expected_invocations[:-1]
+        actual_invocations[:-1], expected_invocations[:-1], strict=True
     ):
       per_invocation_results.append(
           PerInvocationResult(

tests/unittests/evaluation/test_final_response_match_v1.py

@@ -139,3 +139,25 @@ def test_rouge_evaluator_multiple_invocations(
       expected_score, rel=1e-3
   )
   assert evaluation_result.overall_eval_status == expected_status
+
+
+@pytest.mark.parametrize(
+    "actual_count, expected_count",
+    [
+        pytest.param(2, 1, id="extra-actual-turn"),
+        pytest.param(1, 2, id="missing-actual-turn"),
+    ],
+)
+def test_rouge_evaluator_rejects_mismatched_invocation_lengths(
+    actual_count: int, expected_count: int
+):
+  actual, expected = _create_test_invocations("same", "same")
+  rouge_evaluator = _create_test_rouge_evaluator(threshold=0.8)
+
+  with pytest.raises(
+      ValueError,
+      match=f"same length; got {actual_count} and {expected_count}",
+  ):
+    rouge_evaluator.evaluate_invocations(
+        [actual] * actual_count, [expected] * expected_count
+    )