fix: allow invocation-level rubrics

Merge #6161

Defer the missing-rubrics failure until the effective rubric list is built, after invocation rubrics have been merged. Both rubric-based prompt formatters now read rubrics through get_effective_rubrics_list(). CLI pretty printing now treats missing criterion rubrics as an empty lookup and falls back to the rubric id.

PiperOrigin-RevId: 952442674

Author: JaeCoding

src/google/adk/cli/cli_eval.py

@@ -36,6 +36,7 @@
 from ..evaluation.eval_case import get_all_tool_calls
 from ..evaluation.eval_case import IntermediateDataType
 from ..evaluation.eval_metrics import EvalMetric
+from ..evaluation.eval_metrics import RubricsBasedCriterion
 from ..evaluation.eval_result import EvalCaseResult
 from ..evaluation.eval_sets_manager import EvalSetsManager
 from ..utils.context_utils import Aclosing
@@ -219,9 +220,13 @@ def pretty_print_eval_result(eval_result: EvalCaseResult) -> None:
     )
     if metric_result.details and metric_result.details.rubric_scores:
       click.echo("Rubric Scores:")
+      rubrics = (
+          metric_result.criterion.rubrics
+          if isinstance(metric_result.criterion, RubricsBasedCriterion)
+          else None
+      ) or []
       rubrics_by_id = {
-          r["rubric_id"]: r["rubric_content"]["text_property"]
-          for r in metric_result.criterion.rubrics
+          r.rubric_id: r.rubric_content.text_property for r in rubrics
       }
       for rubric_score in metric_result.details.rubric_scores:
         rubric_text = rubrics_by_id.get(rubric_score.rubric_id)
@@ -262,9 +267,13 @@ def pretty_print_eval_result(eval_result: EvalCaseResult) -> None:
           f"Score: {metric_result.score}"
       )
       if metric_result.details and metric_result.details.rubric_scores:
+        rubrics = (
+            metric_result.criterion.rubrics
+            if isinstance(metric_result.criterion, RubricsBasedCriterion)
+            else None
+        ) or []
         rubrics_by_id = {
-            r["rubric_id"]: r["rubric_content"]["text_property"]
-            for r in metric_result.criterion.rubrics
+            r.rubric_id: r.rubric_content.text_property for r in rubrics
         }
         for rubric_score in metric_result.details.rubric_scores:
           rubric = rubrics_by_id.get(rubric_score.rubric_id)

src/google/adk/evaluation/rubric_based_evaluator.py

@@ -300,7 +300,7 @@ def summarize(
     )
 
 
-def _normalize_text(text: str) -> str:
+def _normalize_text(text: object) -> str:
   """Returns a normalized version of the passed in text."""
   if not isinstance(text, str):
     return ""
@@ -352,9 +352,7 @@ def __init__(
     self._per_invocation_results_aggregator = per_invocation_results_aggregator
     self._invocation_results_summarizer = invocation_results_summarizer
 
-    assert self._criterion.rubrics, "Rubrics are required."
-
-    self._rubrics: list[Rubric] = self._criterion.rubrics
+    self._rubrics: list[Rubric] = self._criterion.rubrics or []
     self._effective_rubrics_list: Optional[list[Rubric]] = None
 
     self._normalized_rubric_to_id_map = {
@@ -388,6 +386,8 @@ def _add_rubrics(rubrics_to_add: list[Rubric], scope_name: str) -> None:
       _add_rubrics(filtered_invocation_rubrics, "invocation")
 
     self._effective_rubrics_list = list(rubrics_by_id.values())
+    if not self._effective_rubrics_list:
+      raise ValueError("Rubrics are required.")
 
   def get_effective_rubrics_list(self) -> list[Rubric]:
     """Returns the effective rubrics list."""

src/google/adk/evaluation/rubric_based_final_response_quality_v1.py

@@ -296,7 +296,7 @@ def format_auto_rater_prompt(
 
     rubrics_text = "\n".join([
         f"*  [id: {r.rubric_id}] {r.rubric_content.text_property}"
-        for r in self._effective_rubrics_list
+        for r in self.get_effective_rubrics_list()
     ])
 
     developer_instructions = ""

src/google/adk/evaluation/rubric_based_multi_turn_trajectory_evaluator.py

@@ -353,11 +353,11 @@ def format_auto_rater_prompt(
     self.create_effective_rubrics_list(actual_invocation.rubrics)
     logger.debug(
         "format_auto_rater_prompt called (effective rubrics: %d)",
-        len(self._effective_rubrics_list),
+        len(self.get_effective_rubrics_list()),
     )
 
     rubrics_list = []
-    for r in self._effective_rubrics_list:
+    for r in self.get_effective_rubrics_list():
       rubrics_dict = {
           "id": r.rubric_id,
           "property": r.rubric_content.text_property,

src/google/adk/evaluation/rubric_based_tool_use_quality_v1.py

@@ -185,7 +185,7 @@ def format_auto_rater_prompt(
 
     rubrics_text = "\n".join([
         f"*  [id: {r.rubric_id}] {r.rubric_content.text_property}"
-        for r in self._effective_rubrics_list
+        for r in self.get_effective_rubrics_list()
     ])
 
     app_details = actual_invocation.app_details

tests/unittests/cli/utils/test_cli_eval_pretty_print.py

@@ -0,0 +1,72 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import annotations
+
+from google.adk.cli.cli_eval import pretty_print_eval_result
+from google.adk.evaluation.eval_case import Invocation
+from google.adk.evaluation.eval_metrics import EvalMetricResult
+from google.adk.evaluation.eval_metrics import EvalMetricResultDetails
+from google.adk.evaluation.eval_metrics import EvalMetricResultPerInvocation
+from google.adk.evaluation.eval_metrics import PrebuiltMetrics
+from google.adk.evaluation.eval_metrics import RubricsBasedCriterion
+from google.adk.evaluation.eval_result import EvalCaseResult
+from google.adk.evaluation.eval_rubrics import RubricScore
+from google.adk.evaluation.evaluator import EvalStatus
+from google.genai import types as genai_types
+
+
+def test_pretty_print_eval_result_with_empty_criterion_rubrics(capsys):
+  """Tests pretty printing falls back to rubric id when criterion rubrics are empty."""
+  criterion = RubricsBasedCriterion(threshold=0.5)
+  metric_result = EvalMetricResult(
+      metric_name=PrebuiltMetrics.RUBRIC_BASED_TOOL_USE_QUALITY_V1.value,
+      threshold=0.5,
+      criterion=criterion,
+      score=1.0,
+      eval_status=EvalStatus.PASSED,
+      details=EvalMetricResultDetails(
+          rubric_scores=[
+              RubricScore(
+                  rubric_id="invocation-rubric",
+                  score=1.0,
+                  rationale="The correct tool was used.",
+              )
+          ]
+      ),
+  )
+  invocation = Invocation(
+      user_content=genai_types.Content(
+          parts=[genai_types.Part(text="User input here.")]
+      )
+  )
+  eval_result = EvalCaseResult(
+      eval_set_id="eval-set",
+      eval_id="eval-id",
+      final_eval_status=EvalStatus.PASSED,
+      overall_eval_metric_results=[metric_result],
+      eval_metric_result_per_invocation=[
+          EvalMetricResultPerInvocation(
+              actual_invocation=invocation,
+              eval_metric_results=[metric_result],
+          )
+      ],
+      session_id="session-id",
+  )
+
+  pretty_print_eval_result(eval_result)
+
+  captured = capsys.readouterr()
+  assert "Rubric: invocation-rubric" in captured.out
+  assert "The correct tool was used." in captured.out

tests/unittests/evaluation/test_rubric_based_evaluator.py

@@ -711,6 +711,24 @@ def test_create_effective_rubrics_list_with_no_invocation_rubrics(
     assert len(effective_rubrics) == 2
     assert {r.rubric_id for r in effective_rubrics} == {"1", "2"}
 
+  def test_create_effective_rubrics_list_with_no_rubrics_raises_error(self):
+    judge_model_options = JudgeModelOptions(
+        judge_model_config=None,
+        num_samples=3,
+    )
+    criterion = RubricsBasedCriterion(
+        threshold=0.5, judge_model_options=judge_model_options
+    )
+    metric = EvalMetric(
+        metric_name=PrebuiltMetrics.RUBRIC_BASED_FINAL_RESPONSE_QUALITY_V1.value,
+        threshold=0.5,
+        criterion=criterion,
+    )
+    evaluator = FakeRubricBasedEvaluator(metric)
+
+    with pytest.raises(ValueError, match="Rubrics are required."):
+      evaluator.create_effective_rubrics_list(None)
+
   def test_get_effective_rubrics_list_before_creation_raises_error(
       self, evaluator: RubricBasedEvaluator
   ):
@@ -771,6 +789,31 @@ def test_create_effective_rubrics_filters_by_rubric_type(
         "test_type_rubric",
     }
 
+  def test_create_effective_rubrics_filters_to_empty_raises_error(self):
+    judge_model_options = JudgeModelOptions(
+        judge_model_config=None,
+        num_samples=3,
+    )
+    criterion = RubricsBasedCriterion(
+        threshold=0.5, judge_model_options=judge_model_options
+    )
+    metric = EvalMetric(
+        metric_name=PrebuiltMetrics.RUBRIC_BASED_FINAL_RESPONSE_QUALITY_V1.value,
+        threshold=0.5,
+        criterion=criterion,
+    )
+    evaluator = FakeRubricBasedEvaluator(metric, rubric_type="EXPECTED_TYPE")
+    invocation_rubrics = [
+        Rubric(
+            rubric_id="wrong_type_rubric",
+            rubric_content=RubricContent(text_property="Invocation rubric"),
+            type="WRONG_TYPE",
+        )
+    ]
+
+    with pytest.raises(ValueError, match="Rubrics are required."):
+      evaluator.create_effective_rubrics_list(invocation_rubrics)
+
   def test_convert_matches_by_id_when_text_paraphrased(
       self,
       evaluator: RubricBasedEvaluator,

tests/unittests/evaluation/test_rubric_based_tool_use_quality_v1.py

@@ -79,6 +79,67 @@ def test_format_auto_rater_prompt_with_basic_invocation(
   assert "<response>\nNo intermediate steps were taken.\n</response>" in prompt
 
 
+def test_format_auto_rater_prompt_with_invocation_rubrics_only():
+  """Tests prompt formatting when rubrics are defined on the invocation."""
+  judge_model_options = JudgeModelOptions(
+      judge_model_config=None,
+      num_samples=3,
+  )
+  criterion = RubricsBasedCriterion(
+      threshold=0.5, judge_model_options=judge_model_options
+  )
+  metric = EvalMetric(
+      metric_name=PrebuiltMetrics.RUBRIC_BASED_TOOL_USE_QUALITY_V1.value,
+      threshold=0.5,
+      criterion=criterion,
+  )
+  evaluator = RubricBasedToolUseV1Evaluator(metric)
+  invocation = Invocation(
+      user_content=genai_types.Content(
+          parts=[genai_types.Part(text="User input here.")]
+      ),
+      rubrics=[
+          Rubric(
+              rubric_id="invocation-rubric",
+              rubric_content=RubricContent(
+                  text_property="Did the agent use the lookup tool?"
+              ),
+              type=RubricBasedToolUseV1Evaluator.RUBRIC_TYPE,
+          )
+      ],
+  )
+
+  prompt = evaluator.format_auto_rater_prompt(invocation, None)
+
+  assert "User input here." in prompt
+  assert "Did the agent use the lookup tool?" in prompt
+
+
+def test_format_auto_rater_prompt_without_effective_rubrics_raises_error():
+  """Tests prompt formatting fails when no criterion or invocation rubrics exist."""
+  judge_model_options = JudgeModelOptions(
+      judge_model_config=None,
+      num_samples=3,
+  )
+  criterion = RubricsBasedCriterion(
+      threshold=0.5, judge_model_options=judge_model_options
+  )
+  metric = EvalMetric(
+      metric_name=PrebuiltMetrics.RUBRIC_BASED_TOOL_USE_QUALITY_V1.value,
+      threshold=0.5,
+      criterion=criterion,
+  )
+  evaluator = RubricBasedToolUseV1Evaluator(metric)
+  invocation = Invocation(
+      user_content=genai_types.Content(
+          parts=[genai_types.Part(text="User input here.")]
+      ),
+  )
+
+  with pytest.raises(ValueError, match="Rubrics are required."):
+    evaluator.format_auto_rater_prompt(invocation, None)
+
+
 def test_format_auto_rater_prompt_with_app_details(
     evaluator: RubricBasedToolUseV1Evaluator,
 ):