fix: match rubric verdicts by echoed id so paraphrased rubrics are not dropped

RubricBasedEvaluator mapped each auto-rater verdict back to its rubric by
exact normalized property text. When the judge paraphrased the property
text (e.g. kanji/hiragana rewrites), nothing matched and every verdict was
silently dropped, leaving the invocation unscored. Each rubric now carries
a short id that the judge is asked to echo, and verdicts are matched by id
first, falling back to normalized property text so existing behavior is
preserved.

Close #6171

Co-authored-by: George Weale <gweale@google.com>
PiperOrigin-RevId: 951036528

Author: GWeale

src/google/adk/evaluation/rubric_based_evaluator.py

@@ -42,6 +42,7 @@
 class RubricResponse(EvalBaseModel):
   """Internal data model to represent a rubric's response from the auto-rater."""
 
+  rubric_id: Optional[str] = None
   property_text: Optional[str] = None
   rationale: Optional[str] = None
   score: Optional[float] = None
@@ -56,7 +57,8 @@ def parse(self, auto_rater_response: str) -> list[RubricResponse]:
     raise NotImplementedError
 
 
-_PROPERTY_PATTERN = r"(?<=Property: )(.*)"
+_ID_PATTERN = r"(?m)^\s*ID: (.*)$"
+_PROPERTY_PATTERN = r"(?m)^\s*Property: (.*)$"
 _RATIONALE_PATTERN = r"(?<=Rationale: )(.*)"
 _VERDICT_PATTERN = r"(?<=Verdict: )(.*)"
 
@@ -66,7 +68,8 @@ class DefaultAutoRaterResponseParser(AutoRaterResponseParser):
 
   def parse(self, auto_rater_response: str) -> list[RubricResponse]:
     """Returns a list of RubricResponse parsed from the AutoRater's response."""
-    properties = re.findall(_PROPERTY_PATTERN, auto_rater_response)
+    property_matches = list(re.finditer(_PROPERTY_PATTERN, auto_rater_response))
+    id_matches = list(re.finditer(_ID_PATTERN, auto_rater_response))
     rationales = re.findall(_RATIONALE_PATTERN, auto_rater_response)
     scores = []
 
@@ -81,13 +84,27 @@ def parse(self, auto_rater_response: str) -> list[RubricResponse]:
       scores.append(score)
 
     # A partial parse can silently omit a failed rubric and inflate the score.
-    if not len(properties) == len(rationales) == len(scores):
+    if not len(property_matches) == len(rationales) == len(scores):
       return []
 
     rubric_responses = []
-    for p, r, s in zip(properties, rationales, scores, strict=True):
+    for i, (property_match, rationale, score) in enumerate(
+        zip(property_matches, rationales, scores, strict=True)
+    ):
+      # Match each id to the property it immediately precedes (not by index) so
+      # an omitted id line can't shift a later id onto an earlier property.
+      previous_start = property_matches[i - 1].start() if i > 0 else -1
+      rubric_id = None
+      for id_match in id_matches:
+        if previous_start < id_match.start() < property_match.start():
+          rubric_id = id_match.group(1).strip() or None
       rubric_responses.append(
-          RubricResponse(property_text=p.strip(), rationale=r.strip(), score=s)
+          RubricResponse(
+              rubric_id=rubric_id,
+              property_text=property_match.group(1).strip(),
+              rationale=rationale.strip(),
+              score=score,
+          )
       )
 
     return rubric_responses
@@ -406,14 +423,21 @@ def convert_auto_rater_response_to_score(
     rubric_scores = []
 
     normalized_rubric_to_rubric_map = {}
+    rubric_by_id = {}
     for r in self.get_effective_rubrics_list():
       normalized_rubric_to_rubric_map[
           _normalize_text(r.rubric_content.text_property)
       ] = r
+      rubric_by_id[r.rubric_id] = r
 
     for rubric_response in rubric_responses:
-      normalized_rubric_text = _normalize_text(rubric_response.property_text)
-      rubric = normalized_rubric_to_rubric_map.get(normalized_rubric_text, None)
+      rubric = None
+      if rubric_response.rubric_id:
+        rubric = rubric_by_id.get(rubric_response.rubric_id)
+      if rubric is None:
+        rubric = normalized_rubric_to_rubric_map.get(
+            _normalize_text(rubric_response.property_text)
+        )
       if rubric:
         rubric_scores.append(
             RubricScore(

src/google/adk/evaluation/rubric_based_final_response_quality_v1.py

@@ -70,7 +70,8 @@
 6. Output the final verdict in the required output format.
 
 # Output Format (repeat this format for every property, starting with a new line):
-Property: [Repeat the property, word for word, without making any changes. Keep everything including punctuation and capitalization as-is.]
+ID: [Copy the id shown in the "[id: ...]" tag for this property, verbatim. Omit this line only if no tag is shown.]
+Property: [Repeat the property text that follows the "[id: ...]" tag, word for word, without making any changes and without the tag. Keep everything including punctuation and capitalization as-is.]
 Evidence: [List all trusted evidence from tool calls or the user prompt that is relevant to the property (referencing the Step Index). Alternatively, if either no trusted evidence is required, or no trusted evidence exists (e.g., flawed process, missing tool call, tool error), explain why.]
 Rationale: [Explain your reasoning, detailing how the evidence (or lack thereof) supports or contradicts the final answer, or why the property is not applicable.]
 Verdict: [yes|no]
@@ -149,46 +150,53 @@
 </response>
 
 <properties>
-* The final answer correctly identifies the total number of employees.
-* The final answer correctly identifies the name of Alice Smith's manager, or correctly states that it cannot be determined and why.
-* The final answer correctly states the average salary for the Marketing department.
-* The final answer correctly identifies the employee with the highest salary.
-* The final answer correctly identifies the gender of the employee with the highest salary, or correctly states that it cannot be determined and why.
-* The final answer is formatted as a numbered list.
-* If the company has fewer than 100 employees, then the final answer states that it has fewer than 100 employees.
+*  [id: 1] The final answer correctly identifies the total number of employees.
+*  [id: 2] The final answer correctly identifies the name of Alice Smith's manager, or correctly states that it cannot be determined and why.
+*  [id: 3] The final answer correctly states the average salary for the Marketing department.
+*  [id: 4] The final answer correctly identifies the employee with the highest salary.
+*  [id: 5] The final answer correctly identifies the gender of the employee with the highest salary, or correctly states that it cannot be determined and why.
+*  [id: 6] The final answer is formatted as a numbered list.
+*  [id: 7] If the company has fewer than 100 employees, then the final answer states that it has fewer than 100 employees.
 </properties>
 
 ## Output
+ID: 1
 Property: The final answer correctly identifies the total number of employees.
 Evidence: The trusted evidence is "110 employees". The tool call in Step 0 is procedurally sound and provides the total number of employees (110) by calling the load_hr_data_from_file tool with the correct file name.
 Rationale: The final answer's claim ("110 employees") is fully consistent with the trusted evidence.
 Verdict: yes
 
+ID: 2
 Property: The final answer correctly identifies the name of Alice Smith's manager, or correctly states that it cannot be determined and why.
 Evidence: No trusted evidence exists. The agent did not perform a tool call to determine the manager of Alice Smith, despite having the necessary information (the employee name) and access to the necessary tools (get_manager) to do so.
 Rationale: The agent incorrectly stated that the final answer cannot be determined, despite having the necessary information (the employee name) and tools (get_manager) to determine it.
 Verdict: no
 
+ID: 3
 Property: The final answer correctly states the average salary for the Marketing department.
 Evidence: No trusted evidence exists for the Marketing department's average salary. The tool call in Step 1 is procedurally flawed; the agent searched for "Engineering" instead of "Marketing".
 Rationale: There is no trusted evidence for the Marketing department's average salary.
 Verdict: no
 
+ID: 4
 Property: The final answer correctly identifies the employee with the highest salary.
 Evidence: The trusted evidence is "John Smith". The tool call in Step 2 produces trusted evidence for the employee with the highest salary by calling the load_hr_data_from_file tool with the correct file name and then using the idxmax() method to find the employee with the highest salary.
 Rationale: The final answer's claim ("John Doe") is inconsistent with the trusted evidence ("John Smith").
 Verdict: no
 
+ID: 5
 Property: The final answer correctly identifies the gender of the employee with the highest salary, or correctly states that it cannot be determined and why.
 Evidence: No trusted evidence exists. The agent did not perform a tool call to determine the gender of the employee with the highest salary.
 Rationale: There is no trusted evidence to confirm the gender of the employee with the highest salary that the final answer states (male). Even if the gender is coincidentally actually male, the claim in the final answer cannot be unambiguously verified using the evidence.
 Verdict: no
 
+ID: 7
 Property: If the company has fewer than 100 employees, then the final answer should state that it has fewer than 100 employees.
 Evidence: The trusted evidence is "110 employees". The tool call in Step 0 correctly counts the total number of employees as 110 by calling the load_hr_data_from_file tool with the correct file name.
 Rationale: The total number of employees is 110, so the condition for this property (fewer than 100 employees) was not met. Therefore, the property is not applicable to this response.
 Verdict: yes
 
+ID: 6
 Property: The final answer is formatted as a numbered list.
 Evidence: N/A. Trusted evidence from tool calls or the user prompt is not required in order to determine the format of the final answer.
 Rationale: The final answer is formatted as a numbered list from 1 to 4, e.g. "1. The total number of employees is 110\n2...".
@@ -287,7 +295,7 @@ def format_auto_rater_prompt(
     )
 
     rubrics_text = "\n".join([
-        f"*  {r.rubric_content.text_property}"
+        f"*  [id: {r.rubric_id}] {r.rubric_content.text_property}"
         for r in self._effective_rubrics_list
     ])
 

src/google/adk/evaluation/rubric_based_multi_turn_trajectory_evaluator.py

@@ -59,6 +59,7 @@
 STEP 3: Follow the steps outlined in STEP 2, thinking out loud. As you think, refer to specific assistant responses and count their turn numbers within the <conversation_history>.
 STEP 4: Review the thoughts and the original property.
 STEP 5: Output the final verdict.
+ID: [[Copy the "id" field of this property from the properties list, verbatim. Omit this line only if the property has no "id" field.]]
 Property: [[Repeat the property in STEP 1 again.]]
 Rationale: [[Explain your reasoning for the verdict.]]
 Verdict: [[yes|no]]
@@ -69,6 +70,7 @@
 STEP 3: ...
 STEP 4: ...
 STEP 5: ...
+ID: ...
 Property: ...
 Rationale: ...
 Verdict: ...
@@ -80,6 +82,7 @@
 STEP 3: Reviewing the <conversation_history>, I find that in Assistant Turn 2, the agent includes the function call default_api.grammar_check(sentence="the dog walks on the a park"). This matches the required function name.
 STEP 4: The assistant successfully called the specified function in Turn 2.
 STEP 5: yes
+ID: 1
 Property: Does the agent run function call 'default_api.grammar_check'?
 Rationale: The agent's response in Assistant Turn 2 contains the function call 'default_api.grammar_check' within a proper code block and with the correct function name.
 Verdict: yes
@@ -89,6 +92,7 @@
 STEP 3: In Assistant Turn 2, the agent's response includes the function call default_api.grammar_check(sentence="the dog walks on the a park"). The parameter 'sentence' is present, and the value assigned to it is "the dog walks on the a park", which is identical to the reference value.
 STEP 4: The parameter 'sentence' is present and its value is exactly the same as the reference value in Assistant Turn 2.
 STEP 5: yes
+ID: 2
 Property: The sentence parameter in the grammar_check function call must be 'the dog walks on the a park'.
 Rationale: The agent's response in Assistant Turn 2 includes the 'sentence' parameter in the function call 'default_api.grammar_check', and the value assigned to it is exactly the same as the reference value.
 Verdict: yes
@@ -100,6 +104,7 @@
 STEP 3: Reviewing the <conversation_history>, I find that in Assistant Turn 2, the agent includes a function call default_api.get_web_search_results. This does not match the required 'default_api.search_via_perplexity'. I checked all other assistant turns and no other function call matches.
 STEP 4: The agent did not use the function 'default_api.search_via_perplexity' in any turn.
 STEP 5: no
+ID: 1
 Property: Does the agent run function call 'default_api.search_via_perplexity'?
 Rationale: The agent called 'default_api.get_web_search_results' in Assistant Turn 2, not 'default_api.search_via_perplexity'. The correct function was not called in any turn.
 Verdict: no
@@ -109,6 +114,7 @@
 STEP 3: Based on the previous evaluation, the agent did not include a function call 'default_api.search_via_perplexity' in any of its turns. Therefore, this property cannot be fulfilled.
 STEP 4: The property cannot be fulfilled because the agent did not use the specified function call.
 STEP 5: no
+ID: 2
 Property: Does the agent provide function call 'default_api.search_via_perplexity' with input parameter 'keyword' that is valid compared to the reference 'keyword'= 'GPT-4o vs GPT-3.5 cost comparison' and based on the following guideline? Guideline for 'keyword': 'The wording can differ. The agent response is valid if it conveys similar core content as the reference response. Less efficient and minor inaccurate phrasing is acceptable.'
 Rationale: The agent did not use the function call 'default_api.search_via_perplexity' in any of its responses throughout the conversation.
 Verdict: no
@@ -353,6 +359,7 @@ def format_auto_rater_prompt(
     rubrics_list = []
     for r in self._effective_rubrics_list:
       rubrics_dict = {
+          "id": r.rubric_id,
           "property": r.rubric_content.text_property,
       }
       if r.type:

src/google/adk/evaluation/rubric_based_tool_use_quality_v1.py

@@ -42,12 +42,13 @@
 "no": The agent's response did not fulfill the property.
 
 # For each property started with a new line, follow these steps:
-STEP 1: Repeat the property, word for word, without making any changes. Keep everything including punctuation and capitalization as-is.
+STEP 1: Repeat the property text (excluding the "[id: ...]" tag), word for word, without making any changes. Keep everything including punctuation and capitalization as-is.
 STEP 2: Determine the steps needed to **exactly**, **precisely** and **completely** determine whether the agent's response fulfilled the property.
 STEP 3: Follow the steps outlined in STEP 2, thinking out loud.
 STEP 4: Review the thoughts and the original property.
 STEP 5: Output the final verdict.
-Property: [[Repeat the property in STEP 1 again.]]
+ID: [[Copy the id shown in the "[id: ...]" tag for this property, verbatim. Omit this line only if no tag is shown.]]
+Property: [[Repeat the property in STEP 1 again, without the "[id: ...]" tag.]]
 Rationale: [[Explain your reasoning for the verdict.]]
 Verdict: [[yes|no]]
 
@@ -57,6 +58,7 @@
 STEP 3: ...
 STEP 4: ...
 STEP 5: ...
+ID: ...
 Property: ...
 Rationale: ...
 Verdict: ...
@@ -69,6 +71,7 @@
 STEP 3: The response includes a function call 'default_api.grammar_check'.
 STEP 4: The function call format and the function name are correct.
 STEP 5: yes
+ID: 1
 Property: Does the agent run function call 'default_api.grammar_check'?
 Rationale: The agent's response contains the function call 'default_api.grammar_check' within a proper code block and with the correct function name.
 Verdict: yes
@@ -78,6 +81,7 @@
 STEP 3: The agent's response includes the function call `default_api.grammar_check(sentence="the dog walks on the a park")`. The parameter 'sentence' is present, and the value assigned to it is "the dog walks on the a park", which is identical to the reference value.
 STEP 4: The parameter 'sentence' is present and its value is exactly the same as the reference value.
 STEP 5: yes
+ID: 2
 Property: Does the agent provide function call 'default_api.grammar_check' with input parameter 'sentence' that is valid compared to the reference 'sentence'= 'the dog walks on the a park' and based on the following guideline? Guideline for 'sentence': 'The wording can differ. The agent response is valid if it conveys similar core content as the reference response. Less efficient and minor inaccurate phrasing is acceptable. The default value is None, if the reference response includes this parameter with value equal to the default value but it is not provided in the agent response, then evaluate it as valid.'
 Rationale: The agent's response includes the 'sentence' parameter in the function call 'default_api.grammar_check', and the value assigned to it is exactly the same as the reference value, thus satisfying the given guideline.
 Verdict: yes
@@ -89,6 +93,7 @@
 STEP 3: The response includes a function call `default_api.get_web_search_results`, which does not match 'default_api.search_via_perplexity'.
 STEP 4: The function name does not match.
 STEP 5: no
+ID: 1
 Property: Does the agent run function call 'default_api.search_via_perplexity'?
 Rationale: The agent called 'default_api.get_web_search_results', not 'default_api.search_via_perplexity'.
 Verdict: no
@@ -98,6 +103,7 @@
 STEP 3: N/A
 STEP 4: N/A
 STEP 5: yes
+ID: 2
 Property: Does the agent provide function call 'default_api.search_via_perplexity' with input parameter 'keyword' that is valid compared to the reference 'keyword'= 'GPT-4o vs GPT-3.5 cost comparison' and based on the following guideline? Guideline for 'keyword': 'The wording can differ. The agent response is valid if it conveys similar core content as the reference response. Less efficient and minor inaccurate phrasing is acceptable.'
 Rationale: The agent did not use the function call 'default_api.search_via_perplexity'.
 Verdict: yes
@@ -178,7 +184,7 @@ def format_auto_rater_prompt(
     )
 
     rubrics_text = "\n".join([
-        f"*  {r.rubric_content.text_property}"
+        f"*  [id: {r.rubric_id}] {r.rubric_content.text_property}"
         for r in self._effective_rubrics_list
     ])