diff --git a/src/google/adk/evaluation/final_response_match_v1.py b/src/google/adk/evaluation/final_response_match_v1.py index f7c07cec73..98b509a50a 100644 --- a/src/google/adk/evaluation/final_response_match_v1.py +++ b/src/google/adk/evaluation/final_response_match_v1.py @@ -14,6 +14,7 @@ from __future__ import annotations +import re from typing import Optional from google.genai import types as genai_types @@ -96,6 +97,31 @@ def _get_eval_status(score: float, threshold: float) -> EvalStatus: return EvalStatus.PASSED if score >= threshold else EvalStatus.FAILED +def _contains_non_ascii(text: str) -> bool: + """Returns True if the text contains any non-ASCII characters.""" + return bool(re.search(r"[^\x00-\x7F]", text)) + + +class _UnicodeTokenizer: + """A tokenizer that handles non-ASCII text by splitting on whitespace and + decomposing non-ASCII tokens into individual characters.""" + + def tokenize(self, text: str) -> list[str]: + tokens = [] + for word in text.lower().split(): + if _contains_non_ascii(word): + # For non-ASCII words (e.g. Thai, Chinese, Arabic), treat each + # Unicode character as a separate token so that ROUGE overlap is + # computed at the character level instead of being discarded entirely. + tokens.extend(list(word)) + else: + # Keep ASCII tokens as-is (drop purely punctuation-only tokens). + cleaned = re.sub(r"[^a-z0-9]", "", word) + if cleaned: + tokens.append(cleaned) + return tokens + + def _calculate_rouge_1_scores(candidate: str, reference: str): """Calculates the ROUGE-1 score between a candidate and reference text. @@ -114,7 +140,14 @@ def _calculate_rouge_1_scores(candidate: str, reference: str): Returns: A dictionary containing the ROUGE-1 precision, recall, and f-measure. """ - scorer = rouge_scorer.RougeScorer(["rouge1"], use_stemmer=True) + # Use a Unicode-aware tokenizer when either text contains non-ASCII + # characters (e.g. Thai, Chinese, Arabic) so that the default ASCII-only + # tokenizer does not strip every token and return a spurious score of 0. + if _contains_non_ascii(candidate) or _contains_non_ascii(reference): + tokenizer = _UnicodeTokenizer() + scorer = rouge_scorer.RougeScorer(["rouge1"], tokenizer=tokenizer) + else: + scorer = rouge_scorer.RougeScorer(["rouge1"], use_stemmer=True) # The score method returns a dictionary where keys are the ROUGE types # and values are Score objects (tuples) with precision, recall, and fmeasure. diff --git a/tests/unittests/evaluation/test_final_response_match_v1.py b/tests/unittests/evaluation/test_final_response_match_v1.py index 111ca8415c..2c0e5126bd 100644 --- a/tests/unittests/evaluation/test_final_response_match_v1.py +++ b/tests/unittests/evaluation/test_final_response_match_v1.py @@ -87,6 +87,22 @@ def test_calculate_rouge_1_scores(): assert rouge_1_score.fmeasure == pytest.approx(8 / 11) +def test_calculate_rouge_1_scores_non_ascii_identical(): + # Identical non-ASCII (Thai) text should yield a perfect score, not 0. + candidate = "สวัสดี" + reference = "สวัสดี" + rouge_1_score = _calculate_rouge_1_scores(candidate, reference) + assert rouge_1_score.fmeasure == pytest.approx(1.0) + + +def test_calculate_rouge_1_scores_non_ascii_different(): + # Completely different non-ASCII text should yield a score of 0. + candidate = "สวัสดี" + reference = "ขอบคุณ" + rouge_1_score = _calculate_rouge_1_scores(candidate, reference) + assert rouge_1_score.fmeasure == pytest.approx(0.0) + + @pytest.mark.parametrize( "candidates, references, expected_score, expected_status", [