fix(evaluation): support non-English responses in ROUGE-1 matching

Merge #6292

Closes #3111

Problem:
The `response_match_score` metric (`RougeEvaluator`) always returns 0 for responses in non-Latin scripts, even when the actual and expected responses are identical.

Root Cause:
The default `rouge_score` tokenizer (`DefaultTokenizer`) lowercases text and replaces every character outside `[a-z0-9]` with a space. Consequently, non-Latin scripts (Thai, Chinese, Arabic, Japanese, Cyrillic, etc.) tokenize to an empty token list (`[]`) and every comparison scores 0.0.

Reproduction on `main`:

```python
from rouge_score import rouge_scorer
scorer = rouge_scorer.RougeScorer(["rouge1"], use_stemmer=True)
scorer.score("สวัสดี", "สวัสดี")["rouge1"].fmeasure  # 0.0 — identical strings
scorer.score("hello", "hello")["rouge1"].fmeasure    # 1.0
```

Solution:
Pass a custom `_UnicodeAwareTokenizer` to `RougeScorer` in `final_response_match_v1.py` to handle non-English scripts:

1. CJK (Chinese, Japanese Hiragana/Katakana, Hangul): Characters identified via `_is_cjk()` (Unicode ranges `0x4E00..0x9FFF`, `0x3040..0x309F`, `0x30A0..0x30FF`, `0xAC00..0xD7AF`) are separated into individual character tokens. This enables character-level ROUGE-1 unigram matching for CJK text without external heavy NLP segmentation dependencies.
2. Non-spaced scripts (Thai, Lao, Khmer, Myanmar): Detected via `_is_non_spaced_script()` (`0x0E00..0x0E7F` etc.). Characters are bundled into grapheme clusters where base consonants trigger new word boundaries while combining marks/vowels/tone marks (Unicode category `M`, e.g. `Mn` like Thai vowel signs) stay attached to their base consonant.
3. Spaced Non-ASCII scripts (Arabic, Cyrillic, Hindi, etc.): Non-ASCII word characters (`str.isalnum()` or category `M`) are preserved as distinct tokens instead of being stripped.
4. ASCII compatibility: Pure-ASCII tokens are delegated to `rouge_score`s default `DefaultTokenizer`, keeping lowercasing and Porter stemming identical for English text.

Testing Plan:

Unit Tests:
- [x] I have added or updated unit tests for my change.
- [x] All unit tests pass locally.

New tests in `tests/unittests/evaluation/test_final_response_match_v1.py`:
- Identical non-English text scores 1.0 (Thai, Chinese, Arabic, Japanese, Russian)
- Partially overlapping non-English text scores expected ROUGE-1 fractions (CJK & Russian)
- Mixed English + non-English text handling
- `_UnicodeAwareTokenizer` produces identical tokens to `DefaultTokenizer` for ASCII inputs (regression guard for English scoring)
- Evaluator-level test cases for identical, partially matching (Chinese & Thai), and completely non-overlapping non-English responses

```
$ pytest tests/unittests/evaluation/test_final_response_match_v1.py -q
65 passed in 14.17s
```

Manual End-to-End (E2E) Tests:
Ran the evaluator directly on the scenario from #3111 (agent instructed to reply with the word `สวัสดี`, expected response `สวัสดี`):

```python
ev = RougeEvaluator(EvalMetric(metric_name="response_match_score", threshold=0.8))
result = ev.evaluate_invocations([inv("สวัสดี")], [inv("สวัสดี")])
# before: score=0.0, status=EvalStatus.FAILED
# after:  score=1.0, status=EvalStatus.PASSED
```

Original PR by @agharsallah

Co-authored-by: Yi Liu <yiliuly@google.com>
COPYBARA_INTEGRATE_REVIEW=#6292 from agharsallah:fix/eval-rouge-non-english-3111 4810719975
PiperOrigin-RevId: 954809249

Author: 2 people

src/google/adk/dependencies/rouge_scorer.py

@@ -15,3 +15,4 @@
 from __future__ import annotations
 
 from rouge_score import rouge_scorer as rouge_scorer
+from rouge_score import tokenizers as tokenizers

src/google/adk/evaluation/final_response_match_v1.py

@@ -15,11 +15,13 @@
 from __future__ import annotations
 
 from typing import Optional
+import unicodedata
 
 from google.genai import types as genai_types
 from typing_extensions import override
 
 from ..dependencies.rouge_scorer import rouge_scorer
+from ..dependencies.rouge_scorer import tokenizers
 from .eval_case import ConversationScenario
 from .eval_case import Invocation
 from .eval_metrics import EvalMetric
@@ -96,6 +98,83 @@ def _get_eval_status(score: float, threshold: float) -> EvalStatus:
   return EvalStatus.PASSED if score >= threshold else EvalStatus.FAILED
 
 
+def _is_cjk(char: str) -> bool:
+  """Checks if a character belongs to CJK (Chinese, Japanese, Korean) scripts."""
+  code = ord(char)
+  return (
+      0x4E00 <= code <= 0x9FFF  # CJK Unified Ideographs
+      or 0x3040 <= code <= 0x309F  # Hiragana
+      or 0x30A0 <= code <= 0x30FF  # Katakana
+      or 0xAC00 <= code <= 0xD7AF  # Hangul Syllables
+  )
+
+
+def _is_non_spaced_script(char: str) -> bool:
+  """Checks if a character belongs to non-spaced scripts like Thai, Lao, Khmer, Myanmar."""
+  code = ord(char)
+  return (
+      0x0E00 <= code <= 0x0E7F  # Thai
+      or 0x0E80 <= code <= 0x0EFF  # Lao
+      or 0x1780 <= code <= 0x17FF  # Khmer
+      or 0x1000 <= code <= 0x109F  # Myanmar
+  )
+
+
+def _is_word_char(char: str) -> bool:
+  # Combining marks (e.g. Thai vowel signs, Devanagari matras) are not
+  # alphanumeric on their own but must stay attached to their base character.
+  return char.isalnum() or unicodedata.category(char).startswith("M")
+
+
+class _UnicodeAwareTokenizer:
+  """Tokenizer that keeps non-ASCII word characters and splits CJK/non-spaced characters.
+
+  The default rouge_score tokenizer discards any character outside [a-z0-9],
+  so text in non-Latin scripts (e.g. Thai, Chinese, Arabic) tokenizes to
+  nothing and always scores 0. This tokenizer keeps Unicode word characters,
+  normalizes Unicode variants (NFKC), splits non-spaced CJK characters at the
+  character level, bundles non-spaced scripts by grapheme clusters (base
+  consonant + attached combining marks), and delegates ASCII tokens to the
+  default tokenizer.
+
+  Note:
+      Languages written without spaces (e.g. Thai, Chinese, Japanese) are
+      tokenized at the character or grapheme cluster level rather than at true
+      word granularity, since dictionary-based word segmentation requires heavy
+      external NLP dependencies. As a result, ROUGE-1 unigram overlap for these
+      languages operates on character/grapheme cluster units rather than full words.
+  """
+
+  def __init__(self, use_stemmer: bool = False):
+    self._default_tokenizer = tokenizers.DefaultTokenizer(use_stemmer)
+
+  def tokenize(self, text: str) -> list[str]:
+    text = unicodedata.normalize("NFKC", text).lower()
+    processed_chars = []
+    for char in text:
+      if _is_cjk(char):
+        processed_chars.extend([" ", char, " "])
+      elif _is_non_spaced_script(char):
+        if unicodedata.category(char).startswith("M"):
+          # Combining mark (vowel/tone mark): attach directly to previous base consonant!
+          processed_chars.append(char)
+        else:
+          # Base consonant: start a new grapheme cluster boundary by prepending a space!
+          processed_chars.extend([" ", char])
+      elif _is_word_char(char):
+        processed_chars.append(char)
+      else:
+        processed_chars.append(" ")
+    words = "".join(processed_chars).split()
+    tokens = []
+    for word in words:
+      if word.isascii():
+        tokens.extend(self._default_tokenizer.tokenize(word))
+      else:
+        tokens.append(word)
+    return tokens
+
+
 def _calculate_rouge_1_scores(candidate: str, reference: str):
   """Calculates the ROUGE-1 score between a candidate and reference text.
 
@@ -114,7 +193,9 @@ def _calculate_rouge_1_scores(candidate: str, reference: str):
   Returns:
       A dictionary containing the ROUGE-1 precision, recall, and f-measure.
   """
-  scorer = rouge_scorer.RougeScorer(["rouge1"], use_stemmer=True)
+  scorer = rouge_scorer.RougeScorer(
+      ["rouge1"], tokenizer=_UnicodeAwareTokenizer(use_stemmer=True)
+  )
 
   # The score method returns a dictionary where keys are the ROUGE types
   # and values are Score objects (tuples) with precision, recall, and fmeasure.

tests/unittests/evaluation/test_final_response_match_v1.py

@@ -14,14 +14,21 @@
 
 from __future__ import annotations
 
+import unicodedata
+
 from google.adk.evaluation.eval_case import Invocation
 from google.adk.evaluation.eval_metrics import EvalMetric
 from google.adk.evaluation.eval_metrics import PrebuiltMetrics
 from google.adk.evaluation.evaluator import EvalStatus
 from google.adk.evaluation.final_response_match_v1 import _calculate_rouge_1_scores
+from google.adk.evaluation.final_response_match_v1 import _is_cjk
+from google.adk.evaluation.final_response_match_v1 import _is_non_spaced_script
+from google.adk.evaluation.final_response_match_v1 import _is_word_char
+from google.adk.evaluation.final_response_match_v1 import _UnicodeAwareTokenizer
 from google.adk.evaluation.final_response_match_v1 import RougeEvaluator
 from google.genai import types as genai_types
 import pytest
+from rouge_score import tokenizers
 
 
 def _create_test_rouge_evaluator(threshold: float) -> RougeEvaluator:
@@ -87,6 +94,183 @@ def test_calculate_rouge_1_scores():
   assert rouge_1_score.fmeasure == pytest.approx(8 / 11)
 
 
+@pytest.mark.parametrize(
+    "text",
+    [
+        "สวัสดี",  # Thai
+        "你好世界",  # Chinese
+        "مرحبا بالعالم",  # Arabic
+        "こんにちは",  # Japanese
+        "Здравствуйте",  # Russian
+    ],
+)
+def test_calculate_rouge_1_scores_identical_non_english_text(text: str):
+  rouge_1_score = _calculate_rouge_1_scores(text, text)
+  assert rouge_1_score.precision == pytest.approx(1)
+  assert rouge_1_score.recall == pytest.approx(1)
+  assert rouge_1_score.fmeasure == pytest.approx(1)
+
+
+def test_calculate_rouge_1_scores_different_non_english_text():
+  candidate = "мир привет"
+  reference = "привет только"
+  rouge_1_score = _calculate_rouge_1_scores(candidate, reference)
+  assert rouge_1_score.precision == pytest.approx(1 / 2)
+  assert rouge_1_score.recall == pytest.approx(1 / 2)
+  assert rouge_1_score.fmeasure == pytest.approx(1 / 2)
+
+
+def test_calculate_rouge_1_scores_cjk_partial_overlap_and_inversion():
+  candidate = "天气很好今天"
+  reference = "今天天气很好"
+  rouge_1_score = _calculate_rouge_1_scores(candidate, reference)
+  # Character-level matching: 6/6 characters overlap in unigram space.
+  assert rouge_1_score.precision == pytest.approx(1.0)
+  assert rouge_1_score.recall == pytest.approx(1.0)
+  assert rouge_1_score.fmeasure == pytest.approx(1.0)
+
+
+def test_calculate_rouge_1_scores_mixed_language_text():
+  candidate = "hello สวัสดี"
+  reference = "hello world"
+  rouge_1_score = _calculate_rouge_1_scores(candidate, reference)
+  # Candidate tokens: ['hello', 'สั', 'ส', 'ด', 'ดี'] (5 tokens).
+  # Reference tokens: ['hello', 'world'] (2 tokens).
+  assert rouge_1_score.precision == pytest.approx(1 / 5)
+  assert rouge_1_score.recall == pytest.approx(1 / 2)
+  assert rouge_1_score.fmeasure == pytest.approx(2 / 7)
+
+
+def test_unicode_aware_tokenizer_combining_marks_category_m():
+  """Tests that combining marks (category 'M', e.g. Thai vowel signs) stay attached to base characters."""
+  tokenizer = _UnicodeAwareTokenizer()
+
+  # Thai word "ดี" (Consonant 'ด' + Combining Mark Vowel ' ี' [category Mn]).
+  # Verifies that category 'M' combining marks hit the startswith("M") branch and attach to 'ด'.
+  # Extracting mark from "ดี"[1] ensures clean visual rendering without font overlap.
+  thai_vowel_mark = "ดี"[1]
+  assert unicodedata.category(thai_vowel_mark).startswith("M")
+
+  tokens = tokenizer.tokenize("ดี")
+  assert len(tokens) == 1
+  assert tokens[0] == "ดี"
+
+  # Hindi / Devanagari word "नमस्ते" (contains combining mark matras).
+  tokens_hindi = tokenizer.tokenize("नमस्ते")
+  assert len(tokens_hindi) == 1
+  assert tokens_hindi[0] == "नमस्ते"
+
+
+@pytest.mark.parametrize(
+    "input_text, use_stemmer, expected_tokens",
+    [
+        # Mixed English + Thai (with stemmer)
+        ("hello สวัสดี", True, ["hello", "ส", "วั", "ส", "ดี"]),
+        # Branch 1a: CJK Hanzi
+        ("中文测试", False, ["中", "文", "测", "试"]),
+        ("今天天气很好", False, ["今", "天", "天", "气", "很", "好"]),
+        # Branch 1b: CJK Hiragana
+        ("ひらがな", False, ["ひ", "ら", "が", "な"]),
+        ("こんにちは", False, ["こ", "ん", "に", "ち", "は"]),
+        # Branch 1c: CJK Katakana
+        ("カタカナ", False, ["カ", "タ", "カ", "ナ"]),
+        # Branch 1d: CJK Hangul
+        ("한글", False, ["한", "글"]),
+        # Branch 2a: Non-spaced script (Thai consonant + combining mark M)
+        ("ดี", False, ["ดี"]),
+        ("ฉันรักคุณมาก", False, ["ฉั", "น", "รั", "ก", "คุ", "ณ", "ม", "า", "ก"]),
+        # Branch 2b: Non-spaced script (Lao)
+        ("ດີ", False, ["ດີ"]),
+        # Branch 2c: Non-spaced script (Khmer)
+        ("ល្អ", False, ["ល្", "អ"]),
+        # Branch 2d: Non-spaced script (Myanmar)
+        ("မင်္ဂလာ", False, ["မ", "င်္", "ဂ", "လာ"]),
+        # Branch 3a: Alphanumeric ASCII (with and without stemmer)
+        ("Running jumped 123", True, ["run", "jump", "123"]),
+        ("Running jumped 123", False, ["running", "jumped", "123"]),
+        # Branch 3b & 3c: Non-ASCII spaced script with combining mark M (Arabic Harakat & Hindi Matra)
+        ("مَرْحَبًا", False, ["مَرْحَبًا"]),
+        ("नमस्ते", False, ["नमस्ते"]),
+        ("Hello World! Привет мир", True, ["hello", "world", "привет", "мир"]),
+        # Branch 4: Punctuation and non-word symbols (triggers else: append(" "))
+        ("hello, world! @123 #test", True, ["hello", "world", "123", "test"]),
+    ],
+)
+def test_unicode_aware_tokenizer_all_branches_coverage(
+    input_text: str, use_stemmer: bool, expected_tokens: list[str]
+):
+  """Verifies 100% branch coverage for all script types, combining marks, stemmer flag, and punctuation handling."""
+  tokenizer = _UnicodeAwareTokenizer(use_stemmer=use_stemmer)
+  assert tokenizer.tokenize(input_text) == expected_tokens
+
+
+@pytest.mark.parametrize(
+    "char, expected",
+    [
+        ("中", True),  # Hanzi
+        ("ぁ", True),  # Hiragana
+        ("ァ", True),  # Katakana
+        ("한", True),  # Hangul
+        ("a", False),
+        ("1", False),
+        ("ส", False),
+    ],
+)
+def test_is_cjk(char: str, expected: bool):
+  """Tests _is_cjk helper for Chinese, Hiragana, Katakana, and Hangul boundaries."""
+  assert _is_cjk(char) == expected
+
+
+@pytest.mark.parametrize(
+    "char, expected",
+    [
+        ("ส", True),  # Thai
+        ("ກ", True),  # Lao
+        ("ក", True),  # Khmer
+        ("က", True),  # Myanmar
+        ("中", False),
+        ("a", False),
+    ],
+)
+def test_is_non_spaced_script(char: str, expected: bool):
+  """Tests _is_non_spaced_script helper for Thai, Lao, Khmer, and Myanmar boundaries."""
+  assert _is_non_spaced_script(char) == expected
+
+
+@pytest.mark.parametrize(
+    "char, expected",
+    [
+        ("a", True),
+        ("9", True),
+        ("中", True),
+        ("ส", True),
+        ("ดี"[1], True),  # Combining Mark Category Mn (Thai Vowel)
+        (" ", False),
+        ("!", False),
+    ],
+)
+def test_is_word_char(char: str, expected: bool):
+  """Tests _is_word_char helper for alphanumerics and combining marks."""
+  assert _is_word_char(char) == expected
+
+
+@pytest.mark.parametrize(
+    "text",
+    [
+        "The quick brown fox jumps over the lazy dog.",
+        "Testing stemmed words like running and jumped, don't split!",
+        "Numbers 123 and mixed a1b2 tokens under_scored.",
+        "",
+    ],
+)
+def test_unicode_aware_tokenizer_matches_default_tokenizer_for_ascii(
+    text: str,
+):
+  default_tokens = tokenizers.DefaultTokenizer(use_stemmer=True).tokenize(text)
+  unicode_tokens = _UnicodeAwareTokenizer(use_stemmer=True).tokenize(text)
+  assert unicode_tokens == default_tokens
+
+
 @pytest.mark.parametrize(
     "candidates, references, expected_score, expected_status",
     [
@@ -114,6 +298,30 @@ def test_calculate_rouge_1_scores():
             1.0,
             EvalStatus.PASSED,
         ),
+        (
+            ["สวัสดี", "你好"],
+            ["สวัสดี", "你好"],
+            1.0,
+            EvalStatus.PASSED,
+        ),
+        (
+            ["今天天气不错", "我想吃炒饭"],
+            ["今天天气很好", "我想吃面条"],
+            0.63333,  # (2/3 + 3/5) / 2
+            EvalStatus.FAILED,
+        ),
+        (
+            ["สวัสดีครับ", "ฉันชอบกินข้าวผัด"],
+            ["สวัสดีค่ะ", "ฉันชอบกินก๋วยเตี๋ยว"],
+            0.61538,  # (8/13 + 8/13) / 2
+            EvalStatus.FAILED,
+        ),
+        (
+            ["你好世界", "人工智能"],
+            ["再见", "机器学习"],
+            0.0,
+            EvalStatus.FAILED,
+        ),
     ],
 )
 def test_rouge_evaluator_multiple_invocations(