fix: add missing f-prefix in _normalize_token() error message

prince-shakyaa · prince-shakyaa · commit 7c369a69e7e4 · 2026-05-24T04:20:19.000+05:30
The error message was a plain string, causing {token!r} to be printed literally instead of interpolating the actual token value. This makes debugging stop_tokens / forbidden_tokens misconfigurations impossible. Fixes #658
diff --git a/gemma/gm/text/_sampler.py b/gemma/gm/text/_sampler.py
@@ -577,7 +577,7 @@ def _normalize_token(tokenizer, token: str | int) -> int:
   token_id = tokenizer.encode(token)
   if len(token_id) != 1:
     raise ValueError(
-        'Invalid token: {token!r}. `stop_token`s and `forbidden_token`s must'
+        f'Invalid token: {token!r}. `stop_token`s and `forbidden_token`s must'
         ' map to single token ids in the vocab.'
     )
   (token_id,) = token_id

Original file line number	Diff line number	Diff line change
`@@ -577,7 +577,7 @@ def _normalize_token(tokenizer, token: str \| int) -> int:`
`577`	`577`	`token_id = tokenizer.encode(token)`
`578`	`578`	`if len(token_id) != 1:`
`579`	`579`	`raise ValueError(`
`580`		- 'Invalid token: {token!r}. `stop_token`s and `forbidden_token`s must'
	`580`	+ f'Invalid token: {token!r}. `stop_token`s and `forbidden_token`s must'
`581`	`581`	`' map to single token ids in the vocab.'`
`582`	`582`	`)`
`583`	`583`	`(token_id,) = token_id`