We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
2 parents 0a6fc9e + a59588e commit 09bae21Copy full SHA for 09bae21
1 file changed
tinybpe/tokenizer.py
@@ -153,10 +153,8 @@ def __init__(
153
# so we build it once here and cache the inverse-remapped result.
154
self._vocab_cache: dict[int, bytes] | None = None
155
if bytes_maps is not None:
156
- self._vocab_cache = {
157
- k: self._inv_map(v)
158
- for k, v in self._enc.vocab.items() # type: ignore[union-attr]
159
- }
+ assert self._inv_map is not None # guaranteed when bytes_maps is set
+ self._vocab_cache = {k: self._inv_map(v) for k, v in self._enc.vocab.items()}
160
161
# ------------------------------------------------------------------
162
# Encoding
0 commit comments