We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
1 parent ab2b4ca commit 0224d0cCopy full SHA for 0224d0c
1 file changed
tinybpe/tokenizer.py
@@ -230,8 +230,7 @@ def stream_decode_reset(self) -> None:
230
def __repr__(self) -> str:
231
"""Return a concise string representation."""
232
has_remap = self._bytes_maps is not None
233
- has_special = self._special_tokens is not None
234
- n_special = len(self._special_tokens) if has_special else 0
+ n_special = len(self._special_tokens) if self._special_tokens is not None else 0
235
return f"Tokenizer(n_vocab={self.n_vocab}, byte_remap={has_remap}, special_tokens={n_special})"
236
237
@property
0 commit comments