From cb0c7ee8a84aa29c5cf3fa3d77ac90e93c91f9f2 Mon Sep 17 00:00:00 2001 From: Yingge He Date: Wed, 3 Jun 2026 17:24:04 -0700 Subject: [PATCH] remove tokenizer.all_special_tokens_extended --- python/openai/openai_frontend/engine/utils/tokenizer.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/python/openai/openai_frontend/engine/utils/tokenizer.py b/python/openai/openai_frontend/engine/utils/tokenizer.py index 8c21c63773..73184e843f 100644 --- a/python/openai/openai_frontend/engine/utils/tokenizer.py +++ b/python/openai/openai_frontend/engine/utils/tokenizer.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2025, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2024-2026, NVIDIA CORPORATION. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -47,7 +47,6 @@ def get_cached_tokenizer( function caches these properties for faster access.""" tokenizer_all_special_ids = set(tokenizer.all_special_ids) - tokenizer_all_special_tokens_extended = tokenizer.all_special_tokens_extended tokenizer_all_special_tokens = set(tokenizer.all_special_tokens) tokenizer_len = len(tokenizer) @@ -60,10 +59,6 @@ def all_special_ids(self): def all_special_tokens(self): return tokenizer_all_special_tokens - @property - def all_special_tokens_extended(self): - return tokenizer_all_special_tokens_extended - def __len__(self): return tokenizer_len