2525from fastdeploy .input .text_processor import DataProcessor
2626from fastdeploy .utils import envs
2727
28- MODEL_PATH = os .getenv ("MODEL_PATH" ,"" ) + "/ERNIE-4.5-0.3B-Paddle"
28+ MODEL_PATH = os .getenv ("MODEL_PATH" , "" ) + "/ERNIE-4.5-0.3B-Paddle"
2929
3030
3131class TestDecodeToken (unittest .TestCase ):
@@ -49,22 +49,22 @@ def setUp(self):
4949 self .data_processor .decode_status [self .req_id ] = [0 , 0 , [], "" ]
5050
5151 def _tokenize (self , text ):
52- return self .data_processor_obj .tokenizer (text , add_special_tokens = False )[' input_ids' ]
52+ return self .data_processor_obj .tokenizer (text , add_special_tokens = False )[" input_ids" ]
5353
5454 def _assert_cleaned_up (self ):
5555 self .assertNotIn (self .req_id , self .data_processor .decode_status )
5656
5757 def test_empty_end (self ):
5858 """Empty token_ids with is_end=True should return empty and cleanup"""
59- with patch .object (envs , ' FD_ENABLE_RETURN_TEXT' , True ):
59+ with patch .object (envs , " FD_ENABLE_RETURN_TEXT" , True ):
6060 delta_text , returned_tokens = self .engine ._decode_token ([], self .req_id , is_end = True )
6161 self .assertEqual (delta_text , "" )
6262 self .assertEqual (returned_tokens , [])
6363 self ._assert_cleaned_up ()
6464
6565 def test_incremental_decoding_and_cleanup (self ):
6666 """Tokens added in multiple steps should decode correctly and cleanup at end"""
67- with patch .object (envs , ' FD_ENABLE_RETURN_TEXT' , True ):
67+ with patch .object (envs , " FD_ENABLE_RETURN_TEXT" , True ):
6868 for char in ["你" , "好" ]:
6969 tokens = self ._tokenize (char )
7070 delta_text , _ = self .engine ._decode_token (tokens , self .req_id , is_end = False )
@@ -76,8 +76,9 @@ def test_incremental_decoding_and_cleanup(self):
7676 def test_undecoded_tokens_on_end (self ):
7777 """Test that tokens which produce no visible text during streaming
7878 are force-decoded when is_end=True"""
79- with patch .object (envs , 'FD_ENABLE_RETURN_TEXT' , True ), \
80- patch .dict (os .environ , {'DEBUG_DECODE' : '1' }):
79+ with patch .object (
80+ envs , "FD_ENABLE_RETURN_TEXT" , True
81+ ), patch .dict (os .environ , {"DEBUG_DECODE" : "1" }):
8182 all_delta = ""
8283
8384 delta_text , _ = self .engine ._decode_token ([109584 ], self .req_id , is_end = False )
0 commit comments