up

Author: juncaipeng

fastdeploy/cache_manager/prefix_cache_manager.py

@@ -881,7 +881,7 @@ def request_match_blocks(self, task: Request, block_size, *args):
                     read_storage_task = ReadStorageTask(
                         task_id=req_id,
                         keys=no_match_block_keys,
-                        token_ids=input_token_ids,
+                        token_ids=input_token_ids if self.kvcache_storage_backend == "attention_store" else None,
                         gpu_block_ids=gpu_recv_storage_block_ids,
                         start_read_block_idx=match_token_num // block_size,
                     )
@@ -1162,7 +1162,7 @@ def write_cache_to_storage(self, request: Request):
         write_storage_task = WriteStorageTask(
             task_id=req_id,
             keys=keys,
-            token_ids=input_token_ids,
+            token_ids=input_token_ids if self.kvcache_storage_backend == "attention_store" else None,
             gpu_block_ids=gpu_block_ids,
         )
         logger.debug(f"issue write storage task: {write_storage_task}")
@@ -1241,7 +1241,7 @@ def write_cache_to_storage_decode(self, request: Request):
         write_storage_task = WriteStorageTask(
             task_id=req_id,
             keys=keys,
-            token_ids=input_token_ids,
+            token_ids=input_token_ids if self.kvcache_storage_backend == "attention_store" else None,
             gpu_block_ids=gpu_block_ids,
         )
 
@@ -2171,7 +2171,7 @@ def recv_data_transfer_result(self):
                 event_type = data[0]
 
                 if event_type.value == CacheStatus.STORAGE2GPU.value:
-                    logger.info(f"recv_data_transfer_result: {data}")
+                    logger.debug(f"recv_data_transfer_result: {data}")
                     task_id, hash_keys, block_ids = data[1:]
                     if task_id not in self.storage_prefetch_block_ids:
                         self.storage_prefetch_block_ids[task_id] = []
@@ -2182,7 +2182,7 @@ def recv_data_transfer_result(self):
                         if task_id in self.task_prefetch_event:
                             self.task_prefetch_event[task_id].set()
                 elif event_type.value == CacheStatus.GPU2STORAGE.value:
-                    logger.info(f"recv_data_transfer_result: {data}")
+                    logger.debug(f"recv_data_transfer_result: {data}")
                     task_id, hash_keys, block_ids = data[1:]
                     if task_id in self.task_write_back_event:
                         self.task_write_back_event[task_id].set()

fastdeploy/envs.py

@@ -254,7 +254,7 @@ def _validate_split_kv_size(value: int) -> int:
     # When v1 is enabled, the legacy /clear_load_weight and /update_model_weight
     # will adopt this new communication pattern.
     "FD_ENABLE_V1_UPDATE_WEIGHTS": lambda: bool(int(os.getenv("FD_ENABLE_V1_UPDATE_WEIGHTS", "0"))),
-    # Whether to save the cache of output token for preemted request to radix tree or storage.
+    # Whether to save the cache of output token for preempted request to storage.
     "FD_SAVE_OUTPUT_CACHE_FOR_PREEMPTED_REQUEST": lambda: bool(
         int(os.getenv("FD_SAVE_OUTPUT_CACHE_FOR_PREEMPTED_REQUEST", "1"))
     ),