Merge branch 'release/2.2' into release/2.2

Author: Jiang-Jia-Jun

fastdeploy/engine/common_engine.py

@@ -527,8 +527,14 @@ def _fetch_request():
                 self.cfg.max_prefill_batch,
             )
 
+            if self.cfg.model_config.enable_mm:
+                self.resource_manager.check_and_free_block_tables()
+                available_blocks = self.resource_manager.available_block_num()
+            else:
+                available_blocks = self.cfg.cache_config.max_block_num_per_seq
+
             tasks = self.scheduler.get_requests(
-                available_blocks=self.cfg.cache_config.max_block_num_per_seq,
+                available_blocks=available_blocks,
                 block_size=self.cfg.cache_config.block_size,
                 reserved_output_blocks=self.cfg.cache_config.enc_dec_block_num,
                 max_num_batched_tokens=self.cfg.max_model_len,

fastdeploy/engine/sched/resource_manager_v1.py

@@ -348,6 +348,15 @@ def schedule(self):
                     if request.status == RequestStatus.WAITING:
                         # Enable prefix caching
                         if self.config.cache_config.enable_prefix_caching:
+                            if (
+                                self.config.cache_config.enable_hierarchical_cache
+                                and self.cache_manager.num_cpu_blocks > 0
+                            ):
+                                if not self.cache_manager.can_allocate_gpu_blocks(
+                                    (request.need_prefill_tokens + self.config.cache_config.block_size - 1)
+                                    // self.config.cache_config.block_size
+                                ):  # to prevent block allocation for matching in hierarchical cache and cause dead lock
+                                    break
                             success = self.get_prefix_cached_blocks(request)
                             if not success:
                                 self._free_blocks(request)
@@ -387,6 +396,15 @@ def schedule(self):
                             request.num_total_tokens
                         )  # Before preempted task rescheduled, preempted task has been sent to engine, no more tokens are output, here num_total_tokens should be static and correct
                         if self.config.cache_config.enable_prefix_caching:
+                            if (
+                                self.config.cache_config.enable_hierarchical_cache
+                                and self.cache_manager.num_cpu_blocks > 0
+                            ):
+                                if not self.cache_manager.can_allocate_gpu_blocks(
+                                    (request.need_prefill_tokens + self.config.cache_config.block_size - 1)
+                                    // self.config.cache_config.block_size
+                                ):  # to prevent block allocation for matching in hierarchical cache and cause dead lock
+                                    break
                             success = self.get_prefix_cached_blocks(request)
                             if not success:
                                 self._free_blocks(request)

requirements.txt

@@ -10,7 +10,7 @@ tqdm
 pynvml
 uvicorn==0.29.0
 fastapi
-paddleformers==0.2.1
+paddleformers>=0.2.3
 redis
 etcd3
 httpx

tests/cov_pytest.ini

@@ -22,3 +22,4 @@ addopts =
     --ignore=tests/operators/test_flash_mask_attn.py
     --ignore=tests/operators/test_w4afp8_gemm.py
     --ignore=tests/operators/test_tree_mask.py
+    --ignore=tests/model_loader/test_common_model.py