issue/394 - feat: support flash-attn via MooreThreads/mate for moore gpu

Author: spike-zhu

csrc/pybind11/engine/engine.hpp

@@ -70,8 +70,13 @@ inline void bind_infer_engine(py::module &m) {
             return state_dict_tp_all;
         })
         .def("process_weights_after_loading", &InferEngine::process_weights_after_loading, "Process the weights after loading on all workers (e.g., for quantization)")
-        .def("forward", [](InferEngine &self, const InferEngine::Input &input) -> InferEngine::Output { return self.forward(input); }, "Run inference on all ranks with arbitrary arguments")
-        .def("reset_cache", [](InferEngine &self, std::shared_ptr<cache::CacheConfig> cfg) { self.reset_cache(cfg ? cfg.get() : nullptr); }, py::arg("cache_config") = py::none())
+        .def("forward", [](InferEngine &self, const InferEngine::Input &input) -> InferEngine::Output { return self.forward(input); },
+            "Run inference on all ranks with arbitrary arguments",
+            py::call_guard<py::gil_scoped_release>())
+        .def("reset_cache", [](InferEngine &self, std::shared_ptr<cache::CacheConfig> cfg) { self.reset_cache(cfg ? cfg.get() : nullptr); },
+            py::arg("cache_config") = py::none(),
+            py::call_guard<py::gil_scoped_release>())
+
         .def("get_cache_config", [](const InferEngine &self) -> std::shared_ptr<cache::CacheConfig> {
             auto cfg = self.get_cache_config();
             return cfg ? std::shared_ptr<cache::CacheConfig>(cfg->unique_copy()) : nullptr; })

examples/bench.py

@@ -325,38 +325,38 @@ def run(
     #                                Warmup
     # ---------------------------------------------------------------------------- #
     if cfg.warmup:
-        warmup_steps = 1
+        print("=================== warmup start ===================")
 
-        # warmup cache capacity
-        warmup_cache_len = 128
-        warmup_batch = len(test.input_ids_list)
+        if cache_config is not None:
+            test.model.reset_cache(cache_config)
 
-        test.model.reset_cache(
-            StaticKVCacheConfig(
-                max_batch_size=warmup_batch,
-                max_cache_len=warmup_cache_len,
+        warmup_shapes = []
+        seen = set()
+        for _, case in cases_dict.items():
+            key = (case["batch_size"], case["input_len"])
+            if key in seen:
+                continue
+            seen.add(key)
+            warmup_shapes.append((case["batch_size"], case["input_len"]))
+
+        for w_batch, w_input_len in warmup_shapes:
+            tqdm.write(
+                f"\033[93m[warmup] batch={w_batch}, input_len={w_input_len}, "
+                f"will prefill + 3 decode steps\033[0m"
             )
-        )
-
-        avg_prompt_len = min(64, max(len(ids) for ids in test.input_ids_list))
 
-        warmup_ids = [
-            ids[:avg_prompt_len] if len(ids) >= avg_prompt_len else ids
-            for ids in test.input_ids_list
-        ]
+            warmup_ids = repeat_prompt(test.input_ids_list[0], target_length=w_input_len)
+            warmup_ids_list = [warmup_ids] * w_batch
+            warmup_input = infinicore.from_list(warmup_ids_list)
 
-        input_ids_infini = infinicore.from_list(warmup_ids)
-
-        print("=================== warmup start ===================")
-
-        for _ in range(warmup_steps):
             _ = test.model.generate(
-                input_ids_infini,
+                warmup_input,
                 GenerationConfig(
-                    max_new_tokens=5,  # decode kernel warmup
-                    temperature=cfg.temperature,
+                    max_new_tokens=3,
+                    eos_token_id=[],
                     top_k=cfg.top_k,
                     top_p=cfg.top_p,
+                    temperature=cfg.temperature,
                     stop_on_eos=False,
                 ),
                 _measure_and_log_time=False,