feat: add embedded-python model executor for qwen3 on cuda.

docker/Dockerfile.cuda

@@ -14,6 +14,7 @@ ARG TORCH_INDEX_URL=https://download.pytorch.org/whl/cu128
 ARG TORCH_CUDA_ARCH_LIST="8.0 8.9 9.0a 10.0a 12.0a"
 
 ARG FLASHINFER_VERSION=0.6.2
+ARG FLASHINFER_PYTHON_VERSION=0.6.14
 ARG FLASHINFER_CUDA_ARCH_LIST="8.0 8.9 9.0a 10.0a 12.0a"
 
 ARG GET_PIP_URL="https://bootstrap.pypa.io/get-pip.py"
@@ -149,6 +150,27 @@ RUN set -eux; \
     rm -rf /workspace/flashinfer
 
 ENV FLASHINFER_OPS_PATH=/flashinfer_ops
+
+# Install FlashInfer globally after the AOT build. Point the Python package at
+# the TVM-FFI library linked by xLLM to avoid duplicate global registrations.
+RUN set -eux; \
+    pip install --ignore-installed "packaging>=24.2"; \
+    pip install "flashinfer-python==${FLASHINFER_PYTHON_VERSION}"; \
+    site_packages="$(python -c 'import sysconfig; print(sysconfig.get_paths()["purelib"])')"; \
+    tvm_ffi_lib_dir="${site_packages}/tvm_ffi/lib"; \
+    test -f "${tvm_ffi_lib_dir}/libtvm_ffi.so"; \
+    test -f "${tvm_ffi_lib_dir}/libtvm_ffi_testing.so"; \
+    cp -f "${tvm_ffi_lib_dir}/libtvm_ffi_testing.so" \
+      /usr/local/lib/libtvm_ffi_testing.so; \
+    ln -sfn /usr/local/lib/libtvm_ffi.so \
+      "${tvm_ffi_lib_dir}/libtvm_ffi.so"; \
+    ln -sfn /usr/local/lib/libtvm_ffi_testing.so \
+      "${tvm_ffi_lib_dir}/libtvm_ffi_testing.so"; \
+    test "$(stat -Lc '%d:%i' /usr/local/lib/libtvm_ffi.so)" = \
+      "$(stat -Lc '%d:%i' "${tvm_ffi_lib_dir}/libtvm_ffi.so")"; \
+    test "$(stat -Lc '%d:%i' /usr/local/lib/libtvm_ffi_testing.so)" = \
+      "$(stat -Lc '%d:%i' "${tvm_ffi_lib_dir}/libtvm_ffi_testing.so")"
+
 ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH
 ENV LD_LIBRARY_PATH=/usr/lib/x86_64-linux-gnu:$LD_LIBRARY_PATH
 
@@ -166,4 +188,4 @@ RUN if [[ -n "${GIT_HTTPS_MIRROR_PREFIX}" ]]; then \
     uv cache clean
 
 USER root
-CMD ["/bin/bash"]
+CMD ["/bin/bash"]

setup.py

@@ -461,6 +461,23 @@ def build_cmake_targets(
             os.path.join(os.path.dirname(cmake_dir), "xllm/core/server/"),
         )
 
+        # Stage the Python model-executor package into the wheel under the xLLM
+        # install tree (xllm/python/...). It is deliberately NOT declared as a
+        # top-level site-packages package (that would make ``import python``
+        # shadow globally); it is imported only when the containing xllm/ dir is
+        # prepended to sys.path (see --python_model_path / XLLM_PYTHON_MODEL_PATH,
+        # which launch_server.py defaults to the packaged xllm/ dir).
+        py_pkg_src = os.path.join(self.base_dir, "xllm", "python")
+        if os.path.isdir(py_pkg_src):
+            py_pkg_dst = os.path.join(extdir, "python")
+            if os.path.isdir(py_pkg_dst):
+                shutil.rmtree(py_pkg_dst)
+            shutil.copytree(
+                py_pkg_src,
+                py_pkg_dst,
+                ignore=shutil.ignore_patterns("__pycache__", "*.pyc"),
+            )
+
         _stage_triton_npu_runtime_binaries(self.base_dir, extdir, self.device)
 
         _stage_mlu_triton_kernels(self.base_dir, extdir, self.device)

tests/core/framework/config/config_json_test.cpp

@@ -23,7 +23,9 @@ limitations under the License.
 
 #include "core/common/global_flags.h"
 #include "core/framework/config/config_utils.h"
+#include "core/framework/config/execution_config.h"
 #include "core/framework/config/kv_cache_config.h"
+#include "core/framework/config/model_config.h"
 #include "core/framework/config/scheduler_config.h"
 
 namespace xllm {
@@ -34,7 +36,10 @@ inline constexpr std::string_view kInlineConfig = R"json({
   "max_memory_utilization": 0.5,
   "enable_prefix_cache": false,
   "max_tokens_per_batch": 8192,
-  "max_seqs_per_batch": 64
+  "max_seqs_per_batch": 64,
+  "model_impl": "py",
+  "python_model_path": "/tmp/xllm-python-model",
+  "python_graph_backend": "cudagraphs"
 })json";
 
 inline constexpr std::string_view kUpdatedConfig = R"json({
@@ -77,30 +82,80 @@ class DumpConfigJsonFlagGuard final {
   std::string old_dump_config_json_file_;
 };
 
+class ConfigFlagGuard final {
+ public:
+  ConfigFlagGuard()
+      : old_block_size_(FLAGS_block_size),
+        old_max_memory_utilization_(FLAGS_max_memory_utilization),
+        old_enable_prefix_cache_(FLAGS_enable_prefix_cache),
+        old_max_tokens_per_batch_(FLAGS_max_tokens_per_batch),
+        old_max_seqs_per_batch_(FLAGS_max_seqs_per_batch),
+        old_model_impl_(FLAGS_model_impl),
+        old_python_model_path_(FLAGS_python_model_path),
+        old_python_graph_backend_(FLAGS_python_graph_backend) {}
+
+  ~ConfigFlagGuard() {
+    FLAGS_block_size = old_block_size_;
+    FLAGS_max_memory_utilization = old_max_memory_utilization_;
+    FLAGS_enable_prefix_cache = old_enable_prefix_cache_;
+    FLAGS_max_tokens_per_batch = old_max_tokens_per_batch_;
+    FLAGS_max_seqs_per_batch = old_max_seqs_per_batch_;
+    FLAGS_model_impl = old_model_impl_;
+    FLAGS_python_model_path = old_python_model_path_;
+    FLAGS_python_graph_backend = old_python_graph_backend_;
+  }
+
+ private:
+  int32_t old_block_size_;
+  double old_max_memory_utilization_;
+  bool old_enable_prefix_cache_;
+  int32_t old_max_tokens_per_batch_;
+  int32_t old_max_seqs_per_batch_;
+  std::string old_model_impl_;
+  std::string old_python_model_path_;
+  std::string old_python_graph_backend_;
+};
+
 class StartupConfigGuard final {
  public:
   StartupConfigGuard()
-      : kv_cache_config_(KVCacheConfig::get_instance()),
+      : model_config_(ModelConfig::get_instance()),
+        execution_config_(ExecutionConfig::get_instance()),
+        kv_cache_config_(KVCacheConfig::get_instance()),
         scheduler_config_(SchedulerConfig::get_instance()),
+        old_model_impl_(model_config_.model_impl()),
+        old_python_model_path_(model_config_.python_model_path()),
+        old_python_graph_backend_(execution_config_.python_graph_backend()),
         old_block_size_(kv_cache_config_.block_size()),
         old_enable_prefix_cache_(kv_cache_config_.enable_prefix_cache()),
         old_max_tokens_per_batch_(scheduler_config_.max_tokens_per_batch()),
+        old_max_seqs_per_batch_(scheduler_config_.max_seqs_per_batch()),
         old_enable_chunked_prefill_(
             scheduler_config_.enable_chunked_prefill()) {}
 
   ~StartupConfigGuard() {
+    model_config_.model_impl(old_model_impl_)
+        .python_model_path(old_python_model_path_);
+    execution_config_.python_graph_backend(old_python_graph_backend_);
     kv_cache_config_.block_size(old_block_size_)
         .enable_prefix_cache(old_enable_prefix_cache_);
     scheduler_config_.max_tokens_per_batch(old_max_tokens_per_batch_)
+        .max_seqs_per_batch(old_max_seqs_per_batch_)
         .enable_chunked_prefill(old_enable_chunked_prefill_);
   }
 
  private:
+  ModelConfig& model_config_;
+  ExecutionConfig& execution_config_;
   KVCacheConfig& kv_cache_config_;
   SchedulerConfig& scheduler_config_;
+  std::string old_model_impl_;
+  std::string old_python_model_path_;
+  std::string old_python_graph_backend_;
   int32_t old_block_size_;
   bool old_enable_prefix_cache_;
   int32_t old_max_tokens_per_batch_;
+  int32_t old_max_seqs_per_batch_;
   bool old_enable_chunked_prefill_;
 };
 
@@ -133,6 +188,13 @@ std::filesystem::path config_test_file_path() {
 
 TEST(ConfigJsonTest, FromJsonUsesParsedOverrides) {
   const JsonReader json = config::parse_json_string(kInlineConfig);
+  ConfigFlagGuard flag_guard;
+
+  ModelConfig model_config;
+  model_config.from_json(json);
+
+  ExecutionConfig execution_config;
+  execution_config.from_json(json);
 
   KVCacheConfig kv_cache_config;
   kv_cache_config.from_flags();
@@ -147,12 +209,24 @@ TEST(ConfigJsonTest, FromJsonUsesParsedOverrides) {
   EXPECT_FALSE(kv_cache_config.enable_prefix_cache());
   EXPECT_EQ(scheduler_config.max_tokens_per_batch(), 8192);
   EXPECT_EQ(scheduler_config.max_seqs_per_batch(), 64);
+  // model_impl is no longer canonicalized: the raw "py" alias is preserved and
+  // recognized via is_python_model_impl(). from_json still mirrors it into the
+  // matching gflag.
+  EXPECT_EQ(model_config.model_impl(), "py");
+  EXPECT_TRUE(ModelConfig::is_python_model_impl(model_config.model_impl()));
+  EXPECT_EQ(model_config.python_model_path(), "/tmp/xllm-python-model");
+  EXPECT_EQ(execution_config.python_graph_backend(), "cudagraphs");
+
+  EXPECT_EQ(FLAGS_model_impl, "py");
+  EXPECT_EQ(FLAGS_python_model_path, "/tmp/xllm-python-model");
+  EXPECT_EQ(FLAGS_python_graph_backend, "cudagraphs");
 
   EXPECT_EQ(kv_cache_config.kv_cache_dtype(), "auto");
   EXPECT_EQ(scheduler_config.max_decode_token_per_sequence(), 256);
 }
 
 TEST(ConfigJsonTest, LoadJsonFileReadsConfigFixture) {
+  ConfigFlagGuard flag_guard;
   const std::filesystem::path config_path = config_test_file_path();
   ASSERT_TRUE(std::filesystem::exists(config_path)) << config_path;
 
@@ -194,6 +268,7 @@ TEST(ConfigJsonTest, LoadJsonFileReadsConfigFixture) {
 }
 
 TEST(ConfigJsonTest, InitializeLoadsConfigJsonFileFromFlag) {
+  ConfigFlagGuard config_flag_guard;
   const std::filesystem::path config_path =
       std::filesystem::temp_directory_path() / "xllm_config_json_test.json";
   write_config_file(config_path, kInlineConfig);
@@ -213,6 +288,7 @@ TEST(ConfigJsonTest, InitializeLoadsConfigJsonFileFromFlag) {
 }
 
 TEST(ConfigJsonTest, InitializeReusesCachedConfigJsonForSameFile) {
+  ConfigFlagGuard config_flag_guard;
   const std::filesystem::path config_path =
       std::filesystem::temp_directory_path() /
       "xllm_config_json_test_cached.json";
@@ -295,24 +371,33 @@ TEST(ConfigJsonTest, DumpStartupConfigWritesNonDefaultValuesOnly) {
   DumpConfigJsonFlagGuard flag_guard(dump_path.string());
   StartupConfigGuard startup_config_guard;
 
+  ModelConfig::get_instance().model_impl("python").python_model_path(
+      "/tmp/xllm-python-model");
+  ExecutionConfig::get_instance().python_graph_backend("cudagraphs");
   KVCacheConfig::get_instance().block_size(256).enable_prefix_cache(false);
   SchedulerConfig::get_instance()
       .max_tokens_per_batch(2048)
+      .max_seqs_per_batch(128)
       .enable_chunked_prefill(false);
   FLAGS_enable_dump_config_json = true;
 
   config::dump_startup_config();
 
   ASSERT_TRUE(std::filesystem::exists(dump_path));
   const nlohmann::ordered_json config_json = read_json_file(dump_path);
+  EXPECT_EQ(config_json.at("model_impl").get<std::string>(), "python");
+  EXPECT_EQ(config_json.at("python_model_path").get<std::string>(),
+            "/tmp/xllm-python-model");
+  EXPECT_EQ(config_json.at("python_graph_backend").get<std::string>(),
+            "cudagraphs");
   EXPECT_EQ(config_json.at("block_size").get<int32_t>(), 256);
   EXPECT_FALSE(config_json.at("enable_prefix_cache").get<bool>());
   EXPECT_EQ(config_json.at("max_tokens_per_batch").get<int32_t>(), 2048);
+  EXPECT_EQ(config_json.at("max_seqs_per_batch").get<int32_t>(), 128);
   EXPECT_FALSE(config_json.at("enable_chunked_prefill").get<bool>());
 
   EXPECT_FALSE(config_json.contains("max_cache_size"));
   EXPECT_FALSE(config_json.contains("kv_cache_dtype"));
-  EXPECT_FALSE(config_json.contains("max_seqs_per_batch"));
   EXPECT_FALSE(config_json.contains("priority_strategy"));
 
   std::filesystem::remove(dump_path);

tests/core/kernels/cuda/CMakeLists.txt

@@ -69,3 +69,18 @@ cc_test(
     glog::glog
 )
 target_link_libraries(fp8_quant_test PRIVATE brpc)
+
+cc_test(
+  NAME
+    xllm_ops_test
+  SRCS
+    xllm_ops_test.cpp
+  DEPS
+    :cuda_kernels
+    torch
+    GTest::gtest_main
+    glog::glog
+)
+# Embedded-interpreter test path: needs libtorch_python (THPVariable / at::Tensor
+# pybind caster) and libpython.
+target_link_libraries(xllm_ops_test PRIVATE torch_python Python::Python)