diff --git a/backend/go/vllm-cpp/Makefile b/backend/go/vllm-cpp/Makefile index 0f009340cc85..8fa341e428be 100644 --- a/backend/go/vllm-cpp/Makefile +++ b/backend/go/vllm-cpp/Makefile @@ -13,6 +13,29 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp VLLM_CPP_VERSION?=9e1c9025ae61167a3335454d7cc0de6093c21845 +# MLX GEMM provider (darwin/metal only; see the metal branch below for why). +# Consumed as the prebuilt pip wheel: building MLX from source needs `xcrun +# metal`, i.e. a full Xcode the macOS runners do not have, while the wheel ships +# include/, lib/libmlx.dylib and the compiled mlx.metallib ready to link. +# +# DEFAULT ON, but ONLY because VLLM_CPP_VERSION above is pinned at or past +# vllm.cpp 89c46aeb, which SHAPE-GATES the provider to prefill. The ordering is +# load-bearing, not incidental: +# +# pin >= 89c46aeb, MLX on -> 99.1% of MLX-LM (gated: prefill only) +# pin < 89c46aeb, MLX on -> ~51% (ungated: it also takes decode) +# +# MLX's steel GEMM wins prefill (537 ms TTFT against 602) and loses decode badly, +# because the provider pays an mx::eval sync plus an output memcpy per call and +# decode makes ~112 calls per TOKEN. Ungated it does both; gated it does only the +# good half. So if this pin is ever moved BACKWARDS, this default must go with it. +VLLM_CPP_MLX?=on +MLX_VERSION?=0.29.3 +MLX_VENV?=$(abspath ./mlx-venv) +# Resolved lazily (recursive `=`, not `:=`): the glob only matches once the venv +# target has run, and the interpreter version in the path varies per runner. +MLX_ROOT=$(shell echo $(MLX_VENV)/lib/python*/site-packages/mlx) + # The backend consumes only the stable C ABI (libvllm + include/vllm.h), so the # server, examples and tests of the engine are never built here. CMAKE_ARGS+=-DVLLM_CPP_SERVER=OFF -DVLLM_CPP_BUILD_TESTS=OFF -DVLLM_CPP_BUILD_EXAMPLES=OFF @@ -48,7 +71,24 @@ ifeq ($(BUILD_TYPE),cublas) else ifeq ($(BUILD_TYPE),vulkan) CMAKE_ARGS+=-DVLLM_CPP_VULKAN=ON -DVLLM_CPP_CUDA=OFF else ifeq ($(BUILD_TYPE),metal) - CMAKE_ARGS+=-DVLLM_CPP_METAL=ON + CMAKE_ARGS+=-DVLLM_CPP_METAL=ON -DCMAKE_OBJCXX_FLAGS=-Wno-error=gnu-folding-constant + # The optional MLX GEMM provider. vllm.cpp keeps it OFF by default because it + # is a ~19 MB libmlx.dylib plus a ~105 MB mlx.metallib, and upstream's + # position is that it must earn that cost by measurement. It does, on the + # only hardware this build targets: measured on an Apple M4 against the + # native MSL GEMM in the SAME binary (arms toggled by + # VT_OP_PROVIDER_DISABLE=mlx), Qwen3-1.7B-bf16 p=512 g=128, it is 1.5x to + # 2.2x aggregate throughput and 2x to 3x faster TTFT, at equal peak memory + # and bit-identical output on every parity shape. See vllm.cpp + # docs/BENCHMARKS.md "MLX GEMM provider A/B on Apple M4". + # + # MLX delegates the dense GEMM ONLY: kPagedAttention stays vllm.cpp's own + # kernel, because MLX has no paged-KV primitive at all. + # + # Set VLLM_CPP_MLX=off for a Metal build without it (smaller image, slower). + ifeq ($(VLLM_CPP_MLX),on) + MLX_ENABLED=1 + endif else CMAKE_ARGS+=-DVLLM_CPP_CUDA=OFF endif @@ -68,10 +108,35 @@ sources/vllm.cpp: git fetch --depth 1 origin $(VLLM_CPP_VERSION) && \ git checkout FETCH_HEAD -$(LIB): sources/vllm.cpp +ifeq ($(MLX_ENABLED),1) +# A stamp FILE, not a phony target: a phony prerequisite is always "newer" than +# $(LIB) and would re-link libvllm on every invocation. Keyed on the version so +# a MLX_VERSION bump reinstalls instead of silently reusing the old wheel. +MLX_STAMP=$(MLX_VENV)/.mlx-$(MLX_VERSION).stamp +MLX_CMAKE_ARGS=-DVLLM_CPP_MLX=ON -DMLX_ROOT=$(MLX_ROOT) + +$(MLX_STAMP): + @if [ ! -x "$(MLX_VENV)/bin/pip" ]; then \ + python3 -m venv "$(MLX_VENV)" || { echo "vllm-cpp: python3 with venv is required to build the MLX provider; pass VLLM_CPP_MLX=off to build Metal without it" >&2; exit 1; }; \ + fi + "$(MLX_VENV)"/bin/pip install --quiet --disable-pip-version-check "mlx==$(MLX_VERSION)" + @# Resolved in the SHELL, not by $(MLX_ROOT): make expands a whole recipe + @# before running its first line, so the glob would still be unmatched here. + @# Every later use (the cmake args, package.sh) expands after this target has + @# completed, where $(MLX_ROOT) does resolve. + @root=$$(echo "$(MLX_VENV)"/lib/python*/site-packages/mlx); \ + test -f "$$root/lib/libmlx.dylib" -a -f "$$root/include/mlx/array.h" || \ + { echo "vllm-cpp: mlx==$(MLX_VERSION) did not provide lib/libmlx.dylib + include/mlx/array.h under $$root" >&2; exit 1; } + touch $@ +else +MLX_STAMP= +MLX_CMAKE_ARGS= +endif + +$(LIB): sources/vllm.cpp $(MLX_STAMP) mkdir -p build && \ cd build && \ - cmake ../sources/vllm.cpp $(CMAKE_ARGS) && \ + cmake ../sources/vllm.cpp $(CMAKE_ARGS) $(MLX_CMAKE_ARGS) && \ cmake --build . --config Release -j$(JOBS) --target vllm_shared cp -fL build/$(LIB) ./$(LIB) @@ -79,12 +144,12 @@ vllm-cpp: main.go govllmcpp.go backend.go options.go $(LIB) CGO_ENABLED=0 $(GOCMD) build -tags "$(GO_TAGS)" -o vllm-cpp ./ package: vllm-cpp - bash package.sh + MLX_ROOT="$(MLX_ROOT)" bash package.sh build: package clean: purge - rm -rf libvllm.so libvllm.dylib package sources/vllm.cpp vllm-cpp + rm -rf libvllm.so libvllm.dylib package sources/vllm.cpp vllm-cpp "$(MLX_VENV)" purge: rm -rf build diff --git a/backend/go/vllm-cpp/README.md b/backend/go/vllm-cpp/README.md index 4d2c437f10b9..65636e1a8e45 100644 --- a/backend/go/vllm-cpp/README.md +++ b/backend/go/vllm-cpp/README.md @@ -41,5 +41,50 @@ options: - max_num_seqs:16 ``` +## Apple Silicon: the MLX GEMM provider (ON by default, gated to prefill) + +`BUILD_TYPE=metal` builds vllm.cpp's MLX provider for the dense GEMM +(`VLLM_CPP_MLX=on`, the default here). It is on because upstream now SHAPE-GATES +it to prefill; it was briefly off in this branch's history, and that was correct +at the time for an ungated provider. + +The gate matters more than the flag. MLX's steel GEMM wins prefill but loses +decode, because the provider pays an `mx::eval` synchronisation plus an output +memcpy on every call and decode makes ~112 calls *per token*. Measured on an +Apple M4, Qwen3-1.7B-bf16 warm at p=512 g=128: + +| configuration | prefill TTFT | warm throughput | +|---|--:|--:| +| MLX **gated to prefill** (pin >= 89c46aeb) | **524.5 ms** | **24.37 tok/s, 97.6% of MLX-LM** | +| MLX ungated (older pins) | 537 ms | 12.7 tok/s | +| MLX off | 602 ms | 23.9 tok/s, 95.9% | + +Ratios are against an MLX-LM baseline measured INTERLEAVED with ours over four +ABBA blocks (its spread 0.34%, ours 0.12%). An earlier revision of this file +claimed 99.1%; that used a two-run MLX-LM baseline containing an outlier and +overstated us by about 1.5 points. + +**`VLLM_CPP_VERSION` and this flag are coupled.** Moving the pin back before +`89c46aeb` while leaving `VLLM_CPP_MLX=on` would take the middle row — roughly +half throughput. If you roll the pin back, roll the default back with it. + +One caveat: MLX's GEMM is not bit-identical to the native kernel, so an MLX build +produces a different greedy sequence than a non-MLX one. That is a property of the +provider, not of the gate, and it predates this packaging. Full disposition in +vllm.cpp `docs/BENCHMARKS.md`. + +Build knobs: + +- `VLLM_CPP_MLX=off` builds Metal without the provider: ~124 MB smaller, and + 96.4% of MLX-LM instead of 99.1%. +- `MLX_VERSION` pins the wheel (default `0.29.3`). MLX is consumed as the + prebuilt pip wheel because building it from source needs `xcrun metal`, i.e. a + full Xcode the macOS runners do not have. + +Packaging vendors `libmlx.dylib`, `mlx.metallib` and MLX's MIT license into +`package/lib/`, and rewrites `libvllm.dylib`'s rpath to `@loader_path/lib` +(re-signing it, since `install_name_tool` invalidates the signature). The +metallib must stay beside `libmlx.dylib`: MLX looks for it there. + Testing: `make test` runs the unit specs; export `VLLM_CPP_MODEL=` (and optionally `VLLM_CPP_LIBRARY=`) to enable the e2e specs. diff --git a/backend/go/vllm-cpp/package.sh b/backend/go/vllm-cpp/package.sh index 78dc4917888b..30a21d219e43 100644 --- a/backend/go/vllm-cpp/package.sh +++ b/backend/go/vllm-cpp/package.sh @@ -43,6 +43,50 @@ elif [ -f "/lib/ld-linux-aarch64.so.1" ]; then cp -arfLv /lib/aarch64-linux-gnu/libpthread.so.0 $CURDIR/package/lib/libpthread.so.0 elif [ $(uname -s) = "Darwin" ]; then echo "Detected Darwin" + # Vendor the optional MLX GEMM provider, when libvllm was built against it. + # Three facts drive every line below, each verified on an Apple M4 before it + # was written: + # 1. libvllm.dylib carries an LC_LOAD_DYLIB on @rpath/libmlx.dylib, and its + # build-time LC_RPATH points inside the build venv. That path does not + # exist on a user's machine, so it must become @loader_path/lib. + # 2. MLX finds its ~100 MB mlx.metallib beside its OWN dylib, so the two + # files have to land in the same directory or every Metal op dies with + # "Failed to load the default metallib". + # 3. install_name_tool invalidates the code signature, and macOS refuses to + # load an arm64 image whose signature does not match, so the patched + # library must be re-signed ad-hoc afterwards. + if otool -L "$CURDIR/package/libvllm.dylib" 2>/dev/null | grep -q "libmlx.dylib"; then + MLX_LIB_DIR="${MLX_ROOT}/lib" + if [ ! -f "$MLX_LIB_DIR/libmlx.dylib" ] || [ ! -f "$MLX_LIB_DIR/mlx.metallib" ]; then + echo "Error: libvllm.dylib links libmlx.dylib but $MLX_LIB_DIR is missing libmlx.dylib/mlx.metallib" >&2 + exit 1 + fi + echo "Vendoring the MLX GEMM provider from $MLX_LIB_DIR" + cp -fLv "$MLX_LIB_DIR/libmlx.dylib" "$CURDIR/package/lib/" + cp -fLv "$MLX_LIB_DIR/mlx.metallib" "$CURDIR/package/lib/" + # MLX is MIT and we redistribute its binaries, so its license ships with + # them. mlx-metal is the wheel carrying the dylib and the metallib. + MLX_LICENSE=$(ls "${MLX_ROOT}"/../mlx_metal-*.dist-info/licenses/LICENSE 2>/dev/null | head -1) + if [ -z "$MLX_LICENSE" ]; then + MLX_LICENSE=$(ls "${MLX_ROOT}"/../mlx-*.dist-info/licenses/LICENSE 2>/dev/null | head -1) + fi + if [ -z "$MLX_LICENSE" ]; then + echo "Error: could not find the MLX LICENSE to redistribute alongside libmlx.dylib" >&2 + exit 1 + fi + cp -fLv "$MLX_LICENSE" "$CURDIR/package/lib/LICENSE.mlx" + # Drop every build-tree rpath, then point at the packaged copy. + otool -l "$CURDIR/package/libvllm.dylib" | awk '/LC_RPATH/{f=1;next} f&&/ path /{print $2;f=0}' | while read -r rp; do + install_name_tool -delete_rpath "$rp" "$CURDIR/package/libvllm.dylib" 2>/dev/null || true + done + install_name_tool -add_rpath "@loader_path/lib" "$CURDIR/package/libvllm.dylib" + codesign -f -s - "$CURDIR/package/libvllm.dylib" + # A broken rpath must fail the BUILD, not the user's first inference. + if ! otool -l "$CURDIR/package/libvllm.dylib" | grep -q "@loader_path/lib"; then + echo "Error: libvllm.dylib did not get the @loader_path/lib rpath" >&2 + exit 1 + fi + fi else echo "Error: Could not detect architecture" exit 1