Merge pull request #879 from docker/use-llamacpp-images

Use official llama.cpp images for Linux

Author: ericcurtin

.github/workflows/release.yml

@@ -31,12 +31,6 @@ on:
         description: "SGLang version"
         required: false
         type: string
-      # This can be removed once we have llama.cpp built for MUSA and CANN.
-      buildMusaCann:
-        description: "Build MUSA and CANN images"
-        required: false
-        type: boolean
-        default: false
       imagesOnly:
         description: "Only build and push Docker images (skip CLI releases, pinata bump, docs update, and CE packaging)"
         required: false
@@ -243,7 +237,6 @@ jobs:
       LLAMA_SERVER_VERSION: ${{ inputs.llamaServerVersion }}
       VLLM_VERSION: ${{ inputs.vllmVersion }}
       SGLANG_VERSION: ${{ inputs.sglangVersion }}
-      BUILD_MUSA_CANN: ${{ inputs.buildMusaCann || 'false' }}
     steps:
       - name: Checkout repo
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
@@ -272,15 +265,7 @@ jobs:
           echo "docker/model-runner:$RELEASE_TAG-rocm" >> "$GITHUB_OUTPUT"
           echo "docker/model-runner:latest-rocm" >> "$GITHUB_OUTPUT"
           echo 'EOF' >> "$GITHUB_OUTPUT"
-          echo "musa<<EOF" >> "$GITHUB_OUTPUT"
-          echo "docker/model-runner:$RELEASE_TAG-musa" >> "$GITHUB_OUTPUT"
-          echo "docker/model-runner:latest-musa" >> "$GITHUB_OUTPUT"
-          echo 'EOF' >> "$GITHUB_OUTPUT"
-          echo "cann<<EOF" >> "$GITHUB_OUTPUT"
-          echo "docker/model-runner:$RELEASE_TAG-cann" >> "$GITHUB_OUTPUT"
-          echo "docker/model-runner:latest-cann" >> "$GITHUB_OUTPUT"
-          echo 'EOF' >> "$GITHUB_OUTPUT"
-          
+
       - name: Load versions
         shell: bash
         run: |
@@ -293,6 +278,21 @@ jobs:
           echo "VLLM_VERSION=${VLLM_VERSION:-$VERSIONS_VLLM}"                   >> "$GITHUB_ENV"
           echo "SGLANG_VERSION=${SGLANG_VERSION:-$VERSIONS_SGLANG}"             >> "$GITHUB_ENV"
 
+      - name: Resolve llama.cpp upstream images
+        id: llama-images
+        shell: bash
+        run: |
+          CPU_IMAGE=$(bash scripts/resolve-llama-upstream-image.sh \
+            "$LLAMA_SERVER_VERSION" cpu)
+          CUDA_IMAGE=$(bash scripts/resolve-llama-upstream-image.sh \
+            "$LLAMA_SERVER_VERSION" cuda)
+          ROCM_IMAGE=$(bash scripts/resolve-llama-upstream-image.sh \
+            "$LLAMA_SERVER_VERSION" rocm)
+
+          echo "cpu=$CPU_IMAGE" >> "$GITHUB_OUTPUT"
+          echo "cuda=$CUDA_IMAGE" >> "$GITHUB_OUTPUT"
+          echo "rocm=$ROCM_IMAGE" >> "$GITHUB_OUTPUT"
+
       - name: Log in to DockerHub
         uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121
         with:
@@ -315,6 +315,8 @@ jobs:
           platforms: linux/amd64, linux/arm64
           build-args: |
             "LLAMA_SERVER_VERSION=${{ env.LLAMA_SERVER_VERSION }}"
+            "LLAMA_SERVER_VARIANT=cpu"
+            "LLAMA_UPSTREAM_IMAGE=${{ steps.llama-images.outputs.cpu }}"
             "VERSION=${{ env.RELEASE_TAG }}"
           push: true
           sbom: true
@@ -330,7 +332,7 @@ jobs:
           build-args: |
             "LLAMA_SERVER_VERSION=${{ env.LLAMA_SERVER_VERSION }}"
             "LLAMA_SERVER_VARIANT=cuda"
-            "BASE_IMAGE=nvidia/cuda:12.9.0-runtime-ubuntu24.04"
+            "LLAMA_UPSTREAM_IMAGE=${{ steps.llama-images.outputs.cuda }}"
             "VERSION=${{ env.RELEASE_TAG }}"
           push: true
           sbom: true
@@ -346,7 +348,7 @@ jobs:
           build-args: |
             "LLAMA_SERVER_VERSION=${{ env.LLAMA_SERVER_VERSION }}"
             "LLAMA_SERVER_VARIANT=cuda"
-            "BASE_IMAGE=nvidia/cuda:13.0.2-runtime-ubuntu24.04"
+            "LLAMA_UPSTREAM_IMAGE=${{ steps.llama-images.outputs.cuda }}"
             "VLLM_VERSION=${{ env.VLLM_VERSION }}"
             "VERSION=${{ env.RELEASE_TAG }}"
           push: true
@@ -363,7 +365,7 @@ jobs:
           build-args: |
             "LLAMA_SERVER_VERSION=${{ env.LLAMA_SERVER_VERSION }}"
             "LLAMA_SERVER_VARIANT=cuda"
-            "BASE_IMAGE=nvidia/cuda:12.9.0-runtime-ubuntu24.04"
+            "LLAMA_UPSTREAM_IMAGE=${{ steps.llama-images.outputs.cuda }}"
             "SGLANG_VERSION=${{ env.SGLANG_VERSION }}"
             "VERSION=${{ env.RELEASE_TAG }}"
           push: true
@@ -380,47 +382,13 @@ jobs:
           build-args: |
             "LLAMA_SERVER_VERSION=${{ env.LLAMA_SERVER_VERSION }}"
             "LLAMA_SERVER_VARIANT=rocm"
-            "BASE_IMAGE=rocm/dev-ubuntu-22.04"
+            "LLAMA_UPSTREAM_IMAGE=${{ steps.llama-images.outputs.rocm }}"
             "VERSION=${{ env.RELEASE_TAG }}"
           push: true
           sbom: true
           provenance: mode=max
           tags: ${{ steps.tags.outputs.rocm }}
 
-      - name: Build MUSA image
-        if: ${{ env.BUILD_MUSA_CANN == 'true' }}
-        uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
-        with:
-          file: Dockerfile
-          target: final-llamacpp
-          platforms: linux/amd64
-          build-args: |
-            "LLAMA_SERVER_VERSION=${{ env.LLAMA_SERVER_VERSION }}"
-            "LLAMA_SERVER_VARIANT=musa"
-            "BASE_IMAGE=mthreads/musa:rc4.3.0-runtime-ubuntu22.04-amd64"
-            "VERSION=${{ env.RELEASE_TAG }}"
-          push: true
-          sbom: true
-          provenance: mode=max
-          tags: ${{ steps.tags.outputs.musa }}
-
-      - name: Build CANN image
-        if: ${{ env.BUILD_MUSA_CANN == 'true' }}
-        uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f
-        with:
-          file: Dockerfile
-          target: final-llamacpp
-          platforms: linux/arm64, linux/amd64
-          build-args: |
-            "LLAMA_SERVER_VERSION=${{ env.LLAMA_SERVER_VERSION }}"
-            "LLAMA_SERVER_VARIANT=cann"
-            "BASE_IMAGE=ascendai/cann:8.2.rc2-910b-ubuntu22.04-py3.11"
-            "VERSION=${{ env.RELEASE_TAG }}"
-          push: true
-          sbom: true
-          provenance: mode=max
-          tags: ${{ steps.tags.outputs.cann }}
-
   # ---------------------------------------------------------------------------
   # Release CLI for Docker Desktop — build, sign & push CLI + Desktop module image
   # ---------------------------------------------------------------------------

.versions

@@ -4,5 +4,4 @@ VLLM_UPSTREAM_VERSION=0.19.0
 VLLM_METAL_RELEASE=v0.2.0-20260420-142150
 DIFFUSERS_RELEASE=v0.1.0-20260216-000000
 SGLANG_VERSION=0.5.6
-LLAMA_SERVER_VERSION=latest
-BASE_IMAGE=ubuntu:26.04
+LLAMA_SERVER_VERSION=b8967

Dockerfile

@@ -1,13 +1,9 @@
 # syntax=docker/dockerfile:1
 
 ARG GO_VERSION=1.25
-ARG LLAMA_SERVER_VERSION=latest
+ARG LLAMA_SERVER_VERSION=b8967
 ARG LLAMA_SERVER_VARIANT=cpu
-ARG LLAMA_BINARY_PATH=/com.docker.llama-server.native.linux.${LLAMA_SERVER_VARIANT}.${TARGETARCH}
-
-# only 26.04 for cpu variant for max hardware support with vulkan
-# use 22.04 for gpu variants to match ROCm/CUDA base images
-ARG BASE_IMAGE=ubuntu:26.04
+ARG LLAMA_UPSTREAM_IMAGE=ghcr.io/ggml-org/llama.cpp:server-vulkan-b8967
 
 ARG VERSION=dev
 
@@ -43,11 +39,8 @@ RUN --mount=type=cache,target=/go/pkg/mod \
     --mount=type=cache,target=/root/.cache/go-build \
     CGO_ENABLED=1 GOOS=linux go build -tags=novllm -ldflags="-s -w -X main.Version=${VERSION}" -o model-runner .
 
-# --- Get llama.cpp binary ---
-FROM docker/docker-model-backend-llamacpp:${LLAMA_SERVER_VERSION}-${LLAMA_SERVER_VARIANT} AS llama-server
-
-# --- Final image ---
-FROM docker.io/${BASE_IMAGE} AS llamacpp
+# --- Final image: directly FROM the upstream llama.cpp image ---
+FROM ${LLAMA_UPSTREAM_IMAGE} AS llamacpp
 
 ARG LLAMA_SERVER_VARIANT
 
@@ -57,30 +50,32 @@ RUN groupadd --system modelrunner && useradd --system --gid modelrunner -G video
 
 COPY scripts/ /scripts/
 
-# Install ca-certificates for HTTPS and vulkan
+# Install additional packages not shipped by the upstream image
+# (e.g. ca-certificates for HTTPS, mesa patches for aarch64 virtio-vulkan).
 RUN /scripts/apt-install.sh && rm -rf /scripts
 
 WORKDIR /app
 
-# Create directories for the socket file and llama.cpp binary, and set proper permissions
-RUN mkdir -p /var/run/model-runner /app/bin /models && \
+# Create directories for the socket file and set proper permissions
+RUN mkdir -p /var/run/model-runner /models && \
     chown -R modelrunner:modelrunner /var/run/model-runner /app /models && \
     chmod -R 755 /models
 
-# Copy the llama.cpp binary from the llama-server stage
-ARG LLAMA_BINARY_PATH
-COPY --from=llama-server ${LLAMA_BINARY_PATH}/ /app/.
-RUN chmod +x /app/bin/com.docker.llama-server
-
 USER modelrunner
 
-# Set the environment variable for the socket path and LLaMA server binary path
+# Set the environment variable for the socket path and LLamA server binary path.
+# LLAMA_SERVER_PATH points at the directory containing the llama-server binary
+# and its ggml backend plugins — keeping them together lets llama.cpp discover
+# backends via its default search path (relative to the binary).
 ENV MODEL_RUNNER_SOCK=/var/run/model-runner/model-runner.sock
 ENV MODEL_RUNNER_PORT=12434
-ENV LLAMA_SERVER_PATH=/app/bin
+ENV LLAMA_SERVER_PATH=/app
+# LD_LIBRARY_PATH is required so that backend plugins loaded via dlopen()
+# (e.g. libggml-cpu-*.so, libggml-vulkan.so) can resolve their transitive
+# dependencies on libggml-base.so and other shared libraries in /app.
+ENV LD_LIBRARY_PATH=/app
 ENV HOME=/home/modelrunner
 ENV MODELS_PATH=/models
-ENV LD_LIBRARY_PATH=/app/lib
 
 # Label the image so that it's hidden on cloud engines.
 LABEL com.docker.desktop.service="model-runner"

Makefile

@@ -3,22 +3,36 @@ include .versions
 
 APP_NAME := model-runner
 LLAMA_SERVER_VARIANT := cpu
-VLLM_BASE_IMAGE := nvidia/cuda:13.0.2-runtime-ubuntu24.04
+# Resolved lazily — only evaluated when a Docker target references it.
+LLAMA_UPSTREAM_IMAGE ?= $(shell \
+	bash scripts/resolve-llama-upstream-image.sh \
+	"$(LLAMA_SERVER_VERSION)" "$(LLAMA_SERVER_VARIANT)")
 DOCKER_IMAGE := docker/model-runner:latest
 DOCKER_IMAGE_VLLM := docker/model-runner:latest-vllm-cuda
 DOCKER_IMAGE_SGLANG := docker/model-runner:latest-sglang
 DOCKER_TARGET ?= final-llamacpp
 PORT := 8080
 LLAMA_ARGS ?=
 E2E_TIMEOUT ?= 30m
-DOCKER_BUILD_ARGS := \
-	--load \
-	--platform linux/$(shell docker version --format '{{.Server.Arch}}') \
+
+define check-llama-image
+$(if $(LLAMA_UPSTREAM_IMAGE),,$(error Failed to resolve llama.cpp upstream image. Check LLAMA_SERVER_VERSION and LLAMA_SERVER_VARIANT or set LLAMA_UPSTREAM_IMAGE directly.))
+endef
+
+ifeq ($(LLAMA_SERVER_VARIANT),rocm)
+DOCKER_BUILD_PLATFORMS := linux/amd64
+else
+DOCKER_BUILD_PLATFORMS := linux/amd64,linux/arm64
+endif
+
+LOCAL_DOCKER_PLATFORM ?= linux/$(shell docker version --format '{{.Server.Arch}}')
+
+DOCKER_BUILD_COMMON_ARGS = \
 	--build-arg GO_VERSION=$(GO_VERSION) \
 	--build-arg LLAMA_SERVER_VERSION=$(LLAMA_SERVER_VERSION) \
 	--build-arg LLAMA_SERVER_VARIANT=$(LLAMA_SERVER_VARIANT) \
+	--build-arg LLAMA_UPSTREAM_IMAGE=$(LLAMA_UPSTREAM_IMAGE) \
 	--build-arg SGLANG_VERSION=$(SGLANG_VERSION) \
-	--build-arg BASE_IMAGE=$(BASE_IMAGE) \
 	--build-arg VLLM_VERSION='$(VLLM_VERSION)' \
 	--target $(DOCKER_TARGET) \
 	-t $(DOCKER_IMAGE)
@@ -102,7 +116,7 @@ e2e:
 test-docker-ce-installation:
 	@echo "Testing Docker CE installation..."
 	@echo "Note: This requires Docker to be running"
-	BASE_IMAGE=$(BASE_IMAGE) scripts/test-docker-ce-installation.sh
+	scripts/test-docker-ce-installation.sh
 
 validate:
 	find . -type f -name "*.sh" | grep -v "pkg/go-containerregistry\|llamacpp/native/vendor" | xargs shellcheck
@@ -154,11 +168,13 @@ validate-all:
 
 # Build Docker image
 docker-build:
-	docker buildx build $(DOCKER_BUILD_ARGS) .
+	$(call check-llama-image)
+	docker buildx build --load --platform $(LOCAL_DOCKER_PLATFORM) $(DOCKER_BUILD_COMMON_ARGS) .
 
 # Build multi-platform Docker image
 docker-build-multiplatform:
-	docker buildx build --platform linux/amd64,linux/arm64 $(DOCKER_BUILD_ARGS) .
+	$(call check-llama-image)
+	docker buildx build --platform $(DOCKER_BUILD_PLATFORMS) $(DOCKER_BUILD_COMMON_ARGS) .
 
 # Run in Docker container with TCP port access and mounted model storage
 docker-run: docker-build
@@ -169,8 +185,7 @@ docker-build-vllm:
 	@$(MAKE) docker-build \
 		DOCKER_TARGET=final-vllm \
 		DOCKER_IMAGE=$(DOCKER_IMAGE_VLLM) \
-		LLAMA_SERVER_VARIANT=cuda \
-		BASE_IMAGE=$(VLLM_BASE_IMAGE)
+		LLAMA_SERVER_VARIANT=cuda
 
 # Run vLLM Docker container with TCP port access and mounted model storage
 docker-run-vllm: docker-build-vllm
@@ -181,8 +196,7 @@ docker-build-sglang:
 	@$(MAKE) docker-build \
 		DOCKER_TARGET=final-sglang \
 		DOCKER_IMAGE=$(DOCKER_IMAGE_SGLANG) \
-		LLAMA_SERVER_VARIANT=cuda \
-		BASE_IMAGE=$(VLLM_BASE_IMAGE)
+		LLAMA_SERVER_VARIANT=cuda
 
 # Run SGLang Docker container with TCP port access and mounted model storage
 docker-run-sglang: docker-build-sglang
@@ -379,6 +393,9 @@ help:
 	@echo ""
 	@echo "Backend configuration options:"
 	@echo "  LLAMA_ARGS    - Arguments for llama.cpp (e.g., \"--verbose --jinja -ngl 999 --ctx-size 2048\")"
+	@echo "  LLAMA_SERVER_VERSION - Upstream llama.cpp version (latest or bNNNN)"
+	@echo "  LLAMA_SERVER_VARIANT - Linux backend flavor (cpu, cuda, or rocm)"
+	@echo "  LLAMA_UPSTREAM_IMAGE - Override the resolved upstream image directly"
 	@echo "  LOCAL_LLAMA   - Use local llama.cpp build from llamacpp/install/bin (set to 1 to enable)"
 	@echo ""
 	@echo "Example usage:"