feat: make llama.cpp a deferred backend on macOS/Windows

On macOS/Windows the llama.cpp server binary is now downloaded on demand
into a single, writable per-user install directory
(~/.docker/model-runner/llama.cpp/bin) instead of relying on a binary
vendored by Docker Desktop. This removes the vendored/updated storage
split and the requirement for a bundled com.docker.llama-server.digest.

- Collapse vendoredServerStoragePath/updatedServerStoragePath into a
  single installDir; default to ~/.docker/model-runner/llama.cpp/bin when
  no path is configured.
- Add llamacpp.NeedsDeferredInstall() (true on macOS/Windows) and mark the
  backend Deferred in the routing defs so it installs on first use instead
  of blocking startup.
- Reuse main's mirror-aware dockerhub download path and the existing
  docker/docker-model-backend-llamacpp {version}-{variant} image tags; the
  download no longer hard-fails when there is no Desktop-bundled binary.
- Linux keeps bundling the binary in the container image (not deferred);
  ensureLatestLlamaCpp remains a no-op that reports the bundled binary.
- LlamaServerPath() no longer defaults to the read-only Docker Desktop
  bundle path; it returns the env override or empty.
- Add a `docker model install-runner --backend llama.cpp` trigger for the
  deferred install, mirroring the vllm-metal path.

Signed-off-by: Dorin Geman <dorin.geman@docker.com>

Author: doringeman

cmd/cli/commands/install-runner.go

@@ -291,6 +291,18 @@ func runInstallOrStart(cmd *cobra.Command, opts runnerOptions, debug bool) error
 		return nil
 	}
 
+	// On macOS/Windows the llama.cpp backend is installed on demand: its binary
+	// is downloaded rather than bundled at build time. Trigger installation via
+	// the running model runner, the same way vllm-metal is handled above.
+	if opts.backend == llamacpp.Name && llamacpp.NeedsDeferredInstall() {
+		cmd.Println("Installing llama.cpp backend...")
+		if err := desktopClient.InstallBackend(llamacpp.Name); err != nil {
+			return fmt.Errorf("failed to install llama.cpp backend: %w", err)
+		}
+		cmd.Println("llama.cpp backend installed successfully")
+		return nil
+	}
+
 	// The diffusers backend uses deferred installation: it pulls a Docker
 	// image, extracts a self-contained Python environment, and installs it
 	// to a well-known local folder. Trigger installation via the running

pkg/envconfig/envconfig.go

@@ -6,7 +6,6 @@ import (
 	"net"
 	"os"
 	"path/filepath"
-	"runtime"
 	"strconv"
 	"strings"
 
@@ -99,20 +98,13 @@ func TCPPort() string {
 	return Var("MODEL_RUNNER_PORT")
 }
 
-// LlamaServerPath returns the path to the llama.cpp server binary.
-// Configured via LLAMA_SERVER_PATH. On macOS this defaults to the Docker
-// Desktop bundle location as a last-resort lookup (Desktop vendors its own
-// copy there); on every other platform there is no meaningful default path
-// to check, and the daemon falls back to downloading the pinned llama.cpp
-// release on demand.
+// LlamaServerPath returns the directory that holds the llama.cpp server binary.
+// Configured via LLAMA_SERVER_PATH. On Linux the container image sets this to
+// the bundled binary's location. When it is unset (macOS/Windows), an empty
+// string is returned and the llama.cpp backend falls back to a writable
+// per-user directory into which it downloads the pinned release on demand.
 func LlamaServerPath() string {
-	if s := Var("LLAMA_SERVER_PATH"); s != "" {
-		return s
-	}
-	if runtime.GOOS == "darwin" {
-		return "/Applications/Docker.app/Contents/Resources/model-runner/bin"
-	}
-	return ""
+	return Var("LLAMA_SERVER_PATH")
 }
 
 // LlamaArgs returns custom arguments to pass to the llama.cpp server.

pkg/envconfig/envconfig_test.go

@@ -1,7 +1,6 @@
 package envconfig_test
 
 import (
-	"runtime"
 	"testing"
 
 	"github.com/docker/model-runner/pkg/envconfig"
@@ -15,16 +14,13 @@ func TestLlamaServerPath(t *testing.T) {
 		}
 	})
 
-	t.Run("default depends on platform", func(t *testing.T) {
+	t.Run("empty when unset", func(t *testing.T) {
 		t.Setenv("LLAMA_SERVER_PATH", "")
-		got := envconfig.LlamaServerPath()
-		if runtime.GOOS == "darwin" {
-			want := "/Applications/Docker.app/Contents/Resources/model-runner/bin"
-			if got != want {
-				t.Fatalf("LlamaServerPath() = %q, want %q", got, want)
-			}
-		} else if got != "" {
-			t.Fatalf("LlamaServerPath() = %q, want empty string on %s", got, runtime.GOOS)
+		// When unset there is no meaningful default here: the llama.cpp backend
+		// falls back to a writable per-user directory (macOS/Windows) or the
+		// image sets LLAMA_SERVER_PATH explicitly (Linux).
+		if got := envconfig.LlamaServerPath(); got != "" {
+			t.Fatalf("LlamaServerPath() = %q, want empty string", got)
 		}
 	})
 }

pkg/inference/backends/llamacpp/download.go

@@ -48,7 +48,7 @@ func SetDesiredServerVersion(version string) {
 
 //nolint:unused // Used in platform-specific files (download_darwin.go, download_windows.go)
 func (l *llamaCpp) downloadLatestLlamaCpp(ctx context.Context, log logging.Logger,
-	llamaCppPath, vendoredServerStoragePath, desiredVersion, desiredVariant string,
+	desiredVersion, desiredVariant string,
 ) error {
 	ShouldUpdateServerLock.Lock()
 	shouldUpdateServer := ShouldUpdateServer
@@ -58,7 +58,10 @@ func (l *llamaCpp) downloadLatestLlamaCpp(ctx context.Context, log logging.Logge
 		return errLlamaCppUpdateDisabled
 	}
 
-	log.Info("downloadLatestLlamaCpp", "desiredVersion", desiredVersion, "desiredVariant", desiredVariant, "vendoredServerStoragePath", vendoredServerStoragePath, "llamaCppPath", llamaCppPath)
+	llamaCppPath := filepath.Join(l.installDir, l.downloadBinaryName())
+	versionFile := filepath.Join(l.installDir, ".llamacpp_version")
+
+	log.Info("downloadLatestLlamaCpp", "desiredVersion", desiredVersion, "desiredVariant", desiredVariant, "installDir", l.installDir)
 	desiredTag := desiredVersion + "-" + desiredVariant
 
 	// Resolve the desired tag to a digest via the Registry HTTP API v2. This
@@ -74,30 +77,20 @@ func (l *llamaCpp) downloadLatestLlamaCpp(ctx context.Context, log logging.Logge
 		return fmt.Errorf("could not resolve the %s tag: %w", desiredTag, err)
 	}
 
-	bundledVersionFile := filepath.Join(vendoredServerStoragePath, "com.docker.llama-server.digest")
-	currentVersionFile := filepath.Join(filepath.Dir(llamaCppPath), ".llamacpp_version")
-
-	data, err := os.ReadFile(bundledVersionFile)
-	if err != nil {
-		return fmt.Errorf("failed to read bundled llama.cpp version: %w", err)
-	} else if strings.TrimSpace(string(data)) == latest {
-		l.setRunningStatus(log, filepath.Join(vendoredServerStoragePath, "com.docker.llama-server"), desiredTag, latest)
-		return errLlamaCppUpToDate
-	}
-
-	data, err = os.ReadFile(currentVersionFile)
-	if err != nil {
-		log.Warn("failed to read current llama.cpp version", "error", err)
-		log.Warn("proceeding to update llama.cpp binary")
-	} else if strings.TrimSpace(string(data)) == latest {
-		log.Info("current llama.cpp version is already up to date")
-		if _, statErr := os.Stat(llamaCppPath); statErr == nil {
-			l.setRunningStatus(log, llamaCppPath, desiredTag, latest)
-			return nil
+	// If we have already downloaded this exact digest and the binary is still
+	// present, there is nothing to do. Unlike the previous Docker Desktop
+	// bundled model, there is no vendored binary to compare against here.
+	if data, readErr := os.ReadFile(versionFile); readErr == nil {
+		if strings.TrimSpace(string(data)) == latest {
+			if _, statErr := os.Stat(llamaCppPath); statErr == nil {
+				log.Info("current llama.cpp version is already up to date")
+				l.setRunningStatus(log, llamaCppPath, desiredTag, latest)
+				return errLlamaCppUpToDate
+			}
+			log.Info("llama.cpp binary missing despite version match, proceeding to download")
+		} else {
+			log.Info("current llama.cpp version is outdated, proceeding to update", "current", strings.TrimSpace(string(data)), "latest", latest)
 		}
-		log.Info("llama.cpp binary must be updated, proceeding to update it")
-	} else {
-		log.Info("current llama.cpp version is outdated, proceeding to update it", "current", strings.TrimSpace(string(data)), "latest", latest)
 	}
 
 	image := fmt.Sprintf("registry-1.docker.io/%s/%s@%s", hubNamespace, hubRepo, latest)
@@ -112,32 +105,33 @@ func (l *llamaCpp) downloadLatestLlamaCpp(ctx context.Context, log logging.Logge
 		return fmt.Errorf("could not extract image: %w", extractErr)
 	}
 
-	if err := os.RemoveAll(filepath.Dir(llamaCppPath)); err != nil && !errors.Is(err, os.ErrNotExist) {
+	libDir := filepath.Join(filepath.Dir(l.installDir), "lib")
+	if err := os.RemoveAll(l.installDir); err != nil && !errors.Is(err, os.ErrNotExist) {
 		return fmt.Errorf("failed to clear inference binary dir: %w", err)
 	}
-	if err := os.RemoveAll(filepath.Join(filepath.Dir(filepath.Dir(llamaCppPath)), "lib")); err != nil && !errors.Is(err, os.ErrNotExist) {
+	if err := os.RemoveAll(libDir); err != nil && !errors.Is(err, os.ErrNotExist) {
 		return fmt.Errorf("failed to clear inference library dir: %w", err)
 	}
 
-	if err := os.MkdirAll(filepath.Dir(filepath.Dir(llamaCppPath)), 0o755); err != nil {
+	if err := os.MkdirAll(filepath.Dir(l.installDir), 0o755); err != nil {
 		return fmt.Errorf("could not create directory for llama.cpp artifacts: %w", err)
 	}
 
 	rootDir := fmt.Sprintf("com.docker.llama-server.native.%s.%s.%s", runtime.GOOS, desiredVariant, runtime.GOARCH)
-	if err := os.Rename(filepath.Join(downloadDir, rootDir, "bin"), filepath.Dir(llamaCppPath)); err != nil {
+	if err := os.Rename(filepath.Join(downloadDir, rootDir, "bin"), l.installDir); err != nil {
 		return fmt.Errorf("could not move llama.cpp binary: %w", err)
 	}
 	if err := os.Chmod(llamaCppPath, 0o755); err != nil {
 		return fmt.Errorf("could not chmod llama.cpp binary: %w", err)
 	}
 
-	libDir := filepath.Join(downloadDir, rootDir, "lib")
-	fi, err := os.Stat(libDir)
+	srcLibDir := filepath.Join(downloadDir, rootDir, "lib")
+	fi, err := os.Stat(srcLibDir)
 	if err != nil && !errors.Is(err, os.ErrNotExist) {
 		return fmt.Errorf("failed to stat llama.cpp lib dir: %w", err)
 	}
 	if err == nil && fi.IsDir() {
-		if err := os.Rename(libDir, filepath.Join(filepath.Dir(filepath.Dir(llamaCppPath)), "lib")); err != nil {
+		if err := os.Rename(srcLibDir, libDir); err != nil {
 			return fmt.Errorf("could not move llama.cpp libs: %w", err)
 		}
 	}
@@ -146,7 +140,7 @@ func (l *llamaCpp) downloadLatestLlamaCpp(ctx context.Context, log logging.Logge
 	l.setRunningStatus(log, llamaCppPath, desiredTag, latest)
 	log.Info(l.status)
 
-	if err := os.WriteFile(currentVersionFile, []byte(latest), 0o644); err != nil {
+	if err := os.WriteFile(versionFile, []byte(latest), 0o644); err != nil {
 		log.Warn("failed to save llama.cpp version", "error", err)
 	}
 

pkg/inference/backends/llamacpp/download_darwin.go

@@ -7,11 +7,8 @@ import (
 	"github.com/docker/model-runner/pkg/logging"
 )
 
-func (l *llamaCpp) ensureLatestLlamaCpp(ctx context.Context, log logging.Logger, _ *http.Client,
-	llamaCppPath, vendoredServerStoragePath string,
-) error {
+func (l *llamaCpp) ensureLatestLlamaCpp(ctx context.Context, log logging.Logger, _ *http.Client) error {
 	desiredVersion := GetDesiredServerVersion()
 	desiredVariant := "metal"
-	return l.downloadLatestLlamaCpp(ctx, log, llamaCppPath, vendoredServerStoragePath, desiredVersion,
-		desiredVariant)
+	return l.downloadLatestLlamaCpp(ctx, log, desiredVersion, desiredVariant)
 }

pkg/inference/backends/llamacpp/download_linux.go

@@ -8,9 +8,9 @@ import (
 	"github.com/docker/model-runner/pkg/logging"
 )
 
-func (l *llamaCpp) ensureLatestLlamaCpp(_ context.Context, log logging.Logger, _ *http.Client,
-	_, vendoredServerStoragePath string,
-) error {
-	l.setRunningStatus(log, filepath.Join(vendoredServerStoragePath, resolveLlamaServerBin(vendoredServerStoragePath)), "", "")
+func (l *llamaCpp) ensureLatestLlamaCpp(_ context.Context, log logging.Logger, _ *http.Client) error {
+	// On Linux the binary is bundled into the container image at installDir;
+	// there is no on-demand download.
+	l.setRunningStatus(log, filepath.Join(l.installDir, resolveLlamaServerBin(l.installDir)), "", "")
 	return errLlamaCppUpdateDisabled
 }

pkg/inference/backends/llamacpp/download_windows.go

@@ -11,10 +11,8 @@ import (
 	"github.com/docker/model-runner/pkg/logging"
 )
 
-func (l *llamaCpp) ensureLatestLlamaCpp(ctx context.Context, log logging.Logger, _ *http.Client,
-	llamaCppPath, vendoredServerStoragePath string,
-) error {
-	nvGPUInfoBin := filepath.Join(vendoredServerStoragePath, "com.docker.nv-gpu-info.exe")
+func (l *llamaCpp) ensureLatestLlamaCpp(ctx context.Context, log logging.Logger, _ *http.Client) error {
+	nvGPUInfoBin := filepath.Join(l.installDir, "com.docker.nv-gpu-info.exe")
 	var canUseCUDA11, canUseOpenCL bool
 	var err error
 	ShouldUseGPUVariantLock.Lock()
@@ -43,6 +41,5 @@ func (l *llamaCpp) ensureLatestLlamaCpp(ctx context.Context, log logging.Logger,
 		desiredVariant = "opencl"
 	}
 	l.status = inference.FormatInstalling(fmt.Sprintf("%s llama.cpp %s", inference.DetailCheckingForUpdates, desiredVariant))
-	return l.downloadLatestLlamaCpp(ctx, log, llamaCppPath, vendoredServerStoragePath, desiredVersion,
-		desiredVariant)
+	return l.downloadLatestLlamaCpp(ctx, log, desiredVersion, desiredVariant)
 }