Skip to content

Commit 4e6fb00

Browse files
doringemandoringeman
committed
feat: pin llama.cpp version per release, allow pulling latest on demand
By default the deferred macOS/Windows download now fetches a specific llama.cpp version pinned to this model-runner build instead of the mutable `latest` tag, so a given release always installs the version it was tested against and doesn't silently move when Docker Hub publishes a new build. - Add a pinnedServerVersion constant (a docker-model-backend-llamacpp tag) used as the default DesiredServerVersion; bump it together with the Linux bundle (Dockerfile LLAMA_SERVER_VERSION) when llama.cpp is upgraded. - Allow opting into another version on demand, via LLAMA_SERVER_VERSION or a new `--llama-server-version` flag on install-runner/reinstall-runner (e.g. `--llama-server-version latest`). The flag threads through the InstallBackend API and a new inference.BackendVersionSelector interface, keeping the scheduler decoupled from the concrete backend. - Skip the registry digest round-trip when a pinned (immutable) version is already installed, recording {tag,digest} in .llamacpp_version. This makes startup work offline for pinned versions and removes the per-launch Docker Hub call; only `latest` is always re-resolved. Also report the on-disk binary as running when updates are disabled. Signed-off-by: Dorin Geman <dorin.geman@docker.com>
1 parent 7469d77 commit 4e6fb00

13 files changed

Lines changed: 259 additions & 123 deletions

File tree

‎cmd/cli/commands/install-runner.go‎

Lines changed: 42 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -231,18 +231,19 @@ func getStandaloneRunner(ctx context.Context) (*standaloneRunner, error) {
231231

232232
// runnerOptions holds common configuration for install/start/reinstall commands
233233
type runnerOptions struct {
234-
port uint16
235-
host string
236-
gpuMode string
237-
backend string
238-
doNotTrack bool
239-
pullImage bool
240-
pruneContainers bool
241-
proxyCert string
242-
tls bool
243-
tlsPort uint16
244-
tlsCert string
245-
tlsKey string
234+
port uint16
235+
host string
236+
gpuMode string
237+
backend string
238+
llamaServerVersion string
239+
doNotTrack bool
240+
pullImage bool
241+
pruneContainers bool
242+
proxyCert string
243+
tls bool
244+
tlsPort uint16
245+
tlsCert string
246+
tlsKey string
246247
}
247248

248249
func commandFlagChanged(cmd *cobra.Command, name string) bool {
@@ -284,7 +285,7 @@ func runInstallOrStart(cmd *cobra.Command, opts runnerOptions, debug bool) error
284285
// (on-demand via the running model runner), not as a standalone container.
285286
if opts.backend == vllm.Name && platform.SupportsVLLMMetal() {
286287
cmd.Println("Installing vllm backend...")
287-
if err := desktopClient.InstallBackend(vllm.Name); err != nil {
288+
if err := desktopClient.InstallBackend(vllm.Name, ""); err != nil {
288289
return fmt.Errorf("failed to install vllm backend: %w", err)
289290
}
290291
cmd.Println("vllm backend installed successfully")
@@ -296,7 +297,7 @@ func runInstallOrStart(cmd *cobra.Command, opts runnerOptions, debug bool) error
296297
// the running model runner, the same way vllm-metal is handled above.
297298
if opts.backend == llamacpp.Name && llamacpp.NeedsDeferredInstall() {
298299
cmd.Println("Installing llama.cpp backend...")
299-
if err := desktopClient.InstallBackend(llamacpp.Name); err != nil {
300+
if err := desktopClient.InstallBackend(llamacpp.Name, opts.llamaServerVersion); err != nil {
300301
return fmt.Errorf("failed to install llama.cpp backend: %w", err)
301302
}
302303
cmd.Println("llama.cpp backend installed successfully")
@@ -318,7 +319,7 @@ func runInstallOrStart(cmd *cobra.Command, opts runnerOptions, debug bool) error
318319
}
319320

320321
cmd.Println("Installing diffusers backend...")
321-
if err := desktopClient.InstallBackend(diffusers.Name); err != nil {
322+
if err := desktopClient.InstallBackend(diffusers.Name, ""); err != nil {
322323
return fmt.Errorf("failed to install diffusers backend: %w", err)
323324
}
324325
cmd.Println("diffusers backend installed successfully")
@@ -469,6 +470,7 @@ func newInstallRunner() *cobra.Command {
469470
var host string
470471
var gpuMode string
471472
var backend string
473+
var llamaServerVersion string
472474
var doNotTrack bool
473475
var debug bool
474476
var proxyCert string
@@ -481,34 +483,36 @@ func newInstallRunner() *cobra.Command {
481483
Short: "Install Docker Model Runner (Docker Engine only)",
482484
RunE: func(cmd *cobra.Command, args []string) error {
483485
return runInstallOrStart(cmd, runnerOptions{
484-
port: port,
485-
host: host,
486-
gpuMode: gpuMode,
487-
backend: backend,
488-
doNotTrack: doNotTrack,
489-
pullImage: true,
490-
pruneContainers: false,
491-
proxyCert: proxyCert,
492-
tls: tlsEnabled,
493-
tlsPort: tlsPort,
494-
tlsCert: tlsCert,
495-
tlsKey: tlsKey,
486+
port: port,
487+
host: host,
488+
gpuMode: gpuMode,
489+
backend: backend,
490+
llamaServerVersion: llamaServerVersion,
491+
doNotTrack: doNotTrack,
492+
pullImage: true,
493+
pruneContainers: false,
494+
proxyCert: proxyCert,
495+
tls: tlsEnabled,
496+
tlsPort: tlsPort,
497+
tlsCert: tlsCert,
498+
tlsKey: tlsKey,
496499
}, debug)
497500
},
498501
ValidArgsFunction: completion.NoComplete,
499502
}
500503
addRunnerFlags(c, runnerFlagOptions{
501-
Port: &port,
502-
Host: &host,
503-
GpuMode: &gpuMode,
504-
Backend: &backend,
505-
DoNotTrack: &doNotTrack,
506-
Debug: &debug,
507-
ProxyCert: &proxyCert,
508-
TLS: &tlsEnabled,
509-
TLSPort: &tlsPort,
510-
TLSCert: &tlsCert,
511-
TLSKey: &tlsKey,
504+
Port: &port,
505+
Host: &host,
506+
GpuMode: &gpuMode,
507+
Backend: &backend,
508+
LlamaServerVersion: &llamaServerVersion,
509+
DoNotTrack: &doNotTrack,
510+
Debug: &debug,
511+
ProxyCert: &proxyCert,
512+
TLS: &tlsEnabled,
513+
TLSPort: &tlsPort,
514+
TLSCert: &tlsCert,
515+
TLSKey: &tlsKey,
512516
})
513517
return c
514518
}

‎cmd/cli/commands/reinstall-runner.go‎

Lines changed: 26 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ func newReinstallRunner() *cobra.Command {
1010
var host string
1111
var gpuMode string
1212
var backend string
13+
var llamaServerVersion string
1314
var doNotTrack bool
1415
var debug bool
1516
var proxyCert string
@@ -22,34 +23,36 @@ func newReinstallRunner() *cobra.Command {
2223
Short: "Reinstall Docker Model Runner (Docker Engine only)",
2324
RunE: func(cmd *cobra.Command, args []string) error {
2425
return runInstallOrStart(cmd, runnerOptions{
25-
port: port,
26-
host: host,
27-
gpuMode: gpuMode,
28-
backend: backend,
29-
doNotTrack: doNotTrack,
30-
pullImage: true,
31-
pruneContainers: true,
32-
proxyCert: proxyCert,
33-
tls: tlsEnabled,
34-
tlsPort: tlsPort,
35-
tlsCert: tlsCert,
36-
tlsKey: tlsKey,
26+
port: port,
27+
host: host,
28+
gpuMode: gpuMode,
29+
backend: backend,
30+
llamaServerVersion: llamaServerVersion,
31+
doNotTrack: doNotTrack,
32+
pullImage: true,
33+
pruneContainers: true,
34+
proxyCert: proxyCert,
35+
tls: tlsEnabled,
36+
tlsPort: tlsPort,
37+
tlsCert: tlsCert,
38+
tlsKey: tlsKey,
3739
}, debug)
3840
},
3941
ValidArgsFunction: completion.NoComplete,
4042
}
4143
addRunnerFlags(c, runnerFlagOptions{
42-
Port: &port,
43-
Host: &host,
44-
GpuMode: &gpuMode,
45-
Backend: &backend,
46-
DoNotTrack: &doNotTrack,
47-
Debug: &debug,
48-
ProxyCert: &proxyCert,
49-
TLS: &tlsEnabled,
50-
TLSPort: &tlsPort,
51-
TLSCert: &tlsCert,
52-
TLSKey: &tlsKey,
44+
Port: &port,
45+
Host: &host,
46+
GpuMode: &gpuMode,
47+
Backend: &backend,
48+
LlamaServerVersion: &llamaServerVersion,
49+
DoNotTrack: &doNotTrack,
50+
Debug: &debug,
51+
ProxyCert: &proxyCert,
52+
TLS: &tlsEnabled,
53+
TLSPort: &tlsPort,
54+
TLSCert: &tlsCert,
55+
TLSKey: &tlsKey,
5356
})
5457
return c
5558
}

‎cmd/cli/commands/utils.go‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -197,17 +197,18 @@ func requireMinArgs(n int, cmdName string, usageArgs string) cobra.PositionalArg
197197

198198
// runnerFlagOptions holds common runner configuration options
199199
type runnerFlagOptions struct {
200-
Port *uint16
201-
Host *string
202-
GpuMode *string
203-
Backend *string
204-
DoNotTrack *bool
205-
Debug *bool
206-
ProxyCert *string
207-
TLS *bool
208-
TLSPort *uint16
209-
TLSCert *string
210-
TLSKey *string
200+
Port *uint16
201+
Host *string
202+
GpuMode *string
203+
Backend *string
204+
LlamaServerVersion *string
205+
DoNotTrack *bool
206+
Debug *bool
207+
ProxyCert *string
208+
TLS *bool
209+
TLSPort *uint16
210+
TLSCert *string
211+
TLSKey *string
211212
}
212213

213214
// addRunnerFlags adds common runner flags to a command
@@ -225,6 +226,10 @@ func addRunnerFlags(cmd *cobra.Command, opts runnerFlagOptions) {
225226
if opts.Backend != nil {
226227
cmd.Flags().StringVar(opts.Backend, "backend", "", backendUsage)
227228
}
229+
if opts.LlamaServerVersion != nil {
230+
cmd.Flags().StringVar(opts.LlamaServerVersion, "llama-server-version", "",
231+
`Override the llama.cpp version to install on macOS/Windows (e.g. "latest" or "v0.0.34"); defaults to the version pinned to this release`)
232+
}
228233
if opts.DoNotTrack != nil {
229234
cmd.Flags().BoolVar(opts.DoNotTrack, "do-not-track", false, "Do not track models usage in Docker Model Runner")
230235
}

‎cmd/cli/desktop/desktop.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1076,11 +1076,12 @@ func (c *Client) ShowConfigs(modelFilter string) ([]scheduling.ModelConfigEntry,
10761076
}
10771077

10781078
// InstallBackend triggers on-demand installation of a deferred backend
1079-
func (c *Client) InstallBackend(backend string) error {
1079+
func (c *Client) InstallBackend(backend, version string) error {
10801080
installPath := inference.InferencePrefix + "/install-backend"
10811081
jsonData, err := json.Marshal(struct {
10821082
Backend string `json:"backend"`
1083-
}{Backend: backend})
1083+
Version string `json:"version,omitempty"`
1084+
}{Backend: backend, Version: version})
10841085
if err != nil {
10851086
return fmt.Errorf("error marshaling request: %w", err)
10861087
}

‎cmd/cli/docs/reference/docker_model_install-runner.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,16 @@ options:
5555
experimentalcli: false
5656
kubernetes: false
5757
swarm: false
58+
- option: llama-server-version
59+
value_type: string
60+
description: |
61+
Override the llama.cpp version to install on macOS/Windows (e.g. "latest" or "v0.0.34"); defaults to the version pinned to this release
62+
deprecated: false
63+
hidden: false
64+
experimental: false
65+
experimentalcli: false
66+
kubernetes: false
67+
swarm: false
5868
- option: port
5969
value_type: uint16
6070
default_value: "0"

‎cmd/cli/docs/reference/docker_model_reinstall-runner.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,16 @@ options:
5555
experimentalcli: false
5656
kubernetes: false
5757
swarm: false
58+
- option: llama-server-version
59+
value_type: string
60+
description: |
61+
Override the llama.cpp version to install on macOS/Windows (e.g. "latest" or "v0.0.34"); defaults to the version pinned to this release
62+
deprecated: false
63+
hidden: false
64+
experimental: false
65+
experimentalcli: false
66+
kubernetes: false
67+
swarm: false
5868
- option: port
5969
value_type: uint16
6070
default_value: "0"

‎cmd/cli/docs/reference/model_install-runner.md‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5,19 +5,20 @@ Install Docker Model Runner (Docker Engine only)
55

66
### Options
77

8-
| Name | Type | Default | Description |
9-
|:-----------------|:---------|:------------|:-------------------------------------------------------------------------------------------------------|
10-
| `--backend` | `string` | | Specify backend (llama.cpp\|vllm\|diffusers). Default: llama.cpp |
11-
| `--debug` | `bool` | | Enable debug logging |
12-
| `--do-not-track` | `bool` | | Do not track models usage in Docker Model Runner |
13-
| `--gpu` | `string` | `auto` | Specify GPU support (none\|auto\|cuda\|rocm\|musa\|cann) |
14-
| `--host` | `string` | `127.0.0.1` | Host address to bind Docker Model Runner |
15-
| `--port` | `uint16` | `0` | Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode) |
16-
| `--proxy-cert` | `string` | | Path to a CA certificate file for proxy SSL inspection |
17-
| `--tls` | `bool` | | Enable TLS/HTTPS for Docker Model Runner API |
18-
| `--tls-cert` | `string` | | Path to TLS certificate file (auto-generated if not provided) |
19-
| `--tls-key` | `string` | | Path to TLS private key file (auto-generated if not provided) |
20-
| `--tls-port` | `uint16` | `0` | TLS port for Docker Model Runner (default: 12444 for Docker Engine, 12445 for Cloud mode) |
8+
| Name | Type | Default | Description |
9+
|:-------------------------|:---------|:------------|:----------------------------------------------------------------------------------------------------------------------------------------|
10+
| `--backend` | `string` | | Specify backend (llama.cpp\|vllm\|diffusers). Default: llama.cpp |
11+
| `--debug` | `bool` | | Enable debug logging |
12+
| `--do-not-track` | `bool` | | Do not track models usage in Docker Model Runner |
13+
| `--gpu` | `string` | `auto` | Specify GPU support (none\|auto\|cuda\|rocm\|musa\|cann) |
14+
| `--host` | `string` | `127.0.0.1` | Host address to bind Docker Model Runner |
15+
| `--llama-server-version` | `string` | | Override the llama.cpp version to install on macOS/Windows (e.g. "latest" or "v0.0.34"); defaults to the version pinned to this release |
16+
| `--port` | `uint16` | `0` | Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode) |
17+
| `--proxy-cert` | `string` | | Path to a CA certificate file for proxy SSL inspection |
18+
| `--tls` | `bool` | | Enable TLS/HTTPS for Docker Model Runner API |
19+
| `--tls-cert` | `string` | | Path to TLS certificate file (auto-generated if not provided) |
20+
| `--tls-key` | `string` | | Path to TLS private key file (auto-generated if not provided) |
21+
| `--tls-port` | `uint16` | `0` | TLS port for Docker Model Runner (default: 12444 for Docker Engine, 12445 for Cloud mode) |
2122

2223

2324
<!---MARKER_GEN_END-->

‎cmd/cli/docs/reference/model_reinstall-runner.md‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5,19 +5,20 @@ Reinstall Docker Model Runner (Docker Engine only)
55

66
### Options
77

8-
| Name | Type | Default | Description |
9-
|:-----------------|:---------|:------------|:-------------------------------------------------------------------------------------------------------|
10-
| `--backend` | `string` | | Specify backend (llama.cpp\|vllm\|diffusers). Default: llama.cpp |
11-
| `--debug` | `bool` | | Enable debug logging |
12-
| `--do-not-track` | `bool` | | Do not track models usage in Docker Model Runner |
13-
| `--gpu` | `string` | `auto` | Specify GPU support (none\|auto\|cuda\|rocm\|musa\|cann) |
14-
| `--host` | `string` | `127.0.0.1` | Host address to bind Docker Model Runner |
15-
| `--port` | `uint16` | `0` | Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode) |
16-
| `--proxy-cert` | `string` | | Path to a CA certificate file for proxy SSL inspection |
17-
| `--tls` | `bool` | | Enable TLS/HTTPS for Docker Model Runner API |
18-
| `--tls-cert` | `string` | | Path to TLS certificate file (auto-generated if not provided) |
19-
| `--tls-key` | `string` | | Path to TLS private key file (auto-generated if not provided) |
20-
| `--tls-port` | `uint16` | `0` | TLS port for Docker Model Runner (default: 12444 for Docker Engine, 12445 for Cloud mode) |
8+
| Name | Type | Default | Description |
9+
|:-------------------------|:---------|:------------|:----------------------------------------------------------------------------------------------------------------------------------------|
10+
| `--backend` | `string` | | Specify backend (llama.cpp\|vllm\|diffusers). Default: llama.cpp |
11+
| `--debug` | `bool` | | Enable debug logging |
12+
| `--do-not-track` | `bool` | | Do not track models usage in Docker Model Runner |
13+
| `--gpu` | `string` | `auto` | Specify GPU support (none\|auto\|cuda\|rocm\|musa\|cann) |
14+
| `--host` | `string` | `127.0.0.1` | Host address to bind Docker Model Runner |
15+
| `--llama-server-version` | `string` | | Override the llama.cpp version to install on macOS/Windows (e.g. "latest" or "v0.0.34"); defaults to the version pinned to this release |
16+
| `--port` | `uint16` | `0` | Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode) |
17+
| `--proxy-cert` | `string` | | Path to a CA certificate file for proxy SSL inspection |
18+
| `--tls` | `bool` | | Enable TLS/HTTPS for Docker Model Runner API |
19+
| `--tls-cert` | `string` | | Path to TLS certificate file (auto-generated if not provided) |
20+
| `--tls-key` | `string` | | Path to TLS private key file (auto-generated if not provided) |
21+
| `--tls-port` | `uint16` | `0` | TLS port for Docker Model Runner (default: 12444 for Docker Engine, 12445 for Cloud mode) |
2122

2223

2324
<!---MARKER_GEN_END-->

‎pkg/inference/backend.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -323,3 +323,13 @@ type Backend interface {
323323
// GetDiskUsage returns the disk usage of the backend.
324324
GetDiskUsage() (int64, error)
325325
}
326+
327+
// BackendVersionSelector is an optional interface implemented by backends whose
328+
// install version can be overridden at install time (e.g. to pull a specific
329+
// build or the latest one on demand instead of the version pinned to this
330+
// model-runner release). The override takes effect on the next Install.
331+
type BackendVersionSelector interface {
332+
// SetInstallVersion sets the version the backend should install. An empty
333+
// string leaves the current selection unchanged.
334+
SetInstallVersion(version string)
335+
}

0 commit comments

Comments
 (0)