Skip to content

Commit 23d40b5

Browse files
doringemandoringeman
authored
Merge pull request #737 from doringeman/llamacpp-on-demand
feat: make llama.cpp a deferred backend on macOS/Windows
2 parents d62a635 + 4e6fb00 commit 23d40b5

20 files changed

Lines changed: 367 additions & 238 deletions

‎cmd/cli/commands/install-runner.go‎

Lines changed: 53 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -231,18 +231,19 @@ func getStandaloneRunner(ctx context.Context) (*standaloneRunner, error) {
231231

232232
// runnerOptions holds common configuration for install/start/reinstall commands
233233
type runnerOptions struct {
234-
port uint16
235-
host string
236-
gpuMode string
237-
backend string
238-
doNotTrack bool
239-
pullImage bool
240-
pruneContainers bool
241-
proxyCert string
242-
tls bool
243-
tlsPort uint16
244-
tlsCert string
245-
tlsKey string
234+
port uint16
235+
host string
236+
gpuMode string
237+
backend string
238+
llamaServerVersion string
239+
doNotTrack bool
240+
pullImage bool
241+
pruneContainers bool
242+
proxyCert string
243+
tls bool
244+
tlsPort uint16
245+
tlsCert string
246+
tlsKey string
246247
}
247248

248249
func commandFlagChanged(cmd *cobra.Command, name string) bool {
@@ -284,13 +285,25 @@ func runInstallOrStart(cmd *cobra.Command, opts runnerOptions, debug bool) error
284285
// (on-demand via the running model runner), not as a standalone container.
285286
if opts.backend == vllm.Name && platform.SupportsVLLMMetal() {
286287
cmd.Println("Installing vllm backend...")
287-
if err := desktopClient.InstallBackend(vllm.Name); err != nil {
288+
if err := desktopClient.InstallBackend(vllm.Name, ""); err != nil {
288289
return fmt.Errorf("failed to install vllm backend: %w", err)
289290
}
290291
cmd.Println("vllm backend installed successfully")
291292
return nil
292293
}
293294

295+
// On macOS/Windows the llama.cpp backend is installed on demand: its binary
296+
// is downloaded rather than bundled at build time. Trigger installation via
297+
// the running model runner, the same way vllm-metal is handled above.
298+
if opts.backend == llamacpp.Name && llamacpp.NeedsDeferredInstall() {
299+
cmd.Println("Installing llama.cpp backend...")
300+
if err := desktopClient.InstallBackend(llamacpp.Name, opts.llamaServerVersion); err != nil {
301+
return fmt.Errorf("failed to install llama.cpp backend: %w", err)
302+
}
303+
cmd.Println("llama.cpp backend installed successfully")
304+
return nil
305+
}
306+
294307
// The diffusers backend uses deferred installation: it pulls a Docker
295308
// image, extracts a self-contained Python environment, and installs it
296309
// to a well-known local folder. Trigger installation via the running
@@ -306,7 +319,7 @@ func runInstallOrStart(cmd *cobra.Command, opts runnerOptions, debug bool) error
306319
}
307320

308321
cmd.Println("Installing diffusers backend...")
309-
if err := desktopClient.InstallBackend(diffusers.Name); err != nil {
322+
if err := desktopClient.InstallBackend(diffusers.Name, ""); err != nil {
310323
return fmt.Errorf("failed to install diffusers backend: %w", err)
311324
}
312325
cmd.Println("diffusers backend installed successfully")
@@ -457,6 +470,7 @@ func newInstallRunner() *cobra.Command {
457470
var host string
458471
var gpuMode string
459472
var backend string
473+
var llamaServerVersion string
460474
var doNotTrack bool
461475
var debug bool
462476
var proxyCert string
@@ -469,34 +483,36 @@ func newInstallRunner() *cobra.Command {
469483
Short: "Install Docker Model Runner (Docker Engine only)",
470484
RunE: func(cmd *cobra.Command, args []string) error {
471485
return runInstallOrStart(cmd, runnerOptions{
472-
port: port,
473-
host: host,
474-
gpuMode: gpuMode,
475-
backend: backend,
476-
doNotTrack: doNotTrack,
477-
pullImage: true,
478-
pruneContainers: false,
479-
proxyCert: proxyCert,
480-
tls: tlsEnabled,
481-
tlsPort: tlsPort,
482-
tlsCert: tlsCert,
483-
tlsKey: tlsKey,
486+
port: port,
487+
host: host,
488+
gpuMode: gpuMode,
489+
backend: backend,
490+
llamaServerVersion: llamaServerVersion,
491+
doNotTrack: doNotTrack,
492+
pullImage: true,
493+
pruneContainers: false,
494+
proxyCert: proxyCert,
495+
tls: tlsEnabled,
496+
tlsPort: tlsPort,
497+
tlsCert: tlsCert,
498+
tlsKey: tlsKey,
484499
}, debug)
485500
},
486501
ValidArgsFunction: completion.NoComplete,
487502
}
488503
addRunnerFlags(c, runnerFlagOptions{
489-
Port: &port,
490-
Host: &host,
491-
GpuMode: &gpuMode,
492-
Backend: &backend,
493-
DoNotTrack: &doNotTrack,
494-
Debug: &debug,
495-
ProxyCert: &proxyCert,
496-
TLS: &tlsEnabled,
497-
TLSPort: &tlsPort,
498-
TLSCert: &tlsCert,
499-
TLSKey: &tlsKey,
504+
Port: &port,
505+
Host: &host,
506+
GpuMode: &gpuMode,
507+
Backend: &backend,
508+
LlamaServerVersion: &llamaServerVersion,
509+
DoNotTrack: &doNotTrack,
510+
Debug: &debug,
511+
ProxyCert: &proxyCert,
512+
TLS: &tlsEnabled,
513+
TLSPort: &tlsPort,
514+
TLSCert: &tlsCert,
515+
TLSKey: &tlsKey,
500516
})
501517
return c
502518
}

‎cmd/cli/commands/reinstall-runner.go‎

Lines changed: 26 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ func newReinstallRunner() *cobra.Command {
1010
var host string
1111
var gpuMode string
1212
var backend string
13+
var llamaServerVersion string
1314
var doNotTrack bool
1415
var debug bool
1516
var proxyCert string
@@ -22,34 +23,36 @@ func newReinstallRunner() *cobra.Command {
2223
Short: "Reinstall Docker Model Runner (Docker Engine only)",
2324
RunE: func(cmd *cobra.Command, args []string) error {
2425
return runInstallOrStart(cmd, runnerOptions{
25-
port: port,
26-
host: host,
27-
gpuMode: gpuMode,
28-
backend: backend,
29-
doNotTrack: doNotTrack,
30-
pullImage: true,
31-
pruneContainers: true,
32-
proxyCert: proxyCert,
33-
tls: tlsEnabled,
34-
tlsPort: tlsPort,
35-
tlsCert: tlsCert,
36-
tlsKey: tlsKey,
26+
port: port,
27+
host: host,
28+
gpuMode: gpuMode,
29+
backend: backend,
30+
llamaServerVersion: llamaServerVersion,
31+
doNotTrack: doNotTrack,
32+
pullImage: true,
33+
pruneContainers: true,
34+
proxyCert: proxyCert,
35+
tls: tlsEnabled,
36+
tlsPort: tlsPort,
37+
tlsCert: tlsCert,
38+
tlsKey: tlsKey,
3739
}, debug)
3840
},
3941
ValidArgsFunction: completion.NoComplete,
4042
}
4143
addRunnerFlags(c, runnerFlagOptions{
42-
Port: &port,
43-
Host: &host,
44-
GpuMode: &gpuMode,
45-
Backend: &backend,
46-
DoNotTrack: &doNotTrack,
47-
Debug: &debug,
48-
ProxyCert: &proxyCert,
49-
TLS: &tlsEnabled,
50-
TLSPort: &tlsPort,
51-
TLSCert: &tlsCert,
52-
TLSKey: &tlsKey,
44+
Port: &port,
45+
Host: &host,
46+
GpuMode: &gpuMode,
47+
Backend: &backend,
48+
LlamaServerVersion: &llamaServerVersion,
49+
DoNotTrack: &doNotTrack,
50+
Debug: &debug,
51+
ProxyCert: &proxyCert,
52+
TLS: &tlsEnabled,
53+
TLSPort: &tlsPort,
54+
TLSCert: &tlsCert,
55+
TLSKey: &tlsKey,
5356
})
5457
return c
5558
}

‎cmd/cli/commands/utils.go‎

Lines changed: 16 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -197,17 +197,18 @@ func requireMinArgs(n int, cmdName string, usageArgs string) cobra.PositionalArg
197197

198198
// runnerFlagOptions holds common runner configuration options
199199
type runnerFlagOptions struct {
200-
Port *uint16
201-
Host *string
202-
GpuMode *string
203-
Backend *string
204-
DoNotTrack *bool
205-
Debug *bool
206-
ProxyCert *string
207-
TLS *bool
208-
TLSPort *uint16
209-
TLSCert *string
210-
TLSKey *string
200+
Port *uint16
201+
Host *string
202+
GpuMode *string
203+
Backend *string
204+
LlamaServerVersion *string
205+
DoNotTrack *bool
206+
Debug *bool
207+
ProxyCert *string
208+
TLS *bool
209+
TLSPort *uint16
210+
TLSCert *string
211+
TLSKey *string
211212
}
212213

213214
// addRunnerFlags adds common runner flags to a command
@@ -225,6 +226,10 @@ func addRunnerFlags(cmd *cobra.Command, opts runnerFlagOptions) {
225226
if opts.Backend != nil {
226227
cmd.Flags().StringVar(opts.Backend, "backend", "", backendUsage)
227228
}
229+
if opts.LlamaServerVersion != nil {
230+
cmd.Flags().StringVar(opts.LlamaServerVersion, "llama-server-version", "",
231+
`Override the llama.cpp version to install on macOS/Windows (e.g. "latest" or "v0.0.34"); defaults to the version pinned to this release`)
232+
}
228233
if opts.DoNotTrack != nil {
229234
cmd.Flags().BoolVar(opts.DoNotTrack, "do-not-track", false, "Do not track models usage in Docker Model Runner")
230235
}

‎cmd/cli/desktop/desktop.go‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1076,11 +1076,12 @@ func (c *Client) ShowConfigs(modelFilter string) ([]scheduling.ModelConfigEntry,
10761076
}
10771077

10781078
// InstallBackend triggers on-demand installation of a deferred backend
1079-
func (c *Client) InstallBackend(backend string) error {
1079+
func (c *Client) InstallBackend(backend, version string) error {
10801080
installPath := inference.InferencePrefix + "/install-backend"
10811081
jsonData, err := json.Marshal(struct {
10821082
Backend string `json:"backend"`
1083-
}{Backend: backend})
1083+
Version string `json:"version,omitempty"`
1084+
}{Backend: backend, Version: version})
10841085
if err != nil {
10851086
return fmt.Errorf("error marshaling request: %w", err)
10861087
}

‎cmd/cli/docs/reference/docker_model_install-runner.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,16 @@ options:
5555
experimentalcli: false
5656
kubernetes: false
5757
swarm: false
58+
- option: llama-server-version
59+
value_type: string
60+
description: |
61+
Override the llama.cpp version to install on macOS/Windows (e.g. "latest" or "v0.0.34"); defaults to the version pinned to this release
62+
deprecated: false
63+
hidden: false
64+
experimental: false
65+
experimentalcli: false
66+
kubernetes: false
67+
swarm: false
5868
- option: port
5969
value_type: uint16
6070
default_value: "0"

‎cmd/cli/docs/reference/docker_model_reinstall-runner.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,16 @@ options:
5555
experimentalcli: false
5656
kubernetes: false
5757
swarm: false
58+
- option: llama-server-version
59+
value_type: string
60+
description: |
61+
Override the llama.cpp version to install on macOS/Windows (e.g. "latest" or "v0.0.34"); defaults to the version pinned to this release
62+
deprecated: false
63+
hidden: false
64+
experimental: false
65+
experimentalcli: false
66+
kubernetes: false
67+
swarm: false
5868
- option: port
5969
value_type: uint16
6070
default_value: "0"

‎cmd/cli/docs/reference/model_install-runner.md‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5,19 +5,20 @@ Install Docker Model Runner (Docker Engine only)
55

66
### Options
77

8-
| Name | Type | Default | Description |
9-
|:-----------------|:---------|:------------|:-------------------------------------------------------------------------------------------------------|
10-
| `--backend` | `string` | | Specify backend (llama.cpp\|vllm\|diffusers). Default: llama.cpp |
11-
| `--debug` | `bool` | | Enable debug logging |
12-
| `--do-not-track` | `bool` | | Do not track models usage in Docker Model Runner |
13-
| `--gpu` | `string` | `auto` | Specify GPU support (none\|auto\|cuda\|rocm\|musa\|cann) |
14-
| `--host` | `string` | `127.0.0.1` | Host address to bind Docker Model Runner |
15-
| `--port` | `uint16` | `0` | Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode) |
16-
| `--proxy-cert` | `string` | | Path to a CA certificate file for proxy SSL inspection |
17-
| `--tls` | `bool` | | Enable TLS/HTTPS for Docker Model Runner API |
18-
| `--tls-cert` | `string` | | Path to TLS certificate file (auto-generated if not provided) |
19-
| `--tls-key` | `string` | | Path to TLS private key file (auto-generated if not provided) |
20-
| `--tls-port` | `uint16` | `0` | TLS port for Docker Model Runner (default: 12444 for Docker Engine, 12445 for Cloud mode) |
8+
| Name | Type | Default | Description |
9+
|:-------------------------|:---------|:------------|:----------------------------------------------------------------------------------------------------------------------------------------|
10+
| `--backend` | `string` | | Specify backend (llama.cpp\|vllm\|diffusers). Default: llama.cpp |
11+
| `--debug` | `bool` | | Enable debug logging |
12+
| `--do-not-track` | `bool` | | Do not track models usage in Docker Model Runner |
13+
| `--gpu` | `string` | `auto` | Specify GPU support (none\|auto\|cuda\|rocm\|musa\|cann) |
14+
| `--host` | `string` | `127.0.0.1` | Host address to bind Docker Model Runner |
15+
| `--llama-server-version` | `string` | | Override the llama.cpp version to install on macOS/Windows (e.g. "latest" or "v0.0.34"); defaults to the version pinned to this release |
16+
| `--port` | `uint16` | `0` | Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode) |
17+
| `--proxy-cert` | `string` | | Path to a CA certificate file for proxy SSL inspection |
18+
| `--tls` | `bool` | | Enable TLS/HTTPS for Docker Model Runner API |
19+
| `--tls-cert` | `string` | | Path to TLS certificate file (auto-generated if not provided) |
20+
| `--tls-key` | `string` | | Path to TLS private key file (auto-generated if not provided) |
21+
| `--tls-port` | `uint16` | `0` | TLS port for Docker Model Runner (default: 12444 for Docker Engine, 12445 for Cloud mode) |
2122

2223

2324
<!---MARKER_GEN_END-->

‎cmd/cli/docs/reference/model_reinstall-runner.md‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5,19 +5,20 @@ Reinstall Docker Model Runner (Docker Engine only)
55

66
### Options
77

8-
| Name | Type | Default | Description |
9-
|:-----------------|:---------|:------------|:-------------------------------------------------------------------------------------------------------|
10-
| `--backend` | `string` | | Specify backend (llama.cpp\|vllm\|diffusers). Default: llama.cpp |
11-
| `--debug` | `bool` | | Enable debug logging |
12-
| `--do-not-track` | `bool` | | Do not track models usage in Docker Model Runner |
13-
| `--gpu` | `string` | `auto` | Specify GPU support (none\|auto\|cuda\|rocm\|musa\|cann) |
14-
| `--host` | `string` | `127.0.0.1` | Host address to bind Docker Model Runner |
15-
| `--port` | `uint16` | `0` | Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode) |
16-
| `--proxy-cert` | `string` | | Path to a CA certificate file for proxy SSL inspection |
17-
| `--tls` | `bool` | | Enable TLS/HTTPS for Docker Model Runner API |
18-
| `--tls-cert` | `string` | | Path to TLS certificate file (auto-generated if not provided) |
19-
| `--tls-key` | `string` | | Path to TLS private key file (auto-generated if not provided) |
20-
| `--tls-port` | `uint16` | `0` | TLS port for Docker Model Runner (default: 12444 for Docker Engine, 12445 for Cloud mode) |
8+
| Name | Type | Default | Description |
9+
|:-------------------------|:---------|:------------|:----------------------------------------------------------------------------------------------------------------------------------------|
10+
| `--backend` | `string` | | Specify backend (llama.cpp\|vllm\|diffusers). Default: llama.cpp |
11+
| `--debug` | `bool` | | Enable debug logging |
12+
| `--do-not-track` | `bool` | | Do not track models usage in Docker Model Runner |
13+
| `--gpu` | `string` | `auto` | Specify GPU support (none\|auto\|cuda\|rocm\|musa\|cann) |
14+
| `--host` | `string` | `127.0.0.1` | Host address to bind Docker Model Runner |
15+
| `--llama-server-version` | `string` | | Override the llama.cpp version to install on macOS/Windows (e.g. "latest" or "v0.0.34"); defaults to the version pinned to this release |
16+
| `--port` | `uint16` | `0` | Docker container port for Docker Model Runner (default: 12434 for Docker Engine, 12435 for Cloud mode) |
17+
| `--proxy-cert` | `string` | | Path to a CA certificate file for proxy SSL inspection |
18+
| `--tls` | `bool` | | Enable TLS/HTTPS for Docker Model Runner API |
19+
| `--tls-cert` | `string` | | Path to TLS certificate file (auto-generated if not provided) |
20+
| `--tls-key` | `string` | | Path to TLS private key file (auto-generated if not provided) |
21+
| `--tls-port` | `uint16` | `0` | TLS port for Docker Model Runner (default: 12444 for Docker Engine, 12445 for Cloud mode) |
2122

2223

2324
<!---MARKER_GEN_END-->

0 commit comments

Comments
 (0)