Merge pull request #325 from doringeman/vllm-status-not-found

doringeman · web-flow · commit 1c872eb015bf · 2025-10-27T16:16:11.000+02:00
feat(vllm): handle binary not found
diff --git a/cmd/cli/commands/df.go b/cmd/cli/commands/df.go
@@ -17,8 +17,7 @@ func newDFCmd() *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			df, err := desktopClient.DF()
 			if err != nil {
-				err = handleClientError(err, "Failed to list running models")
-				return handleNotRunningError(err)
+				return handleClientError(err, "Failed to list running models")
 			}
 			cmd.Print(diskUsageTable(df))
 			return nil
diff --git a/cmd/cli/commands/inspect.go b/cmd/cli/commands/inspect.go
@@ -53,15 +53,13 @@ func inspectModel(args []string, openai bool, remote bool, desktopClient *deskto
 	if openai {
 		model, err := desktopClient.InspectOpenAI(modelName)
 		if err != nil {
-			err = handleClientError(err, "Failed to get model "+modelName)
-			return "", handleNotRunningError(err)
+			return "", handleClientError(err, "Failed to get model "+modelName)
 		}
 		return formatter.ToStandardJSON(model)
 	}
 	model, err := desktopClient.Inspect(modelName, remote)
 	if err != nil {
-		err = handleClientError(err, "Failed to get model "+modelName)
-		return "", handleNotRunningError(err)
+		return "", handleClientError(err, "Failed to get model "+modelName)
 	}
 	return formatter.ToStandardJSON(model)
 }
diff --git a/cmd/cli/commands/list.go b/cmd/cli/commands/list.go
@@ -60,15 +60,13 @@ func listModels(openai bool, desktopClient *desktop.Client, quiet bool, jsonForm
 	if openai {
 		models, err := desktopClient.ListOpenAI()
 		if err != nil {
-			err = handleClientError(err, "Failed to list models")
-			return "", handleNotRunningError(err)
+			return "", handleClientError(err, "Failed to list models")
 		}
 		return formatter.ToStandardJSON(models)
 	}
 	models, err := desktopClient.List()
 	if err != nil {
-		err = handleClientError(err, "Failed to list models")
-		return "", handleNotRunningError(err)
+		return "", handleClientError(err, "Failed to list models")
 	}
 
 	if modelFilter != "" {
diff --git a/cmd/cli/commands/ps.go b/cmd/cli/commands/ps.go
@@ -19,8 +19,7 @@ func newPSCmd() *cobra.Command {
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ps, err := desktopClient.PS()
 			if err != nil {
-				err = handleClientError(err, "Failed to list running models")
-				return handleNotRunningError(err)
+				return handleClientError(err, "Failed to list running models")
 			}
 			cmd.Print(psTable(ps))
 			return nil
diff --git a/cmd/cli/commands/pull.go b/cmd/cli/commands/pull.go
@@ -58,7 +58,7 @@ func pullModel(cmd *cobra.Command, desktopClient *desktop.Client, model string,
 	}
 
 	if err != nil {
-		return handleNotRunningError(handleClientError(err, "Failed to pull model"))
+		return handleClientError(err, "Failed to pull model")
 	}
 
 	cmd.Println(response)
diff --git a/cmd/cli/commands/push.go b/cmd/cli/commands/push.go
@@ -45,7 +45,7 @@ func pushModel(cmd *cobra.Command, desktopClient *desktop.Client, model string)
 	}
 
 	if err != nil {
-		return handleNotRunningError(handleClientError(err, "Failed to push model"))
+		return handleClientError(err, "Failed to push model")
 	}
 
 	cmd.Println(response)
diff --git a/cmd/cli/commands/requests.go b/cmd/cli/commands/requests.go
@@ -35,8 +35,7 @@ func newRequestsCmd() *cobra.Command {
 				if model != "" {
 					errMsg = errMsg + " for " + model
 				}
-				err = handleClientError(err, errMsg)
-				return handleNotRunningError(err)
+				return handleClientError(err, errMsg)
 			}
 			defer cancel()
 
diff --git a/cmd/cli/commands/rm.go b/cmd/cli/commands/rm.go
@@ -38,8 +38,7 @@ func newRemoveCmd() *cobra.Command {
 				cmd.Print(response)
 			}
 			if err != nil {
-				err = handleClientError(err, "Failed to remove model")
-				return handleNotRunningError(err)
+				return handleClientError(err, "Failed to remove model")
 			}
 			return nil
 		},
diff --git a/cmd/cli/commands/run.go b/cmd/cli/commands/run.go
@@ -257,7 +257,7 @@ func generateInteractiveWithReadline(cmd *cobra.Command, desktopClient *desktop.
 				if errors.Is(err, context.Canceled) {
 					cmd.Println()
 				} else {
-					cmd.PrintErr(handleClientError(err, "Failed to generate a response"))
+					cmd.PrintErrln(handleClientError(err, "Failed to generate a response"))
 				}
 				sb.Reset()
 				continue
@@ -317,7 +317,7 @@ func generateInteractiveBasic(cmd *cobra.Command, desktopClient *desktop.Client,
 			if errors.Is(err, context.Canceled) {
 				fmt.Println("\nUse Ctrl + d or /bye to exit.")
 			} else {
-				cmd.PrintErr(handleClientError(err, "Failed to generate a response"))
+				cmd.PrintErrln(handleClientError(err, "Failed to generate a response"))
 			}
 			continue
 		}
@@ -628,17 +628,17 @@ func newRunCmd() *cobra.Command {
 				if err != nil {
 					return fmt.Errorf("failed to create Docker client: %w", err)
 				}
-				
+
 				// Run the NIM model
 				if err := runNIMModel(cmd.Context(), dockerClient, model, cmd); err != nil {
 					return fmt.Errorf("failed to run NIM model: %w", err)
 				}
-				
+
 				// If no prompt provided, enter interactive mode
 				if prompt == "" {
 					scanner := bufio.NewScanner(os.Stdin)
 					cmd.Println("Interactive chat mode started. Type '/bye' to exit.")
-					
+
 					for {
 						userInput, err := readMultilineInput(cmd, scanner)
 						if err != nil {
@@ -648,26 +648,26 @@ func newRunCmd() *cobra.Command {
 							}
 							return fmt.Errorf("Error reading input: %v", err)
 						}
-						
+
 						if strings.ToLower(strings.TrimSpace(userInput)) == "/bye" {
 							cmd.Println("Chat session ended.")
 							break
 						}
-						
+
 						if strings.TrimSpace(userInput) == "" {
 							continue
 						}
-						
+
 						if err := chatWithNIM(cmd, model, userInput); err != nil {
 							cmd.PrintErr(fmt.Errorf("failed to chat with NIM: %w", err))
 							continue
 						}
-						
+
 						cmd.Println()
 					}
 					return nil
 				}
-				
+
 				// Single prompt mode
 				if err := chatWithNIM(cmd, model, prompt); err != nil {
 					return fmt.Errorf("failed to chat with NIM: %w", err)
@@ -683,7 +683,7 @@ func newRunCmd() *cobra.Command {
 			_, err := desktopClient.Inspect(model, false)
 			if err != nil {
 				if !errors.Is(err, desktop.ErrNotFound) {
-					return handleNotRunningError(handleClientError(err, "Failed to inspect model"))
+					return handleClientError(err, "Failed to inspect model")
 				}
 				cmd.Println("Unable to find model '" + model + "' locally. Pulling from the server.")
 				if err := pullModel(cmd, desktopClient, model, ignoreRuntimeMemoryCheck); err != nil {
diff --git a/cmd/cli/commands/unload.go b/cmd/cli/commands/unload.go
@@ -25,8 +25,7 @@ func newUnloadCmd() *cobra.Command {
 			}
 			unloadResp, err := desktopClient.Unload(desktop.UnloadRequest{All: all, Backend: backend, Models: normalizedModels})
 			if err != nil {
-				err = handleClientError(err, "Failed to unload models")
-				return handleNotRunningError(err)
+				return handleClientError(err, "Failed to unload models")
 			}
 			unloaded := unloadResp.UnloadedRunners
 			if unloaded == 0 {
diff --git a/cmd/cli/commands/utils.go b/cmd/cli/commands/utils.go
@@ -2,12 +2,13 @@ package commands
 
 import (
 	"bytes"
+	"errors"
 	"fmt"
 	"strings"
 
 	"github.com/docker/cli/cli-plugins/hooks"
 	"github.com/docker/model-runner/cmd/cli/desktop"
-	"github.com/pkg/errors"
+	"github.com/docker/model-runner/pkg/inference/backends/vllm"
 )
 
 const (
@@ -18,24 +19,24 @@ const (
 const (
 	enableViaCLI = "Enable Docker Model Runner via the CLI → docker desktop enable model-runner"
 	enableViaGUI = "Enable Docker Model Runner via the GUI → Go to Settings->AI->Enable Docker Model Runner"
+	enableVLLM   = "It looks like you're trying to use a model for vLLM → docker model install-runner --vllm"
 )
 
 var notRunningErr = fmt.Errorf("Docker Model Runner is not running. Please start it and try again.\n")
 
 func handleClientError(err error, message string) error {
 	if errors.Is(err, desktop.ErrServiceUnavailable) {
-		return notRunningErr
-	}
-	return errors.Wrap(err, message)
-}
-
-func handleNotRunningError(err error) error {
-	if errors.Is(err, notRunningErr) {
+		err = notRunningErr
 		var buf bytes.Buffer
 		hooks.PrintNextSteps(&buf, []string{enableViaCLI, enableViaGUI})
 		return fmt.Errorf("%w\n%s", err, strings.TrimRight(buf.String(), "\n"))
+	} else if strings.Contains(err.Error(), vllm.StatusNotFound.Error()) {
+		// Handle `run` error.
+		var buf bytes.Buffer
+		hooks.PrintNextSteps(&buf, []string{enableVLLM})
+		return fmt.Errorf("%w\n%s", err, strings.TrimRight(buf.String(), "\n"))
 	}
-	return err
+	return errors.Join(err, errors.New(message))
 }
 
 // stripDefaultsFromModelName removes the default "ai/" prefix and ":latest" tag for display.
diff --git a/pkg/inference/backends/vllm/vllm.go b/pkg/inference/backends/vllm/vllm.go
@@ -29,6 +29,8 @@ const (
 	vllmDir = "/opt/vllm-env/bin"
 )
 
+var StatusNotFound = errors.New("vLLM binary not found")
+
 // vLLM is the vLLM-based backend implementation.
 type vLLM struct {
 	// log is the associated logger.
@@ -76,7 +78,8 @@ func (v *vLLM) Install(_ context.Context, _ *http.Client) error {
 	vllmBinaryPath := v.binaryPath()
 	if _, err := os.Stat(vllmBinaryPath); err != nil {
 		if errors.Is(err, fs.ErrNotExist) {
-			return fmt.Errorf("vLLM binary not found at %s", vllmBinaryPath)
+			v.status = StatusNotFound.Error()
+			return StatusNotFound
 		}
 		return fmt.Errorf("failed to check vLLM binary: %w", err)
 	}
diff --git a/pkg/inference/scheduling/scheduler.go b/pkg/inference/scheduling/scheduler.go
@@ -247,6 +247,8 @@ func (s *Scheduler) handleOpenAIInference(w http.ResponseWriter, r *http.Request
 			// shutting down (since that will also cancel the request context).
 			// Either way, provide a response, even if it's ignored.
 			http.Error(w, "service unavailable", http.StatusServiceUnavailable)
+		} else if errors.Is(err, vllm.StatusNotFound) {
+			http.Error(w, err.Error(), http.StatusPreconditionFailed)
 		} else {
 			http.Error(w, fmt.Errorf("backend installation failed: %w", err).Error(), http.StatusServiceUnavailable)
 		}

Original file line number	Diff line number	Diff line change
`@@ -53,15 +53,13 @@ func inspectModel(args []string, openai bool, remote bool, desktopClient *deskto`
`53`	`53`	`if openai {`
`54`	`54`	`model, err := desktopClient.InspectOpenAI(modelName)`
`55`	`55`	`if err != nil {`
`56`		`- err = handleClientError(err, "Failed to get model "+modelName)`
`57`		`- return "", handleNotRunningError(err)`
	`56`	`+ return "", handleClientError(err, "Failed to get model "+modelName)`
`58`	`57`	`}`
`59`	`58`	`return formatter.ToStandardJSON(model)`
`60`	`59`	`}`
`61`	`60`	`model, err := desktopClient.Inspect(modelName, remote)`
`62`	`61`	`if err != nil {`
`63`		`- err = handleClientError(err, "Failed to get model "+modelName)`
`64`		`- return "", handleNotRunningError(err)`
	`62`	`+ return "", handleClientError(err, "Failed to get model "+modelName)`
`65`	`63`	`}`
`66`	`64`	`return formatter.ToStandardJSON(model)`
`67`	`65`	`}`
Original file line number	Diff line number	Diff line change
`@@ -60,15 +60,13 @@ func listModels(openai bool, desktopClient *desktop.Client, quiet bool, jsonForm`
`60`	`60`	`if openai {`
`61`	`61`	`models, err := desktopClient.ListOpenAI()`
`62`	`62`	`if err != nil {`
`63`		`- err = handleClientError(err, "Failed to list models")`
`64`		`- return "", handleNotRunningError(err)`
	`63`	`+ return "", handleClientError(err, "Failed to list models")`
`65`	`64`	`}`
`66`	`65`	`return formatter.ToStandardJSON(models)`
`67`	`66`	`}`
`68`	`67`	`models, err := desktopClient.List()`
`69`	`68`	`if err != nil {`
`70`		`- err = handleClientError(err, "Failed to list models")`
`71`		`- return "", handleNotRunningError(err)`
	`69`	`+ return "", handleClientError(err, "Failed to list models")`
`72`	`70`	`}`
`73`	`71`
`74`	`72`	`if modelFilter != "" {`
Original file line number	Diff line number	Diff line change
`@@ -58,7 +58,7 @@ func pullModel(cmd cobra.Command, desktopClient desktop.Client, model string,`
`58`	`58`	`}`
`59`	`59`
`60`	`60`	`if err != nil {`
`61`		`- return handleNotRunningError(handleClientError(err, "Failed to pull model"))`
	`61`	`+ return handleClientError(err, "Failed to pull model")`
`62`	`62`	`}`
`63`	`63`
`64`	`64`	`cmd.Println(response)`
Original file line number	Diff line number	Diff line change
`@@ -45,7 +45,7 @@ func pushModel(cmd cobra.Command, desktopClient desktop.Client, model string)`
`45`	`45`	`}`
`46`	`46`
`47`	`47`	`if err != nil {`
`48`		`- return handleNotRunningError(handleClientError(err, "Failed to push model"))`
	`48`	`+ return handleClientError(err, "Failed to push model")`
`49`	`49`	`}`
`50`	`50`
`51`	`51`	`cmd.Println(response)`
Original file line number	Diff line number	Diff line change
`@@ -35,8 +35,7 @@ func newRequestsCmd() *cobra.Command {`
`35`	`35`	`if model != "" {`
`36`	`36`	`errMsg = errMsg + " for " + model`
`37`	`37`	`}`
`38`		`- err = handleClientError(err, errMsg)`
`39`		`- return handleNotRunningError(err)`
	`38`	`+ return handleClientError(err, errMsg)`
`40`	`39`	`}`
`41`	`40`	`defer cancel()`
`42`	`41`
Original file line number	Diff line number	Diff line change
`@@ -38,8 +38,7 @@ func newRemoveCmd() *cobra.Command {`
`38`	`38`	`cmd.Print(response)`
`39`	`39`	`}`
`40`	`40`	`if err != nil {`
`41`		`- err = handleClientError(err, "Failed to remove model")`
`42`		`- return handleNotRunningError(err)`
	`41`	`+ return handleClientError(err, "Failed to remove model")`
`43`	`42`	`}`
`44`	`43`	`return nil`
`45`	`44`	`},`
Original file line number	Diff line number	Diff line change
`@@ -25,8 +25,7 @@ func newUnloadCmd() *cobra.Command {`
`25`	`25`	`}`
`26`	`26`	`unloadResp, err := desktopClient.Unload(desktop.UnloadRequest{All: all, Backend: backend, Models: normalizedModels})`
`27`	`27`	`if err != nil {`
`28`		`- err = handleClientError(err, "Failed to unload models")`
`29`		`- return handleNotRunningError(err)`
	`28`	`+ return handleClientError(err, "Failed to unload models")`
`30`	`29`	`}`
`31`	`30`	`unloaded := unloadResp.UnloadedRunners`
`32`	`31`	`if unloaded == 0 {`