Merge pull request #784 from docker/e2e

test(e2e): add multi-backend, API compat, and concurrency tests

Author: doringeman

e2e/api_test.go

@@ -0,0 +1,106 @@
+//go:build e2e
+
+package e2e
+
+import (
+	"encoding/json"
+	"net/http"
+	"strings"
+	"testing"
+
+	"github.com/docker/model-runner/cmd/cli/desktop"
+)
+
+func TestE2E_APIErrors(t *testing.T) {
+	t.Run("InvalidModel", func(t *testing.T) {
+		status, body := doJSON(t, http.MethodPost, serverURL+"/engines/v1/chat/completions",
+			desktop.OpenAIChatRequest{
+				Model:    "nonexistent/model:latest",
+				Messages: []desktop.OpenAIChatMessage{{Role: "user", Content: "hi"}},
+			})
+		if status == http.StatusOK {
+			t.Fatalf("expected error for invalid model, got 200: %s", body)
+		}
+		t.Logf("invalid model: status=%d", status)
+	})
+
+	t.Run("MalformedJSON", func(t *testing.T) {
+		req, _ := http.NewRequestWithContext(t.Context(), http.MethodPost,
+			serverURL+"/engines/v1/chat/completions",
+			strings.NewReader(`{bad json`))
+		req.Header.Set("Content-Type", "application/json")
+		resp, err := http.DefaultClient.Do(req)
+		if err != nil {
+			t.Fatalf("request failed: %v", err)
+		}
+		resp.Body.Close()
+		if resp.StatusCode == http.StatusOK {
+			t.Fatal("expected error for malformed JSON")
+		}
+		t.Logf("malformed JSON: status=%d", resp.StatusCode)
+	})
+
+	t.Run("EmptyMessages", func(t *testing.T) {
+		status, body := doJSON(t, http.MethodPost, serverURL+"/engines/v1/chat/completions",
+			desktop.OpenAIChatRequest{
+				Model:    ggufModel,
+				Messages: []desktop.OpenAIChatMessage{},
+			})
+		// Some backends accept empty messages, some don't — just verify no 500.
+		if status == http.StatusInternalServerError {
+			t.Fatalf("unexpected 500 for empty messages: %s", body)
+		}
+		t.Logf("empty messages: status=%d", status)
+	})
+}
+
+func TestE2E_ModelLifecycle(t *testing.T) {
+	model := ggufModel
+
+	t.Run("PullIdempotent", func(t *testing.T) {
+		pullModel(t, model)
+		output := pullModel(t, model)
+		if !strings.Contains(output, "Using cached model") {
+			t.Errorf("expected 'Using cached model' in second pull, got: %s", output)
+		}
+	})
+
+	t.Run("InspectModel", func(t *testing.T) {
+		status, body := doJSON(t, http.MethodGet,
+			serverURL+"/engines/v1/models/"+model, nil)
+		if status != http.StatusOK {
+			t.Fatalf("inspect: status=%d body=%s", status, body)
+		}
+		var m struct {
+			ID string `json:"id"`
+		}
+		if err := json.Unmarshal(body, &m); err != nil {
+			t.Fatalf("decode: %v", err)
+		}
+		if m.ID == "" {
+			t.Fatal("empty model ID in inspect response")
+		}
+		t.Logf("inspect: id=%s", m.ID)
+	})
+
+	t.Run("OllamaShow", func(t *testing.T) {
+		status, body := doJSON(t, http.MethodPost, serverURL+"/api/show",
+			map[string]string{"name": model})
+		if status != http.StatusOK {
+			t.Fatalf("show: status=%d body=%s", status, body)
+		}
+		var show struct {
+			Details struct {
+				Format string `json:"format"`
+			} `json:"details"`
+		}
+		if err := json.Unmarshal(body, &show); err != nil {
+			t.Fatalf("decode: %v", err)
+		}
+		t.Logf("format: %s", show.Details.Format)
+	})
+
+	t.Run("Remove", func(t *testing.T) {
+		removeModel(t, model)
+	})
+}

e2e/cli_test.go

@@ -7,46 +7,45 @@ import (
 	"testing"
 )
 
-// TestE2E_CLI runs all CLI tests sequentially as subtests to ensure
-// correct ordering (pull → list → run → remove).
 func TestE2E_CLI(t *testing.T) {
-	t.Run("Pull", func(t *testing.T) {
-		out, err := runCLI(t, "pull", testModel)
-		if err != nil {
-			t.Fatalf("cli pull failed: %v\noutput: %s", err, out)
-		}
-		t.Logf("pull output: %s", out)
-	})
-
-	t.Run("List", func(t *testing.T) {
-		out, err := runCLI(t, "ls")
-		if err != nil {
-			t.Fatalf("cli ls failed: %v\noutput: %s", err, out)
-		}
-
-		if !strings.Contains(out, "smollm2") {
-			t.Errorf("expected smollm2 in list output, got:\n%s", out)
-		}
-		t.Logf("ls output:\n%s", out)
-	})
-
-	t.Run("Run", func(t *testing.T) {
-		out, err := runCLI(t, "run", testModel, "Say hi in one word.")
-		if err != nil {
-			t.Fatalf("cli run failed: %v\noutput: %s", err, out)
-		}
-
-		if strings.TrimSpace(out) == "" {
-			t.Fatal("cli run produced empty output")
-		}
-		t.Logf("run output: %s", out)
-	})
-
-	t.Run("Remove", func(t *testing.T) {
-		out, err := runCLI(t, "rm", "-f", testModel)
-		if err != nil {
-			t.Fatalf("cli rm failed: %v\noutput: %s", err, out)
-		}
-		t.Logf("rm output: %s", out)
-	})
+	for _, bc := range backends {
+		bc := bc
+		t.Run(bc.name, func(t *testing.T) {
+			t.Run("Pull", func(t *testing.T) {
+				out, err := runCLI(t, "pull", bc.model)
+				if err != nil {
+					t.Fatalf("cli pull failed: %v\noutput: %s", err, out)
+				}
+				t.Logf("pull output: %s", out)
+			})
+
+			t.Run("List", func(t *testing.T) {
+				out, err := runCLI(t, "ls")
+				if err != nil {
+					t.Fatalf("cli ls failed: %v\noutput: %s", err, out)
+				}
+				if !strings.Contains(out, "smollm2") {
+					t.Errorf("expected smollm2 in list output, got:\n%s", out)
+				}
+			})
+
+			t.Run("Run", func(t *testing.T) {
+				out, err := runCLI(t, "run", bc.model, "Say hi in one word.")
+				if err != nil {
+					t.Fatalf("cli run failed: %v\noutput: %s", err, out)
+				}
+				if strings.TrimSpace(out) == "" {
+					t.Fatal("cli run produced empty output")
+				}
+				t.Logf("run output: %s", out)
+			})
+
+			t.Run("Remove", func(t *testing.T) {
+				out, err := runCLI(t, "rm", "-f", bc.model)
+				if err != nil {
+					t.Fatalf("cli rm failed: %v\noutput: %s", err, out)
+				}
+			})
+		})
+	}
 }

e2e/concurrency_test.go

@@ -0,0 +1,28 @@
+//go:build e2e
+
+package e2e
+
+import (
+	"fmt"
+	"testing"
+)
+
+// TestE2E_ConcurrentRequests sends parallel chat completions to verify
+// the scheduler handles concurrent slot allocation correctly.
+func TestE2E_ConcurrentRequests(t *testing.T) {
+	model := ggufModel
+	pullModel(t, model)
+	t.Cleanup(func() {
+		removeModel(t, model)
+	})
+
+	for i := range 5 {
+		t.Run(fmt.Sprintf("request-%d", i), func(t *testing.T) {
+			t.Parallel()
+			resp := chatCompletion(t, model, "Say a single word.")
+			if len(resp.Choices) == 0 || resp.Choices[0].Message.Content == "" {
+				t.Fatal("empty response")
+			}
+		})
+	}
+}