fix: make assistant E2E tests model-agnostic and upgrade to qwen3 (#498)

* fix: make assistant E2E tests model-agnostic and upgrade to qwen3

The assistant E2E tests were flaky because they asserted on specific
LLM output patterns (regex for sequential numbers, greeting with
username). This breaks with models that have thinking mode (qwen3)
or different output formatting.

Changes:
- Replace two flaky tests with a single deterministic arithmetic
  test ("What is 1 + 1?" → check response contains "2")
- Increase beforeAll warmup timeout from 30s to 120s — qwen3 with
  thinking mode takes ~40s for first inference on CI runners
- Upgrade E2E model from qwen2.5:1.5b to qwen3:1.7b in workflow
  and all source file defaults

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

* fix: use testInfo.setTimeout and scope assertion to sidebar

- Use testInfo.setTimeout in beforeAll with biome-ignore for the
  required empty destructuring (Playwright 1.58 does not support
  the { timeout } option on beforeAll)
- Scope the /\b2\b/ assertion to the assistant sidebar
  (data-side="right") to avoid false positives from unrelated page
  content like pagination or counts

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>

Author: peppescg

.github/workflows/e2e.yml

@@ -26,7 +26,7 @@ jobs:
       BETTER_AUTH_URL: http://localhost:3000
       BETTER_AUTH_SECRET: test-only-not-a-real-better-auth-secret
       USE_E2E_MODEL: "true"
-      E2E_MODEL_NAME: qwen2.5:1.5b
+      E2E_MODEL_NAME: qwen3:1.7b
       OLLAMA_BASE_URL: http://localhost:11434
     steps:
       - name: Checkout
@@ -36,7 +36,7 @@ jobs:
         uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5
         with:
           path: /home/runner/.ollama/models
-          key: ollama-qwen2.5-1.5b-v3
+          key: ollama-qwen3-1.7b-v1
 
       - name: Start Ollama via Docker
         run: |
@@ -53,7 +53,7 @@ jobs:
           done
 
       - name: Pull Ollama model
-        run: docker exec ollama ollama pull qwen2.5:1.5b
+        run: docker exec ollama ollama pull qwen3:1.7b
 
       - name: Setup
         uses: ./.github/actions/setup

playwright.config.mts

@@ -54,7 +54,7 @@ export default defineConfig({
           BETTER_AUTH_RATE_LIMIT: "100",
           // Always use testing model for E2E tests to avoid needing OpenRouter API keys
           USE_E2E_MODEL: "true",
-          E2E_MODEL_NAME: process.env.E2E_MODEL_NAME ?? "qwen2.5:1.5b",
+          E2E_MODEL_NAME: process.env.E2E_MODEL_NAME ?? "qwen3:1.7b",
           OLLAMA_BASE_URL:
             process.env.OLLAMA_BASE_URL ?? "http://localhost:11434",
         },

src/app/api/chat/route.ts

@@ -17,7 +17,7 @@ import {
 } from "@/lib/mcp/client";
 import { SYSTEM_PROMPT } from "./system-prompt";
 
-const E2E_MODEL_NAME = process.env.E2E_MODEL_NAME ?? "qwen2.5:1.5b";
+const E2E_MODEL_NAME = process.env.E2E_MODEL_NAME ?? "qwen3:1.7b";
 
 export const maxDuration = 60;
 

tests/e2e/assistant.spec.ts

@@ -1,7 +1,7 @@
 import { expect, test } from "./fixtures";
 
 const OLLAMA_BASE_URL = process.env.OLLAMA_BASE_URL ?? "http://localhost:11434";
-const E2E_MODEL_NAME = process.env.E2E_MODEL_NAME ?? "qwen2.5:1.5b";
+const E2E_MODEL_NAME = process.env.E2E_MODEL_NAME ?? "qwen3:1.7b";
 
 /**
  * Warms up the testing model (Ollama) by making a simple generation request.
@@ -13,7 +13,7 @@ async function warmupTestingModel(): Promise<void> {
     headers: { "Content-Type": "application/json" },
     body: JSON.stringify({
       model: E2E_MODEL_NAME,
-      prompt: "Say hello",
+      prompt: "What is 1 + 1?",
       stream: false,
     }),
   });
@@ -34,8 +34,12 @@ test.describe("Assistant chat", () => {
   // Triple all timeouts for this describe block since LLM operations are slow
   test.slow();
 
-  // Warmup testing model before running any tests in this describe block
-  test.beforeAll(async () => {
+  // Warmup testing model before running any tests in this describe block.
+  // qwen3 with thinking mode needs >30s for first inference on CI runners,
+  // so we set a generous timeout (120s) for model loading + first generation.
+  // biome-ignore lint/correctness/noEmptyPattern: Playwright requires destructured first arg to access testInfo
+  test.beforeAll(async ({}, testInfo) => {
+    testInfo.setTimeout(120_000);
     console.log("Warming up E2E testing model...");
     const startTime = Date.now();
 
@@ -48,12 +52,9 @@ test.describe("Assistant chat", () => {
     }
   });
 
-  test("responds to user message with expected content", async ({
+  test("responds to a simple arithmetic question", async ({
     authenticatedPage,
   }) => {
-    // Use a unique identifier that we expect to appear in the response
-    const testUsername = `testuser_${Date.now()}`;
-
     // Navigate to catalog page which has the assistant sidebar
     await authenticatedPage.goto("/catalog");
 
@@ -67,53 +68,16 @@ test.describe("Assistant chat", () => {
       authenticatedPage.getByPlaceholder(/type your message/i),
     ).toBeVisible({ timeout: 10_000 });
 
-    // Type a message that includes the unique identifier
     const textarea = authenticatedPage.getByPlaceholder(/type your message/i);
-    await textarea.fill(
-      `Reply with a short greeting for the user named '${testUsername}'. Include their exact username in your response.`,
-    );
+    await textarea.fill("What is 1 + 1? Reply with just the number.");
 
-    // Submit the message
     await authenticatedPage.keyboard.press("Enter");
 
-    // Wait for the assistant's response to appear
-    // The response should contain our unique username in a greeting
-    // Using a generous timeout since model inference can take time
-    // The regex matches any greeting pattern followed by the username
-    await expect(
-      authenticatedPage.getByText(
-        new RegExp(`(hello|hi|hey|greetings).*${testUsername}`, "i"),
-      ),
-    ).toBeVisible({
-      timeout: 60_000,
-    });
-  });
-
-  test("displays streaming response", async ({ authenticatedPage }) => {
-    // Navigate to catalog page which has the assistant sidebar
-    await authenticatedPage.goto("/catalog");
-
-    // Open the assistant sidebar
-    await authenticatedPage
-      .getByRole("button", { name: "Toggle Assistant" })
-      .click();
-
-    // Wait for the sidebar to open and chat input to be visible
-    await expect(
-      authenticatedPage.getByPlaceholder(/type your message/i),
-    ).toBeVisible({ timeout: 10_000 });
-
-    const textarea = authenticatedPage.getByPlaceholder(/type your message/i);
-    await textarea.fill("Count from 1 to 5, one number per line.");
-
-    await authenticatedPage.keyboard.press("Enter");
-
-    // Wait for the assistant's response containing numbers
-    // Look for a pattern that indicates the assistant counted - digits on separate lines
-    // This won't match the user's message "1 to 5" or model name "4.5"
-    await expect(
-      authenticatedPage.getByText(/\b1\s+2\s+3\b/), // Sequential numbers separated by whitespace
-    ).toBeVisible({
+    // Wait for the assistant's response containing "2".
+    // Scope to the assistant sidebar (data-side="right") to avoid matching
+    // unrelated "2" text elsewhere on the page (pagination, counts, etc.).
+    const sidebar = authenticatedPage.locator('[data-side="right"]');
+    await expect(sidebar.getByText(/\b2\b/)).toBeVisible({
       timeout: 60_000,
     });
   });