Fix e2e smoke tests for real leaderboard data

Author: biru-codeastromer

web/playwright.config.ts

@@ -12,6 +12,7 @@ export default defineConfig({
   fullyParallel: true,
   retries: process.env.CI ? 2 : 0,
   reporter: process.env.CI ? "github" : "list",
+  globalSetup: "./tests/e2e/global-setup.ts",
   use: {
     baseURL: "http://127.0.0.1:3100",
     trace: "on-first-retry"
@@ -27,7 +28,7 @@ export default defineConfig({
             WMG_UPLOAD_TOKEN: "e2e-token",
             WMG_AUTO_MIGRATE: "true",
             WMG_ENABLE_METRICS: "false",
-            WMG_SEED_DEMO_DATA: "true"
+            WMG_SEED_DEMO_DATA: "false"
           },
           url: "http://127.0.0.1:8100/healthz",
           reuseExistingServer: !process.env.CI,

web/tests/e2e/global-setup.ts

@@ -0,0 +1,71 @@
+import { execFileSync } from "node:child_process";
+import path from "node:path";
+
+async function waitFor(url: string, timeoutMs: number): Promise<void> {
+  const deadline = Date.now() + timeoutMs;
+
+  while (Date.now() < deadline) {
+    try {
+      const response = await fetch(url);
+      if (response.ok) {
+        return;
+      }
+    } catch {}
+    await new Promise((resolve) => setTimeout(resolve, 1000));
+  }
+
+  throw new Error(`Timed out waiting for ${url}`);
+}
+
+export default async function globalSetup() {
+  const repoRoot = path.resolve(__dirname, "../../..");
+  const pythonExecutable = path.resolve(
+    repoRoot,
+    process.env.WMG_E2E_PYTHON ?? ".venv/bin/python",
+  );
+  const apiBase = process.env.WMG_E2E_API_BASE ?? "http://127.0.0.1:8100";
+  const uploadToken = process.env.WMG_E2E_UPLOAD_TOKEN ?? "e2e-token";
+
+  await waitFor(`${apiBase}/healthz`, 120_000);
+
+  const uniqueSuffix = Date.now().toString(36);
+  const runs = [
+    {
+      runId: `e2e_random_test_${uniqueSuffix}`,
+      agent: "random",
+      env: "memory_maze",
+      track: "test",
+    },
+    {
+      runId: `e2e_oracle_train_${uniqueSuffix}`,
+      agent: "greedy_oracle",
+      env: "switch_quest",
+      track: "train",
+    },
+  ];
+
+  for (const run of runs) {
+    execFileSync(
+      pythonExecutable,
+      [
+        "scripts/demo_run.py",
+        "--api-base",
+        apiBase,
+        "--upload-token",
+        uploadToken,
+        "--agent",
+        run.agent,
+        "--env",
+        run.env,
+        "--track",
+        run.track,
+        "--run-id",
+        run.runId,
+      ],
+      {
+        cwd: repoRoot,
+        stdio: "pipe",
+      },
+    );
+  }
+}

web/tests/e2e/leaderboard.spec.ts

@@ -1,6 +1,6 @@
 import { expect, test } from "@playwright/test";
 
-test("homepage and seeded leaderboard feel populated", async ({ page }) => {
+test("homepage and leaderboard feel populated with uploaded runs", async ({ page }) => {
   await page.goto("/");
 
   await expect(
@@ -13,10 +13,10 @@ test("homepage and seeded leaderboard feel populated", async ({ page }) => {
   await expect(page).toHaveURL(/\/leaderboard/);
   await expect(page.getByText(/Track planning quality/i)).toBeVisible();
   await expect(page.locator("table tbody tr").first()).toBeVisible();
-  await expect(page.getByText("demo-mpc")).toBeVisible();
+  await expect(page.getByRole("cell", { name: "random" }).first()).toBeVisible();
 
   await page.getByRole("button", { name: "train" }).click();
-  await expect(page.getByText("demo-oracle")).toBeVisible();
+  await expect(page.getByRole("cell", { name: "greedy_oracle" }).first()).toBeVisible();
 });
 
 test("run detail pages resolve from leaderboard links", async ({ page }) => {