Make the test suite parallel-safe and speed up CI (#1273)

* Make self-host tests CI-safe

* Await cloud db teardown

* Give local e2e boots more headroom

* Keep main CI runs alive

* Cache CI dependencies and image layers

* Wait for recorded introspection requests in graphql plugin tests

* Size self-host test budget for loaded runners

* Probe test http servers for readiness before use

* Die on recorder wait timeout instead of typing a string error

* Type the recorder wait timeout as a tagged error

* Update e2e locators for the integration copy renames

* Keep stale data rendered while a refresh is waiting

* Guard create-org input against hydration clobbering

* Match the new sign-in heading in the selfhost callback test

Author: RhysSullivan

.github/workflows/ci.yml

@@ -7,8 +7,8 @@ on:
       - main
 
 concurrency:
-  group: ci-${{ github.ref }}
-  cancel-in-progress: true
+  group: ci-${{ github.event_name == 'push' && format('{0}-{1}', github.ref, github.sha) || github.ref }}
+  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
 jobs:
   format:
@@ -21,6 +21,14 @@ jobs:
         with:
           bun-version: 1.3.11
 
+      - name: Cache Bun package cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.bun/install/cache
+          key: ${{ runner.os }}-bun-1.3.11-${{ hashFiles('bun.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-bun-1.3.11-
+
       - run: bun install --frozen-lockfile
 
       - run: bun run format:check
@@ -35,6 +43,14 @@ jobs:
         with:
           bun-version: 1.3.11
 
+      - name: Cache Bun package cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.bun/install/cache
+          key: ${{ runner.os }}-bun-1.3.11-${{ hashFiles('bun.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-bun-1.3.11-
+
       - run: bun install --frozen-lockfile
 
       - run: bun run lint
@@ -49,20 +65,38 @@ jobs:
         with:
           bun-version: 1.3.11
 
+      - name: Cache Bun package cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.bun/install/cache
+          key: ${{ runner.os }}-bun-1.3.11-${{ hashFiles('bun.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-bun-1.3.11-
+
       - run: bun install --frozen-lockfile
 
       - run: bun run typecheck
 
   test:
     name: Test
     runs-on: ubuntu-latest
+    env:
+      TURBO_TEST_CONCURRENCY: 3
     steps:
       - uses: actions/checkout@v4
 
       - uses: oven-sh/setup-bun@v2
         with:
           bun-version: 1.3.11
 
+      - name: Cache Bun package cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.bun/install/cache
+          key: ${{ runner.os }}-bun-1.3.11-${{ hashFiles('bun.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-bun-1.3.11-
+
       # apps/cloud's test script invokes `node` directly; undici 8.x (pulled
       # in by @cloudflare/vitest-pool-workers) calls webidl.markAsUncloneable
       # which only exists in Node 22.10+. Pin a known-good runtime.
@@ -105,6 +139,14 @@ jobs:
         with:
           bun-version: 1.3.11
 
+      - name: Cache Bun package cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.bun/install/cache
+          key: ${{ runner.os }}-bun-1.3.11-${{ hashFiles('bun.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-bun-1.3.11-
+
       # The dev stacks spawn Node sidecars (vite/workerd tooling); pin the
       # same known-good runtime the unit-test job uses.
       - uses: actions/setup-node@v4
@@ -113,6 +155,12 @@ jobs:
 
       - run: bun install --frozen-lockfile
 
+      - name: Cache Playwright browsers
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/ms-playwright
+          key: ${{ runner.os }}-playwright-1.60.0
+
       # Install from e2e so bunx resolves ITS pinned playwright (the version
       # the tests run against) rather than floating to the latest.
       - name: Install Playwright Chromium
@@ -152,6 +200,14 @@ jobs:
         with:
           bun-version: 1.3.11
 
+      - name: Cache Bun package cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.bun/install/cache
+          key: ${{ runner.os }}-bun-1.3.11-${{ hashFiles('bun.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-bun-1.3.11-
+
       # The local scenarios boot a real `executor web` (which spawns a Node
       # sidecar) and some drive a browser, so pin Node 22 and install Chromium.
       - uses: actions/setup-node@v4
@@ -160,6 +216,12 @@ jobs:
 
       - run: bun install --frozen-lockfile
 
+      - name: Cache Playwright browsers
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/ms-playwright
+          key: ${{ runner.os }}-playwright-1.60.0
+
       # `chromium` and the new `chromium-headless-shell` ship as separate
       # downloads; the browser-driven scenarios launch the headless shell.
       # Install from e2e so bunx resolves ITS pinned playwright (the version the
@@ -190,6 +252,14 @@ jobs:
         with:
           bun-version: 1.3.11
 
+      - name: Cache Bun package cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.bun/install/cache
+          key: ${{ runner.os }}-bun-1.3.11-${{ hashFiles('bun.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-bun-1.3.11-
+
       - run: bun install --frozen-lockfile
 
       - name: Build web app
@@ -221,3 +291,5 @@ jobs:
           file: apps/host-selfhost/Dockerfile
           push: false
           tags: executor-selfhost:ci
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

apps/cloud/src/db/db.ts

@@ -73,16 +73,12 @@ const makePostgresResource = (): DbResource => {
     sql,
     db: drizzle(sql, { schema: combinedSchema }) as DrizzleDb,
     close: () =>
-      Effect.sync(() => {
-        void Effect.runFork(
-          Effect.ignore(
-            Effect.tryPromise({
-              try: () => sql.end({ timeout: 0 }),
-              catch: (cause) => cause,
-            }),
-          ),
-        );
-      }),
+      Effect.ignore(
+        Effect.tryPromise({
+          try: () => sql.end({ timeout: 0 }),
+          catch: (cause) => cause,
+        }),
+      ),
   };
 };
 

apps/host-selfhost/src/boot.test.ts

@@ -2,20 +2,26 @@ import { mkdtempSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 
-import { afterAll, expect, test } from "@effect/vitest";
+import { afterAll, beforeAll, expect, test } from "@effect/vitest";
 
 // Config reads the environment, so point it at a throwaway data dir before
 // importing the app graph.
 process.env.EXECUTOR_DATA_DIR = mkdtempSync(join(tmpdir(), "eh-boot-"));
 
-const { makeSelfHostTestApp, singleAdminIdentityLayer } = await import("./testing/test-app");
+let handler!: (request: Request) => Promise<Response>;
+let dispose: () => Promise<void> = async () => {};
 
-const { handler, dispose } = await makeSelfHostTestApp({
-  identity: singleAdminIdentityLayer({
-    userId: "admin",
-    organizationId: "default-org",
-    organizationName: "Default",
-  }),
+beforeAll(async () => {
+  const { makeSelfHostTestApp, singleAdminIdentityLayer } = await import("./testing/test-app");
+  const app = await makeSelfHostTestApp({
+    identity: singleAdminIdentityLayer({
+      userId: "admin",
+      organizationId: "default-org",
+      organizationName: "Default",
+    }),
+  });
+  handler = app.handler;
+  dispose = app.dispose;
 });
 afterAll(() => dispose());
 

apps/host-selfhost/src/multi-user.test.ts

@@ -2,7 +2,7 @@ import { mkdtempSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 
-import { afterAll, expect, test } from "@effect/vitest";
+import { afterAll, beforeAll, expect, test } from "@effect/vitest";
 import { connectionIdentifier } from "@executor-js/sdk/shared";
 
 import { mintInviteCode } from "./testing/mint-invite";
@@ -13,9 +13,15 @@ process.env.BETTER_AUTH_SECRET = "multi-user-secret-0123456789-abcdefghij-klmn";
 process.env.EXECUTOR_BOOTSTRAP_ADMIN_EMAIL = "admin@multi.test";
 process.env.EXECUTOR_BOOTSTRAP_ADMIN_PASSWORD = "admin-pass-123456";
 
-const { makeSelfHostApiHandler } = await import("./app");
+let handler!: (request: Request) => Promise<Response>;
+let dispose: () => Promise<void> = async () => {};
 
-const { handler, dispose } = await makeSelfHostApiHandler();
+beforeAll(async () => {
+  const { makeSelfHostApiHandler } = await import("./app");
+  const app = await makeSelfHostApiHandler();
+  handler = app.handler;
+  dispose = app.dispose;
+});
 afterAll(() => dispose());
 
 const BASE = "http://localhost:4788";

apps/host-selfhost/src/scope-isolation.test.ts

@@ -2,7 +2,7 @@ import { mkdtempSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 
-import { afterAll, expect, test } from "@effect/vitest";
+import { afterAll, beforeAll, expect, test } from "@effect/vitest";
 import { connectionIdentifier } from "@executor-js/sdk/shared";
 
 process.env.EXECUTOR_DATA_DIR = mkdtempSync(join(tmpdir(), "eh-iso-"));
@@ -13,10 +13,16 @@ process.env.EXECUTOR_DATA_DIR = mkdtempSync(join(tmpdir(), "eh-iso-"));
 // request-scoped. Each identity is its own (org, user): in v2 the org is the
 // tenant (catalog partition) and the user is the acting subject (drives
 // `owner: "user"` rows).
-const { makeSelfHostTestApp, headerIdentityLayer } = await import("./testing/test-app");
+let handler!: (request: Request) => Promise<Response>;
+let dispose: () => Promise<void> = async () => {};
 
-const { handler, dispose } = await makeSelfHostTestApp({
-  identity: headerIdentityLayer,
+beforeAll(async () => {
+  const { makeSelfHostTestApp, headerIdentityLayer } = await import("./testing/test-app");
+  const app = await makeSelfHostTestApp({
+    identity: headerIdentityLayer,
+  });
+  handler = app.handler;
+  dispose = app.dispose;
 });
 afterAll(() => dispose());
 
@@ -135,7 +141,7 @@ test("concurrent requests with distinct identities get disjoint, correct executo
       expect(addresses.some((a) => a.includes(connectionNameForUser(other)))).toBe(false);
     }
   });
-}, 30_000);
+});
 
 test("a request with no identity is rejected", async () => {
   const res = await handler(new Request("http://localhost/api/connections"));

apps/host-selfhost/src/secrets-integration.test.ts

@@ -4,7 +4,7 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 
 import { Effect, Layer } from "effect";
-import { afterAll, expect, test } from "@effect/vitest";
+import { afterAll, beforeAll, expect, test } from "@effect/vitest";
 
 import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk";
 import { makeScopedExecutor } from "@executor-js/api/server";
@@ -32,13 +32,20 @@ const createScopedExecutor = (
     Effect.provide(SelfHostScopedExecutorSeams),
   );
 
-const dbHandle = await createSelfHostDb({
-  path: dbPath,
-  namespace: "executor_selfhost",
-  version: "1.0.0",
+let dbLayer!: Layer.Layer<SelfHostDb>;
+let dbHandle: Awaited<ReturnType<typeof createSelfHostDb>> | undefined;
+
+beforeAll(async () => {
+  dbHandle = await createSelfHostDb({
+    path: dbPath,
+    namespace: "executor_selfhost",
+    version: "1.0.0",
+  });
+  dbLayer = Layer.succeed(SelfHostDb)(dbHandle);
+});
+afterAll(async () => {
+  await dbHandle?.close();
 });
-const dbLayer = Layer.succeed(SelfHostDb)(dbHandle);
-afterAll(() => dbHandle.close());
 
 const TINY_SPEC = JSON.stringify({
   openapi: "3.0.0",

apps/host-selfhost/src/sources-mcp.test.ts

@@ -3,7 +3,7 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 
 import { Effect, Layer } from "effect";
-import { afterAll, expect, test } from "@effect/vitest";
+import { afterAll, beforeAll, expect, test } from "@effect/vitest";
 
 import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk";
 import { makeScopedExecutor } from "@executor-js/api/server";
@@ -50,8 +50,15 @@ const TINY_SPEC = JSON.stringify({
   },
 });
 
-const { makeSelfHostApiHandler } = await import("./app");
-const { handler, dispose } = await makeSelfHostApiHandler({ dbPath });
+let handler!: (request: Request) => Promise<Response>;
+let dispose: () => Promise<void> = async () => {};
+
+beforeAll(async () => {
+  const { makeSelfHostApiHandler } = await import("./app");
+  const app = await makeSelfHostApiHandler({ dbPath });
+  handler = app.handler;
+  dispose = app.dispose;
+});
 afterAll(() => dispose());
 
 const BASE = "http://localhost:4788";

apps/host-selfhost/src/sources.test.ts

@@ -3,7 +3,7 @@ import { tmpdir } from "node:os";
 import { join } from "node:path";
 
 import { Effect, Layer } from "effect";
-import { afterAll, expect, test } from "@effect/vitest";
+import { afterAll, beforeAll, expect, test } from "@effect/vitest";
 
 import { AuthTemplateSlug, ConnectionName, IntegrationSlug } from "@executor-js/sdk";
 import { makeScopedExecutor } from "@executor-js/api/server";
@@ -28,13 +28,20 @@ const createScopedExecutor = (
 const dataDir = mkdtempSync(join(tmpdir(), "eh-src-"));
 process.env.EXECUTOR_DATA_DIR = dataDir;
 
-const dbHandle = await createSelfHostDb({
-  path: join(dataDir, "data.db"),
-  namespace: "executor_selfhost",
-  version: "1.0.0",
+let dbLayer!: Layer.Layer<SelfHostDb>;
+let dbHandle: Awaited<ReturnType<typeof createSelfHostDb>> | undefined;
+
+beforeAll(async () => {
+  dbHandle = await createSelfHostDb({
+    path: join(dataDir, "data.db"),
+    namespace: "executor_selfhost",
+    version: "1.0.0",
+  });
+  dbLayer = Layer.succeed(SelfHostDb)(dbHandle);
+});
+afterAll(async () => {
+  await dbHandle?.close();
 });
-const dbLayer = Layer.succeed(SelfHostDb)(dbHandle);
-afterAll(() => dbHandle.close());
 
 // Inline OpenAPI spec so the test doesn't depend on the network to register.
 const TINY_SPEC = JSON.stringify({