ci(e2e): parallelize tests with Nx, block non-essential assets, add timing logs

Author: tannerlinsley

.github/workflows/e2e-opportunistic-matrix.yml

@@ -39,14 +39,12 @@ jobs:
           fetch-depth: 0
       - name: Setup Tools
         uses: tanstack/config/.github/setup@main
-      - name: Build
-        run: pnpm build
       - name: Install Playwright Chrome
         run: pnpm --filter @tanstack/cli exec playwright install --with-deps chrome
       - name: Run Matrix Scenario
         env:
           E2E_MATRIX_SCENARIO: ${{ matrix.scenario }}
-        run: pnpm --filter @tanstack/cli test:e2e:matrix
+        run: pnpm nx run @tanstack/cli:test:e2e:matrix
       - name: Upload Playwright Report
         if: always()
         uses: actions/upload-artifact@v4

.github/workflows/pr.yml

@@ -50,9 +50,11 @@ jobs:
           fetch-depth: 0
       - name: Setup Tools
         uses: tanstack/config/.github/setup@main
-      - name: Build
-        run: pnpm build
+      - name: Get base and head commits for `nx affected`
+        uses: nrwl/nx-set-shas@v4.4.0
+        with:
+          main-branch-name: main
       - name: Install Playwright Chrome
         run: pnpm --filter @tanstack/cli exec playwright install --with-deps chrome
       - name: Test E2E Blocking
-        run: pnpm test:e2e
+        run: pnpm nx affected --target=test:e2e --parallel=3

nx.json

@@ -10,8 +10,25 @@
   },
   "targetDefaults": {
     "build": {
+      "cache": true,
       "inputs": ["production", "^production"],
+      "outputs": ["{projectRoot}/dist"],
       "dependsOn": ["^build"]
+    },
+    "test:e2e": {
+      "cache": true,
+      "dependsOn": ["^build", "build"],
+      "inputs": ["default", "^production"]
+    },
+    "test:e2e:blocking": {
+      "cache": true,
+      "dependsOn": ["^build", "build"],
+      "inputs": ["default", "^production"]
+    },
+    "test:e2e:matrix": {
+      "cache": true,
+      "dependsOn": ["^build", "build"],
+      "inputs": ["default", "^production"]
     }
   }
 }

package.json

@@ -13,8 +13,8 @@
     "dev": "nx run-many --target=dev --parallel 20",
     "test": "pnpm test:unit && pnpm test:e2e",
     "test:unit": "nx run-many --target=test",
-    "test:e2e": "pnpm --filter @tanstack/cli test:e2e",
-    "test:e2e:matrix": "pnpm --filter @tanstack/cli test:e2e:matrix",
+    "test:e2e": "nx run @tanstack/cli:test:e2e",
+    "test:e2e:matrix": "nx run @tanstack/cli:test:e2e:matrix",
     "check-outdated": "node scripts/check-outdated-packages.js",
     "update-outdated": "node scripts/check-outdated-packages.js --update",
     "prepare": "husky install",

packages/cli/package.json

@@ -16,10 +16,12 @@
     "build": "tsc",
     "dev": "tsc --watch",
     "test:e2e": "npm run test:e2e:blocking",
-    "test:e2e:blocking": "npm run build && playwright test --grep @blocking",
-    "test:e2e:matrix": "npm run build && playwright test --grep @matrix",
-    "test:e2e:debug": "npm run build && playwright test --debug",
-    "test:e2e:headed": "npm run build && playwright test --headed",
+    "test:e2e:blocking": "playwright test --grep @blocking",
+    "test:e2e:matrix": "playwright test --grep @matrix",
+    "test:e2e:debug": "playwright test --debug",
+    "test:e2e:headed": "playwright test --headed",
+    "test:e2e:local": "npm run build && npm run test:e2e:blocking",
+    "test:e2e:matrix:local": "npm run build && npm run test:e2e:matrix",
     "test:lint": "eslint ./src",
     "test": "vitest run",
     "test:watch": "vitest",

packages/cli/playwright.config.ts

@@ -1,11 +1,19 @@
 import { defineConfig, devices } from '@playwright/test'
 
+const workers = process.env.PLAYWRIGHT_WORKERS
+  ? Number.isNaN(Number(process.env.PLAYWRIGHT_WORKERS))
+    ? process.env.PLAYWRIGHT_WORKERS
+    : Number(process.env.PLAYWRIGHT_WORKERS)
+  : process.env.CI
+    ? 4
+    : '50%'
+
 export default defineConfig({
   testDir: './tests-e2e',
-  fullyParallel: false,
+  fullyParallel: true,
   forbidOnly: !!process.env.CI,
   retries: process.env.CI ? 1 : 0,
-  workers: 1,
+  workers,
   timeout: 10 * 60 * 1000,
   expect: {
     timeout: 20 * 1000,
@@ -14,8 +22,9 @@ export default defineConfig({
   use: {
     ...devices['Desktop Chrome'],
     channel: 'chrome',
+    serviceWorkers: 'block',
     trace: 'on-first-retry',
     screenshot: 'only-on-failure',
-    video: 'retain-on-failure',
+    video: 'off',
   },
 })

packages/cli/tests-e2e/addons-smoke.spec.ts

@@ -1,6 +1,6 @@
 import { expect, test } from '@playwright/test'
 
-import { attachRuntimeGuards, createReactAppFixture } from './helpers'
+import { attachRuntimeGuards, createReactAppFixture, optimizePageForFastE2E } from './helpers'
 
 test('@blocking creates app with multiple add-ons and renders demo routes', async ({ page }) => {
   const fixture = await createReactAppFixture({
@@ -10,6 +10,7 @@ test('@blocking creates app with multiple add-ons and renders demo routes', asyn
   const guards = attachRuntimeGuards(page, fixture.url)
 
   try {
+    await optimizePageForFastE2E(page)
     await page.goto(`${fixture.url}/demo/form/simple`)
     await expect(page.getByText('Title', { exact: true })).toBeVisible()
     await expect(page.getByRole('button', { name: 'Submit' })).toBeVisible()

packages/cli/tests-e2e/create-smoke.spec.ts

@@ -1,14 +1,16 @@
 import { expect, test } from '@playwright/test'
 
-import { attachRuntimeGuards, createReactAppFixture } from './helpers'
+import { attachRuntimeGuards, createReactAppFixture, optimizePageForFastE2E } from './helpers'
 
 test('@blocking creates a React app and navigates core starter routes', async ({ page }) => {
   const fixture = await createReactAppFixture({
     appName: 'react-smoke-app',
+    runQualityGatesChecks: true,
   })
   const guards = attachRuntimeGuards(page, fixture.url)
 
   try {
+    await optimizePageForFastE2E(page)
     await page.goto(fixture.url)
     await expect(
       page.getByRole('heading', {

packages/cli/tests-e2e/helpers.ts

@@ -28,6 +28,7 @@ type CreateAppFixtureOptions = {
   addOns?: Array<string>
   postCreateAddOns?: Array<string>
   skipDevServer?: boolean
+  runQualityGatesChecks?: boolean
 }
 
 export type RuntimeGuards = {
@@ -291,18 +292,68 @@ async function runQualityGates(
   })
 }
 
+function envBoolean(name: string, defaultValue: boolean) {
+  const value = process.env[name]
+  if (value == null) {
+    return defaultValue
+  }
+
+  const normalized = value.trim().toLowerCase()
+  return normalized === '1' || normalized === 'true' || normalized === 'yes'
+}
+
+const IMAGE_GLOB = '**/*.{png,jpg,jpeg,gif,webp,avif,ico}'
+const FONT_GLOB = '**/*.{woff,woff2,ttf,otf,eot}'
+const MEDIA_GLOB = '**/*.{mp4,webm,mp3,wav,ogg,mov,m4a}'
+const CSS_GLOB = '**/*.css'
+
+const abortRoute = (route: { abort: (reason?: string) => Promise<void> }) =>
+  route.abort('blockedbyclient')
+
+export async function optimizePageForFastE2E(page: Page) {
+  const blockAssets = envBoolean('E2E_BLOCK_NON_ESSENTIAL', true)
+  if (!blockAssets) {
+    return
+  }
+
+  await page.route(IMAGE_GLOB, abortRoute)
+  await page.route(FONT_GLOB, abortRoute)
+  await page.route(MEDIA_GLOB, abortRoute)
+
+  if (envBoolean('E2E_BLOCK_CSS', false)) {
+    await page.route(CSS_GLOB, abortRoute)
+  }
+}
+
 export async function createAppFixture(
   options: CreateAppFixtureOptions,
 ): Promise<E2EApp> {
   await access(cliDistPath)
 
+  const timings: Array<[string, number]> = []
+  const now = () => performance.now()
+  const start = now()
+  const mark = (label: string, startedAt: number) => {
+    timings.push([label, now() - startedAt])
+  }
+  const logTimings = () => {
+    if (!envBoolean('E2E_TIMINGS', true)) {
+      return
+    }
+
+    const totalMs = now() - start
+    const summary = timings.map(([label, ms]) => `${label}=${Math.round(ms)}ms`).join(' | ')
+    console.log(`[e2e:timing] ${options.appName} :: ${summary} | total=${Math.round(totalMs)}ms`)
+  }
+
   const rootDir = await mkdtemp(join(tmpdir(), 'tanstack-cli-e2e-'))
   const {
     appName,
     template,
     addOns,
     postCreateAddOns,
     skipDevServer,
+    runQualityGatesChecks = false,
     framework = 'react',
     packageManager = 'pnpm',
     routerOnly = false,
@@ -332,6 +383,7 @@ export async function createAppFixture(
     createArgs.push('--add-ons', addOns.join(','))
   }
 
+  const createStartedAt = now()
   await runCommand(
     'node',
     createArgs,
@@ -342,23 +394,32 @@ export async function createAppFixture(
       },
     },
   )
+  mark('create', createStartedAt)
 
   await patchViteConfigForE2E(appDir)
 
   if (postCreateAddOns?.length) {
+    const postAddOnsStartedAt = now()
     await runCommand('node', [cliDistPath, 'add', ...postCreateAddOns], {
       cwd: appDir,
       env: {
         CI: '1',
       },
     })
 
+    mark('post-add-ons', postAddOnsStartedAt)
+
     await patchViteConfigForE2E(appDir)
   }
 
-  await runQualityGates(appDir, packageManager)
+  if (runQualityGatesChecks) {
+    const qualityGatesStartedAt = now()
+    await runQualityGates(appDir, packageManager)
+    mark('quality-gates', qualityGatesStartedAt)
+  }
 
   if (skipDevServer) {
+    logTimings()
     return {
       rootDir,
       appDir,
@@ -374,6 +435,7 @@ export async function createAppFixture(
 
   const dev = getPackageManagerCommandForScript(packageManager, 'dev')
 
+  const devServerStartedAt = now()
   const server = spawn(dev.command, dev.args, {
     cwd: appDir,
     env: {
@@ -398,13 +460,16 @@ export async function createAppFixture(
   try {
     url = await waitForDevServerURL(() => serverStdout)
     await waitForServer(url)
+    mark('dev-server-ready', devServerStartedAt)
   } catch (error) {
     await stopChild(server)
     throw new Error(
       `Failed to start app server at ${url}\nstdout:\n${serverStdout}\n\nstderr:\n${serverStderr}\n\n${error}`,
     )
   }
 
+  logTimings()
+
   return {
     rootDir,
     appDir,
@@ -429,6 +494,10 @@ function toSameOrigin(url: string, appOrigin: URL) {
   }
 }
 
+function isBlockedByClientError(value: string) {
+  return value.includes('ERR_BLOCKED_BY_CLIENT')
+}
+
 export function attachRuntimeGuards(page: Page, appUrl: string): RuntimeGuards {
   const appOrigin = new URL(appUrl)
   const pageErrors: Array<string> = []
@@ -442,7 +511,11 @@ export function attachRuntimeGuards(page: Page, appUrl: string): RuntimeGuards {
 
   const onConsole = (message: { type: () => string; text: () => string }) => {
     if (message.type() === 'error') {
-      consoleErrors.push(message.text())
+      const text = message.text()
+      if (isBlockedByClientError(text)) {
+        return
+      }
+      consoleErrors.push(text)
     }
   }
 
@@ -457,7 +530,7 @@ export function attachRuntimeGuards(page: Page, appUrl: string): RuntimeGuards {
     }
     const errorText = request.failure()?.errorText || 'unknown error'
 
-    if (errorText.includes('ERR_ABORTED')) {
+    if (errorText.includes('ERR_ABORTED') || isBlockedByClientError(errorText)) {
       return
     }
 

packages/cli/tests-e2e/matrix-opportunistic.spec.ts

@@ -5,6 +5,7 @@ import {
   attachRuntimeGuards,
   createAppFixture,
   getRepoPath,
+  optimizePageForFastE2E,
   type E2EApp,
 } from './helpers'
 
@@ -19,16 +20,16 @@ type MatrixScenario = {
   assert: (fixture: E2EApp, page: Page) => Promise<void>
 }
 
+const reactStarterHeading = /Island hours, but for product teams\.|Start simple, ship quickly\./
+
 const scenarios: Array<MatrixScenario> = [
   {
     id: 'react-base-pnpm',
     framework: 'react',
     packageManager: 'pnpm',
     visits: ['/'],
     assert: async (_, page) => {
-      await expect(
-        page.getByRole('heading', { name: 'Island hours, but for product teams.' }),
-      ).toBeVisible()
+      await expect(page.getByRole('heading', { name: reactStarterHeading })).toBeVisible()
     },
   },
   {
@@ -37,9 +38,7 @@ const scenarios: Array<MatrixScenario> = [
     packageManager: 'npm',
     visits: ['/'],
     assert: async (_, page) => {
-      await expect(
-        page.getByRole('heading', { name: 'Island hours, but for product teams.' }),
-      ).toBeVisible()
+      await expect(page.getByRole('heading', { name: reactStarterHeading })).toBeVisible()
     },
   },
   {
@@ -97,9 +96,7 @@ const scenarios: Array<MatrixScenario> = [
     addOns: ['biome', 'netlify'],
     visits: ['/'],
     assert: async (_, page) => {
-      await expect(
-        page.getByRole('heading', { name: 'Island hours, but for product teams.' }),
-      ).toBeVisible()
+      await expect(page.getByRole('heading', { name: reactStarterHeading })).toBeVisible()
     },
   },
 ]
@@ -123,6 +120,7 @@ test.describe('@matrix opportunistic matrix', () => {
       const guards = attachRuntimeGuards(page, fixture.url)
 
       try {
+        await optimizePageForFastE2E(page)
         for (const visit of scenario.visits || ['/']) {
           await page.goto(`${fixture.url}${visit}`)
           await expect(page.locator('body')).toBeVisible()