Add CI timeouts and diagnostics

Author: kvz

.github/workflows/ci.yml

@@ -80,6 +80,7 @@ jobs:
       github.event_name != 'pull_request' ||
       github.event.pull_request.head.repo.full_name == github.repository
     runs-on: ubuntu-latest
+    timeout-minutes: 20
     steps:
       - uses: actions/checkout@v4
         with:
@@ -117,8 +118,17 @@ jobs:
           R2_HOST: ${{ secrets.R2_HOST }}
           R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}
           R2_PUBLIC_URL: ${{ secrets.R2_PUBLIC_URL }}
-      - run: yarn exec playwright install --with-deps chromium
-      - run: yarn verify:local
+      - name: Install Playwright Chromium
+        run: |
+          set -euo pipefail
+          echo "::group::Browser test diagnostics"
+          node --version
+          yarn --version
+          df -h
+          echo "::endgroup::"
+          timeout --foreground 10m yarn exec playwright install --with-deps chromium
+      - name: Run local browser verification
+        run: timeout --foreground 12m yarn verify:local
         env:
           PLAYWRIGHT_SKIP_INSTALL: "1"
           TRANSLOADIT_KEY: ${{ secrets.TRANSLOADIT_KEY }}
@@ -137,6 +147,7 @@ jobs:
       github.event_name != 'pull_request' ||
       github.event.pull_request.head.repo.full_name == github.repository
     runs-on: ubuntu-latest
+    timeout-minutes: 25
     steps:
       - uses: actions/checkout@v4
         with:
@@ -154,7 +165,7 @@ jobs:
       - name: Deploy Convex preview (PR)
         if: github.event_name == 'pull_request'
         run: |
-          node scripts/deploy-cloud.ts >> "$GITHUB_ENV"
+          timeout --foreground 12m node scripts/deploy-cloud.ts >> "$GITHUB_ENV"
         env:
           CI_OUTPUT: "1"
           CONVEX_PREVIEW_NAME: ${{ steps.preview_name.outputs.name }}
@@ -171,7 +182,7 @@ jobs:
       - name: Deploy Convex stable (main)
         if: github.ref == 'refs/heads/main'
         run: |
-          node scripts/deploy-cloud.ts >> "$GITHUB_ENV"
+          timeout --foreground 12m node scripts/deploy-cloud.ts >> "$GITHUB_ENV"
         env:
           CI_OUTPUT: "1"
           CONVEX_DEPLOY_KEY: ${{ secrets.CONVEX_DEPLOY_KEY }}
@@ -197,7 +208,9 @@ jobs:
       - name: Resolve Vercel preview URL
         if: env.E2E_REMOTE_APP_URL == ''
         run: |
-          echo "E2E_REMOTE_APP_URL=$(node scripts/resolve-vercel-preview.ts)" >> "$GITHUB_ENV"
+          preview_url="$(timeout --foreground 7m node scripts/resolve-vercel-preview.ts)"
+          echo "Resolved Vercel preview URL: ${preview_url}"
+          echo "E2E_REMOTE_APP_URL=${preview_url}" >> "$GITHUB_ENV"
         env:
           VERCEL_PREVIEW_DEPLOY_HOOK: ${{ secrets.VERCEL_PREVIEW_DEPLOY_HOOK }}
           VERCEL_PROJECT_SLUG: convex
@@ -234,7 +247,8 @@ jobs:
           VERCEL_PROTECTION_BYPASS: ${{ secrets.VERCEL_PROTECTION_BYPASS }}
           CONVEX_DEPLOY_KEY: ${{ secrets.CONVEX_DEPLOY_KEY }}
           CONVEX_PREVIEW_DEPLOY_KEY: ${{ secrets.CONVEX_PREVIEW_DEPLOY_KEY }}
-      - run: yarn verify:cloud
+      - name: Run cloud verification
+        run: timeout --foreground 12m yarn verify:cloud
         env:
           E2E_REMOTE_APP_URL: ${{ env.E2E_REMOTE_APP_URL }}
           E2E_REMOTE_CONVEX_URL: ${{ env.E2E_REMOTE_CONVEX_URL }}

scripts/deploy-cloud.ts

@@ -20,6 +20,7 @@ const log = (...args: Parameters<typeof console.log>) => {
 const runStdio = ciOutput ? 'pipe' : 'inherit'
 
 const rootDir = resolve(fileURLToPath(new URL('..', import.meta.url)))
+const minutes = (value: number) => value * 60 * 1000
 const sleep = (ms: number) =>
   new Promise((resolve) => {
     setTimeout(resolve, ms)
@@ -40,18 +41,26 @@ const deployCloud = async () => {
     await mkdir(projectDir, { recursive: true })
 
     log(`Packing @transloadit/convex into ${tgzPath}...`)
-    run('yarn', ['pack', '-o', tgzPath], { cwd: rootDir, stdio: runStdio })
+    run('yarn', ['pack', '-o', tgzPath], {
+      cwd: rootDir,
+      stdio: runStdio,
+      timeoutMs: minutes(2),
+    })
 
     await writeAppFiles({ projectDir, tgzPath })
 
     log('Installing dependencies...')
     run('npm', ['install', '--no-fund', '--no-audit'], {
       cwd: projectDir,
       stdio: runStdio,
+      timeoutMs: minutes(5),
     })
 
     log('Deploying Convex app...')
     const previewName = process.env.CONVEX_PREVIEW_NAME
+    if (previewName) {
+      log(`Preview deployment name: ${previewName}`)
+    }
     const deployArgs = ['convex', 'deploy', '--typecheck', 'disable', '--yes']
     if (previewName) {
       deployArgs.push('--preview-create', previewName)
@@ -64,6 +73,7 @@ const deployCloud = async () => {
         CONVEX_DEPLOY_KEY: requireEnv('CONVEX_DEPLOY_KEY'),
       },
       stdio: 'pipe',
+      timeoutMs: minutes(8),
     })
 
     const { deploymentName, deploymentUrl } = parseDeployOutput(deployOutput)
@@ -89,6 +99,7 @@ const deployCloud = async () => {
             env: deployEnv,
             stdio: runStdio === 'inherit' ? 'pipe' : runStdio,
             input: value,
+            timeoutMs: minutes(2),
           })
           return
         } catch (error) {

scripts/qa/run.ts

@@ -5,6 +5,7 @@ export type RunOptions = {
   env?: NodeJS.ProcessEnv
   stdio?: 'inherit' | 'pipe'
   input?: string
+  timeoutMs?: number
 }
 
 export const requireEnv = (name: string) => {
@@ -17,20 +18,42 @@ export const requireEnv = (name: string) => {
 
 export const run = (command: string, args: string[], options: RunOptions = {}) => {
   const stdio = options.input !== undefined ? 'pipe' : (options.stdio ?? 'inherit')
+  if (options.timeoutMs) {
+    console.error(
+      `Running with timeout (${Math.round(options.timeoutMs / 1000)}s): ${command} ${args.join(
+        ' ',
+      )}`,
+    )
+  }
   const result = spawnSync(command, args, {
     cwd: options.cwd,
     env: options.env,
     stdio,
     input: options.input,
     encoding: 'utf8',
+    timeout: options.timeoutMs,
+    killSignal: 'SIGTERM',
   })
 
+  const output = `${result.stdout ?? ''}${result.stderr ?? ''}`
+
+  if (result.error) {
+    throw new Error(`Command failed: ${command} ${args.join(' ')}
+${result.error.message}
+${output}`)
+  }
+
+  if (result.signal) {
+    throw new Error(`Command failed: ${command} ${args.join(' ')}
+Exited via signal ${result.signal}${options.timeoutMs ? ` after ${options.timeoutMs}ms` : ''}
+${output}`)
+  }
+
   if (result.status !== 0) {
-    const output = `${result.stdout ?? ''}${result.stderr ?? ''}`
     throw new Error(`Command failed: ${command} ${args.join(' ')}\n${output}`)
   }
 
-  return `${result.stdout ?? ''}${result.stderr ?? ''}`
+  return output
 }
 
 export const parseJson = <T>(output: string): T => {

scripts/resolve-vercel-preview.ts

@@ -12,6 +12,9 @@ const githubRepo = process.env.GITHUB_REPOSITORY ?? ''
 const githubSha = process.env.GITHUB_SHA ?? ''
 const githubEventPath = process.env.GITHUB_EVENT_PATH ?? ''
 const githubHeadRef = process.env.GITHUB_HEAD_REF ?? process.env.GITHUB_REF_NAME ?? ''
+const log = (...args: Parameters<typeof console.error>) => {
+  console.error('[resolve-vercel-preview]', ...args)
+}
 
 if (!githubToken) {
   throw new Error('Missing GITHUB_TOKEN')
@@ -29,6 +32,7 @@ const headers = {
 
 const triggerPreviewDeploy = async () => {
   if (!deployHook) return
+  log('Triggering Vercel preview deploy hook')
   const response = await fetch(deployHook, { method: 'POST' })
   if (!response.ok) {
     throw new Error(`Vercel deploy hook failed: ${response.status}`)
@@ -61,7 +65,10 @@ const fetchDeploymentUrl = async (): Promise<string | null> => {
       target_url?: string
     }>
     const success = statuses.find((status) => status.state === 'success')
-    if (success?.target_url) return success.target_url
+    if (success?.target_url) {
+      log('Found successful GitHub deployment URL')
+      return success.target_url
+    }
   }
 
   return null
@@ -86,7 +93,11 @@ const fetchCheckRunUrl = async (): Promise<string | null> => {
   const vercelCheck = checks.find(
     (check) => check.app?.slug === 'vercel' && check.conclusion === 'success' && check.details_url,
   )
-  return vercelCheck?.details_url ?? null
+  if (vercelCheck?.details_url) {
+    log('Found successful Vercel check-run URL')
+    return vercelCheck.details_url
+  }
+  return null
 }
 
 const fetchPreviewUrlFromComments = async (): Promise<string | null> => {
@@ -115,6 +126,9 @@ const fetchPreviewUrlFromComments = async (): Promise<string | null> => {
       projects?: Array<{ previewUrl?: string }>
     }
     const previewUrl = payload.projects?.find((project) => project.previewUrl)?.previewUrl
+    if (previewUrl) {
+      log('Found Vercel preview URL in PR comments')
+    }
     return previewUrl ?? null
   } catch {
     return null
@@ -161,7 +175,9 @@ const isPreviewReady = async (url: string) => {
 await triggerPreviewDeploy()
 
 const deadline = Date.now() + 6 * 60 * 1000
+let attempts = 0
 while (Date.now() < deadline) {
+  attempts += 1
   const deploymentUrl = await fetchDeploymentUrl()
   if (deploymentUrl) {
     process.stdout.write(normalizeUrl(deploymentUrl))
@@ -183,11 +199,16 @@ while (Date.now() < deadline) {
   if (fallbackUrl) {
     const ready = await isPreviewReady(fallbackUrl)
     if (ready) {
+      log('Using ready fallback Vercel preview URL')
       process.stdout.write(normalizeUrl(fallbackUrl))
       process.exit(0)
     }
   }
 
+  if (attempts === 1 || attempts % 6 === 0) {
+    log(`Still waiting for preview URL after ${attempts} checks`)
+  }
+
   await sleep(5000)
 }
 

scripts/verify.ts

@@ -21,6 +21,7 @@ const parseArgs = (args: string[]) => {
 }
 
 const logger = createDebugLogger({ namespace: 'verify' })
+const minutes = (value: number) => value * 60 * 1000
 
 const runBrowser = async (options: {
   mode: Mode
@@ -33,11 +34,11 @@ const runBrowser = async (options: {
 
   if (!skipInstall) {
     logger.event('playwright-install', { browser: 'chromium' })
-    run('yarn', ['exec', 'playwright', 'install', 'chromium'])
+    run('yarn', ['exec', 'playwright', 'install', 'chromium'], { timeoutMs: minutes(5) })
   }
 
   logger.event('build')
-  run('yarn', ['build'])
+  run('yarn', ['build'], { timeoutMs: minutes(5) })
 
   const testEnv: NodeJS.ProcessEnv = {
     ...process.env,
@@ -53,6 +54,7 @@ const runBrowser = async (options: {
 
   run('yarn', ['exec', 'vitest', 'run', '--config', 'vitest.e2e.config.ts'], {
     env: testEnv,
+    timeoutMs: minutes(8),
   })
 }