chore(test): wrap tests in workflow

.github/workflows/ci.yml

@@ -206,8 +206,9 @@ jobs:
 
       - name: Run Tests (Node)
         if: matrix.node != 'bun'
-        run: pnpm run test
+        run: pnpm -F @temporalio/test run test:ci
         env:
+          TEST_MAX_RETRIES: '3'
           RUN_INTEGRATION_TESTS: true
           REUSE_V8_CONTEXT: ${{ matrix.reuse-v8-context }}
 
@@ -230,8 +231,9 @@ jobs:
 
       - name: Run Tests (Bun)
         if: matrix.node == 'bun'
-        run: bun run -b --cwd packages/test ava ./lib/test-*.js
+        run: bun run -b packages/test/lib/ci/run.js
         env:
+          TEST_MAX_RETRIES: '3'
           RUN_INTEGRATION_TESTS: true
           REUSE_V8_CONTEXT: ${{ matrix.reuse-v8-context }}
 

packages/test/package.json

@@ -9,7 +9,9 @@
     "build:protos": "tsx ./scripts/compile-proto.ts",
     "test": "ava ./lib/test-*.js",
     "test:watch": "ava --watch ./lib/test-*.js",
-    "test:bun": "bun run -b ava ./lib/test-*.js"
+    "test:bun": "bun run -b ava ./lib/test-*.js",
+    "test:ci": "node ./lib/ci/run.js",
+    "test:ci:discover": "node -e \"require('./lib/ci/activities').discoverTests().then(f => f.forEach(t => console.log(t)))\""
   },
   "ava": {
     "timeout": "60s",

packages/test/src/ci/activities.ts

@@ -0,0 +1,89 @@
+import { spawn } from 'node:child_process';
+import fs from 'node:fs';
+import path from 'node:path';
+import { glob } from 'glob';
+import { Context } from '@temporalio/activity';
+import { parseTapOutput } from './tap-parser';
+import type { TestFile, TestBatchResult, FlakyTest } from './types';
+
+const TEST_PKG_DIR = path.resolve(__dirname, '../..');
+
+export async function discoverTests(): Promise<TestFile[]> {
+  const pattern = './lib/test-*.js';
+  const files = await glob(pattern, { cwd: TEST_PKG_DIR });
+  console.log(`Discovered ${files.length} test files`);
+  return files.sort();
+}
+
+export async function runTests(files: TestFile[], env?: Record<string, string>): Promise<TestBatchResult> {
+  const ctx = Context.current();
+
+  console.log(`Running ${files.length} test file(s)`);
+
+  const avaPath = path.resolve(TEST_PKG_DIR, 'node_modules/.bin/ava');
+  const args = ['--tap', '--timeout', '60s', '--concurrency', '1', '--no-worker-threads', ...files];
+
+  const batchResult = await new Promise<TestBatchResult>((resolve, reject) => {
+    const child = spawn(avaPath, args, {
+      cwd: TEST_PKG_DIR,
+      env: {
+        ...process.env,
+        ...env,
+        FORCE_COLOR: '0',
+      },
+      stdio: ['ignore', 'pipe', 'pipe'],
+      timeout: 15 * 60 * 1000,
+      // On Windows, .bin entries are .cmd files that need a shell to execute
+      shell: process.platform === 'win32',
+    });
+
+    const stdoutChunks: Buffer[] = [];
+
+    child.stdout.on('data', (chunk: Buffer) => {
+      stdoutChunks.push(chunk);
+      const text = chunk.toString();
+      for (const line of text.split('\n')) {
+        const trimmed = line.trim();
+        if (trimmed.startsWith('ok ') || trimmed.startsWith('not ok ')) {
+          console.log(trimmed);
+          // Heartbeat on every assertion so the server knows we're alive
+          ctx.heartbeat();
+        }
+      }
+    });
+
+    child.stderr.on('data', (chunk: Buffer) => {
+      process.stderr.write(chunk);
+    });
+
+    child.on('error', reject);
+    child.on('close', (code) => {
+      const stdout = Buffer.concat(stdoutChunks).toString();
+      if (code !== null && code !== 0 && stdout.length === 0) {
+        reject(new Error(`AVA exited with code ${code} and no output`));
+      } else {
+        resolve(parseTapOutput(stdout, files));
+      }
+    });
+  });
+
+  console.log(`Finished: ${batchResult.passed.length} passed, ${batchResult.failed.length} failed`);
+  return batchResult;
+}
+
+export async function alertFlakes(flakes: FlakyTest[]): Promise<void> {
+  if (flakes.length === 0) return;
+
+  const lines = ['## Flaky Tests Detected', ''];
+  for (const flake of flakes) {
+    lines.push(`- **${flake.file}** passed on attempt ${flake.attemptsToPass}`);
+  }
+  const summary = lines.join('\n');
+
+  console.log(summary);
+
+  const summaryPath = process.env.GITHUB_STEP_SUMMARY;
+  if (summaryPath) {
+    await fs.promises.appendFile(summaryPath, summary + '\n');
+  }
+}

packages/test/src/ci/run.ts

@@ -0,0 +1,113 @@
+import { Connection, WorkflowClient, WorkflowFailedError } from '@temporalio/client';
+import { NativeConnection, Worker } from '@temporalio/worker';
+import * as activities from './activities';
+import { testSuiteWorkflow } from './workflows';
+import type { TestSuiteInput, TestSuiteResult } from './types';
+
+const TEMPORAL_ADDRESS = process.env.TEMPORAL_ADDRESS ?? 'localhost:7233';
+
+// Env vars to forward to AVA child processes
+const FORWARDED_ENV_VARS = [
+  'RUN_INTEGRATION_TESTS',
+  'REUSE_V8_CONTEXT',
+  'TEMPORAL_CLOUD_MTLS_TEST_TARGET_HOST',
+  'TEMPORAL_CLOUD_MTLS_TEST_NAMESPACE',
+  'TEMPORAL_CLOUD_MTLS_TEST_CLIENT_CERT',
+  'TEMPORAL_CLOUD_MTLS_TEST_CLIENT_KEY',
+  'TEMPORAL_CLOUD_API_KEY_TEST_TARGET_HOST',
+  'TEMPORAL_CLOUD_API_KEY_TEST_NAMESPACE',
+  'TEMPORAL_CLOUD_API_KEY_TEST_API_KEY',
+  'TEMPORAL_CLOUD_OPS_TEST_TARGET_HOST',
+  'TEMPORAL_CLOUD_OPS_TEST_NAMESPACE',
+  'TEMPORAL_CLOUD_OPS_TEST_API_KEY',
+  'TEMPORAL_CLOUD_OPS_TEST_API_VERSION',
+];
+
+function collectEnv(): Record<string, string> {
+  return Object.fromEntries(
+    FORWARDED_ENV_VARS.filter((k) => process.env[k] !== undefined).map((k) => [k, process.env[k]!])
+  );
+}
+
+function printResults(result: TestSuiteResult): void {
+  console.log('\n=== Test Suite Results ===');
+  console.log(`Total files: ${result.totalFiles}`);
+  console.log(`Passed: ${result.passed.length}`);
+  console.log(`Failed: ${result.failed.length}`);
+  console.log(`Retries used: ${result.retriesUsed}`);
+
+  if (result.flakes.length > 0) {
+    console.log(`\nFlaky tests (${result.flakes.length}):`);
+    for (const flake of result.flakes) {
+      console.log(`  - ${flake.file} (passed on attempt ${flake.attemptsToPass})`);
+    }
+  }
+
+  if (result.failed.length > 0) {
+    console.log(`\nFailed tests:`);
+    for (const file of result.failed) {
+      console.log(`  - ${file}`);
+    }
+  }
+}
+
+function printError(err: unknown): void {
+  console.error('\n=== Test Suite Failed ===');
+
+  if (err instanceof WorkflowFailedError && err.cause) {
+    console.error(`Reason: ${err.cause.message}`);
+    if (err.cause.cause instanceof Error) {
+      console.error(`Root cause: ${err.cause.cause.message}`);
+    }
+  } else if (err instanceof Error) {
+    console.error(`Error: ${err.message}`);
+  } else {
+    console.error(err);
+  }
+}
+
+async function main() {
+  const maxRetries = process.env.TEST_MAX_RETRIES ? parseInt(process.env.TEST_MAX_RETRIES, 10) : 3;
+
+  const input: TestSuiteInput = {
+    maxRetries,
+    env: collectEnv(),
+  };
+
+  const nativeConnection = await NativeConnection.connect({ address: TEMPORAL_ADDRESS });
+  const worker = await Worker.create({
+    connection: nativeConnection,
+    taskQueue: 'test-suite',
+    workflowsPath: require.resolve('./workflows'),
+    activities,
+    maxConcurrentActivityTaskExecutions: 1,
+  });
+
+  const connection = await Connection.connect({ address: TEMPORAL_ADDRESS });
+  const client = new WorkflowClient({ connection });
+
+  console.log(`Starting test suite workflow (maxRetries=${maxRetries})...`);
+
+  try {
+    const result = await worker.runUntil(
+      client.execute(testSuiteWorkflow, {
+        workflowId: `test-suite-${Date.now()}`,
+        taskQueue: 'test-suite',
+        args: [input],
+        workflowExecutionTimeout: '30m',
+      })
+    );
+    printResults(result);
+  } catch (err) {
+    printError(err);
+    process.exit(1);
+  }
+}
+
+main().then(
+  () => void process.exit(0),
+  (err) => {
+    console.error(err);
+    process.exit(1);
+  }
+);

packages/test/src/ci/tap-parser.ts

@@ -0,0 +1,122 @@
+import type { TestBatchResult, TestFile } from './types';
+
+// AVA uses `figures.pointerSmall` as the separator between file and test title.
+// The character varies by platform: › (U+203A) on Unix, » (U+00BB) on Windows.
+const SEPARATOR_RE = / [\u203a\u00bb] /;
+
+type Directive = 'skip' | 'todo' | undefined;
+
+interface AssertLine {
+  ok: boolean;
+  // undefined when AVA omits the file prefix (single-file runs)
+  file: string | undefined;
+  title: string;
+  directive: Directive;
+}
+
+// TAP directives appear as "# SKIP" or "# TODO" (case-insensitive) at end of line
+const DIRECTIVE_RE = /\s+#\s+(SKIP|TODO)(.*)$/i;
+
+function parseAssertLine(line: string): AssertLine | undefined {
+  const match = line.match(/^(not ok|ok)\s+\d+\s+-\s+(.+)$/);
+  if (!match) return undefined;
+
+  const ok = match[1] === 'ok';
+  let fullName = match[2]!;
+
+  let directive: Directive;
+  const directiveMatch = fullName.match(DIRECTIVE_RE);
+  if (directiveMatch) {
+    directive = directiveMatch[1]!.toLowerCase() as 'skip' | 'todo';
+    fullName = fullName.slice(0, directiveMatch.index);
+  }
+
+  const sepMatch = fullName.match(SEPARATOR_RE);
+  if (!sepMatch) {
+    // AVA omits the file prefix when running a single file
+    return { ok, file: undefined, title: fullName, directive };
+  }
+  return {
+    ok,
+    file: fullName.slice(0, sepMatch.index),
+    title: fullName.slice(sepMatch.index! + sepMatch[0].length),
+    directive,
+  };
+}
+
+export function parseTapOutput(output: string, expectedFiles: TestFile[]): TestBatchResult {
+  const fileResults = new Map<TestFile, { seen: boolean; hasFailed: boolean; failedTitles: string[] }>();
+
+  for (const file of expectedFiles) {
+    fileResults.set(file, { seen: false, hasFailed: false, failedTitles: [] });
+  }
+
+  for (const line of output.split('\n')) {
+    const assert = parseAssertLine(line.trim());
+    if (!assert) continue;
+
+    // When AVA runs a single file it omits the file prefix;
+    // attribute all assertions to the sole expected file.
+    const file =
+      assert.file !== undefined
+        ? resolveFile(assert.file, expectedFiles)
+        : expectedFiles.length === 1
+          ? expectedFiles[0]
+          : undefined;
+    if (!file) continue;
+
+    let result = fileResults.get(file);
+    if (!result) {
+      result = { seen: false, hasFailed: false, failedTitles: [] };
+      fileResults.set(file, result);
+    }
+
+    result.seen = true;
+    // skip, todo, and "no tests found" assertions are not real failures
+    if (!assert.ok && !assert.directive && !isNoTestsFound(assert.title)) {
+      result.hasFailed = true;
+      if (assert.title) {
+        result.failedTitles.push(assert.title);
+      }
+    }
+  }
+
+  const passed: TestFile[] = [];
+  const failed: TestFile[] = [];
+  const failureDetails: Record<TestFile, string[]> = {};
+
+  for (const [file, result] of fileResults) {
+    // Files with no output are treated as failures (crashed before tests ran)
+    if (result.seen && !result.hasFailed) {
+      passed.push(file);
+    } else {
+      failed.push(file);
+      failureDetails[file] = result.failedTitles;
+    }
+  }
+
+  return { passed, failed, failureDetails };
+}
+
+// AVA emits "No tests found in $FILE" when all tests in a file are
+// conditionally skipped at registration time (e.g. platform guards).
+function isNoTestsFound(title: string): boolean {
+  return title.startsWith('No tests found in ');
+}
+
+function resolveFile(tapName: string, expectedFiles: TestFile[]): TestFile | undefined {
+  for (const file of expectedFiles) {
+    // AVA strips the directory, the "test-" prefix, and the extension in TAP output.
+    // e.g. "lib/test-time.js" becomes "time", "lib/test-enums-helpers.js" becomes "enums-helpers"
+    const shortName = file.replace(/^.*[/\\]test-/, '').replace(/\.js$/, '');
+    if (shortName === tapName) {
+      return file;
+    }
+    // Also try full path without extension and exact match
+    const withoutExt = file.replace(/\.js$/, '');
+    if (withoutExt === tapName || file === tapName) {
+      return file;
+    }
+  }
+  return undefined;
+}

packages/test/src/ci/types.ts

@@ -0,0 +1,26 @@
+export type TestFile = string;
+
+export interface TestBatchResult {
+  passed: TestFile[];
+  failed: TestFile[];
+  failureDetails: Record<TestFile, string[]>;
+}
+
+export interface FlakyTest {
+  file: TestFile;
+  attemptsToPass: number;
+}
+
+export interface TestSuiteResult {
+  totalFiles: number;
+  passed: TestFile[];
+  failed: TestFile[];
+  flakes: FlakyTest[];
+  retriesUsed: number;
+}
+
+export interface TestSuiteInput {
+  maxRetries?: number;
+  files?: TestFile[];
+  env?: Record<string, string>;
+}