feat(test): GitHub-native CI output for "test run --all" (--gh-output, --summary-file)

src/commands/test.ts

@@ -1,9 +1,11 @@
 import {
+  appendFileSync,
   createWriteStream,
   existsSync,
   readFileSync,
   readdirSync,
   statSync,
+  writeFileSync,
   type WriteStream,
 } from 'node:fs';
 import { rename, stat, unlink } from 'node:fs/promises';
@@ -92,6 +94,7 @@ import {
 import { createTicker } from '../lib/ticker.js';
 import { RateThrottle } from '../lib/rate-throttle.js';
 import { resolvePortalBase, resolvePortalUrl } from '../lib/facade.js';
+import { emitGithubOutputs, summarizeAcceptedPayload } from '../lib/gh-output.js';
 import { loadConfig } from '../lib/config.js';
 import {
   flakyExitCode,
@@ -6363,6 +6366,10 @@ interface RunTestRunAllOptions extends CommonOptions {
   reportFile?: string;
   /** --report-suite-name: optional override for the JUnit <testsuite name=...>. */
   reportSuiteName?: string;
+  /** --gh-output: force the GitHub-native output layer even off-Actions (issue #99). */
+  ghOutput?: boolean;
+  /** --summary-file: also write the reduced machine summary JSON to this path. */
+  summaryFile?: string;
 }
 
 async function writeBatchJUnitReportIfRequested(
@@ -6927,6 +6934,36 @@ export async function runTestRunAll(
   };
   await writeBatchJUnitReportIfRequested(opts, freshRunResults);
   out.print(jsonPayload);
+  // CI-native output layer (issue #99): emitted before the gate throws below so
+  // the artifacts land even when the batch exits non-zero. The summary file is a
+  // machine artifact written regardless of --output mode; stdout stays owned by
+  // the envelope above (plus Actions workflow commands, which Actions parses).
+  {
+    const env = deps.env ?? process.env;
+    const ghEnabled = opts.ghOutput === true || env.GITHUB_ACTIONS === 'true';
+    if (ghEnabled || opts.summaryFile !== undefined) {
+      const ciSummary = summarizeAcceptedPayload(JSON.stringify(jsonPayload));
+      if (opts.summaryFile !== undefined) {
+        try {
+          writeFileSync(opts.summaryFile, `${JSON.stringify(ciSummary, null, 2)}\n`, 'utf8');
+        } catch {
+          stderrFn(`[run] could not write --summary-file ${opts.summaryFile}; continuing`);
+        }
+      }
+      if (ghEnabled) {
+        emitGithubOutputs(
+          ciSummary,
+          env,
+          {
+            stdout: deps.stdout ?? ((line: string) => process.stdout.write(`${line}\n`)),
+            stderr: stderrFn,
+            appendFile: (path: string, content: string) => appendFileSync(path, content, 'utf8'),
+          },
+          { force: opts.ghOutput === true },
+        );
+      }
+    }
+  }
 
   // Rate-deferred tests were never dispatched → the batch is incomplete (exit 7),
   // mirroring `test rerun --all`. Checked before the failed-run throw so the
@@ -9125,6 +9162,14 @@ export function createTestCommand(deps: TestDeps = {}): Command {
       '--report-suite-name <name>',
       'optional JUnit <testsuite name=...> override (default: testsprite:<projectId>)',
     )
+    .option(
+      '--gh-output',
+      'with --all: emit GitHub-native output (::error:: annotations per non-passed run; job-summary table when $GITHUB_STEP_SUMMARY is set). Auto-enabled when GITHUB_ACTIONS=true',
+    )
+    .option(
+      '--summary-file <path>',
+      'with --all: also write the reduced machine summary JSON {total, passed, failed, timedOut, runs[]} to this file',
+    )
     .addHelpText(
       'after',
       '\nDependency-aware fresh run (M4):\n' +
@@ -9173,6 +9218,20 @@ export function createTestCommand(deps: TestDeps = {}): Command {
         wait: cmdOpts.wait === true,
         batchPath: isAll,
       });
+      // --gh-output / --summary-file reduce the batch envelope; on the single-id
+      // path they would be silently ignored — reject loudly (same rule as --filter).
+      if (cmdOpts.ghOutput === true && !isAll) {
+        throw localValidationError(
+          'gh-output',
+          '--gh-output only applies with --all (it reduces the batch envelope). Remove --gh-output, or add --all.',
+        );
+      }
+      if (cmdOpts.summaryFile !== undefined && !isAll) {
+        throw localValidationError(
+          'summary-file',
+          '--summary-file only applies with --all (it reduces the batch envelope). Remove --summary-file, or add --all.',
+        );
+      }
 
       if (isAll) {
         // --all path: wave-ordered fresh batch run.
@@ -9207,6 +9266,8 @@ export function createTestCommand(deps: TestDeps = {}): Command {
             report,
             reportFile: cmdOpts.reportFile,
             reportSuiteName: cmdOpts.reportSuiteName,
+            ghOutput: cmdOpts.ghOutput === true,
+            summaryFile: cmdOpts.summaryFile,
           },
           deps,
         );
@@ -9687,6 +9748,8 @@ interface RunFlagOpts {
   report?: string;
   reportFile?: string;
   reportSuiteName?: string;
+  ghOutput?: boolean;
+  summaryFile?: string;
 }
 
 interface WaitFlagOpts {

src/lib/gh-output.test.ts

@@ -0,0 +1,123 @@
+/**
+ * Unit tests for the CI-native output layer attached to `test run --all`
+ * (issue #99, reshaped from the withdrawn top-level `ci` command). The heavy
+ * lifting (trigger + poll) is the batch command's, already covered by its own
+ * suites; these tests cover the presentation seams: payload reduction, the
+ * job-summary Markdown, and the GitHub gating (env-driven and `--gh-output`
+ * forced).
+ */
+
+import { describe, expect, it } from 'vitest';
+import {
+  emitGithubOutputs,
+  renderJobSummaryMarkdown,
+  summarizeAcceptedPayload,
+  type CiSummary,
+} from './gh-output.js';
+
+const PAYLOAD = JSON.stringify({
+  accepted: [
+    {
+      testId: 'test_a',
+      runId: 'run_a',
+      status: 'passed',
+      dashboardUrl: 'https://portal.example.com/a',
+    },
+    {
+      testId: 'test_b',
+      runId: 'run_b',
+      status: 'failed',
+      error: { code: 'INTERNAL', message: 'boom', exitCode: 1 },
+    },
+    { testId: 'test_c', runId: 'run_c', status: 'timeout' },
+  ],
+  conflicts: [],
+});
+
+describe('summarizeAcceptedPayload', () => {
+  it('reduces accepted[] rows into counts and rows', () => {
+    const summary = summarizeAcceptedPayload(PAYLOAD);
+    expect(summary).toMatchObject({ total: 3, passed: 1, failed: 1, timedOut: 1 });
+    expect(summary.runs[1]).toMatchObject({ testId: 'test_b', status: 'failed', error: 'boom' });
+  });
+
+  it('unparseable or non-batch output reduces to an empty summary (never throws)', () => {
+    expect(summarizeAcceptedPayload('')).toMatchObject({ total: 0, passed: 0 });
+    expect(summarizeAcceptedPayload('{"method":"POST"}')).toMatchObject({ total: 0 });
+    expect(summarizeAcceptedPayload('not json')).toMatchObject({ total: 0 });
+  });
+});
+
+describe('renderJobSummaryMarkdown', () => {
+  it('renders the counts headline and one table row per run', () => {
+    const md = renderJobSummaryMarkdown(summarizeAcceptedPayload(PAYLOAD));
+    expect(md).toContain('**1/3 passed** (1 failed, 1 timed out)');
+    expect(md).toContain('| test_a | passed | [dashboard](https://portal.example.com/a) |');
+    expect(md).toContain('| test_c | timeout | run_c |');
+  });
+});
+
+describe('emitGithubOutputs', () => {
+  const summary: CiSummary = summarizeAcceptedPayload(PAYLOAD);
+
+  function makeSinks() {
+    const stdout: string[] = [];
+    const stderr: string[] = [];
+    const appended: Array<{ path: string; content: string }> = [];
+    return {
+      stdout,
+      stderr,
+      appended,
+      sinks: {
+        stdout: (line: string) => stdout.push(line),
+        stderr: (line: string) => stderr.push(line),
+        appendFile: (path: string, content: string) => appended.push({ path, content }),
+      },
+    };
+  }
+
+  it('appends the job summary and annotates only non-passed runs under Actions', () => {
+    const { stdout, appended, sinks } = makeSinks();
+    emitGithubOutputs(
+      summary,
+      { GITHUB_ACTIONS: 'true', GITHUB_STEP_SUMMARY: '/gh/summary.md' },
+      sinks,
+    );
+    expect(appended).toHaveLength(1);
+    expect(appended[0]!.path).toBe('/gh/summary.md');
+    expect(appended[0]!.content).toContain('TestSprite results');
+    const annotations = stdout.filter(line => line.startsWith('::error'));
+    expect(annotations).toHaveLength(2);
+    expect(annotations[0]).toContain('test_b');
+    expect(annotations[0]).toContain('boom');
+    expect(annotations[1]).toContain('test_c');
+  });
+
+  it('emits nothing off-CI, and a broken summary file downgrades to stderr', () => {
+    const offCi = makeSinks();
+    emitGithubOutputs(summary, {}, offCi.sinks);
+    expect(offCi.stdout).toHaveLength(0);
+    expect(offCi.appended).toHaveLength(0);
+
+    const broken = makeSinks();
+    emitGithubOutputs(
+      summary,
+      { GITHUB_STEP_SUMMARY: '/gh/summary.md' },
+      {
+        ...broken.sinks,
+        appendFile: () => {
+          throw new Error('EROFS');
+        },
+      },
+    );
+    expect(broken.stderr.join('\n')).toContain('could not append');
+  });
+
+  it('force (--gh-output) emits annotations off-Actions; the step summary still needs its env path', () => {
+    const forced = makeSinks();
+    emitGithubOutputs(summary, {}, forced.sinks, { force: true });
+    const annotations = forced.stdout.filter(line => line.startsWith('::error'));
+    expect(annotations).toHaveLength(2);
+    expect(forced.appended).toHaveLength(0);
+  });
+});

src/lib/gh-output.ts

@@ -0,0 +1,122 @@
+/**
+ * CI-native output layer for the batch run path (issue #99, reshaped from
+ * the withdrawn top-level `ci` command per the #264 review).
+ *
+ * `test run --all --wait` presents its result in the formats CI consumes:
+ *   (a) a stable machine summary `{total, passed, failed, timedOut, runs[]}`
+ *       written to `--summary-file <path>` when requested,
+ *   (b) a Markdown results table appended to `$GITHUB_STEP_SUMMARY` when
+ *       running under GitHub Actions,
+ *   (c) one `::error::` workflow-command line per non-passed run so failures
+ *       annotate the PR checks tab.
+ * Activation: `GITHUB_ACTIONS=true` in the environment, or the explicit
+ * `--gh-output` flag (which forces the annotations even off-Actions, so the
+ * behavior is previewable locally). All writes are best-effort: a broken
+ * summary file must never mask the batch gate's exit code.
+ */
+
+export interface CiRunRow {
+  testId: string;
+  runId?: string;
+  status: string;
+  dashboardUrl?: string;
+  error?: string;
+}
+
+export interface CiSummary {
+  total: number;
+  passed: number;
+  failed: number;
+  timedOut: number;
+  runs: CiRunRow[];
+}
+
+/**
+ * Reduce the batch command's JSON payload into the CI summary. The parse is
+ * defensive: it reads the same `accepted[]` rows the automation contract
+ * documents, and anything unparseable (dry-run envelope, partial output
+ * after a timeout) reduces to an empty run list rather than a crash.
+ */
+export function summarizeAcceptedPayload(capturedJson: string): CiSummary {
+  let payload: { accepted?: unknown } = {};
+  try {
+    payload = JSON.parse(capturedJson) as { accepted?: unknown };
+  } catch {
+    // Not a JSON object (dry-run banner path or truncated output): no rows.
+  }
+  const rows: CiRunRow[] = Array.isArray(payload.accepted)
+    ? (payload.accepted as Array<Record<string, unknown>>).map(row => {
+        const errorMessage =
+          row.error !== null && typeof row.error === 'object'
+            ? (row.error as { message?: unknown }).message
+            : undefined;
+        return {
+          testId: String(row.testId ?? ''),
+          ...(typeof row.runId === 'string' ? { runId: row.runId } : {}),
+          status: String(row.status ?? 'unknown'),
+          ...(typeof row.dashboardUrl === 'string' ? { dashboardUrl: row.dashboardUrl } : {}),
+          ...(typeof errorMessage === 'string' ? { error: errorMessage } : {}),
+        };
+      })
+    : [];
+  const passed = rows.filter(row => row.status === 'passed').length;
+  const timedOut = rows.filter(row => row.status === 'timeout').length;
+  const failed = rows.length - passed - timedOut;
+  return { total: rows.length, passed, failed, timedOut, runs: rows };
+}
+
+/** Markdown table for the GitHub job summary. */
+export function renderJobSummaryMarkdown(summary: CiSummary): string {
+  return [
+    '## TestSprite results',
+    '',
+    `**${summary.passed}/${summary.total} passed** (${summary.failed} failed, ${summary.timedOut} timed out)`,
+    '',
+    '| Test | Status | Run |',
+    '| --- | --- | --- |',
+    ...summary.runs.map(
+      row =>
+        `| ${row.testId} | ${row.status} | ${
+          row.dashboardUrl ? `[dashboard](${row.dashboardUrl})` : (row.runId ?? '')
+        } |`,
+    ),
+    '',
+  ].join('\n');
+}
+
+/**
+ * Emit the GitHub-native surfaces. Self-gating on the standard env vars:
+ * `$GITHUB_STEP_SUMMARY` (a file path Actions provides) receives the Markdown
+ * table; `GITHUB_ACTIONS=true` enables one `::error::` workflow command per
+ * non-passed run on stdout (Actions parses workflow commands from stdout).
+ * `force` (the `--gh-output` flag) emits the annotations even off-Actions;
+ * the step summary still requires the env-provided file path to exist.
+ * Both writes are best-effort: a broken summary file must not mask the gate.
+ */
+export function emitGithubOutputs(
+  summary: CiSummary,
+  env: NodeJS.ProcessEnv,
+  sinks: {
+    stdout: (line: string) => void;
+    stderr: (line: string) => void;
+    appendFile: (path: string, content: string) => void;
+  },
+  opts: { force?: boolean } = {},
+): void {
+  const summaryPath = env.GITHUB_STEP_SUMMARY;
+  if (typeof summaryPath === 'string' && summaryPath.length > 0) {
+    try {
+      sinks.appendFile(summaryPath, renderJobSummaryMarkdown(summary));
+    } catch {
+      sinks.stderr('[run] could not append to GITHUB_STEP_SUMMARY; continuing');
+    }
+  }
+  if (env.GITHUB_ACTIONS === 'true' || opts.force === true) {
+    for (const row of summary.runs) {
+      if (row.status === 'passed') continue;
+      const detail = row.error !== undefined ? ` ${row.error}` : '';
+      const link = row.dashboardUrl !== undefined ? ` ${row.dashboardUrl}` : '';
+      sinks.stdout(`::error title=TestSprite ${row.testId}::status=${row.status}${detail}${link}`);
+    }
+  }
+}

test/__snapshots__/help.snapshot.test.ts.snap

@@ -679,6 +679,13 @@ Options:
   --report-file <path>        output path for --report (atomic write)
   --report-suite-name <name>  optional JUnit <testsuite name=...> override
                               (default: testsprite:<projectId>)
+  --gh-output                 with --all: emit GitHub-native output (::error::
+                              annotations per non-passed run; job-summary table
+                              when $GITHUB_STEP_SUMMARY is set). Auto-enabled
+                              when GITHUB_ACTIONS=true
+  --summary-file <path>       with --all: also write the reduced machine
+                              summary JSON {total, passed, failed, timedOut,
+                              runs[]} to this file
   -h, --help                  display help for command
 
 Dependency-aware fresh run (M4):