Skip to content

Commit 07d4e03

Browse files
committed
feat(test): attach GitHub-native CI output to "test run --all" (--gh-output, --summary-file)
1 parent 26821e1 commit 07d4e03

4 files changed

Lines changed: 315 additions & 0 deletions

File tree

src/commands/test.ts

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
import {
2+
appendFileSync,
23
createWriteStream,
34
existsSync,
45
readFileSync,
56
readdirSync,
67
statSync,
8+
writeFileSync,
79
type WriteStream,
810
} from 'node:fs';
911
import { rename, stat, unlink } from 'node:fs/promises';
@@ -92,6 +94,7 @@ import {
9294
import { createTicker } from '../lib/ticker.js';
9395
import { RateThrottle } from '../lib/rate-throttle.js';
9496
import { resolvePortalBase, resolvePortalUrl } from '../lib/facade.js';
97+
import { emitGithubOutputs, summarizeAcceptedPayload } from '../lib/gh-output.js';
9598
import { loadConfig } from '../lib/config.js';
9699
import {
97100
flakyExitCode,
@@ -6363,6 +6366,10 @@ interface RunTestRunAllOptions extends CommonOptions {
63636366
reportFile?: string;
63646367
/** --report-suite-name: optional override for the JUnit <testsuite name=...>. */
63656368
reportSuiteName?: string;
6369+
/** --gh-output: force the GitHub-native output layer even off-Actions (issue #99). */
6370+
ghOutput?: boolean;
6371+
/** --summary-file: also write the reduced machine summary JSON to this path. */
6372+
summaryFile?: string;
63666373
}
63676374

63686375
async function writeBatchJUnitReportIfRequested(
@@ -6927,6 +6934,36 @@ export async function runTestRunAll(
69276934
};
69286935
await writeBatchJUnitReportIfRequested(opts, freshRunResults);
69296936
out.print(jsonPayload);
6937+
// CI-native output layer (issue #99): emitted before the gate throws below so
6938+
// the artifacts land even when the batch exits non-zero. The summary file is a
6939+
// machine artifact written regardless of --output mode; stdout stays owned by
6940+
// the envelope above (plus Actions workflow commands, which Actions parses).
6941+
{
6942+
const env = deps.env ?? process.env;
6943+
const ghEnabled = opts.ghOutput === true || env.GITHUB_ACTIONS === 'true';
6944+
if (ghEnabled || opts.summaryFile !== undefined) {
6945+
const ciSummary = summarizeAcceptedPayload(JSON.stringify(jsonPayload));
6946+
if (opts.summaryFile !== undefined) {
6947+
try {
6948+
writeFileSync(opts.summaryFile, `${JSON.stringify(ciSummary, null, 2)}\n`, 'utf8');
6949+
} catch {
6950+
stderrFn(`[run] could not write --summary-file ${opts.summaryFile}; continuing`);
6951+
}
6952+
}
6953+
if (ghEnabled) {
6954+
emitGithubOutputs(
6955+
ciSummary,
6956+
env,
6957+
{
6958+
stdout: deps.stdout ?? ((line: string) => process.stdout.write(`${line}\n`)),
6959+
stderr: stderrFn,
6960+
appendFile: (path: string, content: string) => appendFileSync(path, content, 'utf8'),
6961+
},
6962+
{ force: opts.ghOutput === true },
6963+
);
6964+
}
6965+
}
6966+
}
69306967

69316968
// Rate-deferred tests were never dispatched → the batch is incomplete (exit 7),
69326969
// mirroring `test rerun --all`. Checked before the failed-run throw so the
@@ -9125,6 +9162,14 @@ export function createTestCommand(deps: TestDeps = {}): Command {
91259162
'--report-suite-name <name>',
91269163
'optional JUnit <testsuite name=...> override (default: testsprite:<projectId>)',
91279164
)
9165+
.option(
9166+
'--gh-output',
9167+
'with --all: emit GitHub-native output (::error:: annotations per non-passed run; job-summary table when $GITHUB_STEP_SUMMARY is set). Auto-enabled when GITHUB_ACTIONS=true',
9168+
)
9169+
.option(
9170+
'--summary-file <path>',
9171+
'with --all: also write the reduced machine summary JSON {total, passed, failed, timedOut, runs[]} to this file',
9172+
)
91289173
.addHelpText(
91299174
'after',
91309175
'\nDependency-aware fresh run (M4):\n' +
@@ -9173,6 +9218,20 @@ export function createTestCommand(deps: TestDeps = {}): Command {
91739218
wait: cmdOpts.wait === true,
91749219
batchPath: isAll,
91759220
});
9221+
// --gh-output / --summary-file reduce the batch envelope; on the single-id
9222+
// path they would be silently ignored — reject loudly (same rule as --filter).
9223+
if (cmdOpts.ghOutput === true && !isAll) {
9224+
throw localValidationError(
9225+
'gh-output',
9226+
'--gh-output only applies with --all (it reduces the batch envelope). Remove --gh-output, or add --all.',
9227+
);
9228+
}
9229+
if (cmdOpts.summaryFile !== undefined && !isAll) {
9230+
throw localValidationError(
9231+
'summary-file',
9232+
'--summary-file only applies with --all (it reduces the batch envelope). Remove --summary-file, or add --all.',
9233+
);
9234+
}
91769235

91779236
if (isAll) {
91789237
// --all path: wave-ordered fresh batch run.
@@ -9207,6 +9266,8 @@ export function createTestCommand(deps: TestDeps = {}): Command {
92079266
report,
92089267
reportFile: cmdOpts.reportFile,
92099268
reportSuiteName: cmdOpts.reportSuiteName,
9269+
ghOutput: cmdOpts.ghOutput === true,
9270+
summaryFile: cmdOpts.summaryFile,
92109271
},
92119272
deps,
92129273
);
@@ -9687,6 +9748,8 @@ interface RunFlagOpts {
96879748
report?: string;
96889749
reportFile?: string;
96899750
reportSuiteName?: string;
9751+
ghOutput?: boolean;
9752+
summaryFile?: string;
96909753
}
96919754

96929755
interface WaitFlagOpts {

src/lib/gh-output.test.ts

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
/**
2+
* Unit tests for the CI-native output layer attached to `test run --all`
3+
* (issue #99, reshaped from the withdrawn top-level `ci` command). The heavy
4+
* lifting (trigger + poll) is the batch command's, already covered by its own
5+
* suites; these tests cover the presentation seams: payload reduction, the
6+
* job-summary Markdown, and the GitHub gating (env-driven and `--gh-output`
7+
* forced).
8+
*/
9+
10+
import { describe, expect, it } from 'vitest';
11+
import {
12+
emitGithubOutputs,
13+
renderJobSummaryMarkdown,
14+
summarizeAcceptedPayload,
15+
type CiSummary,
16+
} from './gh-output.js';
17+
18+
const PAYLOAD = JSON.stringify({
19+
accepted: [
20+
{
21+
testId: 'test_a',
22+
runId: 'run_a',
23+
status: 'passed',
24+
dashboardUrl: 'https://portal.example.com/a',
25+
},
26+
{
27+
testId: 'test_b',
28+
runId: 'run_b',
29+
status: 'failed',
30+
error: { code: 'INTERNAL', message: 'boom', exitCode: 1 },
31+
},
32+
{ testId: 'test_c', runId: 'run_c', status: 'timeout' },
33+
],
34+
conflicts: [],
35+
});
36+
37+
describe('summarizeAcceptedPayload', () => {
38+
it('reduces accepted[] rows into counts and rows', () => {
39+
const summary = summarizeAcceptedPayload(PAYLOAD);
40+
expect(summary).toMatchObject({ total: 3, passed: 1, failed: 1, timedOut: 1 });
41+
expect(summary.runs[1]).toMatchObject({ testId: 'test_b', status: 'failed', error: 'boom' });
42+
});
43+
44+
it('unparseable or non-batch output reduces to an empty summary (never throws)', () => {
45+
expect(summarizeAcceptedPayload('')).toMatchObject({ total: 0, passed: 0 });
46+
expect(summarizeAcceptedPayload('{"method":"POST"}')).toMatchObject({ total: 0 });
47+
expect(summarizeAcceptedPayload('not json')).toMatchObject({ total: 0 });
48+
});
49+
});
50+
51+
describe('renderJobSummaryMarkdown', () => {
52+
it('renders the counts headline and one table row per run', () => {
53+
const md = renderJobSummaryMarkdown(summarizeAcceptedPayload(PAYLOAD));
54+
expect(md).toContain('**1/3 passed** (1 failed, 1 timed out)');
55+
expect(md).toContain('| test_a | passed | [dashboard](https://portal.example.com/a) |');
56+
expect(md).toContain('| test_c | timeout | run_c |');
57+
});
58+
});
59+
60+
describe('emitGithubOutputs', () => {
61+
const summary: CiSummary = summarizeAcceptedPayload(PAYLOAD);
62+
63+
function makeSinks() {
64+
const stdout: string[] = [];
65+
const stderr: string[] = [];
66+
const appended: Array<{ path: string; content: string }> = [];
67+
return {
68+
stdout,
69+
stderr,
70+
appended,
71+
sinks: {
72+
stdout: (line: string) => stdout.push(line),
73+
stderr: (line: string) => stderr.push(line),
74+
appendFile: (path: string, content: string) => appended.push({ path, content }),
75+
},
76+
};
77+
}
78+
79+
it('appends the job summary and annotates only non-passed runs under Actions', () => {
80+
const { stdout, appended, sinks } = makeSinks();
81+
emitGithubOutputs(
82+
summary,
83+
{ GITHUB_ACTIONS: 'true', GITHUB_STEP_SUMMARY: '/gh/summary.md' },
84+
sinks,
85+
);
86+
expect(appended).toHaveLength(1);
87+
expect(appended[0]!.path).toBe('/gh/summary.md');
88+
expect(appended[0]!.content).toContain('TestSprite results');
89+
const annotations = stdout.filter(line => line.startsWith('::error'));
90+
expect(annotations).toHaveLength(2);
91+
expect(annotations[0]).toContain('test_b');
92+
expect(annotations[0]).toContain('boom');
93+
expect(annotations[1]).toContain('test_c');
94+
});
95+
96+
it('emits nothing off-CI, and a broken summary file downgrades to stderr', () => {
97+
const offCi = makeSinks();
98+
emitGithubOutputs(summary, {}, offCi.sinks);
99+
expect(offCi.stdout).toHaveLength(0);
100+
expect(offCi.appended).toHaveLength(0);
101+
102+
const broken = makeSinks();
103+
emitGithubOutputs(
104+
summary,
105+
{ GITHUB_STEP_SUMMARY: '/gh/summary.md' },
106+
{
107+
...broken.sinks,
108+
appendFile: () => {
109+
throw new Error('EROFS');
110+
},
111+
},
112+
);
113+
expect(broken.stderr.join('\n')).toContain('could not append');
114+
});
115+
116+
it('force (--gh-output) emits annotations off-Actions; the step summary still needs its env path', () => {
117+
const forced = makeSinks();
118+
emitGithubOutputs(summary, {}, forced.sinks, { force: true });
119+
const annotations = forced.stdout.filter(line => line.startsWith('::error'));
120+
expect(annotations).toHaveLength(2);
121+
expect(forced.appended).toHaveLength(0);
122+
});
123+
});

src/lib/gh-output.ts

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
/**
2+
* CI-native output layer for the batch run path (issue #99, reshaped from
3+
* the withdrawn top-level `ci` command per the #264 review).
4+
*
5+
* `test run --all --wait` presents its result in the formats CI consumes:
6+
* (a) a stable machine summary `{total, passed, failed, timedOut, runs[]}`
7+
* written to `--summary-file <path>` when requested,
8+
* (b) a Markdown results table appended to `$GITHUB_STEP_SUMMARY` when
9+
* running under GitHub Actions,
10+
* (c) one `::error::` workflow-command line per non-passed run so failures
11+
* annotate the PR checks tab.
12+
* Activation: `GITHUB_ACTIONS=true` in the environment, or the explicit
13+
* `--gh-output` flag (which forces the annotations even off-Actions, so the
14+
* behavior is previewable locally). All writes are best-effort: a broken
15+
* summary file must never mask the batch gate's exit code.
16+
*/
17+
18+
export interface CiRunRow {
19+
testId: string;
20+
runId?: string;
21+
status: string;
22+
dashboardUrl?: string;
23+
error?: string;
24+
}
25+
26+
export interface CiSummary {
27+
total: number;
28+
passed: number;
29+
failed: number;
30+
timedOut: number;
31+
runs: CiRunRow[];
32+
}
33+
34+
/**
35+
* Reduce the batch command's JSON payload into the CI summary. The parse is
36+
* defensive: it reads the same `accepted[]` rows the automation contract
37+
* documents, and anything unparseable (dry-run envelope, partial output
38+
* after a timeout) reduces to an empty run list rather than a crash.
39+
*/
40+
export function summarizeAcceptedPayload(capturedJson: string): CiSummary {
41+
let payload: { accepted?: unknown } = {};
42+
try {
43+
payload = JSON.parse(capturedJson) as { accepted?: unknown };
44+
} catch {
45+
// Not a JSON object (dry-run banner path or truncated output): no rows.
46+
}
47+
const rows: CiRunRow[] = Array.isArray(payload.accepted)
48+
? (payload.accepted as Array<Record<string, unknown>>).map(row => {
49+
const errorMessage =
50+
row.error !== null && typeof row.error === 'object'
51+
? (row.error as { message?: unknown }).message
52+
: undefined;
53+
return {
54+
testId: String(row.testId ?? ''),
55+
...(typeof row.runId === 'string' ? { runId: row.runId } : {}),
56+
status: String(row.status ?? 'unknown'),
57+
...(typeof row.dashboardUrl === 'string' ? { dashboardUrl: row.dashboardUrl } : {}),
58+
...(typeof errorMessage === 'string' ? { error: errorMessage } : {}),
59+
};
60+
})
61+
: [];
62+
const passed = rows.filter(row => row.status === 'passed').length;
63+
const timedOut = rows.filter(row => row.status === 'timeout').length;
64+
const failed = rows.length - passed - timedOut;
65+
return { total: rows.length, passed, failed, timedOut, runs: rows };
66+
}
67+
68+
/** Markdown table for the GitHub job summary. */
69+
export function renderJobSummaryMarkdown(summary: CiSummary): string {
70+
return [
71+
'## TestSprite results',
72+
'',
73+
`**${summary.passed}/${summary.total} passed** (${summary.failed} failed, ${summary.timedOut} timed out)`,
74+
'',
75+
'| Test | Status | Run |',
76+
'| --- | --- | --- |',
77+
...summary.runs.map(
78+
row =>
79+
`| ${row.testId} | ${row.status} | ${
80+
row.dashboardUrl ? `[dashboard](${row.dashboardUrl})` : (row.runId ?? '')
81+
} |`,
82+
),
83+
'',
84+
].join('\n');
85+
}
86+
87+
/**
88+
* Emit the GitHub-native surfaces. Self-gating on the standard env vars:
89+
* `$GITHUB_STEP_SUMMARY` (a file path Actions provides) receives the Markdown
90+
* table; `GITHUB_ACTIONS=true` enables one `::error::` workflow command per
91+
* non-passed run on stdout (Actions parses workflow commands from stdout).
92+
* `force` (the `--gh-output` flag) emits the annotations even off-Actions;
93+
* the step summary still requires the env-provided file path to exist.
94+
* Both writes are best-effort: a broken summary file must not mask the gate.
95+
*/
96+
export function emitGithubOutputs(
97+
summary: CiSummary,
98+
env: NodeJS.ProcessEnv,
99+
sinks: {
100+
stdout: (line: string) => void;
101+
stderr: (line: string) => void;
102+
appendFile: (path: string, content: string) => void;
103+
},
104+
opts: { force?: boolean } = {},
105+
): void {
106+
const summaryPath = env.GITHUB_STEP_SUMMARY;
107+
if (typeof summaryPath === 'string' && summaryPath.length > 0) {
108+
try {
109+
sinks.appendFile(summaryPath, renderJobSummaryMarkdown(summary));
110+
} catch {
111+
sinks.stderr('[run] could not append to GITHUB_STEP_SUMMARY; continuing');
112+
}
113+
}
114+
if (env.GITHUB_ACTIONS === 'true' || opts.force === true) {
115+
for (const row of summary.runs) {
116+
if (row.status === 'passed') continue;
117+
const detail = row.error !== undefined ? ` ${row.error}` : '';
118+
const link = row.dashboardUrl !== undefined ? ` ${row.dashboardUrl}` : '';
119+
sinks.stdout(`::error title=TestSprite ${row.testId}::status=${row.status}${detail}${link}`);
120+
}
121+
}
122+
}

test/__snapshots__/help.snapshot.test.ts.snap

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -679,6 +679,13 @@ Options:
679679
--report-file <path> output path for --report (atomic write)
680680
--report-suite-name <name> optional JUnit <testsuite name=...> override
681681
(default: testsprite:<projectId>)
682+
--gh-output with --all: emit GitHub-native output (::error::
683+
annotations per non-passed run; job-summary table
684+
when $GITHUB_STEP_SUMMARY is set). Auto-enabled
685+
when GITHUB_ACTIONS=true
686+
--summary-file <path> with --all: also write the reduced machine
687+
summary JSON {total, passed, failed, timedOut,
688+
runs[]} to this file
682689
-h, --help display help for command
683690
684691
Dependency-aware fresh run (M4):

0 commit comments

Comments
 (0)