Skip to content

Commit d166e3b

Browse files
authored
feat: support nightly spec tests (#9221)
## Motivation The `ethereum/consensus-specs` CI generates spec test artifacts daily via `tests.yml`, without requiring a release. This also works for forks and feature branches. ## Description Extends `pnpm download-spec-tests` with optional arguments to download from workflow artifacts: ```bash pnpm download-spec-tests # old behavior pnpm download-spec-tests latest # latest available run pnpm download-spec-tests 2026-04-14 # latest run from that date pnpm download-spec-tests latest brech1/consensus-specs # fork pnpm download-spec-tests latest brech1/consensus-specs feat/x # fork + branch ``` Since these are not releases, the artifacts must be downloaded through the GitHub API, this requires a `GITHUB_TOKEN`, it can be set in the local `.env` file. ## Checks I checked locally with an experimental branch: - https://github.com/brech1/consensus-specs/actions/runs/24481515544/ ``` pnpm download-spec-tests latest brech1/consensus-specs test/payload-att-message ```
1 parent b7d6944 commit d166e3b

7 files changed

Lines changed: 132 additions & 9 deletions

File tree

packages/beacon-node/package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,7 @@
173173
"@types/js-yaml": "^4.0.5",
174174
"@types/qs": "^6.9.7",
175175
"@types/tmp": "^0.2.3",
176+
"dotenv": "^16.4.5",
176177
"js-yaml": "^4.1.0",
177178
"rewiremock": "^3.14.5",
178179
"rimraf": "^4.4.1",
Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,27 @@
1+
import path from "node:path";
2+
import {config} from "dotenv";
3+
import {downloadNightlyTests} from "@lodestar/spec-test-util/downloadNightlyTests";
14
import {downloadTests} from "@lodestar/spec-test-util/downloadTests";
25
import {blsSpecTests, ethereumConsensusSpecsTests} from "./specTestVersioning.js";
36

4-
for (const downloadTestOpts of [ethereumConsensusSpecsTests, blsSpecTests]) {
5-
downloadTests(downloadTestOpts, console.log).catch((e: Error) => {
6-
console.error(e);
7-
process.exit(1);
8-
});
7+
const [date, repo, branch] = process.argv.slice(2);
8+
const downloads = [downloadTests(blsSpecTests, console.log)];
9+
10+
if (date) {
11+
config({path: path.join(import.meta.dirname, "../../../../.env")});
12+
13+
const opts = {
14+
...ethereumConsensusSpecsTests,
15+
...(repo && {specTestsRepoUrl: `https://github.com/${repo}`}),
16+
...(branch && {branch}),
17+
};
18+
19+
downloads.push(downloadNightlyTests(opts, console.log, date));
20+
} else {
21+
downloads.push(downloadTests(ethereumConsensusSpecsTests, console.log));
922
}
23+
24+
await Promise.all(downloads).catch((e: Error) => {
25+
console.error(e);
26+
process.exit(1);
27+
});

packages/beacon-node/test/spec/presets/sanity.test.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ const sanity: TestRunnerFn<any, BeaconStateAllForks> = (fork, testName, testSuit
2323
case "slots":
2424
return sanitySlots(fork, testName, testSuite);
2525
case "blocks":
26+
case "epoch_boundary":
2627
return sanityBlocks(fork, testName, testSuite);
2728
default:
2829
throw Error(`Unknown sanity test ${testName}`);

packages/spec-test-util/package.json

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,11 @@
1919
"bun": "./src/downloadTests.ts",
2020
"types": "./lib/downloadTests.d.ts",
2121
"import": "./lib/downloadTests.js"
22+
},
23+
"./downloadNightlyTests": {
24+
"bun": "./src/downloadNightlyTests.ts",
25+
"types": "./lib/downloadNightlyTests.d.ts",
26+
"import": "./lib/downloadNightlyTests.js"
2227
}
2328
},
2429
"files": [
Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
import {fetch} from "@lodestar/utils";
2+
import {downloadGenericSpecTests} from "./downloadTests.js";
3+
4+
type WorkflowRunsResponse = {workflow_runs: {id: number}[]};
5+
type ArtifactsListResponse = {artifacts: {archive_download_url: string; expired: boolean; name: string}[]};
6+
7+
async function ghApiFetch<T>(endpoint: string, token: string): Promise<T> {
8+
const res = await fetch(`https://api.github.com${endpoint}`, {
9+
headers: {Authorization: `token ${token}`, Accept: "application/vnd.github+json"},
10+
signal: AbortSignal.timeout(30_000),
11+
});
12+
13+
if (!res.ok) {
14+
throw new Error(
15+
res.status === 401 ? "GITHUB_TOKEN is invalid or expired" : `GitHub API ${res.status} (${endpoint})`
16+
);
17+
}
18+
19+
return res.json() as Promise<T>;
20+
}
21+
22+
async function resolveNightlyRunId(repo: string, token: string, date?: string, branch?: string): Promise<number> {
23+
const params = new URLSearchParams({status: "success", per_page: "1"});
24+
if (branch) params.append("branch", branch);
25+
if (date) params.append("created", date);
26+
27+
const {workflow_runs} = await ghApiFetch<WorkflowRunsResponse>(
28+
`/repos/${repo}/actions/workflows/tests.yml/runs?${params}`,
29+
token
30+
);
31+
32+
const runId = workflow_runs[0]?.id;
33+
if (!runId) {
34+
throw new Error(`No successful run found${date ? ` on ${date}` : ""} for ${repo}${branch ? ` (${branch})` : ""}`);
35+
}
36+
return runId;
37+
}
38+
39+
export async function downloadNightlyTests(
40+
opts: {specTestsRepoUrl: string; outputDir: string; testsToDownload: string[]; branch?: string},
41+
log: (msg: string) => void,
42+
date?: string
43+
): Promise<void> {
44+
const token = process.env.GITHUB_TOKEN;
45+
if (!token) throw new Error("GITHUB_TOKEN is required for nightly downloads");
46+
47+
const resolvedDate = date === "latest" || !date ? undefined : date;
48+
if (resolvedDate && !/^\d{4}-\d{2}-\d{2}$/.test(resolvedDate)) {
49+
throw new Error(`Invalid date: "${date}". Expected "latest" or YYYY-MM-DD`);
50+
}
51+
52+
const repo = new URL(opts.specTestsRepoUrl).pathname.slice(1).replace(/\/$/, "");
53+
const runId = await resolveNightlyRunId(repo, token, resolvedDate, opts.branch);
54+
log(`Resolved nightly${resolvedDate ? ` ${resolvedDate}` : ""} to run ${runId}`);
55+
56+
const {artifacts} = await ghApiFetch<ArtifactsListResponse>(`/repos/${repo}/actions/runs/${runId}/artifacts`, token);
57+
58+
const urlByTest: Record<string, string> = {};
59+
const available: string[] = [];
60+
for (const test of opts.testsToDownload) {
61+
const artifact = artifacts.find((a) => a.name === `${test}.tar.gz` && !a.expired);
62+
if (artifact) {
63+
urlByTest[test] = artifact.archive_download_url;
64+
available.push(test);
65+
} else {
66+
log(`Skipping ${test} (not found in run ${runId})`);
67+
}
68+
}
69+
70+
if (available.length === 0) throw new Error(`No matching artifacts found in run ${runId}`);
71+
72+
const authInit: RequestInit = {headers: {Authorization: `token ${token}`, Accept: "application/vnd.github+json"}};
73+
74+
await downloadGenericSpecTests(
75+
{
76+
specVersion: `nightly-${runId}`,
77+
specTestsRepoUrl: opts.specTestsRepoUrl,
78+
outputDir: opts.outputDir,
79+
testsToDownload: available,
80+
testUrls: urlByTest,
81+
fetchInit: authInit,
82+
},
83+
log
84+
);
85+
}

packages/spec-test-util/src/downloadTests.ts

Lines changed: 14 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@ export interface DownloadGenericTestsOptions<TestNames extends string> {
2525
outputDir: string;
2626
specTestsRepoUrl: string;
2727
testsToDownload: TestNames[];
28+
testUrls?: Record<string, string>;
29+
fetchInit?: RequestInit;
2830
}
2931

3032
/**
@@ -39,7 +41,14 @@ export async function downloadTests(opts: DownloadTestsOptions, log: (msg: strin
3941
* Used by spec tests and SlashingProtectionInterchangeTest
4042
*/
4143
export async function downloadGenericSpecTests<TestNames extends string>(
42-
{specVersion, specTestsRepoUrl, outputDir, testsToDownload}: DownloadGenericTestsOptions<TestNames>,
44+
{
45+
specVersion,
46+
specTestsRepoUrl,
47+
outputDir,
48+
testsToDownload,
49+
testUrls,
50+
fetchInit,
51+
}: DownloadGenericTestsOptions<TestNames>,
4352
log: (msg: string) => void = logEmpty
4453
): Promise<void> {
4554
log(`outputDir = ${outputDir}`);
@@ -62,12 +71,13 @@ export async function downloadGenericSpecTests<TestNames extends string>(
6271

6372
await Promise.all(
6473
testsToDownload.map(async (test) => {
65-
const url = `${specTestsRepoUrl ?? defaultSpecTestsRepoUrl}/releases/download/${specVersion}/${test}.tar.gz`;
74+
const defaultUrl = `${specTestsRepoUrl ?? defaultSpecTestsRepoUrl}/releases/download/${specVersion}/${test}.tar.gz`;
6675
const tarball = path.join(outputDir, `${test}.tar.gz`);
6776

6877
await retry(
6978
async () => {
70-
const res = await fetch(url, {signal: AbortSignal.timeout(30 * 60 * 1000)});
79+
const url = testUrls?.[test] ?? defaultUrl;
80+
const res = await fetch(url, {signal: AbortSignal.timeout(30 * 60 * 1000), ...fetchInit});
7181

7282
if (!res.ok) {
7383
throw new Error(`Failed to download file from ${url}: ${res.status} ${res.statusText}`);
@@ -95,7 +105,7 @@ export async function downloadGenericSpecTests<TestNames extends string>(
95105
{
96106
retries: 3,
97107
onRetry: (e, attempt) => {
98-
log(`Download attempt ${attempt} for ${url} failed: ${e.message}`);
108+
log(`Download attempt ${attempt} for ${test} failed: ${e.message}`);
99109
},
100110
}
101111
);

pnpm-lock.yaml

Lines changed: 3 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)