Skip to content

Commit a45205c

Browse files
feat(ci): fill mainnet fixtures nightly, draft cached tests@ releases (#3100)
Co-authored-by: danceratopz <danceratopz@gmail.com>
1 parent 693dbbd commit a45205c

10 files changed

Lines changed: 1181 additions & 87 deletions

File tree

.github/actions/build-fixtures/action.yaml

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,9 @@ inputs:
1313
split_label:
1414
description: "Label for this fork-range split. Empty for unsplit builds."
1515
default: ""
16+
split_retention_days:
17+
description: "retention-days for the split fixture artifact. Empty for the repo default."
18+
default: ""
1619
evm:
1720
description: "Override the evm impl. Defaults to the feature's evm-type."
1821
default: ""
@@ -49,6 +52,8 @@ runs:
4952
run: sudo apt-get install -y pigz
5053
- name: Generate fixtures using fill
5154
shell: bash
55+
env:
56+
PYTEST_XDIST_AUTO_NUM_WORKERS: ${{ steps.evm-builder.outputs.xdist }}
5257
run: |
5358
IS_SPLIT="${{ inputs.split_label }}"
5459
@@ -60,13 +65,14 @@ runs:
6065
FORK_ARGS=""
6166
fi
6267
68+
EVM_ARGS=""
69+
if [ "${{ steps.evm-builder.outputs.impl }}" != "eels" ]; then
70+
EVM_ARGS="--evm-bin=${{ steps.evm-builder.outputs.evm-bin }}"
71+
fi
72+
6373
# Allow exit code 5 (NO_TESTS_COLLECTED) for fork ranges with no tests.
6474
EXIT_CODE=0
65-
if [ "${{ steps.evm-builder.outputs.impl }}" = "eels" ]; then
66-
uv run fill -n ${{ steps.evm-builder.outputs.xdist }} ${{ steps.properties.outputs.fill-params }} $FORK_ARGS $OUTPUT_ARG --build-name ${{ inputs.release_name }} --no-html --durations=100 --log-level=DEBUG || EXIT_CODE=$?
67-
else
68-
uv run fill -n ${{ steps.evm-builder.outputs.xdist }} --evm-bin=${{ steps.evm-builder.outputs.evm-bin }} ${{ steps.properties.outputs.fill-params }} $FORK_ARGS $OUTPUT_ARG --build-name ${{ inputs.release_name }} --no-html --durations=100 --log-level=DEBUG || EXIT_CODE=$?
69-
fi
75+
just fill-release $EVM_ARGS ${{ steps.properties.outputs.fill-params }} $FORK_ARGS $OUTPUT_ARG --build-name ${{ inputs.release_name }} || EXIT_CODE=$?
7076
if [ "$EXIT_CODE" -ne 0 ] && [ "$EXIT_CODE" -ne 5 ]; then
7177
exit "$EXIT_CODE"
7278
fi
@@ -95,3 +101,4 @@ runs:
95101
include-hidden-files: true
96102
path: fixtures_${{ inputs.release_name }}/
97103
if-no-files-found: ignore
104+
retention-days: ${{ inputs.split_retention_days }}

.github/configs/evm.yaml

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,12 +17,6 @@ evmone:
1717
evm-bin: evmone
1818
xdist: auto
1919
targets: ["evmone-cli"]
20-
benchmark:
21-
impl: geth
22-
repo: ethereum/go-ethereum
23-
ref: master
24-
evm-bin: evm
25-
xdist: auto
2620
besu:
2721
impl: besu
2822
repo: hyperledger/besu

.github/configs/feature.yaml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@
55
# Any `<feat>-devnet` input resolves to the shared `devnet` entry but keeps
66
# its name in the tag; the devnet number lives in the version (X), not the
77
# feature name, so this file needs no edits for new devnets.
8+
# `tests` is also what the scheduled nightly run of the `release_fixtures`
9+
# workflow fills: mainnet forks only, so the rotating nightly artifact is a
10+
# release-ready rehearsal of the next tests@ release.
811
tests:
912
evm-type: eels
1013
fill-params: --until=BPO4 --generate-all-formats
Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
#!/usr/bin/env -S uv run --script
2+
#
3+
# /// script
4+
# requires-python = ">=3.12"
5+
# ///
6+
"""
7+
Decide whether a scheduled nightly fill has new commits to fill.
8+
9+
Usage: `check_new_commits.py` (all inputs come from the environment).
10+
11+
Compare the current commit against the head SHA of the last scheduled
12+
run of the release workflow that actually filled: the newest successful
13+
*scheduled* run that uploaded artifacts. A quiet nightly skips its
14+
build jobs yet still concludes as a successful run, so plain success is
15+
no evidence of a fill. Anchoring on real fills means a nightly that
16+
fails or skips keeps re-running until a fill goes green and no commit
17+
slips through unfilled; filtering on scheduled runs means manual
18+
releases never advance the nightly baseline. Manual
19+
(`workflow_dispatch`) runs always run.
20+
21+
A quiet stretch with no new commits still refreshes: once the last
22+
fill is `REFRESH_AGE` old -- or its artifact is no longer live -- the
23+
nightly re-runs anyway, so a live artifact always exists within the
24+
workflow's five-day retention and the release pipeline keeps getting
25+
exercised.
26+
27+
Read `GITHUB_EVENT_NAME`, `GITHUB_REPOSITORY` and `GITHUB_SHA` from the
28+
environment and query the GitHub API via the `gh` CLI (authenticated by
29+
`GH_TOKEN`). Print `run=true|false` to stdout for `$GITHUB_OUTPUT` and
30+
append the new-commit list (or a skip notice) to the
31+
`$GITHUB_STEP_SUMMARY` file.
32+
"""
33+
34+
import json
35+
import os
36+
import subprocess
37+
import sys
38+
from datetime import datetime, timedelta, timezone
39+
40+
WORKFLOW_FILE = "release_fixtures.yaml"
41+
42+
# Re-run a quiet nightly once the last successful fill is this old, so
43+
# a fresh artifact is uploaded before the previous one lapses (the
44+
# workflow retains scheduled tarballs for five days).
45+
REFRESH_AGE = timedelta(days=4)
46+
47+
48+
def gh_api(path: str) -> str:
49+
"""Return the stdout of `gh api <path>`, exiting non-zero on error."""
50+
result = subprocess.run(
51+
["gh", "api", path], capture_output=True, text=True
52+
)
53+
if result.returncode != 0:
54+
print(f"Error: gh api {path} failed:", file=sys.stderr)
55+
print(result.stderr, file=sys.stderr)
56+
sys.exit(1)
57+
return result.stdout
58+
59+
60+
def last_real_nightly(repository: str) -> tuple[str, str, bool]:
61+
"""
62+
Return the head SHA, creation time and artifact liveness of the
63+
last scheduled run that actually filled.
64+
65+
A skipped nightly still concludes as a successful scheduled run,
66+
so taking the newest success as the baseline would let skip-runs
67+
keep resetting the refresh clock while the last real artifact
68+
quietly expires. Walk the recent successful scheduled runs and
69+
take the newest one that uploaded artifacts, reporting whether any
70+
of them is still live. Return empty strings when none exists yet.
71+
"""
72+
runs = json.loads(
73+
gh_api(
74+
f"repos/{repository}/actions/workflows/{WORKFLOW_FILE}"
75+
"/runs?status=success&event=schedule&per_page=10"
76+
)
77+
)["workflow_runs"]
78+
for run in runs:
79+
artifacts = json.loads(
80+
gh_api(f"repos/{repository}/actions/runs/{run['id']}/artifacts")
81+
)["artifacts"]
82+
if artifacts:
83+
live = any(not a["expired"] for a in artifacts)
84+
return str(run["head_sha"]), str(run["created_at"]), live
85+
return "", "", False
86+
87+
88+
def is_stale(created_at: str) -> bool:
89+
"""Return whether a run created at *created_at* is due a refresh."""
90+
created = datetime.fromisoformat(created_at)
91+
return datetime.now(timezone.utc) - created >= REFRESH_AGE
92+
93+
94+
def commits_since(repository: str, last_sha: str, head_sha: str) -> list[str]:
95+
"""Return `- <sha> <subject>` lines for commits after *last_sha*."""
96+
compare = json.loads(
97+
gh_api(f"repos/{repository}/compare/{last_sha}...{head_sha}")
98+
)
99+
return [
100+
f"- {c['sha'][:7]} {(c['commit']['message'].splitlines() or [''])[0]}"
101+
for c in compare["commits"]
102+
]
103+
104+
105+
def append_summary(text: str) -> None:
106+
"""Append *text* to the GitHub step summary, or stderr if unset."""
107+
summary_path = os.environ.get("GITHUB_STEP_SUMMARY")
108+
if summary_path:
109+
with open(summary_path, "a") as f:
110+
f.write(text + "\n")
111+
else:
112+
print(text, file=sys.stderr)
113+
114+
115+
def main() -> None:
116+
"""Print `run=true|false` and write the step summary."""
117+
if os.environ["GITHUB_EVENT_NAME"] != "schedule":
118+
# Manual releases always run.
119+
print("run=true")
120+
return
121+
122+
repository = os.environ["GITHUB_REPOSITORY"]
123+
head_sha = os.environ["GITHUB_SHA"]
124+
125+
last_sha, last_created, artifact_live = last_real_nightly(repository)
126+
if last_sha:
127+
commits = commits_since(repository, last_sha, head_sha)
128+
else:
129+
# No prior successful nightly recorded; fill to get a baseline.
130+
commits = ["- (no previous successful nightly fill found)"]
131+
132+
if commits:
133+
print("run=true")
134+
append_summary(
135+
"### Commits since last successful nightly fill\n"
136+
+ "\n".join(commits)
137+
)
138+
elif not artifact_live:
139+
print("run=true")
140+
append_summary(
141+
"No new commits, but no live fixture artifact exists; refilling."
142+
)
143+
elif is_stale(last_created):
144+
print("run=true")
145+
append_summary(
146+
"No new commits, but the last nightly fill is older than "
147+
f"{REFRESH_AGE.days} days; refreshing before its artifact "
148+
"retention lapses."
149+
)
150+
else:
151+
print("run=false")
152+
append_summary(
153+
"No new commits since the last successful nightly fill; skipping."
154+
)
155+
156+
157+
if __name__ == "__main__":
158+
main()

.github/scripts/generate_build_matrix.py

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
Validate release inputs and generate the build matrix for release
1111
fixture workflows.
1212
13-
Usage: `generate_build_matrix.py <feature> <version> [branch]`.
13+
Usage: `generate_build_matrix.py <feature> <version> [branch] [evm]`.
1414
1515
First validate the dispatch inputs (see `validate_inputs`), then read
1616
`.github/configs/feature.yaml` and emit a flat JSON build matrix suitable
@@ -31,6 +31,7 @@
3131

3232
FEATURE_CONFIG = Path(".github/configs/feature.yaml")
3333
FORK_RANGES_CONFIG = Path(".github/configs/fork-ranges.yaml")
34+
EVM_CONFIG = Path(".github/configs/evm.yaml")
3435

3536
VERSION_RE = re.compile(r"^v[0-9]+\.[0-9]+\.[0-9]+$")
3637

@@ -81,12 +82,13 @@ def fail(message: str) -> NoReturn:
8182
sys.exit(1)
8283

8384

84-
def validate_inputs(feature: str, version: str, branch: str) -> None:
85+
def validate_inputs(feature: str, version: str, branch: str, evm: str) -> None:
8586
"""
8687
Validate the release dispatch inputs before building a matrix.
8788
88-
Centralize the feature/version checks here so they are unit-testable
89-
rather than living as inline bash in the release workflow.
89+
Centralize the feature/version/evm checks here so they are
90+
unit-testable rather than living as inline bash in the release
91+
workflow.
9092
9193
For `<feat>-devnet` releases the major version (`X` of `vX.Y.Z`)
9294
must equal the devnet number encoded in the release branch, so a
@@ -97,6 +99,10 @@ def validate_inputs(feature: str, version: str, branch: str) -> None:
9799
if not VERSION_RE.match(version):
98100
fail(f"version '{version}' must match vX.Y.Z (e.g. v20.0.0)")
99101

102+
# An `evm` override must name a key in evm.yaml.
103+
if evm and evm not in load_config(EVM_CONFIG):
104+
fail(f"evm '{evm}' is not a key in {EVM_CONFIG}")
105+
100106
# A bare `devnet` has no friendly `<feat>-` prefix to tag with.
101107
if feature in ("devnet", "-devnet"):
102108
fail("devnet releases require a <feat>- prefix, e.g. bal-devnet")
@@ -207,16 +213,18 @@ def main() -> None:
207213
args = sys.argv[1:]
208214
if len(args) < 2:
209215
print(
210-
"Usage: generate_build_matrix.py <feature> <version> [branch]",
216+
"Usage: generate_build_matrix.py "
217+
"<feature> <version> [branch] [evm]",
211218
file=sys.stderr,
212219
)
213220
sys.exit(1)
214221

215222
name = args[0]
216223
version = args[1]
217224
branch = args[2] if len(args) > 2 else ""
225+
evm = args[3] if len(args) > 3 else ""
218226

219-
validate_inputs(name, version, branch)
227+
validate_inputs(name, version, branch, evm)
220228

221229
config = load_config(FEATURE_CONFIG)
222230
fork_ranges = load_config(FORK_RANGES_CONFIG) or []

0 commit comments

Comments
 (0)