diff --git a/.github/actions/build-evm-base/action.yaml b/.github/actions/build-evm-base/action.yaml index 7a5496259b0..d8ed5780d98 100644 --- a/.github/actions/build-evm-base/action.yaml +++ b/.github/actions/build-evm-base/action.yaml @@ -2,48 +2,61 @@ name: 'Build EVM' description: 'Resolves and builds the requested EVM binary by name' inputs: type: - description: 'Type of EVM binary to build' + description: 'Type of EVM binary to build (key in .github/configs/evm.yaml)' required: true - default: 'main' + repo_override: + description: 'Override the repo from evm.yaml (e.g. ethereum/go-ethereum)' + required: false + default: '' + ref_override: + description: 'Override the ref/branch/commit from evm.yaml' + required: false + default: '' outputs: impl: description: "Implementation of EVM binary to build" value: ${{ steps.config-evm-reader.outputs.impl }} repo: description: "Repository to use to build the EVM binary" - value: ${{ steps.config-evm-reader.outputs.repo }} + value: ${{ steps.resolved.outputs.repo }} ref: description: "Reference to branch, commit, or tag to use to build the EVM binary" - value: ${{ steps.config-evm-reader.outputs.ref }} + value: ${{ steps.resolved.outputs.ref }} evm-bin: description: "Binary name of the evm tool to use" - value: ${{ steps.config-evm-impl-config-reader.outputs.evm-bin }} - x-dist: - description: "Binary name of the evm tool to use" - value: ${{ steps.config-evm-impl-config-reader.outputs.x-dist }} + value: ${{ steps.config-evm-reader.outputs.evm-bin }} + xdist: + description: "Number of parallel pytest-xdist workers to use" + value: ${{ steps.config-evm-reader.outputs.xdist }} runs: using: "composite" steps: - - name: Get the selected EVM version from the .github/configs/evm.yaml + - name: Get the selected EVM configuration from .github/configs/evm.yaml id: config-evm-reader shell: bash run: | awk "/^${{ inputs.type }}:/{flag=1; next} /^[[:alnum:]]/{flag=0} flag" ./.github/configs/evm.yaml \ | sed 's/ //g' | sed 's/:/=/g' >> "$GITHUB_OUTPUT" - - name: Get the EVM implementation configuration from .github/configs/evm-impl-config.yaml - id: config-evm-impl-config-reader + - name: Apply repo/ref overrides + id: resolved shell: bash + env: + DEFAULT_REPO: ${{ steps.config-evm-reader.outputs.repo }} + DEFAULT_REF: ${{ steps.config-evm-reader.outputs.ref }} + REPO_OVERRIDE: ${{ inputs.repo_override }} + REF_OVERRIDE: ${{ inputs.ref_override }} run: | - awk "/^${{ steps.config-evm-reader.outputs.impl }}:/{flag=1; next} /^[[:alnum:]]/{flag=0} flag" ./.github/configs/evm-impl.yaml \ - | sed 's/ //g' | sed 's/:/=/g' >> "$GITHUB_OUTPUT" + echo "repo=${REPO_OVERRIDE:-$DEFAULT_REPO}" >> "$GITHUB_OUTPUT" + echo "ref=${REF_OVERRIDE:-$DEFAULT_REF}" >> "$GITHUB_OUTPUT" - name: Print Variables for the selected EVM type shell: bash run: | + echo "Type: ${{ inputs.type }}" echo "Implementation: ${{ steps.config-evm-reader.outputs.impl }}" - echo "Repository: ${{ steps.config-evm-reader.outputs.repo }}" - echo "Reference: ${{ steps.config-evm-reader.outputs.ref }}" - echo "EVM Binary: ${{ steps.config-evm-impl-config-reader.outputs.evm-bin }}" - echo "X-Dist parameter: ${{ steps.config-evm-impl-config-reader.outputs.x-dist }}" + echo "Repository: ${{ steps.resolved.outputs.repo }}" + echo "Reference: ${{ steps.resolved.outputs.ref }}" + echo "EVM Binary: ${{ steps.config-evm-reader.outputs.evm-bin }}" + echo "X-Dist parameter: ${{ steps.config-evm-reader.outputs.xdist }}" - name: Skip building for EELS if: steps.config-evm-reader.outputs.impl == 'eels' shell: bash @@ -52,25 +65,19 @@ runs: if: steps.config-evm-reader.outputs.impl == 'geth' uses: ./.github/actions/build-evm-client/geth with: - repo: ${{ steps.config-evm-reader.outputs.repo }} - ref: ${{ steps.config-evm-reader.outputs.ref }} + repo: ${{ steps.resolved.outputs.repo }} + ref: ${{ steps.resolved.outputs.ref }} - name: Build the EVM using EVMONE action if: steps.config-evm-reader.outputs.impl == 'evmone' uses: ./.github/actions/build-evm-client/evmone with: - repo: ${{ steps.config-evm-reader.outputs.repo }} - ref: ${{ steps.config-evm-reader.outputs.ref }} - # `targets` in the evm.yaml must be an inline array to not interfere with `config-evm-reader`'s parsing + repo: ${{ steps.resolved.outputs.repo }} + ref: ${{ steps.resolved.outputs.ref }} + # `targets` in evm.yaml must be an inline array to not interfere with `config-evm-reader`'s parsing targets: ${{ join(fromJSON(steps.config-evm-reader.outputs.targets), ' ') }} - name: Build the EVM using Besu action if: steps.config-evm-reader.outputs.impl == 'besu' uses: ./.github/actions/build-evm-client/besu with: - repo: ${{ steps.config-evm-reader.outputs.repo }} - ref: ${{ steps.config-evm-reader.outputs.ref }} - - name: Build the EVM using EthJS action - if: steps.config-evm-reader.outputs.impl == 'ethjs' - uses: ./.github/actions/build-evm-client/ethjs - with: - repo: ${{ steps.config-evm-reader.outputs.repo }} - ref: ${{ steps.config-evm-reader.outputs.ref }} \ No newline at end of file + repo: ${{ steps.resolved.outputs.repo }} + ref: ${{ steps.resolved.outputs.ref }} diff --git a/.github/actions/build-evm-client/ethjs/action.yaml b/.github/actions/build-evm-client/ethjs/action.yaml deleted file mode 100644 index 2bc21dd7bf5..00000000000 --- a/.github/actions/build-evm-client/ethjs/action.yaml +++ /dev/null @@ -1,37 +0,0 @@ -name: 'Build EthereumJS monorepo' -description: 'Builds the EthereumJS monorepo' -inputs: - repo: - description: 'Source repository to use to build EthereumJS' - required: true - default: 'ethereumjs/ethereumjs-monorepo' - ref: - description: 'Reference to branch, commit, or tag to use to build EthereumJS' - required: true - default: 'master' -runs: - using: "composite" - steps: - - name: Checkout EthereumJS monorepo - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - repository: ${{ inputs.repo }} - ref: ${{ inputs.ref }} - path: ethereumjs - - - name: Setup node - uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 - with: - node-version: 18 - - - name: Build monorepo - shell: bash - run: | - cd $GITHUB_WORKSPACE/ethereumjs - npm ci - - - name: Add t8ntool to $PATH - shell: bash - run: | - echo $GITHUB_WORKSPACE/ethereumjs/packages/vm/test/t8n/ >> $GITHUB_PATH - echo $GITHUB_WORKSPACE/ethereumjs/node_modules/.bin >> $GITHUB_PATH \ No newline at end of file diff --git a/.github/actions/build-fixtures/action.yaml b/.github/actions/build-fixtures/action.yaml index 0c2f197e216..719c3221a75 100644 --- a/.github/actions/build-fixtures/action.yaml +++ b/.github/actions/build-fixtures/action.yaml @@ -13,6 +13,15 @@ inputs: split_label: description: "Label for this fork-range split. Empty for unsplit builds." default: "" + evm: + description: "Override the evm impl. Defaults to the feature's evm-type." + default: "" + evm_repo: + description: "Override the t8n tool repo (e.g. ethereum/go-ethereum)" + default: "" + evm_ref: + description: "Override the t8n tool branch / tag / commit" + default: "" runs: using: "composite" steps: @@ -32,7 +41,9 @@ runs: - uses: ./.github/actions/build-evm-base id: evm-builder with: - type: ${{ steps.properties.outputs.evm-type }} + type: ${{ inputs.evm != '' && inputs.evm || steps.properties.outputs.evm-type }} + repo_override: ${{ inputs.evm_repo }} + ref_override: ${{ inputs.evm_ref }} - name: Install pigz for parallel tarball compression if: inputs.split_label == '' shell: bash @@ -53,9 +64,9 @@ runs: # Allow exit code 5 (NO_TESTS_COLLECTED) for fork ranges with no tests. EXIT_CODE=0 if [ "${{ steps.evm-builder.outputs.impl }}" = "eels" ]; then - uv run fill -n ${{ steps.evm-builder.outputs.x-dist }} ${{ steps.properties.outputs.fill-params }} $FORK_ARGS $OUTPUT_ARG --build-name ${{ inputs.release_name }} --no-html --durations=100 --log-level=DEBUG || EXIT_CODE=$? + uv run fill -n ${{ steps.evm-builder.outputs.xdist }} ${{ steps.properties.outputs.fill-params }} $FORK_ARGS $OUTPUT_ARG --build-name ${{ inputs.release_name }} --no-html --durations=100 --log-level=DEBUG || EXIT_CODE=$? else - uv run fill -n ${{ steps.evm-builder.outputs.x-dist }} --evm-bin=${{ steps.evm-builder.outputs.evm-bin }} ${{ steps.properties.outputs.fill-params }} $FORK_ARGS $OUTPUT_ARG --build-name ${{ inputs.release_name }} --no-html --durations=100 --log-level=DEBUG || EXIT_CODE=$? + uv run fill -n ${{ steps.evm-builder.outputs.xdist }} --evm-bin=${{ steps.evm-builder.outputs.evm-bin }} ${{ steps.properties.outputs.fill-params }} $FORK_ARGS $OUTPUT_ARG --build-name ${{ inputs.release_name }} --no-html --durations=100 --log-level=DEBUG || EXIT_CODE=$? fi if [ "$EXIT_CODE" -ne 0 ] && [ "$EXIT_CODE" -ne 5 ]; then exit "$EXIT_CODE" diff --git a/.github/configs/evm-impl.yaml b/.github/configs/evm-impl.yaml deleted file mode 100644 index 4a70077a9e1..00000000000 --- a/.github/configs/evm-impl.yaml +++ /dev/null @@ -1,15 +0,0 @@ -eels: - evm-bin: null - x-dist: auto -geth: - evm-bin: evm - x-dist: auto -evmone: - evm-bin: evmone-t8n - x-dist: auto -besu: - evm-bin: evmtool - x-dist: 0 -ethjs: - evm-bin: ethereumjs-t8ntool.sh - x-dist: auto diff --git a/.github/configs/evm.yaml b/.github/configs/evm.yaml index c295d43bb86..98269b5c38f 100644 --- a/.github/configs/evm.yaml +++ b/.github/configs/evm.yaml @@ -1,13 +1,31 @@ +benchmark: + impl: geth + repo: ethereum/go-ethereum + ref: master + evm-bin: evm + xdist: auto eels: impl: eels repo: null ref: null -static: + evm-bin: null + xdist: auto +evmone: impl: evmone repo: ethereum/evmone ref: master targets: ["evmone-t8n"] -benchmark: + evm-bin: evmone-t8n + xdist: auto +geth: impl: geth repo: ethereum/go-ethereum - ref: master \ No newline at end of file + ref: master + evm-bin: evm + xdist: auto +besu: + impl: besu + repo: hyperledger/besu + ref: main + evm-bin: evmtool + xdist: 0 diff --git a/.github/configs/feature.yaml b/.github/configs/feature.yaml index cf6a92748d9..38a356f60a9 100644 --- a/.github/configs/feature.yaml +++ b/.github/configs/feature.yaml @@ -1,19 +1,19 @@ -# Unless filling for special features, all features should fill for previous forks (starting from Frontier) too -mainnet: +# Release feature definitions consumed by the `release_fixtures` workflow. +# +# Top-level keys are feature names used verbatim in the release tag +# (`tests-@vX.Y.Z`), except `tests`, which tags as `tests@vX.Y.Z`. +# Any `-devnet` input resolves to the shared `devnet` entry but keeps +# its name in the tag; the devnet number lives in the version (X), not the +# feature name, so this file needs no edits for new devnets. +tests: evm-type: eels - fill-params: --until=BPO2 --generate-all-formats + fill-params: --until=BPO4 --generate-all-formats benchmark: evm-type: benchmark - fill-params: --fork=Osaka --generate-all-formats --gas-benchmark-values 1,5,10,30,60,100,150 ./tests/benchmark/compute --maxprocesses=30 --dist=worksteal - feature_only: true + fill-params: --fork=Amsterdam --generate-all-formats --gas-benchmark-values 1,10,30,60,100,150,200,300 ./tests/benchmark -benchmark_fast: - evm-type: benchmark - fill-params: --fork=Osaka --generate-all-formats --gas-benchmark-values 100 ./tests/benchmark/compute - feature_only: true - -bal: +# Shared entry for all `-devnet` releases; matched by `-devnet` suffix. +devnet: evm-type: eels - fill-params: --fork=Amsterdam - feature_only: true + fill-params: --until=Amsterdam --generate-all-formats diff --git a/.github/configs/fork-ranges.yaml b/.github/configs/fork-ranges.yaml index feb5ed48e38..db167855f61 100644 --- a/.github/configs/fork-ranges.yaml +++ b/.github/configs/fork-ranges.yaml @@ -12,10 +12,7 @@ until: Prague - label: osaka from: Osaka - until: Osaka -- label: bpo - from: BPO1 - until: BPO2 + until: BPO5 - label: amsterdam from: Amsterdam until: Amsterdam diff --git a/.github/scripts/create_release_tarball.py b/.github/scripts/create_release_tarball.py index 1f4e7d47a2f..c63b3fe7c60 100644 --- a/.github/scripts/create_release_tarball.py +++ b/.github/scripts/create_release_tarball.py @@ -7,11 +7,11 @@ """ Create a release tarball from a merged fixture directory. -Archive all ``.json`` and ``.ini`` files under a ``fixtures/`` prefix, +Archive all `.json` and `.ini` files under a `fixtures/` prefix, matching the structure produced by -``execution_testing.cli.pytest_commands.plugins.shared.fixture_output``. +`execution_testing.cli.pytest_commands.plugins.shared.fixture_output`. -Use ``pigz`` for parallel compression when available, otherwise fall +Use `pigz` for parallel compression when available, otherwise fall back to Python's built-in gzip. """ diff --git a/.github/scripts/generate_build_matrix.py b/.github/scripts/generate_build_matrix.py index de8b3b5144a..62382e6e457 100644 --- a/.github/scripts/generate_build_matrix.py +++ b/.github/scripts/generate_build_matrix.py @@ -7,26 +7,33 @@ # ] # /// """ -Generate the build matrix for release fixture workflows. +Validate release inputs and generate the build matrix for release +fixture workflows. -Read `.github/configs/feature.yaml` and emit a flat JSON build matrix -suitable for ``strategy.matrix`` in GitHub Actions. +Usage: `generate_build_matrix.py [branch]`. -Features whose ``fill-params`` contain ``--until`` are split across the +First validate the dispatch inputs (see `validate_inputs`), then read +`.github/configs/feature.yaml` and emit a flat JSON build matrix suitable +for `strategy.matrix` in GitHub Actions. + +Features whose `fill-params` contain `--until` are split across the shared fork ranges defined in `.github/configs/fork-ranges.yaml`. -Features using ``--fork`` (single fork) produce a single unsplit entry. +Features using `--fork` (single fork) produce a single unsplit entry. """ import json import re import sys from pathlib import Path +from typing import NoReturn import yaml FEATURE_CONFIG = Path(".github/configs/feature.yaml") FORK_RANGES_CONFIG = Path(".github/configs/fork-ranges.yaml") +VERSION_RE = re.compile(r"^v[0-9]+\.[0-9]+\.[0-9]+$") + # Canonical fork ordering used to filter fork ranges per feature. FORK_ORDER = [ "Frontier", @@ -49,6 +56,9 @@ "Osaka", "BPO1", "BPO2", + "BPO3", + "BPO4", + "BPO5", "Amsterdam", ] @@ -61,11 +71,70 @@ def load_config(path: Path) -> dict: return yaml.safe_load(f) +def fail(message: str) -> NoReturn: + """Print an error to stderr and exit non-zero.""" + print(f"Error: {message}", file=sys.stderr) + sys.exit(1) + + +def validate_inputs(feature: str, version: str, branch: str) -> None: + """ + Validate the release dispatch inputs before building a matrix. + + Centralize the feature/version checks here so they are unit-testable + rather than living as inline bash in the release workflow. + + For `-devnet` releases the major version (`X` of `vX.Y.Z`) + must equal the devnet number encoded in the release branch, so a + `bal-devnet` release from `bal-devnet-7` must be tagged `v7.*.*`. + """ + if not feature: + fail("feature name is empty") + if not VERSION_RE.match(version): + fail(f"version '{version}' must match vX.Y.Z (e.g. v20.0.0)") + + # A bare `devnet` has no friendly `-` prefix to tag with. + if feature in ("devnet", "-devnet"): + fail("devnet releases require a - prefix, e.g. bal-devnet") + + # `-devnet-`: the devnet index belongs in the version (X of + # vX.Y.Z), not in the feature name. + if "-devnet-" in feature: + suggested_feature, _, suggested_index = feature.rpartition("-") + fail( + "devnet index must go in 'version', not the feature name; " + f"did you mean feature={suggested_feature} " + f"version=v{suggested_index}.0.0?" + ) + + if feature.endswith("-devnet"): + if not branch: + fail( + "devnet releases require a 'branch' input, " + "e.g. branch=bal-devnet-7" + ) + match = re.search(r"(\d+)$", branch) + if not match: + fail( + f"could not parse a devnet number from branch '{branch}' " + "(expected a trailing number, e.g. bal-devnet-7)" + ) + devnet_number = int(match.group(1)) + major = int(version.lstrip("v").split(".")[0]) + if major != devnet_number: + minor_patch = version.split(".", 1)[1] + fail( + f"version major (v{major}) must equal the devnet number " + f"({devnet_number}) from branch '{branch}'; " + f"did you mean version=v{devnet_number}.{minor_patch}?" + ) + + def parse_until_fork(fill_params: str) -> str | None: """ - Extract the ``--until`` value from fill-params. + Extract the `--until` value from fill-params. - Return ``None`` when ``--fork`` is used instead (single-fork + Return `None` when `--fork` is used instead (single-fork feature that should not be split). """ if re.search(r"--fork\b", fill_params): @@ -76,9 +145,9 @@ def parse_until_fork(fill_params: str) -> str | None: def applicable_ranges(fork_ranges: list[dict], until_fork: str) -> list[dict]: """ - Return fork ranges whose ``from`` is at or before *until_fork*. + Return fork ranges whose `from` is at or before *until_fork*. - Clamp the last applicable range's ``until`` to *until_fork* so we + Clamp the last applicable range's `until` to *until_fork* so we never fill beyond the feature's declared boundary. """ limit = FORK_INDEX[until_fork] @@ -130,26 +199,38 @@ def build_matrix( def main() -> None: - """Entry point.""" - if len(sys.argv) != 2: + """Validate the inputs and print the build matrix to stdout.""" + args = sys.argv[1:] + if len(args) < 2: print( - "Usage: generate_build_matrix.py ", + "Usage: generate_build_matrix.py [branch]", file=sys.stderr, ) sys.exit(1) + name = args[0] + version = args[1] + branch = args[2] if len(args) > 2 else "" + + validate_inputs(name, version, branch) + config = load_config(FEATURE_CONFIG) fork_ranges = load_config(FORK_RANGES_CONFIG) or [] - name = sys.argv[1] - if name not in config or not isinstance(config[name], dict): + # `-devnet` releases (e.g. bal-devnet) share the `devnet` entry, + # while keeping their friendly name in the matrix and artifact outputs. + lookup = ( + "devnet" if name.endswith("-devnet") and "devnet" in config else name + ) + + if lookup not in config or not isinstance(config[lookup], dict): print( - f"Error: feature '{name}' not found in {FEATURE_CONFIG}.", + f"Error: feature '{lookup}' not found in {FEATURE_CONFIG}.", file=sys.stderr, ) sys.exit(1) - build, labels = build_matrix(config[name], name, fork_ranges) + build, labels = build_matrix(config[lookup], name, fork_ranges) print(f"build_matrix={json.dumps(build)}") print(f"feature_name={name}") diff --git a/.github/scripts/get_release_props.py b/.github/scripts/get_release_props.py index 577979c31bf..f8505675ed3 100644 --- a/.github/scripts/get_release_props.py +++ b/.github/scripts/get_release_props.py @@ -22,8 +22,14 @@ def get_release_props(release: str) -> None: with open(RELEASE_PROPS_FILE) as f: data = yaml.safe_load(f) if release not in data: - print(f"Error: Release {release} not found in {RELEASE_PROPS_FILE}.") - sys.exit(1) + # `-devnet` releases (e.g. bal-devnet) share the `devnet` entry. + if release.endswith("-devnet") and "devnet" in data: + release = "devnet" + else: + print( + f"Error: Release {release} not found in {RELEASE_PROPS_FILE}." + ) + sys.exit(1) print("\n".join(f"{key}={value}" for key, value in data[release].items())) diff --git a/.github/scripts/merge_index_files.py b/.github/scripts/merge_index_files.py index ea41b299a10..4f5e31e3ca0 100644 --- a/.github/scripts/merge_index_files.py +++ b/.github/scripts/merge_index_files.py @@ -3,7 +3,7 @@ Merge multiple .meta/index.json files from split fixture builds. Accept fixture directories as arguments, load each directory's -``.meta/index.json``, merge them via ``IndexFile.merge()``, and write +`.meta/index.json`, merge them via `IndexFile.merge()`, and write the result to the specified output path. """ diff --git a/.github/scripts/tests/test_release_scripts.py b/.github/scripts/tests/test_release_scripts.py index bd3acc17e8d..cfd92d8d7de 100644 --- a/.github/scripts/tests/test_release_scripts.py +++ b/.github/scripts/tests/test_release_scripts.py @@ -1,7 +1,7 @@ """ Test the CI release helper scripts. -Each test invokes the script via ``uv run`` to validate the actual CLI +Each test invokes the script via `uv run` to validate the actual CLI interface, matching how GitHub Actions calls them. """ @@ -43,12 +43,12 @@ class TestGenerateBuildMatrix: def test_split_feature_produces_entries_per_range(self): """Verify a split feature expands into one entry per range.""" - result = run_script(BUILD_MATRIX_SCRIPT, "mainnet") + result = run_script(BUILD_MATRIX_SCRIPT, "tests", "v24.0.0") assert result.returncode == 0 out = parse_matrix_output(result.stdout) matrix = json.loads(out["build_matrix"]) assert len(matrix) > 1 - assert out["feature_name"] == "mainnet" + assert out["feature_name"] == "tests" assert out["combine_labels"] != "" labels = [e["label"] for e in matrix] assert all(lbl != "" for lbl in labels) @@ -57,7 +57,7 @@ def test_split_feature_produces_entries_per_range(self): def test_unsplit_feature_produces_single_entry(self): """Verify a feature without fork-ranges produces one entry.""" - result = run_script(BUILD_MATRIX_SCRIPT, "benchmark") + result = run_script(BUILD_MATRIX_SCRIPT, "benchmark", "v24.0.0") assert result.returncode == 0 out = parse_matrix_output(result.stdout) matrix = json.loads(out["build_matrix"]) @@ -68,19 +68,21 @@ def test_unsplit_feature_produces_single_entry(self): assert matrix[0]["from_fork"] == "" assert matrix[0]["until_fork"] == "" - def test_feature_only_can_be_requested_explicitly(self): - """Verify feature_only entries work when named directly.""" - result = run_script(BUILD_MATRIX_SCRIPT, "bal") + def test_devnet_name_resolves_to_shared_feature(self): + """Verify a -devnet name resolves to the devnet feature.""" + result = run_script( + BUILD_MATRIX_SCRIPT, "bal-devnet", "v7.0.0", "bal-devnet-7" + ) assert result.returncode == 0 out = parse_matrix_output(result.stdout) matrix = json.loads(out["build_matrix"]) - assert len(matrix) == 1 - assert matrix[0]["feature"] == "bal" - assert out["combine_labels"] == "" + assert out["feature_name"] == "bal-devnet" + # Entries keep the friendly name, not the shared "devnet" key. + assert all(e["feature"] == "bal-devnet" for e in matrix) def test_unknown_feature_fails(self): """Verify error exit for unknown feature name.""" - result = run_script(BUILD_MATRIX_SCRIPT, "nonexistent") + result = run_script(BUILD_MATRIX_SCRIPT, "nonexistent", "v1.0.0") assert result.returncode == 1 assert "not found" in result.stderr @@ -92,7 +94,7 @@ def test_no_args_fails(self): def test_output_is_valid_github_actions_format(self): """Verify output lines are key=value for GITHUB_OUTPUT.""" - result = run_script(BUILD_MATRIX_SCRIPT, "mainnet") + result = run_script(BUILD_MATRIX_SCRIPT, "tests", "v24.0.0") assert result.returncode == 0 lines = result.stdout.strip().splitlines() assert len(lines) == 3 @@ -101,6 +103,74 @@ def test_output_is_valid_github_actions_format(self): assert lines[2].startswith("combine_labels=") +class TestValidateInputs: + """Test the release input validation in generate_build_matrix.py.""" + + def test_bad_version_fails(self): + """Verify a non vX.Y.Z version is rejected.""" + result = run_script(BUILD_MATRIX_SCRIPT, "tests", "24.0.0") + assert result.returncode == 1 + assert "must match vX.Y.Z" in result.stderr + + def test_empty_feature_fails(self): + """Verify an empty feature name is rejected.""" + result = run_script(BUILD_MATRIX_SCRIPT, "", "v1.0.0") + assert result.returncode == 1 + assert "feature name is empty" in result.stderr + + def test_bare_devnet_fails(self): + """Verify a bare `devnet` feature name is rejected.""" + result = run_script( + BUILD_MATRIX_SCRIPT, "devnet", "v7.0.0", "bal-devnet-7" + ) + assert result.returncode == 1 + assert "require a - prefix" in result.stderr + + def test_devnet_index_in_feature_name_fails(self): + """Verify `-devnet-` is rejected with a suggestion.""" + result = run_script( + BUILD_MATRIX_SCRIPT, "bal-devnet-7", "v7.0.0", "bal-devnet-7" + ) + assert result.returncode == 1 + assert "did you mean feature=bal-devnet version=v7.0.0" in ( + result.stderr + ) + + def test_devnet_without_branch_fails(self): + """Verify a `-devnet` release requires a branch.""" + result = run_script(BUILD_MATRIX_SCRIPT, "bal-devnet", "v7.0.0") + assert result.returncode == 1 + assert "require a 'branch' input" in result.stderr + + def test_devnet_branch_without_number_fails(self): + """Verify a devnet branch with no trailing number is rejected.""" + result = run_script( + BUILD_MATRIX_SCRIPT, "bal-devnet", "v7.0.0", "bal-devnet" + ) + assert result.returncode == 1 + assert "could not parse a devnet number" in result.stderr + + def test_devnet_major_must_match_branch_number(self): + """Verify the major version must equal the branch devnet number.""" + result = run_script( + BUILD_MATRIX_SCRIPT, "bal-devnet", "v3.0.0", "bal-devnet-7" + ) + assert result.returncode == 1 + assert "must equal the devnet number" in result.stderr + + def test_devnet_matching_major_passes(self): + """Verify a major equal to the branch devnet number passes.""" + result = run_script( + BUILD_MATRIX_SCRIPT, + "glamsterdam-devnet", + "v5.0.0", + "glamsterdam-devnet-5", + ) + assert result.returncode == 0 + out = parse_matrix_output(result.stdout) + assert out["feature_name"] == "glamsterdam-devnet" + + class TestCreateReleaseTarball: """Test create_release_tarball.py.""" diff --git a/.github/workflows/benchmark.yaml b/.github/workflows/benchmark.yaml index 165bfa52039..56706289b4e 100644 --- a/.github/workflows/benchmark.yaml +++ b/.github/workflows/benchmark.yaml @@ -83,18 +83,3 @@ jobs: # TODO: Add execute remote tests with --gas-benchmark-values # TODO: Add execute remote tests with --fixed-opcode-count - - build-artifact: - name: Build Benchmark Fixture Artifact - needs: [sanity-checks] # TODO: Add execute remote jobs when implemented - if: github.event_name == 'push' - runs-on: [self-hosted-ghr, size-gigachungus-x64] - timeout-minutes: 720 - steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - submodules: true - - - uses: ./.github/actions/build-fixtures - with: - release_name: benchmark_fast diff --git a/.github/workflows/benchmark_release_check.yaml b/.github/workflows/benchmark_release_check.yaml new file mode 100644 index 00000000000..f8e16521d70 --- /dev/null +++ b/.github/workflows/benchmark_release_check.yaml @@ -0,0 +1,20 @@ +name: Benchmark Release Sanity Check + +on: + schedule: + - cron: "0 0 * * 0" + workflow_dispatch: + +jobs: + release-check: + name: Benchmark Release Sanity Check + runs-on: [self-hosted-ghr, size-gigachungus-x64] + timeout-minutes: 720 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + submodules: true + + - uses: ./.github/actions/build-fixtures + with: + release_name: benchmark diff --git a/.github/workflows/release_fixture_feature.yaml b/.github/workflows/release_fixtures.yaml similarity index 53% rename from .github/workflows/release_fixture_feature.yaml rename to .github/workflows/release_fixtures.yaml index 2f333ed6ff9..04621149795 100644 --- a/.github/workflows/release_fixture_feature.yaml +++ b/.github/workflows/release_fixtures.yaml @@ -1,10 +1,32 @@ name: Create Fixture Release on: - push: - tags: - - "tests-*@v*" workflow_dispatch: + inputs: + feature: + description: "Feature name, e.g. tests, benchmark, bal-devnet" + required: true + type: string + version: + description: "Release version, e.g. v20.0.0" + required: true + type: string + branch: + description: "Branch to release from (required for *-devnet features)" + required: false + type: string + evm: + description: "Override the evm impl (e.g. geth, evmone). Defaults to the feature's evm-type." + required: false + type: string + evm_repo: + description: "Override the t8n tool repo (e.g. ethereum/go-ethereum)" + required: false + type: string + evm_ref: + description: "Override the t8n tool branch / tag / commit" + required: false + type: string jobs: setup: @@ -13,19 +35,37 @@ jobs: build_matrix: ${{ steps.matrix.outputs.build_matrix }} feature_name: ${{ steps.matrix.outputs.feature_name }} combine_labels: ${{ steps.matrix.outputs.combine_labels }} + target_sha: ${{ steps.target_sha.outputs.sha }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: false + ref: ${{ inputs.branch }} + - name: Resolve target SHA + id: target_sha + shell: bash + run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - uses: ./.github/actions/setup-uv - - name: Generate build matrix + - name: Validate input and generate build matrix id: matrix shell: bash + env: + INPUT_FEATURE: ${{ inputs.feature }} + INPUT_VERSION: ${{ inputs.version }} + INPUT_BRANCH: ${{ inputs.branch }} + INPUT_EVM: ${{ inputs.evm }} run: | - FEATURE_PREFIX="${GITHUB_REF_NAME//@*/}" - FEATURE_NAME="${FEATURE_PREFIX#tests-}" - uv run -q .github/scripts/generate_build_matrix.py "$FEATURE_NAME" >> "$GITHUB_OUTPUT" + # An `evm` override must name a key in evm.yaml; the feature, + # version and devnet-branch validation lives in (and is unit-tested + # via) generate_build_matrix.py. + if [ -n "$INPUT_EVM" ] && ! grep -qE "^${INPUT_EVM}:" .github/configs/evm.yaml; then + echo "::error::evm '$INPUT_EVM' is not a key in .github/configs/evm.yaml" + exit 1 + fi + + uv run -q .github/scripts/generate_build_matrix.py \ + "$INPUT_FEATURE" "$INPUT_VERSION" "$INPUT_BRANCH" >> "$GITHUB_OUTPUT" build: name: fill (${{ matrix.label || matrix.feature }}) @@ -40,26 +80,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: true - - - name: Fetch lllc - uses: ./.github/actions/fetch-binary - with: - version: v1.0.0 - repo_owner: felix314159 - repo_name: lllc-custom - remote_name: lllc - binary_name: lllc - expected_sha256: 865a0d5379acb3b5471337b5dcf686a2dd71587c6b65b9da6c963de627e0b300 - - - name: Fetch Solidity - uses: ./.github/actions/fetch-binary - with: - version: v0.8.24 - repo_owner: ethereum - repo_name: solidity - remote_name: solc-static-linux - binary_name: solc - expected_sha256: fb03a29a517452b9f12bcf459ef37d0a543765bb3bbc911e70a87d6a37c30d5f + ref: ${{ needs.setup.outputs.target_sha }} - uses: ./.github/actions/build-fixtures with: @@ -67,6 +88,9 @@ jobs: from_fork: ${{ matrix.from_fork }} until_fork: ${{ matrix.until_fork }} split_label: ${{ matrix.label }} + evm: ${{ inputs.evm }} + evm_repo: ${{ inputs.evm_repo }} + evm_ref: ${{ inputs.evm_ref }} combine: name: combine (${{ needs.setup.outputs.feature_name }}) @@ -77,6 +101,7 @@ jobs: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: false + ref: ${{ needs.setup.outputs.target_sha }} - uses: ./.github/actions/setup-uv - name: Install pigz run: sudo apt-get install -y pigz @@ -112,19 +137,28 @@ jobs: - name: Free disk space run: rm -rf split_artifacts/ - name: Create release tarball + id: tarball shell: bash + env: + FEATURE_NAME: ${{ needs.setup.outputs.feature_name }} run: | - uv run -q .github/scripts/create_release_tarball.py combined fixtures_${{ needs.setup.outputs.feature_name }}.tar.gz + if [ "$FEATURE_NAME" = "tests" ]; then + TARBALL="fixtures.tar.gz" + else + TARBALL="fixtures_${FEATURE_NAME}.tar.gz" + fi + uv run -q .github/scripts/create_release_tarball.py combined "$TARBALL" + echo "path=$TARBALL" >> "$GITHUB_OUTPUT" - name: Upload combined fixture tarball uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 with: name: fixtures_${{ needs.setup.outputs.feature_name }} - path: fixtures_${{ needs.setup.outputs.feature_name }}.tar.gz + path: ${{ steps.tarball.outputs.path }} release: runs-on: ubuntu-latest needs: [setup, build, combine] - if: always() && needs.build.result == 'success' && (needs.combine.result == 'success' || needs.combine.result == 'skipped') && startsWith(github.ref, 'refs/tags/tests-') + if: always() && needs.build.result == 'success' && (needs.combine.result == 'success' || needs.combine.result == 'skipped') permissions: contents: write steps: @@ -132,6 +166,7 @@ jobs: with: submodules: false fetch-depth: 0 + ref: ${{ needs.setup.outputs.target_sha }} - name: Download release artifacts shell: bash @@ -142,38 +177,50 @@ jobs: env: GH_TOKEN: ${{ github.token }} - - name: Draft release on EELS (canonical) + - name: Draft release + shell: bash + env: + INPUT_FEATURE: ${{ inputs.feature }} + INPUT_VERSION: ${{ inputs.version }} + TARGET_SHA: ${{ needs.setup.outputs.target_sha }} + GH_TOKEN: ${{ github.token }} run: | - FEATURE_PREFIX="${TAG_NAME%%@*}" + # Release title never carries the `tests-` prefix. + RELEASE_NAME="${INPUT_FEATURE}@${INPUT_VERSION}" + # Git tags namespace fixture releases under `tests-@`, except + # the default `tests` feature which tags as `tests@vX.Y.Z` (no doubled + # prefix). + if [ "$INPUT_FEATURE" = "tests" ]; then + TAG_NAME="$RELEASE_NAME" + TAG_GLOB="tests@v*" + else + TAG_NAME="tests-${RELEASE_NAME}" + TAG_GLOB="tests-${INPUT_FEATURE}@v*" + fi + + # Use the prior release of the same feature as the notes baseline. PREV_TAG=$( - git tag --list "${FEATURE_PREFIX}@v*" --sort=-v:refname \ + git tag --list "$TAG_GLOB" --sort=-v:refname \ | grep -v "^${TAG_NAME}$" \ | head -n 1 \ || true ) - RELEASE_ARGS=(--draft --generate-notes) - if [ "$FEATURE_NAME" != "mainnet" ]; then - RELEASE_ARGS+=(--prerelease) - fi + + # Pin the tag to the single SHA resolved in `setup` so the tag + # lands on the same commit every job built from. + TARGET="$TARGET_SHA" + + # All fixture releases are drafted as prereleases; publish manually. + RELEASE_ARGS=( + --draft + --prerelease + --generate-notes + --target "$TARGET" + --title "$RELEASE_NAME" + ) if [ -n "$PREV_TAG" ]; then RELEASE_ARGS+=(--notes-start-tag "$PREV_TAG") fi - gh release create "$TAG_NAME" "${RELEASE_ARGS[@]}" ./artifacts/**/*.tar.gz - env: - TAG_NAME: ${{ github.ref_name }} - FEATURE_NAME: ${{ needs.setup.outputs.feature_name }} - GH_TOKEN: ${{ github.token }} - - name: Draft release on EEST (mirror) - run: | - EEST_TAG="${TAG_NAME#tests-}" - RELEASE_ARGS=(--repo ethereum/execution-spec-tests --draft) - if [ "$FEATURE_NAME" != "mainnet" ]; then - RELEASE_ARGS+=(--prerelease) - fi - RELEASE_ARGS+=(--notes "This release is mirrored from [ethereum/execution-specs ${TAG_NAME}](https://github.com/ethereum/execution-specs/releases/tag/${TAG_NAME}).") - gh release create "$EEST_TAG" "${RELEASE_ARGS[@]}" ./artifacts/**/*.tar.gz - env: - TAG_NAME: ${{ github.ref_name }} - FEATURE_NAME: ${{ needs.setup.outputs.feature_name }} - GH_TOKEN: ${{ secrets.EEST_RELEASE_TOKEN }} + # Creates the tag on the target commit and the release on success. + gh release create "$TAG_NAME" "${RELEASE_ARGS[@]}" ./artifacts/**/*.tar.gz diff --git a/docs/dev/releasing_tests.md b/docs/dev/releasing_tests.md new file mode 100644 index 00000000000..8432b2cc252 --- /dev/null +++ b/docs/dev/releasing_tests.md @@ -0,0 +1,108 @@ +# Releasing Test Fixtures + +This page covers the mechanics of cutting a test fixture release. For the release types, +their versioning, and consumption guidance, see +[EELS Fixture Releases](../running_tests/releases.md). + +Fixture releases are produced by manually dispatching the +[`release_fixtures.yaml`](https://github.com/ethereum/execution-specs/blob/master/.github/workflows/release_fixtures.yaml) +workflow. There is no tag to push by hand. The workflow builds the fixtures and, only on +success, creates the tag and the (draft) GitHub release. + +```bash +gh workflow run release_fixtures.yaml -f feature= -f version=vX.Y.Z [-f branch=] +``` + +## Inputs + +| Input | Required | Description | +| ---------- | ----------------- | ---------------------------------------------------------------------------------------------------- | +| `feature` | yes | Feature name, e.g. `tests`, `benchmark`, or a `-devnet` name. | +| `version` | yes | Release version `vX.Y.Z` (validated against `^v[0-9]+\.[0-9]+\.[0-9]+$`). Tagged as `tests-@` (the `tests` feature tags as `tests@`). | +| `branch` | devnet only | Branch to build and release from. Optional for non-devnet features; **required** for devnet releases. | +| `evm` | no | Override the evm impl (e.g. `geth`, `evmone`). Defaults to the feature's `evm-type` in `feature.yaml`. | +| `evm_repo` | no | Override the t8n tool repo (e.g. `ethereum/go-ethereum`). | +| `evm_ref` | no | Override the t8n tool branch / tag / commit. | + +`` must be a key in +[`.github/configs/feature.yaml`](https://github.com/ethereum/execution-specs/blob/master/.github/configs/feature.yaml) +(e.g. `tests`, `benchmark`), or a `-devnet` name that resolves to the shared `devnet` +feature. + +Input validation runs in +[`generate_build_matrix.py`](https://github.com/ethereum/execution-specs/blob/master/.github/scripts/generate_build_matrix.py) +(unit-tested) before any fixtures are built, and fails fast on: + +- an empty `feature` or a `version` that is not `vX.Y.Z`; +- a bare `devnet` feature name (must carry a `-` prefix, e.g. `bal-devnet`); +- a `-devnet-` feature name — the devnet index belongs in the `version` major, not + the feature name (so `feature=bal-devnet-7` is rejected in favour of + `feature=bal-devnet version=v7.0.0`); +- a `*-devnet` release missing a `branch`, or whose `version` major does not equal the devnet + number in the branch (so `feature=bal-devnet branch=bal-devnet-7` must use `version=v7.*.*`). + +## Devnet releases + +Devnet releases must use a `-devnet` feature name (e.g. `feature=bal-devnet`) and must +specify the branch to release from. The `version` major must match the devnet number in the +branch: + +```bash +gh workflow run release_fixtures.yaml -f feature=bal-devnet -f version=v7.0.0 -f branch=bal-devnet-7 +``` + +## What the workflow produces + +On success the workflow: + +1. Builds `fixtures_.tar.gz` for the resolved feature (per its `evm-type` and + `fill-params` in `feature.yaml`). +2. Creates the git tag `tests-@vX.Y.Z` (the `tests` feature tags as `tests@vX.Y.Z`, + no doubled prefix) on the released commit (the SHA resolved once from the `branch` HEAD when + given, otherwise the dispatch commit). +3. Publishes a **draft pre-release** to + [`ethereum/execution-specs`](https://github.com/ethereum/execution-specs/releases), titled + `@vX.Y.Z` (no `tests-` prefix), with the fixture tarball(s) attached. + +| Example dispatch | Git tag | Release title | Artifact | +| ---------------- | ------- | ------------- | -------- | +| `feature=tests version=v24.0.0` | `tests@v24.0.0` | `tests@v24.0.0` | `fixtures.tar.gz` | +| `feature=bal-devnet version=v7.0.0 branch=bal-devnet-7` | `tests-bal-devnet@v7.0.0` | `bal-devnet@v7.0.0` | `fixtures_bal-devnet.tar.gz` | + +The release is created as a draft; review and publish it from the GitHub releases page. + +## Cutting a release + +1. **Pick the next version** per the + [Versioning Scheme](../running_tests/releases.md#versioning-scheme) for the feature you're + releasing (e.g. the next `tests` release after `tests@v24.1.0` is `tests@v24.1.1` for a + non-breaking/new-tests bump, or `tests@v24.2.0` for a consensus-breaking spec change). +2. **Dispatch the workflow** from the + [Actions tab](https://github.com/ethereum/execution-specs/actions/workflows/release_fixtures.yaml) + or via the CLI: + + ```bash + gh workflow run release_fixtures.yaml -f feature=tests -f version=v24.1.1 + # devnet releases additionally require the branch (major must match its number): + gh workflow run release_fixtures.yaml -f feature=bal-devnet -f version=v7.0.0 -f branch=bal-devnet-7 + ``` + +3. **Wait for the build to succeed.** On success the workflow creates the + `tests-@vX.Y.Z` tag on the target commit and drafts the GitHub release with the + fixture tarball attached. If any job fails, no tag or release is created — fix the cause + and re-dispatch. +4. **Review and publish the draft.** Open the draft on the + [releases page](https://github.com/ethereum/execution-specs/releases), check the + auto-generated notes (anchored at the prior release on the same feature via + `--notes-start-tag`), and click *Publish release* when ready. + +!!! tip "Release features opt into all fixture formats via `feature.yaml`" + Tarball output (`.tar.gz`) does not by itself include the pre-allocation group formats + (`BlockchainEngineXFixture`, `BlockchainEngineStatefulFixture`). A release feature + requests them by adding `--generate-all-formats` to its `fill-params` in + `.github/configs/feature.yaml`: + ```console + # .tar.gz no longer auto-enables all formats (changed in #2702); request + # them explicitly with --generate-all-formats + uv run fill --generate-all-formats --output=fixtures.tar.gz tests/ + ``` diff --git a/docs/navigation.md b/docs/navigation.md index e5344212474..a2b19e69654 100644 --- a/docs/navigation.md +++ b/docs/navigation.md @@ -45,7 +45,7 @@ * [Debugging Transition Tools](filling_tests/debugging_t8n_tools.md) * [Running Tests](running_tests/index.md) * [Methods of Running Tests](running_tests/running.md) - * [EEST Fixture Releases](running_tests/releases.md) + * [EELS Fixture Releases](running_tests/releases.md) * [Fuzzer Bridge](writing_tests/fuzzer_bridge.md) * [Test Fixture Specifications](running_tests/test_formats/index.md) * [State Tests](running_tests/test_formats/state_test.md) @@ -82,6 +82,7 @@ * [Running Github Actions Locally](dev/test_actions_locally.md) * [Dependencies and Packaging](dev/deps_and_packaging.md) * [Releasing](dev/releasing.md) + * [Releasing Test Fixtures](dev/releasing_tests.md) * [Library Reference](library/index.md) * [EEST CLI Tools](library/cli/index.md) * [eest](library/cli/eest.md) diff --git a/docs/running_tests/releases.md b/docs/running_tests/releases.md index 9907c64a81c..b8b7a11b6b2 100644 --- a/docs/running_tests/releases.md +++ b/docs/running_tests/releases.md @@ -1,8 +1,71 @@ -# EEST Fixture Releases - -## Formats and Release Layout - -@ethereum/execution-specs releases contain JSON test fixtures in various formats. Note that transaction type tests are executed directly from Python source using the [`execute`](./execute/index.md) command. +# EELS Fixture Releases + +Test fixtures are published as **feature-scoped releases** on the +[`ethereum/execution-specs`](https://github.com/ethereum/execution-specs/releases) +repository — `tests@vX.Y.Z`, `-devnet@vX.Y.Z`, and `benchmark@vX.Y.Z`. Each is a +self-contained `.tar.gz` of JSON fixtures that execution clients consume in CI. + +This page describes the release types, their versioning, the fixture formats they contain, +and how to consume them. For the release *mechanics* — how to cut a new release — see +[Releasing Test Fixtures](../dev/releasing_tests.md). + +!!! note "Fixture releases vs. the spec-package `vX.Y.Z` tags" + `ethereum/execution-specs` also publishes Python *spec package* releases tagged + `vX.Y.Z` (e.g. [`v2.20.0`](https://github.com/ethereum/execution-specs/releases/tag/v2.20.0)). + Those contain **no test fixtures** — they are the executable specification package only. + Fixture releases are the feature-scoped tags described on this page and are never + attached to the `vX.Y.Z` package tags. + +## Test Release Types + +Fixtures are released as independent types. Each type has its own tag namespace, artifact, +and cadence. + +| Type | Tag | Artifact | Scope | Built from | +| --------- | ---------------------- | ------------------------------- | ------------------------------------------------------------------------------ | ----------------------- | +| Tests | `tests@vX.Y.Z` | `fixtures.tar.gz` | All forks, all tests (eventually including `ethereum/tests` state tests) | latest `forks/*` branch | +| Devnet | `-devnet@vX.Y.Z` | `fixtures_-devnet.tar.gz` | All forks, all tests, for an upcoming-fork feature under active devnet testing | the devnet branch | +| Benchmark | `benchmark@vX.Y.Z` | `fixtures_benchmark.tar.gz` | EVM benchmarking tests | latest `forks/*` branch | + +- **Tests** releases track clients' production branches and are tagged frequently (roughly + once or twice a week). They are the "must pass" release for mainnet CI, and supersede the + old `fixtures_stable` / `fixtures_develop` artifacts. +- **Devnet** releases target a specific feature under active development (e.g. `bal-devnet`). + They are advisory/non-blocking and may not yet cover every EIP; see the corresponding + release notes for the coverage provided. +- **Benchmark** (and, in future, zkEVM) releases are produced separately for their + specialized consumers. + +## Versioning Scheme + +Release tags use the form `@v..`. The underlying git tag is prefixed with +`tests-` to namespace it apart from the spec-package `vX.Y.Z` tags (e.g. +`tests-bal-devnet@v7.0.0`), except the default `tests` feature, which tags as +`tests@v..` directly (no doubled prefix). The GitHub release title always omits the +`tests-` prefix (`bal-devnet@v7.0.0`). + +`X` identifies the fork or devnet a release targets; `Y` and `Z` order changes within that +target: + +| Component | Tests | Devnet | Benchmark | +| --------- | --------------------------------------------------- | ---------------------------------------------------- | ------------------------------- | +| `X` | Fork number | Devnet number | Fork number (mirrors target) | +| `Y` | Consensus-breaking spec change targeting fork `X` | Consensus-breaking spec change targeting devnet `X` | Mirrors the targeted feature | +| `Z` | Non-breaking change (refactor), new/modified tests | Non-breaking change (refactor), new/modified tests | Moves freely at its own pace | + +A client targeting fork/devnet `X` should take the release with **major == `X`**, the latest +**minor**, and ideally the latest **patch**. The major alone tells you whether a release is +relevant to you, and a bump in `Y` (e.g. `v7.0.0` → `v7.1.0`) signals a consensus-breaking +spec change in *your* target — read the release notes before adopting it. + +This also lets two devnets of the same feature be maintained in parallel — e.g. `v3.0.1` +alongside `v7.0.0` — without ambiguity, the same way `2.x` and `3.x` coexist under semver. + +## Fixture Formats + +Fixture releases contain JSON test fixtures in various formats. Note that transaction type +tests are executed directly from Python source using the [`execute`](./execute/index.md) +command. | Format | Consumed by the client | Location in `.tar.gz` release | | -------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------- | @@ -10,118 +73,40 @@ | [Blockchain Tests](./test_formats/blockchain_test.md) | - directly via a `blocktest`-like command
(e.g., [go-ethereum/cmd/evm/blockrunner.go](https://github.com/ethereum/go-ethereum/blob/4bb097b7ffc32256791e55ff16ca50ef83c4609b/cmd/evm/blockrunner.go))
- using the [eels/consume-rlp Simulator](./running.md#rlp) via block import | `./fixtures/blockchain_tests/` | | [Blockchain Engine Tests](./test_formats/blockchain_test_engine.md) | - using the [eels/consume-engine Simulator](./running.md#engine) and the Engine API | `./fixtures/blockchain_tests_engine/` | | [Transaction Tests](./test_formats/transaction_test.md) | - using a new simulator coming soon | None; executed directly from Python source,
using a release tag | -| Blob Transaction Tests | - using the [eels/execute-blobs Simulator](./execute/hive.md#the-eelsexecute-blobs-simulator) and | None; executed directly from Python source,
using a release tag | - -## Fixture Output Directory Structure - -Inside each format directory, fixtures are grouped by **target fork**. - -The top-level subdirectory identifies the fork **under test**. Below it, -fixtures mirror the `./tests/` source layout: each directory corresponds -to the fork where the functionality was originally introduced. Because -tests declare `valid_from`, a single target fork directory contains -fixtures from every prior fork whose tests are still valid at that fork. - -### Consensus fixture layout - -```text -fixtures/ -└── blockchain_tests/ - ├── for_prague/ # filled targeting Prague - │ ├── istanbul/ # tests introduced in Istanbul - │ │ └── eip1344_chainid/... - │ ├── cancun/ # tests introduced in Cancun - │ │ └── eip4844_blobs/... - │ └── prague/ # tests introduced in Prague - │ └── eip7702_set_code_tx/... - └── for_osaka/ # filled targeting Osaka - ├── istanbul/ - │ └── eip1344_chainid/... - ├── cancun/ - │ └── eip4844_blobs/... - ├── prague/ - │ └── eip7702_set_code_tx/... - └── osaka/ # tests introduced in Osaka - └── eip7692_eof_v1/... -``` - -Other format directories (`state_tests/`, `blockchain_tests_engine/`) -follow the same layout. - -### Benchmark fixture layout - -When filling with `--gas-benchmark-values`, benchmark tests additionally -include the gas limit in the subdirectory name (`for_{fork}_at_{gas}M`, -where `{gas}` is in millions, zero-padded to four digits), with one -subdirectory per gas value: - -```text -fixtures/ -└── blockchain_tests/ - ├── for_osaka_at_0001M/ # 1M gas benchmark - │ └── benchmark/compute/... - └── for_osaka_at_0002M/ # 2M gas benchmark - └── benchmark/compute/... -``` - -When filling with `--fixed-opcode-count`, the opcode count replaces the -gas limit in the subdirectory name (`for_{fork}_at_opcount_{N}K`, where -`{N}` is in thousands and may include decimals): - -```text -fixtures/ -└── blockchain_tests/ - ├── for_osaka_at_opcount_10K/ # 10K opcodes - │ └── benchmark/compute/... - └── for_osaka_at_opcount_20K/ # 20K opcodes - └── benchmark/compute/... -``` - -## Release URLs and Tarballs +| Blob Transaction Tests | - using the [eels/execute-blobs Simulator](./execute/hive.md#the-eelsexecute-blobs-simulator) | None; executed directly from Python source,
using a release tag | -### Versioning Scheme +## Pinning Guidance -EEST framework and test sources and fixture releases are tagged use a semantic versioning scheme, `>v..` as following: +Mapped to a typical client CI setup: -- ``: An existing fixture format has changed (potentially breaking change). Action must be taken by client teams to ensure smooth upgrade to the new format. -- ``: Additional coverage (new tests, or a new format) have been added to the release. -- ``: A bug-fix release; an error in the tests or framework has been patched. +- **Blocking gate (current + past forks).** Pin a specific `tests@vX.Y.Z` for reproducible, + no-rug-pull CI on your `master`/production branch, or follow the latest `tests` release if + a moving target is acceptable. This supersedes the old `fixtures_develop` / `fixtures_stable` + artifacts. +- **Non-blocking gate (next fork).** Use the current `-devnet@vX.Y.Z` release for the + upcoming fork's active devnet (e.g. `bal-devnet@vX.Y.Z`). Treat it as advisory — devnet + coverage changes rapidly and should not block merges. -Please see below for an explanation of the optional `` that is used in pre-releases. +!!! note "Devnet vs. tests overlap" + Devnet releases are filled for all forks/tests, so they overlap with the `tests` release. + If your blocking gate already runs a `tests` release, the devnet gate re-runs that shared + coverage. Deduplicating that overlap is a consumer-side concern handled when + resolving/consuming releases. -### Standard Releases +## Downloading Releases -Releases are published on the @ethereum/execution-specs [releases](https://github.com/ethereum/execution-specs/releases) page. Standard releases are tagged using the format `vX.Y.Z` (they don't have a ``). +The [`consume cache`](./consume/cache.md) command resolves EELS release and pre-release tags +to release URLs and downloads them — for example: -For standard releases, two tarballs are available: - -| Release Artifact | Fork/feature scope | -| ------------------------- | ----------------------------------------------------------------------- | -| `fixtures_stable.tar.gz` | Tests for all forks up to and including the last deployed ("stable") mainnet fork ("must pass") | -| `fixtures_develop.tar.gz` | Tests for all forks up to and including the last development fork | - -I.e., `fixtures_develop` are a superset of `fixtures_stable`. - -!!! tip "Release features opt into all fixture formats via `feature.yaml`" - Tarball output (`.tar.gz`) does not by itself include the pre-allocation group formats (`BlockchainEngineXFixture`, `BlockchainEngineStatefulFixture`). A release feature requests them by adding `--generate-all-formats` to its `fill-params` in `.github/configs/feature.yaml`: - ```console - uv run fill --generate-all-formats --output=fixtures_stable.tar.gz tests/ - ``` - -### Pre-Release and Devnet Releases - -Intermediate releases that target specific subsets of features or tests under active development are published at @ethereum/execution-specs [releases](https://github.com/ethereum/execution-specs/releases). - -These releases are tagged using the format `@vX.Y.Z`. - - -Examples: - -- [`fusaka-devnet-1@v1.0.0`](https://github.com/ethereum/execution-spec-tests/releases/tag/fusaka-devnet-1%40v1.0.0) - this fixture release contains tests adhering to the [Fusaka Devnet 1 spec](https://notes.ethereum.org/@ethpandaops/fusaka-devnet-1). -- [`benchmark@v0.0.3`](https://github.com/ethereum/execution-spec-tests/releases/tag/benchmark%40v0.0.3) - this fixture release contains tests specifically aimed at benchmarking EVMs. +```bash +uv run consume cache --input=tests@latest +uv run consume cache --input=bal-devnet@v7.0.0 +``` -Devnet releases should be treated as WIP and may not yet contain full test coverage (or even coverage for all EIPs). The coverage provided by these releases is detailed in the corresponding release notes. +Raw tarballs can also be fetched directly with the GitHub CLI: -### Help Downloading Releases +```bash +gh release download tests-bal-devnet@v7.0.0 --repo ethereum/execution-specs --pattern '*.tar.gz' +``` -The [`consume cache`](./consume/cache.md) command can be used to resolve EEST release and pre-release tags to release URLs and download them. +To create a release, see [Releasing Test Fixtures](../dev/releasing_tests.md). diff --git a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/tests/test_benchmarking.py b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/tests/test_benchmarking.py index f611ccdb4d1..86b7e00dfe8 100644 --- a/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/tests/test_benchmarking.py +++ b/packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/tests/test_benchmarking.py @@ -193,7 +193,7 @@ def test_benchmarking_mode_configured_with_option( ) assert result.ret == 0 - assert any("6 tests collected" in line for line in result.outlines) + assert any("3 tests collected" in line for line in result.outlines) # Check that the test names include the benchmark gas values assert any("benchmark-gas-value_10M" in line for line in result.outlines) assert any("benchmark-gas-value_20M" in line for line in result.outlines) @@ -379,8 +379,8 @@ def test_benchmarking_mode_not_configured_without_option( ) assert result.ret == 0 - # Should generate normal test variants (2) without parametrization - assert any("2 tests collected" in line for line in result.outlines) + # Should generate normal test variants (1) without parametrization + assert any("1 test collected" in line for line in result.outlines) assert not any( "benchmark-gas-value_10M" in line for line in result.outlines ) @@ -523,7 +523,7 @@ def test_repricing_marker_with_kwargs_filters_parametrized_tests( ] # test_parametrized_with_repricing_kwargs should only have ADD variants - # (multiple test types like blockchain_test and blockchain_test_engine) + # (the blockchain_test fixture type) kwargs_test_lines = [ line for line in collected_lines @@ -863,29 +863,29 @@ def test_fixed_opcode_count_invalid_regex_raises_error() -> None: @pytest.mark.parametrize( "config_counts,expected_tests,expected_ids", [ - pytest.param([1], 2, ["opcount_1"], id="single_int"), + pytest.param([1], 1, ["opcount_1"], id="single_int"), pytest.param( [1, 2, 3], - 6, + 3, ["opcount_1", "opcount_2", "opcount_3"], id="multiple_ints", ), - pytest.param([0.5], 2, ["opcount_0.5"], id="single_float"), + pytest.param([0.5], 1, ["opcount_0.5"], id="single_float"), pytest.param( [0.5, 1, 2], - 6, + 3, ["opcount_0.5", "opcount_1", "opcount_2"], id="multiple_floats", ), pytest.param( [1, 0.5, 2], - 6, + 3, ["opcount_1", "opcount_0.5", "opcount_2"], id="mixed_int_float", ), pytest.param( [1, 2, 3, 5], - 8, + 4, ["opcount_1", "opcount_2", "opcount_3", "opcount_5"], id="four_ints", ), @@ -943,7 +943,7 @@ def test_fixed_opcode_count_config_file_parametrized( ) assert result.ret == 0 - # Check expected number of tests (2 test types * len(counts)) + # Check expected number of tests (1 test type * len(counts)) assert any(f"{expected_tests} passed" in line for line in result.outlines) # Check opcode count IDs are present for expected_id in expected_ids: diff --git a/packages/testing/src/execution_testing/specs/benchmark.py b/packages/testing/src/execution_testing/specs/benchmark.py index 197cb5063eb..e9e1fa7345a 100644 --- a/packages/testing/src/execution_testing/specs/benchmark.py +++ b/packages/testing/src/execution_testing/specs/benchmark.py @@ -31,7 +31,6 @@ TransactionPost, ) from execution_testing.fixtures import ( - BlockchainEngineFixture, BlockchainEngineXFixture, BlockchainFixture, FixtureFormat, @@ -316,7 +315,6 @@ class BenchmarkTest(BaseTest): Sequence[FixtureFormat | LabeledFixtureFormat] ] = [ BlockchainFixture, - BlockchainEngineFixture, BlockchainEngineXFixture, ] @@ -329,9 +327,6 @@ class BenchmarkTest(BaseTest): ] supported_markers: ClassVar[Dict[str, str]] = { - "blockchain_test_engine_only": ( - "Only generate a blockchain test engine fixture" - ), "blockchain_test_only": "Only generate a blockchain test fixture", "repricing": "Mark test as reference test for gas repricing analysis", } @@ -430,8 +425,6 @@ def discard_fixture_format_by_marks( if "blockchain_test_only" in [m.name for m in markers]: return fixture_format != BlockchainFixture - if "blockchain_test_engine_only" in [m.name for m in markers]: - return fixture_format != BlockchainEngineFixture return False def get_genesis_environment(self) -> Environment: