feat(ci): overhaul fixture releases - benchmark

.github/actions/build-evm-base/action.yaml

@@ -2,48 +2,61 @@ name: 'Build EVM'
 description: 'Resolves and builds the requested EVM binary by name'
 inputs:
   type:
-    description: 'Type of EVM binary to build'
+    description: 'Type of EVM binary to build (key in .github/configs/evm.yaml)'
     required: true
-    default: 'main'
+  repo_override:
+    description: 'Override the repo from evm.yaml (e.g. ethereum/go-ethereum)'
+    required: false
+    default: ''
+  ref_override:
+    description: 'Override the ref/branch/commit from evm.yaml'
+    required: false
+    default: ''
 outputs:
   impl:
     description: "Implementation of EVM binary to build"
     value: ${{ steps.config-evm-reader.outputs.impl }}
   repo:
     description: "Repository to use to build the EVM binary"
-    value: ${{ steps.config-evm-reader.outputs.repo }}
+    value: ${{ steps.resolved.outputs.repo }}
   ref:
     description: "Reference to branch, commit, or tag to use to build the EVM binary"
-    value: ${{ steps.config-evm-reader.outputs.ref }}
+    value: ${{ steps.resolved.outputs.ref }}
   evm-bin:
     description: "Binary name of the evm tool to use"
-    value: ${{ steps.config-evm-impl-config-reader.outputs.evm-bin }}
-  x-dist:
-    description: "Binary name of the evm tool to use"
-    value: ${{ steps.config-evm-impl-config-reader.outputs.x-dist }}
+    value: ${{ steps.config-evm-reader.outputs.evm-bin }}
+  xdist:
+    description: "Number of parallel pytest-xdist workers to use"
+    value: ${{ steps.config-evm-reader.outputs.xdist }}
 runs:
   using: "composite"
   steps:
-    - name: Get the selected EVM version from the .github/configs/evm.yaml
+    - name: Get the selected EVM configuration from .github/configs/evm.yaml
       id: config-evm-reader
       shell: bash
       run: |
         awk "/^${{ inputs.type }}:/{flag=1; next} /^[[:alnum:]]/{flag=0} flag" ./.github/configs/evm.yaml \
         | sed 's/ //g' | sed 's/:/=/g' >> "$GITHUB_OUTPUT"
-    - name: Get the EVM implementation configuration from .github/configs/evm-impl-config.yaml
-      id: config-evm-impl-config-reader
+    - name: Apply repo/ref overrides
+      id: resolved
       shell: bash
+      env:
+        DEFAULT_REPO: ${{ steps.config-evm-reader.outputs.repo }}
+        DEFAULT_REF: ${{ steps.config-evm-reader.outputs.ref }}
+        REPO_OVERRIDE: ${{ inputs.repo_override }}
+        REF_OVERRIDE: ${{ inputs.ref_override }}
       run: |
-        awk "/^${{ steps.config-evm-reader.outputs.impl }}:/{flag=1; next} /^[[:alnum:]]/{flag=0} flag" ./.github/configs/evm-impl.yaml \
-        | sed 's/ //g' | sed 's/:/=/g' >> "$GITHUB_OUTPUT"
+        echo "repo=${REPO_OVERRIDE:-$DEFAULT_REPO}" >> "$GITHUB_OUTPUT"
+        echo "ref=${REF_OVERRIDE:-$DEFAULT_REF}" >> "$GITHUB_OUTPUT"
     - name: Print Variables for the selected EVM type
       shell: bash
       run: |
+        echo "Type: ${{ inputs.type }}"
         echo "Implementation: ${{ steps.config-evm-reader.outputs.impl }}"
-        echo "Repository: ${{ steps.config-evm-reader.outputs.repo }}"
-        echo "Reference: ${{ steps.config-evm-reader.outputs.ref }}"
-        echo "EVM Binary: ${{ steps.config-evm-impl-config-reader.outputs.evm-bin }}"
-        echo "X-Dist parameter: ${{ steps.config-evm-impl-config-reader.outputs.x-dist }}"
+        echo "Repository: ${{ steps.resolved.outputs.repo }}"
+        echo "Reference: ${{ steps.resolved.outputs.ref }}"
+        echo "EVM Binary: ${{ steps.config-evm-reader.outputs.evm-bin }}"
+        echo "X-Dist parameter: ${{ steps.config-evm-reader.outputs.xdist }}"
     - name: Skip building for EELS
       if: steps.config-evm-reader.outputs.impl == 'eels'
       shell: bash
@@ -52,25 +65,19 @@ runs:
       if: steps.config-evm-reader.outputs.impl == 'geth'
       uses: ./.github/actions/build-evm-client/geth
       with:
-        repo: ${{ steps.config-evm-reader.outputs.repo }}
-        ref: ${{ steps.config-evm-reader.outputs.ref }}
+        repo: ${{ steps.resolved.outputs.repo }}
+        ref: ${{ steps.resolved.outputs.ref }}
     - name: Build the EVM using EVMONE action
       if: steps.config-evm-reader.outputs.impl == 'evmone'
       uses: ./.github/actions/build-evm-client/evmone
       with:
-        repo: ${{ steps.config-evm-reader.outputs.repo }}
-        ref: ${{ steps.config-evm-reader.outputs.ref }}
-        # `targets` in the evm.yaml must be an inline array to not interfere with `config-evm-reader`'s parsing
+        repo: ${{ steps.resolved.outputs.repo }}
+        ref: ${{ steps.resolved.outputs.ref }}
+        # `targets` in evm.yaml must be an inline array to not interfere with `config-evm-reader`'s parsing
         targets: ${{ join(fromJSON(steps.config-evm-reader.outputs.targets), ' ') }}
     - name: Build the EVM using Besu action
       if: steps.config-evm-reader.outputs.impl == 'besu'
       uses: ./.github/actions/build-evm-client/besu
       with:
-        repo: ${{ steps.config-evm-reader.outputs.repo }}
-        ref: ${{ steps.config-evm-reader.outputs.ref }}
-    - name: Build the EVM using EthJS action
-      if: steps.config-evm-reader.outputs.impl == 'ethjs'
-      uses: ./.github/actions/build-evm-client/ethjs
-      with:
-        repo: ${{ steps.config-evm-reader.outputs.repo }}
-        ref: ${{ steps.config-evm-reader.outputs.ref }}
+        repo: ${{ steps.resolved.outputs.repo }}
+        ref: ${{ steps.resolved.outputs.ref }}

.github/actions/build-fixtures/action.yaml

@@ -13,6 +13,15 @@ inputs:
   split_label:
     description: "Label for this fork-range split. Empty for unsplit builds."
     default: ""
+  evm:
+    description: "Override the evm impl. Defaults to the feature's evm-type."
+    default: ""
+  evm_repo:
+    description: "Override the t8n tool repo (e.g. ethereum/go-ethereum)"
+    default: ""
+  evm_ref:
+    description: "Override the t8n tool branch / tag / commit"
+    default: ""
 runs:
   using: "composite"
   steps:
@@ -32,7 +41,9 @@ runs:
     - uses: ./.github/actions/build-evm-base
       id: evm-builder
       with:
-        type: ${{ steps.properties.outputs.evm-type }}
+        type: ${{ inputs.evm != '' && inputs.evm || steps.properties.outputs.evm-type }}
+        repo_override: ${{ inputs.evm_repo }}
+        ref_override: ${{ inputs.evm_ref }}
     - name: Install pigz for parallel tarball compression
       if: inputs.split_label == ''
       shell: bash
@@ -53,9 +64,9 @@ runs:
         # Allow exit code 5 (NO_TESTS_COLLECTED) for fork ranges with no tests.
         EXIT_CODE=0
         if [ "${{ steps.evm-builder.outputs.impl }}" = "eels" ]; then
-          uv run fill -n ${{ steps.evm-builder.outputs.x-dist }} ${{ steps.properties.outputs.fill-params }} $FORK_ARGS $OUTPUT_ARG --build-name ${{ inputs.release_name }} --no-html --durations=100 --log-level=DEBUG || EXIT_CODE=$?
+          uv run fill -n ${{ steps.evm-builder.outputs.xdist }} ${{ steps.properties.outputs.fill-params }} $FORK_ARGS $OUTPUT_ARG --build-name ${{ inputs.release_name }} --no-html --durations=100 --log-level=DEBUG || EXIT_CODE=$?
         else
-          uv run fill -n ${{ steps.evm-builder.outputs.x-dist }} --evm-bin=${{ steps.evm-builder.outputs.evm-bin }} ${{ steps.properties.outputs.fill-params }} $FORK_ARGS $OUTPUT_ARG --build-name ${{ inputs.release_name }} --no-html --durations=100 --log-level=DEBUG || EXIT_CODE=$?
+          uv run fill -n ${{ steps.evm-builder.outputs.xdist }} --evm-bin=${{ steps.evm-builder.outputs.evm-bin }} ${{ steps.properties.outputs.fill-params }} $FORK_ARGS $OUTPUT_ARG --build-name ${{ inputs.release_name }} --no-html --durations=100 --log-level=DEBUG || EXIT_CODE=$?
         fi
         if [ "$EXIT_CODE" -ne 0 ] && [ "$EXIT_CODE" -ne 5 ]; then
           exit "$EXIT_CODE"

.github/configs/evm.yaml

@@ -1,13 +1,31 @@
+benchmark:
+  impl: geth
+  repo: ethereum/go-ethereum
+  ref: master
+  evm-bin: evm
+  xdist: auto
 eels:
   impl: eels
   repo: null
   ref: null
-static:
+  evm-bin: null
+  xdist: auto
+evmone:
   impl: evmone
   repo: ethereum/evmone
   ref: master
   targets: ["evmone-t8n"]
-benchmark:
+  evm-bin: evmone-t8n
+  xdist: auto
+geth:
   impl: geth
   repo: ethereum/go-ethereum
-  ref: master
+  ref: master
+  evm-bin: evm
+  xdist: auto
+besu:
+  impl: besu
+  repo: hyperledger/besu
+  ref: main
+  evm-bin: evmtool
+  xdist: 0

.github/configs/feature.yaml

@@ -1,19 +1,19 @@
-# Unless filling for special features, all features should fill for previous forks (starting from Frontier) too
-mainnet:
+# Release feature definitions consumed by the `release_fixtures` workflow.
+#
+# Top-level keys are feature names used verbatim in the release tag
+# (`tests-<feature>@vX.Y.Z`), except `tests`, which tags as `tests@vX.Y.Z`.
+# Any `<feat>-devnet` input resolves to the shared `devnet` entry but keeps
+# its name in the tag; the devnet number lives in the version (X), not the
+# feature name, so this file needs no edits for new devnets.
+tests:
   evm-type: eels
-  fill-params: --until=BPO2 --generate-all-formats
+  fill-params: --until=BPO4 --generate-all-formats
 
 benchmark:
   evm-type: benchmark
-  fill-params: --fork=Osaka --generate-all-formats --gas-benchmark-values 1,5,10,30,60,100,150 ./tests/benchmark/compute --maxprocesses=30 --dist=worksteal
-  feature_only: true
+  fill-params: --fork=Amsterdam --generate-all-formats --gas-benchmark-values 1,10,30,60,100,150,200,300 ./tests/benchmark
 
-benchmark_fast:
-  evm-type: benchmark
-  fill-params: --fork=Osaka --generate-all-formats --gas-benchmark-values 100 ./tests/benchmark/compute
-  feature_only: true
-
-bal:
+# Shared entry for all `<feat>-devnet` releases; matched by `-devnet` suffix.
+devnet:
   evm-type: eels
-  fill-params: --fork=Amsterdam
-  feature_only: true
+  fill-params: --until=Amsterdam --generate-all-formats

.github/configs/fork-ranges.yaml

@@ -12,10 +12,7 @@
   until: Prague
 - label: osaka
   from: Osaka
-  until: Osaka
-- label: bpo
-  from: BPO1
-  until: BPO2
+  until: BPO5
 - label: amsterdam
   from: Amsterdam
   until: Amsterdam

.github/scripts/create_release_tarball.py

@@ -7,11 +7,11 @@
 """
 Create a release tarball from a merged fixture directory.
 
-Archive all ``.json`` and ``.ini`` files under a ``fixtures/`` prefix,
+Archive all `.json` and `.ini` files under a `fixtures/` prefix,
 matching the structure produced by
-``execution_testing.cli.pytest_commands.plugins.shared.fixture_output``.
+`execution_testing.cli.pytest_commands.plugins.shared.fixture_output`.
 
-Use ``pigz`` for parallel compression when available, otherwise fall
+Use `pigz` for parallel compression when available, otherwise fall
 back to Python's built-in gzip.
 """