From 4e5946fddb88f7a5b807b1180d9b344de72059a9 Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Tue, 30 Jun 2026 16:45:01 +0200 Subject: [PATCH 1/2] feat(eest): build the fill image from an embedded Dockerfile by default Make builder.eest_payloads.fill_image and fill_dockerfile both optional: when neither is set, benchmarkoor builds the fill image from a Dockerfile.eest-filler embedded in the binary (go:embed). The Dockerfile copies nothing from the build context, so it builds against an empty temp dir written by resolveFillDockerfile (cleaned up after the build). Moves Dockerfile.eest-filler into pkg/builder so it can be embedded, and drops the now-unnecessary fill_dockerfile line from the example configs, docs, and config.example.yaml. --- config.example.yaml | 12 ++--- docs/configuration.md | 15 +++--- ...te-actor-eest.full.amsterdam.stateful.yaml | 3 +- ...e-actor-eest.simple.amsterdam.compute.yaml | 6 +-- ...-actor-eest.simple.amsterdam.stateful.yaml | 6 +-- ...state-actor-eest.simple.osaka.compute.yaml | 6 +-- .../builder/Dockerfile.eest-filler | 22 +++++---- pkg/builder/eest_payloads.go | 47 ++++++++++++++++++- pkg/builder/eest_payloads_test.go | 37 +++++++++++++++ pkg/config/config.go | 23 ++++----- pkg/config/config_test.go | 47 +++++++++++++++++-- 11 files changed, 175 insertions(+), 49 deletions(-) rename Dockerfile.eest-filler => pkg/builder/Dockerfile.eest-filler (69%) diff --git a/config.example.yaml b/config.example.yaml index 0ec1ce07..376c3cdf 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -275,15 +275,11 @@ runner: # # Boots a filler client on a copy of a snapshot datadir (e.g. a state_actor # # output_dir above), records engine-API payloads, and writes fixtures that # # `run` consumes via tests.source.eest_fixtures. Runs after state_actor. -# # Only geth supports fill-stateful's testing_buildBlockV1 today. Build the -# # fill image from the repo's Dockerfile.eest-filler (execution-specs #2637): -# # docker build -f Dockerfile.eest-filler -t ghcr.io/your-org/eest-fill-stateful:latest . +# # The fill image (uv/python toolchain) is built from a Dockerfile embedded in +# # the binary by default — nothing to publish. Optionally pull or customise it. # eest_payloads: -# fill_image: ghcr.io/your-org/eest-fill-stateful:latest -# # Instead of a pre-built fill_image, point at a Dockerfile and benchmarkoor -# # builds it (with the container runtime) at build time, tagging it fill_image -# # if set or a default local tag. One of fill_image / fill_dockerfile is required. -# # fill_dockerfile: Dockerfile.eest-filler +# # fill_image: ghcr.io/your-org/eest-fill-stateful:latest # pull a pre-built image instead +# # fill_dockerfile: pkg/builder/Dockerfile.eest-filler # or build from a custom Dockerfile # # pull_policy: always # always | if-not-present | never # # container_runtime: docker # defaults to runner.container_runtime, then docker # # jwt: # Engine API secret shared with the filler (default: built-in) diff --git a/docs/configuration.md b/docs/configuration.md index 40c38efd..af6766ee 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -1646,16 +1646,17 @@ builder: > **Filler client:** `geth` (`ethpandaops/geth:master`) is the production-ready filler. `nethermind` (`nethermindeth/nethermind:master`) also works — it implements `testing_buildBlockV1` with correct EIP-7928 block-access-lists, and `fill-stateful`'s per-test rewind falls back to `debug_resetHead` for it (nethermind has no `debug_setHead`). `besu` works too with an image carrying the merged `TestingBuildBlockV1` coinbase fix (e.g. `ethpandaops/besu:bal-devnet-7`); benchmarkoor auto-pins its session priority fee. > -> **Fill image:** there is no published `fill-stateful` image yet (the command lands in execution-specs [#2637](https://github.com/ethereum/execution-specs/pull/2637)). Build one from the repo's `Dockerfile.eest-filler` (it bundles `uv` + execution-specs) and point `fill_image` at it: +> **Fill image:** by default benchmarkoor builds the fill image (the `uv`/python toolchain that runs `fill-stateful`) from a Dockerfile **embedded in the binary** — nothing to publish or pass. To pull a pre-built image instead, set `fill_image`; to build from a custom Dockerfile, set `fill_dockerfile`. The embedded Dockerfile lives at `pkg/builder/Dockerfile.eest-filler`; to build it by hand: > ```bash -> docker build -f Dockerfile.eest-filler -t ghcr.io/your-org/eest-fill-stateful:latest . +> docker build -f pkg/builder/Dockerfile.eest-filler -t ghcr.io/your-org/eest-fill-stateful:latest . > ``` ```yaml builder: eest_payloads: - fill_image: ghcr.io/your-org/eest-fill-stateful:latest # image carrying `uv run fill-stateful` - # fill_dockerfile: Dockerfile.eest-filler # build the fill image instead of pulling one + # Fill image defaults to a Dockerfile embedded in the binary. Optionally: + # fill_image: ghcr.io/your-org/eest-fill-stateful:latest # pull a pre-built image instead + # fill_dockerfile: pkg/builder/Dockerfile.eest-filler # or build from a custom Dockerfile pull_policy: always # always | if-not-present | never (default: always) container_runtime: docker # docker | podman (default: inherits runner.container_runtime, then docker) # jwt: # Engine API secret, shared with the filler (default: benchmarkoor's DefaultJWT) @@ -1684,9 +1685,9 @@ builder: | Option | Type | Default | Description | |---|---|---|---| -| `fill_image` | string | – | Pre-built container image carrying the uv/python toolchain that runs `fill-stateful`. Required unless `fill_dockerfile` is set. | -| `fill_dockerfile` | string | – | Path to a Dockerfile (e.g. `Dockerfile.eest-filler`) that benchmarkoor builds with the container runtime at build time, instead of pulling a pre-built image. Tagged `fill_image` when set, else `benchmarkoor-eest-fill:local`. Requires the runtime's `build` CLI (docker/podman) on the host. One of `fill_image` / `fill_dockerfile` is required. | -| `pull_policy` | string | `always` | One of `always`, `if-not-present`, `never`. Applies to both the fill image and the filler image (ignored for a locally built `fill_dockerfile`). | +| `fill_image` | string | – | Pre-built container image carrying the uv/python toolchain that runs `fill-stateful`. Optional: when neither this nor `fill_dockerfile` is set, benchmarkoor builds the fill image from a Dockerfile embedded in the binary. | +| `fill_dockerfile` | string | – | Path to a custom Dockerfile that benchmarkoor builds with the container runtime at build time, instead of pulling a pre-built image or using the embedded default. Tagged `fill_image` when set, else `benchmarkoor-eest-fill:local`. Requires the runtime's `build` CLI (docker/podman) on the host. | +| `pull_policy` | string | `always` | One of `always`, `if-not-present`, `never`. Applies to both the fill image and the filler image (ignored for a locally built fill image). | | `container_runtime` | string | runner's runtime, then `docker` | Container runtime for the filler + fill containers. | | `jwt` | string | benchmarkoor's `DefaultJWT` | Engine API JWT secret; shared between the filler client and `fill-stateful`. | | `fill_command` | []string | `[uv, run, fill-stateful]` | argv prefix invoked inside `fill_image` before the `fill-stateful` flags. Override if your image exposes the command differently. | diff --git a/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml b/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml index 2465579d..368e5f82 100644 --- a/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml +++ b/examples/configuration/config.state-actor-eest.full.amsterdam.stateful.yaml @@ -147,7 +147,8 @@ builder: ## Stage 2: Building EEST bloatnet/repricing fixtures eest_payloads: - fill_dockerfile: Dockerfile.eest-filler + # Fill image defaults to a Dockerfile embedded in the binary; set fill_image + # to pull a pre-built one, or fill_dockerfile to build from a custom Dockerfile. pull_policy: always eest_repo: https://github.com/ethereum/execution-specs.git # devnets/bal/7-bench carries the BAL benchmark tooling plus the fill-stateful diff --git a/examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml b/examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml index afc71f69..5cd7117e 100644 --- a/examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml +++ b/examples/configuration/config.state-actor-eest.simple.amsterdam.compute.yaml @@ -91,9 +91,9 @@ builder: ## Stage 2: Building EEST compute fixtures eest_payloads: - # Build the fill image (uv/python toolchain) from the repo's Dockerfile instead - # of pulling a pre-built one. Alternatively set fill_image: . - fill_dockerfile: Dockerfile.eest-filler + # By default benchmarkoor builds the fill image (uv/python toolchain) from a + # Dockerfile embedded in the binary. Set fill_image: to pull + # one instead, or fill_dockerfile: to build from a custom Dockerfile. pull_policy: always eest_repo: https://github.com/ethereum/execution-specs.git # devnets/bal/7-bench carries the BAL benchmark tooling plus the fill-stateful diff --git a/examples/configuration/config.state-actor-eest.simple.amsterdam.stateful.yaml b/examples/configuration/config.state-actor-eest.simple.amsterdam.stateful.yaml index a6b48b19..c09298f0 100644 --- a/examples/configuration/config.state-actor-eest.simple.amsterdam.stateful.yaml +++ b/examples/configuration/config.state-actor-eest.simple.amsterdam.stateful.yaml @@ -86,9 +86,9 @@ builder: ## Stage 2: Building EEST stateful (EIP-7928 / BAL) fixtures eest_payloads: - # Build the fill image (uv/python toolchain) from the repo's Dockerfile instead - # of pulling a pre-built one. Alternatively set fill_image: . - fill_dockerfile: Dockerfile.eest-filler + # By default benchmarkoor builds the fill image (uv/python toolchain) from a + # Dockerfile embedded in the binary. Set fill_image: to pull + # one instead, or fill_dockerfile: to build from a custom Dockerfile. pull_policy: always eest_repo: https://github.com/ethereum/execution-specs.git # devnets/bal/7-bench carries the BAL benchmark tooling plus the fill-stateful diff --git a/examples/configuration/config.state-actor-eest.simple.osaka.compute.yaml b/examples/configuration/config.state-actor-eest.simple.osaka.compute.yaml index 64bc4fbf..ec9eb881 100644 --- a/examples/configuration/config.state-actor-eest.simple.osaka.compute.yaml +++ b/examples/configuration/config.state-actor-eest.simple.osaka.compute.yaml @@ -56,9 +56,9 @@ builder: ## Stage 2: Building EEST stateful fixtures eest_payloads: - # Build the fill image (uv/python toolchain) from the repo's Dockerfile instead - # of pulling a pre-built one. Alternatively set fill_image: . - fill_dockerfile: Dockerfile.eest-filler + # By default benchmarkoor builds the fill image (uv/python toolchain) from a + # Dockerfile embedded in the binary. Set fill_image: to pull + # one instead, or fill_dockerfile: to build from a custom Dockerfile. pull_policy: always eest_repo: https://github.com/ethereum/execution-specs.git eest_ref: forks/amsterdam diff --git a/Dockerfile.eest-filler b/pkg/builder/Dockerfile.eest-filler similarity index 69% rename from Dockerfile.eest-filler rename to pkg/builder/Dockerfile.eest-filler index dee86490..319aa9fb 100644 --- a/Dockerfile.eest-filler +++ b/pkg/builder/Dockerfile.eest-filler @@ -1,18 +1,22 @@ # Dockerfile.eest-filler # -# Builds the toolchain image referenced by `builder.eest_payloads.fill_image`: -# it carries only `uv` + python + the build toolchain — NOT the execution-specs -# repo. benchmarkoor clones execution-specs at build time (at the configured -# `eest_ref`, caching it on disk) and mounts the checkout into this container at -# /eest, then runs `uv run fill-stateful`. uv builds the venv into the mounted -# checkout on first use (cached across runs). This keeps the EEST version in -# config and changeable without rebuilding the image. +# The default fill-image Dockerfile: it carries only `uv` + python + the build +# toolchain — NOT the execution-specs repo. benchmarkoor clones execution-specs +# at build time (at the configured `eest_ref`, caching it on disk) and mounts the +# checkout into this container at /eest, then runs `uv run fill-stateful`. uv +# builds the venv into the mounted checkout on first use (cached across runs). +# This keeps the EEST version in config and changeable without rebuilding the image. +# +# This file is embedded into the benchmarkoor binary (see pkg/builder), so by +# default the fill image is built from it with no Dockerfile needed on disk. Set +# builder.eest_payloads.fill_image to pull a pre-built image, or fill_dockerfile +# to build from a custom one. # # fill-stateful drives a *live* EL client over testing_buildBlockV1, so — unlike # the t8n-based `fill` — it needs NO evmone/eels transition-tool binary. # -# Build: -# docker build -f Dockerfile.eest-filler -t ghcr.io/your-org/eest-fill-stateful:latest . +# Build it by hand (from the repo root): +# docker build -f pkg/builder/Dockerfile.eest-filler -t ghcr.io/your-org/eest-fill-stateful:latest . # # (podman build works identically) # # Then point your config at it: diff --git a/pkg/builder/eest_payloads.go b/pkg/builder/eest_payloads.go index ccccfbe6..556dd3a9 100644 --- a/pkg/builder/eest_payloads.go +++ b/pkg/builder/eest_payloads.go @@ -2,6 +2,7 @@ package builder import ( "context" + _ "embed" "encoding/json" "errors" "fmt" @@ -59,6 +60,15 @@ const ( besuDefaultPriorityFeeWei = "1000000000" ) +// embeddedFillDockerfile is the default fill-image Dockerfile, compiled into the +// binary so the fill image can be built without a Dockerfile on disk. It is used +// when neither fill_image nor fill_dockerfile is configured. The Dockerfile +// copies nothing from the build context (execution-specs is mounted at run +// time), so it builds against an empty context written to a temp dir. +// +//go:embed Dockerfile.eest-filler +var embeddedFillDockerfile []byte + // EESTPayloadsBuilder generates stateful EEST benchmark fixtures. Per target // it boots a filler EL client on a writable copy of a pre-populated snapshot // datadir, runs fill-stateful against the live client, then tears it down. @@ -532,8 +542,16 @@ func (b *EESTPayloadsBuilder) ensureFillImage(ctx context.Context, log logrus.Fi } b.fillImageOnce.Do(func() { + dockerfile, cleanup, err := b.resolveFillDockerfile() + if err != nil { + b.fillImageErr = err + + return + } + defer cleanup() + tag := b.cfg.ResolveFillImageTag() - if err := b.buildFillImage(ctx, log, b.cfg.FillDockerfile, tag); err != nil { + if err := b.buildFillImage(ctx, log, dockerfile, tag); err != nil { b.fillImageErr = err return @@ -545,6 +563,33 @@ func (b *EESTPayloadsBuilder) ensureFillImage(ctx context.Context, log logrus.Fi return b.fillImage, b.fillImageErr } +// resolveFillDockerfile returns the Dockerfile path to build the fill image +// from, plus a cleanup func to run once the build completes. When +// fill_dockerfile is configured it is used directly (no-op cleanup). Otherwise +// the embedded default Dockerfile is written to a temp dir — which doubles as +// the (empty) build context — and cleanup removes it. +func (b *EESTPayloadsBuilder) resolveFillDockerfile() (string, func(), error) { + noop := func() {} + + if b.cfg.FillDockerfile != "" { + return b.cfg.FillDockerfile, noop, nil + } + + dir, err := os.MkdirTemp("", "benchmarkoor-fill-dockerfile-") + if err != nil { + return "", noop, fmt.Errorf("creating temp dir for embedded fill Dockerfile: %w", err) + } + + path := filepath.Join(dir, "Dockerfile.eest-filler") + if err := os.WriteFile(path, embeddedFillDockerfile, 0o600); err != nil { + _ = os.RemoveAll(dir) + + return "", noop, fmt.Errorf("writing embedded fill Dockerfile: %w", err) + } + + return path, func() { _ = os.RemoveAll(dir) }, nil +} + // buildFillImage builds the fill image from dockerfile, tagging it tag, by // shelling out to the configured container runtime's `build`. The build // context is the Dockerfile's directory. diff --git a/pkg/builder/eest_payloads_test.go b/pkg/builder/eest_payloads_test.go index 79983718..287f8665 100644 --- a/pkg/builder/eest_payloads_test.go +++ b/pkg/builder/eest_payloads_test.go @@ -14,6 +14,43 @@ import ( "github.com/stretchr/testify/require" ) +func TestEmbeddedFillDockerfile(t *testing.T) { + require.NotEmpty(t, embeddedFillDockerfile, "Dockerfile.eest-filler must be embedded") + body := string(embeddedFillDockerfile) + assert.Contains(t, body, "FROM python:") + assert.Contains(t, body, "WORKDIR /eest") +} + +func TestResolveFillDockerfile(t *testing.T) { + t.Run("configured path is used as-is with no-op cleanup", func(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "Custom.Dockerfile") + require.NoError(t, os.WriteFile(path, []byte("FROM scratch\n"), 0o600)) + + b := &EESTPayloadsBuilder{cfg: &config.EESTPayloadsConfig{FillDockerfile: path}} + got, cleanup, err := b.resolveFillDockerfile() + require.NoError(t, err) + + assert.Equal(t, path, got) + + cleanup() + assert.FileExists(t, path, "cleanup must not remove a configured Dockerfile") + }) + + t.Run("embedded default is written to a temp context and cleaned up", func(t *testing.T) { + b := &EESTPayloadsBuilder{cfg: &config.EESTPayloadsConfig{}} + got, cleanup, err := b.resolveFillDockerfile() + require.NoError(t, err) + + content, readErr := os.ReadFile(got) //nolint:gosec // path produced by the function under test + require.NoError(t, readErr) + assert.Equal(t, embeddedFillDockerfile, content, "temp Dockerfile must match the embedded one") + + cleanup() + assert.NoFileExists(t, got, "cleanup must remove the embedded temp Dockerfile") + }) +} + func TestBuildFillArgs(t *testing.T) { spec := client.NewGethSpec() prefix := []string{"uv", "run", "fill-stateful"} diff --git a/pkg/config/config.go b/pkg/config/config.go index d1738abf..1ca18f02 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -305,8 +305,9 @@ type EESTPayloadsConfig struct { // FillDockerfile, when set, makes benchmarkoor build the fill image from the // given Dockerfile at build time (using the container runtime), instead of // pulling a pre-built image. The built image is tagged FillImage when set, - // otherwise DefaultFillImageTag. Mutually informative with FillImage: at - // least one of the two must be provided. + // otherwise DefaultFillImageTag. Both FillImage and FillDockerfile are + // optional: when neither is set, benchmarkoor builds the fill image from a + // default Dockerfile embedded in the binary. FillDockerfile string `yaml:"fill_dockerfile,omitempty" mapstructure:"fill_dockerfile"` PullPolicy string `yaml:"pull_policy,omitempty" mapstructure:"pull_policy"` JWT string `yaml:"jwt,omitempty" mapstructure:"jwt"` @@ -338,10 +339,12 @@ const ( DefaultFillImageTag = "benchmarkoor-eest-fill:local" ) -// BuildsFillImage reports whether benchmarkoor should build the fill image from -// FillDockerfile (rather than pulling a pre-built FillImage). +// BuildsFillImage reports whether benchmarkoor should build the fill image +// (rather than pulling a pre-built FillImage). It builds whenever FillDockerfile +// is set, or when no FillImage is configured to pull — in the latter case from +// the Dockerfile embedded in the binary. func (e *EESTPayloadsConfig) BuildsFillImage() bool { - return e.FillDockerfile != "" + return e.FillDockerfile != "" || e.FillImage == "" } // ResolveFillImageTag returns the image reference for the fill container: the @@ -2180,12 +2183,10 @@ func (c *Config) validateEESTPayloads() error { ) } - if ep.FillImage == "" && ep.FillDockerfile == "" { - return fmt.Errorf( - "builder.eest_payloads: one of fill_image (a pre-built image) or " + - "fill_dockerfile (built by benchmarkoor) is required", - ) - } + // Neither fill_image nor fill_dockerfile is required: with both unset, + // benchmarkoor builds the fill image from the Dockerfile embedded in the + // binary. fill_image pulls a pre-built image; fill_dockerfile builds from a + // Dockerfile on disk. // Reject a missing Dockerfile at config time so typos surface early. // Relative paths are resolved against the working directory. diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 125d35d7..1836e74d 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -13,6 +13,49 @@ import ( "github.com/stretchr/testify/require" ) +func TestBuildsFillImage(t *testing.T) { + tests := []struct { + name string + fillImage string + fillDockerfile string + wantBuilds bool + wantTag string + }{ + { + name: "neither set builds from embedded default", + wantBuilds: true, + wantTag: DefaultFillImageTag, + }, + { + name: "fill_image only pulls", + fillImage: "fill:latest", + wantBuilds: false, + wantTag: "fill:latest", + }, + { + name: "fill_dockerfile only builds, default tag", + fillDockerfile: "Dockerfile.eest-filler", + wantBuilds: true, + wantTag: DefaultFillImageTag, + }, + { + name: "both set builds, tagged fill_image", + fillImage: "fill:latest", + fillDockerfile: "Dockerfile.eest-filler", + wantBuilds: true, + wantTag: "fill:latest", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := &EESTPayloadsConfig{FillImage: tt.fillImage, FillDockerfile: tt.fillDockerfile} + assert.Equal(t, tt.wantBuilds, e.BuildsFillImage()) + assert.Equal(t, tt.wantTag, e.ResolveFillImageTag()) + }) + } +} + func TestLoad_InlineAddressStubsKeyCasing(t *testing.T) { // Viper is case-insensitive and lowercases all map keys; EEST resolves stub // names by exact match, so Load must restore the original casing. @@ -3964,12 +4007,10 @@ func TestValidateEESTPayloads(t *testing.T) { }, }, { - name: "neither fill_image nor fill_dockerfile", + name: "neither fill_image nor fill_dockerfile is valid (embedded default)", ep: &EESTPayloadsConfig{ Targets: []EESTPayloadTarget{base(dirA)}, }, - wantErr: true, - errSubstr: "fill_image", }, { name: "fill_dockerfile only is valid", From 25e6ec197e1c6461284cb7910b7e76613323c79d Mon Sep 17 00:00:00 2001 From: Rafael Matias Date: Tue, 30 Jun 2026 16:45:01 +0200 Subject: [PATCH 2/2] ci: add state-actor + eest build/run e2e via the composite action Add an optional build phase to the composite action (build-config / build-config-urls / build-args) that runs `benchmarkoor build` before the existing run step; existing jobs are unaffected. Add a state-actor job to ci.action.yaml that builds OSAKA nethermind+besu snapshots, fills the compute bn128 subset with the besu filler, then replays the fixtures on nethermind and besu (datadir_method: copy, 256 MB snapshot). --- .github/workflows/ci.action.yaml | 119 +++++++++++++++++++++++++++++++ action.yaml | 67 +++++++++++++++++ 2 files changed, 186 insertions(+) diff --git a/.github/workflows/ci.action.yaml b/.github/workflows/ci.action.yaml index 83d14dc7..1987d716 100644 --- a/.github/workflows/ci.action.yaml +++ b/.github/workflows/ci.action.yaml @@ -138,3 +138,122 @@ jobs: instances: - id: reth client: reth + + # Exercises the builder pipeline end-to-end: state-actor builds OSAKA + # snapshots for nethermind + besu, the eest_payloads builder fills the + # compute "bn128" subset with the besu filler, then the runner replays + # those fixtures on nethermind and besu. datadir_method: copy + a small + # 256 MB snapshot keep it within a GitHub-hosted runner. + state-actor: + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + # The client + fill images and the locally built benchmarkoor image are + # several GB; free the runner's preinstalled toolchains to be safe. + - name: Free disk space + shell: bash + run: | + sudo rm -rf /usr/local/lib/android /opt/ghc /usr/local/.ghcup \ + /opt/hostedtoolcache/CodeQL /usr/share/dotnet /usr/local/share/boost + df -h / + + - name: Build snapshots + fill payloads, then run (nethermind + besu) + uses: ./ + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + git-ref: ${{ github.event.pull_request.head.sha }} + git-repo: ${{ github.event.pull_request.head.repo.clone_url }} + build-args: '--force' + build-config: | + builder: + state_actor: + images: + nethermind: ghcr.io/ethereum/state-actor-nethermind:main + besu: ghcr.io/ethereum/state-actor-besu:main + pull_policy: always + config: + seed: 1234 + fork: osaka + chain: 1337 + gas_limit: 300000000 + target_size: 256MB + spec: + entities: + # Predeploy the Arachnid CREATE2 factory so the EEST fill + # bootstrap detects it and skips the keyless deploy. + - kind: contract + template: create2_factory + name: deterministic-deployment-proxy + address: 0x4e59b44847b379578588920cA78FbF26c0B4956C + # Pre-fund fill-stateful's seed (rpc_seed_key=0x..01 below) + # so it skips the withdrawal funding block. + - kind: eoa + name: fill-stateful-seed + address: 0x7e5f4552091a69125d5dfcb7b8c2659029395bdf + balance: "10000000000000000000000000000" + nonce: 0 + targets: + - client: nethermind + output_dir: /tmp/benchmarkoor/state-actor/nethermind + - client: besu + output_dir: /tmp/benchmarkoor/state-actor/besu + eest_payloads: + # Fill image is built from the Dockerfile embedded in the binary. + pull_policy: always + eest_repo: https://github.com/ethereum/execution-specs.git + eest_ref: devnets/bal/7-bench + config: + fork: osaka + rpc_seed_key: "0x0000000000000000000000000000000000000000000000000000000000000001" + gas_benchmark_values: [10] + datadir_method: copy + tests: + - tests/benchmark/compute + filter: bn128 + targets: + - name: payload-generator-besu + filler_client: besu + # besu image carrying testing_buildBlockV1 (drives fill-stateful). + filler_image: ethpandaops/besu:bal-devnet-7 + source_dir: /tmp/benchmarkoor/state-actor/besu + genesis: /tmp/benchmarkoor/state-actor/besu/besu-chainspec.json + output_dir: /tmp/benchmarkoor/eest-fixtures/besu + run-config-urls: >- + https://raw.githubusercontent.com/${{ github.event.pull_request.head.repo.full_name }}/${{ github.event.pull_request.head.sha }}/.github/workflows/config/benchmarkoor-global-cleanup.yaml, + https://raw.githubusercontent.com/${{ github.event.pull_request.head.repo.full_name }}/${{ github.event.pull_request.head.sha }}/.github/workflows/config/benchmarkoor-global-clientstdout.yaml + run-config: | + runner: + benchmark: + tests: + source: + eest_fixtures: + local_fixtures_dir: /tmp/benchmarkoor/eest-fixtures/besu + fixtures_subdir: blockchain_tests_stateful_engine + client: + config: + bootstrap_fcu: + enabled: true + max_retries: 10 + backoff: 1s + genesis: + nethermind: /tmp/benchmarkoor/state-actor/nethermind/parity-chainspec.json + besu: /tmp/benchmarkoor/state-actor/besu/besu-chainspec.json + datadirs: + nethermind: + source_dir: /tmp/benchmarkoor/state-actor/nethermind + method: copy + besu: + source_dir: /tmp/benchmarkoor/state-actor/besu + method: copy + instances: + - id: nethermind + client: nethermind + image: nethermindeth/nethermind:master + extra_args: [--Init.BaseDbPath=/data] + - id: besu + client: besu + image: ethpandaops/besu:bal-devnet-7 + extra_args: [--p2p-enabled=true] diff --git a/action.yaml b/action.yaml index daf16b39..3b43cd75 100644 --- a/action.yaml +++ b/action.yaml @@ -45,6 +45,21 @@ inputs: required: false default: '' + build-config-urls: + description: 'Comma-separated URLs for config files to download and pass to `benchmarkoor build` via --config (merged in order). Setting this or build-config enables the build phase (state-actor snapshots + eest payloads), which runs before the benchmark run.' + required: false + default: '' + + build-config: + description: 'Raw YAML config content to pass to `benchmarkoor build` via --config (appended after URL config). Setting this or build-config-urls enables the build phase.' + required: false + default: '' + + build-args: + description: 'Extra flags passed to the benchmarkoor build command (e.g. --force, --limit-state-actor-target=)' + required: false + default: '' + tmp-dir: description: 'Temporary directory path used for benchmarkoor data (overlayfs) and cache' required: false @@ -109,6 +124,56 @@ runs: echo "binary=${BIN}" >> "$GITHUB_OUTPUT" + - name: Build Benchmarkoor artifacts + id: build-benchmarkoor + if: inputs.build-config != '' || inputs.build-config-urls != '' + shell: bash + env: + INPUT_BUILD_CONFIG: ${{ inputs.build-config }} + INPUT_BUILD_CONFIG_URLS: ${{ inputs.build-config-urls }} + run: | + BIN="${{ steps.extract-binary.outputs.binary }}" + + # Download URL configs if provided (comma-separated) + if [ -n "$INPUT_BUILD_CONFIG_URLS" ]; then + IFS=',' read -ra CONFIG_URLS <<< "$INPUT_BUILD_CONFIG_URLS" + for i in "${!CONFIG_URLS[@]}"; do + URL=$(echo "${CONFIG_URLS[$i]}" | xargs) # trim whitespace + echo "Downloading build config from ${URL}" + curl -fsSL "${URL}" -o "${{ runner.temp }}/benchmarkoor-build-config-url-${i}.yaml" + done + fi + + # Write inline config if provided + if [ -n "$INPUT_BUILD_CONFIG" ]; then + echo "$INPUT_BUILD_CONFIG" > "${{ runner.temp }}/benchmarkoor-build-config-inline.yaml" + fi + + # Build argv as an array so quoted values survive intact. Config files + # apply in order: URL configs, then inline config (last wins). This step + # runs from the workspace, so any relative paths in the config (e.g. a + # custom fill_dockerfile) resolve against the checkout. + ARGS=("build") + for cfg in "${{ runner.temp }}"/benchmarkoor-build-config-url-*.yaml; do + [ -f "$cfg" ] && ARGS+=("--config=$cfg") + done + if [ -f "${{ runner.temp }}/benchmarkoor-build-config-inline.yaml" ]; then + ARGS+=("--config=${{ runner.temp }}/benchmarkoor-build-config-inline.yaml") + fi + + # Append extra build args if provided (whitespace-split, same as run). + if [ -n "${{ inputs.build-args }}" ]; then + # shellcheck disable=SC2206 # intentional word-split for argv + EXTRA_ARGS=(${{ inputs.build-args }}) + ARGS+=("${EXTRA_ARGS[@]}") + fi + + # `sudo -E` for the same reasons as the run step: keeps the built + # snapshots/fixtures root-owned (so the rootful run phase can read + # them) and propagates caller env (e.g. STATE_DIR) into the build. + echo "Running: sudo -E $BIN ${ARGS[*]}" + sudo -E "$BIN" "${ARGS[@]}" + - name: Get Job ID from GH API id: get-job-id shell: bash @@ -253,5 +318,7 @@ runs: rm -rf "${{ runner.temp }}/results" rm -f "${{ runner.temp }}/benchmarkoor-config-url-"*.yaml rm -f "${{ runner.temp }}/benchmarkoor-config-inline.yaml" + rm -f "${{ runner.temp }}/benchmarkoor-build-config-url-"*.yaml + rm -f "${{ runner.temp }}/benchmarkoor-build-config-inline.yaml" rm -f "${{ runner.temp }}/benchmarkoor-results.tar.gz" rm -f "${{ runner.temp }}/benchmarkoor"