diff --git a/.github/workflows/ci.action.yaml b/.github/workflows/ci.action.yaml index 2a83681..912c002 100644 --- a/.github/workflows/ci.action.yaml +++ b/.github/workflows/ci.action.yaml @@ -20,10 +20,11 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} git-ref: ${{ github.event.pull_request.head.sha }} + git-repo: ${{ github.event.pull_request.head.repo.clone_url }} #run-args: '--log-level=debug' run-config-urls: >- - https://raw.githubusercontent.com/ethpandaops/benchmarkoor/${{ github.event.pull_request.head.sha }}/.github/workflows/config/benchmarkoor-global-cleanup.yaml, - https://raw.githubusercontent.com/ethpandaops/benchmarkoor/${{ github.event.pull_request.head.sha }}/.github/workflows/config/benchmarkoor-global-clientstdout.yaml + https://raw.githubusercontent.com/${{ github.event.pull_request.head.repo.full_name }}/${{ github.event.pull_request.head.sha }}/.github/workflows/config/benchmarkoor-global-cleanup.yaml, + https://raw.githubusercontent.com/${{ github.event.pull_request.head.repo.full_name }}/${{ github.event.pull_request.head.sha }}/.github/workflows/config/benchmarkoor-global-clientstdout.yaml run-config: | runner: benchmark: @@ -62,10 +63,11 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} git-ref: ${{ github.event.pull_request.head.sha }} + git-repo: ${{ github.event.pull_request.head.repo.clone_url }} #run-args: '--log-level=debug' run-config-urls: >- - https://raw.githubusercontent.com/ethpandaops/benchmarkoor/${{ github.event.pull_request.head.sha }}/.github/workflows/config/benchmarkoor-global-cleanup.yaml, - https://raw.githubusercontent.com/ethpandaops/benchmarkoor/${{ github.event.pull_request.head.sha }}/.github/workflows/config/benchmarkoor-global-clientstdout.yaml + https://raw.githubusercontent.com/${{ github.event.pull_request.head.repo.full_name }}/${{ github.event.pull_request.head.sha }}/.github/workflows/config/benchmarkoor-global-cleanup.yaml, + https://raw.githubusercontent.com/${{ github.event.pull_request.head.repo.full_name }}/${{ github.event.pull_request.head.sha }}/.github/workflows/config/benchmarkoor-global-clientstdout.yaml run-config: | runner: container_runtime: podman @@ -102,10 +104,11 @@ jobs: with: github-token: ${{ secrets.GITHUB_TOKEN }} git-ref: ${{ github.event.pull_request.head.sha }} + git-repo: ${{ github.event.pull_request.head.repo.clone_url }} #run-args: '--log-level=debug' run-config-urls: >- - https://raw.githubusercontent.com/ethpandaops/benchmarkoor/${{ github.event.pull_request.head.sha }}/.github/workflows/config/benchmarkoor-global-cleanup.yaml, - https://raw.githubusercontent.com/ethpandaops/benchmarkoor/${{ github.event.pull_request.head.sha }}/.github/workflows/config/benchmarkoor-global-clientstdout.yaml + https://raw.githubusercontent.com/${{ github.event.pull_request.head.repo.full_name }}/${{ github.event.pull_request.head.sha }}/.github/workflows/config/benchmarkoor-global-cleanup.yaml, + https://raw.githubusercontent.com/${{ github.event.pull_request.head.repo.full_name }}/${{ github.event.pull_request.head.sha }}/.github/workflows/config/benchmarkoor-global-clientstdout.yaml run-config: | runner: container_runtime: podman diff --git a/action.yaml b/action.yaml index ba4a4d9..7b9fd42 100644 --- a/action.yaml +++ b/action.yaml @@ -20,6 +20,11 @@ inputs: required: false default: '' + git-repo: + description: 'Git repository URL to clone for building the image. Only used when image is not provided. Defaults to ethpandaops/benchmarkoor.' + required: false + default: 'https://github.com/ethpandaops/benchmarkoor.git' + upload-artifacts: description: 'Whether to upload run results as GitHub artifacts' required: false @@ -65,8 +70,10 @@ runs: GIT_REF="master" fi - echo "Cloning https://github.com/ethpandaops/benchmarkoor.git at ref $GIT_REF" - git clone https://github.com/ethpandaops/benchmarkoor.git benchmarkoor-build + GIT_REPO="${{ inputs.git-repo }}" + + echo "Cloning ${GIT_REPO} at ref $GIT_REF" + git clone "${GIT_REPO}" benchmarkoor-build cd benchmarkoor-build git checkout "$GIT_REF" diff --git a/pkg/eest/converter.go b/pkg/eest/converter.go index 063671a..3c84d71 100644 --- a/pkg/eest/converter.go +++ b/pkg/eest/converter.go @@ -63,6 +63,68 @@ func ConvertFixture(name string, fixture *Fixture) (*ConvertedTest, error) { return result, nil } +// ConvertStatefulFixture converts a stateful EEST fixture to JSON-RPC calls. +// Unlike ConvertFixture, phases are explicit: setupEngineNewPayloads become +// setup steps and engineNewPayloads become test steps. +func ConvertStatefulFixture(name string, fixture *StatefulFixture) (*ConvertedTest, error) { + if fixture == nil { + return nil, fmt.Errorf("fixture is nil") + } + + if len(fixture.EngineNewPayloads) == 0 { + return nil, fmt.Errorf("fixture has no test payloads") + } + + result := &ConvertedTest{ + Name: name, + SetupLines: make([]string, 0), + TestLines: make([]string, 0), + GenesisHash: fixture.SnapshotBlockHash, + FinalHash: fixture.LastBlockHash, + PayloadCount: len(fixture.SetupEngineNewPayloads) + len(fixture.EngineNewPayloads), + } + + // Convert setup payloads. + for i, payload := range fixture.SetupEngineNewPayloads { + lines, err := convertPayload(payload, i+1) + if err != nil { + return nil, fmt.Errorf("converting setup payload %d: %w", i, err) + } + + result.SetupLines = append(result.SetupLines, lines...) + } + + // Convert test payloads. + idOffset := len(fixture.SetupEngineNewPayloads) + + for i, payload := range fixture.EngineNewPayloads { + lines, err := convertPayload(payload, idOffset+i+1) + if err != nil { + return nil, fmt.Errorf("converting test payload %d: %w", i, err) + } + + result.TestLines = append(result.TestLines, lines...) + } + + return result, nil +} + +// ConvertPreRunFixture converts a pre-run fixture to JSON-RPC lines. +func ConvertPreRunFixture(fixture *PreRunFixture) ([]string, error) { + var lines []string + + for i, payload := range fixture.EngineNewPayloads { + converted, err := convertPayload(payload, i+1) + if err != nil { + return nil, fmt.Errorf("converting pre-run payload %d: %w", i, err) + } + + lines = append(lines, converted...) + } + + return lines, nil +} + // convertPayload generates JSON-RPC lines for a single payload. func convertPayload(payload *EngineNewPayload, id int) ([]string, error) { if payload.ExecutionPayload == nil { diff --git a/pkg/eest/fixture.go b/pkg/eest/fixture.go index 37cd563..e33015a 100644 --- a/pkg/eest/fixture.go +++ b/pkg/eest/fixture.go @@ -6,9 +6,12 @@ import ( "strconv" ) -// SupportedFixtureFormat is the fixture format we support. +// SupportedFixtureFormat is the fixture format for genesis-based tests. const SupportedFixtureFormat = "blockchain_test_engine_x" +// SupportedStatefulFixtureFormat is the fixture format for snapshot-based stateful tests. +const SupportedStatefulFixtureFormat = "blockchain_test_stateful_engine" + // Fixture represents a single EEST test fixture. type Fixture struct { Info *FixtureInfo `json:"_info"` @@ -33,6 +36,43 @@ func (f *Fixture) IsSupportedFormat() bool { return f.Info != nil && f.Info.FixtureFormat == SupportedFixtureFormat } +// StatefulFixture represents a snapshot-based stateful EEST test fixture. +// Unlike Fixture, it has no genesis or pre-alloc — state comes from an +// external network snapshot referenced by SnapshotBlockNumber/Hash. +type StatefulFixture struct { + Info *FixtureInfo `json:"_info"` + Network string `json:"network"` + LastBlockHash string `json:"lastblockhash"` + Config *FixtureConfig `json:"config"` + SnapshotBlockNumber string `json:"snapshotBlockNumber"` + SnapshotBlockHash string `json:"snapshotBlockHash"` + StartBlockNumber string `json:"startBlockNumber"` + StartBlockHash string `json:"startBlockHash"` + SetupEngineNewPayloads []*EngineNewPayload `json:"setupEngineNewPayloads"` + EngineNewPayloads []*EngineNewPayload `json:"engineNewPayloads"` +} + +// PreRunFixture represents a pre-run payload file for stateful benchmarks. +// Contains global setup blocks (e.g. factory deploy) that bridge the gap +// between the raw snapshot and the start block. +type PreRunFixture struct { + Network string `json:"network"` + SnapshotBlockNumber string `json:"snapshotBlockNumber"` + SnapshotBlockHash string `json:"snapshotBlockHash"` + EngineNewPayloads []*EngineNewPayload `json:"engineNewPayloads"` +} + +// FixtureConfig contains chain configuration for a fixture. +type FixtureConfig struct { + Network string `json:"network"` + ChainID string `json:"chainid"` +} + +// IsSupportedFormat returns true if the stateful fixture has a supported format. +func (f *StatefulFixture) IsSupportedFormat() bool { + return f.Info != nil && f.Info.FixtureFormat == SupportedStatefulFixtureFormat +} + // BlockHeader represents an Ethereum block header. type BlockHeader struct { ParentHash string `json:"parentHash"` @@ -206,7 +246,7 @@ type Consolidate struct { TargetPubkey string `json:"targetPubkey"` } -// ParseFixtureFile parses a fixture JSON file. +// ParseFixtureFile parses a fixture JSON file containing genesis-based fixtures. // The file contains a map of test names to Fixture objects. func ParseFixtureFile(data []byte) (map[string]*Fixture, error) { var fixtures map[string]*Fixture @@ -216,3 +256,37 @@ func ParseFixtureFile(data []byte) (map[string]*Fixture, error) { return fixtures, nil } + +// ParseStatefulFixtureFile parses a fixture JSON file containing stateful fixtures. +// The file contains a map of test names to StatefulFixture objects. +func ParseStatefulFixtureFile(data []byte) (map[string]*StatefulFixture, error) { + var fixtures map[string]*StatefulFixture + if err := json.Unmarshal(data, &fixtures); err != nil { + return nil, err + } + + return fixtures, nil +} + +// DetectFixtureFormat inspects a fixture JSON file and returns the format +// string from the first entry's _info.fixture-format field. +// Returns empty string if the format cannot be determined. +func DetectFixtureFormat(data []byte) string { + var raw map[string]json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + return "" + } + + for _, entry := range raw { + var info struct { + Info *FixtureInfo `json:"_info"` + } + if err := json.Unmarshal(entry, &info); err == nil && info.Info != nil { + return info.Info.FixtureFormat + } + + break // only check the first entry + } + + return "" +} diff --git a/pkg/executor/eest_source.go b/pkg/executor/eest_source.go index b70dd38..aa267d0 100644 --- a/pkg/executor/eest_source.go +++ b/pkg/executor/eest_source.go @@ -593,7 +593,7 @@ func (s *EESTSource) discoverTests() (*PreparedSource, error) { } if info.IsDir() { - if info.Name() == "pre_alloc" { + if info.Name() == "pre_alloc" || info.Name() == "pre_run" { return filepath.SkipDir } @@ -610,83 +610,19 @@ func (s *EESTSource) discoverTests() (*PreparedSource, error) { return fmt.Errorf("reading fixture %s: %w", path, err) } - fixtures, err := eest.ParseFixtureFile(data) - if err != nil { - s.log.WithFields(logrus.Fields{ - "file": path, - "error": err, - }).Warn("Failed to parse fixture file, skipping") - - return nil - } - - // Convert each fixture to tests. - for name, fixture := range fixtures { - // Skip fixtures that don't have the supported format. - if !fixture.IsSupportedFormat() { - format := "" - if fixture.Info != nil { - format = fixture.Info.FixtureFormat - } - - s.log.WithFields(logrus.Fields{ - "file": path, - "fixture": name, - "format": format, - }).Debug("Skipping fixture with unsupported format") + // Detect format and route to the appropriate parser/converter. + format := eest.DetectFixtureFormat(data) - continue - } - - // Apply filter to individual test names too. - if s.filter != "" && !strings.Contains(name, s.filter) { - continue - } - - converted, err := eest.ConvertFixture(name, fixture) - if err != nil { - s.log.WithFields(logrus.Fields{ - "file": path, - "fixture": name, - "error": err, - }).Warn("Failed to convert fixture, skipping") - - continue - } - - // Build test name from the fixture key. - // The fixture key is a pytest node ID like - // "tests/benchmark/.../test_foo.py::test_bar[params]". - // Strip the leading "tests/" directory prefix if present - // since it's a pytest artifact, not part of the test identity. - testName := name - if after, ok := strings.CutPrefix(testName, "tests/"); ok { - testName = after - } - - test := &TestWithSteps{ - Name: testName, - EESTInfo: fixture.Info, - } - - // Create setup step if there are setup lines. - if len(converted.SetupLines) > 0 { - test.Setup = &StepFile{ - Name: testName + "/setup", - Provider: &linesProvider{lines: converted.SetupLines}, - } - } - - // Create test step. - if len(converted.TestLines) > 0 { - test.Test = &StepFile{ - Name: testName + "/test", - Provider: &linesProvider{lines: converted.TestLines}, - } - } - - result.Tests = append(result.Tests, test) - testsByFixtureKey[name] = test + switch format { + case eest.SupportedStatefulFixtureFormat: + s.processStatefulFixtures(data, path, result, testsByFixtureKey) + case eest.SupportedFixtureFormat: + s.processGenesisFixtures(data, path, result, testsByFixtureKey) + default: + s.log.WithFields(logrus.Fields{ + "file": path, + "format": format, + }).Debug("Skipping fixture file with unsupported format") } return nil @@ -696,6 +632,9 @@ func (s *EESTSource) discoverTests() (*PreparedSource, error) { return nil, fmt.Errorf("walking fixtures directory: %w", err) } + // Scan pre_run/ directory for global setup payload files. + s.loadPreRunSteps(searchDir, result) + // Sort tests by name for consistent ordering. sort.Slice(result.Tests, func(i, j int) bool { return result.Tests[i].Name < result.Tests[j].Name @@ -727,6 +666,207 @@ func (s *EESTSource) discoverTests() (*PreparedSource, error) { return result, nil } +// processGenesisFixtures parses and converts genesis-based (blockchain_test_engine_x) fixtures. +func (s *EESTSource) processGenesisFixtures( + data []byte, + path string, + result *PreparedSource, + testsByFixtureKey map[string]*TestWithSteps, +) { + fixtures, err := eest.ParseFixtureFile(data) + if err != nil { + s.log.WithFields(logrus.Fields{ + "file": path, + "error": err, + }).Warn("Failed to parse fixture file, skipping") + + return + } + + for name, fixture := range fixtures { + if !fixture.IsSupportedFormat() { + continue + } + + if s.filter != "" && !strings.Contains(name, s.filter) { + continue + } + + converted, err := eest.ConvertFixture(name, fixture) + if err != nil { + s.log.WithFields(logrus.Fields{ + "file": path, + "fixture": name, + "error": err, + }).Warn("Failed to convert fixture, skipping") + + continue + } + + s.addConvertedTest(name, fixture.Info, converted, result, testsByFixtureKey) + } +} + +// processStatefulFixtures parses and converts stateful (blockchain_test_stateful_engine) fixtures. +func (s *EESTSource) processStatefulFixtures( + data []byte, + path string, + result *PreparedSource, + testsByFixtureKey map[string]*TestWithSteps, +) { + fixtures, err := eest.ParseStatefulFixtureFile(data) + if err != nil { + s.log.WithFields(logrus.Fields{ + "file": path, + "error": err, + }).Warn("Failed to parse stateful fixture file, skipping") + + return + } + + for name, fixture := range fixtures { + if !fixture.IsSupportedFormat() { + continue + } + + if s.filter != "" && !strings.Contains(name, s.filter) { + continue + } + + converted, err := eest.ConvertStatefulFixture(name, fixture) + if err != nil { + s.log.WithFields(logrus.Fields{ + "file": path, + "fixture": name, + "error": err, + }).Warn("Failed to convert stateful fixture, skipping") + + continue + } + + test := s.addConvertedTest(name, fixture.Info, converted, result, testsByFixtureKey) + test.GenesisHash = converted.GenesisHash + } +} + +// addConvertedTest builds a TestWithSteps from a ConvertedTest and adds it to the result. +func (s *EESTSource) addConvertedTest( + name string, + info *eest.FixtureInfo, + converted *eest.ConvertedTest, + result *PreparedSource, + testsByFixtureKey map[string]*TestWithSteps, +) *TestWithSteps { + // Build test name from the fixture key. + // The fixture key is a pytest node ID like + // "tests/benchmark/.../test_foo.py::test_bar[params]". + // Strip the leading "tests/" directory prefix if present + // since it's a pytest artifact, not part of the test identity. + testName := name + if after, ok := strings.CutPrefix(testName, "tests/"); ok { + testName = after + } + + test := &TestWithSteps{ + Name: testName, + EESTInfo: info, + } + + if len(converted.SetupLines) > 0 { + test.Setup = &StepFile{ + Name: testName + "/setup", + Provider: &linesProvider{lines: converted.SetupLines}, + } + } + + if len(converted.TestLines) > 0 { + test.Test = &StepFile{ + Name: testName + "/test", + Provider: &linesProvider{lines: converted.TestLines}, + } + } + + result.Tests = append(result.Tests, test) + testsByFixtureKey[name] = test + + return test +} + +// loadPreRunSteps scans the pre_run/ directory for global setup +// payload files and adds them as PreRunSteps. +func (s *EESTSource) loadPreRunSteps(searchDir string, result *PreparedSource) { + preRunDir := filepath.Join(searchDir, "pre_run") + + entries, err := os.ReadDir(preRunDir) + if err != nil { + if os.IsNotExist(err) { + return + } + + s.log.WithError(err).Warn("Failed to read pre_run directory") + + return + } + + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".json") { + continue + } + + data, err := os.ReadFile(filepath.Join(preRunDir, entry.Name())) + if err != nil { + s.log.WithFields(logrus.Fields{ + "file": entry.Name(), + "error": err, + }).Warn("Failed to read pre-run file, skipping") + + continue + } + + var preRun eest.PreRunFixture + if err := json.Unmarshal(data, &preRun); err != nil { + s.log.WithFields(logrus.Fields{ + "file": entry.Name(), + "error": err, + }).Warn("Failed to parse pre-run file, skipping") + + continue + } + + if len(preRun.EngineNewPayloads) == 0 { + continue + } + + s.log.WithFields(logrus.Fields{ + "file": entry.Name(), + "snapshot_block_number": preRun.SnapshotBlockNumber, + "snapshot_block_hash": preRun.SnapshotBlockHash, + }).Info("Pre-run references snapshot state") + + lines, err := eest.ConvertPreRunFixture(&preRun) + if err != nil { + s.log.WithFields(logrus.Fields{ + "file": entry.Name(), + "error": err, + }).Warn("Failed to convert pre-run fixture, skipping") + + continue + } + + if len(lines) > 0 { + result.PreRunSteps = append(result.PreRunSteps, &StepFile{ + Name: "pre_run/" + entry.Name(), + Provider: &linesProvider{lines: lines}, + }) + + s.log.WithFields(logrus.Fields{ + "file": entry.Name(), + "payloads": len(preRun.EngineNewPayloads), + }).Info("Loaded pre-run step") + } + } +} + // Cleanup is a no-op for EEST sources (we keep the cache). func (s *EESTSource) Cleanup() error { return nil