config.example.yaml

# Environment variables can be used anywhere in the config using ${VAR} or $VAR syntax.
# Example: ${MY_SECRET} or ${DATA_DIR:-/default/path} (with default value).
#
# Multiple config files can be merged with: --config base.yaml --config overrides.yaml
# Later files override earlier ones.

global:
  log_level: ${LOG_LEVEL:-info}

runner:
  # Container runtime: "docker" (default) or "podman".
  # Podman is required for the "container-checkpoint-restore" rollback strategy.
  # When using Podman, ensure the socket is active: sudo systemctl start podman.socket
  # container_runtime: docker
  client_logs_to_stdout: true
  docker_network: benchmarkoor
  cleanup_on_start: false
  # Optional: Global timeout for the entire run (all instances, setup, teardown).
  # If the run exceeds this duration, it will be cancelled.
  # Uses Go duration format (e.g., "1h", "30m", "2h30m").
  # Can also be set via BENCHMARKOOR_RUNNER_RUN_TIMEOUT environment variable.
  # run_timeout: 4h
  # Optional: Arbitrary key-value labels attached to this benchmark run.
  # Labels appear in the run's config.json and can be used for filtering/organization.
  # Can also be set via CLI: --metadata.label=env=production --metadata.label=team=platform
  # metadata:
  #   labels:
  #     env: production
  #     team: platform
  # Optional directory configurations.
  # directories:
  #   # Directory for temporary datadir copies (defaults to system temp).
  #   tmp_datadir: ${TMP_DIR:-/tmp}/benchmarkoor
  #   # Directory for executor cache - git clones, etc (defaults to ~/.cache/benchmarkoor).
  #   tmp_cachedir: ${TMP_DIR:-/tmp}/benchmarkoor-cache
  # Optional: Override path to drop_caches file (default: /proc/sys/vm/drop_caches).
  # Useful when running in containers where the file is mounted at a different path.
  # drop_caches_path: /proc/sys/vm/drop_caches
  # Optional: Override sysfs base path for CPU frequency control (default: /sys/devices/system/cpu).
  # Useful when running in containers where /sys is read-only and the host path is bind-mounted
  # at a different location (e.g., -v /sys/devices/system/cpu:/host_sys_cpu).
  # cpu_sysfs_path: /sys/devices/system/cpu
  # Optional: GitHub token for downloading GitHub Actions artifacts via the REST API.
  # If the gh CLI is installed and authenticated, no token is needed.
  # Otherwise, provide a GitHub token with actions:read scope.
  # Can also be set via BENCHMARKOOR_RUNNER_GITHUB_TOKEN environment variable.
  # github_token: ${GITHUB_TOKEN:-}

  benchmark:
    results_dir: ${RESULTS_DIR:-./results}
    # Optional: Set ownership (user:group) for results files. Useful when running as root.
    # results_owner: "1000:1000"
    # Optional: Skip test execution entirely. When enabled, only post-run operations
    # (index generation, suite stats) are performed. Useful for regenerating stats from
    # S3-backed results without needing Docker or test infrastructure.
    # skip_test_run: false
    # Optional: Enable/disable system resource collection (cgroups/Docker Stats API).
    # When disabled, no CPU/memory/disk metrics will be collected during tests.
    # Useful when running in environments without cgroup access. Default: true
    # system_resource_collection_enabled: true
    # Optional: Generate index.json after benchmark (aggregates all run metadata).
    # generate_results_index: true
    # Optional: Method for index generation: "local" (default, filesystem) or "s3"
    # (read runs from S3, upload index.json back to the bucket).
    # Requires results_upload.s3 to be configured when set to "s3".
    # generate_results_index_method: local
    # Optional: Generate stats.json per suite after benchmark (aggregates test durations
    # across runs for the UI heatmap).
    # generate_suite_stats: true
    # Optional: Method for suite stats generation: "local" (default, filesystem) or "s3"
    # (read runs from S3, upload stats.json back to the bucket).
    # Requires results_upload.s3 to be configured when set to "s3".
    # generate_suite_stats_method: local
    # Optional: Upload results to S3-compatible storage after each instance run.
    # A preflight check verifies connectivity before any benchmarks start.
    # results_upload:
    #   s3:
    #     enabled: true
    #     # Endpoint URL (scheme + host only, no path or bucket name).
    #     # Examples:
    #     #   AWS:    https://s3.us-east-1.amazonaws.com
    #     #   R2:     https://<account_id>.r2.cloudflarestorage.com
    #     #   MinIO:  http://localhost:9000
    #     endpoint_url: https://s3.us-east-1.amazonaws.com
    #     region: us-east-1
    #     bucket: my-benchmark-results
    #     access_key_id: ${AWS_ACCESS_KEY_ID}
    #     secret_access_key: ${AWS_SECRET_ACCESS_KEY}
    #     prefix: results  # Base key prefix (runs at prefix/runs/, suites at prefix/suites/)
    #     # storage_class: STANDARD  # Optional: S3 storage class
    #     # acl: private  # Optional: Canned ACL
    #     # Path-style addressing: required for MinIO and Cloudflare R2.
    #     force_path_style: false
    #     # parallel_uploads: 50  # Number of concurrent file uploads

    # Optional test execution configuration.
    # tests:
    #   # Optional filter to run only tests matching this pattern.
    #   filter: ""
    #   # Optional: Metadata labels for the test suite.
    #   # Labels appear in the suite's summary.json and are shown in the UI.
    #   # The special "name" label is used as the display name for the suite.
    #   # metadata:
    #   #   labels:
    #   #     name: "EIP-7934 BN128 Benchmarks"
    #   #     category: precompile
    #   source:
    #     # Option 1: Local directory source.
    #     # local:
    #     #   base_dir: ./benchmark-tests
    #     #   # Optional: Pre-run steps executed before tests (glob patterns).
    #     #   pre_run_steps:
    #     #     - "warmup/*.txt"
    #     #   # Test step patterns (glob patterns for setup/test/cleanup phases).
    #     #   steps:
    #     #     setup:
    #     #       - "tests/setup/*.txt"
    #     #     test:
    #     #       - "tests/test/*.txt"
    #     #     cleanup:
    #     #       - "tests/cleanup/*.txt"
    #
    #     # Option 2: Git repository source.
    #     # git:
    #     #   repo: https://github.com/example/gas-benchmarks.git
    #     #   version: main  # branch, tag, or commit hash
    #     #   # Optional: Pre-run steps executed before tests (glob patterns).
    #     #   pre_run_steps:
    #     #     - "funding/*.txt"
    #     #   # Test step patterns (glob patterns for setup/test/cleanup phases).
    #     #   steps:
    #     #     setup:
    #     #       - "tests/setup/*.txt"
    #     #     test:
    #     #       - "tests/test/*.txt"
    #     #     cleanup:
    #     #       - "tests/cleanup/*.txt"
    #
    #     # Option 3: EEST (Ethereum Execution Spec Tests) fixtures from GitHub releases.
    #     # Downloads fixtures directly from ethereum/execution-spec-tests releases.
    #     # Genesis files are generated dynamically from hive/{hash}.json files in the
    #     # fixtures using mapper.jq scripts (no separate genesis download needed).
    #     # eest_fixtures:
    #     #   github_repo: ethereum/execution-spec-tests
    #     #   github_release: benchmark@v0.0.7
    #     #   # Optional: Override the subdirectory within fixtures tarball.
    #     #   # fixtures_subdir: fixtures/blockchain_tests_engine_x  # default
    #     #   # Optional: Override URL for fixtures tarball.
    #     #   # fixtures_url: https://example.com/fixtures_benchmark.tar.gz
    #
    #     # Option 3b: EEST fixtures from GitHub Actions artifacts.
    #     # Alternative to releases - downloads from workflow run artifacts.
    #     # Requires runner.github_token or BENCHMARKOOR_RUNNER_GITHUB_TOKEN.
    #     # eest_fixtures:
    #     #   github_repo: ethereum/execution-spec-tests
    #     #   fixtures_artifact_name: fixtures_benchmark_fast  # Required (instead of github_release)
    #     #   # Optional: Specify a specific workflow run ID (uses latest if not specified)
    #     #   # fixtures_artifact_run_id: "12345678901"
    #     #   # fixtures_subdir also works with artifacts
    #
    #     # Option 3c: EEST fixtures from local directories.
    #     # Points directly at already-extracted fixtures directory.
    #     # No downloading or caching — useful for local development with locally-built EEST fixtures.
    #     # eest_fixtures:
    #     #   local_fixtures_dir: /home/user/eest-output/fixtures
    #     #   # fixtures_subdir also works with local directories
    #     #   # fixtures_subdir: fixtures/blockchain_tests_engine_x  # default
    #
    #     # Option 3d: EEST fixtures from local .tar.gz tarball.
    #     # Extracts tarball to a cache directory (keyed by content hash, re-extraction skipped
    #     # if already cached). Useful when you have locally-built tarballs but haven't extracted them.
    #     # eest_fixtures:
    #     #   local_fixtures_tarball: /home/user/eest-output/fixtures_benchmark.tar.gz
    #     #   # fixtures_subdir also works with local tarballs

# Optional: API server for authentication and user management.
# When configured, the UI can integrate with the API for login, admin, and role-based access.
# Start with: benchmarkoor api --config config.yaml
# api:
#   server:
#     listen: ":9090"
#     # CORS origins for the UI. Required when the UI is served from a different origin.
#     # When using credentials (cookies), wildcard "*" is not allowed — list specific origins.
#     # cors_origins:
#     #   - http://localhost:5173
#     #   - https://benchmarkoor.example.com
#     # Optional: Per-IP rate limiting.
#     # rate_limit:
#     #   enabled: true
#     #   auth:
#     #     requests_per_minute: 10   # Login/logout endpoints (strict)
#     #   public:
#     #     requests_per_minute: 60   # Health/config endpoints
#     #   authenticated:
#     #     requests_per_minute: 120  # Admin endpoints
#   auth:
#     session_ttl: 24h  # How long sessions last before expiring
#     # Allow unauthenticated users to access /files/ endpoints (default: false).
#     # When true, the UI allows browsing without login. When false, users must sign in.
#     # anonymous_read: false
#     # Basic (username/password) authentication.
#     basic:
#       enabled: true
#       users:
#         - username: admin
#           password: ${ADMIN_PASSWORD:-changeme}
#           role: admin      # "admin" or "readonly"
#         - username: viewer
#           password: ${VIEWER_PASSWORD:-changeme}
#           role: readonly
#     # GitHub OAuth authentication.
#     # github:
#     #   enabled: true
#     #   client_id: ${GITHUB_CLIENT_ID}
#     #   client_secret: ${GITHUB_CLIENT_SECRET}
#     #   redirect_url: http://localhost:9090/api/v1/auth/github/callback
#     #   # Map GitHub orgs to roles (users in these orgs get this role).
#     #   org_role_mapping:
#     #     my-org: admin
#     #     another-org: readonly
#     #   # Map specific GitHub users to roles (takes precedence over org mapping).
#     #   user_role_mapping:
#     #     admin-user: admin
#   database:
#     driver: sqlite  # "sqlite" or "postgres"
#     sqlite:
#       path: benchmarkoor.db
#     # postgres:
#     #   host: localhost
#     #   port: 5432
#     #   user: benchmarkoor
#     #   password: ${DB_PASSWORD}
#     #   database: benchmarkoor
#     #   ssl_mode: disable  # disable, require, verify-ca, verify-full
#   # Optional: S3 storage for serving benchmark results via presigned URLs.
#   # This is separate from runner.benchmark.results_upload.s3 (which handles uploads during runs).
#   # The API uses this to generate presigned GET URLs so the UI can fetch files directly from S3.
#   # storage:
#   #   s3:
#   #     enabled: true
#   #     endpoint_url: https://s3.us-east-1.amazonaws.com
#   #     region: us-east-1
#   #     bucket: my-benchmark-results
#   #     access_key_id: ${AWS_ACCESS_KEY_ID}
#   #     secret_access_key: ${AWS_SECRET_ACCESS_KEY}
#   #     force_path_style: false  # Required for MinIO/R2
#   #     presigned_urls:
#   #       expiry: 1h  # How long presigned URLs are valid
#   #     # S3 key prefixes the UI can browse. Each path should contain an index.json.
#   #     discovery_paths:
#   #       - results
#   #   # Alternative: Serve files directly from local filesystem directories.
#   #   # Only one backend (s3 or local) may be enabled at a time.
#   #   local:
#   #     enabled: true
#   #     # Named prefixes mapping URL path segments to absolute directories.
#   #     # Each directory should contain an index.json and run/suite sub-dirs.
#   #     # Keys become URL prefixes (e.g. /api/v1/files/results/...).
#   #     discovery_paths:
#   #       results: /data/benchmarkoor/results

  client:
    config:
      # JWT secret for Engine API authentication (optional, has a default).
      # Can be set via environment variable for security.
      # jwt: ${JWT_SECRET:-5a64f13bfb41a147711492237995b437433bcbec80a7eb2daae11132098d7bae}
      # Optional: Drop memory caches during benchmark execution (Linux only, requires root).
      # Values: "disabled" (default), "tests" (between tests), "steps" (between setup/test/cleanup steps)
      # drop_memory_caches: "disabled"
      # Optional: Rollback strategy to reset client state after each test.
      # Values:
      #   "none"                - Do not rollback
      #   "rpc-debug-setHead"   - Capture block number before test, rollback via debug_setHead after (default)
      #   "container-recreate"  - Stop and remove the container after each test, recreate from same datadir
      #   "container-checkpoint-restore"  - Podman CRIU checkpoint/restore for instant per-test restore.
      #                           Requires: container_runtime: "podman"
      #                           With datadir: requires datadir.method: "zfs" (ZFS snapshot rollback)
      #                           Without datadir: uses copy-based rollback (cp -a + rsync)
      # rollback_strategy: "rpc-debug-setHead"
      # Optional: Checkpoint-restore strategy options. Only apply to the "container-checkpoint-restore"
      # rollback strategy.
      # checkpoint_restore_strategy_options:
      #   # Store checkpoint on tmpfs (RAM) when container memory is under this threshold.
      #   # Uses the same format as resource_limits.memory (e.g., "8g", "512m").
      #   tmpfs_threshold: "8g"
      #   # Maximum size of the tmpfs mount for checkpoint storage.
      #   # Defaults to 2x tmpfs_threshold if not set. Same format (e.g., "16g", "512m").
      #   # tmpfs_max_size: "16g"
      #   # How long to wait after dropping TCP connections before the CRIU checkpoint.
      #   # After blocking new outgoing TCP connections and killing existing sockets, this
      #   # delay gives the process time to close the file descriptors.
      #   # Default: 10s. Go duration string (e.g., "10s", "5s", "30s").
      #   wait_after_tcp_drop_connections: "10s"
      #   # Whether to restart the container before taking a CRIU checkpoint.
      #   # Restarting ensures a clean process state (cold caches, clean DB shutdown).
      #   restart_container: false
      # Optional: Wait duration after RPC becomes ready before running tests.
      # Useful for clients like Erigon that need time to complete internal sync pipelines
      # after their RPC endpoint becomes available.
      # wait_after_rpc_ready: 30s
      # Optional: Maximum duration for the test execution phase.
      # If exceeded, the run is cancelled with "timed_out" status. Partial results are kept.
      # run_timeout: 2h
      # Optional: Retry engine_newPayload calls when client returns SYNCING status.
      # Useful for clients with internal sync pipelines (e.g., Erigon) that may return
      # SYNCING while still processing blocks internally.
      # retry_new_payloads_syncing_state:
      #   enabled: true
      #   max_retries: 10
      #   backoff: 1s
      # Optional: Arbitrary RPC calls to execute after each test step.
      # These calls are NOT timed and do NOT affect test results. Useful for collecting
      # debug traces or other post-test data from the client.
      # Template variables are available in params:
      #   {{.BlockHash}}      - Hash of the latest block (e.g., "0xabc...")
      #   {{.BlockNumber}}    - Decimal block number (e.g., "1234")
      #   {{.BlockNumberHex}} - Hex block number with 0x prefix (e.g., "0x4d2")
      # post_test_rpc_calls:
      #   - method: debug_traceBlockByNumber
      #     params: ["{{.BlockNumberHex}}", {"tracer": "callTracer"}]
      #     dump:
      #       enabled: true
      #       filename: debug_traceBlockByNumber  # Written to {resultsDir}/{testName}/post_test_rpc_calls/{filename}.json
      #   - method: debug_traceBlockByHash
      #     params: ["{{.BlockHash}}"]
      #     timeout: 2m  # Per-call timeout (default: 30s)
      #     dump:
      #       enabled: true
      #       filename: debug_traceBlockByHash
      # Optional: Send an engine_forkchoiceUpdatedV3 call after RPC is ready.
      # Uses the latest block hash from eth_getBlockByNumber("latest").
      # Retries until the client accepts the FCU with VALID status, confirming
      # it has finished internal initialization and is ready for test execution.
      # Supports shorthand: `bootstrap_fcu: true` (defaults to max_retries: 30, backoff: 1s)
      # bootstrap_fcu:
      #   enabled: true
      #   max_retries: 30
      #   backoff: 1s
      # Optional: Container resource limits (applied to all instances by default).
      # resource_limits:
      #   # CPU pinning - use ONE of the following:
      #   #   cpuset_count: N    - Pick N random CPUs (new random selection each run)
      #   #   cpuset: [0, 1, 2]  - Pin to specific CPUs
      #   cpuset_count: 4
      #   # Memory limit (supports units: b, k, m, g, e.g., "16g", "4096m")
      #   memory: "16g"
      #   # Disable swap for the container (sets memory-swap equal to memory and swappiness to 0)
      #   swap_disabled: true
      #   # Block I/O throttling (optional)
      #   blkio_config:
      #     # Limit device read bandwidth (supports units: b, k, m, g)
      #     device_read_bps:
      #       - path: /dev/sdb
      #         rate: '12mb'
      #     # Limit device read IOPS (integer)
      #     device_read_iops:
      #       - path: /dev/sdb
      #         rate: '120'
      #     # Limit device write bandwidth
      #     device_write_bps:
      #       - path: /dev/sdb
      #         rate: '1024k'
      #     # Limit device write IOPS
      #     device_write_iops:
      #       - path: /dev/sdb
      #         rate: '30'
      #   # CPU frequency management (Linux only, requires root and cpufreq subsystem)
      #   # These settings are applied to the CPUs specified by cpuset/cpuset_count,
      #   # or all online CPUs if neither is specified.
      #   # Fixed CPU frequency (supports: "2000MHz", "2.4GHz", "MAX")
      #   cpu_freq: "2000MHz"
      #   # Enable/disable turbo boost (Intel: no_turbo, AMD: boost)
      #   cpu_turboboost: false
      #   # CPU frequency governor (common: performance, powersave, schedutil)
      #   # Defaults to "performance" when cpu_freq is set
      #   cpu_freq_governor: performance
      # Genesis file URLs per client type
      genesis:
        besu: https://github.com/nethermindeth/gas-benchmarks/raw/refs/heads/main/scripts/genesisfiles/besu/zkevmgenesis.json
        erigon: https://github.com/nethermindeth/gas-benchmarks/raw/refs/heads/main/scripts/genesisfiles/geth/zkevmgenesis.json
        geth: https://github.com/nethermindeth/gas-benchmarks/raw/refs/heads/main/scripts/genesisfiles/geth/zkevmgenesis.json
        nethermind: https://github.com/nethermindeth/gas-benchmarks/raw/refs/heads/main/scripts/genesisfiles/nethermind/zkevmgenesis.json
        nimbus: https://github.com/nethermindeth/gas-benchmarks/raw/refs/heads/main/scripts/genesisfiles/geth/zkevmgenesis.json
        reth: https://github.com/nethermindeth/gas-benchmarks/raw/refs/heads/main/scripts/genesisfiles/geth/zkevmgenesis.json

    # Default images per client (used when 'image' is not specified):
    #   geth:       ethpandaops/geth:performance
    #   nethermind: ethpandaops/nethermind:performance
    #   besu:       ethpandaops/besu:performance
    #   erigon:     ethpandaops/erigon:performance
    #   nimbus:     statusim/nimbus-eth1:performance
    #   reth:       ethpandaops/reth:performance

    # Optional: Pre-populated data directories per client type.
    # When configured, the source directory is prepared and mounted into the container,
    # and the init container is skipped (data is already initialized).
    # datadirs:
    #   geth:
    #     source_dir: ./data/snapshots/geth
    #     # container_dir: /data  # defaults to client's data directory if omitted
    #     # Method for preparing the data directory:
    #     #   copy           - parallel Go copy (default, works everywhere, has progress)
    #     #   overlayfs      - uses Linux overlayfs for near-instant setup (requires root)
    #     #   fuse-overlayfs - uses fuse-overlayfs for near-instant setup (no root required)
    #     #                    Requires: apt install fuse-overlayfs (or equivalent)
    #     #                    Requires: user_allow_other in /etc/fuse.conf if Docker runs as root
    #     #                    WARNING: fuse-overlayfs is SLOW compared to overlayfs. You'll get
    #     #                             3x Worse results.
    #     #   zfs            - uses ZFS snapshots and clones for near-instant, copy-on-write setup
    #     #                    Requires: source_dir must be on a ZFS filesystem
    #     #                    Requires: root access OR ZFS delegations configured:
    #     #                      zfs allow -u <user> clone,create,destroy,mount,snapshot <dataset>
    #     #                    The dataset is auto-detected from the source_dir mount point.
    #     method: copy
    #   reth:
    #     source_dir: ./data/snapshots/reth
    #     # container_dir defaults to /var/lib/reth for reth
    #     method: fuse-overlayfs  # near-instant, no root required
    #   # Example: ZFS-backed data directory
    #   # nethermind:
    #   #   source_dir: /tank/benchmarkoor/nethermind-snapshot  # Must be on ZFS
    #   #   container_dir: /nethermind/data
    #   #   method: zfs  # near-instant via ZFS clones

  instances:
    - id: geth-latest
      client: geth
      # image: ${GETH_IMAGE:-ethpandaops/geth:performance}
      # pull_policy: always (default)
      # Optional overrides:
      # entrypoint: []
      # command: []
      # extra_args:  # Additional arguments appended to command
      #   - --verbosity=5
      # restart: never
      # environment:
      #   SOME_VAR: ${MY_ENV_VAR}
      # genesis: <override-url>
      # datadir:  # Instance-level datadir (overrides global datadirs)
      #   source_dir: ${DATA_SNAPSHOTS_DIR}/geth
      #   container_dir: /data
      # drop_memory_caches: "steps"  # Instance-level override (optional)
      # rollback_strategy: "rpc-debug-setHead"  # Instance-level override (optional)
      # checkpoint_restore_strategy_options:  # Instance-level override (optional, replaces global)
      #   tmpfs_threshold: "8g"
      #   tmpfs_max_size: "16g"
      #   wait_after_tcp_drop_connections: "10s"
      # wait_after_rpc_ready: 60s  # Instance-level override (optional)
      # run_timeout: 1h  # Instance-level override (optional)
      # retry_new_payloads_syncing_state:  # Instance-level override (optional)
      #   enabled: true
      #   max_retries: 10
      #   backoff: 1s
      # resource_limits:  # Instance-level override (optional)
      #   cpuset_count: 2  # Override global with fewer CPUs
      #   memory: "8g"     # Override global memory limit
      # post_test_rpc_calls:  # Instance-level override (optional, replaces global)
      #   - method: debug_traceBlockByNumber
      #     params: ["{{.BlockNumberHex}}"]
      #     dump:
      #       enabled: true
      #       filename: trace
      # bootstrap_fcu: true  # Instance-level override (optional)

    # Example: running multiple clients
    # - id: nethermind-latest
    #   client: nethermind
    #   # image: ethpandaops/nethermind:performance (default)

    # - id: besu-latest
    #   client: besu
    #   # image: ethpandaops/besu:performance (default)

    # - id: reth-latest
    #   client: reth
    #   # image: ethpandaops/reth:performance (default)

    # - id: erigon-latest
    #   client: erigon
    #   # image: ethpandaops/erigon:performance (default)

    # - id: nimbus-latest
    #   client: nimbus
    #   # image: statusim/nimbus-eth1:performance (default)