refactor(test-benchmark): align test suite with glam-devnet-7 spec (#3187)

Co-authored-by: marioevz <marioevz@gmail.com>
Co-authored-by: Jochem Brouwer <jochembrouwer96@gmail.com>
Co-authored-by: spencer-tb <spencer.tb@ethereum.org>

Author: 4 people

.github/configs/evm.yaml

@@ -1,7 +1,7 @@
 benchmark:
   impl: geth
   repo: ethereum/go-ethereum
-  ref: master
+  ref: glamsterdam-devnet-7
   evm-bin: evm
   xdist: auto
 eels:

.github/configs/feature.yaml

@@ -14,12 +14,12 @@ tests:
 
 benchmark:
   evm-type: benchmark
-  fill-params: --fork=Osaka --generate-all-formats --gas-benchmark-values 1,5,10,30,60,100,150 ./tests/benchmark/compute --maxprocesses=30 --dist=worksteal
+  fill-params: --fork=Amsterdam --generate-all-formats --gas-benchmark-values 1,5,10,30,60,100,150 ./tests/benchmark/compute --maxprocesses=30 --dist=worksteal
   feature_only: true
 
 benchmark_fast:
   evm-type: benchmark
-  fill-params: --fork=Osaka --generate-all-formats --gas-benchmark-values 100 ./tests/benchmark/compute
+  fill-params: --fork=Amsterdam --generate-all-formats --gas-benchmark-values 100 ./tests/benchmark/compute
   feature_only: true
 
 # Shared entry for all `<feat>-devnet` releases; matched by `-devnet` suffix.

Justfile

@@ -261,7 +261,7 @@ bench-gas *args:
         --generate-pre-alloc-groups \
         --evm-bin="{{ evm_bin }}" \
         --gas-benchmark-values 1 \
-        --fork Osaka \
+        --fork Amsterdam \
         -m "not slow" \
         -n auto --maxprocesses 10 --dist=loadgroup \
         --output="{{ output_dir }}/bench-gas/pre-alloc" \
@@ -274,7 +274,7 @@ bench-gas *args:
     uv run fill \
         --evm-bin="{{ evm_bin }}" \
         --gas-benchmark-values 1 \
-        --fork Osaka \
+        --fork Amsterdam \
         -m "blockchain_test and (not derived_test) and (not slow)" \
         -n auto --maxprocesses 10 --dist=loadgroup \
         --durations=20 \
@@ -288,7 +288,7 @@ bench-gas *args:
     @rm -rf tests/json_loader/bench_gas_fixtures
     ln -sfn "{{ output_dir }}/bench-gas/fixtures" tests/json_loader/bench_gas_fixtures
     cd tests/json_loader && uv run --python pypy3.11 --no-dev --group test pytest \
-        --fork Osaka \
+        --fork Amsterdam \
         --allow-post-state-hash \
         -n auto --maxprocesses 10 --dist=loadfile \
         --durations=20 \
@@ -302,8 +302,8 @@ bench-opcode *args:
     uv run fill \
         --evm-bin="{{ evm_bin }}" \
         --fixed-opcode-count 1 \
-        --fork Osaka \
-        -m repricing \
+        --fork Amsterdam \
+        -m "repricing and not slow" \
         -n auto --maxprocesses 10 --dist=loadgroup \
         -k "not test_alt_bn128 and not test_bls12_381 and not test_modexp and not uncachable" \
         --output="{{ output_dir }}/bench-opcode/fixtures" \
@@ -321,10 +321,10 @@ bench-opcode-config *args:
     uv run fill \
         --evm-bin="{{ evm_bin }}" \
         --fixed-opcode-count \
-        --fork Osaka \
-        -m repricing \
+        --fork Amsterdam \
+        -m "repricing and not slow" \
         -n auto --maxprocesses 10 --dist=loadgroup \
-        -k "not test_alt_bn128 and not test_bls12_381 and not test_modexp and not test_point_evaluation_uncachable" \
+        -k "not test_alt_bn128 and not test_bls12_381 and not test_modexp and not uncachable" \
         --output="{{ output_dir }}/bench-opcode-config/fixtures" \
         --basetemp="{{ output_dir }}/bench-opcode-config/tmp" \
         --log-to "{{ output_dir }}/bench-opcode-config/logs" \

packages/testing/src/execution_testing/__init__.py

@@ -110,6 +110,7 @@
     SequentialAddressLayout,
     Switch,
     TransactionWithCost,
+    TxOutcome,
     While,
     WhileGas,
     extend_with_defaults,
@@ -226,6 +227,7 @@
     "TransactionTestFiller",
     "TransactionType",
     "TransactionWithCost",
+    "TxOutcome",
     "TransitionFork",
     "While",
     "WhileGas",

packages/testing/src/execution_testing/benchmark/benchmark_code_generator.py

@@ -100,21 +100,38 @@ def deploy_contracts(self, *, pre: Alloc, fork: Fork) -> Address:
 
         # Create caller contract that repeatedly calls the target contract
         # attack = POP(
-        #             STATICCALL(GAS, target_contract_address, 0, 0, 0, 0)
+        #             (STATIC)CALL(GAS, target_contract_address, ...)
         #          )
         #
         # setup + JUMPDEST + attack + attack + ... + attack +
         # JUMP(setup_length)
-        code_sequence = Op.POP(
-            Op.STATICCALL(
-                Op.GAS,
-                self._target_contract_address,
-                Op.PUSH0,
-                Op.CALLDATASIZE,
-                Op.PUSH0,
-                Op.PUSH0,
+        #
+        # The target must be entered via CALL when it contains
+        # state-changing opcodes: a STATICCALL'd frame faults on the
+        # first one and the target executes nothing.
+        # A state-changing target must be entered via CALL, not STATICCALL.
+        # CALL takes a value argument STATICCALL lacks; push the zero value
+        # with PUSH0, one gas cheaper than the default PUSH1 0x00.
+        if self.uses_state_changing_opcode():
+            wrapper_call = Op.CALL(
+                gas=Op.GAS,
+                address=self._target_contract_address,
+                value=Op.PUSH0,
+                args_offset=Op.PUSH0,
+                args_size=Op.CALLDATASIZE,
+                ret_offset=Op.PUSH0,
+                ret_size=Op.PUSH0,
             )
-        )
+        else:
+            wrapper_call = Op.STATICCALL(
+                gas=Op.GAS,
+                address=self._target_contract_address,
+                args_offset=Op.PUSH0,
+                args_size=Op.CALLDATASIZE,
+                ret_offset=Op.PUSH0,
+                ret_size=Op.PUSH0,
+            )
+        code_sequence = Op.POP(wrapper_call)
 
         caller_code = self.generate_repeated_code(
             setup=Op.CALLDATACOPY(Op.PUSH0, Op.PUSH0, Op.CALLDATASIZE),

packages/testing/src/execution_testing/cli/pytest_commands/plugins/execute/pre_alloc.py

@@ -29,6 +29,7 @@
 )
 from execution_testing.forks import Fork, TransitionFork
 from execution_testing.logging import get_logger
+from execution_testing.recipient_type import RecipientType
 from execution_testing.rpc import EthRPC
 from execution_testing.rpc.rpc_types import TransactionByHashResponse
 from execution_testing.test_types import (
@@ -287,8 +288,11 @@ def _compute_deploy_gas_limit(
     else:
         regular_gas = buffered_regular_gas
 
-    # State portion, from the block reservoir.
+    # State portion, from the block reservoir. The created account's
+    # NEW_ACCOUNT is charged at the top frame for create transactions
+    # and by CREATE2 at access for proxy deploys — same amount.
     state_gas = fork.code_deposit_state_gas(code_size=deploy_code_size)
+    state_gas += fork.transaction_top_frame_state_gas(contract_creation=True)
     state_gas += storage_slots * sstore_state_gas
 
     deploy_gas_limit = regular_gas + state_gas
@@ -630,6 +634,35 @@ def _fund_eoa(
             )
             intrinsic_calc = fork.transaction_intrinsic_cost_calculator()
 
+            worst_case_auth = [
+                AuthorizationTuple(
+                    address=Address(0),
+                    v=0,
+                    r=0,
+                    s=0,
+                    creates_account=True,
+                    writes_delegation=True,
+                    first_write=True,
+                )
+            ]
+            auth_fund_gas_limit = (
+                intrinsic_calc(
+                    authorization_list_or_count=1,
+                    sends_value=True,
+                    recipient_type=RecipientType.EMPTY_ACCOUNT,
+                )
+                + fork.transaction_top_frame_gas_calculator()(
+                    sends_value=True,
+                    recipient_type=RecipientType.EMPTY_ACCOUNT,
+                    authorizations=worst_case_auth,
+                )
+                + fork.transaction_top_frame_state_gas(
+                    sends_value=True,
+                    recipient_type=RecipientType.EMPTY_ACCOUNT,
+                    authorizations=worst_case_auth,
+                )
+            )
+
             if storage is not None:
                 if not isinstance(storage, Storage):
                     storage = Storage.model_validate(storage)
@@ -703,7 +736,7 @@ def _fund_eoa(
                             signer=eoa,
                         ),
                     ],
-                    gas_limit=(intrinsic_calc(authorization_list_or_count=1)),
+                    gas_limit=auth_fund_gas_limit,
                 )
                 eoa.nonce = Number(eoa.nonce + 1)
             else:
@@ -721,7 +754,7 @@ def _fund_eoa(
                             signer=eoa,
                         ),
                     ],
-                    gas_limit=intrinsic_calc(authorization_list_or_count=1),
+                    gas_limit=auth_fund_gas_limit,
                 )
                 eoa.nonce = Number(eoa.nonce + 1)
 

packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/filler.py

@@ -1754,6 +1754,9 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
                 # If operation mode is benchmarking, check the gas used.
                 self.validate_benchmark_gas(
                     benchmark_gas_used=fill_result.benchmark_gas_used,
+                    benchmark_block_gas_used=(
+                        fill_result.benchmark_block_gas_used
+                    ),
                     gas_benchmark_value=gas_benchmark_value,
                 )
 

packages/testing/src/execution_testing/forks/forks/eips/amsterdam/eip_8037.py

@@ -150,6 +150,9 @@ def opcode_state_map(
             Opcodes.SELFDESTRUCT: (
                 lambda op: cls._calculate_selfdestruct_state_gas(op, gas_costs)
             ),
+            Opcodes.CALL: lambda op: cls._calculate_call_state_gas(
+                op, gas_costs
+            ),
         }
 
     @classmethod
@@ -400,3 +403,39 @@ def _calculate_selfdestruct_gas(
         if opcode.metadata["account_new"]:
             gas_cost -= gas_costs.NEW_ACCOUNT
         return gas_cost
+
+    @classmethod
+    def _calculate_call_state_gas(
+        cls, opcode: OpcodeBase, gas_costs: GasCosts
+    ) -> int:
+        """
+        Calculate the CALL state gas cost: `NEW_ACCOUNT` when a value
+        transfer funds a new account. Before EIP-8037 this was folded
+        into the regular CALL cost (EIP-161); under EIP-8037 it is
+        exposed here as state gas, mirroring
+        `_calculate_selfdestruct_state_gas`.
+        """
+        metadata = opcode.metadata
+        if "value_transfer" in metadata and metadata["value_transfer"]:
+            if metadata["account_new"]:
+                return gas_costs.NEW_ACCOUNT
+        return 0
+
+    @classmethod
+    def _calculate_call_gas(
+        cls, opcode: OpcodeBase, gas_costs: GasCosts
+    ) -> int:
+        """
+        Calculate the regular CALL gas cost. The EIP-161 base
+        calculation folds `NEW_ACCOUNT` into the regular cost when a
+        value transfer funds a new account; EIP-8037 moves that charge
+        to the state-gas dimension (see `_calculate_call_state_gas`),
+        so this subtracts the `NEW_ACCOUNT` term back out of the
+        inherited regular cost.
+        """
+        gas_cost = super()._calculate_call_gas(opcode, gas_costs)
+        metadata = opcode.metadata
+        if "value_transfer" in metadata and metadata["value_transfer"]:
+            if metadata["account_new"]:
+                gas_cost -= gas_costs.NEW_ACCOUNT
+        return gas_cost

packages/testing/src/execution_testing/specs/base.py

@@ -94,6 +94,7 @@ class FillResult(BaseModel):
     fixture: BaseFixture
     gas_optimization: int | None
     benchmark_gas_used: int | None = None
+    benchmark_block_gas_used: int | None = None
     benchmark_opcode_count: OpcodeCount | None = None
     post_verifications: PostVerifications | None = None
     metadata: Dict[str, Any] = Field(default_factory=dict)
@@ -259,12 +260,25 @@ def get_genesis_environment(self) -> Environment:
         )
 
     def validate_benchmark_gas(
-        self, *, benchmark_gas_used: int | None, gas_benchmark_value: int
+        self,
+        *,
+        benchmark_gas_used: int | None,
+        gas_benchmark_value: int,
+        benchmark_block_gas_used: int | None = None,
     ) -> None:
         """
         Validates the total consumed gas of the last block in the test matches
         the expectation of the benchmark test.
 
+        ``benchmark_gas_used`` is the combined gas across all dimensions (the
+        receipt ``cumulativeGasUsed``) and is checked against
+        ``expected_benchmark_gas_used``. ``benchmark_block_gas_used`` is the
+        block-header gas, i.e. the maximum across the independent gas
+        dimensions (EIP-8037); it is what must stay within the block gas
+        limit, because the combined value can legitimately exceed it. When it
+        is not available (e.g. execute mode), the combined value is used for
+        the ceiling check instead.
+
         Requires the following fields to be set:
         - expected_benchmark_gas_used
         - operation_mode
@@ -287,9 +301,17 @@ def validate_benchmark_gas(
                 f"({expected_benchmark_gas_used}), "
                 f"difference: {diff}"
             )
-        # Gas used should never exceed the maximum benchmark gas allowed.
-        assert benchmark_gas_used <= gas_benchmark_value, (
-            f"benchmark_gas_used ({benchmark_gas_used}) exceeds maximum "
+        # No single gas dimension may exceed the block gas limit. The
+        # block-header gas is the max across dimensions; the combined
+        # regular+state gas may exceed the target under EIP-8037, so the
+        # ceiling is checked against the header value when available.
+        block_gas_used = (
+            benchmark_block_gas_used
+            if benchmark_block_gas_used is not None
+            else benchmark_gas_used
+        )
+        assert block_gas_used <= gas_benchmark_value, (
+            f"benchmark block gas used ({block_gas_used}) exceeds maximum "
             "benchmark gas allowed for this configuration: "
             f"{gas_benchmark_value}"
         )