diff --git a/packages/testing/src/execution_testing/base_types/tests/test_keccak_dispatch.py b/packages/testing/src/execution_testing/base_types/tests/test_keccak_dispatch.py index 95bd341ba6d..af3fb835d21 100644 --- a/packages/testing/src/execution_testing/base_types/tests/test_keccak_dispatch.py +++ b/packages/testing/src/execution_testing/base_types/tests/test_keccak_dispatch.py @@ -162,13 +162,3 @@ def test_eest_bytes_keccak256_matches_eels() -> None: from_eest = bytes(Bytes(buffer).keccak256()) from_eels = bytes(keccak256(buffer)) assert from_eest == from_eels - - -def test_eest_trie_keccak256_matches_eels() -> None: - """`trie.keccak256` and EELS `keccak256` return identical digests.""" - from ethereum.crypto.hash import keccak256 as eels - - from ...test_types.trie import keccak256 as trie - - for buffer in (b"", b"hashme", bytes(range(256))): - assert bytes(trie(buffer)) == bytes(eels(buffer)) diff --git a/packages/testing/src/execution_testing/client_clis/cli_types.py b/packages/testing/src/execution_testing/client_clis/cli_types.py index 8fec397ebc2..0a272374f68 100644 --- a/packages/testing/src/execution_testing/client_clis/cli_types.py +++ b/packages/testing/src/execution_testing/client_clis/cli_types.py @@ -647,7 +647,12 @@ def model_dump(self, mode: str, **model_dump_config: Any) -> Any: class TransitionToolOutput: """Transition tool output.""" - alloc: LazyAlloc + # External t8ns return JSON; the testing framework wraps them in a + # ``LazyAlloc`` subclass so the multi-MB alloc isn't parsed until a + # consumer asks. The in-process EELS path materializes ``Alloc`` + # eagerly and hands it through directly — no lazy wrapper needed. + # Consumers iterate via ``.get()``, which both branches satisfy. + alloc: LazyAlloc | Alloc result: Result body: Bytes | None = None diff --git a/packages/testing/src/execution_testing/client_clis/clis/besu.py b/packages/testing/src/execution_testing/client_clis/clis/besu.py index ce0789cd399..e54f7c0fcc7 100644 --- a/packages/testing/src/execution_testing/client_clis/clis/besu.py +++ b/packages/testing/src/execution_testing/client_clis/clis/besu.py @@ -278,7 +278,7 @@ def _evaluate( dump_files_to_directory( debug_output_path, { - "output/alloc.json": output.alloc.raw, + "output/alloc.json": output.alloc, "output/result.json": output.result.model_dump( mode="json", **model_dump_config ), diff --git a/packages/testing/src/execution_testing/client_clis/clis/execution_specs.py b/packages/testing/src/execution_testing/client_clis/clis/execution_specs.py index 55434840aeb..4e3f94581a9 100644 --- a/packages/testing/src/execution_testing/client_clis/clis/execution_specs.py +++ b/packages/testing/src/execution_testing/client_clis/clis/execution_specs.py @@ -2,19 +2,20 @@ Ethereum Specs EVM Transition Tool Interface. """ -import json import tempfile -from io import StringIO from pathlib import Path from typing import Any, ClassVar, Dict, Optional import ethereum -from ethereum_spec_tools.evm_tools import create_parser from ethereum_spec_tools.evm_tools.t8n import T8N, ForkCache +from ethereum_spec_tools.evm_tools.t8n.evm_trace.eip3155 import Eip3155Tracer +from ethereum_spec_tools.evm_tools.t8n.evm_trace.group import GroupTracer from ethereum_spec_tools.evm_tools.utils import get_supported_forks from typing_extensions import override -from execution_testing.client_clis.cli_types import TransitionToolOutput +from execution_testing.client_clis.cli_types import ( + TransitionToolOutput, +) from execution_testing.client_clis.file_utils import ( dump_files_to_directory, ) @@ -72,72 +73,53 @@ def _evaluate( profiler: Profiler, ) -> TransitionToolOutput: """ - Evaluate using the EELS T8N entry point. + Evaluate using the EELS T8N entry point in-process. + + ``transition_tool_data`` is handed to ``T8N`` as-is — fork, + chain_id, reward, state_test, blob_schedule all flow through + — and ``T8N.run()`` returns the ``TransitionToolOutput`` + directly. """ del slow_request, profiler - request_data = transition_tool_data.get_request_data() - request_data_json = request_data.model_dump( - mode="json", **model_dump_config - ) temp_dir = tempfile.TemporaryDirectory() - t8n_args = [ - "t8n", - "--input.alloc=stdin", - "--input.env=stdin", - "--input.txs=stdin", - "--output.result=stdout", - "--output.body=stdout", - "--output.alloc=stdout", - f"--output.basedir={temp_dir.name}", - f"--state.fork={request_data_json['state']['fork']}", - f"--state.chainid={request_data_json['state']['chainid']}", - f"--state.reward={request_data_json['state']['reward']}", - ] - - if transition_tool_data.state_test: - t8n_args.append("--state-test") - - if transition_tool_data.blob_params: - fork = transition_tool_data.fork - if fork.bpo_fork() and fork != fork.non_bpo_ancestor(): - # Only send this information for BPO forks. - # TODO: This should be optimized by the t8n tool instead. - t8n_args.append("--input.blobParams=stdin") + tracers = None if self.trace: - t8n_args.extend( - [ - "--trace", - "--trace.memory", - "--trace.returndata", - ] + # TODO: Eip3155 traces still round-trip through tempfile + # JSON — the tracer writes one ``trace-.jsonl`` per tx + # to ``output_basedir`` and ``collect_traces`` reads them + # back. Same JSON round-trip we eliminated for alloc / + # result / body; a follow-up should wire the tracer + # output through memory like the rest of the in-process + # path. + tracers = GroupTracer() + tracers.add( + Eip3155Tracer( + trace_memory=True, + trace_stack=True, + trace_return_data=True, + output_basedir=temp_dir.name, + ) ) - parser = create_parser() - t8n_options = parser.parse_args(t8n_args) - - out_stream = StringIO() - - in_stream = StringIO(json.dumps(request_data_json["input"])) - - t8n = T8N(t8n_options, out_stream, in_stream, self.fork_cache) - t8n.run() - - output_dict = json.loads(out_stream.getvalue()) - output: TransitionToolOutput = TransitionToolOutput.model_validate( - output_dict, context={"exception_mapper": self.exception_mapper} + t8n = T8N( + transition_tool_data, + cache=self.fork_cache, + tracers=tracers, + exception_mapper=self.exception_mapper, ) + output = t8n.run() if debug_output_path: dump_files_to_directory( debug_output_path, { - "input/alloc.json": request_data.input.alloc, - "input/env.json": request_data.input.env, + "input/alloc.json": transition_tool_data.alloc, + "input/env.json": transition_tool_data.env, "input/txs.json": [ tx.model_dump(mode="json", **model_dump_config) - for tx in request_data.input.txs + for tx in transition_tool_data.txs ], }, ) diff --git a/packages/testing/src/execution_testing/client_clis/transition_tool.py b/packages/testing/src/execution_testing/client_clis/transition_tool.py index 93d1d7e17d5..63437d5bf7e 100644 --- a/packages/testing/src/execution_testing/client_clis/transition_tool.py +++ b/packages/testing/src/execution_testing/client_clis/transition_tool.py @@ -666,7 +666,7 @@ def _evaluate_server( dump_files_to_directory( debug_output_path, { - "output/alloc.json": output.alloc.raw, + "output/alloc.json": output.alloc, "output/result.json": output.result, "output/txs.rlp": str(output.body), "response_info.txt": response_info, diff --git a/packages/testing/src/execution_testing/specs/blockchain.py b/packages/testing/src/execution_testing/specs/blockchain.py index 39fc641c3fb..d35da12bbd3 100644 --- a/packages/testing/src/execution_testing/specs/blockchain.py +++ b/packages/testing/src/execution_testing/specs/blockchain.py @@ -387,7 +387,7 @@ class BuiltBlock(CamelModel): header: FixtureHeader env: Environment - alloc: LazyAlloc + alloc: LazyAlloc | Alloc state_root: Hash txs: List[Transaction] ommers: List[FixtureHeader] diff --git a/packages/testing/src/execution_testing/test_types/account_types.py b/packages/testing/src/execution_testing/test_types/account_types.py index aba69bed4aa..dee3e4dcf43 100644 --- a/packages/testing/src/execution_testing/test_types/account_types.py +++ b/packages/testing/src/execution_testing/test_types/account_types.py @@ -1,9 +1,10 @@ """Account-related types for Ethereum tests.""" import json -from dataclasses import dataclass, field +from dataclasses import dataclass from enum import Enum, auto from typing import ( + AbstractSet, Any, Dict, ItemsView, @@ -15,14 +16,20 @@ Tuple, ) +import ethereum.state as spec_state from coincurve.keys import PrivateKey -from ethereum_types.bytes import Bytes20 +from ethereum.crypto.hash import Hash32 +from ethereum.crypto.hash import keccak256 as spec_keccak256 +from ethereum.merkle_patricia_trie import InternalNode +from ethereum_types.bytes import Bytes, Bytes20 from ethereum_types.numeric import U256, Bytes32, Uint +from pydantic import PrivateAttr from execution_testing.base_types import ( Account, Address, Hash, + HashInt, Number, Storage, StorageRootType, @@ -34,82 +41,22 @@ NumberConvertible, ) -from .trie import ( - EMPTY_TRIE_ROOT, - FrontierAccount, - Trie, - root, - trie_get, - trie_set, -) from .utils import keccak256 -FrontierAddress = Bytes20 - - -@dataclass -class State: - """Contains all information that is preserved between transactions.""" - - _main_trie: Trie[Bytes20, Optional[FrontierAccount]] = field( - default_factory=lambda: Trie(secured=True, default=None) - ) - _storage_tries: Dict[Bytes20, Trie[Bytes32, U256]] = field( - default_factory=dict - ) - _snapshots: List[ - Tuple[ - Trie[Bytes20, Optional[FrontierAccount]], - Dict[Bytes20, Trie[Bytes32, U256]], - ] - ] = field(default_factory=list) - -def set_account( - state: State, address: Bytes20, account: Optional[FrontierAccount] -) -> None: +class _Phase(Enum): """ - Set the `Account` object at an address. Setting to `None` deletes the - account (but not its storage, see `destroy_account()`). - """ - trie_set(state._main_trie, address, account) - + Lifecycle phase of an `Alloc` instance used as a `PreState`. -def set_storage( - state: State, address: Bytes20, key: Bytes32, value: U256 -) -> None: - """ - Set a value at a storage key on an account. Setting to `U256(0)` deletes - the key. + See `Alloc` for the rules each phase enforces. """ - assert trie_get(state._main_trie, address) is not None - - trie = state._storage_tries.get(address) - if trie is None: - trie = Trie(secured=True, default=U256(0)) - state._storage_tries[address] = trie - trie_set(trie, key, value) - if trie._data == {}: - del state._storage_tries[address] - - -def storage_root(state: State, address: Bytes20) -> Bytes32: - """Calculate the storage root of an account.""" - assert not state._snapshots - if address in state._storage_tries: - return root(state._storage_tries[address]) - else: - return EMPTY_TRIE_ROOT - -def state_root(state: State) -> Bytes32: - """Calculate the state root.""" - assert not state._snapshots - - def get_storage_root(address: Bytes20) -> Bytes32: - return storage_root(state, address) - - return root(state._main_trie, get_storage_root=get_storage_root) + CONSTRUCTION = auto() + """Free mutations on `self.root` are allowed; no cache exists.""" + LIVE = auto() + """Cache built; only `apply_diff` may mutate.""" + FROZEN = auto() + """No mutations are allowed.""" class EOA(Address): @@ -161,7 +108,20 @@ def copy(self) -> Self: class Alloc(BaseAlloc): - """Allocation of accounts in the state, pre and post test execution.""" + """ + Allocation of accounts in the state, pre and post test execution. + + Doubles as a `PreState` provider for the spec's state transition: once + any `PreState` method is called the instance transitions from + `CONSTRUCTION` to `LIVE` (a code-hash → bytes cache is built once) and + further free mutations via `__setitem__`/`__delitem__` are rejected. + The only mutation entry point in `LIVE` is `apply_diff`, which patches + `self.root` and updates the cache in lockstep. `freeze` locks the + allocation for read-only assertion use. + """ + + _phase: _Phase = PrivateAttr(default=_Phase.CONSTRUCTION) + _code_store: Dict[Hash32, Bytes] = PrivateAttr(default_factory=dict) @dataclass(kw_only=True) class UnexpectedAccountError(Exception): @@ -298,6 +258,7 @@ def __setitem__( account: Account | None, ) -> None: """Set account associated with an address.""" + self._require_construction("__setitem__") if not isinstance(address, Address): address = Address(address) self.root[address] = account @@ -306,6 +267,7 @@ def __delitem__( self, address: Address | FixedSizeBytesConvertible ) -> None: """Delete account associated with an address.""" + self._require_construction("__delitem__") if not isinstance(address, Address): address = Address(address) self.root.pop(address, None) @@ -330,36 +292,20 @@ def empty_accounts(self) -> List[Address]: address for address, account in self.root.items() if not account ] + def get(self) -> "Alloc": + """ + Return self. + + ``TransitionToolOutput.alloc`` is typed ``LazyAlloc | Alloc``; + callers iterate it via ``.get()`` so they don't need to branch + on the concrete type. This satisfies that contract for an + already-materialized ``Alloc``. + """ + return self + def state_root(self) -> Hash: """Return state root of the allocation.""" - state = State() - for address, account in self.root.items(): - if account is None: - continue - set_account( - state=state, - address=FrontierAddress(address), - account=FrontierAccount( - nonce=Uint(account.nonce) - if account.nonce is not None - else Uint(0), - balance=( - U256(account.balance) - if account.balance is not None - else U256(0) - ), - code=account.code if account.code is not None else b"", - ), - ) - if account.storage is not None: - for key, value in account.storage.root.items(): - set_storage( - state=state, - address=FrontierAddress(address), - key=Bytes32(Hash(key)), - value=U256(value), - ) - return Hash(state_root(state)) + return Hash(spec_state.state_root(self._materialize_state())) def verify_post_alloc(self, got_alloc: "Alloc") -> None: """ @@ -388,6 +334,247 @@ def verify_post_alloc(self, got_alloc: "Alloc") -> None: else: raise Alloc.MissingAccountError(address=address) + # ------------------------------------------------------------------ + # PreState protocol implementation + # ------------------------------------------------------------------ + + def _require_construction(self, operation: str) -> None: + """Reject mutations once the allocation has left construction.""" + if self._phase is not _Phase.CONSTRUCTION: + raise RuntimeError( + f"{operation} not allowed: Alloc is in phase " + f"{self._phase.name}. Mutate via apply_diff during LIVE, " + f"or call freeze() to lock the allocation." + ) + + def _build_cache(self) -> None: + """Populate the code-hash → bytes cache from `self.root`.""" + self._code_store = {spec_state.EMPTY_CODE_HASH: Bytes(b"")} + for account in self.root.values(): + if account is None: + continue + code = bytes(account.code) if account.code else b"" + if not code: + continue + self._code_store[spec_keccak256(code)] = Bytes(code) + + def _ensure_live(self) -> None: + """Transition from `CONSTRUCTION` to `LIVE`, building the cache.""" + if self._phase is _Phase.CONSTRUCTION: + self._build_cache() + self._phase = _Phase.LIVE + + def _materialize_state(self) -> spec_state.State: + """ + Build an in-memory `ethereum.state.State` mirror of `self.root`. + + Used as the trie-backed delegate for + `compute_state_root_and_trie_changes` (a cold, once-per-block call). + The materialized state is not retained. + """ + state = spec_state.State() + for address, account in self.root.items(): + if account is None: + continue + addr = Bytes20(address) + code = bytes(account.code) if account.code else b"" + code_hash = ( + spec_keccak256(code) if code else spec_state.EMPTY_CODE_HASH + ) + spec_state.set_account( + state, + addr, + spec_state.Account( + nonce=Uint(int(account.nonce)), + balance=U256(int(account.balance)), + code_hash=code_hash, + ), + ) + for key_hi, value_hi in account.storage.root.items(): + value_int = int(value_hi) + if value_int == 0: + continue + spec_state.set_storage( + state, + addr, + Bytes32(int(key_hi).to_bytes(32, "big")), + U256(value_int), + ) + state._code_store.update(self._code_store) + return state + + def get_account_optional( + self, address: Bytes20 + ) -> Optional[spec_state.Account]: + """ + Return the spec-side `Account` at `address`, or `None`. + + Conforms to `ethereum.state.PreState.get_account_optional`. + """ + self._ensure_live() + account = self.root.get(Address(address)) + if account is None: + return None + code = bytes(account.code) if account.code else b"" + code_hash = ( + spec_keccak256(code) if code else spec_state.EMPTY_CODE_HASH + ) + return spec_state.Account( + nonce=Uint(int(account.nonce)), + balance=U256(int(account.balance)), + code_hash=code_hash, + ) + + def get_storage(self, address: Bytes20, key: Bytes32) -> U256: + """ + Return the storage value at `key` for `address`, or `U256(0)`. + + Conforms to `ethereum.state.PreState.get_storage`. + """ + self._ensure_live() + account = self.root.get(Address(address)) + if account is None: + return U256(0) + key_int = int.from_bytes(bytes(key), "big") + value_hi = account.storage.root.get(HashInt(key_int)) + if value_hi is None: + return U256(0) + return U256(int(value_hi)) + + def get_code(self, code_hash: Hash32) -> Bytes: + """ + Return the bytecode for `code_hash`. + + Conforms to `ethereum.state.PreState.get_code`. + """ + self._ensure_live() + if code_hash == spec_state.EMPTY_CODE_HASH: + return Bytes(b"") + return self._code_store[code_hash] + + def account_has_storage(self, address: Bytes20) -> bool: + """ + Return whether the account at `address` has any storage slots set. + + Conforms to `ethereum.state.PreState.account_has_storage`. + """ + self._ensure_live() + account = self.root.get(Address(address)) + return account is not None and bool(account.storage.root) + + def compute_state_root_and_trie_changes( + self, + account_changes: Dict[Bytes20, Optional[spec_state.Account]], + storage_changes: Dict[Bytes20, Dict[Bytes32, U256]], + storage_clears: AbstractSet[Bytes20] = frozenset(), + ) -> Tuple[Hash32, List["InternalNode"]]: + """ + Compute the state root after applying `*_changes` to the pre-state. + + Conforms to + `ethereum.state.PreState.compute_state_root_and_trie_changes`. + Builds the trie inline; `Alloc` does not cache `Trie` instances. + """ + self._ensure_live() + state = self._materialize_state() + return state.compute_state_root_and_trie_changes( + account_changes, storage_changes, storage_clears + ) + + # ------------------------------------------------------------------ + # Lifecycle: apply_diff and freeze + # ------------------------------------------------------------------ + + def apply_diff(self, diff: spec_state.BlockDiff) -> None: + """ + Apply a `BlockDiff` to mutate the allocation in place. + + The only mutation entry point in the `LIVE` phase. Writes bypass + `__setitem__` intentionally — `_code_store` is updated additively + in lockstep with `self.root`. + """ + if self._phase is _Phase.FROZEN: + raise RuntimeError("apply_diff not allowed: Alloc is FROZEN") + if self._phase is _Phase.CONSTRUCTION: + raise RuntimeError( + "apply_diff not allowed in CONSTRUCTION: the allocation " + "has not been used as a PreState yet, so its cache is not " + "built. Trigger a PreState method (or hand it to a " + "BlockState) before calling apply_diff." + ) + + for code_hash, code in diff.code_changes.items(): + self._code_store[Hash32(code_hash)] = Bytes(code) + + for address in diff.storage_clears: + addr = Address(address) + current = self.root.get(addr) + if current is not None and current.storage.root: + self.root[addr] = current.model_copy( + update={"storage": Storage(root={})} + ) + + for address, spec_account in diff.account_changes.items(): + addr = Address(address) + if spec_account is None: + self.root.pop(addr, None) + continue + code_hash = Hash32(spec_account.code_hash) + if code_hash == spec_state.EMPTY_CODE_HASH: + code = Bytes(b"") + else: + code = self._code_store[code_hash] + existing = self.root.get(addr) + existing_storage = ( + existing.storage if existing is not None else Storage(root={}) + ) + self.root[addr] = Account( + nonce=int(spec_account.nonce), + balance=int(spec_account.balance), + code=code, + storage=existing_storage, + ) + + for address, slots in diff.storage_changes.items(): + addr = Address(address) + account = self.root.get(addr) + if account is None: + continue + merged: Dict[HashInt, HashInt] = dict(account.storage.root) + for key, value in slots.items(): + key_int = HashInt(int.from_bytes(bytes(key), "big")) + value_int = int(value) + if value_int == 0: + merged.pop(key_int, None) + else: + merged[key_int] = HashInt(value_int) + self.root[addr] = account.model_copy( + update={"storage": Storage(root=merged)} + ) + + # Drop zero-valued storage entries from every account. Ethereum + # treats an absent slot as zero, so a literal ``{0x00: 0x00}`` + # pair carried over untouched from the pre-state JSON would + # otherwise survive into the post-state dump and produce noise + # the spec-state-backed pipeline never had (the spec's + # ``set_storage`` drops zeros on insert). + for addr, account in list(self.root.items()): + if account is None or not account.storage.root: + continue + cleaned = { + key: value + for key, value in account.storage.root.items() + if int(value) != 0 + } + if len(cleaned) != len(account.storage.root): + self.root[addr] = account.model_copy( + update={"storage": Storage(root=cleaned)} + ) + + def freeze(self) -> None: + """Lock the allocation: no further mutations allowed.""" + self._phase = _Phase.FROZEN + def deterministic_deploy_contract( self, *, diff --git a/packages/testing/src/execution_testing/test_types/tests/test_alloc_prestate.py b/packages/testing/src/execution_testing/test_types/tests/test_alloc_prestate.py new file mode 100644 index 00000000000..30cb39af3d6 --- /dev/null +++ b/packages/testing/src/execution_testing/test_types/tests/test_alloc_prestate.py @@ -0,0 +1,277 @@ +""" +Unit tests for `Alloc` acting as a `PreState` provider. + +Covers four invariants of the lifecycle phase machinery: + 1. The code-hash → bytes cache is built correctly when the alloc goes + LIVE, and the PreState read methods agree with the source dict. + 2. `compute_state_root_and_trie_changes` on `Alloc` matches the same + call on a freshly built `ethereum.state.State` over the same data. + 3. Mutating an `Alloc` via `__setitem__`/`__delitem__` is rejected + after it has been used as a PreState. + 4. Building alloc B by `apply_diff`ing a diff onto alloc A produces a + post-state whose root matches an alloc independently constructed + to look like the post-state. +""" + +from typing import Dict, Optional + +import ethereum.state as spec_state +import pytest +from ethereum.crypto.hash import keccak256 +from ethereum_types.bytes import Bytes20, Bytes32 +from ethereum_types.numeric import U256, Uint + +from execution_testing.base_types import Account +from execution_testing.test_types import Alloc +from execution_testing.test_types.account_types import _Phase + + +def _b20(hex_str: str) -> Bytes20: + """Build a `Bytes20` address from a 40-char hex string (no `0x`).""" + return Bytes20(bytes.fromhex(hex_str)) + + +# A small, fixed set of addresses for ergonomic reuse in tests. They are +# `Bytes20` so they satisfy the `PreState` protocol's address parameter +# type, and pydantic re-validates them into `Address` when used as `Alloc` +# keys. +ADDR_A = _b20("000000000000000000000000000000000000aaaa") +ADDR_B = _b20("000000000000000000000000000000000000bbbb") +ADDR_C = _b20("000000000000000000000000000000000000cccc") +ADDR_MISSING = _b20("dead000000000000000000000000000000000000") +CODE = bytes.fromhex("60016002") # PUSH1 1 PUSH1 2 + + +def _fixture_alloc() -> Alloc: + """Build a small alloc with one EOA, one contract, and one empty acct.""" + return Alloc.model_validate( + { + ADDR_A: {"balance": 100, "nonce": 1}, + ADDR_B: { + "balance": 7, + "nonce": 3, + "code": "0x" + CODE.hex(), + "storage": {1: 0x42, 2: 0xCAFE}, + }, + ADDR_C: {"balance": 0, "nonce": 0}, + } + ) + + +def _state_from_alloc(alloc: Alloc) -> spec_state.State: + """Build a spec `State` mirroring `alloc` for parity comparisons.""" + state = spec_state.State() + for address, account in alloc.root.items(): + if account is None: + continue + addr = Bytes20(address) + code = bytes(account.code) if account.code else b"" + code_hash = keccak256(code) if code else spec_state.EMPTY_CODE_HASH + spec_state.set_account( + state, + addr, + spec_state.Account( + nonce=Uint(int(account.nonce)), + balance=U256(int(account.balance)), + code_hash=code_hash, + ), + ) + if code: + state._code_store[code_hash] = code + for key_hi, value_hi in account.storage.root.items(): + if int(value_hi) == 0: + continue + spec_state.set_storage( + state, + addr, + Bytes32(int(key_hi).to_bytes(32, "big")), + U256(int(value_hi)), + ) + return state + + +def test_cache_build_and_read_methods_agree_with_source() -> None: + """PreState reads on the alloc agree with the source dict.""" + alloc = _fixture_alloc() + assert alloc._phase is _Phase.CONSTRUCTION + + # The first PreState call must transition the alloc to LIVE. + acct_b = alloc.get_account_optional(ADDR_B) + assert alloc._phase is _Phase.LIVE + assert acct_b is not None + assert acct_b.nonce == Uint(3) + assert acct_b.balance == U256(7) + assert acct_b.code_hash == keccak256(CODE) + + # _code_store contains the empty hash and the only contract's code. + assert alloc._code_store[spec_state.EMPTY_CODE_HASH] == b"" + assert alloc._code_store[keccak256(CODE)] == CODE + # EOA + empty account contribute no code entries. + assert len(alloc._code_store) == 2 + + # Storage reads agree with the source for set and unset keys. + assert alloc.get_storage(ADDR_B, Bytes32(b"\x00" * 31 + b"\x01")) == U256( + 0x42 + ) + assert alloc.get_storage(ADDR_B, Bytes32(b"\x00" * 31 + b"\x02")) == U256( + 0xCAFE + ) + assert alloc.get_storage(ADDR_B, Bytes32(b"\x00" * 31 + b"\x03")) == U256( + 0 + ) + # Account with no storage returns zero for any key. + assert alloc.get_storage(ADDR_A, Bytes32(b"\x00" * 32)) == U256(0) + # Missing account returns zero. + assert alloc.get_storage(ADDR_MISSING, Bytes32(b"\x00" * 32)) == U256(0) + + # get_code round-trips, including the empty-code sentinel. + assert alloc.get_code(spec_state.EMPTY_CODE_HASH) == b"" + assert alloc.get_code(keccak256(CODE)) == CODE + + # account_has_storage distinguishes the contract from EOAs. + assert alloc.account_has_storage(ADDR_B) is True + assert alloc.account_has_storage(ADDR_A) is False + assert alloc.account_has_storage(ADDR_MISSING) is False + + # Missing accounts return None from get_account_optional. + assert alloc.get_account_optional(ADDR_MISSING) is None + + +def test_state_root_parity_against_spec_state() -> None: + """`Alloc.compute_state_root_and_trie_changes` matches spec `State`.""" + alloc = _fixture_alloc() + state = _state_from_alloc(alloc) + + alloc_root, _ = alloc.compute_state_root_and_trie_changes({}, {}) + spec_root, _ = state.compute_state_root_and_trie_changes({}, {}) + assert alloc_root == spec_root + + # Same parity under non-trivial change sets. + account_changes: Dict[Bytes20, Optional[spec_state.Account]] = { + ADDR_A: spec_state.Account( + nonce=Uint(2), balance=U256(200), code_hash=keccak256(CODE) + ), + } + storage_changes: Dict[Bytes20, Dict[Bytes32, U256]] = { + ADDR_B: {Bytes32(b"\x00" * 31 + b"\x01"): U256(0x99)}, + } + alloc_root_changed, _ = alloc.compute_state_root_and_trie_changes( + account_changes, storage_changes + ) + spec_root_changed, _ = state.compute_state_root_and_trie_changes( + account_changes, storage_changes + ) + assert alloc_root_changed == spec_root_changed + assert alloc_root_changed != alloc_root + + +def test_phase_guard_rejects_mutation_after_live() -> None: + """`__setitem__` and `__delitem__` raise once the alloc is LIVE.""" + alloc = _fixture_alloc() + # Still in CONSTRUCTION — mutations are allowed. + alloc[_b20("000000000000000000000000000000000000dddd")] = Account( + balance=1 + ) + + # Any PreState read transitions to LIVE. + _ = alloc.get_account_optional(ADDR_A) + assert alloc._phase is _Phase.LIVE + + with pytest.raises(RuntimeError, match="not allowed"): + alloc[_b20("000000000000000000000000000000000000eeee")] = Account( + balance=1 + ) + + with pytest.raises(RuntimeError, match="not allowed"): + del alloc[ADDR_A] + + # freeze() locks further mutation including apply_diff. + alloc.freeze() + with pytest.raises(RuntimeError, match="FROZEN"): + alloc.apply_diff( + spec_state.BlockDiff( + account_changes={}, storage_changes={}, code_changes={} + ) + ) + + +def test_apply_diff_round_trip_matches_independent_post_state() -> None: + """A.apply_diff(diff) reproduces an independently built post-state.""" + new_code = bytes.fromhex("6005600555") # arbitrary, distinct from CODE + new_code_hash = keccak256(new_code) + + alloc_pre = _fixture_alloc() + + # Independently build the expected post-state: + # - ADDR_A: nonce 1 → 2, balance 100 → 50 + # - ADDR_B: keeps account, storage slot 1 cleared, slot 3 added, + # slot 2 left alone + # - ADDR_C: deleted + # - new ADDR_NEW: brand-new contract with `new_code` and a slot set + addr_new = _b20("000000000000000000000000000000000000ffff") + alloc_post_expected = Alloc.model_validate( + { + ADDR_A: {"balance": 50, "nonce": 2}, + ADDR_B: { + "balance": 7, + "nonce": 3, + "code": "0x" + CODE.hex(), + "storage": {2: 0xCAFE, 3: 0x77}, + }, + addr_new: { + "balance": 1, + "nonce": 1, + "code": "0x" + new_code.hex(), + "storage": {0: 0x11}, + }, + } + ) + + # Build the diff that, applied to alloc_pre, should produce + # alloc_post_expected. + diff = spec_state.BlockDiff( + account_changes={ + ADDR_A: spec_state.Account( + nonce=Uint(2), + balance=U256(50), + code_hash=spec_state.EMPTY_CODE_HASH, + ), + ADDR_C: None, + addr_new: spec_state.Account( + nonce=Uint(1), + balance=U256(1), + code_hash=new_code_hash, + ), + }, + storage_changes={ + ADDR_B: { + Bytes32(b"\x00" * 31 + b"\x01"): U256(0), + Bytes32(b"\x00" * 31 + b"\x03"): U256(0x77), + }, + addr_new: {Bytes32(b"\x00" * 32): U256(0x11)}, + }, + code_changes={new_code_hash: new_code}, + ) + + # Force LIVE so apply_diff is allowed. + _ = alloc_pre.get_account_optional(ADDR_A) + alloc_pre.apply_diff(diff) + + # State roots should match. + pre_root, _ = alloc_pre.compute_state_root_and_trie_changes({}, {}) + expected_root, _ = alloc_post_expected.compute_state_root_and_trie_changes( + {}, {} + ) + assert pre_root == expected_root + + # Cache must be updated additively with the new code. + assert alloc_pre._code_store[new_code_hash] == new_code + # The contract's pre-existing code is still cached too. + assert alloc_pre._code_store[keccak256(CODE)] == CODE + + # apply_diff is still allowed (alloc stays LIVE) for the next block. + alloc_pre.apply_diff( + spec_state.BlockDiff( + account_changes={}, storage_changes={}, code_changes={} + ) + ) diff --git a/packages/testing/src/execution_testing/test_types/trie.py b/packages/testing/src/execution_testing/test_types/trie.py deleted file mode 100644 index 16fd4d5709d..00000000000 --- a/packages/testing/src/execution_testing/test_types/trie.py +++ /dev/null @@ -1,388 +0,0 @@ -""" -The state trie is the structure responsible for storing Ethereum state. -""" - -import copy -from dataclasses import dataclass, field -from typing import ( - Callable, - Dict, - Generic, - List, - Mapping, - MutableMapping, - Optional, - Sequence, - Tuple, - TypeVar, - cast, -) - -from ethereum.crypto.hash import keccak256 -from ethereum_rlp import Extended, rlp -from ethereum_types.bytes import Bytes, Bytes20, Bytes32 -from ethereum_types.frozen import slotted_freezable -from ethereum_types.numeric import U256, Uint -from typing_extensions import assert_type - - -@slotted_freezable -@dataclass -class FrontierAccount: - """State associated with an address.""" - - nonce: Uint - balance: U256 - code: Bytes - - -def encode_account( - raw_account_data: FrontierAccount, storage_root: Bytes -) -> Bytes: - """ - Encode `Account` dataclass. - - Storage is not stored in the `Account` dataclass, so `Accounts` cannot be - encoded without providing a storage root. - """ - return rlp.encode( - ( - raw_account_data.nonce, - raw_account_data.balance, - storage_root, - keccak256(raw_account_data.code), - ) - ) - - -# note: an empty trie (regardless of whether it is secured) has root: -# keccak256(RLP(b'')) == -# 56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421 -# also: -# keccak256(RLP(())) == -# 1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347 -# which is the sha3Uncles hash in block header with no uncles -EMPTY_TRIE_ROOT = Bytes32( - bytes.fromhex( - "56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421" - ) -) - -Node = FrontierAccount | Bytes | Uint | U256 | None -K = TypeVar("K", bound=Bytes) -V = TypeVar( - "V", - Optional[FrontierAccount], - Bytes, - Uint, - U256, -) - - -@slotted_freezable -@dataclass -class LeafNode: - """Leaf node in the Merkle Trie.""" - - rest_of_key: Bytes - value: Extended - - -@slotted_freezable -@dataclass -class ExtensionNode: - """Extension node in the Merkle Trie.""" - - key_segment: Bytes - subnode: Extended - - -BranchSubnodes = Tuple[ - Extended, - Extended, - Extended, - Extended, - Extended, - Extended, - Extended, - Extended, - Extended, - Extended, - Extended, - Extended, - Extended, - Extended, - Extended, - Extended, -] - - -@slotted_freezable -@dataclass -class BranchNode: - """Branch node in the Merkle Trie.""" - - subnodes: BranchSubnodes - value: Extended - - -InternalNode = LeafNode | ExtensionNode | BranchNode - - -def encode_internal_node(node: Optional[InternalNode]) -> Extended: - """ - Encode a Merkle Trie node into its RLP form. - - The RLP will then be serialized into a `Bytes` and hashed unless it is less - that 32 bytes when serialized. - - This function also accepts `None`, representing the absence of a node, - which is encoded to `b""`. - """ - unencoded: Extended - match node: - case None: - unencoded = b"" - case LeafNode(): - unencoded = ( - nibble_list_to_compact(node.rest_of_key, True), - node.value, - ) - case ExtensionNode(): - unencoded = ( - nibble_list_to_compact(node.key_segment, False), - node.subnode, - ) - case BranchNode(): - unencoded = list(node.subnodes) + [node.value] - case _: - raise AssertionError(f"Invalid internal node type {type(node)}!") - - encoded = rlp.encode(unencoded) - if len(encoded) < 32: - return unencoded - else: - return keccak256(encoded) - - -def encode_node(node: Node, storage_root: Optional[Bytes] = None) -> Bytes: - """ - Encode a Node for storage in the Merkle Trie. - - Currently mostly an unimplemented stub. - """ - match node: - case FrontierAccount(): - assert storage_root is not None - return encode_account(node, storage_root) - case U256(): - return rlp.encode(node) - case Bytes(): - return node - case _: - raise AssertionError( - f"encoding for {type(node)} is not currently implemented" - ) - - -@dataclass(slots=True) -class Trie(Generic[K, V]): - """The Merkle Trie.""" - - secured: bool - default: V - _data: Dict[K, V] = field(default_factory=dict) - - -def copy_trie(trie: Trie[K, V]) -> Trie[K, V]: - """ - Create a copy of `trie`. Since only frozen objects may be stored in tries, - the contents are reused. - """ - return Trie(trie.secured, trie.default, copy.copy(trie._data)) - - -def trie_set(trie: Trie[K, V], key: K, value: V) -> None: - """ - Store an item in a Merkle Trie. - - This method deletes the key if `value == trie.default`, because the Merkle - Trie represents the default value by omitting it from the trie. - """ - if value == trie.default: - if key in trie._data: - del trie._data[key] - else: - trie._data[key] = value - - -def trie_get(trie: Trie[K, V], key: K) -> V: - """ - Get an item from the Merkle Trie. - - This method returns `trie.default` if the key is missing. - """ - return trie._data.get(key, trie.default) - - -def common_prefix_length(a: Sequence, b: Sequence) -> int: - """Find the longest common prefix of two sequences.""" - for i in range(len(a)): - if i >= len(b) or a[i] != b[i]: - return i - return len(a) - - -def nibble_list_to_compact(x: Bytes, is_leaf: bool) -> Bytes: - """ - Compresses nibble-list into a standard byte array with a flag. - - A nibble-list is a list of byte values no greater than `15`. The flag is - encoded in high nibble of the highest byte. The flag nibble can be broken - down into two two-bit flags. - - Highest nibble:: - - +---+---+----------+--------+ - | _ | _ | is_leaf | parity | - +---+---+----------+--------+ - 3 2 1 0 - - The lowest bit of the nibble encodes the parity of the length of the - remaining nibbles -- `0` when even and `1` when odd. The second lowest bit - is used to distinguish leaf and extension nodes. The other two bits are not - used. - """ - compact = bytearray() - - if len(x) % 2 == 0: # ie even length - compact.append(16 * (2 * is_leaf)) - for i in range(0, len(x), 2): - compact.append(16 * x[i] + x[i + 1]) - else: - compact.append(16 * ((2 * is_leaf) + 1) + x[0]) - for i in range(1, len(x), 2): - compact.append(16 * x[i] + x[i + 1]) - - return Bytes(compact) - - -def bytes_to_nibble_list(bytes_: Bytes) -> Bytes: - """ - Convert a `Bytes` into to a sequence of nibbles (bytes with value < 16). - """ - nibble_list = bytearray(2 * len(bytes_)) - for byte_index, byte in enumerate(bytes_): - nibble_list[byte_index * 2] = (byte & 0xF0) >> 4 - nibble_list[byte_index * 2 + 1] = byte & 0x0F - return Bytes(nibble_list) - - -def _prepare_trie( - trie: Trie[K, V], - get_storage_root: Optional[Callable[[Bytes20], Bytes32]] = None, -) -> Mapping[Bytes, Bytes]: - """ - Prepare the trie for root calculation. Removes values that are empty, - hashes the keys (if `secured == True`) and encodes all the nodes. - """ - mapped: MutableMapping[Bytes, Bytes] = {} - - for preimage, value in trie._data.items(): - if isinstance(value, FrontierAccount): - assert get_storage_root is not None - address = Bytes20(preimage) - encoded_value = encode_node(value, get_storage_root(address)) - else: - encoded_value = encode_node(value) - if encoded_value == b"": - raise AssertionError - key: Bytes - if trie.secured: - # "secure" tries hash keys once before construction - key = keccak256(preimage) - else: - key = preimage - mapped[bytes_to_nibble_list(key)] = encoded_value - - return mapped - - -def root( - trie: Trie[K, V], - get_storage_root: Optional[Callable[[Bytes20], Bytes32]] = None, -) -> Bytes32: - """Compute the root of a modified merkle patricia trie (MPT).""" - obj = _prepare_trie(trie, get_storage_root) - - root_node = encode_internal_node(patricialize(obj, Uint(0))) - if len(rlp.encode(root_node)) < 32: - return keccak256(rlp.encode(root_node)) - else: - assert isinstance(root_node, Bytes) - return Bytes32(root_node) - - -def patricialize( - obj: Mapping[Bytes, Bytes], level: Uint -) -> Optional[InternalNode]: - """ - Structural composition function. - - Used to recursively patricialize and merkleize a dictionary. Includes - memoization of the tree structure and hashes. - """ - if len(obj) == 0: - return None - - arbitrary_key = next(iter(obj)) - - # if leaf node - if len(obj) == 1: - leaf = LeafNode(arbitrary_key[level:], obj[arbitrary_key]) - return leaf - - # prepare for extension node check by finding max j such that all keys in - # obj have the same key[i:j] - substring = arbitrary_key[level:] - prefix_length = len(substring) - for key in obj: - prefix_length = min( - prefix_length, common_prefix_length(substring, key[level:]) - ) - - # finished searching, found another key at the current level - if prefix_length == 0: - break - - # if extension node - if prefix_length > 0: - prefix = arbitrary_key[int(level) : int(level) + prefix_length] - return ExtensionNode( - prefix, - encode_internal_node( - patricialize(obj, level + Uint(prefix_length)) - ), - ) - - branches: List[MutableMapping[Bytes, Bytes]] = [] - for _ in range(16): - branches.append({}) - value = b"" - for key in obj: - if len(key) == level: - # shouldn't ever have an account or receipt in an internal node - if isinstance(obj[key], (FrontierAccount, Uint)): - raise AssertionError - value = obj[key] - else: - branches[key[level]][key] = obj[key] - - subnodes = tuple( - encode_internal_node(patricialize(branches[k], level + Uint(1))) - for k in range(16) - ) - return BranchNode( - cast(BranchSubnodes, assert_type(subnodes, Tuple[Extended, ...])), - value, - ) diff --git a/src/ethereum_spec_tools/evm_tools/__init__.py b/src/ethereum_spec_tools/evm_tools/__init__.py index 96be3137370..bf854c088cd 100644 --- a/src/ethereum_spec_tools/evm_tools/__init__.py +++ b/src/ethereum_spec_tools/evm_tools/__init__.py @@ -14,7 +14,8 @@ from .b11r import B11R, b11r_arguments from .daemon import Daemon, daemon_arguments from .statetest import StateTest, state_test_arguments -from .t8n import T8N, ForkCache, t8n_arguments +from .t8n import ForkCache +from .t8n.cli import run_t8n_cli, t8n_arguments from .utils import get_supported_forks DESCRIPTION = """ @@ -112,8 +113,7 @@ def main( exit_stack.push(fork_cache) if options.evm_tool == "t8n": - t8n_tool = T8N(options, out_file, in_file, fork_cache) - return t8n_tool.run() + return run_t8n_cli(options, out_file, in_file, fork_cache) elif options.evm_tool == "b11r": b11r_tool = B11R(options, out_file, in_file) return b11r_tool.run() diff --git a/src/ethereum_spec_tools/evm_tools/loaders/fork_loader.py b/src/ethereum_spec_tools/evm_tools/loaders/fork_loader.py index f9ec92d6ded..fda1bac008b 100644 --- a/src/ethereum_spec_tools/evm_tools/loaders/fork_loader.py +++ b/src/ethereum_spec_tools/evm_tools/loaders/fork_loader.py @@ -134,11 +134,6 @@ def signing_hash_155(self) -> Any: """signing_hash_155 function of the fork.""" return self._module("transactions").signing_hash_155 - @property - def has_signing_hash_155(self) -> bool: - """Check if the fork has a `signing_hash_155` function.""" - return hasattr(self._module("transactions"), "signing_hash_155") - @property def build_block_access_list(self) -> Any: """build_block_access_list function of the fork.""" @@ -259,14 +254,6 @@ def LegacyTransaction(self) -> Any: """Legacytransaction class of the fork.""" return self._module("transactions").LegacyTransaction - @property - def has_legacy_transaction(self) -> bool: - """ - Return `True` if the fork has a `LegacyTransaction` class, or `False` - otherwise. - """ - return hasattr(self._module("transactions"), "LegacyTransaction") - @property def Access(self) -> Any: """Access class of the fork.""" @@ -316,11 +303,6 @@ def decode_transaction(self) -> Any: """decode_transaction function of the fork.""" return self._module("transactions").decode_transaction - @property - def has_decode_transaction(self) -> bool: - """Check if this fork has a `decode_transaction`.""" - return hasattr(self._module("transactions"), "decode_transaction") - @property def BlockState(self) -> Any: """BlockState class of the fork.""" diff --git a/src/ethereum_spec_tools/evm_tools/statetest/__init__.py b/src/ethereum_spec_tools/evm_tools/statetest/__init__.py index 2f8246829c6..e9b74e62e65 100644 --- a/src/ethereum_spec_tools/evm_tools/statetest/__init__.py +++ b/src/ethereum_spec_tools/evm_tools/statetest/__init__.py @@ -9,14 +9,28 @@ from copy import deepcopy from dataclasses import dataclass from io import StringIO -from typing import Any, Dict, Generator, Iterable, List, Optional, TextIO +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Generator, + Iterable, + List, + Optional, + TextIO, +) from ethereum.utils.hexadecimal import hex_to_bytes -from ..t8n import T8N, ForkCache -from ..t8n.t8n_types import Result +from ..t8n import ForkCache +from ..t8n.cli import build_t8n_from_cli_options from ..utils import get_supported_forks +if TYPE_CHECKING: + from execution_testing.client_clis.cli_types import ( + Result as TestingResult, + ) + @dataclass class TestCase: @@ -87,7 +101,7 @@ def run_test_case( fork_cache: ForkCache, t8n_extra: Optional[List[str]] = None, output_basedir: Optional[str | TextIO] = None, -) -> Result: +) -> "TestingResult": """ Runs a single general state test. """ @@ -156,7 +170,8 @@ def run_test_case( if output_basedir is not None: t8n_options.output_basedir = output_basedir - t8n = T8N(t8n_options, out_stream, in_stream, fork_cache) + del out_stream # statetest reads ``t8n.result`` directly. + t8n = build_t8n_from_cli_options(t8n_options, in_stream, fork_cache) t8n.run_state_test() return t8n.result diff --git a/src/ethereum_spec_tools/evm_tools/t8n/__init__.py b/src/ethereum_spec_tools/evm_tools/t8n/__init__.py index c5fc7cde359..0d77a76ad9d 100644 --- a/src/ethereum_spec_tools/evm_tools/t8n/__init__.py +++ b/src/ethereum_spec_tools/evm_tools/t8n/__init__.py @@ -1,95 +1,60 @@ """ Create a transition tool for the given fork. + +The ``T8N`` class consumes testing-package pydantic types directly; the +JSON CLI surface lives in :mod:`.cli`. """ -import argparse -import fnmatch -import json -import os from contextlib import AbstractContextManager from dataclasses import astuple, dataclass -from typing import Any, Final, TextIO, Type, TypeVar +from typing import ( + TYPE_CHECKING, + Any, + Dict, + Final, + List, + Optional, + Sequence, + Type, + TypeVar, +) from ethereum_rlp import rlp +from ethereum_types.bytes import Bytes from ethereum_types.numeric import U64, U256, Uint from typing_extensions import override from ethereum import trace from ethereum.exceptions import EthereumException, InvalidBlock from ethereum.fork_criteria import ByBlockNumber, ByTimestamp, Unscheduled -from ethereum.merkle_patricia_trie import copy_trie from ethereum_spec_tools.forks import Hardfork, TemporaryHardfork from ..loaders.fixture_loader import Load -from ..utils import ( - FatalError, - find_fork, - get_stream_logger, - parse_hex_or_int, -) -from .env import Env -from .evm_trace.count import CountTracer -from .evm_trace.eip3155 import Eip3155Tracer +from ..loaders.transaction_loader import TransactionLoad, UnsupportedTxError +from ..utils import get_stream_logger, resolve_fork +from .block_environment import Ommer, build_block_environment from .evm_trace.group import GroupTracer -from .t8n_types import Alloc, Result, Txs - -T = TypeVar("T") -ForkCriteriaArgument = ByBlockNumber | ByTimestamp | Unscheduled | None - - -def t8n_arguments(subparsers: argparse._SubParsersAction) -> None: - """ - Adds the arguments for the t8n tool subparser. - """ - t8n_parser = subparsers.add_parser("t8n", help="This is the t8n tool.") +from .result import build_result, record_rejected_tx - t8n_parser.add_argument( - "--input.alloc", dest="input_alloc", type=str, default="alloc.json" - ) - t8n_parser.add_argument( - "--input.env", dest="input_env", type=str, default="env.json" - ) - t8n_parser.add_argument( - "--input.txs", dest="input_txs", type=str, default="txs.json" - ) - t8n_parser.add_argument( - "--input.blobParams", - dest="blob_parameters", - type=str, - default=None, - ) - t8n_parser.add_argument( - "--output.alloc", dest="output_alloc", type=str, default="alloc.json" +if TYPE_CHECKING: + from execution_testing.client_clis.cli_types import ( + TransitionToolOutput, ) - t8n_parser.add_argument( - "--output.basedir", dest="output_basedir", type=str, default="." + from execution_testing.client_clis.transition_tool import ( + TransitionTool, ) - t8n_parser.add_argument("--output.body", dest="output_body", type=str) - t8n_parser.add_argument( - "--output.result", - dest="output_result", - type=str, - default="result.json", + from execution_testing.exceptions import ExceptionMapper + from execution_testing.test_types import ( + Environment as TestingEnvironment, ) - t8n_parser.add_argument( - "--state.chainid", dest="state_chainid", type=int, default=1 + from execution_testing.test_types import ( + Transaction as TestingTransaction, ) - t8n_parser.add_argument( - "--state.fork", dest="state_fork", type=str, default="Frontier" - ) - t8n_parser.add_argument( - "--state.reward", dest="state_reward", type=int, default=None - ) - t8n_parser.add_argument("--trace", action="store_true") - t8n_parser.add_argument("--trace.memory", action="store_true") - t8n_parser.add_argument("--trace.nomemory", action="store_true") - t8n_parser.add_argument("--trace.noreturndata", action="store_true") - t8n_parser.add_argument("--trace.nostack", action="store_true") - t8n_parser.add_argument("--trace.returndata", action="store_true") - t8n_parser.add_argument("--opcode.count", dest="opcode_count", type=str) + TransitionToolData = TransitionTool.TransitionToolData - t8n_parser.add_argument("--state-test", action="store_true") +T = TypeVar("T") +ForkCriteriaArgument = ByBlockNumber | ByTimestamp | Unscheduled | None @dataclass(frozen=True) @@ -244,66 +209,80 @@ def get( class T8N(Load): - """The class that carries out the transition.""" + """ + Execute the transition function on already-parsed inputs. + + ``T8N`` is JSON-free: callers hand in a testing + ``TransitionTool.TransitionToolData`` (alloc / env / txs / + blob_schedule / fork / chain_id / reward / state_test) plus any + pre-PoS ommer data, and ``run()`` returns a + :class:`~execution_testing.client_clis.cli_types.TransitionToolOutput`. + See :mod:`.cli` for the JSON wrapper used by the + ``ethereum-spec-evm t8n`` entry point. + """ tracers: Final[GroupTracer | None] + alloc: Any + env: "TestingEnvironment" + txs: List["TestingTransaction"] + ommers: List[Ommer] + rejected_transactions: List[Any] + body: Bytes + state_test: bool + state_reward: int + exception_mapper: Optional["ExceptionMapper"] + _block_exception: Optional[str] def __init__( self, - options: Any, - out_file: TextIO, - in_file: TextIO, + t8n_data: "TransitionToolData", + *, cache: ForkCache, + fork_block: Optional[int] = None, + ommers: Sequence[Ommer] = (), + tracers: Optional[GroupTracer] = None, + exception_mapper: Optional["ExceptionMapper"] = None, ) -> None: - self.out_file = out_file - self.in_file = in_file - self.options = options - forks = Hardfork.discover() - - if "stdin" in ( - options.input_env, - options.input_alloc, - options.input_txs, - options.blob_parameters, + # ``resolve_fork`` only maps the testing fork name to a spec + # ``Hardfork`` module — CLI exception aliases like + # ``HomesteadToDaoAt5`` are unfolded by ``find_fork`` in + # :mod:`.cli` before the testing ``Fork`` is constructed. For + # those transition-fork tests the CLI also reports the block + # number at which the resolved fork activates via + # ``fork_block``; the in-process path leaves it ``None``. + fork_module = resolve_fork(t8n_data.fork_name) + fork_criteria: Optional[ByBlockNumber] = None + if fork_block is not None and fork_block != 0: + fork_criteria = ByBlockNumber(fork_block) + + # Translate ``t8n_data.blob_params`` (testing ``ForkBlobSchedule``) + # into the override arguments ``ForkCache.get`` consumes. + # + # Only forward overrides for BPO forks. BPO forks share their + # non-BPO ancestor's spec module and rely on the override to + # differentiate their blob schedule. Non-BPO forks (Cancun, + # Prague, Amsterdam, …) carry the correct schedule built into + # their spec module — overriding here would force ``ForkCache`` + # to clone the fork into a temporary directory whenever the + # override values don't byte-match the constants, attributing + # all opcode coverage to the clone's ``/tmp/...`` paths instead + # of the original ``src/ethereum/forks//`` source. + target_blobs_per_block: Optional[U64] = None + max_blobs_per_block: Optional[U64] = None + base_fee_update_fraction: Optional[Uint] = None + if ( + t8n_data.blob_params is not None + and t8n_data.fork.bpo_fork() + and t8n_data.fork != t8n_data.fork.non_bpo_ancestor() ): - stdin = json.load(in_file) - else: - stdin = None - - fork_module, self.fork_block = find_fork(forks, self.options, stdin) - - fork_criteria = None - if self.fork_block is not None and self.fork_block != 0: - # I can't find where `self.fork_block` is even used, and the vast - # majority of the time it's zero anyway. Not changing the fork - # criteria doesn't seem to break the tests, but changing it - # introduces cloning overhead, so... pretend it didn't happen. - fork_criteria = ByBlockNumber(self.fork_block) - - target_blobs_per_block = None - max_blobs_per_block = None - base_fee_update_fraction = None - - blob_parameters = None - if options.blob_parameters == "stdin": - assert stdin is not None - blob_parameters = stdin["blobParams"] - elif options.blob_parameters is not None: - with open(options.blob_parameters, "r") as f: - blob_parameters = json.load(f) - - if blob_parameters is not None: - target_blobs_per_block = parse_hex_or_int( - blob_parameters["target"], - U64, + target_blobs_per_block = U64( + int(t8n_data.blob_params.target_blobs_per_block) ) - max_blobs_per_block = parse_hex_or_int( - blob_parameters["max"], - U64, + max_blobs_per_block = U64( + int(t8n_data.blob_params.max_blobs_per_block) ) - base_fee_update_fraction = parse_hex_or_int( - blob_parameters["baseFeeUpdateFraction"], - Uint, + base_fee_update_fraction = Uint( + int(t8n_data.blob_params.base_fee_update_fraction) ) fork = cache.get( @@ -314,44 +293,30 @@ def __init__( blob_base_fee_update_fraction=base_fee_update_fraction, ) - tracers = GroupTracer() - - if self.options.trace: - trace_memory = getattr(self.options, "trace.memory", False) - trace_stack = not getattr(self.options, "trace.nostack", False) - trace_return_data = getattr(self.options, "trace.returndata") - tracers.add( - Eip3155Tracer( - trace_memory=trace_memory, - trace_stack=trace_stack, - trace_return_data=trace_return_data, - output_basedir=self.options.output_basedir, - ) - ) - - if self.options.opcode_count is not None: - tracers.add(CountTracer()) - - maybe_tracers: GroupTracer | None - if tracers.tracers: + if tracers is not None: trace.set_evm_trace(tracers) - maybe_tracers = tracers - else: - maybe_tracers = None - - self.tracers = maybe_tracers + self.tracers = tracers self.logger = get_stream_logger("T8N") - super().__init__(fork) - self.chain_id = parse_hex_or_int(self.options.state_chainid, U64) - self.alloc = Alloc(self, stdin) - self.env = Env(self, stdin) - self.txs = Txs(self, stdin) - self.result = Result( - self.env.block_difficulty, self.env.base_fee_per_gas - ) + self.chain_id = U64(int(t8n_data.chain_id)) + self.state_test = t8n_data.state_test + self.state_reward = int(t8n_data.reward) + self.exception_mapper = exception_mapper + + # Take a defensive copy of the input alloc so ``apply_diff`` + # (and any other in-place mutation T8N does) never escapes + # into the caller's Python object. Without this, multi-block + # tests that contain an invalid block would observe a mutated + # pre-state — the testing framework expects ``previous_alloc`` + # to remain unchanged when ``block.exception`` is set. + self.alloc = t8n_data.alloc.get().model_copy(deep=True) + self.env = t8n_data.env + self.txs = list(t8n_data.txs) + self.ommers = list(ommers) + self.body = Bytes(rlp.encode([tx.rlp() for tx in self.txs])) + self.rejected_transactions = [] def _tracer(self, type_: Type[T]) -> T: group = self.tracers @@ -364,70 +329,67 @@ def _tracer(self, type_: Type[T]) -> T: def block_environment(self) -> Any: """ - Create the environment for the transaction. The keyword - arguments are adjusted according to the fork. - """ - kw_arguments = { - "block_hashes": self.env.block_hashes, - "coinbase": self.env.coinbase, - "number": self.env.block_number, - "time": self.env.block_timestamp, - "block_gas_limit": self.env.block_gas_limit, - "chain_id": self.chain_id, - } - - block_state = self.fork.BlockState(pre_state=self.alloc.state) - kw_arguments["state"] = block_state - self._block_state = block_state - - block_environment = self.fork.BlockEnvironment - - if self.fork.has_calculate_base_fee_per_gas: - kw_arguments["base_fee_per_gas"] = self.env.base_fee_per_gas - - if self.fork.hardfork.consensus.is_pos(): - kw_arguments["prev_randao"] = self.env.prev_randao - else: - kw_arguments["difficulty"] = self.env.block_difficulty + Build the fork's ``BlockEnvironment`` for the current block. - if self.fork.has_beacon_roots_address: - kw_arguments["parent_beacon_block_root"] = ( - self.env.parent_beacon_block_root - ) - kw_arguments["excess_blob_gas"] = self.env.excess_blob_gas - - if self.fork.has_hash_block_access_list: - kw_arguments["block_access_list_builder"] = ( - self.fork.BlockAccessListBuilder() - ) - if self.fork.has_slot_number: - kw_arguments["slot_number"] = self.env.slot_number - - return block_environment(**kw_arguments) - - def backup_state(self) -> None: - """Back up the state in order to restore in case of an error.""" - state = self.alloc.state - main_trie = copy_trie(state._main_trie) - storage_tries = { - k: copy_trie(t) for (k, t) in state._storage_tries.items() - } - self.alloc.state_backup = ( - main_trie, - storage_tries, - dict(state._code_store), + Side effect: stores the resulting ``BlockState`` on ``self`` so + ``extract_block_diff`` can be called after execution. + """ + block_env = build_block_environment( + fork=self.fork, + env=self.env, + pre_state=self.alloc, + chain_id=self.chain_id, + state_test=self.state_test, ) + self._block_state = block_env.state + return block_env - def restore_state(self) -> None: - """Restore the state from the backup.""" - state = self.alloc.state - state._main_trie = self.alloc.state_backup[0] - state._storage_tries = self.alloc.state_backup[1] - state._code_store = self.alloc.state_backup[2] + def convert_transaction(self, tx: "TestingTransaction") -> Any: + """ + Convert a testing ``Transaction`` into the fork's tx object. + + TODO: Replace with ``self.fork.decode_transaction(tx.rlp())`` + once two pieces land in a follow-up PR: + + 1. Pre-Berlin forks gain a ``decode_transaction`` (or T8N + branches on ``has_decode_transaction`` and falls back to + ``rlp.decode_to(LegacyTransaction, ...)``). Pre-Berlin forks + predate typed txs and currently expose no decode entry + point — block decoding produces the legacy class directly. + 2. The testing exception_mapper learns to surface + ``DecodingError`` (raised when a contract-creating typed tx + like ``BlobTransaction`` (``to=None``) reaches + ``decode_transaction``) as the canonical + ``TransactionTypeContractCreationError``. Today + ``TransactionLoad`` constructs the tx object even when its + shape is illegal for the fork, so ``check_transaction`` + inside ``process_transaction`` raises the canonical error. + + Until both are in place, we go through ``TransactionLoad`` + (the JSON loader) which handles both concerns. + """ + raw: Dict[str, Any] = tx.model_dump( + mode="json", by_alias=True, exclude_none=True + ) + # Bridge testing-side aliases (geth-compatible) to the names + # ``TransactionLoad`` expects. + if "input" in raw: + raw.setdefault("data", raw["input"]) + if "gas" in raw: + raw.setdefault("gasLimit", raw["gas"]) + # ``to == None`` is dumped as JSON ``null``; ``TransactionLoad`` + # treats the empty string as the contract-creation sentinel. + if raw.get("to") in (None, "0x"): + raw["to"] = "" + # Ensure the ``type`` field is set so ``TransactionLoad`` + # dispatches to the right tx class (testing's dump uses ``ty`` + # which serializes to ``type`` only on some fork variants). + raw.setdefault("type", "0x" + format(int(tx.ty), "02x")) + return TransactionLoad(raw, self.fork).read() def pay_block_rewards(self, block_reward: U256, block_env: Any) -> None: """Apply the block rewards to the block coinbase.""" - ommer_count = U256(len(self.env.ommers)) + ommer_count = U256(len(self.ommers)) miner_reward = block_reward + ( ommer_count * (block_reward // U256(32)) ) @@ -436,42 +398,63 @@ def pay_block_rewards(self, block_reward: U256, block_env: Any) -> None: self.fork.create_ether(rewards_state, block_env.coinbase, miner_reward) - for ommer in self.env.ommers: - # Ommer age with respect to the current block. - ommer_age = U256(block_env.number - ommer.number) + for ommer in self.ommers: + # ``delta`` is the age of the ommer relative to the current block. + ommer_age = U256(int(ommer.delta, 16)) ommer_miner_reward = ( (U256(8) - ommer_age) * block_reward ) // U256(8) self.fork.create_ether( - rewards_state, ommer.coinbase, ommer_miner_reward + rewards_state, ommer.address, ommer_miner_reward ) self.fork.incorporate_tx_into_block(rewards_state) - def run_state_test(self) -> Any: + def _process_txs(self, block_env: Any, block_output: Any) -> None: + """Execute every transaction in ``self.txs`` against ``block_env``.""" + for tx_index, testing_tx in enumerate(self.txs): + try: + fork_tx = self.convert_transaction(testing_tx) + self.fork.process_transaction( + block_env, block_output, fork_tx, Uint(tx_index) + ) + except (EthereumException, UnsupportedTxError) as e: + # `UnsupportedTxError` covers ``convert_transaction`` + # failures when a typed tx is structurally malformed for + # this fork (e.g. a contract-creating BlobTransaction). + record_rejected_tx(self, tx_index, e) + self.logger.warning(f"Transaction {tx_index} failed: {e!r}") + + def run_state_test(self) -> None: """ Apply a single transaction on pre-state. No system operations are performed. """ - block_env = self.block_environment() - block_output = self.fork.BlockOutput() - self.backup_state() - if len(self.txs.transactions) > 0: - tx = self.txs.transactions[0] + self._block_env = self.block_environment() + self._block_output = self.fork.BlockOutput() + + if len(self.txs) > 0: + testing_tx = self.txs[0] try: + fork_tx = self.convert_transaction(testing_tx) self.fork.process_transaction( - block_env=block_env, - block_output=block_output, - tx=tx, + block_env=self._block_env, + block_output=self._block_output, + tx=fork_tx, index=Uint(0), ) - except EthereumException as e: - self.txs.rejected_txs[0] = f"Failed transaction: {e!r}" - self.restore_state() - self.logger.warning(f"Transaction {0} failed: {str(e)}") - - self.result.update(self, block_env, block_output) - self.result.rejected = self.txs.rejected_txs + except (EthereumException, UnsupportedTxError) as e: + record_rejected_tx(self, 0, e) + self.logger.warning(f"Transaction 0 failed: {e!r}") + + self._block_exception = None + self.result = build_result( + self, + self._block_env, + self._block_output, + self._block_exception, + self.rejected_transactions, + ) def _run_blockchain_test(self, block_env: Any, block_output: Any) -> None: if self.fork.has_compute_requests_hash: @@ -488,45 +471,35 @@ def _run_blockchain_test(self, block_env: Any, block_output: Any) -> None: data=block_env.parent_beacon_block_root, ) - for tx_index, (original_idx, tx) in enumerate( - zip( - self.txs.successfully_parsed, - self.txs.transactions, - strict=True, - ) - ): - self.backup_state() - try: - self.fork.process_transaction( - block_env, block_output, tx, Uint(tx_index) - ) - except EthereumException as e: - self.txs.rejected_txs[original_idx] = ( - f"Failed transaction: {e!r}" - ) - self.restore_state() - self.logger.warning( - f"Transaction {original_idx} failed: {e!r}" - ) + self._process_txs(block_env, block_output) # EIP-7928: Post-execution operations use index N+1 - num_txs = len(self.txs.transactions) if self.fork.has_hash_block_access_list: block_env.block_access_list_builder.block_access_index = ( - self.fork.BlockAccessIndex(Uint(num_txs) + Uint(1)) + self.fork.BlockAccessIndex(Uint(len(self.txs)) + Uint(1)) ) - if not self.fork.proof_of_stake: - if self.options.state_reward is None: - self.pay_block_rewards(self.fork.BLOCK_REWARD, block_env) - elif self.options.state_reward != -1: - self.pay_block_rewards( - U256(self.options.state_reward), block_env - ) + if not self.fork.proof_of_stake and self.state_reward != -1: + # ``-1`` is the sentinel for "skip block rewards entirely" + # (testing-side ``TransitionToolData.__post_init__`` sets + # this for genesis blocks; the CLI wrapper resolves a + # ``--state.reward=None`` to the fork's ``BLOCK_REWARD`` + # before constructing the data). + self.pay_block_rewards(U256(self.state_reward), block_env) if self.fork.has_withdrawal: + withdrawals = self.env.withdrawals or [] + fork_withdrawals = tuple( + self.fork.Withdrawal( + Uint(int(w.index)), + Uint(int(w.validator_index)), + self.fork.hex_to_address(w.address.hex()), + U256(int(w.amount)), + ) + for w in withdrawals + ) self.fork.process_withdrawals( - block_env, block_output, self.env.withdrawals + block_env, block_output, fork_withdrawals ) if self.fork.has_compute_requests_hash: @@ -547,102 +520,53 @@ def run_blockchain_test(self) -> None: """ Apply a block on the pre-state. Also includes system operations. """ - block_env = self.block_environment() - block_output = self.fork.BlockOutput() + self._block_env = self.block_environment() + self._block_output = self.fork.BlockOutput() + self._block_exception = None try: - self._run_blockchain_test(block_env, block_output) + self._run_blockchain_test(self._block_env, self._block_output) except InvalidBlock as e: - self.result.block_exception = f"{e}" - - self.result.update(self, block_env, block_output) - self.result.rejected = self.txs.rejected_txs - - def run(self) -> int: - """Run the transition and provide the relevant outputs.""" - # Clear files that may have been created in a previous - # run of the t8n tool. - # Define the specific files and pattern to delete - files_to_delete = [ - self.options.output_result, - self.options.output_alloc, - self.options.output_body, - ] - pattern_to_delete = "trace-*.jsonl" - - # Iterate through the directory - for file in os.listdir(self.options.output_basedir): - file_path = os.path.join(self.options.output_basedir, file) - - # Check if the file matches the specific names or the pattern - if file in files_to_delete or fnmatch.fnmatch( - file, pattern_to_delete - ): - os.remove(file_path) - - try: - if self.options.state_test: - self.run_state_test() - else: - self.run_blockchain_test() - except FatalError as e: - self.logger.error(str(e)) - return 1 - - json_state = self.alloc.to_json() - json_result = self.result.to_json() - - json_output: dict[str, object] = {} - - if self.options.output_body == "stdout": - txs_rlp = "0x" + rlp.encode(self.txs.all_txs).hex() - json_output["body"] = txs_rlp - elif self.options.output_body is not None: - txs_rlp_path = os.path.join( - self.options.output_basedir, - self.options.output_body, - ) - txs_rlp = "0x" + rlp.encode(self.txs.all_txs).hex() - with open(txs_rlp_path, "w") as f: - json.dump(txs_rlp, f) - self.logger.info(f"Wrote transaction rlp to {txs_rlp_path}") + self._block_exception = f"{e}" + + self.result = build_result( + self, + self._block_env, + self._block_output, + self._block_exception, + self.rejected_transactions, + ) - if self.options.output_alloc == "stdout": - json_output["alloc"] = json_state - else: - alloc_output_path = os.path.join( - self.options.output_basedir, - self.options.output_alloc, - ) - with open(alloc_output_path, "w") as f: - json.dump(json_state, f, indent=4) - self.logger.info(f"Wrote alloc to {alloc_output_path}") + def run(self) -> "TransitionToolOutput": + """ + Execute the transition; return the in-memory result. + + The returned ``TransitionToolOutput`` carries the post-state + ``Alloc`` directly (the field is typed ``LazyAlloc | Alloc``; + EELS hands over the already-materialized form), the ``Result`` + (state root, receipts, rejected txs, block exception, …), and + the encoded transaction body as raw RLP bytes. The JSON CLI + surface lives in :func:`.cli.write_t8n_outputs`. + """ + from execution_testing.base_types import Bytes as TestingBytes + from execution_testing.client_clis.cli_types import ( + TransitionToolOutput, + ) - if self.options.output_result == "stdout": - json_output["result"] = json_result + if self.state_test: + self.run_state_test() else: - result_output_path = os.path.join( - self.options.output_basedir, - self.options.output_result, - ) - with open(result_output_path, "w") as f: - json.dump(json_result, f, indent=4) - self.logger.info(f"Wrote result to {result_output_path}") - - if self.options.opcode_count == "stdout": - opcode_count_results = self._tracer(CountTracer).results() - json_output["opcodeCount"] = opcode_count_results - elif self.options.opcode_count is not None: - opcode_count_results = self._tracer(CountTracer).results() - result_output_path = os.path.join( - self.options.output_basedir, - self.options.opcode_count, - ) - with open(result_output_path, "w") as f: - json.dump(opcode_count_results, f, indent=4) - self.logger.info(f"Wrote opcode counts to {result_output_path}") - - if json_output: - json.dump(json_output, self.out_file, indent=4) - - return 0 + self.run_blockchain_test() + + # Apply the block diff in place so ``self.alloc`` is the + # post-state when the caller reads it. Safe to do + # unconditionally — ``self.alloc`` is a defensive copy taken + # in ``__init__``, so mutating it never escapes to the caller. + diff = self.fork.extract_block_diff(self._block_state) + self.alloc.apply_diff(diff) + + return TransitionToolOutput( + alloc=self.alloc, + result=self.result, + body=TestingBytes(self.body), + ) diff --git a/src/ethereum_spec_tools/evm_tools/t8n/block_environment.py b/src/ethereum_spec_tools/evm_tools/t8n/block_environment.py new file mode 100644 index 00000000000..aab52779c9d --- /dev/null +++ b/src/ethereum_spec_tools/evm_tools/t8n/block_environment.py @@ -0,0 +1,233 @@ +""" +Build the spec's per-fork ``BlockEnvironment`` from a testing-package +``Environment``. +""" + +from dataclasses import dataclass +from typing import TYPE_CHECKING, Any, List, Optional + +from ethereum_rlp import rlp +from ethereum_types.bytes import Bytes8, Bytes20, Bytes32, Bytes256 +from ethereum_types.numeric import U64, U256, Uint + +from ethereum.crypto.hash import Hash32, keccak256 + +if TYPE_CHECKING: + from execution_testing.test_types import Environment as TestingEnvironment + + from ..loaders.fork_loader import ForkLoad + + +@dataclass +class Ommer: + """ + Pre-PoS ommer header summary consumed by `pay_block_rewards`. + + Carries the two fields needed for ommer-reward arithmetic + (`block_number - delta` and the ommer coinbase). The testing + `Environment.ommers` field is `List[Hash]` and cannot represent + these — the JSON CLI fallback populates this from the raw env JSON + instead, and the in-process path leaves it empty (PoS has no ommers). + """ + + delta: str + address: Bytes20 + + +def build_block_environment( + fork: "ForkLoad", + env: "TestingEnvironment", + pre_state: Any, + chain_id: U64, + state_test: bool = False, +) -> Any: + """ + Build the fork's `BlockEnvironment` from a testing `Environment`. + + `pre_state` must satisfy the spec's `PreState` protocol (in + practice, a testing `Alloc`). + """ + block_state = fork.BlockState(pre_state=pre_state) + + block_number = Uint(int(env.number)) + block_gas_limit = Uint(int(env.gas_limit)) + block_timestamp = U256(int(env.timestamp)) + coinbase = Bytes20(env.fee_recipient) + + base_fee_per_gas = _resolve_base_fee_per_gas(env, fork, block_gas_limit) + + kw_arguments: dict[str, Any] = { + "block_hashes": _resolve_block_hashes(env.block_hashes, block_number), + "coinbase": coinbase, + "number": block_number, + "time": block_timestamp, + "block_gas_limit": block_gas_limit, + "chain_id": chain_id, + "state": block_state, + } + + if fork.has_calculate_base_fee_per_gas: + assert base_fee_per_gas is not None + kw_arguments["base_fee_per_gas"] = base_fee_per_gas + + if fork.hardfork.consensus.is_pos(): + kw_arguments["prev_randao"] = _resolve_prev_randao(env) + else: + kw_arguments["difficulty"] = _resolve_block_difficulty( + env, fork, block_number, block_timestamp + ) + + if fork.has_beacon_roots_address: + kw_arguments["parent_beacon_block_root"] = ( + None if state_test else _resolve_parent_beacon_block_root(env) + ) + kw_arguments["excess_blob_gas"] = _resolve_excess_blob_gas(env, fork) + + if fork.has_hash_block_access_list: + kw_arguments["block_access_list_builder"] = ( + fork.BlockAccessListBuilder() + ) + + if fork.has_slot_number: + slot_number = env.slot_number + kw_arguments["slot_number"] = ( + U64(int(slot_number)) if slot_number is not None else None + ) + + return fork.BlockEnvironment(**kw_arguments) + + +def _resolve_base_fee_per_gas( + env: "TestingEnvironment", fork: "ForkLoad", block_gas_limit: Uint +) -> Optional[Uint]: + """Use ``currentBaseFee`` if present; otherwise derive from parent.""" + if not fork.has_calculate_base_fee_per_gas: + return None + if env.base_fee_per_gas is not None: + return Uint(int(env.base_fee_per_gas)) + assert env.parent_gas_limit is not None + assert env.parent_gas_used is not None + assert env.parent_base_fee_per_gas is not None + return fork.calculate_base_fee_per_gas( + block_gas_limit, + Uint(int(env.parent_gas_limit)), + Uint(int(env.parent_gas_used)), + Uint(int(env.parent_base_fee_per_gas)), + ) + + +def _resolve_excess_blob_gas( + env: "TestingEnvironment", + fork: "ForkLoad", +) -> Optional[U64]: + """Use ``currentExcessBlobGas`` if present; else derive from parent.""" + if env.excess_blob_gas is not None: + return U64(int(env.excess_blob_gas)) + + parent_blob_gas_used = U64( + int(env.parent_blob_gas_used) if env.parent_blob_gas_used else 0 + ) + parent_excess_blob_gas = U64( + int(env.parent_excess_blob_gas) if env.parent_excess_blob_gas else 0 + ) + # EIP-7918 reads ``parent.base_fee_per_gas`` from the parent header. + parent_base_fee_per_gas = Uint( + int(env.parent_base_fee_per_gas) + if env.parent_base_fee_per_gas is not None + else 0 + ) + + arguments: dict[str, Any] = { + "parent_hash": Hash32(b"\0" * 32), + "ommers_hash": Hash32(b"\0" * 32), + "coinbase": Bytes20(b"\0" * 20), + "state_root": Hash32(b"\0" * 32), + "transactions_root": Hash32(b"\0" * 32), + "receipt_root": Hash32(b"\0" * 32), + "bloom": Bytes256(b"\0" * 256), + "difficulty": Uint(0), + "number": Uint(0), + "gas_limit": Uint(0), + "gas_used": Uint(0), + "timestamp": U256(0), + "extra_data": b"", + "prev_randao": Bytes32(b"\0" * 32), + "nonce": Bytes8(b"\0" * 8), + "withdrawals_root": Hash32(b"\0" * 32), + "parent_beacon_block_root": Hash32(b"\0" * 32), + "base_fee_per_gas": parent_base_fee_per_gas, + "blob_gas_used": parent_blob_gas_used, + "excess_blob_gas": parent_excess_blob_gas, + } + if fork.has_compute_requests_hash: + arguments["requests_hash"] = Hash32(b"\0" * 32) + if fork.has_hash_block_access_list: + arguments["block_access_list_hash"] = Hash32(b"\0" * 32) + if fork.has_slot_number: + arguments["slot_number"] = U64(0) + + parent_header = fork.Header(**arguments) + return fork.calculate_excess_blob_gas(parent_header) + + +def _resolve_block_difficulty( + env: "TestingEnvironment", + fork: "ForkLoad", + block_number: Uint, + block_timestamp: U256, +) -> Optional[Uint]: + """Use ``currentDifficulty`` if present; otherwise derive from parent.""" + if env.difficulty is not None: + return Uint(int(env.difficulty)) + + assert env.parent_timestamp is not None + assert env.parent_difficulty is not None + args: List[Any] = [ + block_number, + block_timestamp, + U256(int(env.parent_timestamp)), + Uint(int(env.parent_difficulty)), + ] + if fork.calculate_block_difficulty_arity > 4: + empty_ommers_hash = keccak256(rlp.encode([])) + parent_ommers_hash = Hash32(env.parent_ommers_hash) + args.append(parent_ommers_hash != empty_ommers_hash) + return fork.calculate_block_difficulty(*args) + + +def _resolve_prev_randao(env: "TestingEnvironment") -> Bytes32: + """Pad the (numeric) ``prev_randao`` field to 32 bytes.""" + value = env.prev_randao + if value is None: + return Bytes32(b"\0" * 32) + return Bytes32(int(value).to_bytes(32, "big")) + + +def _resolve_block_hashes( + block_hashes: Any, block_number: Uint +) -> List[Optional[Hash32]]: + """ + Return up to the last 256 block hashes preceding ``block_number``. + + `block_hashes` is the testing `Environment.block_hashes` dict keyed by + block number; missing entries become `None` placeholders. + """ + result: List[Optional[Hash32]] = [] + if not block_hashes: + return result + normalized = {int(k): Hash32(v) for k, v in block_hashes.items()} + max_blockhash_count = min(Uint(256), block_number) + for number in range( + int(block_number) - int(max_blockhash_count), int(block_number) + ): + result.append(normalized.get(number)) + return result + + +def _resolve_parent_beacon_block_root( + env: "TestingEnvironment", +) -> Optional[Hash32]: + """Return the parent beacon block root, or ``None`` if absent.""" + if env.parent_beacon_block_root is None: + return None + return Hash32(env.parent_beacon_block_root) diff --git a/src/ethereum_spec_tools/evm_tools/t8n/cli.py b/src/ethereum_spec_tools/evm_tools/t8n/cli.py new file mode 100644 index 00000000000..174946276ba --- /dev/null +++ b/src/ethereum_spec_tools/evm_tools/t8n/cli.py @@ -0,0 +1,467 @@ +""" +CLI / JSON wrapper for the ``T8N`` transition tool. + +``T8N`` itself consumes a testing-package +``TransitionTool.TransitionToolData`` and knows nothing about argparse, +stdin/stdout, or JSON. This module provides the bridge used by the +``ethereum-spec-evm t8n`` entry point and by ``statetest``: + +* :func:`build_t8n_from_cli_options` reads the JSON inputs + (stdin / files), resolves the fork, parses everything into testing + pydantic types, bundles them into a ``TransitionToolData``, builds + the tracer group, and returns a constructed ``T8N``. +* :func:`write_t8n_outputs` serialises the t8n output + opcode-count + results to disk / stdout per ``--output.*`` flags. +* :func:`run_t8n_cli` chains the two for the CLI entry point. +""" + +import argparse +import fnmatch +import json +import os +from typing import Any, Dict, List, Optional, TextIO, Tuple + +from ethereum_rlp import rlp +from ethereum_types.bytes import Bytes +from ethereum_types.numeric import U64 + +from ethereum_spec_tools.forks import Hardfork + +from ..loaders.fork_loader import ForkLoad +from ..utils import FatalError, find_fork, parse_hex_or_int +from . import T8N, ForkCache +from .block_environment import Ommer +from .evm_trace.count import CountTracer +from .evm_trace.eip3155 import Eip3155Tracer +from .evm_trace.group import GroupTracer + + +def t8n_arguments(subparsers: argparse._SubParsersAction) -> None: + """ + Adds the arguments for the t8n tool subparser. + """ + t8n_parser = subparsers.add_parser("t8n", help="This is the t8n tool.") + + t8n_parser.add_argument( + "--input.alloc", dest="input_alloc", type=str, default="alloc.json" + ) + t8n_parser.add_argument( + "--input.env", dest="input_env", type=str, default="env.json" + ) + t8n_parser.add_argument( + "--input.txs", dest="input_txs", type=str, default="txs.json" + ) + t8n_parser.add_argument( + "--input.blobParams", + dest="blob_parameters", + type=str, + default=None, + ) + t8n_parser.add_argument( + "--output.alloc", dest="output_alloc", type=str, default="alloc.json" + ) + t8n_parser.add_argument( + "--output.basedir", dest="output_basedir", type=str, default="." + ) + t8n_parser.add_argument("--output.body", dest="output_body", type=str) + t8n_parser.add_argument( + "--output.result", + dest="output_result", + type=str, + default="result.json", + ) + t8n_parser.add_argument( + "--state.chainid", dest="state_chainid", type=int, default=1 + ) + t8n_parser.add_argument( + "--state.fork", dest="state_fork", type=str, default="Frontier" + ) + t8n_parser.add_argument( + "--state.reward", dest="state_reward", type=int, default=None + ) + t8n_parser.add_argument("--trace", action="store_true") + t8n_parser.add_argument("--trace.memory", action="store_true") + t8n_parser.add_argument("--trace.nomemory", action="store_true") + t8n_parser.add_argument("--trace.noreturndata", action="store_true") + t8n_parser.add_argument("--trace.nostack", action="store_true") + t8n_parser.add_argument("--trace.returndata", action="store_true") + + t8n_parser.add_argument("--opcode.count", dest="opcode_count", type=str) + + t8n_parser.add_argument("--state-test", action="store_true") + + +def _read_json_input( + path_or_stdin: str, stdin: Optional[Dict], key: str +) -> Any: + """Read one of the t8n JSON inputs (alloc / env / txs).""" + if path_or_stdin == "stdin": + assert stdin is not None + return stdin[key] + with open(path_or_stdin, "r") as f: + return json.load(f) + + +def _parse_ommers_from_env_json(env_json: Any, fork: Any) -> List[Ommer]: + """Parse the pre-PoS ``ommers`` block from a raw env JSON dict.""" + ommers: List[Ommer] = [] + for raw in env_json.get("ommers", []): + ommers.append( + Ommer( + delta=raw["delta"], + address=fork.hex_to_address(raw["address"]), + ) + ) + return ommers + + +def _normalize_tx_json(tx: Dict[str, Any]) -> Dict[str, Any]: + """ + Drop fields that the testing ``Transaction`` model rejects. + + Two boundary mismatches to smooth over: + + 1. ``yParity`` on authorization tuples. The testing + ``AuthorizationTuple`` serializer emits both ``v`` and + ``yParity`` (they are guaranteed equal — see the model's + ``duplicate_v_as_y_parity``), but its validator binds only + ``v`` and treats ``yParity`` as an extra-forbidden field. + 2. ``secretKey`` on an already-signed tx. The testing + ``Transaction`` retains the private key after auto-signing in + ``model_post_init``, so the dump still carries ``secretKey`` + alongside the populated ``v``/``r``/``s``. On re-validation + the model rejects the pair with + ``InvalidSignaturePrivateKeyError``. Strip ``secretKey`` + whenever ``v`` is set (i.e. the tx is already signed). + """ + auth_list = tx.get("authorizationList") + if isinstance(auth_list, list): + tx["authorizationList"] = [ + {k: v for k, v in entry.items() if k != "yParity"} + if isinstance(entry, dict) + else entry + for entry in auth_list + ] + if "secretKey" in tx and tx.get("v") is not None: + tx = {k: v for k, v in tx.items() if k != "secretKey"} + return tx + + +def _parse_txs_json_to_testing( + raw_txs_json: Any, + fork_module: Hardfork, + transaction_cls: Any, +) -> Tuple[List[Any], Bytes]: + """ + Parse a JSON tx array into signed testing ``Transaction`` objects. + + Unsigned txs carrying only ``secretKey`` are signed in place via + ``Transaction.sign``; pre-Spurious-Dragon forks get + ``protected=False`` so the ``v`` value stays in ``{27, 28}``. + + RLP-string input (a single hex string of an encoded tx list) is + rejected — this path only handles JSON arrays. + """ + if raw_txs_json is None: + return [], Bytes(b"") + if isinstance(raw_txs_json, str): + raise NotImplementedError( + "RLP-encoded `txs` input is not supported by the testing " + "T8N entry point; provide a JSON array instead." + ) + + fork_supports_eip155 = hasattr( + fork_module.module("transactions"), "signing_hash_155" + ) + + normalized = [_normalize_tx_json(dict(tx)) for tx in raw_txs_json] + txs: List[Any] = [] + for tx_dict in normalized: + tx = transaction_cls.model_validate(tx_dict) + if "v" not in tx.model_fields_set and tx.secret_key is not None: + if not fork_supports_eip155 and int(tx.ty) == 0: + tx.protected = False + tx.sign() + txs.append(tx) + body = Bytes(rlp.encode([tx.rlp() for tx in txs])) + return txs, body + + +def _parse_blob_params_from_options( + options: Any, stdin: Optional[Dict] +) -> Any: + """ + Load a testing ``ForkBlobSchedule`` from ``--input.blobParams``. + + Returns ``None`` when the flag is unset. Reads from ``stdin`` + (``"blobParams"`` key) or a file path depending on the flag value. + """ + # Function-scoped: see import-cycle note in ``build_t8n_from_cli_options``. + from execution_testing.base_types.composite_types import ( + ForkBlobSchedule, + ) + + if options.blob_parameters == "stdin": + assert stdin is not None + raw = stdin["blobParams"] + elif options.blob_parameters is not None: + with open(options.blob_parameters, "r") as f: + raw = json.load(f) + else: + return None + return ForkBlobSchedule.model_validate(raw) + + +def _build_tracers_from_options( + options: Any, +) -> Optional[GroupTracer]: + """ + Build the tracer group from CLI ``--trace*`` / ``--opcode.count`` + flags. Returns ``None`` if no tracer would be active. + """ + tracers = GroupTracer() + if options.trace: + trace_memory = getattr(options, "trace.memory", False) + trace_stack = not getattr(options, "trace.nostack", False) + trace_return_data = getattr(options, "trace.returndata") + tracers.add( + Eip3155Tracer( + trace_memory=trace_memory, + trace_stack=trace_stack, + trace_return_data=trace_return_data, + output_basedir=options.output_basedir, + ) + ) + if options.opcode_count is not None: + tracers.add(CountTracer()) + return tracers if tracers.tracers else None + + +# Spec ``Hardfork.title_case_name`` matches the testing-side +# ``Fork.name()`` after stripping spaces, except for a handful of +# legacy outliers where the testing class uses a different +# capitalisation convention. +_TESTING_FORK_NAME_OVERRIDES = { + "DaoFork": "DAOFork", +} + + +def _testing_fork_from_spec_hardfork(hardfork: Hardfork) -> Any: + """Map a spec ``Hardfork`` to the matching testing ``Fork`` class.""" + # Function-scoped: see import-cycle note in ``build_t8n_from_cli_options``. + from execution_testing.forks import get_fork_by_name + + name = hardfork.title_case_name.replace(" ", "") + name = _TESTING_FORK_NAME_OVERRIDES.get(name, name) + fork = get_fork_by_name(name) + if fork is None: + raise ValueError( + f"No testing.Fork class for spec hardfork " + f"{hardfork.short_name!r} (looked for {name!r})" + ) + return fork + + +def _resolve_state_reward( + state_reward: Optional[int], fork_module: Hardfork +) -> int: + """ + Resolve a CLI ``--state.reward`` value into the int that + ``TransitionToolData.reward`` expects. + + ``None`` means "use the fork's default ``BLOCK_REWARD``"; an + explicit ``-1`` means "skip block rewards entirely" (the testing + sentinel); any other int passes through unchanged. + """ + if state_reward is None: + fork_load = ForkLoad(fork_module) + if fork_load.proof_of_stake: + return -1 + return int(fork_load.BLOCK_REWARD) + return state_reward + + +def build_t8n_from_cli_options( + options: Any, + in_file: TextIO, + cache: ForkCache, +) -> T8N: + """ + Construct a ``T8N`` from CLI options + JSON stdin / file inputs. + + Reads ``--input.*`` files (or stdin), validates each piece into + testing pydantic types, bundles them into a ``TransitionToolData``, + builds the tracer group, and hands them to ``T8N``. + """ + # Function-scoped imports: ``execution_testing/__init__`` eagerly + # imports ``.specs`` which transitively imports ``client_clis``, + # which imports ``ExecutionSpecsTransitionTool`` — top-level imports + # from ``execution_testing`` would cycle back into spec-tools. + from execution_testing.base_types.composite_types import BlobSchedule + from execution_testing.client_clis.transition_tool import TransitionTool + from execution_testing.test_types import ( + Alloc as TestingAlloc, + ) + from execution_testing.test_types import ( + Environment as TestingEnvironment, + ) + from execution_testing.test_types import ( + Transaction as TestingTransaction, + ) + + forks = Hardfork.discover() + + if "stdin" in ( + options.input_env, + options.input_alloc, + options.input_txs, + options.blob_parameters, + ): + stdin = json.load(in_file) + else: + stdin = None + + fork_module, fork_block = find_fork(forks, options, stdin) + testing_fork = _testing_fork_from_spec_hardfork(fork_module) + + raw_alloc_json = _read_json_input(options.input_alloc, stdin, "alloc") + raw_env_json = _read_json_input(options.input_env, stdin, "env") + raw_txs_json = _read_json_input(options.input_txs, stdin, "txs") + blob_params = _parse_blob_params_from_options(options, stdin) + + alloc = TestingAlloc.model_validate(raw_alloc_json) + env = TestingEnvironment.model_validate(raw_env_json) + txs, _body = _parse_txs_json_to_testing( + raw_txs_json, fork_module, TestingTransaction + ) + + # Wrap the single per-fork blob schedule into a ``BlobSchedule`` + # collection keyed by fork name (the field TransitionToolData + # expects). + blob_schedule: Any = None + if blob_params is not None: + blob_schedule = BlobSchedule() + blob_schedule.append(fork=testing_fork.name(), schedule=blob_params) + + t8n_data = TransitionTool.TransitionToolData( + alloc=alloc, + env=env, + txs=txs, + fork=testing_fork, + chain_id=int(parse_hex_or_int(options.state_chainid, U64)), + reward=_resolve_state_reward(options.state_reward, fork_module), + blob_schedule=blob_schedule, + state_test=options.state_test, + ) + + # ``Ommer.address`` is parsed via the per-fork ``hex_to_address`` + # helper; construct a temporary ``ForkLoad`` from the resolved + # module just to get the conversion. + fork_load = ForkLoad(fork_module) + ommers = _parse_ommers_from_env_json(raw_env_json, fork_load) + + return T8N( + t8n_data, + cache=cache, + fork_block=fork_block, + ommers=ommers, + tracers=_build_tracers_from_options(options), + ) + + +def write_t8n_outputs( + t8n: T8N, + output: Any, + options: Any, + out_file: TextIO, +) -> None: + """Serialise the t8n output + opcode counts per ``--output.*``.""" + json_state = output.alloc.get().model_dump(mode="json", by_alias=True) + json_result = output.result.model_dump( + mode="json", by_alias=True, exclude_none=True + ) + json_output: Dict[str, object] = {} + body_hex = "0x" + bytes(output.body or b"").hex() + + if options.output_body == "stdout": + json_output["body"] = body_hex + elif options.output_body is not None: + txs_rlp_path = os.path.join( + options.output_basedir, options.output_body + ) + with open(txs_rlp_path, "w") as f: + json.dump(body_hex, f) + t8n.logger.info(f"Wrote transaction rlp to {txs_rlp_path}") + + if options.output_alloc == "stdout": + json_output["alloc"] = json_state + else: + alloc_output_path = os.path.join( + options.output_basedir, options.output_alloc + ) + with open(alloc_output_path, "w") as f: + json.dump(json_state, f, indent=4) + t8n.logger.info(f"Wrote alloc to {alloc_output_path}") + + if options.output_result == "stdout": + json_output["result"] = json_result + else: + result_output_path = os.path.join( + options.output_basedir, options.output_result + ) + with open(result_output_path, "w") as f: + json.dump(json_result, f, indent=4) + t8n.logger.info(f"Wrote result to {result_output_path}") + + if options.opcode_count == "stdout": + json_output["opcodeCount"] = t8n._tracer(CountTracer).results() + elif options.opcode_count is not None: + result_output_path = os.path.join( + options.output_basedir, options.opcode_count + ) + with open(result_output_path, "w") as f: + json.dump(t8n._tracer(CountTracer).results(), f, indent=4) + t8n.logger.info(f"Wrote opcode counts to {result_output_path}") + + if json_output: + json.dump(json_output, out_file, indent=4) + + +def _clean_output_dir(options: Any) -> None: + """Remove prior output files matching ``--output.*`` from the basedir.""" + files_to_delete = [ + options.output_result, + options.output_alloc, + options.output_body, + ] + pattern_to_delete = "trace-*.jsonl" + for file in os.listdir(options.output_basedir): + file_path = os.path.join(options.output_basedir, file) + if file in files_to_delete or fnmatch.fnmatch(file, pattern_to_delete): + os.remove(file_path) + + +def run_t8n_cli( + options: Any, + out_file: TextIO, + in_file: TextIO, + cache: ForkCache, +) -> int: + """End-to-end CLI entry: read JSON, run ``T8N``, write JSON output.""" + _clean_output_dir(options) + t8n = build_t8n_from_cli_options(options, in_file, cache) + try: + output = t8n.run() + except FatalError as e: + t8n.logger.error(str(e)) + return 1 + write_t8n_outputs(t8n, output, options, out_file) + return 0 + + +__all__ = [ + "build_t8n_from_cli_options", + "run_t8n_cli", + "t8n_arguments", + "write_t8n_outputs", +] diff --git a/src/ethereum_spec_tools/evm_tools/t8n/env.py b/src/ethereum_spec_tools/evm_tools/t8n/env.py deleted file mode 100644 index edf3763d573..00000000000 --- a/src/ethereum_spec_tools/evm_tools/t8n/env.py +++ /dev/null @@ -1,333 +0,0 @@ -""" -Define t8n Env class. -""" - -import json -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Dict, List, Optional - -from ethereum_rlp import rlp -from ethereum_types.bytes import Bytes8, Bytes20, Bytes32, Bytes256 -from ethereum_types.numeric import U64, U256, Uint - -from ethereum.crypto.hash import Hash32, keccak256 -from ethereum.utils.byte import left_pad_zero_bytes -from ethereum.utils.hexadecimal import hex_to_bytes - -from ..utils import parse_hex_or_int - -if TYPE_CHECKING: - from ethereum_spec_tools.evm_tools.t8n import T8N - - -@dataclass -class Ommer: - """The Ommer type for the t8n tool.""" - - delta: str - address: Any - - -class Env: - """ - The environment for the transition tool. - """ - - coinbase: Any - block_gas_limit: Uint - block_number: Uint - block_timestamp: U256 - withdrawals: Any - block_difficulty: Optional[Uint] - prev_randao: Optional[Bytes32] - parent_difficulty: Optional[Uint] - parent_timestamp: Optional[U256] - base_fee_per_gas: Optional[Uint] - parent_gas_used: Optional[Uint] - parent_gas_limit: Optional[Uint] - parent_base_fee_per_gas: Optional[Uint] - block_hashes: Optional[List[Any]] - parent_ommers_hash: Optional[Hash32] - ommers: Any - parent_beacon_block_root: Optional[Hash32] - parent_excess_blob_gas: Optional[U64] - parent_blob_gas_used: Optional[U64] - excess_blob_gas: Optional[U64] - slot_number: Optional[U64] - requests: Any - - def __init__(self, t8n: "T8N", stdin: Optional[Dict] = None): - if t8n.options.input_env == "stdin": - assert stdin is not None - data = stdin["env"] - else: - with open(t8n.options.input_env, "r") as f: - data = json.load(f) - - self.coinbase = t8n.fork.hex_to_address(data["currentCoinbase"]) - self.block_gas_limit = parse_hex_or_int(data["currentGasLimit"], Uint) - self.block_number = parse_hex_or_int(data["currentNumber"], Uint) - self.block_timestamp = parse_hex_or_int(data["currentTimestamp"], U256) - - self.read_block_difficulty(data, t8n) - self.read_base_fee_per_gas(data, t8n) - self.read_randao(data, t8n) - self.read_block_hashes(data) - self.read_ommers(data, t8n) - self.read_withdrawals(data, t8n) - - self.parent_beacon_block_root = None - if t8n.fork.has_beacon_roots_address: - if not t8n.options.state_test: - parent_beacon_block_root_hex = data["parentBeaconBlockRoot"] - self.parent_beacon_block_root = ( - Bytes32(hex_to_bytes(parent_beacon_block_root_hex)) - if parent_beacon_block_root_hex is not None - else None - ) - self.read_excess_blob_gas(data, t8n) - - self.read_slot_number(data, t8n) - - def read_excess_blob_gas(self, data: Any, t8n: "T8N") -> None: - """ - Read the excess_blob_gas from the data. If the excess blob gas is - not present, it is calculated from the parent block parameters. - """ - self.parent_blob_gas_used = U64(0) - self.parent_excess_blob_gas = U64(0) - self.excess_blob_gas = None - - if not t8n.fork.has_beacon_roots_address: - return - - if "parentExcessBlobGas" in data: - self.parent_excess_blob_gas = parse_hex_or_int( - data["parentExcessBlobGas"], U64 - ) - - if "parentBlobGasUsed" in data: - self.parent_blob_gas_used = parse_hex_or_int( - data["parentBlobGasUsed"], U64 - ) - - if "currentExcessBlobGas" in data: - self.excess_blob_gas = parse_hex_or_int( - data["currentExcessBlobGas"], U64 - ) - return - - assert self.parent_excess_blob_gas is not None - assert self.parent_blob_gas_used is not None - - arguments = { - # Useless as far as calculate_excess_blob_gas is concerned. - "parent_hash": Hash32(b"\0" * 32), - "ommers_hash": Hash32(b"\0" * 32), - "coinbase": Bytes20(b"\0" * 20), - "state_root": Hash32(b"\0" * 32), - "transactions_root": Hash32(b"\0" * 32), - "receipt_root": Hash32(b"\0" * 32), - "bloom": Bytes256(b"\0" * 256), - "difficulty": Uint(0), - "number": Uint(0), - "gas_limit": Uint(0), - "gas_used": Uint(0), - "timestamp": U256(0), - "extra_data": b"", - "prev_randao": Bytes32(b"\0" * 32), - "nonce": Bytes8(b"\0" * 8), - "withdrawals_root": Hash32(b"\0" * 32), - "parent_beacon_block_root": Hash32(b"\0" * 32), - # Used for calculating excess_blob_gas. - "base_fee_per_gas": self.parent_base_fee_per_gas, - "blob_gas_used": self.parent_blob_gas_used, - "excess_blob_gas": self.parent_excess_blob_gas, - } - - if t8n.fork.has_compute_requests_hash: - arguments["requests_hash"] = Hash32(b"\0" * 32) - - if t8n.fork.has_hash_block_access_list: - arguments["block_access_list_hash"] = Hash32(b"\0" * 32) - if t8n.fork.has_slot_number: - arguments["slot_number"] = U64(0) - - parent_header = t8n.fork.Header(**arguments) - - self.excess_blob_gas = t8n.fork.calculate_excess_blob_gas( - parent_header - ) - - def read_base_fee_per_gas(self, data: Any, t8n: "T8N") -> None: - """ - Read the base_fee_per_gas from the data. If the base fee is - not present, it is calculated from the parent block parameters. - """ - self.parent_gas_used = None - self.parent_gas_limit = None - self.parent_base_fee_per_gas = None - self.base_fee_per_gas = None - - if t8n.fork.has_calculate_base_fee_per_gas: - if "currentBaseFee" in data: - self.base_fee_per_gas = parse_hex_or_int( - data["currentBaseFee"], Uint - ) - - if "parentGasUsed" in data: - self.parent_gas_used = parse_hex_or_int( - data["parentGasUsed"], Uint - ) - - if "parentGasLimit" in data: - self.parent_gas_limit = parse_hex_or_int( - data["parentGasLimit"], Uint - ) - - if "parentBaseFee" in data: - self.parent_base_fee_per_gas = parse_hex_or_int( - data["parentBaseFee"], Uint - ) - - if self.base_fee_per_gas is None: - assert self.parent_gas_limit is not None - assert self.parent_gas_used is not None - assert self.parent_base_fee_per_gas is not None - - parameters: List[object] = [ - self.block_gas_limit, - self.parent_gas_limit, - self.parent_gas_used, - self.parent_base_fee_per_gas, - ] - - self.base_fee_per_gas = t8n.fork.calculate_base_fee_per_gas( - *parameters - ) - - def read_randao(self, data: Any, t8n: "T8N") -> None: - """ - Read the randao from the data. - """ - self.prev_randao = None - if t8n.fork.proof_of_stake: - # tf tool might not always provide an - # even number of nibbles in the randao - # This could create issues in the - # hex_to_bytes function - current_random = data["currentRandom"] - if current_random.startswith("0x"): - current_random = current_random[2:] - - if len(current_random) % 2 == 1: - current_random = "0" + current_random - - self.prev_randao = Bytes32( - left_pad_zero_bytes(hex_to_bytes(current_random), 32) - ) - - def read_slot_number(self, data: Any, t8n: "T8N") -> None: - """ - Read the slot number from the data. - The slot number is provided by the consensus layer. - """ - self.slot_number = None - if t8n.fork.has_slot_number: - if "slotNumber" in data: - self.slot_number = parse_hex_or_int(data["slotNumber"], U64) - - def read_withdrawals(self, data: Any, t8n: "T8N") -> None: - """ - Read the withdrawals from the data. - """ - self.withdrawals = None - if t8n.fork.has_withdrawal: - self.withdrawals = tuple( - t8n.json_to_withdrawals(wd) for wd in data["withdrawals"] - ) - - def read_block_difficulty(self, data: Any, t8n: "T8N") -> None: - """ - Read the block difficulty from the data. - If `currentDifficulty` is present, it is used. Otherwise, - the difficulty is calculated from the parent block. - """ - self.block_difficulty = None - self.parent_timestamp = None - self.parent_difficulty = None - self.parent_ommers_hash = None - if t8n.fork.proof_of_stake: - return - elif "currentDifficulty" in data: - self.block_difficulty = parse_hex_or_int( - data["currentDifficulty"], Uint - ) - else: - self.parent_timestamp = parse_hex_or_int( - data["parentTimestamp"], U256 - ) - self.parent_difficulty = parse_hex_or_int( - data["parentDifficulty"], Uint - ) - args: List[object] = [ - self.block_number, - self.block_timestamp, - self.parent_timestamp, - self.parent_difficulty, - ] - if t8n.fork.calculate_block_difficulty_arity > 4: - if "parentUncleHash" in data: - EMPTY_OMMER_HASH = keccak256(rlp.encode([])) # noqa N806 - self.parent_ommers_hash = Hash32( - hex_to_bytes(data["parentUncleHash"]) - ) - parent_has_ommers = ( - self.parent_ommers_hash != EMPTY_OMMER_HASH - ) - args.append(parent_has_ommers) - else: - args.append(False) - self.block_difficulty = t8n.fork.calculate_block_difficulty(*args) - - def read_block_hashes(self, data: Any) -> None: - """ - Read the block hashes. Returns a maximum of 256 block hashes. - """ - # Read the block hashes - block_hashes: List[Any] = [] - - # The hex key strings provided might not have standard formatting - clean_block_hashes: Dict[int, Hash32] = {} - if "blockHashes" in data: - for key, value in data["blockHashes"].items(): - int_key = int(key, 16) - clean_block_hashes[int_key] = Hash32(hex_to_bytes(value)) - - # Store a maximum of 256 block hashes. - max_blockhash_count = min(Uint(256), self.block_number) - for number in range( - self.block_number - max_blockhash_count, self.block_number - ): - if number in clean_block_hashes.keys(): - block_hashes.append(clean_block_hashes[number]) - else: - block_hashes.append(None) - - self.block_hashes = block_hashes - - def read_ommers(self, data: Any, t8n: "T8N") -> None: - """ - Read the ommers. The ommers data might not have all the details - needed to obtain the Header. - """ - ommers = [] - if "ommers" in data: - for ommer in data["ommers"]: - ommers.append( - Ommer( - ommer["delta"], - t8n.fork.hex_to_address(ommer["address"]), - ) - ) - self.ommers = ommers diff --git a/src/ethereum_spec_tools/evm_tools/t8n/result.py b/src/ethereum_spec_tools/evm_tools/t8n/result.py new file mode 100644 index 00000000000..b8b646e84a8 --- /dev/null +++ b/src/ethereum_spec_tools/evm_tools/t8n/result.py @@ -0,0 +1,146 @@ +""" +Build the testing-side ``Result`` from an executed block. + +All construction of ``Result`` and ``TransactionReceipt`` lives here +so the testing-package pydantic types stay isolated to one boundary +module. +""" + +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +from ethereum_rlp import rlp + +from ethereum.crypto.hash import keccak256 +from ethereum.merkle_patricia_trie import root, trie_get + +if TYPE_CHECKING: + from execution_testing.client_clis.cli_types import ( + Result as TestingResult, + ) + + from . import T8N + + +def get_receipts_from_output(t8n: "T8N", block_output: Any) -> List[Any]: + """Build testing-side `TransactionReceipt`s from the block output tries.""" + # Function-scoped: ``execution_testing/__init__`` eagerly imports + # ``.specs`` which transitively imports ``client_clis``, which + # imports ``ExecutionSpecsTransitionTool`` — top-level import would + # cycle back into ``t8n``. + from execution_testing.test_types.receipt_types import ( + TransactionLog, + TransactionReceipt, + ) + + receipts: List[Any] = [] + for key in block_output.receipt_keys: + tx = trie_get(block_output.transactions_trie, key) + receipt = trie_get(block_output.receipts_trie, key) + assert tx is not None + assert receipt is not None + + tx_hash = t8n.fork.get_transaction_hash(tx) + + if hasattr(t8n.fork, "decode_receipt"): + decoded_receipt = t8n.fork.decode_receipt(receipt) + else: + decoded_receipt = receipt + + receipt_kwargs: Dict[str, Any] = { + "transaction_hash": tx_hash, + "cumulative_gas_used": int(decoded_receipt.cumulative_gas_used), + "bloom": decoded_receipt.bloom, + "logs": [ + TransactionLog( + address=log.address, + topics=list(log.topics), + data=log.data, + ) + for log in decoded_receipt.logs + ], + } + if hasattr(decoded_receipt, "succeeded"): + receipt_kwargs["status"] = int(decoded_receipt.succeeded) + elif hasattr(decoded_receipt, "post_state"): + receipt_kwargs["post_state"] = decoded_receipt.post_state + receipts.append(TransactionReceipt(**receipt_kwargs)) + return receipts + + +def build_result( + t8n: "T8N", + block_env: Any, + block_output: Any, + block_exception: Optional[str], + rejected_transactions: List[Any], +) -> "TestingResult": + """Build the testing-side `Result` from the executed block.""" + # Function-scoped: see import-cycle note in ``get_receipts_from_output``. + from execution_testing.client_clis.cli_types import Result as TestingResult + + diff = t8n.fork.extract_block_diff(t8n._block_state) + state_root, _ = t8n.alloc.compute_state_root_and_trie_changes( + diff.account_changes, diff.storage_changes, diff.storage_clears + ) + + arguments: Dict[str, Any] = { + "state_root": state_root, + "transactions_trie": root(block_output.transactions_trie), + "receipts_root": root(block_output.receipts_trie), + "logs_hash": keccak256(rlp.encode(block_output.block_logs)), + "logs_bloom": t8n.fork.logs_bloom(block_output.block_logs), + "receipts": get_receipts_from_output(t8n, block_output), + "rejected_transactions": rejected_transactions, + "gas_used": int(block_output.block_gas_used), + } + if block_exception is not None: + arguments["block_exception"] = block_exception + if hasattr(block_env, "difficulty"): + arguments["difficulty"] = int(block_env.difficulty) + if hasattr(block_env, "base_fee_per_gas"): + arguments["base_fee_per_gas"] = int(block_env.base_fee_per_gas) + if hasattr(block_output, "withdrawals_trie"): + arguments["withdrawals_root"] = root(block_output.withdrawals_trie) + if hasattr(block_env, "excess_blob_gas"): + arguments["excess_blob_gas"] = int(block_env.excess_blob_gas) + arguments["blob_gas_used"] = int(block_output.blob_gas_used) + if hasattr(block_output, "requests"): + arguments["requests"] = list(block_output.requests) + arguments["requests_hash"] = t8n.fork.compute_requests_hash( + block_output.requests + ) + if hasattr(block_output, "block_access_list"): + arguments["block_access_list"] = rlp.encode( + block_output.block_access_list + ) + arguments["block_access_list_hash"] = t8n.fork.hash_block_access_list( + block_output.block_access_list + ) + + context: Optional[Dict[str, Any]] = None + if t8n.exception_mapper is not None: + context = {"exception_mapper": t8n.exception_mapper} + return TestingResult.model_validate(arguments, context=context) + + +def record_rejected_tx(t8n: "T8N", index: int, error: Exception) -> None: + """Append a ``RejectedTransaction`` to ``t8n.rejected_transactions``.""" + # Function-scoped: see import-cycle note in ``get_receipts_from_output``. + from execution_testing.client_clis.cli_types import RejectedTransaction + + context: Optional[Dict[str, Any]] = None + if t8n.exception_mapper is not None: + context = {"exception_mapper": t8n.exception_mapper} + t8n.rejected_transactions.append( + RejectedTransaction.model_validate( + {"index": index, "error": f"Failed transaction: {error!r}"}, + context=context, + ) + ) + + +__all__ = [ + "build_result", + "get_receipts_from_output", + "record_rejected_tx", +] diff --git a/src/ethereum_spec_tools/evm_tools/t8n/t8n_types.py b/src/ethereum_spec_tools/evm_tools/t8n/t8n_types.py deleted file mode 100644 index fb3139c7b5c..00000000000 --- a/src/ethereum_spec_tools/evm_tools/t8n/t8n_types.py +++ /dev/null @@ -1,440 +0,0 @@ -""" -Define the types used by the t8n tool. -""" - -import json -from dataclasses import dataclass -from typing import TYPE_CHECKING, Any, Dict, List, Optional - -from ethereum_rlp import Simple, rlp -from ethereum_types.bytes import Bytes -from ethereum_types.numeric import U64, U256, Uint - -from ethereum.crypto.hash import Hash32, keccak256 -from ethereum.merkle_patricia_trie import root, trie_get -from ethereum.state import EMPTY_CODE_HASH, apply_changes_to_state -from ethereum.utils.hexadecimal import hex_to_bytes, hex_to_u256, hex_to_uint - -from ..loaders.transaction_loader import TransactionLoad, UnsupportedTxError -from ..utils import FatalError, encode_to_hex, secp256k1_sign - -if TYPE_CHECKING: - from . import T8N - - -class Alloc: - """ - The alloc (state) type for the t8n tool. - """ - - state: Any - state_backup: Any - - def __init__(self, t8n: "T8N", stdin: Optional[Dict] = None): - """Read the alloc file and return the state.""" - if t8n.options.input_alloc == "stdin": - assert stdin is not None - data = stdin["alloc"] - else: - with open(t8n.options.input_alloc, "r") as f: - data = json.load(f) - - # The json_to_state function expects the values to be hex - # strings, so we convert them here. - for address, account in data.items(): - for key, value in account.items(): - if key == "storage" or not value: - continue - elif not value.startswith("0x"): - data[address][key] = "0x" + hex(int(value)) - - state = t8n.json_to_state(data) - if t8n.fork.hardfork.short_name == "dao_fork": - t8n.fork.apply_dao(state) - - self.state = state - - def to_json(self) -> Any: - """Encode the state to JSON.""" - data = {} - for address, account in self.state._main_trie._data.items(): - account_data: Dict[str, Any] = {} - - if account.balance: - account_data["balance"] = hex(account.balance) - - if account.nonce: - account_data["nonce"] = hex(account.nonce) - - if account.code_hash != EMPTY_CODE_HASH: - code = self.state._code_store[account.code_hash] - account_data["code"] = "0x" + code.hex() - - if address in self.state._storage_tries: - account_data["storage"] = { - "0x" + k.hex(): hex(v) - for k, v in self.state._storage_tries[ - address - ]._data.items() - } - - data["0x" + address.hex()] = account_data - - return data - - -class Txs: - """ - Read the transactions file, sort out the valid transactions and - return a list of transactions. - """ - - def __init__(self, t8n: "T8N", stdin: Optional[Dict] = None): - self.t8n = t8n - self.successfully_parsed: List[int] = [] - self.transactions: List[Any] = [] - self.rejected_txs = {} - self.rlp_input = False - self.all_txs = [] - - if t8n.options.input_txs == "stdin": - assert stdin is not None - data = stdin["txs"] - else: - with open(t8n.options.input_txs, "r") as f: - data = json.load(f) - - if data is None: - self.data: Simple = [] - elif isinstance(data, str): - self.rlp_input = True - self.data = rlp.decode(hex_to_bytes(data)) - else: - self.data = data - - for idx, raw_tx in enumerate(self.data): - try: - if self.rlp_input: - self.transactions.append(self.parse_rlp_tx(raw_tx)) - self.successfully_parsed.append(idx) - else: - self.transactions.append(self.parse_json_tx(raw_tx)) - self.successfully_parsed.append(idx) - except UnsupportedTxError as e: - self.t8n.logger.warning( - f"Unsupported transaction at index {idx}: " - f"{e.error_message}" - ) - self.rejected_txs[idx] = ( - f"Unsupported transaction type: {e.error_message}" - ) - if e.encoded_params is not None: - self.all_txs.append(e.encoded_params) - except Exception as e: - msg = f"Failed to parse transaction {idx}: {str(e)}" - self.t8n.logger.warning(msg, exc_info=e) - self.rejected_txs[idx] = msg - - def parse_rlp_tx(self, raw_tx: Any) -> Any: - """ - Read transactions from RLP. - """ - t8n = self.t8n - - tx_rlp = rlp.encode(raw_tx) - if t8n.fork.has_legacy_transaction: - if isinstance(raw_tx, Bytes): - transaction = t8n.fork.decode_transaction(raw_tx) - self.all_txs.append(raw_tx) - else: - transaction = rlp.decode_to(t8n.fork.LegacyTransaction, tx_rlp) - self.all_txs.append(transaction) - else: - transaction = rlp.decode_to(t8n.fork.Transaction, tx_rlp) - self.all_txs.append(transaction) - - return transaction - - def parse_json_tx(self, raw_tx: Any) -> Any: - """ - Read the transactions from json. - If a transaction is unsigned but has a `secretKey` field, the - transaction will be signed. - """ - t8n = self.t8n - - # for idx, json_tx in enumerate(self.data): - raw_tx["gasLimit"] = raw_tx["gas"] - raw_tx["data"] = raw_tx["input"] - if "to" not in raw_tx or raw_tx["to"] is None: - raw_tx["to"] = "" - - # tf tool might provide None instead of 0 - # for v, r, s - raw_tx["v"] = raw_tx.get("v") or raw_tx.get("y_parity") or "0x00" - raw_tx["r"] = raw_tx.get("r") or "0x00" - raw_tx["s"] = raw_tx.get("s") or "0x00" - - v = hex_to_u256(raw_tx["v"]) - r = hex_to_u256(raw_tx["r"]) - s = hex_to_u256(raw_tx["s"]) - - if "secretKey" in raw_tx and v == r == s == 0: - self.sign_transaction(raw_tx) - - tx = TransactionLoad(raw_tx, t8n.fork).read() - self.all_txs.append(tx) - - if t8n.fork.has_decode_transaction: - transaction = t8n.fork.decode_transaction(tx) - else: - transaction = tx - - return transaction - - def sign_transaction(self, json_tx: Any) -> None: - """ - Sign a transaction. This function will be invoked if a `secretKey` - is provided in the transaction. - Post spurious dragon, the transaction is signed according to EIP-155 - if the protected flag is missing or set to true. - """ - t8n = self.t8n - protected = json_tx.get("protected", True) - - tx = TransactionLoad(json_tx, t8n.fork).read() - - if isinstance(tx, bytes): - tx_decoded = t8n.fork.decode_transaction(tx) - else: - tx_decoded = tx - - secret_key = hex_to_uint(json_tx["secretKey"][2:]) - if t8n.fork.has_legacy_transaction: - Transaction = t8n.fork.LegacyTransaction # noqa N806 - else: - Transaction = t8n.fork.Transaction # noqa N806 - - v_addend: U256 - if isinstance(tx_decoded, Transaction): - if t8n.fork.has_signing_hash_155: - if protected: - signing_hash = t8n.fork.signing_hash_155( - tx_decoded, self.t8n.chain_id - ) - # EIP-155: CHAIN_ID * 2 + 35 - v_addend = U256(self.t8n.chain_id) * U256(2) + U256(35) - else: - signing_hash = t8n.fork.signing_hash_pre155(tx_decoded) - v_addend = U256(27) - else: - signing_hash = t8n.fork.signing_hash(tx_decoded) - v_addend = U256(27) - elif isinstance(tx_decoded, t8n.fork.AccessListTransaction): - signing_hash = t8n.fork.signing_hash_2930(tx_decoded) - v_addend = U256(0) - elif isinstance(tx_decoded, t8n.fork.FeeMarketTransaction): - signing_hash = t8n.fork.signing_hash_1559(tx_decoded) - v_addend = U256(0) - elif isinstance(tx_decoded, t8n.fork.BlobTransaction): - signing_hash = t8n.fork.signing_hash_4844(tx_decoded) - v_addend = U256(0) - elif isinstance(tx_decoded, t8n.fork.SetCodeTransaction): - signing_hash = t8n.fork.signing_hash_7702(tx_decoded) - v_addend = U256(0) - else: - raise FatalError("Unknown transaction type") - - r, s, y = secp256k1_sign(signing_hash, int(secret_key)) - json_tx["r"] = hex(r) - json_tx["s"] = hex(s) - json_tx["v"] = hex(y + v_addend) - - if v_addend == 0: - json_tx["y_parity"] = json_tx["v"] - - -@dataclass -class Result: - """Type that represents the result of a transition execution.""" - - difficulty: Any - base_fee: Any - state_root: Any = None - tx_root: Any = None - receipt_root: Any = None - withdrawals_root: Any = None - logs_hash: Any = None - bloom: Any = None - receipts: Any = None - rejected: Any = None - gas_used: Any = None - excess_blob_gas: Optional[U64] = None - blob_gas_used: Optional[Uint] = None - requests_hash: Optional[Hash32] = None - requests: Optional[List[Bytes]] = None - block_exception: Optional[str] = None - block_access_list: Optional[Any] = None - block_access_list_hash: Optional[Hash32] = None - - def get_receipts_from_output( - self, - t8n: Any, - block_output: Any, - ) -> List[Any]: - """ - Get receipts from the transaction and receipts tries. - """ - receipts: List[Any] = [] - for key in block_output.receipt_keys: - tx = trie_get(block_output.transactions_trie, key) - receipt = trie_get(block_output.receipts_trie, key) - - assert tx is not None - assert receipt is not None - - tx_hash = t8n.fork.get_transaction_hash(tx) - - if hasattr(t8n.fork, "decode_receipt"): - decoded_receipt = t8n.fork.decode_receipt(receipt) - else: - decoded_receipt = receipt - - receipts.append((tx_hash, decoded_receipt)) - - return receipts - - def update(self, t8n: "T8N", block_env: Any, block_output: Any) -> None: - """ - Update the result after processing the inputs. - """ - self.gas_used = block_output.block_gas_used - self.tx_root = root(block_output.transactions_trie) - self.receipt_root = root(block_output.receipts_trie) - self.bloom = t8n.fork.logs_bloom(block_output.block_logs) - self.logs_hash = keccak256(rlp.encode(block_output.block_logs)) - block_diff = t8n.fork.extract_block_diff(t8n._block_state) - state_root_value, _ = ( - t8n.alloc.state.compute_state_root_and_trie_changes( - block_diff.account_changes, - block_diff.storage_changes, - block_diff.storage_clears, - ) - ) - self.state_root = state_root_value - # Apply diffs to pre-state for alloc output - apply_changes_to_state(t8n.alloc.state, block_diff) - self.receipts = self.get_receipts_from_output(t8n, block_output) - - if hasattr(block_env, "base_fee_per_gas"): - self.base_fee = block_env.base_fee_per_gas - - if hasattr(block_output, "withdrawals_trie"): - self.withdrawals_root = root(block_output.withdrawals_trie) - - if hasattr(block_env, "excess_blob_gas"): - self.excess_blob_gas = block_env.excess_blob_gas - - if hasattr(block_output, "requests"): - self.requests = block_output.requests - self.requests_hash = t8n.fork.compute_requests_hash(self.requests) - - if hasattr(block_output, "block_access_list"): - self.block_access_list = block_output.block_access_list - self.block_access_list_hash = t8n.fork.hash_block_access_list( - block_output.block_access_list - ) - - def json_encode_receipts(self) -> Any: - """ - Encode receipts to JSON. - """ - receipts_json = [] - for tx_hash, receipt in self.receipts: - receipt_dict = {"transactionHash": "0x" + tx_hash.hex()} - - if hasattr(receipt, "succeeded"): - receipt_dict["succeeded"] = receipt.succeeded - else: - assert hasattr(receipt, "post_state") - receipt_dict["post_state"] = "0x" + receipt.post_state.hex() - - receipt_dict["cumulativeGasUsed"] = hex( - receipt.cumulative_gas_used - ) - receipt_dict["bloom"] = "0x" + receipt.bloom.hex() - - # Add logs to receipts - logs_json = [] - for log in receipt.logs: - log_dict = { - "address": "0x" + log.address.hex(), - "topics": ["0x" + topic.hex() for topic in log.topics], - "data": "0x" + log.data.hex(), - } - logs_json.append(log_dict) - receipt_dict["logs"] = logs_json - - receipts_json.append(receipt_dict) - - return receipts_json - - def to_json(self) -> Any: - """Encode the result to JSON.""" - data = {} - - data["stateRoot"] = "0x" + self.state_root.hex() - data["txRoot"] = "0x" + self.tx_root.hex() - data["receiptsRoot"] = "0x" + self.receipt_root.hex() - if self.withdrawals_root: - data["withdrawalsRoot"] = "0x" + self.withdrawals_root.hex() - data["logsHash"] = "0x" + self.logs_hash.hex() - data["logsBloom"] = "0x" + self.bloom.hex() - data["gasUsed"] = hex(self.gas_used) - if self.difficulty: - data["currentDifficulty"] = hex(self.difficulty) - else: - data["currentDifficulty"] = None - - if self.base_fee: - data["currentBaseFee"] = hex(self.base_fee) - else: - data["currentBaseFee"] = None - - if self.excess_blob_gas is not None: - data["currentExcessBlobGas"] = hex(self.excess_blob_gas) - - if self.blob_gas_used is not None: - data["blobGasUsed"] = hex(self.blob_gas_used) - - data["rejected"] = [ - {"index": idx, "error": error} - for idx, error in self.rejected.items() - ] - - data["receipts"] = self.json_encode_receipts() - - if self.requests_hash is not None: - assert self.requests is not None - - data["requestsHash"] = encode_to_hex(self.requests_hash) - # T8N doesn't consider the request type byte to be part of the - # request - data["requests"] = [encode_to_hex(req) for req in self.requests] - - if self.block_exception is not None: - data["blockException"] = self.block_exception - - if self.block_access_list is not None: - # Output BAL as RLP-encoded hex bytes; the testing framework - # handles JSON serialization. - data["blockAccessList"] = encode_to_hex( - rlp.encode(self.block_access_list) - ) - - if self.block_access_list_hash is not None: - data["blockAccessListHash"] = encode_to_hex( - self.block_access_list_hash - ) - - return data diff --git a/src/ethereum_spec_tools/evm_tools/utils.py b/src/ethereum_spec_tools/evm_tools/utils.py index 8698fd07a7a..7483c38b34f 100644 --- a/src/ethereum_spec_tools/evm_tools/utils.py +++ b/src/ethereum_spec_tools/evm_tools/utils.py @@ -15,13 +15,10 @@ Sequence, Tuple, TypeVar, - Union, ) -import coincurve from ethereum_types.numeric import U64, U256, Uint -from ethereum.crypto.hash import Hash32 from ethereum_spec_tools.forks import Hardfork W = TypeVar("W", Uint, U64, U256) @@ -132,6 +129,44 @@ def find_fork( sys.exit(f"Unsupported state fork: {options.state_fork}") +# Map testing ``Fork.transition_tool_name()`` → spec ``Hardfork.short_name`` +# for cases where CamelCase → snake_case does not produce the spec +# module name: +# * ``Paris`` reports itself as ``"Merge"`` to the t8n protocol. +# * ``DAOFork`` would snake-case to ``d_a_o_fork``. +# * ``ConstantinopleFix`` is a testing-side distinction that the spec +# folds into the ``constantinople`` module. +_SPEC_SHORT_NAME_OVERRIDES: Dict[str, str] = { + "Merge": "paris", + "DAOFork": "dao_fork", + "ConstantinopleFix": "constantinople", +} + + +def resolve_fork(fork_name: str) -> Hardfork: + """ + Resolve a testing ``Fork.transition_tool_name()`` to its matching + spec ``Hardfork``. + + CLI exception aliases like ``HomesteadToDaoAt5`` are resolved by + :func:`find_fork` before the testing ``Fork`` is built, so the name + reaching this function is always post-alias-resolution. + """ + short = _SPEC_SHORT_NAME_OVERRIDES.get(fork_name) + if short is None: + short = re.sub(r"(? List[str]: """ Get the supported forks. @@ -166,29 +201,3 @@ def get_stream_logger(name: str) -> Any: logger.addHandler(stream_handler) return logger - - -def secp256k1_sign(msg_hash: Hash32, secret_key: int) -> Tuple[U256, ...]: - """ - Returns the signature of a message hash given the secret key. - """ - private_key = coincurve.PrivateKey.from_int(secret_key) - signature = private_key.sign_recoverable(msg_hash, hasher=None) - - return ( - U256.from_be_bytes(signature[0:32]), - U256.from_be_bytes(signature[32:64]), - U256(signature[64]), - ) - - -def encode_to_hex(data: Union[bytes, int]) -> str: - """ - Encode the data to a hex string. - """ - if isinstance(data, int): - return hex(data) - elif isinstance(data, bytes): - return "0x" + data.hex() - else: - raise Exception("Invalid data type") diff --git a/tests/evm_tools/test_count_opcodes.py b/tests/evm_tools/test_count_opcodes.py index 4220ffa6586..0ced37e51f5 100644 --- a/tests/evm_tools/test_count_opcodes.py +++ b/tests/evm_tools/test_count_opcodes.py @@ -11,7 +11,8 @@ import pytest from ethereum_spec_tools.evm_tools import create_parser -from ethereum_spec_tools.evm_tools.t8n import T8N, ForkCache +from ethereum_spec_tools.evm_tools.t8n import ForkCache +from ethereum_spec_tools.evm_tools.t8n.cli import run_t8n_cli parser = create_parser() @@ -41,10 +42,7 @@ def test_count_opcodes(root_relative: Callable[[str | Path], Path]) -> None: out_file = StringIO() with ForkCache() as fork_cache: - t8n_tool = T8N( - options, out_file=out_file, in_file=in_file, cache=fork_cache - ) - exit_code = t8n_tool.run() + exit_code = run_t8n_cli(options, out_file, in_file, fork_cache) assert 0 == exit_code results = json.loads(out_file.getvalue()) diff --git a/tests/json_loader/helpers/load_state_tests.py b/tests/json_loader/helpers/load_state_tests.py index 4cfe757d943..52a4557ff53 100644 --- a/tests/json_loader/helpers/load_state_tests.py +++ b/tests/json_loader/helpers/load_state_tests.py @@ -1,7 +1,6 @@ """Helper functions to load and run general state tests for Ethereum forks.""" import json -import sys from io import StringIO from typing import Any, Dict, Final, Iterable, List @@ -14,7 +13,8 @@ from ethereum.utils.hexadecimal import hex_to_bytes from ethereum_spec_tools.evm_tools import create_parser from ethereum_spec_tools.evm_tools.statetest import read_test_case -from ethereum_spec_tools.evm_tools.t8n import T8N, ForkCache +from ethereum_spec_tools.evm_tools.t8n import ForkCache +from ethereum_spec_tools.evm_tools.t8n.cli import build_t8n_from_cli_options from .. import FORKS from ..stash_keys import desired_forks_key, fork_cache_key @@ -144,14 +144,18 @@ def runtest(self) -> None: with ForkCache() as fork_cache: try: - t8n = T8N(t8n_options, sys.stdout, in_stream, fork_cache) + t8n = build_t8n_from_cli_options( + t8n_options, in_stream, fork_cache + ) except StateWithEmptyAccount as e: pytest.xfail(str(e)) t8n.run_state_test() if "expectException" in post: - assert 0 in t8n.txs.rejected_txs + assert any( + int(rej.index) == 0 for rej in t8n.rejected_transactions + ) return assert hex_to_bytes(post_hash) == t8n.result.state_root diff --git a/vulture_whitelist.py b/vulture_whitelist.py index 6b045fb9e8b..d5e686c23dd 100644 --- a/vulture_whitelist.py +++ b/vulture_whitelist.py @@ -19,7 +19,7 @@ from ethereum_spec_tools.evm_tools.loaders.transaction_loader import ( TransactionLoad, ) -from ethereum_spec_tools.evm_tools.t8n.env import Ommer +from ethereum_spec_tools.evm_tools.t8n.block_environment import Ommer from ethereum_spec_tools.evm_tools.t8n.evm_trace.eip3155 import ( FinalTrace, Trace, @@ -116,9 +116,15 @@ TransactionLoad.json_to_r TransactionLoad.json_to_s -# src/ethereum_spec_tools/evm_tools/t8n/env.py +# src/ethereum_spec_tools/evm_tools/t8n/block_environment.py Ommer.delta +# src/ethereum_spec_tools/evm_tools/t8n/__init__.py +# `protected` is a field on the testing-package `Transaction` model; +# T8N flips it to False for pre-EIP-155 forks before calling `sign()`. +_unused_protected_marker = None +_unused_protected_marker.protected # type: ignore[attr-defined] + # src/ethereum_spec_tools/evm_tools/t8n/evm_trace/eip3155.py Trace.gasCost Trace.memSize