diff --git a/packages/testing/pyproject.toml b/packages/testing/pyproject.toml index afd2e37ab02..8dd62d7ffdd 100644 --- a/packages/testing/pyproject.toml +++ b/packages/testing/pyproject.toml @@ -54,6 +54,7 @@ dependencies = [ "tenacity>=9.0.0,<10", "Jinja2>=3,<4", "ijson>=3.3,<4", + "eth-remerkleable==0.1.31", ] [project.urls] diff --git a/packages/testing/src/execution_testing/base_types/ssz.py b/packages/testing/src/execution_testing/base_types/ssz.py new file mode 100644 index 00000000000..e639906c99f --- /dev/null +++ b/packages/testing/src/execution_testing/base_types/ssz.py @@ -0,0 +1,1012 @@ +""" +Native SSZ serialization for base_types models. + +Declare a container once as a pydantic SszModel, in the ordinary base types, +and get SSZ encoding, hash_tree_root, and defaults for them. + +Each field's SSZ type is derived from its Python type, so the model stays the +single source of truth: + +* fixed byte types self-describe by byte_length + (Hash -> ByteVector[32], Address -> ByteVector[20]); +* the width ints defined here carry it (Uint64 -> uint64); +* bool -> boolean; a nested SszModel -> Container; +* the only facts a Python type cannot express -- list / vector / bytelist / bit + caps -- ride as Annotated markers (ssz_list(N), ssz_vector(N), byte_list(N), + bitvector(N), bitlist(N)). Element types are derived from the annotation, so + a marker carries only the cap/length, never a duplicated element spec. + +Each field's SSZ type is described by an SszType value (SszUint, SszByteList, +SszList, SszContainer, ...). The engine turns that into a remerkleable type +on demand (build_ssz_type) and delegates the actual encoding, merkleization, +and default (zero) values to it. + +Fork-scoped models: one model can serve every fork. Future-fork fields are +declared T | None (None == absent in older forks, omitted from JSON), and a +__ssz_schema__ = SszForkSchema(...) table beside the fields says which fork +introduces what, in canonical SSZ order (the class body's order stays free +for JSON). Such models require fork= on encode / hash_tree_root / decode / +ssz_default / describe_schema / build_ssz_type +""" + +import ast +import inspect +import io +import re +import textwrap +import tokenize +from dataclasses import dataclass +from functools import lru_cache +from types import UnionType +from typing import ( + Any, + ClassVar, + FrozenSet, + List, + Mapping, + Optional, + Sequence, + Set, + Tuple, + Type, + TypeVar, + Union, + get_args, + get_origin, +) + +from remerkleable.basic import ( + boolean, + uint8, + uint16, + uint32, + uint64, + uint128, + uint256, +) +from remerkleable.bitfields import Bitlist as RmkBitlist +from remerkleable.bitfields import Bitvector as RmkBitvector +from remerkleable.byte_arrays import ByteList, ByteVector +from remerkleable.complex import Container +from remerkleable.complex import List as RmkList +from remerkleable.complex import Vector as RmkVector +from remerkleable.core import View +from remerkleable.progressive import ( + ProgressiveBitlist as RmkProgressiveBitlist, +) +from remerkleable.progressive import ProgressiveContainer +from remerkleable.progressive import ProgressiveList as RmkProgressiveList + +from .base_types import Bytes, FixedSizeBytes, HexNumber +from .pydantic import CamelModel + +_UINTS = { + 8: uint8, + 16: uint16, + 32: uint32, + 64: uint64, + 128: uint128, + 256: uint256, +} + + +class SszType: + """A description of a field's SSZ type.""" + + +@dataclass(frozen=True) +class SszUint(SszType): + """An unsigned integer of bits width (8/16/32/64/128/256).""" + + bits: int + + +@dataclass(frozen=True) +class SszByteVector(SszType): + """A fixed-length byte vector of length bytes.""" + + length: int + + +@dataclass(frozen=True) +class SszByteList(SszType): + """A variable byte list capped at limit bytes.""" + + limit: int + + +@dataclass(frozen=True) +class SszList(SszType): + """A list of element capped at limit items.""" + + element: SszType + limit: int + + +@dataclass(frozen=True) +class SszVector(SszType): + """A fixed-length vector of exactly length element items.""" + + element: SszType + length: int + + +@dataclass(frozen=True) +class SszBitvector(SszType): + """A fixed-length bit vector of length bits.""" + + length: int + + +@dataclass(frozen=True) +class SszBitlist(SszType): + """A variable bit list capped at limit bits.""" + + limit: int + + +@dataclass(frozen=True) +class SszBool(SszType): + """The SSZ boolean type.""" + + +@dataclass(frozen=True) +class SszContainer(SszType): + """A nested container backed by pydantic model.""" + + model: Type["SszModel"] + + +@dataclass(frozen=True) +class SszProgressiveList(SszType): + """An uncapped progressive list of element (EIP-7916).""" + + element: SszType + + +@dataclass(frozen=True) +class SszProgressiveBitlist(SszType): + """An uncapped progressive bit list.""" + + +@dataclass(frozen=True) +class SszProgressiveContainer(SszType): + """A forward-compatible progressive container backed by model.""" + + model: Type["SszModel"] + + +_M = TypeVar("_M", bound="SszModel") + + +@dataclass(frozen=True, eq=False) +class SszForkSchema: + """ + Fork-scoped field sets for a fork-evolving container. + + One model declares every fork's fields; this table says which fields + exist at which fork and in which SSZ order. base holds the fields of + base_fork; appended maps each later fork (in order) to the fields it + adds, which must be declared Optional (T | None) on the model. + + Fork keys are opaque strings: base_types knows nothing about forks; + """ + + base_fork: str + base: Tuple[str, ...] + appended: Mapping[str, Tuple[str, ...]] + + def forks(self) -> Tuple[str, ...]: + """Every known fork key, oldest first.""" + return (self.base_fork, *self.appended) + + def fields_at(self, fork: str) -> Tuple[str, ...]: + """The SSZ field names of fork, in canonical order.""" + if fork == self.base_fork: + return self.base + if fork not in self.appended: + raise TypeError( + f"unknown fork {fork!r}; known forks: {self.forks()}" + ) + names = list(self.base) + for key, fields in self.appended.items(): + names.extend(fields) + if key == fork: + break + return tuple(names) + + def all_fields(self) -> Tuple[str, ...]: + """Every field of the newest fork, in canonical order.""" + keys = self.forks() + return self.fields_at(keys[-1]) + + +def _unwrap_optional(annotation: Any) -> Tuple[Any, bool]: + """Strip a T | None union; return.""" + if get_origin(annotation) in (Union, UnionType): + args = [a for a in get_args(annotation) if a is not type(None)] + if len(args) != 1: + raise TypeError( + f"only T | None unions are supported: {annotation!r}" + ) + return args[0], True + return annotation, False + + +def _is_fork_optional(model_cls: Type["SszModel"], name: str) -> bool: + ann = model_cls.model_fields[name].annotation + return _unwrap_optional(ann)[1] + + +_SSZ_EXCLUDE_COMMENT = re.compile(r"#\s*ssz_exclude\b") + + +@lru_cache(maxsize=None) +def _comment_excluded_names(klass: type) -> FrozenSet[str]: + """ + Field names in klass's own body marked by a ``# ssz_exclude`` + comment. + + The comment must sit on the line directly above the field (or trail + the field's own line). Classes whose source cannot be retrieved + (built dynamically) contribute nothing; use the Annotated + ssz_exclude() marker there. + """ + try: + source = textwrap.dedent(inspect.getsource(klass)) + tree = ast.parse(source) + tokens = list(tokenize.generate_tokens(io.StringIO(source).readline)) + except (OSError, TypeError, SyntaxError, tokenize.TokenError): + return frozenset() + marked: Set[int] = set() + for tok in tokens: + if tok.type != tokenize.COMMENT: + continue + if not _SSZ_EXCLUDE_COMMENT.match(tok.string): + continue + row, col = tok.start + on_own_line = not tok.line[:col].strip() + marked.add(row + 1 if on_own_line else row) + class_def = tree.body[0] + if not isinstance(class_def, ast.ClassDef): + return frozenset() + return frozenset( + node.target.id + for node in class_def.body + if isinstance(node, ast.AnnAssign) + and isinstance(node.target, ast.Name) + and node.lineno in marked + ) + + +def _is_ssz_excluded(model_cls: Type["SszModel"], name: str) -> bool: + metadata = model_cls.model_fields[name].metadata + if any(isinstance(m, _SszExclude) for m in metadata): + return True + return any( + name in _comment_excluded_names(klass) + for klass in model_cls.__mro__ + if klass is not SszModel and issubclass(klass, SszModel) + ) + + +def _included_fields(model_cls: Type["SszModel"]) -> Tuple[str, ...]: + """Every SSZ-participating field, in declaration order.""" + return tuple( + name + for name in model_cls.model_fields + if not _is_ssz_excluded(model_cls, name) + ) + + +def _check_fork_schema(model_cls: Type["SszModel"]) -> None: + """ + Validate a model's __ssz_schema__ against its fields, at class + definition. + + Optional (T | None) fields require a schema naming their fork; the + schema must cover exactly the model's SSZ fields; base fields must + be required and appended fields Optional with a None default -- so + a mis-declared container fails at import. + """ + schema = model_cls.__ssz_schema__ + included = _included_fields(model_cls) + optional = { + name for name in included if _is_fork_optional(model_cls, name) + } + progressive = globals().get("ProgressiveModel") + if progressive is not None and issubclass(model_cls, progressive): + if schema is not None: + raise TypeError( + f"{model_cls.__name__}: __ssz_schema__ is not supported " + f"on ProgressiveModel (progressive containers evolve via " + f"__active_fields__)" + ) + if optional: + raise TypeError( + f"{model_cls.__name__}: T | None fields are not supported " + f"on ProgressiveModel; reserve future slots with 0s in " + f"__active_fields__ instead" + ) + return + if schema is None: + if optional: + raise TypeError( + f"{model_cls.__name__} has fork-optional fields " + f"{sorted(optional)} but no __ssz_schema__ declaring " + f"which fork introduces them" + ) + return + all_names = schema.all_fields() + dupes = sorted({n for n in all_names if all_names.count(n) > 1}) + if dupes: + raise TypeError( + f"{model_cls.__name__}.__ssz_schema__ names fields more than " + f"once: {dupes}" + ) + declared = set(all_names) + fields = set(included) + if declared != fields: + raise TypeError( + f"{model_cls.__name__}.__ssz_schema__ does not match the " + f"model: schema-only={sorted(declared - fields)} " + f"model-only={sorted(fields - declared)}" + ) + appended = fields - set(schema.base) + if optional != appended: + raise TypeError( + f"{model_cls.__name__}: appended fields must be T | None and " + f"base fields required; non-optional appended=" + f"{sorted(appended - optional)} optional base=" + f"{sorted(optional - appended)}" + ) + no_default = sorted( + name for name in appended if model_cls.model_fields[name].is_required() + ) + if no_default: + raise TypeError( + f"{model_cls.__name__}: appended fields must default to None " + f"(decode of older forks constructs without them): " + f"{no_default}" + ) + + +class _Marker: + """Base for cap-only Annotated markers resolved by spec_of.""" + + +@dataclass(frozen=True) +class _ListCap(_Marker): + limit: int + + +@dataclass(frozen=True) +class _VectorLen(_Marker): + length: int + + +@dataclass(frozen=True) +class _ProgressiveListMark(_Marker): + pass + + +@dataclass(frozen=True) +class _SszExclude(_Marker): + pass + + +def byte_list(limit: int) -> SszByteList: + """Annotate a Bytes field as a capped SSZ byte list.""" + return SszByteList(limit) + + +def ssz_list(limit: int) -> _ListCap: + """Annotate a list[...] field as a capped SSZ list.""" + return _ListCap(limit) + + +def ssz_vector(length: int) -> _VectorLen: + """Annotate a list[...] field as a fixed SSZ vector.""" + return _VectorLen(length) + + +def bitvector(length: int) -> SszBitvector: + """Annotate a list[bool] field as a fixed SSZ bit vector.""" + return SszBitvector(length) + + +def bitlist(limit: int) -> SszBitlist: + """Annotate a list[bool] field as a capped SSZ bit list.""" + return SszBitlist(limit) + + +def progressive_list() -> _ProgressiveListMark: + """Annotate a list[...] field as an uncapped progressive list.""" + return _ProgressiveListMark() + + +def progressive_bitlist() -> SszProgressiveBitlist: + """Annotate a list[bool] field as an uncapped progressive bit list.""" + return SszProgressiveBitlist() + + +def ssz_exclude() -> _SszExclude: + """ + Annotate a field as JSON-only: the SSZ ignores it entirely. + + The usual spelling is a ``# ssz_exclude`` comment on the line + directly above the field; this Annotated marker is the fallback for + classes built dynamically, whose source the comment scan cannot + read. + """ + return _SszExclude() + + +class SszModel(CamelModel): + """ + A pydantic model whose fields carry SSZ types. + + Every field must resolve to an SszType, or be excluded from SSZ + with a ``# ssz_exclude`` comment on the line above it (JSON-only + fields); each Annotated marker must be consistent with the field's + Python type. Both are checked when the subclass is defined, so a + mis-typed container fails at import, not at first encode. + """ + + __ssz_schema__: ClassVar[Optional[SszForkSchema]] = None + + @classmethod + def __pydantic_init_subclass__(cls, **kwargs: Any) -> None: + """Validate every field resolves to a consistent SSZ type.""" + super().__pydantic_init_subclass__(**kwargs) + for name in cls.model_fields: + if _is_ssz_excluded(cls, name): + if cls.model_fields[name].is_required(): + raise TypeError( + f"{cls.__name__}.{name} is SSZ-excluded but has " + f"no default; decode cannot reconstruct it" + ) + continue + spec_of(cls, name) # raises TypeError on unmapped/inconsistent + _check_fork_schema(cls) + + +class ProgressiveModel(SszModel): + """ + A forward-compatible progressive container. + + __active_fields__ is the active-field bitvector; it defaults to all fields + active. A 0 marks a reserved gap with no declared field, so new fields can + be slotted in later without shifting existing roots -- the declared fields + fill the 1 positions in order. The number of 1s, when set, must equal the + declared field count. + """ + + __active_fields__: ClassVar[Sequence[int]] = () + + @classmethod + def __pydantic_init_subclass__(cls, **kwargs: Any) -> None: + """Check the active-field bitvector agrees with the field count.""" + super().__pydantic_init_subclass__(**kwargs) + active = cls.__active_fields__ + if active and sum(active) != len(cls.model_fields): + raise TypeError( + f"{cls.__name__}.__active_fields__ has {sum(active)} active " + f"entries but the container declares " + f"{len(cls.model_fields)} fields" + ) + + +def _marker_in(metadata: Any) -> Any: + """The first SSZ marker in metadata.""" + return next( + (m for m in metadata if isinstance(m, (SszType, _Marker))), None + ) + + +def _spec_for_type_bare(annotation: Any) -> SszType: + """Derive the SSZ type of a plain Python type.""" + ssz = getattr(annotation, "__ssz__", None) + if isinstance(ssz, SszType): + return ssz + if isinstance(annotation, type): + if issubclass(annotation, FixedSizeBytes): + return SszByteVector(annotation.byte_length) + if issubclass(annotation, ProgressiveModel): + return SszProgressiveContainer(annotation) + if issubclass(annotation, SszModel): + return SszContainer(annotation) + if annotation is bool: + return SszBool() + raise TypeError(f"no SSZ type for {annotation!r}") + + +def _spec_for_type(annotation: Any) -> SszType: + """Resolve an SSZ type, honoring an inner Annotated marker if present.""" + meta = getattr(annotation, "__metadata__", None) + if meta is not None: + return _resolve(_marker_in(meta), annotation.__origin__) + return _spec_for_type_bare(annotation) + + +def _element_of(annotation: Any, ctx: str) -> SszType: + """Resolve the element SSZ type of a list[...] annotation.""" + if get_origin(annotation) not in (list, List): + raise TypeError(f"{ctx} requires a list[...] field: {annotation!r}") + args = get_args(annotation) + if len(args) != 1: + raise TypeError(f"{ctx} needs a single list element type") + return _spec_for_type(args[0]) + + +def _resolve(marker: Any, annotation: Any) -> SszType: + """ + Resolve a field/element into an SSZ type. + + Cap-only markers (ssz_list/ssz_vector/progressive_list) derive their + element from the annotation; complete markers (byte_list/bitvector/...) + are checked for consistency with it. Byte-list elements are expressed as + Annotated[Bytes, byte_list(N)] so the inner cap lives on the element. + """ + if marker is None: + return _spec_for_type(annotation) + if isinstance(marker, _ListCap): + return SszList(_element_of(annotation, "ssz_list"), marker.limit) + if isinstance(marker, _VectorLen): + return SszVector(_element_of(annotation, "ssz_vector"), marker.length) + if isinstance(marker, _ProgressiveListMark): + return SszProgressiveList(_element_of(annotation, "progressive_list")) + if isinstance(marker, SszByteList): + is_bytes = isinstance(annotation, type) and issubclass( + annotation, Bytes + ) + if not is_bytes: + raise TypeError( + f"byte_list requires a Bytes field/element: {annotation!r}" + ) + return marker + if isinstance(marker, (SszBitvector, SszBitlist, SszProgressiveBitlist)): + if not isinstance(_element_of(annotation, "bit markers"), SszBool): + raise TypeError( + f"bit markers require a list[bool] field: {annotation!r}" + ) + return marker + if isinstance(marker, _SszExclude): + raise TypeError( + f"field is ssz_exclude()d; it has no SSZ type: {annotation!r}" + ) + # Raw SszType instances (SszUint, SszContainer, ...) as markers would + # bypass the consistency checks above; only the marker helpers are + # supported. + raise TypeError( + f"unsupported Annotated SSZ marker {marker!r}; use the marker " + f"helpers (ssz_list, ssz_vector, byte_list, bitvector, ...)" + ) + + +@lru_cache(maxsize=None) +def spec_of(model_cls: Type["SszModel"], name: str) -> SszType: + """ + The resolved SSZ type of a field. + + An Annotated marker takes precedence over the bare type; cap-only markers + derive their element from the annotation, and every marker is checked for + consistency with it. A T | None union resolves to T's SSZ type -- the + None arm means "absent in older forks" (see SszForkSchema), which is a + schema fact, not an SSZ type. Cached per (model_cls, name). + """ + field = model_cls.model_fields[name] + annotation, _ = _unwrap_optional(field.annotation) + if _is_ssz_excluded(model_cls, name): + raise TypeError( + f"{model_cls.__name__}.{name} is SSZ-excluded; it has no " + f"SSZ type: {annotation!r}" + ) + return _resolve(_marker_in(field.metadata), annotation) + + +def _rmk_type(spec: SszType, fork: Optional[str] = None) -> Type[View]: + if isinstance(spec, SszUint): + return _UINTS[spec.bits] + if isinstance(spec, SszByteVector): + return ByteVector[spec.length] + if isinstance(spec, SszByteList): + return ByteList[spec.limit] + if isinstance(spec, SszList): + return RmkList[_rmk_type(spec.element, fork), spec.limit] + if isinstance(spec, SszVector): + return RmkVector[_rmk_type(spec.element, fork), spec.length] + if isinstance(spec, SszBitvector): + return RmkBitvector[spec.length] + if isinstance(spec, SszBitlist): + return RmkBitlist[spec.limit] + if isinstance(spec, SszProgressiveList): + return RmkProgressiveList[_rmk_type(spec.element, fork)] + if isinstance(spec, SszProgressiveBitlist): + return RmkProgressiveBitlist + if isinstance(spec, (SszContainer, SszProgressiveContainer)): + return build_ssz_type(spec.model, _nested_fork(spec.model, fork)) + if isinstance(spec, SszBool): + return boolean + raise TypeError(f"unhandled SSZ type {spec!r}") + + +def _active_fields(model_cls: Type["SszModel"]) -> Sequence[int]: + declared = getattr(model_cls, "__active_fields__", ()) + return declared if declared else [1] * len(model_cls.model_fields) + + +def _nested_fork( + model_cls: Type["SszModel"], fork: Optional[str] +) -> Optional[str]: + """ + The fork a nested container is projected at. + + One fork propagates down the whole value tree: everything inside one + message is at the same chain fork, so a fork-scoped nested model + inherits the outer fork, while a + complete nested model takes no fork at all. + """ + return fork if model_cls.__ssz_schema__ is not None else None + + +def _schema_fields( + model_cls: Type["SszModel"], fork: Optional[str] +) -> Tuple[str, ...]: + """ + The SSZ field names of model_cls, in canonical order. + + A fork-scoped model requires fork and gets + that fork's fields in the schema's order; a complete model forbids + fork and gets every non-excluded field in declaration order. + """ + schema = model_cls.__ssz_schema__ + if schema is None: + if fork is not None: + raise TypeError( + f"{model_cls.__name__} is not fork-scoped; do not pass fork" + ) + return _included_fields(model_cls) + if fork is None: + raise TypeError( + f"{model_cls.__name__} is fork-scoped; pass fork= " + f"(one of {schema.forks()})" + ) + return schema.fields_at(fork) + + +def ssz_fields( + model_cls: Type["SszModel"], fork: Optional[str] = None +) -> Tuple[str, ...]: + """ + The SSZ field names of model_cls, in canonical (wire) order. + + The public twin of the engine's internal field selection: callers + (vector generators, fixtures tooling) can enumerate exactly the + fields a model encodes -- per fork for fork-scoped models. + """ + return _schema_fields(model_cls, fork) + + +def _check_populated( + model: "SszModel", names: Tuple[str, ...], fork: str +) -> None: + """Raise unless the populated fields exactly match the fork schema.""" + missing = [n for n in names if getattr(model, n) is None] + unexpected = sorted( + n + for n in _included_fields(type(model)) + if n not in names and getattr(model, n) is not None + ) + if missing or unexpected: + raise TypeError( + f"{type(model).__name__} does not fit the {fork!r} SSZ schema: " + f"missing={missing} unexpected={unexpected}; " + f"refusing to drop data" + ) + + +def build_ssz_type( + model_cls: Type["SszModel"], fork: Optional[str] = None +) -> Type[Container]: + """ + Build the remerkleable container type mirroring model_cls. + + Cached per (class object, fork) -- distinct same-named models get + distinct types, and each fork of a fork-scoped model gets its own + genuinely distinct container (different offsets and merkle shape). + """ + return _build_ssz_type(model_cls, fork) + + +@lru_cache(maxsize=None) +def _build_ssz_type( + model_cls: Type["SszModel"], fork: Optional[str] +) -> Type[Container]: + names = _schema_fields(model_cls, fork) + anns = {name: _rmk_type(spec_of(model_cls, name), fork) for name in names} + if issubclass(model_cls, ProgressiveModel): + base: Any = ProgressiveContainer( + active_fields=list(_active_fields(model_cls)) + ) + else: + base = Container + cls_name = model_cls.__name__ + (fork if fork else "") + return type(cls_name, (base,), {"__annotations__": anns}) + + +def _to_rmk(spec: SszType, value: Any, fork: Optional[str] = None) -> Any: + if isinstance(spec, (SszContainer, SszProgressiveContainer)): + return _rmk_instance(value, _nested_fork(spec.model, fork)) + if isinstance(spec, (SszList, SszVector, SszProgressiveList)): + return [_to_rmk(spec.element, v, fork) for v in value] + if isinstance(spec, (SszBitvector, SszBitlist, SszProgressiveBitlist)): + return list(value) + return value # scalar / byte-vector / byte-list: remerkleable coerces + + +def _rmk_instance(model: "SszModel", fork: Optional[str] = None) -> Container: + model_cls: Type[SszModel] = type(model) + names = _schema_fields(model_cls, fork) + if fork is not None: + _check_populated(model, names, fork) + container = build_ssz_type(model_cls, fork) + values = { + name: _to_rmk(spec_of(model_cls, name), getattr(model, name), fork) + for name in names + } + return container(**values) + + +def _to_py(spec: SszType, value: Any, fork: Optional[str] = None) -> Any: + if isinstance(spec, (SszContainer, SszProgressiveContainer)): + nested = _nested_fork(spec.model, fork) + return _view_to_model( + spec.model, value, _schema_fields(spec.model, nested), nested + ) + if isinstance(spec, (SszList, SszVector, SszProgressiveList)): + return [_to_py(spec.element, v, fork) for v in value] + if isinstance(spec, (SszBitvector, SszBitlist, SszProgressiveBitlist)): + return [bool(b) for b in value] + if isinstance(spec, (SszByteVector, SszByteList)): + return bytes(value) + if isinstance(spec, SszUint): + return int(value) + if isinstance(spec, SszBool): + return bool(value) + raise TypeError(f"unhandled SSZ type {spec!r}") + + +def _view_to_model( + model_cls: Type[_M], + view: Container, + names: Optional[Tuple[str, ...]] = None, + fork: Optional[str] = None, +) -> _M: + if names is None: + names = _included_fields(model_cls) + # Fields beyond `names` (older-fork decodes) keep their None default. + return model_cls( + **{ + name: _to_py(spec_of(model_cls, name), getattr(view, name), fork) + for name in names + } + ) + + +def default_value(spec: SszType, fork: Optional[str] = None) -> Any: + """Return the SSZ default (zero) value for spec as a pydantic value.""" + if isinstance(spec, SszUint): + return 0 + if isinstance(spec, SszByteVector): + return b"\x00" * spec.length + if isinstance( + spec, + (SszByteList, SszList, SszBitlist, SszProgressiveList), + ): + return [] + if isinstance(spec, SszProgressiveBitlist): + return [] + if isinstance(spec, SszVector): + # A fresh value per slot: container defaults are mutable, so a shared + # [x] * n would alias one instance across every position. + return [default_value(spec.element, fork) for _ in range(spec.length)] + if isinstance(spec, SszBitvector): + return [False] * spec.length + if isinstance(spec, (SszContainer, SszProgressiveContainer)): + return ssz_default(spec.model, _nested_fork(spec.model, fork)) + if isinstance(spec, SszBool): + return False + raise TypeError(f"no default for SSZ type {spec!r}") + + +def ssz_default(model_cls: Type[_M], fork: Optional[str] = None) -> _M: + """ + Build the SSZ default (all-zero) instance of model_cls. + + Fork-scoped models require fork; fields beyond it stay None. + """ + return model_cls( + **{ + name: default_value(spec_of(model_cls, name), fork) + for name in _schema_fields(model_cls, fork) + } + ) + + +def describe_type(spec: SszType) -> str: + """Render an SSZ type as text (uint64, List[T, N], ...).""" + if isinstance(spec, SszUint): + return f"uint{spec.bits}" + if isinstance(spec, SszByteVector): + return f"ByteVector[{spec.length}]" + if isinstance(spec, SszByteList): + return f"ByteList[{spec.limit}]" + if isinstance(spec, SszList): + return f"List[{describe_type(spec.element)}, {spec.limit}]" + if isinstance(spec, SszVector): + return f"Vector[{describe_type(spec.element)}, {spec.length}]" + if isinstance(spec, SszBitvector): + return f"Bitvector[{spec.length}]" + if isinstance(spec, SszBitlist): + return f"Bitlist[{spec.limit}]" + if isinstance(spec, SszProgressiveList): + return f"ProgressiveList[{describe_type(spec.element)}]" + if isinstance(spec, SszProgressiveBitlist): + return "ProgressiveBitlist" + if isinstance(spec, SszContainer): + return spec.model.__name__ + if isinstance(spec, SszProgressiveContainer): + return f"Progressive[{spec.model.__name__}]" + if isinstance(spec, SszBool): + return "boolean" + raise TypeError(f"unhandled SSZ type {spec!r}") + + +def describe_schema( + model_cls: Type["SszModel"], fork: Optional[str] = None +) -> str: + """ + Render the resolved SSZ layout, one 'field: type' line per field. + + Fork-scoped models require fork and render that fork's projection. + """ + title = model_cls.__name__ + (f" @ {fork}" if fork else "") + lines = [f"{title}:"] + for name in _schema_fields(model_cls, fork): + lines.append(f" {name}: {describe_type(spec_of(model_cls, name))}") + return "\n".join(lines) + + +def encode(model: "SszModel", fork: Optional[str] = None) -> bytes: + """ + Return the SSZ wire bytes of model. + + A fork-scoped model requires fork and is + checked against that fork's schema before encoding. + """ + return _rmk_instance(model, fork).encode_bytes() + + +def hash_tree_root(model: "SszModel", fork: Optional[str] = None) -> bytes: + """ + Return the 32-byte SSZ hash_tree_root of model. + + Fork-scoped models require fork, exactly as encode does. + """ + return bytes(_rmk_instance(model, fork).hash_tree_root()) + + +def decode(model_cls: Type[_M], data: bytes, fork: Optional[str] = None) -> _M: + """ + Decode SSZ data into an instance of model_cls. + + For a fork-scoped model, data is decoded as fork's container and + fields beyond that fork come back as None. + """ + view = build_ssz_type(model_cls, fork).decode_bytes(data) + return _view_to_model( + model_cls, view, _schema_fields(model_cls, fork), fork + ) + + +# width-carrying integer types (base_types.HexNumber underneath) +class _SizedUint(HexNumber): + """ + A width-checked unsigned integer. + """ + + __bits__: ClassVar[int] = 0 + + def __new__(cls, input_number: Any) -> "_SizedUint": + """Create the integer, enforcing 0 <= value < 2**bits.""" + value = super().__new__(cls, input_number) + if not 0 <= int(value) < (1 << cls.__bits__): + raise ValueError(f"{cls.__name__} out of range: {int(value)}") + return value + + +class Uint8(_SizedUint): + """An 8-bit unsigned integer.""" + + __bits__: ClassVar[int] = 8 + __ssz__: ClassVar[SszType] = SszUint(8) + + +class Uint16(_SizedUint): + """A 16-bit unsigned integer.""" + + __bits__: ClassVar[int] = 16 + __ssz__: ClassVar[SszType] = SszUint(16) + + +class Uint32(_SizedUint): + """A 32-bit unsigned integer.""" + + __bits__: ClassVar[int] = 32 + __ssz__: ClassVar[SszType] = SszUint(32) + + +class Uint64(_SizedUint): + """A 64-bit unsigned integer.""" + + __bits__: ClassVar[int] = 64 + __ssz__: ClassVar[SszType] = SszUint(64) + + +class Uint128(_SizedUint): + """A 128-bit unsigned integer.""" + + __bits__: ClassVar[int] = 128 + __ssz__: ClassVar[SszType] = SszUint(128) + + +class Uint256(_SizedUint): + """A 256-bit unsigned integer.""" + + __bits__: ClassVar[int] = 256 + __ssz__: ClassVar[SszType] = SszUint(256) + + +__all__ = [ + "ProgressiveModel", + "SszBitlist", + "SszBitvector", + "SszBool", + "SszByteList", + "SszByteVector", + "SszContainer", + "SszForkSchema", + "SszList", + "SszModel", + "SszProgressiveBitlist", + "SszProgressiveContainer", + "SszProgressiveList", + "SszType", + "SszUint", + "SszVector", + "Uint128", + "Uint16", + "Uint256", + "Uint32", + "Uint64", + "Uint8", + "bitlist", + "bitvector", + "byte_list", + "build_ssz_type", + "decode", + "default_value", + "describe_schema", + "describe_type", + "encode", + "hash_tree_root", + "progressive_bitlist", + "progressive_list", + "spec_of", + "ssz_default", + "ssz_exclude", + "ssz_fields", + "ssz_list", + "ssz_vector", +] diff --git a/packages/testing/src/execution_testing/base_types/tests/test_ssz.py b/packages/testing/src/execution_testing/base_types/tests/test_ssz.py new file mode 100644 index 00000000000..b35dd0a9cf2 --- /dev/null +++ b/packages/testing/src/execution_testing/base_types/tests/test_ssz.py @@ -0,0 +1,961 @@ +""" +Tests for SSZ support in base_types. +""" + +from typing import Annotated, Callable, List, Optional, Tuple + +import pytest +from pydantic import ValidationError +from remerkleable.basic import boolean, uint8, uint64, uint256 +from remerkleable.bitfields import Bitlist as RmkBitlist +from remerkleable.bitfields import Bitvector as RmkBitvector +from remerkleable.byte_arrays import ByteList, ByteVector +from remerkleable.complex import Container +from remerkleable.complex import List as RmkList +from remerkleable.complex import Vector as RmkVector +from remerkleable.progressive import ( + ProgressiveBitlist as RmkProgressiveBitlist, +) +from remerkleable.progressive import ProgressiveContainer +from remerkleable.progressive import ProgressiveList as RmkProgressiveList + +from execution_testing.base_types import Address, Bloom, Bytes, Hash +from execution_testing.base_types.ssz import ( + ProgressiveModel, + SszForkSchema, + SszModel, + SszUint, + Uint8, + Uint16, + Uint32, + Uint64, + Uint128, + Uint256, + bitlist, + bitvector, + build_ssz_type, + byte_list, + decode, + describe_schema, + encode, + hash_tree_root, + progressive_bitlist, + progressive_list, + spec_of, + ssz_default, + ssz_exclude, + ssz_fields, + ssz_list, + ssz_vector, +) + +MAX_EXTRA = 32 +MAX_BYTES_PER_TX = 2**30 +MAX_TXS = 2**20 +MAX_WITHDRAWALS = 16 +CELLS = 128 +BITS = [i % 3 == 0 for i in range(CELLS)] + + +class Withdrawal(SszModel): + """A pydantic model declared to check the SSZ machinery.""" + + index: Uint64 + validator_index: Uint64 + address: Address + amount: Uint64 + + +class ExecutionPayload(SszModel): + """An Amsterdam-shaped payload exercising every field kind.""" + + parent_hash: Hash + fee_recipient: Address + state_root: Hash + logs_bloom: Bloom + block_number: Uint64 + base_fee_per_gas: Uint256 + extra_data: Annotated[Bytes, byte_list(MAX_EXTRA)] + transactions: Annotated[ + List[Annotated[Bytes, byte_list(MAX_BYTES_PER_TX)]], + ssz_list(MAX_TXS), + ] + withdrawals: Annotated[List[Withdrawal], ssz_list(MAX_WITHDRAWALS)] + + +class Status(SszModel): + """A boolean and a fixed bit vector.""" + + ok: bool + columns: Annotated[List[bool], bitvector(CELLS)] + + +class Committee(SszModel): + """A fixed Vector[uint64, N] and a variable Bitlist[N].""" + + seats: Annotated[List[Uint64], ssz_vector(3)] + flags: Annotated[List[bool], bitlist(8)] + + +class Ballot(SszModel): + """An uncapped progressive bit list.""" + + votes: Annotated[List[bool], progressive_bitlist()] + + +class Prog(ProgressiveModel): + """EIP-7916 progressive container with a progressive list.""" + + a: Uint64 + b: Uint8 + items: Annotated[List[Uint64], progressive_list()] + + +class GapProg(ProgressiveModel): + """Two fields around a reserved (0) middle slot.""" + + __active_fields__ = [1, 0, 1] + + a: Uint64 + c: Uint64 + + +class ForkedPayload(SszModel): + """One model for every fork.""" + + parent_hash: Hash + blob_gas_used: Uint64 | None = None + block_number: Uint64 + transactions: Annotated[ # JSON order differs from SSZ. + List[Annotated[Bytes, byte_list(MAX_BYTES_PER_TX)]], + ssz_list(MAX_TXS), + ] + withdrawals: ( + Annotated[List[Withdrawal], ssz_list(MAX_WITHDRAWALS)] | None + ) = None + + __ssz_schema__ = SszForkSchema( + base_fork="Paris", + base=("parent_hash", "block_number", "transactions"), + appended={ + "Shanghai": ("withdrawals",), + "Cancun": ("blob_gas_used",), + }, + ) + + +class Mixed(SszModel): + """An SSZ container carrying a JSON-only (excluded) field.""" + + a: Uint64 + note: Annotated[str, ssz_exclude()] = "json-only" + + +class CommentMixed(SszModel): + """An SSZ container whose JSON-only field is comment-marked.""" + + a: Uint64 + # ssz_exclude + note: str = "json-only" + + +class RefWithdrawal(Container): + """Hand-written twin of Withdrawal.""" + + index: uint64 + validator_index: uint64 + address: ByteVector[20] + amount: uint64 + + +class RefPayload(Container): + """Hand-written twin of ExecutionPayload.""" + + parent_hash: ByteVector[32] + fee_recipient: ByteVector[20] + state_root: ByteVector[32] + logs_bloom: ByteVector[256] + block_number: uint64 + base_fee_per_gas: uint256 + extra_data: ByteList[MAX_EXTRA] + transactions: RmkList[ByteList[MAX_BYTES_PER_TX], MAX_TXS] + withdrawals: RmkList[RefWithdrawal, MAX_WITHDRAWALS] + + +class RefStatus(Container): + """Hand-written twin of Status.""" + + ok: boolean + columns: RmkBitvector[CELLS] + + +class RefCommittee(Container): + """Hand-written twin of Committee.""" + + seats: RmkVector[uint64, 3] + flags: RmkBitlist[8] + + +class RefBallot(Container): + """Hand-written twin of Ballot.""" + + votes: RmkProgressiveBitlist + + +class RefProg(ProgressiveContainer(active_fields=[1, 1, 1])): # type: ignore[misc] + """Hand-written twin of Prog.""" + + a: uint64 + b: uint8 + items: RmkProgressiveList[uint64] + + +class RefGapProg(ProgressiveContainer(active_fields=[1, 0, 1])): # type: ignore[misc] + """Hand-written twin of GapProg.""" + + a: uint64 + c: uint64 + + +class RefForkedParis(Container): + """Hand-written twin of ForkedPayload at Paris.""" + + parent_hash: ByteVector[32] + block_number: uint64 + transactions: RmkList[ByteList[MAX_BYTES_PER_TX], MAX_TXS] + + +class RefForkedShanghai(Container): + """Hand-written twin of ForkedPayload at Shanghai.""" + + parent_hash: ByteVector[32] + block_number: uint64 + transactions: RmkList[ByteList[MAX_BYTES_PER_TX], MAX_TXS] + withdrawals: RmkList[RefWithdrawal, MAX_WITHDRAWALS] + + +class RefMixed(Container): # the excluded field simply does not exist + """Hand-written twin of Mixed (no excluded field).""" + + a: uint64 + + +def _withdrawal() -> Withdrawal: + return Withdrawal( + index=7, + validator_index=42, + address=Address(b"\x11" * 20), + amount=32_000_000_000, + ) + + +def _ref_withdrawal() -> Container: + return RefWithdrawal( + index=7, + validator_index=42, + address=b"\x11" * 20, + amount=32_000_000_000, + ) + + +def _payload() -> ExecutionPayload: + return ExecutionPayload( + parent_hash=Hash(b"\xaa" * 32), + fee_recipient=Address(b"\xbb" * 20), + state_root=Hash(b"\xcc" * 32), + logs_bloom=Bloom(b"\x00" * 256), + block_number=21_000_000, + base_fee_per_gas=10**18, + extra_data=Bytes(b"\xde\xad"), + transactions=[Bytes(b"\x02\xf8"), Bytes(b"\x03" * 5)], + withdrawals=[_withdrawal()], + ) + + +def _ref_payload() -> Container: + return RefPayload( + parent_hash=b"\xaa" * 32, + fee_recipient=b"\xbb" * 20, + state_root=b"\xcc" * 32, + logs_bloom=b"\x00" * 256, + block_number=21_000_000, + base_fee_per_gas=10**18, + extra_data=b"\xde\xad", + transactions=[b"\x02\xf8", b"\x03" * 5], + withdrawals=[_ref_withdrawal()], + ) + + +def _paris_payload() -> ForkedPayload: + return ForkedPayload( + parent_hash=Hash(b"\xaa" * 32), + block_number=100, + transactions=[Bytes(b"\x02\xf8")], + ) + + +def _shanghai_payload() -> ForkedPayload: + return ForkedPayload( + parent_hash=Hash(b"\xaa" * 32), + block_number=100, + transactions=[Bytes(b"\x02\xf8")], + withdrawals=[_withdrawal()], + ) + + +def assert_matches_reference( + model: SszModel, ref: Container, fork: Optional[str] = None +) -> None: + """ + Compare the engine against a hand-written remerkleable twin. + + The twin is the ground truth: everything observable must + """ + model_cls = type(model) + ref_cls = type(ref) + raw = encode(model, fork) + # populated instance: wire bytes + merkle root + assert raw == ref.encode_bytes() + assert hash_tree_root(model, fork) == bytes(ref.hash_tree_root()) + # decode round-trips losslessly, back to an equal pydantic model + restored = decode(model_cls, raw, fork) + assert encode(restored, fork) == raw + assert restored == model + # both sides agree on the zero value + zero = ssz_default(model_cls, fork) + assert encode(zero, fork) == ref_cls().encode_bytes() + assert hash_tree_root(zero, fork) == bytes(ref_cls().hash_tree_root()) + + +TWIN_CASES: List[ + Tuple[ + str, + Callable[[], SszModel], + Callable[[], Container], + Optional[str], + ] +] = [ + ("withdrawal", _withdrawal, _ref_withdrawal, None), + ("payload", _payload, _ref_payload, None), + ( + "bool-bitvector", + lambda: Status(ok=True, columns=BITS), + lambda: RefStatus(ok=True, columns=BITS), + None, + ), + ( + "vector-bitlist", + lambda: Committee(seats=[1, 2, 3], flags=[True, False, True]), + lambda: RefCommittee(seats=[1, 2, 3], flags=[True, False, True]), + None, + ), + ( + "progressive-bitlist", + lambda: Ballot(votes=[True, False, True, True]), + lambda: RefBallot(votes=[True, False, True, True]), + None, + ), + ( + "progressive", + lambda: Prog(a=5, b=9, items=[10, 20, 30]), + lambda: RefProg(a=5, b=9, items=[10, 20, 30]), + None, + ), + ( + "progressive-gap", + lambda: GapProg(a=1, c=3), + lambda: RefGapProg(a=1, c=3), + None, + ), + ( + # default-valued excluded field: decode restores the default, so + # the harness's restored == model leg holds; the non-default case + # is covered by test_excluded_field_is_json_only. + "excluded-field", + lambda: Mixed(a=7), + lambda: RefMixed(a=7), + None, + ), + ( + "forked-paris", + _paris_payload, + lambda: RefForkedParis( + parent_hash=b"\xaa" * 32, + block_number=100, + transactions=[b"\x02\xf8"], + ), + "Paris", + ), + ( + "forked-shanghai", + _shanghai_payload, + lambda: RefForkedShanghai( + parent_hash=b"\xaa" * 32, + block_number=100, + transactions=[b"\x02\xf8"], + withdrawals=[_ref_withdrawal()], + ), + "Shanghai", + ), +] + + +@pytest.mark.parametrize( + "make_model,make_ref,fork", + [pytest.param(m, r, f, id=name) for name, m, r, f in TWIN_CASES], +) +def test_matches_remerkleable_reference( + make_model: Callable[[], SszModel], + make_ref: Callable[[], Container], + fork: Optional[str], +) -> None: + """Every model kind is byte-identical to its hand-written twin.""" + assert_matches_reference(make_model(), make_ref(), fork) + + +def test_full_payload_round_trips() -> None: + """A container with every field kind round-trips pydantic<->SSZ.""" + payload = _payload() + restored = decode(ExecutionPayload, encode(payload)) + assert restored.parent_hash == payload.parent_hash + assert int(restored.base_fee_per_gas) == 10**18 + assert [bytes(t) for t in restored.transactions] == [ + b"\x02\xf8", + b"\x03" * 5, + ] + assert int(restored.withdrawals[0].amount) == 32_000_000_000 + assert len(hash_tree_root(payload)) == 32 + + +def test_ssz_default_matches_remerkleable_zero() -> None: + """ssz_default builds the SSZ zero value, like remerkleable's default.""" + zero = ssz_default(ExecutionPayload) + assert int(zero.block_number) == 0 + assert zero.transactions == [] + assert zero.withdrawals == [] + assert bytes(zero.parent_hash) == b"\x00" * 32 + # zero encodes identically to a freshly-defaulted remerkleable container + assert encode(zero) == build_ssz_type(ExecutionPayload)().encode_bytes() + + +def test_describe_schema_renders_every_field_kind() -> None: + """describe_schema renders the resolved SSZ type of each field.""" + schema = describe_schema(ExecutionPayload) + assert "block_number: uint64" in schema + assert "base_fee_per_gas: uint256" in schema + assert "parent_hash: ByteVector[32]" in schema + assert f"extra_data: ByteList[{MAX_EXTRA}]" in schema + assert ( + f"transactions: List[ByteList[{MAX_BYTES_PER_TX}], {MAX_TXS}]" + in schema + ) + assert f"withdrawals: List[Withdrawal, {MAX_WITHDRAWALS}]" in schema + # progressive kinds render their consensus-style names + assert "items: ProgressiveList[uint64]" in describe_schema(Prog) + assert "votes: ProgressiveBitlist" in describe_schema(Ballot) + + +def test_default_vector_of_container_has_independent_slots() -> None: + """A defaulted Vector-of-container has independent (non-aliased) slots.""" + + class Inner(SszModel): + x: Uint64 + + class Outer(SszModel): + items: Annotated[List[Inner], ssz_vector(3)] + + zero = ssz_default(Outer) + assert len(zero.items) == 3 + zero.items[0].x = Uint64(99) + # Mutating one slot must not bleed into its siblings. + assert int(zero.items[1].x) == 0 + assert int(zero.items[2].x) == 0 + + +def test_forked_model_json_omission_unchanged() -> None: + """The JSON leg keeps today's exclude_none single-model behavior.""" + dumped = _shanghai_payload().model_dump( + mode="json", by_alias=True, exclude_none=True + ) + assert "withdrawals" in dumped + assert "blobGasUsed" not in dumped # pre-Cancun: key simply absent + + +def test_forked_model_decode_fills_none() -> None: + """Decoding an older fork's bytes restores the one model with None.""" + shanghai = _shanghai_payload() + raw = encode(shanghai, fork="Shanghai") + restored = decode(ForkedPayload, raw, fork="Shanghai") + assert restored == shanghai + assert restored.blob_gas_used is None # beyond-fork field stays None + assert restored.withdrawals is not None + + +def test_fork_scoped_nested_in_complete_model_raises() -> None: + """ + A COMPLETE model cannot carry a fork-scoped one. + + Without a fork at the outer encode there is nothing to propagate to + the nested container, so the encode must refuse rather than pick a + schema silently. (Fork-scoped outer models propagate their fork; see + test_fork_propagates_to_nested_containers.) + """ + + class Wrapper(SszModel): + payload: ForkedPayload + + wrapper = Wrapper(payload=_shanghai_payload()) + with pytest.raises(TypeError, match="fork-scoped"): + encode(wrapper) + + +def test_fork_propagates_to_nested_containers() -> None: + """ + One fork projects the whole value tree (envelope contains payload). + + This is the general #793 shape: fork-evolving containers nest other + fork-evolving containers, and everything inside one message is at + the same chain fork. The outer fork= selects every nested projection. + """ + + class Envelope(SszModel): + payload: ForkedPayload + blob_count: Uint64 | None = None # Shanghai-era envelope field + + __ssz_schema__ = SszForkSchema( + base_fork="Paris", + base=("payload",), + appended={"Shanghai": ("blob_count",)}, + ) + + class RefEnvelopeShanghai(Container): + payload: RefForkedShanghai + blob_count: uint64 + + envelope = Envelope(payload=_shanghai_payload(), blob_count=3) + ref = RefEnvelopeShanghai( + payload=RefForkedShanghai( + parent_hash=b"\xaa" * 32, + block_number=100, + transactions=[b"\x02\xf8"], + withdrawals=[_ref_withdrawal()], + ), + blob_count=3, + ) + assert_matches_reference(envelope, ref, fork="Shanghai") + # decode restores both levels, beyond-fork fields None at each level + restored = decode(Envelope, encode(envelope, "Shanghai"), fork="Shanghai") + assert restored.payload.blob_gas_used is None + # a nested payload that does not fit the propagated fork still raises + paris_inside = Envelope(payload=_paris_payload(), blob_count=1) + with pytest.raises(TypeError, match="missing=\\['withdrawals'\\]"): + encode(paris_inside, fork="Shanghai") + + +def test_forked_model_describe_schema_per_fork() -> None: + """describe_schema renders each fork's projection in SSZ order.""" + paris = describe_schema(ForkedPayload, fork="Paris") + cancun = describe_schema(ForkedPayload, fork="Cancun") + assert "blob_gas_used" not in paris + assert cancun.splitlines()[-1].strip() == "blob_gas_used: uint64" + # SSZ order comes from the schema tuples, not the class body: the + # model declares blob_gas_used second, but it encodes LAST. + assert cancun.splitlines()[1].strip() == "parent_hash: ByteVector[32]" + + +def _bad_vector_marker_on_scalar() -> None: + class Bad(SszModel): + seats: Annotated[Uint64, ssz_vector(3)] # not a list + + +def _bad_byte_list_on_int() -> None: + class Bad(SszModel): + data: Annotated[Uint64, byte_list(8)] # not Bytes + + +def _bad_bit_marker_on_ints() -> None: + class Bad(SszModel): + flags: Annotated[List[Uint64], bitlist(8)] # not list[bool] + + +def _bad_raw_ssz_type_marker() -> None: + class Bad(SszModel): + x: Annotated[Uint64, SszUint(32)] # raw SszType, not a helper + + +def _bad_unmapped_str() -> None: + class Bad(SszModel): + s: str # no SSZ mapping and not excluded + + +def _bad_bare_bytes() -> None: + class Bad(SszModel): + data: Bytes # variable bytes need a byte_list cap + + +def _bad_bare_list() -> None: + class Bad(SszModel): + items: List[Uint64] # lists need a cap/length marker + + +def _bad_multi_arm_union() -> None: + class Bad(SszModel): + x: Uint64 | Uint8 | None = None # only T | None supported + + +def _bad_optional_without_schema() -> None: + class Bad(SszModel): + a: Uint64 + b: Uint64 | None = None # optional but no schema + + +def _bad_schema_field_typo() -> None: + class Bad(SszModel): + a: Uint64 + b: Uint64 | None = None + + __ssz_schema__ = SszForkSchema( + base_fork="Paris", + base=("a",), + appended={"Shanghai": ("typo",)}, + ) + + +def _bad_required_appended() -> None: + class Bad(SszModel): + a: Uint64 + b: Uint64 # appended but not optional + + __ssz_schema__ = SszForkSchema( + base_fork="Paris", + base=("a",), + appended={"Shanghai": ("b",)}, + ) + + +def _bad_optional_base() -> None: + class Bad(SszModel): + a: Uint64 + b: Uint64 | None = None # optional but declared in base + + __ssz_schema__ = SszForkSchema( + base_fork="Paris", + base=("a", "b"), + appended={}, + ) + + +def _bad_appended_no_default() -> None: + class Bad(SszModel): + a: Uint64 + b: Uint64 | None # optional type but NO None default + + __ssz_schema__ = SszForkSchema( + base_fork="Paris", + base=("a",), + appended={"Shanghai": ("b",)}, + ) + + +def _bad_duplicate_schema_names() -> None: + class Bad(SszModel): + a: Uint64 + b: Uint64 | None = None + + __ssz_schema__ = SszForkSchema( + base_fork="Paris", + base=("a", "a"), + appended={"Shanghai": ("b",)}, + ) + + +def _bad_required_excluded() -> None: + class Bad(SszModel): + a: Uint64 + note: Annotated[str, ssz_exclude()] # excluded but required + + +def _bad_required_comment_excluded() -> None: + # The class name must be unique file-wide: on Python < 3.13, + # inspect.getsource finds a class by name, so a second "Bad" would + # scan the first one's body. + class BadCommentRequired(SszModel): + a: Uint64 + # ssz_exclude + note: str + + +def _bad_progressive_with_schema() -> None: + class Bad(ProgressiveModel): + a: Uint64 + + __ssz_schema__ = SszForkSchema( + base_fork="Paris", base=("a",), appended={} + ) + + +def _bad_progressive_with_optional() -> None: + class Bad(ProgressiveModel): + a: Uint64 + b: Uint64 | None = None + + +def _bad_progressive_active_count() -> None: + class Bad(ProgressiveModel): + __active_fields__ = [1, 1] # two active, three declared fields + + a: Uint64 + b: Uint64 + c: Uint64 + + +BAD_DECLARATIONS: List[Tuple[str, Callable[[], None], str]] = [ + ("vector-on-scalar", _bad_vector_marker_on_scalar, "requires a list"), + ("byte-list-on-int", _bad_byte_list_on_int, "byte_list requires"), + ("bits-on-ints", _bad_bit_marker_on_ints, "list\\[bool\\]"), + ("raw-marker", _bad_raw_ssz_type_marker, "unsupported Annotated"), + ("unmapped-str", _bad_unmapped_str, "no SSZ type"), + ("bare-bytes", _bad_bare_bytes, "no SSZ type"), + ("bare-list", _bad_bare_list, "no SSZ type"), + ("multi-arm-union", _bad_multi_arm_union, "only T \\| None"), + ("optional-no-schema", _bad_optional_without_schema, "no __ssz_schema__"), + ("schema-typo", _bad_schema_field_typo, "does not match the model"), + ("required-appended", _bad_required_appended, "must be T \\| None"), + ("optional-base", _bad_optional_base, "optional base"), + ("appended-no-default", _bad_appended_no_default, "default to None"), + ("dup-schema-names", _bad_duplicate_schema_names, "more than once"), + ("required-excluded", _bad_required_excluded, "no default"), + ( + "required-comment-excluded", + _bad_required_comment_excluded, + "no default", + ), + ("progressive-schema", _bad_progressive_with_schema, "not supported"), + ("progressive-optional", _bad_progressive_with_optional, "not supported"), + ("progressive-count", _bad_progressive_active_count, "active"), +] + + +@pytest.mark.parametrize( + "define,match", + [pytest.param(fn, match, id=name) for name, fn, match in BAD_DECLARATIONS], +) +def test_bad_declaration_fails_at_import( + define: Callable[[], None], match: str +) -> None: + """Every mis-declared container fails at class definition, named.""" + with pytest.raises(TypeError, match=match): + define() + + +STRICTNESS: List[Tuple[str, Optional[str], str]] = [ + ("bare-encode", None, "fork-scoped"), + ("older-fork", "Paris", "unexpected=\\['withdrawals'\\]"), + ("newer-fork", "Cancun", "missing=\\['blob_gas_used'\\]"), + ("unknown-fork", "Osaka", "unknown fork"), +] + + +@pytest.mark.parametrize( + "fork,match", + [pytest.param(f, m, id=name) for name, f, m in STRICTNESS], +) +def test_forked_model_strictness(fork: Optional[str], match: str) -> None: + """A Shanghai payload only encodes under the Shanghai schema.""" + with pytest.raises(TypeError, match=match): + encode(_shanghai_payload(), fork=fork) + + +NOT_FORK_SCOPED: List[Tuple[str, Callable[[], object]]] = [ + ("encode", lambda: encode(_withdrawal(), fork="Paris")), + ("decode", lambda: decode(Withdrawal, b"", fork="Paris")), + ("describe", lambda: describe_schema(Withdrawal, fork="Paris")), + ("default", lambda: ssz_default(Withdrawal, "Paris")), + ("fields", lambda: ssz_fields(Withdrawal, "Paris")), +] + + +@pytest.mark.parametrize( + "call", + [pytest.param(c, id=name) for name, c in NOT_FORK_SCOPED], +) +def test_fork_on_complete_model_raises(call: Callable[[], object]) -> None: + """Passing fork= to a non-fork-scoped model raises on every path.""" + with pytest.raises(TypeError, match="is not fork-scoped"): + call() + + +def test_ssz_default_per_fork() -> None: + """ssz_default(fork) zeroes that fork's fields, leaves the rest None.""" + zero = ssz_default(ForkedPayload, "Shanghai") + assert zero.withdrawals == [] + assert zero.blob_gas_used is None # beyond Shanghai: absent, not zero + assert encode(zero, "Shanghai") == RefForkedShanghai().encode_bytes() + with pytest.raises(TypeError, match="fork-scoped"): + ssz_default(ForkedPayload) # bare default: must name the fork + + +@pytest.mark.parametrize("mutation", ["truncate", "extend"]) +def test_decode_of_malformed_bytes_raises(mutation: str) -> None: + """Truncated or oversized SSZ data raises, never mis-decodes.""" + raw = encode(_withdrawal()) + data = raw[:-1] if mutation == "truncate" else raw + b"\x00" + with pytest.raises(ValueError): + decode(Withdrawal, data) + + +def test_decode_under_wrong_fork_does_not_silently_succeed() -> None: + """Shanghai bytes decoded as Cancun raise (schema sizes differ).""" + raw = encode(_shanghai_payload(), fork="Shanghai") + with pytest.raises(Exception): # noqa: B017 - remerkleable's error + decode(ForkedPayload, raw, fork="Cancun") + + +def test_build_ssz_type_cache_identity() -> None: + """One cache entry per (class, fork); distinct classes never share.""" + assert build_ssz_type(Withdrawal) is build_ssz_type(Withdrawal, None) + assert build_ssz_type(ForkedPayload, "Paris") is build_ssz_type( + ForkedPayload, "Paris" + ) + assert build_ssz_type(ForkedPayload, "Paris") is not build_ssz_type( + ForkedPayload, "Shanghai" + ) + + def make_dup() -> type: + class Dup(SszModel): + a: Uint64 + + return Dup + + first, second = make_dup(), make_dup() + assert first is not second + assert build_ssz_type(first) is not build_ssz_type(second) + + +UINT_WIDTHS = [ + (Uint8, 8), + (Uint16, 16), + (Uint32, 32), + (Uint64, 64), + (Uint128, 128), + (Uint256, 256), +] + + +@pytest.mark.parametrize( + "uint_cls,bits", + [pytest.param(c, b, id=c.__name__) for c, b in UINT_WIDTHS], +) +def test_uint_width_checked_at_construction(uint_cls: type, bits: int) -> None: + """A wrong-width value fails when built, not at first encode.""" + assert int(uint_cls((1 << bits) - 1)) == (1 << bits) - 1 + with pytest.raises(ValueError, match="out of range"): + uint_cls(1 << bits) + with pytest.raises(ValueError, match="out of range"): + uint_cls(-1) + + +def test_uint_width_checked_at_model_parse() -> None: + """Pydantic parsing of an overflowing value fails loudly.""" + with pytest.raises(ValidationError): + Withdrawal( + index=1, + validator_index=2, + address=Address(b"\x00" * 20), + amount=1 << 64, # one past uint64 + ) + + +def test_excluded_field_is_json_only() -> None: + """ssz_exclude()d fields exist in JSON but are invisible to SSZ.""" + value = Mixed(a=7, note="kept in JSON") + assert "note" in value.model_dump(mode="json") + assert ssz_fields(Mixed) == ("a",) + # decode cannot see the field; it comes back as the default + restored = decode(Mixed, encode(value)) + assert int(restored.a) == 7 + assert restored.note == "json-only" + + +def test_excluded_field_on_fork_scoped_model() -> None: + """Exclusion composes with __ssz_schema__ (schema skips the field).""" + + class ForkedMixed(SszModel): + a: Uint64 + b: Uint64 | None = None + note: Annotated[str, ssz_exclude()] = "aux" + + __ssz_schema__ = SszForkSchema( + base_fork="One", + base=("a",), + appended={"Two": ("b",)}, + ) + + value = ForkedMixed(a=1, note="ride-along") + assert ssz_fields(ForkedMixed, "One") == ("a",) + restored = decode(ForkedMixed, encode(value, "One"), "One") + assert restored.b is None + assert restored.note == "aux" + + +def test_comment_excluded_field_is_json_only() -> None: + """A # ssz_exclude comment above a field hides it from SSZ.""" + value = CommentMixed(a=7, note="kept in JSON") + assert "note" in value.model_dump(mode="json") + assert ssz_fields(CommentMixed) == ("a",) + restored = decode(CommentMixed, encode(value)) + assert int(restored.a) == 7 + assert restored.note == "json-only" + + +def test_trailing_comment_excludes_field() -> None: + """The marker also works as a trailing comment on the field.""" + + class TrailingMixed(SszModel): + a: Uint64 + note: str = "aux" # ssz_exclude + + assert ssz_fields(TrailingMixed) == ("a",) + + +def test_comment_exclusion_is_inherited() -> None: + """A subclass keeps the base's comment-marked exclusions.""" + + class CommentChild(CommentMixed): + b: Uint64 + + assert ssz_fields(CommentChild) == ("a", "b") + + +def test_comment_must_be_directly_above() -> None: + """A blank line detaches the marker; the field must then be SSZ.""" + with pytest.raises(TypeError, match="no SSZ type"): + + class DetachedComment(SszModel): + a: Uint64 + # ssz_exclude + + note: str = "x" + + +def test_spec_of_rejects_excluded_field() -> None: + """spec_of refuses excluded fields instead of resolving the type.""" + with pytest.raises(TypeError, match="SSZ-excluded"): + spec_of(CommentMixed, "note") + with pytest.raises(TypeError, match="SSZ-excluded"): + spec_of(Mixed, "note") + + +def test_single_fork_schema_works_end_to_end() -> None: + """A schema with no appended forks is valid and encodable.""" + + class OnlyFork(SszModel): + a: Uint64 + + __ssz_schema__ = SszForkSchema( + base_fork="Only", base=("a",), appended={} + ) + + value = OnlyFork(a=5) + assert ssz_fields(OnlyFork, "Only") == ("a",) + assert decode(OnlyFork, encode(value, "Only"), "Only") == value diff --git a/packages/testing/src/execution_testing/tools/ssz_vectors.py b/packages/testing/src/execution_testing/tools/ssz_vectors.py new file mode 100644 index 00000000000..2bc26b7a9dc --- /dev/null +++ b/packages/testing/src/execution_testing/tools/ssz_vectors.py @@ -0,0 +1,476 @@ +""" +SSZ static-vector generation on top of the base_types SSZ engine. + +Suites mirror consensus-specs exactly, one per RandomizationMode plus a chaos +suite (ssz_random, ssz_zero, ssz_max, ssz_nil, ssz_one, ssz_lengthy, +ssz_random_chaos). Mode semantics are: +zero/max pin scalar CONTENT (0 / all-ones) but keep collections short +(1-byte byte-lists), while emptiness and saturation are their own modes +(nil_count / max_count). Changing modes (random / one_count / max_count / +chaos) yield several cases; the rest are fully determined by one. +""" + +import hashlib +import random +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import ( + Any, + Dict, + Iterator, + List, + Optional, + Sequence, + Tuple, + Type, + TypeVar, + Union, +) + +import yaml + +from execution_testing.base_types.ssz import ( + SszBitlist, + SszBitvector, + SszBool, + SszByteList, + SszByteVector, + SszContainer, + SszList, + SszModel, + SszProgressiveBitlist, + SszProgressiveContainer, + SszProgressiveList, + SszType, + SszUint, + SszVector, + encode, + hash_tree_root, + spec_of, + ssz_fields, +) + +MAX_LIST_LENGTH = 10 +MAX_BYTES_LENGTH = 1000 + +RANDOM_CASE_COUNT = 30 + +_M = TypeVar("_M", bound=SszModel) + +_MODE_NAMES = ( + "random", + "zero", + "max", + "nil", + "one", + "lengthy", +) + + +class RandomizationMode(Enum): + """ + How a value's scalar and collection fields are filled. + """ + + mode_random = 0 + mode_zero = 1 + mode_max = 2 + mode_nil_count = 3 + mode_one_count = 4 + mode_max_count = 5 + + def is_changing(self) -> bool: + """ + Return whether the mode yields varying values across cases. + + True for random, one_count and max_count -- those randomize content, + so several cases are worth generating; the rest are fully determined + by a single case. + """ + return self in ( + RandomizationMode.mode_random, + RandomizationMode.mode_one_count, + RandomizationMode.mode_max_count, + ) + + def to_name(self) -> str: + """Return the canonical short name for this mode.""" + return _MODE_NAMES[self.value] + + +def deterministic_seed(*parts: object) -> int: + """ + Return a stable integer seed derived from parts. + + Uses SHA-256 over the slash-joined string parts. + """ + joined = "/".join(str(part) for part in parts) + digest = hashlib.sha256(joined.encode("utf-8")).digest() + return int.from_bytes(digest, "big") + + +@dataclass(frozen=True) +class VectorCase: + """One ssz_static case: the value, its SSZ bytes, and its root.""" + + value: Any # value.yaml + serialized: bytes # serialized.ssz + root: bytes # roots.yaml + + +def _random_bytes(rng: random.Random, length: int) -> bytes: + return bytes(rng.getrandbits(8) for _ in range(length)) + + +def _bits(rng: random.Random, length: int, mode: RandomizationMode) -> Any: + if mode == RandomizationMode.mode_zero: + return [False] * length + if mode == RandomizationMode.mode_max: + return [True] * length + return [bool(rng.getrandbits(1)) for _ in range(length)] + + +def _bitlist_length( + rng: random.Random, cap: int, mode: RandomizationMode +) -> int: + # Consensus semantics: only the *count* modes pin the length; + # zero/max keep a random length. + if mode == RandomizationMode.mode_nil_count: + return 0 + if mode == RandomizationMode.mode_one_count: + return min(1, cap) + if mode == RandomizationMode.mode_max_count: + return cap + return rng.randint(0, cap) + + +def random_value( + rng: random.Random, + spec: SszType, + mode: RandomizationMode, + *, + max_bytes_length: int = MAX_BYTES_LENGTH, + max_list_length: int = MAX_LIST_LENGTH, + chaos: bool = False, +) -> Any: + """ + Build a pydantic value of spec filled with random data per mode. + + A port of the consensus-specs get_random_ssz_object, branching on the + engine's SszType descriptors. With chaos, the mode is re-drawn at every + level of the value tree. + """ + if chaos: + mode = rng.choice(list(RandomizationMode)) + if isinstance(spec, SszByteList): + if mode == RandomizationMode.mode_nil_count: + return b"" + if mode == RandomizationMode.mode_max_count: + return _random_bytes(rng, min(max_bytes_length, spec.limit)) + if mode == RandomizationMode.mode_one_count: + return _random_bytes(rng, min(1, spec.limit)) + if mode == RandomizationMode.mode_zero: + return b"\x00" * min(1, spec.limit) + if mode == RandomizationMode.mode_max: + return b"\xff" * min(1, spec.limit) + return _random_bytes( + rng, rng.randint(0, min(max_bytes_length, spec.limit)) + ) + if isinstance(spec, SszByteVector): + # Byte vectors are fixed length; no max-bytes cap applies. + if mode == RandomizationMode.mode_zero: + return b"\x00" * spec.length + if mode == RandomizationMode.mode_max: + return b"\xff" * spec.length + return _random_bytes(rng, spec.length) + if isinstance(spec, SszUint): + if mode == RandomizationMode.mode_zero: + return 0 + if mode == RandomizationMode.mode_max: + return (1 << spec.bits) - 1 + return rng.randint(0, (1 << spec.bits) - 1) + if isinstance(spec, SszBool): + if mode == RandomizationMode.mode_zero: + return False + if mode == RandomizationMode.mode_max: + return True + return bool(rng.getrandbits(1)) + if isinstance(spec, SszBitvector): + # Bit vectors are fixed length; no cap applies. + return _bits(rng, spec.length, mode) + if isinstance(spec, SszBitlist): + # Consensus caps bit lists by the LIST cap, not the byte cap. + cap = min(max_list_length, spec.limit) + length = _bitlist_length(rng, cap, mode) + return _bits(rng, length, mode) + if isinstance(spec, SszProgressiveBitlist): + # Progressive bit lists are uncapped; the list cap bounds them. + length = _bitlist_length(rng, max_list_length, mode) + return _bits(rng, length, mode) + if isinstance(spec, (SszList, SszProgressiveList)): + # Progressive lists are uncapped; the list cap bounds them. + limit = max_list_length + if isinstance(spec, SszList) and spec.limit < limit: + limit = spec.limit + length = rng.randint(0, limit) + if mode == RandomizationMode.mode_one_count: + length = 1 + elif mode == RandomizationMode.mode_max_count: + length = limit + elif mode == RandomizationMode.mode_nil_count: + length = 0 + # Shrink the cap for nested collections, as consensus-specs does. + max_list_length = 1 << (max_list_length.bit_length() >> 1) + return [ + random_value( + rng, + spec.element, + mode, + max_bytes_length=max_bytes_length, + max_list_length=max_list_length, + chaos=chaos, + ) + for _ in range(length) + ] + if isinstance(spec, SszVector): + return [ + random_value( + rng, + spec.element, + mode, + max_bytes_length=max_bytes_length, + max_list_length=max_list_length, + chaos=chaos, + ) + for _ in range(spec.length) + ] + if isinstance(spec, (SszContainer, SszProgressiveContainer)): + return random_model( + rng, + spec.model, + mode, + max_bytes_length=max_bytes_length, + max_list_length=max_list_length, + chaos=chaos, + ) + raise TypeError(f"no random value for SSZ type {spec!r}") + + +def random_model( + rng: random.Random, + model_cls: Type[_M], + mode: RandomizationMode, + *, + fork: Optional[str] = None, + max_bytes_length: int = MAX_BYTES_LENGTH, + max_list_length: int = MAX_LIST_LENGTH, + chaos: bool = False, +) -> _M: + """ + Build a model_cls instance filled with random data per mode. + + For a fork-scoped model, fork selects which fields get values; + fields beyond that fork keep their None default. + """ + return model_cls( + **{ + name: random_value( + rng, + spec_of(model_cls, name), + mode, + max_bytes_length=max_bytes_length, + max_list_length=max_list_length, + chaos=chaos, + ) + for name in ssz_fields(model_cls, fork) + } + ) + + +def make_case(model: SszModel, fork: Optional[str] = None) -> VectorCase: + """Turn a model instance into its ssz_static case triple.""" + return VectorCase( + value=model.model_dump(mode="json", exclude_none=True), + serialized=encode(model, fork), + root=hash_tree_root(model, fork), + ) + + +def suite_name(mode: RandomizationMode, chaos: bool = False) -> str: + """Return the consensus suite name for a mode (ssz_random, ...).""" + return f"ssz_{mode.to_name()}" + ("_chaos" if chaos else "") + + +def suite_plan( + count: int = RANDOM_CASE_COUNT, +) -> List[Tuple[str, RandomizationMode, bool, int]]: + """ + Return every suite as (name, mode, chaos, case_count). + + One suite per RandomizationMode plus ssz_random_chaos; changing modes + get count cases, deterministic ones a single case. + """ + plan = [ + ( + suite_name(mode), + mode, + False, + count if mode.is_changing() else 1, + ) + for mode in RandomizationMode + ] + plan.append( + ( + suite_name(RandomizationMode.mode_random, chaos=True), + RandomizationMode.mode_random, + True, + count, + ) + ) + return plan + + +ModelSpec = Union[Type[SszModel], Tuple[Type[SszModel], str]] + + +def _normalize_models( + models: Sequence[ModelSpec], +) -> List[Tuple[Type[SszModel], Optional[str]]]: + """Normalize entries to (model, fork) and reject output collisions.""" + entries: List[Tuple[Type[SszModel], Optional[str]]] = [ + m if isinstance(m, tuple) else (m, None) for m in models + ] + seen: Dict[Tuple[str, Optional[str]], Type[SszModel]] = {} + for model_cls, fork in entries: + key = (model_cls.__name__, fork) + other = seen.setdefault(key, model_cls) + if other is not model_cls: + raise ValueError( + f"two distinct models would share vector output " + f"{key[0]!r} (fork={fork!r}); rename one" + ) + return entries + + +def generate_cases( + models: Sequence[ModelSpec], + *, + count: int = RANDOM_CASE_COUNT, + max_bytes_length: int = MAX_BYTES_LENGTH, + max_list_length: int = MAX_LIST_LENGTH, +) -> Iterator[Tuple[str, Optional[str], str, int, VectorCase]]: + """ + Yield (container_name, fork, suite, case_index, case) per vector. + + Entries are complete models or (fork-scoped model, fork) pairs. The + RNG is seeded per (container, [fork,] suite, index) so output is + fully deterministic across runs. + """ + for model_cls, fork in _normalize_models(models): + name = model_cls.__name__ + seed_head = (name, fork) if fork else (name,) + for suite, mode, chaos, n in suite_plan(count): + for i in range(n): + rng = random.Random(deterministic_seed(*seed_head, suite, i)) + model = random_model( + rng, + model_cls, + mode, + fork=fork, + max_bytes_length=max_bytes_length, + max_list_length=max_list_length, + chaos=chaos, + ) + yield name, fork, suite, i, make_case(model, fork) + + +class _HexQuotingDumper(yaml.SafeDumper): + """SafeDumper that single-quotes 0x-hex strings (see _yaml_dump).""" + + +def _represent_str(dumper: Any, data: str) -> Any: + style = "'" if data.startswith("0x") else None + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style=style) + + +_HexQuotingDumper.add_representer(str, _represent_str) + + +def _yaml_dump(obj: Any) -> bytes: + """ + Dump YAML with 0x-hex strings explicitly single-quoted. + + PyYAML's emitter is not consistent across interpreters about quoting + strings that look like YAML 1.1 ints (PyPy emits root: 0x... bare, which + a loader would read back as an integer). Consensus vectors always quote + them, so force the style instead of trusting the emitter. + """ + return yaml.dump(obj, Dumper=_HexQuotingDumper, sort_keys=False).encode() + + +def case_files(case: VectorCase) -> Dict[str, bytes]: + """The on-disk files for a case (bytes), mirroring the consensus layout.""" + return { + "value.yaml": _yaml_dump(case.value), + "serialized.ssz": case.serialized, + "roots.yaml": _yaml_dump({"root": "0x" + case.root.hex()}), + } + + +def case_dir( + output_dir: Path, + container_name: str, + suite: str, + case_index: int, + fork: Optional[str] = None, +) -> Path: + """Return the per-case output directory for a given case.""" + base = output_dir / container_name + if fork is not None: + base = base / fork + return base / suite / f"case_{case_index}" + + +def write_case(directory: Path, case: VectorCase) -> None: + """Write a case's value.yaml / serialized.ssz / roots.yaml.""" + directory.mkdir(parents=True, exist_ok=True) + for name, data in case_files(case).items(): + (directory / name).write_bytes(data) + + +def write_vectors( + models: Sequence[ModelSpec], + output_dir: Path, + *, + count: int = RANDOM_CASE_COUNT, +) -> int: + """Write every vector case under output_dir; return the count.""" + written = 0 + for name, fork, suite, case_index, case in generate_cases( + models, count=count + ): + write_case(case_dir(output_dir, name, suite, case_index, fork), case) + written += 1 + return written + + +__all__ = [ + "MAX_BYTES_LENGTH", + "MAX_LIST_LENGTH", + "RANDOM_CASE_COUNT", + "ModelSpec", + "RandomizationMode", + "VectorCase", + "case_dir", + "case_files", + "deterministic_seed", + "generate_cases", + "make_case", + "random_model", + "random_value", + "suite_name", + "suite_plan", + "write_case", + "write_vectors", +] diff --git a/packages/testing/src/execution_testing/tools/tests/test_ssz_vectors.py b/packages/testing/src/execution_testing/tools/tests/test_ssz_vectors.py new file mode 100644 index 00000000000..fa07717bede --- /dev/null +++ b/packages/testing/src/execution_testing/tools/tests/test_ssz_vectors.py @@ -0,0 +1,296 @@ +""" +Tests for SSZ static-vector generation (consensus-specs-style suites). + +Every generated case must be internally consistent with the engine (the +ground truth for bytes/roots), round-trip losslessly, and match pinned +known-answer values; the suites and mode semantics mirror consensus-specs' +ssz_static generator. +""" + +import random +from pathlib import Path +from typing import Annotated, List + +import pytest +import yaml + +from execution_testing.base_types import Address, Bytes, Hash +from execution_testing.base_types.ssz import ( + SszForkSchema, + SszModel, + Uint64, + Uint256, + byte_list, + decode, + encode, + hash_tree_root, + spec_of, + ssz_list, +) +from execution_testing.tools.ssz_vectors import ( + RandomizationMode, + case_files, + deterministic_seed, + generate_cases, + make_case, + random_model, + random_value, + suite_plan, + write_vectors, +) + +MAX_TX = 2**20 +MAX_BYTES_PER_TX = 2**30 +MAX_WITHDRAWALS = 16 + + +class Withdrawal(SszModel): + """A withdrawal container.""" + + index: Uint64 + validator_index: Uint64 + address: Address + amount: Uint64 + + +class Payload(SszModel): + """A container with a byte-list, a capped list, and a nested list.""" + + parent_hash: Hash + base_fee_per_gas: Uint256 + extra_data: Annotated[Bytes, byte_list(32)] + transactions: Annotated[ + List[Annotated[Bytes, byte_list(MAX_BYTES_PER_TX)]], + ssz_list(MAX_TX), + ] + withdrawals: Annotated[List[Withdrawal], ssz_list(MAX_WITHDRAWALS)] + + +class ForkedPayload(SszModel): + """A fork-scoped model, for the generator's fork axis.""" + + parent_hash: Hash + block_number: Uint64 + withdrawals: ( + Annotated[List[Withdrawal], ssz_list(MAX_WITHDRAWALS)] | None + ) = None + + __ssz_schema__ = SszForkSchema( + base_fork="Paris", + base=("parent_hash", "block_number"), + appended={"Shanghai": ("withdrawals",)}, + ) + + +def assert_roundtrip(model: SszModel) -> None: + """Reusable harness: encode -> decode reconstructs the SSZ value.""" + restored = decode(type(model), encode(model)) + assert encode(restored) == encode(model) + assert hash_tree_root(restored) == hash_tree_root(model) + + +def test_case_triple_is_consistent() -> None: + """A case's serialized/root match the engine, and the value round-trips.""" + w = Withdrawal( + index=7, + validator_index=42, + address=Address(b"\x11" * 20), + amount=32_000_000_000, + ) + case = make_case(w) + assert case.serialized == encode(w) + assert case.root == hash_tree_root(w) + assert case.value["amount"] == "0x773594000" + assert_roundtrip(w) + + +def test_suite_plan_mirrors_consensus() -> None: + """The published CL suite names; changing modes get several cases.""" + plan = suite_plan(count=30) + names = [name for name, *_rest in plan] + assert names == [ + "ssz_random", + "ssz_zero", + "ssz_max", + "ssz_nil", + "ssz_one", + "ssz_lengthy", + "ssz_random_chaos", + ] + counts = {name: n for name, _m, _c, n in plan} + # consensus-specs: cases_if_random if chaos or is_changing() else 1 + assert counts["ssz_random"] == 30 + assert counts["ssz_one"] == 30 + assert counts["ssz_lengthy"] == 30 + assert counts["ssz_random_chaos"] == 30 + assert counts["ssz_zero"] == 1 + assert counts["ssz_max"] == 1 + assert counts["ssz_nil"] == 1 + + +CONTENT_MODES = [ + ("zero", RandomizationMode.mode_zero, 0, b"\x00"), + ("max", RandomizationMode.mode_max, 2**256 - 1, b"\xff"), +] + + +@pytest.mark.parametrize( + "mode,fee,fill", + [pytest.param(m, v, b, id=name) for name, m, v, b in CONTENT_MODES], +) +def test_content_modes_pin_values_not_lengths( + mode: RandomizationMode, fee: int, fill: bytes +) -> None: + """zero/max pin scalar CONTENT; collections stay short (1 byte).""" + rng = random.Random(0) + model = random_model(rng, Payload, mode) + assert int(model.base_fee_per_gas) == fee + assert bytes(model.parent_hash) == fill * 32 + # consensus semantics: byte-lists get ONE fill byte, not emptiness + assert bytes(model.extra_data) == fill + + +COUNT_MODES = [ + ("nil", RandomizationMode.mode_nil_count, 0), + ("one", RandomizationMode.mode_one_count, 1), + ("lengthy", RandomizationMode.mode_max_count, 10), +] + + +@pytest.mark.parametrize( + "mode,length", + [pytest.param(m, n, id=name) for name, m, n in COUNT_MODES], +) +def test_count_modes_pin_lengths(mode: RandomizationMode, length: int) -> None: + """nil/one/lengthy pin list LENGTHS (up to the generator cap of 10).""" + rng = random.Random(0) + model = random_model(rng, Payload, mode) + assert len(model.transactions) == length + assert len(model.withdrawals) == length + + +def test_generate_cases_covers_models_and_suites() -> None: + """Every model x suite is emitted with the planned case counts.""" + cases = list(generate_cases([Withdrawal, Payload], count=2)) + names = {name for name, _fork, _suite, _i, _c in cases} + assert names == {"Withdrawal", "Payload"} + # 4 changing suites x 2 cases + 3 deterministic suites x 1 = 11 each + assert len(cases) == 2 * 11 + + # Every generated case is internally consistent and round-trips. + for name, _fork, _suite, _i, case in cases: + assert len(case.root) == 32 + model_cls = Withdrawal if name == "Withdrawal" else Payload + assert encode(decode(model_cls, case.serialized)) == case.serialized + + +def test_value_yaml_matches_serialized() -> None: + """ + The written value.yaml re-encodes to the written serialized.ssz. + + This is the contract an ssz_static consumer relies on: value, + serialized bytes, and root must all describe the same object. + """ + for _n, _f, _s, _i, case in generate_cases([Withdrawal], count=2): + files = case_files(case) + value = yaml.safe_load(files["value.yaml"]) + rebuilt = Withdrawal.model_validate(value) + assert encode(rebuilt) == files["serialized.ssz"] + root = yaml.safe_load(files["roots.yaml"])["root"] + assert hash_tree_root(rebuilt).hex() == root.removeprefix("0x") + + +def test_deterministic() -> None: + """Generation is deterministic across runs.""" + a = [c.root for *_h, c in generate_cases([Payload], count=2)] + b = [c.root for *_h, c in generate_cases([Payload], count=2)] + assert a == b + assert deterministic_seed("a", "b", 0) == deterministic_seed("a", "b", 0) + assert deterministic_seed("a", "b", 0) != deterministic_seed("a", "b", 1) + + +def test_chaos_redraws_modes() -> None: + """Chaos re-draws the mode per node yet still builds valid values.""" + rng = random.Random(deterministic_seed("chaos-test")) + for _ in range(5): + model = random_model( + rng, Payload, RandomizationMode.mode_random, chaos=True + ) + assert_roundtrip(model) + + +@pytest.mark.parametrize("name", list(Payload.model_fields)) +def test_random_value_covers_field_spec(name: str) -> None: + """random_value handles every SszType the test containers use.""" + rng = random.Random(1) + value = random_value( + rng, spec_of(Payload, name), RandomizationMode.mode_random + ) + assert value is not None + + +def test_case_files_layout() -> None: + """A case serializes to the consensus value/serialized/roots files.""" + w = Withdrawal( + index=1, + validator_index=2, + address=Address(b"\x00" * 20), + amount=3, + ) + files = case_files(make_case(w)) + assert set(files) == {"value.yaml", "serialized.ssz", "roots.yaml"} + assert files["serialized.ssz"] == encode(w) + # Hex strings must be single-quoted (a bare 0x... reads back as int). + assert b"root: '0x" in files["roots.yaml"] + assert b"amount: '0x3'" in files["value.yaml"] + assert b"address: '0x" in files["value.yaml"] + + +def test_write_vectors_emits_consensus_tree(tmp_path: Path) -> None: + """write_vectors lays out //case_/ triples.""" + written = write_vectors([Withdrawal], tmp_path, count=2) + assert written == 11 # 4 changing x 2 + 3 deterministic x 1 + zero_case = tmp_path / "Withdrawal" / "ssz_zero" / "case_0" + assert (zero_case / "value.yaml").is_file() + assert (zero_case / "serialized.ssz").is_file() + assert (zero_case / "roots.yaml").is_file() + # The serialized bytes reload and re-encode identically via the engine. + raw = (zero_case / "serialized.ssz").read_bytes() + assert encode(decode(Withdrawal, raw)) == raw + # Changing suites have every planned case on disk. + random_dir = tmp_path / "Withdrawal" / "ssz_random" + assert sorted(p.name for p in random_dir.iterdir()) == [ + "case_0", + "case_1", + ] + + +def test_fork_scoped_vectors(tmp_path: Path) -> None: + """(model, fork) entries emit per-fork projections under fork dirs.""" + written = write_vectors( + [(ForkedPayload, "Paris"), (ForkedPayload, "Shanghai")], + tmp_path, + count=1, + ) + assert written == 2 * 7 # all 7 suites x 1 case, per fork entry + paris_zero = tmp_path / "ForkedPayload" / "Paris" / "ssz_zero" / "case_0" + raw = (paris_zero / "serialized.ssz").read_bytes() + restored = decode(ForkedPayload, raw, fork="Paris") + assert restored.withdrawals is None # beyond-fork field absent + shanghai_zero = ( + tmp_path / "ForkedPayload" / "Shanghai" / "ssz_zero" / "case_0" + ) + assert (shanghai_zero / "roots.yaml").is_file() + + +def test_duplicate_vector_targets_rejected(tmp_path: Path) -> None: + """Two distinct same-named models cannot share an output directory.""" + + def make_dup() -> type: + class Withdrawal(SszModel): # same __name__, different class + a: Uint64 + + return Withdrawal + + with pytest.raises(ValueError, match="share vector output"): + write_vectors([Withdrawal, make_dup()], tmp_path, count=1) diff --git a/pyproject.toml b/pyproject.toml index d7bc28a0acc..f4f23245855 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -531,6 +531,11 @@ exclude = [ ] plugins = ["pydantic.mypy"] +[[tool.mypy.overrides]] +# remerkleable ships no type stubs / py.typed marker. +module = "remerkleable.*" +ignore_missing_imports = true + [tool.uv] required-version = ">=0.7.0" extra-build-dependencies = { ethash = ["setuptools", "cmake>=4.2.1,<5"] } diff --git a/uv.lock b/uv.lock index 5020d13995c..14aae6e9ffe 100644 --- a/uv.lock +++ b/uv.lock @@ -819,6 +819,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/eb/db/f8775490669d28aca24871c67dd56b3e72105cb3bcae9a4ec65dd70859b3/eth_hash-0.7.1-py3-none-any.whl", hash = "sha256:0fb1add2adf99ef28883fd6228eb447ef519ea72933535ad1a0b28c6f65f868a", size = 8028, upload-time = "2025-01-13T21:29:19.365Z" }, ] +[[package]] +name = "eth-remerkleable" +version = "0.1.31" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/51/ac/40fde655f67fd02f07b28e3fe4b9bb4af521388ca8aa48462d622ce4fa03/eth_remerkleable-0.1.31.tar.gz", hash = "sha256:94df4b18a50dfc46f55b2e790cfece384e64032e9cb82d185cff09224682ad1d", size = 49525, upload-time = "2026-06-11T13:23:38.552Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/a1/87f7c996f344c5f213c2bca657e579c3696bb99737238c0cf9f04867386a/eth_remerkleable-0.1.31-py3-none-any.whl", hash = "sha256:7e48ea1b80935977effc02300f3f9b1db19153ab87ffbcea14a6e6429bbdc56b", size = 57192, upload-time = "2026-06-11T13:23:37.208Z" }, +] + [[package]] name = "eth-typing" version = "5.2.1" @@ -1098,6 +1107,7 @@ dependencies = [ { name = "coincurve" }, { name = "colorlog" }, { name = "eth-abi" }, + { name = "eth-remerkleable" }, { name = "ethereum-execution" }, { name = "ethereum-hive" }, { name = "ethereum-rlp" }, @@ -1151,6 +1161,7 @@ requires-dist = [ { name = "coincurve", specifier = ">=20.0.0,<21" }, { name = "colorlog", specifier = ">=6.7.0,<7" }, { name = "eth-abi", specifier = ">=5.2.0" }, + { name = "eth-remerkleable", specifier = "==0.1.31" }, { name = "ethereum-execution", editable = "." }, { name = "ethereum-hive", specifier = ">=0.1.0a5,<1.0.0" }, { name = "ethereum-rlp", specifier = ">=0.1.6,<0.2" },