chore(test-tools): file-based t8n stream; replacing in-memory stdout

Replaces Python's stdout-buffered, multi-pass parse of t8n output
(which held up to ~5 copies of the alloc in flight: raw stdout bytes,
parsed dict, re-serialized string, re-parsed dict, validated Alloc)
with a file-based LazyAllocFile that streams {address: account_dict}
entries incrementally via ``ijson`` and validates each ``Account`` one
at a time, preserving every existing Pydantic validator.

This cuts per-test peak RSS on the unchunkified benchmark from
5.38 GB → 3.40 GB (measured, fixtures byte-identical) — ~2 GB of
Python-heap pressure removed per xdist worker.

Author: fselmo

packages/testing/pyproject.toml

@@ -52,6 +52,7 @@ dependencies = [
     "ckzg>=2.1.3,<3",
     "tenacity>=9.0.0,<10",
     "Jinja2>=3,<4",
+    "ijson>=3.3,<4",
 ]
 
 [project.urls]

packages/testing/src/execution_testing/client_clis/cli_types.py

@@ -14,9 +14,11 @@
     TypeVar,
 )
 
+import ijson  # type: ignore[import-untyped]
 from pydantic import Field, PlainSerializer, PlainValidator
 
 from execution_testing.base_types import (
+    Account,
     Bloom,
     Bytes,
     CamelModel,
@@ -458,6 +460,31 @@ def validate(self) -> Alloc:
         return Alloc.model_validate_json(self.raw)
 
 
+class LazyAllocFile(LazyAlloc[Path]):
+    """
+    Lazy allocation backed by a filesystem path.
+
+    Streams `{address: account_or_null}` entries from the file and validates
+    each `Account` incrementally via `Account.model_validate`. Keeps only one
+    in-flight account dict in memory at a time, avoiding the multi-GB peak
+    that `LazyAllocStr` incurs by holding the full JSON string alongside the
+    validated `Alloc`.
+    """
+
+    def validate(self) -> Alloc:
+        """Validate the alloc by streaming entries from the backing file."""
+        accumulated: Dict[str, Account | None] = {}
+        with open(self.raw, "rb") as f:
+            for address_str, account_data in ijson.kvitems(f, ""):
+                if account_data is None:
+                    accumulated[address_str] = None
+                else:
+                    accumulated[address_str] = Account.model_validate(
+                        account_data
+                    )
+        return Alloc.model_validate(accumulated)
+
+
 @dataclass
 class TransitionToolInput:
     """Transition tool input."""
@@ -478,6 +505,10 @@ def to_files(
             alloc_contents = self.alloc.model_dump_json(**model_dump_config)
         elif isinstance(self.alloc, LazyAllocStr):
             alloc_contents = self.alloc.raw
+        elif isinstance(self.alloc, LazyAllocFile):
+            alloc_contents = self.alloc.get().model_dump_json(
+                **model_dump_config
+            )
         else:
             raise Exception(f"Invalid alloc type: {type(self.alloc)}")
 
@@ -513,6 +544,10 @@ def model_dump_json(self, **model_dump_config: Any) -> str:
             alloc_contents = self.alloc.model_dump_json(**model_dump_config)
         elif isinstance(self.alloc, LazyAllocStr):
             alloc_contents = self.alloc.raw
+        elif isinstance(self.alloc, LazyAllocFile):
+            alloc_contents = self.alloc.get().model_dump_json(
+                **model_dump_config
+            )
         else:
             raise Exception(f"Invalid alloc type: {type(self.alloc)}")
 
@@ -547,6 +582,10 @@ def model_dump(self, mode: str, **model_dump_config: Any) -> Any:
             )
         elif isinstance(self.alloc, LazyAllocJson):
             alloc_contents = self.alloc.raw
+        elif isinstance(self.alloc, LazyAllocFile):
+            alloc_contents = self.alloc.get().model_dump(
+                mode=mode, **model_dump_config
+            )
         else:
             raise Exception(f"Invalid alloc type: {type(self.alloc)}")
 
@@ -582,13 +621,19 @@ def model_validate_files(
         """
         Validate the model from the file system where each key is a
         different JSON file.
+
+        `alloc.json` is referenced by path and parsed incrementally on
+        `.get()` via `LazyAllocFile`, so the full file is never held in
+        memory alongside the validated `Alloc`.
         """
-        alloc_data = (directory_path / "alloc.json").read_text()
         result_data = (directory_path / "result.json").read_text()
         result = Result.model_validate_json(
             json_data=result_data, context=context
         )
-        alloc = LazyAllocStr(raw=alloc_data, _state_root=result.state_root)
+        alloc = LazyAllocFile(
+            raw=directory_path / "alloc.json",
+            _state_root=result.state_root,
+        )
         output = cls(result=result, alloc=alloc)
         return output
 

packages/testing/src/execution_testing/client_clis/file_utils.py

@@ -1,6 +1,7 @@
 """Methods to work with the filesystem and json."""
 
 import os
+import shutil
 import stat
 from json import dump
 from pathlib import Path
@@ -9,6 +10,7 @@
 from pydantic import BaseModel, RootModel
 
 from execution_testing.client_clis.cli_types import (
+    LazyAllocFile,
     LazyAllocJson,
     LazyAllocStr,
     TransitionToolInput,
@@ -28,27 +30,29 @@ def dump_files_to_directory(output_path: Path, files: Dict[str, Any]) -> None:
         if rel_path:
             os.makedirs(output_path / rel_path, exist_ok=True)
         file_path = output_path / file_rel_path
-        with open(file_path, "w") as f:
-            if isinstance(file_contents, (LazyAllocStr, LazyAllocJson)):
-                if isinstance(file_contents, LazyAllocJson):
-                    dump(file_contents.raw, f, ensure_ascii=True, indent=4)
-                else:
-                    f.write(file_contents.raw)
-
-            elif isinstance(
-                file_contents, (BaseModel, RootModel, TransitionToolInput)
-            ):
-                f.write(
-                    file_contents.model_dump_json(
-                        indent=4,
-                        exclude_none=True,
-                        by_alias=True,
+        if isinstance(file_contents, LazyAllocFile):
+            shutil.copyfile(file_contents.raw, file_path)
+        else:
+            with open(file_path, "w") as f:
+                if isinstance(file_contents, (LazyAllocStr, LazyAllocJson)):
+                    if isinstance(file_contents, LazyAllocJson):
+                        dump(file_contents.raw, f, ensure_ascii=True, indent=4)
+                    else:
+                        f.write(file_contents.raw)
+                elif isinstance(
+                    file_contents, (BaseModel, RootModel, TransitionToolInput)
+                ):
+                    f.write(
+                        file_contents.model_dump_json(
+                            indent=4,
+                            exclude_none=True,
+                            by_alias=True,
+                        )
                     )
-                )
-            elif isinstance(file_contents, str):
-                f.write(file_contents)
-            else:
-                dump(file_contents, f, ensure_ascii=True, indent=4)
+                elif isinstance(file_contents, str):
+                    f.write(file_contents)
+                else:
+                    dump(file_contents, f, ensure_ascii=True, indent=4)
         if flags:
             file_mode = os.stat(file_path).st_mode
             if "x" in flags:

packages/testing/src/execution_testing/client_clis/tests/test_transition_tool.py

@@ -1,5 +1,6 @@
 """Test the transition tool and subclasses."""
 
+import json
 import shutil
 import subprocess
 from pathlib import Path
@@ -17,10 +18,14 @@
 )
 from execution_testing.client_clis.cli_types import (
     LazyAlloc,
+    LazyAllocFile,
     LazyAllocJson,
     LazyAllocStr,
+    Result,
+    TransitionToolInput,
+    TransitionToolOutput,
 )
-from execution_testing.test_types import Alloc
+from execution_testing.test_types import Alloc, Environment
 
 
 def test_default_tool() -> None:
@@ -128,3 +133,124 @@ def test_lazy_alloc(ty: Type[LazyAlloc], raw: Any) -> None:
     lazy_instance = ty(raw=raw, _state_root=TEST_ALLOC_STATE_ROOT)
     assert lazy_instance.get() == TEST_ALLOC
     assert lazy_instance.state_root() == TEST_ALLOC_STATE_ROOT
+
+
+def test_lazy_alloc_file(tmp_path: Path) -> None:
+    """LazyAllocFile streams the alloc from disk and yields the same Alloc."""
+    alloc_path = tmp_path / "alloc.json"
+    alloc_path.write_text(TEST_ALLOC.model_dump_json())
+    lazy_instance = LazyAllocFile(
+        raw=alloc_path, _state_root=TEST_ALLOC_STATE_ROOT
+    )
+    assert lazy_instance.get() == TEST_ALLOC
+    assert lazy_instance.state_root() == TEST_ALLOC_STATE_ROOT
+
+
+def test_lazy_alloc_file_handles_mixed_entries(tmp_path: Path) -> None:
+    """
+    LazyAllocFile correctly reconstructs an Alloc containing None entries,
+    non-empty storage, long code, and multiple accounts.
+    """
+    alloc = Alloc.model_validate(
+        {
+            0xA: {
+                "balance": 1,
+                "nonce": 2,
+                "code": "0x" + "ab" * 128,
+                "storage": {"0x01": "0x02", "0x03": "0x04"},
+            },
+            0xB: None,
+            0xC: {"balance": "0xff", "nonce": 0, "code": "0x"},
+        }
+    )
+    state_root = alloc.state_root()
+    alloc_path = tmp_path / "alloc.json"
+    alloc_path.write_text(alloc.model_dump_json())
+    lazy_instance = LazyAllocFile(raw=alloc_path, _state_root=state_root)
+    assert lazy_instance.get() == alloc
+    assert lazy_instance.state_root() == state_root
+
+
+def _write_minimal_result(path: Path, state_root: Any) -> None:
+    """Write a minimal valid Result JSON to `path` using `state_root`."""
+    result = Result.model_validate(
+        {
+            "stateRoot": state_root,
+            "txRoot": bytes(32),
+            "receiptsRoot": bytes(32),
+            "logsHash": bytes(32),
+            "logsBloom": bytes(256),
+            "receipts": [],
+            "gasUsed": 0,
+        }
+    )
+    path.write_text(result.model_dump_json(by_alias=True, exclude_none=True))
+
+
+def test_model_validate_files_uses_lazy_alloc_file(tmp_path: Path) -> None:
+    """
+    model_validate_files backs the alloc with the on-disk file, not a
+    multi-GB string in Python memory.
+    """
+    alloc_path = tmp_path / "alloc.json"
+    alloc_path.write_text(TEST_ALLOC.model_dump_json())
+    _write_minimal_result(tmp_path / "result.json", TEST_ALLOC_STATE_ROOT)
+
+    output = TransitionToolOutput.model_validate_files(tmp_path)
+
+    assert isinstance(output.alloc, LazyAllocFile)
+    assert output.alloc.raw == alloc_path
+    assert output.alloc.get() == TEST_ALLOC
+
+
+def test_transition_tool_input_serializes_lazy_alloc_file(
+    tmp_path: Path,
+) -> None:
+    """
+    A `TransitionToolInput` carrying a `LazyAllocFile` (as happens when
+    one t8n invocation's output is chained into the next's input) can be
+    serialized via both `to_files` and `model_dump_json` without pulling
+    the alloc file into memory at `TransitionToolInput` construction.
+    """
+    source_alloc = tmp_path / "source_alloc.json"
+    source_alloc.write_text(TEST_ALLOC.model_dump_json())
+    lazy_alloc = LazyAllocFile(
+        raw=source_alloc, _state_root=TEST_ALLOC_STATE_ROOT
+    )
+
+    input_data = TransitionToolInput(
+        alloc=lazy_alloc, txs=[], env=Environment()
+    )
+
+    out_dir = tmp_path / "out"
+    out_dir.mkdir()
+    paths = input_data.to_files(out_dir, by_alias=True, exclude_none=True)
+    written_alloc = Alloc.model_validate_json(Path(paths["alloc"]).read_text())
+    assert written_alloc == TEST_ALLOC
+
+    dumped = input_data.model_dump_json(by_alias=True, exclude_none=True)
+    parsed = json.loads(dumped)
+    assert Alloc.model_validate(parsed["alloc"]) == TEST_ALLOC
+
+
+def test_dump_files_to_directory_copies_lazy_alloc_file(
+    tmp_path: Path,
+) -> None:
+    """
+    `dump_files_to_directory` copies the backing file when given a
+    `LazyAllocFile`, preserving exact bytes without round-tripping the
+    alloc through Python memory.
+    """
+    from execution_testing.client_clis.file_utils import (
+        dump_files_to_directory,
+    )
+
+    source = tmp_path / "source_alloc.json"
+    source_text = TEST_ALLOC.model_dump_json()
+    source.write_text(source_text)
+    lazy = LazyAllocFile(raw=source, _state_root=TEST_ALLOC_STATE_ROOT)
+
+    dump_dir = tmp_path / "dump"
+    dump_files_to_directory(dump_dir, {"output/alloc.json": lazy})
+
+    assert (dump_dir / "output" / "alloc.json").read_text() == source_text