feat(test-benchmark): support per block opcode count record (#3183)

* feat: support per block opcode count record

* fix(test-clis): Fix eels opcode count

---------

Co-authored-by: Mario Vega <marioevz@gmail.com>

Author: LouisTsai-Csie

packages/testing/src/execution_testing/cli/pytest_commands/plugins/filler/filler.py

@@ -1787,6 +1787,11 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
                     fill_metadata["opcode_count"] = (
                         t8n.opcode_count.model_dump()
                     )
+                if t8n.opcode_count_per_block:
+                    fill_metadata["opcode_count_per_block"] = [
+                        block_opcode_count.model_dump()
+                        for block_opcode_count in t8n.opcode_count_per_block
+                    ]
                 if fill_result.metadata:
                     fill_metadata.update(fill_result.metadata)
 

packages/testing/src/execution_testing/client_clis/cli_types.py

@@ -329,6 +329,7 @@ def print(self) -> None:
 
 _opcode_synonyms = {
     "KECCAK256": "SHA3",
+    "KECCAK": "SHA3",
     "DIFFICULTY": "PREVRANDAO",
 }
 

packages/testing/src/execution_testing/client_clis/client_backend.py

@@ -167,6 +167,7 @@ class ClientBackend:
 
     # t8n-compatibility stubs — fill's filler reads these on the backend.
     opcode_count: OpcodeCount | None = None
+    opcode_count_per_block: List[OpcodeCount] | None = None
     output_cache: Any = None
     debug_dump_dir: Path | None = None
     call_counter: int = 0

packages/testing/src/execution_testing/client_clis/clis/execution_specs.py

@@ -128,6 +128,9 @@ def _evaluate(
                 # TODO: This should be optimized by the t8n tool instead.
                 t8n_args.append("--input.blobParams=stdin")
 
+        if self.supports_opcode_count:
+            t8n_args.append("--opcode.count=stdout")
+
         if self.trace:
             t8n_args.extend(
                 [
@@ -148,6 +151,12 @@ def _evaluate(
         t8n.run()
 
         output_dict = json.loads(out_stream.getvalue())
+
+        if "opcodeCount" in output_dict and "result" in output_dict:
+            output_dict["result"]["opcodeCount"] = output_dict.pop(
+                "opcodeCount"
+            )
+
         output: TransitionToolOutput = TransitionToolOutput.model_validate(
             output_dict, context={"exception_mapper": self.exception_mapper}
         )

packages/testing/src/execution_testing/client_clis/tests/test_execution_specs.py

@@ -184,6 +184,7 @@ def test_evm_t8n(
             ),
         )
         assert to_json(t8n_output.alloc.get()) == expected.get("alloc")
+        t8n_result = to_json(t8n_output.result)
         if isinstance(default_t8n, ExecutionSpecsTransitionTool):
             # The expected output was generated with geth, instead of deleting
             # any info from this expected output, the fields not returned by
@@ -198,9 +199,11 @@ def test_evm_t8n(
                 for i, _ in enumerate(expected.get("result")["receipts"]):
                     del expected.get("result")["receipts"][i][key]
 
-            t8n_result = to_json(t8n_output.result)
             for i, _ in enumerate(expected.get("result")["rejected"]):
                 del expected.get("result")["rejected"][i]["error"]
                 del t8n_result["rejected"][i]["error"]
 
+            if "opcodeCount" in t8n_result:
+                t8n_result.pop("opcodeCount")
+
         assert t8n_result == expected.get("result")

packages/testing/src/execution_testing/client_clis/tests/test_transition_tool.py

@@ -22,6 +22,7 @@
     LazyAllocFile,
     LazyAllocJson,
     LazyAllocStr,
+    OpcodeCount,
     Result,
     TransitionToolInput,
     TransitionToolOutput,
@@ -436,3 +437,49 @@ def test_lazy_alloc_file_empty_object_yields_empty_alloc(
     lazy = LazyAllocFile(raw=alloc_path, _state_root=TEST_ALLOC_STATE_ROOT)
 
     assert lazy.get() == Alloc.model_validate({})
+
+
+def _output_with_opcode_count(counts: dict) -> TransitionToolOutput:
+    """Build a minimal t8n output carrying the given opcode counts."""
+    result = Result.model_validate(
+        {
+            "stateRoot": "0x" + "00" * 32,
+            "txRoot": "0x" + "00" * 32,
+            "receiptsRoot": "0x" + "00" * 32,
+            "logsHash": "0x" + "00" * 32,
+            "logsBloom": "0x" + "00" * 256,
+            "receipts": [],
+            "gasUsed": "0x0",
+        }
+    )
+    result.opcode_count = OpcodeCount.model_validate(counts)
+    return TransitionToolOutput(
+        alloc=LazyAllocJson(
+            raw=TEST_ALLOC.model_dump(), _state_root=TEST_ALLOC_STATE_ROOT
+        ),
+        result=result,
+    )
+
+
+def test_opcode_count_accumulation() -> None:
+    """
+    `process_result` accumulates the per-test opcode count total and also
+    records each call's (per-block) count separately.
+    """
+    tool = ExecutionSpecsTransitionTool()
+    tool.reset_opcode_count()
+
+    tool.process_result(_output_with_opcode_count({"PUSH1": 5, "SSTORE": 2}))
+    tool.process_result(_output_with_opcode_count({"PUSH1": 3}))
+
+    assert tool.opcode_count == OpcodeCount.model_validate(
+        {"PUSH1": 8, "SSTORE": 2}
+    )
+    assert tool.opcode_count_per_block == [
+        OpcodeCount.model_validate({"PUSH1": 5, "SSTORE": 2}),
+        OpcodeCount.model_validate({"PUSH1": 3}),
+    ]
+
+    tool.reset_opcode_count()
+    assert tool.opcode_count == OpcodeCount({})
+    assert tool.opcode_count_per_block == []

packages/testing/src/execution_testing/client_clis/transition_tool.py

@@ -202,6 +202,7 @@ class TransitionTool(EthereumCLI):
     debug_dump_dir: Path | None = None
     call_counter: int = 0
     opcode_count: OpcodeCount | None = None
+    opcode_count_per_block: List[OpcodeCount] | None = None
 
     supports_opcode_count: ClassVar[bool] = False
     supports_xdist: ClassVar[bool] = True
@@ -314,6 +315,7 @@ def reset_opcode_count(self) -> None:
         Reset the opcode count to zero.
         """
         self.opcode_count = OpcodeCount({})
+        self.opcode_count_per_block = []
 
     @dataclass
     class TransitionToolData:
@@ -987,6 +989,8 @@ def process_result(
             and self.opcode_count is not None
         ):
             self.opcode_count += result.result.opcode_count
+            if self.opcode_count_per_block is not None:
+                self.opcode_count_per_block.append(result.result.opcode_count)
         return result
 
     def evaluate(