feat: sync all tooling changes from ethrex main

Port all tooling changes that landed on ethrex main after the initial
ethrex-tooling repo was created:

New tools/files:
- ef_tests/engine: in-process engine API EF tests runner (#6665)
- ef_tests/state_v2/statetest: goevmlab fuzzing subcommand (#6663)
- migrations/src/bin/bench_migration + seed_migration_test (#6598)
- trace_compare/: opcodeTracer compare script (#6595)
- sync/peer_top.py: snap sync observability (#6470)

Changes to existing tools:
- Add release-fast profile for faster EF test rebuilds
- Share merkle pool in blockchain/state_v2 test runners (#6665)
- Update amsterdam/zkevm Make targets to be idempotent
- Bump fixture URLs for bal-devnet-6/7 (#6574, #6653, #6671)
- Bump zkevm fixtures to v0.3.3 (#6527)
- Harden JSON-RPC defaults in sync/l2 configs (#6627)
- Move getBlockAccessList to eth namespace in REPL (#6709)
- Statetest CLI follow-ups (#6721)
- Docker monitor disk-full handling (#6500)
- Various version bumps and dep updates

Author: azteca1998

Cargo.lock

@@ -2,6 +2,7 @@
 members = [
   "archive_sync",
   "ef_tests/blockchain",
+  "ef_tests/engine",
   "ef_tests/state",
   "ef_tests/state_v2",
   "hive_report",
@@ -88,6 +89,9 @@ crossbeam = "0.8.4"
 rayon = "1.10.0"
 rkyv = { version = "0.8.10", features = ["std", "unaligned"] }
 tempfile = "3.8"
+anyhow = "1.0.86"
+datatest-stable = "0.2.9"
+libc = "0.2"
 uuid = { version = "1.18.1", features = ["v4"] }
 tower-http = { version = "0.6.2", features = ["cors"] }
 indexmap = { version = "2.11.4" }
@@ -106,5 +110,13 @@ codegen-units = 1
 inherits = "release"
 debug = 2
 
+# Release-grade opt-level without thin-LTO so ef_tests rebuilds are fast.
+[profile.release-fast]
+inherits = "release"
+lto = false
+codegen-units = 16
+debug = "line-tables-only"
+incremental = true
+
 [patch.crates-io]
 secp256k1 = { git = "https://github.com/sp1-patches/rust-secp256k1", tag = "patch-0.30.0-sp1-5.0.0" }

Cargo.toml

@@ -1 +1 @@
-https://github.com/ethereum/execution-spec-tests/releases/download/bal%40v5.6.1/fixtures_bal.tar.gz
+https://github.com/ethereum/execution-specs/releases/download/tests-bal%40v7.2.0/fixtures_bal.tar.gz

ef_tests/blockchain/.fixtures_url_amsterdam

@@ -1 +1 @@
-https://github.com/ethereum/execution-spec-tests/releases/download/zkevm%40v0.3.0/fixtures_zkevm.tar.gz
+https://github.com/ethereum/execution-spec-tests/releases/download/zkevm%40v0.3.3/fixtures_zkevm.tar.gz

ef_tests/blockchain/.fixtures_url_zkevm

@@ -24,6 +24,7 @@ lazy_static.workspace = true
 tokio = { workspace = true, features = ["full"] }
 datatest-stable = "0.2.9"
 regex = "1.11.1"
+rayon.workspace = true
 
 [lib]
 path = "./lib.rs"

ef_tests/blockchain/Cargo.toml

@@ -16,6 +16,12 @@ AMSTERDAM_FIXTURES_FILE := .fixtures_url_amsterdam
 AMSTERDAM_ARTIFACT := amsterdam-tests.tar.gz
 AMSTERDAM_URL := $(shell cat $(AMSTERDAM_FIXTURES_FILE))
 
+# zkevm@v0.3.3 ships fixtures filled against an older Amsterdam base
+# (bal@v5.6.1). Extracting them on top of the bal-devnet-7 tree would
+# clobber the newer fixtures with stale gas-accounting expectations, so we
+# keep them in a separate root and only the stateless harness reads from it.
+ZKEVM_VECTORS_ROOT := vectors_zkevm
+ZKEVM_VECTORS_DIR := $(ZKEVM_VECTORS_ROOT)/eest
 ZKEVM_FIXTURES_FILE := .fixtures_url_zkevm
 ZKEVM_ARTIFACT := zkevm-tests.tar.gz
 ZKEVM_URL := $(shell cat $(ZKEVM_FIXTURES_FILE))
@@ -44,36 +50,49 @@ $(LEGACYTEST_VECTORS_DIR): $(LEGACYTEST_ARTIFACT)
 $(AMSTERDAM_ARTIFACT): $(AMSTERDAM_FIXTURES_FILE)
 	curl -L -o $(AMSTERDAM_ARTIFACT) $(AMSTERDAM_URL)
 
-amsterdam-vectors: $(AMSTERDAM_ARTIFACT) $(SPECTEST_VECTORS_DIR)
+$(SPECTEST_VECTORS_DIR)/for_amsterdam: $(AMSTERDAM_ARTIFACT) $(SPECTEST_VECTORS_DIR)
 	tar -xzf $(AMSTERDAM_ARTIFACT) --strip-components=2 -C $(SPECTEST_VECTORS_DIR) fixtures/blockchain_tests/for_amsterdam
 
+amsterdam-vectors: $(SPECTEST_VECTORS_DIR)/for_amsterdam
+
 $(ZKEVM_ARTIFACT): $(ZKEVM_FIXTURES_FILE)
 	curl -L -o $(ZKEVM_ARTIFACT) $(ZKEVM_URL)
 
-zkevm-vectors: $(ZKEVM_ARTIFACT) $(SPECTEST_VECTORS_DIR)
-	tar -xzf $(ZKEVM_ARTIFACT) --strip-components=2 -C $(SPECTEST_VECTORS_DIR) fixtures/blockchain_tests/for_amsterdam/amsterdam/eip8025_optional_proofs
+$(ZKEVM_VECTORS_DIR): $(ZKEVM_ARTIFACT)
+	rm -rf $(ZKEVM_VECTORS_DIR)
+	mkdir -p $(ZKEVM_VECTORS_DIR)
+	tar -xzf $(ZKEVM_ARTIFACT) --strip-components=2 -C $(ZKEVM_VECTORS_DIR) fixtures/blockchain_tests/for_amsterdam
+
+zkevm-vectors: $(ZKEVM_VECTORS_DIR)
 
 help: ## 📚 Show help for each of the Makefile recipes
 	@grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
 
 download-test-vectors: $(VECTORS_TARGETS) amsterdam-vectors zkevm-vectors ## 📥 Download test vectors
 
 clean-vectors: ## 🗑️  Clean test vectors
-	rm -rf $(VECTORS_ROOT)
+	rm -rf $(VECTORS_ROOT) $(ZKEVM_VECTORS_ROOT)
 	rm -f $(SPECTEST_ARTIFACT) $(LEGACYTEST_ARTIFACT) $(AMSTERDAM_ARTIFACT) $(ZKEVM_ARTIFACT)
 
-test-levm: $(VECTORS_TARGETS) amsterdam-vectors zkevm-vectors ## 🧪 Run blockchain tests with LEVM
-	cargo test --profile release-with-debug
+test-levm: $(VECTORS_TARGETS) amsterdam-vectors ## 🧪 Run blockchain tests with LEVM
+	cargo test --profile release-fast
 
-test-sp1: $(VECTORS_TARGETS) amsterdam-vectors zkevm-vectors
-	cargo test --profile release-with-debug --features sp1
+test-sp1: $(VECTORS_TARGETS) amsterdam-vectors
+	cargo test --profile release-fast --features sp1
 
-test-stateless: $(VECTORS_TARGETS) amsterdam-vectors zkevm-vectors
-	cargo test --profile release-with-debug --features stateless
+test-stateless: zkevm-vectors
+	cargo test --profile release-fast --features stateless
 
-test-stateless-zkevm: $(VECTORS_TARGETS) amsterdam-vectors zkevm-vectors
-	cargo test --profile release-with-debug --features stateless -- eip8025_optional_proofs
+test-stateless-zkevm: zkevm-vectors
+	cargo test --profile release-fast --features stateless -- eip8025_optional_proofs
 
-test: ## 🧪 Run blockchain tests with LEVM both with state and stateless 
+test: ## 🧪 Run blockchain tests with LEVM both with state and stateless
 	$(MAKE) test-levm
-	$(MAKE) test-stateless
+	# Narrow stateless coverage to the EIP-8025 optional-proofs suite. The
+	# zkevm@v0.3.3 fixtures are filled against bal@v5.6.1, which is out of
+	# sync with this branch's bal-devnet-6+ (and bal-devnet-7-prep) gas
+	# accounting; the broader `test-stateless` invocation introduced by
+	# #6527 trips ~549 of those fixtures with `GasUsedMismatch` /
+	# `ReceiptsRootMismatch` / `BlockAccessListHashMismatch`. Re-broaden
+	# once the zkevm bundle is regenerated against the current bal spec.
+	$(MAKE) test-stateless-zkevm

ef_tests/blockchain/Makefile

@@ -1,14 +1,30 @@
-use std::{collections::HashMap, path::Path};
+use std::{collections::HashMap, path::Path, sync::Arc};
 
 use crate::{
     fork::Fork,
     types::{BlockChainExpectedException, BlockExpectedException, BlockWithRLP, TestUnit},
 };
 use ethrex_blockchain::{
-    Blockchain, BlockchainOptions,
+    Blockchain,
     error::{ChainError, InvalidBlockError},
     fork_choice::apply_fork_choice,
 };
+
+thread_local! {
+    /// Per-OS-thread merkleization pool, lazily built on first use. Mirrors the
+    /// pattern used by `tooling/ef_tests/engine` so the ~10k+ blockchain tests
+    /// don't each spawn a fresh 17-thread rayon pool inside `Blockchain::new`.
+    /// The merkle protocol's 16 worker jobs cross-communicate via channels, so
+    /// each pool may have only one concurrent `in_place_scope` caller; keying by
+    /// `thread_local!` makes the calling test-runner thread the natural
+    /// exclusive owner.
+    static MERKLE_POOL: std::cell::OnceCell<Arc<rayon::ThreadPool>> =
+        const { std::cell::OnceCell::new() };
+}
+
+fn merkle_pool() -> Arc<rayon::ThreadPool> {
+    MERKLE_POOL.with(|cell| cell.get_or_init(Blockchain::build_merkle_pool).clone())
+}
 #[cfg(feature = "stateless")]
 use ethrex_common::types::block_execution_witness::RpcExecutionWitness;
 use ethrex_common::{
@@ -85,7 +101,7 @@ pub async fn run_ef_test(
     check_prestate_against_db(test_key, test, &store);
 
     // Blockchain EF tests are meant for L1.
-    let blockchain = Blockchain::new(store.clone(), BlockchainOptions::default());
+    let blockchain = Blockchain::default_with_store_and_pool(store.clone(), merkle_pool());
 
     // Early return if the exception is in the rlp decoding of the block
     for bf in &test.blocks {
@@ -158,8 +174,12 @@ async fn run(
                         "Warning: Returned exception {error:?} does not match expected {expected_exception:?}",
                     );
                 }
-                // Expected exception matched — stop processing further blocks of this test.
-                break;
+                // Expected exception matched — block was rejected, but the test may
+                // still expect subsequent blocks to be processed (e.g. fork-transition
+                // tests where a block at the pre-fork timestamp fails and a block at
+                // the post-fork timestamp succeeds, both built on the same parent).
+                // Continue with the next block in the fixture.
+                continue;
             }
             Ok(_) => {
                 if expects_exception {
@@ -188,7 +208,7 @@ async fn run(
 async fn run_two_pass_parallel(test_key: &str, test: &TestUnit) -> Result<(), String> {
     // ---- Pass 1: sequential, collect BALs ----
     let store1 = build_store_for_test(test).await;
-    let blockchain1 = Blockchain::new(store1.clone(), BlockchainOptions::default());
+    let blockchain1 = Blockchain::default_with_store_and_pool(store1.clone(), merkle_pool());
 
     let mut bals: Vec<BlockAccessList> = Vec::with_capacity(test.blocks.len());
 
@@ -220,7 +240,7 @@ async fn run_two_pass_parallel(test_key: &str, test: &TestUnit) -> Result<(), St
 
     // ---- Pass 2: parallel (BAL-driven), verify post-state ----
     let store2 = build_store_for_test(test).await;
-    let blockchain2 = Blockchain::new(store2.clone(), BlockchainOptions::default());
+    let blockchain2 = Blockchain::default_with_store_and_pool(store2.clone(), merkle_pool());
 
     for (block_fixture, bal) in test.blocks.iter().zip(bals.iter()) {
         let block: CoreBlock = block_fixture.block().unwrap().clone().into();
@@ -554,29 +574,71 @@ async fn run_stateless_from_fixture(
         let block: CoreBlock = block_data.clone().into();
         let block_number = block.header.number;
 
+        // Absent bytes means "expected to succeed"; malformed bytes are a hard error.
+        let expected_valid = match block_data.stateless_output_bytes.as_deref() {
+            None => true,
+            Some(bytes) => parse_expected_valid_flag(bytes).map_err(|e| {
+                format!("Malformed statelessOutputBytes for {test_key} block {block_number}: {e}")
+            })?,
+        };
+
+        // Parse and conversion errors must always fail; only the execution outcome is
+        // matched against `expected_valid` so the (false, Err(_)) arm below cannot
+        // absorb regressions in deserialization or witness conversion.
         let rpc_witness: RpcExecutionWitness = serde_json::from_value(witness_json.clone())
             .map_err(|e| {
-                format!("Failed to parse executionWitness for block {block_number}: {e}")
+                format!("executionWitness parse failed for {test_key} block {block_number}: {e}")
             })?;
-
         let execution_witness = rpc_witness
             .into_execution_witness(*chain_config, block_number)
-            .map_err(|e| format!("Witness conversion failed for block {block_number}: {e}"))?;
+            .map_err(|e| {
+                format!("witness conversion failed for {test_key} block {block_number}: {e}")
+            })?;
 
         let program_input = ProgramInput::new(vec![block], execution_witness);
-
-        let execute_result = match backend_type {
+        let exec_result = match backend_type {
             BackendType::Exec => ExecBackend::new().execute(program_input),
             #[cfg(feature = "sp1")]
             BackendType::SP1 => Sp1Backend::new().execute(program_input),
         };
 
-        if let Err(e) = execute_result {
-            return Err(format!(
-                "Stateless execution from fixture failed for {test_key} block {block_number}: {e}"
-            ));
+        match (expected_valid, exec_result) {
+            (true, Ok(_)) | (false, Err(_)) => {}
+            (true, Err(e)) => {
+                return Err(format!(
+                    "Stateless execution from fixture failed for {test_key} block {block_number}: {e}"
+                ));
+            }
+            (false, Ok(_)) => {
+                return Err(format!(
+                    "Stateless execution from fixture succeeded for {test_key} block \
+                     {block_number} but fixture expected it to fail (invalid executionWitness)"
+                ));
+            }
         }
     }
 
     Ok(())
 }
+
+/// Decode the `valid` byte (index 32) from a zkevm-fixture `statelessOutputBytes` hex
+/// string, encoded as `new_payload_request_root (32 B) ++ valid (1 B) ++ padding`.
+#[cfg(feature = "stateless")]
+fn parse_expected_valid_flag(hex: &str) -> Result<bool, String> {
+    let trimmed = hex.strip_prefix("0x").unwrap_or(hex);
+    let byte_hex = trimmed.get(64..66).ok_or_else(|| {
+        format!(
+            "expected at least 33 bytes (66 hex chars), got {} hex chars",
+            trimmed.len()
+        )
+    })?;
+    let byte = u8::from_str_radix(byte_hex, 16)
+        .map_err(|e| format!("invalid hex at byte 32 ({byte_hex:?}): {e}"))?;
+    match byte {
+        0 => Ok(false),
+        1 => Ok(true),
+        n => Err(format!(
+            "invalid validity byte 0x{n:02x} (expected 0x00 or 0x01)"
+        )),
+    }
+}

ef_tests/blockchain/test_runner.rs

@@ -6,6 +6,13 @@ use std::path::Path;
 #[cfg(all(feature = "sp1", feature = "stateless"))]
 compile_error!("Only one of `sp1` and `stateless` can be enabled at a time.");
 
+// test-levm / test-sp1 read snobal-devnet-6 + legacy from `vectors/`.
+// test-stateless reads zkevm@v0.3.3 (the only bundle that ships executionWitness)
+// from a separate `vectors_zkevm/` so its older bal@v5.6.1 base never overlays
+// the snobal fixtures used by the other suites.
+#[cfg(feature = "stateless")]
+const TEST_FOLDER: &str = "vectors_zkevm/";
+#[cfg(not(feature = "stateless"))]
 const TEST_FOLDER: &str = "vectors/";
 
 // Base skips shared by all runs.
@@ -18,23 +25,91 @@ const SKIPPED_BASE: &[&str] = &[
     "ValueOverflowParis",
     // Skip because it's a "Create" Blob Transaction, which doesn't actually exist. It never reaches the EVM because we can't even parse it as an actual Transaction.
     "createBlobhashTx",
+    // EIP-8025 optional-proofs fixtures filled against bal@v5.6.1 (devnets/bal/3),
+    // which predates EELS PR #2711 "immutable intrinsic_state_gas for EIP-7702".
+    // Expected gas assumes the auth refund still deducts from block-accounted state
+    // gas; our devnet-4 (bal@v5.7.0) impl correctly keeps intrinsic_state_gas
+    // immutable and routes the refund to the reservoir only. Re-enable once the
+    // zkevm@v0.4.x release ships fixtures regenerated against devnet-4.
+    "witness_codes_redelegation_old_marker_included_new_marker_excluded",
+    "witness_codes_reset_delegation",
+    "witness_codes_reverted_transaction",
+    "witness_codes_failed_create_includes_factory",
+    "witness_codes_reverted_create_same_hash_then_read",
+    "witness_codes_create_then_selfdestruct_same_tx",
+    // Additional EIP-8025 optional-proofs fixtures whose expected gas magnitudes
+    // disagree with bal-devnet-7 (bal@v7.1.1) state-gas accounting. Same root
+    // cause as the block above: zkevm@v0.3.3 bundle is pinned at an older bal
+    // spec (storage_set / new_account / cpsb constants pre-recalibration plus
+    // earlier refund-channel semantics) and the broader fork.py changes from
+    // EELS PRs #2815/#2816/#2823/#2827/#2828. Re-enable once the zkevm bundle
+    // is regenerated against bal-7.
+    "witness_codes_delegation_set_in_same_block",
+    "witness_codes_auth_nonce_mismatch",
+    "witness_codes_dedup_identical_bytecode",
+    "witness_codes_create2_excludes_new_bytecode",
+    "witness_codes_reverted_inner_call",
+    "witness_codes_create_same_hash_then_read",
+    "witness_codes_create_then_call_same_block",
+    "witness_codes_create_then_call_same_tx",
+    "witness_codes_failed_create_after_initcode_read",
+    "witness_codes_initcode_calls_existing_contract",
+    "witness_excludes_bytecode_created_in_same_block",
+    "witness_keeps_prestate_code_read_even_if_later_created_with_same_hash",
+    "witness_codes_selfdestruct_in_initcode",
+    "witness_codes_selfdestruct_beneficiary_no_code",
+    "witness_state_delete_with_new_dirty_sibling_omits_post_state_node",
+    "witness_state_block_diff_delete_insert_before_delete_order",
+    "witness_state_delete_then_insert_uses_insert_before_delete_order",
+    "witness_state_sstore_into_empty_storage_omits_post_state_nodes",
+    "witness_state_sstore_new_slot_omits_post_state_nodes",
+    "validation_state_missing_absent_slot_proof_leaf_node",
+    "validation_state_missing_storage_proof_node",
 ];
 
 // Extra skips added only for prover backends.
-#[cfg(feature = "sp1")]
+#[cfg(all(feature = "sp1", not(feature = "stateless")))]
 const EXTRA_SKIPS: &[&str] = &[
     // I believe these tests fail because of how much stress they put into the zkVM, they probably cause an OOM though this should be checked
     "static_Call50000",
     "Return50000",
     "static_Call1MB1024Calldepth",
 ];
-#[cfg(not(feature = "sp1"))]
+#[cfg(feature = "stateless")]
+const EXTRA_SKIPS: &[&str] = &[
+    // zkevm@v0.3.3 tolerance tests: the fixture's `statelessOutputBytes` declares `valid = 1`
+    // because the executed path does not actually consume the malformed/extra/missing witness
+    // entry, but our RpcExecutionWitness conversion eagerly validates the full witness and
+    // rejects it. Re-enable once the witness conversion is lazy per EIP-8025 §Tolerance.
+    "validation_headers_malformed_rlp_header",
+    "validation_headers_missing_oldest_blockhash_ancestor",
+    "validation_headers_missing_parent_header",
+    "validation_state_extra_unused_trie_node",
+    // zkevm@v0.3.3 rejection tests: `statelessOutputBytes` declares `valid = 0` so the guest
+    // program must reject the deliberately-incomplete witness, but our stateless path runs
+    // to completion instead of detecting the missing entry. Re-enable once the witness
+    // completeness checks land (missing delegation/external-code bytecodes, non-contiguous
+    // header chain detection).
+    "validation_codes_missing_delegated_code_on_insufficient_balance_call",
+    "validation_codes_missing_external_code_read_target",
+    "validation_codes_missing_redelegation_old_marker",
+    "validation_codes_missing_sender_delegation_marker",
+    "validation_headers_non_contiguous_chain",
+    // zkevm@v0.3.3 conversion-time rejection: `statelessOutputBytes` declares `valid = 0` and
+    // our `into_execution_witness` correctly rejects the witness because it can't extract the
+    // initial state root without the parent header. Since 5a597e67d the runner treats
+    // conversion errors as unconditional regressions, so this correct-rejection-at-the-wrong-
+    // stage trips the test. Re-enable once conversion is lazy enough to defer the parent-
+    // header check to execution.
+    "validation_headers_empty_block_missing_mandatory_parent",
+];
+#[cfg(not(any(feature = "sp1", feature = "stateless")))]
 const EXTRA_SKIPS: &[&str] = &[];
 
 // Select backend
 #[cfg(feature = "stateless")]
 const BACKEND: Option<BackendType> = Some(BackendType::Exec);
-#[cfg(feature = "sp1")]
+#[cfg(all(feature = "sp1", not(feature = "stateless")))]
 const BACKEND: Option<BackendType> = Some(BackendType::SP1);
 #[cfg(not(any(feature = "sp1", feature = "stateless")))]
 const BACKEND: Option<BackendType> = None;

ef_tests/blockchain/tests/all.rs

@@ -0,0 +1 @@
+https://github.com/ethereum/execution-spec-tests/releases/download/v5.3.0/fixtures_develop.tar.gz

ef_tests/engine/.fixtures_url

@@ -0,0 +1 @@
+https://github.com/ethereum/execution-spec-tests/releases/download/snobal-devnet-6%40v1.1.0/fixtures_snobal-devnet-6.tar.gz