Skip to content

Commit cb2b683

Browse files
azteca1998azteca1998
committed
feat: add in-process engine API ef_tests runner
Port the engine EF tests runner from lambdaclass/ethrex PR #6665. - Add ef_tests/engine/ crate with in-process Engine API test harness - Add release-fast profile (release-grade opt without thin-LTO) - Share merkle pool across tests via thread_local! in blockchain and state_v2 test runners to avoid spawning 17 threads per test - Update blockchain/state/state_v2 Makefiles to use release-fast profile - Fix amsterdam-vectors and zkevm-vectors Make targets to be idempotent - Add anyhow and datatest-stable as workspace dependencies
1 parent 215859a commit cb2b683

21 files changed

Lines changed: 2501 additions & 605 deletions

‎Cargo.lock‎

Lines changed: 656 additions & 582 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22
members = [
33
"archive_sync",
44
"ef_tests/blockchain",
5+
"ef_tests/engine",
56
"ef_tests/state",
67
"ef_tests/state_v2",
78
"hive_report",
@@ -88,6 +89,8 @@ crossbeam = "0.8.4"
8889
rayon = "1.10.0"
8990
rkyv = { version = "0.8.10", features = ["std", "unaligned"] }
9091
tempfile = "3.8"
92+
anyhow = "1.0.86"
93+
datatest-stable = "0.2.9"
9194
uuid = { version = "1.18.1", features = ["v4"] }
9295
tower-http = { version = "0.6.2", features = ["cors"] }
9396
indexmap = { version = "2.11.4" }
@@ -106,5 +109,14 @@ codegen-units = 1
106109
inherits = "release"
107110
debug = 2
108111

112+
# See workspace root Cargo.toml for rationale: release-grade opt-level without
113+
# thin-LTO so ef_tests rebuilds are fast.
114+
[profile.release-fast]
115+
inherits = "release"
116+
lto = false
117+
codegen-units = 16
118+
debug = "line-tables-only"
119+
incremental = true
120+
109121
[patch.crates-io]
110122
secp256k1 = { git = "https://github.com/sp1-patches/rust-secp256k1", tag = "patch-0.30.0-sp1-5.0.0" }

‎ef_tests/blockchain/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ lazy_static.workspace = true
2424
tokio = { workspace = true, features = ["full"] }
2525
datatest-stable = "0.2.9"
2626
regex = "1.11.1"
27+
rayon.workspace = true
2728

2829
[lib]
2930
path = "./lib.rs"

‎ef_tests/blockchain/Makefile‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -44,14 +44,22 @@ $(LEGACYTEST_VECTORS_DIR): $(LEGACYTEST_ARTIFACT)
4444
$(AMSTERDAM_ARTIFACT): $(AMSTERDAM_FIXTURES_FILE)
4545
curl -L -o $(AMSTERDAM_ARTIFACT) $(AMSTERDAM_URL)
4646

47-
amsterdam-vectors: $(AMSTERDAM_ARTIFACT) $(SPECTEST_VECTORS_DIR)
47+
$(SPECTEST_VECTORS_DIR)/for_amsterdam: $(AMSTERDAM_ARTIFACT) $(SPECTEST_VECTORS_DIR)
4848
tar -xzf $(AMSTERDAM_ARTIFACT) --strip-components=2 -C $(SPECTEST_VECTORS_DIR) fixtures/blockchain_tests/for_amsterdam
4949

50+
amsterdam-vectors: $(SPECTEST_VECTORS_DIR)/for_amsterdam
51+
5052
$(ZKEVM_ARTIFACT): $(ZKEVM_FIXTURES_FILE)
5153
curl -L -o $(ZKEVM_ARTIFACT) $(ZKEVM_URL)
5254

53-
zkevm-vectors: $(ZKEVM_ARTIFACT) $(SPECTEST_VECTORS_DIR)
54-
tar -xzf $(ZKEVM_ARTIFACT) --strip-components=2 -C $(SPECTEST_VECTORS_DIR) fixtures/blockchain_tests/for_amsterdam/amsterdam/eip8025_optional_proofs
55+
ZKEVM_VECTORS_DIR := $(VECTORS_ROOT)/zkevm
56+
57+
$(ZKEVM_VECTORS_DIR): $(ZKEVM_ARTIFACT)
58+
rm -rf $(ZKEVM_VECTORS_DIR)
59+
mkdir -p $(ZKEVM_VECTORS_DIR)
60+
tar -xzf $(ZKEVM_ARTIFACT) --strip-components=2 -C $(ZKEVM_VECTORS_DIR) fixtures/blockchain_tests/for_amsterdam
61+
62+
zkevm-vectors: $(ZKEVM_VECTORS_DIR)
5563

5664
help: ## 📚 Show help for each of the Makefile recipes
5765
@grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@@ -63,16 +71,16 @@ clean-vectors: ## 🗑️ Clean test vectors
6371
rm -f $(SPECTEST_ARTIFACT) $(LEGACYTEST_ARTIFACT) $(AMSTERDAM_ARTIFACT) $(ZKEVM_ARTIFACT)
6472

6573
test-levm: $(VECTORS_TARGETS) amsterdam-vectors zkevm-vectors ## 🧪 Run blockchain tests with LEVM
66-
cargo test --profile release-with-debug
74+
cargo test --profile release-fast
6775

6876
test-sp1: $(VECTORS_TARGETS) amsterdam-vectors zkevm-vectors
69-
cargo test --profile release-with-debug --features sp1
77+
cargo test --profile release-fast --features sp1
7078

7179
test-stateless: $(VECTORS_TARGETS) amsterdam-vectors zkevm-vectors
72-
cargo test --profile release-with-debug --features stateless
80+
cargo test --profile release-fast --features stateless
7381

7482
test-stateless-zkevm: $(VECTORS_TARGETS) amsterdam-vectors zkevm-vectors
75-
cargo test --profile release-with-debug --features stateless -- eip8025_optional_proofs
83+
cargo test --profile release-fast --features stateless -- eip8025_optional_proofs
7684

7785
test: ## 🧪 Run blockchain tests with LEVM both with state and stateless
7886
$(MAKE) test-levm

‎ef_tests/blockchain/test_runner.rs‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,30 @@
1-
use std::{collections::HashMap, path::Path};
1+
use std::{collections::HashMap, path::Path, sync::Arc};
22

33
use crate::{
44
fork::Fork,
55
types::{BlockChainExpectedException, BlockExpectedException, BlockWithRLP, TestUnit},
66
};
77
use ethrex_blockchain::{
8-
Blockchain, BlockchainOptions,
8+
Blockchain,
99
error::{ChainError, InvalidBlockError},
1010
fork_choice::apply_fork_choice,
1111
};
12+
13+
thread_local! {
14+
/// Per-OS-thread merkleization pool, lazily built on first use. Mirrors the
15+
/// pattern used by `ef_tests/engine` so the ~10k+ blockchain tests
16+
/// don't each spawn a fresh 17-thread rayon pool inside `Blockchain::new`.
17+
/// The merkle protocol's 16 worker jobs cross-communicate via channels, so
18+
/// each pool may have only one concurrent `in_place_scope` caller; keying by
19+
/// `thread_local!` makes the calling test-runner thread the natural
20+
/// exclusive owner.
21+
static MERKLE_POOL: std::cell::OnceCell<Arc<rayon::ThreadPool>> =
22+
const { std::cell::OnceCell::new() };
23+
}
24+
25+
fn merkle_pool() -> Arc<rayon::ThreadPool> {
26+
MERKLE_POOL.with(|cell| cell.get_or_init(Blockchain::build_merkle_pool).clone())
27+
}
1228
#[cfg(feature = "stateless")]
1329
use ethrex_common::types::block_execution_witness::RpcExecutionWitness;
1430
use ethrex_common::{
@@ -85,7 +101,7 @@ pub async fn run_ef_test(
85101
check_prestate_against_db(test_key, test, &store);
86102

87103
// Blockchain EF tests are meant for L1.
88-
let blockchain = Blockchain::new(store.clone(), BlockchainOptions::default());
104+
let blockchain = Blockchain::default_with_store_and_pool(store.clone(), merkle_pool());
89105

90106
// Early return if the exception is in the rlp decoding of the block
91107
for bf in &test.blocks {
@@ -188,7 +204,7 @@ async fn run(
188204
async fn run_two_pass_parallel(test_key: &str, test: &TestUnit) -> Result<(), String> {
189205
// ---- Pass 1: sequential, collect BALs ----
190206
let store1 = build_store_for_test(test).await;
191-
let blockchain1 = Blockchain::new(store1.clone(), BlockchainOptions::default());
207+
let blockchain1 = Blockchain::default_with_store_and_pool(store1.clone(), merkle_pool());
192208

193209
let mut bals: Vec<BlockAccessList> = Vec::with_capacity(test.blocks.len());
194210

@@ -220,7 +236,7 @@ async fn run_two_pass_parallel(test_key: &str, test: &TestUnit) -> Result<(), St
220236

221237
// ---- Pass 2: parallel (BAL-driven), verify post-state ----
222238
let store2 = build_store_for_test(test).await;
223-
let blockchain2 = Blockchain::new(store2.clone(), BlockchainOptions::default());
239+
let blockchain2 = Blockchain::default_with_store_and_pool(store2.clone(), merkle_pool());
224240

225241
for (block_fixture, bal) in test.blocks.iter().zip(bals.iter()) {
226242
let block: CoreBlock = block_fixture.block().unwrap().clone().into();

‎ef_tests/engine/.fixtures_url‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
https://github.com/ethereum/execution-spec-tests/releases/download/v5.3.0/fixtures_develop.tar.gz

‎ef_tests/engine/.fixtures_url_amsterdam‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
https://github.com/ethereum/execution-spec-tests/releases/download/snobal-devnet-6%40v1.1.0/fixtures_snobal-devnet-6.tar.gz

‎ef_tests/engine/.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
vectors/
2+
tests.tar.gz
3+
amsterdam-tests.tar.gz

‎ef_tests/engine/Cargo.toml‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
[package]
2+
name = "ef_tests-engine"
3+
version.workspace = true
4+
edition.workspace = true
5+
authors.workspace = true
6+
documentation.workspace = true
7+
license.workspace = true
8+
9+
[dependencies]
10+
ethrex-common.workspace = true
11+
ethrex-storage.workspace = true
12+
ethrex-rpc.workspace = true
13+
ethrex-blockchain.workspace = true
14+
ethrex-p2p.workspace = true
15+
rayon.workspace = true
16+
17+
serde.workspace = true
18+
serde_json.workspace = true
19+
tokio = { workspace = true, features = ["full"] }
20+
anyhow.workspace = true
21+
bytes.workspace = true
22+
hex.workspace = true
23+
tempfile.workspace = true
24+
regex = "1.11.1"
25+
26+
[dev-dependencies]
27+
datatest-stable.workspace = true
28+
ctor = "0.2"
29+
30+
[lib]
31+
path = "src/lib.rs"
32+
33+
[[test]]
34+
name = "all"
35+
harness = false
36+
37+
[features]
38+
default = ["c-kzg"]
39+
c-kzg = ["ethrex-blockchain/c-kzg"]
40+
rocksdb = ["ethrex-storage/rocksdb"]

‎ef_tests/engine/Makefile‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,48 @@
1+
.PHONY: download-test-vectors clean-vectors test test-rocksdb test-strict amsterdam-vectors help
2+
3+
VECTORS_ROOT := vectors
4+
FIXTURES_FILE := .fixtures_url
5+
SPECTEST_ARTIFACT := tests.tar.gz
6+
SPECTEST_VECTORS_DIR := $(VECTORS_ROOT)/eest
7+
SPECTEST_URL := $(shell cat $(FIXTURES_FILE))
8+
9+
AMSTERDAM_FIXTURES_FILE := .fixtures_url_amsterdam
10+
AMSTERDAM_ARTIFACT := amsterdam-tests.tar.gz
11+
AMSTERDAM_URL := $(shell cat $(AMSTERDAM_FIXTURES_FILE))
12+
13+
VECTORS_TARGETS := $(SPECTEST_VECTORS_DIR)
14+
15+
$(SPECTEST_ARTIFACT): $(FIXTURES_FILE)
16+
$(MAKE) clean-vectors
17+
curl -L -o $(SPECTEST_ARTIFACT) $(SPECTEST_URL)
18+
19+
$(SPECTEST_VECTORS_DIR): $(SPECTEST_ARTIFACT)
20+
rm -rf $(SPECTEST_VECTORS_DIR)
21+
mkdir -p $(SPECTEST_VECTORS_DIR)
22+
tar -xzf $(SPECTEST_ARTIFACT) --strip-components=2 -C $(SPECTEST_VECTORS_DIR) fixtures/blockchain_tests_engine
23+
24+
$(AMSTERDAM_ARTIFACT): $(AMSTERDAM_FIXTURES_FILE)
25+
curl -L -o $(AMSTERDAM_ARTIFACT) $(AMSTERDAM_URL)
26+
27+
$(SPECTEST_VECTORS_DIR)/for_amsterdam: $(AMSTERDAM_ARTIFACT) $(SPECTEST_VECTORS_DIR)
28+
tar -xzf $(AMSTERDAM_ARTIFACT) --strip-components=2 -C $(SPECTEST_VECTORS_DIR) fixtures/blockchain_tests_engine/for_amsterdam
29+
30+
amsterdam-vectors: $(SPECTEST_VECTORS_DIR)/for_amsterdam ## 📥 Overlay snobal-devnet-6 engine fixtures onto vectors/eest
31+
32+
help: ## 📚 Show help for each Makefile target
33+
@grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
34+
35+
download-test-vectors: $(VECTORS_TARGETS) amsterdam-vectors ## 📥 Download engine-format test vectors
36+
37+
clean-vectors: ## 🗑️ Clean test vectors
38+
rm -rf $(VECTORS_ROOT)
39+
rm -f $(SPECTEST_ARTIFACT) $(AMSTERDAM_ARTIFACT)
40+
41+
test: $(VECTORS_TARGETS) amsterdam-vectors ## 🧪 Run engine-format fixture suite (InMemory, release)
42+
cargo test -p ef_tests-engine --release
43+
44+
test-rocksdb: $(VECTORS_TARGETS) amsterdam-vectors ## 🧪 Run with RocksDB backend
45+
cargo test -p ef_tests-engine --release --features rocksdb
46+
47+
test-strict: $(VECTORS_TARGETS) amsterdam-vectors ## 🧪 Run with strict validationError checking
48+
ETHREX_ENGINE_STRICT_EXCEPTIONS=1 cargo test -p ef_tests-engine --release

0 commit comments

Comments
 (0)