From b7fb069ca0f5d0a2a02e5e673f95d1a28490c5ed Mon Sep 17 00:00:00 2001 From: oXtxNt9U <120286271+oXtxNt9U@users.noreply.github.com> Date: Fri, 5 Jun 2026 09:43:58 +0900 Subject: [PATCH 01/10] conditional storage compression --- packages/evm/core/src/compression.rs | 69 ++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/packages/evm/core/src/compression.rs b/packages/evm/core/src/compression.rs index a6563df9e5..2cc4ed0d8d 100644 --- a/packages/evm/core/src/compression.rs +++ b/packages/evm/core/src/compression.rs @@ -1,8 +1,24 @@ use std::{borrow::Cow, ops::Deref}; -const VERSION: u8 = 1; +// First byte of every stored value tags how the remainder is encoded: +// TAG_RAW: [0][raw bincode] (uncompressed) +// TAG_ZSTD: [1][orig_len: u32 LE][zstd payload] (compressed) +const TAG_RAW: u8 = 0; +const TAG_ZSTD: u8 = 1; + const ZSTD_LEVEL: i32 = 3; +// Compression is only attempted for serialized values at least this large. The frequently-written +// values are poor candidates for two independent reasons: +// - small size (accounts ~70 B, proofs ~120 B): too little context, and zstd's frame overhead +// plus the 4-byte length header outweigh any gain; +// - high-entropy content: keccak code hashes and BLS signatures are effectively random, so there +// is no redundancy to exploit at any size. +// The threshold skips the first case cheaply; the "keep raw unless actually smaller" check on the +// result handles the second. Larger structured values (headers with a sparse logs bloom, +// ABI-padded calldata, receipt logs, bytecode) still compress and are kept only when it shrinks them. +const MIN_COMPRESS_LEN: usize = 256; + #[derive(Debug)] pub struct CompressedBincode(pub T); impl<'a, T: serde::Serialize + 'a> heed::BytesEncode<'a> for CompressedBincode { @@ -10,16 +26,27 @@ impl<'a, T: serde::Serialize + 'a> heed::BytesEncode<'a> for CompressedBincode Result, heed::BoxedError> { let raw = bincode::serialize(&item.0)?; - let orig_len = raw.len(); - let compressed = zstd::bulk::compress(&raw, ZSTD_LEVEL)?; - let mut out = Vec::with_capacity(1 + 4 + compressed.len()); + if raw.len() >= MIN_COMPRESS_LEN { + let compressed = zstd::bulk::compress(&raw, ZSTD_LEVEL)?; - // [1 byte version][4 bytes orig_len LE][compressed...] - out.push(VERSION); - out.extend_from_slice(&(orig_len as u32).to_le_bytes()); - out.extend_from_slice(&compressed); + // The compressed layout carries an extra 4-byte original-length header; only keep it + // when the result is genuinely smaller than storing raw (both forms share the 1-byte + // tag, so it cancels out of the comparison). + if compressed.len() + 4 < raw.len() { + let mut out = Vec::with_capacity(1 + 4 + compressed.len()); + out.push(TAG_ZSTD); + out.extend_from_slice(&(raw.len() as u32).to_le_bytes()); + out.extend_from_slice(&compressed); + return Ok(Cow::Owned(out)); + } + } + // Store raw. A value is therefore never persisted larger than its bincode encoding plus the + // single tag byte. + let mut out = Vec::with_capacity(1 + raw.len()); + out.push(TAG_RAW); + out.extend_from_slice(&raw); Ok(Cow::Owned(out)) } } @@ -28,17 +55,23 @@ impl<'a, T: serde::de::DeserializeOwned + 'a> heed::BytesDecode<'a> for Compress type DItem = CompressedBincode; fn bytes_decode(bytes: &'_ [u8]) -> Result { - let version = bytes[0]; - assert_eq!(version, VERSION, "unsupported version"); - - let mut len_bytes = [0u8; 4]; - len_bytes.copy_from_slice(&bytes[1..5]); - let orig_len = u32::from_le_bytes(len_bytes) as usize; - - let payload = &bytes[5..]; - let decompressed = zstd::bulk::decompress(payload, orig_len)?; + let (&tag, payload) = bytes + .split_first() + .ok_or("CompressedBincode: empty value")?; - let deserialized = bincode::deserialize(&decompressed)?; + let deserialized = match tag { + TAG_ZSTD => { + if payload.len() < 4 { + return Err("CompressedBincode: truncated zstd header".into()); + } + let (len_bytes, compressed) = payload.split_at(4); + let orig_len = u32::from_le_bytes(len_bytes.try_into().unwrap()) as usize; + let decompressed = zstd::bulk::decompress(compressed, orig_len)?; + bincode::deserialize(&decompressed)? + } + TAG_RAW => bincode::deserialize(payload)?, + other => return Err(format!("CompressedBincode: unknown tag {other}").into()), + }; Ok(CompressedBincode(deserialized)) } From 500cb2e4825fe741abc77fb6c1816422dacaf3df Mon Sep 17 00:00:00 2001 From: oXtxNt9U <120286271+oXtxNt9U@users.noreply.github.com> Date: Fri, 5 Jun 2026 09:44:03 +0900 Subject: [PATCH 02/10] add tests --- packages/evm/core/src/compression.rs | 74 ++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/packages/evm/core/src/compression.rs b/packages/evm/core/src/compression.rs index 2cc4ed0d8d..b6c45651de 100644 --- a/packages/evm/core/src/compression.rs +++ b/packages/evm/core/src/compression.rs @@ -84,3 +84,77 @@ impl Deref for CompressedBincode { &self.0 } } + +#[cfg(test)] +mod tests { + use heed::{BytesDecode, BytesEncode}; + + use super::*; + + fn encode(value: &Vec) -> Vec { + let item = CompressedBincode(value); + > as BytesEncode>::bytes_encode(&item) + .unwrap() + .into_owned() + } + + fn decode(bytes: &[u8]) -> Vec { + > as BytesDecode>::bytes_decode(bytes) + .unwrap() + .0 + } + + #[test] + fn small_value_is_stored_raw() { + let value = vec![1u8, 2, 3, 4, 5]; + let encoded = encode(&value); + assert_eq!(encoded[0], TAG_RAW); + assert_eq!(decode(&encoded), value); + } + + #[test] + fn large_compressible_value_is_stored_zstd_and_smaller() { + let value = vec![7u8; 4096]; + let encoded = encode(&value); + assert_eq!(encoded[0], TAG_ZSTD); + assert!(encoded.len() < value.len()); + assert_eq!(decode(&encoded), value); + } + + #[test] + fn output_never_exceeds_raw_plus_tag() { + // A large, hard-to-compress value: compression is attempted but must fall back to raw + // rather than store something bigger. + let value: Vec = (0..2048u32) + .map(|i| (i.wrapping_mul(2_654_435_761) >> 13) as u8) + .collect(); + let raw_len = bincode::serialize(&value).unwrap().len(); + + let encoded = encode(&value); + assert!( + encoded.len() <= raw_len + 1, + "encoded {} raw {}", + encoded.len(), + raw_len + ); + assert_eq!(decode(&encoded), value); + } + + #[test] + fn empty_input_is_an_error_not_a_panic() { + assert!(> as BytesDecode>::bytes_decode(&[]).is_err()); + } + + #[test] + fn truncated_zstd_header_is_an_error_not_a_panic() { + // TAG_ZSTD with fewer than the four orig_len header bytes must error, not panic on the slice. + assert!( + > as BytesDecode>::bytes_decode(&[TAG_ZSTD, 1, 2]).is_err() + ); + } + + #[test] + fn unknown_tag_is_an_error_not_a_panic() { + assert!(> as BytesDecode>::bytes_decode(&[9, 0, 0]).is_err()); + } +} From 895e1874de6a340eceaaa76f87febc27878cb250 Mon Sep 17 00:00:00 2001 From: oXtxNt9U <120286271+oXtxNt9U@users.noreply.github.com> Date: Fri, 5 Jun 2026 09:45:49 +0900 Subject: [PATCH 03/10] rename CompressedBincode to CompactBincode --- packages/evm/core/src/compression.rs | 28 +++--- packages/evm/core/src/db.rs | 125 ++++++++++++--------------- packages/evm/core/src/historical.rs | 8 +- 3 files changed, 72 insertions(+), 89 deletions(-) diff --git a/packages/evm/core/src/compression.rs b/packages/evm/core/src/compression.rs index b6c45651de..3a6244dafc 100644 --- a/packages/evm/core/src/compression.rs +++ b/packages/evm/core/src/compression.rs @@ -20,9 +20,9 @@ const ZSTD_LEVEL: i32 = 3; const MIN_COMPRESS_LEN: usize = 256; #[derive(Debug)] -pub struct CompressedBincode(pub T); -impl<'a, T: serde::Serialize + 'a> heed::BytesEncode<'a> for CompressedBincode { - type EItem = CompressedBincode<&'a T>; +pub struct CompactBincode(pub T); +impl<'a, T: serde::Serialize + 'a> heed::BytesEncode<'a> for CompactBincode { + type EItem = CompactBincode<&'a T>; fn bytes_encode(item: &'a Self::EItem) -> Result, heed::BoxedError> { let raw = bincode::serialize(&item.0)?; @@ -51,8 +51,8 @@ impl<'a, T: serde::Serialize + 'a> heed::BytesEncode<'a> for CompressedBincode heed::BytesDecode<'a> for CompressedBincode { - type DItem = CompressedBincode; +impl<'a, T: serde::de::DeserializeOwned + 'a> heed::BytesDecode<'a> for CompactBincode { + type DItem = CompactBincode; fn bytes_decode(bytes: &'_ [u8]) -> Result { let (&tag, payload) = bytes @@ -73,11 +73,11 @@ impl<'a, T: serde::de::DeserializeOwned + 'a> heed::BytesDecode<'a> for Compress other => return Err(format!("CompressedBincode: unknown tag {other}").into()), }; - Ok(CompressedBincode(deserialized)) + Ok(CompactBincode(deserialized)) } } -impl Deref for CompressedBincode { +impl Deref for CompactBincode { type Target = T; fn deref(&self) -> &Self::Target { @@ -92,14 +92,14 @@ mod tests { use super::*; fn encode(value: &Vec) -> Vec { - let item = CompressedBincode(value); - > as BytesEncode>::bytes_encode(&item) + let item = CompactBincode(value); + > as BytesEncode>::bytes_encode(&item) .unwrap() .into_owned() } fn decode(bytes: &[u8]) -> Vec { - > as BytesDecode>::bytes_decode(bytes) + > as BytesDecode>::bytes_decode(bytes) .unwrap() .0 } @@ -142,19 +142,17 @@ mod tests { #[test] fn empty_input_is_an_error_not_a_panic() { - assert!(> as BytesDecode>::bytes_decode(&[]).is_err()); + assert!(> as BytesDecode>::bytes_decode(&[]).is_err()); } #[test] fn truncated_zstd_header_is_an_error_not_a_panic() { // TAG_ZSTD with fewer than the four orig_len header bytes must error, not panic on the slice. - assert!( - > as BytesDecode>::bytes_decode(&[TAG_ZSTD, 1, 2]).is_err() - ); + assert!(> as BytesDecode>::bytes_decode(&[TAG_ZSTD, 1, 2]).is_err()); } #[test] fn unknown_tag_is_an_error_not_a_panic() { - assert!(> as BytesDecode>::bytes_decode(&[9, 0, 0]).is_err()); + assert!(> as BytesDecode>::bytes_decode(&[9, 0, 0]).is_err()); } } diff --git a/packages/evm/core/src/db.rs b/packages/evm/core/src/db.rs index 78954494a8..2be8dc0f5d 100644 --- a/packages/evm/core/src/db.rs +++ b/packages/evm/core/src/db.rs @@ -22,7 +22,7 @@ use serde::{Deserialize, Serialize}; use crate::{ account::{AccountInfoExtended, StoredAccountInfo}, bytecode::StoredBytecode, - compression::CompressedBincode, + compression::CompactBincode, historical::{AccountHistory, HistoricalAccountData}, legacy::{LegacyAccountAttributes, LegacyAddress, LegacyColdWallet}, logger::{LogLevel, Logger}, @@ -151,19 +151,14 @@ pub(crate) struct CommitReceipts { } pub(crate) struct InnerStorage { - pub accounts: heed::Database>, + pub accounts: heed::Database>, pub accounts_history: Option< - heed::Database< - HeedBlockNumber, - CompressedBincode>, - >, + heed::Database>>, >, - pub commits: heed::Database>, - pub contracts: heed::Database>, - pub legacy_attributes: - heed::Database>, - pub legacy_cold_wallets: - heed::Database>, + pub commits: heed::Database>, + pub contracts: heed::Database>, + pub legacy_attributes: heed::Database>, + pub legacy_cold_wallets: heed::Database>, pub storage: heed::Database< AddressWrapper, StorageEntryWrapper, @@ -172,10 +167,10 @@ pub(crate) struct InnerStorage { >, // Carried over from previous database-service.ts lmdb backend pub state: heed::Database>, - pub proofs: heed::Database>, - pub blocks: heed::Database>, + pub proofs: heed::Database>, + pub blocks: heed::Database>, pub blocks_hash_number: heed::Database, - pub transactions: heed::Database>, + pub transactions: heed::Database>, pub transactions_hash_key: heed::Database>, // } @@ -374,36 +369,35 @@ impl PersistentDB { let tx_env = env.clone(); let mut wtxn = tx_env.write_txn()?; - let accounts = env - .create_database::>( - &mut wtxn, - Some("accounts"), - )?; + let accounts = env.create_database::>( + &mut wtxn, + Some("accounts"), + )?; let (accounts_history_db, accounts_history) = match opts.history_size { Some(history_size) if history_size > 0 => { - let db = env.create_database::>>(&mut wtxn, Some("accounts_history")) ?; (Some(db), Some(AccountHistory::new(history_size))) } _ => (None, None), }; - let commits = env.create_database::>( + let commits = env.create_database::>( &mut wtxn, Some("commits"), )?; - let contracts = env.create_database::>( + let contracts = env.create_database::>( &mut wtxn, Some("contracts"), )?; let legacy_attributes = env - .create_database::>( + .create_database::>( &mut wtxn, Some("legacy_attributes"), )?; let legacy_cold_wallets = env - .create_database::>( + .create_database::>( &mut wtxn, Some("legacy_cold_wallets"), )?; @@ -420,11 +414,11 @@ impl PersistentDB { &mut wtxn, Some("state"), )?; - let proofs = env.create_database::>( + let proofs = env.create_database::>( &mut wtxn, Some("proofs"), )?; - let blocks = env.create_database::>( + let blocks = env.create_database::>( &mut wtxn, Some("blocks"), )?; @@ -432,11 +426,10 @@ impl PersistentDB { &mut wtxn, Some("blocks_hash_number"), )?; - let transactions = env - .create_database::>( - &mut wtxn, - Some("transactions"), - )?; + let transactions = env.create_database::>( + &mut wtxn, + Some("transactions"), + )?; let transactions_hash_key = env .create_database::>( &mut wtxn, @@ -480,7 +473,7 @@ impl PersistentDB { inner.accounts.put( &mut wtxn, &AddressWrapper(genesis_info.account), - &CompressedBincode(&StoredAccountInfo::new( + &CompactBincode(&StoredAccountInfo::new( genesis_info.initial_supply, 0, KECCAK_EMPTY, @@ -863,7 +856,7 @@ impl PersistentDB { inner.accounts.put( rwtxn, &address, - &CompressedBincode(&StoredAccountInfo::new( + &CompactBincode(&StoredAccountInfo::new( account.balance, account.nonce, account.code_hash, @@ -893,11 +886,9 @@ impl PersistentDB { // Update legacy attributes for (address, legacy_attributes) in legacy_attributes.into_iter() { let address = AddressWrapper(*address); - inner.legacy_attributes.put( - rwtxn, - &address, - &CompressedBincode(legacy_attributes), - )?; + inner + .legacy_attributes + .put(rwtxn, &address, &CompactBincode(legacy_attributes))?; } // Update legacy cold wallets @@ -906,7 +897,7 @@ impl PersistentDB { inner.legacy_cold_wallets.put( rwtxn, &address, - &CompressedBincode(legacy_cold_wallets), + &CompactBincode(legacy_cold_wallets), )?; } @@ -915,7 +906,7 @@ impl PersistentDB { inner.contracts.put( rwtxn, &HashWrapper(*hash), - &CompressedBincode(&bytecode.clone().into()), + &CompactBincode(&bytecode.clone().into()), )?; } @@ -994,18 +985,16 @@ impl PersistentDB { assert!(legacy_cold_wallet.merge_info.is_none()); legacy_cold_wallet.merge_info.replace((legacy.0, *address)); - inner.legacy_cold_wallets.put( - rwtxn, - key, - &CompressedBincode(&legacy_cold_wallet), - )?; + inner + .legacy_cold_wallets + .put(rwtxn, key, &CompactBincode(&legacy_cold_wallet))?; // The legacy balance has already been applied to the `PendingCommit`, // thus only the legacy attributes need to be moved to a different storage. inner.legacy_attributes.put( rwtxn, &AddressWrapper(*address), - &CompressedBincode(&legacy_cold_wallet.legacy_attributes), + &CompactBincode(&legacy_cold_wallet.legacy_attributes), )?; } @@ -1019,15 +1008,13 @@ impl PersistentDB { } = commit_data; // Update blocks - inner - .blocks - .put(rwtxn, &key.0, &CompressedBincode(header))?; + inner.blocks.put(rwtxn, &key.0, &CompactBincode(header))?; inner .blocks_hash_number .put(rwtxn, &HashWrapper(header.hash), &key.0)?; // Update proofs - inner.proofs.put(rwtxn, &key.0, &CompressedBincode(proof))?; + inner.proofs.put(rwtxn, &key.0, &CompactBincode(proof))?; // Update transactions for (sequence, _) in transactions.iter().enumerate() { @@ -1043,7 +1030,7 @@ impl PersistentDB { inner.transactions.put( rwtxn, &StringWrapper(key), - &CompressedBincode(transaction), + &CompactBincode(transaction), )?; } @@ -1070,7 +1057,7 @@ impl PersistentDB { inner.commits.put( rwtxn, &key.0, - &CompressedBincode(&CommitReceipts { tx_receipts }), + &CompactBincode(&CommitReceipts { tx_receipts }), )?; Ok(()) @@ -1255,7 +1242,7 @@ mod tests { use crate::{ account::StoredAccountInfo, - compression::CompressedBincode, + compression::CompactBincode, db::{ AddressWrapper, BlockHeaderData, CommitData, CommitKey, CommitReceipts, HashWrapper, LegacyAddressWrapper, MAP_SIZE_UNIT, PendingCommit, PersistentDB, PersistentDBOptions, @@ -1680,7 +1667,7 @@ mod tests { .put( &mut wtxn, &AddressWrapper(*address), - &CompressedBincode(&StoredAccountInfo { + &CompactBincode(&StoredAccountInfo { balance: U256::from(index), nonce: index as u64, ..Default::default() @@ -1694,7 +1681,7 @@ mod tests { .put( &mut wtxn, &AddressWrapper(*address), - &CompressedBincode(&LegacyAccountAttributes::default()), + &CompactBincode(&LegacyAccountAttributes::default()), ) .unwrap(); } @@ -1751,7 +1738,7 @@ mod tests { .put( &mut wtxn, &LegacyAddressWrapper(legacy_address), - &CompressedBincode(&LegacyColdWallet { + &CompactBincode(&LegacyColdWallet { address: legacy_address, balance: U256::from(index), legacy_attributes: Default::default(), @@ -1828,7 +1815,7 @@ mod tests { .borrow_mut() .accounts_history .unwrap() - .put(&mut wtxn, &1, &CompressedBincode(&entries)) + .put(&mut wtxn, &1, &CompactBincode(&entries)) .unwrap(); wtxn.commit().unwrap(); @@ -1970,9 +1957,7 @@ mod tests { .put( &mut wtxn, &HashWrapper(hash), - &CompressedBincode( - &Bytecode::new_raw(Bytes::from_static(&[0, 1, 2, 3])).into(), - ), + &CompactBincode(&Bytecode::new_raw(Bytes::from_static(&[0, 1, 2, 3])).into()), ) .unwrap(); @@ -2028,7 +2013,7 @@ mod tests { .put( &mut wtxn, &1, - &CompressedBincode(&BlockHeaderData { + &CompactBincode(&BlockHeaderData { hash: b256!( "0000000000000000000000000000000000000000000000000000000000000001" ), @@ -2101,7 +2086,7 @@ mod tests { .put( &mut wtxn, &1, - &CompressedBincode(&CommitReceipts { + &CompactBincode(&CommitReceipts { tx_receipts, ..Default::default() }), @@ -2158,7 +2143,7 @@ mod tests { .put( &mut wtxn, &block_number, - &CompressedBincode(&CommitReceipts { + &CompactBincode(&CommitReceipts { tx_receipts: receipts, ..Default::default() }), @@ -2216,7 +2201,7 @@ mod tests { .put( &mut wtxn, &AddressWrapper(address), - &CompressedBincode(&LegacyAccountAttributes { + &CompactBincode(&LegacyAccountAttributes { legacy_nonce: Some(1234), second_public_key: Some("key".into()), multi_signature: None, @@ -2252,7 +2237,7 @@ mod tests { .put( &mut wtxn, &1, - &CompressedBincode(&BlockHeaderData { + &CompactBincode(&BlockHeaderData { hash: b256!( "0000000000000000000000000000000000000000000000000000000000000001" ), @@ -2292,7 +2277,7 @@ mod tests { .put( &mut wtxn, &255, - &CompressedBincode(&BlockHeaderData { + &CompactBincode(&BlockHeaderData { number: 255, hash: b256!( "0000000000000000000000000000000000000000000000000000000000000001" @@ -2346,7 +2331,7 @@ mod tests { .put( &mut wtxn, &1, - &CompressedBincode(&BlockHeaderData { + &CompactBincode(&BlockHeaderData { number: 1, hash, ..Default::default() @@ -2382,7 +2367,7 @@ mod tests { .put( &mut wtxn, &1, - &CompressedBincode(&ProofData { + &CompactBincode(&ProofData { round: 1, validator_set: 1234, ..Default::default() @@ -2421,7 +2406,7 @@ mod tests { .put( &mut wtxn, &StringWrapper(key.clone()), - &CompressedBincode(&TransactionData { + &CompactBincode(&TransactionData { tx_hash: hash, ..Default::default() }), diff --git a/packages/evm/core/src/historical.rs b/packages/evm/core/src/historical.rs index 3fbe087e43..f294088065 100644 --- a/packages/evm/core/src/historical.rs +++ b/packages/evm/core/src/historical.rs @@ -6,7 +6,7 @@ use revm::{ state::AccountInfo, }; -use crate::{compression::CompressedBincode, db::Error}; +use crate::{compression::CompactBincode, db::Error}; #[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct HistoricalAccountData { @@ -39,7 +39,7 @@ impl AccountHistory { txn: &mut RwTxn, database: &heed::Database< heed::types::U64, - CompressedBincode>, + CompactBincode>, >, block_number: u64, accounts: Vec<(Address, AccountInfo)>, @@ -58,7 +58,7 @@ impl AccountHistory { .map(|a| (a.0, HistoricalAccountData::from(a.1))) .collect::>(); - database.put(txn, &block_number, &CompressedBincode(&data))?; + database.put(txn, &block_number, &CompactBincode(&data))?; Ok(()) } @@ -68,7 +68,7 @@ impl AccountHistory { txn: &RoTxn, database: &heed::Database< heed::types::U64, - CompressedBincode>, + CompactBincode>, >, block_number: u64, address: &Address, From 20b3ef5c5d054381a7f5f4ccfcbb63944906580c Mon Sep 17 00:00:00 2001 From: oXtxNt9U <120286271+oXtxNt9U@users.noreply.github.com> Date: Fri, 5 Jun 2026 12:10:29 +0900 Subject: [PATCH 04/10] add TransactionKey wrapper --- packages/evm/core/src/db.rs | 99 ++++++++++++++++++++++++++----------- 1 file changed, 71 insertions(+), 28 deletions(-) diff --git a/packages/evm/core/src/db.rs b/packages/evm/core/src/db.rs index 2be8dc0f5d..215345119e 100644 --- a/packages/evm/core/src/db.rs +++ b/packages/evm/core/src/db.rs @@ -77,21 +77,60 @@ impl heed::BytesEncode<'_> for HashWrapper { } } -#[derive(Debug)] -pub(crate) struct StringWrapper(String); -impl heed::BytesEncode<'_> for StringWrapper { - type EItem = StringWrapper; +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct TransactionKey { + pub block_number: u64, + pub index: u16, +} + +impl TransactionKey { + pub fn new(block_number: u64, index: u16) -> Self { + Self { + block_number, + index, + } + } + + // Parses the "-" token exchanged across the napi boundary. + pub fn parse(token: &str) -> Option { + let (block_number, index) = token.split_once('-')?; + Some(Self { + block_number: block_number.parse().ok()?, + index: index.parse().ok()?, + }) + } + + pub fn to_token(&self) -> String { + format!("{}-{}", self.block_number, self.index) + } +} + +impl heed::BytesEncode<'_> for TransactionKey { + type EItem = TransactionKey; fn bytes_encode(item: &Self::EItem) -> Result, heed::BoxedError> { - Ok(Cow::Borrowed(item.0.as_bytes())) + let mut buffer = Vec::with_capacity(10); + buffer.extend_from_slice(&item.block_number.to_be_bytes()); + buffer.extend_from_slice(&item.index.to_be_bytes()); + Ok(Cow::Owned(buffer)) } } -impl heed::BytesDecode<'_> for StringWrapper { - type DItem = StringWrapper; +impl heed::BytesDecode<'_> for TransactionKey { + type DItem = TransactionKey; fn bytes_decode(bytes: &'_ [u8]) -> Result { - Ok(StringWrapper(String::from_utf8(bytes.into())?)) + let Some((block_number, rest)) = bytes.split_first_chunk::<8>() else { + return Err("TransactionKey: truncated key".into()); + }; + let Some((index, _)) = rest.split_first_chunk::<2>() else { + return Err("TransactionKey: truncated key".into()); + }; + + Ok(TransactionKey { + block_number: u64::from_be_bytes(*block_number), + index: u16::from_be_bytes(*index), + }) } } @@ -170,8 +209,8 @@ pub(crate) struct InnerStorage { pub proofs: heed::Database>, pub blocks: heed::Database>, pub blocks_hash_number: heed::Database, - pub transactions: heed::Database>, - pub transactions_hash_key: heed::Database>, + pub transactions: heed::Database>, + pub transactions_hash_key: heed::Database, // } @@ -426,15 +465,14 @@ impl PersistentDB { &mut wtxn, Some("blocks_hash_number"), )?; - let transactions = env.create_database::>( + let transactions = env.create_database::>( &mut wtxn, Some("transactions"), )?; - let transactions_hash_key = env - .create_database::>( - &mut wtxn, - Some("transactions_hash_key"), - )?; + let transactions_hash_key = env.create_database::( + &mut wtxn, + Some("transactions_hash_key"), + )?; wtxn.commit()?; @@ -1018,7 +1056,9 @@ impl PersistentDB { // Update transactions for (sequence, _) in transactions.iter().enumerate() { - let key = format!("{}-{}", key.0, sequence); + debug_assert!(sequence <= u16::MAX as usize); + + let key = TransactionKey::new(key.0, sequence as u16); let transaction = &transactions[sequence]; inner.transactions_hash_key.put( @@ -1027,11 +1067,9 @@ impl PersistentDB { &key, )?; - inner.transactions.put( - rwtxn, - &StringWrapper(key), - &CompactBincode(transaction), - )?; + inner + .transactions + .put(rwtxn, &key, &CompactBincode(transaction))?; } // Update state @@ -1155,15 +1193,19 @@ impl PersistentDB { Ok(inner.blocks.get(&rtxn, &block_number)?.map(|data| data.0)) } - pub fn get_transaction_data(&self, key: String) -> Result, Error> { + pub fn get_transaction(&self, key: TransactionKey) -> Result, Error> { let env = self.env.clone(); let rtxn = env.read_txn().expect("read"); let inner = self.inner.borrow(); - Ok(inner - .transactions - .get(&rtxn, &StringWrapper(key))? - .map(|data| data.0)) + Ok(inner.transactions.get(&rtxn, &key)?.map(|data| data.0)) + } + + pub fn get_transaction_data(&self, key: String) -> Result, Error> { + match TransactionKey::parse(&key) { + Some(key) => self.get_transaction(key), + None => Ok(None), + } } pub fn get_transaction_key_by_hash(&self, tx_hash: B256) -> Result, Error> { @@ -1173,7 +1215,8 @@ impl PersistentDB { Ok(inner .transactions_hash_key - .get(&rtxn, &HashWrapper(tx_hash))?) + .get(&rtxn, &HashWrapper(tx_hash))? + .map(|key| key.to_token())) } } From 9c676e556ecde4e6e1eb9b13681962def4533086 Mon Sep 17 00:00:00 2001 From: oXtxNt9U <120286271+oXtxNt9U@users.noreply.github.com> Date: Fri, 5 Jun 2026 12:11:35 +0900 Subject: [PATCH 05/10] tests --- packages/evm/core/src/db.rs | 105 +++++++++++++++++++++++++++++------- 1 file changed, 86 insertions(+), 19 deletions(-) diff --git a/packages/evm/core/src/db.rs b/packages/evm/core/src/db.rs index 215345119e..354492d79b 100644 --- a/packages/evm/core/src/db.rs +++ b/packages/evm/core/src/db.rs @@ -1289,7 +1289,7 @@ mod tests { db::{ AddressWrapper, BlockHeaderData, CommitData, CommitKey, CommitReceipts, HashWrapper, LegacyAddressWrapper, MAP_SIZE_UNIT, PendingCommit, PersistentDB, PersistentDBOptions, - ProofData, StaticStringWrapper, StorageEntryWrapper, StringWrapper, TransactionData, + ProofData, StaticStringWrapper, StorageEntryWrapper, TransactionData, TransactionKey, next_map_size, }, historical::HistoricalAccountData, @@ -1895,17 +1895,6 @@ mod tests { assert_eq!(legacy_address, deserialized.0); } - #[test] - fn test_string_wrapper() { - let string = "test".to_owned(); - - let wrapper = StringWrapper(string); - let serialized = ::bytes_encode(&wrapper).expect("ok"); - let deserialized = ::bytes_decode(&serialized).expect("ok"); - - assert_eq!("test", deserialized.0); - } - #[test] fn test_static_string_wrapper() { let string = "test"; @@ -2431,12 +2420,87 @@ mod tests { ); } + #[test] + fn test_transaction_key_encode_decode_roundtrip() { + for (block_number, index) in [(0u64, 0u16), (1, 2), (5, 9999), (u64::MAX, u16::MAX)] { + let key = TransactionKey::new(block_number, index); + let encoded = ::bytes_encode(&key).unwrap(); + assert_eq!(encoded.len(), 10, "key is 8-byte block + 2-byte index"); + + let decoded = ::bytes_decode(&encoded).unwrap(); + assert_eq!(decoded, key); + } + } + + #[test] + fn test_transaction_key_orders_by_block_then_index() { + // The transactions DB relies on byte (memcmp) key order matching numeric + // (block_number, index) order, so get_commits_by_block_range can scan by block number. + // Big-endian encoding is what guarantees this (e.g. block 2 sorts before block 10). + let ascending = [ + TransactionKey::new(0, 0), + TransactionKey::new(0, 1), + TransactionKey::new(0, u16::MAX), + TransactionKey::new(1, 0), // block 1 sorts after every transaction of block 0 + TransactionKey::new(2, 0), + TransactionKey::new(10, 0), // numeric order, not lexicographic on decimal + TransactionKey::new(u64::MAX, 0), + TransactionKey::new(u64::MAX, u16::MAX), + ]; + + for window in ascending.windows(2) { + let lo = ::bytes_encode(&window[0]).unwrap(); + let hi = ::bytes_encode(&window[1]).unwrap(); + assert!(lo < hi, "encoded keys must sort by (block, index)"); + // The derived Ord must agree with the on-disk byte order. + assert!(window[0] < window[1]); + } + } + + #[test] + fn test_transaction_key_token_roundtrip_and_lenient_parse() { + assert_eq!(TransactionKey::new(5, 2).to_token(), "5-2"); + + let key = TransactionKey::new(123, 45); + assert_eq!(TransactionKey::parse(&key.to_token()), Some(key)); + + // Malformed or out-of-range tokens parse to None (treated as "no such transaction"). + assert_eq!(TransactionKey::parse("nope"), None); + assert_eq!(TransactionKey::parse("-5"), None); + assert_eq!(TransactionKey::parse("1-2-3"), None); + assert_eq!(TransactionKey::parse("1-70000"), None); // index exceeds u16::MAX + } + + #[test] + fn test_transaction_key_range_bounds_capture_a_block_range() { + // Mirrors the scan bounds get_commits_by_block_range builds for [from, to]. + let from = TransactionKey::new(5, 0); + let to = TransactionKey::new(7, u16::MAX); + + for block in 5..=7u64 { + for index in [0u16, 1, 1000, u16::MAX] { + let key = TransactionKey::new(block, index); + assert!( + key >= from && key <= to, + "{block}-{index} should be within range" + ); + } + } + + // The neighbouring blocks fall outside the range on each side. + assert!(TransactionKey::new(4, u16::MAX) < from); + assert!(TransactionKey::new(8, 0) > to); + } + #[test] fn test_get_transaction_data() { let db = create_temp_database(); - let key = String::from("tx-1"); - assert_eq!(db.get_transaction_data(key.clone()).unwrap(), None); + // Lookups go through the "-" token; before anything is written it is absent, + // and malformed/out-of-range tokens resolve to None rather than erroring. + assert_eq!(db.get_transaction_data("1-0".into()).unwrap(), None); + assert_eq!(db.get_transaction_data("not-a-key".into()).unwrap(), None); + assert_eq!(db.get_transaction_data("1-70000".into()).unwrap(), None); let hash = b256!("0000000000000000000000000000000000000000000000000000000000000001"); @@ -2448,7 +2512,7 @@ mod tests { .transactions .put( &mut wtxn, - &StringWrapper(key.clone()), + &TransactionKey::new(1, 0), &CompactBincode(&TransactionData { tx_hash: hash, ..Default::default() @@ -2460,7 +2524,7 @@ mod tests { } assert_eq!( - db.get_transaction_data(key.clone()).unwrap(), + db.get_transaction_data("1-0".into()).unwrap(), Some(TransactionData { tx_hash: hash, ..Default::default() @@ -2472,7 +2536,6 @@ mod tests { fn test_get_transaction_hash_by_hash() { let db = create_temp_database(); - let key = String::from("tx-1"); let hash = b256!("0000000000000000000000000000000000000000000000000000000000000001"); assert_eq!(db.get_transaction_key_by_hash(hash).unwrap(), None); @@ -2483,13 +2546,17 @@ mod tests { db.inner .borrow_mut() .transactions_hash_key - .put(&mut wtxn, &HashWrapper(hash), &key) + .put(&mut wtxn, &HashWrapper(hash), &TransactionKey::new(1, 0)) .unwrap(); wtxn.commit().unwrap(); } - assert_eq!(db.get_transaction_key_by_hash(hash).unwrap(), Some(key)); + // The stored typed key is returned to the napi boundary as its "-" token. + assert_eq!( + db.get_transaction_key_by_hash(hash).unwrap(), + Some("1-0".to_string()) + ); } #[test] From 29110f9f1f09f70db2d23764f05f0e0dceb8011f Mon Sep 17 00:00:00 2001 From: oXtxNt9U <120286271+oXtxNt9U@users.noreply.github.com> Date: Fri, 5 Jun 2026 14:12:51 +0900 Subject: [PATCH 06/10] add `getCommitsByBlockRange` --- .../contracts/source/contracts/evm/storage.ts | 5 + packages/evm-service/source/instances/evm.ts | 8 + packages/evm/bindings/src/lib.rs | 63 ++++++++ packages/evm/core/src/db.rs | 148 ++++++++++++++++++ 4 files changed, 224 insertions(+) diff --git a/packages/contracts/source/contracts/evm/storage.ts b/packages/contracts/source/contracts/evm/storage.ts index 3f4ae4de01..1957a54334 100644 --- a/packages/contracts/source/contracts/evm/storage.ts +++ b/packages/contracts/source/contracts/evm/storage.ts @@ -55,6 +55,11 @@ export interface Storage { getBlockHeaderData(blockNumber: number): Promise; getBlockNumberByHash(blockHash: string): Promise; getCommitData(blockNumber: number): Promise; + getCommitsByBlockRange( + fromBlockNumber: number, + toBlockNumber: number, + maxBytes: number, + ): Promise; getTransactionData(key: string): Promise; getTransactionKeyByHash(txHash: string): Promise; isEmpty(): Promise; diff --git a/packages/evm-service/source/instances/evm.ts b/packages/evm-service/source/instances/evm.ts index 34be839bbc..a716ccfb54 100644 --- a/packages/evm-service/source/instances/evm.ts +++ b/packages/evm-service/source/instances/evm.ts @@ -208,6 +208,14 @@ export class EvmInstance implements Contracts.Evm.Instance, Contracts.Evm.Storag return result; } + public async getCommitsByBlockRange( + fromBlockNumber: number, + toBlockNumber: number, + maxBytes: number, + ): Promise { + return this.#evm.getCommitsByBlockRange(BigInt(fromBlockNumber), BigInt(toBlockNumber), BigInt(maxBytes)); + } + public async getTransactionData(key: string): Promise { const result = await this.#evm.getTransactionData(key); if (result === null || result === undefined) { diff --git a/packages/evm/bindings/src/lib.rs b/packages/evm/bindings/src/lib.rs index f4936d0340..5c21753cb9 100644 --- a/packages/evm/bindings/src/lib.rs +++ b/packages/evm/bindings/src/lib.rs @@ -915,6 +915,27 @@ impl EvmInner { Ok(Some((proof, header, txs))) } + pub fn get_commits_by_block_range( + &mut self, + from_block_number: u64, + to_block_number: u64, + max_bytes: u64, + ) -> std::result::Result< + Vec<(ProofData, BlockHeaderData, Vec)>, + EVMError, + > { + match self.persistent_db.get_commits_by_block_range( + from_block_number, + to_block_number, + max_bytes, + ) { + Ok(commits) => Ok(commits), + Err(err) => Err(EVMError::Database( + format!("failed reading commits by block range: {}", err).into(), + )), + } + } + pub fn get_transaction_data( &mut self, key: String, @@ -1625,6 +1646,33 @@ impl JsEvmWrapper { ) } + #[napi] + pub fn get_commits_by_block_range<'env>( + &mut self, + env: &'env Env, + from_block_number: BigInt, + to_block_number: BigInt, + max_bytes: BigInt, + ) -> Result>> { + let from_block_number = from_block_number.get_u64().1; + let to_block_number = to_block_number.get_u64().1; + let max_bytes = max_bytes.get_u64().1; + env.spawn_future_with_callback( + Self::get_commits_by_block_range_async( + self.evm.clone(), + from_block_number, + to_block_number, + max_bytes, + ), + |_, result| { + Ok(result + .into_iter() + .map(|(proof, header, txs)| JsCommitData::new(proof, header, txs)) + .collect()) + }, + ) + } + #[napi] pub fn get_transaction_data<'env>( &mut self, @@ -2050,6 +2098,21 @@ impl JsEvmWrapper { } } + async fn get_commits_by_block_range_async( + evm: Arc>, + from_block_number: u64, + to_block_number: u64, + max_bytes: u64, + ) -> Result)>> { + let mut lock = evm.lock().await; + let result = lock.get_commits_by_block_range(from_block_number, to_block_number, max_bytes); + + match result { + Ok(result) => Result::Ok(result), + Err(err) => Result::Err(serde::de::Error::custom(err)), + } + } + async fn get_transaction_data_async( evm: Arc>, key: String, diff --git a/packages/evm/core/src/db.rs b/packages/evm/core/src/db.rs index 354492d79b..294310e7b6 100644 --- a/packages/evm/core/src/db.rs +++ b/packages/evm/core/src/db.rs @@ -642,6 +642,57 @@ impl PersistentDB { } } + pub fn get_commits_by_block_range( + &self, + from_block_number: u64, + to_block_number: u64, + max_bytes: u64, + ) -> Result)>, Error> { + // Per-commit fixed cost charged against the budget on top of the block's transaction payload, + // so that a long run of (near-)empty blocks is still bounded by block count, not just bytes. + const PER_COMMIT_OVERHEAD_BYTES: u64 = 1024; + + let tx_env = self.env.read_txn()?; + let inner = self.inner.borrow(); + + let capacity = to_block_number.saturating_sub(from_block_number).min(512) as usize; + let mut commits = Vec::with_capacity(capacity); + let mut accumulated_bytes: u64 = 0; + + for item in inner + .blocks + .range(&tx_env, &(from_block_number..=to_block_number))? + { + let (block_number, header) = item?; + + // Headers and proofs are written together per commit; a missing proof means the end of + // the available data has been reached. + let Some(proof) = inner.proofs.get(&tx_env, &block_number)? else { + break; + }; + + // Collect this block's transactions via a single range scan over its key prefix; the + // keys sort by (block_number, index), so they arrive in index order. + let mut transactions = Vec::with_capacity(header.0.transactions_count as usize); + let tx_from = TransactionKey::new(block_number, 0); + let tx_to = TransactionKey::new(block_number, u16::MAX); + for tx_item in inner.transactions.range(&tx_env, &(tx_from..=tx_to))? { + let (_, transaction) = tx_item?; + transactions.push(transaction.0); + } + + let estimated_bytes = header.0.payload_size as u64 + PER_COMMIT_OVERHEAD_BYTES; + commits.push((proof.0, header.0, transactions)); + + accumulated_bytes += estimated_bytes; + if accumulated_bytes >= max_bytes { + break; + } + } + + Ok(commits) + } + pub fn get_historical_account_info( &self, block_number: u64, @@ -2094,6 +2145,103 @@ mod tests { assert_eq!(db.genesis_info, Some(Default::default())); } + #[test] + fn test_get_commits_by_block_range() { + let db = create_temp_database(); + + // Empty range before anything is written. + assert!( + db.get_commits_by_block_range(1, 3, u64::MAX) + .unwrap() + .is_empty() + ); + + // Write blocks 1..=3; block N has N transactions, inserted in reverse sequence order to + // prove the reader returns them ordered by (block_number, sequence). + { + let mut wtxn = db.env.write_txn().unwrap(); + let inner = db.inner.borrow(); + + for block_number in 1u64..=3 { + inner + .blocks + .put( + &mut wtxn, + &block_number, + &CompactBincode(&BlockHeaderData { + number: block_number as u32, + transactions_count: block_number as u16, + ..Default::default() + }), + ) + .unwrap(); + + inner + .proofs + .put( + &mut wtxn, + &block_number, + &CompactBincode(&ProofData { + round: block_number as u32, + ..Default::default() + }), + ) + .unwrap(); + + for sequence in (0..block_number).rev() { + inner + .transactions + .put( + &mut wtxn, + &TransactionKey::new(block_number, sequence as u16), + &CompactBincode(&TransactionData { + block_number: block_number as u32, + index: sequence as u32, + tx_hash: B256::from(U256::from(block_number * 100 + sequence)), + ..Default::default() + }), + ) + .unwrap(); + } + } + + wtxn.commit().unwrap(); + } + + // Full range (unbounded budget): blocks ascending, transactions per block in sequence order. + let commits = db.get_commits_by_block_range(1, 3, u64::MAX).unwrap(); + assert_eq!(commits.len(), 3); + + for (index, (proof, header, transactions)) in commits.iter().enumerate() { + let block_number = (index + 1) as u64; + + assert_eq!(header.number, block_number as u32); + assert_eq!(proof.round, block_number as u32); + assert_eq!(transactions.len(), block_number as usize); + + for (sequence, transaction) in transactions.iter().enumerate() { + assert_eq!(transaction.index, sequence as u32); + assert_eq!(transaction.block_number, block_number as u32); + } + } + + // Sub-range returns only the requested block. + let commits = db.get_commits_by_block_range(2, 2, u64::MAX).unwrap(); + assert_eq!(commits.len(), 1); + assert_eq!(commits[0].1.number, 2); + assert_eq!(commits[0].2.len(), 2); + + // Range extending past the tip stops at the last available block. + let commits = db.get_commits_by_block_range(2, 99, u64::MAX).unwrap(); + assert_eq!(commits.len(), 2); + + // A tiny byte budget stops early but always makes progress (returns at least one commit), + // so callers can resume from the last returned block. + let commits = db.get_commits_by_block_range(1, 3, 1).unwrap(); + assert_eq!(commits.len(), 1); + assert_eq!(commits[0].1.number, 1); + } + #[test] fn test_get_receipts() { let db = create_temp_database(); From 8da998f61d9096a1cdaef8ca4c9c9b9a52b4072a Mon Sep 17 00:00:00 2001 From: oXtxNt9U <120286271+oXtxNt9U@users.noreply.github.com> Date: Fri, 5 Jun 2026 14:14:57 +0900 Subject: [PATCH 07/10] use new evm to read commits --- .../contracts/source/contracts/database.ts | 4 +- packages/database/source/database-service.ts | 73 ++++++++----------- 2 files changed, 32 insertions(+), 45 deletions(-) diff --git a/packages/contracts/source/contracts/database.ts b/packages/contracts/source/contracts/database.ts index dd5ba72645..0c2abfa0b8 100644 --- a/packages/contracts/source/contracts/database.ts +++ b/packages/contracts/source/contracts/database.ts @@ -14,8 +14,8 @@ export interface DatabaseService extends CommitHandler { getLastCommit(): Promise; hasCommitByHash(blockHash: string): Promise; - findCommitBuffers(start: number, end: number): Promise; - readCommits(start: number, end: number): AsyncGenerator; + findCommitBuffers(start: number, end: number, maxBytes: number): Promise; + readCommits(start: number, end: number, maxBytes: number): AsyncGenerator; getBlock(blockNumber: number): Promise; getBlockByHash(blockHash: string): Promise; diff --git a/packages/database/source/database-service.ts b/packages/database/source/database-service.ts index 9abf635ba0..57cb7a3457 100644 --- a/packages/database/source/database-service.ts +++ b/packages/database/source/database-service.ts @@ -38,26 +38,14 @@ export class DatabaseService implements Contracts.Database.DatabaseService { return blockNumber !== undefined; } - public async findCommitBuffers(start: number, end: number): Promise { - const blockNumbers: number[] = []; + public async findCommitBuffers(start: number, end: number, maxBytes: number): Promise { + const buffers: Buffer[] = []; - for (const blockNumber of this.#range(start, end)) { - blockNumbers.push(blockNumber); + for await (const commit of this.readCommits(start, end, maxBytes)) { + buffers.push(Buffer.from(commit.serialized, "hex")); } - const buffers = await Promise.all( - blockNumbers.map(async (blockNumber: number) => { - const commitStorage = await this.#readCommitStorage(blockNumber); - if (!commitStorage) { - return; - } - - const commit = await this.commitFactory.fromStorage(commitStorage); - return Buffer.from(commit.serialized, "hex"); - }), - ); - - return buffers.filter((commit) => !!commit); + return buffers; } public async getBlock(blockNumber: number): Promise { @@ -106,12 +94,13 @@ export class DatabaseService implements Contracts.Database.DatabaseService { } public async findBlocks(start: number, end: number): Promise { - const commitBuffers = await this.findCommitBuffers(start, end); + const blocks: Contracts.Crypto.Block[] = []; + + for await (const commit of this.readCommits(start, end, Number.MAX_SAFE_INTEGER)) { + blocks.push(commit.block); + } - return await this.#map( - commitBuffers, - async (buffer: Buffer) => (await this.commitFactory.fromBytes(buffer)).block, - ); + return blocks; } public async getTransactionByHash(transactionHash: string): Promise { @@ -143,16 +132,29 @@ export class DatabaseService implements Contracts.Database.DatabaseService { return this.#readTransaction(`${blockNumber}-${index}`); } - public async *readCommits(start: number, end: number): AsyncGenerator { - for (let blockNumber = start; blockNumber <= end; blockNumber++) { - const data = await this.#readCommitStorage(blockNumber); + public async *readCommits(start: number, end: number, maxBytes: number): AsyncGenerator { + let from = start; + let remainingBytes = maxBytes; - if (!data) { + while (from <= end) { + const commitsData = await this.storage.getCommitsByBlockRange(from, end, remainingBytes); + if (commitsData.length === 0) { return; } - const commit = await this.commitFactory.fromStorage(data); - yield commit; + let lastBlockNumber = from; + for (const data of commitsData) { + const commit = await this.commitFactory.fromStorage(data); + lastBlockNumber = commit.block.number; + yield commit; + + remainingBytes -= commit.serialized.length / 2; + if (remainingBytes <= 0) { + return; + } + } + + from = lastBlockNumber + 1; } } @@ -201,19 +203,4 @@ export class DatabaseService implements Contracts.Database.DatabaseService { return this.transactionFactory.fromStorage({ ...transactionStorageData, blockHash: blockHeaderData.hash }); } - - async #map(data: U[], callback: (...arguments_: U[]) => Promise): Promise { - const result: T[] = []; - for (const [index, datum] of data.entries()) { - result[index] = await callback(datum); - } - - return result; - } - - *#range(start: number, end: number): Generator { - for (let index = start; index <= end; index++) { - yield index; - } - } } From b735c417139c4458ca4de9fc49ec3e89ae64fd76 Mon Sep 17 00:00:00 2001 From: oXtxNt9U <120286271+oXtxNt9U@users.noreply.github.com> Date: Fri, 5 Jun 2026 14:15:38 +0900 Subject: [PATCH 08/10] pass maxBytes --- packages/api-sync/source/restore.ts | 2 +- packages/p2p/source/socket-server/controllers/get-blocks.ts | 3 ++- tests/functional/consensus/source/utilities.ts | 5 ++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/api-sync/source/restore.ts b/packages/api-sync/source/restore.ts index bc80f15e71..9716283bc7 100644 --- a/packages/api-sync/source/restore.ts +++ b/packages/api-sync/source/restore.ts @@ -324,7 +324,7 @@ export class Restore { const fromBlockNumber = Math.min(currentBlockNumber, mostRecentCommit.block.number); const toBlockNumber = Math.min(currentBlockNumber + BATCH_SIZE - 1, mostRecentCommit.block.number); - const commits = this.databaseService.readCommits(fromBlockNumber, toBlockNumber); + const commits = this.databaseService.readCommits(fromBlockNumber, toBlockNumber, Number.MAX_SAFE_INTEGER); const blocks: Models.Block[] = []; const transactions: Models.Transaction[] = []; diff --git a/packages/p2p/source/socket-server/controllers/get-blocks.ts b/packages/p2p/source/socket-server/controllers/get-blocks.ts index 0d8f6bb267..bd6f0dcd31 100644 --- a/packages/p2p/source/socket-server/controllers/get-blocks.ts +++ b/packages/p2p/source/socket-server/controllers/get-blocks.ts @@ -31,14 +31,15 @@ export class GetBlocksController implements Contracts.P2P.Controller { return { blocks: [] }; } + const maxPayload = constants.MAX_PAYLOAD_CLIENT; const commits: Buffer[] = await this.database.findCommitBuffers( requestBlockNumber, requestBlockNumber + requestBlockLimit - 1, + maxPayload, ); // Only return the blocks fetched while we are below the p2p maxPayload limit const blocksToReturn: Buffer[] = []; - const maxPayload = constants.MAX_PAYLOAD_CLIENT; let totalSize = 0; for (const commit of commits) { diff --git a/tests/functional/consensus/source/utilities.ts b/tests/functional/consensus/source/utilities.ts index a81210e21a..dc405df300 100644 --- a/tests/functional/consensus/source/utilities.ts +++ b/tests/functional/consensus/source/utilities.ts @@ -222,9 +222,8 @@ export const getLastCommit = async (app: Contracts.Kernel.Application): Promise< const [serialized] = await databaseService.findCommitBuffers( lasCommit.block.number, lasCommit.block.number, + Number.MAX_SAFE_INTEGER, ); - return app - .get(Identifiers.Cryptography.Commit.Factory) - .fromBytes(serialized); + return app.get(Identifiers.Cryptography.Commit.Factory).fromBytes(serialized); }; From 5aef98e23730c73ca67ed21e4ef439e3db6c3458 Mon Sep 17 00:00:00 2001 From: oXtxNt9U <120286271+oXtxNt9U@users.noreply.github.com> Date: Fri, 5 Jun 2026 14:48:46 +0900 Subject: [PATCH 09/10] fix --- packages/database/source/database-service.test.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/database/source/database-service.test.ts b/packages/database/source/database-service.test.ts index fa5f9e1ca3..633074a365 100644 --- a/packages/database/source/database-service.test.ts +++ b/packages/database/source/database-service.test.ts @@ -83,13 +83,13 @@ describe<{ }); it("findCommitBuffers - should be ok", async ({ databaseService }) => { - const commits = await databaseService.findCommitBuffers(1, 2); + const commits = await databaseService.findCommitBuffers(1, 2, Number.MAX_SAFE_INTEGER); assert.empty(commits); }); it("readCommits - should be ok", async ({ databaseService }) => { const commits = []; - for await (const commit of databaseService.readCommits(1, 2)) { + for await (const commit of databaseService.readCommits(1, 2, Number.MAX_SAFE_INTEGER)) { commits.push(commit); } @@ -197,7 +197,6 @@ describe<{ from: transaction.from, gasLimit: BigInt(transaction.gasLimit), gasPrice: BigInt(transaction.gasPrice), - index: transaction.transactionIndex, nonce: transaction.nonce, specId: Enums.Evm.SpecId.LATEST, to: transaction.to, @@ -237,7 +236,9 @@ describe<{ }); it("#findCommitBuffers - should return commit buffer", async ({ databaseService, genesisCommit }) => { - assert.equal(await databaseService.findCommitBuffers(0, 1), [Buffer.from(genesisCommit.serialized, "hex")]); + assert.equal(await databaseService.findCommitBuffers(0, 1, Number.MAX_SAFE_INTEGER), [ + Buffer.from(genesisCommit.serialized, "hex"), + ]); }); it("#getBlock - should return block", async ({ databaseService, genesisCommit }) => { @@ -269,7 +270,7 @@ describe<{ it("#readCommits - should return commits", async ({ databaseService, genesisCommit }) => { const commits = []; - for await (const commit of databaseService.readCommits(0, 1)) { + for await (const commit of databaseService.readCommits(0, 1, Number.MAX_SAFE_INTEGER)) { commits.push(commit); } assert.equal( From 55368910e0d77ab91a53a72cd5ed6195114b1cab Mon Sep 17 00:00:00 2001 From: oXtxNt9U <120286271+oXtxNt9U@users.noreply.github.com> Date: Fri, 5 Jun 2026 15:38:04 +0900 Subject: [PATCH 10/10] ensure no negative start --- packages/database/source/database-service.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/database/source/database-service.ts b/packages/database/source/database-service.ts index 57cb7a3457..60f0da971b 100644 --- a/packages/database/source/database-service.ts +++ b/packages/database/source/database-service.ts @@ -133,7 +133,7 @@ export class DatabaseService implements Contracts.Database.DatabaseService { } public async *readCommits(start: number, end: number, maxBytes: number): AsyncGenerator { - let from = start; + let from = Math.max(0, start); let remainingBytes = maxBytes; while (from <= end) {