perf(store): precompute account state forest mutations during block application

bin/stress-test/Cargo.toml

@@ -28,3 +28,6 @@ miden-standards  = { workspace = true }
 rand             = { workspace = true }
 rayon            = { workspace = true }
 tokio            = { workspace = true }
+
+[dev-dependencies]
+tempfile = { workspace = true }

bin/stress-test/README.md

@@ -17,6 +17,29 @@ point-in-time comparison rather than current guarantees.
 Use the binary help output for the current command and configuration surface. The help output is the source of truth for
 flags and environment variables.
 
+### Large public account storage map
+
+`seed-store` can create an exact number of accounts with a deterministic storage map on every public account. The
+following command creates one public account containing 250,000 entries in one map, then applies one partial account
+update that changes a single entry in that map:
+
+```sh
+cargo run --release --locked -p miden-node-stress-test -- \
+  seed-store \
+  --data-directory /tmp/miden-large-storage-map \
+  --num-accounts 1 \
+  --public-accounts-percentage 100 \
+  --storage-map-entries 250000 \
+  --vault-entries 1 \
+  --account-update-blocks 1
+```
+
+Account creation and account updates each require a preceding note-emission block. The block metrics label these phases
+separately, so `account-update` rows measure the partial public-account updates rather than their setup work. The
+generated account IDs are written to `accounts.txt` in the data directory. If a full public account state would exceed
+the protocol's transaction account-update size limit, `seed-store` inserts that account at genesis automatically; its
+subsequent partial updates still use normal blocks and appear as `account-update` rows.
+
 ## Benchmark Results
 
 The following reference results were obtained using a store with 100k accounts, half of which are public.

bin/stress-test/src/main.rs

@@ -1,3 +1,4 @@
+use std::num::NonZeroUsize;
 use std::path::PathBuf;
 
 use clap::{Parser, Subcommand};
@@ -34,22 +35,28 @@ pub enum Command {
 
         /// Number of accounts to create.
         #[arg(short, long, value_name = "NUM_ACCOUNTS")]
-        num_accounts: usize,
+        num_accounts: NonZeroUsize,
 
         /// Percentage of accounts that will be created as public accounts. The rest will be private
         /// accounts.
-        #[arg(short, long, value_name = "PUBLIC_ACCOUNTS_PERCENTAGE", default_value = "0")]
+        #[arg(
+            short,
+            long,
+            value_name = "PUBLIC_ACCOUNTS_PERCENTAGE",
+            default_value = "0",
+            value_parser = clap::value_parser!(u8).range(0..=100)
+        )]
         public_accounts_percentage: u8,
 
         /// Number of entries to add to a deterministic storage map on every public account.
         #[arg(long, value_name = "STORAGE_MAP_ENTRIES", default_value = "0")]
-        storage_map_entries: usize,
+        storage_map_entries: u32,
 
         /// Number of distinct vault assets to add to every public account.
         #[arg(long, value_name = "VAULT_ENTRIES", default_value = "1")]
-        vault_entries: usize,
+        vault_entries: NonZeroUsize,
 
-        /// Number of post-initialization blocks to generate with random account updates.
+        /// Number of post-initialization blocks containing public account updates.
         #[arg(long, value_name = "ACCOUNT_UPDATE_BLOCKS", default_value = "0")]
         account_update_blocks: usize,
     },
@@ -124,10 +131,10 @@ async fn main() {
         } => {
             Box::pin(seed_store(
                 data_directory,
-                num_accounts,
+                num_accounts.get(),
                 public_accounts_percentage,
                 storage_map_entries,
-                vault_entries,
+                vault_entries.get(),
                 account_update_blocks,
             ))
             .await;

bin/stress-test/src/seeding/metrics.rs

@@ -25,14 +25,41 @@ const SQLITE_INDEXES: &[&str] = &[
     "idx_transactions_block_num",
 ];
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(super) enum BlockKind {
+    AccountNoteEmission,
+    AccountCreation,
+    UpdateNoteEmission,
+    AccountUpdate,
+}
+
+impl Display for BlockKind {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let name = match self {
+            Self::AccountNoteEmission => "account-note-emission",
+            Self::AccountCreation => "account-creation",
+            Self::UpdateNoteEmission => "update-note-emission",
+            Self::AccountUpdate => "account-update",
+        };
+        f.write_str(name)
+    }
+}
+
+struct BlockMetric {
+    kind: BlockKind,
+    insertion_time: Duration,
+    get_block_inputs_time: Duration,
+    block_size: usize,
+    transaction_count: usize,
+    public_account_updates: usize,
+}
+
 /// Metrics struct to show the results of the stress test
 pub struct SeedingMetrics {
-    insertion_time_per_block: Vec<Duration>,
-    get_block_inputs_time_per_block: Vec<Duration>,
+    blocks: Vec<BlockMetric>,
+    pending_get_block_inputs_time: Option<Duration>,
     get_batch_inputs_time_per_block: Vec<Duration>,
-    bytes_per_block: Vec<usize>,
-    num_insertions: u32,
-    store_file_sizes: Vec<u64>,
+    store_file_sizes: Vec<(usize, u64)>,
     initial_store_size: u64,
     store_file: PathBuf,
 }
@@ -42,32 +69,47 @@ impl SeedingMetrics {
     pub fn new(store_file: PathBuf) -> Self {
         let initial_store_size = get_store_size(&store_file);
         Self {
-            insertion_time_per_block: Vec::new(),
-            get_block_inputs_time_per_block: Vec::new(),
+            blocks: Vec::new(),
+            pending_get_block_inputs_time: None,
             get_batch_inputs_time_per_block: Vec::new(),
-            bytes_per_block: Vec::new(),
-            num_insertions: 0,
             store_file_sizes: Vec::new(),
             initial_store_size,
             store_file,
         }
     }
 
     /// Tracks a new block insertion to the metrics, with the insertion time and size of the block.
-    pub fn track_block_insertion(&mut self, insertion_time: Duration, block_size: usize) {
-        self.insertion_time_per_block.push(insertion_time);
-        self.bytes_per_block.push(block_size);
-        self.num_insertions += 1;
+    pub fn track_block_insertion(
+        &mut self,
+        kind: BlockKind,
+        insertion_time: Duration,
+        block_size: usize,
+        transaction_count: usize,
+        public_account_updates: usize,
+    ) {
+        self.blocks.push(BlockMetric {
+            kind,
+            insertion_time,
+            get_block_inputs_time: self.pending_get_block_inputs_time.take().unwrap_or_default(),
+            block_size,
+            transaction_count,
+            public_account_updates,
+        });
     }
 
     /// Tracks the size of the store file.
     pub fn record_store_size(&mut self) {
-        self.store_file_sizes.push(get_store_size(&self.store_file));
+        if let Some(block_index) = self.blocks.len().checked_sub(1) {
+            self.store_file_sizes.push((block_index, get_store_size(&self.store_file)));
+        }
     }
 
     /// Tracks the time it takes to query the block inputs.
     pub fn add_get_block_inputs(&mut self, query_time: Duration) {
-        self.get_block_inputs_time_per_block.push(query_time);
+        assert!(
+            self.pending_get_block_inputs_time.replace(query_time).is_none(),
+            "block input query time should be consumed before the next query"
+        );
     }
 
     /// Tracks the time it takes to query the batch inputs.
@@ -79,60 +121,67 @@ impl SeedingMetrics {
     #[expect(clippy::cast_precision_loss)]
     fn print_block_metrics(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         writeln!(f, "\nBlock metrics:")?;
-        writeln!(f, "Note: Each block contains 256 transactions (16 batches * 16 transactions).")?;
-        writeln!(
-            f,
-            "The first transaction sends assets from the faucet to 255 accounts, and the remaining 255 transactions consume notes created in the previous block."
-        )?;
         writeln!(
             f,
-            "{:<10} {:<20} {:<30} {:<30} {:<20} {:<20}",
+            "{:<10} {:<24} {:<8} {:<16} {:<18} {:<24} {:<16} {:<16}",
             "Block #",
-            "Insert Time (ms)",
-            "Get Block Inputs Time (ms)",
-            "Get Batch Inputs Time (ms)",
+            "Kind",
+            "Txs",
+            "Public updates",
+            "Insert time (ms)",
+            "Get block inputs (ms)",
             "Block Size (KB)",
             "DB Size (MB)"
         )?;
-        writeln!(f, "{}", "-".repeat(135))?;
-        for (i, store_size) in self.store_file_sizes.iter().enumerate() {
-            let block_index = i * 50;
-            let insertion_time = self
-                .insertion_time_per_block
-                .get(block_index)
-                .unwrap_or(&Duration::default())
-                .as_millis();
-            let block_query_time = self
-                .get_block_inputs_time_per_block
-                .get(block_index)
-                .unwrap_or(&Duration::default())
-                .as_millis();
-            let batch_query_time = self
-                .get_batch_inputs_time_per_block
-                .get(block_index)
-                .unwrap_or(&Duration::default())
-                .as_millis();
-            let block_size_kb =
-                *self.bytes_per_block.get(block_index).unwrap_or(&0) as f64 / 1024.0;
-            let store_size_mb = *store_size as f64 / (1024.0 * 1024.0);
+        writeln!(f, "{}", "-".repeat(150))?;
+        for (block_index, metric) in self.blocks.iter().enumerate().filter(|(index, metric)| {
+            self.blocks.len() <= 20
+                || index % 50 == 0
+                || index + 1 == self.blocks.len()
+                || metric.kind == BlockKind::AccountUpdate
+        }) {
+            let block_size_kb = metric.block_size as f64 / 1024.0;
+            let store_size_mb =
+                self.store_file_sizes.iter().rev().find_map(|(sample_index, size)| {
+                    (*sample_index == block_index).then_some(*size as f64 / (1024.0 * 1024.0))
+                });
+            let store_size =
+                store_size_mb.map_or_else(|| "-".to_string(), |size| format!("{size:.1}"));
 
             writeln!(
                 f,
-                "{block_index:<10} {insertion_time:<20} {block_query_time:<30} {batch_query_time:<30} {block_size_kb:<20.1} {store_size_mb:<20.1}",
+                "{block_index:<10} {:<24} {:<8} {:<16} {:<18} {:<24} {block_size_kb:<16.1} {store_size:<16}",
+                metric.kind,
+                metric.transaction_count,
+                metric.public_account_updates,
+                metric.insertion_time.as_millis(),
+                metric.get_block_inputs_time.as_millis(),
             )?;
         }
+
+        if !self.get_batch_inputs_time_per_block.is_empty() {
+            let average = self
+                .get_batch_inputs_time_per_block
+                .iter()
+                .map(Duration::as_micros)
+                .sum::<u128>()
+                / self.get_batch_inputs_time_per_block.len() as u128;
+            writeln!(f, "Average get-batch-inputs time: {average} us")?;
+        }
         Ok(())
     }
 
     /// Prints the database table stats.
     fn print_table_stats(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         writeln!(f, "\nDatabase stats:")?;
-        writeln!(
-            f,
-            "Note: Database contains {} accounts and {} notes across all blocks.",
-            self.num_insertions * 255,
-            self.num_insertions * 255
-        )?;
+        let account_count = self.table_row_count("accounts");
+        let note_count = self.table_row_count("notes");
+        if let (Some(account_count), Some(note_count)) = (account_count, note_count) {
+            writeln!(
+                f,
+                "Note: Database contains {account_count} accounts and {note_count} notes across all blocks."
+            )?;
+        }
         writeln!(f, "{:<35} {:<15} {:<15}", "Table", "Size (MB)", "KB/Entry")?;
         writeln!(f, "{}", "-".repeat(70))?;
         for table in SQLITE_TABLES {
@@ -162,6 +211,15 @@ impl SeedingMetrics {
         Ok(())
     }
 
+    fn table_row_count(&self, table: &str) -> Option<u64> {
+        let output = Command::new("sqlite3")
+            .arg(&self.store_file)
+            .arg(format!("SELECT COUNT(*) FROM {table};"))
+            .output()
+            .ok()?;
+        String::from_utf8(output.stdout).ok()?.trim().parse().ok()
+    }
+
     /// Prints the index stats.
     fn print_index_stats(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         writeln!(f, "\nIndex stats:")?;
@@ -194,9 +252,9 @@ impl Display for SeedingMetrics {
         writeln!(
             f,
             "Inserted {} blocks with avg insertion time {} ms",
-            self.num_insertions,
-            (self.insertion_time_per_block.iter().map(Duration::as_millis).sum::<u128>()
-                / self.insertion_time_per_block.len() as u128)
+            self.blocks.len(),
+            self.blocks.iter().map(|metric| metric.insertion_time.as_millis()).sum::<u128>()
+                / self.blocks.len().max(1) as u128
         )?;
         writeln!(
             f,
@@ -205,9 +263,12 @@ impl Display for SeedingMetrics {
         )?;
 
         // Print out the average growth rate of the store file
-        let final_size = self.store_file_sizes.last().unwrap();
-        let growth_rate_mb = (final_size - self.initial_store_size) as f64
-            / f64::from(self.num_insertions)
+        let final_size = self
+            .store_file_sizes
+            .last()
+            .map_or_else(|| get_store_size(&self.store_file), |(_, size)| *size);
+        let growth_rate_mb = final_size.saturating_sub(self.initial_store_size) as f64
+            / self.blocks.len().max(1) as f64
             / (1024.0 * 1024.0);
         writeln!(f, "Average DB growth rate: {growth_rate_mb:.1} MB per block")?;