From cdf178454c07a74f0e78fbda63020434ea99a19e Mon Sep 17 00:00:00 2001 From: diegokingston Date: Thu, 9 Apr 2026 11:07:10 -0300 Subject: [PATCH 1/7] perf(tables): cap LT/LOAD/BRANCH/MEMW_R max_rows at 2^20 Tables with effective width < 42 were sized at 2^21, producing single large chunks. Capping at 2^20 produces 2x more chunks of the standard size, improving parallel throughput and keeping peak memory per chunk uniform across all tables. --- prover/src/tables/mod.rs | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/prover/src/tables/mod.rs b/prover/src/tables/mod.rs index 551dc4aa3..fc56ec22b 100644 --- a/prover/src/tables/mod.rs +++ b/prover/src/tables/mod.rs @@ -49,29 +49,29 @@ pub use types::BusId; /// (* MEMW_A formula gives 2^20, but set to 2^19 to match MEMW chunk geometry; /// benchmarks show better parallel throughput with smaller chunks.) /// -/// | Table | Main | Bus | Eff.width | Max rows | -/// |---------|------|-----|-----------|----------| -/// | MEMW | 49 | 26 | 127 | 2^19 | -/// | MEMW_A | 29 | 20 | 89 | 2^19 * | -/// | CPU | 74 | 40 | 194 | 2^19 | -/// | DVRM | 34 | 34 | 136 | 2^19 | -/// | MUL | 26 | 16 | 74 | 2^20 | -/// | LT | 15 | 9 | 42 | 2^21 | -/// | SHIFT | 27 | 15 | 72 | 2^20 | -/// | LOAD | 18 | 5 | 33 | 2^21 | -/// | BRANCH | 14 | 6 | 32 | 2^21 | -/// | MEMW_R | 10 | 7 | 31 | 2^21 | +/// | Table | Main | Bus | Eff.width | Max rows | +/// |---------|------|-----|-----------|-----------------| +/// | MEMW | 49 | 26 | 127 | 2^19 | +/// | MEMW_A | 29 | 20 | 89 | 2^19 * | +/// | CPU | 74 | 40 | 194 | 2^19 | +/// | DVRM | 34 | 34 | 136 | 2^19 | +/// | MUL | 26 | 16 | 74 | 2^20 | +/// | LT | 15 | 9 | 42 | 2^20 (capped) | +/// | SHIFT | 27 | 15 | 72 | 2^20 | +/// | LOAD | 18 | 5 | 33 | 2^20 (capped) | +/// | BRANCH | 14 | 6 | 32 | 2^20 (capped) | +/// | MEMW_R | 10 | 7 | 31 | 2^20 (capped) | pub mod max_rows { pub const CPU: usize = 1 << 19; // 524,288 — eff. width 194 pub const MEMW: usize = 1 << 19; // 524,288 — eff. width 127 (baseline) pub const MEMW_A: usize = 1 << 19; // 524,288 — eff. width 89 pub const DVRM: usize = 1 << 19; // 524,288 — eff. width 136 pub const MUL: usize = 1 << 20; // 1,048,576 — eff. width 74 - pub const LT: usize = 1 << 21; // 2,097,152 — eff. width 42 + pub const LT: usize = 1 << 20; // 1,048,576 — eff. width 42 (capped at 2^20) pub const SHIFT: usize = 1 << 20; // 1,048,576 — eff. width 72 - pub const LOAD: usize = 1 << 21; // 2,097,152 — eff. width 33 - pub const BRANCH: usize = 1 << 21; // 2,097,152 — eff. width 32 - pub const MEMW_R: usize = 1 << 21; // 2,097,152 — eff. width 31 + pub const LOAD: usize = 1 << 20; // 1,048,576 — eff. width 33 (capped at 2^20) + pub const BRANCH: usize = 1 << 20; // 1,048,576 — eff. width 32 (capped at 2^20) + pub const MEMW_R: usize = 1 << 20; // 1,048,576 — eff. width 31 (capped at 2^20) } /// Per-table maximum row limits, configurable for different environments. From a6397106dbcd71eb629993ff3f70b36cc11d35cd Mon Sep 17 00:00:00 2001 From: diegokingston Date: Thu, 9 Apr 2026 11:07:36 -0300 Subject: [PATCH 2/7] perf(stark): deduplicate LdeTwiddles by domain size in multi_prove Tables sharing the same lde_size now reuse a single Arc instead of each allocating their own copy. This avoids redundant twiddle computation and allocation when multiple tables happen to have the same trace length (e.g., all 2^19-row tables after the cap change). --- crypto/stark/src/prover.rs | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/crypto/stark/src/prover.rs b/crypto/stark/src/prover.rs index e1f0aaf74..a59833682 100644 --- a/crypto/stark/src/prover.rs +++ b/crypto/stark/src/prover.rs @@ -660,7 +660,7 @@ pub trait IsStarkProver< air_trace_pairs: &[AirTracePair<'_, Field, FieldExtension, PI>], metadatas: &[Round1Metadata], domains: &[Domain], - twiddle_caches: &[LdeTwiddles], + twiddle_caches: &[Arc>], main_pool: &mut [Vec>], aux_pool: &mut [Vec>], ) where @@ -676,7 +676,7 @@ pub trait IsStarkProver< .zip(domains.iter().zip(twiddle_caches.iter())) { let result = Self::reconstruct_round1( - *air, *trace, domain, metadata, twiddles, main_pool, aux_pool, + *air, *trace, domain, metadata, &**twiddles, main_pool, aux_pool, ) .expect("reconstruct_round1 failed in debug-checks"); temp_results.push(result); @@ -1534,24 +1534,30 @@ pub trait IsStarkProver< let phase_start = Instant::now(); let mut domains = Vec::with_capacity(num_airs); - let mut twiddle_caches: Vec> = Vec::with_capacity(num_airs); + let mut twiddle_caches: Vec>> = Vec::with_capacity(num_airs); let mut max_main_cols = 0usize; let mut max_aux_cols = 0usize; let mut max_lde_size = 0usize; + // Deduplicate twiddle caches: tables with the same lde_size share one Arc. + let mut twiddle_by_size: std::collections::HashMap>> = + std::collections::HashMap::new(); + for (air, trace, _pub_inputs) in &*air_trace_pairs { let trace_length = trace.num_rows(); let domain = new_domain(*air, trace_length); let lde_size = domain.interpolation_domain_size * domain.blowup_factor; - let twiddles = LdeTwiddles::new(&domain); + let twiddles = twiddle_by_size + .entry(lde_size) + .or_insert_with(|| Arc::new(LdeTwiddles::new(&domain))); max_main_cols = max_main_cols.max(trace.num_main_columns); max_aux_cols = max_aux_cols.max(air.num_auxiliary_rap_columns()); max_lde_size = max_lde_size.max(lde_size); domains.push(domain); - twiddle_caches.push(twiddles); + twiddle_caches.push(Arc::clone(twiddles)); } // Spill all main trace tables to mmap before allocating pool buffers. @@ -1631,7 +1637,7 @@ pub trait IsStarkProver< let idx = chunk_start + j; let (air, trace, _) = &air_trace_pairs[idx]; let domain = &domains[idx]; - let twiddles = &twiddle_caches[idx]; + let twiddles = &*twiddle_caches[idx]; #[allow(unused_mut)] let (mut tree, root, pre_tree, pre_root, n_pre) = if air.is_preprocessed() { @@ -1828,7 +1834,7 @@ pub trait IsStarkProver< let idx = chunk_start + j; let (air, trace, _) = &air_trace_pairs[idx]; let domain = &domains[idx]; - let twiddles = &twiddle_caches[idx]; + let twiddles = &*twiddle_caches[idx]; if air.has_aux_trace() { let num_aux_cols = trace.num_aux_columns; @@ -2080,7 +2086,7 @@ pub trait IsStarkProver< let (air, trace, pub_inputs) = &air_trace_pairs[idx]; let metadata = &metadatas[idx]; let domain = &domains[idx]; - let twiddles = &twiddle_caches[idx]; + let twiddles = &*twiddle_caches[idx]; #[cfg(feature = "instruments")] let table_start = Instant::now(); From 4c29c1025a7b79cea53601e1ff5ed1609d9909d7 Mon Sep 17 00:00:00 2001 From: diegokingston Date: Thu, 9 Apr 2026 11:10:12 -0300 Subject: [PATCH 3/7] perf(fri): parallelize fold_evaluations_in_place with Rayon Under the `parallel` feature, fold each conjugate pair concurrently using `par_chunks(2).zip(inv_twiddles.par_iter())`, collecting into a new Vec of half length. The sequential path (no `parallel` feature) is unchanged. This avoids the index aliasing issue from an in-place interleaved write. --- crypto/stark/src/fri/fri_functions.rs | 37 ++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/crypto/stark/src/fri/fri_functions.rs b/crypto/stark/src/fri/fri_functions.rs index 8bd355ec4..e96cbf5d2 100644 --- a/crypto/stark/src/fri/fri_functions.rs +++ b/crypto/stark/src/fri/fri_functions.rs @@ -18,14 +18,37 @@ pub fn fold_evaluations_in_place, E: IsField>( inv_twiddles: &[FieldElement], ) { let half = evals.len() / 2; - for j in 0..half { - let lo = &evals[2 * j]; - let hi = &evals[2 * j + 1]; - let sum = lo + hi; - let diff = lo - hi; - evals[j] = &sum + &(&inv_twiddles[j] * &(zeta * &diff)); + + #[cfg(feature = "parallel")] + { + use rayon::prelude::*; + // Evaluations are stored interleaved: pairs (evals[2j], evals[2j+1]). + // Fold each pair in parallel, collecting into a new Vec of half length. + let folded: Vec> = evals + .par_chunks(2) + .zip(inv_twiddles.par_iter()) + .map(|(pair, tw)| { + let lo = &pair[0]; + let hi = &pair[1]; + let sum = lo + hi; + let diff = lo - hi; + &sum + &(tw * &(zeta * &diff)) + }) + .collect(); + *evals = folded; + } + + #[cfg(not(feature = "parallel"))] + { + for j in 0..half { + let lo = &evals[2 * j]; + let hi = &evals[2 * j + 1]; + let sum = lo + hi; + let diff = lo - hi; + evals[j] = &sum + &(&inv_twiddles[j] * &(zeta * &diff)); + } + evals.truncate(half); } - evals.truncate(half); } /// Compute inverse twiddle factors for evaluation-form FRI folding. From a809ced7e4b4ddbb3678bca9efcff482e0031598 Mon Sep 17 00:00:00 2001 From: diegokingston Date: Thu, 9 Apr 2026 11:10:35 -0300 Subject: [PATCH 4/7] perf(prover): eliminate per-row Vec alloc in commit_columns_bit_reversed Replace the per-row `Vec` allocation inside the iterator with a thread-local buffer via Rayon's `map_init` (parallel path) or a single buffer allocated outside the loop (sequential path). This avoids one heap allocation per row during Merkle tree construction. --- crypto/stark/src/prover.rs | 40 ++++++++++++++++++++++++++------------ 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/crypto/stark/src/prover.rs b/crypto/stark/src/prover.rs index a59833682..f1de46e8f 100644 --- a/crypto/stark/src/prover.rs +++ b/crypto/stark/src/prover.rs @@ -335,19 +335,35 @@ pub trait IsStarkProver< let num_cols = columns.len(); #[cfg(feature = "parallel")] - let iter = (0..num_rows).into_par_iter(); - #[cfg(not(feature = "parallel"))] - let iter = 0..num_rows; + let hashed_leaves: Vec = { + (0..num_rows) + .into_par_iter() + .map_init( + || vec![FieldElement::::zero(); num_cols], + |row_buf, row_idx| { + let br_idx = reverse_index(row_idx, num_rows as u64); + for col_idx in 0..num_cols { + row_buf[col_idx] = columns[col_idx][br_idx].clone(); + } + BatchedMerkleTreeBackend::::hash_data(row_buf) + }, + ) + .collect() + }; - let hashed_leaves: Vec = iter - .map(|row_idx| { - let br_idx = reverse_index(row_idx, num_rows as u64); - let row: Vec> = (0..num_cols) - .map(|col_idx| columns[col_idx][br_idx].clone()) - .collect(); - BatchedMerkleTreeBackend::::hash_data(&row) - }) - .collect(); + #[cfg(not(feature = "parallel"))] + let hashed_leaves: Vec = { + let mut row_buf = vec![FieldElement::::zero(); num_cols]; + (0..num_rows) + .map(|row_idx| { + let br_idx = reverse_index(row_idx, num_rows as u64); + for col_idx in 0..num_cols { + row_buf[col_idx] = columns[col_idx][br_idx].clone(); + } + BatchedMerkleTreeBackend::::hash_data(&row_buf) + }) + .collect() + }; let tree = BatchedMerkleTree::::build_from_hashed_leaves(hashed_leaves)?; let root = tree.root; From 0ce8a63a31c79c2d43e2f1dd5e111276ba34b8a0 Mon Sep 17 00:00:00 2001 From: gabrielbosio Date: Thu, 9 Apr 2026 16:25:08 -0300 Subject: [PATCH 5/7] Fix fmt and clippy after cherry-pick --- crypto/stark/src/fri/fri_functions.rs | 3 +-- crypto/stark/src/prover.rs | 8 +++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/crypto/stark/src/fri/fri_functions.rs b/crypto/stark/src/fri/fri_functions.rs index e96cbf5d2..b02090533 100644 --- a/crypto/stark/src/fri/fri_functions.rs +++ b/crypto/stark/src/fri/fri_functions.rs @@ -17,8 +17,6 @@ pub fn fold_evaluations_in_place, E: IsField>( zeta: &FieldElement, inv_twiddles: &[FieldElement], ) { - let half = evals.len() / 2; - #[cfg(feature = "parallel")] { use rayon::prelude::*; @@ -40,6 +38,7 @@ pub fn fold_evaluations_in_place, E: IsField>( #[cfg(not(feature = "parallel"))] { + let half = evals.len() / 2; for j in 0..half { let lo = &evals[2 * j]; let hi = &evals[2 * j + 1]; diff --git a/crypto/stark/src/prover.rs b/crypto/stark/src/prover.rs index f1de46e8f..3fa5640b3 100644 --- a/crypto/stark/src/prover.rs +++ b/crypto/stark/src/prover.rs @@ -692,7 +692,13 @@ pub trait IsStarkProver< .zip(domains.iter().zip(twiddle_caches.iter())) { let result = Self::reconstruct_round1( - *air, *trace, domain, metadata, &**twiddles, main_pool, aux_pool, + *air, + *trace, + domain, + metadata, + &**twiddles, + main_pool, + aux_pool, ) .expect("reconstruct_round1 failed in debug-checks"); temp_results.push(result); From c6058b956f4774075d197eca6237a74d1a6d3857 Mon Sep 17 00:00:00 2001 From: gabrielbosio Date: Thu, 9 Apr 2026 17:23:50 -0300 Subject: [PATCH 6/7] Remove redundant capped annotations from max_rows --- prover/src/tables/mod.rs | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/prover/src/tables/mod.rs b/prover/src/tables/mod.rs index fc56ec22b..19d14411d 100644 --- a/prover/src/tables/mod.rs +++ b/prover/src/tables/mod.rs @@ -49,29 +49,29 @@ pub use types::BusId; /// (* MEMW_A formula gives 2^20, but set to 2^19 to match MEMW chunk geometry; /// benchmarks show better parallel throughput with smaller chunks.) /// -/// | Table | Main | Bus | Eff.width | Max rows | -/// |---------|------|-----|-----------|-----------------| -/// | MEMW | 49 | 26 | 127 | 2^19 | -/// | MEMW_A | 29 | 20 | 89 | 2^19 * | -/// | CPU | 74 | 40 | 194 | 2^19 | -/// | DVRM | 34 | 34 | 136 | 2^19 | -/// | MUL | 26 | 16 | 74 | 2^20 | -/// | LT | 15 | 9 | 42 | 2^20 (capped) | -/// | SHIFT | 27 | 15 | 72 | 2^20 | -/// | LOAD | 18 | 5 | 33 | 2^20 (capped) | -/// | BRANCH | 14 | 6 | 32 | 2^20 (capped) | -/// | MEMW_R | 10 | 7 | 31 | 2^20 (capped) | +/// | Table | Main | Bus | Eff.width | Max rows | +/// |---------|------|-----|-----------|----------| +/// | MEMW | 49 | 26 | 127 | 2^19 | +/// | MEMW_A | 29 | 20 | 89 | 2^19 * | +/// | CPU | 74 | 40 | 194 | 2^19 | +/// | DVRM | 34 | 34 | 136 | 2^19 | +/// | MUL | 26 | 16 | 74 | 2^20 | +/// | LT | 15 | 9 | 42 | 2^20 | +/// | SHIFT | 27 | 15 | 72 | 2^20 | +/// | LOAD | 18 | 5 | 33 | 2^20 | +/// | BRANCH | 14 | 6 | 32 | 2^20 | +/// | MEMW_R | 10 | 7 | 31 | 2^20 | pub mod max_rows { pub const CPU: usize = 1 << 19; // 524,288 — eff. width 194 pub const MEMW: usize = 1 << 19; // 524,288 — eff. width 127 (baseline) pub const MEMW_A: usize = 1 << 19; // 524,288 — eff. width 89 pub const DVRM: usize = 1 << 19; // 524,288 — eff. width 136 pub const MUL: usize = 1 << 20; // 1,048,576 — eff. width 74 - pub const LT: usize = 1 << 20; // 1,048,576 — eff. width 42 (capped at 2^20) + pub const LT: usize = 1 << 20; // 1,048,576 — eff. width 42 pub const SHIFT: usize = 1 << 20; // 1,048,576 — eff. width 72 - pub const LOAD: usize = 1 << 20; // 1,048,576 — eff. width 33 (capped at 2^20) - pub const BRANCH: usize = 1 << 20; // 1,048,576 — eff. width 32 (capped at 2^20) - pub const MEMW_R: usize = 1 << 20; // 1,048,576 — eff. width 31 (capped at 2^20) + pub const LOAD: usize = 1 << 20; // 1,048,576 — eff. width 33 + pub const BRANCH: usize = 1 << 20; // 1,048,576 — eff. width 32 + pub const MEMW_R: usize = 1 << 20; // 1,048,576 — eff. width 31 } /// Per-table maximum row limits, configurable for different environments. From 48c1be0d8e6759289152934c4f160c3c9bc1a625 Mon Sep 17 00:00:00 2001 From: gabrielbosio Date: Fri, 10 Apr 2026 13:08:20 -0300 Subject: [PATCH 7/7] Document twiddle dedup invariant --- crypto/stark/src/prover.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crypto/stark/src/prover.rs b/crypto/stark/src/prover.rs index 3fa5640b3..5fefbcbd1 100644 --- a/crypto/stark/src/prover.rs +++ b/crypto/stark/src/prover.rs @@ -1561,7 +1561,8 @@ pub trait IsStarkProver< let mut max_aux_cols = 0usize; let mut max_lde_size = 0usize; - // Deduplicate twiddle caches: tables with the same lde_size share one Arc. + // Share twiddles across tables with equal lde_size. Relies on all AIRs + // using the same ProofOptions (blowup_factor, coset_offset). let mut twiddle_by_size: std::collections::HashMap>> = std::collections::HashMap::new();