allow num_ns configuration

Author: mrain

crates/hotshot-builder/legacy/src/service.rs

@@ -3582,7 +3582,7 @@ mod test {
                 BlockInfo {
                     block_payload: payload,
                     metadata: TestMetadata {
-                        num_transactions: 1,
+                        num_transactions: 1, ..Default::default()
                     },
                     vid_trigger: Arc::new(async_lock::RwLock::new(Some(vid_trigger_sender))),
                     offered_fee: 100,
@@ -3820,7 +3820,7 @@ mod test {
         let da_proposal = DaProposal2::<TestTypes> {
             encoded_transactions: Arc::new([1, 2, 3, 4, 5, 6]),
             metadata: TestMetadata {
-                num_transactions: 1,
+                num_transactions: 1, ..Default::default()
             }, // arbitrary
             view_number,
             epoch,
@@ -3879,7 +3879,7 @@ mod test {
         let da_proposal = DaProposal2::<TestTypes> {
             encoded_transactions: Arc::new([1, 2, 3, 4, 5, 6]),
             metadata: TestMetadata {
-                num_transactions: 1,
+                num_transactions: 1, ..Default::default()
             }, // arbitrary
             view_number,
             epoch,
@@ -3929,7 +3929,7 @@ mod test {
         let da_proposal = DaProposal2::<TestTypes> {
             encoded_transactions: Arc::new([1, 2, 3, 4, 5, 6]),
             metadata: TestMetadata {
-                num_transactions: 1,
+                num_transactions: 1, ..Default::default()
             }, // arbitrary
             view_number,
             epoch,

crates/hotshot-builder/legacy/src/testing/basic_test.rs

@@ -151,7 +151,7 @@ mod tests {
                         timestamp: 0,
                         timestamp_millis: 0,
                         metadata: TestMetadata {
-                            num_transactions: 0,
+                            num_transactions: 0, ..Default::default()
                         },
                         random: 1, // arbitrary
                         version: TEST_VERSIONS.test.base,
@@ -266,7 +266,7 @@ mod tests {
                     let da_proposal = DaProposal2 {
                         encoded_transactions: encoded_transactions.clone().into(),
                         metadata: TestMetadata {
-                            num_transactions: encoded_transactions.len() as u64,
+                            num_transactions: encoded_transactions.len() as u64, ..Default::default()
                         },
                         view_number: ViewNumber::new(round as u64),
                         epoch: None, // TODO: check if this is okay

crates/hotshot-builder/legacy/src/testing/finalization_test.rs

@@ -273,7 +273,7 @@ async fn progress_round_with_transactions(
         .expect("should sign encoded transactions hash successfully");
 
         let metadata = TestMetadata {
-            num_transactions: transactions.len() as u64,
+            num_transactions: transactions.len() as u64, ..Default::default()
         };
 
         da_proposal_sender

crates/hotshot-builder/legacy/src/testing/mod.rs

@@ -124,7 +124,7 @@ pub async fn calc_proposal_msg(
     let num_transactions = transactions.len() as u64;
     let encoded_transactions = TestTransaction::encode(&transactions);
     let block_payload = TestBlockPayload { transactions };
-    let metadata = TestMetadata { num_transactions };
+    let metadata = TestMetadata { num_transactions, ..Default::default() };
     let block_vid_commitment = vid_commitment(
         &encoded_transactions,
         &metadata.encode(),
@@ -146,7 +146,7 @@ pub async fn calc_proposal_msg(
         let da_proposal = DaProposal2 {
             encoded_transactions: encoded_transactions.clone().into(),
             metadata: TestMetadata {
-                num_transactions: encoded_transactions.len() as u64,
+                num_transactions: encoded_transactions.len() as u64, ..Default::default()
             },
             view_number: ViewNumber::new(round as u64),
             epoch: None,

crates/hotshot-builder/refactored/src/testing/basic.rs

@@ -199,7 +199,7 @@ async fn test_pruning() {
                 &Default::default(),
                 TestBlockPayload::genesis(),
                 &TestMetadata {
-                    num_transactions: 0,
+                    num_transactions: 0, ..Default::default()
                 },
                 TEST_VERSIONS.test.base,
             ),

crates/hotshot-builder/shared/src/testing/consensus.rs

@@ -54,7 +54,7 @@ impl SimulatedChainState {
         let num_transactions = transactions.len() as u64;
         let encoded_transactions = TestTransaction::encode(&transactions);
         let block_payload = TestBlockPayload { transactions };
-        let metadata = TestMetadata { num_transactions };
+        let metadata = TestMetadata { num_transactions, ..Default::default() };
         let block_vid_commitment = vid_commitment(
             &encoded_transactions,
             &metadata.encode(),

crates/hotshot/example-types/src/block_types.rs

@@ -170,16 +170,44 @@ impl<TYPES: NodeType> TestableBlock<TYPES> for TestBlockPayload {
 }
 
 #[derive(
-    Debug, Display, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash,
+    Debug, Display, Clone, Copy, Default, Serialize, Deserialize,
+    PartialEq, Eq, PartialOrd, Ord, Hash,
 )]
 #[display("{{num_transactions:{num_transactions}}}")]
 pub struct TestMetadata {
     pub num_transactions: u64,
+    /// Total encoded-payload byte length. When this is `> 0` and
+    /// `num_transactions > 1`, `EncodeBytes::encode` emits a well-formed
+    /// namespace table (in the wire format consumed by
+    /// `hotshot_types::data::ns_table::parse_ns_table`) that splits the
+    /// payload into `num_transactions` evenly-sized namespaces. The AvidM
+    /// dispersal pipeline then parallelizes per-namespace via rayon.
+    ///
+    /// Default `0` preserves the legacy single-namespace behaviour for
+    /// every existing call site that doesn't opt in.
+    #[serde(default)]
+    pub payload_byte_len: u64,
 }
 
 impl EncodeBytes for TestMetadata {
     fn encode(&self) -> Arc<[u8]> {
-        Arc::new([])
+        let n_ns = self.num_transactions;
+        let total = self.payload_byte_len as usize;
+        if n_ns <= 1 || total == 0 {
+            return Arc::new([]);
+        }
+        // Wire format consumed by `parse_ns_table`:
+        //   [u32_le num_nss] [u32_le ns_id | u32_le ns_end_offset] × num_nss
+        let n = n_ns as usize;
+        let chunk = total / n;
+        let mut buf = Vec::with_capacity(4 + n * 8);
+        buf.extend_from_slice(&(n_ns as u32).to_le_bytes());
+        for i in 0..n {
+            let end = if i + 1 == n { total } else { (i + 1) * chunk };
+            buf.extend_from_slice(&(i as u32).to_le_bytes()); // ns_id
+            buf.extend_from_slice(&(end as u32).to_le_bytes()); // ns_end_offset
+        }
+        buf.into()
     }
 }
 
@@ -205,6 +233,7 @@ impl<TYPES: NodeType> BlockPayload<TYPES> for TestBlockPayload {
         let txns_vec: Vec<TestTransaction> = transactions.into_iter().collect();
         let metadata = TestMetadata {
             num_transactions: txns_vec.len() as u64,
+            ..Default::default()
         };
         Ok((
             Self {
@@ -240,6 +269,7 @@ impl<TYPES: NodeType> BlockPayload<TYPES> for TestBlockPayload {
             Self::genesis(),
             TestMetadata {
                 num_transactions: 0,
+                ..Default::default()
             },
         )
     }

crates/hotshot/new-protocol/bench/src/config.rs

@@ -38,6 +38,13 @@ pub struct NodeConfig {
     #[arg(long, default_value_t = 0)]
     pub block_size: usize,
 
+    /// Number of namespaces to split each block payload into. AvidmGf2Scheme's
+    /// `ns_disperse` and `recover` parallelize per-namespace via rayon, so
+    /// increasing this should reduce dispersal/recovery latency up to the
+    /// machine's core count. Default 1 = legacy single-namespace behaviour.
+    #[arg(long, default_value_t = 1)]
+    pub namespaces: u32,
+
     /// Period between CPU + network sampler ticks (milliseconds). 50ms is the
     /// default; lower values give finer resolution at the cost of more
     /// /proc reads per second.

crates/hotshot/new-protocol/bench/src/node.rs

@@ -311,7 +311,7 @@ async fn run_instrumented(mut coordinator: BenchCoordinator, cfg: &NodeConfig) -
             if let ConsensusOutput::RequestBlockAndHeader(ref req) = output
                 && cfg.block_size > 0
             {
-                let block = build_test_block(cfg.block_size, cfg.total_nodes);
+                let block = build_test_block(cfg.block_size, cfg.total_nodes, cfg.namespaces);
                 let parent_leaf = req.parent_proposal.clone().into();
                 let version = bench_upgrade_lock().version_infallible(req.view);
                 let header = TestBlockHeader::new::<TestTypes>(
@@ -371,20 +371,26 @@ struct TestBlock {
     builder_commitment: hotshot_types::utils::BuilderCommitment,
 }
 
-fn build_test_block(size: usize, num_nodes: usize) -> TestBlock {
+fn build_test_block(size: usize, num_nodes: usize, n_namespaces: u32) -> TestBlock {
     use hotshot_types::traits::EncodeBytes;
 
     let tx = TestTransaction::new(vec![0u8; size]);
     let block = TestBlockPayload {
         transactions: vec![tx],
     };
+    let encoded = block.encode();
+
+    // `TestMetadata` itself emits the namespace table when `num_transactions
+    // > 1` AND `payload_byte_len > 0`. The bench sets both so AvidM dispersal
+    // splits the payload into N evenly-sized namespaces and parallelizes
+    // per-namespace via rayon (same wire format as production `NsTable`).
+    let n = n_namespaces.max(1);
     let metadata = TestMetadata {
-        num_transactions: 1,
+        num_transactions: n as u64,
+        payload_byte_len: if n > 1 { encoded.len() as u64 } else { 0 },
     };
-    // Use the actual committee size so the commitment matches what
-    // VidDisperse::calculate_vid_disperse will produce.
     let payload_commitment = hotshot_types::data::vid_commitment(
-        &block.encode(),
+        &encoded,
         &metadata.encode(),
         num_nodes,
         versions::NEW_PROTOCOL_VERSION,

crates/hotshot/new-protocol/bench/tests/smoke.rs

@@ -40,6 +40,7 @@ fn node_config(
             .into_owned(),
         block_size,
         sampler_tick_ms: 50,
+        namespaces: 1,
     }
 }