refactor(fast-inbox): make per-block message insertion the only path in the checkpoint builder (A-1384)

Every block now carries its own L1-to-L2 message bundle, so the up-front whole-checkpoint
insertion has no callers left. Delete the `insertMessagesPerBlock` toggle and the checkpoint-wide
`l1ToL2Messages` argument of `startCheckpoint`/`startNewCheckpoint` (which would otherwise be
silently double-counted into the checkpoint's rolling hash), and promote the bundle from
`addBlock`'s options bag to a required argument, since a block always states what it consumes.
`openCheckpoint` keeps the list for the resume path, where it seeds the rolling hash of blocks
already built, and now rejects a non-empty list when starting fresh. Tests that relied on the
up-front path insert the bundle through the checkpoint's first block instead.

Author: spalladino

yarn-project/prover-client/src/light/lightweight_checkpoint_builder.bench.test.ts

@@ -146,19 +146,18 @@ describe('LightweightCheckpointBuilder benchmarks', () => {
       const constants = makeCheckpointConstants(slotNumber);
       const fork = await worldState.fork();
 
-      const builder = await LightweightCheckpointBuilder.startNewCheckpoint(
+      const builder = LightweightCheckpointBuilder.startNewCheckpoint(
         CheckpointNumber(1),
         constants,
         [],
-        [],
         Fr.ZERO,
         fork,
       );
 
       const globalVariables = makeGlobalVariables(blockNumber, slotNumber);
       const txs = await timesAsync(numTxs, i => makeTx(globalVariables, 5000 + i));
 
-      const { timings } = await builder.addBlock(globalVariables, txs, { insertTxsEffects: true });
+      const { timings } = await builder.addBlock(globalVariables, txs, [], { insertTxsEffects: true });
 
       const prefix = `addBlock/${label}/${numTxs} txs`;
       for (const [step, ms] of Object.entries(timings)) {

yarn-project/prover-client/src/light/lightweight_checkpoint_builder.test.ts

@@ -97,24 +97,22 @@ describe('LightweightCheckpointBuilder', () => {
       const blockNumber = BlockNumber(1);
 
       const constants = makeCheckpointConstants(slotNumber);
-      const l1ToL2Messages: Fr[] = [];
       const previousCheckpointOutHashes: Fr[] = [];
 
       const fork = await worldState.fork();
 
       // Use LightweightCheckpointBuilder directly
-      const checkpointBuilder = await LightweightCheckpointBuilder.startNewCheckpoint(
+      const checkpointBuilder = LightweightCheckpointBuilder.startNewCheckpoint(
         checkpointNumber,
         constants,
-        l1ToL2Messages,
         previousCheckpointOutHashes,
         Fr.ZERO,
         fork,
       );
 
       // Build empty block
       const globalVariables = makeGlobalVariables(blockNumber, slotNumber);
-      const { block } = await checkpointBuilder.addBlock(globalVariables, [], { insertTxsEffects: true });
+      const { block } = await checkpointBuilder.addBlock(globalVariables, [], [], { insertTxsEffects: true });
 
       expect(block.header.globalVariables.blockNumber).toEqual(blockNumber);
 
@@ -138,16 +136,14 @@ describe('LightweightCheckpointBuilder', () => {
       const blockNumber = BlockNumber(1);
 
       const constants = makeCheckpointConstants(slotNumber);
-      const l1ToL2Messages: Fr[] = [];
       // There are two checkpoints before this one.
       const previousCheckpointOutHashes = [Fr.random(), Fr.random()];
 
       const fork = await worldState.fork();
 
-      const checkpointBuilder = await LightweightCheckpointBuilder.startNewCheckpoint(
+      const checkpointBuilder = LightweightCheckpointBuilder.startNewCheckpoint(
         checkpointNumber,
         constants,
-        l1ToL2Messages,
         previousCheckpointOutHashes,
         Fr.ZERO,
         fork,
@@ -161,7 +157,7 @@ describe('LightweightCheckpointBuilder', () => {
       tx.txEffect.l2ToL1Msgs.push(...msgs);
 
       // Build block with tx - insertTxsEffects will handle inserting side effects
-      const { block } = await checkpointBuilder.addBlock(globalVariables, [tx], {
+      const { block } = await checkpointBuilder.addBlock(globalVariables, [tx], [], {
         insertTxsEffects: true,
       });
 
@@ -190,15 +186,13 @@ describe('LightweightCheckpointBuilder', () => {
       const blockNumber = BlockNumber(1);
 
       const constants = makeCheckpointConstants(slotNumber);
-      const l1ToL2Messages: Fr[] = [];
       const previousCheckpointOutHashes: Fr[] = [];
 
       const fork = await worldState.fork();
 
-      const checkpointBuilder = await LightweightCheckpointBuilder.startNewCheckpoint(
+      const checkpointBuilder = LightweightCheckpointBuilder.startNewCheckpoint(
         checkpointNumber,
         constants,
-        l1ToL2Messages,
         previousCheckpointOutHashes,
         Fr.ZERO,
         fork,
@@ -209,7 +203,7 @@ describe('LightweightCheckpointBuilder', () => {
       const txs = await timesAsync(3, i => makeProcessedTx(globalVariables, 1000 + i));
 
       // Build block with txs - insertTxsEffects will handle inserting side effects
-      const { block } = await checkpointBuilder.addBlock(globalVariables, txs, {
+      const { block } = await checkpointBuilder.addBlock(globalVariables, txs, [], {
         insertTxsEffects: true,
       });
 
@@ -230,15 +224,13 @@ describe('LightweightCheckpointBuilder', () => {
       const slotNumber = SlotNumber(15);
 
       const constants = makeCheckpointConstants(slotNumber);
-      const l1ToL2Messages: Fr[] = [];
       const previousCheckpointOutHashes: Fr[] = [];
 
       const fork = await worldState.fork();
 
-      const checkpointBuilder = await LightweightCheckpointBuilder.startNewCheckpoint(
+      const checkpointBuilder = LightweightCheckpointBuilder.startNewCheckpoint(
         checkpointNumber,
         constants,
-        l1ToL2Messages,
         previousCheckpointOutHashes,
         Fr.ZERO,
         fork,
@@ -256,7 +248,7 @@ describe('LightweightCheckpointBuilder', () => {
         const txs = await timesAsync(txsPerBlock, j => makeProcessedTx(globalVariables, 2000 + i * 10 + j));
 
         // Build block - insertTxsEffects will handle inserting side effects
-        const { block } = await checkpointBuilder.addBlock(globalVariables, txs, {
+        const { block } = await checkpointBuilder.addBlock(globalVariables, txs, [], {
           insertTxsEffects: true,
         });
 
@@ -284,15 +276,13 @@ describe('LightweightCheckpointBuilder', () => {
       const slotNumber = SlotNumber(15);
 
       const constants = makeCheckpointConstants(slotNumber);
-      const l1ToL2Messages: Fr[] = [];
       const previousCheckpointOutHashes: Fr[] = [];
 
       const fork = await worldState.fork();
 
-      const checkpointBuilder = await LightweightCheckpointBuilder.startNewCheckpoint(
+      const checkpointBuilder = LightweightCheckpointBuilder.startNewCheckpoint(
         checkpointNumber,
         constants,
-        l1ToL2Messages,
         previousCheckpointOutHashes,
         Fr.ZERO,
         fork,
@@ -309,15 +299,13 @@ describe('LightweightCheckpointBuilder', () => {
       const slotNumber = SlotNumber(15);
 
       const constants = makeCheckpointConstants(slotNumber);
-      const l1ToL2Messages: Fr[] = [];
       const previousCheckpointOutHashes: Fr[] = [];
 
       const fork = await worldState.fork();
 
-      const checkpointBuilder = await LightweightCheckpointBuilder.startNewCheckpoint(
+      const checkpointBuilder = LightweightCheckpointBuilder.startNewCheckpoint(
         checkpointNumber,
         constants,
-        l1ToL2Messages,
         previousCheckpointOutHashes,
         Fr.ZERO,
         fork,
@@ -326,13 +314,13 @@ describe('LightweightCheckpointBuilder', () => {
       // Add first block with txs - insertTxsEffects will handle inserting side effects
       const globalVariables1 = makeGlobalVariables(BlockNumber(1), slotNumber);
       const txs1 = await timesAsync(2, i => makeProcessedTx(globalVariables1, 3000 + i));
-      await checkpointBuilder.addBlock(globalVariables1, txs1, {
+      await checkpointBuilder.addBlock(globalVariables1, txs1, [], {
         insertTxsEffects: true,
       });
 
       // Try to add second block with no txs - this should fail
       const globalVariables2 = makeGlobalVariables(BlockNumber(2), slotNumber);
-      await expect(checkpointBuilder.addBlock(globalVariables2, [], { insertTxsEffects: true })).rejects.toThrow(
+      await expect(checkpointBuilder.addBlock(globalVariables2, [], [], { insertTxsEffects: true })).rejects.toThrow(
         /first block/,
       );
 
@@ -344,15 +332,13 @@ describe('LightweightCheckpointBuilder', () => {
       const slotNumber = SlotNumber(15);
 
       const constants = makeCheckpointConstants(slotNumber);
-      const l1ToL2Messages: Fr[] = [];
       const previousCheckpointOutHashes: Fr[] = [];
 
       const fork = await worldState.fork();
 
-      const checkpointBuilder = await LightweightCheckpointBuilder.startNewCheckpoint(
+      const checkpointBuilder = LightweightCheckpointBuilder.startNewCheckpoint(
         checkpointNumber,
         constants,
-        l1ToL2Messages,
         previousCheckpointOutHashes,
         Fr.ZERO,
         fork,
@@ -363,7 +349,7 @@ describe('LightweightCheckpointBuilder', () => {
       const wrongBlockNumber = BlockNumber(5);
       const globalVariables = makeGlobalVariables(wrongBlockNumber, slotNumber);
 
-      await expect(checkpointBuilder.addBlock(globalVariables, [], { insertTxsEffects: true })).rejects.toThrow(
+      await expect(checkpointBuilder.addBlock(globalVariables, [], [], { insertTxsEffects: true })).rejects.toThrow(
         /Archive tree next leaf index mismatch/,
       );
 

yarn-project/prover-client/src/light/lightweight_checkpoint_builder.ts

@@ -30,7 +30,7 @@ import {
 /**
  * Builds a checkpoint and its header and the blocks in it from a set of processed tx without running any circuits.
  *
- * It updates the l1-to-l2 message tree when starting a new checkpoint, and then updates the archive tree when each block is added.
+ * Each added block inserts its own L1-to-L2 message bundle into the message tree and then updates the archive tree.
  * Finally completes the checkpoint by computing its header.
  */
 export class LightweightCheckpointBuilder {
@@ -60,29 +60,24 @@ export class LightweightCheckpointBuilder {
     this.logger.debug('Starting new checkpoint', { constants, l1ToL2Messages, feeAssetPriceModifier });
   }
 
-  static async startNewCheckpoint(
+  /**
+   * Starts a fresh checkpoint. The checkpoint's L1-to-L2 messages are not supplied here: every block brings its own
+   * bundle to {@link addBlock}, which inserts it into the tree and accumulates it into the checkpoint's message list.
+   */
+  static startNewCheckpoint(
     checkpointNumber: CheckpointNumber,
     constants: CheckpointGlobalVariables,
-    l1ToL2Messages: Fr[],
     previousCheckpointOutHashes: Fr[],
     previousInboxRollingHash: Fr,
     db: MerkleTreeWriteOperations,
     bindings?: LoggerBindings,
     feeAssetPriceModifier: bigint = 0n,
-    // Streaming Inbox (AZIP-22 Fast Inbox): messages are inserted per block via `addBlock`, so `l1ToL2Messages` here
-    // is empty and the up-front checkpoint-wide insertion is skipped.
-    insertMessagesPerBlock: boolean = false,
-  ): Promise<LightweightCheckpointBuilder> {
-    // Insert l1-to-l2 messages into the tree (legacy flow: the whole checkpoint's messages up front).
-    if (!insertMessagesPerBlock) {
-      await appendL1ToL2MessagesToTree(db, l1ToL2Messages);
-    }
-
+  ): LightweightCheckpointBuilder {
     return new LightweightCheckpointBuilder(
       checkpointNumber,
       constants,
       feeAssetPriceModifier,
-      l1ToL2Messages,
+      [],
       previousCheckpointOutHashes,
       previousInboxRollingHash,
       db,
@@ -93,8 +88,8 @@ export class LightweightCheckpointBuilder {
   /**
    * Resumes building a checkpoint from existing blocks. This is used for validator re-execution
    * where blocks have already been built and their effects are already in the database.
-   * Unlike startNewCheckpoint, this does NOT append l1ToL2Messages to the tree since they
-   * were already added when the blocks were originally built.
+   * `l1ToL2Messages` is the whole checkpoint's message list as consumed by the existing blocks: it seeds the
+   * checkpoint's rolling hash and is not inserted into the tree, since the blocks already inserted it.
    */
   static async resumeCheckpoint(
     checkpointNumber: CheckpointNumber,
@@ -172,18 +167,20 @@ export class LightweightCheckpointBuilder {
   /**
    * Adds a new block to the checkpoint. The tx effects must have already been inserted into the db if
    * this is called after tx processing, if that's not the case, then set `insertTxsEffects` to true.
+   * @param l1ToL2Messages - The message leaves this block consumes from the Inbox, in insertion order.
    */
   public async addBlock(
     globalVariables: GlobalVariables,
     txs: ProcessedTx[],
-    opts: { insertTxsEffects?: boolean; expectedEndState?: StateReference; l1ToL2Messages?: Fr[] } = {},
+    l1ToL2Messages: Fr[],
+    opts: { insertTxsEffects?: boolean; expectedEndState?: StateReference } = {},
   ): Promise<{ block: L2Block; timings: Record<string, number> }> {
     const timings: Record<string, number> = {};
     const isFirstBlock = this.blocks.length === 0;
 
     // A non-first block with no txs is only allowed when it inserts a non-empty L1-to-L2 message bundle: the
     // message-only block shape (AZIP-22 Fast Inbox), proven by the msgs-only block root.
-    if (!isFirstBlock && txs.length === 0 && (opts.l1ToL2Messages?.length ?? 0) === 0) {
+    if (!isFirstBlock && txs.length === 0 && l1ToL2Messages.length === 0) {
       throw new Error('Cannot add an empty non-first block that carries no L1-to-L2 messages.');
     }
 
@@ -213,9 +210,7 @@ export class LightweightCheckpointBuilder {
     // tree's current next-available index). The logical messages are accumulated only once the block is fully built
     // (below), so a mid-build failure does not pollute the checkpoint's rolling hash; the rolling hash is recomputed
     // over them at checkpoint completion.
-    if (opts.l1ToL2Messages !== undefined) {
-      await appendL1ToL2MessagesToTree(this.db, opts.l1ToL2Messages);
-    }
+    await appendL1ToL2MessagesToTree(this.db, l1ToL2Messages);
 
     const [msGetEndState, endState] = await elapsed(() => this.db.getStateReference());
     timings.getEndState = msGetEndState;
@@ -254,9 +249,7 @@ export class LightweightCheckpointBuilder {
 
     // Accumulate the streaming bundle now that the block is fully built, so a mid-build throw above leaves the
     // checkpoint's message list (and thus its inHash/rolling hash) consistent with the blocks actually built.
-    if (opts.l1ToL2Messages !== undefined) {
-      this.l1ToL2Messages.push(...opts.l1ToL2Messages);
-    }
+    this.l1ToL2Messages.push(...l1ToL2Messages);
 
     const [msSpongeAbsorb] = await elapsed(() => this.spongeBlob.absorb(blockBlobFields));
     timings.spongeAbsorb = msSpongeAbsorb;

yarn-project/prover-client/src/mocks/test_context.ts

@@ -241,10 +241,9 @@ export class TestContext {
 
     const cleanFork = await this.worldState.fork();
     const previousCheckpointOutHashes = this.checkpointOutHashes;
-    const builder = await LightweightCheckpointBuilder.startNewCheckpoint(
+    const builder = LightweightCheckpointBuilder.startNewCheckpoint(
       checkpointNumber,
       { ...constants, timestamp },
-      l1ToL2Messages,
       previousCheckpointOutHashes,
       Fr.ZERO,
       cleanFork,
@@ -256,7 +255,7 @@ export class TestContext {
       const txs = blockTxs[i];
       const state = blockEndStates[i];
 
-      const { block } = await builder.addBlock(blockGlobalVariables[i], txs, {
+      const { block } = await builder.addBlock(blockGlobalVariables[i], txs, i === 0 ? l1ToL2Messages : [], {
         expectedEndState: state,
         insertTxsEffects: true,
       });

yarn-project/sequencer-client/src/publisher/l1_publisher.integration.test.ts

@@ -499,16 +499,15 @@ describe('L1Publisher integration', () => {
 
     // Test uses 1-block-per-checkpoint
     const checkpointNumber = CheckpointNumber.fromBlockNumber(globalVariables.blockNumber);
-    const builder = await LightweightCheckpointBuilder.startNewCheckpoint(
+    const builder = LightweightCheckpointBuilder.startNewCheckpoint(
       checkpointNumber,
       checkpointConstants,
-      l1ToL2Messages,
       previousCheckpointOutHashes,
       previousInboxRollingHash,
       tempFork,
     );
 
-    await builder.addBlock(globalVariables, txs, { insertTxsEffects: true });
+    await builder.addBlock(globalVariables, txs, l1ToL2Messages, { insertTxsEffects: true });
     const checkpoint = await builder.completeCheckpoint();
 
     await tempFork.close();

yarn-project/sequencer-client/src/sequencer/automine/automine_sequencer.ts

@@ -498,12 +498,10 @@ export class AutomineSequencer {
       checkpointNumber,
       checkpointGlobals,
       feeAssetPriceModifier,
-      [],
       previousCheckpointOutHashes,
       previousInboxRollingHash,
       fork,
       this.log.getBindings(),
-      true,
     );
 
     // Block building only executes txs, and automine publishes straight to L1, so the proofs are never needed.

yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts

@@ -1441,11 +1441,9 @@ describe('CheckpointProposalJob', () => {
 
       expect(checkpoint).toBeDefined();
 
-      // Streaming skips the bulk per-checkpoint fetch and tells the builder to insert messages per block.
+      // Streaming skips the bulk per-checkpoint fetch; every message reaches the builder through a block.
       expect(l1ToL2MessageSource.getL1ToL2Messages).not.toHaveBeenCalled();
       expect(checkpointsBuilder.startCheckpointCalls).toHaveLength(1);
-      expect(checkpointsBuilder.startCheckpointCalls[0].l1ToL2Messages).toEqual([]);
-      expect(checkpointsBuilder.startCheckpointCalls[0].insertMessagesPerBlock).toBe(true);
 
       // The first block consumes the selected bundle; the second consumes nothing.
       expect(checkpointBuilder.buildBlockCalls).toHaveLength(2);

yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts

@@ -648,9 +648,8 @@ export class CheckpointProposalJob implements Traceable {
         this.checkpointSimulationOverridesPlan,
       );
 
-      // Under the streaming Inbox (AZIP-22 Fast Inbox) messages are selected per block, so the checkpoint-level bulk
-      // list stays empty and the legacy inHash is fed zero; the running values are computed block by block.
-      const l1ToL2Messages: Fr[] = [];
+      // Under the streaming Inbox (AZIP-22 Fast Inbox) messages are selected per block, so the legacy inHash is fed
+      // zero; the running values are computed block by block.
       const inHash = Fr.ZERO;
 
       // Collect the out hashes of all the checkpoints before this one in the same epoch.
@@ -697,12 +696,10 @@ export class CheckpointProposalJob implements Traceable {
         this.checkpointNumber,
         checkpointGlobalVariables,
         feeAssetPriceModifier,
-        l1ToL2Messages,
         previousCheckpointOutHashes,
         previousInboxRollingHash,
         fork,
         this.log.getBindings(),
-        true,
       );
 
       // Options for the validator client when creating block and checkpoint proposals