feat(fast-inbox): split checkpoint messages per block in the prover (A-1384)

Prove streaming per-block message insertion (AZIP-22 Fast Inbox): each block root
now appends its own real message slice at compact indices instead of the whole
checkpoint being padded into the first block.

- checkpoint-prover: slices the checkpoint's messages per block from the blocks'
  L1-to-L2 leaf-count ranges and feeds each block its slice.
- CheckpointSubTreeOrchestrator.startNewBlock: inserts the block's slice, capturing
  the per-block start/end snapshots and a full-height frontier at the compact start
  index (new getFrontierSiblingPath helper); startCheckpoint no longer inserts
  messages up front.
- BlockProvingState/CheckpointProvingState carry per-block snapshots + the block's
  own slice; getPaddedL1ToL2Messages/getNumRealL1ToL2Messages retired.

Prover orchestrator unit suites verify on CI (bb proving is not local); test mocks
updated for compact per-block insertion.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Author: spalladino

yarn-project/prover-client/src/mocks/test_context.ts

@@ -1,8 +1,7 @@
 import type { BBProverConfig } from '@aztec/bb-prover';
 import { TestCircuitProver } from '@aztec/bb-prover';
-import { NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/constants';
 import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
-import { padArrayEnd, times, timesAsync } from '@aztec/foundation/collection';
+import { times, timesAsync } from '@aztec/foundation/collection';
 import { Fr } from '@aztec/foundation/curves/bn254';
 import type { Logger } from '@aztec/foundation/log';
 import { SerialQueue } from '@aztec/foundation/queue';
@@ -194,12 +193,10 @@ export class TestContext {
 
     const fork = await this.worldState.fork();
 
-    // Build l1 to l2 messages.
+    // Build l1 to l2 messages. Appended unpadded at compact indices (AZIP-22 Fast Inbox); the mock assigns them all to
+    // the checkpoint's first block, matching how the per-block driver slices them.
     const l1ToL2Messages = times(numL1ToL2Messages, i => new Fr(slotNumber * 100 + i));
-    await fork.appendLeaves(
-      MerkleTreeId.L1_TO_L2_MESSAGE_TREE,
-      padArrayEnd<Fr, number>(l1ToL2Messages, Fr.ZERO, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP),
-    );
+    await fork.appendLeaves(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, l1ToL2Messages);
     const newL1ToL2Snapshot = await getTreeSnapshot(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, fork);
 
     const startBlockNumber = this.nextBlockNumber;

yarn-project/prover-client/src/orchestrator/block-building-helpers.ts

@@ -400,6 +400,15 @@ export async function getSubtreeSiblingPath(
   return fullSiblingPath.getSubtreeSiblingPath(subtreeHeight).toFields();
 }
 
+/**
+ * Returns the full-height frontier (left-sibling path) at a tree's next-available leaf index — the hint the append
+ * circuits re-hash against the snapshot root when appending leaves at a compact (unaligned) index (AZIP-22 Fast Inbox).
+ */
+export async function getFrontierSiblingPath(treeId: MerkleTreeId, db: MerkleTreeReadOperations): Promise<Fr[]> {
+  const nextAvailableLeafIndex = await db.getTreeInfo(treeId).then(t => t.size);
+  return (await db.getSiblingPath(treeId, nextAvailableLeafIndex)).toFields();
+}
+
 // Scan a tree searching for a specific value and return a membership witness proof for it
 export async function getMembershipWitnessFor<N extends number>(
   value: Fr,

yarn-project/prover-client/src/orchestrator/block-proving-state.ts

@@ -1,10 +1,7 @@
 import { type BlockBlobData, type BlockEndBlobData, type SpongeBlob, encodeBlockEndBlobData } from '@aztec/blob-lib';
 import {
   type ARCHIVE_HEIGHT,
-  L1_TO_L2_MSG_SUBTREE_HEIGHT,
-  type L1_TO_L2_MSG_SUBTREE_ROOT_SIBLING_PATH_LENGTH,
   type L1_TO_L2_MSG_TREE_HEIGHT,
-  MAX_L1_TO_L2_MSGS_PER_BLOCK,
   type NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH,
 } from '@aztec/constants';
 import { BlockNumber } from '@aztec/foundation/branded-types';
@@ -66,12 +63,14 @@ export class BlockProvingState {
     private readonly timestamp: UInt64,
     public readonly lastArchiveTreeSnapshot: AppendOnlyTreeSnapshot,
     private readonly lastArchiveSiblingPath: Tuple<Fr, typeof ARCHIVE_HEIGHT>,
-    private readonly lastL1ToL2MessageTreeSnapshot: AppendOnlyTreeSnapshot,
-    private readonly lastL1ToL2MessageSubtreeRootSiblingPath: Tuple<
-      Fr,
-      typeof L1_TO_L2_MSG_SUBTREE_ROOT_SIBLING_PATH_LENGTH
-    >,
+    // This block's L1-to-L2 message tree snapshot before and after its own bundle (AZIP-22 Fast Inbox). The start is
+    // the previous block's end (block-merge continuity); the end is this block's own post-bundle snapshot.
+    private readonly startL1ToL2MessageTreeSnapshot: AppendOnlyTreeSnapshot,
     public readonly newL1ToL2MessageTreeSnapshot: AppendOnlyTreeSnapshot,
+    // Full-height frontier (left-sibling path) at the block's start index, pinning the append at a compact index.
+    private readonly l1ToL2MessageFrontierHint: Tuple<Fr, typeof L1_TO_L2_MSG_TREE_HEIGHT>,
+    // This block's own real L1-to-L2 message leaves (unpadded slice).
+    private readonly l1ToL2Messages: Fr[],
     private readonly headerOfLastBlockInPreviousCheckpoint: BlockHeader,
     private readonly startSpongeBlob: SpongeBlob,
     public parentCheckpoint: CheckpointProvingState,
@@ -304,7 +303,7 @@ export class BlockProvingState {
         inputs: new BlockRootSingleTxRollupPrivateInputs(
           leftRollup,
           messageBundle,
-          this.lastL1ToL2MessageTreeSnapshot,
+          this.startL1ToL2MessageTreeSnapshot,
           startMsgSponge,
           frontierHint,
           this.lastArchiveSiblingPath,
@@ -316,7 +315,7 @@ export class BlockProvingState {
         inputs: new BlockRootRollupPrivateInputs(
           [leftRollup, rightRollup],
           messageBundle,
-          this.lastL1ToL2MessageTreeSnapshot,
+          this.startL1ToL2MessageTreeSnapshot,
           startMsgSponge,
           frontierHint,
           this.lastArchiveSiblingPath,
@@ -348,7 +347,7 @@ export class BlockProvingState {
         inputs: new BlockRootSingleTxFirstRollupPrivateInputs(
           leftRollup,
           messageBundle,
-          this.lastL1ToL2MessageTreeSnapshot,
+          this.startL1ToL2MessageTreeSnapshot,
           frontierHint,
           this.lastArchiveSiblingPath,
         ),
@@ -359,7 +358,7 @@ export class BlockProvingState {
         inputs: new BlockRootFirstRollupPrivateInputs(
           [leftRollup, rightRollup],
           messageBundle,
-          this.lastL1ToL2MessageTreeSnapshot,
+          this.startL1ToL2MessageTreeSnapshot,
           frontierHint,
           this.lastArchiveSiblingPath,
         ),
@@ -368,31 +367,21 @@ export class BlockProvingState {
   }
 
   /**
-   * The real-count message bundle this block appends (AZIP-22 Fast Inbox): the real leaves inserted at compact indices
-   * and absorbed into the message sponge.
-   *
-   * TODO(fast-inbox): the prover still assigns the whole checkpoint's messages to the first block. Post-flip a block
-   * carries at most `MAX_L1_TO_L2_MSGS_PER_BLOCK` and a checkpoint drains its consumption across up to four blocks, so
-   * this must split the checkpoint's messages per block (with per-block start/end snapshots and full-height frontier
-   * hints at compact indices). Requires the proving-path rework; verified by epoch-proving e2e on CI.
+   * The real-count message bundle this block appends (AZIP-22 Fast Inbox): the block's own message slice, inserted at
+   * compact indices and absorbed into its block-root message sponge.
    */
   #getMessageBundle(): L1ToL2MessageBundle {
-    if (this.isFirstBlock) {
-      return makeL1ToL2MessageBundle(this.parentCheckpoint.getL1ToL2Messages());
-    }
-    return L1ToL2MessageBundle.empty();
+    return this.l1ToL2Messages.length === 0
+      ? L1ToL2MessageBundle.empty()
+      : makeL1ToL2MessageBundle(this.l1ToL2Messages);
   }
 
   /**
-   * Full-height frontier hint for the bundle append. The l1-to-l2 tree index is always subtree-aligned in the
-   * transitional wiring, so the bottom `L1_TO_L2_MSG_SUBTREE_HEIGHT` levels are left-child (unread, zero) and the top
-   * levels are exactly the subtree-root sibling path already captured for this block.
+   * Full-height frontier hint for the bundle append: the left-sibling path at the block's compact start index, which
+   * the block-root circuit re-hashes against its start snapshot root (AZIP-22 Fast Inbox).
    */
   #getFrontierHint(): Tuple<Fr, typeof L1_TO_L2_MSG_TREE_HEIGHT> {
-    return [
-      ...Array.from({ length: L1_TO_L2_MSG_SUBTREE_HEIGHT }, () => Fr.ZERO),
-      ...this.lastL1ToL2MessageSubtreeRootSiblingPath,
-    ] as Tuple<Fr, typeof L1_TO_L2_MSG_TREE_HEIGHT>;
+    return this.l1ToL2MessageFrontierHint;
   }
 
   // Returns a specific transaction proving state

yarn-project/prover-client/src/orchestrator/checkpoint-proving-state.ts

@@ -1,7 +1,7 @@
 import { SpongeBlob } from '@aztec/blob-lib';
 import type {
   ARCHIVE_HEIGHT,
-  L1_TO_L2_MSG_SUBTREE_ROOT_SIBLING_PATH_LENGTH,
+  L1_TO_L2_MSG_TREE_HEIGHT,
   NESTED_RECURSIVE_PROOF_LENGTH,
   NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH,
 } from '@aztec/constants';
@@ -42,18 +42,6 @@ export class CheckpointProvingState {
     // Inbox rolling hash before this checkpoint's messages (the previous checkpoint's end value; genesis is zero).
     // Threaded into the InboxParity circuit so the resulting checkpoint header rolling hash matches the proposer's.
     private readonly startInboxRollingHash: Fr,
-    // The snapshot and sibling path before the new l1 to l2 message subtree is inserted.
-    private readonly lastL1ToL2MessageTreeSnapshot: AppendOnlyTreeSnapshot,
-    private readonly lastL1ToL2MessageSubtreeRootSiblingPath: Tuple<
-      Fr,
-      typeof L1_TO_L2_MSG_SUBTREE_ROOT_SIBLING_PATH_LENGTH
-    >,
-    // The snapshot and sibling path after the new l1 to l2 message subtree is inserted.
-    private readonly newL1ToL2MessageTreeSnapshot: AppendOnlyTreeSnapshot,
-    private readonly newL1ToL2MessageSubtreeRootSiblingPath: Tuple<
-      Fr,
-      typeof L1_TO_L2_MSG_SUBTREE_ROOT_SIBLING_PATH_LENGTH
-    >,
     // Message-bundle sponge over the checkpoint's real messages (real-count absorb). Equals the InboxParity proof's
     // end sponge and the sponge the block roots accumulate, so it is threaded into non-first block roots as their
     // inherited `startMsgSponge`.
@@ -84,20 +72,18 @@ export class CheckpointProvingState {
     totalNumTxs: number,
     lastArchiveTreeSnapshot: AppendOnlyTreeSnapshot,
     lastArchiveSiblingPath: Tuple<Fr, typeof ARCHIVE_HEIGHT>,
+    // Per-block L1-to-L2 message state (AZIP-22 Fast Inbox): the block's start snapshot (its parent's end), its own
+    // post-bundle end snapshot, the full-height frontier at the start index, and its own real message slice.
+    startL1ToL2MessageTreeSnapshot: AppendOnlyTreeSnapshot,
+    endL1ToL2MessageTreeSnapshot: AppendOnlyTreeSnapshot,
+    l1ToL2MessageFrontierHint: Tuple<Fr, typeof L1_TO_L2_MSG_TREE_HEIGHT>,
+    l1ToL2Messages: Fr[],
   ): BlockProvingState {
     const index = Number(blockNumber) - Number(this.firstBlockNumber);
     if (index >= this.totalNumBlocks) {
       throw new Error(`Unable to start a new block at index ${index}. Expected at most ${this.totalNumBlocks} blocks.`);
     }
 
-    // If this is the first block, we use the snapshot and sibling path before the new l1 to l2 messages are inserted.
-    // Otherwise, we use the snapshot and sibling path after the new l1 to l2 messages are inserted, which will always
-    // happen in the first block.
-    const lastL1ToL2MessageTreeSnapshot =
-      index === 0 ? this.lastL1ToL2MessageTreeSnapshot : this.newL1ToL2MessageTreeSnapshot;
-    const lastL1ToL2MessageSubtreeRootSiblingPath =
-      index === 0 ? this.lastL1ToL2MessageSubtreeRootSiblingPath : this.newL1ToL2MessageSubtreeRootSiblingPath;
-
     const startSpongeBlob = index === 0 ? SpongeBlob.init() : this.blocks[index - 1]?.getEndSpongeBlob();
     if (!startSpongeBlob) {
       throw new Error(
@@ -113,9 +99,10 @@ export class CheckpointProvingState {
       timestamp,
       lastArchiveTreeSnapshot,
       lastArchiveSiblingPath,
-      lastL1ToL2MessageTreeSnapshot,
-      lastL1ToL2MessageSubtreeRootSiblingPath,
-      this.newL1ToL2MessageTreeSnapshot,
+      startL1ToL2MessageTreeSnapshot,
+      endL1ToL2MessageTreeSnapshot,
+      l1ToL2MessageFrontierHint,
+      l1ToL2Messages,
       this.headerOfLastBlockInPreviousCheckpoint,
       startSpongeBlob,
       this,

yarn-project/prover-client/src/orchestrator/checkpoint-sub-tree-orchestrator.test.ts

@@ -58,9 +58,9 @@ describe('prover/orchestrator/checkpoint-sub-tree', () => {
     try {
       const resultPromise = subTree.getSubTreeResult();
 
-      for (const block of blocks) {
+      for (const [blockIndex, block] of blocks.entries()) {
         const { blockNumber, timestamp } = block.header.globalVariables;
-        await subTree.startNewBlock(blockNumber, timestamp, block.txs.length);
+        await subTree.startNewBlock(blockNumber, timestamp, block.txs.length, blockIndex === 0 ? l1ToL2Messages : []);
         if (block.txs.length > 0) {
           await subTree.addTxs(block.txs);
         }
@@ -104,9 +104,9 @@ describe('prover/orchestrator/checkpoint-sub-tree', () => {
     try {
       const resultPromise = subTree.getSubTreeResult();
 
-      for (const block of blocks) {
+      for (const [blockIndex, block] of blocks.entries()) {
         const { blockNumber, timestamp } = block.header.globalVariables;
-        await subTree.startNewBlock(blockNumber, timestamp, block.txs.length);
+        await subTree.startNewBlock(blockNumber, timestamp, block.txs.length, blockIndex === 0 ? l1ToL2Messages : []);
         if (block.txs.length > 0) {
           await subTree.addTxs(block.txs);
         }
@@ -149,9 +149,9 @@ describe('prover/orchestrator/checkpoint-sub-tree', () => {
     try {
       const resultPromise = subTree.getSubTreeResult();
 
-      for (const block of blocks) {
+      for (const [blockIndex, block] of blocks.entries()) {
         const { blockNumber, timestamp } = block.header.globalVariables;
-        await subTree.startNewBlock(blockNumber, timestamp, block.txs.length);
+        await subTree.startNewBlock(blockNumber, timestamp, block.txs.length, blockIndex === 0 ? l1ToL2Messages : []);
         if (block.txs.length > 0) {
           await subTree.addTxs(block.txs);
         }
@@ -197,9 +197,9 @@ describe('prover/orchestrator/checkpoint-sub-tree', () => {
     try {
       const resultPromise = subTree.getSubTreeResult();
 
-      for (const block of blocks) {
+      for (const [blockIndex, block] of blocks.entries()) {
         const { blockNumber, timestamp } = block.header.globalVariables;
-        await subTree.startNewBlock(blockNumber, timestamp, block.txs.length);
+        await subTree.startNewBlock(blockNumber, timestamp, block.txs.length, blockIndex === 0 ? l1ToL2Messages : []);
         if (block.txs.length > 0) {
           await subTree.addTxs(block.txs);
         }

yarn-project/prover-client/src/orchestrator/checkpoint-sub-tree-orchestrator.ts

@@ -1,8 +1,7 @@
 import type { SpongeBlob } from '@aztec/blob-lib/types';
 import {
   type ARCHIVE_HEIGHT,
-  L1_TO_L2_MSG_SUBTREE_HEIGHT,
-  L1_TO_L2_MSG_SUBTREE_ROOT_SIBLING_PATH_LENGTH,
+  L1_TO_L2_MSG_TREE_HEIGHT,
   NESTED_RECURSIVE_ROLLUP_HONK_PROOF_LENGTH,
 } from '@aztec/constants';
 import { BlockNumber, type EpochNumber } from '@aztec/foundation/branded-types';
@@ -51,10 +50,10 @@ import { inspect } from 'util';
 
 import {
   buildHeaderFromCircuitOutputs,
+  getFrontierSiblingPath,
   getLastSiblingPath,
   getPublicChonkVerifierPrivateInputsFromTx,
   getRootTreeSiblingPath,
-  getSubtreeSiblingPath,
   getTreeSnapshot,
   insertSideEffectsAndBuildBaseRollupHints,
   validatePartialState,
@@ -264,7 +263,7 @@ export class CheckpointSubTreeOrchestrator extends ProvingScheduler {
   @trackSpan('CheckpointSubTreeOrchestrator.startNewBlock', blockNumber => ({
     [Attributes.BLOCK_NUMBER]: blockNumber,
   }))
-  public async startNewBlock(blockNumber: BlockNumber, timestamp: UInt64, totalNumTxs: number) {
+  public async startNewBlock(blockNumber: BlockNumber, timestamp: UInt64, totalNumTxs: number, l1ToL2Messages: Fr[]) {
     if (!this.provingState) {
       throw new Error('Empty proving state. The checkpoint sub-tree has not been started.');
     }
@@ -288,12 +287,26 @@ export class CheckpointSubTreeOrchestrator extends ProvingScheduler {
     const lastArchiveTreeSnapshot = await getTreeSnapshot(MerkleTreeId.ARCHIVE, db);
     const lastArchiveSiblingPath = await getRootTreeSiblingPath(MerkleTreeId.ARCHIVE, db);
 
+    // Streaming Inbox (AZIP-22 Fast Inbox): insert this block's own real message slice at compact indices, capturing
+    // the start snapshot + full-height frontier before the append and the block's own post-bundle end snapshot after.
+    const startL1ToL2Snapshot = await getTreeSnapshot(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, db);
+    const l1ToL2FrontierHint = assertLength(
+      await getFrontierSiblingPath(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, db),
+      L1_TO_L2_MSG_TREE_HEIGHT,
+    );
+    await appendL1ToL2MessagesToTree(db, l1ToL2Messages);
+    const endL1ToL2Snapshot = await getTreeSnapshot(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, db);
+
     const blockProvingState = this.provingState.startNewBlock(
       blockNumber,
       timestamp,
       totalNumTxs,
       lastArchiveTreeSnapshot,
       lastArchiveSiblingPath,
+      startL1ToL2Snapshot,
+      endL1ToL2Snapshot,
+      l1ToL2FrontierHint,
+      l1ToL2Messages,
     );
 
     // Because `addTxs` won't be called for a block without txs, and that's where the sponge blob state is computed,
@@ -518,16 +531,9 @@ export class CheckpointSubTreeOrchestrator extends ProvingScheduler {
     // Get archive sibling path before any block in this checkpoint lands.
     const lastArchiveSiblingPath = await getLastSiblingPath(MerkleTreeId.ARCHIVE, db);
 
-    // Insert all the l1 to l2 messages into the db. Get the states before and after the insertion.
-    const {
-      lastL1ToL2MessageTreeSnapshot,
-      lastL1ToL2MessageSubtreeRootSiblingPath,
-      newL1ToL2MessageTreeSnapshot,
-      newL1ToL2MessageSubtreeRootSiblingPath,
-    } = await this.updateL1ToL2MessageTree(l1ToL2Messages, db);
-
-    // The message sponge absorbs the checkpoint's real messages (real-count), matching the checkpoint's single
-    // InboxParity proof; non-first block roots inherit this sponge (AZIP-22 Fast Inbox).
+    // Streaming Inbox (AZIP-22 Fast Inbox): messages are inserted per block in `startNewBlock`, not the whole
+    // checkpoint up front. The message sponge still absorbs the checkpoint's real messages (real-count), matching the
+    // checkpoint's single InboxParity proof; non-first block roots inherit this sponge.
     const checkpointMsgSponge = L1ToL2MessageSponge.empty();
     await checkpointMsgSponge.absorb(l1ToL2Messages);
 
@@ -539,10 +545,6 @@ export class CheckpointSubTreeOrchestrator extends ProvingScheduler {
       lastArchiveSiblingPath,
       l1ToL2Messages,
       startInboxRollingHash,
-      lastL1ToL2MessageTreeSnapshot,
-      lastL1ToL2MessageSubtreeRootSiblingPath,
-      newL1ToL2MessageTreeSnapshot,
-      newL1ToL2MessageSubtreeRootSiblingPath,
       checkpointMsgSponge,
       Number(this.epochNumber),
       /* isAlive */ () => !this.cancelled,
@@ -556,30 +558,6 @@ export class CheckpointSubTreeOrchestrator extends ProvingScheduler {
 
   // ---------------- private: per-block proof orchestration ----------------
 
-  private async updateL1ToL2MessageTree(l1ToL2Messages: Fr[], db: MerkleTreeWriteOperations) {
-    const lastL1ToL2MessageTreeSnapshot = await getTreeSnapshot(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, db);
-    const lastL1ToL2MessageSubtreeRootSiblingPath = assertLength(
-      await getSubtreeSiblingPath(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, L1_TO_L2_MSG_SUBTREE_HEIGHT, db),
-      L1_TO_L2_MSG_SUBTREE_ROOT_SIBLING_PATH_LENGTH,
-    );
-
-    // Update the local trees to include the new l1 to l2 messages.
-    await appendL1ToL2MessagesToTree(db, l1ToL2Messages);
-
-    const newL1ToL2MessageTreeSnapshot = await getTreeSnapshot(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, db);
-    const newL1ToL2MessageSubtreeRootSiblingPath = assertLength(
-      await getSubtreeSiblingPath(MerkleTreeId.L1_TO_L2_MESSAGE_TREE, L1_TO_L2_MSG_SUBTREE_HEIGHT, db),
-      L1_TO_L2_MSG_SUBTREE_ROOT_SIBLING_PATH_LENGTH,
-    );
-
-    return {
-      lastL1ToL2MessageTreeSnapshot,
-      lastL1ToL2MessageSubtreeRootSiblingPath,
-      newL1ToL2MessageTreeSnapshot,
-      newL1ToL2MessageSubtreeRootSiblingPath,
-    };
-  }
-
   // Updates the merkle trees for a transaction. The first enqueued job for a transaction.
   @trackSpan('CheckpointSubTreeOrchestrator.prepareBaseRollupInputs', tx => ({
     [Attributes.TX_HASH]: tx.hash.toString(),