feat(sequencer): add waitForBuildDeadlineOnFinalBlock flag

The proposer seals the final block of a checkpoint as soon as minTxsPerBlock are
available and then snapshots the mempool. When txs are submitted as a burst late
in the slot, they propagate over gossip one at a time, so the snapshot captures
only a subset and the remainder spill into the next checkpoint.

Add an opt-in sequencer flag (default off, so production behaviour is unchanged)
that makes the final block of a checkpoint keep collecting txs until its build
deadline before sealing. The build deadline already reserves the post-build
window for attestation collection, so this does not change checkpoint timing.

Enable the flag in e2e_epochs/epochs_mbps_redistribution, which dumps a burst of
late txs and asserts they all land in the final block; without this it raced the
proposer's snapshot and flaked. Covered by new checkpoint_proposal_job timing
unit tests for both the default (seal early) and flag-on (wait) behaviour.

Author: PhilWindle

yarn-project/end-to-end/src/e2e_epochs/epochs_mbps_redistribution.test.ts

@@ -109,6 +109,10 @@ describe('e2e_epochs/epochs_mbps_redistribution', () => {
       // With 3 blocks and multiplier 1.2: maxTxsPerBlock = ceil(TOTAL_TX_COUNT/3*1.2).
       // The redistribution should cap early blocks, preserving budget for the last block.
       maxTxsPerCheckpoint: TOTAL_TX_COUNT,
+      // Make the final block collect txs until its build deadline rather than sealing on the first late tx.
+      // Otherwise the proposer snapshots the mempool mid-arrival and the late txs (which propagate over gossip
+      // one at a time) spill across two blocks, making the `lateBlockNumbers` assertion flaky.
+      waitForBuildDeadlineOnFinalBlock: true,
       // PXE syncs on checkpointed chain tip.
       pxeOpts: { syncChainTip: 'checkpointed' },
       ...contextConfigOverride,

yarn-project/sequencer-client/src/config.ts

@@ -44,6 +44,7 @@ export const DefaultSequencerConfig = {
   sequencerPollingIntervalMS: 500,
   minTxsPerBlock: 1,
   buildCheckpointIfEmpty: false,
+  waitForBuildDeadlineOnFinalBlock: false,
   publishTxsWithProposals: false,
   perBlockAllocationMultiplier: MIN_PER_BLOCK_ALLOCATION_MULTIPLIER,
   perBlockDAAllocationMultiplier: MIN_PER_BLOCK_DA_ALLOCATION_MULTIPLIER,
@@ -241,6 +242,12 @@ export const sequencerConfigMappings: ConfigMappingsType<SequencerConfig> = {
     description: 'Have sequencer build and publish an empty checkpoint if there are no txs',
     ...booleanConfigHelper(DefaultSequencerConfig.buildCheckpointIfEmpty),
   },
+  waitForBuildDeadlineOnFinalBlock: {
+    description:
+      'Wait until the build deadline before sealing the final block of a checkpoint instead of sealing as ' +
+      'soon as minTxsPerBlock are available (for testing only)',
+    ...booleanConfigHelper(DefaultSequencerConfig.waitForBuildDeadlineOnFinalBlock),
+  },
   skipPushProposedBlocksToArchiver: {
     description: 'Skip pushing proposed blocks to archiver (default: true)',
     ...booleanConfigHelper(DefaultSequencerConfig.skipPushProposedBlocksToArchiver),

yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts

@@ -1352,6 +1352,7 @@ describe('CheckpointProposalJob', () => {
       const result = await job.buildSingleBlock(checkpointBuilder, {
         blockNumber: newBlockNumber,
         indexWithinCheckpoint: IndexWithinCheckpoint(1),
+        isLastBlock: false,
         buildDeadline: undefined,
         blockTimestamp: 0n,
         txHashesAlreadyIncluded: new Set<string>(),
@@ -1373,6 +1374,7 @@ describe('CheckpointProposalJob', () => {
       const result = await job.buildSingleBlock(checkpointBuilder, {
         blockNumber: newBlockNumber,
         indexWithinCheckpoint: IndexWithinCheckpoint(1),
+        isLastBlock: false,
         buildDeadline: undefined,
         blockTimestamp: 0n,
         txHashesAlreadyIncluded: new Set<string>(),
@@ -1742,6 +1744,7 @@ class TestCheckpointProposalJob extends CheckpointProposalJob {
     checkpointBuilder: CheckpointBuilder,
     opts: {
       forceCreate?: boolean;
+      isLastBlock: boolean;
       blockTimestamp: bigint;
       blockNumber: BlockNumber;
       indexWithinCheckpoint: IndexWithinCheckpoint;

yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.timing.test.ts

@@ -561,6 +561,59 @@ describe('CheckpointProposalJob Timing Tests', () => {
       expect(checkpointMetrics.noteCheckpointBroadcast).toHaveBeenCalledWith(expect.any(Number));
     });
 
+    // 30s block duration in the 72s slot derives exactly one block, so the single built block is also the
+    // last block of the checkpoint - the only block affected by waitForBuildDeadlineOnFinalBlock.
+    const singleBlockTimetable = () =>
+      makeProposerTimetable({
+        l1Constants,
+        p2pPropagationTime: P2P_PROPAGATION_TIME,
+        checkpointProposalPrepareTime: CHECKPOINT_ASSEMBLE_TIME,
+        blockDurationMs: 30000,
+      });
+
+    it('seals the final block as soon as txs are available by default', async () => {
+      const { blocks, txs } = await createTestBlocksAndTxs(1);
+      mockP2pWithTxs(txs);
+      checkpointBuilder.seedBlocks(blocks, [[txs[0]]]);
+      checkpointBuilder.setExecutionDurations([1]);
+      validatorClient.collectAttestations.mockResolvedValue(getAttestations(blocks[0]));
+
+      setTimeInSlot(0.5);
+
+      const job = createJob();
+      job.setTimetable(singleBlockTimetable());
+
+      await job.execute();
+
+      expect(checkpointBuilder.buildBlockCalls.length).toBe(1);
+      // A tx is available from the start (0.5s), so the block is built immediately rather than waiting.
+      expect(checkpointBuilder.recordedBuildTimes[0].startTime).toBeLessThan(2);
+    });
+
+    it('waits until the build deadline before sealing the final block when waitForBuildDeadlineOnFinalBlock is set', async () => {
+      const { blocks, txs } = await createTestBlocksAndTxs(1);
+      mockP2pWithTxs(txs);
+      checkpointBuilder.seedBlocks(blocks, [[txs[0]]]);
+      checkpointBuilder.setExecutionDurations([1]);
+      validatorClient.collectAttestations.mockResolvedValue(getAttestations(blocks[0]));
+
+      setTimeInSlot(0.5);
+
+      const job = createJob();
+      job.setTimetable(singleBlockTimetable());
+      job.updateConfig({ waitForBuildDeadlineOnFinalBlock: true });
+
+      await job.execute();
+
+      expect(checkpointBuilder.buildBlockCalls.length).toBe(1);
+      // Even though the tx is available from the start, the final block now waits until its build deadline
+      // (minus min_block_duration) before sealing, so a late burst of txs would still be picked up.
+      const deadline = checkpointBuilder.buildBlockCalls[0].opts.deadline!;
+      const startBuildingDeadline =
+        deadline.getTime() / 1000 - getSlotStartTime(slotNumber) - job.getTimetable().minBlockDuration;
+      expect(checkpointBuilder.recordedBuildTimes[0].startTime).toBeGreaterThanOrEqual(startBuildingDeadline - 0.01);
+    });
+
     it('builds maximum blocks when given enough time', async () => {
       const { blocks, txs } = await createTestBlocksAndTxs(EXPECTED_MAX_BLOCKS);
       mockP2pWithTxs(txs);

yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts

@@ -906,6 +906,7 @@ export class CheckpointProposalJob implements Traceable {
         blockTimestamp: timestamp,
         // Create an empty block if we haven't already and this is the last one
         forceCreate: timingInfo.isLastBlock && blocksBuilt === 0 && this.config.buildCheckpointIfEmpty,
+        isLastBlock: timingInfo.isLastBlock,
         buildDeadline: new Date(timingInfo.deadline * 1000),
         blockNumber,
         indexWithinCheckpoint,
@@ -1030,6 +1031,7 @@ export class CheckpointProposalJob implements Traceable {
     checkpointBuilder: CheckpointBuilder,
     opts: {
       forceCreate?: boolean;
+      isLastBlock: boolean;
       blockTimestamp: bigint;
       blockNumber: BlockNumber;
       indexWithinCheckpoint: IndexWithinCheckpoint;
@@ -1238,11 +1240,12 @@ export class CheckpointProposalJob implements Traceable {
   @trackSpan('CheckpointProposalJob.waitForMinTxs')
   private async waitForMinTxs(opts: {
     forceCreate?: boolean;
+    isLastBlock: boolean;
     blockNumber: BlockNumber;
     indexWithinCheckpoint: IndexWithinCheckpoint;
     buildDeadline: Date | undefined;
   }): Promise<{ canStartBuilding: boolean; availableTxs: number; minTxs: number }> {
-    const { indexWithinCheckpoint, blockNumber, buildDeadline, forceCreate } = opts;
+    const { indexWithinCheckpoint, blockNumber, buildDeadline, forceCreate, isLastBlock } = opts;
 
     // We only allow a block with 0 txs in the first block of the checkpoint
     const minTxs = indexWithinCheckpoint > 0 && this.config.minTxsPerBlock === 0 ? 1 : this.config.minTxsPerBlock;
@@ -1252,19 +1255,33 @@ export class CheckpointProposalJob implements Traceable {
       ? new Date(buildDeadline.getTime() - this.timetable.minBlockDuration * 1000)
       : undefined;
 
+    // When enabled, the final block of a checkpoint keeps collecting txs until its build deadline rather than
+    // sealing as soon as minTxs are available. This lets a burst of txs submitted late in the slot all land in
+    // the final block instead of spilling into the next checkpoint. Opt-in (test only); production seals early.
+    const waitForBuildDeadline = isLastBlock && !!this.config.waitForBuildDeadlineOnFinalBlock;
+
     let availableTxs = await this.p2pClient.getPendingTxCount();
 
-    while (!forceCreate && availableTxs < minTxs) {
-      // If we're past deadline, or we have no deadline, give up
+    while (!forceCreate) {
       const now = this.dateProvider.nowAsDate();
-      if (startBuildingDeadline === undefined || now >= startBuildingDeadline) {
+      const pastDeadline = startBuildingDeadline === undefined || now >= startBuildingDeadline;
+      const haveMinTxs = availableTxs >= minTxs;
+
+      // Earlier blocks (and the final block by default) seal as soon as they reach minTxs. With
+      // waitForBuildDeadlineOnFinalBlock set, the final block instead waits until its build deadline.
+      const doneWaiting = waitForBuildDeadline ? pastDeadline && haveMinTxs : haveMinTxs;
+      if (doneWaiting) {
+        break;
+      }
+      // Out of time without minTxs: give up.
+      if (pastDeadline) {
         return { canStartBuilding: false, availableTxs, minTxs };
       }
 
       // Wait a bit before checking again
       this.setState(SequencerState.WAITING_FOR_TXS);
       this.log.verbose(
-        `Waiting for enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.targetSlot} (have ${availableTxs} but need ${minTxs})`,
+        `Waiting for txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.targetSlot} (have ${availableTxs}, need ${minTxs}${waitForBuildDeadline ? ', final block collecting until build deadline' : ''})`,
         { blockNumber, slot: this.targetSlot, indexWithinCheckpoint },
       );
       await this.waitForTxsPollingInterval();

yarn-project/stdlib/src/interfaces/configs.ts

@@ -108,6 +108,12 @@ export interface SequencerConfig {
   expectedBlockProposalsPerSlot?: number;
   /** Have sequencer build and publish an empty checkpoint if there are no txs */
   buildCheckpointIfEmpty?: boolean;
+  /**
+   * On the final block of a checkpoint, wait until the block build deadline before sealing rather than
+   * sealing as soon as `minTxsPerBlock` are available. This lets a burst of txs submitted late in the slot
+   * all land in the final block instead of spilling into the next checkpoint (for testing only).
+   */
+  waitForBuildDeadlineOnFinalBlock?: boolean;
   /** Skip pushing proposed blocks to archiver (default: false) */
   skipPushProposedBlocksToArchiver?: boolean;
   /** Minimum number of blocks required for a checkpoint proposal (test only, defaults to undefined = no minimum) */
@@ -171,6 +177,7 @@ export const SequencerConfigSchema = zodFor<SequencerConfig>()(
     checkpointProposalSyncGraceSeconds: z.number().nonnegative().optional(),
     expectedBlockProposalsPerSlot: z.number().nonnegative().optional(),
     buildCheckpointIfEmpty: z.boolean().optional(),
+    waitForBuildDeadlineOnFinalBlock: z.boolean().optional(),
     skipPushProposedBlocksToArchiver: z.boolean().optional(),
     minBlocksForCheckpoint: z.number().positive().optional(),
     skipPublishingCheckpointsPercent: z.number().gte(0).lte(100).optional(),