From f0b94287c5f94832e22af6cc22b6548b3bff863d Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Mon, 15 Jun 2026 16:25:58 +0000 Subject: [PATCH 1/2] feat(sequencer): add waitForBuildDeadlineOnFinalBlock flag The proposer seals the final block of a checkpoint as soon as minTxsPerBlock are available and then snapshots the mempool. When txs are submitted as a burst late in the slot, they propagate over gossip one at a time, so the snapshot captures only a subset and the remainder spill into the next checkpoint. Add an opt-in sequencer flag (default off, so production behaviour is unchanged) that makes the final block of a checkpoint keep collecting txs until its build deadline before sealing. The build deadline already reserves the post-build window for attestation collection, so this does not change checkpoint timing. Enable the flag in e2e_epochs/epochs_mbps_redistribution, which dumps a burst of late txs and asserts they all land in the final block; without this it raced the proposer's snapshot and flaked. Covered by new checkpoint_proposal_job timing unit tests for both the default (seal early) and flag-on (wait) behaviour. --- .../epochs_mbps_redistribution.test.ts | 4 ++ yarn-project/sequencer-client/src/config.ts | 7 +++ .../sequencer/checkpoint_proposal_job.test.ts | 3 ++ .../checkpoint_proposal_job.timing.test.ts | 53 +++++++++++++++++++ .../src/sequencer/checkpoint_proposal_job.ts | 27 ++++++++-- yarn-project/stdlib/src/interfaces/configs.ts | 7 +++ 6 files changed, 96 insertions(+), 5 deletions(-) diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_mbps_redistribution.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_mbps_redistribution.test.ts index 3a7b8e7f16d6..82a77ac353b9 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_mbps_redistribution.test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_mbps_redistribution.test.ts @@ -109,6 +109,10 @@ describe('e2e_epochs/epochs_mbps_redistribution', () => { // With 3 blocks and multiplier 1.2: maxTxsPerBlock = ceil(TOTAL_TX_COUNT/3*1.2). // The redistribution should cap early blocks, preserving budget for the last block. maxTxsPerCheckpoint: TOTAL_TX_COUNT, + // Make the final block collect txs until its build deadline rather than sealing on the first late tx. + // Otherwise the proposer snapshots the mempool mid-arrival and the late txs (which propagate over gossip + // one at a time) spill across two blocks, making the `lateBlockNumbers` assertion flaky. + waitForBuildDeadlineOnFinalBlock: true, // PXE syncs on checkpointed chain tip. pxeOpts: { syncChainTip: 'checkpointed' }, ...contextConfigOverride, diff --git a/yarn-project/sequencer-client/src/config.ts b/yarn-project/sequencer-client/src/config.ts index 0d1d24e52b92..69c9fbb6b83f 100644 --- a/yarn-project/sequencer-client/src/config.ts +++ b/yarn-project/sequencer-client/src/config.ts @@ -44,6 +44,7 @@ export const DefaultSequencerConfig = { sequencerPollingIntervalMS: 500, minTxsPerBlock: 1, buildCheckpointIfEmpty: false, + waitForBuildDeadlineOnFinalBlock: false, publishTxsWithProposals: false, perBlockAllocationMultiplier: MIN_PER_BLOCK_ALLOCATION_MULTIPLIER, perBlockDAAllocationMultiplier: MIN_PER_BLOCK_DA_ALLOCATION_MULTIPLIER, @@ -241,6 +242,12 @@ export const sequencerConfigMappings: ConfigMappingsType = { description: 'Have sequencer build and publish an empty checkpoint if there are no txs', ...booleanConfigHelper(DefaultSequencerConfig.buildCheckpointIfEmpty), }, + waitForBuildDeadlineOnFinalBlock: { + description: + 'Wait until the build deadline before sealing the final block of a checkpoint instead of sealing as ' + + 'soon as minTxsPerBlock are available (for testing only)', + ...booleanConfigHelper(DefaultSequencerConfig.waitForBuildDeadlineOnFinalBlock), + }, skipPushProposedBlocksToArchiver: { description: 'Skip pushing proposed blocks to archiver (default: true)', ...booleanConfigHelper(DefaultSequencerConfig.skipPushProposedBlocksToArchiver), diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts index 119b88234b77..851a91e3a76c 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts @@ -1352,6 +1352,7 @@ describe('CheckpointProposalJob', () => { const result = await job.buildSingleBlock(checkpointBuilder, { blockNumber: newBlockNumber, indexWithinCheckpoint: IndexWithinCheckpoint(1), + isLastBlock: false, buildDeadline: undefined, blockTimestamp: 0n, txHashesAlreadyIncluded: new Set(), @@ -1373,6 +1374,7 @@ describe('CheckpointProposalJob', () => { const result = await job.buildSingleBlock(checkpointBuilder, { blockNumber: newBlockNumber, indexWithinCheckpoint: IndexWithinCheckpoint(1), + isLastBlock: false, buildDeadline: undefined, blockTimestamp: 0n, txHashesAlreadyIncluded: new Set(), @@ -1742,6 +1744,7 @@ class TestCheckpointProposalJob extends CheckpointProposalJob { checkpointBuilder: CheckpointBuilder, opts: { forceCreate?: boolean; + isLastBlock: boolean; blockTimestamp: bigint; blockNumber: BlockNumber; indexWithinCheckpoint: IndexWithinCheckpoint; diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.timing.test.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.timing.test.ts index 2acf3e4cdf4c..51d7f294f85d 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.timing.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.timing.test.ts @@ -561,6 +561,59 @@ describe('CheckpointProposalJob Timing Tests', () => { expect(checkpointMetrics.noteCheckpointBroadcast).toHaveBeenCalledWith(expect.any(Number)); }); + // 30s block duration in the 72s slot derives exactly one block, so the single built block is also the + // last block of the checkpoint - the only block affected by waitForBuildDeadlineOnFinalBlock. + const singleBlockTimetable = () => + makeProposerTimetable({ + l1Constants, + p2pPropagationTime: P2P_PROPAGATION_TIME, + checkpointProposalPrepareTime: CHECKPOINT_ASSEMBLE_TIME, + blockDurationMs: 30000, + }); + + it('seals the final block as soon as txs are available by default', async () => { + const { blocks, txs } = await createTestBlocksAndTxs(1); + mockP2pWithTxs(txs); + checkpointBuilder.seedBlocks(blocks, [[txs[0]]]); + checkpointBuilder.setExecutionDurations([1]); + validatorClient.collectAttestations.mockResolvedValue(getAttestations(blocks[0])); + + setTimeInSlot(0.5); + + const job = createJob(); + job.setTimetable(singleBlockTimetable()); + + await job.execute(); + + expect(checkpointBuilder.buildBlockCalls.length).toBe(1); + // A tx is available from the start (0.5s), so the block is built immediately rather than waiting. + expect(checkpointBuilder.recordedBuildTimes[0].startTime).toBeLessThan(2); + }); + + it('waits until the build deadline before sealing the final block when waitForBuildDeadlineOnFinalBlock is set', async () => { + const { blocks, txs } = await createTestBlocksAndTxs(1); + mockP2pWithTxs(txs); + checkpointBuilder.seedBlocks(blocks, [[txs[0]]]); + checkpointBuilder.setExecutionDurations([1]); + validatorClient.collectAttestations.mockResolvedValue(getAttestations(blocks[0])); + + setTimeInSlot(0.5); + + const job = createJob(); + job.setTimetable(singleBlockTimetable()); + job.updateConfig({ waitForBuildDeadlineOnFinalBlock: true }); + + await job.execute(); + + expect(checkpointBuilder.buildBlockCalls.length).toBe(1); + // Even though the tx is available from the start, the final block now waits until its build deadline + // (minus min_block_duration) before sealing, so a late burst of txs would still be picked up. + const deadline = checkpointBuilder.buildBlockCalls[0].opts.deadline!; + const startBuildingDeadline = + deadline.getTime() / 1000 - getSlotStartTime(slotNumber) - job.getTimetable().minBlockDuration; + expect(checkpointBuilder.recordedBuildTimes[0].startTime).toBeGreaterThanOrEqual(startBuildingDeadline - 0.01); + }); + it('builds maximum blocks when given enough time', async () => { const { blocks, txs } = await createTestBlocksAndTxs(EXPECTED_MAX_BLOCKS); mockP2pWithTxs(txs); diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts index 8c60a98e8e5d..ec7d6d5a6453 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts @@ -906,6 +906,7 @@ export class CheckpointProposalJob implements Traceable { blockTimestamp: timestamp, // Create an empty block if we haven't already and this is the last one forceCreate: timingInfo.isLastBlock && blocksBuilt === 0 && this.config.buildCheckpointIfEmpty, + isLastBlock: timingInfo.isLastBlock, buildDeadline: new Date(timingInfo.deadline * 1000), blockNumber, indexWithinCheckpoint, @@ -1030,6 +1031,7 @@ export class CheckpointProposalJob implements Traceable { checkpointBuilder: CheckpointBuilder, opts: { forceCreate?: boolean; + isLastBlock: boolean; blockTimestamp: bigint; blockNumber: BlockNumber; indexWithinCheckpoint: IndexWithinCheckpoint; @@ -1238,11 +1240,12 @@ export class CheckpointProposalJob implements Traceable { @trackSpan('CheckpointProposalJob.waitForMinTxs') private async waitForMinTxs(opts: { forceCreate?: boolean; + isLastBlock: boolean; blockNumber: BlockNumber; indexWithinCheckpoint: IndexWithinCheckpoint; buildDeadline: Date | undefined; }): Promise<{ canStartBuilding: boolean; availableTxs: number; minTxs: number }> { - const { indexWithinCheckpoint, blockNumber, buildDeadline, forceCreate } = opts; + const { indexWithinCheckpoint, blockNumber, buildDeadline, forceCreate, isLastBlock } = opts; // We only allow a block with 0 txs in the first block of the checkpoint const minTxs = indexWithinCheckpoint > 0 && this.config.minTxsPerBlock === 0 ? 1 : this.config.minTxsPerBlock; @@ -1252,19 +1255,33 @@ export class CheckpointProposalJob implements Traceable { ? new Date(buildDeadline.getTime() - this.timetable.minBlockDuration * 1000) : undefined; + // When enabled, the final block of a checkpoint keeps collecting txs until its build deadline rather than + // sealing as soon as minTxs are available. This lets a burst of txs submitted late in the slot all land in + // the final block instead of spilling into the next checkpoint. Opt-in (test only); production seals early. + const waitForBuildDeadline = isLastBlock && !!this.config.waitForBuildDeadlineOnFinalBlock; + let availableTxs = await this.p2pClient.getPendingTxCount(); - while (!forceCreate && availableTxs < minTxs) { - // If we're past deadline, or we have no deadline, give up + while (!forceCreate) { const now = this.dateProvider.nowAsDate(); - if (startBuildingDeadline === undefined || now >= startBuildingDeadline) { + const pastDeadline = startBuildingDeadline === undefined || now >= startBuildingDeadline; + const haveMinTxs = availableTxs >= minTxs; + + // Earlier blocks (and the final block by default) seal as soon as they reach minTxs. With + // waitForBuildDeadlineOnFinalBlock set, the final block instead waits until its build deadline. + const doneWaiting = waitForBuildDeadline ? pastDeadline && haveMinTxs : haveMinTxs; + if (doneWaiting) { + break; + } + // Out of time without minTxs: give up. + if (pastDeadline) { return { canStartBuilding: false, availableTxs, minTxs }; } // Wait a bit before checking again this.setState(SequencerState.WAITING_FOR_TXS); this.log.verbose( - `Waiting for enough txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.targetSlot} (have ${availableTxs} but need ${minTxs})`, + `Waiting for txs to build block ${blockNumber} at index ${indexWithinCheckpoint} in slot ${this.targetSlot} (have ${availableTxs}, need ${minTxs}${waitForBuildDeadline ? ', final block collecting until build deadline' : ''})`, { blockNumber, slot: this.targetSlot, indexWithinCheckpoint }, ); await this.waitForTxsPollingInterval(); diff --git a/yarn-project/stdlib/src/interfaces/configs.ts b/yarn-project/stdlib/src/interfaces/configs.ts index a806a31e0354..e9056794ef85 100644 --- a/yarn-project/stdlib/src/interfaces/configs.ts +++ b/yarn-project/stdlib/src/interfaces/configs.ts @@ -108,6 +108,12 @@ export interface SequencerConfig { expectedBlockProposalsPerSlot?: number; /** Have sequencer build and publish an empty checkpoint if there are no txs */ buildCheckpointIfEmpty?: boolean; + /** + * On the final block of a checkpoint, wait until the block build deadline before sealing rather than + * sealing as soon as `minTxsPerBlock` are available. This lets a burst of txs submitted late in the slot + * all land in the final block instead of spilling into the next checkpoint (for testing only). + */ + waitForBuildDeadlineOnFinalBlock?: boolean; /** Skip pushing proposed blocks to archiver (default: false) */ skipPushProposedBlocksToArchiver?: boolean; /** Minimum number of blocks required for a checkpoint proposal (test only, defaults to undefined = no minimum) */ @@ -171,6 +177,7 @@ export const SequencerConfigSchema = zodFor()( checkpointProposalSyncGraceSeconds: z.number().nonnegative().optional(), expectedBlockProposalsPerSlot: z.number().nonnegative().optional(), buildCheckpointIfEmpty: z.boolean().optional(), + waitForBuildDeadlineOnFinalBlock: z.boolean().optional(), skipPushProposedBlocksToArchiver: z.boolean().optional(), minBlocksForCheckpoint: z.number().positive().optional(), skipPublishingCheckpointsPercent: z.number().gte(0).lte(100).optional(), From 12afdb795ab41af6fd6d83996c983f0916495bc9 Mon Sep 17 00:00:00 2001 From: Phil Windle Date: Mon, 15 Jun 2026 16:28:09 +0000 Subject: [PATCH 2/2] test(ci): remove epochs_mbps_redistribution flake entry The waitForBuildDeadlineOnFinalBlock flag enabled in this PR makes the test deterministic, so it no longer needs to be tolerated as a flake (added in #24098). --- .test_patterns.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.test_patterns.yml b/.test_patterns.yml index 3816c919c8e4..498fc75e22ad 100644 --- a/.test_patterns.yml +++ b/.test_patterns.yml @@ -275,13 +275,6 @@ tests: owners: - *sean - # Asserts all late txs land in the checkpoint's last block, but the proposer seals the last block as - # soon as one tx is available and snapshots the mempool, so late txs still propagating over gossip - # spill into the next checkpoint. Flaky until the sequencer waits for the last block's tx window. - - regex: "src/e2e_epochs/epochs_mbps_redistribution.test.ts" - owners: - - *phil - # Blanket flake patterns for unstable p2p, epoch, and l1 tx utils tests # This is a temporary measure while team bandwidth is constrained # Replaced many specific patterns - see https://github.com/AztecProtocol/aztec-packages/pull/17962 for historical context