test(e2e): enable pipelining in l1-reorgs and mbps redistribution tests (#23009)

No major changes needed

Author: spalladino

yarn-project/end-to-end/src/e2e_epochs/epochs_l1_reorgs.parallel.test.ts

@@ -60,9 +60,6 @@ describe('e2e_epochs/epochs_l1_reorgs', () => {
   };
 
   beforeEach(async () => {
-    // Note: pipelining is NOT enabled for this test because it manipulates L1 state directly
-    // (reorgs, tx cancellation) with cancelTxOnTimeout: false and maxSpeedUpAttempts: 0,
-    // which conflicts with pipelining's assumption that previous checkpoints land on L1 promptly.
     test = await EpochsTestContext.setup({
       numberOfAccounts: 1,
       maxSpeedUpAttempts: 0, // Do not speed up l1 txs, we dont want them to land
@@ -78,6 +75,10 @@ describe('e2e_epochs/epochs_l1_reorgs', () => {
       aztecProofSubmissionEpochs: 1,
       // Use 32 slots/epoch (matching real Ethereum mainnet)
       anvilSlotsInAnEpoch: 32,
+      // Pipelining + multi-blocks-per-slot: 8s blocks fit ~4 blocks per 36s slot, and TX_COUNT=8
+      // ensures multiple checkpoints have multiple blocks
+      enableProposerPipelining: true,
+      inboxLag: 2,
     });
     ({ proverDelayer, sequencerDelayer, context, logger, monitor, L1_BLOCK_TIME_IN_S, L2_SLOT_DURATION_IN_S } = test);
     node = context.aztecNode;
@@ -220,12 +221,12 @@ describe('e2e_epochs/epochs_l1_reorgs', () => {
       const targetProvenCheckpoint = CheckpointNumber(initialProvenCheckpoint + 1);
 
       // Wait until we have proven something and the nodes have caught up
-      // Use a longer timeout since we need to wait for the epoch to complete (~288s) plus proving time
+      // Use a longer timeout since we need to wait for the epoch to complete (~288s) plus proving time.
       const epochDurationSeconds = test.constants.epochDuration * test.constants.slotDuration;
       logger.warn(`Waiting for initial proof to land`);
       const provenCheckpoint = await test.waitUntilProvenCheckpointNumber(
         targetProvenCheckpoint,
-        epochDurationSeconds * 2,
+        epochDurationSeconds * 4,
       );
       await retryUntil(() => getProvenCheckpointNumber(node).then(cp => cp >= provenCheckpoint), 'node sync', 10, 0.1);
 
@@ -270,11 +271,16 @@ describe('e2e_epochs/epochs_l1_reorgs', () => {
       proverDelayer.cancelNextTx();
 
       // Expect pending chain to advance, so there's something to be pruned
-      await retryUntil(() => getCheckpointNumber(node).then(cp => cp > initialCheckpoint), 'node sync', 60, 0.1);
+      await retryUntil(
+        () => getCheckpointNumber(node).then(cp => cp > initialCheckpoint),
+        'node sync',
+        L2_SLOT_DURATION_IN_S * 4,
+        0.1,
+      );
 
       // Wait until the end of the proof submission window for the first unproven epoch
       const firstUnprovenCheckpoint = CheckpointNumber(initialProvenCheckpoint + 1);
-      await test.waitUntilCheckpointNumber(firstUnprovenCheckpoint, 60);
+      await test.waitUntilCheckpointNumber(firstUnprovenCheckpoint, L2_SLOT_DURATION_IN_S * 4);
       const epochToWaitFor = await test.rollup.getEpochNumberForCheckpoint(firstUnprovenCheckpoint);
       await test.waitUntilLastSlotOfProofSubmissionWindow(epochToWaitFor);
       await monitor.run(true);
@@ -339,18 +345,21 @@ describe('e2e_epochs/epochs_l1_reorgs', () => {
 
       // Wait until CHECKPOINT_NUMBER is mined and node synced, and stop the sequencer
       const CHECKPOINT_NUMBER = CheckpointNumber(initialCheckpoint + 3);
-      await test.waitUntilCheckpointNumber(CHECKPOINT_NUMBER, L2_SLOT_DURATION_IN_S * 7);
+      await test.waitUntilCheckpointNumber(CHECKPOINT_NUMBER, L2_SLOT_DURATION_IN_S * 10);
       expect(monitor.checkpointNumber).toEqual(CHECKPOINT_NUMBER);
+      // Stop the sequencer immediately so any in-flight pipelined publish for CHECKPOINT_NUMBER+1
+      // doesn't extend l1BlockNumber before we capture it. setConfig alone is not enough under
+      // pipelining because already-constructed jobs snapshot the old config.
+      await context.sequencer!.stop();
+      logger.warn(`Sequencer stopped`);
       const l1BlockNumber = monitor.l1BlockNumber;
       // Wait for node to sync to the checkpoint.
       await retryUntil(() => getCheckpointNumber(node).then(b => b === CHECKPOINT_NUMBER), 'node sync', 10, 0.1);
+      logger.warn(`Reached checkpoint ${CHECKPOINT_NUMBER}`);
 
       // Verify multi-block checkpoints were built before we do the reorg
       await test.assertMultipleBlocksPerSlot(2);
 
-      logger.warn(`Reached checkpoint ${CHECKPOINT_NUMBER}. Stopping block production.`);
-      await context.aztecNodeAdmin.setConfig({ minTxsPerBlock: 100 });
-
       // Remove the L2 block from L1
       const l1BlocksToReorg = monitor.l1BlockNumber - l1BlockNumber + 1;
       await context.cheatCodes.eth.reorgWithReplacement(l1BlocksToReorg);
@@ -374,19 +383,22 @@ describe('e2e_epochs/epochs_l1_reorgs', () => {
       // Wait until the checkpoint *before* CHECKPOINT_NUMBER is mined and node synced
       const CHECKPOINT_NUMBER = CheckpointNumber(initialCheckpoint + 3);
       const prevCheckpointNumber = CheckpointNumber(CHECKPOINT_NUMBER - 1);
-      await test.waitUntilCheckpointNumber(prevCheckpointNumber, L2_SLOT_DURATION_IN_S * 7);
+      await test.waitUntilCheckpointNumber(prevCheckpointNumber, L2_SLOT_DURATION_IN_S * 10);
       expect(monitor.checkpointNumber).toEqual(prevCheckpointNumber);
       // Wait for node to sync to the checkpoint
       await retryUntil(() => getCheckpointNumber(node).then(b => b === prevCheckpointNumber), 'node sync', 5, 0.1);
 
       // Verify multi-block checkpoints were built before we do the reorg
       await test.assertMultipleBlocksPerSlot(2);
 
-      // Cancel the next tx to be mined and pause the sequencer
+      // Cancel the next tx to be mined (the proposal for CHECKPOINT_NUMBER) and pause the sequencer.
+      // Under pipelining we then stop the sequencer entirely so an in-flight pipelined job for
+      // CHECKPOINT_NUMBER+1 cannot escape and publish onto L1 before our reorg captures the gap.
       sequencerDelayer.cancelNextTx();
       await retryUntil(() => sequencerDelayer.getCancelledTxs().length, 'next block', L2_SLOT_DURATION_IN_S * 2, 0.1);
       const [l2BlockTx] = sequencerDelayer.getCancelledTxs();
-      await context.aztecNodeAdmin.setConfig({ minTxsPerBlock: 100 });
+      await context.sequencer!.stop();
+      logger.warn(`Sequencer stopped`);
 
       // Save the L1 block number when the L2 block would have been mined
       const l1BlockNumber = monitor.l1BlockNumber;
@@ -447,7 +459,7 @@ describe('e2e_epochs/epochs_l1_reorgs', () => {
     it('updates L1 to L2 messages changed due to an L1 reorg', async () => {
       // Send L2 txs to trigger multi-block checkpoints and wait for them to land in a checkpoint
       await sendTransactions(TX_COUNT, 100);
-      await test.waitUntilCheckpointNumber(CheckpointNumber(2), L2_SLOT_DURATION_IN_S * 4);
+      await test.waitUntilCheckpointNumber(CheckpointNumber(2), L2_SLOT_DURATION_IN_S * 6);
 
       // Send 3 messages and wait for archiver sync
       logger.warn(`Sending 3 cross chain messages`);
@@ -484,7 +496,7 @@ describe('e2e_epochs/epochs_l1_reorgs', () => {
     it('handles missed message inserted by an L1 reorg', async () => {
       // Send L2 txs to trigger multi-block checkpoints and wait for them to land in a checkpoint
       await sendTransactions(TX_COUNT, 200);
-      await test.waitUntilCheckpointNumber(CheckpointNumber(2), L2_SLOT_DURATION_IN_S * 4);
+      await test.waitUntilCheckpointNumber(CheckpointNumber(2), L2_SLOT_DURATION_IN_S * 6);
 
       // Send a message and wait for node to sync it
       logger.warn(`Sending first cross chain message`);

yarn-project/end-to-end/src/e2e_epochs/epochs_mbps_redistribution.test.ts

@@ -94,6 +94,8 @@ describe('e2e_epochs/epochs_mbps_redistribution', () => {
     test = await EpochsTestContext.setup({
       numberOfAccounts: 0,
       initialValidators: validators,
+      enableProposerPipelining: true,
+      inboxLag: 2,
       mockGossipSubNetwork: true,
       disableAnvilTestWatcher: true,
       startProverNode: true,

yarn-project/end-to-end/src/e2e_epochs/epochs_proof_fails.parallel.test.ts

@@ -2,13 +2,15 @@ import { getTimestampRangeForEpoch } from '@aztec/aztec.js/block';
 import type { Logger } from '@aztec/aztec.js/log';
 import { BatchedBlob } from '@aztec/blob-lib/types';
 import { RollupContract } from '@aztec/ethereum/contracts';
-import { type Delayer, waitUntilL1Timestamp } from '@aztec/ethereum/l1-tx-utils';
+import type { Delayer } from '@aztec/ethereum/l1-tx-utils';
 import { ChainMonitor } from '@aztec/ethereum/test';
 import type { ViemClient } from '@aztec/ethereum/types';
-import { CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types';
+import { CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
 import { promiseWithResolvers } from '@aztec/foundation/promise';
+import { retryUntil } from '@aztec/foundation/retry';
 import { sleep } from '@aztec/foundation/sleep';
 import type { TestProverNode } from '@aztec/prover-node/test';
+import type { SequencerEvents } from '@aztec/sequencer-client';
 import { type L1RollupConstants, getEpochAtSlot } from '@aztec/stdlib/epoch-helpers';
 import { Proof } from '@aztec/stdlib/proofs';
 import { RootRollupPublicInputs } from '@aztec/stdlib/rollup';
@@ -27,27 +29,27 @@ describe('e2e_epochs/epochs_proof_fails', () => {
   let constants: L1RollupConstants;
   let logger: Logger;
   let proverDelayer: Delayer;
-  let sequencerDelayer: Delayer;
   let monitor: ChainMonitor;
 
-  let L1_BLOCK_TIME_IN_S: number;
   let L2_SLOT_DURATION_IN_S: number;
 
   let test: EpochsTestContext;
 
   beforeEach(async () => {
-    // Note: pipelining is NOT enabled for this test because it deliberately manipulates L1 tx timing
-    // (via proverDelayer/sequencerDelayer with cancelTxOnTimeout: false and maxSpeedUpAttempts: 0),
-    // which conflicts with pipelining's assumption that previous checkpoints land on L1 promptly.
     test = await EpochsTestContext.setup({
       maxSpeedUpAttempts: 0, // No speed ups
       startProverNode: false, // Avoid early proving
       ethereumSlotDuration: 8,
-      aztecEpochDuration: 8, // Bump empoch duration so we can land at least one block in epoch 0
+      aztecEpochDuration: 8, // Bump epoch duration so we can land at least one block in epoch 0
+      aztecSlotDurationInL1Slots: 2,
+      blockDurationMs: 3000, // 3s blocks → 2 blocks per checkpoint under pipelining
       cancelTxOnTimeout: false,
+      enableProposerPipelining: true,
+      enforceTimeTable: true,
+      inboxLag: 2,
     });
-    ({ sequencerDelayer, context, l1Client, rollup, constants, logger, monitor } = test);
-    ({ L1_BLOCK_TIME_IN_S, L2_SLOT_DURATION_IN_S } = test);
+    ({ context, l1Client, rollup, constants, logger, monitor } = test);
+    ({ L2_SLOT_DURATION_IN_S } = test);
   });
 
   afterEach(async () => {
@@ -56,13 +58,14 @@ describe('e2e_epochs/epochs_proof_fails', () => {
   });
 
   it('does not allow submitting proof after epoch end', async () => {
-    // Here we cause a re-org by not publishing the proof for epoch 0 until after the end of epoch 1
-    // The proof will be rejected and a re-org will take place
-
-    // Ensure that there was at least one checkpoint mined in epoch 0, otherwise this test fails, since it
-    // relies on the proof for epoch zero not landing in time, which will never happen if there is
-    // nothing to prove on epoch zero. We need to wait for the checkpoint L1 tx to be mined, not just
-    // for the block to appear in the node's world state, since the propose tx may still be in-flight.
+    // Here we cause a re-org by not publishing the proof for epoch 0 until after the end of epoch 1.
+    // The proof will be rejected and a re-org will take place via the next post-deadline propose tx.
+    const publishedEvents: Parameters<SequencerEvents['checkpoint-published']>[0][] = [];
+    test.context.sequencer!.getSequencer().on('checkpoint-published', args => publishedEvents.push(args));
+
+    // Ensure that there was at least one checkpoint mined in epoch 0, otherwise this test fails, since
+    // it relies on the proof for epoch zero not landing in time, which will never happen if there is
+    // nothing to prove on epoch zero.
     await test.waitUntilCheckpointNumber(CheckpointNumber(1));
     const firstCheckpoint = await rollup.getCheckpoint(CheckpointNumber(1));
     const firstCheckpointEpoch = getEpochAtSlot(firstCheckpoint.slotNumber, test.constants);
@@ -75,38 +78,42 @@ describe('e2e_epochs/epochs_proof_fails', () => {
     // Get the prover delayer from the newly created prover node
     proverDelayer = proverNode.getProverNode()!.getDelayer()!;
 
-    // Hold off prover tx until end epoch 1
+    // Hold off the prover tx until epoch 2 starts (i.e. past the proof submission deadline)
     const [epoch2Start] = getTimestampRangeForEpoch(EpochNumber(2), constants);
     proverDelayer.pauseNextTxUntilTimestamp(epoch2Start);
-    logger.info(`Delayed prover tx until epoch 2 starts at ${epoch2Start}`);
+    logger.warn(`Delayed prover tx until epoch 2 starts at ${epoch2Start}`);
 
-    // Wait until the start of epoch 1 and grab the checkpoint number
+    // Wait until the start of epoch 1 and capture the checkpoint number before the rollback
     await test.waitUntilEpochStarts(EpochNumber(1));
-    const checkpointNumberAtEndOfEpoch0 = await rollup.getCheckpointNumber();
-    logger.info(`Starting epoch 1 after checkpoint ${checkpointNumberAtEndOfEpoch0}`);
-
-    // Wait until the last checkpoint of epoch 1 is published and then hold off the sequencer.
-    await test.waitUntilCheckpointNumber(
-      CheckpointNumber(checkpointNumberAtEndOfEpoch0 + test.epochDuration),
-      test.L2_SLOT_DURATION_IN_S * (test.epochDuration + 4),
+    const checkpointBeforeRollback = await rollup.getCheckpointNumber();
+    logger.warn(`Starting epoch 1 after checkpoint ${checkpointBeforeRollback}`);
+    expect(checkpointBeforeRollback).toBeGreaterThan(CheckpointNumber(1));
+
+    // Wait for the rollback to land via natural sequencer activity in epoch 2. We poll the
+    // checkpoint number rather than a fixed timestamp because the exact slot that triggers the
+    // prune depends on poll timing (see comment above).
+    await test.waitUntilEpochStarts(EpochNumber(2));
+    await retryUntil(
+      async () => (await rollup.getCheckpointNumber()) < checkpointBeforeRollback,
+      'rollup rolled back',
+      L2_SLOT_DURATION_IN_S * 4,
+      0.2,
     );
-    sequencerDelayer.pauseNextTxUntilTimestamp(epoch2Start + BigInt(L1_BLOCK_TIME_IN_S));
 
-    // Next sequencer to publish a block should trigger a rollback to block 1
-    await waitUntilL1Timestamp(l1Client, epoch2Start + BigInt(L1_BLOCK_TIME_IN_S));
-    expect(await rollup.getCheckpointNumber()).toEqual(CheckpointNumber(1));
-    expect(await rollup.getSlotNumber()).toEqual(SlotNumber(2 * test.epochDuration));
-
-    // The prover tx should have been rejected, and mined strictly before the one that triggered the rollback
+    // The prover tx should have been rejected as it was submitted past the deadline
     const lastProverTxHash = proverDelayer.getSentTxHashes().at(-1);
+    expect(lastProverTxHash).toBeDefined();
     const lastProverTxReceipt = await l1Client.getTransactionReceipt({ hash: lastProverTxHash! });
     expect(lastProverTxReceipt.status).toEqual('reverted');
 
-    const lastL2BlockTxHash = sequencerDelayer.getSentTxHashes().at(-1);
-    const lastL2BlockTxReceipt = await l1Client.getTransactionReceipt({ hash: lastL2BlockTxHash! });
-    expect(lastL2BlockTxReceipt.status).toEqual('success');
-    expect(lastL2BlockTxReceipt.blockNumber).toBeGreaterThan(lastProverTxReceipt!.blockNumber);
-    logger.info(`Test succeeded`);
+    // The post-rollback chain tip should be in epoch 2 (the rollback-triggering propose was made
+    // during epoch 2, after the deadline)
+    const checkpointAfterRollback = await rollup.getCheckpointNumber();
+    expect(checkpointAfterRollback).toBeLessThan(checkpointBeforeRollback);
+    const latestCheckpoint = await rollup.getCheckpoint(checkpointAfterRollback);
+    expect(getEpochAtSlot(latestCheckpoint.slotNumber, test.constants)).toEqual(EpochNumber(2));
+
+    logger.warn(`Test succeeded`);
   });
 
   it('aborts proving if end of next epoch is reached', async () => {
@@ -149,17 +156,17 @@ describe('e2e_epochs/epochs_proof_fails', () => {
     context.proverNode = proverNode;
 
     await test.waitUntilEpochStarts(1);
-    logger.info(`Starting epoch 1`);
+    logger.warn(`Starting epoch 1`);
     const proverTxCount = proverDelayer.getSentTxHashes().length;
 
     await test.waitUntilEpochStarts(2);
-    logger.info(`Starting epoch 2`);
+    logger.warn(`Starting epoch 2`);
 
     // No proof for epoch zero should have landed during epoch one
     expect(monitor.provenCheckpointNumber).toEqual(CheckpointNumber(0));
 
     // Wait until the prover job finalizes (and a bit more) and check that it aborted and never attempted to submit a tx
-    logger.info(`Awaiting finalize epoch`);
+    logger.warn(`Awaiting finalize epoch`);
     await finalizeEpochPromise.promise;
     await sleep(1000);
     expect(proverDelayer.getSentTxHashes().length - proverTxCount).toEqual(0);