test(e2e): enable pipelining on p2p tests

spalladino · spalladino · commit f9dcfd7e69d3 · 2026-05-07T18:54:36.000-03:00
diff --git a/yarn-project/end-to-end/src/e2e_p2p/add_rollup.test.ts b/yarn-project/end-to-end/src/e2e_p2p/add_rollup.test.ts
@@ -5,6 +5,7 @@ import { EthAddress } from '@aztec/aztec.js/addresses';
 import { waitForProven } from '@aztec/aztec.js/contracts';
 import { generateClaimSecret } from '@aztec/aztec.js/ethereum';
 import { Fr } from '@aztec/aztec.js/fields';
+import { waitForL1ToL2MessageReady } from '@aztec/aztec.js/messaging';
 import { RollupCheatCodes } from '@aztec/aztec/testing';
 import { FeeAssetHandlerContract, RegistryContract, RollupContract } from '@aztec/ethereum/contracts';
 import { deployRollupForUpgrade } from '@aztec/ethereum/deploy-aztec-l1-contracts';
@@ -13,7 +14,6 @@ import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses'
 import { L1TxUtils, createL1TxUtils } from '@aztec/ethereum/l1-tx-utils';
 import type { ExtendedViemWalletClient } from '@aztec/ethereum/types';
 import { CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types';
-import { retryUntil } from '@aztec/foundation/retry';
 import { sleep } from '@aztec/foundation/sleep';
 import {
   GovernanceAbi,
@@ -53,7 +53,7 @@ const BOOT_NODE_UDP_PORT = 4500;
 const DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'add-rollup-old-'));
 const DATA_DIR_NEW = fs.mkdtempSync(path.join(os.tmpdir(), 'add-rollup-new-'));
 
-jest.setTimeout(1000 * 60 * 10);
+jest.setTimeout(1000 * 60 * 20);
 
 /**
  * This test emulates the addition of a new rollup to the registry and tests that cross-chain messages work.
@@ -80,6 +80,14 @@ describe('e2e_p2p_add_rollup', () => {
         ...SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES,
         listenAddress: '127.0.0.1',
         governanceProposerRoundSize: 10,
+        enableProposerPipelining: true,
+        // Allow validators to build empty checkpoints under pipelining so the chain keeps
+        // advancing while we wait for L1->L2 messages to land in the next checkpoint's inbox tree.
+        minTxsPerBlock: 0,
+        // Pipelining starts cycle for checkpoint N+1 during slot N, but the inbox tree for
+        // checkpoint N is only sealed when checkpoint N is published. inboxLag: 2 sources
+        // L1->L2 messages from checkpoint N-1 (already sealed), avoiding L1ToL2MessagesNotReadyError.
+        inboxLag: 2,
       },
       startProverNode: false, // Start one later using p2p.
     });
@@ -307,8 +315,10 @@ describe('e2e_p2p_add_rollup', () => {
       });
 
       const makeMessageConsumable = async (msgHash: Fr) => {
-        // We poll isL1ToL2MessageSynced endpoint until the message is available
-        await retryUntil(async () => await node.isL1ToL2MessageSynced(msgHash), 'message sync', 10);
+        // Wait until the message is ready to be consumed (the rollup has reached the message's checkpoint).
+        // Using waitForL1ToL2MessageReady rather than isL1ToL2MessageSynced because with `inboxLag > 0`
+        // a synced message is not yet present in the latest checkpoint's inbox tree.
+        await waitForL1ToL2MessageReady(node, msgHash, { timeoutSeconds: 120 });
 
         const { receipt } = await testContract.methods
           .create_l2_to_l1_message_arbitrary_recipient_private(contentOutFromRollup, ethRecipient)
diff --git a/yarn-project/end-to-end/src/e2e_p2p/broadcasted_invalid_block_proposal_slash.test.ts b/yarn-project/end-to-end/src/e2e_p2p/broadcasted_invalid_block_proposal_slash.test.ts
@@ -62,6 +62,8 @@ describe('e2e_p2p_broadcasted_invalid_block_proposal_slash', () => {
         ethereumSlotDuration: ETHEREUM_SLOT_DURATION,
         aztecSlotDuration: AZTEC_SLOT_DURATION,
         aztecTargetCommitteeSize: COMMITTEE_SIZE,
+        enableProposerPipelining: true,
+        inboxLag: 2,
         aztecProofSubmissionEpochs: 1024, // effectively do not reorg
         slashInactivityConsecutiveEpochThreshold: 32, // effectively do not slash for inactivity
         minTxsPerBlock: 0, // always be building
diff --git a/yarn-project/end-to-end/src/e2e_p2p/data_withholding_slash.test.ts b/yarn-project/end-to-end/src/e2e_p2p/data_withholding_slash.test.ts
@@ -73,6 +73,8 @@ describe('e2e_p2p_data_withholding_slash', () => {
         slashAmountLarge: slashingUnit * 3n,
         slashSelfAllowed: true,
         minTxsPerBlock: 0,
+        enableProposerPipelining: true,
+        inboxLag: 2,
       },
     });
 
@@ -165,6 +167,11 @@ describe('e2e_p2p_data_withholding_slash', () => {
 
     // Re-create the nodes.
     // ASSUMING they sync in the middle of the epoch, they will "see" the reorg, and try to slash.
+    // Reset minTxsPerBlock to 0 so re-created validators build empty checkpoints. Under proposer
+    // pipelining, the vote-offenses signature is bound to the target slot and the multicall is only
+    // delayed to the target slot start when a checkpoint is being proposed; without a proposal,
+    // votes would mine in the current wall-clock slot, causing the EIP-712 signature verification to fail.
+    t.ctx.aztecNodeConfig.minTxsPerBlock = 0;
     t.logger.warn('Re-creating nodes');
     nodes = await createNodes(
       t.ctx.aztecNodeConfig,
diff --git a/yarn-project/end-to-end/src/e2e_p2p/duplicate_attestation_slash.test.ts b/yarn-project/end-to-end/src/e2e_p2p/duplicate_attestation_slash.test.ts
@@ -89,6 +89,8 @@ describe('e2e_p2p_duplicate_attestation_slash', () => {
         slashDuplicateProposalPenalty: slashingUnit,
         slashDuplicateAttestationPenalty: slashingUnit,
         slashingOffsetInRounds: 1,
+        enableProposerPipelining: true,
+        inboxLag: 2,
       },
     });
 
diff --git a/yarn-project/end-to-end/src/e2e_p2p/duplicate_proposal_slash.test.ts b/yarn-project/end-to-end/src/e2e_p2p/duplicate_proposal_slash.test.ts
@@ -2,6 +2,7 @@ import type { AztecNodeService } from '@aztec/aztec-node';
 import type { TestAztecNodeService } from '@aztec/aztec-node/test';
 import { EthAddress } from '@aztec/aztec.js/addresses';
 import { EpochNumber, SlotNumber } from '@aztec/foundation/branded-types';
+import { retryUntil } from '@aztec/foundation/retry';
 import { bufferToHex } from '@aztec/foundation/string';
 import { OffenseType } from '@aztec/slasher';
 import { TopicType } from '@aztec/stdlib/p2p';
@@ -79,6 +80,8 @@ describe('e2e_p2p_duplicate_proposal_slash', () => {
         blockDurationMs: BLOCK_DURATION * 1000,
         slashDuplicateProposalPenalty: slashingUnit,
         slashingOffsetInRounds: 1,
+        enableProposerPipelining: true,
+        inboxLag: 2,
       },
     });
 
@@ -224,24 +227,39 @@ describe('e2e_p2p_duplicate_proposal_slash', () => {
     t.logger.warn(`Advancing to target epoch ${targetEpoch}`);
     await t.ctx.cheatCodes.rollup.advanceToEpoch(targetEpoch);
 
-    // Wait for offense to be detected
-    // The honest nodes should detect the duplicate proposal from the malicious validator
+    // Wait for offense to be detected. Under proposer pipelining, checkpoint proposals are broadcast
+    // at the slot boundary while the receivers' wall clocks may have already advanced past the build
+    // slot — when that happens, honest nodes reject the gossip with "invalid slot number" before
+    // duplicate detection runs, so DUPLICATE_PROPOSAL is only observed by whichever node managed to
+    // process both proposals while still in the build slot (often the other malicious node, since
+    // they receive each other's broadcasts immediately). We therefore collect offenses from every
+    // node in the network and assert that at least one of them recorded the duplicate proposal.
     t.logger.warn('Waiting for duplicate proposal offense to be detected...');
-    const offenses = await awaitOffenseDetected({
+    await awaitOffenseDetected({
       epochDuration: t.ctx.aztecNodeConfig.aztecEpochDuration,
       logger: t.logger,
-      nodeAdmin: honestNode1, // Use honest node to check for offenses
+      nodeAdmin: honestNode1,
       slashingRoundSize,
       waitUntilOffenseCount: 1,
       timeoutSeconds: AZTEC_SLOT_DURATION * 16,
     });
 
-    t.logger.warn(`Collected offenses`, { offenses });
+    // Poll every node for DUPLICATE_PROPOSAL offenses, retrying briefly so any node that detected
+    // the duplicate after the initial offense was collected has time to flush it through the
+    // slasher's offenses-collector.
+    const proposalOffenses = await retryUntil(
+      async () => {
+        const allOffenses = (await Promise.all(nodes.map(n => n.getSlashOffenses('all')))).flat();
+        const filtered = allOffenses.filter(o => o.offenseType === OffenseType.DUPLICATE_PROPOSAL);
+        if (filtered.length > 0) {
+          return filtered;
+        }
+      },
+      'duplicate proposal offense',
+      AZTEC_SLOT_DURATION * 4,
+    );
 
-    // Filter to only DUPLICATE_PROPOSAL offenses. The two malicious nodes sharing the same key
-    // will also each self-attest to their own (different) checkpoint proposals, which causes honest
-    // nodes to detect a DUPLICATE_ATTESTATION as well. We only care about proposals here.
-    const proposalOffenses = offenses.filter(o => o.offenseType === OffenseType.DUPLICATE_PROPOSAL);
+    t.logger.warn(`Collected duplicate proposal offenses`, { proposalOffenses });
     expect(proposalOffenses.length).toBeGreaterThan(0);
     for (const offense of proposalOffenses) {
       expect(offense.validator.toString()).toEqual(maliciousValidatorAddress.toString());
diff --git a/yarn-project/end-to-end/src/e2e_p2p/fee_asset_price_oracle_gossip.test.ts b/yarn-project/end-to-end/src/e2e_p2p/fee_asset_price_oracle_gossip.test.ts
@@ -63,6 +63,8 @@ describe('e2e_p2p_network', () => {
         slashingRoundSizeInEpochs: 2,
         slashingQuorum: 5,
         listenAddress: '127.0.0.1',
+        enableProposerPipelining: true,
+        inboxLag: 2,
       },
     });
 
@@ -205,11 +207,23 @@ describe('e2e_p2p_network', () => {
       expect(validatorAddresses).toContain(signer);
     }
 
+    // Allow a tolerance up to the per-checkpoint modifier cap (±100 bps): under proposer pipelining,
+    // the modifier for checkpoint N is computed in slot N-1 from the price visible at prep time, but
+    // the on-chain new price is the parent fee header (one checkpoint ahead) modified by that value.
+    // This stale-read effect causes the price to oscillate around the oracle target with up to ~1%
+    // amplitude rather than land exactly on it. The test still verifies that the price tracks the
+    // oracle within the cap.
+    const TOLERANCE_BPS = 100n;
+    const absDiffBps = (a: bigint, b: bigint) => {
+      const d = diffInBps(a, b);
+      return d < 0n ? -d : d;
+    };
+
     await retryUntil(
       async () => {
         const currentPrice = await rollup.getEthPerFeeAsset();
         t.logger.info(`Current on-chain price: ${currentPrice}, waiting for: ${targetOraclePrice}`);
-        return diffInBps(currentPrice, targetOraclePrice) == 0n;
+        return absDiffBps(currentPrice, targetOraclePrice) <= TOLERANCE_BPS;
       },
       'price convergence toward oracle',
       120, // timeout in seconds
@@ -225,7 +239,7 @@ describe('e2e_p2p_network', () => {
       async () => {
         const currentPrice = await rollup.getEthPerFeeAsset();
         t.logger.info(`Current on-chain price: ${currentPrice}, waiting for: ${targetOraclePrice2}`);
-        return diffInBps(currentPrice, targetOraclePrice2) == 0n;
+        return absDiffBps(currentPrice, targetOraclePrice2) <= TOLERANCE_BPS;
       },
       'price convergence toward oracle',
       120, // timeout in seconds
@@ -237,6 +251,6 @@ describe('e2e_p2p_network', () => {
 
     // Verify the price moved toward the oracle price
     expect(finalPrice).toBeGreaterThan(initialOnChainPrice);
-    expect(diffInBps(finalPrice, targetOraclePrice2)).toBe(0n);
+    expect(absDiffBps(finalPrice, targetOraclePrice2)).toBeLessThanOrEqual(TOLERANCE_BPS);
   });
 });
diff --git a/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts b/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts
@@ -69,6 +69,8 @@ describe('e2e_p2p_network', () => {
         slashingRoundSizeInEpochs: 2,
         slashingQuorum: 5,
         listenAddress: '127.0.0.1',
+        enableProposerPipelining: true,
+        inboxLag: 2,
       },
     });
 
diff --git a/yarn-project/end-to-end/src/e2e_p2p/gossip_network_no_cheat.test.ts b/yarn-project/end-to-end/src/e2e_p2p/gossip_network_no_cheat.test.ts
@@ -68,6 +68,8 @@ describe('e2e_p2p_network', () => {
         // Without this, no blocks are built until txs arrive, and a failed checkpoint during tx
         // submission causes block pruning that invalidates tx references.
         minTxsPerBlock: 0,
+        enableProposerPipelining: true,
+        inboxLag: 2,
       },
     });
 
diff --git a/yarn-project/end-to-end/src/e2e_p2p/inactivity_slash_test.ts b/yarn-project/end-to-end/src/e2e_p2p/inactivity_slash_test.ts
@@ -58,6 +58,8 @@ export class P2PInactivityTest {
       basePort: BOOT_NODE_UDP_PORT,
       startProverNode: true,
       initialConfig: {
+        enableProposerPipelining: true,
+        inboxLag: 2,
         anvilSlotsInAnEpoch: 4,
         proverNodeConfig: { proverNodeEpochProvingDelayMs: AZTEC_SLOT_DURATION * 1000 },
         aztecTargetCommitteeSize: COMMITTEE_SIZE,
diff --git a/yarn-project/end-to-end/src/e2e_p2p/inactivity_slash_with_consecutive_epochs.test.ts b/yarn-project/end-to-end/src/e2e_p2p/inactivity_slash_with_consecutive_epochs.test.ts
@@ -55,7 +55,7 @@ describe('e2e_p2p_inactivity_slash_with_consecutive_epochs', () => {
         return offenses.length > 0 ? offenses : undefined;
       },
       'slash offenses',
-      slashInactivityConsecutiveEpochThreshold * aztecEpochDuration * aztecSlotDuration * 2,
+      slashInactivityConsecutiveEpochThreshold * aztecEpochDuration * aztecSlotDuration * 4,
     );
     expect(unique(offenses.map(o => o.validator.toString()))).toEqual([offlineValidator.toString()]);
     expect(unique(offenses.map(o => o.offenseType))).toEqual([OffenseType.INACTIVITY]);
diff --git a/yarn-project/end-to-end/src/e2e_p2p/multiple_validators_sentinel.parallel.test.ts b/yarn-project/end-to-end/src/e2e_p2p/multiple_validators_sentinel.parallel.test.ts
@@ -58,6 +58,8 @@ describe('e2e_p2p_multiple_validators_sentinel', () => {
         slashingRoundSizeInEpochs: 2,
         sentinelEnabled: true,
         slashInactivityPenalty: 0n, // Set to 0 to disable
+        enableProposerPipelining: true,
+        inboxLag: 2,
       },
     });
 
@@ -103,7 +105,20 @@ describe('e2e_p2p_multiple_validators_sentinel', () => {
   });
 
   it('collects attestations for all validators on a node', async () => {
+    // Ensure all nodes see each other, especially the sentinel, before starting slot counting
+    await t.waitForP2PMeshConnectivity([...nodes, sentinel]);
+
+    // Wait until validator nodes have advanced past their first proposed slot so that the
+    // pipelining warm-up period (where some attestations may be missed) is behind us.
     await t.monitor.run();
+    const warmupSlot = Number(t.monitor.l2SlotNumber) + 1;
+    t.logger.info(`Waiting for warmup slot ${warmupSlot} before establishing initial slot`);
+    await retryUntil(
+      async () => (await t.monitor.run()).l2SlotNumber >= warmupSlot,
+      'warmup slot',
+      AZTEC_SLOT_DURATION * 3,
+    );
+
     const { checkpointNumber: initialBlock, l2SlotNumber: initialSlot } = t.monitor;
 
     const timeout = AZTEC_SLOT_DURATION * SLOT_COUNT * 4;
diff --git a/yarn-project/end-to-end/src/e2e_p2p/preferred_gossip_network.test.ts b/yarn-project/end-to-end/src/e2e_p2p/preferred_gossip_network.test.ts
@@ -142,6 +142,9 @@ describe('e2e_p2p_preferred_network', () => {
         p2pDisableStatusHandshake: false,
         // Just for testing be aggressive here, don't allow any auth handshake failures
         p2pMaxFailedAuthAttemptsAllowed: 0,
+        minTxsPerBlock: 0,
+        enableProposerPipelining: true,
+        inboxLag: 2,
       },
     });
 
diff --git a/yarn-project/end-to-end/src/e2e_p2p/rediscovery.test.ts b/yarn-project/end-to-end/src/e2e_p2p/rediscovery.test.ts
@@ -35,6 +35,8 @@ describe('e2e_p2p_rediscovery', () => {
         ...SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES,
         aztecSlotDuration: 24,
         listenAddress: '127.0.0.1',
+        enableProposerPipelining: true,
+        inboxLag: 2,
       },
     });
     await t.setup();
diff --git a/yarn-project/end-to-end/src/e2e_p2p/reex.test.ts b/yarn-project/end-to-end/src/e2e_p2p/reex.test.ts
@@ -47,6 +47,8 @@ describe('e2e_p2p_reex', () => {
         txTimeoutMs: 30_000,
         listenAddress: '127.0.0.1',
         aztecProofSubmissionEpochs: 1024, // effectively do not reorg
+        enableProposerPipelining: true,
+        inboxLag: 2,
       },
     });
 
diff --git a/yarn-project/end-to-end/src/e2e_p2p/slash_veto_demo.test.ts b/yarn-project/end-to-end/src/e2e_p2p/slash_veto_demo.test.ts
@@ -76,6 +76,8 @@ describe('veto slash', () => {
         aztecProofSubmissionEpochs: 1024, // effectively do not reorg
         listenAddress: '127.0.0.1',
         minTxsPerBlock: 0,
+        enableProposerPipelining: true,
+        inboxLag: 2,
         aztecEpochDuration: EPOCH_DURATION,
         sentinelEnabled: true,
         slashSelfAllowed: true,
diff --git a/yarn-project/end-to-end/src/e2e_p2p/upgrade_governance_proposer.test.ts b/yarn-project/end-to-end/src/e2e_p2p/upgrade_governance_proposer.test.ts
@@ -54,6 +54,9 @@ describe('e2e_p2p_governance_proposer', () => {
         governanceProposerRoundSize: 10,
         activationThreshold: 10n ** 22n,
         ejectionThreshold: 5n ** 22n,
+        enableProposerPipelining: true,
+        inboxLag: 2,
+        minTxsPerBlock: 0,
       },
     });
 
diff --git a/yarn-project/end-to-end/src/e2e_p2p/valid_epoch_pruned_slash.test.ts b/yarn-project/end-to-end/src/e2e_p2p/valid_epoch_pruned_slash.test.ts
@@ -69,6 +69,8 @@ describe('e2e_p2p_valid_epoch_pruned_slash', () => {
         slashAmountMedium: slashingUnit * 2n,
         slashAmountLarge: slashingUnit * 3n,
         aztecTargetCommitteeSize: COMMITTEE_SIZE,
+        enableProposerPipelining: true,
+        inboxLag: 2,
       },
     });
 
@@ -161,6 +163,16 @@ describe('e2e_p2p_valid_epoch_pruned_slash', () => {
     t.logger.warn(`Removing initial node`);
     await t.removeInitialNode();
 
+    // Under proposer pipelining, slashing votes are EIP-712 signed against the pipelined target
+    // slot. The CheckpointProposalJob only delays the multicall to that slot when there is a
+    // checkpoint to broadcast; with minTxsPerBlock=1 and no further live txs, the job submits at
+    // wall-clock time and the EIP-712 signature is bound to the wrong slot, so VoteCast is never
+    // emitted. Drop minTxsPerBlock to 0 on the running validators so they always build (empty)
+    // checkpoints during the slashing voting window.
+    for (const node of nodes) {
+      await node.setConfig({ minTxsPerBlock: 0 });
+    }
+
     // Wait for epoch to be pruned and the offense to be detected
     const offenses = await awaitOffenseDetected({
       logger: t.logger,
diff --git a/yarn-project/end-to-end/src/e2e_p2p/validators_sentinel.test.ts b/yarn-project/end-to-end/src/e2e_p2p/validators_sentinel.test.ts
@@ -50,6 +50,8 @@ describe('e2e_p2p_validators_sentinel', () => {
         slashingRoundSizeInEpochs: 2,
         sentinelEnabled: true,
         slashInactivityPenalty: 0n, // Set to 0 to disable
+        enableProposerPipelining: true,
+        inboxLag: 2,
       },
     });
 
@@ -85,6 +87,20 @@ describe('e2e_p2p_validators_sentinel', () => {
   describe('with an offline validator', () => {
     let stats: ValidatorsStats;
     beforeAll(async () => {
+      // Ensure all running validator nodes see each other before starting block counting.
+      await t.waitForP2PMeshConnectivity(nodes);
+
+      // Wait until validator nodes have advanced past their first proposed slot so that the
+      // pipelining warm-up period (where some attestations may be missed) is behind us.
+      await t.monitor.run();
+      const warmupSlot = Number(t.monitor.l2SlotNumber) + 1;
+      t.logger.info(`Waiting for warmup slot ${warmupSlot} before establishing initial block`);
+      await retryUntil(
+        async () => (await t.monitor.run()).l2SlotNumber >= warmupSlot,
+        'warmup slot',
+        AZTEC_SLOT_DURATION * 3,
+      );
+
       const currentBlock = t.monitor.checkpointNumber;
       const blockCount = BLOCK_COUNT;
       const timeout = AZTEC_SLOT_DURATION * blockCount * 8;
