From 4fa67acd90f8460ecb526d741fe8cc15e3f99c64 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Thu, 7 May 2026 18:53:16 -0300 Subject: [PATCH 1/4] test(e2e): enable pipelining on p2p tests --- .../end-to-end/src/e2e_p2p/add_rollup.test.ts | 70 +++++++++++++++++-- ...asted_invalid_block_proposal_slash.test.ts | 2 + .../e2e_p2p/data_withholding_slash.test.ts | 7 ++ .../duplicate_attestation_slash.test.ts | 2 + .../e2e_p2p/duplicate_proposal_slash.test.ts | 36 +++++++--- .../fee_asset_price_oracle_gossip.test.ts | 20 +++++- .../src/e2e_p2p/gossip_network.test.ts | 2 + .../e2e_p2p/gossip_network_no_cheat.test.ts | 2 + .../src/e2e_p2p/inactivity_slash_test.ts | 2 + ...vity_slash_with_consecutive_epochs.test.ts | 2 +- ...tiple_validators_sentinel.parallel.test.ts | 15 ++++ .../e2e_p2p/preferred_gossip_network.test.ts | 3 + .../src/e2e_p2p/rediscovery.test.ts | 2 + .../end-to-end/src/e2e_p2p/reex.test.ts | 2 + yarn-project/end-to-end/src/e2e_p2p/shared.ts | 5 +- .../src/e2e_p2p/slash_veto_demo.test.ts | 2 + .../upgrade_governance_proposer.test.ts | 3 + .../e2e_p2p/valid_epoch_pruned_slash.test.ts | 12 ++++ .../src/e2e_p2p/validators_sentinel.test.ts | 16 +++++ 19 files changed, 187 insertions(+), 18 deletions(-) diff --git a/yarn-project/end-to-end/src/e2e_p2p/add_rollup.test.ts b/yarn-project/end-to-end/src/e2e_p2p/add_rollup.test.ts index 28a9a5d105ae..4ab5af89b5c1 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/add_rollup.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/add_rollup.test.ts @@ -5,7 +5,8 @@ import { EthAddress } from '@aztec/aztec.js/addresses'; import { waitForProven } from '@aztec/aztec.js/contracts'; import { generateClaimSecret } from '@aztec/aztec.js/ethereum'; import { Fr } from '@aztec/aztec.js/fields'; -import { RollupCheatCodes } from '@aztec/aztec/testing'; +import { waitForL1ToL2MessageReady } from '@aztec/aztec.js/messaging'; +import { EpochTestSettler, RollupCheatCodes } from '@aztec/aztec/testing'; import { FeeAssetHandlerContract, RegistryContract, RollupContract } from '@aztec/ethereum/contracts'; import { deployRollupForUpgrade } from '@aztec/ethereum/deploy-aztec-l1-contracts'; import { deployL1Contract } from '@aztec/ethereum/deploy-l1-contract'; @@ -13,6 +14,7 @@ import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses' import { L1TxUtils, createL1TxUtils } from '@aztec/ethereum/l1-tx-utils'; import type { ExtendedViemWalletClient } from '@aztec/ethereum/types'; import { CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; +import { retryUntil } from '@aztec/foundation/retry'; import { sleep } from '@aztec/foundation/sleep'; import { GovernanceAbi, @@ -42,7 +44,6 @@ import { shouldCollectMetrics } from '../fixtures/fixtures.js'; import { sendL1ToL2Message } from '../fixtures/l1_to_l2_messaging.js'; import { ATTESTER_PRIVATE_KEYS_START_INDEX, createNodes, createProverNode } from '../fixtures/setup_p2p_test.js'; import { setupSharedBlobStorage } from '../fixtures/utils.js'; -import { waitForL1ToL2MessageSeen } from '../shared/wait_for_l1_to_l2_message.js'; import { TestWallet } from '../test-wallet/test_wallet.js'; import { P2PNetworkTest, SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES } from './p2p_network.js'; @@ -53,7 +54,7 @@ const BOOT_NODE_UDP_PORT = 4500; const DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'add-rollup-old-')); const DATA_DIR_NEW = fs.mkdtempSync(path.join(os.tmpdir(), 'add-rollup-new-')); -jest.setTimeout(1000 * 60 * 10); +jest.setTimeout(1000 * 60 * 20); /** * This test emulates the addition of a new rollup to the registry and tests that cross-chain messages work. @@ -67,6 +68,13 @@ describe('e2e_p2p_add_rollup', () => { let nodes: AztecNodeService[]; let proverAztecNode: AztecNodeService; let l1TxUtils: L1TxUtils; + // Cheat-code-driven epoch settlers stand in for real prover-node submission. Pipelining + // currently produces a `Root rollup public inputs mismatch` between the prover's recomputed + // checkpoint header hashes and the on-chain log (see pipeline-review.md), so the real prover + // never moves the proven tip and `waitForProven` would hang indefinitely. The settler advances + // the proven tip and writes the outbox out hash via cheat codes once each epoch is complete. + let oldRollupSettler: EpochTestSettler | undefined; + let newRollupSettler: EpochTestSettler | undefined; beforeAll(async () => { t = await P2PNetworkTest.create({ @@ -80,6 +88,14 @@ describe('e2e_p2p_add_rollup', () => { ...SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES, listenAddress: '127.0.0.1', governanceProposerRoundSize: 10, + enableProposerPipelining: true, + // Allow validators to build empty checkpoints under pipelining so the chain keeps + // advancing while we wait for L1->L2 messages to land in the next checkpoint's inbox tree. + minTxsPerBlock: 0, + // Pipelining starts cycle for checkpoint N+1 during slot N, but the inbox tree for + // checkpoint N is only sealed when checkpoint N is published. inboxLag: 2 sources + // L1->L2 messages from checkpoint N-1 (already sealed), avoiding L1ToL2MessagesNotReadyError. + inboxLag: 2, }, startProverNode: false, // Start one later using p2p. }); @@ -94,6 +110,8 @@ describe('e2e_p2p_add_rollup', () => { }); afterAll(async () => { + await oldRollupSettler?.stop(); + await newRollupSettler?.stop(); await tryStop(proverAztecNode); await t.stopNodes(nodes); await t.teardown(); @@ -260,6 +278,18 @@ describe('e2e_p2p_add_rollup', () => { shouldCollectMetrics(), )); + // Cheat-code-driven epoch settler so the proven tip and outbox advance without depending on + // the real prover, which currently fails to publish under proposer pipelining due to a + // `Root rollup public inputs mismatch`. See add_rollup.pipeline-review.md. + oldRollupSettler = new EpochTestSettler( + t.ctx.cheatCodes.eth, + t.ctx.deployL1ContractsValues.l1ContractAddresses.rollupAddress, + nodes[0].getBlockSource(), + t.logger.createChild('epoch-settler-old'), + { pollingIntervalMs: 200 }, + ); + await oldRollupSettler.start(); + await sleep(4000); t.logger.info('Start progressing time to cast votes'); @@ -307,7 +337,10 @@ describe('e2e_p2p_add_rollup', () => { }); const makeMessageConsumable = async (msgHash: Fr) => { - await waitForL1ToL2MessageSeen(node, msgHash, { timeoutSeconds: 10 }); + // Wait until the message is ready to be consumed (the rollup has reached the message's checkpoint). + // Using waitForL1ToL2MessageReady rather than isL1ToL2MessageSynced because with `inboxLag > 0` + // a synced message is not yet present in the latest checkpoint's inbox tree. + await waitForL1ToL2MessageReady(node, msgHash, { timeoutSeconds: 120 }); const { receipt } = await testContract.methods .create_l2_to_l1_message_arbitrary_recipient_private(contentOutFromRollup, ethRecipient) @@ -572,12 +605,41 @@ describe('e2e_p2p_add_rollup', () => { shouldCollectMetrics(), )); + // Stop the old-rollup settler and spin up one for the new rollup. Same rationale as above: + // the real prover does not publish proofs under pipelining, so we need cheat-code settlement + // for the bridging step's `waitForProven` to make progress. + await oldRollupSettler?.stop(); + oldRollupSettler = undefined; + newRollupSettler = new EpochTestSettler( + t.ctx.cheatCodes.eth, + EthAddress.fromString(newRollup.address), + nodes[0].getBlockSource(), + t.logger.createChild('epoch-settler-new'), + { pollingIntervalMs: 200 }, + ); + await newRollupSettler.start(); + // wait a bit for peers to discover each other await sleep(4000); // The new rollup should have no checkpoints expect(await newRollup.getCheckpointNumber()).toBe(CheckpointNumber(0)); + // Wait for the new rollup to publish its first checkpoint AND for `nodes[0]` to have synced + // it locally, before the second bridging step. The bridge wallet uses + // `syncChainTip: 'checkpointed'`, which falls back to the genesis block when no checkpoint + // exists. After warping ~500 epochs forward, txs anchored at genesis would expire before + // being included. We poll the node's local view (not just the L1 rollup contract) so the PXE + // and the assertion observe the same chain state. + t.logger.info(`Waiting for new rollup to publish its first checkpoint`); + await retryUntil( + async () => Number(await nodes[0].getCheckpointNumber('checkpointed')) > 0, + 'newRollup first checkpoint synced by node', + 300, + 2, + ); + t.logger.info(`New rollup published its first checkpoint`); + // Bridge into and out of the new rollup to ensure that it works. await bridging( nodes[0], diff --git a/yarn-project/end-to-end/src/e2e_p2p/broadcasted_invalid_block_proposal_slash.test.ts b/yarn-project/end-to-end/src/e2e_p2p/broadcasted_invalid_block_proposal_slash.test.ts index 8dc7dd33c2b9..02dd223b3b86 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/broadcasted_invalid_block_proposal_slash.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/broadcasted_invalid_block_proposal_slash.test.ts @@ -62,6 +62,8 @@ describe('e2e_p2p_broadcasted_invalid_block_proposal_slash', () => { ethereumSlotDuration: ETHEREUM_SLOT_DURATION, aztecSlotDuration: AZTEC_SLOT_DURATION, aztecTargetCommitteeSize: COMMITTEE_SIZE, + enableProposerPipelining: true, + inboxLag: 2, aztecProofSubmissionEpochs: 1024, // effectively do not reorg slashInactivityConsecutiveEpochThreshold: 32, // effectively do not slash for inactivity minTxsPerBlock: 0, // always be building diff --git a/yarn-project/end-to-end/src/e2e_p2p/data_withholding_slash.test.ts b/yarn-project/end-to-end/src/e2e_p2p/data_withholding_slash.test.ts index 84171f697fb2..bd63a95eda13 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/data_withholding_slash.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/data_withholding_slash.test.ts @@ -73,6 +73,8 @@ describe('e2e_p2p_data_withholding_slash', () => { slashAmountLarge: slashingUnit * 3n, slashSelfAllowed: true, minTxsPerBlock: 0, + enableProposerPipelining: true, + inboxLag: 2, }, }); @@ -165,6 +167,11 @@ describe('e2e_p2p_data_withholding_slash', () => { // Re-create the nodes. // ASSUMING they sync in the middle of the epoch, they will "see" the reorg, and try to slash. + // Reset minTxsPerBlock to 0 so re-created validators build empty checkpoints. Under proposer + // pipelining, the vote-offenses signature is bound to the target slot and the multicall is only + // delayed to the target slot start when a checkpoint is being proposed; without a proposal, + // votes would mine in the current wall-clock slot, causing the EIP-712 signature verification to fail. + t.ctx.aztecNodeConfig.minTxsPerBlock = 0; t.logger.warn('Re-creating nodes'); nodes = await createNodes( t.ctx.aztecNodeConfig, diff --git a/yarn-project/end-to-end/src/e2e_p2p/duplicate_attestation_slash.test.ts b/yarn-project/end-to-end/src/e2e_p2p/duplicate_attestation_slash.test.ts index 0f6cd6c0f9c3..ce4a8f706999 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/duplicate_attestation_slash.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/duplicate_attestation_slash.test.ts @@ -89,6 +89,8 @@ describe('e2e_p2p_duplicate_attestation_slash', () => { slashDuplicateProposalPenalty: slashingUnit, slashDuplicateAttestationPenalty: slashingUnit, slashingOffsetInRounds: 1, + enableProposerPipelining: true, + inboxLag: 2, }, }); diff --git a/yarn-project/end-to-end/src/e2e_p2p/duplicate_proposal_slash.test.ts b/yarn-project/end-to-end/src/e2e_p2p/duplicate_proposal_slash.test.ts index 51b8e4b4083f..82b6fd7fdc23 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/duplicate_proposal_slash.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/duplicate_proposal_slash.test.ts @@ -2,6 +2,7 @@ import type { AztecNodeService } from '@aztec/aztec-node'; import type { TestAztecNodeService } from '@aztec/aztec-node/test'; import { EthAddress } from '@aztec/aztec.js/addresses'; import { EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; +import { retryUntil } from '@aztec/foundation/retry'; import { bufferToHex } from '@aztec/foundation/string'; import { OffenseType } from '@aztec/slasher'; import { TopicType } from '@aztec/stdlib/p2p'; @@ -79,6 +80,8 @@ describe('e2e_p2p_duplicate_proposal_slash', () => { blockDurationMs: BLOCK_DURATION * 1000, slashDuplicateProposalPenalty: slashingUnit, slashingOffsetInRounds: 1, + enableProposerPipelining: true, + inboxLag: 2, }, }); @@ -224,24 +227,39 @@ describe('e2e_p2p_duplicate_proposal_slash', () => { t.logger.warn(`Advancing to target epoch ${targetEpoch}`); await t.ctx.cheatCodes.rollup.advanceToEpoch(targetEpoch); - // Wait for offense to be detected - // The honest nodes should detect the duplicate proposal from the malicious validator + // Wait for offense to be detected. Under proposer pipelining, checkpoint proposals are broadcast + // at the slot boundary while the receivers' wall clocks may have already advanced past the build + // slot — when that happens, honest nodes reject the gossip with "invalid slot number" before + // duplicate detection runs, so DUPLICATE_PROPOSAL is only observed by whichever node managed to + // process both proposals while still in the build slot (often the other malicious node, since + // they receive each other's broadcasts immediately). We therefore collect offenses from every + // node in the network and assert that at least one of them recorded the duplicate proposal. t.logger.warn('Waiting for duplicate proposal offense to be detected...'); - const offenses = await awaitOffenseDetected({ + await awaitOffenseDetected({ epochDuration: t.ctx.aztecNodeConfig.aztecEpochDuration, logger: t.logger, - nodeAdmin: honestNode1, // Use honest node to check for offenses + nodeAdmin: honestNode1, slashingRoundSize, waitUntilOffenseCount: 1, timeoutSeconds: AZTEC_SLOT_DURATION * 16, }); - t.logger.warn(`Collected offenses`, { offenses }); + // Poll every node for DUPLICATE_PROPOSAL offenses, retrying briefly so any node that detected + // the duplicate after the initial offense was collected has time to flush it through the + // slasher's offenses-collector. + const proposalOffenses = await retryUntil( + async () => { + const allOffenses = (await Promise.all(nodes.map(n => n.getSlashOffenses('all')))).flat(); + const filtered = allOffenses.filter(o => o.offenseType === OffenseType.DUPLICATE_PROPOSAL); + if (filtered.length > 0) { + return filtered; + } + }, + 'duplicate proposal offense', + AZTEC_SLOT_DURATION * 4, + ); - // Filter to only DUPLICATE_PROPOSAL offenses. The two malicious nodes sharing the same key - // will also each self-attest to their own (different) checkpoint proposals, which causes honest - // nodes to detect a DUPLICATE_ATTESTATION as well. We only care about proposals here. - const proposalOffenses = offenses.filter(o => o.offenseType === OffenseType.DUPLICATE_PROPOSAL); + t.logger.warn(`Collected duplicate proposal offenses`, { proposalOffenses }); expect(proposalOffenses.length).toBeGreaterThan(0); for (const offense of proposalOffenses) { expect(offense.validator.toString()).toEqual(maliciousValidatorAddress.toString()); diff --git a/yarn-project/end-to-end/src/e2e_p2p/fee_asset_price_oracle_gossip.test.ts b/yarn-project/end-to-end/src/e2e_p2p/fee_asset_price_oracle_gossip.test.ts index 04b436680edb..aeeb5d3dbdd6 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/fee_asset_price_oracle_gossip.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/fee_asset_price_oracle_gossip.test.ts @@ -63,6 +63,8 @@ describe('e2e_p2p_network', () => { slashingRoundSizeInEpochs: 2, slashingQuorum: 5, listenAddress: '127.0.0.1', + enableProposerPipelining: true, + inboxLag: 2, }, }); @@ -205,11 +207,23 @@ describe('e2e_p2p_network', () => { expect(validatorAddresses).toContain(signer); } + // Allow a tolerance up to the per-checkpoint modifier cap (±100 bps): under proposer pipelining, + // the modifier for checkpoint N is computed in slot N-1 from the price visible at prep time, but + // the on-chain new price is the parent fee header (one checkpoint ahead) modified by that value. + // This stale-read effect causes the price to oscillate around the oracle target with up to ~1% + // amplitude rather than land exactly on it. The test still verifies that the price tracks the + // oracle within the cap. + const TOLERANCE_BPS = 100n; + const absDiffBps = (a: bigint, b: bigint) => { + const d = diffInBps(a, b); + return d < 0n ? -d : d; + }; + await retryUntil( async () => { const currentPrice = await rollup.getEthPerFeeAsset(); t.logger.info(`Current on-chain price: ${currentPrice}, waiting for: ${targetOraclePrice}`); - return diffInBps(currentPrice, targetOraclePrice) == 0n; + return absDiffBps(currentPrice, targetOraclePrice) <= TOLERANCE_BPS; }, 'price convergence toward oracle', 120, // timeout in seconds @@ -225,7 +239,7 @@ describe('e2e_p2p_network', () => { async () => { const currentPrice = await rollup.getEthPerFeeAsset(); t.logger.info(`Current on-chain price: ${currentPrice}, waiting for: ${targetOraclePrice2}`); - return diffInBps(currentPrice, targetOraclePrice2) == 0n; + return absDiffBps(currentPrice, targetOraclePrice2) <= TOLERANCE_BPS; }, 'price convergence toward oracle', 120, // timeout in seconds @@ -237,6 +251,6 @@ describe('e2e_p2p_network', () => { // Verify the price moved toward the oracle price expect(finalPrice).toBeGreaterThan(initialOnChainPrice); - expect(diffInBps(finalPrice, targetOraclePrice2)).toBe(0n); + expect(absDiffBps(finalPrice, targetOraclePrice2)).toBeLessThanOrEqual(TOLERANCE_BPS); }); }); diff --git a/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts b/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts index 2ff865324b5f..aad5e52e0631 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/gossip_network.test.ts @@ -69,6 +69,8 @@ describe('e2e_p2p_network', () => { slashingRoundSizeInEpochs: 2, slashingQuorum: 5, listenAddress: '127.0.0.1', + enableProposerPipelining: true, + inboxLag: 2, }, }); diff --git a/yarn-project/end-to-end/src/e2e_p2p/gossip_network_no_cheat.test.ts b/yarn-project/end-to-end/src/e2e_p2p/gossip_network_no_cheat.test.ts index 242a2fef3e16..9fa76164243d 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/gossip_network_no_cheat.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/gossip_network_no_cheat.test.ts @@ -68,6 +68,8 @@ describe('e2e_p2p_network', () => { // Without this, no blocks are built until txs arrive, and a failed checkpoint during tx // submission causes block pruning that invalidates tx references. minTxsPerBlock: 0, + enableProposerPipelining: true, + inboxLag: 2, }, }); diff --git a/yarn-project/end-to-end/src/e2e_p2p/inactivity_slash_test.ts b/yarn-project/end-to-end/src/e2e_p2p/inactivity_slash_test.ts index efcc6d4833a5..20fff9244f6d 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/inactivity_slash_test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/inactivity_slash_test.ts @@ -58,6 +58,8 @@ export class P2PInactivityTest { basePort: BOOT_NODE_UDP_PORT, startProverNode: true, initialConfig: { + enableProposerPipelining: true, + inboxLag: 2, anvilSlotsInAnEpoch: 4, proverNodeConfig: { proverNodeEpochProvingDelayMs: AZTEC_SLOT_DURATION * 1000 }, aztecTargetCommitteeSize: COMMITTEE_SIZE, diff --git a/yarn-project/end-to-end/src/e2e_p2p/inactivity_slash_with_consecutive_epochs.test.ts b/yarn-project/end-to-end/src/e2e_p2p/inactivity_slash_with_consecutive_epochs.test.ts index 93513f4b879f..78af77bc07d6 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/inactivity_slash_with_consecutive_epochs.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/inactivity_slash_with_consecutive_epochs.test.ts @@ -55,7 +55,7 @@ describe('e2e_p2p_inactivity_slash_with_consecutive_epochs', () => { return offenses.length > 0 ? offenses : undefined; }, 'slash offenses', - slashInactivityConsecutiveEpochThreshold * aztecEpochDuration * aztecSlotDuration * 2, + slashInactivityConsecutiveEpochThreshold * aztecEpochDuration * aztecSlotDuration * 4, ); expect(unique(offenses.map(o => o.validator.toString()))).toEqual([offlineValidator.toString()]); expect(unique(offenses.map(o => o.offenseType))).toEqual([OffenseType.INACTIVITY]); diff --git a/yarn-project/end-to-end/src/e2e_p2p/multiple_validators_sentinel.parallel.test.ts b/yarn-project/end-to-end/src/e2e_p2p/multiple_validators_sentinel.parallel.test.ts index e90f0a918d46..b270212db706 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/multiple_validators_sentinel.parallel.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/multiple_validators_sentinel.parallel.test.ts @@ -58,6 +58,8 @@ describe('e2e_p2p_multiple_validators_sentinel', () => { slashingRoundSizeInEpochs: 2, sentinelEnabled: true, slashInactivityPenalty: 0n, // Set to 0 to disable + enableProposerPipelining: true, + inboxLag: 2, }, }); @@ -103,7 +105,20 @@ describe('e2e_p2p_multiple_validators_sentinel', () => { }); it('collects attestations for all validators on a node', async () => { + // Ensure all nodes see each other, especially the sentinel, before starting slot counting + await t.waitForP2PMeshConnectivity([...nodes, sentinel]); + + // Wait until validator nodes have advanced past their first proposed slot so that the + // pipelining warm-up period (where some attestations may be missed) is behind us. await t.monitor.run(); + const warmupSlot = Number(t.monitor.l2SlotNumber) + 1; + t.logger.info(`Waiting for warmup slot ${warmupSlot} before establishing initial slot`); + await retryUntil( + async () => (await t.monitor.run()).l2SlotNumber >= warmupSlot, + 'warmup slot', + AZTEC_SLOT_DURATION * 3, + ); + const { checkpointNumber: initialBlock, l2SlotNumber: initialSlot } = t.monitor; const timeout = AZTEC_SLOT_DURATION * SLOT_COUNT * 4; diff --git a/yarn-project/end-to-end/src/e2e_p2p/preferred_gossip_network.test.ts b/yarn-project/end-to-end/src/e2e_p2p/preferred_gossip_network.test.ts index 03bd23ebb57e..409d8580e684 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/preferred_gossip_network.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/preferred_gossip_network.test.ts @@ -142,6 +142,9 @@ describe('e2e_p2p_preferred_network', () => { p2pDisableStatusHandshake: false, // Just for testing be aggressive here, don't allow any auth handshake failures p2pMaxFailedAuthAttemptsAllowed: 0, + minTxsPerBlock: 0, + enableProposerPipelining: true, + inboxLag: 2, }, }); diff --git a/yarn-project/end-to-end/src/e2e_p2p/rediscovery.test.ts b/yarn-project/end-to-end/src/e2e_p2p/rediscovery.test.ts index 527750cc83e8..e4ddef8157f5 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/rediscovery.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/rediscovery.test.ts @@ -35,6 +35,8 @@ describe('e2e_p2p_rediscovery', () => { ...SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES, aztecSlotDuration: 24, listenAddress: '127.0.0.1', + enableProposerPipelining: true, + inboxLag: 2, }, }); await t.setup(); diff --git a/yarn-project/end-to-end/src/e2e_p2p/reex.test.ts b/yarn-project/end-to-end/src/e2e_p2p/reex.test.ts index 47f81edb57d4..aa1369606450 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/reex.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/reex.test.ts @@ -47,6 +47,8 @@ describe('e2e_p2p_reex', () => { txTimeoutMs: 30_000, listenAddress: '127.0.0.1', aztecProofSubmissionEpochs: 1024, // effectively do not reorg + enableProposerPipelining: true, + inboxLag: 2, }, }); diff --git a/yarn-project/end-to-end/src/e2e_p2p/shared.ts b/yarn-project/end-to-end/src/e2e_p2p/shared.ts index 2b454d59845e..4c46ec6e5c57 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/shared.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/shared.ts @@ -281,7 +281,10 @@ export async function awaitCommitteeKicked({ expect(attesterInfo.status).toEqual(1); // Validating } - const timeout = slashingRoundSize * 2 * aztecSlotDuration + 30; + // Allow up to four round-lengths so that under proposer pipelining, where individual rounds + // sometimes fail to gather quorum because parts of the committee miss votes due to chain-state + // races, we still see a later round execute the slash. + const timeout = slashingRoundSize * 4 * aztecSlotDuration + 30; logger.info(`Waiting for slash to be executed (timeout ${timeout}s)`); await awaitProposalExecution(slashingProposer, timeout, logger); diff --git a/yarn-project/end-to-end/src/e2e_p2p/slash_veto_demo.test.ts b/yarn-project/end-to-end/src/e2e_p2p/slash_veto_demo.test.ts index f6ffbe12c371..c7fa39824224 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/slash_veto_demo.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/slash_veto_demo.test.ts @@ -76,6 +76,8 @@ describe('veto slash', () => { aztecProofSubmissionEpochs: 1024, // effectively do not reorg listenAddress: '127.0.0.1', minTxsPerBlock: 0, + enableProposerPipelining: true, + inboxLag: 2, aztecEpochDuration: EPOCH_DURATION, sentinelEnabled: true, slashSelfAllowed: true, diff --git a/yarn-project/end-to-end/src/e2e_p2p/upgrade_governance_proposer.test.ts b/yarn-project/end-to-end/src/e2e_p2p/upgrade_governance_proposer.test.ts index f4b5fa4d3acb..205899701ace 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/upgrade_governance_proposer.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/upgrade_governance_proposer.test.ts @@ -54,6 +54,9 @@ describe('e2e_p2p_governance_proposer', () => { governanceProposerRoundSize: 10, activationThreshold: 10n ** 22n, ejectionThreshold: 5n ** 22n, + enableProposerPipelining: true, + inboxLag: 2, + minTxsPerBlock: 0, }, }); diff --git a/yarn-project/end-to-end/src/e2e_p2p/valid_epoch_pruned_slash.test.ts b/yarn-project/end-to-end/src/e2e_p2p/valid_epoch_pruned_slash.test.ts index e0245c2d6e2e..83b1203c6bca 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/valid_epoch_pruned_slash.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/valid_epoch_pruned_slash.test.ts @@ -69,6 +69,8 @@ describe('e2e_p2p_valid_epoch_pruned_slash', () => { slashAmountMedium: slashingUnit * 2n, slashAmountLarge: slashingUnit * 3n, aztecTargetCommitteeSize: COMMITTEE_SIZE, + enableProposerPipelining: true, + inboxLag: 2, }, }); @@ -161,6 +163,16 @@ describe('e2e_p2p_valid_epoch_pruned_slash', () => { t.logger.warn(`Removing initial node`); await t.removeInitialNode(); + // Under proposer pipelining, slashing votes are EIP-712 signed against the pipelined target + // slot. The CheckpointProposalJob only delays the multicall to that slot when there is a + // checkpoint to broadcast; with minTxsPerBlock=1 and no further live txs, the job submits at + // wall-clock time and the EIP-712 signature is bound to the wrong slot, so VoteCast is never + // emitted. Drop minTxsPerBlock to 0 on the running validators so they always build (empty) + // checkpoints during the slashing voting window. + for (const node of nodes) { + await node.setConfig({ minTxsPerBlock: 0 }); + } + // Wait for epoch to be pruned and the offense to be detected const offenses = await awaitOffenseDetected({ logger: t.logger, diff --git a/yarn-project/end-to-end/src/e2e_p2p/validators_sentinel.test.ts b/yarn-project/end-to-end/src/e2e_p2p/validators_sentinel.test.ts index 4d0b38d533dd..189404f772a9 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/validators_sentinel.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/validators_sentinel.test.ts @@ -50,6 +50,8 @@ describe('e2e_p2p_validators_sentinel', () => { slashingRoundSizeInEpochs: 2, sentinelEnabled: true, slashInactivityPenalty: 0n, // Set to 0 to disable + enableProposerPipelining: true, + inboxLag: 2, }, }); @@ -85,6 +87,20 @@ describe('e2e_p2p_validators_sentinel', () => { describe('with an offline validator', () => { let stats: ValidatorsStats; beforeAll(async () => { + // Ensure all running validator nodes see each other before starting block counting. + await t.waitForP2PMeshConnectivity(nodes); + + // Wait until validator nodes have advanced past their first proposed slot so that the + // pipelining warm-up period (where some attestations may be missed) is behind us. + await t.monitor.run(); + const warmupSlot = Number(t.monitor.l2SlotNumber) + 1; + t.logger.info(`Waiting for warmup slot ${warmupSlot} before establishing initial block`); + await retryUntil( + async () => (await t.monitor.run()).l2SlotNumber >= warmupSlot, + 'warmup slot', + AZTEC_SLOT_DURATION * 3, + ); + const currentBlock = t.monitor.checkpointNumber; const blockCount = BLOCK_COUNT; const timeout = AZTEC_SLOT_DURATION * blockCount * 8; From 04915be12d892ee38c04ef1308768c268421764f Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Fri, 8 May 2026 16:35:15 -0300 Subject: [PATCH 2/4] fix(e2e): duplicate proposal slash p2p test --- .../src/e2e_p2p/duplicate_proposal_slash.test.ts | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/yarn-project/end-to-end/src/e2e_p2p/duplicate_proposal_slash.test.ts b/yarn-project/end-to-end/src/e2e_p2p/duplicate_proposal_slash.test.ts index 82b6fd7fdc23..697e2f539039 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/duplicate_proposal_slash.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/duplicate_proposal_slash.test.ts @@ -223,9 +223,13 @@ describe('e2e_p2p_duplicate_proposal_slash', () => { t.logger.warn('Starting all sequencers'); await Promise.all(nodes.map(n => n.getSequencer()!.start())); - // Now warp to the target epoch — sequencers are already running - t.logger.warn(`Advancing to target epoch ${targetEpoch}`); - await t.ctx.cheatCodes.rollup.advanceToEpoch(targetEpoch); + // Now warp to one slot before the target epoch — sequencers are already running. + // Under proposer pipelining, the malicious proposers begin building for the first + // slot of the target epoch one slot earlier; warping to the start of the epoch + // would force both AVM-heavy duplicate proposals to serialize past the slot + // boundary, after which honest receivers reject them as late. + t.logger.warn(`Advancing to one slot before target epoch ${targetEpoch}`); + await t.ctx.cheatCodes.rollup.advanceToEpoch(targetEpoch, { offset: -AZTEC_SLOT_DURATION }); // Wait for offense to be detected. Under proposer pipelining, checkpoint proposals are broadcast // at the slot boundary while the receivers' wall clocks may have already advanced past the build From a0d0f9ec4e7036148eed84f62f017bee8f8545f1 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Fri, 8 May 2026 16:42:42 -0300 Subject: [PATCH 3/4] revert changes to add_rollup and fee_asset_price_oracle_gossip --- .../end-to-end/src/e2e_p2p/add_rollup.test.ts | 70 ++----------------- .../fee_asset_price_oracle_gossip.test.ts | 20 +----- 2 files changed, 7 insertions(+), 83 deletions(-) diff --git a/yarn-project/end-to-end/src/e2e_p2p/add_rollup.test.ts b/yarn-project/end-to-end/src/e2e_p2p/add_rollup.test.ts index 4ab5af89b5c1..28a9a5d105ae 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/add_rollup.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/add_rollup.test.ts @@ -5,8 +5,7 @@ import { EthAddress } from '@aztec/aztec.js/addresses'; import { waitForProven } from '@aztec/aztec.js/contracts'; import { generateClaimSecret } from '@aztec/aztec.js/ethereum'; import { Fr } from '@aztec/aztec.js/fields'; -import { waitForL1ToL2MessageReady } from '@aztec/aztec.js/messaging'; -import { EpochTestSettler, RollupCheatCodes } from '@aztec/aztec/testing'; +import { RollupCheatCodes } from '@aztec/aztec/testing'; import { FeeAssetHandlerContract, RegistryContract, RollupContract } from '@aztec/ethereum/contracts'; import { deployRollupForUpgrade } from '@aztec/ethereum/deploy-aztec-l1-contracts'; import { deployL1Contract } from '@aztec/ethereum/deploy-l1-contract'; @@ -14,7 +13,6 @@ import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses' import { L1TxUtils, createL1TxUtils } from '@aztec/ethereum/l1-tx-utils'; import type { ExtendedViemWalletClient } from '@aztec/ethereum/types'; import { CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; -import { retryUntil } from '@aztec/foundation/retry'; import { sleep } from '@aztec/foundation/sleep'; import { GovernanceAbi, @@ -44,6 +42,7 @@ import { shouldCollectMetrics } from '../fixtures/fixtures.js'; import { sendL1ToL2Message } from '../fixtures/l1_to_l2_messaging.js'; import { ATTESTER_PRIVATE_KEYS_START_INDEX, createNodes, createProverNode } from '../fixtures/setup_p2p_test.js'; import { setupSharedBlobStorage } from '../fixtures/utils.js'; +import { waitForL1ToL2MessageSeen } from '../shared/wait_for_l1_to_l2_message.js'; import { TestWallet } from '../test-wallet/test_wallet.js'; import { P2PNetworkTest, SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES } from './p2p_network.js'; @@ -54,7 +53,7 @@ const BOOT_NODE_UDP_PORT = 4500; const DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'add-rollup-old-')); const DATA_DIR_NEW = fs.mkdtempSync(path.join(os.tmpdir(), 'add-rollup-new-')); -jest.setTimeout(1000 * 60 * 20); +jest.setTimeout(1000 * 60 * 10); /** * This test emulates the addition of a new rollup to the registry and tests that cross-chain messages work. @@ -68,13 +67,6 @@ describe('e2e_p2p_add_rollup', () => { let nodes: AztecNodeService[]; let proverAztecNode: AztecNodeService; let l1TxUtils: L1TxUtils; - // Cheat-code-driven epoch settlers stand in for real prover-node submission. Pipelining - // currently produces a `Root rollup public inputs mismatch` between the prover's recomputed - // checkpoint header hashes and the on-chain log (see pipeline-review.md), so the real prover - // never moves the proven tip and `waitForProven` would hang indefinitely. The settler advances - // the proven tip and writes the outbox out hash via cheat codes once each epoch is complete. - let oldRollupSettler: EpochTestSettler | undefined; - let newRollupSettler: EpochTestSettler | undefined; beforeAll(async () => { t = await P2PNetworkTest.create({ @@ -88,14 +80,6 @@ describe('e2e_p2p_add_rollup', () => { ...SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES, listenAddress: '127.0.0.1', governanceProposerRoundSize: 10, - enableProposerPipelining: true, - // Allow validators to build empty checkpoints under pipelining so the chain keeps - // advancing while we wait for L1->L2 messages to land in the next checkpoint's inbox tree. - minTxsPerBlock: 0, - // Pipelining starts cycle for checkpoint N+1 during slot N, but the inbox tree for - // checkpoint N is only sealed when checkpoint N is published. inboxLag: 2 sources - // L1->L2 messages from checkpoint N-1 (already sealed), avoiding L1ToL2MessagesNotReadyError. - inboxLag: 2, }, startProverNode: false, // Start one later using p2p. }); @@ -110,8 +94,6 @@ describe('e2e_p2p_add_rollup', () => { }); afterAll(async () => { - await oldRollupSettler?.stop(); - await newRollupSettler?.stop(); await tryStop(proverAztecNode); await t.stopNodes(nodes); await t.teardown(); @@ -278,18 +260,6 @@ describe('e2e_p2p_add_rollup', () => { shouldCollectMetrics(), )); - // Cheat-code-driven epoch settler so the proven tip and outbox advance without depending on - // the real prover, which currently fails to publish under proposer pipelining due to a - // `Root rollup public inputs mismatch`. See add_rollup.pipeline-review.md. - oldRollupSettler = new EpochTestSettler( - t.ctx.cheatCodes.eth, - t.ctx.deployL1ContractsValues.l1ContractAddresses.rollupAddress, - nodes[0].getBlockSource(), - t.logger.createChild('epoch-settler-old'), - { pollingIntervalMs: 200 }, - ); - await oldRollupSettler.start(); - await sleep(4000); t.logger.info('Start progressing time to cast votes'); @@ -337,10 +307,7 @@ describe('e2e_p2p_add_rollup', () => { }); const makeMessageConsumable = async (msgHash: Fr) => { - // Wait until the message is ready to be consumed (the rollup has reached the message's checkpoint). - // Using waitForL1ToL2MessageReady rather than isL1ToL2MessageSynced because with `inboxLag > 0` - // a synced message is not yet present in the latest checkpoint's inbox tree. - await waitForL1ToL2MessageReady(node, msgHash, { timeoutSeconds: 120 }); + await waitForL1ToL2MessageSeen(node, msgHash, { timeoutSeconds: 10 }); const { receipt } = await testContract.methods .create_l2_to_l1_message_arbitrary_recipient_private(contentOutFromRollup, ethRecipient) @@ -605,41 +572,12 @@ describe('e2e_p2p_add_rollup', () => { shouldCollectMetrics(), )); - // Stop the old-rollup settler and spin up one for the new rollup. Same rationale as above: - // the real prover does not publish proofs under pipelining, so we need cheat-code settlement - // for the bridging step's `waitForProven` to make progress. - await oldRollupSettler?.stop(); - oldRollupSettler = undefined; - newRollupSettler = new EpochTestSettler( - t.ctx.cheatCodes.eth, - EthAddress.fromString(newRollup.address), - nodes[0].getBlockSource(), - t.logger.createChild('epoch-settler-new'), - { pollingIntervalMs: 200 }, - ); - await newRollupSettler.start(); - // wait a bit for peers to discover each other await sleep(4000); // The new rollup should have no checkpoints expect(await newRollup.getCheckpointNumber()).toBe(CheckpointNumber(0)); - // Wait for the new rollup to publish its first checkpoint AND for `nodes[0]` to have synced - // it locally, before the second bridging step. The bridge wallet uses - // `syncChainTip: 'checkpointed'`, which falls back to the genesis block when no checkpoint - // exists. After warping ~500 epochs forward, txs anchored at genesis would expire before - // being included. We poll the node's local view (not just the L1 rollup contract) so the PXE - // and the assertion observe the same chain state. - t.logger.info(`Waiting for new rollup to publish its first checkpoint`); - await retryUntil( - async () => Number(await nodes[0].getCheckpointNumber('checkpointed')) > 0, - 'newRollup first checkpoint synced by node', - 300, - 2, - ); - t.logger.info(`New rollup published its first checkpoint`); - // Bridge into and out of the new rollup to ensure that it works. await bridging( nodes[0], diff --git a/yarn-project/end-to-end/src/e2e_p2p/fee_asset_price_oracle_gossip.test.ts b/yarn-project/end-to-end/src/e2e_p2p/fee_asset_price_oracle_gossip.test.ts index aeeb5d3dbdd6..04b436680edb 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/fee_asset_price_oracle_gossip.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/fee_asset_price_oracle_gossip.test.ts @@ -63,8 +63,6 @@ describe('e2e_p2p_network', () => { slashingRoundSizeInEpochs: 2, slashingQuorum: 5, listenAddress: '127.0.0.1', - enableProposerPipelining: true, - inboxLag: 2, }, }); @@ -207,23 +205,11 @@ describe('e2e_p2p_network', () => { expect(validatorAddresses).toContain(signer); } - // Allow a tolerance up to the per-checkpoint modifier cap (±100 bps): under proposer pipelining, - // the modifier for checkpoint N is computed in slot N-1 from the price visible at prep time, but - // the on-chain new price is the parent fee header (one checkpoint ahead) modified by that value. - // This stale-read effect causes the price to oscillate around the oracle target with up to ~1% - // amplitude rather than land exactly on it. The test still verifies that the price tracks the - // oracle within the cap. - const TOLERANCE_BPS = 100n; - const absDiffBps = (a: bigint, b: bigint) => { - const d = diffInBps(a, b); - return d < 0n ? -d : d; - }; - await retryUntil( async () => { const currentPrice = await rollup.getEthPerFeeAsset(); t.logger.info(`Current on-chain price: ${currentPrice}, waiting for: ${targetOraclePrice}`); - return absDiffBps(currentPrice, targetOraclePrice) <= TOLERANCE_BPS; + return diffInBps(currentPrice, targetOraclePrice) == 0n; }, 'price convergence toward oracle', 120, // timeout in seconds @@ -239,7 +225,7 @@ describe('e2e_p2p_network', () => { async () => { const currentPrice = await rollup.getEthPerFeeAsset(); t.logger.info(`Current on-chain price: ${currentPrice}, waiting for: ${targetOraclePrice2}`); - return absDiffBps(currentPrice, targetOraclePrice2) <= TOLERANCE_BPS; + return diffInBps(currentPrice, targetOraclePrice2) == 0n; }, 'price convergence toward oracle', 120, // timeout in seconds @@ -251,6 +237,6 @@ describe('e2e_p2p_network', () => { // Verify the price moved toward the oracle price expect(finalPrice).toBeGreaterThan(initialOnChainPrice); - expect(absDiffBps(finalPrice, targetOraclePrice2)).toBeLessThanOrEqual(TOLERANCE_BPS); + expect(diffInBps(finalPrice, targetOraclePrice2)).toBe(0n); }); }); From 18c2c550ecca83d1f696c52b4a408f071eae5509 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Fri, 8 May 2026 16:43:01 -0300 Subject: [PATCH 4/4] remove unneeded fix to epoch prune slash --- .../src/e2e_p2p/valid_epoch_pruned_slash.test.ts | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/yarn-project/end-to-end/src/e2e_p2p/valid_epoch_pruned_slash.test.ts b/yarn-project/end-to-end/src/e2e_p2p/valid_epoch_pruned_slash.test.ts index 83b1203c6bca..c9129bf2f3d6 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/valid_epoch_pruned_slash.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/valid_epoch_pruned_slash.test.ts @@ -163,16 +163,6 @@ describe('e2e_p2p_valid_epoch_pruned_slash', () => { t.logger.warn(`Removing initial node`); await t.removeInitialNode(); - // Under proposer pipelining, slashing votes are EIP-712 signed against the pipelined target - // slot. The CheckpointProposalJob only delays the multicall to that slot when there is a - // checkpoint to broadcast; with minTxsPerBlock=1 and no further live txs, the job submits at - // wall-clock time and the EIP-712 signature is bound to the wrong slot, so VoteCast is never - // emitted. Drop minTxsPerBlock to 0 on the running validators so they always build (empty) - // checkpoints during the slashing voting window. - for (const node of nodes) { - await node.setConfig({ minTxsPerBlock: 0 }); - } - // Wait for epoch to be pruned and the offense to be detected const offenses = await awaitOffenseDetected({ logger: t.logger,