diff --git a/.test_patterns.yml b/.test_patterns.yml index 3816c919c8e4..99686b9b4c6f 100644 --- a/.test_patterns.yml +++ b/.test_patterns.yml @@ -267,18 +267,23 @@ tests: - *charlie - *adam - - regex: "src/e2e_epochs/epochs_l1_reorgs.parallel.test.ts" + - regex: "src/single-node/l1-reorgs/blocks.parallel.test.ts" owners: - *phil - - regex: "epochs_mbps.pipeline.parallel.test.ts.*prunes uncheckpointed" + - regex: "src/single-node/l1-reorgs/messages.parallel.test.ts" + owners: + - *phil + + - regex: "src/multi-node/recovery/pipeline_prune.test.ts.*prunes uncheckpointed" owners: - *sean - # Asserts all late txs land in the checkpoint's last block, but the proposer seals the last block as - # soon as one tx is available and snapshots the mempool, so late txs still propagating over gossip - # spill into the next checkpoint. Flaky until the sequencer waits for the last block's tx window. - - regex: "src/e2e_epochs/epochs_mbps_redistribution.test.ts" + # Asserts a late tx burst is redistributed across the checkpoint's last two blocks, but the proposer + # seals the one-before-last block as soon as one tx is available and snapshots the mempool, so late + # txs still propagating over gossip spill into the next checkpoint. Flaky until the sequencer waits + # for the last block's tx window. + - regex: "src/multi-node/block-production/redistribution.parallel.test.ts.*redistributes checkpoint budget" owners: - *phil @@ -290,7 +295,12 @@ tests: owners: - *phil - - regex: "src/e2e_epochs/.*\\.test\\.ts" + - regex: "src/single-node/.*\\.test\\.ts" + flake_group_id: e2e-p2p-epoch-flakes + owners: + - *palla + + - regex: "src/multi-node/.*\\.test\\.ts" flake_group_id: e2e-p2p-epoch-flakes owners: - *palla @@ -408,7 +418,7 @@ tests: # http://ci.aztec-labs.com/930ddc9ede87059f # Pipelining race: slasher doesn't record the duplicate proposal offense before # the test's wait timeout — same family as #23501. - - regex: "src/e2e_p2p/duplicate_proposal_slash.test.ts" + - regex: "src/multi-node/slashing/duplicate_proposal.test.ts.*duplicate proposals" error_regex: "TimeoutError: Timeout awaiting duplicate proposal offense" owners: - *palla diff --git a/yarn-project/end-to-end/bootstrap.sh b/yarn-project/end-to-end/bootstrap.sh index 3c9806c4985c..8bd0b9c89937 100755 --- a/yarn-project/end-to-end/bootstrap.sh +++ b/yarn-project/end-to-end/bootstrap.sh @@ -34,18 +34,28 @@ function test_cmds { fi echo "$prefix:TIMEOUT=25m:NAME=e2e_block_building $(set_dump_avm e2e_block_building) $run_test_script simple e2e_block_building" echo "$prefix:TIMEOUT=30m:NAME=e2e_avm_simulator $(set_dump_avm e2e_avm_simulator) $run_test_script simple src/e2e_avm_simulator.test.ts" - echo "$prefix:TIMEOUT=15m:NAME=e2e_epochs/epochs_long_proving_time $run_test_script simple src/e2e_epochs/epochs_long_proving_time.test.ts" local tests=( # List all standalone and nested tests, except for the ones listed above. - src/e2e_!(prover|epochs)/*.test.ts - src/e2e_epochs/!(epochs_long_proving_time).test.ts + src/e2e_!(prover)/*.test.ts + src/single-node/proving/*.test.ts + src/single-node/l1-reorgs/*.test.ts + src/single-node/recovery/*.test.ts + src/single-node/partial-proofs/*.test.ts + src/single-node/misc/*.test.ts + src/multi-node/block-production/*.test.ts + src/multi-node/recovery/*.test.ts + src/multi-node/invalid-attestations/*.test.ts + src/multi-node/high-availability/*.test.ts + src/multi-node/slashing/*.test.ts src/e2e_p2p/reqresp/*.test.ts src/e2e_!(block_building|avm_simulator).test.ts ) for test in "${tests[@]}"; do - local name=${test#*e2e_} - name=e2e_${name%.test.ts} + # Derive a CI test name from the path: drop the leading "src/" and trailing ".test.ts". + # This keeps e2e_/ names while also handling the multi-node/ category folder. + local name=${test#src/} + name=${name%.test.ts} # Per-test bash TIMEOUT overrides — keep in sync with the test file's jest.setTimeout. local test_prefix="$prefix" @@ -56,14 +66,20 @@ function test_cmds { e2e_cross_chain_messaging/l1_to_l2) test_prefix="$prefix:TIMEOUT=20m" ;; + single-node/proving/long_proving_time) + # The long-proving-time scenario waits out a multi-epoch prover delay. + test_prefix="$prefix:TIMEOUT=15m" + ;; esac # Check if this is a .parallel.test.ts file if [[ "$test" == *.parallel.test.ts ]]; then # Extract individual test names and create a command for each while IFS= read -r test_name; do - # Create a safe name for the individual test (replace spaces with underscores) - local safe_test_name=$(echo "$test_name" | sed 's/ /_/g') + # Create a safe name for the individual test. This becomes the docker container name via + # docker_isolate, so every character outside docker's allowed set [a-zA-Z0-9_.-] (spaces, + # parentheses, etc.) must be collapsed to an underscore or `docker run --name` rejects it. + local safe_test_name=$(echo "$test_name" | sed 's/[^a-zA-Z0-9_.-]/_/g') local full_name="${name}_${safe_test_name}" echo "$test_prefix:NAME=$full_name $(set_dump_avm $full_name) $run_test_script simple $test \"$test_name\"" done < <(extract_test_names "$test") @@ -276,7 +292,9 @@ function compat_test_cmds { if [[ "$test" == *.parallel.test.ts ]]; then while IFS= read -r test_name; do - local safe_test_name=$(echo "$test_name" | sed 's/ /_/g') + # See the matching note in test_cmds: collapse docker-illegal characters so NAME is a valid + # container name for docker_isolate. + local safe_test_name=$(echo "$test_name" | sed 's/[^a-zA-Z0-9_.-]/_/g') local full_name="compat_${version}_${name}_${safe_test_name}" echo "$prefix:NAME=$full_name $compat_env $run_test_script simple $test \"$test_name\"" done < <(extract_test_names "$test") diff --git a/yarn-project/end-to-end/package.local.json b/yarn-project/end-to-end/package.local.json index d2b8973573d3..f421c171eda2 100644 --- a/yarn-project/end-to-end/package.local.json +++ b/yarn-project/end-to-end/package.local.json @@ -3,6 +3,7 @@ "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests src/fixtures" }, "jest": { - "setupFilesAfterEnv": ["../../foundation/src/jest/setupAfterEnv.mjs", "jest-extended/all", "./shared/jest_setup.ts"] + "setupFilesAfterEnv": ["../../foundation/src/jest/setupAfterEnv.mjs", "jest-extended/all", "./shared/jest_setup.ts"], + "testEnvironment": "./shared/timing_env.mjs" } } diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_l1_reorgs.parallel.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_l1_reorgs.parallel.test.ts deleted file mode 100644 index c0f3c2dcfcf2..000000000000 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_l1_reorgs.parallel.test.ts +++ /dev/null @@ -1,565 +0,0 @@ -import type { Archiver } from '@aztec/archiver'; -import type { AztecNodeService } from '@aztec/aztec-node'; -import { AztecAddress } from '@aztec/aztec.js/addresses'; -import { NO_WAIT } from '@aztec/aztec.js/contracts'; -import { Fr } from '@aztec/aztec.js/fields'; -import type { Logger } from '@aztec/aztec.js/log'; -import { isL1ToL2MessageReady, waitForL1ToL2MessageReady } from '@aztec/aztec.js/messaging'; -import type { AztecNode } from '@aztec/aztec.js/node'; -import { createBlobClient } from '@aztec/blob-client/client'; -import { Blob } from '@aztec/blob-lib'; -import type { Delayer } from '@aztec/ethereum/l1-tx-utils'; -import type { ChainMonitor, ChainMonitorEventMap } from '@aztec/ethereum/test'; -import type { ExtendedViemWalletClient } from '@aztec/ethereum/types'; -import { CheckpointNumber } from '@aztec/foundation/branded-types'; -import { timesAsync } from '@aztec/foundation/collection'; -import { AbortError } from '@aztec/foundation/error'; -import { retryUntil } from '@aztec/foundation/retry'; -import { hexToBuffer } from '@aztec/foundation/string'; -import { executeTimeout } from '@aztec/foundation/timer'; -import type { TestContract } from '@aztec/noir-test-contracts.js/Test'; - -import { jest } from '@jest/globals'; -import 'jest-extended'; -import { keccak256, parseTransaction } from 'viem'; - -import { sendL1ToL2Message } from '../fixtures/l1_to_l2_messaging.js'; -import type { EndToEndContext } from '../fixtures/utils.js'; -import { waitForL1ToL2MessageSeen } from '../shared/wait_for_l1_to_l2_message.js'; -import { proveInteraction } from '../test-wallet/utils.js'; -import { EpochsTestContext } from './epochs_test.js'; - -jest.setTimeout(1000 * 60 * 20); - -// Single-node + prover-node suite exercising L1 reorg behavior for both block data and L1→L2 -// messages. Uses EthCheatCodes reorg/reorgWithReplacement to remove or insert L1 transactions -// and verifies the archiver and node prune/restore their views accordingly. Prover and sequencer -// delayers intercept L1 txs to enable controlled reorg scenarios. Uses EpochsTestContext defaults -// (single initial sequencer, fake prover, no mock gossip); actively drives L1 via cheatcodes. -describe('e2e_epochs/epochs_l1_reorgs', () => { - let context: EndToEndContext; - let logger: Logger; - let node: AztecNode; - let archiver: Archiver; - let monitor: ChainMonitor; - let proverDelayer: Delayer; - let sequencerDelayer: Delayer; - - let L1_BLOCK_TIME_IN_S: number; - let L2_SLOT_DURATION_IN_S: number; - - let test: EpochsTestContext; - let contract: TestContract; - let from: AztecAddress; - - // Number of txs to send at the start of each blocks test to trigger multi-block checkpoints. - const TX_COUNT = 8; - - /** Pre-proves and sends txs to generate L2 activity for multi-block checkpoints. */ - const sendTransactions = async (count: number, offset = 0) => { - logger.warn(`Pre-proving ${count} transactions`); - const txs = await timesAsync(count, i => - proveInteraction(context.wallet, contract.methods.emit_nullifier(new Fr(offset + i + 1)), { from }), - ); - const txHashes = await Promise.all(txs.map(tx => tx.send({ wait: NO_WAIT }))); - logger.warn(`Sent ${txHashes.length} transactions`); - return txHashes; - }; - - beforeEach(async () => { - test = await EpochsTestContext.setup({ - numberOfAccounts: 1, - maxSpeedUpAttempts: 0, // Do not speed up l1 txs, we dont want them to land - cancelTxOnTimeout: false, - aztecEpochDuration: 4, - ethereumSlotDuration: 4, - aztecSlotDuration: 36, - blockDurationMs: 8000, - minTxsPerBlock: 0, - maxTxsPerBlock: 1, - aztecProofSubmissionEpochs: 1, - // Use 32 slots/epoch (matching real Ethereum mainnet) - anvilSlotsInAnEpoch: 32, - // Pipelining + multi-blocks-per-slot: 8s blocks fit ~4 blocks per 36s slot, and TX_COUNT=8 - // ensures multiple checkpoints have multiple blocks - inboxLag: 2, - }); - ({ proverDelayer, sequencerDelayer, context, logger, monitor, L1_BLOCK_TIME_IN_S, L2_SLOT_DURATION_IN_S } = test); - node = context.aztecNode; - archiver = (node as AztecNodeService).getBlockSource() as Archiver; - from = context.accounts[0]; - contract = await test.registerTestContract(context.wallet); - }); - - afterEach(async () => { - await test.teardown(); - }); - - // Suite covering L1 reorg effects on L2 block state: proof removal, proof re-addition via - // reorg, checkpoint removal from pending chain, and checkpoint insertion via reorg. - describe('blocks', () => { - const getBlobs = async (serializedTx: `0x${string}`) => { - const parsedTx = parseTransaction(serializedTx); - if (parsedTx.sidecars === false) { - throw new Error('No sidecars found in tx'); - } - return await Promise.all(parsedTx.sidecars!.map(sidecar => Blob.fromBlobBuffer(hexToBuffer(sidecar.blob)))); - }; - - /** Returns the last synced checkpoint number for a node */ - const getCheckpointNumber = (node: AztecNode) => - node.getChainTips().then(tips => tips.checkpointed.checkpoint.number); - - /** Returns the last proven checkpoint number for a node */ - const getProvenCheckpointNumber = (node: AztecNode) => - node.getChainTips().then(tips => tips.proven.checkpoint.number); - - // Waits for an initial proof to land, stops the prover, reorgs L1 to remove the proof block, - // waits for the proof submission window to expire, spins up a new sync-only node, and verifies - // both the new node and the old node have rolled back to the pre-proof checkpoint number. - it('prunes L2 blocks if a proof is removed due to an L1 reorg', async () => { - /** Logs a full state snapshot: L1 latest/finalized and archiver L2 tips. */ - const logState = async (label: string) => { - const [l1Latest, l1Finalized, archiverTips] = await Promise.all([ - test.l1Client.getBlockNumber(), - test.l1Client.getBlock({ blockTag: 'finalized', includeTransactions: false }).then(b => b.number), - archiver.getL2Tips(), - ]); - logger.warn(`[state:${label}]`, { - l1Latest, - l1Finalized, - l2Proposed: archiverTips.proposed.number, - l2Checkpointed: archiverTips.checkpointed.block.number, - l2Proven: archiverTips.proven.block.number, - provenCheckpoint: archiverTips.proven.checkpoint.number, - l2Finalized: archiverTips.finalized.block.number, - finalizedCheckpoint: archiverTips.finalized.checkpoint.number, - }); - }; - - // Send txs to trigger multi-block checkpoints - await sendTransactions(TX_COUNT); - - // Capture initial chain state - const initialProvenCheckpoint = (await monitor.run(true)).provenCheckpointNumber; - await logState('initial'); - - // Wait until we have proven something and the nodes have caught up - const epochDurationSeconds = test.constants.epochDuration * test.constants.slotDuration; - logger.warn(`Waiting for initial proof to land`); - const provenBlockEvent = await executeTimeout( - signal => { - return new Promise<{ provenCheckpointNumber: number; l1BlockNumber: number }>((res, rej) => { - const handleMsg = (...[ev]: ChainMonitorEventMap['checkpoint-proven']) => { - if (ev.provenCheckpointNumber > initialProvenCheckpoint) { - res(ev); - monitor.off('checkpoint-proven', handleMsg); - } - }; - - signal.onabort = () => { - monitor.off('checkpoint-proven', handleMsg); - rej(new AbortError()); - }; - monitor.on('checkpoint-proven', handleMsg); - }); - }, - epochDurationSeconds * 4 * 1000, - ); - - logger.warn( - `Proof for checkpoint ${provenBlockEvent.provenCheckpointNumber} mined at L1 block ${provenBlockEvent.l1BlockNumber}`, - ); - await logState('proof-landed'); - - // Stop the prover node (by stopping its hosting aztec node) so it doesn't re-submit the proof after we've removed it - logger.warn(`Stopping prover node`); - await test.proverNodes[0].stop(); - await logState('prover-stopped'); - - // And remove the proof from L1 - const reorgTarget = provenBlockEvent.l1BlockNumber - 1; - logger.warn( - `Reorging L1 from current tip to block ${reorgTarget} (removing proof block ${provenBlockEvent.l1BlockNumber})`, - ); - await context.cheatCodes.eth.reorgTo(reorgTarget); - await logState('after-reorg'); - expect((await monitor.run(true)).provenCheckpointNumber).toEqual(initialProvenCheckpoint); - - // Wait until the end of the proof submission window for the epoch of the proven checkpoint - const provenCheckpointEpoch = await test.rollup.getEpochNumberForCheckpoint( - CheckpointNumber(provenBlockEvent.provenCheckpointNumber), - ); - await test.waitUntilLastSlotOfProofSubmissionWindow(provenCheckpointEpoch); - await logState('after-submission-window'); - - // Ensure that a new node sees the reorg - logger.warn(`Syncing new node to test reorg`); - const newNode = await executeTimeout(() => test.createNonValidatorNode(), 10_000, `new node sync`); - expect(await getProvenCheckpointNumber(newNode)).toEqual(initialProvenCheckpoint); - - // Latest checkpointed block seen by the node may be from the current checkpoint, or one less if it was *just* mined. - // This is because the call to createNonValidatorNode will block until the initial sync is completed, - // but the initial sync is done to the latest L1 block _at the time the initial sync starts_. So a new - // checkpoint may have appeared while the initial sync runs, that's why we account for a small span. - const currentCheckpointNumber = (await monitor.run(true)).checkpointNumber; - expect(await getCheckpointNumber(newNode)).toBeWithin(currentCheckpointNumber - 1, currentCheckpointNumber + 1); - - // And check that the old node has processed the reorg as well - logger.warn(`Testing old node after reorg`); - // REFACTOR: hand-rolled poll on proven checkpoint equality; replace with - // test.waitUntilProvenCheckpointNumber(initialProvenCheckpoint, timeout). - await retryUntil( - () => getProvenCheckpointNumber(node).then(cp => cp === initialProvenCheckpoint), - 'prune', - L2_SLOT_DURATION_IN_S * 4, - 0.1, - ); - await logState('old-node-synced'); - expect(await getCheckpointNumber(node)).toBeWithin(monitor.checkpointNumber - 1, monitor.checkpointNumber + 1); - - // Verify multi-block checkpoints were built - await test.assertMultipleBlocksPerSlot(2); - - logger.warn(`Test succeeded`); - await newNode.stop(); - }); - - // Waits for a proof, stops the prover, removes the proof via reorgWithReplacement (same block - // count), starts a fresh prover node, and verifies a new proof lands and the node re-syncs to - // the proven state without having pruned. - it('does not prune if a second proof lands within the submission window after the first one is reorged out', async () => { - // Send txs to trigger multi-block checkpoints - await sendTransactions(TX_COUNT); - - // Capture initial chain state - const initialProvenCheckpoint = (await monitor.run(true)).provenCheckpointNumber; - const targetProvenCheckpoint = CheckpointNumber(initialProvenCheckpoint + 1); - - // Wait until we have proven something and the nodes have caught up - // Use a longer timeout since we need to wait for the epoch to complete (~288s) plus proving time. - const epochDurationSeconds = test.constants.epochDuration * test.constants.slotDuration; - logger.warn(`Waiting for initial proof to land`); - const provenCheckpoint = await test.waitUntilProvenCheckpointNumber( - targetProvenCheckpoint, - epochDurationSeconds * 4, - ); - await retryUntil(() => getProvenCheckpointNumber(node).then(cp => cp >= provenCheckpoint), 'node sync', 10, 0.1); - - // Stop the prover node (by stopping its hosting aztec node) - await test.proverNodes[0].stop(); - - // Remove the proof from L1 but do not change the block number - await context.cheatCodes.eth.reorgWithReplacement(1); - await expect(monitor.run(true).then(m => m.provenCheckpointNumber)).resolves.toEqual(initialProvenCheckpoint); - - // Create another prover node so it submits a proof and wait until it is submitted - await test.createProverNode(); - const provenCheckpointRetry = await test.waitUntilProvenCheckpointNumber(CheckpointNumber(1)); - await expect(monitor.run(true).then(m => m.provenCheckpointNumber)).resolves.toBeGreaterThanOrEqual(1); - - // Check that the node has followed along - logger.warn(`Testing old node`); - await retryUntil( - () => getProvenCheckpointNumber(node).then(cp => cp >= provenCheckpointRetry), - 'proof sync', - 10, - 0.1, - ); - expect(await getCheckpointNumber(node)).toBeWithin(monitor.checkpointNumber - 1, monitor.checkpointNumber + 1); - - // Verify multi-block checkpoints were built - await test.assertMultipleBlocksPerSlot(2); - - logger.warn(`Test succeeded`); - // New prover's aztec node is stopped in test.teardown() - }); - - // Cancels the next prover L1 tx so no proof lands, waits for the end of the submission window - // (triggering pruning), then reorgs L1 to include the previously-cancelled proof tx and - // verifies the node un-prunes and resumes from the proven state. - it('restores L2 blocks if a proof is added due to an L1 reorg', async () => { - // Send txs to trigger multi-block checkpoints - await sendTransactions(TX_COUNT); - - // Capture initial chain state - const initialProvenCheckpoint = (await monitor.run(true)).provenCheckpointNumber; - const initialCheckpoint = monitor.checkpointNumber; - - // Next proof shall not land - proverDelayer.cancelNextTx(); - - // Expect pending chain to advance, so there's something to be pruned - await retryUntil( - () => getCheckpointNumber(node).then(cp => cp > initialCheckpoint), - 'node sync', - L2_SLOT_DURATION_IN_S * 4, - 0.1, - ); - - // Wait until the end of the proof submission window for the first unproven epoch - const firstUnprovenCheckpoint = CheckpointNumber(initialProvenCheckpoint + 1); - await test.waitUntilCheckpointNumber(firstUnprovenCheckpoint, L2_SLOT_DURATION_IN_S * 4); - const epochToWaitFor = await test.rollup.getEpochNumberForCheckpoint(firstUnprovenCheckpoint); - await test.waitUntilLastSlotOfProofSubmissionWindow(epochToWaitFor); - await monitor.run(true); - logger.warn( - `End of epoch ${epochToWaitFor} submission window (L1 block ${await monitor.run(true).then(m => m.l1BlockNumber)}).`, - ); - - // Grab the prover's tx to submit it later as part of a reorg and stop the prover (by stopping its hosting aztec node) - const [proofTx] = proverDelayer.getCancelledTxs(); - expect(proofTx).toBeDefined(); - await test.proverNodes[0].stop(); - logger.warn(`Prover node stopped.`); - - // Wait for the node to prune - const syncTimeout = L2_SLOT_DURATION_IN_S * 2; - await retryUntil( - () => getCheckpointNumber(node).then(cp => cp <= initialProvenCheckpoint + 1), - 'node prune', - syncTimeout, - 0.1, - ); - expect(monitor.provenCheckpointNumber).toEqual(initialProvenCheckpoint); - expect(await getProvenCheckpointNumber(node)).toEqual(initialProvenCheckpoint); - - // But not all is lost, for a reorg gets the proof back on chain! - logger.warn(`Reorging proof back (L1 block ${await monitor.run(true).then(m => m.l1BlockNumber)}).`); - await context.cheatCodes.eth.reorgWithReplacement(4, [[proofTx]]); - const proofTxReceipt = await test.l1Client.getTransactionReceipt({ hash: keccak256(proofTx) }); - expect(proofTxReceipt.status).toEqual('success'); - - // Monitor should update to see the proof - const { checkpointNumber, provenCheckpointNumber } = await monitor.run(true); - expect(checkpointNumber).toBeGreaterThan(initialCheckpoint); - expect(provenCheckpointNumber).toBeGreaterThan(initialProvenCheckpoint); - - // And so the node undoes its reorg - await retryUntil( - () => getCheckpointNumber(node).then(b => b >= checkpointNumber), - 'node resync', - syncTimeout, - 0.1, - ); - await retryUntil( - () => getProvenCheckpointNumber(node).then(b => b >= provenCheckpointNumber), - 'proof sync', - 1, - 0.1, - ); - - // Verify multi-block checkpoints were built - await test.assertMultipleBlocksPerSlot(2); - - logger.warn(`Test succeeded`); - }); - - // Waits until CHECKPOINT_NUMBER is mined and node synced, stops the sequencer, reorgs L1 to - // remove that checkpoint's L1 block, and verifies the node rolls back to checkpoint-1. - it('prunes blocks from pending chain removed from L1 due to an L1 reorg', async () => { - // Send txs to trigger multi-block checkpoints - await sendTransactions(TX_COUNT); - - // Capture initial chain state - const initialCheckpoint = (await monitor.run(true)).checkpointNumber; - - // Wait until CHECKPOINT_NUMBER is mined and node synced, and stop the sequencer - const CHECKPOINT_NUMBER = CheckpointNumber(initialCheckpoint + 3); - await test.waitUntilCheckpointNumber(CHECKPOINT_NUMBER, L2_SLOT_DURATION_IN_S * 10); - expect(monitor.checkpointNumber).toEqual(CHECKPOINT_NUMBER); - const l1BlockNumber = monitor.l1BlockNumber; - // Stop the sequencer immediately so any in-flight pipelined publish for CHECKPOINT_NUMBER+1 - // doesn't extend the reorg range before we calculate it. setConfig alone is not enough under - // pipelining because already-constructed jobs snapshot the old config. - await context.sequencer!.stop(); - logger.warn(`Sequencer stopped`); - // Wait for node to sync to the checkpoint. - await retryUntil(() => getCheckpointNumber(node).then(b => b === CHECKPOINT_NUMBER), 'node sync', 10, 0.1); - logger.warn(`Reached checkpoint ${CHECKPOINT_NUMBER}`); - - // Verify multi-block checkpoints were built before we do the reorg - await test.assertMultipleBlocksPerSlot(2); - - // Remove the L2 block from L1 - await context.cheatCodes.eth.reorgTo(l1BlockNumber - 1); - expect(await monitor.run(true).then(monitor => monitor.checkpointNumber)).toEqual( - CheckpointNumber(CHECKPOINT_NUMBER - 1), - ); - logger.warn(`Removed checkpoint ${CHECKPOINT_NUMBER} via L1 reorg`); - - // And expect the node to prune the block - const expectedCheckpointNumber = CHECKPOINT_NUMBER - 1; - await retryUntil(() => getCheckpointNumber(node).then(b => b === expectedCheckpointNumber), 'node sync', 30, 0.1); - }); - - // Cancels the next sequencer L1 tx (blocking CHECKPOINT_NUMBER from landing), waits for - // several more L1 blocks to pass, then reorgs L1 to include the previously-cancelled checkpoint - // tx and manually sends the blobs to the filestore. Verifies the node sees the new block. - it('sees new blocks added in an L1 reorg', async () => { - // Send txs to trigger multi-block checkpoints - await sendTransactions(TX_COUNT); - - // Capture initial chain state - const initialCheckpoint = (await monitor.run(true)).checkpointNumber; - - // Wait until the checkpoint *before* CHECKPOINT_NUMBER is mined and node synced - const CHECKPOINT_NUMBER = CheckpointNumber(initialCheckpoint + 3); - const prevCheckpointNumber = CheckpointNumber(CHECKPOINT_NUMBER - 1); - await test.waitUntilCheckpointNumber(prevCheckpointNumber, L2_SLOT_DURATION_IN_S * 10); - expect(monitor.checkpointNumber).toEqual(prevCheckpointNumber); - // Wait for node to sync to the checkpoint - await retryUntil(() => getCheckpointNumber(node).then(b => b === prevCheckpointNumber), 'node sync', 5, 0.1); - - // Verify multi-block checkpoints were built before we do the reorg - await test.assertMultipleBlocksPerSlot(2); - - // Cancel the next tx to be mined (the proposal for CHECKPOINT_NUMBER) and pause the sequencer. - // Under pipelining we then stop the sequencer entirely so an in-flight pipelined job for - // CHECKPOINT_NUMBER+1 cannot escape and publish onto L1 before our reorg captures the gap. - sequencerDelayer.cancelNextTx(); - await retryUntil(() => sequencerDelayer.getCancelledTxs().length, 'next block', L2_SLOT_DURATION_IN_S * 2, 0.1); - const [l2BlockTx] = sequencerDelayer.getCancelledTxs(); - await context.sequencer!.stop(); - logger.warn(`Sequencer stopped`); - - // Save the L1 block number when the L2 block would have been mined - const l1BlockNumber = monitor.l1BlockNumber; - - // Wait until a few more L1 blocks go by - await retryUntil(() => monitor.l1BlockNumber > l1BlockNumber + 1, 'l1 block number', L1_BLOCK_TIME_IN_S * 4, 0.1); - await retryUntil(() => archiver.getL1BlockNumber()! > l1BlockNumber + 1, 'archiver sync', 10, 0.1); - expect(await getCheckpointNumber(node)).toEqual(prevCheckpointNumber); - - // Manually update the archiver's L1 syncpoint to ensure we look back when needed - // Otherwise this test just passes because we do not update the L1 syncpoint in the archiver since there are no new blocks - await archiver.dataStores.blocks.setSynchedL1BlockNumber(BigInt(archiver.getL1BlockNumber()!)); - - // Now trigger the reorg. Note that we cannot use reorgWithReplacement here for the reorg, due to an anvil bug with - // blob txs (now fixed, we can just update its version), so we reorg, then replay the tx, and then mine. - const reorgDepth = monitor.l1BlockNumber - l1BlockNumber; - expect(reorgDepth).toBeGreaterThan(0); - logger.warn(`Triggering ${reorgDepth}-block L1 reorg to include L2 block`); - await context.cheatCodes.eth.reorg(reorgDepth); - expect(await context.cheatCodes.eth.blockNumber()).toEqual(l1BlockNumber); - logger.warn(`Sending L2 block tx to L1`); - const txHash = await test.l1Client.sendRawTransaction({ serializedTransaction: l2BlockTx }); - await context.cheatCodes.eth.mine(reorgDepth); - - // Check that the tx was reorged in and succeeded. We log the trace to debug any issues with the tx. - const txReceipt = await test.l1Client.getTransactionReceipt({ hash: txHash }); - logger.warn(`L2 block tx receipt`, { receipt: txReceipt }); - logger.warn(`L2 block tx trace`, { trace: await context.cheatCodes.eth.traceTransaction(txHash) }); - expect(txReceipt.status).toEqual('success'); - expect(txReceipt.blobGasUsed).toBeGreaterThan(0n); - expect(await monitor.run(true).then(m => m.checkpointNumber)).toEqual(CHECKPOINT_NUMBER); - - // We also need to send the blob to the sink, so the node can get it - logger.warn(`Sending blobs to blob client`); - const blobs = await getBlobs(l2BlockTx); - const blobClient = createBlobClient(context.config); - await blobClient.sendBlobsToFilestore(blobs); - - // And wait for the node to see the new block - await retryUntil(() => getCheckpointNumber(node).then(b => b === CHECKPOINT_NUMBER), 'node sync', 20, 0.1); - }); - }); - - // Suite covering L1 reorg effects on L1→L2 cross-chain messages: removal of a sent message - // and insertion of a previously-cancelled message. - describe('messages', () => { - let l1Client: ExtendedViemWalletClient; - let l1ClientDelayer: Delayer; - - beforeEach(async () => { - ({ client: l1Client, delayer: l1ClientDelayer } = await test.createL1Client()); - }); - - const sendMessage = async () => - sendL1ToL2Message( - { recipient: await AztecAddress.random(), content: Fr.random(), secretHash: Fr.random() }, - { l1ContractAddresses: context.deployL1ContractsValues.l1ContractAddresses, l1Client }, - ); - - // Sends 3 L1→L2 messages, waits for the last to be seen, reorgs it out, sends a replacement - // message, and verifies the replacement becomes ready while the removed message is gone. - it('updates L1 to L2 messages changed due to an L1 reorg', async () => { - // Send L2 txs to trigger multi-block checkpoints and wait for them to land in a checkpoint - await sendTransactions(TX_COUNT, 100); - await test.waitUntilCheckpointNumber(CheckpointNumber(2), L2_SLOT_DURATION_IN_S * 6); - - // Send 3 messages and wait for archiver sync - logger.warn(`Sending 3 cross chain messages`); - const msgs = await timesAsync(3, async (i: number) => { - logger.warn(`Sending message ${i + 1}`); - return await sendMessage(); - }); - logger.warn(`Sent messages on L1 blocks ${msgs.map(m => m.txReceipt.blockNumber)}`); - - await waitForL1ToL2MessageSeen(node, msgs.at(-1)!.msgHash, { - timeoutSeconds: msgs.length * L1_BLOCK_TIME_IN_S * 2, - }); - - // Reorg the last message out - logger.warn(`Triggering reorg to remove last message`); - const l1BlockNumber = await monitor.run(true).then(m => m.l1BlockNumber); - const l1BlocksToReorg = l1BlockNumber - Number(msgs.at(-1)!.txReceipt.blockNumber) + 1; - await context.cheatCodes.eth.reorg(l1BlocksToReorg); - const newMsg = await sendMessage(); - logger.warn(`Sent new message on L1 block ${newMsg.txReceipt.blockNumber}`); - - // New msg gets synced, and old one is out - await waitForL1ToL2MessageReady(node, newMsg.msgHash, { timeoutSeconds: L2_SLOT_DURATION_IN_S * 5 }); - expect(await isL1ToL2MessageReady(node, msgs[0].msgHash)).toBe(true); - expect(await isL1ToL2MessageReady(node, msgs.at(-1)!.msgHash)).toBe(false); - - // Verify multi-block checkpoints were built - await test.assertMultipleBlocksPerSlot(2); - }); - - // Sends a first message, cancels a second message's L1 tx via delayer, waits for the archiver - // to advance past the cancelled block, then reorgs to include the cancelled message. Sends a - // third message on top and verifies all three are eventually seen by the node. - it('handles missed message inserted by an L1 reorg', async () => { - // Send L2 txs to trigger multi-block checkpoints and wait for them to land in a checkpoint - await sendTransactions(TX_COUNT, 200); - await test.waitUntilCheckpointNumber(CheckpointNumber(2), L2_SLOT_DURATION_IN_S * 6); - - // Send a message and wait for node to sync it - logger.warn(`Sending first cross chain message`); - const firstMsg = await sendMessage(); - logger.warn(`Sent first message on L1 block ${firstMsg.txReceipt.blockNumber}`); - await waitForL1ToL2MessageSeen(node, firstMsg.msgHash, { timeoutSeconds: L1_BLOCK_TIME_IN_S * 3 }); - logger.warn(`Synced first message`); - - // Next message shall not land - l1ClientDelayer.cancelNextTx(); - const secondMsgPromise = sendMessage(); - await retryUntil(() => l1ClientDelayer.getCancelledTxs().length, 'next msg tx', L1_BLOCK_TIME_IN_S, 0.1); - - // Wait until the archiver moves the syncpoint forward - const l1BlockNumber = await monitor.run(true).then(m => m.l1BlockNumber); - await retryUntil( - () => archiver.getL1BlockNumber()! > l1BlockNumber, - 'archiver sync', - L1_BLOCK_TIME_IN_S * 2, - 0.1, - ); - - // Now trigger the reorg, where we insert the second message - logger.warn(`Triggering reorg to insert second message`); - const reorgDepth = (await monitor.run(true).then(m => m.l1BlockNumber)) - l1BlockNumber; - await context.cheatCodes.eth.reorgWithReplacement(reorgDepth, [[l1ClientDelayer.getCancelledTxs()[0]]]); - const secondMsg = await secondMsgPromise; - await waitForL1ToL2MessageSeen(node, secondMsg.msgHash, { timeoutSeconds: L1_BLOCK_TIME_IN_S * 3 }); - - // Archiver should see the new message and should be able to accept a third one on top, without any rolling hash issues - logger.warn(`Reorged-in second message on L1 block ${secondMsg.txReceipt.blockNumber}. Sending third message.`); - const thirdMsg = await sendMessage(); - await waitForL1ToL2MessageSeen(node, thirdMsg.msgHash, { timeoutSeconds: L1_BLOCK_TIME_IN_S * 3 }); - - // Verify multi-block checkpoints were built - await test.assertMultipleBlocksPerSlot(2); - }); - }); -}); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_manual_rollback.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_manual_rollback.test.ts deleted file mode 100644 index 3c2066bc08dd..000000000000 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_manual_rollback.test.ts +++ /dev/null @@ -1,70 +0,0 @@ -import type { Logger } from '@aztec/aztec.js/log'; -import type { AztecNode } from '@aztec/aztec.js/node'; -import type { RollupContract } from '@aztec/ethereum/contracts'; -import { CheckpointNumber } from '@aztec/foundation/branded-types'; -import { retryUntil } from '@aztec/foundation/retry'; - -import { jest } from '@jest/globals'; - -import type { EndToEndContext } from '../fixtures/utils.js'; -import { EpochsTestContext, type EpochsTestOpts } from './epochs_test.js'; - -jest.setTimeout(1000 * 60 * 10); - -// Single-node suite exercising the aztecNodeAdmin.rollbackTo() API. Default EpochsTestContext with -// a very long epoch (aztecEpochDuration=100) so there are no L2 reorgs, no finalized blocks, and -// the full pending chain is prunable. Actively drives L1 via cheatcodes (reorgTo to remove blocks). -describe('e2e_epochs/manual_rollback', () => { - let context: EndToEndContext; - let logger: Logger; - let node: AztecNode; - let rollup: RollupContract; - - let test: EpochsTestContext; - - const setup = async (opts: Partial = {}) => { - test = await EpochsTestContext.setup({ ...opts }); - ({ context, logger, rollup } = test); - ({ aztecNode: node } = context); - }; - - afterEach(async () => { - jest.restoreAllMocks(); - await test.teardown(); - }); - - // Sub-suite for rolling back to a block that has not been finalized (epoch=100 → no finalization). - describe('to unfinalized block', () => { - beforeEach(async () => { - await setup({ aztecEpochDuration: 100 }); // No L2 reorgs, no finalized blocks - }); - - // Waits for checkpoint 4, pauses node sync, reorgs L1 by 2 blocks, calls rollbackTo on the - // node, and asserts blockNumber equals the rolled-back value. Resumes sync and verifies the - // node re-syncs to the same block. - it('manually rolls back', async () => { - logger.info(`Starting manual rollback test to unfinalized block`); - context.sequencer?.updateConfig({ minTxsPerBlock: 0 }); - const targetCheckpointNumber = CheckpointNumber(4); - // With pipelining, each checkpoint takes ~2 L2 slots on a solo-sequencer setup. - await test.waitUntilCheckpointNumber(targetCheckpointNumber, test.L2_SLOT_DURATION_IN_S * 12); - await retryUntil(async () => await node.getBlockNumber().then(b => b >= 4), 'sync to 4', 10, 0.1); - - logger.info(`Synced to checkpoint 4. Pausing syncing and rolling back the chain.`); - await context.aztecNodeAdmin.pauseSync(); - context.sequencer?.updateConfig({ minTxsPerBlock: 100 }); // Ensure no new blocks are produced - await context.cheatCodes.eth.reorg(2); - const checkpointAfterReorg = await rollup.getCheckpointNumber(); - expect(checkpointAfterReorg).toBeLessThan(targetCheckpointNumber); - logger.info(`Rolled back to checkpoint ${checkpointAfterReorg}.`); - - logger.info(`Manually rolling back node to ${checkpointAfterReorg - 1}.`); - const blockAfterReorg = Number(checkpointAfterReorg - 1); - await context.aztecNodeAdmin.rollbackTo(blockAfterReorg); - expect(await node.getBlockNumber()).toEqual(blockAfterReorg); - - logger.info(`Waiting for node to re-sync to ${blockAfterReorg}.`); - await retryUntil(async () => await node.getBlockNumber().then(b => b >= blockAfterReorg), 'resync', 10, 0.1); - }); - }); -}); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_mbps.parallel.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_mbps.parallel.test.ts deleted file mode 100644 index 72660abcfc85..000000000000 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_mbps.parallel.test.ts +++ /dev/null @@ -1,648 +0,0 @@ -import type { Archiver } from '@aztec/archiver'; -import type { AztecNodeService } from '@aztec/aztec-node'; -import { AztecAddress, EthAddress } from '@aztec/aztec.js/addresses'; -import { NO_WAIT } from '@aztec/aztec.js/contracts'; -import { generateClaimSecret } from '@aztec/aztec.js/ethereum'; -import { Fr } from '@aztec/aztec.js/fields'; -import type { Logger } from '@aztec/aztec.js/log'; -import { isL1ToL2MessageReady } from '@aztec/aztec.js/messaging'; -import { waitForTx } from '@aztec/aztec.js/node'; -import { RollupContract } from '@aztec/ethereum/contracts'; -import type { Operator } from '@aztec/ethereum/deploy-aztec-l1-contracts'; -import { waitUntilL1Timestamp } from '@aztec/ethereum/l1-tx-utils'; -import { asyncMap } from '@aztec/foundation/async-map'; -import { CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types'; -import { times, timesAsync } from '@aztec/foundation/collection'; -import { SecretValue } from '@aztec/foundation/config'; -import { retryUntil } from '@aztec/foundation/retry'; -import { sleep } from '@aztec/foundation/sleep'; -import { bufferToHex } from '@aztec/foundation/string'; -import { executeTimeout } from '@aztec/foundation/timer'; -import { TestContract } from '@aztec/noir-test-contracts.js/Test'; -import { getSlotAtTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; -import { GasFees } from '@aztec/stdlib/gas'; -import { TxStatus } from '@aztec/stdlib/tx'; - -import { jest } from '@jest/globals'; -import { privateKeyToAccount } from 'viem/accounts'; - -import { sendL1ToL2Message } from '../fixtures/l1_to_l2_messaging.js'; -import { type EndToEndContext, getPrivateKeyFromIndex } from '../fixtures/utils.js'; -import { TestWallet } from '../test-wallet/test_wallet.js'; -import { proveInteraction } from '../test-wallet/utils.js'; -import { EpochsTestContext, type TrackedSequencerEvent } from './epochs_test.js'; - -jest.setTimeout(1000 * 60 * 20); - -const NODE_COUNT = 4; -const EXPECTED_BLOCKS_PER_CHECKPOINT = 3; - -// Send enough transactions to trigger multiple blocks within a checkpoint assuming 2 txs per block. -// If we start including txs at the 2nd block of a checkpoint, we can ensure a 3-block checkpoint -// if we produce 10 txs: -// - Checkpoint 1: Block 1 (0 txs), Block 2 (2 txs), Block 3 (2 txs) -// - Checkpoint 2: Block 1 (2 txs), Block 2 (2 txs), Block 3 (2 txs) -const TX_COUNT = 10; - -/** - * E2E tests for Multiple Blocks Per Slot (MBPS) functionality. - * Tests that the system correctly builds multiple blocks within a single slot/checkpoint. - * - * Four-validator suite under mock gossip with a prover node (fake proofs); PXE mode varies per test - * (checkpointed vs proposed). Exercises MBPS with: checkpointed-anchored txs, proposed-anchored txs, - * L2→L1 messages, L1→L2 messages, non-validator re-execution sync, cross-slot contract deploy+call, - * and prover proving MBPS checkpoints. Uses EpochsTestContext with mockGossipSubNetwork and no - * initial sequencer. - */ -describe('e2e_epochs/epochs_mbps', () => { - let context: EndToEndContext; - let logger: Logger; - let rollup: RollupContract; - let archiver: Archiver; - - let test: EpochsTestContext; - let validators: (Operator & { privateKey: `0x${string}` })[]; - let nodes: AztecNodeService[]; - let contract: TestContract; - let crossChainContract: TestContract | undefined; - let wallet: TestWallet; - let from: AztecAddress; - let failEvents: TrackedSequencerEvent[]; - - /** - * Creates validators and sets up the test context with MBPS configuration. - */ - async function setupTest(opts: { - syncChainTip: 'proposed' | 'checkpointed'; - minTxsPerBlock?: number; - maxTxsPerBlock?: number; - buildCheckpointIfEmpty?: boolean; - skipPushProposedBlocksToArchiver?: boolean; - }) { - const { syncChainTip = 'checkpointed', ...setupOpts } = opts; - - validators = times(NODE_COUNT, i => { - const privateKey = bufferToHex(getPrivateKeyFromIndex(i + 3)!); - const attester = EthAddress.fromString(privateKeyToAccount(privateKey).address); - return { attester, withdrawer: attester, privateKey, bn254SecretKey: new SecretValue(Fr.random().toBigInt()) }; - }); - - // Setup context with the given set of validators and MBPS configuration. - // Pipelining is enabled, so we adopt the wider timing used by the dedicated - // epochs_mbps.pipeline.parallel test (72s L2 slots, 12s L1 slots, 5500ms blocks). - // The tighter 36s/4s timing produces CheckpointNumberNotSequentialError on non-proposer - // nodes when the pipelined proposer races ahead of L1 confirmation (see A-914). - test = await EpochsTestContext.setup({ - numberOfAccounts: 0, - initialValidators: validators, - mockGossipSubNetwork: true, - startProverNode: true, - // Mirrors the pipeline-MBPS sibling: more blocks per slot needs a larger per-block gas - // allocation multiplier so each block can fit non-trivial txs. - perBlockAllocationMultiplier: 8, - aztecEpochDuration: 4, - // L1 slot duration - mirrors the pipeline-MBPS test for headroom on the parent's L1 tx - ethereumSlotDuration: 12, - // L2 slot duration - should fit several blocks (5.5s each) with pipelining overhead - aztecSlotDuration: 72, - // Block duration of 5.5s, matches the pipeline sibling - blockDurationMs: 5500, - // Committee size of 3 - aztecTargetCommitteeSize: 3, - // Additional options (minTxsPerBlock, maxTxsPerBlock, etc.) - ...setupOpts, - // PXE options for chain tip syncing - pxeOpts: { syncChainTip }, - skipInitialSequencer: true, - inboxLag: 2, - }); - - ({ context, logger, rollup } = test); - wallet = context.wallet; - from = context.accounts[0]; // auto-created by setup - - // Start the validator nodes - logger.warn(`Initial setup complete. Starting ${NODE_COUNT} validator nodes.`); - nodes = await asyncMap(validators, ({ privateKey }) => - test.createValidatorNode([privateKey], { dontStartSequencer: true }), - ); - logger.warn(`Started ${NODE_COUNT} validator nodes.`, { validators: validators.map(v => v.attester.toString()) }); - ({ failEvents } = test.watchSequencerEvents( - nodes.map(n => n.getSequencer()!), - i => ({ validator: validators[i].attester }), - )); - - // Point the wallet at a validator node. The initial node-0 has all validator keys in its config, - // so it rejects block proposals from validators thinking they come from itself. By redirecting - // the wallet to a validator node, the PXE correctly tracks proposed blocks. - wallet.updateNode(nodes[0]); - archiver = nodes[0].getBlockSource() as Archiver; - - // Register contract for sending txs. - contract = await test.registerTestContract(wallet); - logger.warn(`Test setup completed.`, { validators: validators.map(v => v.attester.toString()) }); - } - - /** Retrieves all checkpoints from the archiver, checks that one has the target block count, and returns its number. */ - async function assertMultipleBlocksPerSlot(targetBlockCount: number, logger: Logger): Promise { - // Wait for the first validator's archiver to index a checkpoint with the target block count. - // waitForTx polls the initial setup node, but this archiver belongs to nodes[0] (the first - // validator). They sync L1 independently, so there's a race window of ~200-400ms. - const waitTimeout = test.L2_SLOT_DURATION_IN_S * 3; - await retryUntil( - async () => { - const checkpoints = await archiver.getCheckpoints({ from: CheckpointNumber(1), limit: 50 }); - return checkpoints.some(pc => pc.checkpoint.blocks.length >= targetBlockCount) || undefined; - }, - `checkpoint with at least ${targetBlockCount} blocks`, - waitTimeout, - 0.5, - ); - - const checkpoints = await archiver.getCheckpoints({ from: CheckpointNumber(1), limit: 50 }); - logger.warn(`Retrieved ${checkpoints.length} checkpoints from archiver`, { - checkpoints: checkpoints.map(pc => pc.checkpoint.getStats()), - }); - - let expectedBlockNumber = checkpoints[0].checkpoint.blocks[0].number; - let multiBlockCheckpointNumber: CheckpointNumber | undefined; - - for (const checkpoint of checkpoints) { - const blockCount = checkpoint.checkpoint.blocks.length; - if (blockCount >= targetBlockCount && multiBlockCheckpointNumber === undefined) { - multiBlockCheckpointNumber = checkpoint.checkpoint.number; - } - logger.warn(`Checkpoint ${checkpoint.checkpoint.number} has ${blockCount} blocks`, { - checkpoint: checkpoint.checkpoint.getStats(), - }); - - for (let i = 0; i < blockCount; i++) { - const block = checkpoint.checkpoint.blocks[i]; - expect(block.indexWithinCheckpoint).toBe(i); - expect(block.checkpointNumber).toBe(checkpoint.checkpoint.number); - expect(block.number).toBe(expectedBlockNumber); - expectedBlockNumber++; - } - } - - expect(multiBlockCheckpointNumber).toBeDefined(); - return multiBlockCheckpointNumber!; - } - - /** Waits until a specific multi-block checkpoint is proven, verifying that proving succeeds with MBPS blocks. */ - async function waitForProvenCheckpoint(targetCheckpoint: CheckpointNumber) { - test.assertNoFailuresFromSequencers(failEvents); - - logger.warn(`Stopping validator sequencers before waiting for checkpoint ${targetCheckpoint} to be proven`); - await Promise.all(nodes.map(n => n.getSequencer()?.stop())); - - const provenTimeout = test.L2_SLOT_DURATION_IN_S * test.epochDuration * 4; - logger.warn(`Waiting for checkpoint ${targetCheckpoint} to be proven (timeout=${provenTimeout}s)`); - await test.waitUntilProvenCheckpointNumber(targetCheckpoint, provenTimeout); - logger.warn(`Proven checkpoint advanced to ${test.monitor.provenCheckpointNumber}`); - } - - afterEach(async () => { - jest.restoreAllMocks(); - await test?.teardown(); - }); - - // Pre-proves and sends TX_COUNT txs, starts sequencers, waits for all txs to be mined, asserts a - // checkpoint with ≥EXPECTED_BLOCKS_PER_CHECKPOINT exists, then waits for that checkpoint to be proven. - it('builds multiple blocks per slot with transactions anchored to checkpointed block', async () => { - await setupTest({ syncChainTip: 'checkpointed', minTxsPerBlock: 1, maxTxsPerBlock: 2 }); - - // Record the current checkpoint number before starting sequencers - const initialCheckpointNumber = await rollup.getCheckpointNumber(); - logger.warn(`Initial checkpoint number: ${initialCheckpointNumber}`); - - // Pre-prove and send transactions - const txs = await timesAsync(TX_COUNT, i => - proveInteraction(context.wallet, contract.methods.emit_nullifier(new Fr(i + 1)), { from }), - ); - const txHashes = await Promise.all(txs.map(tx => tx.send({ wait: NO_WAIT }))); - logger.warn(`Sent ${txHashes.length} transactions`, { txs: txHashes }); - - // Start the sequencers - await Promise.all(nodes.map(n => n.getSequencer()!.start())); - logger.warn(`Started all sequencers`); - - // Wait until all txs are mined - const timeout = test.L2_SLOT_DURATION_IN_S * 5; - await executeTimeout( - () => Promise.all(txHashes.map(txHash => waitForTx(context.aztecNode, txHash, { timeout }))), - timeout * 1000, - ); - logger.warn(`All txs have been mined`); - - const multiBlockCheckpoint = await assertMultipleBlocksPerSlot(EXPECTED_BLOCKS_PER_CHECKPOINT, logger); - await waitForProvenCheckpoint(multiBlockCheckpoint); - }); - - // Starts sequencers then sends txs one at a time, anchoring each to the proposed block containing - // the previous tx (PXE in 'proposed' mode). Verifies tx anchor block numbers are monotonically - // non-decreasing. Asserts ≥2 blocks per checkpoint and waits for the MBPS checkpoint to be proven. - it('builds multiple blocks per slot with transactions anchored to proposed blocks', async () => { - await setupTest({ syncChainTip: 'proposed', minTxsPerBlock: 1, maxTxsPerBlock: 1 }); - - // Record the current checkpoint number before starting sequencers - const initialCheckpointNumber = await rollup.getCheckpointNumber(); - logger.warn(`Initial checkpoint number: ${initialCheckpointNumber}`); - - // Start the sequencers - await Promise.all(nodes.map(n => n.getSequencer()!.start())); - logger.warn(`Started all sequencers`); - - // Now send the txs and wait for them to be mined one at a time - // If the pxe syncs correctly, every tx should be anchored to the block in which the previous one was mined - const txReceipts = []; - let expectedAnchorBlockNumber = undefined; - - while (txReceipts.length < TX_COUNT / 2) { - logger.warn(`Sending transaction ${txReceipts.length}`); - const nullifier = new Fr(txReceipts.length + 1); - const tx = await proveInteraction(context.wallet, contract.methods.emit_nullifier(nullifier), { from }); - const txAnchorBlockNumber = tx.data.constants.anchorBlockHeader.globalVariables.blockNumber; - expect(txAnchorBlockNumber).toBeGreaterThanOrEqual(expectedAnchorBlockNumber ?? txAnchorBlockNumber); - - const txReceipt = await tx.send({ wait: { waitForStatus: TxStatus.PROPOSED } }); - txReceipts.push(txReceipt); - expectedAnchorBlockNumber = txReceipt.blockNumber; - logger.warn(`Transaction ${txReceipts.length} mined on block ${txReceipt.blockNumber}`, { txReceipt }); - - await wallet.sync(); - expect((await wallet.getSyncedBlockHeader()).getBlockNumber()).toBeGreaterThanOrEqual(txReceipt.blockNumber!); - } - logger.warn(`All txs have been mined`); - - // We are fine with at least 2 blocks per checkpoint, since we may lose one sub-slot if assembling a tx is slow - const multiBlockCheckpoint = await assertMultipleBlocksPerSlot(2, logger); - await waitForProvenCheckpoint(multiBlockCheckpoint); - }); - - // Deploys a cross-chain TestContract, pre-proves TX_COUNT L2→L1 message txs, sends them all, waits - // for all to be mined, then asserts the total L2→L1 message count across all blocks ≥ TX_COUNT, - // a MBPS checkpoint exists, and that checkpoint is proven. - it('builds multiple blocks per slot with L2 to L1 messages', async () => { - await setupTest({ syncChainTip: 'proposed', minTxsPerBlock: 1, maxTxsPerBlock: 2 }); - - // Start sequencers first, then deploy cross-chain contract (needs running sequencer to mine). - await Promise.all(nodes.map(n => n.getSequencer()!.start())); - logger.warn(`Started all sequencers`); - - logger.warn(`Deploying cross-chain test contract`); - ({ contract: crossChainContract } = await TestContract.deploy(wallet).send({ from })); - logger.warn(`Cross-chain test contract deployed at ${crossChainContract!.address}`); - - // Pre-prove all L2→L1 message transactions - const l2ToL1Recipient = EthAddress.fromString(context.deployL1ContractsValues.l1Client.account.address); - logger.warn(`Pre-proving ${TX_COUNT} L2→L1 message transactions`); - const txs = await timesAsync(TX_COUNT, () => - proveInteraction( - wallet, - crossChainContract!.methods.create_l2_to_l1_message_arbitrary_recipient_public(Fr.random(), l2ToL1Recipient), - { from }, - ), - ); - logger.warn(`Pre-proved ${txs.length} L2→L1 message transactions`); - - // Send all transactions at once - const txHashes = await Promise.all(txs.map(tx => tx.send({ wait: NO_WAIT }))); - logger.warn(`Sent ${txHashes.length} L2→L1 message transactions`); - - // Wait until all txs are mined - const timeout = test.L2_SLOT_DURATION_IN_S * 5; - const receipts = await Promise.all(txHashes.map(txHash => waitForTx(context.aztecNode, txHash, { timeout }))); - logger.warn(`All L2→L1 message txs have been mined`); - - // wait for the other node to synch - const maxBlockNumber = Math.max(...receipts.map(r => r.blockNumber!)); - await retryUntil( - async () => - ((await archiver.getBlockNumber({ tag: 'checkpointed' })) ?? 0) >= maxBlockNumber ? true : undefined, - `archiver to checkpoint block ${maxBlockNumber}`, - test.L2_SLOT_DURATION_IN_S * 3, - 0.1, - ); - - // Mirror the sibling MBPS tests: we may lose one sub-slot to pipelined overhead, so accept >= 2 - // blocks per checkpoint rather than the legacy 3-block expectation. - const multiBlockCheckpoint = await assertMultipleBlocksPerSlot(2, logger); - - // Verify L2→L1 messages are in the blocks - const checkpoints = await archiver.getCheckpoints({ from: CheckpointNumber(1), limit: 50 }); - const allBlocks = checkpoints.flatMap(pc => pc.checkpoint.blocks); - const allL2ToL1Messages = allBlocks.flatMap(block => block.body.txEffects.flatMap(txEffect => txEffect.l2ToL1Msgs)); - logger.warn(`Found ${allL2ToL1Messages.length} L2→L1 message(s) across all blocks`, { allL2ToL1Messages }); - expect(allL2ToL1Messages.length).toBeGreaterThanOrEqual(TX_COUNT); - await waitForProvenCheckpoint(multiBlockCheckpoint); - }); - - // Seeds L1→L2 messages, sends filler txs to advance the chain so messages become ready, then - // pre-proves and sends consume txs. Verifies all consume txs are mined, a MBPS checkpoint exists, - // and that checkpoint is proven. - it('builds multiple blocks per slot with L1 to L2 messages', async () => { - // L1→L2 messages only become ready once the chain advances `inboxLag` checkpoints past where they - // were inboxed, and a checkpoint only advances when a block is built in a new slot. With - // skipInitialSequencer the chain won't move on its own, and a one-shot burst of filler txs lands - // within a single checkpoint — so let the sequencer keep building (empty) blocks each slot to drive - // the chain forward until the messages are ready. - await setupTest({ syncChainTip: 'proposed', minTxsPerBlock: 0, maxTxsPerBlock: 1, buildCheckpointIfEmpty: true }); - - // Start sequencers first, then deploy cross-chain contract (needs running sequencer to mine). - await Promise.all(nodes.map(n => n.getSequencer()!.start())); - logger.warn(`Started all sequencers`); - - logger.warn(`Deploying cross-chain test contract`); - ({ contract: crossChainContract } = await TestContract.deploy(wallet).send({ from })); - logger.warn(`Cross-chain test contract deployed at ${crossChainContract!.address}`); - - const L1_TO_L2_COUNT = 4; - const FILLER_TX_COUNT = 5; // Enough txs to advance the chain so messages become ready - - // Seed all L1→L2 messages at the beginning - logger.warn(`Seeding ${L1_TO_L2_COUNT} L1→L2 messages`); - const l1ToL2Messages = await timesAsync(L1_TO_L2_COUNT, async i => { - const [secret, secretHash] = await generateClaimSecret(); - const content = Fr.random(); - const message = { recipient: crossChainContract!.address, content, secretHash }; - - const { msgHash, globalLeafIndex } = await sendL1ToL2Message(message, { - l1Client: context.deployL1ContractsValues.l1Client, - l1ContractAddresses: context.deployL1ContractsValues.l1ContractAddresses, - }); - logger.warn(`L1→L2 message ${i + 1} sent with hash ${msgHash} and index ${globalLeafIndex}`); - - return { content, secret, msgHash, globalLeafIndex }; - }); - logger.warn(`Seeded ${l1ToL2Messages.length} L1→L2 messages`); - - // Pre-prove filler txs (using unique nullifiers to avoid conflicts) - logger.warn(`Pre-proving ${FILLER_TX_COUNT} filler txs to advance the chain`); - const fillerTxs = await timesAsync(FILLER_TX_COUNT, i => - proveInteraction(wallet, contract.methods.emit_nullifier(new Fr(1000 + i)), { from }), - ); - logger.warn(`Pre-proved ${fillerTxs.length} filler txs`); - - // Send all filler txs at once (without waiting for them to be mined) - const fillerTxHashes = await Promise.all(fillerTxs.map(tx => tx.send({ wait: NO_WAIT }))); - logger.warn(`Sent ${fillerTxHashes.length} filler txs`); - - // Wait for filler txs to be mined first - this ensures the chain has advanced enough for messages to be ready - const timeout = test.L2_SLOT_DURATION_IN_S * 5; - await executeTimeout( - () => Promise.all(fillerTxHashes.map(txHash => waitForTx(context.aztecNode, txHash, { timeout }))), - timeout * 1000, - ); - logger.warn(`All filler txs have been mined`); - - // Wait for all messages to be ready in parallel (chain has advanced, messages should be available) - const ethAccount = EthAddress.fromString(context.deployL1ContractsValues.l1Client.account.address); - await Promise.all( - l1ToL2Messages.map(async ({ msgHash }, i) => { - logger.warn(`Waiting for L1→L2 message ${i + 1} to be ready`); - await retryUntil( - () => isL1ToL2MessageReady(context.aztecNode, msgHash), - `L1→L2 message ${i + 1} ready`, - test.L2_SLOT_DURATION_IN_S * 5, - ); - logger.warn(`L1→L2 message ${i + 1} is ready`); - }), - ); - logger.warn(`All ${l1ToL2Messages.length} L1→L2 messages are ready`); - - // Pre-prove all consume transactions (to avoid nonce conflicts when sending in parallel) - logger.warn(`Pre-proving ${l1ToL2Messages.length} consume transactions`); - const consumeTxs = await timesAsync(l1ToL2Messages.length, i => { - const { content, secret, globalLeafIndex } = l1ToL2Messages[i]; - return proveInteraction( - wallet, - crossChainContract!.methods.consume_message_from_arbitrary_sender_public( - content, - secret, - ethAccount, - globalLeafIndex, - ), - { from }, - ); - }); - logger.warn(`Pre-proved ${consumeTxs.length} consume transactions`); - - // Send all consume transactions at once - const consumeTxHashes = await Promise.all(consumeTxs.map(tx => tx.send({ wait: NO_WAIT }))); - logger.warn(`Sent ${consumeTxHashes.length} consume transactions`); - - // Wait for all consume txs to be mined - await Promise.all(consumeTxHashes.map(txHash => waitForTx(context.aztecNode, txHash, { timeout }))); - logger.warn(`All ${consumeTxHashes.length} L1→L2 messages consumed`); - - const multiBlockCheckpoint = await assertMultipleBlocksPerSlot(2, logger); - await waitForProvenCheckpoint(multiBlockCheckpoint); - }); - - // Creates an extra non-validator node with alwaysReexecuteBlockProposals=true, sends txs, and - // waits until that node has stored a multi-block proposed slot (≥2 blocks) beyond its checkpointed - // tip. Verifies block effects are valid, then starts a second sync-only node and confirms it - // syncs the multi-block slot from scratch. - it('builds multiple blocks per slot and non-validators re-execute and sync multi-block slots', async () => { - await setupTest({ syncChainTip: 'proposed', minTxsPerBlock: 1, maxTxsPerBlock: 1 }); - - logger.warn(`Creating non-validator reexecuting node`); - const nonValidatorNode = await test.createNonValidatorNode({ - alwaysReexecuteBlockProposals: true, - skipPushProposedBlocksToArchiver: false, - }); - - await Promise.all(nodes.map(n => n.getSequencer()!.start())); - logger.warn(`Started all sequencers`); - - logger.warn(`Pre-proving ${TX_COUNT / 2} transactions`); - const txs = await timesAsync(TX_COUNT / 2, i => { - const nullifier = new Fr(i + 100); - return proveInteraction(context.wallet, contract.methods.emit_nullifier(nullifier), { from }); - }); - logger.warn(`Pre-proved ${txs.length} transactions`); - - const sentTxHashes = await Promise.all(txs.map(tx => tx.send({ wait: NO_WAIT }))); - logger.warn(`Sent ${sentTxHashes.length} transactions`); - - const nonValidatorArchiver = nonValidatorNode.getBlockSource(); - - let multiBlockSlotNumber: number | undefined; - let checkpointedBlockNumber: number | undefined; - await retryUntil( - async () => { - const tips = await nonValidatorArchiver.getL2Tips(); - if (tips.proposed.number <= tips.checkpointed.block.number) { - return false; - } - const blockData = await nonValidatorArchiver.getBlockData({ number: tips.proposed.number }); - if (!blockData) { - return false; - } - const blocksInSlot = await nonValidatorArchiver.getBlocksForSlot(blockData.header.globalVariables.slotNumber); - if (blocksInSlot.length < 2) { - return false; - } - multiBlockSlotNumber = blockData.header.globalVariables.slotNumber; - checkpointedBlockNumber = tips.checkpointed.block.number; - return true; - }, - 'non-validator node to store multi-block proposed slot', - test.L2_SLOT_DURATION_IN_S * 5, - 0.5, - ); - - // Ensure the proposed multi-block slot has valid effects - expect(multiBlockSlotNumber).toBeDefined(); - const blocksInSlot = await nonValidatorArchiver.getBlocksForSlot(SlotNumber(multiBlockSlotNumber!)); - expect(blocksInSlot.length).toBeGreaterThanOrEqual(2); - expect(checkpointedBlockNumber).toBeDefined(); - expect(blocksInSlot.every(block => block.number > checkpointedBlockNumber!)).toBe(true); // ensure the block is proposed - const txHashesInSlot = blocksInSlot.flatMap(block => block.body.txEffects.map(effect => effect.txHash)); - expect(txHashesInSlot.length).toBeGreaterThan(0); - const effectsInSlot = await Promise.all(txHashesInSlot.map(txHash => nonValidatorArchiver.getTxEffect(txHash))); - expect(effectsInSlot.every(effect => effect !== undefined)).toBe(true); - - // Wait until the node syncs to the checkpointed block successfully - const maxBlockNumberInSlot = Math.max(...blocksInSlot.map(block => block.number)); - await retryUntil( - async () => (await nonValidatorArchiver.getL2Tips()).checkpointed.block.number >= maxBlockNumberInSlot!, - 'non-validator node to sync checkpointed block', - test.L2_SLOT_DURATION_IN_S * 5, - 0.5, - ); - - // Start a new node an make sure it can sync from scratch including the multi-block slot - logger.warn(`Creating non-validator syncing node`); - const nonValidatorSyncingNode = await test.createNonValidatorNode({ - alwaysReexecuteBlockProposals: false, - }); - await retryUntil( - async () => - (await nonValidatorSyncingNode.getBlockSource().getL2Tips()).checkpointed.block.number >= maxBlockNumberInSlot!, - 'non-validator syncing node to sync checkpointed block', - test.L2_SLOT_DURATION_IN_S * 10, - 0.5, - ); - - const multiBlockCheckpoint = await assertMultipleBlocksPerSlot(2, logger); - await waitForProvenCheckpoint(multiBlockCheckpoint); - }); - - // Pre-proves a high-priority deploy tx and a low-priority call tx for the same contract. Waits - // until just before the next L2 slot boundary, sends deploy first (then call after 1s), and - // waits for both to be checkpointed. Asserts deploy block < call block and both belong to the - // same checkpoint. Waits for that checkpoint to be proven. - it('deploys a contract and calls it in separate blocks within a slot', async () => { - await setupTest({ - syncChainTip: 'checkpointed', - minTxsPerBlock: 1, - maxTxsPerBlock: 1, - }); - - // Prepare deploy tx for a new TestContract. Get the instance address so we can construct the call tx. - const highPriority = new GasFees(100, 100); - const lowPriority = new GasFees(1, 1); - - const deployMethod = TestContract.deploy(wallet, { deployer: from }); - const deployInstance = await deployMethod.getInstance(); - logger.warn(`Will deploy TestContract at ${deployInstance.address}`); - - // Register the contract on the PXE so we can prove the call interaction against it. - await wallet.registerContract(deployInstance, TestContract.artifact); - const deployedContract = TestContract.at(deployInstance.address, wallet); - - // Pre-prove both txs before starting sequencers. This ensures both arrive in the pool - // at the same time, so the sequencer can sort by priority fee for correct ordering. - logger.warn(`Pre-proving deploy tx (high priority) and call tx (low priority)`); - const deployTx = await proveInteraction(wallet, deployMethod, { - from, - fee: { gasSettings: { maxPriorityFeesPerGas: highPriority } }, - }); - const callTx = await proveInteraction(wallet, deployedContract.methods.emit_nullifier_public(new Fr(42)), { - from, - fee: { gasSettings: { maxPriorityFeesPerGas: lowPriority } }, - }); - logger.warn(`Pre-proved both txs`); - - // Start the sequencers - await Promise.all(nodes.map(n => n.getSequencer()!.start())); - logger.warn(`Started all sequencers`); - - // Wait until one L1 slot before the start of the next L2 slot. - // This ensures both txs land in the pending pool right before the proposer starts building. - // REFACTOR: manual slot-timing arithmetic and waitUntilL1Timestamp call; replace with a helper - // such as test.waitUntilBuildWindowForNextSlot() that encapsulates this pattern. - // REFACTOR: This should go into a shared "waitUntilNextSlotStartsBuilding" utility - const currentL1Block = await test.l1Client.getBlock({ blockTag: 'latest' }); - const currentTimestamp = currentL1Block.timestamp; - const currentSlot = getSlotAtTimestamp(currentTimestamp, test.constants); - const nextSlot = SlotNumber(currentSlot + 1); - const nextSlotTimestamp = getTimestampForSlot(nextSlot, test.constants); - const targetTimestamp = nextSlotTimestamp - BigInt(test.L1_BLOCK_TIME_IN_S); - logger.warn(`Waiting until L1 timestamp ${targetTimestamp} (one L1 slot before L2 slot ${nextSlot})`, { - currentTimestamp, - currentSlot, - nextSlot, - nextSlotTimestamp, - targetTimestamp, - }); - await waitUntilL1Timestamp(test.l1Client, targetTimestamp, undefined, test.L2_SLOT_DURATION_IN_S * 3); - - // Send the deploy tx first and give it time to propagate to all validators, - // then send the call tx. Priority fees are a safety net, but arrival ordering - // ensures the deploy tx is in the pool before the call tx regardless of gossip timing. - const timeout = test.L2_SLOT_DURATION_IN_S * 5; - logger.warn(`Sending deploy tx first, then call tx`); - const deployTxHash = await deployTx.send({ wait: NO_WAIT }); - await sleep(1000); - const callTxHash = await callTx.send({ wait: NO_WAIT }); - const [deployReceipt, callReceipt] = await executeTimeout( - () => - Promise.all([ - waitForTx(context.aztecNode, deployTxHash, { timeout }), - waitForTx(context.aztecNode, callTxHash, { timeout }), - ]), - timeout * 1000, - ); - logger.warn(`Both txs checkpointed`, { - deployBlock: deployReceipt.blockNumber, - callBlock: callReceipt.blockNumber, - }); - - // Both txs should succeed (send throws on revert). Deploy should be in an earlier block. - expect(deployReceipt.blockNumber).toBeLessThan(callReceipt.blockNumber!); - - // Verify both blocks belong to the same checkpoint. - const deployCheckpointedBlock = await retryUntil( - async () => - ( - await context.aztecNode.getBlocks(deployReceipt.blockNumber!, 1, { - includeL1PublishInfo: true, - includeAttestations: true, - onlyCheckpointed: true, - }) - )[0], - 'deploy checkpointed block', - timeout, - ); - const callCheckpointedBlock = await retryUntil( - async () => - ( - await context.aztecNode.getBlocks(callReceipt.blockNumber!, 1, { - includeL1PublishInfo: true, - includeAttestations: true, - onlyCheckpointed: true, - }) - )[0], - 'call checkpointed block', - timeout, - ); - expect(deployCheckpointedBlock.checkpointNumber).toBe(callCheckpointedBlock.checkpointNumber); - logger.warn(`Both blocks in checkpoint ${deployCheckpointedBlock.checkpointNumber}`); - - // Wait for the checkpoint to be proven. - await waitForProvenCheckpoint(deployCheckpointedBlock.checkpointNumber); - }); -}); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_mbps.pipeline.parallel.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_mbps.pipeline.parallel.test.ts deleted file mode 100644 index 2d3a78d39103..000000000000 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_mbps.pipeline.parallel.test.ts +++ /dev/null @@ -1,419 +0,0 @@ -import type { Archiver } from '@aztec/archiver'; -import type { AztecNodeService } from '@aztec/aztec-node'; -import { AztecAddress, EthAddress } from '@aztec/aztec.js/addresses'; -import { NO_WAIT } from '@aztec/aztec.js/contracts'; -import { Fr } from '@aztec/aztec.js/fields'; -import type { Logger } from '@aztec/aztec.js/log'; -import { waitForTx } from '@aztec/aztec.js/node'; -import type { EpochCacheInterface } from '@aztec/epoch-cache'; -import { RollupContract } from '@aztec/ethereum/contracts'; -import type { Operator } from '@aztec/ethereum/deploy-aztec-l1-contracts'; -import { asyncMap } from '@aztec/foundation/async-map'; -import { BlockNumber, CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types'; -import { times, timesAsync } from '@aztec/foundation/collection'; -import { SecretValue } from '@aztec/foundation/config'; -import { retryUntil } from '@aztec/foundation/retry'; -import { bufferToHex } from '@aztec/foundation/string'; -import { executeTimeout } from '@aztec/foundation/timer'; -import { TestContract } from '@aztec/noir-test-contracts.js/Test'; -import type { SequencerEvents } from '@aztec/sequencer-client'; -import { L2BlockSourceEvents } from '@aztec/stdlib/block'; -import { getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; - -import { jest } from '@jest/globals'; -import { privateKeyToAccount } from 'viem/accounts'; - -import { type EndToEndContext, getPrivateKeyFromIndex } from '../fixtures/utils.js'; -import { TestWallet } from '../test-wallet/test_wallet.js'; -import { proveInteraction } from '../test-wallet/utils.js'; -import { EpochsTestContext } from './epochs_test.js'; - -jest.setTimeout(1000 * 60 * 20); - -const NODE_COUNT = 4; -const EXPECTED_BLOCKS_PER_CHECKPOINT = 8; - -// Send enough transactions to trigger multiple blocks within a checkpoint assuming 2 txs per block. -const TX_COUNT = 34; - -/** - * E2E tests for proposer pipelining with Multiple Blocks Per Slot (MBPS). - * Verifies that the block proposer in slot N is the validator scheduled on L1 for slot N+1 - * (the proposer view uses a +1 slot offset). - * - * Four-validator suite with a prover node (fake proofs) and 500ms mock gossip latency to simulate - * adverse network conditions. Two tests: (1) normal pipelining flow asserting build-vs-submission - * slot offsets and blob-fetch promotion; (2) a proposer skips its checkpoint publish, triggering an - * uncheckpointed-blocks prune followed by recovery. Uses EpochsTestContext with mockGossipSubNetwork - * and no initial sequencer. - */ -describe('e2e_epochs/epochs_mbps_pipeline', () => { - let context: EndToEndContext; - let logger: Logger; - let rollup: RollupContract; - let archiver: Archiver; - - let test: EpochsTestContext; - let validators: (Operator & { privateKey: `0x${string}` })[]; - let nodes: AztecNodeService[]; - let contract: TestContract; - let wallet: TestWallet; - let from: AztecAddress; - - /** Creates validators and sets up the test context with MBPS and proposer pipelining. */ - async function setupTest(opts: { - syncChainTip: 'proposed' | 'checkpointed'; - minTxsPerBlock?: number; - maxTxsPerBlock?: number; - }) { - const { syncChainTip = 'checkpointed', ...setupOpts } = opts; - - validators = times(NODE_COUNT, i => { - const privateKey = bufferToHex(getPrivateKeyFromIndex(i + 3)!); - const attester = EthAddress.fromString(privateKeyToAccount(privateKey).address); - return { attester, withdrawer: attester, privateKey, bn254SecretKey: new SecretValue(Fr.random().toBigInt()) }; - }); - - test = await EpochsTestContext.setup({ - numberOfAccounts: 0, - initialValidators: validators, - mockGossipSubNetwork: true, - mockGossipSubNetworkLatency: 500, // adverse network conditions - startProverNode: true, - perBlockAllocationMultiplier: 8, - aztecEpochDuration: 4, - ethereumSlotDuration: 12, - aztecSlotDuration: 72, - blockDurationMs: 5500, - maxTxsPerCheckpoint: 24, - aztecTargetCommitteeSize: 3, - inboxLag: 2, - ...setupOpts, - pxeOpts: { syncChainTip }, - skipInitialSequencer: true, - }); - - ({ context, logger, rollup } = test); - wallet = context.wallet; - from = context.accounts[0]; // auto-created by setup - - logger.warn(`Initial setup complete. Starting ${NODE_COUNT} validator nodes.`); - // Clear inherited coinbase so each validator derives coinbase from its own attester key - nodes = await asyncMap(validators, ({ privateKey }, i) => - test.createValidatorNode([privateKey], { - dontStartSequencer: true, - coinbase: undefined, - // Disable checkpoint promotion on the first node so it always fetches blobs, - // allowing us to assert that other nodes skip blob fetching via promotion. - ...(i === 0 ? { skipPromoteProposedCheckpointDuringL1Sync: true } : {}), - }), - ); - logger.warn(`Started ${NODE_COUNT} validator nodes.`, { validators: validators.map(v => v.attester.toString()) }); - - wallet.updateNode(nodes[0]); - archiver = nodes[0].getBlockSource() as Archiver; - - contract = await test.registerTestContract(wallet); - logger.warn(`Test setup completed.`, { validators: validators.map(v => v.attester.toString()) }); - } - - /** - * Waits until the archiver's checkpointed chain tip has reached `targetBlockNumber`, then retrieves all checkpoints, - * checks that one has the target block count, and returns its number. - */ - async function assertMultipleBlocksPerSlot( - targetBlockCount: number, - targetBlockNumber: BlockNumber, - logger: Logger, - ): Promise { - await retryUntil( - async () => { - const checkpointed = await archiver.getBlockNumber({ tag: 'checkpointed' }); - return checkpointed !== undefined && checkpointed >= targetBlockNumber; - }, - `archiver checkpointed block ${targetBlockNumber}`, - 10, - 0.1, - ); - - const checkpoints = await archiver.getCheckpoints({ from: CheckpointNumber(1), limit: 50 }); - logger.warn(`Retrieved ${checkpoints.length} checkpoints from archiver`, { - checkpoints: checkpoints.map(pc => pc.checkpoint.getStats()), - }); - - let expectedBlockNumber = checkpoints[0].checkpoint.blocks[0].number; - let multiBlockCheckpointNumber: CheckpointNumber | undefined; - - for (const checkpoint of checkpoints) { - const blockCount = checkpoint.checkpoint.blocks.length; - if (blockCount >= targetBlockCount && multiBlockCheckpointNumber === undefined) { - multiBlockCheckpointNumber = checkpoint.checkpoint.number; - } - logger.warn(`Checkpoint ${checkpoint.checkpoint.number} has ${blockCount} blocks`, { - checkpoint: checkpoint.checkpoint.getStats(), - }); - - for (let i = 0; i < blockCount; i++) { - const block = checkpoint.checkpoint.blocks[i]; - expect(block.indexWithinCheckpoint).toBe(i); - expect(block.checkpointNumber).toBe(checkpoint.checkpoint.number); - expect(block.number).toBe(expectedBlockNumber); - expectedBlockNumber++; - } - } - - expect(multiBlockCheckpointNumber).toBeDefined(); - return multiBlockCheckpointNumber!; - } - - /** Waits until a specific multi-block checkpoint is proven. */ - async function waitForProvenCheckpoint(targetCheckpoint: CheckpointNumber) { - const provenTimeout = test.L2_SLOT_DURATION_IN_S * test.epochDuration * 4; - logger.warn(`Waiting for checkpoint ${targetCheckpoint} to be proven (timeout=${provenTimeout}s)`); - await test.waitUntilProvenCheckpointNumber(targetCheckpoint, provenTimeout); - logger.warn(`Proven checkpoint advanced to ${test.monitor.provenCheckpointNumber}`); - } - - /** - * Asserts pipelining by comparing the build slot (from block-proposed events) against - * the submission slot (from block headers). With pipelining, the block is built in slot N - * but its header carries submission slot N+1. - */ - async function assertProposerPipelining( - blockProposedEvents: { blockNumber: BlockNumber; slot: SlotNumber; buildSlot: SlotNumber }[], - logger: Logger, - ) { - const checkpoints = await archiver.getCheckpoints({ from: CheckpointNumber(1), limit: 50 }); - const allBlocks = checkpoints.flatMap(pc => pc.checkpoint.blocks); - - logger.warn(`assertProposerPipelining: ${allBlocks.length} blocks, ${blockProposedEvents.length} events`, { - blockNumbers: allBlocks.map(b => b.number), - eventBlockNumbers: blockProposedEvents.map(e => e.blockNumber), - }); - - let foundPipelining = false; - - for (const block of allBlocks) { - const headerSlot = block.header.globalVariables.slotNumber; // submission slot (N+1) - const coinbase = block.header.globalVariables.coinbase; - - // Find the block-proposed event for this block (use Number() for safe comparison) - const event = blockProposedEvents.find(e => Number(e.blockNumber) === Number(block.number)); - // if there is no event, then it was probably block number one - which was proposed in setup - if (!event) { - continue; - } - - const buildSlot = event.buildSlot; // build slot (N) - - // Verify the pipelining offset: block built in slot N, submitted in slot N+1 - expect(Number(headerSlot)).toBe(Number(buildSlot) + 1); - foundPipelining = true; - - // Verify coinbase matches the expected proposer for the submission slot - const expectedProposer = await rollup.getProposerAt(getTimestampForSlot(headerSlot, test.constants)); - expect(coinbase).toEqual(expectedProposer); - - logger.warn(`Block ${block.number}: buildSlot=${buildSlot}, submissionSlot=${headerSlot}, coinbase=${coinbase}`, { - blockNumber: block.number, - buildSlot, - headerSlot, - coinbase: coinbase.toString(), - expectedProposer: expectedProposer.toString(), - }); - } - - expect(foundPipelining).toBe(true); - logger.warn(`Pipelining assertion passed for ${allBlocks.length} blocks`); - } - - afterEach(async () => { - jest.restoreAllMocks(); - await test?.teardown(); - }); - - // Pre-proves TX_COUNT txs, starts sequencers, waits for all txs to be mined. Asserts a - // MBPS checkpoint with ≥EXPECTED_BLOCKS_PER_CHECKPOINT blocks. Asserts every block's header - // slot equals build-slot+1 (pipelining offset). Verifies node-0 fetches blobs (promotion - // disabled) while nodes 1-3 skip blob fetching (promotion enabled). Waits for the checkpoint - // to be proven. - it('pipelining builds blocks using slot plus 1 proposer and proves them', async () => { - await setupTest({ syncChainTip: 'checkpointed', minTxsPerBlock: 1, maxTxsPerBlock: 2 }); - - // Spy on getBlobSidecar on all validator nodes before sequencers start, so we check that nodes - // promote their proposed checkpoints and don't source data from blobs if they don't need to. - const blobSpies = nodes.map((node, i) => { - const blobClient = node.getBlobClient()!; - const spy = jest.spyOn(blobClient, 'getBlobSidecar'); - logger.warn(`Installed getBlobSidecar spy on validator node ${i}`); - return spy; - }); - - // Subscribe to block-proposed events to capture build slots - const blockProposedEvents: { blockNumber: BlockNumber; slot: SlotNumber; buildSlot: SlotNumber }[] = []; - const sequencers = nodes.map(n => n.getSequencer()!); - for (const sequencer of sequencers) { - sequencer.getSequencer().on('block-proposed', (args: Parameters[0]) => { - logger.warn(`block-proposed event: blockNumber=${args.blockNumber}, slot=${args.slot}`, args); - blockProposedEvents.push({ - blockNumber: args.blockNumber, - slot: args.slot, - buildSlot: args.buildSlot, - }); - }); - } - - const initialCheckpointNumber = await rollup.getCheckpointNumber(); - logger.warn(`Initial checkpoint number: ${initialCheckpointNumber}`); - - // Pre-prove and send transactions - const txs = await timesAsync(TX_COUNT, i => - proveInteraction(context.wallet, contract.methods.emit_nullifier(new Fr(i + 1)), { from }), - ); - const txHashes = await Promise.all(txs.map(tx => tx.send({ wait: NO_WAIT }))); - logger.warn(`Sent ${txHashes.length} transactions`, { txs: txHashes }); - - // Start the sequencers - await Promise.all(sequencers.map(s => s.start())); - logger.warn(`Started all sequencers`); - - // Wait until all txs are mined - const timeout = test.L2_SLOT_DURATION_IN_S * 5; - const receipts = await executeTimeout( - () => Promise.all(txHashes.map(txHash => waitForTx(context.aztecNode, txHash, { timeout }))), - timeout * 1000, - ); - logger.warn(`All txs have been mined`); - - // Verify MBPS works with pipelining; target the highest block number across mined receipts - const maxMinedBlockNumber = BlockNumber(Math.max(...receipts.map(r => r.blockNumber ?? 0))); - const multiBlockCheckpoint = await assertMultipleBlocksPerSlot( - EXPECTED_BLOCKS_PER_CHECKPOINT, - maxMinedBlockNumber, - logger, - ); - - // Verify the pipelining offset: build slot N vs submission slot N+1 - await assertProposerPipelining(blockProposedEvents, logger); - - // Verify blob fetching behavior: node 0 has promotion disabled so it must fetch blobs, - // while all other nodes should promote their proposed checkpoints and skip blob fetching entirely. - for (let i = 0; i < blobSpies.length; i++) { - const calls = blobSpies[i].mock.calls.length; - logger.warn(`Validator ${i} made ${calls} getBlobSidecar calls`); - if (i === 0) { - expect(calls).toBeGreaterThan(0); - } else { - expect(calls).toBe(0); - } - } - - // Verify proving still works end-to-end with pipelined proposers - await waitForProvenCheckpoint(multiBlockCheckpoint); - }); - - // Establishes a baseline at checkpoint 1. Identifies the next proposer and disables its - // checkpoint publishing. Waits for the L2PruneUncheckpointed event on the archiver, then - // re-enables publishing. Waits for all txs to be mined, asserts a MBPS checkpoint exists, - // verifies the pipelining offset, and checks recovery blockNumber > baseline. - it('prunes uncheckpointed blocks when proposer fails to deliver', async () => { - await setupTest({ syncChainTip: 'checkpointed', minTxsPerBlock: 1, maxTxsPerBlock: 2 }); - - const blockProposedEvents: { blockNumber: BlockNumber; slot: SlotNumber; buildSlot: SlotNumber }[] = []; - const sequencers = nodes.map(n => n.getSequencer()!); - - // Pre-prove and send transactions - const txs = await timesAsync(TX_COUNT, i => - proveInteraction(context.wallet, contract.methods.emit_nullifier(new Fr(i + 1)), { from }), - ); - const txHashes = await Promise.all(txs.map(tx => tx.send({ wait: NO_WAIT }))); - logger.warn(`Sent ${txHashes.length} transactions`, { txs: txHashes }); - - await Promise.all(sequencers.map(s => s.start())); - logger.warn(`Started all sequencers`); - - // Assert that at least 1 checkpoint has been reached - const checkpointTimeout = test.L2_SLOT_DURATION_IN_S * test.epochDuration * 3; - await test.waitUntilCheckpointNumber(CheckpointNumber(1), checkpointTimeout); - const checkpointedBlockNumber = await archiver.getBlockNumber(); - logger.warn(`Baseline established: checkpoint 1 reached at block ${checkpointedBlockNumber}`); - // Target a submission slot whose pipelined build has not started yet. - const { slot: currentSlot } = test.epochCache.getEpochAndSlotNow(); - const { proposerIndex, slot: proposerSlotToNotPublish } = await findNextProposerIndex( - test.epochCache, - validators, - SlotNumber(currentSlot + 2), - ); - logger.warn( - `Will skip checkpoint publishing for proposer ${proposerIndex} in slot ${proposerSlotToNotPublish} - current slot ${currentSlot}`, - ); - - const targetSequencer = nodes[proposerIndex].getSequencer(); - if (!targetSequencer) { - throw new Error('Target proposer sequencer not found'); - } - // Subscribe to prune event BEFORE disabling publishing, so we don't miss the event - const prunePromise = new Promise(resolve => { - archiver.events.once(L2BlockSourceEvents.L2PruneUncheckpointed, () => resolve()); - }); - - // The sequencer keeps building blocks and broadcasting via P2P, but won't submit the checkpoint to L1 - targetSequencer.updateConfig({ skipPublishingCheckpointsPercent: 100 }); - - const pruneTimeout = test.L2_SLOT_DURATION_IN_S * 5 * 1000; - logger.warn(`Waiting for uncheckpointed blocks to be pruned (timeout=${pruneTimeout}ms)`); - await executeTimeout(() => prunePromise, pruneTimeout); - - // add block proposed listeners after the prune - for (const sequencer of sequencers) { - sequencer.getSequencer().on('block-proposed', (args: Parameters[0]) => { - logger.warn(`block-proposed event: blockNumber=${args.blockNumber}, slot=${args.slot}`, args); - blockProposedEvents.push({ - blockNumber: args.blockNumber, - slot: args.slot, - buildSlot: args.buildSlot, - }); - }); - } - logger.warn(`Pruning detected, block number now ${await archiver.getBlockNumber()}`); - - // Re-enable checkpoint publishing - logger.warn(`Re-enabling checkpoint publishing for validator ${proposerIndex}`); - targetSequencer.updateConfig({ skipPublishingCheckpointsPercent: 0 }); - - // Wait for a new checkpoint (recovery) - where all txs end up mined - const timeout = test.L2_SLOT_DURATION_IN_S * 5; - const receipts = await executeTimeout( - () => Promise.all(txHashes.map(txHash => waitForTx(context.aztecNode, txHash, { timeout }))), - timeout * 1000, - ); - logger.warn(`All txs have been mined`); - - // Verify MBPS works with pipelining; target the highest block number across mined receipts - const maxMinedBlockNumber = BlockNumber(Math.max(...receipts.map(r => r.blockNumber ?? 0))); - await assertMultipleBlocksPerSlot(EXPECTED_BLOCKS_PER_CHECKPOINT, maxMinedBlockNumber, logger); - - // Verify the pipelining offset: build slot N vs submission slot N+1 - await assertProposerPipelining(blockProposedEvents, logger); - - const recoveredBlockNumber = await archiver.getBlockNumber(); - logger.warn(`Recovery complete: block number ${recoveredBlockNumber} > ${checkpointedBlockNumber}`); - expect(recoveredBlockNumber).toBeGreaterThan(checkpointedBlockNumber); - }); -}); - -/** Scans upcoming slots to find which validator proposes next and returns its index. */ -async function findNextProposerIndex( - epochCache: EpochCacheInterface, - validators: { attester: EthAddress }[], - slotToDisable: SlotNumber, -): Promise<{ proposerIndex: number; slot: SlotNumber }> { - const proposer = await epochCache.getProposerAttesterAddressInSlot(SlotNumber(slotToDisable)); - if (proposer) { - const idx = validators.findIndex(v => v.attester.equals(proposer)); - if (idx >= 0) { - return { proposerIndex: idx, slot: SlotNumber(slotToDisable) }; - } - } - throw new Error(`No proposer found in slot ${slotToDisable}`); -} diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_missed_l1_publish.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_missed_l1_publish.test.ts deleted file mode 100644 index de2e68d376c7..000000000000 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_missed_l1_publish.test.ts +++ /dev/null @@ -1,397 +0,0 @@ -import type { Archiver } from '@aztec/archiver'; -import type { AztecNodeService } from '@aztec/aztec-node'; -import { EthAddress } from '@aztec/aztec.js/addresses'; -import { Fr } from '@aztec/aztec.js/fields'; -import type { Logger } from '@aztec/aztec.js/log'; -import { waitUntilL1Timestamp } from '@aztec/ethereum/l1-tx-utils'; -import { asyncMap } from '@aztec/foundation/async-map'; -import { BlockNumber, SlotNumber } from '@aztec/foundation/branded-types'; -import { times } from '@aztec/foundation/collection'; -import { SecretValue } from '@aztec/foundation/config'; -import { retryUntil } from '@aztec/foundation/retry'; -import { bufferToHex } from '@aztec/foundation/string'; -import { timeoutPromise } from '@aztec/foundation/timer'; -import { type L2Block, L2BlockSourceEvents, type L2Tips } from '@aztec/stdlib/block'; -import { getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; - -import { jest } from '@jest/globals'; -import { privateKeyToAccount } from 'viem/accounts'; - -import { getPrivateKeyFromIndex } from '../fixtures/utils.js'; -import { EpochsTestContext } from './epochs_test.js'; - -jest.setTimeout(1000 * 60 * 15); - -const NODE_COUNT = 4; - -/** - * E2E test for the "missed L1 publish" scenario under proposer pipelining. - * - * Each of 4 nodes holds exactly one validator key. We pick four consecutive slots - * (slotZero, slotOne, slotTwo, slotThree) such that the proposers for slotOne, slotTwo, and - * slotThree are three distinct validators, then warp to one L1 block before slotZero begins. - * The proposer for slotOne is configured to skip its L1 publish. - * - * With pipelining, the proposer for slot N+1 builds and gossips its checkpoint during slot N, - * then publishes that checkpoint to L1 during slot N+1. So gossip-driven `proposed` chain - * advances arrive one slot earlier than the L1-driven `checkpointed` advance. - * - * Expected behavior: - * - During slotZero, the pipelined proposer for slotOne gossips its build → every node's - * `proposed` tip advances to a block at slotOne. - * - During slotOne, the pipelined proposer for slotTwo gossips on top of the slotOne proposal → - * `proposed` advances to a block at slotTwo. Meanwhile the proposer for slotOne attempts L1 - * publish but is configured to skip it, so no checkpoint lands. - * - When slotOne ends with no checkpoint mined, every node's archiver prunes the - * uncheckpointed slotOne and slotTwo blocks; we verify rollback via the prune event. - * We then re-enable publishing on the formerly suppressed node so recovery can proceed. - * - During slotTwo, the pipelined proposer for slotThree builds on top of the (now genesis) - * checkpointed tip → `proposed` advances again. - * - During slotThree, that pipelined work is published → `checkpointed` finally advances. - * - * Uses EpochsTestContext with mockGossipSubNetwork, no initial sequencer, and no prover node. - */ -describe('e2e_epochs/epochs_missed_l1_publish', () => { - let logger: Logger; - let test: EpochsTestContext; - let nodes: AztecNodeService[]; - - afterEach(async () => { - jest.restoreAllMocks(); - await test?.teardown(); - }); - - // Searches for slotOne..slotThree with three distinct proposers (warp on EpochNotStable). Sets - // skipPublishingCheckpointsPercent=100 on proposerOne's node. Warps L1 to slotZero-1 L1 block. - // Subscribes to prune events on all nodes. Starts all sequencers and verifies: proposed tip - // reaches slotOne then slotTwo; all nodes emit L2PruneUncheckpointed at slotOne end; recovery - // produces a checkpointed block at slotThree. Sanity-checks no unexpected fail events. - it('all nodes prune and recover when proposer fails to publish to L1', async () => { - // Build 4 distinct validators (V1..V4). One key per node, no overlap. - const validators = times(NODE_COUNT, i => { - const privateKey = bufferToHex(getPrivateKeyFromIndex(i + 3)!); - const attester = EthAddress.fromString(privateKeyToAccount(privateKey).address); - return { attester, withdrawer: attester, privateKey, bn254SecretKey: new SecretValue(Fr.random().toBigInt()) }; - }); - - test = await EpochsTestContext.setup({ - numberOfAccounts: 0, - initialValidators: validators, - inboxLag: 2, - mockGossipSubNetwork: true, - startProverNode: false, - aztecEpochDuration: 4, - aztecProofSubmissionEpochs: 1024, - ethereumSlotDuration: 6, - aztecSlotDuration: 36, - blockDurationMs: 8000, - attestationPropagationTime: 0.5, - aztecTargetCommitteeSize: NODE_COUNT, - skipInitialSequencer: true, - }); - - logger = test.logger; - - // One node per validator. dontStartSequencer until after the warp so timing is deterministic. - nodes = await asyncMap(validators, ({ privateKey }, i) => - test.createValidatorNode([privateKey], { - dontStartSequencer: true, - coinbase: EthAddress.fromNumber(0xa + i), - buildCheckpointIfEmpty: true, - minTxsPerBlock: 0, - }), - ); - - const attesterAddresses = validators.map(v => v.attester); - logger.warn('Validator nodes created', { - validators: attesterAddresses.map((a, i) => ({ idx: i, attester: a.toString() })), - }); - - // Find slotOne (>=4 ahead) such that proposers for slotOne, slotTwo, slotThree are three - // distinct validators. The +4 margin (vs +2 in equivocation) gives the warp+sequencer-start - // path enough headroom to reach the build window for slotZero even if node creation jitters. - // - // The L1 rollup contract only exposes proposers for epochs whose randao seed is "stable" - // (i.e. queryable on L1 right now). When we look too far into the future the contract - // reverts with `ValidatorSelection__EpochNotStable`. We handle this by warping L1 forward - // one epoch at a time and retrying — after each warp the previously-unstable epoch becomes - // queryable, and we bump the candidate to keep the +4 slot margin from the new "now". - // REFACTOR: hand-rolled slot-search with EpochNotStable warp fallback looking for three - // consecutive distinct-proposer slots; replace with a shared helper such as - // findConsecutiveSlotsWithDistinctProposers(test, fromSlot, count) that encapsulates this pattern. - let slotOne: SlotNumber | undefined; - let proposerOne: EthAddress | undefined; - let proposerTwo: EthAddress | undefined; - let proposerThree: EthAddress | undefined; - let candidate = Number(test.epochCache.getEpochAndSlotNow().slot) + 4; - const maxAttempts = 200; - for (let attempt = 0; attempt < maxAttempts && slotOne === undefined; attempt++) { - try { - const [p1, p2, p3] = await Promise.all([ - test.epochCache.getProposerAttesterAddressInSlot(SlotNumber(candidate)), - test.epochCache.getProposerAttesterAddressInSlot(SlotNumber(candidate + 1)), - test.epochCache.getProposerAttesterAddressInSlot(SlotNumber(candidate + 2)), - ]); - if (p1 && p2 && p3 && !p1.equals(p2) && !p1.equals(p3) && !p2.equals(p3)) { - slotOne = SlotNumber(candidate); - proposerOne = p1; - proposerTwo = p2; - proposerThree = p3; - break; - } - candidate++; - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - if (!msg.includes('EpochNotStable')) { - throw err; - } - const block = await test.l1Client.getBlock({ includeTransactions: false }); - const warpBy = test.epochDuration * test.L2_SLOT_DURATION_IN_S; - const newTs = Number(block.timestamp) + warpBy; - logger.warn(`Hit EpochNotStable at candidate ${candidate}, warping L1 forward by ${warpBy}s to ${newTs}`); - await test.context.cheatCodes.eth.warp(newTs, { resetBlockInterval: true }); - const newCurrentSlot = Number(test.epochCache.getEpochAndSlotNow().slot); - if (candidate < newCurrentSlot + 4) { - candidate = newCurrentSlot + 4; - } - } - } - if (slotOne === undefined || !proposerOne || !proposerTwo || !proposerThree) { - throw new Error(`Could not find a slot with three distinct consecutive proposers after ${maxAttempts} attempts`); - } - - const slotZero = SlotNumber(slotOne - 1); - const slotTwo = SlotNumber(slotOne + 1); - const slotThree = SlotNumber(slotOne + 2); - - const proposerOneNodeIndex = validators.findIndex(v => v.attester.equals(proposerOne!)); - if (proposerOneNodeIndex < 0) { - throw new Error(`No node holds the key for proposer ${proposerOne}`); - } - - logger.warn(`Selected target slotOne=${slotOne}`, { - slotOne, - slotZero, - slotTwo, - slotThree, - proposerOne: proposerOne.toString(), - proposerOneNodeIndex, - proposerTwo: proposerTwo.toString(), - proposerThree: proposerThree.toString(), - }); - - // Prevent the proposer for slotOne from publishing the checkpoint to L1 (build & gossip still happen). - await nodes[proposerOneNodeIndex].setConfig({ skipPublishingCheckpointsPercent: 100 }); - - // Subscribe to the prune event on every node before sequencers start, so we never miss it. - // We capture the L2 tips synchronously inside the handler — the archiver has already removed - // the pruned blocks at emit time, so this snapshot reflects the rolled-back state before any - // new pipelined block can be applied. - type PruneObservation = { slotNumber: SlotNumber; blocks: L2Block[]; tipsAtPrune: L2Tips }; - const prunePromises: Promise[] = nodes.map( - (node, idx) => - new Promise(resolve => { - const archiver = node.getBlockSource() as Archiver; - // eslint-disable-next-line @typescript-eslint/no-misused-promises - archiver.events.once(L2BlockSourceEvents.L2PruneUncheckpointed, async ev => { - const tipsAtPrune = await node.getChainTips(); - logger.warn(`Node ${idx} pruned uncheckpointed blocks`, { - slotNumber: ev.slotNumber, - blocks: ev.blocks.map(b => ({ number: b.number, slot: b.header.globalVariables.slotNumber })), - tipsAtPrune, - }); - resolve({ slotNumber: ev.slotNumber, blocks: ev.blocks, tipsAtPrune }); - }); - }), - ); - - // Warp L1 to one L1 block before slotZero begins. Pipelining will then engage during slotZero. - const slotZeroStart = getTimestampForSlot(slotZero, test.constants); - const warpTo = slotZeroStart - BigInt(test.L1_BLOCK_TIME_IN_S); - logger.warn(`Warping L1 to timestamp ${warpTo} (one L1 block before slot ${slotZero})`); - await test.context.cheatCodes.eth.warp(Number(warpTo), { resetBlockInterval: true }); - - // Check that the chain is empty - const node = nodes[0]; - const blockNumber = await node.getBlockNumber(); - expect(blockNumber).toEqual(0); - - // Start all sequencers. - const sequencers = nodes.map(n => n.getSequencer()!); - const { failEvents } = test.watchSequencerEvents(sequencers, i => ({ validator: `V${i + 1}` })); - - // Subscribe to the proposerTwo pipelined-discard event — this is the most direct signal - // that the pipelined slotTwo work was correctly thrown away because parent slotOne did not land. - const proposerTwoNodeIndex = validators.findIndex(v => v.attester.equals(proposerTwo!)); - const pipelinedDiscardEvents: { slot: SlotNumber; checkpointNumber: number; reason: string }[] = []; - sequencers[proposerTwoNodeIndex].getSequencer().on('pipelined-checkpoint-discarded', args => { - pipelinedDiscardEvents.push({ slot: args.slot, checkpointNumber: args.checkpointNumber, reason: args.reason }); - logger.warn(`proposerTwo (node ${proposerTwoNodeIndex}) discarded pipelined work`, args); - }); - - await Promise.all(sequencers.map(s => s.start())); - logger.warn('All sequencers started'); - - const slotAdvanceTimeout = test.L2_SLOT_DURATION_IN_S * 3; - - // (1) During slotZero: the pipelined proposer for slotOne broadcasts. Every node sees a proposed block at slotOne. - logger.warn(`Waiting for proposed chain to reach slot ${slotOne} on all nodes (build during slotZero)`); - // REFACTOR: duplicated Promise.all+retryUntil block checking proposed slot on all nodes; - // replace with a shared helper such as waitUntilAllNodesProposedSlot(nodes, slot, timeout). - await Promise.all( - nodes.map((node, idx) => - retryUntil( - async () => { - const tips = await node.getChainTips(); - if (tips.proposed.number === 0) { - return false; - } - const block = await node.getBlock(tips.proposed.number); - return !!block && block.header.globalVariables.slotNumber === slotOne; - }, - `node ${idx} proposed advanced to slot ${slotOne}`, - slotAdvanceTimeout, - 0.5, - ), - ), - ); - - // (2) During slotOne: the pipelined proposer for slotTwo broadcasts on top of slotOne → proposed reaches slotTwo. - logger.warn(`Waiting for proposed chain to reach slot ${slotTwo} on all nodes (build during slotOne)`); - // REFACTOR: same pattern as above — duplicated Promise.all+retryUntil; extract to helper. - await Promise.all( - nodes.map((node, idx) => - retryUntil( - async () => { - const tips = await node.getChainTips(); - if (tips.proposed.number === 0) { - return false; - } - const block = await node.getBlock(tips.proposed.number); - return !!block && block.header.globalVariables.slotNumber === slotTwo; - }, - `node ${idx} proposed advanced to slot ${slotTwo}`, - slotAdvanceTimeout, - 0.5, - ), - ), - ); - - // (3) Wait until slotOne has fully ended on L1 — the archiver only prunes once slotAtNextL1Block > slotOne. - // The end-of-slotOne timestamp equals the start-of-slotTwo timestamp. - const slotOneEndTimestamp = getTimestampForSlot(slotTwo, test.constants); - logger.warn(`Waiting until L1 timestamp ${slotOneEndTimestamp} (end of slot ${slotOne})`); - await waitUntilL1Timestamp(test.l1Client, slotOneEndTimestamp, undefined, test.L2_SLOT_DURATION_IN_S * 3); - - // (4) After slotOne ends without a checkpoint, all nodes should prune. - // Verify rollback via the prune event itself: the pruned slot must equal slotOne, and the - // pruned blocks must include the broadcast blocks for slotOne (proposerOne) and slotTwo - // (pipelined proposerTwo, whose work is now invalid because parent slotOne did not land). - logger.warn('Waiting for L2PruneUncheckpointed on every node'); - const pruneTimeoutMs = test.L2_SLOT_DURATION_IN_S * 2 * 1000; - const pruneObservations = await Promise.all( - prunePromises.map((p, idx) => - Promise.race([p, timeoutPromise(pruneTimeoutMs, `Node ${idx} did not emit prune event in time`)]), - ), - ); - - logger.warn('Asserting prune event details on every node'); - for (const [idx, obs] of pruneObservations.entries()) { - expect({ idx, slotNumber: obs.slotNumber }).toEqual({ idx, slotNumber: slotOne }); - // proposerOne broadcasts during slotZero, so its block must always be in the pruned set. - // The pipelined slotTwo broadcast may or may not have arrived in time on every node, so - // we don't strictly require it here. - const prunedSlots = obs.blocks.map(b => b.header.globalVariables.slotNumber); - expect(prunedSlots).toContain(slotOne); - } - - // (5) Allow the formerly suppressed node to publish again so the chain can recover. - logger.warn(`Re-enabling checkpoint publishing on node ${proposerOneNodeIndex}`); - await nodes[proposerOneNodeIndex].setConfig({ skipPublishingCheckpointsPercent: 0 }); - - // (6) During slotTwo: the pipelined proposer for slotThree builds and broadcasts → proposed advances again. - // The chain must have rewound past slotOne and slotTwo and now build on whatever was - // checkpointed before slotZero — genesis, in this test, since no checkpoints have landed yet. - const postPruneProposedNumbers = pruneObservations.map(o => o.tipsAtPrune.proposed.number); - expect(postPruneProposedNumbers[0]).toBe(0); - - logger.warn(`Waiting for proposed chain to advance to slot ${slotThree} on all nodes (build during slotTwo)`); - await Promise.all( - nodes.map((node, idx) => - retryUntil( - async () => { - const tips = await node.getChainTips(); - if (tips.proposed.number === 0) { - return false; - } - const block = await node.getBlock(tips.proposed.number); - return !!block && block.header.globalVariables.slotNumber >= slotThree; - }, - `node ${idx} proposed advanced to slot >= ${slotThree}`, - slotAdvanceTimeout, - 0.5, - ), - ), - ); - - // The first block in the chain after the prune must be the slotThree block — there should be - // nothing between genesis and the new pipelined work, since slotOne and slotTwo were pruned. - for (const node of nodes) { - const blocks = await node.getBlocks(BlockNumber(1), 50); - const firstSlotThreeIdx = blocks.findIndex(b => b.header.globalVariables.slotNumber === slotThree); - expect(firstSlotThreeIdx).toEqual(0); - } - - // (7) During slotThree: proposerThree publishes → checkpointed advances on every node. - logger.warn(`Waiting for checkpointed chain to reach slot >= ${slotThree} on all nodes`); - await Promise.all( - nodes.map((node, idx) => - retryUntil( - async () => { - const tips = await node.getChainTips(); - if (tips.checkpointed.checkpoint.number === 0) { - return false; - } - const block = await node.getBlock(tips.checkpointed.block.number); - return ( - !!block && block.header.globalVariables.slotNumber >= slotThree && tips.checkpointed.block.number > 0 - ); - }, - `node ${idx} checkpointed advanced to slot >= ${slotThree}`, - slotAdvanceTimeout, - 0.5, - ), - ), - ); - - // Sanity: the only fail events we tolerate are the deliberate skip-publish on the suppressed - // node for slotOne, the pipelined-discard knock-on from proposerTwo (its parent slotOne - // never landed), and proposer-rollup-check noise that any non-proposer emits when the rollup - // contract rejects them. - const unexpectedFailEvents = failEvents.filter(e => { - if ( - e.type === 'checkpoint-publish-failed' && - e.sequencerIndex === proposerOneNodeIndex + 2 && - e.slot === slotOne - ) { - return false; - } - if ( - e.type === 'checkpoint-publish-failed' && - e.sequencerIndex === proposerTwoNodeIndex + 2 && - e.slot === slotTwo - ) { - return false; - } - // Expected - if (e.type === 'pipelined-checkpoint-discarded') { - return false; - } - return true; - }); - if (unexpectedFailEvents.length > 0) { - logger.error('Unexpected fail events from sequencers', unexpectedFailEvents); - } - expect(unexpectedFailEvents).toEqual([]); - }); -}); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_orphan_block_prune.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_orphan_block_prune.test.ts deleted file mode 100644 index f90a4c3f9092..000000000000 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_orphan_block_prune.test.ts +++ /dev/null @@ -1,316 +0,0 @@ -import type { Archiver } from '@aztec/archiver'; -import type { AztecNodeService } from '@aztec/aztec-node'; -import { EthAddress } from '@aztec/aztec.js/addresses'; -import { Fr } from '@aztec/aztec.js/fields'; -import type { Logger } from '@aztec/aztec.js/log'; -import { asyncMap } from '@aztec/foundation/async-map'; -import { BlockNumber, CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types'; -import { times } from '@aztec/foundation/collection'; -import { SecretValue } from '@aztec/foundation/config'; -import { retryUntil } from '@aztec/foundation/retry'; -import { bufferToHex } from '@aztec/foundation/string'; -import { timeoutPromise } from '@aztec/foundation/timer'; -import { type L2Block, L2BlockSourceEvents, type L2Tips } from '@aztec/stdlib/block'; -import { getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; - -import { jest } from '@jest/globals'; -import { privateKeyToAccount } from 'viem/accounts'; - -import { getPrivateKeyFromIndex } from '../fixtures/utils.js'; -import { EpochsTestContext } from './epochs_test.js'; - -jest.setTimeout(1000 * 60 * 15); - -const NODE_COUNT = 4; - -/** - * E2E test for the orphan-proposed-block prune path under proposer pipelining. - * - * With pipelining, the proposer for slot N+1 builds and gossips its checkpoint during slot N. The last block in that - * checkpoint is broadcast standalone (so peers can pre-sync the archive) and the enclosing CheckpointProposal is - * broadcast separately. If the CheckpointProposal never arrives, peers are left with a proposed-but-uncheckpointed tip - * — an "orphan" block — and the next proposer must NOT attempt to build on it. - * - * Setup: 4 validators (V1..V4), one node per key, mocked gossip network. We find two consecutive slots S1, S2 with - * distinct proposers P1, P2. P1 is configured via the test-only `skipBroadcastCheckpointProposal` flag to suppress its - * CheckpointProposal broadcast while still letting the held last block reach peers. P2 must (a) prune the orphan on - * every archiver, and (b) build a fresh checkpoint for S2 that lands on L1. - * - * EpochsTestContext with 4 validator nodes, mockGossipSubNetwork, no prover. Timing: ethSlot=6s, - * aztecSlot=36s, epoch=4, proofSubmissionEpochs=1024, blockDurationMs=8000, inboxLag=2 (v5 always - * enforces the timetable, so the former enforceTimeTable/disableAnvilTestWatcher overrides are gone). - * L1 is time-warped to align with the target S1 build slot. - */ -describe('e2e_epochs/epochs_orphan_block_prune', () => { - let logger: Logger; - let test: EpochsTestContext; - let nodes: AztecNodeService[]; - - afterEach(async () => { - jest.restoreAllMocks(); - await test?.teardown(); - }); - - // Finds two consecutive slots S1/S2 with distinct proposers. Suppresses P1's CheckpointProposal - // broadcast, waits for the orphan block to appear on all archivers, asserts L2PruneUncheckpointed - // fires on every node for slot S1, then verifies the rebuilt S2 checkpoint lands on L1 with a - // different archive root from the orphan. - it('all nodes prune the orphan block and S2 rebuilds the checkpoint chain', async () => { - // Build 4 distinct validators (V1..V4). One key per node, no overlap. - const validators = times(NODE_COUNT, i => { - const privateKey = bufferToHex(getPrivateKeyFromIndex(i + 3)!); - const attester = EthAddress.fromString(privateKeyToAccount(privateKey).address); - return { attester, withdrawer: attester, privateKey, bn254SecretKey: new SecretValue(Fr.random().toBigInt()) }; - }); - - test = await EpochsTestContext.setup({ - numberOfAccounts: 1, - initialValidators: validators, - inboxLag: 2, - mockGossipSubNetwork: true, - startProverNode: false, - aztecEpochDuration: 4, - aztecProofSubmissionEpochs: 1024, - ethereumSlotDuration: 6, - aztecSlotDuration: 36, - blockDurationMs: 8000, - attestationPropagationTime: 0.5, - aztecTargetCommitteeSize: NODE_COUNT, - skipInitialSequencer: true, - }); - - ({ logger } = test); - - nodes = await asyncMap(validators, ({ privateKey }, i) => - test.createValidatorNode([privateKey], { - dontStartSequencer: true, - coinbase: EthAddress.fromNumber(0xa + i), - buildCheckpointIfEmpty: true, - minTxsPerBlock: 0, - }), - ); - - logger.warn('Validator nodes created', { - validators: validators.map((v, i) => ({ idx: i, attester: v.attester.toString() })), - }); - - // Find S1 (>=4 ahead) such that proposers for S1 and S2=S1+1 are two distinct validators. The +4 margin gives the - // warp+sequencer-start path enough headroom to reach the build window for S1-1 (the pipelining build slot for S1) - // even if node creation jitters. - // - // The L1 rollup contract only exposes proposers for epochs whose randao seed is "stable" (i.e. queryable on L1 - // right now). When we look too far into the future the contract reverts with `ValidatorSelection__EpochNotStable`. - // We handle this by warping L1 forward one epoch at a time and retrying. - // REFACTOR: hand-rolled slot-search loop with per-epoch warp and EpochNotStable retry; a DSL - // helper like findConsecutiveSlotsWithDistinctProposers(minAhead, maxAttempts) would encapsulate - // the epoch-stable query, warp cadence, and candidate-advance logic. - let S1: SlotNumber | undefined; - let proposerOne: EthAddress | undefined; - let proposerTwo: EthAddress | undefined; - let candidate = Number(test.epochCache.getEpochAndSlotNow().slot) + 4; - const maxAttempts = 200; - for (let attempt = 0; attempt < maxAttempts && S1 === undefined; attempt++) { - try { - const [p1, p2] = await Promise.all([ - test.epochCache.getProposerAttesterAddressInSlot(SlotNumber(candidate)), - test.epochCache.getProposerAttesterAddressInSlot(SlotNumber(candidate + 1)), - ]); - const p1Index = p1 ? validators.findIndex(v => v.attester.equals(p1)) : -1; - const p2Index = p2 ? validators.findIndex(v => v.attester.equals(p2)) : -1; - if (p1 && p2 && !p1.equals(p2) && p1Index >= 0 && p2Index >= 0) { - S1 = SlotNumber(candidate); - proposerOne = p1; - proposerTwo = p2; - break; - } - candidate++; - } catch (err) { - const msg = err instanceof Error ? err.message : String(err); - if (!msg.includes('EpochNotStable')) { - throw err; - } - const block = await test.l1Client.getBlock({ includeTransactions: false }); - const warpBy = test.epochDuration * test.L2_SLOT_DURATION_IN_S; - const newTs = Number(block.timestamp) + warpBy; - logger.warn(`Hit EpochNotStable at candidate ${candidate}, warping L1 forward by ${warpBy}s to ${newTs}`); - await test.context.cheatCodes.eth.warp(newTs, { resetBlockInterval: true }); - const newCurrentSlot = Number(test.epochCache.getEpochAndSlotNow().slot); - if (candidate < newCurrentSlot + 4) { - candidate = newCurrentSlot + 4; - } - } - } - if (S1 === undefined || !proposerOne || !proposerTwo) { - throw new Error(`Could not find a slot with two distinct consecutive proposers after ${maxAttempts} attempts`); - } - - const S2 = SlotNumber(S1 + 1); - const p1Index = validators.findIndex(v => v.attester.equals(proposerOne!)); - const p2Index = validators.findIndex(v => v.attester.equals(proposerTwo!)); - - logger.warn(`Selected target S1=${S1}`, { - S1, - S2, - proposerOne: proposerOne.toString(), - p1Index, - proposerTwo: proposerTwo.toString(), - p2Index, - }); - - // Suppress only the CheckpointProposal broadcast for the proposer of S1. The held last block is still broadcast - // standalone, so peers' archivers ingest the slot-S1 block as a proposed tip but never see a checkpoint proposal - // for it — the exact orphan-block state we want. - await nodes[p1Index].setConfig({ skipBroadcastCheckpointProposal: true }); - - // No tx is needed: nodes are configured with buildCheckpointIfEmpty so the proposer will produce an empty - // checkpoint on its slot. The test verifies the orphan prune + rebuild invariants, not tx flow. - - // Subscribe to the prune event on every node before sequencers start, so we never miss it. We capture the chain - // tips asynchronously inside the handler for log context, but do not assert on them — by the time the snapshot is - // read, P2's rebuild may already have landed. - type PruneObservation = { slotNumber: SlotNumber; blocks: L2Block[]; tipsAtPrune: L2Tips }; - const prunePromises: Promise[] = nodes.map( - (node, idx) => - new Promise(resolve => { - const archiver = node.getBlockSource() as Archiver; - // eslint-disable-next-line @typescript-eslint/no-misused-promises - archiver.events.once(L2BlockSourceEvents.L2PruneUncheckpointed, async ev => { - const tipsAtPrune = await node.getChainTips(); - logger.warn(`Node ${idx} pruned uncheckpointed blocks`, { - slotNumber: ev.slotNumber, - blocks: ev.blocks.map(b => ({ number: b.number, slot: b.header.globalVariables.slotNumber })), - tipsAtPrune, - }); - resolve({ slotNumber: ev.slotNumber, blocks: ev.blocks, tipsAtPrune }); - }); - }), - ); - - // Warp L1 to one L1 block before the build slot for S1 (which is S1-1 under pipelining offset 1). Pipelining will - // then engage during S1-1 and the proposer for S1 builds + would broadcast its CheckpointProposal — except we - // just suppressed it. - const buildSlot = SlotNumber(S1 - 1); - const targetTs = getTimestampForSlot(buildSlot, test.constants) - BigInt(test.L1_BLOCK_TIME_IN_S); - logger.warn(`Warping L1 to timestamp ${targetTs} (one L1 block before build slot ${buildSlot} for S1=${S1})`); - await test.context.cheatCodes.eth.warp(Number(targetTs), { resetBlockInterval: true }); - - expect(await nodes[0].getBlockNumber()).toEqual(0); - - const sequencers = nodes.map(n => n.getSequencer()!); - const { failEvents } = test.watchSequencerEvents(sequencers, i => ({ validator: `V${i + 1}` })); - - await Promise.all(sequencers.map(s => s.start())); - logger.warn('All sequencers started'); - - const slotAdvanceTimeout = test.L2_SLOT_DURATION_IN_S * 3; - - // (1) Orphan appears on every archiver. During build slot S1-1, P1 builds and broadcasts the held last block - // standalone (because of skipBroadcastCheckpointProposal). Every node's proposed tip advances to a block whose - // slotNumber === S1. - logger.warn(`Waiting for proposed chain to reach slot ${S1} on all nodes (orphan tip from P1)`); - // REFACTOR: Promise.all over per-node retryUntil polling getChainTips; a waitForAllNodesToReach - // helper that takes a predicate over chain tips would avoid this hand-rolled fan-out pattern. - await Promise.all( - nodes.map((node, idx) => - retryUntil( - async () => { - const tips = await node.getChainTips(); - if (tips.proposed.number === 0) { - return false; - } - const block = await node.getBlock(tips.proposed.number); - return !!block && block.header.globalVariables.slotNumber === S1; - }, - `node ${idx} proposed advanced to slot ${S1}`, - slotAdvanceTimeout, - 0.5, - ), - ), - ); - - // Capture each node's pre-prune block-1 archive root for the staleness check in (3). - const preBlocks = await Promise.all(nodes.map(node => node.getBlock(BlockNumber(1)))); - const preArchiveRoots = preBlocks.map(block => { - if (!block) { - throw new Error('Expected pre-prune block 1 to exist on every node'); - } - return block.archive.root.toString(); - }); - logger.warn('Captured pre-prune block-1 archive roots', { preArchiveRoots }); - - // (2) Orphan is pruned on every archiver. Since no CheckpointProposal was received for S1, the wall-clock prune - // fires after the checkpoint proposal receive deadline plus local jitter, well inside slot S1 (= the build slot - // for S2). We wait up to 2 slot durations as a margin. - logger.warn('Waiting for L2PruneUncheckpointed on every node'); - const pruneTimeoutMs = test.L2_SLOT_DURATION_IN_S * 2 * 1000; - const pruneObservations = await Promise.all( - prunePromises.map((p, idx) => - Promise.race([p, timeoutPromise(pruneTimeoutMs, `Node ${idx} did not emit prune event in time`)]), - ), - ); - - for (const [idx, obs] of pruneObservations.entries()) { - expect({ idx, slotNumber: obs.slotNumber }).toEqual({ idx, slotNumber: S1 }); - const prunedSlots = obs.blocks.map(b => b.header.globalVariables.slotNumber); - // Only the orphan at slot S1 should have been pruned — nothing earlier or later. - expect(prunedSlots.every(s => s === S1)).toBe(true); - // We do not assert exact equality on tipsAtPrune here. The handler is async and awaits getChainTips(), so P2's - // rebuild could already have landed by the time the snapshot is read. The prune event itself (slotNumber === S1, - // blocks include S1) is sufficient proof. - } - - // (3) S2 builds and the checkpoint lands on L1. After the prune, P2's pipelined build during S1 publishes during - // S2, so L2 block 1 on every node must be the rebuilt block with slot S2. We target block 1 directly rather than - // the live checkpointed tip to avoid an S3-first race where the chain has already advanced past S2 by the time - // we poll. - logger.warn(`Waiting for L2 block 1 to be the rebuilt slot-${S2} block on all nodes`); - await Promise.all( - nodes.map((node, idx) => - retryUntil( - async () => { - const block = await node.getBlock(BlockNumber(1)); - return !!block && block.header.globalVariables.slotNumber === S2; - }, - `node ${idx} block 1 rebuilt at slot ${S2}`, - slotAdvanceTimeout, - 0.5, - ), - ), - ); - - // Independently confirm the checkpoint actually landed on L1 by waiting (bounded) on the chain monitor and - // verifying the block at L2 block number 1 — that is the rebuilt block, and its slot must equal S2. Targeting - // block 1 rather than the live tip avoids a race where the chain has already advanced past S2 by the time we read. - await test.waitUntilCheckpointNumber(CheckpointNumber(1), test.L2_SLOT_DURATION_IN_S * 4); - const rebuiltBlock = await nodes[0].getBlock(BlockNumber(1)); - expect(rebuiltBlock).toBeDefined(); - expect(rebuiltBlock!.header.globalVariables.slotNumber).toEqual(S2); - - // The rebuilt block at number 1 must have a different archive root from the orphan we saw before the prune. This - // guards against accidental pass on stale state. - const postBlocks = await Promise.all(nodes.map(node => node.getBlock(BlockNumber(1)))); - const postArchiveRoots = postBlocks.map(block => { - if (!block) { - throw new Error('Expected post-prune block 1 to exist on every node'); - } - return block.archive.root.toString(); - }); - logger.warn('Captured post-prune block-1 archive roots', { postArchiveRoots }); - for (const [idx, root] of postArchiveRoots.entries()) { - expect({ idx, root }).not.toEqual({ idx, root: preArchiveRoots[idx] }); - } - - // Tolerated fail events, scoped narrowly: P1 at S1 expectedly fails to publish because peers never see the - // CheckpointProposal, so it cannot collect attestations. P2 must not discard or miss its own S2 checkpoint. - const unexpectedFailEvents = failEvents.filter(e => { - if (e.type === 'checkpoint-publish-failed' && e.sequencerIndex === p1Index + 2 && e.slot === S1) { - return false; - } - return true; - }); - if (unexpectedFailEvents.length > 0) { - logger.error('Unexpected fail events from sequencers', unexpectedFailEvents); - } - expect(unexpectedFailEvents).toEqual([]); - }); -}); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_simple_block_building.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_simple_block_building.test.ts deleted file mode 100644 index 64d1f52afd67..000000000000 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_simple_block_building.test.ts +++ /dev/null @@ -1,115 +0,0 @@ -import type { AztecNodeService } from '@aztec/aztec-node'; -import type { AztecAddress } from '@aztec/aztec.js/addresses'; -import { EthAddress } from '@aztec/aztec.js/addresses'; -import { NO_WAIT } from '@aztec/aztec.js/contracts'; -import { Fr } from '@aztec/aztec.js/fields'; -import type { Logger } from '@aztec/aztec.js/log'; -import { waitForTx } from '@aztec/aztec.js/node'; -import type { Operator } from '@aztec/ethereum/deploy-aztec-l1-contracts'; -import { asyncMap } from '@aztec/foundation/async-map'; -import { times, timesAsync } from '@aztec/foundation/collection'; -import { SecretValue } from '@aztec/foundation/config'; -import { bufferToHex } from '@aztec/foundation/string'; -import { executeTimeout } from '@aztec/foundation/timer'; -import type { TestContract } from '@aztec/noir-test-contracts.js/Test'; - -import { jest } from '@jest/globals'; -import { privateKeyToAccount } from 'viem/accounts'; - -import { type EndToEndContext, getPrivateKeyFromIndex } from '../fixtures/utils.js'; -import { proveInteraction } from '../test-wallet/utils.js'; -import { EpochsTestContext } from './epochs_test.js'; - -jest.setTimeout(1000 * 60 * 10); - -const NODE_COUNT = 3; -const TX_COUNT = 8; - -// Suite: verifies that 3 validator nodes can build blocks without sequencer errors. Uses a -// lightweight RPC-only initial node (skipInitialSequencer), mockGossipSubNetwork, no prover. -// Timing: ethSlot=12s, aztecSlot=3×12=36s, epoch=default 6, proofSubmissionEpochs=1024, -// blockDurationMs=6s, inboxLag=2 (v5 always enforces the timetable, so the former enforceTimeTable/ -// disableAnvilTestWatcher overrides are gone). Pre-proved txs sent from hardcoded -// genesis-funded account (no on-chain account deploy needed). -// Sets up a lightweight RPC-only node without any account deployment, registers a test contract -// locally, then spawns NODE_COUNT validator nodes connected via a mocked gossip sub network. -// Mines N txs across N blocks, checking that no sequencer errors occur during block building. -describe('e2e_epochs/epochs_simple_block_building', () => { - let context: EndToEndContext; - let logger: Logger; - - let test: EpochsTestContext; - let validators: (Operator & { privateKey: `0x${string}` })[]; - let nodes: AztecNodeService[]; - let contract: TestContract; - let from: AztecAddress; - - beforeEach(async () => { - validators = times(NODE_COUNT, i => { - const privateKey = bufferToHex(getPrivateKeyFromIndex(i + 3)!); - const attester = EthAddress.fromString(privateKeyToAccount(privateKey).address); - return { attester, withdrawer: attester, privateKey, bn254SecretKey: new SecretValue(Fr.random().toBigInt()) }; - }); - - // Setup context with no initial sequencer (lightweight RPC-only node). - // The hardcoded account is funded via genesis without needing on-chain deployment. - test = await EpochsTestContext.setup({ - numberOfAccounts: 0, - initialValidators: validators, - mockGossipSubNetwork: true, - aztecProofSubmissionEpochs: 1024, - aztecSlotDurationInL1Slots: 3, - ethereumSlotDuration: 12, - blockDurationMs: 6000, - startProverNode: false, - skipInitialSequencer: true, - inboxLag: 2, - }); - - ({ context, logger } = test); - from = context.accounts[0]; // auto-created by setup - - // Register test contract locally for sending txs (no on-chain deployment needed). - contract = await test.registerTestContract(context.wallet); - - // Start the validator nodes. - logger.warn(`Initial setup complete. Starting ${NODE_COUNT} validator nodes.`); - nodes = await asyncMap(validators, ({ privateKey }) => - test.createValidatorNode([privateKey], { minTxsPerBlock: 1, maxTxsPerBlock: 1 }), - ); - logger.warn(`Test setup completed.`, { validators: validators.map(v => v.attester.toString()) }); - }); - - afterEach(async () => { - jest.restoreAllMocks(); - await test.teardown(); - }); - - // Pre-proves TX_COUNT transactions emitting unique nullifiers, sends them, waits for all to mine, - // then asserts no fail events were emitted by any of the 3 sequencers during the run. - it('builds blocks without any errors', async () => { - const sequencers = nodes.map(node => node.getSequencer()!); - const { failEvents } = test.watchSequencerEvents(sequencers, i => ({ validator: validators[i].attester })); - - // Create and submit txs from the hardcoded account. Each tx emits a unique - // nullifier, which is enough side-effect to produce a non-empty block. - const txs = await timesAsync(TX_COUNT, _i => - proveInteraction(context.wallet, contract.methods.emit_nullifier(Fr.random()), { from }), - ); - const txHashes = await Promise.all(txs.map(tx => tx.send({ wait: NO_WAIT }))); - logger.warn(`Sent ${txHashes.length} transactions`, { - txs: txHashes, - }); - - // Wait until all txs are mined - const timeout = test.L2_SLOT_DURATION_IN_S * (TX_COUNT * 2 + 1); - await executeTimeout( - () => Promise.all(txHashes.map(txHash => waitForTx(context.aztecNode, txHash, { timeout }))), - timeout * 1000, - ); - logger.warn(`All txs have been mined`); - - // Expect no failures from sequencers during block building - test.assertNoFailuresFromSequencers(failEvents); - }); -}); diff --git a/yarn-project/end-to-end/src/e2e_genesis_timestamp.test.ts b/yarn-project/end-to-end/src/e2e_genesis_timestamp.test.ts index be5dbb143b7d..26f17f834832 100644 --- a/yarn-project/end-to-end/src/e2e_genesis_timestamp.test.ts +++ b/yarn-project/end-to-end/src/e2e_genesis_timestamp.test.ts @@ -28,6 +28,11 @@ describe('e2e_genesis_timestamp', () => { 0, { ...AUTOMINE_E2E_OPTS, + // This suite pins the proven tip at genesis (no prover node, syncChainTip:'proven', + // advancePastGenesis:false) and asserts on genesis-anchored txs. Mining the L1 setup txs + // instantly shifts how far L1 time advances past the rollup genesis during deployment, which + // breaks those genesis-anchoring assumptions, so keep L1 setup on the anvil block interval. + automineL1Setup: false, advancePastGenesis: false, // This test proves genesis-anchored account deployment txs, so it needs deployable accounts additionallyFundedAccounts: await generateSchnorrAccounts(2, 'schnorr'), diff --git a/yarn-project/end-to-end/src/e2e_p2p/duplicate_attestation_slash.test.ts b/yarn-project/end-to-end/src/e2e_p2p/duplicate_attestation_slash.test.ts deleted file mode 100644 index 605bd8027e21..000000000000 --- a/yarn-project/end-to-end/src/e2e_p2p/duplicate_attestation_slash.test.ts +++ /dev/null @@ -1,300 +0,0 @@ -import type { AztecNodeService } from '@aztec/aztec-node'; -import type { TestAztecNodeService } from '@aztec/aztec-node/test'; -import { EthAddress } from '@aztec/aztec.js/addresses'; -import { EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; -import { bufferToHex } from '@aztec/foundation/string'; -import { OffenseType } from '@aztec/slasher'; -import { TopicType } from '@aztec/stdlib/p2p'; - -import { jest } from '@jest/globals'; -import fs from 'fs'; -import os from 'os'; -import path from 'path'; -import { privateKeyToAccount } from 'viem/accounts'; - -import { shouldCollectMetrics } from '../fixtures/fixtures.js'; -import { ATTESTER_PRIVATE_KEYS_START_INDEX, createNode } from '../fixtures/setup_p2p_test.js'; -import { getPrivateKeyFromIndex } from '../fixtures/utils.js'; -import { P2PNetworkTest } from './p2p_network.js'; -import { advanceToEpochBeforeProposer, awaitCommitteeExists, awaitOffenseDetected } from './shared.js'; - -const TEST_TIMEOUT = 600_000; // 10 minutes - -jest.setTimeout(TEST_TIMEOUT); - -const NUM_VALIDATORS = 4; -const BOOT_NODE_UDP_PORT = 4600; -const COMMITTEE_SIZE = NUM_VALIDATORS; -const ETHEREUM_SLOT_DURATION = 8; -const AZTEC_SLOT_DURATION = ETHEREUM_SLOT_DURATION * 3; -const BLOCK_DURATION = 4; - -const DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'duplicate-attestation-slash-')); - -/** - * Test that slashing occurs when a validator sends duplicate attestations (equivocation). - * - * The setup of the test is as follows: - * 1. Create 4 validator nodes total: - * - 2 honest validators with unique keys - * - 2 "malicious proposer" validators that share the SAME validator key but have DIFFERENT coinbase addresses - * (these will create duplicate proposals for the same slot) - * - The malicious proposer validators also have `attestToEquivocatedProposals: true` which makes them attest - * to BOTH proposals when they receive them - this is the attestation equivocation we want to detect - * 2. The two nodes with the same proposer key will both detect they are proposers for the same slot and race to propose - * 3. Since they have different coinbase addresses, their proposals will have different archives (different content) - * 4. The malicious attester nodes (with attestToEquivocatedProposals enabled) will attest to BOTH proposals - * 5. Honest validators will detect the duplicate attestations and emit a slash event - * - * NOTE: This test triggers BOTH duplicate proposal (from malicious proposers sharing a key) AND duplicate attestation - * (from the malicious proposers attesting to multiple proposals). We verify specifically that the duplicate - * attestation offense is recorded. - * - * Setup: P2PNetworkTest with mockGossipSubNetwork:true (in-memory bus, NOT real libp2p). 4 validators, - * ethSlot=8s, aztecSlot=24s, epoch=2, proofSubEpochs=1024, minTxsPerBlock=0, inboxLag=2 (v5 always enforces - * the timetable, so the former enforceTimeTable/l1PublishingTime overrides are gone). - * Candidate for relocation to e2e_slashing/. - */ -describe('e2e_p2p_duplicate_attestation_slash', () => { - let t: P2PNetworkTest; - let nodes: AztecNodeService[]; - - // Small slashing unit so we don't kick anyone out - const slashingUnit = BigInt(1e14); - const slashingQuorum = 3; - const slashingRoundSize = 4; - const aztecEpochDuration = 2; - - beforeEach(async () => { - t = await P2PNetworkTest.create({ - testName: 'e2e_p2p_duplicate_attestation_slash', - numberOfNodes: 0, - numberOfValidators: NUM_VALIDATORS, - basePort: BOOT_NODE_UDP_PORT, - metricsPort: shouldCollectMetrics(), - initialConfig: { - anvilSlotsInAnEpoch: 4, - listenAddress: '127.0.0.1', - aztecEpochDuration, - ethereumSlotDuration: ETHEREUM_SLOT_DURATION, - aztecSlotDuration: AZTEC_SLOT_DURATION, - aztecTargetCommitteeSize: COMMITTEE_SIZE, - aztecProofSubmissionEpochs: 1024, // effectively do not reorg - slashInactivityConsecutiveEpochThreshold: 32, // effectively do not slash for inactivity - minTxsPerBlock: 0, // always be building - mockGossipSubNetwork: true, // do not worry about p2p connectivity issues - slashingQuorum, - slashingRoundSizeInEpochs: slashingRoundSize / aztecEpochDuration, - slashAmountSmall: slashingUnit, - slashAmountMedium: slashingUnit * 2n, - slashAmountLarge: slashingUnit * 3n, - blockDurationMs: BLOCK_DURATION * 1000, - slashDuplicateProposalPenalty: slashingUnit, - slashDuplicateAttestationPenalty: slashingUnit, - slashingOffsetInRounds: 1, - inboxLag: 2, - }, - }); - - await t.setup(); - await t.applyBaseSetup(); - }); - - afterEach(async () => { - await t.stopNodes(nodes); - await t.teardown(); - for (let i = 0; i < NUM_VALIDATORS; i++) { - fs.rmSync(`${DATA_DIR}-${i}`, { recursive: true, force: true, maxRetries: 3 }); - } - }); - - const debugRollup = async () => { - await t.ctx.cheatCodes.rollup.debugRollup(); - }; - - // Two malicious nodes share a validator key and both attest to each other's proposals - // (attestToEquivocatedProposals:true). Honest nodes detect the DUPLICATE_ATTESTATION offense and verify - // the offending attester is the shared key's address. Also exercises DUPLICATE_PROPOSAL as a side effect - // but asserts specifically that DUPLICATE_ATTESTATION is recorded. - it('slashes validator who sends duplicate attestations', async () => { - const { rollup } = await t.getContracts(); - - // Jump forward to an epoch in the future such that the validator set is not empty - await t.ctx.cheatCodes.rollup.advanceToEpoch(EpochNumber(4)); - await debugRollup(); - - t.logger.warn('Creating nodes'); - - // Get the attester private key that will be shared between two malicious proposer nodes - // We'll use validator index 0 for the "malicious" proposer validator key - const maliciousProposerIndex = 0; - const maliciousProposerPrivateKey = getPrivateKeyFromIndex( - ATTESTER_PRIVATE_KEYS_START_INDEX + maliciousProposerIndex, - )!; - const maliciousProposerAddress = EthAddress.fromString( - privateKeyToAccount(`0x${maliciousProposerPrivateKey.toString('hex')}`).address, - ); - - t.logger.warn(`Malicious proposer address: ${maliciousProposerAddress.toString()}`); - - // Create two nodes with the SAME validator key but DIFFERENT coinbase addresses - // This will cause them to create proposals with different content for the same slot - // Additionally, enable attestToEquivocatedProposals so they will attest to BOTH proposals - const maliciousProposerPrivateKeyHex = bufferToHex(maliciousProposerPrivateKey); - const coinbase1 = EthAddress.random(); - const coinbase2 = EthAddress.random(); - - t.logger.warn(`Creating malicious proposer node 1 with coinbase ${coinbase1.toString()}`); - const maliciousNode1 = await createNode( - { - ...t.ctx.aztecNodeConfig, - validatorPrivateKey: maliciousProposerPrivateKeyHex, - coinbase: coinbase1, - attestToEquivocatedProposals: true, // Attest to all proposals - creates duplicate attestations - broadcastEquivocatedProposals: true, // Don't abort checkpoint building on duplicate block proposals - dontStartSequencer: true, - // Prevent HA peer proposals from being added to the archiver, so both - // malicious nodes build their own blocks instead of one yielding to the other. - skipPushProposedBlocksToArchiver: true, - }, - t.ctx.dateProvider!, - BOOT_NODE_UDP_PORT + 1, - t.bootstrapNodeEnr, - maliciousProposerIndex, - t.genesis, - `${DATA_DIR}-0`, - shouldCollectMetrics(), - ); - - t.logger.warn(`Creating malicious proposer node 2 with coinbase ${coinbase2.toString()}`); - const maliciousNode2 = await createNode( - { - ...t.ctx.aztecNodeConfig, - validatorPrivateKey: maliciousProposerPrivateKeyHex, - coinbase: coinbase2, - attestToEquivocatedProposals: true, // Attest to all proposals - creates duplicate attestations - broadcastEquivocatedProposals: true, // Don't abort checkpoint building on duplicate block proposals - dontStartSequencer: true, - // Prevent HA peer proposals from being added to the archiver, so both - // malicious nodes build their own blocks instead of one yielding to the other. - skipPushProposedBlocksToArchiver: true, - }, - t.ctx.dateProvider!, - BOOT_NODE_UDP_PORT + 2, - t.bootstrapNodeEnr, - maliciousProposerIndex, - t.genesis, - `${DATA_DIR}-1`, - shouldCollectMetrics(), - ); - - // Create honest nodes with unique validator keys (indices 1 and 2) - t.logger.warn('Creating honest nodes'); - const honestNode1 = await createNode( - { - ...t.ctx.aztecNodeConfig, - dontStartSequencer: true, - }, - t.ctx.dateProvider!, - BOOT_NODE_UDP_PORT + 3, - t.bootstrapNodeEnr, - 1, - t.genesis, - `${DATA_DIR}-2`, - shouldCollectMetrics(), - ); - const honestNode2 = await createNode( - { - ...t.ctx.aztecNodeConfig, - dontStartSequencer: true, - }, - t.ctx.dateProvider!, - BOOT_NODE_UDP_PORT + 4, - t.bootstrapNodeEnr, - 2, - t.genesis, - `${DATA_DIR}-3`, - shouldCollectMetrics(), - ); - - nodes = [maliciousNode1, maliciousNode2, honestNode1, honestNode2]; - - // Wait for P2P mesh on all needed topics before starting sequencers - await t.waitForP2PMeshConnectivity(nodes, NUM_VALIDATORS, 30, 0.1, [ - TopicType.tx, - TopicType.block_proposal, - TopicType.checkpoint_proposal, - ]); - await awaitCommitteeExists({ rollup, logger: t.logger }); - - // Find an epoch where the malicious proposer is selected, stopping one epoch before - // so we have time to start sequencers before the target epoch arrives - const epochCache = (honestNode1 as TestAztecNodeService).epochCache; - const { targetEpoch, targetSlot } = await advanceToEpochBeforeProposer({ - epochCache, - cheatCodes: t.ctx.cheatCodes.rollup, - targetProposer: maliciousProposerAddress, - logger: t.logger, - }); - - // Start all sequencers while still one epoch before the target - t.logger.warn('Starting all sequencers'); - await Promise.all(nodes.map(n => n.getSequencer()!.start())); - - // Now warp to one slot before the target epoch — sequencers are already running. The helper - // picks a target slot at least one slot into the epoch, so warping here (rather than to the - // epoch start) leaves the freshly-started sequencers a full warm-up slot before the pipelined - // build for the malicious slot begins. Without that margin the duplicate proposals serialize - // past the slot boundary and receivers reject them as late, so the malicious nodes never get to - // attest to both and no duplicate attestation is produced. - t.logger.warn(`Advancing to one slot before target epoch ${targetEpoch} (target slot ${targetSlot})`); - await t.ctx.cheatCodes.rollup.advanceToEpoch(targetEpoch, { offset: -AZTEC_SLOT_DURATION }); - - // Wait for offenses to be detected - // We expect BOTH duplicate proposal AND duplicate attestation offenses - // The malicious proposer nodes create duplicate proposals (same key, different coinbase) - // The malicious proposer nodes also create duplicate attestations (attestToEquivocatedProposals enabled) - t.logger.warn('Waiting for duplicate attestation offense to be detected...'); - const offenses = await awaitOffenseDetected({ - epochDuration: t.ctx.aztecNodeConfig.aztecEpochDuration, - logger: t.logger, - nodeAdmin: honestNode1, // Use honest node to check for offenses - slashingRoundSize, - waitUntilOffenseCount: 2, // Wait for both duplicate proposal and duplicate attestation - timeoutSeconds: AZTEC_SLOT_DURATION * 16, - }); - - t.logger.warn(`Collected offenses`, { offenses }); - - // Verify we have detected the duplicate attestation offense - const duplicateAttestationOffenses = offenses.filter( - offense => offense.offenseType === OffenseType.DUPLICATE_ATTESTATION, - ); - const duplicateProposalOffenses = offenses.filter( - offense => offense.offenseType === OffenseType.DUPLICATE_PROPOSAL, - ); - - t.logger.info(`Found ${duplicateAttestationOffenses.length} duplicate attestation offenses`); - t.logger.info(`Found ${duplicateProposalOffenses.length} duplicate proposal offenses`); - - // We should have at least one duplicate attestation offense - expect(duplicateAttestationOffenses.length).toBeGreaterThan(0); - - // Verify the duplicate attestation offense is from the malicious proposer address - // (since they are the ones with attestToEquivocatedProposals enabled) - for (const offense of duplicateAttestationOffenses) { - expect(offense.offenseType).toEqual(OffenseType.DUPLICATE_ATTESTATION); - expect(offense.validator.toString()).toEqual(maliciousProposerAddress.toString()); - } - - // Verify that for each duplicate attestation offense, the attester for that slot is the malicious validator - for (const offense of duplicateAttestationOffenses) { - const offenseSlot = SlotNumber(Number(offense.epochOrSlot)); - const committeeInfo = await epochCache.getCommittee(offenseSlot); - t.logger.info(`Offense slot ${offenseSlot}: committee includes attester ${maliciousProposerAddress.toString()}`); - expect(committeeInfo.committee?.map(addr => addr.toString())).toContain(maliciousProposerAddress.toString()); - } - - t.logger.warn('Duplicate attestation offense correctly detected and recorded'); - }); -}); diff --git a/yarn-project/end-to-end/src/e2e_p2p/duplicate_proposal_slash.test.ts b/yarn-project/end-to-end/src/e2e_p2p/duplicate_proposal_slash.test.ts deleted file mode 100644 index 82675b476581..000000000000 --- a/yarn-project/end-to-end/src/e2e_p2p/duplicate_proposal_slash.test.ts +++ /dev/null @@ -1,291 +0,0 @@ -import type { AztecNodeService } from '@aztec/aztec-node'; -import type { TestAztecNodeService } from '@aztec/aztec-node/test'; -import { EthAddress } from '@aztec/aztec.js/addresses'; -import { EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; -import { retryUntil } from '@aztec/foundation/retry'; -import { bufferToHex } from '@aztec/foundation/string'; -import { OffenseType } from '@aztec/slasher'; -import { TopicType } from '@aztec/stdlib/p2p'; - -import { jest } from '@jest/globals'; -import fs from 'fs'; -import os from 'os'; -import path from 'path'; -import { privateKeyToAccount } from 'viem/accounts'; - -import { shouldCollectMetrics } from '../fixtures/fixtures.js'; -import { ATTESTER_PRIVATE_KEYS_START_INDEX, createNode } from '../fixtures/setup_p2p_test.js'; -import { getPrivateKeyFromIndex } from '../fixtures/utils.js'; -import { P2PNetworkTest } from './p2p_network.js'; -import { advanceToEpochBeforeProposer, awaitCommitteeExists, awaitOffenseDetected } from './shared.js'; - -const TEST_TIMEOUT = 600_000; // 10 minutes - -jest.setTimeout(TEST_TIMEOUT); - -const NUM_VALIDATORS = 4; -const BOOT_NODE_UDP_PORT = 4500; -const COMMITTEE_SIZE = NUM_VALIDATORS; -const ETHEREUM_SLOT_DURATION = 8; -const AZTEC_SLOT_DURATION = ETHEREUM_SLOT_DURATION * 3; -const BLOCK_DURATION = 4; - -const DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'duplicate-proposal-slash-')); - -/** - * Test that slashing occurs when a validator sends duplicate proposals (equivocation). - * - * The setup of the test is as follows: - * 1. Create 4 validator nodes total: - * - 2 honest validators with unique keys - * - 2 "malicious" validators that share the SAME validator key but have DIFFERENT coinbase addresses - * 2. The two nodes with the same key will both detect they are proposers for the same slot and naturally race to propose - * 3. Since they have different coinbase addresses, their proposals will have different archives (different content) - * 4. Other validators will detect the duplicate and emit a slash event - * - * Setup: P2PNetworkTest with mockGossipSubNetwork:true (in-memory bus, NOT real libp2p). 4 validators, - * ethSlot=8s, aztecSlot=24s, epoch=2, proofSubEpochs=1024, minTxsPerBlock=0, inboxLag=2 (v5 always enforces - * the timetable, so the former enforceTimeTable override is gone). - * Candidate for relocation to e2e_slashing/. - */ -describe('e2e_p2p_duplicate_proposal_slash', () => { - let t: P2PNetworkTest; - let nodes: AztecNodeService[]; - - // Small slashing unit so we don't kick anyone out - const slashingUnit = BigInt(1e14); - const slashingQuorum = 3; - const slashingRoundSize = 4; - const aztecEpochDuration = 2; - - beforeEach(async () => { - t = await P2PNetworkTest.create({ - testName: 'e2e_p2p_duplicate_proposal_slash', - numberOfNodes: 0, - numberOfValidators: NUM_VALIDATORS, - basePort: BOOT_NODE_UDP_PORT, - metricsPort: shouldCollectMetrics(), - initialConfig: { - anvilSlotsInAnEpoch: 4, - listenAddress: '127.0.0.1', - aztecEpochDuration, - ethereumSlotDuration: ETHEREUM_SLOT_DURATION, - aztecSlotDuration: AZTEC_SLOT_DURATION, - aztecTargetCommitteeSize: COMMITTEE_SIZE, - aztecProofSubmissionEpochs: 1024, // effectively do not reorg - slashInactivityConsecutiveEpochThreshold: 32, // effectively do not slash for inactivity - minTxsPerBlock: 0, // always be building - mockGossipSubNetwork: true, // do not worry about p2p connectivity issues - slashingQuorum, - slashingRoundSizeInEpochs: slashingRoundSize / aztecEpochDuration, - slashAmountSmall: slashingUnit, - slashAmountMedium: slashingUnit * 2n, - slashAmountLarge: slashingUnit * 3n, - blockDurationMs: BLOCK_DURATION * 1000, - slashDuplicateProposalPenalty: slashingUnit, - slashingOffsetInRounds: 1, - inboxLag: 2, - }, - }); - - await t.setup(); - await t.applyBaseSetup(); - }); - - afterEach(async () => { - await t.stopNodes(nodes); - await t.teardown(); - for (let i = 0; i < NUM_VALIDATORS; i++) { - fs.rmSync(`${DATA_DIR}-${i}`, { recursive: true, force: true, maxRetries: 3 }); - } - }); - - const debugRollup = async () => { - await t.ctx.cheatCodes.rollup.debugRollup(); - }; - - // Two malicious nodes share a validator key but have different coinbase addresses so their proposals - // differ. Honest nodes receive both proposals via mock gossip, detect the equivocation, and record a - // DUPLICATE_PROPOSAL offense. The test collects offenses from all nodes (equivocation may only be - // observed by whichever node processed both proposals before the slot closed) and asserts the offense - // is attributed to the shared key's address. - it('slashes validator who sends duplicate proposals', async () => { - const { rollup } = await t.getContracts(); - - // Jump forward to an epoch in the future such that the validator set is not empty - await t.ctx.cheatCodes.rollup.advanceToEpoch(EpochNumber(4)); - await debugRollup(); - - t.logger.warn('Creating nodes'); - - // Get the attester private key that will be shared between two malicious nodes - // We'll use validator index 0 for the "malicious" validator key - const maliciousValidatorIndex = 0; - const maliciousValidatorPrivateKey = getPrivateKeyFromIndex( - ATTESTER_PRIVATE_KEYS_START_INDEX + maliciousValidatorIndex, - )!; - const maliciousValidatorAddress = EthAddress.fromString( - privateKeyToAccount(`0x${maliciousValidatorPrivateKey.toString('hex')}`).address, - ); - - t.logger.warn(`Malicious proposer address: ${maliciousValidatorAddress.toString()}`); - - // Create two nodes with the SAME validator key but DIFFERENT coinbase addresses - // This will cause them to create proposals with different content for the same slot - const maliciousPrivateKeyHex = bufferToHex(maliciousValidatorPrivateKey); - const coinbase1 = EthAddress.random(); - const coinbase2 = EthAddress.random(); - - t.logger.warn(`Creating malicious node 1 with coinbase ${coinbase1.toString()}`); - const maliciousNode1 = await createNode( - { - ...t.ctx.aztecNodeConfig, - validatorPrivateKey: maliciousPrivateKeyHex, - coinbase: coinbase1, - broadcastEquivocatedProposals: true, - dontStartSequencer: true, - // Prevent HA peer proposals from being added to the archiver, so both - // malicious nodes build their own blocks instead of one yielding to the other. - skipPushProposedBlocksToArchiver: true, - }, - t.ctx.dateProvider, - BOOT_NODE_UDP_PORT + 1, - t.bootstrapNodeEnr, - maliciousValidatorIndex, - t.genesis, - `${DATA_DIR}-0`, - shouldCollectMetrics(), - ); - - t.logger.warn(`Creating malicious node 2 with coinbase ${coinbase2.toString()}`); - const maliciousNode2 = await createNode( - { - ...t.ctx.aztecNodeConfig, - validatorPrivateKey: maliciousPrivateKeyHex, - coinbase: coinbase2, - broadcastEquivocatedProposals: true, - dontStartSequencer: true, - // Prevent HA peer proposals from being added to the archiver, so both - // malicious nodes build their own blocks instead of one yielding to the other. - skipPushProposedBlocksToArchiver: true, - }, - t.ctx.dateProvider, - BOOT_NODE_UDP_PORT + 2, - t.bootstrapNodeEnr, - maliciousValidatorIndex, - t.genesis, - `${DATA_DIR}-1`, - shouldCollectMetrics(), - ); - - // Create honest nodes with unique validator keys (indices 1 and 2) - t.logger.warn('Creating honest nodes'); - const honestNode1 = await createNode( - { - ...t.ctx.aztecNodeConfig, - dontStartSequencer: true, - }, - t.ctx.dateProvider, - BOOT_NODE_UDP_PORT + 3, - t.bootstrapNodeEnr, - 1, - t.genesis, - `${DATA_DIR}-2`, - shouldCollectMetrics(), - ); - const honestNode2 = await createNode( - { - ...t.ctx.aztecNodeConfig, - dontStartSequencer: true, - }, - t.ctx.dateProvider, - BOOT_NODE_UDP_PORT + 4, - t.bootstrapNodeEnr, - 2, - t.genesis, - `${DATA_DIR}-3`, - shouldCollectMetrics(), - ); - - nodes = [maliciousNode1, maliciousNode2, honestNode1, honestNode2]; - - // Wait for P2P mesh on all needed topics before starting sequencers - await t.waitForP2PMeshConnectivity(nodes, NUM_VALIDATORS, 30, 0.1, [ - TopicType.tx, - TopicType.block_proposal, - TopicType.checkpoint_proposal, - ]); - await awaitCommitteeExists({ rollup, logger: t.logger }); - - // Find an epoch where the malicious proposer is selected, stopping one epoch before - // so we have time to start sequencers before the target epoch arrives - const epochCache = (honestNode1 as TestAztecNodeService).epochCache; - const { targetEpoch, targetSlot } = await advanceToEpochBeforeProposer({ - epochCache, - cheatCodes: t.ctx.cheatCodes.rollup, - targetProposer: maliciousValidatorAddress, - logger: t.logger, - }); - - // Start all sequencers while still one epoch before the target - t.logger.warn('Starting all sequencers'); - await Promise.all(nodes.map(n => n.getSequencer()!.start())); - - // Now warp to one slot before the target epoch — sequencers are already running. - // Under proposer pipelining, the malicious proposers begin building for their slot one slot - // earlier; warping to the start of the epoch would force both AVM-heavy duplicate proposals to - // serialize past the slot boundary, after which honest receivers reject them as late. The helper - // picks a target slot at least one slot into the epoch, so warping here leaves a full warm-up - // slot before the build begins rather than starting it at the exact instant of the warp. - t.logger.warn(`Advancing to one slot before target epoch ${targetEpoch} (target slot ${targetSlot})`); - await t.ctx.cheatCodes.rollup.advanceToEpoch(targetEpoch, { offset: -AZTEC_SLOT_DURATION }); - - // Wait for offense to be detected. Under proposer pipelining, checkpoint proposals are broadcast - // at the slot boundary while the receivers' wall clocks may have already advanced past the build - // slot — when that happens, honest nodes reject the gossip with "invalid slot number" before - // duplicate detection runs, so DUPLICATE_PROPOSAL is only observed by whichever node managed to - // process both proposals while still in the build slot (often the other malicious node, since - // they receive each other's broadcasts immediately). We therefore collect offenses from every - // node in the network and assert that at least one of them recorded the duplicate proposal. - t.logger.warn('Waiting for duplicate proposal offense to be detected...'); - await awaitOffenseDetected({ - epochDuration: t.ctx.aztecNodeConfig.aztecEpochDuration, - logger: t.logger, - nodeAdmin: honestNode1, - slashingRoundSize, - waitUntilOffenseCount: 1, - timeoutSeconds: AZTEC_SLOT_DURATION * 16, - }); - - // Poll every node for DUPLICATE_PROPOSAL offenses, retrying briefly so any node that detected - // the duplicate after the initial offense was collected has time to flush it through the - // slasher's offenses-collector. - const proposalOffenses = await retryUntil( - async () => { - const allOffenses = (await Promise.all(nodes.map(n => n.getSlashOffenses('all')))).flat(); - const filtered = allOffenses.filter(o => o.offenseType === OffenseType.DUPLICATE_PROPOSAL); - if (filtered.length > 0) { - return filtered; - } - }, - 'duplicate proposal offense', - AZTEC_SLOT_DURATION * 4, - ); - - t.logger.warn(`Collected duplicate proposal offenses`, { proposalOffenses }); - expect(proposalOffenses.length).toBeGreaterThan(0); - for (const offense of proposalOffenses) { - expect(offense.validator.toString()).toEqual(maliciousValidatorAddress.toString()); - } - - // Verify that for each offense, the proposer for that slot is the malicious validator - for (const offense of proposalOffenses) { - const offenseSlot = SlotNumber(Number(offense.epochOrSlot)); - const proposerForSlot = await epochCache.getProposerAttesterAddressInSlot(offenseSlot); - t.logger.info(`Offense slot ${offenseSlot}: proposer is ${proposerForSlot?.toString()}`); - expect(proposerForSlot?.toString()).toEqual(maliciousValidatorAddress.toString()); - } - - t.logger.warn('Duplicate proposal offense correctly detected and recorded'); - }); -}); diff --git a/yarn-project/end-to-end/src/e2e_slashing/attested_invalid_proposal.test.ts b/yarn-project/end-to-end/src/e2e_slashing/attested_invalid_proposal.test.ts index 6f222816300a..2da0dfcfcb62 100644 --- a/yarn-project/end-to-end/src/e2e_slashing/attested_invalid_proposal.test.ts +++ b/yarn-project/end-to-end/src/e2e_slashing/attested_invalid_proposal.test.ts @@ -121,7 +121,7 @@ async function advanceToEpochBeforePipelinedTargetSlot({ cheatCodes, targetProposer, logger, - maxAttempts = 30, + maxAttempts = 100, }: { epochCache: EpochCacheInterface; cheatCodes: RollupCheatCodes; diff --git a/yarn-project/end-to-end/src/fixtures/fixtures.ts b/yarn-project/end-to-end/src/fixtures/fixtures.ts index edc92ab04023..1b43900e2bcd 100644 --- a/yarn-project/end-to-end/src/fixtures/fixtures.ts +++ b/yarn-project/end-to-end/src/fixtures/fixtures.ts @@ -54,7 +54,7 @@ export const PIPELINING_SETUP_OPTS = { * Setup option preset that opts a test into the deterministic AutomineSequencer path. * Use only for single-sequencer tests that don't exercise block-building or consensus * (e.g. e2e_token, e2e_amm, e2e_authwit). Not compatible with `e2e_p2p/*`, - * `e2e_epochs/*`, `e2e_slashing/*`, `e2e_block_building`, or any multi-validator suite. + * `multi-node/*`, `e2e_slashing/*`, `e2e_block_building`, or any multi-validator suite. * * await setup(N, { ...AUTOMINE_E2E_OPTS, ...otherOpts }); * @@ -64,8 +64,10 @@ export const PIPELINING_SETUP_OPTS = { * serial queue (see `sequencer-client/src/sequencer/automine/automine_sequencer.ts`). * - Disables the validator client (the AutomineSequencer needs none). * - Uses `inboxLag: 1` (synchronous) since the AutomineSequencer publishes one block per tx. - * - Switches anvil into automine mode at setup time (no interval mining); each L1 tx - * mines an L1 block immediately. + * - Runs anvil at a 4s interval (`ethereumSlotDuration: 4`); at runtime the AutomineSequencer + * flips anvil into automine so each submitted tx mines its L1 block immediately. Initial L1 + * contract deployment (which runs before the sequencer starts) is mined immediately too via the + * global `automineL1Setup` default in `setup()`, instead of stalling on the 4s interval. * * Requires `aztecTargetCommitteeSize: 0`, which is the e2e default at `setup.ts:317`. */ diff --git a/yarn-project/end-to-end/src/fixtures/index.ts b/yarn-project/end-to-end/src/fixtures/index.ts index ad0e910fb298..b76e874fd0e8 100644 --- a/yarn-project/end-to-end/src/fixtures/index.ts +++ b/yarn-project/end-to-end/src/fixtures/index.ts @@ -3,4 +3,5 @@ export * from './ha_setup.js'; export * from './logging.js'; export * from './utils.js'; export * from './token_utils.js'; +export * from './wait_helpers.js'; export * from './with_telemetry_utils.js'; diff --git a/yarn-project/end-to-end/src/fixtures/setup.ts b/yarn-project/end-to-end/src/fixtures/setup.ts index dd62ceaffc17..a5f76ad14496 100644 --- a/yarn-project/end-to-end/src/fixtures/setup.ts +++ b/yarn-project/end-to-end/src/fixtures/setup.ts @@ -184,7 +184,11 @@ export type SetupOptions = { mockGossipSubNetwork?: boolean; /** Whether to add simulated latency to the mock gossipsub network (in ms) */ mockGossipSubNetworkLatency?: number; - /** Whether to enable anvil automine during deployment of L1 contracts (consider defaulting this to true). */ + /** + * Whether to mine the L1 setup txs (Multicall3 + rollup contract deployment) under anvil automine + * instead of waiting on the block interval. Defaults to `true` (set in `setupInner`); only suites + * that assert on genesis-relative L1 timing need to opt out with `false`. + */ automineL1Setup?: boolean; /** How many accounts to seed and unlock in anvil. */ anvilAccounts?: number; @@ -325,7 +329,12 @@ export async function setup( ): Promise { const setupStart = performance.now(); try { - return await setupInner(numberOfAccounts, opts, pxeOpts, chain); + const ctx = await setupInner(numberOfAccounts, opts, pxeOpts, chain); + if (process.env.EXIT_E2E_AFTER_SETUP) { + ctx.logger.info('EXIT_E2E_AFTER_SETUP is set; aborting before the test body runs'); + throw new Error('EXIT_E2E_AFTER_SETUP'); + } + return ctx; } finally { recordFnSpan('setup', performance.now() - setupStart); } @@ -338,10 +347,16 @@ async function setupInner( chain: Chain, ): Promise { assertContractArtifactsVersion(); + const logger = getLogger(); let anvil: Anvil | undefined; try { opts.aztecTargetCommitteeSize ??= 0; opts.slasherEnabled ??= false; + // Mine the L1 setup txs (Multicall3 + rollup contract deployment) immediately instead of + // waiting on anvil's block interval — this is the dominant cost of e2e setup. Suites that + // assert on genesis-relative L1 timing can opt out by passing `automineL1Setup: false`. + opts.automineL1Setup ??= true; + logger.trace('Starting e2e test setup'); const config: AztecNodeConfig & SetupOptions = { ...getConfigEnvVars(), ...opts }; // use initialValidators for the node config @@ -357,8 +372,6 @@ async function setupInner( config.minTxPoolAgeMs = opts.minTxPoolAgeMs ?? 0; - const logger = getLogger(); - // Create a temp directory for any services that need it and cleanup later const directoryToCleanup = path.join(tmpdir(), randomBytes(8).toString('hex')); await fs.mkdir(directoryToCleanup, { recursive: true }); @@ -382,6 +395,7 @@ async function setupInner( anvil = res.anvil; config.l1RpcUrls = [res.rpcUrl]; } + logger.trace('Started anvil and L1 RPC client'); // Enable logging metrics to a local file named after the test suite if (isMetricsLoggingRequested()) { @@ -398,6 +412,7 @@ async function setupInner( if (opts.l1StartTime) { await ethCheatCodes.warp(opts.l1StartTime, { resetBlockInterval: true }); } + logger.trace('Initialized L1 cheat codes and applied state/time overrides'); let publisherPrivKeyHex: `0x${string}` | undefined = undefined; let publisherHdAccount: HDAccount | PrivateKeyAccount | undefined = undefined; @@ -425,6 +440,7 @@ async function setupInner( if (config.coinbase === undefined) { config.coinbase = EthAddress.fromString(publisherHdAccount.address); } + logger.trace('Resolved L1 publisher account'); // The accounts setup creates itself: `numberOfAccounts` initializerless accounts, generated here and // funded at genesis so they are immediately usable. @@ -438,6 +454,7 @@ async function setupInner( const sponsoredFPCAddress = await getSponsoredFPCAddress(); addressesToFund.push(sponsoredFPCAddress); } + logger.trace('Generated test accounts to fund at genesis'); const genesisTimestamp = BigInt(Math.floor(Date.now() / 1000)); const { genesisArchiveRoot, genesis, fundingNeeded } = await getGenesisValues( @@ -446,6 +463,7 @@ async function setupInner( opts.genesisPublicData, genesisTimestamp, ); + logger.trace('Computed genesis values'); const wasAutomining = await ethCheatCodes.isAutoMining(); const enableAutomine = opts.automineL1Setup && !wasAutomining && isAnvilTestChain(chain.id); @@ -461,6 +479,7 @@ async function setupInner( // Force viem to refresh its nonce cache to avoid "nonce too low" errors in subsequent transactions // This is necessary because deployMulticall3 sends multiple transactions and viem may cache a stale nonce await l1Client.getTransactionCount({ address: l1Client.account.address }); + logger.trace('Deployed Multicall3'); const deployL1ContractsValues: DeployAztecL1ContractsReturnType = await deployAztecL1Contracts( config.l1RpcUrls[0], @@ -515,13 +534,16 @@ async function setupInner( if (opts.l2StartTime) { await ethCheatCodes.warp(opts.l2StartTime, { resetBlockInterval: true }); } + logger.trace('Deployed L1 rollup contracts'); // Use metricsPort-based telemetry if provided, otherwise use the regular telemetry client const telemetryClient = opts.metricsPort ? await getEndToEndTestTelemetryClient(opts.metricsPort) : await getTelemetryClient(opts.telemetryConfig); + logger.trace('Created telemetry client'); await setupSharedBlobStorage(config); + logger.trace('Set up shared blob storage'); logger.verbose('Creating and synching an aztec node', config); @@ -530,12 +552,14 @@ async function setupInner( config.acvmWorkingDirectory = acvmConfig.acvmWorkingDirectory; config.acvmBinaryPath = acvmConfig.acvmBinaryPath; } + logger.trace('Resolved ACVM config'); const bbConfig = await getBBConfig(logger); if (bbConfig) { config.bbBinaryPath = bbConfig.bbBinaryPath; config.bbWorkingDirectory = bbConfig.bbWorkingDirectory; } + logger.trace('Resolved Barretenberg config'); let mockGossipSubNetwork: MockGossipSubNetwork | undefined; let p2pClientDeps: P2PClientDeps | undefined = undefined; @@ -585,6 +609,7 @@ async function setupInner( ...(opts.mockGossipSubNetwork ? {} : { p2pEnabled: false, bootstrapNodes: [] as string[] }), } : config; + logger.trace('Prepared aztec node config'); const aztecNodeService = await withLoggerBindings({ actor: 'node-0' }, () => createAztecNodeService( @@ -594,6 +619,7 @@ async function setupInner( ), ); const sequencerClient = aztecNodeService.getSequencer(); + logger.trace('Created and synced aztec node'); let proverNode: AztecNodeService | undefined = undefined; if (opts.startProverNode) { @@ -617,6 +643,7 @@ async function setupInner( { dateProvider, p2pClientDeps, telemetry: telemetryClient }, { genesis }, )); + logger.trace('Created prover node'); } const sequencerDelayer = sequencerClient?.getDelayer(); @@ -637,8 +664,10 @@ async function setupInner( if (opts.walletMinFeePadding !== undefined) { wallet.setMinFeePadding(opts.walletMinFeePadding); } + logger.trace('Created PXE and test wallet'); const cheatCodes = await CheatCodes.create(config.l1RpcUrls, aztecNodeService, dateProvider); + logger.trace('Created cheat codes'); if ( (opts.aztecTargetCommitteeSize && opts.aztecTargetCommitteeSize > 0) || @@ -652,6 +681,7 @@ async function setupInner( ); await cheatCodes.rollup.setupEpoch(); await cheatCodes.rollup.debugRollup(); + logger.trace('Advanced chain to set up validator committee'); } let accounts: AztecAddress[] = []; @@ -663,6 +693,7 @@ async function setupInner( await createFundedInitializerlessAccounts(wallet, defaultAccounts); accounts = defaultAccounts.map(a => a.address); } + logger.trace('Created funded test accounts'); // Advancing past genesis needs a running sequencer to build the empty block; advancePastGenesis is // already false when skipInitialSequencer is set. @@ -678,6 +709,7 @@ async function setupInner( } else if (opts.skipInitialSequencer) { logger.info('Sequencer not started on initial node, skipping block progression'); } + logger.trace('Advanced chain past genesis'); // Now we restore the original minTxsPerBlock setting if we changed it. if (sequencerClient) { @@ -690,6 +722,7 @@ async function setupInner( config.buildCheckpointIfEmpty = originalBuildCheckpointIfEmpty; } } + logger.trace('Restored sequencer config'); const teardown = async () => { const teardownStart = performance.now(); diff --git a/yarn-project/end-to-end/src/fixtures/wait_helpers.ts b/yarn-project/end-to-end/src/fixtures/wait_helpers.ts new file mode 100644 index 000000000000..326eb9de3d22 --- /dev/null +++ b/yarn-project/end-to-end/src/fixtures/wait_helpers.ts @@ -0,0 +1,163 @@ +import type { WaitOpts } from '@aztec/aztec.js/contracts'; +import type { Fr } from '@aztec/aztec.js/fields'; +import { waitForTx } from '@aztec/aztec.js/node'; +import { INITIAL_L2_BLOCK_NUM } from '@aztec/aztec.js/protocol'; +import type { BlockNumber, CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types'; +import { retryUntil } from '@aztec/foundation/retry'; +import type { L2BlockTag } from '@aztec/stdlib/block'; +import type { AztecNode, CheckpointTag } from '@aztec/stdlib/interfaces/client'; +import type { L2ToL1MembershipWitness } from '@aztec/stdlib/messaging'; +import type { TxHash, TxReceipt } from '@aztec/stdlib/tx'; + +/** Options for the block-number polling helpers. */ +export type WaitForBlockOpts = { + /** Which chain tip to read; defaults to 'proposed'. */ + tag?: L2BlockTag; + /** Seconds before the poll rejects; defaults to 60. */ + timeout?: number; + /** Seconds between polls; defaults to 1. */ + interval?: number; +}; + +/** + * Polls `node.getBlockNumber(tag)` until it reaches `target`. Replaces the ad-hoc + * `retryUntil(() => node.getBlockNumber(tag) >= target)` polls scattered across the suite. + * @returns The block number once it reaches `target`. + */ +export function waitForBlockNumber(node: AztecNode, target: number, opts: WaitForBlockOpts = {}): Promise { + const tag = opts.tag ?? 'proposed'; + // Wrap the matched value: retryUntil treats any falsy return as "keep polling", so a legitimate + // match of block 0 (e.g. a freshly-pruned tip) would otherwise loop until timeout. + return retryUntil( + async () => { + const blockNumber = await node.getBlockNumber(tag); + return blockNumber >= target ? { blockNumber } : undefined; + }, + `block ${tag} >= ${target}`, + opts.timeout ?? 60, + opts.interval ?? 1, + ).then(({ blockNumber }) => blockNumber); +} + +/** Convenience for {@link waitForBlockNumber} on the proven tip. */ +export function waitForProvenBlock( + node: AztecNode, + target: number, + opts: Omit = {}, +): Promise { + return waitForBlockNumber(node, target, { ...opts, tag: 'proven' }); +} + +/** Compares the node's checkpoint number against the target; defaults to `>=`. */ +export type CheckpointComparator = (actual: number, target: number) => boolean; + +/** Options for {@link waitForNodeCheckpoint}. */ +export type WaitForCheckpointOpts = { + /** Which checkpoint tip to read; defaults to 'checkpointed'. */ + tag?: CheckpointTag; + /** How the node's checkpoint number must relate to `target`; defaults to `(actual, target) => actual >= target`. */ + compare?: CheckpointComparator; + /** Seconds before the poll rejects; defaults to 30. */ + timeout?: number; + /** Seconds between polls; defaults to 0.5. */ + interval?: number; +}; + +/** + * Polls a single node's checkpoint number until `opts.compare(actual, target)` holds. Replaces the + * `retryUntil(() => node.getChainTips().then(tips => tips..checkpoint.number target))` polls + * duplicated across the reorg/proving tests, where the node may sync forward, prune backward, or land + * exactly on a value — the caller passes the comparator lambda for whichever it expects. + * @returns The node's checkpoint number once the comparison holds. + */ +export function waitForNodeCheckpoint( + node: AztecNode, + target: number, + opts: WaitForCheckpointOpts = {}, +): Promise { + const tag = opts.tag ?? 'checkpointed'; + const compare = opts.compare ?? ((actual, target) => actual >= target); + // Wrap the matched value: retryUntil treats any falsy return as "keep polling", so a legitimate + // match of checkpoint 0 (e.g. proven === 0 or checkpointed <= 1 after a prune) would otherwise + // loop until timeout instead of resolving. + return retryUntil( + async () => { + const checkpointNumber = await node.getCheckpointNumber(tag); + return compare(checkpointNumber, target) ? { checkpointNumber } : undefined; + }, + `node checkpoint ${tag} ${compare} ${target}`, + opts.timeout ?? 30, + opts.interval ?? 0.5, + ).then(({ checkpointNumber }) => checkpointNumber); +} + +/** Convenience for {@link waitForNodeCheckpoint} on the proven tip. */ +export function waitForNodeProvenCheckpoint( + node: AztecNode, + target: number, + opts: Omit = {}, +): Promise { + return waitForNodeCheckpoint(node, target, { ...opts, tag: 'proven' }); +} + +/** + * Waits for all of `txHashes` to reach the desired status on `node`. The plural form of + * {@link waitForTx}; resolves with the receipts in input order. + */ +export function waitForTxs(node: AztecNode, txHashes: TxHash[], opts?: WaitOpts): Promise { + return Promise.all(txHashes.map(txHash => waitForTx(node, txHash, opts))); +} + +/** Options for {@link waitForBlocksAtSlots}. */ +export type WaitForBlocksAtSlotsOpts = { + /** Block number to start scanning from; defaults to the initial L2 block. */ + from?: BlockNumber; + /** How many blocks to fetch per poll; defaults to 10. */ + limit?: number; + /** Seconds before the poll rejects; defaults to 20. */ + timeout?: number; + /** Seconds between polls; defaults to 1. */ + interval?: number; +}; + +/** + * Polls `node.getBlocks` until every slot in `slots` is present among the fetched blocks' slot + * numbers. Replaces the hand-rolled `retryUntil` over `getBlocks(...).map(getSlot)` membership check. + */ +export async function waitForBlocksAtSlots( + node: AztecNode, + slots: SlotNumber[], + opts: WaitForBlocksAtSlotsOpts = {}, +): Promise { + const from = opts.from ?? INITIAL_L2_BLOCK_NUM; + const limit = opts.limit ?? 10; + await retryUntil( + async () => { + const blocks = await node.getBlocks(from, limit); + const foundSlots = blocks.map(block => block.header.getSlot()); + return slots.every(slot => foundSlots.includes(slot)) || undefined; + }, + `blocks at slots ${slots.join(', ')}`, + opts.timeout ?? 20, + opts.interval ?? 1, + ); +} + +/** + * Polls `node.getL2ToL1MembershipWitness(txHash, message)` until a witness is available, resolving + * with it. Wraps the `retryUntil` the cross-chain tests hand-roll while waiting for an L2-to-L1 + * message's membership witness to become provable. + */ +export function waitForL2ToL1Witness( + node: AztecNode, + txHash: TxHash, + message: Fr, + opts: { timeout?: number; interval?: number } = {}, +): Promise { + return retryUntil( + () => node.getL2ToL1MembershipWitness(txHash, message), + `L2-to-L1 membership witness for ${txHash.toString()}`, + opts.timeout ?? 30, + opts.interval ?? 1, + ); +} diff --git a/yarn-project/end-to-end/src/multi-node/README.md b/yarn-project/end-to-end/src/multi-node/README.md new file mode 100644 index 000000000000..6e61e275752f --- /dev/null +++ b/yarn-project/end-to-end/src/multi-node/README.md @@ -0,0 +1,94 @@ +# `multi-node` e2e test category + +Multi-node tests run N validator nodes sharing an in-memory `MockGossipSubNetwork` bus (no real +libp2p). This is the category for any multi-node test whose subject — proposals, attestations, +checkpointing, pruning/recovery, offense detection — is faithfully reproduced by the mock-gossip bus. +Tests that need *real networking* (peer discovery / discv5, the req/resp protocol, gossip mesh +formation, peer auth/scoring, transport behavior) belong in the `p2p` category instead. + +## Base class + +`MultiNodeTestContext` (`multi_node_test_context.ts`) extends `SingleNodeTestContext` (in +`../single-node/`) with the N-validator topology: + +- `createValidatorNode` / `createValidatorNodeAt(index)` spawn nodes on the mock-gossip bus; passing + the same index to two calls (with different coinbases) models an equivocating proposer that shares + a key across two nodes. Validators with no spawned node stay registered-but-offline. +- The per-validator registration accessors (`validatorAt` / `addressAt` / `privateKeyAt`) for the + validators registered at genesis via `initialValidators`. +- `getSlashingContracts()` for the rollup / slasher / slashing-proposer L1 contracts. +- The committee convergence helpers `waitForAllNodes*` and `findSlotsWithProposers`. + +The environment (in-process anvil + L1 deploy), the prover lifecycle, and the proving / reorg waiters +all live on the parent and are inherited. + +## Shared presets and helpers + +These are exported from `multi_node_test_context.ts` and spread into a `setup(...)` call rather than +copy-pasted: + +- `buildMockGossipValidators(n)` — the deterministic validator set (keys from + `getPrivateKeyFromIndex(i + 3)`), passed as `initialValidators`. +- `MOCK_GOSSIP_MULTI_VALIDATOR_OPTS` — a tight committee on the mock bus with no prover + (`{ mockGossipSubNetwork, skipInitialSequencer, startProverNode: false, aztecProofSubmissionEpochs: + 1024, numberOfAccounts: 0 }`). Tests that want a prover leave `startProverNode` explicit. +- `SLASHER_ENABLED_MULTI_VALIDATOR_OPTS` — the same committee with the slasher turned on, used by the + offense-detection tests. +- `defaultSlashingPenalties(unit?)` / `withOnlyOffense(offense, unit?)` — build the per-offense + `slash*Penalty` knobs; `withOnlyOffense` zeroes all but the named offense to isolate one offense. +- `setupHaPairs(test, validators, { baseOpts, coinbases })` — stands up two HA pairs (nodes 0/1 share + keys pk1+pk2, nodes 2/3 share pk3+pk4), each pair on a shared slashing-protection DB with distinct + per-node coinbases. Used by the `high-availability/` tests. + +## Organizing principle + +The top level groups tests by node topology and setup model; the second level names the primary +behavior under test, not the shared setup or a flag. Multiple-blocks-per-slot and proposer pipelining +are default traits of block production here, not separate categories. Each file has a single top-level +`describe` named to match its path, and a co-located `setup.ts` holds shared setup. A `.parallel` +suffix marks files with more than one top-level `it`; CI splits each `it` into its own job. + +## Subfolders + +| Folder | Base / preset | Contents | +|---|---|---| +| `block-production/` | `MultiNodeTestContext` + wide-slot / block-production timing (`setup.ts`) | Happy-path committee block production with multiple blocks per slot and pipelining as the default cadence: `simple`, `high_tps`, `first_slot` (blocks on the first two slots of an epoch), `deploy_and_call_ordering` (a contract deployed and called in separate blocks of one slot), `cross_chain_messages.parallel` (multi-block slots carrying L2→L1 and L1→L2 messages), `proposed_chain.parallel` (txs anchored to proposed blocks; non-validators re-execute and sync multi-block slots), `proof_boundary.parallel` (the proof-submission deadline vs. the pipelining boundary slot, across five proof-landing scenarios), `redistribution.parallel` (checkpoint budget redistributed so a late tx burst fits across the last blocks), `blob_promotion` (a promotion-disabled node fetches blobs while peers skip them, and a high-block-count checkpoint still proves). | +| `recovery/` | `MultiNodeTestContext` + wide-slot / block-production timing | The chain detects a bad/withheld/conflicting proposal and recovers: `proposal_failure_recovery.parallel` (all nodes prune and recover when a proposer fails to publish to L1), `pipeline_prune` (an uncheckpointed-blocks prune under pipelined MBPS, then recovery to a multi-block checkpoint), `equivocation_recovery` (an L1-confirmed checkpoint overrides a gossip-only equivocating proposal, the chain heals, and observers record the offense). | +| `invalid-attestations/` | `MultiNodeTestContext` (slasher on) | Invalid checkpoints are detected, invalidated on L1, and the chain progresses: `invalidate_block.parallel`, a six-validator suite injecting insufficient/fake/high-s/unrecoverable/shuffled attestations and withheld blobs. | +| `high-availability/` | `MultiNodeTestContext` + `setupHaPairs` | HA-pair sync and handoff between nodes that share validator keys: `ha_sync` (a peer that did not build a block syncs to the proposed chain tip over P2P), `ha_checkpoint_handoff` (a peer records and takes over a pipelined checkpoint when its partner proposes the previous slot). | +| `slashing/` | `MultiNodeTestContext` + `SLASHER_ENABLED_MULTI_VALIDATOR_OPTS` (`setup.ts`) | Pure offense detection: a validator equivocates and the slasher records the offense. `duplicate_proposal` and `duplicate_attestation`. | + +## Helper surface + +Prefer these named waiters over hand-rolled `retryUntil` / raw `.on` / `sleep` polling in test bodies. + +On `SingleNodeTestContext` (inherited): + +- `waitUntilEpochStarts(epoch)` / `waitUntilNextEpochStarts()` — epoch-boundary waiters. +- `waitUntilCheckpointNumber(n)` / `waitUntilProvenCheckpointNumber(n)` — checkpoint waiters. +- `waitUntilLastSlotOfProofSubmissionWindow(epoch)` — proof-window timing. +- `waitForNodeToSync(blockNumber, type)` — single-node sync wait. +- `watchSequencerEvents(sequencers, …)` accumulates state-changes and fail-events across sequencers; + `assertNoFailuresFromSequencers(failEvents)` asserts none fired. +- `waitForSequencerEvent(sequencer, event, match?, opts)` — one-shot wait for a matching event, with + timeout and listener cleanup. +- `assertMultipleBlocksPerSlot(n)` — asserts some checkpoint has at least `n` blocks (MBPS). + +Added by `MultiNodeTestContext`: + +- `waitForAllNodes(predicate, opts)` and the conveniences + `waitForAllNodesToReachProvenCheckpoint(target, opts)` and + `waitForAllNodesToReachBlockAtSlot(slot, tag, match?, opts)` — multi-node convergence. +- `findSlotsWithProposers(count, predicate, opts)` — finds N consecutive slots whose proposers + satisfy `predicate`, warping the L1 clock forward and retrying on `EpochNotStable`. + +On `ChainMonitor` (`@aztec/ethereum/test`): + +- `waitUntilCheckpoint(n)` / `waitUntilCheckpointProven(n)`, `waitUntilL2Slot(slot)`, + `waitUntilL1Block` / `waitUntilL1Timestamp`. + +Node-only / wallet-only waits (no context dependency) live in `../fixtures/wait_helpers.ts`: + +- `waitForBlockNumber(node, target, { tag })` / `waitForProvenBlock(node, target)`. +- `waitForNodeCheckpoint(node, target, opts)` / `waitForNodeProvenCheckpoint(node, target)`. +- `waitForTxs(node, txHashes, opts)` — the plural form of `waitForTx`. diff --git a/yarn-project/end-to-end/src/multi-node/block-production/blob_promotion.test.ts b/yarn-project/end-to-end/src/multi-node/block-production/blob_promotion.test.ts new file mode 100644 index 000000000000..096a59068d1b --- /dev/null +++ b/yarn-project/end-to-end/src/multi-node/block-production/blob_promotion.test.ts @@ -0,0 +1,180 @@ +import type { Archiver } from '@aztec/archiver'; +import type { AztecNodeConfig } from '@aztec/aztec-node'; +import { Fr } from '@aztec/aztec.js/fields'; +import { waitForTx } from '@aztec/aztec.js/node'; +import { asyncMap } from '@aztec/foundation/async-map'; +import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types'; +import { retryUntil } from '@aztec/foundation/retry'; +import { executeTimeout } from '@aztec/foundation/timer'; + +import type { TestWallet } from '../../test-wallet/test_wallet.js'; +import { proveAndSendTxs } from '../../test-wallet/utils.js'; +import { MultiNodeTestContext, buildMockGossipValidators } from '../multi_node_test_context.js'; +import { + type BlockProductionWithProverFixture, + NODE_COUNT, + WIDE_SLOT_TIMING, + jest, + waitForProvenCheckpoint, +} from './setup.js'; + +const PIPELINE_TX_COUNT = 34; +const PIPELINE_EXPECTED_BLOCKS_PER_CHECKPOINT = 8; + +// Blob/checkpoint promotion under stressed multi-block production: a node with promotion disabled +// fetches blobs while promotion-enabled peers fetch zero (the getBlobSidecar spy), and a +// high-block-count checkpoint built under adverse gossip latency still proves. The MBPS and pipelining +// offset assertions live in their behavior-named homes (production tests, pipeline_prune) and are not +// re-checked here. +describe('multi-node/block-production/blob_promotion', () => { + let fixture: BlockProductionWithProverFixture; + + afterEach(async () => { + jest.restoreAllMocks(); + await fixture?.test?.teardown(); + }); + + /** + * Sets up the pipelining wide-slot context: same timing profile as {@link setupBlockProductionWithProver} plus 500ms mock + * gossip latency, a tighter `maxTxsPerCheckpoint`, and node-0 with checkpoint promotion disabled so + * the blob-promotion behavior of the other nodes can be asserted against it. + */ + async function setupBlobPromotion(): Promise { + const validators = buildMockGossipValidators(NODE_COUNT); + + const test = await MultiNodeTestContext.setup({ + ...WIDE_SLOT_TIMING, + numberOfAccounts: 0, + initialValidators: validators, + mockGossipSubNetwork: true, + mockGossipSubNetworkLatency: 500, // adverse network conditions + startProverNode: true, + maxTxsPerCheckpoint: 24, + inboxLag: 2, + minTxsPerBlock: 1, + maxTxsPerBlock: 2, + pxeOpts: { syncChainTip: 'checkpointed' }, + skipInitialSequencer: true, + }); + + const { context, logger, rollup } = test; + const wallet = context.wallet as TestWallet; + const from = context.accounts[0]; // auto-created by setup + + logger.warn(`Initial setup complete. Starting ${NODE_COUNT} validator nodes.`); + // Clear inherited coinbase so each validator derives coinbase from its own attester key + const nodes = await asyncMap(validators, ({ privateKey }, i) => + test.createValidatorNode([privateKey], { + dontStartSequencer: true, + coinbase: undefined, + // Disable checkpoint promotion on the first node so it always fetches blobs, + // allowing us to assert that other nodes skip blob fetching via promotion. + ...(i === 0 ? { skipPromoteProposedCheckpointDuringL1Sync: true } : {}), + } as Partial), + ); + logger.warn(`Started ${NODE_COUNT} validator nodes.`, { validators: validators.map(v => v.attester.toString()) }); + + wallet.updateNode(nodes[0]); + const archiver = nodes[0].getBlockSource() as Archiver; + + const contract = await test.registerTestContract(wallet); + logger.warn(`Test setup completed.`, { validators: validators.map(v => v.attester.toString()) }); + + const { failEvents } = test.watchNodeSequencerEvents(nodes); + + return { test, context, logger, rollup, archiver, validators, nodes, contract, wallet, from, failEvents }; + } + + /** + * Waits until the archiver's checkpointed chain tip has reached `targetBlockNumber`, then retrieves all + * checkpoints and returns the number of the first one with at least `targetBlockCount` blocks. Used to + * pick a high-block-count checkpoint to assert proving against, not to re-assert MBPS itself. + */ + async function findMultiBlockCheckpoint( + targetBlockCount: number, + targetBlockNumber: BlockNumber, + ): Promise { + const { archiver, logger } = fixture; + await retryUntil( + async () => { + const checkpointed = await archiver.getBlockNumber({ tag: 'checkpointed' }); + return checkpointed !== undefined && checkpointed >= targetBlockNumber; + }, + `archiver checkpointed block ${targetBlockNumber}`, + 10, + 0.1, + ); + + const checkpoints = await archiver.getCheckpoints({ from: CheckpointNumber(1), limit: 50 }); + logger.warn(`Retrieved ${checkpoints.length} checkpoints from archiver`, { + checkpoints: checkpoints.map(pc => pc.checkpoint.getStats()), + }); + + const multiBlockCheckpoint = checkpoints.find(pc => pc.checkpoint.blocks.length >= targetBlockCount); + expect(multiBlockCheckpoint).toBeDefined(); + return multiBlockCheckpoint!.checkpoint.number; + } + + // Pre-proves TX_COUNT txs under adverse gossip latency, starts sequencers, waits for all txs to be + // mined, then verifies node-0 (promotion disabled) fetches blobs while nodes 1-3 (promotion enabled) + // skip blob fetching entirely, and that a high-block-count checkpoint built under load still proves. + it('promotion-disabled node fetches blobs while peers skip them, and the checkpoint proves', async () => { + fixture = await setupBlobPromotion(); + const { test, context, logger, nodes, contract, from } = fixture; + + // Spy on getBlobSidecar on all validator nodes before sequencers start, so we check that nodes + // promote their proposed checkpoints and don't source data from blobs if they don't need to. + const blobSpies = nodes.map((node, i) => { + const blobClient = node.getBlobClient()!; + const spy = jest.spyOn(blobClient, 'getBlobSidecar'); + logger.warn(`Installed getBlobSidecar spy on validator node ${i}`); + return spy; + }); + + const initialCheckpointNumber = await fixture.rollup.getCheckpointNumber(); + logger.warn(`Initial checkpoint number: ${initialCheckpointNumber}`); + + // Pre-prove and send transactions + const txHashes = await proveAndSendTxs( + context.wallet, + PIPELINE_TX_COUNT, + i => contract.methods.emit_nullifier(new Fr(i + 1)), + { from }, + ); + logger.warn(`Sent ${txHashes.length} transactions`, { txs: txHashes }); + + // Start the sequencers + await test.startSequencers(nodes); + logger.warn(`Started all sequencers`); + + // Wait until all txs are mined + const timeout = test.L2_SLOT_DURATION_IN_S * 5; + const receipts = await executeTimeout( + () => Promise.all(txHashes.map(txHash => waitForTx(context.aztecNode, txHash, { timeout }))), + timeout * 1000, + ); + logger.warn(`All txs have been mined`); + + // Pick a high-block-count checkpoint to assert proving against; target the highest mined block. + const maxMinedBlockNumber = BlockNumber(Math.max(...receipts.map(r => r.blockNumber ?? 0))); + const multiBlockCheckpoint = await findMultiBlockCheckpoint( + PIPELINE_EXPECTED_BLOCKS_PER_CHECKPOINT, + maxMinedBlockNumber, + ); + + // Verify blob fetching behavior: node 0 has promotion disabled so it must fetch blobs, + // while all other nodes should promote their proposed checkpoints and skip blob fetching entirely. + for (let i = 0; i < blobSpies.length; i++) { + const calls = blobSpies[i].mock.calls.length; + logger.warn(`Validator ${i} made ${calls} getBlobSidecar calls`); + if (i === 0) { + expect(calls).toBeGreaterThan(0); + } else { + expect(calls).toBe(0); + } + } + + // Verify proving still works end-to-end with pipelined proposers under stressed production. + await waitForProvenCheckpoint(fixture, multiBlockCheckpoint); + }); +}); diff --git a/yarn-project/end-to-end/src/multi-node/block-production/cross_chain_messages.parallel.test.ts b/yarn-project/end-to-end/src/multi-node/block-production/cross_chain_messages.parallel.test.ts new file mode 100644 index 000000000000..a2a3cb9f8635 --- /dev/null +++ b/yarn-project/end-to-end/src/multi-node/block-production/cross_chain_messages.parallel.test.ts @@ -0,0 +1,192 @@ +import { EthAddress } from '@aztec/aztec.js/addresses'; +import { generateClaimSecret } from '@aztec/aztec.js/ethereum'; +import { Fr } from '@aztec/aztec.js/fields'; +import { isL1ToL2MessageReady } from '@aztec/aztec.js/messaging'; +import { CheckpointNumber } from '@aztec/foundation/branded-types'; +import { timesAsync } from '@aztec/foundation/collection'; +import { retryUntil } from '@aztec/foundation/retry'; +import { executeTimeout } from '@aztec/foundation/timer'; +import { TestContract } from '@aztec/noir-test-contracts.js/Test'; + +import { sendL1ToL2Message } from '../../fixtures/l1_to_l2_messaging.js'; +import { waitForBlockNumber, waitForTxs } from '../../fixtures/wait_helpers.js'; +import { proveAndSendTxs } from '../../test-wallet/utils.js'; +import { + type BlockProductionWithProverFixture, + jest, + setupBlockProductionWithProver, + waitForProvenCheckpoint, +} from './setup.js'; + +const TX_COUNT = 10; + +// Cross-chain payloads survive multi-block production: L2→L1 message effects are present across the +// produced blocks, and L1→L2 messages become ready after inbox lag and their consume txs mine. Both +// run the shared MBPS pipelining context (4 validators + prover) from setup.ts. +describe('multi-node/block-production/cross_chain_messages', () => { + let fixture: BlockProductionWithProverFixture; + + afterEach(async () => { + jest.restoreAllMocks(); + await fixture?.test?.teardown(); + }); + + // Deploys a cross-chain TestContract, pre-proves TX_COUNT L2→L1 message txs, sends them all, waits + // for all to be mined, then asserts the total L2→L1 message count across all blocks ≥ TX_COUNT, + // a MBPS checkpoint exists, and that checkpoint is proven. + it('builds multiple blocks per slot with L2 to L1 messages', async () => { + fixture = await setupBlockProductionWithProver({ syncChainTip: 'proposed', minTxsPerBlock: 1, maxTxsPerBlock: 2 }); + const { test, context, logger, archiver, nodes, wallet, from } = fixture; + + // Start sequencers first, then deploy cross-chain contract (needs running sequencer to mine). + await test.startSequencers(nodes); + logger.warn(`Started all sequencers`); + + logger.warn(`Deploying cross-chain test contract`); + const { contract: crossChainContract } = await TestContract.deploy(wallet).send({ from }); + logger.warn(`Cross-chain test contract deployed at ${crossChainContract.address}`); + + // Pre-prove and send all L2→L1 message transactions at once + const l2ToL1Recipient = EthAddress.fromString(context.deployL1ContractsValues.l1Client.account.address); + logger.warn(`Pre-proving ${TX_COUNT} L2→L1 message transactions`); + const txHashes = await proveAndSendTxs( + wallet, + TX_COUNT, + () => crossChainContract.methods.create_l2_to_l1_message_arbitrary_recipient_public(Fr.random(), l2ToL1Recipient), + { from }, + ); + logger.warn(`Sent ${txHashes.length} L2→L1 message transactions`); + + // Wait until all txs are mined + const timeout = test.L2_SLOT_DURATION_IN_S * 5; + const receipts = await waitForTxs(context.aztecNode, txHashes, { timeout }); + logger.warn(`All L2→L1 message txs have been mined`); + + // wait for the other node to synch (nodes[0]'s block source is `archiver`) + const maxBlockNumber = Math.max(...receipts.map(r => r.blockNumber!)); + await waitForBlockNumber(nodes[0], maxBlockNumber, { + tag: 'checkpointed', + timeout: test.L2_SLOT_DURATION_IN_S * 3, + interval: 0.1, + }); + + // Mirror the sibling MBPS tests: we may lose one sub-slot to pipelined overhead, so accept >= 2 + // blocks per checkpoint rather than the legacy 3-block expectation. + const multiBlockCheckpoint = await fixture.test.assertMultipleBlocksPerSlot(2, { + wait: true, + archiver: fixture.archiver, + }); + + // Verify L2→L1 messages are in the blocks + const checkpoints = await archiver.getCheckpoints({ from: CheckpointNumber(1), limit: 50 }); + const allBlocks = checkpoints.flatMap(pc => pc.checkpoint.blocks); + const allL2ToL1Messages = allBlocks.flatMap(block => block.body.txEffects.flatMap(txEffect => txEffect.l2ToL1Msgs)); + logger.warn(`Found ${allL2ToL1Messages.length} L2→L1 message(s) across all blocks`, { allL2ToL1Messages }); + expect(allL2ToL1Messages.length).toBeGreaterThanOrEqual(TX_COUNT); + await waitForProvenCheckpoint(fixture, multiBlockCheckpoint); + }); + + // Seeds L1→L2 messages, sends filler txs to advance the chain so messages become ready, then + // pre-proves and sends consume txs. Verifies all consume txs are mined, a MBPS checkpoint exists, + // and that checkpoint is proven. + it('builds multiple blocks per slot with L1 to L2 messages', async () => { + // L1→L2 messages only become ready once the chain advances `inboxLag` checkpoints past where they + // were inboxed, and a checkpoint only advances when a block is built in a new slot. With + // skipInitialSequencer the chain won't move on its own, and a one-shot burst of filler txs lands + // within a single checkpoint — so let the sequencer keep building (empty) blocks each slot to drive + // the chain forward until the messages are ready. + fixture = await setupBlockProductionWithProver({ + syncChainTip: 'proposed', + minTxsPerBlock: 0, + maxTxsPerBlock: 1, + buildCheckpointIfEmpty: true, + }); + const { test, context, logger, nodes, contract, wallet, from } = fixture; + + // Start sequencers first, then deploy cross-chain contract (needs running sequencer to mine). + await test.startSequencers(nodes); + logger.warn(`Started all sequencers`); + + logger.warn(`Deploying cross-chain test contract`); + const { contract: crossChainContract } = await TestContract.deploy(wallet).send({ from }); + logger.warn(`Cross-chain test contract deployed at ${crossChainContract.address}`); + + const L1_TO_L2_COUNT = 4; + const FILLER_TX_COUNT = 5; // Enough txs to advance the chain so messages become ready + + // Seed all L1→L2 messages at the beginning + logger.warn(`Seeding ${L1_TO_L2_COUNT} L1→L2 messages`); + const l1ToL2Messages = await timesAsync(L1_TO_L2_COUNT, async i => { + const [secret, secretHash] = await generateClaimSecret(); + const content = Fr.random(); + const message = { recipient: crossChainContract.address, content, secretHash }; + + const { msgHash, globalLeafIndex } = await sendL1ToL2Message(message, { + l1Client: context.deployL1ContractsValues.l1Client, + l1ContractAddresses: context.deployL1ContractsValues.l1ContractAddresses, + }); + logger.warn(`L1→L2 message ${i + 1} sent with hash ${msgHash} and index ${globalLeafIndex}`); + + return { content, secret, msgHash, globalLeafIndex }; + }); + logger.warn(`Seeded ${l1ToL2Messages.length} L1→L2 messages`); + + // Pre-prove and send all filler txs at once (using unique nullifiers to avoid conflicts) + logger.warn(`Pre-proving ${FILLER_TX_COUNT} filler txs to advance the chain`); + const fillerTxHashes = await proveAndSendTxs( + wallet, + FILLER_TX_COUNT, + i => contract.methods.emit_nullifier(new Fr(1000 + i)), + { from }, + ); + logger.warn(`Sent ${fillerTxHashes.length} filler txs`); + + // Wait for filler txs to be mined first - this ensures the chain has advanced enough for messages to be ready + const timeout = test.L2_SLOT_DURATION_IN_S * 5; + await executeTimeout(() => waitForTxs(context.aztecNode, fillerTxHashes, { timeout }), timeout * 1000); + logger.warn(`All filler txs have been mined`); + + // Wait for all messages to be ready in parallel (chain has advanced, messages should be available) + const ethAccount = EthAddress.fromString(context.deployL1ContractsValues.l1Client.account.address); + await Promise.all( + l1ToL2Messages.map(async ({ msgHash }, i) => { + logger.warn(`Waiting for L1→L2 message ${i + 1} to be ready`); + await retryUntil( + () => isL1ToL2MessageReady(context.aztecNode, msgHash), + `L1→L2 message ${i + 1} ready`, + test.L2_SLOT_DURATION_IN_S * 5, + ); + logger.warn(`L1→L2 message ${i + 1} is ready`); + }), + ); + logger.warn(`All ${l1ToL2Messages.length} L1→L2 messages are ready`); + + // Pre-prove and send all consume transactions at once (proving up front avoids nonce conflicts) + logger.warn(`Pre-proving ${l1ToL2Messages.length} consume transactions`); + const consumeTxHashes = await proveAndSendTxs( + wallet, + l1ToL2Messages.length, + i => { + const { content, secret, globalLeafIndex } = l1ToL2Messages[i]; + return crossChainContract.methods.consume_message_from_arbitrary_sender_public( + content, + secret, + ethAccount, + globalLeafIndex, + ); + }, + { from }, + ); + logger.warn(`Sent ${consumeTxHashes.length} consume transactions`); + + // Wait for all consume txs to be mined + await waitForTxs(context.aztecNode, consumeTxHashes, { timeout }); + logger.warn(`All ${consumeTxHashes.length} L1→L2 messages consumed`); + + const multiBlockCheckpoint = await fixture.test.assertMultipleBlocksPerSlot(2, { + wait: true, + archiver: fixture.archiver, + }); + await waitForProvenCheckpoint(fixture, multiBlockCheckpoint); + }); +}); diff --git a/yarn-project/end-to-end/src/multi-node/block-production/deploy_and_call_ordering.test.ts b/yarn-project/end-to-end/src/multi-node/block-production/deploy_and_call_ordering.test.ts new file mode 100644 index 000000000000..aa9bb3a42ca8 --- /dev/null +++ b/yarn-project/end-to-end/src/multi-node/block-production/deploy_and_call_ordering.test.ts @@ -0,0 +1,127 @@ +import { NO_WAIT } from '@aztec/aztec.js/contracts'; +import { Fr } from '@aztec/aztec.js/fields'; +import { SlotNumber } from '@aztec/foundation/branded-types'; +import { retryUntil } from '@aztec/foundation/retry'; +import { sleep } from '@aztec/foundation/sleep'; +import { executeTimeout } from '@aztec/foundation/timer'; +import { TestContract } from '@aztec/noir-test-contracts.js/Test'; +import { getSlotAtTimestamp } from '@aztec/stdlib/epoch-helpers'; +import { GasFees } from '@aztec/stdlib/gas'; + +import { waitForTxs } from '../../fixtures/wait_helpers.js'; +import { proveInteraction } from '../../test-wallet/utils.js'; +import { + type BlockProductionWithProverFixture, + jest, + setupBlockProductionWithProver, + waitForProvenCheckpoint, +} from './setup.js'; + +describe('multi-node/block-production/deploy_and_call_ordering', () => { + let fixture: BlockProductionWithProverFixture; + + afterEach(async () => { + jest.restoreAllMocks(); + await fixture?.test?.teardown(); + }); + + // Pre-proves a high-priority deploy tx and a low-priority call tx for the same contract. Waits + // until just before the next L2 slot boundary, sends deploy first (then call after 1s), and + // waits for both to be checkpointed. Asserts deploy block < call block and both belong to the + // same checkpoint. Waits for that checkpoint to be proven. + it('deploys a contract and calls it in separate blocks within a slot', async () => { + fixture = await setupBlockProductionWithProver({ + syncChainTip: 'checkpointed', + minTxsPerBlock: 1, + maxTxsPerBlock: 1, + }); + const { test, context, logger, nodes, wallet, from } = fixture; + + // Prepare deploy tx for a new TestContract. Get the instance address so we can construct the call tx. + const highPriority = new GasFees(100, 100); + const lowPriority = new GasFees(1, 1); + + const deployMethod = TestContract.deploy(wallet, { deployer: from }); + const deployInstance = await deployMethod.getInstance(); + logger.warn(`Will deploy TestContract at ${deployInstance.address}`); + + // Register the contract on the PXE so we can prove the call interaction against it. + await wallet.registerContract(deployInstance, TestContract.artifact); + const deployedContract = TestContract.at(deployInstance.address, wallet); + + // Pre-prove both txs before starting sequencers. This ensures both arrive in the pool + // at the same time, so the sequencer can sort by priority fee for correct ordering. + logger.warn(`Pre-proving deploy tx (high priority) and call tx (low priority)`); + const deployTx = await proveInteraction(wallet, deployMethod, { + from, + fee: { gasSettings: { maxPriorityFeesPerGas: highPriority } }, + }); + const callTx = await proveInteraction(wallet, deployedContract.methods.emit_nullifier_public(new Fr(42)), { + from, + fee: { gasSettings: { maxPriorityFeesPerGas: lowPriority } }, + }); + logger.warn(`Pre-proved both txs`); + + // Start the sequencers + await test.startSequencers(nodes); + logger.warn(`Started all sequencers`); + + // Wait until one L1 slot before the start of the next L2 slot. + // This ensures both txs land in the pending pool right before the proposer starts building. + const currentL1Block = await test.l1Client.getBlock({ blockTag: 'latest' }); + const currentSlot = getSlotAtTimestamp(currentL1Block.timestamp, test.constants); + const nextSlot = SlotNumber(currentSlot + 1); + await test.waitForBuildWindowForSlot(nextSlot, { timeout: test.L2_SLOT_DURATION_IN_S * 3 }); + + // Send the deploy tx first and give it time to propagate to all validators, + // then send the call tx. Priority fees are a safety net, but arrival ordering + // ensures the deploy tx is in the pool before the call tx regardless of gossip timing. + const timeout = test.L2_SLOT_DURATION_IN_S * 5; + logger.warn(`Sending deploy tx first, then call tx`); + const deployTxHash = await deployTx.send({ wait: NO_WAIT }); + await sleep(1000); + const callTxHash = await callTx.send({ wait: NO_WAIT }); + const [deployReceipt, callReceipt] = await executeTimeout( + () => waitForTxs(context.aztecNode, [deployTxHash, callTxHash], { timeout }), + timeout * 1000, + ); + logger.warn(`Both txs checkpointed`, { + deployBlock: deployReceipt.blockNumber, + callBlock: callReceipt.blockNumber, + }); + + // Both txs should succeed (send throws on revert). Deploy should be in an earlier block. + expect(deployReceipt.blockNumber).toBeLessThan(callReceipt.blockNumber!); + + // Verify both blocks belong to the same checkpoint. + const deployCheckpointedBlock = await retryUntil( + async () => + ( + await context.aztecNode.getBlocks(deployReceipt.blockNumber!, 1, { + includeL1PublishInfo: true, + includeAttestations: true, + onlyCheckpointed: true, + }) + )[0], + 'deploy checkpointed block', + timeout, + ); + const callCheckpointedBlock = await retryUntil( + async () => + ( + await context.aztecNode.getBlocks(callReceipt.blockNumber!, 1, { + includeL1PublishInfo: true, + includeAttestations: true, + onlyCheckpointed: true, + }) + )[0], + 'call checkpointed block', + timeout, + ); + expect(deployCheckpointedBlock.checkpointNumber).toBe(callCheckpointedBlock.checkpointNumber); + logger.warn(`Both blocks in checkpoint ${deployCheckpointedBlock.checkpointNumber}`); + + // Wait for the checkpoint to be proven. + await waitForProvenCheckpoint(fixture, deployCheckpointedBlock.checkpointNumber); + }); +}); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_first_slot.test.ts b/yarn-project/end-to-end/src/multi-node/block-production/first_slot.test.ts similarity index 62% rename from yarn-project/end-to-end/src/e2e_epochs/epochs_first_slot.test.ts rename to yarn-project/end-to-end/src/multi-node/block-production/first_slot.test.ts index e820d2f05d63..237eb4f72d74 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_first_slot.test.ts +++ b/yarn-project/end-to-end/src/multi-node/block-production/first_slot.test.ts @@ -1,29 +1,26 @@ import type { AztecNodeService } from '@aztec/aztec-node'; import type { AztecAddress } from '@aztec/aztec.js/addresses'; -import { EthAddress } from '@aztec/aztec.js/addresses'; import { getTimestampRangeForEpoch } from '@aztec/aztec.js/block'; -import { NO_WAIT } from '@aztec/aztec.js/contracts'; -import { Fr } from '@aztec/aztec.js/fields'; import type { Logger } from '@aztec/aztec.js/log'; import { waitForTx } from '@aztec/aztec.js/node'; -import { INITIAL_L2_BLOCK_NUM } from '@aztec/aztec.js/protocol'; -import type { Operator } from '@aztec/ethereum/deploy-aztec-l1-contracts'; import { asyncMap } from '@aztec/foundation/async-map'; -import { BlockNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; -import { times, timesAsync } from '@aztec/foundation/collection'; -import { SecretValue } from '@aztec/foundation/config'; -import { retryUntil } from '@aztec/foundation/retry'; -import { bufferToHex } from '@aztec/foundation/string'; +import { EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; import { executeTimeout } from '@aztec/foundation/timer'; import type { SpamContract } from '@aztec/noir-test-contracts.js/Spam'; import { getSlotRangeForEpoch } from '@aztec/stdlib/epoch-helpers'; import { jest } from '@jest/globals'; -import { privateKeyToAccount } from 'viem/accounts'; -import { type EndToEndContext, getPrivateKeyFromIndex } from '../fixtures/utils.js'; -import { proveInteraction } from '../test-wallet/utils.js'; -import { EpochsTestContext } from './epochs_test.js'; +import type { EndToEndContext } from '../../fixtures/utils.js'; +import { waitForBlocksAtSlots } from '../../fixtures/wait_helpers.js'; +import { proveAndSendTxs } from '../../test-wallet/utils.js'; +import { + MOCK_GOSSIP_MULTI_VALIDATOR_OPTS, + MULTI_VALIDATOR_BLOCK_PRODUCTION_TIMING, + MultiNodeTestContext, + type RegisteredValidator, + buildMockGossipValidators, +} from '../multi_node_test_context.js'; jest.setTimeout(1000 * 60 * 10); @@ -36,43 +33,32 @@ const TX_COUNT = 8; // (one per sub-slot, maxTxsPerBlock=1), then warps L1 to just before an epoch boundary so // the pipelined proposer's first build window targets the epoch's first slot. Verifies that // blocks are built on both the first and second slots of the new epoch. -// Uses EpochsTestContext with mockGossipSubNetwork, no initial sequencer, no prover node. -describe('e2e_epochs/epochs_first_slot', () => { +// Uses MultiNodeTestContext with mockGossipSubNetwork, no initial sequencer, no prover node. +describe('multi-node/block-production/first_slot', () => { let context: EndToEndContext; let logger: Logger; - let test: EpochsTestContext; - let validators: (Operator & { privateKey: `0x${string}` })[]; + let test: MultiNodeTestContext; + let validators: RegisteredValidator[]; let nodes: AztecNodeService[]; let contract: SpamContract; let from: AztecAddress; beforeEach(async () => { - validators = times(NODE_COUNT, i => { - const privateKey = bufferToHex(getPrivateKeyFromIndex(i + 3)!); - const attester = EthAddress.fromString(privateKeyToAccount(privateKey).address); - return { attester, withdrawer: attester, privateKey, bn254SecretKey: new SecretValue(Fr.random().toBigInt()) }; - }); + validators = buildMockGossipValidators(NODE_COUNT); // Setup context with the given set of validators, no reorgs, and a mocked gossip sub network. // We expect 4 blocks per checkpoint with this config - test = await EpochsTestContext.setup({ - numberOfAccounts: 0, + test = await MultiNodeTestContext.setup({ + ...MOCK_GOSSIP_MULTI_VALIDATOR_OPTS, + ...MULTI_VALIDATOR_BLOCK_PRODUCTION_TIMING, initialValidators: validators, - mockGossipSubNetwork: true, - aztecProofSubmissionEpochs: 1024, aztecEpochDuration: 32, - aztecSlotDurationInL1Slots: 3, - ethereumSlotDuration: 12, - blockDurationMs: 6000, - startProverNode: false, aztecTargetCommitteeSize: COMMITTEE_SIZE, minTxsPerBlock: 1, maxTxsPerBlock: 1, attestationPropagationTime: 0.5, archiverPollingIntervalMS: 200, - skipInitialSequencer: true, - inboxLag: 2, }); ({ context, logger } = test); @@ -104,16 +90,14 @@ describe('e2e_epochs/epochs_first_slot', () => { it('builds blocks on the first two slots of the epoch', async () => { // Create and submit txs for the first two slots of the epoch // We set maxTxsPerBlock to 1, so two txs mean two consecutive blocks - const txs = await timesAsync(TX_COUNT, i => - proveInteraction(context.wallet, contract.methods.spam(i, 1n, false), { from }), - ); - const txHashes = await Promise.all(txs.map(tx => tx.send({ wait: NO_WAIT }))); + const txHashes = await proveAndSendTxs(context.wallet, TX_COUNT, i => contract.methods.spam(i, 1n, false), { + from, + }); logger.warn(`Sent ${txHashes.length} transactions`, { txs: txHashes, }); - const sequencers = nodes.map(node => node.getSequencer()!); - const { failEvents } = test.watchSequencerEvents(sequencers, i => ({ validator: validators[i].attester })); + const { failEvents } = test.watchNodeSequencerEvents(nodes); // Jump to the beginning of two epochs from now const currentEpoch = (await test.monitor.run()).l2EpochNumber; @@ -131,7 +115,7 @@ describe('e2e_epochs/epochs_first_slot', () => { }); // Start the sequencers - await Promise.all(sequencers.map(sequencer => sequencer.start())); + await test.startSequencers(nodes); logger.warn(`Started all sequencers`); // Wait until all txs are mined @@ -143,19 +127,7 @@ describe('e2e_epochs/epochs_first_slot', () => { const [firstSlot] = getSlotRangeForEpoch(epoch, test.constants); const secondSlot = SlotNumber(firstSlot + 1); logger.warn(`Waiting until blocks are synced for slots ${firstSlot} and ${secondSlot}`); - // REFACTOR: hand-rolled poll checking block slots; replace with a helper such as - // waitUntilBlocksForSlots(nodes[0], [firstSlot, secondSlot], timeout). - await retryUntil( - async () => { - const blocks = await nodes[0].getBlocks(BlockNumber(INITIAL_L2_BLOCK_NUM), 10); - const slots = blocks.map(block => block.header.getSlot()); - logger.info(`Fetched blocks ${blocks.map(b => b.number).join(', ')} with slots ${slots.join(', ')}`); - return slots.includes(firstSlot) && slots.includes(secondSlot); - }, - 'waiting for blocks', - 20, - 1, - ); + await waitForBlocksAtSlots(nodes[0], [firstSlot, secondSlot]); test.assertNoFailuresFromSequencers(failEvents); }); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_high_tps_block_building.test.ts b/yarn-project/end-to-end/src/multi-node/block-production/high_tps.test.ts similarity index 74% rename from yarn-project/end-to-end/src/e2e_epochs/epochs_high_tps_block_building.test.ts rename to yarn-project/end-to-end/src/multi-node/block-production/high_tps.test.ts index 4dbd40134084..10c119356fba 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_high_tps_block_building.test.ts +++ b/yarn-project/end-to-end/src/multi-node/block-production/high_tps.test.ts @@ -1,28 +1,17 @@ import type { AztecNodeService } from '@aztec/aztec-node'; import type { AztecAddress } from '@aztec/aztec.js/addresses'; -import { EthAddress } from '@aztec/aztec.js/addresses'; -import { NO_WAIT } from '@aztec/aztec.js/contracts'; -import { Fr } from '@aztec/aztec.js/fields'; import type { Logger } from '@aztec/aztec.js/log'; import { waitForTx } from '@aztec/aztec.js/node'; -import type { Operator } from '@aztec/ethereum/deploy-aztec-l1-contracts'; -import { asyncMap } from '@aztec/foundation/async-map'; import { BlockNumber, SlotNumber } from '@aztec/foundation/branded-types'; -import { chunkBy, times, timesAsync } from '@aztec/foundation/collection'; -import { SecretValue } from '@aztec/foundation/config'; +import { chunkBy } from '@aztec/foundation/collection'; import { sleepUntil } from '@aztec/foundation/sleep'; -import { bufferToHex } from '@aztec/foundation/string'; import type { SpamContract } from '@aztec/noir-test-contracts.js/Spam'; import { getSlotAtTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; -import { jest } from '@jest/globals'; -import { privateKeyToAccount } from 'viem/accounts'; - -import { type EndToEndContext, getPrivateKeyFromIndex } from '../fixtures/utils.js'; -import { proveInteraction } from '../test-wallet/utils.js'; -import { EpochsTestContext } from './epochs_test.js'; - -jest.setTimeout(1000 * 60 * 10); +import type { EndToEndContext } from '../../fixtures/utils.js'; +import { proveAndSendTxs } from '../../test-wallet/utils.js'; +import type { MultiNodeTestContext, RegisteredValidator } from '../multi_node_test_context.js'; +import { jest, setupSimpleBlockProduction } from './setup.js'; const NODE_COUNT = 3; @@ -32,7 +21,7 @@ const NODE_COUNT = 3; // Config: aztecSlotDuration=36s, ethereumSlotDuration=12s (3 L1 blocks / L2 slot), blockDuration=6s, // fakeProcessingDelayPerTxMs=2500ms, attestationPropagationTime=1s, // txDelayerMaxInclusionTimeIntoSlot=1s. (v5: the explicit l1PublishingTime override was dropped — -// EpochsTestContext no longer takes it; the publish window is now the framework default.) +// MultiNodeTestContext no longer takes it; the publish window is now the framework default.) // // Time inside a build slot (36s total): // T=0-1 (1s) init (checkpointInitializationTime) @@ -62,62 +51,39 @@ const TXS_PER_BLOCK = 2; const CHECKPOINTS_TO_CHECK = 3; // Extra txs beyond the ones we assert on: one partial checkpoint at startup (sequencers start mid-slot with // only one blockDuration of slack) plus a buffer at the tail. -const TX_COUNT = BLOCKS_PER_CHECKPOINT * TXS_PER_BLOCK * (CHECKPOINTS_TO_CHECK + 1); +const TX_COUNT_HIGH = BLOCKS_PER_CHECKPOINT * TXS_PER_BLOCK * (CHECKPOINTS_TO_CHECK + 1); const TX_DURATION_MS = 2500; const BLOCK_DURATION_MS = 6000; -const L2_SLOT_DURATION_S = 36; -const L1_BLOCK_TIME_S = 12; // Multi-block-per-slot suite verifying that 3 validator nodes can build fully-filled checkpoints // (4 blocks × 2 txs each) under proposer pipelining with fake tx processing delays. Asserts that // CHECKPOINTS_TO_CHECK consecutive checkpoints at or after the target slot each have at least // BLOCKS_PER_CHECKPOINT-1 blocks and that the checkpoint tx lands in the 1st or 2nd L1 block of the -// target slot. Uses EpochsTestContext with mockGossipSubNetwork, no initial sequencer, no prover node. -describe('e2e_epochs/epochs_high_tps_block_building', () => { +// target slot. mockGossipSubNetwork, no initial sequencer, no prover node. +describe('multi-node/block-production/high_tps', () => { let context: EndToEndContext; let logger: Logger; - let test: EpochsTestContext; - let validators: (Operator & { privateKey: `0x${string}` })[]; + let test: MultiNodeTestContext; + let validators: RegisteredValidator[]; let nodes: AztecNodeService[]; let contract: SpamContract; let from: AztecAddress; beforeEach(async () => { - validators = times(NODE_COUNT, i => { - const privateKey = bufferToHex(getPrivateKeyFromIndex(i + 3)!); - const attester = EthAddress.fromString(privateKeyToAccount(privateKey).address); - return { attester, withdrawer: attester, privateKey, bn254SecretKey: new SecretValue(Fr.random().toBigInt()) }; - }); - - test = await EpochsTestContext.setup({ - numberOfAccounts: 0, - initialValidators: validators, - mockGossipSubNetwork: true, - aztecProofSubmissionEpochs: 1024, - startProverNode: false, - ethereumSlotDuration: L1_BLOCK_TIME_S, - aztecSlotDuration: L2_SLOT_DURATION_S, - blockDurationMs: BLOCK_DURATION_MS, - fakeProcessingDelayPerTxMs: TX_DURATION_MS, - attestationPropagationTime: 1, - minTxsPerBlock: 1, - maxTxsPerBlock: 100, - skipInitialSequencer: true, - inboxLag: 2, - }); - - ({ context, logger } = test); - from = context.accounts[0]; // auto-created by setup - // Start the validator nodes. Note the txDelayerMaxInclusionTimeIntoSlot is set to 1s, // so the tx delayer will simulate the network not accepting a tx for the next block // unless it is sent within the first second of the L1 slot. - logger.warn(`Initial setup complete. Starting ${NODE_COUNT} validator nodes.`); - nodes = await asyncMap(validators, ({ privateKey }) => - test.createValidatorNode([privateKey], { dontStartSequencer: true, txDelayerMaxInclusionTimeIntoSlot: 1 }), - ); - logger.warn(`Started ${NODE_COUNT} validator nodes.`, { validators: validators.map(v => v.attester.toString()) }); + ({ test, context, logger, validators, nodes, from } = await setupSimpleBlockProduction({ + nodeCount: NODE_COUNT, + setupOpts: { + fakeProcessingDelayPerTxMs: TX_DURATION_MS, + attestationPropagationTime: 1, + minTxsPerBlock: 1, + maxTxsPerBlock: 100, + }, + nodeOpts: { dontStartSequencer: true, txDelayerMaxInclusionTimeIntoSlot: 1 }, + })); // Register spam contract for sending txs. contract = await test.registerSpamContract(context.wallet); @@ -133,16 +99,14 @@ describe('e2e_epochs/epochs_high_tps_block_building', () => { // build window is reachable. Starts all sequencers and waits for all txs to be mined. Groups // blocks by checkpoint number and for each checkpoint at or after the target slot asserts block // count, per-block tx count, and L1 submission offset. Expects zero fail events. - it('builds blocks without any errors', async () => { + it('builds high-tps blocks without any errors', async () => { // Pre-prove and send all txs so the proposer has a full backlog ready in the pool when it starts building. - const txs = await timesAsync(TX_COUNT, i => - proveInteraction(context.wallet, contract.methods.spam(i, 1n, false), { from }), - ); - const txHashes = await Promise.all(txs.map(tx => tx.send({ wait: NO_WAIT }))); + const txHashes = await proveAndSendTxs(context.wallet, TX_COUNT_HIGH, i => contract.methods.spam(i, 1n, false), { + from, + }); logger.warn(`Sent ${txHashes.length} transactions`, { txs: txHashes }); - const sequencers = nodes.map(node => node.getSequencer()!); - const { failEvents } = test.watchSequencerEvents(sequencers, i => ({ validator: validators[i].attester })); + const { failEvents } = test.watchNodeSequencerEvents(nodes); // Wait until `ethereumSlotDuration + blockDuration` seconds before the L2 target slot boundary before // starting the sequencers. The sequencer's timetable treats the build window for slot N as starting at @@ -167,7 +131,7 @@ describe('e2e_epochs/epochs_high_tps_block_building', () => { // such as test.waitUntilBuildWindowForSlot(targetSlot) that encapsulates lead-time arithmetic. await sleepUntil(startSequencersAt, context.dateProvider.nowAsDate()); - await Promise.all(sequencers.map(sequencer => sequencer.start())); + await test.startSequencers(nodes); logger.warn(`Started all sequencers`); // Wait until all txs are mined. diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_proof_at_boundary.parallel.test.ts b/yarn-project/end-to-end/src/multi-node/block-production/proof_boundary.parallel.test.ts similarity index 92% rename from yarn-project/end-to-end/src/e2e_epochs/epochs_proof_at_boundary.parallel.test.ts rename to yarn-project/end-to-end/src/multi-node/block-production/proof_boundary.parallel.test.ts index ba1cabdd509b..784c3e1a98c6 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_proof_at_boundary.parallel.test.ts +++ b/yarn-project/end-to-end/src/multi-node/block-production/proof_boundary.parallel.test.ts @@ -1,23 +1,23 @@ import type { AztecNodeService } from '@aztec/aztec-node'; -import { EthAddress } from '@aztec/aztec.js/addresses'; -import { Fr } from '@aztec/aztec.js/fields'; import type { Logger } from '@aztec/aztec.js/log'; -import type { Operator } from '@aztec/ethereum/deploy-aztec-l1-contracts'; import type { Delayer } from '@aztec/ethereum/l1-tx-utils'; import { asyncMap } from '@aztec/foundation/async-map'; import { CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; -import { times } from '@aztec/foundation/collection'; -import { SecretValue } from '@aztec/foundation/config'; import { retryUntil } from '@aztec/foundation/retry'; -import { bufferToHex } from '@aztec/foundation/string'; import type { SequencerClient, SequencerEvents } from '@aztec/sequencer-client'; import { getEpochAtSlot, getEpochNumberAtTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; import { jest } from '@jest/globals'; -import { privateKeyToAccount } from 'viem/accounts'; -import { type EndToEndContext, getPrivateKeyFromIndex } from '../fixtures/utils.js'; -import { EpochsTestContext, type EpochsTestOpts } from './epochs_test.js'; +import type { EndToEndContext } from '../../fixtures/utils.js'; +import { + MOCK_GOSSIP_MULTI_VALIDATOR_OPTS, + MULTI_VALIDATOR_BLOCK_PRODUCTION_TIMING, + MultiNodeTestContext, + type MultiNodeTestOpts, + type RegisteredValidator, + buildMockGossipValidators, +} from '../multi_node_test_context.js'; jest.setTimeout(1000 * 60 * 10); @@ -27,41 +27,30 @@ type PreparingEvent = Parameters[0]; type PublishedEvent = Parameters[0]; // Suite: 5 parallel scenarios testing the interaction between the proof submission deadline and -// the pipelining boundary slot. EpochsTestContext: 3 validator nodes + 1 prover node, +// the pipelining boundary slot. MultiNodeTestContext: 3 validator nodes + 1 prover node, // mockGossipSubNetwork, skipInitialSequencer. Timing: ethSlot=12s, aztecSlot=3×12=36s, // epoch=default 6, proofSubmissionEpochs=1 (overridden per test via setupTest), blockDurationMs=6s, // inboxLag=2 (v5 always enforces the timetable, so the former enforceTimeTable/disableAnvilTestWatcher // overrides are gone). The Delayer is used to steer proof tx timing. -describe('e2e_epochs/epochs_proof_at_boundary', () => { +describe('multi-node/block-production/proof_boundary', () => { let context: EndToEndContext; let logger: Logger; - let test: EpochsTestContext; - let validators: (Operator & { privateKey: `0x${string}` })[]; + let test: MultiNodeTestContext; + let validators: RegisteredValidator[]; let nodes: AztecNodeService[]; let proverNode: AztecNodeService; const setupTest = async ( - overrides: Partial = {}, + overrides: Partial = {}, validatorOverrides: { minTxsPerBlock?: number; maxTxsPerBlock?: number } = {}, ) => { - validators = times(NODE_COUNT, i => { - const privateKey = bufferToHex(getPrivateKeyFromIndex(i + 3)!); - const attester = EthAddress.fromString(privateKeyToAccount(privateKey).address); - return { attester, withdrawer: attester, privateKey, bn254SecretKey: new SecretValue(Fr.random().toBigInt()) }; - }); + validators = buildMockGossipValidators(NODE_COUNT); - test = await EpochsTestContext.setup({ - numberOfAccounts: 0, + test = await MultiNodeTestContext.setup({ + ...MOCK_GOSSIP_MULTI_VALIDATOR_OPTS, + ...MULTI_VALIDATOR_BLOCK_PRODUCTION_TIMING, initialValidators: validators, - mockGossipSubNetwork: true, - aztecProofSubmissionEpochs: 1024, - aztecSlotDurationInL1Slots: 3, - ethereumSlotDuration: 12, - blockDurationMs: 6000, - startProverNode: false, - skipInitialSequencer: true, - inboxLag: 2, ...overrides, }); @@ -103,7 +92,7 @@ describe('e2e_epochs/epochs_proof_at_boundary', () => { const computeBoundarySlot = async () => { // REFACTOR: hand-rolled retryUntil polling for first checkpoint; replace with - // test.waitUntilCheckpointNumber(CheckpointNumber(1)) from EpochsTestContext. + // test.waitUntilCheckpointNumber(CheckpointNumber(1)) from MultiNodeTestContext. await retryUntil( async () => { await test.monitor.run(true); diff --git a/yarn-project/end-to-end/src/multi-node/block-production/proposed_chain.parallel.test.ts b/yarn-project/end-to-end/src/multi-node/block-production/proposed_chain.parallel.test.ts new file mode 100644 index 000000000000..ec061008cb0d --- /dev/null +++ b/yarn-project/end-to-end/src/multi-node/block-production/proposed_chain.parallel.test.ts @@ -0,0 +1,163 @@ +import { Fr } from '@aztec/aztec.js/fields'; +import { SlotNumber } from '@aztec/foundation/branded-types'; +import { retryUntil } from '@aztec/foundation/retry'; +import { TxStatus } from '@aztec/stdlib/tx'; + +import { proveAndSendTxs, proveInteraction } from '../../test-wallet/utils.js'; +import { + type BlockProductionWithProverFixture, + TX_COUNT, + jest, + setupBlockProductionWithProver, + waitForProvenCheckpoint, +} from './setup.js'; + +// Production of a multi-block proposed slot: txs anchor to the proposed tip and the wallet syncs to it, +// and a non-validator re-executes then cold-syncs the checkpointed multi-block slot. Both share the +// proposed-tip MBPS setup (PXE in 'proposed' mode) from setup.ts. +describe('multi-node/block-production/proposed_chain', () => { + let fixture: BlockProductionWithProverFixture; + + afterEach(async () => { + jest.restoreAllMocks(); + await fixture?.test?.teardown(); + }); + + // Starts sequencers then sends txs one at a time, anchoring each to the proposed block containing + // the previous tx (PXE in 'proposed' mode). Verifies tx anchor block numbers are monotonically + // non-decreasing. Asserts ≥2 blocks per checkpoint and waits for the MBPS checkpoint to be proven. + it('builds multiple blocks per slot with transactions anchored to proposed blocks', async () => { + fixture = await setupBlockProductionWithProver({ syncChainTip: 'proposed', minTxsPerBlock: 1, maxTxsPerBlock: 1 }); + const { test, context, logger, rollup, nodes, contract, wallet, from } = fixture; + + // Record the current checkpoint number before starting sequencers + const initialCheckpointNumber = await rollup.getCheckpointNumber(); + logger.warn(`Initial checkpoint number: ${initialCheckpointNumber}`); + + // Start the sequencers + await test.startSequencers(nodes); + logger.warn(`Started all sequencers`); + + // Now send the txs and wait for them to be mined one at a time + // If the pxe syncs correctly, every tx should be anchored to the block in which the previous one was mined + const txReceipts = []; + let expectedAnchorBlockNumber = undefined; + + while (txReceipts.length < TX_COUNT / 2) { + logger.warn(`Sending transaction ${txReceipts.length}`); + const nullifier = new Fr(txReceipts.length + 1); + const tx = await proveInteraction(context.wallet, contract.methods.emit_nullifier(nullifier), { from }); + const txAnchorBlockNumber = tx.data.constants.anchorBlockHeader.globalVariables.blockNumber; + expect(txAnchorBlockNumber).toBeGreaterThanOrEqual(expectedAnchorBlockNumber ?? txAnchorBlockNumber); + + const txReceipt = await tx.send({ wait: { waitForStatus: TxStatus.PROPOSED } }); + txReceipts.push(txReceipt); + expectedAnchorBlockNumber = txReceipt.blockNumber; + logger.warn(`Transaction ${txReceipts.length} mined on block ${txReceipt.blockNumber}`, { txReceipt }); + + await wallet.sync(); + expect((await wallet.getSyncedBlockHeader()).getBlockNumber()).toBeGreaterThanOrEqual(txReceipt.blockNumber!); + } + logger.warn(`All txs have been mined`); + + // We are fine with at least 2 blocks per checkpoint, since we may lose one sub-slot if assembling a tx is slow + const multiBlockCheckpoint = await fixture.test.assertMultipleBlocksPerSlot(2, { + wait: true, + archiver: fixture.archiver, + }); + await waitForProvenCheckpoint(fixture, multiBlockCheckpoint); + }); + + // Creates an extra non-validator node with alwaysReexecuteBlockProposals=true, sends txs, and + // waits until that node has stored a multi-block proposed slot (≥2 blocks) beyond its checkpointed + // tip. Verifies block effects are valid, then starts a second sync-only node and confirms it + // syncs the multi-block slot from scratch. + it('builds multiple blocks per slot and non-validators re-execute and sync multi-block slots', async () => { + fixture = await setupBlockProductionWithProver({ syncChainTip: 'proposed', minTxsPerBlock: 1, maxTxsPerBlock: 1 }); + const { test, context, logger, nodes, contract, from } = fixture; + + logger.warn(`Creating non-validator reexecuting node`); + const nonValidatorNode = await test.createNonValidatorNode({ + alwaysReexecuteBlockProposals: true, + skipPushProposedBlocksToArchiver: false, + }); + + await test.startSequencers(nodes); + logger.warn(`Started all sequencers`); + + logger.warn(`Pre-proving ${TX_COUNT / 2} transactions`); + const sentTxHashes = await proveAndSendTxs( + context.wallet, + TX_COUNT / 2, + i => contract.methods.emit_nullifier(new Fr(i + 100)), + { from }, + ); + logger.warn(`Sent ${sentTxHashes.length} transactions`); + + const nonValidatorArchiver = nonValidatorNode.getBlockSource(); + + let multiBlockSlotNumber: number | undefined; + let checkpointedBlockNumber: number | undefined; + await retryUntil( + async () => { + const tips = await nonValidatorArchiver.getL2Tips(); + if (tips.proposed.number <= tips.checkpointed.block.number) { + return false; + } + const blockData = await nonValidatorArchiver.getBlockData({ number: tips.proposed.number }); + if (!blockData) { + return false; + } + const blocksInSlot = await nonValidatorArchiver.getBlocksForSlot(blockData.header.globalVariables.slotNumber); + if (blocksInSlot.length < 2) { + return false; + } + multiBlockSlotNumber = blockData.header.globalVariables.slotNumber; + checkpointedBlockNumber = tips.checkpointed.block.number; + return true; + }, + 'non-validator node to store multi-block proposed slot', + test.L2_SLOT_DURATION_IN_S * 5, + 0.5, + ); + + // Ensure the proposed multi-block slot has valid effects + expect(multiBlockSlotNumber).toBeDefined(); + const blocksInSlot = await nonValidatorArchiver.getBlocksForSlot(SlotNumber(multiBlockSlotNumber!)); + expect(blocksInSlot.length).toBeGreaterThanOrEqual(2); + expect(checkpointedBlockNumber).toBeDefined(); + expect(blocksInSlot.every(block => block.number > checkpointedBlockNumber!)).toBe(true); // ensure the block is proposed + const txHashesInSlot = blocksInSlot.flatMap(block => block.body.txEffects.map(effect => effect.txHash)); + expect(txHashesInSlot.length).toBeGreaterThan(0); + const effectsInSlot = await Promise.all(txHashesInSlot.map(txHash => nonValidatorArchiver.getTxEffect(txHash))); + expect(effectsInSlot.every(effect => effect !== undefined)).toBe(true); + + // Wait until the node syncs to the checkpointed block successfully + const maxBlockNumberInSlot = Math.max(...blocksInSlot.map(block => block.number)); + await retryUntil( + async () => (await nonValidatorArchiver.getL2Tips()).checkpointed.block.number >= maxBlockNumberInSlot!, + 'non-validator node to sync checkpointed block', + test.L2_SLOT_DURATION_IN_S * 5, + 0.5, + ); + + // Start a new node an make sure it can sync from scratch including the multi-block slot + logger.warn(`Creating non-validator syncing node`); + const nonValidatorSyncingNode = await test.createNonValidatorNode({ + alwaysReexecuteBlockProposals: false, + }); + await retryUntil( + async () => + (await nonValidatorSyncingNode.getBlockSource().getL2Tips()).checkpointed.block.number >= maxBlockNumberInSlot!, + 'non-validator syncing node to sync checkpointed block', + test.L2_SLOT_DURATION_IN_S * 10, + 0.5, + ); + + const multiBlockCheckpoint = await fixture.test.assertMultipleBlocksPerSlot(2, { + wait: true, + archiver: fixture.archiver, + }); + await waitForProvenCheckpoint(fixture, multiBlockCheckpoint); + }); +}); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_mbps_redistribution.test.ts b/yarn-project/end-to-end/src/multi-node/block-production/redistribution.parallel.test.ts similarity index 60% rename from yarn-project/end-to-end/src/e2e_epochs/epochs_mbps_redistribution.test.ts rename to yarn-project/end-to-end/src/multi-node/block-production/redistribution.parallel.test.ts index 1e0065c58d23..d2eeb78036b8 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_mbps_redistribution.test.ts +++ b/yarn-project/end-to-end/src/multi-node/block-production/redistribution.parallel.test.ts @@ -1,114 +1,59 @@ import type { Archiver } from '@aztec/archiver'; -import type { AztecNodeConfig, AztecNodeService } from '@aztec/aztec-node'; -import { AztecAddress, EthAddress } from '@aztec/aztec.js/addresses'; +import type { AztecNodeConfig } from '@aztec/aztec-node'; import { NO_WAIT } from '@aztec/aztec.js/contracts'; import { Fr } from '@aztec/aztec.js/fields'; -import type { Logger } from '@aztec/aztec.js/log'; import { waitForTx } from '@aztec/aztec.js/node'; -import { RollupContract } from '@aztec/ethereum/contracts'; -import type { Operator } from '@aztec/ethereum/deploy-aztec-l1-contracts'; -import { waitUntilL1Timestamp } from '@aztec/ethereum/l1-tx-utils'; import { asyncMap } from '@aztec/foundation/async-map'; import { CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types'; -import { times, timesAsync } from '@aztec/foundation/collection'; -import { SecretValue } from '@aztec/foundation/config'; import { retryUntil } from '@aztec/foundation/retry'; -import { sleep } from '@aztec/foundation/sleep'; import { executeTimeout } from '@aztec/foundation/timer'; -import { TestContract } from '@aztec/noir-test-contracts.js/Test'; -import { getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; - -import { jest } from '@jest/globals'; -import { privateKeyToAccount } from 'viem/accounts'; - -import { type EndToEndContext, getPrivateKeyFromIndex } from '../fixtures/utils.js'; -import { TestWallet } from '../test-wallet/test_wallet.js'; -import { proveInteraction } from '../test-wallet/utils.js'; -import { EpochsTestContext } from './epochs_test.js'; - -jest.setTimeout(1000 * 60 * 20); - -const NODE_COUNT = 4; - -/** - * Number of txs fed one-by-one during the early sub-slots (blocks 0 and 1), one per block. - * They are sent at the start of each sub-slot so each early block picks up exactly one, leaving - * most of the checkpoint's tx budget unconsumed for the later blocks to inherit. - */ -const EARLY_TX_COUNT = 2; - -/** - * Number of txs dumped into the mempool as a burst once the early blocks are in. They race the - * proposer's mempool snapshot for the one-before-last block, so they split arbitrarily across the - * last two blocks (an x/(LATE_TX_COUNT-x) split). With redistribution working, those two blocks - * together inherit enough budget to hold all of them regardless of the split; without it, each is - * capped at the static per-block limit S and the burst spills into the next checkpoint. - */ -const LATE_TX_COUNT = 7; - -/** Total txs pre-proved before the test begins. */ -const TOTAL_TX_COUNT = EARLY_TX_COUNT + LATE_TX_COUNT; - -// Four-validator MBPS suite verifying that the per-block gas budget redistribution mechanism allows -// late transactions to fill the last blocks of a checkpoint whose earlier blocks were light. Two tests: -// (1) standard redistribution — early blocks consume minimal budget, late txs all fit across the last -// blocks; (2) validators should NOT apply the proposer's fair-share multiplier during re-execution — -// nodes with different perBlockAllocationMultiplier values must still attest for each other's blocks. -// Uses EpochsTestContext with mockGossipSubNetwork, startProverNode, no initial sequencer. -/** - * Verifies that checkpoint budget redistribution lets a burst of late transactions fit into the last - * blocks of a checkpoint when the earlier blocks were light. - * - * Config gives C = maxBlocksPerCheckpoint = 4 blocks per checkpoint with a tight - * maxTxsPerCheckpoint = TOTAL_TX_COUNT = 9. The proposer's per-block cap (proposer mode) is - * `min(remainingTxs, ceil(remainingTxs / remainingBlocks * 1.2))` with the multiplier floor 1.2. - * Blocks 0 and 1 each take a single early tx (caps 3 and 4), so used = 2 and 7 txs remain. The burst - * of LATE_TX_COUNT = 7 then races the snapshot for block 2 (one-before-last): block 2 has cap - * `ceil(7/2*1.2)=5` and takes some x ∈ [1,5]; block 3 (last) then has cap `7-x` and takes the rest. - * Either way the last two blocks jointly absorb all 7 late txs. - * - * The "no redistribution" baseline is the static per-block cap S = ceil(9/4*1.2) = 3: without - * redistribution each of the last two blocks holds at most S = 3, so together at most 6, and the 7th - * tx spills into the first block of the *next* checkpoint (a consecutive global block number). - * - * Success is verified by confirming that, within the checkpoint holding the early txs, the last two - * checkpoint block indices together hold all LATE_TX_COUNT late txs (> 2*S = 6). The check counts by - * checkpoint block index, not by global block number, so a spilled tx in the next checkpoint does not - * masquerade as a redistributed one. - */ -describe('e2e_epochs/epochs_mbps_redistribution', () => { - let context: EndToEndContext; - let logger: Logger; - let rollup: RollupContract; - let archiver: Archiver; - - let test: EpochsTestContext; - let validators: (Operator & { privateKey: `0x${string}` })[]; - let nodes: AztecNodeService[]; - let contract: TestContract; - let wallet: TestWallet; - let from: AztecAddress; + +import type { TestWallet } from '../../test-wallet/test_wallet.js'; +import { proveTxs, startMempoolFeeder } from '../../test-wallet/utils.js'; +import { MultiNodeTestContext, buildMockGossipValidators } from '../multi_node_test_context.js'; +import { type BlockProductionWithProverFixture, NODE_COUNT, jest } from './setup.js'; + +describe('multi-node/block-production/redistribution', () => { + /** + * Number of txs fed one-by-one during the early sub-slots (blocks 0 and 1), one per block. + * They are sent at the start of each sub-slot so each early block picks up exactly one, leaving + * most of the checkpoint's tx budget unconsumed for the later blocks to inherit. + */ + const EARLY_TX_COUNT = 2; + + /** + * Number of txs dumped into the mempool as a burst once the early blocks are in. They race the + * proposer's mempool snapshot for the one-before-last block, so they split arbitrarily across the + * last two blocks (an x/(LATE_TX_COUNT-x) split). With redistribution working, those two blocks + * together inherit enough budget to hold all of them regardless of the split; without it, each is + * capped at the static per-block limit S and the burst spills into the next checkpoint. + */ + const LATE_TX_COUNT = 7; + + /** Total txs pre-proved before the test begins. */ + const TOTAL_TX_COUNT = EARLY_TX_COUNT + LATE_TX_COUNT; + + let fixture: Pick< + BlockProductionWithProverFixture, + 'test' | 'context' | 'logger' | 'rollup' | 'archiver' | 'validators' | 'nodes' | 'contract' | 'wallet' | 'from' + >; /** * Sets up validators and the test context with MBPS + redistribution config. * Uses a tight `maxTxsPerCheckpoint` so that the redistribution logic is exercised. */ - async function setupTest( + async function setupRedistribution( nodeConfigOverride?: (index: number) => Partial, contextConfigOverride?: Record, ) { - validators = times(NODE_COUNT, i => { - const privateKey = `0x${getPrivateKeyFromIndex(i + 3)!.toString('hex')}` as `0x${string}`; - const attester = EthAddress.fromString(privateKeyToAccount(privateKey).address); - return { attester, withdrawer: attester, privateKey, bn254SecretKey: new SecretValue(Fr.random().toBigInt()) }; - }); + const validators = buildMockGossipValidators(NODE_COUNT); // Timing for C = 4 blocks per checkpoint with 6s sub-slots (fast e2e profile, ethereumSlotDuration < 8): // maxBlocksPerCheckpoint = floor((S - init - D - 2P - prepCp) / D). In the fast profile the operational // budgets collapse to init + 2P + prepCp = 1 + 2*0.5 + 0.5 = 2.5s, so floor((36 - 2.5 - 6) / 6) = // floor(27.5/6) = 4. (At the old D = 8s this was floor((36 - 2.5 - 8) / 8) = 3.) The chosen 36s slot // leaves room for the 4 sub-slots plus L1 publish and final-block re-execution. - test = await EpochsTestContext.setup({ + const test = await MultiNodeTestContext.setup({ numberOfAccounts: 0, initialValidators: validators, inboxLag: 2, @@ -132,29 +77,31 @@ describe('e2e_epochs/epochs_mbps_redistribution', () => { skipInitialSequencer: true, }); - ({ context, logger, rollup } = test); - wallet = context.wallet; - from = context.accounts[0]; // auto-created by setup + const { context, logger, rollup } = test; + const wallet = context.wallet as TestWallet; + const from = context.accounts[0]; // auto-created by setup // Start validator nodes. logger.warn(`Starting ${NODE_COUNT} validator nodes.`); - nodes = await asyncMap(validators, ({ privateKey }, i) => + const nodes = await asyncMap(validators, ({ privateKey }, i) => test.createValidatorNode([privateKey], { dontStartSequencer: true, ...nodeConfigOverride?.(i) }), ); logger.warn(`Started ${NODE_COUNT} validator nodes.`, { validators: validators.map(v => v.attester.toString()) }); // Point the wallet at a validator node. wallet.updateNode(nodes[0]); - archiver = nodes[0].getBlockSource() as Archiver; + const archiver = nodes[0].getBlockSource() as Archiver; // Register the test contract. - contract = await test.registerTestContract(wallet); + const contract = await test.registerTestContract(wallet); logger.warn(`Test setup completed.`); + + fixture = { test, context, logger, rollup, archiver, validators, nodes, contract, wallet, from }; } afterEach(async () => { jest.restoreAllMocks(); - await test?.teardown(); + await fixture?.test?.teardown(); }); // Pre-proves TOTAL_TX_COUNT txs. Warps to just before the next L2 slot. Sends the first early tx @@ -162,23 +109,21 @@ describe('e2e_epochs/epochs_mbps_redistribution', () => { // (waiting for each to be proposed), then dumps all late txs at once. Waits for all txs to be // mined and verifies the late txs landed across the last two blocks (redistribution gave them budget). it('redistributes checkpoint budget so a late burst fits across the last two blocks', async () => { - await setupTest(); + await setupRedistribution(); + const { test, logger, rollup, archiver, nodes, contract, wallet, from } = fixture; // Pre-prove all transactions up front. logger.warn(`Pre-proving ${TOTAL_TX_COUNT} transactions`); - const provenTxs = await timesAsync(TOTAL_TX_COUNT, i => - proveInteraction(wallet, contract.methods.emit_nullifier(new Fr(i + 1)), { from }), - ); + const provenTxs = await proveTxs(wallet, TOTAL_TX_COUNT, i => contract.methods.emit_nullifier(new Fr(i + 1)), { + from, + }); logger.warn(`Pre-proved ${provenTxs.length} transactions`); - // Warp to just before the next L2 slot so sequencers start building promptly. + // Warp to just before the next L2 slot so sequencers start building promptly (one L1 slot before + // the L2 slot = the sequencer's build start). Uses the wait form since this test does not warp. const currentSlot = await rollup.getSlotNumber(); const nextSlot = SlotNumber(currentSlot + 1); - const slotStartTimestamp = getTimestampForSlot(nextSlot, test.constants); - // Warp to one L1 slot before the L2 slot starts (= the sequencer's build start). - const warpTo = slotStartTimestamp - BigInt(test.L1_BLOCK_TIME_IN_S); - logger.warn(`Warping to L1 timestamp ${warpTo} (one L1 slot before L2 slot ${nextSlot})`); - await waitUntilL1Timestamp(test.l1Client, warpTo, undefined, 60); + await test.waitForBuildWindowForSlot(nextSlot, { timeout: 60 }); // Send first early tx to the mempool before starting sequencers, so the first block isn't empty. // With skipInitialSequencer, there are no pre-existing blocks, and sequencers build block 1 @@ -188,7 +133,7 @@ describe('e2e_epochs/epochs_mbps_redistribution', () => { const earlyTxHashes = [await provenTxs[0].send({ wait: NO_WAIT })]; // Start sequencers. - await Promise.all(nodes.map(n => n.getSequencer()!.start())); + await test.startSequencers(nodes); logger.warn(`Started all sequencers`); // Wait for the first early tx to be proposed before sending the next. @@ -235,7 +180,7 @@ describe('e2e_epochs/epochs_mbps_redistribution', () => { ); logger.warn(`All transactions have been mined`); - // maxBlocksPerCheckpoint derived from the timing config above (see setupTest): floor((36-2.5-6)/6) = 4. + // maxBlocksPerCheckpoint derived from the timing config above (see setupRedistribution): floor((36-2.5-6)/6) = 4. const MAX_BLOCKS_PER_CHECKPOINT = 4; // Static per-block cap (the "no redistribution" baseline): S = ceil(maxTxsPerCheckpoint / C * 1.2) = 3. const STATIC_PER_BLOCK_CAP = Math.ceil((TOTAL_TX_COUNT / MAX_BLOCKS_PER_CHECKPOINT) * 1.2); @@ -273,34 +218,18 @@ describe('e2e_epochs/epochs_mbps_redistribution', () => { expect(lastTwoLateCount).toBe(LATE_TX_COUNT); }); - /** - * Verifies that validators do NOT apply the proposer's fair-share multiplier when re-executing blocks. - * - * Configures nodes 0/1 with perBlockAllocationMultiplier=10 and nodes 2/3 with the default (1.2), - * and keeps the mempool topped up with a background loop. - * - * Two of the four validator nodes are configured with a very large `perBlockAllocationMultiplier` (10), - * allowing their proposer to pack multiple txs into a single block. The other two keep the default - * multiplier (1.2), which limits them to 1 tx per block given the tight `maxTxsPerCheckpoint`. - * - * With `maxTxsPerCheckpoint = 2` and 4 blocks per checkpoint (shared setupTest uses blockDurationMs=6000 → C=4): - * - Normal multiplier (1.2): first block cap = ceil(2/4 * 1.2) = 1 tx (later blocks may get more via redistribution) - * - High multiplier (10): first block cap = ceil(2/4 * 10) = 5 txs (capped by remaining = 2) - * - * The test watches checkpoints and identifies the proposer for each slot via EpochCache. - * It waits until it observes a checkpoint by a high-multiplier proposer with the initial block having >1 tx - * - * If validators incorrectly applied their own multiplier during re-execution, checkpoints built by - * high-multiplier proposers would fail attestation and the chain would stall. - */ + // Configures nodes 0/1 with a large perBlockAllocationMultiplier and 2/3 with default, keeps the + // mempool topped up via a background loop, watches checkpoints, and asserts that a high-multiplier + // proposer's first block holds >1 tx (validators do not apply their own multiplier on re-execution). it('validators accept blocks built with a larger proposer multiplier (no fair-share re-execution)', async () => { const HIGH_MULTIPLIER = 10; const MAX_TXS_PER_CHECKPOINT = 2; // Nodes 0 and 1 get a very large multiplier; nodes 2 and 3 keep the default (1.2). - await setupTest(i => (i < 2 ? { perBlockAllocationMultiplier: HIGH_MULTIPLIER } : {}), { + await setupRedistribution(i => (i < 2 ? { perBlockAllocationMultiplier: HIGH_MULTIPLIER } : {}), { maxTxsPerCheckpoint: MAX_TXS_PER_CHECKPOINT, }); + const { test, logger, rollup, archiver, validators, nodes, contract, wallet, from } = fixture; logger.warn( `Set perBlockAllocationMultiplier=${HIGH_MULTIPLIER} on nodes 0,1; maxTxsPerCheckpoint=${MAX_TXS_PER_CHECKPOINT}`, ); @@ -309,21 +238,22 @@ describe('e2e_epochs/epochs_mbps_redistribution', () => { const INITIAL_TX_COUNT = 4; let nullifierCounter = 200; logger.warn(`Pre-proving ${INITIAL_TX_COUNT} initial transactions`); - const initialProvenTxs = await timesAsync(INITIAL_TX_COUNT, () => - proveInteraction(wallet, contract.methods.emit_nullifier(new Fr(nullifierCounter++)), { from }), + const initialProvenTxs = await proveTxs( + wallet, + INITIAL_TX_COUNT, + () => contract.methods.emit_nullifier(new Fr(nullifierCounter++)), + { from }, ); logger.warn(`Pre-proved ${initialProvenTxs.length} transactions`); - // Warp to just before the next L2 slot so sequencers start building promptly. + // Warp to just before the next L2 slot so sequencers start building promptly (one L1 slot before + // the L2 slot start). Uses the wait form since this test does not warp. const currentSlot = await rollup.getSlotNumber(); const nextSlot = SlotNumber(currentSlot + 1); - const slotStartTimestamp = getTimestampForSlot(nextSlot, test.constants); - const warpTo = slotStartTimestamp - BigInt(test.L1_BLOCK_TIME_IN_S); - logger.warn(`Warping to L1 timestamp ${warpTo} (one L1 slot before L2 slot ${nextSlot})`); - await waitUntilL1Timestamp(test.l1Client, warpTo, undefined, 60); + await test.waitForBuildWindowForSlot(nextSlot, { timeout: 60 }); // Start sequencers and send the initial batch. - await Promise.all(nodes.map(n => n.getSequencer()!.start())); + await test.startSequencers(nodes); logger.warn(`Started all sequencers`); logger.warn(`Sending ${initialProvenTxs.length} initial transactions`); @@ -331,27 +261,12 @@ describe('e2e_epochs/epochs_mbps_redistribution', () => { logger.warn(`Sent initial transactions`); // Background loop: keep the mempool topped up so proposers always have txs to include. - let done = false; - const keepMempoolFull = async () => { - while (!done) { - try { - const pendingCount = await nodes[0].getPendingTxCount(); - if (pendingCount < 3) { - const tx = await proveInteraction(wallet, contract.methods.emit_nullifier(new Fr(nullifierCounter++)), { - from, - }); - await tx.send({ wait: NO_WAIT }); - logger.verbose(`Topped up mempool (was ${pendingCount}, nullifier=${nullifierCounter - 1})`); - } - } catch (err) { - logger.verbose(`Mempool top-up error (will retry): ${err}`); - } - await sleep(1000); - } - }; - // REFACTOR: hand-rolled background sleep loop keeping the mempool above a threshold; replace - // with a shared test utility such as startMempoolFeeder(wallet, contract, from, minPending). - void keepMempoolFull(); + await using _feeder = startMempoolFeeder( + wallet, + nodes[0], + () => contract.methods.emit_nullifier(new Fr(nullifierCounter++)), + { from, minPending: 3, logger }, + ); // Build a lookup from attester address to validator index for proposer identification. const attesterToIndex = new Map(); @@ -417,7 +332,6 @@ describe('e2e_epochs/epochs_mbps_redistribution', () => { 1, ); - done = true; logger.warn( `Test passed: observed checkpoints from both high-multiplier and normal-multiplier proposers. ` + `High-multiplier proposers packed >1 tx per block; normal proposers respected the fair-share ` + diff --git a/yarn-project/end-to-end/src/multi-node/block-production/setup.ts b/yarn-project/end-to-end/src/multi-node/block-production/setup.ts new file mode 100644 index 000000000000..d04826ca89cf --- /dev/null +++ b/yarn-project/end-to-end/src/multi-node/block-production/setup.ts @@ -0,0 +1,231 @@ +import type { Archiver } from '@aztec/archiver'; +import type { AztecNodeConfig, AztecNodeService } from '@aztec/aztec-node'; +import { AztecAddress, EthAddress } from '@aztec/aztec.js/addresses'; +import { NO_WAIT } from '@aztec/aztec.js/contracts'; +import { generateClaimSecret } from '@aztec/aztec.js/ethereum'; +import { Fr } from '@aztec/aztec.js/fields'; +import type { Logger } from '@aztec/aztec.js/log'; +import { isL1ToL2MessageReady } from '@aztec/aztec.js/messaging'; +import { waitForTx } from '@aztec/aztec.js/node'; +import { RollupContract } from '@aztec/ethereum/contracts'; +import { waitUntilL1Timestamp } from '@aztec/ethereum/l1-tx-utils'; +import { asyncMap } from '@aztec/foundation/async-map'; +import { BlockNumber, CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types'; +import { timesAsync } from '@aztec/foundation/collection'; +import { retryUntil } from '@aztec/foundation/retry'; +import { sleep } from '@aztec/foundation/sleep'; +import { executeTimeout } from '@aztec/foundation/timer'; +import { TestContract } from '@aztec/noir-test-contracts.js/Test'; +import type { SequencerEvents } from '@aztec/sequencer-client'; +import { getSlotAtTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; +import { GasFees } from '@aztec/stdlib/gas'; +import { TxStatus } from '@aztec/stdlib/tx'; + +import { jest } from '@jest/globals'; + +import { sendL1ToL2Message } from '../../fixtures/l1_to_l2_messaging.js'; +import type { EndToEndContext } from '../../fixtures/utils.js'; +import { waitForBlockNumber, waitForTxs } from '../../fixtures/wait_helpers.js'; +import type { TestWallet } from '../../test-wallet/test_wallet.js'; +import { proveInteraction } from '../../test-wallet/utils.js'; +import { + type BlockProposedEvent, + MOCK_GOSSIP_MULTI_VALIDATOR_OPTS, + MULTI_VALIDATOR_BLOCK_PRODUCTION_TIMING, + MultiNodeTestContext, + type MultiNodeTestOpts, + type RegisteredValidator, + type TrackedSequencerEvent, + WIDE_SLOT_TIMING, + buildMockGossipValidators, +} from '../multi_node_test_context.js'; + +jest.setTimeout(1000 * 60 * 20); + +export const NODE_COUNT = 4; + +// Send enough transactions to trigger multiple blocks within a checkpoint assuming 2 txs per block. +// If we start including txs at the 2nd block of a checkpoint, we can ensure a 3-block checkpoint +// if we produce 10 txs: +// - Checkpoint 1: Block 1 (0 txs), Block 2 (2 txs), Block 3 (2 txs) +// - Checkpoint 2: Block 1 (2 txs), Block 2 (2 txs), Block 3 (2 txs) +export const TX_COUNT = 10; + +/** The validator cluster + context produced by {@link setupSimpleBlockProduction}. */ +export type SimpleBlockProductionFixture = { + test: MultiNodeTestContext; + context: EndToEndContext; + logger: Logger; + validators: RegisteredValidator[]; + nodes: AztecNodeService[]; + from: AztecAddress; +}; + +/** State shared by the wide-slot `it`s (handles 4 validators + prover + a wallet pointed at node 0). */ +export type BlockProductionWithProverFixture = { + test: MultiNodeTestContext; + context: EndToEndContext; + logger: Logger; + rollup: RollupContract; + archiver: Archiver; + validators: RegisteredValidator[]; + nodes: AztecNodeService[]; + contract: TestContract; + wallet: TestWallet; + from: AztecAddress; + failEvents: TrackedSequencerEvent[]; +}; + +/** Shared spine: builds N mock-gossip validators, sets up the context, spawns one node per validator. */ +async function buildValidatorCluster(opts: { + nodeCount: number; + setupOpts: Partial; + nodeOpts?: Partial & { dontStartSequencer?: boolean }; +}): Promise { + const validators = buildMockGossipValidators(opts.nodeCount); + + const test = await MultiNodeTestContext.setup({ + ...opts.setupOpts, + initialValidators: validators, + }); + + const { context, logger } = test; + const from = context.accounts[0]; + + logger.warn(`Initial setup complete. Starting ${opts.nodeCount} validator nodes.`); + const nodes = await asyncMap(validators, ({ privateKey }) => + test.createValidatorNode([privateKey], { ...opts.nodeOpts }), + ); + logger.warn(`Started ${opts.nodeCount} validator nodes.`, { validators: validators.map(v => v.attester.toString()) }); + + return { test, context, logger, validators, nodes, from }; +} + +/** + * Stands up the `block-production` validator cluster shared by the `MULTI_VALIDATOR_BLOCK_PRODUCTION_TIMING` tests + * (`simple`, `high_tps`): builds `nodeCount` mock-gossip validators, sets up the context with the + * block-production timing profile, spawns one validator node per validator, and returns the cluster. The + * per-test divergence (`fakeProcessingDelayPerTxMs`, `txDelayerMaxInclusionTimeIntoSlot`, + * min/maxTxsPerBlock, whether sequencers start eagerly, contract type) passes through `opts`. Mirrors + * how {@link setupBlockProductionWithProver} factors out the prover-backed setup; the test still registers its own contract. + */ +export function setupSimpleBlockProduction(opts: { + nodeCount: number; + setupOpts?: Partial; + nodeOpts?: Partial & { dontStartSequencer?: boolean }; +}): Promise { + return buildValidatorCluster({ + nodeCount: opts.nodeCount, + setupOpts: { ...MOCK_GOSSIP_MULTI_VALIDATOR_OPTS, ...MULTI_VALIDATOR_BLOCK_PRODUCTION_TIMING, ...opts.setupOpts }, + nodeOpts: opts.nodeOpts, + }); +} + +/** + * Creates validators and sets up a wide-slot test context with the pipelining timing profile and a prover + * node, then starts (paused) validator nodes and points the wallet at node 0. Mirrors the per-test + * setup from the dissolved `mbps.parallel` file. + */ +export async function setupBlockProductionWithProver(opts: { + syncChainTip: 'proposed' | 'checkpointed'; + minTxsPerBlock?: number; + maxTxsPerBlock?: number; + buildCheckpointIfEmpty?: boolean; + skipPushProposedBlocksToArchiver?: boolean; +}): Promise { + const { syncChainTip = 'checkpointed', ...setupOpts } = opts; + + // WIDE_SLOT_TIMING is the wide 72s/12s pipelining cadence (see A-914 on why the tighter 36s/4s breaks + // non-proposer nodes); the JSDoc on the profile carries the full rationale. + const { test, context, logger, validators, nodes, from } = await buildValidatorCluster({ + nodeCount: NODE_COUNT, + setupOpts: { + ...WIDE_SLOT_TIMING, + numberOfAccounts: 0, + mockGossipSubNetwork: true, + startProverNode: true, + ...setupOpts, + pxeOpts: { syncChainTip }, + skipInitialSequencer: true, + inboxLag: 2, + }, + nodeOpts: { dontStartSequencer: true }, + }); + + const { rollup } = test; + const wallet = context.wallet as TestWallet; + const { failEvents } = test.watchNodeSequencerEvents(nodes); + + // Point the wallet at a validator node. The initial node-0 has all validator keys in its config, + // so it rejects block proposals from validators thinking they come from itself. By redirecting + // the wallet to a validator node, the PXE correctly tracks proposed blocks. + wallet.updateNode(nodes[0]); + const archiver = nodes[0].getBlockSource() as Archiver; + + // Register contract for sending txs. + const contract = await test.registerTestContract(wallet); + logger.warn(`Test setup completed.`, { validators: validators.map(v => v.attester.toString()) }); + + return { test, context, logger, rollup, archiver, validators, nodes, contract, wallet, from, failEvents }; +} + +/** Waits until a specific multi-block checkpoint is proven, verifying that proving succeeds with multiple-blocks-per-slot. */ +export async function waitForProvenCheckpoint( + fixture: BlockProductionWithProverFixture, + targetCheckpoint: CheckpointNumber, +) { + const { test, nodes, logger, failEvents } = fixture; + test.assertNoFailuresFromSequencers(failEvents); + + logger.warn(`Stopping validator sequencers before waiting for checkpoint ${targetCheckpoint} to be proven`); + await Promise.all(nodes.map(n => n.getSequencer()?.stop())); + + const provenTimeout = test.L2_SLOT_DURATION_IN_S * test.epochDuration * 4; + logger.warn(`Waiting for checkpoint ${targetCheckpoint} to be proven (timeout=${provenTimeout}s)`); + await test.waitUntilProvenCheckpointNumber(targetCheckpoint, provenTimeout); + logger.warn(`Proven checkpoint advanced to ${test.monitor.provenCheckpointNumber}`); +} + +export { + type Archiver, + type AztecNodeConfig, + type AztecNodeService, + AztecAddress, + EthAddress, + NO_WAIT, + generateClaimSecret, + Fr, + type Logger, + isL1ToL2MessageReady, + waitForTx, + RollupContract, + waitUntilL1Timestamp, + asyncMap, + BlockNumber, + CheckpointNumber, + SlotNumber, + timesAsync, + retryUntil, + sleep, + executeTimeout, + TestContract, + type SequencerEvents, + getSlotAtTimestamp, + getTimestampForSlot, + GasFees, + TxStatus, + jest, + sendL1ToL2Message, + type EndToEndContext, + waitForBlockNumber, + waitForTxs, + type TestWallet, + proveInteraction, + type BlockProposedEvent, + WIDE_SLOT_TIMING, + MULTI_VALIDATOR_BLOCK_PRODUCTION_TIMING, + MultiNodeTestContext, + type RegisteredValidator, + type TrackedSequencerEvent, + buildMockGossipValidators, +}; diff --git a/yarn-project/end-to-end/src/multi-node/block-production/simple.test.ts b/yarn-project/end-to-end/src/multi-node/block-production/simple.test.ts new file mode 100644 index 000000000000..2f21f3ce519b --- /dev/null +++ b/yarn-project/end-to-end/src/multi-node/block-production/simple.test.ts @@ -0,0 +1,74 @@ +import type { AztecNodeService } from '@aztec/aztec-node'; +import type { AztecAddress } from '@aztec/aztec.js/addresses'; +import { Fr } from '@aztec/aztec.js/fields'; +import type { Logger } from '@aztec/aztec.js/log'; +import { executeTimeout } from '@aztec/foundation/timer'; +import type { TestContract } from '@aztec/noir-test-contracts.js/Test'; + +import type { EndToEndContext } from '../../fixtures/utils.js'; +import { waitForTxs } from '../../fixtures/wait_helpers.js'; +import { proveAndSendTxs } from '../../test-wallet/utils.js'; +import type { MultiNodeTestContext, RegisteredValidator } from '../multi_node_test_context.js'; +import { jest, setupSimpleBlockProduction } from './setup.js'; + +const NODE_COUNT = 3; +const TX_COUNT_SIMPLE = 8; + +// Verifies that 3 validator nodes can build blocks without sequencer errors. Lightweight RPC-only +// initial node (skipInitialSequencer), mockGossipSubNetwork, no prover. Timing: ethSlot=12s, +// aztecSlot=36s, epoch=default 6, proofSubmissionEpochs=1024, blockDurationMs=6s. Pre-proved txs sent +// from the hardcoded genesis-funded account (no on-chain account deploy needed). +describe('multi-node/block-production/simple', () => { + let context: EndToEndContext; + let logger: Logger; + + let test: MultiNodeTestContext; + let validators: RegisteredValidator[]; + let nodes: AztecNodeService[]; + let contract: TestContract; + let from: AztecAddress; + + beforeEach(async () => { + // Setup context with no initial sequencer (lightweight RPC-only node). + // The hardcoded account is funded via genesis without needing on-chain deployment. + ({ test, context, logger, validators, nodes, from } = await setupSimpleBlockProduction({ + nodeCount: NODE_COUNT, + nodeOpts: { minTxsPerBlock: 1, maxTxsPerBlock: 1 }, + })); + + // Register test contract locally for sending txs (no on-chain deployment needed). + contract = await test.registerTestContract(context.wallet); + logger.warn(`Test setup completed.`, { validators: validators.map(v => v.attester.toString()) }); + }); + + afterEach(async () => { + jest.restoreAllMocks(); + await test.teardown(); + }); + + // Pre-proves TX_COUNT transactions emitting unique nullifiers, sends them, waits for all to mine, + // then asserts no fail events were emitted by any of the 3 sequencers during the run. + it('builds simple blocks without any errors', async () => { + const { failEvents } = test.watchNodeSequencerEvents(nodes); + + // Create and submit txs from the hardcoded account. Each tx emits a unique + // nullifier, which is enough side-effect to produce a non-empty block. + const txHashes = await proveAndSendTxs( + context.wallet, + TX_COUNT_SIMPLE, + () => contract.methods.emit_nullifier(Fr.random()), + { from }, + ); + logger.warn(`Sent ${txHashes.length} transactions`, { + txs: txHashes, + }); + + // Wait until all txs are mined + const timeout = test.L2_SLOT_DURATION_IN_S * (TX_COUNT_SIMPLE * 2 + 1); + await executeTimeout(() => waitForTxs(context.aztecNode, txHashes, { timeout }), timeout * 1000); + logger.warn(`All txs have been mined`); + + // Expect no failures from sequencers during block building + test.assertNoFailuresFromSequencers(failEvents); + }); +}); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_ha_checkpoint_handoff.test.ts b/yarn-project/end-to-end/src/multi-node/high-availability/ha_checkpoint_handoff.test.ts similarity index 77% rename from yarn-project/end-to-end/src/e2e_epochs/epochs_ha_checkpoint_handoff.test.ts rename to yarn-project/end-to-end/src/multi-node/high-availability/ha_checkpoint_handoff.test.ts index ded9c6c5808c..9ce8ea5d3eb4 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_ha_checkpoint_handoff.test.ts +++ b/yarn-project/end-to-end/src/multi-node/high-availability/ha_checkpoint_handoff.test.ts @@ -1,25 +1,24 @@ import type { Archiver } from '@aztec/archiver'; import type { AztecNodeService } from '@aztec/aztec-node'; import { EthAddress } from '@aztec/aztec.js/addresses'; -import { Fr } from '@aztec/aztec.js/fields'; import type { Logger } from '@aztec/aztec.js/log'; import { RollupContract } from '@aztec/ethereum/contracts'; -import type { Operator } from '@aztec/ethereum/deploy-aztec-l1-contracts'; import { BlockNumber, CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types'; -import { times } from '@aztec/foundation/collection'; -import { SecretValue } from '@aztec/foundation/config'; import { retryUntil } from '@aztec/foundation/retry'; -import { bufferToHex } from '@aztec/foundation/string'; import type { BlockData } from '@aztec/stdlib/block'; import type { CheckpointData, ProposedCheckpointData } from '@aztec/stdlib/checkpoint'; -import { getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; -import { createSharedSlashingProtectionDb } from '@aztec/validator-ha-signer/factory'; import { jest } from '@jest/globals'; -import { privateKeyToAccount } from 'viem/accounts'; -import { type EndToEndContext, getPrivateKeyFromIndex } from '../fixtures/utils.js'; -import { EpochsTestContext } from './epochs_test.js'; +import type { EndToEndContext } from '../../fixtures/utils.js'; +import { + MOCK_GOSSIP_MULTI_VALIDATOR_OPTS, + MULTI_VALIDATOR_REORG_TIMING, + MultiNodeTestContext, + type RegisteredValidator, + buildMockGossipValidators, + setupHaPairs, +} from '../multi_node_test_context.js'; jest.setTimeout(1000 * 60 * 20); @@ -55,29 +54,25 @@ const VALIDATOR_COUNT = 4; * checkpoint. With the fix, checkpoint 1 (covering S1, built by the builder) and checkpoint 2 (covering * S2, built by the peer) both land on L1, and S2's covered block carries the peer's distinct coinbase. * - * Setup: `EpochsTestContext.setup` with 4 validators (`skipInitialSequencer: true`) wired onto the in-memory + * Setup: `MultiNodeTestContext.setup` with 4 validators (`skipInitialSequencer: true`) wired onto the in-memory * `mockGossipSubNetwork` bus, then 4 validator nodes created via `test.createValidatorNode` in 2 HA pairs. Each pair * shares its two validator keys plus an in-memory `createSharedSlashingProtectionDb` (so only one peer signs per duty) * — explicitly NOT the Postgres-backed docker-compose HA suite, so this is an in-proc `multi-node` test, not infra. - * Production `Sequencer`, no prover node. Timing: ethSlot=6s, aztecSlot=36s, epoch=8, proofSubEpochs=1024, + * Production `Sequencer`, no prover node. Timing: ethSlot=6s, aztecSlot=36s, epoch=4, proofSubEpochs=1024, * blockDurationMs=8s, committeeSize=4, attestationPropagationTime=0.5, inboxLag=2; anvil on interval mining. Nodes build * empty checkpoints (`buildCheckpointIfEmpty` + `minTxsPerBlock: 0`) so no txs are needed, and each node uses a distinct * coinbase so the secondary assertion can prove which peer produced S2. Time is warped with `cheatCodes.eth.warp`: * `findConsecutiveSamePairSlots` recovers from `ValidatorSelection__EpochNotStable` by warping forward one epoch, and * the test warps to one L1 slot before S1's build slot before starting the sequencers. Routing of S1→builder and * S2→peer uses the test-only `pauseProposingForSlots` hook. - * - * Proposed category: `multi-node` (epochs/) — 4 validators on the mock gossip bus (mirrors - * `epochs_orphan_block_prune` / `epochs_simple_block_building`). See the inline REFACTOR markers for hand-rolled - * coordination a DSL helper should replace. */ -describe('e2e_epochs/epochs_ha_checkpoint_handoff', () => { +describe('multi-node/high-availability/ha_checkpoint_handoff', () => { let context: EndToEndContext; let logger: Logger; let rollup: RollupContract; - let test: EpochsTestContext; - let validators: (Operator & { privateKey: `0x${string}` })[]; + let test: MultiNodeTestContext; + let validators: RegisteredValidator[]; let nodes: AztecNodeService[]; /** @@ -95,85 +90,37 @@ describe('e2e_epochs/epochs_ha_checkpoint_handoff', () => { let haPairs: HaPair[]; async function setupTest() { - validators = times(VALIDATOR_COUNT, i => { - const privateKey = bufferToHex(getPrivateKeyFromIndex(i + 3)!); - const attester = EthAddress.fromString(privateKeyToAccount(privateKey).address); - return { attester, withdrawer: attester, privateKey, bn254SecretKey: new SecretValue(Fr.random().toBigInt()) }; - }); + validators = buildMockGossipValidators(VALIDATOR_COUNT); // No initial sequencer: the validator nodes do all the building, and they build empty checkpoints // (buildCheckpointIfEmpty + minTxsPerBlock: 0) so no transactions are needed. We keep checkpoint - // publishing ENABLED (unlike epochs_ha_sync.test.ts): the handoff must produce a real on-chain - // checkpoint. - test = await EpochsTestContext.setup({ + // publishing ENABLED (unlike ha_sync.test.ts): the handoff must produce a real on-chain checkpoint. + test = await MultiNodeTestContext.setup({ + ...MOCK_GOSSIP_MULTI_VALIDATOR_OPTS, + ...MULTI_VALIDATOR_REORG_TIMING, initialValidators: validators, - mockGossipSubNetwork: true, - startProverNode: false, - skipInitialSequencer: true, - aztecEpochDuration: 8, - aztecProofSubmissionEpochs: 1024, - ethereumSlotDuration: 6, - aztecSlotDuration: 36, - blockDurationMs: 8000, - attestationPropagationTime: 0.5, aztecTargetCommitteeSize: VALIDATOR_COUNT, - inboxLag: 2, }); ({ context, logger, rollup } = test); - // Create 4 nodes in 2 HA pairs: each pair shares the same two validator keys. - const pk1 = validators[0].privateKey; - const pk2 = validators[1].privateKey; - const pk3 = validators[2].privateKey; - const pk4 = validators[3].privateKey; - - const addressesA = [pk1, pk2].map(pk => privateKeyToAccount(pk).address.toLowerCase()); - const addressesB = [pk3, pk4].map(pk => privateKeyToAccount(pk).address.toLowerCase()); - - // Use different coinbase addresses per node so HA peers build distinguishable blocks (the secondary - // assertion relies on this to prove which node produced S2's checkpoint). Each HA pair shares a - // slashing protection DB so only one peer signs per duty. buildCheckpointIfEmpty + minTxsPerBlock: 0 - // lets proposers build empty checkpoints without txs. - const baseOpts = { dontStartSequencer: true, buildCheckpointIfEmpty: true, minTxsPerBlock: 0 } as const; - const sharedDb1 = await createSharedSlashingProtectionDb(context.dateProvider); - const sharedDb2 = await createSharedSlashingProtectionDb(context.dateProvider); - - const coinbaseA1 = EthAddress.fromNumber(1); - const coinbaseA2 = EthAddress.fromNumber(2); - const coinbaseB1 = EthAddress.fromNumber(3); - const coinbaseB2 = EthAddress.fromNumber(4); - + // Create 4 nodes in 2 HA pairs (pk1+pk2, pk3+pk4) sharing keys + a per-pair slashing-protection DB. + // Distinct coinbases per node let the secondary assertion prove which peer produced S2's checkpoint; + // buildCheckpointIfEmpty + minTxsPerBlock: 0 lets proposers build empty checkpoints without txs. logger.warn(`Creating 4 validator nodes in 2 HA pairs.`); - nodes = [ - // Pair A: {nodes[0], nodes[1]} share {pk1, pk2} - await test.createValidatorNode([pk1, pk2], { - ...baseOpts, - coinbase: coinbaseA1, - slashingProtectionDb: sharedDb1, - }), - await test.createValidatorNode([pk1, pk2], { - ...baseOpts, - coinbase: coinbaseA2, - slashingProtectionDb: sharedDb1, - }), - // Pair B: {nodes[2], nodes[3]} share {pk3, pk4} - await test.createValidatorNode([pk3, pk4], { - ...baseOpts, - coinbase: coinbaseB1, - slashingProtectionDb: sharedDb2, - }), - await test.createValidatorNode([pk3, pk4], { - ...baseOpts, - coinbase: coinbaseB2, - slashingProtectionDb: sharedDb2, - }), - ]; + const { nodes: haNodes, pairs } = await setupHaPairs(test, validators, { + baseOpts: { dontStartSequencer: true, buildCheckpointIfEmpty: true, minTxsPerBlock: 0 }, + }); + nodes = haNodes; - haPairs = [ - { nodes: [nodes[0], nodes[1]], addresses: addressesA, coinbases: [coinbaseA1, coinbaseA2] }, - { nodes: [nodes[2], nodes[3]], addresses: addressesB, coinbases: [coinbaseB1, coinbaseB2] }, - ]; + haPairs = pairs.map((pair, pairIndex) => ({ + nodes: pair.nodes, + // The two validators backing this pair: pair 0 → validators[0..1], pair 1 → validators[2..3]. + addresses: [validators[pairIndex * 2], validators[pairIndex * 2 + 1]].map(v => + v.attester.toString().toLowerCase(), + ), + coinbases: pair.coinbases, + })); logger.warn(`Created 4 validator nodes.`); logger.warn(`Test setup completed.`); @@ -209,8 +156,8 @@ describe('e2e_epochs/epochs_ha_checkpoint_handoff', () => { : haPairs.find(pair => pair.addresses.includes(proposer.toString().toLowerCase())); // REFACTOR: hand-rolled slot-search loop with manual epoch arithmetic and warp-on-EpochNotStable retry - // (same pattern as epochs_invalidate_block / epochs_orphan_block_prune) — a shared "find slots matching a - // proposer predicate, warping past EpochNotStable" helper should replace it. + // (same pattern as invalid-attestations/invalidate_block and recovery/proposal_failure_recovery) — a shared + // "find slots matching a proposer predicate, warping past EpochNotStable" helper should replace it. let candidate = Number(test.epochCache.getEpochAndSlotNow().slot) + 4; const maxAttempts = 200; for (let attempt = 0; attempt < maxAttempts; attempt++) { @@ -282,16 +229,13 @@ describe('e2e_epochs/epochs_ha_checkpoint_handoff', () => { // Under proposer pipelining the proposer for proposal slot S1 builds during wall-clock slot S1-1. // Warp to 1 L1 slot before the build slot (S1-1) so the builder starts cleanly. const buildSlotForS1 = SlotNumber(slotS1 - 1); - const buildSlotTimestamp = getTimestampForSlot(buildSlotForS1, test.constants); - await context.cheatCodes.eth.warp(Number(buildSlotTimestamp) - test.L1_BLOCK_TIME_IN_S, { - resetBlockInterval: true, - }); + await test.warpToBuildWindowForSlot(buildSlotForS1); logger.warn(`Warped to 1 L1 slot before L2 build slot ${buildSlotForS1} (proposal slot ${slotS1}).`); expect(await builder.getBlockNumber()).toEqual(0); // Start the sequencers on all nodes. - await Promise.all(nodes.map(n => n.getSequencer()!.start())); + await test.startSequencers(nodes); logger.warn(`Started all sequencers.`); // The builder always records its own proposed S1 checkpoint locally (its sequencer pushes it to the diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_ha_sync.test.ts b/yarn-project/end-to-end/src/multi-node/high-availability/ha_sync.test.ts similarity index 54% rename from yarn-project/end-to-end/src/e2e_epochs/epochs_ha_sync.test.ts rename to yarn-project/end-to-end/src/multi-node/high-availability/ha_sync.test.ts index 923ac1b9bff0..d43066a3dfea 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_ha_sync.test.ts +++ b/yarn-project/end-to-end/src/multi-node/high-availability/ha_sync.test.ts @@ -1,27 +1,24 @@ import type { Archiver } from '@aztec/archiver'; import type { AztecNodeService } from '@aztec/aztec-node'; -import { AztecAddress, EthAddress } from '@aztec/aztec.js/addresses'; -import { NO_WAIT } from '@aztec/aztec.js/contracts'; +import type { AztecAddress } from '@aztec/aztec.js/addresses'; import { Fr } from '@aztec/aztec.js/fields'; import type { Logger } from '@aztec/aztec.js/log'; import { RollupContract } from '@aztec/ethereum/contracts'; -import type { Operator } from '@aztec/ethereum/deploy-aztec-l1-contracts'; import { BlockNumber, SlotNumber } from '@aztec/foundation/branded-types'; -import { times, timesAsync } from '@aztec/foundation/collection'; -import { SecretValue } from '@aztec/foundation/config'; -import { retryUntil } from '@aztec/foundation/retry'; -import { bufferToHex } from '@aztec/foundation/string'; import { TestContract } from '@aztec/noir-test-contracts.js/Test'; -import { getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; -import { createSharedSlashingProtectionDb } from '@aztec/validator-ha-signer/factory'; import { jest } from '@jest/globals'; -import { privateKeyToAccount } from 'viem/accounts'; -import { type EndToEndContext, getPrivateKeyFromIndex } from '../fixtures/utils.js'; -import { TestWallet } from '../test-wallet/test_wallet.js'; -import { proveInteraction } from '../test-wallet/utils.js'; -import { EpochsTestContext } from './epochs_test.js'; +import type { EndToEndContext } from '../../fixtures/utils.js'; +import { TestWallet } from '../../test-wallet/test_wallet.js'; +import { proveAndSendTxs } from '../../test-wallet/utils.js'; +import { + MULTI_VALIDATOR_REORG_TIMING, + MultiNodeTestContext, + type RegisteredValidator, + buildMockGossipValidators, + setupHaPairs, +} from '../multi_node_test_context.js'; jest.setTimeout(1000 * 60 * 20); @@ -36,44 +33,36 @@ const TX_COUNT = 6; * Creates two HA pairs (nodes sharing validator keys) with a shared SlashingProtectionDatabase per * pair, and disables checkpoint publishing on all validator nodes so every node — including the HA * peer that did NOT build a given block — must sync to the proposed chain tip via P2P before any - * checkpoint lands on L1. Uses EpochsTestContext with mockGossipSubNetwork and pxeOpts + * checkpoint lands on L1. Uses MultiNodeTestContext with mockGossipSubNetwork and pxeOpts * syncChainTip='proposed'. */ -describe('e2e_epochs/epochs_ha_sync', () => { +describe('multi-node/high-availability/ha_sync', () => { let context: EndToEndContext; let logger: Logger; let rollup: RollupContract; - let test: EpochsTestContext; - let validators: (Operator & { privateKey: `0x${string}` })[]; + let test: MultiNodeTestContext; + let validators: RegisteredValidator[]; let nodes: AztecNodeService[]; let contract: TestContract; let wallet: TestWallet; let from: AztecAddress; async function setupTest() { - validators = times(VALIDATOR_COUNT, i => { - const privateKey = bufferToHex(getPrivateKeyFromIndex(i + 3)!); - const attester = EthAddress.fromString(privateKeyToAccount(privateKey).address); - return { attester, withdrawer: attester, privateKey, bn254SecretKey: new SecretValue(Fr.random().toBigInt()) }; - }); + validators = buildMockGossipValidators(VALIDATOR_COUNT); - // Do NOT set skipPublishingCheckpointsPercent here: the initial sequencer needs to - // publish checkpoints during setup (account deployment). We disable it per-validator-node below. - test = await EpochsTestContext.setup({ + // Do NOT set skipInitialSequencer / skipPublishingCheckpointsPercent here: the initial sequencer + // needs to publish checkpoints during setup (account deployment). We disable publishing + // per-validator-node below. + test = await MultiNodeTestContext.setup({ + ...MULTI_VALIDATOR_REORG_TIMING, numberOfAccounts: 1, initialValidators: validators, mockGossipSubNetwork: true, - aztecEpochDuration: 4, - ethereumSlotDuration: 4, - aztecSlotDuration: 36, - blockDurationMs: 8000, - attestationPropagationTime: 0.5, aztecTargetCommitteeSize: VALIDATOR_COUNT, minTxsPerBlock: 1, maxTxsPerBlock: 2, pxeOpts: { syncChainTip: 'proposed' }, - inboxLag: 2, }); ({ context, logger, rollup } = test); @@ -84,44 +73,13 @@ describe('e2e_epochs/epochs_ha_sync', () => { logger.warn(`Stopping sequencer in initial aztec node.`); await context.sequencer!.stop(); - // Create 4 nodes in 2 HA pairs: each pair shares the same two validator keys. - const pk1 = validators[0].privateKey; - const pk2 = validators[1].privateKey; - const pk3 = validators[2].privateKey; - const pk4 = validators[3].privateKey; - - // Disable checkpoint publishing on validator nodes so we can assert proposed chain sync - // strictly before any checkpoint is published by the validators. - // Use different coinbase addresses per node so HA peers would build different blocks - // if the proposer's block isn't correctly propagated to its HA peer. - // Each HA pair shares a slashing protection DB so only one peer can sign per duty. - const baseOpts = { dontStartSequencer: true, skipPublishingCheckpointsPercent: 100 } as const; - const sharedDb1 = await createSharedSlashingProtectionDb(context.dateProvider); - const sharedDb2 = await createSharedSlashingProtectionDb(context.dateProvider); - + // Create 4 nodes in 2 HA pairs sharing keys. Disable checkpoint publishing on the validator nodes + // so we can assert proposed chain sync strictly before any checkpoint is published by the + // validators; distinct coinbases let us detect if a peer builds a different block than its proposer. logger.warn(`Creating 4 validator nodes in 2 HA pairs.`); - nodes = [ - await test.createValidatorNode([pk1, pk2], { - ...baseOpts, - coinbase: EthAddress.fromNumber(1), - slashingProtectionDb: sharedDb1, - }), - await test.createValidatorNode([pk1, pk2], { - ...baseOpts, - coinbase: EthAddress.fromNumber(2), - slashingProtectionDb: sharedDb1, - }), - await test.createValidatorNode([pk3, pk4], { - ...baseOpts, - coinbase: EthAddress.fromNumber(3), - slashingProtectionDb: sharedDb2, - }), - await test.createValidatorNode([pk3, pk4], { - ...baseOpts, - coinbase: EthAddress.fromNumber(4), - slashingProtectionDb: sharedDb2, - }), - ]; + ({ nodes } = await setupHaPairs(test, validators, { + baseOpts: { dontStartSequencer: true, skipPublishingCheckpointsPercent: 100 }, + })); logger.warn(`Created 4 validator nodes.`); // Point the wallet at a validator node so it tracks proposed blocks. @@ -152,41 +110,31 @@ describe('e2e_epochs/epochs_ha_sync', () => { logger.warn(`Initial state: checkpoint ${initialCheckpointNumber}, checkpointed block ${initialCheckpointedBlock}`); // Pre-prove and send transactions. - const txs = await timesAsync(TX_COUNT, i => - proveInteraction(context.wallet, contract.methods.emit_nullifier(new Fr(i + 1)), { from }), + const txHashes = await proveAndSendTxs( + context.wallet, + TX_COUNT, + i => contract.methods.emit_nullifier(new Fr(i + 1)), + { from }, ); - const txHashes = await Promise.all(txs.map(tx => tx.send({ wait: NO_WAIT }))); logger.warn(`Sent ${txHashes.length} transactions.`); // Warp to 1 L1 slot before the start of the following L2 slot, so sequencers start cleanly. // We don't warp to the next L2 slot because we may already be less than 1 L1 slot before it. const currentSlot = await rollup.getSlotNumber(); const nextSlot = SlotNumber(currentSlot + 2); - const nextSlotTimestamp = getTimestampForSlot(nextSlot, test.constants); - await context.cheatCodes.eth.warp(Number(nextSlotTimestamp) - test.L1_BLOCK_TIME_IN_S, { - resetBlockInterval: true, - }); + await test.warpToBuildWindowForSlot(nextSlot); logger.warn(`Warped to 1 L1 slot before L2 slot ${nextSlot}.`); // Start the sequencers on all nodes. - await Promise.all(nodes.map(n => n.getSequencer()!.start())); + await test.startSequencers(nodes); logger.warn(`Started all sequencers.`); // Wait until all nodes have proposed blocks strictly beyond the checkpointed tip. // This ensures we're checking blocks produced by validators via P2P proposals, // not blocks synced from L1 checkpoints during setup. - // REFACTOR: hand-rolled poll over all archivers checking proposed > checkpointed; replace with - // a test-context helper such as waitUntilAllNodesProposedBeyondCheckpointed(nodes, timeout). - await retryUntil( - async () => { - const tips = await Promise.all(allArchivers.map(a => a.getL2Tips())); - return tips.every( - t => t.proposed.number > initialCheckpointedBlock && t.proposed.number > t.checkpointed.block.number, - ); - }, - 'all nodes to sync proposed blocks beyond checkpointed tip', - test.L2_SLOT_DURATION_IN_S * 5, - 0.5, + await test.waitForAllNodes( + tips => tips.proposed.number > initialCheckpointedBlock && tips.proposed.number > tips.checkpointed.block.number, + { nodes, timeout: test.L2_SLOT_DURATION_IN_S * 5, interval: 0.5 }, ); logger.warn(`All nodes synced proposed blocks beyond checkpointed tip`); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts b/yarn-project/end-to-end/src/multi-node/invalid-attestations/invalidate_block.parallel.test.ts similarity index 95% rename from yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts rename to yarn-project/end-to-end/src/multi-node/invalid-attestations/invalidate_block.parallel.test.ts index 4751d74224ec..c2cfb7e2eeed 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts +++ b/yarn-project/end-to-end/src/multi-node/invalid-attestations/invalidate_block.parallel.test.ts @@ -2,22 +2,18 @@ import { type Archiver, CalldataRetriever } from '@aztec/archiver'; import type { AztecNodeService } from '@aztec/aztec-node'; import type { AztecAddress } from '@aztec/aztec.js/addresses'; import { NO_WAIT } from '@aztec/aztec.js/contracts'; -import { Fr } from '@aztec/aztec.js/fields'; import type { Logger } from '@aztec/aztec.js/log'; import { waitForTx } from '@aztec/aztec.js/node'; import { RollupContract } from '@aztec/ethereum/contracts'; -import type { Operator } from '@aztec/ethereum/deploy-aztec-l1-contracts'; import type { ExtendedViemWalletClient, ViemPublicClient, ViemPublicDebugClient } from '@aztec/ethereum/types'; import { range } from '@aztec/foundation/array'; import { asyncMap } from '@aztec/foundation/async-map'; import { CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types'; import { times, timesAsync } from '@aztec/foundation/collection'; -import { SecretValue } from '@aztec/foundation/config'; import { EthAddress } from '@aztec/foundation/eth-address'; import { createLogger } from '@aztec/foundation/log'; import { promiseWithResolvers } from '@aztec/foundation/promise'; import { retryUntil } from '@aztec/foundation/retry'; -import { bufferToHex } from '@aztec/foundation/string'; import { executeTimeout, timeoutPromise } from '@aztec/foundation/timer'; import type { TestContract } from '@aztec/noir-test-contracts.js/Test'; import { OffenseType } from '@aztec/slasher'; @@ -26,11 +22,15 @@ import { computeQuorum, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers' import { jest } from '@jest/globals'; import type { Log } from 'viem'; -import { privateKeyToAccount } from 'viem/accounts'; -import { getAnvilPort } from '../fixtures/fixtures.js'; -import { type EndToEndContext, getPrivateKeyFromIndex } from '../fixtures/utils.js'; -import { EpochsTestContext } from './epochs_test.js'; +import { getAnvilPort } from '../../fixtures/fixtures.js'; +import type { EndToEndContext } from '../../fixtures/utils.js'; +import { + MOCK_GOSSIP_MULTI_VALIDATOR_OPTS, + MultiNodeTestContext, + type RegisteredValidator, + buildMockGossipValidators, +} from '../multi_node_test_context.js'; jest.setTimeout(1000 * 60 * 10); @@ -44,39 +44,32 @@ const BASE_ANVIL_PORT = getAnvilPort(); // a mocked gossip bus. The setup injects bad configs (insufficient attestations, fake/high-s/ // unrecoverable signatures, shuffled attestations, parent-validity bypasses) to force invalid // checkpoints, then verifies the next good proposer invalidates them and the chain progresses. -// Slasher is enabled. Uses EpochsTestContext with mockGossipSubNetwork, no initial sequencer, no +// Slasher is enabled. Uses MultiNodeTestContext with mockGossipSubNetwork, no initial sequencer, no // prover node; ports are port-bumped per test via anvilPortOffset to support parallel execution. -describe('e2e_epochs/epochs_invalidate_block', () => { +describe('multi-node/invalid-attestations/invalidate_block', () => { let context: EndToEndContext; let logger: Logger; let l1Client: ExtendedViemWalletClient; let rollupContract: RollupContract; let anvilPortOffset = 0; - let test: EpochsTestContext; - let validators: (Operator & { privateKey: `0x${string}` })[]; + let test: MultiNodeTestContext; + let validators: RegisteredValidator[]; let nodes: AztecNodeService[]; let testContract: TestContract; let from: AztecAddress; beforeEach(async () => { - validators = times(VALIDATOR_COUNT, i => { - const privateKey = bufferToHex(getPrivateKeyFromIndex(i + 3)!); - const attester = EthAddress.fromString(privateKeyToAccount(privateKey).address); - return { attester, withdrawer: attester, privateKey, bn254SecretKey: new SecretValue(Fr.random().toBigInt()) }; - }); + validators = buildMockGossipValidators(VALIDATOR_COUNT); // Setup context with the given set of validators and a mocked gossip sub network. // Uses multiple-blocks-per-slot timing configuration. - test = await EpochsTestContext.setup({ + test = await MultiNodeTestContext.setup({ + ...MOCK_GOSSIP_MULTI_VALIDATOR_OPTS, ethereumSlotDuration: 8, - aztecSlotDuration: 32, + aztecSlotDuration: 36, blockDurationMs: 6000, - numberOfAccounts: 0, initialValidators: validators, - mockGossipSubNetwork: true, - aztecProofSubmissionEpochs: 1024, - startProverNode: false, aztecTargetCommitteeSize: VALIDATOR_COUNT, secondsBeforeInvalidatingBlockAsCommitteeMember: Number.MAX_SAFE_INTEGER, archiverPollingIntervalMS: 200, @@ -87,8 +80,6 @@ describe('e2e_epochs/epochs_invalidate_block', () => { slasherEnabled: true, minTxsPerBlock: 1, maxTxsPerBlock: 1, - skipInitialSequencer: true, - inboxLag: 2, }); ({ context, logger, l1Client } = test); @@ -148,7 +139,7 @@ describe('e2e_epochs/epochs_invalidate_block', () => { l1Client as unknown as ViemPublicDebugClient, VALIDATOR_COUNT, undefined, - createLogger('e2e:epochs_invalidate_block:calldata'), + createLogger('e2e:invalidate_block:calldata'), EthAddress.fromString(rollupContract.address), ); const { attestations } = await calldataRetriever.getCheckpointFromRollupTx( @@ -450,9 +441,8 @@ describe('e2e_epochs/epochs_invalidate_block', () => { // Warp to one L1 block before warpSlot, so the sequencers have a full L2 slot to boot and settle // pipelining before the build window for warpSlot+1 opens at the end of warpSlot. - const warpTo = getTimestampForSlot(warpSlot, test.constants) - BigInt(test.L1_BLOCK_TIME_IN_S); - logger.warn(`Warping L1 to ${warpTo}, one L1 block before slot ${warpSlot}`, { warpSlot, badSlot1, badSlot2 }); - await test.context.cheatCodes.eth.warp(Number(warpTo), { resetBlockInterval: true }); + logger.warn(`Warping L1 to one L1 block before slot ${warpSlot}`, { warpSlot, badSlot1, badSlot2 }); + await test.warpToBuildWindowForSlot(warpSlot); // Start all sequencers with default (good) config and wait for the first checkpoint to land, // so the chain is moving before we apply the bad config to the proposers of the bad slots. @@ -686,10 +676,8 @@ describe('e2e_epochs/epochs_invalidate_block', () => { // window for P1, so the first proposer job that can observe the malicious config is the // intended checkpoint, not an earlier slot owned by the same validator. const buildSlot = SlotNumber.add(badSlot1, -1); - const buildSlotStart = getTimestampForSlot(buildSlot, test.constants); - const warpTo = buildSlotStart - BigInt(test.L1_BLOCK_TIME_IN_S); - logger.warn(`Warping L1 to timestamp ${warpTo} (one L1 block before build slot ${buildSlot})`); - await test.context.cheatCodes.eth.warp(Number(warpTo), { resetBlockInterval: true }); + logger.warn(`Warping L1 to one L1 block before build slot ${buildSlot}`); + await test.warpToBuildWindowForSlot(buildSlot); await Promise.all(sequencers.map(s => s.start())); logger.warn(`Started all sequencers after warping to the target build window`); diff --git a/yarn-project/end-to-end/src/multi-node/multi_node_test_context.ts b/yarn-project/end-to-end/src/multi-node/multi_node_test_context.ts new file mode 100644 index 000000000000..f723207955e2 --- /dev/null +++ b/yarn-project/end-to-end/src/multi-node/multi_node_test_context.ts @@ -0,0 +1,430 @@ +import type { AztecNodeConfig, AztecNodeService } from '@aztec/aztec-node'; +import { EthAddress } from '@aztec/aztec.js/addresses'; +import { Fr } from '@aztec/aztec.js/fields'; +import { RollupContract, type SlashingProposerContract } from '@aztec/ethereum/contracts'; +import type { Operator } from '@aztec/ethereum/deploy-aztec-l1-contracts'; +import type { ViemClient } from '@aztec/ethereum/types'; +import { CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types'; +import { times } from '@aztec/foundation/collection'; +import { SecretValue } from '@aztec/foundation/config'; +import { retryUntil } from '@aztec/foundation/retry'; +import { bufferToHex } from '@aztec/foundation/string'; +import { SlasherAbi } from '@aztec/l1-artifacts'; +import type { L2Tips } from '@aztec/stdlib/block'; +import type { AztecNode, BlockResponse } from '@aztec/stdlib/interfaces/client'; +import type { Offense } from '@aztec/stdlib/slashing'; +import { createSharedSlashingProtectionDb } from '@aztec/validator-ha-signer/factory'; +import type { SlashingProtectionDatabase } from '@aztec/validator-ha-signer/types'; + +import { type GetContractReturnType, getAddress, getContract } from 'viem'; +import { privateKeyToAccount } from 'viem/accounts'; + +import { getPrivateKeyFromIndex } from '../fixtures/utils.js'; +import { + SingleNodeTestContext, + type SingleNodeTestOpts, + type TrackedSequencerEvent, +} from '../single-node/single_node_test_context.js'; + +export { + WORLD_STATE_CHECKPOINT_HISTORY, + WORLD_STATE_BLOCK_CHECK_INTERVAL, + ARCHIVER_POLL_INTERVAL, + DEFAULT_L1_BLOCK_TIME, + REORG_TIMING_BASE, + FAST_REORG_TIMING, + MULTI_VALIDATOR_REORG_TIMING, + MULTI_VALIDATOR_BLOCK_PRODUCTION_TIMING, + WIDE_SLOT_TIMING, + type BlockProposedEvent, + type TrackedSequencerEvent, + type SingleNodeTestOpts, +} from '../single-node/single_node_test_context.js'; + +/** Options for {@link MultiNodeTestContext.setup} — superset of {@link SingleNodeTestOpts}. */ +export type MultiNodeTestOpts = SingleNodeTestOpts; + +/** A registered validator with its on-chain operator data and the L1 private key its node signs with. */ +export type RegisteredValidator = Operator & { privateKey: `0x${string}` }; + +/** + * Builds the deterministic validator set used across the multi-validator tests: `count` validators + * keyed from `getPrivateKeyFromIndex(i + 3)` (indices 0..2 are reserved for the setup account, + * bootstrap node, and prover node, matching the `P2PNetworkTest` convention). This replaces the + * `times(N, i => ({ attester, withdrawer, privateKey, bn254SecretKey }))` block copy-pasted in every + * direct multi-validator test. + */ +export function buildMockGossipValidators(count: number): RegisteredValidator[] { + return times(count, i => { + const privateKey = bufferToHex(getPrivateKeyFromIndex(i + 3)!); + const attester = EthAddress.fromString(privateKeyToAccount(privateKey).address); + return { attester, withdrawer: attester, privateKey, bn254SecretKey: new SecretValue(Fr.random().toBigInt()) }; + }); +} + +/** + * The shared `setup` cluster for the multi-validator tests that run a tight committee on the + * in-memory mock-gossip bus without a prover (block-production / recovery tests). Spread into the `setup` + * call alongside `initialValidators` (from {@link buildMockGossipValidators}). Tests that want a + * prover (MBPS / HA-sync) leave `startProverNode` explicit rather than adopting this preset's `false`. + */ +export const MOCK_GOSSIP_MULTI_VALIDATOR_OPTS = { + mockGossipSubNetwork: true, + skipInitialSequencer: true, + startProverNode: false, + aztecProofSubmissionEpochs: 1024, + numberOfAccounts: 0, +} as const; + +/** + * The `setup` preset for a slasher-enabled committee on the in-memory mock-gossip bus, used by the + * offense-detection tests (`slashing/`). Mirrors {@link MOCK_GOSSIP_MULTI_VALIDATOR_OPTS} but turns + * the slasher on; spread alongside `initialValidators` (from {@link buildMockGossipValidators}) and + * the per-test slashing-round/penalty config. + */ +export const SLASHER_ENABLED_MULTI_VALIDATOR_OPTS = { + mockGossipSubNetwork: true, + skipInitialSequencer: true, + slasherEnabled: true, +} as const; + +/** The slasher and slashing-proposer L1 contracts a slashing test interacts with. */ +export type SlashingContracts = { + rollup: RollupContract; + slasherContract: GetContractReturnType; + slashingProposer: SlashingProposerContract | undefined; +}; + +/** The per-offense penalty knobs a slashing test tunes; all default to a single `unit`. */ +export type SlashingPenalties = { + slashInactivityPenalty: bigint; + slashDataWithholdingPenalty: bigint; + slashBroadcastedInvalidBlockPenalty: bigint; + slashBroadcastedInvalidCheckpointProposalPenalty: bigint; + slashDuplicateProposalPenalty: bigint; + slashDuplicateAttestationPenalty: bigint; + slashProposeInvalidAttestationsPenalty: bigint; + slashProposeDescendantOfCheckpointWithInvalidAttestationsPenalty: bigint; + slashAttestInvalidCheckpointProposalPenalty: bigint; + slashUnknownPenalty: bigint; +}; + +/** The names of every per-offense penalty knob, in declaration order. */ +const SLASHING_PENALTY_KEYS: (keyof SlashingPenalties)[] = [ + 'slashInactivityPenalty', + 'slashDataWithholdingPenalty', + 'slashBroadcastedInvalidBlockPenalty', + 'slashBroadcastedInvalidCheckpointProposalPenalty', + 'slashDuplicateProposalPenalty', + 'slashDuplicateAttestationPenalty', + 'slashProposeInvalidAttestationsPenalty', + 'slashProposeDescendantOfCheckpointWithInvalidAttestationsPenalty', + 'slashAttestInvalidCheckpointProposalPenalty', + 'slashUnknownPenalty', +]; + +/** + * Returns every per-offense slashing penalty set to `unit` (default `1e14` — small enough not to + * kick a validator out). Spread alongside the slashing-round/quorum config in a slashing test's + * `setup`. + */ +export function defaultSlashingPenalties(unit: bigint = BigInt(1e14)): SlashingPenalties { + return Object.fromEntries(SLASHING_PENALTY_KEYS.map(key => [key, unit])) as SlashingPenalties; +} + +/** + * Returns the penalties with only `offense` set to `unit` and every other offense zeroed out, so a + * test isolates a single slashing offense. Names the test's intent (which offense is under test) and + * replaces the ~9-line manual zero-out block. + */ +export function withOnlyOffense(offense: keyof SlashingPenalties, unit: bigint = BigInt(1e14)): SlashingPenalties { + return Object.fromEntries(SLASHING_PENALTY_KEYS.map(key => [key, key === offense ? unit : 0n])) as SlashingPenalties; +} + +/** One HA pair: its two member nodes, the two shared validator keys, and the per-node coinbases. */ +export type HaPairNodes = { + nodes: [AztecNodeService, AztecNodeService]; + privateKeys: [`0x${string}`, `0x${string}`]; + coinbases: [EthAddress, EthAddress]; +}; + +/** + * Stands up two HA pairs from the first four registered validators: nodes[0]/nodes[1] share keys + * pk1+pk2, nodes[2]/nodes[3] share pk3+pk4. Each pair shares an in-memory slashing-protection DB (so + * only one peer signs per duty) and each node gets a distinct coinbase. Encapsulates the ~40-line + * pair-wiring duplicated by both HA tests; the per-test divergence (publishing disabled vs. enabled, + * empty-checkpoint building) is passed through `baseOpts`. + * @returns The four nodes flat, plus the two `HaPairNodes` descriptors. + */ +export async function setupHaPairs( + test: MultiNodeTestContext, + validators: RegisteredValidator[], + opts: { baseOpts?: Partial & { dontStartSequencer?: boolean }; coinbases?: EthAddress[] } = {}, +): Promise<{ nodes: AztecNodeService[]; pairs: [HaPairNodes, HaPairNodes] }> { + const baseOpts = opts.baseOpts ?? {}; + const coinbases = opts.coinbases ?? [1, 2, 3, 4].map(n => EthAddress.fromNumber(n)); + const [pk1, pk2, pk3, pk4] = validators.map(v => v.privateKey); + const sharedDb1 = await createSharedSlashingProtectionDb(test.context.dateProvider); + const sharedDb2 = await createSharedSlashingProtectionDb(test.context.dateProvider); + + const nodes = [ + await test.createValidatorNode([pk1, pk2], { + ...baseOpts, + coinbase: coinbases[0], + slashingProtectionDb: sharedDb1, + }), + await test.createValidatorNode([pk1, pk2], { + ...baseOpts, + coinbase: coinbases[1], + slashingProtectionDb: sharedDb1, + }), + await test.createValidatorNode([pk3, pk4], { + ...baseOpts, + coinbase: coinbases[2], + slashingProtectionDb: sharedDb2, + }), + await test.createValidatorNode([pk3, pk4], { + ...baseOpts, + coinbase: coinbases[3], + slashingProtectionDb: sharedDb2, + }), + ]; + + const pairs: [HaPairNodes, HaPairNodes] = [ + { nodes: [nodes[0], nodes[1]], privateKeys: [pk1, pk2], coinbases: [coinbases[0], coinbases[1]] }, + { nodes: [nodes[2], nodes[3]], privateKeys: [pk3, pk4], coinbases: [coinbases[2], coinbases[3]] }, + ]; + + return { nodes, pairs }; +} + +/** + * Multi-validator test base: N validator nodes sharing the in-memory `MockGossipSubNetwork` bus, with + * fast block times and short epochs. Extends {@link SingleNodeTestContext} with validator-node + * spawning and the convergence helpers (`waitForAllNodes*`, `findSlotsWithProposers`) that only make + * sense across a committee. The environment, prover lifecycle, and reorg/proving waiters live on the + * parent so the single-node-topology tests share them. + */ +export class MultiNodeTestContext extends SingleNodeTestContext { + /** + * The validators registered on-chain at genesis (from `opts.initialValidators`). Tests spawn nodes + * for whichever validators they want online via {@link createValidatorNodeAt}; the rest stay + * registered-but-offline. Empty for single-validator-less topologies. + */ + public validators: RegisteredValidator[] = []; + + public override async setup(opts: MultiNodeTestOpts = {}) { + this.validators = (opts.initialValidators as RegisteredValidator[] | undefined) ?? []; + await super.setup(opts); + } + + public createValidatorNode( + privateKeys: `0x${string}`[], + opts: Partial & { + dontStartSequencer?: boolean; + slashingProtectionDb?: SlashingProtectionDatabase; + } = {}, + ) { + this.logger.warn('Creating and syncing a validator node...'); + return this.createNode({ ...opts, disableValidator: false, validatorPrivateKeys: new SecretValue(privateKeys) }); + } + + /** Returns the validator registered at on-chain index `index` (0-based into {@link validators}). */ + public validatorAt(index: number): RegisteredValidator { + return this.validators[index]; + } + + /** The L1 attester address of the validator registered at `index`. */ + public addressAt(index: number): EthAddress { + return this.validators[index].attester; + } + + /** The L1 signing key of the validator registered at `index`. */ + public privateKeyAt(index: number): `0x${string}` { + return this.validators[index].privateKey; + } + + /** + * Spawns a validator node on the mock-gossip bus signing with the validator registered at `index`. + * Pass the same `index` to two calls (with different `coinbase`) to model an equivocating proposer + * that shares a key across two nodes. + */ + public createValidatorNodeAt( + index: number, + opts: Partial & { dontStartSequencer?: boolean } = {}, + ): Promise { + return this.createValidatorNode([this.privateKeyAt(index)], opts); + } + + /** Resolves the rollup, slasher, and slashing-proposer L1 contracts a slashing test interacts with. */ + public async getSlashingContracts(): Promise { + const rollup = this.rollup; + const slasherContract = getContract({ + address: getAddress((await rollup.getSlasherAddress()).toString()), + abi: SlasherAbi, + client: this.l1Client, + }); + const slashingProposer = await rollup.getSlashingProposer(); + return { rollup, slasherContract, slashingProposer }; + } + + /** + * Polls every node until `predicate(tips, node)` holds for all of them. The multi-node + * generalization of {@link SingleNodeTestContext.waitForNodeToSync} — replaces hand-rolled + * `Promise.all(this.nodes.map(node => retryUntil(...)))` fan-out blocks. + * @param nodes - Nodes to poll; defaults to all validator nodes (`this.nodes`). + */ + public async waitForAllNodes( + predicate: (tips: L2Tips, node: AztecNode) => boolean | Promise, + opts: { nodes?: AztecNode[]; timeout?: number; interval?: number; description?: string } = {}, + ): Promise { + const nodes = opts.nodes ?? this.nodes; + const timeout = opts.timeout ?? this.L2_SLOT_DURATION_IN_S * 4; + const interval = opts.interval ?? 0.5; + const description = opts.description ?? 'all nodes to reach target'; + await Promise.all( + nodes.map((node, idx) => + retryUntil( + async () => { + const tips = await node.getChainTips(); + return (await predicate(tips, node)) || undefined; + }, + `node ${idx} ${description}`, + timeout, + interval, + ), + ), + ); + } + + /** Waits until every node's proven checkpoint tip reaches `target`. */ + public waitForAllNodesToReachProvenCheckpoint( + target: CheckpointNumber, + opts: { nodes?: AztecNode[]; timeout?: number; interval?: number } = {}, + ): Promise { + return this.waitForAllNodes(tips => tips.proven.checkpoint.number >= target, { + ...opts, + description: `proven checkpoint >= ${target}`, + }); + } + + /** Waits until every node's checkpointed checkpoint tip reaches `target`. */ + public waitForAllNodesToReachCheckpoint( + target: CheckpointNumber, + opts: { nodes?: AztecNode[]; timeout?: number; interval?: number } = {}, + ): Promise { + return this.waitForAllNodes(tips => tips.checkpointed.checkpoint.number >= target, { + ...opts, + description: `checkpointed checkpoint >= ${target}`, + }); + } + + /** + * Waits until every node's `proposed` or `checkpointed` tip points at a block whose slot + * satisfies `match` (defaults to "slot equals `slot`"). Polls the block referenced by the tip. + */ + public waitForAllNodesToReachBlockAtSlot( + slot: SlotNumber, + tag: 'proposed' | 'checkpointed', + match: (block: BlockResponse) => boolean = block => block.header.globalVariables.slotNumber === slot, + opts: { nodes?: AztecNode[]; timeout?: number; interval?: number } = {}, + ): Promise { + return this.waitForAllNodes( + async (tips, node) => { + const blockNumber = tag === 'proposed' ? tips.proposed.number : tips.checkpointed.block.number; + if (blockNumber === 0) { + return false; + } + const block = await node.getBlock(blockNumber); + return !!block && match(block); + }, + { ...opts, description: `${tag} block at slot ${slot}` }, + ); + } + + /** + * Finds `count` consecutive slots (starting from `opts.fromSlot` or the current slot plus a + * margin) whose proposers satisfy `predicate`, warping the L1 clock forward one epoch and + * retrying when the rollup reports `ValidatorSelection__EpochNotStable` for a future epoch. + * Returns the matched slots and their proposer addresses. Encapsulates the slot-search loop + * duplicated across the multi-validator tests. + */ + public async findSlotsWithProposers( + count: number, + predicate: (proposers: EthAddress[]) => boolean, + opts: { fromSlot?: SlotNumber; margin?: number; maxAttempts?: number } = {}, + ): Promise<{ slots: SlotNumber[]; proposers: EthAddress[] }> { + const margin = opts.margin ?? 4; + const maxAttempts = opts.maxAttempts ?? 200; + let candidate = opts.fromSlot ?? SlotNumber(Number(this.epochCache.getEpochAndSlotNow().slot) + margin); + + for (let attempt = 0; attempt < maxAttempts; attempt++) { + try { + const slots = Array.from({ length: count }, (_, i) => SlotNumber(candidate + i)); + const maybeProposers = await Promise.all( + slots.map(slot => this.epochCache.getProposerAttesterAddressInSlot(slot)), + ); + if (maybeProposers.every((p): p is EthAddress => p !== undefined) && predicate(maybeProposers)) { + return { slots, proposers: maybeProposers }; + } + candidate = SlotNumber(candidate + 1); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + if (!msg.includes('EpochNotStable')) { + throw err; + } + const block = await this.l1Client.getBlock({ includeTransactions: false }); + const warpBy = this.epochDuration * this.L2_SLOT_DURATION_IN_S; + const newTs = Number(block.timestamp) + warpBy; + this.logger.warn(`Hit EpochNotStable at candidate ${candidate}, warping L1 forward by ${warpBy}s to ${newTs}`); + await this.context.cheatCodes.eth.warp(newTs, { resetBlockInterval: true }); + const newCurrentSlot = Number(this.epochCache.getEpochAndSlotNow().slot); + if (candidate < newCurrentSlot + margin) { + candidate = SlotNumber(newCurrentSlot + margin); + } + } + } + throw new Error( + `Could not find ${count} consecutive slots matching the proposer predicate after ${maxAttempts} attempts`, + ); + } + + /** + * Watches the sequencers of `nodes` via {@link SingleNodeTestContext.watchSequencerEvents}, pulling + * the {@link SequencerClient} off each node first. `getMetadata` tags each captured event; it + * defaults to `{ validator: this.validators[i].attester }`. Pass an override for tests that label + * their nodes differently (e.g. `['A','B','C'][i]`). + */ + public watchNodeSequencerEvents( + nodes: AztecNodeService[], + getMetadata: (i: number) => Record = i => ({ validator: this.validators[i].attester }), + ): { failEvents: TrackedSequencerEvent[]; stateChanges: TrackedSequencerEvent[] } { + return this.watchSequencerEvents(this.getSequencers(nodes), getMetadata); + } + + /** + * Waits until matching slash offenses have converged across `nodes`, polling `getSlashOffenses`. + * With `opts.mode === 'all'` (the default) every node must record a matching offense; with `'any'` + * a single node suffices. Resolves with all matching offenses collected across the polled nodes. + */ + public waitForOffenseOnNodes( + nodes: AztecNodeService[], + match: (offense: Offense) => boolean, + opts: { mode?: 'all' | 'any'; timeout?: number; interval?: number } = {}, + ): Promise { + const mode = opts.mode ?? 'all'; + const timeout = opts.timeout ?? this.L2_SLOT_DURATION_IN_S * 4; + const interval = opts.interval ?? 0.5; + return retryUntil( + async () => { + const perNode = await Promise.all(nodes.map(node => node.getSlashOffenses('all').then(os => os.filter(match)))); + const converged = + mode === 'all' ? perNode.every(matches => matches.length > 0) : perNode.some(m => m.length > 0); + return converged ? perNode.flat() : undefined; + }, + `offense on ${mode} node(s)`, + timeout, + interval, + ); + } +} diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_equivocation.test.ts b/yarn-project/end-to-end/src/multi-node/recovery/equivocation_recovery.test.ts similarity index 62% rename from yarn-project/end-to-end/src/e2e_epochs/epochs_equivocation.test.ts rename to yarn-project/end-to-end/src/multi-node/recovery/equivocation_recovery.test.ts index 485ec595b03a..61fac1f986b9 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_equivocation.test.ts +++ b/yarn-project/end-to-end/src/multi-node/recovery/equivocation_recovery.test.ts @@ -1,22 +1,21 @@ import type { AztecNodeService } from '@aztec/aztec-node'; import { EthAddress } from '@aztec/aztec.js/addresses'; -import { Fr } from '@aztec/aztec.js/fields'; import type { Logger } from '@aztec/aztec.js/log'; import { asyncMap } from '@aztec/foundation/async-map'; import { CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types'; -import { times } from '@aztec/foundation/collection'; -import { SecretValue } from '@aztec/foundation/config'; -import { retryUntil } from '@aztec/foundation/retry'; -import { bufferToHex } from '@aztec/foundation/string'; -import { getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; import { tryStop } from '@aztec/stdlib/interfaces/server'; import { OffenseType } from '@aztec/stdlib/slashing'; import { jest } from '@jest/globals'; import { privateKeyToAccount } from 'viem/accounts'; -import { getPrivateKeyFromIndex } from '../fixtures/utils.js'; -import { EpochsTestContext } from './epochs_test.js'; +import { + MOCK_GOSSIP_MULTI_VALIDATOR_OPTS, + MULTI_VALIDATOR_REORG_TIMING, + MultiNodeTestContext, + buildMockGossipValidators, + withOnlyOffense, +} from '../multi_node_test_context.js'; jest.setTimeout(1000 * 60 * 15); @@ -38,11 +37,11 @@ const NODE_COUNT = 4; * It additionally verifies that the chain heals after node A is stopped, and that every observing * validator records a DUPLICATE_PROPOSAL slashing offense. * - * Uses EpochsTestContext with mockGossipSubNetwork, no initial sequencer, and slasherEnabled. + * Uses MultiNodeTestContext with mockGossipSubNetwork, no initial sequencer, and slasherEnabled. */ -describe('e2e_epochs/epochs_equivocation', () => { +describe('multi-node/recovery/equivocation_recovery', () => { let logger: Logger; - let test: EpochsTestContext; + let test: MultiNodeTestContext; let nodes: AztecNodeService[]; afterEach(async () => { @@ -55,12 +54,8 @@ describe('e2e_epochs/epochs_equivocation', () => { // then for A's L1-confirmed checkpoint to override it on those nodes. Stops A, re-enables // publishing on B/C, waits for chain recovery, and asserts DUPLICATE_PROPOSAL offense on B and C. it('L1-confirmed checkpoint overrides gossip-only equivocating proposal', async () => { - // Build 4 validators (V1..V4) using getPrivateKeyFromIndex(i+3), same convention as other epoch tests. - const validators = times(NODE_COUNT, i => { - const privateKey = bufferToHex(getPrivateKeyFromIndex(i + 3)!); - const attester = EthAddress.fromString(privateKeyToAccount(privateKey).address); - return { attester, withdrawer: attester, privateKey, bn254SecretKey: new SecretValue(Fr.random().toBigInt()) }; - }); + // Build 4 validators (V1..V4) using the shared deterministic builder (keys from index 3). + const validators = buildMockGossipValidators(NODE_COUNT); // Timing calculation for 3 blocks per checkpoint with 8s sub-slots: // - initializationOffset = 0.5s (test mode, ethereumSlotDuration < 8) @@ -69,20 +64,11 @@ describe('e2e_epochs/epochs_equivocation', () => { // - finalBlockDuration = 8s (re-execution) // - Total: 0.5 + 24 + 8 + 2.5 = 35s => use 36s const slashingUnit = BigInt(1e14); - test = await EpochsTestContext.setup({ - numberOfAccounts: 0, + test = await MultiNodeTestContext.setup({ + ...MOCK_GOSSIP_MULTI_VALIDATOR_OPTS, + ...MULTI_VALIDATOR_REORG_TIMING, initialValidators: validators, - inboxLag: 2, - mockGossipSubNetwork: true, - startProverNode: false, - aztecEpochDuration: 4, - aztecProofSubmissionEpochs: 1024, - ethereumSlotDuration: 6, - aztecSlotDuration: 36, - blockDurationMs: 8000, - attestationPropagationTime: 0.5, aztecTargetCommitteeSize: 4, - skipInitialSequencer: true, // Enable the slasher so we can assert the equivocating proposer is detected for slashing. // Round size is aztecEpochDuration * slashingRoundSizeInEpochs = 4 slots; the L1 contract // requires QUORUM > ROUND_SIZE / 2, so quorum must be at least 3. @@ -94,17 +80,8 @@ describe('e2e_epochs/epochs_equivocation', () => { slashAmountMedium: slashingUnit * 2n, slashAmountLarge: slashingUnit * 3n, slashSelfAllowed: true, - slashDuplicateProposalPenalty: slashingUnit, - // Disable other offense penalties so we only see the equivocation offense. - slashInactivityPenalty: 0n, - slashDataWithholdingPenalty: 0n, - slashBroadcastedInvalidBlockPenalty: 0n, - slashBroadcastedInvalidCheckpointProposalPenalty: 0n, - slashDuplicateAttestationPenalty: 0n, - slashProposeInvalidAttestationsPenalty: 0n, - slashProposeDescendantOfCheckpointWithInvalidAttestationsPenalty: 0n, - slashAttestInvalidCheckpointProposalPenalty: 0n, - slashUnknownPenalty: 0n, + // Isolate the equivocation offense: only the duplicate-proposal penalty is non-zero. + ...withOnlyOffense('slashDuplicateProposalPenalty', slashingUnit), }); logger = test.logger; @@ -185,70 +162,39 @@ describe('e2e_epochs/epochs_equivocation', () => { logger.warn(`Expected proposer for submission slot`, { submissionSlot, proposerAttester }); // Warp to one L1 slot before the target L2 slot so pipelining's build window engages. - const slotStartTimestamp = getTimestampForSlot(targetSlot, test.constants); - const warpTo = slotStartTimestamp - BigInt(test.L1_BLOCK_TIME_IN_S); - logger.warn(`Warping to L1 timestamp ${warpTo} (one L1 slot before L2 slot ${targetSlot})`); - await test.context.cheatCodes.eth.warp(Number(warpTo), { resetBlockInterval: true }); + await test.warpToBuildWindowForSlot(targetSlot); // Start all sequencers now that the clock is warped. - const sequencers = nodes.slice(0, 3).map(n => n.getSequencer()!); - const { failEvents } = test.watchSequencerEvents(sequencers, i => ({ validator: ['A', 'B', 'C'][i] })); - await Promise.all(sequencers.map(s => s.start())); + const sequencerNodes = nodes.slice(0, 3); + const { failEvents } = test.watchNodeSequencerEvents(sequencerNodes, i => ({ validator: ['A', 'B', 'C'][i] })); + await test.startSequencers(sequencerNodes); logger.warn('All sequencers started'); // Wait until each of B, C, D sees a proposed block for submissionSlot with coinbase B or C. // This confirms the gossip-only equivocating proposal from B or C has propagated. - // REFACTOR: This is candidate for a "wait until all nodes see a block with these properties" helper in the test context. + const observerNodes = [nodeB, nodeC, nodeD]; const gossipTimeout = test.L2_SLOT_DURATION_IN_S * 4; - await Promise.all( - [nodeB, nodeC, nodeD].map(async (node, idx) => { - const nodeName = ['B', 'C', 'D'][idx]; - let observedCoinbase: EthAddress | undefined; - await retryUntil( - async () => { - const block = await node.getBlock('proposed'); - if (!block) { - return false; - } - const slot = block.header.globalVariables.slotNumber; - const cb = block.header.globalVariables.coinbase; - if (slot === submissionSlot && (cb.equals(coinbaseB) || cb.equals(coinbaseC))) { - observedCoinbase = cb; - return true; - } - return false; - }, - `${nodeName} sees gossip-only proposed block for slot ${submissionSlot}`, - gossipTimeout, - 0.5, - ); - logger.warn(`Node ${nodeName} observed gossip-only coinbase for slot ${submissionSlot}`, { observedCoinbase }); - }), + await test.waitForAllNodesToReachBlockAtSlot( + submissionSlot, + 'proposed', + block => + block.header.globalVariables.slotNumber === submissionSlot && + (block.header.globalVariables.coinbase.equals(coinbaseB) || + block.header.globalVariables.coinbase.equals(coinbaseC)), + { nodes: observerNodes, timeout: gossipTimeout, interval: 0.5 }, ); // Now wait until each of B, C, D has a checkpointed block for submissionSlot with coinbaseA. // This confirms A's L1-confirmed checkpoint has overridden the gossip-only proposal. - // REFACTOR: This is candidate for a "wait until all nodes see a block with these properties" helper in the test context. const overrideTimeout = test.L2_SLOT_DURATION_IN_S * 4; logger.warn(`Waiting for L1-sync override on B, C, D (timeout=${overrideTimeout}s)`); - await Promise.all( - [nodeB, nodeC, nodeD].map(async (node, idx) => { - const nodeName = ['B', 'C', 'D'][idx]; - await retryUntil( - async () => { - const block = await node.getBlock('checkpointed'); - if (!block) { - return false; - } - const slot = block.header.globalVariables.slotNumber; - const cb = block.header.globalVariables.coinbase; - return slot >= submissionSlot && cb.equals(coinbaseA); - }, - `${nodeName} checkpointed block for slot ${submissionSlot} with coinbaseA`, - overrideTimeout, - 0.5, - ); - }), + await test.waitForAllNodesToReachBlockAtSlot( + submissionSlot, + 'checkpointed', + block => + block.header.globalVariables.slotNumber >= submissionSlot && + block.header.globalVariables.coinbase.equals(coinbaseA), + { nodes: observerNodes, timeout: overrideTimeout, interval: 0.5 }, ); // Assert no spurious failures on B, C. @@ -285,20 +231,11 @@ describe('e2e_epochs/epochs_equivocation', () => { expect(test.monitor.checkpointNumber).toBeGreaterThanOrEqual(healTarget); logger.warn(`Network healed: checkpoint ${test.monitor.checkpointNumber}`); - // REFACTOR: This is candidate for a "wait until all nodes sync to a chain tip with these properties" helper in the test context. - await Promise.all( - [nodeB, nodeC, nodeD].map((node, idx) => - retryUntil( - async () => { - const tips = await node.getChainTips(); - return tips.checkpointed.checkpoint.number >= healTarget; - }, - `${'BCD'[idx]} synced to checkpoint ${healTarget}`, - healTimeout, - 0.5, - ), - ), - ); + await test.waitForAllNodesToReachCheckpoint(healTarget, { + nodes: observerNodes, + timeout: healTimeout, + interval: 0.5, + }); // Every observing validator should have recorded the equivocation offense. A has been stopped // above and D is a non-validator (no slasher), so we poll only B and C. @@ -306,20 +243,13 @@ describe('e2e_epochs/epochs_equivocation', () => { proposerAttester, submissionSlot, }); - const matchesOffense = (o: { offenseType: OffenseType; validator: { toString(): string }; epochOrSlot: bigint }) => - o.offenseType === OffenseType.DUPLICATE_PROPOSAL && - o.validator.toString() === proposerAttester.toString() && - o.epochOrSlot === BigInt(submissionSlot); - await retryUntil( - async () => { - const found = await Promise.all( - [nodeB, nodeC].map(async n => (await n.getSlashOffenses('all')).some(matchesOffense)), - ); - return found.every(Boolean); - }, - `DUPLICATE_PROPOSAL offense on every observing node`, - test.L2_SLOT_DURATION_IN_S * 4, - 0.5, + await test.waitForOffenseOnNodes( + [nodeB, nodeC], + o => + o.offenseType === OffenseType.DUPLICATE_PROPOSAL && + o.validator.toString() === proposerAttester.toString() && + o.epochOrSlot === BigInt(submissionSlot), + { mode: 'all', timeout: test.L2_SLOT_DURATION_IN_S * 4, interval: 0.5 }, ); }); }); diff --git a/yarn-project/end-to-end/src/multi-node/recovery/pipeline_prune.test.ts b/yarn-project/end-to-end/src/multi-node/recovery/pipeline_prune.test.ts new file mode 100644 index 000000000000..296aa8e06537 --- /dev/null +++ b/yarn-project/end-to-end/src/multi-node/recovery/pipeline_prune.test.ts @@ -0,0 +1,221 @@ +import type { Archiver } from '@aztec/archiver'; +import type { AztecNodeService } from '@aztec/aztec-node'; +import type { AztecAddress, EthAddress } from '@aztec/aztec.js/addresses'; +import { Fr } from '@aztec/aztec.js/fields'; +import type { Logger } from '@aztec/aztec.js/log'; +import { waitForTx } from '@aztec/aztec.js/node'; +import type { EpochCacheInterface } from '@aztec/epoch-cache'; +import { asyncMap } from '@aztec/foundation/async-map'; +import { BlockNumber, CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types'; +import { executeTimeout } from '@aztec/foundation/timer'; +import type { TestContract } from '@aztec/noir-test-contracts.js/Test'; +import type { SequencerEvents } from '@aztec/sequencer-client'; +import { L2BlockSourceEvents } from '@aztec/stdlib/block'; + +import { jest } from '@jest/globals'; + +import type { EndToEndContext } from '../../fixtures/utils.js'; +import type { TestWallet } from '../../test-wallet/test_wallet.js'; +import { proveAndSendTxs } from '../../test-wallet/utils.js'; +import { + type BlockProposedEvent, + MultiNodeTestContext, + type RegisteredValidator, + WIDE_SLOT_TIMING, + buildMockGossipValidators, +} from '../multi_node_test_context.js'; + +jest.setTimeout(1000 * 60 * 20); + +const NODE_COUNT = 4; +const EXPECTED_BLOCKS_PER_CHECKPOINT = 8; + +// Send enough transactions to trigger multiple blocks within a checkpoint assuming 2 txs per block. +const TX_COUNT = 34; + +/** + * E2E prune-and-recover test under proposer pipelining with MBPS. A selected next proposer is configured + * to skip its checkpoint publish mid-run, which triggers an uncheckpointed-blocks prune; publishing is + * then re-enabled and the chain recovers. Asserts that recovery still produces a multi-block checkpoint + * with the correct pipelining build-vs-submission slot offset, and that the recovered block number is + * past the pre-prune baseline. + * + * Four-validator suite with a prover node (fake proofs) and 500ms mock gossip latency to simulate adverse + * network conditions. Relocated from the dissolved `mbps.pipeline.parallel` file. Uses MultiNodeTestContext + * with mockGossipSubNetwork and no initial sequencer. + */ +describe('multi-node/recovery/pipeline_prune', () => { + let context: EndToEndContext; + let logger: Logger; + let archiver: Archiver; + + let test: MultiNodeTestContext; + let validators: RegisteredValidator[]; + let nodes: AztecNodeService[]; + let contract: TestContract; + let wallet: TestWallet; + let from: AztecAddress; + + /** Creates validators and sets up the test context with MBPS and proposer pipelining. */ + async function setupTest(opts: { + syncChainTip: 'proposed' | 'checkpointed'; + minTxsPerBlock?: number; + maxTxsPerBlock?: number; + }) { + const { syncChainTip = 'checkpointed', ...setupOpts } = opts; + + validators = buildMockGossipValidators(NODE_COUNT); + + test = await MultiNodeTestContext.setup({ + ...WIDE_SLOT_TIMING, + numberOfAccounts: 0, + initialValidators: validators, + mockGossipSubNetwork: true, + mockGossipSubNetworkLatency: 500, // adverse network conditions + startProverNode: true, + maxTxsPerCheckpoint: 24, + inboxLag: 2, + ...setupOpts, + pxeOpts: { syncChainTip }, + skipInitialSequencer: true, + }); + + ({ context, logger } = test); + wallet = context.wallet as TestWallet; + from = context.accounts[0]; // auto-created by setup + + logger.warn(`Initial setup complete. Starting ${NODE_COUNT} validator nodes.`); + // Clear inherited coinbase so each validator derives coinbase from its own attester key + nodes = await asyncMap(validators, ({ privateKey }, i) => + test.createValidatorNode([privateKey], { + dontStartSequencer: true, + coinbase: undefined, + // Disable checkpoint promotion on the first node so it always fetches blobs, + // allowing us to assert that other nodes skip blob fetching via promotion. + ...(i === 0 ? { skipPromoteProposedCheckpointDuringL1Sync: true } : {}), + }), + ); + logger.warn(`Started ${NODE_COUNT} validator nodes.`, { validators: validators.map(v => v.attester.toString()) }); + + wallet.updateNode(nodes[0]); + archiver = nodes[0].getBlockSource() as Archiver; + + contract = await test.registerTestContract(wallet); + logger.warn(`Test setup completed.`, { validators: validators.map(v => v.attester.toString()) }); + } + + afterEach(async () => { + jest.restoreAllMocks(); + await test?.teardown(); + }); + + // Establishes a baseline at checkpoint 1. Identifies the next proposer and disables its + // checkpoint publishing. Waits for the L2PruneUncheckpointed event on the archiver, then + // re-enables publishing. Waits for all txs to be mined, asserts a MBPS checkpoint exists, + // verifies the pipelining offset, and checks recovery blockNumber > baseline. + it('prunes uncheckpointed blocks when proposer fails to deliver', async () => { + await setupTest({ syncChainTip: 'checkpointed', minTxsPerBlock: 1, maxTxsPerBlock: 2 }); + + const blockProposedEvents: BlockProposedEvent[] = []; + const sequencers = test.getSequencers(nodes); + + // Pre-prove and send transactions + const txHashes = await proveAndSendTxs( + context.wallet, + TX_COUNT, + i => contract.methods.emit_nullifier(new Fr(i + 1)), + { from }, + ); + logger.warn(`Sent ${txHashes.length} transactions`, { txs: txHashes }); + + await test.startSequencers(nodes); + logger.warn(`Started all sequencers`); + + // Assert that at least 1 checkpoint has been reached + const checkpointTimeout = test.L2_SLOT_DURATION_IN_S * test.epochDuration * 3; + await test.waitUntilCheckpointNumber(CheckpointNumber(1), checkpointTimeout); + const checkpointedBlockNumber = await archiver.getBlockNumber(); + logger.warn(`Baseline established: checkpoint 1 reached at block ${checkpointedBlockNumber}`); + // Target a submission slot whose pipelined build has not started yet. + const { slot: currentSlot } = test.epochCache.getEpochAndSlotNow(); + const { proposerIndex, slot: proposerSlotToNotPublish } = await findNextProposerIndex( + test.epochCache, + validators, + SlotNumber(currentSlot + 2), + ); + logger.warn( + `Will skip checkpoint publishing for proposer ${proposerIndex} in slot ${proposerSlotToNotPublish} - current slot ${currentSlot}`, + ); + + const targetSequencer = nodes[proposerIndex].getSequencer(); + if (!targetSequencer) { + throw new Error('Target proposer sequencer not found'); + } + // Subscribe to prune event BEFORE disabling publishing, so we don't miss the event + const prunePromise = new Promise(resolve => { + archiver.events.once(L2BlockSourceEvents.L2PruneUncheckpointed, () => resolve()); + }); + + // The sequencer keeps building blocks and broadcasting via P2P, but won't submit the checkpoint to L1 + targetSequencer.updateConfig({ skipPublishingCheckpointsPercent: 100 }); + + const pruneTimeout = test.L2_SLOT_DURATION_IN_S * 5 * 1000; + logger.warn(`Waiting for uncheckpointed blocks to be pruned (timeout=${pruneTimeout}ms)`); + await executeTimeout(() => prunePromise, pruneTimeout); + + // add block proposed listeners after the prune + for (const sequencer of sequencers) { + sequencer.getSequencer().on('block-proposed', (args: Parameters[0]) => { + logger.warn(`block-proposed event: blockNumber=${args.blockNumber}, slot=${args.slot}`, args); + blockProposedEvents.push({ + blockNumber: args.blockNumber, + slot: args.slot, + buildSlot: args.buildSlot, + }); + }); + } + logger.warn(`Pruning detected, block number now ${await archiver.getBlockNumber()}`); + + // Re-enable checkpoint publishing + logger.warn(`Re-enabling checkpoint publishing for validator ${proposerIndex}`); + targetSequencer.updateConfig({ skipPublishingCheckpointsPercent: 0 }); + + // Wait for a new checkpoint (recovery) - where all txs end up mined + const timeout = test.L2_SLOT_DURATION_IN_S * 5; + const receipts = await executeTimeout( + () => Promise.all(txHashes.map(txHash => waitForTx(context.aztecNode, txHash, { timeout }))), + timeout * 1000, + ); + logger.warn(`All txs have been mined`); + + // Verify MBPS works with pipelining; target the highest block number across mined receipts + const maxMinedBlockNumber = BlockNumber(Math.max(...receipts.map(r => r.blockNumber ?? 0))); + await test.assertMultipleBlocksPerSlot(EXPECTED_BLOCKS_PER_CHECKPOINT, { + targetBlock: maxMinedBlockNumber, + archiver, + }); + + // Verify the pipelining offset: build slot N vs submission slot N+1 + await test.assertProposerPipelining(archiver, blockProposedEvents, logger); + + const recoveredBlockNumber = await archiver.getBlockNumber(); + logger.warn(`Recovery complete: block number ${recoveredBlockNumber} > ${checkpointedBlockNumber}`); + expect(recoveredBlockNumber).toBeGreaterThan(checkpointedBlockNumber); + }); +}); + +/** Scans upcoming slots to find which validator proposes next and returns its index. */ +async function findNextProposerIndex( + epochCache: EpochCacheInterface, + validators: { attester: EthAddress }[], + slotToDisable: SlotNumber, +): Promise<{ proposerIndex: number; slot: SlotNumber }> { + const proposer = await epochCache.getProposerAttesterAddressInSlot(SlotNumber(slotToDisable)); + if (proposer) { + const idx = validators.findIndex(v => v.attester.equals(proposer)); + if (idx >= 0) { + return { proposerIndex: idx, slot: SlotNumber(slotToDisable) }; + } + } + throw new Error(`No proposer found in slot ${slotToDisable}`); +} diff --git a/yarn-project/end-to-end/src/multi-node/recovery/proposal_failure_recovery.parallel.test.ts b/yarn-project/end-to-end/src/multi-node/recovery/proposal_failure_recovery.parallel.test.ts new file mode 100644 index 000000000000..35387b9d38b8 --- /dev/null +++ b/yarn-project/end-to-end/src/multi-node/recovery/proposal_failure_recovery.parallel.test.ts @@ -0,0 +1,466 @@ +import type { Archiver } from '@aztec/archiver'; +import type { AztecNodeService } from '@aztec/aztec-node'; +import { EthAddress } from '@aztec/aztec.js/addresses'; +import type { Logger } from '@aztec/aztec.js/log'; +import { waitUntilL1Timestamp } from '@aztec/ethereum/l1-tx-utils'; +import { asyncMap } from '@aztec/foundation/async-map'; +import { BlockNumber, CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types'; +import { retryUntil } from '@aztec/foundation/retry'; +import { timeoutPromise } from '@aztec/foundation/timer'; +import { type L2Block, L2BlockSourceEvents, type L2Tips } from '@aztec/stdlib/block'; +import { getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; + +import { jest } from '@jest/globals'; + +import { + MOCK_GOSSIP_MULTI_VALIDATOR_OPTS, + MULTI_VALIDATOR_REORG_TIMING, + MultiNodeTestContext, + type RegisteredValidator, + buildMockGossipValidators, +} from '../multi_node_test_context.js'; + +jest.setTimeout(1000 * 60 * 15); + +const NODE_COUNT = 4; + +/** + * Production-recovery suite under proposer pipelining: a checkpoint proposal fails to land (missed L1 + * publish, or a withheld CheckpointProposal leaving an orphan), every node prunes the uncheckpointed + * blocks, and the next proposer rebuilds a fresh checkpoint that lands on L1. + * + * Both scenarios share the same 4-validator mock-gossip cluster (one key per node, no prover) on the + * multi-validator reorg cadence (ethSlot=6s, aztecSlot=36s, epoch=4, proofSubmissionEpochs=1024, blockDurationMs=8000, + * inboxLag=2 — v5 always enforces the timetable). Each test warps L1 to align with its target build slot. + */ +describe('multi-node/recovery/proposal_failure_recovery', () => { + let logger: Logger; + let test: MultiNodeTestContext; + let validators: RegisteredValidator[]; + let nodes: AztecNodeService[]; + + beforeEach(async () => { + // Build 4 distinct validators (V1..V4). One key per node, no overlap. + validators = buildMockGossipValidators(NODE_COUNT); + + test = await MultiNodeTestContext.setup({ + ...MOCK_GOSSIP_MULTI_VALIDATOR_OPTS, + ...MULTI_VALIDATOR_REORG_TIMING, + initialValidators: validators, + aztecTargetCommitteeSize: NODE_COUNT, + }); + + ({ logger } = test); + + // One node per validator. dontStartSequencer until after the warp so timing is deterministic. + nodes = await asyncMap(validators, ({ privateKey }, i) => + test.createValidatorNode([privateKey], { + dontStartSequencer: true, + coinbase: EthAddress.fromNumber(0xa + i), + buildCheckpointIfEmpty: true, + minTxsPerBlock: 0, + }), + ); + + logger.warn('Validator nodes created', { + validators: validators.map((v, i) => ({ idx: i, attester: v.attester.toString() })), + }); + }); + + afterEach(async () => { + jest.restoreAllMocks(); + await test?.teardown(); + }); + + /** + * Missed L1 publish: each of 4 nodes holds one validator key. We pick four consecutive slots + * (slotZero, slotOne, slotTwo, slotThree) such that the proposers for slotOne, slotTwo, and slotThree + * are three distinct validators, then warp to one L1 block before slotZero begins. The proposer for + * slotOne is configured to skip its L1 publish. + * + * With pipelining, the proposer for slot N+1 builds and gossips its checkpoint during slot N, then + * publishes that checkpoint to L1 during slot N+1. So gossip-driven `proposed` chain advances arrive + * one slot earlier than the L1-driven `checkpointed` advance. + * + * Expected behavior: + * - During slotZero, the pipelined proposer for slotOne gossips its build → every node's `proposed` + * tip advances to a block at slotOne. + * - During slotOne, the pipelined proposer for slotTwo gossips on top of the slotOne proposal → + * `proposed` advances to a block at slotTwo. Meanwhile the proposer for slotOne attempts L1 publish + * but is configured to skip it, so no checkpoint lands. + * - When slotOne ends with no checkpoint mined, every node's archiver prunes the uncheckpointed + * slotOne and slotTwo blocks; we verify rollback via the prune event. We then re-enable publishing + * on the formerly suppressed node so recovery can proceed. + * - During slotTwo, the pipelined proposer for slotThree builds on top of the (now genesis) + * checkpointed tip → `proposed` advances again. + * - During slotThree, that pipelined work is published → `checkpointed` finally advances. + */ + // Searches for slotOne..slotThree with three distinct proposers (warp on EpochNotStable). Sets + // skipPublishingCheckpointsPercent=100 on proposerOne's node. Warps L1 to slotZero-1 L1 block. + // Subscribes to prune events on all nodes. Starts all sequencers and verifies: proposed tip + // reaches slotOne then slotTwo; all nodes emit L2PruneUncheckpointed at slotOne end; recovery + // produces a checkpointed block at slotThree. Sanity-checks no unexpected fail events. + it('all nodes prune and recover when proposer fails to publish to L1', async () => { + const attesterAddresses = validators.map(v => v.attester); + logger.warn('Validator nodes created', { + validators: attesterAddresses.map((a, i) => ({ idx: i, attester: a.toString() })), + }); + + // Find slotOne..slotThree (>=4 slots ahead) with three distinct proposers. The +4 margin (vs +2 + // in equivocation) gives the warp+sequencer-start path enough headroom to reach the build window + // for slotZero even if node creation jitters. findSlotsWithProposers handles the EpochNotStable + // warp-and-retry: the L1 rollup only exposes proposers for epochs whose randao seed is queryable + // now, so the helper warps L1 forward one epoch at a time until the candidate epoch is stable. + const { + slots: [slotOne, slotTwo, slotThree], + proposers: [proposerOne, proposerTwo, proposerThree], + } = await test.findSlotsWithProposers(3, ([p1, p2, p3]) => !p1.equals(p2) && !p1.equals(p3) && !p2.equals(p3)); + + const slotZero = SlotNumber(slotOne - 1); + + const proposerOneNodeIndex = validators.findIndex(v => v.attester.equals(proposerOne)); + if (proposerOneNodeIndex < 0) { + throw new Error(`No node holds the key for proposer ${proposerOne}`); + } + + logger.warn(`Selected target slotOne=${slotOne}`, { + slotOne, + slotZero, + slotTwo, + slotThree, + proposerOne: proposerOne.toString(), + proposerOneNodeIndex, + proposerTwo: proposerTwo.toString(), + proposerThree: proposerThree.toString(), + }); + + // Prevent the proposer for slotOne from publishing the checkpoint to L1 (build & gossip still happen). + await nodes[proposerOneNodeIndex].setConfig({ skipPublishingCheckpointsPercent: 100 }); + + // Subscribe to the prune event on every node before sequencers start, so we never miss it. + // We capture the L2 tips synchronously inside the handler — the archiver has already removed + // the pruned blocks at emit time, so this snapshot reflects the rolled-back state before any + // new pipelined block can be applied. + type PruneObservation = { slotNumber: SlotNumber; blocks: L2Block[]; tipsAtPrune: L2Tips }; + const prunePromises: Promise[] = nodes.map( + (node, idx) => + new Promise(resolve => { + const archiver = node.getBlockSource() as Archiver; + // eslint-disable-next-line @typescript-eslint/no-misused-promises + archiver.events.once(L2BlockSourceEvents.L2PruneUncheckpointed, async ev => { + const tipsAtPrune = await node.getChainTips(); + logger.warn(`Node ${idx} pruned uncheckpointed blocks`, { + slotNumber: ev.slotNumber, + blocks: ev.blocks.map(b => ({ number: b.number, slot: b.header.globalVariables.slotNumber })), + tipsAtPrune, + }); + resolve({ slotNumber: ev.slotNumber, blocks: ev.blocks, tipsAtPrune }); + }); + }), + ); + + // Warp L1 to one L1 block before slotZero begins. Pipelining will then engage during slotZero. + await test.warpToBuildWindowForSlot(slotZero); + + // Check that the chain is empty + const node = nodes[0]; + const blockNumber = await node.getBlockNumber(); + expect(blockNumber).toEqual(0); + + // Start all sequencers. + const { failEvents } = test.watchNodeSequencerEvents(nodes, i => ({ validator: `V${i + 1}` })); + + // The proposerTwo pipelined-discard event (the most direct signal that pipelined slotTwo work was + // thrown away because parent slotOne did not land) is captured by watchNodeSequencerEvents above and + // tolerated in the final fail-event filter. + const proposerTwoNodeIndex = validators.findIndex(v => v.attester.equals(proposerTwo)); + + await test.startSequencers(nodes); + logger.warn('All sequencers started'); + + const slotAdvanceTimeout = test.L2_SLOT_DURATION_IN_S * 3; + + // (1) During slotZero: the pipelined proposer for slotOne broadcasts. Every node sees a proposed block at slotOne. + logger.warn(`Waiting for proposed chain to reach slot ${slotOne} on all nodes (build during slotZero)`); + await test.waitForAllNodesToReachBlockAtSlot(slotOne, 'proposed', undefined, { timeout: slotAdvanceTimeout }); + + // (2) During slotOne: the pipelined proposer for slotTwo broadcasts on top of slotOne → proposed reaches slotTwo. + logger.warn(`Waiting for proposed chain to reach slot ${slotTwo} on all nodes (build during slotOne)`); + await test.waitForAllNodesToReachBlockAtSlot(slotTwo, 'proposed', undefined, { timeout: slotAdvanceTimeout }); + + // (3) Wait until slotOne has fully ended on L1 — the archiver only prunes once slotAtNextL1Block > slotOne. + // The end-of-slotOne timestamp equals the start-of-slotTwo timestamp. + const slotOneEndTimestamp = getTimestampForSlot(slotTwo, test.constants); + logger.warn(`Waiting until L1 timestamp ${slotOneEndTimestamp} (end of slot ${slotOne})`); + await waitUntilL1Timestamp(test.l1Client, slotOneEndTimestamp, undefined, test.L2_SLOT_DURATION_IN_S * 3); + + // (4) After slotOne ends without a checkpoint, all nodes should prune. + // Verify rollback via the prune event itself: the pruned slot must equal slotOne, and the + // pruned blocks must include the broadcast blocks for slotOne (proposerOne) and slotTwo + // (pipelined proposerTwo, whose work is now invalid because parent slotOne did not land). + logger.warn('Waiting for L2PruneUncheckpointed on every node'); + const pruneTimeoutMs = test.L2_SLOT_DURATION_IN_S * 2 * 1000; + const pruneObservations = await Promise.all( + prunePromises.map((p, idx) => + Promise.race([p, timeoutPromise(pruneTimeoutMs, `Node ${idx} did not emit prune event in time`)]), + ), + ); + + logger.warn('Asserting prune event details on every node'); + for (const [idx, obs] of pruneObservations.entries()) { + expect({ idx, slotNumber: obs.slotNumber }).toEqual({ idx, slotNumber: slotOne }); + // proposerOne broadcasts during slotZero, so its block must always be in the pruned set. + // The pipelined slotTwo broadcast may or may not have arrived in time on every node, so + // we don't strictly require it here. + const prunedSlots = obs.blocks.map(b => b.header.globalVariables.slotNumber); + expect(prunedSlots).toContain(slotOne); + } + + // (5) Allow the formerly suppressed node to publish again so the chain can recover. + logger.warn(`Re-enabling checkpoint publishing on node ${proposerOneNodeIndex}`); + await nodes[proposerOneNodeIndex].setConfig({ skipPublishingCheckpointsPercent: 0 }); + + // (6) During slotTwo: the pipelined proposer for slotThree builds and broadcasts → proposed advances again. + // The chain must have rewound past slotOne and slotTwo and now build on whatever was + // checkpointed before slotZero — genesis, in this test, since no checkpoints have landed yet. + const postPruneProposedNumbers = pruneObservations.map(o => o.tipsAtPrune.proposed.number); + expect(postPruneProposedNumbers[0]).toBe(0); + + logger.warn(`Waiting for proposed chain to advance to slot ${slotThree} on all nodes (build during slotTwo)`); + await test.waitForAllNodesToReachBlockAtSlot( + slotThree, + 'proposed', + block => block.header.globalVariables.slotNumber >= slotThree, + { timeout: slotAdvanceTimeout }, + ); + + // The first block in the chain after the prune must be the slotThree block — there should be + // nothing between genesis and the new pipelined work, since slotOne and slotTwo were pruned. + for (const node of nodes) { + const blocks = await node.getBlocks(BlockNumber(1), 50); + const firstSlotThreeIdx = blocks.findIndex(b => b.header.globalVariables.slotNumber === slotThree); + expect(firstSlotThreeIdx).toEqual(0); + } + + // (7) During slotThree: proposerThree publishes → checkpointed advances on every node. + logger.warn(`Waiting for checkpointed chain to reach slot >= ${slotThree} on all nodes`); + await test.waitForAllNodesToReachBlockAtSlot( + slotThree, + 'checkpointed', + block => block.header.globalVariables.slotNumber >= slotThree, + { timeout: slotAdvanceTimeout }, + ); + + // Sanity: the only fail events we tolerate are the deliberate skip-publish on the suppressed + // node for slotOne, the pipelined-discard knock-on from proposerTwo (its parent slotOne + // never landed), and proposer-rollup-check noise that any non-proposer emits when the rollup + // contract rejects them. + const unexpectedFailEvents = failEvents.filter(e => { + if ( + e.type === 'checkpoint-publish-failed' && + e.sequencerIndex === proposerOneNodeIndex + 2 && + e.slot === slotOne + ) { + return false; + } + if ( + e.type === 'checkpoint-publish-failed' && + e.sequencerIndex === proposerTwoNodeIndex + 2 && + e.slot === slotTwo + ) { + return false; + } + // Expected + if (e.type === 'pipelined-checkpoint-discarded') { + return false; + } + return true; + }); + if (unexpectedFailEvents.length > 0) { + logger.error('Unexpected fail events from sequencers', unexpectedFailEvents); + } + expect(unexpectedFailEvents).toEqual([]); + }); + + /** + * Orphan-proposed-block prune: with pipelining, the proposer for slot N+1 builds and gossips its + * checkpoint during slot N. The last block in that checkpoint is broadcast standalone (so peers can + * pre-sync the archive) and the enclosing CheckpointProposal is broadcast separately. If the + * CheckpointProposal never arrives, peers are left with a proposed-but-uncheckpointed tip — an + * "orphan" block — and the next proposer must NOT attempt to build on it. + * + * We find two consecutive slots S1, S2 with distinct proposers P1, P2. P1 is configured via the + * test-only `skipBroadcastCheckpointProposal` flag to suppress its CheckpointProposal broadcast while + * still letting the held last block reach peers. P2 must (a) prune the orphan on every archiver, and + * (b) build a fresh checkpoint for S2 that lands on L1. L1 is time-warped to align with the S1 build slot. + */ + // Finds two consecutive slots S1/S2 with distinct proposers. Suppresses P1's CheckpointProposal + // broadcast, waits for the orphan block to appear on all archivers, asserts L2PruneUncheckpointed + // fires on every node for slot S1, then verifies the rebuilt S2 checkpoint lands on L1 with a + // different archive root from the orphan. + it('all nodes prune the orphan block and S2 rebuilds the checkpoint chain', async () => { + // Find S1 (>=4 ahead) such that proposers for S1 and S2=S1+1 are two distinct validators. The +4 margin gives the + // warp+sequencer-start path enough headroom to reach the build window for S1-1 (the pipelining build slot for S1) + // even if node creation jitters. The context helper handles the per-epoch warp + EpochNotStable retry. + const { + slots: [S1, S2], + proposers: [proposerOne, proposerTwo], + } = await test.findSlotsWithProposers( + 2, + ([p1, p2]) => + !p1.equals(p2) && validators.some(v => v.attester.equals(p1)) && validators.some(v => v.attester.equals(p2)), + ); + + const p1Index = validators.findIndex(v => v.attester.equals(proposerOne)); + const p2Index = validators.findIndex(v => v.attester.equals(proposerTwo)); + + logger.warn(`Selected target S1=${S1}`, { + S1, + S2, + proposerOne: proposerOne.toString(), + p1Index, + proposerTwo: proposerTwo.toString(), + p2Index, + }); + + // Suppress only the CheckpointProposal broadcast for the proposer of S1. The held last block is still broadcast + // standalone, so peers' archivers ingest the slot-S1 block as a proposed tip but never see a checkpoint proposal + // for it — the exact orphan-block state we want. + await nodes[p1Index].setConfig({ skipBroadcastCheckpointProposal: true }); + + // No tx is needed: nodes are configured with buildCheckpointIfEmpty so the proposer will produce an empty + // checkpoint on its slot. The test verifies the orphan prune + rebuild invariants, not tx flow. + + // Subscribe to the prune event on every node before sequencers start, so we never miss it. We capture the chain + // tips asynchronously inside the handler for log context, but do not assert on them — by the time the snapshot is + // read, P2's rebuild may already have landed. + type PruneObservation = { slotNumber: SlotNumber; blocks: L2Block[]; tipsAtPrune: L2Tips }; + const prunePromises: Promise[] = nodes.map( + (node, idx) => + new Promise(resolve => { + const archiver = node.getBlockSource() as Archiver; + // eslint-disable-next-line @typescript-eslint/no-misused-promises + archiver.events.once(L2BlockSourceEvents.L2PruneUncheckpointed, async ev => { + const tipsAtPrune = await node.getChainTips(); + logger.warn(`Node ${idx} pruned uncheckpointed blocks`, { + slotNumber: ev.slotNumber, + blocks: ev.blocks.map(b => ({ number: b.number, slot: b.header.globalVariables.slotNumber })), + tipsAtPrune, + }); + resolve({ slotNumber: ev.slotNumber, blocks: ev.blocks, tipsAtPrune }); + }); + }), + ); + + // Warp L1 to one L1 block before the build slot for S1 (which is S1-1 under pipelining offset 1). Pipelining will + // then engage during S1-1 and the proposer for S1 builds + would broadcast its CheckpointProposal — except we + // just suppressed it. + const buildSlot = SlotNumber(S1 - 1); + await test.warpToBuildWindowForSlot(buildSlot); + + expect(await nodes[0].getBlockNumber()).toEqual(0); + + const { failEvents } = test.watchNodeSequencerEvents(nodes, i => ({ validator: `V${i + 1}` })); + + await test.startSequencers(nodes); + logger.warn('All sequencers started'); + + const slotAdvanceTimeout = test.L2_SLOT_DURATION_IN_S * 3; + + // (1) Orphan appears on every archiver. During build slot S1-1, P1 builds and broadcasts the held last block + // standalone (because of skipBroadcastCheckpointProposal). Every node's proposed tip advances to a block whose + // slotNumber === S1. + logger.warn(`Waiting for proposed chain to reach slot ${S1} on all nodes (orphan tip from P1)`); + await test.waitForAllNodesToReachBlockAtSlot( + S1, + 'proposed', + block => block.header.globalVariables.slotNumber === S1, + { timeout: slotAdvanceTimeout, interval: 0.5 }, + ); + + // Capture each node's pre-prune block-1 archive root for the staleness check in (3). + const preBlocks = await Promise.all(nodes.map(node => node.getBlock(BlockNumber(1)))); + const preArchiveRoots = preBlocks.map(block => { + if (!block) { + throw new Error('Expected pre-prune block 1 to exist on every node'); + } + return block.archive.root.toString(); + }); + logger.warn('Captured pre-prune block-1 archive roots', { preArchiveRoots }); + + // (2) Orphan is pruned on every archiver. Since no CheckpointProposal was received for S1, the wall-clock prune + // fires after the checkpoint proposal receive deadline plus local jitter, well inside slot S1 (= the build slot + // for S2). We wait up to 2 slot durations as a margin. + logger.warn('Waiting for L2PruneUncheckpointed on every node'); + const pruneTimeoutMs = test.L2_SLOT_DURATION_IN_S * 2 * 1000; + const pruneObservations = await Promise.all( + prunePromises.map((p, idx) => + Promise.race([p, timeoutPromise(pruneTimeoutMs, `Node ${idx} did not emit prune event in time`)]), + ), + ); + + for (const [idx, obs] of pruneObservations.entries()) { + expect({ idx, slotNumber: obs.slotNumber }).toEqual({ idx, slotNumber: S1 }); + const prunedSlots = obs.blocks.map(b => b.header.globalVariables.slotNumber); + // Only the orphan at slot S1 should have been pruned — nothing earlier or later. + expect(prunedSlots.every(s => s === S1)).toBe(true); + // We do not assert exact equality on tipsAtPrune here. The handler is async and awaits getChainTips(), so P2's + // rebuild could already have landed by the time the snapshot is read. The prune event itself (slotNumber === S1, + // blocks include S1) is sufficient proof. + } + + // (3) S2 builds and the checkpoint lands on L1. After the prune, P2's pipelined build during S1 publishes during + // S2, so L2 block 1 on every node must be the rebuilt block with slot S2. We target block 1 directly rather than + // the live checkpointed tip to avoid an S3-first race where the chain has already advanced past S2 by the time + // we poll. + logger.warn(`Waiting for L2 block 1 to be the rebuilt slot-${S2} block on all nodes`); + await Promise.all( + nodes.map((node, idx) => + retryUntil( + async () => { + const block = await node.getBlock(BlockNumber(1)); + return !!block && block.header.globalVariables.slotNumber === S2; + }, + `node ${idx} block 1 rebuilt at slot ${S2}`, + slotAdvanceTimeout, + 0.5, + ), + ), + ); + + // Independently confirm the checkpoint actually landed on L1 by waiting (bounded) on the chain monitor and + // verifying the block at L2 block number 1 — that is the rebuilt block, and its slot must equal S2. Targeting + // block 1 rather than the live tip avoids a race where the chain has already advanced past S2 by the time we read. + await test.waitUntilCheckpointNumber(CheckpointNumber(1), test.L2_SLOT_DURATION_IN_S * 4); + const rebuiltBlock = await nodes[0].getBlock(BlockNumber(1)); + expect(rebuiltBlock).toBeDefined(); + expect(rebuiltBlock!.header.globalVariables.slotNumber).toEqual(S2); + + // The rebuilt block at number 1 must have a different archive root from the orphan we saw before the prune. This + // guards against accidental pass on stale state. + const postBlocks = await Promise.all(nodes.map(node => node.getBlock(BlockNumber(1)))); + const postArchiveRoots = postBlocks.map(block => { + if (!block) { + throw new Error('Expected post-prune block 1 to exist on every node'); + } + return block.archive.root.toString(); + }); + logger.warn('Captured post-prune block-1 archive roots', { postArchiveRoots }); + for (const [idx, root] of postArchiveRoots.entries()) { + expect({ idx, root }).not.toEqual({ idx, root: preArchiveRoots[idx] }); + } + + // Tolerated fail events, scoped narrowly: P1 at S1 expectedly fails to publish because peers never see the + // CheckpointProposal, so it cannot collect attestations. P2 must not discard or miss its own S2 checkpoint. + const unexpectedFailEvents = failEvents.filter(e => { + if (e.type === 'checkpoint-publish-failed' && e.sequencerIndex === p1Index + 2 && e.slot === S1) { + return false; + } + return true; + }); + if (unexpectedFailEvents.length > 0) { + logger.error('Unexpected fail events from sequencers', unexpectedFailEvents); + } + expect(unexpectedFailEvents).toEqual([]); + }); +}); diff --git a/yarn-project/end-to-end/src/multi-node/slashing/duplicate_attestation.test.ts b/yarn-project/end-to-end/src/multi-node/slashing/duplicate_attestation.test.ts new file mode 100644 index 000000000000..a1d7549db344 --- /dev/null +++ b/yarn-project/end-to-end/src/multi-node/slashing/duplicate_attestation.test.ts @@ -0,0 +1,195 @@ +import type { AztecNodeService } from '@aztec/aztec-node'; +import type { TestAztecNodeService } from '@aztec/aztec-node/test'; +import { EthAddress } from '@aztec/aztec.js/addresses'; +import { EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; +import { OffenseType } from '@aztec/slasher'; + +import { advanceToEpochBeforeProposer, awaitCommitteeExists, awaitOffenseDetected } from '../../e2e_p2p/shared.js'; +import { + MultiNodeTestContext, + SLASHER_ENABLED_MULTI_VALIDATOR_OPTS, + buildMockGossipValidators, +} from '../multi_node_test_context.js'; +import { + AZTEC_SLOT_DURATION, + NUM_VALIDATORS, + aztecEpochDuration, + baseSlashingOpts, + slashingRoundSize, + slashingUnit, +} from './setup.js'; + +/** + * Test that slashing occurs when a validator sends duplicate attestations (equivocation). + * + * The setup of the test is as follows: + * 1. Create 4 validator nodes total: + * - 2 honest validators with unique keys + * - 2 "malicious proposer" validators that share the SAME validator key but have DIFFERENT coinbase addresses + * (these will create duplicate proposals for the same slot) + * - The malicious proposer validators also have `attestToEquivocatedProposals: true` which makes them attest + * to BOTH proposals when they receive them - this is the attestation equivocation we want to detect + * 2. The two nodes with the same proposer key will both detect they are proposers for the same slot and race to propose + * 3. Since they have different coinbase addresses, their proposals will have different archives (different content) + * 4. The malicious attester nodes (with attestToEquivocatedProposals enabled) will attest to BOTH proposals + * 5. Honest validators will detect the duplicate attestations and emit a slash event + * + * NOTE: This test triggers BOTH duplicate proposal (from malicious proposers sharing a key) AND duplicate attestation + * (from the malicious proposers attesting to multiple proposals). We verify specifically that the duplicate + * attestation offense is recorded. + * + * Setup: MultiNodeTestContext on the in-memory mock-gossip bus (no real libp2p). 4 validators, ethSlot=8s, + * aztecSlot=24s, epoch=2, proofSubEpochs=1024, minTxsPerBlock=0, inboxLag=2 (v5 always enforces the timetable). + */ +describe('multi-node/slashing/duplicate_attestation', () => { + let test: MultiNodeTestContext; + let nodes: AztecNodeService[]; + + beforeEach(async () => { + test = await MultiNodeTestContext.setup({ + ...SLASHER_ENABLED_MULTI_VALIDATOR_OPTS, + ...baseSlashingOpts, + slashDuplicateAttestationPenalty: slashingUnit, + initialValidators: buildMockGossipValidators(NUM_VALIDATORS), + }); + }); + + afterEach(async () => { + await test.teardown(); + }); + + const cheatCodes = () => test.context.cheatCodes; + + const debugRollup = async () => { + await cheatCodes().rollup.debugRollup(); + }; + + // Two malicious nodes share a validator key and both attest to each other's proposals + // (attestToEquivocatedProposals:true). Honest nodes detect the DUPLICATE_ATTESTATION offense and verify + // the offending attester is the shared key's address. Also exercises DUPLICATE_PROPOSAL as a side effect + // but asserts specifically that DUPLICATE_ATTESTATION is recorded. + it('slashes validator who sends duplicate attestations', async () => { + const { rollup } = await test.getSlashingContracts(); + + // Jump forward to an epoch in the future such that the validator set is not empty + await cheatCodes().rollup.advanceToEpoch(EpochNumber(4)); + await debugRollup(); + + test.logger.warn('Creating nodes'); + + // Use validator index 0 for the "malicious" proposer validator key + const maliciousProposerIndex = 0; + const maliciousProposerAddress = test.addressAt(maliciousProposerIndex); + + test.logger.warn(`Malicious proposer address: ${maliciousProposerAddress.toString()}`); + + // Create two nodes with the SAME validator key but DIFFERENT coinbase addresses + // This will cause them to create proposals with different content for the same slot + // Additionally, enable attestToEquivocatedProposals so they will attest to BOTH proposals + const coinbase1 = EthAddress.random(); + const coinbase2 = EthAddress.random(); + + test.logger.warn(`Creating malicious proposer node 1 with coinbase ${coinbase1.toString()}`); + const maliciousNode1 = await test.createValidatorNodeAt(maliciousProposerIndex, { + coinbase: coinbase1, + attestToEquivocatedProposals: true, // Attest to all proposals - creates duplicate attestations + broadcastEquivocatedProposals: true, // Don't abort checkpoint building on duplicate block proposals + dontStartSequencer: true, + // Prevent HA peer proposals from being added to the archiver, so both + // malicious nodes build their own blocks instead of one yielding to the other. + skipPushProposedBlocksToArchiver: true, + }); + + test.logger.warn(`Creating malicious proposer node 2 with coinbase ${coinbase2.toString()}`); + const maliciousNode2 = await test.createValidatorNodeAt(maliciousProposerIndex, { + coinbase: coinbase2, + attestToEquivocatedProposals: true, // Attest to all proposals - creates duplicate attestations + broadcastEquivocatedProposals: true, // Don't abort checkpoint building on duplicate block proposals + dontStartSequencer: true, + // Prevent HA peer proposals from being added to the archiver, so both + // malicious nodes build their own blocks instead of one yielding to the other. + skipPushProposedBlocksToArchiver: true, + }); + + // Create honest nodes with unique validator keys (indices 1 and 2) + test.logger.warn('Creating honest nodes'); + const honestNode1 = await test.createValidatorNodeAt(1, { dontStartSequencer: true }); + const honestNode2 = await test.createValidatorNodeAt(2, { dontStartSequencer: true }); + + nodes = [maliciousNode1, maliciousNode2, honestNode1, honestNode2]; + + await awaitCommitteeExists({ rollup, logger: test.logger }); + + // Find an epoch where the malicious proposer is selected, stopping one epoch before + // so we have time to start sequencers before the target epoch arrives + const epochCache = (honestNode1 as TestAztecNodeService).epochCache; + const { targetEpoch, targetSlot } = await advanceToEpochBeforeProposer({ + epochCache, + cheatCodes: cheatCodes().rollup, + targetProposer: maliciousProposerAddress, + logger: test.logger, + }); + + // Start all sequencers while still one epoch before the target + test.logger.warn('Starting all sequencers'); + await Promise.all(nodes.map(n => n.getSequencer()!.start())); + + // Now warp to one slot before the target epoch — sequencers are already running. The helper + // picks a target slot at least one slot into the epoch, so warping here (rather than to the + // epoch start) leaves the freshly-started sequencers a full warm-up slot before the pipelined + // build for the malicious slot begins. Without that margin the duplicate proposals serialize + // past the slot boundary and receivers reject them as late, so the malicious nodes never get to + // attest to both and no duplicate attestation is produced. + test.logger.warn(`Advancing to one slot before target epoch ${targetEpoch} (target slot ${targetSlot})`); + await cheatCodes().rollup.advanceToEpoch(targetEpoch, { offset: -AZTEC_SLOT_DURATION }); + + // Wait for offenses to be detected + // We expect BOTH duplicate proposal AND duplicate attestation offenses + // The malicious proposer nodes create duplicate proposals (same key, different coinbase) + // The malicious proposer nodes also create duplicate attestations (attestToEquivocatedProposals enabled) + test.logger.warn('Waiting for duplicate attestation offense to be detected...'); + const offenses = await awaitOffenseDetected({ + epochDuration: aztecEpochDuration, + logger: test.logger, + nodeAdmin: honestNode1, // Use honest node to check for offenses + slashingRoundSize, + waitUntilOffenseCount: 2, // Wait for both duplicate proposal and duplicate attestation + timeoutSeconds: AZTEC_SLOT_DURATION * 16, + }); + + test.logger.warn(`Collected offenses`, { offenses }); + + // Verify we have detected the duplicate attestation offense + const duplicateAttestationOffenses = offenses.filter( + offense => offense.offenseType === OffenseType.DUPLICATE_ATTESTATION, + ); + const duplicateProposalOffenses = offenses.filter( + offense => offense.offenseType === OffenseType.DUPLICATE_PROPOSAL, + ); + + test.logger.info(`Found ${duplicateAttestationOffenses.length} duplicate attestation offenses`); + test.logger.info(`Found ${duplicateProposalOffenses.length} duplicate proposal offenses`); + + // We should have at least one duplicate attestation offense + expect(duplicateAttestationOffenses.length).toBeGreaterThan(0); + + // Verify the duplicate attestation offense is from the malicious proposer address + // (since they are the ones with attestToEquivocatedProposals enabled) + for (const offense of duplicateAttestationOffenses) { + expect(offense.offenseType).toEqual(OffenseType.DUPLICATE_ATTESTATION); + expect(offense.validator.toString()).toEqual(maliciousProposerAddress.toString()); + } + + // Verify that for each duplicate attestation offense, the attester for that slot is the malicious validator + for (const offense of duplicateAttestationOffenses) { + const offenseSlot = SlotNumber(Number(offense.epochOrSlot)); + const committeeInfo = await epochCache.getCommittee(offenseSlot); + test.logger.info( + `Offense slot ${offenseSlot}: committee includes attester ${maliciousProposerAddress.toString()}`, + ); + expect(committeeInfo.committee?.map(addr => addr.toString())).toContain(maliciousProposerAddress.toString()); + } + + test.logger.warn('Duplicate attestation offense correctly detected and recorded'); + }); +}); diff --git a/yarn-project/end-to-end/src/multi-node/slashing/duplicate_proposal.test.ts b/yarn-project/end-to-end/src/multi-node/slashing/duplicate_proposal.test.ts new file mode 100644 index 000000000000..cda5fd8cf553 --- /dev/null +++ b/yarn-project/end-to-end/src/multi-node/slashing/duplicate_proposal.test.ts @@ -0,0 +1,176 @@ +import type { AztecNodeService } from '@aztec/aztec-node'; +import type { TestAztecNodeService } from '@aztec/aztec-node/test'; +import { EthAddress } from '@aztec/aztec.js/addresses'; +import { EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; +import { OffenseType } from '@aztec/slasher'; + +import { advanceToEpochBeforeProposer, awaitCommitteeExists, awaitOffenseDetected } from '../../e2e_p2p/shared.js'; +import { + MultiNodeTestContext, + SLASHER_ENABLED_MULTI_VALIDATOR_OPTS, + buildMockGossipValidators, +} from '../multi_node_test_context.js'; +import { + AZTEC_SLOT_DURATION, + NUM_VALIDATORS, + aztecEpochDuration, + baseSlashingOpts, + slashingRoundSize, +} from './setup.js'; + +/** + * Test that slashing occurs when a validator sends duplicate proposals (equivocation). + * + * The setup of the test is as follows: + * 1. Create 4 validator nodes total: + * - 2 honest validators with unique keys + * - 2 "malicious" validators that share the SAME validator key but have DIFFERENT coinbase addresses + * 2. The two nodes with the same key will both detect they are proposers for the same slot and naturally race to propose + * 3. Since they have different coinbase addresses, their proposals will have different archives (different content) + * 4. Other validators will detect the duplicate and emit a slash event + * + * Setup: MultiNodeTestContext on the in-memory mock-gossip bus (no real libp2p). 4 validators, ethSlot=8s, + * aztecSlot=24s, epoch=2, proofSubEpochs=1024, minTxsPerBlock=0, inboxLag=2 (v5 always enforces the timetable). + */ +describe('multi-node/slashing/duplicate_proposal', () => { + let test: MultiNodeTestContext; + let nodes: AztecNodeService[]; + + beforeEach(async () => { + test = await MultiNodeTestContext.setup({ + ...SLASHER_ENABLED_MULTI_VALIDATOR_OPTS, + ...baseSlashingOpts, + initialValidators: buildMockGossipValidators(NUM_VALIDATORS), + }); + }); + + afterEach(async () => { + await test.teardown(); + }); + + const cheatCodes = () => test.context.cheatCodes; + + const debugRollup = async () => { + await cheatCodes().rollup.debugRollup(); + }; + + // Two malicious nodes share a validator key but have different coinbase addresses so their proposals + // differ. Honest nodes receive both proposals via mock gossip, detect the equivocation, and record a + // DUPLICATE_PROPOSAL offense. The test collects offenses from all nodes (equivocation may only be + // observed by whichever node processed both proposals before the slot closed) and asserts the offense + // is attributed to the shared key's address. + it('slashes validator who sends duplicate proposals', async () => { + const { rollup } = await test.getSlashingContracts(); + + // Jump forward to an epoch in the future such that the validator set is not empty + await cheatCodes().rollup.advanceToEpoch(EpochNumber(4)); + await debugRollup(); + + test.logger.warn('Creating nodes'); + + // Use validator index 0 for the "malicious" validator key + const maliciousValidatorIndex = 0; + const maliciousValidatorAddress = test.addressAt(maliciousValidatorIndex); + + test.logger.warn(`Malicious proposer address: ${maliciousValidatorAddress.toString()}`); + + // Create two nodes with the SAME validator key but DIFFERENT coinbase addresses + // This will cause them to create proposals with different content for the same slot + const coinbase1 = EthAddress.random(); + const coinbase2 = EthAddress.random(); + + test.logger.warn(`Creating malicious node 1 with coinbase ${coinbase1.toString()}`); + const maliciousNode1 = await test.createValidatorNodeAt(maliciousValidatorIndex, { + coinbase: coinbase1, + broadcastEquivocatedProposals: true, + dontStartSequencer: true, + // Prevent HA peer proposals from being added to the archiver, so both + // malicious nodes build their own blocks instead of one yielding to the other. + skipPushProposedBlocksToArchiver: true, + }); + + test.logger.warn(`Creating malicious node 2 with coinbase ${coinbase2.toString()}`); + const maliciousNode2 = await test.createValidatorNodeAt(maliciousValidatorIndex, { + coinbase: coinbase2, + broadcastEquivocatedProposals: true, + dontStartSequencer: true, + // Prevent HA peer proposals from being added to the archiver, so both + // malicious nodes build their own blocks instead of one yielding to the other. + skipPushProposedBlocksToArchiver: true, + }); + + // Create honest nodes with unique validator keys (indices 1 and 2) + test.logger.warn('Creating honest nodes'); + const honestNode1 = await test.createValidatorNodeAt(1, { dontStartSequencer: true }); + const honestNode2 = await test.createValidatorNodeAt(2, { dontStartSequencer: true }); + + nodes = [maliciousNode1, maliciousNode2, honestNode1, honestNode2]; + + await awaitCommitteeExists({ rollup, logger: test.logger }); + + // Find an epoch where the malicious proposer is selected, stopping one epoch before + // so we have time to start sequencers before the target epoch arrives + const epochCache = (honestNode1 as TestAztecNodeService).epochCache; + const { targetEpoch, targetSlot } = await advanceToEpochBeforeProposer({ + epochCache, + cheatCodes: cheatCodes().rollup, + targetProposer: maliciousValidatorAddress, + logger: test.logger, + }); + + // Start all sequencers while still one epoch before the target + test.logger.warn('Starting all sequencers'); + await Promise.all(nodes.map(n => n.getSequencer()!.start())); + + // Now warp to one slot before the target epoch — sequencers are already running. + // Under proposer pipelining, the malicious proposers begin building for their slot one slot + // earlier; warping to the start of the epoch would force both AVM-heavy duplicate proposals to + // serialize past the slot boundary, after which honest receivers reject them as late. The helper + // picks a target slot at least one slot into the epoch, so warping here leaves a full warm-up + // slot before the build begins rather than starting it at the exact instant of the warp. + test.logger.warn(`Advancing to one slot before target epoch ${targetEpoch} (target slot ${targetSlot})`); + await cheatCodes().rollup.advanceToEpoch(targetEpoch, { offset: -AZTEC_SLOT_DURATION }); + + // Wait for offense to be detected. Under proposer pipelining, checkpoint proposals are broadcast + // at the slot boundary while the receivers' wall clocks may have already advanced past the build + // slot — when that happens, honest nodes reject the gossip with "invalid slot number" before + // duplicate detection runs, so DUPLICATE_PROPOSAL is only observed by whichever node managed to + // process both proposals while still in the build slot (often the other malicious node, since + // they receive each other's broadcasts immediately). We therefore collect offenses from every + // node in the network and assert that at least one of them recorded the duplicate proposal. + test.logger.warn('Waiting for duplicate proposal offense to be detected...'); + await awaitOffenseDetected({ + epochDuration: aztecEpochDuration, + logger: test.logger, + nodeAdmin: honestNode1, + slashingRoundSize, + waitUntilOffenseCount: 1, + timeoutSeconds: AZTEC_SLOT_DURATION * 16, + }); + + // Poll every node for DUPLICATE_PROPOSAL offenses, retrying briefly so any node that detected + // the duplicate after the initial offense was collected has time to flush it through the + // slasher's offenses-collector. + const proposalOffenses = await test.waitForOffenseOnNodes( + nodes, + o => o.offenseType === OffenseType.DUPLICATE_PROPOSAL, + { mode: 'any', timeout: AZTEC_SLOT_DURATION * 4 }, + ); + + test.logger.warn(`Collected duplicate proposal offenses`, { proposalOffenses }); + expect(proposalOffenses.length).toBeGreaterThan(0); + for (const offense of proposalOffenses) { + expect(offense.validator.toString()).toEqual(maliciousValidatorAddress.toString()); + } + + // Verify that for each offense, the proposer for that slot is the malicious validator + for (const offense of proposalOffenses) { + const offenseSlot = SlotNumber(Number(offense.epochOrSlot)); + const proposerForSlot = await epochCache.getProposerAttesterAddressInSlot(offenseSlot); + test.logger.info(`Offense slot ${offenseSlot}: proposer is ${proposerForSlot?.toString()}`); + expect(proposerForSlot?.toString()).toEqual(maliciousValidatorAddress.toString()); + } + + test.logger.warn('Duplicate proposal offense correctly detected and recorded'); + }); +}); diff --git a/yarn-project/end-to-end/src/multi-node/slashing/setup.ts b/yarn-project/end-to-end/src/multi-node/slashing/setup.ts new file mode 100644 index 000000000000..27171dd3cb2e --- /dev/null +++ b/yarn-project/end-to-end/src/multi-node/slashing/setup.ts @@ -0,0 +1,44 @@ +import { jest } from '@jest/globals'; + +const TEST_TIMEOUT = 600_000; // 10 minutes + +jest.setTimeout(TEST_TIMEOUT); + +export const NUM_VALIDATORS = 4; +export const COMMITTEE_SIZE = NUM_VALIDATORS; +export const ETHEREUM_SLOT_DURATION = 8; +export const AZTEC_SLOT_DURATION = ETHEREUM_SLOT_DURATION * 3; +export const BLOCK_DURATION = 4; + +// Small slashing unit so we don't kick anyone out. +export const slashingUnit = BigInt(1e14); +export const slashingQuorum = 3; +export const slashingRoundSize = 4; +export const aztecEpochDuration = 2; + +/** + * The shared per-test slashing config for the offense-detection suites (`duplicate_proposal`, + * `duplicate_attestation`). Spread into a {@link MultiNodeTestContext.setup} call alongside + * {@link SLASHER_ENABLED_MULTI_VALIDATOR_OPTS} and `initialValidators` (from `buildMockGossipValidators`). + */ +export const baseSlashingOpts = { + anvilSlotsInAnEpoch: 4, + listenAddress: '127.0.0.1', + aztecEpochDuration, + ethereumSlotDuration: ETHEREUM_SLOT_DURATION, + aztecSlotDuration: AZTEC_SLOT_DURATION, + aztecTargetCommitteeSize: COMMITTEE_SIZE, + aztecProofSubmissionEpochs: 1024, // effectively do not reorg + slashInactivityConsecutiveEpochThreshold: 32, // effectively do not slash for inactivity + minTxsPerBlock: 0, // always be building + slashingQuorum, + slashingRoundSizeInEpochs: slashingRoundSize / aztecEpochDuration, + slashAmountSmall: slashingUnit, + slashAmountMedium: slashingUnit * 2n, + slashAmountLarge: slashingUnit * 3n, + blockDurationMs: BLOCK_DURATION * 1000, + slashDuplicateProposalPenalty: slashingUnit, + slashingOffsetInRounds: 1, +}; + +export { jest }; diff --git a/yarn-project/end-to-end/src/single-node/README.md b/yarn-project/end-to-end/src/single-node/README.md new file mode 100644 index 000000000000..bea3825c1145 --- /dev/null +++ b/yarn-project/end-to-end/src/single-node/README.md @@ -0,0 +1,38 @@ +# `single-node` e2e test category + +Single-node tests run one Aztec node with the production sequencer (and, where a test needs proving, +a fake-proof prover node). They cover behavior that only requires a single sequencer and no +multi-validator committee: the proving/epoch lifecycle, partial proofs, L1-reorg handling, and +pending-chain recovery. + +## Base class + +All tests use `SingleNodeTestContext` (`single_node_test_context.ts`), which owns: + +- The environment: an in-process anvil plus the L1 contract deploy. +- Node spawning: `createNonValidatorNode` and `createProverNode` (the latter wires the mock-gossip + `p2pServiceFactory`, which is harmless with a single node). +- The `ChainMonitor`. +- The epoch / checkpoint / proof-window / reorg waiters and assertion helpers (`waitUntilEpochStarts`, + `waitUntilProvenCheckpointNumber`, `waitForNodeToSync`, `verifyHistoricBlock`, …). + +`MultiNodeTestContext` (in `../multi-node/`) extends this base with the N-validator topology, so the +multi-node category inherits the same environment and waiters. + +## Organizing principle + +The top level groups tests by node topology and setup model; the second level names the primary +behavior under test (the proving lifecycle, partial proofs, reorgs, recovery) rather than the shared +setup. Each file has a single top-level `describe` named to match its path, and a co-located +`setup.ts` holds any shared timeout/option wiring. A `.parallel` suffix marks files with more than one +top-level `it`; CI splits each `it` into its own job. + +## Subfolders + +| Path | Contents | +|---|---| +| `proving/` | Epoch and proof lifecycle. `world_state_pruning` (consecutive epochs prove and finalized blocks are purged from world state beyond the checkpoint-history window), `empty_blocks` (a proof is submitted even with no txs), `long_proving_time` (a prover delay spanning multiple epochs), `multi_proof` (multiple prover nodes prove one epoch), `optimistic.parallel` (checkpoint-driven proving across the happy path and several mid-epoch / last-slot / during-proving reorg cases), `proof_fails.parallel` (proof not accepted after epoch end; proving aborts when the next epoch ends), `cross_chain_public_message` (an epoch with a public tx that consumes an L1→L2 message in the block it lands, guarding against a sequencer/prover state-root mismatch), `upload_failed_proof` (a failed proving job's state is uploaded and re-run on a fresh instance). | +| `partial-proofs/` | Manually driven partial-proof submission. `single_root` (the prover node's `startProof` path on a single root) and `multi_root` (three partial-proof roots are staged and messages consume against any covering root, exercising the multi-root Outbox semantics). | +| `l1-reorgs/` | Behavior under L1 reorgs, split by what reorgs. `blocks.parallel` (prune L2 blocks when a reorg drops a proof, hold when a replacement proof lands in the window, restore blocks when a proof reappears, prune pending-chain blocks, and see new blocks added by a reorg) and `messages.parallel` (L1→L2 messages updated by a reorg, and a missed message inserted by one). `setup.ts` holds the shared `FAST_REORG_TIMING` profile and delayer wiring. | +| `recovery/` | Reorg and pending-chain recovery. `manual_rollback` (the `rollbackTo` admin API rolls back to an unfinalized block), `sync_after_reorg` (a fresh node syncs world state past an unpruned reorg window), `prune_when_cannot_build` (a solo sequencer prunes the pending chain via the fallback path when it cannot propose). | +| `misc/` | Genuine single-node outliers. `missed_l1_slot` (the sequencer builds a block after missed L1 slots once the previous checkpoint is synced). | diff --git a/yarn-project/end-to-end/src/single-node/l1-reorgs/blocks.parallel.test.ts b/yarn-project/end-to-end/src/single-node/l1-reorgs/blocks.parallel.test.ts new file mode 100644 index 000000000000..90bfa98f4461 --- /dev/null +++ b/yarn-project/end-to-end/src/single-node/l1-reorgs/blocks.parallel.test.ts @@ -0,0 +1,411 @@ +import type { Archiver } from '@aztec/archiver'; +import type { Logger } from '@aztec/aztec.js/log'; +import type { AztecNode } from '@aztec/aztec.js/node'; +import { createBlobClient } from '@aztec/blob-client/client'; +import { Blob } from '@aztec/blob-lib'; +import type { Delayer } from '@aztec/ethereum/l1-tx-utils'; +import type { ChainMonitor, ChainMonitorEventMap } from '@aztec/ethereum/test'; +import { CheckpointNumber } from '@aztec/foundation/branded-types'; +import { AbortError } from '@aztec/foundation/error'; +import { retryUntil } from '@aztec/foundation/retry'; +import { hexToBuffer } from '@aztec/foundation/string'; +import { executeTimeout } from '@aztec/foundation/timer'; + +import 'jest-extended'; +import { keccak256, parseTransaction } from 'viem'; + +import type { EndToEndContext } from '../../fixtures/utils.js'; +import { waitForNodeCheckpoint, waitForNodeProvenCheckpoint } from '../../fixtures/wait_helpers.js'; +import type { SingleNodeTestContext } from '../single_node_test_context.js'; +import { L1ReorgsTest, TX_COUNT } from './setup.js'; + +// Single-node + prover-node suite exercising L1 reorg behavior for L2 block state: proof removal, proof +// re-addition via reorg, checkpoint removal from the pending chain, and checkpoint insertion via reorg. +// Uses EthCheatCodes reorg/reorgWithReplacement to remove or insert L1 transactions and verifies the +// archiver and node prune/restore their views accordingly. Prover and sequencer delayers intercept L1 +// txs to enable controlled reorg scenarios. Shared setup lives in setup.ts. +describe('single-node/l1-reorgs/blocks', () => { + let t: L1ReorgsTest; + + let context: EndToEndContext; + let logger: Logger; + let node: AztecNode; + let archiver: Archiver; + let monitor: ChainMonitor; + let proverDelayer: Delayer; + let sequencerDelayer: Delayer; + + let L1_BLOCK_TIME_IN_S: number; + let L2_SLOT_DURATION_IN_S: number; + + let test: SingleNodeTestContext; + + const sendTransactions = (count: number, offset = 0) => t.sendTransactions(count, offset); + + beforeEach(async () => { + t = new L1ReorgsTest(); + await t.setup(); + ({ test, context, logger, node, archiver, monitor, proverDelayer, sequencerDelayer } = t); + ({ L1_BLOCK_TIME_IN_S, L2_SLOT_DURATION_IN_S } = t); + }); + + afterEach(async () => { + await t.teardown(); + }); + + const getBlobs = async (serializedTx: `0x${string}`) => { + const parsedTx = parseTransaction(serializedTx); + if (parsedTx.sidecars === false) { + throw new Error('No sidecars found in tx'); + } + return await Promise.all(parsedTx.sidecars!.map(sidecar => Blob.fromBlobBuffer(hexToBuffer(sidecar.blob)))); + }; + + // Waits for an initial proof to land, stops the prover, reorgs L1 to remove the proof block, + // waits for the proof submission window to expire, spins up a new sync-only node, and verifies + // both the new node and the old node have rolled back to the pre-proof checkpoint number. + it('prunes L2 blocks if a proof is removed due to an L1 reorg', async () => { + /** Logs a full state snapshot: L1 latest/finalized and archiver L2 tips. */ + const logState = async (label: string) => { + const [l1Latest, l1Finalized, archiverTips] = await Promise.all([ + test.l1Client.getBlockNumber(), + test.l1Client.getBlock({ blockTag: 'finalized', includeTransactions: false }).then(b => b.number), + archiver.getL2Tips(), + ]); + logger.warn(`[state:${label}]`, { + l1Latest, + l1Finalized, + l2Proposed: archiverTips.proposed.number, + l2Checkpointed: archiverTips.checkpointed.block.number, + l2Proven: archiverTips.proven.block.number, + provenCheckpoint: archiverTips.proven.checkpoint.number, + l2Finalized: archiverTips.finalized.block.number, + finalizedCheckpoint: archiverTips.finalized.checkpoint.number, + }); + }; + + // Send txs to trigger multi-block checkpoints + await sendTransactions(TX_COUNT); + + // Capture initial chain state + const initialProvenCheckpoint = (await monitor.run(true)).provenCheckpointNumber; + await logState('initial'); + + // Wait until we have proven something and the nodes have caught up + const epochDurationSeconds = test.constants.epochDuration * test.constants.slotDuration; + logger.warn(`Waiting for initial proof to land`); + const provenBlockEvent = await executeTimeout( + signal => { + return new Promise<{ provenCheckpointNumber: number; l1BlockNumber: number }>((res, rej) => { + const handleMsg = (...[ev]: ChainMonitorEventMap['checkpoint-proven']) => { + if (ev.provenCheckpointNumber > initialProvenCheckpoint) { + res(ev); + monitor.off('checkpoint-proven', handleMsg); + } + }; + + signal.onabort = () => { + monitor.off('checkpoint-proven', handleMsg); + rej(new AbortError()); + }; + monitor.on('checkpoint-proven', handleMsg); + }); + }, + epochDurationSeconds * 4 * 1000, + ); + + logger.warn( + `Proof for checkpoint ${provenBlockEvent.provenCheckpointNumber} mined at L1 block ${provenBlockEvent.l1BlockNumber}`, + ); + await logState('proof-landed'); + + // Stop the prover node (by stopping its hosting aztec node) so it doesn't re-submit the proof after we've removed it + logger.warn(`Stopping prover node`); + await test.proverNodes[0].stop(); + await logState('prover-stopped'); + + // And remove the proof from L1 + const reorgTarget = provenBlockEvent.l1BlockNumber - 1; + logger.warn( + `Reorging L1 from current tip to block ${reorgTarget} (removing proof block ${provenBlockEvent.l1BlockNumber})`, + ); + await context.cheatCodes.eth.reorgTo(reorgTarget); + await logState('after-reorg'); + expect((await monitor.run(true)).provenCheckpointNumber).toEqual(initialProvenCheckpoint); + + // Wait until the end of the proof submission window for the epoch of the proven checkpoint + const provenCheckpointEpoch = await test.rollup.getEpochNumberForCheckpoint( + CheckpointNumber(provenBlockEvent.provenCheckpointNumber), + ); + await test.waitUntilLastSlotOfProofSubmissionWindow(provenCheckpointEpoch); + await logState('after-submission-window'); + + // Ensure that a new node sees the reorg + logger.warn(`Syncing new node to test reorg`); + const newNode = await executeTimeout(() => test.createNonValidatorNode(), 10_000, `new node sync`); + expect(await newNode.getCheckpointNumber('proven')).toEqual(initialProvenCheckpoint); + + // Latest checkpointed block seen by the node may be from the current checkpoint, or one less if it was *just* mined. + // This is because the call to createNonValidatorNode will block until the initial sync is completed, + // but the initial sync is done to the latest L1 block _at the time the initial sync starts_. So a new + // checkpoint may have appeared while the initial sync runs, that's why we account for a small span. + const currentCheckpointNumber = (await monitor.run(true)).checkpointNumber; + expect(await newNode.getCheckpointNumber('checkpointed')).toBeWithin( + currentCheckpointNumber - 1, + currentCheckpointNumber + 1, + ); + + // And check that the old node has processed the reorg as well + logger.warn(`Testing old node after reorg`); + await waitForNodeProvenCheckpoint(node, initialProvenCheckpoint, { + compare: (actual, target) => actual === target, + timeout: L2_SLOT_DURATION_IN_S * 4, + }); + await logState('old-node-synced'); + expect(await node.getCheckpointNumber('checkpointed')).toBeWithin( + monitor.checkpointNumber - 1, + monitor.checkpointNumber + 1, + ); + + // Verify multi-block checkpoints were built + await test.assertMultipleBlocksPerSlot(2); + + logger.warn(`Test succeeded`); + await newNode.stop(); + }); + + // Waits for a proof, stops the prover, removes the proof via reorgWithReplacement (same block + // count), starts a fresh prover node, and verifies a new proof lands and the node re-syncs to + // the proven state without having pruned. + it('does not prune if a second proof lands within the submission window after the first one is reorged out', async () => { + // Send txs to trigger multi-block checkpoints + await sendTransactions(TX_COUNT); + + // Capture initial chain state + const initialProvenCheckpoint = (await monitor.run(true)).provenCheckpointNumber; + const targetProvenCheckpoint = CheckpointNumber(initialProvenCheckpoint + 1); + + // Wait until we have proven something and the nodes have caught up + // Use a longer timeout since we need to wait for the epoch to complete (~288s) plus proving time. + const epochDurationSeconds = test.constants.epochDuration * test.constants.slotDuration; + logger.warn(`Waiting for initial proof to land`); + const provenCheckpoint = await test.waitUntilProvenCheckpointNumber( + targetProvenCheckpoint, + epochDurationSeconds * 4, + ); + await waitForNodeProvenCheckpoint(node, provenCheckpoint, { timeout: 10 }); + + // Stop the prover node (by stopping its hosting aztec node) + await test.proverNodes[0].stop(); + + // Remove the proof from L1 but do not change the block number + await context.cheatCodes.eth.reorgWithReplacement(1); + await expect(monitor.run(true).then(m => m.provenCheckpointNumber)).resolves.toEqual(initialProvenCheckpoint); + + // Create another prover node so it submits a proof and wait until it is submitted + await test.createProverNode(); + const provenCheckpointRetry = await test.waitUntilProvenCheckpointNumber(CheckpointNumber(1)); + await expect(monitor.run(true).then(m => m.provenCheckpointNumber)).resolves.toBeGreaterThanOrEqual(1); + + // Check that the node has followed along + logger.warn(`Testing old node`); + await waitForNodeProvenCheckpoint(node, provenCheckpointRetry, { timeout: 10 }); + expect(await node.getCheckpointNumber('checkpointed')).toBeWithin( + monitor.checkpointNumber - 1, + monitor.checkpointNumber + 1, + ); + + // Verify multi-block checkpoints were built + await test.assertMultipleBlocksPerSlot(2); + + logger.warn(`Test succeeded`); + // New prover's aztec node is stopped in test.teardown() + }); + + // Cancels the next prover L1 tx so no proof lands, waits for the end of the submission window + // (triggering pruning), then reorgs L1 to include the previously-cancelled proof tx and + // verifies the node un-prunes and resumes from the proven state. + it('restores L2 blocks if a proof is added due to an L1 reorg', async () => { + // Send txs to trigger multi-block checkpoints + await sendTransactions(TX_COUNT); + + // Capture initial chain state + const initialProvenCheckpoint = (await monitor.run(true)).provenCheckpointNumber; + const initialCheckpoint = monitor.checkpointNumber; + + // Next proof shall not land + proverDelayer.cancelNextTx(); + + // Expect pending chain to advance, so there's something to be pruned + await waitForNodeCheckpoint(node, initialCheckpoint, { + compare: (actual, target) => actual > target, + timeout: L2_SLOT_DURATION_IN_S * 4, + }); + + // Wait until the end of the proof submission window for the first unproven epoch + const firstUnprovenCheckpoint = CheckpointNumber(initialProvenCheckpoint + 1); + await test.waitUntilCheckpointNumber(firstUnprovenCheckpoint, L2_SLOT_DURATION_IN_S * 4); + const epochToWaitFor = await test.rollup.getEpochNumberForCheckpoint(firstUnprovenCheckpoint); + await test.waitUntilLastSlotOfProofSubmissionWindow(epochToWaitFor); + await monitor.run(true); + logger.warn( + `End of epoch ${epochToWaitFor} submission window (L1 block ${await monitor.run(true).then(m => m.l1BlockNumber)}).`, + ); + + // Grab the prover's tx to submit it later as part of a reorg and stop the prover (by stopping its hosting aztec node) + const [proofTx] = proverDelayer.getCancelledTxs(); + expect(proofTx).toBeDefined(); + await test.proverNodes[0].stop(); + logger.warn(`Prover node stopped.`); + + // Wait for the node to prune + const syncTimeout = L2_SLOT_DURATION_IN_S * 2; + await waitForNodeCheckpoint(node, initialProvenCheckpoint + 1, { + compare: (actual, target) => actual <= target, + timeout: syncTimeout, + }); + expect(monitor.provenCheckpointNumber).toEqual(initialProvenCheckpoint); + expect(await node.getCheckpointNumber('proven')).toEqual(initialProvenCheckpoint); + + // But not all is lost, for a reorg gets the proof back on chain! + logger.warn(`Reorging proof back (L1 block ${await monitor.run(true).then(m => m.l1BlockNumber)}).`); + await context.cheatCodes.eth.reorgWithReplacement(4, [[proofTx]]); + const proofTxReceipt = await test.l1Client.getTransactionReceipt({ hash: keccak256(proofTx) }); + expect(proofTxReceipt.status).toEqual('success'); + + // Monitor should update to see the proof + const { checkpointNumber, provenCheckpointNumber } = await monitor.run(true); + expect(checkpointNumber).toBeGreaterThan(initialCheckpoint); + expect(provenCheckpointNumber).toBeGreaterThan(initialProvenCheckpoint); + + // And so the node undoes its reorg + await waitForNodeCheckpoint(node, checkpointNumber, { timeout: syncTimeout }); + await waitForNodeProvenCheckpoint(node, provenCheckpointNumber, { timeout: 1 }); + + // Verify multi-block checkpoints were built + await test.assertMultipleBlocksPerSlot(2); + + logger.warn(`Test succeeded`); + }); + + // Waits until CHECKPOINT_NUMBER is mined and node synced, stops the sequencer, reorgs L1 to + // remove that checkpoint's L1 block, and verifies the node rolls back to checkpoint-1. + it('prunes blocks from pending chain removed from L1 due to an L1 reorg', async () => { + // Send txs to trigger multi-block checkpoints + await sendTransactions(TX_COUNT); + + // Capture initial chain state + const initialCheckpoint = (await monitor.run(true)).checkpointNumber; + + // Wait until CHECKPOINT_NUMBER is mined and node synced, and stop the sequencer + const CHECKPOINT_NUMBER = CheckpointNumber(initialCheckpoint + 3); + await test.waitUntilCheckpointNumber(CHECKPOINT_NUMBER, L2_SLOT_DURATION_IN_S * 10); + expect(monitor.checkpointNumber).toEqual(CHECKPOINT_NUMBER); + const l1BlockNumber = monitor.l1BlockNumber; + // Stop the sequencer immediately so any in-flight pipelined publish for CHECKPOINT_NUMBER+1 + // doesn't extend the reorg range before we calculate it. setConfig alone is not enough under + // pipelining because already-constructed jobs snapshot the old config. + await context.sequencer!.stop(); + logger.warn(`Sequencer stopped`); + // Wait for node to sync to the checkpoint. + await waitForNodeCheckpoint(node, CHECKPOINT_NUMBER, { + compare: (actual, target) => actual === target, + timeout: 10, + }); + logger.warn(`Reached checkpoint ${CHECKPOINT_NUMBER}`); + + // Verify multi-block checkpoints were built before we do the reorg + await test.assertMultipleBlocksPerSlot(2); + + // Remove the L2 block from L1 + await context.cheatCodes.eth.reorgTo(l1BlockNumber - 1); + expect(await monitor.run(true).then(monitor => monitor.checkpointNumber)).toEqual( + CheckpointNumber(CHECKPOINT_NUMBER - 1), + ); + logger.warn(`Removed checkpoint ${CHECKPOINT_NUMBER} via L1 reorg`); + + // And expect the node to prune the block + const expectedCheckpointNumber = CHECKPOINT_NUMBER - 1; + await waitForNodeCheckpoint(node, expectedCheckpointNumber, { + compare: (actual, target) => actual === target, + timeout: 30, + }); + }); + + // Cancels the next sequencer L1 tx (blocking CHECKPOINT_NUMBER from landing), waits for + // several more L1 blocks to pass, then reorgs L1 to include the previously-cancelled checkpoint + // tx and manually sends the blobs to the filestore. Verifies the node sees the new block. + it('sees new blocks added in an L1 reorg', async () => { + // Send txs to trigger multi-block checkpoints + await sendTransactions(TX_COUNT); + + // Capture initial chain state + const initialCheckpoint = (await monitor.run(true)).checkpointNumber; + + // Wait until the checkpoint *before* CHECKPOINT_NUMBER is mined and node synced + const CHECKPOINT_NUMBER = CheckpointNumber(initialCheckpoint + 3); + const prevCheckpointNumber = CheckpointNumber(CHECKPOINT_NUMBER - 1); + await test.waitUntilCheckpointNumber(prevCheckpointNumber, L2_SLOT_DURATION_IN_S * 10); + expect(monitor.checkpointNumber).toEqual(prevCheckpointNumber); + // Wait for node to sync to the checkpoint + await waitForNodeCheckpoint(node, prevCheckpointNumber, { + compare: (actual, target) => actual === target, + timeout: 5, + }); + + // Verify multi-block checkpoints were built before we do the reorg + await test.assertMultipleBlocksPerSlot(2); + + // Cancel the next tx to be mined (the proposal for CHECKPOINT_NUMBER) and pause the sequencer. + // Under pipelining we then stop the sequencer entirely so an in-flight pipelined job for + // CHECKPOINT_NUMBER+1 cannot escape and publish onto L1 before our reorg captures the gap. + sequencerDelayer.cancelNextTx(); + await retryUntil(() => sequencerDelayer.getCancelledTxs().length, 'next block', L2_SLOT_DURATION_IN_S * 2, 0.1); + const [l2BlockTx] = sequencerDelayer.getCancelledTxs(); + await context.sequencer!.stop(); + logger.warn(`Sequencer stopped`); + + // Save the L1 block number when the L2 block would have been mined + const l1BlockNumber = monitor.l1BlockNumber; + + // Wait until a few more L1 blocks go by + await retryUntil(() => monitor.l1BlockNumber > l1BlockNumber + 1, 'l1 block number', L1_BLOCK_TIME_IN_S * 4, 0.1); + await retryUntil(() => archiver.getL1BlockNumber()! > l1BlockNumber + 1, 'archiver sync', 10, 0.1); + expect(await node.getCheckpointNumber('checkpointed')).toEqual(prevCheckpointNumber); + + // Manually update the archiver's L1 syncpoint to ensure we look back when needed + // Otherwise this test just passes because we do not update the L1 syncpoint in the archiver since there are no new blocks + await archiver.dataStores.blocks.setSynchedL1BlockNumber(BigInt(archiver.getL1BlockNumber()!)); + + // Now trigger the reorg. Note that we cannot use reorgWithReplacement here for the reorg, due to an anvil bug with + // blob txs (now fixed, we can just update its version), so we reorg, then replay the tx, and then mine. + const reorgDepth = monitor.l1BlockNumber - l1BlockNumber; + expect(reorgDepth).toBeGreaterThan(0); + logger.warn(`Triggering ${reorgDepth}-block L1 reorg to include L2 block`); + await context.cheatCodes.eth.reorg(reorgDepth); + expect(await context.cheatCodes.eth.blockNumber()).toEqual(l1BlockNumber); + logger.warn(`Sending L2 block tx to L1`); + const txHash = await test.l1Client.sendRawTransaction({ serializedTransaction: l2BlockTx }); + await context.cheatCodes.eth.mine(reorgDepth); + + // Check that the tx was reorged in and succeeded. We log the trace to debug any issues with the tx. + const txReceipt = await test.l1Client.getTransactionReceipt({ hash: txHash }); + logger.warn(`L2 block tx receipt`, { receipt: txReceipt }); + logger.warn(`L2 block tx trace`, { trace: await context.cheatCodes.eth.traceTransaction(txHash) }); + expect(txReceipt.status).toEqual('success'); + expect(txReceipt.blobGasUsed).toBeGreaterThan(0n); + expect(await monitor.run(true).then(m => m.checkpointNumber)).toEqual(CHECKPOINT_NUMBER); + + // We also need to send the blob to the sink, so the node can get it + logger.warn(`Sending blobs to blob client`); + const blobs = await getBlobs(l2BlockTx); + const blobClient = createBlobClient(context.config); + await blobClient.sendBlobsToFilestore(blobs); + + // And wait for the node to see the new block + await waitForNodeCheckpoint(node, CHECKPOINT_NUMBER, { + compare: (actual, target) => actual === target, + timeout: 20, + }); + }); +}); diff --git a/yarn-project/end-to-end/src/single-node/l1-reorgs/messages.parallel.test.ts b/yarn-project/end-to-end/src/single-node/l1-reorgs/messages.parallel.test.ts new file mode 100644 index 000000000000..5df8550d3897 --- /dev/null +++ b/yarn-project/end-to-end/src/single-node/l1-reorgs/messages.parallel.test.ts @@ -0,0 +1,137 @@ +import type { Archiver } from '@aztec/archiver'; +import { AztecAddress } from '@aztec/aztec.js/addresses'; +import { Fr } from '@aztec/aztec.js/fields'; +import type { Logger } from '@aztec/aztec.js/log'; +import { isL1ToL2MessageReady, waitForL1ToL2MessageReady } from '@aztec/aztec.js/messaging'; +import type { AztecNode } from '@aztec/aztec.js/node'; +import type { Delayer } from '@aztec/ethereum/l1-tx-utils'; +import type { ChainMonitor } from '@aztec/ethereum/test'; +import type { ExtendedViemWalletClient } from '@aztec/ethereum/types'; +import { CheckpointNumber } from '@aztec/foundation/branded-types'; +import { timesAsync } from '@aztec/foundation/collection'; +import { retryUntil } from '@aztec/foundation/retry'; + +import 'jest-extended'; + +import { sendL1ToL2Message } from '../../fixtures/l1_to_l2_messaging.js'; +import type { EndToEndContext } from '../../fixtures/utils.js'; +import { waitForL1ToL2MessageSeen } from '../../shared/wait_for_l1_to_l2_message.js'; +import type { SingleNodeTestContext } from '../single_node_test_context.js'; +import { L1ReorgsTest, TX_COUNT } from './setup.js'; + +// Single-node + prover-node suite exercising L1 reorg behavior for L1→L2 cross-chain messages: removal +// of a sent message and insertion of a previously-cancelled message. An L1-client delayer holds back a +// message tx so a reorg can drop or replay it. Shared setup lives in setup.ts. +describe('single-node/l1-reorgs/messages', () => { + let t: L1ReorgsTest; + + let context: EndToEndContext; + let logger: Logger; + let node: AztecNode; + let archiver: Archiver; + let monitor: ChainMonitor; + + let L1_BLOCK_TIME_IN_S: number; + let L2_SLOT_DURATION_IN_S: number; + + let test: SingleNodeTestContext; + + let l1Client: ExtendedViemWalletClient; + let l1ClientDelayer: Delayer; + + const sendTransactions = (count: number, offset = 0) => t.sendTransactions(count, offset); + + beforeEach(async () => { + t = new L1ReorgsTest(); + await t.setup(); + ({ test, context, logger, node, archiver, monitor } = t); + ({ L1_BLOCK_TIME_IN_S, L2_SLOT_DURATION_IN_S } = t); + ({ client: l1Client, delayer: l1ClientDelayer } = await test.createL1Client()); + }); + + afterEach(async () => { + await t.teardown(); + }); + + const sendMessage = async () => + sendL1ToL2Message( + { recipient: await AztecAddress.random(), content: Fr.random(), secretHash: Fr.random() }, + { l1ContractAddresses: context.deployL1ContractsValues.l1ContractAddresses, l1Client }, + ); + + // Sends 3 L1→L2 messages, waits for the last to be seen, reorgs it out, sends a replacement + // message, and verifies the replacement becomes ready while the removed message is gone. + it('updates L1 to L2 messages changed due to an L1 reorg', async () => { + // Send L2 txs to trigger multi-block checkpoints and wait for them to land in a checkpoint + await sendTransactions(TX_COUNT, 100); + await test.waitUntilCheckpointNumber(CheckpointNumber(2), L2_SLOT_DURATION_IN_S * 6); + + // Send 3 messages and wait for archiver sync + logger.warn(`Sending 3 cross chain messages`); + const msgs = await timesAsync(3, async (i: number) => { + logger.warn(`Sending message ${i + 1}`); + return await sendMessage(); + }); + logger.warn(`Sent messages on L1 blocks ${msgs.map(m => m.txReceipt.blockNumber)}`); + + await waitForL1ToL2MessageSeen(node, msgs.at(-1)!.msgHash, { + timeoutSeconds: msgs.length * L1_BLOCK_TIME_IN_S * 2, + }); + + // Reorg the last message out + logger.warn(`Triggering reorg to remove last message`); + const l1BlockNumber = await monitor.run(true).then(m => m.l1BlockNumber); + const l1BlocksToReorg = l1BlockNumber - Number(msgs.at(-1)!.txReceipt.blockNumber) + 1; + await context.cheatCodes.eth.reorg(l1BlocksToReorg); + const newMsg = await sendMessage(); + logger.warn(`Sent new message on L1 block ${newMsg.txReceipt.blockNumber}`); + + // New msg gets synced, and old one is out + await waitForL1ToL2MessageReady(node, newMsg.msgHash, { timeoutSeconds: L2_SLOT_DURATION_IN_S * 5 }); + expect(await isL1ToL2MessageReady(node, msgs[0].msgHash)).toBe(true); + expect(await isL1ToL2MessageReady(node, msgs.at(-1)!.msgHash)).toBe(false); + + // Verify multi-block checkpoints were built + await test.assertMultipleBlocksPerSlot(2); + }); + + // Sends a first message, cancels a second message's L1 tx via delayer, waits for the archiver + // to advance past the cancelled block, then reorgs to include the cancelled message. Sends a + // third message on top and verifies all three are eventually seen by the node. + it('handles missed message inserted by an L1 reorg', async () => { + // Send L2 txs to trigger multi-block checkpoints and wait for them to land in a checkpoint + await sendTransactions(TX_COUNT, 200); + await test.waitUntilCheckpointNumber(CheckpointNumber(2), L2_SLOT_DURATION_IN_S * 6); + + // Send a message and wait for node to sync it + logger.warn(`Sending first cross chain message`); + const firstMsg = await sendMessage(); + logger.warn(`Sent first message on L1 block ${firstMsg.txReceipt.blockNumber}`); + await waitForL1ToL2MessageSeen(node, firstMsg.msgHash, { timeoutSeconds: L1_BLOCK_TIME_IN_S * 3 }); + logger.warn(`Synced first message`); + + // Next message shall not land + l1ClientDelayer.cancelNextTx(); + const secondMsgPromise = sendMessage(); + await retryUntil(() => l1ClientDelayer.getCancelledTxs().length, 'next msg tx', L1_BLOCK_TIME_IN_S, 0.1); + + // Wait until the archiver moves the syncpoint forward + const l1BlockNumber = await monitor.run(true).then(m => m.l1BlockNumber); + await retryUntil(() => archiver.getL1BlockNumber()! > l1BlockNumber, 'archiver sync', L1_BLOCK_TIME_IN_S * 2, 0.1); + + // Now trigger the reorg, where we insert the second message + logger.warn(`Triggering reorg to insert second message`); + const reorgDepth = (await monitor.run(true).then(m => m.l1BlockNumber)) - l1BlockNumber; + await context.cheatCodes.eth.reorgWithReplacement(reorgDepth, [[l1ClientDelayer.getCancelledTxs()[0]]]); + const secondMsg = await secondMsgPromise; + await waitForL1ToL2MessageSeen(node, secondMsg.msgHash, { timeoutSeconds: L1_BLOCK_TIME_IN_S * 3 }); + + // Archiver should see the new message and should be able to accept a third one on top, without any rolling hash issues + logger.warn(`Reorged-in second message on L1 block ${secondMsg.txReceipt.blockNumber}. Sending third message.`); + const thirdMsg = await sendMessage(); + await waitForL1ToL2MessageSeen(node, thirdMsg.msgHash, { timeoutSeconds: L1_BLOCK_TIME_IN_S * 3 }); + + // Verify multi-block checkpoints were built + await test.assertMultipleBlocksPerSlot(2); + }); +}); diff --git a/yarn-project/end-to-end/src/single-node/l1-reorgs/setup.ts b/yarn-project/end-to-end/src/single-node/l1-reorgs/setup.ts new file mode 100644 index 000000000000..a57bab604e74 --- /dev/null +++ b/yarn-project/end-to-end/src/single-node/l1-reorgs/setup.ts @@ -0,0 +1,92 @@ +import type { Archiver } from '@aztec/archiver'; +import type { AztecNodeService } from '@aztec/aztec-node'; +import type { AztecAddress } from '@aztec/aztec.js/addresses'; +import { Fr } from '@aztec/aztec.js/fields'; +import type { Logger } from '@aztec/aztec.js/log'; +import type { AztecNode } from '@aztec/aztec.js/node'; +import type { Delayer } from '@aztec/ethereum/l1-tx-utils'; +import type { ChainMonitor } from '@aztec/ethereum/test'; +import type { TestContract } from '@aztec/noir-test-contracts.js/Test'; +import type { TxHash } from '@aztec/stdlib/tx'; + +import { jest } from '@jest/globals'; + +import type { EndToEndContext } from '../../fixtures/utils.js'; +import { proveAndSendTxs } from '../../test-wallet/utils.js'; +import { FAST_REORG_TIMING, SingleNodeTestContext } from '../single_node_test_context.js'; + +jest.setTimeout(1000 * 60 * 20); + +/** Number of txs to send at the start of each reorg test to trigger multi-block checkpoints. */ +export const TX_COUNT = 8; + +/** + * The single-node + prover-node fixture shared by the L1-reorg suites (`blocks`, `messages`). Stands + * up a {@link SingleNodeTestContext} on the {@link FAST_REORG_TIMING} cadence (ethSlot=4s, + * aztecSlot=36s, block=8s, epoch=4, 32 slots/epoch) with L1 speed-ups disabled so prover/sequencer txs + * can be held back and reorged, registers a {@link TestContract}, and exposes the per-test handles plus + * a `sendTransactions` helper that pre-proves and fires `count` lightweight txs to drive multi-block + * checkpoints. Reorgs themselves are driven by `EthCheatCodes.reorg`/`reorgWithReplacement` at the call + * site. The returned object's fields are populated when `setup()` resolves; create it in `beforeEach`. + */ +export class L1ReorgsTest { + public test!: SingleNodeTestContext; + public context!: EndToEndContext; + public logger!: Logger; + public node!: AztecNode; + public archiver!: Archiver; + public monitor!: ChainMonitor; + public proverDelayer!: Delayer; + public sequencerDelayer!: Delayer; + public contract!: TestContract; + public from!: AztecAddress; + + public L1_BLOCK_TIME_IN_S!: number; + public L2_SLOT_DURATION_IN_S!: number; + + public async setup(): Promise { + this.test = await SingleNodeTestContext.setup({ + ...FAST_REORG_TIMING, // ethSlot=4s, aztecSlot=36s, block=8s, epoch=4, 32 slots/epoch (mainnet) + numberOfAccounts: 1, + maxSpeedUpAttempts: 0, // Do not speed up l1 txs, we dont want them to land + cancelTxOnTimeout: false, + minTxsPerBlock: 0, + maxTxsPerBlock: 1, + aztecProofSubmissionEpochs: 1, + // Pipelining + multi-blocks-per-slot: 8s blocks fit ~4 blocks per 36s slot, and TX_COUNT=8 + // ensures multiple checkpoints have multiple blocks + }); + ({ + proverDelayer: this.proverDelayer, + sequencerDelayer: this.sequencerDelayer, + context: this.context, + logger: this.logger, + monitor: this.monitor, + L1_BLOCK_TIME_IN_S: this.L1_BLOCK_TIME_IN_S, + L2_SLOT_DURATION_IN_S: this.L2_SLOT_DURATION_IN_S, + } = this.test); + this.node = this.context.aztecNode; + this.archiver = (this.node as AztecNodeService).getBlockSource() as Archiver; + this.from = this.context.accounts[0]; + this.contract = await this.test.registerTestContract(this.context.wallet); + } + + public teardown(): Promise { + return this.test.teardown(); + } + + /** Pre-proves and sends `count` txs to generate L2 activity for multi-block checkpoints. */ + public async sendTransactions(count: number, offset = 0): Promise { + this.logger.warn(`Pre-proving ${count} transactions`); + const txHashes = await proveAndSendTxs( + this.context.wallet, + count, + i => this.contract.methods.emit_nullifier(new Fr(offset + i + 1)), + { from: this.from }, + ); + this.logger.warn(`Sent ${txHashes.length} transactions`); + return txHashes; + } +} + +export { jest }; diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_missed_l1_slot.test.ts b/yarn-project/end-to-end/src/single-node/misc/missed_l1_slot.test.ts similarity index 71% rename from yarn-project/end-to-end/src/e2e_epochs/epochs_missed_l1_slot.test.ts rename to yarn-project/end-to-end/src/single-node/misc/missed_l1_slot.test.ts index ef219a64807d..434b145b4fbc 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_missed_l1_slot.test.ts +++ b/yarn-project/end-to-end/src/single-node/misc/missed_l1_slot.test.ts @@ -1,20 +1,15 @@ import type { AztecAddress } from '@aztec/aztec.js/addresses'; -import { NO_WAIT } from '@aztec/aztec.js/contracts'; import { Fr } from '@aztec/aztec.js/fields'; -import type { ChainMonitorEventMap } from '@aztec/ethereum/test'; import { CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types'; -import { timesAsync } from '@aztec/foundation/collection'; -import { AbortError } from '@aztec/foundation/error'; import { sleep } from '@aztec/foundation/sleep'; -import { executeTimeout } from '@aztec/foundation/timer'; import type { TestContract } from '@aztec/noir-test-contracts.js/Test'; import { SequencerState } from '@aztec/sequencer-client'; import { getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; import { jest } from '@jest/globals'; -import { proveInteraction } from '../test-wallet/utils.js'; -import { EpochsTestContext } from './epochs_test.js'; +import { proveAndSendTxs } from '../../test-wallet/utils.js'; +import { SingleNodeTestContext } from '../single_node_test_context.js'; jest.setTimeout(1000 * 60 * 10); @@ -57,30 +52,30 @@ jest.setTimeout(1000 * 60 * 10); // state via setStateFn(state, targetSlot)). Slot N+2 is unique to this cycle: the prior cycle // targeted N+1. // Suite: regression test for sequencer sync logic when L1 slot production stalls mid-slot. -// EpochsTestContext with single-node + mockGossipSubNetwork, prod-seq, interval mining (automine -// during L1 deploy only). Timing: ethSlot=8s (12s CI), aztecSlot=6×ethSlot, epoch=default 6, +// SingleNodeTestContext with single-node + mockGossipSubNetwork, prod-seq, interval mining (automine +// during L1 deploy only). Timing: ethSlot=6s, aztecSlot=6×ethSlot=36s, epoch=default 6, // proofSubmissionEpochs=1024, blockDurationMs=8000, inboxLag=2 (v5 always enforces the timetable, so // the former enforceTimeTable/disableAnvilTestWatcher overrides are gone). No prover. -describe('e2e_epochs/epochs_missed_l1_slot', () => { - let test: EpochsTestContext; +describe('single-node/misc/missed_l1_slot', () => { + let test: SingleNodeTestContext; let contract: TestContract; let from: AztecAddress; // Use enough L1 slots per L2 slot to have room for pausing mining mid-slot. - // With 6 L1 slots per L2 slot (L1=8s, L2=48s), we have plenty of time to + // With 6 L1 slots per L2 slot (L1=6s, L2=36s), we have plenty of time to // publish a checkpoint and pause mining without accidentally skipping a slot. const L1_SLOTS_PER_L2_SLOT = 6; // Block duration tuned to reliably produce 2+ blocks per checkpoint under pipelining: // timeAvailableForBlocks = aztecSlotDuration - checkpointInitializationTime - timeReservedAtEnd - // = 48 - 1 - (1 + 4 + 8) = 34s, which fits ~4 blocks of 8s each. + // = 36 - 1 - (1 + 3 + 8) = 23s, which fits ~2-3 blocks of 8s each. const BLOCK_DURATION_MS = 8_000; // Pre-prove this many txs at the start so blocks have content during the test. const TX_COUNT = 12; beforeEach(async () => { - test = await EpochsTestContext.setup({ + test = await SingleNodeTestContext.setup({ numberOfAccounts: 0, // The 8s blockDurationMs leaves a per-block DA gas budget too small to fit an account // deploy, so use the hardcoded-account fast-path (funded via genesis) even though we @@ -89,19 +84,21 @@ describe('e2e_epochs/epochs_missed_l1_slot', () => { minTxsPerBlock: 0, maxTxsPerBlock: 1, blockDurationMs: BLOCK_DURATION_MS, + // Pin the L1 slot to 6s so the L2 slot is 6 L1 slots * 6s = 36s (the multi-node mode), while + // preserving the load-bearing 6-L1-slots-per-L2-slot invariant this test's pause math relies on. + ethereumSlotDuration: 6, aztecSlotDurationInL1Slots: L1_SLOTS_PER_L2_SLOT, startProverNode: false, aztecProofSubmissionEpochs: 1024, - inboxLag: 2, // Required for the proposer's own broadcasts to route through the local // proposal handler (the dummy p2p service drops them). Without this, the // archiver's #proposedCheckpoints map stays empty and the pipelining // override path is never taken. mockGossipSubNetwork: true, - // With L1=12s on CI, aztecSlotDuration=72s and blockDurationMs=8000ms gives only ~1/9 of - // slot mana per block — too small for emit_nullifier's daGas (~196k) under the default - // 1.2 allocation. Bump it so the pre-proved txs actually land and step 6's - // assertMultipleBlocksPerSlot has data to verify against. + // With aztecSlotDuration=36s and blockDurationMs=8000ms each block gets only a few seconds of + // slot mana — too small for emit_nullifier's daGas (~196k) under the default 1.2 allocation. + // Bump it so the pre-proved txs actually land and step 6's assertMultipleBlocksPerSlot has data + // to verify against. perBlockAllocationMultiplier: 8, }); @@ -126,10 +123,12 @@ describe('e2e_epochs/epochs_missed_l1_slot', () => { // Pre-prove a batch of txs and send them so blocks have content while building checkpoints. // Done before waiting for the early checkpoint so that mbps is exercised by the time we pause. logger.info(`Pre-proving ${TX_COUNT} transactions`); - const txs = await timesAsync(TX_COUNT, i => - proveInteraction(context.wallet, contract.methods.emit_nullifier(new Fr(i + 1)), { from }), + const txHashes = await proveAndSendTxs( + context.wallet, + TX_COUNT, + i => contract.methods.emit_nullifier(new Fr(i + 1)), + { from }, ); - const txHashes = await Promise.all(txs.map(tx => tx.send({ wait: NO_WAIT }))); logger.info(`Sent ${txHashes.length} transactions`); // Step 1: Wait for a checkpoint published in the first half of its L2 slot. @@ -138,42 +137,20 @@ describe('e2e_epochs/epochs_missed_l1_slot', () => { // slot (e.g. in the last L1 slot of L2 slot N), slotFromL1Sync would already be N and the // bug would not be exercised. logger.info('Waiting for a checkpoint published in the first half of its L2 slot...'); - // REFACTOR: raw on-off subscription to ChainMonitor 'checkpoint' event; replace with a - // DSL helper that waits for the first checkpoint satisfying a predicate (e.g. inFirstHalfOfSlot). - const checkpointEvent = await executeTimeout( - signal => - new Promise((res, rej) => { - const handleCheckpoint = (...[ev]: ChainMonitorEventMap['checkpoint']) => { - // Skip the genesis checkpoint. - if (ev.checkpointNumber === 0) { - return; - } - const slotStart = getTimestampForSlot(ev.l2SlotNumber, constants); - // Half-slot cutoff keeps slotFromL1Sync at N-1 with comfortable margin: at the cutoff - // the next L1 block lands at slotStart + L2_SLOT_DURATION/2 + L1_BLOCK_TIME, which is - // still well within slot N (since L1 < L2/2). - const cutoff = slotStart + BigInt(Math.floor(L2_SLOT_DURATION / 2)); - if (ev.timestamp < cutoff) { - logger.info( - `Checkpoint ${ev.checkpointNumber} in slot ${ev.l2SlotNumber} at L1 timestamp ${ev.timestamp}`, - { slotStart, cutoff }, - ); - res(ev); - monitor.off('checkpoint', handleCheckpoint); - } else { - logger.info( - `Skipping checkpoint ${ev.checkpointNumber}: published at ${ev.timestamp} (cutoff ${cutoff})`, - ); - } - }; - signal.onabort = () => { - monitor.off('checkpoint', handleCheckpoint); - rej(new AbortError()); - }; - monitor.on('checkpoint', handleCheckpoint); - }), - 120_000, - 'Wait for early checkpoint', + const checkpointEvent = await monitor.waitForCheckpoint( + ev => { + // Skip the genesis checkpoint. + if (ev.checkpointNumber === 0) { + return false; + } + const slotStart = getTimestampForSlot(ev.l2SlotNumber, constants); + // Half-slot cutoff keeps slotFromL1Sync at N-1 with comfortable margin: at the cutoff the + // next L1 block lands at slotStart + L2_SLOT_DURATION/2 + L1_BLOCK_TIME, which is still well + // within slot N (since L1 < L2/2). + const cutoff = slotStart + BigInt(Math.floor(L2_SLOT_DURATION / 2)); + return ev.timestamp < cutoff; + }, + { timeout: 120_000 }, ); const checkpointSlotNumber = checkpointEvent.l2SlotNumber; @@ -218,28 +195,11 @@ describe('e2e_epochs/epochs_missed_l1_slot', () => { `Waiting for sequencer to reach INITIALIZING_CHECKPOINT for target slot ${targetSlotForBugFixCycle} ` + `(build slot ${nextSlotNumber}) during mining pause...`, ); - // REFACTOR: raw on-off subscription to sequencer 'state-changed' event; a DSL helper that - // waits for a specific (state, slot) pair would eliminate the manual Promise + signal cleanup. - await executeTimeout( - signal => - new Promise((res, rej) => { - const stateListener = (args: { newState: SequencerState; targetSlot?: SlotNumber }) => { - if ( - args.newState === SequencerState.INITIALIZING_CHECKPOINT && - args.targetSlot === targetSlotForBugFixCycle - ) { - sequencer.off('state-changed', stateListener); - res(); - } - }; - signal.onabort = () => { - sequencer.off('state-changed', stateListener); - rej(new AbortError()); - }; - sequencer.on('state-changed', stateListener); - }), - L2_SLOT_DURATION * 3 * 1000, - `Wait for sequencer INITIALIZING_CHECKPOINT at target slot ${targetSlotForBugFixCycle}`, + await test.waitForSequencerEvent( + sequencer, + 'state-changed', + args => args.newState === SequencerState.INITIALIZING_CHECKPOINT && args.targetSlot === targetSlotForBugFixCycle, + { timeout: L2_SLOT_DURATION * 3 * 1000 }, ); logger.info( diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_partial_proof_multi_root.test.ts b/yarn-project/end-to-end/src/single-node/partial-proofs/multi_root.test.ts similarity index 92% rename from yarn-project/end-to-end/src/e2e_epochs/epochs_partial_proof_multi_root.test.ts rename to yarn-project/end-to-end/src/single-node/partial-proofs/multi_root.test.ts index 8df67ef56029..3ccc87bb63ae 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_partial_proof_multi_root.test.ts +++ b/yarn-project/end-to-end/src/single-node/partial-proofs/multi_root.test.ts @@ -20,24 +20,22 @@ import { } from '@aztec/stdlib/messaging'; import { type TxReceipt, TxStatus } from '@aztec/stdlib/tx'; -import { jest } from '@jest/globals'; import { type Hex, decodeEventLog } from 'viem'; -import { EpochsTestContext } from './epochs_test.js'; - -jest.setTimeout(1000 * 60 * 10); +import { waitForL2ToL1Witness } from '../../fixtures/wait_helpers.js'; +import { SingleNodeTestContext, jest } from './setup.js'; // Suite: verifies the AZIP-14 partial-proof multi-root Outbox design. Drives an EpochTestSettler // manually to stage progressively deeper partial-proof roots (K=1, 2, 3) for the same epoch, then // asserts: (a) the node picks the smallest covering root, (b) any covering root produces a valid // consume tx, (c) the shared bitmap blocks double-spend, and (d) K=4 can be staged later. -// EpochsTestContext: single node, no prover, prod-seq, interval mining. Timing: ethSlot=default +// SingleNodeTestContext: single node, no prover, prod-seq, interval mining. Timing: ethSlot=default // (8s/12s CI), aztecSlot=default, epoch=1000, proofSubmissionEpochs=1024 (v5: the disableAnvilTestWatcher // override was removed and a perBlockAllocationMultiplier=1.3 was added so the first block of the // now-up-to-5-block checkpoint has enough DA budget for the TestContract deploy tx). The test actively // calls the Outbox L1 contract to consume L2-to-L1 messages → cross-chain. -describe('e2e_epochs/epochs_partial_proof_multi_root', () => { - let test: EpochsTestContext; +describe('single-node/partial-proofs/multi_root', () => { + let test: SingleNodeTestContext; let logger: Logger; let node: AztecNode; let l1Client: ExtendedViemWalletClient; @@ -50,7 +48,7 @@ describe('e2e_epochs/epochs_partial_proof_multi_root', () => { let recipient: EthAddress; beforeEach(async () => { - test = await EpochsTestContext.setup({ + test = await SingleNodeTestContext.setup({ numberOfAccounts: 1, minTxsPerBlock: 1, // With the enforced timetable this setup can have 5 blocks per checkpoint. The default @@ -225,14 +223,7 @@ describe('e2e_epochs/epochs_partial_proof_multi_root', () => { // Consume msg2 against the smallest covering root the node picks (K=2). { - // REFACTOR: hand-rolled retryUntil waiting for L2ToL1 membership witness availability; a - // DSL helper like waitForL2ToL1MembershipWitness(txHash, leaf) would encapsulate the retry. - const witness = await retryUntil( - () => node.getL2ToL1MembershipWitness(sends[1].receipt.txHash, sends[1].leaf), - 'K=2 membership witness', - 30, - 1, - ); + const witness = await waitForL2ToL1Witness(node, sends[1].receipt.txHash, sends[1].leaf); expect(witness).toBeDefined(); expect(witness.epochNumber).toBe(epoch); expect(witness.numCheckpointsInEpoch).toBe(2); @@ -254,12 +245,7 @@ describe('e2e_epochs/epochs_partial_proof_multi_root', () => { // the second consume reverts due to the shared bitmap. { // K=1 path (default node choice). - const witnessK1 = await retryUntil( - () => node.getL2ToL1MembershipWitness(sends[0].receipt.txHash, sends[0].leaf), - 'K=1 membership witness', - 30, - 1, - ); + const witnessK1 = await waitForL2ToL1Witness(node, sends[0].receipt.txHash, sends[0].leaf); expect(witnessK1.numCheckpointsInEpoch).toBe(1); expect(witnessK1.root.toString()).toBe(root1.toString()); await expectConsumeSucceeds(sends[0].msg, sends[0].leaf, witnessK1, epoch); @@ -290,12 +276,7 @@ describe('e2e_epochs/epochs_partial_proof_multi_root', () => { // Replay protection: consuming msg2 again (now under any covering root) reverts. { - const witness = await retryUntil( - () => node.getL2ToL1MembershipWitness(sends[1].receipt.txHash, sends[1].leaf), - 'replay membership witness', - 30, - 1, - ); + const witness = await waitForL2ToL1Witness(node, sends[1].receipt.txHash, sends[1].leaf); await expect( outbox.consume( sends[1].msg, @@ -320,12 +301,7 @@ describe('e2e_epochs/epochs_partial_proof_multi_root', () => { const root4 = computeEpochOutHash(messagesPerCheckpoint.slice(0, 4)); await retryUntil(async () => !(await outbox.getRoots(epoch))[3].isZero(), 'K=4 root visible', 10, 0.1); - const witness = await retryUntil( - () => node.getL2ToL1MembershipWitness(sends[3].receipt.txHash, sends[3].leaf), - 'K=4 membership witness', - 30, - 1, - ); + const witness = await waitForL2ToL1Witness(node, sends[3].receipt.txHash, sends[3].leaf); expect(witness.numCheckpointsInEpoch).toBe(4); expect(witness.root.toString()).toBe(root4.toString()); await expectConsumeSucceeds(sends[3].msg, sends[3].leaf, witness, epoch); diff --git a/yarn-project/end-to-end/src/single-node/partial-proofs/setup.ts b/yarn-project/end-to-end/src/single-node/partial-proofs/setup.ts new file mode 100644 index 000000000000..de507584c2d5 --- /dev/null +++ b/yarn-project/end-to-end/src/single-node/partial-proofs/setup.ts @@ -0,0 +1,7 @@ +import { jest } from '@jest/globals'; + +import { SingleNodeTestContext } from '../single_node_test_context.js'; + +jest.setTimeout(1000 * 60 * 10); + +export { jest, SingleNodeTestContext }; diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_partial_proof.test.ts b/yarn-project/end-to-end/src/single-node/partial-proofs/single_root.test.ts similarity index 52% rename from yarn-project/end-to-end/src/e2e_epochs/epochs_partial_proof.test.ts rename to yarn-project/end-to-end/src/single-node/partial-proofs/single_root.test.ts index 6c486d62ac56..06a6c773932d 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_partial_proof.test.ts +++ b/yarn-project/end-to-end/src/single-node/partial-proofs/single_root.test.ts @@ -1,26 +1,21 @@ import type { Logger } from '@aztec/aztec.js/log'; import type { ChainMonitor } from '@aztec/ethereum/test'; import { CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types'; -import { retryUntil } from '@aztec/foundation/retry'; -import { jest } from '@jest/globals'; +import { SingleNodeTestContext, jest } from './setup.js'; -import { EpochsTestContext } from './epochs_test.js'; - -jest.setTimeout(1000 * 60 * 10); - -// Suite: verifies that manually triggering epoch proving via startProof() results in a partial-proof -// being submitted on L1. EpochsTestContext with single node + fake prover. Timing: ethSlot=default -// (8s/12s CI), aztecSlot=default, epoch=1000 (overridden to a very long epoch so the epoch never -// ends during the test), proofSubmissionEpochs=1 (default). prod-seq, interval mining. -describe('e2e_epochs/epochs_partial_proof', () => { +// Co-located with the multi-root suite: both manually drive partial-epoch proving on a single node +// with a very long epoch. This one is the only coverage of the prover-node `startProof` path (the +// multi-root suite hand-drives an EpochTestSettler with no prover), so it keeps its own setup +// rather than folding into the multi-root beforeEach. +describe('single-node/partial-proofs/single_root', () => { let logger: Logger; let monitor: ChainMonitor; - let test: EpochsTestContext; + let test: SingleNodeTestContext; beforeEach(async () => { - test = await EpochsTestContext.setup({ aztecEpochDuration: 1000 }); + test = await SingleNodeTestContext.setup({ aztecEpochDuration: 1000 }); ({ monitor, logger } = test); }); @@ -38,9 +33,7 @@ describe('e2e_epochs/epochs_partial_proof', () => { logger.info(`Kicking off partial proof`); await test.context.proverNode!.getProverNode()!.startProof(EpochNumber(0)); - // REFACTOR: hand-rolled retryUntil polling ChainMonitor.provenCheckpointNumber; replace with - // test.waitUntilProvenCheckpointNumber(CheckpointNumber(1)) from EpochsTestContext. - await retryUntil(() => monitor.provenCheckpointNumber > CheckpointNumber(0), 'proof', 120, 1); + await test.waitUntilProvenCheckpointNumber(CheckpointNumber(1)); logger.info(`Test succeeded with proven checkpoint number ${monitor.provenCheckpointNumber}`); }); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_proof_public_cross_chain.test.ts b/yarn-project/end-to-end/src/single-node/proving/cross_chain_public_message.test.ts similarity index 81% rename from yarn-project/end-to-end/src/e2e_epochs/epochs_proof_public_cross_chain.test.ts rename to yarn-project/end-to-end/src/single-node/proving/cross_chain_public_message.test.ts index 4c068e530a33..c76ac87a477e 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_proof_public_cross_chain.test.ts +++ b/yarn-project/end-to-end/src/single-node/proving/cross_chain_public_message.test.ts @@ -4,14 +4,14 @@ import type { Logger } from '@aztec/aztec.js/log'; import { waitForL1ToL2MessageReady } from '@aztec/aztec.js/messaging'; import { TxExecutionResult } from '@aztec/aztec.js/tx'; import { EthAddress } from '@aztec/foundation/eth-address'; -import { retryUntil } from '@aztec/foundation/retry'; import { TestContract } from '@aztec/noir-test-contracts.js/Test'; import { jest } from '@jest/globals'; -import { sendL1ToL2Message } from '../fixtures/l1_to_l2_messaging.js'; -import type { EndToEndContext } from '../fixtures/utils.js'; -import { EpochsTestContext } from './epochs_test.js'; +import { sendL1ToL2Message } from '../../fixtures/l1_to_l2_messaging.js'; +import type { EndToEndContext } from '../../fixtures/utils.js'; +import { waitForProvenBlock } from '../../fixtures/wait_helpers.js'; +import { SingleNodeTestContext } from '../single_node_test_context.js'; jest.setTimeout(1000 * 60 * 10); @@ -22,18 +22,18 @@ jest.setTimeout(1000 * 60 * 10); // a message that was added to the L1-to-L2 message tree in the same block — the prover reverts the tx while // the sequencer processes it successfully. // -// EpochsTestContext: 1 node + fake prover, prod-seq, interval mining. Timing: all defaults (ethSlot=8s/12s +// SingleNodeTestContext: 1 node + fake prover, prod-seq, interval mining. Timing: all defaults (ethSlot=8s/12s // CI, aztecSlot=16s/24s, epoch=6, proofSubmissionEpochs=1), minTxsPerBlock=1 (v5: the disableAnvilTestWatcher // override was removed). Cross-chain: writes to L1 Inbox (sendL1ToL2Message), then claims the message in a // public L2 function. -describe('e2e_epochs/epochs_proof_public_cross_chain', () => { +describe('single-node/proving/cross_chain_public_message', () => { let context: EndToEndContext; let logger: Logger; - let test: EpochsTestContext; + let test: SingleNodeTestContext; beforeEach(async () => { - test = await EpochsTestContext.setup({ + test = await SingleNodeTestContext.setup({ numberOfAccounts: 1, minTxsPerBlock: 1, sequencerPublisherAllowInvalidStates: true, @@ -82,17 +82,9 @@ describe('e2e_epochs/epochs_proof_public_cross_chain', () => { // Wait until a proof lands for the transaction logger.warn(`Waiting for proof for tx ${txReceipt.txHash} mined at ${txReceipt.blockNumber!}`); - // REFACTOR: hand-rolled retryUntil polling aztecNode.getBlockNumber('proven'); replace with - // test.waitUntilProvenCheckpointNumber or a waitForProvenBlock(blockNumber) DSL helper. - await retryUntil( - async () => { - const provenBlockNumber = await context.aztecNode.getBlockNumber('proven'); - logger.info(`Proven block number is ${provenBlockNumber}`); - return provenBlockNumber >= txReceipt.blockNumber!; - }, - 'Proof has been submitted', - test.L2_SLOT_DURATION_IN_S * test.epochDuration * 3, - ); + await waitForProvenBlock(context.aztecNode, txReceipt.blockNumber!, { + timeout: test.L2_SLOT_DURATION_IN_S * test.epochDuration * 3, + }); const provenBlockNumber = await context.aztecNode.getBlockNumber('proven'); expect(provenBlockNumber).toBeGreaterThanOrEqual(txReceipt.blockNumber!); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_empty_blocks_proof.test.ts b/yarn-project/end-to-end/src/single-node/proving/empty_blocks.test.ts similarity index 61% rename from yarn-project/end-to-end/src/e2e_epochs/epochs_empty_blocks_proof.test.ts rename to yarn-project/end-to-end/src/single-node/proving/empty_blocks.test.ts index 00d4d5fe688e..5c96a32a9b33 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_empty_blocks_proof.test.ts +++ b/yarn-project/end-to-end/src/single-node/proving/empty_blocks.test.ts @@ -3,18 +3,13 @@ import { RollupContract } from '@aztec/ethereum/contracts'; import { ChainMonitor } from '@aztec/ethereum/test'; import { sleep } from '@aztec/foundation/sleep'; -import { jest } from '@jest/globals'; +import type { EndToEndContext } from '../../fixtures/utils.js'; +import { SingleNodeTestContext, jest } from './setup.js'; -import type { EndToEndContext } from '../fixtures/utils.js'; -import { EpochsTestContext } from './epochs_test.js'; - -jest.setTimeout(1000 * 60 * 15); - -// Single-node epoch suite (default EpochsTestContext, no extra validator nodes). Starts a prover -// node (fake proofs). Sets minTxsPerBlock=1 after setup so blocks are empty, then verifies that -// the prover still submits a proof for those empty-block checkpoints within the proof submission -// window. -describe('e2e_epochs/epochs_empty_blocks_proof', () => { +// Starts a prover node (fake proofs) on the default setup, raises minTxsPerBlock=1 so blocks are +// empty, then verifies the prover still submits a proof for those empty-block checkpoints within the +// proof submission window. +describe('single-node/proving/empty_blocks', () => { let context: EndToEndContext; let rollup: RollupContract; let logger: Logger; @@ -22,10 +17,10 @@ describe('e2e_epochs/epochs_empty_blocks_proof', () => { let L1_BLOCK_TIME_IN_S: number; - let test: EpochsTestContext; + let test: SingleNodeTestContext; beforeEach(async () => { - test = await EpochsTestContext.setup({}); + test = await SingleNodeTestContext.setup({}); ({ context, rollup, logger, monitor, L1_BLOCK_TIME_IN_S } = test); }); @@ -41,9 +36,10 @@ describe('e2e_epochs/epochs_empty_blocks_proof', () => { context.sequencer?.updateConfig({ minTxsPerBlock: 1 }); await test.waitUntilEpochStarts(1); - // REFACTOR: raw sleep to flush pending L1 txs; replace with a helper that waits for the - // sequencer to finish all in-flight L1 publishes (e.g. waitForSequencerIdle). - // Sleep to make sure any pending checkpoints are published + // Sleep to make sure any pending checkpoints are published. We deliberately keep the fixed + // sleep rather than waiting for the sequencer to reach IDLE: the sequencer is typically already + // idle here, so an IDLE wait would return immediately and not give the in-flight L1 publish time + // to land. The window we need is the publish settling, not the sequencer becoming idle. await sleep(L1_BLOCK_TIME_IN_S * 1000); const checkpointNumberAtEndOfEpoch0 = await rollup.getCheckpointNumber(); logger.info(`Starting epoch 1 after checkpoint ${checkpointNumberAtEndOfEpoch0}`); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_long_proving_time.test.ts b/yarn-project/end-to-end/src/single-node/proving/long_proving_time.test.ts similarity index 86% rename from yarn-project/end-to-end/src/e2e_epochs/epochs_long_proving_time.test.ts rename to yarn-project/end-to-end/src/single-node/proving/long_proving_time.test.ts index 6c68e2bff650..f31536bbb0d0 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_long_proving_time.test.ts +++ b/yarn-project/end-to-end/src/single-node/proving/long_proving_time.test.ts @@ -2,39 +2,34 @@ import type { Logger } from '@aztec/aztec.js/log'; import { ChainMonitor } from '@aztec/ethereum/test'; import { sleep } from '@aztec/foundation/sleep'; -import { jest } from '@jest/globals'; - -import { EpochsTestContext } from './epochs_test.js'; - -jest.setTimeout(1000 * 60 * 15); +import { SingleNodeTestContext, jest } from './setup.js'; const MAX_JOB_COUNT = 20; -// Single-node + prover-node suite verifying that a prover node whose proving time spans multiple +// Single-node + prover-node scenario verifying that a prover node whose proving time spans multiple // epochs (proverTestDelayMs ≈ 3 epochs) still eventually submits valid proofs while proving several // epochs concurrently (proverNodeMaxPendingJobs=20, proverBrokerMaxEpochsToKeepResultsFor=10) without // the broker rejecting in-flight jobs as stale. (v5: previously capped at one job at a time with -// proverNodeMaxPendingJobs=1; now exercises concurrent multi-epoch proving.) Uses EpochsTestContext -// default setup (single sequencer, fake prover with delay, no mock gossip). -describe('e2e_epochs/epochs_long_proving_time', () => { +// proverNodeMaxPendingJobs=1; now exercises concurrent multi-epoch proving.) +describe('single-node/proving/long_proving_time', () => { let logger: Logger; let monitor: ChainMonitor; let L1_BLOCK_TIME_IN_S: number; - let test: EpochsTestContext; + let test: SingleNodeTestContext; beforeEach(async () => { // Given empty blocks and 2-block epochs, the circuits needed for proving an epoch are: // 1) base parity, 2) root parity, 3) empty block, and 4) epoch root. // So we delay proving of each circuit such that each epoch takes 3 epochs to prove. const aztecEpochDuration = 2; - const { aztecSlotDuration } = EpochsTestContext.getSlotDurations({ aztecEpochDuration }); + const { aztecSlotDuration } = SingleNodeTestContext.getSlotDurations({ aztecEpochDuration }); const epochDurationInSeconds = aztecSlotDuration * aztecEpochDuration; const proverTestDelayMs = (epochDurationInSeconds * 1000 * 3) / 4; // Each epoch takes ~3 epochs to prove, so the broker needs to keep results for // at least that many epochs to avoid rejecting jobs as stale. - test = await EpochsTestContext.setup({ + test = await SingleNodeTestContext.setup({ aztecEpochDuration, aztecProofSubmissionEpochs: 1000, // Effectively don't re-org proverTestDelayMs, diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_multi_proof.test.ts b/yarn-project/end-to-end/src/single-node/proving/multi_proof.test.ts similarity index 78% rename from yarn-project/end-to-end/src/e2e_epochs/epochs_multi_proof.test.ts rename to yarn-project/end-to-end/src/single-node/proving/multi_proof.test.ts index 61fc9cdfe058..42b981937680 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_multi_proof.test.ts +++ b/yarn-project/end-to-end/src/single-node/proving/multi_proof.test.ts @@ -1,38 +1,35 @@ import type { Logger } from '@aztec/aztec.js/log'; -import { RollupContract } from '@aztec/ethereum/contracts'; import { BlockNumber, EpochNumber } from '@aztec/foundation/branded-types'; -import { retryUntil } from '@aztec/foundation/retry'; import { sleep } from '@aztec/foundation/sleep'; import { type L1RollupConstants, getSlotRangeForEpoch } from '@aztec/stdlib/epoch-helpers'; import { jest } from '@jest/globals'; -import type { EndToEndContext } from '../fixtures/utils.js'; -import { EpochsTestContext } from './epochs_test.js'; +import type { EndToEndContext } from '../../fixtures/utils.js'; +import { SingleNodeTestContext } from '../single_node_test_context.js'; jest.setTimeout(1000 * 60 * 10); // Suite: checks that multiple prover nodes can each submit their own valid proof for the same epoch. -// EpochsTestContext with startProverNode=false (test creates 3 prover nodes manually). Single +// SingleNodeTestContext with startProverNode=false (test creates 3 prover nodes manually). Single // sequencer node. Timing: all defaults (ethSlot=8s/12s CI, aztecSlot=16s/24s, epoch=6, // proofSubmissionEpochs=1, fake prover). Staggered top-tree-prove delays (v5 patches // createTopTreeOrchestrator's prove() per node; pre-v5 it patched finalizeEpoch) ensure provers don't // all land at the same L1 block. -describe('e2e_epochs/epochs_multi_proof', () => { +describe('single-node/proving/multi_proof', () => { let context: EndToEndContext; - let rollup: RollupContract; let constants: L1RollupConstants; let logger: Logger; let L1_BLOCK_TIME_IN_S: number; - let test: EpochsTestContext; + let test: SingleNodeTestContext; beforeEach(async () => { // Don't start prover node during setup - we'll create and manage all prover nodes in the test // This ensures we can apply delay patches before any prover starts proving - test = await EpochsTestContext.setup({ startProverNode: false }); - ({ context, rollup, constants, logger, L1_BLOCK_TIME_IN_S } = test); + test = await SingleNodeTestContext.setup({ startProverNode: false }); + ({ context, constants, logger, L1_BLOCK_TIME_IN_S } = test); }); afterEach(async () => { @@ -89,19 +86,7 @@ describe('e2e_epochs/epochs_multi_proof', () => { logger.info(`Starting epoch 1 with length ${firstEpochLength} after L2 block ${firstEpochLastBlockNum}`); // Wait until all three provers have submitted proofs - // REFACTOR: hand-rolled retryUntil polling loop over Promise.all per-prover submission check; - // a DSL helper like waitForAllProversToSubmit(proverIds, epoch) would centralise this pattern. - await retryUntil( - async () => { - const haveSubmitted = await Promise.all( - proverIds.map(proverId => rollup.getHasSubmittedProof(EpochNumber(0), firstEpochLength, proverId)), - ); - logger.info(`Proof submissions: ${haveSubmitted.join(', ')}`); - return haveSubmitted.every(submitted => submitted); - }, - 'Provers have submitted proofs', - 120, - ); + await test.waitForAllProversToSubmit(EpochNumber(0), firstEpochLength); const provenBlockNumber = await context.aztecNode.getBlockNumber('proven'); expect(provenBlockNumber).toEqual(firstEpochLastBlockNum); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_optimistic_proving.parallel.test.ts b/yarn-project/end-to-end/src/single-node/proving/optimistic.parallel.test.ts similarity index 87% rename from yarn-project/end-to-end/src/e2e_epochs/epochs_optimistic_proving.parallel.test.ts rename to yarn-project/end-to-end/src/single-node/proving/optimistic.parallel.test.ts index 2ec8cca88d30..b2eb15c212b2 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_optimistic_proving.parallel.test.ts +++ b/yarn-project/end-to-end/src/single-node/proving/optimistic.parallel.test.ts @@ -2,9 +2,7 @@ import type { Logger } from '@aztec/aztec.js/log'; import { RollupContract } from '@aztec/ethereum/contracts'; import { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; import { Fr } from '@aztec/foundation/curves/bn254'; -import { promiseWithResolvers } from '@aztec/foundation/promise'; import { retryUntil } from '@aztec/foundation/retry'; -import { executeTimeout } from '@aztec/foundation/timer'; import type { TestProverNode } from '@aztec/prover-node/test'; import { getEpochAtSlot, getSlotRangeForEpoch } from '@aztec/stdlib/epoch-helpers'; import type { AztecNode } from '@aztec/stdlib/interfaces/server'; @@ -12,16 +10,17 @@ import { TxExecutionResult } from '@aztec/stdlib/tx'; import { expect, jest } from '@jest/globals'; -import type { EndToEndContext } from '../fixtures/utils.js'; -import { proveInteraction } from '../test-wallet/utils.js'; -import { EpochsTestContext } from './epochs_test.js'; +import type { EndToEndContext } from '../../fixtures/utils.js'; +import { waitForNodeCheckpoint } from '../../fixtures/wait_helpers.js'; +import { proveInteraction } from '../../test-wallet/utils.js'; +import { FAST_REORG_TIMING, SingleNodeTestContext } from '../single_node_test_context.js'; jest.setTimeout(1000 * 60 * 20); /** * E2E tests for optimistic (checkpoint-driven) proving with reorg scenarios. * - * Setup: a single sequencer/validator node from `EpochsTestContext.setup` plus the context's fake prover-node (no + * Setup: a single sequencer/validator node from `SingleNodeTestContext.setup` plus the context's fake prover-node (no * `mockGossipSubNetwork`, so no gossip bus), making this a `single-node` test on the production `Sequencer`. Each of the * six `describe` blocks builds a fresh context in its own `beforeEach` and tears it down in the shared `afterEach`. The * happy-path pair uses defaults (`numberOfAccounts: 1`; ethSlot=8s local/12s CI, aztecSlot=16s/24s, epoch=6, @@ -31,15 +30,13 @@ jest.setTimeout(1000 * 60 * 20); * sets `startProverNode: false` and spins up the prover via `test.createProverNode()` partway through the epoch. * * L1 reorgs are driven by `cheatCodes.eth.reorgWithReplacement` and treated as `other-active L1` per the rubric — NOT - * cross-chain bridging — so the file stays `single-node` (mirrors `epochs_partial_proof` / `epochs_sync_after_reorg`). + * cross-chain bridging — so the file stays `single-node` (mirrors `partial-proofs/single_root` and + * `recovery/sync_after_reorg`). * Block production is paused/resumed mid-test via the `skipPublishingCheckpointsPercent` node-admin config, and the * `checkpoint reorg during proving` describe gates top-tree proving with the prover's `beforeTopTreeProve` session hook. * Anvil runs on interval mining; time advances naturally (the reorgs and `waitUntilNextEpochStarts` do the warping). - * - * Proposed category: `single-node` (epochs/). Heavy hand-rolled coordination throughout — see the inline REFACTOR - * markers below for the raw-async sites a DSL helper should replace. */ -describe('e2e_epochs/epochs_optimistic_proving', () => { +describe('single-node/proving/optimistic', () => { let context: EndToEndContext; let rollup: RollupContract; let logger: Logger; @@ -47,9 +44,7 @@ describe('e2e_epochs/epochs_optimistic_proving', () => { let L2_SLOT_DURATION_IN_S: number; - let test: EpochsTestContext; - - const getCheckpointNumber = (n: AztecNode) => n.getCheckpointNumber('checkpointed'); + let test: SingleNodeTestContext; /** * Looks up the epoch a given checkpoint sits in by reading its slot from the archiver. @@ -145,7 +140,7 @@ describe('e2e_epochs/epochs_optimistic_proving', () => { describe('happy path', () => { beforeEach(async () => { - test = await EpochsTestContext.setup({ numberOfAccounts: 1 }); + test = await SingleNodeTestContext.setup({ numberOfAccounts: 1 }); ({ rollup, logger, context } = test); ({ L2_SLOT_DURATION_IN_S } = test); node = context.aztecNode; @@ -228,7 +223,8 @@ describe('e2e_epochs/epochs_optimistic_proving', () => { describe('mid-epoch checkpoint reorg with replacement', () => { beforeEach(async () => { - test = await EpochsTestContext.setup({ + test = await SingleNodeTestContext.setup({ + ...FAST_REORG_TIMING, maxSpeedUpAttempts: 0, cancelTxOnTimeout: false, // Use a longer epoch so the replacement checkpoint has room to land in the same @@ -237,13 +233,8 @@ describe('e2e_epochs/epochs_optimistic_proving', () => { // boundary (see CI failure on `+2` reorg, replacement landed two slots into the // next epoch). aztecEpochDuration: 8, - ethereumSlotDuration: 4, - aztecSlotDuration: 36, - blockDurationMs: 8000, minTxsPerBlock: 0, aztecProofSubmissionEpochs: 1000, - anvilSlotsInAnEpoch: 32, - inboxLag: 2, }); ({ rollup, logger, context } = test); ({ L2_SLOT_DURATION_IN_S } = test); @@ -301,12 +292,10 @@ describe('e2e_epochs/epochs_optimistic_proving', () => { logger.info(`After reorg: checkpoint ${afterReorgCheckpoint} (was ${checkpointBeforeReorg})`); // Verify node detects the reorg. - await retryUntil( - () => getCheckpointNumber(node).then(cp => cp <= afterReorgCheckpoint), - 'reorg detected', - 30, - 0.5, - ); + await waitForNodeCheckpoint(node, afterReorgCheckpoint, { + compare: (actual, target) => actual <= target, + timeout: 30, + }); // Verify the prover-node observes the prune. `markPruned()` fires reactively when // the L2BlockStream emits the prune; the SlotWatcher then reaps the (now pruned) @@ -375,17 +364,13 @@ describe('e2e_epochs/epochs_optimistic_proving', () => { describe('mid-epoch checkpoint reorg moving a tx', () => { beforeEach(async () => { - test = await EpochsTestContext.setup({ + test = await SingleNodeTestContext.setup({ + ...FAST_REORG_TIMING, numberOfAccounts: 1, maxSpeedUpAttempts: 0, cancelTxOnTimeout: false, - aztecEpochDuration: 4, - ethereumSlotDuration: 4, - aztecSlotDuration: 36, - blockDurationMs: 8000, minTxsPerBlock: 0, aztecProofSubmissionEpochs: 1000, - anvilSlotsInAnEpoch: 32, }); ({ rollup, logger, context } = test); ({ L2_SLOT_DURATION_IN_S } = test); @@ -424,12 +409,10 @@ describe('e2e_epochs/epochs_optimistic_proving', () => { await context.cheatCodes.eth.reorgWithReplacement(reorgDepth); // The node detects the prune and drops back below the reorged-out checkpoint. - await retryUntil( - () => getCheckpointNumber(node).then(cpNum => cpNum < originalCheckpoint), - 'node detects reorg', - 60, - 0.5, - ); + await waitForNodeCheckpoint(node, originalCheckpoint, { + compare: (actual, target) => actual < target, + timeout: 60, + }); logger.info(`Node observed the reorg removing checkpoint ${originalCheckpoint}`); // The tx returns to the mempool and is remined into a fresh checkpoint. Poll for a @@ -482,16 +465,12 @@ describe('e2e_epochs/epochs_optimistic_proving', () => { describe('mid-epoch checkpoint reorg without replacement', () => { beforeEach(async () => { - test = await EpochsTestContext.setup({ + test = await SingleNodeTestContext.setup({ + ...FAST_REORG_TIMING, maxSpeedUpAttempts: 0, cancelTxOnTimeout: false, - aztecEpochDuration: 4, - ethereumSlotDuration: 4, - aztecSlotDuration: 36, - blockDurationMs: 8000, minTxsPerBlock: 0, aztecProofSubmissionEpochs: 1000, - anvilSlotsInAnEpoch: 32, }); ({ rollup, logger, context } = test); ({ L2_SLOT_DURATION_IN_S } = test); @@ -534,12 +513,10 @@ describe('e2e_epochs/epochs_optimistic_proving', () => { logger.info(`After reorg: checkpoint ${afterReorgCheckpoint} (was ${checkpointBeforeReorg})`); // Verify node detects the reorg. - await retryUntil( - () => getCheckpointNumber(node).then(cp => cp <= afterReorgCheckpoint), - 'reorg detected', - 30, - 0.5, - ); + await waitForNodeCheckpoint(node, afterReorgCheckpoint, { + compare: (actual, target) => actual <= target, + timeout: 30, + }); // The survivor must still be in the epoch we reorged within — otherwise the reorg removed // the only in-epoch checkpoint and the test isn't exercising mid-epoch removal. @@ -566,17 +543,12 @@ describe('e2e_epochs/epochs_optimistic_proving', () => { describe('last-slot checkpoint reorg without replacement', () => { beforeEach(async () => { - test = await EpochsTestContext.setup({ + test = await SingleNodeTestContext.setup({ + ...FAST_REORG_TIMING, maxSpeedUpAttempts: 0, cancelTxOnTimeout: false, - aztecEpochDuration: 4, - ethereumSlotDuration: 4, - aztecSlotDuration: 36, - blockDurationMs: 8000, minTxsPerBlock: 0, aztecProofSubmissionEpochs: 1000, - anvilSlotsInAnEpoch: 32, - inboxLag: 2, // Apply a delay between "epoch complete on L1" and the prover-node hand-off so // the reorg below has time to be processed before finalization starts. proverNodeConfig: { proverNodeEpochProvingDelayMs: 10_000 }, @@ -633,12 +605,10 @@ describe('e2e_epochs/epochs_optimistic_proving', () => { expect(survivor.header.slotNumber).toBeLessThan(epochEndSlot); // Verify node detects the reorg. - await retryUntil( - () => getCheckpointNumber(node).then(cp => cp <= afterReorgCheckpoint), - 'reorg detected', - 30, - 0.5, - ); + await waitForNodeCheckpoint(node, afterReorgCheckpoint, { + compare: (actual, target) => actual <= target, + timeout: 30, + }); // Wait for the next epoch to start, then for proof to land with the surviving checkpoints. await test.waitUntilEpochStarts(epoch + 1); @@ -653,16 +623,12 @@ describe('e2e_epochs/epochs_optimistic_proving', () => { describe('checkpoint reorg during proving', () => { beforeEach(async () => { - test = await EpochsTestContext.setup({ + test = await SingleNodeTestContext.setup({ + ...FAST_REORG_TIMING, maxSpeedUpAttempts: 0, cancelTxOnTimeout: false, - aztecEpochDuration: 4, - ethereumSlotDuration: 4, - aztecSlotDuration: 36, - blockDurationMs: 8000, minTxsPerBlock: 0, aztecProofSubmissionEpochs: 1000, - anvilSlotsInAnEpoch: 32, }); ({ rollup, logger, context } = test); ({ L2_SLOT_DURATION_IN_S } = test); @@ -671,60 +637,64 @@ describe('e2e_epochs/epochs_optimistic_proving', () => { it('handles a reorg arriving while the top of the epoch is proving', async () => { // Gate top-tree proving so it deterministically blocks until we release it. This - // gives us a window where the session is parked at the top-tree boundary (all - // sub-trees proven, root prove not yet started), and we can fire the reorg + // gives us a window where the session is mid-proof, and we can fire the reorg // precisely during that window. We use the session's `beforeTopTreeProve` hook // rather than monkey-patching the orchestrator factory. const proverNode = test.proverNodes[0].getProverNode() as TestProverNode; - const provingGate = promiseWithResolvers(); - // Resolves with the gated epoch once a session is actually parked inside the gate. - // Firing the reorg only after this settles guarantees the session is blocked at the - // top-tree boundary — not racing real proving — so the prune deterministically takes - // the cancel-and-recreate path instead of failing a half-run sub-tree. - const gateEntered = promiseWithResolvers(); - // The hook fires from inside `beforeProve`, by which point the session has already - // transitioned from `awaiting-checkpoints` to `awaiting-root`; query that state to - // find the gateable session. Only gate sessions with at least 2 checkpoints — - // reorging the last checkpoint of a single-checkpoint epoch leaves nothing to prove, - // the session is cancelled without replacement, and the test's "wait for fewer - // checkpoints" check never converges. Sessions with one checkpoint just pass through. - const findGateableEpoch = () => { - const job = proverNode.sessionManager.getJobs().find(j => j.status === 'awaiting-root'); - const session = job && proverNode.sessionManager.getFullSession(job.epochNumber); - return session && session.getCheckpoints().length >= 2 ? job!.epochNumber : undefined; - }; + let releaseProvingGate: () => void = () => {}; + const provingGate = new Promise(resolve => { + releaseProvingGate = resolve; + }); + + // Capture the session the hook actually gates so the test reorgs the right epoch. + // The `beforeTopTreeProve` hook takes no session argument, so we identify the calling + // session by state: `EpochSession.beforeProve` flips the state to `awaiting-root` + // *before* awaiting this hook (see epoch-session.ts), so the gating session is the + // live full session sitting in `awaiting-root`. Matching on `awaiting-checkpoints` + // never fires — that state is already gone by the time the hook runs. + let gatedSession: ReturnType[number] | undefined; + // Only gate sessions with at least 2 checkpoints — reorging the last checkpoint + // of a single-checkpoint epoch leaves nothing to prove, the session is cancelled + // without replacement, and the test's "wait for fewer checkpoints" check never + // converges. Sessions with one checkpoint just pass through. + const findGateableSession = () => + proverNode.sessionManager + .allSessions() + .find(s => s.getKind() === 'full' && s.getState() === 'awaiting-root' && s.getCheckpoints().length >= 2); proverNode.setSessionHooks({ beforeTopTreeProve: async () => { - const epoch = findGateableEpoch(); - if (epoch === undefined) { + const session = findGateableSession(); + if (!session) { return; } - logger.warn(`Top-tree proving gated for epoch ${epoch} — waiting for test to release`); - gateEntered.resolve(epoch); - await provingGate.promise; - logger.warn('Proving gate released'); + // First gateable session to hit the gate is the one we reorg; later recreated + // sessions (over the surviving prefix) also reach this hook but the gate is + // already resolved by then, so they sail through after release. + gatedSession ??= session; + logger.warn('Top-tree proving gated — waiting for test to release', { epoch: session.getEpochNumber() }); + await provingGate; + logger.warn('Proving gate released', { epoch: session.getEpochNumber() }); }, }); - // Wait until a session with at least 2 checkpoints is actually parked inside the - // gate. The session manager opens one full session at a time, starting with the - // lowest unproven epoch; small epochs pass through (see hook above) and we keep - // waiting until a gateable epoch lands. - const gatedEpoch = await executeTimeout( - () => gateEntered.promise, - L2_SLOT_DURATION_IN_S * 12 * 1000, + // Wait for a gateable session (>= 2 checkpoints) to actually block at the gate. The + // session manager opens one full session at a time, starting with the lowest unproven + // epoch; small epochs pass through (see hook above) and we keep proving until a + // gateable epoch lands and parks itself on `provingGate`. The hook records the session + // the moment it blocks, so polling `gatedSession` tells us the gate is engaged. + const inFlightSession = await retryUntil( + () => Promise.resolve(gatedSession), 'gateable session blocks at proving gate', + L2_SLOT_DURATION_IN_S * 12, + 0.5, ); - logger.info(`Job for epoch ${gatedEpoch} is blocked inside proving — firing reorg now`); - - // Capture the in-flight session and the last checkpoint of the gated epoch — - // we'll reorg that checkpoint out and verify the prover recovers with the - // surviving prefix. We take the session's own checkpoint list rather than - // `monitor.checkpointNumber` because the global high may sit in a later epoch. - const inFlightSession = proverNode.sessionManager.getFullSession(gatedEpoch); - if (!inFlightSession) { - throw new Error(`No in-flight session for epoch ${gatedEpoch}`); - } + const gatedEpoch = inFlightSession.getEpochNumber(); + logger.info(`Session for epoch ${gatedEpoch} is blocked inside proving — firing reorg now`); + + // The gated session's own checkpoint list gives us the last checkpoint of the gated + // epoch — we'll reorg that checkpoint out and verify the prover recovers with the + // surviving prefix. We take the session's own list rather than `monitor.checkpointNumber` + // because the global high may sit in a later epoch. const trackedBeforeReorg = inFlightSession.getCheckpoints().length; const epochEndCheckpoint = inFlightSession.getCheckpoints()[trackedBeforeReorg - 1].checkpoint.number; logger.info(`Reorging last checkpoint ${epochEndCheckpoint} of gated epoch ${gatedEpoch}`); @@ -777,7 +747,7 @@ describe('e2e_epochs/epochs_optimistic_proving', () => { // Release the gate. The cancelled top tree #1 short-circuits with // TopTreeCancelledError, the finalize loop restarts with the surviving sub-trees, // and a fresh top tree submits a valid proof for checkpoints 1..afterReorgCheckpoint. - provingGate.resolve(); + releaseProvingGate(); // The in-flight epoch should now be proven on L1 await test.waitUntilProvenCheckpointNumber(afterReorgCheckpoint, 240); @@ -788,18 +758,14 @@ describe('e2e_epochs/epochs_optimistic_proving', () => { describe('prover-node starts mid-epoch', () => { beforeEach(async () => { - test = await EpochsTestContext.setup({ + test = await SingleNodeTestContext.setup({ + ...FAST_REORG_TIMING, // Don't start the prover-node automatically — we spin it up mid-epoch in the test. startProverNode: false, maxSpeedUpAttempts: 0, cancelTxOnTimeout: false, - aztecEpochDuration: 4, - ethereumSlotDuration: 4, - aztecSlotDuration: 36, - blockDurationMs: 8000, minTxsPerBlock: 0, aztecProofSubmissionEpochs: 1000, - anvilSlotsInAnEpoch: 32, }); ({ rollup, logger, context } = test); ({ L2_SLOT_DURATION_IN_S } = test); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_proof_fails.parallel.test.ts b/yarn-project/end-to-end/src/single-node/proving/proof_fails.parallel.test.ts similarity index 96% rename from yarn-project/end-to-end/src/e2e_epochs/epochs_proof_fails.parallel.test.ts rename to yarn-project/end-to-end/src/single-node/proving/proof_fails.parallel.test.ts index 1cb820ae5f63..bbe10bdd95d2 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_proof_fails.parallel.test.ts +++ b/yarn-project/end-to-end/src/single-node/proving/proof_fails.parallel.test.ts @@ -17,17 +17,17 @@ import { RootRollupPublicInputs } from '@aztec/stdlib/rollup'; import { jest } from '@jest/globals'; -import type { EndToEndContext } from '../fixtures/utils.js'; -import { EpochsTestContext } from './epochs_test.js'; +import type { EndToEndContext } from '../../fixtures/utils.js'; +import { SingleNodeTestContext } from '../single_node_test_context.js'; jest.setTimeout(1000 * 60 * 10); -// Suite: 2 parallel scenarios testing proof-submission failure paths. EpochsTestContext with single +// Suite: 2 parallel scenarios testing proof-submission failure paths. SingleNodeTestContext with single // sequencer node, no initial prover (prover nodes created in test bodies). Timing: ethSlot=8s, // aztecSlot=2×8=16s, epoch=8, proofSubmissionEpochs=1 (default), blockDurationMs=3s, // cancelTxOnTimeout=false, inboxLag=2 (v5 always enforces the timetable, so the former enforceTimeTable // override is gone). Prover Delayer steers proof tx timing. -describe('e2e_epochs/epochs_proof_fails', () => { +describe('single-node/proving/proof_fails', () => { let context: EndToEndContext; let l1Client: ViemClient; let rollup: RollupContract; @@ -38,10 +38,10 @@ describe('e2e_epochs/epochs_proof_fails', () => { let L2_SLOT_DURATION_IN_S: number; - let test: EpochsTestContext; + let test: SingleNodeTestContext; beforeEach(async () => { - test = await EpochsTestContext.setup({ + test = await SingleNodeTestContext.setup({ maxSpeedUpAttempts: 0, // No speed ups startProverNode: false, // Avoid early proving ethereumSlotDuration: 8, @@ -49,7 +49,6 @@ describe('e2e_epochs/epochs_proof_fails', () => { aztecSlotDurationInL1Slots: 2, blockDurationMs: 3000, // 3s blocks → 2 blocks per checkpoint under pipelining cancelTxOnTimeout: false, - inboxLag: 2, }); ({ context, l1Client, rollup, constants, logger, monitor } = test); ({ L2_SLOT_DURATION_IN_S } = test); diff --git a/yarn-project/end-to-end/src/single-node/proving/setup.ts b/yarn-project/end-to-end/src/single-node/proving/setup.ts new file mode 100644 index 000000000000..9ac299a9280a --- /dev/null +++ b/yarn-project/end-to-end/src/single-node/proving/setup.ts @@ -0,0 +1,7 @@ +import { jest } from '@jest/globals'; + +import { SingleNodeTestContext, WORLD_STATE_CHECKPOINT_HISTORY } from '../single_node_test_context.js'; + +jest.setTimeout(1000 * 60 * 15); + +export { jest, SingleNodeTestContext, WORLD_STATE_CHECKPOINT_HISTORY }; diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_upload_failed_proof.test.ts b/yarn-project/end-to-end/src/single-node/proving/upload_failed_proof.test.ts similarity index 89% rename from yarn-project/end-to-end/src/e2e_epochs/epochs_upload_failed_proof.test.ts rename to yarn-project/end-to-end/src/single-node/proving/upload_failed_proof.test.ts index 865ddc658179..6596286bdc97 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_upload_failed_proof.test.ts +++ b/yarn-project/end-to-end/src/single-node/proving/upload_failed_proof.test.ts @@ -12,19 +12,19 @@ import { mkdtemp } from 'fs/promises'; import { tmpdir } from 'os'; import { join } from 'path'; -import { getACVMConfig } from '../fixtures/get_acvm_config.js'; -import { getBBConfig } from '../fixtures/get_bb_config.js'; -import type { EndToEndContext } from '../fixtures/utils.js'; -import { EpochsTestContext } from './epochs_test.js'; +import { getACVMConfig } from '../../fixtures/get_acvm_config.js'; +import { getBBConfig } from '../../fixtures/get_bb_config.js'; +import type { EndToEndContext } from '../../fixtures/utils.js'; +import { SingleNodeTestContext } from '../single_node_test_context.js'; jest.setTimeout(1000 * 60 * 10); // Suite: verifies that a failed epoch-proving job uploads its state to a file store and that // rerunEpochProvingJob can re-prove from the downloaded data on a fresh instance. Uses -// EpochsTestContext with a prover configured to use a temp file:// URL as the epoch failure store. +// SingleNodeTestContext with a prover configured to use a temp file:// URL as the epoch failure store. // Timing: all defaults (ethSlot=8s/12s CI, aztecSlot=16s/24s, epoch=6, proofSubmissionEpochs=1, // fake prover). The test tears down mid-run and re-proves via a standalone helper. -describe('e2e_epochs/epochs_upload_failed_proof', () => { +describe('single-node/proving/upload_failed_proof', () => { let context: EndToEndContext; let logger: Logger; let config: AztecNodeConfig; @@ -34,7 +34,7 @@ describe('e2e_epochs/epochs_upload_failed_proof', () => { let rerunDataDir: string; let rerunDownloadDir: string; - let test: EpochsTestContext; + let test: SingleNodeTestContext; beforeEach(async () => { rerunDataDir = await mkdtemp(join(tmpdir(), 'rerun-data-')); @@ -42,7 +42,7 @@ describe('e2e_epochs/epochs_upload_failed_proof', () => { uploadPath = await mkdtemp(join(tmpdir(), 'failed-proofs-')); uploadUrl = `file://${uploadPath}`; - test = await EpochsTestContext.setup({ + test = await SingleNodeTestContext.setup({ proverNodeConfig: { proverNodeFailedEpochStore: uploadUrl }, }); ({ context, logger } = test); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_multiple.test.ts b/yarn-project/end-to-end/src/single-node/proving/world_state_pruning.test.ts similarity index 81% rename from yarn-project/end-to-end/src/e2e_epochs/epochs_multiple.test.ts rename to yarn-project/end-to-end/src/single-node/proving/world_state_pruning.test.ts index 731e0bf95d2e..25728a784846 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_multiple.test.ts +++ b/yarn-project/end-to-end/src/single-node/proving/world_state_pruning.test.ts @@ -2,25 +2,20 @@ import type { Logger } from '@aztec/aztec.js/log'; import { RollupContract } from '@aztec/ethereum/contracts'; import { BlockNumber } from '@aztec/foundation/branded-types'; -import { jest } from '@jest/globals'; +import { SingleNodeTestContext, WORLD_STATE_CHECKPOINT_HISTORY, jest } from './setup.js'; -import { EpochsTestContext, WORLD_STATE_CHECKPOINT_HISTORY } from './epochs_test.js'; - -jest.setTimeout(1000 * 60 * 15); - -// Suite: verifies that multiple consecutive epochs are proven successfully and that world-state -// checkpoints are pruned after finalization. Uses EpochsTestContext defaults: single node, -// prod-seq, interval mining, ethSlot=8s (12s CI), aztecSlot=16s (24s CI), epoch=6, -// proofSubmissionEpochs=1, fake prover. TARGET_PROVEN_EPOCHS env var controls iteration count. -// Assumes one block per checkpoint -describe('e2e_epochs/epochs_multiple', () => { +// Verifies that multiple consecutive epochs are proven successfully and that world-state checkpoints +// are pruned after finalization. SingleNodeTestContext defaults: single node, prod-seq, interval +// mining, ethSlot=8s (12s CI), aztecSlot=16s (24s CI), epoch=6, proofSubmissionEpochs=1, fake prover. +// TARGET_PROVEN_EPOCHS env var controls iteration count. Assumes one block per checkpoint. +describe('single-node/proving/world_state_pruning', () => { let rollup: RollupContract; let logger: Logger; - let test: EpochsTestContext; + let test: SingleNodeTestContext; beforeEach(async () => { - test = await EpochsTestContext.setup({}); + test = await SingleNodeTestContext.setup({}); ({ rollup, logger } = test); }); diff --git a/yarn-project/end-to-end/src/single-node/recovery/manual_rollback.test.ts b/yarn-project/end-to-end/src/single-node/recovery/manual_rollback.test.ts new file mode 100644 index 000000000000..306b3b917af9 --- /dev/null +++ b/yarn-project/end-to-end/src/single-node/recovery/manual_rollback.test.ts @@ -0,0 +1,59 @@ +import type { Logger } from '@aztec/aztec.js/log'; +import type { AztecNode } from '@aztec/aztec.js/node'; +import type { RollupContract } from '@aztec/ethereum/contracts'; +import { CheckpointNumber } from '@aztec/foundation/branded-types'; + +import type { EndToEndContext } from '../../fixtures/utils.js'; +import { waitForBlockNumber } from '../../fixtures/wait_helpers.js'; +import { SingleNodeTestContext, jest } from './setup.js'; + +// Exercises the aztecNodeAdmin.rollbackTo() API. Default SingleNodeTestContext with a very long epoch +// (aztecEpochDuration=100) so there are no L2 reorgs, no finalized blocks, and the full pending chain +// is prunable. Actively drives L1 via cheatcodes (reorgTo to remove blocks). +describe('single-node/recovery/manual_rollback', () => { + let context: EndToEndContext; + let logger: Logger; + let node: AztecNode; + let rollup: RollupContract; + + let test: SingleNodeTestContext; + + beforeEach(async () => { + test = await SingleNodeTestContext.setup({ aztecEpochDuration: 100 }); // No L2 reorgs, no finalized blocks + ({ context, logger, rollup } = test); + ({ aztecNode: node } = context); + }); + + afterEach(async () => { + jest.restoreAllMocks(); + await test.teardown(); + }); + + // Waits for checkpoint 4, pauses node sync, reorgs L1 by 2 blocks, calls rollbackTo on the + // node, and asserts blockNumber equals the rolled-back value. Resumes sync and verifies the + // node re-syncs to the same block. + it('manually rolls back to an unfinalized block', async () => { + logger.info(`Starting manual rollback test to unfinalized block`); + context.sequencer?.updateConfig({ minTxsPerBlock: 0 }); + const targetCheckpointNumber = CheckpointNumber(4); + // With pipelining, each checkpoint takes ~2 L2 slots on a solo-sequencer setup. + await test.waitUntilCheckpointNumber(targetCheckpointNumber, test.L2_SLOT_DURATION_IN_S * 12); + await waitForBlockNumber(node, 4, { timeout: 10 }); + + logger.info(`Synced to checkpoint 4. Pausing syncing and rolling back the chain.`); + await context.aztecNodeAdmin.pauseSync(); + context.sequencer?.updateConfig({ minTxsPerBlock: 100 }); // Ensure no new blocks are produced + await context.cheatCodes.eth.reorg(2); + const checkpointAfterReorg = await rollup.getCheckpointNumber(); + expect(checkpointAfterReorg).toBeLessThan(targetCheckpointNumber); + logger.info(`Rolled back to checkpoint ${checkpointAfterReorg}.`); + + logger.info(`Manually rolling back node to ${checkpointAfterReorg - 1}.`); + const blockAfterReorg = Number(checkpointAfterReorg - 1); + await context.aztecNodeAdmin.rollbackTo(blockAfterReorg); + expect(await node.getBlockNumber()).toEqual(blockAfterReorg); + + logger.info(`Waiting for node to re-sync to ${blockAfterReorg}.`); + await waitForBlockNumber(node, blockAfterReorg, { timeout: 10 }); + }); +}); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_prune_when_cannot_build.test.ts b/yarn-project/end-to-end/src/single-node/recovery/prune_when_cannot_build.test.ts similarity index 92% rename from yarn-project/end-to-end/src/e2e_epochs/epochs_prune_when_cannot_build.test.ts rename to yarn-project/end-to-end/src/single-node/recovery/prune_when_cannot_build.test.ts index c9c03a4ce2de..9907f305646e 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_prune_when_cannot_build.test.ts +++ b/yarn-project/end-to-end/src/single-node/recovery/prune_when_cannot_build.test.ts @@ -6,19 +6,19 @@ import { retryUntil } from '@aztec/foundation/retry'; import { jest } from '@jest/globals'; -import type { EndToEndContext } from '../fixtures/utils.js'; -import { EpochsTestContext } from './epochs_test.js'; +import type { EndToEndContext } from '../../fixtures/utils.js'; +import { SingleNodeTestContext } from '../single_node_test_context.js'; jest.setTimeout(1000 * 60 * 10); -// Single-sequencer suite for the failed-sync prune fallback (A-1260). The proposer cannot build while +// Single-sequencer suite for the failed-sync prune fallback. The proposer cannot build while // its sync is paused, so the only way the pending chain can be wound back to proven is the // `Sequencer.tryVoteAndPruneWhenCannotBuild` path. With no prover node, epoch 0 never proves, so once // its proof-submission window closes the chain becomes prunable and the proposer's fallback must call // `prune()` despite being unable to propose. // // Timing: ethSlot=8s, aztecSlot=2×8=16s, epoch=8, proofSubmissionEpochs=1. -describe('e2e_epochs/epochs_prune_when_cannot_build', () => { +describe('single-node/recovery/prune_when_cannot_build', () => { let context: EndToEndContext; let logger: Logger; let rollup: RollupContract; @@ -26,10 +26,10 @@ describe('e2e_epochs/epochs_prune_when_cannot_build', () => { let L2_SLOT_DURATION_IN_S: number; - let test: EpochsTestContext; + let test: SingleNodeTestContext; beforeEach(async () => { - test = await EpochsTestContext.setup({ + test = await SingleNodeTestContext.setup({ startProverNode: false, // Nothing ever proves epoch 0, so its pending chain stays unproven and becomes prunable. ethereumSlotDuration: 8, aztecEpochDuration: 8, // Long enough to land a few checkpoints in epoch 0. diff --git a/yarn-project/end-to-end/src/single-node/recovery/setup.ts b/yarn-project/end-to-end/src/single-node/recovery/setup.ts new file mode 100644 index 000000000000..de507584c2d5 --- /dev/null +++ b/yarn-project/end-to-end/src/single-node/recovery/setup.ts @@ -0,0 +1,7 @@ +import { jest } from '@jest/globals'; + +import { SingleNodeTestContext } from '../single_node_test_context.js'; + +jest.setTimeout(1000 * 60 * 10); + +export { jest, SingleNodeTestContext }; diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_sync_after_reorg.test.ts b/yarn-project/end-to-end/src/single-node/recovery/sync_after_reorg.test.ts similarity index 70% rename from yarn-project/end-to-end/src/e2e_epochs/epochs_sync_after_reorg.test.ts rename to yarn-project/end-to-end/src/single-node/recovery/sync_after_reorg.test.ts index 42386221e4b7..1c5e2c81eff3 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_sync_after_reorg.test.ts +++ b/yarn-project/end-to-end/src/single-node/recovery/sync_after_reorg.test.ts @@ -3,28 +3,24 @@ import type { Logger } from '@aztec/aztec.js/log'; import { CheckpointNumber } from '@aztec/foundation/branded-types'; import { executeTimeout } from '@aztec/foundation/timer'; -import { jest } from '@jest/globals'; - -import type { EndToEndContext } from '../fixtures/utils.js'; -import { EpochsTestContext } from './epochs_test.js'; - -jest.setTimeout(1000 * 60 * 10); - -// Suite: regression test ensuring a new node can sync world-state after an unpruned reorg -// (issue #12206). EpochsTestContext with single node, no prover, prod-seq, interval mining. -// Timing: all defaults (ethSlot=8s/12s CI, aztecSlot=16s/24s, epoch=6, proofSubmissionEpochs=1). -// The test stops the sequencer mid-run, advances into epoch 2 via waitUntilEpochStarts, then -// creates a second node and verifies it syncs cleanly despite the reorg window. -describe('e2e_epochs/epochs_sync_after_reorg', () => { +import type { EndToEndContext } from '../../fixtures/utils.js'; +import { SingleNodeTestContext, jest } from './setup.js'; + +// Regression test ensuring a new node can sync world-state after an unpruned reorg (issue #12206). +// SingleNodeTestContext with single node, no prover, prod-seq, interval mining. Timing: all defaults +// (ethSlot=8s/12s CI, aztecSlot=16s/24s, epoch=6, proofSubmissionEpochs=1). The test stops the +// sequencer mid-run, advances into epoch 2 via waitUntilEpochStarts, then creates a second node and +// verifies it syncs cleanly despite the reorg window. +describe('single-node/recovery/sync_after_reorg', () => { let context: EndToEndContext; let logger: Logger; let L2_SLOT_DURATION_IN_S: number; - let test: EpochsTestContext; + let test: SingleNodeTestContext; beforeEach(async () => { - test = await EpochsTestContext.setup({ startProverNode: false }); // no prover! + test = await SingleNodeTestContext.setup({ startProverNode: false }); // no prover! ({ context, logger } = test); ({ L2_SLOT_DURATION_IN_S } = test); }); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_test.ts b/yarn-project/end-to-end/src/single-node/single_node_test_context.ts similarity index 60% rename from yarn-project/end-to-end/src/e2e_epochs/epochs_test.ts rename to yarn-project/end-to-end/src/single-node/single_node_test_context.ts index 710508dcd3da..9d9dea248e85 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_test.ts +++ b/yarn-project/end-to-end/src/single-node/single_node_test_context.ts @@ -16,20 +16,25 @@ import { RollupContract } from '@aztec/ethereum/contracts'; import { Delayer, createDelayer, waitUntilL1Timestamp, wrapClientWithDelayer } from '@aztec/ethereum/l1-tx-utils'; import { ChainMonitor } from '@aztec/ethereum/test'; import type { ExtendedViemWalletClient } from '@aztec/ethereum/types'; -import { BlockNumber, CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types'; +import { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; import { SecretValue } from '@aztec/foundation/config'; import { randomBytes } from '@aztec/foundation/crypto/random'; import { withLoggerBindings } from '@aztec/foundation/log/server'; import { retryUntil } from '@aztec/foundation/retry'; import { sleep } from '@aztec/foundation/sleep'; +import { executeTimeout } from '@aztec/foundation/timer'; import { SpamContract } from '@aztec/noir-test-contracts.js/Spam'; import { TestContract } from '@aztec/noir-test-contracts.js/Test'; import { getMockPubSubP2PServiceFactory } from '@aztec/p2p/test-helpers'; import type { ProverNodeConfig } from '@aztec/prover-node'; import type { PXEConfig } from '@aztec/pxe/config'; -import { type SequencerClient, type SequencerEvents, SequencerState } from '@aztec/sequencer-client'; +import { type Sequencer, type SequencerClient, type SequencerEvents, SequencerState } from '@aztec/sequencer-client'; import { type BlockParameter, EthAddress } from '@aztec/stdlib/block'; -import { type L1RollupConstants, getProofSubmissionDeadlineTimestamp } from '@aztec/stdlib/epoch-helpers'; +import { + type L1RollupConstants, + getProofSubmissionDeadlineTimestamp, + getTimestampForSlot, +} from '@aztec/stdlib/epoch-helpers'; import { tryStop } from '@aztec/stdlib/interfaces/server'; import type { SlashingProtectionDatabase } from '@aztec/validator-ha-signer/types'; @@ -55,7 +60,7 @@ export const WORLD_STATE_BLOCK_CHECK_INTERVAL = 50; export const ARCHIVER_POLL_INTERVAL = 50; export const DEFAULT_L1_BLOCK_TIME = process.env.CI ? 12 : 8; -export type EpochsTestOpts = Partial & { +export type SingleNodeTestOpts = Partial & { numberOfAccounts?: number; pxeOpts?: Partial; aztecSlotDurationInL1Slots?: number; @@ -76,12 +81,97 @@ export type TrackedSequencerEvent = { }; }[keyof SequencerEvents]; +/** A `block-proposed` sequencer event captured for the pipelining-offset assertion. */ +export type BlockProposedEvent = { blockNumber: BlockNumber; slot: SlotNumber; buildSlot: SlotNumber }; + +/** + * The 36s-slot reorg cadence shared by every reorg/prune/HA test, regardless of single-node vs + * multi-validator topology: a 36s L2 slot, 8s blocks, and a 4-slot epoch. The two concrete reorg + * profiles ({@link FAST_REORG_TIMING}, {@link MULTI_VALIDATOR_REORG_TIMING}) extend this with their topology's L1 + * slot duration and any extra knobs. Kept timing-only — `maxSpeedUpAttempts`, `cancelTxOnTimeout`, and + * `aztecProofSubmissionEpochs` encode per-test scenario intent and stay explicit at the call site. + */ +export const REORG_TIMING_BASE = { + aztecSlotDuration: 36, + blockDurationMs: 8000, + aztecEpochDuration: 4, +} as const; + +/** + * Timing-only profile shared by the fast single-node L1-reorg tests (`proving/optimistic`'s reorg + * cases and `l1-reorgs/`). Extends {@link REORG_TIMING_BASE} with mainnet-style 32-slot anvil epochs + * and a 4s L1 slot. Note: `ethereumSlotDuration` stays at 4 here (not unified to + * {@link MULTI_VALIDATOR_REORG_TIMING}'s 6) — at eth=6 the proof-submission-window timing in the + * proof-removal/proof-restore reorg assertions in `l1-reorgs/blocks` starves and times out, so the 4s + * L1 slot is required for the single-node reorg path. Tests that need a different epoch length (e.g. 8 + * for the "with replacement" case) override `aztecEpochDuration` after the spread. + */ +export const FAST_REORG_TIMING = { + ...REORG_TIMING_BASE, + ethereumSlotDuration: 4, + anvilSlotsInAnEpoch: 32, +} as const; + +/** + * Timing-only profile naming the 36s/6s reorg-and-prune cadence copied verbatim across the + * multi-validator recovery and high-availability tests (`recovery/proposal_failure_recovery`, + * `recovery/equivocation_recovery`, `high-availability/ha_sync`, + * `high-availability/ha_checkpoint_handoff`). The multi-validator analogue of + * {@link FAST_REORG_TIMING}, adding the 0.5s attestation-propagation budget those committee tests + * need. Timing-only: committee size, `aztecProofSubmissionEpochs`, and the slasher block stay + * per-test. Spread BEFORE per-test overrides so a test can still bump e.g. `aztecEpochDuration`. + */ +export const MULTI_VALIDATOR_REORG_TIMING = { + ...REORG_TIMING_BASE, + ethereumSlotDuration: 6, + attestationPropagationTime: 0.5, +} as const; + /** - * Tests building of epochs using fast block times and short epochs. - * Spawns an aztec node and a prover node with fake proofs. - * Sequencer is allowed to build empty blocks. + * Timing-only profile naming the 36s/12s multi-validator block-production cadence copied across + * `block-production/` (`simple`, `high_tps`, `first_slot`, and `proof_boundary`). Uses + * `aztecSlotDurationInL1Slots: 3` rather than an explicit `aztecSlotDuration: 36` so the L2 slot stays + * coupled to `ethereumSlotDuration` if a test overrides eth. Deliberately omits + * `attestationPropagationTime` (per-scenario: default 2, 0.5, or 1) — set it per test. Spread BEFORE + * per-test overrides. */ -export class EpochsTestContext { +export const MULTI_VALIDATOR_BLOCK_PRODUCTION_TIMING = { + ethereumSlotDuration: 12, + aztecSlotDurationInL1Slots: 3, + blockDurationMs: 6000, +} as const; + +/** + * Timing-only profile naming the 72s wide-slot multiple-blocks-per-slot cadence copied across the + * block-production and recovery tests (`block-production/`'s `setupBlockProductionWithProver`, `block-production/blob_promotion`, + * `recovery/pipeline_prune`). A-914: + * pipelined multiple-blocks-per-slot needs this 72s/12s cadence (not the tighter 36s/4s), otherwise non-proposer nodes hit + * `CheckpointNumberNotSequentialError` when the pipelined proposer races ahead of L1 confirmation. The + * larger `perBlockAllocationMultiplier` lets each of the several blocks per slot fit non-trivial txs. + * Spread BEFORE per-test overrides (e.g. `mockGossipSubNetworkLatency`, `maxTxsPerCheckpoint`). + */ +export const WIDE_SLOT_TIMING = { + ethereumSlotDuration: 12, + aztecSlotDuration: 72, + blockDurationMs: 5500, + aztecEpochDuration: 4, + perBlockAllocationMultiplier: 8, + aztecTargetCommitteeSize: 3, +} as const; + +/** + * Base class for the prod-sequencer single-node test topology: one node running the production + * sequencer with fast block times and short epochs, an optional fake-proof prover node, and the + * environment it runs in (in-proc anvil + L1 deploy). Owns node spawning + * (`createNonValidatorNode` / `createProverNode` incl. the mock-gossip `p2pServiceFactory` wiring, + * which is harmless with a single node), the `ChainMonitor`, and the epoch / proof-window / reorg + * waiters and assertion helpers shared by every test in the category. + * + * {@link MultiNodeTestContext} extends this with the N-validator topology (validator-node spawning, + * committee/proposal/attestation convergence helpers). Single-node-topology tests use this base + * directly from the sibling `single-node/` category. + */ +export class SingleNodeTestContext { public context!: EndToEndContext; public l1Client!: ExtendedViemWalletClient; public rollup!: RollupContract; @@ -100,13 +190,13 @@ export class EpochsTestContext { public L1_BLOCK_TIME_IN_S!: number; public L2_SLOT_DURATION_IN_S!: number; - public static async setup(opts: EpochsTestOpts = {}) { - const test = new EpochsTestContext(); + public static async setup(this: new () => T, opts: SingleNodeTestOpts = {}) { + const test = new this(); await test.setup(opts); return test; } - public static getSlotDurations(opts: EpochsTestOpts = {}) { + public static getSlotDurations(opts: SingleNodeTestOpts = {}) { const envEthereumSlotDuration = process.env.L1_BLOCK_TIME ? parseInt(process.env.L1_BLOCK_TIME) : DEFAULT_L1_BLOCK_TIME; @@ -122,9 +212,9 @@ export class EpochsTestContext { }; } - public async setup(opts: EpochsTestOpts = {}) { + public async setup(opts: SingleNodeTestOpts = {}) { const { ethereumSlotDuration, aztecSlotDuration, aztecEpochDuration, aztecProofSubmissionEpochs } = - EpochsTestContext.getSlotDurations(opts); + SingleNodeTestContext.getSlotDurations(opts); this.L1_BLOCK_TIME_IN_S = ethereumSlotDuration; this.L2_SLOT_DURATION_IN_S = aztecSlotDuration; @@ -136,7 +226,7 @@ export class EpochsTestContext { const useHardcodedAccount = (opts.skipInitialSequencer || opts.useHardcodedAccount) && !opts.skipHardcodedAccount; let hardcodedAccountData: InitialAccountData | undefined; if (useHardcodedAccount) { - hardcodedAccountData = await EpochsTestContext.getHardcodedAccountData(Fr.random(), Fr.random()); + hardcodedAccountData = await SingleNodeTestContext.getHardcodedAccountData(Fr.random(), Fr.random()); } // Set up system without any account nor protocol contracts @@ -161,6 +251,9 @@ export class EpochsTestContext { worldStateCheckpointHistory: WORLD_STATE_CHECKPOINT_HISTORY, exitDelaySeconds: DefaultL1ContractsConfig.exitDelaySeconds, slasherEnabled: false, + // `inboxLag: 2` is the intended value when running with pipelining (the production config + // default of 1 is a separate bug). Set before `...opts` so tests can still override. + inboxLag: 2, ...opts, ...(hardcodedAccountData ? { additionallyFundedAccounts: [hardcodedAccountData], numberOfAccounts: 0 } : {}), }, @@ -286,18 +379,7 @@ export class EpochsTestContext { return this.createNode({ ...opts, disableValidator: true }); } - public createValidatorNode( - privateKeys: `0x${string}`[], - opts: Partial & { - dontStartSequencer?: boolean; - slashingProtectionDb?: SlashingProtectionDatabase; - } = {}, - ) { - this.logger.warn('Creating and syncing a validator node...'); - return this.createNode({ ...opts, disableValidator: false, validatorPrivateKeys: new SecretValue(privateKeys) }); - } - - private async createNode( + protected async createNode( opts: Partial & { dontStartSequencer?: boolean; slashingProtectionDb?: SlashingProtectionDatabase; @@ -337,7 +419,7 @@ export class EpochsTestContext { return node; } - private getNextPrivateKey(): Hex { + protected getNextPrivateKey(): Hex { const key = getPrivateKeyFromIndex(this.nodes.length + this.proverNodes.length + 1); return `0x${key!.toString('hex')}`; } @@ -418,6 +500,45 @@ export class EpochsTestContext { await waitUntilL1Timestamp(this.l1Client, oneSlotBefore, undefined, proofSubmissionWindowDuration * 2); } + /** + * Timestamp one L1 block before L2 `slot` begins — the instant a pipelined proposer starts + * building for that slot. `opts.lead` overrides the default one-L1-block lead (e.g. when a test + * needs an extra block-duration margin to guarantee sub-slot 1 is reachable). + */ + public buildWindowTimestampForSlot(slot: SlotNumber, opts: { lead?: number } = {}): bigint { + const lead = opts.lead ?? this.L1_BLOCK_TIME_IN_S; + return getTimestampForSlot(slot, this.constants) - BigInt(lead); + } + + /** + * Warps the L1 clock to the build window of `slot` (one L1 block before the slot begins) with + * `resetBlockInterval`, so a pipelined proposer engages cleanly when its sequencer starts. + * @returns The slot warped to. + */ + public async warpToBuildWindowForSlot(slot: SlotNumber): Promise { + const target = this.buildWindowTimestampForSlot(slot); + this.logger.info(`Warping L1 to build window of slot ${slot}`, { slot, target }); + await this.context.cheatCodes.eth.warp(Number(target), { resetBlockInterval: true }); + return slot; + } + + /** + * Waits (in wall-clock, without warping) until the L1 clock reaches the build window of `slot`. + * For production-sequencer tests that can't warp. `opts.lead` extends the lead beyond the default + * one L1 block; `opts.timeout` bounds the wait (defaults to three L2 slots). + * @returns The slot waited for. + */ + public async waitForBuildWindowForSlot( + slot: SlotNumber, + opts: { lead?: number; timeout?: number } = {}, + ): Promise { + const target = this.buildWindowTimestampForSlot(slot, { lead: opts.lead }); + const timeout = opts.timeout ?? this.L2_SLOT_DURATION_IN_S * 3; + this.logger.info(`Waiting until L1 reaches build window of slot ${slot}`, { slot, target }); + await waitUntilL1Timestamp(this.l1Client, target, undefined, timeout); + return slot; + } + /** Waits for the aztec node to sync to the target block number. */ public async waitForNodeToSync(blockNumber: BlockNumber, type: 'proven' | 'finalized' | 'historic') { const waitTime = ARCHIVER_POLL_INTERVAL + WORLD_STATE_BLOCK_CHECK_INTERVAL; @@ -439,6 +560,30 @@ export class EpochsTestContext { } } + /** + * Waits until every prover node has submitted a proof for `epoch` (of length `epochLength` + * checkpoints) on L1. Polls `rollup.getHasSubmittedProof` for each prover registered in + * {@link proverNodes}. + */ + public async waitForAllProversToSubmit( + epoch: EpochNumber, + epochLength: number, + opts: { timeout?: number } = {}, + ): Promise { + const proverIds = this.proverNodes.map(node => node.getProverNode()!.getProverId()); + await retryUntil( + async () => { + const haveSubmitted = await Promise.all( + proverIds.map(proverId => this.rollup.getHasSubmittedProof(epoch, epochLength, proverId)), + ); + this.logger.info(`Proof submissions: ${haveSubmitted.join(', ')}`); + return haveSubmitted.every(submitted => submitted); + }, + 'Provers have submitted proofs', + opts.timeout ?? 120, + ); + } + /** Registers the SpamContract on the given wallet. */ public async registerSpamContract(wallet: Wallet, salt = Fr.ZERO) { const instance = await getContractInstanceFromInstantiationParams(SpamContract.artifact, { @@ -490,9 +635,52 @@ export class EpochsTestContext { expect(result).toBe(expectedSuccess); } - /** Verifies at least one checkpoint has the target number of blocks (for MBPS validation). */ - public async assertMultipleBlocksPerSlot(targetBlockCount: number) { - const archiver = (this.context.aztecNode as AztecNodeService).getBlockSource() as Archiver; + /** + * Verifies that at least one checkpoint has `targetBlockCount` blocks and that block numbering is + * contiguous within every checkpoint (MBPS validation). + * + * Two optional wait modes (both poll before reading checkpoints): + * - `opts.wait`: waits until some checkpoint reaches `targetBlockCount` blocks (the proposed-tip + * MBPS setup case), and + * - `opts.targetBlock`: waits until the archiver's checkpointed tip reaches that block number + * (the pipeline-prune recovery case). + * + * Reads from `opts.archiver` when given (e.g. a specific validator node's block source); otherwise + * from the initial node's archiver. + * @returns The number of the first checkpoint with at least `targetBlockCount` blocks. + */ + public async assertMultipleBlocksPerSlot( + targetBlockCount: number, + opts: { wait?: boolean; targetBlock?: BlockNumber; archiver?: Archiver; timeout?: number } = {}, + ): Promise { + const archiver = opts.archiver ?? ((this.context.aztecNode as AztecNodeService).getBlockSource() as Archiver); + const waitTimeout = opts.timeout ?? this.L2_SLOT_DURATION_IN_S * 3; + + if (opts.targetBlock !== undefined) { + const targetBlock = opts.targetBlock; + await retryUntil( + async () => { + const checkpointed = await archiver.getBlockNumber({ tag: 'checkpointed' }); + return checkpointed !== undefined && checkpointed >= targetBlock; + }, + `archiver checkpointed block ${targetBlock}`, + 10, + 0.1, + ); + } + + if (opts.wait) { + await retryUntil( + async () => { + const found = await archiver.getCheckpoints({ from: CheckpointNumber(1), limit: 50 }); + return found.some(pc => pc.checkpoint.blocks.length >= targetBlockCount) || undefined; + }, + `checkpoint with at least ${targetBlockCount} blocks`, + waitTimeout, + 0.5, + ); + } + const checkpoints = await archiver.getCheckpoints({ from: CheckpointNumber(1), limit: 50 }); this.logger.warn(`Retrieved ${checkpoints.length} checkpoints from archiver`, { @@ -500,11 +688,13 @@ export class EpochsTestContext { }); let expectedBlockNumber = checkpoints[0].checkpoint.blocks[0].number; - let targetFound = false; + let multiBlockCheckpointNumber: CheckpointNumber | undefined; for (const checkpoint of checkpoints) { const blockCount = checkpoint.checkpoint.blocks.length; - targetFound = targetFound || blockCount >= targetBlockCount; + if (blockCount >= targetBlockCount && multiBlockCheckpointNumber === undefined) { + multiBlockCheckpointNumber = checkpoint.checkpoint.number; + } this.logger.verbose(`Checkpoint ${checkpoint.checkpoint.number} has ${blockCount} blocks`, { checkpoint: checkpoint.checkpoint.getStats(), @@ -519,7 +709,64 @@ export class EpochsTestContext { } } - expect(targetFound).toBe(true); + expect(multiBlockCheckpointNumber).toBeDefined(); + return multiBlockCheckpointNumber!; + } + + /** + * Asserts pipelining by comparing the build slot (from block-proposed events) against the + * submission slot (from block headers). With pipelining, the block is built in slot N but its + * header carries submission slot N+1. Also checks each block's coinbase matches the expected + * proposer for its submission slot. Reads checkpoints from `archiver` (typically a validator + * node's block source). + */ + public async assertProposerPipelining( + archiver: Archiver, + blockProposedEvents: BlockProposedEvent[], + logger: Logger, + ): Promise { + const checkpoints = await archiver.getCheckpoints({ from: CheckpointNumber(1), limit: 50 }); + const allBlocks = checkpoints.flatMap(pc => pc.checkpoint.blocks); + + logger.warn(`assertProposerPipelining: ${allBlocks.length} blocks, ${blockProposedEvents.length} events`, { + blockNumbers: allBlocks.map(b => b.number), + eventBlockNumbers: blockProposedEvents.map(e => e.blockNumber), + }); + + let foundPipelining = false; + + for (const block of allBlocks) { + const headerSlot = block.header.globalVariables.slotNumber; // submission slot (N+1) + const coinbase = block.header.globalVariables.coinbase; + + // Find the block-proposed event for this block (use Number() for safe comparison) + const event = blockProposedEvents.find(e => Number(e.blockNumber) === Number(block.number)); + // if there is no event, then it was probably block number one - which was proposed in setup + if (!event) { + continue; + } + + const buildSlot = event.buildSlot; // build slot (N) + + // Verify the pipelining offset: block built in slot N, submitted in slot N+1 + expect(Number(headerSlot)).toBe(Number(buildSlot) + 1); + foundPipelining = true; + + // Verify coinbase matches the expected proposer for the submission slot + const expectedProposer = await this.rollup.getProposerAt(getTimestampForSlot(headerSlot, this.constants)); + expect(coinbase).toEqual(expectedProposer); + + logger.warn(`Block ${block.number}: buildSlot=${buildSlot}, submissionSlot=${headerSlot}, coinbase=${coinbase}`, { + blockNumber: block.number, + buildSlot, + headerSlot, + coinbase: coinbase.toString(), + expectedProposer: expectedProposer.toString(), + }); + } + + expect(foundPipelining).toBe(true); + logger.warn(`Pipelining assertion passed for ${allBlocks.length} blocks`); } public watchSequencerEvents( @@ -587,6 +834,68 @@ export class EpochsTestContext { return { failEvents, stateChanges }; } + /** + * Resolves with the event args the first time `sequencer` emits `event` with args matching + * `match`. Rejects after `opts.timeout` ms (default 60s). Wraps the + * `executeTimeout(signal => new Promise(...))` one-shot subscription boilerplate, cleaning up + * the listener on both the resolve and the abort paths. + */ + public waitForSequencerEvent( + sequencer: Sequencer, + event: E, + match: (args: Parameters[0]) => boolean = () => true, + opts: { timeout?: number } = {}, + ): Promise[0]> { + const timeout = opts.timeout ?? 60_000; + return executeTimeout( + signal => + new Promise[0]>(resolve => { + const listener = (args: Parameters[0]) => { + if (match(args)) { + sequencer.off(event, listener as SequencerEvents[E]); + resolve(args); + } + }; + signal.addEventListener('abort', () => sequencer.off(event, listener as SequencerEvents[E]), { once: true }); + sequencer.on(event, listener as SequencerEvents[E]); + }), + timeout, + `wait for sequencer event ${String(event)}`, + ); + } + + /** Returns the {@link SequencerClient} of each given node, throwing if any node has no sequencer. */ + public getSequencers(nodes: AztecNodeService[]): SequencerClient[] { + return nodes.map(node => { + const sequencer = node.getSequencer(); + if (!sequencer) { + throw new Error('Node has no sequencer'); + } + return sequencer; + }); + } + + /** Starts the sequencer on each given node in parallel. */ + public async startSequencers(nodes: AztecNodeService[]): Promise { + await Promise.all(this.getSequencers(nodes).map(sequencer => sequencer.start())); + } + + /** + * Resolves once `sequencer` is in `state`, returning immediately if it is already there. Use to + * flush in-flight work (e.g. wait for `IDLE` so pending L1 publishes have been issued) before + * sampling chain state. Builds on {@link waitForSequencerEvent} for the not-yet-there path. + */ + public async waitForSequencerState( + sequencer: Sequencer, + state: SequencerState, + opts: { timeout?: number } = {}, + ): Promise { + if (sequencer.status().state === state) { + return; + } + await this.waitForSequencerEvent(sequencer, 'state-changed', args => args.newState === state, opts); + } + public assertNoFailuresFromSequencers(failEvents: TrackedSequencerEvent[]) { if (failEvents.length > 0) { this.logger.error(`Failed events from sequencers`, failEvents); diff --git a/yarn-project/end-to-end/src/test-wallet/utils.ts b/yarn-project/end-to-end/src/test-wallet/utils.ts index de361a62823b..28c6e6882b45 100644 --- a/yarn-project/end-to-end/src/test-wallet/utils.ts +++ b/yarn-project/end-to-end/src/test-wallet/utils.ts @@ -1,3 +1,4 @@ +import type { AztecAddress } from '@aztec/aztec.js/addresses'; import { BatchCall, ContractFunctionInteraction, @@ -10,6 +11,9 @@ import { toSendOptions, } from '@aztec/aztec.js/contracts'; import { type AztecNode, waitForTx } from '@aztec/aztec.js/node'; +import { timesAsync } from '@aztec/foundation/collection'; +import type { Logger } from '@aztec/foundation/log'; +import { sleep } from '@aztec/foundation/sleep'; import { SimulationError } from '@aztec/stdlib/errors'; import { type OffchainEffect, type ProvingStats, Tx, TxHash, type TxReceipt } from '@aztec/stdlib/tx'; @@ -70,6 +74,89 @@ export async function proveInteraction( return wallet.proveTx(execPayload, toSendOptions(options)); } +/** Builds an interaction for index `i` of a batch. */ +export type MakeBatchCall = (i: number) => ContractFunctionInteraction | DeployMethod | BatchCall; + +/** + * Pre-proves `count` interactions built by `makeCall(i)` against `wallet`, all with the same + * `options`. Returns the proven txs so the caller can send them individually (e.g. with sleeps + * between sends, or anchored to intermediate tips). For the common "send the whole batch at once" + * case use {@link proveAndSendTxs}. + */ +export function proveTxs( + wallet: TestWallet, + count: number, + makeCall: MakeBatchCall, + options: SendInteractionOptions | DeployOptions, +): Promise { + return timesAsync(count, i => proveInteraction(wallet, makeCall(i), options)); +} + +/** + * Pre-proves `count` interactions built by `makeCall(i)` and sends each one with `NO_WAIT`, all at + * once. Returns the tx hashes in batch order. Pairs with `waitForTxs` for the "send N, wait for N" + * arc. Use {@link proveTxs} instead when the sends need bespoke sequencing. + */ +export async function proveAndSendTxs( + wallet: TestWallet, + count: number, + makeCall: MakeBatchCall, + options: SendInteractionOptions | DeployOptions, +): Promise { + const txs = await proveTxs(wallet, count, makeCall, options); + return Promise.all(txs.map(tx => tx.send({ wait: NO_WAIT }))); +} + +/** Options for {@link startMempoolFeeder}. */ +export type MempoolFeederOpts = { + /** The account the txs are sent from. */ + from: AztecAddress; + /** Keep the pending-tx count at or above this threshold; defaults to 3. */ + minPending?: number; + /** Milliseconds between top-up polls; defaults to 1000. */ + intervalMs?: number; + /** Logger for verbose top-up traces. */ + logger?: Logger; +}; + +/** + * Starts a background loop that keeps `node`'s mempool topped up: whenever the pending-tx count drops + * below `opts.minPending`, it proves and sends one interaction built by `makeCall()` (NO_WAIT). Errors + * are swallowed and retried. Returns an {@link AsyncDisposable}; disposing stops the loop and awaits it. + */ +export function startMempoolFeeder( + wallet: TestWallet, + node: AztecNode, + makeCall: () => ContractFunctionInteraction | DeployMethod | BatchCall, + opts: MempoolFeederOpts, +): AsyncDisposable { + const minPending = opts.minPending ?? 3; + const intervalMs = opts.intervalMs ?? 1000; + let stopped = false; + + const loop = (async () => { + while (!stopped) { + try { + const pendingCount = await node.getPendingTxCount(); + if (pendingCount < minPending) { + await proveAndSendTxs(wallet, 1, makeCall, { from: opts.from }); + opts.logger?.verbose(`Topped up mempool (was ${pendingCount})`); + } + } catch (err) { + opts.logger?.verbose(`Mempool top-up error (will retry): ${err}`); + } + await sleep(intervalMs); + } + })(); + + return { + async [Symbol.asyncDispose]() { + stopped = true; + await loop; + }, + }; +} + /** * Extends AztecNode via declaration merging so instances can be used wherever AztecNode is expected. * The actual method forwarding is handled by a Proxy in the class constructor. diff --git a/yarn-project/ethereum/src/test/chain_monitor.ts b/yarn-project/ethereum/src/test/chain_monitor.ts index 24237a3cae0f..b36f697f09fe 100644 --- a/yarn-project/ethereum/src/test/chain_monitor.ts +++ b/yarn-project/ethereum/src/test/chain_monitor.ts @@ -273,6 +273,53 @@ export class ChainMonitor extends EventEmitter { }); } + /** + * Resolves with the first `checkpoint` event whose payload satisfies `match`. Unlike + * {@link waitUntilCheckpoint} (which waits for a target number), this lets callers wait for an + * arbitrary checkpoint property (e.g. one published in the first half of its slot). Rejects after + * `opts.timeout` ms if provided; otherwise waits indefinitely. + */ + public waitForCheckpoint( + match: (event: ChainMonitorEventMap['checkpoint'][0]) => boolean, + opts: { timeout?: number } = {}, + ): Promise { + return new Promise((resolve, reject) => { + let timer: NodeJS.Timeout | undefined; + const listener = (event: ChainMonitorEventMap['checkpoint'][0]) => { + if (match(event)) { + if (timer) { + clearTimeout(timer); + } + this.off('checkpoint', listener); + resolve(event); + } + }; + if (opts.timeout !== undefined) { + timer = setTimeout(() => { + this.off('checkpoint', listener); + reject(new Error(`Timed out after ${opts.timeout}ms waiting for a matching checkpoint`)); + }, opts.timeout); + } + this.on('checkpoint', listener); + }); + } + + /** Resolves once the proven checkpoint number reaches `checkpointNumber`. */ + public waitUntilCheckpointProven(checkpointNumber: CheckpointNumber): Promise { + if (this.provenCheckpointNumber >= checkpointNumber) { + return Promise.resolve(); + } + return new Promise(resolve => { + const listener = (data: { provenCheckpointNumber: CheckpointNumber; timestamp: bigint }) => { + if (data.provenCheckpointNumber >= checkpointNumber) { + this.off('checkpoint-proven', listener); + resolve(); + } + }; + this.on('checkpoint-proven', listener); + }); + } + private async fetchFeeData(timestamp: bigint): Promise { const [components, minFeePerMana, l1Fees, ethPerFeeAsset, manaTarget] = await Promise.all([ this.rollup.getManaMinFeeComponentsAt(timestamp, true), diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts index 119b88234b77..3b8c14d6e0cf 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts @@ -364,16 +364,17 @@ describe('CheckpointProposalJob', () => { deadline: undefined, isLastBlock: false, }); + const makeSingleBlockTimetable = () => + makeProposerTimetable({ + l1Constants, + blockDurationMs: 9000, + }); describe('single block mode', () => { beforeEach(() => { // Single block mode: a 9s block duration in a 24s slot derives exactly one block sub-slot. - job.setTimetable( - makeProposerTimetable({ - l1Constants, - blockDurationMs: 9000, - }), - ); + timetable = makeSingleBlockTimetable(); + job.setTimetable(timetable); }); it('builds one block with sufficient txs', async () => { @@ -825,6 +826,8 @@ describe('CheckpointProposalJob', () => { targetSlot: SlotNumber(newSlotNumber + 1), proposedCheckpointData, }); + pipelinedJob.setTimetable(makeSingleBlockTimetable()); + dateProvider.setTime(pipelinedJob.getTimetable().getBuildFrameStart(SlotNumber(newSlotNumber + 1)) * 1000); // Listen for mismatch events on this job's emitter mismatchEvents = [];