From 5fbc95628b3a9be533a2e1b89889531a4345f49a Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Fri, 29 May 2026 05:24:07 -0300 Subject: [PATCH 01/24] test(e2e): unskip pipelining related e2e tests (#23642) ## Summary - fix the L1 reorg message tests to wait for pre-reorg message visibility instead of readiness where readiness is not the behavior under test - stabilize the L1 reorg pending-chain prune test by reorging back before the checkpoint publish block - target a future pipelined submission slot in the MBPS prune test before checkpoint publishing is disabled - keep recently unskipped tests listed as flake patterns without skip: true - leave blacklist token contract e2e tests on AUTOMINE_E2E_OPTS and remove stale pipelining migration TODOs ## Verification - yarn lint end-to-end - git diff --check - ANVIL_PORT=8571 e2e_epochs/epochs_l1_reorgs.parallel.test.ts -t 'updates L1 to L2 messages changed due to an L1 reorg' passed locally - ANVIL_PORT=8572 e2e_epochs/epochs_mbps.pipeline.parallel.test.ts -t 'prunes uncheckpointed blocks when proposer fails to deliver' passed locally - Raman: escalated red/green local runs passed for e2e_epochs/epochs_l1_reorgs.parallel.test.ts -t 'handles missed message inserted by an L1 reorg' - Raman: escalated red/green local runs passed for e2e_epochs/epochs_l1_reorgs.parallel.test.ts -t 'prunes blocks from pending chain removed from L1 due to an L1 reorg' - Raman: yarn build passed --- .test_patterns.yml | 20 --------------- .../src/composed/ha/e2e_ha_full.test.ts | 25 +++++++++++++++++++ .../e2e_blacklist_token_contract/burn.test.ts | 3 --- .../minting.test.ts | 3 --- .../shielding.test.ts | 3 --- .../transfer_private.test.ts | 3 --- .../transfer_public.test.ts | 3 --- .../unshielding.test.ts | 3 --- .../epochs_l1_reorgs.parallel.test.ts | 17 +++++++------ .../epochs_mbps.pipeline.parallel.test.ts | 4 +-- 10 files changed, 36 insertions(+), 48 deletions(-) diff --git a/.test_patterns.yml b/.test_patterns.yml index 7652dfda2060..b740808ec72b 100644 --- a/.test_patterns.yml +++ b/.test_patterns.yml @@ -185,15 +185,7 @@ tests: error_regex: "ContractFunctionExecutionError: The contract function" owners: - *mitch - # Under proposer pipelining each validator votes in its own slot and the votes - # don't aggregate into the same round, so the slashing quorum (3) is never - # reached within the 414s budget; the test consistently times out at the docker - # outer 600s (exit 124). The publisher refactor lands all vote-offenses tx's - # on L1 successfully — voteCount on the slasher proposer simply stays at 1 - # per round. This is a slashing-payload aggregation issue independent of - # publisher work; skip until the slashing team addresses it separately. - regex: "e2e_p2p/valid_epoch_pruned_slash.test.ts" - skip: true owners: - *mitch - *palla @@ -276,14 +268,10 @@ tests: - *adam - regex: "src/e2e_epochs/epochs_l1_reorgs.parallel.test.ts" - skip: true owners: - *phil - # Consistently times out — prune detection timing too tight for CI resources - # See https://github.com/AztecProtocol/aztec-packages/pull/22392 - regex: "epochs_mbps.pipeline.parallel.test.ts.*prunes uncheckpointed" - skip: true owners: - *sean @@ -383,15 +371,7 @@ tests: owners: - *palla - # HA full suite is unstable under HA pipelining: multiple distinct failure - # modes (governance vote coord, work distribution, afterAll teardown hook - # timeouts, peer races on checkpoint proposals — see #23344, #23541, - # ci.aztec-labs.com/136431da99834194). Flagging individual error_regex - # entries and even the whole-suite flake match still let it dequeue PRs - # because ci3 retries once and the suite fails both attempts. Skip - # outright on merge-train/spartan until HA pipelining stabilises. - regex: "yarn-project/end-to-end/scripts/run_test.sh ha src/composed/ha/e2e_ha_full.test.ts" - skip: true owners: - *spyros diff --git a/yarn-project/end-to-end/src/composed/ha/e2e_ha_full.test.ts b/yarn-project/end-to-end/src/composed/ha/e2e_ha_full.test.ts index 443adbe3a2e0..d59e3cb189c9 100644 --- a/yarn-project/end-to-end/src/composed/ha/e2e_ha_full.test.ts +++ b/yarn-project/end-to-end/src/composed/ha/e2e_ha_full.test.ts @@ -27,6 +27,7 @@ import { GovernanceProposerAbi } from '@aztec/l1-artifacts/GovernanceProposerAbi import { StatefulTestContractArtifact } from '@aztec/noir-test-contracts.js/StatefulTest'; import { type AttestationInfo, getAttestationInfoFromPublishedCheckpoint } from '@aztec/stdlib/block'; import { Checkpoint } from '@aztec/stdlib/checkpoint'; +import { TopicType } from '@aztec/stdlib/p2p'; import { OffenseType } from '@aztec/stdlib/slashing'; import { TxStatus } from '@aztec/stdlib/tx'; import type { GenesisData } from '@aztec/stdlib/world-state'; @@ -263,6 +264,30 @@ describe('HA Full Setup', () => { } logger.info(`All ${NODE_COUNT} HA peer nodes started and coordinating via PostgreSQL database`); + logger.info('Waiting for HA peer nodes to join the tx gossip mesh before deploying the test account'); + await retryUntil( + async () => { + const meshStates = await Promise.all( + haNodeServices.map(async (service, nodeIndex) => { + const p2p = service.getP2P(); + const [peers, txMeshPeerCount] = await Promise.all([ + p2p.getPeers(), + p2p.getGossipMeshPeerCount(TopicType.tx), + ]); + + return { nodeIndex, peerCount: peers.length, txMeshPeerCount }; + }), + ); + + logger.debug('HA tx gossip mesh status', { meshStates }); + return meshStates.every(({ peerCount, txMeshPeerCount }) => peerCount > 0 && txMeshPeerCount > 0) + ? true + : undefined; + }, + 'HA tx gossip mesh readiness', + 60, + 1, + ); // Now deploy the account - blocks can be built by the HA nodes logger.info('Deploying test account now that validators are running'); diff --git a/yarn-project/end-to-end/src/e2e_blacklist_token_contract/burn.test.ts b/yarn-project/end-to-end/src/e2e_blacklist_token_contract/burn.test.ts index f0d50c5eebcb..3b7e65effb29 100644 --- a/yarn-project/end-to-end/src/e2e_blacklist_token_contract/burn.test.ts +++ b/yarn-project/end-to-end/src/e2e_blacklist_token_contract/burn.test.ts @@ -11,9 +11,6 @@ describe('e2e_blacklist_token_contract burn', () => { let { asset, tokenSim, wallet, adminAddress, otherAddress, blacklistedAddress } = t; beforeAll(async () => { - // TODO(kill-non-pipelined): re-enable pipelining once B1 (world-state fork lifecycle) is - // fixed — BlacklistTokenContractTest.applyBaseSetup runs two 86400s warps which time out - // mineBlock under pipelining. See PIPELINING_GOTCHAS.md. await t.setup({ ...AUTOMINE_E2E_OPTS }); // Beware that we are adding the wallet as minter here, which is very slow because it needs multiple blocks. await t.applyMint(); diff --git a/yarn-project/end-to-end/src/e2e_blacklist_token_contract/minting.test.ts b/yarn-project/end-to-end/src/e2e_blacklist_token_contract/minting.test.ts index 77666a652533..95949cf2138f 100644 --- a/yarn-project/end-to-end/src/e2e_blacklist_token_contract/minting.test.ts +++ b/yarn-project/end-to-end/src/e2e_blacklist_token_contract/minting.test.ts @@ -11,9 +11,6 @@ describe('e2e_blacklist_token_contract mint', () => { let { asset, tokenSim, adminAddress, otherAddress, blacklistedAddress } = t; beforeAll(async () => { - // TODO(kill-non-pipelined): re-enable pipelining once B1 (world-state fork lifecycle) is - // fixed — BlacklistTokenContractTest.applyBaseSetup runs two 86400s warps which time out - // mineBlock under pipelining. See PIPELINING_GOTCHAS.md. await t.setup({ ...AUTOMINE_E2E_OPTS }); // Beware that we are adding the admin as minter here, which is very slow because it needs multiple blocks. await t.applyMint(); diff --git a/yarn-project/end-to-end/src/e2e_blacklist_token_contract/shielding.test.ts b/yarn-project/end-to-end/src/e2e_blacklist_token_contract/shielding.test.ts index 55e22ac57043..92171c469a05 100644 --- a/yarn-project/end-to-end/src/e2e_blacklist_token_contract/shielding.test.ts +++ b/yarn-project/end-to-end/src/e2e_blacklist_token_contract/shielding.test.ts @@ -10,9 +10,6 @@ describe('e2e_blacklist_token_contract shield + redeem_shield', () => { let { asset, tokenSim, wallet, adminAddress, otherAddress, blacklistedAddress } = t; beforeAll(async () => { - // TODO(kill-non-pipelined): re-enable pipelining once B1 (world-state fork lifecycle) is - // fixed — BlacklistTokenContractTest.applyBaseSetup runs two 86400s warps which time out - // mineBlock under pipelining. See PIPELINING_GOTCHAS.md. await t.setup({ ...AUTOMINE_E2E_OPTS }); await t.applyMint(); // Beware that we are adding the admin as minter here // Have to destructure again to ensure we have latest refs. diff --git a/yarn-project/end-to-end/src/e2e_blacklist_token_contract/transfer_private.test.ts b/yarn-project/end-to-end/src/e2e_blacklist_token_contract/transfer_private.test.ts index 1bc2e45f53e7..18481c443e02 100644 --- a/yarn-project/end-to-end/src/e2e_blacklist_token_contract/transfer_private.test.ts +++ b/yarn-project/end-to-end/src/e2e_blacklist_token_contract/transfer_private.test.ts @@ -10,9 +10,6 @@ describe('e2e_blacklist_token_contract transfer private', () => { let { asset, tokenSim, wallet, adminAddress, otherAddress, blacklistedAddress } = t; beforeAll(async () => { - // TODO(kill-non-pipelined): re-enable pipelining once B1 (world-state fork lifecycle) is - // fixed — BlacklistTokenContractTest.applyBaseSetup runs two 86400s warps which time out - // mineBlock under pipelining. See PIPELINING_GOTCHAS.md. await t.setup({ ...AUTOMINE_E2E_OPTS }); // Beware that we are adding the admin as minter here, which is very slow because it needs multiple blocks. await t.applyMint(); diff --git a/yarn-project/end-to-end/src/e2e_blacklist_token_contract/transfer_public.test.ts b/yarn-project/end-to-end/src/e2e_blacklist_token_contract/transfer_public.test.ts index 2c5ff43cae93..1efd41353ddd 100644 --- a/yarn-project/end-to-end/src/e2e_blacklist_token_contract/transfer_public.test.ts +++ b/yarn-project/end-to-end/src/e2e_blacklist_token_contract/transfer_public.test.ts @@ -9,9 +9,6 @@ describe('e2e_blacklist_token_contract transfer public', () => { let { asset, tokenSim, wallet, adminAddress, otherAddress, blacklistedAddress } = t; beforeAll(async () => { - // TODO(kill-non-pipelined): re-enable pipelining once B1 (world-state fork lifecycle) is - // fixed — BlacklistTokenContractTest.applyBaseSetup runs two 86400s warps which time out - // mineBlock under pipelining. See PIPELINING_GOTCHAS.md. await t.setup({ ...AUTOMINE_E2E_OPTS }); // Beware that we are adding the admin as minter here, which is very slow because it needs multiple blocks. await t.applyMint(); diff --git a/yarn-project/end-to-end/src/e2e_blacklist_token_contract/unshielding.test.ts b/yarn-project/end-to-end/src/e2e_blacklist_token_contract/unshielding.test.ts index 1f78751a2d7e..00e1722613f3 100644 --- a/yarn-project/end-to-end/src/e2e_blacklist_token_contract/unshielding.test.ts +++ b/yarn-project/end-to-end/src/e2e_blacklist_token_contract/unshielding.test.ts @@ -10,9 +10,6 @@ describe('e2e_blacklist_token_contract unshielding', () => { let { asset, tokenSim, wallet, adminAddress, otherAddress, blacklistedAddress } = t; beforeAll(async () => { - // TODO(kill-non-pipelined): re-enable pipelining once B1 (world-state fork lifecycle) is - // fixed — BlacklistTokenContractTest.applyBaseSetup runs two 86400s warps which time out - // mineBlock under pipelining. See PIPELINING_GOTCHAS.md. await t.setup({ ...AUTOMINE_E2E_OPTS }); // Beware that we are adding the admin as minter here, which is very slow because it needs multiple blocks. await t.applyMint(); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_l1_reorgs.parallel.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_l1_reorgs.parallel.test.ts index a75838de66ba..ed1682220909 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_l1_reorgs.parallel.test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_l1_reorgs.parallel.test.ts @@ -25,6 +25,7 @@ import { keccak256, parseTransaction } from 'viem'; import { sendL1ToL2Message } from '../fixtures/l1_to_l2_messaging.js'; import type { EndToEndContext } from '../fixtures/utils.js'; +import { waitForL1ToL2MessageSeen } from '../shared/wait_for_l1_to_l2_message.js'; import { proveInteraction } from '../test-wallet/utils.js'; import { EpochsTestContext } from './epochs_test.js'; @@ -350,12 +351,12 @@ describe('e2e_epochs/epochs_l1_reorgs', () => { const CHECKPOINT_NUMBER = CheckpointNumber(initialCheckpoint + 3); await test.waitUntilCheckpointNumber(CHECKPOINT_NUMBER, L2_SLOT_DURATION_IN_S * 10); expect(monitor.checkpointNumber).toEqual(CHECKPOINT_NUMBER); + const l1BlockNumber = monitor.l1BlockNumber; // Stop the sequencer immediately so any in-flight pipelined publish for CHECKPOINT_NUMBER+1 - // doesn't extend l1BlockNumber before we capture it. setConfig alone is not enough under + // doesn't extend the reorg range before we calculate it. setConfig alone is not enough under // pipelining because already-constructed jobs snapshot the old config. await context.sequencer!.stop(); logger.warn(`Sequencer stopped`); - const l1BlockNumber = monitor.l1BlockNumber; // Wait for node to sync to the checkpoint. await retryUntil(() => getCheckpointNumber(node).then(b => b === CHECKPOINT_NUMBER), 'node sync', 10, 0.1); logger.warn(`Reached checkpoint ${CHECKPOINT_NUMBER}`); @@ -364,8 +365,7 @@ describe('e2e_epochs/epochs_l1_reorgs', () => { await test.assertMultipleBlocksPerSlot(2); // Remove the L2 block from L1 - const l1BlocksToReorg = monitor.l1BlockNumber - l1BlockNumber + 1; - await context.cheatCodes.eth.reorgWithReplacement(l1BlocksToReorg); + await context.cheatCodes.eth.reorgTo(l1BlockNumber - 1); expect(await monitor.run(true).then(monitor => monitor.checkpointNumber)).toEqual( CheckpointNumber(CHECKPOINT_NUMBER - 1), ); @@ -472,7 +472,7 @@ describe('e2e_epochs/epochs_l1_reorgs', () => { }); logger.warn(`Sent messages on L1 blocks ${msgs.map(m => m.txReceipt.blockNumber)}`); - await waitForL1ToL2MessageReady(node, msgs.at(-1)!.msgHash, { + await waitForL1ToL2MessageSeen(node, msgs.at(-1)!.msgHash, { timeoutSeconds: msgs.length * L1_BLOCK_TIME_IN_S * 2, }); @@ -485,7 +485,7 @@ describe('e2e_epochs/epochs_l1_reorgs', () => { logger.warn(`Sent new message on L1 block ${newMsg.txReceipt.blockNumber}`); // New msg gets synced, and old one is out - await waitForL1ToL2MessageReady(node, newMsg.msgHash, { timeoutSeconds: L1_BLOCK_TIME_IN_S * 6 }); + await waitForL1ToL2MessageReady(node, newMsg.msgHash, { timeoutSeconds: L2_SLOT_DURATION_IN_S * 5 }); expect(await isL1ToL2MessageReady(node, msgs[0].msgHash)).toBe(true); expect(await isL1ToL2MessageReady(node, msgs.at(-1)!.msgHash)).toBe(false); @@ -502,7 +502,7 @@ describe('e2e_epochs/epochs_l1_reorgs', () => { logger.warn(`Sending first cross chain message`); const firstMsg = await sendMessage(); logger.warn(`Sent first message on L1 block ${firstMsg.txReceipt.blockNumber}`); - await waitForL1ToL2MessageReady(node, firstMsg.msgHash, { timeoutSeconds: L1_BLOCK_TIME_IN_S * 3 }); + await waitForL1ToL2MessageSeen(node, firstMsg.msgHash, { timeoutSeconds: L1_BLOCK_TIME_IN_S * 3 }); logger.warn(`Synced first message`); // Next message shall not land @@ -524,11 +524,12 @@ describe('e2e_epochs/epochs_l1_reorgs', () => { const reorgDepth = (await monitor.run(true).then(m => m.l1BlockNumber)) - l1BlockNumber; await context.cheatCodes.eth.reorgWithReplacement(reorgDepth, [[l1ClientDelayer.getCancelledTxs()[0]]]); const secondMsg = await secondMsgPromise; + await waitForL1ToL2MessageSeen(node, secondMsg.msgHash, { timeoutSeconds: L1_BLOCK_TIME_IN_S * 3 }); // Archiver should see the new message and should be able to accept a third one on top, without any rolling hash issues logger.warn(`Reorged-in second message on L1 block ${secondMsg.txReceipt.blockNumber}. Sending third message.`); const thirdMsg = await sendMessage(); - await waitForL1ToL2MessageReady(node, thirdMsg.msgHash, { timeoutSeconds: L1_BLOCK_TIME_IN_S * 3 }); + await waitForL1ToL2MessageSeen(node, thirdMsg.msgHash, { timeoutSeconds: L1_BLOCK_TIME_IN_S * 3 }); // Verify multi-block checkpoints were built await test.assertMultipleBlocksPerSlot(2); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_mbps.pipeline.parallel.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_mbps.pipeline.parallel.test.ts index 35f3fb1acc40..105286701ed6 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_mbps.pipeline.parallel.test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_mbps.pipeline.parallel.test.ts @@ -325,12 +325,12 @@ describe('e2e_epochs/epochs_mbps_pipeline', () => { await test.waitUntilCheckpointNumber(CheckpointNumber(1), checkpointTimeout); const checkpointedBlockNumber = await archiver.getBlockNumber(); logger.warn(`Baseline established: checkpoint 1 reached at block ${checkpointedBlockNumber}`); - // Find the next proposer and prevent it from publishing checkpoints + // Target a submission slot whose pipelined build has not started yet. const { slot: currentSlot } = test.epochCache.getEpochAndSlotNow(); const { proposerIndex, slot: proposerSlotToNotPublish } = await findNextProposerIndex( test.epochCache, validators, - SlotNumber(currentSlot + 1), + SlotNumber(currentSlot + 2), ); logger.warn( `Will skip checkpoint publishing for proposer ${proposerIndex} in slot ${proposerSlotToNotPublish} - current slot ${currentSlot}`, From a61245222d9200d7201afc8dc7008bc6007a1f34 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Fri, 29 May 2026 05:49:27 -0300 Subject: [PATCH 02/24] fix(archiver): prune blocks without proposed checkpoint by end of build slot (#23606) When the previous proposer sent some block proposals but failed to send the corresponding checkpoint proposal, the current proposer would assume there was no proposed checkpoint to build on top of, but would still use the proposed blocks as chain tip. This meant a failed `canPropose` check against the Rollup contract as soon as it started its slot, since the proposed blocks from the previous proposer meant the proposer had a wrong chain tip. To fix, the sequencer is now aware that there may be proposed blocks without the corresponding checkpoints, and it can't start building until that's resolved. Also, the archiver now prunes proposed blocks without a checkpoint when the corresponding _build_ slot is over. --- ## Motivation Under proposer pipelining a node can receive and reexecute the block-only proposals for a checkpoint before (or without ever) receiving the enclosing proposed checkpoint. This leaves the local tip one checkpoint ahead of the checkpointed tip with no proposed checkpoint backing it. A sequencer that then builds the next checkpoint on top of that orphan tip forks the chain off a parent no other node can follow, which was the root cause behind the sentinel CI flake. ## Approach Two complementary defenses. The sequencer's `checkSync` refuses to proceed when the synced block's checkpoint is ahead of the checkpointed tip and no matching proposed checkpoint exists, holding the line during the window before cleanup. The archiver adds a wall-clock orphan prune that, shortly after a block's build slot ends, removes a block-only tip whose checkpoint was never proposed, restoring liveness even while L1 is quiet. ## Changes - **sequencer-client**: `checkSync` rejects syncing onto a proposed block with no matching proposed-checkpoint tip/data, logging a descriptive warning. - **archiver**: new `pruneOrphanProposedBlocks` on the L1 synchronizer, run from `Archiver.sync()` after the inbound queue drains and before L1 sync; prunes after `start(blockSlot) + grace` using the epoch-cache pipelining offset and emits `L2PruneUncheckpointed`. The existing L1-sync prune is preserved (shared prune/emit helper). - **archiver/stdlib/foundation config**: new `orphanProposedBlockPruneGraceSeconds` in `ArchiverSpecificConfig`, archiver config mappings (`ARCHIVER_ORPHAN_PROPOSED_BLOCK_PRUNE_GRACE_SECONDS`), `mapArchiverConfig`, the synchronizer/archiver config types, and a new `EnvVar`. - **aztec-node**: defaults the grace window from `blockDurationMs / 1000` when unset, falling back to `MIN_EXECUTION_TIME`; the archiver factory also defaults to `MIN_EXECUTION_TIME`. - **sequencer-client (tests)**: orphan tip returns `undefined` and warns; matching proposed checkpoint proceeds. - **archiver (tests)**: no prune before grace; prune + event after grace; no prune when a matching proposed checkpoint exists; queued proposed checkpoint is processed before the prune. --- yarn-project/archiver/README.md | 12 +- .../archiver/src/archiver-misc.test.ts | 11 +- .../archiver/src/archiver-store.test.ts | 5 + .../archiver/src/archiver-sync.test.ts | 160 +++++++++ yarn-project/archiver/src/archiver.ts | 109 +++++- yarn-project/archiver/src/config.ts | 9 + yarn-project/archiver/src/factory.ts | 4 + .../src/modules/data_store_updater.ts | 32 +- .../archiver/src/modules/l1_synchronizer.ts | 1 - .../archiver/src/test/noop_l1_archiver.ts | 7 + .../aztec-node/src/aztec-node/server.ts | 6 + .../epochs_orphan_block_prune.test.ts | 319 ++++++++++++++++++ yarn-project/foundation/src/config/env_var.ts | 1 + yarn-project/sequencer-client/src/config.ts | 6 + .../src/sequencer/checkpoint_proposal_job.ts | 10 +- .../src/sequencer/sequencer.test.ts | 118 ++++++- .../src/sequencer/sequencer.ts | 24 ++ .../stdlib/src/interfaces/archiver.ts | 9 + yarn-project/stdlib/src/interfaces/configs.ts | 10 + 19 files changed, 838 insertions(+), 15 deletions(-) create mode 100644 yarn-project/end-to-end/src/e2e_epochs/epochs_orphan_block_prune.test.ts diff --git a/yarn-project/archiver/README.md b/yarn-project/archiver/README.md index 5d4a8e92a35a..959f16f819dc 100644 --- a/yarn-project/archiver/README.md +++ b/yarn-project/archiver/README.md @@ -26,12 +26,13 @@ The archiver runs a periodic sync loop with two phases: ``` sync() -├── processQueuedBlocks() # Handle blocks pushed via addBlock() +├── processQueuedBlocks() # Handle blocks pushed via addBlock() +├── pruneOrphanProposedBlocks() # Wall-clock prune of orphan block-only tips └── syncFromL1() - ├── handleL1ToL2Messages() # Sync messages from Inbox contract - ├── handleCheckpoints() # Sync checkpoints from Rollup contract + ├── handleL1ToL2Messages() # Sync messages from Inbox contract + ├── handleCheckpoints() # Sync checkpoints from Rollup contract ├── pruneUncheckpointedBlocks() # Prune provisional blocks from expired slots - ├── handleEpochPrune() # Proactive unwind before proof window expires + ├── handleEpochPrune() # Proactive unwind before proof window expires └── checkForNewCheckpointsBeforeL1SyncPoint() # Handle L1 reorg edge case ``` @@ -100,8 +101,9 @@ Queued blocks are processed at the start of each sync iteration. This allows the Blocks added via `addBlock()` are considered "provisional" until they appear in an L1 checkpoint. These provisional blocks may need to be reconciled when: - **Checkpoint mismatch**: A checkpoint lands on L1 with different blocks than stored locally (e.g., a different proposer won the slot) - **Slot expiration**: An L2 slot ends without any checkpoint being mined on L1 +- **Orphan proposed block**: Under proposer pipelining, a proposer can broadcast a block-only proposal but never the matching `CheckpointProposal` (e.g. it crashes before assembling the checkpoint). The provisional block then has no proposed checkpoint backing it. -When `handleCheckpoints()` processes incoming checkpoints, it compares archive roots of local blocks against the checkpoint's blocks. If they differ, local blocks are pruned and replaced with the checkpoint's blocks. After checkpoint sync, `pruneUncheckpointedBlocks()` removes any remaining provisional blocks from slots that have ended. Both cases emit `L2PruneUncheckpointed`. +When `handleCheckpoints()` processes incoming checkpoints, it compares archive roots of local blocks against the checkpoint's blocks. If they differ, local blocks are pruned and replaced with the checkpoint's blocks. After checkpoint sync, `pruneUncheckpointedBlocks()` removes any remaining provisional blocks from slots that have ended. Independently, `pruneOrphanProposedBlocks()` runs on wall-clock time (so it fires during quiet L1 periods) and removes a block-only tip once its build slot ended without a matching proposed checkpoint, plus a grace window configured via `orphanProposedBlockPruneGraceSeconds`. All three cases emit `L2PruneUncheckpointed`. ### Querying Block Data diff --git a/yarn-project/archiver/src/archiver-misc.test.ts b/yarn-project/archiver/src/archiver-misc.test.ts index 700a863c07a8..94cf202dc722 100644 --- a/yarn-project/archiver/src/archiver-misc.test.ts +++ b/yarn-project/archiver/src/archiver-misc.test.ts @@ -8,6 +8,7 @@ import { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/f import { Buffer32 } from '@aztec/foundation/buffer'; import { Fr } from '@aztec/foundation/curves/bn254'; import { EthAddress } from '@aztec/foundation/eth-address'; +import { DateProvider } from '@aztec/foundation/timer'; import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; import type { L2Tips } from '@aztec/stdlib/block'; import type { CheckpointData } from '@aztec/stdlib/checkpoint'; @@ -57,6 +58,7 @@ describe('Archiver misc', () => { const rollupContract = mock(); const epochCache = mock(); epochCache.getCommitteeForEpoch.mockResolvedValue({ committee: [] as EthAddress[] } as EpochCommitteeInfo); + epochCache.pipeliningOffset.mockReturnValue(0); const tracer = getTelemetryClient().getTracer(''); const instrumentation = mock({ isEnabled: () => true, tracer }); @@ -78,7 +80,12 @@ describe('Archiver misc', () => { slashingProposerAddress: EthAddress.random(), }, archiverStore, - { pollingIntervalMs: 1000, batchSize: 1000, maxAllowedEthClientDriftSeconds: 300 }, + { + pollingIntervalMs: 1000, + batchSize: 1000, + maxAllowedEthClientDriftSeconds: 300, + orphanProposedBlockPruneGraceSeconds: 2, + }, blobClient, instrumentation, l1Constants, @@ -87,6 +94,8 @@ describe('Archiver misc', () => { initialHeader, initialBlockHash, l2TipsCache, + epochCache, + new DateProvider(), ); }); diff --git a/yarn-project/archiver/src/archiver-store.test.ts b/yarn-project/archiver/src/archiver-store.test.ts index dd84e9b5a824..bab140c6f758 100644 --- a/yarn-project/archiver/src/archiver-store.test.ts +++ b/yarn-project/archiver/src/archiver-store.test.ts @@ -12,6 +12,7 @@ import { import { Buffer32 } from '@aztec/foundation/buffer'; import { Fr } from '@aztec/foundation/curves/bn254'; import { EthAddress } from '@aztec/foundation/eth-address'; +import { DateProvider } from '@aztec/foundation/timer'; import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; import { L2Block } from '@aztec/stdlib/block'; import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers'; @@ -63,6 +64,7 @@ describe('Archiver Store', () => { blobClient = mock(); epochCache = mock(); epochCache.getCommitteeForEpoch.mockResolvedValue({ committee: [] as EthAddress[] } as EpochCommitteeInfo); + epochCache.pipeliningOffset.mockReturnValue(0); const rollupContract = mock(); Object.defineProperty(rollupContract, 'address', { value: rollupAddress.toString(), writable: true }); @@ -98,6 +100,7 @@ describe('Archiver Store', () => { batchSize: 1000, maxAllowedEthClientDriftSeconds: 300, ethereumAllowNoDebugHosts: true, + orphanProposedBlockPruneGraceSeconds: 2, }; const events = new EventEmitter() as ArchiverEmitter; @@ -120,6 +123,8 @@ describe('Archiver Store', () => { initialHeader, initialBlockHash, l2TipsCache, + epochCache, + new DateProvider(), ); }); diff --git a/yarn-project/archiver/src/archiver-sync.test.ts b/yarn-project/archiver/src/archiver-sync.test.ts index c9294389f062..468297212cf4 100644 --- a/yarn-project/archiver/src/archiver-sync.test.ts +++ b/yarn-project/archiver/src/archiver-sync.test.ts @@ -20,6 +20,7 @@ import type { ProposedCheckpointInput } from '@aztec/stdlib/checkpoint'; import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers'; import { computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging'; import { CheckpointHeader } from '@aztec/stdlib/rollup'; +import { mockCheckpointAndMessages } from '@aztec/stdlib/testing'; import { BlockHeader } from '@aztec/stdlib/tx'; import { getTelemetryClient } from '@aztec/telemetry-client'; @@ -92,6 +93,9 @@ describe('Archiver Sync', () => { // Create epoch cache mock (separate from fake) epochCache = mock(); epochCache.getCommitteeForEpoch.mockResolvedValue({ committee: [] as EthAddress[] } as EpochCommitteeInfo); + // Default to no pipelining offset; the orphan-prune tests below override this. Keeps the prune + // deadline well ahead of wall-clock time for the other tests so it never fires spuriously. + epochCache.pipeliningOffset.mockReturnValue(0); // Create instrumentation mock const tracer = getTelemetryClient().getTracer(''); @@ -118,6 +122,7 @@ describe('Archiver Sync', () => { maxAllowedEthClientDriftSeconds: 300, ethereumAllowNoDebugHosts: true, skipHistoricalLogsCheck: true, + orphanProposedBlockPruneGraceSeconds: 2, }; // Create event emitter shared by archiver and synchronizer @@ -162,6 +167,8 @@ describe('Archiver Sync', () => { initialHeader, initialBlockHash, l2TipsCache, + epochCache, + dateProvider, ); }); @@ -2143,4 +2150,157 @@ describe('Archiver Sync', () => { expect(tips.proposedCheckpoint.block.number).toEqual(tips.checkpointed.block.number); }, 15_000); }); + + describe('pruning orphan proposed blocks', () => { + let pruneSpy: jest.Mock; + + // Slot the orphan block targets. With slotDuration=24, slot S starts at l1GenesisTime + S*24. + const orphanSlot = SlotNumber(1); + // Grace period configured for these tests (see the `config` object above). + const graceSeconds = 2; + + beforeEach(() => { + pruneSpy = jest.fn(); + archiver.events.on(L2BlockSourceEvents.L2PruneUncheckpointed, pruneSpy); + // Normal proposer pipelining: a block targeting slot S is built during slot S-1, so its proposed + // checkpoint is expected by the start of slot S. + epochCache.pipeliningOffset.mockReturnValue(1); + }); + + afterEach(() => { + archiver.events.off(L2BlockSourceEvents.L2PruneUncheckpointed, pruneSpy); + }); + + // Wall-clock time (seconds) at which the orphan tip becomes prunable: start(orphanSlot) + grace. + const pruneDeadline = () => now + Number(orphanSlot) * l1Constants.slotDuration + graceSeconds; + const pruneDeadlineForSlot = (slot: SlotNumber) => now + Number(slot) * l1Constants.slotDuration + graceSeconds; + + // Syncs checkpoint 1 (slot 0), then writes uncheckpointed blocks for slot 1 (checkpoint 2) straight + // into the store as a block-only tip with no matching proposed checkpoint. L1 is held at slot 1 so + // the L1-sync prune (which only fires once the build slot has ended on L1) stays out of the way. + const setupOrphanTip = async () => { + const { checkpoint: cp1 } = await fake.addCheckpoint(CheckpointNumber(1), { + l1BlockNumber: 1n, + messagesL1BlockNumber: 1n, + numL1ToL2Messages: 3, + slotNumber: SlotNumber(0), + }); + const cp1Archive = cp1.blocks.at(-1)!.archive; + fake.setL1BlockNumber(1n); + await archiver.syncImmediate(); + expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(1)); + + const lastBlockInCp1 = cp1.blocks.at(-1)!.number; + const provisionalBlocks = await fake.makeBlocks(CheckpointNumber(2), { + l1BlockNumber: 2n, + previousArchive: cp1Archive, + slotNumber: orphanSlot, + }); + for (const block of provisionalBlocks) { + await archiver.addBlock(block); + } + + // Hold L1 at slot 1 so the slot has not ended from L1's perspective. + fake.setL1BlockNumber(2n); + return { lastBlockInCp1, lastProvisional: provisionalBlocks.at(-1)!.number, provisionalBlocks }; + }; + + const makeProposedCheckpoint = (lastBlockInCp1: BlockNumber, blockCount: number): ProposedCheckpointInput => ({ + checkpointNumber: CheckpointNumber(2), + header: CheckpointHeader.empty({ slotNumber: orphanSlot }), + startBlock: BlockNumber(lastBlockInCp1 + 1), + blockCount, + totalManaUsed: 0n, + feeAssetPriceModifier: 0n, + }); + + it('does not prune before the grace window elapses', async () => { + const { lastProvisional } = await setupOrphanTip(); + + dateProvider.setTime((pruneDeadline() - 1) * 1000); + await archiver.syncImmediate(); + + expect(pruneSpy).not.toHaveBeenCalled(); + expect(await archiver.getBlockNumber()).toEqual(lastProvisional); + }, 15_000); + + it('prunes the orphan tip once the grace window elapses', async () => { + const { lastBlockInCp1, provisionalBlocks } = await setupOrphanTip(); + + dateProvider.setTime((pruneDeadline() + 1) * 1000); + await archiver.syncImmediate(); + + expect(pruneSpy).toHaveBeenCalledWith( + expect.objectContaining({ + type: L2BlockSourceEvents.L2PruneUncheckpointed, + slotNumber: orphanSlot, + blocks: expect.arrayContaining(provisionalBlocks.map(b => expect.objectContaining({ number: b.number }))), + }), + ); + expect(await archiver.getBlockNumber()).toEqual(lastBlockInCp1); + expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(1)); + }, 15_000); + + it('does not prune when a matching proposed checkpoint exists', async () => { + const { lastBlockInCp1, lastProvisional, provisionalBlocks } = await setupOrphanTip(); + + await archiver.addProposedCheckpoint(makeProposedCheckpoint(lastBlockInCp1, provisionalBlocks.length)); + + dateProvider.setTime((pruneDeadline() + 100) * 1000); + await archiver.syncImmediate(); + + expect(pruneSpy).not.toHaveBeenCalled(); + expect(await archiver.getBlockNumber()).toEqual(lastProvisional); + expect(await archiverStore.blocks.getLastProposedCheckpoint()).toBeDefined(); + }, 15_000); + + it('processes a queued proposed checkpoint before pruning, sparing the tip', async () => { + const { lastBlockInCp1, lastProvisional, provisionalBlocks } = await setupOrphanTip(); + + // Past the grace window: without the matching checkpoint the next sync would prune the tip. + dateProvider.setTime((pruneDeadline() + 100) * 1000); + + // Queue the proposed checkpoint. The triggered sync drains the inbound queue (storing the + // checkpoint) before running the orphan prune, so the prune sees it and stands down. If the + // order were reversed, this sync would prune the tip before storing the checkpoint. + await archiver.addProposedCheckpoint(makeProposedCheckpoint(lastBlockInCp1, provisionalBlocks.length)); + await archiver.syncImmediate(); + + expect(pruneSpy).not.toHaveBeenCalled(); + expect(await archiver.getBlockNumber()).toEqual(lastProvisional); + expect(await archiverStore.blocks.getLastProposedCheckpoint()).toBeDefined(); + }, 15_000); + + it('prunes only the orphan suffix after a covered pending checkpoint', async () => { + const { lastBlockInCp1, provisionalBlocks: checkpointTwoBlocks } = await setupOrphanTip(); + + await archiver.addProposedCheckpoint(makeProposedCheckpoint(lastBlockInCp1, checkpointTwoBlocks.length)); + + const orphanSuffixSlot = SlotNumber(orphanSlot + 1); + const { checkpoint: orphanSuffixCheckpoint } = await mockCheckpointAndMessages(CheckpointNumber(3), { + startBlockNumber: BlockNumber(checkpointTwoBlocks.at(-1)!.number + 1), + numBlocks: 1, + previousArchive: checkpointTwoBlocks.at(-1)!.archive, + slotNumber: orphanSuffixSlot, + }); + const orphanSuffixBlocks = orphanSuffixCheckpoint.blocks; + for (const block of orphanSuffixBlocks) { + await archiver.addBlock(block); + } + + dateProvider.setTime((pruneDeadlineForSlot(orphanSuffixSlot) + 1) * 1000); + await archiver.syncImmediate(); + + expect(pruneSpy).toHaveBeenCalledWith( + expect.objectContaining({ + type: L2BlockSourceEvents.L2PruneUncheckpointed, + slotNumber: orphanSuffixSlot, + blocks: expect.arrayContaining(orphanSuffixBlocks.map(b => expect.objectContaining({ number: b.number }))), + }), + ); + expect(await archiver.getBlockNumber()).toEqual(checkpointTwoBlocks.at(-1)!.number); + expect(await archiverStore.blocks.getProposedCheckpointByNumber(CheckpointNumber(2))).toBeDefined(); + expect(await archiverStore.blocks.getProposedCheckpointByNumber(CheckpointNumber(3))).toBeUndefined(); + }, 15_000); + }); }); diff --git a/yarn-project/archiver/src/archiver.ts b/yarn-project/archiver/src/archiver.ts index 9519e29012c0..895d6a2b07ae 100644 --- a/yarn-project/archiver/src/archiver.ts +++ b/yarn-project/archiver/src/archiver.ts @@ -5,7 +5,7 @@ import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses' import type { ViemPublicClient, ViemPublicDebugClient } from '@aztec/ethereum/types'; import { BlockNumber, CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; import { Buffer32 } from '@aztec/foundation/buffer'; -import { merge } from '@aztec/foundation/collection'; +import { merge, pick } from '@aztec/foundation/collection'; import { Fr } from '@aztec/foundation/curves/bn254'; import { EthAddress } from '@aztec/foundation/eth-address'; import { type Logger, createLogger } from '@aztec/foundation/log'; @@ -17,6 +17,7 @@ import { type BlockHash, L2Block, type L2BlockSink, + L2BlockSourceEvents, type L2Tips, type ValidateCheckpointResult, } from '@aztec/stdlib/block'; @@ -110,9 +111,15 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra * @param dataStores - Archiver substores for storage & retrieval of blocks, encrypted logs & contract data. * @param config - Archiver configuration options. * @param blobClient - Client for retrieving blob data. - * @param dateProvider - Provider for current date/time. * @param instrumentation - Instrumentation for metrics and tracing. * @param l1Constants - L1 rollup constants. + * @param synchronizer - L1 synchronizer that handles fetching checkpoints and messages from L1. + * @param events - Event emitter shared with the synchronizer. + * @param initialHeader - Genesis block header. + * @param initialBlockHash - Precomputed hash of the genesis block header. + * @param l2TipsCache - In-memory cache for L2 chain tips. + * @param epochCache - Cache used to compute the proposer pipelining offset. + * @param dateProvider - Provider for current date/time, used for wall-clock orphan-block pruning. * @param log - A logger. */ constructor( @@ -133,6 +140,7 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra maxAllowedEthClientDriftSeconds: number; ethereumAllowNoDebugHosts?: boolean; skipHistoricalLogsCheck?: boolean; + orphanProposedBlockPruneGraceSeconds: number; }, private readonly blobClient: BlobClientInterface, instrumentation: ArchiverInstrumentation, @@ -145,6 +153,8 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra initialHeader: BlockHeader, initialBlockHash: BlockHash, l2TipsCache: L2TipsCache, + private readonly epochCache: EpochCache, + private readonly dateProvider: DateProvider, private readonly log: Logger = createLogger('archiver'), ) { super(dataStores, l1Constants, initialHeader, initialBlockHash, l1Constants.genesisArchiveRoot); @@ -338,6 +348,101 @@ export class Archiver extends ArchiverDataSourceBase implements L2BlockSink, Tra await this.processInboundQueue(); // Now perform L1 sync await this.syncFromL1(); + // Prune proposed blocks with no corresponding proposed checkpoint by the end of their build slot. + await this.pruneOrphanProposedBlocks(); + } + + /** + * Prunes a block-only local tip that was built atop a checkpoint that was never itself proposed. + * + * Under pipelining, a proposer publishes the blocks for a checkpoint (block-only proposals) before + * assembling and publishing the enclosing proposed checkpoint at the end of the build slot. A node + * that received those blocks but never the proposed checkpoint is left with an orphan tip it must + * not build on. We prune it once enough wall-clock time has elapsed that the proposed checkpoint + * should have arrived. This runs on wall-clock time (not L1 block advancement) so it fires during + * quiet L1 periods, and is the liveness counterpart to the sequencer's checkSync guard. + * + * The uncheckpointed suffix is scanned in order. Blocks covered by proposed checkpoints are left in + * place; the first block not covered by a proposed checkpoint starts the orphan suffix to prune. + */ + private async pruneOrphanProposedBlocks(): Promise { + const tips = await this.getL2Tips(); + const now = BigInt(this.dateProvider.nowInSeconds()); + const pipeliningOffset = this.epochCache.pipeliningOffset(); + + // This only applies under pipelining + if (pipeliningOffset === 0) { + this.log.trace(`Pipelining offset is 0, skipping orphan proposed block pruning`); + return; + } + + // The proposed tip is a proposed-checkpointed block, so there are no orphan proposed blocks to prune + if (tips.proposedCheckpoint.block.number === tips.proposed.number) { + this.log.trace( + `No orphan proposed blocks to prune: proposed tip ${tips.proposed.number} is checkpointed`, + pick(tips, 'proposed', 'proposedCheckpoint'), + ); + return; + } + + // Load the blocks that are candidates for pruning (ie blocks without a proposed checkpoint covering them) + const blocksWithoutProposedCheckpoint = await this.stores.blocks.getBlocksData({ + from: BlockNumber(tips.proposedCheckpoint.block.number + 1), + limit: tips.proposed.number - tips.proposedCheckpoint.block.number, + }); + + // Iterate through them in order, the first one with a slot that should have received a proposed checkpoint + // is the first orphan block, and all blocks after it are also orphaned and should be pruned. + let lastSlotChecked = undefined; + for (const blockData of blocksWithoutProposedCheckpoint) { + // No need to recheck if this block had the same slot as the previous one. + const blockSlot = blockData.header.getSlot(); + const blockNumber = blockData.header.getBlockNumber(); + if (lastSlotChecked !== undefined && blockSlot === lastSlotChecked) { + continue; + } + lastSlotChecked = blockSlot; + + // The proposed checkpoint should have landed by the start of the slot after the block's build slot + // (build slot = blockSlot - pipeliningOffset). Wait a grace period beyond that to tolerate propagation. + const expectedCheckpointedBySlot = SlotNumber(Number(blockSlot) - pipeliningOffset + 1); + const expectedCheckpointedByTime = + getTimestampForSlot(expectedCheckpointedBySlot, this.l1Constants) + + BigInt(this.config.orphanProposedBlockPruneGraceSeconds); + + // If it's not checkpointed by the expected time, prune it along with all blocks after it. + if (now >= expectedCheckpointedByTime) { + const pruneAfterBlockNumber = BlockNumber(blockNumber - 1); + this.log.warn( + `Pruning orphan blocks after block ${pruneAfterBlockNumber}: block at slot ${blockSlot} belongs to ` + + `checkpoint ${blockData.checkpointNumber} which has no matching proposed checkpoint`, + { + firstUncheckpointedBlockHeader: blockData.header.toInspect(), + blockCheckpointNumber: blockData.checkpointNumber, + blockNumber, + blockSlot, + pipeliningOffset, + expectedCheckpointedBySlot, + expectedCheckpointedByTime, + now, + }, + ); + + const prunedBlocks = await this.updater.removeBlocksWithoutProposedCheckpointAfter(pruneAfterBlockNumber); + if (prunedBlocks.length > 0) { + this.events.emit(L2BlockSourceEvents.L2PruneUncheckpointed, { + type: L2BlockSourceEvents.L2PruneUncheckpointed, + slotNumber: blockSlot, + blocks: prunedBlocks, + }); + } + return; + } + } + + this.log.trace('No orphan proposed blocks to prune: all uncheckpointed blocks are still within the grace period', { + blocksWithoutProposedCheckpoint: blocksWithoutProposedCheckpoint.map(b => b.header.toInspect()), + }); } private async syncFromL1() { diff --git a/yarn-project/archiver/src/config.ts b/yarn-project/archiver/src/config.ts index 4d7e1197638c..a10fcb40ede1 100644 --- a/yarn-project/archiver/src/config.ts +++ b/yarn-project/archiver/src/config.ts @@ -78,6 +78,14 @@ export const archiverConfigMappings: ConfigMappingsType = { 'Set to true to bypass the check when the connected RPC node is known to prune old logs.', ...booleanConfigHelper(false), }, + orphanProposedBlockPruneGraceSeconds: { + env: 'ARCHIVER_ORPHAN_PROPOSED_BLOCK_PRUNE_GRACE_SECONDS', + description: + 'Grace period in seconds, measured from the end of a proposed block build slot, after which a ' + + 'proposed block with no matching proposed checkpoint is pruned as an orphan. Defaults from the ' + + 'sequencer block duration at the node wiring layer when unset.', + ...optionalNumberConfigHelper(), + }, ...chainConfigMappings, ...l1ReaderConfigMappings, viemPollingIntervalMS: { @@ -107,5 +115,6 @@ export function mapArchiverConfig(config: Partial) { maxAllowedEthClientDriftSeconds: config.maxAllowedEthClientDriftSeconds, ethereumAllowNoDebugHosts: config.ethereumAllowNoDebugHosts, skipHistoricalLogsCheck: config.archiverSkipHistoricalLogsCheck, + orphanProposedBlockPruneGraceSeconds: config.orphanProposedBlockPruneGraceSeconds, }; } diff --git a/yarn-project/archiver/src/factory.ts b/yarn-project/archiver/src/factory.ts index f486dfbbb5c4..b75f356fcbef 100644 --- a/yarn-project/archiver/src/factory.ts +++ b/yarn-project/archiver/src/factory.ts @@ -16,6 +16,7 @@ import { FunctionType, decodeFunctionSignature } from '@aztec/stdlib/abi'; import type { ArchiverEmitter, BlockHash } from '@aztec/stdlib/block'; import { type ContractClassPublicWithCommitment, computePublicBytecodeCommitment } from '@aztec/stdlib/contract'; import type { DataStoreConfig } from '@aztec/stdlib/kv-store'; +import { MIN_EXECUTION_TIME } from '@aztec/stdlib/timetable'; import type { BlockHeader } from '@aztec/stdlib/tx'; import { getTelemetryClient } from '@aztec/telemetry-client'; @@ -129,6 +130,7 @@ export async function createArchiver( maxAllowedEthClientDriftSeconds: 300, ethereumAllowNoDebugHosts: false, skipHistoricalLogsCheck: false, + orphanProposedBlockPruneGraceSeconds: MIN_EXECUTION_TIME, }, mapArchiverConfig(config), ); @@ -179,6 +181,8 @@ export async function createArchiver( initialHeader, initialBlockHash, l2TipsCache, + epochCache, + deps.dateProvider ?? new DateProvider(), ); await archiver.start(opts.blockUntilSync); diff --git a/yarn-project/archiver/src/modules/data_store_updater.ts b/yarn-project/archiver/src/modules/data_store_updater.ts index 56a1369201f9..9b314c2fd7bf 100644 --- a/yarn-project/archiver/src/modules/data_store_updater.ts +++ b/yarn-project/archiver/src/modules/data_store_updater.ts @@ -262,12 +262,36 @@ export class ArchiverDataStoreUpdater { ); } - const result = await this.removeBlocksAfter(blockNumber); + const prunedBlocks = await this.removeBlocksAfter(blockNumber); + await this.evictProposedCheckpointsForPrunedBlocks(prunedBlocks); - // Clear all pending proposed checkpoints since their blocks have been pruned - await this.stores.blocks.deleteProposedCheckpoints(); + return prunedBlocks; + }); + await this.l2TipsCache?.refresh(); + return result; + } + + /** + * Removes all blocks without a proposed checkpoint strictly after the specified block number and cleans up associated contract data. + * This handles removal of provisionally added blocks along with their contract classes/instances. + * Verifies that each block being removed is not part of a stored checkpoint (proposed or not). + * This differs from `removeUncheckpointedBlocksAfter` in that it also checks proposed checkpoints. + * + * @param blockNumber - Remove all blocks with number greater than this. + * @returns The removed blocks. + * @throws Error if any block to be removed is checkpointed. + */ + public async removeBlocksWithoutProposedCheckpointAfter(blockNumber: BlockNumber): Promise { + const result = await this.stores.db.transactionAsync(async () => { + // Verify we're only removing uncheckpointed blocks + const lastCheckpointedBlockNumber = await this.stores.blocks.getProposedCheckpointL2BlockNumber(); + if (blockNumber < lastCheckpointedBlockNumber) { + throw new Error( + `Cannot remove blocks after ${blockNumber} because proposed checkpointed blocks exist up to ${lastCheckpointedBlockNumber}`, + ); + } - return result; + return await this.removeBlocksAfter(blockNumber); }); await this.l2TipsCache?.refresh(); return result; diff --git a/yarn-project/archiver/src/modules/l1_synchronizer.ts b/yarn-project/archiver/src/modules/l1_synchronizer.ts index 3d12a9d58a4a..fd61b33a963b 100644 --- a/yarn-project/archiver/src/modules/l1_synchronizer.ts +++ b/yarn-project/archiver/src/modules/l1_synchronizer.ts @@ -295,7 +295,6 @@ export class ArchiverL1Synchronizer implements Traceable { ); const prunedBlocks = await this.updater.removeUncheckpointedBlocksAfter(lastCheckpointedBlockNumber); - if (prunedBlocks.length > 0) { this.events.emit(L2BlockSourceEvents.L2PruneUncheckpointed, { type: L2BlockSourceEvents.L2PruneUncheckpointed, diff --git a/yarn-project/archiver/src/test/noop_l1_archiver.ts b/yarn-project/archiver/src/test/noop_l1_archiver.ts index e4601b30fdfe..1095ceaeaa2d 100644 --- a/yarn-project/archiver/src/test/noop_l1_archiver.ts +++ b/yarn-project/archiver/src/test/noop_l1_archiver.ts @@ -1,10 +1,12 @@ import type { BlobClientInterface } from '@aztec/blob-client/client'; +import { EpochCache } from '@aztec/epoch-cache'; import type { RollupContract } from '@aztec/ethereum/contracts'; import type { ViemPublicClient, ViemPublicDebugClient } from '@aztec/ethereum/types'; import { SlotNumber } from '@aztec/foundation/branded-types'; import { Buffer32 } from '@aztec/foundation/buffer'; import { Fr } from '@aztec/foundation/curves/bn254'; import { EthAddress } from '@aztec/foundation/eth-address'; +import { DateProvider } from '@aztec/foundation/timer'; import type { FunctionsOf } from '@aztec/foundation/types'; import type { ArchiverEmitter, BlockHash } from '@aztec/stdlib/block'; import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers'; @@ -69,6 +71,8 @@ export class NoopL1Archiver extends Archiver { const events = new EventEmitter() as ArchiverEmitter; const synchronizer = new NoopL1Synchronizer(instrumentation.tracer); + const epochCache = mock(); + epochCache.pipeliningOffset.mockReturnValue(0); super( publicClient, @@ -89,6 +93,7 @@ export class NoopL1Archiver extends Archiver { maxAllowedEthClientDriftSeconds: 300, ethereumAllowNoDebugHosts: true, // Skip trace validation skipHistoricalLogsCheck: true, // Skip historical logs validation + orphanProposedBlockPruneGraceSeconds: 2, }, blobClient, instrumentation, @@ -98,6 +103,8 @@ export class NoopL1Archiver extends Archiver { initialHeader, initialBlockHash, l2TipsCache, + epochCache, + new DateProvider(), ); } diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index f4be06db0b6e..2386d1da467f 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -112,6 +112,7 @@ import type { DebugLogStore, LogFilter, SiloedTag, Tag, TxScopedL2Log } from '@a import { InMemoryDebugLogStore, NullDebugLogStore } from '@aztec/stdlib/logs'; import { InboxLeaf, type L1ToL2MessageSource, appendL1ToL2MessagesToTree } from '@aztec/stdlib/messaging'; import type { Offense } from '@aztec/stdlib/slashing'; +import { MIN_EXECUTION_TIME } from '@aztec/stdlib/timetable'; import type { NullifierLeafPreimage, PublicDataTreeLeafPreimage } from '@aztec/stdlib/trees'; import { MerkleTreeId, NullifierMembershipWitness, PublicDataWitness } from '@aztec/stdlib/trees'; import { @@ -577,6 +578,11 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, AztecNodeDeb // Track started resources so we can clean up on partial failure during node creation. const started: { stop?(): Promise | void }[] = []; try { + // Default the orphan-prune grace window from the block build duration when unset, so the archiver + // waits roughly one build slot for a proposed checkpoint to arrive before pruning a block-only tip. + config.orphanProposedBlockPruneGraceSeconds ??= + config.blockDurationMs !== undefined ? Math.ceil(config.blockDurationMs / 1000) : MIN_EXECUTION_TIME; + // Create world-state first so we can retrieve the initial header before constructing the archiver. const nativeWs = await createWorldState(config, options.genesis); const initialHeader = nativeWs.getInitialHeader(); diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_orphan_block_prune.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_orphan_block_prune.test.ts new file mode 100644 index 000000000000..4842181432ae --- /dev/null +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_orphan_block_prune.test.ts @@ -0,0 +1,319 @@ +import type { Archiver } from '@aztec/archiver'; +import type { AztecNodeService } from '@aztec/aztec-node'; +import { EthAddress } from '@aztec/aztec.js/addresses'; +import { Fr } from '@aztec/aztec.js/fields'; +import type { Logger } from '@aztec/aztec.js/log'; +import { asyncMap } from '@aztec/foundation/async-map'; +import { BlockNumber, CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types'; +import { times } from '@aztec/foundation/collection'; +import { SecretValue } from '@aztec/foundation/config'; +import { retryUntil } from '@aztec/foundation/retry'; +import { bufferToHex } from '@aztec/foundation/string'; +import { timeoutPromise } from '@aztec/foundation/timer'; +import { type L2Block, L2BlockSourceEvents } from '@aztec/stdlib/block'; +import { getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; +import type { ChainTips } from '@aztec/stdlib/interfaces/server'; + +import { jest } from '@jest/globals'; +import { privateKeyToAccount } from 'viem/accounts'; + +import { getPrivateKeyFromIndex } from '../fixtures/utils.js'; +import { EpochsTestContext } from './epochs_test.js'; + +jest.setTimeout(1000 * 60 * 15); + +const NODE_COUNT = 4; + +/** + * E2E test for the orphan-proposed-block prune path under proposer pipelining. + * + * With pipelining, the proposer for slot N+1 builds and gossips its checkpoint during slot N. The last block in that + * checkpoint is broadcast standalone (so peers can pre-sync the archive) and the enclosing CheckpointProposal is + * broadcast separately. If the CheckpointProposal never arrives, peers are left with a proposed-but-uncheckpointed tip + * — an "orphan" block — and the next proposer must NOT attempt to build on it. + * + * Setup: 4 validators (V1..V4), one node per key, mocked gossip network. We find two consecutive slots S1, S2 with + * distinct proposers P1, P2. P1 is configured via the test-only `skipBroadcastCheckpointProposal` flag to suppress its + * CheckpointProposal broadcast while still letting the held last block reach peers. P2 must (a) prune the orphan on + * every archiver, and (b) build a fresh checkpoint for S2 that lands on L1. + */ +describe('e2e_epochs/epochs_orphan_block_prune', () => { + let logger: Logger; + let test: EpochsTestContext; + let nodes: AztecNodeService[]; + + afterEach(async () => { + jest.restoreAllMocks(); + await test?.teardown(); + }); + + it('all nodes prune the orphan block and S2 rebuilds the checkpoint chain', async () => { + // Build 4 distinct validators (V1..V4). One key per node, no overlap. + const validators = times(NODE_COUNT, i => { + const privateKey = bufferToHex(getPrivateKeyFromIndex(i + 3)!); + const attester = EthAddress.fromString(privateKeyToAccount(privateKey).address); + return { attester, withdrawer: attester, privateKey, bn254SecretKey: new SecretValue(Fr.random().toBigInt()) }; + }); + + test = await EpochsTestContext.setup({ + numberOfAccounts: 1, + initialValidators: validators, + enableProposerPipelining: true, + inboxLag: 2, + mockGossipSubNetwork: true, + disableAnvilTestWatcher: true, + startProverNode: false, + aztecEpochDuration: 4, + aztecProofSubmissionEpochs: 1024, + enforceTimeTable: true, + ethereumSlotDuration: 6, + aztecSlotDuration: 36, + blockDurationMs: 8000, + attestationPropagationTime: 0.5, + l1PublishingTime: 2, + aztecTargetCommitteeSize: NODE_COUNT, + skipInitialSequencer: true, + }); + + ({ logger } = test); + + nodes = await asyncMap(validators, ({ privateKey }, i) => + test.createValidatorNode([privateKey], { + dontStartSequencer: true, + coinbase: EthAddress.fromNumber(0xa + i), + buildCheckpointIfEmpty: true, + minTxsPerBlock: 0, + }), + ); + + logger.warn('Validator nodes created', { + validators: validators.map((v, i) => ({ idx: i, attester: v.attester.toString() })), + }); + + // Find S1 (>=4 ahead) such that proposers for S1 and S2=S1+1 are two distinct validators. The +4 margin gives the + // warp+sequencer-start path enough headroom to reach the build window for S1-1 (the pipelining build slot for S1) + // even if node creation jitters. + // + // The L1 rollup contract only exposes proposers for epochs whose randao seed is "stable" (i.e. queryable on L1 + // right now). When we look too far into the future the contract reverts with `ValidatorSelection__EpochNotStable`. + // We handle this by warping L1 forward one epoch at a time and retrying. + let S1: SlotNumber | undefined; + let proposerOne: EthAddress | undefined; + let proposerTwo: EthAddress | undefined; + let candidate = Number(test.epochCache.getEpochAndSlotNow().slot) + 4; + const maxAttempts = 200; + for (let attempt = 0; attempt < maxAttempts && S1 === undefined; attempt++) { + try { + const [p1, p2] = await Promise.all([ + test.epochCache.getProposerAttesterAddressInSlot(SlotNumber(candidate)), + test.epochCache.getProposerAttesterAddressInSlot(SlotNumber(candidate + 1)), + ]); + const p1Index = p1 ? validators.findIndex(v => v.attester.equals(p1)) : -1; + const p2Index = p2 ? validators.findIndex(v => v.attester.equals(p2)) : -1; + if (p1 && p2 && !p1.equals(p2) && p1Index >= 0 && p2Index >= 0) { + S1 = SlotNumber(candidate); + proposerOne = p1; + proposerTwo = p2; + break; + } + candidate++; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + if (!msg.includes('EpochNotStable')) { + throw err; + } + const block = await test.l1Client.getBlock({ includeTransactions: false }); + const warpBy = test.epochDuration * test.L2_SLOT_DURATION_IN_S; + const newTs = Number(block.timestamp) + warpBy; + logger.warn(`Hit EpochNotStable at candidate ${candidate}, warping L1 forward by ${warpBy}s to ${newTs}`); + await test.context.cheatCodes.eth.warp(newTs, { resetBlockInterval: true }); + const newCurrentSlot = Number(test.epochCache.getEpochAndSlotNow().slot); + if (candidate < newCurrentSlot + 4) { + candidate = newCurrentSlot + 4; + } + } + } + if (S1 === undefined || !proposerOne || !proposerTwo) { + throw new Error(`Could not find a slot with two distinct consecutive proposers after ${maxAttempts} attempts`); + } + + const S2 = SlotNumber(S1 + 1); + const p1Index = validators.findIndex(v => v.attester.equals(proposerOne!)); + const p2Index = validators.findIndex(v => v.attester.equals(proposerTwo!)); + + logger.warn(`Selected target S1=${S1}`, { + S1, + S2, + proposerOne: proposerOne.toString(), + p1Index, + proposerTwo: proposerTwo.toString(), + p2Index, + }); + + // Suppress only the CheckpointProposal broadcast for the proposer of S1. The held last block is still broadcast + // standalone, so peers' archivers ingest the slot-S1 block as a proposed tip but never see a checkpoint proposal + // for it — the exact orphan-block state we want. + await nodes[p1Index].setConfig({ skipBroadcastCheckpointProposal: true }); + + // No tx is needed: nodes are configured with buildCheckpointIfEmpty so the proposer will produce an empty + // checkpoint on its slot. The test verifies the orphan prune + rebuild invariants, not tx flow. + + // Subscribe to the prune event on every node before sequencers start, so we never miss it. We capture the chain + // tips asynchronously inside the handler for log context, but do not assert on them — by the time the snapshot is + // read, P2's rebuild may already have landed. + type PruneObservation = { slotNumber: SlotNumber; blocks: L2Block[]; tipsAtPrune: ChainTips }; + const prunePromises: Promise[] = nodes.map( + (node, idx) => + new Promise(resolve => { + const archiver = node.getBlockSource() as Archiver; + // eslint-disable-next-line @typescript-eslint/no-misused-promises + archiver.events.once(L2BlockSourceEvents.L2PruneUncheckpointed, async ev => { + const tipsAtPrune = await node.getChainTips(); + logger.warn(`Node ${idx} pruned uncheckpointed blocks`, { + slotNumber: ev.slotNumber, + blocks: ev.blocks.map(b => ({ number: b.number, slot: b.header.globalVariables.slotNumber })), + tipsAtPrune, + }); + resolve({ slotNumber: ev.slotNumber, blocks: ev.blocks, tipsAtPrune }); + }); + }), + ); + + // Warp L1 to one L1 block before the build slot for S1 (which is S1-1 under pipelining offset 1). Pipelining will + // then engage during S1-1 and the proposer for S1 builds + would broadcast its CheckpointProposal — except we + // just suppressed it. + const buildSlot = SlotNumber(S1 - 1); + const targetTs = getTimestampForSlot(buildSlot, test.constants) - BigInt(test.L1_BLOCK_TIME_IN_S); + logger.warn(`Warping L1 to timestamp ${targetTs} (one L1 block before build slot ${buildSlot} for S1=${S1})`); + await test.context.cheatCodes.eth.warp(Number(targetTs), { resetBlockInterval: true }); + + expect(await nodes[0].getBlockNumber()).toEqual(0); + + const sequencers = nodes.map(n => n.getSequencer()!); + const { failEvents } = test.watchSequencerEvents(sequencers, i => ({ validator: `V${i + 1}` })); + + await Promise.all(sequencers.map(s => s.start())); + logger.warn('All sequencers started'); + + const slotAdvanceTimeout = test.L2_SLOT_DURATION_IN_S * 3; + + // (1) Orphan appears on every archiver. During build slot S1-1, P1 builds and broadcasts the held last block + // standalone (because of skipBroadcastCheckpointProposal). Every node's proposed tip advances to a block whose + // slotNumber === S1. + logger.warn(`Waiting for proposed chain to reach slot ${S1} on all nodes (orphan tip from P1)`); + await Promise.all( + nodes.map((node, idx) => + retryUntil( + async () => { + const tips = await node.getChainTips(); + if (tips.proposed.number === 0) { + return false; + } + const block = await node.getBlock(tips.proposed.number); + return !!block && block.header.globalVariables.slotNumber === S1; + }, + `node ${idx} proposed advanced to slot ${S1}`, + slotAdvanceTimeout, + 0.5, + ), + ), + ); + + // Capture each node's pre-prune block-1 archive root for the staleness check in (3). + const preBlocks = await Promise.all(nodes.map(node => node.getBlock(BlockNumber(1)))); + const preArchiveRoots = preBlocks.map(block => { + if (!block) { + throw new Error('Expected pre-prune block 1 to exist on every node'); + } + return block.archive.root.toString(); + }); + logger.warn('Captured pre-prune block-1 archive roots', { preArchiveRoots }); + + // (2) Orphan is pruned on every archiver. The wall-clock prune in pruneOrphanProposedBlocks fires once + // `now >= getTimestampForSlot(blockSlot - pipeliningOffset + 1) + grace`, which lands well inside slot S1 (= the + // build slot for S2) given a 36s aztecSlotDuration and the default 8s grace. We wait up to 2 slot durations as a + // margin. + logger.warn('Waiting for L2PruneUncheckpointed on every node'); + const pruneTimeoutMs = test.L2_SLOT_DURATION_IN_S * 2 * 1000; + const pruneObservations = await Promise.all( + prunePromises.map((p, idx) => + Promise.race([p, timeoutPromise(pruneTimeoutMs, `Node ${idx} did not emit prune event in time`)]), + ), + ); + + for (const [idx, obs] of pruneObservations.entries()) { + expect({ idx, slotNumber: obs.slotNumber }).toEqual({ idx, slotNumber: S1 }); + const prunedSlots = obs.blocks.map(b => b.header.globalVariables.slotNumber); + // Only the orphan at slot S1 should have been pruned — nothing earlier or later. + expect(prunedSlots.every(s => s === S1)).toBe(true); + // We do not assert exact equality on tipsAtPrune here. The handler is async and awaits getChainTips(), so P2's + // rebuild could already have landed by the time the snapshot is read. The prune event itself (slotNumber === S1, + // blocks include S1) is sufficient proof. + } + + // (3) S2 builds and the checkpoint lands on L1. After the prune, P2's pipelined build during S1 publishes during + // S2, so L2 block 1 on every node must be the rebuilt block with slot S2. We target block 1 directly rather than + // the live checkpointed tip to avoid an S3-first race where the chain has already advanced past S2 by the time + // we poll. + logger.warn(`Waiting for L2 block 1 to be the rebuilt slot-${S2} block on all nodes`); + await Promise.all( + nodes.map((node, idx) => + retryUntil( + async () => { + const block = await node.getBlock(BlockNumber(1)); + return !!block && block.header.globalVariables.slotNumber === S2; + }, + `node ${idx} block 1 rebuilt at slot ${S2}`, + slotAdvanceTimeout, + 0.5, + ), + ), + ); + + // Independently confirm the checkpoint actually landed on L1 by waiting (bounded) on the chain monitor and + // verifying the block at L2 block number 1 — that is the rebuilt block, and its slot must equal S2. Targeting + // block 1 rather than the live tip avoids a race where the chain has already advanced past S2 by the time we read. + await test.waitUntilCheckpointNumber(CheckpointNumber(1), test.L2_SLOT_DURATION_IN_S * 4); + const rebuiltBlock = await nodes[0].getBlock(BlockNumber(1)); + expect(rebuiltBlock).toBeDefined(); + expect(rebuiltBlock!.header.globalVariables.slotNumber).toEqual(S2); + + // The rebuilt block at number 1 must have a different archive root from the orphan we saw before the prune. This + // guards against accidental pass on stale state. + const postBlocks = await Promise.all(nodes.map(node => node.getBlock(BlockNumber(1)))); + const postArchiveRoots = postBlocks.map(block => { + if (!block) { + throw new Error('Expected post-prune block 1 to exist on every node'); + } + return block.archive.root.toString(); + }); + logger.warn('Captured post-prune block-1 archive roots', { postArchiveRoots }); + for (const [idx, root] of postArchiveRoots.entries()) { + expect({ idx, root }).not.toEqual({ idx, root: preArchiveRoots[idx] }); + } + + // Tolerated fail events, scoped narrowly: + // - P1 at S1 expectedly fails to publish (peers never see the CheckpointProposal, so it cannot collect + // attestations). Scoped to that exact (sequencer, slot). + // - P2 discards its own pipelined checkpoint for S2 because S1's parent never landed. No other sequencer + // should be discarding pipelined work in this scenario. + const unexpectedFailEvents = failEvents.filter(e => { + if (e.type === 'checkpoint-publish-failed' && e.sequencerIndex === p1Index + 2 && e.slot === S1) { + return false; + } + if ( + e.type === 'pipelined-checkpoint-discarded' && + e.sequencerIndex === p2Index + 2 && + e.slot === S2 && + e.checkpointNumber === CheckpointNumber(1) + ) { + return false; + } + return true; + }); + if (unexpectedFailEvents.length > 0) { + logger.error('Unexpected fail events from sequencers', unexpectedFailEvents); + } + expect(unexpectedFailEvents).toEqual([]); + }); +}); diff --git a/yarn-project/foundation/src/config/env_var.ts b/yarn-project/foundation/src/config/env_var.ts index 99869e33e6b4..d631360d8b08 100644 --- a/yarn-project/foundation/src/config/env_var.ts +++ b/yarn-project/foundation/src/config/env_var.ts @@ -11,6 +11,7 @@ export type EnvVar = | 'ARCHIVER_URL' | 'ARCHIVER_VIEM_POLLING_INTERVAL_MS' | 'ARCHIVER_BATCH_SIZE' + | 'ARCHIVER_ORPHAN_PROPOSED_BLOCK_PRUNE_GRACE_SECONDS' | 'AZTEC_ADMIN_PORT' | 'AZTEC_NODE_DEBUG' | 'AZTEC_ADMIN_API_KEY_HASH' diff --git a/yarn-project/sequencer-client/src/config.ts b/yarn-project/sequencer-client/src/config.ts index 7801bc99f309..aa306e7a5219 100644 --- a/yarn-project/sequencer-client/src/config.ts +++ b/yarn-project/sequencer-client/src/config.ts @@ -252,6 +252,12 @@ export const sequencerConfigMappings: ConfigMappingsType = { description: 'Skip broadcasting checkpoint and block proposals via gossipsub when proposer (for testing only)', ...booleanConfigHelper(false), }, + skipBroadcastCheckpointProposal: { + description: + 'Skip broadcasting only the CheckpointProposal via gossipsub when proposer; the held last block is broadcast ' + + 'standalone instead so peers still receive it as a proposed-but-uncheckpointed tip (for testing only)', + ...booleanConfigHelper(false), + }, pauseProposingForSlots: { description: 'List of slots for which the sequencer will not produce a proposal (for testing only). Attestation paths are unaffected.', diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts index e86b5533edcc..d054f9882635 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts @@ -775,7 +775,15 @@ export class CheckpointProposalJob implements Traceable { ); const blockProposedAt = this.dateProvider.now(); - if (!this.config.skipBroadcastProposals) { + if (this.config.skipBroadcastCheckpointProposal) { + // Test-only: suppress the CheckpointProposal so peers never see a proposed checkpoint for + // this slot, but still broadcast the held last block standalone so peers' archivers ingest + // it as a proposed-but-uncheckpointed tip — the exact orphan-block state that + // pruneOrphanProposedBlocks / checkSync must handle. + if (blockPendingBroadcast && !this.config.skipBroadcastProposals) { + await this.p2pClient.broadcastProposal(blockPendingBroadcast); + } + } else if (!this.config.skipBroadcastProposals) { await this.p2pClient.broadcastCheckpointProposal(proposal); this.checkpointMetrics.noteCheckpointBroadcast(this.dateProvider.now()); } diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts index d17475a5fee7..c5f84642cd44 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts @@ -45,7 +45,7 @@ import { AppendOnlyTreeSnapshot } from '@aztec/stdlib/trees'; import { BlockHeader, GlobalVariables, type Tx } from '@aztec/stdlib/tx'; import type { FullNodeCheckpointsBuilder, ValidatorClient } from '@aztec/validator-client'; -import { expect } from '@jest/globals'; +import { expect, jest } from '@jest/globals'; import { type MockProxy, mock, mockDeep, mockFn } from 'jest-mock-extended'; import type { GlobalVariableBuilder } from '../global_variable_builder/global_builder.js'; @@ -1312,6 +1312,114 @@ describe('sequencer', () => { }); }); + describe('checkSync orphan-block guard', () => { + // Mocks all sync sources so checkSync passes its earlier equality checks and reaches the orphan + // guard, with the world-state tip at `blockNumber` (in `blockCheckpointNumber`) while the + // checkpointed and proposed-checkpoint tips sit at the given checkpoint numbers. + const setupSyncedToBlock = (opts: { + blockNumber: BlockNumber; + blockCheckpointNumber: CheckpointNumber; + checkpointedCheckpointNumber: CheckpointNumber; + proposedCheckpointTipNumber: CheckpointNumber; + proposedCheckpointData: ProposedCheckpointData | undefined; + }) => { + const hash = Fr.random().toString(); + const checkpointHash = Fr.random().toString(); + const proposedCheckpointHash = Fr.random().toString(); + worldState.status.mockResolvedValue({ + state: WorldStateRunningState.IDLE, + syncSummary: { + latestBlockNumber: opts.blockNumber, + latestBlockHash: hash, + finalizedBlockNumber: BlockNumber.ZERO, + oldestHistoricBlockNumber: BlockNumber.ZERO, + treesAreSynched: true, + }, + } satisfies WorldStateSynchronizerStatus); + const tips = { + proposed: { number: opts.blockNumber, hash }, + proposedCheckpoint: { + block: { number: opts.blockNumber, hash }, + checkpoint: { number: opts.proposedCheckpointTipNumber, hash: proposedCheckpointHash }, + }, + checkpointed: { + block: { number: opts.blockNumber, hash }, + checkpoint: { number: opts.checkpointedCheckpointNumber, hash: checkpointHash }, + }, + proven: { + block: { number: opts.blockNumber, hash }, + checkpoint: { number: opts.checkpointedCheckpointNumber, hash: checkpointHash }, + }, + finalized: { + block: { number: opts.blockNumber, hash }, + checkpoint: { number: opts.checkpointedCheckpointNumber, hash: checkpointHash }, + }, + }; + l2BlockSource.getL2Tips.mockResolvedValue(tips); + l1ToL2MessageSource.getL2Tips.mockResolvedValue(tips); + p2p.getStatus.mockResolvedValue({ syncedToL2Block: { number: opts.blockNumber, hash } } as any); + l2BlockSource.getBlockData.mockResolvedValue({ + header: BlockHeader.empty({ globalVariables: GlobalVariables.empty({ blockNumber: opts.blockNumber }) }), + archive: AppendOnlyTreeSnapshot.empty(), + blockHash: BlockHash.ZERO, + checkpointNumber: opts.blockCheckpointNumber, + indexWithinCheckpoint: IndexWithinCheckpoint(0), + } satisfies BlockData); + l2BlockSource.getProposedCheckpointData.mockResolvedValue(opts.proposedCheckpointData); + }; + + it('returns undefined and warns when the proposed block has no matching proposed checkpoint', async () => { + // Local tip is a block at checkpoint 3, but the checkpointed and proposed-checkpoint tips are + // still at checkpoint 2 and no proposed checkpoint 3 exists: an orphan block-only tip. + setupSyncedToBlock({ + blockNumber: BlockNumber(3), + blockCheckpointNumber: CheckpointNumber(3), + checkpointedCheckpointNumber: CheckpointNumber(2), + proposedCheckpointTipNumber: CheckpointNumber(2), + proposedCheckpointData: undefined, + }); + const warnSpy = jest.spyOn(sequencer.getLogger(), 'warn'); + + const result = await sequencer.checkSyncForTest({ ts: 1000n, slot: SlotNumber(2) }); + + expect(result).toBeUndefined(); + expect(warnSpy).toHaveBeenCalledWith( + 'Sequencer sync check failed: proposed block has no matching proposed checkpoint', + expect.objectContaining({ + blockCheckpointNumber: CheckpointNumber(3), + checkpointedCheckpointNumber: CheckpointNumber(2), + proposedCheckpointTipNumber: CheckpointNumber(2), + proposedCheckpointDataNumber: undefined, + }), + ); + }); + + it('proceeds when a matching proposed checkpoint exists for the block', async () => { + setupSyncedToBlock({ + blockNumber: BlockNumber(3), + blockCheckpointNumber: CheckpointNumber(3), + checkpointedCheckpointNumber: CheckpointNumber(2), + proposedCheckpointTipNumber: CheckpointNumber(3), + proposedCheckpointData: { + checkpointNumber: CheckpointNumber(3), + header: CheckpointHeader.empty(), + archive: AppendOnlyTreeSnapshot.empty(), + checkpointOutHash: Fr.ZERO, + startBlock: BlockNumber(3), + blockCount: 1, + totalManaUsed: 0n, + feeAssetPriceModifier: 0n, + } satisfies ProposedCheckpointData, + }); + + const result = await sequencer.checkSyncForTest({ ts: 1000n, slot: SlotNumber(2) }); + + expect(result).toBeDefined(); + expect(result?.checkpointNumber).toEqual(CheckpointNumber(3)); + expect(result?.checkpointedCheckpointNumber).toEqual(CheckpointNumber(2)); + }); + }); + describe('view-based proposer lookup', () => { it('passes target slot to getProposerAttesterAddressInSlot', async () => { const proposer = signer.address; @@ -1373,4 +1481,12 @@ class TestSequencer extends Sequencer { public checkCanProposeForTest(slot: SlotNumber) { return this.checkCanPropose(slot); } + + public checkSyncForTest(args: { ts: bigint; slot: SlotNumber }) { + return this.checkSync(args); + } + + public getLogger() { + return this.log; + } } diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.ts index 6dda210dad1a..7e4d44da78fb 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.ts @@ -699,6 +699,30 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter l2Tips.checkpointed.checkpoint.number && + (l2Tips.proposedCheckpoint.checkpoint.number !== blockData.checkpointNumber || + proposedCheckpointData?.checkpointNumber !== blockData.checkpointNumber) + ) { + this.log.warn(`Sequencer sync check failed: proposed block has no matching proposed checkpoint`, { + blockCheckpointNumber: blockData.checkpointNumber, + checkpointedCheckpointNumber: l2Tips.checkpointed.checkpoint.number, + proposedCheckpointTipNumber: l2Tips.proposedCheckpoint.checkpoint.number, + proposedCheckpointDataNumber: proposedCheckpointData?.checkpointNumber, + blockNumber: blockData.header.getBlockNumber(), + blockSlot: blockData.header.getSlot(), + syncedL2Slot, + ...args, + }); + return undefined; + } + const hasProposedCheckpoint = l2Tips.proposedCheckpoint.checkpoint.number > l2Tips.checkpointed.checkpoint.number; // The l2Tips and proposedCheckpointData reads above come from independent archiver snapshots diff --git a/yarn-project/stdlib/src/interfaces/archiver.ts b/yarn-project/stdlib/src/interfaces/archiver.ts index b8abf1c873b8..48d937c65cb9 100644 --- a/yarn-project/stdlib/src/interfaces/archiver.ts +++ b/yarn-project/stdlib/src/interfaces/archiver.ts @@ -68,6 +68,14 @@ export type ArchiverSpecificConfig = { /** Skip promoting proposed checkpoints during L1 sync (for testing purposes only) */ skipPromoteProposedCheckpointDuringL1Sync?: boolean; + + /** + * Grace period in seconds, measured from the end of a proposed block's build slot, after which a + * proposed block whose enclosing checkpoint was never proposed is pruned as an orphan. Defaults + * from `blockDurationMs / 1000` at the node wiring layer, falling back to the timetable minimum + * execution time. + */ + orphanProposedBlockPruneGraceSeconds?: number; }; export const ArchiverSpecificConfigSchema = z.object({ @@ -81,6 +89,7 @@ export const ArchiverSpecificConfigSchema = z.object({ archiverSkipHistoricalLogsCheck: z.boolean().optional(), skipValidateCheckpointAttestations: z.boolean().optional(), skipPromoteProposedCheckpointDuringL1Sync: z.boolean().optional(), + orphanProposedBlockPruneGraceSeconds: schemas.Integer.optional(), }); export type ArchiverApi = Omit< diff --git a/yarn-project/stdlib/src/interfaces/configs.ts b/yarn-project/stdlib/src/interfaces/configs.ts index 27de324a3630..3ed6a5c4799a 100644 --- a/yarn-project/stdlib/src/interfaces/configs.ts +++ b/yarn-project/stdlib/src/interfaces/configs.ts @@ -101,6 +101,14 @@ export interface SequencerConfig { skipPublishingCheckpointsPercent?: number; /** Skip broadcasting checkpoint and block proposals via gossipsub when proposer (for testing only) */ skipBroadcastProposals?: boolean; + /** + * Skip broadcasting only the CheckpointProposal via gossipsub when proposer; the held last block is still broadcast + * standalone so peers receive it as a proposed-but-uncheckpointed tip. Used to exercise the orphan-proposed-block + * prune path (for testing only). Narrower variant of `skipBroadcastProposals`: when only this flag is set the held + * last block is still broadcast standalone, but when `skipBroadcastProposals` is also set neither the block nor the + * checkpoint proposal is broadcast. + */ + skipBroadcastCheckpointProposal?: boolean; /** List of slots for which the sequencer will not produce a proposal (for testing only). Attestation paths are unaffected. */ pauseProposingForSlots?: SlotNumber[]; } @@ -149,6 +157,7 @@ export const SequencerConfigSchema = zodFor()( minBlocksForCheckpoint: z.number().positive().optional(), skipPublishingCheckpointsPercent: z.number().gte(0).lte(100).optional(), skipBroadcastProposals: z.boolean().optional(), + skipBroadcastCheckpointProposal: z.boolean().optional(), pauseProposingForSlots: z.array(SlotNumberSchema).optional(), }), ); @@ -174,6 +183,7 @@ type SequencerConfigOptionalKeys = | 'maxDABlockGas' | 'redistributeCheckpointBudget' | 'skipBroadcastProposals' + | 'skipBroadcastCheckpointProposal' | 'pauseProposingForSlots'; export type ResolvedSequencerConfig = Prettify< From 5683aa0b1957b955f1dc6f91c45361d428f80580 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Fri, 29 May 2026 05:57:36 -0300 Subject: [PATCH 03/24] test: migrate benchmarks to pipelining setup (#23647) ## Summary - Run build-block, node RPC, and tx stats benchmarks with `PIPELINING_SETUP_OPTS`. - Run client-flow benchmarks through the Automine setup via `ClientFlowsBenchmark`. - Advance fee-juice bridge setup with explicit debug `mineBlock()` when available so Automine can progress empty blocks. ## Testing - `yarn format end-to-end` - `yarn build` - `yarn lint end-to-end` --- .../client_flows/client_flows_benchmark.ts | 4 ++-- .../src/bench/node_rpc_perf.test.ts | 2 ++ .../src/bench/tx_stats_bench.test.ts | 8 +++---- yarn-project/end-to-end/src/bench/utils.ts | 3 ++- .../src/shared/gas_portal_test_harness.ts | 24 +++++++++++++------ 5 files changed, 27 insertions(+), 14 deletions(-) diff --git a/yarn-project/end-to-end/src/bench/client_flows/client_flows_benchmark.ts b/yarn-project/end-to-end/src/bench/client_flows/client_flows_benchmark.ts index 0858e18e26b3..74ba3a4439d7 100644 --- a/yarn-project/end-to-end/src/bench/client_flows/client_flows_benchmark.ts +++ b/yarn-project/end-to-end/src/bench/client_flows/client_flows_benchmark.ts @@ -28,7 +28,7 @@ import type { ContractInstanceWithAddress } from '@aztec/stdlib/contract'; import { GasSettings } from '@aztec/stdlib/gas'; import { deriveSigningKey } from '@aztec/stdlib/keys'; -import { MNEMONIC, getPaddedMaxFeesPerGas } from '../../fixtures/fixtures.js'; +import { AUTOMINE_E2E_OPTS, MNEMONIC, getPaddedMaxFeesPerGas } from '../../fixtures/fixtures.js'; import { type EndToEndContext, type SetupOptions, deployAccounts, setup, teardown } from '../../fixtures/setup.js'; import { mintTokensToPrivate } from '../../fixtures/token_utils.js'; import { setupSponsoredFPC } from '../../fixtures/utils.js'; @@ -125,7 +125,7 @@ export class ClientFlowsBenchmark { constructor(testName?: string, setupOptions: Partial = {}) { this.logger = createLogger(`bench:client_flows${testName ? `:${testName}` : ''}`); - this.setupOptions = { startProverNode: true, ...setupOptions }; + this.setupOptions = { ...AUTOMINE_E2E_OPTS, startProverNode: true, ...setupOptions }; this.config = BENCHMARK_CONFIG === 'key_flows' ? KEY_FLOWS_CONFIG : FULL_FLOWS_CONFIG; ProxyLogger.create(); this.proxyLogger = ProxyLogger.getInstance(); diff --git a/yarn-project/end-to-end/src/bench/node_rpc_perf.test.ts b/yarn-project/end-to-end/src/bench/node_rpc_perf.test.ts index f2db3a0ba546..a0db5a60f32c 100644 --- a/yarn-project/end-to-end/src/bench/node_rpc_perf.test.ts +++ b/yarn-project/end-to-end/src/bench/node_rpc_perf.test.ts @@ -24,6 +24,7 @@ import { mkdir, writeFile } from 'fs/promises'; import 'jest-extended'; import * as path from 'path'; +import { PIPELINING_SETUP_OPTS } from '../fixtures/fixtures.js'; import { setup } from '../fixtures/utils.js'; import type { TestWallet } from '../test-wallet/test_wallet.js'; import { proveInteraction } from '../test-wallet/utils.js'; @@ -148,6 +149,7 @@ describe('e2e_node_rpc_perf', () => { sequencerPollingIntervalMS: 200, worldStateBlockCheckIntervalMS: 200, blockCheckIntervalMS: 200, + ...PIPELINING_SETUP_OPTS, minTxsPerBlock: 1, })); diff --git a/yarn-project/end-to-end/src/bench/tx_stats_bench.test.ts b/yarn-project/end-to-end/src/bench/tx_stats_bench.test.ts index 1d58f3a3e40e..4b0fda236914 100644 --- a/yarn-project/end-to-end/src/bench/tx_stats_bench.test.ts +++ b/yarn-project/end-to-end/src/bench/tx_stats_bench.test.ts @@ -20,14 +20,14 @@ import { } from 'zlib'; import { FullProverTest } from '../fixtures/e2e_prover_test.js'; +import { PIPELINING_SETUP_OPTS } from '../fixtures/fixtures.js'; import type { TestWallet } from '../test-wallet/test_wallet.js'; import { proveInteraction } from '../test-wallet/utils.js'; -// Set a 3 minute timeout. -const TIMEOUT = 300_000; +const REAL_PROOFS = !parseBooleanEnv(process.env.FAKE_PROOFS); +const TIMEOUT = REAL_PROOFS ? 45 * 60 * 1000 : 15 * 60 * 1000; describe('transaction benchmarks', () => { - const REAL_PROOFS = !parseBooleanEnv(process.env.FAKE_PROOFS); const COINBASE_ADDRESS = EthAddress.random(); const t = new FullProverTest('full_prover', 1, COINBASE_ADDRESS, REAL_PROOFS); @@ -55,7 +55,7 @@ describe('transaction benchmarks', () => { beforeAll(async () => { t.logger.warn(`Running suite with ${REAL_PROOFS ? 'real' : 'fake'} proofs`); - await t.setup(); + await t.setup({ ...PIPELINING_SETUP_OPTS }); ({ provenWallet, diff --git a/yarn-project/end-to-end/src/bench/utils.ts b/yarn-project/end-to-end/src/bench/utils.ts index 7daa924782df..78834612d929 100644 --- a/yarn-project/end-to-end/src/bench/utils.ts +++ b/yarn-project/end-to-end/src/bench/utils.ts @@ -11,6 +11,7 @@ import type { BenchmarkDataPoint, BenchmarkMetricsType, BenchmarkTelemetryClient import { mkdirSync, writeFileSync } from 'fs'; import path from 'path'; +import { PIPELINING_SETUP_OPTS } from '../fixtures/fixtures.js'; import { type EndToEndContext, type SetupOptions, setup } from '../fixtures/utils.js'; /** @@ -23,7 +24,7 @@ export async function benchmarkSetup( benchOutput?: string; }, ) { - const context = await setup(1, { ...opts, telemetryConfig: { benchmark: true } }); + const context = await setup(1, { ...PIPELINING_SETUP_OPTS, ...opts, telemetryConfig: { benchmark: true } }); const defaultAccountAddress = context.accounts[0]; const { contract } = await BenchmarkingContract.deploy(context.wallet).send({ from: defaultAccountAddress }); context.logger.info(`Deployed benchmarking contract at ${contract.address}`); diff --git a/yarn-project/end-to-end/src/shared/gas_portal_test_harness.ts b/yarn-project/end-to-end/src/shared/gas_portal_test_harness.ts index e894e77862a7..0d3285247727 100644 --- a/yarn-project/end-to-end/src/shared/gas_portal_test_harness.ts +++ b/yarn-project/end-to-end/src/shared/gas_portal_test_harness.ts @@ -8,10 +8,13 @@ import type { ExtendedViemWalletClient } from '@aztec/ethereum/types'; import { retryUntil } from '@aztec/foundation/retry'; import { FeeJuiceContract } from '@aztec/noir-contracts.js/FeeJuice'; import { ProtocolContractAddress } from '@aztec/protocol-contracts'; -import type { AztecNodeAdmin } from '@aztec/stdlib/interfaces/client'; +import type { AztecNodeAdmin, AztecNodeDebug } from '@aztec/stdlib/interfaces/client'; import { waitForL1ToL2MessageSeen } from './wait_for_l1_to_l2_message.js'; +/** Aztec node that may expose the debug mining API in local e2e setups. */ +type MiningAztecNode = AztecNode & Partial; + export interface IGasBridgingTestHarness { getL1FeeJuiceBalance(address: EthAddress): Promise; prepareTokensOnL1(owner: AztecAddress): Promise; @@ -21,7 +24,7 @@ export interface IGasBridgingTestHarness { } export interface FeeJuicePortalTestingHarnessFactoryConfig { - aztecNode: AztecNode; + aztecNode: MiningAztecNode; aztecNodeAdmin?: AztecNodeAdmin; l1Client: ExtendedViemWalletClient; wallet: Wallet; @@ -77,7 +80,7 @@ export class GasBridgingTestHarness implements IGasBridgingTestHarness { constructor( /** Aztec node */ - public aztecNode: AztecNode, + public aztecNode: MiningAztecNode, /** Aztec node admin interface */ public aztecNodeAdmin: AztecNodeAdmin | undefined, /** Wallet. */ @@ -164,6 +167,11 @@ export class GasBridgingTestHarness implements IGasBridgingTestHarness { } private async advanceL2Block() { + if (this.aztecNode.mineBlock) { + await this.aztecNode.mineBlock(); + return; + } + const initialBlockNumber = await this.aztecNode.getBlockNumber(); let minTxsPerBlock = undefined; @@ -172,10 +180,12 @@ export class GasBridgingTestHarness implements IGasBridgingTestHarness { await this.aztecNodeAdmin.setConfig({ minTxsPerBlock: 0 }); // Set to 0 to ensure we can advance the block } - await retryUntil(async () => (await this.aztecNode.getBlockNumber()) >= initialBlockNumber + 1); - - if (this.aztecNodeAdmin && minTxsPerBlock !== undefined) { - await this.aztecNodeAdmin.setConfig({ minTxsPerBlock }); + try { + await retryUntil(async () => (await this.aztecNode.getBlockNumber()) >= initialBlockNumber + 1); + } finally { + if (this.aztecNodeAdmin && minTxsPerBlock !== undefined) { + await this.aztecNodeAdmin.setConfig({ minTxsPerBlock }); + } } } } From 11fd9f9545e882e4037b2f1cb0ef65aaf3455872 Mon Sep 17 00:00:00 2001 From: Michal Rzeszutko Date: Fri, 29 May 2026 10:57:50 +0200 Subject: [PATCH 04/24] fix(p2p): fall back to archiver in BLOCK_TXS response validation (#23624) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary `Libp2pService.validateRequestedBlockTxsConsistency` rejects every BLOCK_TXS reqresp response for any block whose proposal is not in the local attestation pool. The responder handler already falls back to the archiver in this case, the validator did not. So any node that doesn't have the proposal locally — but does know the block from the archiver — cannot collect its missing txs, and instead storms its peers until it is rate-limited and disconnected. This PR teaches the validator the same proposal-or-archiver fallback the handler already uses, breaking the storm at its source. ## The defect: validator/handler asymmetry To verify that a peer's BLOCK_TXS response matches the block, the validator needs the canonical tx-hash list for the block. Until this PR it consulted only the attestation pool: ```ts // yarn-project/p2p/src/services/libp2p/libp2p_service.ts (pre-fix) const proposal = await this.mempools.attestationPool.getBlockProposalByArchive(...); if (proposal) { /* check membership/order */ } else { return false; } ``` The responder handler (`p2p/src/services/reqresp/protocols/block_txs/block_txs_handler.ts:40-45`) already serves from either source: ```ts let txHashes = (await attestationPool.getBlockProposalByArchive(...))?.txHashes; if (!txHashes) { txHashes = (await archiver.getBlock({ archive: request.archiveRoot }))?.body.txEffects.map(e => e.txHash); } ``` So peers can produce valid responses for blocks that are only known via the archiver, but the validator at the other end rejects them. ## When this fires Any p2p-enabled node subscribes to `block_proposal` gossip (`libp2p_service.ts:575`) and stores received proposals into its attestation pool (`validateAndStoreBlockProposal` at `libp2p_service.ts:1236`, calling `tryAddBlockProposal` at line 1252). Neither subscription nor storage is gated by `disableValidator` — that flag only controls the validator-client (attestation signing). So in the steady state, a node that was online when a proposal was gossiped does have it locally. The validator's lookup fails whenever the node lacks the proposal in its local attestation pool, yet still needs to collect the block's txs over reqresp. The real-world triggers we've seen and can describe: - **A node joins the mesh late** and misses the proposal gossip for blocks that were proposed before it arrived. This was originally noticed during an e2e run where mesh formation was slower than usual, and it's the scenario the e2e test in this PR reproduces. - A prover-node calling `ProverNode.gatherTxs` (`prover-node/src/prover-node.ts:330`) → `TxProvider.getTxsForBlock` → `TxCollection.collectFastFor({type:'block', ...})` → `BatchTxRequester` for any block whose proposal it doesn't happen to hold: prover restart, gossip drop, mesh churn during the epoch, etc. In every case the prover (or any node) has the mined block in its archiver but no proposal in its attestation pool. Until this PR the validator only consulted the attestation pool, so every otherwise-valid response was rejected. ## The self-inflicted ban-storm In `BatchTxRequester` (`p2p/src/services/reqresp/batch-tx-requester/batch_tx_requester.ts`): ```ts // line 432-438: every response gets rejected because validation has no way to validate it const isValid = await this.p2pService.validateRequestedBlockTxsConsistency(...); if (!isValid) { this.handleFailResponseFromPeer(peerId, ReqRespStatus.INTERNAL_ERROR); return; } // line 461-481: INTERNAL_ERROR correctly does not penalize the peer, but // also does not back off — the dumb worker loop just rotates to the next peer: if (responseStatus === ReqRespStatus.NOT_FOUND || responseStatus === ReqRespStatus.INTERNAL_ERROR) { this.peers.markPeerDumb(peerId); this.txsMetadata.clearPeerData(peerId); return; } ``` The dumb worker loop (`batch_tx_requester.ts:261-304`) has no inter-iteration sleep and ten parallel workers (`dumbParallelWorkerCount: 10`) round-robin the peers. Per-peer in-flight de-duplication caps it at one request in flight per peer, but the steady-state hit-rate per peer easily exceeds the responder's per-peer GCRA cap. The penalty arrives from the **responder** side, on the requester: ```ts // rate_limiter.ts:214-225 if (rateLimitStatus === RateLimitStatus.DeniedPeer) { this.peerScoring.penalizePeer(peerId, PeerErrorSeverity.HighToleranceError); } ``` ``` BLOCK_TXS per-peer cap : 10 req / 1000 ms (rate_limits.ts:55-65) HighToleranceError : -2 score points (peer_scoring.ts:34-36) disconnect threshold : -50 (peer_scoring.ts:57) ban threshold : -100 (peer_scoring.ts:56) ``` With ~1 GCRA denial per second per peer, the prover loses 2 points/sec at each responder. The first responder hits **-50 in ~25 s** and goodbyes the prover via `peer_manager.ts:601-603 pruneUnhealthyPeers` (`GoodByeReason.LOW_SCORE`); **-100 in ~50 s** would ban. ## Fix A four-line change in `yarn-project/p2p/src/services/libp2p/libp2p_service.ts`: fall back to the archiver after the attestation-pool lookup, mirroring the responder handler. If neither source has the block we still return `false`without penalising the peer (it really is unverifiable locally). ```ts const proposal = await this.mempools.attestationPool.getBlockProposalByArchive(...); const blockTxHashes = proposal?.txHashes ?? (await this.archiver.getBlock({ archive: request.archiveRoot }))?.body.txEffects.map(e => e.txHash); if (blockTxHashes) { /* existing membership/order check, against blockTxHashes */ } else { /* unchanged: log warn, return false, no penalty */ } ``` The validator's other checks (archive-root match, bitvector length, no dupes, size bounds, subset-membership, ordering) are unchanged. ## Tests **Unit anchor** — `p2p/src/services/libp2p/libp2p_service.test.ts` - New test: *"should accept when the proposal is missing but the block is known via the archiver"* — verified red before the fix (`Expected: true, Received: false`) and green after. - Existing test renamed and tightened to *"should reject without penalising when the block is unknown (no proposal and not in the archiver)"* — covers the still-correct rejection path. - All 46 tests in the file pass. **End-to-end** — `end-to-end/src/e2e_p2p/late_prover_tx_collection.test.ts` Validators form a mesh and mine a block carrying real txs; a prover joins **after** the block is mined, so it has the block in its archiver but never received the proposal or txs over gossip. The test then drives the exact production path the prover would take to gather txs for proving: ```ts const txCollection = (proverNode as ...).p2pClient.txCollection; const collected = await txCollection.collectFastForBlock(minedBlock, blockTxHashes, { deadline }); expect(collected.map(t => t.getTxHash().toString()).sort()) .toEqual(blockTxHashes.map(h => h.toString()).sort()); ``` - Red on the unfixed source: `collected.length === 0` (validation rejects every response, dumb loop runs until the deadline), assertion fails. - Green with the fix: all of `block.body.txEffects`'s txs are collected. --- .../e2e_p2p/late_prover_tx_collection.test.ts | 165 ++++++++++++++++++ .../services/libp2p/libp2p_service.test.ts | 36 +++- .../p2p/src/services/libp2p/libp2p_service.ts | 23 ++- 3 files changed, 214 insertions(+), 10 deletions(-) create mode 100644 yarn-project/end-to-end/src/e2e_p2p/late_prover_tx_collection.test.ts diff --git a/yarn-project/end-to-end/src/e2e_p2p/late_prover_tx_collection.test.ts b/yarn-project/end-to-end/src/e2e_p2p/late_prover_tx_collection.test.ts new file mode 100644 index 000000000000..2f644ad4f6a6 --- /dev/null +++ b/yarn-project/end-to-end/src/e2e_p2p/late_prover_tx_collection.test.ts @@ -0,0 +1,165 @@ +import type { AztecNodeService } from '@aztec/aztec-node'; +import { waitForTx } from '@aztec/aztec.js/node'; +import { BlockNumber } from '@aztec/foundation/branded-types'; +import { retryUntil } from '@aztec/foundation/retry'; +import { tryStop } from '@aztec/stdlib/interfaces/server'; +import type { Tx } from '@aztec/stdlib/tx'; + +import { jest } from '@jest/globals'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +import { shouldCollectMetrics } from '../fixtures/fixtures.js'; +import { ATTESTER_PRIVATE_KEYS_START_INDEX, createNodes, createProverNode } from '../fixtures/setup_p2p_test.js'; +import { P2PNetworkTest, SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES, WAIT_FOR_TX_TIMEOUT } from './p2p_network.js'; +import { submitTransactions } from './shared.js'; + +// A prover joins the mesh after a block has been mined and its proposal/tx gossip already happened, +// so it knows the block from its archiver (via L1 sync) but has no proposal locally and is missing +// the block's txs. It must fetch them from its (dumb) peers over reqresp BLOCK_TXS — the same path +// ProverNode.gatherTxs takes when preparing an epoch proof — and the test asserts the prover ends up +// holding all of the block's txs. + +const NUM_VALIDATORS = 4; +const NUM_TXS = 2; +const BOOT_NODE_UDP_PORT = process.env.BOOT_NODE_UDP_PORT ? parseInt(process.env.BOOT_NODE_UDP_PORT) : 4900; +const AZTEC_SLOT_DURATION = 12; +const AZTEC_EPOCH_DURATION = 4; + +const DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), 'late-prover-')); + +jest.setTimeout(1000 * 60 * 10); + +describe('e2e_p2p_late_prover_tx_collection', () => { + let t: P2PNetworkTest; + let nodes: AztecNodeService[] = []; + let proverNode: AztecNodeService | undefined; + + beforeEach(async () => { + t = await P2PNetworkTest.create({ + testName: 'e2e_p2p_late_prover_tx_collection', + numberOfNodes: 0, + numberOfValidators: NUM_VALIDATORS, + basePort: BOOT_NODE_UDP_PORT, + metricsPort: shouldCollectMetrics(), + startProverNode: false, // we start our own prover, late, after a block is mined + initialConfig: { + ...SHORTENED_BLOCK_TIME_CONFIG_NO_PRUNES, + aztecSlotDuration: AZTEC_SLOT_DURATION, + aztecEpochDuration: AZTEC_EPOCH_DURATION, + listenAddress: '127.0.0.1', + enableProposerPipelining: true, + // Only build blocks that actually carry txs, so the chain idles after our block is mined and + // the late prover is never auto-triggered to collect for a different block. + minTxsPerBlock: 1, + inboxLag: 2, + }, + }); + + await t.setup(); + await t.applyBaseSetup(); + }); + + afterEach(async () => { + await tryStop(proverNode); + await t.stopNodes(nodes); + await t.teardown(); + for (let i = 0; i < NUM_VALIDATORS; i++) { + fs.rmSync(`${DATA_DIR}-${i}`, { recursive: true, force: true, maxRetries: 3 }); + } + fs.rmSync(`${DATA_DIR}-late-prover`, { recursive: true, force: true, maxRetries: 3 }); + }); + + it("lets a late-joining prover collect a mined block's txs from dumb peers when it has no local proposal", async () => { + if (!t.bootstrapNodeEnr) { + throw new Error('Bootstrap node ENR is not available'); + } + + // 1. Stand up the validator network and let it form the mesh. + t.logger.info('Creating validator nodes'); + nodes = await createNodes( + t.ctx.aztecNodeConfig, + t.ctx.dateProvider, + t.bootstrapNodeEnr, + NUM_VALIDATORS, + BOOT_NODE_UDP_PORT, + t.genesis, + DATA_DIR, + shouldCollectMetrics(), + ); + await t.waitForP2PMeshConnectivity(nodes, NUM_VALIDATORS); + await t.setupAccount(); + + // 2. Submit txs and wait for them to be mined into a block. The validators end up holding the + // proposal, the mined block, and the txs in their pools — everything the prover will be missing. + t.logger.info('Submitting transactions and waiting for them to be mined'); + const txHashes = await submitTransactions(t.logger, nodes[0], NUM_TXS, t.fundedAccount); + await Promise.all(txHashes.map(h => waitForTx(nodes[0], h, { timeout: WAIT_FOR_TX_TIMEOUT }))); + + const receipt = await nodes[0].getTxReceipt(txHashes[0]); + const minedBlockNumber = BlockNumber(receipt.blockNumber!); + const minedBlock = await nodes[0].getBlockSource().getBlock({ number: minedBlockNumber }); + if (!minedBlock) { + throw new Error(`Mined block ${minedBlockNumber} not found on validator`); + } + const blockTxHashes = minedBlock.body.txEffects.map(e => e.txHash); + t.logger.info(`Block ${minedBlockNumber} mined with ${blockTxHashes.length} txs`); + + // 3. Start the prover LATE: after the block was mined and its proposal/tx gossip already happened. + // It learns the mined block via L1/archiver sync, but never received the proposal or the txs. + t.logger.info('Creating late-joining prover node'); + ({ proverNode } = await createProverNode( + t.ctx.aztecNodeConfig, + BOOT_NODE_UDP_PORT + NUM_VALIDATORS + 1, + t.bootstrapNodeEnr, + ATTESTER_PRIVATE_KEYS_START_INDEX + NUM_VALIDATORS + 1, + { dateProvider: t.ctx.dateProvider }, + t.genesis, + `${DATA_DIR}-late-prover`, + shouldCollectMetrics(), + )); + + // The prover syncs the mined block from L1 (no peers required), so wait for both the archive sync + // and for it to actually peer with the network before driving reqresp collection. + await retryUntil( + async () => (await proverNode!.getBlockNumber()) >= minedBlockNumber, + 'prover to sync the mined block via L1', + 60, + 1, + ); + await retryUntil( + async () => (await proverNode!.getP2P().getPeers()).length >= 2, + 'prover to connect to peers', + 60, + 1, + ); + + // Sanity check: the prover does not have the block's txs (it joined after they were gossiped, and + // gossip is not replayed to late joiners). The archiver only carries tx effects, not full txs. + const txsBeforeCollection = await proverNode.getTxsByHash(blockTxHashes); + expect(txsBeforeCollection.length).toBe(0); + + // 4. Drive the exact collection ProverNode.gatherTxs performs to prove the block: fetch the missing + // txs over reqresp BLOCK_TXS from dumb peers, running the prover's real response validation + // against its own (empty) attestation pool and its archiver. + // + // Deadline must be computed against the same DateProvider the prover uses (advanced by the + // harness's cheatCodes). Using Date.now() would land in the past from the prover's view of time, + // and collectFastFor would short-circuit with timeout <= 0. + const txCollection = (proverNode as unknown as { p2pClient: { txCollection: TxCollectionLike } }).p2pClient + .txCollection; + const collected = await txCollection.collectFastForBlock(minedBlock, blockTxHashes, { + deadline: new Date(t.ctx.dateProvider.now() + AZTEC_SLOT_DURATION * 1000 * 4), + }); + + const collectedHashes = collected.map(tx => tx.getTxHash().toString()).sort(); + const expectedHashes = blockTxHashes.map(h => h.toString()).sort(); + expect(collectedHashes).toEqual(expectedHashes); + }); +}); + +/** Minimal shape of the (otherwise private) TxCollection we reach into for this test. */ +interface TxCollectionLike { + collectFastForBlock(block: unknown, txHashes: unknown[], opts: { deadline: Date }): Promise; +} diff --git a/yarn-project/p2p/src/services/libp2p/libp2p_service.test.ts b/yarn-project/p2p/src/services/libp2p/libp2p_service.test.ts index 3aaf4c97b9ce..5f2ce4391d23 100644 --- a/yarn-project/p2p/src/services/libp2p/libp2p_service.test.ts +++ b/yarn-project/p2p/src/services/libp2p/libp2p_service.test.ts @@ -5,7 +5,7 @@ import { Secp256k1Signer } from '@aztec/foundation/crypto/secp256k1-signer'; import { Fr } from '@aztec/foundation/curves/bn254'; import { type Logger, createLogger } from '@aztec/foundation/log'; import { openTmpStore } from '@aztec/kv-store/lmdb'; -import type { L2BlockSource } from '@aztec/stdlib/block'; +import type { L2Block, L2BlockSource } from '@aztec/stdlib/block'; import type { ContractDataSource } from '@aztec/stdlib/contract'; import { GasFees } from '@aztec/stdlib/gas'; import type { ClientProtocolCircuitVerifier } from '@aztec/stdlib/interfaces/server'; @@ -328,6 +328,13 @@ describe('LibP2PService', () => { return new BlockTxsResponse(archiveRoot, txs as TxArray, BitVector.init(length, indices)); } + /** Builds a minimal archived block whose txEffects carry the given tx hashes, for archiver-fallback tests. */ + function makeArchivedBlock(txHashes: string[]): L2Block { + return { + body: { txEffects: txHashes.map(h => ({ txHash: { toString: () => h } })) }, + } as unknown as L2Block; + } + /** Sets up the mempools with a mock attestation pool that returns a proposal with given tx hashes. */ function setProposalTxHashes(svc: TestLibP2PService, txHashes: string[]): void { // Create a partial mock of the attestation pool that only implements getBlockProposalByArchive. @@ -483,22 +490,45 @@ describe('LibP2PService', () => { expect(mockPeerManager.penalizePeer).toHaveBeenCalledWith(mockPeerId, PeerErrorSeverity.LowToleranceError); }); - it('should reject without penalizing when proposal is missing', async () => { + it('should reject without penalizing when the block is unknown (no proposal and not in the archiver)', async () => { const hash = Fr.random(); // Simple valid shape that should pass pre-checks const request = makeRequest(hash, 3, [0, 2]); const response = makeResponse(hash, 3, [0, 2], ['0xgood0']); - // No proposal available - mock attestationPool to return undefined + // Neither the attestation pool nor the archiver knows this block, so we cannot verify the + // membership/order of the returned txs. This is not a peer fault, so no penalty is applied. const mockAttestationPool: MockAttestationPoolForTests = { getBlockProposalByArchive: (_: string) => Promise.resolve(undefined), }; service.setAttestationPool(mockAttestationPool); + mockArchiver.getBlock.mockResolvedValue(undefined); const ok = await service.validateRequestedBlockTxsConsistency(request, response, mockPeerId); expect(ok).toBe(false); expect(mockPeerManager.penalizePeer).not.toHaveBeenCalled(); }); + + // Regression test for the tx-collection ban-storm: a prover (or any node) collecting txs to + // prove an already-mined block has no block proposal in its attestation pool, but it does know + // the block via the archiver. The validator must fall back to the archiver (as the responder + // handler does) so it can accept valid responses instead of rejecting every one and storming peers. + it('should accept when the proposal is missing but the block is known via the archiver', async () => { + const hash = Fr.random(); + const request = makeRequest(hash, 5, [0, 2, 4]); + const response = makeResponse(hash, 5, [0, 2, 4], ['0xgood0', '0xgood2', '0xgood4']); + + // No proposal (the prover never received it), but the mined block is in the archiver. + service.setAttestationPool({ getBlockProposalByArchive: (_: string) => Promise.resolve(undefined) }); + mockArchiver.getBlock.mockResolvedValue( + makeArchivedBlock(['0xgood0', '0xother1', '0xgood2', '0xother3', '0xgood4']), + ); + + const ok = await service.validateRequestedBlockTxsConsistency(request, response, mockPeerId); + expect(ok).toBe(true); + expect(mockPeerManager.penalizePeer).not.toHaveBeenCalled(); + expect(mockArchiver.getBlock).toHaveBeenCalledWith({ archive: hash }); + }); }); describe('processBlockFromPeer', () => { diff --git a/yarn-project/p2p/src/services/libp2p/libp2p_service.ts b/yarn-project/p2p/src/services/libp2p/libp2p_service.ts index 2da0cdecebea..688e7798c8e1 100644 --- a/yarn-project/p2p/src/services/libp2p/libp2p_service.ts +++ b/yarn-project/p2p/src/services/libp2p/libp2p_service.ts @@ -1560,18 +1560,26 @@ export class LibP2PService extends WithTracer implements P2PService { ); } - // Given proposal (should have locally), ensure returned txs are valid subset and match request indices + // To verify membership/order of the returned txs we need the canonical tx hash list for the + // block. Prefer the block proposal (held while a block is in flight), but fall back to the + // archiver for blocks we only know as mined — e.g. a prover collecting txs to prove a block it + // never received a proposal for. This mirrors the responder side (reqRespBlockTxsHandler), + // which serves from proposal-or-archiver. const proposal = await this.mempools.attestationPool.getBlockProposalByArchive(request.archiveRoot.toString()); - if (proposal) { + const blockTxHashes = + proposal?.txHashes ?? + (await this.archiver.getBlock({ archive: request.archiveRoot }))?.body.txEffects.map(e => e.txHash); + + if (blockTxHashes) { // Build intersected indices const intersectIdx = request.txIndices.getTrueIndices().filter(i => response.txIndices.isSet(i)); // Enforce subset membership and preserve increasing order by index. - const hashToIndexInProposal = new Map( - proposal.txHashes.map((h, i) => [h.toString(), i] as [string, number]), + const hashToIndexInBlock = new Map( + blockTxHashes.map((h, i) => [h.toString(), i] as [string, number]), ); const allowedIndexSet = new Set(intersectIdx); - const indices = returnedHashes.map(h => hashToIndexInProposal.get(h)); + const indices = returnedHashes.map(h => hashToIndexInBlock.get(h)); const allAllowed = indices.every(idx => idx !== undefined && allowedIndexSet.has(idx)); const strictlyIncreasing = indices.every((idx, i) => (i === 0 ? idx !== undefined : idx! > indices[i - 1]!)); if (!allAllowed || !strictlyIncreasing) { @@ -1579,9 +1587,10 @@ export class LibP2PService extends WithTracer implements P2PService { throw new ValidationError('Returned txs do not match expected subset/order for requested indices'); } } else { - // No local proposal, cannot check the membership/order of the returned txs + // Neither a local proposal nor an archived block: we cannot verify membership/order of the + // returned txs. This is a local-state gap, not a peer fault, so we do not penalize. this.logger.warn( - `Block proposal not found for archive root ${request.archiveRoot.toString()}; cannot validate membership/order of returned txs`, + `Block ${request.archiveRoot.toString()} not found in attestation pool or archiver; cannot validate membership/order of returned txs`, ); return false; } From 73e45e4acb9a69a089469c7bbaea5cc8c5fc4c8e Mon Sep 17 00:00:00 2001 From: PhilWindle <60546371+PhilWindle@users.noreply.github.com> Date: Fri, 29 May 2026 10:12:54 +0100 Subject: [PATCH 05/24] docs(slashing): align operator and slasher docs with AZIP-7 (#23494) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Closes [A-970](https://linear.app/aztec-labs/issue/A-970). Refresh the operator-facing slashing-configuration guide and the slasher README to match the AZIP-7 end-state, now that the implementation work for AZIP-7 has landed across the Slashing Post-Alpha Improvements project. Operator docs (\`docs/docs-operate/operators/sequencer-management/slashing-configuration.md\`): - Remove the obsolete \"Valid Epoch Not Proven\" section. \`SLASH_PRUNE_PENALTY\` is gone with it. - Rewrite \"Data Withholding\" for the end-of-slot detection rule and add the matching \`SLASH_DATA_WITHHOLDING_TOLERANCE_SLOTS\` env var. - Update \"Inactivity\" to mention end-of-epoch evaluation (no longer waits for proven) and re-execution-based fault attribution. - Flip the descendant offense section to proposer-fault framing to match the rename in #23468. - Add sections for the new offenses: broadcasted invalid block proposal, broadcasted invalid checkpoint proposal, attesting to an invalid checkpoint proposal, duplicate proposal, duplicate attestation. - Sync the env-vars block and the offense-detection bullet list with the current set of watchers. - Convert touched section headings to sentence case per docs style. Slasher README (\`yarn-project/slasher/README.md\`): - Add a note under \`BROADCASTED_INVALID_CHECKPOINT_PROPOSAL\` to make the AZIP-7 \"submitting block proposal after checkpoint\" mapping explicit. That AZIP offense is detected via the existing invalid-checkpoint watcher (a late block makes the prior checkpoint retroactively invalid) rather than having its own offense type. Stacked on #23468 (the \`ATTESTED_DESCENDANT_OF_INVALID\` → \`PROPOSED_DESCENDANT_OF_CHECKPOINT_WITH_INVALID_ATTESTATIONS\` rename) because the new env var name only exists once that PR lands. ## Test plan - \`cd docs && yarn spellcheck\` (clean) - Visual review of the rewritten \"Slashable offenses\" section against the [AZIP-7 spec](https://github.com/AztecProtocol/governance/blob/main/AZIPs/azip-7-update_slashing.md) --- .../slashing-configuration.md | 171 +++++++----------- yarn-project/slasher/README.md | 2 +- 2 files changed, 62 insertions(+), 111 deletions(-) diff --git a/docs/docs-operate/operators/sequencer-management/slashing-configuration.md b/docs/docs-operate/operators/sequencer-management/slashing-configuration.md index 53a8d0008b62..9004789eb74f 100644 --- a/docs/docs-operate/operators/sequencer-management/slashing-configuration.md +++ b/docs/docs-operate/operators/sequencer-management/slashing-configuration.md @@ -55,157 +55,109 @@ The L1 contract defines three fixed slashing tiers that can be configured for di On the current network, **all offenses are currently configured to slash 2,000 tokens (1% of the Activation Threshold - the minimum stake required to join the validator set)**. With the ejection threshold at 98%, validators can be slashed a maximum of **3 times** (totaling 3% of their Activation Threshold) before being automatically ejected from the validator set. ::: -## Slashable Offenses +## Slashable offenses -Your sequencer automatically detects and votes to slash the following offenses: +Your sequencer automatically detects and votes to slash the following offenses. The set of offenses, and the rationale behind them, is specified in [AZIP-7](https://github.com/AztecProtocol/governance/blob/main/AZIPs/azip-7-update_slashing.md). ### 1. Inactivity -**What it is**: A validator fails to attest to block proposals when selected for committee duty, or fails to propose a block when selected as proposer. +**What it is**: A validator fails to attest to checkpoint proposals when selected for committee duty, or fails to produce checkpoints and block proposals when selected as proposer. **Detection criteria**: -- Measured **per epoch** for validators on the committee during that epoch (committees are assigned per epoch and remain constant for all slots in that epoch) -- The Sentinel calculates: `(missed_proposals + missed_attestations) / (total_proposals + total_attestations)` -- A validator is considered inactive for an epoch if this ratio meets or exceeds `SLASH_INACTIVITY_TARGET_PERCENTAGE` (e.g., 0.8 = 80% or more duties missed) -- Requires **consecutive committee participation with inactivity**: Must be inactive for N consecutive epochs where they were on the committee (configured via `SLASH_INACTIVITY_CONSECUTIVE_EPOCH_THRESHOLD=2`). Epochs where the validator was not on the committee are not counted, so a validator inactive in epochs 1, 3, and 5 meets the threshold for 3 consecutive inactive epochs even though epochs 2 and 4 are skipped. +- Measured per epoch by the Sentinel for validators on the committee during that epoch. +- Evaluated at the end of each epoch (plus a small buffer) without waiting for the epoch to be proven on L1, so inactive validators can be slashed regardless of prover availability. +- Block re-execution is used to attribute fault between proposers and attestors based on what actually happened in each slot, rather than using attestation count as a proxy. +- A validator is considered inactive for an epoch if their failure ratio meets or exceeds `SLASH_INACTIVITY_TARGET_PERCENTAGE`. +- Requires consecutive committee participation with inactivity: must be inactive for N consecutive epochs where they were on the committee (configured via `SLASH_INACTIVITY_CONSECUTIVE_EPOCH_THRESHOLD`). Epochs where the validator was not on the committee are not counted, so a validator inactive in epochs 1, 3, and 5 meets the threshold for 3 consecutive inactive epochs even though epochs 2 and 4 are skipped. -**Proposed penalty**: 1% of stake +**Note**: Requires the Sentinel to be enabled (`SENTINEL_ENABLED=true`). -**Note**: Requires the Sentinel to be enabled (`SENTINEL_ENABLED=true`). The Sentinel tracks attestation and proposal activity for all validators. +### 2. Data withholding -### 2. Valid Epoch Not Proven - -**What it is**: An epoch was not proven within the proof submission window, even though all data was available and the epoch was valid. +**What it is**: After a checkpoint is published, the transactions it contains were not made available on the P2P network within the tolerance window. **Detection criteria**: -- An epoch gets pruned (removed from the chain) -- Your node can re-execute all transactions from that epoch -- The state roots match the original epoch (indicating it could have been proven) +- Once `SLASH_DATA_WITHHOLDING_TOLERANCE_SLOTS` full L2 slots have elapsed past the checkpoint's slot, your node checks whether it has all the transactions for that checkpoint in its local mempool. +- If any are missing, the validators who attested to the checkpoint are flagged. +- The check runs regardless of whether the epoch is eventually proven. Slashing still applies if the data was withheld, to prevent committees from striking side deals with specific provers by releasing data only to them. -**Proposed penalty**: 0% (disabled for initial deployment) +**Responsibility**: Validators who attested to the checkpoint. -**Responsibility**: The entire committee of the pruned epoch is slashed. +### 3. Broadcasted invalid block proposal -### 3. Data Withholding +**What it is**: A proposer broadcast an invalid block proposal over the P2P network. -**What it is**: The committee failed to make transaction data publicly available, preventing the epoch from being proven. +**Detection criteria**: Detected by validators during proposal validation, for example when a transaction in the proposal fails validation or the proposed block header is structurally invalid. -**Detection criteria**: -- An epoch gets pruned -- Your node cannot obtain all the transactions needed to re-execute the epoch -- The data was not propagated to the sequencer set before the proof submission window ended +**Responsibility**: The proposer who broadcast the invalid block. -**Proposed penalty**: 0% (disabled for initial deployment) +### 4. Broadcasted invalid checkpoint proposal -**Responsibility**: The entire committee from the pruned epoch is slashed for failing to propagate data. +**What it is**: A proposer broadcast an invalid checkpoint proposal over the P2P network. This includes the AZIP-7 "submitting a block proposal after the checkpoint" case, because a later block signed by the same proposer in the same slot makes the prior checkpoint retroactively invalid. -### 4. Proposed Insufficient Attestations +**Detection criteria**: Detected when the checkpoint terminates before a higher-index block proposal signed by the same proposer in the same slot, when the signed header does not match deterministic validator recomputation, or when the fee asset price modifier is malformed. -**What it is**: A proposer submitted a block to L1 without collecting enough valid committee attestations. - -**Detection criteria**: -- Block published to L1 has fewer than 2/3 + 1 attestations from the committee -- Your node detects this through L1 block validation +**Responsibility**: The proposer who broadcast the invalid checkpoint. -**Proposed penalty**: 1% of stake +### 5. Proposed insufficient attestations -### 5. Proposed Incorrect Attestations - -**What it is**: A proposer submitted a block with invalid signatures or signatures from non-committee members. +**What it is**: A proposer submitted a block to L1 without collecting enough valid committee attestations. **Detection criteria**: -- Block contains attestations with invalid ECDSA signatures -- Block contains signatures from addresses not in the committee +- Block published to L1 has fewer than 2/3 + 1 attestations from the committee. +- Your node detects this through L1 block validation. -**Proposed penalty**: 1% of stake +**Responsibility**: The proposer who published the block. -### 6. Attested to Descendant of Invalid Block +### 6. Proposed incorrect attestations -**What it is**: A validator attested to a block that builds on top of an invalid block. +**What it is**: A proposer submitted a block with invalid signatures, or signatures from non-committee members. **Detection criteria**: -- A validator attests to block B -- Block B's parent block has invalid or insufficient attestations -- Your node has previously identified the parent as invalid +- Block contains attestations with invalid ECDSA signatures. +- Block contains signatures from addresses not in the committee. -**Proposed penalty**: 1% of stake +**Responsibility**: The proposer who published the block. -**Note**: Validators should only attest to blocks that build on valid chains with proper attestations. +### 7. Proposed descendant of checkpoint with invalid attestations -## Configuring Your Sequencer for Slashing +**What it is**: A proposer published a checkpoint to L1 that builds on an earlier checkpoint with invalid or insufficient attestations. -The slashing module runs automatically when your sequencer is enabled. You can configure its behavior using environment variables or the node's admin API. Remember to enable the Sentinel if you want to detect inactivity offenses. +**Detection criteria**: +- Your node has previously identified a checkpoint as having invalid or insufficient attestations. +- A later proposer publishes a descendant checkpoint to L1 on top of it. -### Environment Variables +**Responsibility**: The proposer of the descendant checkpoint. Under pipelining, the next proposer may have started building optimistically before the prior checkpoint's signatures were submitted to L1, so only the proposer who actually publishes the descendant checkpoint to L1 is slashed. -Your sequencer comes pre-configured with default slashing settings. You can optionally override these defaults by setting environment variables before starting your node. +### 8. Attested to invalid checkpoint proposal -**Default configuration:** +**What it is**: A committee member attested to a checkpoint proposal in a slot where your node detected a slashable invalid block proposal. -```bash -# Grace period - offenses during the first N slots are not slashed -SLASH_GRACE_PERIOD_L2_SLOTS=128 # Default: first round is grace period +**Detection criteria**: +- Your node detected an invalid block proposal for the slot via re-execution. +- A committee member subsequently attested to a checkpoint covering that slot. -# Inactivity detection (requires SENTINEL_ENABLED=true) -SLASH_INACTIVITY_TARGET_PERCENTAGE=0.8 # Slash if missed proposals + attestations >= 80% -SLASH_INACTIVITY_CONSECUTIVE_EPOCH_THRESHOLD=2 # Must be inactive for 2+ epochs -SLASH_INACTIVITY_PENALTY=2000000000000000000000 # 2000 tokens (1%) +**Responsibility**: Committee members who attested in the invalid proposal slot. -# Sentinel configuration (required for inactivity detection) -SENTINEL_ENABLED=true # Must be true to detect inactivity offenses -SENTINEL_HISTORY_LENGTH_IN_EPOCHS=100 # Track 100 epochs of history +### 9. Duplicate proposal -# Epoch prune and data withholding penalties (disabled by default) -SLASH_PRUNE_PENALTY=0 # Set to >0 to enable -SLASH_DATA_WITHHOLDING_PENALTY=0 # Set to >0 to enable +**What it is**: A proposer broadcast multiple block or checkpoint proposals for the same position with different content (equivocation). -# Invalid attestations and blocks -SLASH_PROPOSE_INVALID_ATTESTATIONS_PENALTY=2000000000000000000000 # 2000 tokens -SLASH_PROPOSE_DESCENDANT_OF_CHECKPOINT_WITH_INVALID_ATTESTATIONS_PENALTY=2000000000000000000000 # 2000 tokens -SLASH_INVALID_BLOCK_PENALTY=2000000000000000000000 # 2000 tokens +**Detection criteria**: Detected at the P2P layer by the AttestationPool, which tracks proposals by position (slot plus `indexWithinCheckpoint` for blocks, or slot for checkpoints). A second proposal for the same position with a different archive flags the duplicate. -# Offense expiration -SLASH_OFFENSE_EXPIRATION_ROUNDS=4 # Offenses older than 4 rounds are dropped +**Responsibility**: The proposer who broadcast the duplicate proposal. -# Execution behavior -SLASH_EXECUTE_ROUNDS_LOOK_BACK=4 # Check 4 rounds back for executable slashing rounds -``` +### 10. Duplicate attestation -### Runtime Configuration via API +**What it is**: A validator signed attestations for different proposals at the same slot (equivocation). -You can update slashing configuration while your node is running using the `nodeAdmin_setConfig` method: +**Detection criteria**: Detected at the P2P layer when conflicting attestations are observed from the same signer for the same slot. -**CLI Method**: +**Responsibility**: The attestor. -```bash -curl -X POST http://localhost:8880 \ - -H 'Content-Type: application/json' \ - -d '{ - "jsonrpc":"2.0", - "method":"nodeAdmin_setConfig", - "params":[{ - "slashInactivityPenalty":"2000000000000000000000", - "slashInactivityTargetPercentage":0.9 - }], - "id":1 - }' -``` - -**Docker Method**: +## Configuring Your Sequencer for Slashing -```bash -docker exec -it aztec-sequencer curl -X POST http://localhost:8880 \ - -H 'Content-Type: application/json' \ - -d '{ - "jsonrpc":"2.0", - "method":"nodeAdmin_setConfig", - "params":[{ - "slashInactivityPenalty":"2000000000000000000000", - "slashInactivityTargetPercentage":0.9 - }], - "id":1 - }' -``` +The slashing module runs automatically when your sequencer is enabled. ### Excluding Validators from Slashing @@ -251,19 +203,18 @@ docker exec -it aztec-sequencer curl -X POST http://localhost:8880 \ }' ``` -Look for fields starting with `slash` in the response to verify your settings. - -## How Automatic Slashing Works +## Automatic Slashing -Once configured, your sequencer handles slashing automatically: +Your sequencer handles slashing automatically: ### 1. Continuous Offense Detection Watchers run in the background, monitoring: -- Block attestations via the Sentinel (when enabled) -- Invalid blocks from the P2P network -- Chain prunes and epoch validation +- Validator attestations and proposals via the Sentinel (when enabled) +- Invalid block and checkpoint proposals from the P2P network +- Transaction data availability after each checkpoint - L1 block data for attestation validation +- Equivocation (duplicate proposals and attestations) on the P2P network ### 2. Offense Storage diff --git a/yarn-project/slasher/README.md b/yarn-project/slasher/README.md index fca7713ef588..4d24f910a2b2 100644 --- a/yarn-project/slasher/README.md +++ b/yarn-project/slasher/README.md @@ -129,7 +129,7 @@ List of all slashable offenses in the system: **Time Unit**: Slot-based offense. ### BROADCASTED_INVALID_CHECKPOINT_PROPOSAL -**Description**: A proposer broadcast an invalid checkpoint proposal, either one that terminates before a higher-index block proposal signed by the same proposer in the same slot, one whose signed header does not match deterministic validator recomputation, or one with a malformed fee asset price modifier. +**Description**: A proposer broadcast an invalid checkpoint proposal, either one that terminates before a higher-index block proposal signed by the same proposer in the same slot, one whose signed header does not match deterministic validator recomputation, or one with a malformed fee asset price modifier. The first case also covers AZIP-7's _Submitting Block Proposal After Checkpoint_: a later block signed by the same proposer in the same slot makes the prior checkpoint retroactively invalid. **Detection**: BroadcastedInvalidCheckpointProposalWatcher scans retained P2P proposal evidence and compares checkpoint archive roots to signed block proposals from the same slot and signer. ValidatorClient also validates checkpoint proposals during the all-nodes callback and emits this offense when checkpoint header recomputation fails or the signed fee asset price modifier is malformed. **Target**: Proposer who broadcast the invalid checkpoint proposal. **Time Unit**: Slot-based offense. From 5ee92841f4f3506b0b7dbfc78f2228fe1449ccf1 Mon Sep 17 00:00:00 2001 From: Michal Rzeszutko Date: Fri, 29 May 2026 11:41:20 +0200 Subject: [PATCH 06/24] fix(p2p): do not penalize peers that signal a missing block with Fr.ZERO (#23672) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Stacked on:** `[mr/fix-block-txs-validation-archiver-fallback](https://github.com/AztecProtocol/aztec-packages/pull/23624)` (the archiver-fallback PR). --- ## Summary A peer that legitimately can't find the requested block in its attestation pool or archiver, but matches the requested hashes against its tx pool and sends those txs back, signals "I don't have the block" by setting `archiveRoot = Fr.ZERO` in the response (`block_txs_handler.ts:54-58`). The requester's validation currently treats that response the same as a malicious archive-root mismatch and applies a `MidToleranceError` peer penalty. After enough such responses from the same helpful peer, that peer is disconnected by the requester for behaviour the protocol explicitly documents as legitimate. This PR makes the validator recognise `Fr.ZERO` as the "I don't have the block" signal and stop penalising peers for it. ## What's broken ### Responder side (correct, intentional) `block_txs_handler.ts:54-58` — when the responder lacks the block (no proposal, no archived block) but the request carried full tx hashes, it tries to serve from its pool by hash and signals the "I don't know the block" condition with `archiveRoot = Fr.zero()`: ```ts if (!txHashes && requestedTxsHashes !== undefined) { const responseTxs = (await txPool.getTxsByHash(requestedTxsHashes)).filter(tx => !!tx); const response = new BlockTxsResponse(Fr.zero(), new TxArray(...responseTxs), BitVector.init(0, [])); return response.toBuffer(); } ``` ### Requester side (broken) The validator at `libp2p_service.ts:1525-1530` penalizes any archive-root mismatch with `MidToleranceError` (-10 score points) and throws — *including* `Fr.zero`: ```ts if (!response.archiveRoot.equals(request.archiveRoot)) { this.peerManager.penalizePeer(peerId, PeerErrorSeverity.MidToleranceError); throw new ValidationError(...); } ``` After 5 such responses from the same helpful peer, that peer is at -50 in the requester's score book → disconnected by `pruneUnhealthyPeers` (`peer_manager.ts:601-603`). ### Receiver-side code already documented the correct intent The wrongful penalty contradicts what the receiver-side code explicitly says should happen. `batch_tx_requester.ts:586-600` has `handleArchiveRootMismatch`, whose docstring spells out exactly the semantic we're restoring: ```ts /** * Handles an archive root mismatch between local state and peer response. * * - Response archive is Fr.ZERO (peer pruned proposal, legitimate): marks peer dumb. * - Non-zero archive mismatch (malicious response): penalises + marks dumb. */ private handleArchiveRootMismatch(peerId: PeerId, response: BlockTxsResponse): void { if (!response.archiveRoot.isZero()) { this.peers.penalisePeer(peerId, PeerErrorSeverity.LowToleranceError); } this.peers.markPeerDumb(peerId); this.txsMetadata.clearPeerData(peerId); } ``` But this function is only reached from `decideIfPeerIsSmart` → `handleSuccessResponseFromPeer`, which only runs when validation returns `true`. The validator's first check (archive-root equality) rejects every archive-mismatched response — including `Fr.zero` — so `handleArchiveRootMismatch` is never actually invoked. The "Fr.zero is legitimate" exemption it encodes has been unreachable since the validator's archive-root check was added. The flow today: ``` reqresp response │ ▼ validateRequestedBlockTxsConsistency ├─ Fr.zero → reject + Mid penalty (BUG, contradicts the docstring above) ├─ non-zero ≠ → reject + Mid penalty └─ matches archive root ──► handleSuccessResponseFromPeer └─ decideIfPeerIsSmart └─ hasArchiveRootMismatch? always false (validator filtered them all out) ──► handleArchiveRootMismatch ◄── DEAD CODE ``` So the receiver side already knew Fr.zero should not be penalised; the decision was just being made at the wrong layer. This PR moves it to the validator, where it can actually fire. `handleArchiveRootMismatch` itself remains dead code (separate cleanup candidate, not in this PR). ## Fix Special-case `response.archiveRoot.isZero()` at the top of `validateRequestedBlockTxsConsistency` so the archive-root mismatch path is bypassed for `Fr.zero` responses. We still return `false` (the txs are dropped because we can't verify membership/order without the block) but no peer penalty is applied — matching the intent of the `Fr.zero` exemption in `batch_tx_requester.ts:593-600`. ```ts // libp2p_service.ts (inside validateRequestedBlockTxsConsistency) if (response.archiveRoot.isZero()) { this.logger.debug(`Peer ${peerId.toString()} signalled missing block with Fr.zero archive root`); return false; } if (!response.archiveRoot.equals(request.archiveRoot)) { this.peerManager.penalizePeer(peerId, PeerErrorSeverity.MidToleranceError); ... } ``` The validator's other checks are unchanged. The early return prevents the bitvector-length and `maxReturnable` checks downstream from firing on a zero-length bitvector response, which would otherwise also wrongly penalise the peer. ## Tests **Unit** — `p2p/src/services/libp2p/libp2p_service.test.ts` A new test in the `validateRequestedBlockTxsConsistency` describe block: *"should not penalize a peer that signals lacking the block with Fr.ZERO archive root"*. Constructs a response with `archiveRoot = Fr.ZERO` and asserts that `peerManager.penalizePeer` is not called. Verified red before the fix, green after. **Integration** — `p2p/src/client/test/p2p_client.integration_block_txs.test.ts` A new test in the `p2p client integration block txs protocol` describe block: *"requester does not penalize peer that returns Fr.zero (peer lacks proposal but matched by hash)"*. Drives a real reqresp BLOCK_TXS request over libp2p between two clients, lets the responder hit the `Fr.zero` branch in `block_txs_handler`, then runs the response through the requester's real `validateRequestedBlockTxsConsistency` and asserts the requester's `peerManager.penalizePeer` is not called with `MidToleranceError` or `LowToleranceError`. Verified red before the fix, green after. --- .../p2p_client.integration_block_txs.test.ts | 44 ++++++++++++++++++- .../services/libp2p/libp2p_service.test.ts | 15 +++++++ .../p2p/src/services/libp2p/libp2p_service.ts | 10 +++++ 3 files changed, 68 insertions(+), 1 deletion(-) diff --git a/yarn-project/p2p/src/client/test/p2p_client.integration_block_txs.test.ts b/yarn-project/p2p/src/client/test/p2p_client.integration_block_txs.test.ts index 50123143334d..a0f8c6960b4e 100644 --- a/yarn-project/p2p/src/client/test/p2p_client.integration_block_txs.test.ts +++ b/yarn-project/p2p/src/client/test/p2p_client.integration_block_txs.test.ts @@ -7,7 +7,7 @@ import { type Logger, createLogger } from '@aztec/foundation/log'; import { sleep } from '@aztec/foundation/sleep'; import { emptyChainConfig } from '@aztec/stdlib/config'; import type { WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server'; -import { BlockProposal } from '@aztec/stdlib/p2p'; +import { BlockProposal, PeerErrorSeverity } from '@aztec/stdlib/p2p'; import { makeBlockHeader, makeBlockProposal } from '@aztec/stdlib/testing'; import { Tx, TxHash, TxHashArray } from '@aztec/stdlib/tx'; @@ -418,4 +418,46 @@ describe('p2p client integration block txs protocol ', () => { // Should get NOT_FOUND because without full tx hashes, handler can't return txs without proposal expect(response.status).toBe(ReqRespStatus.NOT_FOUND); }); + + // When the responder takes the Fr.zero branch (no proposal/archived block locally, but matched + // the requested hashes against its tx pool) it is acting correctly per block_txs_handler.ts:54-58. + // Drive the response back through the requester's real validation and confirm it does NOT apply + // a peer penalty — Fr.zero is a documented "I don't have the block" signal, not misbehavior. + it('requester does not penalize peer that returns Fr.zero (peer lacks proposal but matched by hash)', async () => { + attestationPool.getBlockProposalByArchive.mockResolvedValue(undefined); + const hashToTx = new Map(txs.map((tx, i) => [txHashes[i].toString(), tx])); + txPool.getTxsByHash.mockImplementation((hashes: TxHash[]) => + Promise.resolve(hashes.map(h => hashToTx.get(h.toString())!)), + ); + + const [client1, client2] = clients as any; + const peerManager = client1.p2pService.peerManager; + const penalizeSpy = jest.spyOn(peerManager, 'penalizePeer'); + + // Build and send a real reqresp BLOCK_TXS request over libp2p (includeFullTxHashes=true so + // the responder hits the Fr.zero branch instead of NOT_FOUND). + const requestedHashes = [txHashes[1], txHashes[3]]; + const sourceProposal = await createBlockProposal(blockNumber, Fr.random(), txHashes); + const request = BlockTxsRequest.fromTxsSourceAndMissingTxs(sourceProposal, requestedHashes, true)!; + const response = await client1.p2pService.reqresp.sendRequestToPeer( + client2.p2pService.node.peerId, + ReqRespSubProtocol.BLOCK_TXS, + request.toBuffer(), + ); + + expect(response.status).toBe(ReqRespStatus.SUCCESS); + const decoded = BlockTxsResponse.fromBuffer(response.data); + expect(decoded.archiveRoot.equals(Fr.ZERO)).toBe(true); + + // Run the response through client1's real validation — this is what BatchTxRequester does in + // production. The peer must not be penalized. + await (client1.p2pService as any).validateRequestedBlockTxsConsistency( + request, + decoded, + client2.p2pService.node.peerId, + ); + + expect(penalizeSpy).not.toHaveBeenCalledWith(expect.anything(), PeerErrorSeverity.MidToleranceError); + expect(penalizeSpy).not.toHaveBeenCalledWith(expect.anything(), PeerErrorSeverity.LowToleranceError); + }); }); diff --git a/yarn-project/p2p/src/services/libp2p/libp2p_service.test.ts b/yarn-project/p2p/src/services/libp2p/libp2p_service.test.ts index 5f2ce4391d23..9ea75ef3fd60 100644 --- a/yarn-project/p2p/src/services/libp2p/libp2p_service.test.ts +++ b/yarn-project/p2p/src/services/libp2p/libp2p_service.test.ts @@ -529,6 +529,21 @@ describe('LibP2PService', () => { expect(mockPeerManager.penalizePeer).not.toHaveBeenCalled(); expect(mockArchiver.getBlock).toHaveBeenCalledWith({ archive: hash }); }); + + // The reqresp BLOCK_TXS responder (block_txs_handler.ts:54-58) returns archiveRoot=Fr.ZERO + // when it doesn't have the block in either the attestation pool or the archiver, but the + // request carried full tx hashes — it matches them against its pool and ships whatever it + // finds. This is a documented "I don't have the block" signal, not misbehavior, so the + // requester must not penalize the peer for it. + it('should not penalize a peer that signals lacking the block with Fr.ZERO archive root', async () => { + const hash = Fr.random(); + const request = makeRequest(hash, 3, [0, 2]); + const response = makeResponse(Fr.ZERO, 0, [], ['0xfound0', '0xfound2']); + + await service.validateRequestedBlockTxsConsistency(request, response, mockPeerId); + + expect(mockPeerManager.penalizePeer).not.toHaveBeenCalled(); + }); }); describe('processBlockFromPeer', () => { diff --git a/yarn-project/p2p/src/services/libp2p/libp2p_service.ts b/yarn-project/p2p/src/services/libp2p/libp2p_service.ts index 688e7798c8e1..a5c2728d4089 100644 --- a/yarn-project/p2p/src/services/libp2p/libp2p_service.ts +++ b/yarn-project/p2p/src/services/libp2p/libp2p_service.ts @@ -1528,6 +1528,16 @@ export class LibP2PService extends WithTracer implements P2PService { peerId: PeerId, ): Promise { try { + // A response with archiveRoot=Fr.zero is the documented "I don't have the block" signal from + // reqRespBlockTxsHandler (block_txs_handler.ts:54-58): the peer lacked the block in its + // attestation pool and archiver, but matched the requested hashes against its tx pool and + // shipped what it found. This is legitimate behaviour, not misbehaviour — we just can't verify + // membership/order without the block, so we drop the response without penalising the peer. + if (response.archiveRoot.isZero()) { + this.logger.debug(`Peer ${peerId.toString()} signalled missing block with Fr.zero archive root`); + return false; + } + if (!response.archiveRoot.equals(request.archiveRoot)) { this.peerManager.penalizePeer(peerId, PeerErrorSeverity.MidToleranceError); throw new ValidationError( From 581acd07f965b5075a862405573a8f839ec72b5a Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Fri, 29 May 2026 10:45:38 +0100 Subject: [PATCH 07/24] chore: adjust metrics deployment (#23676) . --- spartan/metrics/values/prod.yaml | 53 +++++++++++++++----------------- 1 file changed, 24 insertions(+), 29 deletions(-) diff --git a/spartan/metrics/values/prod.yaml b/spartan/metrics/values/prod.yaml index 1fd2331ee772..2230745967cc 100644 --- a/spartan/metrics/values/prod.yaml +++ b/spartan/metrics/values/prod.yaml @@ -2,16 +2,13 @@ opentelemetry-collector: replicaCount: 1 resources: requests: - memory: 12Gi + memory: 1Gi + cpu: "0.5" + limits: + memory: 4Gi cpu: "2" nodeSelector: node-type: infra - # pool: spot - # tolerations: - # - key: "cloud.google.com/gke-spot" - # operator: "Equal" - # value: "true" - # effect: "NoSchedule" ports: jaeger-compact: enabled: false @@ -22,8 +19,8 @@ opentelemetry-collector: processors: memory_limiter: check_interval: 1s - limit_mib: 10000 - spike_limit_mib: 2000 + limit_mib: 3200 + spike_limit_mib: 800 filter/large_histograms: metrics: datapoint: @@ -87,19 +84,23 @@ opentelemetry-collector: - otlp/tempo prometheus: + alertmanager: + enabled: false + prometheus-node-exporter: + enabled: false + prometheus-pushgateway: + enabled: false + server: resources: requests: - memory: 40Gi - cpu: "3.5" + memory: 4Gi + cpu: "1" + limits: + memory: 32Gi + cpu: "8" nodeSelector: node-type: infra - # pool: spot - # tolerations: - # - key: "cloud.google.com/gke-spot" - # operator: "Equal" - # value: "true" - # effect: "NoSchedule" persistentVolume: enabled: true @@ -107,15 +108,6 @@ prometheus: replicaCount: 1 statefulSet: enabled: true - alertmanager: - nodeSelector: - node-type: infra - nodeExporter: - nodeSelector: - node-type: infra - pushgateway: - nodeSelector: - node-type: infra loki: enabled: false @@ -126,7 +118,7 @@ tempo: resources: requests: memory: 4Gi - cpu: "2" + cpu: "0.5" limits: memory: 8Gi cpu: "4" @@ -139,7 +131,7 @@ tempo: enabled: false persistence: enabled: true - size: 256Gi + size: 32Gi storageClassName: standard-rwo nodeSelector: node-type: infra @@ -148,8 +140,11 @@ tempo: grafana: resources: requests: + memory: 1Gi + cpu: "0.5" + limits: memory: 4Gi - cpu: "1.5" + cpu: "2" nodeSelector: node-type: infra service: From f5ba79050bd78542962578756dcbc7aa0b171e9a Mon Sep 17 00:00:00 2001 From: Aztec Bot <49558828+AztecBot@users.noreply.github.com> Date: Fri, 29 May 2026 05:45:51 -0400 Subject: [PATCH 08/24] fix(cheat-codes): warpL2TimeAtLeastBy advances relative to leading clock (#23675) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem CI on `merge-train/spartan` failed in `yarn-project/end-to-end/src/composed/e2e_cheat_codes.test.ts` ([log](http://ci.aztec-labs.com/1780045078587982) → test-engine `358cb51c378c6913` → `4baba53d3b9feb67`): ``` e2e_cheat_codes › warpL2TimeAtLeastBy advances time by at least the duration expect(received).toBeGreaterThanOrEqual(expected) Expected: >= 1780048759 (timestampBefore_L2 + 100) Received: 1780048731 (advanced only ~72s) ``` ## Root cause `CheatCodes.warpL2TimeAtLeastBy(duration)` computed its target as `eth.lastBlockTimestamp() (L1) + duration`, but its documented contract is that the **L2** timestamp advances by at least `duration`, and the test measures advancement against the latest **L2 block** timestamp. In the composed test a live sequencer mines L2 blocks at slot boundaries that can run ahead of anvil's L1 clock. When the latest L2 block timestamp leads L1, adding `duration` to L1 produces a target below `latestL2 + duration`, so the resulting block advances L2 time by less than `duration` and the assertion fails. This is a latent correctness bug in the helper that surfaces non-deterministically depending on slot/L1 alignment. ## Fix Anchor the target to whichever clock leads — `max(currentL1Timestamp, latestL2BlockTimestamp) + duration` — before delegating to `warpL2TimeAtLeastTo`. This guarantees the post-warp L2 block is at least `duration` ahead of the current one, while remaining a strict superset of the old behaviour (it never advances by less than before), so other callers (`e2e_expiration_timestamp`, `e2e_contract_updates`, `blacklist_token_contract`, `e2e_automine_smoke`, `lending_simulator`) are unaffected. Single-file change in `yarn-project/aztec/src/testing/cheat_codes.ts`. --- *Created by [claudebox](https://claudebox.work/v2/sessions/368b5e8b3cef0969) · group: `slackbot`* --- noir/noir-repo | 2 +- yarn-project/aztec/src/testing/cheat_codes.ts | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/noir/noir-repo b/noir/noir-repo index f1a4575adac5..4d0392685f8b 160000 --- a/noir/noir-repo +++ b/noir/noir-repo @@ -1 +1 @@ -Subproject commit f1a4575adac59af0a86b036cf73ff5883d142a91 +Subproject commit 4d0392685f8b9a4a9895d675d3ed730716559860 diff --git a/yarn-project/aztec/src/testing/cheat_codes.ts b/yarn-project/aztec/src/testing/cheat_codes.ts index a78137eb3a1a..9e2f459bbe11 100644 --- a/yarn-project/aztec/src/testing/cheat_codes.ts +++ b/yarn-project/aztec/src/testing/cheat_codes.ts @@ -118,8 +118,15 @@ export class CheatCodes { throw new Error(`warpL2TimeAtLeastBy: duration must be positive, got ${duration} seconds.`); } - const currentTimestamp = await this.eth.lastBlockTimestamp(); - const targetTimestamp = BigInt(currentTimestamp) + BigInt(duration); + // Advance relative to whichever clock leads. A live sequencer mines L2 blocks at slot boundaries that can run + // ahead of anvil's L1 timestamp, so basing the target on L1 alone would advance the L2 timestamp by less than + // `duration`. Anchoring to the latest L2 block timestamp when it leads guarantees the post-warp L2 block is at + // least `duration` ahead of the current one. + const currentL1Timestamp = BigInt(await this.eth.lastBlockTimestamp()); + const latestBlockData = await node.getBlockData('latest'); + const latestL2Timestamp = latestBlockData ? BigInt(latestBlockData.header.globalVariables.timestamp) : 0n; + const baseTimestamp = latestL2Timestamp > currentL1Timestamp ? latestL2Timestamp : currentL1Timestamp; + const targetTimestamp = baseTimestamp + BigInt(duration); await this.warpL2TimeAtLeastTo(node, targetTimestamp); } } From 35093acc25efe6167c0aba468878321efc345f1a Mon Sep 17 00:00:00 2001 From: AztecBot Date: Fri, 29 May 2026 10:05:03 +0000 Subject: [PATCH 09/24] fix: make world-state hash queries reorg-aware to close getWorldState race --- .../aztec-node/src/aztec-node/server.test.ts | 19 +++++++++++++ .../aztec-node/src/aztec-node/server.ts | 28 +++++++++++++------ 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/yarn-project/aztec-node/src/aztec-node/server.test.ts b/yarn-project/aztec-node/src/aztec-node/server.test.ts index 6c35abec8370..b37860749c4e 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.test.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.test.ts @@ -622,6 +622,25 @@ describe('aztec node', () => { expect(worldState.getSnapshot).toHaveBeenCalledWith(BlockNumber(3)); }); + it('drives a reorg-aware sync to the requested block hash', async () => { + // A hash-anchored query resolves the hash against the archiver and then syncs world state to that + // exact (number, hash) so the synchronizer barriers on the archive-tree commit and detects reorgs, + // rather than syncing to bare latest height and racing the snapshot read. + const blockHash = BlockHash.random(); + l2BlockSource.getBlockNumber.mockImplementation(((query?: BlockQuery) => + Promise.resolve(query && 'hash' in query ? BlockNumber(3) : lastBlockNumber)) as L2BlockSource['getBlockNumber']); + snapshotMerkleTreeOps.getLeafValue.mockResolvedValue(blockHash); + + await node.getWorldState(blockHash); + + expect(worldState.syncImmediate).toHaveBeenCalledWith(BlockNumber(3), blockHash); + }); + + it('syncs to latest height without a hash when querying by block number', async () => { + await node.getWorldState(BlockNumber(3)); + expect(worldState.syncImmediate).toHaveBeenCalledWith(lastBlockNumber, undefined); + }); + it('throws when block hash is not found in archiver', async () => { const blockHash = BlockHash.random(); diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index f4be06db0b6e..2c48e8767186 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -2027,21 +2027,31 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, AztecNodeDeb * @returns An instance of a committed MerkleTreeOperations */ protected async getWorldState(block: BlockParameter) { + const query = this.normalizeBlockParameter(block); + + // When the request anchors on a specific block hash, resolve it against the archiver up front and + // drive the world-state sync to that exact block number and hash. Resolving against the archiver + // first fails fast with a clear reorg error if the hash is unknown, and passing the hash to the + // synchronizer makes the sync reorg-aware: it barriers until the archive-tree commit for that block + // has landed and verifies it matches the requested fork, instead of syncing to bare latest height + // and then racing the snapshot read below against an in-flight archive-tree write. + const requestedHash = 'hash' in query ? query.hash : undefined; + const anchorBlockNumber = requestedHash !== undefined ? await this.resolveBlockNumber(query) : undefined; + let blockSyncedTo: BlockNumber = BlockNumber.ZERO; try { // Attempt to sync the world state if necessary - blockSyncedTo = await this.#syncWorldState(); + blockSyncedTo = await this.#syncWorldState(anchorBlockNumber, requestedHash); } catch (err) { this.log.error(`Error getting world state: ${err}`); } - const query = this.normalizeBlockParameter(block); if ('tag' in query && query.tag === 'proposed') { this.log.debug(`Using committed db for latest block, world state synced upto ${blockSyncedTo}`); return this.worldStateSynchronizer.getCommitted(); } - const blockNumber = await this.resolveBlockNumber(block); + const blockNumber = anchorBlockNumber ?? (await this.resolveBlockNumber(query)); // Check it's within world state sync range if (blockNumber > blockSyncedTo) { @@ -2058,7 +2068,6 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, AztecNodeDeb // (size 0), so leaf 0 is not yet inserted from that snapshot's view even though block 0's hash // does live at archive index 0 in the committed tree. The genesis hash is already validated by // the archiver when it resolves the hash query to block number 0. - const requestedHash = 'hash' in query ? query.hash : undefined; if (requestedHash !== undefined && blockNumber !== BlockNumber.ZERO) { const blockHash = await snapshot.getLeafValue(MerkleTreeId.ARCHIVE, BigInt(blockNumber)); if (!blockHash || !requestedHash.equals(blockHash)) { @@ -2090,11 +2099,14 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, AztecNodeDeb } /** - * Ensure we fully sync the world state + * Ensure the world state is synced. + * @param targetBlockNumber - Block to sync up to. Defaults to the latest block known to the archiver. + * @param blockHash - If provided, the synchronizer verifies the block at `targetBlockNumber` matches this + * hash, resyncing (and so detecting reorgs) if it does not yet match or has been reorged away. * @returns A promise that fulfils once the world state is synced */ - async #syncWorldState(): Promise { - const blockSourceHeight = await this.blockSource.getBlockNumber(); - return await this.worldStateSynchronizer.syncImmediate(blockSourceHeight); + async #syncWorldState(targetBlockNumber?: BlockNumber, blockHash?: BlockHash): Promise { + const target = targetBlockNumber ?? (await this.blockSource.getBlockNumber()); + return await this.worldStateSynchronizer.syncImmediate(target, blockHash); } } From 00265d269e57c860767b4c4751ed337ba7a952f6 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Fri, 29 May 2026 11:38:50 +0100 Subject: [PATCH 10/24] chore: tighten node pool sizes (#23678) . --- spartan/terraform/gke-cluster/cluster/main.tf | 10 +++---- .../gke-cluster/cluster/variables.tf | 26 ++++++++++++++++++- spartan/terraform/gke-cluster/main.tf | 10 +++++++ 3 files changed, 40 insertions(+), 6 deletions(-) diff --git a/spartan/terraform/gke-cluster/cluster/main.tf b/spartan/terraform/gke-cluster/cluster/main.tf index c71886ecaab0..231883327214 100644 --- a/spartan/terraform/gke-cluster/cluster/main.tf +++ b/spartan/terraform/gke-cluster/cluster/main.tf @@ -9,7 +9,7 @@ resource "google_container_cluster" "primary" { deletion_protection = true # Kubernetes version - min_master_version = var.node_version + min_master_version = "1.30.5-gke.1713000" release_channel { channel = "UNSPECIFIED" @@ -346,8 +346,8 @@ resource "google_container_node_pool" "infra_nodes_8core_highmem" { version = var.node_version # Enable autoscaling autoscaling { - min_node_count = 0 - max_node_count = 4 + min_node_count = var.infra_8core_pool_size.min + max_node_count = var.infra_8core_pool_size.max } # Node configuration @@ -389,8 +389,8 @@ resource "google_container_node_pool" "infra_nodes_16core_highmem" { version = var.node_version # Enable autoscaling autoscaling { - min_node_count = 0 - max_node_count = 4 + min_node_count = var.infra_16core_pool_size.min + max_node_count = var.infra_16core_pool_size.max } # Node configuration diff --git a/spartan/terraform/gke-cluster/cluster/variables.tf b/spartan/terraform/gke-cluster/cluster/variables.tf index 0727c4fb2c25..27e3edc11010 100644 --- a/spartan/terraform/gke-cluster/cluster/variables.tf +++ b/spartan/terraform/gke-cluster/cluster/variables.tf @@ -18,7 +18,7 @@ variable "service_account" { } variable "node_version" { - default = "1.30.5-gke.1713000" + default = "1.33.10-gke.1067000" } variable "enable_workload_identity" { @@ -27,3 +27,27 @@ variable "enable_workload_identity" { default = false } + +variable "infra_8core_pool_size" { + description = "how many 8 core nodes to schedule for this cluster" + type = object({ + min = number + max = number + }) + default = { + min = 0 + max = 4 + } +} + +variable "infra_16core_pool_size" { + description = "how many 16 core nodes to schedule for this cluster" + type = object({ + min = number + max = number + }) + default = { + min = 0 + max = 4 + } +} diff --git a/spartan/terraform/gke-cluster/main.tf b/spartan/terraform/gke-cluster/main.tf index 35a3f149cae8..790496b624d9 100644 --- a/spartan/terraform/gke-cluster/main.tf +++ b/spartan/terraform/gke-cluster/main.tf @@ -26,6 +26,16 @@ module "gke_cluster_private" { zone = var.zone service_account = google_service_account.gke_sa.email enable_workload_identity = true + + infra_8core_pool_size = { + min = 0 + max = 1 + } + + infra_16core_pool_size = { + min = 0 + max = 0 + } } module "gke_cluster_public" { From 22b735f44f45973b1abe8b9e05f84dc95be03a28 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Fri, 29 May 2026 11:53:23 +0100 Subject: [PATCH 11/24] chore: remove archival nodes (#23630) Fix A-1116 --- spartan/CLAUDE.md | 1 - spartan/environments/testnet.env | 2 - spartan/scripts/deploy_network.sh | 4 -- spartan/terraform/deploy-aztec-infra/main.tf | 43 ------------------- .../values/archive-resources-dev.yaml | 30 ------------- .../values/archive-resources-prod.yaml | 35 --------------- .../deploy-aztec-infra/values/archive.yaml | 5 --- .../terraform/deploy-aztec-infra/variables.tf | 11 ----- 8 files changed, 131 deletions(-) delete mode 100644 spartan/terraform/deploy-aztec-infra/values/archive-resources-dev.yaml delete mode 100644 spartan/terraform/deploy-aztec-infra/values/archive-resources-prod.yaml delete mode 100644 spartan/terraform/deploy-aztec-infra/values/archive.yaml diff --git a/spartan/CLAUDE.md b/spartan/CLAUDE.md index 347a9812eae9..17d1bb8ea682 100644 --- a/spartan/CLAUDE.md +++ b/spartan/CLAUDE.md @@ -302,7 +302,6 @@ This ensures each release uses non-overlapping publisher key ranges while decoup ### RPC Nodes - Serve public API endpoints - Optional ingress with GCP backend config -- Archive nodes for historical data ### Boot Nodes - P2P bootstrap for network discovery diff --git a/spartan/environments/testnet.env b/spartan/environments/testnet.env index 8f0e5c30e7ac..c18eee78635f 100644 --- a/spartan/environments/testnet.env +++ b/spartan/environments/testnet.env @@ -68,8 +68,6 @@ BOT_SWAPS_REPLICAS=0 P2P_TX_POOL_DELETE_TXS_AFTER_REORG=true SEQ_MAX_TX_PER_CHECKPOINT=72 -DEPLOY_ARCHIVAL_NODE=true - RPC_INGRESS_ENABLED=true RPC_INGRESS_HOSTS='["rpc.testnet.aztec-labs.com"]' RPC_INGRESS_STATIC_IP_NAME=testnet-rpc-ip diff --git a/spartan/scripts/deploy_network.sh b/spartan/scripts/deploy_network.sh index bc7e9e346cdd..1029295b7621 100755 --- a/spartan/scripts/deploy_network.sh +++ b/spartan/scripts/deploy_network.sh @@ -149,7 +149,6 @@ OTEL_COLLECTOR_ENDPOINT=${OTEL_COLLECTOR_ENDPOINT:-} OTEL_COLLECT_INTERVAL_MS=${OTEL_COLLECT_INTERVAL_MS:-} OTEL_EXPORT_TIMEOUT_MS=${OTEL_EXPORT_TIMEOUT_MS:-} DEPLOY_INTERNAL_BOOTNODE=${DEPLOY_INTERNAL_BOOTNODE:-} -DEPLOY_ARCHIVAL_NODE=${DEPLOY_ARCHIVAL_NODE:-false} BOT_RESOURCE_PROFILE=${BOT_RESOURCE_PROFILE:-${RESOURCE_PROFILE}} BOT_TRANSFERS_MNEMONIC_START_INDEX=${BOT_TRANSFERS_MNEMONIC_START_INDEX:-7000} @@ -186,7 +185,6 @@ FULL_NODE_RESOURCE_PROFILE=${FULL_NODE_RESOURCE_PROFILE:-${RESOURCE_PROFILE}} P2P_BOOTSTRAP_RESOURCE_PROFILE=${P2P_BOOTSTRAP_RESOURCE_PROFILE:-${RESOURCE_PROFILE}} VALIDATOR_RESOURCE_PROFILE=${VALIDATOR_RESOURCE_PROFILE:-${RESOURCE_PROFILE}} PROVER_RESOURCE_PROFILE=${PROVER_RESOURCE_PROFILE:-${RESOURCE_PROFILE}} -ARCHIVE_RESOURCE_PROFILE=${ARCHIVE_RESOURCE_PROFILE:-${RESOURCE_PROFILE}} BLOB_SINK_RESOURCE_PROFILE=${BLOB_SINK_RESOURCE_PROFILE:-${RESOURCE_PROFILE}} PROVER_NODE_DISABLE_PROOF_PUBLISH=${PROVER_NODE_DISABLE_PROOF_PUBLISH:-false} @@ -555,7 +553,6 @@ VALIDATOR_RESOURCE_PROFILE = "${VALIDATOR_RESOURCE_PROFILE}" PROVER_RESOURCE_PROFILE = "${PROVER_RESOURCE_PROFILE}" RPC_RESOURCE_PROFILE = "${RPC_RESOURCE_PROFILE}" FULL_NODE_RESOURCE_PROFILE = "${FULL_NODE_RESOURCE_PROFILE}" -ARCHIVE_RESOURCE_PROFILE = "${ARCHIVE_RESOURCE_PROFILE}" BLOB_SINK_RESOURCE_PROFILE = "${BLOB_SINK_RESOURCE_PROFILE}" AZTEC_DOCKER_IMAGE = "${AZTEC_DOCKER_IMAGE}" PROVER_AGENT_DOCKER_IMAGE = "${PROVER_AGENT_DOCKER_IMAGE:-$AZTEC_DOCKER_IMAGE}" @@ -664,7 +661,6 @@ FULL_NODE_REPLICAS = ${FULL_NODE_REPLICAS:-1} PROVER_FAILED_PROOF_STORE = "${PROVER_FAILED_PROOF_STORE}" PROVER_PROOF_STORE = "${PROVER_PROOF_STORE:-}" PROVER_BROKER_DEBUG_REPLAY_ENABLED = ${PROVER_BROKER_DEBUG_REPLAY_ENABLED:-false} -DEPLOY_ARCHIVAL_NODE = ${DEPLOY_ARCHIVAL_NODE} PROVER_REPLICAS = ${PROVER_REPLICAS} PROVER_ENABLED = ${PROVER_ENABLED} diff --git a/spartan/terraform/deploy-aztec-infra/main.tf b/spartan/terraform/deploy-aztec-infra/main.tf index d14fd6d69d8a..3c9988f079f5 100644 --- a/spartan/terraform/deploy-aztec-infra/main.tf +++ b/spartan/terraform/deploy-aztec-infra/main.tf @@ -149,7 +149,6 @@ locals { p2p_port_rpc = 40400 + (parseint(substr(md5("${var.NAMESPACE}-rpc"), 0, 4), 16) % 100) p2p_port_fisherman = 40400 + (parseint(substr(md5("${var.NAMESPACE}-fisherman"), 0, 4), 16) % 100) p2p_port_full_node = 40400 + (parseint(substr(md5("${var.NAMESPACE}-full-node"), 0, 4), 16) % 100) - p2p_port_archive = 40400 + (parseint(substr(md5("${var.NAMESPACE}-archive"), 0, 4), 16) % 100) p2p_port_validators = { for idx in range(1 + var.VALIDATOR_HA_REPLICAS) : idx => 40400 + (parseint(substr(md5("${var.NAMESPACE}-validator-${idx}"), 0, 4), 16) % 100) @@ -588,48 +587,6 @@ locals { wait = false } : null - archive = var.DEPLOY_ARCHIVAL_NODE ? { - name = "${var.RELEASE_PREFIX}-archive" - chart = "aztec-node" - values = [ - "common.yaml", - "archive.yaml", - "archive-resources-${var.ARCHIVE_RESOURCE_PROFILE}.yaml" - ] - inline_values = [yamlencode({ - service = { - p2p = { publicIP = var.P2P_PUBLIC_IP } - } - })] - custom_settings = { - "nodeType" = "archive" - "service.p2p.nodePortEnabled" = var.P2P_NODEPORT_ENABLED - "service.p2p.announcePort" = local.p2p_port_archive - "service.p2p.port" = local.p2p_port_archive - "node.env.P2P_ARCHIVED_TX_LIMIT" = "10000000" - "node.proverRealProofs" = var.PROVER_REAL_PROOFS - "node.env.PROVER_TEST_VERIFICATION_DELAY_MS" = var.PROVER_TEST_VERIFICATION_DELAY_MS - "node.env.BB_CHONK_VERIFY_MAX_BATCH" = var.BB_CHONK_VERIFY_MAX_BATCH - "node.env.BB_CHONK_VERIFY_BATCH_CONCURRENCY" = var.BB_CHONK_VERIFY_BATCH_CONCURRENCY - "node.env.DEBUG_FORCE_TX_PROOF_VERIFICATION" = var.DEBUG_FORCE_TX_PROOF_VERIFICATION - "node.env.DEBUG_P2P_INSTRUMENT_MESSAGES" = var.DEBUG_P2P_INSTRUMENT_MESSAGES - "node.env.P2P_TX_POOL_DELETE_TXS_AFTER_REORG" = var.P2P_TX_POOL_DELETE_TXS_AFTER_REORG - "node.env.BLOB_ALLOW_EMPTY_SOURCES" = var.BLOB_ALLOW_EMPTY_SOURCES - "node.env.P2P_GOSSIPSUB_D" = var.P2P_GOSSIPSUB_D - "node.env.P2P_GOSSIPSUB_DLO" = var.P2P_GOSSIPSUB_DLO - "node.env.P2P_GOSSIPSUB_DHI" = var.P2P_GOSSIPSUB_DHI - "node.env.P2P_DROP_TX_CHANCE" = var.P2P_DROP_TX_CHANCE - "node.env.P2P_MAX_PENDING_TX_COUNT" = var.P2P_MAX_PENDING_TX_COUNT - "node.env.WS_NUM_HISTORIC_CHECKPOINTS" = var.WS_NUM_HISTORIC_CHECKPOINTS - "node.env.TX_COLLECTION_FILE_STORE_URLS" = var.TX_COLLECTION_FILE_STORE_URLS - "node.env.BLOB_FILE_STORE_URLS" = var.BLOB_FILE_STORE_URLS - "node.env.SEQ_ENABLE_PROPOSER_PIPELINING" = var.SEQ_ENABLE_PROPOSER_PIPELINING - } - boot_node_host_path = "node.env.BOOT_NODE_HOST" - bootstrap_nodes_path = "node.env.BOOTSTRAP_NODES" - wait = true - } : null - # Blob sink: uploads blobs to filestore as it syncs blob_sink = var.BLOB_FILE_STORE_UPLOAD_URL != null ? { name = "${var.RELEASE_PREFIX}-blob-sink" diff --git a/spartan/terraform/deploy-aztec-infra/values/archive-resources-dev.yaml b/spartan/terraform/deploy-aztec-infra/values/archive-resources-dev.yaml deleted file mode 100644 index bc95d42caebb..000000000000 --- a/spartan/terraform/deploy-aztec-infra/values/archive-resources-dev.yaml +++ /dev/null @@ -1,30 +0,0 @@ -replicaCount: 1 - -node: - resources: - requests: - cpu: "0.25" - memory: "1Gi" - limits: - cpu: "0.5" - memory: "2Gi" - -persistence: - enabled: true - -statefulSet: - enabled: true - volumeClaimTemplates: - - metadata: - name: data - annotations: - "helm.sh/resource-policy": "Retain" - spec: - accessModes: [ReadWriteOnce] - resources: - requests: - storage: "1Ti" - -service: - headless: - enabled: true diff --git a/spartan/terraform/deploy-aztec-infra/values/archive-resources-prod.yaml b/spartan/terraform/deploy-aztec-infra/values/archive-resources-prod.yaml deleted file mode 100644 index b4322de7a540..000000000000 --- a/spartan/terraform/deploy-aztec-infra/values/archive-resources-prod.yaml +++ /dev/null @@ -1,35 +0,0 @@ -nodeSelector: - local-ssd: "false" - node-type: "network" - cores: "2" - -replicaCount: 1 - -node: - resources: - requests: - cpu: "0.5" - memory: "2Gi" - limits: - cpu: "1.5" - memory: "6Gi" - -persistence: - enabled: true - -statefulSet: - enabled: true - volumeClaimTemplates: - - metadata: - name: data - annotations: - "helm.sh/resource-policy": "Retain" - spec: - accessModes: [ReadWriteOnce] - resources: - requests: - storage: "64Gi" - -service: - headless: - enabled: true diff --git a/spartan/terraform/deploy-aztec-infra/values/archive.yaml b/spartan/terraform/deploy-aztec-infra/values/archive.yaml deleted file mode 100644 index b75c474875c2..000000000000 --- a/spartan/terraform/deploy-aztec-infra/values/archive.yaml +++ /dev/null @@ -1,5 +0,0 @@ -node: - env: - OTEL_SERVICE_NAME: "archival-node" - startCmd: - - --node diff --git a/spartan/terraform/deploy-aztec-infra/variables.tf b/spartan/terraform/deploy-aztec-infra/variables.tf index 5b8c00682983..7906d34030a8 100644 --- a/spartan/terraform/deploy-aztec-infra/variables.tf +++ b/spartan/terraform/deploy-aztec-infra/variables.tf @@ -51,11 +51,6 @@ variable "BOT_RESOURCE_PROFILE" { type = string } -variable "ARCHIVE_RESOURCE_PROFILE" { - description = "Resource profile to use for the archive node" - type = string -} - variable "BLOB_SINK_RESOURCE_PROFILE" { description = "Resource profile to use for the blob sink" type = string @@ -559,12 +554,6 @@ variable "EXTERNAL_BOOTNODES" { default = [] } -variable "DEPLOY_ARCHIVAL_NODE" { - description = "Whether to deploy the archival node" - type = bool - default = false -} - variable "NETWORK" { description = "One of the existing network names to use default config for" type = string From 84765424b7b7eedeb7a69c0c5a96aa13442108c9 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Fri, 29 May 2026 11:57:39 +0100 Subject: [PATCH 12/24] chore: merge blob sink duties into RPC node (#23631) Fix A-1117 --- spartan/environments/mainnet.env | 1 - spartan/scripts/deploy_network.sh | 2 - spartan/terraform/deploy-aztec-infra/main.tf | 38 +----------------- .../values/blob-sink-resources-dev.yaml | 34 ---------------- .../values/blob-sink-resources-prod.yaml | 39 ------------------- .../deploy-aztec-infra/values/blob-sink.yaml | 18 --------- .../terraform/deploy-aztec-infra/variables.tf | 5 --- 7 files changed, 1 insertion(+), 136 deletions(-) delete mode 100644 spartan/terraform/deploy-aztec-infra/values/blob-sink-resources-dev.yaml delete mode 100644 spartan/terraform/deploy-aztec-infra/values/blob-sink-resources-prod.yaml delete mode 100644 spartan/terraform/deploy-aztec-infra/values/blob-sink.yaml diff --git a/spartan/environments/mainnet.env b/spartan/environments/mainnet.env index e8b74d9b3efe..fe27dd0dd6c2 100644 --- a/spartan/environments/mainnet.env +++ b/spartan/environments/mainnet.env @@ -27,7 +27,6 @@ FISHERMAN_MNEMONIC_START_INDEX=1 PROVER_NODE_DISABLE_PROOF_PUBLISH=true RPC_RESOURCE_PROFILE=prod -BLOB_SINK_RESOURCE_PROFILE=prod PROVER_RESOURCE_PROFILE=prod LOG_LEVEL=info diff --git a/spartan/scripts/deploy_network.sh b/spartan/scripts/deploy_network.sh index 1029295b7621..5149702f22c6 100755 --- a/spartan/scripts/deploy_network.sh +++ b/spartan/scripts/deploy_network.sh @@ -185,7 +185,6 @@ FULL_NODE_RESOURCE_PROFILE=${FULL_NODE_RESOURCE_PROFILE:-${RESOURCE_PROFILE}} P2P_BOOTSTRAP_RESOURCE_PROFILE=${P2P_BOOTSTRAP_RESOURCE_PROFILE:-${RESOURCE_PROFILE}} VALIDATOR_RESOURCE_PROFILE=${VALIDATOR_RESOURCE_PROFILE:-${RESOURCE_PROFILE}} PROVER_RESOURCE_PROFILE=${PROVER_RESOURCE_PROFILE:-${RESOURCE_PROFILE}} -BLOB_SINK_RESOURCE_PROFILE=${BLOB_SINK_RESOURCE_PROFILE:-${RESOURCE_PROFILE}} PROVER_NODE_DISABLE_PROOF_PUBLISH=${PROVER_NODE_DISABLE_PROOF_PUBLISH:-false} P2P_TX_POOL_DELETE_TXS_AFTER_REORG=${P2P_TX_POOL_DELETE_TXS_AFTER_REORG:-false} @@ -553,7 +552,6 @@ VALIDATOR_RESOURCE_PROFILE = "${VALIDATOR_RESOURCE_PROFILE}" PROVER_RESOURCE_PROFILE = "${PROVER_RESOURCE_PROFILE}" RPC_RESOURCE_PROFILE = "${RPC_RESOURCE_PROFILE}" FULL_NODE_RESOURCE_PROFILE = "${FULL_NODE_RESOURCE_PROFILE}" -BLOB_SINK_RESOURCE_PROFILE = "${BLOB_SINK_RESOURCE_PROFILE}" AZTEC_DOCKER_IMAGE = "${AZTEC_DOCKER_IMAGE}" PROVER_AGENT_DOCKER_IMAGE = "${PROVER_AGENT_DOCKER_IMAGE:-$AZTEC_DOCKER_IMAGE}" VALIDATOR_HA_DOCKER_IMAGE = "${VALIDATOR_HA_DOCKER_IMAGE:-}" diff --git a/spartan/terraform/deploy-aztec-infra/main.tf b/spartan/terraform/deploy-aztec-infra/main.tf index 3c9988f079f5..6f5939cb1e26 100644 --- a/spartan/terraform/deploy-aztec-infra/main.tf +++ b/spartan/terraform/deploy-aztec-infra/main.tf @@ -491,6 +491,7 @@ locals { "node.env.P2P_DROP_TX_CHANCE" = var.P2P_DROP_TX_CHANCE "node.env.P2P_MAX_PENDING_TX_COUNT" = var.P2P_MAX_PENDING_TX_COUNT "node.env.WS_NUM_HISTORIC_CHECKPOINTS" = var.WS_NUM_HISTORIC_CHECKPOINTS + "node.env.BLOB_FILE_STORE_UPLOAD_URL" = var.BLOB_FILE_STORE_UPLOAD_URL "node.env.TX_FILE_STORE_ENABLED" = var.TX_FILE_STORE_ENABLED "node.env.TX_FILE_STORE_URL" = var.TX_FILE_STORE_URL "node.env.TX_COLLECTION_FILE_STORE_URLS" = var.TX_COLLECTION_FILE_STORE_URLS @@ -587,43 +588,6 @@ locals { wait = false } : null - # Blob sink: uploads blobs to filestore as it syncs - blob_sink = var.BLOB_FILE_STORE_UPLOAD_URL != null ? { - name = "${var.RELEASE_PREFIX}-blob-sink" - chart = "aztec-node" - values = [ - "common.yaml", - "blob-sink.yaml", - "blob-sink-resources-${var.BLOB_SINK_RESOURCE_PROFILE}.yaml" - ] - inline_values = [yamlencode({ - service = { - p2p = { publicIP = var.P2P_PUBLIC_IP } - } - })] - custom_settings = { - "nodeType" = "blob-sink" - "service.p2p.nodePortEnabled" = var.P2P_NODEPORT_ENABLED - "node.proverRealProofs" = var.PROVER_REAL_PROOFS - "node.env.BLOB_FILE_STORE_UPLOAD_URL" = var.BLOB_FILE_STORE_UPLOAD_URL - "node.env.AWS_ACCESS_KEY_ID" = var.R2_ACCESS_KEY_ID - "node.env.AWS_SECRET_ACCESS_KEY" = var.R2_SECRET_ACCESS_KEY - "node.env.DEBUG_FORCE_TX_PROOF_VERIFICATION" = var.DEBUG_FORCE_TX_PROOF_VERIFICATION - "node.env.DEBUG_P2P_INSTRUMENT_MESSAGES" = var.DEBUG_P2P_INSTRUMENT_MESSAGES - "node.env.BLOB_ALLOW_EMPTY_SOURCES" = var.BLOB_ALLOW_EMPTY_SOURCES - "node.env.P2P_GOSSIPSUB_D" = var.P2P_GOSSIPSUB_D - "node.env.P2P_GOSSIPSUB_DLO" = var.P2P_GOSSIPSUB_DLO - "node.env.P2P_GOSSIPSUB_DHI" = var.P2P_GOSSIPSUB_DHI - "node.env.P2P_DROP_TX_CHANCE" = var.P2P_DROP_TX_CHANCE - "node.env.P2P_MAX_PENDING_TX_COUNT" = var.P2P_MAX_PENDING_TX_COUNT - "node.env.WS_NUM_HISTORIC_CHECKPOINTS" = var.WS_NUM_HISTORIC_CHECKPOINTS - "node.env.SEQ_ENABLE_PROPOSER_PIPELINING" = var.SEQ_ENABLE_PROPOSER_PIPELINING - } - boot_node_host_path = "node.env.BOOT_NODE_HOST" - bootstrap_nodes_path = "node.env.BOOTSTRAP_NODES" - wait = true - } : null - # Optional: transfer bots bot_transfers = var.BOT_TRANSFERS_REPLICAS > 0 ? { name = "${var.RELEASE_PREFIX}-bot-transfers" diff --git a/spartan/terraform/deploy-aztec-infra/values/blob-sink-resources-dev.yaml b/spartan/terraform/deploy-aztec-infra/values/blob-sink-resources-dev.yaml deleted file mode 100644 index cd8db4b4de0b..000000000000 --- a/spartan/terraform/deploy-aztec-infra/values/blob-sink-resources-dev.yaml +++ /dev/null @@ -1,34 +0,0 @@ -replicaCount: 1 - -node: - resources: - requests: - cpu: "0.25" - memory: "1Gi" - limits: - cpu: "0.5" - memory: "2Gi" - -persistence: - enabled: true - -statefulSet: - enabled: true - volumeClaimTemplates: - - metadata: - name: data - spec: - accessModes: [ReadWriteOnce] - resources: - requests: - storage: 1Gi - -service: - type: ClusterIP - p2p: - enabled: true - nodePortEnabled: false - admin: - enabled: false - headless: - enabled: false diff --git a/spartan/terraform/deploy-aztec-infra/values/blob-sink-resources-prod.yaml b/spartan/terraform/deploy-aztec-infra/values/blob-sink-resources-prod.yaml deleted file mode 100644 index 34193fd66b67..000000000000 --- a/spartan/terraform/deploy-aztec-infra/values/blob-sink-resources-prod.yaml +++ /dev/null @@ -1,39 +0,0 @@ -nodeSelector: - local-ssd: "false" - node-type: "network" - cores: "2" - -replicaCount: 1 - -node: - resources: - requests: - cpu: "0.5" - memory: "2Gi" - limits: - cpu: "1.5" - memory: "6Gi" - -persistence: - enabled: true - -statefulSet: - enabled: true - volumeClaimTemplates: - - metadata: - name: data - spec: - accessModes: [ReadWriteOnce] - resources: - requests: - storage: 16Gi - -service: - type: ClusterIP - p2p: - enabled: true - nodePortEnabled: false - admin: - enabled: false - headless: - enabled: false diff --git a/spartan/terraform/deploy-aztec-infra/values/blob-sink.yaml b/spartan/terraform/deploy-aztec-infra/values/blob-sink.yaml deleted file mode 100644 index 6dc71a519297..000000000000 --- a/spartan/terraform/deploy-aztec-infra/values/blob-sink.yaml +++ /dev/null @@ -1,18 +0,0 @@ -node: - nodeType: "blob-sink" - env: - OTEL_SERVICE_NAME: "blob-sink" - - preStartScript: | - if [ -n "${BOOT_NODE_HOST:-}" ]; then - until curl --silent --head --fail "${BOOT_NODE_HOST}/status" > /dev/null; do - echo "Waiting for boot node..." - sleep 1 - done - echo "Boot node is ready!" - - export BOOTSTRAP_NODES=$(curl -X POST -H "content-type: application/json" --data '{"method": "bootstrap_getEncodedEnr"}' $BOOT_NODE_HOST | jq -r .result) - fi - - startCmd: - - --node diff --git a/spartan/terraform/deploy-aztec-infra/variables.tf b/spartan/terraform/deploy-aztec-infra/variables.tf index 7906d34030a8..c9377b9e3a01 100644 --- a/spartan/terraform/deploy-aztec-infra/variables.tf +++ b/spartan/terraform/deploy-aztec-infra/variables.tf @@ -51,11 +51,6 @@ variable "BOT_RESOURCE_PROFILE" { type = string } -variable "BLOB_SINK_RESOURCE_PROFILE" { - description = "Resource profile to use for the blob sink" - type = string -} - variable "DEBUG_P2P_INSTRUMENT_MESSAGES" { description = "Whether to enable debug instrumentation of P2P messages" type = bool From 43764ab311432f8c969961c814454551f6dc35fd Mon Sep 17 00:00:00 2001 From: Aztec Bot <49558828+AztecBot@users.noreply.github.com> Date: Fri, 29 May 2026 07:12:05 -0400 Subject: [PATCH 13/24] fix: sync avm-transpiler Cargo.lock with noir submodule (#23683) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem CI on `merge-train/spartan` is failing in the `avm-transpiler-native` build step ([log](http://ci.aztec-labs.com/1780052027304932)): ``` error: the lock file /home/aztec-dev/aztec-packages/avm-transpiler/Cargo.lock needs to be updated but --locked was passed to prevent this ``` `cargo build --release --locked --bin avm-transpiler` rejects the stale lock file. ## Root cause The `noir/noir-repo` submodule was bumped on this branch and its `acvm-repo` crates (`acir`, `acir_field`, `brillig`) gained a new path dependency, `msgpack_tagged` (+ `msgpack_tagged_derive`). `avm-transpiler/Cargo.lock` was not regenerated, so it no longer matches `Cargo.toml` and the `--locked` CI build fails. ## Fix Regenerated `avm-transpiler/Cargo.lock` with a plain (non-`--locked`) build so only the required entries change — no bulk update. The diff adds `msgpack_tagged`/`msgpack_tagged_derive` to the relevant dependency lists plus the transitive deps they introduce (`serde_bytes`, `bs58`, `tinyvec`, and a bump of `darling`/`serde_with`). ## Verification - Reproduced the failure: `cargo build --release --locked --bin avm-transpiler` → lock-file error. - After the fix: `cargo build --release --locked --bin avm-transpiler` succeeds. - `./bootstrap.sh build_native` in `avm-transpiler/` (the exact failing CI step) completes cleanly. Only `avm-transpiler/Cargo.lock` is changed. --- *Created by [claudebox](https://claudebox.work/v2/sessions/d9f97447b0ab23ed) · group: `slackbot`* --- avm-transpiler/Cargo.lock | 81 +++++++++++++++++++++++++++++++++------ 1 file changed, 69 insertions(+), 12 deletions(-) diff --git a/avm-transpiler/Cargo.lock b/avm-transpiler/Cargo.lock index e305fd22e1a8..f1e392e03f57 100644 --- a/avm-transpiler/Cargo.lock +++ b/avm-transpiler/Cargo.lock @@ -10,6 +10,7 @@ dependencies = [ "base64 0.22.1", "brillig", "flate2", + "msgpack_tagged", "num-bigint", "num-traits", "num_enum", @@ -29,8 +30,10 @@ dependencies = [ "ark-ff", "cfg-if", "hex", + "msgpack_tagged", "num-bigint", "serde", + "serde_bytes", ] [[package]] @@ -461,6 +464,7 @@ version = "1.0.0-beta.21" dependencies = [ "acir_field", "itertools", + "msgpack_tagged", "serde", ] @@ -476,6 +480,15 @@ dependencies = [ "thiserror", ] +[[package]] +name = "bs58" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" +dependencies = [ + "tinyvec", +] + [[package]] name = "bumpalo" version = "3.19.1" @@ -701,9 +714,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ "darling_core", "darling_macro", @@ -711,11 +724,10 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" dependencies = [ - "fnv", "ident_case", "proc-macro2", "quote", @@ -725,9 +737,9 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.21.3" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ "darling_core", "quote", @@ -914,7 +926,6 @@ version = "1.0.0-beta.21" dependencies = [ "codespan-reporting", "iter-extended", - "itertools", "serde", ] @@ -1219,6 +1230,26 @@ dependencies = [ "simd-adler32", ] +[[package]] +name = "msgpack_tagged" +version = "1.0.0-beta.20" +dependencies = [ + "msgpack_tagged_derive", + "rmp", + "rmp-serde", + "serde", + "smallvec", +] + +[[package]] +name = "msgpack_tagged_derive" +version = "1.0.0-beta.20" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "noirc_abi" version = "1.0.0-beta.21" @@ -1813,6 +1844,16 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_bytes" +version = "0.11.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" +dependencies = [ + "serde", + "serde_core", +] + [[package]] name = "serde_core" version = "1.0.228" @@ -1857,11 +1898,12 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.16.1" +version = "3.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fa237f2807440d238e0364a218270b98f767a00d3dada77b1c53ae88940e2e7" +checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2" dependencies = [ "base64 0.22.1", + "bs58", "chrono", "hex", "indexmap 1.9.3", @@ -1876,9 +1918,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.16.1" +version = "3.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52a8e3ca0ca629121f70ab50f95249e5a6f925cc0f6ffe8256c45b728875706c" +checksum = "b90c488738ecb4fb0262f41f43bc40efc5868d9fb744319ddf5f5317f417bfac" dependencies = [ "darling", "proc-macro2", @@ -2068,6 +2110,21 @@ dependencies = [ "time-core", ] +[[package]] +name = "tinyvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + [[package]] name = "toml" version = "0.8.23" From 422538c1ec00c0f76141df6037f5daca4e20d34d Mon Sep 17 00:00:00 2001 From: spypsy Date: Fri, 29 May 2026 14:22:47 +0300 Subject: [PATCH 14/24] fix(spartan): set validator lag env vars in tps-scenario (#23684) - Replace stale `AZTEC_LAG_IN_EPOCHS` in `tps-scenario.env` with `AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET` and `AZTEC_LAG_IN_EPOCHS_FOR_RANDAO` (value 1 each). - Fixes immediate failure in nightly Spartan `wait-bench-l2-block` ([run #184](https://github.com/AztecProtocol/aztec-packages/actions/runs/26627597396/job/78472605259)). Co-authored-by: PhilWindle <60546371+PhilWindle@users.noreply.github.com> --- spartan/environments/tps-scenario.env | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spartan/environments/tps-scenario.env b/spartan/environments/tps-scenario.env index 4a1f08e156e2..fc9ff68990c0 100644 --- a/spartan/environments/tps-scenario.env +++ b/spartan/environments/tps-scenario.env @@ -6,7 +6,8 @@ GCP_REGION=us-west1-a AZTEC_EPOCH_DURATION=8 AZTEC_SLOT_DURATION=72 AZTEC_PROOF_SUBMISSION_EPOCHS=2 -AZTEC_LAG_IN_EPOCHS=1 +AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET=1 +AZTEC_LAG_IN_EPOCHS_FOR_RANDAO=1 SPONSORED_FPC=true CREATE_ETH_DEVNET=false From 3345feaaad8c5f773dfd73b09f1e827181028d5a Mon Sep 17 00:00:00 2001 From: AztecBot Date: Fri, 29 May 2026 11:24:57 +0000 Subject: [PATCH 15/24] update PR #23677 --- yarn-project/aztec-node/src/aztec-node/server.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/yarn-project/aztec-node/src/aztec-node/server.test.ts b/yarn-project/aztec-node/src/aztec-node/server.test.ts index b37860749c4e..b284a3cb4562 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.test.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.test.ts @@ -628,7 +628,9 @@ describe('aztec node', () => { // rather than syncing to bare latest height and racing the snapshot read. const blockHash = BlockHash.random(); l2BlockSource.getBlockNumber.mockImplementation(((query?: BlockQuery) => - Promise.resolve(query && 'hash' in query ? BlockNumber(3) : lastBlockNumber)) as L2BlockSource['getBlockNumber']); + Promise.resolve( + query && 'hash' in query ? BlockNumber(3) : lastBlockNumber, + )) as L2BlockSource['getBlockNumber']); snapshotMerkleTreeOps.getLeafValue.mockResolvedValue(blockHash); await node.getWorldState(blockHash); From 6622f23563c38db2cc7bcd60819ce4c8c8178024 Mon Sep 17 00:00:00 2001 From: Aztec Bot <49558828+AztecBot@users.noreply.github.com> Date: Fri, 29 May 2026 08:22:45 -0400 Subject: [PATCH 16/24] fix: pin noir submodule to next's version on merge-train/spartan (#23690) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem CI on `merge-train/spartan` is failing in the `aztec-nr` step ([log](http://ci.aztec-labs.com/1780053790683323)) with `BoundedVec::from_parts_unchecked` deprecation errors under `nargo check --deny-warnings`. The train's noir submodule had diverged from `next`: | Branch | noir pin | date | `from_parts_unchecked` deprecated? | |---|---|---|---| | `next` | `f1a4575` | May 11 | no | | `merge-train/spartan` | `4d039268` | May 28 | **yes** | The newer pin (`4d039268`) was pulled onto the train by **PR #23675** ("fix(cheat-codes): warpL2TimeAtLeastBy…"), which bumped `noir/noir-repo` from the May-11 pin to a May-28 nightly. That nightly added `#[deprecated]` to `BoundedVec::from_parts_unchecked`, and since aztec-nr builds with `--deny-warnings`, the two remaining call sites became hard errors. The two noir commits are on divergent lines (neither is an ancestor of the other), so the train was simply ahead of `next` on noir. ## Fix Pin the train's noir back to exactly what `next` uses. Only two files differed from `next`: - `noir/noir-repo` → `f1a4575` (next's pin) - `avm-transpiler/Cargo.lock` → next's version (it had been re-synced to the May-28 noir by #23683; restored to match the May-11 pin) This restores parity with `next` and removes the deprecated API entirely, so no aztec-nr source change is needed. ## Verification Built `nargo` from the `f1a4575` pin and ran the failing check against the **unmodified** aztec-nr source: - `nargo check --deny-warnings` → exit 0 (the deprecation attribute is absent in `f1a4575`). ## Note This is an alternative to #23687, which fixed the same failure by patching the two aztec-nr call sites to use `from_parts` against the newer noir. Pick one: this PR keeps the train aligned with `next`'s noir; #23687 keeps the newer noir and updates the source. Closing whichever isn't chosen. --- *Created by [claudebox](https://claudebox.work/v2/sessions/2f980b2000011f91) · group: `slackbot`* --- avm-transpiler/Cargo.lock | 81 ++++++--------------------------------- noir/noir-repo | 2 +- 2 files changed, 13 insertions(+), 70 deletions(-) diff --git a/avm-transpiler/Cargo.lock b/avm-transpiler/Cargo.lock index f1e392e03f57..e305fd22e1a8 100644 --- a/avm-transpiler/Cargo.lock +++ b/avm-transpiler/Cargo.lock @@ -10,7 +10,6 @@ dependencies = [ "base64 0.22.1", "brillig", "flate2", - "msgpack_tagged", "num-bigint", "num-traits", "num_enum", @@ -30,10 +29,8 @@ dependencies = [ "ark-ff", "cfg-if", "hex", - "msgpack_tagged", "num-bigint", "serde", - "serde_bytes", ] [[package]] @@ -464,7 +461,6 @@ version = "1.0.0-beta.21" dependencies = [ "acir_field", "itertools", - "msgpack_tagged", "serde", ] @@ -480,15 +476,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "bs58" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" -dependencies = [ - "tinyvec", -] - [[package]] name = "bumpalo" version = "3.19.1" @@ -714,9 +701,9 @@ dependencies = [ [[package]] name = "darling" -version = "0.23.0" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" dependencies = [ "darling_core", "darling_macro", @@ -724,10 +711,11 @@ dependencies = [ [[package]] name = "darling_core" -version = "0.23.0" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9865a50f7c335f53564bb694ef660825eb8610e0a53d3e11bf1b0d3df31e03b0" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" dependencies = [ + "fnv", "ident_case", "proc-macro2", "quote", @@ -737,9 +725,9 @@ dependencies = [ [[package]] name = "darling_macro" -version = "0.23.0" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ "darling_core", "quote", @@ -926,6 +914,7 @@ version = "1.0.0-beta.21" dependencies = [ "codespan-reporting", "iter-extended", + "itertools", "serde", ] @@ -1230,26 +1219,6 @@ dependencies = [ "simd-adler32", ] -[[package]] -name = "msgpack_tagged" -version = "1.0.0-beta.20" -dependencies = [ - "msgpack_tagged_derive", - "rmp", - "rmp-serde", - "serde", - "smallvec", -] - -[[package]] -name = "msgpack_tagged_derive" -version = "1.0.0-beta.20" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "noirc_abi" version = "1.0.0-beta.21" @@ -1844,16 +1813,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_bytes" -version = "0.11.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" -dependencies = [ - "serde", - "serde_core", -] - [[package]] name = "serde_core" version = "1.0.228" @@ -1898,12 +1857,11 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.20.0" +version = "3.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e72c1c2cb7b223fafb600a619537a871c2818583d619401b785e7c0b746ccde2" +checksum = "4fa237f2807440d238e0364a218270b98f767a00d3dada77b1c53ae88940e2e7" dependencies = [ "base64 0.22.1", - "bs58", "chrono", "hex", "indexmap 1.9.3", @@ -1918,9 +1876,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.20.0" +version = "3.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b90c488738ecb4fb0262f41f43bc40efc5868d9fb744319ddf5f5317f417bfac" +checksum = "52a8e3ca0ca629121f70ab50f95249e5a6f925cc0f6ffe8256c45b728875706c" dependencies = [ "darling", "proc-macro2", @@ -2110,21 +2068,6 @@ dependencies = [ "time-core", ] -[[package]] -name = "tinyvec" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e61e67053d25a4e82c844e8424039d9745781b3fc4f32b8d55ed50f5f667ef3" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - [[package]] name = "toml" version = "0.8.23" diff --git a/noir/noir-repo b/noir/noir-repo index 4d0392685f8b..f1a4575adac5 160000 --- a/noir/noir-repo +++ b/noir/noir-repo @@ -1 +1 @@ -Subproject commit 4d0392685f8b9a4a9895d675d3ed730716559860 +Subproject commit f1a4575adac59af0a86b036cf73ff5883d142a91 From c7703c17e20fc33408226896ea835ffcff50d1d7 Mon Sep 17 00:00:00 2001 From: spypsy Date: Fri, 29 May 2026 16:15:17 +0300 Subject: [PATCH 17/24] fix: ensure image ref is used by bench runner (#23682) Using the nightly tag as `source_ref` to ensure the code that the runner uses is the same as what's deployed on GKE --- .github/workflows/nightly-bench-10tps.yml | 43 +++++++++++++--- .github/workflows/nightly-spartan-bench.yml | 57 ++++++++++++++------- .github/workflows/weekly-proving-bench.yml | 30 ++++++++--- 3 files changed, 99 insertions(+), 31 deletions(-) diff --git a/.github/workflows/nightly-bench-10tps.yml b/.github/workflows/nightly-bench-10tps.yml index 47a4df7db8e0..09281040dbb6 100644 --- a/.github/workflows/nightly-bench-10tps.yml +++ b/.github/workflows/nightly-bench-10tps.yml @@ -13,6 +13,10 @@ on: description: "Nightly tag to use (e.g., 2.3.4-nightly.20251209). Ignored if docker_image is set. Leave empty to auto-detect." required: false type: string + source_ref: + description: "Git ref to checkout (e.g., v5.0.0-nightly.20260529). Required when docker_image is not a standard nightly tag." + required: false + type: string concurrency: group: nightly-bench-10tps-${{ github.ref }} @@ -24,6 +28,7 @@ jobs: outputs: docker_image: ${{ steps.docker-image.outputs.docker_image }} image_label: ${{ steps.docker-image.outputs.image_label }} + source_ref: ${{ steps.docker-image.outputs.source_ref }} steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd @@ -36,19 +41,43 @@ jobs: if [[ -n "${{ inputs.docker_image }}" ]]; then docker_image="${{ inputs.docker_image }}" image_label="${{ inputs.docker_image }}" + image_tag="${docker_image##*:}" elif [[ -n "${{ inputs.nightly_tag }}" ]]; then nightly_tag="${{ inputs.nightly_tag }}" docker_image="aztecprotocol/aztec:${nightly_tag}" image_label="${nightly_tag}" + image_tag="${nightly_tag}" else current_version=$(jq -r '."."' .release-please-manifest.json) nightly_tag="${current_version}-nightly.$(date -u +%Y%m%d)" docker_image="aztecprotocol/aztec:${nightly_tag}" image_label="${nightly_tag}" + image_tag="${nightly_tag}" fi + + if [[ -n "${{ inputs.source_ref }}" ]]; then + source_ref="${{ inputs.source_ref }}" + elif [[ "${image_tag:-}" =~ ^[0-9]+\.[0-9]+\.[0-9]+-nightly\.[0-9]{8}$ ]]; then + source_ref="v${image_tag}" + else + echo "Could not infer git ref from docker image. Pass workflow input source_ref." + exit 1 + fi + echo "docker_image=$docker_image" >> "$GITHUB_OUTPUT" echo "image_label=$image_label" >> "$GITHUB_OUTPUT" + echo "source_ref=$source_ref" >> "$GITHUB_OUTPUT" echo "Using docker image: $docker_image" + echo "Using source ref: $source_ref" + + - name: Verify source git ref + id: verify-source + run: | + set -euo pipefail + source_ref="${{ steps.docker-image.outputs.source_ref }}" + git fetch --depth 1 origin "refs/tags/${source_ref}:refs/tags/${source_ref}" + source_sha=$(git rev-parse "${source_ref}^{}") + echo "Nightly source commit: $source_sha" - name: Check if Docker image exists run: | @@ -68,19 +97,21 @@ jobs: network: bench-10tps namespace: bench-10tps aztec_docker_image: ${{ needs.select-image.outputs.docker_image }} - ref: next + ref: ${{ needs.select-image.outputs.source_ref }} notify_on_failure: false secrets: inherit wait-for-first-l2-block: - needs: deploy-bench-10tps-network + needs: + - select-image + - deploy-bench-10tps-network runs-on: ubuntu-latest timeout-minutes: 120 steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: ${{ github.sha }} + ref: ${{ needs.select-image.outputs.source_ref }} - name: Authenticate to Google Cloud uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f @@ -108,7 +139,7 @@ jobs: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: ${{ github.sha }} + ref: ${{ needs.select-image.outputs.source_ref }} - name: Run 10 TPS benchmark timeout-minutes: 240 @@ -139,7 +170,7 @@ jobs: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: next + ref: ${{ needs.select-image.outputs.source_ref }} - name: Cleanup network resources env: @@ -165,7 +196,7 @@ jobs: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: next + ref: ${{ needs.select-image.outputs.source_ref }} - name: Notify Slack on failure env: diff --git a/.github/workflows/nightly-spartan-bench.yml b/.github/workflows/nightly-spartan-bench.yml index 8b2c83f487d3..d1f3303ff17c 100644 --- a/.github/workflows/nightly-spartan-bench.yml +++ b/.github/workflows/nightly-spartan-bench.yml @@ -20,6 +20,7 @@ jobs: outputs: nightly_tag: ${{ steps.nightly-tag.outputs.nightly_tag }} docker_image: ${{ steps.nightly-tag.outputs.docker_image }} + source_ref: ${{ steps.nightly-tag.outputs.source_ref }} steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd @@ -36,9 +37,20 @@ jobs: nightly_tag="${current_version}-nightly.$(date -u +%Y%m%d)" fi docker_image="aztecprotocol/aztec:${nightly_tag}" + source_ref="v${nightly_tag}" echo "nightly_tag=$nightly_tag" >> "$GITHUB_OUTPUT" echo "docker_image=$docker_image" >> "$GITHUB_OUTPUT" + echo "source_ref=$source_ref" >> "$GITHUB_OUTPUT" echo "Using nightly tag: $nightly_tag" + echo "Using source ref: $source_ref" + + - name: Verify source git ref + run: | + set -euo pipefail + source_ref="${{ steps.nightly-tag.outputs.source_ref }}" + git fetch --depth 1 origin "refs/tags/${source_ref}:refs/tags/${source_ref}" + source_sha=$(git rev-parse "${source_ref}^{}") + echo "Nightly source commit: $source_sha" - name: Check if Docker image exists run: | @@ -61,19 +73,21 @@ jobs: network: tps-scenario namespace: nightly-bench aztec_docker_image: ${{ needs.select-image.outputs.docker_image }} - ref: next + ref: ${{ needs.select-image.outputs.source_ref }} notify_on_failure: false secrets: inherit wait-bench-l2-block: - needs: deploy-bench-network + needs: + - select-image + - deploy-bench-network runs-on: ubuntu-latest timeout-minutes: 120 steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: next + ref: ${{ needs.select-image.outputs.source_ref }} - name: Authenticate to Google Cloud uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f @@ -102,7 +116,7 @@ jobs: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: next + ref: ${{ needs.select-image.outputs.source_ref }} - name: Run benchmarks timeout-minutes: 240 @@ -152,6 +166,7 @@ jobs: cleanup-bench: if: always() needs: + - select-image - deploy-bench-network - wait-bench-l2-block - benchmark @@ -160,7 +175,7 @@ jobs: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: next + ref: ${{ needs.select-image.outputs.source_ref }} - name: Cleanup network resources env: @@ -186,7 +201,7 @@ jobs: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: next + ref: ${{ needs.select-image.outputs.source_ref }} - name: Notify Slack and dispatch ClaudeBox on failure env: @@ -210,19 +225,21 @@ jobs: network: prove-n-tps-fake namespace: prove-n-tps-fake aztec_docker_image: ${{ needs.select-image.outputs.docker_image }} - ref: next + ref: ${{ needs.select-image.outputs.source_ref }} notify_on_failure: false secrets: inherit wait-proving-l2-block: - needs: deploy-proving-network + needs: + - select-image + - deploy-proving-network runs-on: ubuntu-latest timeout-minutes: 120 steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: next + ref: ${{ needs.select-image.outputs.source_ref }} - name: Authenticate to Google Cloud uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f @@ -251,7 +268,7 @@ jobs: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: next + ref: ${{ needs.select-image.outputs.source_ref }} - name: Run proving benchmarks timeout-minutes: 180 @@ -299,6 +316,7 @@ jobs: cleanup-proving: if: always() needs: + - select-image - deploy-proving-network - wait-proving-l2-block - proving-benchmark @@ -307,7 +325,7 @@ jobs: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: next + ref: ${{ needs.select-image.outputs.source_ref }} - name: Cleanup network resources env: @@ -333,7 +351,7 @@ jobs: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: next + ref: ${{ needs.select-image.outputs.source_ref }} - name: Notify Slack and dispatch ClaudeBox on failure env: @@ -357,19 +375,21 @@ jobs: network: block-capacity namespace: nightly-block-capacity aztec_docker_image: ${{ needs.select-image.outputs.docker_image }} - ref: next + ref: ${{ needs.select-image.outputs.source_ref }} notify_on_failure: false secrets: inherit wait-block-capacity-l2-block: - needs: deploy-block-capacity-network + needs: + - select-image + - deploy-block-capacity-network runs-on: ubuntu-latest timeout-minutes: 120 steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: next + ref: ${{ needs.select-image.outputs.source_ref }} - name: Authenticate to Google Cloud uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f @@ -398,7 +418,7 @@ jobs: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: next + ref: ${{ needs.select-image.outputs.source_ref }} - name: Run block capacity benchmarks timeout-minutes: 240 @@ -447,6 +467,7 @@ jobs: cleanup-block-capacity: if: always() needs: + - select-image - deploy-block-capacity-network - wait-block-capacity-l2-block - block-capacity-benchmark @@ -455,7 +476,7 @@ jobs: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: next + ref: ${{ needs.select-image.outputs.source_ref }} - name: Cleanup network resources env: @@ -481,7 +502,7 @@ jobs: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: next + ref: ${{ needs.select-image.outputs.source_ref }} - name: Notify Slack and dispatch ClaudeBox on failure env: diff --git a/.github/workflows/weekly-proving-bench.yml b/.github/workflows/weekly-proving-bench.yml index ff5b060e82ff..2f487158faa8 100644 --- a/.github/workflows/weekly-proving-bench.yml +++ b/.github/workflows/weekly-proving-bench.yml @@ -20,6 +20,7 @@ jobs: outputs: nightly_tag: ${{ steps.nightly-tag.outputs.nightly_tag }} docker_image: ${{ steps.nightly-tag.outputs.docker_image }} + source_ref: ${{ steps.nightly-tag.outputs.source_ref }} steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd @@ -37,9 +38,20 @@ jobs: fi docker_image="aztecprotocol/aztec:${nightly_tag}" + source_ref="v${nightly_tag}" echo "nightly_tag=$nightly_tag" >> "$GITHUB_OUTPUT" echo "docker_image=$docker_image" >> "$GITHUB_OUTPUT" + echo "source_ref=$source_ref" >> "$GITHUB_OUTPUT" echo "Using nightly tag: $nightly_tag" + echo "Using source ref: $source_ref" + + - name: Verify source git ref + run: | + set -euo pipefail + source_ref="${{ steps.nightly-tag.outputs.source_ref }}" + git fetch --depth 1 origin "refs/tags/${source_ref}:refs/tags/${source_ref}" + source_sha=$(git rev-parse "${source_ref}^{}") + echo "Nightly source commit: $source_sha" - name: Check if Docker image exists run: | @@ -59,19 +71,21 @@ jobs: network: prove-n-tps-real namespace: prove-n-tps-real aztec_docker_image: ${{ needs.select-image.outputs.docker_image }} - ref: next + ref: ${{ needs.select-image.outputs.source_ref }} notify_on_failure: false secrets: inherit wait-for-first-l2-block: - needs: deploy-real-proving-network + needs: + - select-image + - deploy-real-proving-network runs-on: ubuntu-latest timeout-minutes: 120 steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: ${{ github.sha }} + ref: ${{ needs.select-image.outputs.source_ref }} - name: Authenticate to Google Cloud uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f @@ -91,13 +105,15 @@ jobs: ./bootstrap.sh wait_for_l2_block prove-n-tps-real benchmark: - needs: wait-for-first-l2-block + needs: + - select-image + - wait-for-first-l2-block runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: next + ref: ${{ needs.select-image.outputs.source_ref }} - name: Run real proving benchmarks timeout-minutes: 180 @@ -154,7 +170,7 @@ jobs: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: next + ref: ${{ needs.select-image.outputs.source_ref }} - name: Cleanup network resources env: @@ -180,7 +196,7 @@ jobs: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: next + ref: ${{ needs.select-image.outputs.source_ref }} - name: Notify Slack and dispatch ClaudeBox on failure env: From ffa2c6c13740d03326a2e8a9b80d5facb8def997 Mon Sep 17 00:00:00 2001 From: Aztec Bot <49558828+AztecBot@users.noreply.github.com> Date: Fri, 29 May 2026 10:34:34 -0400 Subject: [PATCH 18/24] fix(ci): retry aztec-nr nargo dependency clone on transient network flake (#23653) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Why The `merge-train/spartan` train PR (#23580) was dequeued from the merge queue. The merge-queue CI3 run ([run 26608568295](https://github.com/AztecProtocol/aztec-packages/actions/runs/26608568295)) failed in the `x2-full amd64 ci-full-no-test-cache` grind during the **aztec-nr warnings check**, after only ~11s: ``` Checking aztec-nr for warnings... Cloning into '.../noir-lang/poseidon/v0.3.0'... Cloning into '.../noir-lang/sha256/v0.3.0'... fatal: unable to access 'https://github.com/noir-lang/sha256/': Could not resolve host: github.com Cannot read file .../noir-lang/sha256/v0.3.0/Nargo.toml - does it exist? make: *** [Makefile:303: aztec-nr] Error 1 ``` A transient DNS/network flake on the runner — not a code defect. `aztec-nr/aztec/Nargo.toml` declares external git dependencies (`noir-lang/sha256`, `noir-lang/poseidon`, pinned at `v0.3.0`) which `nargo check` resolves by cloning from `github.com` on a cold cache. When the runner momentarily can't resolve `github.com`, the clone fails and dequeues the whole train. ## What A blanket `retry` around `nargo check` would also re-run on genuine check failures (type errors, denied warnings) — wasting CI time and masking intent. So instead: - **`ci3/retry` gains a `-p ` option.** It captures the command's combined output and only retries when a failure matches the regex; any non-matching failure exits immediately with the original code. Without `-p`, behavior is unchanged (the heavily-used default path is untouched). `pipefail` ensures the wrapped command's exit code (not `tee`'s) is what's checked, and `tee` finishes before inspection so the captured output is complete. - **`aztec-nr/bootstrap.sh`** wraps its two network-touching nargo calls (`check`, `doc --check`) with `retry -p ""`, matching only `Could not resolve host`, `unable to access`, `Connection timed out/refused`, `Failed to connect`, `TLS connect error`, `early EOF`, `RPC failed`. These never overlap with nargo's `error:`/`warning:` output, so a genuine check failure still fails on the first attempt. `nargo` has no standalone dependency-install/fetch command (its subcommands are `check, compile, dap, debug, doc, execute, expand, export, fmt, fuzz, info, init, interpret, lsp, new, test`); resolution only happens inside `check`/`compile`/`test`, so the regex-gated retry is the workable option of the two suggested. ## Verification - `bash -n` on all three files; `ci3/tests/retry_test` (new, auto-discovered by the ci3 test runner) passes all 6 cases: - default mode retries a transient failure then succeeds / gives up after 3 attempts - pattern mode retries a matching network failure then succeeds - **pattern mode fails fast (1 attempt) on a non-matching genuine error** ← the behavior requested - pattern mode gives up after 3 attempts on a persistent matching failure - `RETRY_DISABLED` runs the command exactly once The full `./bootstrap.sh ci` run is the same orchestrated remote-EC2 CI that failed here and isn't reproducible on a dev host; the transient DNS failure also can't be reproduced where DNS works. Verification is therefore at the retry/wrapper level, which is exactly what this change touches. --- ci3/retry | 27 +++++++++++++++++++++++++-- noir-projects/aztec-nr/bootstrap.sh | 9 +++++++-- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/ci3/retry b/ci3/retry index bc153364de71..1d70eb641732 100755 --- a/ci3/retry +++ b/ci3/retry @@ -2,6 +2,16 @@ set -u # not -e [ "${BUILD_SYSTEM_DEBUG:-}" = 1 ] && set -x +# By default retries any non-zero exit. With "-p " only failures whose (combined) output +# matches the extended regex are retried; any other failure exits immediately so genuine errors +# (e.g. a real compile error) are not masked by retries. +# Usage: retry [-p ] "" +pattern= +if [ "${1:-}" = "-p" ]; then + pattern=$2 + shift 2 +fi + if [ -n "${RETRY_DISABLED:-}" ]; then set -e eval "$1" @@ -9,10 +19,19 @@ if [ -n "${RETRY_DISABLED:-}" ]; then fi ATTEMPTS=3 +out= +[ -n "$pattern" ] && out=$(mktemp) # Retries up to 3 times with 5 second intervals -trap 'kill -SIGTERM $pid &>/dev/null || true' SIGTERM +trap 'kill -SIGTERM $pid &>/dev/null || true; [ -n "$out" ] && rm -f "$out"' SIGTERM EXIT for i in $(seq 1 $ATTEMPTS); do - bash -c "$1" & + if [ -n "$pattern" ]; then + # Capture combined output (while still streaming it) so we can decide whether the failure is + # retryable. pipefail makes the subshell exit with the command's code, not tee's, and the tee + # finishes before the subshell exits so the captured file is complete before we inspect it. + ( set -o pipefail; bash -c "$1" 2>&1 | tee "$out" ) & + else + bash -c "$1" & + fi pid=$! # First wait might be SIGTERM received by this script. # Second should be the actual exit status. @@ -20,6 +39,10 @@ for i in $(seq 1 $ATTEMPTS); do wait $pid code=$? [ $code -eq 0 ] || [ $code -eq 143 ] && exit 0 + if [ -n "$pattern" ] && ! grep -Eq "$pattern" "$out"; then + >&2 echo "Not retrying: failure (code $code) did not match retry pattern: $pattern" + exit $code + fi [ "$i" != "$ATTEMPTS" ] && sleep ${RETRY_SLEEP:-5} done diff --git a/noir-projects/aztec-nr/bootstrap.sh b/noir-projects/aztec-nr/bootstrap.sh index 1c058e07851c..e84f5bb558c5 100755 --- a/noir-projects/aztec-nr/bootstrap.sh +++ b/noir-projects/aztec-nr/bootstrap.sh @@ -16,10 +16,15 @@ function build { # Being a library, aztec-nr does not technically need to be built. But we can still run nargo check to find any type # errors and prevent warnings echo_stderr "Checking aztec-nr for warnings..." - $NARGO check --deny-warnings + # nargo resolves git dependencies (e.g. noir-lang/sha256, noir-lang/poseidon) by cloning from + # github.com on a cold cache, which intermittently fails with transient DNS/network errors. Retry + # only on those transport failures so a flaky clone does not dequeue the merge train, while genuine + # check failures (type errors, warnings) still fail immediately. + local git_net_flake="Could not resolve host|unable to access|Connection timed out|Connection refused|Failed to connect|TLS connect error|early EOF|RPC failed" + retry -p "$git_net_flake" "$NARGO check --deny-warnings" # We also check that no docstring links are broken - $NARGO doc --check + retry -p "$git_net_flake" "$NARGO doc --check" } function test_cmds { From e8eb17d58e1b83a257345c9e37b82554cdc04b64 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Fri, 29 May 2026 15:41:19 +0100 Subject: [PATCH 19/24] chore: run one-off jobs on network nodes (#23701) . --- spartan/aztec-keystore/templates/batchjob.yaml | 4 ++++ spartan/aztec-keystore/values.yaml | 2 ++ spartan/aztec-snapshots/templates/cronjob.yaml | 4 ++++ spartan/aztec-snapshots/values.yaml | 4 +++- spartan/terraform/deploy-aztec-infra/main.tf | 1 + spartan/terraform/deploy-rollup-contracts/main.tf | 3 +++ spartan/terraform/modules/web3signer/main.tf | 3 +++ 7 files changed, 20 insertions(+), 1 deletion(-) diff --git a/spartan/aztec-keystore/templates/batchjob.yaml b/spartan/aztec-keystore/templates/batchjob.yaml index e02b62914c49..94c98082f365 100644 --- a/spartan/aztec-keystore/templates/batchjob.yaml +++ b/spartan/aztec-keystore/templates/batchjob.yaml @@ -9,6 +9,10 @@ spec: template: spec: serviceAccountName: {{ include "aztec-keystore.serviceAccountName" . }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} restartPolicy: OnFailure volumes: - name: shared diff --git a/spartan/aztec-keystore/values.yaml b/spartan/aztec-keystore/values.yaml index d227584b6bd0..51c5af37f094 100644 --- a/spartan/aztec-keystore/values.yaml +++ b/spartan/aztec-keystore/values.yaml @@ -50,6 +50,8 @@ provers: keystores: secretName: "" # if empty, will be generated as {Release.Name}-keystores +nodeSelector: {} + serviceAccount: create: true name: "" # leave empty to generate a name based on Release.Name diff --git a/spartan/aztec-snapshots/templates/cronjob.yaml b/spartan/aztec-snapshots/templates/cronjob.yaml index 4e90be0daeec..1e81d89d1054 100644 --- a/spartan/aztec-snapshots/templates/cronjob.yaml +++ b/spartan/aztec-snapshots/templates/cronjob.yaml @@ -10,6 +10,10 @@ spec: spec: template: spec: + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 12 }} + {{- end }} restartPolicy: Never containers: - name: snapshot diff --git a/spartan/aztec-snapshots/values.yaml b/spartan/aztec-snapshots/values.yaml index 0fd650d63514..9b336af27a7f 100644 --- a/spartan/aztec-snapshots/values.yaml +++ b/spartan/aztec-snapshots/values.yaml @@ -3,7 +3,7 @@ nameOverride: "" # -- Overrides the chart computed fullname fullnameOverride: "" -snapshot: +snapshots: uploadLocation: null frequency: "0 0 * * *" # daily uploads at midnight aztecNodeAdminUrl: null @@ -11,6 +11,8 @@ snapshot: s3SecretAccessKey: s3SessionToken: +nodeSelector: {} + image: repository: curlimages/curl tag: "7.81.0" diff --git a/spartan/terraform/deploy-aztec-infra/main.tf b/spartan/terraform/deploy-aztec-infra/main.tf index 6f5939cb1e26..f436eb52772a 100644 --- a/spartan/terraform/deploy-aztec-infra/main.tf +++ b/spartan/terraform/deploy-aztec-infra/main.tf @@ -287,6 +287,7 @@ locals { "snapshots.aztecNodeAdminUrl" = local.internal_rpc_admin_url "snapshots.uploadLocation" = var.STORE_SNAPSHOT_URL "snapshots.frequency" = var.SNAPSHOT_CRON + "nodeSelector.node-type" = "network" } boot_node_host_path = "" bootstrap_nodes_path = "" diff --git a/spartan/terraform/deploy-rollup-contracts/main.tf b/spartan/terraform/deploy-rollup-contracts/main.tf index 660d1632ed79..23147f1cf70a 100644 --- a/spartan/terraform/deploy-rollup-contracts/main.tf +++ b/spartan/terraform/deploy-rollup-contracts/main.tf @@ -99,6 +99,9 @@ resource "kubernetes_job_v1" "deploy_rollup_contracts" { spec { restart_policy = "Never" + node_selector = { + "node-type" = "network" + } container { name = "deploy-rollup-contracts" diff --git a/spartan/terraform/modules/web3signer/main.tf b/spartan/terraform/modules/web3signer/main.tf index 7cf255f87410..dc9eccb89724 100644 --- a/spartan/terraform/modules/web3signer/main.tf +++ b/spartan/terraform/modules/web3signer/main.tf @@ -52,6 +52,9 @@ resource "helm_release" "keystore_setup" { publishersPerProver = var.PUBLISHERS_PER_PROVER mnemonicStartIndex = var.PROVER_PUBLISHER_MNEMONIC_START_INDEX } + nodeSelector = { + "node-type" = "network" + } }) ] From 0ceeeb2e88a1f12e9a0aff949dc84a0634b91075 Mon Sep 17 00:00:00 2001 From: spypsy Date: Fri, 29 May 2026 18:00:16 +0300 Subject: [PATCH 20/24] fix: simulate proposals inside target slot (#23692) ## Summary - Use the last L1 slot timestamp inside the target L2 slot for proposal/header simulations. - Keep bundle simulation and pre-broadcast header validation on the same timestamp rule to avoid `eth_simulateV1` timestamp-order failures. - Add regression coverage for both simulation paths. --- .../publisher/sequencer-bundle-simulator.ts | 15 +++--- .../src/publisher/sequencer-publisher.test.ts | 46 ++++++++++++++++++- .../src/publisher/sequencer-publisher.ts | 8 +++- 3 files changed, 59 insertions(+), 10 deletions(-) diff --git a/yarn-project/sequencer-client/src/publisher/sequencer-bundle-simulator.ts b/yarn-project/sequencer-client/src/publisher/sequencer-bundle-simulator.ts index 5ae7c1647117..6c3361d31e03 100644 --- a/yarn-project/sequencer-client/src/publisher/sequencer-bundle-simulator.ts +++ b/yarn-project/sequencer-client/src/publisher/sequencer-bundle-simulator.ts @@ -4,7 +4,7 @@ import { type L1TxUtils, MAX_L1_TX_LIMIT } from '@aztec/ethereum/l1-tx-utils'; import { formatViemError } from '@aztec/ethereum/utils'; import type { SlotNumber } from '@aztec/foundation/branded-types'; import { type Logger, createLogger } from '@aztec/foundation/log'; -import { getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; +import { getLastL1SlotTimestampForL2Slot } from '@aztec/stdlib/epoch-helpers'; import type { Hex, StateOverride } from 'viem'; @@ -64,7 +64,7 @@ export class SequencerBundleSimulator { } /** - * Simulates the given bundle at the target slot's start timestamp and filters out entries + * Simulates the given bundle near the end of the target slot and filters out entries * that revert. * * - If all entries pass on the first pass, returns `success` with the gasLimit. @@ -75,14 +75,15 @@ export class SequencerBundleSimulator { * - If eth_simulateV1 is unavailable, returns `fallback`. The caller is expected to send the * bundle as-is with a safe gas limit. * - * The simulation `block.timestamp` is always the target L2 slot's start timestamp, since - * propose's `validateHeader` and EIP-712 signature checks both derive a slot from - * `block.timestamp` and compare against the slot the validator signed for. + * The simulation `block.timestamp` is the last L1 slot timestamp inside the target L2 slot. + * This still maps to the target L2 slot for propose's `validateHeader` and EIP-712 signature + * checks, while avoiding eth_simulateV1 rejecting a child block whose timestamp is not strictly + * greater than the current L1 head. * * Known limitation: on networks where L1 is mining behind cadence (missed L1 slots, anvil with * overridden timestamps), the actual `block.timestamp` at send time can land in the prior L2 * slot. In that case `propose` would revert silently inside the multicall. The simulator does - * not detect this case because it simulates AT the target timestamp — the prior implementation + * not detect this case because it simulates inside the target slot — the prior implementation * used `min(predictedNextL1Ts, targetTimestamp)` to surface this failure mode at simulate time. */ public async simulate(validRequests: RequestWithExpiry[], targetSlot: SlotNumber): Promise { @@ -94,7 +95,7 @@ export class SequencerBundleSimulator { const l1TxUtils = this.deps.getL1TxUtils(); const proposeRequest = validRequests.find(r => r.action === 'propose'); - const simulateTimestamp = getTimestampForSlot(targetSlot, this.deps.epochCache.getL1Constants()); + const simulateTimestamp = getLastL1SlotTimestampForL2Slot(targetSlot, this.deps.epochCache.getL1Constants()); const firstPassOverrides = await this.buildStateOverrides(!!proposeRequest); const firstPass = await this.simulateAndDecode(l1TxUtils, validRequests, simulateTimestamp, firstPassOverrides); diff --git a/yarn-project/sequencer-client/src/publisher/sequencer-publisher.test.ts b/yarn-project/sequencer-client/src/publisher/sequencer-publisher.test.ts index 2a9bac671e44..bf7b7f2fafb8 100644 --- a/yarn-project/sequencer-client/src/publisher/sequencer-publisher.test.ts +++ b/yarn-project/sequencer-client/src/publisher/sequencer-publisher.test.ts @@ -68,6 +68,7 @@ describe('SequencerPublisher', () => { let rollup: MockProxy; let slashingProposerContract: MockProxy; let governanceProposerContract: MockProxy; + let epochCache: MockProxy; let l1TxUtils: MockProxy; let l1Metrics: MockProxy; let forwardSpy: jest.SpiedFunction; @@ -137,7 +138,7 @@ describe('SequencerPublisher', () => { governanceProposerContract = mock(); - const epochCache = mock(); + epochCache = mock(); epochCache.getEpochAndSlotNow.mockReturnValue({ epoch: EpochNumber(1), slot: SlotNumber(2), ts: 3n, nowMs: 3000n }); epochCache.getL1Constants.mockReturnValue(EmptyL1RollupConstants); epochCache.getSlotNow.mockReturnValue(SlotNumber(2)); @@ -525,6 +526,49 @@ describe('SequencerPublisher', () => { expect(l1TxUtils.simulate).toHaveBeenCalledTimes(1); }); + it('validates block headers when L1 head is already at the L2 slot boundary', async () => { + epochCache.getL1Constants.mockReturnValue({ + ...EmptyL1RollupConstants, + l1GenesisTime: 1000n, + slotDuration: 72, + ethereumSlotDuration: 12, + }); + const slotStartTimestamp = 1360n; + (l1TxUtils as any).simulate.mockImplementationOnce((_call: unknown, blockOverrides: { time?: bigint }) => { + if ((blockOverrides.time ?? 0n) <= slotStartTimestamp) { + throw new Error(`simulated block timestamp must be greater than parent timestamp`); + } + return Promise.resolve({ gasUsed: 1_000_000n, result: '0x' }); + }); + + await expect( + publisher.validateBlockHeader(CheckpointHeader.random({ slotNumber: SlotNumber(5) })), + ).resolves.toBeUndefined(); + }); + + it('simulates request bundles at the last L1 timestamp within the target L2 slot', async () => { + epochCache.getL1Constants.mockReturnValue({ + ...EmptyL1RollupConstants, + l1GenesisTime: 1000n, + slotDuration: 72, + ethereumSlotDuration: 12, + }); + publisher.addRequest({ + action: 'invalidate-by-invalid-attestation', + request: { to: mockRollupAddress, data: '0xdeadbeef' }, + lastValidL2Slot: SlotNumber(5), + checkSuccess: () => true, + }); + forwardSpy.mockResolvedValue({ receipt: proposeTxReceipt, stats: undefined, multicallData: '0x' }); + + await publisher.sendRequests(SlotNumber(5)); + + expect(l1TxUtils.simulate.mock.calls[0][1]).toEqual({ + time: 1420n, + gasLimit: MAX_L1_TX_LIMIT * 2n, + }); + }); + describe('bundleSimulate second-pass re-decode', () => { const addTwoRequests = () => { const currentL2Slot = publisher.getCurrentL2Slot(); diff --git a/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts b/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts index cf4146867c7e..d69c75dae81b 100644 --- a/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts +++ b/yarn-project/sequencer-client/src/publisher/sequencer-publisher.ts @@ -45,7 +45,11 @@ import { EmpireBaseAbi, ErrorsAbi, RollupAbi, SlashingProposerAbi } from '@aztec import { type ProposerSlashAction, encodeSlashConsensusVotes } from '@aztec/slasher'; import { CommitteeAttestationsAndSigners, type ValidateCheckpointResult } from '@aztec/stdlib/block'; import type { Checkpoint } from '@aztec/stdlib/checkpoint'; -import { getNextL1SlotTimestamp, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; +import { + getLastL1SlotTimestampForL2Slot, + getNextL1SlotTimestamp, + getTimestampForSlot, +} from '@aztec/stdlib/epoch-helpers'; import type { CheckpointHeader } from '@aztec/stdlib/rollup'; import type { L1PublishCheckpointStats } from '@aztec/stdlib/stats'; import { type TelemetryClient, type Tracer, getTelemetryClient, trackSpan } from '@aztec/telemetry-client'; @@ -727,7 +731,7 @@ export class SequencerPublisher { ] as const; const l1Constants = this.epochCache.getL1Constants(); - const ts = getTimestampForSlot(header.slotNumber, l1Constants); + const ts = getLastL1SlotTimestampForL2Slot(header.slotNumber, l1Constants); const stateOverrides = await buildSimulationOverridesStateOverride(this.rollupContract, simulationOverridesPlan); let balance = 0n; if (this.config.fishermanMode) { From 00bc25ed2e8a67f9b626c1ce0ab301fc082f8b20 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Fri, 29 May 2026 16:31:55 +0100 Subject: [PATCH 21/24] chore: smaller eth-devnet (#23704) Fix A-1122 --- .../values/resources-prod.yaml | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/spartan/terraform/deploy-eth-devnet/values/resources-prod.yaml b/spartan/terraform/deploy-eth-devnet/values/resources-prod.yaml index 5b4da2364cc5..21c507178721 100644 --- a/spartan/terraform/deploy-eth-devnet/values/resources-prod.yaml +++ b/spartan/terraform/deploy-eth-devnet/values/resources-prod.yaml @@ -10,8 +10,11 @@ ethereum: storageSize: "10Gi" resources: requests: - memory: "12Gi" - cpu: "2.5" + memory: "2Gi" + cpu: "1" + limits: + memory: "8Gi" + cpu: "4" beacon: service: @@ -23,8 +26,11 @@ ethereum: storageSize: "10Gi" resources: requests: - memory: "12Gi" - cpu: "2.5" + memory: "1Gi" + cpu: "500m" + limits: + memory: "4Gi" + cpu: "2" validator: nodeSelector: @@ -33,5 +39,8 @@ ethereum: storageSize: "10Gi" resources: requests: - memory: "12Gi" - cpu: "2.5" + memory: "512Mi" + cpu: "250m" + limits: + memory: "2Gi" + cpu: "1" From 7f1e9f2d1d31e322b3afe033b8d4398dd2e7e6e3 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Fri, 29 May 2026 16:49:26 +0100 Subject: [PATCH 22/24] chore: enable testnet autoscaling (#23705) . --- spartan/environments/testnet.env | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spartan/environments/testnet.env b/spartan/environments/testnet.env index c18eee78635f..eae96f2f61f7 100644 --- a/spartan/environments/testnet.env +++ b/spartan/environments/testnet.env @@ -87,8 +87,8 @@ PROVER_FAILED_PROOF_STORE=gs://aztec-develop/testnet/failed-proofs L1_TX_FAILED_STORE=gs://aztec-develop/testnet/failed-l1-txs PROVER_REPLICAS=4 PROVER_RESOURCE_PROFILE="prod" -PROVER_AGENT_KEDA_ENABLED=false -PROVER_AGENT_KEDA_PROMETHEUS_SERVER_ADDRESS= +PROVER_AGENT_KEDA_ENABLED=true +PROVER_AGENT_KEDA_PROMETHEUS_SERVER_ADDRESS=REPLACE_WITH_GCP_SECRET PROVER_AGENT_KEDA_MIN_REPLICAS=0 PROVER_AGENT_KEDA_MAX_REPLICAS=8 PROVER_AGENT_KEDA_SCALING_BANDS='[ @@ -97,7 +97,7 @@ PROVER_AGENT_KEDA_SCALING_BANDS='[ replicas = 4 }, { - queueSize = 50 + queueSize = 100 replicas = 8 } ]' From 1aa8f1031758e7fdef2dcf4a5e9e76bdd5dcc15e Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Fri, 29 May 2026 13:27:37 -0300 Subject: [PATCH 23/24] feat(api)!: redesign node log retrieval API around tag-based queries (#23625) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation The node exposed four log-retrieval methods with three filter shapes and two return shapes, while the private index was only tag-keyed — so a `(tag, narrow block range)` query loaded the entire per-tag history into memory. Pagination was a single global `page` counter shared across all tags, and the public path was split between a `LogFilter` method and a tag-based method. v5 is already a breaking release, so this collapses everything to a single, fast, tag-based surface with no back-compat. Fixes A-1111 Fixes A-1031 ## Approach Two methods, `getPrivateLogsByTags(query)` and `getPublicLogsByTags(query)`, replace the four old ones; `getContractClassLogs` and `getPublicLogs(LogFilter)` are gone. The archiver stores each log under a fixed-width composite hex-string key `[contractHex] - tagHex - blockHex8 - txIndexHex8 - logIndexHex8` in an LMDB map, so every supported filter (tag, block range, txHash, per-tag `afterLog` cursor, `referenceBlock` reorg cap) reduces to a single ordered range scan. Note hashes and nullifiers are never copied into the log index — they're fetched on demand from the block store via a partial deserializer that reads only the relevant prefix of the stored `IndexedTxEffect`. `ARCHIVER_DB_VERSION` bumps 6 → 7, so the archiver self-wipes and re-syncs from L1 on first start. ## API Two node methods replace the previous four. Each returns one inner array per element of `query.tags`, in input order; an empty inner array means that tag matched nothing. ```ts getPrivateLogsByTags(query: PrivateLogsQuery): Promise; getPublicLogsByTags(query: PublicLogsQuery): Promise; ``` **Input** ```ts // Filters shared by both queries. type LogsQueryBase = { fromBlock?: BlockNumber; // inclusive lower bound toBlock?: BlockNumber; // exclusive upper bound txHash?: TxHash; // restrict to one tx; mutually exclusive with fromBlock/toBlock referenceBlock?: BlockHash; // reorg anchor: throws if that block is no longer present includeEffects?: boolean; // also attach each log's tx noteHashes + nullifiers limitPerTag?: number; // page size, 1..MAX_LOGS_PER_TAG (default & max = 20) }; // A tag to query, optionally resuming strictly after a previously-seen log. // The bare `T` form starts from the beginning. type TagQuery = T | { tag: T; afterLog?: LogCursor }; type PrivateLogsQuery = LogsQueryBase & { tags: TagQuery[]; // 1..MAX_RPC_LEN (100) entries }; type PublicLogsQuery = LogsQueryBase & { contractAddress: AztecAddress; // required for public queries tags: TagQuery[]; // 1..MAX_RPC_LEN (100) entries }; ``` **Output** ```ts type LogResult = { logData: Fr[]; // log fields; the tag is logData[0] blockNumber: BlockNumber; blockHash: BlockHash; blockTimestamp: UInt64; txHash: TxHash; txIndexWithinBlock: number; // 0-based index of the tx within its block logIndexWithinTx: number; // 0-based index of the log within its tx } & (Opts extends { includeEffects: true } ? { noteHashes: Fr[]; nullifiers: Fr[] } // present only when includeEffects: true : {}); // Opaque per-tag pagination cursor. // String form: `--`. class LogCursor { blockNumber: BlockNumber; txIndexWithinBlock: number; logIndexWithinTx: number; static fromLog(log: LogResult): LogCursor; } ``` Pagination is per-tag: feed a tag's last `LogResult` back as the next query's `afterLog` (`{ tag, afterLog: LogCursor.fromLog(last) }`). A tag is exhausted once it returns fewer than `limitPerTag` results. The stdlib helpers `queryAllPrivateLogsByTags` / `queryAllPublicLogsByTags` drive this loop and return the fully-drained results. ## Changes - **stdlib**: new `LogResult`, `LogCursor`, `PrivateLogsQuery` / `PublicLogsQuery` types with zod schemas; `txHash` ⊕ `fromBlock`/`toBlock` enforced via `.refine` (but `txHash` + `afterLog` is allowed, to paginate within a tx's logs). `L2LogsSource` / `AztecNode` / `Archiver` interfaces and schemas reduced to the two new methods. Deleted `LogFilter`, `LogId`, `TxScopedL2Log`, `ExtendedPublicLog`, `ExtendedContractClassLog`, `GetPublicLogsResponse`, `GetContractClassLogsResponse`, and the dead `Tx.getPublicLogs(logsSource)`. - **archiver**: full `LogStore` rewrite — two hex-string-keyed `AztecAsyncMap` primary maps (keys are fixed-width zero-padded lowercase hex, so `ordered-binary`'s string ordering matches the canonical `(contract, tag, block, txIndex, logIndex)` tuple and every filter is a single ordered range scan) plus two `blockNumber → string[]` secondary indices driving `deleteLogs` (replaces the buggy per-block tag-union list). All reads, including the `referenceBlock` existence check, run inside one `db.transactionAsync` across `BlockStore` + `LogStore`; a `referenceBlock` equal to the (synthetic, unindexed) genesis block hash resolves to the genesis block number rather than throwing. New `BlockStore.getNoteHashesAndNullifiers(txHashes)` is a batched partial deserializer for `includeEffects`, and `getTxLocation` reads only the 40-byte header instead of the full `TxEffect`. Contract-class-log storage removed entirely. `ARCHIVER_DB_VERSION` 6 → 7. `OutOfOrderLogInsertionError` and the `ARCHIVER_MAX_LOGS` env var dropped. - **aztec-node**: four RPC handlers collapsed to two thin forwarders; `referenceBlock` resolution moved into the store so it shares the transaction. - **pxe**: `getAllPages` rewritten from a global `page` counter to per-tag `afterLog` cursors — each round re-queries only tags that returned a full page, and tags drop out as soon as they return a short page. `fromBlock` / `toBlock` are pushed down into the node, eliminating the in-memory `#extractLogs` range filter. - **aztec.js**: `getPublicEvents` migrated to the new query shape; `PublicEventFilter.contractAddress` is now required; `EventFilterBase.afterLog: LogId → LogCursor`. - **cli**: `get-logs` requires `--contract-address` and `--tag`; `--after-log` parses a `LogCursor` string `--`. - **end-to-end**: `e2e_ordering` rewritten to read `getBlock().body.txEffects[*].publicLogs` directly (the new API drops the tag-less, contract-less query shape). - **docs**: migration notes for client consumers; operator changelog covering the DB version bump and one-time resync. --- .../docs/aztec-js/how_to_read_data.md | 6 +- .../framework-description/events_and_logs.md | 13 +- .../docs/resources/migration_notes.md | 78 + docs/examples/ts/aztecjs_advanced/index.ts | 43 +- .../archiver/src/archiver-misc.test.ts | 4 +- .../archiver/src/archiver-store.test.ts | 4 +- .../archiver/src/archiver-sync.test.ts | 20 +- yarn-project/archiver/src/config.ts | 5 - yarn-project/archiver/src/errors.ts | 16 - yarn-project/archiver/src/factory.ts | 7 +- .../archiver/src/modules/data_source_base.ts | 28 +- .../src/modules/data_store_updater.test.ts | 51 +- .../archiver/src/store/block_store.test.ts | 69 + .../archiver/src/store/block_store.ts | 35 +- .../archiver/src/store/data_stores.ts | 20 +- .../archiver/src/store/log_store.test.ts | 1557 ++++++----------- yarn-project/archiver/src/store/log_store.ts | 946 ++++------ .../src/store/log_store_codec.test.ts | 250 +++ .../archiver/src/store/log_store_codec.ts | 132 ++ .../aztec-node/src/aztec-node/server.ts | 59 +- yarn-project/aztec.js/src/api/events.ts | 67 +- yarn-project/aztec.js/src/api/log.ts | 2 +- yarn-project/aztec.js/src/wallet/wallet.ts | 43 +- .../cli/src/cmds/aztec_node/get_logs.ts | 108 +- yarn-project/cli/src/cmds/aztec_node/index.ts | 21 +- yarn-project/cli/src/utils/commands.ts | 31 +- .../src/bench/node_rpc_perf.test.ts | 22 +- .../contract_class_registration.test.ts | 10 +- .../e2e_deploy_contract/deploy_method.test.ts | 4 +- .../end-to-end/src/e2e_event_logs.test.ts | 2 + .../src/e2e_large_public_event.test.ts | 1 + .../e2e_nested_contract/manual_public.test.ts | 11 +- .../end-to-end/src/e2e_orderbook.test.ts | 4 +- .../end-to-end/src/e2e_ordering.test.ts | 18 +- .../end-to-end/src/e2e_synching.test.ts | 6 - yarn-project/foundation/src/config/env_var.ts | 1 - .../node-lib/src/actions/snapshot-sync.ts | 9 +- .../src/actions/rerun-epoch-proving-job.ts | 3 +- .../oracle/private_execution.test.ts | 3 +- .../oracle/utility_execution.test.ts | 2 +- yarn-project/pxe/src/events/event_service.ts | 4 +- yarn-project/pxe/src/logs/log_service.test.ts | 91 +- yarn-project/pxe/src/logs/log_service.ts | 135 +- yarn-project/pxe/src/notes/note_service.ts | 4 +- yarn-project/pxe/src/pxe.test.ts | 3 +- .../src/tagging/get_all_logs_by_tags.test.ts | 146 +- .../pxe/src/tagging/get_all_logs_by_tags.ts | 126 +- .../sync_tagged_private_logs.test.ts | 74 +- .../sync_tagged_private_logs.ts | 24 +- .../utils/find_highest_indexes.test.ts | 8 +- .../utils/find_highest_indexes.ts | 4 +- .../sync_sender_tagging_indexes.test.ts | 37 +- ...load_and_store_new_tagging_indexes.test.ts | 44 +- .../load_and_store_new_tagging_indexes.ts | 2 +- .../stdlib/src/interfaces/api_limit.ts | 2 +- .../stdlib/src/interfaces/archiver.test.ts | 92 +- .../stdlib/src/interfaces/archiver.ts | 28 +- .../stdlib/src/interfaces/aztec-node.test.ts | 88 +- .../stdlib/src/interfaces/aztec-node.ts | 95 +- yarn-project/stdlib/src/interfaces/client.ts | 1 - .../src/interfaces/get_logs_response.test.ts | 11 - .../src/interfaces/get_logs_response.ts | 35 - .../stdlib/src/interfaces/l2_logs_source.ts | 55 +- .../src/logs/extended_contract_class_log.ts | 87 - .../stdlib/src/logs/extended_public_log.ts | 94 - yarn-project/stdlib/src/logs/index.ts | 9 +- .../stdlib/src/logs/log_cursor.test.ts | 60 + yarn-project/stdlib/src/logs/log_cursor.ts | 110 ++ yarn-project/stdlib/src/logs/log_filter.ts | 36 - yarn-project/stdlib/src/logs/log_id.test.ts | 28 - yarn-project/stdlib/src/logs/log_id.ts | 124 -- .../stdlib/src/logs/log_result.test.ts | 31 + yarn-project/stdlib/src/logs/log_result.ts | 104 ++ .../stdlib/src/logs/logs_query.test.ts | 47 + yarn-project/stdlib/src/logs/logs_query.ts | 138 ++ .../stdlib/src/logs/query_all_logs_by_tags.ts | 98 ++ .../stdlib/src/logs/tx_scoped_l2_log.test.ts | 17 - .../stdlib/src/logs/tx_scoped_l2_log.ts | 114 -- yarn-project/stdlib/src/tests/factories.ts | 87 +- yarn-project/stdlib/src/tx/tx.ts | 11 - .../txe/src/state_machine/archiver.ts | 2 +- .../src/validator.integration.test.ts | 15 +- 82 files changed, 2912 insertions(+), 3120 deletions(-) create mode 100644 yarn-project/archiver/src/store/log_store_codec.test.ts create mode 100644 yarn-project/archiver/src/store/log_store_codec.ts delete mode 100644 yarn-project/stdlib/src/interfaces/get_logs_response.test.ts delete mode 100644 yarn-project/stdlib/src/interfaces/get_logs_response.ts delete mode 100644 yarn-project/stdlib/src/logs/extended_contract_class_log.ts delete mode 100644 yarn-project/stdlib/src/logs/extended_public_log.ts create mode 100644 yarn-project/stdlib/src/logs/log_cursor.test.ts create mode 100644 yarn-project/stdlib/src/logs/log_cursor.ts delete mode 100644 yarn-project/stdlib/src/logs/log_filter.ts delete mode 100644 yarn-project/stdlib/src/logs/log_id.test.ts delete mode 100644 yarn-project/stdlib/src/logs/log_id.ts create mode 100644 yarn-project/stdlib/src/logs/log_result.test.ts create mode 100644 yarn-project/stdlib/src/logs/log_result.ts create mode 100644 yarn-project/stdlib/src/logs/logs_query.test.ts create mode 100644 yarn-project/stdlib/src/logs/logs_query.ts create mode 100644 yarn-project/stdlib/src/logs/query_all_logs_by_tags.ts delete mode 100644 yarn-project/stdlib/src/logs/tx_scoped_l2_log.test.ts delete mode 100644 yarn-project/stdlib/src/logs/tx_scoped_l2_log.ts diff --git a/docs/docs-developers/docs/aztec-js/how_to_read_data.md b/docs/docs-developers/docs/aztec-js/how_to_read_data.md index 2cc3b2ecca88..a9e03a51b79f 100644 --- a/docs/docs-developers/docs/aztec-js/how_to_read_data.md +++ b/docs/docs-developers/docs/aztec-js/how_to_read_data.md @@ -65,18 +65,18 @@ Contracts emit data in two forms you can read: | ------------------ | --------------------------- | -------------------------------------------------- | | **What** | Raw field arrays (untyped) | Decoded domain objects with type info | | **Storage** | Archiver (node-level) | PXE (client-level) for private events | -| **API** | `aztecNode.getPublicLogs()` | `wallet.getPrivateEvents()` or `getPublicEvents()` | +| **API** | `aztecNode.getBlock()` tx effects | `wallet.getPrivateEvents()` or `getPublicEvents()` | | **Type awareness** | None - raw `Fr[]` data | Requires ABI metadata to decode | **Logs** are the low-level transport layer, while **events** are the semantic application layer decoded using ABI metadata from your contract. ## Reading raw public logs -Use `aztecNode.getPublicLogs()` to retrieve raw log data: +Raw public logs are carried on each block's transaction effects. Fetch a block with `includeTransactions: true` and read `body.txEffects[*].publicLogs`: #include_code read_public_logs /docs/examples/ts/aztecjs_advanced/index.ts typescript -You can also filter by transaction hash or block range: +You can scope this to a single transaction (by locating its block and matching its tx hash) or to a block range (by reading each block's tx effects): #include_code read_logs_by_filter /docs/examples/ts/aztecjs_advanced/index.ts typescript diff --git a/docs/docs-developers/docs/aztec-nr/framework-description/events_and_logs.md b/docs/docs-developers/docs/aztec-nr/framework-description/events_and_logs.md index 1f7c9af2d67d..4d8a25ad57b8 100644 --- a/docs/docs-developers/docs/aztec-nr/framework-description/events_and_logs.md +++ b/docs/docs-developers/docs/aztec-nr/framework-description/events_and_logs.md @@ -95,15 +95,14 @@ self.context.emit_public_log([1, 2, 3]); ## Query public logs -Query public logs from offchain applications using the Aztec node: +Query public logs from offchain applications using the Aztec node. Raw public logs are +attached to each block's transaction effects — fetch a block with `includeTransactions: true` +and read `body.txEffects[*].publicLogs`: ```typescript -const fromBlock = await node.getBlockNumber(); -const logFilter = { - fromBlock, - toBlock: fromBlock + 1, -}; -const publicLogs = (await node.getPublicLogs(logFilter)).logs; +const blockNumber = await node.getBlockNumber(); +const block = await node.getBlock(blockNumber, { includeTransactions: true }); +const publicLogs = block?.body.txEffects.flatMap(tx => tx.publicLogs) ?? []; ``` ## Cost considerations diff --git a/docs/docs-developers/docs/resources/migration_notes.md b/docs/docs-developers/docs/resources/migration_notes.md index 5fba9d75ef01..24a104714ca9 100644 --- a/docs/docs-developers/docs/resources/migration_notes.md +++ b/docs/docs-developers/docs/resources/migration_notes.md @@ -93,6 +93,84 @@ The `privately_check_timestamp`, `privately_check_block_number`, and related cal + use aztec::public_checks::privately_check_timestamp; ``` +### [Aztec Node / Aztec.js / CLI] Log retrieval API consolidated to two tag-based methods + +The four log-retrieval methods on `AztecNode` have been collapsed into two. `getContractClassLogs` and the `LogFilter`-shaped `getPublicLogs` are removed entirely; the surviving methods are `getPrivateLogsByTags(query)` and `getPublicLogsByTags(query)`, both taking a single query object and returning `LogResult[][]` (one inner array per requested tag, in input order). + +**Removed methods on `AztecNode`:** + +| Removed | Replacement | +| --------------------------------------------------------------------------- | -------------------------------------------------------- | +| `getPublicLogs(filter: LogFilter)` | `getPublicLogsByTags({ contractAddress, tags, ... })` | +| `getContractClassLogs(filter: LogFilter)` | none — RPC removed; no production consumer existed | +| `getPrivateLogsByTags(tags, page?, referenceBlock?)` | `getPrivateLogsByTags({ tags, ... })` | +| `getPublicLogsByTagsFromContract(contract, tags, page?, referenceBlock?)` | `getPublicLogsByTags({ contractAddress, tags, ... })` | + +**New query and response shapes:** + +```ts +// Query +type TagQuery = T | { tag: T; afterLog?: LogCursor }; + +type LogsQueryBase = { + fromBlock?: BlockNumber; // inclusive + toBlock?: BlockNumber; // exclusive + txHash?: TxHash; // mutually exclusive with fromBlock/toBlock + referenceBlock?: BlockHash; // reorg-safety anchor; throws if missing + includeEffects?: boolean; // attach noteHashes + all nullifiers +}; + +type PrivateLogsQuery = LogsQueryBase & { tags: TagQuery[] }; +type PublicLogsQuery = LogsQueryBase & { contractAddress: AztecAddress; tags: TagQuery[] }; + +// Response (per log) +type LogResult = { + logData: Fr[]; + blockNumber: BlockNumber; + blockHash: BlockHash; + blockTimestamp: UInt64; + txHash: TxHash; + logIndexWithinTx: number; + noteHashes?: Fr[]; // present only when includeEffects is set + nullifiers?: Fr[]; // all nullifiers of the tx, not just the first +}; +``` + +**Public queries now require a contract address.** Tag-only / contract-less public queries are no longer supported (the public log index is keyed on `(contract, tag)`). + +**Per-tag `afterLog` cursors replace the global `page` argument.** Each tag advances independently — pass `{ tag, afterLog: LogCursor.fromLog(lastLog) }` to resume that tag, and omit it for tags that are already exhausted. + +**Aztec.js wallet — `PublicEventFilter.contractAddress` is now required, and `afterLog` is a `LogCursor`:** + +```diff +- type PublicEventFilter = EventFilterBase & { contractAddress?: AztecAddress }; ++ type PublicEventFilter = EventFilterBase & { contractAddress: AztecAddress }; + + type EventFilterBase = { + txHash?: TxHash; + fromBlock?: BlockNumber; + toBlock?: BlockNumber; +- afterLog?: LogId; ++ afterLog?: LogCursor; + }; +``` + +`LogId`, `LogFilter`, `TxScopedL2Log`, `ExtendedPublicLog`, `ExtendedContractClassLog`, `GetPublicLogsResponse`, and `GetContractClassLogsResponse` are no longer exported from `@aztec/aztec.js`. Build cursors with `LogCursor.fromLog(log)` and decode public-event payloads from `result.logData.slice(1)` (the tag is field 0). + +**CLI — `aztec get-logs` now requires `--contract-address` and `--tag`:** + +```diff +- aztec get-logs [--tx-hash ] [--from-block ] [--to-block ] [--after-log ] ++ aztec get-logs --contract-address
--tag \ ++ [--tx-hash ] [--from-block ] [--to-block ] [--after-log ] +``` + +`--after-log` now takes a `LogCursor` of the form `--` (formerly a `LogId`). + +**Mutual exclusion**: setting both `txHash` and `fromBlock`/`toBlock` is rejected (a `txHash` already pins a block). `txHash` + `afterLog` is allowed and paginates within the tx's logs for a tag. + +**Impact**: Any consumer of `getPublicLogs(LogFilter)`, `getContractClassLogs`, the old tag-based methods, `PublicEventFilter` without a `contractAddress`, or `EventFilterBase.afterLog: LogId` must be updated. The CLI rejects calls missing `--contract-address` or `--tag`. + ### [Aztec.js] `AccountManager.create` takes an options bag `AccountManager.create` no longer takes `salt` as a positional argument. The trailing `salt?: Salt` parameter has been folded into a new `AccountManagerCreateOptions` bag alongside `immutablesHash` and `deployer`: diff --git a/docs/examples/ts/aztecjs_advanced/index.ts b/docs/examples/ts/aztecjs_advanced/index.ts index 90f60cc5c675..fbcdcdb029db 100644 --- a/docs/examples/ts/aztecjs_advanced/index.ts +++ b/docs/examples/ts/aztecjs_advanced/index.ts @@ -306,6 +306,7 @@ async function pollForTransferEvents() { node, TokenContract.events.Transfer, { + contractAddress: token.address, fromBlock: BlockNumber(lastProcessedBlock + 1), toBlock: BlockNumber(currentBlock + 1), // toBlock is exclusive }, @@ -361,12 +362,14 @@ console.log("DA gas limit:", metaResult.estimatedGas.gasLimits.daGas); // docs:end:simulate_with_metadata // docs:start:read_public_logs -const publicLogs = await node.getPublicLogs({ - fromBlock: 1, - toBlock: (await node.getBlockNumber()) + 1, +// Raw public logs are carried on each block's transaction effects. +const latestBlockNumber = await node.getBlockNumber(); +const block = await node.getBlock(latestBlockNumber, { + includeTransactions: true, }); -if (publicLogs.logs.length > 0) { - const rawFields = publicLogs.logs[0].log.getEmittedFields(); // Fr[] +const publicLogs = block?.body.txEffects.flatMap((tx) => tx.publicLogs) ?? []; +if (publicLogs.length > 0) { + const rawFields = publicLogs[0].getEmittedFields(); // Fr[] console.log("Raw log fields:", rawFields.length); } // docs:end:read_public_logs @@ -434,14 +437,30 @@ const { receipt: gsReceipt } = await token.methods // docs:end:send_with_gas_settings // docs:start:read_logs_by_filter -// Get logs for a specific transaction -const txLogs = await node.getPublicLogs({ txHash: gsReceipt.txHash }); - -// Get logs for a block range -const rangeLogs = await node.getPublicLogs({ - fromBlock: 1, - toBlock: (await node.getBlockNumber()) + 1, +// Get raw public logs for a specific transaction by locating its block and tx effect. +const txReceiptForLogs = await node.getTxReceipt(gsReceipt.txHash); +const txBlock = await node.getBlock(txReceiptForLogs.blockNumber!, { + includeTransactions: true, }); +const txLogs = + txBlock?.body.txEffects + .filter((tx) => tx.txHash.equals(gsReceipt.txHash)) + .flatMap((tx) => tx.publicLogs) ?? []; + +// Get raw public logs for a block range by reading each block's tx effects. +const tipBlockNumber = await node.getBlockNumber(); +const rangeLogs = ( + await Promise.all( + Array.from({ length: tipBlockNumber }, (_, i) => BlockNumber(i + 1)).map( + async (blockNumber) => { + const rangeBlock = await node.getBlock(blockNumber, { + includeTransactions: true, + }); + return rangeBlock?.body.txEffects.flatMap((tx) => tx.publicLogs) ?? []; + }, + ), + ) +).flat(); // docs:end:read_logs_by_filter // docs:start:auto_gas_estimation diff --git a/yarn-project/archiver/src/archiver-misc.test.ts b/yarn-project/archiver/src/archiver-misc.test.ts index 94cf202dc722..264b00b40db5 100644 --- a/yarn-project/archiver/src/archiver-misc.test.ts +++ b/yarn-project/archiver/src/archiver-misc.test.ts @@ -10,7 +10,7 @@ import { Fr } from '@aztec/foundation/curves/bn254'; import { EthAddress } from '@aztec/foundation/eth-address'; import { DateProvider } from '@aztec/foundation/timer'; import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; -import type { L2Tips } from '@aztec/stdlib/block'; +import { GENESIS_BLOCK_HEADER_HASH, type L2Tips } from '@aztec/stdlib/block'; import type { CheckpointData } from '@aztec/stdlib/checkpoint'; import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers'; import { CheckpointHeader } from '@aztec/stdlib/rollup'; @@ -62,7 +62,7 @@ describe('Archiver misc', () => { const tracer = getTelemetryClient().getTracer(''); const instrumentation = mock({ isEnabled: () => true, tracer }); - const archiverStore = createArchiverDataStores(await openTmpStore('archiver_misc_test'), { logsMaxPageSize: 1000 }); + const archiverStore = createArchiverDataStores(await openTmpStore('archiver_misc_test'), GENESIS_BLOCK_HEADER_HASH); const events = new EventEmitter() as ArchiverEmitter; const initialHeader = BlockHeader.empty(); const initialBlockHash = await initialHeader.hash(); diff --git a/yarn-project/archiver/src/archiver-store.test.ts b/yarn-project/archiver/src/archiver-store.test.ts index bab140c6f758..49addc842576 100644 --- a/yarn-project/archiver/src/archiver-store.test.ts +++ b/yarn-project/archiver/src/archiver-store.test.ts @@ -14,7 +14,7 @@ import { Fr } from '@aztec/foundation/curves/bn254'; import { EthAddress } from '@aztec/foundation/eth-address'; import { DateProvider } from '@aztec/foundation/timer'; import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; -import { L2Block } from '@aztec/stdlib/block'; +import { GENESIS_BLOCK_HEADER_HASH, L2Block } from '@aztec/stdlib/block'; import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers'; import { CheckpointHeader } from '@aztec/stdlib/rollup'; import { makeStateReference } from '@aztec/stdlib/testing'; @@ -72,7 +72,7 @@ describe('Archiver Store', () => { const tracer = getTelemetryClient().getTracer(''); instrumentation = mock({ isEnabled: () => true, tracer }); - archiverStore = createArchiverDataStores(await openTmpStore('archiver_test'), { logsMaxPageSize: 1000 }); + archiverStore = createArchiverDataStores(await openTmpStore('archiver_test'), GENESIS_BLOCK_HEADER_HASH); l1Constants = { l1GenesisTime: BigInt(now), diff --git a/yarn-project/archiver/src/archiver-sync.test.ts b/yarn-project/archiver/src/archiver-sync.test.ts index 468297212cf4..f3822fcd45c6 100644 --- a/yarn-project/archiver/src/archiver-sync.test.ts +++ b/yarn-project/archiver/src/archiver-sync.test.ts @@ -15,7 +15,7 @@ import { type Logger, createLogger } from '@aztec/foundation/log'; import { retryFastUntil } from '@aztec/foundation/retry'; import { TestDateProvider } from '@aztec/foundation/timer'; import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; -import { L2BlockSourceEvents } from '@aztec/stdlib/block'; +import { GENESIS_BLOCK_HEADER_HASH, L2BlockSourceEvents } from '@aztec/stdlib/block'; import type { ProposedCheckpointInput } from '@aztec/stdlib/checkpoint'; import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers'; import { computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging'; @@ -102,7 +102,7 @@ describe('Archiver Sync', () => { instrumentation = mock({ isEnabled: () => true, tracer }); // Create archiver store - archiverStore = createArchiverDataStores(await openTmpStore('archiver_sync_test'), { logsMaxPageSize: 1000 }); + archiverStore = createArchiverDataStores(await openTmpStore('archiver_sync_test'), GENESIS_BLOCK_HEADER_HASH); const contractAddresses = { rollupAddress, @@ -229,24 +229,15 @@ describe('Archiver Sync', () => { expect(await archiver.getL1ToL2Messages(CheckpointNumber(3))).toEqual(msgs3); await expect(archiver.getL1ToL2Messages(CheckpointNumber(4))).rejects.toThrow(L1ToL2MessagesNotReadyError); - // Verify logs for each block in the checkpoints + // Verify private logs are surfaced through the block body. for (const checkpoint of [cp1, cp2, cp3]) { for (const block of checkpoint.blocks) { const blockNumber = block.number; - const expectedTotalNumLogs = (name: 'private' | 'public' | 'contractClass') => + const expectedTotalNumLogs = (name: 'private') => sum(block.body.txEffects.map(txEffect => txEffect[`${name}Logs`].length)); const privateLogs = (await archiver.getBlock({ number: blockNumber }))!.getPrivateLogs(); expect(privateLogs.length).toBe(expectedTotalNumLogs('private')); - - const publicLogs = (await archiver.getPublicLogs({ fromBlock: blockNumber, toBlock: blockNumber + 1 })).logs; - expect(publicLogs.length).toBe(expectedTotalNumLogs('public')); - - const contractClassLogs = await archiver.getContractClassLogs({ - fromBlock: blockNumber, - toBlock: blockNumber + 1, - }); - expect(contractClassLogs.logs.length).toBe(expectedTotalNumLogs('contractClass')); } } @@ -980,9 +971,6 @@ describe('Archiver Sync', () => { const txHash = cp2.blocks[0].body.txEffects[0].txHash; expect(await archiver.getTxEffect(txHash)).toBeUndefined(); expect(await archiver.getCheckpoints({ from: CheckpointNumber(2), limit: 1 })).toEqual([]); - - expect((await archiver.getPublicLogs({ fromBlock: 2, toBlock: 3 })).logs).toEqual([]); - expect((await archiver.getContractClassLogs({ fromBlock: 2, toBlock: 3 })).logs).toEqual([]); }, 10_000); it('handles updated messages due to L1 reorg', async () => { diff --git a/yarn-project/archiver/src/config.ts b/yarn-project/archiver/src/config.ts index a10fcb40ede1..d99d6c90b5e2 100644 --- a/yarn-project/archiver/src/config.ts +++ b/yarn-project/archiver/src/config.ts @@ -43,11 +43,6 @@ export const archiverConfigMappings: ConfigMappingsType = { description: 'The number of L2 blocks the archiver will attempt to download at a time.', ...numberConfigHelper(100), }, - maxLogs: { - env: 'ARCHIVER_MAX_LOGS', - description: 'The max number of logs that can be obtained in 1 "getPublicLogs" call.', - ...numberConfigHelper(1_000), - }, archiverStoreMapSizeKb: { env: 'ARCHIVER_STORE_MAP_SIZE_KB', ...optionalNumberConfigHelper(), diff --git a/yarn-project/archiver/src/errors.ts b/yarn-project/archiver/src/errors.ts index 6de0234d9585..db3be133c0b0 100644 --- a/yarn-project/archiver/src/errors.ts +++ b/yarn-project/archiver/src/errors.ts @@ -102,22 +102,6 @@ export class BlockAlreadyCheckpointedError extends Error { } } -/** Thrown when logs are added for a tag whose last stored log has a higher block number than the new log. */ -export class OutOfOrderLogInsertionError extends Error { - constructor( - public readonly logType: 'private' | 'public', - public readonly tag: string, - public readonly lastBlockNumber: number, - public readonly newBlockNumber: number, - ) { - super( - `Out-of-order ${logType} log insertion for tag ${tag}: ` + - `last existing log is from block ${lastBlockNumber} but new log is from block ${newBlockNumber}`, - ); - this.name = 'OutOfOrderLogInsertionError'; - } -} - /** Thrown when L1 to L2 messages are requested for a checkpoint whose message tree hasn't been sealed yet. */ export class L1ToL2MessagesNotReadyError extends Error { constructor( diff --git a/yarn-project/archiver/src/factory.ts b/yarn-project/archiver/src/factory.ts index b75f356fcbef..1b1e5ec83b7f 100644 --- a/yarn-project/archiver/src/factory.ts +++ b/yarn-project/archiver/src/factory.ts @@ -34,14 +34,15 @@ export const ARCHIVER_STORE_NAME = 'archiver'; /** Creates an archiver store. */ export async function createArchiverStore( - userConfig: Pick & DataStoreConfig, + userConfig: Pick & DataStoreConfig, + genesisBlockHash: BlockHash, ): Promise { const config = { ...userConfig, dataStoreMapSizeKb: userConfig.archiverStoreMapSizeKb ?? userConfig.dataStoreMapSizeKb, }; const store = await createStore(ARCHIVER_STORE_NAME, ARCHIVER_DB_VERSION, config); - return createArchiverDataStores(store, { logsMaxPageSize: config.maxLogs }); + return createArchiverDataStores(store, genesisBlockHash); } /** @@ -61,7 +62,7 @@ export async function createArchiver( initialHeader: BlockHeader, initialBlockHash: BlockHash, ): Promise { - const archiverStore = await createArchiverStore(config); + const archiverStore = await createArchiverStore(config, initialBlockHash); await registerProtocolContracts(archiverStore); // Create Ethereum clients diff --git a/yarn-project/archiver/src/modules/data_source_base.ts b/yarn-project/archiver/src/modules/data_source_base.ts index f4fbdff572c2..a0cc8f61f1a1 100644 --- a/yarn-project/archiver/src/modules/data_source_base.ts +++ b/yarn-project/archiver/src/modules/data_source_base.ts @@ -38,9 +38,8 @@ import { getProofSubmissionDeadlineEpoch, getSlotRangeForEpoch, } from '@aztec/stdlib/epoch-helpers'; -import type { GetContractClassLogsResponse, GetPublicLogsResponse } from '@aztec/stdlib/interfaces/client'; import type { L2LogsSource } from '@aztec/stdlib/interfaces/server'; -import type { LogFilter, SiloedTag, Tag, TxScopedL2Log } from '@aztec/stdlib/logs'; +import type { LogResult, PrivateLogsQuery, PublicLogsQuery } from '@aztec/stdlib/logs'; import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging'; import { AppendOnlyTreeSnapshot } from '@aztec/stdlib/trees'; import type { BlockHeader, IndexedTxEffect, TxHash, TxReceipt } from '@aztec/stdlib/tx'; @@ -299,29 +298,12 @@ export abstract class ArchiverDataSourceBase return (await this.stores.blocks.getPendingChainValidationStatus()) ?? { valid: true }; } - public getPrivateLogsByTags( - tags: SiloedTag[], - page?: number, - upToBlockNumber?: BlockNumber, - ): Promise { - return this.stores.logs.getPrivateLogsByTags(tags, page, upToBlockNumber); + public getPrivateLogsByTags(query: PrivateLogsQuery): Promise { + return this.stores.logs.getPrivateLogsByTags(query); } - public getPublicLogsByTagsFromContract( - contractAddress: AztecAddress, - tags: Tag[], - page?: number, - upToBlockNumber?: BlockNumber, - ): Promise { - return this.stores.logs.getPublicLogsByTagsFromContract(contractAddress, tags, page, upToBlockNumber); - } - - public getPublicLogs(filter: LogFilter): Promise { - return this.stores.logs.getPublicLogs(filter); - } - - public getContractClassLogs(filter: LogFilter): Promise { - return this.stores.logs.getContractClassLogs(filter); + public getPublicLogsByTags(query: PublicLogsQuery): Promise { + return this.stores.logs.getPublicLogsByTags(query); } public getContractClass(id: Fr): Promise { diff --git a/yarn-project/archiver/src/modules/data_store_updater.test.ts b/yarn-project/archiver/src/modules/data_store_updater.test.ts index fa3a8ec9be60..6163a406e06c 100644 --- a/yarn-project/archiver/src/modules/data_store_updater.test.ts +++ b/yarn-project/archiver/src/modules/data_store_updater.test.ts @@ -4,7 +4,7 @@ import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; import { ContractClassPublishedEvent } from '@aztec/protocol-contracts/class-registry'; import { ContractInstancePublishedEvent } from '@aztec/protocol-contracts/instance-registry'; import { AztecAddress } from '@aztec/stdlib/aztec-address'; -import { L2Block } from '@aztec/stdlib/block'; +import { GENESIS_BLOCK_HEADER_HASH, L2Block } from '@aztec/stdlib/block'; import { ContractClassLog, PrivateLog } from '@aztec/stdlib/logs'; import { CheckpointHeader } from '@aztec/stdlib/rollup'; import '@aztec/stdlib/testing/jest'; @@ -46,7 +46,7 @@ describe('ArchiverDataStoreUpdater', () => { let instanceAddress: AztecAddress; beforeEach(async () => { - store = createArchiverDataStores(await openTmpStore('data_store_updater_test'), { logsMaxPageSize: 1000 }); + store = createArchiverDataStores(await openTmpStore('data_store_updater_test'), GENESIS_BLOCK_HEADER_HASH); updater = new ArchiverDataStoreUpdater(store); // Create contract class log from sample fixture data @@ -279,8 +279,18 @@ describe('ArchiverDataStoreUpdater', () => { }); describe('logs handling', () => { + /** + * Counts how many indexed public logs at `block.number` come from `block`'s txs. Compares the indexed + * logs' `txHash` against the block's tx-effect hashes, so an orphan write from a different block at + * the same number (e.g. after a slot conflict swap) doesn't get counted. + */ + async function countIndexedPublicLogs(block: L2Block): Promise { + const expectedTxHashes = new Set(block.body.txEffects.map(tx => tx.txHash.toString())); + const indexed = await store.logs.getPublicLogsForBlock(block.number); + return indexed.filter(log => expectedTxHashes.has(log.txHash.toString())).length; + } + it('does not duplicate logs when checkpoint contains same block as provisional', async () => { - // Add provisional block with some logs const block = await L2Block.random(BlockNumber(1), { checkpointNumber: CheckpointNumber(1), indexWithinCheckpoint: IndexWithinCheckpoint(0), @@ -294,24 +304,23 @@ describe('ArchiverDataStoreUpdater', () => { await updater.addCheckpoints([publishedCheckpoint]); - // Verify logs are NOT duplicated - const publicLogs = await store.logs.getPublicLogs({ fromBlock: block.number, toBlock: block.number + 1 }); - expect(publicLogs.logs.length).toBe(block.body.txEffects.flatMap(tx => tx.publicLogs).length); - expect(publicLogs.logs.length).toBeGreaterThan(0); + const expected = block.body.txEffects.flatMap(tx => tx.publicLogs).length; + const indexed = await countIndexedPublicLogs(block); + expect(indexed).toBe(expected); + expect(indexed).toBeGreaterThan(0); }); it('replaces logs when checkpoint conflicts with provisional block', async () => { - // Add local provisional block const localBlock = await L2Block.random(BlockNumber(1), { checkpointNumber: CheckpointNumber(1), indexWithinCheckpoint: IndexWithinCheckpoint(0), slotNumber: SlotNumber(100), }); await updater.addProposedBlock(localBlock); - const publicLogsBefore = await store.logs.getPublicLogs({}); - expect(publicLogsBefore.logs.map(l => l.log)).toEqual(localBlock.body.txEffects.flatMap(tx => tx.publicLogs)); + expect(await countIndexedPublicLogs(localBlock)).toBe( + localBlock.body.txEffects.flatMap(tx => tx.publicLogs).length, + ); - // Create checkpoint with DIFFERENT block (different archive root) const checkpointBlock = await L2Block.random(BlockNumber(1), { checkpointNumber: CheckpointNumber(1), indexWithinCheckpoint: IndexWithinCheckpoint(0), @@ -321,30 +330,30 @@ describe('ArchiverDataStoreUpdater', () => { await updater.addCheckpoints([makePublishedCheckpoint(makeCheckpoint([checkpointBlock]), 10)]); - // Verify checkpoint block is stored const storedBlock = await store.blocks.getBlock({ number: BlockNumber(1) }); expect(storedBlock?.archive.root.equals(checkpointBlock.archive.root)).toBe(true); - const publicLogsAfter = await store.logs.getPublicLogs({}); - expect(publicLogsAfter.logs.map(l => l.log)).toEqual(checkpointBlock.body.txEffects.flatMap(tx => tx.publicLogs)); + + expect(await countIndexedPublicLogs(checkpointBlock)).toBe( + checkpointBlock.body.txEffects.flatMap(tx => tx.publicLogs).length, + ); + // The old (now-removed) block's logs are no longer indexed. + expect(await countIndexedPublicLogs(localBlock)).toBe(0); }); it('removes logs when removing uncheckpointed blocks', async () => { - // Add local provisional block const localBlock = await L2Block.random(BlockNumber(1), { checkpointNumber: CheckpointNumber(1), indexWithinCheckpoint: IndexWithinCheckpoint(0), slotNumber: SlotNumber(100), }); await updater.addProposedBlock(localBlock); - const publicLogsBefore = await store.logs.getPublicLogs({}); - expect(publicLogsBefore.logs.map(l => l.log)).toEqual(localBlock.body.txEffects.flatMap(tx => tx.publicLogs)); + expect(await countIndexedPublicLogs(localBlock)).toBe( + localBlock.body.txEffects.flatMap(tx => tx.publicLogs).length, + ); - // Remove the uncheckpointed block await updater.removeUncheckpointedBlocksAfter(BlockNumber.ZERO); - // Verify logs are removed - const publicLogsAfter = await store.logs.getPublicLogs({}); - expect(publicLogsAfter.logs.length).toBe(0); + expect(await countIndexedPublicLogs(localBlock)).toBe(0); }); }); diff --git a/yarn-project/archiver/src/store/block_store.test.ts b/yarn-project/archiver/src/store/block_store.test.ts index d093bbf30784..f434af7b07bf 100644 --- a/yarn-project/archiver/src/store/block_store.test.ts +++ b/yarn-project/archiver/src/store/block_store.test.ts @@ -1895,6 +1895,75 @@ describe('BlockStore', () => { }); }); + describe('getTxLocation', () => { + beforeEach(async () => { + await blockStore.addCheckpoints(publishedCheckpoints); + }); + + it.each([ + [1, 0], + [3, 1], + [9, 3], + [5, 2], + ])('returns (blockNumber, txIndex) for known tx at block %i tx-index %i', async (blockIdx, txIdx) => { + const block = publishedCheckpoints[blockIdx - 1].checkpoint.blocks[0]; + const txEffect = block.body.txEffects[txIdx]; + + const loc = await blockStore.getTxLocation(txEffect.txHash); + expect(loc).toEqual([block.number, txIdx]); + }); + + it('returns undefined for an unknown tx hash', async () => { + await expect(blockStore.getTxLocation(TxHash.random())).resolves.toBeUndefined(); + }); + }); + + describe('getNoteHashesAndNullifiers', () => { + beforeEach(async () => { + await blockStore.addCheckpoints(publishedCheckpoints); + }); + + it('returns one [noteHashes, nullifiers] tuple per input txHash in input order', async () => { + const txEffects = [ + publishedCheckpoints[0].checkpoint.blocks[0].body.txEffects[0], + publishedCheckpoints[5].checkpoint.blocks[0].body.txEffects[2], + publishedCheckpoints[9].checkpoint.blocks[0].body.txEffects[1], + ]; + const result = await blockStore.getNoteHashesAndNullifiers(txEffects.map(tx => tx.txHash)); + + expect(result.length).toBe(txEffects.length); + for (let i = 0; i < txEffects.length; i++) { + const [noteHashes, nullifiers] = result[i]; + expect(noteHashes).toEqual(txEffects[i].noteHashes); + expect(nullifiers).toEqual(txEffects[i].nullifiers); + } + }); + + it('returns [[], []] for unknown txHashes (preserves input order)', async () => { + const known = publishedCheckpoints[2].checkpoint.blocks[0].body.txEffects[0]; + const unknown = TxHash.random(); + const result = await blockStore.getNoteHashesAndNullifiers([unknown, known.txHash, unknown]); + + expect(result.length).toBe(3); + expect(result[0]).toEqual([[], []]); + expect(result[1][0]).toEqual(known.noteHashes); + expect(result[1][1]).toEqual(known.nullifiers); + expect(result[2]).toEqual([[], []]); + }); + + it('returns [] for an empty input array', async () => { + await expect(blockStore.getNoteHashesAndNullifiers([])).resolves.toEqual([]); + }); + + it('the partial deserializer matches the full getTxEffect for noteHashes/nullifiers', async () => { + const tx = publishedCheckpoints[4].checkpoint.blocks[0].body.txEffects[2]; + const [[noteHashes, nullifiers]] = await blockStore.getNoteHashesAndNullifiers([tx.txHash]); + const full = await blockStore.getTxEffect(tx.txHash); + expect(noteHashes).toEqual(full!.data.noteHashes); + expect(nullifiers).toEqual(full!.data.nullifiers); + }); + }); + describe('pendingChainValidationStatus', () => { it('should return undefined when no status is set', async () => { const status = await blockStore.getPendingChainValidationStatus(); diff --git a/yarn-project/archiver/src/store/block_store.ts b/yarn-project/archiver/src/store/block_store.ts index 0db47a55eafb..673ec8a2f2ce 100644 --- a/yarn-project/archiver/src/store/block_store.ts +++ b/yarn-project/archiver/src/store/block_store.ts @@ -1152,10 +1152,43 @@ export class BlockStore { if (!txEffect) { return undefined; } - const { l2BlockNumber, txIndexInBlock } = deserializeIndexedTxEffect(txEffect); + // Read only the IndexedTxEffect header (`blockHash(32) + l2BlockNumber(4) + txIndexInBlock(4)`); the + // large tail (the full TxEffect with logs etc.) is irrelevant here. + const view = Buffer.from(txEffect.buffer, txEffect.byteOffset, txEffect.byteLength); + const l2BlockNumber = view.readUInt32BE(32); + const txIndexInBlock = view.readUInt32BE(36); return [l2BlockNumber, txIndexInBlock]; } + /** + * Batched, partial deserializer that fetches `noteHashes` and `nullifiers` (all of them) for the given + * txs. For each input txHash, returns a `[noteHashes, nullifiers]` tuple. Returns `[[], []]` for any + * unknown txHash. Preserves input order. Used by the log read path when `includeEffects` is set to + * attach effect data on demand without paying for a full {@link TxEffect} deserialization. + * + * The on-disk `IndexedTxEffect` layout starts with a fixed-length header + * (`blockHash(32) + l2BlockNumber(4) + txIndexInBlock(4) + revertCode(1) + txHash(32) + transactionFee(32)` = + * 105 bytes), followed by `noteHashes` and `nullifiers` (both u8-length-prefixed `Fr` vectors). We + * skip the header, then read the two vectors, and stop — the large tail (`l2ToL1Msgs`, + * `publicDataWrites`, `privateLogs`, `publicLogs`, `contractClassLogs`) is never touched. + */ + public getNoteHashesAndNullifiers(txHashes: TxHash[]): Promise<[Fr[], Fr[]][]> { + return Promise.all( + txHashes.map(async (txHash): Promise<[Fr[], Fr[]]> => { + const buffer = await this.#txEffects.getAsync(txHash.toString()); + if (!buffer) { + return [[], []]; + } + const reader = BufferReader.asReader(buffer); + // Skip the fixed-length header: blockHash + l2BlockNumber + txIndexInBlock + revertCode + txHash + transactionFee. + reader.readBytes(32 + 4 + 4 + 1 + 32 + 32); + const noteHashes = reader.readVectorUint8Prefix(Fr); + const nullifiers = reader.readVectorUint8Prefix(Fr); + return [noteHashes, nullifiers]; + }), + ); + } + /** * Looks up which block deployed a particular contract. * @param contractAddress - The address of the contract to look up. diff --git a/yarn-project/archiver/src/store/data_stores.ts b/yarn-project/archiver/src/store/data_stores.ts index 86fe761a5962..d34d9497b152 100644 --- a/yarn-project/archiver/src/store/data_stores.ts +++ b/yarn-project/archiver/src/store/data_stores.ts @@ -1,5 +1,6 @@ import type { L1BlockId } from '@aztec/ethereum/l1-types'; import type { AztecAsyncKVStore } from '@aztec/kv-store'; +import type { BlockHash } from '@aztec/stdlib/block'; import type { ContractDataSource } from '@aztec/stdlib/contract'; import { join } from 'path'; @@ -12,7 +13,7 @@ import { FunctionNamesCache } from './function_names_cache.js'; import { LogStore } from './log_store.js'; import { MessageStore } from './message_store.js'; -export const ARCHIVER_DB_VERSION = 6; +export const ARCHIVER_DB_VERSION = 7; /** * Represents the latest L1 block processed by the archiver for various objects in L2. @@ -48,25 +49,20 @@ export type ArchiverDataStores = { functionNames: FunctionNamesCache; }; -/** Options used by {@link createArchiverDataStores}. */ -export type CreateArchiverDataStoresOptions = { - /** Maximum number of logs returned per page when paginating tagged log queries. */ - logsMaxPageSize?: number; -}; - /** * Wires up the archiver substores against a shared KV store and returns the * {@link ArchiverDataStores} bundle. + * + * @param genesisBlockHash - Hash of the synthetic genesis block, forwarded to the {@link LogStore} so it + * can resolve a genesis `referenceBlock` (used by the PXE during early sync) instead of treating it as a + * reorg. */ -export function createArchiverDataStores( - db: AztecAsyncKVStore, - opts: CreateArchiverDataStoresOptions = {}, -): ArchiverDataStores { +export function createArchiverDataStores(db: AztecAsyncKVStore, genesisBlockHash: BlockHash): ArchiverDataStores { const blocks = new BlockStore(db); return { db, blocks, - logs: new LogStore(db, blocks, opts.logsMaxPageSize ?? 1000), + logs: new LogStore(db, blocks, genesisBlockHash), messages: new MessageStore(db), contractClasses: new ContractClassStore(db), contractInstances: new ContractInstanceStore(db), diff --git a/yarn-project/archiver/src/store/log_store.test.ts b/yarn-project/archiver/src/store/log_store.test.ts index b4212aa6b166..1b684535f063 100644 --- a/yarn-project/archiver/src/store/log_store.test.ts +++ b/yarn-project/archiver/src/store/log_store.test.ts @@ -1,40 +1,47 @@ -import { INITIAL_L2_BLOCK_NUM } from '@aztec/constants'; -import { BlockNumber, CheckpointNumber, IndexWithinCheckpoint } from '@aztec/foundation/branded-types'; -import { randomInt } from '@aztec/foundation/crypto/random'; +import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types'; +import { Fr } from '@aztec/foundation/curves/bn254'; import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; import { AztecAddress } from '@aztec/stdlib/aztec-address'; -import { BlockHash, L2Block } from '@aztec/stdlib/block'; -import { Checkpoint, PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; +import { BlockHash, GENESIS_BLOCK_HEADER_HASH } from '@aztec/stdlib/block'; +import { Checkpoint, type PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; import { MAX_LOGS_PER_TAG } from '@aztec/stdlib/interfaces/api-limit'; -import { ContractClassLog, LogId } from '@aztec/stdlib/logs'; +import { LogCursor, SiloedTag, Tag, queryAllPrivateLogsByTags, queryAllPublicLogsByTags } from '@aztec/stdlib/logs'; import '@aztec/stdlib/testing/jest'; -import { TxHash } from '@aztec/stdlib/tx'; +import type { AppendOnlyTreeSnapshot } from '@aztec/stdlib/trees'; + +import { jest } from '@jest/globals'; -import { OutOfOrderLogInsertionError } from '../errors.js'; import { + type MockCheckpointWithLogsOptions, makeCheckpointWithLogs, - makePrivateLog, makePrivateLogTag, - makePublicLog, - makePublicLogTag, makePublishedCheckpoint, makeStateForBlock, } from '../test/mock_structs.js'; import { BlockStore } from './block_store.js'; import { LogStore } from './log_store.js'; -async function addProposedBlocks( - blockStore: BlockStore, - blocks: L2Block[], - opts?: { force?: boolean }, -): Promise { - let result = true; - for (const block of blocks) { - result = (await blockStore.addProposedBlock(block, opts)) && result; +/** + * Builds a list of `PublishedCheckpoint`s for sequential blocks `[1..count]`, chaining each + * block's `lastArchive` to the previous block's `archive` so they pass `BlockStore.addCheckpoints` + * validation. The caller may mutate the returned blocks (e.g. overriding log fields) before adding. + */ +async function buildChainedCheckpointsWithLogs( + count: number, + options: MockCheckpointWithLogsOptions, +): Promise { + const checkpoints: PublishedCheckpoint[] = []; + let previousArchive: AppendOnlyTreeSnapshot | undefined; + for (let b = 1; b <= count; b++) { + const ckpt = await makeCheckpointWithLogs(b, { ...options, previousArchive }); + previousArchive = ckpt.checkpoint.blocks[0].archive; + checkpoints.push(ckpt); } - return result; + return checkpoints; } +const CONTRACT = AztecAddress.fromNumber(543254); + describe('LogStore', () => { let blockStore: BlockStore; let logStore: LogStore; @@ -43,10 +50,11 @@ describe('LogStore', () => { beforeEach(async () => { const db = await openTmpStore('log_store_test'); blockStore = new BlockStore(db); - logStore = new LogStore(db, blockStore, 1000); - // Create checkpoints sequentially to ensure archive roots are chained properly. + logStore = new LogStore(db, blockStore, GENESIS_BLOCK_HEADER_HASH); + + // Build 10 sequential single-block checkpoints, each with 2 txs and 2 logs/tx (private + public). publishedCheckpoints = []; - const txsPerBlock = 4; + const txsPerBlock = 2; for (let i = 0; i < 10; i++) { const blockNumber = i + 1; const previousArchive = i > 0 ? publishedCheckpoints[i - 1].checkpoint.blocks[0].archive : undefined; @@ -56,1077 +64,634 @@ describe('LogStore', () => { previousArchive, txsPerBlock, state: makeStateForBlock(blockNumber, txsPerBlock), - txOptions: { numPublicCallsPerTx: 2, numPublicLogsPerCall: 2 }, + // Ensure each tx emits at least one nullifier (needed by validator). + txOptions: { numNullifiers: 2, numPublicCallsPerTx: 1, numPublicLogsPerCall: 1 }, }); publishedCheckpoints.push(makePublishedCheckpoint(checkpoint, i + 10)); } }); - describe('addLogs', () => { - it('adds private & public logs', async () => { - const checkpoint = publishedCheckpoints[0]; - await blockStore.addCheckpoints([checkpoint]); - await expect(logStore.addLogs(checkpoint.checkpoint.blocks)).resolves.toEqual(true); - }); - }); - - describe('deleteLogs', () => { - it('deletes public logs for a block', async () => { + describe('addLogs / deleteLogs', () => { + it('adds private & public logs and returns true', async () => { const block = publishedCheckpoints[0].checkpoint.blocks[0]; await blockStore.addProposedBlock(block); await expect(logStore.addLogs([block])).resolves.toEqual(true); - - expect((await logStore.getPublicLogs({ fromBlock: BlockNumber(1) })).logs.length).toEqual( - block.body.txEffects.map(txEffect => txEffect.publicLogs).flat().length, - ); - - await logStore.deleteLogs([block]); - - expect((await logStore.getPublicLogs({ fromBlock: BlockNumber(1) })).logs.length).toEqual(0); }); - it('deletes contract class logs for a block', async () => { - // Create a block that explicitly has contract class logs - const block = await L2Block.random(BlockNumber(1), { - txsPerBlock: 2, - txOptions: { numContractClassLogs: 1 }, - state: makeStateForBlock(1, 2), + it('deletes all logs for the given blocks (reorg trim)', async () => { + // Build a custom checkpoint where every log has a known tag so we can query it back. + const ckpt = await makeCheckpointWithLogs(1, { + numTxsPerBlock: 2, + privateLogs: { numLogsPerTx: 1 }, + publicLogs: { numLogsPerTx: 1, contractAddress: CONTRACT }, }); + const block = ckpt.checkpoint.blocks[0]; await blockStore.addProposedBlock(block); await logStore.addLogs([block]); - const logsBefore = await logStore.getContractClassLogs({ fromBlock: BlockNumber(1) }); - expect(logsBefore.logs.length).toBeGreaterThan(0); + // Sanity: logs are present. + const tag = makePrivateLogTag(1, 0, 0); + const before = await logStore.getPrivateLogsByTags({ tags: [tag] }); + expect(before[0].length).toBe(1); await logStore.deleteLogs([block]); - const logsAfter = await logStore.getContractClassLogs({ fromBlock: BlockNumber(1) }); - expect(logsAfter.logs.length).toEqual(0); + const after = await logStore.getPrivateLogsByTags({ tags: [tag] }); + expect(after[0].length).toBe(0); }); + }); - it('retains private logs from non-reorged block when same tag appears in reorged block', async () => { - const sharedTag = makePrivateLogTag(1, 0, 0); - - // Block 1 with a private log using sharedTag - const cp1 = await makeCheckpointWithLogs(1, { - numTxsPerBlock: 1, - privateLogs: { numLogsPerTx: 1 }, - }); - const block1 = cp1.checkpoint.blocks[0]; - - // Block 2 with a private log using the SAME tag - const cp2 = await makeCheckpointWithLogs(2, { - previousArchive: block1.archive, - numTxsPerBlock: 1, - privateLogs: { numLogsPerTx: 1 }, + describe('getPrivateLogsByTags', () => { + it('returns empty for an unknown tag', async () => { + const ckpt = publishedCheckpoints[0]; + await blockStore.addCheckpoints([ckpt]); + await logStore.addLogs(ckpt.checkpoint.blocks); + + const unseen = SiloedTag.random(); + const result = await logStore.getPrivateLogsByTags({ tags: [unseen] }); + expect(result).toEqual([[]]); + }); + + it('preserves canonical key order across many shuffled inserts (key encoding regression)', async () => { + // Stamp a shared tag across 1000 (txIndex, logIndex) slots spread over sequential blocks. The + // hex-string key encoding must yield entries in canonical composite order even though + // higher block numbers produce lex-greater keys (where naive padding would break ordering). + const sharedTag = new SiloedTag(new Fr(0x5050n)); + const NUM_BLOCKS = 10; + const TXS_PER_BLOCK = 10; + const LOGS_PER_TX = 10; + + const ckpts = await buildChainedCheckpointsWithLogs(NUM_BLOCKS, { + numTxsPerBlock: TXS_PER_BLOCK, + privateLogs: { numLogsPerTx: LOGS_PER_TX }, }); - const block2 = cp2.checkpoint.blocks[0]; - // Override block2's private log tag to match block1's - block2.body.txEffects[0].privateLogs[0] = makePrivateLog(sharedTag); - - await addProposedBlocks(blockStore, [block1, block2], { force: true }); - await logStore.addLogs([block1, block2]); - - // Both blocks' logs should be present - const logsBefore = await logStore.getPrivateLogsByTags([sharedTag]); - expect(logsBefore[0]).toHaveLength(2); - - // Reorg: delete block 2 - await logStore.deleteLogs([block2]); + for (const ckpt of ckpts) { + for (const tx of ckpt.checkpoint.blocks[0].body.txEffects) { + for (const log of tx.privateLogs) { + (log.fields as Fr[])[0] = sharedTag.value; + } + } + } + const blocks = ckpts.map(c => c.checkpoint.blocks[0]); + await blockStore.addCheckpoints(ckpts); + await logStore.addLogs(blocks); + + // Drain via pagination. + const all: { block: number; txIdx: number; logIdx: number }[] = []; + let afterLog: LogCursor | undefined; + while (true) { + const [page] = await logStore.getPrivateLogsByTags({ + tags: afterLog ? [{ tag: sharedTag, afterLog }] : [sharedTag], + }); + for (const log of page) { + all.push({ + block: Number(log.blockNumber), + txIdx: log.txIndexWithinBlock, + logIdx: log.logIndexWithinTx, + }); + } + if (page.length < MAX_LOGS_PER_TAG) { + break; + } + afterLog = LogCursor.fromLog(page[page.length - 1]); + } - // Block 1's log should still be present - const logsAfter = await logStore.getPrivateLogsByTags([sharedTag]); - expect(logsAfter[0]).toHaveLength(1); - expect(logsAfter[0][0].blockNumber).toEqual(1); + const total = NUM_BLOCKS * TXS_PER_BLOCK * LOGS_PER_TX; + expect(all.length).toBe(total); + + // Verify canonical ordering: (block, txIdx, logIdx) strictly increasing tuple by tuple. + for (let i = 1; i < all.length; i++) { + const a = all[i - 1]; + const b = all[i]; + const greater = + b.block > a.block || + (b.block === a.block && (b.txIdx > a.txIdx || (b.txIdx === a.txIdx && b.logIdx > a.logIdx))); + expect(greater).toBe(true); + } }); - it('retains public logs from non-reorged block when same tag appears in reorged block', async () => { - const contractAddress = AztecAddress.fromNumber(543254); - const sharedTag = makePublicLogTag(1, 0, 0); - - // Block 1 with a public log using sharedTag - const cp1 = await makeCheckpointWithLogs(1, { - numTxsPerBlock: 1, - publicLogs: { numLogsPerTx: 1, contractAddress }, - }); - const block1 = cp1.checkpoint.blocks[0]; - - // Block 2 with a public log using the SAME tag from the same contract - const cp2 = await makeCheckpointWithLogs(2, { - previousArchive: block1.archive, - numTxsPerBlock: 1, - publicLogs: { numLogsPerTx: 1, contractAddress }, + it('returns logs in canonical (block, txIndex, logIndex) order for a known tag', async () => { + // 3 blocks, each with 2 txs, each with 4 private logs (24 total), all sharing a fixed tag — more than + // MAX_LOGS_PER_TAG so the first page fills and spans multiple blocks. + const sharedTag = new SiloedTag(new Fr(0xdeadbeefn)); + const ckpts = await buildChainedCheckpointsWithLogs(3, { + numTxsPerBlock: 2, + privateLogs: { numLogsPerTx: 4 }, }); - const block2 = cp2.checkpoint.blocks[0]; - // Override block2's public log tag to match block1's - block2.body.txEffects[0].publicLogs[0] = makePublicLog(sharedTag, contractAddress); - - await addProposedBlocks(blockStore, [block1, block2], { force: true }); - await logStore.addLogs([block1, block2]); - - // Both blocks' logs should be present - const logsBefore = await logStore.getPublicLogsByTagsFromContract(contractAddress, [sharedTag]); - expect(logsBefore[0]).toHaveLength(2); + // Override the first field of every private log to the shared tag. + for (const ckpt of ckpts) { + for (const txEffect of ckpt.checkpoint.blocks[0].body.txEffects) { + for (const log of txEffect.privateLogs) { + (log.fields as Fr[])[0] = sharedTag.value; + } + } + } + const blocks = ckpts.map(c => c.checkpoint.blocks[0]); + await blockStore.addCheckpoints(ckpts); + await logStore.addLogs(blocks); - // Reorg: delete block 2 - await logStore.deleteLogs([block2]); + // 3 blocks * 2 txs * 4 logs = 24, but we cap at MAX_LOGS_PER_TAG. + const [page1] = await logStore.getPrivateLogsByTags({ tags: [sharedTag] }); + expect(page1.length).toBe(MAX_LOGS_PER_TAG); - // Block 1's log should still be present - const logsAfter = await logStore.getPublicLogsByTagsFromContract(contractAddress, [sharedTag]); - expect(logsAfter[0]).toHaveLength(1); - expect(logsAfter[0][0].blockNumber).toEqual(1); + // Verify order: blockNumber non-decreasing; within a block, logIndexWithinTx ascending. + for (let i = 1; i < page1.length; i++) { + expect(page1[i].blockNumber >= page1[i - 1].blockNumber).toBe(true); + } }); - it('deletes multiple blocks at once', async () => { - const cp1 = await makeCheckpointWithLogs(1, { - numTxsPerBlock: 2, - privateLogs: { numLogsPerTx: 1 }, - publicLogs: { numLogsPerTx: 1 }, - }); - const block1 = cp1.checkpoint.blocks[0]; - - const cp2 = await makeCheckpointWithLogs(2, { - previousArchive: block1.archive, - numTxsPerBlock: 2, + it('respects fromBlock / toBlock', async () => { + const sharedTag = new SiloedTag(new Fr(0x1234n)); + const ckpts = await buildChainedCheckpointsWithLogs(5, { + numTxsPerBlock: 1, privateLogs: { numLogsPerTx: 1 }, - publicLogs: { numLogsPerTx: 1 }, }); - const block2 = cp2.checkpoint.blocks[0]; - - await addProposedBlocks(blockStore, [block1, block2], { force: true }); - await logStore.addLogs([block1, block2]); - - // Verify logs exist - expect((await logStore.getPublicLogs({ fromBlock: BlockNumber(1) })).logs.length).toBeGreaterThan(0); - - // Delete both blocks at once - await logStore.deleteLogs([block1, block2]); - - expect((await logStore.getPublicLogs({ fromBlock: BlockNumber(1) })).logs.length).toEqual(0); - }); - - it('is a no-op when deleting blocks with no logs', async () => { - const block = publishedCheckpoints[0].checkpoint.blocks[0]; - // Don't add logs, just try to delete - await expect(logStore.deleteLogs([block])).resolves.toEqual(true); - }); - }); - - describe('getPrivateLogsByTags', () => { - const numBlocksForLogs = 3; - const numTxsPerBlock = 4; - const numPrivateLogsPerTx = 3; - - let logsCheckpoints: PublishedCheckpoint[]; - - beforeEach(async () => { - // Create checkpoints sequentially to chain archive roots - logsCheckpoints = []; - for (let i = 0; i < numBlocksForLogs; i++) { - const previousArchive = i > 0 ? logsCheckpoints[i - 1].checkpoint.blocks[0].archive : undefined; - logsCheckpoints.push( - await makeCheckpointWithLogs(i + 1, { - previousArchive, - numTxsPerBlock, - privateLogs: { numLogsPerTx: numPrivateLogsPerTx }, - }), - ); + for (const ckpt of ckpts) { + ckpt.checkpoint.blocks[0].body.txEffects[0].privateLogs[0].fields[0] = sharedTag.value; } - - await blockStore.addCheckpoints(logsCheckpoints); - await logStore.addLogs(logsCheckpoints.flatMap(p => p.checkpoint.blocks)); - }); - - it('is possible to batch request private logs via tags', async () => { - const tags = [makePrivateLogTag(2, 1, 2), makePrivateLogTag(1, 2, 0)]; - - const logsByTags = await logStore.getPrivateLogsByTags(tags); - - expect(logsByTags).toEqual([ - [ - expect.objectContaining({ - blockNumber: 2, - logData: makePrivateLog(tags[0]).getEmittedFields(), - }), - ], - [ - expect.objectContaining({ - blockNumber: 1, - logData: makePrivateLog(tags[1]).getEmittedFields(), - }), - ], - ]); - }); - - it('is possible to batch request logs that have the same tag but different content', async () => { - const tags = [makePrivateLogTag(1, 2, 1)]; - - // Create a checkpoint containing logs that have the same tag as the checkpoints before. - // Chain from the last checkpoint's archive - const newBlockNumber = numBlocksForLogs + 1; - const previousArchive = logsCheckpoints[logsCheckpoints.length - 1].checkpoint.blocks[0].archive; - const newCheckpoint = await makeCheckpointWithLogs(newBlockNumber, { - previousArchive, - numTxsPerBlock, - privateLogs: { numLogsPerTx: numPrivateLogsPerTx }, + const blocks = ckpts.map(c => c.checkpoint.blocks[0]); + await blockStore.addCheckpoints(ckpts); + await logStore.addLogs(blocks); + + // Block range [2, 4) → blocks 2, 3 only. + const [res] = await logStore.getPrivateLogsByTags({ + tags: [sharedTag], + fromBlock: BlockNumber(2), + toBlock: BlockNumber(4), }); - const newLog = newCheckpoint.checkpoint.blocks[0].body.txEffects[1].privateLogs[1]; - newLog.fields[0] = tags[0].value; - newCheckpoint.checkpoint.blocks[0].body.txEffects[1].privateLogs[1] = newLog; - await blockStore.addCheckpoints([newCheckpoint]); - await logStore.addLogs([newCheckpoint.checkpoint.blocks[0]]); - - const logsByTags = await logStore.getPrivateLogsByTags(tags); - - expect(logsByTags).toEqual([ - [ - expect.objectContaining({ - blockNumber: 1, - logData: makePrivateLog(tags[0]).getEmittedFields(), - }), - expect.objectContaining({ - blockNumber: newBlockNumber, - logData: newLog.getEmittedFields(), - }), - ], - ]); + expect(res.map(l => l.blockNumber)).toEqual([BlockNumber(2), BlockNumber(3)]); }); - it('throws on out-of-order private log insertion', async () => { - const sharedTag = makePrivateLogTag(99, 0, 0); - - // Create blocks 4 and 5 with the same shared tag - const prevArchive1 = logsCheckpoints[logsCheckpoints.length - 1].checkpoint.blocks[0].archive; - const checkpoint4 = await makeCheckpointWithLogs(numBlocksForLogs + 1, { - previousArchive: prevArchive1, - numTxsPerBlock, - privateLogs: { numLogsPerTx: numPrivateLogsPerTx }, + it('returns empty when the range is before the first log', async () => { + // Logs live only on block 5; the chained checkpoints 1-4 carry no logs for `sharedTag`. + const sharedTag = new SiloedTag(new Fr(0xaa11n)); + const ckpts = await buildChainedCheckpointsWithLogs(5, { + numTxsPerBlock: 1, + privateLogs: { numLogsPerTx: 1 }, }); - checkpoint4.checkpoint.blocks[0].body.txEffects[0].privateLogs[0] = makePrivateLog(sharedTag); - - const prevArchive2 = checkpoint4.checkpoint.blocks[0].archive; - const checkpoint5 = await makeCheckpointWithLogs(numBlocksForLogs + 2, { - previousArchive: prevArchive2, - numTxsPerBlock, - privateLogs: { numLogsPerTx: numPrivateLogsPerTx }, + ckpts[4].checkpoint.blocks[0].body.txEffects[0].privateLogs[0].fields[0] = sharedTag.value; + const blocks = ckpts.map(c => c.checkpoint.blocks[0]); + await blockStore.addCheckpoints(ckpts); + await logStore.addLogs(blocks); + + // Range [1, 5) excludes block 5 → no hits. + const [res] = await logStore.getPrivateLogsByTags({ + tags: [sharedTag], + fromBlock: BlockNumber(1), + toBlock: BlockNumber(5), }); - checkpoint5.checkpoint.blocks[0].body.txEffects[0].privateLogs[0] = makePrivateLog(sharedTag); - - // Store block 5's logs first (higher block number), then try to store block 4's logs - // (lower block number) — this should fail. - await logStore.addLogs([checkpoint5.checkpoint.blocks[0]]); - await expect(logStore.addLogs([checkpoint4.checkpoint.blocks[0]])).rejects.toThrow(OutOfOrderLogInsertionError); - }); - - it('is possible to request logs for non-existing tags and determine their position', async () => { - const tags = [makePrivateLogTag(99, 88, 77), makePrivateLogTag(1, 1, 1)]; - - const logsByTags = await logStore.getPrivateLogsByTags(tags); - - expect(logsByTags).toEqual([ - [ - // No logs for the first tag. - ], - [ - expect.objectContaining({ - blockNumber: 1, - logData: makePrivateLog(tags[1]).getEmittedFields(), - }), - ], - ]); - }); - - it('filters logs up to specified block number', async () => { - // Tags are unique per block, so create a shared tag across blocks by adding logs with the same tag - const sharedTag = makePrivateLogTag(1, 2, 1); - - // Add extra blocks with logs sharing the same tag - for (let blockNum = numBlocksForLogs + 1; blockNum <= numBlocksForLogs + 2; blockNum++) { - const previousArchive = logsCheckpoints[logsCheckpoints.length - 1].checkpoint.blocks[0].archive; - const newCheckpoint = await makeCheckpointWithLogs(blockNum, { - previousArchive, - numTxsPerBlock, - privateLogs: { numLogsPerTx: numPrivateLogsPerTx }, - }); - const newLog = newCheckpoint.checkpoint.blocks[0].body.txEffects[1].privateLogs[1]; - newLog.fields[0] = sharedTag.value; - newCheckpoint.checkpoint.blocks[0].body.txEffects[1].privateLogs[1] = newLog; - await blockStore.addCheckpoints([newCheckpoint]); - await logStore.addLogs([newCheckpoint.checkpoint.blocks[0]]); - logsCheckpoints.push(newCheckpoint); - } - - // Without filter, should return logs from block 1 and the extra blocks - const allLogs = await logStore.getPrivateLogsByTags([sharedTag]); - expect(allLogs[0].some(log => log.blockNumber > numBlocksForLogs)).toBe(true); - - // With upToBlockNumber=numBlocksForLogs, should only return the original log from block 1 - const filteredLogs = await logStore.getPrivateLogsByTags([sharedTag], 0, BlockNumber(numBlocksForLogs)); - expect(filteredLogs[0].length).toBeGreaterThan(0); - for (const log of filteredLogs[0]) { - expect(log.blockNumber).toBeLessThanOrEqual(numBlocksForLogs); - } - expect(filteredLogs[0].length).toBeLessThan(allLogs[0].length); + expect(res).toEqual([]); }); - it('returns all logs when upToBlockNumber is not set', async () => { - const tag = makePrivateLogTag(1, 2, 1); - - const logsWithoutFilter = await logStore.getPrivateLogsByTags([tag]); - const logsWithUndefined = await logStore.getPrivateLogsByTags([tag], 0, undefined); - - expect(logsWithoutFilter).toEqual(logsWithUndefined); - }); - - describe('pagination', () => { - const paginationTag = makePrivateLogTag(1, 2, 1); - - beforeEach(async () => { - // Add more blocks with the same tag to exceed MAX_LOGS_PER_TAG - for (let i = numBlocksForLogs; i < numBlocksForLogs + MAX_LOGS_PER_TAG + 5; i++) { - const previousArchive = logsCheckpoints[logsCheckpoints.length - 1].checkpoint.blocks[0].archive; - const newCheckpoint = await makeCheckpointWithLogs(i + 1, { - previousArchive, - numTxsPerBlock, - privateLogs: { numLogsPerTx: numPrivateLogsPerTx }, - }); - const newLog = newCheckpoint.checkpoint.blocks[0].body.txEffects[1].privateLogs[1]; - newLog.fields[0] = paginationTag.value; - newCheckpoint.checkpoint.blocks[0].body.txEffects[1].privateLogs[1] = newLog; - await blockStore.addCheckpoints([newCheckpoint]); - await logStore.addLogs([newCheckpoint.checkpoint.blocks[0]]); - logsCheckpoints.push(newCheckpoint); - } - }); - - it('pagination works correctly with upToBlockNumber', async () => { - // With a low upToBlockNumber, the filtered set should be smaller than MAX_LOGS_PER_TAG - const filteredPage0 = await logStore.getPrivateLogsByTags([paginationTag], 0, BlockNumber(5)); - for (const log of filteredPage0[0]) { - expect(log.blockNumber).toBeLessThanOrEqual(5); - } - - // Page 1 with the same filter should only contain remaining filtered logs - const filteredPage1 = await logStore.getPrivateLogsByTags([paginationTag], 1, BlockNumber(5)); - for (const log of filteredPage1[0]) { - expect(log.blockNumber).toBeLessThanOrEqual(5); - } - }); - - it('returns first page of logs when page=0', async () => { - const logsByTags = await logStore.getPrivateLogsByTags([paginationTag], 0); - - expect(logsByTags[0]).toHaveLength(MAX_LOGS_PER_TAG); - expect(logsByTags[0][0].blockNumber).toBe(1); // First log from block 1 + it('returns empty when the range is past the last log', async () => { + // Logs live only on block 3; querying from block 10 yields nothing. + const sharedTag = new SiloedTag(new Fr(0xbb22n)); + const ckpts = await buildChainedCheckpointsWithLogs(3, { + numTxsPerBlock: 1, + privateLogs: { numLogsPerTx: 1 }, }); - - it('returns second page of logs when page=1', async () => { - const logsByTags = await logStore.getPrivateLogsByTags([paginationTag], 1); - - // Should have the remaining logs (total was MAX_LOGS_PER_TAG + 6, so page 1 has 6) - expect(logsByTags[0]).toHaveLength(6); + ckpts[2].checkpoint.blocks[0].body.txEffects[0].privateLogs[0].fields[0] = sharedTag.value; + const blocks = ckpts.map(c => c.checkpoint.blocks[0]); + await blockStore.addCheckpoints(ckpts); + await logStore.addLogs(blocks); + + const [res] = await logStore.getPrivateLogsByTags({ + tags: [sharedTag], + fromBlock: BlockNumber(10), }); + expect(res).toEqual([]); + }); - it('returns empty array when page is beyond available logs', async () => { - const logsByTags = await logStore.getPrivateLogsByTags([paginationTag], 100); - - expect(logsByTags).toEqual([[]]); + it('paginates correctly across a page boundary via afterLog', async () => { + const sharedTag = new SiloedTag(new Fr(0xcc33n)); + const ckpts = await buildChainedCheckpointsWithLogs(6, { + numTxsPerBlock: 2, + privateLogs: { numLogsPerTx: 4 }, }); - - /** - * Verifies that logs are stored and returned in block order (ascending). - * This ordering guarantee is critical for pagination safety: if logs were not ordered - * by block number, new logs added between paginated calls could be inserted in the - * middle of the result set, causing callers to receive duplicate logs or miss logs - * entirely. By maintaining block order, new logs always appear at the end of the - * result set, meaning previously fetched pages remain stable. - */ - it('maintains stable pagination when new logs are added between page fetches', async () => { - // Fetch page 0 and record the block numbers - const page0Before = await logStore.getPrivateLogsByTags([paginationTag], 0); - expect(page0Before[0]).toHaveLength(MAX_LOGS_PER_TAG); - const blockNumbersBefore = page0Before[0].map(log => log.blockNumber); - - // Verify block numbers are in ascending order - for (let i = 1; i < blockNumbersBefore.length; i++) { - expect(blockNumbersBefore[i]).toBeGreaterThanOrEqual(blockNumbersBefore[i - 1]); - } - - // Add more blocks with the same tag - const additionalBlocks = 3; - for (let i = 0; i < additionalBlocks; i++) { - const previousArchive = logsCheckpoints[logsCheckpoints.length - 1].checkpoint.blocks[0].archive; - const blockNumber = logsCheckpoints.length + 1; - const newCheckpoint = await makeCheckpointWithLogs(blockNumber, { - previousArchive, - numTxsPerBlock, - privateLogs: { numLogsPerTx: numPrivateLogsPerTx }, - }); - const newLog = newCheckpoint.checkpoint.blocks[0].body.txEffects[1].privateLogs[1]; - newLog.fields[0] = paginationTag.value; - newCheckpoint.checkpoint.blocks[0].body.txEffects[1].privateLogs[1] = newLog; - await blockStore.addCheckpoints([newCheckpoint]); - await logStore.addLogs([newCheckpoint.checkpoint.blocks[0]]); - logsCheckpoints.push(newCheckpoint); - } - - // Fetch page 0 again - should return the exact same logs - const page0After = await logStore.getPrivateLogsByTags([paginationTag], 0); - expect(page0After[0]).toHaveLength(MAX_LOGS_PER_TAG); - const blockNumbersAfter = page0After[0].map(log => log.blockNumber); - expect(blockNumbersAfter).toEqual(blockNumbersBefore); - - // Fetch page 1 - should include the newly added logs - const page1 = await logStore.getPrivateLogsByTags([paginationTag], 1); - expect(page1[0].length).toBeGreaterThan(0); - - // Verify all logs across both pages are in ascending block order - const allBlockNumbers = [...blockNumbersAfter, ...page1[0].map(log => log.blockNumber)]; - for (let i = 1; i < allBlockNumbers.length; i++) { - expect(allBlockNumbers[i]).toBeGreaterThanOrEqual(allBlockNumbers[i - 1]); - } - - // The new logs should appear on page 1, meaning their block numbers - // should be greater than or equal to the max block number on page 0 - const maxBlockOnPage0 = Math.max(...blockNumbersAfter); - const newLogBlockNumbers = page1[0].slice(-additionalBlocks).map(log => log.blockNumber); - for (const blockNum of newLogBlockNumbers) { - expect(blockNum).toBeGreaterThanOrEqual(maxBlockOnPage0); + for (const ckpt of ckpts) { + for (const tx of ckpt.checkpoint.blocks[0].body.txEffects) { + for (const log of tx.privateLogs) { + (log.fields as Fr[])[0] = sharedTag.value; + } } + } + const blocks = ckpts.map(c => c.checkpoint.blocks[0]); + await blockStore.addCheckpoints(ckpts); + await logStore.addLogs(blocks); + + // 6 blocks * 2 txs * 4 logs = 48 total — over two full pages. First page returns MAX_LOGS_PER_TAG. + const [page1] = await logStore.getPrivateLogsByTags({ tags: [sharedTag] }); + expect(page1.length).toBe(MAX_LOGS_PER_TAG); + + const cursor = LogCursor.fromLog(page1[page1.length - 1]); + const [page2b] = await logStore.getPrivateLogsByTags({ tags: [{ tag: sharedTag, afterLog: cursor }] }); + expect(page2b.length).toBe(MAX_LOGS_PER_TAG); + // No overlap between page1 and page2b: every page2b entry must be strictly after the page1 cursor. + const lastP1 = page1[page1.length - 1]; + const firstP2 = page2b[0]; + expect( + firstP2.blockNumber > lastP1.blockNumber || + (firstP2.blockNumber === lastP1.blockNumber && + (firstP2.txIndexWithinBlock > lastP1.txIndexWithinBlock || + (firstP2.txIndexWithinBlock === lastP1.txIndexWithinBlock && + firstP2.logIndexWithinTx > lastP1.logIndexWithinTx))), + ).toBe(true); + }); + + it('handles overlapping ranges across multiple tags', async () => { + const tagA = new SiloedTag(new Fr(101)); + const tagB = new SiloedTag(new Fr(102)); + const ckpts = await buildChainedCheckpointsWithLogs(3, { + numTxsPerBlock: 1, + privateLogs: { numLogsPerTx: 2 }, }); - }); - }); - - // Note that a lot of tests here are basically duplicates of the ones in getPrivateLogsByTags but the types used - // by each of the endpoints are different and that makes improving code reuse here not straightforward. - - describe('getPublicLogsByTagsFromContract', () => { - const numBlocksForLogs = 3; - const numTxsPerBlock = 4; - const numPublicLogsPerTx = 2; - const contractAddress = AztecAddress.fromNumber(543254); - - let logsCheckpoints: PublishedCheckpoint[]; - - beforeEach(async () => { - // Create checkpoints sequentially to chain archive roots - logsCheckpoints = []; - for (let i = 0; i < numBlocksForLogs; i++) { - const previousArchive = i > 0 ? logsCheckpoints[i - 1].checkpoint.blocks[0].archive : undefined; - logsCheckpoints.push( - await makeCheckpointWithLogs(i + 1, { - previousArchive, - numTxsPerBlock, - publicLogs: { numLogsPerTx: numPublicLogsPerTx, contractAddress }, - }), - ); + for (const ckpt of ckpts) { + // log 0 → tagA, log 1 → tagB + ckpt.checkpoint.blocks[0].body.txEffects[0].privateLogs[0].fields[0] = tagA.value; + ckpt.checkpoint.blocks[0].body.txEffects[0].privateLogs[1].fields[0] = tagB.value; } - - await blockStore.addCheckpoints(logsCheckpoints); - await logStore.addLogs(logsCheckpoints.flatMap(p => p.checkpoint.blocks)); - }); - - it('is possible to batch request public logs via tags', async () => { - const tags = [makePublicLogTag(2, 1, 1), makePublicLogTag(1, 2, 0)]; - - const logsByTags = await logStore.getPublicLogsByTagsFromContract(contractAddress, tags); - - expect(logsByTags).toEqual([ - [ - expect.objectContaining({ - blockNumber: 2, - logData: makePublicLog(tags[0]).getEmittedFields(), - }), - ], - [ - expect.objectContaining({ - blockNumber: 1, - logData: makePublicLog(tags[1]).getEmittedFields(), - }), - ], - ]); - }); - - it('is possible to batch request logs that have the same tag but different content', async () => { - const tags = [makePublicLogTag(1, 2, 1)]; - - // Create a checkpoint containing logs that have the same tag as the checkpoints before. - // Chain from the last checkpoint's archive - const newBlockNumber = numBlocksForLogs + 1; - const previousArchive = logsCheckpoints[logsCheckpoints.length - 1].checkpoint.blocks[0].archive; - const newCheckpoint = await makeCheckpointWithLogs(newBlockNumber, { - previousArchive, - numTxsPerBlock, - publicLogs: { numLogsPerTx: numPublicLogsPerTx, contractAddress }, + const blocks = ckpts.map(c => c.checkpoint.blocks[0]); + await blockStore.addCheckpoints(ckpts); + await logStore.addLogs(blocks); + + const result = await logStore.getPrivateLogsByTags({ tags: [tagA, tagB] }); + expect(result[0].length).toBe(3); + expect(result[1].length).toBe(3); + // Tag A logs all share tagA's tag in field 0; same for tag B. + expect(result[0].every(l => l.logData[0].equals(tagA.value))).toBe(true); + expect(result[1].every(l => l.logData[0].equals(tagB.value))).toBe(true); + }); + + it('filters by txHash', async () => { + const tag = new SiloedTag(new Fr(0x5555)); + const ckpt = await makeCheckpointWithLogs(1, { numTxsPerBlock: 3, privateLogs: { numLogsPerTx: 1 } }); + for (const tx of ckpt.checkpoint.blocks[0].body.txEffects) { + tx.privateLogs[0].fields[0] = tag.value; + } + await blockStore.addProposedBlock(ckpt.checkpoint.blocks[0]); + await logStore.addLogs([ckpt.checkpoint.blocks[0]]); + + const target = ckpt.checkpoint.blocks[0].body.txEffects[1].txHash; + const [res] = await logStore.getPrivateLogsByTags({ tags: [tag], txHash: target }); + expect(res.length).toBe(1); + expect(res[0].txHash.equals(target)).toBe(true); + }); + + it('rejects txHash combined with fromBlock/toBlock', () => { + // Validation throws synchronously before returning the transaction promise, + // so use the sync `.toThrow` matcher rather than `.rejects.toThrow`. + const tag = SiloedTag.random(); + expect(() => + logStore.getPrivateLogsByTags({ + tags: [tag], + txHash: { equals: () => false } as any, + fromBlock: BlockNumber(1), + }), + ).toThrow(/mutually exclusive/i); + }); + + it('paginates within a single tx via txHash + afterLog', async () => { + const tag = new SiloedTag(new Fr(0x7777)); + // One tx with MAX_LOGS_PER_TAG + 5 logs sharing the same tag; first page returns MAX_LOGS_PER_TAG, + // resume returns the remaining 5. + const ckpt = await makeCheckpointWithLogs(1, { + numTxsPerBlock: 1, + privateLogs: { numLogsPerTx: MAX_LOGS_PER_TAG + 5 }, }); - const newLog = newCheckpoint.checkpoint.blocks[0].body.txEffects[1].publicLogs[1]; - newLog.fields[0] = tags[0].value; - newCheckpoint.checkpoint.blocks[0].body.txEffects[1].publicLogs[1] = newLog; - await blockStore.addCheckpoints([newCheckpoint]); - await logStore.addLogs([newCheckpoint.checkpoint.blocks[0]]); - - const logsByTags = await logStore.getPublicLogsByTagsFromContract(contractAddress, tags); - - expect(logsByTags).toEqual([ - [ - expect.objectContaining({ - blockNumber: 1, - logData: makePublicLog(tags[0]).getEmittedFields(), - }), - expect.objectContaining({ - blockNumber: newBlockNumber, - logData: newLog.getEmittedFields(), - }), - ], - ]); - }); - - it('throws on out-of-order public log insertion', async () => { - const sharedTag = makePublicLogTag(99, 0, 0); - - // Create blocks 4 and 5 with the same shared tag - const prevArchive1 = logsCheckpoints[logsCheckpoints.length - 1].checkpoint.blocks[0].archive; - const checkpoint4 = await makeCheckpointWithLogs(numBlocksForLogs + 1, { - previousArchive: prevArchive1, - numTxsPerBlock, - publicLogs: { numLogsPerTx: numPublicLogsPerTx, contractAddress }, + const txEffect = ckpt.checkpoint.blocks[0].body.txEffects[0]; + for (const log of txEffect.privateLogs) { + log.fields[0] = tag.value; + } + await blockStore.addProposedBlock(ckpt.checkpoint.blocks[0]); + await logStore.addLogs([ckpt.checkpoint.blocks[0]]); + + const target = txEffect.txHash; + const [page1] = await logStore.getPrivateLogsByTags({ tags: [tag], txHash: target }); + expect(page1.length).toBe(MAX_LOGS_PER_TAG); + const cursor = LogCursor.fromLog(page1[page1.length - 1]); + const [page2] = await logStore.getPrivateLogsByTags({ + tags: [{ tag, afterLog: cursor }], + txHash: target, }); - checkpoint4.checkpoint.blocks[0].body.txEffects[0].publicLogs[0] = makePublicLog(sharedTag, contractAddress); + expect(page2.length).toBe(5); + // All from the same tx + expect(page1.every(l => l.txHash.equals(target))).toBe(true); + expect(page2.every(l => l.txHash.equals(target))).toBe(true); + }); - const prevArchive2 = checkpoint4.checkpoint.blocks[0].archive; - const checkpoint5 = await makeCheckpointWithLogs(numBlocksForLogs + 2, { - previousArchive: prevArchive2, - numTxsPerBlock, - publicLogs: { numLogsPerTx: numPublicLogsPerTx, contractAddress }, + it('referenceBlock caps a txHash query past the anchor', async () => { + const tag = new SiloedTag(new Fr(0x9999)); + // Block 3 has the tx with the tag. + const ckpts = await buildChainedCheckpointsWithLogs(3, { + numTxsPerBlock: 1, + privateLogs: { numLogsPerTx: 1 }, }); - checkpoint5.checkpoint.blocks[0].body.txEffects[0].publicLogs[0] = makePublicLog(sharedTag, contractAddress); + for (const ckpt of ckpts) { + ckpt.checkpoint.blocks[0].body.txEffects[0].privateLogs[0].fields[0] = tag.value; + } + await blockStore.addCheckpoints(ckpts); + await logStore.addLogs(ckpts.map(c => c.checkpoint.blocks[0])); - // Store block 5's logs first (higher block number), then try to store block 4's logs - // (lower block number) — this should fail. - await logStore.addLogs([checkpoint5.checkpoint.blocks[0]]); - await expect(logStore.addLogs([checkpoint4.checkpoint.blocks[0]])).rejects.toThrow(OutOfOrderLogInsertionError); - }); + const blockTwoHash = await ckpts[1].checkpoint.blocks[0].hash(); + const targetTx = ckpts[2].checkpoint.blocks[0].body.txEffects[0].txHash; - it('is possible to request logs for non-existing tags and determine their position', async () => { - const tags = [makePublicLogTag(99, 88, 77), makePublicLogTag(1, 1, 0)]; - - const logsByTags = await logStore.getPublicLogsByTagsFromContract(contractAddress, tags); - - expect(logsByTags).toEqual([ - [ - // No logs for the first tag. - ], - [ - expect.objectContaining({ - blockNumber: 1, - logData: makePublicLog(tags[1]).getEmittedFields(), - }), - ], - ]); + // referenceBlock = block 2; tx is in block 3 → past anchor → empty. + const [res] = await logStore.getPrivateLogsByTags({ + tags: [tag], + txHash: targetTx, + referenceBlock: blockTwoHash, + }); + expect(res).toEqual([]); }); - it('filters logs up to specified block number', async () => { - const sharedTag = makePublicLogTag(1, 2, 1); - - // Add extra blocks with logs sharing the same tag - for (let blockNum = numBlocksForLogs + 1; blockNum <= numBlocksForLogs + 2; blockNum++) { - const previousArchive = logsCheckpoints[logsCheckpoints.length - 1].checkpoint.blocks[0].archive; - const newCheckpoint = await makeCheckpointWithLogs(blockNum, { - previousArchive, - numTxsPerBlock, - publicLogs: { numLogsPerTx: numPublicLogsPerTx, contractAddress }, - }); - const newLog = newCheckpoint.checkpoint.blocks[0].body.txEffects[1].publicLogs[1]; - newLog.fields[0] = sharedTag.value; - newCheckpoint.checkpoint.blocks[0].body.txEffects[1].publicLogs[1] = newLog; - await blockStore.addCheckpoints([newCheckpoint]); - await logStore.addLogs([newCheckpoint.checkpoint.blocks[0]]); - logsCheckpoints.push(newCheckpoint); - } - - // Without filter, should return logs from block 1 and the extra blocks - const allLogs = await logStore.getPublicLogsByTagsFromContract(contractAddress, [sharedTag]); - expect(allLogs[0].some(log => log.blockNumber > numBlocksForLogs)).toBe(true); - - // With upToBlockNumber=numBlocksForLogs, should only return the original log from block 1 - const filteredLogs = await logStore.getPublicLogsByTagsFromContract( - contractAddress, - [sharedTag], - 0, - BlockNumber(numBlocksForLogs), + it('throws when referenceBlock points to a missing block', async () => { + const tag = SiloedTag.random(); + const missing = BlockHash.random(); + await expect(logStore.getPrivateLogsByTags({ tags: [tag], referenceBlock: missing })).rejects.toThrow( + /not found/i, ); - expect(filteredLogs[0].length).toBeGreaterThan(0); - for (const log of filteredLogs[0]) { - expect(log.blockNumber).toBeLessThanOrEqual(numBlocksForLogs); - } - expect(filteredLogs[0].length).toBeLessThan(allLogs[0].length); }); - it('returns all logs when upToBlockNumber is not set', async () => { - const tag = makePublicLogTag(1, 2, 1); + it('attaches noteHashes + nullifiers only when includeEffects is set', async () => { + const tag = new SiloedTag(new Fr(0xeeee)); + const ckpt = await makeCheckpointWithLogs(1, { numTxsPerBlock: 1, privateLogs: { numLogsPerTx: 1 } }); + ckpt.checkpoint.blocks[0].body.txEffects[0].privateLogs[0].fields[0] = tag.value; + await blockStore.addProposedBlock(ckpt.checkpoint.blocks[0]); + await logStore.addLogs([ckpt.checkpoint.blocks[0]]); + + const [withoutEffects] = await logStore.getPrivateLogsByTags({ tags: [tag] }); + expect(withoutEffects[0].noteHashes).toBeUndefined(); + expect(withoutEffects[0].nullifiers).toBeUndefined(); + + const [withEffects] = await logStore.getPrivateLogsByTags({ tags: [tag], includeEffects: true }); + expect(withEffects[0].noteHashes).toBeDefined(); + expect(withEffects[0].nullifiers).toBeDefined(); + // All nullifiers, not just the first. + const txEffect = ckpt.checkpoint.blocks[0].body.txEffects[0]; + expect(withEffects[0].nullifiers!.length).toBe(txEffect.nullifiers.length); + }); + + it('batches effect lookups (one fetch per unique tx in the page)', async () => { + // Build a single block with 2 txs, each emitting 3 logs of the same tag → 6 logs, 2 unique txs. + const tag = new SiloedTag(new Fr(0xabcd)); + const ckpt = await makeCheckpointWithLogs(1, { numTxsPerBlock: 2, privateLogs: { numLogsPerTx: 3 } }); + for (const tx of ckpt.checkpoint.blocks[0].body.txEffects) { + for (const log of tx.privateLogs) { + log.fields[0] = tag.value; + } + } + await blockStore.addProposedBlock(ckpt.checkpoint.blocks[0]); + await logStore.addLogs([ckpt.checkpoint.blocks[0]]); - const logsWithoutFilter = await logStore.getPublicLogsByTagsFromContract(contractAddress, [tag]); - const logsWithUndefined = await logStore.getPublicLogsByTagsFromContract(contractAddress, [tag], 0, undefined); + const spy = jest.spyOn(blockStore, 'getNoteHashesAndNullifiers'); - expect(logsWithoutFilter).toEqual(logsWithUndefined); + const [res] = await logStore.getPrivateLogsByTags({ tags: [tag], includeEffects: true }); + expect(res.length).toBe(6); + // Exactly one batched call. + expect(spy).toHaveBeenCalledTimes(1); + // …with the 2 distinct txHashes. + const argTxHashes = spy.mock.calls[0][0]; + expect(argTxHashes.length).toBe(2); + spy.mockRestore(); }); + }); - describe('pagination', () => { - const paginationTag = makePublicLogTag(1, 2, 1); - - beforeEach(async () => { - // Add more blocks with the same tag to exceed MAX_LOGS_PER_TAG - for (let i = numBlocksForLogs; i < numBlocksForLogs + MAX_LOGS_PER_TAG + 5; i++) { - const previousArchive = logsCheckpoints[logsCheckpoints.length - 1].checkpoint.blocks[0].archive; - const newCheckpoint = await makeCheckpointWithLogs(i + 1, { - previousArchive, - numTxsPerBlock, - publicLogs: { numLogsPerTx: numPublicLogsPerTx, contractAddress }, - }); - const newLog = newCheckpoint.checkpoint.blocks[0].body.txEffects[1].publicLogs[1]; - newLog.fields[0] = paginationTag.value; - newCheckpoint.checkpoint.blocks[0].body.txEffects[1].publicLogs[1] = newLog; - await blockStore.addCheckpoints([newCheckpoint]); - await logStore.addLogs([newCheckpoint.checkpoint.blocks[0]]); - logsCheckpoints.push(newCheckpoint); - } + describe('getPublicLogsByTags', () => { + it('requires contractAddress and filters on it', async () => { + const tag = new Tag(new Fr(0xf00d)); + const ckpt = await makeCheckpointWithLogs(1, { + numTxsPerBlock: 1, + publicLogs: { numLogsPerTx: 1, contractAddress: CONTRACT }, }); + ckpt.checkpoint.blocks[0].body.txEffects[0].publicLogs[0].fields[0] = tag.value; + await blockStore.addProposedBlock(ckpt.checkpoint.blocks[0]); + await logStore.addLogs([ckpt.checkpoint.blocks[0]]); - it('pagination works correctly with upToBlockNumber', async () => { - const filteredPage0 = await logStore.getPublicLogsByTagsFromContract( - contractAddress, - [paginationTag], - 0, - BlockNumber(5), - ); - for (const log of filteredPage0[0]) { - expect(log.blockNumber).toBeLessThanOrEqual(5); - } + // Same tag, different contract → no hits. + const otherContract = AztecAddress.fromNumber(99); + const [missing] = await logStore.getPublicLogsByTags({ contractAddress: otherContract, tags: [tag] }); + expect(missing).toEqual([]); - const filteredPage1 = await logStore.getPublicLogsByTagsFromContract( - contractAddress, - [paginationTag], - 1, - BlockNumber(5), - ); - for (const log of filteredPage1[0]) { - expect(log.blockNumber).toBeLessThanOrEqual(5); - } - }); - - it('returns first page of logs when page=0', async () => { - const logsByTags = await logStore.getPublicLogsByTagsFromContract(contractAddress, [paginationTag], 0); + const [found] = await logStore.getPublicLogsByTags({ contractAddress: CONTRACT, tags: [tag] }); + expect(found.length).toBe(1); + expect(found[0].logData[0].equals(tag.value)).toBe(true); + }); - expect(logsByTags[0]).toHaveLength(MAX_LOGS_PER_TAG); - expect(logsByTags[0][0].blockNumber).toBe(1); // First log from block 1 + it('handles range filtering', async () => { + const tag = new Tag(new Fr(0xfeed)); + const ckpts = await buildChainedCheckpointsWithLogs(5, { + numTxsPerBlock: 1, + publicLogs: { numLogsPerTx: 1, contractAddress: CONTRACT }, }); - - it('returns second page of logs when page=1', async () => { - const logsByTags = await logStore.getPublicLogsByTagsFromContract(contractAddress, [paginationTag], 1); - - // Should have the remaining logs (total was MAX_LOGS_PER_TAG + 6, so page 1 has 6) - expect(logsByTags[0]).toHaveLength(6); + for (const ckpt of ckpts) { + ckpt.checkpoint.blocks[0].body.txEffects[0].publicLogs[0].fields[0] = tag.value; + } + const blocks = ckpts.map(c => c.checkpoint.blocks[0]); + await blockStore.addCheckpoints(ckpts); + await logStore.addLogs(blocks); + + const [res] = await logStore.getPublicLogsByTags({ + contractAddress: CONTRACT, + tags: [tag], + fromBlock: BlockNumber(2), + toBlock: BlockNumber(5), }); + expect(res.map(l => l.blockNumber)).toEqual([BlockNumber(2), BlockNumber(3), BlockNumber(4)]); + }); + + it('rejects txHash combined with fromBlock/toBlock', () => { + // Validation throws synchronously before returning the transaction promise, + // so use the sync `.toThrow` matcher rather than `.rejects.toThrow`. + const tag = Tag.random(); + expect(() => + logStore.getPublicLogsByTags({ + contractAddress: CONTRACT, + tags: [tag], + txHash: { equals: () => false } as any, + toBlock: BlockNumber(5), + }), + ).toThrow(/mutually exclusive/i); + }); + }); - it('returns empty array when page is beyond available logs', async () => { - const logsByTags = await logStore.getPublicLogsByTagsFromContract(contractAddress, [paginationTag], 100); - - expect(logsByTags).toEqual([[]]); + describe('genesis referenceBlock handling', () => { + // During early sync the PXE anchors to the genesis block and passes its hash as the query + // referenceBlock. The genesis block is synthetic and never indexed in the block store, so without the + // wired genesis hash the reorg check throws "not found" — the regression that broke e2e on this branch. + // The store is now always constructed with the genesis hash (see beforeEach), so it resolves a genesis + // anchor to the genesis block number instead. + it('resolves a genesis anchor to an empty result (no reorg) instead of treating it as a missing block', async () => { + // Genesis is a valid anchor with no logs at or before it, so both query kinds return empty without + // throwing — the upper bound collapses to the genesis block, which precedes the first indexable block. + const [privateResult] = await logStore.getPrivateLogsByTags({ + tags: [SiloedTag.random()], + referenceBlock: GENESIS_BLOCK_HEADER_HASH, }); + expect(privateResult).toEqual([]); - /** - * Verifies that logs are stored and returned in block order (ascending). - * This ordering guarantee is critical for pagination safety: if logs were not ordered - * by block number, new logs added between paginated calls could be inserted in the - * middle of the result set, causing callers to receive duplicate logs or miss logs - * entirely. By maintaining block order, new logs always appear at the end of the - * result set, meaning previously fetched pages remain stable. - */ - it('maintains stable pagination when new logs are added between page fetches', async () => { - // Fetch page 0 and record the block numbers - const page0Before = await logStore.getPublicLogsByTagsFromContract(contractAddress, [paginationTag], 0); - expect(page0Before[0]).toHaveLength(MAX_LOGS_PER_TAG); - const blockNumbersBefore = page0Before[0].map(log => log.blockNumber); - - // Verify block numbers are in ascending order - for (let i = 1; i < blockNumbersBefore.length; i++) { - expect(blockNumbersBefore[i]).toBeGreaterThanOrEqual(blockNumbersBefore[i - 1]); - } - - // Add more blocks with the same tag - const additionalBlocks = 3; - for (let i = 0; i < additionalBlocks; i++) { - const previousArchive = logsCheckpoints[logsCheckpoints.length - 1].checkpoint.blocks[0].archive; - const blockNumber = logsCheckpoints.length + 1; - const newCheckpoint = await makeCheckpointWithLogs(blockNumber, { - previousArchive, - numTxsPerBlock, - publicLogs: { numLogsPerTx: numPublicLogsPerTx, contractAddress }, - }); - const newLog = newCheckpoint.checkpoint.blocks[0].body.txEffects[1].publicLogs[1]; - newLog.fields[0] = paginationTag.value; - newCheckpoint.checkpoint.blocks[0].body.txEffects[1].publicLogs[1] = newLog; - await blockStore.addCheckpoints([newCheckpoint]); - await logStore.addLogs([newCheckpoint.checkpoint.blocks[0]]); - logsCheckpoints.push(newCheckpoint); - } - - // Fetch page 0 again - should return the exact same logs - const page0After = await logStore.getPublicLogsByTagsFromContract(contractAddress, [paginationTag], 0); - expect(page0After[0]).toHaveLength(MAX_LOGS_PER_TAG); - const blockNumbersAfter = page0After[0].map(log => log.blockNumber); - expect(blockNumbersAfter).toEqual(blockNumbersBefore); - - // Fetch page 1 - should include the newly added logs - const page1 = await logStore.getPublicLogsByTagsFromContract(contractAddress, [paginationTag], 1); - expect(page1[0].length).toBeGreaterThan(0); - - // Verify all logs across both pages are in ascending block order - const allBlockNumbers = [...blockNumbersAfter, ...page1[0].map(log => log.blockNumber)]; - for (let i = 1; i < allBlockNumbers.length; i++) { - expect(allBlockNumbers[i]).toBeGreaterThanOrEqual(allBlockNumbers[i - 1]); - } - - // The new logs should appear on page 1, meaning their block numbers - // should be greater than or equal to the max block number on page 0 - const maxBlockOnPage0 = Math.max(...blockNumbersAfter); - const newLogBlockNumbers = page1[0].slice(-additionalBlocks).map(log => log.blockNumber); - for (const blockNum of newLogBlockNumbers) { - expect(blockNum).toBeGreaterThanOrEqual(maxBlockOnPage0); - } + const [publicResult] = await logStore.getPublicLogsByTags({ + contractAddress: CONTRACT, + tags: [Tag.random()], + referenceBlock: GENESIS_BLOCK_HEADER_HASH, }); + expect(publicResult).toEqual([]); }); }); - describe('getPublicLogs', () => { - const numBlocksForPublicLogs = 10; - - // Helper to get total public logs per tx from a block - const getPublicLogsPerTx = (block: L2Block, txIndex: number) => block.body.txEffects[txIndex].publicLogs.length; - - // Helper to get number of txs in a block - const getTxsPerBlock = (block: L2Block) => block.body.txEffects.length; - - beforeEach(async () => { - // Use the outer publishedCheckpoints for log tests - for (let i = 0; i < numBlocksForPublicLogs; i++) { - await blockStore.addCheckpoints([publishedCheckpoints[i]]); - await logStore.addLogs(publishedCheckpoints[i].checkpoint.blocks); - } - }); - - it('no logs returned if deleted ("txHash" filter param is respected variant)', async () => { - // get random tx - const targetBlockIndex = randomInt(numBlocksForPublicLogs); - const targetBlock = publishedCheckpoints[targetBlockIndex].checkpoint.blocks[0]; - const targetTxIndex = randomInt(getTxsPerBlock(targetBlock)); - const targetTxHash = targetBlock.body.txEffects[targetTxIndex].txHash; - - await Promise.all([ - blockStore.removeCheckpointsAfter(CheckpointNumber(0)), - logStore.deleteLogs(publishedCheckpoints.slice(0, numBlocksForPublicLogs).flatMap(b => b.checkpoint.blocks)), - ]); - - const response = await logStore.getPublicLogs({ txHash: targetTxHash }); - const logs = response.logs; - - expect(response.maxLogsHit).toBeFalsy(); - expect(logs.length).toEqual(0); - }); - - it('"txHash" filter param is respected', async () => { - // get random tx - const targetBlockIndex = randomInt(numBlocksForPublicLogs); - const targetBlock = publishedCheckpoints[targetBlockIndex].checkpoint.blocks[0]; - const targetTxIndex = randomInt(getTxsPerBlock(targetBlock)); - const targetTxHash = targetBlock.body.txEffects[targetTxIndex].txHash; - - const response = await logStore.getPublicLogs({ txHash: targetTxHash }); - const logs = response.logs; - - expect(response.maxLogsHit).toBeFalsy(); - - const expectedNumLogs = getPublicLogsPerTx(targetBlock, targetTxIndex); - expect(logs.length).toEqual(expectedNumLogs); - - const targeBlockNumber = targetBlockIndex + INITIAL_L2_BLOCK_NUM; - for (const log of logs) { - expect(log.id.blockNumber).toEqual(targeBlockNumber); - expect(log.id.txIndex).toEqual(targetTxIndex); - } - }); - - it('returns block hash on public log ids', async () => { - const targetBlock = publishedCheckpoints[0].checkpoint.blocks[0]; - const expectedBlockHash = await targetBlock.header.hash(); - - const logs = (await logStore.getPublicLogs({ fromBlock: targetBlock.number, toBlock: targetBlock.number + 1 })) - .logs; - - expect(logs.length).toBeGreaterThan(0); - expect(logs.every(log => log.id.blockHash.equals(expectedBlockHash))).toBe(true); - }); - - it('returns tx hash on public log ids', async () => { - const targetBlock = publishedCheckpoints[0].checkpoint.blocks[0]; - - const logs = (await logStore.getPublicLogs({ fromBlock: targetBlock.number, toBlock: targetBlock.number + 1 })) - .logs; + describe('getPrivateLogsForBlock / getPublicLogsForBlock', () => { + it('returns all private and public logs indexed for a block in the same order they were written', async () => { + // 2 txs * (2 private + 2 public) per block; the secondary index records every write for this block. + const ckpt = await makeCheckpointWithLogs(1, { + numTxsPerBlock: 2, + privateLogs: { numLogsPerTx: 2 }, + publicLogs: { numLogsPerTx: 2, contractAddress: CONTRACT }, + }); + const block = ckpt.checkpoint.blocks[0]; + await blockStore.addProposedBlock(block); + await logStore.addLogs([block]); - expect(logs.length).toBeGreaterThan(0); - const expectedTxHashes = targetBlock.body.txEffects.map(txEffect => txEffect.txHash); - for (const log of logs) { - const expectedTxHash = expectedTxHashes[log.id.txIndex]; - expect(log.id.txHash.equals(expectedTxHash)).toBe(true); + const priv = await logStore.getPrivateLogsForBlock(block.number); + const pub = await logStore.getPublicLogsForBlock(block.number); + expect(priv.length).toBe(2 * 2); + expect(pub.length).toBe(2 * 2); + // Every returned log carries this block's number, with non-decreasing (txIndex, logIndex). + for (const arr of [priv, pub]) { + expect(arr.every(l => l.blockNumber === block.number)).toBe(true); + for (let i = 1; i < arr.length; i++) { + const prev = arr[i - 1]; + const cur = arr[i]; + expect( + cur.txIndexWithinBlock > prev.txIndexWithinBlock || + (cur.txIndexWithinBlock === prev.txIndexWithinBlock && cur.logIndexWithinTx >= prev.logIndexWithinTx), + ).toBe(true); + } } }); - it('"fromBlock" and "toBlock" filter params are respected', async () => { - // Set "fromBlock" and "toBlock" - const fromBlock = 3; - const toBlock = 7; + it('assigns logIndexWithinTx independently for private and public logs (each kind starts at 0)', async () => { + // A tx with 2 private + 2 public logs: private logs should be indexed 0,1 and public logs + // should also be indexed 0,1, not 2,3 (which would be the old combined-counter behavior). + const ckpt = await makeCheckpointWithLogs(1, { + numTxsPerBlock: 1, + privateLogs: { numLogsPerTx: 2 }, + publicLogs: { numLogsPerTx: 2, contractAddress: CONTRACT }, + }); + const block = ckpt.checkpoint.blocks[0]; + await blockStore.addProposedBlock(block); + await logStore.addLogs([block]); - const response = await logStore.getPublicLogs({ fromBlock, toBlock }); - const logs = response.logs; + const priv = await logStore.getPrivateLogsForBlock(block.number); + const pub = await logStore.getPublicLogsForBlock(block.number); - expect(response.maxLogsHit).toBeFalsy(); + expect(priv.length).toBe(2); + expect(pub.length).toBe(2); - // Compute expected logs from the blocks in range - let expectedNumLogs = 0; - for (let i = fromBlock - 1; i < toBlock - 1; i++) { - const block = publishedCheckpoints[i].checkpoint.blocks[0]; - expectedNumLogs += block.body.txEffects.reduce((sum, tx) => sum + tx.publicLogs.length, 0); - } - expect(logs.length).toEqual(expectedNumLogs); + // Private log indices: 0, 1. + expect(priv[0].logIndexWithinTx).toBe(0); + expect(priv[1].logIndexWithinTx).toBe(1); - for (const log of logs) { - const blockNumber = log.id.blockNumber; - expect(blockNumber).toBeGreaterThanOrEqual(fromBlock); - expect(blockNumber).toBeLessThan(toBlock); - } + // Public log indices: 0, 1 (not 2, 3). + expect(pub[0].logIndexWithinTx).toBe(0); + expect(pub[1].logIndexWithinTx).toBe(1); }); - it('"contractAddress" filter param is respected', async () => { - // Get a random contract address from the logs - const targetBlockIndex = randomInt(numBlocksForPublicLogs); - const targetBlock = publishedCheckpoints[targetBlockIndex].checkpoint.blocks[0]; - const targetTxIndex = randomInt(getTxsPerBlock(targetBlock)); - const targetLogIndex = randomInt(getPublicLogsPerTx(targetBlock, targetTxIndex)); - const targetContractAddress = - targetBlock.body.txEffects[targetTxIndex].publicLogs[targetLogIndex].contractAddress; - - const response = await logStore.getPublicLogs({ contractAddress: targetContractAddress }); - - expect(response.maxLogsHit).toBeFalsy(); - - for (const extendedLog of response.logs) { - expect(extendedLog.log.contractAddress.equals(targetContractAddress)).toBeTruthy(); - } + it('returns empty arrays for blocks with no indexed logs', async () => { + expect(await logStore.getPrivateLogsForBlock(BlockNumber(99))).toEqual([]); + expect(await logStore.getPublicLogsForBlock(BlockNumber(99))).toEqual([]); }); + }); - it('"tag" filter param is respected', async () => { - // Get a random tag from the logs - const targetBlockIndex = randomInt(numBlocksForPublicLogs); - const targetBlock = publishedCheckpoints[targetBlockIndex].checkpoint.blocks[0]; - const targetTxIndex = randomInt(getTxsPerBlock(targetBlock)); - const targetLogIndex = randomInt(getPublicLogsPerTx(targetBlock, targetTxIndex)); - const targetTag = targetBlock.body.txEffects[targetTxIndex].publicLogs[targetLogIndex].fields[0]; - - const response = await logStore.getPublicLogs({ tag: targetTag }); - - expect(response.maxLogsHit).toBeFalsy(); - expect(response.logs.length).toBeGreaterThan(0); - - for (const extendedLog of response.logs) { - expect(extendedLog.log.fields[0].equals(targetTag)).toBeTruthy(); - } - }); - - it('"afterLog" filter param is respected', async () => { - // Get a random log as reference - const targetBlockIndex = randomInt(numBlocksForPublicLogs); - const targetBlock = publishedCheckpoints[targetBlockIndex].checkpoint.blocks[0]; - const targetTxIndex = randomInt(getTxsPerBlock(targetBlock)); - const numLogsInTx = targetBlock.body.txEffects[targetTxIndex].publicLogs.length; - const targetLogIndex = numLogsInTx > 0 ? randomInt(numLogsInTx) : 0; - const targetBlockHash = await targetBlock.header.hash(); - - const targetTxHash = targetBlock.body.txEffects[targetTxIndex].txHash; - const afterLog = new LogId( - BlockNumber(targetBlockIndex + INITIAL_L2_BLOCK_NUM), - targetBlockHash, - targetTxHash, - targetTxIndex, - targetLogIndex, - ); - - const response = await logStore.getPublicLogs({ afterLog }); - const logs = response.logs; - - expect(response.maxLogsHit).toBeFalsy(); - - for (const log of logs) { - const logId = log.id; - expect(logId.blockNumber).toBeGreaterThanOrEqual(afterLog.blockNumber); - if (logId.blockNumber === afterLog.blockNumber) { - expect(logId.txIndex).toBeGreaterThanOrEqual(afterLog.txIndex); - if (logId.txIndex === afterLog.txIndex) { - expect(logId.logIndex).toBeGreaterThan(afterLog.logIndex); + describe('queryAllPrivate/PublicLogsByTags helpers', () => { + /** + * Stamps a single shared `tag` on every private log across `count` chained checkpoints. Used as the + * fixture for the pagination drivers. + */ + async function stampPrivateTagAcrossChain(count: number, numLogsPerTx: number, tag: SiloedTag) { + const ckpts = await buildChainedCheckpointsWithLogs(count, { + numTxsPerBlock: 2, + privateLogs: { numLogsPerTx }, + }); + for (const ckpt of ckpts) { + for (const tx of ckpt.checkpoint.blocks[0].body.txEffects) { + for (const log of tx.privateLogs) { + (log.fields as Fr[])[0] = tag.value; } } } - }); - - it('"txHash" filter param is respected when "afterLog" is set', async () => { - // A random txHash should match nothing, even with afterLog set - const txHash = TxHash.random(); - const afterLog = new LogId(BlockNumber(1), BlockHash.random(), TxHash.random(), 0, 0); - - const response = await logStore.getPublicLogs({ txHash, afterLog }); - expect(response.logs.length).toBe(0); - }); - - it('intersecting works', async () => { - let logs = (await logStore.getPublicLogs({ fromBlock: -10 as BlockNumber, toBlock: -5 as BlockNumber })).logs; - expect(logs.length).toBe(0); - - // "fromBlock" gets correctly trimmed to range and "toBlock" is exclusive - logs = (await logStore.getPublicLogs({ fromBlock: -10 as BlockNumber, toBlock: BlockNumber(5) })).logs; - let blockNumbers = new Set(logs.map(log => log.id.blockNumber)); - expect(blockNumbers).toEqual(new Set([1, 2, 3, 4])); - - // "toBlock" should be exclusive - logs = (await logStore.getPublicLogs({ fromBlock: BlockNumber(1), toBlock: BlockNumber(1) })).logs; - expect(logs.length).toBe(0); + const blocks = ckpts.map(c => c.checkpoint.blocks[0]); + await blockStore.addCheckpoints(ckpts); + await logStore.addLogs(blocks); + return blocks; + } - logs = (await logStore.getPublicLogs({ fromBlock: BlockNumber(10), toBlock: BlockNumber(5) })).logs; - expect(logs.length).toBe(0); + it('paginates per-tag using afterLog and drops exhausted tags between rounds (private)', async () => { + const tagA = new SiloedTag(new Fr(0xa1)); + const tagB = new SiloedTag(new Fr(0xb2)); + // 4 blocks * 2 txs * 1 log = 8 logs share tagA (over 3 rounds with limitPerTag=3). + // tagB matches nothing. + await stampPrivateTagAcrossChain(4, 1, tagA); + + const calls: number[] = []; + const node = { + getPrivateLogsByTags: jest.fn((q: Parameters[0]) => { + calls.push(q.tags.length); + return logStore.getPrivateLogsByTags(q); + }), + }; + const result = await queryAllPrivateLogsByTags(node, { + tags: [tagA, tagB], + limitPerTag: 3, + }); - // both "fromBlock" and "toBlock" get correctly capped to range and logs from all blocks are returned - logs = (await logStore.getPublicLogs({ fromBlock: -100 as BlockNumber, toBlock: +100 })).logs; - blockNumbers = new Set(logs.map(log => log.id.blockNumber)); - expect(blockNumbers.size).toBe(numBlocksForPublicLogs); + expect(result[0].length).toBe(8); + expect(result[1].length).toBe(0); - // intersecting with "afterLog" works - logs = ( - await logStore.getPublicLogs({ - fromBlock: BlockNumber(2), - toBlock: BlockNumber(5), - afterLog: new LogId(BlockNumber(4), BlockHash.random(), TxHash.random(), 0, 0), - }) - ).logs; - blockNumbers = new Set(logs.map(log => log.id.blockNumber)); - expect(blockNumbers).toEqual(new Set([4])); - - logs = ( - await logStore.getPublicLogs({ - toBlock: BlockNumber(5), - afterLog: new LogId(BlockNumber(5), BlockHash.random(), TxHash.random(), 1, 0), - }) - ).logs; - expect(logs.length).toBe(0); - - logs = ( - await logStore.getPublicLogs({ - fromBlock: BlockNumber(2), - toBlock: BlockNumber(5), - afterLog: new LogId(BlockNumber(100), BlockHash.random(), TxHash.random(), 0, 0), - }) - ).logs; - expect(logs.length).toBe(0); + // tagA needs 3 rounds (3 + 3 + 2). tagB returns 0 on round 1 and drops out. Round 1 queries both + // tags (length 2), rounds 2 and 3 only re-query tagA (length 1 each). + expect(calls).toEqual([2, 1, 1]); + expect(node.getPrivateLogsByTags).toHaveBeenCalledTimes(3); }); - it('"txIndex" and "logIndex" are respected when "afterLog.blockNumber" is equal to "fromBlock"', async () => { - // Get a random log as reference - const targetBlockIndex = randomInt(numBlocksForPublicLogs); - const targetBlock = publishedCheckpoints[targetBlockIndex].checkpoint.blocks[0]; - const targetTxIndex = randomInt(getTxsPerBlock(targetBlock)); - const numLogsInTx = targetBlock.body.txEffects[targetTxIndex].publicLogs.length; - const targetLogIndex = numLogsInTx > 0 ? randomInt(numLogsInTx) : 0; - const targetBlockHash = await targetBlock.header.hash(); - const targetTxHash = targetBlock.body.txEffects[targetTxIndex].txHash; - - const afterLog = new LogId( - BlockNumber(targetBlockIndex + INITIAL_L2_BLOCK_NUM), - targetBlockHash, - targetTxHash, - targetTxIndex, - targetLogIndex, - ); - - const response = await logStore.getPublicLogs({ afterLog, fromBlock: afterLog.blockNumber }); - const logs = response.logs; + it('preserves input tag order under pagination (public)', async () => { + // Three distinct public tags on the same contract, only the second one gets a full page so it + // forces an extra round; the helper must still return results indexed in input tag order. + const tagX = new Tag(new Fr(0x11)); + const tagY = new Tag(new Fr(0x22)); + const tagZ = new Tag(new Fr(0x33)); - expect(response.maxLogsHit).toBeFalsy(); - - for (const log of logs) { - const logId = log.id; - expect(logId.blockNumber).toBeGreaterThanOrEqual(afterLog.blockNumber); - if (logId.blockNumber === afterLog.blockNumber) { - expect(logId.txIndex).toBeGreaterThanOrEqual(afterLog.txIndex); - if (logId.txIndex === afterLog.txIndex) { - expect(logId.logIndex).toBeGreaterThan(afterLog.logIndex); - } + const ckpts = await buildChainedCheckpointsWithLogs(3, { + numTxsPerBlock: 2, + publicLogs: { numLogsPerTx: 1, contractAddress: CONTRACT }, + }); + // Round-robin tags: tagX in block 1, tagY in blocks 2-3 (6 logs total), tagZ on one tx of block 3. + for (const tx of ckpts[0].checkpoint.blocks[0].body.txEffects) { + tx.publicLogs[0].fields[0] = tagX.value; + } + for (const ckpt of ckpts.slice(1, 3)) { + for (const tx of ckpt.checkpoint.blocks[0].body.txEffects) { + tx.publicLogs[0].fields[0] = tagY.value; } } - }); - }); - - describe('getContractClassLogs', () => { - let targetBlock: L2Block; - let expectedContractClassLog: ContractClassLog; - - beforeEach(async () => { - await blockStore.addCheckpoints(publishedCheckpoints); - - targetBlock = publishedCheckpoints[0].checkpoint.blocks[0]; - expectedContractClassLog = await ContractClassLog.random(); - targetBlock.body.txEffects.forEach((txEffect, index) => { - txEffect.contractClassLogs = index === 0 ? [expectedContractClassLog] : []; + // Overwrite one of block 3's tagY entries with tagZ. + ckpts[2].checkpoint.blocks[0].body.txEffects[0].publicLogs[0].fields[0] = tagZ.value; + const blocks = ckpts.map(c => c.checkpoint.blocks[0]); + await blockStore.addCheckpoints(ckpts); + await logStore.addLogs(blocks); + + const result = await queryAllPublicLogsByTags(logStore, { + contractAddress: CONTRACT, + tags: [tagX, tagY, tagZ], + limitPerTag: 2, }); - await logStore.addLogs([targetBlock]); - }); - - it('returns block hash on contract class log ids', async () => { - const result = await logStore.getContractClassLogs({ - fromBlock: targetBlock.number, - toBlock: targetBlock.number + 1, - }); - - expect(result.maxLogsHit).toBeFalsy(); - expect(result.logs).toHaveLength(1); - - const [{ id, log }] = result.logs; - const expectedBlockHash = await targetBlock.header.hash(); - - expect(id.blockHash.equals(expectedBlockHash)).toBe(true); - expect(id.blockNumber).toEqual(targetBlock.number); - expect(log).toEqual(expectedContractClassLog); - }); - }); - - describe('idempotency', () => { - it('does not duplicate logs when addLogs is called twice with same block', async () => { - const block = await L2Block.random(BlockNumber(1), { - checkpointNumber: CheckpointNumber(1), - indexWithinCheckpoint: IndexWithinCheckpoint(0), - }); - - // Add logs first time - await logStore.addLogs([block]); - - // Get initial log count - const initialLogs = await logStore.getPublicLogs({ fromBlock: BlockNumber(1), toBlock: BlockNumber(2) }); - const initialCount = initialLogs.logs.length; - expect(initialCount).toBeGreaterThan(0); - - // Add logs second time (same block) - await logStore.addLogs([block]); - - // Verify logs are NOT duplicated - const finalLogs = await logStore.getPublicLogs({ fromBlock: BlockNumber(1), toBlock: BlockNumber(2) }); - expect(finalLogs.logs.length).toBe(initialCount); + // Input-order preserved: [tagX results, tagY results, tagZ results]. + expect(result.length).toBe(3); + expect(result[0].every(l => l.logData[0].equals(tagX.value))).toBe(true); + expect(result[1].every(l => l.logData[0].equals(tagY.value))).toBe(true); + expect(result[2].every(l => l.logData[0].equals(tagZ.value))).toBe(true); + // tagY had 3 logs across blocks 2-3 (one block 3 tagY entry got rewritten to tagZ). + expect(result[1].length).toBe(3); + expect(result[2].length).toBe(1); }); }); }); diff --git a/yarn-project/archiver/src/store/log_store.ts b/yarn-project/archiver/src/store/log_store.ts index debd3a873d38..49d4d719084a 100644 --- a/yarn-project/archiver/src/store/log_store.ts +++ b/yarn-project/archiver/src/store/log_store.ts @@ -1,733 +1,379 @@ import { INITIAL_L2_BLOCK_NUM } from '@aztec/constants'; -import { BlockNumber } from '@aztec/foundation/branded-types'; -import { compactArray, filterAsync } from '@aztec/foundation/collection'; +import { Fr } from '@aztec/foundation/curves/bn254'; import { createLogger } from '@aztec/foundation/log'; -import { BufferReader, numToUInt32BE } from '@aztec/foundation/serialize'; import type { AztecAsyncKVStore, AztecAsyncMap } from '@aztec/kv-store'; -import type { AztecAddress } from '@aztec/stdlib/aztec-address'; -import { BlockHash, L2Block } from '@aztec/stdlib/block'; +import type { BlockHash, L2Block } from '@aztec/stdlib/block'; import { MAX_LOGS_PER_TAG } from '@aztec/stdlib/interfaces/api-limit'; -import type { GetContractClassLogsResponse, GetPublicLogsResponse } from '@aztec/stdlib/interfaces/client'; -import { - ContractClassLog, - ExtendedContractClassLog, - ExtendedPublicLog, - type LogFilter, - LogId, - PublicLog, - type SiloedTag, +import type { + LogCursor, + LogResult, + PrivateLogsQuery, + PublicLogsQuery, + SiloedTag, Tag, - TxScopedL2Log, + TagQuery, } from '@aztec/stdlib/logs'; import { TxHash } from '@aztec/stdlib/tx'; -import { OutOfOrderLogInsertionError } from '../errors.js'; import type { BlockStore } from './block_store.js'; +import { + decodeKeyTail, + decodeValue, + encodeKey, + encodePublicPrefix, + encodeValue, + endOfTagRange, + endOfTxRange, + fieldHex, + incKey, +} from './log_store_codec.js'; /** - * A store for logs + * Indexes every emitted private and public log under a composite hex-string key + * `[contractAddress (public only)]-tag-blockNumber-txIndexWithinBlock-logIndexWithinTx`, + * where each numeric segment is zero-padded to 8 lowercase hex digits (4 bytes BE) and + * `contractAddress` / `tag` are the bare 64-hex-char field representations (no `0x` prefix). The + * fixed-width zero-padded hex segments sort lexicographically in the same order as the canonical + * `(contract, tag, blockNumber, txIndexWithinBlock, logIndexWithinTx)` tuple, so a single ordered + * range scan answers every {@link PrivateLogsQuery} / {@link PublicLogsQuery}. + * + * Per-block secondary indices (`#privateKeysByBlock`, `#publicKeysByBlock`) record the exact primary + * keys written for each block so {@link deleteLogs} can drop them on reorg without having to range + * scan by block (block isn't the leading key segment). + * + * Contract-class logs are no longer stored or served by the log store. */ export class LogStore { - // `tag` --> private logs - #privateLogsByTag: AztecAsyncMap; - // `{contractAddress}_${tag}` --> public logs - #publicLogsByContractAndTag: AztecAsyncMap; - #privateLogKeysByBlock: AztecAsyncMap; - #publicLogKeysByBlock: AztecAsyncMap; - #publicLogsByBlock: AztecAsyncMap; - #contractClassLogsByBlock: AztecAsyncMap; - #logsMaxPageSize: number; + /** Primary map: composite private key (tag + tail = 96 hex chars + separators) -> serialized {@link StoredLogValue}. */ + #privateLogs: AztecAsyncMap; + /** Primary map: composite public key (contract + tag + tail) -> serialized {@link StoredLogValue}. */ + #publicLogs: AztecAsyncMap; + + /** Secondary deletion index: blockNumber -> the exact primary keys written for that block. */ + #privateKeysByBlock: AztecAsyncMap; + #publicKeysByBlock: AztecAsyncMap; + #log = createLogger('archiver:log_store'); + /** + * @param genesisBlockHash - Hash of the synthetic genesis block. During early sync the PXE anchors to + * genesis and passes its hash as a query `referenceBlock`; since the archiver never indexes the + * genesis block, the store recognizes this hash directly and resolves it to the genesis block number + * rather than mistaking it for a reorg. + */ constructor( private db: AztecAsyncKVStore, private blockStore: BlockStore, - logsMaxPageSize: number = 1000, + private readonly genesisBlockHash: BlockHash, ) { - this.#privateLogsByTag = db.openMap('archiver_private_tagged_logs_by_tag'); - this.#publicLogsByContractAndTag = db.openMap('archiver_public_tagged_logs_by_tag'); - this.#privateLogKeysByBlock = db.openMap('archiver_private_log_keys_by_block'); - this.#publicLogKeysByBlock = db.openMap('archiver_public_log_keys_by_block'); - this.#publicLogsByBlock = db.openMap('archiver_public_logs_by_block'); - this.#contractClassLogsByBlock = db.openMap('archiver_contract_class_logs_by_block'); - - this.#logsMaxPageSize = logsMaxPageSize; + this.#privateLogs = db.openMap('archiver_private_logs'); + this.#publicLogs = db.openMap('archiver_public_logs'); + this.#privateKeysByBlock = db.openMap('archiver_private_log_keys_by_block'); + this.#publicKeysByBlock = db.openMap('archiver_public_log_keys_by_block'); } /** - * Extracts tagged logs from a single block, grouping them into private and public maps. + * Indexes every emitted private and public log from the given blocks. Wraps the write in a single + * `db.transactionAsync` so the primary entries and the per-block secondary indices stay consistent. * - * @param block - The L2 block to extract logs from. - * @returns An object containing the private and public tagged logs for the block. - */ - #extractTaggedLogsFromBlock(block: L2Block) { - // SiloedTag (as string) -> array of log buffers. - const privateTaggedLogs = new Map(); - // "{contractAddress}_{tag}" (as string) -> array of log buffers. - const publicTaggedLogs = new Map(); - - block.body.txEffects.forEach(txEffect => { - const txHash = txEffect.txHash; - - txEffect.privateLogs.forEach(log => { - // Private logs use SiloedTag (already siloed by kernel) - const tag = log.fields[0]; - this.#log.debug(`Found private log with tag ${tag.toString()} in block ${block.number}`); - - const currentLogs = privateTaggedLogs.get(tag.toString()) ?? []; - currentLogs.push( - new TxScopedL2Log( - txHash, - block.number, - block.timestamp, - log.getEmittedFields(), - txEffect.noteHashes, - txEffect.nullifiers[0], - ).toBuffer(), - ); - privateTaggedLogs.set(tag.toString(), currentLogs); - }); - - txEffect.publicLogs.forEach(log => { - // Public logs use Tag directly (not siloed) and are stored with contract address - const tag = log.fields[0]; - const contractAddress = log.contractAddress; - const key = `${contractAddress.toString()}_${tag.toString()}`; - this.#log.debug( - `Found public log with tag ${tag.toString()} from contract ${contractAddress.toString()} in block ${block.number}`, - ); - - const currentLogs = publicTaggedLogs.get(key) ?? []; - currentLogs.push( - new TxScopedL2Log( - txHash, - block.number, - block.timestamp, - log.getEmittedFields(), - txEffect.noteHashes, - txEffect.nullifiers[0], - ).toBuffer(), - ); - publicTaggedLogs.set(key, currentLogs); - }); - }); - - return { privateTaggedLogs, publicTaggedLogs }; - } - - /** - * Extracts and aggregates tagged logs from a list of blocks. - * @param blocks - The blocks to extract logs from. - * @returns A map from tag (as string) to an array of serialized private logs belonging to that tag, and a map from - * "{contractAddress}_{tag}" (as string) to an array of serialized public logs belonging to that key. + * A block is only ever added once; on reorg the archiver calls {@link deleteLogs} first, so we write + * the secondary index entries with a plain `set` (overwrite) rather than read-modify-append. */ - #extractTaggedLogs(blocks: L2Block[]): { - privateTaggedLogs: Map; - publicTaggedLogs: Map; - } { - const taggedLogsInBlocks = blocks.map(block => this.#extractTaggedLogsFromBlock(block)); - - // Now we merge the maps from each block into a single map. - const privateTaggedLogs = taggedLogsInBlocks.reduce((acc, { privateTaggedLogs }) => { - for (const [tag, logs] of privateTaggedLogs.entries()) { - const currentLogs = acc.get(tag) ?? []; - acc.set(tag, currentLogs.concat(logs)); - } - return acc; - }, new Map()); - - const publicTaggedLogs = taggedLogsInBlocks.reduce((acc, { publicTaggedLogs }) => { - for (const [key, logs] of publicTaggedLogs.entries()) { - const currentLogs = acc.get(key) ?? []; - acc.set(key, currentLogs.concat(logs)); - } - return acc; - }, new Map()); - - return { privateTaggedLogs, publicTaggedLogs }; - } - - async #addPrivateLogs(blocks: L2Block[]): Promise { - const newBlocks = await filterAsync( - blocks, - async block => !(await this.#privateLogKeysByBlock.hasAsync(block.number)), - ); - - const { privateTaggedLogs } = this.#extractTaggedLogs(newBlocks); - const keysOfPrivateLogsToUpdate = Array.from(privateTaggedLogs.keys()); - - const currentPrivateTaggedLogs = await Promise.all( - keysOfPrivateLogsToUpdate.map(async key => ({ - tag: key, - logBuffers: await this.#privateLogsByTag.getAsync(key), - })), - ); - - for (const taggedLogBuffer of currentPrivateTaggedLogs) { - if (taggedLogBuffer.logBuffers && taggedLogBuffer.logBuffers.length > 0) { - const newLogs = privateTaggedLogs.get(taggedLogBuffer.tag)!; - if (newLogs.length === 0) { - continue; - } - const lastExisting = TxScopedL2Log.fromBuffer(taggedLogBuffer.logBuffers.at(-1)!); - const firstNew = TxScopedL2Log.fromBuffer(newLogs[0]); - if (lastExisting.blockNumber > firstNew.blockNumber) { - throw new OutOfOrderLogInsertionError( - 'private', - taggedLogBuffer.tag, - lastExisting.blockNumber, - firstNew.blockNumber, - ); + addLogs(blocks: L2Block[]): Promise { + return this.db.transactionAsync(async () => { + for (const block of blocks) { + const blockHash = await block.hash(); + const blockNumber = block.number; + const blockTimestamp = block.timestamp; + + const privateKeys: string[] = []; + const privateValues: Buffer[] = []; + const publicKeys: string[] = []; + const publicValues: Buffer[] = []; + + for (let txIndexWithinBlock = 0; txIndexWithinBlock < block.body.txEffects.length; txIndexWithinBlock++) { + const txEffect = block.body.txEffects[txIndexWithinBlock]; + const txHash = txEffect.txHash; + + // Private and public log indices are counted independently per tx, each starting at 0. + let privateLogIndexWithinTx = 0; + let publicLogIndexWithinTx = 0; + + for (const log of txEffect.privateLogs) { + const tagHex = fieldHex(log.fields[0]); + const key = encodeKey(tagHex, blockNumber, txIndexWithinBlock, privateLogIndexWithinTx); + const value = encodeValue({ + txHash, + blockHash, + blockTimestamp, + logData: log.getEmittedFields(), + }); + privateKeys.push(key); + privateValues.push(value); + privateLogIndexWithinTx++; + } + + for (const log of txEffect.publicLogs) { + const contractHex = fieldHex(log.contractAddress); + const tagHex = fieldHex(log.fields[0]); + const key = encodeKey( + encodePublicPrefix(contractHex, tagHex), + blockNumber, + txIndexWithinBlock, + publicLogIndexWithinTx, + ); + const value = encodeValue({ + txHash, + blockHash, + blockTimestamp, + logData: log.getEmittedFields(), + }); + publicKeys.push(key); + publicValues.push(value); + publicLogIndexWithinTx++; + } } - privateTaggedLogs.set(taggedLogBuffer.tag, taggedLogBuffer.logBuffers.concat(newLogs)); - } - } - for (const block of newBlocks) { - const privateTagsInBlock: string[] = []; - for (const [tag, logs] of privateTaggedLogs.entries()) { - await this.#privateLogsByTag.set(tag, logs); - privateTagsInBlock.push(tag); - } - await this.#privateLogKeysByBlock.set(block.number, privateTagsInBlock); - } - } - - async #addPublicLogs(blocks: L2Block[]): Promise { - const newBlocks = await filterAsync( - blocks, - async block => !(await this.#publicLogKeysByBlock.hasAsync(block.number)), - ); - - const { publicTaggedLogs } = this.#extractTaggedLogs(newBlocks); - const keysOfPublicLogsToUpdate = Array.from(publicTaggedLogs.keys()); - - const currentPublicTaggedLogs = await Promise.all( - keysOfPublicLogsToUpdate.map(async key => ({ - tag: key, - logBuffers: await this.#publicLogsByContractAndTag.getAsync(key), - })), - ); - - for (const taggedLogBuffer of currentPublicTaggedLogs) { - if (taggedLogBuffer.logBuffers && taggedLogBuffer.logBuffers.length > 0) { - const newLogs = publicTaggedLogs.get(taggedLogBuffer.tag)!; - if (newLogs.length === 0) { - continue; + for (let i = 0; i < privateKeys.length; i++) { + await this.#privateLogs.set(privateKeys[i], privateValues[i]); } - const lastExisting = TxScopedL2Log.fromBuffer(taggedLogBuffer.logBuffers.at(-1)!); - const firstNew = TxScopedL2Log.fromBuffer(newLogs[0]); - if (lastExisting.blockNumber > firstNew.blockNumber) { - throw new OutOfOrderLogInsertionError( - 'public', - taggedLogBuffer.tag, - lastExisting.blockNumber, - firstNew.blockNumber, - ); + for (let i = 0; i < publicKeys.length; i++) { + await this.#publicLogs.set(publicKeys[i], publicValues[i]); } - publicTaggedLogs.set(taggedLogBuffer.tag, taggedLogBuffer.logBuffers.concat(newLogs)); - } - } - for (const block of newBlocks) { - const blockHash = await block.hash(); - const publicTagsInBlock: string[] = []; - for (const [tag, logs] of publicTaggedLogs.entries()) { - await this.#publicLogsByContractAndTag.set(tag, logs); - publicTagsInBlock.push(tag); - } - await this.#publicLogKeysByBlock.set(block.number, publicTagsInBlock); - - const publicLogsInBlock = block.body.txEffects - .map((txEffect, txIndex) => - [ - numToUInt32BE(txIndex), - txEffect.txHash.toBuffer(), - numToUInt32BE(txEffect.publicLogs.length), - txEffect.publicLogs.map(log => log.toBuffer()), - ].flat(), - ) - .flat(); - - await this.#publicLogsByBlock.set(block.number, this.#packWithBlockHash(blockHash, publicLogsInBlock)); - } - } - - async #addContractClassLogs(blocks: L2Block[]): Promise { - const newBlocks = await filterAsync( - blocks, - async block => !(await this.#contractClassLogsByBlock.hasAsync(block.number)), - ); - - for (const block of newBlocks) { - const blockHash = await block.hash(); - - const contractClassLogsInBlock = block.body.txEffects - .map((txEffect, txIndex) => - [ - numToUInt32BE(txIndex), - txEffect.txHash.toBuffer(), - numToUInt32BE(txEffect.contractClassLogs.length), - txEffect.contractClassLogs.map(log => log.toBuffer()), - ].flat(), - ) - .flat(); - - await this.#contractClassLogsByBlock.set( - block.number, - this.#packWithBlockHash(blockHash, contractClassLogsInBlock), - ); - } - } + await this.#privateKeysByBlock.set(blockNumber, privateKeys); + await this.#publicKeysByBlock.set(blockNumber, publicKeys); - /** - * Append new logs to the store's list. - * @param blocks - The blocks for which to add the logs. - * @returns True if the operation is successful. - */ - addLogs(blocks: L2Block[]): Promise { - return this.db.transactionAsync(async () => { - await Promise.all([ - this.#addPrivateLogs(blocks), - this.#addPublicLogs(blocks), - this.#addContractClassLogs(blocks), - ]); + this.#log.debug(`Indexed logs for block ${blockNumber}`, { + blockNumber, + privateCount: privateKeys.length, + publicCount: publicKeys.length, + }); + } return true; }); } - #packWithBlockHash(blockHash: BlockHash, data: Buffer[]): Buffer { - return Buffer.concat([blockHash.toBuffer(), ...data]); - } - - #unpackBlockHash(reader: BufferReader): BlockHash { - if (reader.remainingBytes() === 0) { - throw new Error('Failed to read block hash from log entry buffer'); - } - - return BlockHash.fromBuffer(reader); - } - + /** + * Deletes every log indexed under any of the given blocks. Secondary-index driven, so it doesn't + * have to range-scan the primary maps. + */ deleteLogs(blocks: L2Block[]): Promise { return this.db.transactionAsync(async () => { - const blockNumbers = new Set(blocks.map(block => block.number)); - const firstBlockToDelete = Math.min(...blockNumbers); - - // Collect all unique private tags across all blocks being deleted - const allPrivateTags = new Set( - compactArray(await Promise.all(blocks.map(block => this.#privateLogKeysByBlock.getAsync(block.number)))).flat(), - ); - - // Trim private logs: for each tag, delete all instances including and after the first block being deleted. - // This hinges on the invariant that logs for a given tag are always inserted in order of block number, which is enforced in #addPrivateLogs. - for (const tag of allPrivateTags) { - const existing = await this.#privateLogsByTag.getAsync(tag); - if (existing === undefined || existing.length === 0) { - continue; + for (const block of blocks) { + const blockNumber = block.number; + + const [privateKeys, publicKeys] = await Promise.all([ + this.#privateKeysByBlock.getAsync(blockNumber), + this.#publicKeysByBlock.getAsync(blockNumber), + ]); + + if (privateKeys) { + for (const key of privateKeys) { + await this.#privateLogs.delete(key); + } + await this.#privateKeysByBlock.delete(blockNumber); } - const lastIndexToKeep = existing.findLastIndex( - buf => TxScopedL2Log.getBlockNumberFromBuffer(buf) < firstBlockToDelete, - ); - const remaining = existing.slice(0, lastIndexToKeep + 1); - await (remaining.length > 0 ? this.#privateLogsByTag.set(tag, remaining) : this.#privateLogsByTag.delete(tag)); - } - - // Collect all unique public keys across all blocks being deleted - const allPublicKeys = new Set( - compactArray(await Promise.all(blocks.map(block => this.#publicLogKeysByBlock.getAsync(block.number)))).flat(), - ); - - // And do the same as we did with private logs - for (const key of allPublicKeys) { - const existing = await this.#publicLogsByContractAndTag.getAsync(key); - if (existing === undefined || existing.length === 0) { - continue; + if (publicKeys) { + for (const key of publicKeys) { + await this.#publicLogs.delete(key); + } + await this.#publicKeysByBlock.delete(blockNumber); } - const lastIndexToKeep = existing.findLastIndex( - buf => TxScopedL2Log.getBlockNumberFromBuffer(buf) < firstBlockToDelete, - ); - const remaining = existing.slice(0, lastIndexToKeep + 1); - await (remaining.length > 0 - ? this.#publicLogsByContractAndTag.set(key, remaining) - : this.#publicLogsByContractAndTag.delete(key)); } - - // After trimming the tagged logs, we can delete the block-level keys that track which tags are in which blocks. - await Promise.all( - blocks.map(block => - Promise.all([ - this.#publicLogsByBlock.delete(block.number), - this.#privateLogKeysByBlock.delete(block.number), - this.#publicLogKeysByBlock.delete(block.number), - this.#contractClassLogsByBlock.delete(block.number), - ]), - ), - ); - return true; }); } - /** - * Gets private logs that match any of the `tags`. For each tag, an array of matching logs is returned. An empty - * array implies no logs match that tag. - * @param tags - The tags to search for. - * @param page - The page number (0-indexed) for pagination. - * @param upToBlockNumber - If set, only return logs from blocks up to and including this block number. - * @returns An array of log arrays, one per tag. Returns at most MAX_LOGS_PER_TAG logs per tag per page. If - * MAX_LOGS_PER_TAG logs are returned for a tag, the caller should fetch the next page to check for more logs. - */ - async getPrivateLogsByTags( - tags: SiloedTag[], - page: number = 0, - upToBlockNumber?: BlockNumber, - ): Promise { - const logs = await Promise.all(tags.map(tag => this.#privateLogsByTag.getAsync(tag.toString()))); - - const start = page * MAX_LOGS_PER_TAG; - const end = start + MAX_LOGS_PER_TAG; - - return logs.map(logBuffers => { - const deserialized = logBuffers?.slice(start, end).map(buf => TxScopedL2Log.fromBuffer(buf)) ?? []; - if (upToBlockNumber !== undefined) { - const cutoff = deserialized.findIndex(log => log.blockNumber > upToBlockNumber); - if (cutoff !== -1) { - return deserialized.slice(0, cutoff); - } - } - return deserialized; - }); + /** Returns one inner array per element of `query.tags`, in input order. */ + getPrivateLogsByTags(query: PrivateLogsQuery): Promise { + LogStore.#validateQuery(query); + return this.db.transactionAsync(() => this.#runQuery(query, /* contractHex */ undefined)); } - /** - * Gets public logs that match any of the `tags` from the specified contract. For each tag, an array of matching - * logs is returned. An empty array implies no logs match that tag. - * @param contractAddress - The contract address to search logs for. - * @param tags - The tags to search for. - * @param page - The page number (0-indexed) for pagination. - * @param upToBlockNumber - If set, only return logs from blocks up to and including this block number. - * @returns An array of log arrays, one per tag. Returns at most MAX_LOGS_PER_TAG logs per tag per page. If - * MAX_LOGS_PER_TAG logs are returned for a tag, the caller should fetch the next page to check for more logs. - */ - async getPublicLogsByTagsFromContract( - contractAddress: AztecAddress, - tags: Tag[], - page: number = 0, - upToBlockNumber?: BlockNumber, - ): Promise { - const logs = await Promise.all( - tags.map(tag => { - const key = `${contractAddress.toString()}_${tag.value.toString()}`; - return this.#publicLogsByContractAndTag.getAsync(key); - }), - ); - const start = page * MAX_LOGS_PER_TAG; - const end = start + MAX_LOGS_PER_TAG; - - return logs.map(logBuffers => { - const deserialized = logBuffers?.slice(start, end).map(buf => TxScopedL2Log.fromBuffer(buf)) ?? []; - if (upToBlockNumber !== undefined) { - const cutoff = deserialized.findIndex(log => log.blockNumber > upToBlockNumber); - if (cutoff !== -1) { - return deserialized.slice(0, cutoff); - } - } - return deserialized; - }); + /** Returns one inner array per element of `query.tags`, in input order. */ + getPublicLogsByTags(query: PublicLogsQuery): Promise { + LogStore.#validateQuery(query); + return this.db.transactionAsync(() => this.#runQuery(query, fieldHex(query.contractAddress))); } - /** - * Gets public logs based on the provided filter. - * @param filter - The filter to apply to the logs. - * @returns The requested logs. - */ - getPublicLogs(filter: LogFilter): Promise { - if (filter.afterLog) { - return this.#filterPublicLogsBetweenBlocks(filter); - } else if (filter.txHash) { - return this.#filterPublicLogsOfTx(filter); - } else { - return this.#filterPublicLogsBetweenBlocks(filter); + static #validateQuery(query: { txHash?: TxHash; fromBlock?: unknown; toBlock?: unknown }): void { + if (query.txHash !== undefined && (query.fromBlock !== undefined || query.toBlock !== undefined)) { + throw new Error('`txHash` is mutually exclusive with `fromBlock`/`toBlock`'); } } - async #filterPublicLogsOfTx(filter: LogFilter): Promise { - if (!filter.txHash) { - throw new Error('Missing txHash'); + async #runQuery(query: PrivateLogsQuery | PublicLogsQuery, contractHex: string | undefined): Promise { + const isPublic = contractHex !== undefined; + const tags = (query.tags as ReadonlyArray>) ?? []; + const primaryMap = isPublic ? this.#publicLogs : this.#privateLogs; + + // referenceBlock reorg check, in-transaction, against the same db the log primary maps live on. The + // genesis block is a valid anchor during early sync but is synthetic and never indexed in the block + // store, so resolve it directly to the genesis block number rather than mistaking it for a reorg. + let referenceBlockNumber: number | undefined; + if (query.referenceBlock) { + if (query.referenceBlock.equals(this.genesisBlockHash)) { + referenceBlockNumber = INITIAL_L2_BLOCK_NUM - 1; + } else { + const refBlk = await this.blockStore.getBlockData({ hash: query.referenceBlock }); + if (!refBlk) { + throw new Error( + `Reference block ${query.referenceBlock.toString()} not found in the node. This might indicate a reorg has occurred.`, + ); + } + referenceBlockNumber = refBlk.header.globalVariables.blockNumber; + } } - const [blockNumber, txIndex] = (await this.blockStore.getTxLocation(filter.txHash)) ?? []; - if (typeof blockNumber !== 'number' || typeof txIndex !== 'number') { - return { logs: [], maxLogsHit: false }; + // Compute the exclusive upper-block bound across `toBlock` and `referenceBlock`. + // `toBlock` is already exclusive; `referenceBlock` caps inclusively, so its exclusive form is +1. + let upperExclusive: number | undefined; + if (query.toBlock !== undefined) { + upperExclusive = query.toBlock; + } + if (referenceBlockNumber !== undefined) { + const refExclusive = referenceBlockNumber + 1; + upperExclusive = upperExclusive === undefined ? refExclusive : Math.min(upperExclusive, refExclusive); } - const buffer = (await this.#publicLogsByBlock.getAsync(blockNumber)) ?? Buffer.alloc(0); - const publicLogsInBlock: { txHash: TxHash; logs: PublicLog[] }[] = []; - const reader = new BufferReader(buffer); - - const blockHash = this.#unpackBlockHash(reader); - - while (reader.remainingBytes() > 0) { - const indexOfTx = reader.readNumber(); - const txHash = reader.readObject(TxHash); - const numLogsInTx = reader.readNumber(); - publicLogsInBlock[indexOfTx] = { txHash, logs: [] }; - for (let i = 0; i < numLogsInTx; i++) { - publicLogsInBlock[indexOfTx].logs.push(reader.readObject(PublicLog)); + // Resolve txHash -> (blockNumber, txIndexInBlock) once for the whole query. + let txLocation: [number, number] | undefined; + if (query.txHash) { + const loc = await this.blockStore.getTxLocation(query.txHash); + if (!loc) { + return tags.map(() => []); + } + txLocation = loc; + if (upperExclusive !== undefined && txLocation[0] >= upperExclusive) { + return tags.map(() => []); } } - const txData = publicLogsInBlock[txIndex]; - - const logs: ExtendedPublicLog[] = []; - const maxLogsHit = this.#accumulatePublicLogs( - logs, - blockNumber, - blockHash, - txIndex, - txData.txHash, - txData.logs, - filter, - ); - - return { logs, maxLogsHit }; - } - - async #filterPublicLogsBetweenBlocks(filter: LogFilter): Promise { - const start = - filter.afterLog?.blockNumber ?? Math.max(filter.fromBlock ?? INITIAL_L2_BLOCK_NUM, INITIAL_L2_BLOCK_NUM); - const end = filter.toBlock; + const fromBlock = query.fromBlock ?? INITIAL_L2_BLOCK_NUM; + const includeEffects = query.includeEffects === true; + + const perTagResults: LogResult[][] = []; + for (const tagEntry of tags) { + const { tagHex, afterLog } = normalizeTagEntry(tagEntry); + const prefix = contractHex !== undefined ? encodePublicPrefix(contractHex, tagHex) : tagHex; + + const end = txLocation + ? endOfTxRange(prefix, txLocation[0], txLocation[1]) + : endOfTagRange(prefix, upperExclusive); + + let start: string; + if (afterLog) { + // Cursor wins as the start; `fromBlock` is ignored (fine if the cursor sits below it). The cursor + // carries `(blockNumber, txIndexWithinBlock, logIndexWithinTx)`, which slot directly into the + // composite key — no tx-hash lookup needed. + start = incKey(encodeKey(prefix, afterLog.blockNumber, afterLog.txIndexWithinBlock, afterLog.logIndexWithinTx)); + } else if (txLocation) { + start = encodeKey(prefix, txLocation[0], txLocation[1], 0); + } else { + start = encodeKey(prefix, fromBlock, 0, 0); + } - if (typeof end === 'number' && end < start) { - return { - logs: [], - maxLogsHit: true, - }; + const limit = query.limitPerTag ?? MAX_LOGS_PER_TAG; + const out: LogResult[] = []; + for await (const [rawKey, rawVal] of primaryMap.entriesAsync({ start, end, limit })) { + const tail = decodeKeyTail(rawKey); + const value = decodeValue(rawVal); + out.push({ + logData: value.logData, + blockNumber: tail.blockNumber, + blockHash: value.blockHash, + blockTimestamp: value.blockTimestamp, + txHash: value.txHash, + txIndexWithinBlock: tail.txIndexWithinBlock, + logIndexWithinTx: tail.logIndexWithinTx, + }); + } + perTagResults.push(out); } - const logs: ExtendedPublicLog[] = []; - - let maxLogsHit = false; - loopOverBlocks: for await (const [blockNumber, logBuffer] of this.#publicLogsByBlock.entriesAsync({ start, end })) { - const publicLogsInBlock: { txHash: TxHash; logs: PublicLog[] }[] = []; - const reader = new BufferReader(logBuffer); - - const blockHash = this.#unpackBlockHash(reader); - - while (reader.remainingBytes() > 0) { - const indexOfTx = reader.readNumber(); - const txHash = reader.readObject(TxHash); - const numLogsInTx = reader.readNumber(); - publicLogsInBlock[indexOfTx] = { txHash, logs: [] }; - for (let i = 0; i < numLogsInTx; i++) { - publicLogsInBlock[indexOfTx].logs.push(reader.readObject(PublicLog)); + if (includeEffects) { + // Dedupe by txHash across the entire page so a tx with many tagged logs costs one fetch. + const txHashByKey = new Map(); + for (const arr of perTagResults) { + for (const log of arr) { + txHashByKey.set(log.txHash.toString(), log.txHash); } } - for (let txIndex = filter.afterLog?.txIndex ?? 0; txIndex < publicLogsInBlock.length; txIndex++) { - const txData = publicLogsInBlock[txIndex]; - maxLogsHit = this.#accumulatePublicLogs( - logs, - blockNumber, - blockHash, - txIndex, - txData.txHash, - txData.logs, - filter, - ); - if (maxLogsHit) { - this.#log.debug(`Max logs hit at block ${blockNumber}`); - break loopOverBlocks; + const uniqueTxs = Array.from(txHashByKey.values()); + if (uniqueTxs.length > 0) { + const effects = await this.blockStore.getNoteHashesAndNullifiers(uniqueTxs); + const byTxHash = new Map(); + uniqueTxs.forEach((tx, i) => byTxHash.set(tx.toString(), effects[i])); + for (let i = 0; i < perTagResults.length; i++) { + perTagResults[i] = perTagResults[i].map(log => { + const [noteHashes, nullifiers] = byTxHash.get(log.txHash.toString()) ?? [[], []]; + return { ...log, noteHashes, nullifiers }; + }); } } } - return { logs, maxLogsHit }; + return perTagResults; } /** - * Gets contract class logs based on the provided filter. - * @param filter - The filter to apply to the logs. - * @returns The requested logs. + * Reads back every private log indexed for the given block via the per-block secondary index. Order + * matches the canonical composite-key order (`tag`, `blockNumber`, `txIndexWithinBlock`, + * `logIndexWithinTx`). Used by the data-store-updater test suite to verify the indexed-vs-block-body + * counts without depending on the removed `getPublicLogs(LogFilter)` API. */ - getContractClassLogs(filter: LogFilter): Promise { - if (filter.afterLog) { - return this.#filterContractClassLogsBetweenBlocks(filter); - } else if (filter.txHash) { - return this.#filterContractClassLogsOfTx(filter); - } else { - return this.#filterContractClassLogsBetweenBlocks(filter); - } - } - - async #filterContractClassLogsOfTx(filter: LogFilter): Promise { - if (!filter.txHash) { - throw new Error('Missing txHash'); - } - - const [blockNumber, txIndex] = (await this.blockStore.getTxLocation(filter.txHash)) ?? []; - if (typeof blockNumber !== 'number' || typeof txIndex !== 'number') { - return { logs: [], maxLogsHit: false }; - } - const contractClassLogsBuffer = (await this.#contractClassLogsByBlock.getAsync(blockNumber)) ?? Buffer.alloc(0); - const contractClassLogsInBlock: { txHash: TxHash; logs: ContractClassLog[] }[] = []; - - const reader = new BufferReader(contractClassLogsBuffer); - const blockHash = this.#unpackBlockHash(reader); - - while (reader.remainingBytes() > 0) { - const indexOfTx = reader.readNumber(); - const txHash = reader.readObject(TxHash); - const numLogsInTx = reader.readNumber(); - contractClassLogsInBlock[indexOfTx] = { txHash, logs: [] }; - for (let i = 0; i < numLogsInTx; i++) { - contractClassLogsInBlock[indexOfTx].logs.push(reader.readObject(ContractClassLog)); - } - } - - const txData = contractClassLogsInBlock[txIndex]; - - const logs: ExtendedContractClassLog[] = []; - const maxLogsHit = this.#accumulateContractClassLogs( - logs, - blockNumber, - blockHash, - txIndex, - txData.txHash, - txData.logs, - filter, + getPrivateLogsForBlock(blockNumber: number): Promise { + return this.db.transactionAsync(() => + this.#readBlockLogs(this.#privateKeysByBlock, this.#privateLogs, blockNumber), ); - - return { logs, maxLogsHit }; } - async #filterContractClassLogsBetweenBlocks(filter: LogFilter): Promise { - const start = - filter.afterLog?.blockNumber ?? Math.max(filter.fromBlock ?? INITIAL_L2_BLOCK_NUM, INITIAL_L2_BLOCK_NUM); - const end = filter.toBlock; - - if (typeof end === 'number' && end < start) { - return { - logs: [], - maxLogsHit: true, - }; - } - - const logs: ExtendedContractClassLog[] = []; - - let maxLogsHit = false; - loopOverBlocks: for await (const [blockNumber, logBuffer] of this.#contractClassLogsByBlock.entriesAsync({ - start, - end, - })) { - const contractClassLogsInBlock: { txHash: TxHash; logs: ContractClassLog[] }[] = []; - const reader = new BufferReader(logBuffer); - const blockHash = this.#unpackBlockHash(reader); - while (reader.remainingBytes() > 0) { - const indexOfTx = reader.readNumber(); - const txHash = reader.readObject(TxHash); - const numLogsInTx = reader.readNumber(); - contractClassLogsInBlock[indexOfTx] = { txHash, logs: [] }; - for (let i = 0; i < numLogsInTx; i++) { - contractClassLogsInBlock[indexOfTx].logs.push(reader.readObject(ContractClassLog)); - } - } - for (let txIndex = filter.afterLog?.txIndex ?? 0; txIndex < contractClassLogsInBlock.length; txIndex++) { - const txData = contractClassLogsInBlock[txIndex]; - maxLogsHit = this.#accumulateContractClassLogs( - logs, - blockNumber, - blockHash, - txIndex, - txData.txHash, - txData.logs, - filter, - ); - if (maxLogsHit) { - this.#log.debug(`Max logs hit at block ${blockNumber}`); - break loopOverBlocks; - } - } - } - - return { logs, maxLogsHit }; + /** {@inheritDoc LogStore.getPrivateLogsForBlock} */ + getPublicLogsForBlock(blockNumber: number): Promise { + return this.db.transactionAsync(() => this.#readBlockLogs(this.#publicKeysByBlock, this.#publicLogs, blockNumber)); } - #accumulatePublicLogs( - results: ExtendedPublicLog[], + async #readBlockLogs( + keysByBlock: AztecAsyncMap, + primaryMap: AztecAsyncMap, blockNumber: number, - blockHash: BlockHash, - txIndex: number, - txHash: TxHash, - txLogs: PublicLog[], - filter: LogFilter = {}, - ): boolean { - if (filter.fromBlock && blockNumber < filter.fromBlock) { - return false; - } - if (filter.toBlock && blockNumber >= filter.toBlock) { - return false; - } - if (filter.txHash && !txHash.equals(filter.txHash)) { - return false; + ): Promise { + const keys = await keysByBlock.getAsync(blockNumber); + if (!keys || keys.length === 0) { + return []; } - - let maxLogsHit = false; - let logIndex = typeof filter.afterLog?.logIndex === 'number' ? filter.afterLog.logIndex + 1 : 0; - for (; logIndex < txLogs.length; logIndex++) { - const log = txLogs[logIndex]; - if ( - (!filter.contractAddress || log.contractAddress.equals(filter.contractAddress)) && - (!filter.tag || log.fields[0]?.equals(filter.tag)) - ) { - results.push( - new ExtendedPublicLog(new LogId(BlockNumber(blockNumber), blockHash, txHash, txIndex, logIndex), log), - ); - - if (results.length >= this.#logsMaxPageSize) { - maxLogsHit = true; - break; - } + const results: LogResult[] = []; + for (const key of keys) { + const raw = await primaryMap.getAsync(key); + if (!raw) { + continue; } + const tail = decodeKeyTail(key); + const value = decodeValue(raw); + results.push({ + logData: value.logData, + blockNumber: tail.blockNumber, + blockHash: value.blockHash, + blockTimestamp: value.blockTimestamp, + txHash: value.txHash, + txIndexWithinBlock: tail.txIndexWithinBlock, + logIndexWithinTx: tail.logIndexWithinTx, + }); } - - return maxLogsHit; + return results; } +} - #accumulateContractClassLogs( - results: ExtendedContractClassLog[], - blockNumber: number, - blockHash: BlockHash, - txIndex: number, - txHash: TxHash, - txLogs: ContractClassLog[], - filter: LogFilter = {}, - ): boolean { - if (filter.fromBlock && blockNumber < filter.fromBlock) { - return false; - } - if (filter.toBlock && blockNumber >= filter.toBlock) { - return false; - } - if (filter.txHash && !txHash.equals(filter.txHash)) { - return false; - } - - let maxLogsHit = false; - let logIndex = typeof filter.afterLog?.logIndex === 'number' ? filter.afterLog.logIndex + 1 : 0; - for (; logIndex < txLogs.length; logIndex++) { - const log = txLogs[logIndex]; - if (!filter.contractAddress || log.contractAddress.equals(filter.contractAddress)) { - results.push( - new ExtendedContractClassLog(new LogId(BlockNumber(blockNumber), blockHash, txHash, txIndex, logIndex), log), - ); - - if (results.length >= this.#logsMaxPageSize) { - maxLogsHit = true; - break; - } - } - } - - return maxLogsHit; +/** Pulls `{ tagHex, afterLog }` out of a {@link TagQuery}, normalizing the bare-tag form. */ +function normalizeTagEntry( + entry: TagQuery, +): { + tagHex: string; + afterLog: LogCursor | undefined; +} { + if (typeof entry === 'object' && entry !== null && 'tag' in entry) { + return { tagHex: fieldHex(entry.tag.value), afterLog: entry.afterLog }; } + return { tagHex: fieldHex((entry as T).value), afterLog: undefined }; } diff --git a/yarn-project/archiver/src/store/log_store_codec.test.ts b/yarn-project/archiver/src/store/log_store_codec.test.ts new file mode 100644 index 000000000000..b087312746e1 --- /dev/null +++ b/yarn-project/archiver/src/store/log_store_codec.test.ts @@ -0,0 +1,250 @@ +import { BlockNumber } from '@aztec/foundation/branded-types'; +import { Fr } from '@aztec/foundation/curves/bn254'; +import { AztecAddress } from '@aztec/stdlib/aztec-address'; +import { BlockHash } from '@aztec/stdlib/block'; +import '@aztec/stdlib/testing/jest'; +import { TxHash } from '@aztec/stdlib/tx'; + +import { + decodeKeyTail, + decodeValue, + encodeKey, + encodePublicPrefix, + encodeValue, + endOfTagRange, + endOfTxRange, + fieldHex, + incKey, + u32Hex, +} from './log_store_codec.js'; + +describe('log_store_codec', () => { + describe('u32Hex', () => { + it('zero-pads to 8 lowercase hex chars', () => { + expect(u32Hex(0)).toBe('00000000'); + expect(u32Hex(1)).toBe('00000001'); + expect(u32Hex(255)).toBe('000000ff'); + expect(u32Hex(0xdeadbeef)).toBe('deadbeef'); + }); + + it('preserves big-endian lex ordering', () => { + expect(u32Hex(2) < u32Hex(16)).toBe(true); + expect(u32Hex(16) < u32Hex(256)).toBe(true); + expect(u32Hex(256) < u32Hex(0xffff)).toBe(true); + }); + }); + + describe('fieldHex', () => { + it('strips 0x prefix and returns 64 lowercase hex chars for Fr', () => { + const fr = new Fr(0xabcdn); + const hex = fieldHex(fr); + expect(hex).toHaveLength(64); + expect(hex).not.toMatch(/^0x/); + expect(hex).toMatch(/^[0-9a-f]+$/); + expect(hex).toBe(fr.toString().slice(2)); + }); + + it('strips 0x prefix for AztecAddress', () => { + const addr = AztecAddress.fromNumber(12345); + const hex = fieldHex(addr); + expect(hex).toHaveLength(64); + expect(hex).not.toMatch(/^0x/); + expect(hex).toBe(addr.toString().slice(2)); + }); + + it('handles Fr.ZERO', () => { + const hex = fieldHex(Fr.ZERO); + expect(hex).toBe('0'.repeat(64)); + }); + }); + + describe('encodeKey / decodeKeyTail', () => { + it('round-trips a private-style prefix (single tag hex)', () => { + const tagHex = fieldHex(new Fr(0x1234n)); + const key = encodeKey(tagHex, 5, 3, 7); + const tail = decodeKeyTail(key); + expect(tail.blockNumber).toBe(BlockNumber(5)); + expect(tail.txIndexWithinBlock).toBe(3); + expect(tail.logIndexWithinTx).toBe(7); + }); + + it('round-trips a public-style prefix (contract-tag)', () => { + const contractHex = fieldHex(AztecAddress.fromNumber(99)); + const tagHex = fieldHex(new Fr(0x5678n)); + const prefix = encodePublicPrefix(contractHex, tagHex); + const key = encodeKey(prefix, 10, 0, 2); + const tail = decodeKeyTail(key); + expect(tail.blockNumber).toBe(BlockNumber(10)); + expect(tail.txIndexWithinBlock).toBe(0); + expect(tail.logIndexWithinTx).toBe(2); + }); + + it('decodeKeyTail returns a branded BlockNumber', () => { + const key = encodeKey('somepfx', 42, 1, 0); + const tail = decodeKeyTail(key); + // BlockNumber is a branded number; verify it equals the numeric value. + expect(tail.blockNumber).toEqual(BlockNumber(42)); + }); + + it('handles zero values in the tail triple', () => { + const key = encodeKey('pfx', 0, 0, 0); + const tail = decodeKeyTail(key); + expect(tail.blockNumber).toEqual(BlockNumber(0)); + expect(tail.txIndexWithinBlock).toBe(0); + expect(tail.logIndexWithinTx).toBe(0); + }); + }); + + describe('lexicographic key ordering', () => { + it('sorts encoded keys in canonical (block, txIdx, logIdx) order', () => { + const prefix = fieldHex(new Fr(0xdeadn)); + const tuples: [number, number, number][] = [ + [3, 0, 0], + [1, 5, 2], + [2, 0, 9], + [1, 5, 0], + [1, 0, 1], + [3, 1, 0], + [2, 3, 0], + [1, 0, 0], + ]; + const encodedKeys = tuples.map(([b, t, l]) => encodeKey(prefix, b, t, l)); + const sortedKeys = [...encodedKeys].sort(); + const decodedTails = sortedKeys.map(k => decodeKeyTail(k)); + + // Verify that the decoded tails are in canonical tuple order. + for (let i = 1; i < decodedTails.length; i++) { + const a = decodedTails[i - 1]; + const b = decodedTails[i]; + const isOrdered = + b.blockNumber > a.blockNumber || + (b.blockNumber === a.blockNumber && b.txIndexWithinBlock > a.txIndexWithinBlock) || + (b.blockNumber === a.blockNumber && + b.txIndexWithinBlock === a.txIndexWithinBlock && + b.logIndexWithinTx > a.logIndexWithinTx); + expect(isOrdered).toBe(true); + } + }); + }); + + describe('endOfTagRange', () => { + it('with no upper bound returns a key that sorts after all real keys under the prefix', () => { + const prefix = fieldHex(new Fr(0xaabbccn)); + const end = endOfTagRange(prefix, undefined); + // Any real key at a high block number should sort before the end bound. + const highKey = encodeKey(prefix, 0xffffffff, 0xffffffff, 0xffffffff); + expect(highKey < end).toBe(true); + }); + + it('with upper bound returns encodeKey(prefix, upper, 0, 0)', () => { + const prefix = fieldHex(new Fr(0x11n)); + const end = endOfTagRange(prefix, 10); + expect(end).toBe(encodeKey(prefix, 10, 0, 0)); + }); + + it('with upper bound sorts after all real keys within the block range', () => { + const prefix = fieldHex(new Fr(0x22n)); + const end = endOfTagRange(prefix, 5); + // Block 4 is the last included block (exclusive upper = 5). + const lastKey = encodeKey(prefix, 4, 0xffff, 0xffff); + expect(lastKey < end).toBe(true); + // Block 5 is excluded. + const excludedKey = encodeKey(prefix, 5, 0, 0); + expect(excludedKey >= end).toBe(true); + }); + }); + + describe('endOfTxRange', () => { + it('sorts strictly after every real log key for the given tx', () => { + const prefix = fieldHex(new Fr(0x33n)); + const end = endOfTxRange(prefix, 7, 2); + // All log indices within tx (7, 2) must be before the end. + const lastLog = encodeKey(prefix, 7, 2, 0xffffffff); + expect(lastLog < end).toBe(true); + }); + + it('sorts strictly before the next tx first key', () => { + const prefix = fieldHex(new Fr(0x44n)); + const end = endOfTxRange(prefix, 7, 2); + // The next tx (7, 3) should be strictly after our end bound. + const nextTx = encodeKey(prefix, 7, 3, 0); + expect(end < nextTx).toBe(true); + }); + }); + + describe('incKey', () => { + it('returns the smallest string strictly greater than the input key', () => { + const prefix = fieldHex(new Fr(0x55n)); + const key = encodeKey(prefix, 1, 2, 3); + const next = incKey(key); + expect(next > key).toBe(true); + }); + + it('an inclusive cursor becomes exclusive: no real key falls between key and incKey(key)', () => { + const prefix = fieldHex(new Fr(0x66n)); + const key = encodeKey(prefix, 5, 1, 3); + const nextKey = encodeKey(prefix, 5, 1, 4); + const inc = incKey(key); + // nextKey is the first real key after key; it must sort after inc or at inc. + // In our scheme `inc = key + 'g'`, and `nextKey` ends in a hex digit, so nextKey > inc + // cannot happen — nextKey should be strictly greater than key but less than inc only if + // we haven't skipped a valid key. Since the last segment of key ends in hex and 'g' > 'f', + // anything with a valid hex tail for the same position would sort before inc. + expect(inc > key).toBe(true); + expect(nextKey > inc).toBe(true); + }); + }); + + describe('encodeValue / decodeValue', () => { + it('round-trips a value with empty logData', () => { + const txHash = TxHash.random(); + const blockHash = BlockHash.random(); + const blockTimestamp = 1234567890n; + const original = { txHash, blockHash, blockTimestamp, logData: [] }; + const buf = encodeValue(original); + const decoded = decodeValue(buf); + expect(decoded.txHash.equals(txHash)).toBe(true); + expect(decoded.blockHash.equals(blockHash)).toBe(true); + expect(decoded.blockTimestamp).toBe(blockTimestamp); + expect(decoded.logData).toHaveLength(0); + }); + + it('round-trips a value with multiple logData fields', () => { + const txHash = TxHash.random(); + const blockHash = BlockHash.random(); + const blockTimestamp = 9999999999n; + const logData = [Fr.random(), Fr.random(), Fr.random()]; + const original = { txHash, blockHash, blockTimestamp, logData }; + const buf = encodeValue(original); + const decoded = decodeValue(buf); + expect(decoded.txHash.equals(txHash)).toBe(true); + expect(decoded.blockHash.equals(blockHash)).toBe(true); + expect(decoded.blockTimestamp).toBe(blockTimestamp); + expect(decoded.logData).toHaveLength(3); + for (let i = 0; i < logData.length; i++) { + expect(decoded.logData[i].equals(logData[i])).toBe(true); + } + }); + + it('handles a Uint8Array input in decodeValue', () => { + const txHash = TxHash.random(); + const blockHash = BlockHash.random(); + const blockTimestamp = 0n; + const original = { txHash, blockHash, blockTimestamp, logData: [Fr.random()] }; + const buf = encodeValue(original); + // Convert to Uint8Array (not a Buffer). + const uint8 = new Uint8Array(buf.buffer, buf.byteOffset, buf.byteLength); + const decoded = decodeValue(uint8); + expect(decoded.txHash.equals(txHash)).toBe(true); + expect(decoded.blockTimestamp).toBe(0n); + }); + }); + + describe('encodePublicPrefix', () => { + it('produces contractHex-tagHex', () => { + const contractHex = fieldHex(AztecAddress.fromNumber(1)); + const tagHex = fieldHex(new Fr(2n)); + expect(encodePublicPrefix(contractHex, tagHex)).toBe(`${contractHex}-${tagHex}`); + }); + }); +}); diff --git a/yarn-project/archiver/src/store/log_store_codec.ts b/yarn-project/archiver/src/store/log_store_codec.ts new file mode 100644 index 000000000000..0a1edfc8bdb2 --- /dev/null +++ b/yarn-project/archiver/src/store/log_store_codec.ts @@ -0,0 +1,132 @@ +import { BlockNumber } from '@aztec/foundation/branded-types'; +import { Fr } from '@aztec/foundation/curves/bn254'; +import { BlockHash } from '@aztec/stdlib/block'; +import { TxHash } from '@aztec/stdlib/tx'; + +export const NUMERIC_HEX_LEN = 8; +export const SEP = '-'; + +/** + * Sentinel appended after a numeric hex segment to build an end bound strictly greater than any + * real key for that namespace. `'g'` sorts lexicographically after every hex digit (`0`-`9`, `a`-`f`), + * so `prefix + '-g'` is a clean exclusive upper bound. + */ +export const HEX_SENTINEL = 'g'; + +export type ParsedKeyTail = { + blockNumber: BlockNumber; + txIndexWithinBlock: number; + logIndexWithinTx: number; +}; + +/** + * Per-kind stored value layout (no msgpackr): + * txHash(32) ++ blockHash(32) ++ blockTimestamp(u64 BE = 8) ++ logDataLen(u32 BE = 4) ++ logData[i].toBuffer()... + * `blockNumber`, `txIndexWithinBlock`, and `logIndexWithinTx` are decoded from the composite key. + */ +export type StoredLogValue = { + txHash: TxHash; + blockHash: BlockHash; + blockTimestamp: bigint; + logData: Fr[]; +}; + +/** Returns the 64-char lowercase hex representation of a field, stripping the `0x` prefix. */ +export function fieldHex(value: Fr | { toString: () => string }): string { + // Fr.toString() and AztecAddress.toString() both return `0x` + 64 lowercase hex chars. + return value.toString().slice(2); +} + +/** Encodes a number as 8-char zero-padded lowercase hex (matches a u32 big-endian byte buffer's lex order). */ +export function u32Hex(n: number): string { + return n.toString(16).padStart(NUMERIC_HEX_LEN, '0'); +} + +/** + * Encodes the composite primary key as `prefix-block-txIdx-logIdx` where `prefix` is the leading + * segment (`tag` for private; `contract-tag` for public) and the trailing triple is fixed-width + * 8-char zero-padded hex so byte-order matches `(blockNumber, txIndexWithinBlock, logIndexWithinTx)`. + */ +export function encodeKey(prefix: string, blockNumber: number, txIndex: number, logIndex: number): string { + return `${prefix}${SEP}${u32Hex(blockNumber)}${SEP}${u32Hex(txIndex)}${SEP}${u32Hex(logIndex)}`; +} + +/** + * Decodes the trailing `(blockNumber, txIndexWithinBlock, logIndexWithinTx)` triple from a composite + * key. The leading prefix segments are ignored — we only ever read them off the input query, never + * back off the key. + */ +export function decodeKeyTail(key: string): ParsedKeyTail { + const parts = key.split(SEP); + const len = parts.length; + return { + blockNumber: BlockNumber(parseInt(parts[len - 3], 16)), + txIndexWithinBlock: parseInt(parts[len - 2], 16), + logIndexWithinTx: parseInt(parts[len - 1], 16), + }; +} + +/** + * Exclusive end bound for a `(contract, tag)`-prefix scan. With an `upperBlockExclusive` we cut at + * `(prefix, upper, 0, 0)`. With no bound we use `prefix + '-' + HEX_SENTINEL`, which sorts strictly + * after every real key under `prefix` (`g` is greater than any hex digit). + */ +export function endOfTagRange(prefix: string, upperBlockExclusive: number | undefined): string { + if (upperBlockExclusive === undefined) { + return `${prefix}${SEP}${HEX_SENTINEL}`; + } + return encodeKey(prefix, upperBlockExclusive, 0, 0); +} + +/** + * Exclusive end bound for a tx-strict scan: every key strictly inside `(prefix, txBlk, txIdx, *)`. + * `prefix-block-txIdx-` followed by the hex sentinel is the first key past every real logIndex for + * this tx and strictly less than the next tx's first key. + */ +export function endOfTxRange(prefix: string, txBlk: number, txIdx: number): string { + return `${prefix}${SEP}${u32Hex(txBlk)}${SEP}${u32Hex(txIdx)}${SEP}${HEX_SENTINEL}`; +} + +/** + * Returns the smallest string strictly greater than a fully-encoded composite key. The encoded key + * ends in a hex digit, and `'g'` sorts strictly after any hex digit, so appending `'g'` is the + * smallest possible successor in our key alphabet. Used to turn an inclusive cursor into an + * exclusive `start`. + */ +export function incKey(key: string): string { + return key + HEX_SENTINEL; +} + +export function encodeValue(value: StoredLogValue): Buffer { + const head = Buffer.allocUnsafe(32 + 32 + 8 + 4); + value.txHash.toBuffer().copy(head, 0); + value.blockHash.toBuffer().copy(head, 32); + head.writeBigUInt64BE(value.blockTimestamp, 64); + head.writeUInt32BE(value.logData.length, 72); + const fieldBufs = value.logData.map(f => f.toBuffer()); + return Buffer.concat([head, ...fieldBufs]); +} + +export function decodeValue(buffer: Buffer | Uint8Array): StoredLogValue { + const buf = Buffer.isBuffer(buffer) ? buffer : Buffer.from(buffer.buffer, buffer.byteOffset, buffer.byteLength); + let off = 0; + const txHash = TxHash.fromBuffer(buf.subarray(off, off + 32)); + off += 32; + const blockHash = BlockHash.fromBuffer(buf.subarray(off, off + 32)); + off += 32; + const blockTimestamp = buf.readBigUInt64BE(off); + off += 8; + const logDataLen = buf.readUInt32BE(off); + off += 4; + const logData: Fr[] = new Array(logDataLen); + for (let i = 0; i < logDataLen; i++) { + logData[i] = Fr.fromBuffer(buf.subarray(off, off + 32)); + off += 32; + } + return { txHash, blockHash, blockTimestamp, logData }; +} + +/** Encodes the public-log key prefix as `contractHex-tagHex`. */ +export function encodePublicPrefix(contractHex: string, tagHex: string): string { + return `${contractHex}${SEP}${tagHex}`; +} diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index aec62c243b0e..6fd0e7675cd1 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -95,8 +95,6 @@ import type { CheckpointIncludeOptions, CheckpointParameter, CheckpointResponse, - GetContractClassLogsResponse, - GetPublicLogsResponse, } from '@aztec/stdlib/interfaces/client'; import { AztecNodeAdminConfigSchema } from '@aztec/stdlib/interfaces/client'; import { @@ -108,7 +106,7 @@ import { type WorldStateSynchronizer, tryStop, } from '@aztec/stdlib/interfaces/server'; -import type { DebugLogStore, LogFilter, SiloedTag, Tag, TxScopedL2Log } from '@aztec/stdlib/logs'; +import type { DebugLogStore, LogResult, PrivateLogsQuery, PublicLogsQuery } from '@aztec/stdlib/logs'; import { InMemoryDebugLogStore, NullDebugLogStore } from '@aztec/stdlib/logs'; import { InboxLeaf, type L1ToL2MessageSource, appendL1ToL2MessagesToTree } from '@aztec/stdlib/messaging'; import type { Offense } from '@aztec/stdlib/slashing'; @@ -1137,59 +1135,12 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, AztecNodeDeb return this.contractDataSource.getContract(address); } - public async getPrivateLogsByTags( - tags: SiloedTag[], - page?: number, - referenceBlock?: BlockHash, - ): Promise { - let upToBlockNumber: BlockNumber | undefined; - if (referenceBlock) { - const data = await this.blockSource.getBlockData({ hash: referenceBlock }); - if (!data) { - throw new Error( - `Block ${referenceBlock.toString()} not found in the node. This might indicate a reorg has occurred.`, - ); - } - upToBlockNumber = data.header.globalVariables.blockNumber; - } - return this.logsSource.getPrivateLogsByTags(tags, page, upToBlockNumber); - } - - public async getPublicLogsByTagsFromContract( - contractAddress: AztecAddress, - tags: Tag[], - page?: number, - referenceBlock?: BlockHash, - ): Promise { - let upToBlockNumber: BlockNumber | undefined; - if (referenceBlock) { - const data = await this.blockSource.getBlockData({ hash: referenceBlock }); - if (!data) { - throw new Error( - `Block ${referenceBlock.toString()} not found in the node. This might indicate a reorg has occurred.`, - ); - } - upToBlockNumber = data.header.globalVariables.blockNumber; - } - return this.logsSource.getPublicLogsByTagsFromContract(contractAddress, tags, page, upToBlockNumber); - } - - /** - * Gets public logs based on the provided filter. - * @param filter - The filter to apply to the logs. - * @returns The requested logs. - */ - getPublicLogs(filter: LogFilter): Promise { - return this.logsSource.getPublicLogs(filter); + public getPrivateLogsByTags(query: PrivateLogsQuery): Promise { + return this.logsSource.getPrivateLogsByTags(query); } - /** - * Gets contract class logs based on the provided filter. - * @param filter - The filter to apply to the logs. - * @returns The requested logs. - */ - getContractClassLogs(filter: LogFilter): Promise { - return this.logsSource.getContractClassLogs(filter); + public getPublicLogsByTags(query: PublicLogsQuery): Promise { + return this.logsSource.getPublicLogsByTags(query); } /** diff --git a/yarn-project/aztec.js/src/api/events.ts b/yarn-project/aztec.js/src/api/events.ts index 9e616c0822f9..40b77fb5484a 100644 --- a/yarn-project/aztec.js/src/api/events.ts +++ b/yarn-project/aztec.js/src/api/events.ts @@ -1,7 +1,9 @@ import { DomainSeparator } from '@aztec/constants'; import { type EventMetadataDefinition, decodeFromAbi } from '@aztec/stdlib/abi'; import { computeLogTag } from '@aztec/stdlib/hash'; +import { MAX_LOGS_PER_TAG } from '@aztec/stdlib/interfaces/api-limit'; import type { AztecNode } from '@aztec/stdlib/interfaces/client'; +import { type PublicLogsQuery, Tag } from '@aztec/stdlib/logs'; import type { PublicEvent, PublicEventFilter } from '../wallet/wallet.js'; @@ -9,20 +11,22 @@ import type { PublicEvent, PublicEventFilter } from '../wallet/wallet.js'; export type GetPublicEventsResult = { /** The decoded events with metadata. */ events: PublicEvent[]; - /** Whether the log limit was reached, indicating more results may be available. */ + /** Whether the per-tag log limit was reached, indicating more results may be available — pass back the last + * event's metadata as an `afterLog` cursor to continue. */ maxLogsHit: boolean; }; /** * Returns decoded public events given search parameters. - * @param node - The node to request events from - * @param eventMetadataDef - Metadata of the event. This should be the class generated from the contract. e.g. Contract.events.Event + * @param node - The node to request events from. + * @param eventMetadataDef - Metadata of the event. This should be the class generated from the contract. + * e.g. `Contract.events.Event`. * @param filter - Filter options for the event query: - * - `contractAddress`: The address of the contract that emitted the events. - * - `txHash`: Transaction in which the events were emitted. - * - `fromBlock`: The block number from which to start fetching events (inclusive). Defaults to 1. - * - `toBlock`: The block number until which to fetch events (not inclusive). Defaults to latest + 1. - * - `afterLog`: Log id after which to start fetching logs. Used for pagination. + * - `contractAddress`: The address of the contract that emitted the events. Required. + * - `txHash`: Transaction in which the events were emitted (mutually exclusive with `fromBlock`/`toBlock`). + * - `fromBlock`: The block number from which to start fetching events (inclusive). Optional. + * - `toBlock`: The block number until which to fetch events (not inclusive). Optional. + * - `afterLog`: Log cursor after which to start fetching logs. Used for pagination. * @returns The decoded events with metadata and a flag indicating if more results are available. */ export async function getPublicEvents( @@ -32,32 +36,27 @@ export async function getPublicEvents( ): Promise> { // Public events are tagged with a domain-separated hash of their event type ID, so we compute // the same hash here to filter for logs of the requested event type. - const logTag = await computeLogTag(eventMetadataDef.eventSelector.toField(), DomainSeparator.EVENT_LOG_TAG); + const logTagField = await computeLogTag(eventMetadataDef.eventSelector.toField(), DomainSeparator.EVENT_LOG_TAG); + const logTag = new Tag(logTagField); - const { logs, maxLogsHit } = await node.getPublicLogs({ - fromBlock: filter.fromBlock ? Number(filter.fromBlock) : undefined, - toBlock: filter.toBlock ? Number(filter.toBlock) : undefined, - txHash: filter.txHash, + const query: PublicLogsQuery = { contractAddress: filter.contractAddress, - afterLog: filter.afterLog, - tag: logTag, - }); - - const events: PublicEvent[] = []; - - for (const log of logs) { - const logFieldsWithoutTag = log.log.getEmittedFieldsWithoutTag(); - - events.push({ - event: decodeFromAbi([eventMetadataDef.abiType], logFieldsWithoutTag) as T, - metadata: { - l2BlockNumber: log.id.blockNumber, - l2BlockHash: log.id.blockHash, - txHash: log.id.txHash, - contractAddress: log.log.contractAddress, - }, - }); - } - - return { events, maxLogsHit }; + tags: [filter.afterLog !== undefined ? { tag: logTag, afterLog: filter.afterLog } : logTag], + fromBlock: filter.fromBlock, + toBlock: filter.toBlock, + txHash: filter.txHash, + }; + + const [logsForTag] = await node.getPublicLogsByTags(query); + const events: PublicEvent[] = logsForTag.map(log => ({ + event: decodeFromAbi([eventMetadataDef.abiType], log.logData.slice(1)) as T, + metadata: { + l2BlockNumber: log.blockNumber, + l2BlockHash: log.blockHash, + txHash: log.txHash, + contractAddress: filter.contractAddress, + }, + })); + + return { events, maxLogsHit: logsForTag.length === MAX_LOGS_PER_TAG }; } diff --git a/yarn-project/aztec.js/src/api/log.ts b/yarn-project/aztec.js/src/api/log.ts index f813e0bf5699..198d6684afcb 100644 --- a/yarn-project/aztec.js/src/api/log.ts +++ b/yarn-project/aztec.js/src/api/log.ts @@ -1,2 +1,2 @@ export { createLogger, type Logger } from '@aztec/foundation/log'; -export { LogId, type LogFilter } from '@aztec/stdlib/logs'; +export { LogCursor, type LogResult } from '@aztec/stdlib/logs'; diff --git a/yarn-project/aztec.js/src/wallet/wallet.ts b/yarn-project/aztec.js/src/wallet/wallet.ts index 6ae8bddebdde..040c98c287c1 100644 --- a/yarn-project/aztec.js/src/wallet/wallet.ts +++ b/yarn-project/aztec.js/src/wallet/wallet.ts @@ -13,7 +13,7 @@ import { AuthWitness } from '@aztec/stdlib/auth-witness'; import type { AztecAddress } from '@aztec/stdlib/aztec-address'; import { type ContractInstanceWithAddress, ContractInstanceWithAddressSchema } from '@aztec/stdlib/contract'; import { Gas, ManaUsageEstimate } from '@aztec/stdlib/gas'; -import { LogId } from '@aztec/stdlib/logs'; +import { LogCursor, refineTxHashAndRange } from '@aztec/stdlib/logs'; import { AbiDecodedSchema, type ApiSchemaFor, @@ -164,8 +164,8 @@ export type EventFilterBase = { * Optional. If provided, it must be greater than fromBlock. */ toBlock?: BlockNumber; - /** Log id after which to start fetching logs. Used for pagination. */ - afterLog?: LogId; + /** Log cursor after which to start fetching logs. Used for pagination. */ + afterLog?: LogCursor; }; /** @@ -179,11 +179,12 @@ export type PrivateEventFilter = EventFilterBase & { }; /** - * Filter options when querying public events. + * Filter options when querying public events. The contract address is required because the public log index is + * keyed on `(contract, tag)`; tag-only queries are not supported. */ export type PublicEventFilter = EventFilterBase & { - /** The address of the contract that emitted the events. */ - contractAddress?: AztecAddress; + /** The address of the contract that emitted the events. Required. */ + contractAddress: AztecAddress; }; /** @@ -374,21 +375,31 @@ export const EventMetadataDefinitionSchema = z.object({ fieldNames: z.array(z.string()), }); -const EventFilterBaseSchema = z.object({ +// Event filters share `txHash ⊕ block-range` semantics with `LogsQueryBase` (see stdlib `logs_query.ts`) +// but diverge structurally: wallet filters are scoped to a single ABI event so they carry one optional +// `afterLog` cursor inline, whereas the stdlib query batches many tags and stores cursors per-tag inside +// `TagQuery`. We share only the refinement helper from stdlib; the field schemas stay local. +const eventFilterBaseShape = { txHash: optional(TxHash.schema), fromBlock: optional(BlockNumberPositiveSchema), toBlock: optional(BlockNumberPositiveSchema), - afterLog: optional(LogId.schema), -}); + afterLog: optional(LogCursor.schema), +}; -export const PrivateEventFilterSchema = EventFilterBaseSchema.extend({ - contractAddress: schemas.AztecAddress, - scopes: z.array(schemas.AztecAddress), -}); +export const PrivateEventFilterSchema = refineTxHashAndRange( + z.object({ + ...eventFilterBaseShape, + contractAddress: schemas.AztecAddress, + scopes: z.array(schemas.AztecAddress), + }), +); -export const PublicEventFilterSchema = EventFilterBaseSchema.extend({ - contractAddress: optional(schemas.AztecAddress), -}); +export const PublicEventFilterSchema = refineTxHashAndRange( + z.object({ + ...eventFilterBaseShape, + contractAddress: schemas.AztecAddress, + }), +); export const PrivateEventSchema: z.ZodType = zodFor>()( z.object({ diff --git a/yarn-project/cli/src/cmds/aztec_node/get_logs.ts b/yarn-project/cli/src/cmds/aztec_node/get_logs.ts index d58edac91062..8a7ae5c137ee 100644 --- a/yarn-project/cli/src/cmds/aztec_node/get_logs.ts +++ b/yarn-project/cli/src/cmds/aztec_node/get_logs.ts @@ -1,22 +1,47 @@ import type { AztecAddress } from '@aztec/aztec.js/addresses'; -import type { LogFilter, LogId } from '@aztec/aztec.js/log'; import { createAztecNodeClient } from '@aztec/aztec.js/node'; import type { TxHash } from '@aztec/aztec.js/tx'; -import { BlockNumber } from '@aztec/foundation/branded-types'; +import type { BlockNumber } from '@aztec/foundation/branded-types'; import type { LogFn } from '@aztec/foundation/log'; import { sleep } from '@aztec/foundation/sleep'; +import { + LogCursor, + type PublicLogsQuery, + type Tag, + logResultToHumanReadable, + queryAllPublicLogsByTags, +} from '@aztec/stdlib/logs'; -export async function getLogs( - txHash: TxHash, - fromBlock: BlockNumber, - toBlock: BlockNumber, - afterLog: LogId, - contractAddress: AztecAddress, - nodeUrl: string, - follow: boolean, - log: LogFn, -) { - const node = createAztecNodeClient(nodeUrl); +/** Options for the `get-logs` CLI command. */ +export type GetLogsOptions = { + /** Contract address that emitted the logs (required). */ + contractAddress: AztecAddress; + /** Tag to filter logs by (required). */ + tag: Tag; + /** Restrict the search to this tx hash. Mutually exclusive with `fromBlock`/`toBlock`. */ + txHash?: TxHash; + /** Lower block bound, inclusive. */ + fromBlock?: BlockNumber; + /** Upper block bound, exclusive. */ + toBlock?: BlockNumber; + /** Log cursor to resume pagination strictly after a previously-seen log. */ + afterLog?: LogCursor; + /** Node RPC URL. */ + nodeUrl: string; + /** When set, polls indefinitely for new logs. Incompatible with `txHash` and `toBlock`. */ + follow: boolean; + /** Log function. */ + log: LogFn; +}; + +/** + * Fetches public logs for a (contract, tag) pair, draining all pages via the stdlib pagination helper. + * In `--follow` mode, polls indefinitely: each round drains all currently-available logs, then sleeps + * until the next poll if nothing new was found. + */ +export async function getLogs(options: GetLogsOptions): Promise { + const { txHash, fromBlock, toBlock, contractAddress, tag, nodeUrl, follow, log } = options; + let afterLog = options.afterLog; if (follow) { if (txHash) { @@ -26,43 +51,50 @@ export async function getLogs( throw Error('Cannot use --follow with --to-block'); } } + if (txHash !== undefined && (fromBlock !== undefined || toBlock !== undefined)) { + throw Error('Cannot combine --tx-hash with --from-block / --to-block'); + } - const filter: LogFilter = { txHash, fromBlock, toBlock, afterLog, contractAddress }; - - const fetchLogs = async () => { - const response = await node.getPublicLogs(filter); - const logs = response.logs; + const node = createAztecNodeClient(nodeUrl); - if (!logs.length) { - const filterOptions = Object.entries(filter) - .filter(([, value]) => value !== undefined) - .map(([key, value]) => `${key}: ${value}`) - .join(', '); - if (!follow) { - log(`No logs found for filter: {${filterOptions}}`); - } - } else { - if (!follow && !filter.afterLog) { - log('Logs found: \n'); - } - logs.forEach(publicLog => log(publicLog.toHumanReadable())); - // Set the continuation parameter for the following requests - filter.afterLog = logs[logs.length - 1].id; + const drainLogs = async () => { + const query: PublicLogsQuery = { + contractAddress, + tags: [afterLog !== undefined ? { tag, afterLog } : tag], + fromBlock, + toBlock, + txHash, + }; + const [logsForTag] = await queryAllPublicLogsByTags(node, query); + if (logsForTag.length > 0) { + afterLog = LogCursor.fromLog(logsForTag[logsForTag.length - 1]); } - return response.maxLogsHit; + return logsForTag; }; if (follow) { log('Fetching logs...'); while (true) { - const maxLogsHit = await fetchLogs(); - if (!maxLogsHit) { + const results = await drainLogs(); + if (results.length === 0) { await sleep(1000); + } else { + results.forEach(r => log(logResultToHumanReadable(r))); } } } else { - while (await fetchLogs()) { - // Keep fetching logs until we reach the end. + const results = await drainLogs(); + if (results.length === 0) { + log( + `No logs found for {contractAddress: ${contractAddress.toString()}, tag: ${tag.toString()}` + + `${txHash ? `, txHash: ${txHash.toString()}` : ''}` + + `${fromBlock !== undefined ? `, fromBlock: ${fromBlock}` : ''}` + + `${toBlock !== undefined ? `, toBlock: ${toBlock}` : ''}` + + `${afterLog ? `, afterLog: ${afterLog.toString()}` : ''}}`, + ); + } else { + log('Logs found: \n'); + results.forEach(r => log(logResultToHumanReadable(r))); } } } diff --git a/yarn-project/cli/src/cmds/aztec_node/index.ts b/yarn-project/cli/src/cmds/aztec_node/index.ts index 71aa8494dc00..9b9b8e7a6b70 100644 --- a/yarn-project/cli/src/cmds/aztec_node/index.ts +++ b/yarn-project/cli/src/cmds/aztec_node/index.ts @@ -7,10 +7,10 @@ import { nodeOption, parseAztecAddress, parseField, - parseOptionalAztecAddress, parseOptionalInteger, - parseOptionalLogId, + parseOptionalLogCursor, parseOptionalTxHash, + parseTag, } from '../../utils/commands.js'; export function injectCommands(program: Command, log: LogFn, debugLogger: Logger) { @@ -47,21 +47,26 @@ export function injectCommands(program: Command, log: LogFn, debugLogger: Logger program .command('get-logs') - .description('Gets all the public logs from an intersection of all the filter params.') - .option('-tx, --tx-hash ', 'A transaction hash to get the receipt for.', parseOptionalTxHash) + .description('Gets public logs for a contract and tag, optionally restricted by block range or tx hash.') + .requiredOption('-ca, --contract-address
', 'Contract address that emitted the logs.', parseAztecAddress) + .requiredOption('--tag ', 'Tag (Fr value) to filter logs by.', parseTag) + .option('-tx, --tx-hash ', 'A transaction hash to restrict the search to.', parseOptionalTxHash) .option( '-fb, --from-block ', 'Initial block number for getting logs (defaults to 1).', parseOptionalInteger, ) .option('-tb, --to-block ', 'Up to which block to fetch logs (defaults to latest).', parseOptionalInteger) - .option('-al --after-log ', 'ID of a log after which to fetch the logs.', parseOptionalLogId) - .option('-ca, --contract-address
', 'Contract address to filter logs by.', parseOptionalAztecAddress) + .option( + '-al --after-log ', + 'Log cursor of the form -- to resume pagination after.', + parseOptionalLogCursor, + ) .addOption(nodeOption) .option('--follow', 'If set, will keep polling for new logs until interrupted.') - .action(async ({ txHash, fromBlock, toBlock, afterLog, contractAddress, aztecNodeRpcUrl: nodeUrl, follow }) => { + .action(async ({ txHash, fromBlock, toBlock, afterLog, contractAddress, tag, nodeUrl, follow }) => { const { getLogs } = await import('./get_logs.js'); - await getLogs(txHash, fromBlock, toBlock, afterLog, contractAddress, nodeUrl, follow, log); + await getLogs({ txHash, fromBlock, toBlock, afterLog, contractAddress, tag, nodeUrl, follow, log }); }); program diff --git a/yarn-project/cli/src/utils/commands.ts b/yarn-project/cli/src/utils/commands.ts index 6cc0009f568a..1a3c7e1d6cb1 100644 --- a/yarn-project/cli/src/utils/commands.ts +++ b/yarn-project/cli/src/utils/commands.ts @@ -5,13 +5,15 @@ import type { PXE } from '@aztec/pxe/server'; import { FunctionSelector } from '@aztec/stdlib/abi/function-selector'; import { AztecAddress } from '@aztec/stdlib/aztec-address'; import { PublicKeys } from '@aztec/stdlib/keys'; -import { LogId } from '@aztec/stdlib/logs/log-id'; +import { LogCursor, Tag } from '@aztec/stdlib/logs'; import { TxHash } from '@aztec/stdlib/tx/tx-hash'; import { type Command, CommanderError, InvalidArgumentError, Option } from 'commander'; import { lookup } from 'dns/promises'; import { rename, writeFile } from 'fs/promises'; +export { LogCursor }; + /** * If we can successfully resolve 'host.docker.internal', then we are running in a container, and we should treat * localhost as being host.docker.internal. @@ -227,16 +229,27 @@ export function parseOptionalAztecAddress(address: string): AztecAddress | undef } /** - * Parses an optional log ID string into a LogId object. - * - * @param logId - The log ID string to parse. - * @returns The parsed LogId object, or undefined if the log ID is missing or empty. + * Parses an optional `--` triple into a {@link LogCursor}, + * used as the `--after-log` argument of `get-logs` to resume pagination strictly after a previously-seen log. + * Thin wrapper over {@link LogCursor.parseOptional} that surfaces parse errors as commander's + * {@link InvalidArgumentError}. */ -export function parseOptionalLogId(logId: string): LogId | undefined { - if (!logId) { - return undefined; +export function parseOptionalLogCursor(value: string): LogCursor | undefined { + try { + return LogCursor.parseOptional(value); + } catch (err) { + throw new InvalidArgumentError(err instanceof Error ? err.message : String(err)); } - return LogId.fromString(logId); +} + +/** + * Parses a log tag from a string. Tags are field-element values; we delegate to the {@link parseField} parser. + * + * @param tag - A hex string, integer, or boolean string representing the tag. + * @returns A {@link Tag} wrapping the parsed field. + */ +export function parseTag(tag: string): Tag { + return new Tag(parseField(tag)); } /** diff --git a/yarn-project/end-to-end/src/bench/node_rpc_perf.test.ts b/yarn-project/end-to-end/src/bench/node_rpc_perf.test.ts index a0db5a60f32c..c2186f40ec23 100644 --- a/yarn-project/end-to-end/src/bench/node_rpc_perf.test.ts +++ b/yarn-project/end-to-end/src/bench/node_rpc_perf.test.ts @@ -560,31 +560,19 @@ describe('e2e_node_rpc_perf', () => { }); describe('log APIs', () => { - it('benchmarks getPublicLogs', async () => { - const { stats } = await benchmark('getPublicLogs', () => aztecNode.getPublicLogs({})); - addResult('getPublicLogs', stats); - expect(stats.avg).toBeLessThan(3000); - }); - - it('benchmarks getContractClassLogs', async () => { - const { stats } = await benchmark('getContractClassLogs', () => aztecNode.getContractClassLogs({})); - addResult('getContractClassLogs', stats); - expect(stats.avg).toBeLessThan(3000); - }); - it('benchmarks getPrivateLogsByTags', async () => { const tags = [SiloedTag.random()]; - const { stats } = await benchmark('getPrivateLogsByTags', () => aztecNode.getPrivateLogsByTags(tags)); + const { stats } = await benchmark('getPrivateLogsByTags', () => aztecNode.getPrivateLogsByTags({ tags })); addResult('getPrivateLogsByTags', stats); expect(stats.avg).toBeLessThan(3000); }); - it('benchmarks getPublicLogsByTagsFromContract', async () => { + it('benchmarks getPublicLogsByTags', async () => { const tags = [Tag.random()]; - const { stats } = await benchmark('getPublicLogsByTagsFromContract', () => - aztecNode.getPublicLogsByTagsFromContract(contractAddress, tags), + const { stats } = await benchmark('getPublicLogsByTags', () => + aztecNode.getPublicLogsByTags({ contractAddress, tags }), ); - addResult('getPublicLogsByTagsFromContract', stats); + addResult('getPublicLogsByTags', stats); expect(stats.avg).toBeLessThan(3000); }); }); diff --git a/yarn-project/end-to-end/src/e2e_deploy_contract/contract_class_registration.test.ts b/yarn-project/end-to-end/src/e2e_deploy_contract/contract_class_registration.test.ts index 71344027a78d..86b465607e15 100644 --- a/yarn-project/end-to-end/src/e2e_deploy_contract/contract_class_registration.test.ts +++ b/yarn-project/end-to-end/src/e2e_deploy_contract/contract_class_registration.test.ts @@ -57,14 +57,14 @@ describe('e2e_deploy_contract contract class registration', () => { const { receipt: publicationTxReceipt } = await publishContractClass(wallet, TestContract.artifact).then(c => c.send({ from: defaultAccountAddress }), ); - const logs = await aztecNode.getContractClassLogs({ txHash: publicationTxReceipt.txHash }); - expect(logs.logs.length).toEqual(1); + const txEffect = await aztecNode.getTxEffect(publicationTxReceipt.txHash); + expect(txEffect?.data.contractClassLogs.length).toEqual(1); }); it('registers the contract class on the node', async () => { - const logs = await aztecNode.getContractClassLogs({ txHash: publicationTxReceipt.txHash }); - expect(logs.logs.length).toEqual(1); - const logData = logs.logs[0].log.toBuffer(); + const txEffect = await aztecNode.getTxEffect(publicationTxReceipt.txHash); + expect(txEffect?.data.contractClassLogs.length).toEqual(1); + const logData = txEffect!.data.contractClassLogs[0].toBuffer(); // To actually trigger this write: // From `yarn-project/end-to-end/` diff --git a/yarn-project/end-to-end/src/e2e_deploy_contract/deploy_method.test.ts b/yarn-project/end-to-end/src/e2e_deploy_contract/deploy_method.test.ts index e9799ed57b7d..c194d6677265 100644 --- a/yarn-project/end-to-end/src/e2e_deploy_contract/deploy_method.test.ts +++ b/yarn-project/end-to-end/src/e2e_deploy_contract/deploy_method.test.ts @@ -150,8 +150,8 @@ describe('e2e_deploy_contract deploy method', () => { const { receipt } = await contract.methods .emit_public(arbitraryTag, arbitraryValue) .send({ from: defaultAccountAddress }); - const logs = await aztecNode.getPublicLogs({ txHash: receipt.txHash }); - expect(logs.logs[0].log.getEmittedFields()).toEqual([new Fr(arbitraryTag), new Fr(arbitraryValue)]); + const txEffect = await aztecNode.getTxEffect(receipt.txHash); + expect(txEffect?.data.publicLogs[0].getEmittedFields()).toEqual([new Fr(arbitraryTag), new Fr(arbitraryValue)]); }); it('refuses to deploy a contract with no constructor and no public deployment', async () => { diff --git a/yarn-project/end-to-end/src/e2e_event_logs.test.ts b/yarn-project/end-to-end/src/e2e_event_logs.test.ts index eada81d5429d..0afbd83f49f5 100644 --- a/yarn-project/end-to-end/src/e2e_event_logs.test.ts +++ b/yarn-project/end-to-end/src/e2e_event_logs.test.ts @@ -140,6 +140,7 @@ describe('Logs', () => { // docs:start:get_public_events const publicEventFilter: PublicEventFilter = { + contractAddress: testLogContract.address, fromBlock: BlockNumber(firstTx.blockNumber!), toBlock: BlockNumber(lastTx.blockNumber! + 1), }; @@ -193,6 +194,7 @@ describe('Logs', () => { aztecNode, TestLogContract.events.ExampleNestedEvent, { + contractAddress: testLogContract.address, fromBlock: BlockNumber(tx.blockNumber!), toBlock: BlockNumber(tx.blockNumber! + 1), }, diff --git a/yarn-project/end-to-end/src/e2e_large_public_event.test.ts b/yarn-project/end-to-end/src/e2e_large_public_event.test.ts index 34db71a443f1..f8247b01d179 100644 --- a/yarn-project/end-to-end/src/e2e_large_public_event.test.ts +++ b/yarn-project/end-to-end/src/e2e_large_public_event.test.ts @@ -41,6 +41,7 @@ describe('LargePublicEvent', () => { const { receipt: tx } = await contract.methods.emit_large_event(data).send({ from: accountAddress }); const { events } = await getPublicEvents(aztecNode, LargePublicEventContract.events.LargeEvent, { + contractAddress: contract.address, fromBlock: BlockNumber(tx.blockNumber!), toBlock: BlockNumber(tx.blockNumber! + 1), }); diff --git a/yarn-project/end-to-end/src/e2e_nested_contract/manual_public.test.ts b/yarn-project/end-to-end/src/e2e_nested_contract/manual_public.test.ts index 53ddaa2c06f6..bf3ede62be4b 100644 --- a/yarn-project/end-to-end/src/e2e_nested_contract/manual_public.test.ts +++ b/yarn-project/end-to-end/src/e2e_nested_contract/manual_public.test.ts @@ -50,14 +50,9 @@ describe('e2e_nested_contract manual', () => { ]; const { receipt: tx } = await new BatchCall(wallet, actions).send({ from: defaultAccountAddress }); - const extendedLogs = ( - await aztecNode.getPublicLogs({ - fromBlock: tx.blockNumber!, - }) - ).logs; - const processedLogs = extendedLogs.map(extendedLog => - toBigIntBE(serializeToBuffer(extendedLog.log.getEmittedFields())), - ); + const block = (await aztecNode.getBlock({ number: tx.blockNumber! }, { includeTransactions: true }))!; + const allPublicLogs = block.body.txEffects.flatMap(tx => tx.publicLogs); + const processedLogs = allPublicLogs.map(log => toBigIntBE(serializeToBuffer(log.getEmittedFields()))); expect(processedLogs).toEqual([20n, 40n]); expect(await getChildStoredValue(childContract)).toEqual(new Fr(40n)); }); diff --git a/yarn-project/end-to-end/src/e2e_orderbook.test.ts b/yarn-project/end-to-end/src/e2e_orderbook.test.ts index 6a959b27de86..e467aa789096 100644 --- a/yarn-project/end-to-end/src/e2e_orderbook.test.ts +++ b/yarn-project/end-to-end/src/e2e_orderbook.test.ts @@ -85,7 +85,7 @@ describe('Orderbook', () => { const { events: orderCreatedEvents } = await getPublicEvents( aztecNode, OrderbookContract.events.OrderCreated, - {}, + { contractAddress: orderbook.address }, ); expect(orderCreatedEvents.length).toBe(1); @@ -141,7 +141,7 @@ describe('Orderbook', () => { const { events: orderFulfilledEvents } = await getPublicEvents( aztecNode, OrderbookContract.events.OrderFulfilled, - {}, + { contractAddress: orderbook.address }, ); expect(orderFulfilledEvents.length).toBe(1); expect(orderFulfilledEvents[0].event.order_id).toEqual(orderId); diff --git a/yarn-project/end-to-end/src/e2e_ordering.test.ts b/yarn-project/end-to-end/src/e2e_ordering.test.ts index c7f15e8cc36b..342d2ae132d5 100644 --- a/yarn-project/end-to-end/src/e2e_ordering.test.ts +++ b/yarn-project/end-to-end/src/e2e_ordering.test.ts @@ -4,6 +4,7 @@ import { AztecAddress } from '@aztec/aztec.js/addresses'; import { Fr } from '@aztec/aztec.js/fields'; import type { AztecNode } from '@aztec/aztec.js/node'; import { toBigIntBE } from '@aztec/foundation/bigint-buffer'; +import { BlockNumber } from '@aztec/foundation/branded-types'; import { serializeToBuffer } from '@aztec/foundation/serialize'; import { ChildContract } from '@aztec/noir-test-contracts.js/Child'; import { ParentContract } from '@aztec/noir-test-contracts.js/Parent'; @@ -27,14 +28,15 @@ describe('e2e_ordering', () => { let defaultAccountAddress: AztecAddress; let teardown: () => Promise; - const expectLogsFromBlockToBe = async (logMessages: bigint[], fromBlock: number) => { - const logFilter = { - fromBlock, - toBlock: fromBlock + 1, - }; - const publicLogs = (await aztecNode.getPublicLogs(logFilter)).logs; - - const bigintLogs = publicLogs.map(extendedLog => toBigIntBE(serializeToBuffer(extendedLog.log.getEmittedFields()))); + const expectLogsFromBlockToBe = async (logMessages: bigint[], blockNumber: number) => { + // The log RPC is tag-based and per-contract; fetch the block's tx effects directly to assert ordering across all + // public logs in the block in canonical (txIndex, logIndexWithinTx) order. + const block = await aztecNode.getBlock(BlockNumber(blockNumber), { includeTransactions: true }); + if (!block) { + throw new Error(`Block ${blockNumber} not found`); + } + const publicLogs = block.body.txEffects.flatMap(txEffect => txEffect.publicLogs); + const bigintLogs = publicLogs.map(publicLog => toBigIntBE(serializeToBuffer(publicLog.getEmittedFields()))); expect(bigintLogs).toStrictEqual(logMessages); }; diff --git a/yarn-project/end-to-end/src/e2e_synching.test.ts b/yarn-project/end-to-end/src/e2e_synching.test.ts index 99e8c3101e7f..ed4cfb10adbe 100644 --- a/yarn-project/end-to-end/src/e2e_synching.test.ts +++ b/yarn-project/end-to-end/src/e2e_synching.test.ts @@ -637,9 +637,6 @@ describe('e2e_synching', () => { } expect(await archiver.getTxEffect(txHash)).not.toBeUndefined(); - expect( - await archiver.getPublicLogs({ fromBlock: blockTip.number, toBlock: blockTip.number + 1 }), - ).not.toEqual([]); await rollup.write.prune(); @@ -661,9 +658,6 @@ describe('e2e_synching', () => { ); expect(await archiver.getTxEffect(txHash)).toBeUndefined(); - expect(await archiver.getPublicLogs({ fromBlock: blockTip.number, toBlock: blockTip.number + 1 })).toEqual( - [], - ); // Check world state reverted as well const latestBlockNumber = await archiver.getBlockNumber(); diff --git a/yarn-project/foundation/src/config/env_var.ts b/yarn-project/foundation/src/config/env_var.ts index d631360d8b08..a0488d7d3bf7 100644 --- a/yarn-project/foundation/src/config/env_var.ts +++ b/yarn-project/foundation/src/config/env_var.ts @@ -5,7 +5,6 @@ export type EnvVar = | 'ACVM_WORKING_DIRECTORY' | 'API_KEY' | 'API_PREFIX' - | 'ARCHIVER_MAX_LOGS' | 'ARCHIVER_POLLING_INTERVAL_MS' | 'ARCHIVER_SKIP_HISTORICAL_LOGS_CHECK' | 'ARCHIVER_URL' diff --git a/yarn-project/node-lib/src/actions/snapshot-sync.ts b/yarn-project/node-lib/src/actions/snapshot-sync.ts index c6e51f33e12b..043ec102dcbe 100644 --- a/yarn-project/node-lib/src/actions/snapshot-sync.ts +++ b/yarn-project/node-lib/src/actions/snapshot-sync.ts @@ -12,6 +12,7 @@ import type { EthAddress } from '@aztec/foundation/eth-address'; import { tryRmDir } from '@aztec/foundation/fs'; import type { Logger } from '@aztec/foundation/log'; import { P2P_STORE_NAME } from '@aztec/p2p'; +import { GENESIS_BLOCK_HEADER_HASH } from '@aztec/stdlib/block'; import type { ChainConfig } from '@aztec/stdlib/config'; import { DatabaseVersionManager } from '@aztec/stdlib/database-version/manager'; import { type ReadOnlyFileStore, createReadOnlyFileStore } from '@aztec/stdlib/file-store'; @@ -36,7 +37,7 @@ const MIN_L1_BLOCKS_TO_TRIGGER_REPLACE = 86400 / 2 / 12; type SnapshotSyncConfig = Pick & Pick & Pick & - Pick & + Pick & DataStoreConfig & Required> & EthereumClientConfig & { @@ -65,9 +66,11 @@ export async function trySnapshotSync(config: SnapshotSyncConfig, log: Logger) { return false; } - // Create an archiver store to check the current state (do this only once) + // Create an archiver store to check the current state (do this only once). This temporary store is only + // read for its sync point and never serves tagged-log queries, so the genesis block hash it carries is + // immaterial — pass the protocol constant. log.verbose(`Creating temporary archiver data store`); - const archiverStore = await createArchiverStore(config); + const archiverStore = await createArchiverStore(config, GENESIS_BLOCK_HEADER_HASH); let archiverL1BlockNumber: bigint | undefined; let archiverL2BlockNumber: number | undefined; try { diff --git a/yarn-project/prover-node/src/actions/rerun-epoch-proving-job.ts b/yarn-project/prover-node/src/actions/rerun-epoch-proving-job.ts index ea0495973bca..486b53c3026b 100644 --- a/yarn-project/prover-node/src/actions/rerun-epoch-proving-job.ts +++ b/yarn-project/prover-node/src/actions/rerun-epoch-proving-job.ts @@ -32,7 +32,8 @@ export async function rerunEpochProvingJob( const telemetry = getTelemetryClient(); const metrics = new ProverNodeJobMetrics(telemetry.getMeter('prover-job'), telemetry.getTracer('prover-job')); const worldState = await createWorldState(config, genesis); - const archiver = await createArchiverStore(config); + const initialBlockHash = await worldState.getInitialHeader().hash(); + const archiver = await createArchiverStore(config, initialBlockHash); const publicProcessorFactory = new PublicProcessorFactory( createContractDataSource(archiver), undefined, diff --git a/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution.test.ts b/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution.test.ts index 2288551091e8..1019189ef0ca 100644 --- a/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution.test.ts +++ b/yarn-project/pxe/src/contract_function_simulator/oracle/private_execution.test.ts @@ -40,7 +40,6 @@ import type { AztecNode } from '@aztec/stdlib/interfaces/client'; import type { MerkleTreeWriteOperations } from '@aztec/stdlib/interfaces/server'; import { KeyValidationRequest } from '@aztec/stdlib/kernel'; import { computeAppNullifierHidingKey, deriveKeys } from '@aztec/stdlib/keys'; -import type { SiloedTag } from '@aztec/stdlib/logs'; import { L1Actor, L1ToL2Message, L2Actor } from '@aztec/stdlib/messaging'; import { Note, NoteDao } from '@aztec/stdlib/note'; import { makeBlockHeader, makeL2Tips, randomContractInstanceWithAddress } from '@aztec/stdlib/testing'; @@ -313,7 +312,7 @@ describe('Private Execution test suite', () => { // Mock aztec node methods - the return array needs to have the same length as the number of tags // on the input. - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => Promise.resolve(tags.map(() => []))); + aztecNode.getPrivateLogsByTags.mockImplementation(query => Promise.resolve(query.tags.map(() => []))); // Mock getL2Tips and getBlockHeader for syncTaggedPrivateLogs l2TipsStore.getL2Tips.mockResolvedValue(makeL2Tips(anchorBlockHeader.globalVariables.blockNumber)); diff --git a/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution.test.ts b/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution.test.ts index 97f827b410fd..8128f9670504 100644 --- a/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution.test.ts +++ b/yarn-project/pxe/src/contract_function_simulator/oracle/utility_execution.test.ts @@ -87,7 +87,7 @@ describe('Utility Execution test suite', () => { senderAddressBookStore.getSenders.mockResolvedValue([]); l2TipsStore.getL2Tips.mockResolvedValue(makeL2Tips(anchorBlockHeader.globalVariables.blockNumber)); - aztecNode.getPrivateLogsByTags.mockImplementation((tags: any[]) => Promise.resolve(tags.map(() => []))); + aztecNode.getPrivateLogsByTags.mockImplementation(query => Promise.resolve(query.tags.map(() => []))); capsuleStore.setCapsuleArray.mockImplementation((address, slot, content) => { capsuleArrays.set(`${address.toString()}:${slot.toString()}`, content); diff --git a/yarn-project/pxe/src/events/event_service.ts b/yarn-project/pxe/src/events/event_service.ts index ada53c685031..6f717a22f584 100644 --- a/yarn-project/pxe/src/events/event_service.ts +++ b/yarn-project/pxe/src/events/event_service.ts @@ -74,8 +74,8 @@ export class EventService { const txEffect = txEffects.get(txHash.toString()); if (!txEffect) { // We error out instead of just logging a warning and skipping the event because this would indicate a bug. This - // is because the node has already served info about this tx either when obtaining the log (TxScopedL2Log contain - // tx info) or when getting metadata for the offchain message (before the message got passed to `process_log`). + // is because the node has already served info about this tx either when obtaining the log (LogResult carries + // the tx info) or when getting metadata for the offchain message (before the message got passed to `process_log`). throw new Error(`Could not find tx effect for tx hash ${txHash} when processing an event.`); } diff --git a/yarn-project/pxe/src/logs/log_service.test.ts b/yarn-project/pxe/src/logs/log_service.test.ts index 7eb33274d934..f3b8385328aa 100644 --- a/yarn-project/pxe/src/logs/log_service.test.ts +++ b/yarn-project/pxe/src/logs/log_service.test.ts @@ -7,7 +7,7 @@ import { AztecAddress } from '@aztec/stdlib/aztec-address'; import type { L2TipsProvider } from '@aztec/stdlib/block'; import type { AztecNode } from '@aztec/stdlib/interfaces/server'; import { SiloedTag, Tag } from '@aztec/stdlib/logs'; -import { makeBlockHeader, randomTxScopedPrivateL2Log } from '@aztec/stdlib/testing'; +import { makeBlockHeader, randomPrivateLogResult } from '@aztec/stdlib/testing'; import { type MockProxy, mock } from 'jest-mock-extended'; @@ -40,7 +40,7 @@ describe('LogService', () => { aztecNode = mock(); aztecNode.getPrivateLogsByTags.mockReset(); - aztecNode.getPublicLogsByTagsFromContract.mockReset(); + aztecNode.getPublicLogsByTags.mockReset(); aztecNode.getTxEffect.mockReset(); // Set up anchor block header (required for bulkRetrieveLogs) @@ -60,17 +60,17 @@ describe('LogService', () => { it('returns empty arrays if no logs are found', async () => { aztecNode.getPrivateLogsByTags.mockResolvedValue([[]]); - aztecNode.getPublicLogsByTagsFromContract.mockResolvedValue([[]]); + aztecNode.getPublicLogsByTags.mockResolvedValue([[]]); const request = new LogRetrievalRequest(contractAddress, tag); const responses = await logService.fetchLogsByTag(contractAddress, [request]); expect(responses).toEqual([[]]); }); it('returns all logs when multiple public logs exist for a single tag', async () => { - const scopedLog1 = randomTxScopedPrivateL2Log(); - const scopedLog2 = randomTxScopedPrivateL2Log(); + const scopedLog1 = randomPrivateLogResult({ includeEffects: true }); + const scopedLog2 = randomPrivateLogResult({ includeEffects: true }); - aztecNode.getPublicLogsByTagsFromContract.mockResolvedValue([[scopedLog1, scopedLog2]]); + aztecNode.getPublicLogsByTags.mockResolvedValue([[scopedLog1, scopedLog2]]); aztecNode.getPrivateLogsByTags.mockResolvedValue([[]]); const request = new LogRetrievalRequest(contractAddress, tag); @@ -82,10 +82,10 @@ describe('LogService', () => { }); it('returns all logs when multiple private logs exist for a single tag', async () => { - const scopedLog1 = randomTxScopedPrivateL2Log(); - const scopedLog2 = randomTxScopedPrivateL2Log(); + const scopedLog1 = randomPrivateLogResult({ includeEffects: true }); + const scopedLog2 = randomPrivateLogResult({ includeEffects: true }); - aztecNode.getPublicLogsByTagsFromContract.mockResolvedValue([[]]); + aztecNode.getPublicLogsByTags.mockResolvedValue([[]]); aztecNode.getPrivateLogsByTags.mockResolvedValue([[scopedLog1, scopedLog2]]); const request = new LogRetrievalRequest(contractAddress, tag); @@ -97,10 +97,10 @@ describe('LogService', () => { }); it('returns combined public and private logs for a single tag', async () => { - const publicLog = randomTxScopedPrivateL2Log(); - const privateLog = randomTxScopedPrivateL2Log(); + const publicLog = randomPrivateLogResult({ includeEffects: true }); + const privateLog = randomPrivateLogResult({ includeEffects: true }); - aztecNode.getPublicLogsByTagsFromContract.mockResolvedValue([[publicLog]]); + aztecNode.getPublicLogsByTags.mockResolvedValue([[publicLog]]); aztecNode.getPrivateLogsByTags.mockResolvedValue([[privateLog]]); const request = new LogRetrievalRequest(contractAddress, tag); @@ -126,10 +126,10 @@ describe('LogService', () => { const tag2 = Tag.random(); const tag3 = Tag.random(); - const publicLog1 = randomTxScopedPrivateL2Log(); - const privateLog2 = randomTxScopedPrivateL2Log(); + const publicLog1 = randomPrivateLogResult({ includeEffects: true }); + const privateLog2 = randomPrivateLogResult({ includeEffects: true }); - aztecNode.getPublicLogsByTagsFromContract.mockResolvedValue([[publicLog1], [], []]); + aztecNode.getPublicLogsByTags.mockResolvedValue([[publicLog1], [], []]); aztecNode.getPrivateLogsByTags.mockResolvedValue([[], [privateLog2], []]); const requests = [ @@ -147,37 +147,40 @@ describe('LogService', () => { expect(responses[1][0].txHash).toEqual(privateLog2.txHash); expect(responses[2]).toEqual([]); - expect(aztecNode.getPublicLogsByTagsFromContract).toHaveBeenCalledTimes(1); + expect(aztecNode.getPublicLogsByTags).toHaveBeenCalledTimes(1); expect(aztecNode.getPrivateLogsByTags).toHaveBeenCalledTimes(1); }); it('returns empty array for empty requests', async () => { const responses = await logService.fetchLogsByTag(contractAddress, []); expect(responses).toEqual([]); - expect(aztecNode.getPublicLogsByTagsFromContract).not.toHaveBeenCalled(); + expect(aztecNode.getPublicLogsByTags).not.toHaveBeenCalled(); expect(aztecNode.getPrivateLogsByTags).not.toHaveBeenCalled(); }); describe('block range filtering', () => { - it('fromBlock is inclusive', async () => { - const logBefore = randomTxScopedPrivateL2Log({ blockNumber: 9 }); - const logAtBoundary = randomTxScopedPrivateL2Log({ blockNumber: 10 }); + // Range filtering happens in the node (Phase 2 pushed it down). These tests just verify the + // service forwards `fromBlock`/`toBlock` and stitches whatever the node returns. + it('forwards fromBlock to the node', async () => { + const logAtBoundary = randomPrivateLogResult({ blockNumber: 10, includeEffects: true }); - aztecNode.getPublicLogsByTagsFromContract.mockResolvedValue([[logBefore, logAtBoundary]]); + aztecNode.getPublicLogsByTags.mockResolvedValue([[logAtBoundary]]); aztecNode.getPrivateLogsByTags.mockResolvedValue([[]]); const request = new LogRetrievalRequest(contractAddress, tag, LogSource.PUBLIC_AND_PRIVATE, BlockNumber(10)); const responses = await logService.fetchLogsByTag(contractAddress, [request]); + expect(aztecNode.getPublicLogsByTags).toHaveBeenCalledWith( + expect.objectContaining({ fromBlock: BlockNumber(10) }), + ); expect(responses[0]).toHaveLength(1); expect(responses[0][0].txHash).toEqual(logAtBoundary.txHash); }); - it('toBlock is exclusive', async () => { - const logBeforeBoundary = randomTxScopedPrivateL2Log({ blockNumber: 9 }); - const logAtBoundary = randomTxScopedPrivateL2Log({ blockNumber: 10 }); + it('forwards toBlock to the node', async () => { + const logBeforeBoundary = randomPrivateLogResult({ blockNumber: 9, includeEffects: true }); - aztecNode.getPublicLogsByTagsFromContract.mockResolvedValue([[logBeforeBoundary, logAtBoundary]]); + aztecNode.getPublicLogsByTags.mockResolvedValue([[logBeforeBoundary]]); aztecNode.getPrivateLogsByTags.mockResolvedValue([[]]); const request = new LogRetrievalRequest( @@ -189,16 +192,17 @@ describe('LogService', () => { ); const responses = await logService.fetchLogsByTag(contractAddress, [request]); + expect(aztecNode.getPublicLogsByTags).toHaveBeenCalledWith( + expect.objectContaining({ toBlock: BlockNumber(10) }), + ); expect(responses[0]).toHaveLength(1); expect(responses[0][0].txHash).toEqual(logBeforeBoundary.txHash); }); - it('filters with both fromBlock and toBlock', async () => { - const logBefore = randomTxScopedPrivateL2Log({ blockNumber: 3 }); - const logInRange = randomTxScopedPrivateL2Log({ blockNumber: 15 }); - const logAfter = randomTxScopedPrivateL2Log({ blockNumber: 25 }); + it('forwards both fromBlock and toBlock to the node', async () => { + const logInRange = randomPrivateLogResult({ blockNumber: 15, includeEffects: true }); - aztecNode.getPublicLogsByTagsFromContract.mockResolvedValue([[logBefore, logInRange, logAfter]]); + aztecNode.getPublicLogsByTags.mockResolvedValue([[logInRange]]); aztecNode.getPrivateLogsByTags.mockResolvedValue([[]]); const request = new LogRetrievalRequest( @@ -210,6 +214,9 @@ describe('LogService', () => { ); const responses = await logService.fetchLogsByTag(contractAddress, [request]); + expect(aztecNode.getPublicLogsByTags).toHaveBeenCalledWith( + expect.objectContaining({ fromBlock: BlockNumber(10), toBlock: BlockNumber(20) }), + ); expect(responses[0]).toHaveLength(1); expect(responses[0][0].txHash).toEqual(logInRange.txHash); }); @@ -217,9 +224,9 @@ describe('LogService', () => { describe('source filtering', () => { it('returns only public logs and skips private RPC when source is PUBLIC', async () => { - const publicLog = randomTxScopedPrivateL2Log(); + const publicLog = randomPrivateLogResult({ includeEffects: true }); - aztecNode.getPublicLogsByTagsFromContract.mockResolvedValue([[publicLog]]); + aztecNode.getPublicLogsByTags.mockResolvedValue([[publicLog]]); const request = new LogRetrievalRequest(contractAddress, tag, LogSource.PUBLIC); const responses = await logService.fetchLogsByTag(contractAddress, [request]); @@ -230,7 +237,7 @@ describe('LogService', () => { }); it('returns only private logs and skips public RPC when source is PRIVATE', async () => { - const privateLog = randomTxScopedPrivateL2Log(); + const privateLog = randomPrivateLogResult({ includeEffects: true }); aztecNode.getPrivateLogsByTags.mockResolvedValue([[privateLog]]); @@ -239,7 +246,7 @@ describe('LogService', () => { expect(responses[0]).toHaveLength(1); expect(responses[0][0].txHash).toEqual(privateLog.txHash); - expect(aztecNode.getPublicLogsByTagsFromContract).not.toHaveBeenCalled(); + expect(aztecNode.getPublicLogsByTags).not.toHaveBeenCalled(); }); it('only sends relevant tags per source in a mixed batch', async () => { @@ -247,12 +254,12 @@ describe('LogService', () => { const tag2 = Tag.random(); const tag3 = Tag.random(); - const publicLog1 = randomTxScopedPrivateL2Log(); - const privateLog2 = randomTxScopedPrivateL2Log(); - const publicLog3 = randomTxScopedPrivateL2Log(); - const privateLog3 = randomTxScopedPrivateL2Log(); + const publicLog1 = randomPrivateLogResult({ includeEffects: true }); + const privateLog2 = randomPrivateLogResult({ includeEffects: true }); + const publicLog3 = randomPrivateLogResult({ includeEffects: true }); + const privateLog3 = randomPrivateLogResult({ includeEffects: true }); - aztecNode.getPublicLogsByTagsFromContract.mockResolvedValue([[publicLog1], [publicLog3]]); + aztecNode.getPublicLogsByTags.mockResolvedValue([[publicLog1], [publicLog3]]); aztecNode.getPrivateLogsByTags.mockResolvedValue([[privateLog2], [privateLog3]]); const requests = [ @@ -264,14 +271,14 @@ describe('LogService', () => { const responses = await logService.fetchLogsByTag(contractAddress, requests); // Public RPC receives tag1 and tag3, private RPC receives tag2 and tag3 - expect(aztecNode.getPublicLogsByTagsFromContract).toHaveBeenCalledTimes(1); - const publicCallTags = aztecNode.getPublicLogsByTagsFromContract.mock.calls[0][1]; + expect(aztecNode.getPublicLogsByTags).toHaveBeenCalledTimes(1); + const publicCallTags = aztecNode.getPublicLogsByTags.mock.calls[0][0].tags as Tag[]; expect(publicCallTags).toHaveLength(2); expect(publicCallTags[0]).toEqual(tag1); expect(publicCallTags[1]).toEqual(tag3); expect(aztecNode.getPrivateLogsByTags).toHaveBeenCalledTimes(1); - const privateCallTags = aztecNode.getPrivateLogsByTags.mock.calls[0][0]; + const privateCallTags = aztecNode.getPrivateLogsByTags.mock.calls[0][0].tags as SiloedTag[]; expect(privateCallTags).toHaveLength(2); const expectedSiloedTag2 = await SiloedTag.computeFromTagAndApp(tag2, contractAddress); const expectedSiloedTag3 = await SiloedTag.computeFromTagAndApp(tag3, contractAddress); diff --git a/yarn-project/pxe/src/logs/log_service.ts b/yarn-project/pxe/src/logs/log_service.ts index ddbcf205a57c..027c6dc7a267 100644 --- a/yarn-project/pxe/src/logs/log_service.ts +++ b/yarn-project/pxe/src/logs/log_service.ts @@ -4,7 +4,7 @@ import type { KeyStore } from '@aztec/key-store'; import { AztecAddress } from '@aztec/stdlib/aztec-address'; import type { BlockHash, L2TipsProvider } from '@aztec/stdlib/block'; import type { AztecNode } from '@aztec/stdlib/interfaces/server'; -import { AppTaggingSecret, PendingTaggedLog, SiloedTag, type TxScopedL2Log } from '@aztec/stdlib/logs'; +import { AppTaggingSecret, type LogResult, PendingTaggedLog, SiloedTag } from '@aztec/stdlib/logs'; import type { BlockHeader } from '@aztec/stdlib/tx'; import { @@ -21,6 +21,10 @@ import { syncTaggedPrivateLogs, } from '../tagging/index.js'; +/** Key used to group requests by their (fromBlock, toBlock) range so each group becomes a single node call. */ +type RangeKey = string; +const rangeKey = (fromBlock?: BlockNumber, toBlock?: BlockNumber): RangeKey => `${fromBlock ?? ''}-${toBlock ?? ''}`; + export class LogService { private log: Logger; @@ -55,14 +59,14 @@ export class LogService { const anchorBlockHash = await this.anchorBlockHeader.hash(); - const [publicLogsPerTag, privateLogsPerTag] = await Promise.all([ + const [publicLogsPerRequest, privateLogsPerRequest] = await Promise.all([ this.#fetchPublicLogs(contractAddress, logRetrievalRequests, anchorBlockHash), this.#fetchPrivateLogs(logRetrievalRequests, anchorBlockHash), ]); - return logRetrievalRequests.map((request, i) => [ - ...this.#extractLogs(publicLogsPerTag[i], request.fromBlock, request.toBlock), - ...this.#extractLogs(privateLogsPerTag[i], request.fromBlock, request.toBlock), + return logRetrievalRequests.map((_request, i) => [ + ...publicLogsPerRequest[i].map(LogService.#toLogRetrievalResponse), + ...privateLogsPerRequest[i].map(LogService.#toLogRetrievalResponse), ]); } @@ -70,64 +74,95 @@ export class LogService { contractAddress: AztecAddress, requests: LogRetrievalRequest[], anchorBlockHash: BlockHash, - ): Promise { + ): Promise { const indices = requests.flatMap((r, i) => (r.source !== LogSource.PRIVATE ? [i] : [])); if (indices.length === 0) { return requests.map(() => []); } - const results = await getAllPublicLogsByTagsFromContract( - this.aztecNode, - contractAddress, - indices.map(i => requests[i].tag), - anchorBlockHash, + const resultsPerRequest: LogResult[][] = requests.map(() => []); + const groups = LogService.#groupByRange(indices.map(i => ({ index: i, request: requests[i] }))); + + await Promise.all( + Array.from(groups.values()).map(async group => { + const tags = group.entries.map(e => e.request.tag); + const results = await getAllPublicLogsByTagsFromContract( + this.aztecNode, + contractAddress, + tags, + anchorBlockHash, + { fromBlock: group.fromBlock, toBlock: group.toBlock, includeEffects: true }, + ); + group.entries.forEach((entry, i) => { + resultsPerRequest[entry.index] = results[i]; + }); + }), ); - const logsPerTag: TxScopedL2Log[][] = requests.map(() => []); - indices.forEach((originalIdx, resultIdx) => { - logsPerTag[originalIdx] = results[resultIdx]; - }); - return logsPerTag; + return resultsPerRequest; } - async #fetchPrivateLogs(requests: LogRetrievalRequest[], anchorBlockHash: BlockHash): Promise { + async #fetchPrivateLogs(requests: LogRetrievalRequest[], anchorBlockHash: BlockHash): Promise { const indices = requests.flatMap((r, i) => (r.source !== LogSource.PUBLIC ? [i] : [])); if (indices.length === 0) { return requests.map(() => []); } - const siloedTags = await Promise.all( - indices.map(i => SiloedTag.computeFromTagAndApp(requests[i].tag, requests[i].contractAddress)), + const resultsPerRequest: LogResult[][] = requests.map(() => []); + const groups = LogService.#groupByRange(indices.map(i => ({ index: i, request: requests[i] }))); + + await Promise.all( + Array.from(groups.values()).map(async group => { + const siloedTags = await Promise.all( + group.entries.map(e => SiloedTag.computeFromTagAndApp(e.request.tag, e.request.contractAddress)), + ); + const results = await getAllPrivateLogsByTags(this.aztecNode, siloedTags, anchorBlockHash, { + fromBlock: group.fromBlock, + toBlock: group.toBlock, + includeEffects: true, + }); + group.entries.forEach((entry, i) => { + resultsPerRequest[entry.index] = results[i]; + }); + }), ); - const results = await getAllPrivateLogsByTags(this.aztecNode, siloedTags, anchorBlockHash); + return resultsPerRequest; + } - const logsPerTag: TxScopedL2Log[][] = requests.map(() => []); - indices.forEach((originalIdx, resultIdx) => { - logsPerTag[originalIdx] = results[resultIdx]; - }); - return logsPerTag; + /** + * Groups requests by their (fromBlock, toBlock) range so each distinct range becomes a single node call with + * the range pushed down into the query (no in-memory filter). + */ + static #groupByRange( + entries: Array<{ index: number; request: LogRetrievalRequest }>, + ): Map { + const groups = new Map(); + for (const entry of entries) { + const key = rangeKey(entry.request.fromBlock, entry.request.toBlock); + const existing = groups.get(key); + if (existing) { + existing.entries.push(entry); + } else { + groups.set(key, { fromBlock: entry.request.fromBlock, toBlock: entry.request.toBlock, entries: [entry] }); + } + } + return groups; } - #extractLogs(logsForTag: TxScopedL2Log[], fromBlock?: BlockNumber, toBlock?: BlockNumber): LogRetrievalResponse[] { - // TODO(F-650): push the block range filter down to the node query instead of filtering in memory. - const filtered = - fromBlock !== undefined || toBlock !== undefined - ? logsForTag.filter( - log => - (fromBlock === undefined || log.blockNumber >= fromBlock) && - (toBlock === undefined || log.blockNumber < toBlock), - ) - : logsForTag; - - return filtered.map( - scopedLog => - new LogRetrievalResponse( - scopedLog.logData.slice(1), // Skip the tag - scopedLog.txHash, - scopedLog.noteHashes, - scopedLog.firstNullifier, - ), + static #toLogRetrievalResponse(log: LogResult): LogRetrievalResponse { + // includeEffects: true was used, so noteHashes and nullifiers are populated. Every tx has at least one nullifier + // (the first nullifier derived from the tx hash); empty here would indicate a buggy node. + const noteHashes = log.noteHashes!; + const nullifiers = log.nullifiers!; + if (nullifiers.length === 0) { + throw new Error(`Log for tx ${log.txHash} returned no nullifiers from the node`); + } + return new LogRetrievalResponse( + log.logData.slice(1), // Skip the tag + log.txHash, + noteHashes, + nullifiers[0], ); } @@ -147,10 +182,14 @@ export class LogService { this.jobId, ); - return logs.map( - scopedLog => - new PendingTaggedLog(scopedLog.logData, scopedLog.txHash, scopedLog.noteHashes, scopedLog.firstNullifier), - ); + return logs.map(log => { + const noteHashes = log.noteHashes!; + const nullifiers = log.nullifiers!; + if (nullifiers.length === 0) { + throw new Error(`Log for tx ${log.txHash} returned no nullifiers from the node`); + } + return new PendingTaggedLog(log.logData, log.txHash, noteHashes, nullifiers[0]); + }); } async #getSecretsForSenders(contractAddress: AztecAddress, recipient: AztecAddress): Promise { diff --git a/yarn-project/pxe/src/notes/note_service.ts b/yarn-project/pxe/src/notes/note_service.ts index 7695d8614307..cbe1ff82624b 100644 --- a/yarn-project/pxe/src/notes/note_service.ts +++ b/yarn-project/pxe/src/notes/note_service.ts @@ -162,8 +162,8 @@ export class NoteService { const txEffect = txEffects.get(txHash.toString()); if (!txEffect) { // We error out instead of just logging a warning and skipping the note because this would indicate a bug. This - // is because the node has already served info about this tx either when obtaining the log (TxScopedL2Log - // contain tx info) or when getting metadata for the offchain message (before the message got passed to + // is because the node has already served info about this tx either when obtaining the log (LogResult carries + // the tx info) or when getting metadata for the offchain message (before the message got passed to // `process_log`). throw new Error(`Could not find tx effect for tx hash ${txHash} when processing a note.`); } diff --git a/yarn-project/pxe/src/pxe.test.ts b/yarn-project/pxe/src/pxe.test.ts index a88900347f8a..4528144f5bc3 100644 --- a/yarn-project/pxe/src/pxe.test.ts +++ b/yarn-project/pxe/src/pxe.test.ts @@ -18,7 +18,6 @@ import { import { emptyChainConfig } from '@aztec/stdlib/config'; import { getContractClassFromArtifact } from '@aztec/stdlib/contract'; import type { AztecNode, BlockResponse } from '@aztec/stdlib/interfaces/client'; -import { SiloedTag } from '@aztec/stdlib/logs'; import { randomContractArtifact, randomContractInstanceWithAddress, @@ -274,7 +273,7 @@ describe('PXE', () => { // Used to sync private logs from the node - the return array needs to have the same length as the number of tags // on the input. - node.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => Promise.resolve(tags.map(() => []))); + node.getPrivateLogsByTags.mockImplementation(query => Promise.resolve(query.tags.map(() => []))); // Necessary to sync contract private state await pxe.registerContractClass(TestContractArtifact); diff --git a/yarn-project/pxe/src/tagging/get_all_logs_by_tags.test.ts b/yarn-project/pxe/src/tagging/get_all_logs_by_tags.test.ts index c6b44ff48047..c55f0e1abe81 100644 --- a/yarn-project/pxe/src/tagging/get_all_logs_by_tags.test.ts +++ b/yarn-project/pxe/src/tagging/get_all_logs_by_tags.test.ts @@ -1,19 +1,30 @@ +import { BlockNumber } from '@aztec/foundation/branded-types'; import { AztecAddress } from '@aztec/stdlib/aztec-address'; import { BlockHash } from '@aztec/stdlib/block'; import { MAX_LOGS_PER_TAG, MAX_RPC_LEN } from '@aztec/stdlib/interfaces/api-limit'; import type { AztecNode } from '@aztec/stdlib/interfaces/server'; -import { SiloedTag, Tag } from '@aztec/stdlib/logs'; -import { randomTxScopedPrivateL2Log } from '@aztec/stdlib/testing'; +import { LogCursor, type LogResult, type PrivateLogsQuery, SiloedTag, Tag, randomLogResult } from '@aztec/stdlib/logs'; import { type MockProxy, mock } from 'jest-mock-extended'; import { getAllPrivateLogsByTags } from './get_all_logs_by_tags.js'; // We don't bother testing getAllPublicLogsByTagsFromContract because both of the functions are a simple wrapper around -// getAllPages function so just testing the private logs function is enough. +// the same per-tag pagination loop, so testing the private logs function is enough. const MOCK_ANCHOR_BLOCK_HASH = BlockHash.random(); +/** Builds a log with a stable blockNumber/logIndexWithinTx so we can assert cursor wiring. */ +function makeLog({ blockNumber = 1, logIndexWithinTx = 0 }: { blockNumber?: number; logIndexWithinTx?: number } = {}) { + return { ...randomLogResult(), blockNumber: BlockNumber(blockNumber), logIndexWithinTx }; +} + +/** Convenience: build a same-length array of cloned logs that share a cursor target (the last one). */ +function fillPage(size: number, lastLog: LogResult): LogResult[] { + const filler = Array.from({ length: size - 1 }, (_, i) => makeLog({ logIndexWithinTx: i })); + return [...filler, lastLog]; +} + describe('getAllPrivateLogsByTags', () => { let aztecNode: MockProxy; let tags: SiloedTag[]; @@ -34,11 +45,17 @@ describe('getAllPrivateLogsByTags', () => { const result = await getAllPrivateLogsByTags(aztecNode, tags, MOCK_ANCHOR_BLOCK_HASH); expect(result).toEqual([[], [], []]); - expect(aztecNode.getPrivateLogsByTags).toHaveBeenCalledWith(tags, 0, MOCK_ANCHOR_BLOCK_HASH); + expect(aztecNode.getPrivateLogsByTags).toHaveBeenCalledWith({ + tags, + referenceBlock: MOCK_ANCHOR_BLOCK_HASH, + fromBlock: undefined, + toBlock: undefined, + includeEffects: false, + } satisfies PrivateLogsQuery); }); it('returns logs when all fit in a single page', async () => { - const logsPerTag = tags.map((tag, i) => Array(i + 1).fill(randomTxScopedPrivateL2Log({ tag: tag.value }))); + const logsPerTag = tags.map((_tag, i) => Array.from({ length: i + 1 }, () => makeLog())); aztecNode.getPrivateLogsByTags.mockResolvedValue(logsPerTag); const result = await getAllPrivateLogsByTags(aztecNode, tags, MOCK_ANCHOR_BLOCK_HASH); @@ -47,21 +64,35 @@ describe('getAllPrivateLogsByTags', () => { expect(aztecNode.getPrivateLogsByTags).toHaveBeenCalledTimes(1); }); - it('paginates when any tag has MAX_LOGS_PER_TAG logs', async () => { - const firstPage = [ - Array(MAX_LOGS_PER_TAG).fill(randomTxScopedPrivateL2Log({ tag: tags[0].value })), - [randomTxScopedPrivateL2Log({ tag: tags[1].value })], - [], - ]; - const secondPage = [Array(5).fill(randomTxScopedPrivateL2Log({ tag: tags[0].value })), [], []]; + it('paginates only tags that returned a full page, using their afterLog cursor', async () => { + const lastLogOfFirstPage = makeLog({ blockNumber: 42, logIndexWithinTx: 9 }); + const firstPage = [fillPage(MAX_LOGS_PER_TAG, lastLogOfFirstPage), [makeLog()], []]; + const secondPage = [Array.from({ length: 5 }, () => makeLog())]; aztecNode.getPrivateLogsByTags.mockResolvedValueOnce(firstPage).mockResolvedValueOnce(secondPage); const result = await getAllPrivateLogsByTags(aztecNode, tags, MOCK_ANCHOR_BLOCK_HASH); expect(result.map(logs => logs.length)).toEqual([MAX_LOGS_PER_TAG + 5, 1, 0]); - expect(aztecNode.getPrivateLogsByTags).toHaveBeenNthCalledWith(1, tags, 0, MOCK_ANCHOR_BLOCK_HASH); - expect(aztecNode.getPrivateLogsByTags).toHaveBeenNthCalledWith(2, tags, 1, MOCK_ANCHOR_BLOCK_HASH); + expect(aztecNode.getPrivateLogsByTags).toHaveBeenCalledTimes(2); + + // Round 1: all tags queried with bare tags + expect(aztecNode.getPrivateLogsByTags).toHaveBeenNthCalledWith(1, { + tags, + referenceBlock: MOCK_ANCHOR_BLOCK_HASH, + fromBlock: undefined, + toBlock: undefined, + includeEffects: false, + }); + + // Round 2: only tag[0] re-queried, with an afterLog cursor pointing at the last log of round 1 + expect(aztecNode.getPrivateLogsByTags).toHaveBeenNthCalledWith(2, { + tags: [{ tag: tags[0], afterLog: LogCursor.fromLog(lastLogOfFirstPage) }], + referenceBlock: MOCK_ANCHOR_BLOCK_HASH, + fromBlock: undefined, + toBlock: undefined, + includeEffects: false, + }); }); it('handles empty tags array', async () => { @@ -70,6 +101,27 @@ describe('getAllPrivateLogsByTags', () => { const result = await getAllPrivateLogsByTags(aztecNode, [], MOCK_ANCHOR_BLOCK_HASH); expect(result).toEqual([]); + expect(aztecNode.getPrivateLogsByTags).not.toHaveBeenCalled(); + }); + + it('forwards options (fromBlock/toBlock/includeEffects/limitPerTag) to the node', async () => { + aztecNode.getPrivateLogsByTags.mockResolvedValue(tags.map(() => [])); + + await getAllPrivateLogsByTags(aztecNode, tags, MOCK_ANCHOR_BLOCK_HASH, { + fromBlock: BlockNumber(5), + toBlock: BlockNumber(10), + includeEffects: true, + limitPerTag: 3, + }); + + expect(aztecNode.getPrivateLogsByTags).toHaveBeenCalledWith({ + tags, + referenceBlock: MOCK_ANCHOR_BLOCK_HASH, + fromBlock: BlockNumber(5), + toBlock: BlockNumber(10), + includeEffects: true, + limitPerTag: 3, + }); }); describe('batching when tags exceed MAX_RPC_LEN', () => { @@ -83,8 +135,8 @@ describe('getAllPrivateLogsByTags', () => { const batch1Tags = manyTags.slice(0, MAX_RPC_LEN); const batch2Tags = manyTags.slice(MAX_RPC_LEN); - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { - return Promise.resolve(tags.map(tag => [randomTxScopedPrivateL2Log({ tag: tag.value })])); + aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => { + return Promise.resolve(query.tags.map(() => [makeLog()])); }); const result = await getAllPrivateLogsByTags(aztecNode, manyTags, MOCK_ANCHOR_BLOCK_HASH); @@ -94,30 +146,42 @@ describe('getAllPrivateLogsByTags', () => { // Should have been called twice: once per batch expect(aztecNode.getPrivateLogsByTags).toHaveBeenCalledTimes(2); - expect(aztecNode.getPrivateLogsByTags).toHaveBeenNthCalledWith(1, batch1Tags, 0, MOCK_ANCHOR_BLOCK_HASH); - expect(aztecNode.getPrivateLogsByTags).toHaveBeenNthCalledWith(2, batch2Tags, 0, MOCK_ANCHOR_BLOCK_HASH); + expect(aztecNode.getPrivateLogsByTags).toHaveBeenNthCalledWith(1, { + tags: batch1Tags, + referenceBlock: MOCK_ANCHOR_BLOCK_HASH, + fromBlock: undefined, + toBlock: undefined, + includeEffects: false, + }); + expect(aztecNode.getPrivateLogsByTags).toHaveBeenNthCalledWith(2, { + tags: batch2Tags, + referenceBlock: MOCK_ANCHOR_BLOCK_HASH, + fromBlock: undefined, + toBlock: undefined, + includeEffects: false, + }); }); - it('paginates within each batch', async () => { - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[], page: number) => { - if (tags.length === MAX_RPC_LEN && page === 0) { - // First batch, first page: first tag has MAX_LOGS_PER_TAG logs (triggers pagination) + it('paginates within each batch and only re-queries tags with a full page', async () => { + const lastLog = makeLog({ blockNumber: 99, logIndexWithinTx: 9 }); + + aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => { + // First batch (MAX_RPC_LEN tags), round 1: index 0 returns a full page (last log pinned to `lastLog`), + // every other tag returns a single log. + if (query.tags.length === MAX_RPC_LEN) { return Promise.resolve( - tags.map((tag, i) => - i === 0 - ? Array(MAX_LOGS_PER_TAG).fill(randomTxScopedPrivateL2Log({ tag: tag.value })) - : [randomTxScopedPrivateL2Log({ tag: tag.value })], - ), + query.tags.map((_t: unknown, i: number) => (i === 0 ? fillPage(MAX_LOGS_PER_TAG, lastLog) : [makeLog()])), ); } - if (tags.length === MAX_RPC_LEN && page === 1) { - // First batch, second page: 3 more logs for first tag - return Promise.resolve( - tags.map((tag, i) => (i === 0 ? Array(3).fill(randomTxScopedPrivateL2Log({ tag: tag.value })) : [])), - ); + // First batch, round 2: only tag[0] is re-queried with afterLog cursor. + if (query.tags.length === 1) { + const tagEntry = query.tags[0]; + expect(typeof tagEntry).toBe('object'); + expect((tagEntry as { afterLog: LogCursor }).afterLog.equals(LogCursor.fromLog(lastLog))).toBe(true); + return Promise.resolve([Array.from({ length: 3 }, () => makeLog())]); } - // Second batch (50 tags), single page - return Promise.resolve(tags.map(tag => [randomTxScopedPrivateL2Log({ tag: tag.value })])); + // Second batch (50 tags), single round + return Promise.resolve(query.tags.map(() => [makeLog()])); }); const result = await getAllPrivateLogsByTags(aztecNode, manyTags, MOCK_ANCHOR_BLOCK_HASH); @@ -130,20 +194,8 @@ describe('getAllPrivateLogsByTags', () => { // Tags in batch 2 got 1 log each expect(result[MAX_RPC_LEN]).toHaveLength(1); - // 2 pages for first batch + 1 page for second batch = 3 calls + // 2 rounds for first batch + 1 round for second batch = 3 calls expect(aztecNode.getPrivateLogsByTags).toHaveBeenCalledTimes(3); }); - - it('does not batch when tags fit within MAX_RPC_LEN', async () => { - const exactlyMaxTags = manyTags.slice(0, MAX_RPC_LEN); - - aztecNode.getPrivateLogsByTags.mockResolvedValue(exactlyMaxTags.map(() => [])); - - const result = await getAllPrivateLogsByTags(aztecNode, exactlyMaxTags, MOCK_ANCHOR_BLOCK_HASH); - - expect(result).toHaveLength(MAX_RPC_LEN); - expect(aztecNode.getPrivateLogsByTags).toHaveBeenCalledTimes(1); - expect(aztecNode.getPrivateLogsByTags).toHaveBeenCalledWith(exactlyMaxTags, 0, MOCK_ANCHOR_BLOCK_HASH); - }); }); }); diff --git a/yarn-project/pxe/src/tagging/get_all_logs_by_tags.ts b/yarn-project/pxe/src/tagging/get_all_logs_by_tags.ts index a3c832d2a25b..c18663a24fa5 100644 --- a/yarn-project/pxe/src/tagging/get_all_logs_by_tags.ts +++ b/yarn-project/pxe/src/tagging/get_all_logs_by_tags.ts @@ -1,44 +1,45 @@ +import type { BlockNumber } from '@aztec/foundation/branded-types'; import type { AztecAddress } from '@aztec/stdlib/aztec-address'; import type { BlockHash } from '@aztec/stdlib/block'; -import { MAX_LOGS_PER_TAG, MAX_RPC_LEN } from '@aztec/stdlib/interfaces/api-limit'; +import { MAX_RPC_LEN } from '@aztec/stdlib/interfaces/api-limit'; import type { AztecNode } from '@aztec/stdlib/interfaces/client'; -import type { SiloedTag, Tag, TxScopedL2Log } from '@aztec/stdlib/logs'; +import { + type LogIncludeOptions, + type LogResult, + type SiloedTag, + type Tag, + queryAllPrivateLogsByTags, + queryAllPublicLogsByTags, +} from '@aztec/stdlib/logs'; -/** - * Generic pagination helper that fetches all pages of results. - * @param numTags - The number of tags being queried (determines result array size). - * @param fetchPage - Function that fetches a single page of results given a page number. - * @returns An array of arrays, one per tag, containing all results across all pages. - */ -async function getAllPages(numTags: number, fetchPage: (page: number) => Promise): Promise { - const allResultsPerTag: T[][] = Array.from({ length: numTags }, () => []); - let page = 0; - let hasMore = true; - - while (hasMore) { - const resultsPage = await fetchPage(page); - hasMore = false; - - for (let i = 0; i < resultsPage.length; i++) { - allResultsPerTag[i].push(...resultsPage[i]); - if (resultsPage[i].length === MAX_LOGS_PER_TAG) { - hasMore = true; - } - } - page++; - } - - return allResultsPerTag; -} +/** Optional block-range, effects opt-in, and pagination cap shared by both wrappers. */ +export type GetAllLogsByTagsOptions = { + /** Lower block bound, inclusive. */ + fromBlock?: BlockNumber; + /** Upper block bound, exclusive. */ + toBlock?: BlockNumber; + /** + * When set, each log also carries `noteHashes` and all `nullifiers` of its tx. Defaults to off — sender + * sync only needs `txHash`. The recipient-sync/log-service paths flip this on to build `PendingTaggedLog` + * / `LogRetrievalResponse` from note data. + */ + includeEffects?: boolean; + /** + * Maximum number of logs returned per tag per round-trip. Capped at `MAX_LOGS_PER_TAG` on the node + * (rejected if higher). Defaults to `MAX_LOGS_PER_TAG`. Mainly useful for tests that need to force + * pagination at a small page size. + */ + limitPerTag?: number; +}; /** - * Splits tags into chunks of MAX_RPC_LEN, fetches logs for each chunk using getAllPages, then stitches the results - * back into a single array preserving the original tag order. + * Splits tags into chunks of MAX_RPC_LEN, paginates each chunk via the stdlib per-tag cursor helper, + * then stitches the results back into a single array preserving the original tag order. */ -async function getAllPagesInBatches( - tags: Tag[], - fetchAllPagesForBatch: (batch: Tag[]) => Promise, -): Promise { +async function getAllPagesInBatches( + tags: T[], + fetchAllPagesForBatch: (batch: T[]) => Promise[][]>, +): Promise[][]> { if (tags.length === 0) { return []; } @@ -47,7 +48,7 @@ async function getAllPagesInBatches( return fetchAllPagesForBatch(tags); } - const batches: Tag[][] = []; + const batches: T[][] = []; for (let i = 0; i < tags.length; i += MAX_RPC_LEN) { batches.push(tags.slice(i, i + MAX_RPC_LEN)); } @@ -56,41 +57,64 @@ async function getAllPagesInBatches( } /** - * Fetches all private logs for the given tags, automatically paginating through all pages. + * Fetches all private logs for the given tags, automatically paginating per-tag via `afterLog` cursors. + * * @param aztecNode - The Aztec node to query. * @param tags - The siloed tags to search for. - * @param anchorBlockHash - reference block for the Aztec node query, throws if block is not found there (typically - * because of reorgs). + * @param anchorBlockHash - Reference block for the Aztec node query, throws if block is not found there (typically + * because of reorgs). + * @param options - Optional `fromBlock`/`toBlock` range and `includeEffects` opt-in. * @returns An array of log arrays, one per tag, containing all logs across all pages. */ -export function getAllPrivateLogsByTags( +export function getAllPrivateLogsByTags( aztecNode: AztecNode, tags: SiloedTag[], anchorBlockHash: BlockHash, -): Promise { - return getAllPagesInBatches(tags, batch => - getAllPages(batch.length, page => aztecNode.getPrivateLogsByTags(batch, page, anchorBlockHash)), + options: Opts = {} as Opts, +): Promise[][]> { + return getAllPagesInBatches( + tags, + batch => + queryAllPrivateLogsByTags(aztecNode, { + tags: batch, + referenceBlock: anchorBlockHash, + fromBlock: options.fromBlock, + toBlock: options.toBlock, + includeEffects: options.includeEffects ?? false, + limitPerTag: options.limitPerTag, + }) as Promise[][]>, ); } /** - * Fetches all public logs for the given tags from a contract, automatically paginating through all pages. + * Fetches all public logs for the given tags from a contract, automatically paginating per-tag via `afterLog` cursors. + * * @param aztecNode - The Aztec node to query. * @param contractAddress - The contract address to search logs for. * @param tags - The tags to search for. - * @param anchorBlockHash - reference block for the Aztec node query, throws if block is not found there (typically - * because of reorgs). + * @param anchorBlockHash - Reference block for the Aztec node query, throws if block is not found there (typically + * because of reorgs). + * @param options - Optional `fromBlock`/`toBlock` range and `includeEffects` opt-in. * @returns An array of log arrays, one per tag, containing all logs across all pages. */ -export function getAllPublicLogsByTagsFromContract( +export function getAllPublicLogsByTagsFromContract( aztecNode: AztecNode, contractAddress: AztecAddress, tags: Tag[], anchorBlockHash: BlockHash, -): Promise { - return getAllPagesInBatches(tags, batch => - getAllPages(batch.length, page => - aztecNode.getPublicLogsByTagsFromContract(contractAddress, batch, page, anchorBlockHash), - ), + options: Opts = {} as Opts, +): Promise[][]> { + return getAllPagesInBatches( + tags, + batch => + queryAllPublicLogsByTags(aztecNode, { + contractAddress, + tags: batch, + referenceBlock: anchorBlockHash, + fromBlock: options.fromBlock, + toBlock: options.toBlock, + includeEffects: options.includeEffects ?? false, + limitPerTag: options.limitPerTag, + }) as Promise[][]>, ); } diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts index 7ec2deb61771..df5dd20f3b2e 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.test.ts @@ -1,10 +1,17 @@ import { MAX_TX_LIFETIME } from '@aztec/constants'; import { BlockNumber } from '@aztec/foundation/branded-types'; -import { Fr } from '@aztec/foundation/curves/bn254'; +import type { Fr } from '@aztec/foundation/curves/bn254'; import { openTmpStore } from '@aztec/kv-store/lmdb-v2'; import type { AztecNode } from '@aztec/stdlib/interfaces/server'; -import { type AppTaggingSecret, AppTaggingSecretKind, SiloedTag } from '@aztec/stdlib/logs'; -import { randomAppTaggingSecret, randomTxScopedPrivateL2Log } from '@aztec/stdlib/testing'; +import { + type AppTaggingSecret, + AppTaggingSecretKind, + type PrivateLogsQuery, + SiloedTag, + type TagQuery, + randomLogResult, +} from '@aztec/stdlib/logs'; +import { randomAppTaggingSecret } from '@aztec/stdlib/testing'; import { BlockHeader } from '@aztec/stdlib/tx'; import { type MockProxy, mock } from 'jest-mock-extended'; @@ -25,8 +32,15 @@ describe('syncTaggedPrivateLogs', () => { return SiloedTag.compute({ extendedSecret: secret, index }); } - function makeLog(blockNumber: number, blockTimestamp: bigint, tag: Fr) { - return randomTxScopedPrivateL2Log({ blockNumber, blockTimestamp, tag }); + function makeLog(blockNumber: number, blockTimestamp: bigint, _tag: Fr) { + return { ...randomLogResult(/* includeEffects */ true), blockNumber: BlockNumber(blockNumber), blockTimestamp }; + } + + /** + * Extracts the bare-tag set from a query, defaulting `afterLog`-wrapped entries to their inner tag. + */ + function extractTags(query: PrivateLogsQuery): SiloedTag[] { + return query.tags.map((entry: TagQuery) => (entry instanceof SiloedTag ? entry : entry.tag)); } beforeEach(async () => { @@ -43,7 +57,8 @@ describe('syncTaggedPrivateLogs', () => { it('returns empty array when no logs found for any secret', async () => { const secrets = await makeSecrets(3); - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { + aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => { + const tags = extractTags(query); return Promise.resolve(tags.map(() => [])); }); @@ -63,7 +78,8 @@ describe('syncTaggedPrivateLogs', () => { const secrets = await makeSecrets(3); const finalizedBlockNumber = BlockNumber(10); - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { + aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => { + const tags = extractTags(query); return Promise.resolve(tags.map(() => [])); }); @@ -82,7 +98,8 @@ describe('syncTaggedPrivateLogs', () => { const log1Tag = await computeSiloedTagForIndex(secrets[0], log1Index); const log2Tag = await computeSiloedTagForIndex(secrets[1], log2Index); - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { + aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => { + const tags = extractTags(query); return Promise.resolve( tags.map((t: SiloedTag) => { if (t.equals(log1Tag)) { @@ -122,7 +139,8 @@ describe('syncTaggedPrivateLogs', () => { const logIndex = 5; const logTag = await computeSiloedTagForIndex(secret, logIndex); - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { + aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => { + const tags = extractTags(query); return Promise.resolve( tags.map((t: SiloedTag) => t.equals(logTag) ? [makeLog(Number(finalizedBlockNumber), logBlockTimestamp, logTag.value)] : [], @@ -150,7 +168,8 @@ describe('syncTaggedPrivateLogs', () => { const newWindowIndex = lastIndexInInitialWindow + 3; const log2Tag = await computeSiloedTagForIndex(secret, newWindowIndex); - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { + aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => { + const tags = extractTags(query); return Promise.resolve( tags.map((t: SiloedTag) => { if (t.equals(log1Tag)) { @@ -186,11 +205,13 @@ describe('syncTaggedPrivateLogs', () => { await taggingStore.updateHighestAgedIndex(secret, existingAgedIndex, JOB_ID); await taggingStore.updateHighestFinalizedIndex(secret, existingFinalizedIndex, JOB_ID); - aztecNode.getPrivateLogsByTags.mockResolvedValue([]); + aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => + Promise.resolve(query.tags.map(() => [])), + ); await syncTaggedPrivateLogs([secret], aztecNode, taggingStore, ANCHOR_BLOCK_HEADER, finalizedBlockNumber, JOB_ID); - const calledTags = aztecNode.getPrivateLogsByTags.mock.calls[0][0]; + const calledTags = extractTags(aztecNode.getPrivateLogsByTags.mock.calls[0][0]); // The query window must start at existingAgedIndex+1 and end at existingFinalizedIndex+WINDOW_LEN (inclusive). const expectedStart = existingAgedIndex + 1; @@ -213,7 +234,8 @@ describe('syncTaggedPrivateLogs', () => { const logTag = await computeSiloedTagForIndex(secret, logIndex); // Two logs returned for the same tag - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { + aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => { + const tags = extractTags(query); return Promise.resolve( tags.map((t: SiloedTag) => t.equals(logTag) @@ -238,7 +260,7 @@ describe('syncTaggedPrivateLogs', () => { expect(logs).toHaveLength(2); }); - it('filters out logs from blocks after the anchor block', async () => { + it('caps the node query at anchorBlockNumber + 1 (toBlock exclusive)', async () => { const [secret] = await makeSecrets(1); const anchorBlock = BlockNumber(10); const header = BlockHeader.random({ blockNumber: anchorBlock, timestamp: CURRENT_TIMESTAMP }); @@ -248,25 +270,29 @@ describe('syncTaggedPrivateLogs', () => { const logIndex = 3; const logTag = await computeSiloedTagForIndex(secret, logIndex); - // Three logs: one before anchor, one at anchor, one after anchor - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { + // The mock simulates the node honoring `toBlock` (exclusive). Recipient sync now relies on the node + // for this filter rather than dropping post-anchor logs client-side. + aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => { + const tags = extractTags(query); + const toBlockExclusive = Number(query.toBlock ?? Infinity); + const allCandidates = [ + makeLog(Number(anchorBlock) - 1, logBlockTimestamp, logTag.value), + makeLog(Number(anchorBlock), logBlockTimestamp, logTag.value), + makeLog(Number(anchorBlock) + 1, logBlockTimestamp, logTag.value), + ]; return Promise.resolve( tags.map((t: SiloedTag) => - t.equals(logTag) - ? [ - makeLog(Number(anchorBlock) - 1, logBlockTimestamp, logTag.value), - makeLog(Number(anchorBlock), logBlockTimestamp, logTag.value), - makeLog(Number(anchorBlock) + 1, logBlockTimestamp, logTag.value), - ] - : [], + t.equals(logTag) ? allCandidates.filter(l => Number(l.blockNumber) < toBlockExclusive) : [], ), ); }); const logs = await syncTaggedPrivateLogs([secret], aztecNode, taggingStore, header, finalizedBlockNumber, JOB_ID); - // Only logs at or before the anchor block should be included + // Only logs at or before the anchor block should be included — node-side filter drops the post-anchor log. expect(logs).toHaveLength(2); + // Verify the node was called with toBlock = anchorBlock + 1 (exclusive upper bound). + expect(aztecNode.getPrivateLogsByTags.mock.calls[0][0].toBlock).toBe(BlockNumber(Number(anchorBlock) + 1)); }); }); diff --git a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts index bb8e4d94e134..92b87cc9679f 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/sync_tagged_private_logs.ts @@ -1,8 +1,8 @@ -import type { BlockNumber } from '@aztec/foundation/branded-types'; +import { BlockNumber } from '@aztec/foundation/branded-types'; import { isDefined } from '@aztec/foundation/types'; import type { BlockHash } from '@aztec/stdlib/block'; import type { AztecNode } from '@aztec/stdlib/interfaces/client'; -import type { AppTaggingSecret, TxScopedL2Log } from '@aztec/stdlib/logs'; +import type { AppTaggingSecret, LogResult } from '@aztec/stdlib/logs'; import { SiloedTag } from '@aztec/stdlib/logs'; import type { BlockHeader } from '@aztec/stdlib/tx'; @@ -67,7 +67,7 @@ export async function syncTaggedPrivateLogs( anchorBlockHeader: BlockHeader, finalizedBlockNumber: BlockNumber, jobId: string, -): Promise { +): Promise { if (secrets.length === 0) { return []; } @@ -78,7 +78,7 @@ export async function syncTaggedPrivateLogs( // Read stored indexes from the db and compute the initial [start, end) range for each secret let pending = await getIndexRangesForSecrets(secrets, taggingStore, jobId); - const allLogs: TxScopedL2Log[] = []; + const allLogs: LogResult[] = []; while (pending.length > 0) { // Compute tags for all pending secrets and fetch logs in batched RPC calls @@ -156,8 +156,14 @@ async function fetchLogsForSecrets( const allTags = tagsPerSecret.flat(); - // getAllPrivateLogsByTags handles MAX_RPC_LEN chunking internally - const allResults = await getAllPrivateLogsByTags(aztecNode, allTags, anchorBlockHash); + // getAllPrivateLogsByTags handles MAX_RPC_LEN chunking internally. Recipient sync builds `PendingTaggedLog` from + // each log's note hashes and first nullifier, so we opt into effects. The `toBlock` cap (anchor block + 1, + // exclusive) tells the node to skip any logs in blocks past the anchor — the same guard previously enforced + // by an in-memory filter on the response. + const allResults = await getAllPrivateLogsByTags(aztecNode, allTags, anchorBlockHash, { + includeEffects: true, + toBlock: BlockNumber(anchorBlockNumber + 1), + }); // Split flat results back per secret using the known lengths const logsPerSecret: LogWithIndex[][] = []; @@ -166,9 +172,7 @@ async function fetchLogsForSecrets( const logsForSecret: LogWithIndex[] = []; for (let i = 0; i < indexes.length; i++) { for (const log of allResults[offset + i]) { - if (log.blockNumber <= anchorBlockNumber) { - logsForSecret.push({ log, taggingIndex: indexes[i] }); - } + logsForSecret.push({ log, taggingIndex: indexes[i] }); } } logsPerSecret.push(logsForSecret); @@ -231,6 +235,6 @@ type PendingSecret = { }; type LogWithIndex = { - log: TxScopedL2Log; + log: LogResult; taggingIndex: number; }; diff --git a/yarn-project/pxe/src/tagging/recipient_sync/utils/find_highest_indexes.test.ts b/yarn-project/pxe/src/tagging/recipient_sync/utils/find_highest_indexes.test.ts index 2667029abaf7..4b22e4f03c37 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/utils/find_highest_indexes.test.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/utils/find_highest_indexes.test.ts @@ -1,14 +1,14 @@ import { MAX_TX_LIFETIME } from '@aztec/constants'; -import { TxScopedL2Log } from '@aztec/stdlib/logs'; -import { randomTxScopedPrivateL2Log } from '@aztec/stdlib/testing'; +import { BlockNumber } from '@aztec/foundation/branded-types'; +import { type LogResult, randomLogResult } from '@aztec/stdlib/logs'; import { findHighestIndexes } from './find_highest_indexes.js'; describe('findHighestIndexes', () => { const currentTimestamp = BigInt(Math.floor(Date.now() / 1000)); - function makeLog(blockNumber: number, blockTimestamp: bigint): TxScopedL2Log { - return randomTxScopedPrivateL2Log({ blockNumber, blockTimestamp }); + function makeLog(blockNumber: number, blockTimestamp: bigint): LogResult { + return { ...randomLogResult(/* includeEffects */ true), blockNumber: BlockNumber(blockNumber), blockTimestamp }; } it('returns undefined for highestAgedIndex when no logs are at least 24 hours old', () => { diff --git a/yarn-project/pxe/src/tagging/recipient_sync/utils/find_highest_indexes.ts b/yarn-project/pxe/src/tagging/recipient_sync/utils/find_highest_indexes.ts index f863365a80e7..154959d35a88 100644 --- a/yarn-project/pxe/src/tagging/recipient_sync/utils/find_highest_indexes.ts +++ b/yarn-project/pxe/src/tagging/recipient_sync/utils/find_highest_indexes.ts @@ -1,11 +1,11 @@ import { MAX_TX_LIFETIME } from '@aztec/constants'; -import type { TxScopedL2Log } from '@aztec/stdlib/logs'; +import type { LogResult } from '@aztec/stdlib/logs'; /** * Finds the highest aged and the highest finalized tagging indexes. */ export function findHighestIndexes( - privateLogsWithIndexes: Array<{ log: TxScopedL2Log; taggingIndex: number }>, + privateLogsWithIndexes: Array<{ log: LogResult; taggingIndex: number }>, currentTimestamp: bigint, finalizedBlockNumber: number, ): { highestAgedIndex: number | undefined; highestFinalizedIndex: number | undefined } { diff --git a/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.test.ts b/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.test.ts index 29edfb3c441a..2c0887287015 100644 --- a/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.test.ts +++ b/yarn-project/pxe/src/tagging/sender_sync/sync_sender_tagging_indexes.test.ts @@ -5,7 +5,7 @@ import { RevertCode } from '@aztec/stdlib/avm'; import { BlockHash } from '@aztec/stdlib/block'; import type { AztecNode } from '@aztec/stdlib/interfaces/client'; import { AppTaggingSecretKind, PrivateLog } from '@aztec/stdlib/logs'; -import { randomAppTaggingSecret, randomTxScopedPrivateL2Log } from '@aztec/stdlib/testing'; +import { randomAppTaggingSecret, randomPrivateLogResult } from '@aztec/stdlib/testing'; import { type IndexedTxEffect, TxEffect, TxExecutionResult, TxHash, TxReceipt, TxStatus } from '@aztec/stdlib/tx'; import { type MockProxy, mock } from 'jest-mock-extended'; @@ -28,7 +28,7 @@ describe('syncSenderTaggingIndexes', () => { } function makeLog(txHash: TxHash, tag: Fr) { - return randomTxScopedPrivateL2Log({ txHash, tag }); + return randomPrivateLogResult({ txHash, tag }); } async function setUp() { @@ -41,7 +41,8 @@ describe('syncSenderTaggingIndexes', () => { it('no new logs found for a given secret', async () => { await setUp(); - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { + aztecNode.getPrivateLogsByTags.mockImplementation(query => { + const tags = query.tags as SiloedTag[]; // No log found for any tag return Promise.resolve(tags.map((_tag: SiloedTag) => [])); }); @@ -62,7 +63,8 @@ describe('syncSenderTaggingIndexes', () => { const finalizedTag = await computeSiloedTagForIndex(finalizedIndex); const finalizedTxHash = TxHash.random(); - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { + aztecNode.getPrivateLogsByTags.mockImplementation(query => { + const tags = query.tags as SiloedTag[]; return Promise.resolve( tags.map((tag: SiloedTag) => (tag.equals(finalizedTag) ? [makeLog(finalizedTxHash, finalizedTag.value)] : [])), ); @@ -102,7 +104,8 @@ describe('syncSenderTaggingIndexes', () => { const index3Tag = await computeSiloedTagForIndex(finalizedIndexStep1); const finalizedTxHash = TxHash.random(); - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { + aztecNode.getPrivateLogsByTags.mockImplementation(query => { + const tags = query.tags as SiloedTag[]; // Return empty arrays for all tags except the one at index 3 return Promise.resolve( tags.map((tag: SiloedTag) => (tag.equals(index3Tag) ? [makeLog(finalizedTxHash, index3Tag.value)] : [])), @@ -134,7 +137,8 @@ describe('syncSenderTaggingIndexes', () => { it('step 2: pending log is synced', async () => { const pendingTag = await computeSiloedTagForIndex(pendingIndexStep2); - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { + aztecNode.getPrivateLogsByTags.mockImplementation(query => { + const tags = query.tags as SiloedTag[]; // Return empty arrays for all tags except the one at the pending index return Promise.resolve( tags.map((tag: SiloedTag) => (tag.equals(pendingTag) ? [makeLog(pendingTxHashStep2, pendingTag.value)] : [])), @@ -176,7 +180,8 @@ describe('syncSenderTaggingIndexes', () => { const newHighestUsedTag = await computeSiloedTagForIndex(newHighestUsedIndex); // New pending log // Mock getPrivateLogsByTags to return logs for multiple indices - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { + aztecNode.getPrivateLogsByTags.mockImplementation(query => { + const tags = query.tags as SiloedTag[]; return Promise.resolve( tags.map((tag: SiloedTag) => { if (tag.equals(nowFinalizedTag)) { @@ -258,7 +263,8 @@ describe('syncSenderTaggingIndexes', () => { const index3Tag = await computeSiloedTagForIndex(pendingAndFinalizedIndex); - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { + aztecNode.getPrivateLogsByTags.mockImplementation(query => { + const tags = query.tags as SiloedTag[]; // Return both the pending and finalized logs for the tag at index 3 return Promise.resolve( tags.map((tag: SiloedTag) => @@ -326,7 +332,8 @@ describe('syncSenderTaggingIndexes', () => { ); // No new logs surfaced in this window. - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { + aztecNode.getPrivateLogsByTags.mockImplementation(query => { + const tags = query.tags as SiloedTag[]; return Promise.resolve(tags.map(() => [])); }); @@ -361,7 +368,8 @@ describe('syncSenderTaggingIndexes', () => { it('does not call getTxReceipt when no pending entries exist and no new logs are found', async () => { await setUp(); - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { + aztecNode.getPrivateLogsByTags.mockImplementation(query => { + const tags = query.tags as SiloedTag[]; return Promise.resolve(tags.map(() => [])); }); @@ -391,7 +399,8 @@ describe('syncSenderTaggingIndexes', () => { 'test', ); - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { + aztecNode.getPrivateLogsByTags.mockImplementation(query => { + const tags = query.tags as SiloedTag[]; return Promise.resolve( tags.map((tag: SiloedTag) => tag.equals(newlyDiscoveredTag) ? [makeLog(newlyDiscoveredTxHash, newlyDiscoveredTag.value)] : [], @@ -446,7 +455,8 @@ describe('syncSenderTaggingIndexes', () => { ); // Logs query returns the same tx for the same tag — `storePendingIndexes` will treat this as a no-op duplicate. - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { + aztecNode.getPrivateLogsByTags.mockImplementation(query => { + const tags = query.tags as SiloedTag[]; return Promise.resolve( tags.map((tag: SiloedTag) => (tag.equals(pendingTag) ? [makeLog(pendingTxHash, pendingTag.value)] : [])), ); @@ -483,7 +493,8 @@ describe('syncSenderTaggingIndexes', () => { const tag4 = await computeSiloedTagForIndex(4); const tag6 = await computeSiloedTagForIndex(6); - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { + aztecNode.getPrivateLogsByTags.mockImplementation(query => { + const tags = query.tags as SiloedTag[]; return Promise.resolve( tags.map((tag: SiloedTag) => { if (tag.equals(tag4)) { diff --git a/yarn-project/pxe/src/tagging/sender_sync/utils/load_and_store_new_tagging_indexes.test.ts b/yarn-project/pxe/src/tagging/sender_sync/utils/load_and_store_new_tagging_indexes.test.ts index 63284b3157bd..accc15e25667 100644 --- a/yarn-project/pxe/src/tagging/sender_sync/utils/load_and_store_new_tagging_indexes.test.ts +++ b/yarn-project/pxe/src/tagging/sender_sync/utils/load_and_store_new_tagging_indexes.test.ts @@ -1,8 +1,15 @@ import type { Fr } from '@aztec/foundation/curves/bn254'; import { BlockHash } from '@aztec/stdlib/block'; import type { AztecNode } from '@aztec/stdlib/interfaces/server'; -import { type AppTaggingSecret, AppTaggingSecretKind, SiloedTag } from '@aztec/stdlib/logs'; -import { randomAppTaggingSecret, randomTxScopedPrivateL2Log } from '@aztec/stdlib/testing'; +import { + type AppTaggingSecret, + AppTaggingSecretKind, + type PrivateLogsQuery, + SiloedTag, + type TagQuery, + randomLogResult, +} from '@aztec/stdlib/logs'; +import { randomAppTaggingSecret } from '@aztec/stdlib/testing'; import { TxHash } from '@aztec/stdlib/tx'; import { type MockProxy, mock } from 'jest-mock-extended'; @@ -21,8 +28,16 @@ describe('loadAndStoreNewTaggingIndexes', () => { return SiloedTag.compute({ extendedSecret: secret, index }); } - function makeLog(txHash: TxHash, tag: Fr) { - return randomTxScopedPrivateL2Log({ txHash, tag }); + function makeLog(txHash: TxHash, _tag: Fr) { + return { ...randomLogResult(), txHash }; + } + + /** + * Extracts the bare-tag set from a query, defaulting `afterLog`-wrapped entries to their inner tag. Sender sync + * never paginates within a tag (one log per index), so the bare-tag path is the only one exercised here. + */ + function extractTags(query: PrivateLogsQuery): SiloedTag[] { + return query.tags.map((entry: TagQuery) => (entry instanceof SiloedTag ? entry : entry.tag)); } beforeAll(async () => { @@ -35,7 +50,8 @@ describe('loadAndStoreNewTaggingIndexes', () => { }); it('no logs found for the given window', async () => { - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { + aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => { + const tags = extractTags(query); return Promise.resolve(tags.map(() => [])); }); @@ -49,7 +65,8 @@ describe('loadAndStoreNewTaggingIndexes', () => { const index = 5; const tag = await computeSiloedTagForIndex(index); - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { + aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => { + const tags = extractTags(query); return Promise.resolve(tags.map((t: SiloedTag) => (t.equals(tag) ? [makeLog(txHash, tag.value)] : []))); }); @@ -70,7 +87,8 @@ describe('loadAndStoreNewTaggingIndexes', () => { const tag1 = await computeSiloedTagForIndex(index1); const tag2 = await computeSiloedTagForIndex(index2); - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { + aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => { + const tags = extractTags(query); return Promise.resolve( tags.map((t: SiloedTag) => { if (t.equals(tag1)) { @@ -101,7 +119,8 @@ describe('loadAndStoreNewTaggingIndexes', () => { const tag1 = await computeSiloedTagForIndex(index1); const tag2 = await computeSiloedTagForIndex(index2); - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { + aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => { + const tags = extractTags(query); return Promise.resolve( tags.map((t: SiloedTag) => { if (t.equals(tag1)) { @@ -136,7 +155,8 @@ describe('loadAndStoreNewTaggingIndexes', () => { const index = 4; const tag = await computeSiloedTagForIndex(index); - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { + aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => { + const tags = extractTags(query); return Promise.resolve( tags.map((t: SiloedTag) => (t.equals(tag) ? [makeLog(txHash1, tag.value), makeLog(txHash2, tag.value)] : [])), ); @@ -172,7 +192,8 @@ describe('loadAndStoreNewTaggingIndexes', () => { const tag8 = await computeSiloedTagForIndex(8); const tag9 = await computeSiloedTagForIndex(9); - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { + aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => { + const tags = extractTags(query); return Promise.resolve( tags.map((t: SiloedTag) => { if (t.equals(tag1)) { @@ -223,7 +244,8 @@ describe('loadAndStoreNewTaggingIndexes', () => { const tagAtStart = await computeSiloedTagForIndex(start); const tagAtEnd = await computeSiloedTagForIndex(end); - aztecNode.getPrivateLogsByTags.mockImplementation((tags: SiloedTag[]) => { + aztecNode.getPrivateLogsByTags.mockImplementation((query: PrivateLogsQuery) => { + const tags = extractTags(query); return Promise.resolve( tags.map((t: SiloedTag) => { if (t.equals(tagAtStart)) { diff --git a/yarn-project/pxe/src/tagging/sender_sync/utils/load_and_store_new_tagging_indexes.ts b/yarn-project/pxe/src/tagging/sender_sync/utils/load_and_store_new_tagging_indexes.ts index 545ce56ed241..88e32f7fca10 100644 --- a/yarn-project/pxe/src/tagging/sender_sync/utils/load_and_store_new_tagging_indexes.ts +++ b/yarn-project/pxe/src/tagging/sender_sync/utils/load_and_store_new_tagging_indexes.ts @@ -52,7 +52,7 @@ async function getTxsContainingTags( anchorBlockHash: BlockHash, ): Promise { // We use the utility function below to retrieve all logs for the tags across all pages, so we don't need to handle - // pagination here. + // pagination here. Sender sync only needs `txHash` from each log, so we leave `includeEffects` off. const allLogs = await getAllPrivateLogsByTags(aztecNode, tags, anchorBlockHash); return allLogs.map(logs => logs.map(log => log.txHash)); } diff --git a/yarn-project/stdlib/src/interfaces/api_limit.ts b/yarn-project/stdlib/src/interfaces/api_limit.ts index 34936566d6c2..c4145410410e 100644 --- a/yarn-project/stdlib/src/interfaces/api_limit.ts +++ b/yarn-project/stdlib/src/interfaces/api_limit.ts @@ -2,4 +2,4 @@ export const MAX_RPC_LEN = 100; export const MAX_RPC_TXS_LEN = 50; export const MAX_RPC_BLOCKS_LEN = 50; export const MAX_RPC_CHECKPOINTS_LEN = 50; -export const MAX_LOGS_PER_TAG = 10; +export const MAX_LOGS_PER_TAG = 20; diff --git a/yarn-project/stdlib/src/interfaces/archiver.test.ts b/yarn-project/stdlib/src/interfaces/archiver.test.ts index 56a5396927ec..995d2f9c95cd 100644 --- a/yarn-project/stdlib/src/interfaces/archiver.test.ts +++ b/yarn-project/stdlib/src/interfaces/archiver.test.ts @@ -32,14 +32,11 @@ import { } from '../contract/index.js'; import { EmptyL1RollupConstants, type L1RollupConstants } from '../epoch-helpers/index.js'; import { PublicKeys } from '../keys/public_keys.js'; -import { ExtendedContractClassLog } from '../logs/extended_contract_class_log.js'; -import { ExtendedPublicLog } from '../logs/extended_public_log.js'; -import type { LogFilter } from '../logs/log_filter.js'; +import { type LogResult, randomLogResult } from '../logs/log_result.js'; +import type { PrivateLogsQuery, PublicLogsQuery } from '../logs/logs_query.js'; import { SiloedTag } from '../logs/siloed_tag.js'; import { Tag } from '../logs/tag.js'; -import { TxScopedL2Log } from '../logs/tx_scoped_l2_log.js'; import { CheckpointHeader } from '../rollup/checkpoint_header.js'; -import { randomTxScopedPrivateL2Log } from '../tests/factories.js'; import { getTokenContractArtifact } from '../tests/fixtures.js'; import { AppendOnlyTreeSnapshot } from '../trees/append_only_tree_snapshot.js'; import type { IndexedTxEffect } from '../tx/indexed_tx_effect.js'; @@ -47,7 +44,6 @@ import { TxEffect } from '../tx/tx_effect.js'; import { TxHash } from '../tx/tx_hash.js'; import { TxReceipt } from '../tx/tx_receipt.js'; import { type ArchiverApi, ArchiverApiSchema } from './archiver.js'; -import type { GetContractClassLogsResponse, GetPublicLogsResponse } from './get_logs_response.js'; describe('ArchiverApiSchema', () => { let handler: MockArchiver; @@ -165,41 +161,18 @@ describe('ArchiverApiSchema', () => { }); it('getPrivateLogsByTags', async () => { - const result = await context.client.getPrivateLogsByTags([SiloedTag.random()]); - expect(result).toEqual([[expect.any(TxScopedL2Log)]]); - - const resultWithOptionals = await context.client.getPrivateLogsByTags([SiloedTag.random()], 3, BlockNumber(4)); - expect(resultWithOptionals).toEqual([[expect.any(TxScopedL2Log)]]); + const result = await context.client.getPrivateLogsByTags({ tags: [SiloedTag.random()] }); + expect(result).toHaveLength(1); + expect(result[0]).toHaveLength(1); + expect(result[0][0].txHash).toBeDefined(); }); - it('getPublicLogsByTagsFromContract', async () => { + it('getPublicLogsByTags', async () => { const contractAddress = await AztecAddress.random(); - const result = await context.client.getPublicLogsByTagsFromContract(contractAddress, [Tag.random()]); - expect(result).toEqual([[expect.any(TxScopedL2Log)]]); - - const resultWithOptionals = await context.client.getPublicLogsByTagsFromContract( - contractAddress, - [Tag.random()], - 3, - BlockNumber(4), - ); - expect(resultWithOptionals).toEqual([[expect.any(TxScopedL2Log)]]); - }); - - it('getPublicLogs', async () => { - const result = await context.client.getPublicLogs({ - txHash: TxHash.random(), - contractAddress: await AztecAddress.random(), - }); - expect(result).toEqual({ logs: [expect.any(ExtendedPublicLog)], maxLogsHit: true }); - }); - - it('getContractClassLogs', async () => { - const result = await context.client.getContractClassLogs({ - txHash: TxHash.random(), - contractAddress: await AztecAddress.random(), - }); - expect(result).toEqual({ logs: [expect.any(ExtendedContractClassLog)], maxLogsHit: true }); + const result = await context.client.getPublicLogsByTags({ contractAddress, tags: [Tag.random()] }); + expect(result).toHaveLength(1); + expect(result[0]).toHaveLength(1); + expect(result[0][0].txHash).toBeDefined(); }); it('getContractClass', async () => { @@ -514,41 +487,14 @@ class MockArchiver implements ArchiverApi { expect(blockNumber).toEqual(BlockNumber(1)); return Promise.resolve(`0x01`); } - getPrivateLogsByTags(tags: SiloedTag[], page?: number, upToBlockNumber?: BlockNumber): Promise { - expect(tags[0]).toBeInstanceOf(SiloedTag); - if (page !== undefined) { - expect(page).toBe(3); - } - if (upToBlockNumber !== undefined) { - expect(upToBlockNumber).toBe(BlockNumber(4)); - } - return Promise.resolve([tags.map(() => randomTxScopedPrivateL2Log())]); - } - getPublicLogsByTagsFromContract( - contractAddress: AztecAddress, - tags: Tag[], - page?: number, - upToBlockNumber?: BlockNumber, - ): Promise { - expect(contractAddress).toBeInstanceOf(AztecAddress); - expect(tags[0]).toBeInstanceOf(Tag); - if (page !== undefined) { - expect(page).toBe(3); - } - if (upToBlockNumber !== undefined) { - expect(upToBlockNumber).toBe(BlockNumber(4)); - } - return Promise.resolve([tags.map(() => randomTxScopedPrivateL2Log())]); - } - async getPublicLogs(filter: LogFilter): Promise { - expect(filter.txHash).toBeInstanceOf(TxHash); - expect(filter.contractAddress).toBeInstanceOf(AztecAddress); - return { logs: [await ExtendedPublicLog.random()], maxLogsHit: true }; - } - async getContractClassLogs(filter: LogFilter): Promise { - expect(filter.txHash).toBeInstanceOf(TxHash); - expect(filter.contractAddress).toBeInstanceOf(AztecAddress); - return Promise.resolve({ logs: [await ExtendedContractClassLog.random()], maxLogsHit: true }); + getPrivateLogsByTags(query: PrivateLogsQuery): Promise { + expect(Array.isArray(query.tags)).toBe(true); + return Promise.resolve([query.tags.map(() => randomLogResult())]); + } + getPublicLogsByTags(query: PublicLogsQuery): Promise { + expect(query.contractAddress).toBeInstanceOf(AztecAddress); + expect(Array.isArray(query.tags)).toBe(true); + return Promise.resolve([query.tags.map(() => randomLogResult())]); } async getContractClass(id: Fr): Promise { expect(id).toBeInstanceOf(Fr); diff --git a/yarn-project/stdlib/src/interfaces/archiver.ts b/yarn-project/stdlib/src/interfaces/archiver.ts index 48d937c65cb9..ffcc8ae15803 100644 --- a/yarn-project/stdlib/src/interfaces/archiver.ts +++ b/yarn-project/stdlib/src/interfaces/archiver.ts @@ -23,16 +23,13 @@ import { ContractInstanceWithAddressSchema, } from '../contract/index.js'; import { L1RollupConstantsSchema } from '../epoch-helpers/index.js'; -import { LogFilterSchema } from '../logs/log_filter.js'; -import { SiloedTag } from '../logs/siloed_tag.js'; -import { Tag } from '../logs/tag.js'; -import { TxScopedL2Log } from '../logs/tx_scoped_l2_log.js'; +import { LogResultSchema } from '../logs/log_result.js'; +import { PrivateLogsQuerySchema, PublicLogsQuerySchema } from '../logs/logs_query.js'; import type { L1ToL2MessageSource } from '../messaging/l1_to_l2_message_source.js'; import { optional, schemas } from '../schemas/schemas.js'; import { indexedTxSchema } from '../tx/indexed_tx_effect.js'; import { TxHash } from '../tx/tx_hash.js'; import { TxReceipt } from '../tx/tx_receipt.js'; -import { GetContractClassLogsResponseSchema, GetPublicLogsResponseSchema } from './get_logs_response.js'; import type { L2LogsSource } from './l2_logs_source.js'; /** @@ -48,9 +45,6 @@ export type ArchiverSpecificConfig = { /** The polling interval viem uses in ms */ viemPollingIntervalMS?: number; - /** The max number of logs that can be obtained in 1 "getPublicLogs" call. */ - maxLogs?: number; - /** The maximum possible size of the archiver DB in KB. Overwrites the general dataStoreMapSizeKb. */ archiverStoreMapSizeKb?: number; @@ -82,7 +76,6 @@ export const ArchiverSpecificConfigSchema = z.object({ archiverPollingIntervalMS: schemas.Integer.optional(), archiverBatchSize: schemas.Integer.optional(), viemPollingIntervalMS: schemas.Integer.optional(), - maxLogs: schemas.Integer.optional(), archiverStoreMapSizeKb: schemas.Integer.optional(), maxAllowedEthClientDriftSeconds: schemas.Integer.optional(), ethereumAllowNoDebugHosts: z.boolean().optional(), @@ -114,20 +107,13 @@ export const ArchiverApiSchema: ApiSchemaFor = { isEpochComplete: z.function({ input: z.tuple([EpochNumberSchema]), output: z.boolean() }), getL2Tips: z.function({ input: z.tuple([]), output: L2TipsSchema }), getPrivateLogsByTags: z.function({ - input: z.tuple([z.array(SiloedTag.schema), optional(z.number().gte(0)), optional(BlockNumberSchema)]), - output: z.array(z.array(TxScopedL2Log.schema)), + input: z.tuple([PrivateLogsQuerySchema]), + output: z.array(z.array(LogResultSchema)), }), - getPublicLogsByTagsFromContract: z.function({ - input: z.tuple([ - schemas.AztecAddress, - z.array(Tag.schema), - optional(z.number().gte(0)), - optional(BlockNumberSchema), - ]), - output: z.array(z.array(TxScopedL2Log.schema)), + getPublicLogsByTags: z.function({ + input: z.tuple([PublicLogsQuerySchema]), + output: z.array(z.array(LogResultSchema)), }), - getPublicLogs: z.function({ input: z.tuple([LogFilterSchema]), output: GetPublicLogsResponseSchema }), - getContractClassLogs: z.function({ input: z.tuple([LogFilterSchema]), output: GetContractClassLogsResponseSchema }), getContractClass: z.function({ input: z.tuple([schemas.Fr]), output: ContractClassPublicSchema.optional() }), getBytecodeCommitment: z.function({ input: z.tuple([schemas.Fr]), output: schemas.Fr }), getContract: z.function({ diff --git a/yarn-project/stdlib/src/interfaces/aztec-node.test.ts b/yarn-project/stdlib/src/interfaces/aztec-node.test.ts index 53edceea3df1..45a65decfe00 100644 --- a/yarn-project/stdlib/src/interfaces/aztec-node.test.ts +++ b/yarn-project/stdlib/src/interfaces/aztec-node.test.ts @@ -28,13 +28,10 @@ import { } from '../contract/index.js'; import { GasFees } from '../gas/gas_fees.js'; import { PublicKeys } from '../keys/public_keys.js'; -import { ExtendedContractClassLog } from '../logs/extended_contract_class_log.js'; -import { ExtendedPublicLog } from '../logs/extended_public_log.js'; -import type { LogFilter } from '../logs/log_filter.js'; +import { type LogResult, randomLogResult } from '../logs/log_result.js'; +import type { PrivateLogsQuery, PublicLogsQuery } from '../logs/logs_query.js'; import { SiloedTag } from '../logs/siloed_tag.js'; import { Tag } from '../logs/tag.js'; -import { TxScopedL2Log } from '../logs/tx_scoped_l2_log.js'; -import { randomTxScopedPrivateL2Log } from '../tests/factories.js'; import { getTokenContractArtifact } from '../tests/fixtures.js'; import { MerkleTreeId } from '../trees/merkle_tree_id.js'; import { NullifierMembershipWitness } from '../trees/nullifier_membership_witness.js'; @@ -55,7 +52,6 @@ import type { ChainTip, ChainTips } from './chain_tips.js'; import type { CheckpointParameter } from './checkpoint_parameter.js'; import type { CheckpointIncludeOptions, CheckpointResponse } from './checkpoint_response.js'; import type { SequencerConfig } from './configs.js'; -import type { GetContractClassLogsResponse, GetPublicLogsResponse } from './get_logs_response.js'; import type { ProverConfig } from './prover-client.js'; import type { WorldStateSyncStatus } from './world_state.js'; @@ -268,40 +264,19 @@ describe('AztecNodeApiSchema', () => { await context.client.registerContractFunctionSignatures(['test()']); }); - it('getPublicLogs', async () => { - const response = await context.client.getPublicLogs({ contractAddress: await AztecAddress.random() }); - expect(response).toEqual({ logs: [expect.any(ExtendedPublicLog)], maxLogsHit: true }); - }); - - it('getContractClassLogs', async () => { - const response = await context.client.getContractClassLogs({ contractAddress: await AztecAddress.random() }); - expect(response).toEqual({ logs: [expect.any(ExtendedContractClassLog)], maxLogsHit: true }); - }); - it('getPrivateLogsByTags', async () => { - const response = await context.client.getPrivateLogsByTags([SiloedTag.random()]); - expect(response).toEqual([[expect.any(TxScopedL2Log)]]); - - const responseWithOptionals = await context.client.getPrivateLogsByTags( - [SiloedTag.random()], - 3, - BlockHash.random(), - ); - expect(responseWithOptionals).toEqual([[expect.any(TxScopedL2Log)]]); + const response = await context.client.getPrivateLogsByTags({ tags: [SiloedTag.random()] }); + expect(response).toHaveLength(1); + expect(response[0]).toHaveLength(1); + expect(response[0][0].txHash).toBeDefined(); }); - it('getPublicLogsByTagsFromContract', async () => { + it('getPublicLogsByTags', async () => { const contractAddress = await AztecAddress.random(); - const response = await context.client.getPublicLogsByTagsFromContract(contractAddress, [Tag.random()]); - expect(response).toEqual([[expect.any(TxScopedL2Log)]]); - - const responseWithOptionals = await context.client.getPublicLogsByTagsFromContract( - contractAddress, - [Tag.random()], - 3, - BlockHash.random(), - ); - expect(responseWithOptionals).toEqual([[expect.any(TxScopedL2Log)]]); + const response = await context.client.getPublicLogsByTags({ contractAddress, tags: [Tag.random()] }); + expect(response).toHaveLength(1); + expect(response[0]).toHaveLength(1); + expect(response[0][0].txHash).toBeDefined(); }); it('sendTx', async () => { @@ -730,41 +705,14 @@ class MockAztecNode implements AztecNode { registerContractFunctionSignatures(_signatures: string[]): Promise { return Promise.resolve(); } - async getPublicLogs(filter: LogFilter): Promise { - expect(filter.contractAddress).toBeInstanceOf(AztecAddress); - return { logs: [await ExtendedPublicLog.random()], maxLogsHit: true }; - } - async getContractClassLogs(filter: LogFilter): Promise { - expect(filter.contractAddress).toBeInstanceOf(AztecAddress); - return Promise.resolve({ logs: [await ExtendedContractClassLog.random()], maxLogsHit: true }); + getPrivateLogsByTags(query: PrivateLogsQuery): Promise { + expect(Array.isArray(query.tags)).toBe(true); + return Promise.resolve([query.tags.map(() => randomLogResult())]); } - getPrivateLogsByTags(tags: SiloedTag[], page?: number, referenceBlock?: BlockHash): Promise { - expect(tags).toHaveLength(1); - expect(tags[0]).toBeInstanceOf(SiloedTag); - if (page !== undefined) { - expect(page).toBe(3); - } - if (referenceBlock !== undefined) { - expect(referenceBlock).toBeInstanceOf(BlockHash); - } - return Promise.resolve([[randomTxScopedPrivateL2Log()]]); - } - getPublicLogsByTagsFromContract( - contractAddress: AztecAddress, - tags: Tag[], - page?: number, - referenceBlock?: BlockHash, - ): Promise { - expect(contractAddress).toBeInstanceOf(AztecAddress); - expect(tags).toHaveLength(1); - expect(tags[0]).toBeInstanceOf(Tag); - if (page !== undefined) { - expect(page).toBe(3); - } - if (referenceBlock !== undefined) { - expect(referenceBlock).toBeInstanceOf(BlockHash); - } - return Promise.resolve([[randomTxScopedPrivateL2Log()]]); + getPublicLogsByTags(query: PublicLogsQuery): Promise { + expect(query.contractAddress).toBeInstanceOf(AztecAddress); + expect(Array.isArray(query.tags)).toBe(true); + return Promise.resolve([query.tags.map(() => randomLogResult())]); } sendTx(tx: Tx): Promise { expect(tx).toBeInstanceOf(Tx); diff --git a/yarn-project/stdlib/src/interfaces/aztec-node.ts b/yarn-project/stdlib/src/interfaces/aztec-node.ts index 9ca1e204be9f..c2ef8a737409 100644 --- a/yarn-project/stdlib/src/interfaces/aztec-node.ts +++ b/yarn-project/stdlib/src/interfaces/aztec-node.ts @@ -37,8 +37,13 @@ import { } from '../contract/index.js'; import { ManaUsageEstimate } from '../gas/fee_math.js'; import { GasFees } from '../gas/gas_fees.js'; -import { SiloedTag, Tag, TxScopedL2Log } from '../logs/index.js'; -import { type LogFilter, LogFilterSchema } from '../logs/log_filter.js'; +import { type LogResult, LogResultSchema } from '../logs/log_result.js'; +import { + type PrivateLogsQuery, + PrivateLogsQuerySchema, + type PublicLogsQuery, + PublicLogsQuerySchema, +} from '../logs/logs_query.js'; import { type ApiSchemaFor, optional, schemas } from '../schemas/schemas.js'; import { MerkleTreeId } from '../trees/merkle_tree_id.js'; import { NullifierMembershipWitness } from '../trees/nullifier_membership_witness.js'; @@ -75,12 +80,6 @@ import { type CheckpointResponse, CheckpointResponseSchema, } from './checkpoint_response.js'; -import { - type GetContractClassLogsResponse, - GetContractClassLogsResponseSchema, - type GetPublicLogsResponse, - GetPublicLogsResponseSchema, -} from './get_logs_response.js'; import { type WorldStateSyncStatus, WorldStateSyncStatusSchema } from './world_state.js'; /** @@ -335,52 +334,25 @@ export interface AztecNode { registerContractFunctionSignatures(functionSignatures: string[]): Promise; /** - * Gets public logs based on the provided filter. - * @param filter - The filter to apply to the logs. - * @returns The requested logs. - */ - getPublicLogs(filter: LogFilter): Promise; - - /** - * Gets contract class logs based on the provided filter. - * @param filter - The filter to apply to the logs. - * @returns The requested logs. - */ - getContractClassLogs(filter: LogFilter): Promise; - - /** - * Gets private logs that match any of the `tags`. For each tag, an array of matching logs is returned. An empty - * array implies no logs match that tag. - * @param tags - The tags to search for. - * @param page - The page number (0-indexed) for pagination. - * @param referenceBlock - Optional block hash used to ensure the block still exists before logs are retrieved. - * This block is expected to represent the latest block to which the client has synced (called anchor block in PXE). - * If specified and the block is not found, an error is thrown. This helps detect reorgs, which could result in - * undefined behavior in the client's code. - * @returns An array of log arrays, one per tag. Returns at most 10 logs per tag per page. If 10 logs are returned - * for a tag, the caller should fetch the next page to check for more logs. + * Gets private logs matching the given tags. Returns one inner array per element of `query.tags`, in + * input order. An empty inner array means no logs matched that tag. Set `query.includeEffects` to also + * receive the tx's note hashes and nullifiers. + * + * The return type is the widest {@link LogResult} shape — `noteHashes`/`nullifiers` are typed as + * optional even when `includeEffects: true` is set. JSON-RPC validation can't preserve a stricter + * narrowing across the wire. Callers that want a narrowed type at the call site should use the typed + * helpers in `pxe/src/tagging/get_all_logs_by_tags.ts`. */ - getPrivateLogsByTags(tags: SiloedTag[], page?: number, referenceBlock?: BlockHash): Promise; + getPrivateLogsByTags(query: PrivateLogsQuery): Promise; /** - * Gets public logs that match any of the `tags` from the specified contract. For each tag, an array of matching - * logs is returned. An empty array implies no logs match that tag. - * @param contractAddress - The contract address to search logs for. - * @param tags - The tags to search for. - * @param page - The page number (0-indexed) for pagination. - * @param referenceBlock - Optional block hash used to ensure the block still exists before logs are retrieved. - * This block is expected to represent the latest block to which the client has synced (called anchor block in PXE). - * If specified and the block is not found, an error is thrown. This helps detect reorgs, which could result in - * undefined behavior in the client's code. - * @returns An array of log arrays, one per tag. Returns at most 10 logs per tag per page. If 10 logs are returned - * for a tag, the caller should fetch the next page to check for more logs. + * Gets public logs matching the given tags for the given contract. Returns one inner array per element + * of `query.tags`, in input order. An empty inner array means no logs matched that tag. Set + * `query.includeEffects` to also receive the tx's note hashes and nullifiers. + * + * The return type is the widest {@link LogResult} shape — see {@link getPrivateLogsByTags}. */ - getPublicLogsByTagsFromContract( - contractAddress: AztecAddress, - tags: Tag[], - page?: number, - referenceBlock?: BlockHash, - ): Promise; + getPublicLogsByTags(query: PublicLogsQuery): Promise; /** * Method to submit a transaction to the p2p pool. @@ -617,27 +589,14 @@ export const AztecNodeApiSchema: ApiSchemaFor = { output: z.void(), }), - getPublicLogs: z.function({ input: z.tuple([LogFilterSchema]), output: GetPublicLogsResponseSchema }), - - getContractClassLogs: z.function({ input: z.tuple([LogFilterSchema]), output: GetContractClassLogsResponseSchema }), - getPrivateLogsByTags: z.function({ - input: z.tuple([ - z.array(SiloedTag.schema).max(MAX_RPC_LEN), - optional(z.number().gte(0)), - optional(BlockHash.schema), - ]), - output: z.array(z.array(TxScopedL2Log.schema)), + input: z.tuple([PrivateLogsQuerySchema]), + output: z.array(z.array(LogResultSchema)), }), - getPublicLogsByTagsFromContract: z.function({ - input: z.tuple([ - schemas.AztecAddress, - z.array(Tag.schema).max(MAX_RPC_LEN), - optional(z.number().gte(0)), - optional(BlockHash.schema), - ]), - output: z.array(z.array(TxScopedL2Log.schema)), + getPublicLogsByTags: z.function({ + input: z.tuple([PublicLogsQuerySchema]), + output: z.array(z.array(LogResultSchema)), }), sendTx: z.function({ input: z.tuple([Tx.schema]), output: z.void() }), diff --git a/yarn-project/stdlib/src/interfaces/client.ts b/yarn-project/stdlib/src/interfaces/client.ts index 88fa0f93b402..be668152d079 100644 --- a/yarn-project/stdlib/src/interfaces/client.ts +++ b/yarn-project/stdlib/src/interfaces/client.ts @@ -7,6 +7,5 @@ export * from './checkpoint_parameter.js'; export * from './checkpoint_response.js'; export * from './l1_publish_info.js'; export * from './private_kernel_prover.js'; -export * from './get_logs_response.js'; export * from './api_limit.js'; export * from './public_storage_override.js'; diff --git a/yarn-project/stdlib/src/interfaces/get_logs_response.test.ts b/yarn-project/stdlib/src/interfaces/get_logs_response.test.ts deleted file mode 100644 index c2a32c8d5a02..000000000000 --- a/yarn-project/stdlib/src/interfaces/get_logs_response.test.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { jsonStringify } from '@aztec/foundation/json-rpc'; - -import { TxScopedL2Log } from '../logs/tx_scoped_l2_log.js'; -import { randomTxScopedPrivateL2Log } from '../tests/factories.js'; - -describe('TxScopedL2Log', () => { - it('serializes to JSON', () => { - const log = randomTxScopedPrivateL2Log(); - expect(TxScopedL2Log.schema.parse(JSON.parse(jsonStringify(log)))).toEqual(log); - }); -}); diff --git a/yarn-project/stdlib/src/interfaces/get_logs_response.ts b/yarn-project/stdlib/src/interfaces/get_logs_response.ts deleted file mode 100644 index 052dd054fc6d..000000000000 --- a/yarn-project/stdlib/src/interfaces/get_logs_response.ts +++ /dev/null @@ -1,35 +0,0 @@ -import { z } from 'zod'; - -import { ExtendedContractClassLog } from '../logs/extended_contract_class_log.js'; -import { ExtendedPublicLog } from '../logs/extended_public_log.js'; -import { zodFor } from '../schemas/index.js'; - -/** Response for the getContractClassLogs archiver call. */ -export type GetContractClassLogsResponse = { - /** An array of ExtendedContractClassLog elements. */ - logs: ExtendedContractClassLog[]; - /** Indicates if a limit has been reached. */ - maxLogsHit: boolean; -}; - -export const GetContractClassLogsResponseSchema = zodFor()( - z.object({ - logs: z.array(ExtendedContractClassLog.schema), - maxLogsHit: z.boolean(), - }), -); - -/** Response for the getPublicLogs archiver call. */ -export type GetPublicLogsResponse = { - /** An array of ExtendedPublicLog elements. */ - logs: ExtendedPublicLog[]; - /** Indicates if a limit has been reached. */ - maxLogsHit: boolean; -}; - -export const GetPublicLogsResponseSchema = zodFor()( - z.object({ - logs: z.array(ExtendedPublicLog.schema), - maxLogsHit: z.boolean(), - }), -); diff --git a/yarn-project/stdlib/src/interfaces/l2_logs_source.ts b/yarn-project/stdlib/src/interfaces/l2_logs_source.ts index 8d66368c4a15..23567c5c79ab 100644 --- a/yarn-project/stdlib/src/interfaces/l2_logs_source.ts +++ b/yarn-project/stdlib/src/interfaces/l2_logs_source.ts @@ -1,57 +1,28 @@ import type { BlockNumber } from '@aztec/foundation/branded-types'; -import type { AztecAddress } from '../aztec-address/index.js'; -import type { LogFilter } from '../logs/log_filter.js'; -import type { SiloedTag } from '../logs/siloed_tag.js'; -import type { Tag } from '../logs/tag.js'; -import type { TxScopedL2Log } from '../logs/tx_scoped_l2_log.js'; -import type { GetContractClassLogsResponse, GetPublicLogsResponse } from './get_logs_response.js'; +import type { LogResult } from '../logs/log_result.js'; +import type { PrivateLogsQuery, PublicLogsQuery } from '../logs/logs_query.js'; /** * Interface of classes allowing for the retrieval of logs. + * + * The return type is always the widest {@link LogResult} shape (noteHashes/nullifiers optional). To + * narrow at the call site after passing `includeEffects: true`, use the typed wrapper functions in + * `pxe/src/tagging/get_all_logs_by_tags.ts` (or cast — the wire payload is the widest shape, so a + * stricter generic on this interface would not survive the JSON-RPC boundary anyway). */ export interface L2LogsSource { /** - * Gets private logs that match any of the `tags`. For each tag, an array of matching logs is returned. An empty - * array implies no logs match that tag. - * @param tags - The tags to search for. - * @param page - The page number (0-indexed) for pagination. - * @param upToBlockNumber - If set, only return logs from blocks up to and including this block number. - * @returns An array of log arrays, one per tag. Returns at most 10 logs per tag per page. If 10 logs are returned - * for a tag, the caller should fetch the next page to check for more logs. + * Gets private logs matching the given tags. Returns one inner array per element of `query.tags`, in + * input order. An empty inner array means no logs matched that tag. */ - getPrivateLogsByTags(tags: SiloedTag[], page?: number, upToBlockNumber?: BlockNumber): Promise; + getPrivateLogsByTags(query: PrivateLogsQuery): Promise; /** - * Gets public logs that match any of the `tags` from the specified contract. For each tag, an array of matching - * logs is returned. An empty array implies no logs match that tag. - * @param contractAddress - The contract address to search logs for. - * @param tags - The tags to search for. - * @param page - The page number (0-indexed) for pagination. - * @param upToBlockNumber - If set, only return logs from blocks up to and including this block number. - * @returns An array of log arrays, one per tag. Returns at most 10 logs per tag per page. If 10 logs are returned - * for a tag, the caller should fetch the next page to check for more logs. + * Gets public logs matching the given tags for the given contract. Returns one inner array per element + * of `query.tags`, in input order. An empty inner array means no logs matched that tag. */ - getPublicLogsByTagsFromContract( - contractAddress: AztecAddress, - tags: Tag[], - page?: number, - upToBlockNumber?: BlockNumber, - ): Promise; - - /** - * Gets public logs based on the provided filter. - * @param filter - The filter to apply to the logs. - * @returns The requested logs. - */ - getPublicLogs(filter: LogFilter): Promise; - - /** - * Gets contract class logs based on the provided filter. - * @param filter - The filter to apply to the logs. - * @returns The requested logs. - */ - getContractClassLogs(filter: LogFilter): Promise; + getPublicLogsByTags(query: PublicLogsQuery): Promise; /** * Gets the number of the latest L2 block processed by the implementation. diff --git a/yarn-project/stdlib/src/logs/extended_contract_class_log.ts b/yarn-project/stdlib/src/logs/extended_contract_class_log.ts deleted file mode 100644 index 425ee5bbb6f5..000000000000 --- a/yarn-project/stdlib/src/logs/extended_contract_class_log.ts +++ /dev/null @@ -1,87 +0,0 @@ -import type { ZodFor } from '@aztec/foundation/schemas'; -import { BufferReader } from '@aztec/foundation/serialize'; -import { bufferToHex, hexToBuffer } from '@aztec/foundation/string'; -import type { FieldsOf } from '@aztec/foundation/types'; - -import isEqual from 'lodash.isequal'; -import { z } from 'zod'; - -import { ContractClassLog } from './contract_class_log.js'; -import { LogId } from './log_id.js'; - -/** - * Represents an individual contract class log entry extended with info about the block and tx it was emitted in. - */ -export class ExtendedContractClassLog { - constructor( - /** Globally unique id of the log. */ - public readonly id: LogId, - /** The data contents of the log. */ - public readonly log: ContractClassLog, - ) {} - - static async random() { - return new ExtendedContractClassLog(LogId.random(), await ContractClassLog.random()); - } - - static get schema(): ZodFor { - return z - .object({ - id: LogId.schema, - log: ContractClassLog.schema, - }) - .transform(ExtendedContractClassLog.from); - } - - static from(fields: FieldsOf) { - return new ExtendedContractClassLog(fields.id, fields.log); - } - - /** - * Serializes log to a buffer. - * @returns A buffer containing the serialized log. - */ - public toBuffer(): Buffer { - return Buffer.concat([this.id.toBuffer(), this.log.toBuffer()]); - } - - /** - * Serializes log to a string. - * @returns A string containing the serialized log. - */ - public toString(): string { - return bufferToHex(this.toBuffer()); - } - - /** - * Checks if two ExtendedContractClassLog objects are equal. - * @param other - Another ExtendedContractClassLog object to compare with. - * @returns True if the two objects are equal, false otherwise. - */ - public equals(other: ExtendedContractClassLog): boolean { - return isEqual(this, other); - } - - /** - * Deserializes log from a buffer. - * @param buffer - The buffer or buffer reader containing the log. - * @returns Deserialized instance of `Log`. - */ - public static fromBuffer(buffer: Buffer | BufferReader): ExtendedContractClassLog { - const reader = BufferReader.asReader(buffer); - - const logId = LogId.fromBuffer(reader); - const log = ContractClassLog.fromBuffer(reader); - - return new ExtendedContractClassLog(logId, log); - } - - /** - * Deserializes `ExtendedContractClassLog` object from a hex string representation. - * @param data - A hex string representation of the log. - * @returns An `ExtendedContractClassLog` object. - */ - public static fromString(data: string): ExtendedContractClassLog { - return ExtendedContractClassLog.fromBuffer(hexToBuffer(data)); - } -} diff --git a/yarn-project/stdlib/src/logs/extended_public_log.ts b/yarn-project/stdlib/src/logs/extended_public_log.ts deleted file mode 100644 index adca18ec340a..000000000000 --- a/yarn-project/stdlib/src/logs/extended_public_log.ts +++ /dev/null @@ -1,94 +0,0 @@ -import { BufferReader } from '@aztec/foundation/serialize'; -import { bufferToHex, hexToBuffer } from '@aztec/foundation/string'; -import type { FieldsOf } from '@aztec/foundation/types'; - -import isEqual from 'lodash.isequal'; -import { z } from 'zod'; - -import { LogId } from './log_id.js'; -import { PublicLog } from './public_log.js'; - -/** - * Represents an individual public log entry extended with info about the block and tx it was emitted in. - */ -export class ExtendedPublicLog { - constructor( - /** Globally unique id of the log. */ - public readonly id: LogId, - /** The data contents of the log. */ - public readonly log: PublicLog, - ) {} - - static async random() { - return new ExtendedPublicLog(LogId.random(), await PublicLog.random()); - } - - static get schema() { - return z - .object({ - id: LogId.schema, - log: PublicLog.schema, - }) - .transform(ExtendedPublicLog.from); - } - - static from(fields: FieldsOf) { - return new ExtendedPublicLog(fields.id, fields.log); - } - - /** - * Serializes log to a buffer. - * @returns A buffer containing the serialized log. - */ - public toBuffer(): Buffer { - return Buffer.concat([this.id.toBuffer(), this.log.toBuffer()]); - } - - /** - * Serializes log to a string. - * @returns A string containing the serialized log. - */ - public toString(): string { - return bufferToHex(this.toBuffer()); - } - - /** - * Serializes log to a human readable string. - * @returns A human readable representation of the log. - */ - public toHumanReadable(): string { - return `${this.id.toHumanReadable()}, ${this.log.toHumanReadable()}`; - } - - /** - * Checks if two ExtendedPublicLog objects are equal. - * @param other - Another ExtendedPublicLog object to compare with. - * @returns True if the two objects are equal, false otherwise. - */ - public equals(other: ExtendedPublicLog): boolean { - return isEqual(this, other); - } - - /** - * Deserializes log from a buffer. - * @param buffer - The buffer or buffer reader containing the log. - * @returns Deserialized instance of `Log`. - */ - public static fromBuffer(buffer: Buffer | BufferReader): ExtendedPublicLog { - const reader = BufferReader.asReader(buffer); - - const logId = LogId.fromBuffer(reader); - const log = PublicLog.fromBuffer(reader); - - return new ExtendedPublicLog(logId, log); - } - - /** - * Deserializes `ExtendedPublicLog` object from a hex string representation. - * @param data - A hex string representation of the log. - * @returns An `ExtendedPublicLog` object. - */ - public static fromString(data: string): ExtendedPublicLog { - return ExtendedPublicLog.fromBuffer(hexToBuffer(data)); - } -} diff --git a/yarn-project/stdlib/src/logs/index.ts b/yarn-project/stdlib/src/logs/index.ts index c79476b6e513..5b5adf950c22 100644 --- a/yarn-project/stdlib/src/logs/index.ts +++ b/yarn-project/stdlib/src/logs/index.ts @@ -6,12 +6,11 @@ export * from './contract_class_log.js'; export * from './public_log.js'; export * from './private_log.js'; export * from './pending_tagged_log.js'; -export * from './log_id.js'; -export * from './log_filter.js'; -export * from './extended_public_log.js'; -export * from './extended_contract_class_log.js'; +export * from './log_result.js'; +export * from './log_cursor.js'; +export * from './logs_query.js'; +export * from './query_all_logs_by_tags.js'; export * from './shared_secret_derivation.js'; -export * from './tx_scoped_l2_log.js'; export * from './message_context.js'; export * from './debug_log.js'; export * from './debug_log_store.js'; diff --git a/yarn-project/stdlib/src/logs/log_cursor.test.ts b/yarn-project/stdlib/src/logs/log_cursor.test.ts new file mode 100644 index 000000000000..df19f0dcaecd --- /dev/null +++ b/yarn-project/stdlib/src/logs/log_cursor.test.ts @@ -0,0 +1,60 @@ +import { BlockNumber } from '@aztec/foundation/branded-types'; +import { jsonStringify } from '@aztec/foundation/json-rpc'; + +import { LogCursor } from './log_cursor.js'; +import { randomLogResult } from './log_result.js'; + +describe('LogCursor', () => { + it('round-trips through toBuffer/fromBuffer', () => { + const cursor = LogCursor.random(); + const parsed = LogCursor.fromBuffer(cursor.toBuffer()); + expect(parsed.equals(cursor)).toBe(true); + }); + + it('round-trips through the zod schema', () => { + const cursor = LogCursor.random(); + const parsed = LogCursor.schema.parse(JSON.parse(jsonStringify(cursor))); + expect(parsed.equals(cursor)).toBe(true); + }); + + it('fromLog reads the cursor fields from a log', () => { + const log = randomLogResult(); + const cursor = LogCursor.fromLog(log); + expect(cursor.blockNumber).toBe(log.blockNumber); + expect(cursor.txIndexWithinBlock).toBe(log.txIndexWithinBlock); + expect(cursor.logIndexWithinTx).toBe(log.logIndexWithinTx); + }); + + it('cursors are equal iff all three fields match', () => { + const a = LogCursor.random(); + const b = new LogCursor(a.blockNumber, a.txIndexWithinBlock, a.logIndexWithinTx); + expect(a.equals(b)).toBe(true); + + const differentLogIndex = new LogCursor(a.blockNumber, a.txIndexWithinBlock, a.logIndexWithinTx + 1); + expect(a.equals(differentLogIndex)).toBe(false); + + const differentTxIndex = new LogCursor(a.blockNumber, a.txIndexWithinBlock + 1, a.logIndexWithinTx); + expect(a.equals(differentTxIndex)).toBe(false); + }); + + describe('toString / parseOptional', () => { + it('round-trips toString via parseOptional', () => { + const cursor = new LogCursor(BlockNumber(42), 3, 7); + expect(cursor.toString()).toBe('42-3-7'); + const parsed = LogCursor.parseOptional(cursor.toString())!; + expect(parsed.equals(cursor)).toBe(true); + }); + + it('returns undefined for empty input', () => { + expect(LogCursor.parseOptional('')).toBeUndefined(); + }); + + it('rejects malformed strings', () => { + expect(() => LogCursor.parseOptional('1-2')).toThrow(/Invalid log cursor/); + expect(() => LogCursor.parseOptional('a-b-c')).toThrow(/block number/); + expect(() => LogCursor.parseOptional('1-x-3')).toThrow(/tx index/); + expect(() => LogCursor.parseOptional('1-2-y')).toThrow(/log index/); + expect(() => LogCursor.parseOptional('0-0-0')).toThrow(/block number/); + }); + }); +}); diff --git a/yarn-project/stdlib/src/logs/log_cursor.ts b/yarn-project/stdlib/src/logs/log_cursor.ts new file mode 100644 index 000000000000..abd719bfd7e9 --- /dev/null +++ b/yarn-project/stdlib/src/logs/log_cursor.ts @@ -0,0 +1,110 @@ +import { BlockNumber, BlockNumberSchema } from '@aztec/foundation/branded-types'; +import { BufferReader, numToUInt32BE } from '@aztec/foundation/serialize'; + +import { z } from 'zod'; + +import { schemas } from '../schemas/schemas.js'; +import type { LogResultBase } from './log_result.js'; + +/** + * Cursor identifying a position in a tag's ordered log stream. Used as `afterLog` on `TagQuery` to resume + * pagination strictly after a previously-seen log. + * + * `(blockNumber, txIndexWithinBlock, logIndexWithinTx)` is sufficient to uniquely identify a position + * and matches the composite-key ordering of the archiver's log index, so the cursor maps directly to a + * key without a tx-hash lookup. All three fields are always present on {@link LogResult}, so a cursor is + * constructible from any returned log. + */ +export class LogCursor { + constructor( + /** The block the cursor points to. */ + public readonly blockNumber: BlockNumber, + /** The tx index within the block the cursor points to. */ + public readonly txIndexWithinBlock: number, + /** The log index within the tx the cursor points to. */ + public readonly logIndexWithinTx: number, + ) {} + + static get schema() { + return z + .object({ + blockNumber: BlockNumberSchema, + txIndexWithinBlock: schemas.Integer, + logIndexWithinTx: schemas.Integer, + }) + .transform( + ({ blockNumber, txIndexWithinBlock, logIndexWithinTx }) => + new LogCursor(blockNumber, txIndexWithinBlock, logIndexWithinTx), + ); + } + + /** Builds a cursor that points at the given log. Pagination resumes strictly after this position. */ + static fromLog(log: Pick): LogCursor { + return new LogCursor(log.blockNumber, log.txIndexWithinBlock, log.logIndexWithinTx); + } + + static random(): LogCursor { + return new LogCursor( + BlockNumber(Math.floor(Math.random() * 100000) + 1), + Math.floor(Math.random() * 100), + Math.floor(Math.random() * 100), + ); + } + + toBuffer(): Buffer { + return Buffer.concat([ + numToUInt32BE(this.blockNumber), + numToUInt32BE(this.txIndexWithinBlock), + numToUInt32BE(this.logIndexWithinTx), + ]); + } + + static fromBuffer(buffer: Buffer | BufferReader): LogCursor { + const reader = BufferReader.asReader(buffer); + const blockNumber = BlockNumber(reader.readNumber()); + const txIndexWithinBlock = reader.readNumber(); + const logIndexWithinTx = reader.readNumber(); + return new LogCursor(blockNumber, txIndexWithinBlock, logIndexWithinTx); + } + + equals(other: LogCursor): boolean { + return ( + this.blockNumber === other.blockNumber && + this.txIndexWithinBlock === other.txIndexWithinBlock && + this.logIndexWithinTx === other.logIndexWithinTx + ); + } + + toString(): string { + return `${this.blockNumber}-${this.txIndexWithinBlock}-${this.logIndexWithinTx}`; + } + + /** + * Parses a `--` triple into a {@link LogCursor}. + * Throws on malformed input. Returns `undefined` for an empty / missing value so callers can use + * this directly as an optional CLI option parser. + */ + static parseOptional(value: string): LogCursor | undefined { + if (!value) { + return undefined; + } + const parts = value.split('-'); + if (parts.length !== 3) { + throw new Error(`Invalid log cursor "${value}". Expected --.`); + } + const [blockNumberStr, txIndexStr, logIndexStr] = parts; + const blockNumber = Number(blockNumberStr); + const txIndexWithinBlock = Number(txIndexStr); + const logIndexWithinTx = Number(logIndexStr); + if (!Number.isInteger(blockNumber) || blockNumber < 1) { + throw new Error(`Invalid log cursor block number: ${blockNumberStr}`); + } + if (!Number.isInteger(txIndexWithinBlock) || txIndexWithinBlock < 0) { + throw new Error(`Invalid log cursor tx index: ${txIndexStr}`); + } + if (!Number.isInteger(logIndexWithinTx) || logIndexWithinTx < 0) { + throw new Error(`Invalid log cursor log index: ${logIndexStr}`); + } + return new LogCursor(BlockNumber(blockNumber), txIndexWithinBlock, logIndexWithinTx); + } +} diff --git a/yarn-project/stdlib/src/logs/log_filter.ts b/yarn-project/stdlib/src/logs/log_filter.ts deleted file mode 100644 index 75f97cd1ee20..000000000000 --- a/yarn-project/stdlib/src/logs/log_filter.ts +++ /dev/null @@ -1,36 +0,0 @@ -import type { Fr } from '@aztec/foundation/curves/bn254'; - -import { z } from 'zod'; - -import type { AztecAddress } from '../aztec-address/index.js'; -import { type ZodFor, schemas } from '../schemas/index.js'; -import { TxHash } from '../tx/tx_hash.js'; -import { LogId } from './log_id.js'; - -/** - * Log filter used to fetch L2 logs. - * @remarks This filter is applied as an intersection of all it's params. - */ -export type LogFilter = { - /** Hash of a transaction from which to fetch the logs. */ - txHash?: TxHash; - /** The block number from which to start fetching logs (inclusive). */ - fromBlock?: number; - /** The block number until which to fetch logs (not inclusive). */ - toBlock?: number; - /** Log id after which to start fetching logs. */ - afterLog?: LogId; - /** The contract address to filter logs by. */ - contractAddress?: AztecAddress; - /** The tag (first field of the log) to filter logs by. */ - tag?: Fr; -}; - -export const LogFilterSchema: ZodFor = z.object({ - txHash: TxHash.schema.optional(), - fromBlock: schemas.Integer.optional(), - toBlock: schemas.Integer.optional(), - afterLog: LogId.schema.optional(), - contractAddress: schemas.AztecAddress.optional(), - tag: schemas.Fr.optional(), -}); diff --git a/yarn-project/stdlib/src/logs/log_id.test.ts b/yarn-project/stdlib/src/logs/log_id.test.ts deleted file mode 100644 index 0a3a6c8ed3a1..000000000000 --- a/yarn-project/stdlib/src/logs/log_id.test.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { LogId } from './log_id.js'; - -describe('LogId', () => { - let logId: LogId; - beforeEach(() => { - logId = LogId.random(); - }); - - it('toBuffer and fromBuffer works', () => { - const buffer = logId.toBuffer(); - const parsedLogId = LogId.fromBuffer(buffer); - - expect(parsedLogId).toEqual(logId); - }); - - it('toString and fromString works', () => { - const str = logId.toString(); - const parsedLogId = LogId.fromString(str); - - expect(parsedLogId).toEqual(logId); - }); - - it('human readable string includes block hash', () => { - const human = logId.toHumanReadable(); - - expect(human).toContain(logId.blockHash.toString()); - }); -}); diff --git a/yarn-project/stdlib/src/logs/log_id.ts b/yarn-project/stdlib/src/logs/log_id.ts deleted file mode 100644 index 8db8f6ce6f9c..000000000000 --- a/yarn-project/stdlib/src/logs/log_id.ts +++ /dev/null @@ -1,124 +0,0 @@ -import { INITIAL_L2_BLOCK_NUM } from '@aztec/constants'; -import { toBufferBE } from '@aztec/foundation/bigint-buffer'; -import { BlockNumber, BlockNumberSchema } from '@aztec/foundation/branded-types'; -import { BufferReader } from '@aztec/foundation/serialize'; - -import { z } from 'zod'; - -import { BlockHash } from '../block/block_hash.js'; -import { schemas } from '../schemas/index.js'; -import { TxHash } from '../tx/tx_hash.js'; - -/** A globally unique log id. */ -export class LogId { - constructor( - /** The block number the log was emitted in. */ - public readonly blockNumber: BlockNumber, - /** The hash of the block the log was emitted in. */ - public readonly blockHash: BlockHash, - /** The hash of the transaction the log was emitted in. */ - public readonly txHash: TxHash, - /** The index of a tx in a block the log was emitted in. */ - public readonly txIndex: number, - /** The index of a log the tx was emitted in. */ - public readonly logIndex: number, - ) { - if (!Number.isInteger(blockNumber) || blockNumber < INITIAL_L2_BLOCK_NUM) { - throw new Error(`Invalid block number: ${blockNumber}`); - } - if (!Number.isInteger(txIndex)) { - throw new Error(`Invalid tx index: ${txIndex}`); - } - if (!Number.isInteger(logIndex)) { - throw new Error(`Invalid log index: ${logIndex}`); - } - } - - static random() { - return new LogId( - BlockNumber(Math.floor(Math.random() * 1000) + 1), - BlockHash.random(), - TxHash.random(), - Math.floor(Math.random() * 1000), - Math.floor(Math.random() * 100), - ); - } - - static get schema() { - return z - .object({ - blockNumber: BlockNumberSchema, - blockHash: BlockHash.schema, - txHash: TxHash.schema, - txIndex: schemas.Integer, - logIndex: schemas.Integer, - }) - .transform( - ({ blockNumber, blockHash, txHash, txIndex, logIndex }) => - new LogId(blockNumber, blockHash, txHash, txIndex, logIndex), - ); - } - - /** - * Serializes log id to a buffer. - * @returns A buffer containing the serialized log id. - */ - public toBuffer(): Buffer { - return Buffer.concat([ - toBufferBE(BigInt(this.blockNumber), 4), - this.blockHash.toBuffer(), - this.txHash.toBuffer(), - toBufferBE(BigInt(this.txIndex), 4), - toBufferBE(BigInt(this.logIndex), 4), - ]); - } - - /** - * Creates a LogId from a buffer. - * @param buffer - A buffer containing the serialized log id. - * @returns A log id. - */ - static fromBuffer(buffer: Buffer | BufferReader): LogId { - const reader = BufferReader.asReader(buffer); - - const blockNumber = BlockNumber(reader.readNumber()); - const blockHash = BlockHash.fromBuffer(reader); - const txHash = reader.readObject(TxHash); - const txIndex = reader.readNumber(); - const logIndex = reader.readNumber(); - - return new LogId(blockNumber, blockHash, txHash, txIndex, logIndex); - } - - /** - * Converts the LogId instance to a string. - * @returns A string representation of the log id. - */ - public toString(): string { - return `${this.blockNumber}-${this.txIndex}-${this.logIndex}-${this.blockHash.toString()}-${this.txHash.toString()}`; - } - - /** - * Creates a LogId from a string. - * @param data - A string representation of a log id. - * @returns A log id. - */ - static fromString(data: string): LogId { - const [rawBlockNumber, rawTxIndex, rawLogIndex, rawBlockHash, rawTxHash] = data.split('-'); - const blockNumber = BlockNumber(Number(rawBlockNumber)); - const blockHash = BlockHash.fromString(rawBlockHash); - const txHash = TxHash.fromString(rawTxHash); - const txIndex = Number(rawTxIndex); - const logIndex = Number(rawLogIndex); - - return new LogId(blockNumber, blockHash, txHash, txIndex, logIndex); - } - - /** - * Serializes log id to a human readable string. - * @returns A human readable representation of the log id. - */ - public toHumanReadable(): string { - return `logId: (blockNumber: ${this.blockNumber}, blockHash: ${this.blockHash.toString()}, txHash: ${this.txHash.toString()}, txIndex: ${this.txIndex}, logIndex: ${this.logIndex})`; - } -} diff --git a/yarn-project/stdlib/src/logs/log_result.test.ts b/yarn-project/stdlib/src/logs/log_result.test.ts new file mode 100644 index 000000000000..0e84479d3f31 --- /dev/null +++ b/yarn-project/stdlib/src/logs/log_result.test.ts @@ -0,0 +1,31 @@ +import { jsonStringify } from '@aztec/foundation/json-rpc'; + +import { LogResultSchema, logResultToHumanReadable, randomLogResult } from './log_result.js'; + +describe('LogResult', () => { + it('round-trips through the zod schema without effects', () => { + const log = randomLogResult(false); + const parsed = LogResultSchema.parse(JSON.parse(jsonStringify(log))); + expect(parsed).toEqual(log); + expect(parsed.noteHashes).toBeUndefined(); + expect(parsed.nullifiers).toBeUndefined(); + }); + + it('round-trips through the zod schema with effects', () => { + const log = randomLogResult(true); + const parsed = LogResultSchema.parse(JSON.parse(jsonStringify(log))); + expect(parsed).toEqual(log); + expect(parsed.noteHashes).toEqual(log.noteHashes); + expect(parsed.nullifiers).toEqual(log.nullifiers); + }); + + it('logResultToHumanReadable includes effect fields only when present', () => { + const without = randomLogResult(false); + expect(logResultToHumanReadable(without)).not.toContain('noteHashes'); + expect(logResultToHumanReadable(without)).not.toContain('nullifiers'); + + const withEffects = randomLogResult(true); + expect(logResultToHumanReadable(withEffects)).toContain('noteHashes'); + expect(logResultToHumanReadable(withEffects)).toContain('nullifiers'); + }); +}); diff --git a/yarn-project/stdlib/src/logs/log_result.ts b/yarn-project/stdlib/src/logs/log_result.ts new file mode 100644 index 000000000000..b65adcd92f10 --- /dev/null +++ b/yarn-project/stdlib/src/logs/log_result.ts @@ -0,0 +1,104 @@ +import { BlockNumber, BlockNumberSchema } from '@aztec/foundation/branded-types'; +import { times } from '@aztec/foundation/collection'; +import { Fr } from '@aztec/foundation/curves/bn254'; +import { schemas as foundationSchemas } from '@aztec/foundation/schemas'; +import type { IfFlag, Prettify } from '@aztec/foundation/types'; + +import { z } from 'zod'; + +import { BlockHash } from '../block/block_hash.js'; +import { schemas } from '../schemas/schemas.js'; +import { TxHash } from '../tx/tx_hash.js'; +import type { UInt64 } from '../types/shared.js'; + +/** Options for narrowing the shape of a {@link LogResult}. */ +export type LogIncludeOptions = { + /** When set, the log carries `noteHashes` + all `nullifiers` from its tx effect. Off by default. */ + includeEffects?: boolean; +}; + +export const LogIncludeOptionsSchema: z.ZodType = z.object({ + includeEffects: z.boolean().optional(), +}); + +/** Required metadata always present on a {@link LogResult}. */ +export type LogResultBase = { + /** The data contents of the log, with the tag as the first field. */ + logData: Fr[]; + /** The block this log was emitted in. */ + blockNumber: BlockNumber; + /** The hash of the block this log was emitted in. */ + blockHash: BlockHash; + /** The timestamp of the block this log was emitted in. */ + blockTimestamp: UInt64; + /** The hash of the tx this log was emitted in. */ + txHash: TxHash; + /** The 0-based index of this log's tx within its block. */ + txIndexWithinBlock: number; + /** The 0-based index of this log within its tx (across both private and public logs for the tx). */ + logIndexWithinTx: number; +}; + +/** + * A single log returned from {@link L2LogsSource.getPrivateLogsByTags} or {@link L2LogsSource.getPublicLogsByTags}. + * + * Generic over the include-options so that flagged fields become required when the caller passes a + * literal `true`. The default type argument ({@link LogIncludeOptions}) yields the widest shape + * (all include-fields optional) — this is what the JSON-RPC wire layer validates against. + * + * `logData` is the raw field-element payload, with the tag in field 0 (consumers slice it off). + * + * @example + * const l: LogResult<{ includeEffects: true }> = (await node.getPrivateLogsByTags({ tags, includeEffects: true }))[0][0]; + * l.nullifiers; // required, not optional + */ +export type LogResult = Prettify< + LogResultBase & IfFlag +>; + +/** Zod schema for the widest {@link LogResult} shape (all include-gated fields optional). */ +export const LogResultSchema: z.ZodType = z.object({ + logData: z.array(foundationSchemas.Fr), + blockNumber: BlockNumberSchema, + blockHash: BlockHash.schema, + blockTimestamp: schemas.UInt64, + txHash: TxHash.schema, + txIndexWithinBlock: schemas.Integer, + logIndexWithinTx: schemas.Integer, + noteHashes: z.array(foundationSchemas.Fr).optional(), + nullifiers: z.array(foundationSchemas.Fr).optional(), +}); + +/** Builds a random {@link LogResult} for tests. `includeEffects` populates `noteHashes` + `nullifiers`. */ +export function randomLogResult(includeEffects = false): LogResult { + const base: LogResultBase = { + logData: times(3, Fr.random), + blockNumber: BlockNumber(Math.floor(Math.random() * 100000) + 1), + blockHash: BlockHash.random(), + blockTimestamp: BigInt(Math.floor(Date.now() / 1000)), + txHash: TxHash.random(), + txIndexWithinBlock: Math.floor(Math.random() * 100), + logIndexWithinTx: Math.floor(Math.random() * 100), + }; + if (includeEffects) { + return { ...base, noteHashes: times(3, Fr.random), nullifiers: times(3, Fr.random) }; + } + return base; +} + +/** Human-readable single-line representation, primarily for the CLI `get-logs` command. */ +export function logResultToHumanReadable(log: LogResult): string { + const head = + `block ${log.blockNumber} (${log.blockHash.toString()}) ` + + `tx ${log.txHash.toString()} txIndex ${log.txIndexWithinBlock} logIndex ${log.logIndexWithinTx} ` + + `ts ${log.blockTimestamp}`; + const data = `data [${log.logData.map(f => f.toString()).join(', ')}]`; + const parts = [head, data]; + if (log.noteHashes !== undefined) { + parts.push(`noteHashes [${log.noteHashes.map(f => f.toString()).join(', ')}]`); + } + if (log.nullifiers !== undefined) { + parts.push(`nullifiers [${log.nullifiers.map(f => f.toString()).join(', ')}]`); + } + return parts.join(' | '); +} diff --git a/yarn-project/stdlib/src/logs/logs_query.test.ts b/yarn-project/stdlib/src/logs/logs_query.test.ts new file mode 100644 index 000000000000..4f5850af1d02 --- /dev/null +++ b/yarn-project/stdlib/src/logs/logs_query.test.ts @@ -0,0 +1,47 @@ +import { jsonStringify } from '@aztec/foundation/json-rpc'; + +import { AztecAddress } from '../aztec-address/index.js'; +import { MAX_RPC_LEN } from '../interfaces/api_limit.js'; +import { PrivateLogsQuerySchema, PublicLogsQuerySchema } from './logs_query.js'; +import { SiloedTag } from './siloed_tag.js'; +import { Tag } from './tag.js'; + +/** Serialize a query through the JSON wire format the schemas are designed to parse. */ +function wire(value: T): unknown { + return JSON.parse(jsonStringify(value)); +} + +describe('PrivateLogsQuerySchema', () => { + it('accepts a tags array of exactly MAX_RPC_LEN entries', () => { + const tags = Array.from({ length: MAX_RPC_LEN }, () => SiloedTag.random()); + expect(() => PrivateLogsQuerySchema.parse(wire({ tags }))).not.toThrow(); + }); + + it('rejects a tags array longer than MAX_RPC_LEN', () => { + const tags = Array.from({ length: MAX_RPC_LEN + 1 }, () => SiloedTag.random()); + expect(() => PrivateLogsQuerySchema.parse(wire({ tags }))).toThrow(/at most/); + }); + + it('rejects an empty tags array', () => { + expect(() => PrivateLogsQuerySchema.parse(wire({ tags: [] }))).toThrow(); + }); +}); + +describe('PublicLogsQuerySchema', () => { + it('accepts a tags array of exactly MAX_RPC_LEN entries', async () => { + const contractAddress = await AztecAddress.random(); + const tags = Array.from({ length: MAX_RPC_LEN }, () => Tag.random()); + expect(() => PublicLogsQuerySchema.parse(wire({ contractAddress, tags }))).not.toThrow(); + }); + + it('rejects a tags array longer than MAX_RPC_LEN', async () => { + const contractAddress = await AztecAddress.random(); + const tags = Array.from({ length: MAX_RPC_LEN + 1 }, () => Tag.random()); + expect(() => PublicLogsQuerySchema.parse(wire({ contractAddress, tags }))).toThrow(/at most/); + }); + + it('rejects an empty tags array', async () => { + const contractAddress = await AztecAddress.random(); + expect(() => PublicLogsQuerySchema.parse(wire({ contractAddress, tags: [] }))).toThrow(); + }); +}); diff --git a/yarn-project/stdlib/src/logs/logs_query.ts b/yarn-project/stdlib/src/logs/logs_query.ts new file mode 100644 index 000000000000..2ab4fdd94ac8 --- /dev/null +++ b/yarn-project/stdlib/src/logs/logs_query.ts @@ -0,0 +1,138 @@ +import { BlockNumberSchema } from '@aztec/foundation/branded-types'; +import type { BlockNumber } from '@aztec/foundation/branded-types'; + +import { z } from 'zod'; + +import type { AztecAddress } from '../aztec-address/index.js'; +import { BlockHash } from '../block/block_hash.js'; +import { MAX_LOGS_PER_TAG, MAX_RPC_LEN } from '../interfaces/api_limit.js'; +import { type ZodFor, schemas, zodFor } from '../schemas/index.js'; +import { TxHash } from '../tx/tx_hash.js'; +import { LogCursor } from './log_cursor.js'; +import { SiloedTag } from './siloed_tag.js'; +import { Tag } from './tag.js'; + +/** + * A tag to query in {@link PrivateLogsQuery} / {@link PublicLogsQuery}, optionally resuming strictly + * after a previously-seen log via `afterLog`. The bare `T` form means "from the beginning". + */ +export type TagQuery = T | { tag: T; afterLog?: LogCursor }; + +/** + * Shared fields for {@link PrivateLogsQuery} and {@link PublicLogsQuery}. + */ +export type LogsQueryBase = { + /** Lower block bound, inclusive. */ + fromBlock?: BlockNumber; + /** Upper block bound, exclusive. */ + toBlock?: BlockNumber; + /** + * Restrict results to logs emitted in this transaction. Mutually exclusive with `fromBlock`/`toBlock` + * (a txHash already pins a block). `txHash` + `afterLog` is allowed and paginates within the tx's logs + * for that tag. + */ + txHash?: TxHash; + /** + * Reorg-safety anchor: the latest block hash the caller has synced to. If set and the block is no + * longer present, the call throws. Distinct from `toBlock`, which is a filter, not a safety check. + */ + referenceBlock?: BlockHash; + /** When set, each log also carries `noteHashes` and all `nullifiers` for note-nonce discovery. */ + includeEffects?: boolean; + /** + * Maximum number of logs returned per tag. Capped at {@link MAX_LOGS_PER_TAG} (rejected if higher). + * Defaults to {@link MAX_LOGS_PER_TAG} when unset. Mainly useful for tests that need to force + * pagination at a small page size. + */ + limitPerTag?: number; +}; + +/** + * Query for {@link L2LogsSource.getPrivateLogsByTags}. Returns one inner array per element of `tags`, + * in input order. + */ +export type PrivateLogsQuery = LogsQueryBase & { + /** Tags to query. Between 1 and {@link MAX_RPC_LEN} entries (inclusive). */ + tags: TagQuery[]; +}; + +/** + * Query for {@link L2LogsSource.getPublicLogsByTags}. Returns one inner array per element of `tags`, + * in input order. + */ +export type PublicLogsQuery = LogsQueryBase & { + /** Contract address that emitted the logs. Required for public queries. */ + contractAddress: AztecAddress; + /** Tags to query. Between 1 and {@link MAX_RPC_LEN} entries (inclusive). */ + tags: TagQuery[]; +}; + +function tagQuerySchema(tagSchema: ZodFor) { + return z.union([ + tagSchema, + z.object({ + tag: tagSchema, + afterLog: LogCursor.schema.optional(), + }), + ]) as ZodFor>; +} + +const logsQueryBaseShape = { + fromBlock: BlockNumberSchema.optional(), + toBlock: BlockNumberSchema.optional(), + txHash: TxHash.schema.optional(), + referenceBlock: BlockHash.schema.optional(), + includeEffects: z.boolean().optional(), + limitPerTag: z + .number() + .int() + .positive() + .max(MAX_LOGS_PER_TAG, { message: `limitPerTag must be <= ${MAX_LOGS_PER_TAG}` }) + .optional(), +}; + +/** Minimal shape required by {@link refineTxHashAndRange}. */ +type TxHashAndRangeFields = { + /** Tx hash filter. */ + txHash?: TxHash; + /** Lower block bound. */ + fromBlock?: BlockNumber; + /** Upper block bound (exclusive). */ + toBlock?: BlockNumber; +}; + +/** + * Refinement: a `txHash` already pins a block, so combining it with a block range is contradictory. + * (`txHash` + `afterLog` is still allowed and is enforced per-tag inside `TagQuery`.) Exported so + * `aztec.js`'s wallet event-filter schemas can reuse the same rule. + */ +export function refineTxHashAndRange(schema: z.ZodType) { + return schema.refine(q => !(q.txHash !== undefined && (q.fromBlock !== undefined || q.toBlock !== undefined)), { + message: '`txHash` is mutually exclusive with `fromBlock`/`toBlock`', + }); +} + +export const PrivateLogsQuerySchema: ZodFor = refineTxHashAndRange( + zodFor()( + z.object({ + ...logsQueryBaseShape, + tags: z + .array(tagQuerySchema(SiloedTag.schema)) + .min(1) + .max(MAX_RPC_LEN, { message: `tags must have at most ${MAX_RPC_LEN} entries` }), + }), + ), +); + +export const PublicLogsQuerySchema: ZodFor = refineTxHashAndRange( + zodFor()( + z.object({ + ...logsQueryBaseShape, + contractAddress: schemas.AztecAddress, + tags: z + .array(tagQuerySchema(Tag.schema)) + .min(1) + .max(MAX_RPC_LEN, { message: `tags must have at most ${MAX_RPC_LEN} entries` }), + }), + ), +); diff --git a/yarn-project/stdlib/src/logs/query_all_logs_by_tags.ts b/yarn-project/stdlib/src/logs/query_all_logs_by_tags.ts new file mode 100644 index 000000000000..b949d2d6be33 --- /dev/null +++ b/yarn-project/stdlib/src/logs/query_all_logs_by_tags.ts @@ -0,0 +1,98 @@ +import { MAX_LOGS_PER_TAG } from '../interfaces/api_limit.js'; +import { LogCursor } from './log_cursor.js'; +import type { LogResult } from './log_result.js'; +import type { PrivateLogsQuery, PublicLogsQuery, TagQuery } from './logs_query.js'; +import type { SiloedTag } from './siloed_tag.js'; +import type { Tag } from './tag.js'; + +/** Minimal node surface needed by {@link queryAllPrivateLogsByTags}. */ +export type PrivateLogsByTagsFetcher = { + getPrivateLogsByTags(query: PrivateLogsQuery): Promise; +}; + +/** Minimal node surface needed by {@link queryAllPublicLogsByTags}. */ +export type PublicLogsByTagsFetcher = { + getPublicLogsByTags(query: PublicLogsQuery): Promise; +}; + +/** + * Drives the per-tag `afterLog` cursor loop for {@link PrivateLogsByTagsFetcher.getPrivateLogsByTags}. + * + * Each round re-queries only the tags whose previous page was full (`length === effective limit`), passing + * the cursor of the last seen log. Tags drop out as soon as they return a short page. Results are stitched + * back into one inner array per input tag, preserving the original input order. + * + * Honors {@link PrivateLogsQuery.limitPerTag} when set (caller is responsible for keeping it `<=` + * {@link MAX_LOGS_PER_TAG}; the query schema enforces this on the RPC boundary). + */ +export function queryAllPrivateLogsByTags( + node: PrivateLogsByTagsFetcher, + query: PrivateLogsQuery, +): Promise { + return queryAllByTags( + query.tags, + tagQueries => node.getPrivateLogsByTags({ ...query, tags: tagQueries }), + query.limitPerTag, + ); +} + +/** {@inheritDoc queryAllPrivateLogsByTags} */ +export function queryAllPublicLogsByTags( + node: PublicLogsByTagsFetcher, + query: PublicLogsQuery, +): Promise { + return queryAllByTags( + query.tags, + tagQueries => node.getPublicLogsByTags({ ...query, tags: tagQueries }), + query.limitPerTag, + ); +} + +/** + * Generic per-tag pagination loop, abstracted over the query shape (private vs public). Caller supplies + * a `fetchPage` that runs one round against the node with the still-active subset of tag queries. + * + * @param tags - Input tag queries in original order. Returned outer array has the same length and order. + * @param fetchPage - Per-round fetch hook; receives the active subset and returns one page per active tag. + * @param limitPerTag - Effective page cap used to detect "full page" → keep paginating; defaults to + * {@link MAX_LOGS_PER_TAG} (the server-side cap when the query doesn't override). + */ +async function queryAllByTags( + tags: TagQuery[], + fetchPage: (tagQueries: TagQuery[]) => Promise, + limitPerTag: number | undefined, +): Promise { + const effectiveLimit = limitPerTag ?? MAX_LOGS_PER_TAG; + const allResultsPerTag: LogResult[][] = tags.map(() => []); + let activeIndexes = tags.map((_, i) => i); + let nextQueries: TagQuery[] = tags.slice(); + + while (activeIndexes.length > 0) { + const pageResults = await fetchPage(nextQueries); + + const stillActive: number[] = []; + const followups: TagQuery[] = []; + for (let i = 0; i < activeIndexes.length; i++) { + const originalIdx = activeIndexes[i]; + const pageForTag = pageResults[i]; + allResultsPerTag[originalIdx].push(...pageForTag); + if (pageForTag.length === effectiveLimit) { + const lastLog = pageForTag[pageForTag.length - 1]; + stillActive.push(originalIdx); + followups.push({ tag: tagOf(tags[originalIdx]), afterLog: LogCursor.fromLog(lastLog) }); + } + } + activeIndexes = stillActive; + nextQueries = followups; + } + + return allResultsPerTag; +} + +/** Unwraps a `TagQuery` to its underlying `Tag` / `SiloedTag` regardless of the bare-vs-object form. */ +function tagOf(entry: TagQuery): T { + if (typeof entry === 'object' && entry !== null && 'tag' in entry) { + return entry.tag; + } + return entry as T; +} diff --git a/yarn-project/stdlib/src/logs/tx_scoped_l2_log.test.ts b/yarn-project/stdlib/src/logs/tx_scoped_l2_log.test.ts deleted file mode 100644 index d7b42331e3cf..000000000000 --- a/yarn-project/stdlib/src/logs/tx_scoped_l2_log.test.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { TxScopedL2Log } from './tx_scoped_l2_log.js'; - -describe('TxScopedL2Log', () => { - it('should serialize and deserialize correctly', () => { - const log = TxScopedL2Log.random(); - const buffer = log.toBuffer(); - const deserializedLog = TxScopedL2Log.fromBuffer(buffer); - expect(deserializedLog.equals(log)).toBe(true); - }); - - it('should extract block number from buffer correctly', () => { - const log = TxScopedL2Log.random(); - const buffer = log.toBuffer(); - const blockNumber = TxScopedL2Log.getBlockNumberFromBuffer(buffer); - expect(blockNumber).toBe(log.blockNumber); - }); -}); diff --git a/yarn-project/stdlib/src/logs/tx_scoped_l2_log.ts b/yarn-project/stdlib/src/logs/tx_scoped_l2_log.ts deleted file mode 100644 index 7e9264ac94de..000000000000 --- a/yarn-project/stdlib/src/logs/tx_scoped_l2_log.ts +++ /dev/null @@ -1,114 +0,0 @@ -import { BlockNumber, BlockNumberSchema } from '@aztec/foundation/branded-types'; -import { times } from '@aztec/foundation/collection'; -import { Fr } from '@aztec/foundation/curves/bn254'; -import { schemas as foundationSchemas } from '@aztec/foundation/schemas'; -import { - BufferReader, - bigintToUInt64BE, - numToUInt32BE, - serializeArrayOfBufferableToVector, -} from '@aztec/foundation/serialize'; - -import { z } from 'zod'; - -import { schemas } from '../schemas/schemas.js'; -import { TxHash } from '../tx/tx_hash.js'; -import type { UInt64 } from '../types/shared.js'; - -export class TxScopedL2Log { - constructor( - /* - * Hash of the tx where the log is included - */ - public txHash: TxHash, - /* - * The block this log is included in - */ - public blockNumber: BlockNumber, - /* - * The timestamp of the block this log is included in - */ - public blockTimestamp: UInt64, - /* - * The log data as an array of field elements - */ - public logData: Fr[], - /* - * The note hashes from the tx effect - */ - public noteHashes: Fr[], - /* - * The first nullifier from the tx effect. Used for nonce discovery when processing notes from logs. - * - * (Note nonces are computed as `hash(firstNullifier, noteIndexInTx)`.) - */ - public firstNullifier: Fr, - ) {} - - static get schema() { - return z - .object({ - txHash: TxHash.schema, - blockNumber: BlockNumberSchema, - blockTimestamp: schemas.UInt64, - logData: z.array(foundationSchemas.Fr), - noteHashes: z.array(foundationSchemas.Fr), - firstNullifier: foundationSchemas.Fr, - }) - .transform( - ({ txHash, blockNumber, blockTimestamp, logData, noteHashes, firstNullifier }) => - new TxScopedL2Log(txHash, blockNumber, blockTimestamp, logData, noteHashes, firstNullifier), - ); - } - - toBuffer() { - return Buffer.concat([ - this.txHash.toBuffer(), - numToUInt32BE(this.blockNumber), - bigintToUInt64BE(this.blockTimestamp), - serializeArrayOfBufferableToVector(this.logData), - serializeArrayOfBufferableToVector(this.noteHashes), - this.firstNullifier.toBuffer(), - ]); - } - - static fromBuffer(buffer: Buffer) { - const reader = BufferReader.asReader(buffer); - const txHash = reader.readObject(TxHash); - const blockNumber = BlockNumber(reader.readNumber()); - const blockTimestamp = reader.readUInt64(); - const logData = reader.readVector(Fr); - const noteHashes = reader.readVector(Fr); - const firstNullifier = reader.readObject(Fr); - - return new TxScopedL2Log(txHash, blockNumber, blockTimestamp, logData, noteHashes, firstNullifier); - } - - static getBlockNumberFromBuffer(buffer: Buffer) { - return BlockNumber(buffer.readUint32BE(Fr.SIZE_IN_BYTES)); - } - - static random() { - return new TxScopedL2Log( - TxHash.fromField(Fr.random()), - BlockNumber(Math.floor(Math.random() * 100000) + 1), - BigInt(Math.floor(Date.now() / 1000)), - times(3, Fr.random), - times(3, Fr.random), - Fr.random(), - ); - } - - equals(other: TxScopedL2Log) { - return ( - this.txHash.equals(other.txHash) && - this.blockNumber === other.blockNumber && - this.blockTimestamp === other.blockTimestamp && - this.logData.length === other.logData.length && - this.logData.every((f, i) => f.equals(other.logData[i])) && - this.noteHashes.length === other.noteHashes.length && - this.noteHashes.every((h, i) => h.equals(other.noteHashes[i])) && - this.firstNullifier.equals(other.firstNullifier) - ); - } -} diff --git a/yarn-project/stdlib/src/tests/factories.ts b/yarn-project/stdlib/src/tests/factories.ts index f0c25eb6eb3d..d29a1c805b9b 100644 --- a/yarn-project/stdlib/src/tests/factories.ts +++ b/yarn-project/stdlib/src/tests/factories.ts @@ -84,6 +84,7 @@ import { import { PublicDataRead } from '../avm/public_data_read.js'; import { PublicDataWrite } from '../avm/public_data_write.js'; import { AztecAddress } from '../aztec-address/index.js'; +import { BlockHash } from '../block/block_hash.js'; import type { L2Tips } from '../block/l2_block_source.js'; import { type ContractClassPublic, @@ -127,9 +128,9 @@ import { PublicKey, PublicKeys, computeAddress, hashPublicKey } from '../keys/in import { AppTaggingSecret } from '../logs/app_tagging_secret.js'; import { AppTaggingSecretKind } from '../logs/app_tagging_secret_kind.js'; import { ContractClassLog, ContractClassLogFields } from '../logs/index.js'; +import type { LogResult } from '../logs/log_result.js'; import { PrivateLog } from '../logs/private_log.js'; import { FlatPublicLogs, PublicLog } from '../logs/public_log.js'; -import { TxScopedL2Log } from '../logs/tx_scoped_l2_log.js'; import { CountedL2ToL1Message, L2ToL1Message, ScopedL2ToL1Message } from '../messaging/l2_to_l1_message.js'; import { ParityBasePrivateInputs } from '../parity/parity_base_private_inputs.js'; import { ParityPublicInputs } from '../parity/parity_public_inputs.js'; @@ -1642,47 +1643,71 @@ export function fr(n: number): Fr { return new Fr(BigInt(n)); } -/** - * Creates a random TxScopedL2Log with private log data. - */ -export function randomTxScopedPrivateL2Log(opts?: { +/** Creates a random {@link LogResult} with private-log-shaped data. `includeEffects` populates `noteHashes` + `nullifiers`. */ +export function randomPrivateLogResult(opts?: { tag?: Fr; txHash?: TxHash; blockNumber?: number; + blockHash?: BlockHash; blockTimestamp?: bigint; + txIndexWithinBlock?: number; + logIndexWithinTx?: number; noteHashes?: Fr[]; - firstNullifier?: Fr; -}) { + nullifiers?: Fr[]; + includeEffects?: boolean; +}): LogResult { const log = PrivateLog.random(opts?.tag); - return new TxScopedL2Log( - opts?.txHash ?? TxHash.random(), - BlockNumber(opts?.blockNumber ?? 1), - opts?.blockTimestamp ?? 1n, - log.getEmittedFields(), - opts?.noteHashes ?? [Fr.random(), Fr.random()], - opts?.firstNullifier ?? Fr.random(), - ); -} - -/** - * Creates a random TxScopedL2Log with public log data. - */ -export async function randomTxScopedPublicL2Log(opts?: { + const includeEffects = opts?.includeEffects ?? (opts?.noteHashes !== undefined || opts?.nullifiers !== undefined); + const base: LogResult = { + logData: log.getEmittedFields(), + blockNumber: BlockNumber(opts?.blockNumber ?? 1), + blockHash: opts?.blockHash ?? BlockHash.random(), + blockTimestamp: opts?.blockTimestamp ?? 1n, + txHash: opts?.txHash ?? TxHash.random(), + txIndexWithinBlock: opts?.txIndexWithinBlock ?? 0, + logIndexWithinTx: opts?.logIndexWithinTx ?? 0, + }; + if (includeEffects) { + return { + ...base, + noteHashes: opts?.noteHashes ?? [Fr.random(), Fr.random()], + nullifiers: opts?.nullifiers ?? [Fr.random()], + }; + } + return base; +} + +/** Creates a random {@link LogResult} with public-log-shaped data. `includeEffects` populates `noteHashes` + `nullifiers`. */ +export async function randomPublicLogResult(opts?: { txHash?: TxHash; blockNumber?: number; + blockHash?: BlockHash; blockTimestamp?: bigint; + txIndexWithinBlock?: number; + logIndexWithinTx?: number; noteHashes?: Fr[]; - firstNullifier?: Fr; -}) { + nullifiers?: Fr[]; + includeEffects?: boolean; +}): Promise { const log = await PublicLog.random(); - return new TxScopedL2Log( - opts?.txHash ?? TxHash.random(), - BlockNumber(opts?.blockNumber ?? 1), - opts?.blockTimestamp ?? 1n, - log.getEmittedFields(), - opts?.noteHashes ?? [Fr.random(), Fr.random()], - opts?.firstNullifier ?? Fr.random(), - ); + const includeEffects = opts?.includeEffects ?? (opts?.noteHashes !== undefined || opts?.nullifiers !== undefined); + const base: LogResult = { + logData: log.getEmittedFields(), + blockNumber: BlockNumber(opts?.blockNumber ?? 1), + blockHash: opts?.blockHash ?? BlockHash.random(), + blockTimestamp: opts?.blockTimestamp ?? 1n, + txHash: opts?.txHash ?? TxHash.random(), + txIndexWithinBlock: opts?.txIndexWithinBlock ?? 0, + logIndexWithinTx: opts?.logIndexWithinTx ?? 0, + }; + if (includeEffects) { + return { + ...base, + noteHashes: opts?.noteHashes ?? [Fr.random(), Fr.random()], + nullifiers: opts?.nullifiers ?? [Fr.random()], + }; + } + return base; } /** diff --git a/yarn-project/stdlib/src/tx/tx.ts b/yarn-project/stdlib/src/tx/tx.ts index fa5fe768c974..3a70c3c9f5dd 100644 --- a/yarn-project/stdlib/src/tx/tx.ts +++ b/yarn-project/stdlib/src/tx/tx.ts @@ -8,8 +8,6 @@ import { type FieldsOf, unfreeze } from '@aztec/foundation/types'; import { z } from 'zod'; import type { GasSettings } from '../gas/gas_settings.js'; -import type { GetPublicLogsResponse } from '../interfaces/get_logs_response.js'; -import type { L2LogsSource } from '../interfaces/l2_logs_source.js'; import type { PublicCallRequest } from '../kernel/index.js'; import { PrivateKernelTailCircuitPublicInputs } from '../kernel/private_kernel_tail_circuit_public_inputs.js'; import { ContractClassLog, ContractClassLogFields } from '../logs/contract_class_log.js'; @@ -180,15 +178,6 @@ export class Tx extends Gossipable { return this.txHash.equals(expectedHash); } - /** - * Gets public logs emitted by this tx. - * @param logsSource - An instance of `L2LogsSource` which can be used to obtain the logs. - * @returns The requested logs. - */ - public getPublicLogs(logsSource: L2LogsSource): Promise { - return logsSource.getPublicLogs({ txHash: this.getTxHash() }); - } - getContractClassLogs(): ContractClassLog[] { const logHashes = this.data.getNonEmptyContractClassLogsHashes(); return logHashes.map((logHash, i) => diff --git a/yarn-project/txe/src/state_machine/archiver.ts b/yarn-project/txe/src/state_machine/archiver.ts index ec153566daa1..ab58512250b9 100644 --- a/yarn-project/txe/src/state_machine/archiver.ts +++ b/yarn-project/txe/src/state_machine/archiver.ts @@ -26,7 +26,7 @@ export class TXEArchiver extends ArchiverDataSourceBase { constructor(db: AztecAsyncKVStore) { super( - createArchiverDataStores(db, { logsMaxPageSize: 9999 }), + createArchiverDataStores(db, GENESIS_BLOCK_HEADER_HASH), undefined, BlockHeader.empty(), GENESIS_BLOCK_HEADER_HASH, diff --git a/yarn-project/validator-client/src/validator.integration.test.ts b/yarn-project/validator-client/src/validator.integration.test.ts index 99d4caf2e609..dd373dd3b84f 100644 --- a/yarn-project/validator-client/src/validator.integration.test.ts +++ b/yarn-project/validator-client/src/validator.integration.test.ts @@ -20,7 +20,7 @@ import type { P2P, PeerId } from '@aztec/p2p'; import { TestTxProvider } from '@aztec/p2p/test-helpers'; import { protocolContractsHash } from '@aztec/protocol-contracts'; import { AztecAddress } from '@aztec/stdlib/aztec-address'; -import { CommitteeAttestation, L2Block } from '@aztec/stdlib/block'; +import { CommitteeAttestation, GENESIS_BLOCK_HEADER_HASH, L2Block } from '@aztec/stdlib/block'; import { CheckpointReexecutionTracker, L1PublishedData, PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; import { type L1RollupConstants, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; import { Gas, GasFees } from '@aztec/stdlib/gas'; @@ -98,11 +98,14 @@ describe('ValidatorClient Integration', () => { /** Creates a new validator and dependencies */ const createValidatorContext = async (privateKey: Hex<32>): Promise => { // Create archiver store and NoopL1Archiver - const archiverStore = await createArchiverStore({ - archiverStoreMapSizeKb: 1024 * 1024, - dataDirectory: undefined, - dataStoreMapSizeKb: 1024 * 1024, - }); + const archiverStore = await createArchiverStore( + { + archiverStoreMapSizeKb: 1024 * 1024, + dataDirectory: undefined, + dataStoreMapSizeKb: 1024 * 1024, + }, + GENESIS_BLOCK_HEADER_HASH, + ); await registerProtocolContracts(archiverStore); // Construct world-state first so we can pass its initial header to the archiver, mirroring From 69d7b586d8e831dc4d92ea839db611f3cc0ec52e Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Fri, 29 May 2026 14:20:54 -0300 Subject: [PATCH 24/24] fix(sequencer): set own proposed checkpoint locally instead of via p2p loopback (#23659) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation The proposer relied on looping its own checkpoint proposal back through the p2p receive path to advance its local proposed-checkpoint tip before propagating. Under `broadcastInvalidBlockProposal` the broadcast checkpoint archive is deliberately corrupted, so the loopback handler's archive-based block lookup (`getBlockData({ archive })`) found nothing and retried until the next slot. By the time the proposer returned from broadcast, propagation had slipped past the p2p validator's stale window — producing intermittent failures (e.g. peers rejecting a late slot proposal). ## Approach The proposer's optimistic proposed-checkpoint tip is the proposer's own local state, so it is now set directly in the sequencer's checkpoint proposal job rather than via a p2p loopback. The job adds the proposed checkpoint to the archiver from local checkpoint data (block numbers and counts, never the possibly-corrupted broadcast archive) immediately before gossiping, failing closed if the local insert fails. Because every block is already added to the archiver's FIFO queue (and awaited) during block building, the checkpoint insert needs no retry. The `notifyOwnCheckpointProposal` loopback is removed entirely, so the path is identical whether p2p is enabled or not. ## Changes - **stdlib**: New `ProposedCheckpointSink` interface alongside `L2BlockSink`. - **sequencer-client**: `CheckpointProposalJob` now pushes the proposed checkpoint to the archiver from local data before broadcast, gated on proposer pipelining and skipped when block-push is disabled (`skipPushProposedBlocksToArchiver`, fisherman mode); widened the sequencer/client `l2BlockSource` types to `ProposedCheckpointSink`. - **p2p**: Removed `notifyOwnCheckpointProposal` from the `P2PService` interface, the libp2p and dummy services, and the `P2PClient.broadcastCheckpointProposal` call site (own proposals are still stored in the attestation pool before propagation). - **validator-client**: The all-nodes own-proposal branch now skips validation and returns; removed the now-dead `setProposedCheckpointFromBlocks` and narrowed the archiver `Pick`. - **tests**: Added job tests (push-from-local-data and order-before-gossip, abort-on-push-failure, no-push-when-pipelining-disabled, fisherman) and a proposal_handler own-proposal test; removed the obsolete libp2p loopback test and the e2e slash-test stub; widened affected mock types. --- .../duplicate_attestation_slash.test.ts | 12 ---- yarn-project/p2p/src/client/p2p_client.ts | 2 - .../p2p/src/services/dummy_service.ts | 8 +-- .../services/libp2p/libp2p_service.test.ts | 10 ---- .../p2p/src/services/libp2p/libp2p_service.ts | 4 -- yarn-project/p2p/src/services/service.ts | 3 - .../src/client/sequencer-client.ts | 4 +- .../sequencer/checkpoint_proposal_job.test.ts | 55 +++++++++++++++++- .../checkpoint_proposal_job.timing.test.ts | 6 +- .../src/sequencer/checkpoint_proposal_job.ts | 49 +++++++++++++++- .../src/sequencer/sequencer.test.ts | 5 +- .../src/sequencer/sequencer.ts | 10 +++- .../stdlib/src/block/l2_block_source.ts | 15 ++++- .../src/proposal_handler.test.ts | 26 +++++++++ .../validator-client/src/proposal_handler.ts | 57 +++---------------- 15 files changed, 165 insertions(+), 101 deletions(-) diff --git a/yarn-project/end-to-end/src/e2e_p2p/duplicate_attestation_slash.test.ts b/yarn-project/end-to-end/src/e2e_p2p/duplicate_attestation_slash.test.ts index 9c27e88a5f4f..ce4a8f706999 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/duplicate_attestation_slash.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/duplicate_attestation_slash.test.ts @@ -213,18 +213,6 @@ describe('e2e_p2p_duplicate_attestation_slash', () => { nodes = [maliciousNode1, maliciousNode2, honestNode1, honestNode2]; - // Stub the proposer's own-checkpoint-proposal loopback on the malicious nodes. The default - // path awaits a local handleCheckpointProposal → validateCheckpointProposal that retries - // until the proposed block lands in the archiver — but skipPushProposedBlocksToArchiver - // means it never does, so the await hangs until the retry deadline (~one slot). By the - // time the proposer returns from broadcast, the wallclock is in the target slot and the - // staleness gate refuses the self-attestation, so no duplicate attestations are ever - // broadcast. - for (const node of [maliciousNode1, maliciousNode2]) { - const p2pService: any = (node as any).p2pClient.p2pService; - jest.spyOn(p2pService, 'notifyOwnCheckpointProposal').mockResolvedValue(undefined); - } - // Wait for P2P mesh on all needed topics before starting sequencers await t.waitForP2PMeshConnectivity(nodes, NUM_VALIDATORS, 30, 0.1, [ TopicType.tx, diff --git a/yarn-project/p2p/src/client/p2p_client.ts b/yarn-project/p2p/src/client/p2p_client.ts index 8a4ba30a701f..77fc6e8be11c 100644 --- a/yarn-project/p2p/src/client/p2p_client.ts +++ b/yarn-project/p2p/src/client/p2p_client.ts @@ -377,8 +377,6 @@ export class P2PClient extends WithTracer implements P2P { throw new Error(`Attempted to broadcast a duplicate checkpoint proposal for slot ${proposal.slotNumber}`); } } - // Gossipsub doesn't deliver own messages, so fire the all-nodes handler locally - await this.p2pService.notifyOwnCheckpointProposal(checkpointCore); return this.p2pService.propagate(proposal); } diff --git a/yarn-project/p2p/src/services/dummy_service.ts b/yarn-project/p2p/src/services/dummy_service.ts index 6078c3e484cd..7e132b28ef95 100644 --- a/yarn-project/p2p/src/services/dummy_service.ts +++ b/yarn-project/p2p/src/services/dummy_service.ts @@ -1,6 +1,6 @@ import type { EthAddress } from '@aztec/foundation/eth-address'; import type { PeerInfo } from '@aztec/stdlib/interfaces/server'; -import type { CheckpointProposalCore, Gossipable, PeerErrorSeverity, TopicType } from '@aztec/stdlib/p2p'; +import type { Gossipable, PeerErrorSeverity, TopicType } from '@aztec/stdlib/p2p'; import { Tx, TxHash } from '@aztec/stdlib/tx'; import type { PeerId } from '@libp2p/interface'; @@ -93,12 +93,6 @@ export class DummyP2PService implements P2PService { this.allNodesCheckpointReceivedCallback = callback; } - // Mirror libp2p's own-proposal loopback so the proposer's pipelined `canProposeAt` override sees its own - // in-flight parent checkpoint when running in p2p-disabled (single-node e2e) mode. - public async notifyOwnCheckpointProposal(checkpoint: CheckpointProposalCore): Promise { - await this.allNodesCheckpointReceivedCallback?.(checkpoint, undefined as unknown as PeerId); - } - /** * Register a callback for when a duplicate proposal is detected */ diff --git a/yarn-project/p2p/src/services/libp2p/libp2p_service.test.ts b/yarn-project/p2p/src/services/libp2p/libp2p_service.test.ts index 9ea75ef3fd60..fbb6dac2b763 100644 --- a/yarn-project/p2p/src/services/libp2p/libp2p_service.test.ts +++ b/yarn-project/p2p/src/services/libp2p/libp2p_service.test.ts @@ -1071,16 +1071,6 @@ describe('LibP2PService', () => { expect(reportMessageValidationResultSpy).toHaveBeenCalledWith('msg-1', MOCK_PEER_ID, TopicValidatorResult.Reject); }); - it('notifyOwnCheckpointProposal fires allNodesCheckpointReceivedCallback', async () => { - const checkpointHeader = makeCheckpointHeader(1, { slotNumber: targetSlot }); - const proposal = await makeCheckpointProposal({ signer, checkpointHeader }); - - await service.notifyOwnCheckpointProposal(proposal.toCore()); - - expect(allNodesCheckpointReceivedCallback).toHaveBeenCalledTimes(1); - expect(allNodesCheckpointReceivedCallback).toHaveBeenCalledWith(expect.any(Object), expect.anything()); - }); - // Regression for A-1013: payloads sharing (slot, archive) but differing on feeAssetPriceModifier // used to dedup by archive only and silently drop the second one. The pool now dedups by // signed-payload hash, so the equivocation surfaces. diff --git a/yarn-project/p2p/src/services/libp2p/libp2p_service.ts b/yarn-project/p2p/src/services/libp2p/libp2p_service.ts index a5c2728d4089..be2ca52ed8f9 100644 --- a/yarn-project/p2p/src/services/libp2p/libp2p_service.ts +++ b/yarn-project/p2p/src/services/libp2p/libp2p_service.ts @@ -722,10 +722,6 @@ export class LibP2PService extends WithTracer implements P2PService { this.allNodesCheckpointReceivedCallback = callback; } - public async notifyOwnCheckpointProposal(checkpoint: CheckpointProposalCore): Promise { - await this.allNodesCheckpointReceivedCallback(checkpoint, this.node.peerId); - } - /** * Registers a callback to be invoked when a duplicate proposal is detected. * This callback is triggered on the first duplicate (when count goes from 1 to 2). diff --git a/yarn-project/p2p/src/services/service.ts b/yarn-project/p2p/src/services/service.ts index f5428c9a9f19..a2f550da5259 100644 --- a/yarn-project/p2p/src/services/service.ts +++ b/yarn-project/p2p/src/services/service.ts @@ -102,9 +102,6 @@ export interface P2PService { registerAllNodesCheckpointReceivedCallback(callback: P2PCheckpointReceivedCallback): void; - /** Fires the all-nodes checkpoint callback for our own proposal (gossipsub doesn't deliver own messages). */ - notifyOwnCheckpointProposal(checkpoint: CheckpointProposalCore): Promise; - /** * Registers a callback invoked when a duplicate proposal is detected (equivocation). * The callback is triggered on the first duplicate (when count goes from 1 to 2). diff --git a/yarn-project/sequencer-client/src/client/sequencer-client.ts b/yarn-project/sequencer-client/src/client/sequencer-client.ts index 8198590eea3e..7b54296b91a6 100644 --- a/yarn-project/sequencer-client/src/client/sequencer-client.ts +++ b/yarn-project/sequencer-client/src/client/sequencer-client.ts @@ -12,7 +12,7 @@ import type { DateProvider } from '@aztec/foundation/timer'; import type { KeystoreManager } from '@aztec/node-keystore'; import type { P2P } from '@aztec/p2p'; import type { SlasherClientInterface } from '@aztec/slasher'; -import type { L2BlockSink, L2BlockSource } from '@aztec/stdlib/block'; +import type { L2BlockSink, L2BlockSource, ProposedCheckpointSink } from '@aztec/stdlib/block'; import type { ValidatorClientFullConfig, WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server'; import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging'; import { L1Metrics, type TelemetryClient } from '@aztec/telemetry-client'; @@ -56,7 +56,7 @@ export class SequencerClient { worldStateSynchronizer: WorldStateSynchronizer; slasherClient: SlasherClientInterface | undefined; checkpointsBuilder: FullNodeCheckpointsBuilder; - l2BlockSource: L2BlockSource & L2BlockSink; + l2BlockSource: L2BlockSource & L2BlockSink & ProposedCheckpointSink; l1ToL2MessageSource: L1ToL2MessageSource; telemetry: TelemetryClient; publisherFactory?: SequencerPublisherFactory; diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts index 8661d7c0850b..20ddb7d695a8 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.test.ts @@ -24,6 +24,7 @@ import { L2Block, type L2BlockSink, type L2BlockSource, + type ProposedCheckpointSink, type ValidateCheckpointResult, } from '@aztec/stdlib/block'; import { @@ -87,7 +88,7 @@ describe('CheckpointProposalJob', () => { let checkpointBuilder: MockCheckpointBuilder; let l1ToL2MessageSource: MockProxy; let l2BlockSource: MockProxy; - let blockSink: MockProxy; + let blockSink: MockProxy; let slasherClient: MockProxy; let dateProvider: TestDateProvider; let metrics: MockProxy; @@ -244,8 +245,9 @@ describe('CheckpointProposalJob', () => { l2BlockSource = mock(); l2BlockSource.getCheckpointsData.mockResolvedValue([]); - blockSink = mock(); + blockSink = mock(); blockSink.addBlock.mockResolvedValue(undefined); + blockSink.addProposedCheckpoint.mockResolvedValue(undefined); validatorClient = mock(); validatorClient.collectAttestations.mockImplementation(() => Promise.resolve([])); @@ -1134,6 +1136,40 @@ describe('CheckpointProposalJob', () => { expect(mismatchEvents).toHaveLength(0); }); + it('pushes the proposed checkpoint to the archiver from local data before broadcasting', async () => { + const pipelinedJob = await createPipelinedJobWithBlock(proposedParent); + mockL2BlockSource({ checkpointedNumber: CheckpointNumber(1), checkpointedHash: parentCheckpointHash }); + + await pipelinedJob.executeAndAwait(); + + // Built from local checkpoint data: startBlock = syncedToBlockNumber + 1, blockCount = blocks built, + // checkpointNumber from the job — never derived from the (possibly corrupted) broadcast proposal archive. + expect(blockSink.addProposedCheckpoint).toHaveBeenCalledTimes(1); + expect(blockSink.addProposedCheckpoint).toHaveBeenCalledWith( + expect.objectContaining({ + checkpointNumber: CheckpointNumber(2), + startBlock: BlockNumber(lastBlockNumber + 1), + blockCount: 1, + }), + ); + // The proposed checkpoint must be pushed locally before the proposal is gossiped. + expect(blockSink.addProposedCheckpoint.mock.invocationCallOrder[0]).toBeLessThan( + p2p.broadcastCheckpointProposal.mock.invocationCallOrder[0], + ); + }); + + it('aborts the checkpoint without broadcasting when the proposed checkpoint push fails', async () => { + blockSink.addProposedCheckpoint.mockRejectedValue(new Error('proposed checkpoint slot expired')); + const pipelinedJob = await createPipelinedJobWithBlock(proposedParent); + mockL2BlockSource({ checkpointedNumber: CheckpointNumber(1), checkpointedHash: parentCheckpointHash }); + + const checkpoint = await pipelinedJob.execute(); + + expect(checkpoint).toBeUndefined(); + expect(blockSink.addProposedCheckpoint).toHaveBeenCalledTimes(1); + expect(p2p.broadcastCheckpointProposal).not.toHaveBeenCalled(); + }); + it('skips proposal with archiver-sync-timeout when archiver does not sync in time', async () => { const pipelinedJob = await createPipelinedJobWithBlock(proposedParent); l2BlockSource.getSyncedL2SlotNumber.mockResolvedValue(SlotNumber(0)); @@ -1705,6 +1741,21 @@ describe('CheckpointProposalJob', () => { expect(checkpointBuilder.buildBlockCalls).toHaveLength(1); // But must NOT push to the archiver — that was the bug causing reorgs on mainnet expect(blockSink.addBlock).not.toHaveBeenCalled(); + expect(blockSink.addProposedCheckpoint).not.toHaveBeenCalled(); + }); + + it('does not push the proposed checkpoint when pipelining is disabled', async () => { + // The proposed-checkpoint tip is a pipelining-only concept, so a non-pipelining proposer + // must still broadcast but must not advance it. + epochCache.isProposerPipeliningEnabled.mockReturnValue(false); + const { txs, block } = await setupTxsAndBlock(p2p, globalVariables, 1, chainId); + checkpointBuilder.seedBlocks([block], [txs]); + validatorClient.collectAttestations.mockResolvedValue(getAttestations(block)); + + await job.executeAndAwait(); + + expect(blockSink.addProposedCheckpoint).not.toHaveBeenCalled(); + expect(p2p.broadcastCheckpointProposal).toHaveBeenCalledTimes(1); }); it('handles empty committee gracefully', async () => { diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.timing.test.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.timing.test.ts index 23ed57fb1e9a..9d0f7eb8446e 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.timing.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.timing.test.ts @@ -10,7 +10,7 @@ import type { TypedEventEmitter } from '@aztec/foundation/types'; import { type P2P, P2PClientState } from '@aztec/p2p'; import type { SlasherClientInterface } from '@aztec/slasher'; import { AztecAddress } from '@aztec/stdlib/aztec-address'; -import type { L2Block, L2BlockSink, L2BlockSource } from '@aztec/stdlib/block'; +import type { L2Block, L2BlockSink, L2BlockSource, ProposedCheckpointSink } from '@aztec/stdlib/block'; import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers'; import { GasFees } from '@aztec/stdlib/gas'; import type { @@ -206,7 +206,7 @@ describe('CheckpointProposalJob Timing Tests', () => { let worldState: MockProxy; let l1ToL2MessageSource: MockProxy; let l2BlockSource: MockProxy; - let blockSink: MockProxy; + let blockSink: MockProxy; let slasherClient: MockProxy; let metrics: MockProxy; let checkpointMetrics: MockProxy; @@ -438,7 +438,7 @@ describe('CheckpointProposalJob Timing Tests', () => { l2BlockSource = mock(); l2BlockSource.getCheckpointsData.mockResolvedValue([]); - blockSink = mock(); + blockSink = mock(); blockSink.addBlock.mockResolvedValue(undefined); validatorClient = mock(); diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts index d054f9882635..8e1a84e8af32 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts @@ -31,6 +31,7 @@ import { type L2BlockSink, type L2BlockSource, MaliciousCommitteeAttestationsAndSigners, + type ProposedCheckpointSink, type ValidateCheckpointResult, } from '@aztec/stdlib/block'; import { @@ -137,7 +138,7 @@ export class CheckpointProposalJob implements Traceable { private readonly l1ToL2MessageSource: L1ToL2MessageSource, private readonly l2BlockSource: L2BlockSource, private readonly checkpointsBuilder: FullNodeCheckpointsBuilder, - private readonly blockSink: L2BlockSink, + private readonly blockSink: L2BlockSink & ProposedCheckpointSink, private readonly l1Constants: SequencerRollupConstants, private readonly signatureContext: CoordinationSignatureContext, protected config: ResolvedSequencerConfig, @@ -774,6 +775,13 @@ export class CheckpointProposalJob implements Traceable { checkpointProposalOptions, ); + // Advance our own optimistic proposed-checkpoint tip locally before gossiping. Gossipsub + // doesn't echo our own messages back, so this is how the proposer makes its own proposed + // checkpoint visible for pipelining the next slot. Built from local checkpoint data — never + // from the broadcast proposal archive, which may be deliberately corrupted under test flags. + // Fail closed: if this throws, the outer catch aborts the slot before gossiping. + await this.syncProposedCheckpointToArchiver(checkpoint, blocksInCheckpoint.length, feeAssetPriceModifier); + const blockProposedAt = this.dateProvider.now(); if (this.config.skipBroadcastCheckpointProposal) { // Test-only: suppress the CheckpointProposal so peers never see a proposed checkpoint for @@ -1489,6 +1497,45 @@ export class CheckpointProposalJob implements Traceable { await this.blockSink.addBlock(block); } + /** + * Adds the proposed checkpoint to the archiver so the proposer's optimistic proposed-checkpoint + * tip advances locally. Gossip doesn't echo our own messages back, so without this the proposer + * would never see its own proposed checkpoint and couldn't pipeline the next slot. + * + * Only runs under proposer pipelining (the proposed tip is a pipelining-only concept) and is + * skipped whenever proposed blocks aren't pushed (`skipPushProposedBlocksToArchiver`, fisherman + * mode): the archiver derives the checkpoint archive from its stored blocks, so without them the + * push would fail. All blocks were already added (and awaited) during block building, so this + * needs no retry — they are guaranteed present by the time we get here. + */ + private async syncProposedCheckpointToArchiver( + checkpoint: Checkpoint, + blockCount: number, + feeAssetPriceModifier: bigint, + ): Promise { + if (this.config.skipPushProposedBlocksToArchiver || this.config.fishermanMode) { + return; + } + if (!this.epochCache.isProposerPipeliningEnabled()) { + return; + } + const startBlock = BlockNumber(this.syncedToBlockNumber + 1); + this.log.debug(`Syncing proposed checkpoint ${this.checkpointNumber} to archiver`, { + checkpointNumber: this.checkpointNumber, + slot: this.targetSlot, + startBlock, + blockCount, + }); + await this.blockSink.addProposedCheckpoint({ + header: checkpoint.header, + checkpointNumber: this.checkpointNumber, + startBlock, + blockCount, + totalManaUsed: checkpoint.header.totalManaUsed.toBigInt(), + feeAssetPriceModifier, + }); + } + /** Runs fee analysis and logs checkpoint outcome as fisherman */ private async handleCheckpointEndAsFisherman(checkpoint: Checkpoint | undefined) { // Perform L1 fee analysis before clearing requests diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts index c5f84642cd44..8921b309bf12 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.test.ts @@ -26,6 +26,7 @@ import { L2Block, type L2BlockSink, type L2BlockSource, + type ProposedCheckpointSink, type ValidateCheckpointNegativeResult, } from '@aztec/stdlib/block'; import { Checkpoint, type ProposedCheckpointData } from '@aztec/stdlib/checkpoint'; @@ -65,7 +66,7 @@ describe('sequencer', () => { let worldState: MockProxy; let checkpointsBuilder: MockCheckpointsBuilder; let checkpointBuilder: MockCheckpointBuilder; - let l2BlockSource: MockProxy; + let l2BlockSource: MockProxy; let l1ToL2MessageSource: MockProxy; let slasherClient: MockProxy; let publisherFactory: MockProxy; @@ -292,7 +293,7 @@ describe('sequencer', () => { // Use blockProvider so the mock returns whatever `block` is set to at call time checkpointBuilder.setBlockProvider(() => block); - l2BlockSource = mock({ + l2BlockSource = mock({ getBlockData: mockFn().mockResolvedValue({ header: BlockHeader.empty(), archive: AppendOnlyTreeSnapshot.empty(), diff --git a/yarn-project/sequencer-client/src/sequencer/sequencer.ts b/yarn-project/sequencer-client/src/sequencer/sequencer.ts index 7e4d44da78fb..1150bf44c2d8 100644 --- a/yarn-project/sequencer-client/src/sequencer/sequencer.ts +++ b/yarn-project/sequencer-client/src/sequencer/sequencer.ts @@ -11,7 +11,13 @@ import type { DateProvider } from '@aztec/foundation/timer'; import type { TypedEventEmitter } from '@aztec/foundation/types'; import type { P2P } from '@aztec/p2p'; import type { SlasherClientInterface } from '@aztec/slasher'; -import type { BlockData, L2BlockSink, L2BlockSource, ValidateCheckpointResult } from '@aztec/stdlib/block'; +import type { + BlockData, + L2BlockSink, + L2BlockSource, + ProposedCheckpointSink, + ValidateCheckpointResult, +} from '@aztec/stdlib/block'; import type { Checkpoint, ProposedCheckpointData } from '@aztec/stdlib/checkpoint'; import type { ChainConfig } from '@aztec/stdlib/config'; import { getSlotStartBuildTimestamp } from '@aztec/stdlib/epoch-helpers'; @@ -95,7 +101,7 @@ export class Sequencer extends (EventEmitter as new () => TypedEventEmitter; } +/** + * Interface for classes that can receive and store proposed (not-yet-L1-confirmed) checkpoints. + */ +export interface ProposedCheckpointSink { + /** + * Adds a proposed checkpoint to the store. The archive and checkpointOutHash are computed + * internally from the already-stored blocks, so every block in the checkpoint must be added + * (via {@link L2BlockSink.addBlock}) before calling this. + * @param checkpoint - The proposed checkpoint metadata. + */ + addProposedCheckpoint(checkpoint: ProposedCheckpointInput): Promise; +} + /** * L2BlockSource that emits events upon pending / proven chain changes. * see L2BlockSourceEvents for the events emitted. diff --git a/yarn-project/validator-client/src/proposal_handler.test.ts b/yarn-project/validator-client/src/proposal_handler.test.ts index e6634968ee6d..9c5426cfc1cf 100644 --- a/yarn-project/validator-client/src/proposal_handler.test.ts +++ b/yarn-project/validator-client/src/proposal_handler.test.ts @@ -3,6 +3,7 @@ import type { BlobClientInterface } from '@aztec/blob-client/client'; import type { EpochCache } from '@aztec/epoch-cache'; import { MAX_FEE_ASSET_PRICE_MODIFIER_BPS } from '@aztec/ethereum/contracts'; import { CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types'; +import { Secp256k1Signer } from '@aztec/foundation/crypto/secp256k1-signer'; import { Fr } from '@aztec/foundation/curves/bn254'; import { TestDateProvider } from '@aztec/foundation/timer'; import { type FieldsOf, unfreeze } from '@aztec/foundation/types'; @@ -276,6 +277,31 @@ describe('ProposalHandler checkpoint validation', () => { }); }); + describe('own checkpoint proposal handling', () => { + it('skips validation and does not touch the archiver for own proposals', async () => { + // The proposer's own proposed checkpoint is now set locally by the sequencer job, not via the + // p2p loopback, so the all-nodes handler must not re-validate or re-add it. + const signer = Secp256k1Signer.random(); + const proposal = await makeProposal({ signer }); + + const p2p = mock(); + let checkpointHandler: ((proposal: any, sender: any) => Promise) | undefined; + p2p.registerAllNodesCheckpointProposalHandler.mockImplementation(handler => { + checkpointHandler = handler; + }); + + const archiver = mock>(); + const handleSpy = jest.spyOn(handler, 'handleCheckpointProposal'); + + handler.register(p2p, true, archiver, () => [signer.address.toString()]); + await checkpointHandler!(proposal, {} as any); + + expect(handleSpy).not.toHaveBeenCalled(); + expect(archiver.addProposedCheckpoint).not.toHaveBeenCalled(); + expect(blockSource.getBlockData).not.toHaveBeenCalled(); + }); + }); + describe('deep validation (openCheckpoint + completeCheckpoint)', () => { const archiveRoot = Fr.random(); const checkpointOutHash = Fr.random(); diff --git a/yarn-project/validator-client/src/proposal_handler.ts b/yarn-project/validator-client/src/proposal_handler.ts index 15a236a6df43..51035ec1c297 100644 --- a/yarn-project/validator-client/src/proposal_handler.ts +++ b/yarn-project/validator-client/src/proposal_handler.ts @@ -165,7 +165,7 @@ export class ProposalHandler { }; /** Archiver reference for setting proposed checkpoints (pipelining). Set via register(). */ - private archiver?: Pick; + private archiver?: Pick; /** Returns current validator addresses for own-proposal detection. Set via register(). */ private getOwnValidatorAddresses?: () => string[]; @@ -231,7 +231,7 @@ export class ProposalHandler { register( p2pClient: P2P, shouldReexecute: boolean, - archiver?: Pick, + archiver?: Pick, getOwnValidatorAddresses?: () => string[], ): ProposalHandler { this.p2pClient = p2pClient; @@ -297,16 +297,17 @@ export class ProposalHandler { return undefined; } - // For own proposals, skip validation — the proposer already built and validated the checkpoint + // For own proposals, skip validation and return: the proposer already built and validated the + // checkpoint, and the sequencer's checkpoint proposal job pushed the proposed checkpoint to the + // archiver from local data before broadcasting. Gossipsub doesn't echo our own messages back, so + // this branch is normally unreachable — it remains as defense if an own proposal arrives by some + // other path. const proposer = proposal.getSender(); const ownAddresses = this.getOwnValidatorAddresses?.(); const isOwnProposal = proposer && ownAddresses?.some(addr => addr === proposer.toString()); if (isOwnProposal) { this.log.debug(`Skipping validation for own checkpoint proposal at slot ${proposal.slotNumber}`); - if (this.archiver && this.epochCache.isProposerPipeliningEnabled()) { - await this.setProposedCheckpointFromBlocks(proposal); - } return undefined; } @@ -1211,48 +1212,4 @@ export class ProposalHandler { }); return true; } - - /** - * Sets proposed checkpoint from blocks for own proposals (skips full validation). - * Retries fetching block data since the checkpoint proposal often arrives before the last block - * finishes re-execution. - */ - private async setProposedCheckpointFromBlocks(proposal: CheckpointProposalCore): Promise { - if (!this.archiver) { - return false; - } - let blockData = await this.blockSource.getBlockData({ archive: proposal.archive }); - - if (!blockData) { - // The checkpoint proposal often arrives before the last block finishes re-execution. - // Retry until we find the data or give up at the end of the slot. - const nextSlot = this.epochCache.getSlotNow() + 1; - const timeOfNextSlot = getTimestampForSlot(SlotNumber(nextSlot), await this.archiver.getL1Constants()); - const timeoutSeconds = Math.max(1, Number(timeOfNextSlot) - Math.floor(this.dateProvider.now() / 1000)); - - blockData = await retryUntil( - () => this.blockSource.getBlockData({ archive: proposal.archive }), - 'block data for own checkpoint proposal', - timeoutSeconds, - 0.25, - ).catch(() => undefined); - } - - if (blockData) { - await this.archiver.addProposedCheckpoint({ - header: proposal.checkpointHeader, - checkpointNumber: blockData.checkpointNumber, - startBlock: BlockNumber(blockData.header.getBlockNumber() - blockData.indexWithinCheckpoint), - blockCount: blockData.indexWithinCheckpoint + 1, - totalManaUsed: proposal.checkpointHeader.totalManaUsed.toBigInt(), - feeAssetPriceModifier: proposal.feeAssetPriceModifier, - }); - return true; - } else { - this.log.debug(`Block data not found for own checkpoint proposal archive, cannot set proposed checkpoint`, { - archive: proposal.archive.toString(), - }); - return false; - } - } }