diff --git a/.github/workflows/deploy-staging-public.yml b/.github/workflows/deploy-staging-public.yml index 5c521e6c0e7e..e26a2538b650 100644 --- a/.github/workflows/deploy-staging-public.yml +++ b/.github/workflows/deploy-staging-public.yml @@ -26,28 +26,27 @@ jobs: token: ${{ secrets.AZTEC_BOT_GITHUB_TOKEN }} fetch-depth: 0 - - name: Read version from manifest - id: manifest - run: | - VERSION=$(jq -r '."."' .release-please-manifest.json) - echo "version=$VERSION" - echo "version=$VERSION" >> $GITHUB_OUTPUT - - - name: Poll for tag at HEAD + - name: Poll for semver tag at HEAD id: poll-tag run: | - # wait for tag to be pushed (either RC or stable release) - VERSION="${{ steps.manifest.outputs.version }}" HEAD_SHA=$(git rev-parse HEAD) MAX_ATTEMPTS=60 - echo "Looking for tag matching v${VERSION} or v${VERSION}-rc.* at HEAD ($HEAD_SHA)" + echo "Looking for any semver tag at HEAD ($HEAD_SHA)" for i in $(seq 1 $MAX_ATTEMPTS); do git fetch --tags --force - TAG=$(git tag --points-at HEAD | grep -E "^v${VERSION}(-rc\.[0-9]+)?$" | sort -V | tail -n 1 || true) + # Collect all valid semver tags pointing at HEAD + SEMVER_TAGS=() + for t in $(git tag --points-at HEAD); do + if ci3/semver check "$t"; then + SEMVER_TAGS+=("$t") + fi + done - if [ -n "$TAG" ]; then + # If we found valid semver tags, pick the highest + if [ ${#SEMVER_TAGS[@]} -gt 0 ]; then + TAG=$(ci3/semver sort "${SEMVER_TAGS[@]}" | tail -n 1) echo "Found tag: $TAG" SEMVER="${TAG#v}" echo "tag=$TAG" >> $GITHUB_OUTPUT @@ -55,11 +54,11 @@ jobs: exit 0 fi - echo "Attempt $i/$MAX_ATTEMPTS: No matching tag yet, waiting 10s..." + echo "Attempt $i/$MAX_ATTEMPTS: No semver tag yet, waiting 10s..." sleep 10 done - echo "Error: No tag found for v${VERSION} at HEAD after 10 minutes" + echo "Error: No semver tag found at HEAD after 10 minutes" exit 1 wait-for-ci3: diff --git a/bootstrap.sh b/bootstrap.sh index 0350add48a52..c631883130e2 100755 --- a/bootstrap.sh +++ b/bootstrap.sh @@ -446,36 +446,22 @@ function bench { cache_upload bench-$(git rev-parse HEAD^{tree}).tar.gz bench-out/bench.json } -function release_github { - # Add an easy link for comparing to previous release. - local compare_link="" - if gh release view "v$CURRENT_VERSION" &>/dev/null; then - compare_link=$(echo -e "See changes: https://github.com/AztecProtocol/aztec-packages/compare/v${CURRENT_VERSION}...${COMMIT_HASH}") - fi - # Legacy releases. TODO: Eventually remove. - if gh release view "aztec-packages-v$CURRENT_VERSION" &>/dev/null; then - compare_link=$(echo -e "See changes: https://github.com/AztecProtocol/aztec-packages/compare/aztec-packages-v${CURRENT_VERSION}...${COMMIT_HASH}") +function release_bb_github { + # Create a GitHub release in AztecProtocol/barretenberg for bb artifacts. + # Users can manually create releases in aztec-packages via the GitHub UI if needed. + local bb_repo="AztecProtocol/barretenberg" + if gh release view "$REF_NAME" --repo "$bb_repo" &>/dev/null; then + return fi - # Determine if this is a prerelease (has a prerelease tag like -rc.1, -alpha, etc.) - local is_prerelease=false + local prerelease_flag="" if [ -n "$(semver prerelease $REF_NAME)" ]; then - is_prerelease=true - fi - # Ensure we have a commit release. - if ! gh release view "$REF_NAME" &>/dev/null; then - local prerelease_flag="" - if $is_prerelease; then - prerelease_flag="--prerelease" - fi - do_or_dryrun gh release create "$REF_NAME" \ - $prerelease_flag \ - --target $COMMIT_HASH \ - --title "$REF_NAME" \ - --notes "$compare_link" - elif ! $is_prerelease; then - # Release exists but this is not a prerelease version - ensure it's marked as a full release - do_or_dryrun gh release edit "$REF_NAME" --prerelease=false + prerelease_flag="--prerelease" fi + do_or_dryrun gh release create "$REF_NAME" \ + --repo "$bb_repo" \ + $prerelease_flag \ + --title "$REF_NAME" \ + --notes "Release $REF_NAME — see https://github.com/AztecProtocol/aztec-packages/commits/$COMMIT_HASH" } function release { @@ -495,9 +481,9 @@ function release { echo_header "release all" set -x - # Ensure we have a github release for our REF_NAME. - # This is in case were are not going through release-please. - release_github + # Ensure we have a github release in AztecProtocol/barretenberg for bb artifacts. + # Users can create aztec-packages releases manually via the GitHub "Create a release" button. + release_bb_github projects=( barretenberg/cpp diff --git a/yarn-project/archiver/src/archiver-sync.test.ts b/yarn-project/archiver/src/archiver-sync.test.ts index 3b82c701e67c..13bd543c247b 100644 --- a/yarn-project/archiver/src/archiver-sync.test.ts +++ b/yarn-project/archiver/src/archiver-sync.test.ts @@ -1223,6 +1223,56 @@ describe('Archiver Sync', () => { expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(2)); }, 15_000); + + it('handles L1 reorg that moves a checkpoint to a later L1 block', async () => { + expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(0)); + + // Sync checkpoints 1 and 2 + await fake.addCheckpoint(CheckpointNumber(1), { + l1BlockNumber: 70n, + messagesL1BlockNumber: 50n, + numL1ToL2Messages: 3, + }); + const { checkpoint: cp2 } = await fake.addCheckpoint(CheckpointNumber(2), { + l1BlockNumber: 80n, + messagesL1BlockNumber: 60n, + numL1ToL2Messages: 3, + }); + + fake.setL1BlockNumber(90n); + await archiver.syncImmediate(); + expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(2)); + + // Verify checkpoint 2's blocks are stored + const lastBlockNumber = cp2.blocks.at(-1)!.number; + const tips = await archiver.getL2Tips(); + expect(tips.checkpointed.checkpoint.number).toEqual(CheckpointNumber(2)); + expect(tips.checkpointed.block.number).toEqual(lastBlockNumber); + + // Simulate L1 reorg: checkpoint 2 moves from L1 block 80 to L1 block 85. + // The checkpoint content (blocks, archive) stays the same — only the L1 block changes. + // This causes the archiver to re-discover checkpoint 2 when scanning from block 81 onward. + fake.moveCheckpointToL1Block(CheckpointNumber(2), 85n); + + // Advance L1 and sync. The archiver's sync point is at L1 block 80 (from checkpoint 2's + // original insertion). The scan starts from 81, finds checkpoint 2 at block 85, and must + // accept it as a duplicate with updated L1 info rather than throwing. + fake.setL1BlockNumber(95n); + await archiver.syncImmediate(); + + // The archiver should still be at checkpoint 2 and healthy + expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(2)); + + // Add checkpoint 3 to verify the archiver can continue syncing after the duplicate + await fake.addCheckpoint(CheckpointNumber(3), { + l1BlockNumber: 100n, + messagesL1BlockNumber: 90n, + numL1ToL2Messages: 3, + }); + fake.setL1BlockNumber(110n); + await archiver.syncImmediate(); + expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(3)); + }, 15_000); }); describe('finalized checkpoint', () => { diff --git a/yarn-project/archiver/src/store/block_store.ts b/yarn-project/archiver/src/store/block_store.ts index fc534436c3f5..fa611f0d8470 100644 --- a/yarn-project/archiver/src/store/block_store.ts +++ b/yarn-project/archiver/src/store/block_store.ts @@ -227,21 +227,34 @@ export class BlockStore { } return await this.db.transactionAsync(async () => { - // Check that the checkpoint immediately before the first block to be added is present in the store. const firstCheckpointNumber = checkpoints[0].checkpoint.number; const previousCheckpointNumber = await this.getLatestCheckpointNumber(); - if (previousCheckpointNumber !== firstCheckpointNumber - 1 && !opts.force) { + // Handle already-stored checkpoints at the start of the batch. + // This can happen after an L1 reorg re-includes a checkpoint in a different L1 block. + // We accept them if archives match (same content) and update their L1 metadata. + if (!opts.force && firstCheckpointNumber <= previousCheckpointNumber) { + checkpoints = await this.skipOrUpdateAlreadyStoredCheckpoints(checkpoints, previousCheckpointNumber); + if (checkpoints.length === 0) { + return true; + } + // Re-check sequentiality after skipping + const newFirstNumber = checkpoints[0].checkpoint.number; + if (previousCheckpointNumber !== newFirstNumber - 1) { + throw new InitialCheckpointNumberNotSequentialError(newFirstNumber, previousCheckpointNumber); + } + } else if (previousCheckpointNumber !== firstCheckpointNumber - 1 && !opts.force) { throw new InitialCheckpointNumberNotSequentialError(firstCheckpointNumber, previousCheckpointNumber); } // Extract the previous checkpoint if there is one + const currentFirstCheckpointNumber = checkpoints[0].checkpoint.number; let previousCheckpointData: CheckpointData | undefined = undefined; - if (previousCheckpointNumber !== INITIAL_CHECKPOINT_NUMBER - 1) { + if (currentFirstCheckpointNumber - 1 !== INITIAL_CHECKPOINT_NUMBER - 1) { // There should be a previous checkpoint - previousCheckpointData = await this.getCheckpointData(previousCheckpointNumber); + previousCheckpointData = await this.getCheckpointData(CheckpointNumber(currentFirstCheckpointNumber - 1)); if (previousCheckpointData === undefined) { - throw new CheckpointNotFoundError(previousCheckpointNumber); + throw new CheckpointNotFoundError(CheckpointNumber(currentFirstCheckpointNumber - 1)); } } @@ -331,6 +344,50 @@ export class BlockStore { }); } + /** + * Handles checkpoints at the start of a batch that are already stored (e.g. due to L1 reorg). + * Verifies the archive root matches, updates L1 metadata, and returns only the new checkpoints. + */ + private async skipOrUpdateAlreadyStoredCheckpoints( + checkpoints: PublishedCheckpoint[], + latestStored: CheckpointNumber, + ): Promise { + let i = 0; + for (; i < checkpoints.length && checkpoints[i].checkpoint.number <= latestStored; i++) { + const incoming = checkpoints[i]; + const stored = await this.getCheckpointData(incoming.checkpoint.number); + if (!stored) { + // Should not happen if latestStored is correct, but be safe + break; + } + // Verify the checkpoint content matches (archive root) + if (!stored.archive.root.equals(incoming.checkpoint.archive.root)) { + throw new Error( + `Checkpoint ${incoming.checkpoint.number} already exists in store but with a different archive root. ` + + `Stored: ${stored.archive.root}, incoming: ${incoming.checkpoint.archive.root}`, + ); + } + // Update L1 metadata and attestations for the already-stored checkpoint + this.#log.warn( + `Checkpoint ${incoming.checkpoint.number} already stored, updating L1 info ` + + `(L1 block ${stored.l1.blockNumber} -> ${incoming.l1.blockNumber})`, + ); + await this.#checkpoints.set(incoming.checkpoint.number, { + header: incoming.checkpoint.header.toBuffer(), + archive: incoming.checkpoint.archive.toBuffer(), + checkpointOutHash: incoming.checkpoint.getCheckpointOutHash().toBuffer(), + l1: incoming.l1.toBuffer(), + attestations: incoming.attestations.map(a => a.toBuffer()), + checkpointNumber: incoming.checkpoint.number, + startBlock: incoming.checkpoint.blocks[0].number, + blockCount: incoming.checkpoint.blocks.length, + }); + // Update the sync point to reflect the new L1 block + await this.#lastSynchedL1Block.set(incoming.l1.blockNumber); + } + return checkpoints.slice(i); + } + private async addBlockToDatabase(block: L2Block, checkpointNumber: number, indexWithinCheckpoint: number) { const blockHash = await block.hash(); diff --git a/yarn-project/archiver/src/store/kv_archiver_store.test.ts b/yarn-project/archiver/src/store/kv_archiver_store.test.ts index 7dbe66146951..0ba931ca1f27 100644 --- a/yarn-project/archiver/src/store/kv_archiver_store.test.ts +++ b/yarn-project/archiver/src/store/kv_archiver_store.test.ts @@ -58,6 +58,7 @@ import { makeInboxMessage, makeInboxMessages, makeInboxMessagesWithFullBlocks, + makeL1PublishedData, makePrivateLog, makePrivateLogTag, makePublicLog, @@ -134,10 +135,56 @@ describe('KVArchiverDataStore', () => { await expect(store.addCheckpoints(publishedCheckpoints)).resolves.toBe(true); }); - it('throws on duplicate checkpoints', async () => { - await store.addCheckpoints(publishedCheckpoints); - await expect(store.addCheckpoints(publishedCheckpoints)).rejects.toThrow( - InitialCheckpointNumberNotSequentialError, + it('accepts duplicate checkpoints with matching archives and updates L1 info', async () => { + // Add first 3 checkpoints + const first3 = publishedCheckpoints.slice(0, 3); + await store.addCheckpoints(first3); + + // Verify initial L1 block number for checkpoint 3 + const beforeData = await store.getCheckpointData(CheckpointNumber(3)); + expect(beforeData).toBeDefined(); + const originalL1Block = beforeData!.l1.blockNumber; + + // Re-add checkpoint 3 with the same content but different L1 published data + // This simulates an L1 reorg that moved the checkpoint to a different L1 block + const cp3WithNewL1 = new PublishedCheckpoint( + first3[2].checkpoint, + makeL1PublishedData(999), + first3[2].attestations, + ); + // Also add checkpoint 4 (the next one) in the same batch + await store.addCheckpoints([cp3WithNewL1, publishedCheckpoints[3]]); + + // Checkpoint 3's L1 info should be updated + const afterData = await store.getCheckpointData(CheckpointNumber(3)); + expect(afterData).toBeDefined(); + expect(afterData!.l1.blockNumber).toEqual(999n); + expect(afterData!.l1.blockNumber).not.toEqual(originalL1Block); + + // Checkpoint 4 should be stored + expect(await store.getSynchedCheckpointNumber()).toEqual(CheckpointNumber(4)); + }); + + it('accepts a batch that is entirely already-stored checkpoints', async () => { + const first3 = publishedCheckpoints.slice(0, 3); + await store.addCheckpoints(first3); + + // Re-add the same 3 checkpoints — should succeed without error + await expect(store.addCheckpoints(first3)).resolves.toBe(true); + }); + + it('throws on duplicate checkpoints with mismatching archives', async () => { + const first3 = publishedCheckpoints.slice(0, 3); + await store.addCheckpoints(first3); + + // Create a fake checkpoint 3 with a different archive root (content mismatch) + const differentCheckpoint3 = await Checkpoint.random(CheckpointNumber(3), { + numBlocks: 1, + startBlockNumber: 3, + }); + const mismatchedCp3 = makePublishedCheckpoint(differentCheckpoint3, 999); + await expect(store.addCheckpoints([mismatchedCp3])).rejects.toThrow( + 'already exists in store but with a different archive', ); }); @@ -274,7 +321,7 @@ describe('KVArchiverDataStore', () => { await expect(store.addCheckpoints([publishedCheckpoint])).resolves.toBe(true); }); - it('throws on duplicate initial checkpoint', async () => { + it('throws on duplicate checkpoint with different content', async () => { const block1 = await L2Block.random(BlockNumber(1), { checkpointNumber: CheckpointNumber(1), indexWithinCheckpoint: IndexWithinCheckpoint(0), @@ -303,7 +350,7 @@ describe('KVArchiverDataStore', () => { await expect(store.addCheckpoints([publishedCheckpoint])).resolves.toBe(true); await expect(store.addCheckpoints([publishedCheckpoint2])).rejects.toThrow( - InitialCheckpointNumberNotSequentialError, + 'already exists in store but with a different archive', ); }); }); diff --git a/yarn-project/archiver/src/test/fake_l1_state.ts b/yarn-project/archiver/src/test/fake_l1_state.ts index 71e9f84c65f7..55f0f118ee88 100644 --- a/yarn-project/archiver/src/test/fake_l1_state.ts +++ b/yarn-project/archiver/src/test/fake_l1_state.ts @@ -331,6 +331,21 @@ export class FakeL1State { this.updatePendingCheckpointNumber(); } + /** + * Moves a checkpoint to a different L1 block number (simulates L1 reorg that + * re-includes the same checkpoint transaction in a different block). + * The checkpoint content stays the same — only the L1 metadata changes. + * Auto-updates pending status. + */ + moveCheckpointToL1Block(checkpointNumber: CheckpointNumber, newL1BlockNumber: bigint): void { + for (const cpData of this.checkpoints) { + if (cpData.checkpointNumber === checkpointNumber) { + cpData.l1BlockNumber = newL1BlockNumber; + } + } + this.updatePendingCheckpointNumber(); + } + /** * Removes messages after a given total index (simulates L1 reorg). * Auto-updates rolling hash. diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 6dafbb6f2fb1..f76e7da19a18 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -108,7 +108,7 @@ import { FullNodeCheckpointsBuilder, NodeKeystoreAdapter, ValidatorClient, - createBlockProposalHandler, + createProposalHandler, createValidatorClient, } from '@aztec/validator-client'; import type { SlashingProtectionDatabase } from '@aztec/validator-ha-signer/types'; @@ -393,19 +393,21 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, Traceable { } } - // If there's no validator client, create a BlockProposalHandler to handle block proposals + // If there's no validator client, create a ProposalHandler to handle block and checkpoint proposals // for monitoring or reexecution. Reexecution (default) allows us to follow the pending chain, // while non-reexecution is used for validating the proposals and collecting their txs. + // Checkpoint proposals are handled if the blob client can upload blobs. if (!validatorClient) { const reexecute = !!config.alwaysReexecuteBlockProposals; - log.info(`Setting up block proposal handler` + (reexecute ? ' with reexecution of proposals' : '')); - createBlockProposalHandler(config, { + log.info(`Setting up proposal handler` + (reexecute ? ' with reexecution of proposals' : '')); + createProposalHandler(config, { checkpointsBuilder: validatorCheckpointsBuilder, worldState: worldStateSynchronizer, epochCache, blockSource: archiver, l1ToL2MessageSource: archiver, p2pClient, + blobClient, dateProvider, telemetry, }).register(p2pClient, reexecute); diff --git a/yarn-project/bootstrap.sh b/yarn-project/bootstrap.sh index a7992576d9e5..47f62ba45613 100755 --- a/yarn-project/bootstrap.sh +++ b/yarn-project/bootstrap.sh @@ -224,7 +224,7 @@ function test_cmds { # Aztec CLI tests aztec/bootstrap.sh test_cmds - if [[ "${TARGET_BRANCH:-}" =~ ^v[0-9]+$ ]]; then + if [[ "${TARGET_BRANCH:-}" =~ ^(v[0-9]+(-next)?|backport-to-v[0-9]+-(staging|next))$ ]]; then echo "$hash yarn-project/scripts/run_test.sh aztec/src/testnet_compatibility.test.ts" echo "$hash yarn-project/scripts/run_test.sh aztec/src/mainnet_compatibility.test.ts" fi diff --git a/yarn-project/foundation/src/collection/array.test.ts b/yarn-project/foundation/src/collection/array.test.ts index f71c739354d4..9348a90d2191 100644 --- a/yarn-project/foundation/src/collection/array.test.ts +++ b/yarn-project/foundation/src/collection/array.test.ts @@ -8,6 +8,7 @@ import { mean, median, partition, + partitionAsync, removeArrayPaddingEnd, stdDev, times, @@ -380,3 +381,40 @@ describe('partition', () => { expect(odd).toEqual([{ a: 1 }, { a: 3 }]); }); }); + +describe('partitionAsync', () => { + it('partitions an array into pass and fail arrays based on the predicate', async () => { + const input = [1, 2, 3, 4, 5]; + const [even, odd] = await partitionAsync(input, x => Promise.resolve(x % 2 === 0)); + expect(even).toEqual([2, 4]); + expect(odd).toEqual([1, 3, 5]); + }); + + it('returns all items in the first array if all pass the predicate', async () => { + const input = [2, 4, 6]; + const [pass, fail] = await partitionAsync(input, x => Promise.resolve(x % 2 === 0)); + expect(pass).toEqual([2, 4, 6]); + expect(fail).toEqual([]); + }); + + it('returns all items in the second array if none pass the predicate', async () => { + const input = [1, 3, 5]; + const [pass, fail] = await partitionAsync(input, x => Promise.resolve(x % 2 === 0)); + expect(pass).toEqual([]); + expect(fail).toEqual([1, 3, 5]); + }); + + it('handles an empty array', async () => { + const input: number[] = []; + const [pass, fail] = await partitionAsync(input, x => Promise.resolve(x > 0)); + expect(pass).toEqual([]); + expect(fail).toEqual([]); + }); + + it('works with objects and custom predicates', async () => { + const input = [{ a: 1 }, { a: 2 }, { a: 3 }]; + const [even, odd] = await partitionAsync(input, obj => Promise.resolve(obj.a % 2 === 0)); + expect(even).toEqual([{ a: 2 }]); + expect(odd).toEqual([{ a: 1 }, { a: 3 }]); + }); +}); diff --git a/yarn-project/foundation/src/collection/array.ts b/yarn-project/foundation/src/collection/array.ts index 69dde35053a5..e2636c27d1fc 100644 --- a/yarn-project/foundation/src/collection/array.ts +++ b/yarn-project/foundation/src/collection/array.ts @@ -315,3 +315,17 @@ export function partition(items: T[], predicate: (item: T) => boolean): [T[], } return [pass, fail]; } + +/** Partitions the given iterable into two arrays based on the predicate. */ +export async function partitionAsync(items: T[], predicate: (item: T) => Promise): Promise<[T[], T[]]> { + const pass: T[] = []; + const fail: T[] = []; + for (const item of items) { + if (await predicate(item)) { + pass.push(item); + } else { + fail.push(item); + } + } + return [pass, fail]; +} diff --git a/yarn-project/p2p/src/client/factory.ts b/yarn-project/p2p/src/client/factory.ts index 361438feeae4..1afa87c148c2 100644 --- a/yarn-project/p2p/src/client/factory.ts +++ b/yarn-project/p2p/src/client/factory.ts @@ -19,6 +19,7 @@ import type { TxPoolV2 } from '../mem_pools/tx_pool_v2/interfaces.js'; import { AztecKVTxPoolV2 } from '../mem_pools/tx_pool_v2/tx_pool_v2.js'; import { createCheckAllowedSetupCalls, + createTxValidatorForReqResponseReceivedTxs, createTxValidatorForTransactionsEnteringPendingTxPool, getDefaultAllowedSetupFunctions, } from '../msg_validators/index.js'; @@ -147,9 +148,12 @@ export async function createP2PClient( telemetry, ); + const txValidatorForTxCollection = createTxValidatorForReqResponseReceivedTxs(proofVerifier, config); const nodeSources = [ - ...createNodeRpcTxSources(config.txCollectionNodeRpcUrls, config), - ...(deps.rpcTxProviders ?? []).map((node, i) => new NodeRpcTxSource(node, `node-rpc-provider-${i}`)), + ...createNodeRpcTxSources(config.txCollectionNodeRpcUrls, txValidatorForTxCollection, config), + ...(deps.rpcTxProviders ?? []).map( + (node, i) => new NodeRpcTxSource(node, txValidatorForTxCollection, `node-rpc-provider-${i}`), + ), ...(deps.txCollectionNodeSources ?? []), ]; if (nodeSources.length > 0) { @@ -161,6 +165,7 @@ export async function createP2PClient( const fileStoreSources = await createFileStoreTxSources( config.txCollectionFileStoreUrls, txFileStoreBasePath, + txValidatorForTxCollection, logger.createChild('file-store-tx-source'), telemetry, ); diff --git a/yarn-project/p2p/src/services/tx_collection/file_store_tx_source.ts b/yarn-project/p2p/src/services/tx_collection/file_store_tx_source.ts index d682e303a0cf..b47927e8cb44 100644 --- a/yarn-project/p2p/src/services/tx_collection/file_store_tx_source.ts +++ b/yarn-project/p2p/src/services/tx_collection/file_store_tx_source.ts @@ -1,7 +1,8 @@ +import { partitionAsync } from '@aztec/foundation/collection'; import { type Logger, createLogger } from '@aztec/foundation/log'; import { Timer } from '@aztec/foundation/timer'; import { type ReadOnlyFileStore, createReadOnlyFileStore } from '@aztec/stdlib/file-store'; -import { Tx, type TxHash } from '@aztec/stdlib/tx'; +import { Tx, type TxHash, type TxValidator } from '@aztec/stdlib/tx'; import { type Histogram, Metrics, @@ -23,6 +24,7 @@ export class FileStoreTxSource implements TxSource { private readonly fileStore: ReadOnlyFileStore, private readonly baseUrl: string, private readonly basePath: string, + private readonly txValidator: TxValidator, private readonly log: Logger, telemetry: TelemetryClient, ) { @@ -44,6 +46,7 @@ export class FileStoreTxSource implements TxSource { public static async create( url: string, basePath: string, + txValidator: TxValidator, log: Logger = createLogger('p2p:file_store_tx_source'), telemetry: TelemetryClient = getTelemetryClient(), ): Promise { @@ -53,7 +56,7 @@ export class FileStoreTxSource implements TxSource { log.warn(`Failed to create file store for URL: ${url}`); return undefined; } - return new FileStoreTxSource(fileStore, url, basePath, log, telemetry); + return new FileStoreTxSource(fileStore, url, basePath, txValidator, log, telemetry); } catch (err) { log.warn(`Error creating file store for URL: ${url}`, { error: err }); return undefined; @@ -65,35 +68,41 @@ export class FileStoreTxSource implements TxSource { } public async getTxsByHash(txHashes: TxHash[]): Promise { - const invalidTxHashes: string[] = []; + const results = await Promise.all( + txHashes.map(async txHash => { + const path = `${this.basePath}/txs/${txHash.toString()}.bin`; + const timer = new Timer(); + try { + const buffer = await this.fileStore.read(path); + const tx = Tx.fromBuffer(buffer); + return { tx, downloadDuration: timer.ms(), downloadSize: buffer.length }; + } catch { + this.downloadsFailed.add(1); + return undefined; + } + }), + ); + + const txs = results.filter(tx => tx !== undefined); + const [validTxs, invalidTxs] = await partitionAsync( + txs, + async ({ tx, downloadDuration, downloadSize }): Promise => { + const valid = await this.txValidator.validateTx(tx); + if (valid.result === 'valid') { + this.downloadsSuccess.add(1); + this.downloadDuration.record(Math.ceil(downloadDuration)); + this.downloadSize.record(downloadSize); + return true; + } else { + this.downloadsFailed.add(1); + return false; + } + }, + ); + return { - validTxs: ( - await Promise.all( - txHashes.map(async txHash => { - const path = `${this.basePath}/txs/${txHash.toString()}.bin`; - const timer = new Timer(); - try { - const buffer = await this.fileStore.read(path); - const tx = Tx.fromBuffer(buffer); - if ((await tx.validateTxHash()) && txHash.equals(tx.txHash)) { - this.downloadsSuccess.add(1); - this.downloadDuration.record(Math.ceil(timer.ms())); - this.downloadSize.record(buffer.length); - return tx; - } else { - invalidTxHashes.push(tx.txHash.toString()); - this.downloadsFailed.add(1); - return undefined; - } - } catch { - // Tx not found or error reading - return undefined - this.downloadsFailed.add(1); - return undefined; - } - }), - ) - ).filter(tx => tx !== undefined), - invalidTxHashes: invalidTxHashes, + validTxs: validTxs.map(({ tx }) => tx), + invalidTxHashes: invalidTxs.map(({ tx }) => tx.getTxHash().toString()), }; } } @@ -109,9 +118,12 @@ export class FileStoreTxSource implements TxSource { export async function createFileStoreTxSources( urls: string[], basePath: string, + txValidator: TxValidator, log: Logger = createLogger('p2p:file_store_tx_source'), telemetry: TelemetryClient = getTelemetryClient(), ): Promise { - const sources = await Promise.all(urls.map(url => FileStoreTxSource.create(url, basePath, log, telemetry))); + const sources = await Promise.all( + urls.map(url => FileStoreTxSource.create(url, basePath, txValidator, log, telemetry)), + ); return sources.filter((s): s is FileStoreTxSource => s !== undefined); } diff --git a/yarn-project/p2p/src/services/tx_collection/tx_source.test.ts b/yarn-project/p2p/src/services/tx_collection/tx_source.test.ts new file mode 100644 index 000000000000..4767b9fc2481 --- /dev/null +++ b/yarn-project/p2p/src/services/tx_collection/tx_source.test.ts @@ -0,0 +1,62 @@ +import type { AztecNode } from '@aztec/stdlib/interfaces/client'; +import { Tx, type TxValidator } from '@aztec/stdlib/tx'; + +import { type MockProxy, mock } from 'jest-mock-extended'; + +import { NodeRpcTxSource } from './tx_source.js'; + +describe('NodeRpcTxSource', () => { + let mockClient: MockProxy>; + let mockValidator: MockProxy; + + const makeTx = async () => { + const tx = Tx.random(); + await tx.recomputeHash(); + return tx; + }; + + beforeEach(() => { + mockClient = mock>(); + mockValidator = mock(); + mockValidator.validateTx.mockResolvedValue({ result: 'valid' }); + }); + + const createSource = () => new NodeRpcTxSource(mockClient, mockValidator, 'test'); + + it('returns valid txs when validator accepts', async () => { + const tx1 = await makeTx(); + const tx2 = await makeTx(); + mockClient.getTxsByHash.mockResolvedValue([tx1, tx2]); + + const result = await createSource().getTxsByHash([tx1.getTxHash(), tx2.getTxHash()]); + + expect(result.validTxs).toHaveLength(2); + expect(result.invalidTxHashes).toHaveLength(0); + }); + + it('returns invalid tx hashes when validator rejects', async () => { + const tx1 = await makeTx(); + const tx2 = await makeTx(); + mockClient.getTxsByHash.mockResolvedValue([tx1, tx2]); + mockValidator.validateTx.mockResolvedValue({ result: 'invalid', reason: ['bad'] }); + + const result = await createSource().getTxsByHash([tx1.getTxHash(), tx2.getTxHash()]); + + expect(result.validTxs).toHaveLength(0); + expect(result.invalidTxHashes).toEqual([tx1.getTxHash().toString(), tx2.getTxHash().toString()]); + }); + + it('partitions txs based on validator result', async () => { + const tx1 = await makeTx(); + const tx2 = await makeTx(); + mockClient.getTxsByHash.mockResolvedValue([tx1, tx2]); + mockValidator.validateTx + .mockResolvedValueOnce({ result: 'valid' }) + .mockResolvedValueOnce({ result: 'invalid', reason: ['bad'] }); + + const result = await createSource().getTxsByHash([tx1.getTxHash(), tx2.getTxHash()]); + + expect(result.validTxs).toEqual([tx1]); + expect(result.invalidTxHashes).toEqual([tx2.getTxHash().toString()]); + }); +}); diff --git a/yarn-project/p2p/src/services/tx_collection/tx_source.ts b/yarn-project/p2p/src/services/tx_collection/tx_source.ts index 68e2ad8c1c43..20ef0dc923e4 100644 --- a/yarn-project/p2p/src/services/tx_collection/tx_source.ts +++ b/yarn-project/p2p/src/services/tx_collection/tx_source.ts @@ -2,7 +2,7 @@ import { getVKTreeRoot } from '@aztec/noir-protocol-circuits-types/vk-tree'; import { protocolContractsHash } from '@aztec/protocol-contracts'; import type { ChainConfig } from '@aztec/stdlib/config'; import { type AztecNode, createAztecNodeClient } from '@aztec/stdlib/interfaces/client'; -import type { Tx, TxHash } from '@aztec/stdlib/tx'; +import type { Tx, TxHash, TxValidator } from '@aztec/stdlib/tx'; import { type ComponentsVersions, getComponentsVersionsFromConfig } from '@aztec/stdlib/versioning'; import { makeTracedFetch } from '@aztec/telemetry-client'; @@ -16,12 +16,13 @@ export interface TxSource { export class NodeRpcTxSource implements TxSource { constructor( private readonly client: Pick, + private readonly txValidator: TxValidator, private readonly info: string, ) {} - public static fromUrl(nodeUrl: string, versions: ComponentsVersions): NodeRpcTxSource { + public static fromUrl(nodeUrl: string, txValidator: TxValidator, versions: ComponentsVersions): NodeRpcTxSource { const client = createAztecNodeClient(nodeUrl, versions, makeTracedFetch([1, 2, 3], false)); - return new NodeRpcTxSource(client, nodeUrl); + return new NodeRpcTxSource(client, txValidator, nodeUrl); } public getInfo() { @@ -38,8 +39,8 @@ export class NodeRpcTxSource implements TxSource { const invalidTxHashes: string[] = []; await Promise.all( txs.map(async tx => { - const isValid = await tx.validateTxHash(); - if (isValid) { + const validation = await this.txValidator.validateTx(tx); + if (validation.result === 'valid') { validTxs.push(tx); } else { invalidTxHashes.push(tx.getTxHash().toString()); @@ -50,7 +51,7 @@ export class NodeRpcTxSource implements TxSource { } } -export function createNodeRpcTxSources(urls: string[], chainConfig: ChainConfig) { +export function createNodeRpcTxSources(urls: string[], txValidator: TxValidator, chainConfig: ChainConfig) { const versions = getComponentsVersionsFromConfig(chainConfig, protocolContractsHash, getVKTreeRoot()); - return urls.map(url => NodeRpcTxSource.fromUrl(url, versions)); + return urls.map(url => NodeRpcTxSource.fromUrl(url, txValidator, versions)); } diff --git a/yarn-project/p2p/src/services/tx_file_store/tx_file_store.test.ts b/yarn-project/p2p/src/services/tx_file_store/tx_file_store.test.ts index 893a0b54aa9d..2fbefb183222 100644 --- a/yarn-project/p2p/src/services/tx_file_store/tx_file_store.test.ts +++ b/yarn-project/p2p/src/services/tx_file_store/tx_file_store.test.ts @@ -1,10 +1,11 @@ import { createLogger } from '@aztec/foundation/log'; import { sleep } from '@aztec/foundation/sleep'; import { type FileStore, createFileStore } from '@aztec/stdlib/file-store'; -import { Tx } from '@aztec/stdlib/tx'; +import { Tx, type TxValidator } from '@aztec/stdlib/tx'; import { jest } from '@jest/globals'; import { mkdtemp, readdir, rm } from 'fs/promises'; +import { type MockProxy, mock } from 'jest-mock-extended'; import { tmpdir } from 'os'; import { join } from 'path'; @@ -19,6 +20,7 @@ describe('TxFileStore', () => { let txPool: InMemoryTxPool; let config: TxFileStoreConfig; let txFileStore: TxFileStore | undefined; + let mockValidator: MockProxy; const log = createLogger('test:tx_file_store'); const basePath = 'aztec-1-1-0x1234'; @@ -52,6 +54,8 @@ describe('TxFileStore', () => { fileStore = await createFileStore(`file://${tmpDir}`); txPool = new InMemoryTxPool(); + mockValidator = mock(); + mockValidator.validateTx.mockResolvedValue({ result: 'valid' }); config = { txFileStoreEnabled: true, @@ -310,50 +314,44 @@ describe('TxFileStore', () => { }); describe('tx download validation', () => { - it('rejects tx with invalid hash when reading from file store', async () => { - // Write a tx with a mismatched hash directly to the file store - const invalidTx = Tx.random(); // random hash does not match computed hash - await fileStore.save(`${basePath}/txs/${invalidTx.txHash.toString()}.bin`, invalidTx.toBuffer(), { - compress: false, - }); + it('rejects tx when validator returns invalid', async () => { + const tx = await makeTx(); + await fileStore.save(`${basePath}/txs/${tx.txHash.toString()}.bin`, tx.toBuffer(), { compress: false }); - // Read it back via FileStoreTxSource - const source = (await FileStoreTxSource.create(`file://${tmpDir}`, basePath, log))!; - const result = await source.getTxsByHash([invalidTx.txHash]); + mockValidator.validateTx.mockResolvedValueOnce({ result: 'invalid', reason: ['invalid'] }); + const source = (await FileStoreTxSource.create(`file://${tmpDir}`, basePath, mockValidator, log))!; + const result = await source.getTxsByHash([tx.txHash]); expect(result.validTxs).toHaveLength(0); - expect(result.invalidTxHashes).toEqual([invalidTx.txHash.toString()]); + expect(result.invalidTxHashes).toEqual([tx.txHash.toString()]); }); - it('rejects tx when tx with wrong hash is returned', async () => { - // Write a tx with a mismatched hash directly to the file store - const invalidTx = Tx.random(); // random hash does not match computed hash - const validTx = await makeTx(); - await fileStore.save(`${basePath}/txs/${invalidTx.txHash.toString()}.bin`, validTx.toBuffer(), { - compress: false, - }); + it('accepts tx when validator returns valid', async () => { + const tx = await makeTx(); + await fileStore.save(`${basePath}/txs/${tx.txHash.toString()}.bin`, tx.toBuffer(), { compress: false }); - // Read it back via FileStoreTxSource - const source = (await FileStoreTxSource.create(`file://${tmpDir}`, basePath, log))!; - const result = await source.getTxsByHash([invalidTx.txHash]); + const source = (await FileStoreTxSource.create(`file://${tmpDir}`, basePath, mockValidator, log))!; + const result = await source.getTxsByHash([tx.txHash]); - expect(result.validTxs).toHaveLength(0); - expect(result.invalidTxHashes).toEqual([validTx.txHash.toString()]); + expect(result.validTxs).toHaveLength(1); + expect(result.invalidTxHashes).toHaveLength(0); }); - it('accepts correct tx', async () => { - // Write a tx with a correct hash directly to the file store - const validTx = await makeTx(); - await fileStore.save(`${basePath}/txs/${validTx.txHash.toString()}.bin`, validTx.toBuffer(), { - compress: false, - }); + it('partitions txs based on validator result', async () => { + const tx1 = await makeTx(); + const tx2 = await makeTx(); + await fileStore.save(`${basePath}/txs/${tx1.txHash.toString()}.bin`, tx1.toBuffer(), { compress: false }); + await fileStore.save(`${basePath}/txs/${tx2.txHash.toString()}.bin`, tx2.toBuffer(), { compress: false }); + + mockValidator.validateTx + .mockResolvedValueOnce({ result: 'valid' }) + .mockResolvedValueOnce({ result: 'invalid', reason: ['bad'] }); - // Read it back via FileStoreTxSource - const source = (await FileStoreTxSource.create(`file://${tmpDir}`, basePath, log))!; - const result = await source.getTxsByHash([validTx.txHash]); + const source = (await FileStoreTxSource.create(`file://${tmpDir}`, basePath, mockValidator, log))!; + const result = await source.getTxsByHash([tx1.txHash, tx2.txHash]); expect(result.validTxs).toHaveLength(1); - expect(result.invalidTxHashes).toHaveLength(0); + expect(result.invalidTxHashes).toHaveLength(1); }); }); @@ -388,7 +386,7 @@ describe('TxFileStore', () => { await txFileStore!.flush(); // Read back via FileStoreTxSource using the same local file store - const txSource = await FileStoreTxSource.create(`file://${tmpDir}`, basePath, log); + const txSource = await FileStoreTxSource.create(`file://${tmpDir}`, basePath, mockValidator, log); expect(txSource).toBeDefined(); const results = await txSource!.getTxsByHash([tx.getTxHash()]); diff --git a/yarn-project/validator-client/src/factory.ts b/yarn-project/validator-client/src/factory.ts index 6d9db7607753..259d3081ab2f 100644 --- a/yarn-project/validator-client/src/factory.ts +++ b/yarn-project/validator-client/src/factory.ts @@ -9,12 +9,12 @@ import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging'; import type { TelemetryClient } from '@aztec/telemetry-client'; import type { SlashingProtectionDatabase } from '@aztec/validator-ha-signer/types'; -import { BlockProposalHandler } from './block_proposal_handler.js'; import type { FullNodeCheckpointsBuilder } from './checkpoint_builder.js'; import { ValidatorMetrics } from './metrics.js'; +import { ProposalHandler } from './proposal_handler.js'; import { ValidatorClient } from './validator.js'; -export function createBlockProposalHandler( +export function createProposalHandler( config: ValidatorClientFullConfig, deps: { checkpointsBuilder: FullNodeCheckpointsBuilder; @@ -23,6 +23,7 @@ export function createBlockProposalHandler( l1ToL2MessageSource: L1ToL2MessageSource; p2pClient: P2PClient; epochCache: EpochCache; + blobClient: BlobClientInterface; dateProvider: DateProvider; telemetry: TelemetryClient; }, @@ -32,7 +33,7 @@ export function createBlockProposalHandler( txsPermitted: !config.disableTransactions, maxTxsPerBlock: config.validateMaxTxsPerBlock ?? config.validateMaxTxsPerCheckpoint, }); - return new BlockProposalHandler( + return new ProposalHandler( deps.checkpointsBuilder, deps.worldState, deps.blockSource, @@ -41,6 +42,7 @@ export function createBlockProposalHandler( blockProposalValidator, deps.epochCache, config, + deps.blobClient, metrics, deps.dateProvider, deps.telemetry, diff --git a/yarn-project/validator-client/src/index.ts b/yarn-project/validator-client/src/index.ts index e1bb317f9f81..1cef663abc9b 100644 --- a/yarn-project/validator-client/src/index.ts +++ b/yarn-project/validator-client/src/index.ts @@ -1,4 +1,4 @@ -export * from './block_proposal_handler.js'; +export * from './proposal_handler.js'; export * from './checkpoint_builder.js'; export * from './config.js'; export * from './factory.js'; diff --git a/yarn-project/validator-client/src/metrics.ts b/yarn-project/validator-client/src/metrics.ts index 160ac8c17280..7142399023df 100644 --- a/yarn-project/validator-client/src/metrics.ts +++ b/yarn-project/validator-client/src/metrics.ts @@ -11,7 +11,7 @@ import { createUpDownCounterWithDefault, } from '@aztec/telemetry-client'; -import type { BlockProposalValidationFailureReason } from './block_proposal_handler.js'; +import type { BlockProposalValidationFailureReason } from './proposal_handler.js'; export class ValidatorMetrics { private failedReexecutionCounter: UpDownCounter; diff --git a/yarn-project/validator-client/src/block_proposal_handler.ts b/yarn-project/validator-client/src/proposal_handler.ts similarity index 67% rename from yarn-project/validator-client/src/block_proposal_handler.ts rename to yarn-project/validator-client/src/proposal_handler.ts index 642ec9410144..e74e19d31410 100644 --- a/yarn-project/validator-client/src/block_proposal_handler.ts +++ b/yarn-project/validator-client/src/proposal_handler.ts @@ -1,20 +1,29 @@ +import type { BlobClientInterface } from '@aztec/blob-client/client'; +import { type Blob, encodeCheckpointBlobDataFromBlocks, getBlobsPerL1Block } from '@aztec/blob-lib'; import { INITIAL_L2_BLOCK_NUM } from '@aztec/constants'; import type { EpochCache } from '@aztec/epoch-cache'; +import { validateFeeAssetPriceModifier } from '@aztec/ethereum/contracts'; import { BlockNumber, CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types'; import { pick } from '@aztec/foundation/collection'; import { Fr } from '@aztec/foundation/curves/bn254'; import { TimeoutError } from '@aztec/foundation/error'; +import type { LogData } from '@aztec/foundation/log'; import { createLogger } from '@aztec/foundation/log'; import { retryUntil } from '@aztec/foundation/retry'; import { DateProvider, Timer } from '@aztec/foundation/timer'; import type { P2P, PeerId } from '@aztec/p2p'; import { BlockProposalValidator } from '@aztec/p2p/msg_validators'; import type { BlockData, L2Block, L2BlockSink, L2BlockSource } from '@aztec/stdlib/block'; +import { validateCheckpoint } from '@aztec/stdlib/checkpoint'; import { getEpochAtSlot, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; import { Gas } from '@aztec/stdlib/gas'; import type { ITxProvider, ValidatorClientFullConfig, WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server'; -import { type L1ToL2MessageSource, computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging'; -import type { BlockProposal } from '@aztec/stdlib/p2p'; +import { + type L1ToL2MessageSource, + accumulateCheckpointOutHashes, + computeInHashFromL1ToL2Messages, +} from '@aztec/stdlib/messaging'; +import type { BlockProposal, CheckpointProposalCore } from '@aztec/stdlib/p2p'; import { MerkleTreeId } from '@aztec/stdlib/trees'; import type { CheckpointGlobalVariables, FailedTx, Tx } from '@aztec/stdlib/tx'; import { @@ -66,11 +75,14 @@ export type BlockProposalValidationFailureResult = { export type BlockProposalValidationResult = BlockProposalValidationSuccessResult | BlockProposalValidationFailureResult; +export type CheckpointProposalValidationResult = { isValid: true } | { isValid: false; reason: string }; + type CheckpointComputationResult = | { checkpointNumber: CheckpointNumber; reason?: undefined } | { checkpointNumber?: undefined; reason: 'invalid_proposal' | 'global_variables_mismatch' }; -export class BlockProposalHandler { +/** Handles block and checkpoint proposals for both validator and non-validator nodes. */ +export class ProposalHandler { public readonly tracer: Tracer; constructor( @@ -82,21 +94,26 @@ export class BlockProposalHandler { private blockProposalValidator: BlockProposalValidator, private epochCache: EpochCache, private config: ValidatorClientFullConfig, + private blobClient: BlobClientInterface, private metrics?: ValidatorMetrics, private dateProvider: DateProvider = new DateProvider(), telemetry: TelemetryClient = getTelemetryClient(), - private log = createLogger('validator:block-proposal-handler'), + private log = createLogger('validator:proposal-handler'), ) { if (config.fishermanMode) { this.log = this.log.createChild('[FISHERMAN]'); } - this.tracer = telemetry.getTracer('BlockProposalHandler'); + this.tracer = telemetry.getTracer('ProposalHandler'); } - register(p2pClient: P2P, shouldReexecute: boolean): BlockProposalHandler { + /** + * Registers non-validator handlers for block and checkpoint proposals on the p2p client. + * Block proposals are always registered. Checkpoint proposals are registered if the blob client can upload. + */ + register(p2pClient: P2P, shouldReexecute: boolean): ProposalHandler { // Non-validator handler that processes or re-executes for monitoring but does not attest. // Returns boolean indicating whether the proposal was valid. - const handler = async (proposal: BlockProposal, proposalSender: PeerId): Promise => { + const blockHandler = async (proposal: BlockProposal, proposalSender: PeerId): Promise => { try { const { slotNumber, blockNumber } = proposal; const result = await this.handleBlockProposal(proposal, proposalSender, shouldReexecute); @@ -123,7 +140,35 @@ export class BlockProposalHandler { } }; - p2pClient.registerBlockProposalHandler(handler); + p2pClient.registerBlockProposalHandler(blockHandler); + + // Register checkpoint proposal handler if blob uploads are enabled and we are reexecuting + if (this.blobClient.canUpload() && shouldReexecute) { + const checkpointHandler = async (checkpoint: CheckpointProposalCore, _sender: PeerId) => { + try { + const proposalInfo = { + proposalSlotNumber: checkpoint.slotNumber, + archive: checkpoint.archive.toString(), + proposer: checkpoint.getSender()?.toString(), + }; + const result = await this.handleCheckpointProposal(checkpoint, proposalInfo); + if (result.isValid) { + this.log.info(`Non-validator checkpoint proposal at slot ${checkpoint.slotNumber} handled`, proposalInfo); + } else { + this.log.warn( + `Non-validator checkpoint proposal at slot ${checkpoint.slotNumber} failed: ${result.reason}`, + proposalInfo, + ); + } + } catch (error) { + this.log.error('Error processing checkpoint proposal in non-validator handler', error); + } + // Non-validators don't attest + return undefined; + }; + p2pClient.registerCheckpointProposalHandler(checkpointHandler); + } + return this; } @@ -625,4 +670,234 @@ export class BlockProposalHandler { totalManaUsed, }; } + + /** + * Validates a checkpoint proposal and uploads blobs if configured. + * Used by both non-validator nodes (via register) and the validator client (via delegation). + */ + async handleCheckpointProposal( + proposal: CheckpointProposalCore, + proposalInfo: LogData, + ): Promise { + const proposer = proposal.getSender(); + if (!proposer) { + this.log.warn(`Received checkpoint proposal with invalid signature for slot ${proposal.slotNumber}`); + return { isValid: false, reason: 'invalid_signature' }; + } + + if (!validateFeeAssetPriceModifier(proposal.feeAssetPriceModifier)) { + this.log.warn( + `Received checkpoint proposal with invalid feeAssetPriceModifier ${proposal.feeAssetPriceModifier} for slot ${proposal.slotNumber}`, + ); + return { isValid: false, reason: 'invalid_fee_asset_price_modifier' }; + } + + const result = await this.validateCheckpointProposal(proposal, proposalInfo); + + // Upload blobs to filestore if validation passed (fire and forget) + if (result.isValid) { + this.tryUploadBlobsForCheckpoint(proposal, proposalInfo); + } + + return result; + } + + /** + * Validates a checkpoint proposal by building the full checkpoint and comparing it with the proposal. + * @returns Validation result with isValid flag and reason if invalid. + */ + async validateCheckpointProposal( + proposal: CheckpointProposalCore, + proposalInfo: LogData, + ): Promise { + const slot = proposal.slotNumber; + + // Timeout block syncing at the start of the next slot + const config = this.checkpointsBuilder.getConfig(); + const nextSlotTimestampSeconds = Number(getTimestampForSlot(SlotNumber(slot + 1), config)); + const timeoutSeconds = Math.max(1, nextSlotTimestampSeconds - Math.floor(this.dateProvider.now() / 1000)); + + // Wait for last block to sync by archive + let lastBlockHeader; + try { + lastBlockHeader = await retryUntil( + async () => { + await this.blockSource.syncImmediate(); + return this.blockSource.getBlockHeaderByArchive(proposal.archive); + }, + `waiting for block with archive ${proposal.archive.toString()} for slot ${slot}`, + timeoutSeconds, + 0.5, + ); + } catch (err) { + if (err instanceof TimeoutError) { + this.log.warn(`Timed out waiting for block with archive matching checkpoint proposal`, proposalInfo); + return { isValid: false, reason: 'last_block_not_found' }; + } + this.log.error(`Error fetching last block for checkpoint proposal`, err, proposalInfo); + return { isValid: false, reason: 'block_fetch_error' }; + } + + if (!lastBlockHeader) { + this.log.warn(`Last block not found for checkpoint proposal`, proposalInfo); + return { isValid: false, reason: 'last_block_not_found' }; + } + + // Get all full blocks for the slot and checkpoint + const blocks = await this.blockSource.getBlocksForSlot(slot); + if (blocks.length === 0) { + this.log.warn(`No blocks found for slot ${slot}`, proposalInfo); + return { isValid: false, reason: 'no_blocks_for_slot' }; + } + + // Ensure the last block for this slot matches the archive in the checkpoint proposal + if (!blocks.at(-1)?.archive.root.equals(proposal.archive)) { + this.log.warn(`Last block archive mismatch for checkpoint proposal`, proposalInfo); + return { isValid: false, reason: 'last_block_archive_mismatch' }; + } + + this.log.debug(`Found ${blocks.length} blocks for slot ${slot}`, { + ...proposalInfo, + blockNumbers: blocks.map(b => b.number), + }); + + // Get checkpoint constants from first block + const firstBlock = blocks[0]; + const constants = this.extractCheckpointConstants(firstBlock); + const checkpointNumber = firstBlock.checkpointNumber; + + // Get L1-to-L2 messages for this checkpoint + const l1ToL2Messages = await this.l1ToL2MessageSource.getL1ToL2Messages(checkpointNumber); + + // Collect the out hashes of all the checkpoints before this one in the same epoch + const epoch = getEpochAtSlot(slot, this.epochCache.getL1Constants()); + const previousCheckpointOutHashes = (await this.blockSource.getCheckpointsDataForEpoch(epoch)) + .filter(c => c.checkpointNumber < checkpointNumber) + .map(c => c.checkpointOutHash); + + // Fork world state at the block before the first block + const parentBlockNumber = BlockNumber(firstBlock.number - 1); + const fork = await this.worldState.fork(parentBlockNumber); + + try { + // Create checkpoint builder with all existing blocks + const checkpointBuilder = await this.checkpointsBuilder.openCheckpoint( + checkpointNumber, + constants, + proposal.feeAssetPriceModifier, + l1ToL2Messages, + previousCheckpointOutHashes, + fork, + blocks, + this.log.getBindings(), + ); + + // Complete the checkpoint to get computed values + const computedCheckpoint = await checkpointBuilder.completeCheckpoint(); + + // Compare checkpoint header with proposal + if (!computedCheckpoint.header.equals(proposal.checkpointHeader)) { + this.log.warn(`Checkpoint header mismatch`, { + ...proposalInfo, + computed: computedCheckpoint.header.toInspect(), + proposal: proposal.checkpointHeader.toInspect(), + }); + return { isValid: false, reason: 'checkpoint_header_mismatch' }; + } + + // Compare archive root with proposal + if (!computedCheckpoint.archive.root.equals(proposal.archive)) { + this.log.warn(`Archive root mismatch`, { + ...proposalInfo, + computed: computedCheckpoint.archive.root.toString(), + proposal: proposal.archive.toString(), + }); + return { isValid: false, reason: 'archive_mismatch' }; + } + + // Check that the accumulated epoch out hash matches the value in the proposal. + // The epoch out hash is the accumulated hash of all checkpoint out hashes in the epoch. + const checkpointOutHash = computedCheckpoint.getCheckpointOutHash(); + const computedEpochOutHash = accumulateCheckpointOutHashes([...previousCheckpointOutHashes, checkpointOutHash]); + const proposalEpochOutHash = proposal.checkpointHeader.epochOutHash; + if (!computedEpochOutHash.equals(proposalEpochOutHash)) { + this.log.warn(`Epoch out hash mismatch`, { + proposalEpochOutHash: proposalEpochOutHash.toString(), + computedEpochOutHash: computedEpochOutHash.toString(), + checkpointOutHash: checkpointOutHash.toString(), + previousCheckpointOutHashes: previousCheckpointOutHashes.map(h => h.toString()), + ...proposalInfo, + }); + return { isValid: false, reason: 'out_hash_mismatch' }; + } + + // Final round of validations on the checkpoint, just in case. + try { + validateCheckpoint(computedCheckpoint, { + rollupManaLimit: this.checkpointsBuilder.getConfig().rollupManaLimit, + maxDABlockGas: this.config.validateMaxDABlockGas, + maxL2BlockGas: this.config.validateMaxL2BlockGas, + maxTxsPerBlock: this.config.validateMaxTxsPerBlock, + maxTxsPerCheckpoint: this.config.validateMaxTxsPerCheckpoint, + }); + } catch (err) { + this.log.warn(`Checkpoint validation failed: ${err}`, proposalInfo); + return { isValid: false, reason: 'checkpoint_validation_failed' }; + } + + this.log.verbose(`Checkpoint proposal validation successful for slot ${slot}`, proposalInfo); + return { isValid: true }; + } finally { + await fork.close(); + } + } + + /** Extracts checkpoint global variables from a block. */ + private extractCheckpointConstants(block: L2Block): CheckpointGlobalVariables { + const gv = block.header.globalVariables; + return { + chainId: gv.chainId, + version: gv.version, + slotNumber: gv.slotNumber, + timestamp: gv.timestamp, + coinbase: gv.coinbase, + feeRecipient: gv.feeRecipient, + gasFees: gv.gasFees, + }; + } + + /** Triggers blob upload for a checkpoint if the blob client can upload (fire and forget). */ + protected tryUploadBlobsForCheckpoint(proposal: CheckpointProposalCore, proposalInfo: LogData): void { + if (this.blobClient.canUpload()) { + void this.uploadBlobsForCheckpoint(proposal, proposalInfo); + } + } + + /** Uploads blobs for a checkpoint to the filestore. */ + protected async uploadBlobsForCheckpoint(proposal: CheckpointProposalCore, proposalInfo: LogData): Promise { + try { + const lastBlockHeader = await this.blockSource.getBlockHeaderByArchive(proposal.archive); + if (!lastBlockHeader) { + this.log.warn(`Failed to get last block header for blob upload`, proposalInfo); + return; + } + + const blocks = await this.blockSource.getBlocksForSlot(proposal.slotNumber); + if (blocks.length === 0) { + this.log.warn(`No blocks found for blob upload`, proposalInfo); + return; + } + + const blockBlobData = blocks.map(b => b.toBlockBlobData()); + const blobFields = encodeCheckpointBlobDataFromBlocks(blockBlobData); + const blobs: Blob[] = await getBlobsPerL1Block(blobFields); + await this.blobClient.sendBlobsToFilestore(blobs); + this.log.debug(`Uploaded ${blobs.length} blobs to filestore for checkpoint at slot ${proposal.slotNumber}`, { + ...proposalInfo, + numBlobs: blobs.length, + }); + } catch (err) { + this.log.warn(`Failed to upload blobs for checkpoint: ${err}`, proposalInfo); + } + } } diff --git a/yarn-project/validator-client/src/validator.ha.integration.test.ts b/yarn-project/validator-client/src/validator.ha.integration.test.ts index 80c7bd532974..185b9c734556 100644 --- a/yarn-project/validator-client/src/validator.ha.integration.test.ts +++ b/yarn-project/validator-client/src/validator.ha.integration.test.ts @@ -33,12 +33,12 @@ import { afterEach, beforeEach, describe, expect, it } from '@jest/globals'; import { type MockProxy, mock } from 'jest-mock-extended'; import { type PrivateKeyAccount, generatePrivateKey, privateKeyToAccount } from 'viem/accounts'; -import { BlockProposalHandler } from './block_proposal_handler.js'; import type { FullNodeCheckpointsBuilder } from './checkpoint_builder.js'; import type { ValidatorClientConfig } from './config.js'; import { HAKeyStore } from './key_store/ha_key_store.js'; import { NodeKeystoreAdapter } from './key_store/node_keystore_adapter.js'; import { ValidatorMetrics } from './metrics.js'; +import { ProposalHandler } from './proposal_handler.js'; import { ValidatorClient } from './validator.js'; describe('ValidatorClient HA Integration', () => { @@ -195,7 +195,7 @@ describe('ValidatorClient HA Integration', () => { txsPermitted: true, maxTxsPerBlock: undefined, }); - const blockProposalHandler = new BlockProposalHandler( + const proposalHandler = new ProposalHandler( checkpointsBuilder, worldState, blockSource, @@ -204,6 +204,7 @@ describe('ValidatorClient HA Integration', () => { blockProposalValidator, epochCache, config, + blobClient, metrics, dateProvider, getTelemetryClient(), @@ -215,13 +216,10 @@ describe('ValidatorClient HA Integration', () => { haKeyStore, epochCache, p2pClient, - blockProposalHandler, - blockSource, - checkpointsBuilder, - worldState, - l1ToL2MessageSource, + proposalHandler, config, blobClient, + haSigner, dateProvider, getTelemetryClient(), ) as ValidatorClient; diff --git a/yarn-project/validator-client/src/validator.test.ts b/yarn-project/validator-client/src/validator.test.ts index 6fb29264ffac..196e9a37b5a5 100644 --- a/yarn-project/validator-client/src/validator.test.ts +++ b/yarn-project/validator-client/src/validator.test.ts @@ -23,7 +23,7 @@ import { } from '@aztec/p2p'; import { OffenseType, WANT_TO_SLASH_EVENT } from '@aztec/slasher'; import { AztecAddress } from '@aztec/stdlib/aztec-address'; -import type { BlockData, L2Block, L2BlockSink, L2BlockSource } from '@aztec/stdlib/block'; +import { type BlockData, L2Block, type L2BlockSink, type L2BlockSource } from '@aztec/stdlib/block'; import type { getEpochAtSlot } from '@aztec/stdlib/epoch-helpers'; import type { SlasherConfig, WorldStateSynchronizer } from '@aztec/stdlib/interfaces/server'; import { type L1ToL2MessageSource, computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging'; @@ -51,6 +51,7 @@ import type { } from './checkpoint_builder.js'; import { type ValidatorClientConfig, validatorClientConfigMappings } from './config.js'; import { HAKeyStore } from './key_store/ha_key_store.js'; +import { ProposalHandler } from './proposal_handler.js'; import { ValidatorClient } from './validator.js'; function makeKeyStore(validator: { @@ -84,7 +85,7 @@ describe('ValidatorClient', () => { > & { disableTransactions: boolean; }; - let validatorClient: TestValidatorClient; + let validatorClient: ValidatorClient; let p2pClient: MockProxy; let blockSource: MockProxy; let l1ToL2MessageSource: MockProxy; @@ -92,7 +93,7 @@ describe('ValidatorClient', () => { let checkpointsBuilder: MockProxy; let worldState: MockProxy; let validatorAccounts: PrivateKeyAccount[]; - let validatorPrivateKeys: `0x${string}`[]; + let validatorPrivateKeys: ReturnType[]; let dateProvider: TestDateProvider; let txProvider: MockProxy; let keyStoreManager: KeystoreManager; @@ -173,7 +174,7 @@ describe('ValidatorClient', () => { keyStoreManager, blobClient, dateProvider, - )) as TestValidatorClient; + )) as ValidatorClient; }); describe('createBlockProposal', () => { @@ -398,7 +399,7 @@ describe('ValidatorClient', () => { epochCache.getProposerAttesterAddressInSlot.mockResolvedValue(selfSigner.address); - const handleSpy = jest.spyOn(validatorClient.getBlockProposalHandler(), 'handleBlockProposal'); + const handleSpy = jest.spyOn(validatorClient.getProposalHandler(), 'handleBlockProposal'); const isValid = await validatorClient.validateBlockProposal(selfProposal, sender); expect(isValid).toBe(true); expect(handleSpy).toHaveBeenCalled(); @@ -407,7 +408,7 @@ describe('ValidatorClient', () => { it('should return early when escape hatch is open', async () => { epochCache.isEscapeHatchOpenAtSlot.mockResolvedValueOnce(true); - const handleSpy = jest.spyOn(validatorClient.getBlockProposalHandler(), 'handleBlockProposal'); + const handleSpy = jest.spyOn(validatorClient.getProposalHandler(), 'handleBlockProposal'); const isValid = await validatorClient.validateBlockProposal(proposal, sender); expect(isValid).toBe(false); @@ -476,7 +477,10 @@ describe('ValidatorClient', () => { it('should attest to a checkpoint proposal after validating a block for that slot', async () => { const addCheckpointAttestationsSpy = jest.spyOn(p2pClient, 'addOwnCheckpointAttestations'); - const uploadBlobsSpy = jest.spyOn(validatorClient, 'uploadBlobsForCheckpoint'); + const uploadBlobsSpy = jest.spyOn( + validatorClient.getProposalHandler() as TestProposalHandler, + 'tryUploadBlobsForCheckpoint', + ); const didValidate = await validatorClient.validateBlockProposal(proposal, sender); expect(didValidate).toBe(true); @@ -491,10 +495,15 @@ describe('ValidatorClient', () => { }, }); + // Mock validateCheckpointProposal to pass, so handleCheckpointProposal runs its + // own checks (signature, fee modifier) and then proceeds to blob upload. + const validateCheckpointSpy = jest + .spyOn(validatorClient.getProposalHandler(), 'validateCheckpointProposal') + .mockResolvedValue({ isValid: true }); + // Enable blob upload for this attestation blobClient.canUpload.mockReturnValue(true); - validatorClient.updateConfig({ skipCheckpointProposalValidation: true }); const attestations = await validatorClient.attestToCheckpointProposal(checkpointProposal, sender); expect(attestations).toBeDefined(); @@ -503,6 +512,7 @@ describe('ValidatorClient', () => { expect(uploadBlobsSpy).toHaveBeenCalled(); uploadBlobsSpy.mockRestore(); + validateCheckpointSpy.mockRestore(); }); it('should not attest to a checkpoint proposal that references a middle block instead of the last', async () => { @@ -794,7 +804,7 @@ describe('ValidatorClient', () => { // blocks in the same checkpoint share the same checkpointNumber, they will always // compute the same inHash from the same L1 messages. If a malicious proposal has a // different inHash, it will fail the existing validation at lines 192-200 in - // block_proposal_handler.ts. + // proposal_handler.ts. }); it('should validate proposals in fisherman mode but not create or broadcast attestations', async () => { @@ -893,13 +903,15 @@ describe('ValidatorClient', () => { const proposalInfo = { slotNumber: 1, archive: '0x00', proposer: '0x00', txCount: 0 }; it('should send blobs from blocks in the slot to filestore', async () => { - const blobFields = [Fr.random(), Fr.random()]; - const mockBlock = { toBlobFields: () => blobFields } as unknown as L2Block; + const mockBlock = L2Block.empty(); blockSource.getBlockHeaderByArchive.mockResolvedValue(makeBlockHeader()); blockSource.getBlocksForSlot.mockResolvedValue([mockBlock]); const proposal = await makeCheckpointProposal({ lastBlock: {} }); - await validatorClient.uploadBlobsForCheckpoint(proposal, proposalInfo); + await (validatorClient.getProposalHandler() as TestProposalHandler).uploadBlobsForCheckpoint( + proposal, + proposalInfo, + ); expect(blockSource.getBlocksForSlot).toHaveBeenCalledWith(proposal.slotNumber); expect(blobClient.sendBlobsToFilestore).toHaveBeenCalled(); @@ -909,19 +921,24 @@ describe('ValidatorClient', () => { blockSource.getBlockHeaderByArchive.mockResolvedValue(undefined); const proposal = await makeCheckpointProposal({ lastBlock: {} }); - await validatorClient.uploadBlobsForCheckpoint(proposal, proposalInfo); + await (validatorClient.getProposalHandler() as TestProposalHandler).uploadBlobsForCheckpoint( + proposal, + proposalInfo, + ); expect(blobClient.sendBlobsToFilestore).not.toHaveBeenCalled(); }); it('should not throw when blob upload fails', async () => { - const mockBlock = { toBlobFields: () => [Fr.random()] } as unknown as L2Block; + const mockBlock = L2Block.empty(); blockSource.getBlockHeaderByArchive.mockResolvedValue(makeBlockHeader()); blockSource.getBlocksForSlot.mockResolvedValue([mockBlock]); blobClient.sendBlobsToFilestore.mockRejectedValue(new Error('upload failed')); const proposal = await makeCheckpointProposal({ lastBlock: {} }); - await expect(validatorClient.uploadBlobsForCheckpoint(proposal, proposalInfo)).resolves.toBeUndefined(); + await expect( + (validatorClient.getProposalHandler() as TestProposalHandler).uploadBlobsForCheckpoint(proposal, proposalInfo), + ).resolves.toBeUndefined(); }); }); @@ -1086,8 +1103,11 @@ describe('ValidatorClient', () => { }); /** Exposes protected methods for direct testing */ -class TestValidatorClient extends ValidatorClient { +class TestProposalHandler extends ProposalHandler { declare public uploadBlobsForCheckpoint: ( - ...args: Parameters + ...args: Parameters ) => Promise; + declare public tryUploadBlobsForCheckpoint: ( + ...args: Parameters + ) => void; } diff --git a/yarn-project/validator-client/src/validator.ts b/yarn-project/validator-client/src/validator.ts index 0947205959a5..b422c7ca4ac0 100644 --- a/yarn-project/validator-client/src/validator.ts +++ b/yarn-project/validator-client/src/validator.ts @@ -1,7 +1,5 @@ import type { BlobClientInterface } from '@aztec/blob-client/client'; -import { type Blob, getBlobsPerL1Block } from '@aztec/blob-lib'; import type { EpochCache } from '@aztec/epoch-cache'; -import { validateFeeAssetPriceModifier } from '@aztec/ethereum/contracts'; import { BlockNumber, CheckpointNumber, @@ -10,11 +8,9 @@ import { SlotNumber, } from '@aztec/foundation/branded-types'; import { Fr } from '@aztec/foundation/curves/bn254'; -import { TimeoutError } from '@aztec/foundation/error'; import type { EthAddress } from '@aztec/foundation/eth-address'; import type { Signature } from '@aztec/foundation/eth-signature'; -import { type LogData, type Logger, createLogger } from '@aztec/foundation/log'; -import { retryUntil } from '@aztec/foundation/retry'; +import { type Logger, createLogger } from '@aztec/foundation/log'; import { RunningPromise } from '@aztec/foundation/running-promise'; import { sleep } from '@aztec/foundation/sleep'; import { DateProvider } from '@aztec/foundation/timer'; @@ -23,9 +19,8 @@ import type { DuplicateAttestationInfo, DuplicateProposalInfo, P2P, PeerId } fro import { AuthRequest, AuthResponse, BlockProposalValidator, ReqRespSubProtocol } from '@aztec/p2p'; import { OffenseType, WANT_TO_SLASH_EVENT, type Watcher, type WatcherEmitter } from '@aztec/slasher'; import type { AztecAddress } from '@aztec/stdlib/aztec-address'; -import type { CommitteeAttestationsAndSigners, L2Block, L2BlockSink, L2BlockSource } from '@aztec/stdlib/block'; -import { validateCheckpoint } from '@aztec/stdlib/checkpoint'; -import { getEpochAtSlot, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; +import type { CommitteeAttestationsAndSigners, L2BlockSink, L2BlockSource } from '@aztec/stdlib/block'; +import { getEpochAtSlot } from '@aztec/stdlib/epoch-helpers'; import type { CreateCheckpointProposalLastBlockData, ITxProvider, @@ -33,7 +28,7 @@ import type { ValidatorClientFullConfig, WorldStateSynchronizer, } from '@aztec/stdlib/interfaces/server'; -import { type L1ToL2MessageSource, accumulateCheckpointOutHashes } from '@aztec/stdlib/messaging'; +import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging'; import { type BlockProposal, type BlockProposalOptions, @@ -43,7 +38,7 @@ import { type CheckpointProposalOptions, } from '@aztec/stdlib/p2p'; import type { CheckpointHeader } from '@aztec/stdlib/rollup'; -import type { BlockHeader, CheckpointGlobalVariables, Tx } from '@aztec/stdlib/tx'; +import type { BlockHeader, Tx } from '@aztec/stdlib/tx'; import { AttestationTimeoutError } from '@aztec/stdlib/validators'; import { type TelemetryClient, type Tracer, getTelemetryClient } from '@aztec/telemetry-client'; import { createHASigner, createSignerFromSharedDb } from '@aztec/validator-ha-signer/factory'; @@ -53,13 +48,13 @@ import type { ValidatorHASigner } from '@aztec/validator-ha-signer/validator-ha- import { EventEmitter } from 'events'; import type { TypedDataDefinition } from 'viem'; -import { BlockProposalHandler, type BlockProposalValidationFailureReason } from './block_proposal_handler.js'; import type { FullNodeCheckpointsBuilder } from './checkpoint_builder.js'; import { ValidationService } from './duties/validation_service.js'; import { HAKeyStore } from './key_store/ha_key_store.js'; import type { ExtendedValidatorKeyStore } from './key_store/interface.js'; import { NodeKeystoreAdapter } from './key_store/node_keystore_adapter.js'; import { ValidatorMetrics } from './metrics.js'; +import { type BlockProposalValidationFailureReason, ProposalHandler } from './proposal_handler.js'; // We maintain a set of proposers who have proposed invalid blocks. // Just cap the set to avoid unbounded growth. @@ -102,11 +97,7 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) private keyStore: ExtendedValidatorKeyStore, private epochCache: EpochCache, private p2pClient: P2P, - private blockProposalHandler: BlockProposalHandler, - private blockSource: L2BlockSource, - private checkpointsBuilder: FullNodeCheckpointsBuilder, - private worldState: WorldStateSynchronizer, - private l1ToL2MessageSource: L1ToL2MessageSource, + private proposalHandler: ProposalHandler, private config: ValidatorClientFullConfig, private blobClient: BlobClientInterface, private haSigner: ValidatorHASigner | undefined, @@ -204,7 +195,7 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) txsPermitted: !config.disableTransactions, maxTxsPerBlock: config.validateMaxTxsPerBlock, }); - const blockProposalHandler = new BlockProposalHandler( + const proposalHandler = new ProposalHandler( checkpointsBuilder, worldState, blockSource, @@ -213,6 +204,7 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) blockProposalValidator, epochCache, config, + blobClient, metrics, dateProvider, telemetry, @@ -241,11 +233,7 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) validatorKeyStore, epochCache, p2pClient, - blockProposalHandler, - blockSource, - checkpointsBuilder, - worldState, - l1ToL2MessageSource, + proposalHandler, config, blobClient, haSigner, @@ -262,8 +250,8 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) .filter(addr => !this.config.disabledValidators.some(disabled => disabled.equals(addr))); } - public getBlockProposalHandler() { - return this.blockProposalHandler; + public getProposalHandler() { + return this.proposalHandler; } public signWithAddress(addr: EthAddress, msg: TypedDataDefinition, context: SigningContext) { @@ -421,7 +409,7 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) alwaysReexecuteBlockProposals || this.blobClient.canUpload(); - const validationResult = await this.blockProposalHandler.handleBlockProposal( + const validationResult = await this.proposalHandler.handleBlockProposal( proposal, proposalSender, !!shouldReexecute && !escapeHatchOpen, @@ -495,14 +483,8 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) return undefined; } - // Reject proposals with invalid signatures - if (!proposer) { - this.log.warn(`Received checkpoint proposal with invalid signature for slot ${slotNumber}`); - return undefined; - } - // Ignore proposals from ourselves (may happen in HA setups) - if (this.getValidatorAddresses().some(addr => addr.equals(proposer))) { + if (proposer && this.getValidatorAddresses().some(addr => addr.equals(proposer))) { this.log.debug(`Ignoring block proposal from self for slot ${slotNumber}`, { proposer: proposer.toString(), slotNumber, @@ -510,44 +492,31 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) return undefined; } - // Validate fee asset price modifier is within allowed range - if (!validateFeeAssetPriceModifier(proposal.feeAssetPriceModifier)) { - this.log.warn( - `Received checkpoint proposal with invalid feeAssetPriceModifier ${proposal.feeAssetPriceModifier} for slot ${slotNumber}`, - ); - return undefined; - } - - // Check that I have any address in current committee before attesting + // Check that I have any address in the committee where this checkpoint will land before attesting const inCommittee = await this.epochCache.filterInCommittee(slotNumber, this.getValidatorAddresses()); const partOfCommittee = inCommittee.length > 0; const proposalInfo = { slotNumber, archive: proposal.archive.toString(), - proposer: proposer.toString(), + proposer: proposer?.toString(), }; this.log.info(`Received checkpoint proposal for slot ${slotNumber}`, { ...proposalInfo, fishermanMode: this.config.fishermanMode || false, }); - // Validate the checkpoint proposal before attesting (unless skipCheckpointProposalValidation is set) + // Validate the checkpoint proposal and upload blobs (unless skipCheckpointProposalValidation is set) if (this.config.skipCheckpointProposalValidation) { this.log.warn(`Skipping checkpoint proposal validation for slot ${slotNumber}`, proposalInfo); } else { - const validationResult = await this.validateCheckpointProposal(proposal, proposalInfo); + const validationResult = await this.proposalHandler.handleCheckpointProposal(proposal, proposalInfo); if (!validationResult.isValid) { this.log.warn(`Checkpoint proposal validation failed: ${validationResult.reason}`, proposalInfo); return undefined; } } - // Upload blobs to filestore if we can (fire and forget) - if (this.blobClient.canUpload()) { - void this.uploadBlobsForCheckpoint(proposal, proposalInfo); - } - // Check that I have any address in current committee before attesting // In fisherman mode, we still create attestations for validation even if not in committee if (!partOfCommittee && !this.config.fishermanMode) { @@ -642,201 +611,6 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) return attestations; } - /** - * Validates a checkpoint proposal by building the full checkpoint and comparing it with the proposal. - * @returns Validation result with isValid flag and reason if invalid. - */ - private async validateCheckpointProposal( - proposal: CheckpointProposalCore, - proposalInfo: LogData, - ): Promise<{ isValid: true } | { isValid: false; reason: string }> { - const slot = proposal.slotNumber; - - // Timeout block syncing at the start of the next slot - const config = this.checkpointsBuilder.getConfig(); - const nextSlotTimestampSeconds = Number(getTimestampForSlot(SlotNumber(slot + 1), config)); - const timeoutSeconds = Math.max(1, nextSlotTimestampSeconds - Math.floor(this.dateProvider.now() / 1000)); - - // Wait for last block to sync by archive - let lastBlockHeader: BlockHeader | undefined; - try { - lastBlockHeader = await retryUntil( - async () => { - await this.blockSource.syncImmediate(); - return this.blockSource.getBlockHeaderByArchive(proposal.archive); - }, - `waiting for block with archive ${proposal.archive.toString()} for slot ${slot}`, - timeoutSeconds, - 0.5, - ); - } catch (err) { - if (err instanceof TimeoutError) { - this.log.warn(`Timed out waiting for block with archive matching checkpoint proposal`, proposalInfo); - return { isValid: false, reason: 'last_block_not_found' }; - } - this.log.error(`Error fetching last block for checkpoint proposal`, err, proposalInfo); - return { isValid: false, reason: 'block_fetch_error' }; - } - - if (!lastBlockHeader) { - this.log.warn(`Last block not found for checkpoint proposal`, proposalInfo); - return { isValid: false, reason: 'last_block_not_found' }; - } - - // Get all full blocks for the slot and checkpoint - const blocks = await this.blockSource.getBlocksForSlot(slot); - if (blocks.length === 0) { - this.log.warn(`No blocks found for slot ${slot}`, proposalInfo); - return { isValid: false, reason: 'no_blocks_for_slot' }; - } - - // Ensure the last block for this slot matches the archive in the checkpoint proposal - if (!blocks.at(-1)?.archive.root.equals(proposal.archive)) { - this.log.warn(`Last block archive mismatch for checkpoint proposal`, proposalInfo); - return { isValid: false, reason: 'last_block_archive_mismatch' }; - } - - this.log.debug(`Found ${blocks.length} blocks for slot ${slot}`, { - ...proposalInfo, - blockNumbers: blocks.map(b => b.number), - }); - - // Get checkpoint constants from first block - const firstBlock = blocks[0]; - const constants = this.extractCheckpointConstants(firstBlock); - const checkpointNumber = firstBlock.checkpointNumber; - - // Get L1-to-L2 messages for this checkpoint - const l1ToL2Messages = await this.l1ToL2MessageSource.getL1ToL2Messages(checkpointNumber); - - // Collect the out hashes of all the checkpoints before this one in the same epoch - const epoch = getEpochAtSlot(slot, this.epochCache.getL1Constants()); - const previousCheckpointOutHashes = (await this.blockSource.getCheckpointsDataForEpoch(epoch)) - .filter(c => c.checkpointNumber < checkpointNumber) - .map(c => c.checkpointOutHash); - - // Fork world state at the block before the first block - const parentBlockNumber = BlockNumber(firstBlock.number - 1); - const fork = await this.worldState.fork(parentBlockNumber); - - try { - // Create checkpoint builder with all existing blocks - const checkpointBuilder = await this.checkpointsBuilder.openCheckpoint( - checkpointNumber, - constants, - proposal.feeAssetPriceModifier, - l1ToL2Messages, - previousCheckpointOutHashes, - fork, - blocks, - this.log.getBindings(), - ); - - // Complete the checkpoint to get computed values - const computedCheckpoint = await checkpointBuilder.completeCheckpoint(); - - // Compare checkpoint header with proposal - if (!computedCheckpoint.header.equals(proposal.checkpointHeader)) { - this.log.warn(`Checkpoint header mismatch`, { - ...proposalInfo, - computed: computedCheckpoint.header.toInspect(), - proposal: proposal.checkpointHeader.toInspect(), - }); - return { isValid: false, reason: 'checkpoint_header_mismatch' }; - } - - // Compare archive root with proposal - if (!computedCheckpoint.archive.root.equals(proposal.archive)) { - this.log.warn(`Archive root mismatch`, { - ...proposalInfo, - computed: computedCheckpoint.archive.root.toString(), - proposal: proposal.archive.toString(), - }); - return { isValid: false, reason: 'archive_mismatch' }; - } - - // Check that the accumulated epoch out hash matches the value in the proposal. - // The epoch out hash is the accumulated hash of all checkpoint out hashes in the epoch. - const checkpointOutHash = computedCheckpoint.getCheckpointOutHash(); - const computedEpochOutHash = accumulateCheckpointOutHashes([...previousCheckpointOutHashes, checkpointOutHash]); - const proposalEpochOutHash = proposal.checkpointHeader.epochOutHash; - if (!computedEpochOutHash.equals(proposalEpochOutHash)) { - this.log.warn(`Epoch out hash mismatch`, { - proposalEpochOutHash: proposalEpochOutHash.toString(), - computedEpochOutHash: computedEpochOutHash.toString(), - checkpointOutHash: checkpointOutHash.toString(), - previousCheckpointOutHashes: previousCheckpointOutHashes.map(h => h.toString()), - ...proposalInfo, - }); - return { isValid: false, reason: 'out_hash_mismatch' }; - } - - // Final round of validations on the checkpoint, just in case. - try { - validateCheckpoint(computedCheckpoint, { - rollupManaLimit: this.checkpointsBuilder.getConfig().rollupManaLimit, - maxDABlockGas: this.config.validateMaxDABlockGas, - maxL2BlockGas: this.config.validateMaxL2BlockGas, - maxTxsPerBlock: this.config.validateMaxTxsPerBlock, - maxTxsPerCheckpoint: this.config.validateMaxTxsPerCheckpoint, - }); - } catch (err) { - this.log.warn(`Checkpoint validation failed: ${err}`, proposalInfo); - return { isValid: false, reason: 'checkpoint_validation_failed' }; - } - - this.log.verbose(`Checkpoint proposal validation successful for slot ${slot}`, proposalInfo); - return { isValid: true }; - } finally { - await fork.close(); - } - } - - /** - * Extract checkpoint global variables from a block. - */ - private extractCheckpointConstants(block: L2Block): CheckpointGlobalVariables { - const gv = block.header.globalVariables; - return { - chainId: gv.chainId, - version: gv.version, - slotNumber: gv.slotNumber, - timestamp: gv.timestamp, - coinbase: gv.coinbase, - feeRecipient: gv.feeRecipient, - gasFees: gv.gasFees, - }; - } - - /** - * Uploads blobs for a checkpoint to the filestore (fire and forget). - */ - protected async uploadBlobsForCheckpoint(proposal: CheckpointProposalCore, proposalInfo: LogData): Promise { - try { - const lastBlockHeader = await this.blockSource.getBlockHeaderByArchive(proposal.archive); - if (!lastBlockHeader) { - this.log.warn(`Failed to get last block header for blob upload`, proposalInfo); - return; - } - - const blocks = await this.blockSource.getBlocksForSlot(proposal.slotNumber); - if (blocks.length === 0) { - this.log.warn(`No blocks found for blob upload`, proposalInfo); - return; - } - - const blobFields = blocks.flatMap(b => b.toBlobFields()); - const blobs: Blob[] = await getBlobsPerL1Block(blobFields); - await this.blobClient.sendBlobsToFilestore(blobs); - this.log.debug(`Uploaded ${blobs.length} blobs to filestore for checkpoint at slot ${proposal.slotNumber}`, { - ...proposalInfo, - numBlobs: blobs.length, - }); - } catch (err) { - this.log.warn(`Failed to upload blobs for checkpoint: ${err}`, proposalInfo); - } - } - private slashInvalidBlock(proposal: BlockProposal) { const proposer = proposal.getSender();