From 983038ed90f20e16bdf6850e457a74e05926202e Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 27 May 2026 13:18:30 +0200 Subject: [PATCH 01/27] fix(archiver): skip descendants of invalid-attestations checkpoints (#23502) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Motivation `archiver/src/modules/l1_synchronizer.ts` skipped checkpoints with insufficient/invalid attestations under the assumption that the next proposer would invalidate them before publishing. When that assumption was violated — i.e., proposer P2 published a valid-attestations checkpoint that extended P1's invalid one — the archiver hit `InitialCheckpointNumberNotSequentialError` in `block_store.addCheckpoints`, the catch handler rolled back the L1 sync point, and the next poll re-fetched the same range and re-threw. The archiver looped indefinitely. The protocol already defines `OffenseType.PROPOSED_DESCENDANT_OF_CHECKPOINT_WITH_INVALID_ATTESTATIONS` for exactly this case but the slasher couldn't see valid-attestations descendants because the archiver threw before emitting any event. ### Human Note This is particularly relevant under pipelining. Attestors now attest to a checkpoint _before_ the previous one is pushed to L1, so they can be inadvertently attesting to a checkpoint built on top of one that became invalid as it was published to the rollup the contract with wrong attestations. So an honest attestor could get slashed if the proposer was malicious. ## Approach In the synchronizer, persist rejected ancestors in the block store keyed by archive root. On each new checkpoint, before attestation validation, compare its `header.lastArchiveRoot` against the persisted set — if it matches, skip the checkpoint as a descendant of an invalid ancestor and emit a new `L2BlockSourceEvents.CheckpointBuiltOnInvalidAncestorDetected` event with enough metadata to resolve the proposer. The slasher's `AttestationsBlockWatcher` is fixed to slash the proposer (not the attestors) under the new event. Fixes A-1072 --- .../archiver/src/archiver-sync.test.ts | 90 +++++++- yarn-project/archiver/src/errors.ts | 6 +- .../archiver/src/modules/l1_synchronizer.ts | 156 ++++++++++---- .../archiver/src/store/block_store.test.ts | 83 ++++++++ .../archiver/src/store/block_store.ts | 133 +++++++++++- .../aztec-node/src/aztec-node/server.ts | 2 +- .../epochs_invalidate_block.parallel.test.ts | 195 +++++++++++++++++- .../src/branded-types/checkpoint_number.ts | 10 + yarn-project/sequencer-client/src/config.ts | 7 + .../src/sequencer/checkpoint_proposal_job.ts | 7 + .../attestations_block_watcher.test.ts | 189 ++++++++++++----- .../watchers/attestations_block_watcher.ts | 142 +++++++------ .../stdlib/src/block/l2_block_source.ts | 21 ++ yarn-project/stdlib/src/interfaces/configs.ts | 6 + 14 files changed, 875 insertions(+), 172 deletions(-) diff --git a/yarn-project/archiver/src/archiver-sync.test.ts b/yarn-project/archiver/src/archiver-sync.test.ts index f4898e6791f8..626bae272e39 100644 --- a/yarn-project/archiver/src/archiver-sync.test.ts +++ b/yarn-project/archiver/src/archiver-sync.test.ts @@ -689,6 +689,13 @@ describe('Archiver Sync', () => { const invalidCheckpointDetectedSpy = jest.fn(); archiver.events.on(L2BlockSourceEvents.InvalidAttestationsCheckpointDetected, invalidCheckpointDetectedSpy); + // And another spy for DescendentOfInvalidAttestationsCheckpointDetected, which fires only for a + // checkpoint with VALID attestations that builds on a rejected ancestor. CP3 here has invalid + // attestations of its own, so it is caught by the attestation check first and should never + // reach the descendant path — this spy must not fire in this test. + const descendantOfInvalidSpy = jest.fn(); + archiver.events.on(L2BlockSourceEvents.DescendentOfInvalidAttestationsCheckpointDetected, descendantOfInvalidSpy); + // Add valid checkpoint 1 with correct attestations const { checkpoint: cp1 } = await fake.addCheckpoint(CheckpointNumber(1), { l1BlockNumber: 70n, @@ -780,21 +787,24 @@ describe('Archiver Sync', () => { expect(validationStatus.checkpoint.checkpointNumber).toEqual(2); expect(validationStatus.checkpoint.archive.toString()).toEqual(badCp2b.archive.root.toString()); - // Check that event was also emitted for bad CP3 + // CP3 has invalid attestations of its own, so it is caught by the attestation check (which + // runs before the descendant-of-invalid check) and surfaced as an + // InvalidAttestationsCheckpointDetected event — NOT a descendant event — even though it also + // builds on the rejected bad CP2b. + expect(descendantOfInvalidSpy).not.toHaveBeenCalled(); + + // Should have been called 3 times for invalid attestations: bad CP2, bad CP2b, bad CP3 + expect(invalidCheckpointDetectedSpy).toHaveBeenCalledTimes(3); expect(invalidCheckpointDetectedSpy).toHaveBeenCalledWith( expect.objectContaining({ type: L2BlockSourceEvents.InvalidAttestationsCheckpointDetected, validationResult: expect.objectContaining({ valid: false, - reason: 'invalid-attestation', checkpoint: expect.objectContaining({ checkpointNumber: 3 }), }), }), ); - // Should have been called 3 times: bad CP2, bad CP2b, bad CP3 - expect(invalidCheckpointDetectedSpy).toHaveBeenCalledTimes(3); - // Now recover: remove bad checkpoints and add good CP2 and CP3 with valid attestations // Good checkpoints have messages that the archiver will validate logger.warn('Fourth sync: adding good CP2 and CP3 with correct attestations'); @@ -831,6 +841,76 @@ describe('Archiver Sync', () => { // With a valid pending chain validation status expect(await archiver.getPendingChainValidationStatus()).toEqual(expect.objectContaining({ valid: true })); }, 15_000); + + it('skips a valid-attestations checkpoint that builds on a rejected ancestor', async () => { + // Regression for the archiver "non-consecutive checkpoint" retry loop: when a checkpoint + // with insufficient/invalid attestations is followed by a valid-attestations descendant, + // addCheckpoints used to throw InitialCheckpointNumberNotSequentialError and loop on the + // catch handler's L1-sync-point rollback. Now the descendant is detected, skipped, and + // surfaced via DescendentOfInvalidAttestationsCheckpointDetected. + expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(0)); + + fake.setTargetCommitteeSize(3); + const signers = times(3, Secp256k1Signer.random); + const committee = signers.map(s => s.address); + epochCache.getCommitteeForEpoch.mockResolvedValue({ committee, seed: 0n } as EpochCommitteeInfo); + + const descendantOfInvalidSpy = jest.fn(); + archiver.events.on(L2BlockSourceEvents.DescendentOfInvalidAttestationsCheckpointDetected, descendantOfInvalidSpy); + + // Valid CP1 + const { checkpoint: cp1 } = await fake.addCheckpoint(CheckpointNumber(1), { + l1BlockNumber: 70n, + messagesL1BlockNumber: 50n, + numL1ToL2Messages: 3, + signers, + }); + const cp1Archive = cp1.blocks.at(-1)!.archive; + + // Bad CP2 (insufficient attestations — random signers not in committee) + const badSigners = times(3, Secp256k1Signer.random); + const { checkpoint: badCp2 } = await fake.addCheckpoint(CheckpointNumber(2), { + l1BlockNumber: 80n, + numL1ToL2Messages: 0, + signers: badSigners, + previousArchive: cp1Archive, + }); + + // Valid-attestations CP3 chained from bad CP2: this is the case that used to wedge the + // synchronizer. + const { checkpoint: validCp3 } = await fake.addCheckpoint(CheckpointNumber(3), { + l1BlockNumber: 82n, + numL1ToL2Messages: 0, + signers, + previousArchive: badCp2.blocks.at(-1)!.archive, + }); + + fake.setL1BlockNumber(85n); + await archiver.syncImmediate(); + + // Archiver should have stayed at CP1 (skipped both CP2 and CP3) without throwing. + expect(await archiver.getCheckpointNumber()).toEqual(CheckpointNumber(1)); + + // The descendant event should have fired for CP3 with the bad CP2 ancestor. + expect(descendantOfInvalidSpy).toHaveBeenCalledWith( + expect.objectContaining({ + type: L2BlockSourceEvents.DescendentOfInvalidAttestationsCheckpointDetected, + checkpoint: expect.objectContaining({ + checkpointNumber: 3, + archive: validCp3.archive.root, + }), + ancestorArchiveRoot: badCp2.archive.root, + ancestorCheckpointNumber: 2, + }), + ); + expect(descendantOfInvalidSpy).toHaveBeenCalledTimes(1); + + // The rejected entries should persist in the store, keyed by their own archive roots. + const rejectedBad = await archiverStore.blocks.getRejectedCheckpointByArchiveRoot(badCp2.archive.root); + const rejectedValid = await archiverStore.blocks.getRejectedCheckpointByArchiveRoot(validCp3.archive.root); + expect(rejectedBad).toBeDefined(); + expect(rejectedValid).toBeDefined(); + }, 15_000); }); describe('reorg handling', () => { diff --git a/yarn-project/archiver/src/errors.ts b/yarn-project/archiver/src/errors.ts index 64fae2aa69dd..6de0234d9585 100644 --- a/yarn-project/archiver/src/errors.ts +++ b/yarn-project/archiver/src/errors.ts @@ -31,13 +31,13 @@ export class InitialCheckpointNumberNotSequentialError extends Error { export class CheckpointNumberNotSequentialError extends Error { constructor( - newCheckpointNumber: CheckpointNumber, - previous: CheckpointNumber | undefined, + public readonly newCheckpointNumber: CheckpointNumber, + public readonly previousCheckpointNumber: CheckpointNumber | undefined, source?: 'proposed' | 'confirmed', ) { const qualifier = source ? `${source} ` : ''; super( - `Cannot insert new checkpoint ${newCheckpointNumber} given previous ${qualifier}checkpoint number is ${previous ?? 'undefined'}`, + `Cannot insert new checkpoint ${newCheckpointNumber} given previous ${qualifier}checkpoint number is ${previousCheckpointNumber ?? 'undefined'}`, ); this.name = 'CheckpointNumberNotSequentialError'; } diff --git a/yarn-project/archiver/src/modules/l1_synchronizer.ts b/yarn-project/archiver/src/modules/l1_synchronizer.ts index a0daa21d7c83..3d12a9d58a4a 100644 --- a/yarn-project/archiver/src/modules/l1_synchronizer.ts +++ b/yarn-project/archiver/src/modules/l1_synchronizer.ts @@ -17,7 +17,7 @@ import { count } from '@aztec/foundation/string'; import { DateProvider, Timer, elapsed } from '@aztec/foundation/timer'; import { isDefined, isErrorClass } from '@aztec/foundation/types'; import { type ArchiverEmitter, L2BlockSourceEvents, type ValidateCheckpointResult } from '@aztec/stdlib/block'; -import { Checkpoint, PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; +import { Checkpoint, type CheckpointData, PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; import { type L1RollupConstants, getEpochAtSlot, getSlotAtNextL1Block } from '@aztec/stdlib/epoch-helpers'; import { computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging'; import type { CoordinationSignatureContext } from '@aztec/stdlib/p2p'; @@ -32,6 +32,7 @@ import { retrieveL1ToL2Messages, retrievedToPublishedCheckpoint, } from '../l1/data_retrieval.js'; +import type { RejectedCheckpoint } from '../store/block_store.js'; import { type ArchiverDataStores, getArchiverSynchPoint } from '../store/data_stores.js'; import type { L2TipsCache } from '../store/l2_tips_cache.js'; import { MessageStoreError } from '../store/message_store.js'; @@ -46,13 +47,15 @@ type RollupStatus = { pendingCheckpointNumber: CheckpointNumber; pendingArchive: string; validationResult: ValidateCheckpointResult | undefined; + /** Last valid checkpoint observed on L1 and synced on this iteration */ lastRetrievedCheckpoint?: PublishedCheckpoint; - lastL1BlockWithCheckpoint?: bigint; + /** Last checkpoint observed on L1 across both valid and rejected entries on this iteration */ + lastSeenCheckpoint?: PublishedCheckpoint; }; /** * Handles L1 synchronization for the archiver. - * Responsible for fetching checkpoints, L1→L2 messages, and handling L1 reorgs. + * Responsible for fetching checkpoints, L1 to L2 messages, and handling L1 reorgs. */ export class ArchiverL1Synchronizer implements Traceable { private l1BlockNumber: bigint | undefined; @@ -202,18 +205,10 @@ export class ArchiverL1Synchronizer implements Traceable { currentL1Timestamp, ); - // If the last checkpoint we processed had an invalid attestation, we manually advance the L1 syncpoint - // past it, since otherwise we'll keep downloading it and reprocessing it on every iteration until - // we get a valid checkpoint to advance the syncpoint. - if (!rollupStatus.validationResult?.valid && rollupStatus.lastL1BlockWithCheckpoint !== undefined) { - await this.stores.blocks.setSynchedL1BlockNumber(rollupStatus.lastL1BlockWithCheckpoint); - } - // And lastly we check if we are missing any checkpoints behind us due to a possible L1 reorg. // We only do this if rollup cant prune on the next submission. Otherwise we will end up - // re-syncing the checkpoints we have just unwound above. We also dont do this if the last checkpoint is invalid, - // since the archiver will rightfully refuse to sync up to it. - if (!rollupCanPrune && rollupStatus.validationResult?.valid) { + // re-syncing the checkpoints we have just unwound above. + if (!rollupCanPrune) { await this.checkForNewCheckpointsBeforeL1SyncPoint(rollupStatus, blocksSynchedTo, currentL1BlockNumber); } @@ -796,7 +791,7 @@ export class ArchiverL1Synchronizer implements Traceable { let searchStartBlock: bigint = blocksSynchedTo; let searchEndBlock: bigint = blocksSynchedTo; let lastRetrievedCheckpoint: PublishedCheckpoint | undefined; - let lastL1BlockWithCheckpoint: bigint | undefined = undefined; + let lastSeenCheckpoint: PublishedCheckpoint | undefined; do { [searchStartBlock, searchEndBlock] = this.nextRange(searchEndBlock, currentL1BlockNumber); @@ -865,6 +860,9 @@ export class ArchiverL1Synchronizer implements Traceable { // Now loop through all checkpoints and validate their attestations for (const published of publishedCheckpoints) { + // Check the attestations uploaded by the publisher to L1 are correct + // Rollup contract does not validate attestations to save on gas, so this + // falls on the nodes to verify offchain and skip those checkpoints. const validationResult = this.config.skipValidateCheckpointAttestations ? { valid: true as const } : await validateCheckpointAttestations( @@ -875,17 +873,29 @@ export class ArchiverL1Synchronizer implements Traceable { this.log, ); - // Only update the validation result if it has changed, so we can keep track of the first invalid checkpoint + // Also skip the checkpoint if it builds on a previously-rejected ancestor. Without + // this, addCheckpoints would throw InitialCheckpointNumberNotSequentialError when the + // ancestor was skipped earlier (e.g. due to invalid attestations), the catch handler + // would roll back the L1 sync point, and the next iteration would re-fetch and re-throw. + const rejectedAncestor = await this.stores.blocks.getRejectedCheckpointByArchiveRoot( + published.checkpoint.header.lastArchiveRoot, + ); + + // Update the validation result if it has changed, so we can keep track of the first invalid checkpoint // in case there is a sequence of more than one invalid checkpoint, as we need to invalidate the first one. // There is an exception though: if a checkpoint is invalidated and replaced with another invalid checkpoint, // we need to update the validation result, since we need to be able to invalidate the new one. // See test 'chain progresses if an invalid checkpoint is invalidated with an invalid one' for more info. - if ( - rollupStatus.validationResult?.valid !== validationResult.valid || - (!rollupStatus.validationResult.valid && - !validationResult.valid && - rollupStatus.validationResult.checkpoint.checkpointNumber === validationResult.checkpoint.checkpointNumber) - ) { + // Do not update the validation result if there is a rejected ancestor, since in that case we want to keep the + // original invalidation, as the new checkpoint is extending from a previous invalid one. + const validStatusChanged = rollupStatus.validationResult?.valid !== validationResult.valid; + const invalidStatusWithSameCheckpointNumber = + !validationResult.valid && + rollupStatus.validationResult && + !rollupStatus.validationResult.valid && + rollupStatus.validationResult.checkpoint.checkpointNumber === validationResult.checkpoint.checkpointNumber; + + if (!rejectedAncestor && (validStatusChanged || invalidStatusWithSameCheckpointNumber)) { rollupStatus.validationResult = validationResult; } @@ -902,9 +912,55 @@ export class ArchiverL1Synchronizer implements Traceable { validationResult, }); - // We keep consuming checkpoints if we find an invalid one, since we do not listen for CheckpointInvalidated events - // We just pretend the invalid ones are not there and keep consuming the next checkpoints - // Note that this breaks if the committee ever attests to a descendant of an invalid checkpoint + // Persist a rejected-ancestor entry so any later checkpoint that builds on this one + // is detected and skipped (rather than tripping the addCheckpoints consecutive-number + // check and causing the sync point to roll back in a loop). + await this.stores.blocks.addRejectedCheckpoint({ + checkpointNumber: published.checkpoint.number, + archiveRoot: published.checkpoint.archive.root, + parentArchiveRoot: published.checkpoint.header.lastArchiveRoot, + slotNumber: published.checkpoint.header.slotNumber, + l1: published.l1, + reason: 'invalid-attestations' as const, + }); + + continue; + } + + if (rejectedAncestor) { + const descendantInfo = published.checkpoint.toCheckpointInfo(); + this.log.warn( + `Skipping checkpoint ${published.checkpoint.number} as it is a descendant of ` + + `rejected checkpoint ${rejectedAncestor.checkpointNumber} (${rejectedAncestor.reason})`, + { + checkpointNumber: published.checkpoint.number, + checkpointHash: published.checkpoint.hash(), + l1BlockNumber: published.l1.blockNumber, + l1BlockHash: published.l1.blockHash, + ancestorCheckpointNumber: rejectedAncestor.checkpointNumber, + ancestorArchiveRoot: rejectedAncestor.archiveRoot.toString(), + ancestorReason: rejectedAncestor.reason, + }, + ); + + this.events.emit(L2BlockSourceEvents.DescendentOfInvalidAttestationsCheckpointDetected, { + type: L2BlockSourceEvents.DescendentOfInvalidAttestationsCheckpointDetected, + checkpoint: descendantInfo, + ancestorArchiveRoot: rejectedAncestor.archiveRoot, + ancestorCheckpointNumber: rejectedAncestor.checkpointNumber, + }); + + // Persist this chainpoint as rejected as well, so we can construct a chain of + // skipped checkpoints starting from the first one with invalid attestations. + await this.stores.blocks.addRejectedCheckpoint({ + checkpointNumber: published.checkpoint.number, + archiveRoot: published.checkpoint.archive.root, + parentArchiveRoot: published.checkpoint.header.lastArchiveRoot, + slotNumber: published.checkpoint.header.slotNumber, + l1: published.l1, + reason: 'descends-from-invalid-attestations' as const, + }); + continue; } @@ -1005,12 +1061,21 @@ export class ArchiverL1Synchronizer implements Traceable { const previousCheckpoint = previousCheckpointNumber ? await this.stores.blocks.getCheckpointData(CheckpointNumber(previousCheckpointNumber)) : undefined; - const updatedL1SyncPoint = previousCheckpoint?.l1.blockNumber ?? this.l1Constants.l1StartBlock; + const lastFinalizedCheckpoint = await this.stores.blocks.getCheckpointData( + await this.stores.blocks.getFinalizedCheckpointNumber(), + ); + const updatedL1SyncPoint = + previousCheckpoint?.l1.blockNumber ?? + lastFinalizedCheckpoint?.l1.blockNumber ?? + this.l1Constants.l1StartBlock; await this.stores.blocks.setSynchedL1BlockNumber(updatedL1SyncPoint); this.log.warn( `Attempting to insert checkpoint ${newCheckpointNumber} with previous block ${previousCheckpointNumber}. Rolling back L1 sync point to ${updatedL1SyncPoint} to try and fetch the missing blocks.`, { previousCheckpointNumber, + previousCheckpoint: previousCheckpoint?.header.toInspect(), + lastFinalizedCheckpoint: lastFinalizedCheckpoint?.header.toInspect(), + l1StartBlock: this.l1Constants.l1StartBlock, newCheckpointNumber, updatedL1SyncPoint, }, @@ -1031,13 +1096,13 @@ export class ArchiverL1Synchronizer implements Traceable { }); } lastRetrievedCheckpoint = validCheckpoints.at(-1) ?? lastRetrievedCheckpoint; - lastL1BlockWithCheckpoint = calldataCheckpoints.at(-1)?.l1.blockNumber ?? lastL1BlockWithCheckpoint; + lastSeenCheckpoint = publishedCheckpoints.at(-1) ?? lastSeenCheckpoint; } while (searchEndBlock < currentL1BlockNumber); // Important that we update AFTER inserting the blocks. await updateProvenCheckpoint(); - return { ...rollupStatus, lastRetrievedCheckpoint, lastL1BlockWithCheckpoint }; + return { ...rollupStatus, lastRetrievedCheckpoint, lastSeenCheckpoint }; } /** @@ -1137,35 +1202,38 @@ export class ArchiverL1Synchronizer implements Traceable { blocksSynchedTo: bigint, currentL1BlockNumber: bigint, ): Promise { - const { lastRetrievedCheckpoint, pendingCheckpointNumber } = status; - // Compare the last checkpoint we have (either retrieved in this round or loaded from store) with what the - // rollup contract told us was the latest one (pinned at the currentL1BlockNumber). + const { lastSeenCheckpoint, pendingCheckpointNumber } = status; + // Compare the last checkpoint (valid or not) we have (either retrieved in this round or loaded from store) + // with what the rollup contract told us was the latest one (pinned at the currentL1BlockNumber). const latestLocalCheckpointNumber = - lastRetrievedCheckpoint?.checkpoint.number ?? (await this.stores.blocks.getLatestCheckpointNumber()); + lastSeenCheckpoint?.checkpoint.number ?? + CheckpointNumber.max( + await this.stores.blocks.getLatestCheckpointNumber(), + await this.stores.blocks.getLatestRejectedCheckpointNumber(), + ) ?? + CheckpointNumber.ZERO; + if (latestLocalCheckpointNumber < pendingCheckpointNumber) { // Here we have consumed all logs until the `currentL1Block` we pinned at the beginning of the archiver loop, // but still haven't reached the pending checkpoint according to the call to the rollup contract. // We suspect an L1 reorg that added checkpoints *behind* us. If that is the case, it must have happened between // the last checkpoint we saw and the current one, so we reset the last synched L1 block number. In the edge case // we don't have one, we go back 2 L1 epochs, which is the deepest possible reorg (assuming Casper is working). - let latestLocalCheckpointArchive: string | undefined = undefined; - let targetL1BlockNumber = maxBigint(currentL1BlockNumber - 64n, 0n); - if (lastRetrievedCheckpoint) { - latestLocalCheckpointArchive = lastRetrievedCheckpoint.checkpoint.archive.root.toString(); - targetL1BlockNumber = lastRetrievedCheckpoint.l1.blockNumber; - } else if (latestLocalCheckpointNumber > 0) { - const checkpoint = await this.stores.blocks - .getRangeOfCheckpoints(latestLocalCheckpointNumber, 1) - .then(([c]) => c); - latestLocalCheckpointArchive = checkpoint.archive.root.toString(); - targetL1BlockNumber = checkpoint.l1.blockNumber; - } + const latestLocalCheckpoint: PublishedCheckpoint | CheckpointData | RejectedCheckpoint | undefined = + lastSeenCheckpoint ?? + (await this.stores.blocks.getCheckpointData(latestLocalCheckpointNumber)) ?? + (await this.stores.blocks.getRejectedCheckpointByNumber(latestLocalCheckpointNumber)); + + const targetL1BlockNumber = + latestLocalCheckpoint?.l1.blockNumber ?? + maxBigint(currentL1BlockNumber - 64n, this.l1Constants.l1StartBlock, 0n); + this.log.warn( `Failed to reach checkpoint ${pendingCheckpointNumber} at ${currentL1BlockNumber} (latest is ${latestLocalCheckpointNumber}). ` + `Rolling back last synched L1 block number to ${targetL1BlockNumber}.`, { latestLocalCheckpointNumber, - latestLocalCheckpointArchive, + latestLocalCheckpointL1: latestLocalCheckpoint?.l1, blocksSynchedTo, currentL1BlockNumber, ...status, diff --git a/yarn-project/archiver/src/store/block_store.test.ts b/yarn-project/archiver/src/store/block_store.test.ts index 29c6126a1ec0..d093bbf30784 100644 --- a/yarn-project/archiver/src/store/block_store.test.ts +++ b/yarn-project/archiver/src/store/block_store.test.ts @@ -2833,4 +2833,87 @@ describe('BlockStore', () => { expect(await blockStore.getBlock({ number: BlockNumber(2) })).toBeUndefined(); }); }); + + describe('rejected checkpoints', () => { + const makeEntry = (overrides: { archiveRoot?: Fr; l1BlockNumber?: number; checkpointNumber?: number } = {}) => ({ + checkpointNumber: CheckpointNumber(overrides.checkpointNumber ?? 1), + archiveRoot: overrides.archiveRoot ?? Fr.random(), + parentArchiveRoot: Fr.random(), + slotNumber: SlotNumber(1), + l1: makeL1PublishedData(overrides.l1BlockNumber ?? 100), + reason: 'invalid-attestations' as const, + }); + + it('returns an empty result when no rejected checkpoints have been recorded', async () => { + expect(await blockStore.getRejectedCheckpointByArchiveRoot(Fr.random())).toBeUndefined(); + expect(await blockStore.getLatestRejectedCheckpointNumber()).toEqual( + CheckpointNumber(INITIAL_CHECKPOINT_NUMBER - 1), + ); + }); + + it('round-trips an added rejected entry', async () => { + const entry = makeEntry(); + await blockStore.addRejectedCheckpoint(entry); + + const stored = await blockStore.getRejectedCheckpointByArchiveRoot(entry.archiveRoot); + expect(stored).toBeDefined(); + expect(stored!.checkpointNumber).toEqual(entry.checkpointNumber); + expect(stored!.archiveRoot.toString()).toEqual(entry.archiveRoot.toString()); + expect(stored!.parentArchiveRoot.toString()).toEqual(entry.parentArchiveRoot.toString()); + expect(stored!.slotNumber).toEqual(entry.slotNumber); + expect(stored!.l1.blockNumber).toEqual(entry.l1.blockNumber); + expect(stored!.l1.blockHash).toEqual(entry.l1.blockHash); + expect(stored!.reason).toEqual(entry.reason); + }); + + it('updates an existing entry when re-added with the same archive root', async () => { + const archiveRoot = Fr.random(); + await blockStore.addRejectedCheckpoint(makeEntry({ archiveRoot, l1BlockNumber: 100 })); + await blockStore.addRejectedCheckpoint(makeEntry({ archiveRoot, l1BlockNumber: 110 })); + + const stored = await blockStore.getRejectedCheckpointByArchiveRoot(archiveRoot); + expect(stored).toBeDefined(); + expect(stored!.l1.blockNumber).toEqual(110n); + }); + + it('preserves the descends-from-invalid-attestations reason', async () => { + const entry = { + ...makeEntry(), + reason: 'descends-from-invalid-attestations' as const, + }; + await blockStore.addRejectedCheckpoint(entry); + const stored = await blockStore.getRejectedCheckpointByArchiveRoot(entry.archiveRoot); + expect(stored!.reason).toEqual('descends-from-invalid-attestations'); + }); + + it('returns the latest rejected checkpoint number across all entries', async () => { + await blockStore.addRejectedCheckpoint(makeEntry({ checkpointNumber: 1 })); + await blockStore.addRejectedCheckpoint(makeEntry({ checkpointNumber: 5 })); + await blockStore.addRejectedCheckpoint(makeEntry({ checkpointNumber: 3 })); + + expect(await blockStore.getLatestRejectedCheckpointNumber()).toEqual(CheckpointNumber(5)); + }); + + it('looks up a rejected entry by checkpoint number', async () => { + const entry = makeEntry({ checkpointNumber: 7 }); + await blockStore.addRejectedCheckpoint(entry); + + const stored = await blockStore.getRejectedCheckpointByNumber(CheckpointNumber(7)); + expect(stored?.archiveRoot.toString()).toEqual(entry.archiveRoot.toString()); + expect(await blockStore.getRejectedCheckpointByNumber(CheckpointNumber(8))).toBeUndefined(); + }); + + it('removes a rejected entry by archive root', async () => { + const entry = makeEntry({ checkpointNumber: 4 }); + await blockStore.addRejectedCheckpoint(entry); + expect(await blockStore.getRejectedCheckpointByArchiveRoot(entry.archiveRoot)).toBeDefined(); + + await blockStore.removeRejectedCheckpointByArchiveRoot(entry.archiveRoot); + expect(await blockStore.getRejectedCheckpointByArchiveRoot(entry.archiveRoot)).toBeUndefined(); + expect(await blockStore.getRejectedCheckpointByNumber(CheckpointNumber(4))).toBeUndefined(); + expect(await blockStore.getLatestRejectedCheckpointNumber()).toEqual( + CheckpointNumber(INITIAL_CHECKPOINT_NUMBER - 1), + ); + }); + }); }); diff --git a/yarn-project/archiver/src/store/block_store.ts b/yarn-project/archiver/src/store/block_store.ts index 6e68cd2e41c8..0db47a55eafb 100644 --- a/yarn-project/archiver/src/store/block_store.ts +++ b/yarn-project/archiver/src/store/block_store.ts @@ -73,6 +73,41 @@ type BlockStorage = { indexWithinCheckpoint: number; }; +/** Reason a checkpoint was rejected during sync. */ +export type RejectedCheckpointReason = 'invalid-attestations' | 'descends-from-invalid-attestations'; + +/** + * A checkpoint observed on L1 that the archiver decided not to ingest, recorded so that + * any descendant that builds on top of it can also be skipped (rather than throwing + * `InitialCheckpointNumberNotSequentialError` and looping). An entry is dropped via + * {@link BlockStore.removeRejectedCheckpointByArchiveRoot} once a checkpoint with the same + * archive root is later ingested as valid (e.g. it gathered enough attestations), which + * re-enables its descendants. + */ +export type RejectedCheckpoint = { + /** Checkpoint number this entry represents. */ + checkpointNumber: CheckpointNumber; + /** Archive root produced by this rejected checkpoint (matched against descendants' `lastArchiveRoot`). */ + archiveRoot: Fr; + /** `lastArchiveRoot` from this checkpoint's header (the ancestor it built on). */ + parentArchiveRoot: Fr; + /** Slot number of the rejected checkpoint. */ + slotNumber: SlotNumber; + /** L1 publication data for the rejected checkpoint (block number, hash, timestamp). */ + l1: L1PublishedData; + /** Why the entry was recorded. */ + reason: RejectedCheckpointReason; +}; + +type RejectedCheckpointStorage = { + checkpointNumber: number; + archiveRoot: Buffer; + parentArchiveRoot: Buffer; + slotNumber: number; + l1: Buffer; + reason: RejectedCheckpointReason; +}; + /** Checkpoint Storage shared between Checkpoints + Proposed Checkpoints */ type CommonCheckpointStorage = { header: Buffer; @@ -153,6 +188,12 @@ export class BlockStore { /** Index mapping block archive to block number */ #blockArchiveIndex: AztecAsyncMap; + /** Map rejected checkpoints (due to invalid attestations) by archive root */ + #rejectedCheckpoints: AztecAsyncMap; + + /** Index mapping a rejected checkpoint's number to its archive root, so the latest can be read in reverse order */ + #rejectedCheckpointsByNumber: AztecAsyncMap; + #log = createLogger('archiver:block_store'); constructor(private db: AztecAsyncKVStore) { @@ -169,6 +210,8 @@ export class BlockStore { this.#checkpoints = db.openMap('archiver_checkpoints'); this.#slotToCheckpoint = db.openMap('archiver_slot_to_checkpoint'); this.#proposedCheckpoints = db.openMap('archiver_proposed_checkpoints'); + this.#rejectedCheckpoints = db.openMap('archiver_rejected_checkpoints'); + this.#rejectedCheckpointsByNumber = db.openMap('archiver_rejected_checkpoints_by_number'); } /** @@ -340,9 +383,13 @@ export class BlockStore { // Remove proposed checkpoint if it exists, since L1 is authoritative await this.#proposedCheckpoints.delete(checkpoint.checkpoint.number); + + // Drop any rejected entry for this archive root: a checkpoint that was previously rejected + // (e.g. invalid attestations) is now being ingested as valid, so its descendants are allowed. + await this.removeRejectedCheckpointByArchiveRoot(checkpoint.checkpoint.archive.root); } - await this.#lastSynchedL1Block.set(checkpoints[checkpoints.length - 1].l1.blockNumber); + await this.advanceSynchedL1BlockNumber(checkpoints[checkpoints.length - 1].l1.blockNumber); return true; }); } @@ -387,7 +434,7 @@ export class BlockStore { feeAssetPriceModifier: incoming.checkpoint.feeAssetPriceModifier.toString(), }); // Update the sync point to reflect the new L1 block - await this.#lastSynchedL1Block.set(incoming.l1.blockNumber); + await this.advanceSynchedL1BlockNumber(incoming.l1.blockNumber); } return checkpoints.slice(i); } @@ -776,8 +823,12 @@ export class BlockStore { // Remove only this pending entry — remaining entries N+1, N+2, ... stay valid await this.#proposedCheckpoints.delete(proposed.checkpointNumber); + // Drop any rejected entry for this archive root: a checkpoint that was previously rejected + // (e.g. invalid attestations) is now being promoted as valid, so its descendants are allowed. + await this.removeRejectedCheckpointByArchiveRoot(proposed.archive.root); + // Update the last synced L1 block - await this.#lastSynchedL1Block.set(l1.blockNumber); + await this.advanceSynchedL1BlockNumber(l1.blockNumber); }); } @@ -1427,4 +1478,80 @@ export class BlockStore { await this.#pendingChainValidationStatus.delete(); } } + + /** Records a rejected-checkpoint entry, keyed by its own archive root. */ + async addRejectedCheckpoint(entry: RejectedCheckpoint): Promise { + const archiveRootHex = entry.archiveRoot.toString(); + await this.#rejectedCheckpoints.set(archiveRootHex, { + checkpointNumber: entry.checkpointNumber, + archiveRoot: entry.archiveRoot.toBuffer(), + parentArchiveRoot: entry.parentArchiveRoot.toBuffer(), + slotNumber: entry.slotNumber, + l1: entry.l1.toBuffer(), + reason: entry.reason, + }); + await this.#rejectedCheckpointsByNumber.set(entry.checkpointNumber, archiveRootHex); + await this.advanceSynchedL1BlockNumber(entry.l1.blockNumber); + } + + /** Returns the rejected-checkpoint entry with the given archive root, or undefined if not present. */ + async getRejectedCheckpointByArchiveRoot(archiveRoot: Fr): Promise { + const stored = await this.#rejectedCheckpoints.getAsync(archiveRoot.toString()); + return stored ? this.rejectedCheckpointFromStorage(stored) : undefined; + } + + /** Returns the rejected-checkpoint entry recorded for the given checkpoint number, or undefined if none. */ + async getRejectedCheckpointByNumber(checkpointNumber: CheckpointNumber): Promise { + const archiveRootHex = await this.#rejectedCheckpointsByNumber.getAsync(checkpointNumber); + if (archiveRootHex === undefined) { + return undefined; + } + const stored = await this.#rejectedCheckpoints.getAsync(archiveRootHex); + return stored ? this.rejectedCheckpointFromStorage(stored) : undefined; + } + + /** Returns the highest checkpoint number recorded across all rejected entries, or `INITIAL_CHECKPOINT_NUMBER - 1` if none. */ + async getLatestRejectedCheckpointNumber(): Promise { + const [latest] = await toArray(this.#rejectedCheckpointsByNumber.keysAsync({ reverse: true, limit: 1 })); + return CheckpointNumber(latest ?? INITIAL_CHECKPOINT_NUMBER - 1); + } + + /** Removes a rejected-checkpoint entry by its archive root (used when an entry no longer matches L1). */ + async removeRejectedCheckpointByArchiveRoot(archiveRoot: Fr): Promise { + const archiveRootHex = archiveRoot.toString(); + const stored = await this.#rejectedCheckpoints.getAsync(archiveRootHex); + await this.#rejectedCheckpoints.delete(archiveRootHex); + if (stored) { + // Only clear the by-number index if it still points at this archive root, so a distinct + // entry that shares the checkpoint number (e.g. an L1 reorg replacement) is not dropped. + const indexed = await this.#rejectedCheckpointsByNumber.getAsync(stored.checkpointNumber); + if (indexed === archiveRootHex) { + await this.#rejectedCheckpointsByNumber.delete(stored.checkpointNumber); + } + } + } + + /** + * Advances the stored last-synched L1 block number to `l1BlockNumber` only if it is strictly + * greater than the current value. Use this whenever ingesting checkpoint-shaped data so the + * sync pointer never walks backwards on out-of-order writes (e.g. an invalid checkpoint + * advance followed by a valid-checkpoint commit landing at an earlier L1 block). + */ + private async advanceSynchedL1BlockNumber(l1BlockNumber: bigint): Promise { + const current = await this.#lastSynchedL1Block.getAsync(); + if (current === undefined || l1BlockNumber > current) { + await this.#lastSynchedL1Block.set(l1BlockNumber); + } + } + + private rejectedCheckpointFromStorage(stored: RejectedCheckpointStorage): RejectedCheckpoint { + return { + checkpointNumber: CheckpointNumber(stored.checkpointNumber), + archiveRoot: Fr.fromBuffer(stored.archiveRoot), + parentArchiveRoot: Fr.fromBuffer(stored.parentArchiveRoot), + slotNumber: SlotNumber(stored.slotNumber), + l1: L1PublishedData.fromBuffer(stored.l1), + reason: stored.reason, + }; + } } diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index edba020f1e27..88b8e03a6e98 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -779,7 +779,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, AztecNodeDeb config.slashProposeInvalidAttestationsPenalty > 0n || config.slashProposeDescendantOfCheckpointWithInvalidAttestationsPenalty > 0n ) { - attestationsBlockWatcher = new AttestationsBlockWatcher(archiver, epochCache, config); + attestationsBlockWatcher = new AttestationsBlockWatcher(archiver, epochCache, config, log.getBindings()); watchers.push(attestationsBlockWatcher); } } diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts index 431632c757b2..58e3c0609ee0 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts @@ -1,4 +1,4 @@ -import { CalldataRetriever } from '@aztec/archiver'; +import { type Archiver, CalldataRetriever } from '@aztec/archiver'; import type { AztecNodeService } from '@aztec/aztec-node'; import type { AztecAddress } from '@aztec/aztec.js/addresses'; import { NO_WAIT } from '@aztec/aztec.js/contracts'; @@ -8,6 +8,7 @@ import { waitForTx } from '@aztec/aztec.js/node'; import { RollupContract } from '@aztec/ethereum/contracts'; import type { Operator } from '@aztec/ethereum/deploy-aztec-l1-contracts'; import type { ExtendedViemWalletClient, ViemPublicClient, ViemPublicDebugClient } from '@aztec/ethereum/types'; +import { range } from '@aztec/foundation/array'; import { asyncMap } from '@aztec/foundation/async-map'; import { CheckpointNumber, SlotNumber } from '@aztec/foundation/branded-types'; import { times, timesAsync } from '@aztec/foundation/collection'; @@ -17,9 +18,10 @@ import { createLogger } from '@aztec/foundation/log'; import { promiseWithResolvers } from '@aztec/foundation/promise'; import { retryUntil } from '@aztec/foundation/retry'; import { bufferToHex } from '@aztec/foundation/string'; -import { timeoutPromise } from '@aztec/foundation/timer'; +import { executeTimeout, timeoutPromise } from '@aztec/foundation/timer'; import type { TestContract } from '@aztec/noir-test-contracts.js/Test'; import { OffenseType } from '@aztec/slasher'; +import { L2BlockSourceEvents } from '@aztec/stdlib/block'; import { computeQuorum, getTimestampForSlot } from '@aztec/stdlib/epoch-helpers'; import { jest } from '@jest/globals'; @@ -472,6 +474,195 @@ describe('e2e_epochs/epochs_invalidate_block', () => { logger.warn(`Test succeeded '${expect.getState().currentTestName}'`); }); + // P1 publishes a checkpoint with insufficient attestations; the next proposer P2 publishes a + // valid descendant without first invalidating P1. Before the fix, the archiver tripped its + // `InitialCheckpointNumberNotSequentialError` consecutive-number guard, rolled back the L1 + // sync point, and looped indefinitely. The fix records P1 as a rejected ancestor and skips P2 + // (its valid descendant) outright, emitting `DescendentOfInvalidAttestationsCheckpointDetected` + // so the slasher can target P2's proposer. This test verifies the chain advances past P2 and + // that both proposers end up flagged for slashing. + it('archiver skips a descendant of an invalid-attestations checkpoint', async () => { + const sequencers = nodes.map(node => node.getSequencer()!); + + // The committee invalidation fallback is already disabled by the fixture-level + // `secondsBeforeInvalidatingBlockAsCommitteeMember`. We also need to disable the non-committee + // fallback (`considerInvalidatingCheckpoint` at sequencer.ts:950, called from L345) on every + // node, otherwise any sequencer whose pending chain is invalid will eventually invalidate P1 + // and break the loop we're trying to reproduce. + sequencers.forEach(s => + s.updateConfig({ + secondsBeforeInvalidatingBlockAsNonCommitteeMember: Number.MAX_SAFE_INTEGER, + minTxsPerBlock: 0, + }), + ); + await Promise.all(sequencers.map(s => s.start())); + logger.warn(`Started all sequencers, waiting for first checkpoint before applying malicious config`); + + // Wait for at least one good checkpoint to be mined so any in-progress slot has completed. + const initialCheckpointNumber = (await nodes[0].getChainTips()).checkpointed.checkpoint.number; + await test.waitUntilCheckpointNumber(CheckpointNumber(initialCheckpointNumber + 1), test.L2_SLOT_DURATION_IN_S * 4); + + // Align to the start of an L2 slot, then pick two slots with a 3-slot gap so the malicious + // config has time to land on each proposer's job snapshot under pipelining, and P1's proposal + // has time to propagate to P2 before P2 starts pipelined building. + await test.monitor.waitUntilNextL2Slot(); + const { l2SlotNumber: currentSlot } = await test.monitor.run(); + logger.warn(`First checkpoint mined, current slot is ${currentSlot}`); + + let badSlot1 = SlotNumber.add(currentSlot, 3); + let badSlot2 = SlotNumber.add(currentSlot, 4); + let p1Proposer = await test.epochCache.getProposerAttesterAddressInSlot(badSlot1); + let p2Proposer = await test.epochCache.getProposerAttesterAddressInSlot(badSlot2); + + // Ensure the two slots belong to different proposers; retry by walking forward one slot at + // a time. With committee size 6 and random shuffling this should usually succeed first try. + let attempts = 0; + while (p1Proposer && p2Proposer && p1Proposer.equals(p2Proposer)) { + attempts += 1; + if (attempts > 6) { + throw new Error(`Could not find two consecutive slots with different proposers`); + } + badSlot1 = SlotNumber.add(badSlot1, 1); + badSlot2 = SlotNumber.add(badSlot2, 1); + p1Proposer = await test.epochCache.getProposerAttesterAddressInSlot(badSlot1); + p2Proposer = await test.epochCache.getProposerAttesterAddressInSlot(badSlot2); + } + if (!p1Proposer || !p2Proposer) { + throw new Error(`Could not resolve proposers for slots ${badSlot1} and ${badSlot2}`); + } + + const p1NodeIndex = nodes.findIndex(n => n.getSequencer()!.validatorAddresses!.some(a => a.equals(p1Proposer!))); + const p2NodeIndex = nodes.findIndex(n => n.getSequencer()!.validatorAddresses!.some(a => a.equals(p2Proposer!))); + if (p1NodeIndex === -1 || p2NodeIndex === -1) { + throw new Error(`Could not find nodes for proposers P1=${p1Proposer} P2=${p2Proposer}`); + } + const p1Node = nodes[p1NodeIndex]; + const p2Node = nodes[p2NodeIndex]; + logger.warn(`Applying malicious configs`, { + p1NodeIndex, + p1Proposer: p1Proposer.toString(), + badSlot1, + p2NodeIndex, + p2Proposer: p2Proposer.toString(), + badSlot2, + }); + + // P1 publishes its checkpoint with only its own self-attestation (insufficient) and skips + // any invalidation of earlier checkpoints. + await p1Node.setConfig({ + skipCollectingAttestations: true, + skipInvalidateBlockAsProposer: true, + minTxsPerBlock: 0, + }); + + // P2 collects attestations normally so its checkpoint lands valid, but bypasses the + // parent-validity gate, so it ends up pushing a valid checkpoint with valid attestations + // that descends from the invalid P1, which is the scenario we want to test. + await p2Node.setConfig({ + skipWaitForValidParentCheckpointOnL1: true, + skipInvalidateBlockAsProposer: true, + minTxsPerBlock: 0, + }); + + // Subscribe to the new archiver event so we can assert P2 was surfaced through it. + const observerIndex = range(nodes.length).find(i => i !== p1NodeIndex && i !== p2NodeIndex)!; + const observerArchiver = nodes[observerIndex].getBlockSource() as Archiver; + const descendantEvents: { checkpointNumber: CheckpointNumber; ancestorCheckpointNumber: CheckpointNumber }[] = []; + const onDescendant = (event: { + checkpoint: { checkpointNumber: CheckpointNumber }; + ancestorCheckpointNumber: CheckpointNumber; + }) => { + descendantEvents.push({ + checkpointNumber: event.checkpoint.checkpointNumber, + ancestorCheckpointNumber: event.ancestorCheckpointNumber, + }); + }; + + observerArchiver.events.on(L2BlockSourceEvents.DescendentOfInvalidAttestationsCheckpointDetected, onDescendant); + + // Send a couple of txs so there's content for both checkpoints. + logger.warn('Sending transactions to fill the bad checkpoints'); + await Promise.all(times(4, i => testContract.methods.emit_nullifier(BigInt(i + 1)).send({ from, wait: NO_WAIT }))); + + // Watch for both CheckpointProposed events at the targeted slots. + const p1CheckpointPromise = promiseWithResolvers(); + const p2CheckpointPromise = promiseWithResolvers(); + test.monitor.on('checkpoint', ({ checkpointNumber, l2SlotNumber }) => { + if (l2SlotNumber === badSlot1) { + p1CheckpointPromise.resolve(checkpointNumber); + } + if (l2SlotNumber === badSlot2) { + p2CheckpointPromise.resolve(checkpointNumber); + } + }); + + logger.warn(`Waiting for two checkpoints to be mined on slots ${badSlot1} and ${badSlot2}`); + const [p1Checkpoint, p2Checkpoint] = await executeTimeout( + () => Promise.all([p1CheckpointPromise.promise, p2CheckpointPromise.promise]), + test.L2_SLOT_DURATION_IN_S * 8 * 1000, + 'Waiting for both checkpoints', + ); + logger.warn(`Observed checkpoints`, { p1Checkpoint, p2Checkpoint, badSlot1, badSlot2 }); + expect(p2Checkpoint).toEqual(CheckpointNumber(p1Checkpoint + 1)); + + // P1 must have landed with insufficient attestations (the trigger for the archiver skip). + await assertCheckpointInsufficientAttestations(p1Checkpoint); + + // Restore P2 to a healthy config so a later proposer (or P2 in a future slot) can resume + // the chain by invalidating P1 and posting fresh checkpoints. + await p2Node.setConfig({ skipWaitForValidParentCheckpointOnL1: false, skipInvalidateBlockAsProposer: false }); + + // The archiver should no longer stall: wait for the chain to advance past P2 within a + // handful of slots. Note we wait on local checkpoint progress here (i.e. for the chain to + // get unstuck), not specifically on observing P2 in the checkpointed tip — P1 and P2 will + // both be skipped, the chain will be invalidated, and progress comes from later slots. + const targetCheckpoint = CheckpointNumber(p2Checkpoint + 1); + logger.warn(`Waiting for node ${observerIndex} to advance past checkpoint ${p2Checkpoint}`); + await retryUntil( + async () => { + const tips = await nodes[observerIndex].getChainTips(); + return tips.checkpointed.checkpoint.number >= targetCheckpoint; + }, + 'archiver advances past P2', + test.L2_SLOT_DURATION_IN_S * 8, + 0.5, + ); + + // Confirm the descendant-of-invalid event fired for P2 at least once. + logger.warn(`Observed ${descendantEvents.length} DescendentOfInvalidAttestationsCheckpointDetected events`); + expect(descendantEvents.some(e => e.checkpointNumber === p2Checkpoint)).toBe(true); + + // Both proposers should be flagged for slashing: P1 under PROPOSED_INSUFFICIENT_ATTESTATIONS + // and P2 under PROPOSED_DESCENDANT_OF_CHECKPOINT_WITH_INVALID_ATTESTATIONS. + const offenses = await nodes[observerIndex].getSlashOffenses('all'); + logger.warn(`Collected ${offenses.length} offenses`, { + offenses: offenses.map(o => ({ + offenseType: o.offenseType, + validator: o.validator.toString(), + slot: o.epochOrSlot, + })), + }); + const insufficient = offenses.find( + o => o.offenseType === OffenseType.PROPOSED_INSUFFICIENT_ATTESTATIONS && o.epochOrSlot === BigInt(badSlot1), + ); + expect(insufficient).toBeDefined(); + expect(insufficient!.validator.equals(p1Proposer!)).toBeTrue(); + + const descendant = offenses.find( + o => + o.offenseType === OffenseType.PROPOSED_DESCENDANT_OF_CHECKPOINT_WITH_INVALID_ATTESTATIONS && + o.epochOrSlot === BigInt(badSlot2), + ); + expect(descendant).toBeDefined(); + expect(descendant!.validator.equals(p2Proposer!)).toBeTrue(); + + logger.warn(`Test succeeded '${expect.getState().currentTestName}'`); + observerArchiver.events.removeListener( + L2BlockSourceEvents.DescendentOfInvalidAttestationsCheckpointDetected, + onDescendant, + ); + }); + // All tests but this one disable invalidation by committee. This test disables invalidation by proposer and // instead waits for a committee member to invalidate the block after several proposers not doing so. it('committee member invalidates a block if proposer does not come through', async () => { diff --git a/yarn-project/foundation/src/branded-types/checkpoint_number.ts b/yarn-project/foundation/src/branded-types/checkpoint_number.ts index 97c1f1f14b95..194b3062cf8a 100644 --- a/yarn-project/foundation/src/branded-types/checkpoint_number.ts +++ b/yarn-project/foundation/src/branded-types/checkpoint_number.ts @@ -1,5 +1,6 @@ import { z } from 'zod'; +import { isDefined } from '../types/index.js'; import type { BlockNumber } from './block_number.js'; import type { Branded } from './types.js'; @@ -90,6 +91,15 @@ CheckpointNumber.add = function (n: CheckpointNumber, increment: number): Checkp return CheckpointNumber(n + increment); }; +/** Computes max of a set of checkpoint numbers, ignoring undefined values. */ +CheckpointNumber.max = function (...values: (CheckpointNumber | undefined)[]): CheckpointNumber | undefined { + const filtered = values.filter(isDefined); + if (filtered.length === 0) { + return undefined; + } + return CheckpointNumber(Math.max(...filtered)); +}; + /** The zero checkpoint value. */ CheckpointNumber.ZERO = CheckpointNumber(0); diff --git a/yarn-project/sequencer-client/src/config.ts b/yarn-project/sequencer-client/src/config.ts index 16f703bccc9c..7801bc99f309 100644 --- a/yarn-project/sequencer-client/src/config.ts +++ b/yarn-project/sequencer-client/src/config.ts @@ -52,6 +52,7 @@ export const DefaultSequencerConfig = { secondsBeforeInvalidatingBlockAsNonCommitteeMember: 432, // 36 L1 blocks skipCollectingAttestations: false, skipInvalidateBlockAsProposer: false, + skipWaitForValidParentCheckpointOnL1: false, broadcastInvalidBlockProposal: false, broadcastInvalidCheckpointProposalOnly: false, injectFakeAttestation: false, @@ -188,6 +189,12 @@ export const sequencerConfigMappings: ConfigMappingsType = { description: 'Do not invalidate the previous block if invalid when we are the proposer (for testing only)', ...booleanConfigHelper(DefaultSequencerConfig.skipInvalidateBlockAsProposer), }, + skipWaitForValidParentCheckpointOnL1: { + description: + 'Bypass the parent checkpoint validity check before submitting a pipelined checkpoint, ' + + 'allowing the proposer to publish even when the parent landed on L1 with invalid attestations (for testing only)', + ...booleanConfigHelper(DefaultSequencerConfig.skipWaitForValidParentCheckpointOnL1), + }, broadcastInvalidBlockProposal: { description: 'Broadcast invalid block proposals with corrupted state (for testing only)', ...booleanConfigHelper(DefaultSequencerConfig.broadcastInvalidBlockProposal), diff --git a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts index 209e04b4856b..e86b5533edcc 100644 --- a/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts +++ b/yarn-project/sequencer-client/src/sequencer/checkpoint_proposal_job.ts @@ -396,6 +396,13 @@ export class CheckpointProposalJob implements Traceable { * If the parent has invalid attestations, enqueues an invalidation. Returns whether to proceed with the proposal. */ protected async waitForValidParentCheckpointOnL1(): Promise { + if (this.config.skipWaitForValidParentCheckpointOnL1) { + this.log.warn(`Skipping waitForValidParentCheckpointOnL1 due to test configuration`, { + checkpointNumber: this.checkpointNumber, + }); + return true; + } + const parentCheckpointNumber = CheckpointNumber(this.checkpointNumber - 1); // Wait until archiver has synced L1 past the parent's slot (slotNow) diff --git a/yarn-project/slasher/src/watchers/attestations_block_watcher.test.ts b/yarn-project/slasher/src/watchers/attestations_block_watcher.test.ts index 933362d00cde..54a26f58d642 100644 --- a/yarn-project/slasher/src/watchers/attestations_block_watcher.test.ts +++ b/yarn-project/slasher/src/watchers/attestations_block_watcher.test.ts @@ -1,13 +1,15 @@ -import type { EpochCache } from '@aztec/epoch-cache'; +import type { EpochCache, EpochCommitteeInfo } from '@aztec/epoch-cache'; import { CheckpointNumber, EpochNumber, SlotNumber } from '@aztec/foundation/branded-types'; import { Fr } from '@aztec/foundation/curves/bn254'; import { EthAddress } from '@aztec/foundation/eth-address'; import type { + DescendentOfInvalidAttestationsCheckpointEvent, InvalidCheckpointDetectedEvent, L2BlockSourceEventEmitter, ValidateCheckpointNegativeResult, } from '@aztec/stdlib/block'; import type { CheckpointInfo } from '@aztec/stdlib/checkpoint'; +import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers'; import { OffenseType } from '@aztec/stdlib/slashing'; import { jest } from '@jest/globals'; @@ -45,8 +47,15 @@ describe('AttestationsBlockWatcher', () => { proposer = EthAddress.fromString('0x0000000000000000000000000000000000000abc'); committee = [proposer, EthAddress.fromString('0x0000000000000000000000000000000000000def')]; - // Default mock return value + // Default mock return values epochCache.getProposerFromEpochCommittee.mockReturnValue(proposer); + epochCache.getL1Constants.mockReturnValue({ epochDuration: 32 } as L1RollupConstants); + epochCache.getCommitteeForEpoch.mockResolvedValue({ + committee, + seed: 0n, + epoch: EpochNumber(0), + isEscapeHatchOpen: false, + } as EpochCommitteeInfo); }); it('should emit WANT_TO_SLASH_EVENT for proposer when invalid checkpoint detected due to insufficient attestations', () => { @@ -110,36 +119,19 @@ describe('AttestationsBlockWatcher', () => { expect(handler).toHaveBeenCalledTimes(1); }); - it('should emit WANT_TO_SLASH_EVENT for attestors when checkpoint built on invalid parent', () => { - // First, handle an invalid checkpoint using the pre-configured data - const invalidCheckpointValidationResult: ValidateCheckpointNegativeResult = { - valid: false, - reason: 'insufficient-attestations', - checkpoint: checkpointInfo, - committee, - epoch: EpochNumber(1), - seed: 0n, - attestors: [], - attestations: [], - }; - - const invalidCheckpointEvent: InvalidCheckpointDetectedEvent = { - type: 'invalidCheckpointDetected', - validationResult: invalidCheckpointValidationResult, - }; - - watcher.handleInvalidCheckpoint(invalidCheckpointEvent); - - // Now handle a checkpoint that builds on the invalid checkpoint + it('emits both an invalid-attestations slash and a descendant slash for a checkpoint that has invalid attestations and builds on an invalid ancestor', async () => { + // A checkpoint that both has invalid attestations of its own and extends a previously-rejected + // ancestor produces two offenses. The archiver reports each via its own event, so the watcher sees + // an invalidCheckpointDetected (own attestations) and a descendentOfInvalidAttestationsCheckpointDetected + // (extends a rejected ancestor) for the same checkpoint. const childCheckpointInfo: CheckpointInfo = { archive: Fr.random(), - lastArchive: checkpointInfo.archive, // Parent archive + lastArchive: checkpointInfo.archive, // Parent archive (the rejected ancestor) slotNumber: SlotNumber(2), checkpointNumber: CheckpointNumber(2), timestamp: BigInt(Math.floor(Date.now() / 1000)), }; const proposer2 = EthAddress.fromString('0x0000000000000000000000000000000000000def'); - epochCache.getProposerFromEpochCommittee.mockReturnValue(proposer2); const attestor1 = EthAddress.fromString('0x0000000000000000000000000000000000000111'); @@ -156,13 +148,11 @@ describe('AttestationsBlockWatcher', () => { attestations: [], }; - const childEvent: InvalidCheckpointDetectedEvent = { + // First event: slash the proposer for the invalid attestations on its own checkpoint. + watcher.handleInvalidCheckpoint({ type: 'invalidCheckpointDetected', validationResult: childValidationResult, - }; - - handler.mockClear(); - watcher.handleInvalidCheckpoint(childEvent); + }); expect(handler).toHaveBeenCalledWith([ { @@ -173,25 +163,28 @@ describe('AttestationsBlockWatcher', () => { } satisfies WantToSlashArgs, ]); + // Second event: slash the same proposer for building on the rejected ancestor. + await watcher.handleDescendantOfInvalid({ + type: 'descendentOfInvalidAttestationsCheckpointDetected', + checkpoint: childCheckpointInfo, + ancestorArchiveRoot: checkpointInfo.archive, + ancestorCheckpointNumber: checkpointInfo.checkpointNumber, + }); + expect(handler).toHaveBeenCalledWith([ { - validator: attestor1, - amount: config.slashProposeDescendantOfCheckpointWithInvalidAttestationsPenalty, - offenseType: OffenseType.PROPOSED_DESCENDANT_OF_CHECKPOINT_WITH_INVALID_ATTESTATIONS, - epochOrSlot: 2n, - }, - { - validator: attestor2, + validator: proposer2, amount: config.slashProposeDescendantOfCheckpointWithInvalidAttestationsPenalty, offenseType: OffenseType.PROPOSED_DESCENDANT_OF_CHECKPOINT_WITH_INVALID_ATTESTATIONS, epochOrSlot: 2n, - }, - ] satisfies WantToSlashArgs[]); + } satisfies WantToSlashArgs, + ]); expect(handler).toHaveBeenCalledTimes(2); }); - it('should not process the same invalid checkpoint twice', () => { - const validationResult: ValidateCheckpointNegativeResult = { + it('emits WANT_TO_SLASH_EVENT for proposer when a valid-attestations descendant of an invalid checkpoint is detected', async () => { + // Seed the watcher with one invalid ancestor so the descendant event hits the cache. + const invalidValidationResult: ValidateCheckpointNegativeResult = { valid: false, reason: 'insufficient-attestations', checkpoint: checkpointInfo, @@ -201,20 +194,120 @@ describe('AttestationsBlockWatcher', () => { attestors: [], attestations: [], }; - - const event: InvalidCheckpointDetectedEvent = { + watcher.handleInvalidCheckpoint({ type: 'invalidCheckpointDetected', - validationResult, + validationResult: invalidValidationResult, + }); + + handler.mockClear(); + + const descendantCheckpointInfo: CheckpointInfo = { + archive: Fr.random(), + lastArchive: checkpointInfo.archive, + slotNumber: SlotNumber(2), + checkpointNumber: CheckpointNumber(2), + timestamp: BigInt(Math.floor(Date.now() / 1000)), + }; + const descendantProposer = EthAddress.fromString('0x0000000000000000000000000000000000000def'); + epochCache.getProposerFromEpochCommittee.mockReturnValue(descendantProposer); + + const event: DescendentOfInvalidAttestationsCheckpointEvent = { + type: 'descendentOfInvalidAttestationsCheckpointDetected', + checkpoint: descendantCheckpointInfo, + ancestorArchiveRoot: checkpointInfo.archive, + ancestorCheckpointNumber: checkpointInfo.checkpointNumber, }; - // Handle the same event twice - watcher.handleInvalidCheckpoint(event); - watcher.handleInvalidCheckpoint(event); + await watcher.handleDescendantOfInvalid(event); - // Should only emit once (duplicate was skipped) + expect(handler).toHaveBeenCalledWith([ + { + validator: descendantProposer, + amount: config.slashProposeDescendantOfCheckpointWithInvalidAttestationsPenalty, + offenseType: OffenseType.PROPOSED_DESCENDANT_OF_CHECKPOINT_WITH_INVALID_ATTESTATIONS, + epochOrSlot: 2n, + } satisfies WantToSlashArgs, + ]); expect(handler).toHaveBeenCalledTimes(1); }); + it('slashes a further descendant that both has invalid attestations and extends another descendant', async () => { + // The archiver tracks the rejected chain and reports each descendant via its own event. A further + // descendant (D2) that both has its own invalid attestations and extends an earlier descendant (D1) + // is reported through two events; the watcher slashes its proposer for each offense independently. + const d2: CheckpointInfo = { + archive: Fr.random(), + lastArchive: Fr.random(), + slotNumber: SlotNumber(3), + checkpointNumber: CheckpointNumber(3), + timestamp: BigInt(Math.floor(Date.now() / 1000)), + }; + const d2Proposer = EthAddress.fromString('0x0000000000000000000000000000000000000bbb'); + epochCache.getProposerFromEpochCommittee.mockReturnValue(d2Proposer); + + // D2 has invalid attestations of its own. + watcher.handleInvalidCheckpoint({ + type: 'invalidCheckpointDetected', + validationResult: { + valid: false, + reason: 'insufficient-attestations', + checkpoint: d2, + committee, + epoch: EpochNumber(1), + seed: 0n, + attestors: [], + attestations: [], + }, + }); + + expect(handler).toHaveBeenCalledWith([ + { + validator: d2Proposer, + amount: config.slashProposeInvalidAttestationsPenalty, + offenseType: OffenseType.PROPOSED_INSUFFICIENT_ATTESTATIONS, + epochOrSlot: 3n, + } satisfies WantToSlashArgs, + ]); + + // D2 also extends an earlier descendant of an invalid checkpoint. + await watcher.handleDescendantOfInvalid({ + type: 'descendentOfInvalidAttestationsCheckpointDetected', + checkpoint: d2, + ancestorArchiveRoot: Fr.random(), + ancestorCheckpointNumber: CheckpointNumber(1), + }); + + expect(handler).toHaveBeenCalledWith([ + { + validator: d2Proposer, + amount: config.slashProposeDescendantOfCheckpointWithInvalidAttestationsPenalty, + offenseType: OffenseType.PROPOSED_DESCENDANT_OF_CHECKPOINT_WITH_INVALID_ATTESTATIONS, + epochOrSlot: 3n, + } satisfies WantToSlashArgs, + ]); + expect(handler).toHaveBeenCalledTimes(2); + }); + + it('handles descendant-of-invalid event when no proposer is found', async () => { + epochCache.getProposerFromEpochCommittee.mockReturnValue(undefined); + + const descendant: CheckpointInfo = { + archive: Fr.random(), + lastArchive: Fr.random(), + slotNumber: SlotNumber(2), + checkpointNumber: CheckpointNumber(2), + timestamp: BigInt(Math.floor(Date.now() / 1000)), + }; + await watcher.handleDescendantOfInvalid({ + type: 'descendentOfInvalidAttestationsCheckpointDetected', + checkpoint: descendant, + ancestorArchiveRoot: Fr.random(), + ancestorCheckpointNumber: CheckpointNumber(1), + }); + + expect(handler).not.toHaveBeenCalled(); + }); + it('should handle case when no proposer is found', () => { epochCache.getProposerFromEpochCommittee.mockReturnValue(undefined); diff --git a/yarn-project/slasher/src/watchers/attestations_block_watcher.ts b/yarn-project/slasher/src/watchers/attestations_block_watcher.ts index cd02e7312c5b..f49d56ac9c1e 100644 --- a/yarn-project/slasher/src/watchers/attestations_block_watcher.ts +++ b/yarn-project/slasher/src/watchers/attestations_block_watcher.ts @@ -1,14 +1,15 @@ import { EpochCache } from '@aztec/epoch-cache'; -import { SlotNumber } from '@aztec/foundation/branded-types'; +import { EpochNumber } from '@aztec/foundation/branded-types'; import { merge, pick } from '@aztec/foundation/collection'; -import { FifoSet } from '@aztec/foundation/fifo-set'; -import { type Logger, createLogger } from '@aztec/foundation/log'; +import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log'; import { + type DescendentOfInvalidAttestationsCheckpointEvent, type InvalidCheckpointDetectedEvent, type L2BlockSourceEventEmitter, L2BlockSourceEvents, type ValidateCheckpointNegativeResult, } from '@aztec/stdlib/block'; +import { getEpochAtSlot } from '@aztec/stdlib/epoch-helpers'; import { OffenseType } from '@aztec/stdlib/slashing'; import EventEmitter from 'node:events'; @@ -20,22 +21,28 @@ const AttestationsBlockWatcherConfigKeys = [ 'slashProposeDescendantOfCheckpointWithInvalidAttestationsPenalty', 'slashProposeInvalidAttestationsPenalty', ] as const; -const MAX_INVALID_CHECKPOINTS = 100; type AttestationsBlockWatcherConfig = Pick; /** - * This watcher is responsible for detecting invalid blocks and creating slashing arguments for offenders. - * An invalid block is one that doesn't have enough attestations or has incorrect attestations. - * The proposer of an invalid block should be slashed. - * If there's another block consecutive to the invalid one, its proposer and attestors should also be slashed. + * Watches the archiver for checkpoints whose publication is itself a slashable offense. + * + * Two cases are handled, both targeting the proposer of the offending checkpoint: + * + * - Invalid-attestations checkpoint: the proposer published a checkpoint to L1 whose + * attestations are either insufficient (below quorum) or incorrect (signature from a + * non-committee member, malformed signature, etc.). Slashed via + * {@link OffenseType.PROPOSED_INSUFFICIENT_ATTESTATIONS} or + * {@link OffenseType.PROPOSED_INCORRECT_ATTESTATIONS}. + * + * - Descendant of an invalid checkpoint: the proposer published a checkpoint that extends a + * previously-rejected one. The descendant may itself have valid attestations, but it is still + * unusable. Triggered by the archiver's `CheckpointBuiltOnInvalidAncestorDetected` event + * when the descendant has valid attestations (skipped before ingestion). Slashes the descendant's + * proposer via {@link OffenseType.PROPOSED_DESCENDANT_OF_CHECKPOINT_WITH_INVALID_ATTESTATIONS}. */ export class AttestationsBlockWatcher extends (EventEmitter as new () => WatcherEmitter) implements Watcher { - private log: Logger = createLogger('attestations-block-watcher'); - - // Recently seen invalid archive roots. - private invalidArchiveRoots = FifoSet.withLimit(MAX_INVALID_CHECKPOINTS); - + private log: Logger; private config: AttestationsBlockWatcherConfig; private boundHandleInvalidCheckpoint = (event: InvalidCheckpointDetectedEvent) => { @@ -49,12 +56,23 @@ export class AttestationsBlockWatcher extends (EventEmitter as new () => Watcher } }; + private boundHandleDescendantOfInvalid = (event: DescendentOfInvalidAttestationsCheckpointEvent) => { + this.handleDescendantOfInvalid(event).catch(err => { + this.log.error('Error handling descendant of invalid checkpoint', err, { + checkpointNumber: event.checkpoint.checkpointNumber, + ancestorCheckpointNumber: event.ancestorCheckpointNumber, + }); + }); + }; + constructor( private l2BlockSource: L2BlockSourceEventEmitter, private epochCache: EpochCache, config: AttestationsBlockWatcherConfig, + bindings?: LoggerBindings, ) { super(); + this.log = createLogger('slasher:attestations-block-watcher', bindings); this.config = pick(config, ...AttestationsBlockWatcherConfigKeys); this.log.info('AttestationsBlockWatcher initialized'); } @@ -69,6 +87,10 @@ export class AttestationsBlockWatcher extends (EventEmitter as new () => Watcher L2BlockSourceEvents.InvalidAttestationsCheckpointDetected, this.boundHandleInvalidCheckpoint, ); + this.l2BlockSource.events.on( + L2BlockSourceEvents.DescendentOfInvalidAttestationsCheckpointDetected, + this.boundHandleDescendantOfInvalid, + ); return Promise.resolve(); } @@ -77,71 +99,25 @@ export class AttestationsBlockWatcher extends (EventEmitter as new () => Watcher L2BlockSourceEvents.InvalidAttestationsCheckpointDetected, this.boundHandleInvalidCheckpoint, ); + this.l2BlockSource.events.removeListener( + L2BlockSourceEvents.DescendentOfInvalidAttestationsCheckpointDetected, + this.boundHandleDescendantOfInvalid, + ); return Promise.resolve(); } /** Event handler for invalid checkpoints as reported by the archiver. Public for testing purposes. */ public handleInvalidCheckpoint(event: InvalidCheckpointDetectedEvent): void { const { validationResult } = event; - const checkpoint = validationResult.checkpoint; - - // Check if we already have processed this checkpoint, archiver may emit the same event multiple times - if (this.invalidArchiveRoots.has(checkpoint.archive.toString())) { - this.log.trace(`Already processed invalid checkpoint ${checkpoint.checkpointNumber}`); - return; - } + const { reason, checkpoint } = validationResult; this.log.verbose(`Detected invalid checkpoint ${checkpoint.checkpointNumber}`, { ...checkpoint, reason: validationResult.valid === false ? validationResult.reason : 'unknown', }); - this.invalidArchiveRoots.add(checkpoint.archive.toString()); - - // Slash the proposer of the invalid checkpoint - this.slashProposer(event.validationResult); - - // Check if the parent of this checkpoint is invalid as well, if so, we will slash its attestors as well - this.slashAttestorsOnAncestorInvalid(event.validationResult); - } - - private slashAttestorsOnAncestorInvalid(validationResult: ValidateCheckpointNegativeResult) { - const checkpoint = validationResult.checkpoint; - - const parentArchive = checkpoint.lastArchive.toString(); - if (this.invalidArchiveRoots.has(parentArchive)) { - const attestors = validationResult.attestors; - this.log.info( - `Want to slash attestors of checkpoint ${checkpoint.checkpointNumber} built on invalid checkpoint`, - { - ...checkpoint, - ...attestors, - parentArchive, - }, - ); - - this.emit( - WANT_TO_SLASH_EVENT, - attestors.map(attestor => ({ - validator: attestor, - amount: this.config.slashProposeDescendantOfCheckpointWithInvalidAttestationsPenalty, - offenseType: OffenseType.PROPOSED_DESCENDANT_OF_CHECKPOINT_WITH_INVALID_ATTESTATIONS, - epochOrSlot: BigInt(SlotNumber(checkpoint.slotNumber)), - })), - ); - } - } - - private slashProposer(validationResult: ValidateCheckpointNegativeResult) { - const { reason, checkpoint } = validationResult; - const checkpointNumber = checkpoint.checkpointNumber; - const slot = checkpoint.slotNumber; - const epochCommitteeInfo = { - committee: validationResult.committee, - seed: validationResult.seed, - epoch: validationResult.epoch, - isEscapeHatchOpen: false, - }; + const { checkpointNumber, slotNumber: slot } = checkpoint; + const epochCommitteeInfo = { ...validationResult, isEscapeHatchOpen: false }; const proposer = this.epochCache.getProposerFromEpochCommittee(epochCommitteeInfo, slot); if (!proposer) { @@ -166,6 +142,40 @@ export class AttestationsBlockWatcher extends (EventEmitter as new () => Watcher this.emit(WANT_TO_SLASH_EVENT, [args]); } + /** + * Event handler for valid-attestations checkpoints that build on a previously-rejected ancestor. + * The archiver emits this when ingesting the descendant, and we slash its proposer. + */ + public async handleDescendantOfInvalid(event: DescendentOfInvalidAttestationsCheckpointEvent): Promise { + const { checkpoint, ancestorCheckpointNumber, ancestorArchiveRoot } = event; + + const slot = checkpoint.slotNumber; + const epoch = EpochNumber(getEpochAtSlot(slot, this.epochCache.getL1Constants())); + const epochCommitteeInfo = await this.epochCache.getCommitteeForEpoch(epoch); + const proposer = this.epochCache.getProposerFromEpochCommittee({ ...epochCommitteeInfo, epoch }, slot); + + if (!proposer) { + this.log.warn( + `No proposer found for invalid descendant checkpoint ${checkpoint.checkpointNumber} at slot ${slot}`, + ); + return; + } + + const args: WantToSlashArgs = { + validator: proposer, + amount: this.config.slashProposeDescendantOfCheckpointWithInvalidAttestationsPenalty, + offenseType: OffenseType.PROPOSED_DESCENDANT_OF_CHECKPOINT_WITH_INVALID_ATTESTATIONS, + epochOrSlot: BigInt(slot), + }; + + this.log.info( + `Want to slash proposer of checkpoint ${checkpoint.checkpointNumber} built on invalid checkpoint ${ancestorCheckpointNumber}`, + { ...checkpoint, ancestorArchiveRoot: ancestorArchiveRoot.toString(), ...args }, + ); + + this.emit(WANT_TO_SLASH_EVENT, [args]); + } + private getOffenseFromInvalidationReason(reason: ValidateCheckpointNegativeResult['reason']): OffenseType { switch (reason) { case 'invalid-attestation': diff --git a/yarn-project/stdlib/src/block/l2_block_source.ts b/yarn-project/stdlib/src/block/l2_block_source.ts index 429e36da59d7..9d6d46323a0d 100644 --- a/yarn-project/stdlib/src/block/l2_block_source.ts +++ b/yarn-project/stdlib/src/block/l2_block_source.ts @@ -16,6 +16,7 @@ import type { TypedEventEmitter } from '@aztec/foundation/types'; import { z } from 'zod'; import type { CheckpointData, ProposedCheckpointData } from '../checkpoint/checkpoint_data.js'; +import type { CheckpointInfo } from '../checkpoint/checkpoint_info.js'; import type { PublishedCheckpoint } from '../checkpoint/published_checkpoint.js'; import type { L1RollupConstants } from '../epoch-helpers/index.js'; import { CheckpointHeader } from '../rollup/checkpoint_header.js'; @@ -294,6 +295,9 @@ export type ArchiverEmitter = TypedEventEmitter<{ [L2BlockSourceEvents.InvalidAttestationsCheckpointDetected]: (args: InvalidCheckpointDetectedEvent) => void; [L2BlockSourceEvents.L2BlocksCheckpointed]: (args: L2CheckpointEvent) => void; [L2BlockSourceEvents.CheckpointEquivocationDetected]: (args: CheckpointEquivocationDetectedEvent) => void; + [L2BlockSourceEvents.DescendentOfInvalidAttestationsCheckpointDetected]: ( + args: DescendentOfInvalidAttestationsCheckpointEvent, + ) => void; }>; export interface L2BlockSourceEventEmitter extends L2BlockSource { events: ArchiverEmitter; @@ -376,6 +380,7 @@ export enum L2BlockSourceEvents { L2BlocksCheckpointed = 'l2BlocksCheckpointed', InvalidAttestationsCheckpointDetected = 'invalidCheckpointDetected', CheckpointEquivocationDetected = 'checkpointEquivocationDetected', + DescendentOfInvalidAttestationsCheckpointDetected = 'descendentOfInvalidAttestationsCheckpointDetected', } export type L2BlockProvenEvent = { @@ -418,3 +423,19 @@ export type CheckpointEquivocationDetectedEvent = { l1ArchiveRoot: Fr; proposedArchiveRoot: Fr; }; + +/** + * Emitted when the archiver observes a checkpoint that builds on a previously-rejected + * ancestor (typically one with insufficient/invalid attestations). The descendant itself may + * have valid attestations, but it cannot be ingested because the chain it extends was + * skipped. The slasher uses this to slash the proposer of the descendant. + */ +export type DescendentOfInvalidAttestationsCheckpointEvent = { + type: 'descendentOfInvalidAttestationsCheckpointDetected'; + /** The descendant checkpoint being rejected. */ + checkpoint: CheckpointInfo; + /** Archive root of the rejected ancestor this descendant builds on. */ + ancestorArchiveRoot: Fr; + /** Checkpoint number of the rejected ancestor. */ + ancestorCheckpointNumber: CheckpointNumber; +}; diff --git a/yarn-project/stdlib/src/interfaces/configs.ts b/yarn-project/stdlib/src/interfaces/configs.ts index 83453f405fe4..27de324a3630 100644 --- a/yarn-project/stdlib/src/interfaces/configs.ts +++ b/yarn-project/stdlib/src/interfaces/configs.ts @@ -62,6 +62,11 @@ export interface SequencerConfig { skipCollectingAttestations?: boolean; /** Do not invalidate the previous block if invalid when we are the proposer (for testing only) */ skipInvalidateBlockAsProposer?: boolean; + /** + * Bypass the parent checkpoint validity check before submitting a pipelined checkpoint, allowing + * the proposer to publish even when the parent landed on L1 with invalid attestations (for testing only). + */ + skipWaitForValidParentCheckpointOnL1?: boolean; /** Broadcast invalid block proposals with corrupted state (for testing only) */ broadcastInvalidBlockProposal?: boolean; /** Broadcast an invalid block proposal only at this indexWithinCheckpoint (for testing only) */ @@ -126,6 +131,7 @@ export const SequencerConfigSchema = zodFor()( attestationPropagationTime: z.number().optional(), skipCollectingAttestations: z.boolean().optional(), skipInvalidateBlockAsProposer: z.boolean().optional(), + skipWaitForValidParentCheckpointOnL1: z.boolean().optional(), secondsBeforeInvalidatingBlockAsCommitteeMember: z.number(), secondsBeforeInvalidatingBlockAsNonCommitteeMember: z.number(), broadcastInvalidBlockProposal: z.boolean().optional(), From d31c7a5b6d0807669c70ccbfa2ebeec16a535591 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Wed, 27 May 2026 14:27:28 +0100 Subject: [PATCH 02/27] chore: scale network validators (#23579) . --- spartan/environments/mainnet.env | 3 ++- spartan/environments/next-net.env | 5 +++-- spartan/environments/prove-n-tps-fake.env | 4 ++-- spartan/environments/prove-n-tps-real.env | 4 ++-- spartan/environments/staging-public.env | 4 ++-- spartan/environments/testnet.env | 4 ++-- spartan/scripts/deploy_network.sh | 2 ++ spartan/terraform/deploy-aztec-infra/main.tf | 4 ++-- spartan/terraform/deploy-aztec-infra/variables.tf | 6 ++++++ 9 files changed, 23 insertions(+), 13 deletions(-) diff --git a/spartan/environments/mainnet.env b/spartan/environments/mainnet.env index 97d6b69748f5..e8b74d9b3efe 100644 --- a/spartan/environments/mainnet.env +++ b/spartan/environments/mainnet.env @@ -12,7 +12,8 @@ VERIFY_CONTRACTS=false DEPLOY_INTERNAL_BOOTNODE=false VALIDATOR_REPLICAS=0 RPC_REPLICAS=1 -PROVER_REPLICAS=4 +PROVER_REPLICAS=0 +PROVER_ENABLED=false CREATE_RPC_INGRESS=true CREATE_RPC_DNS=true diff --git a/spartan/environments/next-net.env b/spartan/environments/next-net.env index f8541c8b5229..7e24b13ef493 100644 --- a/spartan/environments/next-net.env +++ b/spartan/environments/next-net.env @@ -44,13 +44,14 @@ AZTEC_LAG_IN_EPOCHS_FOR_VALIDATOR_SET=2 AZTEC_LAG_IN_EPOCHS_FOR_RANDAO=2 AZTEC_INBOX_LAG=2 -VALIDATOR_REPLICAS=4 -VALIDATORS_PER_NODE=12 +VALIDATOR_REPLICAS=2 +VALIDATORS_PER_NODE=24 VALIDATOR_PUBLISHERS_PER_REPLICA=4 VALIDATOR_PUBLISHER_MNEMONIC_START_INDEX=5000 PUBLISHERS_PER_PROVER=2 PROVER_PUBLISHER_MNEMONIC_START_INDEX=8000 +PROVER_REPLICAS=4 BOT_TRANSFERS_REPLICAS=1 BOT_TRANSFERS_TX_INTERVAL_SECONDS=250 diff --git a/spartan/environments/prove-n-tps-fake.env b/spartan/environments/prove-n-tps-fake.env index 9aa4fc5f6359..46ea329e2582 100644 --- a/spartan/environments/prove-n-tps-fake.env +++ b/spartan/environments/prove-n-tps-fake.env @@ -22,8 +22,8 @@ FUNDING_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf OTEL_COLLECTOR_ENDPOINT=REPLACE_WITH_GCP_SECRET -VALIDATOR_REPLICAS=4 -VALIDATORS_PER_NODE=12 +VALIDATOR_REPLICAS=2 +VALIDATORS_PER_NODE=24 VALIDATOR_PUBLISHERS_PER_REPLICA=4 VALIDATOR_PUBLISHER_MNEMONIC_START_INDEX=5000 diff --git a/spartan/environments/prove-n-tps-real.env b/spartan/environments/prove-n-tps-real.env index 16e0548e91e9..0713b0e4eef2 100644 --- a/spartan/environments/prove-n-tps-real.env +++ b/spartan/environments/prove-n-tps-real.env @@ -22,8 +22,8 @@ FUNDING_PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf OTEL_COLLECTOR_ENDPOINT=REPLACE_WITH_GCP_SECRET -VALIDATOR_REPLICAS=4 -VALIDATORS_PER_NODE=12 +VALIDATOR_REPLICAS=2 +VALIDATORS_PER_NODE=24 VALIDATOR_PUBLISHERS_PER_REPLICA=4 VALIDATOR_PUBLISHER_MNEMONIC_START_INDEX=5000 diff --git a/spartan/environments/staging-public.env b/spartan/environments/staging-public.env index b3f8b318c4a4..ef03a277c131 100644 --- a/spartan/environments/staging-public.env +++ b/spartan/environments/staging-public.env @@ -47,11 +47,11 @@ CREATE_ROLLUP_CONTRACTS=${CREATE_ROLLUP_CONTRACTS:-false} P2P_TX_POOL_DELETE_TXS_AFTER_REORG=true VALIDATOR_REPLICAS=2 -VALIDATORS_PER_NODE=64 +VALIDATORS_PER_NODE=128 VALIDATOR_PUBLISHERS_PER_REPLICA=4 VALIDATOR_PUBLISHER_MNEMONIC_START_INDEX=5000 VALIDATOR_HA_REPLICAS=1 -VALIDATOR_HA_REPLICA_COUNT=4 +VALIDATOR_HA_REPLICA_COUNT=1 VALIDATOR_RESOURCE_PROFILE="prod" PROVER_FAILED_PROOF_STORE=gs://aztec-develop/staging-public/failed-proofs diff --git a/spartan/environments/testnet.env b/spartan/environments/testnet.env index 10cf468f1341..1097fe11f818 100644 --- a/spartan/environments/testnet.env +++ b/spartan/environments/testnet.env @@ -76,8 +76,8 @@ RPC_INGRESS_STATIC_IP_NAME=testnet-rpc-ip RPC_INGRESS_SSL_CERT_NAMES='["testnet-rpc-cert"]' -VALIDATOR_REPLICAS=4 -VALIDATORS_PER_NODE=64 +VALIDATOR_REPLICAS=2 +VALIDATORS_PER_NODE=128 VALIDATOR_PUBLISHERS_PER_REPLICA=8 VALIDATOR_PUBLISHER_MNEMONIC_START_INDEX=5000 VALIDATOR_HA_REPLICAS=1 diff --git a/spartan/scripts/deploy_network.sh b/spartan/scripts/deploy_network.sh index 9ef173dfdacd..26a2da1623d4 100755 --- a/spartan/scripts/deploy_network.sh +++ b/spartan/scripts/deploy_network.sh @@ -131,6 +131,7 @@ AZTEC_EPOCHS_LAG=${AZTEC_EPOCHS_LAG:-} SEQ_ENFORCE_TIME_TABLE=${SEQ_ENFORCE_TIME_TABLE:-} SEQ_SKIP_CHECKPOINT_PUBLISH_PERCENT=${SEQ_SKIP_CHECKPOINT_PUBLISH_PERCENT:-0} PROVER_REPLICAS=${PROVER_REPLICAS:-4} +PROVER_ENABLED=${PROVER_ENABLED:-true} PROVER_AGENTS_PER_PROVER=${PROVER_AGENTS_PER_PROVER:-1} R2_ACCESS_KEY_ID=${R2_ACCESS_KEY_ID:-} R2_SECRET_ACCESS_KEY=${R2_SECRET_ACCESS_KEY:-} @@ -649,6 +650,7 @@ PROVER_PROOF_STORE = "${PROVER_PROOF_STORE:-}" PROVER_BROKER_DEBUG_REPLAY_ENABLED = ${PROVER_BROKER_DEBUG_REPLAY_ENABLED:-false} DEPLOY_ARCHIVAL_NODE = ${DEPLOY_ARCHIVAL_NODE} PROVER_REPLICAS = ${PROVER_REPLICAS} +PROVER_ENABLED = ${PROVER_ENABLED} PROVER_TEST_DELAY_TYPE = "${PROVER_TEST_DELAY_TYPE}" PROVER_TEST_VERIFICATION_DELAY_MS = ${PROVER_TEST_VERIFICATION_DELAY_MS} diff --git a/spartan/terraform/deploy-aztec-infra/main.tf b/spartan/terraform/deploy-aztec-infra/main.tf index 46583be5f6fc..b4d66a49a142 100644 --- a/spartan/terraform/deploy-aztec-infra/main.tf +++ b/spartan/terraform/deploy-aztec-infra/main.tf @@ -317,7 +317,7 @@ locals { wait = true } : null - prover = { + prover = var.PROVER_ENABLED ? { name = "${var.RELEASE_PREFIX}-prover" chart = "aztec-prover-stack" values = [ @@ -410,7 +410,7 @@ locals { boot_node_host_path = "node.node.env.BOOT_NODE_HOST" bootstrap_nodes_path = "node.node.env.BOOTSTRAP_NODES" wait = var.WAIT_FOR_PROVER_DEPLOY - } + } : null rpc = { name = "${var.RELEASE_PREFIX}-rpc" diff --git a/spartan/terraform/deploy-aztec-infra/variables.tf b/spartan/terraform/deploy-aztec-infra/variables.tf index 189ec5309838..dd3f00859491 100644 --- a/spartan/terraform/deploy-aztec-infra/variables.tf +++ b/spartan/terraform/deploy-aztec-infra/variables.tf @@ -264,6 +264,12 @@ variable "PROVER_REPLICAS" { default = 4 } +variable "PROVER_ENABLED" { + description = "Whether to deploy the prover stack" + type = bool + default = true +} + variable "PROVER_TEST_DELAY_TYPE" { description = "The type of test delay to introduce in the prover (fixed, realistic)" type = string From 0be92049b9573a73604ce089715483320276945b Mon Sep 17 00:00:00 2001 From: spypsy Date: Wed, 27 May 2026 16:56:27 +0300 Subject: [PATCH 03/27] fix(ci): nightly 10 TPS bench GCP auth and checkout (#23586) ## Summary - Run `gcp_auth` before `setup_gcp_secrets` in `source_network_env` so EC2 benchmark jobs can read Secret Manager (e.g. `otel-collector-url`). - Improve `setup_gcp_secrets.sh` diagnostics and activate the CI service account before secret fetches. - Install Terraform on Linux in `install_deps.sh`; add `setup-terraform` on nightly wait jobs. - Fix `deploy-network` checkout for pinned submodules (`fetch-depth: 0`, `lfs: true`). - Checkout `github.sha` on the benchmark job so workflow_dispatch from a feature branch runs that branch on EC2 (not `next`). Validated manually via Nightly Bench 10 TPS workflow_dispatch on this branch (run succeeded). ## Test plan - [x] Nightly Bench 10 TPS workflow_dispatch from `spy/10tps-bench-terraform` (deploy, wait, benchmark) --- .github/workflows/deploy-network.yml | 4 +++- .github/workflows/nightly-bench-10tps.yml | 4 ++-- .github/workflows/weekly-proving-bench.yml | 2 +- spartan/scripts/install_deps.sh | 13 +++++++++++++ spartan/scripts/source_network_env.sh | 9 +++++++++ 5 files changed, 28 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy-network.yml b/.github/workflows/deploy-network.yml index 0a278740d88d..7c84658ebabc 100644 --- a/.github/workflows/deploy-network.yml +++ b/.github/workflows/deploy-network.yml @@ -111,7 +111,9 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd with: ref: ${{ steps.checkout-ref.outputs.ref }} - fetch-depth: 1 + # Full history so recursive submodules can checkout pinned commits (not only branch tips). + fetch-depth: 0 + lfs: true persist-credentials: false submodules: recursive # Initialize git submodules for l1-contracts dependencies diff --git a/.github/workflows/nightly-bench-10tps.yml b/.github/workflows/nightly-bench-10tps.yml index 68d79cb190cb..47a4df7db8e0 100644 --- a/.github/workflows/nightly-bench-10tps.yml +++ b/.github/workflows/nightly-bench-10tps.yml @@ -80,7 +80,7 @@ jobs: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: next + ref: ${{ github.sha }} - name: Authenticate to Google Cloud uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f @@ -108,7 +108,7 @@ jobs: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: next + ref: ${{ github.sha }} - name: Run 10 TPS benchmark timeout-minutes: 240 diff --git a/.github/workflows/weekly-proving-bench.yml b/.github/workflows/weekly-proving-bench.yml index 654a38c610a5..ff5b060e82ff 100644 --- a/.github/workflows/weekly-proving-bench.yml +++ b/.github/workflows/weekly-proving-bench.yml @@ -71,7 +71,7 @@ jobs: - name: Checkout uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 with: - ref: next + ref: ${{ github.sha }} - name: Authenticate to Google Cloud uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f diff --git a/spartan/scripts/install_deps.sh b/spartan/scripts/install_deps.sh index f1fac04c4de9..536ccffd820a 100755 --- a/spartan/scripts/install_deps.sh +++ b/spartan/scripts/install_deps.sh @@ -134,6 +134,19 @@ if ! command -v cast &> /dev/null; then sudo chmod +x /usr/local/bin/cast fi +TERRAFORM_VERSION="1.7.5" +if ! command -v terraform &> /dev/null; then + log "Installing terraform ${TERRAFORM_VERSION}..." + tf_os="$(os)" + if [ "$tf_os" = "macos" ]; then + tf_os="darwin" + fi + curl -fsSL "https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_${tf_os}_$(arch).zip" -o terraform.zip + unzip -qo terraform.zip + sudo mv terraform /usr/local/bin/terraform + rm terraform.zip +fi + require_cmd git require_cmd kubectl require_cmd terraform diff --git a/spartan/scripts/source_network_env.sh b/spartan/scripts/source_network_env.sh index 9a99c22c5481..6257490e8149 100755 --- a/spartan/scripts/source_network_env.sh +++ b/spartan/scripts/source_network_env.sh @@ -24,6 +24,15 @@ function source_network_env { if grep -q "REPLACE_WITH_GCP_SECRET" "$env_file" && command -v gcloud &> /dev/null; then echo "Environment file contains GCP secret placeholders. Processing secrets..." + # Activate GCP credentials before Secret Manager reads (same order as network_deploy.sh). + if declare -f gcp_auth >/dev/null 2>&1; then + gcp_auth + else + # shellcheck source=scripts/gcp_auth.sh + source "$spartan/scripts/gcp_auth.sh" + gcp_auth + fi + # Process GCP secrets source $spartan/scripts/setup_gcp_secrets.sh "$env_file" From a4e7179bd874460b386e1fc5a0646e3e0a30693f Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Wed, 27 May 2026 15:08:43 +0100 Subject: [PATCH 04/27] chore: set eth node resource profile (#23583) Fix A-1104 --- .../terraform/deploy-ethereum-nodes/main.tf | 31 +++++++++--- .../deploy-ethereum-nodes/mainnet.tfvars | 14 +++++- .../deploy-ethereum-nodes/sepolia.tfvars | 14 +++++- .../deploy-ethereum-nodes/variables.tf | 48 +++++++++++++++++++ 4 files changed, 96 insertions(+), 11 deletions(-) diff --git a/spartan/terraform/deploy-ethereum-nodes/main.tf b/spartan/terraform/deploy-ethereum-nodes/main.tf index 5383738cc6e5..1df88be712d2 100644 --- a/spartan/terraform/deploy-ethereum-nodes/main.tf +++ b/spartan/terraform/deploy-ethereum-nodes/main.tf @@ -49,13 +49,6 @@ locals { # we have to mark it as non-sensitive otherwise we can't run for_each on helm_releases :( jwt = nonsensitive(random_bytes.jwt.hex) - resources = { - requests = { - cpu = 2 - memory = "60Gi" - } - } - nodeSelector = { "node-type" = "infra" } @@ -75,6 +68,18 @@ locals { repository = split(":", var.reth_image)[0] tag = split(":", var.reth_image)[1] } + + resources = { + requests = { + cpu = var.reth_cpu_request + memory = var.reth_memory_request + } + limits = { + cpu = var.reth_cpu_limit + memory = var.reth_memory_limit + } + } + p2pNodePort = { enabled = true port = var.reth_p2p_port @@ -105,6 +110,18 @@ locals { repository = split(":", var.lighthouse_image)[0] tag = split(":", var.lighthouse_image)[1] } + + resources = { + requests = { + cpu = var.lighthouse_cpu_request + memory = var.lighthouse_memory_request + } + limits = { + cpu = var.lighthouse_cpu_limit + memory = var.lighthouse_memory_limit + } + } + checkpointSync = { enabled = true url = var.checkpoint_sync_url diff --git a/spartan/terraform/deploy-ethereum-nodes/mainnet.tfvars b/spartan/terraform/deploy-ethereum-nodes/mainnet.tfvars index 089b015f251a..50f72d6568e3 100644 --- a/spartan/terraform/deploy-ethereum-nodes/mainnet.tfvars +++ b/spartan/terraform/deploy-ethereum-nodes/mainnet.tfvars @@ -1,7 +1,17 @@ namespace = "eth-mainnet" chain = "mainnet" checkpoint_sync_url = "https://mainnet.checkpoint.sigp.io" + reth_p2p_port = 32100 reth_storage = "4Ti" -lighthouse_p2p_port = 32101 -lighthouse_storage = "2Ti" +reth_cpu_request = "1" +reth_cpu_limit = "8" +reth_memory_request = "64Gi" +reth_memory_limit = "112Gi" + +lighthouse_p2p_port = 32101 +lighthouse_storage = "2Ti" +lighthouse_cpu_request = "1" +lighthouse_cpu_limit = "8" +lighthouse_memory_request = "24Gi" +lighthouse_memory_limit = "48Gi" diff --git a/spartan/terraform/deploy-ethereum-nodes/sepolia.tfvars b/spartan/terraform/deploy-ethereum-nodes/sepolia.tfvars index 3d049919ea69..bcae9e1c8389 100644 --- a/spartan/terraform/deploy-ethereum-nodes/sepolia.tfvars +++ b/spartan/terraform/deploy-ethereum-nodes/sepolia.tfvars @@ -1,7 +1,17 @@ namespace = "sepolia" chain = "sepolia" checkpoint_sync_url = "https://checkpoint-sync.sepolia.ethpandaops.io" + reth_p2p_port = 32000 reth_storage = "2Ti" -lighthouse_p2p_port = 32001 -lighthouse_storage = "2Ti" +reth_cpu_request = "1" +reth_cpu_limit = "4" +reth_memory_request = "12Gi" +reth_memory_limit = "24Gi" + +lighthouse_p2p_port = 32001 +lighthouse_storage = "2Ti" +lighthouse_cpu_request = "1" +lighthouse_cpu_limit = "4" +lighthouse_memory_request = "24Gi" +lighthouse_memory_limit = "48Gi" diff --git a/spartan/terraform/deploy-ethereum-nodes/variables.tf b/spartan/terraform/deploy-ethereum-nodes/variables.tf index 691cbbf8255b..5afc321b0da9 100644 --- a/spartan/terraform/deploy-ethereum-nodes/variables.tf +++ b/spartan/terraform/deploy-ethereum-nodes/variables.tf @@ -63,6 +63,30 @@ variable "reth_storage" { default = "4Ti" } +variable "reth_cpu_request" { + description = "Reth CPU requests" + type = string + default = "2" +} + +variable "reth_cpu_limit" { + description = "Reth CPU limit" + type = string + default = "4" +} + +variable "reth_memory_request" { + description = "Reth RAM requests" + type = string + default = "16Gi" +} + +variable "reth_memory_limit" { + description = "Reth RAM limit" + type = string + default = "32Gi" +} + variable "lighthouse_image" { description = "Lighthouse Docker image" type = string @@ -80,3 +104,27 @@ variable "lighthouse_storage" { type = string default = "1Ti" } + +variable "lighthouse_cpu_request" { + description = "Lighthouse CPU requests" + type = string + default = "2" +} + +variable "lighthouse_cpu_limit" { + description = "Lighthouse CPU limit" + type = string + default = "4" +} + +variable "lighthouse_memory_request" { + description = "Lighthouse RAM requests" + type = string + default = "16Gi" +} + +variable "lighthouse_memory_limit" { + description = "Lighthouse RAM limit" + type = string + default = "32Gi" +} From 3f9b67e187d09dfea18d71eab05b172555ebad1f Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 27 May 2026 11:36:38 -0300 Subject: [PATCH 05/27] fix: wait for checkpoint before sentinel assertions (#23573) ## Summary - Stabilizes the multiple-validator sentinel e2e by waiting for a post-warmup checkpoint before recording the assertion window. - Reuses the same warm-up helper in the second test so isolated runs avoid the same fresh-network startup noise before stopping a validator. ## Failed run Failed CI run: http://ci.aztec-labs.com/07fb31bc0706159f The failing test was `e2e_p2p_multiple_validators_sentinel > collects attestations for all validators on a node`. The test expected no `attestation-missed` entries, but the assertion window started while the network was still in the first pipelined slots after startup. In the failed run, slot 8 was built on a pending, not-yet-checkpointed parent, so some remote validators could not validate/attest in time and the sentinel recorded a missed attestation. ## Fix The test now waits for one warm-up slot and then waits for the observed checkpoint number to advance before capturing `initialSlot`. That keeps startup pipelining behavior out of the strict sentinel assertion window while preserving the test's actual coverage: once the network is past warm-up, every validator should be observed attesting or proposing as expected. ## Verification - `yarn format end-to-end` - `yarn build` - `yarn workspace @aztec/end-to-end test:e2e e2e_p2p/multiple_validators_sentinel.parallel.test.ts -t 'collects attestations for all validators on a node'` --- ...tiple_validators_sentinel.parallel.test.ts | 28 ++++++++++++++----- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/yarn-project/end-to-end/src/e2e_p2p/multiple_validators_sentinel.parallel.test.ts b/yarn-project/end-to-end/src/e2e_p2p/multiple_validators_sentinel.parallel.test.ts index eb907ce5571e..6d984cefb9b9 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/multiple_validators_sentinel.parallel.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/multiple_validators_sentinel.parallel.test.ts @@ -104,21 +104,33 @@ describe('e2e_p2p_multiple_validators_sentinel', () => { } }); - it('collects attestations for all validators on a node', async () => { - // Ensure all nodes see each other, especially the sentinel, before starting slot counting - await t.waitForP2PMeshConnectivity([...nodes, sentinel]); - - // Wait until validator nodes have advanced past their first proposed slot so that the - // pipelining warm-up period (where some attestations may be missed) is behind us. + const waitForPostWarmupCheckpoint = async (action: string): Promise => { await t.monitor.run(); const warmupSlot = Number(t.monitor.l2SlotNumber) + 1; - t.logger.info(`Waiting for warmup slot ${warmupSlot} before establishing initial slot`); + t.logger.info(`Waiting for warmup slot ${warmupSlot} before ${action}`); await retryUntil( async () => (await t.monitor.run()).l2SlotNumber >= warmupSlot, 'warmup slot', AZTEC_SLOT_DURATION * 3, ); + const warmupCheckpoint = t.monitor.checkpointNumber; + t.logger.info(`Waiting for checkpoint after warmup before ${action}`, { warmupCheckpoint }); + await retryUntil( + async () => (await t.monitor.run()).checkpointNumber > warmupCheckpoint, + 'post-warmup checkpoint', + AZTEC_SLOT_DURATION * (SLOT_COUNT + 1) * 3, + ); + }; + + it('collects attestations for all validators on a node', async () => { + // Ensure all nodes see each other, especially the sentinel, before starting slot counting + await t.waitForP2PMeshConnectivity([...nodes, sentinel]); + + // Wait until validator nodes have advanced past their first proposed slot and landed a checkpoint so that the + // pipelining warm-up period (where some attestations may be missed) is behind us. + await waitForPostWarmupCheckpoint('establishing initial slot'); + const { checkpointNumber: initialBlock, l2SlotNumber: initialSlot } = t.monitor; const timeout = AZTEC_SLOT_DURATION * SLOT_COUNT * 4; @@ -156,6 +168,8 @@ describe('e2e_p2p_multiple_validators_sentinel', () => { // Ensure all nodes see each other, especially the sentinel await t.waitForP2PMeshConnectivity([...nodes, sentinel]); + await waitForPostWarmupCheckpoint('stopping a validator node and establishing initial slot'); + // Stop the second node, this means the first node won't be able to propose since won't achieve quorum await tryStop(nodes[1]); From a30432546dcf98a88c1c79bd9b75aee66941cb79 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Wed, 27 May 2026 15:39:17 +0100 Subject: [PATCH 06/27] fix: slash attestations for invalid checkpoint proposals (#23506) --- .../aztec-node/src/aztec-node/server.test.ts | 15 +- .../aztec-node/src/aztec-node/server.ts | 62 ++++--- ...asted_invalid_block_proposal_slash.test.ts | 24 +-- .../attested_invalid_proposal.test.ts | 41 ++++- yarn-project/slasher/src/index.ts | 1 + .../attested_invalid_proposal_watcher.test.ts | 142 ++++++++++++++++ .../attested_invalid_proposal_watcher.ts | 159 ++++++++++++++++++ yarn-project/txe/src/state_machine/index.ts | 3 +- .../validator-client/src/validator.test.ts | 138 ++++++++------- .../validator-client/src/validator.ts | 79 ++------- 10 files changed, 479 insertions(+), 185 deletions(-) create mode 100644 yarn-project/slasher/src/watchers/attested_invalid_proposal_watcher.test.ts create mode 100644 yarn-project/slasher/src/watchers/attested_invalid_proposal_watcher.ts diff --git a/yarn-project/aztec-node/src/aztec-node/server.test.ts b/yarn-project/aztec-node/src/aztec-node/server.test.ts index 462fa2cba60e..6c35abec8370 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.test.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.test.ts @@ -222,8 +222,7 @@ describe('aztec node', () => { undefined, undefined, undefined, - undefined, - undefined, + async () => {}, 12345, rollupVersion.toNumber(), globalVariablesBuilder, @@ -739,8 +738,7 @@ describe('aztec node', () => { undefined, slasherClient, undefined, - undefined, - undefined, + async () => {}, 12345, rollupVersion.toNumber(), globalVariablesBuilder, @@ -930,8 +928,7 @@ describe('aztec node', () => { undefined, slasherClient, undefined, - undefined, - undefined, + async () => {}, 12345, rollupVersion.toNumber(), globalVariablesBuilder, @@ -1002,8 +999,7 @@ describe('aztec node', () => { undefined, undefined, undefined, - undefined, - undefined, + async () => {}, 12345, rollupVersion.toNumber(), globalVariablesBuilder, @@ -1056,8 +1052,7 @@ describe('aztec node', () => { undefined, undefined, undefined, - undefined, - undefined, + async () => {}, 12345, rollupVersion.toNumber(), globalVariablesBuilder, diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index 88b8e03a6e98..f3070003ed83 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -45,6 +45,7 @@ import { import { PublicContractsDB, PublicProcessorFactory } from '@aztec/simulator/server'; import { AttestationsBlockWatcher, + AttestedInvalidProposalWatcher, BroadcastedInvalidCheckpointProposalWatcher, CheckpointEquivocationWatcher, DataWithholdingWatcher, @@ -186,8 +187,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, AztecNodeDeb protected readonly proverNode: ProverNode | undefined, protected readonly slasherClient: SlasherClientInterface | undefined, protected readonly validatorsSentinel: Sentinel | undefined, - protected readonly dataWithholdingWatcher: DataWithholdingWatcher | undefined, - protected readonly attestationsBlockWatcher: AttestationsBlockWatcher | undefined, + private readonly stopStartedWatchers: () => Promise, protected readonly l1ChainId: number, protected readonly version: number, protected readonly globalVariableBuilder: GlobalVariableBuilderInterface, @@ -737,6 +737,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, AztecNodeDeb let validatorsSentinel: Awaited> | undefined; let dataWithholdingWatcher: DataWithholdingWatcher | undefined; let attestationsBlockWatcher: AttestationsBlockWatcher | undefined; + let attestedInvalidProposalWatcher: AttestedInvalidProposalWatcher | undefined; let broadcastedInvalidCheckpointProposalWatcher: BroadcastedInvalidCheckpointProposalWatcher | undefined; let checkpointEquivocationWatcher: CheckpointEquivocationWatcher | undefined; @@ -769,6 +770,18 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, AztecNodeDeb watchers.push(broadcastedInvalidCheckpointProposalWatcher); } + if (validatorClient && config.slashAttestInvalidCheckpointProposalPenalty > 0n) { + attestedInvalidProposalWatcher = new AttestedInvalidProposalWatcher( + p2pClient, + validatorClient, + archiver, + epochCache, + config, + { log: log.createChild('attested-invalid-proposal-watcher') }, + ); + watchers.push(attestedInvalidProposalWatcher); + } + if (config.slashDuplicateProposalPenalty > 0n) { checkpointEquivocationWatcher = new CheckpointEquivocationWatcher(archiver, epochCache, config); watchers.push(checkpointEquivocationWatcher); @@ -784,33 +797,33 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, AztecNodeDeb } } + const watchersToStart = compactArray([ + validatorsSentinel, + dataWithholdingWatcher, + attestationsBlockWatcher, + broadcastedInvalidCheckpointProposalWatcher, + attestedInvalidProposalWatcher, + checkpointEquivocationWatcher, + ]); + const startedWatchers: Watcher[] = []; + const stopStartedWatchers = async () => { + for (const watcher of startedWatchers) { + await tryStop(watcher); + } + }; + // Start p2p-related services once the archiver has completed sync void archiver .waitForInitialSync() .then(async () => { - if (validatorsSentinel) { - await validatorsSentinel.start(); - started.push(validatorsSentinel); - } - if (dataWithholdingWatcher) { - await dataWithholdingWatcher.start(); - started.push(dataWithholdingWatcher); - } - if (attestationsBlockWatcher) { - await attestationsBlockWatcher.start(); - started.push(attestationsBlockWatcher); - } - if (broadcastedInvalidCheckpointProposalWatcher) { - await broadcastedInvalidCheckpointProposalWatcher.start(); - started.push(broadcastedInvalidCheckpointProposalWatcher); - } - if (checkpointEquivocationWatcher) { - await checkpointEquivocationWatcher.start(); - started.push(checkpointEquivocationWatcher); + for (const watcher of watchersToStart) { + await watcher.start(); + startedWatchers.push(watcher); } log.info(`All p2p services started`); }) .catch(err => log.error('Failed to start p2p services after archiver sync', err)); + started.push({ stop: stopStartedWatchers }); // Validator enabled, create/start relevant service let sequencer: SequencerClient | undefined; @@ -976,8 +989,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, AztecNodeDeb proverNode, slasherClient, validatorsSentinel, - dataWithholdingWatcher, - attestationsBlockWatcher, + stopStartedWatchers, ethereumChain.chainInfo.id, config.rollupVersion, globalVariableBuilder, @@ -1259,9 +1271,7 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, AztecNodeDeb */ public async stop() { this.log.info(`Stopping Aztec Node`); - await tryStop(this.attestationsBlockWatcher); - await tryStop(this.validatorsSentinel); - await tryStop(this.dataWithholdingWatcher); + await this.stopStartedWatchers(); await tryStop(this.slasherClient); await Promise.all([tryStop(this.peerProofVerifier), tryStop(this.rpcProofVerifier)]); await tryStop(this.sequencer); diff --git a/yarn-project/end-to-end/src/e2e_p2p/broadcasted_invalid_block_proposal_slash.test.ts b/yarn-project/end-to-end/src/e2e_p2p/broadcasted_invalid_block_proposal_slash.test.ts index 2915bca2ce86..335ecc5aec4e 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/broadcasted_invalid_block_proposal_slash.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/broadcasted_invalid_block_proposal_slash.test.ts @@ -14,7 +14,7 @@ import path from 'path'; import { shouldCollectMetrics } from '../fixtures/fixtures.js'; import { createNodes } from '../fixtures/setup_p2p_test.js'; import { P2PNetworkTest } from './p2p_network.js'; -import { advanceToEpochBeforeProposer, awaitCommitteeExists, awaitOffenseDetected } from './shared.js'; +import { advanceToEpochBeforeProposer, awaitCommitteeExists } from './shared.js'; const TEST_TIMEOUT = 1_000_000; @@ -141,7 +141,7 @@ describe('e2e_p2p_broadcasted_invalid_block_proposal_slash', () => { // Create remaining honest nodes, also with sequencers stopped, for the same reason. const honestNodes = await createNodes( - { ...t.ctx.aztecNodeConfig, dontStartSequencer: true }, + { ...t.ctx.aztecNodeConfig, dontStartSequencer: true, skipBroadcastProposals: true }, t.ctx.dateProvider, t.bootstrapNodeEnr, NUM_VALIDATORS - 1, @@ -183,18 +183,8 @@ describe('e2e_p2p_broadcasted_invalid_block_proposal_slash', () => { // Wait for offense to be detected. Under proposer pipelining, the invalid block proposal is // broadcast at the slot boundary while a receiver's wall clock may have already advanced - // past the build slot — when that happens, the honest node rejects the gossip with "invalid - // slot number" before slashing logic runs. Collect offenses from every node so we catch - // whichever node managed to process the proposal while still in the build slot. - await awaitOffenseDetected({ - epochDuration: t.ctx.aztecNodeConfig.aztecEpochDuration, - logger: t.logger, - nodeAdmin: nodes[1], // Use honest node to check for offenses - slashingRoundSize, - waitUntilOffenseCount: 1, - timeoutSeconds: AZTEC_SLOT_DURATION * 16, - }); - + // past the build slot. Honest sequencers are running so their validator clients emit offenses, + // but they do not broadcast proposals until after the offense is detected. const invalidBlockOffenses = await retryUntil( async () => { const allOffenses = (await Promise.all(nodes.map(n => n.getSlashOffenses('all')))).flat(); @@ -204,7 +194,7 @@ describe('e2e_p2p_broadcasted_invalid_block_proposal_slash', () => { } }, 'broadcasted invalid block proposal offense', - AZTEC_SLOT_DURATION * 4, + AZTEC_SLOT_DURATION * 16, ); t.logger.warn(`Collected broadcasted invalid block proposal offenses`, { invalidBlockOffenses }); @@ -219,6 +209,10 @@ describe('e2e_p2p_broadcasted_invalid_block_proposal_slash', () => { t.logger.warn(`Slashed ${args.attester.toString()}`); slashPromise.resolve(args); }); + + t.logger.warn('Re-enabling honest proposal broadcasts'); + await Promise.all(honestNodes.map(n => n.setConfig({ skipBroadcastProposals: false }))); + const { amount, attester } = await slashPromise.promise; expect(invalidProposerAddress.toString()).toEqual(attester.toString()); expect(amount).toEqual(slashingAmount); diff --git a/yarn-project/end-to-end/src/e2e_slashing/attested_invalid_proposal.test.ts b/yarn-project/end-to-end/src/e2e_slashing/attested_invalid_proposal.test.ts index 3dc0f67ad3e5..fe7b1bda51f0 100644 --- a/yarn-project/end-to-end/src/e2e_slashing/attested_invalid_proposal.test.ts +++ b/yarn-project/end-to-end/src/e2e_slashing/attested_invalid_proposal.test.ts @@ -202,7 +202,15 @@ describe('e2e_slashing_attested_invalid_proposal', () => { async function createInvalidProposalSlashingScenario({ badProposerConfig = {}, - }: { badProposerConfig?: Partial[0]> } = {}) { + corruptBlockProposal = true, + expectBadProposerOffense = true, + expectedBadProposerOffenseType = OffenseType.BROADCASTED_INVALID_BLOCK_PROPOSAL, + }: { + badProposerConfig?: Partial[0]>; + corruptBlockProposal?: boolean; + expectBadProposerOffense?: boolean; + expectedBadProposerOffenseType?: OffenseType; + } = {}) { const { rollup } = await t.getContracts(); await t.ctx.cheatCodes.rollup.advanceToEpoch(EpochNumber(4)); @@ -212,7 +220,9 @@ describe('e2e_slashing_attested_invalid_proposal', () => { { ...t.ctx.aztecNodeConfig, dontStartSequencer: true, - invalidBlockProposalIndexWithinCheckpoint: BAD_BLOCK_INDEX_WITHIN_CHECKPOINT, + ...(corruptBlockProposal + ? { invalidBlockProposalIndexWithinCheckpoint: BAD_BLOCK_INDEX_WITHIN_CHECKPOINT } + : {}), ...badProposerConfig, }, t.ctx.dateProvider!, @@ -389,11 +399,18 @@ describe('e2e_slashing_attested_invalid_proposal', () => { }); const expectedSlashOffenses = [ - { - description: 'bad proposer broadcasted invalid block proposal', - validator: badProposer, - offenseType: OffenseType.BROADCASTED_INVALID_BLOCK_PROPOSAL, - }, + ...(expectBadProposerOffense + ? [ + { + description: + expectedBadProposerOffenseType === OffenseType.BROADCASTED_INVALID_CHECKPOINT_PROPOSAL + ? 'bad proposer broadcasted invalid checkpoint proposal' + : 'bad proposer broadcasted invalid block proposal', + validator: badProposer, + offenseType: expectedBadProposerOffenseType, + }, + ] + : []), { description: 'lazy validator attested to invalid checkpoint proposal', validator: lazyValidator, @@ -439,6 +456,16 @@ describe('e2e_slashing_attested_invalid_proposal', () => { }; } + it('slashes a lazy attester for an invalid checkpoint proposal', async () => { + await createInvalidProposalSlashingScenario({ + badProposerConfig: { + broadcastInvalidCheckpointProposalOnly: true, + }, + corruptBlockProposal: false, + expectBadProposerOffense: false, + }); + }); + it('slashes a lazy attester for an invalid checkpoint and clears it on delayed equivocation', async () => { const { rollup, diff --git a/yarn-project/slasher/src/index.ts b/yarn-project/slasher/src/index.ts index d947aad82568..d383a639287f 100644 --- a/yarn-project/slasher/src/index.ts +++ b/yarn-project/slasher/src/index.ts @@ -1,6 +1,7 @@ export * from './config.js'; export * from './watchers/data_withholding_watcher.js'; export * from './watchers/attestations_block_watcher.js'; +export * from './watchers/attested_invalid_proposal_watcher.js'; export * from './watchers/broadcasted_invalid_checkpoint_proposal_watcher.js'; export * from './watchers/checkpoint_equivocation_watcher.js'; export * from './slasher_client.js'; diff --git a/yarn-project/slasher/src/watchers/attested_invalid_proposal_watcher.test.ts b/yarn-project/slasher/src/watchers/attested_invalid_proposal_watcher.test.ts new file mode 100644 index 000000000000..1c377785d01b --- /dev/null +++ b/yarn-project/slasher/src/watchers/attested_invalid_proposal_watcher.test.ts @@ -0,0 +1,142 @@ +import type { EpochCacheInterface } from '@aztec/epoch-cache'; +import { SlotNumber } from '@aztec/foundation/branded-types'; +import { Secp256k1Signer } from '@aztec/foundation/crypto/secp256k1-signer'; +import type { L2BlockSource } from '@aztec/stdlib/block'; +import type { P2PClient } from '@aztec/stdlib/interfaces/server'; +import type { CheckpointAttestation } from '@aztec/stdlib/p2p'; +import { OffenseType } from '@aztec/stdlib/slashing'; +import { + makeCheckpointAttestationFromProposal, + makeCheckpointHeader, + makeCheckpointProposal, +} from '@aztec/stdlib/testing'; + +import { jest } from '@jest/globals'; +import { type MockProxy, mock } from 'jest-mock-extended'; + +import { DefaultSlasherConfig, type SlasherConfig } from '../config.js'; +import { WANT_TO_SLASH_EVENT, type WantToSlashArgs } from '../watcher.js'; +import { AttestedInvalidProposalWatcher, type InvalidProposalSlotSource } from './attested_invalid_proposal_watcher.js'; + +describe('AttestedInvalidProposalWatcher', () => { + let p2pClient: MockProxy>; + let l2BlockSource: MockProxy>; + let epochCache: MockProxy>; + let invalidProposalSlots: Set; + let proposalEquivocationSlots: Set; + let invalidProposalSlotSource: InvalidProposalSlotSource; + let config: SlasherConfig; + let watcher: AttestedInvalidProposalWatcher; + let handler: jest.MockedFunction<(args: WantToSlashArgs[]) => void>; + + beforeEach(() => { + p2pClient = mock>(); + p2pClient.getCheckpointAttestationsForSlot.mockResolvedValue([]); + l2BlockSource = mock>(); + l2BlockSource.getSyncedL2SlotNumber.mockResolvedValue(SlotNumber(11)); + epochCache = mock>(); + epochCache.getSlotNow.mockReturnValue(SlotNumber(11)); + epochCache.getL1Constants.mockReturnValue({ ethereumSlotDuration: 4 } as ReturnType< + EpochCacheInterface['getL1Constants'] + >); + invalidProposalSlots = new Set(); + proposalEquivocationSlots = new Set(); + invalidProposalSlotSource = { + hasInvalidProposals: slot => invalidProposalSlots.has(slot), + hasProposalEquivocation: slot => proposalEquivocationSlots.has(slot), + }; + config = { + ...DefaultSlasherConfig, + slashAttestInvalidCheckpointProposalPenalty: 13n, + }; + watcher = new AttestedInvalidProposalWatcher( + p2pClient, + invalidProposalSlotSource, + l2BlockSource, + epochCache, + config, + ); + handler = jest.fn(); + watcher.on(WANT_TO_SLASH_EVENT, handler); + }); + + const makeAttestation = async ( + slot: SlotNumber, + attesterSigner = Secp256k1Signer.random(), + ): Promise => { + const checkpointProposal = await makeCheckpointProposal({ + checkpointHeader: makeCheckpointHeader(1, { slotNumber: slot }), + }); + return makeCheckpointAttestationFromProposal(checkpointProposal, attesterSigner); + }; + + it('slashes checkpoint attestations already in the pool for a marked invalid proposal slot', async () => { + const slot = SlotNumber(10); + const attesterSigner = Secp256k1Signer.random(); + invalidProposalSlots.add(slot); + p2pClient.getCheckpointAttestationsForSlot.mockResolvedValue([await makeAttestation(slot, attesterSigner)]); + + await watcher.scanSlot(slot); + + expect(handler).toHaveBeenCalledWith([ + { + validator: attesterSigner.address, + amount: 13n, + offenseType: OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL, + epochOrSlot: 10n, + }, + ]); + }); + + it('scans only marked invalid proposal slots once they are past the scan lag', async () => { + watcher = new AttestedInvalidProposalWatcher( + p2pClient, + invalidProposalSlotSource, + l2BlockSource, + epochCache, + config, + { scanSlotLookback: 2 }, + ); + invalidProposalSlots = new Set([SlotNumber(8), SlotNumber(9), SlotNumber(10)]); + + await watcher.scan(); + + expect(p2pClient.getCheckpointAttestationsForSlot.mock.calls.map(([slot]) => slot)).toEqual([ + SlotNumber(9), + SlotNumber(10), + ]); + }); + + it('does not rescan completed slots', async () => { + invalidProposalSlots = new Set([SlotNumber(9), SlotNumber(10)]); + + await watcher.scan(); + await watcher.scan(); + + expect(p2pClient.getCheckpointAttestationsForSlot.mock.calls.map(([slot]) => slot)).toEqual([ + SlotNumber(9), + SlotNumber(10), + ]); + + l2BlockSource.getSyncedL2SlotNumber.mockResolvedValue(SlotNumber(12)); + invalidProposalSlots = new Set([SlotNumber(9), SlotNumber(10), SlotNumber(11)]); + await watcher.scan(); + + expect(p2pClient.getCheckpointAttestationsForSlot.mock.calls.map(([slot]) => slot)).toEqual([ + SlotNumber(9), + SlotNumber(10), + SlotNumber(11), + ]); + }); + + it('does not slash attestations once proposal equivocation has been detected for the slot', async () => { + const slot = SlotNumber(10); + invalidProposalSlots.add(slot); + proposalEquivocationSlots.add(slot); + p2pClient.getCheckpointAttestationsForSlot.mockResolvedValue([await makeAttestation(slot)]); + + await watcher.scanSlot(slot); + + expect(handler).not.toHaveBeenCalled(); + }); +}); diff --git a/yarn-project/slasher/src/watchers/attested_invalid_proposal_watcher.ts b/yarn-project/slasher/src/watchers/attested_invalid_proposal_watcher.ts new file mode 100644 index 000000000000..400b47b863a8 --- /dev/null +++ b/yarn-project/slasher/src/watchers/attested_invalid_proposal_watcher.ts @@ -0,0 +1,159 @@ +import type { EpochCacheInterface } from '@aztec/epoch-cache'; +import { SlotNumber } from '@aztec/foundation/branded-types'; +import { merge, pick } from '@aztec/foundation/collection'; +import type { EthAddress } from '@aztec/foundation/eth-address'; +import { type Logger, createLogger } from '@aztec/foundation/log'; +import { RunningPromise } from '@aztec/foundation/running-promise'; +import type { L2BlockSource } from '@aztec/stdlib/block'; +import type { P2PClient, SlasherConfig } from '@aztec/stdlib/interfaces/server'; +import type { CheckpointAttestation } from '@aztec/stdlib/p2p'; +import { OffenseType } from '@aztec/stdlib/slashing'; + +import EventEmitter from 'node:events'; + +import { WANT_TO_SLASH_EVENT, type WantToSlashArgs, type Watcher, type WatcherEmitter } from '../watcher.js'; + +const AttestedInvalidProposalWatcherConfigKeys = ['slashAttestInvalidCheckpointProposalPenalty'] as const; + +const SCAN_SLOT_LAG = 1; +const DEFAULT_SCAN_SLOT_LOOKBACK = 4; + +type AttestedInvalidProposalWatcherConfig = Pick< + SlasherConfig, + (typeof AttestedInvalidProposalWatcherConfigKeys)[number] +>; + +type P2PCheckpointAttestationSource = Pick; + +type AttestedInvalidProposalWatcherOptions = { + scanSlotLookback?: number; + log?: Logger; +}; + +export type InvalidProposalSlotSource = { + hasInvalidProposals(slot: SlotNumber): boolean; + hasProposalEquivocation(slot: SlotNumber): boolean; +}; + +export class AttestedInvalidProposalWatcher extends (EventEmitter as new () => WatcherEmitter) implements Watcher { + private readonly log: Logger; + private readonly runningPromise: RunningPromise; + private readonly scanSlotLookback: number; + private config: AttestedInvalidProposalWatcherConfig; + private lastScannedSlot: SlotNumber | undefined; + + constructor( + private readonly p2pClient: P2PCheckpointAttestationSource, + private readonly invalidProposalSlotSource: InvalidProposalSlotSource, + private readonly l2BlockSource: Pick, + private readonly epochCache: Pick, + config: AttestedInvalidProposalWatcherConfig, + options: AttestedInvalidProposalWatcherOptions = {}, + ) { + super(); + const constants = epochCache.getL1Constants(); + this.log = options.log ?? createLogger('attested-invalid-proposal-watcher'); + this.config = pick(config, ...AttestedInvalidProposalWatcherConfigKeys); + this.scanSlotLookback = Math.max(1, options.scanSlotLookback ?? DEFAULT_SCAN_SLOT_LOOKBACK); + + const intervalMs = Math.max(1000, (constants.ethereumSlotDuration * 1000) / 4); + this.runningPromise = new RunningPromise(() => this.scan(), this.log, intervalMs); + this.log.info('AttestedInvalidProposalWatcher initialized', { scanSlotLookback: this.scanSlotLookback }); + } + + public updateConfig(config: Partial): void { + this.config = merge(this.config, pick(config, ...AttestedInvalidProposalWatcherConfigKeys)); + this.log.verbose('AttestedInvalidProposalWatcher config updated', this.config); + } + + public start(): Promise { + this.runningPromise.start(); + return Promise.resolve(); + } + + public stop(): Promise { + return this.runningPromise.stop(); + } + + public async scan(): Promise { + if (this.config.slashAttestInvalidCheckpointProposalPenalty <= 0n) { + return; + } + + const currentSlot = (await this.l2BlockSource.getSyncedL2SlotNumber()) ?? this.epochCache.getSlotNow(); + // genesis + if (currentSlot <= SlotNumber(SCAN_SLOT_LAG)) { + return; + } + + const newestSlotToConsider = SlotNumber(currentSlot - SCAN_SLOT_LAG); + const oldestSlot = + this.lastScannedSlot === undefined + ? SlotNumber(Math.max(0, newestSlotToConsider - this.scanSlotLookback + 1)) + : SlotNumber(this.lastScannedSlot + 1); + if (oldestSlot > newestSlotToConsider) { + return; + } + + for (let slot = oldestSlot; slot <= newestSlotToConsider; slot++) { + await this.scanSlot(slot); + } + + this.lastScannedSlot = newestSlotToConsider; + } + + /** Scans a single invalid-proposal slot. */ + public async scanSlot(slot: SlotNumber): Promise { + if ( + this.config.slashAttestInvalidCheckpointProposalPenalty <= 0n || + this.invalidProposalSlotSource.hasProposalEquivocation(slot) || + !this.invalidProposalSlotSource.hasInvalidProposals(slot) + ) { + return; + } + + let attestations: CheckpointAttestation[]; + try { + attestations = await this.p2pClient.getCheckpointAttestationsForSlot(slot); + } catch (err) { + this.log.warn('Error getting checkpoint attestations for invalid proposal slot', { err, slot }); + return; + } + + const slashArgs = attestations + .map(attestation => this.getSlashArgs(slot, attestation)) + .filter((args): args is WantToSlashArgs => args !== undefined); + + if (slashArgs.length === 0) { + return; + } + + this.log.warn('Slashing attesters for attesting to invalid checkpoint proposal', { + slot, + attesters: slashArgs.map(args => args.validator.toString()), + }); + this.emit(WANT_TO_SLASH_EVENT, slashArgs); + } + + private getSlashArgs(slot: SlotNumber, attestation: CheckpointAttestation): WantToSlashArgs | undefined { + const attester = attestation.getSender(); + if (!attester) { + this.log.warn('Cannot slash checkpoint attestation with invalid signature', { + slot, + archive: attestation.archive.toString(), + }); + return undefined; + } + + return this.getSlashArgsForAttester(slot, attester); + } + + private getSlashArgsForAttester(slot: SlotNumber, attester: EthAddress): WantToSlashArgs { + return { + validator: attester, + amount: this.config.slashAttestInvalidCheckpointProposalPenalty, + offenseType: OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL, + epochOrSlot: BigInt(slot), + }; + } +} diff --git a/yarn-project/txe/src/state_machine/index.ts b/yarn-project/txe/src/state_machine/index.ts index 746f79f0b9dd..9aeb59c21046 100644 --- a/yarn-project/txe/src/state_machine/index.ts +++ b/yarn-project/txe/src/state_machine/index.ts @@ -52,8 +52,7 @@ export class TXEStateMachine { undefined, undefined, undefined, - undefined, - undefined, + async () => {}, VERSION, CHAIN_ID, new TXEGlobalVariablesBuilder(), diff --git a/yarn-project/validator-client/src/validator.test.ts b/yarn-project/validator-client/src/validator.test.ts index d582a73c3575..d344399a14ac 100644 --- a/yarn-project/validator-client/src/validator.test.ts +++ b/yarn-project/validator-client/src/validator.test.ts @@ -417,7 +417,6 @@ describe('ValidatorClient', () => { Array.isArray(args) && args[0]?.offenseType === OffenseType.BROADCASTED_INVALID_CHECKPOINT_PROPOSAL, ); - beforeEach(async () => { const emptyInHash = computeInHashFromL1ToL2Messages([]); const blockHeader = makeBlockHeader(1, { blockNumber: BlockNumber(100), slotNumber: SlotNumber(100) }); @@ -909,42 +908,17 @@ describe('ValidatorClient', () => { expect(emitSpy).not.toHaveBeenCalled(); }); - it('slashes checkpoint attestations received after an invalid proposal slot is marked only once', async () => { - await validatorClient.registerHandlers(); - const attestationCallback = p2pClient.registerCheckpointAttestationCallback.mock.calls[0][0]; - const emitSpy = jest.spyOn(validatorClient, 'emit'); + it('marks invalid block proposal slots for delayed attestation slashing', async () => { blockBuildResult.block.archive.root = Fr.random(); - await validatorClient.validateBlockProposal(proposal, sender); - - const attesterSigner = Secp256k1Signer.random(); - const attestation = makeCheckpointAttestation({ - header: makeCheckpointHeader(1, { slotNumber: proposal.slotNumber }), - attesterSigner, - }); - attestationCallback(attestation); - attestationCallback(attestation); + const isValid = await validatorClient.validateBlockProposal(proposal, sender); - const badAttestationEvents = emitSpy.mock.calls.filter( - ([event, args]) => - event === WANT_TO_SLASH_EVENT && - Array.isArray(args) && - args[0]?.offenseType === OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL, - ); - expect(badAttestationEvents).toHaveLength(1); - expect(badAttestationEvents[0][1]).toEqual([ - { - validator: attesterSigner.address, - amount: config.slashAttestInvalidCheckpointProposalPenalty, - offenseType: OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL, - epochOrSlot: BigInt(proposal.slotNumber), - }, - ]); + expect(isValid).toBe(false); + expect(validatorClient.hasInvalidProposals(proposal.slotNumber)).toBe(true); }); - it('clears and suppresses bad attestation offenses when proposal equivocation is detected', async () => { + it('records proposal equivocation and emits clear event', async () => { await validatorClient.registerHandlers(); - const attestationCallback = p2pClient.registerCheckpointAttestationCallback.mock.calls[0][0]; const duplicateProposalCallback = p2pClient.registerDuplicateProposalCallback.mock.calls[0][0]; const emitSpy = jest.spyOn(validatorClient, 'emit'); blockBuildResult.block.archive.root = Fr.random(); @@ -956,58 +930,39 @@ describe('ValidatorClient', () => { type: 'block', }); - const attestation = makeCheckpointAttestation({ - header: makeCheckpointHeader(1, { slotNumber: proposal.slotNumber }), - attesterSigner: Secp256k1Signer.random(), - }); - attestationCallback(attestation); - + expect(validatorClient.hasProposalEquivocation(proposal.slotNumber)).toBe(true); expect(emitSpy).toHaveBeenCalledWith(WANT_TO_CLEAR_SLASH_EVENT, [ { offenseType: OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL, epochOrSlot: BigInt(proposal.slotNumber), }, ]); - expect( - emitSpy.mock.calls.some( - ([event, args]) => - event === WANT_TO_SLASH_EVENT && - Array.isArray(args) && - args[0]?.offenseType === OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL, - ), - ).toBe(false); }); - it('reexecutes for bad attestation slashing when invalid block proposer slashing is disabled', async () => { - validatorClient.updateConfig({ slashBroadcastedInvalidBlockPenalty: 0n }); - epochCache.filterInCommittee.mockResolvedValue([]); + it('does not mark invalid proposal slots when the bad attestation penalty is disabled', async () => { + validatorClient.updateConfig({ + slashBroadcastedInvalidBlockPenalty: 0n, + slashAttestInvalidCheckpointProposalPenalty: 0n, + }); + const emitSpy = jest.spyOn(validatorClient, 'emit'); blockBuildResult.block.archive.root = Fr.random(); const isValid = await validatorClient.validateBlockProposal(proposal, sender); expect(isValid).toBe(false); - expect(checkpointsBuilder.openCheckpoint).toHaveBeenCalled(); + expect(validatorClient.hasInvalidProposals(proposal.slotNumber)).toBe(false); + expect(emitSpy).not.toHaveBeenCalled(); }); - it('does not emit bad attestation offenses when the bad attestation penalty is disabled', async () => { - await validatorClient.registerHandlers(); - const attestationCallback = p2pClient.registerCheckpointAttestationCallback.mock.calls[0][0]; - validatorClient.updateConfig({ - slashBroadcastedInvalidBlockPenalty: 0n, - slashAttestInvalidCheckpointProposalPenalty: 0n, - }); - const emitSpy = jest.spyOn(validatorClient, 'emit'); - const attestation = makeCheckpointAttestation({ - header: makeCheckpointHeader(1, { slotNumber: proposal.slotNumber }), - attesterSigner: Secp256k1Signer.random(), - }); + it('reexecutes for bad attestation slashing when invalid block proposer slashing is disabled', async () => { + validatorClient.updateConfig({ slashBroadcastedInvalidBlockPenalty: 0n }); + epochCache.filterInCommittee.mockResolvedValue([]); blockBuildResult.block.archive.root = Fr.random(); const isValid = await validatorClient.validateBlockProposal(proposal, sender); - attestationCallback(attestation); expect(isValid).toBe(false); - expect(emitSpy).not.toHaveBeenCalled(); + expect(checkpointsBuilder.openCheckpoint).toHaveBeenCalled(); }); it('emits WANT_TO_SLASH_EVENT for checkpoint_header_mismatch checkpoint proposals', async () => { @@ -1127,6 +1082,63 @@ describe('ValidatorClient', () => { expect(getBroadcastedInvalidCheckpointProposalSlashEvents(emitSpy)).toHaveLength(1); }); + it('marks invalid checkpoint proposal slots for delayed attestation slashing', async () => { + const checkpointHandler = registerAllNodesCheckpointHandler(); + const { checkpointProposal } = await makeCheckpointProposalWithHeaderMismatch(); + + await checkpointHandler(checkpointProposal, sender); + + expect(validatorClient.hasInvalidProposals(checkpointProposal.slotNumber)).toBe(true); + }); + + it('marks invalid checkpoint proposal slots when proposer slashing is disabled', async () => { + validatorClient.updateConfig({ slashBroadcastedInvalidCheckpointProposalPenalty: 0n }); + const checkpointHandler = registerAllNodesCheckpointHandler(); + const { checkpointProposal } = await makeCheckpointProposalWithHeaderMismatch(); + const emitSpy = jest.spyOn(validatorClient, 'emit'); + + await checkpointHandler(checkpointProposal, sender); + + expect(getBroadcastedInvalidCheckpointProposalSlashEvents(emitSpy)).toHaveLength(0); + expect(validatorClient.hasInvalidProposals(checkpointProposal.slotNumber)).toBe(true); + }); + + it('records checkpoint proposal equivocation and emits clear event', async () => { + await validatorClient.registerHandlers(); + const checkpointHandler = registerAllNodesCheckpointHandler(); + const duplicateProposalCallback = p2pClient.registerDuplicateProposalCallback.mock.calls[0][0]; + const { checkpointProposal } = await makeCheckpointProposalWithHeaderMismatch(); + const emitSpy = jest.spyOn(validatorClient, 'emit'); + + await checkpointHandler(checkpointProposal, sender); + duplicateProposalCallback({ + slot: checkpointProposal.slotNumber, + proposer: checkpointProposal.getSender()!, + type: 'checkpoint', + }); + + expect(validatorClient.hasProposalEquivocation(checkpointProposal.slotNumber)).toBe(true); + expect(emitSpy).toHaveBeenCalledWith(WANT_TO_CLEAR_SLASH_EVENT, [ + { + offenseType: OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL, + epochOrSlot: BigInt(checkpointProposal.slotNumber), + }, + ]); + }); + + it('does not mark invalid proposal slots after a non-slashable invalid checkpoint proposal', async () => { + const checkpointHandler = registerAllNodesCheckpointHandler(); + const checkpointProposal = await makeCheckpointProposalForSlot(); + jest.spyOn(validatorClient.getProposalHandler(), 'handleCheckpointProposal').mockResolvedValue({ + isValid: false, + reason: 'last_block_not_found', + }); + + await checkpointHandler(checkpointProposal, sender); + + expect(validatorClient.hasInvalidProposals(checkpointProposal.slotNumber)).toBe(false); + }); + it('emits slash event even if validator is not in the current committee', async () => { epochCache.filterInCommittee.mockResolvedValue([]); const checkpointHandler = registerAllNodesCheckpointHandler(); diff --git a/yarn-project/validator-client/src/validator.ts b/yarn-project/validator-client/src/validator.ts index c7721e1ea66f..57f7253bfe6b 100644 --- a/yarn-project/validator-client/src/validator.ts +++ b/yarn-project/validator-client/src/validator.ts @@ -73,7 +73,6 @@ import { const MAX_PROPOSERS_OF_INVALID_BLOCKS = 1000; const MAX_TRACKED_INVALID_PROPOSAL_SLOTS = 1000; const MAX_TRACKED_INVALID_CHECKPOINT_PROPOSALS = 1000; -const MAX_TRACKED_BAD_ATTESTATIONS = 10_000; // What errors from the block proposal handler result in slashing const SLASHABLE_BLOCK_PROPOSAL_VALIDATION_RESULT: BlockProposalValidationFailureReason[] = [ @@ -128,10 +127,9 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) private lastAttestedEpochByAttester: Map = new Map(); private proposersOfInvalidBlocks = FifoSet.withLimit(MAX_PROPOSERS_OF_INVALID_BLOCKS); - private slotsWithInvalidBlockProposals = FifoSet.withLimit(MAX_TRACKED_INVALID_PROPOSAL_SLOTS); + private slotsWithInvalidProposals = FifoSet.withLimit(MAX_TRACKED_INVALID_PROPOSAL_SLOTS); private invalidCheckpointProposalOffenseKeys = FifoSet.withLimit(MAX_TRACKED_INVALID_CHECKPOINT_PROPOSALS); - private slotsWithProposalEquivocation = FifoSet.withLimit(MAX_TRACKED_INVALID_PROPOSAL_SLOTS); - private badAttestationOffenseKeys = FifoSet.withLimit(MAX_TRACKED_BAD_ATTESTATIONS); + private slotsWithProposalEquivocation = FifoSet.withLimit(MAX_TRACKED_INVALID_PROPOSAL_SLOTS); /** Tracks the last checkpoint proposal we attested to, to prevent equivocation. */ private lastAttestedProposal?: CheckpointProposalCore; @@ -171,7 +169,6 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) // Refresh epoch cache every second to trigger alert if participation in committee changes this.epochCacheUpdateLoop = new RunningPromise(this.handleEpochCommitteeUpdate.bind(this), this.log, 1000); - const myAddresses = this.getValidatorAddresses(); this.log.verbose(`Initialized validator with addresses: ${myAddresses.map(a => a.toString()).join(', ')}`); } @@ -354,6 +351,14 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) return this.config; } + public hasProposalEquivocation(slotNumber: SlotNumber): boolean { + return this.slotsWithProposalEquivocation.has(slotNumber); + } + + public hasInvalidProposals(slotNumber: SlotNumber): boolean { + return this.slotsWithInvalidProposals.has(slotNumber); + } + public updateConfig(config: Partial) { this.config = { ...this.config, ...config }; this.proposalHandler.updateConfig(config); @@ -425,10 +430,6 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) this.handleDuplicateAttestation(info); }); - this.p2pClient.registerCheckpointAttestationCallback((attestation: CheckpointAttestation) => { - this.handleCheckpointAttestation(attestation); - }); - const myAddresses = this.getValidatorAddresses(); this.p2pClient.registerThisValidatorAddresses(myAddresses); @@ -768,6 +769,10 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) return; } + if (this.config.slashAttestInvalidCheckpointProposalPenalty > 0n) { + this.markInvalidProposalSlot(proposal.slotNumber); + } + if (this.slashInvalidCheckpointProposal(proposal)) { this.log.warn(`Slashing proposer for invalid checkpoint proposal`, { ...proposalInfo, @@ -791,7 +796,7 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) } const offenseType = OffenseType.BROADCASTED_INVALID_CHECKPOINT_PROPOSAL; - const offenseKey = `${proposer.toString()}:${offenseType}:${this.getSlotKey(proposal.slotNumber)}`; + const offenseKey = `${proposer.toString()}:${offenseType}:${proposal.slotNumber}`; if (!this.invalidCheckpointProposalOffenseKeys.addIfAbsent(offenseKey)) { return false; } @@ -808,52 +813,7 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) } private markInvalidProposalSlot(slotNumber: SlotNumber): void { - const slotKey = this.getSlotKey(slotNumber); - this.slotsWithInvalidBlockProposals.add(slotKey); - } - - private handleCheckpointAttestation(attestation: CheckpointAttestation): void { - const slotNumber = attestation.slotNumber; - const slotKey = this.getSlotKey(slotNumber); - if (!this.slotsWithInvalidBlockProposals.has(slotKey) || this.slotsWithProposalEquivocation.has(slotKey)) { - return; - } - - const attester = attestation.getSender(); - if (!attester) { - this.log.warn(`Cannot slash checkpoint attestation with invalid signature`, { - slotNumber, - archive: attestation.archive.toString(), - }); - return; - } - - this.slashAttestedToInvalidCheckpointProposal(slotNumber, attester); - } - - private slashAttestedToInvalidCheckpointProposal(slotNumber: SlotNumber, attester: EthAddress): void { - if (this.config.slashAttestInvalidCheckpointProposalPenalty <= 0n) { - return; - } - - const offenseKey = `${this.getSlotKey(slotNumber)}:${attester.toString()}`; - if (!this.badAttestationOffenseKeys.addIfAbsent(offenseKey)) { - return; - } - - this.log.warn(`Slashing attester for attesting to invalid checkpoint proposal`, { - attester: attester.toString(), - slotNumber, - }); - - this.emit(WANT_TO_SLASH_EVENT, [ - { - validator: attester, - amount: this.config.slashAttestInvalidCheckpointProposalPenalty, - offenseType: OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL, - epochOrSlot: BigInt(slotNumber), - }, - ]); + this.slotsWithInvalidProposals.add(slotNumber); } /** @@ -862,8 +822,7 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) */ private handleDuplicateProposal(info: DuplicateProposalInfo): void { const { slot, proposer, type } = info; - const slotKey = this.getSlotKey(slot); - this.slotsWithProposalEquivocation.add(slotKey); + this.slotsWithProposalEquivocation.add(slot); this.log.warn(`Triggering slash event for duplicate ${type} proposal from ${proposer.toString()} at slot ${slot}`, { proposer: proposer.toString(), @@ -911,10 +870,6 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) ]); } - private getSlotKey(slot: SlotNumber): string { - return slot.toString(); - } - async createBlockProposal( blockHeader: BlockHeader, checkpointNumber: CheckpointNumber, From 740b7e7e52fdc73124e5a7d26ce49d69dd47c4d7 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 27 May 2026 11:51:47 -0300 Subject: [PATCH 07/27] test: fix web3signer pipelining `e2e_multi_validator_node_key_store.test.ts` (#23568) Fix web3signer e2e `e2e_multi_validator_node_key_store.test.ts` by removing the minTxsPerBlock override so the pipelining preset can publish empty checkpoints while txs arrive. Also anchors the test PXE to the checkpointed chain tip to prevent checkpoint prunes from killing sent txs. --- .test_patterns.yml | 14 -------- ...e2e_multi_validator_node_key_store.test.ts | 33 ++++++++++--------- 2 files changed, 18 insertions(+), 29 deletions(-) diff --git a/.test_patterns.yml b/.test_patterns.yml index 299706444e55..0b6c6b0375f2 100644 --- a/.test_patterns.yml +++ b/.test_patterns.yml @@ -379,20 +379,6 @@ tests: owners: - *spyros - # Multi-validator web3signer suite is unstable under proposer pipelining: - # two distinct failure modes have been seen in the same "should build blocks - # & attest with multiple validator keys" case — Promise.all index-0 - # waitForTx aborting with "Tx dropped by P2P node" - # (ci.aztec-labs.com/9a5fa90aa18f62e7), and the proposer missing the slot's - # 5-attestation deadline ("AttestationTimeoutError" / "Block .* not found .* - # reorg") on PR #23344 run 26370196367 (ci.aztec-labs.com/b91d3218b5e88ae4). - # Error-regex flake matches still let ci3 retry-and-fail both attempts. Skip - # outright on merge-train/spartan until proposer pipelining stabilises. - - regex: "yarn-project/end-to-end/scripts/run_test.sh web3signer src/composed/web3signer/e2e_multi_validator_node_key_store.test.ts" - skip: true - owners: - - *palla - # http://ci.aztec-labs.com/98d59d04f85223f8 # Build-cache flake: module not found during Jest startup - regex: "src/e2e_sequencer/gov_proposal.parallel.test.ts" diff --git a/yarn-project/end-to-end/src/composed/web3signer/e2e_multi_validator_node_key_store.test.ts b/yarn-project/end-to-end/src/composed/web3signer/e2e_multi_validator_node_key_store.test.ts index 018c4a8e2f15..af674e9dcdad 100644 --- a/yarn-project/end-to-end/src/composed/web3signer/e2e_multi_validator_node_key_store.test.ts +++ b/yarn-project/end-to-end/src/composed/web3signer/e2e_multi_validator_node_key_store.test.ts @@ -284,21 +284,24 @@ describe('e2e_multi_validator_node', () => { deployL1ContractsValues, aztecNode, sequencer: sequencerClient, - } = await setup(1, { - ...PIPELINING_SETUP_OPTS, - initialValidators, - aztecTargetCommitteeSize: COMMITTEE_SIZE, - keyStoreDirectory, - minTxsPerBlock: 1, - maxTxsPerBlock: 1, - archiverPollingIntervalMS: 200, - sequencerPollingIntervalMS: 200, - worldStateBlockCheckIntervalMS: 200, - blockCheckIntervalMS: 200, - startProverNode: true, - aztecEpochDuration: 8, - aztecProofSubmissionEpochs: 4, - })); + } = await setup( + 1, + { + ...PIPELINING_SETUP_OPTS, + initialValidators, + aztecTargetCommitteeSize: COMMITTEE_SIZE, + keyStoreDirectory, + maxTxsPerBlock: 1, + archiverPollingIntervalMS: 200, + sequencerPollingIntervalMS: 200, + worldStateBlockCheckIntervalMS: 200, + blockCheckIntervalMS: 200, + startProverNode: true, + aztecEpochDuration: 8, + aztecProofSubmissionEpochs: 4, + }, + { syncChainTip: 'checkpointed' }, + )); sequencer = (sequencerClient! as TestSequencerClient).getSequencer(); publisherFactory = (sequencer as TestSequencer).publisherFactory; From 7137a683f6a89354a7df2c03b4b107579237afc6 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 27 May 2026 12:10:49 -0300 Subject: [PATCH 08/27] fix: cap CI devbox hostname (#23591) Running ci.sh grind was failing with `sethostname: invalid argument`. Codex attributed the failure to a long branch name, causing a long instance name, which was too long for `sethostname`. Confirmed that switching to a shorter branch name fixed the issue. ``` --- request build instance (SSH) --- Requesting m6a.48xlarge spot instance (name: spl_fix-web3signer-pipelining-test_amd64_grind-test-cdfb13e6637062de) (type: m6a.48xlarge) (ami: ami-067627aa971a1dcbb) (bid: 8.3136)... Waiting for instance id for spot request: sir-dvtzjepj... Timeout waiting for spot request. Requesting m6a.48xlarge on-demand instance (name: spl_fix-web3signer-pipelining-test_amd64_grind-test-cdfb13e6637062de) (type: m6a.48xlarge) (ami: ami-067627aa971a1dcbb) (bid: 8.3136)... Instance id: i-0fd2be01d28ec47e5 Waiting for SSH at 13.58.96.227... --- connect via SSH --- Stdout is not a tty, running in background... Host processes pinned to OS CPUs: 88-95,184-191 HOST: fetching EC2 metadata token... HOST: metadata token acquired. HOST: decoding credentials... HOST: starting devbox container... HOST: devbox container launched (pid=10513). Monitoring for spot termination... HOST: preparing devbox (uid/gid, docker run)... docker: Error response from daemon: failed to create task for container: failed to create shim task: OCI runtime create failed: runc create failed: unable to start container process: error during container init: sethostname: invalid argument ``` --- ci3/bootstrap_ec2 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci3/bootstrap_ec2 b/ci3/bootstrap_ec2 index 39fb276d0f2c..2ebc53569961 100755 --- a/ci3/bootstrap_ec2 +++ b/ci3/bootstrap_ec2 @@ -83,6 +83,7 @@ else fi [ -n "${INSTANCE_POSTFIX:-}" ] && instance_name+="_$INSTANCE_POSTFIX" +docker_hostname=$(echo -n "$instance_name" | tr '_' '-' | cut -c 1-63) if [ "$use_ssh" -eq 1 ]; then echo_header "request build instance (SSH)" @@ -298,7 +299,7 @@ start_build() { docker run --privileged --rm \${docker_args:-} \ --name aztec_build \ - --hostname $instance_name \ + --hostname $docker_hostname \ -v bootstrap_ci_local_docker:/var/lib/docker \ -v bootstrap_ci_repo:/home/aztec-dev/aztec-packages \ -v \$HOME/.ssh:/home/aztec-dev/.ssh:ro \ From 65dee1a97456a205490bc11dadeeaa2aaefee5ba Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 27 May 2026 12:32:13 -0300 Subject: [PATCH 09/27] test: stabilize invalid checkpoint descendant e2e (#23582) Fixes the invalid checkpoint descendant e2e timing by keeping sequencers stopped until the test has selected adjacent target proposers, installed listeners, applied malicious configs, and warped to the intended pipelined build window. This avoids applying malicious config to an earlier slot owned by the same validator, which is what caused the CI run for PR #23502 to miss the intended P1/P2 checkpoint pair. --- .../epochs_invalidate_block.parallel.test.ts | 87 +++++++++++-------- 1 file changed, 51 insertions(+), 36 deletions(-) diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts index 58e3c0609ee0..3be1b9aa50c3 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts @@ -495,41 +495,44 @@ describe('e2e_epochs/epochs_invalidate_block', () => { minTxsPerBlock: 0, }), ); - await Promise.all(sequencers.map(s => s.start())); - logger.warn(`Started all sequencers, waiting for first checkpoint before applying malicious config`); - - // Wait for at least one good checkpoint to be mined so any in-progress slot has completed. - const initialCheckpointNumber = (await nodes[0].getChainTips()).checkpointed.checkpoint.number; - await test.waitUntilCheckpointNumber(CheckpointNumber(initialCheckpointNumber + 1), test.L2_SLOT_DURATION_IN_S * 4); - - // Align to the start of an L2 slot, then pick two slots with a 3-slot gap so the malicious - // config has time to land on each proposer's job snapshot under pipelining, and P1's proposal - // has time to propagate to P2 before P2 starts pipelined building. - await test.monitor.waitUntilNextL2Slot(); - const { l2SlotNumber: currentSlot } = await test.monitor.run(); - logger.warn(`First checkpoint mined, current slot is ${currentSlot}`); - - let badSlot1 = SlotNumber.add(currentSlot, 3); - let badSlot2 = SlotNumber.add(currentSlot, 4); - let p1Proposer = await test.epochCache.getProposerAttesterAddressInSlot(badSlot1); - let p2Proposer = await test.epochCache.getProposerAttesterAddressInSlot(badSlot2); - - // Ensure the two slots belong to different proposers; retry by walking forward one slot at - // a time. With committee size 6 and random shuffling this should usually succeed first try. - let attempts = 0; - while (p1Proposer && p2Proposer && p1Proposer.equals(p2Proposer)) { - attempts += 1; - if (attempts > 6) { - throw new Error(`Could not find two consecutive slots with different proposers`); + let badSlot1: SlotNumber | undefined; + let p1Proposer: EthAddress | undefined; + let p2Proposer: EthAddress | undefined; + let candidate = Number(test.epochCache.getEpochAndSlotNow().slot) + 4; + const maxAttempts = 200; + for (let attempt = 0; attempt < maxAttempts && badSlot1 === undefined; attempt++) { + try { + const [p1, p2] = await Promise.all([ + test.epochCache.getProposerAttesterAddressInSlot(SlotNumber(candidate)), + test.epochCache.getProposerAttesterAddressInSlot(SlotNumber(candidate + 1)), + ]); + if (p1 && p2 && !p1.equals(p2)) { + badSlot1 = SlotNumber(candidate); + p1Proposer = p1; + p2Proposer = p2; + break; + } + candidate++; + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + if (!msg.includes('EpochNotStable')) { + throw err; + } + const block = await test.l1Client.getBlock({ includeTransactions: false }); + const warpBy = test.epochDuration * test.L2_SLOT_DURATION_IN_S; + const newTs = Number(block.timestamp) + warpBy; + logger.warn(`Hit EpochNotStable at candidate ${candidate}, warping L1 forward by ${warpBy}s to ${newTs}`); + await test.context.cheatCodes.eth.warp(newTs, { resetBlockInterval: true }); + const newCurrentSlot = Number(test.epochCache.getEpochAndSlotNow().slot); + if (candidate < newCurrentSlot + 4) { + candidate = newCurrentSlot + 4; + } } - badSlot1 = SlotNumber.add(badSlot1, 1); - badSlot2 = SlotNumber.add(badSlot2, 1); - p1Proposer = await test.epochCache.getProposerAttesterAddressInSlot(badSlot1); - p2Proposer = await test.epochCache.getProposerAttesterAddressInSlot(badSlot2); } - if (!p1Proposer || !p2Proposer) { - throw new Error(`Could not resolve proposers for slots ${badSlot1} and ${badSlot2}`); + if (badSlot1 === undefined || !p1Proposer || !p2Proposer) { + throw new Error(`Could not find two consecutive slots with different proposers after ${maxAttempts} attempts`); } + const badSlot2 = SlotNumber.add(badSlot1, 1); const p1NodeIndex = nodes.findIndex(n => n.getSequencer()!.validatorAddresses!.some(a => a.equals(p1Proposer!))); const p2NodeIndex = nodes.findIndex(n => n.getSequencer()!.validatorAddresses!.some(a => a.equals(p2Proposer!))); @@ -580,10 +583,6 @@ describe('e2e_epochs/epochs_invalidate_block', () => { observerArchiver.events.on(L2BlockSourceEvents.DescendentOfInvalidAttestationsCheckpointDetected, onDescendant); - // Send a couple of txs so there's content for both checkpoints. - logger.warn('Sending transactions to fill the bad checkpoints'); - await Promise.all(times(4, i => testContract.methods.emit_nullifier(BigInt(i + 1)).send({ from, wait: NO_WAIT }))); - // Watch for both CheckpointProposed events at the targeted slots. const p1CheckpointPromise = promiseWithResolvers(); const p2CheckpointPromise = promiseWithResolvers(); @@ -596,6 +595,22 @@ describe('e2e_epochs/epochs_invalidate_block', () => { } }); + // Send a couple of txs so there's content for both checkpoints. + logger.warn('Sending transactions to fill the bad checkpoints'); + await Promise.all(times(4, i => testContract.methods.emit_nullifier(BigInt(i + 1)).send({ from, wait: NO_WAIT }))); + + // Sequencers are still stopped. Warp to the L1 block immediately before the pipelined build + // window for P1, so the first proposer job that can observe the malicious config is the + // intended checkpoint, not an earlier slot owned by the same validator. + const buildSlot = SlotNumber.add(badSlot1, -1); + const buildSlotStart = getTimestampForSlot(buildSlot, test.constants); + const warpTo = buildSlotStart - BigInt(test.L1_BLOCK_TIME_IN_S); + logger.warn(`Warping L1 to timestamp ${warpTo} (one L1 block before build slot ${buildSlot})`); + await test.context.cheatCodes.eth.warp(Number(warpTo), { resetBlockInterval: true }); + + await Promise.all(sequencers.map(s => s.start())); + logger.warn(`Started all sequencers after warping to the target build window`); + logger.warn(`Waiting for two checkpoints to be mined on slots ${badSlot1} and ${badSlot2}`); const [p1Checkpoint, p2Checkpoint] = await executeTimeout( () => Promise.all([p1CheckpointPromise.promise, p2CheckpointPromise.promise]), From 94a2f579dc2e35232086ed5c9e376d50588dd219 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 27 May 2026 12:34:06 -0300 Subject: [PATCH 10/27] test(e2e): stabilize invalidation slots in `proposer invalidates multiple checkpoints` (#23590) Summary: - Scan for consecutive bad checkpoint slots whose prior pipelined target slot is not owned by either intended bad proposer. - Keep the malicious-config injection tied to the selected bad proposers and remove the now-unnecessary non-null assertion. - Add an inline comment documenting why the prior pipelined target slot matters. Why: The test applies malicious checkpoint config while sequencers are already running. With proposer pipelining, the previous target slot can snapshot that config before the intended bad slots are built. If that prior proposer is one of the intended bad proposers, the test may spend the malicious config on the wrong checkpoint and stop validating the intended two-checkpoint invalidation path. This mirrors the slot-selection issue fixed for the invalid proposal slashing test, but applies it to the consecutive checkpoint invalidation scenario. Testing: - yarn format end-to-end - yarn build - LOG_LEVEL="info; debug:sequencer,publisher,validator" yarn workspace @aztec/end-to-end test:e2e e2e_epochs/epochs_invalidate_block.parallel.test.ts -t "proposer invalidates multiple checkpoints" --- .../epochs_invalidate_block.parallel.test.ts | 39 ++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts index 3be1b9aa50c3..d3ecedaac027 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts @@ -390,17 +390,46 @@ describe('e2e_epochs/epochs_invalidate_block', () => { const { l2SlotNumber: currentSlot } = await test.monitor.run(); logger.warn(`First checkpoint mined, current slot is ${currentSlot}`); - // Pick the next two slots with a 2-slot gap to account for pipelining plus a margin - const badSlot1 = SlotNumber.add(currentSlot, 3); - const badSlot2 = SlotNumber.add(currentSlot, 4); + // The bad config is applied while sequencers are already running; skip pairs where the prior pipelined + // target slot could snapshot that config before the intended bad slots. + let badSlot1: SlotNumber | undefined; + let badSlot2: SlotNumber | undefined; + let badProposers: EthAddress[] = []; + const firstCandidateSlot = Number(currentSlot) + 3; + const maxBadSlotSearchAttempts = 20; + for (let attempt = 0; attempt < maxBadSlotSearchAttempts && badSlot1 === undefined; attempt++) { + const candidateSlot1 = SlotNumber(firstCandidateSlot + attempt); + const candidateSlot2 = SlotNumber.add(candidateSlot1, 1); + const priorPipelinedTargetSlot = SlotNumber.add(candidateSlot1, -1); + const [priorProposer, p1, p2] = await Promise.all([ + test.epochCache.getProposerAttesterAddressInSlot(priorPipelinedTargetSlot), + test.epochCache.getProposerAttesterAddressInSlot(candidateSlot1), + test.epochCache.getProposerAttesterAddressInSlot(candidateSlot2), + ]); + + logger.warn(`Checking bad checkpoint slots ${candidateSlot1} and ${candidateSlot2}`, { + priorPipelinedTargetSlot, + priorProposer: priorProposer?.toString(), + p1: p1?.toString(), + p2: p2?.toString(), + }); + + if (p1 && p2 && !priorProposer?.equals(p1) && !priorProposer?.equals(p2)) { + badSlot1 = candidateSlot1; + badSlot2 = candidateSlot2; + badProposers = [p1, p2]; + } + } + if (badSlot1 === undefined || badSlot2 === undefined) { + throw new Error(`Could not find bad checkpoint slots after ${maxBadSlotSearchAttempts} attempts`); + } const badSlots = [badSlot1, badSlot2]; - const badProposers = await Promise.all(badSlots.map(s => test.epochCache.getProposerAttesterAddressInSlot(s))); const badNodes = []; for (let badProposerIndex = 0; badProposerIndex < badProposers.length; badProposerIndex++) { const badProposer = badProposers[badProposerIndex]; logger.warn(`Disabling invalidation checks and attestation gathering for proposer ${badProposer}`); - const nodeIndex = nodes.findIndex(n => n.getSequencer()!.validatorAddresses!.some(a => a.equals(badProposer!))); + const nodeIndex = nodes.findIndex(n => n.getSequencer()!.validatorAddresses!.some(a => a.equals(badProposer))); if (nodeIndex === -1) { throw new Error(`Could not find node for proposer ${badProposer}`); } From 07f92ccb03d3da8ccf3f3dff70e7a59ab9b704ef Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 27 May 2026 12:40:06 -0300 Subject: [PATCH 11/27] test(e2e): stabilize invalid proposal slashing target slot in `attested_invalid_proposal` (#23589) ## Summary - skip target slots in attested invalid proposal slashing when the previous pipelined target slot has the same bad proposer - log the previous pipelined target proposer while selecting the test slot ## Why CI run http://ci.aztec-labs.com/bf99262466eae1dd selected slot 21 for the invalid checkpoint scenario, but the same bad proposer could first run a prior pipelined slot and build only a partial checkpoint. That left the test waiting for block-proposed events on the intended slot that never arrived. Requiring the previous pipelined target slot to have a different proposer keeps the malicious config from being consumed by the wrong slot after the epoch warp. ## Testing - yarn format end-to-end - yarn build - LOG_LEVEL='info; debug:sequencer,publisher,validator' yarn workspace @aztec/end-to-end test:e2e e2e_slashing/attested_invalid_proposal.test.ts --- .../src/e2e_slashing/attested_invalid_proposal.test.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/yarn-project/end-to-end/src/e2e_slashing/attested_invalid_proposal.test.ts b/yarn-project/end-to-end/src/e2e_slashing/attested_invalid_proposal.test.ts index fe7b1bda51f0..ae15e396f0fd 100644 --- a/yarn-project/end-to-end/src/e2e_slashing/attested_invalid_proposal.test.ts +++ b/yarn-project/end-to-end/src/e2e_slashing/attested_invalid_proposal.test.ts @@ -135,15 +135,18 @@ async function advanceToEpochBeforePipelinedTargetSlot({ const currentEpoch = await cheatCodes.getEpoch(); const nextEpoch = Number(currentEpoch) + 1; const firstSlotOfNextEpoch = nextEpoch * Number(epochDuration); + // The prior pipelined target can start first after the epoch warp and consume the bad proposer config. + const priorPipelinedTargetSlot = SlotNumber(firstSlotOfNextEpoch); const pipelinedTargetSlot = SlotNumber(firstSlotOfNextEpoch + 1); + const priorProposer = await epochCache.getProposerAttesterAddressInSlot(priorPipelinedTargetSlot); const proposer = await epochCache.getProposerAttesterAddressInSlot(pipelinedTargetSlot); logger.info( `Checking pipelined target slot ${pipelinedTargetSlot} in epoch ${nextEpoch} for proposer ${targetProposer}`, - { proposer: proposer?.toString() }, + { proposer: proposer?.toString(), priorPipelinedTargetSlot, priorProposer: priorProposer?.toString() }, ); - if (proposer?.equals(targetProposer)) { + if (proposer?.equals(targetProposer) && !priorProposer?.equals(targetProposer)) { return { targetEpoch: EpochNumber(nextEpoch), targetSlot: pipelinedTargetSlot }; } From dcf73542f884836d57e4e05ec84b0120ecdb1c28 Mon Sep 17 00:00:00 2001 From: Facundo Date: Wed, 27 May 2026 12:50:30 -0300 Subject: [PATCH 12/27] chore(foundation): faster toBufferBE via zero fast-path (#23592) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Summary Adds a zero fast-path to `toBufferBE`, the bigint→big-endian-buffer conversion underlying `Fr.toBuffer()`. Field elements serialized in protocol structs are overwhelmingly zero (kernel public inputs are mostly fixed-size zero-padding), so short-circuiting the zero case avoids a wasteful `bigint → hex string → Buffer.from(hex)` round-trip. ```ts if (num === 0n) { return Buffer.alloc(width); } ``` ## Why Profiling `Tx.toBuffer()` showed it spends ~6.7ms almost entirely in per-field `Fr.toBuffer()` across ~3900 fields, and **96% of those fields are zero**. The scalar conversion is already near-optimal otherwise — a 64-bit-words variant (`writeBigUInt64BE`×4) is actually *slower* on real (non-zero) field elements because V8's bigint shifts allocate. Micro-benchmark of `toBufferBE` variants (width=32, correctness-checked against current): | variant | 96%-zero (real) | all-random (worst case) | |---|---|---| | current | 452 ns | 382 ns | | 64-bit words | 215 ns | 503 ns (slower) | | **zero fast-path** | **55 ns** | 387 ns (free) | The fast-path is ~8× on the real workload and costs one `=== 0n` compare on the worst case. ## Impact End-to-end on `mockTx(42)`: | | before | after | |---|---|---| | `tx.toBuffer()` total | 6.66 ms | 4.20 ms (−37%) | | `data.toBuffer()` | 4.34 ms | 2.25 ms (−48%) | `data.toBuffer()` (the kernel public inputs) is the production-relevant figure: the mock serializes an uncompressed proof, whereas real txs carry a compressed proof that serializes as a single blob. The benefit applies to every `Fr.toBuffer()` / serialization path in the monorepo, not just txs. The remaining cost is structural — a Buffer is allocated per field and then `Buffer.concat`'d across thousands of them. Eliminating that needs a single-preallocated-buffer serializer; this change is the safe, broadly-beneficial first step. ## Testing `toBufferBE` previously had no direct unit tests; added coverage for the zero path, big-endian left-padding, exact-width values, and the negative-input throw. The conversion is otherwise byte-identical to before. --- .../src/bigint-buffer/bigint-buffer.test.ts | 21 ++++++++++++++++++- .../foundation/src/bigint-buffer/index.ts | 7 ++++++- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/yarn-project/foundation/src/bigint-buffer/bigint-buffer.test.ts b/yarn-project/foundation/src/bigint-buffer/bigint-buffer.test.ts index 474c26fffb7b..baabd3d20381 100644 --- a/yarn-project/foundation/src/bigint-buffer/bigint-buffer.test.ts +++ b/yarn-project/foundation/src/bigint-buffer/bigint-buffer.test.ts @@ -1,6 +1,25 @@ -import { fromHex, toHex } from './index.js'; +import { fromHex, toBufferBE, toHex } from './index.js'; describe('bigint-buffer', () => { + describe('toBufferBE', () => { + it('serializes zero to a zeroed buffer of the requested width', () => { + expect(toBufferBE(0n, 32)).toEqual(Buffer.alloc(32)); + }); + + it('big-endian pads small values on the left', () => { + expect(toBufferBE(1n, 4)).toEqual(Buffer.from([0, 0, 0, 1])); + expect(toBufferBE(0x0102n, 4)).toEqual(Buffer.from([0, 0, 1, 2])); + }); + + it('serializes a value that exactly fills the width', () => { + expect(toBufferBE(0xdeadbeefn, 4)).toEqual(Buffer.from('deadbeef', 'hex')); + }); + + it('throws on negative values', () => { + expect(() => toBufferBE(-1n, 32)).toThrow('negative'); + }); + }); + describe('toHex', () => { it('does not pad even length', () => { expect(toHex(16n)).toEqual('0x10'); diff --git a/yarn-project/foundation/src/bigint-buffer/index.ts b/yarn-project/foundation/src/bigint-buffer/index.ts index a97045ac4bec..69a11323cab8 100644 --- a/yarn-project/foundation/src/bigint-buffer/index.ts +++ b/yarn-project/foundation/src/bigint-buffer/index.ts @@ -49,9 +49,14 @@ export function toBufferLE(num: bigint, width: number): Buffer { * @returns A big-endian buffer representation of num. */ export function toBufferBE(num: bigint, width: number): Buffer { - if (num < BigInt(0)) { + if (num < 0n) { throw new Error(`Cannot convert negative bigint ${num.toString()} to buffer with toBufferBE.`); } + // The values serialized in the hot paths are overwhelmingly zero (field elements are mostly + // zero-padding in protocol structs), and the hex round-trip is wasteful for them. + if (num === 0n) { + return Buffer.alloc(width); + } const hex = num.toString(16); const buffer = Buffer.from(hex.padStart(width * 2, '0').slice(0, width * 2), 'hex'); if (buffer.length > width) { From 2a126f643fc72bc4b0cbc253276af793e5088aac Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Wed, 27 May 2026 17:22:07 +0100 Subject: [PATCH 13/27] fix: honour BB_BINARY_PATH (#23570) . --- barretenberg/ts/src/bb_backends/node/platform.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/barretenberg/ts/src/bb_backends/node/platform.ts b/barretenberg/ts/src/bb_backends/node/platform.ts index adc92efff1bc..d312cc43ad15 100644 --- a/barretenberg/ts/src/bb_backends/node/platform.ts +++ b/barretenberg/ts/src/bb_backends/node/platform.ts @@ -83,7 +83,8 @@ export function detectPlatform(): Platform | null { * * Search order: * 1. If customPath is provided and exists, return it - * 2. Otherwise search in /build//bb + * 2. If BB_BINARY_PATH is set and exists, return it + * 3. Otherwise search in /build//bb */ export function findBbBinary(customPath?: string): string | null { // Check custom path first if provided @@ -95,6 +96,14 @@ export function findBbBinary(customPath?: string): string | null { return null; } + const envPath = process.env.BB_BINARY_PATH; + if (envPath) { + if (fs.existsSync(envPath)) { + return path.resolve(envPath); + } + return null; + } + // Automatic detection const platform = detectPlatform(); if (!platform) { From 8e01bed389a711118cd5f31a41096906815b6adb Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Wed, 27 May 2026 18:10:48 +0100 Subject: [PATCH 14/27] chore: bump reth and lighthouse (#23588) Fix A-1109 --- .../terraform/deploy-ethereum-nodes/main.tf | 28 +++++++++++++++++++ .../deploy-ethereum-nodes/variables.tf | 6 ++-- 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/spartan/terraform/deploy-ethereum-nodes/main.tf b/spartan/terraform/deploy-ethereum-nodes/main.tf index 1df88be712d2..2da78bf96ef3 100644 --- a/spartan/terraform/deploy-ethereum-nodes/main.tf +++ b/spartan/terraform/deploy-ethereum-nodes/main.tf @@ -86,6 +86,7 @@ locals { } extraArgs = [ "--chain=${var.chain}", + # "--storage.v2=true", ] persistence = { @@ -93,6 +94,33 @@ locals { size = var.reth_storage storageClassName = "premium-rwo" } + + # disabled because sepolia db has already been migrated, mainnet has not been migrated yet + #initContainers = [ + # { + # name = "migrate-storage-v2" + # image = var.reth_image + # imagePullPolicy = "Always" + # command = [ + # "sh", + # "-ac", + # <<-EOT + # if [ ! -f /data/db/database.version ]; then + # echo "No existing Reth database found, skipping storage v2 migration." + # exit 0 + # fi + + # reth db --datadir=/data --chain=${var.chain} migrate-v2 + # EOT + # ] + # volumeMounts = [ + # { + # name = "storage" + # mountPath = "/data" + # } + # ] + # } + #] }) ] } diff --git a/spartan/terraform/deploy-ethereum-nodes/variables.tf b/spartan/terraform/deploy-ethereum-nodes/variables.tf index 5afc321b0da9..7af9f3e7dec5 100644 --- a/spartan/terraform/deploy-ethereum-nodes/variables.tf +++ b/spartan/terraform/deploy-ethereum-nodes/variables.tf @@ -48,7 +48,7 @@ variable "lighthouse_p2p_port" { variable "reth_image" { description = "Reth Docker image" type = string - default = "ghcr.io/paradigmxyz/reth:v2.0.0" + default = "ghcr.io/paradigmxyz/reth:v2.2.0" } variable "reth_chart_version" { @@ -90,13 +90,13 @@ variable "reth_memory_limit" { variable "lighthouse_image" { description = "Lighthouse Docker image" type = string - default = "sigp/lighthouse:v8.0.1" + default = "sigp/lighthouse:v8.1.3" } variable "lighthouse_chart_version" { description = "Lighthouse Helm chart version" type = string - default = "1.1.7" + default = "1.1.8" } variable "lighthouse_storage" { From 59e48dcb6fa845129ce4e6faf93564b80a313d73 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Wed, 27 May 2026 18:38:56 +0100 Subject: [PATCH 15/27] chore: add web3signer and postgres node selectors (#23598) Fix A-1110 --- spartan/aztec-postgres/templates/statefulset.yaml | 4 ++++ spartan/aztec-postgres/values.yaml | 2 ++ .../terraform/modules/validator-ha-postgres/main.tf | 6 ++++++ spartan/terraform/modules/web3signer/main.tf | 13 +++++++++++++ 4 files changed, 25 insertions(+) diff --git a/spartan/aztec-postgres/templates/statefulset.yaml b/spartan/aztec-postgres/templates/statefulset.yaml index bd8974ced5bb..8f4f7e680ac8 100644 --- a/spartan/aztec-postgres/templates/statefulset.yaml +++ b/spartan/aztec-postgres/templates/statefulset.yaml @@ -18,6 +18,10 @@ spec: app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/component: {{ .Values.component | default .Chart.Name }} spec: + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} containers: - name: postgres image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" diff --git a/spartan/aztec-postgres/values.yaml b/spartan/aztec-postgres/values.yaml index 349a60ac0cc3..f6187304a373 100644 --- a/spartan/aztec-postgres/values.yaml +++ b/spartan/aztec-postgres/values.yaml @@ -28,3 +28,5 @@ persistence: service: port: 5432 + +nodeSelector: {} diff --git a/spartan/terraform/modules/validator-ha-postgres/main.tf b/spartan/terraform/modules/validator-ha-postgres/main.tf index 73e63981bd94..e7f4ba3e4b3a 100644 --- a/spartan/terraform/modules/validator-ha-postgres/main.tf +++ b/spartan/terraform/modules/validator-ha-postgres/main.tf @@ -51,6 +51,9 @@ resource "helm_release" "postgres" { enabled = true size = var.STORAGE_SIZE } + nodeSelector = { + "node-type" = "network" + } })] timeout = 300 @@ -68,6 +71,9 @@ resource "kubernetes_job_v1" "migrations" { template { metadata {} spec { + node_selector = { + "node-type" = "network" + } container { name = "migrate" image = var.AZTEC_DOCKER_IMAGE diff --git a/spartan/terraform/modules/web3signer/main.tf b/spartan/terraform/modules/web3signer/main.tf index af5ce6ff56f4..7cf255f87410 100644 --- a/spartan/terraform/modules/web3signer/main.tf +++ b/spartan/terraform/modules/web3signer/main.tf @@ -77,6 +77,19 @@ resource "helm_release" "web3signer" { repository = split(":", var.WEB3SIGNER_DOCKER_IMAGE)[0] tag = split(":", var.WEB3SIGNER_DOCKER_IMAGE)[1] } + nodeSelector = { + "node-type" = "network" + } + resources = { + requests = { + cpu = "100m" + memory = "512Mi" + } + limits = { + cpu = "1" + memory = "2Gi" + } + } extraVolumes = [ { name = "keystores" From 129eb13899812388d4a4a08bf15493c3180e39bb Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 27 May 2026 14:39:55 -0300 Subject: [PATCH 16/27] fix: do not symlink .codex folders (#23593) This causes Codex sandbox to fail and the apply_patch command to fail. Fix is to remove the symlinks for all the .codex folders, and instead create actual folders with symlinks in their contents. A pre-commit hook checks that all contents are symlinked. > The issue is the tracked symlink: > > yarn-project/.codex -> .claude > > The sandbox is trying to enforce /home/santiago/Projects/aztec-4/yarn-project/.codex as a read-only > path, but yarn-project is also a writable root. Since .codex is a symlink inside that writable root, > bubblewrap refuses to set up the sandbox: > > Fatal error: cannot enforce sandbox read-only path .../.codex > because it crosses writable symlink .../.codex > > So apply_patch is not uniquely broken. I reproduced the same sandbox setup failure with simple > sandboxed commands like pwd and ls. Commands that are already approved or explicitly escalated can > still run because they bypass that sandbox path setup. This issue had been introduced in #23400. --- barretenberg/.codex | 1 - barretenberg/.codex/agents | 1 + barretenberg/.codex/skills | 1 + barretenberg/cpp/.codex | 1 - barretenberg/cpp/.codex/agents | 1 + docs/.codex | 1 - docs/.codex/agents | 1 + docs/.codex/commands | 1 + yarn-project/.codex | 1 - yarn-project/.codex/agents | 1 + yarn-project/.codex/scripts | 1 + yarn-project/.codex/settings.json | 1 + yarn-project/.codex/skills | 1 + yarn-project/precommit.sh | 49 +++++++++++++++++++++++++++++++ 14 files changed, 58 insertions(+), 4 deletions(-) delete mode 120000 barretenberg/.codex create mode 120000 barretenberg/.codex/agents create mode 120000 barretenberg/.codex/skills delete mode 120000 barretenberg/cpp/.codex create mode 120000 barretenberg/cpp/.codex/agents delete mode 120000 docs/.codex create mode 120000 docs/.codex/agents create mode 120000 docs/.codex/commands delete mode 120000 yarn-project/.codex create mode 120000 yarn-project/.codex/agents create mode 120000 yarn-project/.codex/scripts create mode 120000 yarn-project/.codex/settings.json create mode 120000 yarn-project/.codex/skills diff --git a/barretenberg/.codex b/barretenberg/.codex deleted file mode 120000 index c8161850a43d..000000000000 --- a/barretenberg/.codex +++ /dev/null @@ -1 +0,0 @@ -.claude \ No newline at end of file diff --git a/barretenberg/.codex/agents b/barretenberg/.codex/agents new file mode 120000 index 000000000000..0efb85ec44f4 --- /dev/null +++ b/barretenberg/.codex/agents @@ -0,0 +1 @@ +../.claude/agents \ No newline at end of file diff --git a/barretenberg/.codex/skills b/barretenberg/.codex/skills new file mode 120000 index 000000000000..454b8427cd75 --- /dev/null +++ b/barretenberg/.codex/skills @@ -0,0 +1 @@ +../.claude/skills \ No newline at end of file diff --git a/barretenberg/cpp/.codex b/barretenberg/cpp/.codex deleted file mode 120000 index c8161850a43d..000000000000 --- a/barretenberg/cpp/.codex +++ /dev/null @@ -1 +0,0 @@ -.claude \ No newline at end of file diff --git a/barretenberg/cpp/.codex/agents b/barretenberg/cpp/.codex/agents new file mode 120000 index 000000000000..0efb85ec44f4 --- /dev/null +++ b/barretenberg/cpp/.codex/agents @@ -0,0 +1 @@ +../.claude/agents \ No newline at end of file diff --git a/docs/.codex b/docs/.codex deleted file mode 120000 index c8161850a43d..000000000000 --- a/docs/.codex +++ /dev/null @@ -1 +0,0 @@ -.claude \ No newline at end of file diff --git a/docs/.codex/agents b/docs/.codex/agents new file mode 120000 index 000000000000..0efb85ec44f4 --- /dev/null +++ b/docs/.codex/agents @@ -0,0 +1 @@ +../.claude/agents \ No newline at end of file diff --git a/docs/.codex/commands b/docs/.codex/commands new file mode 120000 index 000000000000..0b631dd5b721 --- /dev/null +++ b/docs/.codex/commands @@ -0,0 +1 @@ +../.claude/commands \ No newline at end of file diff --git a/yarn-project/.codex b/yarn-project/.codex deleted file mode 120000 index c8161850a43d..000000000000 --- a/yarn-project/.codex +++ /dev/null @@ -1 +0,0 @@ -.claude \ No newline at end of file diff --git a/yarn-project/.codex/agents b/yarn-project/.codex/agents new file mode 120000 index 000000000000..0efb85ec44f4 --- /dev/null +++ b/yarn-project/.codex/agents @@ -0,0 +1 @@ +../.claude/agents \ No newline at end of file diff --git a/yarn-project/.codex/scripts b/yarn-project/.codex/scripts new file mode 120000 index 000000000000..26e538548009 --- /dev/null +++ b/yarn-project/.codex/scripts @@ -0,0 +1 @@ +../.claude/scripts \ No newline at end of file diff --git a/yarn-project/.codex/settings.json b/yarn-project/.codex/settings.json new file mode 120000 index 000000000000..11a726369cef --- /dev/null +++ b/yarn-project/.codex/settings.json @@ -0,0 +1 @@ +../.claude/settings.json \ No newline at end of file diff --git a/yarn-project/.codex/skills b/yarn-project/.codex/skills new file mode 120000 index 000000000000..454b8427cd75 --- /dev/null +++ b/yarn-project/.codex/skills @@ -0,0 +1 @@ +../.claude/skills \ No newline at end of file diff --git a/yarn-project/precommit.sh b/yarn-project/precommit.sh index a4863d268296..1f5bd7c8420a 100755 --- a/yarn-project/precommit.sh +++ b/yarn-project/precommit.sh @@ -10,6 +10,50 @@ cd $(dirname $0) export FORCE_COLOR=true +# Verify every .codex directory mirrors its sibling .claude via child symlinks, so that adding a +# file or folder to a .claude config does not silently leave the sandboxed .codex path behind. +# Only immediate children are checked: a symlinked folder (e.g. .codex/skills) already covers its +# contents, and a .codex that is itself a symlink (the repo root) mirrors .claude inherently. +check_codex_symlinks() { + local repo_root claude_dirs claude_dir codex_dir path name + local -a errors=() + repo_root=$(git rev-parse --show-toplevel) + claude_dirs=$(cd "$repo_root" && git ls-files -- '.claude/*' '*/.claude/*' | sed -E 's#(.*/)?\.claude/.*#\1.claude#' | sort -u) + + for claude_dir in $claude_dirs; do + codex_dir="${claude_dir%.claude}.codex" + if [ -L "$repo_root/$codex_dir" ]; then + continue + fi + if [ ! -d "$repo_root/$codex_dir" ]; then + errors+=("missing directory $codex_dir (should mirror $claude_dir)") + continue + fi + while IFS= read -r path; do + name=$(basename "$path") + if [ ! -L "$repo_root/$codex_dir/$name" ] || [ ! "$repo_root/$codex_dir/$name" -ef "$path" ]; then + errors+=("$codex_dir/$name should be a symlink to ../.claude/$name") + fi + done < <(find "$repo_root/$claude_dir" -mindepth 1 -maxdepth 1) + while IFS= read -r path; do + name=$(basename "$path") + if [ ! -e "$repo_root/$claude_dir/$name" ] && [ ! -L "$repo_root/$claude_dir/$name" ]; then + errors+=("$codex_dir/$name is stale; no matching $claude_dir/$name") + fi + done < <(find "$repo_root/$codex_dir" -mindepth 1 -maxdepth 1) + done + + if (( ${#errors[@]} > 0 )); then + echo -e "\033[31mError:\033[0m .codex directories are out of sync with their .claude siblings:" + for e in "${errors[@]}"; do echo " - $e"; done + echo "Each entry under a .claude folder needs a sibling symlink in .codex, e.g.:" + echo " (cd /.codex && ln -s ../.claude/ && git add )" + return 1 + fi +} + +check_codex_symlinks + # Get all staged files (excluding deleted), relative to yarn-project staged_files=$(git diff-index --diff-filter=d --relative --cached --name-only HEAD) @@ -18,6 +62,11 @@ staged_files=$(git diff-index --diff-filter=d --relative --cached --name-only HE # Filter for formattable files staged_format_files=$(echo "$staged_files" | grep -E '\.(json|js|mjs|cjs|ts)$' || true) +# Drop symlinks; prettier errors when handed a symbolic link (e.g. .codex/settings.json). +if [[ -n "$staged_format_files" ]]; then + staged_format_files=$(echo "$staged_format_files" | while IFS= read -r f; do [ -L "$f" ] || printf '%s\n' "$f"; done) +fi + # Get unstaged changes for formattable files unstaged_format_files=$(git diff --relative --name-only --diff-filter=d | grep -E '\.(json|js|mjs|cjs|ts)$' || true) From 5fa487b25e262029290926a4b76c7e0f16ce1481 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 27 May 2026 15:33:00 -0300 Subject: [PATCH 17/27] chore: fix claude and codex symlinking tests (#23599) Fixes issue introduced in #23593. Also fixes the content hash so they run on any change to claude or codex folders, which caused the test failure to go unnoticed in the PR where it was introduced. --- .claude/bootstrap.sh | 4 ++- .claude/tests/agents_symlink_test | 40 +++++++++++++++++++++++----- yarn-project/precommit.sh | 44 ------------------------------- 3 files changed, 36 insertions(+), 52 deletions(-) diff --git a/.claude/bootstrap.sh b/.claude/bootstrap.sh index 021f5270f3fb..078ddd1c22a2 100755 --- a/.claude/bootstrap.sh +++ b/.claude/bootstrap.sh @@ -4,7 +4,9 @@ # `test`. Keeps hook scripts and their tests as a self-contained component. source $(git rev-parse --show-toplevel)/ci3/source_bootstrap -hash=$(cache_content_hash ^.claude) +# Hash everything agents_symlink_test inspects, not just .claude/: the .codex mirrors and the root +# AGENTS.md/CLAUDE.md symlinks live outside .claude/, so a change there must still rerun the test. +hash=$(cache_content_hash "^.*.claude" "^.*.codex" "^AGENTS.md" "^CLAUDE.md") function test_cmds { # source_base cd's us into .claude/, so glob relative-to-here, but emit paths diff --git a/.claude/tests/agents_symlink_test b/.claude/tests/agents_symlink_test index 48a13ff77f16..4c3f2244bcf1 100755 --- a/.claude/tests/agents_symlink_test +++ b/.claude/tests/agents_symlink_test @@ -82,17 +82,43 @@ while IFS= read -r dir; do "$ROOT"/noir/noir-repo/*) continue;; esac codex="$(dirname "$dir")/.codex" - if [[ ! -L "$codex" ]]; then - fail "${codex#$ROOT/} is missing or is not a symlink to .claude" + rel=${codex#$ROOT/} + # A .codex may either be a symlink to its sibling .claude (the repo root keeps this form), or a + # real directory whose immediate children symlink into .claude (the per-package form, required + # because a sandbox cannot bind-mount a path that is itself a symlink). Either way Codex must see + # exactly the same contents as .claude. + if [[ -L "$codex" ]]; then + resolved=$(cd "$(dirname "$codex")" && cd "$(readlink "$codex")" 2>/dev/null && pwd -P) || resolved="" + claude_resolved=$(cd "$dir" && pwd -P) + if [[ "$resolved" != "$claude_resolved" ]]; then + fail "$rel is a symlink to ${resolved:-?}, expected its sibling .claude ($claude_resolved)" + else + pass "$rel -> .claude" + fi continue fi - resolved=$(cd "$(dirname "$codex")" && cd "$(readlink "$codex")" 2>/dev/null && pwd -P) || resolved="" - claude_resolved=$(cd "$dir" && pwd -P) - if [[ "$resolved" != "$claude_resolved" ]]; then - fail "${codex#$ROOT/} resolves to $resolved, expected $claude_resolved" + if [[ ! -d "$codex" ]]; then + fail "$rel is missing (expected a directory mirroring .claude via child symlinks)" continue fi - pass "${codex#$ROOT/}" + codex_ok=1 + # Forward: every entry in .claude must have a matching symlink in .codex. + while IFS= read -r child; do + name=$(basename "$child") + if [[ ! -L "$codex/$name" || ! "$codex/$name" -ef "$child" ]]; then + fail "$rel/$name should be a symlink to ../.claude/$name" + codex_ok=0 + fi + done < <(find "$dir" -mindepth 1 -maxdepth 1) + # Reverse: every entry in .codex must correspond to a .claude entry (no stale or dangling links). + while IFS= read -r entry; do + name=$(basename "$entry") + if [[ ! -e "$dir/$name" && ! -L "$dir/$name" ]]; then + fail "$rel/$name is stale; no matching .claude/$name" + codex_ok=0 + fi + done < <(find "$codex" -mindepth 1 -maxdepth 1) + (( codex_ok )) && pass "$rel" done < <(find "$ROOT" -type d -name .claude -not -path "$ROOT/noir/*" -not -path "$ROOT/**/node_modules/*") echo diff --git a/yarn-project/precommit.sh b/yarn-project/precommit.sh index 1f5bd7c8420a..4407f3045225 100755 --- a/yarn-project/precommit.sh +++ b/yarn-project/precommit.sh @@ -10,50 +10,6 @@ cd $(dirname $0) export FORCE_COLOR=true -# Verify every .codex directory mirrors its sibling .claude via child symlinks, so that adding a -# file or folder to a .claude config does not silently leave the sandboxed .codex path behind. -# Only immediate children are checked: a symlinked folder (e.g. .codex/skills) already covers its -# contents, and a .codex that is itself a symlink (the repo root) mirrors .claude inherently. -check_codex_symlinks() { - local repo_root claude_dirs claude_dir codex_dir path name - local -a errors=() - repo_root=$(git rev-parse --show-toplevel) - claude_dirs=$(cd "$repo_root" && git ls-files -- '.claude/*' '*/.claude/*' | sed -E 's#(.*/)?\.claude/.*#\1.claude#' | sort -u) - - for claude_dir in $claude_dirs; do - codex_dir="${claude_dir%.claude}.codex" - if [ -L "$repo_root/$codex_dir" ]; then - continue - fi - if [ ! -d "$repo_root/$codex_dir" ]; then - errors+=("missing directory $codex_dir (should mirror $claude_dir)") - continue - fi - while IFS= read -r path; do - name=$(basename "$path") - if [ ! -L "$repo_root/$codex_dir/$name" ] || [ ! "$repo_root/$codex_dir/$name" -ef "$path" ]; then - errors+=("$codex_dir/$name should be a symlink to ../.claude/$name") - fi - done < <(find "$repo_root/$claude_dir" -mindepth 1 -maxdepth 1) - while IFS= read -r path; do - name=$(basename "$path") - if [ ! -e "$repo_root/$claude_dir/$name" ] && [ ! -L "$repo_root/$claude_dir/$name" ]; then - errors+=("$codex_dir/$name is stale; no matching $claude_dir/$name") - fi - done < <(find "$repo_root/$codex_dir" -mindepth 1 -maxdepth 1) - done - - if (( ${#errors[@]} > 0 )); then - echo -e "\033[31mError:\033[0m .codex directories are out of sync with their .claude siblings:" - for e in "${errors[@]}"; do echo " - $e"; done - echo "Each entry under a .claude folder needs a sibling symlink in .codex, e.g.:" - echo " (cd /.codex && ln -s ../.claude/ && git add )" - return 1 - fi -} - -check_codex_symlinks - # Get all staged files (excluding deleted), relative to yarn-project staged_files=$(git diff-index --diff-filter=d --relative --cached --name-only HEAD) From c3351463c0364bcb5e59fa5541518ffedd9da5a9 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 27 May 2026 18:51:57 -0300 Subject: [PATCH 18/27] test(e2e): narrow down sentinel check in `multiple_validators_sentinel` (#23604) Instead of checking a range of slots, we only check the slot we're interested in. This prevents any build errors that occured until things got stable from interfering. For instance, the sequencer we stop could cause the _next_ sequencer to miss their block. Looking just into the `sentinelSlot` removes this indeterminism. --- ...multiple_validators_sentinel.parallel.test.ts | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/yarn-project/end-to-end/src/e2e_p2p/multiple_validators_sentinel.parallel.test.ts b/yarn-project/end-to-end/src/e2e_p2p/multiple_validators_sentinel.parallel.test.ts index 6d984cefb9b9..b833d3ac6fc3 100644 --- a/yarn-project/end-to-end/src/e2e_p2p/multiple_validators_sentinel.parallel.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p/multiple_validators_sentinel.parallel.test.ts @@ -214,11 +214,14 @@ describe('e2e_p2p_multiple_validators_sentinel', () => { const stats = await sentinel.getValidatorsStats(); t.logger.info(`Collected validator stats at slot ${t.monitor.l2SlotNumber}`, { stats }); - // Check that all of the first node validators have attestations recorded + const historyForSlot = (validator: (typeof firstNodeValidators)[number]) => + stats.stats[validator.toString().toLowerCase()]?.history.filter(h => h.slot === slotForSentinel) ?? []; + + // Check that all of the first node validators have attestations recorded for the selected proposer slot. for (const validator of firstNodeValidators) { - const validatorStats = stats.stats[validator.toString().toLowerCase()]; - const history = validatorStats?.history.filter(h => h.slot > initialSlot && h.slot <= slotForSentinel) ?? []; + const history = historyForSlot(validator); t.logger.info(`Asserting stats for online validator ${validator}`, { history }); + expect(history).not.toBeEmpty(); expect( history.filter( h => h.status === 'attestation-missed' || h.status === 'blocks-missed' || h.status === 'checkpoint-missed', @@ -229,14 +232,13 @@ describe('e2e_p2p_multiple_validators_sentinel', () => { // At least one of the first node validators must have been seen as proposer const firstNodeBlockProposedHistory = firstNodeValidators .flatMap(v => stats.stats[v.toString().toLowerCase()].history) - .filter(h => h.slot > initialSlot && h.slot <= slotForSentinel) + .filter(h => h.slot === slotForSentinel) .filter(h => h.status === 'checkpoint-valid' || h.status === 'checkpoint-mined'); expect(firstNodeBlockProposedHistory).not.toBeEmpty(); - // And all of the proposers for the offline node must be seen as missed attestation or proposal + // And all of the validators for the offline node must be seen as missed attestation or proposal. for (const validator of offlineValidators) { - const validatorStats = stats.stats[validator.toString().toLowerCase()]; - const history = validatorStats.history?.filter(h => h.slot > initialSlot && h.slot <= slotForSentinel) ?? []; + const history = historyForSlot(validator); t.logger.info(`Asserting stats for offline validator ${validator}`, { history }); expect( history.filter( From 206eb0fd3b3807b61305152d3fa6c39dc3bd5103 Mon Sep 17 00:00:00 2001 From: Santiago Palladino Date: Wed, 27 May 2026 18:53:02 -0300 Subject: [PATCH 19/27] test(e2e): fix `proposer invalidates multiple checkpoints` timeout (#23608) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes flake in `proposer invalidates multiple checkpoints` `e2e_epochs/epochs_invalidate_block.parallel.test.ts` test that caused a timeout (see [this run](http://ci.aztec-labs.com/8b1c0f4ec6031f2b)). See below for the Codex analysis and fix. --- **Test Summary** `proposer invalidates multiple checkpoints` verifies that two intended bad checkpoints land with insufficient attestations, a later good proposer invalidates the first bad checkpoint, and the chain then progresses. **Failed Run Error** CI run `8b1c0f4ec6031f2b` timed out at Jest’s 600s limit. The failure was not the shutdown L1 send error; that happened after the timeout while teardown was interrupting pending work. **Failed vs Successful Divergence** First meaningful divergence: checkpoint 4 at slot 23. Failed log: slot 23 published checkpoint 4 with only 1 attestation, then archivers reported `Insufficient attestations ... actualAttestations:1`. Successful log: slot 23 collected all 5 attestations before publishing checkpoint 4, so the first intentionally bad checkpoints were later. **Timeline** Failed: - `15:59:11` selected intended bad slots 25/26, applied bad config to proposer `0x15...` - `15:59:35` slot 23 job prepared by that same proposer - `16:00:15` checkpoint 4 at slot 23 landed with 1 attestation - repeated rollback/retry consumed enough time to hit Jest timeout Successful: - slot 23 checkpoint landed cleanly with 5 attestations - intended bad checkpoints at slots 24/25 landed with 1 attestation - checkpoint 5 was invalidated - test completed successfully **Hypothesis** High confidence: the test’s bad-slot selection only excluded `candidateSlot1 - 1` as a pre-bad pipelined target. In the failed run, `candidateSlot1 - 2` was still unsnapshotted and owned by a bad proposer, so applying malicious config leaked into slot 23. **Evidence** - Logs: failed run selected slots 25/26 but slot 23 later published with 1 attestation from the newly bad proposer. - Source: pipelined checkpoint jobs snapshot sequencer config when the target-slot job is created, so applying config while sequencers are running can affect any not-yet-created pre-bad job. - Skeptic check: no contradiction found; it also caught a broken local timeout race. **Proposed Fix** Implemented in [epochs_invalidate_block.parallel.test.ts](/home/santiago/Projects/aztec-1/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts:393): the selector now excludes bad proposers from every pre-bad target slot from `currentSlot + 2` through `candidateSlot1 - 1`, not just the immediately prior slot. Also fixed the broken timeout race at [line 475](/home/santiago/Projects/aztec-1/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts:475) by removing the accidental inner `await`. --- .../epochs_invalidate_block.parallel.test.ts | 27 ++++++++++++------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts b/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts index d3ecedaac027..2d8323251021 100644 --- a/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts +++ b/yarn-project/end-to-end/src/e2e_epochs/epochs_invalidate_block.parallel.test.ts @@ -390,31 +390,40 @@ describe('e2e_epochs/epochs_invalidate_block', () => { const { l2SlotNumber: currentSlot } = await test.monitor.run(); logger.warn(`First checkpoint mined, current slot is ${currentSlot}`); - // The bad config is applied while sequencers are already running; skip pairs where the prior pipelined - // target slot could snapshot that config before the intended bad slots. + // The bad config is applied while sequencers are already running; skip pairs where a pipelined + // pre-bad target slot could snapshot that config before the intended bad slots. let badSlot1: SlotNumber | undefined; let badSlot2: SlotNumber | undefined; let badProposers: EthAddress[] = []; const firstCandidateSlot = Number(currentSlot) + 3; + const firstUnsnapshottedTargetSlot = SlotNumber.add(currentSlot, 2); const maxBadSlotSearchAttempts = 20; for (let attempt = 0; attempt < maxBadSlotSearchAttempts && badSlot1 === undefined; attempt++) { const candidateSlot1 = SlotNumber(firstCandidateSlot + attempt); const candidateSlot2 = SlotNumber.add(candidateSlot1, 1); - const priorPipelinedTargetSlot = SlotNumber.add(candidateSlot1, -1); - const [priorProposer, p1, p2] = await Promise.all([ - test.epochCache.getProposerAttesterAddressInSlot(priorPipelinedTargetSlot), + const preBadTargetSlots = range( + Math.max(0, Number(candidateSlot1) - Number(firstUnsnapshottedTargetSlot)), + Number(firstUnsnapshottedTargetSlot), + ).map(SlotNumber); + const [preBadProposers, p1, p2] = await Promise.all([ + Promise.all(preBadTargetSlots.map(slot => test.epochCache.getProposerAttesterAddressInSlot(slot))), test.epochCache.getProposerAttesterAddressInSlot(candidateSlot1), test.epochCache.getProposerAttesterAddressInSlot(candidateSlot2), ]); logger.warn(`Checking bad checkpoint slots ${candidateSlot1} and ${candidateSlot2}`, { - priorPipelinedTargetSlot, - priorProposer: priorProposer?.toString(), + preBadTargetSlots, + preBadProposers: preBadProposers.map(proposer => proposer?.toString()), p1: p1?.toString(), p2: p2?.toString(), }); - if (p1 && p2 && !priorProposer?.equals(p1) && !priorProposer?.equals(p2)) { + const badProposerHasUnsnapshottedPreBadSlot = + p1 !== undefined && + p2 !== undefined && + preBadProposers.some(proposer => proposer !== undefined && (proposer.equals(p1) || proposer.equals(p2))); + + if (p1 && p2 && !badProposerHasUnsnapshottedPreBadSlot) { badSlot1 = candidateSlot1; badSlot2 = candidateSlot2; badProposers = [p1, p2]; @@ -464,7 +473,7 @@ describe('e2e_epochs/epochs_invalidate_block', () => { // Wait for both checkpoints to be mined logger.warn(`Waiting for two checkpoints to be mined on slots ${expectedFirstSlot} and ${expectedSecondSlot}`); const [firstCheckpoint, secondCheckpoint] = await Promise.race([ - await Promise.all([firstCheckpointPromise.promise, secondCheckpointPromise.promise]), + Promise.all([firstCheckpointPromise.promise, secondCheckpointPromise.promise]), timeoutPromise(test.L2_SLOT_DURATION_IN_S * 8 * 1000).then(() => [CheckpointNumber(0), CheckpointNumber(0)]), ]); From fcd4ec2d6a8bd7fea45629c8851dddf746893df2 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 28 May 2026 09:34:39 +0100 Subject: [PATCH 20/27] fix: record zero-amount slashing offenses (#23556) - record zero-amount slashing offenses instead of treating penalty 0 as disabled - keep slash vote signaling gated by existing summed vote output - upgrade stored duplicate offenses when a later observation has a higher amount Fix A-1075 --- .../aztec-node/src/aztec-node/server.ts | 58 ++++------ .../aztec-node/src/sentinel/sentinel.test.ts | 18 +++ .../aztec-node/src/sentinel/sentinel.ts | 8 +- yarn-project/slasher/src/config.ts | 13 +-- .../slasher/src/slasher_client.test.ts | 18 +++ yarn-project/slasher/src/slasher_client.ts | 10 +- .../attested_invalid_proposal_watcher.test.ts | 45 ++++++++ .../attested_invalid_proposal_watcher.ts | 25 +++-- ...nvalid_checkpoint_proposal_watcher.test.ts | 35 ++++++ ...ted_invalid_checkpoint_proposal_watcher.ts | 9 +- .../checkpoint_equivocation_watcher.test.ts | 13 ++- .../checkpoint_equivocation_watcher.ts | 6 +- .../watchers/data_withholding_watcher.test.ts | 24 +++- .../src/watchers/data_withholding_watcher.ts | 8 +- .../validator-client/src/validator.test.ts | 92 +++++++++++++-- .../validator-client/src/validator.ts | 105 +++++++++++------- 16 files changed, 359 insertions(+), 128 deletions(-) diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index f3070003ed83..f4be06db0b6e 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -743,34 +743,30 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, AztecNodeDeb if (!proverOnly) { validatorsSentinel = await createSentinel(epochCache, archiver, p2pClient, reexecutionTracker, config); - if (validatorsSentinel && config.slashInactivityPenalty > 0n) { + if (validatorsSentinel) { watchers.push(validatorsSentinel); } - if (config.slashDataWithholdingPenalty > 0n) { - dataWithholdingWatcher = new DataWithholdingWatcher( - epochCache, - archiver, - p2pClient.getTxProvider(), - p2pClient, - reexecutionTracker, - { chainId: config.l1ChainId, rollupAddress: config.rollupAddress }, - config, - ); - watchers.push(dataWithholdingWatcher); - } + dataWithholdingWatcher = new DataWithholdingWatcher( + epochCache, + archiver, + p2pClient.getTxProvider(), + p2pClient, + reexecutionTracker, + { chainId: config.l1ChainId, rollupAddress: config.rollupAddress }, + config, + ); + watchers.push(dataWithholdingWatcher); - if (config.slashBroadcastedInvalidCheckpointProposalPenalty > 0n) { - broadcastedInvalidCheckpointProposalWatcher = new BroadcastedInvalidCheckpointProposalWatcher( - p2pClient, - archiver, - epochCache, - config, - ); - watchers.push(broadcastedInvalidCheckpointProposalWatcher); - } + broadcastedInvalidCheckpointProposalWatcher = new BroadcastedInvalidCheckpointProposalWatcher( + p2pClient, + archiver, + epochCache, + config, + ); + watchers.push(broadcastedInvalidCheckpointProposalWatcher); - if (validatorClient && config.slashAttestInvalidCheckpointProposalPenalty > 0n) { + if (validatorClient) { attestedInvalidProposalWatcher = new AttestedInvalidProposalWatcher( p2pClient, validatorClient, @@ -782,19 +778,11 @@ export class AztecNodeService implements AztecNode, AztecNodeAdmin, AztecNodeDeb watchers.push(attestedInvalidProposalWatcher); } - if (config.slashDuplicateProposalPenalty > 0n) { - checkpointEquivocationWatcher = new CheckpointEquivocationWatcher(archiver, epochCache, config); - watchers.push(checkpointEquivocationWatcher); - } + checkpointEquivocationWatcher = new CheckpointEquivocationWatcher(archiver, epochCache, config); + watchers.push(checkpointEquivocationWatcher); - // We assume we want to slash for invalid attestations unless all max penalties are set to 0 - if ( - config.slashProposeInvalidAttestationsPenalty > 0n || - config.slashProposeDescendantOfCheckpointWithInvalidAttestationsPenalty > 0n - ) { - attestationsBlockWatcher = new AttestationsBlockWatcher(archiver, epochCache, config, log.getBindings()); - watchers.push(attestationsBlockWatcher); - } + attestationsBlockWatcher = new AttestationsBlockWatcher(archiver, epochCache, config, log.getBindings()); + watchers.push(attestationsBlockWatcher); } const watchersToStart = compactArray([ diff --git a/yarn-project/aztec-node/src/sentinel/sentinel.test.ts b/yarn-project/aztec-node/src/sentinel/sentinel.test.ts index d28d49bc81a8..966a72b34fb9 100644 --- a/yarn-project/aztec-node/src/sentinel/sentinel.test.ts +++ b/yarn-project/aztec-node/src/sentinel/sentinel.test.ts @@ -986,6 +986,24 @@ describe('sentinel', () => { ]); }); + it('emits zero-amount inactivity offenses when the penalty is zero', async () => { + sentinel.updateConfig({ slashInactivityPenalty: 0n, slashInactivityConsecutiveEpochThreshold: 1 }); + const emitSpy = jest.spyOn(sentinel, 'emit'); + + await sentinel.handleEpochPerformance(EpochNumber(5), { + [validator1.toString()]: { missed: 8, total: 10 }, + }); + + expect(emitSpy).toHaveBeenCalledWith(WANT_TO_SLASH_EVENT, [ + { + validator: validator1, + amount: 0n, + offenseType: OffenseType.INACTIVITY, + epochOrSlot: 5n, + }, + ]); + }); + it('should not slash when no validators meet consecutive threshold', async () => { // Update config to require 3 consecutive epochs sentinel.updateConfig({ slashInactivityConsecutiveEpochThreshold: 3 }); diff --git a/yarn-project/aztec-node/src/sentinel/sentinel.ts b/yarn-project/aztec-node/src/sentinel/sentinel.ts index 53d1dcc70099..a9746cfa0643 100644 --- a/yarn-project/aztec-node/src/sentinel/sentinel.ts +++ b/yarn-project/aztec-node/src/sentinel/sentinel.ts @@ -335,10 +335,6 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme } protected async handleEpochPerformance(epoch: EpochNumber, performance: ValidatorsEpochPerformance) { - if (this.config.slashInactivityPenalty === 0n) { - return; - } - const inactiveValidators = getEntries(performance) .filter(([_, { missed, total }]) => total > 0 && missed / total >= this.config.slashInactivityTargetPercentage) .map(([address]) => address); @@ -363,8 +359,8 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme if (criminals.length > 0) { this.logger.verbose( - `Identified ${criminals.length} validators to slash due to inactivity in at least ${epochThreshold} consecutive epochs`, - { ...args, epochThreshold }, + `Identified ${criminals.length} inactivity offenses in at least ${epochThreshold} consecutive epochs`, + { offenses: args, epochThreshold }, ); this.emit(WANT_TO_SLASH_EVENT, args); } diff --git a/yarn-project/slasher/src/config.ts b/yarn-project/slasher/src/config.ts index ddc934f02b62..64ca1e97c6c1 100644 --- a/yarn-project/slasher/src/config.ts +++ b/yarn-project/slasher/src/config.ts @@ -71,7 +71,7 @@ export const slasherConfigMappings: ConfigMappingsType = { }, slashDataWithholdingPenalty: { env: 'SLASH_DATA_WITHHOLDING_PENALTY', - description: 'Penalty amount for slashing validators for data withholding (set to 0 to disable).', + description: 'Penalty for data withholding (0 records offenses without slash votes).', ...bigintConfigHelper(DefaultSlasherConfig.slashDataWithholdingPenalty), }, slashDataWithholdingToleranceSlots: { @@ -125,29 +125,28 @@ export const slasherConfigMappings: ConfigMappingsType = { }, slashInactivityPenalty: { env: 'SLASH_INACTIVITY_PENALTY', - description: 'Penalty amount for slashing an inactive validator (set to 0 to disable).', + description: 'Penalty for an inactive validator (0 records offenses without slash votes).', ...bigintConfigHelper(DefaultSlasherConfig.slashInactivityPenalty), }, slashProposeInvalidAttestationsPenalty: { env: 'SLASH_PROPOSE_INVALID_ATTESTATIONS_PENALTY', - description: 'Penalty amount for slashing a proposer that proposed invalid attestations (set to 0 to disable).', + description: 'Penalty for proposing invalid attestations (0 records offenses without slash votes).', ...bigintConfigHelper(DefaultSlasherConfig.slashProposeInvalidAttestationsPenalty), }, slashProposeDescendantOfCheckpointWithInvalidAttestationsPenalty: { env: 'SLASH_PROPOSE_DESCENDANT_OF_CHECKPOINT_WITH_INVALID_ATTESTATIONS_PENALTY', description: - 'Penalty amount for slashing a proposer that published a checkpoint building on an invalid checkpoint (set to 0 to disable).', + 'Penalty for publishing a checkpoint building on an invalid checkpoint (0 records offenses without slash votes).', ...bigintConfigHelper(DefaultSlasherConfig.slashProposeDescendantOfCheckpointWithInvalidAttestationsPenalty), }, slashAttestInvalidCheckpointProposalPenalty: { env: 'SLASH_ATTEST_INVALID_CHECKPOINT_PROPOSAL_PENALTY', - description: - 'Penalty amount for slashing a validator that attested to an invalid checkpoint proposal (set to 0 to disable).', + description: 'Penalty for attesting to an invalid checkpoint proposal (0 records offenses without slash votes).', ...bigintConfigHelper(DefaultSlasherConfig.slashAttestInvalidCheckpointProposalPenalty), }, slashUnknownPenalty: { env: 'SLASH_UNKNOWN_PENALTY', - description: 'Penalty amount for slashing a validator for an unknown offense (set to 0 to disable).', + description: 'Penalty for an unknown offense (0 records offenses without slash votes).', ...bigintConfigHelper(DefaultSlasherConfig.slashUnknownPenalty), }, slashOffenseExpirationRounds: { diff --git a/yarn-project/slasher/src/slasher_client.test.ts b/yarn-project/slasher/src/slasher_client.test.ts index fa657c55bdc5..beb78674a5f7 100644 --- a/yarn-project/slasher/src/slasher_client.test.ts +++ b/yarn-project/slasher/src/slasher_client.test.ts @@ -301,6 +301,24 @@ describe('SlasherClient', () => { const actions = await slasherClient.getProposerActions(SlotNumber.fromBigInt(currentSlot)); expect(actions).toHaveLength(0); }); + + it('should not return any action for zero-amount offenses', async () => { + const currentRound = 5n; + const currentSlot = currentRound * BigInt(roundSize); + const targetRound = 3n; + + await offensesStore.addOffense( + createOffense({ + validator: committee[0], + epochOrSlot: targetRound * BigInt(roundSize), + offenseType: OffenseType.PROPOSED_INSUFFICIENT_ATTESTATIONS, + amount: 0n, + }), + ); + + const actions = await slasherClient.getProposerActions(SlotNumber.fromBigInt(currentSlot)); + expect(actions).toHaveLength(0); + }); }); describe('execute-slash', () => { diff --git a/yarn-project/slasher/src/slasher_client.ts b/yarn-project/slasher/src/slasher_client.ts index 4ec78bb55b79..f0b748a9cec7 100644 --- a/yarn-project/slasher/src/slasher_client.ts +++ b/yarn-project/slasher/src/slasher_client.ts @@ -343,7 +343,7 @@ export class SlasherClient implements ProposerSlashActionProvider, SlasherClient return undefined; } - this.log.info(`Voting to slash ${offensesToSlash.length} offenses`, { + this.log.debug(`Computing slash votes for ${offensesToSlash.length} offenses`, { slotNumber, currentRound, slashedRound, @@ -371,6 +371,14 @@ export class SlasherClient implements ProposerSlashActionProvider, SlasherClient return undefined; } + this.log.info(`Voting to slash ${offensesToSlash.length} offenses`, { + slotNumber, + slashedRound, + currentRound, + votes, + offensesToSlash, + }); + this.log.debug(`Computed votes for slashing ${offensesToSlash.length} offenses`, { slashedRound, currentRound, diff --git a/yarn-project/slasher/src/watchers/attested_invalid_proposal_watcher.test.ts b/yarn-project/slasher/src/watchers/attested_invalid_proposal_watcher.test.ts index 1c377785d01b..639b73ca4153 100644 --- a/yarn-project/slasher/src/watchers/attested_invalid_proposal_watcher.test.ts +++ b/yarn-project/slasher/src/watchers/attested_invalid_proposal_watcher.test.ts @@ -88,6 +88,51 @@ describe('AttestedInvalidProposalWatcher', () => { ]); }); + it('emits zero-amount offenses when the penalty is zero', async () => { + const slot = SlotNumber(10); + const attesterSigner = Secp256k1Signer.random(); + invalidProposalSlots.add(slot); + watcher.updateConfig({ slashAttestInvalidCheckpointProposalPenalty: 0n }); + p2pClient.getCheckpointAttestationsForSlot.mockResolvedValue([await makeAttestation(slot, attesterSigner)]); + + await watcher.scanSlot(slot); + + expect(handler).toHaveBeenCalledWith([ + { + validator: attesterSigner.address, + amount: 0n, + offenseType: OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL, + epochOrSlot: 10n, + }, + ]); + }); + + it('deduplicates repeated scans for the same attester and slot', async () => { + const slot = SlotNumber(10); + invalidProposalSlots.add(slot); + const attestation = await makeAttestation(slot); + p2pClient.getCheckpointAttestationsForSlot.mockResolvedValue([attestation]); + + await watcher.scanSlot(slot); + await watcher.scanSlot(slot); + + expect(handler).toHaveBeenCalledTimes(1); + }); + + it('deduplicates repeated scans for the same offense when the penalty changes', async () => { + const slot = SlotNumber(10); + invalidProposalSlots.add(slot); + const attestation = await makeAttestation(slot); + p2pClient.getCheckpointAttestationsForSlot.mockResolvedValue([attestation]); + + watcher.updateConfig({ slashAttestInvalidCheckpointProposalPenalty: 0n }); + await watcher.scanSlot(slot); + watcher.updateConfig({ slashAttestInvalidCheckpointProposalPenalty: 13n }); + await watcher.scanSlot(slot); + + expect(handler).toHaveBeenCalledTimes(1); + }); + it('scans only marked invalid proposal slots once they are past the scan lag', async () => { watcher = new AttestedInvalidProposalWatcher( p2pClient, diff --git a/yarn-project/slasher/src/watchers/attested_invalid_proposal_watcher.ts b/yarn-project/slasher/src/watchers/attested_invalid_proposal_watcher.ts index 400b47b863a8..1d6fbe082431 100644 --- a/yarn-project/slasher/src/watchers/attested_invalid_proposal_watcher.ts +++ b/yarn-project/slasher/src/watchers/attested_invalid_proposal_watcher.ts @@ -2,6 +2,7 @@ import type { EpochCacheInterface } from '@aztec/epoch-cache'; import { SlotNumber } from '@aztec/foundation/branded-types'; import { merge, pick } from '@aztec/foundation/collection'; import type { EthAddress } from '@aztec/foundation/eth-address'; +import { FifoSet } from '@aztec/foundation/fifo-set'; import { type Logger, createLogger } from '@aztec/foundation/log'; import { RunningPromise } from '@aztec/foundation/running-promise'; import type { L2BlockSource } from '@aztec/stdlib/block'; @@ -17,6 +18,7 @@ const AttestedInvalidProposalWatcherConfigKeys = ['slashAttestInvalidCheckpointP const SCAN_SLOT_LAG = 1; const DEFAULT_SCAN_SLOT_LOOKBACK = 4; +const MAX_TRACKED_BAD_ATTESTATIONS = 10_000; type AttestedInvalidProposalWatcherConfig = Pick< SlasherConfig, @@ -38,6 +40,7 @@ export type InvalidProposalSlotSource = { export class AttestedInvalidProposalWatcher extends (EventEmitter as new () => WatcherEmitter) implements Watcher { private readonly log: Logger; private readonly runningPromise: RunningPromise; + private readonly emittedOffenses = FifoSet.withLimit(MAX_TRACKED_BAD_ATTESTATIONS); private readonly scanSlotLookback: number; private config: AttestedInvalidProposalWatcherConfig; private lastScannedSlot: SlotNumber | undefined; @@ -76,10 +79,6 @@ export class AttestedInvalidProposalWatcher extends (EventEmitter as new () => W } public async scan(): Promise { - if (this.config.slashAttestInvalidCheckpointProposalPenalty <= 0n) { - return; - } - const currentSlot = (await this.l2BlockSource.getSyncedL2SlotNumber()) ?? this.epochCache.getSlotNow(); // genesis if (currentSlot <= SlotNumber(SCAN_SLOT_LAG)) { @@ -105,7 +104,6 @@ export class AttestedInvalidProposalWatcher extends (EventEmitter as new () => W /** Scans a single invalid-proposal slot. */ public async scanSlot(slot: SlotNumber): Promise { if ( - this.config.slashAttestInvalidCheckpointProposalPenalty <= 0n || this.invalidProposalSlotSource.hasProposalEquivocation(slot) || !this.invalidProposalSlotSource.hasInvalidProposals(slot) ) { @@ -122,15 +120,21 @@ export class AttestedInvalidProposalWatcher extends (EventEmitter as new () => W const slashArgs = attestations .map(attestation => this.getSlashArgs(slot, attestation)) - .filter((args): args is WantToSlashArgs => args !== undefined); + .filter((args): args is WantToSlashArgs => args !== undefined) + .filter(args => this.markAsNewOffense(args)); if (slashArgs.length === 0) { return; } - this.log.warn('Slashing attesters for attesting to invalid checkpoint proposal', { + this.log.warn('Detected attestations to invalid checkpoint proposal', { slot, - attesters: slashArgs.map(args => args.validator.toString()), + offenses: slashArgs.map(args => ({ + validator: args.validator.toString(), + amount: args.amount, + offenseType: args.offenseType, + epochOrSlot: args.epochOrSlot, + })), }); this.emit(WANT_TO_SLASH_EVENT, slashArgs); } @@ -156,4 +160,9 @@ export class AttestedInvalidProposalWatcher extends (EventEmitter as new () => W epochOrSlot: BigInt(slot), }; } + + private markAsNewOffense(args: WantToSlashArgs): boolean { + const key = `${args.validator.toString()}-${args.offenseType}-${args.epochOrSlot}`; + return this.emittedOffenses.addIfAbsent(key); + } } diff --git a/yarn-project/slasher/src/watchers/broadcasted_invalid_checkpoint_proposal_watcher.test.ts b/yarn-project/slasher/src/watchers/broadcasted_invalid_checkpoint_proposal_watcher.test.ts index 5d8432b9d3d2..e8611f430582 100644 --- a/yarn-project/slasher/src/watchers/broadcasted_invalid_checkpoint_proposal_watcher.test.ts +++ b/yarn-project/slasher/src/watchers/broadcasted_invalid_checkpoint_proposal_watcher.test.ts @@ -195,6 +195,26 @@ describe('BroadcastedInvalidCheckpointProposalWatcher', () => { expect(handler).not.toHaveBeenCalled(); }); + it('emits zero-amount offenses when the penalty is zero', async () => { + const signer = Secp256k1Signer.random(); + const slot = SlotNumber(10); + const blocks = await makeBlocks(signer, slot, 4); + const checkpoint = await makeCheckpointCore(signer, slot, blocks[1]); + watcher.updateConfig({ slashBroadcastedInvalidCheckpointProposalPenalty: 0n }); + mockProposals(slot, blocks, [checkpoint]); + + await watcher.scanSlot(slot); + + expect(handler).toHaveBeenCalledWith([ + { + validator: signer.address, + amount: 0n, + offenseType: OffenseType.BROADCASTED_INVALID_CHECKPOINT_PROPOSAL, + epochOrSlot: 10n, + }, + ]); + }); + it('does not emit duplicate offenses on repeated scans', async () => { const signer = Secp256k1Signer.random(); const slot = SlotNumber(10); @@ -208,6 +228,21 @@ describe('BroadcastedInvalidCheckpointProposalWatcher', () => { expect(handler).toHaveBeenCalledTimes(1); }); + it('deduplicates repeated scans for the same offense when the penalty changes', async () => { + const signer = Secp256k1Signer.random(); + const slot = SlotNumber(10); + const blocks = await makeBlocks(signer, slot, 4); + const checkpoint = await makeCheckpointCore(signer, slot, blocks[1]); + mockProposals(slot, blocks, [checkpoint]); + + watcher.updateConfig({ slashBroadcastedInvalidCheckpointProposalPenalty: 0n }); + await watcher.scanSlot(slot); + watcher.updateConfig({ slashBroadcastedInvalidCheckpointProposalPenalty: 11n }); + await watcher.scanSlot(slot); + + expect(handler).toHaveBeenCalledTimes(1); + }); + it('scans a lookback of closed slots', async () => { const signer = Secp256k1Signer.random(); const slot = SlotNumber(10); diff --git a/yarn-project/slasher/src/watchers/broadcasted_invalid_checkpoint_proposal_watcher.ts b/yarn-project/slasher/src/watchers/broadcasted_invalid_checkpoint_proposal_watcher.ts index 1e9249b2cdf4..64dacf75e059 100644 --- a/yarn-project/slasher/src/watchers/broadcasted_invalid_checkpoint_proposal_watcher.ts +++ b/yarn-project/slasher/src/watchers/broadcasted_invalid_checkpoint_proposal_watcher.ts @@ -89,10 +89,6 @@ export class BroadcastedInvalidCheckpointProposalWatcher * `currentSlot` at the archiver's last synced L2 slot. */ public async scan(): Promise { - if (this.config.slashBroadcastedInvalidCheckpointProposalPenalty <= 0n) { - return; - } - const currentSlot = (await this.l2BlockSource.getSyncedL2SlotNumber()) ?? this.epochCache.getSlotNow(); if (currentSlot <= SlotNumber(SCAN_SLOT_LAG)) { return; @@ -111,10 +107,6 @@ export class BroadcastedInvalidCheckpointProposalWatcher /** Scans a single slot. Public for tests. */ public async scanSlot(slot: SlotNumber): Promise { - if (this.config.slashBroadcastedInvalidCheckpointProposalPenalty <= 0n) { - return; - } - const proposals = await this.p2pClient.getProposalsForSlot(slot); const slashArgs = this.getSlashArgsForProposals(slot, proposals).filter(args => this.markAsNewOffense(args)); if (slashArgs.length === 0) { @@ -125,6 +117,7 @@ export class BroadcastedInvalidCheckpointProposalWatcher slot, offenses: slashArgs.map(args => ({ validator: args.validator.toString(), + amount: args.amount, offenseType: args.offenseType, epochOrSlot: args.epochOrSlot, })), diff --git a/yarn-project/slasher/src/watchers/checkpoint_equivocation_watcher.test.ts b/yarn-project/slasher/src/watchers/checkpoint_equivocation_watcher.test.ts index 8b6fe48f2fde..11c00a2d271b 100644 --- a/yarn-project/slasher/src/watchers/checkpoint_equivocation_watcher.test.ts +++ b/yarn-project/slasher/src/watchers/checkpoint_equivocation_watcher.test.ts @@ -86,9 +86,11 @@ describe('CheckpointEquivocationWatcher', () => { expect(handler).not.toHaveBeenCalled(); }); - it('does not emit when the penalty is zero', async () => { + it('emits a zero-amount offense when the penalty is zero', async () => { await watcher.stop(); + const proposer = EthAddress.random(); config = { ...config, slashDuplicateProposalPenalty: 0n }; + epochCache.getProposerAttesterAddressInSlot.mockResolvedValueOnce(proposer); watcher = new CheckpointEquivocationWatcher(l2BlockSource, epochCache, config); handler = jest.fn(); watcher.on(WANT_TO_SLASH_EVENT, handler); @@ -96,7 +98,14 @@ describe('CheckpointEquivocationWatcher', () => { await emitAndFlush(makeEvent()); - expect(handler).not.toHaveBeenCalled(); + expect(handler).toHaveBeenCalledWith([ + { + validator: proposer, + amount: 0n, + offenseType: OffenseType.DUPLICATE_PROPOSAL, + epochOrSlot: 10n, + }, + ]); }); it('emits separately for distinct slots', async () => { diff --git a/yarn-project/slasher/src/watchers/checkpoint_equivocation_watcher.ts b/yarn-project/slasher/src/watchers/checkpoint_equivocation_watcher.ts index 887986a8ba7e..6e611bcc12d2 100644 --- a/yarn-project/slasher/src/watchers/checkpoint_equivocation_watcher.ts +++ b/yarn-project/slasher/src/watchers/checkpoint_equivocation_watcher.ts @@ -66,10 +66,6 @@ export class CheckpointEquivocationWatcher extends (EventEmitter as new () => Wa /** Public for tests. */ public async onEquivocationDetected(event: CheckpointEquivocationDetectedEvent): Promise { - if (this.config.slashDuplicateProposalPenalty <= 0n) { - return; - } - const proposer = await this.epochCache.getProposerAttesterAddressInSlot(event.slotNumber); if (!proposer) { this.log.warn(`Cannot attribute checkpoint equivocation: no proposer for slot ${event.slotNumber}`, { @@ -89,6 +85,8 @@ export class CheckpointEquivocationWatcher extends (EventEmitter as new () => Wa this.log.info(`Detected checkpoint equivocation offense`, { slotNumber: event.slotNumber, checkpointNumber: event.checkpointNumber, + amount: slashArgs.amount, + offenseType: slashArgs.offenseType, l1ArchiveRoot: event.l1ArchiveRoot.toString(), proposedArchiveRoot: event.proposedArchiveRoot.toString(), validator: proposer.toString(), diff --git a/yarn-project/slasher/src/watchers/data_withholding_watcher.test.ts b/yarn-project/slasher/src/watchers/data_withholding_watcher.test.ts index 310eaf3c3b62..5db77003fa20 100644 --- a/yarn-project/slasher/src/watchers/data_withholding_watcher.test.ts +++ b/yarn-project/slasher/src/watchers/data_withholding_watcher.test.ts @@ -291,16 +291,32 @@ describe('DataWithholdingWatcher', () => { expect(l2BlockSource.getCheckpoint).toHaveBeenCalledTimes(1); }); - it('respects penalty=0 as a disable switch', async () => { + it('emits zero-amount offenses when the penalty is zero', async () => { watcher.updateConfig({ slashDataWithholdingPenalty: 0n }); await startAtSlot(10); - setSyncedSlot(10 + TOLERANCE + 5); + setSyncedSlot(11 + TOLERANCE + 1); + + const slot = 11; + const published = makePublished(slot, 1); + const missing = published.checkpoint.blocks[0].body.txEffects[0].txHash; + const attester = EthAddress.random(); + l2BlockSource.getCheckpoint.mockResolvedValue(published); + mockMissing([missing]); + watcher.attestersBySlot.set(slot, [attester]); const captured = captureEmits(); await watcher.work(); - expect(l2BlockSource.getCheckpoint).not.toHaveBeenCalled(); - expect(captured).toHaveLength(0); + expect(captured).toEqual([ + [ + { + validator: attester, + amount: 0n, + offenseType: OffenseType.DATA_WITHHOLDING, + epochOrSlot: BigInt(slot), + }, + ], + ]); }); it('does not slash a checkpoint with no recoverable attesters even if txs are missing', async () => { diff --git a/yarn-project/slasher/src/watchers/data_withholding_watcher.ts b/yarn-project/slasher/src/watchers/data_withholding_watcher.ts index 091c2be823f6..30588400309c 100644 --- a/yarn-project/slasher/src/watchers/data_withholding_watcher.ts +++ b/yarn-project/slasher/src/watchers/data_withholding_watcher.ts @@ -85,10 +85,6 @@ export class DataWithholdingWatcher extends (EventEmitter as new () => WatcherEm return; } - if (this.config.slashDataWithholdingPenalty === 0n) { - return; // disabled - } - // tolerance is the number of full slots that must elapse after the checkpoint's slot // before we declare its data missing. For checkpoint slot S, we therefore process S // only once we are in slot `S + tolerance + 1` or later. Drive this off the archiver's @@ -175,9 +171,11 @@ export class DataWithholdingWatcher extends (EventEmitter as new () => WatcherEm return; } - this.log.warn(`Detected data withholding at slot ${slot}. Slashing ${attesters.length} attesters.`, { + this.log.warn(`Detected data withholding offense at slot ${slot}`, { slot, checkpointNumber, + amount: this.config.slashDataWithholdingPenalty, + offenseType: OffenseType.DATA_WITHHOLDING, missingTxs: missingTxs.map(h => h.toString()), records: collectionRecords, attesters: attesters.map(a => a.toString()), diff --git a/yarn-project/validator-client/src/validator.test.ts b/yarn-project/validator-client/src/validator.test.ts index d344399a14ac..a4af734ccc5b 100644 --- a/yarn-project/validator-client/src/validator.test.ts +++ b/yarn-project/validator-client/src/validator.test.ts @@ -417,6 +417,15 @@ describe('ValidatorClient', () => { Array.isArray(args) && args[0]?.offenseType === OffenseType.BROADCASTED_INVALID_CHECKPOINT_PROPOSAL, ); + const getAttestedToInvalidCheckpointProposalSlashEvents = ( + emitSpy: jest.SpiedFunction, + ) => + emitSpy.mock.calls.filter( + ([event, args]) => + event === WANT_TO_SLASH_EVENT && + Array.isArray(args) && + args[0]?.offenseType === OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL, + ); beforeEach(async () => { const emptyInHash = computeInHashFromL1ToL2Messages([]); const blockHeader = makeBlockHeader(1, { blockNumber: BlockNumber(100), slotNumber: SlotNumber(100) }); @@ -897,15 +906,24 @@ describe('ValidatorClient', () => { expect(blockSource.getBlockData).toHaveBeenCalledWith({ number: blockNumber }); }); - it('should not emit WANT_TO_SLASH_EVENT if slashing is disabled', async () => { + it('emits zero-amount invalid block proposal offenses when the penalty is zero', async () => { validatorClient.updateConfig({ slashBroadcastedInvalidBlockPenalty: 0n }); const emitSpy = jest.spyOn(validatorClient, 'emit'); blockBuildResult.block.archive.root = Fr.random(); const isValid = await validatorClient.validateBlockProposal(proposal, sender); + const proposer = proposal.getSender(); expect(isValid).toBe(false); - expect(emitSpy).not.toHaveBeenCalled(); + expect(proposer).toBeDefined(); + expect(emitSpy).toHaveBeenCalledWith(WANT_TO_SLASH_EVENT, [ + { + validator: proposer!, + amount: 0n, + offenseType: OffenseType.BROADCASTED_INVALID_BLOCK_PROPOSAL, + epochOrSlot: expect.any(BigInt), + }, + ]); }); it('marks invalid block proposal slots for delayed attestation slashing', async () => { @@ -939,19 +957,17 @@ describe('ValidatorClient', () => { ]); }); - it('does not mark invalid proposal slots when the bad attestation penalty is disabled', async () => { + it('marks invalid proposal slots when the bad attestation penalty is zero', async () => { validatorClient.updateConfig({ slashBroadcastedInvalidBlockPenalty: 0n, slashAttestInvalidCheckpointProposalPenalty: 0n, }); - const emitSpy = jest.spyOn(validatorClient, 'emit'); blockBuildResult.block.archive.root = Fr.random(); const isValid = await validatorClient.validateBlockProposal(proposal, sender); expect(isValid).toBe(false); - expect(validatorClient.hasInvalidProposals(proposal.slotNumber)).toBe(false); - expect(emitSpy).not.toHaveBeenCalled(); + expect(validatorClient.hasInvalidProposals(proposal.slotNumber)).toBe(true); }); it('reexecutes for bad attestation slashing when invalid block proposer slashing is disabled', async () => { @@ -965,6 +981,40 @@ describe('ValidatorClient', () => { expect(checkpointsBuilder.openCheckpoint).toHaveBeenCalled(); }); + it('emits zero-amount bad attestation offenses when the bad attestation penalty is zero', async () => { + await validatorClient.registerHandlers(); + const attestationCallback = p2pClient.registerCheckpointAttestationCallback.mock.calls[0][0]; + validatorClient.updateConfig({ + slashBroadcastedInvalidBlockPenalty: 0n, + slashAttestInvalidCheckpointProposalPenalty: 0n, + }); + const emitSpy = jest.spyOn(validatorClient, 'emit'); + const attesterSigner = Secp256k1Signer.random(); + const attestation = makeCheckpointAttestation({ + header: makeCheckpointHeader(1, { slotNumber: proposal.slotNumber }), + attesterSigner, + }); + blockBuildResult.block.archive.root = Fr.random(); + + const isValid = await validatorClient.validateBlockProposal(proposal, sender); + attestationCallback(attestation); + + expect(isValid).toBe(false); + expect(getAttestedToInvalidCheckpointProposalSlashEvents(emitSpy)).toEqual([ + [ + WANT_TO_SLASH_EVENT, + [ + { + validator: attesterSigner.address, + amount: 0n, + offenseType: OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL, + epochOrSlot: BigInt(proposal.slotNumber), + }, + ], + ], + ]); + }); + it('emits WANT_TO_SLASH_EVENT for checkpoint_header_mismatch checkpoint proposals', async () => { const checkpointHandler = registerAllNodesCheckpointHandler(); const { checkpointProposal, disposeFork } = await makeCheckpointProposalWithHeaderMismatch(); @@ -1041,7 +1091,7 @@ describe('ValidatorClient', () => { ]); }); - it('does not emit checkpoint proposal slash event when the penalty is disabled', async () => { + it('emits zero-amount checkpoint proposal offenses when the penalty is zero', async () => { validatorClient.updateConfig({ slashBroadcastedInvalidCheckpointProposalPenalty: 0n }); const checkpointHandler = registerAllNodesCheckpointHandler(); const { checkpointProposal } = await makeCheckpointProposalWithHeaderMismatch(); @@ -1051,7 +1101,19 @@ describe('ValidatorClient', () => { const attestations = await validatorClient.attestToCheckpointProposal(checkpointProposal, sender); expect(attestations).toBeUndefined(); - expect(getBroadcastedInvalidCheckpointProposalSlashEvents(emitSpy)).toHaveLength(0); + expect(getBroadcastedInvalidCheckpointProposalSlashEvents(emitSpy)).toEqual([ + [ + WANT_TO_SLASH_EVENT, + [ + { + validator: checkpointProposal.getSender()!, + amount: 0n, + offenseType: OffenseType.BROADCASTED_INVALID_CHECKPOINT_PROPOSAL, + epochOrSlot: BigInt(checkpointProposal.slotNumber), + }, + ], + ], + ]); }); it.each(['last_block_not_found', 'checkpoint_already_published'])( @@ -1099,7 +1161,19 @@ describe('ValidatorClient', () => { await checkpointHandler(checkpointProposal, sender); - expect(getBroadcastedInvalidCheckpointProposalSlashEvents(emitSpy)).toHaveLength(0); + expect(getBroadcastedInvalidCheckpointProposalSlashEvents(emitSpy)).toEqual([ + [ + WANT_TO_SLASH_EVENT, + [ + { + validator: checkpointProposal.getSender()!, + amount: 0n, + offenseType: OffenseType.BROADCASTED_INVALID_CHECKPOINT_PROPOSAL, + epochOrSlot: BigInt(checkpointProposal.slotNumber), + }, + ], + ], + ]); expect(validatorClient.hasInvalidProposals(checkpointProposal.slotNumber)).toBe(true); }); diff --git a/yarn-project/validator-client/src/validator.ts b/yarn-project/validator-client/src/validator.ts index 57f7253bfe6b..5baa390ae054 100644 --- a/yarn-project/validator-client/src/validator.ts +++ b/yarn-project/validator-client/src/validator.ts @@ -73,6 +73,7 @@ import { const MAX_PROPOSERS_OF_INVALID_BLOCKS = 1000; const MAX_TRACKED_INVALID_PROPOSAL_SLOTS = 1000; const MAX_TRACKED_INVALID_CHECKPOINT_PROPOSALS = 1000; +const MAX_TRACKED_BAD_ATTESTATIONS = 10_000; // What errors from the block proposal handler result in slashing const SLASHABLE_BLOCK_PROPOSAL_VALIDATION_RESULT: BlockProposalValidationFailureReason[] = [ @@ -129,6 +130,7 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) private proposersOfInvalidBlocks = FifoSet.withLimit(MAX_PROPOSERS_OF_INVALID_BLOCKS); private slotsWithInvalidProposals = FifoSet.withLimit(MAX_TRACKED_INVALID_PROPOSAL_SLOTS); private invalidCheckpointProposalOffenseKeys = FifoSet.withLimit(MAX_TRACKED_INVALID_CHECKPOINT_PROPOSALS); + private badAttestationOffenseKeys = FifoSet.withLimit(MAX_TRACKED_BAD_ATTESTATIONS); private slotsWithProposalEquivocation = FifoSet.withLimit(MAX_TRACKED_INVALID_PROPOSAL_SLOTS); /** Tracks the last checkpoint proposal we attested to, to prevent equivocation. */ @@ -430,6 +432,10 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) this.handleDuplicateAttestation(info); }); + this.p2pClient.registerCheckpointAttestationCallback((attestation: CheckpointAttestation) => { + this.handleCheckpointAttestation(attestation); + }); + const myAddresses = this.getValidatorAddresses(); this.p2pClient.registerThisValidatorAddresses(myAddresses); @@ -476,27 +482,8 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) fishermanMode: this.config.fishermanMode || false, }); - // Reexecute txs if we are part of the committee, or if slashing is enabled, or if we are configured to always reexecute. - // In fisherman mode, we always reexecute to validate proposals. - const { - slashBroadcastedInvalidBlockPenalty, - slashAttestInvalidCheckpointProposalPenalty, - alwaysReexecuteBlockProposals, - fishermanMode, - } = this.config; - const shouldReexecute = - fishermanMode || - slashBroadcastedInvalidBlockPenalty > 0n || - slashAttestInvalidCheckpointProposalPenalty > 0n || - partOfCommittee || - alwaysReexecuteBlockProposals || - this.blobClient.canUpload(); - - const validationResult = await this.proposalHandler.handleBlockProposal( - proposal, - proposalSender, - !!shouldReexecute && !escapeHatchOpen, - ); + // Reexecute outside the escape hatch so slashing observers can detect invalid proposals even when penalties are 0. + const validationResult = await this.proposalHandler.handleBlockProposal(proposal, proposalSender, !escapeHatchOpen); if (!validationResult.isValid) { const reason = validationResult.reason || 'unknown'; @@ -524,13 +511,13 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) validationResult.reason && SLASHABLE_BLOCK_PROPOSAL_VALIDATION_RESULT.includes(validationResult.reason) ) { - if (slashBroadcastedInvalidBlockPenalty > 0n) { - this.log.warn(`Slashing proposer for invalid block proposal`, proposalInfo); - this.slashInvalidBlock(proposal); - } - if (slashAttestInvalidCheckpointProposalPenalty > 0n) { - this.markInvalidProposalSlot(proposal.slotNumber); - } + this.log.warn(`Detected invalid block proposal offense`, { + ...proposalInfo, + amount: this.config.slashBroadcastedInvalidBlockPenalty, + offenseType: OffenseType.BROADCASTED_INVALID_BLOCK_PROPOSAL, + }); + this.slashInvalidBlock(proposal); + this.markInvalidProposalSlot(proposal.slotNumber); } return false; } @@ -769,23 +756,19 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) return; } - if (this.config.slashAttestInvalidCheckpointProposalPenalty > 0n) { - this.markInvalidProposalSlot(proposal.slotNumber); - } + this.markInvalidProposalSlot(proposal.slotNumber); if (this.slashInvalidCheckpointProposal(proposal)) { - this.log.warn(`Slashing proposer for invalid checkpoint proposal`, { + this.log.warn(`Detected invalid checkpoint proposal offense`, { ...proposalInfo, reason: result.reason, + amount: this.config.slashBroadcastedInvalidCheckpointProposalPenalty, + offenseType: OffenseType.BROADCASTED_INVALID_CHECKPOINT_PROPOSAL, }); } } private slashInvalidCheckpointProposal(proposal: CheckpointProposalCore): boolean { - if (this.config.slashBroadcastedInvalidCheckpointProposalPenalty <= 0n) { - return false; - } - const proposer = proposal.getSender(); if (!proposer) { this.log.warn(`Cannot slash checkpoint proposal with invalid signature`, { @@ -816,6 +799,47 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) this.slotsWithInvalidProposals.add(slotNumber); } + private handleCheckpointAttestation(attestation: CheckpointAttestation): void { + const slotNumber = attestation.slotNumber; + if (!this.slotsWithInvalidProposals.has(slotNumber) || this.slotsWithProposalEquivocation.has(slotNumber)) { + return; + } + + const attester = attestation.getSender(); + if (!attester) { + this.log.warn(`Cannot slash checkpoint attestation with invalid signature`, { + slotNumber, + archive: attestation.archive.toString(), + }); + return; + } + + this.slashAttestedToInvalidCheckpointProposal(slotNumber, attester); + } + + private slashAttestedToInvalidCheckpointProposal(slotNumber: SlotNumber, attester: EthAddress): void { + const offenseKey = `${attester.toString()}:${OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL}:${slotNumber}`; + if (!this.badAttestationOffenseKeys.addIfAbsent(offenseKey)) { + return; + } + + this.log.warn(`Detected attestation to invalid checkpoint proposal offense`, { + attester: attester.toString(), + slotNumber, + amount: this.config.slashAttestInvalidCheckpointProposalPenalty, + offenseType: OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL, + }); + + this.emit(WANT_TO_SLASH_EVENT, [ + { + validator: attester, + amount: this.config.slashAttestInvalidCheckpointProposalPenalty, + offenseType: OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL, + epochOrSlot: BigInt(slotNumber), + }, + ]); + } + /** * Handle detection of a duplicate proposal (equivocation). * Emits a slash event when a proposer sends multiple proposals for the same position. @@ -824,13 +848,14 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) const { slot, proposer, type } = info; this.slotsWithProposalEquivocation.add(slot); - this.log.warn(`Triggering slash event for duplicate ${type} proposal from ${proposer.toString()} at slot ${slot}`, { + this.log.warn(`Detected duplicate ${type} proposal offense from ${proposer.toString()} at slot ${slot}`, { proposer: proposer.toString(), slot, type, + amount: this.config.slashDuplicateProposalPenalty, + offenseType: OffenseType.DUPLICATE_PROPOSAL, }); - // Emit slash event this.emit(WANT_TO_SLASH_EVENT, [ { validator: proposer, @@ -855,9 +880,11 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) private handleDuplicateAttestation(info: DuplicateAttestationInfo): void { const { slot, attester } = info; - this.log.warn(`Triggering slash event for duplicate attestation from ${attester.toString()} at slot ${slot}`, { + this.log.warn(`Detected duplicate attestation offense from ${attester.toString()} at slot ${slot}`, { attester: attester.toString(), slot, + amount: this.config.slashDuplicateAttestationPenalty, + offenseType: OffenseType.DUPLICATE_ATTESTATION, }); this.emit(WANT_TO_SLASH_EVENT, [ From 8bcfe11e35c3ee77eb21edcea25e9dc3caca7982 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 28 May 2026 10:23:20 +0100 Subject: [PATCH 21/27] fix: log slashing offense names (#23565) Fix A-1075 --- .../aztec-node/src/sentinel/sentinel.ts | 13 +++++++-- .../slasher/src/slash_offenses_collector.ts | 23 +++++++++++---- yarn-project/slasher/src/slasher_client.ts | 29 +++++++++++++++---- .../watchers/attestations_block_watcher.ts | 23 ++++++++++----- .../attested_invalid_proposal_watcher.test.ts | 14 --------- .../attested_invalid_proposal_watcher.ts | 6 ++-- ...nvalid_checkpoint_proposal_watcher.test.ts | 15 ---------- ...ted_invalid_checkpoint_proposal_watcher.ts | 4 +-- .../checkpoint_equivocation_watcher.ts | 4 +-- .../src/watchers/data_withholding_watcher.ts | 6 ++-- .../validator-client/src/validator.ts | 21 +++++++------- 11 files changed, 88 insertions(+), 70 deletions(-) diff --git a/yarn-project/aztec-node/src/sentinel/sentinel.ts b/yarn-project/aztec-node/src/sentinel/sentinel.ts index a9746cfa0643..78a50ff8c075 100644 --- a/yarn-project/aztec-node/src/sentinel/sentinel.ts +++ b/yarn-project/aztec-node/src/sentinel/sentinel.ts @@ -18,6 +18,7 @@ import { type WantToSlashArgs, type Watcher, type WatcherEmitter, + getOffenseTypeName, } from '@aztec/slasher'; import type { SlasherConfig } from '@aztec/slasher/config'; import { @@ -358,9 +359,17 @@ export class Sentinel extends (EventEmitter as new () => WatcherEmitter) impleme })); if (criminals.length > 0) { - this.logger.verbose( + this.logger.info( `Identified ${criminals.length} inactivity offenses in at least ${epochThreshold} consecutive epochs`, - { offenses: args, epochThreshold }, + { + offenses: args.map(arg => ({ + validator: arg.validator.toString(), + amount: arg.amount, + offenseType: getOffenseTypeName(arg.offenseType), + epochOrSlot: arg.epochOrSlot, + })), + epochThreshold, + }, ); this.emit(WANT_TO_SLASH_EVENT, args); } diff --git a/yarn-project/slasher/src/slash_offenses_collector.ts b/yarn-project/slasher/src/slash_offenses_collector.ts index f92e5dd790ad..22833112fe0a 100644 --- a/yarn-project/slasher/src/slash_offenses_collector.ts +++ b/yarn-project/slasher/src/slash_offenses_collector.ts @@ -4,7 +4,7 @@ import { SerialQueue } from '@aztec/foundation/queue'; import type { Prettify } from '@aztec/foundation/types'; import type { L1RollupConstants } from '@aztec/stdlib/epoch-helpers'; import type { SlasherConfig } from '@aztec/stdlib/interfaces/server'; -import { type Offense, getSlotForOffense } from '@aztec/stdlib/slashing'; +import { type Offense, getOffenseTypeName, getSlotForOffense } from '@aztec/stdlib/slashing'; import type { SlasherOffensesStore } from './stores/offenses_store.js'; import { @@ -89,7 +89,7 @@ export class SlashOffensesCollector { }; if (this.shouldSkipOffense(offense)) { - this.log.verbose('Skipping offense during grace period', offense); + this.log.verbose('Skipping offense during grace period', this.getOffenseLogData(offense)); continue; } @@ -98,13 +98,16 @@ export class SlashOffensesCollector { if (this.settings.slashingAmounts) { const minSlash = this.settings.slashingAmounts[0]; if (arg.amount < minSlash) { - this.log.warn(`Offense amount ${arg.amount} is below minimum slashing amount ${minSlash}`); + this.log.warn( + `Offense amount ${arg.amount} is below minimum slashing amount ${minSlash}`, + this.getOffenseLogData(offense), + ); } } - this.log.info(`Adding pending offense for validator ${arg.validator}`, offense); + this.log.info(`Adding pending offense for validator ${arg.validator}`, this.getOffenseLogData(offense)); } else { - this.log.debug('Skipping repeated offense', offense); + this.log.debug('Skipping repeated offense', this.getOffenseLogData(offense)); } } } @@ -114,7 +117,7 @@ export class SlashOffensesCollector { const cleared = await this.offensesStore.clearOffenses(arg); if (cleared > 0) { this.log.info(`Cleared ${cleared} pending offenses`, { - offenseType: arg.offenseType, + offenseType: getOffenseTypeName(arg.offenseType), epochOrSlot: arg.epochOrSlot, validators: arg.validators?.map(validator => validator.toString()), }); @@ -139,6 +142,14 @@ export class SlashOffensesCollector { return offenseSlot < this.settings.rollupRegisteredAtL2Slot + this.config.slashGracePeriodL2Slots; } + private getOffenseLogData(offense: Offense) { + return { + ...offense, + validator: offense.validator.toString(), + offenseType: getOffenseTypeName(offense.offenseType), + }; + } + private enqueueStoreMutation(label: string, callback: () => Promise) { void this.storeMutationQueue.put(callback).catch(err => this.log.error(`Error handling ${label}`, err)); } diff --git a/yarn-project/slasher/src/slasher_client.ts b/yarn-project/slasher/src/slasher_client.ts index f0b748a9cec7..f6dce62add58 100644 --- a/yarn-project/slasher/src/slasher_client.ts +++ b/yarn-project/slasher/src/slasher_client.ts @@ -14,6 +14,7 @@ import { type ProposerSlashAction, type ProposerSlashActionProvider, getEpochsForRound, + getOffenseTypeName, getSlashConsensusVotesFromOffenses, } from '@aztec/stdlib/slashing'; @@ -50,6 +51,14 @@ export type SlasherClientConfig = SlashOffensesCollectorConfig & 'slashValidatorsAlways' | 'slashValidatorsNever' | 'slashExecuteRoundsLookBack' | 'slashMaxPayloadSize' >; +type AlwaysSlashOffense = { + validator: EthAddress; + amount: bigint; + offenseType: OffenseType.UNKNOWN; +}; + +type SlashVoteOffense = Offense | AlwaysSlashOffense; + /** * The Slasher client is responsible for managing slashable offenses using * the consensus-based slashing model where proposers vote on individual validator offenses. @@ -309,7 +318,7 @@ export class SlasherClient implements ProposerSlashActionProvider, SlasherClient // Compute offenses to slash, by loading the offenses for this round, adding synthetic offenses // for validators that should always be slashed, and removing the ones that should never be slashed. const offensesForRound = await this.gatherOffensesForRound(currentRound); - const offensesFromAlwaysSlash = (this.config.slashValidatorsAlways ?? []).map(validator => ({ + const offensesFromAlwaysSlash: AlwaysSlashOffense[] = (this.config.slashValidatorsAlways ?? []).map(validator => ({ validator, amount: this.settings.slashingAmounts[2], offenseType: OffenseType.UNKNOWN, @@ -323,7 +332,7 @@ export class SlasherClient implements ProposerSlashActionProvider, SlasherClient slotNumber, currentRound, slashedRound, - offensesToForgive, + offensesFromAlwaysSlash: offensesFromAlwaysSlash.map(getOffenseLogData), slashValidatorsAlways: this.config.slashValidatorsAlways, }); } @@ -333,7 +342,7 @@ export class SlasherClient implements ProposerSlashActionProvider, SlasherClient slotNumber, currentRound, slashedRound, - offensesToForgive, + offensesToForgive: offensesToForgive.map(getOffenseLogData), slashValidatorsNever: this.config.slashValidatorsNever, }); } @@ -347,7 +356,7 @@ export class SlasherClient implements ProposerSlashActionProvider, SlasherClient slotNumber, currentRound, slashedRound, - offensesToSlash, + offensesToSlash: offensesToSlash.map(getOffenseLogData), }); const committees = await this.collectCommitteesActiveDuringRound(slashedRound); @@ -365,7 +374,7 @@ export class SlasherClient implements ProposerSlashActionProvider, SlasherClient slotNumber, currentRound, slashedRound, - offensesToSlash, + offensesToSlash: offensesToSlash.map(getOffenseLogData), committees, }); return undefined; @@ -376,7 +385,7 @@ export class SlasherClient implements ProposerSlashActionProvider, SlasherClient slashedRound, currentRound, votes, - offensesToSlash, + offensesToSlash: offensesToSlash.map(getOffenseLogData), }); this.log.debug(`Computed votes for slashing ${offensesToSlash.length} offenses`, { @@ -437,3 +446,11 @@ export class SlasherClient implements ProposerSlashActionProvider, SlasherClient return round - BigInt(this.settings.slashingOffsetInRounds); } } + +function getOffenseLogData(offense: SlashVoteOffense) { + return { + ...offense, + validator: offense.validator.toString(), + offenseType: getOffenseTypeName(offense.offenseType), + }; +} diff --git a/yarn-project/slasher/src/watchers/attestations_block_watcher.ts b/yarn-project/slasher/src/watchers/attestations_block_watcher.ts index f49d56ac9c1e..886797b2f445 100644 --- a/yarn-project/slasher/src/watchers/attestations_block_watcher.ts +++ b/yarn-project/slasher/src/watchers/attestations_block_watcher.ts @@ -10,7 +10,7 @@ import { type ValidateCheckpointNegativeResult, } from '@aztec/stdlib/block'; import { getEpochAtSlot } from '@aztec/stdlib/epoch-helpers'; -import { OffenseType } from '@aztec/stdlib/slashing'; +import { OffenseType, getOffenseTypeName } from '@aztec/stdlib/slashing'; import EventEmitter from 'node:events'; @@ -134,9 +134,13 @@ export class AttestationsBlockWatcher extends (EventEmitter as new () => Watcher epochOrSlot: BigInt(slot), }; - this.log.info(`Want to slash proposer of checkpoint ${checkpointNumber} due to ${reason}`, { + this.log.info(`Detected invalid attestations checkpoint proposer offense`, { ...checkpoint, - ...args, + reason, + validator: args.validator.toString(), + amount: args.amount, + offenseType: getOffenseTypeName(args.offenseType), + epochOrSlot: args.epochOrSlot, }); this.emit(WANT_TO_SLASH_EVENT, [args]); @@ -168,10 +172,15 @@ export class AttestationsBlockWatcher extends (EventEmitter as new () => Watcher epochOrSlot: BigInt(slot), }; - this.log.info( - `Want to slash proposer of checkpoint ${checkpoint.checkpointNumber} built on invalid checkpoint ${ancestorCheckpointNumber}`, - { ...checkpoint, ancestorArchiveRoot: ancestorArchiveRoot.toString(), ...args }, - ); + this.log.info(`Detected invalid descendant checkpoint proposer offense`, { + ...checkpoint, + ancestorCheckpointNumber, + ancestorArchiveRoot: ancestorArchiveRoot.toString(), + validator: args.validator.toString(), + amount: args.amount, + offenseType: getOffenseTypeName(args.offenseType), + epochOrSlot: args.epochOrSlot, + }); this.emit(WANT_TO_SLASH_EVENT, [args]); } diff --git a/yarn-project/slasher/src/watchers/attested_invalid_proposal_watcher.test.ts b/yarn-project/slasher/src/watchers/attested_invalid_proposal_watcher.test.ts index 639b73ca4153..db2696a3b0a0 100644 --- a/yarn-project/slasher/src/watchers/attested_invalid_proposal_watcher.test.ts +++ b/yarn-project/slasher/src/watchers/attested_invalid_proposal_watcher.test.ts @@ -119,20 +119,6 @@ describe('AttestedInvalidProposalWatcher', () => { expect(handler).toHaveBeenCalledTimes(1); }); - it('deduplicates repeated scans for the same offense when the penalty changes', async () => { - const slot = SlotNumber(10); - invalidProposalSlots.add(slot); - const attestation = await makeAttestation(slot); - p2pClient.getCheckpointAttestationsForSlot.mockResolvedValue([attestation]); - - watcher.updateConfig({ slashAttestInvalidCheckpointProposalPenalty: 0n }); - await watcher.scanSlot(slot); - watcher.updateConfig({ slashAttestInvalidCheckpointProposalPenalty: 13n }); - await watcher.scanSlot(slot); - - expect(handler).toHaveBeenCalledTimes(1); - }); - it('scans only marked invalid proposal slots once they are past the scan lag', async () => { watcher = new AttestedInvalidProposalWatcher( p2pClient, diff --git a/yarn-project/slasher/src/watchers/attested_invalid_proposal_watcher.ts b/yarn-project/slasher/src/watchers/attested_invalid_proposal_watcher.ts index 1d6fbe082431..440b23413f34 100644 --- a/yarn-project/slasher/src/watchers/attested_invalid_proposal_watcher.ts +++ b/yarn-project/slasher/src/watchers/attested_invalid_proposal_watcher.ts @@ -8,7 +8,7 @@ import { RunningPromise } from '@aztec/foundation/running-promise'; import type { L2BlockSource } from '@aztec/stdlib/block'; import type { P2PClient, SlasherConfig } from '@aztec/stdlib/interfaces/server'; import type { CheckpointAttestation } from '@aztec/stdlib/p2p'; -import { OffenseType } from '@aztec/stdlib/slashing'; +import { OffenseType, getOffenseTypeName } from '@aztec/stdlib/slashing'; import EventEmitter from 'node:events'; @@ -127,12 +127,12 @@ export class AttestedInvalidProposalWatcher extends (EventEmitter as new () => W return; } - this.log.warn('Detected attestations to invalid checkpoint proposal', { + this.log.info('Detected attestations to invalid checkpoint proposal', { slot, offenses: slashArgs.map(args => ({ validator: args.validator.toString(), amount: args.amount, - offenseType: args.offenseType, + offenseType: getOffenseTypeName(args.offenseType), epochOrSlot: args.epochOrSlot, })), }); diff --git a/yarn-project/slasher/src/watchers/broadcasted_invalid_checkpoint_proposal_watcher.test.ts b/yarn-project/slasher/src/watchers/broadcasted_invalid_checkpoint_proposal_watcher.test.ts index e8611f430582..f9f5b25ca669 100644 --- a/yarn-project/slasher/src/watchers/broadcasted_invalid_checkpoint_proposal_watcher.test.ts +++ b/yarn-project/slasher/src/watchers/broadcasted_invalid_checkpoint_proposal_watcher.test.ts @@ -228,21 +228,6 @@ describe('BroadcastedInvalidCheckpointProposalWatcher', () => { expect(handler).toHaveBeenCalledTimes(1); }); - it('deduplicates repeated scans for the same offense when the penalty changes', async () => { - const signer = Secp256k1Signer.random(); - const slot = SlotNumber(10); - const blocks = await makeBlocks(signer, slot, 4); - const checkpoint = await makeCheckpointCore(signer, slot, blocks[1]); - mockProposals(slot, blocks, [checkpoint]); - - watcher.updateConfig({ slashBroadcastedInvalidCheckpointProposalPenalty: 0n }); - await watcher.scanSlot(slot); - watcher.updateConfig({ slashBroadcastedInvalidCheckpointProposalPenalty: 11n }); - await watcher.scanSlot(slot); - - expect(handler).toHaveBeenCalledTimes(1); - }); - it('scans a lookback of closed slots', async () => { const signer = Secp256k1Signer.random(); const slot = SlotNumber(10); diff --git a/yarn-project/slasher/src/watchers/broadcasted_invalid_checkpoint_proposal_watcher.ts b/yarn-project/slasher/src/watchers/broadcasted_invalid_checkpoint_proposal_watcher.ts index 64dacf75e059..0ac35019774b 100644 --- a/yarn-project/slasher/src/watchers/broadcasted_invalid_checkpoint_proposal_watcher.ts +++ b/yarn-project/slasher/src/watchers/broadcasted_invalid_checkpoint_proposal_watcher.ts @@ -8,7 +8,7 @@ import { RunningPromise } from '@aztec/foundation/running-promise'; import type { L2BlockSource } from '@aztec/stdlib/block'; import type { P2PClient, SlasherConfig } from '@aztec/stdlib/interfaces/server'; import type { BlockProposal, CheckpointProposalCore } from '@aztec/stdlib/p2p'; -import { OffenseType } from '@aztec/stdlib/slashing'; +import { OffenseType, getOffenseTypeName } from '@aztec/stdlib/slashing'; import EventEmitter from 'node:events'; @@ -118,7 +118,7 @@ export class BroadcastedInvalidCheckpointProposalWatcher offenses: slashArgs.map(args => ({ validator: args.validator.toString(), amount: args.amount, - offenseType: args.offenseType, + offenseType: getOffenseTypeName(args.offenseType), epochOrSlot: args.epochOrSlot, })), }); diff --git a/yarn-project/slasher/src/watchers/checkpoint_equivocation_watcher.ts b/yarn-project/slasher/src/watchers/checkpoint_equivocation_watcher.ts index 6e611bcc12d2..0070de630f98 100644 --- a/yarn-project/slasher/src/watchers/checkpoint_equivocation_watcher.ts +++ b/yarn-project/slasher/src/watchers/checkpoint_equivocation_watcher.ts @@ -7,7 +7,7 @@ import { L2BlockSourceEvents, } from '@aztec/stdlib/block'; import type { SlasherConfig } from '@aztec/stdlib/interfaces/server'; -import { OffenseType } from '@aztec/stdlib/slashing'; +import { OffenseType, getOffenseTypeName } from '@aztec/stdlib/slashing'; import EventEmitter from 'node:events'; @@ -86,7 +86,7 @@ export class CheckpointEquivocationWatcher extends (EventEmitter as new () => Wa slotNumber: event.slotNumber, checkpointNumber: event.checkpointNumber, amount: slashArgs.amount, - offenseType: slashArgs.offenseType, + offenseType: getOffenseTypeName(slashArgs.offenseType), l1ArchiveRoot: event.l1ArchiveRoot.toString(), proposedArchiveRoot: event.proposedArchiveRoot.toString(), validator: proposer.toString(), diff --git a/yarn-project/slasher/src/watchers/data_withholding_watcher.ts b/yarn-project/slasher/src/watchers/data_withholding_watcher.ts index 30588400309c..65f578f9007c 100644 --- a/yarn-project/slasher/src/watchers/data_withholding_watcher.ts +++ b/yarn-project/slasher/src/watchers/data_withholding_watcher.ts @@ -9,7 +9,7 @@ import { getAttestationInfoFromPublishedCheckpoint } from '@aztec/stdlib/block'; import type { CheckpointReexecutionTracker, PublishedCheckpoint } from '@aztec/stdlib/checkpoint'; import type { ITxProvider, P2PApi, SlasherConfig } from '@aztec/stdlib/interfaces/server'; import { ConsensusPayload, type CoordinationSignatureContext } from '@aztec/stdlib/p2p'; -import { OffenseType } from '@aztec/stdlib/slashing'; +import { OffenseType, getOffenseTypeName } from '@aztec/stdlib/slashing'; import type { TxHash } from '@aztec/stdlib/tx'; import EventEmitter from 'node:events'; @@ -171,11 +171,11 @@ export class DataWithholdingWatcher extends (EventEmitter as new () => WatcherEm return; } - this.log.warn(`Detected data withholding offense at slot ${slot}`, { + this.log.info(`Detected data withholding offense at slot ${slot}`, { slot, checkpointNumber, amount: this.config.slashDataWithholdingPenalty, - offenseType: OffenseType.DATA_WITHHOLDING, + offenseType: getOffenseTypeName(OffenseType.DATA_WITHHOLDING), missingTxs: missingTxs.map(h => h.toString()), records: collectionRecords, attesters: attesters.map(a => a.toString()), diff --git a/yarn-project/validator-client/src/validator.ts b/yarn-project/validator-client/src/validator.ts index 5baa390ae054..7b155518933a 100644 --- a/yarn-project/validator-client/src/validator.ts +++ b/yarn-project/validator-client/src/validator.ts @@ -19,6 +19,7 @@ import { WANT_TO_SLASH_EVENT, type Watcher, type WatcherEmitter, + getOffenseTypeName, } from '@aztec/slasher'; import type { AztecAddress } from '@aztec/stdlib/aztec-address'; import type { CommitteeAttestationsAndSigners, L2BlockSink, L2BlockSource } from '@aztec/stdlib/block'; @@ -511,10 +512,10 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) validationResult.reason && SLASHABLE_BLOCK_PROPOSAL_VALIDATION_RESULT.includes(validationResult.reason) ) { - this.log.warn(`Detected invalid block proposal offense`, { + this.log.info(`Detected invalid block proposal offense`, { ...proposalInfo, amount: this.config.slashBroadcastedInvalidBlockPenalty, - offenseType: OffenseType.BROADCASTED_INVALID_BLOCK_PROPOSAL, + offenseType: getOffenseTypeName(OffenseType.BROADCASTED_INVALID_BLOCK_PROPOSAL), }); this.slashInvalidBlock(proposal); this.markInvalidProposalSlot(proposal.slotNumber); @@ -759,11 +760,11 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) this.markInvalidProposalSlot(proposal.slotNumber); if (this.slashInvalidCheckpointProposal(proposal)) { - this.log.warn(`Detected invalid checkpoint proposal offense`, { + this.log.info(`Detected invalid checkpoint proposal offense`, { ...proposalInfo, reason: result.reason, amount: this.config.slashBroadcastedInvalidCheckpointProposalPenalty, - offenseType: OffenseType.BROADCASTED_INVALID_CHECKPOINT_PROPOSAL, + offenseType: getOffenseTypeName(OffenseType.BROADCASTED_INVALID_CHECKPOINT_PROPOSAL), }); } } @@ -823,11 +824,11 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) return; } - this.log.warn(`Detected attestation to invalid checkpoint proposal offense`, { + this.log.info(`Detected attestation to invalid checkpoint proposal offense`, { attester: attester.toString(), slotNumber, amount: this.config.slashAttestInvalidCheckpointProposalPenalty, - offenseType: OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL, + offenseType: getOffenseTypeName(OffenseType.ATTESTED_TO_INVALID_CHECKPOINT_PROPOSAL), }); this.emit(WANT_TO_SLASH_EVENT, [ @@ -848,12 +849,12 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) const { slot, proposer, type } = info; this.slotsWithProposalEquivocation.add(slot); - this.log.warn(`Detected duplicate ${type} proposal offense from ${proposer.toString()} at slot ${slot}`, { + this.log.info(`Detected duplicate ${type} proposal offense from ${proposer.toString()} at slot ${slot}`, { proposer: proposer.toString(), slot, type, amount: this.config.slashDuplicateProposalPenalty, - offenseType: OffenseType.DUPLICATE_PROPOSAL, + offenseType: getOffenseTypeName(OffenseType.DUPLICATE_PROPOSAL), }); this.emit(WANT_TO_SLASH_EVENT, [ @@ -880,11 +881,11 @@ export class ValidatorClient extends (EventEmitter as new () => WatcherEmitter) private handleDuplicateAttestation(info: DuplicateAttestationInfo): void { const { slot, attester } = info; - this.log.warn(`Detected duplicate attestation offense from ${attester.toString()} at slot ${slot}`, { + this.log.info(`Detected duplicate attestation offense from ${attester.toString()} at slot ${slot}`, { attester: attester.toString(), slot, amount: this.config.slashDuplicateAttestationPenalty, - offenseType: OffenseType.DUPLICATE_ATTESTATION, + offenseType: getOffenseTypeName(OffenseType.DUPLICATE_ATTESTATION), }); this.emit(WANT_TO_SLASH_EVENT, [ From c22eb86126a75d06fffa0ab6f794acfd4cf79c9a Mon Sep 17 00:00:00 2001 From: Facundo Date: Thu, 28 May 2026 07:52:54 -0300 Subject: [PATCH 22/27] feat(p2p): tx validation cache (#23585) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Overview Adds a tx validation cache to the p2p layer so that repeated validation of the same transaction by the same validator reuses the prior result instead of redoing the work (notably the expensive proof verification). Downside: Using this cache for validations adds up to 7ms overhead for each validation, when the object needs to be hashed. This is actually entirely (+90%) dominated by `.toBuffer()` time. Cached validation is added for on-demand tx collection, but NOT for gossip and RPC ingress. ## Changes **Cache core (`p2p/src/msg_validators/tx_validator/`)** - `TxValidationCache` — bounded, LRU-evicting cache keyed by `(validatorSymbol, txHash)`. Stores the in-flight promise before awaiting, so concurrent validations of the same tx coalesce into a single call. `get`/`set`/`delete` take the cache key directly; `key(validatorSymbol, tx)` builds it. - `CachedTxValidator` — wraps any `TxValidator` to route `validateTx` through the cache using the validator's `identifier` symbol. `DataTxValidator` and `TxProofValidator` gained stable `identifier`s. - `factory.ts` — threads an optional `TxValidationCache` through the gossip (first/second stage), block-proposal, on-demand, and RPC validator builders, wrapping the state-independent validators (`DataTxValidator`, `TxProofValidator`, and the minimum-integrity aggregate) in `CachedTxValidator`. **LRU map extracted to foundation (`foundation/src/collection/lru_map.ts`)** - The hand-rolled doubly-linked-list LRU bookkeeping was factored out of `TxValidationCache` into a generic `LruMap`, mirroring the existing `LruSet`. `TxValidationCache` now composes an `LruMap>`. Added `LruMap` unit tests. **Wiring** - New `P2P_TX_VALIDATION_CACHE_SIZE` env var / `txValidationCacheSize` config (cache disabled when `0`). - `createP2PClient` constructs the cache and passes it to `LibP2PService` (gossip + block-proposal paths) and to the batch-tx-requester's on-demand validator config. **Benchmarks** - Added a sha256-based TX hash benchmark. Closes https://linear.app/aztec-labs/issue/A-934/dont-repeatedly-verify-retrieved-transactions . --- .../foundation/src/collection/index.ts | 1 + .../foundation/src/collection/lru_map.test.ts | 163 +++++++++++++++ .../foundation/src/collection/lru_map.ts | 143 +++++++++++++ yarn-project/foundation/src/config/env_var.ts | 1 + yarn-project/p2p/src/client/factory.ts | 14 +- yarn-project/p2p/src/config.ts | 8 + .../tx_validator/cached_tx_validator.test.ts | 71 +++++++ .../tx_validator/cached_tx_validator.ts | 31 +++ .../tx_validator/data_validator.ts | 2 + .../msg_validators/tx_validator/factory.ts | 19 +- .../src/msg_validators/tx_validator/index.ts | 2 + .../tx_validator/tx_proof_validator.ts | 2 + .../tx_validator/tx_validation_cache.test.ts | 190 ++++++++++++++++++ .../tx_validator/tx_validation_cache.ts | 102 ++++++++++ .../p2p/src/services/libp2p/libp2p_service.ts | 7 + .../batch-tx-requester/tx_validator.ts | 15 +- yarn-project/stdlib/src/tx/tx_bench.test.ts | 77 +++---- 17 files changed, 802 insertions(+), 46 deletions(-) create mode 100644 yarn-project/foundation/src/collection/lru_map.test.ts create mode 100644 yarn-project/foundation/src/collection/lru_map.ts create mode 100644 yarn-project/p2p/src/msg_validators/tx_validator/cached_tx_validator.test.ts create mode 100644 yarn-project/p2p/src/msg_validators/tx_validator/cached_tx_validator.ts create mode 100644 yarn-project/p2p/src/msg_validators/tx_validator/tx_validation_cache.test.ts create mode 100644 yarn-project/p2p/src/msg_validators/tx_validator/tx_validation_cache.ts diff --git a/yarn-project/foundation/src/collection/index.ts b/yarn-project/foundation/src/collection/index.ts index 64ebf402e3ba..b62ed9c1f586 100644 --- a/yarn-project/foundation/src/collection/index.ts +++ b/yarn-project/foundation/src/collection/index.ts @@ -1,3 +1,4 @@ export * from './array.js'; +export * from './lru_map.js'; export * from './lru_set.js'; export * from './object.js'; diff --git a/yarn-project/foundation/src/collection/lru_map.test.ts b/yarn-project/foundation/src/collection/lru_map.test.ts new file mode 100644 index 000000000000..9c25853c07f6 --- /dev/null +++ b/yarn-project/foundation/src/collection/lru_map.test.ts @@ -0,0 +1,163 @@ +import { LruMap } from './lru_map.js'; + +describe('LruMap', () => { + it('stores and retrieves values', () => { + const map = new LruMap(3); + map.set('a', 1); + map.set('b', 2); + expect(map.get('a')).toBe(1); + expect(map.get('b')).toBe(2); + expect(map.get('c')).toBeUndefined(); + }); + + it('reports correct size', () => { + const map = new LruMap(5); + expect(map.size).toBe(0); + map.set(1, 'a'); + expect(map.size).toBe(1); + map.set(2, 'b'); + map.set(3, 'c'); + expect(map.size).toBe(3); + }); + + it('overwrites the value of an existing key without growing', () => { + const map = new LruMap(5); + map.set('a', 1); + map.set('a', 2); + expect(map.get('a')).toBe(2); + expect(map.size).toBe(1); + }); + + it('does not grow beyond maxSize', () => { + const map = new LruMap(3); + map.set(1, 1); + map.set(2, 2); + map.set(3, 3); + map.set(4, 4); + expect(map.size).toBe(3); + expect(map.get(1)).toBeUndefined(); // evicted (least recent) + expect(map.get(2)).toBe(2); + expect(map.get(3)).toBe(3); + expect(map.get(4)).toBe(4); + }); + + it('evicts least recently used, not least recently added', () => { + const map = new LruMap(3); + map.set('a', 1); + map.set('b', 2); + map.set('c', 3); + + // Access 'a' via get(), making it the most recently used + expect(map.get('a')).toBe(1); + + // Now 'b' is the least recently used. Adding 'd' should evict 'b'. + map.set('d', 4); + expect(map.get('b')).toBeUndefined(); // evicted + expect(map.get('a')).toBe(1); // kept (was refreshed) + expect(map.get('c')).toBe(3); + expect(map.get('d')).toBe(4); + }); + + it('refreshes recency on set() of existing key', () => { + const map = new LruMap(3); + map.set('a', 1); + map.set('b', 2); + map.set('c', 3); + + // Re-set 'a', refreshing its recency + map.set('a', 10); + + // 'b' is now least recent. Adding 'd' should evict 'b'. + map.set('d', 4); + expect(map.get('b')).toBeUndefined(); // evicted + expect(map.get('a')).toBe(10); + expect(map.size).toBe(3); + }); + + it('has() does not refresh recency', () => { + const map = new LruMap(2); + map.set('a', 1); + map.set('b', 2); + + // has('a') must not refresh recency, so 'a' stays the LRU entry + expect(map.has('a')).toBe(true); + map.set('c', 3); + expect(map.has('a')).toBe(false); // evicted + expect(map.has('b')).toBe(true); + expect(map.has('c')).toBe(true); + }); + + it('deletes entries', () => { + const map = new LruMap(5); + map.set('a', 1); + map.set('b', 2); + expect(map.delete('a')).toBe(true); + expect(map.delete('a')).toBe(false); + expect(map.get('a')).toBeUndefined(); + expect(map.get('b')).toBe(2); + expect(map.size).toBe(1); + }); + + it('reuses freed capacity after delete', () => { + const map = new LruMap(2); + map.set(1, 1); + map.set(2, 2); + map.delete(1); + map.set(3, 3); + expect(map.size).toBe(2); + expect(map.get(2)).toBe(2); + expect(map.get(3)).toBe(3); + }); + + it('clears all entries', () => { + const map = new LruMap(5); + map.set(1, 1); + map.set(2, 2); + map.set(3, 3); + map.clear(); + expect(map.size).toBe(0); + expect(map.get(1)).toBeUndefined(); + }); + + it('works correctly after clear and re-add', () => { + const map = new LruMap(2); + map.set('a', 1); + map.set('b', 2); + map.clear(); + map.set('c', 3); + map.set('d', 4); + expect(map.size).toBe(2); + expect(map.get('a')).toBeUndefined(); + expect(map.get('c')).toBe(3); + expect(map.get('d')).toBe(4); + }); + + it('works with maxSize of 1', () => { + const map = new LruMap(1); + map.set(1, 1); + expect(map.get(1)).toBe(1); + map.set(2, 2); + expect(map.get(1)).toBeUndefined(); + expect(map.get(2)).toBe(2); + expect(map.size).toBe(1); + }); + + it('throws on invalid maxSize', () => { + expect(() => new LruMap(0)).toThrow('LruMap maxSize must be at least 1'); + expect(() => new LruMap(-1)).toThrow('LruMap maxSize must be at least 1'); + }); + + it('handles sequential evictions correctly', () => { + const map = new LruMap(3); + // Fill to capacity + for (let i = 0; i < 3; i++) { + map.set(i, i); + } + // Evict each one in FIFO order (no access refreshes) + for (let i = 3; i < 10; i++) { + map.set(i, i); + expect(map.size).toBe(3); + expect(map.get(i - 3)).toBeUndefined(); // oldest was evicted + } + }); +}); diff --git a/yarn-project/foundation/src/collection/lru_map.ts b/yarn-project/foundation/src/collection/lru_map.ts new file mode 100644 index 000000000000..0ae66c6bb57b --- /dev/null +++ b/yarn-project/foundation/src/collection/lru_map.ts @@ -0,0 +1,143 @@ +/** Node in a doubly-linked list used by {@link LruMap}. */ +type LruNode = { + key: K; + value: V; + prev: LruNode | undefined; + next: LruNode | undefined; +}; + +/** + * A bounded key-value map with Least Recently Used (LRU) eviction. + * Both {@link get} and {@link set} count as an access and refresh the entry's + * recency, so entries that are actively used stay in the map longest. + * + * Uses a doubly-linked list for O(1) ordering and a Map for O(1) lookup. + * Head = least recent, tail = most recent. + */ +export class LruMap { + /** Map from key to its linked-list node for O(1) lookup. */ + private readonly map = new Map>(); + private head: LruNode | undefined; + private tail: LruNode | undefined; + + constructor(private readonly maxSize: number) { + if (maxSize < 1) { + throw new Error('LruMap maxSize must be at least 1'); + } + } + + /** Number of entries in the map. */ + get size(): number { + return this.map.size; + } + + /** Returns true if the key is present, without refreshing its recency. */ + has(key: K): boolean { + return this.map.has(key); + } + + /** + * Returns the value for the key, or undefined if absent. + * Refreshes the entry's recency so it becomes the most recently used. + */ + get(key: K): V | undefined { + const node = this.map.get(key); + if (!node) { + return undefined; + } + this.moveToTail(node); + return node.value; + } + + /** + * Stores a value for the key, refreshing its recency. If the key already exists, overwrites the value. + * If the map is at capacity, evicts the least recently used entry. + */ + set(key: K, value: V): void { + const existing = this.map.get(key); + if (existing) { + existing.value = value; + this.moveToTail(existing); + return; + } + + if (this.map.size >= this.maxSize) { + this.evictHead(); + } + + const node: LruNode = { key, value, prev: this.tail, next: undefined }; + if (this.tail) { + this.tail.next = node; + } else { + this.head = node; + } + this.tail = node; + this.map.set(key, node); + } + + /** Removes the entry for the key, returning true if it was present. */ + delete(key: K): boolean { + const node = this.map.get(key); + if (!node) { + return false; + } + this.unlink(node); + this.map.delete(key); + return true; + } + + /** Removes all entries from the map. */ + clear(): void { + this.map.clear(); + this.head = undefined; + this.tail = undefined; + } + + /** Unlinks a node from its current position and relinks it at the tail. */ + private moveToTail(node: LruNode): void { + if (node === this.tail) { + return; + } + this.unlink(node); + + node.prev = this.tail; + node.next = undefined; + if (this.tail) { + this.tail.next = node; + } else { + this.head = node; + } + this.tail = node; + } + + /** Detaches a node from the linked list, fixing up its neighbours and the head/tail pointers. */ + private unlink(node: LruNode): void { + if (node.prev) { + node.prev.next = node.next; + } else { + this.head = node.next; + } + if (node.next) { + node.next.prev = node.prev; + } else { + this.tail = node.prev; + } + node.prev = undefined; + node.next = undefined; + } + + /** Evicts the head (least recently used) node. */ + private evictHead(): void { + const oldHead = this.head; + if (!oldHead) { + return; + } + this.head = oldHead.next; + if (this.head) { + this.head.prev = undefined; + } else { + this.tail = undefined; + } + this.map.delete(oldHead.key); + } +} diff --git a/yarn-project/foundation/src/config/env_var.ts b/yarn-project/foundation/src/config/env_var.ts index c62136c29ca7..99869e33e6b4 100644 --- a/yarn-project/foundation/src/config/env_var.ts +++ b/yarn-project/foundation/src/config/env_var.ts @@ -164,6 +164,7 @@ export type EnvVar = | 'P2P_MIN_TX_POOL_AGE_MS' | 'P2P_MISSING_TX_COLLECTION_DEADLINE_SLOTS' | 'P2P_RPC_PRICE_BUMP_PERCENTAGE' + | 'P2P_TX_VALIDATION_CACHE_SIZE' | 'DEBUG_P2P_INSTRUMENT_MESSAGES' | 'PEER_ID_PRIVATE_KEY' | 'PEER_ID_PRIVATE_KEY_PATH' diff --git a/yarn-project/p2p/src/client/factory.ts b/yarn-project/p2p/src/client/factory.ts index 721ed3129afb..3f271e43744f 100644 --- a/yarn-project/p2p/src/client/factory.ts +++ b/yarn-project/p2p/src/client/factory.ts @@ -24,6 +24,7 @@ import { createTxValidatorForTransactionsEnteringPendingTxPool, getDefaultAllowedSetupFunctions, } from '../msg_validators/index.js'; +import { TxValidationCache } from '../msg_validators/tx_validator/tx_validation_cache.js'; import { DummyP2PService } from '../services/dummy_service.js'; import { LibP2PService } from '../services/index.js'; import { createFileStoreTxSources } from '../services/tx_collection/file_store_tx_source.js'; @@ -137,6 +138,9 @@ export async function createP2PClient( attestationPool: deps.attestationPool ?? new AttestationPool(attestationStore, telemetry), }; + const txValidationCache = + config.txValidationCacheSize > 0 ? new TxValidationCache(config.txValidationCacheSize) : undefined; + const p2pService = await createP2PService( config, archiver, @@ -151,9 +155,15 @@ export async function createP2PClient( packageVersion, logger.createChild('libp2p_service'), telemetry, + txValidationCache, ); - const txValidatorForTxCollection = createTxValidatorForOnDemandReceivedTxs(proofVerifier, config); + const txValidatorForTxCollection = createTxValidatorForOnDemandReceivedTxs( + proofVerifier, + config, + /*bindings=*/ undefined, + txValidationCache, + ); const nodeSources = [ ...createNodeRpcTxSources(config.txCollectionNodeRpcUrls, txValidatorForTxCollection, config), ...(deps.rpcTxProviders ?? []).map( @@ -230,6 +240,7 @@ async function createP2PService( packageVersion: string, logger: Logger, telemetry: TelemetryClient, + txValidationCache?: TxValidationCache, ) { if (!config.p2pEnabled) { logger.verbose('P2P is disabled. Using dummy P2P service.'); @@ -253,6 +264,7 @@ async function createP2PService( blockMinFeesProvider, telemetry, logger: logger.createChild(`libp2p_service`), + txValidationCache, }); return p2pService; diff --git a/yarn-project/p2p/src/config.ts b/yarn-project/p2p/src/config.ts index b8e388b98747..e9206ebbc705 100644 --- a/yarn-project/p2p/src/config.ts +++ b/yarn-project/p2p/src/config.ts @@ -189,6 +189,9 @@ export interface P2PConfig /** The node's seen message ID cache size */ seenMessageCacheSize: number; + /** Maximum number of (validator, tx) pairs to keep in the tx validation LRU cache. */ + txValidationCacheSize: number; + /** True to disable the status handshake on peer connected. */ p2pDisableStatusHandshake?: boolean; @@ -512,6 +515,11 @@ export const p2pConfigMappings: ConfigMappingsType = { description: 'The number of messages to keep in the seen message cache', ...numberConfigHelper(100_000), // 100K }, + txValidationCacheSize: { + env: 'P2P_TX_VALIDATION_CACHE_SIZE', + description: 'Maximum number of items to keep in the tx validation LRU cache.', + ...numberConfigHelper(5_000), + }, p2pDisableStatusHandshake: { env: 'P2P_DISABLE_STATUS_HANDSHAKE', description: 'True to disable the status handshake on peer connected.', diff --git a/yarn-project/p2p/src/msg_validators/tx_validator/cached_tx_validator.test.ts b/yarn-project/p2p/src/msg_validators/tx_validator/cached_tx_validator.test.ts new file mode 100644 index 000000000000..0c0b44bb97df --- /dev/null +++ b/yarn-project/p2p/src/msg_validators/tx_validator/cached_tx_validator.test.ts @@ -0,0 +1,71 @@ +import { mockTx } from '@aztec/stdlib/testing'; +import type { Tx, TxValidationResult, TxValidator } from '@aztec/stdlib/tx'; + +import { jest } from '@jest/globals'; + +import { CachedTxValidator } from './cached_tx_validator.js'; +import type { ITxValidationCache } from './tx_validation_cache.js'; + +describe('CachedTxValidator', () => { + class TestValidator implements TxValidator { + public readonly identifier = Symbol('TestValidator'); + + constructor(private readonly validateImpl: (tx: Tx) => Promise) {} + + public validateTx(tx: Tx): Promise { + return this.validateImpl(tx); + } + } + + class TestTxValidatorCache implements ITxValidationCache { + public readonly getOrValidate: jest.MockedFunction; + + constructor(impl?: ITxValidationCache['getOrValidate']) { + this.getOrValidate = jest.fn(impl ?? ((_s, _h, validate) => validate())); + } + } + + it('returns inner validator unchanged when cache is not provided', () => { + const inner = new TestValidator(() => Promise.resolve({ result: 'valid' })); + + const wrapped = CachedTxValidator.new(inner, undefined); + + expect(wrapped).toBe(inner); + }); + + it('delegates validation to cache.getOrValidate using validator identifier and tx hash', async () => { + const tx = await mockTx(1); + const validate = jest.fn<(tx: Tx) => Promise>().mockResolvedValue({ result: 'valid' }); + const inner = new TestValidator(txArg => validate(txArg)); + const cache = new TestTxValidatorCache(); + + const wrapped = CachedTxValidator.new(inner, cache); + await wrapped.validateTx(tx); + + expect(cache.getOrValidate).toHaveBeenCalledTimes(1); + expect(cache.getOrValidate).toHaveBeenCalledWith(inner.identifier, tx, expect.any(Function)); + expect(validate).toHaveBeenCalledTimes(1); + }); + + it('returns the value produced by cache.getOrValidate', async () => { + const tx = await mockTx(2); + const result: TxValidationResult = { result: 'invalid', reason: ['cache-hit'] }; + const validate = jest.fn<(tx: Tx) => Promise>().mockResolvedValue({ result: 'valid' }); + const inner = new TestValidator(txArg => validate(txArg)); + const cache = new TestTxValidatorCache(() => Promise.resolve(result)); + + const wrapped = CachedTxValidator.new(inner, cache); + + await expect(wrapped.validateTx(tx)).resolves.toEqual(result); + expect(validate).not.toHaveBeenCalled(); + }); + + it('propagates rejections from cache.getOrValidate', async () => { + const tx = await mockTx(3); + const error = new Error('cache failed'); + const cache = new TestTxValidatorCache(() => Promise.reject(error)); + const wrapped = CachedTxValidator.new(new TestValidator(() => Promise.resolve({ result: 'valid' })), cache); + + await expect(wrapped.validateTx(tx)).rejects.toThrow(error.message); + }); +}); diff --git a/yarn-project/p2p/src/msg_validators/tx_validator/cached_tx_validator.ts b/yarn-project/p2p/src/msg_validators/tx_validator/cached_tx_validator.ts new file mode 100644 index 000000000000..226faad79bb3 --- /dev/null +++ b/yarn-project/p2p/src/msg_validators/tx_validator/cached_tx_validator.ts @@ -0,0 +1,31 @@ +import type { Tx, TxValidationResult, TxValidator } from '@aztec/stdlib/tx'; + +import type { ITxValidationCache } from './tx_validation_cache.js'; + +/** Wraps a {@link TxValidator} to cache its results in a shared {@link ITxValidationCache}. */ +export class CachedTxValidator implements TxValidator { + constructor( + private readonly inner: TxValidator, + private readonly validatorSymbol: symbol, + private readonly cache: ITxValidationCache, + ) {} + + public static new( + inner: TxValidator & { identifier: symbol }, + cache?: ITxValidationCache, + ): TxValidator { + return CachedTxValidator.newWithIdentifier(inner, inner.identifier, cache); + } + + public static newWithIdentifier( + inner: TxValidator, + identifier: symbol, + cache?: ITxValidationCache, + ): TxValidator { + return cache ? new CachedTxValidator(inner, identifier, cache) : inner; + } + + public validateTx(tx: T): Promise { + return this.cache.getOrValidate(this.validatorSymbol, tx, () => this.inner.validateTx(tx)); + } +} diff --git a/yarn-project/p2p/src/msg_validators/tx_validator/data_validator.ts b/yarn-project/p2p/src/msg_validators/tx_validator/data_validator.ts index 7c284b6d0ce3..d1dcd7c75ead 100644 --- a/yarn-project/p2p/src/msg_validators/tx_validator/data_validator.ts +++ b/yarn-project/p2p/src/msg_validators/tx_validator/data_validator.ts @@ -20,6 +20,8 @@ import { } from '@aztec/stdlib/tx'; export class DataTxValidator implements TxValidator { + public readonly identifier: symbol = Symbol('DataTxValidator'); + #log: Logger; constructor(bindings?: LoggerBindings) { diff --git a/yarn-project/p2p/src/msg_validators/tx_validator/factory.ts b/yarn-project/p2p/src/msg_validators/tx_validator/factory.ts index 33ba7a6ffb59..e15078e1bb6d 100644 --- a/yarn-project/p2p/src/msg_validators/tx_validator/factory.ts +++ b/yarn-project/p2p/src/msg_validators/tx_validator/factory.ts @@ -53,6 +53,7 @@ import type { TxMetaData } from '../../mem_pools/tx_pool_v2/tx_metadata.js'; import { AggregateTxValidator } from './aggregate_tx_validator.js'; import { ArchiveCache } from './archive_cache.js'; import { type ArchiveSource, BlockHeaderTxValidator } from './block_header_validator.js'; +import { CachedTxValidator } from './cached_tx_validator.js'; import { ContractInstanceTxValidator } from './contract_instance_validator.js'; import { DataTxValidator } from './data_validator.js'; import { DoubleSpendTxValidator, type NullifierSource } from './double_spend_validator.js'; @@ -64,6 +65,7 @@ import { SizeTxValidator } from './size_validator.js'; import { TimestampTxValidator } from './timestamp_validator.js'; import { TxPermittedValidator } from './tx_permitted_validator.js'; import { TxProofValidator } from './tx_proof_validator.js'; +import { TxValidationCache } from './tx_validation_cache.js'; /** * A validator paired with a peer penalty severity. @@ -210,8 +212,9 @@ function createTxValidatorForMinimumTxIntegrityChecks( rollupVersion: number; }, bindings?: LoggerBindings, + cache?: TxValidationCache, ): TxValidator { - return new AggregateTxValidator( + const aggregate = new AggregateTxValidator( new MetadataTxValidator( { l1ChainId: new Fr(l1ChainId), @@ -222,10 +225,14 @@ function createTxValidatorForMinimumTxIntegrityChecks( bindings, ), new SizeTxValidator(bindings), - new DataTxValidator(bindings), + CachedTxValidator.new(new DataTxValidator(bindings), cache), new ContractInstanceTxValidator(bindings), - new TxProofValidator(verifier, bindings), + CachedTxValidator.new(new TxProofValidator(verifier, bindings), cache), ); + + // This validator is not state-dependent so we can cache it. + const identifier = Symbol('TxValidatorForMinimumTxIntegrityChecks'); + return CachedTxValidator.newWithIdentifier(aggregate, identifier, cache); } /** @@ -244,8 +251,9 @@ export function createTxValidatorForOnDemandReceivedTxs( rollupVersion: number; }, bindings?: LoggerBindings, + cache?: TxValidationCache, ): TxValidator { - return createTxValidatorForMinimumTxIntegrityChecks(verifier, { l1ChainId, rollupVersion }, bindings); + return createTxValidatorForMinimumTxIntegrityChecks(verifier, { l1ChainId, rollupVersion }, bindings, cache); } /** @@ -263,8 +271,9 @@ export function createTxValidatorForBlockProposalReceivedTxs( rollupVersion: number; }, bindings?: LoggerBindings, + cache?: TxValidationCache, ): TxValidator { - return createTxValidatorForMinimumTxIntegrityChecks(verifier, { l1ChainId, rollupVersion }, bindings); + return createTxValidatorForMinimumTxIntegrityChecks(verifier, { l1ChainId, rollupVersion }, bindings, cache); } /** diff --git a/yarn-project/p2p/src/msg_validators/tx_validator/index.ts b/yarn-project/p2p/src/msg_validators/tx_validator/index.ts index 893796772a3b..4a21303b725e 100644 --- a/yarn-project/p2p/src/msg_validators/tx_validator/index.ts +++ b/yarn-project/p2p/src/msg_validators/tx_validator/index.ts @@ -14,3 +14,5 @@ export * from './tx_permitted_validator.js'; export * from './timestamp_validator.js'; export * from './size_validator.js'; export * from './factory.js'; +export * from './tx_validation_cache.js'; +export * from './cached_tx_validator.js'; diff --git a/yarn-project/p2p/src/msg_validators/tx_validator/tx_proof_validator.ts b/yarn-project/p2p/src/msg_validators/tx_validator/tx_proof_validator.ts index eae4cbf33c3c..d461097a738e 100644 --- a/yarn-project/p2p/src/msg_validators/tx_validator/tx_proof_validator.ts +++ b/yarn-project/p2p/src/msg_validators/tx_validator/tx_proof_validator.ts @@ -3,6 +3,8 @@ import type { ClientProtocolCircuitVerifier } from '@aztec/stdlib/interfaces/ser import { TX_ERROR_INVALID_PROOF, Tx, type TxValidationResult, type TxValidator } from '@aztec/stdlib/tx'; export class TxProofValidator implements TxValidator { + public readonly identifier: symbol = Symbol('TxProofValidator'); + #log: Logger; constructor( diff --git a/yarn-project/p2p/src/msg_validators/tx_validator/tx_validation_cache.test.ts b/yarn-project/p2p/src/msg_validators/tx_validator/tx_validation_cache.test.ts new file mode 100644 index 000000000000..4402ba4abaa9 --- /dev/null +++ b/yarn-project/p2p/src/msg_validators/tx_validator/tx_validation_cache.test.ts @@ -0,0 +1,190 @@ +import { sleep } from '@aztec/foundation/sleep'; +import { mockTx } from '@aztec/stdlib/testing'; +import type { Tx, TxValidationResult } from '@aztec/stdlib/tx'; + +import { jest } from '@jest/globals'; + +import { TxValidationCache } from './tx_validation_cache.js'; + +describe('TxValidationCache', () => { + const validatorA = Symbol('validatorA'); + const validatorB = Symbol('validatorB'); + + let cache: TxValidationCache; + let tx: Tx; + + beforeEach(async () => { + cache = new TxValidationCache(100); + tx = await mockTx(1); + }); + + // The cache key is derived asynchronously, so a call's promise is only registered after its key + // resolves. This waits until a call has been registered before issuing a follow-up, making the + // coalescing assertions deterministic rather than dependent on the order in which key digests resolve. + const waitUntilCached = async (validatorSymbol: symbol, forTx: Tx) => { + const key = await cache.key(validatorSymbol, forTx); + while (cache.get(key) === undefined) { + await sleep(1); + } + }; + + describe('get / set', () => { + it('returns undefined on a cache miss', async () => { + expect(cache.get(await cache.key(validatorA, tx))).toBeUndefined(); + }); + + it('returns the stored promise on a cache hit', async () => { + const result: TxValidationResult = { result: 'valid' }; + cache.set(await cache.key(validatorA, tx), Promise.resolve(result)); + + await expect(cache.get(await cache.key(validatorA, tx))).resolves.toEqual(result); + }); + + it('does not share entries across different validator symbols', async () => { + cache.set(await cache.key(validatorA, tx), Promise.resolve({ result: 'valid' })); + + expect(cache.get(await cache.key(validatorB, tx))).toBeUndefined(); + }); + + it('does not share entries across different txs', async () => { + const otherTx = await mockTx(2); + cache.set(await cache.key(validatorA, tx), Promise.resolve({ result: 'valid' })); + + expect(cache.get(await cache.key(validatorA, otherTx))).toBeUndefined(); + }); + }); + + describe('LRU eviction', () => { + it('evicts the least-recently-used entry when the cache is full', async () => { + const smallCache = new TxValidationCache(2); + const tx1 = await mockTx(10); + const tx2 = await mockTx(11); + const tx3 = await mockTx(12); + const result: TxValidationResult = { result: 'valid' }; + + smallCache.set(await smallCache.key(validatorA, tx1), Promise.resolve(result)); + smallCache.set(await smallCache.key(validatorA, tx2), Promise.resolve(result)); + // tx1 is now the LRU entry; adding tx3 should evict it + smallCache.set(await smallCache.key(validatorA, tx3), Promise.resolve(result)); + + expect(smallCache.get(await smallCache.key(validatorA, tx1))).toBeUndefined(); + expect(smallCache.get(await smallCache.key(validatorA, tx2))).toBeDefined(); + expect(smallCache.get(await smallCache.key(validatorA, tx3))).toBeDefined(); + }); + + it('refreshes recency on get so that accessed entries are not evicted first', async () => { + const smallCache = new TxValidationCache(2); + const tx1 = await mockTx(20); + const tx2 = await mockTx(21); + const tx3 = await mockTx(22); + const result: TxValidationResult = { result: 'valid' }; + + smallCache.set(await smallCache.key(validatorA, tx1), Promise.resolve(result)); + smallCache.set(await smallCache.key(validatorA, tx2), Promise.resolve(result)); + // Access tx1 so tx2 becomes the LRU entry + void smallCache.get(await smallCache.key(validatorA, tx1)); + smallCache.set(await smallCache.key(validatorA, tx3), Promise.resolve(result)); + + expect(smallCache.get(await smallCache.key(validatorA, tx1))).toBeDefined(); + expect(smallCache.get(await smallCache.key(validatorA, tx2))).toBeUndefined(); + expect(smallCache.get(await smallCache.key(validatorA, tx3))).toBeDefined(); + }); + + it('throws when constructed with maxSize < 1', () => { + expect(() => new TxValidationCache(0)).toThrow(); + }); + }); + + describe('getOrValidate', () => { + it('calls validate and caches the result on a miss', async () => { + const expected: TxValidationResult = { result: 'invalid', reason: ['bad'] }; + const validate = jest.fn<() => Promise>().mockResolvedValue(expected); + + await expect(cache.getOrValidate(validatorA, tx, validate)).resolves.toEqual(expected); + expect(validate).toHaveBeenCalledTimes(1); + }); + + it('returns the cached promise on a hit without calling validate again', async () => { + const expected: TxValidationResult = { result: 'valid' }; + const validate = jest.fn<() => Promise>().mockResolvedValue(expected); + + await cache.getOrValidate(validatorA, tx, validate); + await expect(cache.getOrValidate(validatorA, tx, validate)).resolves.toEqual(expected); + expect(validate).toHaveBeenCalledTimes(1); + }); + + it('shares an in-flight validation so concurrent calls for the same key validate once', async () => { + const expected: TxValidationResult = { result: 'invalid', reason: ['bad proof'] }; + + let resolveValidation!: (v: TxValidationResult) => void; + const inFlight = new Promise(resolve => { + resolveValidation = resolve; + }); + const validate = jest.fn<() => Promise>().mockReturnValue(inFlight); + + const first = cache.getOrValidate(validatorA, tx, validate); + await waitUntilCached(validatorA, tx); + const second = cache.getOrValidate(validatorA, tx, validate); + const third = cache.getOrValidate(validatorA, tx, validate); + + resolveValidation(expected); + + await expect(first).resolves.toEqual(expected); + await expect(second).resolves.toEqual(expected); + await expect(third).resolves.toEqual(expected); + + expect(validate).toHaveBeenCalledTimes(1); + }); + + it('scopes validation results by validator symbol', async () => { + const resultA: TxValidationResult = { result: 'valid' }; + const resultB: TxValidationResult = { result: 'invalid', reason: ['nope'] }; + + const validateA = jest.fn<() => Promise>().mockResolvedValue(resultA); + const validateB = jest.fn<() => Promise>().mockResolvedValue(resultB); + + await expect(cache.getOrValidate(validatorA, tx, validateA)).resolves.toEqual(resultA); + await expect(cache.getOrValidate(validatorB, tx, validateB)).resolves.toEqual(resultB); + + expect(validateA).toHaveBeenCalledTimes(1); + expect(validateB).toHaveBeenCalledTimes(1); + }); + + it('caches a rejected validation so a later call reuses the failure without retrying', async () => { + const error = new Error('temporary failure'); + const success: TxValidationResult = { result: 'valid' }; + const validate = jest + .fn<() => Promise>() + .mockRejectedValueOnce(error) + .mockResolvedValueOnce(success); + + await expect(cache.getOrValidate(validatorA, tx, validate)).rejects.toThrow(error.message); + await expect(cache.getOrValidate(validatorA, tx, validate)).rejects.toThrow(error.message); + expect(validate).toHaveBeenCalledTimes(1); + }); + + it('caches a rejected in-flight validation so a later call reuses the failure', async () => { + const error = new Error('downstream unavailable'); + const success: TxValidationResult = { result: 'invalid', reason: ['bad tx'] }; + + let rejectValidation!: (err: Error) => void; + const firstInFlight = new Promise((_, reject) => { + rejectValidation = reject; + }); + + const validate = jest + .fn<() => Promise>() + .mockReturnValueOnce(firstInFlight) + .mockResolvedValueOnce(success); + + const first = cache.getOrValidate(validatorA, tx, validate); + await waitUntilCached(validatorA, tx); + + rejectValidation(error); + await expect(first).rejects.toThrow(error.message); + + await expect(cache.getOrValidate(validatorA, tx, validate)).rejects.toThrow(error.message); + expect(validate).toHaveBeenCalledTimes(1); + }); + }); +}); diff --git a/yarn-project/p2p/src/msg_validators/tx_validator/tx_validation_cache.ts b/yarn-project/p2p/src/msg_validators/tx_validator/tx_validation_cache.ts new file mode 100644 index 000000000000..f4d89590689d --- /dev/null +++ b/yarn-project/p2p/src/msg_validators/tx_validator/tx_validation_cache.ts @@ -0,0 +1,102 @@ +import { LruMap } from '@aztec/foundation/collection'; +import { type Logger, createLogger } from '@aztec/foundation/log'; +import type { Tx, TxValidationResult } from '@aztec/stdlib/tx'; + +import { webcrypto } from 'node:crypto'; + +/** + * Minimal interface consumed by {@link CachedTxValidator}. + * Keeping the dependency on an interface lets callers (and tests) substitute any cache implementation. + */ +export interface ITxValidationCache { + /** Returns the cached promise if present, otherwise calls `validate`, caches its promise, and returns it. */ + getOrValidate( + validatorSymbol: symbol, + tx: Tx, + validate: () => Promise, + ): Promise; +} + +/** + * Caches per-validator tx validation results to avoid redundant work across repeated validation calls. + * + * The cache key is composed from the validator symbol and tx hash, ensuring results are + * scoped to the specific validator that produced them. + * + * Promises are stored before they are awaited, so concurrent calls for the same pair share + * a single in-flight validation rather than launching duplicate work. + * + * Entries are evicted in least-recently-used order once the cache reaches `maxSize`. + */ +export class TxValidationCache implements ITxValidationCache { + #log: Logger; + + private readonly entries: LruMap>; + // Remember hashes for known Tx object references to skip rehashing on subsequent lookups. + // WeakMap holds keys weakly, so an entry doesn't keep the Tx alive once nothing else references it. + private readonly txHashesCache: WeakMap = new WeakMap(); + + constructor(maxSize: number) { + this.entries = new LruMap(maxSize); + this.#log = createLogger('p2p:tx_validation_cache'); + } + + /** + * Computes the cache key scoping a validation result to a specific validator and tx. + * + * @param validatorSymbol - The symbol of the validator. + * @param tx - The tx to compute the key for. + * @returns The cache key. + * + * Note: the key should NOT use the tx.hash because it can't be trusted at this point. + */ + public async key(validatorSymbol: symbol, tx: Tx): Promise { + // Hashing the whole tx takes a few milliseconds. So if we have already hashed + // this particular object, we avoid rehashing it. + let hash = this.txHashesCache.get(tx); + if (hash === undefined) { + hash = Buffer.from(await webcrypto.subtle.digest('SHA-256', tx.toBuffer())).toString('hex'); + this.txHashesCache.set(tx, hash); + } + return `${Symbol.keyFor(validatorSymbol) ?? validatorSymbol.toString()}:${hash}`; + } + + /** Returns the cached promise for the given key, or undefined if not cached. Refreshes recency. */ + public get(key: string): Promise | undefined { + return this.entries.get(key); + } + + /** Stores a validation promise under the given key, evicting the LRU entry if at capacity. */ + public set(key: string, result: Promise): void { + this.entries.set(key, result); + } + + /** Removes the cached validation promise for the given key. */ + public delete(key: string): void { + this.entries.delete(key); + } + + /** + * Returns the cached promise if present, otherwise calls `validate`, stores its promise + * immediately (before awaiting), and returns it. + */ + public async getOrValidate( + validatorSymbol: symbol, + tx: Tx, + validate: () => Promise, + ): Promise { + const key = await this.key(validatorSymbol, tx); + const cached = this.get(key); + if (cached !== undefined) { + this.#log.debug('Returning cached tx validation result', { + validator: validatorSymbol.toString(), + txHash: tx.txHash.toString(), + key: key, + }); + return cached; + } + const promise = validate(); + this.set(key, promise); + return promise; + } +} diff --git a/yarn-project/p2p/src/services/libp2p/libp2p_service.ts b/yarn-project/p2p/src/services/libp2p/libp2p_service.ts index 9c00dc517ba6..2da0cdecebea 100644 --- a/yarn-project/p2p/src/services/libp2p/libp2p_service.ts +++ b/yarn-project/p2p/src/services/libp2p/libp2p_service.ts @@ -75,6 +75,7 @@ import { createSecondStageTxValidationsForGossipedTransactions, createTxValidatorForBlockProposalReceivedTxs, } from '../../msg_validators/tx_validator/factory.js'; +import { TxValidationCache } from '../../msg_validators/tx_validator/tx_validation_cache.js'; import { GossipSubEvent } from '../../types/index.js'; import { type PubSubLibp2p, convertToMultiaddr } from '../../util.js'; import { getVersions } from '../../versioning.js'; @@ -202,6 +203,7 @@ export class LibP2PService extends WithTracer implements P2PService { private blockMinFeesProvider: BlockMinFeesProvider, telemetry: TelemetryClient, logger: Logger = createLogger('p2p:libp2p_service'), + private txValidationCache?: TxValidationCache, ) { super(telemetry, 'LibP2PService'); this.telemetry = telemetry; @@ -302,6 +304,7 @@ export class LibP2PService extends WithTracer implements P2PService { telemetry: TelemetryClient; logger: Logger; packageVersion: string; + txValidationCache?: TxValidationCache; }, ) { const { @@ -315,6 +318,7 @@ export class LibP2PService extends WithTracer implements P2PService { telemetry, logger, packageVersion, + txValidationCache, } = deps; const { p2pPort, maxPeerCount, listenAddress } = config; const bindAddrTcp = convertToMultiaddr(listenAddress, p2pPort, 'tcp'); @@ -522,6 +526,7 @@ export class LibP2PService extends WithTracer implements P2PService { blockMinFeesProvider, telemetry, logger, + txValidationCache, ); } @@ -1608,6 +1613,7 @@ export class LibP2PService extends WithTracer implements P2PService { l1ChainId: this.config.l1ChainId, rollupVersion: this.config.rollupVersion, proofVerifier: this.proofVerifier, + txValidationCache: this.txValidationCache, }, peerScoring: this.peerManager, validateRequestedBlockTxsConsistency: this.validateRequestedBlockTxsConsistency.bind(this), @@ -1619,6 +1625,7 @@ export class LibP2PService extends WithTracer implements P2PService { this.proofVerifier, { l1ChainId: this.config.l1ChainId, rollupVersion: this.config.rollupVersion }, this.logger.getBindings(), + this.txValidationCache, ); const results = await Promise.all( diff --git a/yarn-project/p2p/src/services/reqresp/batch-tx-requester/tx_validator.ts b/yarn-project/p2p/src/services/reqresp/batch-tx-requester/tx_validator.ts index dd8015a76a8a..e603482e051d 100644 --- a/yarn-project/p2p/src/services/reqresp/batch-tx-requester/tx_validator.ts +++ b/yarn-project/p2p/src/services/reqresp/batch-tx-requester/tx_validator.ts @@ -2,16 +2,23 @@ import type { ClientProtocolCircuitVerifier } from '@aztec/stdlib/interfaces/ser import type { TxValidator } from '@aztec/stdlib/tx'; import { createTxValidatorForOnDemandReceivedTxs } from '../../../msg_validators/index.js'; +import type { TxValidationCache } from '../../../msg_validators/tx_validator/tx_validation_cache.js'; export interface BatchRequestTxValidatorConfig { l1ChainId: number; rollupVersion: number; proofVerifier: ClientProtocolCircuitVerifier; + txValidationCache?: TxValidationCache; } export function createBatchRequestTxValidator(config: BatchRequestTxValidatorConfig): TxValidator { - return createTxValidatorForOnDemandReceivedTxs(config.proofVerifier, { - l1ChainId: config.l1ChainId, - rollupVersion: config.rollupVersion, - }); + return createTxValidatorForOnDemandReceivedTxs( + config.proofVerifier, + { + l1ChainId: config.l1ChainId, + rollupVersion: config.rollupVersion, + }, + /*bindings=*/ undefined, + config.txValidationCache, + ); } diff --git a/yarn-project/stdlib/src/tx/tx_bench.test.ts b/yarn-project/stdlib/src/tx/tx_bench.test.ts index 9f5f02840376..d8d0133809a3 100644 --- a/yarn-project/stdlib/src/tx/tx_bench.test.ts +++ b/yarn-project/stdlib/src/tx/tx_bench.test.ts @@ -1,5 +1,6 @@ import { Timer } from '@aztec/foundation/timer'; +import { webcrypto } from 'node:crypto'; import fs from 'node:fs/promises'; import path from 'node:path'; import { type RecordableHistogram, createHistogram } from 'perf_hooks'; @@ -11,46 +12,29 @@ const RUNS = 100; describe('Tx', () => { let privateTxHistogram: RecordableHistogram; let publicTxHistogram: RecordableHistogram; + let privateSha256Histogram: RecordableHistogram; + let publicSha256Histogram: RecordableHistogram; beforeAll(() => { privateTxHistogram = createHistogram(); publicTxHistogram = createHistogram(); + privateSha256Histogram = createHistogram(); + publicSha256Histogram = createHistogram(); }); afterAll(async () => { if (process.env.BENCH_OUTPUT) { const data: any[] = []; - data.push({ - name: `Tx/private/getTxHash/avg`, - value: privateTxHistogram.mean, - unit: 'ms', - }); - data.push({ - name: `Tx/private/getTxHash/p50`, - value: privateTxHistogram.percentile(50), - unit: 'ms', - }); - data.push({ - name: `Tx/private/getTxHash/p95`, - value: privateTxHistogram.percentile(95), - unit: 'ms', - }); + const recordHistogram = (name: string, histogram: RecordableHistogram) => { + data.push({ name: `${name}/avg`, value: histogram.mean, unit: 'ms' }); + data.push({ name: `${name}/p50`, value: histogram.percentile(50), unit: 'ms' }); + data.push({ name: `${name}/p95`, value: histogram.percentile(95), unit: 'ms' }); + }; - data.push({ - name: `Tx/public/getTxHash/avg`, - value: publicTxHistogram.mean, - unit: 'ms', - }); - data.push({ - name: `Tx/public/getTxHash/p50`, - value: publicTxHistogram.percentile(50), - unit: 'ms', - }); - data.push({ - name: `Tx/public/getTxHash/p95`, - value: publicTxHistogram.percentile(95), - unit: 'ms', - }); + recordHistogram('Tx/private/getTxHash', privateTxHistogram); + recordHistogram('Tx/public/getTxHash', publicTxHistogram); + recordHistogram('Tx/private/sha256', privateSha256Histogram); + recordHistogram('Tx/public/sha256', publicSha256Histogram); await fs.mkdir(path.dirname(process.env.BENCH_OUTPUT), { recursive: true }); await fs.writeFile(process.env.BENCH_OUTPUT, JSON.stringify(data, null, 2)); @@ -59,12 +43,15 @@ describe('Tx', () => { await using f = await fs.open(process.env.BENCH_OUTPUT_MD!, 'w'); await f.write('|TYPE|MIN|AVG|P50|P90|MAX|\n'); await f.write('|----|---|---|---|---|---|\n'); - await f.write( - `|PRV|${privateTxHistogram.min}|${privateTxHistogram.mean}|${privateTxHistogram.percentile(50)}|${privateTxHistogram.percentile(90)}|${privateTxHistogram.max}|\n`, - ); - await f.write( - `|PUB|${publicTxHistogram.min}|${publicTxHistogram.mean}|${publicTxHistogram.percentile(50)}|${publicTxHistogram.percentile(90)}|${publicTxHistogram.max}|\n`, - ); + const writeRow = async (type: string, histogram: RecordableHistogram) => { + await f.write( + `|${type}|${histogram.min}|${histogram.mean}|${histogram.percentile(50)}|${histogram.percentile(90)}|${histogram.max}|\n`, + ); + }; + await writeRow('PRV', privateTxHistogram); + await writeRow('PUB', publicTxHistogram); + await writeRow('PRV-SHA256', privateSha256Histogram); + await writeRow('PUB-SHA256', publicSha256Histogram); } }); @@ -85,4 +72,22 @@ describe('Tx', () => { publicTxHistogram.record(Math.max(1, Math.ceil(timer.ms()))); } }); + + it('calculates SHA-256 of a private-only tx buffer', async () => { + const tx = await mockTxForRollup(42); + for (let i = 0; i < RUNS; i++) { + const timer = new Timer(); + await webcrypto.subtle.digest('SHA-256', tx.toBuffer()); + privateSha256Histogram.record(Math.max(1, Math.ceil(timer.ms()))); + } + }); + + it('calculates SHA-256 of a tx buffer with enqueued public calls', async () => { + const tx = await mockTx(42); + for (let i = 0; i < RUNS; i++) { + const timer = new Timer(); + await webcrypto.subtle.digest('SHA-256', tx.toBuffer()); + publicSha256Histogram.record(Math.max(1, Math.ceil(timer.ms()))); + } + }); }); From e6760405753a627140996d4259c89de30b957eab Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 28 May 2026 11:56:13 +0100 Subject: [PATCH 23/27] chore: add KEDA deployment module (#23553) Stack: 1. #23553 (this): Add KEDA deployment module 2. #23554: Add KEDA prover agent autoscaling Adds a Terraform module for deploying KEDA to Spartan clusters. --- spartan/terraform/deploy-keda/keda-2.19.0.tgz | Bin 0 -> 113079 bytes spartan/terraform/deploy-keda/main.tf | 39 ++++++++++++++++++ .../terraform/deploy-keda/private.tfbackend | 2 + spartan/terraform/deploy-keda/private.tfvars | 1 + .../terraform/deploy-keda/public.tfbackend | 2 + spartan/terraform/deploy-keda/public.tfvars | 1 + spartan/terraform/deploy-keda/update-chart.sh | 14 +++++++ spartan/terraform/deploy-keda/variables.tf | 18 ++++++++ 8 files changed, 77 insertions(+) create mode 100644 spartan/terraform/deploy-keda/keda-2.19.0.tgz create mode 100644 spartan/terraform/deploy-keda/main.tf create mode 100644 spartan/terraform/deploy-keda/private.tfbackend create mode 100644 spartan/terraform/deploy-keda/private.tfvars create mode 100644 spartan/terraform/deploy-keda/public.tfbackend create mode 100644 spartan/terraform/deploy-keda/public.tfvars create mode 100755 spartan/terraform/deploy-keda/update-chart.sh create mode 100644 spartan/terraform/deploy-keda/variables.tf diff --git a/spartan/terraform/deploy-keda/keda-2.19.0.tgz b/spartan/terraform/deploy-keda/keda-2.19.0.tgz new file mode 100644 index 0000000000000000000000000000000000000000..419c21ab4168b5f9b66a8c75ee1ce61f6431cf4e GIT binary patch literal 113079 zcmV)WK(4Dc zVQyr3R8em|NM&qo0POwicH=gdFo5=NJq6vii%H zZbKv_AtnJ10LrDL)9bv(dA;)_XJO;Qi%5x-WS1);epFc`v9WR6*mvBL5FeaQu~5S~ zj-x-^M>#w^Jbd~5IsAWkcv$}b@YRcFe>i$^^!)Jf)yrqkU;W|m=;_hX(I3#^yi?2Qi(i1IG9j1O-I9k#|L`p1zZsL zqelY`-+dpW4nL(^XzE&^x5!maJbvkAQYyI2zlHaAQEF59iyMPn2>*}R0tgM&!$}I$3r4} z2r;ZX^p*YqU7HWGdOwfmjNE>9OF}xH4=MNG|6h24E_ph_5xvc|gjvPsUtxy6=RY%w zZ+qqmjFRN32nZm$aztmPiL9J|7{wWmQX=n-04&}AM@P?(4j1bgT`uoY-H-j-3F;?2w&InTyYPk2uh^0}~vmjo-yFvkHV>*GJELIGeJcaki=$Buimfl{Hh#3$7 z{Q#i>IvPG5J{vxV-;!VvOiAu7lSe~23x+|IN=3vVAgxMKV+Zq3~n@!VrA}1A?PIn$Q_x2qy`$^hD`k)Qro( z;yJ(xmD%?-Bee*9t7CRYMpMpjc*<3Ld{?;evP&|TDfME zG>R^GM1why8yrOyBq#gm7@b6ScrJTAfRS-Y$Wjs*ry=^*P){@_SfzqUB-3DuutX%L zN>MgJrx8y>;KpRn)_rH#1P^oIu&-^*kS$mLHT*huZ-)^NaMbGsBD91`lUFjPj7W4h zC2C3pQd43&e0up>qA8vcdje1<6^-g@G(0 znUH{vX@H~1jH`y;LkNc<$*OqD7O z%^Nd~?nE%ZkO6iEur4(y7=sqA-sYx#vE}S-tJeJ)lRsdVd|jljH6Ql!jTcss-($H0 zHxmg4xu;O=k|Yrg@D)*J2$W{xUXvu^^O!KD`7v~d87xUbwB&>K>>@u#M~8Zz1({Jz z7{8=aQ{y)@rtr>hS)AYzji{PyAcoya1u<(WjYkAlmN1COj1lwaJplTCN+Sa31ehU_ zG8A(`vTA&cH0$Up^$};$9C1ecBSTSGK+qkHB29KQ1`)v_5eTyo>4Q~MOC|7lOas&I z-}5b|k)lbIwc1CcRB14C1Cw!uXq*T>Gr$uffDP7-+ViH5s6qixqtIX|NSFo$h5U{w z8WR+zLgRqZ7}JRBqZ`n8L$I)2rVRa{yBhK!)pG=d?Z-VQ;Q>1?rw1Vqc9*E zIk3d~;K`IurUN{~G_q5(KZKctZk_U!=^3;W_?k)}7lO!yGfCjM(#ugFhOliumHMU9 z>;`2hh^Q9(C){9es_Q=G>q#0X(HzADGl|p`E2L7v^fa0!oCqH1&B@0o#<#k5h!sXM z*2jcVA$(oaEW8|JIJ1<%Oxp8ZqZ69GoMxEV^JR%Z;MY&W!%C4r6O~glvY%5S)%R3Q zzvNQABMQ+mx=AImKQ&1)4c>T(JR>VZ%FV0=oCPFN@&Hz3K;*!ZBg!TRA(MlIhXecW z;0gQ~n8%vT7f;_1hEw+|$v6-FC`SoGCLPC-??>sl@Z(_5p(&Tj(a9b{k}!pZvAE^YHNGVhC4vMbB(QyEmVvmz7Ss(IQxfSZaZir3gFWHl z2~*TH>sVF2#{V~^f`qSOpIvz*&L@of$7kgER(fUR>*LbX!2%I?Zxv5?#3%EwbxS+NHKiv?9|743 z>^UR7*sVGXD16E^-g)Q$>xmaTrzDE_AjWKNUtlqj`QaP{ zxgR@sK;HGv_-_<)M1m~5*oZ?q_*iE1MF z$g(`XV|zbn(vv$z7Ri#r@)n4bIlzGm`H{;#k8C>wL2yXnFR8FlX)-V^AM9(s!F_h2 zpP4@C$)GrTO(HzMA^~TiJVsAnz`KM9%0vGY_zQ&enaNX?J%*?H>`Vo@o(dwTJPMD| zGk7l3K%XkP=SLYMWO4oQqw^#L%ANbbyx2F~mVLN#*bb|P?b((_t+&9`K6ZA}AdHOm zq!mwDsF`Cdi~J;L0UR4iIwtaW%0zyg%wS=OujUA=*knr(ER0%aIpZe_ZQc)4S)~Z= zILii{Wv#xsHLqF4bSzHGBW6E79QW~|hiN-}Nagr#jSb0b%buWfg>WRfIg1sy>P)2h zc60=||Lgj4& z?LeX_!I7Gp_0dE2Ck*}t0RHsw)1zhdxdP-88eO$*DdA%6d2Re>AUk)jK_kmy%N=& z*1ggq`Pc=f%1%{t_f7SCRg|*h$CmJ8wWjRme3+l2&&~FTuJsAfWw(bC@ zYRcf>pAX1;MVO?ch~&`IluB?dR>^}WmJnDGWZ&&Y*k$!g%hqyohnc2cFHI%_hvZ6W z;d7E1G3%D~54qiA>mq3@|5(uK^z!rK*OQtj2i#8Z5_XzL!79>UN#RJ z0a?A35*}vb=YiRQZ`y(K=WTH>*I$ixPmFNjQNA)RJ~>yS%fFqRA~^*W=!k&8*Fz`} zL=%L9x7qN{51uhJ3qSS3;MHVC!K4P<|uDJm2O!Ktdg%_J+mIeMvV<(`-mVM zOevWee>z~VbAD)s#Xw1f89^~-ctQjkM|gq;gH#fy1RdZY(Cq0T5SpXZl8s&btA(&^ zYOSqp5NSzDaUN-b%e~Y!b1cumB6Fap&OW}-SY8>7P7Fda=lKN?Vu`shyw zwrkEsCtH$F5aW7pXY7Rp2v-U#GEO59p|Busq_lLGdIRo=zwaUAwU@2rp60V1PRD5H zPsY%COIhfCJt=&F#e^q<|72WNpZISzkHS;c>&e1|ANR^$Vl@;ABU=&KwOhF<7wDT4 zqNkb1tnOoht0{D>+b%o>iBN=9k*+)ITnpHc8wv848|C|c)z4a~554mv=)=lSh z+$Z{Lp>kseiV?RCO!rO#MP~%PrUFzmb94&!S~v%;3BydG^U$eC z1ks0`t`NdyzG0pNH1) zkoBUb3w9J&bOEIJhwO#|v>%<_fRT%X`4N> zt;%DxqmR`_^V5>fYz^;)ZT#uW=dh0_L9&ud_~P`Uf@H7a4xSE=4ok#l;MScym;}ip z%{f@gA++gnC7Uo9Kv(q6DA6Bu7GoNryD1H(#(D-!UPNz6{#0`a`;BB8n*_=E$=f2U zmkZN^K?NJ)7+`C0GnL?Kr>9;Kp%<}DbD)_LXS&lXK4p6BNbiZUa9zK-LVF_H3;RPe zSS~smO?ebbuU5&Horw^eV%2-%M;16#;@QoEiAaKg2qjU1)zl601rK}r&GnnBQ~lC} ziZ$xrmsn2eDHjR)4wP3PvU4-Qpxm*k!w?X>esg8|74gZ0vPq37{>HpT5t)%F(@%br z3q8_b;wa*GBy_eqbQ6+MI=Shi8_LF9|4)SA;-;@pBdj?9MJ7a8f-wO}N1ybVp~Jol zSR|p|3{nZoCeRKbaf~#`MGQ?ieG3$SlAPUWd~O2HB#+3AG13E^V`LvLHUti&gate- zm~NN12P@@$|@0h7L2S3UtPmTXJv9yqd!<&8BXc#T(Mr0L=e5` z!~NPxIJ|+XdP@7~=DGO=i)SlKd98oHJ-hs3*A8)^-D3pbX88SIoU9zvhuv;J}^T`@p&iH^jAO-~Eh}GXtz*@98F^+Wpt0;q7g}pK)?svj;2mKOZjvU9>^{K1iuK z>3_H)K`OKW0qK(7s~>$zf-D%S_XK_BtW~BCK_e3JyC3(S=$;Iy97qj^3`TR8Kw!To zE$CAN*<0}{+AN1x`L?0XI*pf)`M{aCDe)oIq2P)8dw@>fyfOMXJq;Iv&S*p?;u-vcsqMpr0hh# z<4liVLH7ehudgqzz(K$4<5GfS-T1!*EeU%5{8@7ddE(?@ z7$f{KdYJ=NWkqr`REN}pE$IpRo;cm0k+Z{M<~DFDxYLhiASmSEK{g;z9`~M9D}rhw z0efKb&$}uc4$Zi5WR47bkUT(vB3zKz1vpyF@z5afSTA{F`%6w#O}`Qt)HGfPK`#Lj%|&VV3E zom-tI95JN5(!1a-wMPpyQu_YzFkY+zHQJ(DNBb6O*yF;xHtMBFp6(NA6EeUw`dTl>Lxi41l^IUqW22hr7K^KYN4(96Z z0i&=^4wrsa{GukRb&(*wQgBCQXvcbi9W`Mc%Yr&8Lpo{$I;z4smZfcJ7kJw&ETmOX z2+U|ACh;$K5%35`3i!Gl>c;&9yl_V;jnsg$0?0mW4KPnzq(V+4 zN59A}IGrG`%t*54NGY9`!2k7&fesuZQ9;ef@IT}RSq_r88}b@bFOWGay)CShholmv#>+D#LX z$Q_;2UAA&lwns9W%0d@a>{3%2!2lACOgT5}-Z(!SkKI&~aT=Mepu`-OI7gJth*Wff z6+xgLmBx%|*b5X9d@D7(dwTJ${*TNELlNh<2rD$X2m<9`rpUSQH9bWmKpvJ-5I8g8{DG_(8h5CriAgzU&q4!5+HzP0l{vm~4XB2b z%s(~TJV|tC6%V+vDLO449FQ^)ILWu;oW`6vElFHub47?%tpkX+oY7p)-rQhI=SgHg zF%If3Jr#JFWF7e?f4VL6I{b={ApFkC{zesN9HhsB$HtN&B@ZZ8B+Q39Wqqc>)YCMR zWz2y2iH{s8%aqs=8KNIbeYr|V@T0}+CyP?qx6kMz-`J2$fteg@Ud_esIc#$%-q==d z42=bT$awe{<`h}x&c-3S?S*`N;PRH8hgh!N3m&d)GbzF?H~X$4$iN0z(QHYXSSrm! z4bj_FrPhZxh*C*s)*;h$?!ukKZod?<$+anRSr>reH=HjtZXapEeMF|ziUf9;jmHao z7pGEzN1eBXn-muQIEAkgp}8u!kV9UbuG3A@U@f8ioJM!c5!)jdUAdCdgx>RNTz%+ zJd8=)7%AA^PFcKr=~y3Z%idqZ?Y-Fpm?}4KQ_B zt%tE$gLVZ?u|!jxB!rQ5`eGWptnX2t@h!&8FXC$aHJgsHZDk?Y-i7w(k9!$RNlY>} z!X3mu+ZAASqmSGhzY%nm+%fES@|!CktDGIO<&)h}rl^>p%E1Zn=QJY6=zyr;KzD8c zF0TiIa|M466B6gYCqh-%(EkpCDW}ot)$fnL!B_x2}6s*kcZU? zMV(O82}KWoa(`%{sFR60nP_8~sK}PxuS~RITkFU~8#cL~OtfL6tM=VSM7P9-cS2Ft zn+%k>XRvZp7Qfl@&TO!D7utWY1>CabRjW9M0FEG1d)o*A)MHPT{hCE4shNI#jJ_rz zh8%9<3_MJ7-xR&d3oj5V-1D!2*xYL0K$);D+qO*)s;gJ@OK;R;4FZkkXs?KQ-?q1p_5@aB@0Nu4pZ4ti zCpI1l|1_Wc)1dyt)A+~zeT4pE!*N|JV=j)-)41>~@~JrMeMV{>GqUeOR`6!55c`O9E{z7p(w@{O*(dF;T?tdva(?d4(*E^rmmy3znqW&HMtfpGD5TpiGEPPWvWd721 zj?saGqyIOtqlj;!8M4$yj;++@X7NU!SC>CumF^Ctu1k z4RV2v|FmqV4O^5*roj|psh2HYNq?QP+X zSW$+4`2`IxNkp(DLw`F7`t{e0Tde6wMiYg=b3I=s51T|GVH`pV@8H@J+ty-Ypep?A z4UGz}1cb|*@(J^ShDFk4OfyN3`h_Df*ALckV{4@ zv$G}7JfG+TON*^#!S58PzO*;`K;jS{kr8Dfj0h;@{D6k#G`G@4iyEwq(gF_0#Zk+8 z#(mO@%0$Fw!T4ME%oQjp3oC9hKuXYwJ{8U8$wu#dT)Nc9N7$10R4O}JFpmM&P3feE2WMLbTeC}nRt$`7$6iSfLMe(sRTNjK%M-j$ZJS++2~ zU_8iTkZ~0Gbi!t6>UJGW&q)+j45#ac=f;cMgqX<8Po-ul>DWcPo7gcEduc2YGE3!v za6)~=vP;M_JHk;i#YcsUXe@!hHZyusAgcC5cO6=mqQDF;zLWVIlL~L$r9~vf_PI<* zkU=QjT;cK+xE#?2IJuSX!nVSdb1D^j=N<6z7@b_5o}VMLc+i)6DHRqG<62n2GNANC zxI9t@=J#N9$aIMW2UE=;heVTz+$kPKCaC-dC&{pwAHf`8;b1gBMt}T8ukK%OvfC`E z8lrC@P`htVL?d_tNrhFfEYoGS90a^`$eJ&Cl$uQ7<^VD>NcgvTF7>jN1l?wiD7yv1 zknm6r(MhDHJe^FlDV6qKHgz zFh^2hl}hx(8}l>MQ3EEWZ2ln+Ziz6N{Mm#ua$vKr9KhEH`#poF-5ziBLpxi4=*q`i zGyynp%UmA3`{w%WYN+0;ZMJcEczF2o`E&UH@bIwo|IyLWi=#gry*PS)c=+n&v*)k= zaCr3W<>8Azpu=q+XbNC%3xfwb_jDG#KcYJ*Ok0DU~ z;j#Yv82&yk7XKQ`5G|JBKPsN~Uj6m2fBiSS)qnl<-{u?mJ1hF(vH$hz)hA_8!&iTO zwFun-2hgwn?LPSTCxzmH>mSf?_Nj&a*9zzccmeC00>EK_j{jYPb!1V{9`%r ze*o+!7q1qf`w7kAxN6MpAb*m<{_4qRUsa5|UyA1YB~$8w56y&{Z*`Yn{rx{ad!qYT zhVFseLFVquuU`nR6S7oUCh*%x1Z_4lvJxM#r8@RMxdi<{_GX%l_& zZ-;~a`Ne0pyeO}XUVcCOt-f^o4?Sw|=&LWj_~LJ$efHUZyefPSz?aQo^YYivK6_HJ z9JU1g;jTV3Ukc%U=5ke2pTf8^XB{Vz^zL#z{nmz1|l@X+Bmcm&8cRRA6WY4=-c| zE6w@Nc`A_2Uux7Z19Z+zMt5jr#8XwScf2$!s!v5|!ChgEmdq+msG6oD;F@*)fD3XU z%|`}zx`05INrr?L8z&w15L_Y~s0cygtlTZH=!d}|liL5s|M{06O*B+575t|HnQalx|3BG#;42 zg*h`y?wniEmt4@FwfGf*hmqS4L`byv<;BT=?;2;!M{uyvq|t$mfiE6V+$i$Sp}E+#=5M4eT5Zb!WK%B^wdL`fKxQvZdQQpx`um zz-e?_s;GH4#?c3uy2^H^SfUXjY{eE~(?=hmH*B#O}~SQ_#Jp$_zNnvA*xfoon~FWV9Im;!_>xDPG_0;$Mfq7a(I4bW>IKo&-8!6+~5-T+;hEwcE|?j8Nj=QJX4i<6WoS&%2Lw*{&PyAga|oQNwl>#mg|2Hk6yei>wllWIDFdae;=a! z^2@=W(2T~%aFy$rMnugM^5<9&OE4wJ=uZdc#5wp=<t6DcHE+u+YvGj9stqg%6fu>Gu9A3*+B#Q4z70`qn-aF2meRzpdlOt;KShlwr4&17ZN0iw1Bho zpF3#hpF8MZNQsmM=;br`lg3x+cue1;ok0fQ>MCq!oM#XkVMzlYZNmjRy?kAVun3LE zlLX#qN}^cPN$AHh`sEiySrDaRseeQL?V@SYA5O=aDSEj9^Os-zkzcYpqJMd(2=Dyi z%-)raCs?Qf9~WSH3CS@Kj?FNGf086_{3}8FGgX7Es#1~(GG?Z3HfmgCgy9sPfZVpqE+!LHY)lTVw8o1DfHZYp4*hpV`d38v>TyUaG$Aj zw(FsS*X$yd$|%&oP#!DigcI0A;&~tXJo&F1`Ml3_Bf_(C|Eoymos-ug1S|IcvzISl z75D$sqvwa+{(q2CI_*~O{ia(v=PO^>&#U)YX~D^d+HpLNIa}Ci3rrD0A%pvJprygG zZ-+~eU9aL$&Vr395A#ICXs=%L7*o~&tHq0q*6v^nP))kr5?q@>@Cw|vbK1CKRR>#D zUeTQ;88aSG8 zLDe)qMweG6N=ee6$uW9*`25peJ~FGxk?FoRm=yxBnpDqKFc1u%a7o5FO4NYBR`5&+ zwc<;h$8atdq&AkrazByu4YkBGBZn0@wy9j;!djyNfkC#a>3s_{@|8D31IF3awS(RQ zkwWzfOzu&NT*m)FRAGQOlKuYBME`eK*8jbHb=3KPK1jJ2`7ak^dT+Ag`V1z7-#r9N zNo6j`wK;#esmyo($M^aN^7Ch(9@>~@1c5s*AYNLQPu`>FFa38<(9!TEeAY)t2hT+4 z%!Rpd?9!pTdJ$;aW&)2|97~c|9f%N(f@}iA1(cV)Vjhd>hcgL zj&An~+J!U$4aih^|BDd*83YkeLoy>w*#PVfD8P#NpO-HWpOy4~htHlq@Am&gl(L{B zLOHYnn&qvC6H4AI!XUzac>Ad=@?&6;ytGl7uL)f6tq#JYO3FC_*0c_GVg{=Jxp3E2YZ1sYPKBKe)ZXro+7 zV`?Zw(fnDY5{_~$?rr!%*<@&4e>_34{LjLBOU{mST+^wB3V#1r02N+ERI0BlU%#Q! z-g6tJ0!I~CSw50Y(g=&i_nO=VJR!%(lhCk-oI8Tm&SrCSAt1ZqDUZ^adDYWTk~5Q1 zXlTM^hXH4%iu~cPd!G%#8{*G9CxN0fva|oA{j8AJ!@P7^)8-l_lR6>Ci{@gxZF)BB zazyZiu#=1P@19*19+xummdDT9#3h>YJ7g~ghSBk97$kwz0p@AW0ZvkO>wnr*m6YW2 z0i%3$dLd|}fYHAOrGOIM32W|m$Z0m+&Ik!@zYY6NB@!eNL^|Iz{TMT3LJx)|(8dwZ zC%z0gn-QTP9D2g&&mOk47lR|?scAA=G8`FyLbzbO1UbfYBnU)5q$~%|I3Sksc*_NW zh=+m?oFj%>#!DGh^8=u3bd)MC9Srxvey#WQ#ddn|85dum1Qb*<(#2Y9Fyo&vr-Q`S9(_DKuaDL!G3>=l)wTF6EHzzD%0LjT z0KY>kgLL5X`z0AK+)7katSZ`cnPe5u_kq|b8pt2+lc(ga8()A_KQyRv<5i9d>^G~b z@Y`ubAm1q&6CnW6EQ@DNc%I_bLseex_Ur1^=L#G~?JK%8TZ>f(`J6gsXbbCF7i*$= zSBc3IEJ7j`Wj>T^;bk`Pud^?U2N4#OrxM-aIjlkwQ)N!a#j#($0`SkNAh55#5H!5z z2@R@8bvYOWc?S(;tb4z5N|((Jf*C60X|IM}Ia9i5%DHaWQbF&rSE-=wJGA5_EQG6c zlr5y?tL&`&y{oXw!LJqjewF={Aod zo~j@sn5D_k##^rsAmDKlk&Qq!9N8kHGOUc|9oh_@+X3eI(5(P#x*Wb7kLG9uVdONU>MN3bKxdb~Lo3p&bqFXy|%0w3b({=FA6q39XTm12m~^)Ep+E zVP2%;BXAxJ0T|gp<&mjQX)7bjLRh41Qq0m)FAh$Lxq*t- z>oyt*jaXJmZWXv+gIY#^?r-|=gSRVDgr(XBMpdi41|z$^t;)zF(?^`a(!PeA(4U)R z>EE(j#_!h03qEKxPV)p>_xP;W;xci5fC^TX;wufe;za{7f03+JR$9bt01DJzZ7oA{ za_Qmq{?mIV_aq&8WxfM!mqOk$KlsodMD_N6ICr+<2C&Nh|NPZL{LiaruR8nxLzK?` zkA8>t|C+tM9X9{~zS0|@@~aA`fNE3?jm`N6RDPDZ2-x<1TmAu^?Y^_!ceeY^c3&u+ z?Y^_!7fNTlFP6@B-`Vav+kIzu>g-OP-Dv}0o!zOk-FGy!qoExQT_0FSLp$4jXS?rg z_oj5V`=)qxw)=X_en+VmQ5l)!cmlQYMoC?=abU; zqG!wLT)|%GC1b3&5AFPXa;U{KZj? z$iLcZj)QGNW)2SgXxN}mf}Yr$v(4Ec*Bz9#cZkD&G8>T#Wfhzd>P2mwwu)<<+eYvQ=5H3S8Zk*&|mD`EI9Fm2RhW zJH_r4xK6a&B~_){CBOS!vdYuYb7V{3`_riMs})G(G&>DEi!1%Dk@dOG>44m=$EnnE zkm5*^$aqf^ltfsKxrhli$!o$eQ|GUjK#o(X z5RN1_c{7)oI#&}0*W2?YOTPt}Z|+!?oa9Pju+Q^#P{7%kPEtW6x|`Bq>OsOK-HI<6 z4=o^TIV(&-u*a$5D5Nsb6E^RIE*M$6tI=?wJ1%Y`ju!;#e0iIy6i3nA_{d7Va>1it z@Q9pe>rm0RTk925bKOHLU3+Ft@bYohrNG zfaD0=g{=uTAWWH2GDK7=#K#_PU1Y`TRRMnfgO!s%K$9R@2Ij6F1$a9kx68g@oYkM@ z4F=e5j)voVX};F8YR2>A`z!xYaksIDelIRrA*EefATI0oXw`vTvW-+huRVzcnN<4z zH$coVKkN9JHsNhoah{F%%>)Y^6SZuwHH0Kr3*QYPSzY`xg4hnym0IGz)DYMz)^jO? zJ;=tn=8Kg)=`zQ>Wzkr64KBP{(jXAIugM&YV5_vL`EOULQsKm@W{AJ5JgrvfRwJI) z%tsTgs^)TEQ$Yg7#r$mZs4I{$p&!m+;oHkM8^K%VlC=e@SmQ{waxrc}r){h&8*;M- zh!L|oT}D;ubQ!;UT?Vo&{ziLk6(w=odFktd%SibXhh(QKsVJSU#FtK2(&zMPtCru&RkJ$mtAJm1vYSSrmd8ykTMvT@Bq}~zE079VZs#pdyyJL_ z@h|F>QZ+qYfVNz@R5_N|931(&0 z(y|;_T@tMP@O_+%>>(GEixOF(xvpF|E|>%+l8z~n2m?SjMOGS{W{69KCHm^>n|GS5 z{9wlCNAq8mEcU&Y6hLOkz(|Az#2$wz1ch`ocV6jKn&v5!id0E=OQvNKBr6=F@7;H~ zQ_-#}or?B%ucB?HU0tF+?ewFae$JN;;`bo$XwKiaBCAFqCtZtgJk@o7hC z-Ol;Ae6ET)308=%Ulu-Iq1%9u11_p)$U0tEHeJG64w1!t7!8wHE~?glT0>l(2G<5aUu5E<7~0Z$=fx(2uVU%sArX%&tkB(!`mhan^Aje$9cnMW$363ULxGFX0n&7ya60uc?O4XbUYrskc-6C#{RA0+o zkLw8uB!|os6J@hSf??N^i2LRV>ot_CnZm&`Qg?40V=)I+;mu;^v*HJ9Fq=yig3T`S z*}fvMUJ1CI47O1I-HYPyew2QV6n-l<%PK`*!|F>Ed>bhD(5O!9u&!cnmCB%nQmSw3jRJ)>ht`Wm-qp+AHqd+N>1-)>^U4m38-_s9QGE4HR_s%DH;Q+;XMd z5`|m?aiUfccP~n~)(W__9mZ;vZc9$NQX0!mypgVopjK#Hs>wsi6{(gJ>fDeD&mYcn zq^8o!TM4QY-8HzHpoCd{s#{m;mqwNnba8Z-OiLkMX^WbbL)E2C?}{mP`QNiuy3UR7 z#j-5_`_tz~FJ3);R?Po?^z!Aaqb~pZLzFK6JNn(_f6wjh_Q~FT>S}nFWp?+^7t*+w za$qm!Y?nWo7d|t)uaTu;f9KTq;l`QRW6CZ`643yk^3>m~YKG=5b;r0l0X00ANwxwK zeqRp>zi$X>*}d|Y1c&pQYvtixujcrFkgMyPH*x`Yveh(VUbbn(W+s<#H(&vl<;+@T zf-l`Hun6>|;_87_upQYm7m`uVu4zo9!f{drJt>ZE=#nXKDc!r@mRbkFydLn5`fm)*2iJjGq97g`CMnJf8DmxsSB5n#=g zmS6Fa{E}+nasGzJv_kuDt&e(-K6_fzK>jez%1?Z>M1@oss%((-!;!u6LJ!qXe1yn* z5~M(U?r0RDi1S+%(c6Wkw9@~^BrYO2j3^^lM4yc{s-k9o-B&);1!grBhD1_^qydfu ziH!Ni5Bz*-P?qvkMswQ-G$LbnO`)7iMPj4`;K7k1BBo3) zEUNz7P>(wJsf1mda7m4g$;>0@vN5u!luTtpSV%&&hr!dvo&%bA2%b)+2-o@d8gj7T zH)KZ9mqjBs$v$K#sqf!16sER9D*=)Et9nL0bAF&4Lm##0Ha-`+i21NF5rQozi=?4y)L zM9Kor7et)1ki18GoJDg4f0IxT>^@AdMXR<22)n8Tn6KS0eBx{;fY#Ha=dYfB`t0TN zS1YHeg2J?&B8T1frYeFSISy@c;_j%LQdYkUC=6EB^qMzVx;ZvuO&A_UBrLX816x-F zOJPEU1xF>iC5dwJmI)7c>-NtGt6*A^bbsQb0RS3U*aPrH%jpV+=v=`iM0!2dK?c*S z*~VY-(XK@H7G}!XTON`>+hqv69^iSQ@zgt7Q^*>q36E$n*FAhkW|~f2PYD`R5&_3V z<2Y4%=0xLqwg-q&W?}CT94@`uk5;(ZkE5r_ct@-fg9cOlIZ_u9IhWnaY z-?+!aUX!{p!C(%wr<6Sm0?s52iDq8hwc4Im3xcM2R)^S#5N74{eCK2e8NbJA6wN)z zP>+EeqB9&!OT$rr7)*K0BR=6N5FMhcf^SlNckBA98wWY3rlKB;2~luqv6MWZn&&Ox z2pbXD>O+mp-eSK8=xD!R10T;Yjo=PoyN9#ASG#31*kUms3JHO0WYkpE3_r03@7 z@E90&*dP3H55`0PAutPhJTPqe{!QI7 zuuBo#Vk#P~24BCsLD0%A(wK)tT4JKRNkX^W#;KO3Y6oTP2|}hWb|yM4w`#yD>T5eSO zll&__s_x?|2li}jZc%u!B4Nf}6gJ+NGAgGe)WrEh zIAUoOl}Vm$%xHJ&4)uhGNYV+zkzA-0MSz+)&#hZqZWroy_2HnQdwHX#8K4S9l#lZG z=D|{F*fZl%aOAm(9BR5w`wm&jtZoxjXf28~Ipl#n2sjH!qU3?|gpmi?T9uk^%YjJQ zK=B;?KDa0DC5BW6;omljWPa;_IJV1NPHz`+jvl~FnI@%?B@MTB`>8}_Jx5k@05rj1 z0gVVqLCzH}np&Sswu)K09%}G+fW~e`m$r%eD6*#tbfxGg*?WT4H7}dRLgM*R#UV`hLa(} zK}EXH`2JcaT#Q+YqbqPm=1p7*Yw1N1k{b$%CWH~;4GNkL8!IUga3IUJO6@$oA({x> z6dSlfDbsSNRfEiV3erDa7k)q-Qwi`S&!9?W@1evdb01dBm+825*l(0{$x>zZZhYI zgV~YZw?{WM?VMbke+WL{Nj6i1IZwra-?2f$!-1QeC0l#Mc|V6c29l^D`Uads3)(qTd6`O%NP93yvvF-* zV+d1JctKestffwg%*erVrWJuBrex*{&1X1mywmPQWUMqO;KEhzh@dP5j9lefogc6| zV~v3C2R1fCi>P1cH>Xx!zj5N45oT*eyG*ZF6u4K;z;b{iV!EqIX`*+3PJ{xL7M0!y z68d6Uf*Po`g8XxXD3t~M*#x$~Fg5DM{mnV-oM5u$U6sZa0-_TIb3$JoW12GHk=Yzg z=uDp~G$w{Y!GTH(dlsOIltI2W$`B+^n_E-m9gU)Tjc+zD`TYb_vRA1B^VjlwQFkBx z4_Z*+)GEs8kPI^(NfTGIy%a|w2ieeL6yx{#`S}gWbqNpc5q$yb@$D@=t8V9GG{r0g z&fQY<1k|(VOb@A|K^kGvM^rh%zcvApJ<%+S0YN4>uv%^wW(3NKdyE*tzCKz8D=L9R zX&wsSno}UBl{Mg(=(0$hsCmB-@G~M)Jfl1nnp=zXXbh})n=(~6*+At{D-BvV48xGy zjVgDgU-lLV2mpPMHdr7?Y^sS7#H4<+=)UlmkK>SL?i;xW+;Fve0*b&D8aJk*tX}4 zJ#)skZQHhO+qP}nwr$%!vvcp;_hbKTL{?W-euaqc%I@mS%B04QC=N%Ls|@yk+Cx3t zFUH$0FodM7!lImRW^qkkfClAT$nf zCtU|Nd)u);W$T<9vmv*_$)zhUrER|Stu?B0=zZXQ;#kRLo5!4AtgOP}1m%wlANH$pZ&Mg+_V1Nqym$Uo_EvZ`g&C=iQ%%9p&kO@* z3rK-oX52@c(|2xID$ll){Mf1f;TZHjg>+=pa}51p!hmLwe#+2Vc}0lpVJC**LS`8v z;lwbry%ArqHj!7RO-eg7=*~<_)TCnr(6Mcx>UHLY?H+cU2Z84SD;As%SFLYaQCY$5 zk>9r(dyK`5gJ_~tX?>FUmF(j|&;@Pl-hifp$l~W`L8LI|&`esP<{lGym;@;BY)kNR zdUdBrw6SGkK7eH01EtD|Pwiss;Td?~hyaNO$msS*U8PSX;Bk#R8# z0aayl?Qw$yw&u6@Hi`c6gHN1!Cb&loxXUWd^jC<6mLUC|V#|YIV++>(5A01oIfXz= zb&47dDKS^AUdu&4q4ZR_l`2aZ$p9!xQD`($6$&D=b<(fa<=O~)Bw+d}eoEaIYT7n0 zl(BgKpSf(?WJbGfgyg}%3*F+f{;sLCL*DN6jYl~qmRfbCxLD_>MMcBbQnE8#=Y#zz zY%zLpjd~_f$B9bDz;F+%RU}$;eR^QoX{eOo$PFdBYl>izzUTW}v1L3Nfq13WVr6|be&OA~v@WsF{xn^dJunPc%d z8%$M&b-H@7eat%Qwt68^C9NI6oKb1ATsHS>3pbPu8mQLdvx@8a~vOJaxAJNfAWJKY+5(_9NHDBhrH4Mey-@m)IA zpDIgtw=)1#`NK0f#bAXJvm)`hKa3VYk=cn`!Sacjb%9e^`gP(ZYAd&rn<(gKl(0Yo z0!TNUrF6)wQ=}xIKcMiI+?M>>W)DCq-Z&pMG?(MBKuD1%F~jxrAk2Q1B{GtNH?vX7 z4?yW6#Zf{s_Hnc)%zurIIXg*K-en?%;hVVb@;>)M~;qAoA`k>7Sdc~or6mb0*d6=8Hg<8S3h)^oH&d+f}PVtJI~kfqEK zXU*RD%1BlEMg4WydY~Uxy(Xhh9?4AcS-c=R*12e^#9Tb&=JPVlHPfi)8EKYItUU{W zX!8WiRhWZ3#vE)$qqIM8y1QW#(Z|{hQq<*+p)jdryMdwYHOjl89@OT){gJ{ zb8R$}BfbJhUFd11R>4q3!tBp#kPd6c_{k=7Xd(N6LDpIIMA(136>8)vB_3yqd5S?? zV4k=6whViFOLS!ufQR-V3+%^yb`Jo9gNGB{#Okecyt%_Qx0i^5^nOSH zwg8ZjIH5_evyibVSK2LeO!oeFFLD`&9tt7xzNVk!*+_E+8!HqPlSg6Gqcmz`@TXBu z7ZDjyWg)q*l(Ous2f4(Bzf)vaImDC`!MZpFR>YECTed zrbZ4sNK~s^6}RNc?cSeG3v-9V;Y(55^|Exo5#iWd(F1VpHGL8?v%8ap!yFn}E5m!0 zL~g~ItjY!gbkcUgogT|j`_aKhz);Q%`RSa1!IPN_CZn=o05y;vYxz8gZl{)^Bz}Y< z)p^N}Mto??#D<5YSvLLwYB2}n%l9jimBN~^*iL<5(JW~PxL>E(v(qdHOJJ!6B)bxI zZ-^%FkJ|*y44<|?LsM|{8FWnT8dhI8U6}PdkKX7dV3buz#CpClV6(0tsNB2mEsYvC z^!Xw)#nYp5PysG6E*}8EJ?rlJZVa9-joQl>Z!;5NX%Oav@wnF*XR|I))~8)TW!D{$ zM(U6|Ij8SjAT#!gjyNp9v4^sLg4vrOkk9E!4F&!2n279H7J-liQFWLe>j!yI106+= z!5UJ$?LY?DrO_uhSjWaLwn~RxO}`p>9Sz889K2K9_%2O#V2wr&WQCri?HC+@QHl+e zu(&HLQ-5a6G}K-V$+{_Q^Ws~%LbxMVEgn92$;2R!5_WTOB>_qggL<#N3-j%2x`ic^bI=rd*#Dby#PjI1^9`CO%D6VxG4>BC5GBS`t;@usti zEK>6GCfo58SA?4Q4x?*LVM%x{3QfJAx?*(zYHB}rL{|F#Xs4}a7yfJPkNj7eEe%=j z-V&9?!r{(BRI1rq{zTM3xG6Q1Sq=&yG_L~V)ul0Ie!8%RO;s}vMZx5cRq(iR1jEy5 zz;hmEn}Xph6Q96iu+=xzjzZ#+$TA+MoLY<-i;Ae8bHWXg5Sop@51C< znz=_)rrRoCGsR~_2%BfwAQ5=bZ@S0wnf~b%^6jN@!Ql)vUTVWpEt~#p{Idb=%R>$Jh4OS3<-qRn2e@8b07$Tn7;?vX{o)4L+7Fm~DCYEg z-}R4YZ9K5Im^+QT&s`D>Y}8nCD@ao=vUUxM2?1I@iDxH+MoD`C7 zn{LI+Uj5Wyrywhgvy%M91-TAajQ{_MSS~baSK2- z;tePdS$#SII7^ucfN1cPc1@ca8XfR>D%w+@9jU);OsT`Vez$c$kL;U5`)N*j(9Hut zqg~xVbt6H&nGdJvr~LA9=1_=E9xvs_hWH+5e^aTyA4a66Dn%OT=5KlTx= zoEkjXkj^IoM+K`Zw0Z3UWp_&|w)%Ee`KCcSn% z4T67Qb+6ClLcvLaL2*UA>G!oTfC3&fbLv&B_GNE=%5~YqJlEP_9bck7SLVI7*zSa)l4k z7HTc!%ZOu^c{@$^%T!@UJlF{9vg8&bTPpcVaHJ_@`i<)!X2ghy$s>G)6`a2RsW59SjB`gLVMOK_lqf^?4Fpoocp!R9!v*vGc0~RwB;tDf+loT!+~`6D zsKr#?9+X)Vfu?|RwHOK_`|qKAyyi|jNl92(;~8r=RCHwBIdybm7?{%fa10!FtA!G= z(xi?DKD=84uMioj@v(&v#;cne9~S#KqZ%xtx#+Yut}Cn0Bge7k$F#!~mv?yBXC*58 z@-SWJK?IB9g43V5j6i>lcURTag*moBQT;HD0O}5J%YnlDYYD35W=zy27a|kmZJPAt z2Z8mN#y6}mjLkC*+M{LT?O_uMVAsbBg|viA(A&pmJ+wvF);sZ9z!loL2CWU^F?H%p zI(4NenB9y>PiOO^w;eiwmGs4K88=&gOVk+O(Ae=oc2%-ZQOXH#IdKGkLc5ZT`sLTjVkL^vD)Uo4Cju zElyHr4GZ-7!@p@#Vl>D_%4{@fB_#&Zx!6!Z6ZSP@trd2oidN*bvqj^>vv7Dey^p_j zix5&v1r2o}Bi!kAL_2UGlSR5e*&iiM8iE502N$m;AL=?$W`k6-BuB{7s+oVLV6H79 z{k()_y~VQOnrFl9b!q(%{*IK*tgh`Rb(kG9i3H7V@o=KAdjm z*D~gb;7~fV#fA5h({=4~rk~znELQ;eO38^^FmbFiJh|cx-|F^w%_^hn$p+^zePe6j z?dc*ObMvTWi+rE2M_!dId^%!@0*1z8;mzd9intkyOo8X$Yq^;rWiQczXbuqL>;C?l{G&qBpRlv@2iN6gf?-9kcxuM zo)gH}H=~?fpG3@rk#KlX5eD32gTbBT6-m>IYpLe!6l$vM6f5`hCL>shQW8r|4jT2} zOoM|a{g+?GxNP!!e>q8=)X&rA41SaGf2&P~0+y1aK^?yj13$LhCHtRk{^>Q^s@{Cr z{D9Azfe|bABHA z;8^UKp+RC1Dc4rwCoJK9N7%`6airM|VMgP_7-7bK2zmDaETbC!UKWv8MvU*#{j|sY z=CPTB|L3rgNS-@L^}RdczOcPl)xqddTu5ahG9g&iyYME&L#%veTW8beRTv9Mvr3#p zJ*Va7?c)qYup_)cil)7daccuSW&?~yR%J1(|8lCd@kPYuV3*5aJeRbwwT@I90H?Y` zsE#)~ekQ`FmdNx2hbszj$xw-?)#e;&N4qpPl3bwW;7Y>9MGY2}4tWdOnez>3IUP2{ ztmXoocNx|6#Uu;!teN38xKoP=JPTBeOjky>UT+EFVX zJ(uWZQ+P(<*KQN*FgPCC3f>DI_X${;bx}PQQSy=p;Kc<7AY689X&FpaT;cC&`K93KdhUV_fR4gSvrebw-M;%U zYQ^?QByJA|MJz$F8&bR}S|&_#wm!)0YEI+!OE!mS><&ejPP6R>9uC*Z+T`zRguGTH(^2jb7S?U$~(EZ@x^(i$69qerCJ)3G{~QaWz!A{Ad|}Q@-pdnF-pXn)#Wcc>}ptSKZ9KG9r1mTti{~ z%0;&o-GX&nVwILjM#dd0*v_&eG81|}XYua#vbevEvd1U_u7S>L3NVdmWx_m?I#p}I zVY7@IA2y|0##;nIzAs$@C4t_UR6?^T%KRps5<y3*Qj}6FG2uRfHG#8#JCbvq#Gd z0gNyxeY$=uoq6Sq3{1vS=_@4^v82cK+VvCj>VfM(Y2a%^^LA4}%MrY0kP806thKCl zCB0q0=QIWT!N%R=Y?)&Zfp=pg% z@6XpPgf}ko-+XLbEBHO;+MqNLWyq^j|Wn_@y@{o4m z{mhYr%LhC1j12yA^+2uvC~l3q1Zs%4Q+G&LBd+!)_!R1U8O(ccvr)Vz!`5!- z{6E($^9NvzGyDa%pLx@aFTatu-!J~poX8t(vSI4PJ(1LLTfDqCC)hLW%1uepBO=B; zzCYl1J;vXFl+BMzv@tkl@GoAT6m0?4lv)= zlkqk@&3cj)gF=MWXtRUC(3XEDV+UXa)wsr9OiA{e1Xb|ZJs+I;vDm>CHtPcpxdavX zq+HKC0*bs^Y8TzXMJ_27%f*dK&M6fF=hVuQlBVSs)XLL;�BNi!1)iNBbXmMxnW; z*rK~`MtbEs6z22`AwTOCb>zKYkj*FnL^Qtb2a;f)tok{nm=`4MF|@?#55~}Ts*m(uKs3c5lnC2Wa&5m z?0Iead%JyHivRq3w$Jf>k2PttMqgRFl*e12D-f9;Bd$%%AJ(N7iW)JB z_WCc4Z3~y-KTmsaH+%QCTYmnI%sVl)_RH)AblA`i`p_&Y`%epc?Q2?r{~>Kqm=)O`G4w@zheGhwk1$e#z(>jT#~J;{`w9KWS+#(raz#6QBYyaIo?T?)s~RC|#p+I^=3jFE zo%Z>E*TAxRMTNxwXN@fX4pQW#THx~kQp;fNe&}zQtP=`FRiP;}KA8Z%pN^AOBjF=$ zf1NC9z(8_na{#QyP73+OWWW>*abQq=U@!{5L`3vH4g6k&OHkNa4Q~@Jkt|UVy2csx zlE^SrGGR`K_Ynt3OeZEnL{nsJ5);Us3RtHM<<^zTKd1qJQK$9Q_9g`6Fo(TbJa~10 zj~yL2Rmcg|4#&kp%1W{^w3+&*GZ&@Qb|avo;SLY59DxA;3ILM#Ik*8$XFWO{qje01 zoT5_zQ^Md{{e81mYF3lNvd6l_=^!&mz#7@5#TiIS6Z|q}1oGmhnX-){Gg2*BY^`Nx z2S?^@ZcWp8$bsM~)K~oYhl_?1OGr}`75+!6(am@3^XBLw_f}>n=Q^-WV23R2d^lr* zI%LbBi3(Alk&aWM3lTSm8{%>gVXd&M*Z)wmB0Xj z$F$;hJnuimI5c9Zz>w4^tl}#5THmYAK~4O`g{>qi;H8jHie8*as z3{My0081sBM8ypkdKbW0rBHe5OXK%=@&%5QR;#amcwCNl zBwAdQ5o&{tv|u+AU{!Yd zOSzJ`_<8^CZr&oaqM=S>QZ={D$npt?UcErQTOJb7vP*&%70)QM>D<-*3NlDxYM5(@9*ky#rU8M5vWG=x)2{BS>FlB-Y0t6$kh#1c)kyeW z=1y7Dur)dz{f?qeGiCl2?&`x`lI-RA>j`LP#OVQzOgHhJ1hch~8xFv@Jr{>sN?JO9Ubi9CO!}G>!uWRn+%5^w|_t$SMdzzy+rBpp9wrMTq|X?nHObW|hM%7Xngccf^5 zPploa|9J|l5yPeDz+|)o#mbTy*RfoVFNmk1ki7%hD|@L(dbrMyaoT$vm}ilUyPdP` z@IYl!xx*0HFeV;c&xgwm=!g3VZZG^EZ^I5-*B{qehi!|CS;Nq(-S5aNgMrO3=y+%a zCu`XxXSg8;T+8#ewRwBBC7Bqvtk0xW-clh2S2qMqJtF1q4r822yiE>iSZE)Cifie! zK=%uBuW&0lKsVjN1BM=i}MuE?qt5 z<CP?M<78#5NEn=7+usz;W*V#C#ZSa8gdp_saLoQPU_JQfe<{>9^? z1Dxq8E}p%+DoLsDJRd7!WO0>ukr8MF|XsJSs(a72`4 z)wqJrzpu&0gpfhHU6RKm-DYIr+IG~ERIM1vRXoWI4_&2up8$2mF`Cxs+gg4H2|L^3 zE+;f2Xt{0I{-H#rj-2e+BY*;-ALqrB`w$>s&q~(F?7D1ueDeNJ7@iP2Y+M(5-Z75S z3R>IinkVzgXRc1-7Z-MZ;LgGQn*IoAM<0GvIpmn+iLCRe>qW4d$P180M)&-@h8BLOZp#<_pO0ddT62^etFALxF;4 zk3R~{$;PiRVft}J-q1D}iD{ATKjH}#;o*Jre3Z0zK~LU>B^f`)lxTD6Rn}4^>4$a8 zy<9l>x1o;EM#ih2Fc{B0yD64(DfH+rhhsi2Oo8hQb*v7r1md=0rb;!5fdGx~tTe80 z4EsoyTLDdKzSDVUP_T^64CJ!P2{no3gIeIn$NR64sd_`s==4>CK&Q?!qSw*_kSY-n zE>ZvLUw{5R>IQ!I-dum%vAxye|2W?M!0yNLUjI1%=-!O_-rU|^f8RZC($&z-_!NEq zeE1jNSO1!0y79->V`2TTZhxL*tL14BoF|t2BK_014E07eft&_=&SQF=z->iJS?AKY z_fm9H4ehqS+F~5Uf%uoaF?wK+QXWk(FBQJdL<`#;B*=X;XRrkQpxoh_?HH$okfQh^ zn^$-i+RUyvEdNC;s6TRA`jm6YSpHk?cq(Q*W~JC{&)`C zW|5L-Dw3 z!xFGZxAxfXgmy#2f{9V&pbaG4Hqg2Uju#hKmzR%r&zV>`8og{*Biq5q(pZWIrHJe& z$S}F5an0rA6sj}6LdJ@I1gK?-O+VCdQ76~sh|$K7Ias_dzn*jY!qZxP6a#9qS*T@@ zjxDGqt6?^~Tr)XuGcs)xnD{NHbp?K)*E-nE12^L=UE|Ch^St9F%epVOdMLAcfYI7d zLmCiu{uvs!`Y}Q=gX#{Cb}bYa^f$M43KI|#uB3N*j;#=3{pyY0)X4QkDf#B=d?rWK z@ujMm;(Z&33%&}&EEwMiw4E;gyD7~J#O#vgk^4Tb#Ii1MZq$R1Ew4J4Xa zM9aV!5LLDZ^0lg+!Ih}^q(5xUEJi&CmkyvC?D|Zb4jufCF1Mk2pO?@@``H6xJ96c zR1UoTy|?jTr6L!vusx*$7)V0w8R9dlZHh-+VE~C(5OvdPjf$0ZyYd=TG|Cr=8D(tI z8iMX-Xf5R;p{5#f@%jZy{2X#Z1UTbFCh|1^cJ9HdsxUresY;L|v0QqAN*AG}B!9t$ zy(WO2a$ia)S_w%A@nol7i3FsNcaRD3I^7g~D>JS81C@dl3XSxBdmrcuYtU1wj`AI{oawrP2vk2REOD4o+Zz1Si!ciE8$y=DeoS1hPYUNAXI0N{Ayv%X8yHJ5I~qUT z*NS*FnW8Fxn!DnC&X?C67UWdlxlmS5xaOKClGnYngQ4|A%t3a1&O zStb}al3%gd48)Znlsa3yURMOY2!{4M+SRt*XP-bmvU9;G zpC&gvy*|cN!#Z+Ynn~NLY59oZZCCU1`nwla54RCdEGajqKL?=q=9-#INdm&cNR%^~ zU}D#bq<36!YYf!G9-^qKXXtMdz|bm*SOM9dhG5KDrS*6hehY`gF}TUh`5_xj{4VPy zXI^)%mGC)PS4(aZ(0p=}f2lYMsW_5c_DkqpfN-dGYnn*H#bKT1LcqdLghV%DXTck) zqv2x!jBw%TxTE#niB!bi;95sxLE@=)suUyOKf4oBOM&@&cQnbu`8v?`1NNosR%>FF`x!#PI z=KluD%;>zvfFUMM+4UDv4SAyPnBr*O&Jjb%DYOk@=YDTOAsnP%7>Q~JPq%(l_39qg z(`y({_kK+E^(}(C=iuHnws!51YK8ZgEvUPBNVPTj@)lA2KiuEandvQx@7MYvRmU&u zZK`opm73erE7<$x4eIlEu$y0m=RXDd+228f%M*Sv-`{S_qLe#Yi{?NiayKzuFEr{S z_~w-o;*a@`aN?8INCh-N^_-*{mMhyf3PQKTAOgC8#i2WSF}=Yr7*d}ZnaJNs=Fi+c z!+KqyCM6~@GI-xY$0G1WIei4rzNb;z_jPnnV4vA_1|jBM!i?P3}*fHKVyj@nLkc(!2p#A*ZnfeObUEZ z^#Gvg2C+O0?R!Bu)s(rG!0(}s4z+rbtd=Q-gf39%6;t=~Lm!u!J&SVb+WLNB| z)gDYp!?NxvKc-G|USpHyJF5K?mGLb(Uj3xcWvaVFrPPqeG$jeeAbm?+!Q<$s?{Jc` zTgV|O6qmB+=-)!~Z{zc4i;%j*y_NSo-UnC3p_2@#rG*n+Kf5}yvY&16-}bn(vKlcd zY-+8vydzn~D=;_UkIW>iZclRu(!$af)f~KDxw=tP8o$UlFHOL+S<=~$Iy1M9R!b6h zsIbzOTYmfqD(v=E^*VeMNtr##A@4Jek}A|W@zA<>=0!)pOttRs#iNS}1*lSAOxS6i zB9o*;o!yhv3f>1emZ(v_01wTjnz9ZI#v>L2W!5b)(d8>pbBz4SbQ#0dTHJ6lfx(A6 zLlfz(*d6GW$=b*BJ@KYC68D!IlB%S#G*Con*6rS?d1%0GQ}Qc#A{Zab>V-!40x=qK zhSYd8s9V!^6;XVBk%HDDg+i?rUb*2>U(|)xM8ZmOS2U$Z9FO(d8SJ%Y&AdtsTU-1$ zB}T1`uXB|qN!i;{@YN>aDy#`hh#TPxB>7igHk89+#z|Bi(_$vcHV%gbgz zlq$K2QF-R>74#~A5DZOfPv(ZAUYk+RxcXkQb${IGemeH#@1LbQZu}^>#>aD2+T_h@ zQr=a!##wXExnLF44gZF1x{akZA!&etp!3B|oOkyQ>>s@RkXLpzB>5u`C}{7<5)ipJ z%n^VGP|^-qnvN_BEU|ex@v<(ay2EpK)tX1uOWjPD_v}Vy$1l>Ugz64z7=dF@VF*p;v!-^ejas*;^W@E}9$}jp5aHcpU{S zQhM;HhalLdunfY)+VG>;_7T)J_)-o7Yv5Ox#@rR3NgnzG4`wb`?e0LD@2AJ!=zrXa1XKhfVk%*vq^JVB>N5$FV zbNqjfjFQO8e`88YKr12Bla%#Wq~&8t)I-m+kODRhL5(&)(V&;P3`jwvRR8+Ojea>H z(SDI7H;_CWO<(qUeV2+=l3nT%3*CIGfwTtsxnCNuZ!m{H%b(o_CdDO5R~miNZ;#3c zY0pxdMU5@ivJq+Ri8DrjTzHz;+tTegBxY$pm)RC_ITmK>rBlYr|LiQ}iu2N||?HYS8nPV92z zYF?1=VyOkH6hf2RW9z{2V@jDR$JW5u1kIjB3l?^Lw1HXK(Lh5@8K6QvLNE{rq$S1Q z5A^#WtMCd!F$GZR+?_E^17LYOqQ%mV>;A-F|B?C|gyZ%1w6nF){|0nPzL!TbGC7px zurjl8zLG}9F{AaltrlI#XOzY$>p1r$Um94X8mPcx3uiY$I7ugo=cA~eH)y58{_(68 zw4cO#rAavza?dj%w^!qlog(`EU`Il2-L69U)LY^5H0SqnILlM+oV%2sHv`AOMFd^o=r!cXta6TQi8?411_YRKK!ZOQ@Zzog;(ow%Km8=Q1EPWEr z;#jk9dtq0?-@~<7ZF^sbaxXIpyG11?5*pxqOWg^Cnd&bvRB28==JIwi02-5afqqnk z-u~%C-dfcXepdAjAM*?g{iJHz{LNV!=TI_8;yMl2$ch|x&++ACW^f%j1JSne!`=lN zy)D?b&7+v2zPwDoYLe#M`wnx|^Kg?!5s}Lx2IZt{($f4rD}mA#E{<~gK}Clv)5|tV zIIeIrZsiX&)RKFEM6Go@uxh>xA8>Gxx>o$tzQBVrL^Q!ys>fsgfSwpsHOIV)8ubq* z)NS$;67#T?dp9i7v+IEU3^W_b92Hc$DCLdSO~;O+>Cg@vR6JFqFxsV!0>AB>@Z{j| zu)<~yMCx>L3u38Up&11P_#leKH+S}X=3D1H}x4Y zR$N%RP|Fw*4?yeffw0sM6EtFKdn;J3H5uj}P;9))n~4!eDXQ*$By26?ZLvpI8#$Gn zq80WXD|#BU_E1k+6D=gLQ^Zalcit*a;^O$>^0Dej$N+&{V4)Ks>Lr*0$(&J8IKt$z z5#u@!;GyhDvU>J7Iu&~xs$(jn$R-e=N26*B4h1U2;$&@NR#5Y1^%80c+-1`CS_lXi z3oKF5B7CimqbB(f02!F~BQP(9L2lSz&C(0($L*l+_rKJ`!d_gwe|H~myuM%G`@JD| zu|I4doAY!xp}Y9F|3O}3Z*4?Qc^3+Gee|roEqy+Y@4y}Y86HW-#Qb|vc0$i3FG-*R z=k3#h+#Au*T6)9FQf)h-Q6z0%FT2zMX0kh{F_6wtnipYD7?2qM=HI|C3LC9Jq7ohd z@aJ_+5^4ZGGFWI9B{^VLj!055g-obQ&7yg!1GWSJt0ohJyYXV0-@fyJY>UMxa8``Z z_zN2PvbG+!CyDLCO&e7#a|10Pv01KLjU)c-Q0flo=yIvN7DhK>yDW4E`EkX{TUed_TdzrG7>9c6bkUWihpxO{g6*G8M+Z=CtF)@Y!7o*cKA+$B-v60hDAPW*QE$68bbl))3N(o^+Y?CfOyATZQgMYB}wN6L0LWoPs^IT2N@o4I#?l*hi+)m=36HJ#$N zez&>PGIZgH@Iwx`#WyFIwv`KiDV!Iei4*zr(oE|AgqbmeGVq#bxg8cPZBB+!yT#i> za#&Sf1BvwqFgnt~Tumz(P81ZX_QX0(v;oBKg3#~SHu@H&XhiogdM9~YUvJTbn*XlH zCa5)APL(Q9D#9iZ%!`p?B#g3Z>L`UF8zJkBEJ`KE^10R8Kz2z6rC7u(ue+H~OH>7B zdVplZT?cuj5;OnGJR%J-nKq2Y24cS|U?8bENup(4x#$jbF!^PheO@0|Z(zXSo}PJ% z4|~;DEm!$-LG&BCLfYq@E~>F08~IvkJ>BUy?JDygFNQJ|R5i@#3)xudwC{#Uv472> zOP3@J*Mz~gkc**a&S!9vfFh>Z0Nnoi=iZ^H$sW6yh)_BwA({o&@NOQ)=-0_;hcZz` z#b>5N_zYdx9x5|#=?7K6K9KL1Q+mSu4*~guieQy=+}#G>&l?^9(!StSv@Y1L=hEqa zk`V(LH}dG^U4!bLU)9Coevfj_Az@$h(SU6+3Ki~@faYgpVERVUuCY#nL!t1wif#F8 zGNOe+l|HKU=|PxDQyEI!{3B7#jQ-Pp`GMCIGMpGi$hc*OsKdGS?9FdIYSWs-;?D>0 zWC!5?S6o}Y7W(^KHVmUyD}?7L2gW4PtAHj+x+Sm2~GT_FiW5P^s{4EYBv9(!_UpIjVJZu#A(HRvX2hcm6DFsL-p75 zI;+TUM&l;Tu??&G$XA17G#TxBn^h!^8o}qBg5D8oa&zWKCI(x_<>m#MmH7^_U3If> z(!UHFlA{zgJ;703uh7&Ao8R?=ue0tcKy?9oX6+cxf0%tZi61o$BzcyFP`Uw08Hri# z%-BWzH~1a`P*8wUmRw7TVgoQzqo<=Rtu8Qupb8_N-JAKby=F05)bhSnOdmvFA0@)3K z5Z#Zj^g!bV@EkivKoIBS8~C+a6_ickr3lpRxWg!6N@ z;JrC2-e*-G4hCb01A-3mk@nhiU`KQQliG>WYwp@q!oH$C46VfvnBHV8jD68Dc@4Ji zy+(sp!A2X+xe}I=z@}i~N(IzGU%ro>I1YO2dj6|h{fDGs5mWf4p?CAXHtMq>|AaK4 z=o;r+y-n)`L`uu1(sbQy7A1Ndn7I-}h>`w+=w&Y2IUF^6i;%ePDESN{#Sy=kvb@bk z+D&Hf>jlpon&mWJ_xg=w#=ZVpj(Fte%vqXYrAnjNL>hB`>~q0Y1iru3z(oOSJH@@9 zaIx43_%U8akX!G}@`RNGHKha|thR2}8-b0G`hNUZM$$WPkX_Pku|%uRv}XN}C`vSs zw(i&N(BjUn)qUw0m#z*4XwZ}Vgw7L{roCzJmF`Ld8ML=8hi!J^RfX~Q$B6b~nal#R zNDe^pPYQx6F~oxJ01D`m%HEs&*3#%Wk0F)iF)+Z0frISRVW~&1zox$xF94(jlofve0Br@oT zg-O`81+}fXIgeP8`h60mADbPcYa!X>v3|hixL&ZIE6Fg-+PRI?g#73+nF>t}o9ga| zfb!-}`p+Fi&RjPT9xaSEM@TSeJ7UtT1S+-%kS~pbVAGXS4{}$u3Ij_$>W+37X#chb zDP_qJP{mJm%D${itB)yr*Fa*f6nfB{Xcu2w!d55UTE|!bwC0&^Y)l7wd6k8CB{*dTX9EZ6y5xEZbf^@zmFw)qxbjQ+l*^bWWv|08`n{MA@ zv<>jBH8Fs`IMVyS`K^Qa+Kv`2MDkZo_+Lq(M6I|0_2k*2Bev7Y$J=EJes&z%<$473YrVaRjvMG>u|H3Ep8bsMIbvr+Kh zvf4*69)G(g2oV*rAinNe%Ws8JcJb(ZL$a6_l0v1b>3hPq@vhw#9G|K>V~RcaH@xiB z#or8pdU(Up6b#USzTtA0&PT z-hIvJis_HNfcW@#Y=4K;(Eo1AsQ5E?O&NG-rK_-@~&-zoH68u{BYLY z!m&b`?yBEkwSnuie=MHn&&=!3J7%A|koAS*^M7kN`LixeEiQl8v5rW3G6DeR99jRC z`Vaq*crWePZDeALGv@5w7j%^bqU)G*sU;<<`JQGC{fomUEfuB?lI`O!+pWi5DjlFn zp|U_4*}xyvwtfqix^qIugQL;v`d9cpuQs^0!TJ1XnABa7vMgsYoJZEwTpIyW-YK*Y zTq1vtJ}zAr4O91(T&$|Zr|)OA?Cb0ZwyzlmkKa3&n$J~vJC*iMhc{~o`U5w;?wf{| z9j(lE0*{%|)K<|z=y7SK;Y!KnzE^R*T8kXNutD^4mTGLOqM^NA0{6F?ga^5ZiQx4z z1f>&c_?AZ}_eB$|dGsL&>5Y59Sh^Tw_b=uwTd%amft|GbKY(h2bETZMLzgPD0|r{C%SFlML21~rs$q?&quNtI+lPq$zd$5pDmHz$U@w$u{_((||Ut`s*a^^_`EI)VaOjVaBKB=QEzPN!5_ zJfF$}56b_C>i`h`Yt8dF+XLFxm%sdVrex&P)C-wV$pru*wv03^uOnnB=ib;pDA5N8 z<;v;MP^lQsYqd!c_VAI7D1K;nhLs}U$__Q#7iP3fI!P04$?WVfoV}`&GxblyZSTql-v+_$tOaUN`j7Bc-`qZPwFID?RK#!;RPK*EZ2vmXzB+Jbnv z6BS(U5f#T6W_bl*g2wbfO!<;m#gABc%U_@1!9ZO&u&}ZNh;4h9yeM$=ZNKZdCGg#| zfL6JQCF;OaT-rL7OwGd2^e&pWefS4YX$%dWr!zKJb`TVJ<#cHo35OhCB~bDSauSDf5hj-e}70b}eyt zE^(JR^F&DOZ|o<^7?O2wsvpu`mh3Ck(|Rh?PxuXoVCY_;#pg_h>3l*+e#@?$kF&I& zC@LPF@_Hp9v*U#w>e0Sk;dLjqR8dIb0XaS2D!c601ULLW`qe-m`~7Td6|2?A$C#u1 zmEGN~s2xPq2nI0x0@qX4nj=xKtz^!;XO`ogs8Z$Ojiq3V7_2x4MiHp$bXX5(Vw{9S z116!LBO0xGH!YeuTo!gKiY)UF_u@?NV%d6~&0phM$r!*|(&np@tF?*jqdFC8?fjI> z_7p!{dtv3YF$+nX6A4lg{X~`bPm1z#GP0owS>DcYnI9loH`@A*rtuUCS!*d>IY+bm z!^KV42xQm=B(goz#htlptjv%V-s4Gef}3+$OD_T?t1t1ci(2d>e8 zi$Zfs_m8%}i=N{4a3Tq@QfYW4u2hSC{4tS^gQ)81TTg0G8%8MSTe0NbTiNdtA7cy? zNBO5E_GWb%$CaOI(*TZuHG1Ip^97{lJ52@cz*aUD#))dw)V*42F8y`WL&X#yv?*82 z+>(`+8JJbx9>}y+u)ijY+Ag+kuJ7}Y&&FI@S>Bw|u8VMsbOPc?%yn(AM=T9!ahNJF z%p`kp!)!iDS~PDzU&WRf*qm+>R2{9vfJTQ)w-!m?vm*03)jsXXDVl81F|OS!cR<+fowY#IJ9cAy_ z`B_Z~?=A{U%35m)AZKE4fb>6DS|D^P2#9 zXD*!&n6zJvOTDT{NDj_3dq6QR6qN1!ffAdnXik~Hd=@b93)$bQevzh~W?@dCx#e;& zDN`aPgr%o?B`U{g0hnpqa!fYdEVxDSF4`z=sF!F6t98Wm??B;+7khAMX{!n1Z<8Ij z2Rla)%17AbmXLIicoS6po^+=BkIpo;(*6JJOf$~p(Lfy7dWu-z z7UPS3ooAhsrsv33G2G*Ig_1Z#^$>fw+@;E@HE#QKR>Yhleq3JdaLeq|O+{g(rU?oB zXGi+yE*Xlg2a3Sak6}s(Mx;vsrxCW@$Ru3qd<($unXVKIpo^EG;NRWorL$7jQqvh= z2QDvEfK<177eMaP%^&4~iftmSOlLPQ)&hkOZ=`h*7s24d*?5~}rYsH862|}0gqCWW zVtsr_Kfde6DCa_S(4H$XU|{Fm2Q(2$??j4Oj#wKFgg|(o$wi6DCB1hc;F-EJs)Z1O z^@W%_#N?R?G+yvKrSccULozykpKrG3qzWNnpuPcq{rElTr^BLI?JVBy@+T}zG93iu zPHlqDa6tW0$xqW-3PP|0A8ZS8kxeA7&RiHl1qzK(Q+TUNIoZpQH&y~cE1j9KFf^Av z5W2Te$XSzo^*MR4RS9ArT_S#c=Ma;xLR(`x=PVXSr+S_E83*jOxxi+jFW_xY=@}A5 zAvZ5CkbKQikRHeyF5_|NJhTvpfwEV`B7q6rb*scvS-g-r))+hQCuf>={k6?N&Hbe} z3wI4JsB|##fL9m)YAdLXmTMGP=R~uNGHA}lSZOe9?}*lwrUF7G#ttmpO?>taU*oe_ z!GIz|SOnrB_VJW1Q15_^dhV%)u^aU}DSiQG1CGV~Yzb;6oEK?{0*DLji~z8VK`K=- zJWjMncV@%&$E%4^A)s+IBqp>3iJ1T(3Y^=iUvA0(Kti7a5@3IwN+y<$(T4>m53z>I zcFrS+9PJEb8j#7_Dx1%*k8B^2y&x4@K3Y|+oiXN6ioPNbWj>?9%yFI9&g}EBC8bnI zx4VZ6HuLg;U+mK>?xcTR{{iMv(0i;iK|GN-FH<2SQ-+_7t}{8SOa9*p?bp?|E%b=JVx? zdeuJ0y+~6+EMR#YTZLtFHiy+P;YX_AwG)><~|-qy=;rC zhRz84>~6&Q^2I2fYCd|dB$}cWId8LVZC)!{D!8^2-$KvUjNv5LRM|WJl0=n1qOxCs z-=VQw7m_e3X^5jqA*^mjq_yV6`z1Y?PD*c8XY-*2`02%}?DbN~EB<5qP@D6|+RP|y zfs2A2W?(yxrmo0Sgn2zFP<;<(r(l0~%1q8CNE4#d^<=NgU~{T?FBX?H1x43rAWbty z9kzKmw6`r#Y~%-}f(y1iV-{0781}Bpga%S@_#PZB0&0h`_#gJ|gnOBTXvu)q{fL%l zxiL*!4h}X7X|f20dz0rwXNRpSu~{hbqx~^dLn!PNw0OkixF||K63VD4>4VZmzh`v9 zh|usDS<+n5ki)=0Hmr}nK!6>aL?pEk;=+9fL2`Nq`|^CJ#wHhg6J8B?9aX9DHUsAC z+02}=pXHnskQ^XumgZ|jvG&$-Wwaqul2o{9+7qXz;un0sT$ds)P1eT-FN0I{RaC@_ zXQS&hh=`wbq~yq?DpYxLYBKbHql+^4`fB*?x~q8KyS3b|F{MvJT+1nG^gbp@P>$UH zZq!JH{9I%k{Q-SglK!qALZxogid-sb%`4mvtZ6LHx(u|k>)042uMjcFHQs-N);FfM z^fcp2mpeHe!T@3P8+a_GCyH32G3SFRiTQ})-sX;6Ur6ul!;uXm0Gde6A*oFt_Ya=N zPaf}`v}$4|7}k4yG|A%0SL)@B&h!Wi^5!k8Dc@odSqIgsMA`S*^;2WAwHrqa5S2S{ zP-ic)3;s2{JDy(;P&jF>KhX_}F3J>LY^JZ$16Q$L?mWG>^g<7rrx|9ygiYLJe4E2m z)RcU!+F2BRhbjdECrh_0me#Nw;vPm(LCh|p2xe77wz72qfu2Wh-~Ze+ASsL_y<&G< zvOfYl7|xGFKb2#ZUwUqoEh*`B(Uz%2k^n3Lyk4y2lp?{ChP2xdF3 zPx~v@2&kb=y0->fnh0mI#bo)A1U|1~G0-qUF2s?Pk0u_e-9P|BUyO&|4E`)CF+}vj zGl&+r#1B2T-pry+nzpa-uQ7WYpd>JRfRMU_N?BTliH5ho~~8KzM4h3q zMU;u$L+2A#WeipgE0KzUUR`P|T4VQw5kaL7F{^3!d%Wh?_dgA+!tg%`i0^y|Ym?f@ z7#LGZKy;C?r!XZ|e0IQ4s#!J&$b9Y$Dg2 ziR0UcxSS z39!k07B-LG=b+dSqWPweNHs|q2?}!*#>ZbBtLHKR24r5m;U1P7@=UN;|2X@|_vybv# z5T*Zj#?sHr(g*-L<_;M101feh+G2Ns6;uQ}b;57CV4mD~KDTvv+Z$%=(Y^iMM&xHW z8=Ak~HWIM(Vjz?Gb5mOKq48eO4Qwm-ZvEgasGyb_l7d#!@hsg_8b*0;g{cgFuYJ4eK>{MZ&arr+4RE#r;5a zL^b zTojER!OWDn9)~hRopDZCo(VOvs7B57fjYk;d~YoqOiXa&6(Ln9Q0FUqbz}z{D)D1S z%g(hUo1X`;?4ohSoE~qiM{*JBhB$*GT9UoL8g^Kn==oQ4XXUw%2bvmntRebbN7$9g zEE{+LHRSkov?l1%Pk|qknmzDzy+AWj3#`J$%;4!<|4f`zYt3vq*jP@cj{vdS9PgZ? zXYOGMg&2kEc%i(XqTJJ!J&4$CD5s~u=9icFD_&Fl+C#`Z^9Bf>s8)_C=e$(0@C8XV zkXIpMAdi#nfHG}SIj{2FgKpJ|DA(5+Ax>fv(08Yljc{<&G6;&!HLHu&Bps@Ewi+kE zG4xD}`jFvz$uxdvANs5&;~W~66C=yZhUB}Z0vGdkWBkp^o;#xwh0b|Jq1fjed>mI^ zg_IpqLz-X<1-kZfFuBMpnoUDv`*>?jyY=@=5^mq&TAcM}n$D5UG1{G&^z0vVi#hgm zpXkQ0bqE6fgM~CV))Fz+ZeHR?=55?#I?}Z<=~cWZJV;_!jBBGR&6^(}E215AMx7?W z`b~V1Ia0m%nL63pS~&*6pfo*;rP2)|xdC6D8w&*k4wx^DsScwg9Zl^~Sp0_x&2<{j zVX9}-#}>ir1Rd@>;bE25dg6d;uC1g@&4$c9EU!Pt_yjyH7lPdq@p3b%4tDSd@p3o1 zpkl8Fc|M)q;Dq*&sii;jW-^Qjbf_t7@g<&o5O zsT#E}+r6JL>C9Otx^wj%2A{$nk@e?uq^iUd+6^xI^wbnJ(*mCb+-k3~{}+@0MB~FL zAAiyZI<)Yelk8eR6@e!j1Y?t9r!&0%Q-^hLp@Rl9V}&U5!{g{`rNOSehYisOtGdDc zue%3b|ELS4U$%KrngMi4z8)`U={6(Ir>t^yJQbpv9#@xF3XLjjt}+!*`&kmqbXrH3 zd=s_T`U@^v^6Zi*u!pTF&x7i_E>9rx8xKXpZ42d6pvKmlZqW;Q;nNWSKFhtKab#FL zyfXb0^_6~Aa#j#|8o9y}=1~u!$*4f7)wKI+x+jGxlXpzJ-2P)d?iKIkV-5r>isWL( zVL*V$(Lp)v3hWy_y>@PT9)s4CKzkUo*;h>#91{ZvhlA&~K(8}4VT^|}F`Fx+4!MC^ z%10PV>q?`ZFmHG_lpA$k>R}DQZr*PdJYfhXtKrXH0ltc(ADgEPb7%tu3Wck6A%`ga zdFJ4c&i!c%T2$}RKVdwnP|)0hUks`_%ePqY=kvbxO=W71wqos*8*h$m zz6xw7I!C*!H^WqXbrLPM%N2fCX2F%Y5>|%7>k@Ysgihj2#ZhE7ul0DX$OWz!X7RVJ zY0PsC`1*Vkx0Qwf-mAg}$3%R?&ddmyFGVkU-4uh`w{I`>l|#*ALoZQ6EmO z_nn6upU>yFejnH!+%Nn4=6v0C_-(cxE_zwL3`|wCI zHqOUI`F<6T{G#Ghi}}|5KzqT7EeCt2->_APD2J+{UH^Msm4i4G{{jop*5Tyuvn9l- zH7c)AvS0LB-wVeas-82*`M`1vFVL;mVL0PK2(Jx{GYL1)HrlM21G>tptX|R=7|BTE$O&?>GVv4tI0LZdw(l(M?}N z!~?sSTW1w|>9lfJN?6(2X-ZC_3ipFXp`U)>%HdDLL1OPfZ2`e`d=Gx2ji=g=Jo}Az z4=p-;1&XXLPj8Z=j*7xzOd9LY&fz-asYzJHZUHJ82VrWP&3JACSLHYH@M8zz{vatMNDKaP>09_dsX3jc_2WLN@$xOp231=x|<8 zSxhQ_?Pp@PR#v|pZ~5~&uuZ{Md3h}WH$$9?OwWfFFIgormPbM! zXw37uTw12kU)ZLiR$?6=8LcJgAp-Sp{lo+6TR-V@=(=_-no0hS-NXz8`i@>l+J=)f zRwQCT(W|XIL7lZi`W?+C^gU^Qkb zJ~l+L`Z_MUm5u#6s@*ynZbQ4>cipJo4lt{y(v~NEDiD)5c>V13Q~)uisx`cq2$EBi zsge?{cX!x3fh028sW*RqoS6EN=rVFR9L8W!Lcb9qrr6~dIFR%@oJb%Mb%)nVS!KO} z*4ru&0!|7pN*O*|?EvRuJPRu@*XB2;U%+OU2o&#`C)}w26eO={(F7QWP3|XY;RSeI z`bkKvA-U=4c0Jw;fT)a}%Q*%2L+T&%#7Oxc^8{gB<@BYt5kR=pH!%$Zon-I)c}vI+FT?PWwRDn%?XKT6V8$Vh;3p3j#lg6?Sc6&v zmyc-Pl}Q5?>gGSqaS5@1n&YGNW^y>(;s4V+DK`ykWgLmy{|Df+8Nz8&_No={@VRzG4ZM+!wXN8% zM>pTDXbC%DBjV{<|*TEK2N`902`;^EyLl%^E5YmVesv}nIDN8eDSu;vl+q}iLO022pAtSIlu6zHp{_cFm3`be){?CmjZgNV2 zG%L$n4WIVmga`@ENmlB8)_#jo3=3!-{x27OH3p{VrYkFWN->UT;wa*};eCOk^>~7rcR{Ezz{z3EqXkwXNG{zT$UAy%%&7(sEl>iFxm=DkL;=sXdH-dMY~z zD&H1b@Qv!4gS4t}X&8^eMWvhkgounD4TP@4#)3CkL0wGt&E~+-c1z~D8Z3ofPN!s2 zn`PJkU)+Z>9G^W6!Tw}#uXt7x*eFv=|(B~LHlD8kS#x% zk1LH{p<~9A8_!kpNuaIA2`FVt=suW-R(V9cCL!c)XFk^|(v4saEXUElALPZ2G#?SGTgy%BHGp?P~jC~%m_4*jXcH^vlDSEC7myi5=}iE9wXem5d|X3Go&?jp|Rnvg8o z7K5A_20PMrWA|v^2Rn3R3XAZ4CwZEIh8*aQzhw(1^Nwza{7jv z1s~bXHVgBMr8+Ez$kg(}ctyi(;U^wTpO*tQ_v3cSXyp75)Tc=6SUcKSJg#tk>jA&o z`My&Xv-dPM9MkL&SCFph^2n)->5BG*GGtJ`ncttB*TiEVkFMjE`D-MZ+Mm=mD(!8} zal-BICvN@*|C|bi<_%hO#%iok9}p*>bqwf0^{Z&LlGuI%-(Q?Q=B?tT-yh z^9Rz}(3Vn-^k?AF;SBQ*exOOB2L5y!QyFgwRiK_BI%9lE6Cbc2G;G19BiUq5i`W{A z^(_5tk%rs2+iuK)DNigkGB#+)yH+0!?l-MfXICqSQGSq*NEDSK<{11rpn8x~cpIzG zy11N`r|}e@ldnU>U=c5{;T~EtotMiZ@OECqXTp#~4nCalTi-)uj}fQ20d_)xLxNU+mi@@u=ID=KwP&47a9S$(Obh)$OPh#8CdBYAYUu zwroz9zMj}|A2P*n9uZ%GTbic-8#}s5I>q@0t9*i=Z0Mf1l*h*a><|a78D+ST%e>`9 zHrWS*NQZBi?YL(({K}O=dmQpSRMj+rviBlkq2i|7_JU z(+9;h9T_x7O`2wHErGYB`pm;NGPDXYan4yC%t*Og?@(|_hZU4ac62Q3MM>6K*fo4aVm@d+M7~C z#0m7=U!%Or-x7VhLD-kEVQMqzzSbrn&fDwW{cVYmXrMT%&`kY_v}+Nw)Rh!(X&`cF z@+4G7ms?@=VVy-A2wv}7s7CR=P9w|V8#<+ z9>6{H7Z`33l(`U1ATWB-MdmX09sYELJ8*m<5rlAbk4f^4zGl4@$|d`*=VS*tDKYpW zhIw`&LaudE1bx>nbEbej_NEJ2|Qx8B!!2;OuorHRB- zF9Tj{)HGC{c#R_~jpJT8y1WdD3)p8iyTXuW3vTD6G{=j1ue@t5)N%0a>9nBT_3fBnw4za4`Z({0J^^{@_3P2{Zd~IX4vaNyoBPG{fBX2bjLLYj77F&W* z|ADc9kFaGIRJJ7IGPZ)%^hVG3{Q>t6Tpvh4$jaMhw$<=YhDAH|q|V;qa}rhFect^nY2*jKopfZnXg4(k%?Z!<-hoF9 zmp7PpuVC$g`so@?kADuGr+NBqvh|y%Yu?WPI_;%{`mUF)|MX(|_sO#?ui%Zb@4}kt z3itgvCc)U|%WPN*; ze=dl+&g@t`#<(I3@i3Fah3+MpC4rRBy`k!QwiX3$8qzuwinKTf-|=_wRDdI8H^1;a zoSd`;4H93tdX=0V=zy)=-Tw4hqWO$;Qj>04kp{YP`8fH5qvXbpI1pq7YEN#N>6DIx zv+<-KT0#?o&Fyow6EwOj@Fj~gJ~`bn2_E+WivzL?!Are5h<-2thwdGDAg@ur?(Q4* zBA^ze|2QZUG%*=O4p*7vhEv_;%D%lY znAso=O?67@M9{+gpd|~Y8Y7a8C_1k+Hc5U;grWg~s{lmW>IaqMPS%~%31EoAfB+U? zFy|<`_(MzsZFa&9Co_ziL$d?%LBTO|IdFIV_r5 z{%r5K6QA`Q7DjPu)V4nc8a?*#^;#1xL@-l?E?&1hO0HrOgyD+vs%Xd{i??8*6Tj6; zFoc3ZSZBH13NIxi$Mqh-LOGwL3>@)vE9TZz#*$M}%(s31)-uK@5cLza%^5(68`a@R z|I#wlQz0a9mzbiCg?L+>ejDe*0A`?9jlwP&1zY_8*^W4-;gRPeQib#O?L^`WYHTUF z;bzOSo74m|$lrFJFXQ8NDSKSvQUEnY=4hl3jK^f;A|CE~*^^P`_@g2>{69PmyT3h+ zyWM|z8e?glY*H~g4hbv$GNdB%1q@Q>7w?q+Xd2%CUDE*nOVgm6AL3}Gg&?;s_HJ|_ zS`^GY>>YPX57cQ#sjk+MVw9C}4di_~vHImNM&%o}n6}s9m27ZaUktdk)eG_DfEk#UE&uIYXkf8_N@@djhDe)o0$z?!t zpH;SFdc;+hx$f*978?39LZ^@qWcpeA`~pWRHVc{>$228zHmB3+2^4I7l?MA^`6qp` zY=b!(`7g;7o!7g6Rbt5;4n`&!LqDcdoS z(rvKh|3LSt^!MA;xf4X;-s4xy(CQg#+wn0USg3_C6_B^0W74+#qY+@7XiCw?q$|pr))}6h+)RNb}+5LG!bKsVLv-`U$0QUl*f1&$-6oV{i zP+nhUeFO%`E=dxStk&{71`dAxgiV+%^sBog!gZM_+@C&j6Z!`Dvs9UwTO#-HpJ>^I zsKsZY%1|}JiJs(hLo)6*_+P5uU{~qNI4=%eLs^3t&iI=yWddw5&{gb79}Mzb%X&lo z72x?d4rth!ip@X&f%u=kA%0(YnC^y1JywX^*c?ZVSaSAcP!`Odrv9yB_y%ZbJU8Om zjffHq#MLC`9Bqh08G7TALxRsd8!f`QoWRn!l3oGXCPCnIgxi$6F{E|n@G4v;o;ZkE z>Q7`sSKbp9;xr|UwXon1FakhPzZ`)d^W z-R}>GMgm!xQQmEzM-Q^aw!Au8B0ev1lm=X;e%e-*#+>0lUzqkCAa;cGy!D8?!?VN* zd+*bbu}KZ==QG06!eh6QHH0ZkT1kyc+0nSU8*yE4$B9s@{W*Zdh44R=x=A+ARszzo zYTgu*cUdo@mY9Yw%iR1Zw~1vnRT^?p+tzIW%7<-gAjM&~ z&>@&RwstIXyVhJSHNvnj?Q>$HALdS2O>{86gSj%D=D ziwwe+y9yQ2$u?~I-d=dz+*(T_g2$X-M}DzLID|v(%J`$ zN4Y%I4}Ne*R6Ek?aW$E(CIR~}Os`H}H$KR+o<8-1*#vjL+_8rsPVy5vk5vl}?>_2$ z?8mdH?mJI9oEc)uSzwywP?;i6iMaLm8Kd89AtiGm{qM8sgzc571)jopgHw3QMdaM- zrR<1xHy>hb?pgfNaNWmmSA>vUd~G2T1MQ73WM({HK)l0YT1fHcPB%s&DSD*sF5c)$ z=gA{&HY5m^w_b3TOyjmGQ9+VPc)}Ma=*AY*Hsj`g#iVSVJEj6QUYcBr$RXEG<7n5>EpzDJwds(&^eu;qoVGM$TbrvTOC8b zH;Tc|)+xToT(fA5t_-NSx!$0HxLT6qr67d|D;8xm>jS%d=f0H}8(JwJT|2XqudJ1Mj(!~iw_zHI{~hmGIg84Q<157B$Q_2iRAXNbZ{v4mkimLGtX$Ns_1oqoG{nhOa_` zDs0I`3_;m_f+89?s5|c;hi}5%m;axO>xtQRGi7>`)TZJ19i}=r0;BR zF+FRYlq;PT3`V?SZW0x^EB%iFRD*f5w%S=%I{c`WYyxUmLlWG*rZ;McC zNlD@wHvT5!Zmoa>Zh2u4b_O#}=HNHC&9DbS6oPT$v?!n`l;Ru*Z9I4g6fcLGs_!3< zfz{J%KXq7_p{45eMP=FiJngEcPH!qOv}`6@d?fb^Y|M0nH)fSc7Y&3|T|?&lgi7nW zd^UhOq<6hbWT8yB9GO!Pi><5(D+KBi3qluUuPg19^Z+<(>hEuj6b^;jIYS3vma<_e$S!$ zr6$jamoaZ(sq}@{2!Pns*aja2_sN<$h|C(l z=!Aw>n#*3p)RB9_XNPuPSiJI#0b#d@Fn_f!YZJcQODx}fp1KmRRk|;)^4s?7m()a| zbv8FKF2vrLb8%D!I-Qt;az0Iek*9R(m11{U-lR$)EH30QR!lb^3pxsxI1RQ$TiIk+ zJ-*)F-!?Peg*So_XG}%e03fPNw*d6wp$okFO@#!vzSC-T5TS&Uu%ZIQna6WSu2h9) ztshbM-_iIt=ndc9#Q#9!eMhMROtmN*q(?O!?bG(z6^MUe&;5nQYj~8P-rp}Dwy`W8 z)dmXq{RbMK*t;+m`xhFoSta|0)sw$grarBaH}|+R3zb-eWZca`tW-&4Y~we^+wr~| zuQa*&Zu)EH`@%e^f?op_k_ccckZr8!n3s3W=F z?Xm88&O8Y+%PZ^gVmZ#nU!f-48V>CaOL9yfPSpkUc7CdG{D)*#mrBa6oOhUx{cu+y1fY0mb@>LLUMy~&HqlqTcE$>VEzXQ?|PQ@FB0Cs?r##_NtEd? z5f!b-)^X-Je}wDGG@OqdIHJA+i;_ zzA;oLMy|cNmS)rw0L_?0#)`$PMTO`2#Ax9>J>BG*Aq-_?Rh!=(;H=g@N8UKrekAtY zcluO2{+`LP`%$YlxrmeQRfFFdw2fwrb47-#M!(j|U$eIGdE^6g1;^IMeuAP}9Z^Zk zMyg_Wx?kwL-gf3}A0^(Ze=Ib?M{|~4;L5C)T~)BaoquDBg7DDZz!z`%;p!)SHQxa% z%ClN`yDal4PL3=LTRLH7X;_qT%NQ862urpBf5)uAaeN*)%VaS*?&^ok^y;W?)>c_n zTh5YRub+*Xx805_yun<<*Kuz`YQ@abD1(xc$(ch3=gJsfi)JD!q}AKDOUx)eInAjo z_POOw1epNWeXpLorYOg21(<2ubV@Vdb05ThDcC4#tCx!isd2*(=||y#BFznD=a##qyQ_;?{|a59p1S--+x-bTKH3{^28R~ z&&FwDWKmaBQNjvK0(89j~CtxCk^!9;J$4LbMKe9AiI#Ma`@6UjkP^ zCQc(^F$|bUpfszCHw&`7VNwR~)dB3*Cs#qWD9(PaJ!5XnR16dE7V~jo=;)Fe;xyFc0qN|tM`fC91n?z z_na$8i6=N6d@l4W<7m_XF%%OZJWG1PBL`Aa6sVBeD${vv=B3zn&qeJ!Zx_?z=fClG zjQD@>c7DMB7rebb_Y&68a2~hPZ4^Vs@yTYN}8Gk6#kg zs3YAHpw@*QF`!D+#jTUVd)pPAkAJgG9=f2oDhw`1wj>4$1%+7ZPZpLpn3>~t1FMNB z)kTatB#S~mk^eg2e-iFm(t77?5|e%Hi2ZZzZ-4H4ynirZEcl;5#TR92;-NYpnaxOI zohm5NFBedE)4KZAX z0aCRgIPU~|O{?p^HSh2xYA)#@rR~K&UcmjG;3gpw+j}z!)yV&&Gu~8=zLLQT{WG2} z8J{W5@-Om!W3vlci5zGbX{11(l3MZn-jO74)0UL_SGNkJx_KSq5J)5_F)~wb3>@HZ z5MV*6-+=g`g+xHX#v2$mZuS>n{ZX>NlTg@7)Bpovo&1YpD)b`Xc)LEQdGl`s&KB)R zeyFxMz~+V3lJVrz_Tf}uzcKPti{GYeZY)R zI_s5r)$-|k)9 zB7GJov${$be~EJ+ouDoHhSgz37UO4TweHsq?O_c8&5W*&U{NYIIv`|O!jhk*mU6AD zo!dZHWL0EC0AXR1i$|XIMtZtW-=u6J(V~$agL7y58Nk8{Fy-7{eND5i*OdF>(mr+S zJOR!2a{AzAb0%j;9+Fe6{a`qoK*U@Zp6?b7noJmmJ5bJZul#+^0JBZnvm3??GQ=-a zFdJ~t;k3uF=V59u5A-D_^m3opE0DXH&K_!G;IH6NAAxPlL4uo!eQ%lVJXyCw9Y_Sr z8}HtTP*cqJHdfaOQP}p|pee5Kr(xiG5rtCQKUeyyS;{F(`~zm5{j!zx8y86P^T6EMo6{v{Afm}jyPruk)PJr|_(}EFn5Zh-fObjvx4F-Z!S;>-yj=cKo7g+#b5}R2nWRQlAO8Q}($m!v) zh|Mh?;s^+hxyuydwChT>(Lw{j}2QqC2vGy;? zd^oQudtPXnUZ_DsOt!<=IV2txNPw$sxsO2sIHxAe3|t0F#Iov3p_Db%&Tl((7bFzqEuE7ipcGm)G|Aq3Ni(7A9E2?o`B zgH;%eCa4W4uz@;)*59lEO5O;*U6PCsJ$dKQjG{-sWb|+)6oyK{OW3`U>h!1Un22{U zFqN&i+kUpWJItz-3<9s=G!55}%D=8;zWNs&6M9W1=c47;7bj$IslVxVzw!~R{=oTp zUmJ8u7Y|4(5J*YCsbIS#*y`W9`w9v8MBbwau-4aBpem))=Sx zYCk_P1`8H_OwEdA7d{(+#LaAQCT*j`^6A>+J;8Zu!BJeTG+dbr_1=Y-g5O z?9Iyl+)=tWHT)<=WqSsqC07Nrf$UsVqk=8?HVl2^w7fuodD*|ehkIH^lvn%f?zK2N zd(-=AN8SN2oJz3HYQ`vmv6RNgR*_)3~oLzG=?qUM!qh*(q#i zJky;O6=dsHCD&b(p5Qy0Lg_3#bOMr*4B-QGqqOE7%B!Nb#v`gyI)_xeIS~5HxAq6) z^|;6kd*w)i8|rU#sA}&}4An^(-1Ob~rY691Ukp<%^yv=$)p|_!ed_f7-LT0NqIk>X z5B+$atn(V)pM%nzXUjZIhbB}L1en}mbedO|#}zPc>74a0^#(>cg2PeRn76+S50B*! z7MNc~O%8b*;gGu8rzg)Tgq?)l0tC@yz>-Svf$Ek7;(P-m$KLp7ovmx;j!^Jn4yqB9 zkvcm&MErkfB}8K?ebE`%OCN&)llu8pYXZfKPWpx~zVgz^Vbh(w1Z)wFom-LpWCRqJ?p25)w}yu#BJ!iffIH0NXN!k_!>?Ngf2MNLwh)dstM)Xakw^b%Q8Qz`yEPWD(a^i$qiw zV^N@y#ZlYJ-v=UIT$jS~tUO?{k8S>Z;ySuSMC10e*bXv-D@XA9QSOda)^SoJAr7sb zzywoUAZMyzD+~F#n>6Z~PN~x}{ zdg+Vu?WChA1q>t{Ukb(5THn-{mr%NH>kadFXbS~30a5UB3o-pW~fnP9l_VpxxM0BO|~ zzt#K2UAabwAGqS!wAk|7HP^d#7In;Q7~{G?bq(CpSv^kY)t0am2-;GLVyX!6!#+^N zqqpQ#-5g;lq|Q0V(M}#H8JIa|RmW0pkU}rA=bV0+D#eLQRQR;4u5DF) zD6W2lFA%yCK=fP%j-$S?na;eGWESIxq?fTl(>kr4W@wwJfF4StXqqL)M|KjPmSZI` z(ITy`&TmXwzPk|d=k4Q&hCdawZ-9Id<~_b#e7LU(Bf#8ImV(t5ydLX3fV+U3Vg713 zwJr}Os>R}0DAovc$b<8eK;meXzM1;#rO~vrZhj;ht+fzw7@1R{N>iM_fKIAO0720h zx401y0cRz^`Acs;_C_c4UNnK&40}eJT$cab?vCTX=6LEyiOS`6@#05AYKY<2Zeua^ zI*71KZlE;zdd@<5ZK*)l;EX9);sZe{+`a7GJnjR0m^f*9#6+r%xF&KyZo#YJ3i-^P|?AdS7*4R<4NN+u<-#FJz8Ht_k-Uaf^7IS>Qw-zHx zTY-J3hP0>i^)U}Lw%vC_)>!iMhxe8xzQSZ?{(N0CykgoYJ%R=A$ya@GgY`Di!&O((0EXr^_l)ZvCu`=AFQEpqX?a^6wiy4$NEZV zk=t$MNg2>0s`8-ZUe$f9aI>wK;6}pFjZn{NLmLvXF?fnpV_`zK0oT7}0qAT9+bKu5 z6D3wdd+FeXeo&VFNNU!gF-Li98GynCuiG1N(~7b-UpKUxRwnI}@itZ0jjsKoEW6Mq zBY9ivWh45bIfZC;q;2l{S4fz{B6+cn2x-E#Htp^dt!>pO`>T_^dwsH#r+D+^z~Q9 zwy~_3l(yz;;-Y1ingW1|_I2d6wOUL)EH5ZyyxdM^3&9SFm>HR*+v^yEm>C=0AhBly z-e!w`s~_vX;CyQ@&9Dm!4lz6U+{iVAyD9l|C!s^(g}Yg*scOUpo;OWjfNIS z@9V~|bCw@#J}#Ik+BO`#Qfoe0jAm;-ENp;qKkIksyP8T~|K(*okcfeS$JeE)9qtz( z7N?Ll&plF$ml|YhGkM12d42(s*Zi?gpc-pC8}}#B=5D>k5-r4+m;OOh+b8Hn1NMPS zZT0rQ85seIP@j0U<9zhfZsIh#tk?O8qUw0(D*lKq^ykbc&auB?43}v{U<|dnW8L<}bF}W{ zXo;^33JA(`Z!5mw4?(>8-k0uD)qIJ6p4ANY;Q0UKVC1#>d^;FOJ?G`qjHP``coEd1 z4et-qtos}fSaHL+OZi?nSWm8{8s!)5;|hS?3wR}m{fA~zLGbxadtqbhWoC5Kg9QOM zPn2HnxAT`C^M=}|%0>QlNyw$5D@%i>QcDsLFP?SwXe6E%YurZ3-6cldxZ>kc{%ac%c&PI#+Bn!K)ourh0;a7T zqc2IMGZwu4$3O1HGsQ;0pIyb>QGPt;IZ4gxH}56AWsel?t_Jeh#O6F^mCc`BwmA%C zyj|>=O4CR8B-}&`dWE5STRC4m@5@&{>^!e)J3H{z(O-ANTezQ=5MR&H-_evGyWYVqG{Nydz$buKnXOrqBHL>n?DiXN6jI=I<0~HqT&h!$2U<7m zioP&>!>jbGEf_8&zYmBG$X}M_`(|j^1bsuxzQ;y6pSN0DT z|BU+JSnb8;uLPQ9)VNatIK(di9yvo;(hc(`?$sA%(NBR8sfbM1x2TTEZK#B?#OA1* zE;|b;LrDG~zV0zPlD1voeaxBIwr$(CZ6^~>CY*3$O+2wVv2As1+qQE$&-=W4?{&^v z=Sz20-&OgRm8|>ffBlBx!+qi`3^|tnlVQaBpVDx=!2haR)r-P>F66az@jA(=VTa|) z|Nn0=3hcsvD7K8BF*i!P4~EyAwYKH6NOm$$=twQtxUeq+>XwQo&9Y=8d?>!&Th}cl z=QLZ`RCwZ?%vTkqTNQgQTPF=XqAi<{4h5}WkQk0C!;y-g6X&kMOe1OVG{9j(K*T2- zn%n&O_RG%wErr+F@1fl5-ecnZ+>_ZuF{x;m)8*=PkvUeQxnN*MDje_?E-=+XsKFr^ z-#NH~eR>+PKP$$K)&b)8mIdmNQyFdq+#E1xnCv_1bm<>zy2M*nrf00K@sGOr-nW?? zpqbcFPWwZ4Qg>R-TUj^lCyG98`A~jEWmtGX1jR7igr?mhcw;@ge+yQzkFu#1Twce0 zSa*Sk&e~QmUbZX?rHMZSMY3L0WE{amCKKyo+m|WW=fa+FsbI8BFs$l4+WpU>W6nKd zJ%VzRE@Yn1^a@oqk3@|-bD*NS4f|PA)Yftl;q8hw=(Td-D_0_wXuwbitMrfC41LL* z9?{j3VZC~_irOs|PUC(jxIWHr{ z^DxXjMl?={krQtneOO)?HsLl`)G0syHKMK+kcO3ZTal*N;WVs4tG7GptC+VgIZH6L zM5%%!V=e`wf&5Lje+TTCde%YU#8u?8zgxTYq}5Y6gww5Z-tR3<><27QPefO-nepgv zvdG7y?0Z6gG9?(sc4m4*rv`BnRtAeIDAxpPD`4ZAuBvY_B_9(x8#$sg@NV}J_llRo zb;0QlH&v8L1&4w9nn$RkJ*6no6!hp(CSl`0o=>li;N_)^+~Kq2Nw+ZF3!7?02mCPf z{BLb6`%o)hw1qd`RVvga3FB7L7aES>)e5<}1$YJYbs|max_-rbm_6;i5}`5D7Bh|2 zm})BVHB(A#nq-h+)$g_93eDA|?XMnNU6wq>SK@j}{=xh3<|C4`EZS8LH~l)pO5&k} zPt;7<0UUv4_n|PrV#rbN6A6Xc+5JQ)RsGaSuET88iEPPc%YsQxA^nUjSta24Ma9Zu zO&%3BWH--I?-_xeNS}C*WgwMLnmnoi?lNbDz7*SgB;QAf_y)kNMCNd94ZtbqadMu5 zILw*sF}&qHDNkOa#UJO^j5x)WKx2Jr2JegvPdPn(m5W855WCuP{!LfH#WWC`y4T7XL-S$=rLKKy){sqRG=A)qiLcqz=GJNr7w}*omxy|1~=grTb%WG9ngF&bP&H5MFmTEO?4I zOx8{uWJaqbWO9Y!jxvRbdO@yp$Fi=0I#L`PU|F~MVa9DuLrwRKU^Xn0h9k%+Lg-r_ z7z0-=Z~bp4=ob{u#Zh1V6)IYyR4XR$q+FUUvxh$xhvV-DIfVI9`Q1$|f+w8Q>nTl` zIn;NuqWb$cT{PHTpua-I7^{EH(LT7pXs~=bB?iqB>E<~0zb)%ZHU}0><;7H^wb!n( z=6-krytJk!77qMc-`>Hfm{koM z=U$13FlCZ%(islu>u@yQYjr%7^r#D&&aP2gmH=BDGt}oUieHRB&X+UQAWhXBZLjBF zTSR%}uM466RZ1Gil{9JA>#7iYRjTE}y;;EzFgCXEtPF7OnoKUm2gGDDK(<-wgnxhB zFSIv&e`@$t*J6at9Ypzu|yrnZaCcOhd2(U*gsu~yxx(-lsO0_Xy zDgT0)xxn$`N&7J@dWGaQ=c6c;(yQWAD7nj)@!kDLC}};v%*9h1{;EElTnbA5kzcx0 zku*i4da=pHaU*BtJw{Pu5m9|b6k-n=535vnlTLdvot(kNKD}Q;k=_gvYb29VQdLJj z^j2tbh5T4*Q6A)pT^9*z!6mmDDrbo`(Uexgf4n~KJfo*l?A+&bOLzkCEg|aJ@E2VmoCjF;L776@SyMW4$DFrf=l#w;93%4 zfw|jg5$tavACKJ%$6=OtDqVG%t3`}Kgd{~$Pu6oaDL2O!WSA~ z%R|gQWz!ze%FI0vitc8V?2weo8b`cu9{KKLaQ9hNnC* z8K`-A;Hd2AO(&cCat7rbNj`sfqi8Hw?z>sN`*^<8fj)2fB<=V&9p81??4EdIZI`06 zU*X4JD{Q?=|5!eC_B707%r;)N(h~6f_we-?9{#J}j!5)`OetGgl(V&C zlCg!(t_cs#%}gH(KyyCFB4={~OVQL~_jfsFITh>S?~3^y5}Ep*1iH&zTMFb+j2Kgw z&Ft?FPAnl6F{`q7(hkmt0$(lcZNJwW$Y?137bu|;s6bpV4=ruMu-sQR4U6u!WVY}@ zGJTse@rih{&wG5)Kr&%O!EcaWd+;r;iXrLY5aBzkUbkV%EJI8xql`X5#mPEatkB>Ip?MlQR+1%@_(gUoc|O`*c?6IM*bB_%4R7)3zBl6P{OE?mU~hz0qriR zcy|)_x9=Ti`9U%{{Erq?P;zJV@L^kJ_vV({S~1PK7Oa$7m(r^KYM1z>3BK?I$jqw$ z%vji&|1?5NjSt6kq6Z@1i^pW5VbU_@A0AgV02~3(-D>5@i3tGh#lmT_sh0P=p#$^}8->U_N5Hg+N3pfb zF5$!<3d~2Q$dY5W+4i!HhGOHHXX?avc(n^*P+j#>YC)=cS`*==C?bywEzD%xfcZ!c z?)DFF?{p{gw|CTOnEu?k0)bR&el!^amZd{QJ1bHhPA1sz!)S5x0UP;wrz`(V&sY8x zN=hc6U`G+4m+TR+{eePBrQ^r_+EW%oXN2ycY#KnshF_R!NI5o)^|=d%JZkg9if(zk zuTxNk<}Top;%H35N&vWIg z#@HG4xgk-QNy+cfbs>jdc7lX)F3f|1xt~G_d?g$Be}od}TN{WE4>7CEg@I3@#10uK zl%S+xd+R|Ykf%URTFt_*kE{Z1Jt8D$kv-|7G#idRY-(+O^ZZ<%^f(lm>IFAdV=yZP z_9)vUTkV8tSXNS{$PkZKoSPY$rc1$Qm9MIne+}QxBR0fz_}dXAM0PtO#GPbeQW_PK zD)bYy2!o&arC|H}>!cb8XMFIq7wUW)?~C2j1jg?pIf%@9Up zg@|L{dze=@w53JS?>2Y{b&nzchbW;z9hf$yr^uUO1GlZ>L*39W0&UIMDF#CL zcPkNfTAo&{1Z(=McxO&&Gi?>Ul?F&pxQ(&jlKj#_Ilf}mfB8jY&E95tL)A{@b1QA? zhftwDZFBb5R>I|7dPYN^!v8}0<8@9&pb8Iw`Jh2@E#sWWLm00kc#jvavmh9nKQF3c ztAY!B4?c*U{UK$ee+OJZl&%1N5+eY7**h>N_C=7cAoK&i0MGBa$?6&Sfd`^0CoU7j zK-$C-c%r`{1fA{s09+%8*Lic1vf(GIe~0g}1pXk}OyHHoC3ms2&#Rvcha0o7nV6z@SCwW~dTxNWd@K6P#In05Ek9bWf zc5HlNjRFY$y>oB?Es<~F&WxYIhH0QfP2`)3dOu!b1;?jR;_;7BqURBD63y*Jf9GY3 z(3ag7?UXS1k+0v?gzMBfj4%CDm=ly&)}}Moe9mef8PFtA?B<+EGjg9W6D!TUrD{u% zk*GFFjr+WW(=hzg61k#CW}`ftkHBbJlS5C4Hnwo12Nd-5Y>@jhn=87tSi?d*-s;S`=4+71lzGIh0{J_O)EY83we;N%6rx3H zPdxlMPpk9z8HXP*n|-m08~%zDPKx677WI$euMfkDozI@%p9d2X2o1k}oc!(Nd9sH1 zxEh@i%zQt6zbc!V0j_+!-lw;wk9Jl2sG>hctqU4n@!0TfqyeMQ=u=Iu+fQ=g*nAq!574-ItXl0uGACr^rw zzXSI>rJ5T?h%J*|Aq>F>_sRCMXSps;%>T|67%RiA>R~02})M) zmZ4z7Th6IP?oQ^g@%ocfFrLXW;Ji}er_CLIyr*yjcE40{(JSH2>{~_=pg3vSISkvk zNAP`plEh5F*^KwwD3rSs=QNBm+Xgg8K%Rk+7dkQp(NADG<%q^YuK?f$Sh5%o>)Cm-r-};UOTLuaw=z z8AKMRMgbIX99KQfszVqIk$n)HIc&jWtI2K;7dDN=&zX?-XQ{P{`#16~iRt%FsSxPf zIEjXOv{Df(yvkH?F7W(-J{mFqJuB};I(>^C9R$m z=_HaA1sXqonu$QbamdZY_>C1ajHr; z|B5zv61XmjW&i$4AvWZYzD8O*%#ls8N3c;L=m)jo*LN@LV`rd5qmE@e+P1Ozc% z<$o@Z^iZ8FMp?N}yu&m@|KZ%n1*?)+V{ZFDqLt^S5 zqQqx`fujnp%I~hKh&mA+$$jEh=eoe7msI8g6rI%xW(A z8MBzM4rzcOwk8iC{N7!7s$!0I7pYHrIG$yVnbeUKd>z8-)+}oDO@SIy36es1uQrNG zPGegqE_~i1RbK^#hjLH_BgwKcrSWa^%82YTbGQ@s{*iO0lq4ZQLD#al8Gw=4u-jH_ zNr@n3f$QSQKizD@+9Q&s+cxRr0|y;Vv%M1TGLUu?4G+*L*?3w_WGn5N2uHr29M>F$cQ%Y!YnW)kpJL05WTiQQk%V1mSp)jA5a?n7(B?wI>JT_8eoj^< zYw-YG4UZk8ABx-Wj|4Ud+(*;B>kqOCq#%rbsfJZ`6C$&t2-{4lS}H8r?I+gkSM6jUtd^Deby8m9>r-S!ZO+VPU7<*$(yxN3d&Y0z(`|m-kvFS&hfd=t>vhza zr@^)yL&)Tu7Y={lvw%p+8W1Ug$(s248;p3L>dxTyp<#R6cuDZDxgy5%y6ZYO<j`e7d;*KKV$&4_)mmdvQ*Ija4SEiDb_3X8Op+F1ktXxtrA6wbNPF7}@!Da9O=^2$ z)S;wJQWdzjD4C^*jjcz(IxXd;X29d7|3s||C1o*)M|b^2V6{S#*jkIS?000ZgGQ$< z=>W+V3${Tjra4x!2HKJqOK#s-soHI7b&zkH7U$z~Nu#|G3#calA|)rENJ-HXukzT2 zY)R*RaH)GctKWZ-5;%RFo}PeJm=Vi@@UN^tKWJ3)9QvA{3zOvQ;;X|dE&+46NUKri zxhO4}qIaEpEU&}o^N2~kw7}-cRXC`vz(oX}y!}O_Je;nnD){L}>v&0}jRXVPHgoR* z5e%%1BXg`@g%yXzye!R}&XS}6Zz|J$*Y`2m2-8s3xDQbIaFzIQwzvfELEm&#&YKCQ zi3O7`QPAaENr6PzC4YHfF$PzQ)kcLkgpg;{mai&Ne4lP0+Y@AC(^?s~X_5jT>eZnC z;dB?$MnYM50i%?vWB{_30xkdyp8AA~5<*}O;)ig=kNjbz)e-<;L^%)9+J#&B5bb^u zrmMi(Nr?`JxIV1uqe}D}Vd>@mcQ;C`35j7gKuO^}j*}tRX>R|4;W3NSDKA4Br5Wj3 z1pMtiiVj(-E)4eLZN@_l^!rU|tv&RF7B>+&^>r)A_pETeB-soW3oKb`DZ)NVMCh#< zUO(dCoJPt^UMc_05l8s3UsQ)F{d;@O(A?3nQ4WC>Ysmqk4oD9^jLck$Z)7P8KJG46 z%4|2|A%-j5AcKGd4ruuRC{c`qu42J8ZV<@ro3ubP%xV*DicuTIloVcwFKHO2SBz_n zGyfnZg*X@ZlDDrP@RhPi-y=MV?dh>?f!#FY<3f|M0ZSLf=3(Au+ljxS0CY-}GOTfC zioe=s`0-L#VFVu&rQrG0x?sr77SASvWCVu@LuZr{cUdggW5$Vx2XO)+x{KMMR5n7_ z!HvWT-S~3@Fm>uQjG9|o`zLkE{Y;&ZM`~Zk30Oatc09{BH}be&z#9(j%Ey@fiYN06 z#HVO;8%GM=ih2^D8CHDPc#x?pDyk&~8F_1&j*l7K{H;q4#ewZe^M(f9N;1{^{@Iyl z<7ecgNEO(LnwG3befhQaN^ayOpF%<7_W|N0PPpP+D!+p$D{@g#M9)E|KPhC*)6Ot% zPj}o{48lCRQ3FAB1D`jMWpGCI4Xuag+zi$grp9|}H4%e=^|}bz^$53Ey2(Sqx-u^y z<=F7JT~gd<#cW8en=*s$(P184EF(HcF8%H3o8kJ(O)e9vBE7r_$OtHY^C;!dFY*!* zU%fI69G)STzC@zcrDj6!3lJSdda}kQiy_2I3bu>&}*}{(UsZ z_0sEBnzbqN=BH7TH?<%i14u=gBsNyE%u`Ko5+v?2U13Yko568Mza*wb>gI)ijeXpl zz|ddcRc&s6*!n5+{9}l|zE*D`wlkbj^3Kt$L(F#Xm}W2;p*byRmP15GmWRr1EK_UV zDoNL!`vh0iqTJzRioS)9MOQ-ctIoPxnS@tIF6a|stnSD^MhWNpLA5Akm###x>P(Il zQ_H_b39HIfTGGCTYM;VGiahuJ|20Z%Rj!Y|On~PoeMEElixFEZ2(b;N@lCg|HBtjC z{2Q_ZcB8mKA*AN_MGHzS{%e#VLh?1})6>qGFzGfXi~o4)s|mhtHiwm?n*KCO`n~3< z&P)lG9L`Uo+pB7OG9Uv!*f^w_R4WIBEio>K(!ZMAvr@>8E-%dg(_#G@&!64heLQ`1 zwcd_V|5e(zo9zw~o~+IsDnMv@lrPqT9;$eajz1fT-tSM9XQ4c(7M_*Hc+@gH`Pa$$ z_2$Z-DI+6Zk~*6Ghtn|rB7(HfWz@Jic!AtEY`zO8iKyk(?DNp4ta2%;HS5e#%1Qua zF)kCkVa~v?KOlxCcH*D8Rk~LNW|;!bG|Wr#ZwfCUQG%PVw)k9|RkzrB*1N28@m<#A zlPKxB?kHIA^!_AD+~zLspZ4DX;s^IX5j%ntT$oLl-d^g`NPPY=N-{uKhjdW4M*Y(= zn1h+CpMMrnzqb}ouO-u#5Guxh?%(!Ee1zlmSCh3G-gOQ;Rwra^^SSTYeF*C^SnUIu z=^v_v+-hj1X2Az3cg0NI9m-z}5TR{-1I zz0W!Go$alyPa#K2zhNI^yz9|W;*KS^7_T!XZ?Tc!X#k>>?TV{ zw_@=$IZ3ZEm%7kp2mAlX zkOAYYKY;?f(UlyS4l+(hG%B@liNH7uzpPZ263P;YloXH!0+EuWMIci0B_d&ECRr^5 zC(GBhC|?sGR0Bjxsy~qutPVDOY4Z!Bal>s}HYH-niOF7xG#NNk1@$N>UdFueuY^_r zz*L`1rT_!on8CD4K-?}V>SFJ$!yTK`ubu5xC-Ec-PW~hj4s3Cr2U2N9k5EBrLhTq9 z&t26SE;Y4@=du^IiGPt2=1-&~GPfD}U!zVmTS%cbWBvK+^@qIhJaZ*Rh#E5+;J5m_;F~2@N^4` zU&Cps*KdQ6bK+Be+;}~O5zn*LQ*ml-)ZH*X?dlFqKYXHjUck5kZv)RU0YquOz|JzdO;Z#IOeMJjb7NHXlXJ$+iBk?>~r3!#Op`KzW22k=Xw& zxuTH~;VH&~OP0 zJL_^J|7T~q8gHV5c0;CME_o1RkW%N|m)a$w_2P1b6CGPw{|~-S2oWs#4g$VOhU) zEnM&~-|6#HdW=ef!v)moSnbUq^Kwh74EH&*Mhl^JW+8K4?HQZo-^BuMW6p;iYG6GN zXhadWBdkykVDz#r2PeFHN+Cbe&4blq4n6!3FTRmTN-?9^g@*-c%#wmYuZVvC70N|J zU#1W|39Yb$9)u0=0m6+OKQuBF^Ut32Pg;s}kk*b`Vo{j^In|Rpw)9!nLs0HaZg_vW zHr;0bOf|><_il6;o$1S#Ff*kT$1^JKa*7Ie3PhV(_)sCG?t$&gv;6IoAMPhol5qn> zN_^ahSjQqcyvT|EVsj?ImI+^qh9(JxWKOPxJ#tMX^pi00gOEt2+dj&nCMiRg*3Qa- z#$^*FUvxyT82ea}%%BKnZ(NAs9h%qP=_KCTCezdA*GLO7#Vkd_b^kG+U~ifmh=e7f z4S9?%9-vm%lY^Hio@Yy6qIq_&SU$yl37clmBd^3xA_>`t0oatFQKyUCjC@91Jw$^_ z#?GOi5m$pXe_#VJ;>vJN21Z;doi#PnE{Nwbs#}z88fj*bmLe;er2Q~;+)m5sRiqm3 zLnP3+CDU-s=N-xqrGIE`Ny@my_}$oAoBl;g1_%~o0w{V>wgonb&Q`%VD|saY74~(@ zO2H=GkJNfYkWRZ#b(5I$u&01X$^3*aKxC8)w7&iy zq(q;SBv1GZHrrqml?8IFq5hli+i|C_?0`$2fxkgzL1D0MESj7gpwxdToHse+)=O{+ zlCgOJ_WN7^+kcRf(WPGPs@+}@1t3x~MY%&h++VldCp34ipMgDiD`RnuY`?eniIiy6 z*uY_NJuS}!vxU=B{2!#mN_H&7?k&-R+V?B(u@S1f-Vi;DRNcm}#4Byhr zL+`8Oh0fky88dm`2RIamSrACFRVoQ|Dr@CF9>QMIdpFm_Jbpat_SzZLitCNFH^^I` zN_1KZ@Uc&%gn4y;@E@e)!T{DUR$R4)^Yo|Xw{J1ybnd=qZ2^#)u{8V)W*Cz8aVwY* zRDJy@Qp}`^UFrh7;Rv}k`ReF|9R86LX#(3_XW5%tHwsf9O$CZE62^hm&&e%wgZ1h&PQBg614?dlTWa%WdtkK5_{t8h)Zw_*kF5BWym;9%d3T>scMhCW1(=z|G zQVwPgE1+8u=#C90i?=OzFL0L^D<#JxjyKY|AcG!f%66iZoUzN&-PTCw?sodvYs`h; z6d^2rvj31A>I5PsOF*Q=`E~9}p`UZyjGmU~U!2Px4^z%4-|#PpY&kJmHVE7|)8DUq5d&QcJGQMmj>N^U-p65mgx1hQba zB~&GVw77(HfZu!e=625FL=7OuQI8Vn7bW0B!)Qan+dXN#*}%V@-Vb(7M*ETFqhXQK9Q1P zwgr&IZPWT_X_ZLGtp2S>cw=k2GOt74l*#?)oG)P7+x{$Ai`8N!7-GZX z0duoIKJXNvzWt1(p;qBMP5WxHyY&o@>6bLgv^q&yW58uU@7R2NzGkKd-~86Z4W5a^ zI&9yp!F?s^Qk0xyGmtexRFx)3Ra_=&J+TbMf6<#q>Kt&vCt*^!_Gy$bW|abslDC70 z>ylg~K$*a>*hd)P5FC7+#$d7q#B7CL0Ti1vjp?e(mRhE^otSdz&uJPcKz{)3DJJ1! zVgnrParP4_VS`f{!uUi=R8OI|2P~2>&owzUW*%y$zG@ixj)V^kz2pBEDbe+uQITj5 zV+E)TwQnhhc1(Qhgi+{5G1g~0*n_J~C3oA_@_bK4+O~BZYMLZdk~Pstpq;4RMFzuI zPDb8|;FnV#BzYF(N@2GvPYYv#vbQSF7`E_n+^Pmf=}56sFzAB^V17wlauGMTiFEz_ig zt#r?NZolilFc)vRABUd)6vbG_Ru5O~uZn+~WOKQPzxh~j*c;k$Sa%zRi%loo=!{qpod1GX`nwsyLJWrLi~W>{awo#maE?3uaF!^ zVZT^PbF1>?yv>P!f8yMc<0?YJuBeCA$fb9=GRK4zNYqaA5|Q* z2a^=^7sA;wJ6$QpZF+8eGP`ojnugGbc|J80;Z)8zabiN^DD0IldIl#mo}o5Bl{<7V zi~JDJyr9zW5m}A1!3O5i9~lkT^!ogs0!{9CCgR?H0DjvY+u7#^zddiCPXDthx1!_n zW-v|9y!wD|%s=3Jky7kvvs6B_CHTG*fsKwJz2rjXl6XYoj$=`0(U26P%%LV;o0<1u z&dBlj1a(uCR!Wmnywg{p#y2EuU_Zq ze(9Xr)6G*5V5&~jLSXb%s#>GqU?0lmsI(~|=KW%=-j?&otZbvpTLcn&9%}oY#c-vR zH4LHSB)INqJHH(CglZBH<4hd5HR4e`yTNV-6^n6Qyt0FtpNThkfVwJlNZcWYr zby5UQmzNDgIsZz69mI~G@vZODl*fow10bwF_sdQ7QN-&k9$f-oyNCei1fvSoSnzvM zDchv!Pu2W8JkB1 zn(vfl3nELJTn2@DG3S2J2AC8UYG&Er%4)qdN?7}g4$Yp6F z2{k83f*sUZHF~h4rS)WauQ}l_+<%0;C36o~dNQ+z>TdmH+$ANOakf?~y{j}-(y3em zFN3ZGT#V5=kZd3g&}zO3#LJOhi#Y8p@5`CCe_zq%f+M}=uV}XWJ-3nVosLH2&a6Ei zA?l<<^6a|Czn^wVrqv7xdQf#c?;q*6X&zXhK2Zk{=yBR63q;Lj1{&lEw5a^zj_Vj7 zgDV8V`DO}&jD+e6B4H=smJh*nap(VbOJHH%vNc z9unLNRPu)ek0D|YZyhyOT<1gm3Cw*kCQ zO@bM5dbst@qoW)b#2cA!)o#3=qg#=8aP}VQ#Nj3))hVgb_z^lFaTblcR&^ELar~{D z56oJuPge5Yj`2IlSh_Cp{~d9;z(p4P7Ij7|VDAJcYr3Pz?1R|{(}ynn8&DQByF*XP z!;^4#l!G6aBVm~@^1iQ8F#9r|ezvyBP!lZpgF`XBxZ@m5oCc4odXu|Pe@efy$9dmh zTOyH$BWW*)vgYsMK$K&z9XXgh?}JdZSHgj&F3WX zC#Q?BY-Y2z;Gi`WM-oD?VOvqR>t8xt`@;PlNd|omi6#$WZzx@=pnw|6v?CLQ%-Qi| zsGx6B7eZ~i)dXBrwyh?`!>;S@gClUo!sRxFv7|Db^-kry@SR#KdLY5S{38slMQ-$v zk6}}rB@#=KUkSg`?gt&n9dg`nWYlEGo&lb_{QU9i$dr`g=4v2x4f|Pb11h1E9wRX@ zaxM#2RjR`*eJ10P#I*P`hH8TeI_$oQkjA1t)|EI)yvA`ZH2S@qzbYng*k^d>lO8VU z%0J$w);vUfrH3-cPzbJtIc5KAa@Et*Ut0~47+53zcF%%@>|B!z3{flj&N97uO7U-Z z&)n&IGtbQBc~!5j>fH>ZY|K0f?dk@EUiZvhL4-n1&0IfJNEUat$$wt2zLK~{4c2tq z9Ct(<^1@Z;a?_@`C)qC73;j_6Gvhbf>1kD}0F|+Cv}_k+$--Ju3NJCY3%4l*0Im({ zyQg=B0f;vJBn=rVhp$UV9QLzx32xCGIjK@?9}*e(e(+h$=XQlTWl&u7nXd6FCQwL1> z<0Ve2U(UqmX|tN6XRxsI9m6S#GeLTGs}!({YUYcE$ zI%+fRG0N2nJvYLT)6)V2(EcB8iX0OP+>Oi7OQ^=I1E8|{v9ceZeaJ0>t%GO z7udR1E2{kvs^gRjWt4CjA_S*(ixzZ6F+La25Adipp_ZPUq6SFA0;rG1(#cpQ?>qbu zSebz1s(i%dxO_BoS;m1Ry`Z!HT*6_F()0pZ^0M&8*$^R1(X4D&Ia^k!bNt6xCBr}M?<7IZkhPymrh^a@ne2+DqV@1O@(@MGXkQqXdXKx4{=KalU$CZ?91 zRVQra^F=TN0X<-J_YFA_7ntj7KUZC2kJhB%$rec2!sTq2)W#_{Sa`5y?J@$@)4Cy% zks!|8aPQ}Fv69N!b-Rd^ZK~ouDjmcigpjUIY8=9Nu()X!e_Pz-IWTHeO=(Re#TiMU zm^kW{M7U*g7AJ#FBg9E#1!}r~M~FQ<_B7U+)mf?KEPdzfd;v_7`*t%mR6FwiCU5O! zao6x0lNwit6^@A$p} zjp{%NIwAzI${v>-sNJ$Rzh&d7n4M))aI`A@$Dt_4uFWEv@`rpGeO@#svcwx5Z4=q5 zVV*Ht#7*O2SU%2FB2q=|k?PPdxb6`Z0TB7H<{OH%xtO}~g;nMDs@ye)9-78xGonpL z5*r39>B6rSE+AJ8wh!7rzu5&yRO#=OO&F-Sl)3qZMJxE@Fb35TWA7kMzXlNlf>&=U zMgZq!!*UqhR1CcV{8c@i_Cd0M!Ken$(Nwxoha+jZP3^Lj26oWie7e}*q%-(XqiE4! zv%sm7HZlWcP^vs$HTRC<8JJ@~LytMUJXg$J^IM7(Sre1$G*v^N(yB$UB-lwMIoYrh zeh3=N#JeRKuNj-rtw?EHl~&>5>3dz1JTa>dNQvSpTEe>V&;Eu0G76iPuUXjXgIBi9 zgAtN<0V9Mb_Jd^n-nG3f8tm{jhP+>;i83R7o>m-)M(+8{T&v_M+b0X~6gSnpoPAs% znn@%#nx*-|y=-#_~gGwtn5rAiKy}t z5fy$2rb$?|i6D#Czj=;LK?N0h{-j*~Yxb`GMP!>DjAxHDR>a%oZXiqUCh4!E>0#~I z4P=KTGyfhp_KFOYTmou0LA6K{WY_a-E3~FO+{(>bV^pR@-E%>3Ud^&OLril8)$RFP zwQ1XTy!n{kEhj8i!toN;!}8ypFEXh&!`{a2k!+5wAaJPYF1#I>IouGiQ=M;ayV6nc z>*bp%EB)2_Q;`$rIJMY5n-V$co1QGZPt9}R9z1TBYElhS8pkqeihnv7<8z{1PU_R3 zv5l3%J?@g?9Kyh!K!|5&;j&|^&J+_)5BG#bko@edX~im6Acjqs(}LP9El;bCpQ}4` z@5h*aA4wsv=SC{lNpz+EN>IJzY-F{|2X`HS>DA@ycB{~qtkSs$cgqByAr|^NWfhzN zc%v_%S{O$(2+uf|9(VudADO~7z1BKLpI;iUceq!Y-*E3chUcp!=P6k`li-|i40?i5qHJTTnuWx}rXTA7$ z;*r50-S>|H(kU%7p+%E1}02Q{w1-@xxw{J5Nmk16cY48 zFvs3Z{6AA~3)?3xquuzNiDmbP-Z|Xs@IONQh=zk}lqifE1pYK~`{;4K*UU7!W3p0c zXYP9Y3a&I-{WjT0zrg#wqX#)OOa|wE-;gD@#xD}OlT?dZW^)s+QTNdsu*kcVmL%@l z8#mtl{h;F<5YA*2OoI8o(DD{;(dHmnOW~p=d32A`5u^8xlq5nJU%RM3wO^&-t5(BJ zHQdl;Tt)z&+KNk=Y)lY+0fV9C2KaH*aar%H!-RC5OSnX=a^8#2W~ z^0+CnKVpggif?4e$47LhloM*c92A^))E#ls?-fg(O_JAZEnul5To}Xi=!UZj*f3Z%sBi7@HruCoWXc_u?D9_C*k$+g$C(u2hy^lxfQ+H5RhLce z9`{FHJ18+L9YXY&?n>ZI?cEs7N@+Osfou`_I+1qP2(r0gEq?{|=#jz|cfw^_1%$Z7&KR2`3F}5{_N|24h97g2!0Zl_{GL_YtjT&caeUFKpaG4W`_7yO=BN zT_8K-k1f+PSm>r&t6=wXqzAtxQ~O*!HnVT#glJ{0rjU-qvIMpwvF*CQEj2xo?Ds$# zbnZ@(?9eBLl&UUxor;s;n!yBkSZTkcM$wd7f;(why>DX0>%mU%C{%zqPU+NP$4lIb zSyhObZ+TR1tOfr@o{iIxWbO7f7^>*^TqAIzy-qIsrrWx>@lC5zoJ@0K0K-^+)KNBV zYXWMeUhj(|Clh0GDA-VCbPU$wq374x8zk1Bk8qg3oDmYVOe4Q0%eA|Mxg^h!I36 zq?nQ(usT;#`PD|N%`M>+p~FCzpy%o{Rf^714zuqr_SLP&Dc6iOgWiLjBI#vnJm5q3 zNa39Dn)v4fe-iT1j_fVTY=IR=ew~xLPM3|UOoG(A^K~5gQZSPjDnxIRa@k0KfN~+d zcu>U$cb<4iee(~fTrGZc4hT2+{v^%3s?!G)z7EO*arV5T-9YDW_^X)$3Mk}{UAU11 z%(Nh+KCNYpo|BIJ%|a1FF6tNdwT#=R1za@M@fQs4NAw*yHEA$+4}Xhh0Qx%>n(HnM z>yI|-QI~X~%mGs!SG&zZd(_lVu7&)$h1pR<7K|oy&sT$6bT5yPJwKP79-xj7 z&Es}kf^4{ktGil%;L;#JPOO4GCp{g?{IyT_Z>29H!p_^B8>OCcZ!wFXS!cNktfo_A z+tmQTDSjNi7+ZTym=wa-66@5DeX!4be(`Mo)&0Y_^6h#RLjcN zD?qM?=9Jz?_@@Yl+&J|I}#r|@p28L9QkrTA>>+8;hA`a_fs- zIXKCpTNE8>VUSR(sIwX~$5Ww6QVo(li!N1kI7`m3$0IcO#Q)9ki{=M6O;jid!g?z! zl!b5aLaM$t=a&<(R6#w{Fz!DQOki;Y?v{55#L?f0VrdNKElCQYQ%GILB62Cy;hNU`~H!#r*s4He}dxUBV`K90(p~`Bt z=sEA>IHgd<+xtUML2noMocn?kikbYW5pRMSb`Kvh*Gdc3N>_W@p!EG^2Az$hj}iX# z1ji5Wr{qqbhx6^%*3NeR&d2-X=j);@hYx2jkEi>hEQYie3y~F)jYheT+k~~7{I_T&%}&r;25YOZA%j_Z|f#KW{djUnC@fvd~_|5QU!)9LehgN z@=vai{>grOOH1nAHrER8|54v6JYE1|8F~>wuyxpF1J z99A}4C#F$<7BwH~qOHQRe@Kzh@j3U&9_)U+3+HCv^UhPU-3Lo$vO$Abu)<8% z7&)qR3$=1(iV?Y2{CCBHnT5SQzzyeT>DRflnl$Bn_o{+(lTpEJ#>K6iNKeW|v@017 zn=mr+@A5`I-(%{Pbu@yK$w#^8%2N`cJ1VUR^t`$p!g_YZq)3R(rePf3CTk@U^aR51 zG&AYlwrZsz7@_dtv{UF*x^djguEniWt#aH9OwilZBSOrN={l3p!`w*y(+kaGL;Uki zOfneLLUgC(lTo8#>?Csl7$yb(6SAdg<6z^AKeq^Pu;44$5%;srW-O}qvlM@~3er$D zc8pwRN|vuv$I;0%>~KzayfW+$-7p2e(82hk$+GA4cXyLtV{1h~ILuOW1tsZx+ynBI zvx1uG9!ELA5A??XH*;eNeSjTx;X-$R3@)xDAFW}bRM5~&q$%)oV z^Sck{o$pohQ0jJ`ns}wjDYB#B&rlYYfWjr4Fj4$CS)v9BdtBJZNF5Ac?rH6tKzRH` z`h9OV(84Vpop4hiErNttWw)moT*sm`T$*7TR2Fedwf$IBq> zvnM#CBf_kEuZB!8!Yo?L-@=G=q_rC@=izPM+cISzdm|%>rj$9=o{gAH$t#sf0GOyV znQ5ISgmW9=^=8;&ZFN@2ZgiUv=W@C_Z8)RS8U^ukedIfv(9(V=C(WJPp5OX7?5?xT zl$Uu;rV)%Rv`;Z1f|CI`7khD-o)We}PT9L8GCJ3MW|h&`$S-Aq_38f)0Cqr$zfIYo zI;}O&nu6BENQ|khqcUXE+=~7rM5(6u^jZJA#fHL-pl4n$GPWmB%5;zE3F5Ks>4|Dl zPMl!%5-iNe>{h(vQz2`Xp9Z&@mmMSWC(lnGQ=P?J%Pu$qiL18n_F&u9Zbj2X*6e}R zeB5NyBG~e z>9YyHcM5(x`KX#M>^5H{LneuQFXXwha-$PL@IgO!C-}YiKM&K?C?9p;;}J z@_k;XYJ)F9ja+pD=U?*G5ugq5rajWR6%2ID^A633>*~54aBeP)XUA!-!Sm}k>jG}+ zCd6}7>lICy*@&yh&9PVTVRn#z!%W4LKN!o@Hn&YxAF>y3S;4X$syyF?jd&skEy;w7 zg+a%OnU<5lv2t_e2@-5GfO3|Nu;nQ{1_8rKO*v+@+v{;xnWX?(sJ8DG#qRi+{-;7H zVT_kgMoCa{*fxAROSSkC3_?0sF3erzYe_rD7KjKnfY7ZKeqxp&`kfI6w0)w%viS5Z zZVQP+-CY%o4`A&T|He+l=N^F(8)D3~pDx2|2 z49K+lGOn2>$|1WmN&08+ij0Le0z=dweZ6d+{k*J!J%no~7-i zM8R)I*>C%$Vb{ZFBOH7Cq`>^~85$Xp@NKuFqJNSE{SSYVGKpK5KG1eR?Tp?neaj%@ z(_@eI6;;95Pz5O|SZ)8?hYtP^sVJ}2h@3l^L-;ovISXEs&~&xz0HlI2n-?c1OM2$O zGVz(4-#eL(?%Dh(HX|9IQ1~5PiK70Rt+vBlz3VYSlDNSOU>H6qYL^bH(XW=Mqc@|pw@XSQkW1i>WX28yPB=N$+n{(H(=;Myi!dq5k_AI&y z#IcS+3gH7UwY0qEhkgi)Lw5^lW;5!|W9FRVMxctOD13=uw4c$R4Fbo8s-{K+>F(x; zB3!hrkZ`ufOjE}XDHigdyOVh(Ze-qdMtvj^nw+@*XsgXeB^x9tUwtTTqIHA~^@EZ= z-QKgu-HAzf{X*saoK0(nEJ$#w;gnlCZs~V^ME0K_=?rKnmVp@Eba8RB+Tyb&3!`0l8JyRGDTRohw_Loj z-tVYpzBbb2iI!Et2Ivw0a`;av7_msdf!|UVN~$G!jp|S36XhM`GpfaoT=M>XMZP%u z;_P+Y0l%{7){H7!C1ZJY*5XF-`I(r}zPA7Sh5g?z$p}Of=+t7i=Lxh8>>|k3OqL9$?u-8wr`xpEf4L@7*tfktf#IK$M~IG<^d4Q+=b3J3 zVLEv7k)reUs3sq{H2Q)Y>B;NJm+EE2*Lb(X!gHmX0`Ma1$G6c+2BrMLYVhIdQeq7!;nYPNbClLDOxmnhlNE^S zHoT1`3(fI>{OIuH*d>B1KLrC8z#6ScP~9bdm2k}3zDte$|6x&Dd&hA{^Ncik`Y1qO z3>N0ee*T+X=|x|}=|x*tX7GQbb>#M-tUMpTIcaC@6mWV=0t-^cTc}jz4!7!{;<}-j zi7wcq%`yp=VPT2`CxBBS;?C`uWwcS|5S5iNmxwak!#PO4Jo%Cg$*WUyngkwQ*dn0< z2Slx?*z1`w&CEJQN)7I6!FAQDPk*fonX~iBgbS|cu>^GeuJ`MumV#@3CLhV8O>cs| zWVsl^?wy%`BstFDGVph&)j<7yW>2fx2P2yblu!|TNWPUq)?cBrk2RkBo%w9Ht$fXj zCJ#6t+uJ?zdKi)`@GpI}t|jk-OkqXv->xf)4 z!aG24?S3nV?R<12BbPPe$`&_dRBOA~w73_Hpp}iby&Di(NN^e(@oPR}x&7iqI^Xi5 zZ#1ge8PK$}Akjf*7xvlmZbBnA`HnX<_G(u<+O7MacbD#Q*EZoCWcLhiI+$qmeUD(z zhed{{4d2+ynyctQG8F6r?xKtqq26iqTb{ETuiG)}Bqql~8S(j{eH$h%LzWaa_6KAQ zIvO|GJ<~2MSVQ7t)o3!N1r?du`=A-cIsPWjb*Lpah;NVXE%Gt?lv?a!kj3CCgJF6@ z+GLy;N)C)va*ZrGH?cYU-xz(hHyUHwG7$u!P#MEEy-K&R1V&xy88-+vngXn4VY?o6 zDL7nK^w9uU!R%T~o`Qn|1&->}85B7XB`r4Y-_q@cQ4C!Vz%1Fk2dmvj+ZH!VKDAv5 zon>>ddxZl0mKH8?)?9&Y-C{7i(ft8iFm^C!sp54YqESNEi5 zN;Y+7d8ij?KGreu;L}zkgCbQW&1Pt&*#IDWs-$Ww4@K6u>+(FF|8ToJB(4HVNpzne zW)_@EEeH^bZug%#s18P93Ugi~ZH*lZ8>>;K_=rBLQ6|fv_;w^haHE6v3;L)Sc6V&L zYyKQF&eXi9iMop}z}6q+jjcEr4piiaymXv@8@O;q;mu~9%Z5Bs^kjkTi%+b$ zQmCe8Ei=eslScD$r8r!MF6?l-$;31va~_{dREO19$PNh?Q_dZUeLOj55l== z-)97_YZ$ysua3wGGJ1iEQZPqE?n{OMq&S-PJ2jF>lB0^-?c}f2%1-bmwt$k7SHvaUhv7@p(ZbGAt;mBkm`p>3jMqAyP!t5GY z)RN8T-)a!kB}V`8YU;^Z)cC@ z<~tq1H3(mX>TIov65WzH@$a`}$|ToeO44B6g&O_k5AUybqU{=Kyi;x0G_X?Jg+JKZ z8+%ZAp}8IkFZ^yph1XnDdsTS%?pnzIm5;U&=vmDoN;0)iXGYtfJ7v-rUBW*lOhd?o zh6m}^-iEGd6SNawk___ucXctrz1M%<+^^lV;k6APlSuK|quXhsyj|S5ue&jATbyUC znoV|=b769{CfEsoq!e7sx*u<33Nevw#|-Vg+n(fY@F^vkEDDyn)JIqx_p?nLh89_+ zM)$u1m!WkIp5cU_y}vwv^Y(I7=D(Q9$Dx+PMzJA}m1v2iV?C&#J_$!wMfI$oQD%N= zmb0zV^g;P5aS7W65gN6Wdbyt5q$vvAALiu56WayRq_Y=f%512*bSC?=GEw$_T>ENH zu~r7zU_slKx=VW7`5YsYdS({*p3MocEgHrDgJSidv&EUtVAjqW#p+R_qV{Q4CPg!Y z@w{h~{jAG-Hd(bS_G2gs<3ZI%(QEV!6JM~R6nuI0Iw7Cti(QyL+`uA1VJ4wF=jbTu zSN-{1!vKI?yhP&KyH05gUo5`~VDTug^%FhjH?)EU%0Jo^RwPeLzxYhOvErkn9Kn#aiNT}w5nN6rXhY zRQMe;$sKt`^D>eoi@f}MB+XP#Fh|l%UoP^vCGTbW!rQy1ul>wIlRT7_I)5ebH95d) z7q8mG{>8@9oR_wT8+tDBE#SM96*F=&yC2DVx(xv8;yy0t>L0}rTO3w-yODQbx4#6W z>B|m$z99VQHa80SENs%x%ALN)HA}ZV_if;``amUtc)^yD0Q&{6Gv~jDl)ARmykp-O4I zOe6EQt-&S*$t>3%D9ZsOdk=+M5SeL5!sCg$U%h(u3R!6X^FRNS$eK`+vy7L^UxaV{ z=5-{|qN%8if=!RguMFguYaBJ zORynj%`I`3%X6jX=BgR8^}^Na%oU^_(9uMu#9h@TpU$);Kr-j1;F-oN64U&p<^95n z$+iNzHN_HwX*nB^g5NVTQQy?EsoDZ5$foKNqp{I;Z$?*G*jLI?Qn_5x# z)0X_GDEArAkoSOES66$>BB>?JbK%9NqpGZG`9HYHfNUzb{(!{E>;XsN)Iv<6+jF%7 zjWTa57PckD$<-vRhz~{jxH4S)YUI( z+d^tPOIK^Z>sHY(Kzu+sSQ5o_W0#KAwXDrpm{OLRyrlI#%f0*`ksCAUq)Oey)q3O) zm@;0{ISj*EW6+u7(dZZjm8^-1bQ<)v!XOXGp2TK zKbtz1cD#1C@1+i6nV`h1=zMY&#L{Xq@6t3GTSvY~SWA39p#p=?r&eR|xtd_1{H{p0 zq9;@`-Q8ip4-tsVc6g3X+)S4W-MjH( z;z9~g2|;GfWm6*%$>T;cS=UAxg$__fYBuFc1F5Vlet<-oJIiF=D<;E(FFumv9sN#g zUR1Sw;JLFd!b^cGQc)Ff(@~{l#=$$?u5xsH=!OAXC1={0ih_0co^XUgsbw1B%Q6!kbT|LTOqG&HVrb^tf-0WpHYXHqgJ!^!8AL8|z zl@qUm>R(d0)fpJA%l*KwAd(T_xj6tE4#0-!Mi0P-eF7Wy4vuqPJQVjd9|C6whRqv< z!+NbIRZCiv|Ft8Vz@awIwXIPdExl--bCl&bJtSX`Ew&jfr5zyH0!-OUb7ycM}LGQ&6j}^&f)|5=>cFVLdTJ$=nEqL_7OXLcPRSuJSk;_M6MEv

}s%2|**rf4)Q7Fr*h>w<}QS75ol{SL2QA-Du%mF2H zK*{VJC3A()7o(XJwtzyiP$d4gOj&?UPzP^XagNsI0%uc#(z;+Gqikv8XFbes;_nAA zUv`XTfHo2w82EH499n)3tPq4?qm;hm;|PLfZfmDPMPE$!8rX7!@mL#tFdGp}k&0&M zUTAm@5~n3fi6wWyr{7GQhgST1R;vUC_V($P?!dRWDO1^umGKv5`G1@2f~y%)nI*uAxMXM8Z&1n_Cg>!$dki6$vT8BCBE_acr-djm~LwxH+nMs z9bqDq(S@?x=MF|%D-CLN>uC>QV5xi7jXDGoL%-8>tDg6oDFD#%WF72RJDmO^rHqX^2U71!~=PtwMyH~WHUuJ z=MP-TXh25$P%(d(cm@cTdF(0GyT9)>*o9@i^+6AFl4PT+CQXCH2QN5xAYH*ce-Km6 zNz(dI(YAS=5)#FyqFS?Bd`15IFLSIkAHsq&E3uAU-{|JKZ>Tj$AA5N@*}(aYvfB#PyhHI$zAPe;aupUtcSR`UqM-= z#2310-}J<{IzU@wvA<=u$0Oa>_;SPioNTW?pyq!;m~v7t0SoYk?~K{Wy}z4)tyx*g zn$~lYTOa*=3xviSmg0i0<~-&SO6D+TId96HGZ;fF;Y>e5Jxp73(?LV+*K8PYOtX7* z3izc+iG6emARZlA++4}1@Ub_*T*{msEB{^WLa*=_f>&&|#EE_3sWErnf~L8*%54eT zh4f+%<^N>g87(^+sF0$9^;5ihsaw5Cf7}E(;I5Y7?y+3x6H>@5Ks>iq+yR)v6}fb> zCl#V%wzX*;tsDq86$(L~JcCO9;6j4^mb{nE%S-yi%cdlaGCh}~`>wX*k9b7!43%8K zk2>zO_`7Sy$gQ8z)+dYrFzf444VdP(H`Zj76Tl7zb;wVXH2=@Je;xK?%q9rS#ovI_D$*E42Y10V&;u}E(;l(IrF<}_Bu9;afAP%o(NtB?K znn?{_O?wu}KgPzs0na+Su0Kb(LR3TMF96D-c z8`e9fNIn;|5dlu;;B>ZDOfp#(}c~xdJbd_NO@P6o{3zxc*&@DpWj@m(3giY z!4@8dKiEcd#H_4K48YjUM*J?G<){PiK1g$RM(W>QgIKUDx?i zY7|#+PwQBphmCsSBs|hNl<=|XGaW#vt8a?UOk~{UTe^$hZnnsYI0rG6Q2|y71Q5;s z0~xeD3$Z6SC@$@Pin@%VW`{>wuc;R0L%V(NSXmWR?;KwFLlDpmkVi)LQIQX6Vb0La z_ZOC61SM53i1?km--Oi?QJr5Yq@X;(nB{OG*&e0@yyT-`wcCg{-(L&}AF+}9mw%pMq%DgAWVV`i36e~)}xBO4$N~d2z%lRsCjWEU{R^f(R4bcY254+PyJlU_*f6j z$S;M|zx0R4rA~nIC<%R#JSU9;B}I@%Kv*;(oMcyfJSQbrn%=V!xi$JFstI0K)JAuN zGV?4ajnF(U1(;at(f13cD$qTFimS;-r zR9Kh*r)sqs1cq@11JYED- zrd6A8hRS|oE7}^A*kdqbo$R;1|CIwwlfiI{C~1{?W8UmV^k+sHdn@zwBfc4AuhVo( z)qsp53gTAy1;Ey-4sR=}lQa5|%uU%lDAIa0hAuvW-5WyphOhr+?~twV#JR_4inqN0 zNSp4qwq>i^Tl}`q46xi@j2qo9ePw>xv7N0)#(U|JkmSU|Wl^virqsIX7ZPpE>=Oh~rQ`iv@W}hu1bg%*K2=!v z%z=F&2j13lSINz@YqeRQ#TkqK< zeQF>Q7QP~`EwpMnKOD6Qde*_yqmY(+?5^p&vYQ~X(0WK5zOu)uedrI2h7>_a`AGle z=3;&3(ZptbA!|75UZ#ib>u(!1J>B~fLslxVFr1G?<=+9i_K1s?kw3++#{N@UK z*{=^@gGiZMvs%x1oewKo>p8|F2Av*ya;P3nZG?@!=OPg`ice=tz!W<84&u;yP1&=% zVBfRS$K)c5vvS=QLvX=>&v{n*VT^%Nm`6t0!XG7t4)E!1VQ()jlyq-^Pp1!>>qm7{ z9Z+2ZW6{RjWJnL=EfH|Hn>P&zpBTkNiXp(JlDpFaO04xWvN@0F8VBhZ^eSko6~5#| z$dqCY_<#V%cb{UK>NA*tae9RW(rQnkPd&I`oEpbPZcZ$y0pM*{O_nj{V`Y71W%mNJ zQ&>`+&6pxeLR~VYX;HA;gGz9rC1|=9aFX0^pfIOTSrCD+;lrw+HX1W&M25~euIC;v zuxQ+XOI$$^DqY>4AQm8SZ~Mo?ZO0Aqi*aeZ5a#$4r|$*;#=44D50sHKKVq&NbYv|`%Re5T7?V!^7k(XynPXAryMlK^g0Vg%ouY==Tcy$G~3 zyAC~%;cA6^3lka4on58wG`V31`eV;=+&O|qqGvT@gqOzLq(YONPdtmK!&ZH4P5apR zKLElncq>K*i$Xpsq93LAf3`>(OUXFsHr_)av8|Y^!0dxSZV${^e`G~r{`X_jJ2|<3 z<(TlP`Hf#Ci3JFQJxS>zlzR~ z9-TNZs#(yB(3jw=wsq&O)4z*aDs}CCvcs+({0x!L!Osx;9Q+Kco`auZ)zdo$O|~RM zu0xHoE2rgTtMf$2wbT}Z2!;WD*1t?aus_oINHA^rv0bR;z3am_wNVBR=a$vs8C#ruPHCgMspjkjhc~0 zvAT<>3qrGAn%E6eCK83><0-z)`)&S7RkH^!8|6|-8;$CR($?yq@#)Nc+2vaS zS8RxQnzO@UJ{;yxf0*Cf-MDz1t)w> z%nLKE*i%*TjO(BVO=PzA;sZ{PN3PI}5rmf&7VQNgGPLXxE{@}SpFwf}VNVc^!w&vi z%edY@MXIxOF2T-0a4)fh|8&d>@{fkq=@Dn6n77&4((FpbbIdw+@z{NOLm?=8`DBa} z)h82$EKb$p3;RU^NKjV+r`s<= zdfi~&d%u-nWqHRri;1;3kX5bQHI;Q|Nq5o_+X?VE z<6pp&4sa!x_zHv=HPvj|14q?9C99{vi=NH;@6HXbNPM=iy*Ek4X`d}7<+!J>-yNT! zk!3x9>2`0gpKs)^*mNv# zMF3SmP#5M)%G%NcEoCm4f77-+vULF__oPK=)l5|qk^CzMZZ*XjRe;o-ScFB@)Rk1W zME~O9=#?*OlY1R20J7`&lVi@Cqt}7V*^ok8*1*PV!J{JQnROsgO+b02#$B5m;E8D7ib9eQRIx9;bcSN`@hbdzS2J-%#GC zl*o3^b_6`st8)a#otgn{p4OJ`qN@KNd+*xbxQ*nG?(6;(I8OFyd&ZKl-JS8->>S5- zCZ4pry_S=leX{fKg-A$3Oc4wK+O2eQKKpraBf&dJO0wM%ULeVK4Du!a5+yd z(s5{M8AyKs@C3wR>$soROFoF56HjL|ZUu2DrXFKhQPLXaqod*j&_FN#(X6s52Z>uA#Kz+<@v3N zFvJy-WT=k@S7I?aD|c#e$AkpP5$09#$WK4LbG8I#`7Zswg8iJw;epV=f55&D5&+wR z=#DEZwlv*$*oz+_H5mX857_TEGc6cjR+61NV+Rm(0_3D%AB9{9Kgz(i{K6eB>H;ea za9NMcv^yW;FvYY_c?gESkXOkK5<^_!088*6!e@Ztc^q&UA`-K}IYRieWRG^LRe(M! zk+Q0UGieQ3jZDVkrdc`CjF3v*a-y9ACir`adX_+dqmRhbSYJ^PPqCw|;as%RgB`QIY2~L`DDqZg^~8M@sDyM$g0OaZ z30kkAjrv+&YSdRdUzpVlA4Mn>A~&@8TaDYwrcvkFcTU6C>r%3Be!Z8vwOQpQHGw)e zWJUs?0rZhmD7Lq`*vJyv69B`-&{dUbec*pw1=;_+6=a4yig*h^kk0uVO-0$^&HP+2 zYpJIuPE}v;wy`fU6f(^5eI`SY1hj&TCH4xrscON>2kj6{%YkJOLrewc3u(O>Q?b zuQ*qd+;7iS@;Deu=p03CSE7Wdf2%8Q(MQWFBYWvSa(XlzKj`8b+BT|PFU1qE)h6kA zAEWp?OS>>3(e3P_*?bykGC`hTN13z3B&-kz06$h?#|Awriw4!!d)ta1P5sm^L&l*5 z2$DS#DuUR@b=DNk!Rp$}zznslbS^Xi4^hFS!2&!`($(J502tIIEjaweNhtKn-q1Dg9~7G?ZvY@51dBFWYRtLd(0^jgWEsfW^w z&vVS~RwaoF6=h}8Kh7?I{xxg-$%-eZs;ckf`rNf;Wz;{;E^^(;8HG-Rz?|g83D2Nn zY3bE*$fT?jdMQOdrpk@P^uix{3@3>JrgvzFiRIGe*2SJFzo#Bbo&9{w?mmM%%)HUY zB(=4&WG_ez9(1z3md2r59G!rRki`_G0aW=a#rbFHrXrc;k)5phzVlI740PFoB1pNk zJFS}4l8VeuW&IE?lh-aMCXjqr=gCp_K+F@I6n}033?Fb_jmZJGNH66c_n&<5h1w_1SAJfYxrb&6!`9 zYE}hBH;HF5WA3KlcJ>6d0+z1NV}64ovNA1Rk)LXv^{t%_-GW2WcDSTHyLwM^wBuihUEmy3l><=v%VU0)5Lbu;0SPr4fT%&= z_)am`Nr_a2;<2ITRCStlkoYfI+M|^iFyz63(3Q)Tq@n~zR(YGHzqeB{u1r#5lfG9) zFi@`M{pE1@awwIYm&2DvWe$?y7Of|wm3PT)pXKx+0PhVbj!fW@i>FCKWr84Sv&2cx zu9p@!3i93~9^0MJVK`HKukra;c5RzBxfAEuJ=PYPo42cLs}c_#TdHM?Zl_1)bCp?p zCU(Ylvz35Sw-bY{``m8xNhf3J8<5JEK5>#powguTLC%1b{H#H_!t|IdMK_y4io~0d zl(7MxAdX#^OJ-L}YN|I-rwv`Ri5i7|L~!U1qL-P5ypkP!Jf%xb+CpIaaask99y<%Pg+yY?82iMe?&R~d>@u*ly)r22kx(WKJT z1c95Yu#5+v2z^*tjp0Rmzsq#RG6$9F&!%bbxiqn%b_^h&Srz;SBSZEALq z&Y+A0_o5aqPCd$65=YvNM}k#^Izj1m90b=Sz}_NBd!~$RzL-F#x(DDicz_FbV)p_- z`5S2TGNwI7jF_ac%hJYHr)Rm&@Cfq~G07p68by*_4ly_%ey zYwh80%ZyXR1v?HBj?{Guu9VMw(K)>yDAM36yP7%Y2!;kh1w88!1AQ2gdLTbP-QgDle|7aX?JV!bBa3C%D1$P}$mBw0^ z@qQY_!X$IVzHVQ^J0dT7rTegZuPr$)xzltMi&ko=xKu;imPi`ur2qVve+}>LJ2LsX zPz!cM`{syFO;*A=4Dkdp!2t`wF4{sYQ=UEuMp2$ax$Zd1zuTM*+M2QSsJu*SgD+{nq1ROCSY#OR^!OAcpRy5LldFv91M?++&izF z#$_E0c`?O&Ap{}%$cGaq6ZM@*fG552&7d^VDGzhCfKsLwjJ676U6gFRt5#BVFDF`> z2^LKGoDIP>VG0~70R(IjTJXfSeo>z3<`3lY?P+iSuKlo&HHt$-Z6^{^odXW1xx{;f zX1bo{oLJ$dmhSbcSKo~}4S~x%Q)Pp;Gzg&kAb*2q@E(=!9TD+|B_dX`wR;8PY>-4n zCCGD(p}EYtR1;v>P3(xV3kY!{49^^1{Jr9vu|s#b)&k{4EH zs<3i|sD0Pntuqs|M0BiPngCMggA33-l6S)sJOwI6$y_7E8s|JhWu^^^eRpd@UR@AV z^g!rcK%j4eH3G3C2lx(wa})(+p;XqxZA@?0q47z5<2Do5AQb=CgCmWb(#tdQE|f1*eC^uHd{Fw6+PbnN>~i?7P!z z0An!+Tmw+9=h4^jW^s!tC;O=Ym7zL?yV^-M9P7cBhBe*5|*-u@LE0tZ!zNv&`7;3Q6Ut5oKW7c z#gPd|AE58k6Q${xy9Vlx6CzG+JW1JY4rAMV5n&Go!2;Z#T}z4#eD!)?YV${julJpK z^0S1Sdm_GKPatMx-6%pC3?P}yLFn@%1(DOnXIaZ>wcrF-Cn`v{)4>MRro3!GZCcUT zfZDjrV6}MFX1Q|5C#x=*cA&UltF8L_`EJ!Kb$i0Ux=7N2BSiNgGa<`1#JbAmF3ey$4p(YeU8<~U#cSc}f^5QEcuff2eLHFyS+fQ! z_7~yAWmKQ6v#*ts$qWep(CrDwsuDn+7)5C6{;z< zz^H{(QF{Ru`hn045MyR%_?*_oO5CFddU%$-^+#pt(N?wjlvbt_q?tT#R?tqyd&2Eb zwmjRL?Z_ZE>~BaW>2(9Fr@xjwHM@K<-^q2hc{ zz4V@his@gTFI@l<${z?_abBTpExC|x{j7j?y&@TxZAvRp*nB_4NINy40ulD3KQt7F z9+_wId(V(kvd%`AS9q!#WZDiU2=bVs!4z`024Du>WwjYw=B;41NwG`00q= zwJWZldA1oP#yW0i=3{Lh7rc>dW{yW^AR~z~GW98JUZ!VD-@qvOMM~Aydsp(M^7W9y z9tMHn6cVOM?%b4!Boso$3d~`o!yaL-P9#7FT{#5A<1o;--DAWb5DLNm{=T|4+<)`tjSSoY zKJxIq_@aF4)xm*#)&G5Z^%g)+S{qdQO4R~6w{|6L7yNVd{+(GCEcaBhS2A2JPUAX? z70a08^s)@{SrV@arFViA=c|QY3;$@D4hm0aa+q-_**L(ShHg~Xwp)b~`%(x}1U6H$7ci*1tq`#R+_f(j}fP_<N=qAog^j$LMUnp=00U7W4}@DwuHEsYufCKfZ*=IeeARREk7YFFM$ z`d(Su_B{_V#ijqN!Wwz{_W!m_$EXUKlvQl74sa;81Q>HNh$9~+s9=ux7il>Sq8Z!; zQ)sFD#dhKR+-D!%J#)L8#7Ag!~}@XJUCz zum`yc)CWHDV{zbvjz&(w6nhNLS#F%4$kCc_)fw%aP{FwjcKud7`Mg<-wr1Y!kE|xh642h!2HM%wre?ivjv5HU_cK|Dvgy0Toxhqbq(qgDz?%XJfBp`>v89@t4T-DcYr2DQERql|p&9JOwW=!--oo{+j z)VqZwmIV3sTqvqQnW%sPPPz^_=;ohZEQX_&HuOCfi`|~=rOq8%ykv@Rh1{FrNF^^7 zyM{Qs_F`UDew6THrh27hU52~h9pU1C7auXxirP6r>>c6qmq(vJ)%B+`d)hXIr3bML zMU$rk*+nx;2I481Y0SW-OzxqEx>p0FJGp_g64jgYv_w18@h`>t_3RRwncNjMMK?p* z$kY=F-5CYYL%u;8gksbnM^ncjkg76AbSlkzUJHcZV(GP+sTSXCQALB z!$^qePeRL-(&`ToK}^{YoEqRu`3SY$dkB$9y;zsC3;1{wnkh5#v$P|>FRX=fKa z%m`B|YZ6Jhb%z#PyFD~LdmGrg47bwjgN*P_6muCguqB^u6^}Z74=ue`VXu$-$04Sr zZ9uOo%QLPR&Mh&Fju!^#RU0;nc8?AfOG}AmRS_(dF*UX4h8_aLz|TF#6bR*t#%LG# zBz(!WQPYM^&ZAHIC9|~6b~;oIDUzU=1`=9ZtrU>5w03jj1ttD0tGI1^wXRr4S%*YQ zF_}=rt~_N7s|i^PdWTx;*iFjzH1=B=+5RS~9GBBTtU^ z`kq^2ba&e^^pzhaYO1A=k(|!49GSH)^OOOefg#(vA?mU-Hdv=`XDGe>ArsjubJ?{V zmQN6{;!xpnDTj^jmoPo1zS}v&Ii@V%!qy6f@>#lqQIGrNgoX^>p>riD;h4m2RT~Xc zrm6XmLkUc+?q1h)?bPx3+<%oSSG26_=Pp3dn+Y{~pxl{+!f!PDb1^uJ#?&7p^b}9L1X&1fCFVp4H-U@f_&@QPNt`esmq!=hLVgJh*CmD z{}5vU%Wcn)Pxxe5uoG6>|o+uCTN5E3y*UQDxz%!oP}N0=HRTcgYV+l>{cQ8)a-ve9lx7UKU7-^_vsF%jd|96Z zZoaI=Ava&TIOyg}dxzb8>2^?6Wm${gPE|}Zr(UhjNd-&V1fs+NpwB(@mEL~UvWH_v zf|$3>id;q1-qU3#jCr`*MxG~cX$DeLQ(JQ%+C4XrJ~v2RZ;v{h3sqTyt-9N7(Nb@U zy4tF{Kv0@*`!4nVaFnV%4k)E6j{;3;s_qX^smfD;sZ{0huM3i-H%iA@v`$o!mdw#q zVn%YNPRiZSj5Ruh;!LdI>Gfp~OPobzDYiJ9-t)D_T`I$x?MZu{cC2MK z=(KcFy_aL3C^Y}uuEziy65nDNPITyIx0`XJyK0@#WYko_8to&2z!K~qvfdUD-uaRl z|CedTICZzu)wKeEoCP-`{)zHifMG~f0#?@DxlP_QvR!?rY5{hIUFJZQ82c*ZB?0z% z*^$;VT#;dfe?+uR=BBcenO#k<+f{E`BliPc{I4;*qN|mQzG45(9-c5X(vlidZ4cSdc;TmQ0Jzn zjLV~1A04nbM{_+Uok`l)<8lnTYTmtd1Ot)u?J|#5J+?)C43m)4IxVnf>LlT^K365+ zP9VT@tmDMQK7m+|W8AhPHuR!+R}&u2(VWo5ZbDo<%+#je*pbI6FQycQJXo|tOf4(R zI9DaZ*3D@#Em;|uWV`RBGs}yBL`#*sxrTo3cM?kMqJkm{@4-E!tm_&B%Q_DE_#U%< zVV^Kuyzjh5fy!Rfdne6O>K$E{VXTo>!TJXhlbA;_*UE~p}{7(J#`}WW1;$MzLc6R>$WsgQ@PIWls140Lil}=+qP}o*iLS2+s=RTP1RIQ z)zmz6Ri9J+&{h4=XZ`ltYs;kZ;cp1=oBQ+MmH7VHa)#OrQdc5EN(1^e0{O&BQuwkX ziIc!=LIPqLfHx3_U*gNgaM8S!C#`xghxX(mZ4nXmbK8wW6RUlMK6-q(LG=7hHOOS| z@0zQp2)tw!!^;C3K;CM{4(HRUD^T8P5v)`@W~O#y7Q$h9S|i_-94U|_B+WbL5-1rW zDLQVNSFM+|OXN*uz+=`I3&FkEF+ng{lLPcExg9;%^PxK~)c~i+Lo{h+96$Z3x(1>p zFjKp{AjZK$%Z4R%ll6g@RE;?M+J+EWg>95LQs_Vt!2eS2nmcT$#G>d9p9q1S3Ml#N zAynxhV^0e0^hIBrtJMd;EuWbvZ4@|@;KXt6(_#A%uet=0BmZ4CKV5O<5|8Zhoc{!z zKm8={k%X$Pc_s-mY)Sn}8ey{O+mdkpu0^6mL}3!Vvy-%Sw;;6F_M%N;Zcrk@3~5>O zO15MEVgu1E;3x?qO;_CqlQPw>6Eywm=xw*sECJVFu85~`?4Y%J`?h%;R_`pt-SFUA zML*o48p1MW=%Y$5pa0D_xo!Ep+!|&oDlpowFl0$7}i4K2sa1h~c_eA%^B;QykB0#xp#UBd{0_@@=hLmJV7aj9xi zazqU&)M%yCY>A>fJLa()QkID+8`Uo9Fo@wJ5yvJ43x`zMnTsZZ7)HPQA=g{}A|y{A z6F+zwa_eK}aaRUw+RXlx$Z&o^H)|0UyEvhbou*(}imrJ3IKLg7UHy9f-tSwU?P^^l zVeu;3cHv&~4nxZkr&g6yojt}v1m`Fy^YY1aDc&U$%Ju`&;nic*^0^&^?6(rJS> zaxKUYI~f}_+OVk8iM$7$T(?^Y0XCUnq>QXJy?^Xht7v8(ihs?4gs)MJFznH4|}*aw%{xPY1Kgu-8Ja@gL)VQR=mT- z{^#v-RZyR+L-93ZkMrx@zHzW>=NKXq#JYGu1EqCXL*(?HI8WROK|eAHZ0mw{inDF< zb!&CC>t=*0QV>-{G5}~L{(2X0p7D_BqGQ`%3ifaHm!o#_qZ?_1P%&l>*VCaBNI2Gv zZU(a}5OH;_{}PpP_yu)VogV(pjXgeI5M~7B2IjHejY3;1lvIikJwF?9GBK1teqhn+ z$A`g(fLePIZA~byi8m#V`$EBzj3M~8_tX2W#?oCI=tDQROq@?J3XfelMTq-lK2(x1`%^{331Zo~KlnIgSh1Be$ z6uQv@z_iHrxBW^z4$(lpmB;ThI}?~ZtO~lIYNf=@Al&KbG37xfe>KDT)iI*j;9zjm1pG$-0Gcjj++638wYIML|RjDUEuvCvW1w zJW{?LiI{ir>>H@Shl-9hf`{c~SSEeZpPQpeP&wg`dOyNO_gCt9Xz-)PEv~SU$|HWJd7(FLlhc6nhzZt2k!8=oO_bMz0r0E z(>9bkxwV6^60*~D=}b1HsPWjQQ75j5b*8ecS}z1O#3}wGeQX5K8Jx47=PLZb$+t7m z9#IFzY+(mzihGA(*M07C-y0`+s#s5rcXb&wBf~<2#4(F6Zm}gM0ZUZ6)p|OvFpEYm zg|UM{P_waDpHpg{s?j#L95ywLwuzeBvxF*GaVj4=s)x&^EFIFW zReB@Z)SmVq=+txWiA-z`mn`fCNVd2Kdqs7>;$FhBQk9S4XXTKy-ydJLIUB*`UV!$lZW_ zcp1~Cun=+XyHom|S}v(-j#$T-+og2nw;SQ`TllG7_z2ppj_MdO={JSex)_?s@?@6* z`tRbyogx+J%#M{wBQk73D#O0gFXjjCdm|wyg+!PbUWrRVn72uJ$bZQGyJX9vh^KbM znsv8o@hf~u4K_y!iQByIf=b+kbi}}-){+;#K@EYyi+(PM*s^v18_mg|=(G(yNhL7or^@6TCTnpobCPqGxi;Dlu0k-!1~DlKxX5C+$uBbOXy)!zj0lEKp`J4FnAe2;Ip z*r!_sXag_&49pPUM8RDDG{fir(4->YJ3rNLEuAG2+7(WwnDtc##9}&u3tVtb6~>y5 zTa;dhQTD#XbB;m2S7G5VZ_5#zw2x{Ha$!hvQ+G&%VSr#jW^^+E6?)>~(rX=|+Hu0! zHm?X0QDp16@!hp9yr2H7sL|!fjQzUAa{%@E+t1GQe2X(q{9K9Jmt{YLoy<2s!Ak9o zyg~+3TE(E)l8$T){g^Z~pppCAX%3wP9A06ad#Ssrc zY?2bas0^W+1tr0`2OWQp2O5DJSo@Tie|!CxHa@w`C;cIED=S#q14TdH@c_uKLm-80 zZRHXKr@homrnVj?qZ~zcA#YEBkKkzShOxC^^oc{K({W-7NvTjr?XvupQ)^l@(SkBp z_lHz$5ncl?*Yk@Vo!o34U8pl43eywAfhimlizBSJ8E>4sIlex2#<+q$=>_5CnFnS1 znI!~OKD3t97-wHZuW5WI1k3AT`VGE1CSZf>9QFO7L*_O+I34-i#^Z)duoSOk--85F zB|0VkuQ^2~DLv18fbALQ#=!9l^jU>5 zS;)RvjK(%02TnyX@V*dss1EUDBYeg=FZ$+~G7gVI<*nhc!kQah_PQ=Uc8f3GE4640-XrkbWSPNt&pF?}v-qQ+@?mFt?N8S=Oh zIs43@{>EwVsUYA^?>;K!JhD92mzc*m#6ocK35*ubux%OagDF=mC{u-YElC2B-YV^I z+Z$%rm5oms=e+hUdHSa~Wc7+REM>z#&IU*VYWXeRpu6>dA6G>MB%!H0F@X%AIlcV1 zX6W9vnMIwJShfYR?HSIZYBB(i??|7*3A&nT#8T}{uGvYPLo<4l8NDYF>4J0XzGXA| zAq?OlzEIv*8n4s^dZ_SbSn{_J3ulJ9QbwJ)pCTwr|Bz?Ge; zmfo#2!4-#6mbj(7inNuJnrLu=XwiLQWnnj=&BK#_=X$QuZ1tAc$|9-!mnp#U9kv6uA1qY#*Wc&_i&}T{ z7IcKn8zFMhHsY9-DpB{F&SlZ4>9(^Mu?yy`BMDxo>z-m@OkzAapKq4P8pU5e70|VI zEQfog8v^CbDryfcoMSoUsH9^wvg+k}UOMV4p2Hiq@dur?oz1169;DPXH^gQF$t?C2 zj8+BHLZ&-Z4(Mj%nW-HWxxuR$e9~M!Sr+wn_Fb{s+#4y(w>spDDGP&5GZW|RSCM2P zjv|E|3JqH0#hue;^74essi+!7G|ic+brRSZWxRz6n#IsPUfm3bT>pwz;4PYIfQa3- z^}tUloRfF_e7rY>bo)^}vnAX;VUHH0`8@{*ab zgGH~r#<7;i<;-JqgG0B8TV0|@^w_D4h>se1V?jn+59sQ!ukT|qqID5dyM;(uz$eYw z8`eTGI!16rPz#Fdu#p*&H*Df{72uv5N+*Y%0I$s94NL>KO=Dzj=q8@buOo@pkiKfI zR;{^eKrLoU)l`zH2aNxEQu0J^g7i{~9OBJsQ8xksxk9($md>`qa9p6{#yn8@btBFx z-*|Yk%pFgv%Gw^EQSej#Gr^9lSA_`T^YO#;JD5$P{QB~QNn-fZrmzvna~j`}6U&B5 zRkoTo(^4`{zgB+<_A_4W>5WPA%4S)sKKe9Ej1iZZVKf=35*PpSnzGM| zC&y7F1vR2ohKEN;$GPCpNX}&cNdd4iWIR!uh3gpoQx8sR|Ft(*#ZqY+aB%BXO{cGT*~Z9QF$-ck(HqKT(qpzOm_%q4T8c zT=$}+w3)~9~O3yrX z>ZPL8Neu0s!O>zp6Nas*ZS_tWvI;mp7{E?#;Z#P9+2NFiqI1BzoQEtzi9c%N7_n7Z zrEDB)2_k`c&@}-`sFUSf>2`_&lA@Q9vIHBxDp#q)f2@X4C0(CvfA96)qo<0193!qj zXorCB-4||~hW!6l;zm`j|0dTNl?(?zcWcnSQVs^;89opgWKxe0Wh8eVV=b+wPZlg_ z`ZAG|4rDMhh#oqw!@rsML(eF*{kN)nId3HXZ#?l(KO2@@SfLL?e;?t58&Zsqn2;k4 z6=p0)H8#RlBLq^fN~DY@`vkZq%&2{MBH6m}A)=$kEL>3m?5%gKcS8gCZ(jIxa%F|y zKP=psOWepR^kcxg7I?qn(aDRXMV%(-EYQZp8uS%b*$yrBX9@pPZC;WxuOHJkUzWTNc-GnAA9GqqTa zfRFhavc&gr>!^x!lai=^p_9;xO6J33ApOgmaGh8*at~mtwK*h(`tFn;nR5{{f$5GS z)VXo4W_TJkCI?GxOXsR)Ly3d*&^L^v1spG`bIy!_&As|(gapi{8%K+d&ngM_8pZK7 z)zCN=w32TI!3l`Q>;7{WHS5ks7lH*BqjKJImR#iHaAABW4N=^8DV>4cbwhC6&c%O) zorKQF{<{3Jm|RKVThv38&sakaElmPj(Xfe#^gbGf6*aicR0aCm(E)SCc7!=-#552Q z$v%4uoCoov9gI|wI)FSAoFNHrpAsWnJAa|4@~awn-=ury`o03q+PFGM} zp0iD8Dc^>5L0X1Yk}@d^G$$Tw0SVdyKUTGf=2aF%q^nl3TcUh+=^QobCQ!uv5nlv6 zN~-NEJ1kZ|3C%#y3DR!&xMsO3l~h{kx!B2^l0Wut`Sb~Fi2#%2$Uc2VkY0ao6ojPE z{QUCrqK}V13AnkrIr#>hpWg1??!f8c z;@sTydHT3~9`8KEZ{zfEeX`f@^hmuRF~gnpUA@#&?{t_ef$i)vDxZx2(0K9;Htwai@ia z)BTPEp8pkRp>i|WLSGO<7#}nzM{5;gl*%_|KjY8Dg? za?&UQxE_LM#D7TAn$7O{6!rmxuec^z`Q>J-t)J1;di9W=g{)N06b$^Wwp5wVXKVGz6{Is#>gM7J8pgnoTHI_7Pk* zbOp0l;8pj1D!}17l|cBB3hEdA`nnW_X&qBz@~*hJloiZk(c-M%&bMFaeoFPx&La7(V=27=EKLrp!n?d#RMqgzberj1L#STIy`^=|N?U3`h(@crcjBLF zbGdNuwaI>GUK1aMkGACW=a~aZvg`y5WQzFTKO&;;JWQ~6_E1+t;Iyj3bfW%C+om%9 z(NwdFxzfzxIN2P{rxgZ?-rQ2&A7$y-!$QJB_rtkMh_@Wa!HvB5TSW$OduKpO_Mx2j z9y0TI(YFF_B+1$A*&q5O>rN4TdJ0QXrDyMF2`iS|ZRL?k5@rKnXPY|FcFGSe*_1}@ zqd0|?$KuhYk9Ui!cga&NjN9sOuO$Hl{YHM~cN$)X;h=0r_wNMq&oWf{_t(VtlRUok z!Vcgk6dXar3`?l9De0(e2uZsv+DBqnRNSR2)>Ne{dJ9F6o)KBuuWgf+5G`eDr#pjl z)J+~HwifR#%usn2o}U*t)4KJ7krWD5{Oq%k!+ia2NSg0b54?%bgLUI}``0u&^L6$vsFazc(zU}wBbKH8405Ou8i9#5>9qZl<2H`C&B$_?=U`4Yx{v^78GzYiGFpG2h z9=O&3;n!Ehtd*SiY$vIT^F$@H!R7D102t^^402GWGF z7>GU=%~P+X@I2<%9Uhk{m<6^h5@BRS-50Mqu1aDB4SG(QY3M(UTOHV7EW%%JgyLG% z+v5%$5TQEeR&1}7%9sPBxH>{-C6q6SP|4(eL@4?C#u$)d+^<%C_%`@gzYW(2d=?t< zfF?R8%e}{Ca)2i#@)uZSVp>eQW^1wwmwz!quC?61Dt4$^(L4Ky7dh|CM~g@UVUw2P zv>BzX#{dDPb!Q^xA^5dKm5k9imeJ!F--27Y;cVNWa;|}ERic-p2m{}HiN zX>`fQt$dLftB)%e3V~fvR>lFopxNcl{w0E``ckTL`7cX%P|xF0>oSBy#6ZmGycxSz zjDAbq90nnA6AFxZHwl50wWi!GGAM?q+PhzRhAmxl6rZf26o<;1yzb(eTW);|eI5u?+ez2D((8~Dk5&gnN+^CH@M-A8ehSwjgfI6nzqyl@IFSC}#NI_imEo1~69aVcQ=VJmRk@b1SrQ>?jlcnCU&=vd(d~q?? z>r0LyQmk&*dzjZS*A-6(?|?GiN)SW@m?1n}bfEx_b?ag`@U8)1$32uD=Nz$0r7U~v zmj_W%eYC<*1{l?Z7x1x#+^NvSvIiceJ-?*`63C~49{{?`z?!_YB!DHY*l$E}o%787 z`F!VSG01OLBi9Xi{0dH~wl42bE*}%lOGhSdD+3A!1bjpyI|A^04m~OPc}jT5>`?FL z5Rw!6q*hjfT=qhci8<#np9IFEVOGvW4!Eh6pMBIAbo95I#^6|D6^tK6880TEUSJB$ zCdSJ32Q+>0`+fP;KfL_ZEMFk}K34VH0Ok~cyR@9Xn@_v%*Nd5%=jYiS{jY=P_kDej zlb5_f58txycb^Y)mpgiKB5rO!Kw}P7xG*1qOa5Wfk*A6(4yuanhE8YCU0C_R`m6BL z&WeSNuaE(&J(M3X$0J1`43s2?`gJ=4OKTCN;w{&j4Q{!lG%8htatrZOtw@c$8)_)@(XH$MJd)z zlM3bWBgmXl!47-VCScJKm?{T&N?9+#*wfO-!UDGxFA1X;UBzFiRkh=DZ`C=W7ZzSU zDU>SMq-qIu3r{6wzeB<_l-E+D5jKAiVcARgUrHuFWaw7k-H3J791_rwyKcDpg;MPY ztuJTv%kj+jMo?}eJU<&+r&n2AJkSD}zkIV9)gLHdl^ z?jQn%Ro5Hx9V%rpQur<%DE6JSRt@rJ$aFAg1{srBkX_Go!i@1!I>J&cEM~9t0L7HmvR>(GB*iP0<=ChuHPTxP6W@0SFYK@rL;MsG z#I`nwC&Hk+6f4KP&xoLo<(n3mWes4r75-fc`HF))f}RxZl#9axMPLdF!PYGs>690b zcRuDtsySQYu|nfF?(IPRCDEOz#=Hzy*2IWEf$}vZgW$;=%wqC5$?ao-pq}r3if_wG zic$JJY)IF6O)7i*)+-Y^kzK%o?7RHFAV7C)c~r{N?y`oak262(cylS%8(Dk3Kvsi zA#>$DXTRyGo{x|oEe;ibavINP{od4}0-L=@bI71?q|bDMU#>;W;-_yh3+Ce(;M#Y6 zA~n!{Y4CAQ^Ed>e)BO!HuWxMO9pH(eAXA}8+0FMM?TdHRgayd>xe@e{9PZs;h=IOy z6T<^xxurVRJWm@5IfC+b2jFcB!1FVo?k%LyGlnx?z$wWXf#dB49`XKWYPP;^#YI>DdX%AV|#`653Ca6>Tj-76IWX}l3J4KoK0&VGb9m%~V#Ut1& zIT-QcCX5U_MWQkK(&(Lqo6AqDbhMx$SvU+_NN#ZfxZ$;&-OK6y_{|X}uGTvQ%q1;# zT+5p}7X&|}BiwN+m8?xnu=+G9O--AW*r-M8lC_<}tOOR=t4x}tzK5!cdou+q6PEep z-HI04u&@j<^(qUsb-eLHo^3NMPAgkzm3j`lo8TF1c?%k2(&| zurj6h-Ha{Wa1f;XbuZ|f5M-h1bA7e0K6Zvs6=2r$9q%Cw&QWu)Xqe!aW#i=`PHV4Y z;)8pH6rc7TaCWetTGLfobIbT+67f<0<_SO#;tIw^|ukdOGT zuqsQE^QhV)xoiZ{x7?r#NPtA*|8nlTT2Ch+S@U^wD=oCXas~X0T?k+0(BVe9Ky_u7 z812}P5zA?Y3$32%lX@`Nlavin8uN)myRWPo&$Q~>Fp)~(mg^AuG(NLiHqeHZ&|$=n z#ehB4DWxQKaV)R}^ek0z^TLhD?}lZyeF+z-k)u-c^kHW>Lb2Jpb}cApsfsKyS{qMN zP-AP4q}bSys;&K#VrxxQ zEejAbYkx%sl3JlKH44%<=;FTD=?0FGeJ-bB+E~Tc1IVwgPI~%tsH-^is=Lv)7@* zl}-=23G9q9fSY8?Cc`V|bge_HDWQK@c9waI=u)~|YSK_&^1tn+Zc@QH0fW~qCQyM* zmHb^R9e}1-Rg*kKCg=A2V2fRbxlpew!*$Oid$sBob9`o z2qJQEikKl%<0LDFw8sB}r$8Nb?UC`ytiqo{%u9$QDTd{hoE?Gd;gHsw8R8?iU8!lB z*+A{NRSXu=pQ@DOR$niO)pA+k*py~>VR!3#8(0>vVpX!xIDIJfR&DV_AX$(qs~eu_ zFzegj-&Y!)fYF~ZM1jI$YIjVZ6n{N+&ordg!J5@A>-10?N0t+&06mZF8e zfz&a0!piZHA2JRF-VNg0B;4gM^pTrX@4MAXQsF3;CJHA`tPx7Ooq)5)ERrUb@DQ0( z09up2Pue;?{-}E6r9&%0qE}BV(;nKEir%FqQmsW^-KxMw9TLQ)*v1sY7{!v}sT7Pq z**hn=R?P~@h7GE23|Y+vvJy3bHqw`Z@vUb3=*;xWN|Q<^oOf`-3f_x1(sRzzrVIN= zmMjQOtf#ZtKV_sFTsZf@Vio(A3;PBoOD&@1Ih1=aD+R(|4oBO*T05km)~*y6#$|b= z!)j2cN3nK<;KU)joMotH2y%5DXm!o7df`;9Y4S*c?8AdNrm+}{hVE#e@FY^D0?KwX zo~f^H*134UXyj4$NWdMsZEGcXTthIjYD#dJX4xkOn6X)pvCiE$~X8V^na-*(#hj=c&PxcDmCeBbeVdY?A?sw=W!@xgeH0Gn&_tuez5aiD=q*f?E{WG4ge zCP%TM%w1a_KY^WtC&>rwrAnd``28_Qo-l!S80eF~K|73Mrx1uQFa)`--A3X~o^Yqt z>+@=h&+C_hVW1&;U)_6S4|?em{G_L66;Raz3R1G^&kR0wmU$k4gD>JJthIImT#xL&-0UMopkRY$c$SLzd z;F!Jh6qx9(STA;%Ja}&ov8&$ewVfS(eERa%vp=ZHB$A|h^QU`aJ06Il4#qz?f zXOYbM8cEk8m|I1EDJ+!UxhAuYC?g;uGD=jG2OM%S^f{(6K3bHlZ} z;W^;TIbWDQaj~C}N;mbwDjjj5G6T?>a%~ljjki@+AD-ECyZ$)?rlJ^XS}KcGs5&NL zGri-4^Gb>vA@tvHsk^mK^E9rH$#1z|S;=&M$LW;u`+y)06XyBX_?iDnyhp=;mdyC7 zwzCxT$^_?f4rO)k!8%@uzMrR4`Atk&@8tSQ0X=3!w~}+-xep4*nK=Y3yfQm^?q@6v zE$fgNHSA?1C$f<;POqM`i7Po~h=91$T~Vtx5fT>%QVdwDdNKa8tL5cdoVU72y+;jO zG|Z%?>1+E%@|=Pp$8?bVX?5eG^{eK+@zYSsKvPF`twe^5@?*9K3eM2qPq``A)5w)R z?~3Y0niKJ1u24CHv*uj!hL9?$&4H`L0-Cdv^yHjNl_DSl*iq7}c%f}5I-3ChA$vq! zmcw$(KMDBpgo6iITZnzl>Y<>V>0#?!b&Be#!7OJyp~^5)skLRT{(eIYQ)xGY6gpB$ zbFM&@+HmYoG#*|SA#NYC>=uJl6|Y}Qw6-%5v${KH!G?=cIZq;~5vQtuU-9nY1*y_? zeQzZ#xIQ0j(m)#S>Ok5uTaZ*DsOK%Ngr(T2K)06d(@GrDA9w7appO7<12SG7kez+yu-}**H$(533v~H^<)**n!_^*p@>Ql0w ztELEPR@=kJ)`G<452Qlw$n%>Bc-P^NtLw`no?la)eE_DnXeeD)xVt(7Y_Wv&XC(=RE7xkj@~)FrT8LkuHc)KTi&Qh|53eDc5x|gz7U}%c3V;8way>hiyzfl;1kV1O zeHeEA@AYy!;i};~(~V`tCqd~W@mtd^Qwi+8z{oHDmduyFZ~K1$czxWSwb?B|pRbR1 z`x;E9hsOZBAfnV5SemWTHb5SUJ}FuOBgJ{SWTn7vfq zhL|7`bz(AFWDoSmJljt6WzNdwgc${uQnDnA3ZedACPXkXZRNZ?`yqx%K^A}So{d78 z#7-9~U5A6$U9}ekU55ny$5IbNBs^E453wOyZbeTLY)=I(K_~hgZH2BOh@JhTKarCv zQ1jb3x+f#!_7DDclNyUOz!e4r4{Mw0GVRF({l$uklb)^Mnr8yp=_;nW(Ti=V*{&3L z9A~tSC_Xj6>>n=&4KZ?lU%SVwakG+|Rx$;a;poK>pBq-#F${1iHJBvy$SS2Cg0}O- zw$|3j<-G`1B#ZdArb8#Hc; zIPy^RWYtt!qR&0GH`TU<;4bCBHyi}nar@g2hqXYn!+_J(paw-0tsTJDwW_kobb2; za@Q)q0)TpGEzf$*Y?P@&a12o2cg;T(CjiWuljOm3jzY#vT4CZMy(< z-yal;Da5jW#-OP62QuVubP-RK3T!y4lFWu3m91Mo!{F}=RB!3xXCno9;{%v%h_~Z7 z`_US6@-s(4Sg{cl8N__TjLbdXTt(BFWop?8lUS92daK+Kjz4QfB(_U)qDzCp+*jS281A<}*s_aaj9d0lWkLxf6Joz?hbO$Qr z&%2zZ)vpb1I`qhY7!bW`X(_)HcfvLchUs3FKwdXG6Qg6eOBVOvG}whHSllHnAPXro z{e}M|KBPlWOj5K-^@S&ft}6#6l`WxSU3SdA)4=6c&1e zdbI*n1x+b7q=j`$EQ|5%MZOutO zd_g}c!|(0t>G^i>y}LStx3}lt)YH@L2JE#abjQb;J+aH`XiY93b-}mUi{txtw`1z| z`kkmv;)JyUOWOjQmX=>aN%^{kDY#Iz22FKTSYptLm+f@-yxx8d4?l%1wx#i_!MI%G zfvD8{Eu?USS*x}=o*q7ZKEA(v9z5N@Jl=mEJX>2EwpijwlzUQ}D$<^9gl-oHaaCoH zrmQ`m``CzT!>9Y+xp-l3Lubd|-s$~$)4S`>_phTaGw|xFT6KN}j~fFG6noR?2+*{N zYe|IJ`4pEtK1ttciL0TR_%quQ%0iLCjTq4S`US-Fg|RLlZZ6^ECPgUm z%roR0No(O&tSX2<`zfE;bY*om_n{kYMfYe^FxvouKtgsFvKO~Bw+bkg6AahtxB1M4 z{W8v4%hye#a-Xmub>NYj=J9cD+sk`WeA;oMJOw-JDEdh3-yTOa9k&+kDixpA(_iMM z%yhz2lbM0ceR8nl^AWTF?^ zNEO)TZ%60Vg-6aRLnvM=sV)?PY$bcnCE_T9`&56v+Oe{^(Rwwk=*uOJohpVhH zkP&f>|C9<%c+DZXfq~%;;?NSNbbDosrC2!Vs!v*JP$duI`a-K~lijT{a zW-X?<+0x z0A6syJm7)?TpDgJfVU9t0|xY+#NS7$kEw8v4;7HTnq@m}iZ>;0eL9!*U>lWt1jd?4o$UQsI=nVHut71e z@SH%ks9@{%spb#&I&wBX!rHmM4#!CI&bhxoFipqWArx85CXYuJtki(Al|M!r=LXa) zkf}iK-?SF2@YmmKlsX56V+SkTsF*$5vu^{|fsu^L&@B7fd3-HC9P0utKC^G%ydv^W zx}mJ@>bVA*$bbT!IA0#wduJCt6db?xm5$+6v1S0H5aMsIH|*kd>~}lXl%yN3tDpE4 z0@TMq^ZH}F6y&+=oC3KDSGIZNKz5>*(PH??~``@s7SXiZ5iLw*}lnzk!GU93W^Je@402`D(jYJ?9PQpqP~x z!CPJwsPE!cnP;Jy@FlGV?DWpFXnQAa{F@1|FtUgpu z6CcO&34cjTDcmW)5-$opOwYt5Ita@66HdYI@LN6#4W0;H$&ahDOVLN zH+TNEvWkP2LXW-0&fl2|doQ)tMkiw@>MCC@cj!h^1Gm%aXSE^mV^+8^H1M+dZpGHL zTD4M!dZ37&tAVRq5@F(}YvGC{5#MvjoN9eq9FMX|9A2*;xuRB2a#Xev+6$OGpJXkw zAeVHF9Qr-=8lZ{;!ge06TKNnxi7f$@;wQ~F1n^Vb0eqa>#{qq*jZy5%{fMuQF4EJeEl%&6v#fTnBL zt-?=oZf zRjV{ATM6B+2@Tij8ZVVCBTZkmGvM6&DKp(b$rR`Oy`QMUSs?9}rwYtu0^62n{f3SC zg{u-p^gu0CRhiWOBcRX5H<5n1 zN_BvIZ;jmAT@PpKjw}G$>rVOfDST#;YG!rbSRZsT;c$HF# z6Srlw=@der z5X&UOw%W&A=jZI)c)YQrbA?Oq*QN6G8*>tms_?ijGO|;(z^3q-W4$jul>1&e4P0cX zkG7fc#dy*zpDC;hoZF*7m3Df+WYA03*t=GNuV}P`_zo{d%dJYNPdqF2@R#N`&j2~IibTp~ztet|UKR5g;WQb4x?S?~ zKBF$D_RVx&S&_Wx_yr&XCGHO;lWHEZfbr+aw9I9R2f$G^`%SMo9UE>jgK)^3Z?eRF>Yhnpnm))1RjnI zMaG3m?J`+L4;6}Xmlb?0^f=~ym!%K)CYHjF-UY@+X_V*4cv<;Vv-sZc=Eoq0g9kG& zZkY%DNH(L?-$dujDyuThDE?b(Sp)*5#an6Xu_;@(zh9mCLMFtg8=P7)% z4E-m>{5Z=x5^C7nF+}*MH;sJPz>nW&i$|58zZK?_Iqm&xXAn5{#AjePbDuwU`|u7Q z4nCIc_2%an_S{$dY9FZ?`Op46K!p^p4ThrZvj*Z>y8_6&*TMnyz8hJ=f|B$lX3|Dt z1Y-#sp|)9+4KEdAJ}TZgRuOzea$niBs{qfUx`f9!`|lT3kqO`Hrx~%f+MX$#H#J^iV zll>`*zDZ$&zAU_NQ^(&vl5*zWuR4FR%LY{T1px68#`)%VYvxA`SXELZ1>`X=@M$x| z9)r%W2E>7|pvtS)LnW^^iJ8M%6z^#U=^5W6+DrR50adDz4W zyNx~%xuL%YJ-+2io&ig#AAXu#S6 zjU=CD#zay{u5m#0DYHmVB0R9l@*5OH$)pxegAGCph<7}Y$z`!1m zP`gS;*)x^OSfm*k>MS84xFA6h?}2ZTJ#}+WIV|I)*hWqPZ?+OWz>6YE{4mbbZ{V6$ z0DPR@59rwNgL`5B?z5&``v8QOeEjufdchw2Tx6P3{-GSv|3f((pPAIvi&0_>*Djne zC?Y{F1=T)kypL#N4k@P;abzhU=fjo9=3NXSv&A&#=W?~)9WOf0V^-9;F2(k9F!H*C z`$VCfESb6P$DH1VGSsVsPfP<0tR53B?+u3#r~MKkrYqm4q?&Sz6vq5_zDzvzUpPiy88w5>7o8T1XzG`U2*okmc^{Si< z{g!(~5ec19Bzv8iKb z-0u+Y0G{>x7aXI~IQn2R<_%VC)iq%jdwXDszMn#T zzZzeNE*@&s_AEg z-92(pZxom|s>R+rc~FbHSWcxBb%|V0DAt=C7wgH6pcnP8wh@-h){#c|1(9g*8 zL%RK6QQOdDw1=r?ac=ehidyytCSIAGa^}1(LX;e6&Lq5mENl;7POh)xtKqO4db(Qv z?B1Tw2BURna#MwtJhUf!k&|@Bi|EW-8f3q_0%)sB+w=cN;bzXbs!x?^!z+5+9QDm(S973-s)3EeU4j`{LEFYmQjMs(XqdyZ456fS$UqJtuRB#WFuhK#kM-HLRZp z2NN8iudBlQpwJ^eqApY})T`xm#NMKa;c)6g?H13` zg?_*E^k^-4jkcWS|6IVUY0lPQkbE&1}xb#_rT zd?^ae*C_~1p|(O3a@oHXwZyPe0Ihabo3(d@cqOljSELNDuh?B=gR%(Fy3e!L*rsOQ zlOUd>E9p^{-Di?TnyJ}k9h(Lh0)tHvy6yWgSe$^|29o3QDUH0_x1%#eIUALz6h@&C z?@6hXt;XuXo_HnyMkNzlO_)J8!)Js>Kf*ZRw{J(~Es(EyghqpV@m@n?a&LEOmzp1l~IqT>G7a$E$G;P zi=Z}qT@fFhfaAmctSMJV_*XATRzLx0wWey;Q?aQP?Wv<*br-qsby#q$4EU#_abNnOLE^FY+MHsXyi&AjYuk7FO1|q~mOM&+Iw^MaOS?9gn>IgW zCC1u95ArtK8DU(94cB}aGxS~hV)}JR{2bnpC|L&$!cIVd!}z1s^2}AFO{D5-$Z-&I ze3tD5oJ??txz%~|OZgRNJ(21L3E5~nBXq0Qp`zLgUQQwskm+I+302{Ygp5;&L!RqU zLZI^fUiZHx%?c3BJ7ictcC78O{r*g;_y6pR{Gdux{MYOKH+lchH%A-%r?r$N;ygDqwIgQ81?k#_9UN&u4^C;VXaefMn@B|Y=!Sv?(ELo{`_QZT=y3AT*624Ajkz9Th z`&Y5{L`PL0fky(-iT1pklMq?`#M(P^0z(|V)jlKhKBwy}f41pLupXlDDI5pt_5Xc{1zsg?2PGqi}`M@}u5ESx>91KP~7jVUU~X61=u0>uU!tP>x94(%z@8j<8HZ411HxT zhZ$N=MyRUVJ7$Cu6x38-xF;;Cb((3JJgBNaOHUqDRZ*|xLEWlsO8t{f{o;^txorc~ z@4Zv%ls9Qz*=Ty`Dl3uy_51%4lD{+?Niqrnz z^A4!V|NGUO{Qb|t{+r{?{=bg0g8QF}V|cqepizd%<{oGz_dvOsdW5^6Tnjqh2X(FP z7c!PTRfNr76_s_bMjadNF|~g&9dP8Yu8oL@e_Lor?#Gr*K}))P>o=~~H3#$xg}Kjd zU#<tgG`CF9+{r}+T zbs_)9n>U;Me`_fv`oH0Ug2Mka6N4wsZD+Z88X=%1jd$y!lRPT@uDvgK=S1Ko^>5OC zKdn+J|Cf^aTa`xne|((x|2aH-vyuPnD4#a@zf*NhbN>olj11+f{N5(%p1PjS<@8Py z^hLx^)AO-xYVQCuF8?9%e2Z1cKCoGlyrkw6l`XIxO&Ce9g&e^&Yn*avU>tHZi35i6 z^rwhB)w8u#tR=v6%%$C{tkIkM3Af@P|8?eXwle+y@GzhMd;iVx{zm?N zW^l{^RW$ap^sClcR2_TL$?dawv^S|g zA0hQ;cZC=%HSK5Gjz@4Bb*$VPBLTH}xfD@}r>Gkh7Fg`FFtm=9~qA@@(xa+5r0tBgcV+Mk5L zgwQ$6XTwP-G$#B>hcKJzr$DV$4yD4^Dj(?}qGXJ!^Zg`^QLoEHgV*J`ek_TSd{vfS zY^#`^EdPEZYuHJ;>T(qs$D*1BRmKRGoon;`Xp=Chbtju0b<c6@x4Z)+OVpPjb~#Nuj?=%%g0Dal*S9%Y(pd!W0twjk9X zISFtX+2!T`t*!b|@(+JnU&`1goutoXvr5-V=L~c;BvzbxuEnvqZ}&on()(*BMqx{BXF-=tv0)%&~?f~7x{r}j(c?x|+=+qiH%$5Q40 zv7E>sRhr`e58mwO{Xbs4+2H@IrK}_VXZ4BUHqIrhb}s3X2(F~+Wu|>A&GjVH2_?4) zAak3ru3TO&CX4pAhv%Q~t<)e=5*pvidjDM0yfqKTCd1q3mf@`?1DbU5fGbRWmOW4R z%X(H#>**&v+muYH(*Ku}`AwAu{r~XjxETMl!GB**arFQG^vG%AY-`UWq=+lgWS?j- z(~{KvqC4Iu{hJGKw)&v&5Z2Hs>w_b${)1XetI)C~dij^C8jmeupLVI-|C3{z3|a4G z0c?u@eRcdgzyBW`ZSa5AQrhkR6)>Td7Q@oRfAX6=GIY%mm!ZcSXyv4= z2MV;Z`i?lDQwuV}ZEQ6jNAc|IG)MuW(oMm>l741;P?h{&PUbIF8sz`c;bA`i*XyHA z{)hFH4)MRWX~If^{~Y#-_F1#4C3A~yN@b&}d$TH~l#w4x%qW$=HP~d6T3f3gb<1AX zWoCL)*O27O*r$tT?27zGF85ib*{s@La@A+Qs#=tpG)rx-pPCb3Rl4yyX*Ru5T{Y*u zsvk$^lvgc>ruwUz1AUv*`(y8lf1c8y|MkrLrAm|ifB#j%|L654|HpdDr~2GcX?*w7 z&;8XWAisXYex7;1tl0cDh0EM*jPu_$|go=+rBQY{CAQSFtI7{W^G6Q8rkIzBVIN&gCNhx+ct=9?!VN<#2O5@I#TBZ~^X8;q$avLk^a~k3 za~}GIj5iEaU?5KMbc!g9`3!|zY(*;ia#hS!THmwftkT|WxU9fsI>>k3%vM+Jv*o$E zPHk8Z-HXu(r)4}yVqZR11G+lC%h$AgRx`aEu8rriJ=*YD9iPedxlVQ~t?+Z;y3&rV zkolgm4Rcev7is?;&QLJN(~wZKiaHwnKlWc8y~+8193CBP;y>3?z5v&db3{W4P@{MN zJj_rC#xV|Lriuu9cW{c>@WmJ4c7_>XaU_Eb0Gpv80Mmesm8izybXQ)faeR;DzG3|i zL;uAWAVgCs62W#v(FA`)zPeoczjlV;eHbhN3FVifNgzTL1UN*)7sK<>hY=?fz4!v0 zk@=j2;K#EO@G)gChEvS<yfCQ^My_@bpEgT>m#0r{`A}!@0kjI%?Pd!Qtzo|JT9s@y7pqE#*HV00TS?zuxlx zfA+q9xs6-dbN}X3pvI}SmEw?c?33%mO;W{3a;%MPYbE8n?rL2{a0Vn14+nDrjAS}a z)xO5Q-ag4KbOV@gijpW<&WWqe2}RDJ8$hE!erPmUs+kOa`hhR!M9JvK0FLpDhC@j& z&FWg@(_AqbiBz*xLq3*~o=8XrABIcH(;?g?qVQIqM9f8~B~4XErFkby=K;Cov0nTb zJlow3hClr<%zo%sn*&?FI;3W{ybc*@9JQl7H7eDywgg6O>YsKpRU$$er{?7&;# zhmjcRl**7@3i%-sG*$ybfD+UsQ-mr)%g?>GCHPLUP$5lY$CDXW(7!PQ*2*k_S4wha zK2s9281_u!a>G%sv{-s45?g`&j)CoaL{lR3lo)L%^uksj6L}0=n7>#&XMMS7*{1>^ID{xSGBBc?y~P|!>_dXFBFv!nm9f70tsCX+~{N+hgj zJS-wvCJ2pV;;0bEko&@NHQ-_xEvVG0^kSZw@T=GRkWX2p6Jk$V6zOuN5qzGfO4B5P zZ%Cy}Bn+!BkTATkS9pClc>2%5uHhOwahPf;;yePRst1-Icp_6Fw9=Ah3gw-rGf9<} zd8Bj6jF462-j%sS83tJ~JX*JcxZTBGZ>&gW+7~mT7gYBN*W{8XiSLg2pLt)h|IE2$ zG1)`OKXH=3?dcG--qJ=STMAHPc6{1Z z098sv92T4(Ivh#U2C}5e1Bx?JO$GqEGhvh=bP%cqC0uEI(`TTVuQMv?l4+wbK~5OT zpr_f>3&y1nZZ9wjHbfthUtpI#DYen-eJ025FY=k3tujVF8##iv%_kK}4%Lmw=icYA z`1iT;pUNlrtM{2qTKAnNc@Q|1}=PAm$m1eXCsW4&(?N{<2jAi}B z=32yU6b~Tz1LaP!WJYXehH2FI`0VJka@B^R&0|&_juz@&*%yYe($b9TXEK4z03qjL zL<92+K8K3va=|1n@x3uIp_qw@V)Df0kHpheC1-;O3El;n8?R;O3Z7!SAZz@*__yzO z7rV<{1?=nz746!Ln(}zhXrd|uzE|ejjC83rYa^D%|5{I}$h~QsTsQN^F*B1J zDn2(Z;NBROSDA+aIrNjyaIk^rz7-9a6R-02#^B*-L8i>DVy6-(^&}eb`ofq?7U>W( zY9O%+%SKGD=ptqe)z)j^I1NUok$wQBaDq`#qaE~v4?K;7QLu+r`4CzQ`hgLzU=+-R z2(9da4me>mCRrLW(>LpA;yW-v_4K96D5d^q!%W}(Wf%axVE+ceHg29b>zfRKr+LLh zrke zR`(Zx)dmmR#gT*S!O)(x8#S{ep`F$hsjjqMo>JL>p@wwLVrW zHb_K76QI))bHwmwrLi`a3rNBGja}`bVE?8V6!D%hY9gK@5N1oRAuxq_6@UrX6B&1; zA#Ugns*FW2I7zYs`?z7*2{bSz(~%lGstGlvpw%pu4VGzU>Vg%u9Cz& zXQn(8u^NzZq8B2cFNpPQ`eezagyV|j!wco24~Ss7A`3oWBr77BiLK0_!~phiWnS)+ zIEM+~T9Jg!X|y6rQ=KbveuRJe>n4$pLV*~CF^EObF_7t(5&Yr#3z;-19m|tyjZWr8woV za(;m)@vdazZ*v#3j&@}hogJMTRj*LTwJnWPril?|!e?Y9a!{)l&PxkC46!nBi!N!Z z!P^qcWuBr55hj*0I`TduIt<0oTz$U6%Ffue<0fmjb zHuj>7V_`M_f~p1I6EgdEBi3g}r`PM~EB88q_rF*#FMxV`4~5d>gj2;@g!WbOqmP7( zi;KvUnpv-^H8ARtNF$bMf39AI#rs1V$019#Mf;BxfWROhKW)l$jQ(-y|qB_L=YeMp^Vo~ey{UBCRjHs|BylCIE?I|Q{@x+dQBxK%s)3+GWawk)eR zcKN!ci^He|kU%A@>fi2zv2#BHq3Sgy5l?u&tcS-AlnF+E1ocZxg3Fy;K7(6&=t#<}8isk%`i3K$rBBNX?UxSUO9K22ew( z9``PE;_&2jY%QiUgNkX`j7&=bg)@MD!_s)UuG-rgKG(_gsrCPhNaK9`?vxykUy~Ez z=FK|Z#1cddf0FDYd@$V98lH6TroM%dCZ2KE<ejrmAZ=&We~qYz3i*_3cgf@D%$ z@R-RvoFApt37dmQf8eS#)OzAj#zKtjN~0+`Jv#-z*G2@`0K|oi#^xF8OOumoECXh) z(U2@k}&z9j3X~v|P2x;UnNZ zbomdVlk-+ZSLx|ROx@KlOru7Wo zx@ua@_YY-iyI0|}?{N|CyGy*nyMeU+ zvUl7z6`fN0-QekNjmyv(3pJcaS(Djehos(E5t*LOqwH|}x>^pfO<+y`jHRm?b8@hx zJRz409xVU@xD~v)sTN=uI@@??ecU6bG#b$P zT(UXU>`d5P0`Pz~bem3o9`6+xuUfz;P8m7(&BklpQt3ABh8RzVMzm7+|4&21-|$h< zk~_5luGEgrLW3X|z9CO8IJ@i<7IWQ)U3|BV3>`ItD@Agg(3Dl>M){*8EoghFR!XyF zY3qo8Vjk1Mlo&AsEY(IIp}pT10YX42F(nBGhb9WQUZVwP7q|t}NLm3-NoH6jey2I1 za}tKRVs#HPj3fj3q=_2UcW%}Wy3L>o6P~hA3z4X>_z`+<8iw=RfG+ih#VCt_X+$(dD1wp+fEW{^@~H>; z3ufGYSiz5Ku;3wqKLU5^CKv_Z78}*y27dxtaAY17vwF6p<;muxWA;-EQ#6f%@2T-q z?OcGAOe(bkRK_fmSRUu^y_sOXhknte)-D7d<~5?>J+Fj8{b!OF84?Y0nlITw8i|;vb%Bwy=-IPp|Evrb zWUes#{MEw-x_!bj5iQJ!EqRh~Wj>`eg)!txy!d|isrhKh;ndnc;4$RfbG*fb&;dE4 zADAMUWRY>DBD=!}HoBlQ<1C45T-d$wJ+aIrQ*JA&k=TGw8DAh1se;^P_dJ5cFO>OF zkB<*cw2A0SI2Qx^s$skQj^fKKS&=29sr4&#E>lIs3{7Sx1?ZO~1Ox@JlpOEIk{h6VQ2 z`q@G30S8r$?22*Zsnl*U9v1M(kbYI&l0mJ}5GWST?Nx5D;Pp$T7RQe0re8I$1rq>=aP zAS(-D1K>`wIFA^K#bpZiH_jyxy66t zNz1(>dywi{HTRx8?hF6QVp__TFaeaqA@S;xrfJ-OCy2p-z!6t^P2w7=I7MQX3dr@m z1pL9K05^4;_ZH2_!AwwaD!C#W+#@)Z<943Frz{_OVsRrZ9l!%FG~lrR1egM$z-0In zNd<8|Ds|+^83w zLgq2q>QuweOj-OB&cca=O?eu_#KW;*&q7>>)`4|1+Bg{qhJ(bn2nTfiKXayZ`yM0? zsaa$GsL531Y20>Wk1dNuYz-U}!&CuPpAy7Aa;2>d!c0UIa%>h1Q)z7&T0T&v1*pRU zsV`+PcY*=OBCg~xcP&g2?{g({2#|ly<2ln6Frtwi-C@Q6_Ago@*wlMZ1W(JY<<_2; z>t?jJLDA0Vt#>L~oh0wydu*_4l!Ca*ri7#&RiCg&beb#nXP2~vCCL`^)$YRjY6gc|GKd;vFE1}b-8718ecJp6I< zJf=+_sIS)bRMuRywHeA!8w4%N?wcm#sMzdSlGr(3Iu@*vGg4OQ?Rc|qxUj$#Tj@Y(v@#c_i0T%z#Lv|HFULVTs zqvREd-xw;dyj8->t1vk@OAZ}p|B~2veFEbM93`)6z>T2tio0-29Jb~zw&8YH zM^K#zUOk@b1oFyVx&tt8vo6$Oy)N$S=w2uDTSfSsVfL|H8`QUE9KSH!w}$;X?zfHc z?8N@cjd$RGH^KYdETPP}<>cV0O6!@B`jt_3B~A4F`7`s^tFyD?Q@AO&E`^)5i0P=M zcE8`a;f|rutE`U7xq3p!SpKA&M-7-x3+7i*V0Q|xRw+FcfVINaeSoYLf?YssHHChg zaBIaj-UxGDqdn^2YaOiGE(q&wLdGjL(;=n}sO%QdY}GaB!f4k~+z78-L#C8v443EJ zEodt;mTHzT!_aCFi3Cqw4|2)d(8k{+H}df|W9gZDi|o0#uoi$?5Z|KtriD(CId=6y zeAA%e!HJ_$NNFbrP0+q{DRN~=Za}&^OUJ$;&kOsQ30E+!i%uruuG{;sK*p;jLYrcP zC0vU3NhD+Pq~OC3#LmVXZc-2OxJ-~01vsHle^PM(#(A0Z+E|! z;b`3T-CW!5@C*Csa43sC*^1%zP>kcXxYoQ>F`*`B)gI89S@?LbJe0Cr&@5vq>*(8n zoLheFJ>SNnz2CUU>AE8pein+%g?pI>xrGVXVBojPGfvo9QM0CVFkM}kyBtbcc2HmJ zzSwmRD*C4YVn8K*qtMFQ)(uTl_gzV`<gri-O0^uQLo;4UysS^S~wW}@n=7< znlBh(mDIo~nqU+J{or>og@^I4hFSBRTVgyV&3K4^2y!83nCHBv5}k*AlAsDJPHKxVd5xG|Sv; zBRC~}ozcG=qjy@Nanp5C<8%|_*C2db16bo>oe?~AwvTF?yyn-5k4R8dVsbbGP_fts zD;WF#|M&mdVj<3|c%y1Je+SoM*{zK>mj9>y&6e}QEKDrf0LglgjaL+^FP+^hWYcui zWhEloip~5krmiV_n$Zr&fSlK+X#*Qr>(m#q{4G*_57xid^O_cG_WQW{0Pcs)m{e}< zy6OhSWKNDnJi+RekU?w1<{Kz;Z_7HY%A1zR*&aXUC6xvzjg6KWkk`4+ZDETj$rZnF z8J%9AUW>nb@>MoffLjoVvVCfHY5(B(OleZc+FBx-qIMeCKkyWS=}um{PVTE_{SM3U6?D zTB4o69dQ2L>-1NzRV?eZinkYKy<+8LOS+|4@@;d0szsJ|8llh|7K8*HW4}5U#uP8O zt@~(s#3_O2mbubkti5#xjZ?tE@Kh#^nCn%yips95oV-ui=7d3^@AB<06!#zsgF35sq|>TIZZsZsJ`*NeYseS8;Uiyww~aJ!gY2!; zZ;gYkse#)M&fiGbJtJWqBmclqur4m|3jyn*|3J~NE_T0K0_no+KskBCQLWC--2tNIufg3E(Q4kH zdkt*MQ@cUXxf$VZigyxzwlHqI;q@5hxI3&HxVfs5bz31SxW8@!L zaN5P?mV;9-&zsVJpuuSuyX%6}-yd>t+SQ@Ic5vF&>jw~=cC+^7gVQeFzRuwEir#$9 z!D(05?m9T_ru)|mPP_X00D{x?tbOIdX=mrQ5}d+;pff3Ql(xL%Q$d*&GoBP> zQq<;WX8fz2P>mhu2&=b~|u66$YWug;2X_gZI+noKti5!IV*a{Ap3IlT_v z(86gWMC)w7$@25fSq3p2ds^>oqjxNcSPaYe;O51?{3wN0$G<^fqat6;*1D>EZAf?H z>Rv7FZL_?WXXwT^Sd1Hp!42w5`B%THyDW;?{8}!TsFj^?9cZ&xodqYF6W(2OVjY27 z)|)j|_3Fk7w4L`V0&aD|CB0J^Kh#BE2hv%EwL6YGo2`gMAz zbr1?F52Xfz=jcA_A9!M2wGZqA+bnue7O`vBJ6I>n^@0aC(qmT^I_OeKuIxbJDBirl zK}#>YsvNYD+o-z11~iJrar;}1n&hruz|g;W?c#mw#Acmm?^ayxNn0$gwbI|=JQWhX zuk)AqoGtEm^GiqBN6k{jrwJ=LwZ$!%YWT=<6e6V^S1${~;kCxg_i4dsq8Csb@3;|y zJoTHLHp(yEN88mrHi$*;YP25ooj@&J`6AT|uhh*Q@lE5Npev`o1tSiwKjFpYj9ig_ zK7amfP$ToOFDClZ(|%P2#&Yz#_2P?HipV_{46Fob#pE@pIqbJ|5Blc)*}M*+!)sb9 z5ZXIHxB};!OBNH6l7nAQEpCZ0{@$BI^10VTr)@Jv)-WBmGv%s5;-ON-3{+%dILz1yRvQ*ptHX;dVlX>SVn+y#$WOfI2*sU3MY#Q^VA z=BrLuO>OizS z0&6t9*1z63*J936sP!A8K0~B<$ux!f=x9dHjRFa)PRpF@MLr!wVmVBOyc8n-edr(C z>Gg{E4HJ@X_lKZ>BhvoREh+Kz{T!X#y zU-pj4JNrC21k|P4&2mNM?*>nIhbG)Ik3}Gaq1EP64d+o7I?^GxO$~b;u;&INDf2X> z8Td3m=;xlqvf#nVBBkKP(2ce8P*So7f$@Zr#7)5U-*nEGap*B2RsDkHuB_V>TP|NZ^% W?|)y(zyAsV0RR6{w!GK?vJL/dev/null +helm pull kedacore/keda --version "$VERSION" --destination "$DIR" + +sed -i "s/keda-.*\.tgz/keda-${VERSION}.tgz/" "$DIR/main.tf" + +echo "Updated to KEDA $VERSION" diff --git a/spartan/terraform/deploy-keda/variables.tf b/spartan/terraform/deploy-keda/variables.tf new file mode 100644 index 000000000000..c6ac69101569 --- /dev/null +++ b/spartan/terraform/deploy-keda/variables.tf @@ -0,0 +1,18 @@ +variable "GKE_CLUSTER_CONTEXT" { + description = "GKE cluster context" + type = string + default = "gke_testnet-440309_us-west1-a_aztec-gke-private" +} + +variable "RELEASE_NAME" { + description = "Helm release name for KEDA" + type = string + default = "keda" +} + +variable "KEDA_NAMESPACE" { + description = "Namespace to install KEDA into" + type = string + default = "keda" +} + From 008caa121d20bd6335f007348d5df5c4a4e84dee Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 28 May 2026 16:45:07 +0100 Subject: [PATCH 24/27] chore: add KEDA prover agent autoscaling (#23554) Stack: 1. #23553: Add KEDA deployment module 2. #23554 (this): Add KEDA prover agent autoscaling Adds KEDA-based prover agent autoscaling bands for next-net and testnet. --- .../templates/agent-scaledobject.yaml | 40 ++++ spartan/aztec-prover-stack/values.yaml | 11 ++ spartan/environments/next-net.env | 11 +- spartan/environments/testnet.env | 14 ++ spartan/scripts/deploy_network.sh | 18 +- spartan/scripts/network_deploy.sh | 1 + spartan/scripts/setup_gcp_secrets.sh | 1 + spartan/terraform/deploy-aztec-infra/main.tf | 19 +- .../terraform/deploy-aztec-infra/variables.tf | 45 +++++ spartan/terraform/deploy-metrics/main.tf | 32 +++ spartan/terraform/deploy-telemetry/main.tf | 186 ------------------ spartan/terraform/deploy-telemetry/outputs.tf | 9 - .../values/public-otel-collector.yaml | 155 --------------- .../values/public-prometheus.yaml | 39 ---- .../terraform/deploy-telemetry/variables.tf | 27 --- 15 files changed, 188 insertions(+), 420 deletions(-) create mode 100644 spartan/aztec-prover-stack/templates/agent-scaledobject.yaml delete mode 100644 spartan/terraform/deploy-telemetry/main.tf delete mode 100644 spartan/terraform/deploy-telemetry/outputs.tf delete mode 100644 spartan/terraform/deploy-telemetry/values/public-otel-collector.yaml delete mode 100644 spartan/terraform/deploy-telemetry/values/public-prometheus.yaml delete mode 100644 spartan/terraform/deploy-telemetry/variables.tf diff --git a/spartan/aztec-prover-stack/templates/agent-scaledobject.yaml b/spartan/aztec-prover-stack/templates/agent-scaledobject.yaml new file mode 100644 index 000000000000..98f55a7cf5b4 --- /dev/null +++ b/spartan/aztec-prover-stack/templates/agent-scaledobject.yaml @@ -0,0 +1,40 @@ +{{- if .Values.agent.autoscaling.keda.enabled }} +{{- $agentChartName := default "agent" .Values.agent.nameOverride }} +{{- $agentDefaultName := ternary .Release.Name (printf "%s-%s" .Release.Name $agentChartName) (contains $agentChartName .Release.Name) }} +{{- $agentName := default $agentDefaultName .Values.agent.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- $queueQuery := printf "sum(aztec_proving_queue_size{k8s_namespace_name=%q})" .Release.Namespace }} +apiVersion: keda.sh/v1alpha1 +kind: ScaledObject +metadata: + name: {{ $agentName }} + labels: + {{- include "chart.labels" . | nindent 4 }} +spec: + scaleTargetRef: + kind: Deployment + name: {{ $agentName }} + pollingInterval: {{ .Values.agent.autoscaling.keda.pollingInterval }} + cooldownPeriod: {{ .Values.agent.autoscaling.keda.cooldownPeriod }} + minReplicaCount: {{ .Values.agent.autoscaling.keda.minReplicaCount }} + maxReplicaCount: {{ .Values.agent.autoscaling.keda.maxReplicaCount }} + triggers: + {{- if .Values.agent.autoscaling.keda.scalingBands }} + {{- range $band := .Values.agent.autoscaling.keda.scalingBands }} + - type: prometheus + metadata: + serverAddress: {{ $.Values.agent.autoscaling.keda.prometheus.serverAddress | quote }} + metricName: {{ printf "aztec_proving_queue_size_agents_%v_over_%v" $band.replicas $band.queueSize | replace "." "_" | quote }} + query: {{ printf "((%s or vector(0)) > bool %v) * %v" $queueQuery $band.queueSize $band.replicas | quote }} + threshold: "1" + activationThreshold: "0" + {{- end }} + {{- else }} + - type: prometheus + metadata: + serverAddress: {{ .Values.agent.autoscaling.keda.prometheus.serverAddress | quote }} + metricName: "aztec_proving_queue_size" + query: {{ $queueQuery | quote }} + threshold: "1" + activationThreshold: "0" + {{- end }} +{{- end }} diff --git a/spartan/aztec-prover-stack/values.yaml b/spartan/aztec-prover-stack/values.yaml index 2e8a433e5eb0..ae014973bcd7 100644 --- a/spartan/aztec-prover-stack/values.yaml +++ b/spartan/aztec-prover-stack/values.yaml @@ -84,6 +84,17 @@ agent: nodeType: "prover-agent" replicaCount: 1 + autoscaling: + keda: + enabled: false + pollingInterval: 30 + cooldownPeriod: 300 + minReplicaCount: 0 + maxReplicaCount: 1 + scalingBands: [] + prometheus: + serverAddress: "" + persistence: enabled: false diff --git a/spartan/environments/next-net.env b/spartan/environments/next-net.env index 7e24b13ef493..efd92c9f1d64 100644 --- a/spartan/environments/next-net.env +++ b/spartan/environments/next-net.env @@ -51,7 +51,16 @@ VALIDATOR_PUBLISHER_MNEMONIC_START_INDEX=5000 PUBLISHERS_PER_PROVER=2 PROVER_PUBLISHER_MNEMONIC_START_INDEX=8000 -PROVER_REPLICAS=4 +PROVER_AGENT_KEDA_ENABLED=true +PROVER_AGENT_KEDA_PROMETHEUS_SERVER_ADDRESS=REPLACE_WITH_GCP_SECRET +PROVER_AGENT_KEDA_MIN_REPLICAS=0 +PROVER_AGENT_KEDA_MAX_REPLICAS=4 +PROVER_AGENT_KEDA_SCALING_BANDS='[ + { + queueSize = 0 + replicas = 4 + } +]' BOT_TRANSFERS_REPLICAS=1 BOT_TRANSFERS_TX_INTERVAL_SECONDS=250 diff --git a/spartan/environments/testnet.env b/spartan/environments/testnet.env index 1097fe11f818..8f0e5c30e7ac 100644 --- a/spartan/environments/testnet.env +++ b/spartan/environments/testnet.env @@ -89,3 +89,17 @@ PROVER_FAILED_PROOF_STORE=gs://aztec-develop/testnet/failed-proofs L1_TX_FAILED_STORE=gs://aztec-develop/testnet/failed-l1-txs PROVER_REPLICAS=4 PROVER_RESOURCE_PROFILE="prod" +PROVER_AGENT_KEDA_ENABLED=false +PROVER_AGENT_KEDA_PROMETHEUS_SERVER_ADDRESS= +PROVER_AGENT_KEDA_MIN_REPLICAS=0 +PROVER_AGENT_KEDA_MAX_REPLICAS=8 +PROVER_AGENT_KEDA_SCALING_BANDS='[ + { + queueSize = 0 + replicas = 4 + }, + { + queueSize = 50 + replicas = 8 + } +]' diff --git a/spartan/scripts/deploy_network.sh b/spartan/scripts/deploy_network.sh index 26a2da1623d4..bc7e9e346cdd 100755 --- a/spartan/scripts/deploy_network.sh +++ b/spartan/scripts/deploy_network.sh @@ -132,6 +132,15 @@ SEQ_ENFORCE_TIME_TABLE=${SEQ_ENFORCE_TIME_TABLE:-} SEQ_SKIP_CHECKPOINT_PUBLISH_PERCENT=${SEQ_SKIP_CHECKPOINT_PUBLISH_PERCENT:-0} PROVER_REPLICAS=${PROVER_REPLICAS:-4} PROVER_ENABLED=${PROVER_ENABLED:-true} +PROVER_AGENT_KEDA_ENABLED=${PROVER_AGENT_KEDA_ENABLED:-false} +PROVER_AGENT_KEDA_MIN_REPLICAS=${PROVER_AGENT_KEDA_MIN_REPLICAS:-0} +PROVER_AGENT_KEDA_MAX_REPLICAS=${PROVER_AGENT_KEDA_MAX_REPLICAS:-$PROVER_REPLICAS} +PROVER_AGENT_KEDA_SCALING_BANDS=${PROVER_AGENT_KEDA_SCALING_BANDS:-[]} +PROVER_AGENT_KEDA_PROMETHEUS_SERVER_ADDRESS=${PROVER_AGENT_KEDA_PROMETHEUS_SERVER_ADDRESS:-} +if [[ "$PROVER_ENABLED" == "true" && "$PROVER_AGENT_KEDA_ENABLED" == "true" && -z "$PROVER_AGENT_KEDA_PROMETHEUS_SERVER_ADDRESS" ]]; then + die "PROVER_AGENT_KEDA_ENABLED=true requires PROVER_AGENT_KEDA_PROMETHEUS_SERVER_ADDRESS. Set it explicitly, for example via GCP secret replacement." +fi +PROVER_AGENT_REPLICA_CAPACITY=$([[ "$PROVER_ENABLED" == "true" ]] && echo "$PROVER_AGENT_KEDA_MAX_REPLICAS" || echo 0) PROVER_AGENTS_PER_PROVER=${PROVER_AGENTS_PER_PROVER:-1} R2_ACCESS_KEY_ID=${R2_ACCESS_KEY_ID:-} R2_SECRET_ACCESS_KEY=${R2_SECRET_ACCESS_KEY:-} @@ -253,7 +262,7 @@ if (( TOTAL_VALIDATOR_PUBLISHERS > 0 )); then fi # Add prover publishers to prefunding list -TOTAL_PROVER_PUBLISHERS=$((PROVER_REPLICAS * PUBLISHERS_PER_PROVER)) +TOTAL_PROVER_PUBLISHERS=$((PROVER_AGENT_REPLICA_CAPACITY * PUBLISHERS_PER_PROVER)) if (( TOTAL_PROVER_PUBLISHERS > 0 )); then PROVER_PUBLISHER_RANGE=$(seq "$PROVER_PUBLISHER_MNEMONIC_START_INDEX" $((PROVER_PUBLISHER_MNEMONIC_START_INDEX + TOTAL_PROVER_PUBLISHERS - 1)) | tr '\n' ',' | sed 's/,$//') @@ -630,6 +639,13 @@ BOT_CROSS_CHAIN_L2_PRIVATE_KEY = "${BOT_CROSS_CHAIN_L2_PRIVATE_KEY:-0xcafe03}" PROVER_AGENTS_PER_PROVER = ${PROVER_AGENTS_PER_PROVER} PROVER_AGENT_POLL_INTERVAL_MS = ${PROVER_AGENT_POLL_INTERVAL_MS} +PROVER_AGENT_KEDA_ENABLED = ${PROVER_AGENT_KEDA_ENABLED:-false} +PROVER_AGENT_KEDA_PROMETHEUS_SERVER_ADDRESS = "${PROVER_AGENT_KEDA_PROMETHEUS_SERVER_ADDRESS:-}" +PROVER_AGENT_KEDA_MIN_REPLICAS = ${PROVER_AGENT_KEDA_MIN_REPLICAS:-0} +PROVER_AGENT_KEDA_MAX_REPLICAS = ${PROVER_AGENT_KEDA_MAX_REPLICAS:-$PROVER_REPLICAS} +PROVER_AGENT_KEDA_SCALING_BANDS = ${PROVER_AGENT_KEDA_SCALING_BANDS:-[]} +PROVER_AGENT_KEDA_POLLING_INTERVAL_SECONDS = ${PROVER_AGENT_KEDA_POLLING_INTERVAL_SECONDS:-30} +PROVER_AGENT_KEDA_COOLDOWN_PERIOD_SECONDS = ${PROVER_AGENT_KEDA_COOLDOWN_PERIOD_SECONDS:-300} RPC_INGRESS_ENABLED = ${RPC_INGRESS_ENABLED} RPC_INGRESS_HOSTS = ${RPC_INGRESS_HOSTS} diff --git a/spartan/scripts/network_deploy.sh b/spartan/scripts/network_deploy.sh index eab3b12962f7..8fa7fcaf3e12 100755 --- a/spartan/scripts/network_deploy.sh +++ b/spartan/scripts/network_deploy.sh @@ -28,6 +28,7 @@ gcp_auth # Second pass: source environment with GCP secret processing source_network_env "$env_file" + # Optional: provision per-network IP + managed cert (+ DNS record in the delegated # rpc.aztec-labs.com zone) via the network-frontend terraform module. The module's # outputs are exported as env vars that deploy_network.sh already consumes. diff --git a/spartan/scripts/setup_gcp_secrets.sh b/spartan/scripts/setup_gcp_secrets.sh index 9eadb39ecdc1..10b2cb30e05c 100755 --- a/spartan/scripts/setup_gcp_secrets.sh +++ b/spartan/scripts/setup_gcp_secrets.sh @@ -88,6 +88,7 @@ declare -A SECRET_MAPPINGS=( ["FUNDING_PRIVATE_KEY"]="${L1_NETWORK}-funding-private-key" ["ROLLUP_DEPLOYMENT_PRIVATE_KEY"]="${L1_NETWORK}-labs-rollup-private-key" ["OTEL_COLLECTOR_ENDPOINT"]="otel-collector-url" + ["PROVER_AGENT_KEDA_PROMETHEUS_SERVER_ADDRESS"]="prometheus-internal-read-url" ["ETHERSCAN_API_KEY"]="etherscan-api-key" ["LABS_INFRA_MNEMONIC"]="${MNEMONIC_SECRET}" ["STORE_SNAPSHOT_URL"]="r2-account-id" diff --git a/spartan/terraform/deploy-aztec-infra/main.tf b/spartan/terraform/deploy-aztec-infra/main.tf index b4d66a49a142..d14fd6d69d8a 100644 --- a/spartan/terraform/deploy-aztec-infra/main.tf +++ b/spartan/terraform/deploy-aztec-infra/main.tf @@ -52,7 +52,7 @@ module "web3signer" { VALIDATOR_MNEMONIC_START_INDEX = tonumber(var.VALIDATOR_MNEMONIC_START_INDEX) VALIDATOR_PUBLISHER_MNEMONIC_START_INDEX = tonumber(var.VALIDATOR_PUBLISHER_MNEMONIC_START_INDEX) VALIDATOR_PUBLISHERS_PER_REPLICA = var.VALIDATOR_PUBLISHERS_PER_REPLICA - PROVER_COUNT = tonumber(var.PROVER_REPLICAS) + PROVER_COUNT = local.prover_agent_replica_capacity PUBLISHERS_PER_PROVER = tonumber(var.PROVER_PUBLISHERS_PER_PROVER) PROVER_PUBLISHER_MNEMONIC_START_INDEX = tonumber(var.PROVER_PUBLISHER_MNEMONIC_START_INDEX) @@ -94,6 +94,8 @@ locals { tag = split(":", var.VALIDATOR_HA_DOCKER_IMAGE)[1] } : local.aztec_image + prover_agent_replica_capacity = var.PROVER_ENABLED ? (var.PROVER_AGENT_KEDA_ENABLED ? var.PROVER_AGENT_KEDA_MAX_REPLICAS : tonumber(var.PROVER_REPLICAS)) : 0 + # Max node count: max of primary (VALIDATOR_REPLICAS) and HA pod counts # Determines how many attester keystores and publisher key ranges to generate effective_ha_count = var.VALIDATOR_HA_REPLICAS > 0 ? coalesce(var.VALIDATOR_HA_REPLICA_COUNT, tonumber(var.VALIDATOR_REPLICAS)) : 0 @@ -343,6 +345,19 @@ locals { node = { logLevel = var.LOG_LEVEL } + autoscaling = { + keda = { + enabled = var.PROVER_AGENT_KEDA_ENABLED && var.PROVER_ENABLED + pollingInterval = var.PROVER_AGENT_KEDA_POLLING_INTERVAL_SECONDS + cooldownPeriod = var.PROVER_AGENT_KEDA_COOLDOWN_PERIOD_SECONDS + minReplicaCount = var.PROVER_AGENT_KEDA_MIN_REPLICAS + maxReplicaCount = var.PROVER_AGENT_KEDA_MAX_REPLICAS + scalingBands = var.PROVER_AGENT_KEDA_SCALING_BANDS + prometheus = { + serverAddress = var.PROVER_AGENT_KEDA_PROMETHEUS_SERVER_ADDRESS + } + } + } } })], local.is_kind ? [yamlencode({ agent = { @@ -377,7 +392,7 @@ locals { "agent.node.env.CRS_PATH" = "/usr/src/crs" "agent.node.proverRealProofs" = var.PROVER_REAL_PROOFS "agent.node.env.PROVER_AGENT_POLL_INTERVAL_MS" = var.PROVER_AGENT_POLL_INTERVAL_MS - "agent.replicaCount" = var.PROVER_REPLICAS + "agent.replicaCount" = var.PROVER_AGENT_KEDA_ENABLED ? "0" : var.PROVER_REPLICAS "agent.node.env.BOOTSTRAP_NODES" = "asdf" "agent.node.env.PROVER_AGENT_COUNT" = var.PROVER_AGENTS_PER_PROVER "agent.node.env.PROVER_TEST_DELAY_TYPE" = var.PROVER_TEST_DELAY_TYPE diff --git a/spartan/terraform/deploy-aztec-infra/variables.tf b/spartan/terraform/deploy-aztec-infra/variables.tf index dd3f00859491..5b8c00682983 100644 --- a/spartan/terraform/deploy-aztec-infra/variables.tf +++ b/spartan/terraform/deploy-aztec-infra/variables.tf @@ -851,6 +851,51 @@ variable "PROVER_AGENT_POLL_INTERVAL_MS" { default = 1000 } +variable "PROVER_AGENT_KEDA_ENABLED" { + description = "Whether KEDA should scale prover agent pods from proving queue depth" + type = bool + default = false +} + +variable "PROVER_AGENT_KEDA_MIN_REPLICAS" { + description = "Minimum prover agent pods managed by KEDA" + type = number + default = 0 +} + +variable "PROVER_AGENT_KEDA_MAX_REPLICAS" { + description = "Maximum prover agent pods managed by KEDA" + type = number + default = 1 +} + +variable "PROVER_AGENT_KEDA_SCALING_BANDS" { + description = "Step scaling bands for prover agents. Each band scales to replicas when total proving queue size is greater than queueSize." + type = list(object({ + queueSize = number + replicas = number + })) + default = [] +} + +variable "PROVER_AGENT_KEDA_PROMETHEUS_SERVER_ADDRESS" { + description = "Prometheus server URL queried by KEDA for prover queue depth" + type = string + default = "" +} + +variable "PROVER_AGENT_KEDA_POLLING_INTERVAL_SECONDS" { + description = "KEDA polling interval for prover agent queue-depth scaling" + type = number + default = 30 +} + +variable "PROVER_AGENT_KEDA_COOLDOWN_PERIOD_SECONDS" { + description = "KEDA cooldown period before scaling prover agents back down" + type = number + default = 300 +} + variable "PROVER_AGENT_INCLUDE_METRICS" { description = "Metrics whitelist in the prover agent" type = string diff --git a/spartan/terraform/deploy-metrics/main.tf b/spartan/terraform/deploy-metrics/main.tf index bf30ce3d59ac..d1a98c03d1a5 100644 --- a/spartan/terraform/deploy-metrics/main.tf +++ b/spartan/terraform/deploy-metrics/main.tf @@ -29,6 +29,11 @@ data "terraform_remote_state" "ssl" { } } +data "google_compute_subnetwork" "default" { + name = "default" + region = var.region +} + resource "google_compute_address" "grafana_ip" { provider = google name = "grafana-ip" @@ -51,6 +56,18 @@ resource "google_compute_address" "otel_collector_ip" { } } +resource "google_compute_address" "prometheus_ip" { + provider = google + name = "prometheus-ip" + address_type = "INTERNAL" + region = var.region + subnetwork = data.google_compute_subnetwork.default.id + + lifecycle { + prevent_destroy = true + } +} + provider "kubernetes" { alias = "gke-cluster" config_path = "~/.kube/config" @@ -209,6 +226,21 @@ resource "helm_release" "aztec-gke-cluster" { value = google_compute_address.otel_collector_ip.address } + set { + name = "prometheus.server.service.type" + value = "LoadBalancer" + } + + set { + name = "prometheus.server.service.annotations.networking\\.gke\\.io\\/load-balancer-type" + value = "Internal" + } + + set { + name = "prometheus.server.service.loadBalancerIP" + value = google_compute_address.prometheus_ip.address + } + set { name = "prometheus.serverFiles.prometheus\\.yml.scrape_configs[0].job_name" value = "prometheus" diff --git a/spartan/terraform/deploy-telemetry/main.tf b/spartan/terraform/deploy-telemetry/main.tf deleted file mode 100644 index b44f1bc0cc28..000000000000 --- a/spartan/terraform/deploy-telemetry/main.tf +++ /dev/null @@ -1,186 +0,0 @@ -terraform { - backend "gcs" { - bucket = "aztec-terraform" - prefix = "metrics-deploy/us-west1-a/aztec-gke-private/telemetry/terraform.tfstate" - } - required_providers { - helm = { - source = "hashicorp/helm" - version = "~> 2.16.1" - } - kubernetes = { - source = "hashicorp/kubernetes" - version = "~> 2.24.0" - } - } -} - -provider "google" { - project = var.project - region = var.region -} - -provider "kubernetes" { - alias = "gke-cluster" - config_path = "~/.kube/config" - config_context = var.cluster -} - -provider "helm" { - alias = "gke-cluster" - kubernetes { - config_path = "~/.kube/config" - config_context = var.cluster - } -} - -resource "google_compute_global_address" "otel_collector_ingress" { - provider = google - name = "${var.RELEASE_NAME}-otel-collector-ingress" - address_type = "EXTERNAL" - - lifecycle { - prevent_destroy = true - } -} - -resource "kubernetes_namespace" "ns" { - provider = kubernetes.gke-cluster - metadata { - name = var.RELEASE_NAME - } -} - -resource "kubernetes_manifest" "otel_ingress_certificate" { - provider = kubernetes.gke-cluster - - manifest = { - "apiVersion" = "networking.gke.io/v1" - "kind" = "ManagedCertificate" - "metadata" = { - "name" = "otel-ingress-cert" - "namespace" = kubernetes_namespace.ns.metadata[0].name - } - "spec" = { - "domains" = var.HOSTS - } - } -} - -resource "kubernetes_manifest" "otel_ingress_backend" { - provider = kubernetes.gke-cluster - - manifest = { - "apiVersion" = "cloud.google.com/v1" - "kind" = "BackendConfig" - "metadata" = { - "name" = "otel-ingress-backend" - "namespace" = kubernetes_namespace.ns.metadata[0].name - } - "spec" = { - "healthCheck" = { - "checkIntervalSec" = 15 - "timeoutSec" = 5 - "type" = "HTTP" - "port" = 13133 - "requestPath" = "/" - } - } - } -} - -locals { - prefixes = jsondecode(file("../../../yarn-project/cli/public_include_metric_prefixes.json")) - registries = ["0xec4156431d0f3df66d4e24ba3d30dcb4c85fa309", "0xf299347e765cfb27f913bde8e4983fd0f195676f", "0x2e48addca360da61e4d6c21ff2b1961af56eb83b", "0xc2f24280f5c7f4897370dfdeb30f79ded14f1c81"] - roles = ["sequencer"] - - otel_metric_allowlist = join(" or ", formatlist("HasPrefix(name, %q)", local.prefixes)) - otel_registry_allowlist = join(" or ", formatlist("resource.attributes[\"aztec.registry_address\"] == %q", local.registries)) - otel_role_allowlist = join(" or ", formatlist("resource.attributes[\"aztec.node_role\"] == %q", local.roles)) -} - -resource "helm_release" "otel_collector" { - provider = helm.gke-cluster - name = "otel" - namespace = kubernetes_namespace.ns.metadata[0].name - repository = "https://open-telemetry.github.io/opentelemetry-helm-charts" - chart = "opentelemetry-collector" - version = "0.127.2" - create_namespace = false - upgrade_install = true - dependency_update = true - force_update = true - reuse_values = false - reset_values = true - - # base values file - values = [ - file("./values/public-otel-collector.yaml"), - yamlencode({ - ingress = { - hosts = [ - for index, host in var.HOSTS : ({ - host = host - paths = [ - { - path = "/" - pathType = "Prefix" - port = 4318 - } - ] - }) - ] - } - }), - # have to use a heredoc because of quotation issues with OTTL - <<-EOF -config: - processors: - filter: - metrics: - metric: - - 'not (${local.otel_registry_allowlist})' - - 'not (${local.otel_role_allowlist})' - - 'not (${local.otel_metric_allowlist})' -EOF - ] - - set { - name = "ingress.annotations.kubernetes\\.io\\/ingress\\.global-static-ip-name" - value = google_compute_global_address.otel_collector_ingress.name - } - - set { - name = "ingress.annotations.networking\\.gke\\.io\\/managed-certificates" - value = "otel-ingress-cert" - } - - timeout = 300 - wait = true - wait_for_jobs = true - atomic = true - cleanup_on_fail = true -} - -resource "helm_release" "public_prometheus" { - provider = helm.gke-cluster - name = "prometheus" - namespace = kubernetes_namespace.ns.metadata[0].name - repository = "https://prometheus-community.github.io/helm-charts" - chart = "prometheus" - version = "25.27.0" - create_namespace = false - upgrade_install = true - dependency_update = true - force_update = true - reuse_values = false - reset_values = true - - values = [file("./values/public-prometheus.yaml")] - - timeout = 300 - wait = true - wait_for_jobs = true - atomic = true - cleanup_on_fail = true -} diff --git a/spartan/terraform/deploy-telemetry/outputs.tf b/spartan/terraform/deploy-telemetry/outputs.tf deleted file mode 100644 index 693227f05012..000000000000 --- a/spartan/terraform/deploy-telemetry/outputs.tf +++ /dev/null @@ -1,9 +0,0 @@ -output "otel_ingress_hostname" { - description = "Public otel ingress" - value = "https://${var.HOSTS[0]}" -} - -output "otel_ingress_ip" { - description = "Public otel ingress IP address" - value = google_compute_global_address.otel_collector_ingress.address -} diff --git a/spartan/terraform/deploy-telemetry/values/public-otel-collector.yaml b/spartan/terraform/deploy-telemetry/values/public-otel-collector.yaml deleted file mode 100644 index 2e18a2d16667..000000000000 --- a/spartan/terraform/deploy-telemetry/values/public-otel-collector.yaml +++ /dev/null @@ -1,155 +0,0 @@ -mode: statefulset -replicaCount: 1 - -nodeSelector: - node-type: infra - -image: - repository: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-contrib - tag: "0.128.0" - -resources: - requests: - memory: 12Gi - cpu: "2" - limits: - memory: 60Gi - cpu: "7" - -ports: - otlp: - enabled: false - otlp-http: - enabled: true - containerPort: 4318 - servicePort: 4318 - hostPort: 4318 - protocol: TCP - jaeger-compact: - enabled: false - jaeger-thrift: - enabled: false - jaeger-grpc: - enabled: false - zipkin: - enabled: false - -config: - extensions: - health_check: - endpoint: ${env:MY_POD_IP}:13133 - - receivers: - jaeger: {} - prometheus: {} - zipkin: {} - otlp: - protocols: - grpc: {} - http: - endpoint: ${env:MY_POD_IP}:4318 - - processors: - memory_limiter: - check_interval: 1s - limit_mib: 12000 - spike_limit_mib: 2000 - - filter: - metrics: - metric: [] # placeholder - datapoint: - - 'metric.type == METRIC_DATA_TYPE_HISTOGRAM and Len(explicit_bounds) > 20' - - resource: - attributes: - - pattern: "(k8s|os|telemetry|service|exported).*" - action: delete - - transform: - metric_statements: - - context: datapoint - statements: - - set(attributes["aztec.node_role"], resource.attributes["aztec.node_role"]) - - set(attributes["aztec.registry_address"], resource.attributes["aztec.registry_address"]) - - batch: {} - - exporters: - prometheus: - endpoint: ${env:MY_POD_IP}:8889 - namespace: external - metric_expiration: 5m - resource_to_telemetry_conversion: - enabled: false - - service: - telemetry: - metrics: - address: ${env:MY_POD_IP}:8888 - - pipelines: - logs: null - traces: null - - metrics: - receivers: - - otlp - processors: - - memory_limiter - - resource - - filter - - transform - - batch - exporters: - - prometheus - -ports: - otlp: - enabled: false - otlp-http: - enabled: true - jaeger-compact: - enabled: false - jaeger-thrift: - enabled: false - jaeger-grpc: - enabled: false - zipkin: - enabled: false - metrics: - enabled: false - healthcheck: - enabled: true - containerPort: 13133 - servicePort: 13133 - hostPort: 13133 - protocol: TCP - prom-otel: - enabled: true - containerPort: 8888 - servicePort: 8888 - hostPort: 8888 - protocol: TCP - prom-aztec: - enabled: true - containerPort: 8889 - servicePort: 8889 - hostPort: 8889 - protocol: TCP -service: - enabled: true - annotations: - cloud.google.com/backend-config: "{\"default\":\"otel-ingress-backend\"}" - cloud.google.com/neg: "{\"ingress\": true}" - -ingress: - enabled: true - annotations: - kubernetes.io/ingress.allow-http: "true" - kubernetes.io/ingress.class: gce - kubernetes.io/ingress.global-static-ip-name: "" - cloud.google.com/healthcheck-port: "13133" - cloud.google.com/healthcheck-path: "/" - - # networking.gke.io/managed-certificates: null diff --git a/spartan/terraform/deploy-telemetry/values/public-prometheus.yaml b/spartan/terraform/deploy-telemetry/values/public-prometheus.yaml deleted file mode 100644 index c25335185818..000000000000 --- a/spartan/terraform/deploy-telemetry/values/public-prometheus.yaml +++ /dev/null @@ -1,39 +0,0 @@ -server: - global: - evaluation_interval: 30s - scrape_interval: 1m - scrape_timeout: 20s - resources: - requests: - memory: 40Gi - cpu: "4" - limits: - memory: 60Gi - cpu: "7" - nodeSelector: - node-type: infra - persistentVolume: - enabled: true - size: 100Gi - replicaCount: 1 - statefulSet: - enabled: true - -serverFiles: - prometheus.yml: - scrape_configs: - - job_name: public_telemetry - kubernetes_sd_configs: - - role: pod - namespaces: - own_namespace: true - names: [] - -alertmanager: - enabled: false -prometheus-node-exporter: - enabled: false -prometheus-pushgateway: - enabled: false -kube-state-metrics: - enabled: false diff --git a/spartan/terraform/deploy-telemetry/variables.tf b/spartan/terraform/deploy-telemetry/variables.tf deleted file mode 100644 index 55f7f8c4cca4..000000000000 --- a/spartan/terraform/deploy-telemetry/variables.tf +++ /dev/null @@ -1,27 +0,0 @@ -variable "cluster" { - description = "GKE cluster context" - type = string - default = "gke_testnet-440309_us-west1-a_aztec-gke-private" -} - -variable "project" { - default = "testnet-440309" - type = string -} - -variable "region" { - default = "us-west1" - type = string -} - -variable "RELEASE_NAME" { - description = "Name of helm deployment and k8s namespace" - type = string - default = "public-telemetry" -} - -variable "HOSTS" { - description = "The public hostname for the ingress" - type = list(string) - default = ["telemetry.alpha-testnet.aztec-labs.com"] -} From 56d75c2b4abb9112b415c7bab933955ccda43f53 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Thu, 28 May 2026 17:22:29 +0100 Subject: [PATCH 25/27] chore: update destroy_bootnode.sh (#23626) delete vm + ip + private-keys + dry run --- iac/network/scripts/destroy_bootnodes.sh | 174 +++++++++++++++++++---- 1 file changed, 150 insertions(+), 24 deletions(-) diff --git a/iac/network/scripts/destroy_bootnodes.sh b/iac/network/scripts/destroy_bootnodes.sh index b4598f634b36..58a715f1e5cf 100755 --- a/iac/network/scripts/destroy_bootnodes.sh +++ b/iac/network/scripts/destroy_bootnodes.sh @@ -3,20 +3,86 @@ set -e # This script should be run from the root of iac/network. It will destroy all VMs for the provided network -# Only the VMs are destroyed. No other infrastructure is modified - -# Usage: ./scripts/destroy_bootnodes.sh - -NETWORK_NAME=${1:-} -PROJECT_ID=${2:-} +# By default, only the VMs are destroyed. Use the optional flags to delete other per-network bootnode resources. + +# Usage: ./scripts/destroy_bootnodes.sh [--destroy-ip] [--destroy-private-key] [--dry-run] + +usage() { + echo "Usage: ./scripts/destroy_bootnodes.sh [--destroy-ip] [--destroy-private-key] [--dry-run]" + echo "" + echo "Options:" + echo " --destroy-ip Delete the reserved static IP addresses and remove them from Terraform state" + echo " --destroy-private-key Delete the per-region bootnode private-key secrets" + echo " --dry-run Show the planned VM destroy and print other destructive commands without running them" +} + +print_command() { + printf '+' + printf ' %q' "$@" + printf '\n' +} + +run_destructive() { + print_command "$@" + + if [[ "$DRY_RUN" != "true" ]]; then + "$@" + fi +} + +DESTROY_IP=false +DESTROY_PRIVATE_KEY=false +DRY_RUN=false +POSITIONAL_ARGS=() + +while [[ $# -gt 0 ]]; do + case "$1" in + --destroy-ip) + DESTROY_IP=true + shift + ;; + --destroy-private-key) + DESTROY_PRIVATE_KEY=true + shift + ;; + --dry-run) + DRY_RUN=true + shift + ;; + -h | --help) + usage + exit 0 + ;; + *) + POSITIONAL_ARGS+=("$1") + shift + ;; + esac +done + +NETWORK_NAME=${POSITIONAL_ARGS[0]:-} +PROJECT_ID=${POSITIONAL_ARGS[1]:-} + +if [[ ${#POSITIONAL_ARGS[@]} -gt 2 ]]; then + echo "Unexpected arguments: ${POSITIONAL_ARGS[*]:2}" + usage + exit 1 +fi if [[ -z "$NETWORK_NAME" ]]; then echo "NETWORK_NAME is required" + usage exit 1 fi if [[ -z "$PROJECT_ID" ]]; then echo "PROJECT_ID is required" + usage + exit 1 +fi + +if [[ "$DESTROY_IP" == "true" || "$DESTROY_PRIVATE_KEY" == "true" ]] && ! command -v gcloud >/dev/null 2>&1; then + echo "gcloud is required when --destroy-ip or --destroy-private-key is used" exit 1 fi @@ -27,20 +93,22 @@ TAG=latest ROOT=$(git rev-parse --show-toplevel)/iac/network -cd $ROOT/bootnode/ip/gcp +cd "$ROOT/bootnode/ip/gcp" -terraform init -backend-config="prefix=network/$NETWORK_NAME/bootnode/ip/gcp" +terraform init -reconfigure -backend-config="prefix=network/$NETWORK_NAME/bootnode/ip/gcp" >/dev/null OUTPUT=$(terraform output -json ip_addresses) echo "IP Addresses output: $OUTPUT" GCP_REGIONS_ARRAY=() +GCP_IPS_ARRAY=() while read -r REGION IP; do echo "IP: $IP is in region $REGION" GCP_REGIONS_ARRAY+=("$REGION") + GCP_IPS_ARRAY+=("$IP") done < <(echo "$OUTPUT" | jq -r 'to_entries | .[] | "\(.key) \(.value)"') @@ -52,22 +120,80 @@ PRIVATE_KEYS_TF_ARG=$GCP_REGIONS_TF_ARG echo "GCP_REGIONS: $GCP_REGIONS_TF_ARG" -cd $ROOT +cd "$ROOT" BOOTNODE_START_SCRIPT="$ROOT/scripts/bootnode_startup.sh" -cd $ROOT/bootnode/vm/gcp - -terraform init -backend-config="prefix=network/$NETWORK_NAME/bootnode/vm/gcp" - -terraform apply \ - -var="regions=$GCP_REGIONS_TF_ARG" \ - -var="start_script=$BOOTNODE_START_SCRIPT" \ - -var="network_name=$NETWORK_NAME" \ - -var="peer_id_private_keys=$PRIVATE_KEYS_TF_ARG" \ - -var="machine_type=" \ - -var="project_id=$PROJECT_ID" \ - -var="p2p_port=$P2P_PORT" \ - -var="l1_chain_id=$L1_CHAIN_ID" \ - -var="image_tag=$TAG" \ - --destroy +cd "$ROOT/bootnode/vm/gcp" + +terraform init -reconfigure -backend-config="prefix=network/$NETWORK_NAME/bootnode/vm/gcp" >/dev/null + +VM_TERRAFORM_ARGS=( + -var="regions=$GCP_REGIONS_TF_ARG" + -var="start_script=$BOOTNODE_START_SCRIPT" + -var="network_name=$NETWORK_NAME" + -var="peer_id_private_keys=$PRIVATE_KEYS_TF_ARG" + -var="machine_type=" + -var="project_id=$PROJECT_ID" + -var="p2p_port=$P2P_PORT" + -var="l1_chain_id=$L1_CHAIN_ID" + -var="image_tag=$TAG" +) + +if [[ "$DRY_RUN" == "true" ]]; then + terraform plan "${VM_TERRAFORM_ARGS[@]}" -destroy +else + terraform apply "${VM_TERRAFORM_ARGS[@]}" --destroy +fi + +if [[ "$DESTROY_IP" == "true" ]]; then + cd "$ROOT/bootnode/ip/gcp" + + terraform init -reconfigure -backend-config="prefix=network/$NETWORK_NAME/bootnode/ip/gcp" >/dev/null + + for INDEX in "${!GCP_REGIONS_ARRAY[@]}"; do + REGION=${GCP_REGIONS_ARRAY[$INDEX]} + IP=${GCP_IPS_ARRAY[$INDEX]} + ADDRESS_NAME="$NETWORK_NAME-bootnodes-$REGION" + STATE_ADDRESS="module.ip.google_compute_address.static_ip[\"$REGION\"]" + + echo "Deleting static IP $ADDRESS_NAME ($IP) in $REGION" + + if DESCRIBE_OUTPUT=$(gcloud compute addresses describe "$ADDRESS_NAME" --region "$REGION" --project "$PROJECT_ID" 2>&1); then + run_destructive gcloud compute addresses delete "$ADDRESS_NAME" --region "$REGION" --project "$PROJECT_ID" --quiet + elif [[ "${DESCRIBE_OUTPUT,,}" == *"not found"* ]]; then + echo "Static IP $ADDRESS_NAME was not found; removing Terraform state entry if present." + else + echo "$DESCRIBE_OUTPUT" + exit 1 + fi + + if terraform state show "$STATE_ADDRESS" >/dev/null 2>&1; then + run_destructive terraform state rm "$STATE_ADDRESS" + else + echo "Terraform state entry $STATE_ADDRESS not found; skipping state removal." + fi + done + + run_destructive terraform apply -auto-approve \ + -var="regions=[]" \ + -var="name=$NETWORK_NAME-bootnodes" \ + -var="project_id=$PROJECT_ID" +fi + +if [[ "$DESTROY_PRIVATE_KEY" == "true" ]]; then + for REGION in "${GCP_REGIONS_ARRAY[@]}"; do + SECRET_NAME="$NETWORK_NAME-$REGION-bootnode-private-key" + + echo "Deleting secret $SECRET_NAME" + + if DESCRIBE_OUTPUT=$(gcloud secrets describe "$SECRET_NAME" --project "$PROJECT_ID" 2>&1); then + run_destructive gcloud secrets delete "$SECRET_NAME" --project "$PROJECT_ID" --quiet + elif [[ "${DESCRIBE_OUTPUT,,}" == *"not found"* ]]; then + echo "Secret $SECRET_NAME was not found; skipping." + else + echo "$DESCRIBE_OUTPUT" + exit 1 + fi + done +fi From c0ec7ad27a3e476f6e804c816853a9445a7c9039 Mon Sep 17 00:00:00 2001 From: Aztec Bot <49558828+AztecBot@users.noreply.github.com> Date: Thu, 28 May 2026 16:45:23 -0400 Subject: [PATCH 26/27] chore: skip failing chonk_pinned_inputs.test in CI (#23643) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds `bbapi/chonk_pinned_inputs.test` to `.test_patterns.yml` with `skip: true`. The test fails in CI with `No execution steps in ivc-inputs.msgpack` (pinned Chonk IVC inputs missing/stale), e.g. http://ci.aztec-labs.com/d7647a841ee811a0 — both the native and wasm backend cases fail at `loadPinnedFlow`. Requested by Santiago Palladino to unblock the merge train; owner set to palla for follow-up. --- *Created by [claudebox](https://claudebox.work/v2/sessions/08e2c5b593fedd23) · group: `slackbot`* --- .test_patterns.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.test_patterns.yml b/.test_patterns.yml index 0b6c6b0375f2..3d2f2e2e2d67 100644 --- a/.test_patterns.yml +++ b/.test_patterns.yml @@ -91,6 +91,13 @@ tests: owners: - *nico + # Fails in CI: "No execution steps in ivc-inputs.msgpack" — pinned Chonk IVC + # inputs missing/stale. http://ci.aztec-labs.com/d7647a841ee811a0 + - regex: "bbapi/chonk_pinned_inputs.test" + skip: true + owners: + - *palla + # e2e tests skipped - regex: "testbench/port_change.test.ts" skip: true From 47d5eb972d23180e07a2ef2831d79e4770d892ca Mon Sep 17 00:00:00 2001 From: Aztec Bot <49558828+AztecBot@users.noreply.github.com> Date: Thu, 28 May 2026 18:28:45 -0400 Subject: [PATCH 27/27] chore(ci): tolerate public authwit P2P receipt flake (#23648) --- .test_patterns.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.test_patterns.yml b/.test_patterns.yml index 3d2f2e2e2d67..7652dfda2060 100644 --- a/.test_patterns.yml +++ b/.test_patterns.yml @@ -349,6 +349,15 @@ tests: owners: - *adam + # http://ci.aztec-labs.com/93754bdf832ef9bd + # P2P receipt race in the compose CLI-wallet flow: the account deployment tx + # can be reported as dropped after the 5s grace window before inclusion is + # visible to the queried node. + - regex: "yarn-project/end-to-end/scripts/run_test.sh compose ../cli-wallet/test/flows/public_authwit_transfer.sh" + error_regex: "Transaction .* was dropped\\. Reason: Tx dropped by P2P node" + owners: + - *palla + - regex: "cd l1-contracts && forge test" error_regex: "Encountered 1 failing test in test/staking_asset_handler/claim.t.sol:ClaimTest" owners: