From a89868523f75ad85b8edeaa3f1ddc5c06bac254c Mon Sep 17 00:00:00 2001 From: Test User Date: Sat, 23 May 2026 16:40:49 -0700 Subject: [PATCH 1/2] fix(pds): include prevData in firehose #commit events prevData is the MST root CID of the previous commit (the data field at the since rev) and is effectively required for the inductive firehose. Without it, relays running strict commit validation (indigo with LenientSyncValidation off) fail with "missing prevData field" and reject every commit after the first, freezing the repo. Populated from the pre-write repo's commit.data at each write path (create/delete/put/applyWrites). Closes #169 Co-Authored-By: Claude Opus 4.7 --- .changeset/firehose-prevdata.md | 7 +++++ packages/pds/scripts/verify-firehose.ts | 1 + packages/pds/src/account-do.ts | 4 +++ packages/pds/src/sequencer.ts | 6 ++++ packages/pds/test/firehose.test.ts | 39 +++++++++++++++++++++++++ 5 files changed, 57 insertions(+) create mode 100644 .changeset/firehose-prevdata.md diff --git a/.changeset/firehose-prevdata.md b/.changeset/firehose-prevdata.md new file mode 100644 index 00000000..d1f423b3 --- /dev/null +++ b/.changeset/firehose-prevdata.md @@ -0,0 +1,7 @@ +--- +"@getcirrus/pds": patch +--- + +Include `prevData` in firehose `#commit` events. + +`prevData` is the MST root CID of the previous commit (the `data` field at the `since` rev) and is effectively required for the inductive version of the firehose. Without it, relays running strict commit validation (e.g. indigo with `LenientSyncValidation` off) fail verification with "missing prevData field" and reject every commit after the first, freezing the repo on that relay. It is now populated from the pre-write repo's `commit.data` at each write path. diff --git a/packages/pds/scripts/verify-firehose.ts b/packages/pds/scripts/verify-firehose.ts index 9c8ca985..e536db19 100644 --- a/packages/pds/scripts/verify-firehose.ts +++ b/packages/pds/scripts/verify-firehose.ts @@ -32,6 +32,7 @@ interface CommitEvent { commit: Cid; rev: string; since: string | null; + prevData: Cid; blocks: Uint8Array; ops: CommitOp[]; blobs: Cid[]; diff --git a/packages/pds/src/account-do.ts b/packages/pds/src/account-do.ts index 3bfb5f92..9cb550eb 100644 --- a/packages/pds/src/account-do.ts +++ b/packages/pds/src/account-do.ts @@ -415,6 +415,7 @@ export class AccountDurableObject extends DurableObject { commit: updatedRepo.cid, rev: updatedRepo.commit.rev, since: prevRev, + prevData: repo.commit.data, newBlocks, ops: [opWithCid], }; @@ -484,6 +485,7 @@ export class AccountDurableObject extends DurableObject { commit: updatedRepo.cid, rev: updatedRepo.commit.rev, since: prevRev, + prevData: repo.commit.data, newBlocks, ops: [deleteOp], }; @@ -589,6 +591,7 @@ export class AccountDurableObject extends DurableObject { commit: updatedRepo.cid, rev: updatedRepo.commit.rev, since: prevRev, + prevData: repo.commit.data, newBlocks, ops: [opWithCid], }; @@ -791,6 +794,7 @@ export class AccountDurableObject extends DurableObject { commit: updatedRepo.cid, rev: updatedRepo.commit.rev, since: prevRev, + prevData: repo.commit.data, newBlocks, ops: opsWithCids, }; diff --git a/packages/pds/src/sequencer.ts b/packages/pds/src/sequencer.ts index e6a96a63..52684417 100644 --- a/packages/pds/src/sequencer.ts +++ b/packages/pds/src/sequencer.ts @@ -14,6 +14,10 @@ export interface CommitEvent { commit: CID; rev: string; since: string | null; + // Root CID of the MST for the previous commit (the `data` field of the + // commit at the `since` rev). Required for relays doing inductive firehose + // verification (com.atproto.sync.subscribeRepos#commit `prevData`). + prevData: CID; blocks: Uint8Array; ops: RepoOp[]; blobs: CID[]; @@ -72,6 +76,7 @@ export interface CommitData { commit: CID; rev: string; since: string | null; + prevData: CID; newBlocks: BlockMap; ops: Array; } @@ -102,6 +107,7 @@ export class Sequencer { commit: data.commit, rev: data.rev, since: data.since, + prevData: data.prevData, blocks: carBytes, ops: data.ops.map( (op): RepoOp => ({ diff --git a/packages/pds/test/firehose.test.ts b/packages/pds/test/firehose.test.ts index 7487ecb0..a07e137a 100644 --- a/packages/pds/test/firehose.test.ts +++ b/packages/pds/test/firehose.test.ts @@ -343,6 +343,45 @@ describe("Firehose (subscribeRepos)", () => { }); }); + it("should include prevData matching the previous commit's MST root", async () => { + const id = env.ACCOUNT.idFromName("account"); + const stub = env.ACCOUNT.get(id); + + await runInDurableObject(stub, async (instance: AccountDurableObject) => { + await instance.getStorage(); + const sequencer = (instance as any).sequencer; + const encodeEventFrame = (instance as any).encodeEventFrame.bind( + instance, + ); + + // Load the repo and capture the current (soon-to-be-previous) state. + await instance.rpcGetRepoStatus(); + const prevRepo = (instance as any).repo; + const expectedPrevData = prevRepo.commit.data.toString(); + const expectedSince = prevRepo.commit.rev; + + const seqBefore = sequencer.getLatestSeq(); + await instance.rpcCreateRecord("app.bsky.feed.post", "prevdata-test", { + text: "prevData test", + createdAt: new Date().toISOString(), + }); + + const events = await sequencer.getEventsSince(seqBefore, 1); + const frame = encodeEventFrame(events[0] as SeqCommitEvent); + const { body } = decodeFrame(frame); + const commitBody = body as { + prevData?: { toString(): string }; + since?: string; + }; + + // prevData must be present (relays require it for verification) + // and equal the data CID of the commit at the `since` rev. + expect(commitBody.prevData).toBeDefined(); + expect(commitBody.prevData!.toString()).toBe(expectedPrevData); + expect(commitBody.since).toBe(expectedSince); + }); + }); + it("should encode identity events with #identity frame type", async () => { const id = env.ACCOUNT.idFromName("account"); const stub = env.ACCOUNT.get(id); From dc3e13b845e011e26d9900ce39cccc84549b00b2 Mon Sep 17 00:00:00 2001 From: Test User Date: Sat, 23 May 2026 17:18:42 -0700 Subject: [PATCH 2/2] refactor(pds): make prevData nullable to mirror since MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review feedback: type prevData as CID | null in the sequencer (consistent with the nullable since field) and optional in the verify-firehose script (tolerates legacy pre-prevData / since=null streams). No behavior change — every write path still has a prior commit and sets it. Co-Authored-By: Claude Opus 4.7 --- packages/pds/scripts/verify-firehose.ts | 4 +++- packages/pds/src/sequencer.ts | 7 ++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/pds/scripts/verify-firehose.ts b/packages/pds/scripts/verify-firehose.ts index e536db19..71854568 100644 --- a/packages/pds/scripts/verify-firehose.ts +++ b/packages/pds/scripts/verify-firehose.ts @@ -32,7 +32,9 @@ interface CommitEvent { commit: Cid; rev: string; since: string | null; - prevData: Cid; + // Optional: absent on legacy (pre-prevData) firehose data and on any + // initial commit where `since` is null. + prevData?: Cid; blocks: Uint8Array; ops: CommitOp[]; blobs: Cid[]; diff --git a/packages/pds/src/sequencer.ts b/packages/pds/src/sequencer.ts index 52684417..6fb1e3b1 100644 --- a/packages/pds/src/sequencer.ts +++ b/packages/pds/src/sequencer.ts @@ -16,8 +16,9 @@ export interface CommitEvent { since: string | null; // Root CID of the MST for the previous commit (the `data` field of the // commit at the `since` rev). Required for relays doing inductive firehose - // verification (com.atproto.sync.subscribeRepos#commit `prevData`). - prevData: CID; + // verification (com.atproto.sync.subscribeRepos#commit `prevData`). Nullable + // to mirror `since`; in practice every write has a prior commit so it is set. + prevData: CID | null; blocks: Uint8Array; ops: RepoOp[]; blobs: CID[]; @@ -76,7 +77,7 @@ export interface CommitData { commit: CID; rev: string; since: string | null; - prevData: CID; + prevData: CID | null; newBlocks: BlockMap; ops: Array; }