feat(storage): verifyFileReferences — the executable R4 acceptance gate

ADR-0104 D3 wave 2 planned to gate enabling collection on "reference counting is
verified", which is aspirational rather than checkable. This makes it runnable.

verifyFileReferences() compares GROUND TRUTH (what records' file-class fields
actually hold) against RECORDED OWNERSHIP (sys_file.ref_object/ref_id/ref_field)
and classifies every disagreement by whether it can cause DATA LOSS once
collection is on, because only that class should block a release:

  blocking
    unowned_reference  a record holds a file nothing owns — with collection on
                       it looks free and its bytes go
    foreign_owner      a record holds a file owned by a different slot; that
                       reference is invisible to the lifecycle, so the bytes go
                       when the recorded owner releases
    shared_reference   one file held by two slots — exclusivity was violated,
                       so copy-on-claim did not run, and one file's ACL is
                       serving two parents

  advisory
    stale_owner        owned but no longer held; fails toward retention
    unreferenced_file  committed and unpointed-at — storage cost, not risk

The scan is read-only: it never writes, tombstones or deletes. The point is that
a ledger may not be handed authority over an irreversible delete until it has
been shown to agree with reality, so this is meant to be run repeatedly, on real
tenant data, and to report zero blocking discrepancies on consecutive runs
before the gated collection change may merge.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SHpGw3GBA9aFpfwVArRWfd

Author: claude

.changeset/adr-0104-d3w2-pr4-governed-download.md

@@ -31,3 +31,22 @@ an inline blob or an external URL, never a `sys_file` id, so no existing file
 has an owner recorded and none of them start being gated. The gate engages only
 for files a record's field has actually claimed, and disengages again when
 ownership is released.
+
+---
+
+Also adds `verifyFileReferences()` — the executable form of ADR-0104's R4
+acceptance gate. It compares ground truth (what records' file fields actually
+hold) against recorded ownership, and classifies disagreements by whether they
+could cause data loss once collection is enabled:
+
+- **blocking** — `unowned_reference` (a held file nothing owns), `foreign_owner`
+  (a record holds a file owned by another slot), `shared_reference` (one file
+  held by two slots, i.e. exclusivity was violated). Each would let a later reap
+  delete bytes a record still points at.
+- **advisory** — `stale_owner` (owned but no longer held; fails toward
+  retention) and `unreferenced_file` (storage cost, not a correctness problem).
+
+The scan is read-only — it never writes, tombstones, or deletes. A ledger may
+not be given authority over irreversible deletes until it has been shown to
+agree with reality, so this must report zero blocking discrepancies on real
+tenant data, on consecutive runs, before the gated collection change may merge.

packages/services/service-storage/src/index.ts

@@ -16,5 +16,17 @@ export { installAttachmentLifecycleHooks, createSysFileReapGuard, createUploadSe
 export type { AttachmentLifecycleEngine, AttachmentLifecycleLogger } from './attachment-lifecycle.js';
 export { installFileReferenceHooks, FileReferenceCopyError } from './file-reference-lifecycle.js';
 export type { FileReferenceEngine, FileReferenceLogger } from './file-reference-lifecycle.js';
+export {
+  verifyFileReferences,
+  formatFileReferenceReport,
+  BLOCKING_ISSUE_KINDS,
+} from './verify-file-references.js';
+export type {
+  FileReferenceIssue,
+  FileReferenceIssueKind,
+  FileReferenceReport,
+  VerifyReferencesEngine,
+  VerifyReferencesOptions,
+} from './verify-file-references.js';
 export { installAttachmentAccessHooks, installAttachmentReadVisibility } from './attachment-access-hooks.js';
 export type { AttachmentSharingLike } from './attachment-access-hooks.js';

packages/services/service-storage/src/verify-file-references.test.ts

@@ -0,0 +1,237 @@
+// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
+
+import { describe, it, expect } from 'vitest';
+import {
+  verifyFileReferences,
+  formatFileReferenceReport,
+  BLOCKING_ISSUE_KINDS,
+  type VerifyReferencesEngine,
+} from './verify-file-references.js';
+
+const REGISTRY: Record<string, any> = {
+  sys_file: { fields: { id: { type: 'text' } } },
+  product: {
+    fields: {
+      id: { type: 'text' },
+      name: { type: 'text' },
+      image: { type: 'image' },
+      gallery: { type: 'image', multiple: true },
+    },
+  },
+  tag: { fields: { id: { type: 'text' }, label: { type: 'text' } } },
+};
+
+function fakeEngine(tables: Record<string, Array<Record<string, unknown>>>): VerifyReferencesEngine {
+  const matches = (row: Record<string, unknown>, where: Record<string, unknown>) =>
+    Object.entries(where).every(([k, v]) => {
+      if (v && typeof v === 'object' && '$ne' in (v as any)) return row[k] !== (v as any).$ne;
+      return row[k] === v;
+    });
+  return {
+    getObject: (name) => REGISTRY[name],
+    getConfigs: () => REGISTRY,
+    async find(object, options: any) {
+      const rows = (tables[object] ?? []).filter((r) => matches(r, options?.where ?? {}));
+      const start = typeof options?.offset === 'number' ? options.offset : 0;
+      const end = typeof options?.limit === 'number' ? start + options.limit : undefined;
+      return rows.slice(start, end);
+    },
+  };
+}
+
+const file = (id: string, owner?: { object: string; recordId: string; field: string }) => ({
+  id,
+  status: 'committed',
+  scope: 'user',
+  ref_object: owner?.object ?? null,
+  ref_id: owner?.recordId ?? null,
+  ref_field: owner?.field ?? null,
+});
+
+describe('verifyFileReferences (ADR-0104 D3 wave 2 — R4 gate)', () => {
+  it('reports a clean bill when ownership matches what records hold', async () => {
+    const engine = fakeEngine({
+      product: [{ id: 'p1', image: 'file_a' }],
+      sys_file: [file('file_a', { object: 'product', recordId: 'p1', field: 'image' })],
+    });
+
+    const report = await verifyFileReferences(engine);
+
+    expect(report.ok).toBe(true);
+    expect(report.blocking).toBe(0);
+    expect(report.issues).toEqual([]);
+    expect(report.scannedObjects).toEqual(['product']);
+    expect(report.heldReferences).toBe(1);
+    expect(report.ownedFiles).toBe(1);
+    expect(formatFileReferenceReport(report)).toContain('No discrepancies');
+  });
+
+  it('only scans objects that declare a file-class field', async () => {
+    const engine = fakeEngine({
+      product: [{ id: 'p1' }],
+      tag: [{ id: 't1', label: 'x' }],
+      sys_file: [],
+    });
+
+    const report = await verifyFileReferences(engine);
+
+    expect(report.scannedObjects).toEqual(['product']);
+    expect(report.scannedRecords).toBe(1);
+  });
+
+  // ── Blocking: these would delete bytes something still holds ──────
+  it('BLOCKS on a held file with no recorded owner', async () => {
+    const engine = fakeEngine({
+      product: [{ id: 'p1', image: 'file_a' }],
+      sys_file: [file('file_a')], // uploaded, never claimed
+    });
+
+    const report = await verifyFileReferences(engine);
+
+    expect(report.ok).toBe(false);
+    expect(report.counts.unowned_reference).toBe(1);
+    expect(report.issues[0]).toMatchObject({
+      kind: 'unowned_reference',
+      fileId: 'file_a',
+      object: 'product',
+      recordId: 'p1',
+      field: 'image',
+    });
+    expect(formatFileReferenceReport(report)).toContain('BLOCKING');
+  });
+
+  it('BLOCKS when a record holds a file owned by a different slot', async () => {
+    const engine = fakeEngine({
+      product: [
+        { id: 'p1', image: 'file_a' },
+        { id: 'p2', image: 'file_a' },
+      ],
+      sys_file: [file('file_a', { object: 'product', recordId: 'p1', field: 'image' })],
+    });
+
+    const report = await verifyFileReferences(engine);
+
+    expect(report.ok).toBe(false);
+    // p2's reference is invisible to the lifecycle: when p1 releases, the
+    // bytes go while p2 still points at them.
+    expect(report.counts.foreign_owner).toBe(1);
+    expect(report.issues.find((i) => i.kind === 'foreign_owner')).toMatchObject({ recordId: 'p2' });
+    // …and the exclusivity violation itself is reported too.
+    expect(report.counts.shared_reference).toBe(1);
+  });
+
+  it('BLOCKS on a file held by two slots (exclusivity violated)', async () => {
+    const engine = fakeEngine({
+      product: [{ id: 'p1', image: 'file_a', gallery: ['file_a'] }],
+      sys_file: [file('file_a', { object: 'product', recordId: 'p1', field: 'image' })],
+    });
+
+    const report = await verifyFileReferences(engine);
+
+    expect(report.counts.shared_reference).toBe(1);
+    expect(report.ok).toBe(false);
+  });
+
+  // ── Advisory: safe directions ─────────────────────────────────────
+  it('reports a stale owner as advisory, not blocking', async () => {
+    const engine = fakeEngine({
+      product: [{ id: 'p1' }], // field cleared without the release landing
+      sys_file: [file('file_a', { object: 'product', recordId: 'p1', field: 'image' })],
+    });
+
+    const report = await verifyFileReferences(engine);
+
+    expect(report.counts.stale_owner).toBe(1);
+    // Fails toward retention — the file is simply never collected.
+    expect(report.blocking).toBe(0);
+    expect(report.ok).toBe(true);
+  });
+
+  it('reports unreferenced committed files only when asked, and never as blocking', async () => {
+    const tables = {
+      product: [{ id: 'p1' }],
+      sys_file: [file('file_orphan')],
+    };
+
+    const without = await verifyFileReferences(fakeEngine(tables));
+    expect(without.counts.unreferenced_file).toBe(0);
+
+    const withSweep = await verifyFileReferences(fakeEngine(tables), { includeUnreferenced: true });
+    expect(withSweep.counts.unreferenced_file).toBe(1);
+    expect(withSweep.ok).toBe(true);
+  });
+
+  it('ignores attachments-scope files, which sys_attachment governs', async () => {
+    const engine = fakeEngine({
+      product: [{ id: 'p1' }],
+      sys_file: [{ ...file('file_att'), scope: 'attachments' }],
+    });
+
+    const report = await verifyFileReferences(engine, { includeUnreferenced: true });
+
+    expect(report.counts.unreferenced_file).toBe(0);
+  });
+
+  // ── Dual-mode ─────────────────────────────────────────────────────
+  it('ignores inline blobs and URL values — a pre-cutover tenant reads clean', async () => {
+    const engine = fakeEngine({
+      product: [
+        { id: 'p1', image: { url: 'https://cdn.example.com/a.png', name: 'a.png' } },
+        { id: 'p2', image: 'https://cdn.example.com/b.png' },
+        { id: 'p3', image: 'data:image/svg+xml,<svg/>' },
+      ],
+      sys_file: [],
+    });
+
+    const report = await verifyFileReferences(engine);
+
+    expect(report.heldReferences).toBe(0);
+    expect(report.issues).toEqual([]);
+    expect(report.ok).toBe(true);
+  });
+
+  // ── Mechanics ─────────────────────────────────────────────────────
+  it('pages through records rather than reading one unbounded page', async () => {
+    const many = Array.from({ length: 1200 }, (_, i) => ({ id: `p${i}`, image: `file_${i}` }));
+    const engine = fakeEngine({
+      product: many,
+      sys_file: many.map((r, i) => file(`file_${i}`, { object: 'product', recordId: `p${i}`, field: 'image' })),
+    });
+
+    const report = await verifyFileReferences(engine);
+
+    expect(report.scannedRecords).toBe(1200);
+    expect(report.ownedFiles).toBe(1200);
+    expect(report.ok).toBe(true);
+  });
+
+  it('marks the verdict truncated when a scan bound is hit', async () => {
+    const many = Array.from({ length: 1200 }, (_, i) => ({ id: `p${i}` }));
+    const engine = fakeEngine({ product: many, sys_file: [] });
+
+    const report = await verifyFileReferences(engine, { maxRecordsPerObject: 500 });
+
+    expect(report.truncated).toBe(true);
+    expect(formatFileReferenceReport(report)).toContain('truncated');
+  });
+
+  it('can be scoped to specific objects', async () => {
+    const engine = fakeEngine({
+      product: [{ id: 'p1', image: 'file_a' }],
+      sys_file: [file('file_a')],
+    });
+
+    const report = await verifyFileReferences(engine, { objects: ['tag'] });
+
+    expect(report.scannedObjects).toEqual([]);
+    expect(report.ok).toBe(true);
+  });
+
+  it('classifies exactly the data-loss kinds as blocking', () => {
+    expect([...BLOCKING_ISSUE_KINDS].sort()).toEqual([
+      'foreign_owner',
+      'shared_reference',
+      'unowned_reference',
+    ]);
+  });
+});