feat(storage): legacy file-value backfill — ADR-0104 D3 wave 2 (PR-6)

backfillFileReferences() converts the pre-reference forms a file/image/avatar/
video/audio field may hold — an inline metadata blob ({url, name, size, …}) or
a bare URL string — into the reference form: an opaque sys_file id, owned by the
record's field.

What it will and will not convert:

  - A URL naming this platform's own resolver (…/storage/files/:id) already
    identifies a sys_file; the field is rewritten to the bare id, no bytes move.
  - A data: URI carries its bytes inline; they are uploaded, a sys_file is
    registered, and the field is rewritten to its id.
  - An external URL is REPORTED, NEVER CONVERTED. Re-hosting third-party
    content is a bandwidth, licensing and privacy decision that is not a
    migration's to make. ADR-0104 R7 retires these toward an explicit `url`
    field, which under AI authoring is the point: it stops "managed file" and
    "external link" being the same declaration.

Dry run by default — nothing is written unless apply is set, and the dry-run
report has the same shape as the applied one so the plan can be reviewed and
diffed against what actually happened. Idempotent — a value already in
reference form is recorded as already_id and left alone, so a partially
completed run is safe to repeat.

The backfill never writes the ref_* columns itself: it rewrites the record, and
the claim hooks observe that write and record ownership. One claiming path, so
there is nothing that can disagree with itself — asserted by a test that fails
if the backfill ever starts touching sys_file directly.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01SHpGw3GBA9aFpfwVArRWfd

Author: claude

.changeset/adr-0104-d3w2-pr6-backfill.md

@@ -0,0 +1,34 @@
+---
+"@objectstack/service-storage": minor
+---
+
+feat(storage): legacy file-value backfill — ADR-0104 D3 wave 2 (PR-6)
+
+`backfillFileReferences()` converts the pre-reference forms a `file`/`image`/
+`avatar`/`video`/`audio` field may hold — an inline metadata blob
+(`{url, name, size, …}`) or a bare URL string — into the reference form: an
+opaque `sys_file` id, owned by the record's field.
+
+What it will and will not convert:
+
+- **A URL naming this platform's own resolver** (`…/storage/files/:id`) already
+  identifies a `sys_file`; the field is rewritten to the bare id and no bytes
+  move.
+- **A `data:` URI** carries its bytes inline; they are uploaded, a `sys_file` is
+  registered, and the field is rewritten to its id.
+- **An external URL** is reported, never converted. Re-hosting third-party
+  content is a bandwidth, licensing and privacy decision that is not a
+  migration's to make — ADR-0104 R7 retires these toward an explicit `url`
+  field, which under AI authoring is the point: it stops "managed file" and
+  "external link" being the same declaration.
+
+**Dry run by default** — nothing is written unless `apply` is set, and the
+dry-run report has the same shape as the applied one so the plan can be reviewed
+and diffed. **Idempotent** — a value already in reference form is recorded and
+left alone, so a partially-completed run is safe to repeat.
+
+The backfill never writes the ownership columns itself: it rewrites the record,
+and the claim hooks observe that write and record ownership. One claiming path,
+so there is nothing that can disagree with itself. Run
+`verifyFileReferences()` afterwards to confirm the two agree — that
+reconciliation is the gate the irreversible collection change must pass.

packages/services/service-storage/src/backfill-file-references.test.ts

@@ -0,0 +1,251 @@
+// Copyright (c) 2025 ObjectStack. Licensed under the Apache-2.0 license.
+
+import { describe, it, expect, vi } from 'vitest';
+import {
+  backfillFileReferences,
+  formatBackfillReport,
+  type BackfillEngine,
+} from './backfill-file-references.js';
+
+const silentLogger = () => ({ info: vi.fn(), warn: vi.fn(), debug: vi.fn() });
+
+const REGISTRY: Record<string, any> = {
+  sys_file: { fields: { id: { type: 'text' } } },
+  product: {
+    fields: {
+      id: { type: 'text' },
+      name: { type: 'text' },
+      image: { type: 'image' },
+      gallery: { type: 'image', multiple: true },
+    },
+  },
+  tag: { fields: { id: { type: 'text' }, label: { type: 'text' } } },
+};
+
+function fakeEngine(tables: Record<string, Array<Record<string, unknown>>>) {
+  const calls: Array<{ op: string; object: string; arg: any }> = [];
+  const engine: BackfillEngine & { tables: typeof tables; calls: typeof calls } = {
+    getObject: (name) => REGISTRY[name],
+    getConfigs: () => REGISTRY,
+    async find(object, options: any) {
+      const rows = tables[object] ?? [];
+      const start = typeof options?.offset === 'number' ? options.offset : 0;
+      const end = typeof options?.limit === 'number' ? start + options.limit : undefined;
+      return rows.slice(start, end);
+    },
+    async insert(object, data: any) {
+      calls.push({ op: 'insert', object, arg: data });
+      (tables[object] ??= []).push({ ...data });
+      return data;
+    },
+    async update(object, data: any) {
+      calls.push({ op: 'update', object, arg: data });
+      const row = (tables[object] ?? []).find((r) => String(r.id) === String(data.id));
+      if (row) Object.assign(row, data);
+      return row;
+    },
+    tables,
+    calls,
+  };
+  return engine;
+}
+
+const fakeStorage = () =>
+  ({
+    upload: vi.fn(async () => {}),
+    download: vi.fn(async () => Buffer.from('x')),
+    delete: vi.fn(async () => {}),
+    exists: vi.fn(async () => true),
+    getInfo: vi.fn(async () => ({ key: 'k', size: 1, contentType: 'text/plain', lastModified: new Date() })),
+  }) as any;
+
+const run = (engine: any, opts: any = {}, storage: any = fakeStorage()) =>
+  backfillFileReferences(engine, () => storage, silentLogger(), opts);
+
+describe('backfillFileReferences (ADR-0104 D3 wave 2)', () => {
+  it('rewrites a URL that already names a sys_file to the bare id, moving no bytes', async () => {
+    const engine = fakeEngine({
+      product: [{ id: 'p1', image: 'https://app.example.com/api/v1/storage/files/file_a' }],
+      sys_file: [],
+    });
+    const storage = fakeStorage();
+
+    const report = await run(engine, { apply: true }, storage);
+
+    expect(engine.tables.product[0].image).toBe('file_a');
+    expect(report.converted).toBe(1);
+    expect(storage.upload).not.toHaveBeenCalled();
+    expect(report.actions[0].kind).toBe('resolved_local_url');
+  });
+
+  it('resolves the same URL when it is wrapped in a legacy inline blob', async () => {
+    const engine = fakeEngine({
+      product: [
+        { id: 'p1', image: { url: '/api/v1/storage/files/file_b', name: 'b.png', size: 12 } },
+      ],
+      sys_file: [],
+    });
+
+    await run(engine, { apply: true });
+
+    expect(engine.tables.product[0].image).toBe('file_b');
+  });
+
+  it('uploads inline data: bytes and rewrites the field to the new id', async () => {
+    const engine = fakeEngine({
+      product: [{ id: 'p1', image: { url: 'data:image/png;base64,aGVsbG8=', name: 'hi.png' } }],
+      sys_file: [],
+    });
+    const storage = fakeStorage();
+
+    const report = await run(engine, { apply: true }, storage);
+
+    expect(storage.upload).toHaveBeenCalledTimes(1);
+    const created = engine.tables.sys_file[0];
+    expect(created).toMatchObject({ name: 'hi.png', mime_type: 'image/png', status: 'committed' });
+    expect(created.size).toBe(5); // "hello"
+    expect(engine.tables.product[0].image).toBe(created.id);
+    expect(report.converted).toBe(1);
+  });
+
+  /**
+   * Converting an external URL would mean fetching third-party content and
+   * silently re-hosting it — a bandwidth/licensing/privacy decision that is not
+   * a migration's to make.
+   */
+  it('reports an external URL and never re-hosts it', async () => {
+    const engine = fakeEngine({
+      product: [{ id: 'p1', image: 'https://cdn.example.com/logo.png' }],
+      sys_file: [],
+    });
+    const storage = fakeStorage();
+
+    const report = await run(engine, { apply: true }, storage);
+
+    expect(engine.tables.product[0].image).toBe('https://cdn.example.com/logo.png');
+    expect(storage.upload).not.toHaveBeenCalled();
+    expect(report.externalUrls).toBe(1);
+    expect(report.converted).toBe(0);
+    expect(formatBackfillReport(report)).toContain('NOT re-hosted');
+  });
+
+  it('handles a multiple:true field element-wise', async () => {
+    const engine = fakeEngine({
+      product: [
+        {
+          id: 'p1',
+          gallery: [
+            '/api/v1/storage/files/file_a',
+            'https://cdn.example.com/x.png',
+            'file_already',
+          ],
+        },
+      ],
+      sys_file: [],
+    });
+
+    const report = await run(engine, { apply: true });
+
+    expect(engine.tables.product[0].gallery).toEqual([
+      'file_a',
+      'https://cdn.example.com/x.png',
+      'file_already',
+    ]);
+    expect(report.converted).toBe(1);
+    expect(report.externalUrls).toBe(1);
+    expect(report.alreadyReferences).toBe(1);
+  });
+
+  // ── Dry run ───────────────────────────────────────────────────────
+  it('writes nothing on a dry run but reports the same plan', async () => {
+    const engine = fakeEngine({
+      product: [
+        { id: 'p1', image: '/api/v1/storage/files/file_a' },
+        { id: 'p2', image: 'data:text/plain;base64,aGk=' },
+      ],
+      sys_file: [],
+    });
+    const storage = fakeStorage();
+
+    const report = await run(engine, {}, storage);
+
+    expect(report.applied).toBe(false);
+    expect(report.converted).toBe(2);
+    expect(engine.calls).toHaveLength(0);
+    expect(engine.tables.product[0].image).toBe('/api/v1/storage/files/file_a');
+    expect(storage.upload).not.toHaveBeenCalled();
+    expect(formatBackfillReport(report)).toContain('DRY RUN');
+  });
+
+  // ── Idempotence / safety ──────────────────────────────────────────
+  it('is idempotent — a second run converts nothing', async () => {
+    const engine = fakeEngine({
+      product: [{ id: 'p1', image: '/api/v1/storage/files/file_a' }],
+      sys_file: [],
+    });
+
+    await run(engine, { apply: true });
+    const second = await run(engine, { apply: true });
+
+    expect(second.converted).toBe(0);
+    expect(second.alreadyReferences).toBe(1);
+  });
+
+  it('never writes the ref_* columns itself — claiming stays on one path', async () => {
+    const engine = fakeEngine({
+      product: [{ id: 'p1', image: '/api/v1/storage/files/file_a' }],
+      sys_file: [],
+    });
+
+    await run(engine, { apply: true });
+
+    // The record write is what the claim hooks observe; the backfill must not
+    // record ownership behind their back, or there would be two claiming paths
+    // that could disagree.
+    for (const c of engine.calls) {
+      expect(c.arg).not.toHaveProperty('ref_object');
+      expect(c.arg).not.toHaveProperty('ref_id');
+      expect(c.arg).not.toHaveProperty('ref_field');
+    }
+    expect(engine.calls.filter((c) => c.object === 'sys_file')).toHaveLength(0);
+  });
+
+  it('leaves data: values alone when no storage service is available', async () => {
+    const engine = fakeEngine({
+      product: [{ id: 'p1', image: 'data:text/plain;base64,aGk=' }],
+      sys_file: [],
+    });
+
+    const report = await backfillFileReferences(engine, () => null, silentLogger(), { apply: true });
+
+    expect(report.unresolvable).toBe(1);
+    expect(report.converted).toBe(0);
+    expect(engine.tables.product[0].image).toBe('data:text/plain;base64,aGk=');
+  });
+
+  it('skips objects with no file-class fields', async () => {
+    const engine = fakeEngine({ tag: [{ id: 't1', label: 'x' }], product: [], sys_file: [] });
+
+    const report = await run(engine, { apply: true });
+
+    expect(report.scannedObjects).toEqual(['product']);
+  });
+
+  it('pages through large objects and flags truncation at the bound', async () => {
+    const many = Array.from({ length: 500 }, (_, i) => ({
+      id: `p${i}`,
+      image: `/api/v1/storage/files/file_${i}`,
+    }));
+    const engine = fakeEngine({ product: many, sys_file: [] });
+
+    const full = await run(engine, {});
+    expect(full.scannedRecords).toBe(500);
+    expect(full.truncated).toBe(false);
+
+    const bounded = await run(fakeEngine({ product: many, sys_file: [] }), {
+      maxRecordsPerObject: 200,
+    });
+    expect(bounded.truncated).toBe(true);
+    expect(formatBackfillReport(bounded)).toContain('truncated');
+  });
+});