feat(attachments): sys_file orphan lifecycle + non-admin permission matrix (#2755)

.changeset/attachments-2755-followups.md

@@ -0,0 +1,38 @@
+---
+'@objectstack/objectql': minor
+'@objectstack/service-storage': minor
+'@objectstack/platform-objects': minor
+'@objectstack/verify': minor
+'@objectstack/rest': patch
+---
+
+feat(attachments): sys_file orphan lifecycle + parent-derived attachment access (#2755)
+
+**Orphan lifecycle (ADR-0057).** Deleting a `sys_attachment` join row used to
+orphan the backing `sys_file` row and its storage bytes forever. `sys_file`
+now declares a lifecycle (`ttl 30d` on a new `deleted_at` tombstone for
+orphans; `retention 7d onlyWhen status=pending` for abandoned uploads), the
+storage plugin's new hooks tombstone a file when its LAST join row is deleted
+(attachments scope only — `Field.file`/`Field.image`/avatar scopes are never
+touched) and un-tombstone on re-attach, and a new LifecycleService **reap
+guard** seam (`registerReapGuard`) re-verifies zero references at sweep time
+and deletes the storage bytes before confirming each row reap. A guarded
+object is never blind-deleted; an erroring guard fails safe (rows retained).
+
+**Attachment access (ADR-0049, Salesforce parent-derived semantics).**
+`sys_attachment` create now requires caller READ visibility of the parent
+record (403 `ATTACHMENT_PARENT_ACCESS`) and server-stamps `uploaded_by` from
+the session (client value ignored); delete requires uploader-or-parent-editor
+(403 `ATTACHMENT_DELETE_DENIED`). The storage upload routes require an
+authenticated session when an auth service is wired (401 `AUTH_REQUIRED`;
+bare kernels stay open) and stamp `owner_id` on new files.
+
+**REMOVED — `sys_attachment.share_type` / `sys_attachment.visibility`.**
+Both fields were modeled in v1 with zero runtime consumers (ADR-0049
+parsed-but-unenforced). There is no replacement key: attachment access is
+derived from the parent record by the hooks above. Writers of these fields
+should simply stop sending them (unknown-field validation will reject them);
+existing DB columns are left as unmanaged leftovers, no migration needed.
+
+`@objectstack/verify` gains `BootOptions.extraPlugins` for booting optional
+service pairs (e.g. storage + audit) in dogfood fixtures.

docs/adr/0057-system-data-lifecycle-and-retention.md

@@ -170,6 +170,28 @@ registered ⇒ rows are retained (today's behavior), reported as
 `archive-pending` — a compliance ledger cannot be destroyed by declaring a
 lifecycle.
 
+#### Amendment (#2755): reap guards — domain re-verification + external-resource reclaim
+
+Some reapable rows are pointers to state the Reaper cannot see: `sys_file`
+rows reference storage bytes (disk/S3), and a tombstoned orphan may have
+regained `sys_attachment` references since it was marked. For these, a plugin
+may register a **reap guard** at runtime
+(`lifecycle.registerReapGuard(object, guard)`): the Reaper fetches candidate
+rows in batches, the guard confirms each id — performing external cleanup
+(byte deletion) and sweep-time re-verification first — or vetoes it (kept,
+retried next sweep). Rules:
+
+- A guarded object is **never blind-deleted**: an erroring guard fails safe
+  (rows retained), and an engine without row reads skips the object
+  (`reap-guard-unsupported`) rather than degrading to a blind delete.
+- Guards are runtime wiring, not spec surface — detection and scheduling stay
+  inside the single platform sweep (no bespoke sweepers, per the alternatives
+  below); the guard is a domain callback, not a scheduler.
+- First consumer: `sys_file` (service-storage) — attachment orphans are
+  tombstoned by hooks when their last `sys_attachment` reference is deleted,
+  reaped `30d` later by TTL with byte reclaim, with zero-reference
+  re-verification at sweep time; abandoned `pending` uploads reap after `7d`.
+
 ### 3.4 Reclaim — driver space hygiene
 
 SQLite driver defaults to `auto_vacuum=INCREMENTAL` (shipped P0); the Reaper

packages/dogfood/package.json

@@ -12,8 +12,10 @@
     "@objectstack/example-crm": "workspace:*",
     "@objectstack/example-showcase": "workspace:*",
     "@objectstack/objectql": "workspace:*",
+    "@objectstack/plugin-audit": "workspace:*",
     "@objectstack/plugin-auth": "workspace:*",
     "@objectstack/plugin-security": "workspace:*",
+    "@objectstack/service-storage": "workspace:*",
     "@objectstack/spec": "workspace:*",
     "@objectstack/verify": "workspace:*"
   },

packages/dogfood/test/fixtures/attachments-fixture.ts

@@ -0,0 +1,97 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+//
+// Attachments permission-matrix fixture (#2755).
+//
+// Three tiny objects spanning the enforcement axes of the generic
+// Attachments surface (#2727):
+//
+//   att_case    — enable.files + public sharing model: any member can read
+//                 (and per Salesforce "can edit parent ⇒ can manage its
+//                 attachments", any member can detach from it).
+//   att_secret  — enable.files + DEFAULT sharing model (omitted ⇒ private,
+//                 ADR-0090) + an `owner_id` field: the sharing service
+//                 read-filters rows to the owner, so a fresh member cannot
+//                 read (→ cannot attach to) another user's record, and
+//                 `canEdit` denies non-owners (→ cannot delete another
+//                 user's attachments on it).
+//   att_nofiles — NO enable.files: the #2727 opt-in gate (FILES_DISABLED).
+//
+// No custom SecurityPlugin: a fresh signUp member falls back to the real
+// `member_default` wildcard-CRUD set — exactly the posture the issue's
+// permission matrix questions (the gates under test are the ones layered on
+// TOP of that wildcard).
+
+import { defineStack } from '@objectstack/spec';
+import { ObjectSchema, Field } from '@objectstack/spec/data';
+import { PermissionSetSchema, type PermissionSet } from '@objectstack/spec/security';
+import { SecurityPlugin, securityDefaultPermissionSets } from '@objectstack/plugin-security';
+
+export const AttCase = ObjectSchema.create({
+  name: 'att_case',
+  label: 'Attachment Case',
+  pluralLabel: 'Attachment Cases',
+  sharingModel: 'public_read_write',
+  enable: { files: true },
+  fields: {
+    name: Field.text({ label: 'Name', required: true }),
+  },
+});
+
+export const AttSecret = ObjectSchema.create({
+  name: 'att_secret',
+  label: 'Attachment Secret',
+  pluralLabel: 'Attachment Secrets',
+  // sharingModel omitted — custom object defaults to PRIVATE (ADR-0090);
+  // owner_id is the sharing service's owner anchor.
+  enable: { files: true },
+  fields: {
+    name: Field.text({ label: 'Name', required: true }),
+    owner_id: Field.text({ label: 'Owner' }),
+  },
+});
+
+export const AttNoFiles = ObjectSchema.create({
+  name: 'att_nofiles',
+  label: 'No Files Here',
+  pluralLabel: 'No Files Here',
+  sharingModel: 'public_read_write',
+  fields: {
+    name: Field.text({ label: 'Name', required: true }),
+  },
+});
+
+/**
+ * The domain grant a real app ships when it turns the attachments panel on
+ * for members: `member_default` (the `everyone` anchor baseline) carries NO
+ * `allowDelete` (ADR-0090 D5 — delete is not a baseline right), so managing
+ * attachments requires an ordinary position-distributed set with the delete
+ * bit on `sys_attachment`. The matrix grants this to some members and pins
+ * the no-grant baseline with another.
+ */
+export const attachmentManagerSet: PermissionSet = PermissionSetSchema.parse({
+  name: 'att_attachment_manager',
+  label: 'Attachments Fixture — attachment manager',
+  objects: {
+    sys_attachment: { allowRead: true, allowCreate: true, allowEdit: true, allowDelete: true },
+  },
+});
+
+/** SecurityPlugin carrying the platform defaults + the fixture's domain set. */
+export function attachmentsFixtureSecurity(): SecurityPlugin {
+  return new SecurityPlugin({
+    defaultPermissionSets: [...securityDefaultPermissionSets, attachmentManagerSet],
+  });
+}
+
+export const attachmentsFixtureStack = defineStack({
+  manifest: {
+    id: 'com.dogfood.attachments_fixture',
+    namespace: 'att',
+    version: '0.0.0',
+    type: 'app',
+    name: 'Attachments Permission Matrix Fixture',
+    description:
+      'Three-object app exercising the #2755 attachment permission matrix: parent visibility, uploader/editor delete, enable.files gate.',
+  },
+  objects: [AttCase, AttSecret, AttNoFiles],
+});

packages/objectql/src/index.ts

@@ -63,6 +63,7 @@ export type {
   LifecycleObjectLike,
   LifecycleSettingsLike,
   LifecycleGovernanceAlert,
+  LifecycleReapGuard,
 } from './lifecycle/lifecycle-service.js';
 export { parseLifecycleDuration } from './lifecycle/duration.js';
 export { lifecycleSettingsManifest } from './lifecycle/lifecycle-settings.js';

packages/objectql/src/lifecycle/lifecycle-service.test.ts

@@ -17,10 +17,13 @@ function captureEngine(
     deleteImpl?: (object: string, options: any) => any;
     driver?: Record<string, unknown>;
     datasources?: Record<string, unknown>;
+    /** When set, the engine exposes `find` (guarded-reap candidate reads). */
+    findImpl?: (object: string, options: any) => any;
   } = {},
 ) {
   const deletes: Array<{ object: string; where: any; multi: any; context: any }> = [];
-  const engine = {
+  const finds: Array<{ object: string; where: any; limit: any; context: any }> = [];
+  const engine: any = {
     registry: { getAllObjects: () => objects },
     async delete(object: string, options: any) {
       deletes.push({ object, where: options?.where, multi: options?.multi, context: options?.context });
@@ -33,7 +36,13 @@ function captureEngine(
       return ds;
     },
   };
-  return { engine, deletes };
+  if (opts.findImpl) {
+    engine.find = async (object: string, options: any) => {
+      finds.push({ object, where: options?.where, limit: options?.limit, context: options?.context });
+      return opts.findImpl!(object, options);
+    };
+  }
+  return { engine, deletes, finds };
 }
 
 function service(engine: any, extra: Partial<ConstructorParameters<typeof LifecycleService>[0]> = {}) {
@@ -307,6 +316,105 @@ describe('LifecycleService.sweep — Reaper', () => {
   });
 });
 
+describe('LifecycleService.sweep — reap guard', () => {
+  const guarded: LifecycleObjectLike[] = [
+    { name: 'sys_file', lifecycle: { class: 'transient', ttl: { field: 'deleted_at', expireAfter: '30d' } } },
+  ];
+
+  it('deletes only guard-confirmed ids, per id, after fetching candidates with the cutoff filter', async () => {
+    const rows = [
+      { id: 'f1', deleted_at: '2020-01-01T00:00:00Z' },
+      { id: 'f2', deleted_at: '2020-01-02T00:00:00Z' },
+      { id: 'f3', deleted_at: '2020-01-03T00:00:00Z' },
+    ];
+    const { engine, deletes, finds } = captureEngine(guarded, { findImpl: () => rows });
+    const svc = service(engine);
+    const guard = vi.fn(async () => ['f1', 'f3']); // vetoes f2
+    svc.registerReapGuard('sys_file', guard);
+
+    const report = await svc.sweep();
+
+    expect(finds).toHaveLength(1);
+    expect(finds[0].where).toEqual({ deleted_at: { $lt: isoCutoff('30d') } });
+    expect(finds[0].context).toEqual({ isSystem: true, positions: [], permissions: [] });
+    expect(guard).toHaveBeenCalledWith('sys_file', rows);
+    // Per-id deletes — the engine's delete path reads `where.id` as a scalar
+    // target; a `{$in}` there would be bound as an object by the driver.
+    expect(deletes.map((d) => d.where)).toEqual([{ id: 'f1' }, { id: 'f3' }]);
+    expect(report.swept).toEqual([
+      { object: 'sys_file', class: 'transient', policy: 'ttl', cutoff: isoCutoff('30d'), deleted: 2 },
+    ]);
+    expect(report.errors).toEqual([]);
+  });
+
+  it('an erroring guard fails safe: no rows deleted, error reported', async () => {
+    const { engine, deletes } = captureEngine(guarded, { findImpl: () => [{ id: 'f1' }] });
+    const svc = service(engine);
+    svc.registerReapGuard('sys_file', async () => {
+      throw new Error('storage unreachable');
+    });
+
+    const report = await svc.sweep();
+
+    expect(deletes).toHaveLength(0);
+    expect(report.swept).toEqual([]);
+    expect(report.errors).toEqual([{ object: 'sys_file', error: 'storage unreachable' }]);
+  });
+
+  it('a guard on one object never changes the blind reap of others (regression pin)', async () => {
+    const { engine, deletes } = captureEngine(
+      [
+        ...guarded,
+        { name: 'sys_job_run', lifecycle: { class: 'telemetry', retention: { maxAge: '30d' } } },
+      ],
+      { findImpl: () => [] },
+    );
+    const svc = service(engine);
+    svc.registerReapGuard('sys_file', async () => []);
+
+    const report = await svc.sweep();
+
+    // sys_file: no candidates → no delete. sys_job_run: classic blind reap.
+    expect(deletes).toHaveLength(1);
+    expect(deletes[0].object).toBe('sys_job_run');
+    expect(deletes[0].where).toEqual({ created_at: { $lt: isoCutoff('30d') } });
+    expect(report.errors).toEqual([]);
+  });
+
+  it('a guarded object on an engine without find is skipped, never blind-deleted', async () => {
+    const { engine, deletes } = captureEngine(guarded); // no findImpl → no engine.find
+    const svc = service(engine);
+    svc.registerReapGuard('sys_file', async () => ['f1']);
+
+    const report = await svc.sweep();
+
+    expect(deletes).toHaveLength(0);
+    expect(report.swept).toEqual([]);
+    expect(report.skipped).toEqual([{ object: 'sys_file', reason: 'reap-guard-unsupported' }]);
+  });
+
+  it('drains full batches but stops the pass when a batch is not fully confirmed', async () => {
+    // Two "pages" of 500, then a short page. All of page 1 confirmed → loop
+    // continues; page 2 only partially confirmed → pass ends (vetoed rows
+    // would be re-fetched forever within one sweep).
+    const page = (n: number, prefix: string) =>
+      Array.from({ length: n }, (_, i) => ({ id: `${prefix}${i}`, deleted_at: '2020-01-01T00:00:00Z' }));
+    const pages = [page(500, 'a'), page(500, 'b')];
+    let call = 0;
+    const { engine, deletes, finds } = captureEngine(guarded, { findImpl: () => pages[call++] ?? [] });
+    const svc = service(engine);
+    svc.registerReapGuard('sys_file', async (_object, rows) =>
+      rows[0].id === 'a0' ? rows.map((r: any) => r.id) : rows.slice(0, 10).map((r: any) => r.id),
+    );
+
+    const report = await svc.sweep();
+
+    expect(finds).toHaveLength(2);
+    expect(deletes).toHaveLength(510); // 500 confirmed from page 1 + 10 from page 2, per id
+    expect(report.swept[0].deleted).toBe(510);
+  });
+});
+
 describe('LifecycleService.sweep — Archiver (P3)', () => {
   const AUDIT_OBJ: LifecycleObjectLike = {
     name: 'sys_audit_log',