feat(rest): surface silently-dropped write fields on PATCH/POST /data (#3431)

.changeset/rest-patch-data-dropped-fields.md

@@ -0,0 +1,45 @@
+---
+"@objectstack/spec": minor
+"@objectstack/metadata-protocol": minor
+"@objectstack/rest": minor
+---
+
+feat(rest): surface silently-dropped write fields on PATCH/POST /data (#3431)
+
+#3413 (closes #3407) built the engine-level strip-observability channel
+(`WriteObservabilityOptions.onFieldsDropped`) and wired the flow side
+(`update_record` / `create_record` emit a step warning + `droppedFields`). The
+**REST write path was never wired**, so an external API caller writing N fields
+still got a bare `200 + record` when `readonly` (#2948) / `readonlyWhen` (#3042)
+stripping meant `< N` actually landed — the same silent-success class #3407
+fixed flow-side, just on HTTP. The only way to notice was a per-field diff of
+the returned row (which need not echo every field). This wires the channel
+through the protocol → REST, on both write verbs.
+
+**Passthrough (metadata-protocol).** `updateData` now registers an
+`onFieldsDropped` collector on `engine.update` and returns the events on the
+response as `droppedFields`. `createData` surfaces the #3043 static-`readonly`
+INGRESS strip too — that strip runs at the protocol ingress
+(`stripReadonlyForInsert`), *before* the engine, so it is recovered by diffing
+the supplied payload against the stripped one (the engine's `onFieldsDropped` is
+also wired for a future insert-side engine strip). A faulty listener never
+breaks the write — the engine catches and logs.
+
+**Contract (spec).** `UpdateDataResponseSchema` / `CreateDataResponseSchema`
+gain an **optional** `droppedFields: DroppedFieldsEvent[]` — present only when
+≥1 field was dropped. Optional + omit-when-empty keeps the response shape
+backward-compatible for clients that only read `record`.
+
+**REST surface.** PATCH `/data/:object/:id` and POST `/data/:object` echo the
+drops as an `X-ObjectStack-Dropped-Fields` response header
+(`field;reason=<reason>` tokens, comma-joined — e.g.
+`approval_status;reason=readonly`) and keep the structured `droppedFields` on
+the body. **Status/success semantics are unchanged** (200 update / 201 create) —
+a strip is legitimate semantics, not a failure (same principle as #3413). The
+FLS write gate is untouched (it already fails closed with 403).
+
+Out of scope (issue #3431 D2 open questions, deferred): bulk
+(`updateManyData` / `createManyData` / `batchData`) and GraphQL mutation wiring,
+typed `@objectstack/client` warnings, and adding the header to the Hono CORS
+`exposeHeaders` allow-list for cross-origin browser reads (the body
+`droppedFields` is the cross-origin-safe channel meanwhile).

content/docs/references/api/protocol.mdx

@@ -219,6 +219,7 @@ const result = AiInsightsRequest.parse(data);
 | **object** | `string` | ✅ | The object name. |
 | **id** | `string` | ✅ | The ID of the newly created record. |
 | **record** | `Record<string, any>` | ✅ | The created record, including server-generated fields (created_at, owner). |
+| **droppedFields** | `{ object: string; fields: string[]; reason: Enum<'readonly' \| 'readonly_when'> }[]` | optional | Write-observability (#3407/#3431): caller-supplied fields that were LEGALLY stripped before the record was written — a non-system create cannot seed a static `readonly` column (#3043 ingress strip), so those keys are dropped and the field re-derives its default. Present ONLY when ≥1 field was dropped; the create still succeeded without them (status/success semantics unchanged). REST additionally surfaces this as the `X-ObjectStack-Dropped-Fields` response header. Optional — omit-when-empty keeps the shape backward-compatible for existing clients. |
 
 
 ---
@@ -1179,6 +1180,7 @@ const result = AiInsightsRequest.parse(data);
 | **object** | `string` | ✅ | Object name |
 | **id** | `string` | ✅ | Updated record ID |
 | **record** | `Record<string, any>` | ✅ | Updated record |
+| **droppedFields** | `{ object: string; fields: string[]; reason: Enum<'readonly' \| 'readonly_when'> }[]` | optional | Write-observability (#3407/#3431): caller-supplied fields the engine LEGALLY stripped from the write before persisting — static `readonly` (#2948) or a TRUE `readonlyWhen` predicate (#3042). Present ONLY when ≥1 field was dropped; the update still succeeded without them (status/success semantics unchanged — stripping is legitimate, not an error). REST additionally surfaces this as the `X-ObjectStack-Dropped-Fields` response header. Optional — omit-when-empty keeps the shape backward-compatible for existing clients that only read `record`. |
 
 
 ---

packages/metadata-protocol/src/protocol.dropped-fields.test.ts

@@ -0,0 +1,124 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+//
+// [#3431] REST/API write paths must not SILENTLY drop caller-supplied fields.
+// #3413 built the engine-level `onFieldsDropped` channel and wired the flow
+// (`update_record`) side; this locks the DataProtocol passthrough that carries
+// the strip back to the REST layer:
+//   - updateData forwards the engine's onFieldsDropped events (readonly /
+//     readonly_when) onto the response as `droppedFields`;
+//   - createData surfaces the #3043 static-`readonly` INGRESS strip, which runs
+//     BEFORE the engine (so it is recovered by diffing the payload, not via the
+//     engine listener) — symmetric with update;
+//   - no strip → NO `droppedFields` key, so the response shape stays
+//     backward-compatible for clients that only read `record`.
+
+import { describe, it, expect, vi } from 'vitest';
+import { ObjectStackProtocolImplementation } from './protocol.js';
+
+const SCHEMA = {
+  name: 'approval_case',
+  fields: {
+    title: { name: 'title', type: 'text' },
+    approval_status: { name: 'approval_status', type: 'text', readonly: true, defaultValue: 'draft' },
+  },
+};
+
+describe('updateData — forwards engine write strips as droppedFields (#3431)', () => {
+  it('surfaces a readonly strip the engine reports via onFieldsDropped', async () => {
+    const engine = {
+      registry: { getObject: () => SCHEMA },
+      // Stand in for the engine stripping `approval_status` and reporting it.
+      update: vi.fn(async (object: string, data: any, options?: any) => {
+        options?.onFieldsDropped?.({ object, fields: ['approval_status'], reason: 'readonly' });
+        return { id: 'rec-1', title: data.title };
+      }),
+      findOne: vi.fn(async () => null),
+    };
+    const p = new ObjectStackProtocolImplementation(engine as any);
+    const res: any = await p.updateData({
+      object: 'approval_case',
+      id: 'rec-1',
+      data: { title: 'B', approval_status: 'approved' },
+      context: { userId: 'u1' },
+    });
+    expect(res.droppedFields).toEqual([
+      { object: 'approval_case', fields: ['approval_status'], reason: 'readonly' },
+    ]);
+    // The write still succeeded; the returned record is unchanged in shape.
+    expect(res.record).toEqual({ id: 'rec-1', title: 'B' });
+  });
+
+  it('forwards multiple strip passes in order (readonly_when then readonly)', async () => {
+    const engine = {
+      registry: { getObject: () => SCHEMA },
+      update: vi.fn(async (object: string, _data: any, options?: any) => {
+        options?.onFieldsDropped?.({ object, fields: ['locked'], reason: 'readonly_when' });
+        options?.onFieldsDropped?.({ object, fields: ['approval_status'], reason: 'readonly' });
+        return { id: 'rec-1' };
+      }),
+      findOne: vi.fn(async () => null),
+    };
+    const p = new ObjectStackProtocolImplementation(engine as any);
+    const res: any = await p.updateData({ object: 'approval_case', id: 'rec-1', data: {} });
+    expect(res.droppedFields).toEqual([
+      { object: 'approval_case', fields: ['locked'], reason: 'readonly_when' },
+      { object: 'approval_case', fields: ['approval_status'], reason: 'readonly' },
+    ]);
+  });
+
+  it('omits droppedFields entirely when the engine stripped nothing', async () => {
+    const engine = {
+      registry: { getObject: () => SCHEMA },
+      update: vi.fn(async (_o: string, data: any) => ({ id: 'rec-1', ...data })),
+      findOne: vi.fn(async () => null),
+    };
+    const p = new ObjectStackProtocolImplementation(engine as any);
+    const res: any = await p.updateData({ object: 'approval_case', id: 'rec-1', data: { title: 'B' } });
+    expect(res).not.toHaveProperty('droppedFields');
+  });
+});
+
+describe('createData — surfaces the #3043 ingress readonly strip as droppedFields (#3431)', () => {
+  function makeProtocol() {
+    const engine = {
+      registry: { getObject: (n: string) => (n === 'approval_case' ? SCHEMA : undefined) },
+      insert: vi.fn(async (_object: string, data: any) => ({ id: 'rec-1', ...data })),
+    };
+    return { p: new ObjectStackProtocolImplementation(engine as any), engine };
+  }
+
+  it('reports the forged readonly field a non-system create dropped', async () => {
+    const { p } = makeProtocol();
+    const res: any = await p.createData({
+      object: 'approval_case',
+      data: { title: 'A', approval_status: 'approved' },
+      context: { userId: 'u1' },
+    });
+    expect(res.droppedFields).toEqual([
+      { object: 'approval_case', fields: ['approval_status'], reason: 'readonly' },
+    ]);
+    // Stripped field is absent from the persisted payload (existing #3043 behaviour).
+    expect(res.record).not.toHaveProperty('approval_status');
+  });
+
+  it('omits droppedFields when the create seeds no readonly field', async () => {
+    const { p } = makeProtocol();
+    const res: any = await p.createData({
+      object: 'approval_case',
+      data: { title: 'A' },
+      context: { userId: 'u1' },
+    });
+    expect(res).not.toHaveProperty('droppedFields');
+  });
+
+  it('a system-context create keeps the field — no strip, no droppedFields', async () => {
+    const { p } = makeProtocol();
+    const res: any = await p.createData({
+      object: 'approval_case',
+      data: { title: 'A', approval_status: 'approved' },
+      context: { isSystem: true },
+    });
+    expect(res).not.toHaveProperty('droppedFields');
+    expect(res.record.approval_status).toBe('approved');
+  });
+});

packages/metadata-protocol/src/protocol.ts

@@ -16,7 +16,7 @@ import type {
 } from '@objectstack/spec/api';
 import type { MetadataCacheRequest, MetadataCacheResponse, ServiceInfo, ApiRoutes, WellKnownCapabilities } from '@objectstack/spec/api';
 import { readServiceSelfInfo } from '@objectstack/spec/api';
-import { parseFilterAST, isFilterAST } from '@objectstack/spec/data';
+import { parseFilterAST, isFilterAST, type DroppedFieldsEvent } from '@objectstack/spec/data';
 import { PLURAL_TO_SINGULAR, SINGULAR_TO_PLURAL } from '@objectstack/spec/shared';
 import { type FormView, isAggregatedViewContainer } from '@objectstack/spec/ui';
 import { METADATA_FORM_REGISTRY } from '@objectstack/spec/system';
@@ -588,6 +588,34 @@ function stripReadonlyForInsert(schema: any, data: any, context: any): any {
     return Array.isArray(data) ? data.map(stripRow) : stripRow(data);
 }
 
+/**
+ * [#3431] Recover a `DroppedFieldsEvent` from a before/after write-payload diff.
+ *
+ * The UPDATE strips (static `readonly` / `readonlyWhen`) run INSIDE the engine,
+ * which reports them via the `onFieldsDropped` listener (wired in `updateData`).
+ * The CREATE `readonly` strip, however, runs at THIS protocol ingress
+ * (`stripReadonlyForInsert`, #3043) — BEFORE the engine — so the engine listener
+ * never sees it. Diffing the caller-supplied keys against the stripped payload
+ * recovers exactly which supplied fields the ingress strip removed, so the create
+ * path can surface them symmetrically with update.
+ *
+ * Returns `null` when nothing was dropped (same reference, non-object, array, or
+ * no key delta) so callers can `if (ev) dropped.push(ev)` without emitting empty
+ * events. Mirrors the engine's own before/after key-set diff (`reportDroppedFields`
+ * in objectql/engine.ts) so both channels agree on what "dropped" means.
+ */
+function diffDroppedFields(
+    object: string,
+    before: unknown,
+    after: unknown,
+    reason: DroppedFieldsEvent['reason'],
+): DroppedFieldsEvent | null {
+    if (before === after || before == null || typeof before !== 'object' || Array.isArray(before)) return null;
+    const afterObj = (after ?? {}) as Record<string, unknown>;
+    const fields = Object.keys(before as Record<string, unknown>).filter((k) => !(k in afterObj));
+    return fields.length > 0 ? { object, fields, reason } : null;
+}
+
 /**
  * Service Configuration for Discovery
  * Maps service names to their routes and plugin providers.
@@ -2829,15 +2857,24 @@ export class ObjectStackProtocolImplementation implements ObjectStackProtocol {
             request.data,
             request.context,
         );
-        const result = await this.engine.insert(
-            request.object,
-            data,
-            request.context !== undefined ? { context: request.context } as any : undefined,
-        );
+        // [#3431] The #3043 ingress strip above is SILENT by contract; surface it
+        // so a REST/API caller learns which supplied fields were dropped, symmetric
+        // with `updateData`. The strip lives at THIS ingress (not the engine, which
+        // is INSERT-readonly-exempt, #3413), so recover it by diffing the supplied
+        // payload against the stripped one. The engine's `onFieldsDropped` is ALSO
+        // wired below so a FUTURE insert-side engine strip surfaces automatically
+        // through the same list instead of going silent.
+        const dropped: DroppedFieldsEvent[] = [];
+        const ingressDropped = diffDroppedFields(request.object, request.data, data, 'readonly');
+        if (ingressDropped) dropped.push(ingressDropped);
+        const opts: any = { onFieldsDropped: (e: DroppedFieldsEvent) => { dropped.push(e); } };
+        if (request.context !== undefined) opts.context = request.context;
+        const result = await this.engine.insert(request.object, data, opts);
         return {
             object: request.object,
             id: result.id,
-            record: result
+            record: result,
+            ...(dropped.length > 0 ? { droppedFields: dropped } : {}),
         };
     }
 
@@ -2928,11 +2965,20 @@ export class ObjectStackProtocolImplementation implements ObjectStackProtocol {
         await this.assertVersionMatch(request.object, request.id, request.expectedVersion, request.context);
         const opts: any = { where: { id: request.id } };
         if (request.context !== undefined) opts.context = request.context;
+        // [#3407/#3431] Capture the engine's LEGAL write strips (static `readonly`
+        // (#2948) / TRUE `readonlyWhen` (#3042)) so a REST/API caller is not left
+        // to field-by-field diff the returned row to discover a value never
+        // landed. The write still succeeds — this only makes the strip observable,
+        // mirroring service-automation's `update_record` wiring (#3413). A faulty
+        // listener never breaks the write (the engine catches + logs).
+        const dropped: DroppedFieldsEvent[] = [];
+        opts.onFieldsDropped = (e: DroppedFieldsEvent) => { dropped.push(e); };
         const result = await this.engine.update(request.object, request.data, opts);
         return {
             object: request.object,
             id: request.id,
-            record: result
+            record: result,
+            ...(dropped.length > 0 ? { droppedFields: dropped } : {}),
         };
     }