From 69360b9be1a1f6d791e20acb88e57d8002ff785c Mon Sep 17 00:00:00 2001 From: os-zhuang Date: Thu, 18 Jun 2026 14:00:14 +0800 Subject: [PATCH] test(dogfood): field-type capability-matrix round-trip (#2004 + AI-authoring) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extends the dogfood gate from "pin one example app" toward the platform's real risk: third parties have an AI author arbitrary metadata, so every authorable primitive needs a runtime proof — not a static shape check. `showcase_field_zoo` carries one field of (almost) every protocol FieldType but was only static-checked (the bundle registers it). This adds the first capability-matrix block: write one record covering many field types over the real REST data API, read it back, assert each round-trips with type fidelity. - Headliner coverage = the #2004 regressions: array-typed fields (multiselect/checkboxes/tags) and Field.time (time-of-day) — all proven to round-trip end-to-end on the in-memory (WASM) driver path `objectstack dev` uses for :memory:. - On first run it surfaced three real type-fidelity gaps — rating/slider/toggle persist but read back as strings ('4'/'25'/'1') while number/currency/percent/ boolean round-trip correctly. They are quarantined with `it.fails` (passes while broken, turns RED when fixed) rather than hidden by coercion, and filed as a separate task. Private package — no changeset. 21 passed + 3 expected-fail. Co-Authored-By: Claude Opus 4.8 --- packages/dogfood/README.md | 27 ++++ packages/dogfood/package.json | 3 +- .../test/field-zoo-roundtrip.dogfood.test.ts | 132 ++++++++++++++++++ pnpm-lock.yaml | 3 + 4 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 packages/dogfood/test/field-zoo-roundtrip.dogfood.test.ts diff --git a/packages/dogfood/README.md b/packages/dogfood/README.md index 4febd3ea6c..bd5b930386 100644 --- a/packages/dogfood/README.md +++ b/packages/dogfood/README.md @@ -36,4 +36,31 @@ sockets, CI-stable). Tests act as a browser client would: sign in, hit 4. **Prove it catches the bug**: temporarily revert the relevant fix and confirm the test goes red. A green-on-the-bug test is not a gate. +## Capability matrix (the AI-authoring angle) + +This is a **development platform**: third parties have an AI author arbitrary +metadata. The risk is not "a platform change broke the CRM example" — it's "the +AI used a valid primitive the examples don't exercise, and it silently breaks at +runtime." So beyond pinning the example apps, the gate is growing into a +**capability matrix**: every authorable primitive gets a runtime round-trip +proof. + +`test/field-zoo-roundtrip.dogfood.test.ts` is the first block — it writes one +record covering many field types over the real REST API and asserts each reads +back with type fidelity. On its first run it surfaced three real gaps +(`rating`/`slider`/`toggle` read back as strings), which are **quarantined**, not +hidden: + +```ts +// Known type-fidelity gap → it.fails passes while broken and turns RED the day +// it's fixed, forcing the quarantine to be lifted instead of rotting. +const runner = c.xfail ? it.fails : it; +``` + +Roadmap (each its own slice, see the ADR): flow-node matrix, form-widget matrix, +RLS-pattern matrix; then a generative pass that emits random valid metadata from +the spec's zod surface; then run the AI **template corpus** through the harness. +The binding policy — every authorable+live primitive must carry a runtime proof +— is the subject of a dedicated ADR. + Runs in CI as the `Dogfood Regression Gate` job (and under `turbo run test`). diff --git a/packages/dogfood/package.json b/packages/dogfood/package.json index 5390f5d0f9..4909305c6a 100644 --- a/packages/dogfood/package.json +++ b/packages/dogfood/package.json @@ -20,7 +20,8 @@ "@objectstack/plugin-security": "workspace:*", "@objectstack/service-settings": "workspace:*", "@objectstack/service-analytics": "workspace:*", - "@objectstack/example-crm": "workspace:*" + "@objectstack/example-crm": "workspace:*", + "@objectstack/example-showcase": "workspace:*" }, "devDependencies": { "@types/node": "^25.9.3", diff --git a/packages/dogfood/test/field-zoo-roundtrip.dogfood.test.ts b/packages/dogfood/test/field-zoo-roundtrip.dogfood.test.ts new file mode 100644 index 0000000000..95239ff1d9 --- /dev/null +++ b/packages/dogfood/test/field-zoo-roundtrip.dogfood.test.ts @@ -0,0 +1,132 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// CAPABILITY-MATRIX golden test — every authorable field type must survive a +// real HTTP write → read round-trip. +// +// `showcase_field_zoo` carries one field of (almost) every protocol FieldType. +// Until now it was only *static*-checked (the metadata bundle registers it); +// nothing wrote a record and read it back. But the platform's value is that an +// AI can author ANY field and it works at runtime — so each field type needs a +// runtime proof, not a shape assertion. This is the first block of that matrix +// and the direct guard for #2004 (array-typed fields silently failed to persist; +// Field.time rejected time-of-day). + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import showcaseStack from '@objectstack/example-showcase'; +import { bootDogfoodStack, type DogfoodStack } from '../src/harness.js'; + +// A field-type coverage entry. `write` is the value POSTed; `expect` describes +// how the value must come back. `equal` = exact (or set-equal for arrays); +// `computed`/`present` cover server-owned fields you don't write. +type Check = + | { kind: 'equal'; write: unknown } + | { kind: 'setEqual'; write: unknown[] } + | { kind: 'present' } // server-assigned, just must be non-null + | { kind: 'computed'; expected: unknown }; // derived, asserted not written + +interface FieldCase { + field: string; + type: string; + check: Check; + // KNOWN GAP: the schema→SQL-column mapping / read coercion doesn't yet cover + // this type, so it round-trips with the wrong JS type (e.g. '4' not 4). The + // value persists (no data loss), but fidelity leaks. Quarantined via it.fails + // so it stays visible AND auto-flags the day it's fixed. Tracked separately. + xfail?: boolean; +} + +// The #2004 headliners are the three array types + f_time. The rest broaden the +// matrix across scalars, temporals, structured JSON, and computed/system fields. +const MATRIX: FieldCase[] = [ + // text-ish + { field: 'f_textarea', type: 'textarea', check: { kind: 'equal', write: 'line1\nline2' } }, + { field: 'f_email', type: 'email', check: { kind: 'equal', write: 'zoo@example.com' } }, + { field: 'f_url', type: 'url', check: { kind: 'equal', write: 'https://objectstack.ai' } }, + { field: 'f_phone', type: 'phone', check: { kind: 'equal', write: '+14155550123' } }, + // numbers + { field: 'f_number', type: 'number', check: { kind: 'equal', write: 42 } }, + { field: 'f_currency', type: 'currency', check: { kind: 'equal', write: 1234.56 } }, + { field: 'f_percent', type: 'percent', check: { kind: 'equal', write: 75 } }, + { field: 'f_rating', type: 'rating', check: { kind: 'equal', write: 4 }, xfail: true }, + { field: 'f_slider', type: 'slider', check: { kind: 'equal', write: 25 }, xfail: true }, + // temporal — f_time is a #2004 fix (time-of-day) + { field: 'f_date', type: 'date', check: { kind: 'equal', write: '2024-03-15' } }, + { field: 'f_time', type: 'time', check: { kind: 'equal', write: '14:30:00' } }, + // logic + { field: 'f_boolean', type: 'boolean', check: { kind: 'equal', write: true } }, + { field: 'f_toggle', type: 'toggle', check: { kind: 'equal', write: true }, xfail: true }, + // scalar selection + { field: 'f_select', type: 'select', check: { kind: 'equal', write: 'high' } }, + { field: 'f_radio', type: 'radio', check: { kind: 'equal', write: 'yes' } }, + // ── #2004 array headliners — these silently dropped before the fix ── + { field: 'f_multiselect', type: 'multiselect', check: { kind: 'setEqual', write: ['red', 'blue'] } }, + { field: 'f_checkboxes', type: 'checkboxes', check: { kind: 'setEqual', write: ['email', 'push'] } }, + { field: 'f_tags', type: 'tags', check: { kind: 'setEqual', write: ['alpha', 'beta', 'gamma'] } }, + // structured JSON + { field: 'f_json', type: 'json', check: { kind: 'equal', write: { a: 1, b: [2, 3] } } }, + { field: 'f_color', type: 'color', check: { kind: 'equal', write: '#FF8800' } }, + // computed / system — not written, must materialize + { field: 'f_autonumber', type: 'autonumber', check: { kind: 'present' } }, + // f_number(42) * f_percent(75) / 100 = 31.5 + { field: 'f_formula', type: 'formula', check: { kind: 'computed', expected: 31.5 } }, +]; + +describe('dogfood: field-type capability matrix round-trips over HTTP (#2004)', () => { + let stack: DogfoodStack; + let record: Record; + + beforeAll(async () => { + stack = await bootDogfoodStack(showcaseStack); + const token = await stack.signIn(); + + // Build the create body from every `equal`/`setEqual` entry (+ required name). + const body: Record = { name: 'zoo-roundtrip' }; + for (const c of MATRIX) { + if (c.check.kind === 'equal') body[c.field] = c.check.write; + else if (c.check.kind === 'setEqual') body[c.field] = c.check.write; + } + + const created = await stack.apiAs(token, 'POST', '/data/showcase_field_zoo', body); + expect(created.status, `create failed: ${created.status} ${await created.clone().text()}`).toBeLessThan(300); + const createdJson = (await created.json()) as { id?: string; record?: { id?: string } }; + const id = createdJson.id ?? createdJson.record?.id; + expect(id, 'no id returned from create').toBeTruthy(); + + const got = await stack.apiAs(token, 'GET', `/data/showcase_field_zoo/${id}`); + expect(got.status).toBe(200); + const gotJson = (await got.json()) as { record?: Record } & Record; + record = (gotJson.record ?? gotJson) as Record; + }, 60_000); + + afterAll(async () => { + await stack?.stop(); + }); + + for (const c of MATRIX) { + // xfail entries are KNOWN type-fidelity gaps: `it.fails` passes while the + // assertion throws and turns RED the moment the gap is fixed — forcing the + // quarantine to be lifted rather than silently rotting. + const runner = c.xfail ? it.fails : it; + runner(`${c.type} (${c.field}) round-trips`, () => { + const actual = record[c.field]; + switch (c.check.kind) { + case 'equal': + expect(actual).toEqual(c.check.write); + break; + case 'setEqual': { + // Array-typed fields: persisted as a JSON array; order is not + // guaranteed, so compare as sets (the #2004 break returned null/[]). + expect(Array.isArray(actual), `${c.field} not an array: ${JSON.stringify(actual)}`).toBe(true); + expect([...(actual as unknown[])].sort()).toEqual([...c.check.write].sort()); + break; + } + case 'present': + expect(actual ?? null, `${c.field} should be server-assigned`).not.toBeNull(); + break; + case 'computed': + expect(Number(actual)).toBeCloseTo(Number(c.check.expected), 5); + break; + } + }); + } +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bb28941280..c2cee6c7d7 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -823,6 +823,9 @@ importers: '@objectstack/example-crm': specifier: workspace:* version: link:../../examples/app-crm + '@objectstack/example-showcase': + specifier: workspace:* + version: link:../../examples/app-showcase '@objectstack/objectql': specifier: workspace:* version: link:../objectql