diff --git a/packages/dogfood/src/derive.ts b/packages/dogfood/src/derive.ts new file mode 100644 index 0000000000..25d93922e8 --- /dev/null +++ b/packages/dogfood/src/derive.ts @@ -0,0 +1,127 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// Metadata-driven proof derivation. +// +// The platform's apps are 100% declarative metadata, so a baseline runtime +// contract can be DERIVED from the metadata itself — no hand-written tests. This +// is the seed of `objectstack verify`: point it at any app (a framework example +// OR a third-party app like hotcrm) and it auto-generates "author this object → +// write it → read it back → assert type fidelity" for every object, then runs +// it against the real in-process stack. +// +// v0 derives per-object CRUD round-trip cases. RLS cross-owner denial (the +// #1994 invariant) is v1 — it needs the multi-user harness + sharing service. + +/* eslint-disable @typescript-eslint/no-explicit-any */ + +const COMPUTED = new Set(['formula', 'summary', 'autonumber', 'rollup', 'vector']); +const RELATIONAL = new Set(['lookup', 'master_detail', 'master-detail', 'masterdetail', 'tree']); +const STRUCTURED = new Set(['composite', 'repeater', 'record', 'location', 'address']); +const MEDIA = new Set(['image', 'file', 'avatar', 'video', 'audio', 'signature', 'qrcode']); +const SYSTEM_NAMES = new Set([ + 'id', 'created_at', 'updated_at', 'created_by', 'updated_by', 'owner', + 'space', 'instance_state', 'record_id', 'is_deleted', +]); + +export type AssertKind = 'equal' | 'set' | 'none'; +export interface DerivedAssert { + field: string; + type: string; + value: unknown; + kind: AssertKind; +} +export interface CrudCase { + object: string; + blocked?: string; // why this object can't be auto-CRUD'd (e.g. required lookup) + body?: Record; + asserts?: DerivedAssert[]; + skippedFields?: Array<{ name: string; type: string; reason: string }>; +} + +function clampNum(f: any, fallback: number): number { + const { min, max, step } = f; + let v = fallback; + if (typeof min === 'number' && v < min) v = min; + if (typeof max === 'number' && v > max) v = max; + if (typeof step === 'number' && typeof min === 'number') { + v = min + step * Math.round((v - min) / step); + } + return v; +} + +/** Synthesize a valid value for a field type, or null if not synthesizable. */ +function synth(type: string, f: any): { value: unknown; kind: AssertKind } | null { + switch (type) { + case 'text': case 'textarea': case 'string': + case 'markdown': case 'html': case 'richtext': case 'code': + return { value: 'verify-sample', kind: 'equal' }; + case 'email': return { value: 'verify@example.com', kind: 'equal' }; + case 'url': return { value: 'https://example.com', kind: 'equal' }; + case 'phone': return { value: '+14155550100', kind: 'equal' }; + case 'color': return { value: '#3366CC', kind: 'equal' }; + case 'number': return { value: clampNum(f, 7), kind: 'equal' }; + case 'currency': return { value: clampNum(f, 100), kind: 'equal' }; + case 'percent': return { value: clampNum(f, 50), kind: 'equal' }; + case 'rating': return { value: clampNum(f, Math.min(3, f.max ?? 5)), kind: 'equal' }; + case 'slider': case 'progress': return { value: clampNum(f, 25), kind: 'equal' }; + case 'boolean': case 'toggle': return { value: true, kind: 'equal' }; + case 'date': return { value: '2024-03-15', kind: 'equal' }; + case 'datetime': return { value: '2024-03-15T08:30:00.000Z', kind: 'equal' }; + case 'time': return { value: '14:30:00', kind: 'equal' }; + case 'json': return { value: { sample: true }, kind: 'equal' }; + case 'select': case 'radio': { + const opt = f.options?.[0]?.value; + return opt != null ? { value: opt, kind: 'equal' } : null; + } + case 'multiselect': case 'checkboxes': { + const opt = f.options?.[0]?.value; + return opt != null ? { value: [opt], kind: 'set' } : null; + } + case 'tags': return { value: ['alpha', 'beta'], kind: 'set' }; + // Opaque-on-read: write a value but don't assert a round-trip (hashed/encrypted). + case 'password': case 'secret': return { value: 'Sample-Secret-123', kind: 'none' }; + default: return null; + } +} + +/** + * Derive one CRUD round-trip case per authorable object in the config. + * An object whose REQUIRED fields can't be synthesized (e.g. a required lookup + * needing a target record) is reported `blocked` rather than silently skipped. + */ +export function deriveCrudCases(config: any): CrudCase[] { + const cases: CrudCase[] = []; + for (const obj of config?.objects ?? []) { + const fields: Record = obj?.fields ?? {}; + const body: Record = {}; + const asserts: DerivedAssert[] = []; + const skippedFields: Array<{ name: string; type: string; reason: string }> = []; + let blocked: string | undefined; + + for (const [name, f] of Object.entries(fields)) { + const type = String((f as any)?.type ?? '').toLowerCase(); + if (SYSTEM_NAMES.has(name) || (f as any)?.system || (f as any)?.readonly) continue; + if (COMPUTED.has(type)) continue; + + if (RELATIONAL.has(type) || STRUCTURED.has(type) || MEDIA.has(type)) { + if ((f as any)?.required) { blocked = `required ${type} field "${name}" (needs target/structured value)`; break; } + skippedFields.push({ name, type, reason: 'unsynthesizable-optional' }); + continue; + } + + const s = synth(type, f); + if (!s) { + if ((f as any)?.required) { blocked = `required field "${name}" of type "${type}" is not synthesizable`; break; } + skippedFields.push({ name, type, reason: 'no-synth' }); + continue; + } + body[name] = s.value; + if (s.kind !== 'none') asserts.push({ field: name, type, value: s.value, kind: s.kind }); + } + + // Every object needs a name-ish required text; synth already covers `name`. + if (blocked) cases.push({ object: obj.name, blocked }); + else cases.push({ object: obj.name, body, asserts, skippedFields }); + } + return cases; +} diff --git a/packages/dogfood/src/harness.ts b/packages/dogfood/src/harness.ts index cf0858fa41..e10c8475ed 100644 --- a/packages/dogfood/src/harness.ts +++ b/packages/dogfood/src/harness.ts @@ -22,7 +22,7 @@ import { HonoServerPlugin } from '@objectstack/plugin-hono-server'; import { createRestApiPlugin } from '@objectstack/rest'; import { AuthPlugin } from '@objectstack/plugin-auth'; import { SecurityPlugin } from '@objectstack/plugin-security'; -import { SettingsServicePlugin } from '@objectstack/service-settings'; +import { SettingsServicePlugin, LocalCryptoProvider } from '@objectstack/service-settings'; import { AnalyticsServicePlugin } from '@objectstack/service-analytics'; /** A Hono app exposes `.request(path, init)` returning a standard `Response`. */ @@ -94,6 +94,18 @@ export async function bootDogfoodStack( // Fire the ready lifecycle: seed data, dev-admin bootstrap, route registration. await kernel.bootstrap(); + // Secret fields (Field.secret) refuse to persist without a crypto provider — + // mirror `objectstack dev`, which wires LocalCryptoProvider in development so + // an app with an encrypted field is exercisable end-to-end. + try { + const engine = await kernel.getServiceAsync<{ setCryptoProvider?: (p: unknown) => void }>('objectql'); + if (engine && typeof engine.setCryptoProvider === 'function') { + engine.setCryptoProvider(new LocalCryptoProvider()); + } + } catch { + /* no engine / no crypto support — secret fields will fail closed, as in prod */ + } + const httpServer = await kernel.getServiceAsync<{ getRawApp(): InjectableApp; close?(): Promise }>( 'http-server', ); diff --git a/packages/dogfood/src/verify.ts b/packages/dogfood/src/verify.ts new file mode 100644 index 0000000000..c28fa5f590 --- /dev/null +++ b/packages/dogfood/src/verify.ts @@ -0,0 +1,145 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// The verify runner — executes derived proofs against a live dogfood stack. +// +// This is what a consumer (a framework example OR a third-party app like hotcrm) +// would run: boot my app in-process, auto-derive a runtime contract from my +// metadata, exercise it through the real HTTP surface, and tell me where the +// declared behavior doesn't actually hold at runtime. + +/* eslint-disable @typescript-eslint/no-explicit-any */ + +import type { DogfoodStack } from './harness.js'; +import { deriveCrudCases, type CrudCase } from './derive.js'; + +export interface ObjectVerifyResult { + object: string; + status: 'verified' | 'fidelity-gaps' | 'create-failed' | 'read-failed' | 'skipped' | 'needs-fixture'; + checked?: number; + reason?: string; + code?: number; + detail?: string; + mismatches?: Array<{ field: string; type: string; wrote: unknown; read: unknown }>; +} + +export interface VerifyReport { + app: string; + results: ObjectVerifyResult[]; + summary: { + objects: number; + verified: number; + fidelityGaps: number; + createFailed: number; + needsFixture: number; + readFailed: number; + skipped: number; + mismatchTotal: number; + }; +} + +function setEqual(a: unknown, b: unknown[]): boolean { + if (!Array.isArray(a)) return false; + return JSON.stringify([...a].sort()) === JSON.stringify([...b].sort()); +} +function deepEqual(a: unknown, b: unknown): boolean { + return JSON.stringify(a) === JSON.stringify(b); +} + +/** + * Run auto-derived CRUD round-trip proofs for every object in `config`, as the + * authenticated `token`, against the live `stack`. Returns a structured report; + * never throws on a per-object failure (collects them). + */ +export async function runCrudVerification( + stack: DogfoodStack, + token: string, + config: any, +): Promise { + const cases = deriveCrudCases(config); + const results: ObjectVerifyResult[] = []; + + for (const c of cases as CrudCase[]) { + if (c.blocked) { + results.push({ object: c.object, status: 'skipped', reason: c.blocked }); + continue; + } + let created: Response; + try { + created = await stack.apiAs(token, 'POST', `/data/${c.object}`, c.body); + } catch (e: any) { + results.push({ object: c.object, status: 'create-failed', detail: String(e?.message ?? e).slice(0, 200) }); + continue; + } + if (created.status >= 300) { + const text = (await created.text()).slice(0, 280); + // A 400 validation failure means our auto-derived record didn't satisfy the + // app's own validation rules (record-level format/json-schema/CEL) — that's + // a fixture gap, not a platform finding. A 5xx (or crypto/driver error) is a + // real runtime failure the app's author needs to see. + const isValidation = created.status === 400 && /VALIDATION_FAILED|Validation failed/i.test(text); + results.push({ + object: c.object, + status: isValidation ? 'needs-fixture' : 'create-failed', + code: created.status, + detail: text, + }); + continue; + } + const cj = (await created.json()) as any; + const id = cj?.id ?? cj?.record?.id; + if (!id) { + results.push({ object: c.object, status: 'create-failed', detail: 'no id returned' }); + continue; + } + + const got = await stack.apiAs(token, 'GET', `/data/${c.object}/${id}`); + if (got.status !== 200) { + results.push({ object: c.object, status: 'read-failed', code: got.status }); + continue; + } + const rec = ((await got.json()) as any)?.record ?? {}; + + const mismatches: ObjectVerifyResult['mismatches'] = []; + for (const a of c.asserts ?? []) { + const actual = rec[a.field]; + const ok = a.kind === 'set' ? setEqual(actual, a.value as unknown[]) : deepEqual(actual, a.value); + if (!ok) mismatches.push({ field: a.field, type: a.type, wrote: a.value, read: actual }); + } + results.push({ + object: c.object, + status: mismatches.length ? 'fidelity-gaps' : 'verified', + checked: (c.asserts ?? []).length, + ...(mismatches.length ? { mismatches } : {}), + }); + } + + const summary = { + objects: results.length, + verified: results.filter((r) => r.status === 'verified').length, + fidelityGaps: results.filter((r) => r.status === 'fidelity-gaps').length, + createFailed: results.filter((r) => r.status === 'create-failed').length, + needsFixture: results.filter((r) => r.status === 'needs-fixture').length, + readFailed: results.filter((r) => r.status === 'read-failed').length, + skipped: results.filter((r) => r.status === 'skipped').length, + mismatchTotal: results.reduce((n, r) => n + (r.mismatches?.length ?? 0), 0), + }; + return { app: config?.manifest?.id ?? config?.manifest?.namespace ?? 'app', results, summary }; +} + +/** Pretty one-line-per-object summary for logs. */ +export function formatReport(report: VerifyReport): string { + const lines: string[] = [`\n=== objectstack verify — ${report.app} ===`]; + for (const r of report.results) { + if (r.status === 'verified') lines.push(` ✓ ${r.object} (${r.checked} fields)`); + else if (r.status === 'fidelity-gaps') { + lines.push(` ⚠ ${r.object} ${r.mismatches!.length} fidelity gap(s):`); + for (const m of r.mismatches!) lines.push(` ${m.field} <${m.type}>: wrote ${JSON.stringify(m.wrote)} → read ${JSON.stringify(m.read)}`); + } + else if (r.status === 'skipped') lines.push(` – ${r.object} skipped: ${r.reason}`); + else if (r.status === 'needs-fixture') lines.push(` ~ ${r.object} needs-fixture (app validation rejected the auto-record): ${(r.detail ?? '').slice(0,120)}`); + else lines.push(` ✗ ${r.object} ${r.status}${r.code ? ` (${r.code})` : ''}: ${r.detail ?? ''}`); + } + const s = report.summary; + lines.push(` ── ${s.verified} verified, ${s.fidelityGaps} gaps, ${s.createFailed + s.readFailed} FAILED, ${s.needsFixture} needs-fixture, ${s.skipped} skipped (${s.mismatchTotal} mismatches)`); + return lines.join('\n'); +} diff --git a/packages/dogfood/test/auto-verify.dogfood.test.ts b/packages/dogfood/test/auto-verify.dogfood.test.ts new file mode 100644 index 0000000000..fef89decd0 --- /dev/null +++ b/packages/dogfood/test/auto-verify.dogfood.test.ts @@ -0,0 +1,54 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// objectstack verify — metadata-driven runtime verification, proven against the +// framework's own example apps. From each app's metadata ALONE it auto-derives a +// CRUD round-trip contract (no hand-written cases) and runs it over real HTTP. +// The same runner points at any third-party app's built artifact via +// `test/verify-external.dogfood.test.ts`. + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import crmStack from '@objectstack/example-crm'; +import showcaseStack from '@objectstack/example-showcase'; +import { bootDogfoodStack, type DogfoodStack } from '../src/harness.js'; +import { runCrudVerification, formatReport, type VerifyReport } from '../src/verify.js'; + +const APPS: Array<[string, unknown]> = [ + ['crm', crmStack], + ['showcase', showcaseStack], +]; + +for (const [name, config] of APPS) { + describe(`objectstack verify: ${name} (auto-derived CRUD round-trip)`, () => { + let stack: DogfoodStack; + let report: VerifyReport; + + beforeAll(async () => { + stack = await bootDogfoodStack(config as never); + const token = await stack.signIn(); + report = await runCrudVerification(stack, token, config); + // eslint-disable-next-line no-console + console.error(formatReport(report)); + }, 60_000); + + afterAll(async () => { + await stack?.stop(); + }); + + it('derives a runtime contract from metadata and boots', () => { + expect(report.summary.objects).toBeGreaterThan(0); + }); + + it('verifies objects end-to-end over real HTTP', () => { + expect(report.summary.verified).toBeGreaterThan(0); + }); + + it('has no object that fails to create or read (the hard runtime invariant)', () => { + // create/read failures = the app's metadata produces a record the runtime + // refuses or can't read back — a real platform/integration finding (vs + // `needs-fixture`, the auto-record tripping the app's own validation rules, + // and `fidelity-gaps`, type leaks tracked separately). + const hard = report.results.filter((r) => r.status === 'create-failed' || r.status === 'read-failed'); + expect(hard, JSON.stringify(hard, null, 2)).toHaveLength(0); + }); + }); +} diff --git a/packages/dogfood/test/verify-external.dogfood.test.ts b/packages/dogfood/test/verify-external.dogfood.test.ts new file mode 100644 index 0000000000..2ab1725276 --- /dev/null +++ b/packages/dogfood/test/verify-external.dogfood.test.ts @@ -0,0 +1,40 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// Point the verifier at ANY app's built artifact — the embryonic +// `objectstack verify `. This is the consumer-facing use: a third-party app +// (e.g. hotcrm) runs it against its OWN metadata to learn where its declared +// behavior doesn't hold at runtime. +// +// Gated on OS_VERIFY_ARTIFACT so it never runs in framework CI (the external app +// isn't in the workspace). Run locally: +// OS_VERIFY_ARTIFACT=/abs/path/to/app/dist/objectstack.json \ +// pnpm --filter @objectstack/dogfood exec vitest run test/verify-external.dogfood.test.ts + +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { readFileSync, existsSync } from 'node:fs'; +import { bootDogfoodStack, type DogfoodStack } from '../src/harness.js'; +import { runCrudVerification, formatReport, type VerifyReport } from '../src/verify.js'; + +const ARTIFACT = process.env.OS_VERIFY_ARTIFACT; + +describe.skipIf(!ARTIFACT || !existsSync(ARTIFACT))('objectstack verify: external app artifact', () => { + let stack: DogfoodStack; + let report: VerifyReport; + + beforeAll(async () => { + const config = JSON.parse(readFileSync(ARTIFACT as string, 'utf8')); + stack = await bootDogfoodStack(config); + const token = await stack.signIn(); + report = await runCrudVerification(stack, token, config); + // eslint-disable-next-line no-console + console.error(formatReport(report)); + }, 120_000); + + afterAll(async () => { + await stack?.stop(); + }); + + it('boots the external app and auto-derives a runtime contract', () => { + expect(report.summary.objects).toBeGreaterThan(0); + }); +});