feat(dogfood): metadata-driven verifier (objectstack verify) — auto-derive runtime proofs (#2027)

os-zhuang · claude · web-flow · commit 311bcc63afc0 · 2026-06-18T14:51:52.000+08:00
The dogfood gate pins the framework's own example apps; it never sees a third
party's metadata. But the platform's value is that an AI authors arbitrary
metadata and it works at runtime — for a SPECIFIC consumer app the framework CI
can't reach.

This adds the seed of `objectstack verify`: derive a runtime contract from an
app's metadata ALONE (no hand-written tests), then run it against the real
in-process stack. v0 derives per-object CRUD round-trip — synthesize a valid
create body from each object's field types, POST it, GET it back, assert type
fidelity.

- src/derive.ts — metadata → CRUD cases. Synthesizes values per field type;
  reports objects blocked by un-synthesizable required fields (e.g. a required
  lookup) rather than silently skipping.
- src/verify.ts — runs the derived cases over real HTTP, returns a structured
  report. Classifies outcomes: verified / fidelity-gaps (type leaks) /
  needs-fixture (the auto-record tripped the app's own validation rules) /
  create-failed|read-failed (real runtime failures) / skipped.
- harness.ts — wire LocalCryptoProvider (as `objectstack dev` does) so apps with
  a Field.secret are exercisable end-to-end.
- test/auto-verify.dogfood.test.ts — runs it on CRM + showcase; gates on "no
  object fails to create/read". On showcase it auto-found f_progress's
  type-fidelity gap too — which the hand-written field-zoo matrix had missed.
- test/verify-external.dogfood.test.ts — OS_VERIFY_ARTIFACT=&lt;app&gt;/dist/objectstack.json
  points it at ANY app's built artifact (the consumer entry point; skipped in CI).

Proven against hotcrm (15-object third-party app, separate repo, zero dogfood
dependency): booted in-process, seeded 162 records, auto-verified — found the
platform rating type-leak in crm_case/crm_lead AND a hotcrm-specific write
divergence (crm_product.sku normalized to uppercase) that no framework test
could reach.

Private package — no changeset.

Co-authored-by: Claude Opus 4.8 &lt;noreply@anthropic.com&gt;
diff --git a/packages/dogfood/src/derive.ts b/packages/dogfood/src/derive.ts
@@ -0,0 +1,127 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+//
+// Metadata-driven proof derivation.
+//
+// The platform's apps are 100% declarative metadata, so a baseline runtime
+// contract can be DERIVED from the metadata itself — no hand-written tests. This
+// is the seed of `objectstack verify`: point it at any app (a framework example
+// OR a third-party app like hotcrm) and it auto-generates "author this object →
+// write it → read it back → assert type fidelity" for every object, then runs
+// it against the real in-process stack.
+//
+// v0 derives per-object CRUD round-trip cases. RLS cross-owner denial (the
+// #1994 invariant) is v1 — it needs the multi-user harness + sharing service.
+
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+const COMPUTED = new Set(['formula', 'summary', 'autonumber', 'rollup', 'vector']);
+const RELATIONAL = new Set(['lookup', 'master_detail', 'master-detail', 'masterdetail', 'tree']);
+const STRUCTURED = new Set(['composite', 'repeater', 'record', 'location', 'address']);
+const MEDIA = new Set(['image', 'file', 'avatar', 'video', 'audio', 'signature', 'qrcode']);
+const SYSTEM_NAMES = new Set([
+  'id', 'created_at', 'updated_at', 'created_by', 'updated_by', 'owner',
+  'space', 'instance_state', 'record_id', 'is_deleted',
+]);
+
+export type AssertKind = 'equal' | 'set' | 'none';
+export interface DerivedAssert {
+  field: string;
+  type: string;
+  value: unknown;
+  kind: AssertKind;
+}
+export interface CrudCase {
+  object: string;
+  blocked?: string; // why this object can't be auto-CRUD'd (e.g. required lookup)
+  body?: Record<string, unknown>;
+  asserts?: DerivedAssert[];
+  skippedFields?: Array<{ name: string; type: string; reason: string }>;
+}
+
+function clampNum(f: any, fallback: number): number {
+  const { min, max, step } = f;
+  let v = fallback;
+  if (typeof min === 'number' && v < min) v = min;
+  if (typeof max === 'number' && v > max) v = max;
+  if (typeof step === 'number' && typeof min === 'number') {
+    v = min + step * Math.round((v - min) / step);
+  }
+  return v;
+}
+
+/** Synthesize a valid value for a field type, or null if not synthesizable. */
+function synth(type: string, f: any): { value: unknown; kind: AssertKind } | null {
+  switch (type) {
+    case 'text': case 'textarea': case 'string':
+    case 'markdown': case 'html': case 'richtext': case 'code':
+      return { value: 'verify-sample', kind: 'equal' };
+    case 'email': return { value: 'verify@example.com', kind: 'equal' };
+    case 'url': return { value: 'https://example.com', kind: 'equal' };
+    case 'phone': return { value: '+14155550100', kind: 'equal' };
+    case 'color': return { value: '#3366CC', kind: 'equal' };
+    case 'number': return { value: clampNum(f, 7), kind: 'equal' };
+    case 'currency': return { value: clampNum(f, 100), kind: 'equal' };
+    case 'percent': return { value: clampNum(f, 50), kind: 'equal' };
+    case 'rating': return { value: clampNum(f, Math.min(3, f.max ?? 5)), kind: 'equal' };
+    case 'slider': case 'progress': return { value: clampNum(f, 25), kind: 'equal' };
+    case 'boolean': case 'toggle': return { value: true, kind: 'equal' };
+    case 'date': return { value: '2024-03-15', kind: 'equal' };
+    case 'datetime': return { value: '2024-03-15T08:30:00.000Z', kind: 'equal' };
+    case 'time': return { value: '14:30:00', kind: 'equal' };
+    case 'json': return { value: { sample: true }, kind: 'equal' };
+    case 'select': case 'radio': {
+      const opt = f.options?.[0]?.value;
+      return opt != null ? { value: opt, kind: 'equal' } : null;
+    }
+    case 'multiselect': case 'checkboxes': {
+      const opt = f.options?.[0]?.value;
+      return opt != null ? { value: [opt], kind: 'set' } : null;
+    }
+    case 'tags': return { value: ['alpha', 'beta'], kind: 'set' };
+    // Opaque-on-read: write a value but don't assert a round-trip (hashed/encrypted).
+    case 'password': case 'secret': return { value: 'Sample-Secret-123', kind: 'none' };
+    default: return null;
+  }
+}
+
+/**
+ * Derive one CRUD round-trip case per authorable object in the config.
+ * An object whose REQUIRED fields can't be synthesized (e.g. a required lookup
+ * needing a target record) is reported `blocked` rather than silently skipped.
+ */
+export function deriveCrudCases(config: any): CrudCase[] {
+  const cases: CrudCase[] = [];
+  for (const obj of config?.objects ?? []) {
+    const fields: Record<string, any> = obj?.fields ?? {};
+    const body: Record<string, unknown> = {};
+    const asserts: DerivedAssert[] = [];
+    const skippedFields: Array<{ name: string; type: string; reason: string }> = [];
+    let blocked: string | undefined;
+
+    for (const [name, f] of Object.entries(fields)) {
+      const type = String((f as any)?.type ?? '').toLowerCase();
+      if (SYSTEM_NAMES.has(name) || (f as any)?.system || (f as any)?.readonly) continue;
+      if (COMPUTED.has(type)) continue;
+
+      if (RELATIONAL.has(type) || STRUCTURED.has(type) || MEDIA.has(type)) {
+        if ((f as any)?.required) { blocked = `required ${type} field "${name}" (needs target/structured value)`; break; }
+        skippedFields.push({ name, type, reason: 'unsynthesizable-optional' });
+        continue;
+      }
+
+      const s = synth(type, f);
+      if (!s) {
+        if ((f as any)?.required) { blocked = `required field "${name}" of type "${type}" is not synthesizable`; break; }
+        skippedFields.push({ name, type, reason: 'no-synth' });
+        continue;
+      }
+      body[name] = s.value;
+      if (s.kind !== 'none') asserts.push({ field: name, type, value: s.value, kind: s.kind });
+    }
+
+    // Every object needs a name-ish required text; synth already covers `name`.
+    if (blocked) cases.push({ object: obj.name, blocked });
+    else cases.push({ object: obj.name, body, asserts, skippedFields });
+  }
+  return cases;
+}
diff --git a/packages/dogfood/src/harness.ts b/packages/dogfood/src/harness.ts
@@ -22,7 +22,7 @@ import { HonoServerPlugin } from '@objectstack/plugin-hono-server';
 import { createRestApiPlugin } from '@objectstack/rest';
 import { AuthPlugin } from '@objectstack/plugin-auth';
 import { SecurityPlugin } from '@objectstack/plugin-security';
-import { SettingsServicePlugin } from '@objectstack/service-settings';
+import { SettingsServicePlugin, LocalCryptoProvider } from '@objectstack/service-settings';
 import { AnalyticsServicePlugin } from '@objectstack/service-analytics';
 
 /** A Hono app exposes `.request(path, init)` returning a standard `Response`. */
@@ -94,6 +94,18 @@ export async function bootDogfoodStack(
   // Fire the ready lifecycle: seed data, dev-admin bootstrap, route registration.
   await kernel.bootstrap();
 
+  // Secret fields (Field.secret) refuse to persist without a crypto provider —
+  // mirror `objectstack dev`, which wires LocalCryptoProvider in development so
+  // an app with an encrypted field is exercisable end-to-end.
+  try {
+    const engine = await kernel.getServiceAsync<{ setCryptoProvider?: (p: unknown) => void }>('objectql');
+    if (engine && typeof engine.setCryptoProvider === 'function') {
+      engine.setCryptoProvider(new LocalCryptoProvider());
+    }
+  } catch {
+    /* no engine / no crypto support — secret fields will fail closed, as in prod */
+  }
+
   const httpServer = await kernel.getServiceAsync<{ getRawApp(): InjectableApp; close?(): Promise<void> }>(
     'http-server',
   );
diff --git a/packages/dogfood/src/verify.ts b/packages/dogfood/src/verify.ts
@@ -0,0 +1,145 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+//
+// The verify runner — executes derived proofs against a live dogfood stack.
+//
+// This is what a consumer (a framework example OR a third-party app like hotcrm)
+// would run: boot my app in-process, auto-derive a runtime contract from my
+// metadata, exercise it through the real HTTP surface, and tell me where the
+// declared behavior doesn't actually hold at runtime.
+
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+import type { DogfoodStack } from './harness.js';
+import { deriveCrudCases, type CrudCase } from './derive.js';
+
+export interface ObjectVerifyResult {
+  object: string;
+  status: 'verified' | 'fidelity-gaps' | 'create-failed' | 'read-failed' | 'skipped' | 'needs-fixture';
+  checked?: number;
+  reason?: string;
+  code?: number;
+  detail?: string;
+  mismatches?: Array<{ field: string; type: string; wrote: unknown; read: unknown }>;
+}
+
+export interface VerifyReport {
+  app: string;
+  results: ObjectVerifyResult[];
+  summary: {
+    objects: number;
+    verified: number;
+    fidelityGaps: number;
+    createFailed: number;
+    needsFixture: number;
+    readFailed: number;
+    skipped: number;
+    mismatchTotal: number;
+  };
+}
+
+function setEqual(a: unknown, b: unknown[]): boolean {
+  if (!Array.isArray(a)) return false;
+  return JSON.stringify([...a].sort()) === JSON.stringify([...b].sort());
+}
+function deepEqual(a: unknown, b: unknown): boolean {
+  return JSON.stringify(a) === JSON.stringify(b);
+}
+
+/**
+ * Run auto-derived CRUD round-trip proofs for every object in `config`, as the
+ * authenticated `token`, against the live `stack`. Returns a structured report;
+ * never throws on a per-object failure (collects them).
+ */
+export async function runCrudVerification(
+  stack: DogfoodStack,
+  token: string,
+  config: any,
+): Promise<VerifyReport> {
+  const cases = deriveCrudCases(config);
+  const results: ObjectVerifyResult[] = [];
+
+  for (const c of cases as CrudCase[]) {
+    if (c.blocked) {
+      results.push({ object: c.object, status: 'skipped', reason: c.blocked });
+      continue;
+    }
+    let created: Response;
+    try {
+      created = await stack.apiAs(token, 'POST', `/data/${c.object}`, c.body);
+    } catch (e: any) {
+      results.push({ object: c.object, status: 'create-failed', detail: String(e?.message ?? e).slice(0, 200) });
+      continue;
+    }
+    if (created.status >= 300) {
+      const text = (await created.text()).slice(0, 280);
+      // A 400 validation failure means our auto-derived record didn't satisfy the
+      // app's own validation rules (record-level format/json-schema/CEL) — that's
+      // a fixture gap, not a platform finding. A 5xx (or crypto/driver error) is a
+      // real runtime failure the app's author needs to see.
+      const isValidation = created.status === 400 && /VALIDATION_FAILED|Validation failed/i.test(text);
+      results.push({
+        object: c.object,
+        status: isValidation ? 'needs-fixture' : 'create-failed',
+        code: created.status,
+        detail: text,
+      });
+      continue;
+    }
+    const cj = (await created.json()) as any;
+    const id = cj?.id ?? cj?.record?.id;
+    if (!id) {
+      results.push({ object: c.object, status: 'create-failed', detail: 'no id returned' });
+      continue;
+    }
+
+    const got = await stack.apiAs(token, 'GET', `/data/${c.object}/${id}`);
+    if (got.status !== 200) {
+      results.push({ object: c.object, status: 'read-failed', code: got.status });
+      continue;
+    }
+    const rec = ((await got.json()) as any)?.record ?? {};
+
+    const mismatches: ObjectVerifyResult['mismatches'] = [];
+    for (const a of c.asserts ?? []) {
+      const actual = rec[a.field];
+      const ok = a.kind === 'set' ? setEqual(actual, a.value as unknown[]) : deepEqual(actual, a.value);
+      if (!ok) mismatches.push({ field: a.field, type: a.type, wrote: a.value, read: actual });
+    }
+    results.push({
+      object: c.object,
+      status: mismatches.length ? 'fidelity-gaps' : 'verified',
+      checked: (c.asserts ?? []).length,
+      ...(mismatches.length ? { mismatches } : {}),
+    });
+  }
+
+  const summary = {
+    objects: results.length,
+    verified: results.filter((r) => r.status === 'verified').length,
+    fidelityGaps: results.filter((r) => r.status === 'fidelity-gaps').length,
+    createFailed: results.filter((r) => r.status === 'create-failed').length,
+    needsFixture: results.filter((r) => r.status === 'needs-fixture').length,
+    readFailed: results.filter((r) => r.status === 'read-failed').length,
+    skipped: results.filter((r) => r.status === 'skipped').length,
+    mismatchTotal: results.reduce((n, r) => n + (r.mismatches?.length ?? 0), 0),
+  };
+  return { app: config?.manifest?.id ?? config?.manifest?.namespace ?? 'app', results, summary };
+}
+
+/** Pretty one-line-per-object summary for logs. */
+export function formatReport(report: VerifyReport): string {
+  const lines: string[] = [`\n=== objectstack verify — ${report.app} ===`];
+  for (const r of report.results) {
+    if (r.status === 'verified') lines.push(`  ✓ ${r.object}  (${r.checked} fields)`);
+    else if (r.status === 'fidelity-gaps') {
+      lines.push(`  ⚠ ${r.object}  ${r.mismatches!.length} fidelity gap(s):`);
+      for (const m of r.mismatches!) lines.push(`      ${m.field} <${m.type}>: wrote ${JSON.stringify(m.wrote)} → read ${JSON.stringify(m.read)}`);
+    }
+    else if (r.status === 'skipped') lines.push(`  – ${r.object}  skipped: ${r.reason}`);
+    else if (r.status === 'needs-fixture') lines.push(`  ~ ${r.object}  needs-fixture (app validation rejected the auto-record): ${(r.detail ?? '').slice(0,120)}`);
+    else lines.push(`  ✗ ${r.object}  ${r.status}${r.code ? ` (${r.code})` : ''}: ${r.detail ?? ''}`);
+  }
+  const s = report.summary;
+  lines.push(`  ── ${s.verified} verified, ${s.fidelityGaps} gaps, ${s.createFailed + s.readFailed} FAILED, ${s.needsFixture} needs-fixture, ${s.skipped} skipped (${s.mismatchTotal} mismatches)`);
+  return lines.join('\n');
+}
diff --git a/packages/dogfood/test/auto-verify.dogfood.test.ts b/packages/dogfood/test/auto-verify.dogfood.test.ts
@@ -0,0 +1,54 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+//
+// objectstack verify — metadata-driven runtime verification, proven against the
+// framework's own example apps. From each app's metadata ALONE it auto-derives a
+// CRUD round-trip contract (no hand-written cases) and runs it over real HTTP.
+// The same runner points at any third-party app's built artifact via
+// `test/verify-external.dogfood.test.ts`.
+
+import { describe, it, expect, beforeAll, afterAll } from 'vitest';
+import crmStack from '@objectstack/example-crm';
+import showcaseStack from '@objectstack/example-showcase';
+import { bootDogfoodStack, type DogfoodStack } from '../src/harness.js';
+import { runCrudVerification, formatReport, type VerifyReport } from '../src/verify.js';
+
+const APPS: Array<[string, unknown]> = [
+  ['crm', crmStack],
+  ['showcase', showcaseStack],
+];
+
+for (const [name, config] of APPS) {
+  describe(`objectstack verify: ${name} (auto-derived CRUD round-trip)`, () => {
+    let stack: DogfoodStack;
+    let report: VerifyReport;
+
+    beforeAll(async () => {
+      stack = await bootDogfoodStack(config as never);
+      const token = await stack.signIn();
+      report = await runCrudVerification(stack, token, config);
+      // eslint-disable-next-line no-console
+      console.error(formatReport(report));
+    }, 60_000);
+
+    afterAll(async () => {
+      await stack?.stop();
+    });
+
+    it('derives a runtime contract from metadata and boots', () => {
+      expect(report.summary.objects).toBeGreaterThan(0);
+    });
+
+    it('verifies objects end-to-end over real HTTP', () => {
+      expect(report.summary.verified).toBeGreaterThan(0);
+    });
+
+    it('has no object that fails to create or read (the hard runtime invariant)', () => {
+      // create/read failures = the app's metadata produces a record the runtime
+      // refuses or can't read back — a real platform/integration finding (vs
+      // `needs-fixture`, the auto-record tripping the app's own validation rules,
+      // and `fidelity-gaps`, type leaks tracked separately).
+      const hard = report.results.filter((r) => r.status === 'create-failed' || r.status === 'read-failed');
+      expect(hard, JSON.stringify(hard, null, 2)).toHaveLength(0);
+    });
+  });
+}
diff --git a/packages/dogfood/test/verify-external.dogfood.test.ts b/packages/dogfood/test/verify-external.dogfood.test.ts
@@ -0,0 +1,40 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+//
+// Point the verifier at ANY app's built artifact — the embryonic
+// `objectstack verify <app>`. This is the consumer-facing use: a third-party app
+// (e.g. hotcrm) runs it against its OWN metadata to learn where its declared
+// behavior doesn't hold at runtime.
+//
+// Gated on OS_VERIFY_ARTIFACT so it never runs in framework CI (the external app
+// isn't in the workspace). Run locally:
+//   OS_VERIFY_ARTIFACT=/abs/path/to/app/dist/objectstack.json \
+//     pnpm --filter @objectstack/dogfood exec vitest run test/verify-external.dogfood.test.ts
+
+import { describe, it, expect, beforeAll, afterAll } from 'vitest';
+import { readFileSync, existsSync } from 'node:fs';
+import { bootDogfoodStack, type DogfoodStack } from '../src/harness.js';
+import { runCrudVerification, formatReport, type VerifyReport } from '../src/verify.js';
+
+const ARTIFACT = process.env.OS_VERIFY_ARTIFACT;
+
+describe.skipIf(!ARTIFACT || !existsSync(ARTIFACT))('objectstack verify: external app artifact', () => {
+  let stack: DogfoodStack;
+  let report: VerifyReport;
+
+  beforeAll(async () => {
+    const config = JSON.parse(readFileSync(ARTIFACT as string, 'utf8'));
+    stack = await bootDogfoodStack(config);
+    const token = await stack.signIn();
+    report = await runCrudVerification(stack, token, config);
+    // eslint-disable-next-line no-console
+    console.error(formatReport(report));
+  }, 120_000);
+
+  afterAll(async () => {
+    await stack?.stop();
+  });
+
+  it('boots the external app and auto-derives a runtime contract', () => {
+    expect(report.summary.objects).toBeGreaterThan(0);
+  });
+});
