feat(verify): extract dogfood engine into public @objectstack/verify + CLI

Productize the internal dogfood regression engine as a published library and an
`objectstack verify` CLI so third-party and template authors can run the same
runtime proofs (auto-derived CRUD round-trip fidelity + the cross-owner RLS
invariant) against their own apps.

- new public @objectstack/verify: bootStack / deriveCrudCases /
  runCrudVerification / runRlsProofs, moved+generalized from packages/dogfood/src
  (tsup ESM+CJS+DTS, publishConfig public)
- new `objectstack verify` oclif command (--app/--rls/--multi-tenant/--json);
  exits 1 on create-failed/read-failed/fidelity-gaps/rls-hole
- dogfood now consumes @objectstack/verify; drop 3 redundant auto-verify tests
  (replaced by an `objectstack verify` CI step over the example apps); keep the
  hand-written golden regressions (#2018 tz, #1994 RLS, #2004 field fidelity)
- fix: the harness used a port-less loopback inject origin (http://localhost)
  that fails better-auth's default dev trusted-origins (http://localhost:*);
  use a ported base so dev-admin sign-in passes in a bare node CLI and the test
  runner alike
- add @objectstack/verify to the changeset `fixed` group; changeset for verify+cli

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: os-zhuang

.changeset/config.json

@@ -80,7 +80,8 @@
       "@objectstack/account",
       "create-objectstack",
       "objectstack-vscode",
-      "@objectstack/connector-openapi"
+      "@objectstack/connector-openapi",
+      "@objectstack/verify"
     ]
   ],
   "linked": [],

.changeset/verify-package.md

@@ -0,0 +1,8 @@
+---
+"@objectstack/verify": minor
+"@objectstack/cli": minor
+---
+
+Add `@objectstack/verify` — boot any ObjectStack app in-process and verify it through the real HTTP stack: auto-derived CRUD round-trip fidelity (`runCrudVerification`) plus the cross-owner RLS invariant (`runRlsProofs`, "you can't write what you can't read"). Also adds an `objectstack verify` CLI command that runs these proofs against an app config and exits non-zero on real failures.
+
+Extracted from the internal dogfood regression gate so third-party and template authors can run the same runtime proofs against their own apps. The private `@objectstack/dogfood` package now consumes this library for its golden regression tests.

.github/workflows/ci.yml

@@ -161,6 +161,19 @@ jobs:
       - name: Boot example apps and exercise real user flows
         run: pnpm turbo run test --filter=@objectstack/dogfood
 
+      # Replaces the former auto-verify dogfood tests: runs the published
+      # `objectstack verify` engine over each example app through the CLI —
+      # auto-derived CRUD round-trip fidelity + the cross-owner RLS invariant.
+      # Exits non-zero on a real runtime failure, so it gates like the tests did.
+      - name: Verify example apps via the `objectstack verify` CLI
+        run: |
+          pnpm turbo run build --filter=@objectstack/cli
+          for app in examples/app-crm examples/app-showcase; do
+            echo "::group::objectstack verify $app --rls"
+            OS_LOG_LEVEL=error node packages/cli/bin/run.js verify --app "$app/objectstack.config.ts" --rls
+            echo "::endgroup::"
+          done
+
   build-core:
     name: Build Core
     needs: filter

packages/cli/package.json

@@ -88,6 +88,7 @@
     "@objectstack/service-storage": "workspace:*",
     "@objectstack/spec": "workspace:*",
     "@objectstack/types": "workspace:*",
+    "@objectstack/verify": "workspace:*",
     "@oclif/core": "^4.11.4",
     "bundle-require": "^5.1.0",
     "chalk": "^5.6.2",

packages/cli/src/commands/verify.ts

@@ -0,0 +1,102 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+
+import { Command, Flags } from '@oclif/core';
+import chalk from 'chalk';
+import { readEnvWithDeprecation } from '@objectstack/types';
+import {
+  bootStack,
+  runCrudVerification,
+  formatReport,
+  runRlsProofs,
+  formatRlsReport,
+  type VerifyReport,
+  type RlsReport,
+} from '@objectstack/verify';
+import { loadConfig } from '../utils/config.js';
+
+/**
+ * `objectstack verify` — boot the app in-process and exercise it through the
+ * real HTTP stack, asserting runtime behavior the static gates can't see:
+ *   - data fidelity: author → write → read → assert, per object/field type
+ *   - authorization (--rls): "you can't write what you can't read" (#1994 class)
+ *
+ * Exits non-zero on real failures so it drops straight into CI.
+ */
+export default class Verify extends Command {
+  static override description =
+    'Boot the app in-process and verify it through the real HTTP stack (CRUD round-trip fidelity + the cross-owner RLS invariant)';
+
+  static override examples = [
+    '<%= config.bin %> verify',
+    '<%= config.bin %> verify --app ./objectstack.config.ts --rls',
+    '<%= config.bin %> verify --rls --multi-tenant --json',
+  ];
+
+  static override flags = {
+    app: Flags.string({
+      char: 'a',
+      description: 'Path to the app config (defaults to ./objectstack.config.{ts,js,mjs})',
+    }),
+    rls: Flags.boolean({
+      description: 'Also run the cross-owner RLS invariant (a fresh member must not write what it cannot read)',
+      default: false,
+    }),
+    'multi-tenant': Flags.boolean({
+      description: 'Boot org-scoped (register plugin-org-scoping) so tenant-isolation RLS policies apply (also honors $OS_MULTI_ORG_ENABLED)',
+      default: false,
+    }),
+    json: Flags.boolean({ description: 'Emit the structured report as JSON', default: false }),
+  };
+
+  async run(): Promise<void> {
+    const { flags } = await this.parse(Verify);
+
+    const { config, absolutePath } = await loadConfig(flags.app);
+
+    const multiTenant =
+      flags['multi-tenant'] ||
+      String(readEnvWithDeprecation('OS_MULTI_ORG_ENABLED', 'OS_MULTI_TENANT') ?? 'false').toLowerCase() !==
+        'false';
+
+    const stack = await bootStack(config, { multiTenant });
+
+    let crud: VerifyReport;
+    let rls: RlsReport | undefined;
+    try {
+      const adminToken = await stack.signIn();
+      crud = await runCrudVerification(stack, adminToken, config);
+
+      if (flags.rls) {
+        const memberToken = await stack.signUp('verify-member@objectstack.test');
+        rls = await runRlsProofs(stack, adminToken, memberToken, config);
+      }
+    } finally {
+      await stack.stop();
+    }
+
+    // Failure contract: a "real" runtime break the app's author must see.
+    const hardFailures =
+      crud.summary.createFailed +
+      crud.summary.readFailed +
+      crud.summary.fidelityGaps +
+      (rls?.summary.holes ?? 0);
+
+    if (flags.json) {
+      this.log(JSON.stringify({ app: crud.app, config: absolutePath, multiTenant, crud, rls, hardFailures }, null, 2));
+    } else {
+      this.log(formatReport(crud));
+      if (rls) this.log(formatRlsReport(rls));
+      this.log('');
+      this.log(
+        hardFailures > 0
+          ? chalk.red(`✗ verify FAILED — ${hardFailures} runtime failure(s)`)
+          : chalk.green('✓ verify passed — no runtime failures'),
+      );
+    }
+
+    // Force process exit: the in-process stack leaves handles open (http server,
+    // sqlite-wasm, better-auth timers) that keep the event loop alive after
+    // stop(), so a bare return would hang. exit() also encodes the CI contract.
+    this.exit(hardFailures > 0 ? 1 : 0);
+  }
+}

packages/dogfood/README.md

@@ -23,15 +23,21 @@ sockets, CI-stable). Tests act as a browser client would: sign in, hit
 
 ## Layout
 
-- `src/harness.ts` — `bootDogfoodStack(config)` → `{ kernel, api, raw, signIn, apiAs, stop }`.
+The in-process harness + auto-derived verifiers (`bootStack`, `runCrudVerification`,
+`runRlsProofs`) now live in the published **[`@objectstack/verify`](../verify)**
+package — point it at any app. This package is the framework's own consumer: it
+holds the **hand-written golden tests** that pin specific historical regressions
+the generic verifier cannot auto-derive.
+
+- depends on `@objectstack/verify` for `bootStack(config)` → `{ kernel, api, raw, signIn, signUp, apiAs, stop }`.
 - `test/*.dogfood.test.ts` — golden flows. Each should assert on **observable
   output** (a number, a bucket label, a row count), not just "no error".
 
 ## Adding a golden test
 
 1. Pick a real user flow that a static test can't cover (it spans engine +
    service + HTTP, or depends on seeded/written data).
-2. `bootDogfoodStack(<appConfig>)`, `signIn()`, drive it via `api()/apiAs()`.
+2. `bootStack(<appConfig>)`, `signIn()`, drive it via `api()/apiAs()`.
 3. Assert on the concrete result.
 4. **Prove it catches the bug**: temporarily revert the relevant fix and confirm
    the test goes red. A green-on-the-bug test is not a gate.
@@ -67,7 +73,7 @@ The binding policy — every authorable+live primitive must carry a runtime proo
 
 The capability matrix above proves *data* round-trips. The authorization
 dimension proves a record the caller must not touch stays untouched. The
-app-agnostic invariant (`src/rls.ts`, `runRlsProofs`):
+app-agnostic invariant (`runRlsProofs`, from `@objectstack/verify`):
 
 > **A user who cannot READ a record must not be able to WRITE it.**
 
@@ -87,9 +93,10 @@ changed. Verdicts: `rls-consistent` (can't read **and** can't write — good),
 `rls-hole` (can't read **yet** wrote — the #1994 bug), `member-visible`
 (member *can* read it — inconclusive, not a cross-owner scenario).
 
-`auto-verify-rls.dogfood.test.ts` runs this over the example apps, but they boot
-**single-tenant**, where every object comes back `member-visible` — so the
-by-id-write path is never actually exercised. Two ways to create real isolation:
+`objectstack verify <app> --rls` runs this over any app in CI, but the example
+apps boot **single-tenant**, where every object comes back `member-visible` — so
+the by-id-write path is never actually exercised. Two ways to create real
+isolation, both pinned as golden tests here:
 
 ### 1. Owner-scoped fixture — `test/rls-fixture.dogfood.test.ts` (hard gate)
 
@@ -98,11 +105,11 @@ permission set carries `RLS.ownerPolicy('rls_note', 'created_by')`. The predicat
 is `created_by = current_user.id` — keyed on the column the engine stamps on
 every record and referencing `current_user.id`, **not**
 `current_user.organization_id`, so it survives single-tenant policy stripping. A
-fresh member genuinely can't read the admin's note. `bootDogfoodStack` takes a
+fresh member genuinely can't read the admin's note. `bootStack` takes a
 `security:` override so the fixture's permission set is the member's fallback:
 
 ```ts
-bootDogfoodStack(rlsFixtureStack, { security: rlsFixtureSecurity(ownerScopedMemberSet) })
+bootStack(rlsFixtureStack, { security: rlsFixtureSecurity(ownerScopedMemberSet) })
 ```
 
 - **Green gate** (owner policy on `all` ops) → `rls-consistent`. Safe *only*
@@ -139,7 +146,7 @@ Faithful fix — boot multi-tenant so `@objectstack/plugin-org-scoping` register
 before `SecurityPlugin` and the `organization_id` policies apply:
 
 ```ts
-bootDogfoodStack(crmStack, { multiTenant: true })
+bootStack(crmStack, { multiTenant: true })
 ```
 
 The dev admin is bound to the seeded default org; a fresh `signUp` member is not,

packages/dogfood/package.json

@@ -3,27 +3,18 @@
   "version": "0.0.1",
   "private": true,
   "license": "Apache-2.0",
-  "description": "Dogfood regression gate — boots real example apps in-process and exercises them through the real HTTP/service stack, catching runtime regressions that static checks (build, unit tests, spec liveness) miss.",
+  "description": "Dogfood regression gate — hand-written golden tests that boot real example apps through @objectstack/verify's in-process HTTP stack, pinning historical runtime regressions (#2018 timezone bucketing, #1994 cross-owner RLS, #2004 field fidelity) that static checks miss.",
   "type": "module",
   "scripts": {
     "test": "vitest run"
   },
   "dependencies": {
-    "@objectstack/core": "workspace:*",
-    "@objectstack/runtime": "workspace:*",
+    "@objectstack/verify": "workspace:*",
     "@objectstack/objectql": "workspace:*",
     "@objectstack/spec": "workspace:*",
-    "@objectstack/driver-sqlite-wasm": "workspace:*",
-    "@objectstack/plugin-hono-server": "workspace:*",
-    "@objectstack/rest": "workspace:*",
-    "@objectstack/plugin-auth": "workspace:*",
     "@objectstack/plugin-security": "workspace:*",
-    "@objectstack/service-settings": "workspace:*",
-    "@objectstack/service-analytics": "workspace:*",
     "@objectstack/example-crm": "workspace:*",
-    "@objectstack/example-showcase": "workspace:*",
-    "@objectstack/plugin-sharing": "workspace:*",
-    "@objectstack/plugin-org-scoping": "workspace:*"
+    "@objectstack/example-showcase": "workspace:*"
   },
   "devDependencies": {
     "@types/node": "^25.9.3",

packages/dogfood/test/analytics-timezone.dogfood.test.ts

@@ -10,7 +10,7 @@
 
 import { describe, it, expect, beforeAll, afterAll } from 'vitest';
 import crmStack from '@objectstack/example-crm';
-import { bootDogfoodStack, type DogfoodStack } from '../src/harness.js';
+import { bootStack, type VerifyStack } from '@objectstack/verify';
 
 // 03:00 UTC on 2024-03-01 is still 2024-02-29 (19:00) in America/Los_Angeles
 // (PST = UTC-8, before DST). So a *day* bucket labels this instant 2024-03-01
@@ -29,11 +29,11 @@ const leadByDay = {
 };
 
 describe('dogfood: org timezone drives analytics date bucketing (#1982/#2018)', () => {
-  let stack: DogfoodStack;
+  let stack: VerifyStack;
   let token: string;
 
   beforeAll(async () => {
-    stack = await bootDogfoodStack(crmStack);
+    stack = await bootStack(crmStack);
 
     // Deterministic fixture: N leads pinned to the tz-boundary instant, inserted
     // as system so the write path's defaults/validation don't fight the setup.