diff --git a/.changeset/spec-liveness-gate.md b/.changeset/spec-liveness-gate.md new file mode 100644 index 0000000000..939d0f3211 --- /dev/null +++ b/.changeset/spec-liveness-gate.md @@ -0,0 +1,4 @@ +--- +--- + +chore(spec): add a spec liveness gate — every authorable property in a governed category must declare a runtime-liveness status (live/experimental/planned/dead) with evidence in `packages/spec/liveness/.json`, enforced by CI on PRs touching `packages/spec/**`. Seeds the `security` category from the liveness audit (93 props: 66 dead, 26 live, 1 experimental); the dead set is the worklist for the security enforce-or-remove ADR. Repo-internal tooling; no package version impact. diff --git a/.github/workflows/spec-liveness-check.yml b/.github/workflows/spec-liveness-check.yml new file mode 100644 index 0000000000..6575900c46 --- /dev/null +++ b/.github/workflows/spec-liveness-check.yml @@ -0,0 +1,42 @@ +name: Spec Liveness Check + +# For a metadata-driven platform the spec is the product surface — a parsed-but- +# unenforced property is a silent no-op (and, for security props, false compliance). +# This gate requires every authorable property in a GOVERNED category to declare a +# liveness status with evidence in packages/spec/liveness/.json. A new +# unclassified property fails the check (the ratchet: no new undeclared surface). +# Governed categories are rolled out highest-risk-first; see packages/spec/liveness/README.md. + +on: + pull_request: + types: [opened, synchronize, reopened] + paths: + - 'packages/spec/**' + +permissions: + contents: read + +jobs: + liveness: + name: Spec property liveness + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v6 + + - name: Setup Node.js + uses: actions/setup-node@v6 + with: + node-version: '20' + + - name: Enable Corepack + run: corepack enable + + - name: Install dependencies + run: pnpm install --frozen-lockfile + + - name: Generate JSON schemas + run: pnpm --filter @objectstack/spec gen:schema + + - name: Check spec liveness + run: node packages/spec/scripts/liveness/check-liveness.mjs diff --git a/packages/spec/liveness/README.md b/packages/spec/liveness/README.md new file mode 100644 index 0000000000..9c45c63875 --- /dev/null +++ b/packages/spec/liveness/README.md @@ -0,0 +1,66 @@ +# Spec liveness ledger + +For a metadata-driven platform, **the spec is the product surface**: authors write +metadata against these Zod schemas. A property that is parsed but has no runtime +consumer is a silent no-op — and for a *security* property, a silent no-op is +**false compliance** (e.g. `forceMfa: true` accepted and ignored). The +metadata-liveness audits (`docs/audits/2026-06-*-property-liveness.md`) found that +large swaths of the declared surface are DEAD. + +This ledger makes that classification **explicit and regression-proof**: in a +*governed* category, every authorable property must declare a liveness status with +evidence, or CI fails (the ratchet — you can't add new undeclared surface). + +## Status vocabulary + +| Status | Meaning | +|---|---| +| `live` | Has a runtime consumer. Cite it in `evidence` (`file:line`, or a test). | +| `experimental` / `planned` | Declared, intentionally not enforced yet. Also recognised from a spec `.describe()` marker like `[EXPERIMENTAL — not enforced]`. | +| `dead` | Parsed, no consumer. Tracked for **enforce-or-remove** (cite the audit/grep). | +| `internal` *(schema-level)* | Not authorable metadata (runtime result/DTO, context, enum). Exempt. | + +Resolution order per property: **ledger entry → spec `.describe()` marker → UNCLASSIFIED**. +A schema-level `"_schema": ""` applies to all its properties (used for +wholesale-dead subtrees like `PasswordPolicy`, or `internal` runtime types). +Caveat: a `_schema`-classified schema also absorbs *new* properties at that status, +so the ratchet does not flag additions to a wholesale-dead/internal subtree — only +additions to per-property schemas (the mixed ones like `ObjectPermission`, +`PermissionSet`). Use `_schema` only for subtrees that are genuinely all-one-status. + +## Files + +- `.json` — the ledger for a governed category (currently: `security`). +- `../scripts/liveness/check-liveness.mjs` — the gate. Reads the generated + `packages/spec/json-schema//*.json`, resolves each authorable + property's status, and exits non-zero on any UNCLASSIFIED property. + +## Usage + +```bash +pnpm --filter @objectstack/spec gen:schema # produce json-schema/ (the source of truth) +pnpm --filter @objectstack/spec check:liveness # run the gate +node packages/spec/scripts/liveness/check-liveness.mjs --dump security # inventory a category (seeding aid) +``` + +CI: `.github/workflows/spec-liveness-check.yml` runs the gate on PRs touching +`packages/spec/**`. + +## Rolling out the next category + +Governed categories are listed in `GOVERNED` at the top of `check-liveness.mjs`, +rolled out **highest-risk-first**. To add one (e.g. `automation`, `data`): + +1. `--dump ` to inventory its authorable properties. +2. Seed `.json` from that category's liveness audit (file:line evidence) + and targeted greps for anything the audit didn't cover. **Classify only with + evidence** — `live` needs a cited consumer; `dead` needs a confirmed absence. +3. Add the category to `GOVERNED` and confirm the gate is green. + +## Current state — `security` + +93 authorable properties: **66 dead, 26 live, 1 experimental.** ~71% of the +authorable security surface is parsed-but-unenforced. The `dead` entries are the +worklist for the security enforce-or-remove ADR — most urgently the destructive +`ObjectPermission.allow{Transfer,Restore,Purge}` (ungated) and the entirely-dead +`Policy` tree (password/session/`forceMfa`/network/audit). diff --git a/packages/spec/liveness/security.json b/packages/spec/liveness/security.json new file mode 100644 index 0000000000..37edc90a88 --- /dev/null +++ b/packages/spec/liveness/security.json @@ -0,0 +1,77 @@ +{ + "category": "security", + "_note": "Liveness classification for authorable security metadata. Seeded from docs/audits/2026-06-security-identity-property-liveness.md (file:line evidence) plus targeted greps for schemas the audit did not cover. Statuses: live | experimental | planned | dead | (schema-level) internal. 'dead' = parsed, no runtime consumer → enforce-or-remove (security ADR). Security-critical: a 'dead' boundary prop = false compliance.", + "schemas": { + "ObjectPermission": { + "props": { + "allowCreate": { "status": "live", "evidence": "packages/plugins/plugin-security/src/permission-evaluator.ts:8" }, + "allowRead": { "status": "live", "evidence": "packages/plugins/plugin-security/src/permission-evaluator.ts:8" }, + "allowEdit": { "status": "live", "evidence": "packages/plugins/plugin-security/src/permission-evaluator.ts:8" }, + "allowDelete": { "status": "live", "evidence": "packages/plugins/plugin-security/src/permission-evaluator.ts:8" }, + "viewAllRecords": { "status": "live", "evidence": "packages/plugins/plugin-security/src/permission-evaluator.ts" }, + "modifyAllRecords": { "status": "live", "evidence": "packages/plugins/plugin-security/src/permission-evaluator.ts" }, + "allowTransfer": { "status": "dead", "evidence": "permission-evaluator.ts:8-16 (absent from OPERATION_TO_PERMISSION)", "note": "destructive op (ownership transfer) NOT gated by RBAC — enforce-or-remove. Spec marks experimental." }, + "allowRestore": { "status": "dead", "evidence": "permission-evaluator.ts:8-16 (absent)", "note": "undelete NOT gated — enforce-or-remove." }, + "allowPurge": { "status": "dead", "evidence": "permission-evaluator.ts:8-16 (absent)", "note": "hard-delete/GDPR purge NOT gated — enforce-or-remove." } + } + }, + "FieldPermission": { + "props": { + "readable": { "status": "live", "evidence": "packages/plugins/plugin-security/src/permission-evaluator.ts", "note": "FLS read-mask enforced." }, + "editable": { "status": "live", "evidence": "packages/plugins/plugin-security/src/permission-evaluator.ts", "note": "FLS write-deny enforced." } + } + }, + "PermissionSet": { + "props": { + "name": { "status": "live", "evidence": "permission-evaluator.ts", "note": "registration/assignment key." }, + "label": { "status": "live", "note": "display metadata (admin forms), not a security boundary." }, + "objects": { "status": "live", "evidence": "packages/plugins/plugin-security/src/permission-evaluator.ts:8" }, + "fields": { "status": "live", "evidence": "packages/plugins/plugin-security/src/permission-evaluator.ts" }, + "rowLevelSecurity": { "status": "live", "evidence": "packages/plugins/plugin-security/src/rls-compiler.ts", "note": "enforced on find + analytics raw-SQL." }, + "systemPermissions": { "status": "live", "evidence": "packages/plugins/plugin-hono-server/src/hono-plugin.ts", "note": "PARTIAL — enforced only for app-entry/nav visibility, NOT as a general capability gate (e.g. manage_users unchecked in data path)." }, + "tabPermissions": { "status": "live", "note": "PARTIAL — only 'hidden' is read; default_on/default_off ignored; UI-only, not a boundary." }, + "isProfile": { "status": "dead", "evidence": "audit: profile-vs-permset never gates anything", "note": "enforce-or-remove." }, + "contextVariables": { "status": "dead", "evidence": "rls-compiler.ts never reads it", "note": "RLS uses only current_user.* built-ins; doc claim of runtime evaluation is false." } + } + }, + "RowLevelSecurityPolicy": { + "props": { + "name": { "status": "live", "evidence": "packages/plugins/plugin-security/src/rls-compiler.ts" }, + "object": { "status": "live", "evidence": "packages/plugins/plugin-security/src/rls-compiler.ts" }, + "operation": { "status": "live", "evidence": "packages/plugins/plugin-security/src/rls-compiler.ts" }, + "using": { "status": "live", "evidence": "packages/plugins/plugin-security/src/rls-compiler.ts", "note": "compiled into find + analytics SQL." }, + "check": { "status": "live", "evidence": "packages/plugins/plugin-security/src/rls-compiler.ts" }, + "roles": { "status": "live", "evidence": "packages/plugins/plugin-security/src/rls-compiler.ts", "note": "flat role match — no subordinate rollup." }, + "enabled": { "status": "live", "evidence": "packages/plugins/plugin-security/src/rls-compiler.ts" }, + "priority": { "status": "live", "evidence": "packages/plugins/plugin-security/src/rls-compiler.ts", "note": "policy ordering." }, + "label": { "status": "live", "note": "display metadata." }, + "description": { "status": "live", "note": "display metadata." }, + "tags": { "status": "live", "note": "organizational metadata (not a boundary)." } + } + }, + "Policy": { + "_note": "PolicySchema is 100% DEAD — not even registered as a metadata type; better-auth runs its own hardcoded config (packages/plugins/plugin-auth/src/index.ts). Authoring a compliance Policy = false compliance.", + "props": { + "name": { "status": "experimental", "evidence": "spec describe(): [EXPERIMENTAL — not enforced]" }, + "password": { "status": "dead", "evidence": "auth: better-auth hardcoded scrypt/session", "note": "false compliance — enforce or remove." }, + "network": { "status": "dead", "evidence": "no consumer", "note": "IP allow-list/VPN unenforced." }, + "session": { "status": "dead", "evidence": "auth: better-auth hardcoded session", "note": "idle/absolute timeout + forceMfa unenforced." }, + "audit": { "status": "dead", "evidence": "no consumer", "note": "retention/redaction unenforced." }, + "isDefault": { "status": "dead", "evidence": "no consumer" }, + "assignedProfiles": { "status": "dead", "evidence": "no consumer" } + } + }, + "PasswordPolicy": { "_schema": "dead", "_note": "Policy.password subtree — better-auth hardcoded; unenforced. enforce-or-remove.", "props": {} }, + "SessionPolicy": { "_schema": "dead", "_note": "Policy.session subtree — idle/absoluteTimeout/forceMfa all unenforced; better-auth hardcoded. forceMfa=true is a silent no-op (false compliance).", "props": {} }, + "NetworkPolicy": { "_schema": "dead", "_note": "Policy.network subtree — trustedRanges/blockUnknown/vpnRequired unenforced.", "props": {} }, + "AuditPolicy": { "_schema": "dead", "_note": "Policy.audit subtree — logRetentionDays/sensitiveFields/captureRead unenforced.", "props": {} }, + "OwnerSharingRule": { "_schema": "dead", "_note": "Spec SharingRule has NO runtime consumer; runtime enforces a divergent sys_sharing_rule model (packages/plugins/plugin-sharing/src/sharing-plugin.ts). Authoring the spec rule has no effect — reconcile to one contract.", "props": {} }, + "RLSConfig": { "_schema": "dead", "_note": "No runtime consumer (grep). RLS enforcement reads RowLevelSecurityPolicy via rls-compiler.ts, not this config object.", "props": {} }, + "RLSAuditConfig": { "_schema": "dead", "_note": "No runtime consumer (grep).", "props": {} }, + "Territory": { "_schema": "dead", "_note": "Salesforce-style territory management — no runtime consumer (grep).", "props": {} }, + "TerritoryModel": { "_schema": "dead", "_note": "No runtime consumer (grep).", "props": {} }, + "RLSEvaluationResult": { "_schema": "internal", "_note": "RLS runtime evaluation result — not authorable metadata." }, + "RLSAuditEvent": { "_schema": "internal", "_note": "RLS runtime audit event — not authorable metadata." }, + "RLSUserContext": { "_schema": "internal", "_note": "RLS runtime user context — not authorable metadata." } + } +} diff --git a/packages/spec/package.json b/packages/spec/package.json index da6d7a626a..385f1b6f89 100644 --- a/packages/spec/package.json +++ b/packages/spec/package.json @@ -191,7 +191,8 @@ "gen:sbom": "tsx scripts/generate-sbom.ts", "test": "vitest run", "test:watch": "vitest", - "test:coverage": "vitest run --coverage" + "test:coverage": "vitest run --coverage", + "check:liveness": "node scripts/liveness/check-liveness.mjs" }, "keywords": [ "objectstack", diff --git a/packages/spec/scripts/liveness/check-liveness.mjs b/packages/spec/scripts/liveness/check-liveness.mjs new file mode 100644 index 0000000000..a59120388a --- /dev/null +++ b/packages/spec/scripts/liveness/check-liveness.mjs @@ -0,0 +1,150 @@ +#!/usr/bin/env node +// Spec liveness gate. +// +// For a metadata-driven platform the spec IS the product surface: authors write +// metadata against these schemas. A property that is parsed but has no runtime +// consumer is a silent no-op (worst case: a *security* no-op — "false compliance"). +// The metadata-liveness audits (docs/audits/2026-06-*-property-liveness.md) found +// large swaths of such DEAD props. This gate makes the classification explicit and +// keeps it from regressing: in a GOVERNED category, every authorable property must +// declare a liveness status with evidence, or CI fails. +// +// Statuses (per property, or per schema via "_schema"): +// live — has a runtime consumer (cite it in `evidence`: file:line / test) +// experimental| planned — declared, intentionally not enforced yet (mark in spec +// `.describe()` as "[EXPERIMENTAL — not enforced]" or via the ledger) +// dead — parsed, no consumer; tracked for enforce-or-remove (cite the audit) +// internal — (schema-level only) not authorable metadata (runtime result/DTO, +// enum, helper); exempt from property classification +// +// Resolution order per property: ledger entry > spec `.describe()` marker > UNCLASSIFIED +// A GOVERNED category with any UNCLASSIFIED property → non-zero exit (the ratchet: +// you cannot add a new spec property without classifying it). +// +// Usage: +// node check-liveness.mjs # check all governed categories +// node check-liveness.mjs --dump security # inventory a category's props (seeding aid) +// node check-liveness.mjs --json # machine-readable report + +import { readFileSync, readdirSync, existsSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join, resolve } from 'node:path'; + +const here = dirname(fileURLToPath(import.meta.url)); +const specRoot = resolve(here, '../..'); // packages/spec +const repoRoot = resolve(specRoot, '../..'); // repo root +const schemaRoot = join(specRoot, 'json-schema'); +const ledgerRoot = join(specRoot, 'liveness'); + +// Categories whose authorable schemas must be fully classified. Extend +// highest-risk-first as each category's ledger is seeded from its audit. +const GOVERNED = ['security']; + +const args = process.argv.slice(2); +const asJson = args.includes('--json'); +const dumpIdx = args.indexOf('--dump'); +const dumpCategory = dumpIdx !== -1 ? args[dumpIdx + 1] : null; + +const MARKER_RE = { + experimental: /\[experimental|not enforced|aspirational/i, + planned: /\[planned|not yet implemented|coming soon/i, +}; + +function loadSchemas(category) { + const dir = join(schemaRoot, category); + if (!existsSync(dir)) return []; + return readdirSync(dir) + .filter((f) => f.endsWith('.json')) + .map((f) => { + const schema = JSON.parse(readFileSync(join(dir, f), 'utf8')); + return { name: f.replace('.json', ''), schema }; + }); +} + +// top-level authorable properties of an object schema (one level; a nested object +// property represents its whole subtree, matching how the audits reason) +function topProps(schema) { + if (!schema || schema.type !== 'object' || !schema.properties) return []; + return Object.entries(schema.properties).map(([key, def]) => ({ + key, + description: (def && def.description) || '', + })); +} + +function markerStatus(description) { + if (MARKER_RE.planned.test(description)) return 'planned'; + if (MARKER_RE.experimental.test(description)) return 'experimental'; + return null; +} + +function loadLedger(category) { + const f = join(ledgerRoot, `${category}.json`); + if (!existsSync(f)) return null; + return JSON.parse(readFileSync(f, 'utf8')); +} + +if (dumpCategory) { + const rows = []; + for (const { name, schema } of loadSchemas(dumpCategory)) { + const kind = schema.type === 'object' && schema.properties ? 'object' : (schema.enum ? 'enum' : schema.type || 'other'); + if (kind !== 'object') { rows.push({ schema: name, kind, prop: '', marker: '' }); continue; } + for (const p of topProps(schema)) rows.push({ schema: name, kind, prop: p.key, marker: markerStatus(p.description) || '' }); + } + process.stdout.write(JSON.stringify(rows, null, 2) + '\n'); + process.exit(0); +} + +const report = { categories: {}, totals: { classified: 0, unclassified: 0, byStatus: {} }, unclassified: [], staleEvidence: [] }; + +for (const category of GOVERNED) { + const ledger = loadLedger(category) || { schemas: {} }; + const cat = { classified: 0, unclassified: 0, byStatus: {} }; + for (const { name, schema } of loadSchemas(category)) { + const props = topProps(schema); + if (props.length === 0) continue; // enums / scalars carry no authorable props + const entry = ledger.schemas[name] || {}; + if (entry._schema === 'internal') continue; // exempt: not authorable + for (const { key, description } of props) { + const led = entry.props && entry.props[key]; + const status = (led && led.status) || entry._schema || markerStatus(description); + if (!status) { + cat.unclassified++; + report.unclassified.push(`${category}/${name}.${key}`); + continue; + } + cat.classified++; + cat.byStatus[status] = (cat.byStatus[status] || 0) + 1; + report.totals.byStatus[status] = (report.totals.byStatus[status] || 0) + 1; + // soft evidence sanity for `live` + if (status === 'live' && led && led.evidence) { + const file = String(led.evidence).split(':')[0]; + if (/\//.test(file) && !existsSync(join(repoRoot, file))) report.staleEvidence.push(`${category}/${name}.${key} → ${led.evidence}`); + } + } + } + report.categories[category] = cat; + report.totals.classified += cat.classified; + report.totals.unclassified += cat.unclassified; +} + +if (asJson) { + process.stdout.write(JSON.stringify(report, null, 2) + '\n'); +} else { + console.log('Spec liveness gate — governed categories:', GOVERNED.join(', ')); + for (const [c, v] of Object.entries(report.categories)) { + const parts = Object.entries(v.byStatus).map(([s, n]) => `${s} ${n}`).join(', '); + console.log(` ${c}: ${v.classified} classified (${parts || '—'}), ${v.unclassified} unclassified`); + } + if (report.staleEvidence.length) { + console.log(`\n⚠ ${report.staleEvidence.length} 'live' entr(ies) cite a missing file (evidence may be stale):`); + report.staleEvidence.forEach((s) => console.log(` ${s}`)); + } + if (report.unclassified.length) { + console.log(`\n✗ ${report.unclassified.length} UNCLASSIFIED propert(ies) in governed categories — classify in packages/spec/liveness/.json:`); + report.unclassified.forEach((s) => console.log(` ${s}`)); + } else { + console.log('\n✓ all governed-category properties are classified.'); + } +} + +process.exit(report.unclassified.length > 0 ? 1 : 0);