diff --git a/.changeset/liveness-evidence-path-resolution.md b/.changeset/liveness-evidence-path-resolution.md new file mode 100644 index 0000000000..8760978270 --- /dev/null +++ b/.changeset/liveness-evidence-path-resolution.md @@ -0,0 +1,49 @@ +--- +"@objectstack/spec": patch +--- + +fix(spec): the liveness gate's stale-evidence check was ~100% false positives — and it was burying a real one + +The check was one line: + +```ts +const file = String(led.evidence).split(':')[0]; +if (/\//.test(file) && !existsSync(join(repoRoot, file))) → flag +``` + +i.e. it assumed every `evidence` string is exactly `path/to/file.ts:123`. Almost +none are — they carry prose (`packages/spec/src/stack.zod.ts (mergeActionsIntoObjects +stable-sorts each group)`), multiple pointers, or a cross-repo attribution +(`objectui: packages/app-shell/…`). Taking everything before the first colon +turns that prose into the "filename", which never exists. + +Result: **48 of 227 entries flagged, every one a parse artefact or a deliberate +cross-repo pointer.** A permanently non-empty, ~100%-false warning is a warning +nobody reads — which is exactly how the one genuine rot in that list went +unnoticed: + +- **`object.enable.clone`** cited `packages/objectql/src/protocol.ts:2259`. That + file no longer exists; `cloneData()`'s `enable.clone` gate moved to + `packages/metadata-protocol/src/protocol.ts:2938`. The claim stayed true, the + pointer rotted, and the check that exists to catch precisely this could not be + heard over the noise. Pointer repaired and dated. + +**New `evidence.mts`** extracts repo-rooted paths properly and honours the +cross-repo attribution entries already write in prose: + +- a realm marker (`objectui`, `cloud`, `ee`) attributes the paths after it, up to + the next clause boundary, so one string can cite both repos; `framework` + switches back explicitly; +- `packages/services/service-ai/…` is always foreign — the closed cloud runtime, + the one sibling missing from this repo's `packages/services/`; +- non-repo-rooted tokens (`app-shell/MetadataProvider.tsx`, + `action-button/-group`) read as prose, neither resolved nor reported. + +The gate now resolves **156 evidence paths** against the checkout, attributes 36 +to another repo, and reports **zero** stale — down from 48 warnings that said +nothing. Each run prints the two counts, so the check degrading to "extracts +nothing" is visible rather than silently green (a unit test asserts it too). + +Also updates the ledger README, whose advice to write objectui paths "as prose to +avoid false stale-flags" was a workaround for this bug: write the full path with +a realm prefix instead. diff --git a/packages/spec/liveness/README.md b/packages/spec/liveness/README.md index 7969140ecd..139dc86777 100644 --- a/packages/spec/liveness/README.md +++ b/packages/spec/liveness/README.md @@ -38,7 +38,7 @@ type back onto the registry and drop the override. | Status | Meaning | |---|---| -| `live` | Has a runtime consumer. Cite it in `evidence` (`file:line`; objectui-repo paths as prose to avoid false stale-flags). | +| `live` | Has a runtime consumer. Cite it in `evidence` as `file:line`; for another repo's path, prefix the realm — `objectui: packages/app-shell/…` (see below). | | `experimental` / `planned` | Declared, intentionally not enforced yet. Also read from a spec `.describe()` marker like `[EXPERIMENTAL — not enforced]`. | | `dead` | Parsed, no consumer. Tracked for **enforce-or-remove** (ADR-0049). | @@ -46,6 +46,38 @@ Resolution per property: **ledger entry → spec `.describe()` marker → UNCLAS Framework provenance/lock fields (`_lock*`, `_provenance`, `_packageId/Version`, `protection` — ADR-0010) are auto-classified `live`. +### Writing `evidence` so the gate can check it + +The gate extracts every **repo-rooted** path from an `evidence` string (one +starting with `packages/`, `apps/`, `content/`, …) and resolves it against this +checkout. Prose around the paths is fine and encouraged — `packages/spec/src/stack.zod.ts +(mergeActionsIntoObjects stable-sorts each group)` resolves the path and ignores +the parenthetical. + +**A path in another repo must say so**, or the gate will report it as rot: + +```jsonc +"evidence": "objectui: packages/app-shell/src/views/RecordDetailView.tsx:573" +"evidence": "registered into ActionEngine (objectui packages/core/…/ActionEngine.ts:150) but no caller" +``` + +A realm marker (`objectui`, `cloud`, `ee`) attributes the paths that follow it, +up to the next clause boundary (`;` or `)`), so a string can cite both repos — +`framework` switches back explicitly. `packages/services/service-ai/…` is always +treated as foreign: that is the closed cloud runtime, and `packages/services/` +here ships every sibling service except it. A path that is not repo-rooted +(`app-shell/MetadataProvider.tsx`, `action-button/-group`) reads as prose and is +neither resolved nor reported. + +> This replaces the old advice to write objectui paths "as prose to avoid false +> stale-flags." That was a workaround for a parser bug — the check took +> `evidence.split(':')[0]` as the filename, so any prose made it fail. It flagged +> **48 of 227 entries and every one was a false positive**, which buried the one +> real rotted pointer in the list (`object.enable.clone`, whose consumer had +> moved from `@objectstack/objectql` to `@objectstack/metadata-protocol`). A +> permanently-noisy check is a check nobody reads — the same way a stale ledger +> entry is a claim nobody re-tests. + ### `verifiedAt` — the re-verification clock An entry may carry `"verifiedAt": "YYYY-MM-DD"`: the date a human last closed the diff --git a/packages/spec/liveness/object.json b/packages/spec/liveness/object.json index ed86cee003..0edc63b130 100644 --- a/packages/spec/liveness/object.json +++ b/packages/spec/liveness/object.json @@ -179,8 +179,9 @@ }, "clone": { "status": "live", - "evidence": "packages/objectql/src/protocol.ts:2259", - "note": "cloneData() gates on schema.enable.clone (explicit false ⇒ 403 CLONE_DISABLED); exposed at POST /data/:object/:id/clone (rest-server.ts registerDataActionEndpoints)." + "verifiedAt": "2026-07-28", + "evidence": "packages/metadata-protocol/src/protocol.ts:2938", + "note": "cloneData() gates on schema.enable.clone (explicit false ⇒ 403 CLONE_DISABLED); exposed at POST /data/:object/:id/clone (rest-server.ts registerDataActionEndpoints). POINTER REPAIRED 2026-07 — cited packages/objectql/src/protocol.ts:2259 until the protocol moved to @objectstack/metadata-protocol. The claim never stopped being true, but the stale-evidence check could not surface the rot: 47 parse-artefact false positives were burying it (fixed in the same pass). Behaviour guarded by packages/objectql/src/protocol-clone-real-engine.test.ts." } } }, diff --git a/packages/spec/scripts/liveness/check-liveness.mts b/packages/spec/scripts/liveness/check-liveness.mts index 75563766ae..12c8b633ce 100644 --- a/packages/spec/scripts/liveness/check-liveness.mts +++ b/packages/spec/scripts/liveness/check-liveness.mts @@ -63,6 +63,7 @@ import { type VerificationEntry, type VerificationReport, } from './verification.mts'; +import { checkEvidence } from './evidence.mts'; const here = dirname(fileURLToPath(import.meta.url)); const specRoot = resolve(here, '../..'); // packages/spec @@ -202,6 +203,8 @@ const report: any = { proofMissing: [] as string[], // a bound high-risk `live` entry with no proof at all orphanProofs: [] as string[], // a dogfood `@proof:` tag not registered in proof-registry.mts verification: null as VerificationReport | null, // `verifiedAt` ages — the re-verification worklist + evidenceLocal: 0, // repo-rooted evidence paths actually resolved against this checkout + evidenceForeign: 0, // evidence paths attributed to objectui / cloud — not resolvable here }; // Every classified entry, for the `verifiedAt` fold below. Collected during the @@ -217,8 +220,14 @@ function classify(type: string, path: string, status: string, led: any, cat: any // Framework-auto entries (`led === null`) have no ledger row to date-stamp. if (led !== null) verificationEntries.push({ key: `${type}/${path}`, status, verifiedAt: led?.verifiedAt }); if (status === 'live' && led?.evidence) { - const file = String(led.evidence).split(':')[0]; - if (/\//.test(file) && !existsSync(join(repoRoot, file))) report.staleEvidence.push(`${type}/${path} → ${led.evidence}`); + // Extract every repo-rooted path the evidence claims and resolve the ones + // attributed to THIS repo. Cross-repo pointers (objectui / cloud) are + // counted, not resolved — see evidence.mts for why the old + // `split(':')[0]` heuristic reported ~100% false positives. + const ev = checkEvidence(led.evidence, (p) => existsSync(join(repoRoot, p))); + report.evidenceForeign += ev.foreign.length; + report.evidenceLocal += ev.local.length; + for (const miss of ev.missing) report.staleEvidence.push(`${type}/${path} → ${miss}`); } // ── ADR-0054 prove-it-runs ── const boundClass = BOUND_PROOF_PATHS.get(`${type}/${path}`); @@ -316,6 +325,10 @@ if (asJson) { } const boundClasses = HIGH_RISK_CLASSES.filter((c) => c.bound).map((c) => c.label); console.log(`\nprove-it-runs (ADR-0054): proof REQUIRED for bound high-risk classes — ${boundClasses.join(', ') || 'none'}`); + console.log( + `\nevidence paths: ${report.evidenceLocal} resolved against this checkout, ` + + `${report.evidenceForeign} attributed to another repo (objectui / cloud — not resolvable here).`, + ); if (report.staleEvidence.length) { console.log(`\n⚠ ${report.staleEvidence.length} 'live' entr(ies) cite a missing file:`); report.staleEvidence.forEach((s: string) => console.log(` ${s}`)); diff --git a/packages/spec/scripts/liveness/evidence.mts b/packages/spec/scripts/liveness/evidence.mts new file mode 100644 index 0000000000..03ee5d4910 --- /dev/null +++ b/packages/spec/scripts/liveness/evidence.mts @@ -0,0 +1,118 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// Evidence path resolution for the liveness gate. +// +// WHY THIS EXISTS. The gate's stale-evidence check used to be one line: +// +// const file = String(led.evidence).split(':')[0]; +// if (/\//.test(file) && !existsSync(join(repoRoot, file))) → flag +// +// i.e. it assumed every `evidence` string is exactly `path/to/file.ts:123`. +// Almost none are. Real entries look like: +// +// "packages/spec/src/stack.zod.ts (mergeActionsIntoObjects stable-sorts …)" +// "packages/objectql/src/validation/rule-validator.ts (UPDATE strip); packages/…" +// "objectui: packages/app-shell/src/views/RecordDetailView.tsx + utils/…" +// +// Taking everything before the first colon turns the prose into the "filename", +// which never exists — so the check flagged 48 of 227 entries, and **every one +// of the 48 was either a parse artefact or a deliberate cross-repo pointer**. A +// warning list that is permanently non-empty and ~100% false is a warning +// nobody reads: the one genuinely rotted pointer in that list +// (`object.enable.clone` → a file that moved repos) sat there unnoticed. +// +// So: extract path-shaped tokens properly, and honour the cross-repo attribution +// the entries already write ("objectui: …"). What's left is signal. + +/** Top-level directories of THIS repo — the anchor for a repo-relative path claim. */ +export const REPO_ROOTS = ['apps', 'content', 'docker', 'docs', 'examples', 'packages', 'scripts', 'skills']; + +/** + * Realm markers an evidence string may use to attribute a path to another repo. + * `objectui` is the renderer repo; `cloud` is the closed EE runtime. `framework` + * switches back explicitly. These are already the house convention in prose — + * this makes them machine-read instead of decorative. + */ +export const FOREIGN_REALMS = ['objectui', 'cloud', 'ee']; +export const LOCAL_REALM = 'framework'; + +/** + * Paths that are repo-rooted in shape but never present in the OPEN edition. + * `@objectstack/service-ai` is the closed cloud runtime — `packages/services/` + * here has every sibling service EXCEPT service-ai. Entries cite it because that + * runtime is what consumes the property (see the `_note` in action.json). + */ +export const FOREIGN_PATH_PREFIXES = ['packages/services/service-ai/']; + +const PATH_RE = new RegExp(`^(?:${REPO_ROOTS.join('|')})/[\\w.@-]+(?:/[\\w.@-]+)*\\.[a-zA-Z]{1,5}$`); + +export interface EvidenceScan { + /** Repo-rooted paths attributed to THIS repo — these must resolve. */ + local: string[]; + /** Paths attributed to another repo (realm marker or foreign prefix) — not resolved here. */ + foreign: string[]; +} + +/** + * Strip surrounding punctuation and any `:123` / `:12-34` line suffix from a + * token. The trailing class includes `:` so a realm marker written `objectui:` + * reduces to `objectui`; a line suffix (`file.ts:150`) ends in a digit, so it + * survives that pass and is removed by the line-number rule after it. + */ +function bareToken(raw: string): string { + return raw + .replace(/^[([{<"'`,;]+/, '') + .replace(/[)\]}>"'`,;.:]+$/, '') + .replace(/:\d+(?:-\d+)?$/, ''); +} + +/** + * Split an evidence string into local vs foreign path claims. + * + * A realm marker (`objectui:`, `(objectui`, `cloud:`) attributes the paths that + * FOLLOW it, and its scope ends at the next clause boundary (`;` or a closing + * paren) — so `"objectui X gates … (plugin-audit, packages/plugins/…/y.ts) …"` + * still resolves the framework path in the trailing clause. Anything that is not + * repo-rooted (`app-shell/MetadataProvider.tsx`, `action-button/-group`) is + * prose and is neither resolved nor reported. + */ +export function scanEvidence(evidence: string): EvidenceScan { + const local: string[] = []; + const foreign: string[] = []; + let realm = LOCAL_REALM; + + for (const raw of String(evidence).split(/\s+/)) { + const token = bareToken(raw); + const asRealm = token.toLowerCase(); + + if (FOREIGN_REALMS.includes(asRealm)) { realm = asRealm; continue; } + if (asRealm === LOCAL_REALM) { realm = LOCAL_REALM; continue; } + + if (PATH_RE.test(token)) { + const isForeignPath = FOREIGN_PATH_PREFIXES.some((p) => token.startsWith(p)); + if (realm !== LOCAL_REALM || isForeignPath) foreign.push(token); + else local.push(token); + } + + // A clause boundary ends a realm's scope; the path above is classified first. + if (/[;)]/.test(raw)) realm = LOCAL_REALM; + } + + return { local: dedupe(local), foreign: dedupe(foreign) }; +} + +function dedupe(xs: string[]): string[] { + return [...new Set(xs)]; +} + +export interface EvidenceCheck extends EvidenceScan { + /** Local paths that do not exist — genuinely rotted pointers. */ + missing: string[]; +} + +/** Scan an evidence string and resolve its local paths against the filesystem. */ +export function checkEvidence(evidence: unknown, exists: (path: string) => boolean): EvidenceCheck { + if (typeof evidence !== 'string') return { local: [], foreign: [], missing: [] }; + const scan = scanEvidence(evidence); + return { ...scan, missing: scan.local.filter((p) => !exists(p)) }; +} diff --git a/packages/spec/scripts/liveness/evidence.test.ts b/packages/spec/scripts/liveness/evidence.test.ts new file mode 100644 index 0000000000..f6c3fcdfb4 --- /dev/null +++ b/packages/spec/scripts/liveness/evidence.test.ts @@ -0,0 +1,128 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. +// +// Unit tests for evidence path extraction (the stale-evidence check). + +import { describe, it, expect } from 'vitest'; +import { readFileSync, existsSync, readdirSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { dirname, join, resolve } from 'node:path'; +import { checkEvidence, scanEvidence } from './evidence.mts'; + +const here = dirname(fileURLToPath(import.meta.url)); +const specRoot = resolve(here, '../..'); +const repoRoot = resolve(specRoot, '../..'); +const ledgerRoot = join(specRoot, 'liveness'); + +const none = () => false; +const all = () => true; + +describe('scanEvidence — path extraction', () => { + it('extracts a bare `path:line` pointer', () => { + expect(scanEvidence('packages/runtime/src/http-dispatcher.ts:120').local) + .toEqual(['packages/runtime/src/http-dispatcher.ts']); + }); + + it('extracts a path followed by parenthetical prose (the old parser\'s blind spot)', () => { + // `split(':')[0]` returned the whole string here, which never exists. + const r = scanEvidence('packages/spec/src/stack.zod.ts (mergeActionsIntoObjects stable-sorts each group)'); + expect(r.local).toEqual(['packages/spec/src/stack.zod.ts']); + }); + + it('extracts every path from a multi-pointer string', () => { + const r = scanEvidence( + 'packages/objectql/src/validation/rule-validator.ts (UPDATE strip); packages/metadata-protocol/src/protocol.ts (INSERT ingress strip)', + ); + expect(r.local).toEqual([ + 'packages/objectql/src/validation/rule-validator.ts', + 'packages/metadata-protocol/src/protocol.ts', + ]); + }); + + it('ignores prose that merely contains a slash', () => { + const r = scanEvidence('objectui: components action-button/-group/-icon/-menu gate the Button'); + expect(r.local).toEqual([]); + expect(r.foreign).toEqual([]); + }); + + it('ignores a path that is not repo-rooted (another repo\'s internal layout)', () => { + expect(scanEvidence('objectui: app-shell/MetadataProvider.tsx reads it').local).toEqual([]); + }); +}); + +describe('scanEvidence — cross-repo attribution', () => { + it('attributes paths after an `objectui:` marker to objectui', () => { + const r = scanEvidence('objectui: packages/app-shell/src/views/RecordDetailView.tsx:573'); + expect(r.foreign).toEqual(['packages/app-shell/src/views/RecordDetailView.tsx']); + expect(r.local).toEqual([]); + }); + + it('handles the parenthesised marker form', () => { + const r = scanEvidence('registered into ActionEngine.shortcuts[] (objectui packages/core/src/actions/ActionEngine.ts:150) but no caller'); + expect(r.foreign).toEqual(['packages/core/src/actions/ActionEngine.ts']); + expect(r.local).toEqual([]); + }); + + it('ends a realm at the clause boundary so a later framework path is still resolved', () => { + const r = scanEvidence( + 'objectui RecordDetailView gates the tab (historyEnabled memo); pairs with packages/plugins/plugin-audit/src/audit-writers.ts', + ); + expect(r.foreign).toEqual([]); + expect(r.local).toEqual(['packages/plugins/plugin-audit/src/audit-writers.ts']); + }); + + it('treats the closed cloud runtime as foreign wherever it appears', () => { + const r = scanEvidence('packages/services/service-ai/src/agent-runtime.ts:264'); + expect(r.foreign).toEqual(['packages/services/service-ai/src/agent-runtime.ts']); + expect(r.local).toEqual([]); + // …but its open-edition siblings are local. + expect(scanEvidence('packages/services/service-automation/src/engine.ts:44').local).toHaveLength(1); + }); + + it('lets `framework` switch attribution back explicitly', () => { + const r = scanEvidence('objectui packages/app-shell/src/x.tsx framework packages/runtime/src/y.ts'); + expect(r.foreign).toEqual(['packages/app-shell/src/x.tsx']); + expect(r.local).toEqual(['packages/runtime/src/y.ts']); + }); +}); + +describe('checkEvidence', () => { + it('reports only unresolvable LOCAL paths', () => { + const r = checkEvidence('packages/a/src/x.ts (prose) objectui: packages/b/src/y.tsx', none); + expect(r.missing).toEqual(['packages/a/src/x.ts']); + }); + + it('is silent when every local path resolves', () => { + expect(checkEvidence('packages/a/src/x.ts', all).missing).toEqual([]); + }); + + it('tolerates a non-string evidence value', () => { + expect(checkEvidence(undefined, none)).toEqual({ local: [], foreign: [], missing: [] }); + expect(checkEvidence(42, none).missing).toEqual([]); + }); +}); + +// Contract test against the REAL ledgers: the gate reports these, so a rotted +// pointer committed to a ledger fails here too. +describe('shipped ledgers', () => { + it('every local evidence path resolves', () => { + const missing: string[] = []; + let local = 0; + for (const f of readdirSync(ledgerRoot).filter((x) => x.endsWith('.json'))) { + const ledger = JSON.parse(readFileSync(join(ledgerRoot, f), 'utf8')); + const visit = (key: string, entry: any) => { + if (entry?.status !== 'live' || typeof entry?.evidence !== 'string') return; + const r = checkEvidence(entry.evidence, (p) => existsSync(join(repoRoot, p))); + local += r.local.length; + r.missing.forEach((m) => missing.push(`${ledger.type}/${key} → ${m}`)); + }; + for (const [key, entry] of Object.entries(ledger.props || {})) { + visit(key, entry); + for (const [ck, centry] of Object.entries(entry?.children || {})) visit(`${key}.${ck}`, centry); + } + } + expect(missing).toEqual([]); + // Guard against the parser silently degrading to "extracts nothing" — that + // would make the assertion above vacuously true. + expect(local).toBeGreaterThan(100); + }); +});