diff --git a/.changeset/ai-liveness.md b/.changeset/ai-liveness.md new file mode 100644 index 0000000000..f9987a2a93 --- /dev/null +++ b/.changeset/ai-liveness.md @@ -0,0 +1,4 @@ +--- +--- + +chore(spec): govern the `ai` category (Agent/Tool/Skill) in the spec-liveness gate, and add two gate capabilities: an **allowlist mode** (`"mode":"allowlist"`) so categories dominated by protocol/engine DTOs only classify their authorable subset, and **auto-classification of ADR-0010 framework provenance/lock fields** (`_lock*`/`_provenance`/`_packageId`/`protection`). Seeded from the agent/tool/skill liveness audits. Repo-internal tooling; no package version impact. diff --git a/packages/spec/liveness/README.md b/packages/spec/liveness/README.md index 9c45c63875..25c9f17c6a 100644 --- a/packages/spec/liveness/README.md +++ b/packages/spec/liveness/README.md @@ -28,9 +28,30 @@ so the ratchet does not flag additions to a wholesale-dead/internal subtree — additions to per-property schemas (the mixed ones like `ObjectPermission`, `PermissionSet`). Use `_schema` only for subtrees that are genuinely all-one-status. +## Two governance modes + +A category's ledger picks how the gate scopes it: + +- **default** — *every* authorable object schema in the category must be classified + (`"_schema": "internal"` exempts non-authorable ones). Right for clean, + fully-authorable categories: `security`, `identity`. +- **allowlist** (`"mode": "allowlist"` + `"governed": ["Agent","Tool","Skill"]`) — + only the named schemas are checked; the rest of the category is out of scope. + Right for categories dominated by protocol/engine/runtime DTOs where the + authorable types are a small subset: `ai` (Agent/Tool/Skill among embedder/ + knowledge/model DTOs). A `governed` name that no longer resolves to a schema is + reported (catches renames that would silently drop coverage). + +## Framework fields (auto-classified) + +The ADR-0010 provenance/lock overlay fields — `_lock`, `_lockReason`, `_lockSource`, +`_lockDocsUrl`, `_provenance`, `_packageId`, `_packageVersion`, `protection` — appear +on every authorable type and are system-stamped, not type-specific surface. The gate +auto-classifies them `live`, so ledgers don't repeat them. + ## Files -- `.json` — the ledger for a governed category (currently: `security`). +- `.json` — the ledger for a governed category (`security`, `identity`, `ai`). - `../scripts/liveness/check-liveness.mjs` — the gate. Reads the generated `packages/spec/json-schema//*.json`, resolves each authorable property's status, and exits non-zero on any UNCLASSIFIED property. @@ -49,18 +70,32 @@ CI: `.github/workflows/spec-liveness-check.yml` runs the gate on PRs touching ## Rolling out the next category Governed categories are listed in `GOVERNED` at the top of `check-liveness.mjs`, -rolled out **highest-risk-first**. To add one (e.g. `automation`, `data`): +rolled out **highest-risk-first**. To add one (e.g. `automation`, `ui`, `data`): -1. `--dump ` to inventory its authorable properties. +1. `--dump ` to inventory its authorable properties. **The json-schema + categories do NOT map to "authorable types"** — most (`data`, `automation`, `ui`, + `kernel`) are dominated by ObjectQL/engine/protocol DTOs, and some authorable + types live elsewhere (Agent/Tool/Skill in `ai`, Dataset in `ui`). Decide the + handful of authorable schemas and use **allowlist mode** unless the whole + category is authorable. 2. Seed `.json` from that category's liveness audit (file:line evidence) and targeted greps for anything the audit didn't cover. **Classify only with evidence** — `live` needs a cited consumer; `dead` needs a confirmed absence. 3. Add the category to `GOVERNED` and confirm the gate is green. -## Current state — `security` +## Current state + +| Category | Mode | Properties | Notes | +|---|---|---|---| +| `security` | default | 93 (26 live / 1 exp / 66 dead)* | ~71% parsed-but-unenforced; enforce-or-remove worklist | +| `identity` | default | 4 (3 live / 1 dead) | `Role` authorable; rest internal (SCIM/auth runtime) | +| `ai` | allowlist | 63 (46 live / 5 exp / 12 dead) | Agent/Tool/Skill; `Tool` is write-only, agent access-control dead | + +\* security numbers shift to 26 / 35 / 32 once the PolicySchema experimental +markers (ADR-0049 #1882) land. -93 authorable properties: **66 dead, 26 live, 1 experimental.** ~71% of the -authorable security surface is parsed-but-unenforced. The `dead` entries are the -worklist for the security enforce-or-remove ADR — most urgently the destructive -`ObjectPermission.allow{Transfer,Restore,Purge}` (ungated) and the entirely-dead -`Policy` tree (password/session/`forceMfa`/network/audit). +The `dead` entries are the cross-category enforce-or-remove worklist (ADR-0049). +Highest-signal: the destructive `ObjectPermission.allow{Transfer,Restore,Purge}` +(ungated), the entirely-dead `Policy` tree, and **agent `access`/`permissions`/ +`visibility`** — "who can chat with this agent" is a no-op (the chat route hardcodes +`['ai:chat','ai:agents']`). diff --git a/packages/spec/liveness/ai.json b/packages/spec/liveness/ai.json new file mode 100644 index 0000000000..7271711eae --- /dev/null +++ b/packages/spec/liveness/ai.json @@ -0,0 +1,215 @@ +{ + "category": "ai", + "mode": "allowlist", + "governed": [ + "Agent", + "Tool", + "Skill" + ], + "_note": "The `ai` category is dominated by embedder/knowledge/model/runtime DTOs; the authorable metadata types are Agent, Tool, Skill (allowlist). Seeded from docs/audits/2026-06-{agent,tool,skill}schema-property-liveness.md (file:line evidence). Framework provenance/lock fields (_lock*, _provenance, _packageId, protection) are auto-classified by the gate (ADR-0010).", + "schemas": { + "Agent": { + "_note": "agent-runtime.ts is the runtime consumer; AgentPreview is display-only (not a runtime reader).", + "props": { + "name": { + "status": "live", + "evidence": "packages/services/service-ai/src/agent-runtime.ts", + "note": "core identity." + }, + "label": { + "status": "live", + "note": "display/core." + }, + "avatar": { + "status": "live", + "note": "display." + }, + "role": { + "status": "live", + "evidence": "agent-runtime.ts", + "note": "persona → system prompt; also derives chatbot picker description." + }, + "instructions": { + "status": "live", + "evidence": "agent-runtime.ts", + "note": "system prompt." + }, + "model": { + "status": "live", + "evidence": "packages/services/service-ai/src/agent-runtime.ts:264", + "note": "PARTIAL — model/temperature/maxTokens applied; model.provider + model.topP are DEAD (provider comes from the configured adapter)." + }, + "skills": { + "status": "live", + "evidence": "agent-runtime.ts", + "note": "Agent→Skill→Tool; contributes tools + prompt." + }, + "tools": { + "status": "live", + "evidence": "agent-runtime.ts", + "note": "legacy direct-tool fallback." + }, + "active": { + "status": "live", + "evidence": "agent-runtime.ts", + "note": "gates listing + 403 on chat." + }, + "planning": { + "status": "live", + "evidence": "packages/services/service-ai/src/agent-runtime.ts", + "note": "PARTIAL — only planning.maxIterations is live (3 call sites); planning.strategy/allowReplan are DEAD." + }, + "access": { + "status": "dead", + "evidence": "packages/services/service-ai/src/routes/agent-routes.ts:109", + "note": "'who can chat' is a no-op — the chat route hardcodes ['ai:chat','ai:agents'] regardless. Latent access-control gap (ADR-0049 #1884: enforce or remove)." + }, + "permissions": { + "status": "dead", + "evidence": "agent-routes.ts:109 (hardcoded perms)", + "note": "display-only; not enforced. Security gap." + }, + "visibility": { + "status": "dead", + "evidence": "no runtime reader", + "note": "defaults 'organization' but gates nothing." + }, + "tenantId": { + "status": "dead", + "evidence": "no runtime reader", + "note": "not read at runtime." + }, + "knowledge": { + "status": "dead", + "evidence": "no runtime reader; AgentPreview.tsx:213 reads knowledge.sources but spec defines {topics,indexes}", + "note": "shape drift; RAG is wired via service-knowledge, not agent.knowledge." + }, + "lifecycle": { + "status": "experimental", + "evidence": "no runtime reader (StateMachine)", + "note": "aspirational autonomy — prune or keep as roadmap (ADR-0049 enforce-or-remove)." + }, + "memory": { + "status": "experimental", + "evidence": "no runtime reader", + "note": "aspirational autonomy (shortTerm/longTerm/reflectionInterval)." + }, + "guardrails": { + "status": "experimental", + "evidence": "no runtime reader", + "note": "aspirational — real limits enforced by an unrelated quota service, not these." + }, + "structuredOutput": { + "status": "experimental", + "evidence": "no runtime reader", + "note": "aspirational." + } + } + }, + "Tool": { + "_note": "Tool metadata is WRITE-ONLY — projected one-way from the code-built ToolRegistry into metadata for Studio display (plugin.ts:813). No code reads `tool` metadata back; the runtime contract is a separate AIToolDefinition. The LIVE props are live via that same-named AIToolDefinition surface, not via metadata read-back.", + "props": { + "name": { + "status": "live", + "evidence": "packages/services/service-ai/src/adapters/vercel-adapter.ts:46", + "note": "selection + LLM function key." + }, + "label": { + "status": "live", + "note": "display." + }, + "description": { + "status": "live", + "evidence": "packages/services/service-ai/src/adapters/vercel-adapter.ts:46", + "note": "sent to the LLM." + }, + "parameters": { + "status": "live", + "evidence": "packages/services/service-ai/src/adapters/vercel-adapter.ts:46", + "note": "LLM function schema." + }, + "objectName": { + "status": "live", + "evidence": "packages/services/service-ai/src/tools/action-tools.ts:535", + "note": "action-tool dispatch." + }, + "outputSchema": { + "status": "experimental", + "evidence": "tools/action-tools.ts:437 (keys folded into description only)", + "note": "docstring claims output validation + chaining, but no validation exists." + }, + "category": { + "status": "dead", + "evidence": "listing/preview tag only; enum drift vs AIToolDefinition free-string", + "note": "not sent to the model." + }, + "requiresConfirmation": { + "status": "dead", + "evidence": "tools/action-tools.ts:239 (HITL re-derives from action.ai.requiresConfirmation)", + "note": "never read off the def — redundant mirror." + }, + "permissions": { + "status": "dead", + "evidence": "tool.form.ts only; not on AIToolDefinition, no consumer" + }, + "active": { + "status": "dead", + "evidence": "never set by any registration path" + }, + "builtIn": { + "status": "dead", + "evidence": "never set by any registration path" + } + } + }, + "Skill": { + "_note": "skill-registry.ts + agent-runtime.ts are the runtime consumers.", + "props": { + "name": { + "status": "live", + "evidence": "packages/services/service-ai/src/skill-registry.ts" + }, + "label": { + "status": "live", + "evidence": "packages/services/service-ai/src/skill-registry.ts:247", + "note": "injected into the agent system prompt." + }, + "description": { + "status": "live", + "evidence": "packages/services/service-ai/src/skill-registry.ts:247", + "note": "injected into prompt." + }, + "instructions": { + "status": "live", + "evidence": "packages/services/service-ai/src/skill-registry.ts:247", + "note": "injected into prompt." + }, + "tools": { + "status": "live", + "evidence": "packages/services/service-ai/src/skill-registry.ts:206", + "note": "tool-contribution path incl. action_* wildcard expand." + }, + "triggerConditions": { + "status": "live", + "evidence": "packages/services/service-ai/src/skill-registry.ts:153", + "note": "THE activation gate — AND of {field,operator,value}." + }, + "active": { + "status": "live", + "evidence": "packages/services/service-ai/src/skill-registry.ts:93", + "note": "inactive skills dropped." + }, + "triggerPhrases": { + "status": "dead", + "evidence": "no intent/phrase matcher; only populates the slash-command palette summary (toSummary)", + "note": "schema frames them as activating the skill, but nothing matches user text — intent routing unimplemented." + }, + "permissions": { + "status": "dead", + "evidence": "no code restricts a skill by skill.permissions; naming drift (preview calls it requiredPermissions)", + "note": "enforce-or-remove + fix naming (ADR-0049)." + } + } + } + } +} diff --git a/packages/spec/scripts/liveness/check-liveness.mjs b/packages/spec/scripts/liveness/check-liveness.mjs index e4b7a2b0e4..4a9f95183b 100644 --- a/packages/spec/scripts/liveness/check-liveness.mjs +++ b/packages/spec/scripts/liveness/check-liveness.mjs @@ -38,7 +38,7 @@ const ledgerRoot = join(specRoot, 'liveness'); // Categories whose authorable schemas must be fully classified. Extend // highest-risk-first as each category's ledger is seeded from its audit. -const GOVERNED = ['security', 'identity']; +const GOVERNED = ['security', 'identity', 'ai']; const args = process.argv.slice(2); const asJson = args.includes('--json'); @@ -50,6 +50,16 @@ const MARKER_RE = { planned: /\[planned|not yet implemented|coming soon/i, }; +// Framework-managed provenance/lock overlay fields (ADR-0010 + metadata +// provenance stamping). These appear on EVERY authorable metadata type, are +// system-stamped (not type-specific authorable surface), and are consumed by the +// overlay/lock loader. Auto-classified `live` so each category's ledger doesn't +// have to repeat them. +const FRAMEWORK_FIELDS = new Set([ + '_lock', '_lockReason', '_lockSource', '_lockDocsUrl', + '_provenance', '_packageId', '_packageVersion', 'protection', +]); + function loadSchemas(category) { const dir = join(schemaRoot, category); if (!existsSync(dir)) return []; @@ -98,13 +108,30 @@ const report = { categories: {}, totals: { classified: 0, unclassified: 0, bySta for (const category of GOVERNED) { const ledger = loadLedger(category) || { schemas: {} }; + // Two governance modes: + // - default: every authorable object schema in the category must be classified + // (`internal` exempts non-authorable ones). Right for clean, fully-authorable + // categories (security, identity). + // - allowlist (`"mode":"allowlist"` + `"governed":[...]`): only the named schemas + // are checked; the rest of the category is out of scope. Right for categories + // dominated by protocol/engine DTOs where the authorable types are a small + // subset (ai: Agent/Tool/Skill among embedder/knowledge/model DTOs). + const allowlist = ledger.mode === 'allowlist' ? new Set(ledger.governed || []) : null; const cat = { classified: 0, unclassified: 0, byStatus: {} }; + const seen = new Set(); for (const { name, schema } of loadSchemas(category)) { + if (allowlist && !allowlist.has(name)) continue; // allowlist: skip un-governed schemas + seen.add(name); const props = topProps(schema); if (props.length === 0) continue; // enums / scalars carry no authorable props const entry = ledger.schemas[name] || {}; if (entry._schema === 'internal') continue; // exempt: not authorable for (const { key, description } of props) { + if (FRAMEWORK_FIELDS.has(key)) { // ADR-0010 provenance/lock — auto-live + cat.classified++; cat.byStatus.live = (cat.byStatus.live || 0) + 1; + report.totals.byStatus.live = (report.totals.byStatus.live || 0) + 1; + continue; + } const led = entry.props && entry.props[key]; const status = (led && led.status) || entry._schema || markerStatus(description); if (!status) { @@ -122,6 +149,16 @@ for (const category of GOVERNED) { } } } + // allowlist sanity: a governed schema name that no longer exists is a typo or a + // rename that silently drops coverage — surface it as unclassified. + if (allowlist) { + for (const want of allowlist) { + if (!seen.has(want)) { + cat.unclassified++; + report.unclassified.push(`${category}/${want} (governed schema not found — renamed or removed?)`); + } + } + } report.categories[category] = cat; report.totals.classified += cat.classified; report.totals.unclassified += cat.unclassified;