feat(spec): compile-check skills/ TypeScript examples (anti-drift, #3094)

.github/workflows/lint.yml

@@ -199,3 +199,16 @@ jobs:
       # step above (reads the built dist).
       - name: Check @objectstack/spec public API surface
         run: pnpm --filter @objectstack/spec run check:api-surface
+
+      # Anti-drift for the skill EXAMPLES, not just the skill reference indexes
+      # (#3094). The TypeScript in skills/ is the first thing an AI copies when
+      # authoring metadata, yet nothing type-checked it — so it rotted silently
+      # (`ObjectSchema`/`Data`/`Field` imported from the wrong entry point, a
+      # `defineStack` key that no longer exists). Each block tagged with an
+      # `<!-- os:check -->` comment is extracted and run through `tsc --noEmit`
+      # against the built `@objectstack/spec` declarations — the exact surface a
+      # consumer's import resolves to — so a renamed export or tightened union
+      # fails here instead of in a third party's editor. Reads the built dist,
+      # so it runs after the build step alongside the other consumer gates.
+      - name: Check skills TypeScript examples compile
+        run: pnpm --filter @objectstack/spec run check:skill-examples

.gitignore

@@ -63,6 +63,9 @@ packages/spec/json-schema/
 # Generated bundle size reports
 packages/spec/bundle-size-report.json
 
+# Throwaway build dir for `check:skill-examples` (extracted skill code blocks)
+packages/spec/.examples-build/
+
 # Generated SBOM (rebuilt during release)
 sbom.json
 

packages/spec/package.json

@@ -207,7 +207,8 @@
     "check:liveness": "tsx scripts/liveness/check-liveness.mts",
     "gen:react-blocks": "tsx scripts/build-react-blocks-contract.ts",
     "check:react-blocks": "tsx scripts/build-react-blocks-contract.ts --check",
-    "check:react-conformance": "tsx scripts/check-react-blocks-conformance.ts"
+    "check:react-conformance": "tsx scripts/check-react-blocks-conformance.ts",
+    "check:skill-examples": "tsx scripts/check-skill-examples.ts"
   },
   "keywords": [
     "objectstack",

packages/spec/scripts/check-skill-examples.ts

@@ -0,0 +1,357 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+
+/**
+ * Check Skill Examples (anti-drift, #3094)
+ *
+ * The TypeScript examples inside `skills/` are the first thing an AI reads when
+ * authoring ObjectStack metadata, yet nothing type-checks them. When the spec
+ * renames an export or tightens a discriminated union, the examples silently rot
+ * (`defineDataset` → `defineSeed`, the removed `unique`/`async` validation
+ * types, kanban's top-level `groupBy`) and the platform's headline
+ * AI-native surface starts teaching code that no longer compiles. A third party
+ * following the skill hits the wall first.
+ *
+ * Full extraction of every ```ts block is infeasible: most are *fragments*
+ * (a `columns: [...]` subtree, a `kanban: {...}` literal) that would need a
+ * hand-authored wrapper to compile, and wrapping them yields high false-positive
+ * noise. So this gate is **opt-in**: a self-contained, should-compile block is
+ * marked by a `<!-- os:check -->` HTML comment on the line directly above its
+ * fence. The marker is an inert comment (renders to nothing) and — crucially —
+ * leaves the fence info-string a bare ` ```ts ` / ` ```typescript `, so the
+ * existing `check:doc-authoring` scanner (which keys on `^```(ts|typescript|tsx)$`)
+ * still sees the block. A fence-meta tag like ` ```ts check ` would have punched
+ * a hole in that gate.
+ *
+ * Each marked block is written verbatim to a throwaway build dir and type-checked
+ * with `tsc --noEmit` against the built `@objectstack/spec` declarations — the
+ * exact surface a consumer's `import { … } from '@objectstack/spec'` resolves to.
+ * Module resolution is wired via a `paths` map derived from the package's own
+ * `exports` field, so it self-updates as the spec adds/removes subpath exports.
+ *
+ * Because it reads the built `dist/*.d.ts`, this runs AFTER the workspace build
+ * step in CI — alongside `check:api-surface` / the example-app typecheck, its
+ * fellow "real consumer" gates — not before it like `check:skill-refs`.
+ *
+ * Usage:
+ *   tsx scripts/check-skill-examples.ts            # extract + type-check (CI)
+ *   tsx scripts/check-skill-examples.ts --keep     # also leave the build dir for inspection
+ */
+
+import { spawnSync } from 'child_process';
+import fs from 'fs';
+import path from 'path';
+
+// ── Paths ────────────────────────────────────────────────────────────────────
+
+const REPO_ROOT = path.resolve(__dirname, '../../..');
+const SPEC_DIR = path.resolve(__dirname, '..');
+const SKILLS_DIR = path.resolve(REPO_ROOT, 'skills');
+const BUILD_DIR = path.resolve(SPEC_DIR, '.examples-build');
+const SPEC_PKG_JSON = path.resolve(SPEC_DIR, 'package.json');
+
+/** Opt-in marker: the line directly above a fence opts that block into the gate. */
+const MARKER = '<!-- os:check -->';
+
+const KEEP = process.argv.includes('--keep');
+
+const rel = (p: string) => path.relative(REPO_ROOT, p);
+
+// ── Extraction ───────────────────────────────────────────────────────────────
+
+interface Example {
+  /** Source markdown file (absolute). */
+  source: string;
+  /** 1-based line in the source of the FIRST code line inside the fence. */
+  bodyStartLine: number;
+  /** Raw fence body. */
+  code: string;
+  /** Flat file name written into the build dir. */
+  fileName: string;
+}
+
+/** Every `*.md` under a skill folder — SKILL.md plus references/rules notes. */
+function skillMarkdownFiles(): string[] {
+  if (!fs.existsSync(SKILLS_DIR)) return [];
+  const out: string[] = [];
+  const walk = (dir: string) => {
+    for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
+      const full = path.join(dir, e.name);
+      if (e.isDirectory()) walk(full);
+      else if (e.name.endsWith('.md')) out.push(full);
+    }
+  };
+  for (const e of fs.readdirSync(SKILLS_DIR, { withFileTypes: true })) {
+    if (e.isDirectory()) walk(path.join(SKILLS_DIR, e.name));
+  }
+  return out.sort();
+}
+
+/**
+ * Pull every marked ```ts / ```typescript block out of one markdown file.
+ * A block is marked when the line immediately above its opening fence is
+ * exactly the MARKER (ignoring surrounding whitespace).
+ *
+ * Also reports `orphans`: MARKER lines that are NOT directly above a ts fence.
+ * A misplaced marker (a blank line slipped in between, or it precedes a bash /
+ * json block) silently checks nothing — exactly the failure mode this gate
+ * exists to prevent — so the caller treats an orphan as an error, not a no-op.
+ */
+function extractFromFile(source: string): { examples: Example[]; orphans: number[] } {
+  const lines = fs.readFileSync(source, 'utf-8').split('\n');
+  const skillDir = path.relative(SKILLS_DIR, source).split(path.sep)[0];
+  const base = path.basename(source, '.md');
+  const examples: Example[] = [];
+  const claimed = new Set<number>(); // MARKER line indices that opened a real block
+  let n = 0;
+
+  for (let i = 0; i < lines.length; i++) {
+    const open = lines[i].match(/^```(ts|typescript)\s*$/);
+    if (!open) continue;
+    const marked = i > 0 && lines[i - 1].trim() === MARKER;
+    // Find the matching close fence regardless of marking, so `i` advances past
+    // this block and we never treat its body as top-level markdown.
+    let close = i + 1;
+    while (close < lines.length && !/^```\s*$/.test(lines[close])) close++;
+    if (marked) {
+      claimed.add(i - 1);
+      const body = lines.slice(i + 1, close);
+      n += 1;
+      examples.push({
+        source,
+        bodyStartLine: i + 2, // 1-based line of body[0]
+        code: body.join('\n'),
+        fileName: `${skillDir}__${base}__${n}.ts`,
+      });
+    }
+    i = close; // skip to the close fence
+  }
+
+  const orphans: number[] = [];
+  for (let i = 0; i < lines.length; i++) {
+    if (lines[i].trim() === MARKER && !claimed.has(i)) orphans.push(i + 1); // 1-based
+  }
+  return { examples, orphans };
+}
+
+// ── Module resolution derived from the spec's own `exports` ──────────────────
+
+interface ExportEntry {
+  import?: { types?: string };
+  require?: { types?: string };
+}
+
+/**
+ * Build a tsconfig `paths` map from `@objectstack/spec`'s published `exports`,
+ * pointing each specifier at its built `.d.ts`. Deriving it from the real
+ * exports means a new subpath export (or a removed one) is reflected here for
+ * free — the map cannot drift from what consumers actually resolve.
+ *
+ * Returns the map plus the list of declaration files that must exist; a missing
+ * root declaration means the spec was not built (or built with OS_SKIP_DTS), and
+ * the caller fails loudly rather than checking against a stale/absent surface.
+ */
+function specPaths(): { paths: Record<string, string[]>; missing: string[] } {
+  const pkg = JSON.parse(fs.readFileSync(SPEC_PKG_JSON, 'utf-8'));
+  const exportsMap = pkg.exports as Record<string, ExportEntry | string>;
+  const paths: Record<string, string[]> = {};
+  const missing: string[] = [];
+
+  for (const [key, entry] of Object.entries(exportsMap)) {
+    const types =
+      typeof entry === 'string'
+        ? entry
+        : (entry.require?.types ?? entry.import?.types);
+    if (!types || !types.endsWith('.d.ts')) continue; // skip non-type conditions (css, etc.)
+    const specifier = key === '.' ? '@objectstack/spec' : `@objectstack/spec/${key.slice(2)}`;
+    const abs = path.resolve(SPEC_DIR, types);
+    paths[specifier] = [abs];
+    if (!fs.existsSync(abs)) missing.push(rel(abs));
+  }
+  return { paths, missing };
+}
+
+// ── tsc harness ──────────────────────────────────────────────────────────────
+
+function writeBuildDir(examples: Example[], paths: Record<string, string[]>): void {
+  fs.rmSync(BUILD_DIR, { recursive: true, force: true });
+  fs.mkdirSync(BUILD_DIR, { recursive: true });
+
+  for (const ex of examples) {
+    // Written verbatim (no prepended wrapper) so a tsc line N maps to source
+    // line (bodyStartLine + N - 1) with zero arithmetic guesswork. A block with
+    // no import/export is a script, not a module; append `export {}` so two such
+    // files can't collide on a global — appended at the end, it never shifts the
+    // line of any real diagnostic.
+    const isModule = /^\s*(import|export)\b/m.test(ex.code);
+    fs.writeFileSync(
+      path.join(BUILD_DIR, ex.fileName),
+      ex.code + (isModule ? '' : '\nexport {};\n'),
+    );
+  }
+
+  const tsconfig = {
+    compilerOptions: {
+      // A consumer-faithful, illustrative-code-friendly profile: strict enough
+      // to catch real type drift, lax on the two rules that punish example code
+      // (an import shown for context, an unused binding).
+      target: 'ES2020',
+      module: 'ESNext',
+      moduleResolution: 'bundler',
+      lib: ['ES2020', 'DOM', 'DOM.Iterable'],
+      types: [],
+      strict: true,
+      skipLibCheck: true,
+      esModuleInterop: true,
+      resolveJsonModule: true,
+      noEmit: true,
+      noUnusedLocals: false,
+      noUnusedParameters: false,
+      // `paths` values are absolute, so no `baseUrl` is needed — and omitting it
+      // sidesteps TS 6.0's `baseUrl` deprecation (TS5101).
+      paths,
+    },
+    include: ['*.ts'],
+  };
+  fs.writeFileSync(
+    path.join(BUILD_DIR, 'tsconfig.json'),
+    JSON.stringify(tsconfig, null, 2),
+  );
+}
+
+interface Diagnostic {
+  file: string; // build-dir file name
+  line: number;
+  col: number;
+  text: string; // full tsc line, from the code after the location
+}
+
+/** Parse `file.ts(line,col): error TSxxxx: message` lines from `tsc --pretty false`. */
+function parseDiagnostics(output: string): Diagnostic[] {
+  const diags: Diagnostic[] = [];
+  const re = /^(.+?)\((\d+),(\d+)\):\s+(error|warning)\s+(TS\d+:.*)$/;
+  for (const raw of output.split('\n')) {
+    const m = raw.match(re);
+    if (!m) continue;
+    diags.push({
+      file: path.basename(m[1]),
+      line: Number(m[2]),
+      col: Number(m[3]),
+      text: `${m[4]} ${m[5]}`,
+    });
+  }
+  return diags;
+}
+
+function runTsc(): { code: number; output: string } {
+  const tscBin = require.resolve('typescript/bin/tsc');
+  const res = spawnSync(
+    process.execPath,
+    [tscBin, '--noEmit', '--pretty', 'false', '-p', path.join(BUILD_DIR, 'tsconfig.json')],
+    { cwd: BUILD_DIR, encoding: 'utf-8' },
+  );
+  return { code: res.status ?? 1, output: `${res.stdout ?? ''}${res.stderr ?? ''}` };
+}
+
+// ── Main ─────────────────────────────────────────────────────────────────────
+
+function fail(message: string): never {
+  console.error(`\n✗ ${message}\n`);
+  process.exit(1);
+}
+
+function main() {
+  console.log('🧪 Type-checking skill TypeScript examples...\n');
+
+  const files = skillMarkdownFiles();
+  const examples: Example[] = [];
+  const orphans: string[] = [];
+  for (const file of files) {
+    const { examples: found, orphans: bad } = extractFromFile(file);
+    examples.push(...found);
+    for (const line of bad) orphans.push(`${rel(file)}:${line}`);
+  }
+
+  // A marker that is not directly above a ```ts fence checks nothing. Fail loudly
+  // rather than let it read as covered — a placed-but-inert marker is worse than
+  // no marker, because it looks intentional.
+  if (orphans.length > 0) {
+    fail(
+      `Found ${MARKER} not directly above a \`\`\`ts / \`\`\`typescript fence:\n\n` +
+        orphans.map((o) => `  - ${o}`).join('\n') +
+        `\n\n  The marker must be the line IMMEDIATELY above the code fence (no blank\n` +
+        `  line between). Move it, or remove it if the block should not be checked.`,
+    );
+  }
+
+  // Vacuous-green guard: opt-in tagging means "zero blocks" is far more likely
+  // to be "the marker got renamed / stripped" than "no examples worth checking".
+  // A gate that checks nothing must not report success.
+  if (examples.length === 0) {
+    fail(
+      `No skill examples are marked for type-checking.\n\n` +
+        `  Mark a self-contained, compilable block by putting\n\n` +
+        `    ${MARKER}\n\n` +
+        `  on the line directly above its \`\`\`ts fence in a skills/**/*.md file.\n` +
+        `  (If you just removed the last marker, that is almost certainly a mistake.)`,
+    );
+  }
+
+  const { paths, missing } = specPaths();
+  if (missing.some((m) => m.endsWith('index.d.ts')) && !fs.existsSync(paths['@objectstack/spec']?.[0] ?? '')) {
+    fail(
+      `@objectstack/spec is not built — no declarations to check examples against:\n\n` +
+        missing.map((m) => `  - ${m} (missing)`).join('\n') +
+        `\n\n  Build the spec first (CI does this in the "Build workspace packages" step):\n\n` +
+        `    pnpm --filter @objectstack/spec build`,
+    );
+  }
+
+  console.log(`   ${examples.length} marked example(s) across ${new Set(examples.map((e) => e.source)).size} file(s):`);
+  for (const ex of examples) {
+    console.log(`     • ${rel(ex.source)}:${ex.bodyStartLine} → ${ex.fileName}`);
+  }
+  console.log('');
+
+  writeBuildDir(examples, paths);
+  const { code, output } = runTsc();
+
+  const byFile = new Map(examples.map((e) => [e.fileName, e]));
+  const diags = parseDiagnostics(output);
+
+  if (code === 0 && diags.length === 0) {
+    console.log(`✅ ${examples.length} skill examples type-check against @objectstack/spec`);
+    if (!KEEP) fs.rmSync(BUILD_DIR, { recursive: true, force: true });
+    return;
+  }
+
+  // Remap every diagnostic back to skills/**/SKILL.md:<real line> so the author
+  // reads the error against the file they actually edit, not the throwaway copy.
+  console.error(`\n✗ Skill TypeScript examples do not compile against @objectstack/spec:\n`);
+  const grouped = new Map<string, string[]>();
+  for (const d of diags) {
+    const ex = byFile.get(d.file);
+    const loc = ex ? `${rel(ex.source)}:${ex.bodyStartLine + d.line - 1}:${d.col}` : d.file;
+    const key = ex ? rel(ex.source) : d.file;
+    if (!grouped.has(key)) grouped.set(key, []);
+    grouped.get(key)!.push(`  ${loc}\n      ${d.text}`);
+  }
+  for (const [, entries] of grouped) {
+    for (const e of entries) console.error(e);
+  }
+
+  // A non-zero exit with no parseable diagnostics (e.g. a tsconfig error) must
+  // still surface — print the raw tail so it is never a silent failure.
+  if (diags.length === 0) {
+    console.error(`\n  tsc exited ${code} but produced no parseable diagnostics. Raw output:\n`);
+    console.error(output.split('\n').map((l) => `    ${l}`).join('\n'));
+  }
+
+  console.error(
+    `\n  These are examples an AI copies verbatim. Fix the example to match the\n` +
+      `  current spec, or drop its ${MARKER} marker if it is an intentional fragment.\n`,
+  );
+  if (!KEEP) fs.rmSync(BUILD_DIR, { recursive: true, force: true });
+  process.exit(1);
+}
+
+main();