check-skill-examples.ts

// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.

/**
 * Check Skill Examples (anti-drift, #3094)
 *
 * The TypeScript examples inside `skills/` are the first thing an AI reads when
 * authoring ObjectStack metadata, yet nothing type-checks them. When the spec
 * renames an export or tightens a discriminated union, the examples silently rot
 * (`defineDataset` → `defineSeed`, the removed `unique`/`async` validation
 * types, kanban's top-level `groupBy`) and the platform's headline
 * AI-native surface starts teaching code that no longer compiles. A third party
 * following the skill hits the wall first.
 *
 * Full extraction of every ```ts block is infeasible: most are *fragments*
 * (a `columns: [...]` subtree, a `kanban: {...}` literal) that would need a
 * hand-authored wrapper to compile, and wrapping them yields high false-positive
 * noise. So this gate is **opt-in**: a self-contained, should-compile block is
 * marked by a `<!-- os:check -->` HTML comment on the line directly above its
 * fence. The marker is an inert comment (renders to nothing) and — crucially —
 * leaves the fence info-string a bare ` ```ts ` / ` ```typescript `, so the
 * existing `check:doc-authoring` scanner (which keys on `^```(ts|typescript|tsx)$`)
 * still sees the block. A fence-meta tag like ` ```ts check ` would have punched
 * a hole in that gate.
 *
 * Each marked block is written verbatim to a throwaway build dir and type-checked
 * with `tsc --noEmit` against the built `@objectstack/spec` declarations — the
 * exact surface a consumer's `import { … } from '@objectstack/spec'` resolves to.
 * Module resolution is wired via a `paths` map derived from the package's own
 * `exports` field, so it self-updates as the spec adds/removes subpath exports.
 *
 * Because it reads the built `dist/*.d.ts`, this runs AFTER the workspace build
 * step in CI — alongside `check:api-surface` / the example-app typecheck, its
 * fellow "real consumer" gates — not before it like `check:skill-refs`.
 *
 * Usage:
 *   tsx scripts/check-skill-examples.ts            # extract + type-check (CI)
 *   tsx scripts/check-skill-examples.ts --keep     # also leave the build dir for inspection
 */

import { spawnSync } from 'child_process';
import fs from 'fs';
import path from 'path';

// ── Paths ────────────────────────────────────────────────────────────────────

const REPO_ROOT = path.resolve(__dirname, '../../..');
const SPEC_DIR = path.resolve(__dirname, '..');
const SKILLS_DIR = path.resolve(REPO_ROOT, 'skills');
const BUILD_DIR = path.resolve(SPEC_DIR, '.examples-build');
const SPEC_PKG_JSON = path.resolve(SPEC_DIR, 'package.json');

/** Opt-in marker: the line directly above a fence opts that block into the gate. */
const MARKER = '<!-- os:check -->';

const KEEP = process.argv.includes('--keep');

const rel = (p: string) => path.relative(REPO_ROOT, p);

// ── Extraction ───────────────────────────────────────────────────────────────

interface Example {
  /** Source markdown file (absolute). */
  source: string;
  /** 1-based line in the source of the FIRST code line inside the fence. */
  bodyStartLine: number;
  /** Raw fence body. */
  code: string;
  /** Flat file name written into the build dir. */
  fileName: string;
}

/** Every `*.md` under a skill folder — SKILL.md plus references/rules notes. */
function skillMarkdownFiles(): string[] {
  if (!fs.existsSync(SKILLS_DIR)) return [];
  const out: string[] = [];
  const walk = (dir: string) => {
    for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
      const full = path.join(dir, e.name);
      if (e.isDirectory()) walk(full);
      else if (e.name.endsWith('.md')) out.push(full);
    }
  };
  for (const e of fs.readdirSync(SKILLS_DIR, { withFileTypes: true })) {
    if (e.isDirectory()) walk(path.join(SKILLS_DIR, e.name));
  }
  return out.sort();
}

/**
 * Pull every marked ```ts / ```typescript block out of one markdown file.
 * A block is marked when the line immediately above its opening fence is
 * exactly the MARKER (ignoring surrounding whitespace).
 *
 * Also reports `orphans`: MARKER lines that are NOT directly above a ts fence.
 * A misplaced marker (a blank line slipped in between, or it precedes a bash /
 * json block) silently checks nothing — exactly the failure mode this gate
 * exists to prevent — so the caller treats an orphan as an error, not a no-op.
 */
function extractFromFile(source: string): { examples: Example[]; orphans: number[] } {
  const lines = fs.readFileSync(source, 'utf-8').split('\n');
  const skillDir = path.relative(SKILLS_DIR, source).split(path.sep)[0];
  const base = path.basename(source, '.md');
  const examples: Example[] = [];
  const claimed = new Set<number>(); // MARKER line indices that opened a real block
  let n = 0;

  for (let i = 0; i < lines.length; i++) {
    const open = lines[i].match(/^```(ts|typescript)\s*$/);
    if (!open) continue;
    const marked = i > 0 && lines[i - 1].trim() === MARKER;
    // Find the matching close fence regardless of marking, so `i` advances past
    // this block and we never treat its body as top-level markdown.
    let close = i + 1;
    while (close < lines.length && !/^```\s*$/.test(lines[close])) close++;
    if (marked) {
      claimed.add(i - 1);
      const body = lines.slice(i + 1, close);
      n += 1;
      examples.push({
        source,
        bodyStartLine: i + 2, // 1-based line of body[0]
        code: body.join('\n'),
        fileName: `${skillDir}__${base}__${n}.ts`,
      });
    }
    i = close; // skip to the close fence
  }

  const orphans: number[] = [];
  for (let i = 0; i < lines.length; i++) {
    if (lines[i].trim() === MARKER && !claimed.has(i)) orphans.push(i + 1); // 1-based
  }
  return { examples, orphans };
}

// ── Module resolution derived from the spec's own `exports` ──────────────────

interface ExportEntry {
  import?: { types?: string };
  require?: { types?: string };
}

/**
 * Build a tsconfig `paths` map from `@objectstack/spec`'s published `exports`,
 * pointing each specifier at its built `.d.ts`. Deriving it from the real
 * exports means a new subpath export (or a removed one) is reflected here for
 * free — the map cannot drift from what consumers actually resolve.
 *
 * Returns the map plus the list of declaration files that must exist; a missing
 * root declaration means the spec was not built (or built with OS_SKIP_DTS), and
 * the caller fails loudly rather than checking against a stale/absent surface.
 */
function specPaths(): { paths: Record<string, string[]>; missing: string[] } {
  const pkg = JSON.parse(fs.readFileSync(SPEC_PKG_JSON, 'utf-8'));
  const exportsMap = pkg.exports as Record<string, ExportEntry | string>;
  const paths: Record<string, string[]> = {};
  const missing: string[] = [];

  for (const [key, entry] of Object.entries(exportsMap)) {
    const types =
      typeof entry === 'string'
        ? entry
        : (entry.require?.types ?? entry.import?.types);
    if (!types || !types.endsWith('.d.ts')) continue; // skip non-type conditions (css, etc.)
    const specifier = key === '.' ? '@objectstack/spec' : `@objectstack/spec/${key.slice(2)}`;
    const abs = path.resolve(SPEC_DIR, types);
    paths[specifier] = [abs];
    if (!fs.existsSync(abs)) missing.push(rel(abs));
  }
  return { paths, missing };
}

// ── tsc harness ──────────────────────────────────────────────────────────────

function writeBuildDir(examples: Example[], paths: Record<string, string[]>): void {
  fs.rmSync(BUILD_DIR, { recursive: true, force: true });
  fs.mkdirSync(BUILD_DIR, { recursive: true });

  for (const ex of examples) {
    // Written verbatim (no prepended wrapper) so a tsc line N maps to source
    // line (bodyStartLine + N - 1) with zero arithmetic guesswork. A block with
    // no import/export is a script, not a module; append `export {}` so two such
    // files can't collide on a global — appended at the end, it never shifts the
    // line of any real diagnostic.
    const isModule = /^\s*(import|export)\b/m.test(ex.code);
    fs.writeFileSync(
      path.join(BUILD_DIR, ex.fileName),
      ex.code + (isModule ? '' : '\nexport {};\n'),
    );
  }

  const tsconfig = {
    compilerOptions: {
      // A consumer-faithful, illustrative-code-friendly profile: strict enough
      // to catch real type drift, lax on the two rules that punish example code
      // (an import shown for context, an unused binding).
      target: 'ES2020',
      module: 'ESNext',
      moduleResolution: 'bundler',
      lib: ['ES2020', 'DOM', 'DOM.Iterable'],
      types: [],
      strict: true,
      skipLibCheck: true,
      esModuleInterop: true,
      resolveJsonModule: true,
      noEmit: true,
      noUnusedLocals: false,
      noUnusedParameters: false,
      // `paths` values are absolute, so no `baseUrl` is needed — and omitting it
      // sidesteps TS 6.0's `baseUrl` deprecation (TS5101).
      paths,
    },
    include: ['*.ts'],
  };
  fs.writeFileSync(
    path.join(BUILD_DIR, 'tsconfig.json'),
    JSON.stringify(tsconfig, null, 2),
  );
}

interface Diagnostic {
  file: string; // build-dir file name
  line: number;
  col: number;
  text: string; // full tsc line, from the code after the location
}

/** Parse `file.ts(line,col): error TSxxxx: message` lines from `tsc --pretty false`. */
function parseDiagnostics(output: string): Diagnostic[] {
  const diags: Diagnostic[] = [];
  const re = /^(.+?)\((\d+),(\d+)\):\s+(error|warning)\s+(TS\d+:.*)$/;
  for (const raw of output.split('\n')) {
    const m = raw.match(re);
    if (!m) continue;
    diags.push({
      file: path.basename(m[1]),
      line: Number(m[2]),
      col: Number(m[3]),
      text: `${m[4]} ${m[5]}`,
    });
  }
  return diags;
}

function runTsc(): { code: number; output: string } {
  const tscBin = require.resolve('typescript/bin/tsc');
  const res = spawnSync(
    process.execPath,
    [tscBin, '--noEmit', '--pretty', 'false', '-p', path.join(BUILD_DIR, 'tsconfig.json')],
    { cwd: BUILD_DIR, encoding: 'utf-8' },
  );
  return { code: res.status ?? 1, output: `${res.stdout ?? ''}${res.stderr ?? ''}` };
}

// ── Main ─────────────────────────────────────────────────────────────────────

function fail(message: string): never {
  console.error(`\n✗ ${message}\n`);
  process.exit(1);
}

function main() {
  console.log('🧪 Type-checking skill TypeScript examples...\n');

  const files = skillMarkdownFiles();
  const examples: Example[] = [];
  const orphans: string[] = [];
  for (const file of files) {
    const { examples: found, orphans: bad } = extractFromFile(file);
    examples.push(...found);
    for (const line of bad) orphans.push(`${rel(file)}:${line}`);
  }

  // A marker that is not directly above a ```ts fence checks nothing. Fail loudly
  // rather than let it read as covered — a placed-but-inert marker is worse than
  // no marker, because it looks intentional.
  if (orphans.length > 0) {
    fail(
      `Found ${MARKER} not directly above a \`\`\`ts / \`\`\`typescript fence:\n\n` +
        orphans.map((o) => `  - ${o}`).join('\n') +
        `\n\n  The marker must be the line IMMEDIATELY above the code fence (no blank\n` +
        `  line between). Move it, or remove it if the block should not be checked.`,
    );
  }

  // Vacuous-green guard: opt-in tagging means "zero blocks" is far more likely
  // to be "the marker got renamed / stripped" than "no examples worth checking".
  // A gate that checks nothing must not report success.
  if (examples.length === 0) {
    fail(
      `No skill examples are marked for type-checking.\n\n` +
        `  Mark a self-contained, compilable block by putting\n\n` +
        `    ${MARKER}\n\n` +
        `  on the line directly above its \`\`\`ts fence in a skills/**/*.md file.\n` +
        `  (If you just removed the last marker, that is almost certainly a mistake.)`,
    );
  }

  const { paths, missing } = specPaths();
  if (missing.some((m) => m.endsWith('index.d.ts')) && !fs.existsSync(paths['@objectstack/spec']?.[0] ?? '')) {
    fail(
      `@objectstack/spec is not built — no declarations to check examples against:\n\n` +
        missing.map((m) => `  - ${m} (missing)`).join('\n') +
        `\n\n  Build the spec first (CI does this in the "Build workspace packages" step):\n\n` +
        `    pnpm --filter @objectstack/spec build`,
    );
  }

  console.log(`   ${examples.length} marked example(s) across ${new Set(examples.map((e) => e.source)).size} file(s):`);
  for (const ex of examples) {
    console.log(`     • ${rel(ex.source)}:${ex.bodyStartLine} → ${ex.fileName}`);
  }
  console.log('');

  writeBuildDir(examples, paths);
  const { code, output } = runTsc();

  const byFile = new Map(examples.map((e) => [e.fileName, e]));
  const diags = parseDiagnostics(output);

  if (code === 0 && diags.length === 0) {
    console.log(`✅ ${examples.length} skill examples type-check against @objectstack/spec`);
    if (!KEEP) fs.rmSync(BUILD_DIR, { recursive: true, force: true });
    return;
  }

  // Remap every diagnostic back to skills/**/SKILL.md:<real line> so the author
  // reads the error against the file they actually edit, not the throwaway copy.
  console.error(`\n✗ Skill TypeScript examples do not compile against @objectstack/spec:\n`);
  const grouped = new Map<string, string[]>();
  for (const d of diags) {
    const ex = byFile.get(d.file);
    const loc = ex ? `${rel(ex.source)}:${ex.bodyStartLine + d.line - 1}:${d.col}` : d.file;
    const key = ex ? rel(ex.source) : d.file;
    if (!grouped.has(key)) grouped.set(key, []);
    grouped.get(key)!.push(`  ${loc}\n      ${d.text}`);
  }
  for (const [, entries] of grouped) {
    for (const e of entries) console.error(e);
  }

  // A non-zero exit with no parseable diagnostics (e.g. a tsconfig error) must
  // still surface — print the raw tail so it is never a silent failure.
  if (diags.length === 0) {
    console.error(`\n  tsc exited ${code} but produced no parseable diagnostics. Raw output:\n`);
    console.error(output.split('\n').map((l) => `    ${l}`).join('\n'));
  }

  console.error(
    `\n  These are examples an AI copies verbatim. Fix the example to match the\n` +
      `  current spec, or drop its ${MARKER} marker if it is an intentional fragment.\n`,
  );
  if (!KEEP) fs.rmSync(BUILD_DIR, { recursive: true, force: true });
  process.exit(1);
}

main();