|
| 1 | +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. |
| 2 | + |
| 3 | +/** |
| 4 | + * Check Skill Examples (anti-drift, #3094) |
| 5 | + * |
| 6 | + * The TypeScript examples inside `skills/` are the first thing an AI reads when |
| 7 | + * authoring ObjectStack metadata, yet nothing type-checks them. When the spec |
| 8 | + * renames an export or tightens a discriminated union, the examples silently rot |
| 9 | + * (`defineDataset` → `defineSeed`, the removed `unique`/`async` validation |
| 10 | + * types, kanban's top-level `groupBy`) and the platform's headline |
| 11 | + * AI-native surface starts teaching code that no longer compiles. A third party |
| 12 | + * following the skill hits the wall first. |
| 13 | + * |
| 14 | + * Full extraction of every ```ts block is infeasible: most are *fragments* |
| 15 | + * (a `columns: [...]` subtree, a `kanban: {...}` literal) that would need a |
| 16 | + * hand-authored wrapper to compile, and wrapping them yields high false-positive |
| 17 | + * noise. So this gate is **opt-in**: a self-contained, should-compile block is |
| 18 | + * marked by a `<!-- os:check -->` HTML comment on the line directly above its |
| 19 | + * fence. The marker is an inert comment (renders to nothing) and — crucially — |
| 20 | + * leaves the fence info-string a bare ` ```ts ` / ` ```typescript `, so the |
| 21 | + * existing `check:doc-authoring` scanner (which keys on `^```(ts|typescript|tsx)$`) |
| 22 | + * still sees the block. A fence-meta tag like ` ```ts check ` would have punched |
| 23 | + * a hole in that gate. |
| 24 | + * |
| 25 | + * Each marked block is written verbatim to a throwaway build dir and type-checked |
| 26 | + * with `tsc --noEmit` against the built `@objectstack/spec` declarations — the |
| 27 | + * exact surface a consumer's `import { … } from '@objectstack/spec'` resolves to. |
| 28 | + * Module resolution is wired via a `paths` map derived from the package's own |
| 29 | + * `exports` field, so it self-updates as the spec adds/removes subpath exports. |
| 30 | + * |
| 31 | + * Because it reads the built `dist/*.d.ts`, this runs AFTER the workspace build |
| 32 | + * step in CI — alongside `check:api-surface` / the example-app typecheck, its |
| 33 | + * fellow "real consumer" gates — not before it like `check:skill-refs`. |
| 34 | + * |
| 35 | + * Usage: |
| 36 | + * tsx scripts/check-skill-examples.ts # extract + type-check (CI) |
| 37 | + * tsx scripts/check-skill-examples.ts --keep # also leave the build dir for inspection |
| 38 | + */ |
| 39 | + |
| 40 | +import { spawnSync } from 'child_process'; |
| 41 | +import fs from 'fs'; |
| 42 | +import path from 'path'; |
| 43 | + |
| 44 | +// ── Paths ──────────────────────────────────────────────────────────────────── |
| 45 | + |
| 46 | +const REPO_ROOT = path.resolve(__dirname, '../../..'); |
| 47 | +const SPEC_DIR = path.resolve(__dirname, '..'); |
| 48 | +const SKILLS_DIR = path.resolve(REPO_ROOT, 'skills'); |
| 49 | +const BUILD_DIR = path.resolve(SPEC_DIR, '.examples-build'); |
| 50 | +const SPEC_PKG_JSON = path.resolve(SPEC_DIR, 'package.json'); |
| 51 | + |
| 52 | +/** Opt-in marker: the line directly above a fence opts that block into the gate. */ |
| 53 | +const MARKER = '<!-- os:check -->'; |
| 54 | + |
| 55 | +const KEEP = process.argv.includes('--keep'); |
| 56 | + |
| 57 | +const rel = (p: string) => path.relative(REPO_ROOT, p); |
| 58 | + |
| 59 | +// ── Extraction ─────────────────────────────────────────────────────────────── |
| 60 | + |
| 61 | +interface Example { |
| 62 | + /** Source markdown file (absolute). */ |
| 63 | + source: string; |
| 64 | + /** 1-based line in the source of the FIRST code line inside the fence. */ |
| 65 | + bodyStartLine: number; |
| 66 | + /** Raw fence body. */ |
| 67 | + code: string; |
| 68 | + /** Flat file name written into the build dir. */ |
| 69 | + fileName: string; |
| 70 | +} |
| 71 | + |
| 72 | +/** Every `*.md` under a skill folder — SKILL.md plus references/rules notes. */ |
| 73 | +function skillMarkdownFiles(): string[] { |
| 74 | + if (!fs.existsSync(SKILLS_DIR)) return []; |
| 75 | + const out: string[] = []; |
| 76 | + const walk = (dir: string) => { |
| 77 | + for (const e of fs.readdirSync(dir, { withFileTypes: true })) { |
| 78 | + const full = path.join(dir, e.name); |
| 79 | + if (e.isDirectory()) walk(full); |
| 80 | + else if (e.name.endsWith('.md')) out.push(full); |
| 81 | + } |
| 82 | + }; |
| 83 | + for (const e of fs.readdirSync(SKILLS_DIR, { withFileTypes: true })) { |
| 84 | + if (e.isDirectory()) walk(path.join(SKILLS_DIR, e.name)); |
| 85 | + } |
| 86 | + return out.sort(); |
| 87 | +} |
| 88 | + |
| 89 | +/** |
| 90 | + * Pull every marked ```ts / ```typescript block out of one markdown file. |
| 91 | + * A block is marked when the line immediately above its opening fence is |
| 92 | + * exactly the MARKER (ignoring surrounding whitespace). |
| 93 | + * |
| 94 | + * Also reports `orphans`: MARKER lines that are NOT directly above a ts fence. |
| 95 | + * A misplaced marker (a blank line slipped in between, or it precedes a bash / |
| 96 | + * json block) silently checks nothing — exactly the failure mode this gate |
| 97 | + * exists to prevent — so the caller treats an orphan as an error, not a no-op. |
| 98 | + */ |
| 99 | +function extractFromFile(source: string): { examples: Example[]; orphans: number[] } { |
| 100 | + const lines = fs.readFileSync(source, 'utf-8').split('\n'); |
| 101 | + const skillDir = path.relative(SKILLS_DIR, source).split(path.sep)[0]; |
| 102 | + const base = path.basename(source, '.md'); |
| 103 | + const examples: Example[] = []; |
| 104 | + const claimed = new Set<number>(); // MARKER line indices that opened a real block |
| 105 | + let n = 0; |
| 106 | + |
| 107 | + for (let i = 0; i < lines.length; i++) { |
| 108 | + const open = lines[i].match(/^```(ts|typescript)\s*$/); |
| 109 | + if (!open) continue; |
| 110 | + const marked = i > 0 && lines[i - 1].trim() === MARKER; |
| 111 | + // Find the matching close fence regardless of marking, so `i` advances past |
| 112 | + // this block and we never treat its body as top-level markdown. |
| 113 | + let close = i + 1; |
| 114 | + while (close < lines.length && !/^```\s*$/.test(lines[close])) close++; |
| 115 | + if (marked) { |
| 116 | + claimed.add(i - 1); |
| 117 | + const body = lines.slice(i + 1, close); |
| 118 | + n += 1; |
| 119 | + examples.push({ |
| 120 | + source, |
| 121 | + bodyStartLine: i + 2, // 1-based line of body[0] |
| 122 | + code: body.join('\n'), |
| 123 | + fileName: `${skillDir}__${base}__${n}.ts`, |
| 124 | + }); |
| 125 | + } |
| 126 | + i = close; // skip to the close fence |
| 127 | + } |
| 128 | + |
| 129 | + const orphans: number[] = []; |
| 130 | + for (let i = 0; i < lines.length; i++) { |
| 131 | + if (lines[i].trim() === MARKER && !claimed.has(i)) orphans.push(i + 1); // 1-based |
| 132 | + } |
| 133 | + return { examples, orphans }; |
| 134 | +} |
| 135 | + |
| 136 | +// ── Module resolution derived from the spec's own `exports` ────────────────── |
| 137 | + |
| 138 | +interface ExportEntry { |
| 139 | + import?: { types?: string }; |
| 140 | + require?: { types?: string }; |
| 141 | +} |
| 142 | + |
| 143 | +/** |
| 144 | + * Build a tsconfig `paths` map from `@objectstack/spec`'s published `exports`, |
| 145 | + * pointing each specifier at its built `.d.ts`. Deriving it from the real |
| 146 | + * exports means a new subpath export (or a removed one) is reflected here for |
| 147 | + * free — the map cannot drift from what consumers actually resolve. |
| 148 | + * |
| 149 | + * Returns the map plus the list of declaration files that must exist; a missing |
| 150 | + * root declaration means the spec was not built (or built with OS_SKIP_DTS), and |
| 151 | + * the caller fails loudly rather than checking against a stale/absent surface. |
| 152 | + */ |
| 153 | +function specPaths(): { paths: Record<string, string[]>; missing: string[] } { |
| 154 | + const pkg = JSON.parse(fs.readFileSync(SPEC_PKG_JSON, 'utf-8')); |
| 155 | + const exportsMap = pkg.exports as Record<string, ExportEntry | string>; |
| 156 | + const paths: Record<string, string[]> = {}; |
| 157 | + const missing: string[] = []; |
| 158 | + |
| 159 | + for (const [key, entry] of Object.entries(exportsMap)) { |
| 160 | + const types = |
| 161 | + typeof entry === 'string' |
| 162 | + ? entry |
| 163 | + : (entry.require?.types ?? entry.import?.types); |
| 164 | + if (!types || !types.endsWith('.d.ts')) continue; // skip non-type conditions (css, etc.) |
| 165 | + const specifier = key === '.' ? '@objectstack/spec' : `@objectstack/spec/${key.slice(2)}`; |
| 166 | + const abs = path.resolve(SPEC_DIR, types); |
| 167 | + paths[specifier] = [abs]; |
| 168 | + if (!fs.existsSync(abs)) missing.push(rel(abs)); |
| 169 | + } |
| 170 | + return { paths, missing }; |
| 171 | +} |
| 172 | + |
| 173 | +// ── tsc harness ────────────────────────────────────────────────────────────── |
| 174 | + |
| 175 | +function writeBuildDir(examples: Example[], paths: Record<string, string[]>): void { |
| 176 | + fs.rmSync(BUILD_DIR, { recursive: true, force: true }); |
| 177 | + fs.mkdirSync(BUILD_DIR, { recursive: true }); |
| 178 | + |
| 179 | + for (const ex of examples) { |
| 180 | + // Written verbatim (no prepended wrapper) so a tsc line N maps to source |
| 181 | + // line (bodyStartLine + N - 1) with zero arithmetic guesswork. A block with |
| 182 | + // no import/export is a script, not a module; append `export {}` so two such |
| 183 | + // files can't collide on a global — appended at the end, it never shifts the |
| 184 | + // line of any real diagnostic. |
| 185 | + const isModule = /^\s*(import|export)\b/m.test(ex.code); |
| 186 | + fs.writeFileSync( |
| 187 | + path.join(BUILD_DIR, ex.fileName), |
| 188 | + ex.code + (isModule ? '' : '\nexport {};\n'), |
| 189 | + ); |
| 190 | + } |
| 191 | + |
| 192 | + const tsconfig = { |
| 193 | + compilerOptions: { |
| 194 | + // A consumer-faithful, illustrative-code-friendly profile: strict enough |
| 195 | + // to catch real type drift, lax on the two rules that punish example code |
| 196 | + // (an import shown for context, an unused binding). |
| 197 | + target: 'ES2020', |
| 198 | + module: 'ESNext', |
| 199 | + moduleResolution: 'bundler', |
| 200 | + lib: ['ES2020', 'DOM', 'DOM.Iterable'], |
| 201 | + types: [], |
| 202 | + strict: true, |
| 203 | + skipLibCheck: true, |
| 204 | + esModuleInterop: true, |
| 205 | + resolveJsonModule: true, |
| 206 | + noEmit: true, |
| 207 | + noUnusedLocals: false, |
| 208 | + noUnusedParameters: false, |
| 209 | + // `paths` values are absolute, so no `baseUrl` is needed — and omitting it |
| 210 | + // sidesteps TS 6.0's `baseUrl` deprecation (TS5101). |
| 211 | + paths, |
| 212 | + }, |
| 213 | + include: ['*.ts'], |
| 214 | + }; |
| 215 | + fs.writeFileSync( |
| 216 | + path.join(BUILD_DIR, 'tsconfig.json'), |
| 217 | + JSON.stringify(tsconfig, null, 2), |
| 218 | + ); |
| 219 | +} |
| 220 | + |
| 221 | +interface Diagnostic { |
| 222 | + file: string; // build-dir file name |
| 223 | + line: number; |
| 224 | + col: number; |
| 225 | + text: string; // full tsc line, from the code after the location |
| 226 | +} |
| 227 | + |
| 228 | +/** Parse `file.ts(line,col): error TSxxxx: message` lines from `tsc --pretty false`. */ |
| 229 | +function parseDiagnostics(output: string): Diagnostic[] { |
| 230 | + const diags: Diagnostic[] = []; |
| 231 | + const re = /^(.+?)\((\d+),(\d+)\):\s+(error|warning)\s+(TS\d+:.*)$/; |
| 232 | + for (const raw of output.split('\n')) { |
| 233 | + const m = raw.match(re); |
| 234 | + if (!m) continue; |
| 235 | + diags.push({ |
| 236 | + file: path.basename(m[1]), |
| 237 | + line: Number(m[2]), |
| 238 | + col: Number(m[3]), |
| 239 | + text: `${m[4]} ${m[5]}`, |
| 240 | + }); |
| 241 | + } |
| 242 | + return diags; |
| 243 | +} |
| 244 | + |
| 245 | +function runTsc(): { code: number; output: string } { |
| 246 | + const tscBin = require.resolve('typescript/bin/tsc'); |
| 247 | + const res = spawnSync( |
| 248 | + process.execPath, |
| 249 | + [tscBin, '--noEmit', '--pretty', 'false', '-p', path.join(BUILD_DIR, 'tsconfig.json')], |
| 250 | + { cwd: BUILD_DIR, encoding: 'utf-8' }, |
| 251 | + ); |
| 252 | + return { code: res.status ?? 1, output: `${res.stdout ?? ''}${res.stderr ?? ''}` }; |
| 253 | +} |
| 254 | + |
| 255 | +// ── Main ───────────────────────────────────────────────────────────────────── |
| 256 | + |
| 257 | +function fail(message: string): never { |
| 258 | + console.error(`\n✗ ${message}\n`); |
| 259 | + process.exit(1); |
| 260 | +} |
| 261 | + |
| 262 | +function main() { |
| 263 | + console.log('🧪 Type-checking skill TypeScript examples...\n'); |
| 264 | + |
| 265 | + const files = skillMarkdownFiles(); |
| 266 | + const examples: Example[] = []; |
| 267 | + const orphans: string[] = []; |
| 268 | + for (const file of files) { |
| 269 | + const { examples: found, orphans: bad } = extractFromFile(file); |
| 270 | + examples.push(...found); |
| 271 | + for (const line of bad) orphans.push(`${rel(file)}:${line}`); |
| 272 | + } |
| 273 | + |
| 274 | + // A marker that is not directly above a ```ts fence checks nothing. Fail loudly |
| 275 | + // rather than let it read as covered — a placed-but-inert marker is worse than |
| 276 | + // no marker, because it looks intentional. |
| 277 | + if (orphans.length > 0) { |
| 278 | + fail( |
| 279 | + `Found ${MARKER} not directly above a \`\`\`ts / \`\`\`typescript fence:\n\n` + |
| 280 | + orphans.map((o) => ` - ${o}`).join('\n') + |
| 281 | + `\n\n The marker must be the line IMMEDIATELY above the code fence (no blank\n` + |
| 282 | + ` line between). Move it, or remove it if the block should not be checked.`, |
| 283 | + ); |
| 284 | + } |
| 285 | + |
| 286 | + // Vacuous-green guard: opt-in tagging means "zero blocks" is far more likely |
| 287 | + // to be "the marker got renamed / stripped" than "no examples worth checking". |
| 288 | + // A gate that checks nothing must not report success. |
| 289 | + if (examples.length === 0) { |
| 290 | + fail( |
| 291 | + `No skill examples are marked for type-checking.\n\n` + |
| 292 | + ` Mark a self-contained, compilable block by putting\n\n` + |
| 293 | + ` ${MARKER}\n\n` + |
| 294 | + ` on the line directly above its \`\`\`ts fence in a skills/**/*.md file.\n` + |
| 295 | + ` (If you just removed the last marker, that is almost certainly a mistake.)`, |
| 296 | + ); |
| 297 | + } |
| 298 | + |
| 299 | + const { paths, missing } = specPaths(); |
| 300 | + if (missing.some((m) => m.endsWith('index.d.ts')) && !fs.existsSync(paths['@objectstack/spec']?.[0] ?? '')) { |
| 301 | + fail( |
| 302 | + `@objectstack/spec is not built — no declarations to check examples against:\n\n` + |
| 303 | + missing.map((m) => ` - ${m} (missing)`).join('\n') + |
| 304 | + `\n\n Build the spec first (CI does this in the "Build workspace packages" step):\n\n` + |
| 305 | + ` pnpm --filter @objectstack/spec build`, |
| 306 | + ); |
| 307 | + } |
| 308 | + |
| 309 | + console.log(` ${examples.length} marked example(s) across ${new Set(examples.map((e) => e.source)).size} file(s):`); |
| 310 | + for (const ex of examples) { |
| 311 | + console.log(` • ${rel(ex.source)}:${ex.bodyStartLine} → ${ex.fileName}`); |
| 312 | + } |
| 313 | + console.log(''); |
| 314 | + |
| 315 | + writeBuildDir(examples, paths); |
| 316 | + const { code, output } = runTsc(); |
| 317 | + |
| 318 | + const byFile = new Map(examples.map((e) => [e.fileName, e])); |
| 319 | + const diags = parseDiagnostics(output); |
| 320 | + |
| 321 | + if (code === 0 && diags.length === 0) { |
| 322 | + console.log(`✅ ${examples.length} skill examples type-check against @objectstack/spec`); |
| 323 | + if (!KEEP) fs.rmSync(BUILD_DIR, { recursive: true, force: true }); |
| 324 | + return; |
| 325 | + } |
| 326 | + |
| 327 | + // Remap every diagnostic back to skills/**/SKILL.md:<real line> so the author |
| 328 | + // reads the error against the file they actually edit, not the throwaway copy. |
| 329 | + console.error(`\n✗ Skill TypeScript examples do not compile against @objectstack/spec:\n`); |
| 330 | + const grouped = new Map<string, string[]>(); |
| 331 | + for (const d of diags) { |
| 332 | + const ex = byFile.get(d.file); |
| 333 | + const loc = ex ? `${rel(ex.source)}:${ex.bodyStartLine + d.line - 1}:${d.col}` : d.file; |
| 334 | + const key = ex ? rel(ex.source) : d.file; |
| 335 | + if (!grouped.has(key)) grouped.set(key, []); |
| 336 | + grouped.get(key)!.push(` ${loc}\n ${d.text}`); |
| 337 | + } |
| 338 | + for (const [, entries] of grouped) { |
| 339 | + for (const e of entries) console.error(e); |
| 340 | + } |
| 341 | + |
| 342 | + // A non-zero exit with no parseable diagnostics (e.g. a tsconfig error) must |
| 343 | + // still surface — print the raw tail so it is never a silent failure. |
| 344 | + if (diags.length === 0) { |
| 345 | + console.error(`\n tsc exited ${code} but produced no parseable diagnostics. Raw output:\n`); |
| 346 | + console.error(output.split('\n').map((l) => ` ${l}`).join('\n')); |
| 347 | + } |
| 348 | + |
| 349 | + console.error( |
| 350 | + `\n These are examples an AI copies verbatim. Fix the example to match the\n` + |
| 351 | + ` current spec, or drop its ${MARKER} marker if it is an intentional fragment.\n`, |
| 352 | + ); |
| 353 | + if (!KEEP) fs.rmSync(BUILD_DIR, { recursive: true, force: true }); |
| 354 | + process.exit(1); |
| 355 | +} |
| 356 | + |
| 357 | +main(); |
0 commit comments