|
| 1 | +// ───────────────────────────────────────────────────────────────────────────── |
| 2 | +// scope-gap.ts — UNIFIED "true gap" harness. Score ANY language's OFFICIAL TextMate |
| 3 | +// grammar AND Monogram's derived grammar against the PARSER as the neutral oracle, and |
| 4 | +// report the gap. Generalizes highlight-bench.ts: a per-language adapter supplies a |
| 5 | +// roleOracle (parser → per-token structural ROLE) + the two grammars; this core tokenizes |
| 6 | +// both with vscode-textmate, grades each oracle token's scope via the FROZEN neutral table |
| 7 | +// in scope-roles.ts, and reports official% vs Monogram% correctness + the gap + the |
| 8 | +// divergent tokens (only-Monogram-correct / only-official-correct). |
| 9 | +// |
| 10 | +// Anchored on microsoft/vscode#203212: VS Code's official grammars for these languages are |
| 11 | +// unmaintained textmate/*.tmbundle repos; the maintained PARSER is the authority. This is |
| 12 | +// the comparative "true gap" — how far each grammar is from the parser — on ONE scale, for |
| 13 | +// ANY language with a parser-role oracle (TS via oracle.ts; YAML/HTML/… plug in their own). |
| 14 | +// ───────────────────────────────────────────────────────────────────────────── |
| 15 | +import { readFileSync, existsSync } from 'node:fs'; |
| 16 | +import { createRequire } from 'node:module'; |
| 17 | +import vsctm from 'vscode-textmate'; |
| 18 | +import onig from 'vscode-oniguruma'; |
| 19 | +import { gradeScope, isCorrect, ROLE_SPEC } from './scope-roles.ts'; |
| 20 | +import type { RoleName } from './scope-roles.ts'; |
| 21 | + |
| 22 | +// A parser-assigned token role over a span. The per-language oracle returns these (e.g. |
| 23 | +// oracle.ts for TS/JS). Only start/end/text/role are required by the harness. |
| 24 | +export interface GoldToken { start: number; end: number; text: string; role: RoleName } |
| 25 | + |
| 26 | +export interface ScopeGapAdapter { |
| 27 | + name: string; |
| 28 | + scopeName: string; // grammar scope, e.g. 'source.ts' |
| 29 | + officialPath: string; // the official .tmLanguage.json (the #203212 bundle) |
| 30 | + monogramPath: string; // Monogram's derived .tmLanguage.json |
| 31 | + loadCorpus: () => { name: string; text: string }[]; |
| 32 | + roleOracle: (text: string) => GoldToken[]; // parser → per-token role (the neutral oracle) |
| 33 | + isGradable?: (text: string) => boolean; // skip inputs the oracle can't judge (default: all) |
| 34 | +} |
| 35 | + |
| 36 | +const { INITIAL, Registry, parseRawGrammar } = vsctm; |
| 37 | +const { loadWASM, OnigScanner, OnigString } = onig; |
| 38 | + |
| 39 | +let wasmReady: Promise<unknown> | null = null; |
| 40 | +function ensureWasm(): Promise<unknown> { |
| 41 | + if (!wasmReady) { |
| 42 | + const require = createRequire(import.meta.url); |
| 43 | + const bin = readFileSync(require.resolve('vscode-oniguruma/release/onig.wasm')); |
| 44 | + wasmReady = loadWASM(bin.buffer.slice(bin.byteOffset, bin.byteOffset + bin.byteLength)); |
| 45 | + } |
| 46 | + return wasmReady; |
| 47 | +} |
| 48 | +function loadGrammar(scopeName: string, path: string) { |
| 49 | + const content = readFileSync(path, 'utf-8'); |
| 50 | + const reg = new Registry({ |
| 51 | + onigLib: Promise.resolve({ |
| 52 | + createOnigScanner: (p: string[]) => new OnigScanner(p), |
| 53 | + createOnigString: (s: string) => new OnigString(s), |
| 54 | + }), |
| 55 | + loadGrammar: async (sn: string) => (sn === scopeName ? parseRawGrammar(content, 'g.json') : null), |
| 56 | + }); |
| 57 | + return reg.loadGrammar(scopeName); |
| 58 | +} |
| 59 | + |
| 60 | +interface TmToken { start: number; end: number; scope: string } |
| 61 | +function tmTokenize(grammar: vsctm.IGrammar, text: string): TmToken[] { |
| 62 | + const lines = text.split('\n'); |
| 63 | + const toks: TmToken[] = []; |
| 64 | + let ruleStack = INITIAL, offset = 0; |
| 65 | + for (const line of lines) { |
| 66 | + const r = grammar.tokenizeLine(line, ruleStack); |
| 67 | + for (const t of r.tokens) toks.push({ start: offset + t.startIndex, end: offset + t.endIndex, scope: t.scopes[t.scopes.length - 1] }); |
| 68 | + ruleStack = r.ruleStack; offset += line.length + 1; |
| 69 | + } |
| 70 | + return toks; |
| 71 | +} |
| 72 | +// deepest scope of the TM token covering `pos` (binary search; '' if none) |
| 73 | +function scopeAt(toks: TmToken[], pos: number): string { |
| 74 | + let lo = 0, hi = toks.length - 1, ans = -1; |
| 75 | + while (lo <= hi) { const mid = (lo + hi) >> 1; if (toks[mid].start <= pos) { ans = mid; lo = mid + 1; } else hi = mid - 1; } |
| 76 | + return ans >= 0 && toks[ans].end > pos ? toks[ans].scope : ''; |
| 77 | +} |
| 78 | + |
| 79 | +export async function run(adapter: ScopeGapAdapter): Promise<void> { |
| 80 | + if (!existsSync(adapter.officialPath)) { console.error(`Official grammar not found:\n ${adapter.officialPath}`); process.exit(1); } |
| 81 | + if (!existsSync(adapter.monogramPath)) { console.error(`Monogram grammar not found: ${adapter.monogramPath} (run: node src/cli.ts ${adapter.name}.ts)`); process.exit(1); } |
| 82 | + await ensureWasm(); |
| 83 | + const official = await loadGrammar(adapter.scopeName, adapter.officialPath); |
| 84 | + const monogram = await loadGrammar(adapter.scopeName, adapter.monogramPath); |
| 85 | + if (!official || !monogram) throw new Error('failed to load a grammar'); |
| 86 | + |
| 87 | + const corpus = adapter.loadCorpus(); |
| 88 | + const gradable = adapter.isGradable ?? (() => true); |
| 89 | + |
| 90 | + let nFiles = 0; |
| 91 | + const tally = { oCorrect: 0, oExact: 0, mCorrect: 0, mExact: 0, total: 0 }; |
| 92 | + const perRole = new Map<RoleName, { n: number; oC: number; mC: number }>(); |
| 93 | + const onlyMono: { text: string; role: RoleName; o: string; m: string }[] = []; |
| 94 | + const onlyOff: { text: string; role: RoleName; o: string; m: string }[] = []; |
| 95 | + // per-snippet: did the grammar get EVERY graded token right in this input? |
| 96 | + const snip = { o: 0, m: 0, n: 0 }; |
| 97 | + |
| 98 | + for (const { text } of corpus) { |
| 99 | + if (!gradable(text)) continue; |
| 100 | + let gold: GoldToken[], tmO: TmToken[], tmM: TmToken[]; |
| 101 | + try { gold = adapter.roleOracle(text); tmO = tmTokenize(official, text); tmM = tmTokenize(monogram, text); } catch { continue; } |
| 102 | + nFiles++; |
| 103 | + let okO = true, okM = true, gradedHere = 0; |
| 104 | + for (const t of gold) { |
| 105 | + const tier = ROLE_SPEC[t.role]?.tier; |
| 106 | + if (!tier || tier === 'lexical') continue; // lexical floor: excluded from the headline |
| 107 | + const so = scopeAt(tmO, t.start), sm = scopeAt(tmM, t.start); |
| 108 | + const oc = isCorrect(gradeScope(t.role, so)), mc = isCorrect(gradeScope(t.role, sm)); |
| 109 | + tally.total++; gradedHere++; |
| 110 | + if (oc) tally.oCorrect++; if (gradeScope(t.role, so) === 'exact') tally.oExact++; |
| 111 | + if (mc) tally.mCorrect++; if (gradeScope(t.role, sm) === 'exact') tally.mExact++; |
| 112 | + const pr = perRole.get(t.role) ?? { n: 0, oC: 0, mC: 0 }; pr.n++; if (oc) pr.oC++; if (mc) pr.mC++; perRole.set(t.role, pr); |
| 113 | + if (!oc) okO = false; if (!mc) okM = false; |
| 114 | + if (mc && !oc && onlyMono.length < 40) onlyMono.push({ text: t.text, role: t.role, o: so || '(none)', m: sm || '(none)' }); |
| 115 | + if (oc && !mc && onlyOff.length < 40) onlyOff.push({ text: t.text, role: t.role, o: so || '(none)', m: sm || '(none)' }); |
| 116 | + } |
| 117 | + if (gradedHere) { snip.n++; if (okO) snip.o++; if (okM) snip.m++; } |
| 118 | + } |
| 119 | + |
| 120 | + const pct = (n: number, d = tally.total) => (d ? (100 * n / d).toFixed(1) : 'n/a'); |
| 121 | + const gap = tally.total ? (100 * (tally.mCorrect - tally.oCorrect) / tally.total).toFixed(1) : 'n/a'; |
| 122 | + console.log('='.repeat(78)); |
| 123 | + console.log(` Scope-gap vs the PARSER oracle — ${adapter.name} (vscode#203212)`); |
| 124 | + console.log(` official: ${adapter.officialPath.replace(/^.*\//, '')} monogram: ${adapter.monogramPath}`); |
| 125 | + console.log('='.repeat(78)); |
| 126 | + console.log(` ${nFiles} files · ${tally.total} graded tokens (lexical-floor roles excluded)`); |
| 127 | + console.log(` OFFICIAL correct ${pct(tally.oCorrect)}% (exact ${pct(tally.oExact)}%)`); |
| 128 | + console.log(` MONOGRAM correct ${pct(tally.mCorrect)}% (exact ${pct(tally.mExact)}%)`); |
| 129 | + console.log(` ══ GAP (Monogram − official) = ${gap} pts ══`); |
| 130 | + console.log(` per-snippet all-tokens-correct: official ${pct(snip.o, snip.n)}% monogram ${pct(snip.m, snip.n)}% (n=${snip.n})`); |
| 131 | + |
| 132 | + const rows = [...perRole.entries()] |
| 133 | + .map(([role, r]) => ({ role, n: r.n, o: r.oC, m: r.mC, d: r.mC - r.oC })) |
| 134 | + .filter((r) => r.o !== r.m).sort((a, b) => Math.abs(b.d) - Math.abs(a.d)); |
| 135 | + if (rows.length) { |
| 136 | + console.log(`\n per-role differences (correct official→monogram / occurrences):`); |
| 137 | + for (const r of rows.slice(0, 15)) console.log(` ${r.role.padEnd(16)} ${String(r.o).padStart(6)} →${String(r.m).padStart(6)} / ${r.n} ${r.d > 0 ? '+' : ''}${r.d}`); |
| 138 | + } |
| 139 | + console.log(`\n only-Monogram-correct tokens (official wrong vs the parser) — ${onlyMono.length} shown:`); |
| 140 | + for (const x of onlyMono.slice(0, 12)) console.log(` «${x.text.slice(0, 18)}» ${x.role}: official «${x.o}» → monogram «${x.m}»`); |
| 141 | + if (onlyOff.length) { |
| 142 | + console.log(`\n only-official-correct tokens (Monogram wrong) — ${onlyOff.length} shown:`); |
| 143 | + for (const x of onlyOff.slice(0, 12)) console.log(` «${x.text.slice(0, 18)}» ${x.role}: official «${x.o}» → monogram «${x.m}»`); |
| 144 | + } |
| 145 | + console.log('\nDone.'); |
| 146 | +} |
0 commit comments