Skip to content

Commit ed0f22e

Browse files
committed
docs: auto-generate the per-grammar alignment table in the README Status section
test/coverage-table.ts runs each src-coverage + scope-gap adapter, parses their ##COV## / ##SCOPEGAP## summary lines, and writes a compact per-grammar alignment table into the README Status section (between <!-- coverage:start/end -->). Both harnesses now emit a machine-readable summary line. Regenerate with: node test/coverage-table.ts --write (needs the corpora + a VS Code install for the official grammars; TS/JS use deterministic stride samples). Replaces the standalone alignment section — the numbers now live in Status next to the per-language descriptions. Fixed a ReferenceError (nFiles -> results.length) in the src-coverage summary line.
1 parent 86759d0 commit ed0f22e

4 files changed

Lines changed: 96 additions & 0 deletions

File tree

README.md

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,19 @@ Write a language's grammar **once**, as an executable definition. Monogram runs
1111
- **HTML** ([`html.ts`](html.ts)) — the engine reaching *past token streams into markup*; ~95 lines, validated against [`parse5`](https://github.com/inikulin/parse5).
1212
- **Vue** ([`vue.ts`](vue.ts)) — a dialect of `html.ts`: SFC blocks that embed Monogram's own TS/JS/CSS, plus directives and `{{ }}` interpolation.
1313

14+
<!-- coverage:start -->
15+
Per-grammar alignment vs the **official parser** as the neutral oracle (`node test/coverage-table.ts --write`). *Parser* = Monogram's parser vs the official parser: `branch` = source-coverage-anchored branch alignment, `agree` = bidirectional accept/reject (tree-equality for structural oracles) — `test/src-coverage.ts`. *Highlighter* = Monogram's derived TextMate grammar vs the official one, both graded against the parser's token roles — `test/scope-gap.ts`, the [vscode#203212](https://github.com/microsoft/vscode/issues/203212) comparison.
16+
17+
| Grammar | Parser (branch · agree) | Highlighter — Monogram vs official |
18+
|---|---|---|
19+
| TypeScript | 97.7% · 97.1% | 99.3% vs 99.0% |
20+
| JavaScript | 97.3% · 92.2% ||
21+
| JSX | 100.0% · 97.1% ||
22+
| TSX | 99.4% · 96.7% ||
23+
| HTML | 84.3% · 77.9% | 100.0% vs 97.6% |
24+
| YAML | 83.0% · 63.1% | 46.5% vs 89.7% |
25+
<!-- coverage:end -->
26+
1427
## Quick start
1528

1629
Requires Node 24+ (runs `.ts` directly — no build step, no `tsx`).
@@ -175,6 +188,7 @@ _Each hand-written **official** grammar vs Monogram's **derived** one, on the bu
175188

176189
<sub>A sampled ledger of real tracker issues, not an exhaustive audit. Run `npm run bench:issues` to regenerate (needs the official grammars: VS Code's installed TS/JS/HTML, and the Vue fixtures — see [`test/vue-bench.ts`](test/vue-bench.ts)). Sources: [`test/issue-cases.ts`](test/issue-cases.ts), [`test/html-issue-cases.ts`](test/html-issue-cases.ts), [`test/vue-issue-cases.ts`](test/vue-issue-cases.ts).</sub>
177190

191+
178192
### The ceiling — and the bar for claiming it
179193

180194
Deriving from a proven parser wins the disambiguation that is *TextMate-expressible but infeasible to hand-write* — regex-vs-division, generic-vs-comparison, whitespace-fragile multiline generics — the **only-Monogram** column. The **both-miss** cases are ones neither grammar gets *today* — not, by default, ones TextMate *can't*.

test/coverage-table.ts

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
// coverage-table.ts — regenerate the README per-language alignment tables. Runs each
2+
// src-coverage + scope-gap adapter as a subprocess, parses its ##COV## / ##SCOPEGAP## summary
3+
// line, and writes two tables into the README between <!-- coverage:start --> / <!-- coverage:end -->.
4+
// node test/coverage-table.ts # print the tables (don't touch the README)
5+
// node test/coverage-table.ts --write # rewrite the README region
6+
// Needs the corpora the adapters use (/tmp/ts-repo, /tmp/test262, /tmp/yaml-test-suite) and a VS
7+
// Code install for the official grammars; an adapter whose inputs are missing renders as "—".
8+
import { execFileSync } from 'node:child_process';
9+
import { readFileSync, writeFileSync } from 'node:fs';
10+
11+
const WRITE = process.argv.includes('--write');
12+
13+
function runAdapter(script: string, args: string[], marker: string): any | null {
14+
try {
15+
const out = execFileSync('node', [script, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], maxBuffer: 128 * 1024 * 1024 });
16+
const line = out.split('\n').find((l) => l.startsWith(marker));
17+
return line ? JSON.parse(line.slice(marker.length).trim()) : null;
18+
} catch { return null; }
19+
}
20+
21+
// TS/JS use deterministic stride subsets for speed; the rest run their full corpus.
22+
const COV = [
23+
{ lang: 'TypeScript', script: 'test/src-coverage-ts.ts', args: ['1500'] },
24+
{ lang: 'JavaScript', script: 'test/src-coverage-js.ts', args: ['800'] },
25+
{ lang: 'JSX', script: 'test/src-coverage-jsx.ts', args: [] },
26+
{ lang: 'TSX', script: 'test/src-coverage-tsx.ts', args: [] },
27+
{ lang: 'HTML', script: 'test/src-coverage-html.ts', args: [] },
28+
{ lang: 'YAML', script: 'test/src-coverage-yaml.ts', args: [] },
29+
];
30+
const GAP = [
31+
{ lang: 'TypeScript', script: 'test/scope-gap-ts.ts', args: ['800'] },
32+
{ lang: 'HTML', script: 'test/scope-gap-html.ts', args: [] },
33+
{ lang: 'YAML', script: 'test/scope-gap-yaml.ts', args: [] },
34+
];
35+
36+
const pct = (v: number | null | undefined) => (v == null ? '—' : v.toFixed(1) + '%');
37+
38+
console.error('Running src-coverage adapters…');
39+
const covRows = COV.map((a) => { console.error(' ' + a.lang); return { lang: a.lang, r: runAdapter(a.script, a.args, '##COV##') }; });
40+
console.error('Running scope-gap adapters…');
41+
const gapRows = GAP.map((a) => { console.error(' ' + a.lang); return { lang: a.lang, r: runAdapter(a.script, a.args, '##SCOPEGAP##') }; });
42+
43+
const covBy = new Map(covRows.map((x) => [x.lang, x.r]));
44+
const gapBy = new Map(gapRows.map((x) => [x.lang, x.r]));
45+
const LANGS = ['TypeScript', 'JavaScript', 'JSX', 'TSX', 'HTML', 'YAML'];
46+
47+
let md = '';
48+
md += "Per-grammar alignment vs the **official parser** as the neutral oracle (`node test/coverage-table.ts --write`). *Parser* = Monogram's parser vs the official parser: `branch` = source-coverage-anchored branch alignment, `agree` = bidirectional accept/reject (tree-equality for structural oracles) — `test/src-coverage.ts`. *Highlighter* = Monogram's derived TextMate grammar vs the official one, both graded against the parser's token roles — `test/scope-gap.ts`, the [vscode#203212](https://github.com/microsoft/vscode/issues/203212) comparison.\n\n";
49+
md += '| Grammar | Parser (branch · agree) | Highlighter — Monogram vs official |\n|---|---|---|\n';
50+
for (const lang of LANGS) {
51+
const c = covBy.get(lang), g = gapBy.get(lang);
52+
const parser = c ? `${pct(c.denoms?.[c.denoms.length - 1]?.alignment)} · ${pct(c.agreePct)}` : '—';
53+
const hl = g ? `${pct(g.monogramPct)} vs ${pct(g.officialPct)}` : '—';
54+
md += `| ${lang} | ${parser} | ${hl} |\n`;
55+
}
56+
57+
const block = '<!-- coverage:start -->\n' + md + '<!-- coverage:end -->';
58+
if (!WRITE) { console.log('\n' + md); process.exit(0); }
59+
60+
const README = 'README.md';
61+
let txt = readFileSync(README, 'utf8');
62+
if (!/<!-- coverage:start -->[\s\S]*?<!-- coverage:end -->/.test(txt)) {
63+
console.error('No <!-- coverage:start/end --> markers in README.md — add them first.');
64+
process.exit(1);
65+
}
66+
txt = txt.replace(/<!-- coverage:start -->[\s\S]*?<!-- coverage:end -->/, () => block);
67+
writeFileSync(README, txt);
68+
console.error('✓ README coverage region updated.');

test/scope-gap.ts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -153,5 +153,11 @@ export async function run(adapter: ScopeGapAdapter): Promise<void> {
153153
console.log(`\n only-official-correct tokens (Monogram wrong) — ${onlyOff.length} shown:`);
154154
for (const x of onlyOff.slice(0, 12)) console.log(` «${x.text.slice(0, 18)}» ${x.role}: official «${x.o}» → monogram «${x.m}»`);
155155
}
156+
// Machine-readable summary for the README coverage-table generator (test/coverage-table.ts).
157+
console.log('##SCOPEGAP## ' + JSON.stringify({
158+
name: adapter.name, official: adapter.officialPath.replace(/^.*\//, ''), tokens: tally.total,
159+
officialPct: tally.total ? (100 * tally.oCorrect) / tally.total : null,
160+
monogramPct: tally.total ? (100 * tally.mCorrect) / tally.total : null,
161+
}));
156162
console.log('\nDone.');
157163
}

test/src-coverage.ts

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,7 @@ export async function run(adapter: Adapter): Promise<void> {
157157
adapter.renderHeader?.(results, corpus);
158158

159159
const ledgerTop = adapter.ledgerTop ?? 15;
160+
const denomSummary: { label: string; alignment: number; completeness: number }[] = [];
160161
for (const den of adapter.denominators) {
161162
let totalSeen = 0, reachable = 0, agreed = 0, disagreed = 0;
162163
const dis: BranchState[] = [];
@@ -171,6 +172,7 @@ export async function run(adapter: Adapter): Promise<void> {
171172
const uncovered = totalSeen - reachable;
172173
const alignment = reachable ? (agreed / reachable) * 100 : 0;
173174
const completeness = totalSeen ? (reachable / totalSeen) * 100 : 0;
175+
denomSummary.push({ label: den.label, alignment, completeness });
174176
console.log(`\n────── denominator: ${den.label} ──────`);
175177
console.log(` total branches seen (reachable+uncovered) : ${totalSeen}`);
176178
console.log(` reachable (hit >=1 file) : ${reachable}`);
@@ -194,5 +196,11 @@ export async function run(adapter: Adapter): Promise<void> {
194196
}
195197

196198
adapter.renderFooter?.(results, corpus);
199+
// Machine-readable summary line for the README coverage-table generator (test/coverage-table.ts).
200+
console.log('##COV## ' + JSON.stringify({
201+
name: adapter.name, oracle: adapter.oracle, files: results.length,
202+
agreePct: results.length ? (100 * agreeCount) / results.length : null,
203+
denoms: denomSummary,
204+
}));
197205
console.log('\nDone.');
198206
}

0 commit comments

Comments
 (0)