Generative testing: grammar-derived inputs + by-construction consistency (#25) (#28)

Author: johnsoncodehk

.github/workflows/ci.yml

@@ -57,6 +57,15 @@ jobs:
           node test/vue-interp-expr.ts
           node test/yaml-issue12-regressions.ts
           node test/yaml-depth-witnesses.ts
+          node test/generative.ts
+          # The gap ledger is the deterministic, oracle-classified record of the divergences the
+          # generative check DISCOVERS. Its self-test asserts the ddmin keep-path + the oracle
+          # drop-path + determinism; --check fails if the committed KNOWN-GAPS.md is stale (the
+          # ledger is a pure function of the grammar, so it must be regenerated when a grammar
+          # changes — `npm run gap-ledger`). This is the deterministic source of truth; a later
+          # layer can turn rows into issues, but the committed artifact is gated here first.
+          node test/gap-ledger-selftest.ts
+          node test/gap-ledger.ts --check
 
   # The derived tree-sitter highlighter is the strongest thesis proof (a real GLR
   # parser from the same grammar, beating the official hand-written one). Build its

KNOWN-GAPS.md

@@ -0,0 +1,62 @@
+# KNOWN-GAPS — Monogram flat-highlighter divergences (auto-generated)
+
+<!-- AUTO-GENERATED by `node test/gap-ledger.ts --write`. Do not edit by hand. -->
+
+A **gap** is a position where, on **valid input** (accepted by the language’s external
+authority — typescript / yaml / parse5), the **flat TextMate highlighter** paints a token a
+different visual role than the **Monogram parser** assigns it by construction. These are the
+floor-blind divergences the generative scope≡role check (`test/generative.ts`) DISCOVERS over
+grammar-derived inputs — the monogram#23/#24 class — which the corpus-bound scope-gap metric is
+blind to (a small/clean corpus may never contain the shape, and the role-graded metric ignores
+punctuation-floor mis-paints). Each gap’s input is **minimized** (delta-debugged to a minimal
+repro that still parses and still diverges) and **fingerprinted** (a content hash, stable across
+commits) so the ledger is deterministic and commit-trackable.
+
+Regenerate: `node test/gap-ledger.ts --write` · verify up-to-date: `node test/gap-ledger.ts --check`.
+
+**2 gaps** across 7 grammars · 0 dropped.
+
+## `525e867dc205` — html: #24 structural-literal→content
+
+- **Language:** html
+- **Minimal repro:** `<A A=""/>`
+- **Divergent token:** `/` (parser token `$punct`)
+- **Role vs scope:** want **punct**, got **string** (highlighter scope `string.unquoted.html`)
+- **Fingerprint:** `525e867dc205`
+
+```json
+{
+  "id": "525e867dc205",
+  "language": "html",
+  "kind": "#24 structural-literal→content",
+  "repro": "<A A=\"\"/>",
+  "tokenType": "$punct",
+  "tokenText": "/",
+  "want": "punct",
+  "got": "string",
+  "gotScope": "string.unquoted.html"
+}
+```
+
+## `85c793d02a86` — vue: #24 structural-literal→content
+
+- **Language:** vue
+- **Minimal repro:** `<A A=""/>`
+- **Divergent token:** `/` (parser token `$punct`)
+- **Role vs scope:** want **punct**, got **string** (highlighter scope `string.unquoted.vue`)
+- **Fingerprint:** `85c793d02a86`
+
+```json
+{
+  "id": "85c793d02a86",
+  "language": "vue",
+  "kind": "#24 structural-literal→content",
+  "repro": "<A A=\"\"/>",
+  "tokenType": "$punct",
+  "tokenText": "/",
+  "want": "punct",
+  "got": "string",
+  "gotScope": "string.unquoted.vue"
+}
+```
+

package.json

@@ -5,6 +5,10 @@
   "scripts": {
     "gen": "node src/cli.ts typescript.ts && node src/cli.ts javascript.ts && node src/cli.ts typescriptreact.ts && node src/cli.ts javascriptreact.ts && node src/cli.ts html.ts && node src/cli.ts vue.ts && node src/cli.ts yaml.ts",
     "test": "node test/sanity-check.ts",
+    "generative": "node test/generative.ts",
+    "gap-ledger": "node test/gap-ledger.ts --write",
+    "gap-ledger:check": "node test/gap-ledger.ts --check",
+    "gap-ledger:selftest": "node test/gap-ledger-selftest.ts",
     "conformance": "node test/run-conformance.ts",
     "conformance:js": "node test/js-conformance.ts",
     "conformance:tsx": "node test/tsx-conformance.ts",
@@ -30,19 +34,19 @@
     "bench:perf": "node test/perf-bench.ts",
     "coverage": "node test/scope-coverage.ts",
     "compat": "node test/repo-compat.ts",
-    "src-coverage:ts": "node test/src-coverage-ts.ts",
-    "src-coverage:js": "node test/src-coverage-js.ts",
-    "src-coverage:jsx": "node test/src-coverage-jsx.ts",
-    "src-coverage:tsx": "node test/src-coverage-tsx.ts",
-    "src-coverage:html": "node test/src-coverage-html.ts",
-    "src-coverage:yaml": "node test/src-coverage-yaml.ts",
-    "scope-gap:ts": "node test/scope-gap-ts.ts",
-    "scope-gap:js": "node test/scope-gap-js.ts",
-    "scope-gap:jsx": "node test/scope-gap-jsx.ts",
-    "scope-gap:tsx": "node test/scope-gap-tsx.ts",
-    "scope-gap:html": "node test/scope-gap-html.ts",
-    "scope-gap:yaml": "node test/scope-gap-yaml.ts",
-    "scope-gap:vue": "node test/scope-gap-vue.ts",
+    "src-coverage:ts": "node test/src-coverage-run.ts ts",
+    "src-coverage:js": "node test/src-coverage-run.ts js",
+    "src-coverage:jsx": "node test/src-coverage-run.ts jsx",
+    "src-coverage:tsx": "node test/src-coverage-run.ts tsx",
+    "src-coverage:html": "node test/src-coverage-run.ts html",
+    "src-coverage:yaml": "node test/src-coverage-run.ts yaml",
+    "scope-gap:ts": "node test/scope-gap-run.ts ts",
+    "scope-gap:js": "node test/scope-gap-run.ts js",
+    "scope-gap:jsx": "node test/scope-gap-run.ts jsx",
+    "scope-gap:tsx": "node test/scope-gap-run.ts tsx",
+    "scope-gap:html": "node test/scope-gap-run.ts html",
+    "scope-gap:yaml": "node test/scope-gap-run.ts yaml",
+    "scope-gap:vue": "node test/scope-gap-run.ts vue",
     "coverage:table": "node test/coverage-table.ts --write"
   },
   "devDependencies": {

test/bench-vs-ts-agg.ts

@@ -18,27 +18,29 @@ function runAdapter(script: string, args: string[], marker: string, env?: NodeJS
   } catch { return null; }
 }
 
-// TS/JS use deterministic stride subsets for speed; the rest run their full corpus.
+// Both metrics now run through ONE data-driven driver each, parameterised by the `<lang>` code
+// (test/scope-gap-run.ts, test/src-coverage-run.ts). TS/JS use deterministic stride subsets for
+// speed; the rest run their full corpus.
 const COV = [
-  { lang: 'TypeScript', script: 'test/src-coverage-ts.ts', args: ['1500'] },
-  { lang: 'JavaScript', script: 'test/src-coverage-js.ts', args: ['800'] },
-  { lang: 'JSX', script: 'test/src-coverage-jsx.ts', args: [] },
-  { lang: 'TSX', script: 'test/src-coverage-tsx.ts', args: [] },
-  { lang: 'HTML', script: 'test/src-coverage-html.ts', args: [] },
-  { lang: 'YAML', script: 'test/src-coverage-yaml.ts', args: [] },
+  { lang: 'TypeScript', script: 'test/src-coverage-run.ts', args: ['ts', '1500'] },
+  { lang: 'JavaScript', script: 'test/src-coverage-run.ts', args: ['js', '800'] },
+  { lang: 'JSX', script: 'test/src-coverage-run.ts', args: ['jsx'] },
+  { lang: 'TSX', script: 'test/src-coverage-run.ts', args: ['tsx'] },
+  { lang: 'HTML', script: 'test/src-coverage-run.ts', args: ['html'] },
+  { lang: 'YAML', script: 'test/src-coverage-run.ts', args: ['yaml'] },
 ];
-// The 4 TS-family scope-gap adapters all read ONE shared env var (MONOGRAM_OFFICIAL_TM) for
-// the official grammar, so each needs its OWN grammar mapped in (CI sets MONOGRAM_OFFICIAL_TS/
-// TSX/JS/JSX). html/yaml read their own var (MONOGRAM_OFFICIAL_HTML/_YAML), inherited as-is;
-// vue is vendored. Absent (local, no env) → each adapter's VS Code-install fallback path.
+// The 4 TS-family scope-gap entries all read ONE shared env var (MONOGRAM_OFFICIAL_TM) for the
+// official grammar, so each needs its OWN grammar mapped in (CI sets MONOGRAM_OFFICIAL_TS/TSX/JS/JSX).
+// html/yaml read their own var (MONOGRAM_OFFICIAL_HTML/_YAML), inherited as-is; vue is vendored.
+// Absent (local, no env) → the driver's VS Code-install fallback path.
 const GAP = [
-  { lang: 'TypeScript', script: 'test/scope-gap-ts.ts', args: ['800'], officialEnv: 'MONOGRAM_OFFICIAL_TS' },
-  { lang: 'JavaScript', script: 'test/scope-gap-js.ts', args: ['800'], officialEnv: 'MONOGRAM_OFFICIAL_JS' },
-  { lang: 'JSX', script: 'test/scope-gap-jsx.ts', args: [], officialEnv: 'MONOGRAM_OFFICIAL_JSX' },
-  { lang: 'TSX', script: 'test/scope-gap-tsx.ts', args: [], officialEnv: 'MONOGRAM_OFFICIAL_TSX' },
-  { lang: 'HTML', script: 'test/scope-gap-html.ts', args: [] },
-  { lang: 'YAML', script: 'test/scope-gap-yaml.ts', args: [] },
-  { lang: 'Vue', script: 'test/scope-gap-vue.ts', args: [] },
+  { lang: 'TypeScript', script: 'test/scope-gap-run.ts', args: ['ts', '800'], officialEnv: 'MONOGRAM_OFFICIAL_TS' },
+  { lang: 'JavaScript', script: 'test/scope-gap-run.ts', args: ['js', '800'], officialEnv: 'MONOGRAM_OFFICIAL_JS' },
+  { lang: 'JSX', script: 'test/scope-gap-run.ts', args: ['jsx'], officialEnv: 'MONOGRAM_OFFICIAL_JSX' },
+  { lang: 'TSX', script: 'test/scope-gap-run.ts', args: ['tsx'], officialEnv: 'MONOGRAM_OFFICIAL_TSX' },
+  { lang: 'HTML', script: 'test/scope-gap-run.ts', args: ['html'] },
+  { lang: 'YAML', script: 'test/scope-gap-run.ts', args: ['yaml'] },
+  { lang: 'Vue', script: 'test/scope-gap-run.ts', args: ['vue'] },
 ] as { lang: string; script: string; args: string[]; officialEnv?: string }[];
 
 const pct = (v: number | null | undefined) => (v == null ? '—' : v.toFixed(1) + '%');

test/bench-vs-ts.ts

@@ -1,18 +1,7 @@
-// scope-gap-jsx.ts — JSX (.jsx) adapter for the unified scope-gap harness. Grades VS Code's
-// OFFICIAL JavaScriptReact.tmLanguage.json AND Monogram's javascriptreact.tmLanguage.json against
-// the parser oracle (oracle.ts with ScriptKind.JSX). Both grammars declare scopeName `source.js.jsx`.
-// Neither the TS suite nor Test262 ships a .jsx corpus, so this uses a CURATED set exercising both
-// halves (plain JS + JSX), copied verbatim from src-coverage-jsx.ts. It is small, so token counts
-// are low; a real .jsx corpus is a follow-up. Run (bare node): node test/scope-gap-jsx.ts
-import ts from 'typescript';
-import { run } from './scope-gap.ts';
-import { oracle } from './oracle.ts';
+// curated-corpora.ts — small hand-written corpora shared by the folded scope-gap / src-coverage drivers.
+// JSX (plain-JS + JSX halves) and realistic HTML — the languages with no public single-file corpus.
 
-const OFFICIAL = process.env.MONOGRAM_OFFICIAL_TM
-  ?? '/Applications/Visual Studio Code.app/Contents/Resources/app/extensions/javascript/syntaxes/JavaScriptReact.tmLanguage.json';
-
-// No TS types — these are .jsx (JavaScript + JSX) only. Copied verbatim from src-coverage-jsx.ts.
-const JSX_CASES: string[] = [
+export const JSX_CASES: string[] = [
   // --- plain JS half ---
   'const x = 1, y = 2;',
   'function f(a, b = 1, ...rest) { return a + b + rest.length; }',
@@ -52,15 +41,25 @@ const JSX_CASES: string[] = [
   'const boolAttr = <button autofocus formNoValidate>ok</button>;',
 ];
 
-await run({
-  name: 'JavaScriptReact (.jsx)',
-  scopeName: 'source.js.jsx',
-  officialPath: OFFICIAL,
-  monogramPath: 'javascriptreact.tmLanguage.json',
-  loadCorpus: () => JSX_CASES.map((code, i) => ({ name: `<curated #${i}>`, text: code })),
-  roleOracle: (text) => oracle(text, ts.ScriptKind.JSX),
-  isGradable: (text) => {
-    const sf = ts.createSourceFile('c.jsx', text, ts.ScriptTarget.Latest, true, ts.ScriptKind.JSX);
-    return (((sf as any).parseDiagnostics?.length ?? 0) === 0);
-  },
-});
+export const HTML_GENERAL: string[] = [
+  '<div class="container" id="main"><p>Hello <a href="/x">world</a>.</p></div>',
+  '<ul><li>one</li><li>two</li><li>three</li></ul>',
+  '<img src="a.png" alt="a picture" width="100" height="80">',
+  '<input type="text" name="q" placeholder="Search" disabled>',
+  '<button type="submit" class="btn btn-primary" data-id="42">Go</button>',
+  '<section><h1>Title</h1><p>Body with <strong>bold</strong> and <em>italic</em>.</p></section>',
+  '<nav><a href="/">Home</a> | <a href="/about">About</a></nav>',
+  '<form action="/submit" method="post"><label for="n">Name</label><input id="n"></form>',
+  '<table><thead><tr><th>A</th><th>B</th></tr></thead><tbody><tr><td>1</td><td>2</td></tr></tbody></table>',
+  '<!-- a comment --><div><!-- inline --><span>x</span></div>',
+  '<meta charset="utf-8"><meta name="viewport" content="width=device-width, initial-scale=1">',
+  '<br><hr>',
+  '<select><option value="1">One</option><option value="2" selected>Two</option></select>',
+  '<video controls width="320"><source src="m.mp4" type="video/mp4"></video>',
+  '<article data-index=3 hidden><header>H</header><footer>F</footer></article>',
+  '<span class="a b c" title="x y z">text</span>',
+  '<div\n  class="multi-line"\n  id="tag"\n  data-x="1">body</div>',
+  '<a href="https://example.com?a=1&b=2" target="_blank" rel="noopener">link</a>',
+  '<label>Email <input type="email" required></label>',
+  '<figure><img src="p.jpg" alt="photo"><figcaption>cap</figcaption></figure>',
+];