YAML compact-nesting + TS generic depth correctness, with an IR-driven depth-verification gate (#48)

Author: johnsoncodehk

src/gen-tm.ts

@@ -98,6 +98,19 @@ export interface RegexContext {
 export interface RawEmbed {
   default: string;                  // embed scope when no (or an unlisted) lang= attribute
   lang?: Record<string, string>;    // lang attribute value → embed scope (e.g. { ts: 'source.ts' })
+  // Whether the embedded language can swallow the close tag mid-line or leave an open construct at
+  // the close that misreads it — i.e. whether the body must FORCE-CLOSE mid-line and FORCE-UNWIND an
+  // open embedded region at the close tag. TRUE for `<script>` (JS `//</script>` on one line must
+  // still close — tmbundle#85; an unterminated `type T =` must unwind before `</script>` is read as
+  // type-args — #5538/#2060): the body uses a `begin/while` open region whose `.*` drops at the first
+  // line CONTAINING the close, plus separate close-LINE rules. That `while` drop is what makes a non-
+  // first DIALECT's close-line content land on a lang-INDEPENDENT close rule (only the first fires),
+  // so a multi-dialect `forceClose` embed cannot keep per-dialect close lines — script accepts this
+  // (mid-line force-close is the priority for JS). FALSE (the default for a `{ default, lang }` embed)
+  // is for a well-behaved embed like CSS — no greedy line-comment swallows `</style>`, nothing leaves
+  // an open construct — so the body uses a lookahead-`end` region (matching the official Vue grammar):
+  // the embed stays active up to `</tag` so a WITH-CONTENT close line keeps its DIALECT (issue #43).
+  forceClose?: boolean;
 }
 
 /**

src/types.ts

@@ -0,0 +1,141 @@
+// angle-depth-probe.ts — the depth gate for the DELIMITER + LOOKAHEAD nesting class (TS/TSX angle
+// brackets), the category that had ZERO depth coverage and where the type-cast `\g<TC>` cliff lived.
+//
+// Two structural checks over the GENERATED typescript/typescriptreact/javascript/javascriptreact
+// grammars, so a new construct cannot be added without this gate seeing it:
+//
+//   PART 1 — DEPTH SWEEP (self-baselining). The angle-bracket disambiguations (generic call, type
+//   cast, generic arrow, JSX tag type-args, generic type) confirm `<…>` is a type span. A confirm
+//   built on an Oniguruma `\g<>` subroutine has a ~20-level recursion cap, so a deep nested generic
+//   flips the discriminating scope at d=20 (the cast did exactly this: `<` → relational at d=20).
+//   For each construct we sweep d=1,5,19,20,24,30 and assert the probe token's scope at every depth
+//   EQUALS its shallow (d=1) value — a flip anywhere is a depth cliff. Self-baselining, so it is
+//   robust to scope renames; it would have caught the cast cliff (gen-tm 1967) and the TSX arrow
+//   cliff (the formerly-recursive arrowEndConfirm) before they shipped.
+//
+//   PART 2 — `\g<>` CENSUS. Every Oniguruma subroutine `\g<>` reachable from a begin/match/while/end
+//   in the emitted grammars is enumerated and matched against an ALLOWLIST of graceful-degraders
+//   (a `\g<>` inside a negative lookahead or an optional group, whose overflow is benign). A `\g<>`
+//   NOT on the list is a latent sole-confirmer cliff → FAIL, forcing a flat-confirm migration (as
+//   done for generic-call/arrow/cast) or an explicit, depth-probed allowlist entry. This makes the
+//   cast-cliff class structurally un-missable: it walks the EMITTED output, not a hand list.
+//
+// Run (bare node): node test/angle-depth-probe.ts   ·   Exit 0 iff no cliff AND census clean.
+import { readFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import vsctm from 'vscode-textmate';
+import onig from 'vscode-oniguruma';
+
+const { INITIAL, Registry } = vsctm;
+const { loadWASM, OnigScanner, OnigString } = onig;
+const require = createRequire(import.meta.url);
+const wasm = readFileSync(require.resolve('vscode-oniguruma/release/onig.wasm'));
+await loadWASM(wasm.buffer.slice(wasm.byteOffset, wasm.byteOffset + wasm.byteLength));
+
+async function load(path: string) {
+  const raw = JSON.parse(readFileSync(path, 'utf8'));
+  const reg = new Registry({
+    onigLib: Promise.resolve({
+      createOnigScanner: (ps: string[]) => new OnigScanner(ps),
+      createOnigString: (s: string) => new OnigString(s),
+    }),
+    loadGrammar: async (scope: string) => (scope === raw.scopeName ? raw : null),
+  });
+  return reg.loadGrammar(raw.scopeName);
+}
+
+const TS = await load('./typescript.tmLanguage.json');
+const TSX = await load('./typescriptreact.tmLanguage.json');
+
+// scope of the FIRST token whose text === `needle` (optionally the nth), innermost scope.
+function scopeOf(g: any, src: string, needle: string, nth = 1): string | null {
+  const lines = src.split('\n');
+  let rs = INITIAL;
+  let seen = 0;
+  for (const line of lines) {
+    const r = g.tokenizeLine(line, rs);
+    for (const t of r.tokens) {
+      if (line.slice(t.startIndex, t.endIndex) === needle && ++seen === nth) return t.scopes[t.scopes.length - 1];
+    }
+    rs = r.ruleStack;
+  }
+  return null;
+}
+
+// A d-deep nested generic type span: A0<A1<…<X>…>>  (no spaces, the densest form).
+const nest = (d: number) => {
+  let s = 'X';
+  for (let i = 0; i < d; i++) s = `A${i}<${s}>`;
+  return s;
+};
+
+interface Construct {
+  name: string;
+  g: any;
+  build: (d: number) => string;
+  needle: string;   // token whose scope must stay stable across depth
+  nth?: number;
+}
+
+const CONSTRUCTS: Construct[] = [
+  { name: 'generic-call (ts)', g: TS, build: (d) => `const z = f<${nest(d)}>(0);\n`, needle: '<' },
+  { name: 'type-cast (ts)', g: TS, build: (d) => `const x = <${nest(d)}>v;\n`, needle: '<' },
+  { name: 'generic-type-annotation (ts)', g: TS, build: (d) => `let a: ${nest(d)};\n`, needle: '<' },
+  { name: 'generic-arrow (tsx)', g: TSX, build: (d) => `const f = <T extends ${nest(d)}>(p: T) => p;\n`, needle: 'p', nth: 1 },
+  { name: 'jsx-tag-type-args (tsx)', g: TSX, build: (d) => `const e = <Comp<${nest(d)}> a={1} />;\n`, needle: 'Comp' },
+];
+
+const DEPTHS = [1, 5, 19, 20, 24, 30];
+const cliffs: string[] = [];
+
+for (const c of CONSTRUCTS) {
+  const baseline = scopeOf(c.g, c.build(1), c.needle, c.nth);
+  const row: string[] = [];
+  for (const d of DEPTHS) {
+    const sc = scopeOf(c.g, c.build(d), c.needle, c.nth);
+    const ok = sc === baseline;
+    row.push(`d${d}:${ok ? 'ok' : 'FLIP'}`);
+    if (!ok) cliffs.push(`${c.name} «${c.needle}» d=${d}: ${sc}  (shallow was ${baseline})`);
+  }
+  console.log(`  ${c.name.padEnd(30)} «${c.needle}»→${baseline}\n      ${row.join('  ')}`);
+}
+
+// PART 2 — \g<> census over the emitted grammars.
+// Allowlist: a graceful-degrader is a `\g<>` whose overflow is benign — inside a NEGATIVE lookahead
+// (the bail-out just doesn't fire) or an OPTIONAL group (a flat fallback follows). Each entry names
+// the repository key + subroutine and WHY it is safe.
+const ALLOW: { grammar: RegExp; keyRe: RegExp; sub: string; why: string }[] = [
+  { grammar: /typescript(react)?/, keyRe: /generic-call-multiline/, sub: 'B', why: 'inside a NEGATIVE lookahead (?!…) — overflow makes the bail-out not fire (benign), verified stable to d=30' },
+  { grammar: /typescriptreact/, keyRe: /jsx/, sub: 'TA', why: 'inside an OPTIONAL group (?:…)? with a flat [^>]* fallback — overflow falls back to the region, verified stable to d=30' },
+];
+const census: string[] = [];
+const unrecognized: string[] = [];
+for (const [gf] of [['typescript.tmLanguage.json'], ['typescriptreact.tmLanguage.json'], ['javascript.tmLanguage.json'], ['javascriptreact.tmLanguage.json']] as const) {
+  const g = JSON.parse(readFileSync(`./${gf}`, 'utf8'));
+  const walk = (node: any, key: string) => {
+    if (!node || typeof node !== 'object') return;
+    for (const f of ['begin', 'match', 'while', 'end'] as const) {
+      const re = node[f];
+      if (typeof re === 'string') {
+        for (const m of re.matchAll(/\\g<([^>]*)>/g)) {
+          const sub = m[1];
+          census.push(`${gf}:${key}.${f}:\\g<${sub}>`);
+          const ok = ALLOW.some((a) => a.grammar.test(gf) && a.keyRe.test(key) && a.sub === sub);
+          if (!ok) unrecognized.push(`${gf} ${key}.${f} \\g<${sub}>`);
+        }
+      }
+    }
+    for (const k in node) if (k !== 'repository' && typeof node[k] === 'object') walk(node[k], key);
+  };
+  for (const k in (g.repository || {})) walk(g.repository[k], k);
+}
+
+console.log(`\n  \\g<> census: ${census.length} subroutine use(s); ${unrecognized.length} unrecognized (not a known graceful-degrader)`);
+for (const u of unrecognized) console.log(`    UNRECOGNIZED \\g<> (latent depth cliff — flat-migrate or allowlist with a depth-probe): ${u}`);
+
+const fail = cliffs.length + unrecognized.length;
+if (cliffs.length) { console.log('\n  DEPTH CLIFFS:'); for (const c of cliffs) console.log(`    ${c}`); }
+console.log(fail === 0
+  ? '\n✓ no angle-bracket depth cliff (stable to d=30) and \\g<> census clean'
+  : `\n✗ ${cliffs.length} cliff(s) + ${unrecognized.length} unrecognized \\g<>`);
+process.exit(fail === 0 ? 0 : 1);

test/angle-depth-probe.ts

@@ -34,20 +34,24 @@ const GATES: Gate[] = [
   { group: 'conformance', name: 'html', args: ['test/html-conformance.ts'] },
   { group: 'highlighter', name: 'tm-guards', args: ['test/tm-highlight-guards.ts'] },
   { group: 'highlighter', name: 'tm-diagnostics', args: ['test/redcmd-tm-diagnostics.ts'] },
+  { group: 'highlighter', name: 'angle-depth', args: ['test/angle-depth-probe.ts'] },
   { group: 'highlighter', name: 'html-monarch', args: ['test/html-monarch.ts'] },
   { group: 'highlighter', name: 'html-embed-js', args: ['test/html-embed-js.ts'] },
   { group: 'highlighter', name: 'html-lexer-spike', args: ['test/html-lexer-spike.ts'] },
   { group: 'highlighter', name: 'self-close-sites', args: ['test/self-close-sites.ts'] },
   { group: 'highlighter', name: 'raw-text-case-sites', args: ['test/raw-text-case-sites.ts'] },
   { group: 'vue', name: 'directives', args: ['test/vue-directives.ts'] },
   { group: 'vue', name: 'embed-boundary', args: ['test/vue-embed-boundary.ts'] },
+  { group: 'vue', name: 'raw-style-embed', args: ['test/vue-raw-style-embed-sites.ts'] },
   { group: 'vue', name: 'interp-expr', args: ['test/vue-interp-expr.ts'] },
   { group: 'core', name: 'indent-extensions', args: ['test/indent-extensions.ts'] },
   { group: 'yaml', name: 'issue12-regressions', args: ['test/yaml-issue12-regressions.ts'] },
   { group: 'yaml', name: 'depth-witnesses', args: ['test/yaml-depth-witnesses.ts'] },
   { group: 'yaml', name: 'depth-sites', args: ['test/depth-sites.ts'] },
   { group: 'yaml', name: 'flow-sites', args: ['test/flow-sites.ts'] },
   { group: 'yaml', name: 'compact-nest-sites', args: ['test/compact-nest-sites.ts'] },
+  { group: 'yaml', name: 'deepest-sibling', args: ['test/yaml-deepest-sibling-probe.ts'] },
+  { group: 'yaml', name: 'blockscalar-depth', args: ['test/yaml-blockscalar-depth-probe.ts'] },
   { group: 'generative', name: 'scope≡role', args: ['test/generative.ts'] },
   { group: 'generative', name: 'gap-ledger-selftest', args: ['test/gap-ledger-selftest.ts'] },
   { group: 'generative', name: 'gap-ledger-check', args: ['test/gap-ledger.ts', '--check'] },

test/check.ts

@@ -0,0 +1,70 @@
+// vue-raw-style-embed-sites.ts — an enumerator of the <style lang="…"> RAW-STYLE EMBED across
+// (dialect × structural position), tokenised with the REAL oniguruma engine and graded against an
+// INDEPENDENT oracle: the dialect scope the GRAMMAR ITSELF declares for the embed.
+//
+// THE CLASS (issue #43): a <style lang="X"> body delegates to the CSS dialect grammar source.css.X
+// the grammar's own embed map names. The derived TextMate grammar splits that delegation into per-
+// dialect open + close rules; the CLOSE rule's match is language-INDEPENDENT (the lang lives on the
+// OPEN tag, not on `</style>`), so every dialect's close rule shares one regex and only the first-
+// listed fires — a non-first dialect's CLOSE-LINE content (the pre-`</style>` text) is then embedded
+// in the WRONG dialect. A bug is exactly: content that should be source.css.X carries some other
+// source.css.* instead.
+//
+// WHY THE EXISTING GATES MISSED IT (see issue #43 discussion): scope-gap-vue grades the Vue shell
+// (tags/directives/interpolation) against @vue/compiler-sfc+parse5+tsc — it has NO CSS oracle, so the
+// embedded dialect is structurally ungraded. This test adds that missing axis: an oracle that is the
+// grammar's DECLARED embed scope (not Monogram's parser, which raw-texts the body as a blob), and
+// DERIVED witnesses (each dialect × each structural position) rather than a thin corpus.
+//
+// CLOSED LOOP / no hardcoding: the dialects + their expected scopes come from the SAME grammar source
+// the emitter derives the rules from (`grammar.markup.rawText.embed.style`).
+//
+// Run: node test/vue-raw-style-embed-sites.ts
+import { tokenize } from './vue-grammar-harness.ts';
+import grammar from '../vue.ts';
+
+const style = (grammar as any).markup?.rawText?.embed?.style;
+if (!style) { console.error('vue grammar has no markup.rawText.embed.style'); process.exit(1); }
+
+// [langAttr | null (default), expectedScope]; closed-loop from the grammar's own embed map.
+const dialects: [string | null, string][] = [
+  [null, style.default],
+  ...Object.entries(style.lang as Record<string, string>).map(([k, v]) => [k, v] as [string, string]),
+];
+
+// Structural positions. Each builds a <style> block; `find` is the CSS-content selector token whose
+// scope chain MUST contain the dialect's declared scope, and `pos` labels where it sits.
+function witnesses(lang: string | null): { pos: string; src: string; find: string }[] {
+  const open = lang === null ? '<style>' : `<style lang="${lang}">`;
+  return [
+    // baseline: content on its OWN line, close on its own line (the path that already works).
+    { pos: 'content-line', src: `${open}\n.midline { a: 1 }\n</style>`, find: 'midline' },
+    // THE BUG: content on the SAME line as the close `</style>` — the per-dialect close rule's capture.
+    { pos: 'close-line  ', src: `${open}\n.firstline { a: 1 }\n.closeline { b: 2 }</style>`, find: 'closeline' },
+    // single-line: open, content, and close all on one line.
+    { pos: 'single-line ', src: `${open}.oneline { c: 3 }</style>`, find: 'oneline' },
+  ];
+}
+
+const cssScope = (chain: string) => chain.split(' ').find(s => s.startsWith('source.css') || s === 'source.sass' || s === 'source.stylus' || s === 'source.postcss') ?? '(none)';
+
+let cells = 0, wrong = 0;
+const fails: string[] = [];
+for (const [lang, expected] of dialects) {
+  for (const w of witnesses(lang)) {
+    cells++;
+    const toks = await tokenize('mono', w.src);
+    const t = toks.find(x => x.text.includes(w.find));
+    const got = t ? cssScope(t.scopes) : '(token not found)';
+    const ok = t !== undefined && t.scopes.split(' ').includes(expected);
+    if (!ok) { wrong++; fails.push(`<style ${lang === null ? '(default)' : `lang="${lang}"`}> @ ${w.pos.trim()}: want ${expected}, got ${got}`); }
+    console.log(`  ${ok ? '✓ ok  ' : '✗ BUG '}[${(lang ?? 'default').padEnd(7)} × ${w.pos}] want ${expected.padEnd(16)} got ${got}`);
+  }
+}
+
+console.log(`\n  ${cells} raw-style-embed cells · ${cells - wrong} ok · ${wrong} wrong`);
+if (wrong) {
+  console.error(`\n  RAW-STYLE EMBED BUG — a <style lang="X"> body embedded in the wrong CSS dialect:\n    ${fails.join('\n    ')}`);
+  process.exit(1);
+}
+console.log('  ✓ every <style lang="X"> body — at every structural position — embeds the dialect the grammar declares.');