Derive YAML plain-fold region literals from the indent config

The multi-line plain-scalar fold regions (gen-tm §2a′/§2a″) and the block-scalar
value-prefix hardcoded YAML structural literals (`#`, `-`/`?`, `---`/`...`, the flow
brackets, `:`) in their regexes. Derive them from the indent config instead so the
YAML region code is data-driven, not hardcoded: comment from `indent.comment`,
compact indicators from `indent.compactIndicators`, document markers from
`blockScalar.documentMarkers`, the flow-bracket exclusion from `indent.flowOpen` /
`indent.flowClose`, and the mapping key/value separator from a new
`indent.keyValueSeparator` field (defaults to ':'). The one remaining inline literal
— node-property `&`/`!` in the block-scalar prop prefix — is anchor/tag-specific
(it derives from the Anchor/Tag tokens, not the indent config) and is left as is.

These regions are gated on `grammar.indent.blockScalar`, so the other six grammars
regenerate byte-identical (verified).

The regenerated YAML grammar is tokenization-IDENTICAL to before: the only byte
changes are two semantically-equivalent char-class normalizations from the derivation
(`[-?]` → `[\-?]`, `[^\n{}[]]` → `[^\n[{]}]`), and a full-corpus tokenization diff over
the yaml-test-suite (406 inputs) shows 0 differences. yaml-issue12-regressions stays
6 pass / 4 known-bug / 0 regression; scope-gap Monogram-wrong unchanged at 8; agnostic
guard 9/9; sanity 15/15; scope-roles 47/47 + 9/9; tm-highlight-guards 22/0; RedCMD
Onigmo diagnostics clean.

Refs #12

Author: johnsoncodehk

src/gen-tm.ts

@@ -4645,6 +4645,20 @@ export function generateTmLanguage(grammar: CstGrammar, langName: string): TmGra
   let bsIndicatorScope = '';
   let bsContentScope = '';
   if (blockScalar) {
+    // YAML structural literals DERIVED from the indent config (kept data-driven, NOT hardcoded in the
+    // generator) — used by the block-scalar value-prefix (`bsVp`) and the multi-line plain-scalar fold
+    // regions (§2a′/§2a″): the comment introducer, the compact indicators (`-`/`?`) as an alternation
+    // and a char class, the document markers (`---`/`...`), the flow brackets (for the key-scan
+    // exclusion class), and the mapping key/value separator (`:`). The one structural literal WITHOUT a
+    // dedicated field — node-property `&`/`!` in `bsProp` below — is anchor/tag-specific (it comes from
+    // the Anchor/Tag tokens, not the indent config) and stays inline.
+    const ind = grammar.indent!;
+    const cmtLit = escapeRegex(ind.comment ?? '#');
+    const compactAlt = (ind.compactIndicators ?? []).map((c) => `${escapeRegex(c)}[\\t ]`).join('|');
+    const compactCls = `[${(ind.compactIndicators ?? []).map(escapeForCharClass).join('')}]`;
+    const docAlt = (blockScalar.documentMarkers ?? []).map(escapeRegex).join('|');
+    const flowEx = `[^\\n${[...(ind.flowOpen ?? []), ...(ind.flowClose ?? [])].map(escapeForCharClass).join('')}]`;
+    const kvSep = escapeRegex(ind.keyValueSeparator ?? ':');
     const bsTok = grammar.tokens.find(t => t.name === blockScalar.token);
     const bsKey = blockScalar.token.toLowerCase();
     const bsScope = bsTok?.scope ?? 'string.unquoted.block';
@@ -4739,7 +4753,7 @@ export function generateTmLanguage(grammar: CstGrammar, langName: string): TmGra
     // END in a pipe) from opening a region: after `a: ` the next token is `foo`, not a separator/
     // key-colon/property, so the lookahead fails. The key arm matches up to the FIRST `: ` separator.
     const bsProp = '(?:[&!][^\\t\\n\\f\\r \\[\\]{},]*[\\t ]+)*';
-    const bsVp = `(?:(?:---|\\.\\.\\.)[\\t ]+)?(?:[-?][\\t ]+)*(?:[^\\n]*?:[\\t ]+)?${bsProp}`;
+    const bsVp = `(?:(?:${docAlt})[\\t ]+)?(?:${compactCls}[\\t ]+)*(?:[^\\n]*?${kvSep}[\\t ]+)?${bsProp}`;
     // Expose the introducer / inner-rule / header-includes to §2b (the `? |` explicit-key variant).
     bsIntro = intro;
     bsFunkyIntroRule = bsIntroRule;
@@ -4832,7 +4846,7 @@ export function generateTmLanguage(grammar: CstGrammar, langName: string): TmGra
       // flow bracket (a `{`/`[` is flow, handled elsewhere), and `:(?:[\t ]|$)` requires the colon to be
       // a real key separator (`http://x` keeps its glued `:` → still plain content). Used as a NEGATIVE
       // lookahead to bound the fold at the first sibling/comment line, matching the parser's foldedPlain.
-      const structAhead = `(?:#|-[\\t ]|\\?[\\t ]|[^\\n{}\\[\\]]*?:(?:[\\t ]|$))`;
+      const structAhead = `(?:${cmtLit}|${compactAlt}|${flowEx}*?${kvSep}(?:[\\t ]|$))`;
       // Value-position lookahead: after the node indent is stripped, the line must carry an INLINE
       // BLOCK plain value — either `<key>: <plain>` (mapping value) or a `-`/`?` indicator + `<plain>`
       // (sequence entry / explicit key). The plain value is confirmed by `(?=plainSrc)`, which only
@@ -4844,7 +4858,7 @@ export function generateTmLanguage(grammar: CstGrammar, langName: string): TmGra
       // a flow collection (`{ a: b,\n  c }` / `a: { b: c,\n  d }`) is a multi-line begin/end region of
       // its own — a `{`/`[` before the `:` means the `:` is a FLOW separator, not a block one, so the
       // region must NOT open and steal those lines from #flow-mapping/#flow-sequence.
-      const plainVp = `(?:(?:---|\\.\\.\\.)[\\t ]+)?(?:(?:[-?][\\t ]+)+(?:[^\\n{}\\[\\]]*?:[\\t ]+)?|[^\\n{}\\[\\]]*?:[\\t ]+)(?=${plainSrc})`;
+      const plainVp = `(?:(?:${docAlt})[\\t ]+)?(?:(?:${compactCls}[\\t ]+)+(?:${flowEx}*?${kvSep}[\\t ]+)?|${flowEx}*?${kvSep}[\\t ]+)(?=${plainSrc})`;
       // Header-line token includes: the same shape any plain `key: value` line gets, so the header is
       // scoped identically to the top level (only the CONTINUATION changes). Includes the typed-value
       // tokens (`#num`/`#boolnull`) so a SINGLE-line `a: 1` keeps `constant.numeric`, and the full
@@ -4897,7 +4911,7 @@ export function generateTmLanguage(grammar: CstGrammar, langName: string): TmGra
       // an inner `begin:$`/`while:\G` body swallows each following line as `string.unquoted` (stopping
       // before an inline ` #` comment, which then falls to `#comment`). structRelease extends §2a′'s
       // `structAhead` with the doc markers (a `---`/`...` ends a doc-body fold).
-      const structRelease = `(?:#|-[\\t ]|\\?[\\t ]|(?:---|\\.\\.\\.)(?:[\\t ]|$)|[^\\n{}\\[\\]]*?:(?:[\\t ]|$))`;
+      const structRelease = `(?:${cmtLit}|${compactAlt}|(?:${docAlt})(?:[\\t ]|$)|${flowEx}*?${kvSep}(?:[\\t ]|$))`;
       // The HEADER line is scoped by the normal token includes (so a standalone bare `42`/`true` keeps
       // its `#num`/`#boolnull` typing). A CONTINUATION line that opens with a non-token char (`%`, which
       // no header include matches) would leave that char unscoped and let a LATER `#plain` claim only the

src/types.ts

@@ -311,6 +311,11 @@ export interface IndentConfig {
     explicitKey?: string;  // the flow `?` explicit-key indicator (e.g. punctuation.definition.key-value)
   };
   comment?: string;       // line-comment introducer ignored for indentation (e.g. '#')
+  // The mapping KEY/VALUE separator literal (YAML `:`). Used by the derived highlighter's multi-line
+  // plain-scalar fold regions (gen-tm §2a′/§2a″) to recognise a `key:`-led line as STRUCTURAL (a
+  // sibling that ends a fold) vs a bare plain-scalar continuation. Declared here (not hardcoded in the
+  // generator) so the YAML region code stays data-driven. Absent → defaults to ':'.
+  keyValueSeparator?: string;
   // Block scalars (YAML `|` / `>`): when the rest of a line is an introducer + indicators, the
   // following more-indented lines are verbatim content emitted as ONE token (like raw-text, but
   // bounded by indentation rather than a close tag). `introducers` are the leading chars (['|','>']).

yaml.tmLanguage.json

@@ -318,7 +318,7 @@
       "match": "!(?:<[^>]*>|[^\\t\\n\\f\\r \\[\\]{},]*)"
     },
     "blockscalar": {
-      "begin": "^([ \\t]*)(?=(?:(?:---|\\.\\.\\.)[\\t ]+)?(?:[-?][\\t ]+)*(?:[^\\n]*?:[\\t ]+)?(?:[&!][^\\t\\n\\f\\r \\[\\]{},]*[\\t ]+)*[|>](?:[1-9][-+]?|[-+][1-9]?|[-+])?[\\t ]*(?:#|$))",
+      "begin": "^([ \\t]*)(?=(?:(?:---|\\.\\.\\.)[\\t ]+)?(?:[\\-?][\\t ]+)*(?:[^\\n]*?:[\\t ]+)?(?:[&!][^\\t\\n\\f\\r \\[\\]{},]*[\\t ]+)*[|>](?:[1-9][-+]?|[-+][1-9]?|[-+])?[\\t ]*(?:#|$))",
       "while": "\\G(?=\\1[ \\t]|[ \\t]*$)",
       "patterns": [
         {
@@ -606,8 +606,8 @@
       ]
     },
     "plain-continuation": {
-      "begin": "^([ \\t]*)(?=(?:(?:---|\\.\\.\\.)[\\t ]+)?(?:(?:[-?][\\t ]+)+(?:[^\\n{}\\[\\]]*?:[\\t ]+)?|[^\\n{}\\[\\]]*?:[\\t ]+)(?=(?:[^\\t\\n\\f\\r \\-?:,\\[\\]{}#&*!|>'\"%@`]|[\\-?:](?=[^\\t\\n\\f\\r ,\\[\\]{}]))(?:[^:#\\n,\\[\\]{}]|:(?=[^\\t\\n\\f\\r ,\\]}])|#(?<=[^\\t\\n\\f\\r ]#))*))",
-      "while": "\\G(?=[ \\t]*$|\\1[ \\t]+(?!(?:#|-[\\t ]|\\?[\\t ]|[^\\n{}\\[\\]]*?:(?:[\\t ]|$))))",
+      "begin": "^([ \\t]*)(?=(?:(?:---|\\.\\.\\.)[\\t ]+)?(?:(?:[\\-?][\\t ]+)+(?:[^\\n\\[{\\]}]*?:[\\t ]+)?|[^\\n\\[{\\]}]*?:[\\t ]+)(?=(?:[^\\t\\n\\f\\r \\-?:,\\[\\]{}#&*!|>'\"%@`]|[\\-?:](?=[^\\t\\n\\f\\r ,\\[\\]{}]))(?:[^:#\\n,\\[\\]{}]|:(?=[^\\t\\n\\f\\r ,\\]}])|#(?<=[^\\t\\n\\f\\r ]#))*))",
+      "while": "\\G(?=[ \\t]*$|\\1[ \\t]+(?!(?:#|-[\\t ]|\\?[\\t ]|[^\\n\\[{\\]}]*?:(?:[\\t ]|$))))",
       "patterns": [
         {
           "match": "\\G[\\t ]+(?:[^#\\n]|#(?<=[^\\t\\n\\f\\r ]#))*",
@@ -661,8 +661,8 @@
       ]
     },
     "plain-bare-fold": {
-      "begin": "^([ \\t]*)(?=(?:[^\\t\\n\\f\\r \\-?:,\\[\\]{}#&*!|>'\"%@`]|[\\-?:](?=[^\\t\\n\\f\\r ,\\[\\]{}]))(?:[^:#\\n,\\[\\]{}]|:(?=[^\\t\\n\\f\\r ,\\]}])|#(?<=[^\\t\\n\\f\\r ]#))*)(?!(?:#|-[\\t ]|\\?[\\t ]|(?:---|\\.\\.\\.)(?:[\\t ]|$)|[^\\n{}\\[\\]]*?:(?:[\\t ]|$)))",
-      "while": "\\G(?=[ \\t]*$|\\1(?=[ \\t]*\\S)(?![ \\t]*(?:#|-[\\t ]|\\?[\\t ]|(?:---|\\.\\.\\.)(?:[\\t ]|$)|[^\\n{}\\[\\]]*?:(?:[\\t ]|$))))",
+      "begin": "^([ \\t]*)(?=(?:[^\\t\\n\\f\\r \\-?:,\\[\\]{}#&*!|>'\"%@`]|[\\-?:](?=[^\\t\\n\\f\\r ,\\[\\]{}]))(?:[^:#\\n,\\[\\]{}]|:(?=[^\\t\\n\\f\\r ,\\]}])|#(?<=[^\\t\\n\\f\\r ]#))*)(?!(?:#|-[\\t ]|\\?[\\t ]|(?:---|\\.\\.\\.)(?:[\\t ]|$)|[^\\n\\[{\\]}]*?:(?:[\\t ]|$)))",
+      "while": "\\G(?=[ \\t]*$|\\1(?=[ \\t]*\\S)(?![ \\t]*(?:#|-[\\t ]|\\?[\\t ]|(?:---|\\.\\.\\.)(?:[\\t ]|$)|[^\\n\\[{\\]}]*?:(?:[\\t ]|$))))",
       "patterns": [
         {
           "include": "#docstart"

yaml.ts

@@ -614,6 +614,7 @@ const indent: IndentConfig = {
     explicitKey: 'punctuation.definition.key-value',
   },
   comment: '#',
+  keyValueSeparator: ':',
   blockScalar: { introducers: ['|', '>'], token: 'BlockScalar', documentMarkers: ['---', '...'], indicatorScope: 'keyword.control.flow.block-scalar' },
   compactIndicators: ['-', '?'],
   // Tag-handle per-document membership (§6.8.2 / §6.9.1): a named handle `!h!` used by a Tag must