feat(gen-tm): continuation brackets — multi-line constructs inside rich comments

A bracket left OPEN in a rich comment continues the construct across
consecutive introducer-prefixed lines (env-spec multi-line decorator calls and
literals):

    # @import(
    #   ./.env.shared,
    #   pick=[
    #     KEY1, # note
    #   ],
    # )

Declared via `lineComment.continuationBrackets` (bracket pairs). Each pair
emits a begin/end region nested inside the line-scoped rich region — a
TextMate child region suspends its parent's $ end, so the construct spans
lines while single-line comments still close at EOL. Inside a construct:
a line-start introducer is a CONTINUATION MARKER (comment punctuation, not a
new comment); any other introducer opens an embedded comment that dims to
end-of-line; brackets nest recursively; everything else highlights via $self.

An unclosed bracket runs the region to the next closer — the standard
hand-written-grammar hazard; the parser stays the authority on validity.

Opt-in as before: `npm run gen` produces no diffs for the built-in grammars;
41/41 gates pass (env-spec regression contracts extended to 18).

Author: theoephraim

src/api.ts

@@ -24,7 +24,7 @@ interface TokenOptions {
   // Generators emit a to-EOL region in the token's comment scope so prose dims; `richStarters`
   // lists tokens that keep full token highlighting when one opens the comment body
   // (e.g. env-spec `# @decorator(...)`). See TokenDecl.lineComment.
-  lineComment?: { richStarters?: TokenRef[] };
+  lineComment?: { richStarters?: TokenRef[]; continuationBrackets?: [string, string][] };
   // A regex matching exactly one well-formed escape sequence. Engine-scanned tokens
   // (templates) validate each `\`-escape against it and reject any that don't match —
   // unlike `escape` (highlight-only), this drives tokenization. Skipped in tag
@@ -561,6 +561,7 @@ export function defineGrammar(config: GrammarConfig): CstGrammar & { name: strin
             if (!refName) throw new Error(`lineComment.richStarters on token '${name}' references an undeclared token`);
             return refName;
           }),
+          continuationBrackets: tok.opts.lineComment.continuationBrackets?.map((pair) => [...pair] as [string, string]),
         }
         : undefined,
       embed: tok.opts.embed,

src/gen-tm.ts

@@ -5034,12 +5034,54 @@ export function generateTmLanguage(grammar: CstGrammar): TmGrammar {
         .filter((t): t is TokenDecl => !!t);
       if (richStarters.length) {
         const startLookahead = richStarters.map((t) => `(?:${tokenPatternSource(t)})`).join('|');
+        // ── Multi-line constructs (`continuationBrackets`) ──
+        // A bracket left open in a rich comment continues the construct across consecutive
+        // introducer-prefixed lines (`# @import(` … `#   KEY1,` … `# )`). Each pair becomes a
+        // begin/end region nested INSIDE the line-scoped rich region — a TextMate child region
+        // suspends its parent's `$` end, so the construct spans lines while the parent still
+        // closes single-line comments. Inside a construct:
+        //   - a line-start introducer is a CONTINUATION MARKER (comment punctuation, not a new
+        //     comment) — matched first (anchored `^`, earliest-match wins);
+        //   - any other introducer opens an embedded comment to end-of-line that dims
+        //     (`#   KEY1, # note`), popping at `$` so the construct resumes next line;
+        //   - brackets nest recursively; everything else highlights via $self.
+        // An unclosed bracket runs the region to the next closer (the same hazard every
+        // hand-written TextMate grammar with multi-line constructs has) — the parser is the
+        // authority on validity, the highlighter degrades soft.
+        const contBrackets = tok.lineComment.continuationBrackets ?? [];
+        const bracketKeyOf = (open: string) => `${key}-rich-cont-${[...open].map((c) => c.charCodeAt(0).toString(16)).join('')}`;
+        const bracketIncludes = contBrackets.map(([open]) => ({ include: `#${bracketKeyOf(open)}` }));
+        if (contBrackets.length) {
+          repository[`${key}-rich-cont-marker`] = {
+            match: `^[ \\t]*(${introducer})`,
+            captures: { '1': { name: `${scope}.${langName} ${commentPunct}` } },
+          };
+          repository[`${key}-rich-cont-comment`] = {
+            name: `${scope}.${langName}`,
+            begin: `(${introducer})`,
+            beginCaptures: { '1': { name: commentPunct } },
+            end: '$',
+            patterns: [],
+          };
+          for (const [open, close] of contBrackets) {
+            repository[bracketKeyOf(open)] = {
+              begin: escapeRegex(open),
+              end: escapeRegex(close),
+              patterns: [
+                { include: `#${key}-rich-cont-marker` },
+                { include: `#${key}-rich-cont-comment` },
+                ...bracketIncludes,
+                { include: '$self' },
+              ],
+            };
+          }
+        }
         repository[`${key}-rich`] = {
           name: `${scope}.${langName}`,
           begin: `(${introducer})(?=[ \\t]*(?:${startLookahead}))`,
           beginCaptures: { '1': { name: commentPunct } },
           end: '$',
-          patterns: [{ include: '$self' }],
+          patterns: [...bracketIncludes, { include: '$self' }],
         };
         topPatterns.push({ include: `#${key}-rich` });
       }

src/types.ts

@@ -30,7 +30,12 @@ export interface TokenDecl {
   // `richStarters` names tokens that keep FULL token highlighting when one of them (after
   // optional blanks) opens the comment body (`# @decorator(...)`). The lexer/parser are
   // unaffected — exactly like `interpolation`, this is generator metadata.
-  lineComment?: { richStarters?: string[] };
+  // `continuationBrackets`: bracket pairs that, when left OPEN inside a rich comment, continue
+  // the construct across consecutive introducer-prefixed lines (env-spec multi-line decorator
+  // calls/literals — `# @import(` … `#   KEY1,` … `# )`). Each opens a begin/end region that
+  // outlives the line-scoped comment region (a TextMate child region suspends its parent's end),
+  // with the line-start introducer scoped as a continuation marker rather than a new comment.
+  lineComment?: { richStarters?: string[]; continuationBrackets?: [string, string][] };
   escapeValidPattern?: TokenPattern; // one well-formed escape; engine-scanned tokens reject non-matching `\`-escapes (skipped in tag position)
   embed?: string;         // @embed(lang) — embedded language scope name
   // ── Lexer hints (keep the engine language-agnostic; all optional) ──

test/env-spec-regressions.ts

@@ -136,6 +136,40 @@ const check = (label: string, cond: boolean) => {
   }
   check('parser: lineComment metadata does not change parsing', !threw);
 
+  // continuationBrackets: a bracket left open in a rich comment continues the construct
+  // across introducer-prefixed lines via nested begin/end regions
+  const HASH_ML = token(seq(notPrecededBy(noneOf(' ', '\t', '\n', '\r')), '#'), {
+    scope: 'comment.line',
+    lineComment: { richStarters: [DEC_NAME], continuationBrackets: [['(', ')'], ['[', ']']] },
+  });
+  const CommentMl = rule(() => [[HASH_ML, many(Part)]]);
+  const FileMl = rule(() => [[many(CommentMl)]]);
+  const mlGrammar = defineGrammar({
+    name: 'env-spec-ml',
+    tokens: { WS, HASH_ML, DEC_NAME, KEY, TEXT },
+    rules: { Part, CommentMl, FileMl },
+    entry: FileMl,
+  });
+  const tmMl = generateTmLanguage(mlGrammar);
+  const parenKey = Object.keys(tmMl.repository).find((k) => k.startsWith('hash_ml-rich-cont-') && tmMl.repository[k].begin === '\\(');
+  check('tm: continuation bracket pair emits a begin/end region', !!parenKey && tmMl.repository[parenKey!].end === '\\)');
+  const parenRegion = parenKey ? tmMl.repository[parenKey] : undefined;
+  const parenIncludes = (parenRegion?.patterns ?? []).map((pp) => (pp as { include?: string }).include);
+  check('tm: construct interior tries marker, embedded comment, nested brackets, then $self',
+    parenIncludes[0] === '#hash_ml-rich-cont-marker'
+    && parenIncludes[1] === '#hash_ml-rich-cont-comment'
+    && parenIncludes.includes('$self')
+    && parenIncludes.filter((n) => n?.startsWith('#hash_ml-rich-cont-') && n !== '#hash_ml-rich-cont-marker' && n !== '#hash_ml-rich-cont-comment').length === 2);
+  const marker = tmMl.repository['hash_ml-rich-cont-marker'];
+  check('tm: continuation marker is line-anchored and scoped as comment punctuation',
+    typeof marker?.match === 'string' && marker.match.startsWith('^[ \\t]*')
+    && JSON.stringify(marker?.captures?.['1']?.name ?? '').includes('punctuation.definition.comment'));
+  const embedded = tmMl.repository['hash_ml-rich-cont-comment'];
+  check('tm: embedded comment inside a construct dims to end-of-line', embedded?.end === '$' && Array.isArray(embedded?.patterns) && embedded.patterns.length === 0);
+  const richMl = tmMl.repository['hash_ml-rich'];
+  const richIncludes = (richMl?.patterns ?? []).map((pp) => (pp as { include?: string }).include);
+  check('tm: rich region tries construct brackets before $self', richIncludes[richIncludes.length - 1] === '$self' && richIncludes.length === 3);
+
   // a comment token WITHOUT the metadata still emits the flat rule (no behavior change)
   const HASH2 = token(seq(notPrecededBy(noneOf(' ', '\t', '\n', '\r')), '#'), { scope: 'comment.line' });
   const Comment2 = rule(() => [[HASH2, many(TEXT)]]);