Give YAML flow punctuation specific scopes (grammar-declared, agnostic)

Flow `{ } [ ] ,` were all scoped a generic `punctuation`, so the strict flowPunct
role graded family-not-exact — the dominant exact% drain (Monogram correct 99.2%
but exact only 82.2%). Emit the TextMate-convention scopes instead:
`punctuation.definition.{mapping,sequence}.{begin,end}` for the brackets,
`punctuation.separator.{mapping,sequence}` for the comma, and
`punctuation.separator.key-value` for the flow `:`.

Kept gen-tm AGNOSTIC: the language-specific names (`mapping`/`sequence` — a JS `{}`
is an "object") live in the grammar via a new `IndentConfig.flowScopes`, declared in
yaml.ts and threaded through detectFlowCollections (same pattern as blockScalar.
indicatorScope); gen-tm only appends `.${lang}` and falls back to the generic
`punctuation` when nothing is declared, so a grammar with no flowScopes is
byte-identical. Block-context `:`/`-` stay generic — the specificity is flow-only.

YAML exact 82.2% -> 98.5% (now also ahead of official's 98.4%); correct% unchanged
at 99.2%, gap +0.2; parser 100% aligned; agnostic 9/9; other six grammars
byte-identical. (The ~14 flow-comma tokens both grammars "miss" are an answer-key
over-emission — a `,` inside a tag-URI / %TAG / comment — not a grammar gap.)

Author: johnsoncodehk

src/gen-tm.ts

@@ -3388,7 +3388,10 @@ function detectExplicitKey(grammar: CstGrammar): { indicator: string; keyScope:
 //   • the KEY scope + plain-scalar shape come from the same Key/Plain token pair detectExplicitKey
 //     uses (a scalar token whose pattern is `<plain>(?=…sep…)` over a broader plain scalar).
 // Returns null when the family has no flow collections (every non-YAML grammar).
-interface FlowColl { open: string; close: string; sep: string; colon: string | null; }
+// `punct` (when present) are the SPECIFIC open/close/separator scopes the grammar declared for THIS
+// collection via `indent.flowScopes.byOpen[open]` (null → fall back to the generic `punctuation.*`).
+// Scope strings are bare (no `.${lang}` suffix); the region builder appends it.
+interface FlowColl { open: string; close: string; sep: string; colon: string | null; punct: { begin: string; end: string; separator: string } | null; }
 function detectFlowCollections(grammar: CstGrammar): {
   colls: FlowColl[];
   plainStart: string;          // plain-scalar LEADING char class (no enclosing group)
@@ -3400,6 +3403,8 @@ function detectFlowCollections(grammar: CstGrammar): {
   sqScope: string | null;      // single-quoted scalar scope
   dqEscape: string | null;     // in-string escape sub-pattern for the double quote
   sqEscape: string | null;     // in-string escape sub-pattern for the single quote
+  keyValueScope: string | null;   // declared `:` key/value separator scope (indent.flowScopes.keyValue), bare
+  explicitKeyScope: string | null; // declared `?` explicit-key indicator scope (indent.flowScopes.explicitKey), bare
 } | null {
   const indent = grammar.indent;
   if (!indent || !indent.flowOpen?.length || !indent.flowClose?.length) return null;
@@ -3447,7 +3452,10 @@ function detectFlowCollections(grammar: CstGrammar): {
       const c = eLits.find(l => l.length === 1 && l !== sep && l !== open && l !== close && !/[\w\s]/.test(l));
       if (c) { colon = c; break; }
     }
-    colls.push({ open, close, sep, colon });
+    // The grammar may DECLARE specific open/close/separator scopes for this collection (keyed by the
+    // open bracket); absent → null → the region builder uses the generic `punctuation.${lang}`.
+    const punct = indent.flowScopes?.byOpen?.[open] ?? null;
+    colls.push({ open, close, sep, colon, punct });
   }
   if (!colls.length) return null;
 
@@ -3484,6 +3492,8 @@ function detectFlowCollections(grammar: CstGrammar): {
     dq: dqTok ? tokenPatternSource(dqTok) : null, sq: sqTok ? tokenPatternSource(sqTok) : null,
     dqScope: dqTok?.scope ?? null, sqScope: sqTok?.scope ?? null,
     dqEscape: quoteEscape(dqTok), sqEscape: quoteEscape(sqTok),
+    keyValueScope: indent.flowScopes?.keyValue ?? null,
+    explicitKeyScope: indent.flowScopes?.explicitKey ?? null,
   };
 }
 
@@ -4861,6 +4871,12 @@ export function generateTmLanguage(grammar: CstGrammar, langName: string): TmGra
   if (flow) {
     const ln = langName;
     const P = `punctuation.${ln}`;
+    // Resolve a DECLARED bare scope (from indent.flowScopes) to a full scope, or fall back to the
+    // generic `P`. The grammar supplies the language-flavoured names (mapping/sequence/key-value);
+    // the engine only appends the language suffix — so gen-tm stays agnostic (no hardcoded names).
+    const withLang = (bare: string | null): string => bare ? `${bare}.${ln}` : P;
+    const kvScope = withLang(flow.keyValueScope);          // the flow-mapping `:` key/value separator
+    const ekScope = withLang(flow.explicitKeyScope);       // the flow `?` explicit-key indicator
     // Comment includes (derived from the grammar's comment-scoped tokens — not a hardcoded key),
     // spread into every flow sub-region so a `#…` comment is scoped even mid-entry / mid-value.
     const commentIncs = commentIncludeKeys.map(k => ({ include: `#${k}` }));
@@ -4905,12 +4921,12 @@ export function generateTmLanguage(grammar: CstGrammar, langName: string): TmGra
     // The VALUE colon (a flow separator `:`); and the JSON-style value colon glued after a closed
     // key (`{"a":b}` — `:` preceded by a quote/closer/line-start, where no space-separator exists).
     repository['flow-map-value'] = {
-      begin: colonSep, beginCaptures: { '0': { name: P } }, end: beforeClose,
+      begin: colonSep, beginCaptures: { '0': { name: kvScope } }, end: beforeClose,
       patterns: [...commentIncs, { include: '#flow-node' }],
     };
     repository['flow-map-value-json'] = {
       begin: `(?<=[${quoteCls}${closeCls}])[\\t ]*${eColon}|(?<=^)[\\t ]*${eColon}`,
-      beginCaptures: { '0': { name: P } }, end: beforeClose,
+      beginCaptures: { '0': { name: kvScope } }, end: beforeClose,
       patterns: [...commentIncs, { include: '#flow-node' }],
     };
     // A plain scalar as a VALUE / as a KEY (entity.name.tag); both end at the flow boundary.
@@ -4931,24 +4947,29 @@ export function generateTmLanguage(grammar: CstGrammar, langName: string): TmGra
     // boundary opens a key region. Derived from the explicit-key indicator when present.
     const qmark = explicitKey ? escapeRegex(explicitKey.indicator) : null;
     const explicitKeyEntry: TmPattern[] = qmark ? [{
-      begin: `${qmark}(?=[\\s${closeCls}]|$)`, beginCaptures: { '0': { name: P } }, end: beforeClose,
+      begin: `${qmark}(?=[\\s${closeCls}]|$)`, beginCaptures: { '0': { name: ekScope } }, end: beforeClose,
       patterns: [...commentIncs, { include: '#flow-mapping-map-key' }, { include: '#flow-map-value' }, { include: '#flow-node' }],
     }] : [];
 
     // Emit one region per collection. A MAPPING (`colon != null`): every entry-leading scalar is a
     // key. A SEQUENCE: a scalar is a key only when a `:` separator follows (single-pair map element).
     for (const c of flow.colls) {
       const eOpen = escapeRegex(c.open), eClose = escapeRegex(c.close), eSep = escapeRegex(c.sep);
+      // The SPECIFIC open/close/separator scopes the grammar declared for THIS collection (mapping vs
+      // sequence — the names are the grammar's, not the engine's); null → the generic `P`.
+      const beginScope = c.punct ? `${c.punct.begin}.${ln}` : P;
+      const endScope = c.punct ? `${c.punct.end}.${ln}` : P;
+      const sepScope = c.punct ? `${c.punct.separator}.${ln}` : P;
       if (c.colon) {
         // A YAML flow mapping `{ … }` is a bracket region. CALLER predicate: detectFlowCollections
         // found a collection rule (`OPEN … CLOSE` seq) whose entry rule carries a `:` key/value
         // separator (`c.colon != null`). Recurse is via the body's `#flow-node` include (which
         // re-includes #flow-mapping/#flow-sequence for nested collections).
         repository['flow-mapping'] = emitBracketRegion({
           name: `meta.flow.mapping.${ln}`,
-          openLit: eOpen, closeLit: eClose, beginCapName: P, endCapName: P,
+          openLit: eOpen, closeLit: eClose, beginCapName: beginScope, endCapName: endScope,
           bodyPatterns: [
-            ...commentIncs, { match: eSep, name: P },
+            ...commentIncs, { match: eSep, name: sepScope },
             { include: '#flow-mapping-map-key' }, { include: '#flow-map-value-json' }, { include: '#flow-map-value' }, { include: '#flow-node' },
           ],
         });
@@ -4968,9 +4989,9 @@ export function generateTmLanguage(grammar: CstGrammar, langName: string): TmGra
         // shape, but the entry rule has NO `:` separator (`c.colon == null`). Same #flow-node recurse.
         repository['flow-sequence'] = emitBracketRegion({
           name: `meta.flow.sequence.${ln}`,
-          openLit: eOpen, closeLit: eClose, beginCapName: P, endCapName: P,
+          openLit: eOpen, closeLit: eClose, beginCapName: beginScope, endCapName: endScope,
           bodyPatterns: [
-            ...commentIncs, { match: eSep, name: P },
+            ...commentIncs, { match: eSep, name: sepScope },
             { include: '#flow-sequence-map-key' }, { include: '#flow-map-value-json' }, { include: '#flow-map-value' }, { include: '#flow-node' },
           ],
         });

src/types.ts

@@ -275,6 +275,23 @@ export interface IndentConfig {
   newlineToken: string;   // token TYPE emitted at a same-column line boundary (sibling separator)
   flowOpen?: string[];    // punctuation that suspends indentation while open (e.g. ['[', '{'])
   flowClose?: string[];   // matching closers (e.g. [']', '}'])
+  // Per-collection SCOPES for the flow structural punctuation, keyed by the OPEN bracket. The flow
+  // region the highlighter derives (gen-tm §2c) otherwise paints every `{ } [ ] ,` as a generic
+  // `punctuation.${lang}` — graded only at the FAMILY tier. Declaring a `begin`/`end`/`separator`
+  // scope here lets the open/close brackets and the in-collection comma carry the SPECIFIC
+  // convention (a `{…}` is a "mapping", a `[…]` a "sequence" — language-FLAVOURED names that must
+  // come from the grammar, not the neutral engine). `keyValue` (optional) re-scopes the `:`
+  // key/value separator inside a flow mapping, and `explicitKey` the `?` explicit-key indicator.
+  // Scope strings are WITHOUT the trailing `.${lang}` segment (gen-tm appends it, like
+  // `blockScalar.indicatorScope`). Absent → the generic `punctuation.${lang}` (legacy). A bracket
+  // pair with no entry in `byOpen` likewise falls back to the generic scope, so partial
+  // declarations are safe. (e.g. YAML: `{` → punctuation.definition.mapping.begin, `}` → …end,
+  // `,` in `{…}` → punctuation.separator.mapping; `[` → …sequence.begin, etc.)
+  flowScopes?: {
+    byOpen: Record<string, { begin: string; end: string; separator: string }>;
+    keyValue?: string;     // the flow-mapping `:` key/value separator (e.g. punctuation.separator.key-value)
+    explicitKey?: string;  // the flow `?` explicit-key indicator (e.g. punctuation.definition.key-value)
+  };
   comment?: string;       // line-comment introducer ignored for indentation (e.g. '#')
   // Block scalars (YAML `|` / `>`): when the rest of a line is an introducer + indicators, the
   // following more-indented lines are verbatim content emitted as ONE token (like raw-text, but

yaml.tmLanguage.json

@@ -755,7 +755,7 @@
       "begin": ":(?=[\\s\\[\\],{}]|$)",
       "beginCaptures": {
         "0": {
-          "name": "punctuation.yaml"
+          "name": "punctuation.separator.key-value.yaml"
         }
       },
       "end": "(?=[\\[\\],{}])",
@@ -772,7 +772,7 @@
       "begin": "(?<=[\"'\\[\\],{}])[\\t ]*:|(?<=^)[\\t ]*:",
       "beginCaptures": {
         "0": {
-          "name": "punctuation.yaml"
+          "name": "punctuation.separator.key-value.yaml"
         }
       },
       "end": "(?=[\\[\\],{}])",
@@ -837,13 +837,13 @@
       "begin": "\\[",
       "beginCaptures": {
         "0": {
-          "name": "punctuation.yaml"
+          "name": "punctuation.definition.sequence.begin.yaml"
         }
       },
       "end": "\\]",
       "endCaptures": {
         "0": {
-          "name": "punctuation.yaml"
+          "name": "punctuation.definition.sequence.end.yaml"
         }
       },
       "patterns": [
@@ -852,7 +852,7 @@
         },
         {
           "match": ",",
-          "name": "punctuation.yaml"
+          "name": "punctuation.separator.sequence.yaml"
         },
         {
           "include": "#flow-sequence-map-key"
@@ -874,7 +874,7 @@
           "begin": "\\?(?=[\\s\\[\\],{}]|$)",
           "beginCaptures": {
             "0": {
-              "name": "punctuation.yaml"
+              "name": "punctuation.definition.key-value.yaml"
             }
           },
           "end": "(?=[\\[\\],{}])",
@@ -936,13 +936,13 @@
       "begin": "\\{",
       "beginCaptures": {
         "0": {
-          "name": "punctuation.yaml"
+          "name": "punctuation.definition.mapping.begin.yaml"
         }
       },
       "end": "\\}",
       "endCaptures": {
         "0": {
-          "name": "punctuation.yaml"
+          "name": "punctuation.definition.mapping.end.yaml"
         }
       },
       "patterns": [
@@ -951,7 +951,7 @@
         },
         {
           "match": ",",
-          "name": "punctuation.yaml"
+          "name": "punctuation.separator.mapping.yaml"
         },
         {
           "include": "#flow-mapping-map-key"
@@ -973,7 +973,7 @@
           "begin": "\\?(?=[\\s\\[\\],{}]|$)",
           "beginCaptures": {
             "0": {
-              "name": "punctuation.yaml"
+              "name": "punctuation.definition.key-value.yaml"
             }
           },
           "end": "(?=[\\[\\],{}])",

yaml.ts

@@ -597,6 +597,22 @@ const indent: IndentConfig = {
   newlineToken: 'Newline',
   flowOpen: ['[', '{'],
   flowClose: [']', '}'],
+  // SPECIFIC scopes for the flow structural punctuation `{ } [ ] ,` (the TextMate / maintained-RedCMD
+  // convention), keyed by the OPEN bracket. A `{…}` is a MAPPING, a `[…]` a SEQUENCE — these are
+  // YAML-flavoured names (a JS `{}` is an "object", a `[]` an "array"), so they belong in the grammar,
+  // not the agnostic engine; gen-tm only threads them onto the flow region's begin/end/separator
+  // captures and appends the `.yaml` suffix. Without this every flow bracket/comma was a generic
+  // `punctuation.yaml` (graded only FAMILY); declaring them makes the flowPunct role grade EXACT and
+  // gives bracket-pair colourisers the begin/end pairing. The `:` key/value separator and `?`
+  // explicit-key indicator (NOT graded by the oracle, but coloured for consistency with official).
+  flowScopes: {
+    byOpen: {
+      '{': { begin: 'punctuation.definition.mapping.begin', end: 'punctuation.definition.mapping.end', separator: 'punctuation.separator.mapping' },
+      '[': { begin: 'punctuation.definition.sequence.begin', end: 'punctuation.definition.sequence.end', separator: 'punctuation.separator.sequence' },
+    },
+    keyValue: 'punctuation.separator.key-value',
+    explicitKey: 'punctuation.definition.key-value',
+  },
   comment: '#',
   blockScalar: { introducers: ['|', '>'], token: 'BlockScalar', documentMarkers: ['---', '...'], indicatorScope: 'keyword.control.flow.block-scalar' },
   compactIndicators: ['-', '?'],