Extend YAML role-oracle to grade block-indicator/tag/directive/flow scopes

The scope-gap role-oracle emitted tokens for only 7 coarse roles, so four
YAML construct classes were never graded — any grammar scope silently
passed: block-scalar indicator (|/>), node tags (!!str), directives
(%YAML), and flow punctuation ([ ] { } ,). Add a role + neutral grading
entry for each (public TextMate convention = exact, the coarse alternative
each grammar may use = family) and split the block-scalar introducer from
its body so the introducer's scope is graded on its own.

The YAML headline drops from a blind-spot-inflated 100% to an honest 99.1%
(Monogram, vs official 98.9%) over 2034 graded tokens (was 1479): the
block-indicator role confirms the keyword.control.flow fix, tag/directive/
flow are family-correct (coarse but right family), and one real bug
surfaces — a document-root block scalar with column-0 content (--- | or
bare |) leaks its body to the directive rule.

Author: johnsoncodehk

test/scope-roles.ts

@@ -84,6 +84,13 @@ export const R = {
   alias: 'alias',
   docMarker: 'doc.marker',
   escape: 'escape',
+  // YAML scopes the coarse key/value/comment oracle never emitted a token for, so ANY grammar
+  // scope silently passed: block-scalar indicator (|/>), node tags (!!str), directives (%YAML),
+  // flow punctuation ([ ] { } ,). Emitting these makes those blind spots graded.
+  blockIndicator: 'block.indicator',
+  tagType: 'tag.type',
+  directive: 'directive',
+  flowPunct: 'flow.punct',
   // lexical floor
   op: 'op',
   punct: 'punct',
@@ -255,6 +262,16 @@ export const ROLE_SPEC: Record<RoleName, RoleSpec> = {
   // A string escape (`\n`, `\t`, `\\`, `\uXXXX`): both engines use constant.character.escape — for a
   // VALUE; a mis-scope (e.g. inside a quoted KEY painted flat entity.name.tag) reads WRONG.
   [R.escape]: { tier: 'strict', desc: 'string escape sequence (\\n, \\t, …)', exact: ['constant.character.escape'], family: ['constant.character', 'constant.other'] },
+  // ── YAML blind-spot roles (block indicator / tag / directive / flow punctuation) ───────────────
+  // Written from the public YAML TextMate convention, NOT from either grammar: the block-scalar
+  // `|`/`>` (+chomp/indent) is keyword.control.flow.block-scalar; a node tag is storage.type.tag;
+  // a directive is keyword.other.directive; flow brackets/separators are punctuation.definition/
+  // .separator. family accepts the defensible alternatives each grammar may legitimately use
+  // (e.g. a coarse generic `punctuation` for flow, the split punctuation.definition.tag for a tag).
+  [R.blockIndicator]: { tier: 'strict', desc: 'YAML block-scalar indicator (|/>, chomping, indent)', exact: ['keyword.control.flow.block-scalar'], family: ['keyword', 'storage.modifier', 'constant.numeric', 'punctuation.definition'] },
+  [R.tagType]: { tier: 'strict', desc: 'YAML node tag (!!str / !foo / !<verbatim>)', exact: ['storage.type.tag'], family: ['storage.type', 'storage', 'entity.name.type', 'support.type', 'keyword.other', 'punctuation.definition.tag'] },
+  [R.directive]: { tier: 'strict', desc: 'YAML directive name (%YAML / %TAG / %…)', exact: ['keyword.other.directive'], family: ['keyword.other', 'keyword', 'punctuation.definition.directive'] },
+  [R.flowPunct]: { tier: 'strict', desc: 'YAML flow punctuation ([ ] { } ,)', exact: ['punctuation.definition', 'punctuation.separator'], family: ['punctuation'] },
 
   // ── lexical floor: reported, excluded from the headline ───────────────────────
   [R.op]: { tier: 'lexical', desc: 'operator punctuation (+ - * = => …)', exact: ['keyword.operator'], family: ['keyword', 'punctuation', 'storage', 'meta'] },
@@ -363,7 +380,9 @@ export function roleFamily(role: RoleName): Family {
     case R.propAccess: case R.propDecl: case R.methodCall: return 'property';
     case R.litString: case R.litNumber: case R.litRegex: case R.litBigint: case R.litTemplate: case R.escape: return 'literal';
     case R.comment: return 'comment';
-    case R.docMarker: return 'keyword';
+    case R.docMarker: case R.blockIndicator: case R.directive: return 'keyword';
+    case R.tagType: return 'type';
+    case R.flowPunct: return 'punct';
     case R.kwControl: case R.kwOperator: case R.kwStorage: case R.kwOther: case R.constBuiltin: case R.thisSuper: return 'keyword';
     case R.op: case R.punct: return 'punct';
     default: return 'value'; // funcDecl, parameter, varDecl, valueRef, classRef, namespace, enumMember, importBinding
@@ -436,6 +455,14 @@ if (import.meta.url === `file://${process.argv[1]}`) {
     [R.opCompare, 'string.regexp.ts', 'wrong'],
     // class.ref is the one value spot where a type scope is right (new X)
     [R.classRef, 'entity.name.type.ts', 'exact'],
+    // YAML blind-spot roles: public convention = exact; the coarse alternative each grammar may use = family
+    [R.blockIndicator, 'keyword.control.flow.block-scalar.yaml', 'exact'],
+    [R.blockIndicator, 'storage.modifier.chomping-indicator.yaml', 'family'],
+    [R.tagType, 'storage.type.tag.yaml', 'exact'],
+    [R.tagType, 'punctuation.definition.tag.begin.yaml', 'family'],
+    [R.directive, 'keyword.other.directive.yaml', 'exact'],
+    [R.flowPunct, 'punctuation.definition.sequence.begin.yaml', 'exact'],
+    [R.flowPunct, 'punctuation.yaml', 'family'],
   ];
   let pass = 0;
   const fails: string[] = [];

test/yaml-oracle.ts

@@ -37,9 +37,29 @@ export function yamlOracle(text: string): GoldToken[] {
     ESCAPE.lastIndex = 0;
     while ((m = ESCAPE.exec(seg))) out.push({ start: r[0] + m.index, end: r[0] + m.index + m[0].length, text: m[0], role: R.escape });
   };
+  // A VALUE-position block scalar (`|`/`>`): split the introducer (a structural control sigil) from
+  // the verbatim body. introducer = `|`/`>` + chomping/indent on the header line; body = the lines
+  // below. (A block scalar in KEY position stays ONE tagName token — the whole scalar is the key name.)
+  const isBlockScalar = (n: any) => n?.type === 'BLOCK_LITERAL' || n?.type === 'BLOCK_FOLDED';
+  const pushBlockScalar = (node: any): void => {
+    const r = node?.range;
+    if (!r || r[1] <= r[0]) return;
+    const introLen = /^[|>][-+0-9]*/.exec(text.slice(r[0], r[1]))?.[0].length ?? 1;
+    out.push({ start: r[0], end: r[0] + introLen, text: text.slice(r[0], r[0] + introLen), role: R.blockIndicator });
+    // content body = the first NON-BLANK char after the header line (skip leading indent + blank
+    // lines — they are structural whitespace; grading them would test the INDENT's scope, not the
+    // scalar content, and whitespace is visually colourless either way).
+    const nl = text.indexOf('\n', r[0]);
+    let contentStart = nl >= 0 ? nl + 1 : r[0] + introLen;
+    while (contentStart < r[1] && (text[contentStart] === ' ' || text[contentStart] === '\t' || text[contentStart] === '\n')) contentStart++;
+    if (r[1] > contentStart) out.push({ start: contentStart, end: r[1], text: text.slice(contentStart, r[1]), role: R.litString });
+  };
   const walk = (node: any, isKey: boolean): void => {
     if (!node) return;
-    if (isScalar(node)) { push(node, isKey ? R.tagName : valueRole(node.value)); pushEscapes(node); }
+    if (isScalar(node)) {
+      if (!isKey && isBlockScalar(node)) pushBlockScalar(node);
+      else { push(node, isKey ? R.tagName : valueRole(node.value)); pushEscapes(node); }
+    }
     else if (isMap(node)) for (const p of node.items) { walk(p.key, true); walk(p.value, false); }
     else if (isSeq(node)) for (const it of node.items) walk(it, false);
   };
@@ -84,6 +104,26 @@ export function yamlOracle(text: string): GoldToken[] {
     if (inSpan(m.index)) continue;
     out.push({ start: m.index, end: m.index + m[0].length, text: m[0], role: R.comment });
   }
+
+  // Node tags (!!str / !foo / !<verbatim> / ! non-specific): the sigil + handle/suffix at a node
+  // boundary (line start / whitespace / flow open). A `!` inside a scalar span is content, not a tag.
+  for (const tm of text.matchAll(/(?<=^|[\s[{,])!(?:<[^>\n]*>|[^\s[\]{},]*)/gm)) {
+    if (tm.index === undefined || inSpan(tm.index)) continue;
+    out.push({ start: tm.index, end: tm.index + tm[0].length, text: tm[0], role: R.tagType });
+  }
+  // Directives (%YAML / %TAG / %FOO): the directive NAME after `%`, line-start only (a directive
+  // owns its line). A `%`-led line folded into a plain scalar body sits in a span → inSpan skips it.
+  for (const dm of text.matchAll(/^%(\w[\w-]*)/gm)) {
+    if (dm.index === undefined || inSpan(dm.index)) continue;
+    out.push({ start: dm.index + 1, end: dm.index + 1 + dm[1].length, text: dm[1], role: R.directive });
+  }
+  // Flow punctuation ([ ] { } ,): structural only OUTSIDE a scalar span — a `,` inside a plain scalar
+  // is content (`a, b` is one plain scalar), and `[`/`{` cannot start a plain scalar, so a bare one
+  // always opens a flow collection. inSpan excludes the in-scalar occurrences.
+  for (const fm of text.matchAll(/[[\]{},]/g)) {
+    if (fm.index === undefined || inSpan(fm.index)) continue;
+    out.push({ start: fm.index, end: fm.index + 1, text: fm[0], role: R.flowPunct });
+  }
   out.sort((a, b) => a.start - b.start);
   return out;
 }