Highlight YAML document markers (--- / ...) as document structure

`---` and `...` were painted as plain scalars (string.unquoted): the DocStart/DocEnd
tokens existed but the plain-scalar catch-all (rank 8.8) out-ranked them, and they
carried a punctuation scope rather than the document scope. Now they:
  - require trailing whitespace/EOL so `---foo` / `...bar` stay plain scalars,
  - scope as entity.other.document.begin / .end (the maintained-grammar convention),
  - out-rank the plain scalar in scopeOrder so the marker wins the `---`/`...` match.

Also fixes the identToken fallback that mis-tagged `---` as the bare identifier (it
classified as variable.other and got `.readwrite` appended to its scope): the
fallback now requires the token's RESOLVED scope to be variable.other, so an
explicitly scoped token can never be mistaken for the identifier. And the Monarch /
tree-sitter generators learn the new entity.other.document scope (delimiter /
@punctuation.delimiter) so all three targets keep painting the markers.

YAML scope-gap doc.marker 0/149 -> 149/149; parser unchanged (the lookahead only
narrows where the marker matches). Other six grammars byte-identical.

Refs #12

Author: johnsoncodehk

src/gen-monarch.ts

@@ -125,6 +125,7 @@ function scopeToMonarch(scope: string): string {
   if (scope.startsWith('storage')) return 'keyword';
   if (scope.startsWith('keyword.operator')) return 'operator';
   if (scope.startsWith('keyword')) return 'keyword';
+  if (scope.startsWith('entity.other.document')) return 'delimiter';  // YAML --- / ... markers
   if (scope.startsWith('entity.name.function.decorator')) return 'annotation';
   if (scope.startsWith('entity.name.function')) return 'identifier';
   if (scope.startsWith('entity.name.type')) return 'type.identifier';

src/gen-tm.ts

@@ -4262,8 +4262,15 @@ export function generateTmLanguage(grammar: CstGrammar, langName: string): TmGra
   // punctuation tokens (e.g. JSX's `/>` / `</`, which also classify as
   // `variable.other` and would otherwise be mistaken for the identifier pattern,
   // corrupting every ident-derived pattern).
+  // The fallback (a token whose PATTERN classifies as variable.other) also requires the token's
+  // RESOLVED scope to be variable.other — else a grammar with no identifier token (YAML) picks the
+  // first punctuation token whose pattern is unrecognised (e.g. `---`, classified variable.other),
+  // mis-tagging it as the identifier and appending `.readwrite` to its real scope. An explicitly
+  // scoped token (entity.other.document, variable.other.alias, …) is excluded; a true bare
+  // identifier (JS `Ident`, no explicit scope) still qualifies.
   const identToken = grammar.tokens.find(t => t.identifier)
-    ?? grammar.tokens.find(t => classifyToken(t, { explicitScope: false }).scope === 'variable.other');
+    ?? grammar.tokens.find(t => classifyToken(t, { explicitScope: false }).scope === 'variable.other'
+      && classifyToken(t).scope === 'variable.other');
   // Widen the identifier pattern so non-ASCII names (`Ω`, Cyrillic `А`) are scoped,
   // matching the parser lexer's Unicode fallback. This widened form is used only in
   // TextMate (Oniguruma) output, never by the lexer.
@@ -7137,6 +7144,10 @@ export function generateTmLanguage(grammar: CstGrammar, langName: string): TmGra
     // scalar it overlaps, so it must be tried first. (Markup tag names live inside begin/end
     // regions, never as a top-level include, so this only ranks the YAML-style key scalar.)
     if (entry?.match && scope.startsWith('entity.name.tag')) return 0.9;
+    // A document marker (`---`/`...`, scoped entity.other.document) is a `lit`+lookahead token that
+    // OVERLAPS the plain scalar (`---` also matches PLAIN_HEAD), so it must out-rank the plain-scalar
+    // catch-all (8.8) — else `---` paints as string.unquoted. Its lookahead pins it to a real marker.
+    if (entry?.match && scope.startsWith('entity.other.document')) return 0.95;
     // An UNQUOTED plain-scalar catch-all (`string.unquoted`) is the least-specific scalar shape
     // (it has no opening delimiter and matches almost any bare run), so — unlike a quoted/regex
     // string — it must rank AFTER the typed-literal scalars (constant.numeric / constant.language)

src/gen-treesitter.ts

@@ -902,6 +902,7 @@ function scopeToCapture(scope: string): string | null {
     ['entity.name.function', '@function'],
     ['entity.name.type', '@type'],
     ['entity.name', '@constructor'],
+    ['entity.other.document', '@punctuation.delimiter'],  // YAML --- / ... markers
     ['entity.other.property', '@property'],
     ['support.type.primitive', '@type.builtin'],
     ['support.type', '@type.builtin'],

yaml.monarch.json

@@ -711,14 +711,14 @@
         }
       ],
       [
-        "---",
+        "---(?=[\\t ]|\\r|\\n|$)",
         {
           "token": "delimiter",
           "switchTo": "@value"
         }
       ],
       [
-        "\\.\\.\\.",
+        "\\.\\.\\.(?=[\\t ]|\\r|\\n|$)",
         {
           "token": "delimiter",
           "switchTo": "@value"

yaml.tmLanguage.json

@@ -27,6 +27,12 @@
     {
       "include": "#key"
     },
+    {
+      "include": "#docstart"
+    },
+    {
+      "include": "#docend"
+    },
     {
       "include": "#dquote"
     },
@@ -51,12 +57,6 @@
     {
       "include": "#plain"
     },
-    {
-      "include": "#docstart"
-    },
-    {
-      "include": "#docend"
-    },
     {
       "include": "#punctuation"
     },
@@ -81,12 +81,12 @@
   ],
   "repository": {
     "docstart": {
-      "name": "punctuation.definition.directives-end.readwrite.yaml",
-      "match": "---"
+      "name": "entity.other.document.begin.yaml",
+      "match": "---(?=[\\t ]|\\r|\\n|$)"
     },
     "docend": {
-      "name": "punctuation.definition.document-end.yaml",
-      "match": "\\.\\.\\."
+      "name": "entity.other.document.end.yaml",
+      "match": "\\.\\.\\.(?=[\\t ]|\\r|\\n|$)"
     },
     "yamldirective": {
       "name": "keyword.other.directive.yaml",
@@ -203,7 +203,7 @@
       "match": "(?:[^\\t\\n\\f\\r \\-?:,\\[\\]{}#&*!|>'\"%@`]|[\\-?:](?=[^\\t\\n\\f\\r ,\\[\\]{}]))(?:[^:#\\n]|:(?=[^\\t\\n\\f\\r ])|#(?<=[^\\t\\n\\f\\r ]#))*"
     },
     "indent": {
-      "name": "variable.other.yaml",
+      "name": "variable.other.readwrite.yaml",
       "match": "(?!)"
     },
     "dedent": {

yaml.ts

@@ -30,8 +30,14 @@ const hexDigit = oneOf(digit, range('A', 'F'), range('a', 'f'));
 const whitespace = oneOf('\t', '\n', '\f', '\r', ' ');
 const nonWhitespace = noneOf(whitespace);
 const hashAfterNonSpace = seq('#', precededBy(seq(nonWhitespace, '#')));
-const DocStart = token(lit('---'), { scope: 'punctuation.definition.directives-end' });
-const DocEnd = token(lit('...'), { scope: 'punctuation.definition.document-end' });
+// Document markers: `---` (directives end / document begin) and `...` (document end). Both must be
+// followed by whitespace or EOL — `---foo` / `...bar` are plain scalars, not markers — so the
+// lookahead keeps the marker from stealing a plain scalar's leading dashes/dots. Scoped
+// `entity.other.document.*` (the maintained-grammar convention) so the highlighter paints them as
+// document structure, not as a string.
+const docMarkerEnd = followedBy(alt(oneOf('\t', ' '), '\r', '\n', end()));
+const DocStart = token(seq('---', docMarkerEnd), { scope: 'entity.other.document.begin' });
+const DocEnd = token(seq('...', docMarkerEnd), { scope: 'entity.other.document.end' });
 const Comment = token(seq('#', star(noneOf('\n'))), { skip: true, scope: 'comment.line.number-sign' });
 // Double-quoted scalar body. The escape set is FIXED (YAML 1.2 §5.7): a `\` may only precede one
 // of `0 a b t n v f r e " / \ N _ L P`, a literal space/tab, a `x`+2 / `u`+4 / `U`+8 hex escape, or