Highlight YAML invalid escapes (#5) and glued directive '#' (#8)

Two more monogram#12 items, highlighter-only:

#5 — a double-quoted scalar's INVALID escape (`"quoted \' scalar"`: `\'` is not a
valid YAML escape) was left as plain string content. The derived quoted-string
region now emits an `invalid.illegal.constant.character.escape` pattern after the
valid-escape pattern, so an unrecognised `\.` is highlighted (the valid escapes
still win the leftmost tie). Only for backslash-escape strings, not doubled-
delimiter (`''`) ones, where a lone `\` is literal.

#8 — `%YAML 1.1#...`: the glued `#...` (no preceding space) was scoped as a comment.
Per YAML §6.6 a `#` is a comment only at line start or after whitespace, so the
Comment token gains a `notPrecededBy(nonWhitespace)` guard (a portable fixed-width
`(?<!\S)`). Plain scalars already keep a glued `#` as content; this stops the Comment
token from claiming a glued `#` a directive left behind. Parser is unaffected
(src-coverage-yaml alignment still 100%) — the lexer skips comments via the indent
config, so this only changes the highlighter's Comment pattern.

Also un-flag the now-fixed regression cases (#5/#6/#7/#8/#9 drop `bug:true` → hard-
gated; #6/#7/#9 were stale from the earlier fixes). yaml-issue12-regressions now
8 pass / 2 known-bug (#4, #10) / 0 regression. The other six grammars regenerate
byte-identical; agnostic 9/9; sanity 15/15; RedCMD Onigmo diagnostics clean.

Refs #12

Author: johnsoncodehk

src/gen-tm.ts

@@ -4582,17 +4582,26 @@ export function generateTmLanguage(grammar: CstGrammar, langName: string): TmGra
       // JS/TS strings declare an @escape and take the escapePattern branch above, so they are
       // untouched; this fires only for the otherwise-flat quoted-string token.
       const { delim, escape } = tokenPatternQuoteDelimAndEscape(tok)!;
+      const isDoubledDelim = escape === escapeRegex(delim) + escapeRegex(delim);
+      // A BACKSLASH-based escape string (e.g. YAML double-quoted): the valid escapes are scoped
+      // constant.character.escape; any OTHER `\.` is an INVALID escape and must still be highlighted
+      // (monogram#12 #5 — `"quoted \' scalar"`: `\'` is not a valid YAML escape but must not read as
+      // plain string content). The invalid catch is listed AFTER the valid pattern (same start → the
+      // valid escape wins the tie; only an unrecognised `\.` falls to it). NOT added for a doubled-
+      // delimiter escape (`''`), where a lone `\` is literal content, not an escape.
+      const escapePatterns: TmPattern[] = [{ match: escape, name: `constant.character.escape.${langName}` }];
+      if (!isDoubledDelim) escapePatterns.push({ match: '\\\\.', name: `invalid.illegal.constant.character.escape.${langName}` });
       const region: TmPattern = {
         name: `${scope}.${langName}`,
         begin: escapeRegex(delim),
         beginCaptures: { '0': { name: `punctuation.definition.string.begin.${langName}` } },
         end: escapeRegex(delim),
         endCaptures: { '0': { name: `punctuation.definition.string.end.${langName}` } },
-        patterns: [{ match: escape, name: `constant.character.escape.${langName}` }],
+        patterns: escapePatterns,
       };
       // A doubled-delimiter escape (`''`) shares its first char with the region's `end`, so the
       // escape pattern must be tried BEFORE the end (else `'a''b'` closes at the inner pair).
-      if (escape === escapeRegex(delim) + escapeRegex(delim)) region.applyEndPatternLast = true;
+      if (isDoubledDelim) region.applyEndPatternLast = true;
       repository[key] = region;
       topPatterns.push({ include: `#${key}` });
       rememberLiteralKey(scope, key, tok.name);

test/yaml-issue12-regressions.ts

@@ -57,23 +57,23 @@ export const cases: Issue12Case[] = [
 
   { id: '#5', title: 'an escape inside a double-quoted scalar is highlighted',
     src: 'double: "quoted \\\' scalar"\n', at: "\\'", col: 0,
-    should: (s) => isEscapeOrInvalid(s), why: '`\\\'` in a double-quoted scalar is an (invalid) escape; it should be constant.character.escape or invalid, not plain string', bug: true },
+    should: (s) => isEscapeOrInvalid(s), why: '`\\\'` in a double-quoted scalar is an (invalid) escape; it should be constant.character.escape or invalid, not plain string' },
 
   { id: '#6', title: 'a `!` opening a plain-scalar CONTINUATION line is string content, not a tag',
     src: 'safe: a!"#$%&\'()*+,-./09:;<=>?@AZ\n     !"#$%&\'()*+,-./09:;<=>?@AZ\n', at: '     !', col: 5,
-    should: (s) => isString(s), why: 'the 2nd line folds into the one multi-line plain scalar (CST: one `scalar` token); the leading `!` is content, not storage.type.tag', bug: true },
+    should: (s) => isString(s), why: 'the 2nd line folds into the one multi-line plain scalar (CST: one `scalar` token); the leading `!` is content, not storage.type.tag' },
 
   { id: '#7', title: '`42` inside a multi-line plain-scalar KEY is string content, not a number',
     src: '? a\n  true\n: null\n  d\n? e\n  42\n', at: '  42', col: 2,
-    should: (s) => notNumber(s), why: 'the explicit key is the plain scalar "e 42" (CST/AST: one scalar that resolves to the STRING "e 42"); `42` is not a numeric literal', bug: true },
+    should: (s) => notNumber(s), why: 'the explicit key is the plain scalar "e 42" (CST/AST: one scalar that resolves to the STRING "e 42"); `42` is not a numeric literal' },
 
   { id: '#8', title: '`#...` immediately after `%YAML 1.1` is directive content, not a comment',
     src: '%YAML 1.1#...\n', at: '#...', col: 0,
-    should: (s) => notComment(s), why: 'no whitespace precedes the `#`, so it is not a comment; CST emits one `directive` token `%YAML 1.1#...`', bug: true },
+    should: (s) => notComment(s), why: 'no whitespace precedes the `#`, so it is not a comment; CST emits one `directive` token `%YAML 1.1#...`' },
 
   { id: '#9', title: '`%YAML` after document content is plain-scalar text, not a directive',
     src: '---\nscalar\n%YAML 1.2\n', at: '%YAML', col: 0,
-    should: (s) => notDirective(s) && isString(s), why: 'a directive cannot appear after content; `scalar\\n%YAML 1.2` folds into one plain scalar (CST: one `scalar` token)', bug: true },
+    should: (s) => notDirective(s) && isString(s), why: 'a directive cannot appear after content; `scalar\\n%YAML 1.2` folds into one plain scalar (CST: one `scalar` token)' },
 
   { id: '#10', title: 'a `#` line inside a `|5` block scalar body is string content, not a comment',
     src: 'abc: |5\n      # string 6\n     # string 5\n    #comment 4\n   #comment 3\n', at: '# string 5', col: 0,

yaml.tmLanguage.json

@@ -217,7 +217,7 @@
     },
     "comment": {
       "name": "comment.line.number-sign.yaml",
-      "match": "#[^\\n]*"
+      "match": "(?<![^\\t\\n\\f\\r ])#[^\\n]*"
     },
     "dquotekey": {
       "name": "entity.name.tag.yaml",
@@ -280,6 +280,10 @@
         {
           "match": "\\\\(?:[0abtnvfre\"/\\\\N_LP \\t]|x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|U[0-9A-Fa-f]{8}|\\r?\\n)",
           "name": "constant.character.escape.yaml"
+        },
+        {
+          "match": "\\\\.",
+          "name": "invalid.illegal.constant.character.escape.yaml"
         }
       ]
     },

yaml.ts

@@ -10,7 +10,7 @@
 import {
   token, rule, defineGrammar, alt, many, many1, opt, not, noCommentBefore, noMultilineFlowBefore,
   altPattern, optPattern, seq, oneOf, noneOf, range, star, plus, repeat, followedBy, notFollowedBy,
-  precededBy, never, end,
+  precededBy, notPrecededBy, never, end,
 } from './src/api.ts';
 import type { IndentConfig } from './src/types.ts';
 
@@ -38,7 +38,12 @@ const hashAfterNonSpace = seq('#', precededBy(seq(nonWhitespace, '#')));
 const docMarkerEnd = followedBy(altPattern(oneOf('\t', ' '), '\r', '\n', end()));
 const DocStart = token(seq('---', docMarkerEnd), { scope: 'entity.other.document.begin' });
 const DocEnd = token(seq('...', docMarkerEnd), { scope: 'entity.other.document.end' });
-const Comment = token(seq('#', star(noneOf('\n'))), { skip: true, scope: 'comment.line.number-sign' });
+// A `#` is a comment indicator only at line start or AFTER whitespace (YAML §6.6); a `#` glued to a
+// non-space char is content, not a comment (`a#b` is a plain scalar, `%YAML 1.1#…` keeps the `#…` as
+// directive content — monogram#12 #8). The `notPrecededBy(nonWhitespace)` guard (a fixed-width, portable
+// `(?<!\S)`) enforces that. Plain scalars already keep a glued `#` via `hashAfterNonSpace`; this stops
+// the Comment token from claiming a glued `#` that a non-scalar token (a directive) left behind.
+const Comment = token(seq(notPrecededBy(nonWhitespace), '#', star(noneOf('\n'))), { skip: true, scope: 'comment.line.number-sign' });
 // Double-quoted scalar body. The escape set is FIXED (YAML 1.2 §5.7): a `\` may only precede one
 // of `0 a b t n v f r e " / \ N _ L P`, a literal space/tab, a `x`+2 / `u`+4 / `U`+8 hex escape, or
 // a LINE BREAK (`\`-at-EOL = line continuation). Any other `\.` (`\.`, `\'`, `\q`, `\x4`) is illegal