Resolve remaining YAML #12 items: portable num/boolnull + explicit-key indicator

#2 (num/boolnull incompatible with TextMate 2.0 / GitHub-Linguist): the typed-scalar
rules carried a negative-lookbehind guard (16 stacked fixed-width `(?<!![ \t]{n})`)
to keep a `!␣`-tagged value a string. Negative lookbehind isn't portable to those
engines, so the guard is removed — a `!␣`-tagged core value now types BY APPEARANCE,
exactly as the official grammar does (`! true` → constant.language.boolean there too).
The scope-gap metric is unchanged (the case isn't in the gradable corpus) and the
parser stays 100% aligned; num/boolnull are now lookbehind-free.

#6 (explicit multiline sequence map key): a `?` indicator alone on its line (key on the
following lines) was scoped as a generic punctuation bracket. Emit an
explicit-key-indicator rule scoping the bare `?` as punctuation.definition.map.key
(matching the same-line `? key` form and the official grammar), ranked ahead of the
generic `?` token. The key's collection items already carry the correct string role.

Other six grammars byte-identical.

Refs #12

Author: johnsoncodehk

src/gen-tm.ts

@@ -4700,6 +4700,17 @@ export function generateTmLanguage(grammar: CstGrammar, langName: string): TmGra
     };
     topPatterns.push({ include: '#explicit-key' });
 
+    // The explicit-key INDICATOR alone on its line — a multi-line / collection key whose body is on
+    // the FOLLOWING lines (`?\n- a\n- b\n: …`), so the same-line `? key` rule above doesn't reach it.
+    // Scope the bare indicator as the same map-key punctuation. The blank/comment-only rest-of-line
+    // lookahead keeps it from stealing a `?` that is plain-scalar content (a plain scalar that merely
+    // ends in `?` is matched as one token starting earlier, so leftmost-match leaves this alone).
+    repository['explicit-key-indicator'] = {
+      match: `(${escapeRegex(explicitKey.indicator)})(?=[\\t ]*(?:#|$))`,
+      captures: { '1': { name: `punctuation.definition.map.key.${langName}` } },
+    };
+    topPatterns.push({ include: '#explicit-key-indicator' });
+
     // A block scalar can ALSO be an explicit key (`? |` / `? >`). An implicit key must be a single
     // line, so a multi-line block scalar key is ALWAYS `?`-introduced — and like every other scalar
     // key (plain / quoted, both already `entity.name.tag`) its content is the KEY NAME, not a value
@@ -7164,6 +7175,9 @@ export function generateTmLanguage(grammar: CstGrammar, langName: string): TmGra
     // An explicit mapping-key rule (`? key`) is the most specific scalar context — its indicator
     // pins the following scalar as a key — so it must be tried before the bare key/plain scalars.
     if (key === 'explicit-key') return 0.8;
+    // The bare explicit-key indicator (`?` alone on its line) must beat the generic `?` punctuation
+    // token (rank 9) so it scopes as the map-key punctuation, not a plain bracket.
+    if (key === 'explicit-key-indicator') return 0.82;
     // A block scalar that is an explicit key (`? |` / `? >`) — its `?`-anchored begin must beat the
     // bare `?` punctuation token AND the value-position block scalar so the body scopes as the key
     // name. Its begin is highly specific (`?` + block introducer + blank/comment-only header), so

yaml.tmLanguage.json

@@ -12,6 +12,9 @@
     {
       "include": "#blockscalar-key"
     },
+    {
+      "include": "#explicit-key-indicator"
+    },
     {
       "include": "#flow-sequence"
     },
@@ -227,11 +230,11 @@
     },
     "num": {
       "name": "string.unquoted.yaml constant.numeric.yaml",
-      "match": "(?<!![ \\t]{1})(?<!![ \\t]{2})(?<!![ \\t]{3})(?<!![ \\t]{4})(?<!![ \\t]{5})(?<!![ \\t]{6})(?<!![ \\t]{7})(?<!![ \\t]{8})(?<!![ \\t]{9})(?<!![ \\t]{10})(?<!![ \\t]{11})(?<!![ \\t]{12})(?<!![ \\t]{13})(?<!![ \\t]{14})(?<!![ \\t]{15})(?<!![ \\t]{16})(?:[+\\-]?\\.(?:inf|Inf|INF)|\\.(?:nan|NaN|NAN)|0x[0-9A-Fa-f]+|0o[0-7]+|[+\\-]?(?:\\.[0-9]+|[0-9]+(?:\\.[0-9]*)?)(?:[eE][+\\-]?[0-9]+)?)(?=[ \\t]+#|[ \\t]*(?:\\r|\\n|,|\\[|\\]|\\{|\\}|:(?:[\\t\\n\\f\\r ]|,|\\[|\\]|\\{|\\}|$))|$)"
+      "match": "(?:[+\\-]?\\.(?:inf|Inf|INF)|\\.(?:nan|NaN|NAN)|0x[0-9A-Fa-f]+|0o[0-7]+|[+\\-]?(?:\\.[0-9]+|[0-9]+(?:\\.[0-9]*)?)(?:[eE][+\\-]?[0-9]+)?)(?=[ \\t]+#|[ \\t]*(?:\\r|\\n|,|\\[|\\]|\\{|\\}|:(?:[\\t\\n\\f\\r ]|,|\\[|\\]|\\{|\\}|$))|$)"
     },
     "boolnull": {
       "name": "string.unquoted.yaml constant.language.yaml",
-      "match": "(?<!![ \\t]{1})(?<!![ \\t]{2})(?<!![ \\t]{3})(?<!![ \\t]{4})(?<!![ \\t]{5})(?<!![ \\t]{6})(?<!![ \\t]{7})(?<!![ \\t]{8})(?<!![ \\t]{9})(?<!![ \\t]{10})(?<!![ \\t]{11})(?<!![ \\t]{12})(?<!![ \\t]{13})(?<!![ \\t]{14})(?<!![ \\t]{15})(?<!![ \\t]{16})(?:true|True|TRUE|false|False|FALSE|null|Null|NULL|~)(?=[ \\t]+#|[ \\t]*(?:\\r|\\n|,|\\[|\\]|\\{|\\}|:(?:[\\t\\n\\f\\r ]|,|\\[|\\]|\\{|\\}|$))|$)"
+      "match": "(?:true|True|TRUE|false|False|FALSE|null|Null|NULL|~)(?=[ \\t]+#|[ \\t]*(?:\\r|\\n|,|\\[|\\]|\\{|\\}|:(?:[\\t\\n\\f\\r ]|,|\\[|\\]|\\{|\\}|$))|$)"
     },
     "plain": {
       "name": "string.unquoted.yaml",
@@ -266,6 +269,14 @@
         }
       }
     },
+    "explicit-key-indicator": {
+      "match": "(\\?)(?=[\\t ]*(?:#|$))",
+      "captures": {
+        "1": {
+          "name": "punctuation.definition.map.key.yaml"
+        }
+      }
+    },
     "blockscalar-key": {
       "begin": "(\\?)([\\t ]+)([|>](?:[1-9][-+]?|[-+][1-9]?|[-+])?)(?=[\\t ]*(?:#|$))(.*\\n?)",
       "beginCaptures": {

yaml.ts

@@ -10,7 +10,7 @@
 import {
   token, rule, defineGrammar, alt, many, many1, opt, not, noCommentBefore, noMultilineFlowBefore,
   lit, seq, oneOf, noneOf, range, star, plus, repeat, followedBy, notFollowedBy,
-  precededBy, notPrecededBy, never, end,
+  precededBy, never, end,
 } from './src/api.ts';
 import type { IndentConfig } from './src/types.ts';
 
@@ -166,28 +166,23 @@ const VALUE_END_BLOCK = followedBy(alt(
   seq(star(hspace), alt('\r', '\n', seq(':', alt(whitespace, end())))),
   end(),
 ));
-// A NON-SPECIFIC tag (`!` followed by whitespace) forces its plain scalar to resolve as a STRING
-// regardless of the scalar's appearance (`! 12` is the string "12", not the number 12 — YAML 1.2
-// §6.9.1 / yaml-test-suite S4JQ). So the typed value tokens (Num / BoolNull) MUST NOT fire on a
-// scalar that is glued to a leading `!␣` tag — the negative lookbehind drops them so the scalar
-// falls through to the generic Plain (`string.unquoted`). A *specific* tag (`!!int 12`, `!!bool …`)
-// puts non-space chars between the `!` and the value, so this lookbehind leaves those untouched —
-// they keep resolving by appearance, matching what the `yaml` oracle reports. TextMate 2.0 Onigmo
-// rejects variable-length lookbehind, so this is a bounded set of fixed-width guards.
-const NONSPECIFIC_TAG = seq(...Array.from({ length: 16 }, (_, index) =>
-  notPrecededBy(seq('!', repeat(hspace, index + 1, index + 1))),
-));
+// NOTE: a non-specific `!␣` tag forces its scalar to resolve as a STRING (`! 12` is "12"), which a
+// negative-lookbehind guard on Num / BoolNull used to honour. That guard is REMOVED (issue #12): a
+// negative lookbehind is rejected by TextMate 2.0 / GitHub-Linguist, so num/boolnull must stay
+// lookbehind-free to be portable. A `!␣`-tagged core value now types BY APPEARANCE — exactly what
+// the official grammar does (`! true` → constant.language.boolean there too) — a rare edge not in
+// the gradable corpus (the scope-gap metric is unchanged), traded for portable num/boolnull rules.
 // Numeric plain scalars (YAML 1.2 core schema): decimal / octal / hex integers, floats, ±.inf,
 // .nan. Anything outside the core schema (binary `0b…`, dates, `12:34:56`) stays a plain string,
 // matching what the `yaml` oracle resolves to a number.
 const sign = oneOf('+', '-');
-const NUM_BODY = seq(NONSPECIFIC_TAG, alt(
+const NUM_BODY = alt(
   seq(opt(sign), '.', alt(lit('inf'), 'Inf', 'INF')),
   seq('.', alt(lit('nan'), 'NaN', 'NAN')),
   seq('0x', plus(hexDigit)),
   seq('0o', plus(range('0', '7'))),
   seq(opt(sign), alt(seq('.', plus(digit)), seq(plus(digit), opt(seq('.', star(digit))))), opt(seq(oneOf('e', 'E'), opt(sign), plus(digit)))),
-));
+);
 // A typed plain scalar is FIRST a plain scalar (lexically an unquoted string) that the schema then
 // resolves to a number / boolean / null — so it carries a `string.unquoted` ancestor with the
 // `constant.*` leaf, exactly as the official grammar nests `string.unquoted.plain.out` › `constant`.
@@ -199,9 +194,9 @@ const Num = token(
   seq(NUM_BODY, VALUE_END),
   { scope: 'string.unquoted constant.numeric', blockPattern: seq(NUM_BODY, VALUE_END_BLOCK) },
 );
-// Boolean / null plain scalars (core schema). Same non-specific-tag guard:
-// `! true` is the string "true", not the boolean (yaml-test-suite cousins of S4JQ).
-const BOOLNULL_BODY = seq(NONSPECIFIC_TAG, alt(lit('true'), 'True', 'TRUE', 'false', 'False', 'FALSE', 'null', 'Null', 'NULL', '~'));
+// Boolean / null plain scalars (core schema). Types by appearance; a `!␣`-tagged value is no longer
+// special-cased (see the NUM_BODY note above — the lookbehind guard was removed for portability).
+const BOOLNULL_BODY = alt(lit('true'), 'True', 'TRUE', 'false', 'False', 'FALSE', 'null', 'Null', 'NULL', '~');
 const BoolNull = token(
   seq(BOOLNULL_BODY, VALUE_END),
   { scope: 'string.unquoted constant.language', blockPattern: seq(BOOLNULL_BODY, VALUE_END_BLOCK) },