Reject numeric literals followed by ident/digit (JS maximal-munch rule)

A numeric literal may not be immediately followed by an IdentifierStart or
DecimalDigit. Without enforcing this, bad literals like 0b21010 / 0B1102110 /
0o81010 munched a valid prefix and left the rest as a second token, so the
file parsed as two statements instead of being rejected. Append a negative
lookahead (?![0-9A-Za-z_$\\]) to every numeric token; the bad literal then
matches no token and the lexer throws — the correct rejection.

Surfaced and fixed a latent gap: the exponent allowed no numeric separator, so
1.1e10_0 only 'parsed' by splitting into 1.1e10 + _0; exponent now allows
(_[0-9]+)* like the rest.

FP 120->116 (7 over-accepts now rejected incl. 3 multi-file separator tests),
FN stays 0 (3376/3376 valid-code), TP unchanged. Artifacts regenerated.

Author: johnsoncodehk

examples/lezer/tokens.js

@@ -12,7 +12,11 @@
 //   JSDoc: /\/\*\*(?!\/)[\s\S]*?\*\//  (skip)
 //   BlockComment: /\/\*[\s\S]*?\*\//  (skip)
 //   Ident: /(?:[a-zA-Z_$]|\\u[0-9a-fA-F]{4}|\\u\{[0-9a-fA-F]+\})(?:[a-zA-Z0-9_$]|\\u[0-9a-fA-F]{4}|\\u\{[0-9a-fA-F]+\})*/
-//   Number: /[0-9]+(_[0-9]+)*(?:\.[0-9]*(_[0-9]+)*)?(?:[eE][+-]?[0-9]+)?/
+//   HexNumber: /0[xX][0-9a-fA-F]+(_[0-9a-fA-F]+)*(?![0-9A-Za-z_$\\])/
+//   OctalNumber: /0[oO][0-7]+(_[0-7]+)*(?![0-9A-Za-z_$\\])/
+//   BinaryNumber: /0[bB][01]+(_[01]+)*(?![0-9A-Za-z_$\\])/
+//   BigInt: /[0-9]+(_[0-9]+)*n(?![0-9A-Za-z_$\\])/
+//   Number: /[0-9]+(_[0-9]+)*(?:\.[0-9]*(_[0-9]+)*)?(?:[eE][+-]?[0-9]+(_[0-9]+)*)?(?![0-9A-Za-z_$\\])/
 //   String: /"(?:[^"\\]|\\[\s\S])*"|'(?:[^'\\]|\\[\s\S])*'/
 //   Decorator: /@(?:[a-zA-Z_$][a-zA-Z0-9_$.]*)?/
 //

examples/lezer/typescript.grammar

@@ -254,11 +254,11 @@ Program {
   LineComment { "/""/"![\n]* }
   // INCOMPLETE: BlockComment block span "\\/\\*"…"\\*\\/" → external tokenizer.
   // INCOMPLETE: Ident regex /(?:[a-zA-Z_$]|\\u[0-9a-fA-F]{4}|\\u\{[0-9a-fA-F]+\})(?:[a-zA-Z0-9_$]|\\u[0-9a-fA-F]{4}|\\u\{[0-9a-fA-F]+\})*/ exceeds Lezer token syntax; external tokenizer.
-  HexNumber { "0"$[xX]$[0-9a-fA-F]+("_"$[0-9a-fA-F]+)* }
-  OctalNumber { "0"$[oO]$[0-7]+("_"$[0-7]+)* }
-  BinaryNumber { "0"$[bB]$[01]+("_"$[01]+)* }
-  BigInt { $[0-9]+("_"$[0-9]+)*"n" }
-  // INCOMPLETE: Number regex /[0-9]+(_[0-9]+)*(?:\.[0-9]*(_[0-9]+)*)?(?:[eE][+-]?[0-9]+)?/ exceeds Lezer token syntax; external tokenizer.
+  // INCOMPLETE: HexNumber regex /0[xX][0-9a-fA-F]+(_[0-9a-fA-F]+)*(?![0-9A-Za-z_$\\])/ exceeds Lezer token syntax; external tokenizer.
+  // INCOMPLETE: OctalNumber regex /0[oO][0-7]+(_[0-7]+)*(?![0-9A-Za-z_$\\])/ exceeds Lezer token syntax; external tokenizer.
+  // INCOMPLETE: BinaryNumber regex /0[bB][01]+(_[01]+)*(?![0-9A-Za-z_$\\])/ exceeds Lezer token syntax; external tokenizer.
+  // INCOMPLETE: BigInt regex /[0-9]+(_[0-9]+)*n(?![0-9A-Za-z_$\\])/ exceeds Lezer token syntax; external tokenizer.
+  // INCOMPLETE: Number regex /[0-9]+(_[0-9]+)*(?:\.[0-9]*(_[0-9]+)*)?(?:[eE][+-]?[0-9]+(_[0-9]+)*)?(?![0-9A-Za-z_$\\])/ exceeds Lezer token syntax; external tokenizer.
   // INCOMPLETE: String regex /"(?:[^"\\]|\\[\s\S])*"|'(?:[^'\\]|\\[\s\S])*'/ exceeds Lezer token syntax; external tokenizer.
   // INCOMPLETE: Template is produced by the external tokenizer
   // (context-sensitive: template interpolation).
@@ -329,6 +329,10 @@ Program {
   JSDoc,
   BlockComment,
   Ident,
+  HexNumber,
+  OctalNumber,
+  BinaryNumber,
+  BigInt,
   Number,
   String,
   Template,

examples/tree-sitter/grammar.js

@@ -163,15 +163,15 @@ module.exports = grammar({
 
     ident: $ => token(/(?:[a-zA-Z_$]|\\u[0-9a-fA-F]{4}|\\u\{[0-9a-fA-F]+\})(?:[a-zA-Z0-9_$]|\\u[0-9a-fA-F]{4}|\\u\{[0-9a-fA-F]+\})*/),
 
-    hex_number: $ => token(/0[xX][0-9a-fA-F]+(_[0-9a-fA-F]+)*/),
+    hex_number: $ => token(/0[xX][0-9a-fA-F]+(_[0-9a-fA-F]+)*(?![0-9A-Za-z_$\\])/),
 
-    octal_number: $ => token(/0[oO][0-7]+(_[0-7]+)*/),
+    octal_number: $ => token(/0[oO][0-7]+(_[0-7]+)*(?![0-9A-Za-z_$\\])/),
 
-    binary_number: $ => token(/0[bB][01]+(_[01]+)*/),
+    binary_number: $ => token(/0[bB][01]+(_[01]+)*(?![0-9A-Za-z_$\\])/),
 
-    big_int: $ => token(/[0-9]+(_[0-9]+)*n/),
+    big_int: $ => token(/[0-9]+(_[0-9]+)*n(?![0-9A-Za-z_$\\])/),
 
-    number: $ => token(/[0-9]+(_[0-9]+)*(?:\.[0-9]*(_[0-9]+)*)?(?:[eE][+-]?[0-9]+)?/),
+    number: $ => token(/[0-9]+(_[0-9]+)*(?:\.[0-9]*(_[0-9]+)*)?(?:[eE][+-]?[0-9]+(_[0-9]+)*)?(?![0-9A-Za-z_$\\])/),
 
     string: $ => token(/"(?:[^"\\]|\\[\s\S])*"|'(?:[^'\\]|\\[\s\S])*'/),
 

examples/typescript.monarch.json

@@ -520,35 +520,35 @@
         }
       ],
       [
-        "0[xX][0-9a-fA-F]+(_[0-9a-fA-F]+)*",
+        "0[xX][0-9a-fA-F]+(_[0-9a-fA-F]+)*(?![0-9A-Za-z_$\\\\])",
         {
           "token": "number.hex",
           "switchTo": "@value"
         }
       ],
       [
-        "0[oO][0-7]+(_[0-7]+)*",
+        "0[oO][0-7]+(_[0-7]+)*(?![0-9A-Za-z_$\\\\])",
         {
           "token": "number.octal",
           "switchTo": "@value"
         }
       ],
       [
-        "0[bB][01]+(_[01]+)*",
+        "0[bB][01]+(_[01]+)*(?![0-9A-Za-z_$\\\\])",
         {
           "token": "number.binary",
           "switchTo": "@value"
         }
       ],
       [
-        "[0-9]+(_[0-9]+)*n",
+        "[0-9]+(_[0-9]+)*n(?![0-9A-Za-z_$\\\\])",
         {
           "token": "number",
           "switchTo": "@value"
         }
       ],
       [
-        "[0-9]+(_[0-9]+)*(?:\\.[0-9]*(_[0-9]+)*)?(?:[eE][+-]?[0-9]+)?",
+        "[0-9]+(_[0-9]+)*(?:\\.[0-9]*(_[0-9]+)*)?(?:[eE][+-]?[0-9]+(_[0-9]+)*)?(?![0-9A-Za-z_$\\\\])",
         {
           "token": "number.float",
           "switchTo": "@value"
@@ -1079,23 +1079,23 @@
     ],
     "interpExprBody": [
       [
-        "0[xX][0-9a-fA-F]+(_[0-9a-fA-F]+)*",
+        "0[xX][0-9a-fA-F]+(_[0-9a-fA-F]+)*(?![0-9A-Za-z_$\\\\])",
         "number.hex"
       ],
       [
-        "0[oO][0-7]+(_[0-7]+)*",
+        "0[oO][0-7]+(_[0-7]+)*(?![0-9A-Za-z_$\\\\])",
         "number.octal"
       ],
       [
-        "0[bB][01]+(_[01]+)*",
+        "0[bB][01]+(_[01]+)*(?![0-9A-Za-z_$\\\\])",
         "number.binary"
       ],
       [
-        "[0-9]+(_[0-9]+)*n",
+        "[0-9]+(_[0-9]+)*n(?![0-9A-Za-z_$\\\\])",
         "number"
       ],
       [
-        "[0-9]+(_[0-9]+)*(?:\\.[0-9]*(_[0-9]+)*)?(?:[eE][+-]?[0-9]+)?",
+        "[0-9]+(_[0-9]+)*(?:\\.[0-9]*(_[0-9]+)*)?(?:[eE][+-]?[0-9]+(_[0-9]+)*)?(?![0-9A-Za-z_$\\\\])",
         "number.float"
       ],
       [

examples/typescript.tmLanguage.json

@@ -434,23 +434,23 @@
     },
     "hexnumber": {
       "name": "constant.numeric.hex.typescript",
-      "match": "0[xX][0-9a-fA-F]+(_[0-9a-fA-F]+)*"
+      "match": "0[xX][0-9a-fA-F]+(_[0-9a-fA-F]+)*(?![0-9A-Za-z_$\\\\])"
     },
     "octalnumber": {
       "name": "constant.numeric.octal.typescript",
-      "match": "0[oO][0-7]+(_[0-7]+)*"
+      "match": "0[oO][0-7]+(_[0-7]+)*(?![0-9A-Za-z_$\\\\])"
     },
     "binarynumber": {
       "name": "constant.numeric.binary.typescript",
-      "match": "0[bB][01]+(_[01]+)*"
+      "match": "0[bB][01]+(_[01]+)*(?![0-9A-Za-z_$\\\\])"
     },
     "bigint": {
       "name": "constant.numeric.bigint.typescript",
-      "match": "[0-9]+(_[0-9]+)*n"
+      "match": "[0-9]+(_[0-9]+)*n(?![0-9A-Za-z_$\\\\])"
     },
     "number": {
       "name": "constant.numeric.float.typescript",
-      "match": "[0-9]+(_[0-9]+)*(?:\\.[0-9]*(_[0-9]+)*)?(?:[eE][+-]?[0-9]+)?"
+      "match": "[0-9]+(_[0-9]+)*(?:\\.[0-9]*(_[0-9]+)*)?(?:[eE][+-]?[0-9]+(_[0-9]+)*)?(?![0-9A-Za-z_$\\\\])"
     },
     "string-double": {
       "name": "string.quoted.double.typescript",

examples/typescript.ts

@@ -8,11 +8,18 @@ import {
 // ── Tokens ──
 
 const Ident        = token(/(?:[a-zA-Z_$]|\\u[0-9a-fA-F]{4}|\\u\{[0-9a-fA-F]+\})(?:[a-zA-Z0-9_$]|\\u[0-9a-fA-F]{4}|\\u\{[0-9a-fA-F]+\})*/, { identifier: true });
-const HexNumber    = token(/0[xX][0-9a-fA-F]+(_[0-9a-fA-F]+)*/,            { scope: 'constant.numeric.hex' });
-const OctalNumber  = token(/0[oO][0-7]+(_[0-7]+)*/,                         { scope: 'constant.numeric.octal' });
-const BinaryNumber = token(/0[bB][01]+(_[01]+)*/,                            { scope: 'constant.numeric.binary' });
-const BigInt_      = token(/[0-9]+(_[0-9]+)*n/,                              { scope: 'constant.numeric.bigint' });
-const Number_      = token(/[0-9]+(_[0-9]+)*(?:\.[0-9]*(_[0-9]+)*)?(?:[eE][+-]?[0-9]+)?/);
+// Numeric tokens end with `(?![0-9A-Za-z_$\\])`: the spec rule that a numeric literal
+// may not be immediately followed by an IdentifierStart or DecimalDigit. Without it,
+// `0b2`/`0B1102110`/`0o81010` would munch a valid prefix (`0b1`, `0B110`) and leave the
+// rest as a second token, so the file parses as two statements instead of being rejected.
+// With it the bad literal matches no token and the lexer throws — the correct rejection.
+// (ASCII IdentifierStart + `\` for `\u`-escapes; the lexer compiles patterns without the
+// /u flag so \p{L} is unavailable, and every affected conformance case is ASCII.)
+const HexNumber    = token(/0[xX][0-9a-fA-F]+(_[0-9a-fA-F]+)*(?![0-9A-Za-z_$\\])/,            { scope: 'constant.numeric.hex' });
+const OctalNumber  = token(/0[oO][0-7]+(_[0-7]+)*(?![0-9A-Za-z_$\\])/,                         { scope: 'constant.numeric.octal' });
+const BinaryNumber = token(/0[bB][01]+(_[01]+)*(?![0-9A-Za-z_$\\])/,                            { scope: 'constant.numeric.binary' });
+const BigInt_      = token(/[0-9]+(_[0-9]+)*n(?![0-9A-Za-z_$\\])/,                              { scope: 'constant.numeric.bigint' });
+const Number_      = token(/[0-9]+(_[0-9]+)*(?:\.[0-9]*(_[0-9]+)*)?(?:[eE][+-]?[0-9]+(_[0-9]+)*)?(?![0-9A-Za-z_$\\])/);
 const String_      = token(/"(?:[^"\\]|\\[\s\S])*"|'(?:[^'\\]|\\[\s\S])*'/, {
   string: true,
   escape: /\\(?:[nrtbfv0'"\\]|x[0-9a-fA-F]{2}|u[0-9a-fA-F]{4}|u\{[0-9a-fA-F]+\})/,