Reverse JS<->TS dependency: javascript.ts is the standalone base, typescript.ts imports it

JavaScript is the subset/base of the ECMAScript family; the subset must not depend on the
superset. Move the shared vocabulary -- the 16 token consts, notReserved/notReservedExpr,
the precedence ladder (ecmaPrec), and the base scope map (jsScopes) -- into
examples/javascript.ts, which now imports ONLY ../src/api.ts and stands alone.
examples/typescript.ts imports that vocabulary and extends it: its scopes reuse jsScopes'
shared keys by reference and interleave the TS-only entries (interface/type/enum/namespace,
the keyof/as/satisfies/... operators, the primitive type names) at their original
positions; its rules add the type layer.

Behavior-preserving for TypeScript: typescript.tmLanguage.json byte-identical, conformance
100%/97.84%, coverage 99.3%, sanity 15/15, agnostic 5/5, refactor-guard 112/112. JavaScript
unchanged (js-conformance 61/61 accepted, 8/10 rejected). javascript.ts has no import from
typescript.ts.

NOTE (do not "simplify" to a spread): the TS scopes object is explicitly ordered, reusing
jsScopes by key -- NOT { ...jsScopes, ...tsOnly }. A spread (a) appends the TS-only keys at
the end, reordering the keyword groups emitted in negative-lookahead patterns, and (b) flips
the first-occurrence scope of the shared literal "module" (support.variable in JS vs
storage.type.namespace in TS), which scopeOverrides keys by first occurrence -- dropping the
module-declaration construct. Both change the generated grammar; the explicit form is the
only byte-identical one.

(Dependency reversal implemented by a delegated agent; independently re-verified
byte-identical + all gates green.)

Author: johnsoncodehk

examples/javascript.ts

@@ -1,7 +1,20 @@
-// JavaScript grammar for Monogram.
+// JavaScript grammar for Monogram — the STANDALONE BASE of the ECMAScript family.
 //
-// JavaScript is a syntactic subset of TypeScript, so this grammar is the TS
-// grammar (examples/typescript.ts) with every type construct removed: no type
+// JavaScript is the syntactic SUBSET of TypeScript (TS = JS + a type layer), so
+// this file owns the shared, type-free ECMAScript *vocabulary* — the token set,
+// the `notReserved`/`notReservedExpr` reserved-word guards, the precedence ladder
+// (`ecmaPrec`), and the JS scope map (`jsScopes`) — and exports it. The TS grammar
+// (the sibling superset file) then imports that vocabulary and EXTENDS it with the
+// type layer. The dependency runs subset → superset only: this file imports nothing
+// from the TS grammar (it has no type knowledge) and must stand alone — its only
+// import is the engine's combinator API.
+//
+// The rules are NOT shared either direction: combinator rules bind their
+// references at definition time, so a JS rule must reference the OTHER JS rule
+// consts — it can't reuse TS's rule objects (and vice-versa). Each file therefore
+// keeps its own rule consts; only the vocabulary above is shared.
+//
+// This grammar is the TS grammar with every type construct removed: no type
 // annotations, type parameters/arguments, `interface`/`type`/`enum`/`namespace`/
 // `declare`/`module`-as-type declarations, `as`/`satisfies`/`<T>`-cast/non-null
 // (`!`) expressions, and no type-optional `?` on params/members. Everything that
@@ -10,24 +23,136 @@
 // operator set (`??`, `?.`, `**`, optional chaining), destructuring, template
 // literals (tagged + interpolated), regex, every module import/export VALUE form,
 // control flow, numeric literals, and Stage-3 decorators on classes.
-//
-// The type-free *vocabulary* (tokens, the precedence ladder) is imported from
-// typescript.ts rather than duplicated; the rules are copied and stripped here
-// because combinator rules bind their references at definition time — a JS rule
-// must reference the OTHER JS rule consts, so it can't reuse the TS rule objects.
 
 import {
-  rule, defineGrammar,
+  token, rule, defineGrammar,
   left, right, none, noUnaryLhs,
   op, prefix, postfix, sameLine,
   sep, opt, many, many1, alt, exclude, not,
 } from '../src/api.ts';
-import {
-  ecmaTokens, ecmaPrec,
+
+// ── Tokens ──
+
+const Ident        = token(/(?:[a-zA-Z_$]|\\u[0-9a-fA-F]{4}|\\u\{[0-9a-fA-F]+\})(?:[a-zA-Z0-9_$]|\\u[0-9a-fA-F]{4}|\\u\{[0-9a-fA-F]+\})*/, { identifier: true });
+// Numeric tokens end with `(?![0-9A-Za-z_$\\])`: the spec rule that a numeric literal
+// may not be immediately followed by an IdentifierStart or DecimalDigit. Without it,
+// `0b2`/`0B1102110`/`0o81010` would munch a valid prefix (`0b1`, `0B110`) and leave the
+// rest as a second token, so the file parses as two statements instead of being rejected.
+// With it the bad literal matches no token and the lexer throws — the correct rejection.
+// (ASCII IdentifierStart + `\` for `\u`-escapes; the lexer compiles patterns without the
+// /u flag so \p{L} is unavailable, and every affected conformance case is ASCII.)
+const HexNumber    = token(/0[xX][0-9a-fA-F]+(_[0-9a-fA-F]+)*(?![0-9A-Za-z_$\\])/,            { scope: 'constant.numeric.hex' });
+const OctalNumber  = token(/0[oO][0-7]+(_[0-7]+)*(?![0-9A-Za-z_$\\])/,                         { scope: 'constant.numeric.octal' });
+const BinaryNumber = token(/0[bB][01]+(_[01]+)*(?![0-9A-Za-z_$\\])/,                            { scope: 'constant.numeric.binary' });
+const BigInt_      = token(/[0-9]+(_[0-9]+)*n(?![0-9A-Za-z_$\\])/,                              { scope: 'constant.numeric.bigint' });
+const Number_      = token(/[0-9]+(_[0-9]+)*(?:\.[0-9]*(_[0-9]+)*)?(?:[eE][+-]?[0-9]+(_[0-9]+)*)?(?![0-9A-Za-z_$\\])/);
+// A well-formed JS escape, used in the string-body pattern below. `\u`/`\x` must
+// match their strict forms — a `\u{cp}` with cp ≤ 0x10FFFF, a 4-hex `\uXXXX`, or a
+// 2-hex `\xXX` — while `\` + any *other* char (\n, \\, \q non-escape, line
+// continuation) stays valid via `[^ux]`. A malformed `\u`/`\x` (e.g. `\u{110000}`,
+// `\u{r}`, `\u{}`, `\u{67`) matches no escape, so the string matches no token and the
+// lexer throws — TS's exact rejection. The in-range codepoint is `0*` leading zeros
+// then 1–5 hex (0–0xFFFFF) or `10`+4 hex (0x100000–0x10FFFF).
+const codePoint = String.raw`0*(?:[0-9a-fA-F]{1,5}|10[0-9a-fA-F]{4})`;
+const escape    = String.raw`\\(?:u\{${codePoint}\}|u[0-9a-fA-F]{4}|x[0-9a-fA-F]{2}|[^ux])`;
+const String_      = token(new RegExp(`"(?:[^"\\\\]|${escape})*"|'(?:[^'\\\\]|${escape})*'`), {
+  string: true,
+  escape: /\\(?:[nrtbfv0'"\\]|x[0-9a-fA-F]{2}|u[0-9a-fA-F]{4}|u\{[0-9a-fA-F]+\})/,
+});
+const Template     = token(/`(?:[^`\\$]|\\.|\$(?!\{))*`/, {
+  escape: /\\(?:[nrtbfv0'"\\`$]|x[0-9a-fA-F]{2}|u[0-9a-fA-F]{4}|u\{[0-9a-fA-F]+\})/,
+  // Same well-formed-escape rule as strings; the lexer rejects a malformed `\u`/`\x`
+  // in an *untagged* template (`\u{110000}`, `\u{r}`), but allows it when tagged.
+  escapeValid: new RegExp(escape),
+  template: { open: '`', interpOpen: '${', interpClose: '}' },
+});
+const Regex_       = token(/\/(?:[^\/\\\[\n]|\\.|\[(?:[^\]\\\n]|\\.)*\])+\/[gimsuydv]*/, {
+  regex: true,
+  regexContext: {
+    divisionAfterTypes: ['Ident', 'Number', 'String', 'Template', 'BigInt'],
+    divisionAfterTexts: [')', ']', '++', '--', 'this', 'super', 'true', 'false', 'null', 'undefined'],
+    regexAfterTexts: ['in', 'of', 'instanceof', 'typeof', 'delete', 'void', 'await', 'yield', 'throw', 'return', 'case', 'do', 'else', 'new'],
+    // `kw ( … )` heads (control-flow): the closing `)` is a statement head, not a
+    // value, so `if (a) /re/` parses `/re/` as a regex rather than division.
+    regexAfterParenKeywords: ['if', 'while', 'for', 'with'],
+    // member accessors: after one, those keywords are property NAMES, so
+    // `obj.for(x) / y` stays a method call + division.
+    memberAccessTexts: ['.', '?.'],
+  },
+});
+const Decorator    = token(/@(?:[a-zA-Z_$][a-zA-Z0-9_$.]*)?/,                { scope: 'entity.name.function.decorator' });
+const PrivateField = token(/#[a-zA-Z_$][a-zA-Z0-9_$]*/,                     { scope: 'variable.other.property' });
+const Shebang      = token(/^#![^\n]*/,           { skip: true, scope: 'comment.line.shebang' });
+const JSDoc        = token(/\/\*\*(?!\/)[\s\S]*?\*\//,  { skip: true, scope: 'comment.block.documentation', embed: 'jsdoc' });
+const TripleSlash  = token(/\/\/\/\s*<[^\n]*/,    { skip: true, scope: 'comment.line.triple-slash' });
+const LineComment  = token(/\/\/[^\n]*/,           { skip: true });
+const BlockComment = token(/\/\*[\s\S]*?\*\//,     { skip: true });
+
+// The token consts, reserved-word guards, precedence ladder, and scope map are
+// pure ECMAScript vocabulary — no rule wiring — so the TS grammar imports them from
+// here and extends them rather than duplicating them.
+export {
+  Shebang, JSDoc, TripleSlash, LineComment, BlockComment,
   Ident, HexNumber, OctalNumber, BinaryNumber, BigInt_,
   Number_, String_, Template, Regex_, Decorator, PrivateField,
-  notReserved, notReservedExpr,
-} from './typescript.ts';
+};
+
+// ── Always-reserved words ──
+// The `Ident` token deliberately swallows keywords (they lex as identifiers), so
+// every keyword can otherwise fall back to a bare identifier. These words are
+// reserved in EVERY context (ECMAScript ReservedWord ∪ TS's always-reserved), so
+// they are valid as an identifier NOWHERE — not as an expression, a shorthand
+// property, or a binding name. `notReserved` is a zero-width guard placed before an
+// identifier position to forbid exactly these. Excluded on purpose: contextual
+// keywords (as/async/from/type/of/…) and strict-mode-only reserved words
+// (let/static/implements/yield/await/…) — those ARE valid identifiers in some
+// context a CFG can't detect (sloppy mode, non-generator/non-async), so forbidding
+// them here would reject valid code (`var let = 1`, `function f(yield) {}`).
+export const notReserved = not(alt(
+  'break', 'case', 'catch', 'class', 'const', 'continue', 'debugger', 'default',
+  'delete', 'do', 'else', 'enum', 'export', 'extends', 'false', 'finally', 'for',
+  'function', 'if', 'import', 'in', 'instanceof', 'new', 'null', 'return', 'super',
+  'switch', 'this', 'throw', 'true', 'try', 'typeof', 'var', 'void', 'while', 'with',
+));
+
+// A NARROWER guard for the *expression* identifier-NUD only. The full `notReserved`
+// set can NOT be used at expression position: most always-reserved words legitimately
+// begin an expression via their own dedicated forms (`new`/`new.target`, `class`/
+// `function` expressions, `import(…)`/`import<T>`, `super`, `this`, `true`/`false`/
+// `null`, …), and TS's own error-recovery tolerates several reserved words sliding into
+// the bare-identifier fallback inside otherwise-valid files (e.g. `export default …`,
+// undeclared `for (x in …)`, `class … extends (e)`, a decorator before `export`). The
+// words below have NO such role: they are the prefix operators `void`/`typeof`/`delete`
+// (which must take an operand) plus the `catch`/`throw` keywords and `enum`. Forbidding
+// the bare-identifier fallback for exactly these rejects `catch(x){}` with no `try`,
+// `void ;`/`typeof ;`/`delete ;` (operatorless prefix op), and `throw ;` — while leaving
+// every valid expression (and TS's recovery cases) untouched. Verified: widening this
+// set to other reserved words regresses valid code; these five are the FN-safe maximum.
+export const notReservedExpr = not(alt(
+  'catch', 'delete', 'enum', 'throw', 'typeof', 'void',
+));
+
+// ── Precedence ladder (shared ECMAScript operator precedence) ──
+
+export const ecmaPrec = [
+  right('=', '+=', '-=', '*=', '/=', '%=', '**=', '<<=', '>>=', '>>>=', '&=', '|=', '^='),
+  right('??=', '||=', '&&='),
+  left('??'),
+  left('||'),
+  left('&&'),
+  left('|'),
+  left('^'),
+  left('&'),
+  none('==', '!=', '===', '!=='),
+  none('<', '>', '<=', '>='),
+  left('<<', '>>', '>>>'),
+  left('+', '-'),
+  left('*', '/', '%'),
+  right(noUnaryLhs('**')),   // `-x ** y` is a syntax error: a unary-prefix expr can't be a `**` LHS
+  right(prefix('!', '~', '+', '-', 'typeof', 'void', 'delete', 'await', 'yield')),
+  right(prefix('++', '--')),
+  left(postfix('++', '--')),
+];
 
 // ── Decorators ──
 // Stage-3 / JS-real decorators: `@dec` and `@dec(args)`. The TS-only
@@ -296,13 +421,63 @@ const Program = rule($ => [
   many(alt(Decl, Stmt)),   // Decl first: prefer declaration over IIFE expression-statement
 ]);
 
+// ── Scope map ──
+// The JS scope map: scopes every real-JS token. The TS grammar imports this and
+// extends it with the type-only scope keys (storage.type.interface/type/enum/
+// namespace, the keyof/as/satisfies/is/infer/asserts keyword.operator.expression
+// entries, support.type.primitive) — those names are plain identifiers in JS, not
+// types, so they are deliberately absent here.
+export const jsScopes = {
+  'keyword.control.conditional': ['if', 'else', 'switch', 'case', 'default'],
+  'keyword.control.loop': ['for', 'while', 'do', 'in', 'of'],
+  'keyword.control.flow': ['return', 'break', 'continue', 'await', 'yield'],
+  'keyword.control.trycatch': ['try', 'catch', 'finally', 'throw'],
+  'keyword.control': ['debugger', 'with'],
+  'keyword.control.import': ['import', 'export', 'from'],
+  'storage.type': ['let', 'const', 'var', 'using'],
+  'storage.type.function': ['function', 'constructor'],
+  'storage.type.class': ['class'],
+  'storage.modifier': ['static', 'async', 'accessor'],
+  'storage.type.property': ['get', 'set'],
+  'keyword.other.extends': ['extends'],
+  'keyword.operator.expression': ['instanceof', 'new', 'delete', 'void', 'typeof'],
+  'keyword.operator.assignment': ['=', '+=', '-=', '*=', '/=', '%=', '**=', '<<=', '>>=', '>>>=', '&=', '|=', '^=', '??=', '||=', '&&='],
+  'keyword.operator.comparison': ['==', '!=', '===', '!=='],
+  'keyword.operator.logical': ['||', '&&', '??'],
+  'keyword.operator.arithmetic': ['+', '-', '*', '/', '%', '**'],
+  'keyword.operator.increment-decrement': ['++', '--'],
+  'keyword.operator.logical.prefix': ['!', '~'],
+  'keyword.operator.bitwise': ['|', '&', '^'],
+  'keyword.operator.bitwise.shift': ['<<', '>>', '>>>'],
+  'storage.type.function.arrow': ['=>'],
+  'punctuation.bracket.round': ['(', ')'],
+  'punctuation.bracket.curly': ['{', '}'],
+  'punctuation.bracket.square': ['[', ']'],
+  'punctuation.accessor': ['.'],
+  'punctuation.accessor.optional': ['?.'],
+  'punctuation.terminator.statement': [';'],
+  'punctuation.separator.comma': [','],
+  'constant.language.boolean': ['true', 'false'],
+  'constant.language.null': ['null', 'undefined'],
+  'variable.language': ['this', 'super'],
+  'support.class': ['Promise', 'Array', 'Map', 'Set', 'WeakMap', 'WeakSet', 'Error', 'RegExp', 'Date', 'Object', 'Function', 'Symbol'],
+  'support.variable': ['console', 'window', 'document', 'process', 'module', 'require', 'exports', 'global', 'globalThis'],
+  'support.variable.property': ['.length', '.prototype', '.constructor'],
+};
+
 // ── Grammar ──
 
 export default defineGrammar({
   name: 'javascript',
   scopeName: 'source.js',
 
-  tokens: ecmaTokens,
+  tokens: {
+    // Comments must come before Regex_ to avoid /** ... */ being matched as regex
+    Shebang, JSDoc, TripleSlash, LineComment, BlockComment,
+    Ident, HexNumber, OctalNumber, BinaryNumber, BigInt: BigInt_,
+    Number: Number_, String: String_, Template, Regex: Regex_,
+    Decorator, PrivateField,
+  },
 
   prec: ecmaPrec,
 
@@ -317,47 +492,7 @@ export default defineGrammar({
     Program,
   },
 
-  // JS subset of ecmaScopes: dropped storage.type.interface/type/enum/namespace,
-  // the type-operator keyword.operator.expression entries (keyof/as/satisfies/is/
-  // infer/asserts), and support.type.primitive (those names are plain identifiers
-  // in JS, not types). Kept everything that scopes real JS tokens.
-  scopes: {
-    'keyword.control.conditional': ['if', 'else', 'switch', 'case', 'default'],
-    'keyword.control.loop': ['for', 'while', 'do', 'in', 'of'],
-    'keyword.control.flow': ['return', 'break', 'continue', 'await', 'yield'],
-    'keyword.control.trycatch': ['try', 'catch', 'finally', 'throw'],
-    'keyword.control': ['debugger', 'with'],
-    'keyword.control.import': ['import', 'export', 'from'],
-    'storage.type': ['let', 'const', 'var', 'using'],
-    'storage.type.function': ['function', 'constructor'],
-    'storage.type.class': ['class'],
-    'storage.modifier': ['static', 'async', 'accessor'],
-    'storage.type.property': ['get', 'set'],
-    'keyword.other.extends': ['extends'],
-    'keyword.operator.expression': ['instanceof', 'new', 'delete', 'void', 'typeof'],
-    'keyword.operator.assignment': ['=', '+=', '-=', '*=', '/=', '%=', '**=', '<<=', '>>=', '>>>=', '&=', '|=', '^=', '??=', '||=', '&&='],
-    'keyword.operator.comparison': ['==', '!=', '===', '!=='],
-    'keyword.operator.logical': ['||', '&&', '??'],
-    'keyword.operator.arithmetic': ['+', '-', '*', '/', '%', '**'],
-    'keyword.operator.increment-decrement': ['++', '--'],
-    'keyword.operator.logical.prefix': ['!', '~'],
-    'keyword.operator.bitwise': ['|', '&', '^'],
-    'keyword.operator.bitwise.shift': ['<<', '>>', '>>>'],
-    'storage.type.function.arrow': ['=>'],
-    'punctuation.bracket.round': ['(', ')'],
-    'punctuation.bracket.curly': ['{', '}'],
-    'punctuation.bracket.square': ['[', ']'],
-    'punctuation.accessor': ['.'],
-    'punctuation.accessor.optional': ['?.'],
-    'punctuation.terminator.statement': [';'],
-    'punctuation.separator.comma': [','],
-    'constant.language.boolean': ['true', 'false'],
-    'constant.language.null': ['null', 'undefined'],
-    'variable.language': ['this', 'super'],
-    'support.class': ['Promise', 'Array', 'Map', 'Set', 'WeakMap', 'WeakSet', 'Error', 'RegExp', 'Date', 'Object', 'Function', 'Symbol'],
-    'support.variable': ['console', 'window', 'document', 'process', 'module', 'require', 'exports', 'global', 'globalThis'],
-    'support.variable.property': ['.length', '.prototype', '.constructor'],
-  },
+  scopes: jsScopes,
 
   entry: Program,
 });