Improve JS highlighter fidelity: bare-param scoping + regex after contextual keywords

Two language-agnostic gen-tm fixes surfaced by the JavaScript coverage gate:

- Unannotated parameters (plain function f(x), methods, constructors) now get
  variable.parameter instead of variable.other. Previously only arrow params and
  type-annotated params (x: T) triggered param scoping, so the copy+strip'd JS
  grammar -- which has no annotations -- lost it. A structural declaration-param-name
  matcher inside the params scope handles bare params; annotated params still defer
  to the type-annotation path. JS highlighter visual accuracy 96.4% -> 98.0%
  (real-gap 16 -> 9 tokens).

- Regex-vs-division detection now flattens alt()/opt()/group() when collecting the
  keywords that may precede an expression, so a keyword reachable only through an
  alternative (e.g. alt('in','of') Expr in the for-head) is seen. Fixes regex
  literals after `of` (for (x of /re/.exec(s)), upstream #883) in both the JS and
  TS TextMate output; a / b still parses as division.

Both engine changes stay agnostic (no hardcoded language tokens; agnostic 5/5).
TS unaffected: coverage 99.3%, valid-code 100% (FN=0), bidirectional 97.84%.
Regenerated both grammars' artifacts.

Author: johnsoncodehk

examples/javascript.tmLanguage.json

@@ -184,7 +184,7 @@
   "repository": {
     "regex-literal": {
       "name": "string.regexp.javascript",
-      "begin": "(?:(?<=[=|\\^&<>+\\-*%!~(,.\\[?:{;])|(?<=\\binstanceof)|(?<=\\bin)|(?<=\\bnew)|(?<=\\bextends)|(?<=\\belse)|(?<=\\bdo)|(?<=\\bthrow)|(?<=\\btry)|(?<=\\bfinally)|(?<=\\bcase)|(?<=\\bstatic)|(?<=\\btypeof)|(?<=\\bvoid)|(?<=\\bdelete)|(?<=\\bawait)|(?<=\\byield)|(?<=^))\\s*(/)(?![*/])",
+      "begin": "(?:(?<=[=|\\^&<>+\\-*%!~(,.\\[?:{;])|(?<=\\binstanceof)|(?<=\\bin)|(?<=\\bnew)|(?<=\\bextends)|(?<=\\byield)|(?<=\\bget)|(?<=\\bset)|(?<=\\basync)|(?<=\\belse)|(?<=\\bdo)|(?<=\\breturn)|(?<=\\bthrow)|(?<=\\btry)|(?<=\\bfinally)|(?<=\\bcatch)|(?<=\\bof)|(?<=\\bcase)|(?<=\\bexport)|(?<=\\bdefault)|(?<=\\bimport)|(?<=\\bstatic)|(?<=\\baccessor)|(?<=\\btypeof)|(?<=\\bvoid)|(?<=\\bdelete)|(?<=\\bawait)|(?<=^))\\s*(/)(?![*/])",
       "beginCaptures": {
         "1": {
           "name": "punctuation.definition.string.begin.regexp.javascript"
@@ -418,6 +418,17 @@
         }
       ]
     },
+    "declaration-param-name": {
+      "match": "(?<=[,(])\\s*(\\.\\.\\.)?\\s*((?:[a-zA-Z_$]|\\\\u[0-9a-fA-F]{4}|\\\\u\\{[0-9a-fA-F]+\\})(?:[a-zA-Z0-9_$]|\\\\u[0-9a-fA-F]{4}|\\\\u\\{[0-9a-fA-F]+\\})*)(?=\\s*[,)=])",
+      "captures": {
+        "1": {
+          "name": "keyword.operator.spread.javascript"
+        },
+        "2": {
+          "name": "variable.parameter.javascript"
+        }
+      }
+    },
     "declaration-params": {
       "name": "meta.parameters.javascript",
       "begin": "\\(",
@@ -433,6 +444,9 @@
         }
       },
       "patterns": [
+        {
+          "include": "#declaration-param-name"
+        },
         {
           "include": "#nested-parens"
         },

examples/typescript.tmLanguage.json

@@ -399,7 +399,7 @@
     },
     "regex-literal": {
       "name": "string.regexp.typescript",
-      "begin": "(?:(?<=[=|\\^&<>+\\-*%!~,\\[(?:{;.])|(?<=\\bis)|(?<=\\bkeyof)|(?<=\\btypeof)|(?<=\\breadonly)|(?<=\\bnew)|(?<=\\bextends)|(?<=\\bin)|(?<=\\bas)|(?<=\\binstanceof)|(?<=\\bclass)|(?<=\\bsatisfies)|(?<=\\belse)|(?<=\\bdo)|(?<=\\bthrow)|(?<=\\btry)|(?<=\\bfinally)|(?<=\\bcase)|(?<=\\btype)|(?<=\\bstatic)|(?<=\\bvoid)|(?<=\\bdelete)|(?<=\\bawait)|(?<=\\byield)|(?<=^))\\s*(/)(?![*/])",
+      "begin": "(?:(?<=[=|\\^&<>+\\-*%!~,\\[(?:{;.])|(?<=\\bis)|(?<=\\bkeyof)|(?<=\\btypeof)|(?<=\\breadonly)|(?<=\\bnew)|(?<=\\bextends)|(?<=\\bin)|(?<=\\bas)|(?<=\\binstanceof)|(?<=\\bclass)|(?<=\\basync)|(?<=\\byield)|(?<=\\bsatisfies)|(?<=\\bfunction)|(?<=\\bget)|(?<=\\bset)|(?<=\\belse)|(?<=\\bdo)|(?<=\\breturn)|(?<=\\bthrow)|(?<=\\btry)|(?<=\\bfinally)|(?<=\\bcatch)|(?<=\\bpublic)|(?<=\\bprivate)|(?<=\\bprotected)|(?<=\\bof)|(?<=\\bcase)|(?<=\\bdeclare)|(?<=\\bexport)|(?<=\\bdefault)|(?<=\\bimport)|(?<=\\btype)|(?<=\\bstatic)|(?<=\\babstract)|(?<=\\boverride)|(?<=\\baccessor)|(?<=\\bvoid)|(?<=\\bdelete)|(?<=\\bawait)|(?<=^))\\s*(/)(?![*/])",
       "beginCaptures": {
         "1": {
           "name": "punctuation.definition.string.begin.regexp.typescript"
@@ -794,6 +794,17 @@
         }
       ]
     },
+    "declaration-param-name": {
+      "match": "(?<=[,(])\\s*(\\.\\.\\.)?\\s*((?:[a-zA-Z_$]|\\\\u[0-9a-fA-F]{4}|\\\\u\\{[0-9a-fA-F]+\\})(?:[a-zA-Z0-9_$]|\\\\u[0-9a-fA-F]{4}|\\\\u\\{[0-9a-fA-F]+\\})*)(?=\\s*[,)=])",
+      "captures": {
+        "1": {
+          "name": "keyword.operator.spread.typescript"
+        },
+        "2": {
+          "name": "variable.parameter.typescript"
+        }
+      }
+    },
     "declaration-params": {
       "name": "meta.parameters.typescript",
       "begin": "\\(",
@@ -812,6 +823,9 @@
         {
           "include": "#param-type-annotation"
         },
+        {
+          "include": "#declaration-param-name"
+        },
         {
           "include": "#nested-parens"
         },

src/gen-tm.ts

@@ -981,8 +981,15 @@ function detectRegexLiteral(grammar: CstGrammar, tokenNames: Set<string>): Regex
     }
   }
 
+  // Walk every expression body, checking each flattened alternative for a
+  // keyword→expr-ref adjacency. `expandAlts` resolves `alt`/`opt`/`group` so a
+  // keyword tucked inside an alternative (e.g. `alt('in','of') Expr` in a for-head)
+  // is seen adjacent to the following ref — without it, only keywords written as a
+  // bare literal directly before the ref (`'in' Expr` as an infix op) are collected,
+  // so `of` would be missed. `sep`/quantifier inner bodies stay opaque to
+  // `expandAlts`, so recurse into them to keep nested adjacencies reachable.
   function walk(expr: RuleExpr) {
-    if (expr.type === 'seq') checkSeq(expr.items);
+    for (const seq of expandAlts(expr)) checkSeq(seq);
     if (expr.type === 'alt' || expr.type === 'seq') expr.items.forEach(walk);
     if (expr.type === 'quantifier' || expr.type === 'group') walk(expr.body);
     if (expr.type === 'sep') walk(expr.element);
@@ -1777,6 +1784,28 @@ export function generateTmLanguage(grammar: CstGrammar, langName: string): TmGra
         };
       }
 
+      // Bare parameter name (no type annotation): the identifier at a
+      // param-start position — directly after '(' or ',', and immediately
+      // followed by a delimiter that ends a *bare* param: ',' (next param),
+      // ')' (end of list), or '=' (default value). The lookbehind keeps this
+      // from matching default-value expressions (e.g. `x = foo` → only `x`
+      // matches; `foo` is preceded by '=', not '('/','). This gives
+      // unannotated params in function/method/constructor signatures the
+      // variable.parameter scope (the type-annotation path handles annotated
+      // ones). The lookahead deliberately omits ':' and '?' so an annotated
+      // param (`x: T`, `x?: T`, `...args: T[]`) is left for
+      // #param-type-annotation — which is listed first AND, because this
+      // matcher swallows leading whitespace (TextMate prefers the leftmost
+      // match), would otherwise be pre-empted on rest params like `...args:`.
+      repository['declaration-param-name'] = {
+        match: `(?<=[,(])\\s*(\\.\\.\\.)?\\s*(${identPattern})(?=\\s*[,)=])`,
+        captures: {
+          '1': { name: `keyword.operator.spread.${langName}` },
+          '2': { name: `variable.parameter.${langName}` },
+        },
+      };
+      paramsInnerPatterns.push({ include: '#declaration-param-name' });
+
       paramsInnerPatterns.push({ include: '#nested-parens' });
       paramsInnerPatterns.push({ include: '$self' });
 

test/test-issues.ts

@@ -315,7 +315,7 @@ const tests: TestCase[] = [
     input: 'function f(a = 1) {}',
     checks: [
       { text: 'f', scope: 'entity.name.function' },
-      { text: 'a', scope: 'variable.other' },
+      { text: 'a', scope: 'variable.parameter' },
       { text: '=', scope: 'keyword.operator.assignment' },
       { text: '1', scope: 'constant.numeric' },
     ],