gen-lezer: fix @Specialize and @top emission (valid syntax, still LR-blocked)

Two real syntax bugs in the Lezer output:

- `@specialize<Ident, keyword> { "abstract" kw_abstract, … }` is not valid Lezer —
  there is no block form. Emit one rule per keyword instead
  (`kw_abstract { @Specialize<Ident, "abstract"> }`), the inline-expression form
  @lezer/javascript uses.
- The entry rule is named `Program`, so `@top Program { Program }` collided with the
  rule's own definition ("Duplicate definition of rule 'Program'"). Promote the entry
  rule to @top in place (prefix its definition) rather than emitting a wrapper.

The grammar is now syntactically valid but still does not build: the optional-modifier
soup in rules like ClassMember (716 choices) / TypeMember (402) blows up Lezer's LR
table generation combinatorially — and behind that lies the `<`/type-vs-expression
LR(1) shift/reduce wall. Both TS and JS hang on the choice blowup. tree-sitter's GLR
absorbs all of this (derived highlighter at 95.9%); Lezer's LR(1) cannot, so there is
still no derived Lezer number. Refine the README note accordingly.

Author: johnsoncodehk

README.md

@@ -77,7 +77,7 @@ TypeScript — token-family accuracy (higher = more correct)
 <sub>Graded over 102 ambiguity-rich snippets ([`test/issue-cases.ts`](test/issue-cases.ts)) against tsc's own parse tree. The three parser-based highlighters (Monogram, tree-sitter, Lezer) cluster well above the hand-written TextMate grammar — that gap is the whole point. Per-engine vocabulary→family maps (frozen, auditable): [`test/highlight-engines.ts`](test/highlight-engines.ts). JavaScript: not yet on the bench. Regenerate: `npm run bench:readme`.</sub>
 <!-- bench:end -->
 
-> **Why the Monogram row is its *TextMate* output.** That's the one CI can rebuild on every commit. Monogram *also* derives a full **tree-sitter** grammar + query from the same definition; built locally to wasm and graded through this *same* `tsc` oracle, it scores **95.9%** — **above the official parsers** (tree-sitter 92.7%, Lezer 89.5%) and both TextMate grammars (its own 87.6%, official 76.2%). The lift is *structural*: every capture is derived from grammar shape — type references by node position (`(type (ident) @type)`), member keys from the member-name rule, type parameters from the `< … >` binder, declaration names keyword-anchored — never a hardcoded vocabulary, so it never paints a value as a type. It's opt-in (needs a local wasm build), hence out of the CI chart: `MONOGRAM_TS_WASM=… node test/highlight-bench.ts --corpus adversarial --engines`. *(There is no Monogram-**Lezer** number: the grammar is expressive enough to trip Lezer's LR(1) shift/reduce wall — it parses under tree-sitter's GLR but won't build under Lezer.)*
+> **Why the Monogram row is its *TextMate* output.** That's the one CI can rebuild on every commit. Monogram *also* derives a full **tree-sitter** grammar + query from the same definition; built locally to wasm and graded through this *same* `tsc` oracle, it scores **95.9%** — **above the official parsers** (tree-sitter 92.7%, Lezer 89.5%) and both TextMate grammars (its own 87.6%, official 76.2%). The lift is *structural*: every capture is derived from grammar shape — type references by node position (`(type (ident) @type)`), member keys from the member-name rule, type parameters from the `< … >` binder, declaration names keyword-anchored — never a hardcoded vocabulary, so it never paints a value as a type. It's opt-in (needs a local wasm build), hence out of the CI chart: `MONOGRAM_TS_WASM=… node test/highlight-bench.ts --corpus adversarial --engines`. *(There is no Monogram-**Lezer** number: the optional-modifier-heavy rules blow up Lezer's LR table generation combinatorially, and behind that the `<`/type-vs-expression ambiguity is a genuine LR(1) shift/reduce conflict. tree-sitter's GLR absorbs both; Lezer's LR(1) does not.)*
 
 > **The other side of the ledger (honesty check).** On the *broad* TS parser-conformance corpus — not just the documented bugs — the two are now neck-and-neck: token-role accuracy is **tied at ~99%**, Monogram **leads on per-cell coverage** (100% of strict cells vs official's ~97%), and whole-file-perfect snippets are essentially level (~88% vs ~89%). The small residual is niche multiline type-annotation scoping, **not** the ambiguity class above. Clone the TS corpus to `/tmp/ts-repo` and run `node test/highlight-bench.ts` to see both corpora.
 

examples/lezer/javascript.grammar

@@ -22,7 +22,6 @@
   l0 @right
 }
 
-@top Program { Program }
 DecoratorExpr {
   Decorator (ParenL (Expr (Comma Expr)*)? ParenR)?
 }
@@ -140,63 +139,61 @@ ImportClause {
 ImportSpecifier {
   Ident (kw_as Ident)?
 }
-Program {
+@top Program {
   (Decl | Stmt)*
 }
 
 // Keyword specialization over the identifier token:
-@specialize<Ident, keyword> {
-  "accessor" kw_accessor,
-  "as" kw_as,
-  "async" kw_async,
-  "await" kw_await,
-  "break" kw_break,
-  "case" kw_case,
-  "catch" kw_catch,
-  "class" kw_class,
-  "const" kw_const,
-  "constructor" kw_constructor,
-  "continue" kw_continue,
-  "debugger" kw_debugger,
-  "default" kw_default,
-  "delete" kw_delete,
-  "do" kw_do,
-  "else" kw_else,
-  "export" kw_export,
-  "extends" kw_extends,
-  "false" kw_false,
-  "finally" kw_finally,
-  "for" kw_for,
-  "from" kw_from,
-  "function" kw_function,
-  "get" kw_get,
-  "if" kw_if,
-  "import" kw_import,
-  "in" kw_in,
-  "instanceof" kw_instanceof,
-  "let" kw_let,
-  "meta" kw_meta,
-  "new" kw_new,
-  "null" kw_null,
-  "of" kw_of,
-  "return" kw_return,
-  "set" kw_set,
-  "static" kw_static,
-  "super" kw_super,
-  "switch" kw_switch,
-  "this" kw_this,
-  "throw" kw_throw,
-  "true" kw_true,
-  "try" kw_try,
-  "typeof" kw_typeof,
-  "undefined" kw_undefined,
-  "using" kw_using,
-  "var" kw_var,
-  "void" kw_void,
-  "while" kw_while,
-  "with" kw_with,
-  "yield" kw_yield
-}
+kw_accessor { @specialize<Ident, "accessor"> }
+kw_as { @specialize<Ident, "as"> }
+kw_async { @specialize<Ident, "async"> }
+kw_await { @specialize<Ident, "await"> }
+kw_break { @specialize<Ident, "break"> }
+kw_case { @specialize<Ident, "case"> }
+kw_catch { @specialize<Ident, "catch"> }
+kw_class { @specialize<Ident, "class"> }
+kw_const { @specialize<Ident, "const"> }
+kw_constructor { @specialize<Ident, "constructor"> }
+kw_continue { @specialize<Ident, "continue"> }
+kw_debugger { @specialize<Ident, "debugger"> }
+kw_default { @specialize<Ident, "default"> }
+kw_delete { @specialize<Ident, "delete"> }
+kw_do { @specialize<Ident, "do"> }
+kw_else { @specialize<Ident, "else"> }
+kw_export { @specialize<Ident, "export"> }
+kw_extends { @specialize<Ident, "extends"> }
+kw_false { @specialize<Ident, "false"> }
+kw_finally { @specialize<Ident, "finally"> }
+kw_for { @specialize<Ident, "for"> }
+kw_from { @specialize<Ident, "from"> }
+kw_function { @specialize<Ident, "function"> }
+kw_get { @specialize<Ident, "get"> }
+kw_if { @specialize<Ident, "if"> }
+kw_import { @specialize<Ident, "import"> }
+kw_in { @specialize<Ident, "in"> }
+kw_instanceof { @specialize<Ident, "instanceof"> }
+kw_let { @specialize<Ident, "let"> }
+kw_meta { @specialize<Ident, "meta"> }
+kw_new { @specialize<Ident, "new"> }
+kw_null { @specialize<Ident, "null"> }
+kw_of { @specialize<Ident, "of"> }
+kw_return { @specialize<Ident, "return"> }
+kw_set { @specialize<Ident, "set"> }
+kw_static { @specialize<Ident, "static"> }
+kw_super { @specialize<Ident, "super"> }
+kw_switch { @specialize<Ident, "switch"> }
+kw_this { @specialize<Ident, "this"> }
+kw_throw { @specialize<Ident, "throw"> }
+kw_true { @specialize<Ident, "true"> }
+kw_try { @specialize<Ident, "try"> }
+kw_typeof { @specialize<Ident, "typeof"> }
+kw_undefined { @specialize<Ident, "undefined"> }
+kw_using { @specialize<Ident, "using"> }
+kw_var { @specialize<Ident, "var"> }
+kw_void { @specialize<Ident, "void"> }
+kw_while { @specialize<Ident, "while"> }
+kw_with { @specialize<Ident, "with"> }
+kw_yield { @specialize<Ident, "yield"> }
 
 @skip { Shebang | JSDoc | TripleSlash | LineComment | BlockComment }
 

examples/lezer/typescript.grammar

@@ -22,7 +22,6 @@
   l0 @right
 }
 
-@top Program { Program }
 Type {
   (Ident (kw_is Type)? | Type AngleL (Type (Comma Type)*)? AngleR | Type BracketL BracketR | Type Op_7c Type | Type Op_26 Type | Op_7c Type | Op_26 Type | kw_keyof Type | kw_typeof TypeofRef | kw_readonly Type | ParenL Type ParenR | TypeParams? ParenL (Param (Comma Param)*)? ParenR Arrow Type | kw_abstract? kw_new TypeParams? ParenL (Param (Comma Param)*)? ParenR Arrow Type | BracketL (Spread? (Ident Question? Colon)? Spread? Type Question? Comma?)* BracketR | BraceL (TypeMember (Semi | Comma)?)* BraceR | kw_asserts Ident (kw_is Type)? | Type kw_extends Type Question Type Colon Type | kw_infer Ident (kw_extends Type (Question Type Colon Type)?)? | String | Number | HexNumber | OctalNumber | BinaryNumber | BigInt | Op_2d (Number | BigInt) | kw_true | kw_false | kw_null | kw_undefined | kw_void | kw_this | kw_unique kw_symbol | kw_import ParenL Type ParenR | Template | Type BracketL Type BracketR | Type Dot Ident)
 }
@@ -167,84 +166,82 @@ ImportClause {
 ImportSpecifier {
   Ident (kw_as Ident)?
 }
-Program {
+@top Program {
   (Decl | Stmt)*
 }
 
 // Keyword specialization over the identifier token:
-@specialize<Ident, keyword> {
-  "abstract" kw_abstract,
-  "accessor" kw_accessor,
-  "as" kw_as,
-  "asserts" kw_asserts,
-  "async" kw_async,
-  "await" kw_await,
-  "break" kw_break,
-  "case" kw_case,
-  "catch" kw_catch,
-  "class" kw_class,
-  "const" kw_const,
-  "constructor" kw_constructor,
-  "continue" kw_continue,
-  "debugger" kw_debugger,
-  "declare" kw_declare,
-  "default" kw_default,
-  "delete" kw_delete,
-  "do" kw_do,
-  "else" kw_else,
-  "enum" kw_enum,
-  "export" kw_export,
-  "extends" kw_extends,
-  "false" kw_false,
-  "finally" kw_finally,
-  "for" kw_for,
-  "from" kw_from,
-  "function" kw_function,
-  "get" kw_get,
-  "if" kw_if,
-  "implements" kw_implements,
-  "import" kw_import,
-  "in" kw_in,
-  "infer" kw_infer,
-  "instanceof" kw_instanceof,
-  "interface" kw_interface,
-  "is" kw_is,
-  "keyof" kw_keyof,
-  "let" kw_let,
-  "meta" kw_meta,
-  "module" kw_module,
-  "namespace" kw_namespace,
-  "new" kw_new,
-  "null" kw_null,
-  "of" kw_of,
-  "out" kw_out,
-  "override" kw_override,
-  "private" kw_private,
-  "protected" kw_protected,
-  "public" kw_public,
-  "readonly" kw_readonly,
-  "return" kw_return,
-  "satisfies" kw_satisfies,
-  "set" kw_set,
-  "static" kw_static,
-  "super" kw_super,
-  "switch" kw_switch,
-  "symbol" kw_symbol,
-  "this" kw_this,
-  "throw" kw_throw,
-  "true" kw_true,
-  "try" kw_try,
-  "type" kw_type,
-  "typeof" kw_typeof,
-  "undefined" kw_undefined,
-  "unique" kw_unique,
-  "using" kw_using,
-  "var" kw_var,
-  "void" kw_void,
-  "while" kw_while,
-  "with" kw_with,
-  "yield" kw_yield
-}
+kw_abstract { @specialize<Ident, "abstract"> }
+kw_accessor { @specialize<Ident, "accessor"> }
+kw_as { @specialize<Ident, "as"> }
+kw_asserts { @specialize<Ident, "asserts"> }
+kw_async { @specialize<Ident, "async"> }
+kw_await { @specialize<Ident, "await"> }
+kw_break { @specialize<Ident, "break"> }
+kw_case { @specialize<Ident, "case"> }
+kw_catch { @specialize<Ident, "catch"> }
+kw_class { @specialize<Ident, "class"> }
+kw_const { @specialize<Ident, "const"> }
+kw_constructor { @specialize<Ident, "constructor"> }
+kw_continue { @specialize<Ident, "continue"> }
+kw_debugger { @specialize<Ident, "debugger"> }
+kw_declare { @specialize<Ident, "declare"> }
+kw_default { @specialize<Ident, "default"> }
+kw_delete { @specialize<Ident, "delete"> }
+kw_do { @specialize<Ident, "do"> }
+kw_else { @specialize<Ident, "else"> }
+kw_enum { @specialize<Ident, "enum"> }
+kw_export { @specialize<Ident, "export"> }
+kw_extends { @specialize<Ident, "extends"> }
+kw_false { @specialize<Ident, "false"> }
+kw_finally { @specialize<Ident, "finally"> }
+kw_for { @specialize<Ident, "for"> }
+kw_from { @specialize<Ident, "from"> }
+kw_function { @specialize<Ident, "function"> }
+kw_get { @specialize<Ident, "get"> }
+kw_if { @specialize<Ident, "if"> }
+kw_implements { @specialize<Ident, "implements"> }
+kw_import { @specialize<Ident, "import"> }
+kw_in { @specialize<Ident, "in"> }
+kw_infer { @specialize<Ident, "infer"> }
+kw_instanceof { @specialize<Ident, "instanceof"> }
+kw_interface { @specialize<Ident, "interface"> }
+kw_is { @specialize<Ident, "is"> }
+kw_keyof { @specialize<Ident, "keyof"> }
+kw_let { @specialize<Ident, "let"> }
+kw_meta { @specialize<Ident, "meta"> }
+kw_module { @specialize<Ident, "module"> }
+kw_namespace { @specialize<Ident, "namespace"> }
+kw_new { @specialize<Ident, "new"> }
+kw_null { @specialize<Ident, "null"> }
+kw_of { @specialize<Ident, "of"> }
+kw_out { @specialize<Ident, "out"> }
+kw_override { @specialize<Ident, "override"> }
+kw_private { @specialize<Ident, "private"> }
+kw_protected { @specialize<Ident, "protected"> }
+kw_public { @specialize<Ident, "public"> }
+kw_readonly { @specialize<Ident, "readonly"> }
+kw_return { @specialize<Ident, "return"> }
+kw_satisfies { @specialize<Ident, "satisfies"> }
+kw_set { @specialize<Ident, "set"> }
+kw_static { @specialize<Ident, "static"> }
+kw_super { @specialize<Ident, "super"> }
+kw_switch { @specialize<Ident, "switch"> }
+kw_symbol { @specialize<Ident, "symbol"> }
+kw_this { @specialize<Ident, "this"> }
+kw_throw { @specialize<Ident, "throw"> }
+kw_true { @specialize<Ident, "true"> }
+kw_try { @specialize<Ident, "try"> }
+kw_type { @specialize<Ident, "type"> }
+kw_typeof { @specialize<Ident, "typeof"> }
+kw_undefined { @specialize<Ident, "undefined"> }
+kw_unique { @specialize<Ident, "unique"> }
+kw_using { @specialize<Ident, "using"> }
+kw_var { @specialize<Ident, "var"> }
+kw_void { @specialize<Ident, "void"> }
+kw_while { @specialize<Ident, "while"> }
+kw_with { @specialize<Ident, "with"> }
+kw_yield { @specialize<Ident, "yield"> }
 
 @skip { Shebang | JSDoc | TripleSlash | LineComment | BlockComment }
 

src/gen-lezer.ts

@@ -902,19 +902,15 @@ export function generateLezer(grammar: LezerGrammar): LezerOutput {
 
   const ruleTexts: string[] = [];
   const entry = findEntryRule(grammar);
-  // Lezer needs a top-level `@top` rule. Reuse the grammar's entry rule.
-  ruleTexts.push(`@top Program { ${entry} }`);
-  if (entry !== 'Program') {
-    // entry rule will also be emitted below as a normal rule
-  }
-
+  // Lezer needs exactly one `@top` rule. PROMOTE the grammar's entry rule to @top
+  // (prefix its own definition) rather than emitting a separate wrapper — a wrapper
+  // named `Program` would collide with an entry rule that is itself named `Program`.
   for (const rule of grammar.rules) {
     const localCtx = { ...exprCtx, selfName: rule.name };
-    if (prattRuleNames.has(rule.name)) {
-      ruleTexts.push(emitPrattRule(rule, grammar, prec, localCtx, incomplete));
-    } else {
-      ruleTexts.push(emitPlainRule(rule, localCtx));
-    }
+    const text = prattRuleNames.has(rule.name)
+      ? emitPrattRule(rule, grammar, prec, localCtx, incomplete)
+      : emitPlainRule(rule, localCtx);
+    ruleTexts.push(rule.name === entry ? `@top ${text}` : text);
   }
 
   // ── Keyword specialization ──
@@ -928,10 +924,12 @@ export function generateLezer(grammar: LezerGrammar): LezerOutput {
 
   const specializeLines: string[] = [];
   if (identToken && keywordLits.size > 0) {
-    const mapping = [...keywordLits].sort()
-      .map(kw => `  ${lezerString(kw)} ${keywordSpecialName(kw)}`)
-      .join(',\n');
-    specializeLines.push(`@specialize<${identToken.name}, keyword> {\n${mapping}\n}`);
+    // Lezer has no `@specialize<tok, name> { … }` block form — each keyword is its own
+    // rule that specializes the identifier token (the inline `@specialize<tok, "word">`
+    // expression, as in @lezer/javascript's `kw<term>` template).
+    for (const kw of [...keywordLits].sort()) {
+      specializeLines.push(`${keywordSpecialName(kw)} { @specialize<${identToken.name}, ${lezerString(kw)}> }`);
+    }
   } else if (keywordLits.size > 0) {
     incomplete.push('Keywords present but no @identifier token to @specialize over.');
   }