Prune longest-match alternatives by their deep FIRST set (#8) (#34)

The three longest-match dispatch loops (parseNonRec, parseLeftRec atoms,
parsePratt nuds) decided whether to try an alternative from its SHALLOW
first token (firstTokenOf): a leading rule ref defeated it (→ always
tried), so a rule-ref-led alt was speculatively parsed and usually failed
at every position — ~57% of all alternative attempts on the TS corpus.

Resolve each alternative's FIRST set through the transitive firstSets
(already computed, previously used only for rule-ref guards) and skip the
alt when the lookahead is provably outside it. Sound by construction:
exprFirst is a complete over-approximation (never omits a startable
token), and a nullable alt is always tried — its only extra match is the
empty one, which never wins the longest-match comparison. Mirrored in
both createParser (gen-parser.ts) and emitParser (emit-parser.ts). The
per-alt FIRST descriptor arrays are precomputed (interpreter) / hoisted to
deduped module-level consts (emitter) so the guard costs nothing per call
— which also removes the same per-call array allocation from the existing
rule-ref firstGuard, and shrinks the emitted TS parser 328KB → 314KB.

Parser-layer ~9% faster (up to ~15% on expression-dense files). Zero
behaviour change: byte-identical CST + accept/reject across the full
18,805-file corpus (createParser ≡ emitParser), interpreter CST identical
to HEAD, run-conformance 5386/5659 unchanged, 26/26 gates pass.

Author: johnsoncodehk

src/emit-parser.ts

@@ -178,13 +178,6 @@ function analyze(grammar: CstGrammar) {
     contMeta.set(ruleName, continuations.map(c => mixfixOf(c, ruleName)));
   }
 
-  // Per-alt first-token (non-recursive + left-rec atoms + pratt nuds use these).
-  const altFirst = new Map<RuleExpr, FirstTok>();
-  for (const rule of grammar.rules) {
-    const alts = rule.body.type === 'alt' ? rule.body.items : [rule.body];
-    for (const alt of alts) altFirst.set(alt, firstTokenOf(alt));
-  }
-
   // Nullability.
   const nullableRules = new Set<string>();
   function exprNullable(e: RuleExpr): boolean {
@@ -257,6 +250,19 @@ function analyze(grammar: CstGrammar) {
     }
   }
 
+  // Deep per-alternative FIRST set + nullability for the longest-match dispatch — the
+  // emitted mirror of gen-parser.ts's altMightStart. An alternative whose FIRST element
+  // is a rule ref (`Decl …`, `Expr …`) is pruned when the lookahead can't begin that
+  // rule (resolved through the transitive firstSets), not only when it begins with a
+  // known literal/token. Sound: exprFirst over-approximates (never omits a startable
+  // token) and a nullable alt is always tried (its empty match never wins longest-match).
+  const altDeepFirst = new Map<RuleExpr, Set<string> | null>();
+  const altNullable = new Map<RuleExpr, boolean>();
+  for (const rule of grammar.rules) {
+    const alts = rule.body.type === 'alt' ? rule.body.items : [rule.body];
+    for (const alt of alts) { altDeepFirst.set(alt, exprFirst(alt)); altNullable.set(alt, exprNullable(alt)); }
+  }
+
   // ── Lever 1: integer token kinds ──
   // Replace the per-call string dispatch in keyMatchesTok / canStartFT /
   // ruleMightStart / matchLiteral / matchToken with integer compares. Two int fields
@@ -350,7 +356,7 @@ function analyze(grammar: CstGrammar) {
   return {
     grammar, tokenNames, opTable, prefixOps, noUnaryLhsOps, postfixOpValues,
     prattRules, leftRecSet, ruleByName, prattClassified, leftRecClassified,
-    maxBp, templateTokenName, templateTokenNames, firstTokenOf, altFirst,
+    maxBp, templateTokenName, templateTokenNames, firstTokenOf, altDeepFirst, altNullable,
     ledMeta, contMeta, nullableRules, firstSets, symtab,
   };
 }
@@ -382,6 +388,12 @@ class Emitter {
   // loops (quantifier/sep) never clash with an enclosing construct's loop.
   private helpers = new Map<string, string>();   // structural key → fn name
   private helperDefs: string[] = [];
+  // Deduped FIRST-set descriptor arrays hoisted to module-level consts (keyed by the
+  // baked literal text). The longest-match alt guards and the rule-ref guards sit on the
+  // hottest dispatch paths; referencing a shared frozen array avoids allocating a fresh
+  // [{…},…] on every call. Spliced at the same `//${HELPERS}` sentinel (top-level, above
+  // the rule fns); only read at runtime inside those fns, so const TDZ is a non-issue.
+  private descConsts = new Map<string, string>();   // baked array literal → const name
   private a: ReturnType<typeof analyze>;
   constructor(a: ReturnType<typeof analyze>) { this.a = a; }
   private id() { return `_t${this.tmp++}`; }
@@ -573,12 +585,35 @@ class Emitter {
     const a = this.a;
     if (a.nullableRules.has(name)) return '';
     const fs = a.firstSets.get(name);
-    if (!fs) return '';
-    // ruleMightStart: true iff some key in fs matches peek(); guard = NOT that.
-    // Pre-classify each FIRST key into an int descriptor (same split keyMatchesTok
-    // does) and bake the descriptor array; the runtime uses integer compares.
-    const descs = [...fs].map(k => this.keyDescLiteral(k));
-    return `!ruleMightStartDescs([${descs.join(', ')}], peek())`;
+    if (!fs || fs.size === 0) return '';
+    // ruleMightStart: true iff some key in fs matches peek(); guard = NOT that. Each FIRST
+    // key is pre-classified into an int descriptor (same split keyMatchesTok does); the
+    // hoisted shared array means the runtime does integer compares with no allocation.
+    return `!ruleMightStartDescs(${this.descArrayConst(fs)}, peek())`;
+  }
+
+  // Deep per-alternative dispatch condition (mirrors gen-parser.ts altMightStart): the
+  // POSITIVE "this alt might start at startTok" test for the longest-match loops. `true`
+  // when the alt is nullable or its FIRST set is unknown/empty (always tried — an empty
+  // match never wins longest-match); else a membership test over the alt's transitive
+  // FIRST set, baked as a hoisted int-descriptor array (same encoding firstGuard uses).
+  altGuard(alt: RuleExpr): string {
+    const a = this.a;
+    if (a.altNullable.get(alt)) return 'true';
+    const fs = a.altDeepFirst.get(alt);
+    if (!fs || fs.size === 0) return 'true';
+    return `ruleMightStartDescs(${this.descArrayConst(fs)}, startTok)`;
+  }
+
+  // Register (deduped) a FIRST-set's baked int-descriptor array as a module-level const
+  // and return its NAME. Keys are sorted so two FIRST sets with the same members share
+  // one const regardless of traversal order (dedup is best-effort; the boolean membership
+  // result `ruleMightStartDescs` computes is order-independent either way).
+  descArrayConst(fs: Set<string>): string {
+    const lit = `[${[...fs].sort().map(k => this.keyDescLiteral(k)).join(', ')}]`;
+    let nm = this.descConsts.get(lit);
+    if (!nm) { nm = `_fs${this.descConsts.size}`; this.descConsts.set(lit, nm); this.helperDefs.push(`const ${nm} = ${lit};`); }
+    return nm;
   }
 
   // ── Lever 1 emit helpers ──
@@ -896,9 +931,8 @@ function emitNonRecRule(e: Emitter, a: ReturnType<typeof analyze>, rule: RuleDec
   e.emit(`  let bestNode = null; let bestPos = saved;`);
   e.emit(`  const startTok = tokens[saved] ?? null;`);
   alts.forEach((alt, i) => {
-    const ft = a.altFirst.get(alt) ?? null;
     e.emit(`  // alt ${i}`);
-    e.emit(`  if (canStartFT(${e.firstTokDescLiteral(ft)}, startTok)) {`);
+    e.emit(`  if (${e.altGuard(alt)}) {`);
     e.emit(`    pos = saved;`);
     e.emit(`    const children = arm_${sanitize(rule.name)}_${i}();`);
     e.emit(`    if (children !== null && pos > bestPos) {`);
@@ -931,8 +965,7 @@ function emitLeftRecRule(e: Emitter, a: ReturnType<typeof analyze>, rule: RuleDe
   e.emit(`  let node = null; let bestAtomPos = saved;`);
   e.emit(`  const startTok = tokens[saved] ?? null;`);
   atoms.forEach((atom, i) => {
-    const ft = a.altFirst.get(atom) ?? null;
-    e.emit(`  if (canStartFT(${e.firstTokDescLiteral(ft)}, startTok)) {`);
+    e.emit(`  if (${e.altGuard(atom)}) {`);
     e.emit(`    pos = saved;`);
     e.emit(`    const children = atom_${sanitize(rule.name)}_${i}();`);
     e.emit(`    if (children !== null && pos > bestAtomPos) {`);
@@ -987,9 +1020,8 @@ function emitPrattRule(e: Emitter, a: ReturnType<typeof analyze>, rule: RuleDecl
   // NUD loop.
   nuds.forEach((nud, i) => {
     const items = nud.type === 'seq' ? nud.items : [nud];
-    const ft = a.altFirst.get(nud) ?? null;
     e.emit(`  // nud ${i}`);
-    e.emit(`  if (canStartFT(${e.firstTokDescLiteral(ft)}, startTok)) {`);
+    e.emit(`  if (${e.altGuard(nud)}) {`);
     e.emit(`    pos = saved;`);
     if (items[0]?.type === 'prefix') {
       // prefix $ pattern: identical to parsePratt's prefix branch.

src/gen-parser.ts

@@ -337,11 +337,12 @@ export function createParser(grammar: CstGrammar) {
   const templateTokenNames = new Set<string>(grammar.tokens.filter(t => t.template).map(t => t.name));
 
   // ── First-token dispatch ──
-  // For each alternative, the token it MUST begin with, if that is statically
-  // knowable (a leading literal or token ref). When known, the alternative can
-  // be skipped outright if the current token can't be its first token — so the
-  // alternative loops branch on the lookahead instead of trying every arm.
-  // `null` = not statically knowable (rule ref / prefix / optional first) → always try.
+  // The single token an expression MUST begin with, if statically knowable (a leading
+  // literal or token ref); `null` = not knowable (rule ref / prefix / optional first) →
+  // always try. The alternative loops now use the deeper `altMightStart` (which resolves
+  // a leading rule ref through `firstSets`); `firstTokenOf` remains the dispatch for the
+  // Pratt LED continuations (`ledFirst`), whose connectors are operator literals for
+  // which the single-token form is already exact.
   type FirstTok = { lit: string } | { tok: string } | null;
   function firstTokenOf(alt: RuleExpr): FirstTok {
     const items = alt.type === 'seq' ? alt.items : [alt];
@@ -351,11 +352,6 @@ export function createParser(grammar: CstGrammar) {
     if (first.type === 'ref' && tokenNames.has(first.name)) return { tok: first.name };
     return null;
   }
-  const firstOf = new Map<RuleExpr, FirstTok>();
-  for (const rule of grammar.rules) {
-    const alts = rule.body.type === 'alt' ? rule.body.items : [rule.body];
-    for (const alt of alts) firstOf.set(alt, firstTokenOf(alt));
-  }
   // Does a FIRST-set key (a token name, or a literal keyword/punctuation) match a token?
   function keyMatchesTok(key: string, tok: Token): boolean {
     if (tokenNames.has(key)) {
@@ -520,6 +516,42 @@ export function createParser(grammar: CstGrammar) {
     return false;
   }
 
+  // ── Deep per-alternative dispatch ──
+  // The shallow `firstTokenOf` above only prunes an alternative when its FIRST element
+  // is a literal or a *token* ref; a leading *rule* ref (e.g. an alt `Decl …` / `Expr …`)
+  // defeats it (→ null → always tried). But the longest-match loops try EVERY alt that
+  // isn't pruned, so a rule-ref-led alt is speculatively parsed (and usually fails) at
+  // every position — measured at ~57% of all alternative attempts on real TS. The full
+  // transitive `firstSets` already knows what each rule can begin with, so resolve the
+  // alt's FIRST set through it and prune the alt when the lookahead is provably outside.
+  //
+  // Sound by construction: `exprFirst` is a sound OVER-approximation (it never omits a
+  // token a non-empty match could begin with), so an alt pruned here genuinely cannot
+  // match non-empty at this token. A NULLABLE alt is always tried — its only extra match
+  // is the empty one, and an empty match never wins the longest-match comparison
+  // (`pos === saved`, never `> bestPos`), so behaviour is identical with or without it.
+  // Strictly dominates `canStart(firstOf…)`: whenever the shallow check pruned, the deep
+  // FIRST set (whose leading member is that same literal/token) prunes too.
+  // Precompute each alt's dispatch keys ONCE: `null` = always try (nullable / unknowable
+  // / empty FIRST — collapsed so the hot path is a single branch), else the flat array of
+  // FIRST-set keys to test the lookahead against. Doing the nullability + FIRST resolution
+  // here keeps `altMightStart` to a map lookup + a bounded scan — no per-call tree walk.
+  const altDispatch = new Map<RuleExpr, string[] | null>();
+  for (const rule of grammar.rules) {
+    const alts = rule.body.type === 'alt' ? rule.body.items : [rule.body];
+    for (const alt of alts) {
+      const fs = exprNullable(alt) ? null : exprFirst(alt);
+      altDispatch.set(alt, fs && fs.size > 0 ? [...fs] : null);
+    }
+  }
+  function altMightStart(alt: RuleExpr, tok: Token | null): boolean {
+    if (!tok) return true;
+    const keys = altDispatch.get(alt);
+    if (!keys) return true;                                      // always try (nullable / top / empty)
+    for (let i = 0; i < keys.length; i++) if (keyMatchesTok(keys[i], tok)) return true;
+    return false;
+  }
+
   // ── Parser core ──
 
   const profCounts = new Map<string, number>();
@@ -703,7 +735,7 @@ export function createParser(grammar: CstGrammar) {
       const startTok = tokens[saved] ?? null;
 
       for (const alt of alts) {
-        if (!canStart(firstOf.get(alt), startTok)) continue;
+        if (!altMightStart(alt, startTok)) continue;
         pos = saved;
         // The markup container arm (HTML element with children) is matched by a
         // dedicated path that honours optional end tags — content stops at a trigger
@@ -735,7 +767,7 @@ export function createParser(grammar: CstGrammar) {
       let bestAtomPos = saved;
       const startTok = tokens[saved] ?? null;
       for (const atom of atoms) {
-        if (!canStart(firstOf.get(atom), startTok)) continue;
+        if (!altMightStart(atom, startTok)) continue;
         pos = saved;
         const children = matchExpr(atom);
         if (children !== null && pos > bestAtomPos) {
@@ -791,7 +823,7 @@ export function createParser(grammar: CstGrammar) {
       let bestNudPos = saved;
       const startTok = tokens[saved] ?? null;
       for (const nud of nuds) {
-        if (!canStart(firstOf.get(nud), startTok)) continue;
+        if (!altMightStart(nud, startTok)) continue;
         pos = saved;
         const items = nud.type === 'seq' ? nud.items : [nud];