Add O(1) lid_of keyword early-exit prefilter across portable targets.

Ident-shaped texts miss a length×first-byte bit table derived from keyword
literals before any string match; rust splits kw/punct match tables so
non-keyword Idents skip punct arms too.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: johnsoncodehk

src/emit-portable.ts

@@ -676,6 +676,60 @@ export function lidOf(plan: LexIdPlan, text: string): number {
   return i >= 0 ? i : 0;
 }
 
+/**
+ * O(1) early-exit prefilter for runtime `lid_of` on Ident-shaped texts.
+ *
+ * - `maxByteLen` spans ALL literals (so oversize still ⇒ lid 0, puncts included).
+ * - `firstByLenBits` is filled only from `isKeywordLiteral` texts (Ident / `@kw` shapes).
+ * Runtime applies the bit probe only when the first byte is Ident-start or `@`; punct first
+ * bytes skip the probe and go straight to the slow path — no tax on the punct hot path.
+ * Conservatively correct: every keyword-shaped literal hits; non-keywords may fall through.
+ *
+ * Layout: `(maxByteLen + 1)` rows × 32 bytes (256 bits).
+ */
+export type LidPrefilterPlan = {
+  maxByteLen: number;
+  /** Flat bit table, length `(maxByteLen + 1) * 32` — keyword-shaped (len × first) only. */
+  firstByLenBits: Uint8Array;
+};
+
+/** True if first byte is Ident-start (`A-Za-z_$`) or `@` (decorator-keyword shape). */
+export function lidPrefilterAppliesToFirst(b0: number): boolean {
+  return (b0 >= 65 && b0 <= 90) || (b0 >= 97 && b0 <= 122) || b0 === 95 || b0 === 36 || b0 === 64;
+}
+
+/** Build length×first-byte prefilter: maxLen from all lids; bits from keyword-shaped lids only. */
+export function buildLidPrefilter(plan: LexIdPlan): LidPrefilterPlan {
+  let maxByteLen = 0;
+  const kw: { len: number; first: number }[] = [];
+  for (let i = 1; i < plan.lids.length; i++) {
+    const text = plan.lids[i]!;
+    const buf = Buffer.from(text, 'utf8');
+    if (buf.length === 0) continue;
+    maxByteLen = Math.max(maxByteLen, buf.length);
+    if (isKeywordLiteral(text)) kw.push({ len: buf.length, first: buf[0]! });
+  }
+  const firstByLenBits = new Uint8Array((maxByteLen + 1) * 32);
+  for (const { len, first } of kw) {
+    firstByLenBits[len * 32 + (first >> 3)] |= 1 << (first & 7);
+  }
+  return { maxByteLen, firstByLenBits };
+}
+
+/**
+ * Probe the prefilter (mirrors emitted runtime).
+ * Non-Ident-shaped texts are never rejected here (punct path).
+ * True ⇒ may be a literal (must consult slow path); false ⇒ definite lid 0 for Ident-shaped.
+ */
+export function lidPrefilterMayHit(pf: LidPrefilterPlan, text: string): boolean {
+  const buf = Buffer.from(text, 'utf8');
+  const n = buf.length;
+  if (n === 0 || n > pf.maxByteLen) return false;
+  const b0 = buf[0]!;
+  if (!lidPrefilterAppliesToFirst(b0)) return true; // punct-shaped — defer to slow path
+  return (pf.firstByLenBits[n * 32 + (b0 >> 3)]! & (1 << (b0 & 7))) !== 0;
+}
+
 /** Resolve kid for a known kind; 0 if not in the plan. */
 export function kidOf(plan: LexIdPlan, kind: string): number {
   const i = plan.kids.indexOf(kind);

src/target-go.ts

@@ -10,7 +10,7 @@
 // slices to saved lengths; the slices keep their capacity across parses (reset to len 0), so a
 // warmed parser allocates ~nothing per parse.
 import { type ParserIR, type RdRule, type PrattRule, type Step, type Bracket, type CharRange, type LexTok, type TplCfg, type NewlineCfg, type FirstSig, type LexFirstBytes, type LexIdPlan, type ArenaIdPlan } from './emit-portable.ts';
-import { portableIR, buildLexDispatchPlan, lexTokFirstBytes, punctFirstBytes, buildLexIdPlan, buildArenaIdPlan, lidOf, kidOf, lidFlagTable, kidFlagTable, ttIdOf, ruleIdOf, TT_SKIP_PUNCT, rangesHaveNonAscii, isFirstGuardable, groupByPreserveOrder } from './emit-portable.ts';
+import { portableIR, buildLexDispatchPlan, lexTokFirstBytes, punctFirstBytes, buildLexIdPlan, buildLidPrefilter, buildArenaIdPlan, lidOf, kidOf, lidFlagTable, kidFlagTable, ttIdOf, ruleIdOf, TT_SKIP_PUNCT, rangesHaveNonAscii, isFirstGuardable, groupByPreserveOrder } from './emit-portable.ts';
 import type { Target } from './emit.ts';
 import type { TokenPattern, CstGrammar } from './types.ts';
 
@@ -44,17 +44,31 @@ const isGuardable = (f: FirstSig, nAlts?: number): f is NonNullable<FirstSig> =>
 function renderIdTablesGo(ids: LexIdPlan): string {
   const kidsLit = ids.kids.map(J).join(', ');
   const lidsLit = ids.lids.map(J).join(', ');
+  const pf = buildLidPrefilter(ids);
+  const bitsLit = [...pf.firstByLenBits].join(', ');
   return `var KIND_STR = []string{${kidsLit}}
 var _lids = []string{${lidsLit}}
 var _kidMap = map[string]uint16{}
 var _lidMap = map[string]uint16{}
+const _lidMaxLen = ${pf.maxByteLen}
+var _lidFirstBits = [...]byte{${bitsLit}}
 
 func init() {
 \tfor i, k := range KIND_STR { _kidMap[k] = uint16(i) }
 \tfor i, t := range _lids { _lidMap[t] = uint16(i) }
 }
 func kidOf(kind string) uint16 { if v, ok := _kidMap[kind]; ok { return v }; return 0 }
-func lidOf(text string) uint16 { if v, ok := _lidMap[text]; ok { return v }; return 0 }
+func lidOf(text string) uint16 {
+\tn := len(text)
+\tif n == 0 || n > _lidMaxLen { return 0 }
+\tb0 := text[0]
+\t// Ident/@-keyword shape: O(1) length×first-byte miss ⇒ lid 0. Punct first bytes skip the probe.
+\tif (b0 >= 'A' && b0 <= 'Z') || (b0 >= 'a' && b0 <= 'z') || b0 == '_' || b0 == '$' || b0 == '@' {
+\t\tif _lidFirstBits[n*32+(int(b0)>>3)]&(1<<(b0&7)) == 0 { return 0 }
+\t}
+\tif v, ok := _lidMap[text]; ok { return v }
+\treturn 0
+}
 func tokKind(t *Tok) string { return KIND_STR[t.Kid] }
 func tokText(src string, t *Tok) string { return src[t.Off:t.End] }
 func mkTok(off, end int, nl bool, kid, lid uint16) Tok { return Tok{Off: uint32(off), End: uint32(end), Kid: kid, Lid: lid, Nl: nl} }

src/target-rust.ts

@@ -12,7 +12,8 @@
 // combinators take non-capturing `fn(&mut Parser) -> bool` pointers (the kids vec is now on
 // the Parser as `scratch`, so the second param the old owned-tree version threaded is gone).
 import { type ParserIR, type RdRule, type PrattRule, type Step, type Bracket, type CharRange, type LexTok, type TplCfg, type NewlineCfg, type FirstSig, type LexFirstBytes, type LexIdPlan, type ArenaIdPlan } from './emit-portable.ts';
-import { portableIR, buildLexDispatchPlan, lexTokFirstBytes, punctFirstBytes, buildLexIdPlan, buildArenaIdPlan, lidOf, kidOf, lidFlagTable, kidFlagTable, ttIdOf, ruleIdOf, TT_SKIP_PUNCT, rangesHaveNonAscii, isFirstGuardable, groupByPreserveOrder } from './emit-portable.ts';
+import { portableIR, buildLexDispatchPlan, lexTokFirstBytes, punctFirstBytes, buildLexIdPlan, buildLidPrefilter, buildArenaIdPlan, lidOf, kidOf, lidFlagTable, kidFlagTable, ttIdOf, ruleIdOf, TT_SKIP_PUNCT, rangesHaveNonAscii, isFirstGuardable, groupByPreserveOrder } from './emit-portable.ts';
+import { isKeywordLiteral } from './grammar-utils.ts';
 import type { Target } from './emit.ts';
 import type { TokenPattern, CstGrammar } from './types.ts';
 
@@ -48,28 +49,59 @@ function renderIdTablesRust(ids: LexIdPlan): string {
   const kidsLit = ids.kids.map(J).join(', ');
   const lidsLit = ids.lids.map(J).join(', ');
   const kidArms = ids.kids.map((k, i) => `${J(k)} => ${i}`).join(', ');
-  // Group lids[1..] by UTF-8 byte length for a two-level match (len → text). Rust `str::len` is bytes.
-  const byLen = new Map<number, { text: string; id: number }[]>();
+  // Group lids[1..] by UTF-8 byte length; split keyword-shaped vs punct for separate match tables.
+  const byLenKw = new Map<number, { text: string; id: number }[]>();
+  const byLenPu = new Map<number, { text: string; id: number }[]>();
   for (let i = 1; i < ids.lids.length; i++) {
-    const text = ids.lids[i];
+    const text = ids.lids[i]!;
     const blen = Buffer.byteLength(text);
-    const arr = byLen.get(blen) ?? [];
-    arr.push({ text, id: i });
-    byLen.set(blen, arr);
+    const ent = { text, id: i };
+    const map = isKeywordLiteral(text) ? byLenKw : byLenPu;
+    const arr = map.get(blen) ?? [];
+    arr.push(ent);
+    map.set(blen, arr);
   }
-  const lenArms = [...byLen.entries()].sort((a, b) => a[0] - b[0]).map(([len, ents]) => {
-    const arms = ents.map((e) => `${J(e.text)} => ${e.id}`).join(', ');
-    return `        ${len} => match text { ${arms}, _ => 0 },`;
-  }).join('\n');
+  const lenArmsOf = (byLen: Map<number, { text: string; id: number }[]>) =>
+    [...byLen.entries()].sort((a, b) => a[0] - b[0]).map(([len, ents]) => {
+      const arms = ents.map((e) => `${J(e.text)} => ${e.id}`).join(', ');
+      return `        ${len} => match text { ${arms}, _ => 0 },`;
+    }).join('\n');
+  const kwArms = lenArmsOf(byLenKw);
+  const puArms = lenArmsOf(byLenPu);
+  const pf = buildLidPrefilter(ids);
+  const bitsLit = [...pf.firstByLenBits].join(', ');
   return `const KIND_STR: &[&str] = &[${kidsLit}];
 const _LIDS: &[&str] = &[${lidsLit}];
+const _LID_MAX_LEN: usize = ${pf.maxByteLen};
+const _LID_FIRST_BITS: &[u8] = &[${bitsLit}];
 #[inline(always)] fn tok_kind(t: &Tok) -> &'static str { KIND_STR[t.kid as usize] }
 #[inline(always)] fn tok_text<'a>(src: &'a str, t: &Tok) -> &'a str { &src[t.off as usize..t.end as usize] }
 #[inline(always)] fn mk_tok(off: usize, end: usize, nl: bool, kid: u16, lid: u16) -> Tok { Tok { off: off as u32, end: end as u32, kid, lid, nl } }
 fn kid_of(kind: &str) -> u16 { match kind { ${kidArms}, _ => 0 } }
+/// Ident/@-keyword: O(1) length×first-byte prefilter, then keyword-only match (no punct arms).
+#[inline(always)]
 fn lid_of(text: &str) -> u16 {
-    match text.len() {
-${lenArms || '        // no non-empty lids'}
+    let n = text.len();
+    if n == 0 || n > _LID_MAX_LEN { return 0; }
+    let b0 = text.as_bytes()[0];
+    if matches!(b0, b'A'..=b'Z' | b'a'..=b'z' | b'_' | b'$' | b'@') {
+        let b0u = b0 as usize;
+        if (_LID_FIRST_BITS[n * 32 + (b0u >> 3)] & (1u8 << (b0u & 7))) == 0 { return 0; }
+        return lid_of_kw(text, n);
+    }
+    lid_of_punct(text, n)
+}
+#[inline(never)]
+fn lid_of_kw(text: &str, n: usize) -> u16 {
+    match n {
+${kwArms || '        // no keyword lids'}
+        _ => 0,
+    }
+}
+#[inline(never)]
+fn lid_of_punct(text: &str, n: usize) -> u16 {
+    match n {
+${puArms || '        // no punct lids'}
         _ => 0,
     }
 }

src/target-ts.ts

@@ -5,7 +5,7 @@
 // is checked byte-for-byte against the interpreter (createParser), so a divergence in the
 // portable logic surfaces here before Go/Rust are compiled.
 import { type ParserIR, type RdRule, type PrattRule, type Step, type Bracket, type CharRange, type LexTok, type TplCfg, type NewlineCfg, type FirstSig, type LexFirstBytes, type LexIdPlan } from './emit-portable.ts';
-import { portableIR, buildLexDispatchPlan, lexTokFirstBytes, punctFirstBytes, buildLexIdPlan, lidOf, kidOf, lidFlagTable, kidFlagTable, rangesHaveNonAscii, isFirstGuardable, groupByPreserveOrder } from './emit-portable.ts';
+import { portableIR, buildLexDispatchPlan, lexTokFirstBytes, punctFirstBytes, buildLexIdPlan, buildLidPrefilter, lidOf, kidOf, lidFlagTable, kidFlagTable, rangesHaveNonAscii, isFirstGuardable, groupByPreserveOrder } from './emit-portable.ts';
 import type { Target } from './emit.ts';
 import type { CstGrammar } from './types.ts';
 
@@ -38,12 +38,25 @@ const isGuardable = (f: FirstSig, nAlts?: number): f is NonNullable<FirstSig> =>
 
 /** Emit kid/lid lookup tables into generated lexer source. */
 function renderIdTablesTS(ids: LexIdPlan): string {
+  const pf = buildLidPrefilter(ids);
+  const bitsLit = [...pf.firstByLenBits].join(', ');
   return `const KIND_STR: string[] = ${J(ids.kids)};
 const _LIDS: string[] = ${J(ids.lids)};
 const _KID_MAP = new Map<string, number>(KIND_STR.map((k, i) => [k, i]));
 const _LID_MAP = new Map<string, number>(_LIDS.map((t, i) => [t, i]));
+const _LID_MAX_LEN = ${pf.maxByteLen};
+const _LID_FIRST_BITS = new Uint8Array([${bitsLit}]);
 function kid_of(kind: string): number { return _KID_MAP.get(kind) ?? 0; }
-function lid_of(text: string): number { return _LID_MAP.get(text) ?? 0; }
+function lid_of(text: string): number {
+  const n = text.length;
+  if (n === 0 || n > _LID_MAX_LEN) return 0;
+  const b0 = text.charCodeAt(0);
+  // Ident/@-keyword shape: O(1) length×first-byte miss ⇒ lid 0. Punct first bytes skip the probe.
+  if ((b0 >= 65 && b0 <= 90) || (b0 >= 97 && b0 <= 122) || b0 === 95 || b0 === 36 || b0 === 64) {
+    if ((_LID_FIRST_BITS[n * 32 + (b0 >> 3)]! & (1 << (b0 & 7))) === 0) return 0;
+  }
+  return _LID_MAP.get(text) ?? 0;
+}
 function tok_kind(t: Tok): string { return KIND_STR[t.kid]!; }
 function tok_text(src: string, t: Tok): string { return src.slice(t.off, t.end); }
 function mk_tok(off: number, end: number, nl: boolean, kid: number, lid: number): Tok { return { off, end, nl, kid, lid }; }