Skip to content

Commit c066d1b

Browse files
johnsoncodehkcursoragent
johnsoncodehk
and
cursoragent
committed
Add first-byte dispatch to portable lexer emission across ts/go/rust targets.
Lexer cascade now switches on the current byte before trying matchers, cutting linear scans on hot paths like punctuation while preserving token stream semantics. Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent 6847134 commit c066d1b

4 files changed

Lines changed: 319 additions & 25 deletions

File tree

‎src/emit-portable.ts‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -521,3 +521,165 @@ function buildPratt(
521521
: [];
522522
return { kind: 'pratt', name, cstName, nudToks, nudBrackets, nudSeqs, nudSeqFirst: [], nudSeqPredictive: false, nudCapped, nudCappedFirst: [], prefix, binary, leds, ledAccessTail, ledLbp, ledSameLine, ledNotLeftLeaf, postfixToks, postfix };
523523
}
524+
525+
// ── Lexer first-byte dispatch (IR analysis; targets render buckets) ──
526+
527+
export type LexFirstBytes = { bytes: number[]; nonAscii: boolean };
528+
529+
/** Conservative FIRST-byte analysis for a punct literal. */
530+
export function punctFirstBytes(value: string): LexFirstBytes | null {
531+
if (value.length === 0) return null;
532+
const c = value.charCodeAt(0);
533+
if (c >= 128) return { bytes: [], nonAscii: true };
534+
return { bytes: [c], nonAscii: false };
535+
}
536+
537+
function expandRangeToAscii(lo: number, hi: number, out: Set<number>): boolean {
538+
let nonAscii = false;
539+
if (lo >= 128 || hi >= 128) nonAscii = true;
540+
for (let c = Math.max(0, lo); c <= Math.min(127, hi); c++) out.add(c);
541+
return nonAscii;
542+
}
543+
544+
function rangesToFirstBytes(ranges: CharRange[]): LexFirstBytes {
545+
const bytes = new Set<number>();
546+
let nonAscii = false;
547+
for (const [lo, hi] of ranges) nonAscii = expandRangeToAscii(lo, hi, bytes) || nonAscii;
548+
return { bytes: [...bytes].sort((a, b) => a - b), nonAscii };
549+
}
550+
551+
function patternNullable(p: TokenPattern): boolean {
552+
if (typeof p === 'string') return false;
553+
switch (p.type) {
554+
case 'repeat': return p.min === 0;
555+
case 'lookahead': case 'lookbehind': case 'anchor': case 'never': return true;
556+
default: return false;
557+
}
558+
}
559+
560+
function patternFirst(p: TokenPattern): LexFirstBytes | null {
561+
if (typeof p === 'string') {
562+
const c = p.charCodeAt(0);
563+
if (c >= 128) return { bytes: [], nonAscii: true };
564+
return { bytes: [c], nonAscii: false };
565+
}
566+
switch (p.type) {
567+
case 'seq': {
568+
const bytes = new Set<number>();
569+
let nonAscii = false;
570+
for (const item of p.items) {
571+
const fb = patternFirst(item);
572+
if (fb === null) return null;
573+
fb.bytes.forEach((b) => bytes.add(b));
574+
nonAscii = nonAscii || fb.nonAscii;
575+
if (!patternNullable(item)) break;
576+
}
577+
return { bytes: [...bytes].sort((a, b) => a - b), nonAscii };
578+
}
579+
case 'alt': {
580+
const bytes = new Set<number>();
581+
let nonAscii = false;
582+
for (const item of p.items) {
583+
const fb = patternFirst(item);
584+
if (fb === null) return null;
585+
fb.bytes.forEach((b) => bytes.add(b));
586+
nonAscii = nonAscii || fb.nonAscii;
587+
}
588+
return { bytes: [...bytes].sort((a, b) => a - b), nonAscii };
589+
}
590+
case 'charClass': {
591+
if (p.negate) {
592+
const bytes = new Set<number>();
593+
for (let c = 0; c <= 127; c++) bytes.add(c);
594+
return { bytes: [...bytes], nonAscii: true };
595+
}
596+
const bytes = new Set<number>();
597+
let nonAscii = false;
598+
for (const item of p.items) {
599+
if (item.type === 'char') {
600+
const c = item.value.charCodeAt(0);
601+
if (c <= 127) bytes.add(c); else nonAscii = true;
602+
} else {
603+
nonAscii = expandRangeToAscii(item.from.charCodeAt(0), item.to.charCodeAt(0), bytes) || nonAscii;
604+
}
605+
}
606+
return { bytes: [...bytes].sort((a, b) => a - b), nonAscii };
607+
}
608+
case 'repeat': return patternFirst(p.body);
609+
case 'anyChar': return null;
610+
case 'lookahead': case 'lookbehind': case 'anchor': case 'never':
611+
return { bytes: [], nonAscii: false };
612+
default: return null;
613+
}
614+
}
615+
616+
/** Which ASCII bytes (0..127) may start this token matcher; nonAscii ⇒ also in fallback. null ⇒ all bytes + nonAscii. */
617+
export function lexTokFirstBytes(t: LexTok): LexFirstBytes | null {
618+
switch (t.kind) {
619+
case 'run': return rangesToFirstBytes(t.first);
620+
case 'string': return punctFirstBytes(t.delim);
621+
case 'line': return punctFirstBytes(t.prefix);
622+
case 'block': return punctFirstBytes(t.open);
623+
case 'pattern': return patternFirst(t.pattern);
624+
}
625+
}
626+
627+
export type LexDispatchMeta = {
628+
key: string;
629+
shape: LexTok['kind'] | 'punct';
630+
firstBytes: LexFirstBytes | null;
631+
};
632+
633+
export type LexDispatchArm = { bytes: number[]; indices: number[] };
634+
635+
/** Build per-byte buckets (0..127) over a fixed global candidate sequence; fallback = full sequence. */
636+
export function buildLexDispatchPlan(firsts: (LexFirstBytes | null)[]): { arms: LexDispatchArm[]; fallbackIndices: number[] } {
637+
const n = firsts.length;
638+
const fallbackIndices = [...Array(n).keys()];
639+
const asciiBuckets: number[][] = Array.from({ length: 128 }, () => []);
640+
for (let i = 0; i < n; i++) {
641+
const fb = firsts[i];
642+
if (fb === null) {
643+
for (let b = 0; b < 128; b++) asciiBuckets[b].push(i);
644+
} else {
645+
for (const b of fb.bytes) if (b >= 0 && b < 128) asciiBuckets[b].push(i);
646+
}
647+
}
648+
const keyToBytes = new Map<string, number[]>();
649+
const keyToIndices = new Map<string, number[]>();
650+
for (let b = 0; b < 128; b++) {
651+
const indices = asciiBuckets[b];
652+
if (!indices.length) continue;
653+
const key = indices.join(',');
654+
if (!keyToIndices.has(key)) { keyToIndices.set(key, indices); keyToBytes.set(key, []); }
655+
keyToBytes.get(key)!.push(b);
656+
}
657+
const arms: LexDispatchArm[] = [...keyToIndices.entries()].map(([key, indices]) => ({
658+
bytes: keyToBytes.get(key)!.sort((a, b) => a - b),
659+
indices,
660+
}));
661+
arms.sort((a, b) => a.bytes[0] - b.bytes[0]);
662+
return { arms, fallbackIndices };
663+
}
664+
665+
/** Interval notation for report tables, e.g. "a-z, A-Z, 48". */
666+
export function formatFirstBytes(fb: LexFirstBytes | null): string {
667+
if (fb === null) return 'null (all ASCII + nonAscii)';
668+
if (!fb.bytes.length && fb.nonAscii) return '(empty ASCII) nonAscii';
669+
const parts: string[] = [];
670+
const sorted = [...fb.bytes].sort((a, b) => a - b);
671+
for (let i = 0; i < sorted.length; i++) {
672+
const lo = sorted[i];
673+
let hi = lo;
674+
while (i + 1 < sorted.length && sorted[i + 1] === hi + 1) { hi = sorted[++i]; }
675+
if (lo === hi) {
676+
parts.push(lo >= 32 && lo <= 126 ? `'${String.fromCharCode(lo)}'(${lo})` : String(lo));
677+
} else {
678+
const ls = lo >= 32 && lo <= 126 ? `'${String.fromCharCode(lo)}'` : String(lo);
679+
const hs = hi >= 32 && hi <= 126 ? `'${String.fromCharCode(hi)}'` : String(hi);
680+
parts.push(`${ls}..${hs}`);
681+
}
682+
}
683+
if (fb.nonAscii) parts.push('nonAscii');
684+
return parts.join(', ');
685+
}

‎src/target-go.ts‎

Lines changed: 43 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@
99
// stack. A node is an int32 index, never a heap pointer. Backtracking truncates the three
1010
// slices to saved lengths; the slices keep their capacity across parses (reset to len 0), so a
1111
// warmed parser allocates ~nothing per parse.
12-
import type { ParserIR, RdRule, PrattRule, Step, Bracket, CharRange, LexTok, TplCfg, NewlineCfg, FirstSig } from './emit-portable.ts';
13-
import { portableIR } from './emit-portable.ts';
12+
import type { ParserIR, RdRule, PrattRule, Step, Bracket, CharRange, LexTok, TplCfg, NewlineCfg, FirstSig, LexFirstBytes } from './emit-portable.ts';
13+
import { portableIR, buildLexDispatchPlan, lexTokFirstBytes, punctFirstBytes } from './emit-portable.ts';
1414
import type { Target } from './emit.ts';
1515
import type { TokenPattern, CstGrammar } from './types.ts';
1616

@@ -81,6 +81,42 @@ function scanTok(t: LexTok, defs: string[], stateful: boolean, rxTok?: string, t
8181
return `\t\tif ${gate ? gate + 'true' : 'true'} { if e := ${m}(pos); e > pos { ${push('e')}pos = e; continue } }`;
8282
}
8383

84+
function buildLexCandidates(
85+
ir: ParserIR, defs: string[], stateful: boolean, rxTok: string | undefined, tplTok: string | undefined,
86+
punctLine: (p: string) => string,
87+
): { codes: string[]; firsts: (LexFirstBytes | null)[] } {
88+
const codes: string[] = [];
89+
const firsts: (LexFirstBytes | null)[] = [];
90+
for (const t of ir.tokens) {
91+
const code = scanTok(t, defs, stateful, rxTok, tplTok);
92+
if (!code) continue;
93+
codes.push(code);
94+
firsts.push(lexTokFirstBytes(t));
95+
}
96+
for (const p of ir.puncts) {
97+
codes.push(punctLine(p));
98+
firsts.push(punctFirstBytes(p));
99+
}
100+
return { codes, firsts };
101+
}
102+
103+
/** Shared first-byte dispatch for all lexFrom variants in this target. */
104+
function renderLexByteDispatchGo(codes: string[], firsts: (LexFirstBytes | null)[], indent: string): string {
105+
const { arms, fallbackIndices } = buildLexDispatchPlan(firsts);
106+
const fallback = fallbackIndices.map((i) => codes[i]).join('\n');
107+
let switchArms = '';
108+
for (const arm of arms) {
109+
switchArms += `${indent}\tcase ${arm.bytes.join(', ')}:\n`;
110+
switchArms += arm.indices.map((i) => codes[i]).join('\n') + '\n';
111+
}
112+
return `${indent}if c >= 128 {
113+
${fallback}
114+
${indent}} else {
115+
${indent}\tswitch c {
116+
${switchArms}${indent}\t}
117+
${indent}}`;
118+
}
119+
84120
function newlinePartsGo(nl: NewlineCfg, pushFn: string): { state: string; stateFrom: string; boundary: string; ws: string; hooks: string } {
85121
const commentSkip = nl.comment
86122
? `\t\tif strings.HasPrefix(src[p:], ${J(nl.comment)}) { e := p; for e < n && src[e] != 10 { e++ }; pos = e; continue }\n`
@@ -142,10 +178,11 @@ function lexer(ir: ParserIR): string {
142178
const rxOrTpl = !!(rx || tpl) && !rxOnly && !tplOnly && !rxTpl;
143179
const stateful = !!(rx || tpl);
144180
const newlineOnly = !!(nl && !rx && !tpl);
145-
const toks = ir.tokens.map((t) => scanTok(t, defs, stateful, rx?.regexToken, tpl?.token)).join('\n');
146181
const pushPunct = stateful ? (p: string) => `emit("", ${J(p)}, pos, pos + ${p.length})` : (p: string) => `pushTok("", ${J(p)}, pos, pos + ${p.length})`;
147-
const puncts = ir.puncts.map((p) =>
148-
`\t\tif strings.HasPrefix(src[pos:], ${J(p)}) { ${pushPunct(p)}; pos += ${p.length}; continue }`).join('\n');
182+
const punctLine = (p: string) =>
183+
`\t\tif strings.HasPrefix(src[pos:], ${J(p)}) { ${pushPunct(p)}; pos += ${p.length}; continue }`;
184+
const { codes: lexCodes, firsts: lexFirsts } = buildLexCandidates(ir, defs, stateful, rx?.regexToken, tpl?.token, punctLine);
185+
const cascade = renderLexByteDispatchGo(lexCodes, lexFirsts, '\t');
149186
const goMap = (a: string[]) => `map[string]bool{${a.map((x) => `${J(x)}: true`).join(', ')}}`;
150187
const rxState = rx ? `\tprevText, prevKind, bpText := "", "", ""
151188
\thasPrev, hasPrev2 := false, false
@@ -292,8 +329,7 @@ ${pushHooks}\t\t*acc = append(*acc, Tok{kind, text, off, end, pendingNl}); pendi
292329
\t_ = pushTok
293330
`;
294331
const loopBody = `${nlBoundary}\t\tc := int(src[pos])
295-
${nlWs}${tplDispatch}${toks}
296-
${puncts}
332+
${nlWs}${tplDispatch}${cascade}
297333
\t\tpanic(fmt.Sprintf("lex error at %d", pos))`;
298334
if (rxOnly) {
299335
return `${defs.length ? 'var _s string\n' + defs.join('\n') + '\n' : ''}func lexFrom(src string, pos int, pendingNl bool, prevText, prevKind, bpText string, hasPrev, hasPrev2 bool, parenHead []bool, lastClose, lastBang bool, acc *[]Tok, limit int) (int, bool, string, string, string, bool, bool, []bool, bool, bool) {

‎src/target-rust.ts‎

Lines changed: 70 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,8 @@
1111
// parser allocates ~nothing per parse. Rule fns return `i32` (-1 = fail); sub-sequence
1212
// combinators take non-capturing `fn(&mut Parser) -> bool` pointers (the kids vec is now on
1313
// the Parser as `scratch`, so the second param the old owned-tree version threaded is gone).
14-
import type { ParserIR, RdRule, PrattRule, Step, Bracket, CharRange, LexTok, TplCfg, NewlineCfg, FirstSig } from './emit-portable.ts';
15-
import { portableIR } from './emit-portable.ts';
14+
import type { ParserIR, RdRule, PrattRule, Step, Bracket, CharRange, LexTok, TplCfg, NewlineCfg, FirstSig, LexFirstBytes } from './emit-portable.ts';
15+
import { portableIR, buildLexDispatchPlan, lexTokFirstBytes, punctFirstBytes } from './emit-portable.ts';
1616
import type { Target } from './emit.ts';
1717
import type { TokenPattern, CstGrammar } from './types.ts';
1818

@@ -86,6 +86,67 @@ function scanTok(t: LexTok, defs: string[], stateful: boolean, rxTok?: string, t
8686
return ` if ${gate}true { let e = ${m}(src, pos as i64); if e > pos as i64 { let e = e as usize; ${push('e')}pos = e; continue; } }`;
8787
}
8888

89+
function rustByteLit(b: number): string {
90+
if ((b >= 97 && b <= 122) || (b >= 65 && b <= 90) || (b >= 48 && b <= 57)) return `b'${String.fromCharCode(b)}'`;
91+
if ([33, 35, 36, 37, 38, 42, 43, 44, 45, 46, 47, 58, 59, 61, 63, 64, 94, 95].includes(b)) return `b'${String.fromCharCode(b)}'`;
92+
return String(b);
93+
}
94+
95+
function rustMatchLabels(bytes: number[]): string {
96+
const sorted = [...bytes].sort((a, b) => a - b);
97+
const parts: string[] = [];
98+
for (let i = 0; i < sorted.length; i++) {
99+
const lo = sorted[i];
100+
let hi = lo;
101+
while (i + 1 < sorted.length && sorted[i + 1] === hi + 1) hi = sorted[++i];
102+
if (lo === hi) {
103+
parts.push(rustByteLit(lo));
104+
} else {
105+
const ls = rustByteLit(lo), hs = rustByteLit(hi);
106+
parts.push(ls.startsWith('b') && hs.startsWith('b') ? `${ls}..=${hs}` : `${lo}..=${hi}`);
107+
}
108+
}
109+
return parts.join(' | ');
110+
}
111+
112+
function buildLexCandidates(
113+
ir: ParserIR, defs: string[], stateful: boolean, rxTok: string | undefined, tplTok: string | undefined,
114+
punctLine: (p: string) => string,
115+
): { codes: string[]; firsts: (LexFirstBytes | null)[] } {
116+
const codes: string[] = [];
117+
const firsts: (LexFirstBytes | null)[] = [];
118+
for (const t of ir.tokens) {
119+
const code = scanTok(t, defs, stateful, rxTok, tplTok);
120+
if (!code) continue;
121+
codes.push(code);
122+
firsts.push(lexTokFirstBytes(t));
123+
}
124+
for (const p of ir.puncts) {
125+
codes.push(punctLine(p));
126+
firsts.push(punctFirstBytes(p));
127+
}
128+
return { codes, firsts };
129+
}
130+
131+
/** Shared first-byte dispatch for all lexFrom variants in this target. */
132+
function renderLexByteDispatchRust(codes: string[], firsts: (LexFirstBytes | null)[], indent: string): string {
133+
const { arms, fallbackIndices } = buildLexDispatchPlan(firsts);
134+
const fallback = fallbackIndices.map((i) => codes[i]).join('\n');
135+
let matchArms = '';
136+
for (const arm of arms) {
137+
matchArms += `${indent} ${rustMatchLabels(arm.bytes)} => {\n`;
138+
matchArms += arm.indices.map((i) => codes[i]).join('\n') + '\n';
139+
matchArms += `${indent} }\n`;
140+
}
141+
return `${indent} if c >= 128 {
142+
${fallback}
143+
${indent} } else {
144+
${indent} match b[pos] {
145+
${matchArms}${indent} _ => {}
146+
${indent} }
147+
${indent} }`;
148+
}
149+
89150
function newlinePartsRs(nl: NewlineCfg): { consts: string; fields: string; init: string; boundary: string; ws: string; hooks: string; boundaryFrom: string; wsFrom: string; hooksFrom: string } {
90151
const commentSkip = nl.comment
91152
? ` if src[p..].starts_with(${J(nl.comment)}) { let mut e = p; while e < n && b[e] != 10 { e += 1; } pos = e; continue; }\n`
@@ -183,9 +244,10 @@ function lexer(ir: ParserIR): string {
183244
const rxOrTpl = !!(rx || tpl) && !rxOnly && !tplOnly && !rxTpl;
184245
const stateful = !!(rx || tpl);
185246
const newlineOnly = !!(nl && !rx && !tpl);
186-
const toks = ir.tokens.map((t) => scanTok(t, defs, stateful, rx?.regexToken, tpl?.token)).join('\n');
187-
const puncts = ir.puncts.map((p) =>
188-
` if src[pos..].starts_with(${J(p)}) { ${stateful ? `st.emit("", &src[pos..pos + ${p.length}], pos, pos + ${p.length});` : `toks.push(Tok { kind: "", text: &src[pos..pos + ${p.length}], off: pos, end: pos + ${p.length}, nl: pending_nl }); pending_nl = false;`} pos += ${p.length}; continue; }`).join('\n');
247+
const punctLine = (p: string) =>
248+
` if src[pos..].starts_with(${J(p)}) { ${stateful ? `st.emit("", &src[pos..pos + ${p.length}], pos, pos + ${p.length});` : `toks.push(Tok { kind: "", text: &src[pos..pos + ${p.length}], off: pos, end: pos + ${p.length}, nl: pending_nl }); pending_nl = false;`} pos += ${p.length}; continue; }`;
249+
const { codes: lexCodes, firsts: lexFirsts } = buildLexCandidates(ir, defs, stateful, rx?.regexToken, tpl?.token, punctLine);
250+
const cascade = renderLexByteDispatchRust(lexCodes, lexFirsts, ' ');
189251
const rsArr = (a: string[]) => `&[${a.map(J).join(', ')}]`;
190252
// Struct fields / emit hooks / init are assembled per-feature so a grammar can have regex,
191253
// templates, or both share one LexState.
@@ -308,13 +370,11 @@ impl<'a, 'b> RxTplScan<'a, 'b> {
308370
if c == 10 || c == 13 { ${nlVar} = true; pos += 1; continue; } // LF/CR
309371
`;
310372
const loopBody = `${nlBoundary} let c = b[pos] as u32;
311-
${nlWs}${tplDispatch}${toks}
312-
${puncts}
373+
${nlWs}${tplDispatch}${cascade}
313374
panic!("lex error at {}", pos);`;
314375
if (rxOnly) {
315376
const rxLoopBody = `${nlBoundary} let c = b[pos] as u32;
316-
${nlWs}${toks}
317-
${puncts}
377+
${nlWs}${cascade}
318378
panic!("lex error at {}", pos);`;
319379
return `${defs.length ? defs.join('\n') + '\n' : ''}${rxConsts}${tplFn}${rxScanImpl}fn lex_from<'a>(src: &'a str, mut pos: usize, mut pending_nl: bool, mut prev_text: &'a str, mut prev_kind: &'static str, mut bp_text: &'a str, mut has_prev: bool, mut has_prev2: bool, mut paren_head: Vec<bool>, mut last_close: bool, mut last_bang: bool, acc: &mut Vec<Tok<'a>>, limit: usize) -> (usize, bool, &'a str, &'static str, &'a str, bool, bool, Vec<bool>, bool, bool) {
320380
let b = src.as_bytes();
@@ -383,8 +443,7 @@ ${loopBody}
383443
.replace(/toks\.push\(Tok \{ kind: ([^,]+), text: ([^,]+), off: pos, end: ([^,]+), nl: pending_nl \}\); pending_nl = false; ?/g, 'st.push_tok($1, $2, pos, $3); ')
384444
.replace(/pending_nl/g, 'st.pending_nl');
385445
const nlLoopBody = `${nlRs!.boundaryFrom} let c = b[pos] as u32;
386-
${nlRs!.wsFrom}${rustNlScan(toks)}
387-
${rustNlScan(puncts)}
446+
${nlRs!.wsFrom}${rustNlScan(cascade)}
388447
panic!("lex error at {}", pos);`;
389448
return `${defs.length ? defs.join('\n') + '\n' : ''}${rxConsts}${tplFn}struct NlScan<'a, 'b> { acc: &'a mut Vec<Tok<'b>>, pending_nl: bool, line_start: bool, emitted_content: bool, flow_depth: i64 }
390449
impl<'a, 'b> NlScan<'a, 'b> {

0 commit comments

Comments
 (0)