Skip to content

Commit e6bd91c

Browse files
johnsoncodehkcursoragent
johnsoncodehk
and
cursoragent
committed
Add FIRST pre-filter guards on non-predictive RD/inline alts (K=4).
Skip doomed backtracking attempts when a non-null FIRST set is small enough, cutting Decl fail attempts ~86% vs D1 without changing accept/reject. Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent 9ea9dda commit e6bd91c

4 files changed

Lines changed: 90 additions & 20 deletions

File tree

‎src/emit-portable.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,9 @@ export type Lit = { value: string; ttype: '$keyword' | '$punct' };
5858
// backtracking for it. Computed conservatively in buildIR; targets only render a predictive
5959
// dispatch when every branch in an alt-list is non-null AND the signatures are pairwise disjoint.
6060
export type FirstSig = { lits: string[]; toks: string[] } | null;
61+
// Cap on FIRST-set size for partial pre-filter guards on non-predictive rd/inline alts.
62+
// Larger sets skip the guard (|| chains can exceed a failed attempt). Tunable; see P4b.
63+
export const FIRST_GUARD_K = 4;
6164
export type Step =
6265
| { t: 'lit'; value: string; ttype: '$keyword' | '$punct' } // match a literal by text
6366
| { t: 'tok'; name: string } // match a token kind

‎src/target-go.ts‎

Lines changed: 31 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
// stack. A node is an int32 index, never a heap pointer. Backtracking truncates the three
1010
// slices to saved lengths; the slices keep their capacity across parses (reset to len 0), so a
1111
// warmed parser allocates ~nothing per parse.
12-
import type { ParserIR, RdRule, PrattRule, Step, Bracket, CharRange, LexTok, TplCfg, NewlineCfg, FirstSig, LexFirstBytes, LexIdPlan, ArenaIdPlan } from './emit-portable.ts';
12+
import { FIRST_GUARD_K, type ParserIR, type RdRule, type PrattRule, type Step, type Bracket, type CharRange, type LexTok, type TplCfg, type NewlineCfg, type FirstSig, type LexFirstBytes, type LexIdPlan, type ArenaIdPlan } from './emit-portable.ts';
1313
import { portableIR, buildLexDispatchPlan, lexTokFirstBytes, punctFirstBytes, buildLexIdPlan, buildArenaIdPlan, lidOf, kidOf, ttIdOf, ruleIdOf, TT_SKIP_PUNCT, rangesHaveNonAscii } from './emit-portable.ts';
1414
import type { Target } from './emit.ts';
1515
import type { TokenPattern, CstGrammar } from './types.ts';
@@ -36,6 +36,9 @@ function emitAsciiBoolTableGo(name: string, rs: CharRange[]): string {
3636
const firstCond = (f: FirstSig, t: string, ids: LexIdPlan) => f
3737
? `(${f.lits.map((l) => `${t}.Lid == ${lidOf(ids, l)}`).join(' || ') || 'false'} || ${f.toks.map((k) => `${t}.Kid == ${kidOf(ids, k)}`).join(' || ') || 'false'})`
3838
: 'false';
39+
/** Non-null FirstSig small enough to pre-filter before a backtracking attempt. */
40+
const isGuardable = (f: FirstSig): f is NonNullable<FirstSig> =>
41+
f !== null && f.lits.length + f.toks.length > 0 && f.lits.length + f.toks.length <= FIRST_GUARD_K;
3942

4043
/** Emit kid/lid lookup tables into generated lexer source. */
4144
function renderIdTablesGo(ids: LexIdPlan): string {
@@ -498,7 +501,19 @@ function stepCond(s: Step, ids: LexIdPlan, ar: ArenaIdPlan): string {
498501
case 'opt': return `opt(func() bool { return ${s.steps.map((x) => stepCond(x, ids, ar)).join(' && ')} })`;
499502
case 'sep': return `sepBy(func() bool { return ${stepCond(s.elem, ids, ar)} }, ${lidOf(ids, s.delim)})`;
500503
case 'altlit': return `altLit([]struct{ Lid uint16; TtId uint8 }{${s.opts.map((o) => `{${lidOf(ids, o.value)}, ${ttIdOf(ar, o.ttype)}}`).join(', ')}})`;
501-
case 'alt': return s.predictive ? `func() bool { ${predAltBody(s.branches, ids, s.firsts, ar)} }()` : `func() bool { ${s.branches.map((br) => `{ save := pos; sb := len(scratch); nb := len(nodes); kb := len(kids); if ${br.length ? br.map((x) => stepCond(x, ids, ar)).join(' && ') : 'true'} { return true }; pos = save; scratch = scratch[:sb]; nodes = nodes[:nb]; kids = kids[:kb] }`).join('; ')}; return false }()`;
504+
case 'alt': {
505+
if (s.predictive) return `func() bool { ${predAltBody(s.branches, ids, s.firsts, ar)} }()`;
506+
const firsts = s.firsts ?? [];
507+
const needPeek = s.branches.some((_, i) => isGuardable(firsts[i] ?? null));
508+
const peekInit = needPeek ? `_ft := peek(); ` : '';
509+
const tries = s.branches.map((br, i) => {
510+
const body = `{ save := pos; sb := len(scratch); nb := len(nodes); kb := len(kids); if ${br.length ? br.map((x) => stepCond(x, ids, ar)).join(' && ') : 'true'} { return true }; pos = save; scratch = scratch[:sb]; nodes = nodes[:nb]; kids = kids[:kb] }`;
511+
const f = firsts[i] ?? null;
512+
if (!isGuardable(f)) return body;
513+
return `if _ft != nil && ${firstCond(f, '_ft', ids)} ${body}`;
514+
}).join('; ');
515+
return `func() bool { ${peekInit}${tries}; return false }()`;
516+
}
502517
case 'not': return `func() bool { save := pos; sb := len(scratch); nb := len(nodes); kb := len(kids); m := ${s.steps.length ? s.steps.map((x) => stepCond(x, ids, ar)).join(' && ') : 'true'}; pos = save; scratch = scratch[:sb]; nodes = nodes[:nb]; kids = kids[:kb]; return !m }()`;
503518
case 'seq': return `(${s.steps.length ? s.steps.map((x) => stepCond(x, ids, ar)).join(' && ') : 'true'})`;
504519
case 'sameLine': return `func() bool { t := peek(); return t != nil && !t.Nl }()`;
@@ -565,12 +580,22 @@ ${r.alts.map(arm).join(' ')}
565580
\treturn -1
566581
}`;
567582
}
568-
const alt = (steps: Step[]) =>
569-
`\tif ${steps.map((x) => stepCond(x, ids, ar)).join(' && ')} { return finish(${ruleIdOf(ar, r.cstName)}, sb, offAt(save), save) }
570-
\tpos = save; scratch = scratch[:sb]; nodes = nodes[:nb]; kids = kids[:kb]`;
583+
const alt = (steps: Step[], i: number) => {
584+
const cond = steps.map((x) => stepCond(x, ids, ar)).join(' && ');
585+
const restore = `pos = save; scratch = scratch[:sb]; nodes = nodes[:nb]; kids = kids[:kb]`;
586+
if (!isGuardable(r.altFirst[i])) {
587+
return `\tif ${cond} { return finish(${ruleIdOf(ar, r.cstName)}, sb, offAt(save), save) }
588+
\t${restore}`;
589+
}
590+
return `\tif _ft != nil && ${firstCond(r.altFirst[i], '_ft', ids)} {
591+
\t\tif ${cond} { return finish(${ruleIdOf(ar, r.cstName)}, sb, offAt(save), save) }
592+
\t\t${restore}
593+
\t}`;
594+
};
595+
const needPeek = r.alts.some((_, i) => isGuardable(r.altFirst[i]));
571596
return `func parse${r.name}() int32 {
572597
\tsave := pos; sb := len(scratch); nb := len(nodes); kb := len(kids)
573-
${r.alts.map(alt).join('\n')}
598+
${needPeek ? '\t_ft := peek()\n' : ''}${r.alts.map(alt).join('\n')}
574599
\treturn -1
575600
}`;
576601
}

‎src/target-rust.ts‎

Lines changed: 32 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
// parser allocates ~nothing per parse. Rule fns return `i32` (-1 = fail); sub-sequence
1212
// combinators take non-capturing `fn(&mut Parser) -> bool` pointers (the kids vec is now on
1313
// the Parser as `scratch`, so the second param the old owned-tree version threaded is gone).
14-
import type { ParserIR, RdRule, PrattRule, Step, Bracket, CharRange, LexTok, TplCfg, NewlineCfg, FirstSig, LexFirstBytes, LexIdPlan, ArenaIdPlan } from './emit-portable.ts';
14+
import { FIRST_GUARD_K, type ParserIR, type RdRule, type PrattRule, type Step, type Bracket, type CharRange, type LexTok, type TplCfg, type NewlineCfg, type FirstSig, type LexFirstBytes, type LexIdPlan, type ArenaIdPlan } from './emit-portable.ts';
1515
import { portableIR, buildLexDispatchPlan, lexTokFirstBytes, punctFirstBytes, buildLexIdPlan, buildArenaIdPlan, lidOf, kidOf, ttIdOf, ruleIdOf, TT_SKIP_PUNCT, rangesHaveNonAscii } from './emit-portable.ts';
1616
import type { Target } from './emit.ts';
1717
import type { TokenPattern, CstGrammar } from './types.ts';
@@ -39,6 +39,9 @@ function emitAsciiBoolTableRs(name: string, rs: CharRange[]): string {
3939
const firstCond = (f: FirstSig, t: string, ids: LexIdPlan) => f
4040
? `(${f.lits.map((l) => `${t}.lid == ${lidOf(ids, l)}`).join(' || ') || 'false'} || ${f.toks.map((k) => `${t}.kid == ${kidOf(ids, k)}`).join(' || ') || 'false'})`
4141
: 'false';
42+
/** Non-null FirstSig small enough to pre-filter before a backtracking attempt. */
43+
const isGuardable = (f: FirstSig): f is NonNullable<FirstSig> =>
44+
f !== null && f.lits.length + f.toks.length > 0 && f.lits.length + f.toks.length <= FIRST_GUARD_K;
4245

4346
/** Emit kid/lid lookup tables into generated lexer source (length-bucketed lid_of match). */
4447
function renderIdTablesRust(ids: LexIdPlan): string {
@@ -575,7 +578,7 @@ function stepCond(s: Step, ids: LexIdPlan, ar: ArenaIdPlan): string {
575578
case 'opt': return `self.opt(|p| ${s.steps.map((x) => stepCondP(x, ids, ar)).join(' && ')})`;
576579
case 'sep': return `self.sep_by(|p| ${stepCondP(s.elem, ids, ar)}, ${lidOf(ids, s.delim)})`;
577580
case 'altlit': return `self.alt_lit(&[${s.opts.map((o) => `(${lidOf(ids, o.value)}, ${ttIdOf(ar, o.ttype)})`).join(', ')}])`;
578-
case 'alt': return s.predictive ? `(|p: &mut Parser<'a>| -> bool { ${predAltBody(s.branches, ids, ar, s.firsts)} })(self)` : `(|p: &mut Parser<'a>| -> bool { ${altBody(s.branches, ids, ar)} })(self)`;
581+
case 'alt': return s.predictive ? `(|p: &mut Parser<'a>| -> bool { ${predAltBody(s.branches, ids, ar, s.firsts)} })(self)` : `(|p: &mut Parser<'a>| -> bool { ${altBody(s.branches, ids, ar, s.firsts)} })(self)`;
579582
case 'not': return `(|p: &mut Parser<'a>| -> bool { ${notBody(s.steps, ids, ar)} })(self)`;
580583
case 'seq': return `(${s.steps.length ? s.steps.map((x) => stepCond(x, ids, ar)).join(' && ') : 'true'})`;
581584
case 'sameLine': return `matches!(self.peek(), Some(t) if !t.nl)`;
@@ -584,8 +587,18 @@ function stepCond(s: Step, ids: LexIdPlan, ar: ArenaIdPlan): string {
584587
}
585588
// A backtracking inline alternation rendered as an immediately-applied closure over p,
586589
// so it composes identically whether it sits at top level or already inside a closure.
587-
function altBody(branches: Step[][], ids: LexIdPlan, ar: ArenaIdPlan): string {
588-
return `${branches.map((br) => `{ let sp = p.pos; let sb = p.scratch.len(); let nb = p.nodes.len(); let kb = p.kids.len(); if ${br.length ? br.map((x) => stepCondP(x, ids, ar)).join(' && ') : 'true'} { return true; } p.pos = sp; p.scratch.truncate(sb); p.nodes.truncate(nb); p.kids.truncate(kb); }`).join(' ')} false`;
590+
// Non-null FirstSig branches get a FIRST pre-filter (skip without save/restore) when |sig|≤K.
591+
function altBody(branches: Step[][], ids: LexIdPlan, ar: ArenaIdPlan, firsts?: FirstSig[]): string {
592+
const fs = firsts ?? [];
593+
const needPeek = branches.some((_, i) => isGuardable(fs[i] ?? null));
594+
const peekInit = needPeek ? `let _ft = p.peek(); ` : '';
595+
const tries = branches.map((br, i) => {
596+
const body = `{ let sp = p.pos; let sb = p.scratch.len(); let nb = p.nodes.len(); let kb = p.kids.len(); if ${br.length ? br.map((x) => stepCondP(x, ids, ar)).join(' && ') : 'true'} { return true; } p.pos = sp; p.scratch.truncate(sb); p.nodes.truncate(nb); p.kids.truncate(kb); }`;
597+
const f = fs[i] ?? null;
598+
if (!isGuardable(f)) return body;
599+
return `if let Some(t) = _ft { if ${firstCond(f, 't', ids)} ${body} }`;
600+
}).join(' ');
601+
return `${peekInit}${tries} false`;
589602
}
590603
// Zero-width negative lookahead: try the steps, restore, succeed iff they did NOT all match.
591604
function notBody(steps: Step[], ids: LexIdPlan, ar: ArenaIdPlan): string {
@@ -602,7 +615,7 @@ function stepCondP(s: Step, ids: LexIdPlan, ar: ArenaIdPlan): string {
602615
case 'opt': return `p.opt(|p| ${s.steps.map((x) => stepCondP(x, ids, ar)).join(' && ')})`;
603616
case 'sep': return `p.sep_by(|p| ${stepCondP(s.elem, ids, ar)}, ${lidOf(ids, s.delim)})`;
604617
case 'altlit': return `p.alt_lit(&[${s.opts.map((o) => `(${lidOf(ids, o.value)}, ${ttIdOf(ar, o.ttype)})`).join(', ')}])`;
605-
case 'alt': return s.predictive ? `(|p: &mut Parser<'a>| -> bool { ${predAltBody(s.branches, ids, ar, s.firsts)} })(p)` : `(|p: &mut Parser<'a>| -> bool { ${altBody(s.branches, ids, ar)} })(p)`;
618+
case 'alt': return s.predictive ? `(|p: &mut Parser<'a>| -> bool { ${predAltBody(s.branches, ids, ar, s.firsts)} })(p)` : `(|p: &mut Parser<'a>| -> bool { ${altBody(s.branches, ids, ar, s.firsts)} })(p)`;
606619
case 'not': return `(|p: &mut Parser<'a>| -> bool { ${notBody(s.steps, ids, ar)} })(p)`;
607620
case 'seq': return `(${s.steps.length ? s.steps.map((x) => stepCondP(x, ids, ar)).join(' && ') : 'true'})`;
608621
case 'sameLine': return `matches!(p.peek(), Some(t) if !t.nl)`;
@@ -750,12 +763,22 @@ ${r.alts.map(arm).join('\n')}
750763
None
751764
}`;
752765
}
753-
const alt = (steps: Step[]) =>
754-
` if ${steps.map((x) => stepCond(x, ids, ar)).join(' && ')} { return Some(self.finish(${ruleIdOf(ar, r.cstName)}, sb, self.off_at(save), save)); }
755-
self.pos = save; self.scratch.truncate(sb); self.nodes.truncate(nb); self.kids.truncate(kb);`;
766+
const alt = (steps: Step[], i: number) => {
767+
const cond = steps.map((x) => stepCond(x, ids, ar)).join(' && ');
768+
const restore = `self.pos = save; self.scratch.truncate(sb); self.nodes.truncate(nb); self.kids.truncate(kb);`;
769+
if (!isGuardable(r.altFirst[i])) {
770+
return ` if ${cond} { return Some(self.finish(${ruleIdOf(ar, r.cstName)}, sb, self.off_at(save), save)); }
771+
${restore}`;
772+
}
773+
return ` if let Some(t) = _ft { if ${firstCond(r.altFirst[i], 't', ids)} {
774+
if ${cond} { return Some(self.finish(${ruleIdOf(ar, r.cstName)}, sb, self.off_at(save), save)); }
775+
${restore}
776+
} }`;
777+
};
778+
const needPeek = r.alts.some((_, i) => isGuardable(r.altFirst[i]));
756779
return ` fn parse_${r.name}(&mut self) -> Option<i32> {
757780
let save = self.pos; let sb = self.scratch.len(); let nb = self.nodes.len(); let kb = self.kids.len();
758-
${r.alts.map(alt).join('\n')}
781+
${needPeek ? ' let _ft = self.peek();\n' : ''}${r.alts.map(alt).join('\n')}
759782
None
760783
}`;
761784
}

‎src/target-ts.ts‎

Lines changed: 24 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
// index LEDs), and a CST→JSON printer over stdin. It is the reference rendering — its CST
55
// is checked byte-for-byte against the interpreter (createParser), so a divergence in the
66
// portable logic surfaces here before Go/Rust are compiled.
7-
import type { ParserIR, RdRule, PrattRule, Step, Bracket, CharRange, LexTok, TplCfg, NewlineCfg, FirstSig, LexFirstBytes, LexIdPlan } from './emit-portable.ts';
7+
import { FIRST_GUARD_K, type ParserIR, type RdRule, type PrattRule, type Step, type Bracket, type CharRange, type LexTok, type TplCfg, type NewlineCfg, type FirstSig, type LexFirstBytes, type LexIdPlan } from './emit-portable.ts';
88
import { portableIR, buildLexDispatchPlan, lexTokFirstBytes, punctFirstBytes, buildLexIdPlan, lidOf, kidOf, rangesHaveNonAscii } from './emit-portable.ts';
99
import type { Target } from './emit.ts';
1010
import type { CstGrammar } from './types.ts';
@@ -32,6 +32,9 @@ function emitAsciiBoolTableTS(name: string, rs: CharRange[]): string {
3232
const firstCond = (f: FirstSig, t: string, ids: LexIdPlan) => f
3333
? `(${f.lits.map((l) => `${t}.lid === ${lidOf(ids, l)}`).join(' || ') || 'false'} || ${f.toks.map((k) => `${t}.kid === ${kidOf(ids, k)}`).join(' || ') || 'false'})`
3434
: 'false';
35+
/** Non-null FirstSig small enough to pre-filter before a backtracking attempt. */
36+
const isGuardable = (f: FirstSig): f is NonNullable<FirstSig> =>
37+
f !== null && f.lits.length + f.toks.length > 0 && f.lits.length + f.toks.length <= FIRST_GUARD_K;
3538

3639
/** Emit kid/lid lookup tables into generated lexer source. */
3740
function renderIdTablesTS(ids: LexIdPlan): string {
@@ -454,7 +457,19 @@ function stepCond(s: Step, ids: LexIdPlan): string {
454457
case 'opt': return `opt(() => ${s.steps.map((x) => stepCond(x, ids)).join(' && ')}, kids)`;
455458
case 'sep': return `sepBy(() => ${stepCond(s.elem, ids)}, ${lidOf(ids, s.delim)}, kids)`;
456459
case 'altlit': return `altLit([${s.opts.map((o) => `[${lidOf(ids, o.value)}, ${J(o.ttype)}]`).join(', ')}], kids)`;
457-
case 'alt': return s.predictive ? `(() => { ${predAltBody(s.branches, ids, s.firsts)} })()` : `(() => { ${s.branches.map((br) => `{ const sp = pos; const bk = kids.length; if (${br.length ? br.map((x) => stepCond(x, ids)).join(' && ') : 'true'}) return true; pos = sp; kids.length = bk; }`).join(' ')} return false; })()`;
460+
case 'alt': {
461+
if (s.predictive) return `(() => { ${predAltBody(s.branches, ids, s.firsts)} })()`;
462+
const firsts = s.firsts ?? [];
463+
const needPeek = s.branches.some((_, i) => isGuardable(firsts[i] ?? null));
464+
const peekInit = needPeek ? `const _ft = peek(); ` : '';
465+
const tries = s.branches.map((br, i) => {
466+
const body = `{ const sp = pos; const bk = kids.length; if (${br.length ? br.map((x) => stepCond(x, ids)).join(' && ') : 'true'}) return true; pos = sp; kids.length = bk; }`;
467+
const f = firsts[i] ?? null;
468+
if (!isGuardable(f)) return body;
469+
return `if (_ft !== null && ${firstCond(f, '_ft', ids)}) ${body}`;
470+
}).join(' ');
471+
return `(() => { ${peekInit}${tries} return false; })()`;
472+
}
458473
case 'not': return `(() => { const sp = pos; const bk = kids.length; const m = ${s.steps.length ? s.steps.map((x) => stepCond(x, ids)).join(' && ') : 'true'}; pos = sp; kids.length = bk; return !m; })()`;
459474
case 'seq': return `(${s.steps.length ? s.steps.map((x) => stepCond(x, ids)).join(' && ') : 'true'})`;
460475
case 'sameLine': return `(() => { const t = peek(); return t !== null && !t.nl; })()`;
@@ -524,11 +539,15 @@ ${r.alts.map(arm).join(' ')}
524539
return null;
525540
}`;
526541
}
527-
const alt = (steps: Step[]) =>
528-
` { const kids: Cst[] = []; if (${steps.map((x) => stepCond(x, ids)).join(' && ')}) return branch(${J(r.cstName)}, kids, save); pos = save; }`;
542+
const alt = (steps: Step[], i: number) => {
543+
const body = `{ const kids: Cst[] = []; if (${steps.map((x) => stepCond(x, ids)).join(' && ')}) return branch(${J(r.cstName)}, kids, save); pos = save; }`;
544+
if (!isGuardable(r.altFirst[i])) return ` ${body}`;
545+
return ` if (_ft !== null && ${firstCond(r.altFirst[i], '_ft', ids)}) ${body}`;
546+
};
547+
const needPeek = r.alts.some((_, i) => isGuardable(r.altFirst[i]));
529548
return `function parse${r.name}(): Node | null {
530549
const save = pos;
531-
${r.alts.map(alt).join('\n')}
550+
${needPeek ? ' const _ft = peek();\n' : ''}${r.alts.map(alt).join('\n')}
532551
return null;
533552
}`;
534553
}

0 commit comments

Comments
 (0)