Skip to content

Commit 4efbe84

Browse files
johnsoncodehkcursoragent
johnsoncodehk
and
cursoragent
committed
Jump-table Pratt LED/nud dispatch; relax FIRST K for large RD rules.
Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent e6bd91c commit 4efbe84

4 files changed

Lines changed: 213 additions & 65 deletions

File tree

‎src/emit-portable.ts‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,46 @@ export type Lit = { value: string; ttype: '$keyword' | '$punct' };
5959
// dispatch when every branch in an alt-list is non-null AND the signatures are pairwise disjoint.
6060
export type FirstSig = { lits: string[]; toks: string[] } | null;
6161
// Cap on FIRST-set size for partial pre-filter guards on non-predictive rd/inline alts.
62-
// Larger sets skip the guard (|| chains can exceed a failed attempt). Tunable; see P4b.
62+
// Larger sets skip the guard (|| chains can exceed a failed attempt). Tunable; see P4b/P5c.
6363
export const FIRST_GUARD_K = 4;
64+
/** toks-only sigs may use a larger K (kid compares cheaper than long lid || chains). 0 = off. */
65+
export const FIRST_GUARD_K_TOK = 0;
66+
/** When a non-predictive rule has ≥ this many alts, allow FIRST_GUARD_K_BIG. 0 = off. */
67+
export const FIRST_GUARD_BIG_NALTS = 10;
68+
export const FIRST_GUARD_K_BIG = 8;
69+
70+
/** Whether a FirstSig is small enough to pre-filter before a backtracking attempt. */
71+
export function isFirstGuardable(f: FirstSig, nAlts?: number): f is NonNullable<FirstSig> {
72+
if (f === null) return false;
73+
const n = f.lits.length + f.toks.length;
74+
if (n === 0) return false;
75+
if (n <= FIRST_GUARD_K) return true;
76+
if (FIRST_GUARD_K_TOK > 0 && f.lits.length === 0 && f.toks.length <= FIRST_GUARD_K_TOK) return true;
77+
if (
78+
FIRST_GUARD_BIG_NALTS > 0 &&
79+
nAlts != null &&
80+
nAlts >= FIRST_GUARD_BIG_NALTS &&
81+
n <= FIRST_GUARD_K_BIG
82+
) return true;
83+
return false;
84+
}
85+
86+
/** Group items by key, preserving first-seen key order and within-group original order. */
87+
export type Grouped<T> = { key: string | number; members: Array<{ item: T; index: number }> };
88+
export function groupByPreserveOrder<T>(
89+
items: readonly T[],
90+
keyOf: (item: T, index: number) => string | number,
91+
): Grouped<T>[] {
92+
const map = new Map<string | number, Array<{ item: T; index: number }>>();
93+
const order: Array<string | number> = [];
94+
for (let i = 0; i < items.length; i++) {
95+
const key = keyOf(items[i]!, i);
96+
let arr = map.get(key);
97+
if (!arr) { arr = []; map.set(key, arr); order.push(key); }
98+
arr.push({ item: items[i]!, index: i });
99+
}
100+
return order.map((key) => ({ key, members: map.get(key)! }));
101+
}
64102
export type Step =
65103
| { t: 'lit'; value: string; ttype: '$keyword' | '$punct' } // match a literal by text
66104
| { t: 'tok'; name: string } // match a token kind

‎src/target-go.ts‎

Lines changed: 59 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@
99
// stack. A node is an int32 index, never a heap pointer. Backtracking truncates the three
1010
// slices to saved lengths; the slices keep their capacity across parses (reset to len 0), so a
1111
// warmed parser allocates ~nothing per parse.
12-
import { FIRST_GUARD_K, type ParserIR, type RdRule, type PrattRule, type Step, type Bracket, type CharRange, type LexTok, type TplCfg, type NewlineCfg, type FirstSig, type LexFirstBytes, type LexIdPlan, type ArenaIdPlan } from './emit-portable.ts';
13-
import { portableIR, buildLexDispatchPlan, lexTokFirstBytes, punctFirstBytes, buildLexIdPlan, buildArenaIdPlan, lidOf, kidOf, ttIdOf, ruleIdOf, TT_SKIP_PUNCT, rangesHaveNonAscii } from './emit-portable.ts';
12+
import { type ParserIR, type RdRule, type PrattRule, type Step, type Bracket, type CharRange, type LexTok, type TplCfg, type NewlineCfg, type FirstSig, type LexFirstBytes, type LexIdPlan, type ArenaIdPlan } from './emit-portable.ts';
13+
import { portableIR, buildLexDispatchPlan, lexTokFirstBytes, punctFirstBytes, buildLexIdPlan, buildArenaIdPlan, lidOf, kidOf, ttIdOf, ruleIdOf, TT_SKIP_PUNCT, rangesHaveNonAscii, isFirstGuardable, groupByPreserveOrder } from './emit-portable.ts';
1414
import type { Target } from './emit.ts';
1515
import type { TokenPattern, CstGrammar } from './types.ts';
1616

@@ -37,8 +37,8 @@ const firstCond = (f: FirstSig, t: string, ids: LexIdPlan) => f
3737
? `(${f.lits.map((l) => `${t}.Lid == ${lidOf(ids, l)}`).join(' || ') || 'false'} || ${f.toks.map((k) => `${t}.Kid == ${kidOf(ids, k)}`).join(' || ') || 'false'})`
3838
: 'false';
3939
/** Non-null FirstSig small enough to pre-filter before a backtracking attempt. */
40-
const isGuardable = (f: FirstSig): f is NonNullable<FirstSig> =>
41-
f !== null && f.lits.length + f.toks.length > 0 && f.lits.length + f.toks.length <= FIRST_GUARD_K;
40+
const isGuardable = (f: FirstSig, nAlts?: number): f is NonNullable<FirstSig> =>
41+
isFirstGuardable(f, nAlts);
4242

4343
/** Emit kid/lid lookup tables into generated lexer source. */
4444
function renderIdTablesGo(ids: LexIdPlan): string {
@@ -504,12 +504,13 @@ function stepCond(s: Step, ids: LexIdPlan, ar: ArenaIdPlan): string {
504504
case 'alt': {
505505
if (s.predictive) return `func() bool { ${predAltBody(s.branches, ids, s.firsts, ar)} }()`;
506506
const firsts = s.firsts ?? [];
507-
const needPeek = s.branches.some((_, i) => isGuardable(firsts[i] ?? null));
507+
const nAlts = s.branches.length;
508+
const needPeek = s.branches.some((_, i) => isGuardable(firsts[i] ?? null, nAlts));
508509
const peekInit = needPeek ? `_ft := peek(); ` : '';
509510
const tries = s.branches.map((br, i) => {
510511
const body = `{ save := pos; sb := len(scratch); nb := len(nodes); kb := len(kids); if ${br.length ? br.map((x) => stepCond(x, ids, ar)).join(' && ') : 'true'} { return true }; pos = save; scratch = scratch[:sb]; nodes = nodes[:nb]; kids = kids[:kb] }`;
511512
const f = firsts[i] ?? null;
512-
if (!isGuardable(f)) return body;
513+
if (!isGuardable(f, nAlts)) return body;
513514
return `if _ft != nil && ${firstCond(f, '_ft', ids)} ${body}`;
514515
}).join('; ');
515516
return `func() bool { ${peekInit}${tries}; return false }()`;
@@ -583,7 +584,7 @@ ${r.alts.map(arm).join(' ')}
583584
const alt = (steps: Step[], i: number) => {
584585
const cond = steps.map((x) => stepCond(x, ids, ar)).join(' && ');
585586
const restore = `pos = save; scratch = scratch[:sb]; nodes = nodes[:nb]; kids = kids[:kb]`;
586-
if (!isGuardable(r.altFirst[i])) {
587+
if (!isGuardable(r.altFirst[i], r.alts.length)) {
587588
return `\tif ${cond} { return finish(${ruleIdOf(ar, r.cstName)}, sb, offAt(save), save) }
588589
\t${restore}`;
589590
}
@@ -592,7 +593,7 @@ ${r.alts.map(arm).join(' ')}
592593
\t\t${restore}
593594
\t}`;
594595
};
595-
const needPeek = r.alts.some((_, i) => isGuardable(r.altFirst[i]));
596+
const needPeek = r.alts.some((_, i) => isGuardable(r.altFirst[i], r.alts.length));
596597
return `func parse${r.name}() int32 {
597598
\tsave := pos; sb := len(scratch); nb := len(nodes); kb := len(kids)
598599
${needPeek ? '\t_ft := peek()\n' : ''}${r.alts.map(alt).join('\n')}
@@ -700,28 +701,63 @@ function prattRule(r: PrattRule, tpl: TplCfg | null, ids: LexIdPlan, ar: ArenaId
700701
const bin = r.binary.map((b) => `${lidOf(ids, b.op)}: {${b.lbp}, ${b.rbp}}`).join(', ');
701702
const pre = r.prefix.map((p) => `${lidOf(ids, p.op)}: ${p.rbp}`).join(', ');
702703
const atoms = r.nudToks.map((k) => `${kidOf(ids, k)}: true`).join(', ');
703-
const bracketNud = (b: Bracket) => `\tif t.Lid == ${lidOf(ids, b.first)} {
704+
const bracketNudBody = (b: Bracket) => `{
704705
\t\tsave := pos; sb := len(scratch); nb := len(nodes); kb := len(kids)
705706
\t\tif ${b.steps.map((x) => stepCond(x, ids, ar)).join(' && ')} { return finish(${ruleIdOf(ar, r.cstName)}, sb, t.Off, save) }
706707
\t\tpos = save; scratch = scratch[:sb]; nodes = nodes[:nb]; kids = kids[:kb]
707708
\t}`;
708-
const ledArm = (b: Bracket, accessTail: boolean, lbp: number | null, sameLine: boolean, nll: string[] | null) => `\t\tif ${accessTail ? '!tailClosed && ' : ''}${lbp !== null ? `${lbp} > minBp && ` : ''}${sameLine ? '!t.Nl && ' : ''}${nll ? `!_inW([]string{${nll.map(J).join(', ')}}, headLeafText(left)) && ` : ''}!_suppressCur[${lidOf(ids, b.first)}] && t.Lid == ${lidOf(ids, b.first)} {
709+
const bracketNudSwitch = (() => {
710+
if (r.nudBrackets.length === 0) return '';
711+
const groups = groupByPreserveOrder(r.nudBrackets, (b) => lidOf(ids, b.first));
712+
return `\tswitch t.Lid {
713+
${groups.map((g) => `\tcase ${g.key}:
714+
${g.members.map(({ item: b }) => `\t\t${bracketNudBody(b)}`).join('\n')}`).join('\n')}
715+
\t}`;
716+
})();
717+
const ledGuard = (accessTail: boolean, lbp: number | null, sameLine: boolean, nll: string[] | null, lid: number) => {
718+
const parts: string[] = [];
719+
if (accessTail) parts.push('!tailClosed');
720+
if (lbp !== null) parts.push(`${lbp} > minBp`);
721+
if (sameLine) parts.push('!t.Nl');
722+
if (nll) parts.push(`!_inW([]string{${nll.map(J).join(', ')}}, headLeafText(left))`);
723+
parts.push(`!_suppressCur[${lid}]`);
724+
return parts.join(' && ');
725+
};
726+
const ledBody = (b: Bracket) => `{
709727
\t\t\tledSave := pos; sb := len(scratch); nb := len(nodes); kb := len(kids)
710728
\t\t\tscratch = append(scratch, left)
711-
\t\t\tif ${b.steps.map((x) => stepCond(x, ids, ar)).join(' && ')} { left = finish(${ruleIdOf(ar, r.cstName)}, sb, int(nodes[left].Offset), int(nodes[left].TokStart)); continue }
712-
\t\t\tpos = ledSave; scratch = scratch[:sb]; nodes = nodes[:nb]; kids = kids[:kb]; break
729+
\t\t\tif ${b.steps.map((x) => stepCond(x, ids, ar)).join(' && ')} { left = finish(${ruleIdOf(ar, r.cstName)}, sb, int(nodes[left].Offset), int(nodes[left].TokStart)); continue LedLoop }
730+
\t\t\tpos = ledSave; scratch = scratch[:sb]; nodes = nodes[:nb]; kids = kids[:kb]; break LedLoop
731+
\t\t}`;
732+
const ledSwitch = (() => {
733+
if (r.leds.length === 0) return '';
734+
const groups = groupByPreserveOrder(r.leds, (b) => lidOf(ids, b.first));
735+
return `\t\tswitch t.Lid {
736+
${groups.map((g) => {
737+
const lid = g.key as number;
738+
const arms = g.members.map(({ item: b, index: i }) =>
739+
`\t\t\tif ${ledGuard(r.ledAccessTail[i]!, r.ledLbp[i]!, r.ledSameLine[i]!, r.ledNotLeftLeaf[i]!, lid)} ${ledBody(b)}`);
740+
return `\t\tcase ${lid}:\n${arms.join('\n')}`;
741+
}).join('\n')}
713742
\t\t}`;
714-
const postfixArm = (tok: string) => {
715-
const tplPart = tpl && tok === tpl.token ? `
743+
})();
744+
const postfixTokSwitch = (() => {
745+
if (r.postfixToks.length === 0) return '';
746+
const groups = groupByPreserveOrder(r.postfixToks, (tok) => kidOf(ids, tok));
747+
const hasTpl = !!(tpl && r.postfixToks.includes(tpl.token));
748+
const tplPart = hasTpl ? `
716749
\t\tif !tailClosed && t.Kind == "$templateHead" {
717750
\t\t\tnode := matchTemplate()
718-
\t\t\tif node >= 0 { sb := len(scratch); scratch = append(scratch, left, node); left = finish(${ruleIdOf(ar, r.cstName)}, sb, int(nodes[left].Offset), int(nodes[left].TokStart)); continue }
751+
\t\t\tif node >= 0 { sb := len(scratch); scratch = append(scratch, left, node); left = finish(${ruleIdOf(ar, r.cstName)}, sb, int(nodes[left].Offset), int(nodes[left].TokStart)); continue LedLoop }
719752
\t\t}` : '';
720-
return `\t\tif !tailClosed && t.Kid == ${kidOf(ids, tok)} {
721-
\t\t\tsb := len(scratch); scratch = append(scratch, left); pushLeaf(uint8(t.Kid), uint32(pos)); pos++
722-
\t\t\tleft = finish(${ruleIdOf(ar, r.cstName)}, sb, int(nodes[left].Offset), int(nodes[left].TokStart)); continue
753+
return `\t\tswitch t.Kid {
754+
${groups.map((g) => `\t\tcase ${g.key}:
755+
\t\t\tif !tailClosed {
756+
\t\t\t\tsb := len(scratch); scratch = append(scratch, left); pushLeaf(uint8(t.Kid), uint32(pos)); pos++
757+
\t\t\t\tleft = finish(${ruleIdOf(ar, r.cstName)}, sb, int(nodes[left].Offset), int(nodes[left].TokStart)); continue LedLoop
758+
\t\t\t}`).join('\n')}
723759
\t\t}${tplPart}`;
724-
};
760+
})();
725761
const post = r.postfix.map((p) => `${lidOf(ids, p.op)}: ${p.lbp}`).join(', ');
726762
return `var ${r.name}BIN = map[uint16]bp{${bin}}
727763
var ${r.name}PRE = map[uint16]int{${pre}}
@@ -740,11 +776,11 @@ func ${r.name}bp(minBp int) int32 {
740776
\tif left < 0 { return -1 }
741777
\tif _capped { return left }
742778
\ttailClosed := false
743-
\tfor {
779+
${(r.leds.length > 0 || r.postfixToks.length > 0) ? 'LedLoop:\n' : ''}\tfor {
744780
\t\tt := peek()
745781
\t\tif t == nil { break }
746-
${r.leds.map((b, i) => ledArm(b, r.ledAccessTail[i], r.ledLbp[i], r.ledSameLine[i], r.ledNotLeftLeaf[i])).join('\n')}
747-
${r.postfixToks.map(postfixArm).join('\n')}
782+
${ledSwitch}
783+
${postfixTokSwitch}
748784
\t\tif post, ok := ${r.name}POST[t.Lid]; ok && !tailClosed && post > minBp {
749785
\t\t\tsb := len(scratch); scratch = append(scratch, left); pushLeaf(${ttIdOf(ar, '$operator')}, uint32(pos)); pos++; tailClosed = true
750786
\t\t\tleft = finish(${ruleIdOf(ar, r.cstName)}, sb, int(nodes[left].Offset), int(nodes[left].TokStart)); continue
@@ -771,7 +807,7 @@ ${tplNud}\tif ${r.name}ATOM[t.Kid] {
771807
\t\tsb := len(scratch); ts := pos; pushLeaf(uint8(t.Kid), uint32(pos)); pos++
772808
\t\treturn finish(${ruleIdOf(ar, r.cstName)}, sb, t.Off, ts)
773809
\t}
774-
${r.nudBrackets.map(bracketNud).join('\n')}
810+
${bracketNudSwitch}
775811
\tif pbp, ok := ${r.name}PRE[t.Lid]; ok {
776812
\t\tsave := pos; sb := len(scratch); nb := len(nodes); kb := len(kids)
777813
\t\tpushLeaf(${ttIdOf(ar, '$operator')}, uint32(pos)); pos++

0 commit comments

Comments
 (0)