Skip to content

Commit 4761151

Browse files
johnsoncodehkjohnsoncodehk
authored
Arena CST: nodes as typed-array rows, leaves as token refs, visit-based traversal (#36)
1 parent 9c165f4 commit 4761151

6 files changed

Lines changed: 884 additions & 648 deletions

File tree

‎src/emit-parser.ts‎

Lines changed: 363 additions & 219 deletions
Large diffs are not rendered by default.

‎src/gen-cst-match.ts‎

Lines changed: 109 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,12 @@
11
// Generate per-rule, per-ARM destructurers for a grammar's CST — the VALUE-level
22
// sibling of gen-ast-types.ts. For every rule it emits
33
//
4-
// export type <Rule>Match = { arm: 'if', expr: ExprNode, … } | { arm: 'block', … } | …
5-
// export function match<Rule>(n: <Rule>Node, src: string): <Rule>Match
4+
// export type <Rule>Match = { arm: 'if', expr: NodeEntry<'Expr'>, … } | …
5+
// export function match<Rule>(t: TreeAccess, n: NodeEntry<'Rule'>, src: string): <Rule>Match
6+
//
7+
// ARENA-NATIVE: matchers read the emitted parser's arena through the TreeAccess
8+
// interface (the emitted module's `tree` export satisfies it) — a node is an id, a
9+
// leaf is a token-encoded entry; nothing is materialized to match.
610
//
711
// The matcher re-derives WHICH grammar alternative a node matched and binds its
812
// children to named fields — the discrimination the parser performed and the CST does
@@ -73,7 +77,15 @@ export function generateCstMatch(grammar: CstGrammar, importFrom: string): strin
7377
const tokenNames = new Set(grammar.tokens.map(t => t.name));
7478
const templateTokenNames = new Set(grammar.tokens.filter(t => t.template).map(t => t.name));
7579
const ruleNames = new Set(grammar.rules.map(r => r.name));
76-
const usedIfaces = new Set<string>(['CstChild', 'CstLeaf']);
80+
// Int worlds, replicated from emit-parser's symtab (same deterministic derivation
81+
// from the grammar; any drift is caught instantly by cst-match-totality):
82+
// token TYPE kinds — '' punct = 1, $template spans = 2/3/4, named tokens from 5
83+
// rule ids — declaration order; the synthetic '$template' node = rules.length
84+
const typeKind = new Map<string, number>([['', 1], ['$templateHead', 2], ['$templateMiddle', 3], ['$templateTail', 4]]);
85+
{ let next = 5; for (const t of grammar.tokens) if (!typeKind.has(t.name)) typeKind.set(t.name, next++); }
86+
const ruleId = new Map<string, number>(grammar.rules.map((r, i) => [r.name, i]));
87+
ruleId.set('$template', grammar.rules.length);
88+
7789

7890
// Pratt / leftRec classification (mirrors the engines' classifyAlts/classifyLeftRec:
7991
// a rule is op-bearing if any alt contains op/prefix/postfix markers; an alt whose
@@ -131,7 +143,7 @@ export function generateCstMatch(grammar: CstGrammar, importFrom: string): strin
131143
return { kind: 'node', rule: self, cap: { name: nm, field: nm, tsType: nodeType(self), card: 'one' } };
132144
}
133145
function opLeafCap(nm: string): Step {
134-
return { kind: 'tok', name: '$operator', template: false, cap: { name: nm, field: nm, tsType: 'CstLeaf', card: 'one' } };
146+
return { kind: 'tok', name: '$operator', template: false, cap: { name: nm, field: nm, tsType: 'LeafEntry', card: 'one' } };
135147
}
136148
function opPlan(name: string, steps: Step[], _self: string): ArmPlan {
137149
const captures = steps.map(s => (s as { cap: Capture }).cap);
@@ -149,9 +161,7 @@ export function generateCstMatch(grammar: CstGrammar, importFrom: string): strin
149161
}
150162

151163
function nodeType(rule: string): string {
152-
const t = `${rule}Node`;
153-
usedIfaces.add(t);
154-
return t;
164+
return `NodeEntry<${J(rule)}>`;
155165
}
156166

157167
// Name an arm from its first significant item; `sig` is already self-stripped for leds.
@@ -193,9 +203,8 @@ export function generateCstMatch(grammar: CstGrammar, importFrom: string): strin
193203
if (tokenNames.has(it.name)) {
194204
steps.push({
195205
kind: 'tok', name: it.name, template: templateTokenNames.has(it.name),
196-
cap: addCap(captures, used, lowerFirst(it.name), templateTokenNames.has(it.name) ? `CstLeaf | $templateNode` : 'CstLeaf', card),
206+
cap: addCap(captures, used, lowerFirst(it.name), templateTokenNames.has(it.name) ? `LeafEntry | NodeEntry<'$template'>` : 'LeafEntry', card),
197207
});
198-
if (templateTokenNames.has(it.name)) usedIfaces.add('$templateNode');
199208
} else {
200209
steps.push({ kind: 'node', rule: it.name, cap: addCap(captures, used, lowerFirst(it.name), nodeType(it.name), card) });
201210
}
@@ -213,7 +222,7 @@ export function generateCstMatch(grammar: CstGrammar, importFrom: string): strin
213222
// capture it as <kw>Tok.
214223
const lead = inner[0];
215224
if (it.kind === '?' && lead !== undefined && lead.kind === 'lit' && lead.tt === '$keyword' && lead.cap === undefined) {
216-
lead.cap = addCap(captures, used, lead.text + 'Tok', 'CstLeaf', innerCard);
225+
lead.cap = addCap(captures, used, lead.text + 'Tok', 'LeafEntry', innerCard);
217226
}
218227
if (it.kind === '?') steps.push({ kind: 'opt', min1: false, body: inner });
219228
else if (it.kind === '*') steps.push({ kind: 'many', body: inner });
@@ -306,13 +315,13 @@ export function generateCstMatch(grammar: CstGrammar, importFrom: string): strin
306315
}
307316
w(` let i = 0;`);
308317
renderSteps(plan.steps, w, ' ', () => `return null;`);
309-
w(` if (i !== c.length) return null;`);
318+
w(` if (i !== cc) return null;`);
310319
const fields = plan.captures.map(c => {
311320
if (c.card === 'one') return `${c.field}: ${c.name}!`;
312321
return c.field === c.name ? c.name : `${c.field}: ${c.name}`;
313322
});
314323
w(` return { arm: ${J(plan.name)}${fields.length ? ', ' + fields.join(', ') : ''} };`);
315-
emit(`function ${fn}(c: readonly CstChild[], src: string): ${matchTypeName(rule.name)} | null {`);
324+
emit(`function ${fn}(t: TreeAccess, n: number, cc: number, src: string): ${matchTypeName(rule.name)} | null {`);
316325
for (const line of body) emit(line);
317326
emit(`}`);
318327
return fn;
@@ -324,35 +333,37 @@ export function generateCstMatch(grammar: CstGrammar, importFrom: string): strin
324333
}
325334

326335
function litCond(text: string, tt: string): string {
327-
return `__lit(c, i, src, ${J(text)}, ${J(tt)})`;
336+
return `__lit(t, cc, i, src, ${J(text)}, ${tt === '$keyword' ? 1 : 0})`;
328337
}
329338

330339
function renderStep(st: Step, w: (s: string) => void, ind: string, fail: () => string): void {
331340
switch (st.kind) {
332341
case 'lit':
333342
w(`${ind}if (!${litCond(st.text, st.tt)}) ${fail()}`);
334-
if (st.cap) assign(st.cap, `c[i] as CstLeaf`, w, ind);
343+
if (st.cap) assign(st.cap, `__SC[i] as LeafEntry`, w, ind);
335344
w(`${ind}i++;`);
336345
return;
337346
case 'litAlt': {
338347
const conds = st.texts.map((t, k) => litCond(t, st.tt[k]));
339348
w(`${ind}if (!(${conds.join(' || ')})) ${fail()}`);
340-
if (st.cap) assign(st.cap, `src.slice(c[i].offset, c[i].end) as ${st.cap.tsType}`, w, ind);
349+
if (st.cap) assign(st.cap, `src.slice(t.offsetOf(__SC[i]), t.endOf(__SC[i])) as ${st.cap.tsType}`, w, ind);
341350
w(`${ind}i++;`);
342351
return;
343352
}
344353
case 'tok': {
345-
const cond = st.template
346-
? `__tok(c, i, ${J(st.name)}) || __nodeOf(c, i, '$template')`
347-
: `__tok(c, i, ${J(st.name)})`;
354+
const cond = st.name === '$operator'
355+
? `__opTok(t, cc, i)`
356+
: st.template
357+
? `__tok(t, cc, i, ${typeKind.get(st.name)}) || __nodeOf(t, cc, i, ${ruleId.get('$template')})`
358+
: `__tok(t, cc, i, ${typeKind.get(st.name)})`;
348359
w(`${ind}if (!(${cond})) ${fail()}`);
349-
if (st.cap) assign(st.cap, `c[i] as ${st.cap.tsType}`, w, ind);
360+
if (st.cap) assign(st.cap, `__SC[i] as ${st.cap.tsType}`, w, ind);
350361
w(`${ind}i++;`);
351362
return;
352363
}
353364
case 'node':
354-
w(`${ind}if (!__nodeOf(c, i, ${J(st.rule)})) ${fail()}`);
355-
if (st.cap) assign(st.cap, `c[i] as ${st.cap.tsType}`, w, ind);
365+
w(`${ind}if (!__nodeOf(t, cc, i, ${ruleId.get(st.rule)})) ${fail()}`);
366+
if (st.cap) assign(st.cap, `__SC[i] as ${st.cap.tsType}`, w, ind);
356367
w(`${ind}i++;`);
357368
return;
358369
case 'opt': {
@@ -553,7 +564,7 @@ export function generateCstMatch(grammar: CstGrammar, importFrom: string): strin
553564
// ("always") arms appear in every bucket at their declaration position; the buckets
554565
// are superset filters (each arm fn re-checks exactly).
555566
const admits = plans.map(p => firstAdmit(p.steps));
556-
const tryLine = (k: number) => ` { const m = ${fns[k]}(c, src); if (m !== null) return m; }`;
567+
const tryLine = (k: number) => ` { const m = ${fns[k]}(t, n, cc, src); if (m !== null) return m; }`;
557568
const bucketLines = (pred: (keys: Set<string>) => boolean): string[] =>
558569
plans.map((_, k) => (admits[k].keys.size === 0 || pred(admits[k].keys) ? tryLine(k) : ''))
559570
.filter(Boolean);
@@ -592,13 +603,12 @@ export function generateCstMatch(grammar: CstGrammar, importFrom: string): strin
592603
else cset.add(Number(key.slice(2)));
593604
}
594605
}
595-
lines.push(`${pad}const k1 = c[1] as (CstChild & { tokenType?: string; rule?: string }) | undefined;`);
596-
lines.push(`${pad}if (k1 === undefined) {`);
606+
lines.push(`${pad}if (cc < 2) {`);
597607
lines.push(...memberIdx.map((k, i) => (restAdmit[i] === null || restAdmit[i]!.canEmpty ? pad + ' ' + tryLine(k).trim() : '')).filter(Boolean));
598-
lines.push(`${pad}} else if (k1.tokenType === undefined) {`);
599-
lines.push(`${pad} switch (k1.rule) {`);
608+
lines.push(`${pad}} else if ((e1 = __SC[1]) >= 0) {`);
609+
lines.push(`${pad} switch (t.ruleIdOf(e1)) {`);
600610
for (const r of [...nset].sort()) {
601-
lines.push(`${pad} case ${J(r)}: {`);
611+
lines.push(`${pad} case ${ruleId.get(r)}: { // ${r}`);
602612
lines.push(...subTry(i => restAdmit[i]!.keys.has('n:' + r)).map(l => ' ' + l));
603613
lines.push(`${pad} break;`);
604614
lines.push(`${pad} }`);
@@ -608,8 +618,8 @@ export function generateCstMatch(grammar: CstGrammar, importFrom: string): strin
608618
lines.push(`${pad} break;`);
609619
lines.push(`${pad} }`);
610620
lines.push(`${pad} }`);
611-
lines.push(`${pad}} else if (k1.tokenType === '$keyword' || k1.tokenType === '$punct') {`);
612-
lines.push(`${pad} switch (src.charCodeAt(k1.offset)) {`);
621+
lines.push(`${pad}} else if ((_k1 = t.leafKindOf(e1)) === 1 || (_k1 === 0 && t.leafTokKindOf(e1) === 1)) {`);
622+
lines.push(`${pad} switch (src.charCodeAt(t.offsetOf(e1))) {`);
613623
for (const cc of [...cset].sort((a, b) => a - b)) {
614624
lines.push(`${pad} case ${cc}: {`);
615625
lines.push(...subTry(i => restAdmit[i]!.keys.has('c:' + cc)).map(l => ' ' + l));
@@ -621,10 +631,13 @@ export function generateCstMatch(grammar: CstGrammar, importFrom: string): strin
621631
lines.push(`${pad} break;`);
622632
lines.push(`${pad} }`);
623633
lines.push(`${pad} }`);
634+
lines.push(`${pad}} else if (_k1 === 2) {`);
635+
lines.push(...subTry(i => restAdmit[i]!.keys.has('t:$operator')));
624636
lines.push(`${pad}} else {`);
625-
lines.push(`${pad} switch (k1.tokenType) {`);
637+
lines.push(`${pad} switch (t.leafTokKindOf(e1)) {`);
626638
for (const t of [...tset].sort()) {
627-
lines.push(`${pad} case ${J(t)}: {`);
639+
if (t === '$operator') continue; // handled by the kind-2 branch above
640+
lines.push(`${pad} case ${typeKind.get(t)}: { // ${t}`);
628641
lines.push(...subTry(i => restAdmit[i]!.keys.has('t:' + t)).map(l => ' ' + l));
629642
lines.push(`${pad} break;`);
630643
lines.push(`${pad} }`);
@@ -639,15 +652,16 @@ export function generateCstMatch(grammar: CstGrammar, importFrom: string): strin
639652
};
640653

641654
const disp: string[] = [];
642-
disp.push(`export function match${sanitizeIdent(rule.name)}(n: ${nName}, src: string): ${tName} {`);
643-
disp.push(` const c = n.children;`);
644-
disp.push(` const k0 = c[0] as (CstChild & { tokenType?: string; rule?: string }) | undefined;`);
645-
disp.push(` if (k0 === undefined) {`);
655+
disp.push(`export function match${sanitizeIdent(rule.name)}(t: TreeAccess, n: NodeEntry<${J(rule.name)}>, src: string): ${tName} {`);
656+
disp.push(` const cc = __load(t, n);`);
657+
disp.push(` let e1 = 0; let _k1 = 0;`);
658+
disp.push(` if (cc === 0) {`);
646659
for (let k = 0; k < plans.length; k++) if (admits[k].canEmpty || admits[k].keys.size === 0) disp.push(tryLine(k));
647-
disp.push(` } else if (k0.tokenType === undefined) {`);
648-
disp.push(` switch (k0.rule) {`);
660+
disp.push(` } else { const e0 = __SC[0];`);
661+
disp.push(` if (e0 >= 0) {`);
662+
disp.push(` switch (t.ruleIdOf(e0)) {`);
649663
for (const r of [...nodeRules].sort()) {
650-
disp.push(` case ${J(r)}: {`);
664+
disp.push(` case ${ruleId.get(r)}: { // ${r}`);
651665
const members = plans.map((_, k) => k).filter(k => admits[k].keys.size === 0 || admits[k].keys.has('n:' + r));
652666
const concrete = members.filter(k => admits[k].keys.size !== 0);
653667
const oneStep = concrete.every(k => plans[k].steps[0]?.kind === 'node');
@@ -666,8 +680,9 @@ export function generateCstMatch(grammar: CstGrammar, importFrom: string): strin
666680
disp.push(` }`);
667681
}
668682
disp.push(` }`);
669-
disp.push(` } else if (k0.tokenType === '$keyword' || k0.tokenType === '$punct') {`);
670-
disp.push(` switch (src.charCodeAt(k0.offset)) {`);
683+
disp.push(` } else { const _k0 = t.leafKindOf(e0);`);
684+
disp.push(` if (_k0 === 1 || (_k0 === 0 && t.leafTokKindOf(e0) === 1)) {`);
685+
disp.push(` switch (src.charCodeAt(t.offsetOf(e0))) {`);
671686
for (const cc of [...charCodes].sort((a, b) => a - b)) {
672687
disp.push(` case ${cc}: {`);
673688
for (const l of bucketLines(keys => keys.has('c:' + cc))) disp.push(' ' + l);
@@ -681,10 +696,13 @@ export function generateCstMatch(grammar: CstGrammar, importFrom: string): strin
681696
disp.push(` }`);
682697
}
683698
disp.push(` }`);
699+
disp.push(` } else if (_k0 === 2) {`);
700+
for (const l of bucketLines(keys => keys.has('t:$operator'))) disp.push(l);
684701
disp.push(` } else {`);
685-
disp.push(` switch (k0.tokenType) {`);
702+
disp.push(` switch (t.leafTokKindOf(e0)) {`);
686703
for (const t of [...tokNames].sort()) {
687-
disp.push(` case ${J(t)}: {`);
704+
if (t === '$operator') continue; // handled by the kind-2 branch above
705+
disp.push(` case ${typeKind.get(t)}: { // ${t}`);
688706
for (const l of bucketLines(keys => keys.has('t:' + t))) disp.push(' ' + l);
689707
disp.push(` break;`);
690708
disp.push(` }`);
@@ -696,37 +714,73 @@ export function generateCstMatch(grammar: CstGrammar, importFrom: string): strin
696714
disp.push(` }`);
697715
}
698716
disp.push(` }`);
699-
disp.push(` }`);
700-
disp.push(` throw new Error(${J(`match${sanitizeIdent(rule.name)}: no arm matches`)} + ' @' + n.offset);`);
717+
disp.push(` } } }`);
718+
disp.push(` throw new Error(${J(`match${sanitizeIdent(rule.name)}: no arm matches`)} + ' @' + t.offsetOf(n));`);
701719
disp.push(`}`);
702720
bodyParts.push(disp.join('\n'));
703721
matcherMapEntries.push(` ${J(rule.name)}: match${sanitizeIdent(rule.name)},`);
704722
}
705723

706724
header.push(`// GENERATED by src/gen-cst-match.ts — do not edit. Per-arm CST destructurers for ${J(grammar.name ?? '')}.`);
707725
header.push(`/* eslint-disable */`);
708-
header.push(`import type { ${[...usedIfaces].sort().join(', ')} } from ${J(importFrom)};`);
709726
header.push(``);
710-
header.push(`const __lit = (c: readonly CstChild[], i: number, src: string, text: string, tt: string): boolean => {`);
711-
header.push(` const k = c[i] as CstLeaf | undefined;`);
712-
header.push(` return k !== undefined && k.tokenType === tt && k.end - k.offset === text.length && src.startsWith(text, k.offset);`);
727+
header.push(`// The arena accessor surface the matchers read through (the emitted parser's`);
728+
header.push(`// \`tree\` export satisfies it). An ENTRY is a node id (>= 0) or a leaf (< 0).`);
729+
header.push(`export interface TreeAccess {`);
730+
header.push(` ruleNameOf(id: number): string;`);
731+
header.push(` ruleIdOf(id: number): number;`);
732+
header.push(` childCount(id: number): number;`);
733+
header.push(` childAt(id: number, i: number): number;`);
734+
header.push(` childrenInto(id: number, out: number[]): number;`);
735+
header.push(` leafTokenType(entry: number): string;`);
736+
header.push(` leafKindOf(entry: number): number;`);
737+
header.push(` leafTokKindOf(entry: number): number;`);
738+
header.push(` offsetOf(entry: number): number;`);
739+
header.push(` endOf(entry: number): number;`);
740+
header.push(`}`);
741+
header.push(`// Branded entry aliases — compile-time discrimination over plain numbers.`);
742+
header.push(`export type NodeEntry<R extends string = string> = number & { readonly __node?: R };`);
743+
header.push(`export type LeafEntry = number & { readonly __leaf?: true };`);
744+
header.push(``);
745+
header.push(`// Per-call child-entry scratch: matchers are non-reentrant (one node at a time),`);
746+
header.push(`// so one bulk load replaces a childAt round-trip per probe.`);
747+
header.push(`const __SC: number[] = [];`);
748+
header.push(`const __load = (t: TreeAccess, n: number): number => t.childrenInto(n, __SC);`);
749+
header.push(`// kind: 1 = '$keyword' (leaf kind bit), 0 = '$punct' (type-derived + tok-kind 1).`);
750+
header.push(`const __lit = (t: TreeAccess, cc: number, i: number, src: string, text: string, kind: number): boolean => {`);
751+
header.push(` if (i >= cc) return false;`);
752+
header.push(` const e = __SC[i];`);
753+
header.push(` if (e >= 0 || t.leafKindOf(e) !== kind || (kind === 0 && t.leafTokKindOf(e) !== 1)) return false;`);
754+
header.push(` const off = t.offsetOf(e);`);
755+
header.push(` return t.endOf(e) - off === text.length && src.startsWith(text, off);`);
756+
header.push(`};`);
757+
header.push(`const __tok = (t: TreeAccess, cc: number, i: number, k: number): boolean => {`);
758+
header.push(` if (i >= cc) return false;`);
759+
header.push(` const e = __SC[i];`);
760+
header.push(` return e < 0 && t.leafKindOf(e) === 0 && t.leafTokKindOf(e) === k;`);
713761
header.push(`};`);
714-
header.push(`const __tok = (c: readonly CstChild[], i: number, name: string): boolean => {`);
715-
header.push(` const k = c[i] as CstLeaf | undefined;`);
716-
header.push(` return k !== undefined && k.tokenType === name;`);
762+
header.push(`const __opTok = (t: TreeAccess, cc: number, i: number): boolean => {`);
763+
header.push(` if (i >= cc) return false;`);
764+
header.push(` const e = __SC[i];`);
765+
header.push(` return e < 0 && t.leafKindOf(e) === 2;`);
717766
header.push(`};`);
718-
header.push(`const __nodeOf = (c: readonly CstChild[], i: number, rule: string): boolean => {`);
719-
header.push(` const k = c[i] as { rule?: string } | undefined;`);
720-
header.push(` return k !== undefined && k.rule === rule;`);
767+
header.push(`const __nodeOf = (t: TreeAccess, cc: number, i: number, rid: number): boolean => {`);
768+
header.push(` if (i >= cc) return false;`);
769+
header.push(` const e = __SC[i];`);
770+
header.push(` return e >= 0 && t.ruleIdOf(e) === rid;`);
721771
header.push(`};`);
722772
header.push(``);
723773

724774
const footer = [
725775
``,
726776
`/** rule name → its matcher (generic walking; the totality gate uses this). */`,
727-
`export const MATCHERS: Record<string, (n: never, src: string) => { arm: string }> = {`,
777+
`export const MATCHERS: Record<string, (t: TreeAccess, n: never, src: string) => { arm: string }> = {`,
728778
...matcherMapEntries,
729779
`};`,
780+
`/** rule ID → matcher (the emitted parser's rowRule ids — declaration order). */`,
781+
`export const MATCHERS_BY_ID: ((t: TreeAccess, n: never, src: string) => { arm: string })[] = [`,
782+
...grammar.rules.map(r => ` match${sanitizeIdent(r.name)},`),
783+
`];`,
730784
];
731785

732786
return [...header, ...bodyParts.join('\n\n').split('\n'), ...footer].join('\n') + '\n';

0 commit comments

Comments
 (0)