Skip to content

Commit 80a952c

Browse files
johnsoncodehkcursoragent
johnsoncodehk
and
cursoragent
committed
emit-portable: reuse top-level subtrees across edits in TS target (S7b, #57)
Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent 9ae3155 commit 80a952c

4 files changed

Lines changed: 241 additions & 30 deletions

File tree

‎src/target-go.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -990,6 +990,7 @@ type Align struct {
990990
\tPrefix int \`json:"prefix"\`
991991
\tSuffix int \`json:"suffix"\`
992992
\tRelexed int \`json:"relexed"\`
993+
\tReused int \`json:"reused"\`
993994
\tStreamEq *bool \`json:"streamEq,omitempty"\`
994995
}
995996
${toMetaFn}
@@ -1041,7 +1042,7 @@ func (d *Doc) Edit(edits []Edit) int32 {
10411042
\trelexed := 0
10421043
${editBody}
10431044
\toldN, newN, prefix, suffix := computeAlignCore(oldText, oldToks, d.text, d.toks)
1044-
\ta := &Align{OldN: oldN, NewN: newN, Prefix: prefix, Suffix: suffix, Relexed: relexed}
1045+
\ta := &Align{OldN: oldN, NewN: newN, Prefix: prefix, Suffix: suffix, Relexed: relexed, Reused: 0}
10451046
\tif d.validate {
10461047
\t\tv := checkStreamEq(d.text, d.toks)
10471048
\t\ta.StreamEq = &v

‎src/target-rust.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1074,7 +1074,7 @@ fn check_stream_eq(text: &str, meta: &[AlignMeta]) -> bool {
10741074
return `pub struct Edit { pub start: usize, pub end: usize, pub text: String }
10751075
#[derive(Clone)]
10761076
struct AlignMeta { kind: &'static str, off: usize, end: usize, nl: bool, fd: i64, pd: i64, lc: bool, lb: bool, hd: bool, td: i64 }
1077-
struct Align { old_n: usize, new_n: usize, prefix: usize, suffix: usize, relexed: usize, stream_eq: Option<bool> }
1077+
struct Align { old_n: usize, new_n: usize, prefix: usize, suffix: usize, relexed: usize, reused: usize, stream_eq: Option<bool> }
10781078
${toMetaFn}
10791079
fn compute_align_core(old_text: &str, old_toks: &[AlignMeta], new_text: &str, new_toks: &[AlignMeta]) -> (usize, usize, usize, usize) {
10801080
let old_n = old_toks.len();
@@ -1116,7 +1116,7 @@ impl Doc {
11161116
${editBody}
11171117
let (old_n, new_n, prefix, suffix) = compute_align_core(&old_text, &old_toks, &self.text, &self.toks);
11181118
let stream_eq = if self.validate { Some(check_stream_eq(&self.text, &self.toks)) } else { None };
1119-
self.align = Some(Align { old_n, new_n, prefix, suffix, relexed, stream_eq });
1119+
self.align = Some(Align { old_n, new_n, prefix, suffix, relexed, reused: 0, stream_eq });
11201120
}
11211121
pub fn parse(&self) -> Option<(Parser<'_>, i32)> {
11221122
let toks = toks_from_meta(&self.text, &self.toks);
@@ -1392,8 +1392,8 @@ fn main() {
13921392
}
13931393
if let Some(a) = doc.alignment() {
13941394
match a.stream_eq {
1395-
Some(eq) => eprintln!("{{\\"oldN\\":{},\\"newN\\":{},\\"prefix\\":{},\\"suffix\\":{},\\"relexed\\":{},\\"streamEq\\":{}}}", a.old_n, a.new_n, a.prefix, a.suffix, a.relexed, eq),
1396-
None => eprintln!("{{\\"oldN\\":{},\\"newN\\":{},\\"prefix\\":{},\\"suffix\\":{},\\"relexed\\":{}}}", a.old_n, a.new_n, a.prefix, a.suffix, a.relexed),
1395+
Some(eq) => eprintln!("{{\\"oldN\\":{},\\"newN\\":{},\\"prefix\\":{},\\"suffix\\":{},\\"relexed\\":{},\\"reused\\":{},\\"streamEq\\":{}}}", a.old_n, a.new_n, a.prefix, a.suffix, a.relexed, a.reused, eq),
1396+
None => eprintln!("{{\\"oldN\\":{},\\"newN\\":{},\\"prefix\\":{},\\"suffix\\":{},\\"relexed\\":{},\\"reused\\":{}}}", a.old_n, a.new_n, a.prefix, a.suffix, a.relexed, a.reused),
13971397
}
13981398
}
13991399
match doc.parse() {

‎src/target-ts.ts‎

Lines changed: 169 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -376,6 +376,28 @@ function predAltBody(branches: Step[][], firsts?: FirstSig[]): string {
376376
return `const t = peek(); if (t === null) return false; ${arms} return false;`;
377377
}
378378

379+
/** Entry shape eligible for top-level subtree reuse: single alt whose sole step is
380+
* `star(rule)` or `star(alt(rule, rule, …))` (calc / javascript). */
381+
function topReusePlan(ir: ParserIR): { topOneBody: string } | null {
382+
const entry = ir.rules.find((r) => r.name === ir.entry);
383+
if (!entry || entry.kind !== 'rd' || entry.alts.length !== 1) return null;
384+
const alt = entry.alts[0];
385+
if (alt.length !== 1 || alt[0].t !== 'star') return null;
386+
const step = alt[0].step;
387+
if (step.t === 'rule') return { topOneBody: ` return parse${step.name}();` };
388+
if (step.t === 'alt') {
389+
for (const br of step.branches) {
390+
if (br.length !== 1 || br[0].t !== 'rule') return null;
391+
}
392+
const tries = step.branches.map((br) => {
393+
const name = (br[0] as { t: 'rule'; name: string }).name;
394+
return ` { const sp = pos; const n = parse${name}(); if (n !== null) return n; pos = sp; }`;
395+
}).join('\n');
396+
return { topOneBody: `${tries}\n return null;` };
397+
}
398+
return null;
399+
}
400+
379401
function rdRule(r: RdRule): string {
380402
if (r.predictive) {
381403
const arm = (steps: Step[], i: number) => ` ${i === 0 ? 'if' : 'else if'} (${firstCond(r.altFirst[i], 't')}) { const kids: Cst[] = []; if (${steps.map(stepCond).join(' && ')}) return branch(${J(r.cstName)}, kids, save); }`;
@@ -396,6 +418,26 @@ ${r.alts.map(alt).join('\n')}
396418
}`;
397419
}
398420

421+
/** Entry rule that records per-top-kid lookahead ext via parseTopOne. */
422+
function rdEntryWithReuse(r: RdRule, plan: { topOneBody: string }): string {
423+
return `function parseTopOne(): Node | null {
424+
${plan.topOneBody}
425+
}
426+
function parse${r.name}(): Node | null {
427+
const save = pos;
428+
const kids: Cst[] = [];
429+
for (;;) {
430+
const sp = pos;
431+
maxLook = 0;
432+
const n = parseTopOne();
433+
if (n === null) { pos = sp; break; }
434+
n.ext = Math.max(n.tokEnd, maxLook);
435+
kids.push(n);
436+
}
437+
return branch(${J(r.cstName)}, kids, save);
438+
}`;
439+
}
440+
399441
function prattRule(r: PrattRule, tpl: TplCfg | null): string {
400442
const tplNud = tpl && r.nudToks.includes(tpl.token)
401443
? ` if (t.kind === '$templateHead') { const node = matchTemplate(); return node === null ? null : { rule: ${J(r.cstName)}, children: [node], offset: node.offset, end: node.end, tokStart: node.tokStart, tokEnd: node.tokEnd }; }\n`
@@ -488,6 +530,7 @@ function docEditBlock(ir: ParserIR): string {
488530
const rxOnly = !!(ir.regexCtx && !ir.tpl && !ir.newlineCfg);
489531
const tplOnly = !!(ir.tpl && !ir.regexCtx && !ir.newlineCfg);
490532
const rxTpl = !!(ir.regexCtx && ir.tpl && !ir.newlineCfg);
533+
const topReuse = topReusePlan(ir);
491534
const zeroMeta = ', fd: 0, pd: 0, lc: false, lb: false, hd: false, td: 0';
492535
const adoptSuffix = `for (let j = oIdx + 1; j < oldToks.length; j++) {
493536
const ot = oldToks[j];
@@ -887,11 +930,121 @@ function checkStreamEq(text: string, meta: AlignMeta[]): boolean {
887930
}
888931
`;
889932
const initToks = (hasNewline || rxOnly || tplOnly || rxTpl) ? 'scanMeta(src)' : 'toMeta(tokenize(src))';
933+
const reuseFns = topReuse ? `
934+
function cstJSON(n: Cst): string {
935+
return JSON.stringify(n, (k, v) => (k === 'tokStart' || k === 'tokEnd' || k === 'ext' ? undefined : v));
936+
}
937+
function checkTreeEq(text: string, root: Node | null): boolean {
938+
const fresh = parse(tokenize(text));
939+
if (root === null || fresh === null) return root === fresh;
940+
return cstJSON(root) === cstJSON(fresh as Cst);
941+
}
942+
function shiftSubtree(n: Cst, byteDelta: number, tokDelta: number): void {
943+
n.offset += byteDelta;
944+
n.end += byteDelta;
945+
n.tokStart += tokDelta;
946+
n.tokEnd += tokDelta;
947+
if ('ext' in n && typeof (n as Node).ext === 'number') (n as Node).ext! += tokDelta;
948+
if ('children' in n) for (const c of n.children) shiftSubtree(c, byteDelta, tokDelta);
949+
}
950+
function tryReuseTop(oldRoot: Node, newText: string, newMeta: AlignMeta[], byteDelta: number, oldN: number, newN: number, prefix: number, suffix: number): { root: Node; reused: number } | null {
951+
const oldKids = oldRoot.children as Node[];
952+
let prefixLen = 0;
953+
while (prefixLen < oldKids.length) {
954+
const k = oldKids[prefixLen]!;
955+
const ext = k.ext ?? k.tokEnd;
956+
if (ext <= prefix) prefixLen++;
957+
else break;
958+
}
959+
let suffixStart = oldKids.length;
960+
for (let i = oldKids.length - 1; i >= prefixLen; i--) {
961+
if (oldKids[i]!.tokStart >= oldN - suffix) suffixStart = i;
962+
else break;
963+
}
964+
const prefixKids = oldKids.slice(0, prefixLen);
965+
const suffixCand = oldKids.slice(suffixStart);
966+
const tokDelta = newN - oldN;
967+
_src = newText;
968+
toks = toksFromMeta(newText, newMeta);
969+
pos = prefixLen > 0 ? prefixKids[prefixLen - 1]!.tokEnd : 0;
970+
const mid: Node[] = [];
971+
const suffixBound = newN - suffix;
972+
const maxCand = suffixCand.length > 0 ? Math.max(...suffixCand.map((k) => k.tokStart + tokDelta)) : -1;
973+
const finish = (adoptFrom: number): { root: Node; reused: number } => {
974+
const adopted = suffixCand.slice(adoptFrom);
975+
for (const s of adopted) shiftSubtree(s, byteDelta, tokDelta);
976+
const children: Cst[] = [...prefixKids, ...mid, ...adopted];
977+
const offset = children.length > 0 ? children[0]!.offset : 0;
978+
const end = children.length > 0 ? children[children.length - 1]!.end : offset;
979+
const tokStart = children.length > 0 ? (children[0] as Node).tokStart : 0;
980+
const tokEnd = children.length > 0 ? (children[children.length - 1] as Node).tokEnd : 0;
981+
return { root: { rule: oldRoot.rule, children, offset, end, tokStart, tokEnd }, reused: prefixKids.length + adopted.length };
982+
};
983+
const tryHit = (): { root: Node; reused: number } | null => {
984+
if (pos < suffixBound) return null;
985+
if (suffixCand.length === 0) {
986+
if (pos === newN) return finish(0);
987+
return null;
988+
}
989+
const hit = suffixCand.findIndex((k) => k.tokStart + tokDelta === pos);
990+
if (hit >= 0) return finish(hit);
991+
return null;
992+
};
993+
{
994+
const early = tryHit();
995+
if (early) return early;
996+
if (suffixCand.length > 0 && maxCand >= 0 && pos > maxCand) return null;
997+
}
998+
for (;;) {
999+
if (pos >= toks.length) {
1000+
if (suffixCand.length === 0 && pos === newN) return finish(0);
1001+
return tryHit() ?? null;
1002+
}
1003+
maxLook = 0;
1004+
const sp = pos;
1005+
const n = parseTopOne();
1006+
if (n === null) { pos = sp; return null; }
1007+
n.ext = Math.max(n.tokEnd, maxLook);
1008+
mid.push(n);
1009+
const hit = tryHit();
1010+
if (hit) return hit;
1011+
if (suffixCand.length > 0 && maxCand >= 0 && pos > maxCand) return null;
1012+
}
1013+
}
1014+
` : `
1015+
function cstJSON(n: Cst): string {
1016+
return JSON.stringify(n, (k, v) => (k === 'tokStart' || k === 'tokEnd' || k === 'ext' ? undefined : v));
1017+
}
1018+
function checkTreeEq(text: string, root: Node | null): boolean {
1019+
const fresh = parse(tokenize(text));
1020+
if (root === null || fresh === null) return root === fresh;
1021+
return cstJSON(root) === cstJSON(fresh as Cst);
1022+
}
1023+
`;
1024+
const editParse = topReuse
1025+
? ` const byteDelta = text.length - oldText.length;
1026+
let reused = 0;
1027+
let next: Node | null = null;
1028+
if (root !== null) {
1029+
const got = tryReuseTop(root, text, prevToks, byteDelta, core.oldN, core.newN, core.prefix, core.suffix);
1030+
if (got) { next = got.root; reused = got.reused; }
1031+
}
1032+
if (next === null) { _src = text; next = parse(toksFromMeta(text, prevToks)) as Node | null; reused = 0; }
1033+
root = next;
1034+
align = validate
1035+
? { ...core, reused, streamEq: checkStreamEq(text, prevToks), treeEq: checkTreeEq(text, root) }
1036+
: { ...core, reused };`
1037+
: ` const reused = 0;
1038+
_src = text;
1039+
root = parse(toksFromMeta(text, prevToks)) as Node | null;
1040+
align = validate
1041+
? { ...core, reused, streamEq: checkStreamEq(text, prevToks), treeEq: checkTreeEq(text, root) }
1042+
: { ...core, reused };`;
8901043
return `export type Edit = { start: number; end: number; text: string };
8911044
type AlignMeta = { kind: string; off: number; end: number; nl: boolean; fd: number; pd: number; lc: boolean; lb: boolean; hd: boolean; td: number };
892-
type Align = { oldN: number; newN: number; prefix: number; suffix: number; relexed: number; streamEq?: boolean };
1045+
type Align = { oldN: number; newN: number; prefix: number; suffix: number; relexed: number; reused: number; streamEq?: boolean; treeEq?: boolean };
8931046
${toMetaFn}
894-
function computeAlign(oldText: string, oldToks: AlignMeta[], newText: string, newToks: AlignMeta[]): Omit<Align, 'relexed' | 'streamEq'> {
1047+
function computeAlign(oldText: string, oldToks: AlignMeta[], newText: string, newToks: AlignMeta[]): Omit<Align, 'relexed' | 'reused' | 'streamEq' | 'treeEq'> {
8951048
const oldN = oldToks.length, newN = newToks.length;
8961049
let prefix = 0;
8971050
while (prefix < oldN && prefix < newN) {
@@ -914,12 +1067,12 @@ function computeAlign(oldText: string, oldToks: AlignMeta[], newText: string, ne
9141067
function toksFromMeta(text: string, meta: AlignMeta[]): Tok[] {
9151068
return meta.map((m) => ({ kind: m.kind, text: text.slice(m.off, m.end), off: m.off, end: m.end, nl: m.nl }));
9161069
}
917-
${checkStreamEqFn}${windowHelpers}export function createDoc(src: string, opts?: { validate?: boolean }): { text(): string; root(): Node | null; align(): Align | null; edit(edits: Edit[]): Node | null } {
1070+
${checkStreamEqFn}${windowHelpers}${reuseFns}export function createDoc(src: string, opts?: { validate?: boolean }): { text(): string; root(): Node | null; align(): Align | null; edit(edits: Edit[]): Node | null } {
9181071
const validate = opts?.validate === true;
9191072
let text = src;
9201073
let prevToks = ${initToks};
9211074
let align: Align | null = null;
922-
let root: Node | null = parse(tokenize(src));
1075+
let root: Node | null = parse(tokenize(src)) as Node | null;
9231076
return {
9241077
text(): string { return text; },
9251078
root(): Node | null { return root; },
@@ -929,8 +1082,7 @@ ${checkStreamEqFn}${windowHelpers}export function createDoc(src: string, opts?:
9291082
let relexed = 0;
9301083
${editBody}
9311084
const core = { ...computeAlign(oldText, oldToks, text, prevToks), relexed };
932-
align = validate ? { ...core, streamEq: checkStreamEq(text, prevToks) } : core;
933-
root = parse(toksFromMeta(text, prevToks));
1085+
${editParse}
9341086
return root;
9351087
},
9361088
};
@@ -956,7 +1108,12 @@ export function tokenize(src: string): Tok[] { return lex(src); }
9561108
},
9571109
emitParser(grammar: CstGrammar, lexerSrc: string | null): string {
9581110
const ir = portableIR(grammar);
959-
const ruleFns = ir.rules.map((r) => (r.kind === 'pratt' ? prattRule(r, ir.tpl) : rdRule(r))).join('\n\n');
1111+
const reuse = topReusePlan(ir);
1112+
const ruleFns = ir.rules.map((r) => {
1113+
if (r.kind === 'pratt') return prattRule(r, ir.tpl);
1114+
if (reuse && r.name === ir.entry) return rdEntryWithReuse(r, reuse);
1115+
return rdRule(r);
1116+
}).join('\n\n');
9601117
const matchTemplate = ir.tpl ? `function matchTemplate(): Cst | null {
9611118
const t = peek();
9621119
if (t === null || t.kind !== '$templateHead') return null;
@@ -981,18 +1138,19 @@ export function tokenize(src: string): Tok[] { return lex(src); }
9811138
9821139
type Tok = { kind: string; text: string; off: number; end: number; nl: boolean };
9831140
type Leaf = { tokenType: string; offset: number; end: number; tokStart: number; tokEnd: number };
984-
type Node = { rule: string; children: Cst[]; offset: number; end: number; tokStart: number; tokEnd: number };
1141+
type Node = { rule: string; children: Cst[]; offset: number; end: number; tokStart: number; tokEnd: number; ext?: number };
9851142
type Cst = Node | Leaf;
9861143
9871144
${lexerSrc ?? ''}
9881145
9891146
let toks: Tok[] = [];
9901147
let pos = 0;
1148+
let maxLook = 0;
9911149
let _capped = false;
9921150
let _suppressNext: Set<string> | null = null;
9931151
let _suppressCur: Set<string> | null = null;
9941152
let _src = '';
995-
function peek(): Tok | null { return pos < toks.length ? toks[pos] : null; }
1153+
function peek(): Tok | null { maxLook = Math.max(maxLook, pos + 1); return pos < toks.length ? toks[pos] : null; }
9961154
function headLeafText(node: Cst): string {
9971155
let n: Cst = node;
9981156
while ('children' in n && n.children.length > 0) n = n.children[0];
@@ -1054,6 +1212,7 @@ export function tokenize(src: string): Tok[] { _src = src; return lex(src); }
10541212
export function parse(tokens: Tok[]): Cst | null {
10551213
toks = tokens;
10561214
pos = 0;
1215+
maxLook = 0;
10571216
const root = parse${ir.entry}();
10581217
return root !== null && pos === toks.length ? root : null;
10591218
}
@@ -1067,7 +1226,7 @@ ${docEditBlock(ir)}
10671226
import { readFileSync } from 'node:fs';
10681227
const _raw = readFileSync(0, 'utf8');
10691228
const _editFast = process.argv.includes('edit-session-fast');
1070-
const _cstJSON = (n: Cst) => JSON.stringify(n, (k, v) => (k === 'tokStart' || k === 'tokEnd' ? undefined : v));
1229+
const _cstJSON = (n: Cst) => JSON.stringify(n, (k, v) => (k === 'tokStart' || k === 'tokEnd' || k === 'ext' ? undefined : v));
10711230
if (_editFast || process.argv.includes('edit-session')) {
10721231
const { init, batches } = JSON.parse(_raw) as { init: string; batches: [number, number, string][][] };
10731232
const doc = createDoc(init, { validate: !_editFast });

0 commit comments

Comments
 (0)