Skip to content

Commit ba88041

Browse files
johnsoncodehkcursoragent
johnsoncodehk
and
cursoragent
committed
Make streamEq self-check optional via validate flag (S6a, #57).
Default Doc/createDoc off; edit-session keeps validate on, edit-session-fast skips the full rescan; gate asserts fast ≡ validated five-field align. Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent 7f8a73f commit ba88041

4 files changed

Lines changed: 56 additions & 25 deletions

File tree

‎src/target-go.ts‎

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -985,12 +985,12 @@ func checkStreamEq(text string, meta []alignMeta) bool {
985985
return `type Edit struct { Start, End int; Text string }
986986
type alignMeta struct { Kind string; Off, End int; Nl bool; Fd, Pd int; Lc, Lb, Hd bool; Td int }
987987
type Align struct {
988-
\tOldN int \`json:"oldN"\`
989-
\tNewN int \`json:"newN"\`
990-
\tPrefix int \`json:"prefix"\`
991-
\tSuffix int \`json:"suffix"\`
992-
\tRelexed int \`json:"relexed"\`
993-
\tStreamEq bool \`json:"streamEq"\`
988+
\tOldN int \`json:"oldN"\`
989+
\tNewN int \`json:"newN"\`
990+
\tPrefix int \`json:"prefix"\`
991+
\tSuffix int \`json:"suffix"\`
992+
\tRelexed int \`json:"relexed"\`
993+
\tStreamEq *bool \`json:"streamEq,omitempty"\`
994994
}
995995
${toMetaFn}
996996
func computeAlignCore(oldText string, oldToks []alignMeta, newText string, newToks []alignMeta) (oldN, newN, prefix, suffix int) {
@@ -1016,13 +1016,14 @@ func toksFromMeta(text string, meta []alignMeta) []Tok {
10161016
\tfor i, m := range meta { t[i] = Tok{m.Kind, text[m.Off:m.End], m.Off, m.End, m.Nl} }
10171017
\treturn t
10181018
}
1019-
${checkStreamEqFn}${windowHelpers}type Doc struct { text string; root int32; toks []alignMeta; align *Align }
1019+
${checkStreamEqFn}${windowHelpers}type Doc struct { text string; root int32; toks []alignMeta; align *Align; validate bool }
10201020
func NewDoc(src string) *Doc {
10211021
\td := &Doc{text: src}
10221022
\td.toks = ${initToks}
10231023
\td.root = parse(tokenize(src))
10241024
\treturn d
10251025
}
1026+
func (d *Doc) SetValidate(v bool) { d.validate = v }
10261027
func (d *Doc) Text() string { return d.text }
10271028
func (d *Doc) Root() int32 { return d.root }
10281029
func (d *Doc) Align() *Align { return d.align }
@@ -1039,9 +1040,13 @@ func (d *Doc) Edit(edits []Edit) int32 {
10391040
\toldText, oldToks := d.text, d.toks
10401041
\trelexed := 0
10411042
${editBody}
1042-
\tstreamEq := checkStreamEq(d.text, d.toks)
10431043
\toldN, newN, prefix, suffix := computeAlignCore(oldText, oldToks, d.text, d.toks)
1044-
\td.align = &Align{oldN, newN, prefix, suffix, relexed, streamEq}
1044+
\ta := &Align{OldN: oldN, NewN: newN, Prefix: prefix, Suffix: suffix, Relexed: relexed}
1045+
\tif d.validate {
1046+
\t\tv := checkStreamEq(d.text, d.toks)
1047+
\t\ta.StreamEq = &v
1048+
\t}
1049+
\td.align = a
10451050
\td.root = parse(toksFromMeta(d.text, d.toks))
10461051
\treturn d.root
10471052
}`;
@@ -1247,8 +1252,10 @@ import (
12471252
func main() {
12481253
\tdata, _ := io.ReadAll(os.Stdin)
12491254
\tsrc := string(data)
1255+
\teditFast := len(os.Args) > 1 && os.Args[1] == "edit-session-fast"
1256+
\teditSess := editFast || (len(os.Args) > 1 && os.Args[1] == "edit-session")
12501257
\t// Self-bench: a numeric arg N times the lex+parse loop and prints ms/iteration.
1251-
\tif len(os.Args) > 1 && os.Args[1] != "edit-session" {
1258+
\tif len(os.Args) > 1 && !editSess {
12521259
\t\tif iters, err := strconv.Atoi(os.Args[1]); err == nil && iters > 0 {
12531260
\t\t\tfor i := 0; i < 3; i++ { parse(tokenize(src)) }
12541261
\t\t\tt0 := time.Now()
@@ -1257,13 +1264,14 @@ func main() {
12571264
\t\t\treturn
12581265
\t\t}
12591266
\t}
1260-
\tif len(os.Args) > 1 && os.Args[1] == "edit-session" {
1267+
\tif editSess {
12611268
\t\tvar sess struct {
12621269
\t\t\tInit string \`json:"init"\`
12631270
\t\t\tBatches [][][3]interface{} \`json:"batches"\`
12641271
\t\t}
12651272
\t\tif json.Unmarshal(data, &sess) != nil { os.Exit(1) }
12661273
\t\td := NewDoc(sess.Init)
1274+
\t\tif !editFast { d.SetValidate(true) }
12671275
\t\tfor _, batch := range sess.Batches {
12681276
\t\t\tedits := make([]Edit, len(batch))
12691277
\t\t\tfor i, t := range batch {

‎src/target-rust.ts‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1075,7 +1075,7 @@ fn check_stream_eq(text: &str, meta: &[AlignMeta]) -> bool {
10751075
return `pub struct Edit { pub start: usize, pub end: usize, pub text: String }
10761076
#[derive(Clone)]
10771077
struct AlignMeta { kind: &'static str, off: usize, end: usize, nl: bool, fd: i64, pd: i64, lc: bool, lb: bool, hd: bool, td: i64 }
1078-
struct Align { old_n: usize, new_n: usize, prefix: usize, suffix: usize, relexed: usize, stream_eq: bool }
1078+
struct Align { old_n: usize, new_n: usize, prefix: usize, suffix: usize, relexed: usize, stream_eq: Option<bool> }
10791079
${toMetaFn}
10801080
fn compute_align_core(old_text: &str, old_toks: &[AlignMeta], new_text: &str, new_toks: &[AlignMeta]) -> (usize, usize, usize, usize) {
10811081
let old_n = old_toks.len();
@@ -1104,18 +1104,19 @@ fn compute_align_core(old_text: &str, old_toks: &[AlignMeta], new_text: &str, ne
11041104
fn toks_from_meta<'a>(text: &'a str, meta: &[AlignMeta]) -> Vec<Tok<'a>> {
11051105
meta.iter().map(|m| Tok { kind: m.kind, text: &text[m.off..m.end], off: m.off, end: m.end, nl: m.nl }).collect()
11061106
}
1107-
${checkStreamEqFn}${windowHelpers}pub struct Doc { text: String, toks: Vec<AlignMeta>, align: Option<Align> }
1107+
${checkStreamEqFn}${windowHelpers}pub struct Doc { text: String, toks: Vec<AlignMeta>, align: Option<Align>, validate: bool }
11081108
impl Doc {
1109-
pub fn new(text: String) -> Doc { Doc { text: text.clone(), toks: ${initToks}, align: None } }
1109+
pub fn new(text: String) -> Doc { Doc { text: text.clone(), toks: ${initToks}, align: None, validate: false } }
1110+
pub fn set_validate(&mut self, v: bool) { self.validate = v; }
11101111
pub fn text(&self) -> &str { &self.text }
11111112
pub fn alignment(&self) -> Option<&Align> { self.align.as_ref() }
11121113
pub fn edit(&mut self, edits: &[Edit]) {
11131114
let old_text = self.text.clone();
11141115
let old_toks = self.toks.clone();
11151116
let mut relexed = 0usize;
11161117
${editBody}
1117-
let stream_eq = check_stream_eq(&self.text, &self.toks);
11181118
let (old_n, new_n, prefix, suffix) = compute_align_core(&old_text, &old_toks, &self.text, &self.toks);
1119+
let stream_eq = if self.validate { Some(check_stream_eq(&self.text, &self.toks)) } else { None };
11191120
self.align = Some(Align { old_n, new_n, prefix, suffix, relexed, stream_eq });
11201121
}
11211122
pub fn parse(&self) -> Option<(Parser<'_>, i32)> {
@@ -1382,15 +1383,19 @@ fn main() {
13821383
let mut src = String::new();
13831384
std::io::stdin().read_to_string(&mut src).unwrap();
13841385
let args: Vec<String> = std::env::args().collect();
1385-
if args.len() > 1 && args[1] == "edit-session" {
1386+
if args.len() > 1 && (args[1] == "edit-session" || args[1] == "edit-session-fast") {
13861387
let (init, batches) = parse_edit_session(&src).unwrap();
13871388
let mut doc = Doc::new(init);
1389+
if args[1] == "edit-session" { doc.set_validate(true); }
13881390
for batch in &batches {
13891391
let edits: Vec<Edit> = batch.iter().map(|&(s, e, ref t)| Edit { start: s, end: e, text: t.clone() }).collect();
13901392
doc.edit(&edits);
13911393
}
13921394
if let Some(a) = doc.alignment() {
1393-
eprintln!("{{\\"oldN\\":{},\\"newN\\":{},\\"prefix\\":{},\\"suffix\\":{},\\"relexed\\":{},\\"streamEq\\":{}}}", a.old_n, a.new_n, a.prefix, a.suffix, a.relexed, a.stream_eq);
1395+
match a.stream_eq {
1396+
Some(eq) => eprintln!("{{\\"oldN\\":{},\\"newN\\":{},\\"prefix\\":{},\\"suffix\\":{},\\"relexed\\":{},\\"streamEq\\":{}}}", a.old_n, a.new_n, a.prefix, a.suffix, a.relexed, eq),
1397+
None => eprintln!("{{\\"oldN\\":{},\\"newN\\":{},\\"prefix\\":{},\\"suffix\\":{},\\"relexed\\":{}}}", a.old_n, a.new_n, a.prefix, a.suffix, a.relexed),
1398+
}
13941399
}
13951400
match doc.parse() {
13961401
Some((p, root)) => {

‎src/target-ts.ts‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -888,7 +888,7 @@ function checkStreamEq(text: string, meta: AlignMeta[]): boolean {
888888
const initToks = (hasNewline || rxOnly || tplOnly || rxTpl) ? 'scanMeta(src)' : 'toMeta(tokenize(src))';
889889
return `export type Edit = { start: number; end: number; text: string };
890890
type AlignMeta = { kind: string; off: number; end: number; nl: boolean; fd: number; pd: number; lc: boolean; lb: boolean; hd: boolean; td: number };
891-
type Align = { oldN: number; newN: number; prefix: number; suffix: number; relexed: number; streamEq: boolean };
891+
type Align = { oldN: number; newN: number; prefix: number; suffix: number; relexed: number; streamEq?: boolean };
892892
${toMetaFn}
893893
function computeAlign(oldText: string, oldToks: AlignMeta[], newText: string, newToks: AlignMeta[]): Omit<Align, 'relexed' | 'streamEq'> {
894894
const oldN = oldToks.length, newN = newToks.length;
@@ -913,7 +913,8 @@ function computeAlign(oldText: string, oldToks: AlignMeta[], newText: string, ne
913913
function toksFromMeta(text: string, meta: AlignMeta[]): Tok[] {
914914
return meta.map((m) => ({ kind: m.kind, text: text.slice(m.off, m.end), off: m.off, end: m.end, nl: m.nl }));
915915
}
916-
${checkStreamEqFn}${windowHelpers}export function createDoc(src: string): { text(): string; root(): Node | null; align(): Align | null; edit(edits: Edit[]): Node | null } {
916+
${checkStreamEqFn}${windowHelpers}export function createDoc(src: string, opts?: { validate?: boolean }): { text(): string; root(): Node | null; align(): Align | null; edit(edits: Edit[]): Node | null } {
917+
const validate = opts?.validate === true;
917918
let text = src;
918919
let prevToks = ${initToks};
919920
let align: Align | null = null;
@@ -926,8 +927,8 @@ ${checkStreamEqFn}${windowHelpers}export function createDoc(src: string): { text
926927
const oldText = text, oldToks = prevToks;
927928
let relexed = 0;
928929
${editBody}
929-
const streamEq = checkStreamEq(text, prevToks);
930-
align = { ...computeAlign(oldText, oldToks, text, prevToks), relexed, streamEq };
930+
const core = { ...computeAlign(oldText, oldToks, text, prevToks), relexed };
931+
align = validate ? { ...core, streamEq: checkStreamEq(text, prevToks) } : core;
931932
root = parse(toksFromMeta(text, prevToks));
932933
return root;
933934
},
@@ -1062,9 +1063,10 @@ ${docEditBlock(ir)}
10621063
// NOT part of the emitted parser. The import is hoisted, so it may follow the library code.
10631064
import { readFileSync } from 'node:fs';
10641065
const _raw = readFileSync(0, 'utf8');
1065-
if (process.argv.includes('edit-session')) {
1066+
const _editFast = process.argv.includes('edit-session-fast');
1067+
if (_editFast || process.argv.includes('edit-session')) {
10661068
const { init, batches } = JSON.parse(_raw) as { init: string; batches: [number, number, string][][] };
1067-
const doc = createDoc(init);
1069+
const doc = createDoc(init, { validate: !_editFast });
10681070
for (const batch of batches) doc.edit(batch.map(([start, end, text]) => ({ start, end, text })));
10691071
const a = doc.align();
10701072
if (a) process.stderr.write(JSON.stringify(a) + '\\n');

‎test/portable-targets.ts‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -446,13 +446,16 @@ for (const c of CASES) {
446446

447447
const editScenarios = EDIT_SCENARIOS[c.grammar];
448448
if (editScenarios) {
449-
let editOk = 0, alignOk = 0;
449+
let editOk = 0, alignOk = 0, fastOk = 0;
450450
const editCmd = r.label === 'typescript' ? 'node' : r.label === 'go' ? `${dir}/go/p` : `${dir}/pr`;
451451
const editArgs = r.label === 'typescript' ? [`${dir}/p.ts`, 'edit-session'] : ['edit-session'];
452+
const fastArgs = r.label === 'typescript' ? [`${dir}/p.ts`, 'edit-session-fast'] : ['edit-session-fast'];
452453
const runEdit = (json: string) => runEditSession(editCmd, editArgs, json);
454+
const runFast = (json: string) => runEditSession(editCmd, fastArgs, json);
453455
for (const sc of editScenarios) {
454456
const final = applyEdits(sc.init, sc.batches);
455-
const a = runEdit(JSON.stringify({ init: sc.init, batches: sc.batches }));
457+
const payload = JSON.stringify({ init: sc.init, batches: sc.batches });
458+
const a = runEdit(payload);
456459
const b = r.run(final);
457460
if (a.ok === b.ok && (!a.ok || a.cst === b.cst) && a.ok === oracleOut(final).ok) editOk++;
458461
else {
@@ -482,9 +485,22 @@ for (const c of CASES) {
482485
failures++;
483486
console.log(` ${c.grammar}/${r.label}: token-align mismatch want=${JSON.stringify(want)} got=${JSON.stringify(a.align)}`);
484487
}
488+
const f = runFast(payload);
489+
const fiveEq = !!(a.align && f.align
490+
&& a.align.oldN === f.align.oldN && a.align.newN === f.align.newN
491+
&& a.align.prefix === f.align.prefix && a.align.suffix === f.align.suffix
492+
&& a.align.relexed === f.align.relexed);
493+
const noStreamEq = f.align !== undefined && !('streamEq' in f.align);
494+
const outEq = a.ok === f.ok && (!a.ok || a.cst === f.cst);
495+
if (fiveEq && noStreamEq && outEq) fastOk++;
496+
else {
497+
failures++;
498+
console.log(` ${c.grammar}/${r.label}: fast≢validated fiveEq=${fiveEq} noStreamEq=${noStreamEq} outEq=${outEq} validated=${JSON.stringify(a.align)} fast=${JSON.stringify(f.align)}`);
499+
}
485500
}
486501
console.log(` ${c.grammar}/${r.label}: ${editOk}/${editScenarios.length} edit-sessions ≡ fresh`);
487502
console.log(` ${c.grammar}/${r.label}: ${alignOk}/${editScenarios.length} token-alignments ≡ oracle`);
503+
console.log(` ${c.grammar}/${r.label}: ${fastOk}/${editScenarios.length} fast ≡ validated`);
488504
}
489505
}
490506
}

0 commit comments

Comments
 (0)