Add S2 token alignment with safe edit bounds and large-doc gate.

Author: johnsoncodehk

src/target-go.ts

@@ -552,17 +552,65 @@ func parse(t []Tok) int32 {
 }
 
 type Edit struct { Start, End int; Text string }
-type Doc struct { text string; root int32 }
+type alignMeta struct { Kind string; Off, End int; Nl bool }
+type Align struct {
+\tOldN    int \`json:"oldN"\`
+\tNewN    int \`json:"newN"\`
+\tPrefix  int \`json:"prefix"\`
+\tSuffix  int \`json:"suffix"\`
+}
+func toMeta(toks []Tok) []alignMeta {
+\tm := make([]alignMeta, len(toks))
+\tfor i, t := range toks { m[i] = alignMeta{t.Kind, t.Off, t.End, t.Nl} }
+\treturn m
+}
+func computeAlign(oldText string, oldToks []alignMeta, newText string, newToks []alignMeta) Align {
+\toldN, newN := len(oldToks), len(newToks)
+\tprefix := 0
+\tfor prefix < oldN && prefix < newN {
+\t\to, n := oldToks[prefix], newToks[prefix]
+\t\tif o.Kind != n.Kind || o.Off != n.Off || o.End != n.End || o.Nl != n.Nl { break }
+\t\tif oldText[o.Off:o.End] != newText[n.Off:n.End] { break }
+\t\tprefix++
+\t}
+\tdelta := len(newText) - len(oldText)
+\tminN := oldN; if newN < minN { minN = newN }
+\tsuffix := 0
+\tfor prefix+suffix < minN {
+\t\to, n := oldToks[oldN-1-suffix], newToks[newN-1-suffix]
+\t\tif o.Kind != n.Kind || o.Nl != n.Nl || n.Off != o.Off+delta || n.End != o.End+delta { break }
+\t\tif oldText[o.Off:o.End] != newText[n.Off:n.End] { break }
+\t\tsuffix++
+\t}
+\treturn Align{oldN, newN, prefix, suffix}
+}
+type Doc struct { text string; root int32; toks []alignMeta; align *Align }
 func NewDoc(src string) *Doc {
 \td := &Doc{text: src}
+\td.toks = toMeta(tokenize(src))
 \td.root = parse(tokenize(src))
 \treturn d
 }
 func (d *Doc) Text() string { return d.text }
 func (d *Doc) Root() int32 { return d.root }
+func (d *Doc) Align() *Align { return d.align }
+func applyEdit(text string, e Edit) string {
+\tn := len(text)
+\tstart, end := e.Start, e.End
+\tif start < 0 { start = 0 }
+\tif start > n { start = n }
+\tif end < start { end = start }
+\tif end > n { end = n }
+\treturn text[:start] + e.Text + text[end:]
+}
 func (d *Doc) Edit(edits []Edit) int32 {
-\tfor _, e := range edits { d.text = d.text[:e.Start] + e.Text + d.text[e.End:] }
-\td.root = parse(tokenize(d.text))
+\toldText, oldToks := d.text, d.toks
+\tfor _, e := range edits { d.text = applyEdit(d.text, e) }
+\tnewToks := tokenize(d.text)
+\td.toks = toMeta(newToks)
+\ta := computeAlign(oldText, oldToks, d.text, d.toks)
+\td.align = &a
+\td.root = parse(newToks)
 \treturn d.root
 }
 `;
@@ -610,6 +658,7 @@ func main() {
 \t\t\t}
 \t\t\td.Edit(edits)
 \t\t}
+\t\tif a := d.Align(); a != nil { b, _ := json.Marshal(a); os.Stderr.Write(append(b, '\\n')) }
 \t\troot := d.Root()
 \t\tif root < 0 || pos != len(toks) { fmt.Fprintf(os.Stderr, "parse error (pos %d/%d)\\n", pos, len(toks)); os.Exit(1) }
 \t\tvar b strings.Builder

src/target-rust.ts

@@ -548,14 +548,65 @@ fn parse<'a>(tokens: Tokens<'a>) -> Option<(Parser<'a>, i32)> {
 }
 
 pub struct Edit { pub start: usize, pub end: usize, pub text: String }
-pub struct Doc { text: String }
+#[derive(Clone)]
+struct AlignMeta { kind: &'static str, off: usize, end: usize, nl: bool }
+struct Align { old_n: usize, new_n: usize, prefix: usize, suffix: usize }
+fn to_meta(toks: &[Tok<'_>]) -> Vec<AlignMeta> {
+    toks.iter().map(|t| AlignMeta { kind: t.kind, off: t.off, end: t.end, nl: t.nl }).collect()
+}
+fn compute_align(old_text: &str, old_toks: &[AlignMeta], new_text: &str, new_toks: &[AlignMeta]) -> Align {
+    let old_n = old_toks.len();
+    let new_n = new_toks.len();
+    let mut prefix = 0usize;
+    while prefix < old_n && prefix < new_n {
+        let o = &old_toks[prefix];
+        let n = &new_toks[prefix];
+        if o.kind != n.kind || o.off != n.off || o.end != n.end || o.nl != n.nl { break; }
+        if old_text[o.off..o.end] != new_text[n.off..n.end] { break; }
+        prefix += 1;
+    }
+    let delta = new_text.len() as isize - old_text.len() as isize;
+    let min_n = old_n.min(new_n);
+    let mut suffix = 0usize;
+    while prefix + suffix < min_n {
+        let o = &old_toks[old_n - 1 - suffix];
+        let n = &new_toks[new_n - 1 - suffix];
+        if o.kind != n.kind || o.nl != n.nl { break; }
+        if n.off != (o.off as isize + delta) as usize || n.end != (o.end as isize + delta) as usize { break; }
+        if old_text[o.off..o.end] != new_text[n.off..n.end] { break; }
+        suffix += 1;
+    }
+    Align { old_n, new_n, prefix, suffix }
+}
+fn toks_from_meta<'a>(text: &'a str, meta: &[AlignMeta]) -> Vec<Tok<'a>> {
+    meta.iter().map(|m| Tok { kind: m.kind, text: &text[m.off..m.end], off: m.off, end: m.end, nl: m.nl }).collect()
+}
+pub struct Doc { text: String, toks: Vec<AlignMeta>, align: Option<Align> }
 impl Doc {
-    pub fn new(text: String) -> Doc { Doc { text } }
+    pub fn new(text: String) -> Doc { Doc { text: text.clone(), toks: to_meta(&lex(&text)), align: None } }
     pub fn text(&self) -> &str { &self.text }
+    pub fn alignment(&self) -> Option<&Align> { self.align.as_ref() }
     pub fn edit(&mut self, edits: &[Edit]) {
-        for e in edits { let s = e.start; let en = e.end; self.text = format!("{}{}{}", &self.text[..s], e.text, &self.text[en..]); }
+        let old_text = self.text.clone();
+        let old_toks = self.toks.clone();
+        for e in edits {
+            let n = self.text.len();
+            let start = e.start.min(n);
+            let end = e.end.max(start).min(n);
+            self.text = format!("{}{}{}", &self.text[..start], e.text, &self.text[end..]);
+        }
+        self.toks = to_meta(&lex(&self.text));
+        self.align = Some(compute_align(&old_text, &old_toks, &self.text, &self.toks));
+    }
+    pub fn parse(&self) -> Option<(Parser<'_>, i32)> {
+        let toks = toks_from_meta(&self.text, &self.toks);
+        let n = toks.len();
+        let mut p = Parser { toks, pos: 0, capped: false, suppress_next: Vec::new(), suppress_cur: Vec::new(), src: &self.text, nodes: Vec::new(), kids: Vec::new(), scratch: Vec::new() };
+        match p.parse_${ir.entry}() {
+            Some(root) if p.pos == n => Some((p, root)),
+            _ => None,
+        }
     }
-    pub fn parse(&self) -> Option<(Parser<'_>, i32)> { parse(tokenize(&self.text)) }
 }
 `;
   },
@@ -665,8 +716,13 @@ fn main() {
             let edits: Vec<Edit> = batch.iter().map(|&(s, e, ref t)| Edit { start: s, end: e, text: t.clone() }).collect();
             doc.edit(&edits);
         }
+        if let Some(a) = doc.alignment() {
+            eprintln!("{{\\"oldN\\":{},\\"newN\\":{},\\"prefix\\":{},\\"suffix\\":{}}}", a.old_n, a.new_n, a.prefix, a.suffix);
+        }
         match doc.parse() {
-            Some((p, root)) => { let mut out = String::new(); write_json(&p, root, &mut out); print!("{}", out); }
+            Some((p, root)) => {
+                let mut out = String::new(); write_json(&p, root, &mut out); print!("{}", out);
+            }
             None => { eprintln!("parse error"); std::process::exit(1); }
         }
         return;

src/target-ts.ts

@@ -473,15 +473,48 @@ export function parse(tokens: Tok[]): Cst | null {
 }
 
 export type Edit = { start: number; end: number; text: string };
-export function createDoc(src: string): { text(): string; root(): Node | null; edit(edits: Edit[]): Node | null } {
+type AlignMeta = { kind: string; off: number; end: number; nl: boolean };
+type Align = { oldN: number; newN: number; prefix: number; suffix: number };
+const toMeta = (toks: Tok[]): AlignMeta[] => toks.map((t) => ({ kind: t.kind, off: t.off, end: t.end, nl: t.nl }));
+function computeAlign(oldText: string, oldToks: AlignMeta[], newText: string, newToks: AlignMeta[]): Align {
+  const oldN = oldToks.length, newN = newToks.length;
+  let prefix = 0;
+  while (prefix < oldN && prefix < newN) {
+    const o = oldToks[prefix], n = newToks[prefix];
+    if (o.kind !== n.kind || o.off !== n.off || o.end !== n.end || o.nl !== n.nl) break;
+    if (oldText.slice(o.off, o.end) !== newText.slice(n.off, n.end)) break;
+    prefix++;
+  }
+  const delta = newText.length - oldText.length;
+  const minN = Math.min(oldN, newN);
+  let suffix = 0;
+  while (prefix + suffix < minN) {
+    const o = oldToks[oldN - 1 - suffix], n = newToks[newN - 1 - suffix];
+    if (o.kind !== n.kind || o.nl !== n.nl || n.off !== o.off + delta || n.end !== o.end + delta) break;
+    if (oldText.slice(o.off, o.end) !== newText.slice(n.off, n.end)) break;
+    suffix++;
+  }
+  return { oldN, newN, prefix, suffix };
+}
+export function createDoc(src: string): { text(): string; root(): Node | null; align(): Align | null; edit(edits: Edit[]): Node | null } {
   let text = src;
+  let prevToks = toMeta(tokenize(src));
+  let align: Align | null = null;
   let root: Node | null = parse(tokenize(src));
   return {
     text(): string { return text; },
     root(): Node | null { return root; },
+    align(): Align | null { return align; },
     edit(edits: Edit[]): Node | null {
-      for (const e of edits) text = text.slice(0, e.start) + e.text + text.slice(e.end);
-      root = parse(tokenize(text));
+      const oldText = text, oldToks = prevToks;
+      for (const e of edits) {
+        const n = text.length, start = Math.max(0, Math.min(e.start, n)), end = Math.max(start, Math.min(e.end, n));
+        text = text.slice(0, start) + e.text + text.slice(end);
+      }
+      const newToks = tokenize(text);
+      prevToks = toMeta(newToks);
+      align = computeAlign(oldText, oldToks, text, prevToks);
+      root = parse(newToks);
       return root;
     },
   };
@@ -497,6 +530,8 @@ if (process.argv.includes('edit-session')) {
   const { init, batches } = JSON.parse(_raw) as { init: string; batches: [number, number, string][][] };
   const doc = createDoc(init);
   for (const batch of batches) doc.edit(batch.map(([start, end, text]) => ({ start, end, text })));
+  const a = doc.align();
+  if (a) process.stderr.write(JSON.stringify(a) + '\\n');
   const root = doc.root();
   if (root === null) { process.stderr.write('parse error\\n'); process.exit(1); }
   process.stdout.write(JSON.stringify(root));

test/portable-targets.ts

@@ -17,9 +17,10 @@
 //
 // Go/Rust toolchains are optional: a missing `go`/`rustc` is logged and skipped (the TS
 // rendering, which needs only node, always runs).
-import { execFileSync } from 'node:child_process';
+import { execFileSync, spawnSync } from 'node:child_process';
 import { mkdirSync, rmSync, writeFileSync } from 'node:fs';
 import { createParser } from '../src/gen-parser.ts';
+import { createLexer } from '../src/gen-lexer.ts';
 import { emitParser, tsTarget, goTarget, rustTarget } from '../src/emit.ts';
 import type { CstGrammar } from '../src/types.ts';
 
@@ -270,21 +271,75 @@ function runProc(cmd: string, args: string[], src: string): Outcome {
   try { return { ok: true, cst: canon(JSON.parse(execFileSync(cmd, args, { input: src, stdio: ['pipe', 'pipe', 'pipe'] }).toString())) }; }
   catch { return { ok: false }; }
 }
+type Align = { oldN: number; newN: number; prefix: number; suffix: number };
+type EditOutcome = Outcome & { align?: Align };
+function parseAlignStderr(stderr: string): Align | undefined {
+  for (const line of stderr.split('\n')) {
+    const t = line.trim();
+    if (!t.startsWith('{')) continue;
+    try { return JSON.parse(t) as Align; } catch { /* keep scanning */ }
+  }
+  return undefined;
+}
+function runEditSession(cmd: string, args: string[], json: string): EditOutcome {
+  const r = spawnSync(cmd, args, { input: json, encoding: 'utf8' });
+  const align = parseAlignStderr(r.stderr ?? '');
+  if (r.status !== 0) return { ok: false, align };
+  try {
+    return { ok: true, cst: canon(JSON.parse(r.stdout)), align };
+  } catch { return { ok: false, align }; }
+}
+
+type AlignMeta = { kind: string; off: number; end: number; nl: boolean };
+const oracleToks = (g: CstGrammar, src: string): AlignMeta[] => {
+  const { tokenize } = createLexer(g);
+  return tokenize(src).map((t) => ({ kind: t.type, off: t.offset, end: t.offset + t.text.length, nl: t.newlineBefore }));
+};
+const computeAlign = (oldText: string, oldToks: AlignMeta[], newText: string, newToks: AlignMeta[]): Align => {
+  const oldN = oldToks.length, newN = newToks.length;
+  let prefix = 0;
+  while (prefix < oldN && prefix < newN) {
+    const o = oldToks[prefix], n = newToks[prefix];
+    if (o.kind !== n.kind || o.off !== n.off || o.end !== n.end || o.nl !== n.nl) break;
+    if (oldText.slice(o.off, o.end) !== newText.slice(n.off, n.end)) break;
+    prefix++;
+  }
+  const delta = newText.length - oldText.length;
+  const minN = Math.min(oldN, newN);
+  let suffix = 0;
+  while (prefix + suffix < minN) {
+    const o = oldToks[oldN - 1 - suffix], n = newToks[newN - 1 - suffix];
+    if (o.kind !== n.kind || o.nl !== n.nl || n.off !== o.off + delta || n.end !== o.end + delta) break;
+    if (oldText.slice(o.off, o.end) !== newText.slice(n.off, n.end)) break;
+    suffix++;
+  }
+  return { oldN, newN, prefix, suffix };
+};
+const expectAlign = (g: CstGrammar, init: string, batches: EditBatch[]): Align => {
+  let text = init;
+  for (let i = 0; i < batches.length - 1; i++) for (const [s, e, r] of batches[i]) text = text.slice(0, s) + r + text.slice(e);
+  const oldText = text;
+  const batch = batches[batches.length - 1];
+  for (const [s, e, r] of batch) text = text.slice(0, s) + r + text.slice(e);
+  return computeAlign(oldText, oracleToks(g, oldText), text, oracleToks(g, text));
+};
 
 type EditBatch = [number, number, string][];
-type EditScenario = { init: string; batches: EditBatch[] };
+type EditScenario = { init: string; batches: EditBatch[]; large?: boolean };
 const EDIT_SCENARIOS: Record<string, EditScenario[]> = {
   calc: [
     { init: '1+2*3', batches: [[[3, 3, '4']]] },
     { init: '1+2*3', batches: [[[1, 3, '']]] },
     { init: '1+2*3', batches: [[[2, 5, '(7-8)']]] },
     { init: '1+2*3', batches: [[[0, 0, '9-']], [[7, 8, '']]] },
+    { init: '1+2*3+'.repeat(199) + '1+2*3', batches: [[[600, 601, '9']]], large: true },
   ],
   javascript: [
     { init: 'let a = 1;\nf(a);', batches: [[[8, 9, '42']]] },
     { init: 'let a = 1;\nf(a);', batches: [[[11, 16, '']]] },
     { init: 'let a = 1;\nf(a);', batches: [[[4, 5, 'b'], [12, 13, 'b']]] },
     { init: 'let a = 1;\nf(a);', batches: [[[16, 16, '\ng(b);']], [[0, 4, 'var']]] },
+    { init: 'let a = 1;\nf(a);\n'.repeat(80), batches: [[[688, 689, '9']]], large: true },
   ],
 };
 const applyEdits = (init: string, batches: EditBatch[]): string => {
@@ -358,12 +413,10 @@ for (const c of CASES) {
 
     const editScenarios = EDIT_SCENARIOS[c.grammar];
     if (editScenarios) {
-      let editOk = 0;
-      const runEdit = (json: string) => runProc(
-        r.label === 'typescript' ? 'node' : r.label === 'go' ? `${dir}/go/p` : `${dir}/pr`,
-        r.label === 'typescript' ? [`${dir}/p.ts`, 'edit-session'] : ['edit-session'],
-        json,
-      );
+      let editOk = 0, alignOk = 0;
+      const editCmd = r.label === 'typescript' ? 'node' : r.label === 'go' ? `${dir}/go/p` : `${dir}/pr`;
+      const editArgs = r.label === 'typescript' ? [`${dir}/p.ts`, 'edit-session'] : ['edit-session'];
+      const runEdit = (json: string) => runEditSession(editCmd, editArgs, json);
       for (const sc of editScenarios) {
         const final = applyEdits(sc.init, sc.batches);
         const a = runEdit(JSON.stringify({ init: sc.init, batches: sc.batches }));
@@ -373,8 +426,20 @@ for (const c of CASES) {
           failures++;
           console.log(`  ${c.grammar}/${r.label}: edit-session mismatch (final=${JSON.stringify(final)}) A ok=${a.ok} B ok=${b.ok}`);
         }
+        const want = expectAlign(grammar, sc.init, sc.batches);
+        if (a.align && a.align.oldN === want.oldN && a.align.newN === want.newN && a.align.prefix === want.prefix && a.align.suffix === want.suffix) {
+          alignOk++;
+          if (sc.large && want.prefix + want.suffix < want.oldN - 8) {
+            failures++;
+            console.log(`  ${c.grammar}/${r.label}: large-doc align too narrow prefix+suffix=${want.prefix + want.suffix} oldN=${want.oldN}`);
+          }
+        } else {
+          failures++;
+          console.log(`  ${c.grammar}/${r.label}: token-align mismatch want=${JSON.stringify(want)} got=${JSON.stringify(a.align)}`);
+        }
       }
       console.log(`  ${c.grammar}/${r.label}: ${editOk}/${editScenarios.length} edit-sessions ≡ fresh`);
+      console.log(`  ${c.grammar}/${r.label}: ${alignOk}/${editScenarios.length} token-alignments ≡ oracle`);
     }
   }
 }