Remove toObject from the engine: visit + tree accessors are the only surface

The arena design's premise (PR #36) is that parse() hands out a tree to
TRAVERSE, not an object tree to materialize - toObject was the
materialization back door left on both the module API and the handle
API, and its only real consumer was the gate layer's byte-identical
JSON comparison. That is a test concern: gates now build the comparison
object through visit + tree accessors (test/emitted-obj.ts, mirroring
the interpreter's object shape and key order exactly, so the emit ≡
interp and incremental ≡ fresh comparisons are unchanged). The unused
emitted getText went with it; the interpreter keeps returning its
native object trees (that IS its representation, not a conversion).

31/31 gates, emit-parser-verify 0 mismatches, multi-doc contract 5/5,
handle-API keystroke median 0.028ms (9MB) / 0.089ms (8MB nested).

Author: johnsoncodehk

src/emit-parser.ts

@@ -2247,10 +2247,6 @@ export function tokenAt(i) {
 }
 
 // The CST is span-only: a node's text is derived from the source it was parsed from.
-export function getText(node, source) {
-  return source.slice(node.offset, node.end);
-}
-
 // ── Arena tree access ──
 // The arena IS the tree: parse() returns the root node id and consumers traverse
 // via visit()/the accessors — nothing is materialized on the parse path. All views
@@ -2323,20 +2319,6 @@ function visitCore(entry, fns, charBase, tokBase) {
   }
   if (fns.leave) fns.leave(entry, charBase, tokBase);
 }
-// Materialize the classic object CST from a node id — a BRIDGE for tests/debugging
-// (the byte-identical gate against the interpreter), not a parse-path product.
-function toObjectCore(id, charBase, tokBase) {
-  if (charBase === undefined) { charBase = rootCharBase; tokBase = rootTokBase; }
-  const n = rowCount[id];
-  const cs = rowStart[id];
-  const children = new Array(n);
-  for (let i = 0; i < n; i++) {
-    const entry = kids[cs + i];
-    children[i] = entry >= 0 ? toObjectCore(entry, charBase + kcr(id, cs + i), tokBase + ktr(id, cs + i))
-      : { tokenType: leafTokenType(entry, tokBase), offset: toff(tokBase + ((~entry) >>> 2)), end: tend(tokBase + ((~entry) >>> 2)) };
-  }
-  return { rule: RULE_NAMES[rowRule[id]], children, offset: charBase, end: charBase + rowLen[id] };
-}
 
 // Parse to the ARENA: returns the root node id.
 function lexInto(source) {
@@ -3159,7 +3141,6 @@ export { tokenize };
 export function parse(source, entryRule) { activate(docDefault); return parseCore(source, entryRule); }
 export function parseEdited(source, entryRule, edits) { activate(docDefault); return editCore(source, entryRule, edits); }
 export function visit(entry, fns, charBase, tokBase) { activate(docDefault); return visitCore(entry, fns, charBase, tokBase); }
-export function toObject(id, charBase, tokBase) { activate(docDefault); return toObjectCore(id, charBase, tokBase); }
 // ── Handle API: explicit trees over per-instance documents ──
 // const p = createParser(); const cst = p.parse(text); p.edit(cst, next[, edits]);
 // The handle is the STABLE IDENTITY of this document's tree: edit() mutates it in
@@ -3194,7 +3175,6 @@ export function createParser() {
       cst.root = editCore(source, entryUsed, edits);
     },
     visit(cst, fns) { chk(cst); activate(d); return visitCore(cst.root, fns); },
-    toObject(cst) { chk(cst); activate(d); return toObjectCore(cst.root); },
     tree: view,
   };
 }

test/emit-parser-verify.ts

@@ -9,6 +9,7 @@
 //   node test/emit-parser-verify.ts            # 4 bench files + ~400-file corpus sample
 //   node test/emit-parser-verify.ts <N>        # sample stride N (default ~ to hit ~400)
 //   node test/emit-parser-verify.ts all        # every .ts file under conformance
+import { objectify } from './emitted-obj.ts';
 import { createParser } from '../src/gen-parser.ts';
 import { emitParser } from '../src/emit-parser.ts';
 import { readdir } from 'fs/promises';
@@ -41,7 +42,7 @@ function compare(code: string): { verdict: string; detail?: string } {
   const o = run(oracle.parse, code);
   // The emitted parser returns an arena node id; materialize the object view for the
   // byte-identical comparison against the interpreter's object tree.
-  const e = run((s: string) => emitted.toObject(emitted.parse(s)), code);
+  const e = run((s: string) => { const r = emitted.parse(s); return objectify(emitted.tree, (fns) => emitted.visit(r, fns)); }, code);
   if (!o.ok && o.err.includes('Maximum call stack')) {
     // The interpreter recursed out of stack — a CAPACITY limit, not a parse verdict;
     // the emitted parser's flatter frames can legitimately survive deeper inputs

test/emitted-obj.ts

@@ -0,0 +1,39 @@
+// Materialize an emitted-engine tree as a plain object — TEST-SIDE ONLY. The engine
+// deliberately exposes a single consumption surface (visit + tree accessors); full
+// materialization is a consumer choice, and the only consumer that needs it is the
+// gate layer's byte-identical JSON comparison (incremental ≡ fresh, emit ≡ interp).
+// The shape (and KEY ORDER — JSON.stringify equality depends on it) mirrors the
+// interpreter's native object trees: nodes { rule, children, offset, end }, leaves
+// { tokenType, offset, end }.
+export interface TreeView {
+  ruleNameOf(id: number): string;
+  lenOf(id: number): number;
+  leafTokenType(entry: number, tokBase: number): string;
+  leafOffsetOf(entry: number, tokBase: number): number;
+  leafEndOf(entry: number, tokBase: number): number;
+}
+type VisitFns = {
+  enter?(id: number, charBase: number, tokBase: number): boolean | void;
+  leave?(id: number, charBase: number, tokBase: number): void;
+  leaf?(entry: number, tok: number): void;
+};
+export type ObjNode = { rule: string; children: (ObjNode | ObjLeaf)[]; offset: number; end: number };
+export type ObjLeaf = { tokenType: string; offset: number; end: number };
+
+export function objectify(tree: TreeView, runVisit: (fns: VisitFns) => void): ObjNode {
+  const rootHolder: { children: (ObjNode | ObjLeaf)[] } = { children: [] };
+  const stack: { children: (ObjNode | ObjLeaf)[] }[] = [rootHolder];
+  runVisit({
+    enter(id, charBase) {
+      const node: ObjNode = { rule: tree.ruleNameOf(id), children: [], offset: charBase, end: charBase + tree.lenOf(id) };
+      stack[stack.length - 1].children.push(node);
+      stack.push(node);
+    },
+    leave() { stack.pop(); },
+    leaf(entry, tok) {
+      const tb = tok - ((~entry) >>> 2);
+      stack[stack.length - 1].children.push({ tokenType: tree.leafTokenType(entry, tb), offset: tree.leafOffsetOf(entry, tb), end: tree.leafEndOf(entry, tb) });
+    },
+  });
+  return rootHolder.children[0] as ObjNode;
+}

test/incremental-verify.ts

@@ -6,6 +6,7 @@
 // and the arena growth, so reuse is MEASURED, not assumed.
 //
 //   node test/incremental-verify.ts
+import { objectify } from './emitted-obj.ts';
 import { existsSync, readFileSync, writeFileSync } from 'node:fs';
 import { emitParser } from '../src/emit-parser.ts';
 
@@ -17,11 +18,13 @@ type Cst = { root: number };
 type Parser = {
   parse(s: string): Cst;
   edit(cst: Cst, s: string, edits?: Edit[]): void;
-  toObject(cst: Cst): unknown;
+  visit(cst: Cst, fns: object): void;
+  tree: import('./emitted-obj.ts').TreeView;
 };
 type Em = {
   parse(s: string): number;
-  toObject(id: number): unknown;
+  visit(entry: number, fns: object): void;
+  tree: import('./emitted-obj.ts').TreeView;
   createParser(): Parser;
 };
 const session = ((await import(emPath + '?session=' + process.pid)) as Em).createParser();
@@ -109,8 +112,8 @@ for (const [base, edited] of GLUE) {
     else bothReject++;
     continue;
   }
-  const a = JSON.stringify(fresh.toObject(fr));
-  const b = JSON.stringify(session.toObject(c0));
+  const a = JSON.stringify(objectify(fresh.tree, (fns) => fresh.visit(fr, fns)));
+  const b = JSON.stringify(objectify(session.tree, (fns) => session.visit(c0, fns)));
   if (a === b) equal++;
   else { mismatch++; if (failures.length < 5) failures.push(`glue «${edited.slice(0, 30)}»: tree diverges`); }
 }
@@ -139,8 +142,8 @@ for (const f of FILES) {
       continue;
     }
     tFresh += tf1 - tf0; tInc += ti1 - ti0;
-    const a = JSON.stringify(fresh.toObject(freshRoot));
-    const b = JSON.stringify(session.toObject(cst));
+    const a = JSON.stringify(objectify(fresh.tree, (fns) => fresh.visit(freshRoot, fns)));
+    const b = JSON.stringify(objectify(session.tree, (fns) => session.visit(cst, fns)));
     if (a === b) equal++;
     else {
       mismatch++;

test/multi-doc.ts

@@ -8,6 +8,7 @@
 // an in-place-mutated tree, and a REJECTED edit leaves the old handle valid.
 //
 //   node test/multi-doc.ts
+import { objectify } from './emitted-obj.ts';
 import { writeFileSync } from 'node:fs';
 import { emitParser } from '../src/emit-parser.ts';
 
@@ -16,8 +17,8 @@ const emPath = '/tmp/emitted-multidoc.mjs';
 writeFileSync(emPath, emitParser(grammar));
 type Edit = { start: number; oldEnd: number; newEnd: number };
 type Cst = { root: number };
-type Parser = { parse(s: string): Cst; edit(cst: Cst, s: string, edits?: Edit[]): void; toObject(cst: Cst): unknown; visit(cst: Cst, fns: object): void };
-type Em = { parse(s: string): number; toObject(id: number): unknown; createParser(): Parser };
+type Parser = { parse(s: string): Cst; edit(cst: Cst, s: string, edits?: Edit[]): void; visit(cst: Cst, fns: object): void; tree: import('./emitted-obj.ts').TreeView };
+type Em = { parse(s: string): number; createParser(): Parser };
 const em = (await import(emPath + '?v=' + process.pid)) as Em;
 
 // Two synthetic documents (no corpus dependency — the gate always exercises).
@@ -65,8 +66,9 @@ for (let k = 0; k < 60; k++) {
   }
   // mix the module-level default doc in between: it must not disturb either instance
   if (k % 5 === 0) em.parse('const mix = ' + k + ';');
-  const a = JSON.stringify(f.toObject(fc!));
-  const b = JSON.stringify((onA ? p1 : p2).toObject(onA ? cstA : cstB));
+  const a = JSON.stringify(objectify(f.tree, (fns) => f.visit(fc!, fns)));
+  const q = onA ? p1 : p2;
+  const b = JSON.stringify(objectify(q.tree, (fns) => q.visit(onA ? cstA : cstB, fns)));
   if (a === b) equal++;
   else {
     mismatch++;
@@ -84,22 +86,23 @@ let contract = 0;
 {
   const p = em.createParser();
   const c1 = p.parse('const a = 1;');
-  const before = JSON.stringify(p.toObject(c1));
+  const obj = (h: Cst) => JSON.stringify(objectify(p.tree, (fns) => p.visit(h, fns)));
+  const before = obj(c1);
   p.edit(c1, 'const ab = 1;');
-  const after = JSON.stringify(p.toObject(c1));
+  const after = obj(c1);
   if (after !== before && after.includes('"end":8')) contract++;   // same handle, new tree
   else failures.push('in-place edit did not update the handle');
   try { p2.edit(c1, 'const y = 3;'); failures.push('foreign handle did not throw'); } catch { contract++; }
   let rejected = false;
   try { p.edit(c1, 'const ] = ;'); } catch { rejected = true; }
-  if (rejected && JSON.stringify(p.toObject(c1)) === after) contract++;   // reject keeps the tree
+  if (rejected && obj(c1) === after) contract++;   // reject keeps the tree
   else failures.push('reject-then-read flow broke');
   const c2 = p.parse('let q = 1;');
-  try { p.toObject(c1); failures.push('re-opened document: old handle did not throw'); } catch { contract++; }
+  try { obj(c1); failures.push('re-opened document: old handle did not throw'); } catch { contract++; }
   // a REJECTING parse() resets the arena too — it must invalidate prior handles
   try { p.parse('const ] = ;'); } catch { /* expected reject */ }
   let dead = false;
-  try { p.toObject(c2); } catch { dead = true; }
+  try { obj(c2); } catch { dead = true; }
   if (dead) contract++;
   else failures.push('rejecting parse() left the old handle readable over a reset arena');
 }