CST leaves are span-only: drop the text field from the contract

A leaf is now {kind, tokenType, offset, end}. Its text was redundant data —
text === source.slice(offset, end) held everywhere once the yaml flow-merge
anomaly was fixed (previous commit), and the cst-text-invariant gate now pins
the span-only shape itself (no text property, sane spans; 43,999 leaves across
all seven grammars + a TS corpus sample). Consumers derive text from the
source they parsed: both engines export getText(node, source), and the
in-repo consumers (html-conformance tree extraction, the generative net's
leafRoles, gap-ledger probes) take the input alongside the CST.

Both engines change together: matchers and the pratt operator paths stop
materializing text (matchKwLit/matchPuLit lose their value parameter — the
leaf no longer needs it), the no-unary-LHS head check and the markup
open/close tag-name comparisons slice the source span, and the generated
*.cst-types.ts leaf interface drops the field. interp ≡ emit holds on the
full 18,805-file corpus; lexer streams and reject messages identical; 27/27
gates.

Side effect on the PR#4 bench: dropping ~10k leaf-text slices per parse and
shrinking every leaf by a field measures +2.1..+14.7% aggregate, 6/8 runs
positive (mean ~+7%).

Author: johnsoncodehk

html.cst-types.ts

@@ -32,7 +32,6 @@ export type TokenType =
 export interface CstLeaf extends CstPos {
   kind: 'leaf';
   tokenType: TokenType;
-  text: string;
 }
 
 /** `Element` node. Children (flattened, in source order) are drawn from: */

javascript.cst-types.ts

@@ -41,7 +41,6 @@ export type TokenType =
 export interface CstLeaf extends CstPos {
   kind: 'leaf';
   tokenType: TokenType;
-  text: string;
 }
 
 /** Synthetic node the parser builds for an interpolated template literal. */

javascriptreact.cst-types.ts

@@ -43,7 +43,6 @@ export type TokenType =
 export interface CstLeaf extends CstPos {
   kind: 'leaf';
   tokenType: TokenType;
-  text: string;
 }
 
 /** Synthetic node the parser builds for an interpolated template literal. */

src/emit-parser.ts

@@ -1127,8 +1127,8 @@ class Emitter {
   // with the value's baked int (so the runtime does int compares, not string work).
   matchLiteralCall(value: string): string {
     const d = this.a.symtab.classifyKey(value);
-    if (d.kind === 'kw') return `matchKwLit(${J(value)}, ${d.t})`;
-    if (d.kind === 'punct') return value === '>' ? `matchPuLitGT(${d.t})` : `matchPuLit(${J(value)}, ${d.t})`;
+    if (d.kind === 'kw') return `matchKwLit(${d.t})`;
+    if (d.kind === 'punct') return value === '>' ? `matchPuLitGT(${d.t})` : `matchPuLit(${d.t})`;
     // A literal key that classifies as a token-name (a token name used as a literal):
     // unreachable for real grammars, but stay safe via the generic matchLiteral.
     return `matchLiteral(${J(value)})`;
@@ -1288,8 +1288,6 @@ function resolveLexerImport(): string { return pathResolve(__dir, 'gen-lexer.ts'
 // ONLY change: where the interpreter called matchExpr(alt)/matchSeq(items) per arm,
 // these call the GENERATED per-arm matcher functions (installed via the rule fns).
 function emitRuntime(e: Emitter) {
-  // Per-site text materialization (soa: slice the source span; fallback: text column).
-  const TXT_OE = e.soa ? 'src.slice(off, end)' : 'tkText[pos]';
   // Column element type: Uint8 when the kind/literal id spaces fit a byte.
   const st = e.a.symtab;
   let tMax = 1;
@@ -1346,34 +1344,34 @@ function childEnd(c) { return c.end; }
 // && tok.text === value. With interned kinds that is tok.k >= K_NAMED_MIN (a declared
 // token name; '' is PUNCT, templates are below NAMED_MIN) && tok.t === KW(value).
 // Returns the SAME $keyword leaf as before. value/kw are baked by the caller.
-function matchKwLit(value, kw) {
+function matchKwLit(kw) {
   // A kw-range t can only come from a named token (template spans never intern to a
   // keyword), so the old k >= K_NAMED_MIN guard was redundant — one int compare.
   if (pos >= cap || tkT[pos] !== kw) return null;
   const off = tkOff[pos];
   const end = tkEnd[pos];
   if (++pos > maxPos) maxPos = pos;
-  return { kind: 'leaf', tokenType: '$keyword', text: value, offset: off, end };
+  return { kind: 'leaf', tokenType: '$keyword', offset: off, end };
 }
 // Punct literal: tok.type === '' && tok.text === value, with the gt-splice fallback.
 // tok.t === PU(value) is the exact-text fast path; the splice handles a longer
 // gt-led token matching the gt key. value/pu are baked by the caller.
-function matchPuLit(value, pu) {
+function matchPuLit(pu) {
   // A pu-range t can only come from a punct token, so the old k === K_PUNCT guard was
   // redundant — one int compare. The '>'-split lives only in matchPuLitGT ('>' sites).
   if (pos >= cap || tkT[pos] !== pu) return null;
   const off = tkOff[pos];
   const end = tkEnd[pos];
   if (++pos > maxPos) maxPos = pos;
-  return { kind: 'leaf', tokenType: '$punct', text: value, offset: off, end };
+  return { kind: 'leaf', tokenType: '$punct', offset: off, end };
 }
 function matchPuLitGT(pu) {
   if (pos >= cap) return null;
   const off = tkOff[pos];
   if (tkT[pos] === pu) {
     const end = tkEnd[pos];
     if (++pos > maxPos) maxPos = pos;
-    return { kind: 'leaf', tokenType: '$punct', text: '>', offset: off, end };
+    return { kind: 'leaf', tokenType: '$punct', offset: off, end };
   }
   // Split multi-'>' tokens: '>>', '>>>', '>>=', '>>>=' can yield a single '>': shift the
   // columns up one slot and write the '>' + rest pair in place (both born flag-less,
@@ -1397,17 +1395,17 @@ function matchPuLitGT(pu) {
     memoNode.fill(undefined);
     memoEnd.fill(undefined);
     if (++pos > maxPos) maxPos = pos;
-    return { kind: 'leaf', tokenType: '$punct', text: '>', offset: off, end: off + 1 };
+    return { kind: 'leaf', tokenType: '$punct', offset: off, end: off + 1 };
   }
   return null;
 }
 // Generic matchLiteral kept for any unspecialized site: classify value via the baked
 // tables (no per-call isKeywordLiteral / string compares) and delegate.
 function matchLiteral(value) {
   const kw = LIT_KW.get(value);
-  if (kw !== undefined) return matchKwLit(value, kw);
+  if (kw !== undefined) return matchKwLit(kw);
   if (value === '>') return matchPuLitGT(LIT_PU.get(value) ?? 0);
-  return matchPuLit(value, LIT_PU.get(value) ?? 0);
+  return matchPuLit(LIT_PU.get(value) ?? 0);
 }
 
 // Match a token ref by its baked TYPE kind: tok.type === name  ⟺  tok.k === nameKind.
@@ -1416,9 +1414,8 @@ function matchTokK(name, nameKind) {
   if (pos >= cap || tkK[pos] !== nameKind) return null;
   const off = tkOff[pos];
   const end = tkEnd[pos];
-  const text = ${TXT_OE};
   if (++pos > maxPos) maxPos = pos;
-  return { kind: 'leaf', tokenType: name, text, offset: off, end };
+  return { kind: 'leaf', tokenType: name, offset: off, end };
 }
 
 // (First-token / FIRST-set gating is baked at emit time: per-set _qN byte-table fns
@@ -1428,31 +1425,31 @@ function parseTemplateExpr() {
   if (pos >= cap) return null;
   const k = tkK[pos];
   if (k === K_TPL_TOKEN) {
-    const off = tkOff[pos]; const end = tkEnd[pos]; const text = ${TXT_OE};
+    const off = tkOff[pos]; const end = tkEnd[pos];
     if (++pos > maxPos) maxPos = pos;
-    return { kind: 'leaf', tokenType: templateTokenName, text, offset: off, end };
+    return { kind: 'leaf', tokenType: templateTokenName, offset: off, end };
   }
   if (k === K_TEMPLATE_HEAD) {
     const children = [];
-    { const off = tkOff[pos]; const end = tkEnd[pos]; const text = ${TXT_OE};
+    { const off = tkOff[pos]; const end = tkEnd[pos];
       if (++pos > maxPos) maxPos = pos;
-      children.push({ kind: 'leaf', tokenType: '$templateHead', text, offset: off, end }); }
+      children.push({ kind: 'leaf', tokenType: '$templateHead', offset: off, end }); }
     const interpRule = currentPrattContext ?? EXPR_RULE;
     while (true) {
       const exprNode = RULES[interpRule]();
       if (exprNode) children.push(exprNode);
       if (pos >= cap) break;
       const nk = tkK[pos];
       if (nk === K_TEMPLATE_MIDDLE) {
-        const off = tkOff[pos]; const end = tkEnd[pos]; const text = ${TXT_OE};
+        const off = tkOff[pos]; const end = tkEnd[pos];
         if (++pos > maxPos) maxPos = pos;
-        children.push({ kind: 'leaf', tokenType: '$templateMiddle', text, offset: off, end });
+        children.push({ kind: 'leaf', tokenType: '$templateMiddle', offset: off, end });
         continue;
       }
       if (nk === K_TEMPLATE_TAIL) {
-        const off = tkOff[pos]; const end = tkEnd[pos]; const text = ${TXT_OE};
+        const off = tkOff[pos]; const end = tkEnd[pos];
         if (++pos > maxPos) maxPos = pos;
-        children.push({ kind: 'leaf', tokenType: '$templateTail', text, offset: off, end });
+        children.push({ kind: 'leaf', tokenType: '$templateTail', offset: off, end });
         break;
       }
       break;
@@ -1589,9 +1586,9 @@ function emitPrattRule(e: Emitter, a: ReturnType<typeof analyze>, rule: RuleDecl
       e.emit(`    if (pos < cap) {`);
       e.emit(`      const info = PREFIX_BY_T[tkT[pos]];`);
       e.emit(`      if (info) {`);
-      e.emit(`        const _o = tkOff[pos]; const _e = tkEnd[pos]; const _tx = ${e.textAt('pos')};`);
+      e.emit(`        const _o = tkOff[pos]; const _e = tkEnd[pos];`);
       e.emit(`        if (++pos > maxPos) maxPos = pos;`);
-      e.emit(`        const opLeaf = { kind: 'leaf', tokenType: '$operator', text: _tx, offset: _o, end: _e };`);
+      e.emit(`        const opLeaf = { kind: 'leaf', tokenType: '$operator', offset: _o, end: _e };`);
       e.emit(`        const rhs = ${ruleFn}_pratt(info.rbp);`);
       e.emit(`        if (rhs && pos > bestNudPos) { lhs = { kind: 'node', rule: ${J(rule.name)}, children: [opLeaf, rhs], offset: opLeaf.offset, end: rhs.end }; bestNudPos = pos; }`);
       e.emit(`      }`);
@@ -1656,20 +1653,20 @@ function emitPrattRule(e: Emitter, a: ReturnType<typeof analyze>, rule: RuleDecl
   e.emit(`    if (info && info.lbp > minBp) {`);
   e.emit(`      if (info.position === 'postfix') {`);
   e.emit(`        if (!tailClosed) {`);
-  e.emit(`          const _o = tkOff[pos]; const _e = tkEnd[pos]; const _tx = ${e.textAt('pos')};`);
+  e.emit(`          const _o = tkOff[pos]; const _e = tkEnd[pos];`);
   e.emit(`          if (++pos > maxPos) maxPos = pos;`);
-  e.emit(`          const opLeaf = { kind: 'leaf', tokenType: '$operator', text: _tx, offset: _o, end: _e };`);
+  e.emit(`          const opLeaf = { kind: 'leaf', tokenType: '$operator', offset: _o, end: _e };`);
   e.emit(`          lhs = { kind: 'node', rule: ${J(rule.name)}, children: [lhs, opLeaf], offset: lhs.offset, end: opLeaf.end };`);
   e.emit(`          tailClosed = true; matched = true;`);
   e.emit(`        }`);
   e.emit(`      } else {`);
   e.emit(`        if (NOUNARY_T[tkT[pos]] !== 0 && lhs.kind === 'node') {`);
   e.emit(`          const head = lhs.children[0];`);
-  e.emit(`          if (head && head.kind === 'leaf' && head.tokenType === '$operator' && prefixOps.has(head.text) && !postfixOpValues.has(head.text)) { return null; }`);
+  e.emit(`          if (head && head.kind === 'leaf' && head.tokenType === '$operator' && prefixOps.has(src.slice(head.offset, head.end)) && !postfixOpValues.has(src.slice(head.offset, head.end))) { return null; }`);
   e.emit(`        }`);
-  e.emit(`        const _o = tkOff[pos]; const _e = tkEnd[pos]; const _tx = ${e.textAt('pos')};`);
+  e.emit(`        const _o = tkOff[pos]; const _e = tkEnd[pos];`);
   e.emit(`        if (++pos > maxPos) maxPos = pos;`);
-  e.emit(`        const opLeaf = { kind: 'leaf', tokenType: '$operator', text: _tx, offset: _o, end: _e };`);
+  e.emit(`        const opLeaf = { kind: 'leaf', tokenType: '$operator', offset: _o, end: _e };`);
   e.emit(`        const rhs = ${ruleFn}_pratt(info.rbp);`);
   e.emit(`        if (rhs) { lhs = { kind: 'node', rule: ${J(rule.name)}, children: [lhs, opLeaf, rhs], offset: lhs.offset, end: rhs.end }; matched = true; }`);
   e.emit(`        else { pos = ledSaved; }`);
@@ -1829,6 +1826,11 @@ export function tokenAt(i) {
   };
 }
 
+// The CST is span-only: a node's text is derived from the source it was parsed from.
+export function getText(node, source) {
+  return source.slice(node.offset, node.end);
+}
+
 export function parse(source, entryRule) {
 ${e.soa ? `  tokenize(source);` : String.raw`  src = source;
   const _toks = tokenize(source);

src/gen-ast-types.ts

@@ -97,7 +97,6 @@ export function generateAstTypes(grammar: CstGrammar): string {
   lines.push('export interface CstLeaf extends CstPos {');
   lines.push('  kind: \'leaf\';');
   lines.push('  tokenType: TokenType;');
-  lines.push('  text: string;');
   lines.push('}');
   lines.push('');
 

src/gen-parser.ts

@@ -15,7 +15,6 @@ export interface CstNode {
 export interface CstLeaf {
   kind: 'leaf';
   tokenType: string;
-  text: string;
   offset: number;
   end: number;
 }
@@ -33,6 +32,11 @@ interface OpInfo {
 
 // ── Parser ──
 
+// The CST is span-only: a node's text is derived from the source it was parsed from.
+export function getText(node: { offset: number; end: number }, source: string): string {
+  return source.slice(node.offset, node.end);
+}
+
 export function createParser(grammar: CstGrammar) {
   const tokenNames = new Set(grammar.tokens.map(t => t.name));
 
@@ -760,14 +764,14 @@ export function createParser(grammar: CstGrammar) {
         // Keyword literal: match against Ident token with same text
         if (tok.type !== '' && tokenNames.has(tok.type) && tok.text === value) {
           pos++;
-          return { kind: 'leaf', tokenType: '$keyword', text: value, offset: tok.offset, end: tok.offset + tok.text.length };
+          return { kind: 'leaf', tokenType: '$keyword', offset: tok.offset, end: tok.offset + tok.text.length };
         }
         return null;
       }
       // Punctuation literal
       if (tok.type === '' && tok.text === value) {
         pos++;
-        return { kind: 'leaf', tokenType: '$punct', text: value, offset: tok.offset, end: tok.offset + tok.text.length };
+        return { kind: 'leaf', tokenType: '$punct', offset: tok.offset, end: tok.offset + tok.text.length };
       }
       // Split multi-`>` tokens: `>>`, `>>>`, `>>=`, `>>>=` can yield a single `>`
       if (value === '>' && tok.type === '' && tok.text.length > 1 && tok.text[0] === '>') {
@@ -778,7 +782,7 @@ export function createParser(grammar: CstGrammar) {
         );
         memo.clear();   // splice shifts later token indices → memo entries are stale
         pos++;
-        return { kind: 'leaf', tokenType: '$punct', text: '>', offset: tok.offset, end: tok.offset + 1 };
+        return { kind: 'leaf', tokenType: '$punct', offset: tok.offset, end: tok.offset + 1 };
       }
       return null;
     }
@@ -789,7 +793,7 @@ export function createParser(grammar: CstGrammar) {
       if (!tok) return null;
       if (tok.type === name) {
         pos++;
-        return { kind: 'leaf', tokenType: name, text: tok.text, offset: tok.offset, end: tok.offset + tok.text.length };
+        return { kind: 'leaf', tokenType: name, offset: tok.offset, end: tok.offset + tok.text.length };
       }
       return null;
     }
@@ -808,12 +812,12 @@ export function createParser(grammar: CstGrammar) {
       if (!tok) return null;
       if (tok.type === templateTokenName) {
         pos++;
-        return { kind: 'leaf', tokenType: templateTokenName, text: tok.text, offset: tok.offset, end: tok.offset + tok.text.length };
+        return { kind: 'leaf', tokenType: templateTokenName, offset: tok.offset, end: tok.offset + tok.text.length };
       }
       if (tok.type === '$templateHead') {
         const children: CstChild[] = [];
         pos++;
-        children.push({ kind: 'leaf', tokenType: '$templateHead', text: tok.text, offset: tok.offset, end: tok.offset + tok.text.length });
+        children.push({ kind: 'leaf', tokenType: '$templateHead', offset: tok.offset, end: tok.offset + tok.text.length });
         const interpRule = currentPrattContext ?? findExprRule();
         while (true) {
           const exprNode = parseRule(interpRule);
@@ -822,12 +826,12 @@ export function createParser(grammar: CstGrammar) {
           if (!next) break;
           if (next.type === '$templateMiddle') {
             pos++;
-            children.push({ kind: 'leaf', tokenType: '$templateMiddle', text: next.text, offset: next.offset, end: next.offset + next.text.length });
+            children.push({ kind: 'leaf', tokenType: '$templateMiddle', offset: next.offset, end: next.offset + next.text.length });
             continue;
           }
           if (next.type === '$templateTail') {
             pos++;
-            children.push({ kind: 'leaf', tokenType: '$templateTail', text: next.text, offset: next.offset, end: next.offset + next.text.length });
+            children.push({ kind: 'leaf', tokenType: '$templateTail', offset: next.offset, end: next.offset + next.text.length });
             break;
           }
           break;
@@ -1008,7 +1012,7 @@ export function createParser(grammar: CstGrammar) {
             const info = prefixOps.get(key);
             if (info) {
               pos++;
-              const opLeaf: CstLeaf = { kind: 'leaf', tokenType: '$operator', text: tok.text, offset: tok.offset, end: tok.offset + tok.text.length };
+              const opLeaf: CstLeaf = { kind: 'leaf', tokenType: '$operator', offset: tok.offset, end: tok.offset + tok.text.length };
               const rhs = parsePratt(rule, info.rbp);
               if (rhs && pos > bestNudPos) {
                 lhs = { kind: 'node', rule: rule.name, children: [opLeaf, rhs], offset: opLeaf.offset, end: rhs.end };
@@ -1086,7 +1090,7 @@ export function createParser(grammar: CstGrammar) {
           if (info.position === 'postfix') {
             if (!tailClosed) {                                   // can't postfix an update expr (`a++ --`)
               pos++;
-              const opLeaf: CstLeaf = { kind: 'leaf', tokenType: '$operator', text: tok.text, offset: tok.offset, end: tok.offset + tok.text.length };
+              const opLeaf: CstLeaf = { kind: 'leaf', tokenType: '$operator', offset: tok.offset, end: tok.offset + tok.text.length };
               lhs = { kind: 'node', rule: rule.name, children: [lhs, opLeaf], offset: lhs.offset, end: opLeaf.end };
               tailClosed = true;
               matched = true;
@@ -1102,12 +1106,12 @@ export function createParser(grammar: CstGrammar) {
             if (noUnaryLhsOps.has(tokKey) && lhs.kind === 'node') {
               const head = lhs.children[0];
               if (head?.kind === 'leaf' && head.tokenType === '$operator'
-                  && prefixOps.has(head.text) && !postfixOpValues.has(head.text)) {
+                  && prefixOps.has(source.slice(head.offset, head.end)) && !postfixOpValues.has(source.slice(head.offset, head.end))) {
                 return null;
               }
             }
             pos++;
-            const opLeaf: CstLeaf = { kind: 'leaf', tokenType: '$operator', text: tok.text, offset: tok.offset, end: tok.offset + tok.text.length };
+            const opLeaf: CstLeaf = { kind: 'leaf', tokenType: '$operator', offset: tok.offset, end: tok.offset + tok.text.length };
             const rhs = parsePratt(rule, info.rbp);
             if (rhs) {
               lhs = { kind: 'node', rule: rule.name, children: [lhs, opLeaf, rhs], offset: lhs.offset, end: rhs.end };
@@ -1243,7 +1247,7 @@ export function createParser(grammar: CstGrammar) {
         if (result === null) { pos = saved; return null; }
         // The open-tag name leaf (a name token) — drives the optional-end lookup.
         if (item.type === 'ref' && c.nameTokens.has(item.name) && result[0]?.kind === 'leaf') {
-          openName = result[0].text;
+          openName = source.slice(result[0].offset, result[0].end);
         }
         children.push(...result);
       }
@@ -1275,7 +1279,7 @@ export function createParser(grammar: CstGrammar) {
       const close = matchSeq(c.items.slice(c.closeStart));
       if (close !== null) {
         const closeName = close.find((ch, i) => i > 0 && ch.kind === 'leaf' && c.nameTokens.has((ch as CstLeaf).tokenType)) as CstLeaf | undefined;
-        if (!closeName || closeName.text.toLowerCase() === openName.toLowerCase()) {
+        if (!closeName || source.slice(closeName.offset, closeName.end).toLowerCase() === openName.toLowerCase()) {
           children.push(...close);
           return children;
         }