Fix word parts, lazy compute

Author: webpro

bench/lazy-word.ts

@@ -0,0 +1,150 @@
+import { parse } from "../src/parser.ts";
+import type { Node, Statement, Word } from "../src/types.ts";
+import { bench, group, run, summary } from "mitata";
+import { short, advanced, specialized, installers, large } from "./fixtures.ts";
+
+function visitWords(node: Node | Statement): number {
+  let count = 0;
+  switch (node.type) {
+    case "Command":
+      if (node.name) count += touchWord(node.name);
+      for (const w of node.suffix) count += touchWord(w);
+      for (const a of node.prefix) {
+        if (a.value) count += touchWord(a.value);
+        if (a.array) for (const w of a.array) count += touchWord(w);
+      }
+      for (const r of node.redirects) count += visitRedirect(r);
+      break;
+    case "Pipeline":
+      for (const c of node.commands) count += visitWords(c);
+      break;
+    case "AndOr":
+      for (const c of node.commands) count += visitWords(c);
+      break;
+    case "If":
+      count += visitNode(node.clause);
+      count += visitNode(node.then);
+      if (node.else) count += visitNode(node.else);
+      break;
+    case "For":
+      count += touchWord(node.name);
+      for (const w of node.wordlist) count += touchWord(w);
+      count += visitCompoundList(node.body);
+      break;
+    case "While":
+      count += visitNode(node.clause);
+      count += visitCompoundList(node.body);
+      break;
+    case "Case":
+      count += touchWord(node.word);
+      for (const item of node.items) {
+        for (const w of item.pattern) count += touchWord(w);
+        count += visitCompoundList(item.body);
+      }
+      break;
+    case "Function":
+      count += touchWord(node.name);
+      count += visitNode(node.body);
+      for (const r of node.redirects) count += visitRedirect(r);
+      break;
+    case "Select":
+      count += touchWord(node.name);
+      for (const w of node.wordlist) count += touchWord(w);
+      count += visitCompoundList(node.body);
+      break;
+    case "Subshell":
+    case "BraceGroup":
+      count += visitCompoundList(node.body);
+      break;
+    case "CompoundList":
+      count += visitCompoundList(node);
+      break;
+    case "Statement":
+      count += visitWords(node.command);
+      for (const r of node.redirects) count += visitRedirect(r);
+      break;
+    case "Coproc":
+      if (node.name) count += touchWord(node.name);
+      count += visitNode(node.body);
+      for (const r of node.redirects) count += visitRedirect(r);
+      break;
+    case "TestCommand":
+      count += visitTestExpr(node.expression);
+      break;
+    case "ArithmeticCommand":
+    case "ArithmeticFor":
+      break;
+  }
+  return count;
+}
+
+function visitNode(node: Node): number {
+  return visitWords(node);
+}
+
+function visitCompoundList(cl: { commands: Statement[] }): number {
+  let count = 0;
+  for (const s of cl.commands) count += visitWords(s);
+  return count;
+}
+
+function visitRedirect(r: { target?: Word; body?: Word }): number {
+  let count = 0;
+  if (r.target) count += touchWord(r.target);
+  if (r.body) count += touchWord(r.body);
+  return count;
+}
+
+function visitTestExpr(expr: import("../src/types.ts").TestExpression): number {
+  let count = 0;
+  switch (expr.type) {
+    case "TestUnary":
+      count += touchWord(expr.operand);
+      break;
+    case "TestBinary":
+      count += touchWord(expr.left);
+      count += touchWord(expr.right);
+      break;
+    case "TestLogical":
+      count += visitTestExpr(expr.left);
+      count += visitTestExpr(expr.right);
+      break;
+    case "TestNot":
+      count += visitTestExpr(expr.operand);
+      break;
+    case "TestGroup":
+      count += visitTestExpr(expr.expression);
+      break;
+  }
+  return count;
+}
+
+function touchWord(w: Word): number {
+  w.parts; // trigger lazy computation
+  return 1;
+}
+
+for (const [label, scripts] of [
+  ["short", short],
+  ["advanced", advanced],
+  ...specialized,
+  ["medium", installers],
+  ["large", large],
+] as const) {
+  group(label, () => {
+    summary(() => {
+      bench("parse only", () => {
+        for (const s of scripts) parse(s);
+      }).baseline();
+
+      bench("parse + parts", () => {
+        for (const s of scripts) {
+          const ast = parse(s);
+          for (const stmt of ast.commands) visitWords(stmt);
+        }
+      });
+    });
+  });
+}
+
+await run();

src/parser.ts

@@ -37,9 +37,36 @@ import type {
   TestUnaryExpression,
   While,
   Word,
+  WordPart,
 } from "./types.ts";
 import { LexContext, Token, Lexer, TokenValue } from "./lexer.ts";
 import { parseArithmeticExpression } from "./arithmetic.ts";
+import { computeWordParts } from "./parts.ts";
+
+class WordImpl implements Word {
+  text: string;
+  pos: number;
+  end: number;
+  #source: string;
+  #parts: WordPart[] | undefined | null = null; // null = not yet computed
+
+  constructor(text: string, pos: number, end: number, source: string) {
+    this.text = text;
+    this.pos = pos;
+    this.end = end;
+    this.#source = source;
+  }
+
+  get parts(): WordPart[] | undefined {
+    if (this.#parts === null) {
+      this.#parts = computeWordParts(this.#source, this) ?? undefined;
+    }
+    return this.#parts;
+  }
+  set parts(v: WordPart[] | undefined) {
+    this.#parts = v ?? undefined;
+  }
+}
 
 const CASE_TERMINATORS: Record<number, CaseTerminator> = {
   [Token.DoubleSemi]: ";;",
@@ -175,11 +202,13 @@ export function parse(source: string): Script & { errors?: ParseError[] } {
 
 class Parser {
   private tok: Lexer;
+  private source: string;
   private errors: ParseError[] = [];
   private _redirects: Redirect[] = [];
 
   constructor(source: string) {
     this.tok = new Lexer(source);
+    this.source = source;
   }
 
   parse(sourceLen: number): Script & { errors?: ParseError[] } {
@@ -961,7 +990,7 @@ class Parser {
       body: undefined,
     };
     if (t.content != null) {
-      r.target = { text: t.content, pos: t.targetPos, end: t.targetEnd, parts: undefined };
+      r.target = new WordImpl(t.content, t.targetPos, t.targetEnd, this.source);
     }
     if (t.value === "<<" || t.value === "<<-") this.tok.registerHereDocTarget(r);
     redirects.push(r);
@@ -982,11 +1011,11 @@ class Parser {
   }
 
   private toWord(tok: TokenValue): Word {
-    return { text: tok.value, pos: tok.pos, end: tok.end, parts: undefined };
+    return new WordImpl(tok.value, tok.pos, tok.end, this.source);
   }
 
   private toWordFromPosEnd(tok: TokenValue, pos: number, end: number): Word {
-    return { text: tok.value, pos, end, parts: undefined };
+    return new WordImpl(tok.value, pos, end, this.source);
   }
 
   private parseAssignment(tok: TokenValue): AssignmentPrefix {
@@ -1045,7 +1074,7 @@ class Parser {
       const elements = this.parseArrayElements(inner, arrayOffset);
       result.array = elements;
     } else {
-      result.value = { text: valText, pos: tokPos + valStart, end: tokEnd, parts: undefined };
+      result.value = new WordImpl(valText, tokPos + valStart, tokEnd, this.source);
     }
 
     return result;