regression test performance (#73)

* regression test

* round

Author: larshp

.github/workflows/regression.yml

@@ -0,0 +1,74 @@
+name: regression
+
+# Performance regression test: runs perf/benchmarks/ through the transpiled
+# interpreter built from the PR branch and from the base branch, then posts
+# a before/after comparison as a PR comment.
+# Inspired by https://github.com/abaplint/abaplint/blob/main/.github/workflows/regression.yml
+
+on:
+  pull_request:
+
+permissions:
+  contents: read
+  pull-requests: write
+
+concurrency:
+  group: regression-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  perf:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        path: after
+    - uses: actions/checkout@v6
+      with:
+        ref: ${{ github.event.pull_request.base.sha }}
+        path: before
+    - uses: actions/setup-node@v6
+
+    - name: build PR branch
+      working-directory: after
+      run: |
+        npm ci
+        npx abap_transpile
+
+    - name: build base branch
+      working-directory: before
+      run: |
+        npm ci
+        npx abap_transpile
+
+    # alternate base/PR passes so both sides sample the same machine conditions
+    - name: run benchmarks
+      run: |
+        for i in 1 2 3 4 5 6 7; do
+          node after/perf/run.mjs before/output before.json
+          node after/perf/run.mjs after/output after.json
+        done
+
+    - name: compare
+      run: |
+        node after/perf/compare.mjs before.json after.json | tee comment-body.md
+        cat comment-body.md >> "$GITHUB_STEP_SUMMARY"
+
+    - name: find existing comment
+      if: github.event.pull_request.head.repo.full_name == github.repository
+      uses: peter-evans/find-comment@v3
+      id: fc
+      with:
+        issue-number: ${{ github.event.pull_request.number }}
+        comment-author: 'github-actions[bot]'
+        body-includes: Performance regression results
+
+    - name: create or update comment
+      if: github.event.pull_request.head.repo.full_name == github.repository
+      uses: peter-evans/create-or-update-comment@v4
+      with:
+        comment-id: ${{ steps.fc.outputs.comment-id }}
+        issue-number: ${{ github.event.pull_request.number }}
+        body-path: comment-body.md
+        edit-mode: replace

.gitignore

@@ -11,3 +11,7 @@ output
 #                      current score: PASS=230 FAIL=36 TOTAL=266
 #   (add your own .js files here freely)
 _tmp/
+
+# perf regression result files (perf/run.mjs)
+before.json
+after.json

perf/benchmarks/array.js

@@ -0,0 +1,9 @@
+const arr = [];
+for (let i = 0; i < 2000; i++) {
+  arr.push(i);
+}
+let total = 0;
+for (let i = 0; i < arr.length; i++) {
+  total = total + arr[i];
+}
+console.log(total);

perf/benchmarks/class.js

@@ -0,0 +1,13 @@
+class Counter {
+  constructor() {
+    this.n = 0;
+  }
+  inc() {
+    this.n = this.n + 1;
+  }
+}
+const c = new Counter();
+for (let i = 0; i < 1000; i++) {
+  c.inc();
+}
+console.log(c.n);

perf/benchmarks/fib.js

@@ -0,0 +1,5 @@
+function fib(n) {
+  if (n < 2) return n;
+  return fib(n - 1) + fib(n - 2);
+}
+console.log(fib(15));

perf/benchmarks/loop.js

@@ -0,0 +1,5 @@
+let sum = 0;
+for (let i = 0; i < 3000; i++) {
+  sum = sum + i;
+}
+console.log(sum);

perf/benchmarks/object.js

@@ -0,0 +1,5 @@
+const obj = { count: 0, name: "bench" };
+for (let i = 0; i < 2000; i++) {
+  obj.count = obj.count + 1;
+}
+console.log(obj.count);

perf/benchmarks/string.js

@@ -0,0 +1,5 @@
+let s = "";
+for (let i = 0; i < 1000; i++) {
+  s = s + "ab";
+}
+console.log(s.length);

perf/compare.mjs

@@ -0,0 +1,74 @@
+// Compares two result files produced by perf/run.mjs (one or more passes
+// each) and prints a markdown summary with the median time per benchmark.
+//
+// usage: node perf/compare.mjs <before.json> <after.json>
+
+import * as fs from "node:fs";
+
+const [beforeFile, afterFile] = process.argv.slice(2);
+if (beforeFile === undefined || afterFile === undefined) {
+  console.error("usage: node perf/compare.mjs <before.json> <after.json>");
+  process.exit(1);
+}
+
+// [{name, ms, value} | {name, error}] per pass -> {name: {median, value, error}}
+function aggregate(file) {
+  const passes = JSON.parse(fs.readFileSync(file, "utf-8"));
+  const byName = new Map();
+  for (const pass of passes) {
+    for (const r of pass) {
+      const agg = byName.get(r.name) ?? {times: []};
+      if (r.error !== undefined) {
+        agg.error = r.error;
+      } else {
+        agg.times.push(r.ms);
+        agg.value = r.value;
+      }
+      byName.set(r.name, agg);
+    }
+  }
+  for (const agg of byName.values()) {
+    agg.times.sort((x, y) => x - y);
+    agg.median = agg.times[Math.floor(agg.times.length / 2)];
+  }
+  return byName;
+}
+
+const before = aggregate(beforeFile);
+const after = aggregate(afterFile);
+const passCount = JSON.parse(fs.readFileSync(afterFile, "utf-8")).length;
+
+const lines = [];
+lines.push("## Performance regression results");
+lines.push("");
+lines.push("| Benchmark | Base | PR | Change |");
+lines.push("|---|---:|---:|---:|");
+
+for (const [name, a] of after) {
+  const b = before.get(name);
+  const col = agg => agg === undefined ? "n/a" : agg.error !== undefined ? "ERROR" : `${Math.round(agg.median)}ms`;
+  let changeCol = "";
+  if (b !== undefined && b.error === undefined && a.error === undefined) {
+    const delta = ((a.median - b.median) / b.median) * 100;
+    const sign = delta >= 0 ? "+" : "";
+    let marker = "";
+    if (delta > 15) {
+      marker = " 🔺";
+    } else if (delta < -15) {
+      marker = " 🟢";
+    }
+    changeCol = `${sign}${delta.toFixed(1)}%${marker}`;
+    if (a.value !== b.value) {
+      const fmt = v => String(v).replace(/\r?\n/g, "\\n").replace(/\|/g, "\\|");
+      changeCol += ` ⚠️ result changed: \`${fmt(b.value)}\` → \`${fmt(a.value)}\``;
+    }
+  } else if (a.error !== undefined || b?.error !== undefined) {
+    changeCol = `⚠️ ${a.error ?? b.error}`;
+  }
+  lines.push(`| ${name} | ${col(b)} | ${col(a)} | ${changeCol} |`);
+}
+
+lines.push("");
+lines.push(`Median of ${passCount} interleaved passes of the transpiled interpreter — small deltas are noise.`);
+
+console.log(lines.join("\n"));

perf/run.mjs

@@ -0,0 +1,55 @@
+// Runs one timing pass of every perf/benchmarks/*.js file through the
+// transpiled ZCL_MJS interpreter and appends the pass to the result file.
+//
+// Invoke multiple times, alternating between the two builds under comparison,
+// so both sides sample the same machine conditions (see regression.yml).
+// perf/compare.mjs computes the median per benchmark across all passes.
+//
+// usage: node perf/run.mjs <transpiled-output-dir> [result.json]
+
+import * as fs from "node:fs";
+import * as path from "node:path";
+import {fileURLToPath, pathToFileURL} from "node:url";
+import {performance} from "node:perf_hooks";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+const outputDir = process.argv[2];
+const resultFile = process.argv[3];
+if (outputDir === undefined) {
+  console.error("usage: node perf/run.mjs <transpiled-output-dir> [result.json]");
+  process.exit(1);
+}
+
+const {initializeABAP} = await import(pathToFileURL(path.resolve(outputDir, "init.mjs")));
+await initializeABAP();
+
+const benchmarkDir = path.join(__dirname, "benchmarks");
+const files = fs.readdirSync(benchmarkDir).filter(f => f.endsWith(".js")).sort();
+const pass = [];
+
+for (const file of files) {
+  const name = path.basename(file, ".js");
+  const source = fs.readFileSync(path.join(benchmarkDir, file), "utf-8");
+  try {
+    const start = performance.now();
+    const res = await abap.Classes["ZCL_MJS"].eval({iv_source: new abap.types.String().set(source)});
+    const ms = Math.round((performance.now() - start) * 10) / 10;
+    const value = res.get().trim();
+    console.log(`${name}: ${ms}ms = ${value}`);
+    pass.push({name, ms, value});
+  } catch (e) {
+    const error = "" + (e?.message ?? e);
+    console.log(`${name}: ERROR ${error}`);
+    pass.push({name, error});
+  }
+}
+
+if (resultFile) {
+  let passes = [];
+  if (fs.existsSync(resultFile)) {
+    passes = JSON.parse(fs.readFileSync(resultFile, "utf-8"));
+  }
+  passes.push(pass);
+  fs.writeFileSync(resultFile, JSON.stringify(passes, null, 2));
+}