Merge pull request #87 from githubnext/copilot/add-autoloop-performance-test

mrjf · web-flow · commit d2d283677867 · 2026-04-12T04:35:14.000-07:00
Add autoloop perf-comparison program for tsb vs pandas benchmarking
diff --git a/.autoloop/programs/perf-comparison/program.md b/.autoloop/programs/perf-comparison/program.md
@@ -0,0 +1,74 @@
+---
+schedule: every 6h
+---
+
+# Performance Comparison: tsb (TypeScript) vs pandas (Python)
+
+## Goal
+
+Systematically benchmark every tsb function against its pandas equivalent, one function per iteration. Each iteration picks a function that has not yet been benchmarked, writes a matching performance test for both tsb (TypeScript/Bun) and pandas (Python), runs both, and records the timing results. The benchmark results are displayed on the playground pages doc site.
+
+This is an open-ended program — it runs continuously, always adding the next benchmark comparison.
+
+### How each iteration works
+
+1. **Read existing benchmarks** — check `benchmarks/tsb/` and `benchmarks/pandas/` to see which functions are already benchmarked.
+2. **Pick ONE function** from `src/` that has no benchmark yet. Prioritize core operations (Series, DataFrame, GroupBy, etc.).
+3. **Write a TypeScript benchmark** in `benchmarks/tsb/bench_{function}.ts` that:
+   - Creates a realistic dataset (e.g. 100,000 rows)
+   - Runs the operation in a tight loop (warm-up + measured iterations)
+   - Outputs JSON: `{"function": "...", "mean_ms": ..., "iterations": ..., "total_ms": ...}`
+4. **Write a matching Python benchmark** in `benchmarks/pandas/bench_{function}.py` that:
+   - Creates the same dataset as the TypeScript version
+   - Runs the same operation with the same loop structure
+   - Outputs the same JSON format
+5. **Run both benchmarks** via `benchmarks/run_benchmarks.sh` and capture results.
+6. **Update `benchmarks/results.json`** with the new timing data.
+7. **Update `playground/benchmarks.html`** to display the new function's comparison metrics.
+
+### Key constraints
+
+- **Matching datasets** — both benchmarks must use identical data (same size, same values where possible).
+- **Fair comparison** — same number of warm-up and measured iterations for both.
+- **JSON output** — every benchmark script must output a single JSON line to stdout.
+- **No modifications to `src/`** — benchmark code is separate from library code.
+- **Python environment** — install pandas via pip if not present.
+
+## Target
+
+Only modify these files:
+- `benchmarks/**` — benchmark scripts and results
+- `playground/benchmarks.html` — performance comparison playground page
+- `playground/index.html` — add/update link to benchmarks page
+
+Do NOT modify:
+- `src/**` — library source code
+- `tests/**` — test files
+- `README.md` — read-only
+- `.autoloop/programs/**` — program definitions (except this file's code/ dir)
+- `.github/workflows/autoloop*` — autoloop workflow files
+
+## Evaluation
+
+```bash
+# Set up Python environment if needed
+if ! command -v python3 &>/dev/null; then
+  echo "Python3 not found, skipping"
+fi
+pip3 install pandas --quiet 2>/dev/null || true
+
+# Count the number of benchmark pairs (functions with both TS and Python benchmarks)
+ts_benchmarks=$(ls benchmarks/tsb/bench_*.ts 2>/dev/null | wc -l | tr -d ' ')
+py_benchmarks=$(ls benchmarks/pandas/bench_*.py 2>/dev/null | wc -l | tr -d ' ')
+
+# The metric is the minimum of the two (both must exist for a complete benchmark)
+if [ "$ts_benchmarks" -lt "$py_benchmarks" ]; then
+  count=$ts_benchmarks
+else
+  count=$py_benchmarks
+fi
+
+echo "{\"benchmarked_functions\": ${count:-0}}"
+```
+
+The metric is `benchmarked_functions`. **Higher is better.**
diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml
@@ -36,6 +36,13 @@ jobs:
       - name: Bundle TypeScript compiler for offline playground
         run: cp node_modules/typescript/lib/typescript.js ./playground/dist/typescript.js
 
+      - name: Copy benchmark results to playground
+        run: |
+          mkdir -p ./playground/benchmarks
+          if [ -f benchmarks/results.json ]; then
+            cp benchmarks/results.json ./playground/benchmarks/results.json
+          fi
+
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
diff --git a/benchmarks/pandas/bench_series_creation.py b/benchmarks/pandas/bench_series_creation.py
@@ -0,0 +1,47 @@
+"""
+Benchmark: Series creation
+
+Creates a Series from a large numeric array and measures the time.
+Outputs JSON: {"function": "series_creation", "mean_ms": ..., "iterations": ..., "total_ms": ...}
+"""
+
+import json
+import time
+
+import pandas as pd
+
+SIZE = 100_000
+WARMUP = 5
+ITERATIONS = 50
+
+
+def generate_data(n: int) -> "list[float]":
+    """Generate a deterministic numeric array of the given size."""
+    return [i * 1.1 + 0.5 for i in range(n)]
+
+
+data = generate_data(SIZE)
+
+# Warm-up
+for _ in range(WARMUP):
+    pd.Series(list(data))
+
+# Measured runs
+times: "list[float]" = []
+for _ in range(ITERATIONS):
+    start = time.perf_counter()
+    pd.Series(list(data))
+    end = time.perf_counter()
+    times.append((end - start) * 1000)  # convert to ms
+
+total_ms = sum(times)
+mean_ms = total_ms / ITERATIONS
+
+result = {
+    "function": "series_creation",
+    "mean_ms": round(mean_ms, 3),
+    "iterations": ITERATIONS,
+    "total_ms": round(total_ms, 3),
+}
+
+print(json.dumps(result))
diff --git a/benchmarks/results.json b/benchmarks/results.json
@@ -0,0 +1 @@
+{ "benchmarks": [], "timestamp": null }
diff --git a/benchmarks/run_benchmarks.sh b/benchmarks/run_benchmarks.sh
@@ -0,0 +1,129 @@
+#!/usr/bin/env bash
+#
+# Run all tsb (TypeScript) and pandas (Python) benchmarks and collect results.
+#
+# Usage: ./benchmarks/run_benchmarks.sh
+#
+# Outputs: benchmarks/results.json with all benchmark results
+#
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+# Ensure Python and pandas are available
+if ! command -v python3 &>/dev/null; then
+  echo "ERROR: python3 is required but not found" >&2
+  exit 1
+fi
+
+python3 -c "import pandas" 2>/dev/null || {
+  echo "Installing pandas..."
+  pip3 install pandas --quiet
+}
+
+# Ensure Bun is available
+if ! command -v bun &>/dev/null; then
+  echo "ERROR: bun is required but not found" >&2
+  exit 1
+fi
+
+# Collect results
+results='{"benchmarks": [], "timestamp": "'$(date -u +"%Y-%m-%dT%H:%M:%SZ")'"}'
+
+echo "=== Running Performance Benchmarks ==="
+echo ""
+
+# Find all TypeScript benchmark files
+for ts_bench in "$SCRIPT_DIR"/tsb/bench_*.ts; do
+  [ -f "$ts_bench" ] || continue
+  bench_name=$(basename "$ts_bench" .ts | sed 's/^bench_//')
+
+  # Check for matching Python benchmark
+  py_bench="$SCRIPT_DIR/pandas/bench_${bench_name}.py"
+  if [ ! -f "$py_bench" ]; then
+    echo "SKIP: $bench_name (no matching Python benchmark)"
+    continue
+  fi
+
+  echo "--- Benchmarking: $bench_name ---"
+
+  # Run TypeScript benchmark
+  echo "  Running tsb (TypeScript)..."
+  ts_result=$(cd "$REPO_ROOT" && bun run "$ts_bench" 2>/dev/null) || {
+    echo "  ERROR: TypeScript benchmark failed"
+    continue
+  }
+  echo "  tsb result: $ts_result"
+
+  # Run Python benchmark
+  echo "  Running pandas (Python)..."
+  py_result=$(cd "$REPO_ROOT" && python3 "$py_bench" 2>/dev/null) || {
+    echo "  ERROR: Python benchmark failed"
+    continue
+  }
+  echo "  pandas result: $py_result"
+
+  # Extract mean_ms from both
+  ts_mean=$(echo "$ts_result" | python3 -c "import sys, json; d=json.load(sys.stdin); print(d['mean_ms'])" 2>/dev/null) || {
+    echo "  ERROR: could not parse tsb benchmark result"
+    continue
+  }
+  py_mean=$(echo "$py_result" | python3 -c "import sys, json; d=json.load(sys.stdin); print(d['mean_ms'])" 2>/dev/null) || {
+    echo "  ERROR: could not parse pandas benchmark result"
+    continue
+  }
+
+  # Calculate ratio (tsb / pandas) — < 1.0 means tsb is faster
+  ratio=$(python3 -c "
+ts, py = $ts_mean, $py_mean
+if py <= 0:
+    print('null')
+else:
+    print(round(ts / py, 3))
+")
+  if [ "$ratio" = "null" ]; then
+    echo "  ERROR: pandas mean_ms is zero, cannot compute ratio"
+    continue
+  fi
+
+  echo "  Ratio (tsb/pandas): ${ratio}x"
+  echo ""
+
+  # Add to results JSON
+  results=$(echo "$results" | python3 -c "
+import sys, json
+data = json.load(sys.stdin)
+data['benchmarks'].append({
+    'function': '$bench_name',
+    'tsb': $ts_result,
+    'pandas': $py_result,
+    'ratio': $ratio
+})
+print(json.dumps(data, indent=2))
+")
+done
+
+# Write results
+echo "$results" > "$SCRIPT_DIR/results.json"
+echo "=== Results written to benchmarks/results.json ==="
+echo ""
+
+# Summary
+echo "=== Summary ==="
+echo "$results" | python3 -c "
+import sys, json
+data = json.load(sys.stdin)
+benchmarks = data.get('benchmarks', [])
+if not benchmarks:
+    print('No benchmarks found.')
+else:
+    print(f'Functions benchmarked: {len(benchmarks)}')
+    for b in benchmarks:
+        fn = b['function']
+        ts = b['tsb']['mean_ms']
+        py = b['pandas']['mean_ms']
+        ratio = b['ratio']
+        faster = 'tsb' if ratio < 1 else 'pandas'
+        print(f'  {fn}: tsb={ts}ms, pandas={py}ms, ratio={ratio}x ({faster} is faster)')
+"
diff --git a/benchmarks/tsb/bench_series_creation.ts b/benchmarks/tsb/bench_series_creation.ts
@@ -0,0 +1,49 @@
+/**
+ * Benchmark: Series creation
+ *
+ * Creates a Series from a large numeric array and measures the time.
+ * Outputs JSON: {"function": "series_creation", "mean_ms": ..., "iterations": ..., "total_ms": ...}
+ */
+
+import { Series } from "../../src/index.ts";
+
+const SIZE = 100_000;
+const WARMUP = 5;
+const ITERATIONS = 50;
+
+/** Generate a deterministic numeric array of the given size. */
+function generateData(n: number): readonly number[] {
+  const arr: number[] = [];
+  for (let i = 0; i < n; i++) {
+    arr.push(i * 1.1 + 0.5);
+  }
+  return arr;
+}
+
+const data = generateData(SIZE);
+
+// Warm-up
+for (let i = 0; i < WARMUP; i++) {
+  new Series({ data: [...data] });
+}
+
+// Measured runs
+const times: number[] = [];
+for (let i = 0; i < ITERATIONS; i++) {
+  const start = performance.now();
+  new Series({ data: [...data] });
+  const end = performance.now();
+  times.push(end - start);
+}
+
+const totalMs = times.reduce((a, b) => a + b, 0);
+const meanMs = totalMs / ITERATIONS;
+
+const result = {
+  function: "series_creation",
+  mean_ms: Math.round(meanMs * 1000) / 1000,
+  iterations: ITERATIONS,
+  total_ms: Math.round(totalMs * 1000) / 1000,
+};
+
+console.log(JSON.stringify(result));
diff --git a/playground/benchmarks.html b/playground/benchmarks.html
diff --git a/playground/index.html b/playground/index.html
diff --git a/tests/stats/rank.test.ts b/tests/stats/rank.test.ts

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+{ "benchmarks": [], "timestamp": null }`