Add v4 vs v5 benchmark comparison suite

bench/compare.py orchestrates side-by-side benchmarking: clones the old
datafog-python repo, sets up isolated venvs, runs benchmarks in parallel,
and prints a comparison table. bench/runner.py is the shared benchmark
runner that auto-detects which package version is installed.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: sidmohan0

.gitignore

@@ -32,3 +32,4 @@ Thumbs.db
 .pytest_cache/
 htmlcov/
 *.prof
+bench/.workspaces/

bench/compare.py

@@ -0,0 +1,254 @@
+#!/usr/bin/env python3
+"""
+Compare datafog v5 (Rust) vs v4 (Python) side-by-side.
+
+Sets up two isolated workspaces, installs each package, runs benchmarks
+in parallel, and prints a comparison table.
+
+Usage:
+    python bench/compare.py              # full setup + benchmark
+    python bench/compare.py --no-setup   # skip setup, reuse existing workspaces
+    python bench/compare.py --clean      # wipe workspaces and start fresh
+"""
+
+import argparse
+import json
+import os
+import shutil
+import subprocess
+import sys
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+WORKSPACE_DIR = REPO_ROOT / "bench" / ".workspaces"
+OLD_DIR = WORKSPACE_DIR / "old"
+NEW_DIR = WORKSPACE_DIR / "new"
+RUNNER = REPO_ROOT / "bench" / "runner.py"
+
+OLD_REPO_URL = "https://github.com/DataFog/datafog-python.git"
+
+
+def run(cmd, cwd=None, check=True):
+    """Run a shell command, printing it for visibility."""
+    print(f"  $ {cmd}")
+    result = subprocess.run(
+        cmd, shell=True, cwd=cwd,
+        capture_output=True, text=True,
+    )
+    if check and result.returncode != 0:
+        print(f"    FAILED (exit {result.returncode})")
+        if result.stderr:
+            for line in result.stderr.strip().splitlines()[-3:]:
+                print(f"    {line}")
+        sys.exit(1)
+    return result
+
+
+def find_python(max_version=(3, 13)):
+    """Find a Python interpreter that satisfies max_version constraint.
+
+    The old datafog package requires Python <3.13. We need to find
+    a compatible interpreter. Returns the path or None.
+    """
+    # Check common locations
+    candidates = [
+        "python3.12", "python3.11", "python3.10",
+        "/opt/homebrew/bin/python3.12",
+        "/opt/homebrew/bin/python3.11",
+        "/opt/homebrew/bin/python3.10",
+        "/usr/local/bin/python3.12",
+        "/usr/local/bin/python3.11",
+        "/usr/local/bin/python3.10",
+    ]
+    for candidate in candidates:
+        try:
+            result = subprocess.run(
+                [candidate, "--version"],
+                capture_output=True, text=True,
+            )
+            if result.returncode == 0:
+                version_str = result.stdout.strip().split()[-1]
+                parts = tuple(int(x) for x in version_str.split(".")[:2])
+                if parts < max_version:
+                    return candidate
+        except FileNotFoundError:
+            continue
+
+    # Fall back to system python3 if it happens to be compatible
+    try:
+        result = subprocess.run(
+            ["python3", "--version"], capture_output=True, text=True
+        )
+        version_str = result.stdout.strip().split()[-1]
+        parts = tuple(int(x) for x in version_str.split(".")[:2])
+        if parts < max_version:
+            return "python3"
+    except FileNotFoundError:
+        pass
+
+    return None
+
+
+def setup_old():
+    """Clone the old datafog-python repo and install it in a venv."""
+    print("\n--- Setting up OLD workspace (v4 Python) ---\n")
+
+    # The old package requires Python <3.13
+    old_python = find_python(max_version=(3, 13))
+    if old_python is None:
+        print("  ERROR: datafog v4 requires Python <3.13 but none found.")
+        print("  Install Python 3.11 or 3.12 (e.g. `brew install python@3.12`)")
+        sys.exit(1)
+
+    result = subprocess.run([old_python, "--version"], capture_output=True, text=True)
+    print(f"  Using {old_python} ({result.stdout.strip()})")
+
+    if not (OLD_DIR / ".git").exists():
+        OLD_DIR.mkdir(parents=True, exist_ok=True)
+        run(f"git clone --depth 1 {OLD_REPO_URL} {OLD_DIR}")
+    else:
+        print(f"  (reusing existing clone at {OLD_DIR})")
+
+    venv = OLD_DIR / ".venv"
+    if not venv.exists():
+        run(f"{old_python} -m venv {venv}")
+
+    pip = venv / "bin" / "pip"
+    run(f"{pip} install -q --upgrade pip")
+    run(f"{pip} install -q {OLD_DIR}")
+
+    shutil.copy2(RUNNER, OLD_DIR / "runner.py")
+    print("  done.\n")
+
+
+def setup_new():
+    """Install the new package from the current repo into a venv."""
+    print("\n--- Setting up NEW workspace (v5 Rust) ---\n")
+
+    NEW_DIR.mkdir(parents=True, exist_ok=True)
+    venv = NEW_DIR / ".venv"
+    if not venv.exists():
+        run(f"python3 -m venv {venv}")
+
+    pip = venv / "bin" / "pip"
+    run(f"{pip} install -q --upgrade pip")
+    run(f"{pip} install -q {REPO_ROOT}")
+
+    shutil.copy2(RUNNER, NEW_DIR / "runner.py")
+    print("  done.\n")
+
+
+def run_benchmark(workspace, label):
+    """Run runner.py in the workspace's venv and return parsed JSON."""
+    python = workspace / ".venv" / "bin" / "python"
+    runner = workspace / "runner.py"
+    result = subprocess.run(
+        [str(python), str(runner)],
+        capture_output=True, text=True, cwd=str(workspace),
+    )
+    if result.returncode != 0:
+        print(f"\n  {label} benchmark failed:")
+        if result.stderr:
+            print(f"    {result.stderr.strip()}")
+        if result.stdout:
+            print(f"    {result.stdout.strip()}")
+        return None
+    return json.loads(result.stdout)
+
+
+def print_table(old, new):
+    """Print a side-by-side comparison table."""
+    tests = [
+        ("detect_single",    "detect single (228 B)"),
+        ("detect_batch_10",  "detect batch x10 (398 B)"),
+        ("detect_large",     "detect large (22.8 KB)"),
+        ("anonymize_redact", "anonymize redact (167 B)"),
+        ("anonymize_hash",   "anonymize hash (167 B)"),
+    ]
+
+    col_test = 28
+    col_val = 16
+    col_speedup = 10
+
+    sep = f"  {'─' * col_test}┬{'─' * col_val}┬{'─' * col_val}┬{'─' * col_speedup}"
+
+    print(f"\n{'='*72}")
+    print(f"  datafog v4 (Python) vs v5 (Rust) — median latency, 200 rounds")
+    print(f"{'='*72}\n")
+
+    hdr = (
+        f"  {'Test':<{col_test}}│"
+        f"{'v4 Python (μs)':^{col_val}}│"
+        f"{'v5 Rust (μs)':^{col_val}}│"
+        f"{'Speedup':^{col_speedup}}"
+    )
+    print(hdr)
+    print(sep)
+
+    for key, label in tests:
+        old_val = old.get(key, {}).get("median_us", 0)
+        new_val = new.get(key, {}).get("median_us", 0)
+        speedup = old_val / new_val if new_val > 0 else float("inf")
+
+        old_str = f"{old_val:,.1f}"
+        new_str = f"{new_val:,.1f}"
+        speedup_str = f"{speedup:,.1f}x"
+
+        print(
+            f"  {label:<{col_test}}│"
+            f"{old_str:^{col_val}}│"
+            f"{new_str:^{col_val}}│"
+            f"{speedup_str:^{col_speedup}}"
+        )
+
+    print(sep)
+
+    # Entity coverage
+    old_ents = old.get("entities_found", [])
+    new_ents = new.get("entities_found", [])
+    print(f"\n  v4 entities: {old_ents}")
+    print(f"  v5 entities: {new_ents}")
+    print(f"  v4 count: {old.get('entity_count', '?')}   v5 count: {new.get('entity_count', '?')}")
+    print()
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Compare datafog v4 vs v5 performance")
+    parser.add_argument("--no-setup", action="store_true", help="Skip setup, reuse existing workspaces")
+    parser.add_argument("--clean", action="store_true", help="Wipe workspaces and start fresh")
+    args = parser.parse_args()
+
+    if args.clean:
+        print(f"Cleaning {WORKSPACE_DIR} ...")
+        shutil.rmtree(WORKSPACE_DIR, ignore_errors=True)
+
+    if not args.no_setup:
+        # Run setup in parallel — old clone + new build happen concurrently
+        with ThreadPoolExecutor(max_workers=2) as pool:
+            futures = [pool.submit(setup_old), pool.submit(setup_new)]
+            for f in as_completed(futures):
+                f.result()
+
+    for label, d in [("old", OLD_DIR), ("new", NEW_DIR)]:
+        if not (d / ".venv" / "bin" / "python").exists():
+            print(f"Error: {label} workspace not set up. Run without --no-setup.", file=sys.stderr)
+            sys.exit(1)
+
+    # Run benchmarks in parallel
+    print("\n--- Running benchmarks ---\n")
+    with ThreadPoolExecutor(max_workers=2) as pool:
+        f_old = pool.submit(run_benchmark, OLD_DIR, "v4")
+        f_new = pool.submit(run_benchmark, NEW_DIR, "v5")
+        old_results = f_old.result()
+        new_results = f_new.result()
+
+    if old_results is None or new_results is None:
+        print("Benchmark failed.", file=sys.stderr)
+        sys.exit(1)
+
+    print_table(old_results, new_results)
+
+
+if __name__ == "__main__":
+    main()