feat: add codspeed benchmarks for valgrind

Author: not-matthias

.gitattributes

@@ -0,0 +1 @@
+bench/testdata/* filter=lfs diff=lfs merge=lfs -text

.github/workflows/codspeed.yml

@@ -0,0 +1,66 @@
+name: CodSpeed Benchmarks
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  benchmarks:
+    runs-on: codspeed-macro
+    strategy:
+      matrix:
+        benchmark:
+          - testdata/take_strings
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          lfs: true
+
+      # Build and install Valgrind
+      #
+      - name: Update apt-get cache
+        run: sudo apt-get update
+
+      - name: Install build dependencies
+        run: |
+          sudo apt-get install -y \
+            build-essential \
+            automake \
+            autoconf \
+            libc6-dev \
+            gcc-multilib \
+            libc6-dev-i386 \
+            gdb \
+            docbook \
+            docbook-xsl \
+            docbook-xml \
+            xsltproc
+
+      - name: Run autogen
+        run: ./autogen.sh
+
+      - name: Configure
+        run: ./configure
+
+      - name: Build Valgrind
+        run: make -j$(nproc)
+
+      - name: Install valgrind
+        run: sudo make install
+
+      # Setup benchmarks and run them
+      #
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Run benchmark
+        working-directory: bench
+        run: uv run bench.py --binary-path ${{ matrix.benchmark }}
+
+      - name: Run the benchmarks
+        uses: CodSpeedHQ/action@main
+        with:
+          mode: walltime

bench/bench.py

@@ -0,0 +1,168 @@
+#!/usr/bin/env python3
+# /// script
+# requires-python = ">=3.9"
+# dependencies = [
+#     "pytest>=8.0",
+#     "pytest-codspeed>=4.2.0",
+# ]
+# ///
+
+import argparse
+import os
+import subprocess
+from pathlib import Path
+
+import pytest
+
+
+class ValgrindRunner:
+    """Run Valgrind with different configurations."""
+
+    def __init__(
+        self,
+        binary_path: str,
+        valgrind_path: str = "valgrind",
+        output_dir: str = "/tmp",
+    ):
+        """Initialize valgrind runner.
+
+        Args:
+            binary_path: Path to the binary to profile
+            valgrind_path: Path to valgrind executable
+            output_dir: Directory for callgrind output files
+        """
+        self.binary_path = Path(binary_path)
+        self.valgrind_path = valgrind_path
+        self.output_dir = Path(output_dir)
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+
+        if not self.binary_path.exists():
+            raise FileNotFoundError(f"Binary not found: {self.binary_path}")
+
+        # Verify valgrind is available
+        result = subprocess.run(
+            [self.valgrind_path, "--version"],
+            capture_output=True,
+            text=True,
+        )
+        if result.returncode != 0:
+            raise RuntimeError(f"Valgrind not found at: {self.valgrind_path}")
+        self.valgrind_version = result.stdout.strip()
+
+    def run_valgrind(self, *args: str) -> None:
+        """Execute valgrind with given arguments.
+
+        Args:
+            *args: Valgrind arguments
+        """
+        callgrind_output = self.output_dir / f"callgrind.{os.getpid()}"
+
+        cmd = [
+            self.valgrind_path,
+            "--tool=callgrind",
+            f"--callgrind-out-file={callgrind_output}",
+            *args,
+            str(self.binary_path),
+        ]
+
+        result = subprocess.run(cmd)
+        if result.returncode != 0:
+            raise RuntimeError(f"Valgrind execution failed with code {result.returncode}")
+
+        # Clean up
+        if callgrind_output.exists():
+            callgrind_output.unlink()
+
+
+# Store runner in pytest config
+_runner = None
+
+
+@pytest.fixture
+def runner(request):
+    """Fixture to provide runner instance to tests."""
+    return request.config._valgrind_runner
+
+
+def test_baseline(runner):
+    if runner:
+        runner.run_valgrind("--read-inline-info=no")
+
+
+def test_with_inline_info(runner):
+    if runner:
+        runner.run_valgrind("--read-inline-info=yes")
+
+
+def test_full_inline(runner):
+    if runner:
+        runner.run_valgrind("--trace-children=yes", "--cache-sim=yes", "--I1=32768,8,64", "--D1=32768,8,64", "--LL=8388608,16,64", "--collect-systime=nsec", "--compress-strings=no", "--combine-dumps=yes", "--dump-line=no", "--read-inline-info=yes")
+
+def test_full(runner):
+    if runner:
+        runner.run_valgrind("--trace-children=yes", "--cache-sim=yes", "--I1=32768,8,64", "--D1=32768,8,64", "--LL=8388608,16,64", "--collect-systime=nsec", "--compress-strings=no", "--combine-dumps=yes", "--dump-line=no")
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Benchmark Valgrind with pytest-codspeed",
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        epilog="""
+Examples:
+  # Run with default binary
+  uv run bench.py --binary-path /path/to/binary
+
+  # Run with custom valgrind installation
+  uv run bench.py --binary-path /path/to/binary --valgrind-path /usr/local/bin/valgrind
+        """,
+    )
+
+    parser.add_argument(
+        "--binary-path",
+        type=str,
+        required=True,
+        help="Path to the binary to profile",
+    )
+    parser.add_argument(
+        "--valgrind-path",
+        type=str,
+        default="valgrind",
+        help="Path to valgrind executable (default: valgrind)",
+    )
+    parser.add_argument(
+        "--output-dir",
+        type=str,
+        default="/tmp",
+        help="Directory for callgrind files (default: /tmp)",
+    )
+
+    args = parser.parse_args()
+
+    # Create runner instance
+    runner = ValgrindRunner(
+        binary_path=args.binary_path,
+        valgrind_path=args.valgrind_path,
+        output_dir=args.output_dir,
+    )
+    print(f"Valgrind version: {runner.valgrind_version}")
+    print(f"Binary: {args.binary_path}")
+
+    # Plugin to pass runner to tests
+    class RunnerPlugin:
+        def pytest_configure(self, config):
+            config._valgrind_runner = runner
+
+    exit_code = pytest.main(
+        [
+            __file__,
+            "-v",
+            "--codspeed"
+        ],
+        plugins=[RunnerPlugin()],
+    )
+    if exit_code != 0 and exit_code != 5:
+        print(f"Benchmark execution returned exit code: {exit_code}")
+
+
+if __name__ == "__main__":
+    main()

bench/testdata/take_strings

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c184f81f7046a8a78cb272ac4a1c7ad616b5e3dd20dcc40638f1db485abc5b22
+size 272199232