ci(codspeed): discover benches via cargo metadata, declare skips in manifests

adriangb · claude · adriangb · commit c4e266f2e43d · 2026-05-26T18:24:58.000-05:00
Replace the per-workflow awk/jq Cargo.toml parsing and hardcoded crate
lists with a single shared script (.github/workflows/codspeed-matrix.sh)
that discovers every [[bench]] target across the workspace via
`cargo metadata`. New crates and bench targets are picked up automatically.

The known-broken exclusion list, previously duplicated as an
EXCLUDED_BENCHES env in both workflows, now lives next to each bench in
its crate's Cargo.toml:

    [package.metadata.codspeed.benches]
    merge_kernels = { skip = true }

cargo surfaces that table at .packages[].metadata.codspeed.benches and the
script drops any target flagged `skip = true`. Unknown `bench:&lt;crate&gt;`
label suffixes now error against the workspace member list rather than a
loose regex.

Co-Authored-By: Claude Opus 4.7 (1M context) &lt;noreply@anthropic.com&gt;
diff --git a/.github/workflows/codspeed-matrix.sh b/.github/workflows/codspeed-matrix.sh
@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Generate the CodSpeed benchmark matrix.
+#
+# Emits to stdout a compact JSON array of {"crate","bench"} objects — one per
+# `[[bench]]` target in the selected workspace crates. Each object becomes one
+# CodSpeed shard (`cargo codspeed run -p <crate> --bench <bench>`). Sharding
+# one job per bench target keeps every shard under CodSpeed's hard per-upload
+# limit of 1000 benchmarks, on the assumption that no single bench target
+# defines >1000 benchmark cases. If a target ever crosses that line, split the
+# bench source rather than reworking the sharding here.
+#
+# Targets are discovered structurally via `cargo metadata` (no Cargo.toml text
+# parsing, no hardcoded crate list), so new crates and new `[[bench]]` targets
+# are picked up automatically.
+#
+# A bench target is dropped from the matrix when its crate's Cargo.toml marks
+# it skipped:
+#
+#   [package.metadata.codspeed.benches]
+#   merge_kernels = { skip = true }   # broken at runtime, fix and remove
+#
+# cargo surfaces that table at .packages[].metadata.codspeed.benches, so the
+# skip list lives next to the bench in the crate that owns it.
+#
+# Usage:
+#   codspeed-matrix.sh                # every workspace crate
+#   codspeed-matrix.sh arrow parquet  # only the named crates (must be members)
+
+set -euo pipefail
+
+metadata="$(cargo metadata --format-version 1 --no-deps)"
+
+# Reject explicitly-requested crates that are not workspace members, so a typo
+# in a `bench:<crate>` label fails loudly instead of silently benching nothing.
+if [ "$#" -gt 0 ]; then
+  members="$(jq -r '.packages[].name' <<<"$metadata")"
+  for crate in "$@"; do
+    if ! grep -qxF "$crate" <<<"$members"; then
+      echo "::error::Unknown workspace crate '$crate'" >&2
+      exit 1
+    fi
+  done
+fi
+
+selected="$(printf '%s\n' "$@" | jq -Rsc 'split("\n") | map(select(length > 0))')"
+
+jq -c \
+  --argjson selected "$selected" '
+  [ .packages[]
+    | .name as $crate
+    | (.metadata.codspeed.benches // {}) as $cfg
+    | select(($selected | length) == 0 or ($selected | index($crate)))
+    | .targets[]
+    | select(.kind | index("bench"))
+    | select(($cfg[.name].skip // false) | not)
+    | { crate: $crate, bench: .name }
+  ]
+' <<<"$metadata"
diff --git a/.github/workflows/codspeed-pr.yml b/.github/workflows/codspeed-pr.yml
@@ -80,60 +80,27 @@ jobs:
         id: gen
         env:
           LABELS: ${{ toJSON(github.event.pull_request.labels.*.name) }}
-          # Keep this list in sync with codspeed.yml — bench targets that
-          # currently panic/error at runtime and should not be benched
-          # until fixed in their respective crates.
-          EXCLUDED_BENCHES: |
-            arrow merge_kernels
-            arrow buffer_bit_ops
-            arrow buffer_create
-            arrow sort_kernel
-            arrow string_run_builder
-            arrow primitive_run_accessor
-            arrow-array union_array
-            arrow-cast parse_date
-            parquet row_selection_cursor
-            parquet-variant-compute variant_kernels
+        # Discovery + the known-broken exclusion list live in the shared
+        # codspeed-matrix.sh (also used by codspeed.yml). `bench:all` passes
+        # no crate args (every crate); otherwise each `bench:<crate>` suffix
+        # is forwarded as an arg and validated against the workspace members
+        # by the script.
         run: |
-          all_crates="arrow arrow-array arrow-avro arrow-buffer arrow-cast arrow-ipc arrow-json arrow-schema parquet parquet-variant parquet-variant-compute"
+          suffixes="$(jq -r '.[] | select(startswith("bench:")) | sub("^bench:"; "")' <<<"$LABELS")"
 
-          suffixes=$(jq -r '.[] | select(startswith("bench:")) | sub("^bench:"; "")' <<<"$LABELS")
-
-          if echo "$suffixes" | grep -qx "all"; then
-            selected_crates="$all_crates"
+          if grep -qx "all" <<<"$suffixes"; then
             scope="full workspace (bench:all)"
+            matrix="$(bash .github/workflows/codspeed-matrix.sh)"
           else
-            for pkg in $suffixes; do
-              if ! [[ "$pkg" =~ ^[a-z][a-z0-9_-]*$ ]]; then
-                echo "::error::Invalid bench label suffix 'bench:$pkg'"
-                exit 1
-              fi
-            done
-            selected_crates="$(echo $suffixes | tr '\n' ' ')"
-            scope="$selected_crates"
+            scope="$(echo $suffixes | tr '\n' ' ')"
+            # Intentionally unquoted: each whitespace-separated suffix is a
+            # separate crate argument.
+            matrix="$(bash .github/workflows/codspeed-matrix.sh $suffixes)"
           fi
 
-          {
-            for crate in $selected_crates; do
-              if [ ! -f "$crate/Cargo.toml" ]; then
-                echo "::warning::No Cargo.toml found for '$crate' (bench:$crate); skipping"
-                continue
-              fi
-              awk -v crate="$crate" '
-                /^\[\[bench\]\]/ { in_bench=1; next }
-                /^\[/             { in_bench=0 }
-                in_bench && /^name = / {
-                  sub(/^name = "/, ""); sub(/"$/, "");
-                  printf "%s %s\n", crate, $0
-                }
-              ' "$crate/Cargo.toml"
-            done
-          } | grep -vxF -f <(printf '%s\n' "$EXCLUDED_BENCHES" | sed '/^$/d') \
-            | jq -Rcs 'split("\n") | map(select(length>0) | split(" ") | {crate: .[0], bench: .[1]})' > matrix.json
-
-          echo "matrix=$(cat matrix.json)" >> "$GITHUB_OUTPUT"
+          echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
           echo "scope=$scope" >> "$GITHUB_OUTPUT"
-          echo "::notice::Scope: $scope ($(jq length matrix.json) bench shards after excluding known-broken targets)"
+          echo "::notice::Scope: $scope ($(jq length <<<"$matrix") bench shards, known-broken targets excluded)"
 
   build:
     # Gate on the same label condition as setup so we don't build when
diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml
@@ -28,11 +28,12 @@
 # Topology:
 #
 #   setup ─┐
-#          ├──→ bench (matrix, ~88 jobs)
+#          ├──→ bench (matrix, ~78 jobs)
 #   build ─┘
 #
-# `setup` parses every workspace Cargo.toml's `[[bench]]` entries and
-# emits a {crate, bench} matrix. `build` does the full-workspace
+# `setup` discovers every `[[bench]]` target via `cargo metadata` (see
+# codspeed-matrix.sh) and emits a {crate, bench} matrix. `build` does the
+# full-workspace
 # `cargo codspeed build` exactly once and uploads
 # `target/codspeed/<mode>/` as an artifact. `bench` shards download the
 # artifact and run a single bench target each via the CodSpeed action;
@@ -74,46 +75,15 @@ jobs:
     steps:
       - uses: actions/checkout@v6
 
-      - name: Generate matrix of {crate, bench} from workspace Cargo.tomls
+      - name: Generate {crate, bench} matrix across the workspace
         id: gen
-        env:
-          # Bench targets known to panic / error at runtime as of writing.
-          # These are pre-existing issues in the bench targets themselves,
-          # not the CodSpeed integration; they should be fixed and removed
-          # from this list one by one.
-          #   - arrow / merge_kernels:           panics at arrow-data/src/transform/primitive.rs:31
-          #   - arrow / buffer_bit_ops, buffer_create, sort_kernel,
-          #     string_run_builder, primitive_run_accessor:  fail at runtime
-          #   - arrow-array / union_array, arrow-cast / parse_date:        fail at runtime
-          #   - parquet / row_selection_cursor:                            fails at runtime
-          #   - parquet-variant-compute / variant_kernels:                 intermittent
-          EXCLUDED_BENCHES: |
-            arrow merge_kernels
-            arrow buffer_bit_ops
-            arrow buffer_create
-            arrow sort_kernel
-            arrow string_run_builder
-            arrow primitive_run_accessor
-            arrow-array union_array
-            arrow-cast parse_date
-            parquet row_selection_cursor
-            parquet-variant-compute variant_kernels
+        # Discovery + the known-broken exclusion list live in the shared
+        # codspeed-matrix.sh (also used by codspeed-pr.yml) so they stay in
+        # one place. No args = every workspace crate.
         run: |
-          {
-            for crate in arrow arrow-array arrow-avro arrow-buffer arrow-cast arrow-ipc arrow-json arrow-schema parquet parquet-variant parquet-variant-compute; do
-              awk -v crate="$crate" '
-                /^\[\[bench\]\]/ { in_bench=1; next }
-                /^\[/             { in_bench=0 }
-                in_bench && /^name = / {
-                  sub(/^name = "/, ""); sub(/"$/, "");
-                  printf "%s %s\n", crate, $0
-                }
-              ' "$crate/Cargo.toml"
-            done
-          } | grep -vxF -f <(printf '%s\n' "$EXCLUDED_BENCHES" | sed '/^$/d') \
-            | jq -Rcs 'split("\n") | map(select(length>0) | split(" ") | {crate: .[0], bench: .[1]})' > matrix.json
-          echo "matrix=$(cat matrix.json)" >> "$GITHUB_OUTPUT"
-          echo "::notice::Generated $(jq length matrix.json) bench shards (after excluding known-broken targets)"
+          matrix="$(bash .github/workflows/codspeed-matrix.sh)"
+          echo "matrix=$matrix" >> "$GITHUB_OUTPUT"
+          echo "::notice::Generated $(jq length <<<"$matrix") bench shards (one per bench target, known-broken targets excluded)"
 
   build:
     name: Build workspace benchmarks
diff --git a/arrow-array/Cargo.toml b/arrow-array/Cargo.toml
@@ -61,6 +61,11 @@ hashbrown = { version = "0.17.0", default-features = false }
 [package.metadata.docs.rs]
 all-features = true
 
+[package.metadata.codspeed.benches]
+# Skipped on CodSpeed (read by .github/workflows/codspeed-matrix.sh):
+# currently fails at runtime. Fix the bench and remove this entry.
+union_array = { skip = true }
+
 [features]
 async = ["dep:futures"]
 ffi = ["arrow-schema/ffi", "arrow-data/ffi"]
diff --git a/arrow-cast/Cargo.toml b/arrow-cast/Cargo.toml
@@ -35,6 +35,11 @@ bench = false
 [package.metadata.docs.rs]
 all-features = true
 
+[package.metadata.codspeed.benches]
+# Skipped on CodSpeed (read by .github/workflows/codspeed-matrix.sh):
+# currently fails at runtime. Fix the bench and remove this entry.
+parse_date = { skip = true }
+
 [features]
 prettyprint = ["comfy-table"]
 force_validate = []
diff --git a/arrow/Cargo.toml b/arrow/Cargo.toml
@@ -60,6 +60,19 @@ half = { version = "2.1", default-features = false, features = ["rand_distr"], o
 [package.metadata.docs.rs]
 all-features = true
 
+[package.metadata.codspeed.benches]
+# Per-bench CodSpeed config, read by .github/workflows/codspeed-matrix.sh.
+# `skip = true` drops the bench target from the CodSpeed matrix; these
+# currently panic/error at runtime, so fix the bench and remove its entry,
+# one at a time. (A per-bench `mode = "simulation" | "walltime"` could live
+# here too, once the workflow builds both measurement modes.)
+merge_kernels = { skip = true }
+buffer_bit_ops = { skip = true }
+buffer_create = { skip = true }
+sort_kernel = { skip = true }
+string_run_builder = { skip = true }
+primitive_run_accessor = { skip = true }
+
 [features]
 default = ["csv", "ipc", "json"]
 async = ["arrow-array/async"]
diff --git a/parquet-variant-compute/Cargo.toml b/parquet-variant-compute/Cargo.toml
@@ -27,6 +27,11 @@ keywords = ["arrow", "parquet", "variant"]
 edition = { workspace = true }
 rust-version = { workspace = true }
 
+[package.metadata.codspeed.benches]
+# Skipped on CodSpeed (read by .github/workflows/codspeed-matrix.sh):
+# intermittently fails at runtime. Fix the bench and remove this entry.
+variant_kernels = { skip = true }
+
 [dependencies]
 arrow = { workspace = true, features = ["canonical_extension_types"] }
 arrow-schema = { workspace = true }
diff --git a/parquet/Cargo.toml b/parquet/Cargo.toml
@@ -98,6 +98,11 @@ sysinfo = { version = "0.38.1", default-features = false, features = ["system"]
 [package.metadata.docs.rs]
 all-features = true
 
+[package.metadata.codspeed.benches]
+# Skipped on CodSpeed (read by .github/workflows/codspeed-matrix.sh):
+# currently fails at runtime. Fix the bench and remove this entry.
+row_selection_cursor = { skip = true }
+
 [features]
 default = ["arrow", "snap", "brotli", "flate2-zlib-rs", "lz4", "zstd", "base64", "simdutf8"]
 # Enable lz4
