feat: add clickhouse-bench for ClickBench Parquet benchmarks

Introduce a new clickhouse-bench benchmark crate that runs ClickBench queries
against Parquet data via clickhouse-local, providing a baseline for comparing
Vortex performance against ClickHouse.

Key design decisions:
- The ClickHouse binary is resolved at runtime: via CLICKHOUSE_BINARY env var
  or from $PATH. CI installs it via the official installer before benchmarks.
- ClickHouse-dialect SQL queries use the same query file as other engines
  (ClickHouse handles quoted identifiers correctly).
- Each query spawns a fresh clickhouse-local process for complete isolation
  (no cross-query caching, no warm-up effects).
- CI workflows updated to include clickhouse:parquet target in ClickBench
  benchmarks and conditionally build clickhouse-bench.

Closes #6425

Signed-off-by: fastio <niclas@fastio.me>

Author: fastio

.github/scripts/run-sql-bench.sh

@@ -2,22 +2,22 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright the Vortex contributors
 #
-# Runs SQL benchmarks (datafusion-bench, duckdb-bench, lance-bench) for the given targets.
-# This script is used by the sql-benchmarks.yml workflow.
+# Runs SQL benchmarks (datafusion-bench, duckdb-bench, lance-bench, clickhouse-bench)
+# for the given targets. This script is used by the sql-benchmarks.yml workflow.
 #
 # Usage:
 #   run-sql-bench.sh <subcommand> <targets> [options]
 #
 # Arguments:
 #   subcommand   The benchmark subcommand (e.g., tpch, clickbench, tpcds)
 #   targets      Comma-separated list of engine:format pairs
-#                (e.g., "datafusion:parquet,datafusion:vortex,duckdb:parquet")
+#                (e.g., "datafusion:parquet,datafusion:vortex,duckdb:parquet,clickhouse:parquet")
 #
 # Options:
 #   --scale-factor <sf>       Scale factor for the benchmark (e.g., 1.0, 10.0)
 #   --iterations <n>          Number of iterations to pass to each benchmark binary
 #   --remote-storage <url>    Remote storage URL (e.g., s3://bucket/path/)
-#                             If provided, runs in remote mode (no lance support).
+#                             If provided, runs in remote mode (no lance/clickhouse support).
 #   --benchmark-id <id>       Benchmark ID for error messages (e.g., tpch-s3)
 
 set -Eeu -o pipefail
@@ -71,6 +71,13 @@ if $is_remote && echo "$targets" | grep -q 'lance'; then
     exit 1
 fi
 
+# ClickHouse on remote storage is not supported. clickhouse-local reads local files only.
+if $is_remote && echo "$targets" | grep -q 'clickhouse:'; then
+    echo "ERROR: ClickHouse benchmarks are not supported for remote storage."
+    echo "Remove 'clickhouse:' targets for benchmark '${benchmark_id:-unknown}'."
+    exit 1
+fi
+
 # Extract formats for each engine from the targets string.
 # Example input: "datafusion:parquet,datafusion:vortex,datafusion:lance,duckdb:parquet"
 #
@@ -84,6 +91,7 @@ fi
 df_formats=$(echo "$targets" | tr ',' '\n' | (grep '^datafusion:' | grep -v ':lance$' || true) | sed 's/datafusion://' | tr '\n' ',' | sed 's/,$//')
 ddb_formats=$(echo "$targets" | tr ',' '\n' | (grep '^duckdb:' || true) | sed 's/duckdb://' | tr '\n' ',' | sed 's/,$//')
 has_lance=$(echo "$targets" | grep -q 'datafusion:lance' && echo "true" || echo "false")
+has_clickhouse=$(echo "$targets" | grep -q 'clickhouse:' && echo "true" || echo "false")
 
 # Build options string.
 opts=""
@@ -136,3 +144,14 @@ if ! $is_remote && [[ "$has_lance" == "true" ]] && [[ -f "target/release_debug/l
 
     cat lance-results.json >> results.json
 fi
+
+# ClickHouse-bench only runs for local benchmarks (clickhouse-local reads local files).
+if ! $is_remote && [[ "$has_clickhouse" == "true" ]] && [[ -f "target/release_debug/clickhouse-bench" ]]; then
+    # shellcheck disable=SC2086
+    target/release_debug/clickhouse-bench "$subcommand" \
+        -d gh-json \
+        $opts \
+        -o ch-results.json
+
+    cat ch-results.json >> results.json
+fi

.github/workflows/bench.yml

@@ -121,7 +121,7 @@ jobs:
             "id": "clickbench-nvme",
             "subcommand": "clickbench",
             "name": "Clickbench on NVME",
-            "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,datafusion:lance,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb",
+            "targets": "datafusion:parquet,datafusion:vortex,datafusion:vortex-compact,datafusion:lance,duckdb:parquet,duckdb:vortex,duckdb:vortex-compact,duckdb:duckdb,clickhouse:parquet",
             "build_lance": true
           },
           {

.github/workflows/sql-benchmarks.yml

@@ -21,7 +21,7 @@ on:
               "id": "clickbench-nvme",
               "subcommand": "clickbench",
               "name": "Clickbench on NVME",
-              "targets": "datafusion:parquet,datafusion:vortex,duckdb:parquet,duckdb:vortex,duckdb:duckdb"
+              "targets": "datafusion:parquet,datafusion:vortex,duckdb:parquet,duckdb:vortex,duckdb:duckdb,clickhouse:parquet"
             },
             {
               "id": "tpch-nvme",
@@ -135,6 +135,16 @@ jobs:
 
       - uses: ./.github/actions/system-info
 
+      - name: Install ClickHouse
+        if: contains(matrix.targets, 'clickhouse:')
+        env:
+          CLICKHOUSE_VERSION: "25.8.18.1"
+        run: |
+          wget -qO- "https://github.com/ClickHouse/ClickHouse/releases/download/v${CLICKHOUSE_VERSION}-lts/clickhouse-common-static-${CLICKHOUSE_VERSION}-amd64.tgz" | tar xz
+          cp clickhouse-common-static-${CLICKHOUSE_VERSION}/usr/bin/clickhouse .
+          chmod +x clickhouse
+          echo "CLICKHOUSE_BINARY=$PWD/clickhouse" >> $GITHUB_ENV
+
       - name: Build binaries
         shell: bash
         env:
@@ -144,6 +154,9 @@ jobs:
           if [ "${{ matrix.build_lance }}" = "true" ]; then
             packages="$packages --bin lance-bench"
           fi
+          if echo "${{ matrix.targets }}" | grep -q 'clickhouse:'; then
+            packages="$packages --bin clickhouse-bench"
+          fi
           cargo build $packages --profile release_debug
 
       - name: Generate data

Cargo.lock

@@ -49,6 +49,7 @@ members = [
     "encodings/zstd",
     "encodings/bytebool",
     # Benchmarks
+    "benchmarks/clickhouse-bench",
     "benchmarks/lance-bench",
     "benchmarks/compress-bench",
     "benchmarks/datafusion-bench",

Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "clickhouse-bench"
+description = "ClickHouse (clickhouse-local) benchmark runner for Vortex"
+authors.workspace = true
+edition.workspace = true
+homepage.workspace = true
+license.workspace = true
+readme.workspace = true
+repository.workspace = true
+rust-version.workspace = true
+version.workspace = true
+publish = false
+
+[dependencies]
+anyhow = { workspace = true }
+clap = { workspace = true, features = ["derive"] }
+parking_lot = { workspace = true }
+tokio = { workspace = true }
+tracing = { workspace = true }
+vortex-bench = { workspace = true }
+
+[lints]
+workspace = true

benchmarks/clickhouse-bench/Cargo.toml

@@ -0,0 +1,18 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+//! Build script that exports the ClickHouse binary path.
+//!
+//! Resolution order:
+//! 1. `CLICKHOUSE_BINARY` env var — use as-is.
+//! 2. Falls back to `"clickhouse"` (i.e., resolve from `$PATH` at runtime).
+//!
+//! Users must install ClickHouse themselves for local runs.
+//! In CI, it is installed via the workflow before the benchmark step.
+
+fn main() {
+    println!("cargo:rerun-if-env-changed=CLICKHOUSE_BINARY");
+
+    let binary = std::env::var("CLICKHOUSE_BINARY").unwrap_or_else(|_| "clickhouse".to_string());
+    println!("cargo:rustc-env=CLICKHOUSE_BINARY={binary}");
+}