Own SQL benchmark suite (#8719)

myrrc · web-flow · commit 1c419e530cd9 · 2026-07-16T10:47:21.000Z
Testing SQL queries' correctness is done in vortex-sqllogictest, but we
also
want to test relative performance of queries not covered in existing
benchmarks.
One example is aggregation with filter which should be much more
performant
once we use zone maps statistics, but this type of query isn't present
is
existing benchmarks.

This PR introduces a new query-per-file dataset and support for running
it in CI

Signed-off-by: Mikhail Kot &lt;mikhail@spiraldb.com&gt;
diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
@@ -171,3 +171,31 @@ jobs:
     secrets: inherit
     with:
       mode: "develop"
+
+  sql-vortex:
+    uses: ./.github/workflows/sql-benchmarks.yml
+    secrets: inherit
+    with:
+      mode: "develop"
+      benchmark_matrix: |
+        [
+          {
+            "id": "vortex-queries",
+            "subcommand": "vortex",
+            "name": "Vortex queries",
+            "data_formats": ["parquet", "vortex"],
+            "pr_targets": [
+              {"engine": "datafusion", "format": "parquet"},
+              {"engine": "datafusion", "format": "vortex"},
+              {"engine": "duckdb", "format": "parquet"},
+              {"engine": "duckdb", "format": "vortex"}
+            ],
+            "develop_targets": [
+              {"engine": "datafusion", "format": "parquet"},
+              {"engine": "datafusion", "format": "vortex"},
+              {"engine": "duckdb", "format": "parquet"},
+              {"engine": "duckdb", "format": "vortex"}
+            ],
+            "iterations": "100"
+          }
+        ]
diff --git a/.github/workflows/sql-vortex-pr.yml b/.github/workflows/sql-vortex-pr.yml
@@ -0,0 +1,47 @@
+# Run Vortex SQL benchmark on every commit in a PR.
+# This is not gated behind action/benchmark-sql
+
+name: PR Vortex Query Benchmark
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: false
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    branches: ["develop"]
+
+permissions:
+  contents: read
+  pull-requests: write
+  id-token: write
+
+jobs:
+  sql:
+    uses: ./.github/workflows/sql-benchmarks.yml
+    secrets: inherit
+    with:
+      mode: "pr"
+      benchmark_matrix: |
+        [
+          {
+            "id": "vortex-queries",
+            "subcommand": "vortex",
+            "name": "Vortex queries",
+            "data_formats": ["parquet", "vortex"],
+            "pr_targets": [
+              {"engine": "datafusion", "format": "parquet"},
+              {"engine": "datafusion", "format": "vortex"},
+              {"engine": "duckdb", "format": "parquet"},
+              {"engine": "duckdb", "format": "vortex"}
+            ],
+            "develop_targets": [
+              {"engine": "datafusion", "format": "parquet"},
+              {"engine": "datafusion", "format": "vortex"},
+              {"engine": "duckdb", "format": "parquet"},
+              {"engine": "duckdb", "format": "vortex"}
+            ],
+            "iterations": "100"
+          }
+        ]
diff --git a/bench-orchestrator/bench_orchestrator/config.py b/bench-orchestrator/bench_orchestrator/config.py
@@ -54,6 +54,7 @@ class Benchmark(Enum):
     PUBLIC_BI = "public-bi"
     STATPOPGEN = "statpopgen"
     SPATIALBENCH = "spatialbench"
+    VORTEX_QUERIES = "vortex"
 
 
 # Engine to supported formats mapping.
diff --git a/vortex-bench/sql/vortex/0_sum-with-filter.sql b/vortex-bench/sql/vortex/0_sum-with-filter.sql
@@ -0,0 +1,4 @@
+-- As this aggregation has a filter, Vortex has to use a linear scan. Once stats
+-- are propagated to arrays, this should use zone maps to aggregate instead of
+-- decoding and processing each row.
+SELECT sum(col) FROM test WHERE col2 > 0 AND col2 < 1000;
diff --git a/vortex-bench/sql/vortex/1_sum.sql b/vortex-bench/sql/vortex/1_sum.sql
@@ -0,0 +1,3 @@
+-- When Footer changes land, vortex-duckdb should populate statistics from
+-- Footer without loading and decoding the data.
+SELECT sum(col) FROM test;
diff --git a/vortex-bench/sql/vortex/init.sql b/vortex-bench/sql/vortex/init.sql
@@ -0,0 +1,8 @@
+-- Script that prepares Parquet data for our SQL microbenchmarks.
+
+COPY (
+    SELECT
+        i % 1000 AS col,
+        (i * 2654435761) % 100000 AS col2
+    FROM range(25000000) t(i)
+) TO 'test.parquet' (FORMAT parquet);
diff --git a/vortex-bench/src/datasets/mod.rs b/vortex-bench/src/datasets/mod.rs
@@ -81,6 +81,8 @@ pub enum BenchmarkDataset {
     Fineweb,
     #[serde(rename = "gharchive")]
     GhArchive,
+    #[serde(rename = "vortex")]
+    VortexQueries,
 }
 
 impl BenchmarkDataset {
@@ -97,6 +99,7 @@ impl BenchmarkDataset {
             BenchmarkDataset::PolarSignals { .. } => "polarsignals",
             BenchmarkDataset::Fineweb => "fineweb",
             BenchmarkDataset::GhArchive => "gharchive",
+            BenchmarkDataset::VortexQueries => "vortex",
         }
     }
 }
@@ -122,6 +125,7 @@ impl Display for BenchmarkDataset {
             }
             BenchmarkDataset::Fineweb => write!(f, "fineweb"),
             BenchmarkDataset::GhArchive => write!(f, "gharchive"),
+            BenchmarkDataset::VortexQueries => write!(f, "vortex"),
         }
     }
 }
@@ -179,6 +183,8 @@ impl BenchmarkDataset {
             BenchmarkDataset::PolarSignals { .. } => &["stacktraces"],
             BenchmarkDataset::Fineweb => &["fineweb"],
             BenchmarkDataset::GhArchive => &["events"],
+            // See VortexBenchmark::table_specs
+            BenchmarkDataset::VortexQueries => &[],
         }
     }
 }
diff --git a/vortex-bench/src/lib.rs b/vortex-bench/src/lib.rs
@@ -36,6 +36,7 @@ use vortex::file::WriteStrategyBuilder;
 use vortex::utils::aliases::hash_map::HashMap;
 
 use crate::spatialbench::SpatialBenchBenchmark;
+use crate::vortex_queries::VortexBenchmark;
 
 pub mod appian;
 pub mod benchmark;
@@ -61,6 +62,7 @@ pub mod tpch;
 pub mod utils;
 pub mod v3;
 pub mod vector_dataset;
+pub mod vortex_queries;
 
 pub use benchmark::Benchmark;
 pub use benchmark::TableSpec;
@@ -281,6 +283,8 @@ pub enum BenchmarkArg {
     PublicBi,
     #[clap(name = "spatialbench")]
     SpatialBench,
+    #[clap(name = "vortex")]
+    VortexQueries,
 }
 
 /// Default scale factor for TPC-related benchmarks
@@ -353,6 +357,13 @@ pub fn create_benchmark(b: BenchmarkArg, opts: &Opts) -> anyhow::Result<Box<dyn
             let benchmark = SpatialBenchBenchmark::new(scale_factor.to_string(), remote_data_dir)?;
             Ok(Box::new(benchmark) as _)
         }
+        BenchmarkArg::VortexQueries => {
+            let mut benchmark = VortexBenchmark::new()?;
+            if let Some(query) = opts.get("query") {
+                benchmark = benchmark.with_query(query)?;
+            }
+            Ok(Box::new(benchmark) as _)
+        }
     }
 }
 
diff --git a/vortex-bench/src/utils/file.rs b/vortex-bench/src/utils/file.rs
@@ -56,8 +56,12 @@ pub trait IdempotentPath {
     fn to_data_path(&self) -> PathBuf;
 }
 
+pub fn bench_dir() -> PathBuf {
+    workspace_root().join("vortex-bench")
+}
+
 pub fn data_dir() -> PathBuf {
-    workspace_root().join("vortex-bench").join("data")
+    bench_dir().join("data")
 }
 
 /// Find the workspace's root by looking for Cargo's lock file
diff --git a/vortex-bench/src/v3.rs b/vortex-bench/src/v3.rs
@@ -296,6 +296,7 @@ fn canonical_tpc_scale_factor(scale_factor: &str) -> String {
 /// | `Appian`                    | `appian`       | `None`              | `None`                                              | Static dataset; no scale factor. |
 /// | `PublicBi { name }`         | `public-bi`    | dataset name (e.g. `cms-provider`) | `None`               | Sub-dataset name lives in `dataset_variant`. |
 /// | `SpatialBench { scale_factor }` | `spatialbench` | `None`         | SF as string | Same canonicalization as TPC-H; no historical v2 records to merge with. |
+/// | `VortexQueries` | `vortex` | `None` | `None` | Own microbenchmarks |
 pub fn benchmark_dataset_dims(d: &BenchmarkDataset) -> (String, Option<String>, Option<String>) {
     match d {
         BenchmarkDataset::TpcH { scale_factor } => (
@@ -331,6 +332,7 @@ pub fn benchmark_dataset_dims(d: &BenchmarkDataset) -> (String, Option<String>,
         BenchmarkDataset::Fineweb => ("fineweb".to_string(), None, None),
         BenchmarkDataset::GhArchive => ("gharchive".to_string(), None, None),
         BenchmarkDataset::Appian => ("appian".to_string(), None, None),
+        BenchmarkDataset::VortexQueries => ("vortex".to_string(), None, None),
     }
 }
 
diff --git a/vortex-bench/src/vortex_queries.rs b/vortex-bench/src/vortex_queries.rs
@@ -0,0 +1,179 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+//! Benchmark that runs queries which aren't part of any existing benchmark
+//! suite but which performance we want to track.
+
+use std::fs;
+use std::path::PathBuf;
+use std::process::Command;
+
+use anyhow::Context;
+use anyhow::Result;
+use anyhow::anyhow;
+use anyhow::bail;
+use glob::Pattern;
+use tracing::debug;
+use url::Url;
+use vortex::error::VortexExpect;
+
+use crate::Benchmark;
+use crate::BenchmarkDataset;
+use crate::Format;
+use crate::TableSpec;
+use crate::bench_dir;
+use crate::resolve_data_url;
+
+// Path to script that creates Parquet data
+const INIT_SQL: &str = "init.sql";
+
+pub struct VortexBenchmark {
+    data_url: Url,
+    queries_dir: PathBuf,
+    query: Option<PathBuf>,
+}
+
+impl VortexBenchmark {
+    pub fn new() -> Result<Self> {
+        let queries_dir = bench_dir().join("sql").join("vortex");
+        let data_url = resolve_data_url(None, "vortex")?;
+        Ok(Self {
+            data_url,
+            queries_dir,
+            query: None,
+        })
+    }
+
+    // Same as new(), but run only for "query" SQL file
+    pub fn with_query(mut self, query: &str) -> Result<Self> {
+        let as_path = PathBuf::from(query);
+        let path = if as_path.is_file() {
+            as_path
+        } else if query.ends_with(".sql") {
+            self.queries_dir.join(query)
+        } else {
+            self.queries_dir.join(format!("{query}.sql"))
+        };
+        if !path.is_file() {
+            bail!("{query} file not found in {}", self.queries_dir.display());
+        }
+        self.query = Some(path);
+        Ok(self)
+    }
+
+    fn query_files(&self) -> Result<Vec<PathBuf>> {
+        if let Some(query) = &self.query {
+            return Ok(vec![query.clone()]);
+        }
+
+        let entries = fs::read_dir(&self.queries_dir)
+            .with_context(|| format!("cannot list queries in {}", self.queries_dir.display()))?
+            .collect::<std::io::Result<Vec<_>>>()?;
+
+        let mut files: Vec<PathBuf> = entries
+            .into_iter()
+            .map(|entry| entry.path())
+            .filter(|path| {
+                path.extension().is_some_and(|ext| ext == "sql")
+                    && path.file_name().is_some_and(|name| name != INIT_SQL)
+            })
+            .collect();
+        files.sort();
+
+        if files.is_empty() {
+            bail!("no query files found in {}", self.queries_dir.display());
+        }
+        Ok(files)
+    }
+}
+
+#[async_trait::async_trait]
+impl Benchmark for VortexBenchmark {
+    fn queries(&self) -> Result<Vec<(usize, String)>> {
+        self.query_files()?
+            .iter()
+            .map(|path| {
+                let idx = path
+                    .file_name()
+                    .vortex_expect("no file name")
+                    .to_str()
+                    .vortex_expect("not utf-8")
+                    .split_once("_")
+                    .vortex_expect("query without a number")
+                    .0
+                    .parse::<usize>()?;
+                let query = fs::read_to_string(path)
+                    .with_context(|| format!("cannot read query {}", path.display()))?;
+                debug!(idx, file = %path.display(), "Loaded vortex query");
+                Ok((idx, query))
+            })
+            .collect()
+    }
+
+    async fn generate_base_data(&self) -> Result<()> {
+        let data_dir = self
+            .data_url
+            .to_file_path()
+            .map_err(|_| anyhow!("Invalid file URL: {}", self.data_url.as_str()))?;
+        let parquet_dir = data_dir.join(Format::Parquet.name());
+        fs::create_dir_all(&parquet_dir)?;
+        let parquet_file = parquet_dir.join("test.parquet");
+
+        if parquet_file.exists() {
+            debug!("Parquet data present in {}", parquet_dir.display());
+            return Ok(());
+        }
+
+        let init_path = self.queries_dir.join(INIT_SQL);
+        let script = fs::read_to_string(&init_path)
+            .with_context(|| format!("cannot read {}", init_path.display()))?;
+
+        let output = Command::new("duckdb")
+            .current_dir(&parquet_dir)
+            .arg("-c")
+            .arg(&script)
+            .output()
+            .context("cannot run duckdb")?;
+        if !output.status.success() {
+            bail!(
+                "duckdb {INIT_SQL} failed: stdout={:?} stderr={:?}",
+                String::from_utf8_lossy(&output.stdout),
+                String::from_utf8_lossy(&output.stderr),
+            );
+        }
+
+        if !parquet_file.exists() {
+            bail!("{INIT_SQL} did not create Parquet files");
+        }
+
+        debug!("Parquet data generated in {}", parquet_dir.display());
+        Ok(())
+    }
+
+    fn dataset(&self) -> BenchmarkDataset {
+        BenchmarkDataset::VortexQueries
+    }
+
+    fn dataset_name(&self) -> &str {
+        "vortex"
+    }
+
+    fn dataset_display(&self) -> String {
+        "vortex".to_owned()
+    }
+
+    fn data_url(&self) -> &Url {
+        &self.data_url
+    }
+
+    fn table_specs(&self) -> Vec<TableSpec> {
+        vec![TableSpec::new("test", None)]
+    }
+
+    fn pattern(&self, table_name: &str, format: Format) -> Option<Pattern> {
+        Some(
+            Pattern::new(&format!("{table_name}.{}", format.ext()))
+                .expect("table name is a valid identifier"),
+        )
+    }
+}

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,3 @@`
	`1`	`+-- When Footer changes land, vortex-duckdb should populate statistics from`
	`2`	`+-- Footer without loading and decoding the data.`
	`3`	`+SELECT sum(col) FROM test;`
Original file line number	Diff line number	Diff line change
`@@ -81,6 +81,8 @@ pub enum BenchmarkDataset {`
`81`	`81`	`Fineweb,`
`82`	`82`	`#[serde(rename = "gharchive")]`
`83`	`83`	`GhArchive,`
	`84`	`+ #[serde(rename = "vortex")]`
	`85`	`+ VortexQueries,`
`84`	`86`	`}`
`85`	`87`
`86`	`88`	`impl BenchmarkDataset {`
`@@ -97,6 +99,7 @@ impl BenchmarkDataset {`
`97`	`99`	`BenchmarkDataset::PolarSignals { .. } => "polarsignals",`
`98`	`100`	`BenchmarkDataset::Fineweb => "fineweb",`
`99`	`101`	`BenchmarkDataset::GhArchive => "gharchive",`
	`102`	`+ BenchmarkDataset::VortexQueries => "vortex",`
`100`	`103`	`}`
`101`	`104`	`}`
`102`	`105`	`}`
`@@ -122,6 +125,7 @@ impl Display for BenchmarkDataset {`
`122`	`125`	`}`
`123`	`126`	`BenchmarkDataset::Fineweb => write!(f, "fineweb"),`
`124`	`127`	`BenchmarkDataset::GhArchive => write!(f, "gharchive"),`
	`128`	`+ BenchmarkDataset::VortexQueries => write!(f, "vortex"),`
`125`	`129`	`}`
`126`	`130`	`}`
`127`	`131`	`}`
`@@ -179,6 +183,8 @@ impl BenchmarkDataset {`
`179`	`183`	`BenchmarkDataset::PolarSignals { .. } => &["stacktraces"],`
`180`	`184`	`BenchmarkDataset::Fineweb => &["fineweb"],`
`181`	`185`	`BenchmarkDataset::GhArchive => &["events"],`
	`186`	`+ // See VortexBenchmark::table_specs`
	`187`	`+ BenchmarkDataset::VortexQueries => &[],`
`182`	`188`	`}`
`183`	`189`	`}`
`184`	`190`	`}`
Original file line number	Diff line number	Diff line change
`@@ -56,8 +56,12 @@ pub trait IdempotentPath {`
`56`	`56`	`fn to_data_path(&self) -> PathBuf;`
`57`	`57`	`}`
`58`	`58`
	`59`	`+pub fn bench_dir() -> PathBuf {`
	`60`	`+ workspace_root().join("vortex-bench")`
	`61`	`+}`
	`62`	`+`
`59`	`63`	`pub fn data_dir() -> PathBuf {`
`60`		`- workspace_root().join("vortex-bench").join("data")`
	`64`	`+ bench_dir().join("data")`
`61`	`65`	`}`
`62`	`66`
`63`	`67`	`/// Find the workspace's root by looking for Cargo's lock file`
Original file line number	Diff line number	Diff line change
`@@ -296,6 +296,7 @@ fn canonical_tpc_scale_factor(scale_factor: &str) -> String {`
`296`	`296`	/// \| `Appian` \| `appian` \| `None` \| `None` \| Static dataset; no scale factor. \|
`297`	`297`	/// \| `PublicBi { name }` \| `public-bi` \| dataset name (e.g. `cms-provider`) \| `None` \| Sub-dataset name lives in `dataset_variant`. \|
`298`	`298`	/// \| `SpatialBench { scale_factor }` \| `spatialbench` \| `None` \| SF as string \| Same canonicalization as TPC-H; no historical v2 records to merge with. \|
	`299`	+/// \| `VortexQueries` \| `vortex` \| `None` \| `None` \| Own microbenchmarks \|
`299`	`300`	`pub fn benchmark_dataset_dims(d: &BenchmarkDataset) -> (String, Option<String>, Option<String>) {`
`300`	`301`	`match d {`
`301`	`302`	`BenchmarkDataset::TpcH { scale_factor } => (`
`@@ -331,6 +332,7 @@ pub fn benchmark_dataset_dims(d: &BenchmarkDataset) -> (String, Option<String>,`
`331`	`332`	`BenchmarkDataset::Fineweb => ("fineweb".to_string(), None, None),`
`332`	`333`	`BenchmarkDataset::GhArchive => ("gharchive".to_string(), None, None),`
`333`	`334`	`BenchmarkDataset::Appian => ("appian".to_string(), None, None),`
	`335`	`+ BenchmarkDataset::VortexQueries => ("vortex".to_string(), None, None),`
`334`	`336`	`}`
`335`	`337`	`}`
`336`	`338`