Skip to content

Commit c16907b

Browse files
committed
initial
Signed-off-by: Mikhail Kot <mikhail@spiraldb.com>
1 parent d96e4ad commit c16907b

9 files changed

Lines changed: 242 additions & 1 deletion

File tree

.github/workflows/sql-benchmarks.yml

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -298,6 +298,25 @@ on:
298298
{"engine": "duckdb", "format": "duckdb"}
299299
],
300300
"iterations": "10"
301+
},
302+
{
303+
"id": "vortex-queries",
304+
"subcommand": "vortex",
305+
"name": "Vortex queries",
306+
"data_formats": ["parquet", "vortex"],
307+
"pr_targets": [
308+
{"engine": "datafusion", "format": "parquet"},
309+
{"engine": "datafusion", "format": "vortex"},
310+
{"engine": "duckdb", "format": "parquet"},
311+
{"engine": "duckdb", "format": "vortex"}
312+
],
313+
"develop_targets": [
314+
{"engine": "datafusion", "format": "parquet"},
315+
{"engine": "datafusion", "format": "vortex"},
316+
{"engine": "duckdb", "format": "parquet"},
317+
{"engine": "duckdb", "format": "vortex"}
318+
],
319+
"iterations": "100"
301320
}
302321
]
303322
base_benchmark_matrix:
@@ -499,6 +518,25 @@ on:
499518
{"engine": "datafusion", "format": "vortex"}
500519
],
501520
"scale_factor": "1"
521+
},
522+
{
523+
"id": "vortex-queries",
524+
"subcommand": "vortex",
525+
"name": "Vortex queries",
526+
"data_formats": ["parquet", "vortex"],
527+
"pr_targets": [
528+
{"engine": "datafusion", "format": "parquet"},
529+
{"engine": "datafusion", "format": "vortex"},
530+
{"engine": "duckdb", "format": "parquet"},
531+
{"engine": "duckdb", "format": "vortex"}
532+
],
533+
"develop_targets": [
534+
{"engine": "datafusion", "format": "parquet"},
535+
{"engine": "datafusion", "format": "vortex"},
536+
{"engine": "duckdb", "format": "parquet"},
537+
{"engine": "duckdb", "format": "vortex"}
538+
],
539+
"iterations": "100"
502540
}
503541
]
504542
benchmark_profile:

bench-orchestrator/bench_orchestrator/config.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ class Benchmark(Enum):
5454
PUBLIC_BI = "public-bi"
5555
STATPOPGEN = "statpopgen"
5656
SPATIALBENCH = "spatialbench"
57+
VORTEX_QUERIES = "vortex"
5758

5859

5960
# Engine to supported formats mapping.

vortex-bench/.gitignore

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,6 @@
1-
data/
1+
data/*
2+
!data/vortex/
3+
data/vortex/*
4+
!data/vortex/*.sql
5+
!data/vortex/README.md
26
public_bi/
Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
SELECT sum(col) from 'test.vortex' WHERE col2 > 0 AND col2 < 1000;

vortex-bench/data/vortex/init.sql

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
-- Script that prepares Parquet data for our SQL microbenchmarks.
2+
3+
COPY (
4+
SELECT
5+
i % 1000 AS col,
6+
(i * 2654435761) % 100000 AS col2
7+
FROM range(25000000) t(i)
8+
) TO 'test.parquet' (FORMAT parquet);

vortex-bench/src/datasets/mod.rs

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,8 @@ pub enum BenchmarkDataset {
8181
Fineweb,
8282
#[serde(rename = "gharchive")]
8383
GhArchive,
84+
#[serde(rename = "vortex")]
85+
VortexQueries,
8486
}
8587

8688
impl BenchmarkDataset {
@@ -97,6 +99,7 @@ impl BenchmarkDataset {
9799
BenchmarkDataset::PolarSignals { .. } => "polarsignals",
98100
BenchmarkDataset::Fineweb => "fineweb",
99101
BenchmarkDataset::GhArchive => "gharchive",
102+
BenchmarkDataset::VortexQueries => "vortex",
100103
}
101104
}
102105
}
@@ -122,6 +125,7 @@ impl Display for BenchmarkDataset {
122125
}
123126
BenchmarkDataset::Fineweb => write!(f, "fineweb"),
124127
BenchmarkDataset::GhArchive => write!(f, "gharchive"),
128+
BenchmarkDataset::VortexQueries => write!(f, "vortex"),
125129
}
126130
}
127131
}
@@ -179,6 +183,8 @@ impl BenchmarkDataset {
179183
BenchmarkDataset::PolarSignals { .. } => &["stacktraces"],
180184
BenchmarkDataset::Fineweb => &["fineweb"],
181185
BenchmarkDataset::GhArchive => &["events"],
186+
// See VortexBenchmark::table_specs
187+
BenchmarkDataset::VortexQueries => &[],
182188
}
183189
}
184190
}

vortex-bench/src/lib.rs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ use vortex::file::WriteStrategyBuilder;
3636
use vortex::utils::aliases::hash_map::HashMap;
3737

3838
use crate::spatialbench::SpatialBenchBenchmark;
39+
use crate::vortex_queries::VortexBenchmark;
3940

4041
pub mod appian;
4142
pub mod benchmark;
@@ -61,6 +62,7 @@ pub mod tpch;
6162
pub mod utils;
6263
pub mod v3;
6364
pub mod vector_dataset;
65+
pub mod vortex_queries;
6466

6567
pub use benchmark::Benchmark;
6668
pub use benchmark::TableSpec;
@@ -281,6 +283,8 @@ pub enum BenchmarkArg {
281283
PublicBi,
282284
#[clap(name = "spatialbench")]
283285
SpatialBench,
286+
#[clap(name = "vortex")]
287+
VortexQueries,
284288
}
285289

286290
/// Default scale factor for TPC-related benchmarks
@@ -353,6 +357,13 @@ pub fn create_benchmark(b: BenchmarkArg, opts: &Opts) -> anyhow::Result<Box<dyn
353357
let benchmark = SpatialBenchBenchmark::new(scale_factor.to_string(), remote_data_dir)?;
354358
Ok(Box::new(benchmark) as _)
355359
}
360+
BenchmarkArg::VortexQueries => {
361+
let mut benchmark = VortexBenchmark::new()?;
362+
if let Some(query) = opts.get("query") {
363+
benchmark = benchmark.with_query(query)?;
364+
}
365+
Ok(Box::new(benchmark) as _)
366+
}
356367
}
357368
}
358369

vortex-bench/src/v3.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -296,6 +296,7 @@ fn canonical_tpc_scale_factor(scale_factor: &str) -> String {
296296
/// | `Appian` | `appian` | `None` | `None` | Static dataset; no scale factor. |
297297
/// | `PublicBi { name }` | `public-bi` | dataset name (e.g. `cms-provider`) | `None` | Sub-dataset name lives in `dataset_variant`. |
298298
/// | `SpatialBench { scale_factor }` | `spatialbench` | `None` | SF as string | Same canonicalization as TPC-H; no historical v2 records to merge with. |
299+
/// | `VortexQueries` | `vortex` | `None` | `None` | Own microbenchmarks |
299300
pub fn benchmark_dataset_dims(d: &BenchmarkDataset) -> (String, Option<String>, Option<String>) {
300301
match d {
301302
BenchmarkDataset::TpcH { scale_factor } => (
@@ -331,6 +332,7 @@ pub fn benchmark_dataset_dims(d: &BenchmarkDataset) -> (String, Option<String>,
331332
BenchmarkDataset::Fineweb => ("fineweb".to_string(), None, None),
332333
BenchmarkDataset::GhArchive => ("gharchive".to_string(), None, None),
333334
BenchmarkDataset::Appian => ("appian".to_string(), None, None),
335+
BenchmarkDataset::VortexQueries => ("vortex".to_string(), None, None),
334336
}
335337
}
336338

vortex-bench/src/vortex_queries.rs

Lines changed: 170 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,170 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
use std::fs;
5+
use std::path::PathBuf;
6+
use std::process::Command;
7+
8+
use anyhow::Context;
9+
use anyhow::Result;
10+
use anyhow::anyhow;
11+
use anyhow::bail;
12+
use glob::Pattern;
13+
use tracing::debug;
14+
use url::Url;
15+
use vortex::error::VortexExpect;
16+
17+
use crate::Benchmark;
18+
use crate::BenchmarkDataset;
19+
use crate::Format;
20+
use crate::TableSpec;
21+
use crate::utils::file::data_dir;
22+
23+
// Path to script that creates Parquet data
24+
const INIT_SQL: &str = "init.sql";
25+
26+
pub struct VortexBenchmark {
27+
data_url: Url,
28+
queries_dir: PathBuf,
29+
query: Option<PathBuf>,
30+
}
31+
32+
impl VortexBenchmark {
33+
pub fn new() -> Result<Self> {
34+
let queries_dir = data_dir().join("vortex");
35+
let data_url = Url::from_directory_path(&queries_dir)
36+
.map_err(|_| anyhow!("cannot build URL for {}", queries_dir.display()))?;
37+
Ok(Self {
38+
data_url,
39+
queries_dir,
40+
query: None,
41+
})
42+
}
43+
44+
// Same as new(), but run only for "query" SQL file
45+
pub fn with_query(mut self, query: &str) -> Result<Self> {
46+
let as_path = PathBuf::from(query);
47+
let path = if as_path.is_file() {
48+
as_path
49+
} else if query.ends_with(".sql") {
50+
self.queries_dir.join(query)
51+
} else {
52+
self.queries_dir.join(format!("{query}.sql"))
53+
};
54+
if !path.is_file() {
55+
bail!("{query} file not found in {}", self.queries_dir.display());
56+
}
57+
self.query = Some(path);
58+
Ok(self)
59+
}
60+
61+
fn query_files(&self) -> Result<Vec<PathBuf>> {
62+
if let Some(query) = &self.query {
63+
return Ok(vec![query.clone()]);
64+
}
65+
66+
let entries = fs::read_dir(&self.queries_dir)
67+
.with_context(|| format!("cannot list queries in {}", self.queries_dir.display()))?
68+
.collect::<std::io::Result<Vec<_>>>()?;
69+
70+
let mut files: Vec<PathBuf> = entries
71+
.into_iter()
72+
.map(|entry| entry.path())
73+
.filter(|path| {
74+
path.extension().is_some_and(|ext| ext == "sql")
75+
&& path.file_name().is_some_and(|name| name != INIT_SQL)
76+
})
77+
.collect();
78+
files.sort();
79+
80+
if files.is_empty() {
81+
bail!("no query files found in {}", self.queries_dir.display());
82+
}
83+
Ok(files)
84+
}
85+
}
86+
87+
#[async_trait::async_trait]
88+
impl Benchmark for VortexBenchmark {
89+
fn queries(&self) -> Result<Vec<(usize, String)>> {
90+
self.query_files()?
91+
.iter()
92+
.map(|path| {
93+
let idx: usize = path
94+
.to_string_lossy()
95+
.split_once("_")
96+
.vortex_expect("query without a number")
97+
.0
98+
.parse::<usize>()?;
99+
let query = fs::read_to_string(path)
100+
.with_context(|| format!("cannot read query {}", path.display()))?
101+
.replace("test.vortex", "$1");
102+
debug!(idx, file = %path.display(), "Loaded vortex query");
103+
Ok((idx, query))
104+
})
105+
.collect()
106+
}
107+
108+
async fn generate_base_data(&self) -> Result<()> {
109+
let parquet_dir = self.queries_dir.join(Format::Parquet.name());
110+
fs::create_dir_all(&parquet_dir)?;
111+
let parquet_file = parquet_dir.join(format!("test.parquet"));
112+
113+
if parquet_file.exists() {
114+
debug!("Parquet data present in {}", parquet_dir.display());
115+
return Ok(());
116+
}
117+
118+
let init_path = self.queries_dir.join(INIT_SQL);
119+
let script = fs::read_to_string(&init_path)
120+
.with_context(|| format!("cannot read {}", init_path.display()))?;
121+
122+
let output = Command::new("duckdb")
123+
.current_dir(&parquet_dir)
124+
.arg("-c")
125+
.arg(&script)
126+
.output()
127+
.context("cannot run duckdb")?;
128+
if !output.status.success() {
129+
bail!(
130+
"duckdb {INIT_SQL} failed: stdout={:?} stderr={:?}",
131+
String::from_utf8_lossy(&output.stdout),
132+
String::from_utf8_lossy(&output.stderr),
133+
);
134+
}
135+
136+
if !parquet_file.exists() {
137+
bail!("{INIT_SQL} did not create Parquet files");
138+
}
139+
140+
debug!("Parquet data generated in {}", parquet_dir.display());
141+
Ok(())
142+
}
143+
144+
fn dataset(&self) -> BenchmarkDataset {
145+
BenchmarkDataset::VortexQueries
146+
}
147+
148+
fn dataset_name(&self) -> &str {
149+
"vortex"
150+
}
151+
152+
fn dataset_display(&self) -> String {
153+
"vortex".to_owned()
154+
}
155+
156+
fn data_url(&self) -> &Url {
157+
&self.data_url
158+
}
159+
160+
fn table_specs(&self) -> Vec<TableSpec> {
161+
vec![TableSpec::new("test", None)]
162+
}
163+
164+
fn pattern(&self, table_name: &str, format: Format) -> Option<Pattern> {
165+
Some(
166+
Pattern::new(&format!("{table_name}.{}", format.ext()))
167+
.expect("table name is a valid identifier"),
168+
)
169+
}
170+
}

0 commit comments

Comments
 (0)