Skip to content

Commit 39d3ca5

Browse files
committed
do this right
Signed-off-by: Adam Gutglick <adam@spiraldb.com>
1 parent 5c70d76 commit 39d3ca5

1 file changed

Lines changed: 36 additions & 40 deletions

File tree

  • benchmarks/datafusion-bench/src

benchmarks/datafusion-bench/src/main.rs

Lines changed: 36 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,11 @@ use datafusion_bench::tracer::set_labels;
2424
use datafusion_physical_plan::ExecutionPlan;
2525
use datafusion_physical_plan::collect;
2626
use parking_lot::Mutex;
27+
use vortex::file::multi::MultiFileDataSource;
2728
use vortex::io::filesystem::FileSystemRef;
29+
use vortex::io::object_store::ObjectStoreFileSystem;
30+
use vortex::io::session::RuntimeSessionExt;
31+
use vortex::scan::DataSource as _;
2832
use vortex::scan::DataSourceRef;
2933
use vortex_arrow::ToArrowType;
3034
use vortex_bench::Benchmark;
@@ -46,6 +50,7 @@ use vortex_bench::runner::filter_queries;
4650
use vortex_bench::setup_logging_and_tracing;
4751
use vortex_bench::v3;
4852
use vortex_datafusion::metrics::VortexMetricsFinder;
53+
use vortex_datafusion::v2::VortexTable;
4954

5055
/// Common arguments shared across benchmarks
5156
#[derive(Parser)]
@@ -253,42 +258,39 @@ async fn register_benchmark_tables<B: Benchmark + ?Sized>(
253258
benchmark: &B,
254259
format: Format,
255260
) -> anyhow::Result<()> {
256-
match format {
257-
_ if use_scan_api() && matches!(format, Format::OnDiskVortex | Format::VortexCompact) => {
258-
register_v2_tables(session, benchmark, format).await
261+
if use_scan_api() && matches!(format, Format::OnDiskVortex | Format::VortexCompact) {
262+
register_v2_tables(session, benchmark, format).await
263+
} else {
264+
let benchmark_base = benchmark.data_url().join(&format!("{}/", format.name()))?;
265+
let file_format = format_to_df_format(format);
266+
267+
for table in benchmark.table_specs().iter() {
268+
let pattern = benchmark.pattern(table.name, format);
269+
let table_url = ListingTableUrl::try_new(benchmark_base.clone(), pattern)?;
270+
271+
let listing_options = ListingOptions::new(Arc::clone(&file_format))
272+
.with_session_config_options(session.state().config());
273+
let mut config =
274+
ListingTableConfig::new(table_url).with_listing_options(listing_options);
275+
276+
config = match table.schema.as_ref() {
277+
Some(schema) => config.with_schema(Arc::new(schema.clone())),
278+
None => config.infer_schema(&session.state()).await?,
279+
};
280+
281+
let listing_table = Arc::new(
282+
ListingTable::try_new(config)?.with_cache(
283+
session
284+
.runtime_env()
285+
.cache_manager
286+
.get_file_statistic_cache(),
287+
),
288+
);
289+
290+
session.register_table(table.name, listing_table)?;
259291
}
260-
_ => {
261-
let benchmark_base = benchmark.data_url().join(&format!("{}/", format.name()))?;
262-
let file_format = format_to_df_format(format);
263-
264-
for table in benchmark.table_specs().iter() {
265-
let pattern = benchmark.pattern(table.name, format);
266-
let table_url = ListingTableUrl::try_new(benchmark_base.clone(), pattern)?;
267-
268-
let listing_options = ListingOptions::new(Arc::clone(&file_format))
269-
.with_session_config_options(session.state().config());
270-
let mut config =
271-
ListingTableConfig::new(table_url).with_listing_options(listing_options);
272-
273-
config = match table.schema.as_ref() {
274-
Some(schema) => config.with_schema(Arc::new(schema.clone())),
275-
None => config.infer_schema(&session.state()).await?,
276-
};
277-
278-
let listing_table = Arc::new(
279-
ListingTable::try_new(config)?.with_cache(
280-
session
281-
.runtime_env()
282-
.cache_manager
283-
.get_file_statistic_cache(),
284-
),
285-
);
286-
287-
session.register_table(table.name, listing_table)?;
288-
}
289292

290-
Ok(())
291-
}
293+
Ok(())
292294
}
293295
}
294296

@@ -298,12 +300,6 @@ async fn register_v2_tables<B: Benchmark + ?Sized>(
298300
benchmark: &B,
299301
format: Format,
300302
) -> anyhow::Result<()> {
301-
use vortex::file::multi::MultiFileDataSource;
302-
use vortex::io::object_store::ObjectStoreFileSystem;
303-
use vortex::io::session::RuntimeSessionExt;
304-
use vortex::scan::DataSource as _;
305-
use vortex_datafusion::v2::VortexTable;
306-
307303
let benchmark_base = benchmark.data_url().join(&format!("{}/", format.name()))?;
308304

309305
for table in benchmark.table_specs().iter() {

0 commit comments

Comments
 (0)