Skip to content

Commit 9fe20c5

Browse files
authored
Pass ExecutionCtx through the compressor (#7578)
## Summary Tracking issue: #7216 In order to have a "less breaking change" before, I just put an execution context in an arc'd mutex in the compressor itself. This is really stupid, and the caller should pass an execution context whenever they want to compress something (because we need to look at buffers to canonicalize and do compute in the compressor). ## Testing N/A since this is just moving where things are coming from. --------- Signed-off-by: Connor Tsui <connor.tsui20@gmail.com>
1 parent 6d4a7f8 commit 9fe20c5

36 files changed

Lines changed: 905 additions & 642 deletions

File tree

Cargo.lock

Lines changed: 2 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

fuzz/src/array/mod.rs

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,7 @@ impl<'a> Arbitrary<'a> for FuzzArrayAction {
237237
};
238238

239239
let compressed = BtrBlocksCompressor::default()
240-
.compress(&indices_array)
240+
.compress(&indices_array, &mut ctx)
241241
.vortex_expect("BtrBlocksCompressor compress should succeed in fuzz test");
242242
(
243243
Action::Take(compressed),
@@ -541,14 +541,15 @@ fn random_action_from_list(
541541
/// Compress an array using the given strategy.
542542
#[cfg(feature = "zstd")]
543543
pub fn compress_array(array: &ArrayRef, strategy: CompressorStrategy) -> ArrayRef {
544+
let mut ctx = SESSION.create_execution_ctx();
544545
match strategy {
545546
CompressorStrategy::Default => BtrBlocksCompressor::default()
546-
.compress(array)
547+
.compress(array, &mut ctx)
547548
.vortex_expect("BtrBlocksCompressor compress should succeed in fuzz test"),
548549
CompressorStrategy::Compact => BtrBlocksCompressorBuilder::default()
549550
.with_compact()
550551
.build()
551-
.compress(array)
552+
.compress(array, &mut ctx)
552553
.vortex_expect("Compact compress should succeed in fuzz test"),
553554
}
554555
}
@@ -557,7 +558,7 @@ pub fn compress_array(array: &ArrayRef, strategy: CompressorStrategy) -> ArrayRe
557558
#[cfg(not(feature = "zstd"))]
558559
pub fn compress_array(array: &ArrayRef, _strategy: CompressorStrategy) -> ArrayRef {
559560
BtrBlocksCompressor::default()
560-
.compress(array)
561+
.compress(array, &mut SESSION.create_execution_ctx())
561562
.vortex_expect("BtrBlocksCompressor compress should succeed in fuzz test")
562563
}
563564

vortex-btrblocks/Cargo.toml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ divan = { workspace = true }
4444
rstest = { workspace = true }
4545
test-with = { workspace = true }
4646
vortex-array = { workspace = true, features = ["_test-harness"] }
47+
vortex-session = { workspace = true }
4748

4849
[features]
4950
# This feature enabled unstable encodings for which we don't guarantee stability.

vortex-btrblocks/benches/compress.rs

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@
55

66
#[cfg(not(codspeed))]
77
mod benchmarks {
8+
use std::sync::LazyLock;
9+
810
use divan::Bencher;
911
use divan::counter::BytesCount;
1012
use divan::counter::ItemsCount;
@@ -13,13 +15,17 @@ mod benchmarks {
1315
use rand::prelude::StdRng;
1416
use vortex_array::ArrayRef;
1517
use vortex_array::IntoArray;
16-
use vortex_array::LEGACY_SESSION;
1718
use vortex_array::VortexSessionExecute;
1819
use vortex_array::arrays::PrimitiveArray;
20+
use vortex_array::session::ArraySession;
1921
use vortex_btrblocks::BtrBlocksCompressor;
2022
use vortex_buffer::buffer_mut;
23+
use vortex_session::VortexSession;
2124
use vortex_utils::aliases::hash_set::HashSet;
2225

26+
static SESSION: LazyLock<VortexSession> =
27+
LazyLock::new(|| VortexSession::empty().with::<ArraySession>());
28+
2329
fn make_clickbench_window_name() -> ArrayRef {
2430
// A test that's meant to mirror the WindowName column from ClickBench.
2531
let mut values = buffer_mut![-1i32; 65_536];
@@ -41,16 +47,20 @@ mod benchmarks {
4147

4248
#[divan::bench]
4349
fn btrblocks(bencher: Bencher) {
44-
let mut ctx = LEGACY_SESSION.create_execution_ctx();
50+
let mut ctx = SESSION.create_execution_ctx();
4551
let array = make_clickbench_window_name()
4652
.execute::<PrimitiveArray>(&mut ctx)
4753
.unwrap();
4854
let compressor = BtrBlocksCompressor::default();
4955
bencher
50-
.with_inputs(|| &array)
51-
.input_counter(|array| ItemsCount::new(array.len()))
52-
.input_counter(|array| BytesCount::of_many::<i32>(array.len()))
53-
.bench_refs(|array| compressor.compress(&array.clone().into_array()).unwrap());
56+
.with_inputs(|| (&array, SESSION.create_execution_ctx()))
57+
.input_counter(|(array, _)| ItemsCount::new(array.len()))
58+
.input_counter(|(array, _)| BytesCount::of_many::<i32>(array.len()))
59+
.bench_refs(|(array, ctx)| {
60+
compressor
61+
.compress(&array.clone().into_array(), ctx)
62+
.unwrap()
63+
});
5464
}
5565
}
5666

vortex-btrblocks/benches/compress_listview.rs

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66

77
#[cfg(not(codspeed))]
88
mod benchmarks {
9+
use std::sync::LazyLock;
10+
911
use divan::Bencher;
1012
use divan::counter::BytesCount;
1113
use divan::counter::ItemsCount;
@@ -14,17 +16,23 @@ mod benchmarks {
1416
use rand::prelude::StdRng;
1517
use vortex_array::ArrayRef;
1618
use vortex_array::IntoArray;
19+
use vortex_array::VortexSessionExecute;
1720
use vortex_array::arrays::ListViewArray;
1821
use vortex_array::arrays::StructArray;
1922
use vortex_array::arrays::VarBinViewArray;
2023
use vortex_array::dtype::FieldNames;
24+
use vortex_array::session::ArraySession;
2125
use vortex_array::validity::Validity;
2226
use vortex_btrblocks::BtrBlocksCompressor;
2327
use vortex_buffer::buffer_mut;
28+
use vortex_session::VortexSession;
2429

2530
const NUM_ROWS: usize = 8192;
2631
const SEED: u64 = 42;
2732

33+
static SESSION: LazyLock<VortexSession> =
34+
LazyLock::new(|| VortexSession::empty().with::<ArraySession>());
35+
2836
const SHORT_STRINGS: &[&str] = &[
2937
"alpha_one",
3038
"bravo_two",
@@ -177,10 +185,10 @@ mod benchmarks {
177185
let nbytes = array.nbytes();
178186
let compressor = BtrBlocksCompressor::default();
179187
bencher
180-
.with_inputs(|| &array)
188+
.with_inputs(|| (&array, SESSION.create_execution_ctx()))
181189
.input_counter(|_| ItemsCount::new(NUM_ROWS))
182190
.input_counter(move |_| BytesCount::new(nbytes as usize))
183-
.bench_refs(|array| compressor.compress(array).unwrap());
191+
.bench_refs(|(array, ctx)| compressor.compress(array, ctx).unwrap());
184192
}
185193
}
186194

0 commit comments

Comments
 (0)