Skip to content

Commit 9e3ae2f

Browse files
perf: branchless boolean zip kernel (#8275)
## Summary Adds a dedicated, branchless `ZipKernel for Bool`. --------- Signed-off-by: Joe Isaacs <joe.isaacs@live.co.uk>
1 parent ab78257 commit 9e3ae2f

6 files changed

Lines changed: 289 additions & 0 deletions

File tree

vortex-array/Cargo.toml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,10 @@ harness = false
192192
name = "varbinview_zip"
193193
harness = false
194194

195+
[[bench]]
196+
name = "bool_zip"
197+
harness = false
198+
195199
[[bench]]
196200
name = "take_primitive"
197201
harness = false

vortex-array/benches/bool_zip.rs

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
#![expect(clippy::unwrap_used)]
5+
6+
use divan::Bencher;
7+
use vortex_array::ArrayRef;
8+
use vortex_array::IntoArray;
9+
use vortex_array::LEGACY_SESSION;
10+
use vortex_array::RecursiveCanonical;
11+
use vortex_array::VortexSessionExecute;
12+
use vortex_array::arrays::BoolArray;
13+
use vortex_array::builtins::ArrayBuiltins;
14+
use vortex_mask::Mask;
15+
16+
fn main() {
17+
divan::main();
18+
}
19+
20+
const LEN: usize = 65_536;
21+
22+
/// Fragmented (alternating) mask: the worst case for the generic per-run builder this kernel
23+
/// replaces. The branchless bitmap blend is mask-shape-independent, so one shape suffices.
24+
fn mask() -> Mask {
25+
Mask::from_iter((0..LEN).map(|i| i.is_multiple_of(2)))
26+
}
27+
28+
#[divan::bench]
29+
fn nonnull(bencher: Bencher) {
30+
let if_true = BoolArray::from_iter((0..LEN).map(|i| i.is_multiple_of(2))).into_array();
31+
let if_false = BoolArray::from_iter((0..LEN).map(|i| i.is_multiple_of(3))).into_array();
32+
run(bencher, if_true, if_false);
33+
}
34+
35+
#[divan::bench]
36+
fn nullable(bencher: Bencher) {
37+
let if_true = BoolArray::from_iter(
38+
(0..LEN).map(|i| (!i.is_multiple_of(7)).then_some(i.is_multiple_of(2))),
39+
)
40+
.into_array();
41+
let if_false = BoolArray::from_iter(
42+
(0..LEN).map(|i| (!i.is_multiple_of(5)).then_some(i.is_multiple_of(3))),
43+
)
44+
.into_array();
45+
run(bencher, if_true, if_false);
46+
}
47+
48+
fn run(bencher: Bencher, if_true: ArrayRef, if_false: ArrayRef) {
49+
let mask = mask();
50+
bencher
51+
.with_inputs(|| {
52+
(
53+
if_true.clone(),
54+
if_false.clone(),
55+
mask.clone().into_array(),
56+
LEGACY_SESSION.create_execution_ctx(),
57+
)
58+
})
59+
.bench_refs(|(t, f, m, ctx)| {
60+
m.zip(t.clone(), f.clone())
61+
.unwrap()
62+
.execute::<RecursiveCanonical>(ctx)
63+
.unwrap();
64+
});
65+
}

vortex-array/src/arrays/bool/compute/mod.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ mod mask;
88
pub mod rules;
99
mod slice;
1010
mod take;
11+
mod zip;
1112

1213
#[cfg(test)]
1314
mod tests {
Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
use vortex_buffer::BitBuffer;
5+
use vortex_buffer::BufferMut;
6+
use vortex_error::VortexResult;
7+
use vortex_mask::Mask;
8+
9+
use crate::ArrayRef;
10+
use crate::ExecutionCtx;
11+
use crate::IntoArray;
12+
use crate::array::ArrayView;
13+
use crate::arrays::Bool;
14+
use crate::arrays::BoolArray;
15+
use crate::arrays::bool::BoolArrayExt;
16+
use crate::scalar_fn::fns::zip::ZipKernel;
17+
use crate::scalar_fn::fns::zip::zip_validity;
18+
19+
/// A branchless boolean zip kernel that blends the two value bitmaps with the mask in one pass.
20+
///
21+
/// Booleans are bit-packed, so selecting `if_true` where the mask is set and `if_false` where it is
22+
/// not is a single bitwise blend over the packed words — `(true & mask) | (false & !mask)` — instead
23+
/// of the generic per-run builder. Validity is combined with the shared `zip_validity`, which itself
24+
/// reuses this kernel (terminating immediately, since validity bitmaps are non-nullable).
25+
impl ZipKernel for Bool {
26+
fn zip(
27+
if_true: ArrayView<'_, Bool>,
28+
if_false: &ArrayRef,
29+
mask: &ArrayRef,
30+
ctx: &mut ExecutionCtx,
31+
) -> VortexResult<Option<ArrayRef>> {
32+
let Some(if_false) = if_false.as_opt::<Bool>() else {
33+
return Ok(None);
34+
};
35+
36+
// Null mask entries select `if_false`, matching `Zip`'s SQL ELSE semantics.
37+
let mask = mask.try_to_mask_fill_null_false(ctx)?;
38+
let mask_values = match &mask {
39+
// Defer trivial masks to the generic zip, which just casts the surviving side.
40+
Mask::AllTrue(_) | Mask::AllFalse(_) => return Ok(None),
41+
Mask::Values(values) => values,
42+
};
43+
let mask_bits = mask_values.bit_buffer();
44+
45+
let values = zip_value_bits(
46+
&if_true.to_bit_buffer(),
47+
&if_false.to_bit_buffer(),
48+
mask_bits,
49+
);
50+
51+
let validity = zip_validity(if_true.validity()?, if_false.validity()?, &mask)?;
52+
53+
Ok(Some(BoolArray::new(values, validity).into_array()))
54+
}
55+
}
56+
57+
fn zip_value_bits(if_true: &BitBuffer, if_false: &BitBuffer, mask: &BitBuffer) -> BitBuffer {
58+
assert_eq!(if_true.len(), if_false.len());
59+
assert_eq!(if_true.len(), mask.len());
60+
61+
let true_chunks = if_true.chunks();
62+
let false_chunks = if_false.chunks();
63+
let mask_chunks = mask.chunks();
64+
65+
let mut values = BufferMut::<u64>::with_capacity(true_chunks.num_u64s());
66+
for ((true_bits, false_bits), mask_bits) in true_chunks
67+
.iter()
68+
.zip(false_chunks.iter())
69+
.zip(mask_chunks.iter())
70+
{
71+
values.push((true_bits & mask_bits) | (false_bits & !mask_bits));
72+
}
73+
74+
if true_chunks.remainder_len() != 0 {
75+
let true_bits = true_chunks.remainder_bits();
76+
let false_bits = false_chunks.remainder_bits();
77+
let mask_bits = mask_chunks.remainder_bits();
78+
values.push((true_bits & mask_bits) | (false_bits & !mask_bits));
79+
}
80+
81+
BitBuffer::new(values.freeze().into_byte_buffer(), if_true.len())
82+
}
83+
84+
#[cfg(test)]
85+
mod tests {
86+
use vortex_error::VortexResult;
87+
use vortex_mask::Mask;
88+
89+
use super::zip_value_bits;
90+
use crate::ArrayRef;
91+
use crate::IntoArray;
92+
use crate::LEGACY_SESSION;
93+
use crate::VortexSessionExecute;
94+
use crate::arrays::Bool;
95+
use crate::arrays::BoolArray;
96+
use crate::assert_arrays_eq;
97+
use crate::builtins::ArrayBuiltins;
98+
99+
#[test]
100+
fn blend_value_bits_boundaries() {
101+
for len in [0usize, 1, 2, 7, 8, 9, 63, 64, 65, 127, 128] {
102+
let if_true = (0..len).map(|i| i.is_multiple_of(2)).collect();
103+
let if_false = (0..len).map(|i| i.is_multiple_of(3)).collect();
104+
let mask = (0..len).map(|i| i % 3 != 1).collect();
105+
106+
let values = zip_value_bits(&if_true, &if_false, &mask);
107+
108+
assert_eq!(values.len(), len);
109+
assert_eq!(
110+
values.iter().collect::<Vec<_>>(),
111+
(0..len)
112+
.map(|i| {
113+
if i % 3 != 1 {
114+
i.is_multiple_of(2)
115+
} else {
116+
i.is_multiple_of(3)
117+
}
118+
})
119+
.collect::<Vec<_>>(),
120+
"failed for len {len}",
121+
);
122+
}
123+
}
124+
125+
/// Blend two non-nullable bool arrays across the 64-bit mask chunk boundary + remainder.
126+
#[test]
127+
fn zip_nonnull_spans_mask_chunks() -> VortexResult<()> {
128+
let len = 150usize;
129+
let if_true = BoolArray::from_iter((0..len).map(|i| i.is_multiple_of(2))).into_array();
130+
let if_false = BoolArray::from_iter((0..len).map(|i| i.is_multiple_of(3))).into_array();
131+
132+
let bits: Vec<bool> = (0..len).map(|i| i.is_multiple_of(5) || i == 64).collect();
133+
let mask = Mask::from_iter(bits.iter().copied());
134+
135+
let mut ctx = LEGACY_SESSION.create_execution_ctx();
136+
let result = mask
137+
.into_array()
138+
.zip(if_true, if_false)?
139+
.execute::<ArrayRef>(&mut ctx)?;
140+
assert!(result.is::<Bool>());
141+
142+
let expected = BoolArray::from_iter((0..len).map(|i| {
143+
if bits[i] {
144+
i.is_multiple_of(2)
145+
} else {
146+
i.is_multiple_of(3)
147+
}
148+
}))
149+
.into_array();
150+
assert_arrays_eq!(result, expected);
151+
Ok(())
152+
}
153+
154+
/// With `Validity::Array` on both sides, select values and validity from the chosen side.
155+
#[test]
156+
fn zip_nullable_selects_values_and_validity() -> VortexResult<()> {
157+
let len = 130usize;
158+
let if_true = BoolArray::from_iter(
159+
(0..len).map(|i| (!i.is_multiple_of(4)).then_some(i.is_multiple_of(2))),
160+
)
161+
.into_array();
162+
let if_false = BoolArray::from_iter(
163+
(0..len).map(|i| (!i.is_multiple_of(5)).then_some(i.is_multiple_of(3))),
164+
)
165+
.into_array();
166+
167+
let bits: Vec<bool> = (0..len).map(|i| i.is_multiple_of(2)).collect();
168+
let mask = Mask::from_iter(bits.iter().copied());
169+
170+
let mut ctx = LEGACY_SESSION.create_execution_ctx();
171+
let result = mask
172+
.into_array()
173+
.zip(if_true, if_false)?
174+
.execute::<ArrayRef>(&mut ctx)?;
175+
assert!(result.is::<Bool>());
176+
177+
let expected = BoolArray::from_iter((0..len).map(|i| {
178+
if bits[i] {
179+
(!i.is_multiple_of(4)).then_some(i.is_multiple_of(2))
180+
} else {
181+
(!i.is_multiple_of(5)).then_some(i.is_multiple_of(3))
182+
}
183+
}))
184+
.into_array();
185+
assert_arrays_eq!(result, expected);
186+
Ok(())
187+
}
188+
}

vortex-array/src/arrays/bool/vtable/kernel.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,9 +6,11 @@ use crate::arrays::dict::TakeExecuteAdaptor;
66
use crate::kernel::ParentKernelSet;
77
use crate::scalar_fn::fns::cast::CastExecuteAdaptor;
88
use crate::scalar_fn::fns::fill_null::FillNullExecuteAdaptor;
9+
use crate::scalar_fn::fns::zip::ZipExecuteAdaptor;
910

1011
pub(super) const PARENT_KERNELS: ParentKernelSet<Bool> = ParentKernelSet::new(&[
1112
ParentKernelSet::lift(&CastExecuteAdaptor(Bool)),
1213
ParentKernelSet::lift(&FillNullExecuteAdaptor(Bool)),
1314
ParentKernelSet::lift(&TakeExecuteAdaptor(Bool)),
15+
ParentKernelSet::lift(&ZipExecuteAdaptor(Bool)),
1416
]);

vortex-array/src/scalar_fn/fns/zip/mod.rs

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ use crate::scalar_fn::ScalarFnId;
3232
use crate::scalar_fn::ScalarFnVTable;
3333
use crate::scalar_fn::SimplifyCtx;
3434
use crate::scalar_fn::fns::literal::Literal;
35+
use crate::validity::Validity;
3536

3637
/// An expression that conditionally selects between two arrays based on a boolean mask.
3738
///
@@ -236,6 +237,34 @@ fn zip_impl_with_builder(
236237
Ok(builder.finish())
237238
}
238239

240+
/// Combine two validities for a row-wise zip: take `if_true`'s validity where `mask` is set and
241+
/// `if_false`'s where it is not.
242+
///
243+
/// That selection is itself a zip over the two boolean validity bitmaps, so it is built as a (lazy)
244+
/// zip array — reusing the zip machinery rather than re-deriving the mask algebra. Trivial cases
245+
/// where both sides' validity already agrees skip the zip. `mask` must already be null-filled so the
246+
/// selection matches the accompanying value selection. Shared by the per-encoding zip kernels (e.g.
247+
/// `Bool`, `Primitive`) that build their result directly.
248+
pub(crate) fn zip_validity(
249+
if_true: Validity,
250+
if_false: Validity,
251+
mask: &Mask,
252+
) -> VortexResult<Validity> {
253+
match (&if_true, &if_false) {
254+
(Validity::NonNullable, Validity::NonNullable) => return Ok(Validity::NonNullable),
255+
(Validity::AllValid, Validity::AllValid) => return Ok(Validity::AllValid),
256+
(Validity::AllInvalid, Validity::AllInvalid) => return Ok(Validity::AllInvalid),
257+
_ => {}
258+
}
259+
260+
let len = mask.len();
261+
let validity = mask
262+
.clone()
263+
.into_array()
264+
.zip(if_true.to_array(len), if_false.to_array(len))?;
265+
Ok(Validity::Array(validity))
266+
}
267+
239268
#[cfg(test)]
240269
mod tests {
241270
use arrow_array::cast::AsArray;

0 commit comments

Comments
 (0)