Skip to content

fix(query): skip inactive rows in vectorized LIKE #20211

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
dantengsky wants to merge 1 commit into
base: main
Choose a base branch
from
+211 −4
Open

fix(query): skip inactive rows in vectorized LIKE #20211

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
215 changes: 211 additions & 4 deletions src/query/functions/src/scalars/comparison.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2062,7 +2062,7 @@ fn vectorize_like(
&mut EvalContext,
) -> Value<BooleanType>
+ Copy {
move |arg1, arg2, arg3, _ctx| {
move |arg1, arg2, arg3, ctx| {
let Value::Scalar(escape) = arg3 else {
unreachable!()
};
Expand All @@ -2078,7 +2078,26 @@ fn vectorize_like(
let pattern = convert_escape_pattern(&escape, arg2);
let pattern_type =
generate_like_pattern(pattern.as_bytes(), arg1.total_bytes_len());
if let LikePattern::SurroundByPercent(searcher) = pattern_type {
let sparse_validity = ctx
.validity
.as_ref()
.filter(|validity| validity.null_count() > 0);
if let Some(validity) = sparse_validity {
if let LikePattern::SurroundByPercent(searcher) = pattern_type {
for (index, arg1) in arg1_iter.enumerate() {
builder.push(
validity.get_bit(index)
&& searcher.search(arg1.as_bytes()).is_some(),
);
}
} else {
for (index, arg1) in arg1_iter.enumerate() {
builder.push(
validity.get_bit(index) && func(arg1.as_bytes(), &pattern_type),
);
}
}
} else if let LikePattern::SurroundByPercent(searcher) = pattern_type {
for arg1 in arg1_iter {
builder.push(searcher.search(arg1.as_bytes()).is_some());
}
Expand All @@ -2093,7 +2112,18 @@ fn vectorize_like(
(Value::Scalar(arg1), Value::Column(arg2)) => {
let arg2_iter = StringType::iter_column(&arg2);
let mut builder = MutableBitmap::with_capacity(arg2.len());
for arg2 in arg2_iter {
let sparse_validity = ctx
.validity
.as_ref()
.filter(|validity| validity.null_count() > 0);
for (index, arg2) in arg2_iter.enumerate() {
if sparse_validity
.map(|validity| !validity.get_bit(index))
.unwrap_or(false)
{
builder.push(false);
continue;
}
let pattern = convert_escape_pattern(&escape, arg2.to_string());
let pattern_type = generate_like_pattern(pattern.as_bytes(), 1);
builder.push(func(arg1.as_bytes(), &pattern_type));
Expand All @@ -2104,7 +2134,18 @@ fn vectorize_like(
let arg1_iter = StringType::iter_column(&arg1);
let arg2_iter = StringType::iter_column(&arg2);
let mut builder = MutableBitmap::with_capacity(arg2.len());
for (arg1, arg2) in arg1_iter.zip(arg2_iter) {
let sparse_validity = ctx
.validity
.as_ref()
.filter(|validity| validity.null_count() > 0);
for (index, (arg1, arg2)) in arg1_iter.zip(arg2_iter).enumerate() {
if sparse_validity
.map(|validity| !validity.get_bit(index))
.unwrap_or(false)
{
builder.push(false);
continue;
}
let pattern = convert_escape_pattern(&escape, arg2.to_string());
let pattern_type = generate_like_pattern(pattern.as_bytes(), 1);
builder.push(func(arg1.as_bytes(), &pattern_type));
Expand Down Expand Up @@ -2471,6 +2512,10 @@ fn compare_bitmap_bytes(lhs: &[u8], rhs: &[u8], ctx: &mut EvalContext, row: usiz

#[cfg(test)]
mod tests {
use std::sync::atomic::AtomicUsize;
use std::sync::atomic::Ordering as AtomicOrdering;

use databend_common_expression::FromData;
use databend_common_expression::FunctionContext;
use databend_common_expression::stat_distribution::BorrowedDistribution;
use databend_common_expression::stat_distribution::NdvEstimate;
Expand All @@ -2483,9 +2528,171 @@ mod tests {
use databend_common_expression::types::nullable::NullableDomain;
use databend_common_expression::types::string::StringDomain;
use jsonb::OwnedJsonb;
use proptest::prelude::*;

use super::*;

fn assert_boolean_column(value: Value<BooleanType>, expected: &[bool]) {
let Value::Column(column) = value else {
panic!("expected a boolean column")
};
assert_eq!(column.iter().collect::<Vec<_>>(), expected);
}

fn string_column(values: Vec<&str>) -> StringColumn {
let Column::String(column) = StringType::from_data(values) else {
unreachable!()
};
column
}

#[test]
fn test_vectorized_like_skips_rows_excluded_by_validity() {
let calls = AtomicUsize::new(0);
let like = vectorize_like(|value, pattern| {
calls.fetch_add(1, AtomicOrdering::Relaxed);
pattern.compare(value)
});
let func_ctx = FunctionContext::default();
let mut ctx = EvalContext {
generics: &[],
num_rows: 4,
func_ctx: &func_ctx,
validity: Some(Bitmap::from_iter([true, false, true, false])),
errors: None,
suppress_error: false,
strict_eval: false,
};
let escape = Value::<StringType>::Scalar("".to_string());

let result = like(
Value::<StringType>::Column(string_column(vec![
"prefix-abc-suffix",
"prefix-abc-suffix",
"prefix-axc-suffix",
"prefix-axc-suffix",
])),
Value::<StringType>::Scalar("%a_c%".to_string()),
escape.clone(),
&mut ctx,
);
assert_boolean_column(result, &[true, false, true, false]);
assert_eq!(calls.load(AtomicOrdering::Relaxed), 2);

calls.store(0, AtomicOrdering::Relaxed);
let result = like(
Value::<StringType>::Scalar("prefix-abc-suffix".to_string()),
Value::<StringType>::Column(string_column(vec!["%a_c%", "%a_c%", "%z_z%", "%a_c%"])),
escape.clone(),
&mut ctx,
);
assert_boolean_column(result, &[true, false, false, false]);
assert_eq!(calls.load(AtomicOrdering::Relaxed), 2);

calls.store(0, AtomicOrdering::Relaxed);
let result = like(
Value::<StringType>::Column(string_column(vec![
"prefix-abc-suffix",
"prefix-abc-suffix",
"prefix-zzz-suffix",
"prefix-abc-suffix",
])),
Value::<StringType>::Column(string_column(vec!["%a_c%", "%a_c%", "%z_z%", "%a_c%"])),
escape,
&mut ctx,
);
assert_boolean_column(result, &[true, false, true, false]);
assert_eq!(calls.load(AtomicOrdering::Relaxed), 2);
}

#[test]
fn test_vectorized_like_keeps_dense_and_surround_paths() {
let calls = AtomicUsize::new(0);
let like = vectorize_like(|value, pattern| {
calls.fetch_add(1, AtomicOrdering::Relaxed);
pattern.compare(value)
});
let func_ctx = FunctionContext::default();
let mut ctx = EvalContext {
generics: &[],
num_rows: 4,
func_ctx: &func_ctx,
validity: Some(Bitmap::new_constant(true, 4)),
errors: None,
suppress_error: false,
strict_eval: false,
};
let values = vec!["prefix-abc-suffix", "zzzz", "abc", "yyyy"];

let result = like(
Value::<StringType>::Column(string_column(values.clone())),
Value::<StringType>::Scalar("%a_c%".to_string()),
Value::<StringType>::Scalar("".to_string()),
&mut ctx,
);
assert_boolean_column(result, &[true, false, true, false]);
assert_eq!(calls.load(AtomicOrdering::Relaxed), 4);

calls.store(0, AtomicOrdering::Relaxed);
ctx.validity = Some(Bitmap::from_iter([true, false, true, false]));
let result = like(
Value::<StringType>::Column(string_column(values)),
Value::<StringType>::Scalar("%abc%".to_string()),
Value::<StringType>::Scalar("".to_string()),
&mut ctx,
);
assert_boolean_column(result, &[true, false, true, false]);
assert_eq!(calls.load(AtomicOrdering::Relaxed), 0);
}

proptest! {
#![proptest_config(ProptestConfig::with_cases(128))]

#[test]
fn vectorized_like_validity_preserves_active_row_results(
rows in prop::collection::vec(("[a-z]{0,48}", any::<bool>()), 1..64),
pattern in "[a-z_%\\\\]{0,24}",
) {
let values = rows.iter().map(|(value, _)| value.as_str()).collect::<Vec<_>>();
let validity = rows.iter().map(|(_, valid)| *valid).collect::<Vec<_>>();
let like = vectorize_like(|value, pattern| pattern.compare(value));
let func_ctx = FunctionContext::default();
let mut dense_ctx = EvalContext {
generics: &[],
num_rows: rows.len(),
func_ctx: &func_ctx,
validity: None,
errors: None,
suppress_error: false,
strict_eval: false,
};
let mut sparse_ctx = EvalContext {
validity: Some(Bitmap::from_iter(validity.iter().copied())),
..dense_ctx.clone()
};

let dense = like(
Value::<StringType>::Column(string_column(values.clone())),
Value::<StringType>::Scalar(pattern.clone()),
Value::<StringType>::Scalar("".to_string()),
&mut dense_ctx,
);
let sparse = like(
Value::<StringType>::Column(string_column(values)),
Value::<StringType>::Scalar(pattern),
Value::<StringType>::Scalar("".to_string()),
&mut sparse_ctx,
);
let (Value::Column(dense), Value::Column(sparse)) = (dense, sparse) else {
unreachable!()
};

for (index, dense_result) in dense.iter().enumerate() {
prop_assert_eq!(sparse.get_bit(index), validity[index] && dense_result);
}
}
}

#[test]
fn test_numeric_histogram_partial_bucket_reserves_equality_mass() {
let bucket = TypedHistogramBucket::new(F64::from(0.0), F64::from(10.0), 10.0, 2.0);
Expand Down
Loading