diff --git a/src/query/functions/src/scalars/comparison.rs b/src/query/functions/src/scalars/comparison.rs index 5373b1fb6a3e5..659b6d4f7cd4e 100644 --- a/src/query/functions/src/scalars/comparison.rs +++ b/src/query/functions/src/scalars/comparison.rs @@ -2062,7 +2062,7 @@ fn vectorize_like( &mut EvalContext, ) -> Value + Copy { - move |arg1, arg2, arg3, _ctx| { + move |arg1, arg2, arg3, ctx| { let Value::Scalar(escape) = arg3 else { unreachable!() }; @@ -2078,7 +2078,26 @@ fn vectorize_like( let pattern = convert_escape_pattern(&escape, arg2); let pattern_type = generate_like_pattern(pattern.as_bytes(), arg1.total_bytes_len()); - if let LikePattern::SurroundByPercent(searcher) = pattern_type { + let sparse_validity = ctx + .validity + .as_ref() + .filter(|validity| validity.null_count() > 0); + if let Some(validity) = sparse_validity { + if let LikePattern::SurroundByPercent(searcher) = pattern_type { + for (index, arg1) in arg1_iter.enumerate() { + builder.push( + validity.get_bit(index) + && searcher.search(arg1.as_bytes()).is_some(), + ); + } + } else { + for (index, arg1) in arg1_iter.enumerate() { + builder.push( + validity.get_bit(index) && func(arg1.as_bytes(), &pattern_type), + ); + } + } + } else if let LikePattern::SurroundByPercent(searcher) = pattern_type { for arg1 in arg1_iter { builder.push(searcher.search(arg1.as_bytes()).is_some()); } @@ -2093,7 +2112,18 @@ fn vectorize_like( (Value::Scalar(arg1), Value::Column(arg2)) => { let arg2_iter = StringType::iter_column(&arg2); let mut builder = MutableBitmap::with_capacity(arg2.len()); - for arg2 in arg2_iter { + let sparse_validity = ctx + .validity + .as_ref() + .filter(|validity| validity.null_count() > 0); + for (index, arg2) in arg2_iter.enumerate() { + if sparse_validity + .map(|validity| !validity.get_bit(index)) + .unwrap_or(false) + { + builder.push(false); + continue; + } let pattern = convert_escape_pattern(&escape, arg2.to_string()); let pattern_type = generate_like_pattern(pattern.as_bytes(), 1); builder.push(func(arg1.as_bytes(), &pattern_type)); @@ -2104,7 +2134,18 @@ fn vectorize_like( let arg1_iter = StringType::iter_column(&arg1); let arg2_iter = StringType::iter_column(&arg2); let mut builder = MutableBitmap::with_capacity(arg2.len()); - for (arg1, arg2) in arg1_iter.zip(arg2_iter) { + let sparse_validity = ctx + .validity + .as_ref() + .filter(|validity| validity.null_count() > 0); + for (index, (arg1, arg2)) in arg1_iter.zip(arg2_iter).enumerate() { + if sparse_validity + .map(|validity| !validity.get_bit(index)) + .unwrap_or(false) + { + builder.push(false); + continue; + } let pattern = convert_escape_pattern(&escape, arg2.to_string()); let pattern_type = generate_like_pattern(pattern.as_bytes(), 1); builder.push(func(arg1.as_bytes(), &pattern_type)); @@ -2471,6 +2512,10 @@ fn compare_bitmap_bytes(lhs: &[u8], rhs: &[u8], ctx: &mut EvalContext, row: usiz #[cfg(test)] mod tests { + use std::sync::atomic::AtomicUsize; + use std::sync::atomic::Ordering as AtomicOrdering; + + use databend_common_expression::FromData; use databend_common_expression::FunctionContext; use databend_common_expression::stat_distribution::BorrowedDistribution; use databend_common_expression::stat_distribution::NdvEstimate; @@ -2483,9 +2528,171 @@ mod tests { use databend_common_expression::types::nullable::NullableDomain; use databend_common_expression::types::string::StringDomain; use jsonb::OwnedJsonb; + use proptest::prelude::*; use super::*; + fn assert_boolean_column(value: Value, expected: &[bool]) { + let Value::Column(column) = value else { + panic!("expected a boolean column") + }; + assert_eq!(column.iter().collect::>(), expected); + } + + fn string_column(values: Vec<&str>) -> StringColumn { + let Column::String(column) = StringType::from_data(values) else { + unreachable!() + }; + column + } + + #[test] + fn test_vectorized_like_skips_rows_excluded_by_validity() { + let calls = AtomicUsize::new(0); + let like = vectorize_like(|value, pattern| { + calls.fetch_add(1, AtomicOrdering::Relaxed); + pattern.compare(value) + }); + let func_ctx = FunctionContext::default(); + let mut ctx = EvalContext { + generics: &[], + num_rows: 4, + func_ctx: &func_ctx, + validity: Some(Bitmap::from_iter([true, false, true, false])), + errors: None, + suppress_error: false, + strict_eval: false, + }; + let escape = Value::::Scalar("".to_string()); + + let result = like( + Value::::Column(string_column(vec![ + "prefix-abc-suffix", + "prefix-abc-suffix", + "prefix-axc-suffix", + "prefix-axc-suffix", + ])), + Value::::Scalar("%a_c%".to_string()), + escape.clone(), + &mut ctx, + ); + assert_boolean_column(result, &[true, false, true, false]); + assert_eq!(calls.load(AtomicOrdering::Relaxed), 2); + + calls.store(0, AtomicOrdering::Relaxed); + let result = like( + Value::::Scalar("prefix-abc-suffix".to_string()), + Value::::Column(string_column(vec!["%a_c%", "%a_c%", "%z_z%", "%a_c%"])), + escape.clone(), + &mut ctx, + ); + assert_boolean_column(result, &[true, false, false, false]); + assert_eq!(calls.load(AtomicOrdering::Relaxed), 2); + + calls.store(0, AtomicOrdering::Relaxed); + let result = like( + Value::::Column(string_column(vec![ + "prefix-abc-suffix", + "prefix-abc-suffix", + "prefix-zzz-suffix", + "prefix-abc-suffix", + ])), + Value::::Column(string_column(vec!["%a_c%", "%a_c%", "%z_z%", "%a_c%"])), + escape, + &mut ctx, + ); + assert_boolean_column(result, &[true, false, true, false]); + assert_eq!(calls.load(AtomicOrdering::Relaxed), 2); + } + + #[test] + fn test_vectorized_like_keeps_dense_and_surround_paths() { + let calls = AtomicUsize::new(0); + let like = vectorize_like(|value, pattern| { + calls.fetch_add(1, AtomicOrdering::Relaxed); + pattern.compare(value) + }); + let func_ctx = FunctionContext::default(); + let mut ctx = EvalContext { + generics: &[], + num_rows: 4, + func_ctx: &func_ctx, + validity: Some(Bitmap::new_constant(true, 4)), + errors: None, + suppress_error: false, + strict_eval: false, + }; + let values = vec!["prefix-abc-suffix", "zzzz", "abc", "yyyy"]; + + let result = like( + Value::::Column(string_column(values.clone())), + Value::::Scalar("%a_c%".to_string()), + Value::::Scalar("".to_string()), + &mut ctx, + ); + assert_boolean_column(result, &[true, false, true, false]); + assert_eq!(calls.load(AtomicOrdering::Relaxed), 4); + + calls.store(0, AtomicOrdering::Relaxed); + ctx.validity = Some(Bitmap::from_iter([true, false, true, false])); + let result = like( + Value::::Column(string_column(values)), + Value::::Scalar("%abc%".to_string()), + Value::::Scalar("".to_string()), + &mut ctx, + ); + assert_boolean_column(result, &[true, false, true, false]); + assert_eq!(calls.load(AtomicOrdering::Relaxed), 0); + } + + proptest! { + #![proptest_config(ProptestConfig::with_cases(128))] + + #[test] + fn vectorized_like_validity_preserves_active_row_results( + rows in prop::collection::vec(("[a-z]{0,48}", any::()), 1..64), + pattern in "[a-z_%\\\\]{0,24}", + ) { + let values = rows.iter().map(|(value, _)| value.as_str()).collect::>(); + let validity = rows.iter().map(|(_, valid)| *valid).collect::>(); + let like = vectorize_like(|value, pattern| pattern.compare(value)); + let func_ctx = FunctionContext::default(); + let mut dense_ctx = EvalContext { + generics: &[], + num_rows: rows.len(), + func_ctx: &func_ctx, + validity: None, + errors: None, + suppress_error: false, + strict_eval: false, + }; + let mut sparse_ctx = EvalContext { + validity: Some(Bitmap::from_iter(validity.iter().copied())), + ..dense_ctx.clone() + }; + + let dense = like( + Value::::Column(string_column(values.clone())), + Value::::Scalar(pattern.clone()), + Value::::Scalar("".to_string()), + &mut dense_ctx, + ); + let sparse = like( + Value::::Column(string_column(values)), + Value::::Scalar(pattern), + Value::::Scalar("".to_string()), + &mut sparse_ctx, + ); + let (Value::Column(dense), Value::Column(sparse)) = (dense, sparse) else { + unreachable!() + }; + + for (index, dense_result) in dense.iter().enumerate() { + prop_assert_eq!(sparse.get_bit(index), validity[index] && dense_result); + } + } + } + #[test] fn test_numeric_histogram_partial_bucket_reserves_equality_mass() { let bucket = TypedHistogramBucket::new(F64::from(0.0), F64::from(10.0), 10.0, 2.0);