diff --git a/Cargo.lock b/Cargo.lock index afeb3dc8697ca..ac495656e121d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5672,6 +5672,16 @@ dependencies = [ "databend-common-exception", ] +[[package]] +name = "databend-expression-domain-fuzz" +version = "0.0.0" +dependencies = [ + "arbitrary", + "databend-common-column", + "databend-common-expression", + "libfuzzer-sys", +] + [[package]] name = "databend-functions-scalar-arithmetic" version = "0.1.0" @@ -11610,6 +11620,16 @@ dependencies = [ "rle-decode-fast", ] +[[package]] +name = "libfuzzer-sys" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fd2f41a1cba099f79a0b6b6c35656cf7c03351a7bae8ff0f28f25270f929d2" +dependencies = [ + "arbitrary", + "cc", +] + [[package]] name = "libloading" version = "0.8.7" diff --git a/Cargo.toml b/Cargo.toml index 90cb56e84cd12..9ce80521b793c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ members = [ "src/meta/store", "src/meta/ver", "src/query/codegen", + "src/query/expression/fuzz", "src/tests/planner_replay", "tests/sqllogictests", ] diff --git a/src/common/column/src/binview/mod.rs b/src/common/column/src/binview/mod.rs index a572b2c3d7522..3a8ac0cf55bfb 100644 --- a/src/common/column/src/binview/mod.rs +++ b/src/common/column/src/binview/mod.rs @@ -515,6 +515,37 @@ impl BinaryViewColumnGeneric { } } + /// Return the minimum and maximum values without materializing every view. + /// + /// Comparisons use the four-byte prefixes stored in [`View`]. The backing + /// buffers are only accessed when two prefixes are equal and when the final + /// extrema are returned. + pub fn min_max(&self) -> Option<(&T, &T)> { + if self.is_empty() { + return None; + } + + let mut min_index = 0; + let mut max_index = 0; + for index in 1..self.len() { + if Self::compare(self, index, self, min_index).is_lt() { + min_index = index; + continue; + } + if Self::compare(self, index, self, max_index).is_gt() { + max_index = index; + } + } + + unsafe { + Some(( + self.value_unchecked(min_index), + self.value_unchecked(max_index), + )) + } + } + + #[inline] pub fn compare(col_i: &Self, i: usize, col_j: &Self, j: usize) -> std::cmp::Ordering { let view_i = unsafe { col_i.views().as_slice().get_unchecked(i) }; let view_j = unsafe { col_j.views().as_slice().get_unchecked(j) }; diff --git a/src/common/column/tests/it/binview/mod.rs b/src/common/column/tests/it/binview/mod.rs index c6be8ac800d6e..3820754aa9b02 100644 --- a/src/common/column/tests/it/binview/mod.rs +++ b/src/common/column/tests/it/binview/mod.rs @@ -190,4 +190,19 @@ fn test_compare() { assert_eq!(min, min_expect); assert_eq!(max, max_expect); + assert_eq!(array.min_max(), Some((min, max))); + assert_eq!(Utf8ViewColumn::new_empty().min_max(), None); + + // Equal view prefixes must fall back to comparing the complete strings. + let same_prefix: Utf8ViewColumn = [ + "same-prefix-middle", + "same-prefix-maximum", + "same-prefix-minimum", + ] + .into_iter() + .collect(); + assert_eq!( + same_prefix.min_max(), + Some(("same-prefix-maximum", "same-prefix-minimum")) + ); } diff --git a/src/query/expression/fuzz/.gitignore b/src/query/expression/fuzz/.gitignore new file mode 100644 index 0000000000000..be4bb88220e27 --- /dev/null +++ b/src/query/expression/fuzz/.gitignore @@ -0,0 +1,6 @@ +artifacts +corpus +coverage +target + + diff --git a/src/query/expression/fuzz/Cargo.toml b/src/query/expression/fuzz/Cargo.toml new file mode 100644 index 0000000000000..bbcc43aab7db1 --- /dev/null +++ b/src/query/expression/fuzz/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "databend-expression-domain-fuzz" +version = "0.0.0" +publish = false +edition = "2024" + +[package.metadata] +cargo-fuzz = true + +[dependencies] +arbitrary = "1.4" +databend-common-column = { path = "../../../common/column" } +databend-common-expression = { path = ".." } +libfuzzer-sys = "0.4" + +[lib] +doctest = false + +[[bin]] +name = "column_domain" +path = "fuzz_targets/column_domain.rs" +test = false +doc = false +bench = false diff --git a/src/query/expression/fuzz/fuzz_targets/column_domain.rs b/src/query/expression/fuzz/fuzz_targets/column_domain.rs new file mode 100644 index 0000000000000..da1ed71495c09 --- /dev/null +++ b/src/query/expression/fuzz/fuzz_targets/column_domain.rs @@ -0,0 +1,35 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![no_main] + +use databend_expression_domain_fuzz::ColumnCase; +use databend_expression_domain_fuzz::run_column_case; +use libfuzzer_sys::fuzz_target; + +// This target validates the fundamental `Column::domain` invariant: every value physically present +// in a generated column must be contained by the domain returned for that column. + +// The typed `Arbitrary` generator covers empty and NULL columns, all number and decimal widths, +// Boolean, String, temporal and interval columns, Binary, and recursive Nullable, Array, Map, and +// Tuple shapes. Array and Map offsets may start above zero so sliced-column behavior is exercised as +// well. + +// For every row, the oracle recursively checks the actual scalar value against the column domain. +// Nullable checks NULL membership, Array and Map check their real elements, and Tuple checks each +// field. Primitive values use singleton-domain containment. Undefined domains accept unsupported +// scalar types by definition. +// +// cargo fuzz run --dev --sanitizer none --strip-dead-code column_domain -- -runs=10000 -max_len=4096 +fuzz_target!(|case: ColumnCase| run_column_case(case)); diff --git a/src/query/expression/fuzz/src/lib.rs b/src/query/expression/fuzz/src/lib.rs new file mode 100644 index 0000000000000..e0490ee07c9b0 --- /dev/null +++ b/src/query/expression/fuzz/src/lib.rs @@ -0,0 +1,335 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use arbitrary::Arbitrary; +use arbitrary::Error; +use arbitrary::Unstructured; +use databend_common_column::types::months_days_micros; +use databend_common_column::types::timestamp_tz; +use databend_common_expression::Column; +use databend_common_expression::ColumnBuilder; +use databend_common_expression::Domain; +use databend_common_expression::Scalar; +use databend_common_expression::ScalarRef; +use databend_common_expression::types::ALL_NUMERICS_TYPES; +use databend_common_expression::types::DataType; +use databend_common_expression::types::DecimalScalar; +use databend_common_expression::types::DecimalSize; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::NumberScalar; +use databend_common_expression::types::date::DATE_MAX; +use databend_common_expression::types::date::DATE_MIN; +use databend_common_expression::types::i256; +use databend_common_expression::types::timestamp::TIMESTAMP_MAX; +use databend_common_expression::types::timestamp::TIMESTAMP_MIN; + +const MAX_ROWS: usize = 8; +const MAX_DEPTH: usize = 3; +const MAX_ARRAY_LEN: usize = 8; +const MAX_STRING_LEN: usize = 50; +const MAX_BINARY_LEN: usize = 50; + +#[derive(Debug)] +pub struct ColumnCase { + column: Column, +} + +struct ColumnGenerator<'a, 'u> { + u: &'u mut Unstructured<'a>, +} + +impl<'a, 'u> ColumnGenerator<'a, 'u> { + fn new(u: &'u mut Unstructured<'a>) -> Self { + Self { u } + } + + fn data_type(&mut self, depth: usize, allow_nullable: bool) -> arbitrary::Result { + let max_tag = if depth < MAX_DEPTH { 15 } else { 11 }; + let mut tag = self.u.int_in_range(0_u8..=max_tag)?; + if !allow_nullable && matches!(tag, 0 | 12) { + tag = 3; + } + + Ok(match tag { + 0 => DataType::Null, + 1 => DataType::EmptyArray, + 2 => DataType::EmptyMap, + 3 => DataType::Boolean, + 4 => DataType::Number(*self.u.choose(ALL_NUMERICS_TYPES)?), + 5 => DataType::Decimal(self.decimal_size()?), + 6 => DataType::String, + 7 => DataType::Timestamp, + 8 => DataType::TimestampTz, + 9 => DataType::Date, + 10 => DataType::Interval, + 11 => DataType::Binary, + 12 => DataType::Nullable(Box::new(self.data_type(depth + 1, false)?)), + 13 => DataType::Array(Box::new(self.data_type(depth + 1, true)?)), + 14 => DataType::Map(Box::new(DataType::Tuple(vec![ + DataType::String, + self.data_type(depth + 1, true)?, + ]))), + 15 => { + let fields = self.u.int_in_range(1_usize..=3)?; + let fields = (0..fields) + .map(|_| self.data_type(depth + 1, true)) + .collect::>>()?; + DataType::Tuple(fields) + } + _ => unreachable!(), + }) + } + + fn column(&mut self, data_type: &DataType, rows: usize) -> arbitrary::Result { + let mut builder = ColumnBuilder::with_capacity(data_type, rows); + for _ in 0..rows { + let scalar = self.scalar(data_type)?; + if !scalar.as_ref().is_value_of_type(data_type) { + return Err(Error::IncorrectFormat); + } + builder.push(scalar.as_ref()); + } + Ok(builder.build()) + } + + fn scalar(&mut self, data_type: &DataType) -> arbitrary::Result { + Ok(match data_type { + DataType::Null => Scalar::Null, + DataType::EmptyArray => Scalar::EmptyArray, + DataType::EmptyMap => Scalar::EmptyMap, + DataType::Boolean => Scalar::Boolean(bool::arbitrary(self.u)?), + DataType::Number(data_type) => Scalar::Number(self.number(data_type)?), + DataType::Decimal(size) => Scalar::Decimal(self.decimal(*size)?), + DataType::String => Scalar::String(self.string()?), + DataType::Timestamp => { + Scalar::Timestamp(self.u.int_in_range(TIMESTAMP_MIN..=TIMESTAMP_MAX)?) + } + DataType::TimestampTz => { + let timestamp = self.u.int_in_range(TIMESTAMP_MIN..=TIMESTAMP_MAX)?; + let offset = self.u.int_in_range(-86_400_i32..=86_400_i32)?; + Scalar::TimestampTz(timestamp_tz::new(timestamp, offset)) + } + DataType::Date => Scalar::Date(self.u.int_in_range(DATE_MIN..=DATE_MAX)?), + DataType::Interval => Scalar::Interval(months_days_micros::new( + i32::arbitrary(self.u)?, + i32::arbitrary(self.u)?, + i64::arbitrary(self.u)?, + )), + DataType::Binary => Scalar::Binary(self.bytes(MAX_BINARY_LEN)?), + DataType::Nullable(inner) => { + if bool::arbitrary(self.u)? { + Scalar::Null + } else { + self.scalar(inner)? + } + } + DataType::Array(inner) => { + let rows = self.u.int_in_range(0_usize..=MAX_ARRAY_LEN)?; + Scalar::Array(self.column(inner, rows)?) + } + DataType::Map(inner) => { + let rows = self.u.int_in_range(0_usize..=MAX_ARRAY_LEN)?; + Scalar::Map(self.column(inner, rows)?) + } + DataType::Tuple(fields) => Scalar::Tuple( + fields + .iter() + .map(|field| self.scalar(field)) + .collect::>>()?, + ), + _ => return Err(Error::IncorrectFormat), + }) + } + + fn number(&mut self, data_type: &NumberDataType) -> arbitrary::Result { + Ok(match data_type { + NumberDataType::UInt8 => NumberScalar::UInt8(u8::arbitrary(self.u)?), + NumberDataType::UInt16 => NumberScalar::UInt16(u16::arbitrary(self.u)?), + NumberDataType::UInt32 => NumberScalar::UInt32(u32::arbitrary(self.u)?), + NumberDataType::UInt64 => NumberScalar::UInt64(u64::arbitrary(self.u)?), + NumberDataType::Int8 => NumberScalar::Int8(i8::arbitrary(self.u)?), + NumberDataType::Int16 => NumberScalar::Int16(i16::arbitrary(self.u)?), + NumberDataType::Int32 => NumberScalar::Int32(i32::arbitrary(self.u)?), + NumberDataType::Int64 => NumberScalar::Int64(i64::arbitrary(self.u)?), + NumberDataType::Float32 => NumberScalar::Float32(f32::arbitrary(self.u)?.into()), + NumberDataType::Float64 => NumberScalar::Float64(f64::arbitrary(self.u)?.into()), + }) + } + + fn decimal_size(&mut self) -> arbitrary::Result { + let precision = self.u.int_in_range(1_u8..=76)?; + let scale = self.u.int_in_range(0_u8..=precision)?; + Ok(DecimalSize::new(precision, scale).unwrap()) + } + + fn decimal(&mut self, size: DecimalSize) -> arbitrary::Result { + let precision = size.precision(); + Ok(match precision { + 1..=18 => { + let max = 10_i64.pow(precision as u32) - 1; + DecimalScalar::Decimal64(self.u.int_in_range(-max..=max)?, size) + } + 19..=38 => { + let max = 10_i128.pow(precision as u32) - 1; + DecimalScalar::Decimal128(self.u.int_in_range(-max..=max)?, size) + } + 39..=76 => DecimalScalar::Decimal256(i256::from(i128::arbitrary(self.u)?), size), + _ => unreachable!(), + }) + } + + fn string(&mut self) -> arbitrary::Result { + let len = self.u.int_in_range(0_usize..=MAX_STRING_LEN)?; + (0..len).map(|_| char::arbitrary(self.u)).collect() + } + + fn bytes(&mut self, max_len: usize) -> arbitrary::Result> { + let len = self.u.int_in_range(0_usize..=max_len)?; + (0..len).map(|_| u8::arbitrary(self.u)).collect() + } +} + +impl<'a> Arbitrary<'a> for ColumnCase { + fn arbitrary(u: &mut Unstructured<'a>) -> arbitrary::Result { + let rows = u.int_in_range(0_usize..=MAX_ROWS)?; + let mut generator = ColumnGenerator::new(u); + let data_type = generator.data_type(0, true)?; + let column = generator.column(&data_type, rows)?; + column.check_valid().map_err(|_| Error::IncorrectFormat)?; + Ok(Self { column }) + } + + fn size_hint(_depth: usize) -> (usize, Option) { + (2, None) + } +} + +/// Decode and run the same typed input used by libFuzzer from raw bytes. +pub fn run_fuzz_bytes(data: &[u8]) { + let unstructured = Unstructured::new(data); + if let Ok(case) = ColumnCase::arbitrary_take_rest(unstructured) { + run_column_case(case); + } +} + +/// Assert that every scalar physically present in the column belongs to its domain. +pub fn run_column_case(case: ColumnCase) { + assert_column_domain(&case.column); +} + +fn assert_column_domain(column: &Column) { + let data_type = column.data_type(); + let domain = column.domain(); + assert!( + domain.matches_data_type(&data_type), + "column domain has the wrong type: type={data_type}, domain={domain:?}, column={column:?}" + ); + + for (index, value) in column.iter().enumerate() { + let value_domain = value.domain(&data_type); + assert!( + value_belongs_to_domain(&value, &data_type, &domain), + "column value escaped its domain: index={index}, type={data_type}, value={value:?}, \ + domain={domain:?}, value_domain={value_domain:?}, column={column:?}" + ); + } +} + +fn value_belongs_to_domain(value: &ScalarRef<'_>, data_type: &DataType, domain: &Domain) -> bool { + match (value, data_type, domain) { + (ScalarRef::Null, DataType::Null, Domain::Nullable(domain)) => domain.has_null, + (ScalarRef::Null, DataType::Nullable(_), Domain::Nullable(domain)) => domain.has_null, + (value, DataType::Nullable(data_type), Domain::Nullable(domain)) => domain + .value + .as_deref() + .is_some_and(|domain| value_belongs_to_domain(value, data_type, domain)), + (ScalarRef::EmptyArray, DataType::EmptyArray, Domain::Array(None)) => true, + (ScalarRef::EmptyMap, DataType::EmptyMap, Domain::Map(None)) => true, + (ScalarRef::Array(values), DataType::Array(data_type), Domain::Array(domain)) => { + match domain { + None => values.len() == 0, + Some(domain) => values + .iter() + .all(|value| value_belongs_to_domain(&value, data_type, domain)), + } + } + (ScalarRef::Map(values), DataType::Map(data_type), Domain::Map(domain)) => match domain { + None => values.len() == 0, + Some(domain) => values + .iter() + .all(|value| value_belongs_to_domain(&value, data_type, domain)), + }, + (ScalarRef::Tuple(values), DataType::Tuple(data_types), Domain::Tuple(domains)) => { + values.len() == data_types.len() + && values.len() == domains.len() + && values + .iter() + .zip(data_types) + .zip(domains) + .all(|((value, data_type), domain)| { + value_belongs_to_domain(value, data_type, domain) + }) + } + ( + ScalarRef::Binary(_) + | ScalarRef::Bitmap(_) + | ScalarRef::Variant(_) + | ScalarRef::Geometry(_) + | ScalarRef::Geography(_) + | ScalarRef::Vector(_) + | ScalarRef::Opaque(_), + _, + Domain::Undefined, + ) => true, + _ => primitive_domain_contains(domain, &value.domain(data_type)), + } +} + +fn primitive_domain_contains(domain: &Domain, other: &Domain) -> bool { + match (domain, other) { + (Domain::Number(domain), Domain::Number(other)) => { + let mut merged = *domain; + merged.merge(other).is_ok() && merged == *domain + } + (Domain::Decimal(domain), Domain::Decimal(other)) => { + let mut merged = *domain; + merged.merge(other).is_ok() && merged == *domain + } + (Domain::Boolean(domain), Domain::Boolean(other)) => { + (!other.has_false || domain.has_false) && (!other.has_true || domain.has_true) + } + (Domain::String(domain), Domain::String(other)) => { + domain.min <= other.min + && match (&domain.max, &other.max) { + (None, _) => true, + (Some(_), None) => false, + (Some(domain), Some(other)) => other <= domain, + } + } + (Domain::Timestamp(domain), Domain::Timestamp(other)) => { + domain.min <= other.min && other.max <= domain.max + } + (Domain::TimestampTz(domain), Domain::TimestampTz(other)) => { + domain.min <= other.min && other.max <= domain.max + } + (Domain::Date(domain), Domain::Date(other)) => { + domain.min <= other.min && other.max <= domain.max + } + (Domain::Interval(domain), Domain::Interval(other)) => { + domain.min <= other.min && other.max <= domain.max + } + (Domain::Undefined, Domain::Undefined) => true, + _ => false, + } +} diff --git a/src/query/expression/src/constant_folder.rs b/src/query/expression/src/constant_folder.rs index 2322640f365f5..64124b1fa4844 100644 --- a/src/query/expression/src/constant_folder.rs +++ b/src/query/expression/src/constant_folder.rs @@ -17,9 +17,7 @@ use std::collections::HashMap; use databend_common_ast::Span; use log::error; -use crate::ColumnBuilder; use crate::ColumnIndex; -use crate::EvalContext; use crate::FunctionContext; use crate::FunctionDomain; use crate::FunctionEval; @@ -44,6 +42,8 @@ use crate::types::number::NumberScalar; const MAX_FUNCTION_ARGS_TO_FOLD: usize = 4096; +mod monotonicity; + pub struct ConstantFolder<'a, Index: ColumnIndex> { input_domains: &'a HashMap, func_ctx: &'a FunctionContext, @@ -440,16 +440,7 @@ impl<'a, Index: ColumnIndex> ConstantFolder<'a, Index> { } let all_args_is_scalar = args_expr.iter().all(|arg| arg.as_constant().is_some()); - let is_monotonicity = self - .fn_registry - .properties - .get(&function.signature.name) - .map(|p| { - args_expr.len() == 1 - && (p.monotonicity - || p.monotonicity_by_type.contains(args_expr[0].data_type())) - }) - .unwrap_or_default(); + let is_monotonicity = self.is_monotonic(&function.signature.name, &args_expr); // Check for mutually exclusive ranges in AND function if function.signature.name == "and" @@ -495,69 +486,13 @@ impl<'a, Index: ColumnIndex> ConstantFolder<'a, Index> { let func_domain = args_domain.and_then(|domains: Vec| { let res = calc_domain.domain_eval(self.func_ctx, &domains); match (res, is_monotonicity) { - (FunctionDomain::MayThrow | FunctionDomain::Full, true) => { - let domain = domains.first().unwrap(); - if args[0].data_type().is_nullable_or_null() { - return None; - } - - let (min, max) = domain.to_minmax(); - if min.is_null() || max.is_null() { - return None; - } - - { - let mut ctx = EvalContext { - generics, - num_rows: 2, - validity: None, - errors: None, - func_ctx: self.func_ctx, - suppress_error: false, - strict_eval: true, - }; - let mut builder = - ColumnBuilder::with_capacity(args[0].data_type(), 2); - builder.push(min.as_ref()); - builder.push(max.as_ref()); - - let input = Value::Column(builder.build()); - let result = eval.eval(&[input], &mut ctx); - - if result.is_scalar() { - None - } else { - // if error happens, domain maybe incorrect - // min, max: String("2024-09-02 00:00") String("2024-09-02 00:0�") - // to_date(s) > to_date('2024-01-1') - let col = result.as_column().unwrap(); - let d = if ctx.has_error(0) || ctx.has_error(1) { - let (full_min, full_max) = - Domain::full(return_type).to_minmax(); - if full_min.is_null() || full_max.is_null() { - return None; - } - - let mut builder = - ColumnBuilder::with_capacity(return_type, 2); - - for (i, (v, f)) in - col.iter().zip([full_min, full_max].iter()).enumerate() - { - if ctx.has_error(i) { - builder.push(f.as_ref()); - } else { - builder.push(v); - } - } - builder.build().domain() - } else { - result.as_column().unwrap().domain() - }; - Some(d) - } - } - } + (FunctionDomain::MayThrow | FunctionDomain::Full, true) => self + .calculate_monotonicity_domain( + return_type, + domains.first().unwrap(), + generics, + eval.as_ref(), + ), (FunctionDomain::MayThrow, _) => None, (FunctionDomain::Full, _) => Some(Domain::full(return_type)), (FunctionDomain::Domain(domain), _) => Some(domain), diff --git a/src/query/expression/src/constant_folder/monotonicity.rs b/src/query/expression/src/constant_folder/monotonicity.rs new file mode 100644 index 0000000000000..7a515b8fdb99c --- /dev/null +++ b/src/query/expression/src/constant_folder/monotonicity.rs @@ -0,0 +1,154 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use databend_common_column::bitmap::Bitmap; + +use super::ConstantFolder; +use crate::Column; +use crate::ColumnBuilder; +use crate::ColumnIndex; +use crate::EvalContext; +use crate::Expr; +use crate::Value; +use crate::function::ScalarFunction; +use crate::property::Domain; +use crate::types::boolean::BooleanDomain; +use crate::types::nullable::NullableDomain; +use crate::types::string::StringDomain; +use crate::types::*; + +impl<'a, Index: ColumnIndex> ConstantFolder<'a, Index> { + pub(super) fn is_monotonic(&self, function_name: &str, args: &[Expr]) -> bool { + self.fn_registry + .properties + .get(function_name) + .is_some_and(|property| { + if let [arg] = args { + property.monotonicity || property.monotonicity_by_type.contains(arg.data_type()) + } else { + false + } + }) + } + + pub(super) fn calculate_monotonicity_domain( + &self, + return_type: &DataType, + input_domain: &Domain, + generics: &[DataType], + eval: &dyn ScalarFunction, + ) -> Option { + let input = input_domain.boundary_column()?; + let mut ctx = EvalContext { + generics, + num_rows: 2, + validity: None, + errors: None, + func_ctx: self.func_ctx, + suppress_error: false, + strict_eval: true, + }; + let Value::Column(col) = eval.eval(&[Value::Column(input)], &mut ctx) else { + return None; + }; + + // if error happens, domain maybe incorrect + // min, max: String("2024-09-02 00:00") String("2024-09-02 00:0�") + // to_date(s) > to_date('2024-01-1') + let domain = if ctx.has_error(0) || ctx.has_error(1) { + // Preserve the successful boundary and widen only the failed side. + // For example, a malformed minimum string must not discard a valid + // maximum timestamp that can still prove an upper bound. + // This assumes all currently registered monotonic functions are + // non-decreasing. Supporting a decreasing function here requires + // recording its direction in `FunctionProperty` and reversing the + // fallback boundary. + let full_domain = Domain::full(return_type); + let Some(fallback) = full_domain.boundary_column() else { + return Some(full_domain); + }; + let mut builder = ColumnBuilder::with_capacity(return_type, 2); + for (index, (value, fallback)) in col.iter().zip(fallback.iter()).enumerate() { + if ctx.has_error(index) { + builder.push(fallback); + } else { + builder.push(value); + } + } + builder.build().domain() + } else { + col.domain() + }; + + if !return_type.is_nullable_or_null() { + return Some(domain); + } + + Some(match domain { + Domain::Nullable(mut domain) => { + domain.has_null = true; + Domain::Nullable(domain) + } + domain => Domain::Nullable(NullableDomain { + has_null: true, + value: Some(Box::new(domain)), + }), + }) + } +} + +impl Domain { + /// Materialize the finite, non-NULL endpoints used for monotonicity probing. + fn boundary_column(&self) -> Option { + Some(match self { + Domain::Number(domain) => crate::with_number_type!(|NUM| match domain { + NumberDomain::NUM(SimpleDomain { min, max }) => + Column::Number(NumberColumn::NUM(Buffer::from(vec![*min, *max])),), + }), + Domain::Decimal(domain) => crate::with_decimal_type!(|DECIMAL| match domain { + DecimalDomain::DECIMAL(SimpleDomain { min, max }, size) => Column::Decimal( + DecimalColumn::DECIMAL(Buffer::from(vec![*min, *max]), *size), + ), + }), + Domain::Boolean(BooleanDomain { + has_false, + has_true, + }) => match (*has_false, *has_true) { + (true, true) => Column::Boolean(Bitmap::from([false, true])), + (true, false) => Column::Boolean(Bitmap::from([false, false])), + (false, true) => Column::Boolean(Bitmap::from([true, true])), + (false, false) => return None, + }, + Domain::String(StringDomain { + min, + max: Some(max), + }) => Column::String(StringColumn::from_slice([min.as_str(), max.as_str()])), + Domain::Timestamp(SimpleDomain { min, max }) => { + Column::Timestamp(Buffer::from(vec![*min, *max])) + } + Domain::TimestampTz(SimpleDomain { min, max }) => { + Column::TimestampTz(Buffer::from(vec![*min, *max])) + } + Domain::Date(SimpleDomain { min, max }) => Column::Date(Buffer::from(vec![*min, *max])), + Domain::Interval(SimpleDomain { min, max }) => { + Column::Interval(Buffer::from(vec![*min, *max])) + } + Domain::Nullable(NullableDomain { + value: Some(domain), + .. + }) => NullableColumn::new_column(domain.boundary_column()?, Bitmap::new_trued(2)), + _ => return None, + }) + } +} diff --git a/src/query/expression/src/property.rs b/src/query/expression/src/property.rs index b1c00639beabc..d7d4d9fae4179 100644 --- a/src/query/expression/src/property.rs +++ b/src/query/expression/src/property.rs @@ -14,9 +14,9 @@ use databend_common_column::types::months_days_micros; use databend_common_column::types::timestamp_tz; +use databend_common_exception::ErrorCode; use enum_as_inner::EnumAsInner; -use crate::ColumnBuilder; use crate::Scalar; use crate::types::AccessType; use crate::types::AnyType; @@ -136,6 +136,178 @@ pub enum Domain { Undefined, } +/// Type-erased exact extrema observed in a set of values. +/// +/// The typed payloads are shared with [`Domain`], but this is not a domain: its +/// bounds are actual non-NULL values observed in a column. The boolean in each +/// variant records whether NULL also occurred in the same set. +#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub enum MinMax { + Number(NumberDomain, bool), + Decimal(DecimalDomain, bool), + Boolean(BooleanDomain, bool), + String(SimpleDomain, bool), + Timestamp(SimpleDomain, bool), + TimestampTz(SimpleDomain, bool), + Date(SimpleDomain, bool), + Interval(SimpleDomain, bool), +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ColumnMinMax { + Empty, + AllNull, + Values(MinMax), +} + +impl ColumnMinMax { + pub fn merge(&mut self, other: &Self) -> Result<(), ErrorCode> { + match (self, other) { + (_, ColumnMinMax::Empty) => Ok(()), + (this @ ColumnMinMax::Empty, other) => { + *this = other.clone(); + Ok(()) + } + (ColumnMinMax::AllNull, ColumnMinMax::AllNull) => Ok(()), + (this @ ColumnMinMax::AllNull, ColumnMinMax::Values(min_max)) => { + let mut min_max = min_max.clone(); + min_max.set_has_null(); + *this = ColumnMinMax::Values(min_max); + Ok(()) + } + (ColumnMinMax::Values(min_max), ColumnMinMax::AllNull) => { + min_max.set_has_null(); + Ok(()) + } + (ColumnMinMax::Values(lhs), ColumnMinMax::Values(rhs)) => lhs.merge(rhs), + } + } + + pub fn into_option(self) -> Option { + match self { + ColumnMinMax::Values(min_max) => Some(min_max), + ColumnMinMax::Empty | ColumnMinMax::AllNull => None, + } + } +} + +impl MinMax { + pub fn has_null(&self) -> bool { + match self { + MinMax::Number(_, has_null) + | MinMax::Decimal(_, has_null) + | MinMax::Boolean(_, has_null) + | MinMax::String(_, has_null) + | MinMax::Timestamp(_, has_null) + | MinMax::TimestampTz(_, has_null) + | MinMax::Date(_, has_null) + | MinMax::Interval(_, has_null) => *has_null, + } + } + + fn set_has_null(&mut self) { + match self { + MinMax::Number(_, value) + | MinMax::Decimal(_, value) + | MinMax::Boolean(_, value) + | MinMax::String(_, value) + | MinMax::Timestamp(_, value) + | MinMax::TimestampTz(_, value) + | MinMax::Date(_, value) + | MinMax::Interval(_, value) => *value = true, + } + } + + pub fn with_null(mut self) -> Self { + self.set_has_null(); + self + } + + pub fn merge(&mut self, other: &Self) -> Result<(), ErrorCode> { + match (self, other) { + (MinMax::Number(lhs, lhs_has_null), MinMax::Number(rhs, rhs_has_null)) => { + lhs.merge(rhs)?; + *lhs_has_null |= *rhs_has_null; + Ok(()) + } + (MinMax::Decimal(lhs, lhs_has_null), MinMax::Decimal(rhs, rhs_has_null)) => { + lhs.merge(rhs)?; + *lhs_has_null |= *rhs_has_null; + Ok(()) + } + (MinMax::Boolean(lhs, lhs_has_null), MinMax::Boolean(rhs, rhs_has_null)) => { + lhs.merge(rhs); + *lhs_has_null |= *rhs_has_null; + Ok(()) + } + (MinMax::String(lhs, lhs_has_null), MinMax::String(rhs, rhs_has_null)) => { + lhs.merge(rhs); + *lhs_has_null |= *rhs_has_null; + Ok(()) + } + (MinMax::Timestamp(lhs, lhs_has_null), MinMax::Timestamp(rhs, rhs_has_null)) => { + lhs.merge(rhs); + *lhs_has_null |= *rhs_has_null; + Ok(()) + } + (MinMax::TimestampTz(lhs, lhs_has_null), MinMax::TimestampTz(rhs, rhs_has_null)) => { + lhs.merge(rhs); + *lhs_has_null |= *rhs_has_null; + Ok(()) + } + (MinMax::Date(lhs, lhs_has_null), MinMax::Date(rhs, rhs_has_null)) => { + lhs.merge(rhs); + *lhs_has_null |= *rhs_has_null; + Ok(()) + } + (MinMax::Interval(lhs, lhs_has_null), MinMax::Interval(rhs, rhs_has_null)) => { + lhs.merge(rhs); + *lhs_has_null |= *rhs_has_null; + Ok(()) + } + (lhs, rhs) => Err(ErrorCode::InvalidArgument(format!( + "cannot merge min/max values {lhs:?} and {rhs:?}" + ))), + } + } + + pub fn scalars(&self) -> (Scalar, Scalar) { + with_number_type!(|NUM| match self { + MinMax::Number(NumberDomain::NUM(values), _) => ( + Scalar::Number(NumberScalar::NUM(values.min)), + Scalar::Number(NumberScalar::NUM(values.max)), + ), + MinMax::Decimal(decimal, _) => with_decimal_type!(|DECIMAL| match decimal { + DecimalDomain::DECIMAL(values, size) => ( + Scalar::Decimal(DecimalScalar::DECIMAL(values.min, *size)), + Scalar::Decimal(DecimalScalar::DECIMAL(values.max, *size)), + ), + }), + MinMax::Boolean(values, _) => match (values.has_false, values.has_true) { + (true, true) => (Scalar::Boolean(false), Scalar::Boolean(true)), + (true, false) => (Scalar::Boolean(false), Scalar::Boolean(false)), + (false, true) => (Scalar::Boolean(true), Scalar::Boolean(true)), + (false, false) => unreachable!("MinMax cannot contain an empty boolean range"), + }, + MinMax::String(values, _) => ( + Scalar::String(values.min.clone()), + Scalar::String(values.max.clone()), + ), + MinMax::Timestamp(values, _) => + (Scalar::Timestamp(values.min), Scalar::Timestamp(values.max),), + MinMax::TimestampTz(values, _) => ( + Scalar::TimestampTz(values.min), + Scalar::TimestampTz(values.max), + ), + MinMax::Date(values, _) => { + (Scalar::Date(values.min), Scalar::Date(values.max)) + } + MinMax::Interval(values, _) => + (Scalar::Interval(values.min), Scalar::Interval(values.max),), + }) + } +} + impl FunctionDomain { pub fn map(self, f: impl Fn(T::Domain) -> U::Domain) -> FunctionDomain { match self { @@ -160,13 +332,6 @@ impl FunctionDomain { } impl Domain { - pub fn from_min_max(min: Scalar, max: Scalar, t: &DataType) -> Self { - let mut builder = ColumnBuilder::with_capacity(t, 2); - builder.push(min.as_ref()); - builder.push(max.as_ref()); - builder.build().domain() - } - pub fn check_data_type(&self, data_type: &DataType) -> Result<(), String> { if self.matches_data_type(data_type) { Ok(()) @@ -327,32 +492,24 @@ impl Domain { pub fn merge(&self, other: &Domain) -> Domain { match (self, other) { (Domain::Number(this), Domain::Number(other)) => { - with_number_type!(|TYPE| match (this, other) { - (NumberDomain::TYPE(this), NumberDomain::TYPE(other)) => - Domain::Number(NumberDomain::TYPE(SimpleDomain { - min: this.min.min(other.min), - max: this.max.max(other.max), - })), - _ => unreachable!("unable to merge {this:?} with {other:?}"), - }) + let mut merged = *this; + merged + .merge(other) + .unwrap_or_else(|_| unreachable!("unable to merge {this:?} with {other:?}")); + Domain::Number(merged) } (Domain::Decimal(this), Domain::Decimal(other)) => { - with_decimal_type!(|TYPE| match (this, other) { - (DecimalDomain::TYPE(x, size), DecimalDomain::TYPE(y, _)) => - Domain::Decimal(DecimalDomain::TYPE( - SimpleDomain { - min: x.min.min(y.min), - max: x.max.max(y.max), - }, - *size - ),), - _ => unreachable!("unable to merge {this:?} with {other:?}"), - }) + let mut merged = *this; + merged + .merge(other) + .unwrap_or_else(|_| unreachable!("unable to merge {this:?} with {other:?}")); + Domain::Decimal(merged) + } + (Domain::Boolean(this), Domain::Boolean(other)) => { + let mut merged = *this; + merged.merge(other); + Domain::Boolean(merged) } - (Domain::Boolean(this), Domain::Boolean(other)) => Domain::Boolean(BooleanDomain { - has_false: this.has_false || other.has_false, - has_true: this.has_true || other.has_true, - }), (Domain::String(this), Domain::String(other)) => Domain::String(StringDomain { min: this.min.as_str().min(&other.min).to_string(), max: this @@ -362,19 +519,20 @@ impl Domain { .map(|(self_max, other_max)| self_max.max(other_max).to_string()), }), (Domain::Timestamp(this), Domain::Timestamp(other)) => { - Domain::Timestamp(SimpleDomain { - min: this.min.min(other.min), - max: this.max.max(other.max), - }) + let mut merged = *this; + merged.merge(other); + Domain::Timestamp(merged) + } + (Domain::Date(this), Domain::Date(other)) => { + let mut merged = *this; + merged.merge(other); + Domain::Date(merged) + } + (Domain::Interval(this), Domain::Interval(other)) => { + let mut merged = *this; + merged.merge(other); + Domain::Interval(merged) } - (Domain::Date(this), Domain::Date(other)) => Domain::Date(SimpleDomain { - min: this.min.min(other.min), - max: this.max.max(other.max), - }), - (Domain::Interval(this), Domain::Interval(other)) => Domain::Interval(SimpleDomain { - min: this.min.min(other.min), - max: this.max.max(other.max), - }), ( Domain::Nullable(NullableDomain { has_null: true, @@ -535,110 +693,6 @@ impl Domain { _ => None, } } - - pub fn to_minmax(&self) -> (Scalar, Scalar) { - match self { - Domain::Number(NumberDomain::Int8(SimpleDomain { min, max })) => ( - Scalar::Number(NumberScalar::Int8(*min)), - Scalar::Number(NumberScalar::Int8(*max)), - ), - Domain::Number(NumberDomain::Int16(SimpleDomain { min, max })) => ( - Scalar::Number(NumberScalar::Int16(*min)), - Scalar::Number(NumberScalar::Int16(*max)), - ), - Domain::Number(NumberDomain::Int32(SimpleDomain { min, max })) => ( - Scalar::Number(NumberScalar::Int32(*min)), - Scalar::Number(NumberScalar::Int32(*max)), - ), - Domain::Number(NumberDomain::Int64(SimpleDomain { min, max })) => ( - Scalar::Number(NumberScalar::Int64(*min)), - Scalar::Number(NumberScalar::Int64(*max)), - ), - Domain::Number(NumberDomain::UInt8(SimpleDomain { min, max })) => ( - Scalar::Number(NumberScalar::UInt8(*min)), - Scalar::Number(NumberScalar::UInt8(*max)), - ), - Domain::Number(NumberDomain::UInt16(SimpleDomain { min, max })) => ( - Scalar::Number(NumberScalar::UInt16(*min)), - Scalar::Number(NumberScalar::UInt16(*max)), - ), - Domain::Number(NumberDomain::UInt32(SimpleDomain { min, max })) => ( - Scalar::Number(NumberScalar::UInt32(*min)), - Scalar::Number(NumberScalar::UInt32(*max)), - ), - Domain::Number(NumberDomain::UInt64(SimpleDomain { min, max })) => ( - Scalar::Number(NumberScalar::UInt64(*min)), - Scalar::Number(NumberScalar::UInt64(*max)), - ), - Domain::Number(NumberDomain::Float32(SimpleDomain { min, max })) => ( - Scalar::Number(NumberScalar::Float32(*min)), - Scalar::Number(NumberScalar::Float32(*max)), - ), - Domain::Number(NumberDomain::Float64(SimpleDomain { min, max })) => ( - Scalar::Number(NumberScalar::Float64(*min)), - Scalar::Number(NumberScalar::Float64(*max)), - ), - Domain::Decimal(decimal_domain) => match decimal_domain { - DecimalDomain::Decimal64(SimpleDomain { min, max }, size) => ( - Scalar::Decimal(DecimalScalar::Decimal64(*min, *size)), - Scalar::Decimal(DecimalScalar::Decimal64(*max, *size)), - ), - DecimalDomain::Decimal128(SimpleDomain { min, max }, size) => ( - Scalar::Decimal(DecimalScalar::Decimal128(*min, *size)), - Scalar::Decimal(DecimalScalar::Decimal128(*max, *size)), - ), - DecimalDomain::Decimal256(SimpleDomain { min, max }, size) => ( - Scalar::Decimal(DecimalScalar::Decimal256(*min, *size)), - Scalar::Decimal(DecimalScalar::Decimal256(*max, *size)), - ), - }, - Domain::Boolean(BooleanDomain { - has_false, - has_true, - }) => (Scalar::Boolean(!*has_false), Scalar::Boolean(*has_true)), - Domain::String(StringDomain { min, max }) => { - let max = if let Some(max) = max { - Scalar::String(max.clone()) - } else { - Scalar::Null - }; - (Scalar::String(min.clone()), max) - } - Domain::Timestamp(SimpleDomain { min, max }) => { - (Scalar::Timestamp(*min), Scalar::Timestamp(*max)) - } - Domain::TimestampTz(SimpleDomain { min, max }) => { - (Scalar::TimestampTz(*min), Scalar::TimestampTz(*max)) - } - Domain::Date(SimpleDomain { min, max }) => (Scalar::Date(*min), Scalar::Date(*max)), - Domain::Interval(SimpleDomain { min, max }) => { - (Scalar::Interval(*min), Scalar::Interval(*max)) - } - Domain::Nullable(NullableDomain { has_null, value }) => { - if let Some(v) = value { - let (min, mut max) = v.to_minmax(); - if *has_null { - max = Scalar::Null; - } - (min, max) - } else { - (Scalar::Null, Scalar::Null) - } - } - Domain::Tuple(fields) => { - let mut mins = Vec::with_capacity(fields.len()); - let mut maxs = Vec::with_capacity(fields.len()); - for field in fields { - let (min, max) = field.to_minmax(); - mins.push(min); - maxs.push(max); - } - (Scalar::Tuple(mins), Scalar::Tuple(maxs)) - } - // cluster key only allow number|string|boolean|date|timestamp|decimal, so unreachable. - _ => (Scalar::Null, Scalar::Null), - } - } } pub trait SimpleDomainCmp { diff --git a/src/query/expression/src/types/boolean.rs b/src/query/expression/src/types/boolean.rs index ff84b198887b2..0921b3636aa44 100644 --- a/src/query/expression/src/types/boolean.rs +++ b/src/query/expression/src/types/boolean.rs @@ -333,12 +333,19 @@ pub fn take_nullable_boolean_from_views( NullableType::::upcast_column(column).into() } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct BooleanDomain { pub has_false: bool, pub has_true: bool, } +impl BooleanDomain { + pub fn merge(&mut self, other: &Self) { + self.has_false |= other.has_false; + self.has_true |= other.has_true; + } +} + impl ColumnView { pub fn and_bitmap(&self, rhs: Option<&Bitmap>) -> Self { debug_assert!(rhs.map(|rhs| rhs.len() == self.len()).unwrap_or(true)); diff --git a/src/query/expression/src/types/date.rs b/src/query/expression/src/types/date.rs index 560f671563c68..87bf0d04091ed 100644 --- a/src/query/expression/src/types/date.rs +++ b/src/query/expression/src/types/date.rs @@ -20,6 +20,7 @@ use databend_common_column::buffer::Buffer; use databend_common_exception::ErrorCode; use databend_common_io::cursor_ext::BufferReadDateTimeExt; use databend_common_io::cursor_ext::ReadBytesExt; +use jiff::SignedDuration; use jiff::civil::Date; use jiff::fmt::strtime; use jiff::tz::TimeZone; @@ -32,7 +33,6 @@ use super::SimpleValueType; use super::number::SimpleDomain; use crate::ColumnBuilder; use crate::ScalarRef; -use crate::date_helper::DateConverter; use crate::property::Domain; use crate::values::Column; use crate::values::Scalar; @@ -45,6 +45,11 @@ pub const DATE_MIN: i32 = -719162; /// 9999-12-31 pub const DATE_MAX: i32 = 2932896; +pub fn date_from_days(days: impl AsPrimitive) -> Date { + let duration = SignedDuration::from_hours(days.as_() * 24); + Date::constant(1970, 1, 1).checked_add(duration).unwrap() +} + /// Check if date is within range. /// /// If days is invalid convert to DATE_MIN. #[inline] @@ -161,7 +166,7 @@ pub fn string_to_date( } #[inline] -pub fn date_to_string(date: impl AsPrimitive, tz: &TimeZone) -> impl Display { - let res = date.as_().to_date(tz); +pub fn date_to_string(date: impl AsPrimitive, _tz: &TimeZone) -> impl Display { + let res = date_from_days(date); strtime::format(DATE_FORMAT, res).unwrap() } diff --git a/src/query/expression/src/types/decimal.rs b/src/query/expression/src/types/decimal.rs index 50a2b5de7ed8d..18badc51887d7 100644 --- a/src/query/expression/src/types/decimal.rs +++ b/src/query/expression/src/types/decimal.rs @@ -460,7 +460,7 @@ pub enum DecimalColumnBuilder { Decimal256(Vec, DecimalSize), } -#[derive(Debug, Clone, Copy, PartialEq, Eq, EnumAsInner)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, EnumAsInner, Serialize, Deserialize)] pub enum DecimalDomain { Decimal64(SimpleDomain, DecimalSize), Decimal128(SimpleDomain, DecimalSize), @@ -468,6 +468,26 @@ pub enum DecimalDomain { } impl DecimalDomain { + pub fn merge(&mut self, other: &Self) -> Result<()> { + with_decimal_type!(|DECIMAL_TYPE| match (self, other) { + ( + DecimalDomain::DECIMAL_TYPE(lhs, lhs_size), + DecimalDomain::DECIMAL_TYPE(rhs, rhs_size), + ) => { + if lhs_size != rhs_size { + return Err(ErrorCode::InvalidArgument(format!( + "cannot merge decimal domains with sizes {lhs_size:?} and {rhs_size:?}" + ))); + } + lhs.merge(rhs); + Ok(()) + } + (lhs, rhs) => Err(ErrorCode::InvalidArgument(format!( + "cannot merge decimal domains {lhs:?} and {rhs:?}" + ))), + }) + } + pub fn decimal_size(&self) -> DecimalSize { with_decimal_type!(|DECIMAL| match self { DecimalDomain::DECIMAL(_, size) => *size, diff --git a/src/query/expression/src/types/number.rs b/src/query/expression/src/types/number.rs index 2c7deb68275d7..d97c0b5f10364 100644 --- a/src/query/expression/src/types/number.rs +++ b/src/query/expression/src/types/number.rs @@ -286,7 +286,7 @@ pub enum NumberColumnBuilder { Float64(Vec), } -#[derive(Debug, Clone, PartialEq, Eq, EnumAsInner)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, EnumAsInner, Serialize, Deserialize)] pub enum NumberDomain { UInt8(SimpleDomain), UInt16(SimpleDomain), @@ -300,7 +300,7 @@ pub enum NumberDomain { Float64(SimpleDomain), } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct SimpleDomain { pub min: T, pub max: T, @@ -321,7 +321,30 @@ where T: PartialOrd + Debug } } +impl SimpleDomain { + pub fn merge(&mut self, other: &Self) { + if other.min < self.min { + self.min = other.min.clone() + } + if other.max > self.max { + self.max = other.max.clone() + } + } +} + impl NumberDomain { + pub fn merge(&mut self, other: &Self) -> Result<()> { + crate::with_number_type!(|NUM_TYPE| match (self, other) { + (NumberDomain::NUM_TYPE(lhs), NumberDomain::NUM_TYPE(rhs)) => { + lhs.merge(rhs); + Ok(()) + } + (lhs, rhs) => Err(ErrorCode::InvalidArgument(format!( + "cannot merge number domains {lhs:?} and {rhs:?}" + ))), + }) + } + pub fn check_valid(&self) -> Result<()> { match self { NumberDomain::UInt8(domain) => domain.check_valid(), @@ -604,19 +627,6 @@ impl NumberColumn { }) } - pub fn domain(&self) -> NumberDomain { - assert!(self.len() > 0); - crate::with_number_type!(|NUM_TYPE| match self { - NumberColumn::NUM_TYPE(col) => { - let (min, max) = col.iter().minmax().into_option().unwrap(); - NumberDomain::NUM_TYPE(SimpleDomain { - min: *min, - max: *max, - }) - } - }) - } - pub fn arrow_buffer(&self) -> arrow_buffer::Buffer { match self { NumberColumn::UInt8(buffer) => buffer.clone().into(), diff --git a/src/query/expression/src/types/timestamp.rs b/src/query/expression/src/types/timestamp.rs index bfc2d9cf4c989..4fcae60924452 100644 --- a/src/query/expression/src/types/timestamp.rs +++ b/src/query/expression/src/types/timestamp.rs @@ -21,9 +21,11 @@ use databend_common_exception::ErrorCode; use databend_common_io::cursor_ext::BufferReadDateTimeExt; use databend_common_io::cursor_ext::DateTimeResType; use databend_common_io::cursor_ext::ReadBytesExt; +use jiff::Timestamp; use jiff::Zoned; use jiff::fmt::strtime; use jiff::tz::TimeZone; +use num_traits::AsPrimitive; use super::ArgType; use super::DataType; @@ -33,7 +35,6 @@ use super::number::SimpleDomain; use crate::ColumnBuilder; use crate::ScalarRef; use crate::property::Domain; -use crate::utils::date_helper::DateConverter; use crate::values::Column; use crate::values::Scalar; @@ -46,6 +47,32 @@ pub const TIMESTAMP_MAX: i64 = 253402300799999999; pub const MICROS_PER_SEC: i64 = 1_000_000; pub const MICROS_PER_MILLI: i64 = 1_000; +// jiff's `Timestamp` only accepts UTC seconds in +// [-377705023201, 253402207200] so that any +/-25:59:59 offset still +// yields a valid civil datetime. Clamp after splitting into seconds +// and sub-second nanoseconds to avoid constructing out-of-range values. +const JIFF_TIMESTAMP_MIN_SEC: i64 = -377705023201; +const JIFF_TIMESTAMP_MAX_SEC: i64 = 253402207200; + +pub fn timestamp_from_micros(micros: impl AsPrimitive, tz: &TimeZone) -> Zoned { + // Can't use `tz.timestamp_nanos(micros.as_() * 1000)` directly, as it may overflow. + let micros = micros.as_(); + let (mut secs, mut nanos) = (micros / MICROS_PER_SEC, (micros % MICROS_PER_SEC) * 1_000); + if nanos < 0 { + secs -= 1; + nanos += 1_000_000_000; + } + if secs > JIFF_TIMESTAMP_MAX_SEC { + secs = JIFF_TIMESTAMP_MAX_SEC; + nanos = 0; + } else if secs < JIFF_TIMESTAMP_MIN_SEC { + secs = JIFF_TIMESTAMP_MIN_SEC; + nanos = 0; + } + let ts = Timestamp::new(secs, nanos as i32).unwrap(); + ts.to_zoned(tz.clone()) +} + pub const PRECISION_MICRO: u8 = 6; pub const PRECISION_MILLI: u8 = 3; pub const PRECISION_SEC: u8 = 0; @@ -194,6 +221,6 @@ pub fn string_to_timestamp( #[inline] pub fn timestamp_to_string(ts: i64, tz: &TimeZone) -> impl Display { - let zdt = ts.to_timestamp(tz); + let zdt = timestamp_from_micros(ts, tz); strtime::format(TIMESTAMP_FORMAT, &zdt).unwrap() } diff --git a/src/query/expression/src/utils/mod.rs b/src/query/expression/src/utils/mod.rs index 03a21207bfd2e..67b3c48a47fba 100644 --- a/src/query/expression/src/utils/mod.rs +++ b/src/query/expression/src/utils/mod.rs @@ -19,7 +19,6 @@ pub mod bitmap; pub mod block_debug; pub mod block_thresholds; mod column_from; -pub mod date_helper; pub mod display; pub mod filter_helper; pub mod serialize; diff --git a/src/query/expression/src/values.rs b/src/query/expression/src/values.rs index 84b5d02865b09..d230b222ec550 100755 --- a/src/query/expression/src/values.rs +++ b/src/query/expression/src/values.rs @@ -53,9 +53,11 @@ use serde::de::Visitor; use string::StringColumnBuilder; use crate::BlockEntry; +use crate::ColumnMinMax; use crate::ColumnView; use crate::bitmap::is_hybrid_encoding; use crate::property::Domain; +use crate::property::MinMax; use crate::types::array::ArrayColumn; use crate::types::array::ArrayColumnBuilder; use crate::types::binary::BinaryColumn; @@ -117,6 +119,7 @@ use crate::with_opaque_size_mapped; use crate::with_opaque_type; pub const LARGE_STRING_BYTES_THRESHOLD: usize = 256; +mod domain; #[derive(Debug, Clone, PartialEq, EnumAsInner)] pub enum Value { @@ -1383,109 +1386,6 @@ impl Column { } } - pub fn domain(&self) -> Domain { - if self.len() == 0 { - if matches!(self, Column::Array(_)) { - return Domain::Array(None); - } - if matches!(self, Column::Map(_)) { - return Domain::Map(None); - } - return Domain::full(&self.data_type()); - } - - match self { - Column::Null { .. } => Domain::Nullable(NullableDomain { - has_null: true, - value: None, - }), - Column::EmptyArray { .. } => Domain::Array(None), - Column::EmptyMap { .. } => Domain::Map(None), - - Column::Number(col) => Domain::Number(col.domain()), - Column::Decimal(col) => Domain::Decimal(col.domain()), - Column::Boolean(col) => Domain::Boolean(BooleanDomain { - has_false: col.null_count() > 0, - has_true: col.len() - col.null_count() > 0, - }), - Column::String(col) => { - let (min, max) = StringType::iter_column(col).minmax().into_option().unwrap(); - Domain::String(StringDomain { - min: min.to_string(), - max: Some(max.to_string()), - }) - } - Column::Timestamp(col) => { - let (min, max) = col.iter().minmax().into_option().unwrap(); - Domain::Timestamp(SimpleDomain { - min: *min, - max: *max, - }) - } - Column::TimestampTz(col) => { - let (min, max) = col.iter().minmax().into_option().unwrap(); - Domain::TimestampTz(SimpleDomain { - min: *min, - max: *max, - }) - } - Column::Date(col) => { - let (min, max) = col.iter().minmax().into_option().unwrap(); - Domain::Date(SimpleDomain { - min: *min, - max: *max, - }) - } - Column::Interval(col) => { - let (min, max) = col.iter().minmax().into_option().unwrap(); - Domain::Interval(SimpleDomain { - min: *min, - max: *max, - }) - } - Column::Array(col) => { - if col.len() == 0 { - Domain::Array(None) - } else { - let inner_domain = col.underlying_column().domain(); - Domain::Array(Some(Box::new(inner_domain))) - } - } - Column::Map(col) => { - if col.len() == 0 { - Domain::Map(None) - } else { - let inner_domain = col.underlying_column().domain(); - Domain::Map(Some(Box::new(inner_domain))) - } - } - Column::Nullable(col) => { - let inner_domain = if col.validity.null_count() > 0 { - // goes into the slower path, we will create a new column without nulls - let inner = col.column.clone().filter(&col.validity); - inner.domain() - } else { - col.column.domain() - }; - Domain::Nullable(NullableDomain { - has_null: col.validity.null_count() > 0, - value: Some(Box::new(inner_domain)), - }) - } - Column::Tuple(fields) => { - let domains = fields.iter().map(|col| col.domain()).collect::>(); - Domain::Tuple(domains) - } - Column::Binary(_) - | Column::Bitmap(_) - | Column::Variant(_) - | Column::Geometry(_) - | Column::Geography(_) - | Column::Vector(_) - | Column::Opaque(_) => Domain::Undefined, - } - } - pub fn data_type(&self) -> DataType { match self { Column::Null { .. } => DataType::Null, @@ -1537,12 +1437,13 @@ impl Column { Column::Geography(x) => Ok(x.check_valid()?), Column::Bitmap(x) => Ok(x.check_valid()?), Column::Map(x) => { - for y in x.iter() { - y.check_valid()?; - } - Ok(()) + x.check_valid()?; + x.values().check_valid() + } + Column::Array(x) => { + x.check_valid()?; + x.values().check_valid() } - Column::Array(x) => x.check_valid(), Column::Nullable(x) => { if x.column.len() != x.validity.len() { return Err(ErrorCode::Internal( diff --git a/src/query/expression/src/values/domain.rs b/src/query/expression/src/values/domain.rs new file mode 100644 index 0000000000000..d0cc65125c0a0 --- /dev/null +++ b/src/query/expression/src/values/domain.rs @@ -0,0 +1,447 @@ +// Copyright 2021 Datafuse Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use databend_common_column::bitmap::Bitmap; +use databend_common_column::bitmap::MutableBitmap; +use databend_common_column::buffer::Buffer; +use databend_common_column::types::months_days_micros; +use databend_common_column::types::timestamp_tz; +use databend_common_exception::ErrorCode; +use databend_common_exception::Result; + +use super::Column; +use super::Scalar; +use crate::ColumnMinMax; +use crate::property::Domain; +use crate::property::MinMax; +use crate::types::boolean::BooleanDomain; +use crate::types::nullable::NullableDomain; +use crate::types::string::StringDomain; +use crate::types::*; +use crate::visitor::ValueVisitor; + +fn extrema_from_iter<'a, T, I>(mut values: I) -> SimpleDomain +where + T: AccessType, + T::ScalarRef<'a>: Copy, + I: Iterator>, +{ + let first = values.next().unwrap(); + let mut min = first; + let mut max = first; + + for value in values { + if T::compare(value, min).is_lt() { + min = value; + } + if T::compare(value, max).is_gt() { + max = value; + } + } + + SimpleDomain { + min: T::to_owned_scalar(min), + max: T::to_owned_scalar(max), + } +} + +fn column_extrema( + column: &T::Column, + validity: Option<&Bitmap>, +) -> SimpleDomain +where + for<'a> T::ScalarRef<'a>: Copy, +{ + match validity { + None => extrema_from_iter::(T::iter_column(column)), + Some(validity) => extrema_from_iter::( + T::iter_column(column) + .zip(validity.iter()) + .filter_map(|(value, valid)| valid.then_some(value)), + ), + } +} + +fn boolean_domain(column: &Bitmap, validity: Option<&Bitmap>) -> BooleanDomain { + match validity { + None => { + let true_count = column.true_count(); + BooleanDomain { + has_false: true_count < column.len(), + has_true: true_count > 0, + } + } + Some(validity) => { + let column_chunks = column.chunks::(); + let validity_chunks = validity.chunks::(); + let remainder = (column_chunks.remainder(), validity_chunks.remainder()); + let mut has_false = false; + let mut has_true = false; + for (column, validity) in column_chunks + .zip(validity_chunks) + .chain(std::iter::once(remainder)) + { + has_false |= !column & validity != 0; + has_true |= column & validity != 0; + if has_false && has_true { + break; + } + } + + BooleanDomain { + has_false, + has_true, + } + } + } +} + +fn string_domain(column: &StringColumn, validity: Option<&Bitmap>) -> SimpleDomain { + let (min, max) = match validity { + None => column.min_max().unwrap(), + Some(validity) => { + let mut indices = validity + .iter() + .enumerate() + .filter_map(|(index, valid)| valid.then_some(index)); + let first = indices.next().unwrap(); + let mut min = first; + let mut max = first; + for index in indices { + if StringColumn::compare(column, index, column, min).is_lt() { + min = index; + continue; + } + if StringColumn::compare(column, index, column, max).is_gt() { + max = index; + } + } + (column.value(min), column.value(max)) + } + }; + + SimpleDomain { + min: min.to_owned(), + max: max.to_owned(), + } +} + +struct ColumnMinMaxVisitor { + validity: Option, + has_null: bool, +} + +impl ValueVisitor for ColumnMinMaxVisitor { + type U = ColumnMinMax; + + fn visit_scalar(&mut self, _scalar: Scalar) -> Result { + unreachable!("ColumnMinMaxVisitor only visits columns") + } + + fn visit_null(&mut self, len: usize) -> Result { + Ok(ColumnMinMax::AllNull) + } + + fn visit_number(&mut self, column: Buffer) -> Result { + let domain = column_extrema::>(&column, self.validity.as_ref()); + Ok(ColumnMinMax::Values(MinMax::Number( + T::upcast_domain(domain), + self.has_null, + ))) + } + + fn visit_decimal( + &mut self, + column: Buffer, + size: DecimalSize, + ) -> Result { + let domain = column_extrema::>(&column, self.validity.as_ref()); + let domain = T::upcast_domain(domain, size).into_decimal().unwrap(); + Ok(ColumnMinMax::Values(MinMax::Decimal(domain, self.has_null))) + } + + fn visit_boolean(&mut self, column: Bitmap) -> Result { + Ok(ColumnMinMax::Values(MinMax::Boolean( + boolean_domain(&column, self.validity.as_ref()), + self.has_null, + ))) + } + + fn visit_string(&mut self, column: StringColumn) -> Result { + Ok(ColumnMinMax::Values(MinMax::String( + string_domain(&column, self.validity.as_ref()), + self.has_null, + ))) + } + + fn visit_timestamp(&mut self, column: Buffer) -> Result { + let domain = column_extrema::(&column, self.validity.as_ref()); + Ok(ColumnMinMax::Values(MinMax::Timestamp( + domain, + self.has_null, + ))) + } + + fn visit_timestamp_tz(&mut self, column: Buffer) -> Result { + let domain = column_extrema::(&column, self.validity.as_ref()); + Ok(ColumnMinMax::Values(MinMax::TimestampTz( + domain, + self.has_null, + ))) + } + + fn visit_date(&mut self, column: Buffer) -> Result { + let domain = column_extrema::(&column, self.validity.as_ref()); + Ok(ColumnMinMax::Values(MinMax::Date(domain, self.has_null))) + } + + fn visit_interval(&mut self, column: Buffer) -> Result { + let domain = column_extrema::(&column, self.validity.as_ref()); + Ok(ColumnMinMax::Values(MinMax::Interval( + domain, + self.has_null, + ))) + } + + fn visit_nullable(&mut self, column: Box>) -> Result { + let null_count = column.validity.null_count(); + if null_count == column.len() { + return Ok(ColumnMinMax::AllNull); + } + + self.has_null = null_count > 0; + self.validity = self.has_null.then_some(column.validity); + self.visit_column(column.column) + } + + fn visit_typed_column( + &mut self, + _column: T::Column, + data_type: &DataType, + ) -> Result { + Err(ErrorCode::UnsupportedDataType(format!( + "min/max is not supported for column type {}", + data_type + ))) + } +} + +struct ColumnDomainVisitor { + validity: Option, +} + +impl ColumnDomainVisitor { + fn visible_count(&self, len: usize) -> usize { + self.validity + .as_ref() + .map_or(len, |validity| validity.true_count()) + } + + fn combined_validity(&self, validity: &Bitmap) -> Bitmap { + self.validity + .as_ref() + .map_or_else(|| validity.clone(), |current| current & validity) + } + + fn array_values_validity(&self, column: &ArrayColumn) -> Option { + let validity = self.validity.as_ref()?; + if validity.null_count() == 0 { + return None; + } + + let offsets = column.offsets(); + let values_len = (offsets.last().unwrap() - offsets.first().unwrap()) as usize; + let mut values_validity = MutableBitmap::with_capacity(values_len); + for (index, offsets) in offsets.windows(2).enumerate() { + values_validity + .extend_constant((offsets[1] - offsets[0]) as usize, validity.get_bit(index)); + } + let values_validity: Bitmap = values_validity.into(); + (values_validity.null_count() > 0).then_some(values_validity) + } +} + +impl ValueVisitor for ColumnDomainVisitor { + type U = Domain; + type Error = !; + + fn visit_scalar(&mut self, _scalar: Scalar) -> std::result::Result { + unreachable!("ColumnDomainVisitor only visits columns") + } + + fn visit_column(&mut self, column: Column) -> std::result::Result { + if self.visible_count(column.len()) == 0 { + return Ok(match &column { + Column::Array(_) => Domain::Array(None), + Column::Map(_) => Domain::Map(None), + _ => Domain::full(&column.data_type()), + }); + } + + Self::default_visit_column(column, self) + } + + fn visit_null(&mut self, _len: usize) -> std::result::Result { + Ok(Domain::Nullable(NullableDomain { + has_null: true, + value: None, + })) + } + + fn visit_empty_array(&mut self, _len: usize) -> std::result::Result { + Ok(Domain::Array(None)) + } + + fn visit_empty_map(&mut self, _len: usize) -> std::result::Result { + Ok(Domain::Map(None)) + } + + fn visit_number(&mut self, column: Buffer) -> std::result::Result { + let domain = column_extrema::>(&column, self.validity.as_ref()); + Ok(Domain::Number(T::upcast_domain(domain))) + } + + fn visit_decimal( + &mut self, + column: Buffer, + size: DecimalSize, + ) -> std::result::Result { + let domain = column_extrema::>(&column, self.validity.as_ref()); + Ok(T::upcast_domain(domain, size)) + } + + fn visit_boolean(&mut self, column: Bitmap) -> std::result::Result { + Ok(Domain::Boolean(boolean_domain( + &column, + self.validity.as_ref(), + ))) + } + + fn visit_string(&mut self, column: StringColumn) -> std::result::Result { + let SimpleDomain { min, max } = string_domain(&column, self.validity.as_ref()); + Ok(Domain::String(StringDomain { + min, + max: Some(max), + })) + } + + fn visit_timestamp(&mut self, column: Buffer) -> std::result::Result { + Ok(Domain::Timestamp(column_extrema::( + &column, + self.validity.as_ref(), + ))) + } + + fn visit_timestamp_tz( + &mut self, + column: Buffer, + ) -> std::result::Result { + Ok(Domain::TimestampTz(column_extrema::( + &column, + self.validity.as_ref(), + ))) + } + + fn visit_date(&mut self, column: Buffer) -> std::result::Result { + Ok(Domain::Date(column_extrema::( + &column, + self.validity.as_ref(), + ))) + } + + fn visit_interval( + &mut self, + column: Buffer, + ) -> std::result::Result { + Ok(Domain::Interval(column_extrema::( + &column, + self.validity.as_ref(), + ))) + } + + fn visit_array( + &mut self, + column: Box>, + ) -> std::result::Result { + let validity = self.array_values_validity(&column); + Ok(Domain::Array(Some(Box::new( + column.underlying_column().domain_with_validity(validity), + )))) + } + + fn visit_map(&mut self, column: Box>) -> std::result::Result { + let validity = self.array_values_validity(&column); + Ok(Domain::Map(Some(Box::new( + column.underlying_column().domain_with_validity(validity), + )))) + } + + fn visit_tuple(&mut self, columns: Vec) -> std::result::Result { + Ok(Domain::Tuple( + columns + .into_iter() + .map(|column| column.domain_with_validity(self.validity.clone())) + .collect(), + )) + } + + fn visit_nullable( + &mut self, + column: Box>, + ) -> std::result::Result { + let visible_count = self.visible_count(column.len()); + let validity = self.combined_validity(&column.validity); + let has_null = validity.true_count() < visible_count; + let inner_validity = (validity.null_count() > 0).then_some(validity); + let inner_domain = column.column.domain_with_validity(inner_validity); + Ok(Domain::Nullable(NullableDomain { + has_null, + value: Some(Box::new(inner_domain)), + })) + } + + fn visit_typed_column( + &mut self, + _column: T::Column, + _data_type: &DataType, + ) -> std::result::Result { + Ok(Domain::Undefined) + } +} + +impl Column { + /// Return exact extrema for comparable scalar columns. + /// + /// Empty and all-NULL columns are represented separately. Types without a + /// supported ordering return an error. + pub fn min_max(&self) -> Result { + if self.len() == 0 { + return Ok(ColumnMinMax::Empty); + } + + ColumnMinMaxVisitor { + validity: None, + has_null: false, + } + .visit_column(self.clone()) + } + + pub fn domain(&self) -> Domain { + self.clone().domain_with_validity(None) + } + + fn domain_with_validity(self, validity: Option) -> Domain { + ColumnDomainVisitor { validity }.visit_column(self).unwrap() + } +} diff --git a/src/query/expression/tests/it/constant_folder.rs b/src/query/expression/tests/it/constant_folder.rs index 411a752d0fab7..eb0afc78c8198 100644 --- a/src/query/expression/tests/it/constant_folder.rs +++ b/src/query/expression/tests/it/constant_folder.rs @@ -204,7 +204,7 @@ fn fold(expr: &Expr) -> Expr { } #[test] -fn test_monotonic_nullable_domain_rejects_boundary_probe() { +fn test_monotonic_nullable_domain_probes_non_null_boundaries() { let mut registry = FunctionRegistry::empty(); registry.register_passthrough_nullable_1_arg::( "identity", @@ -213,7 +213,7 @@ fn test_monotonic_nullable_domain_rejects_boundary_probe() { ); registry.properties.insert( "identity".to_string(), - FunctionProperty::default().monotonicity(), + FunctionProperty::default().monotonicity_type(DataType::Number(NumberDataType::UInt64)), ); let data_type = DataType::Number(NumberDataType::UInt64).wrap_nullable(); @@ -245,7 +245,72 @@ fn test_monotonic_nullable_domain_rejects_boundary_probe() { ); assert_eq!(folded, expr); - assert_eq!(output_domain, None); + assert_eq!( + output_domain, + Some(Domain::Nullable(NullableDomain { + has_null: true, + value: Some(Box::new(Domain::Number(NumberDomain::UInt64( + SimpleDomain { min: 10, max: 20 }, + )))), + })) + ); +} + +#[test] +fn test_monotonic_domain_preserves_successful_boundary_on_error() { + let mut registry = FunctionRegistry::empty(); + registry.register_1_arg::( + "fallible_identity", + |_, _| FunctionDomain::Full, + |value, ctx| { + if value == 10 { + ctx.set_error(0, "lower boundary failed"); + } + value + }, + ); + registry.properties.insert( + "fallible_identity".to_string(), + FunctionProperty::default().monotonicity(), + ); + + let data_type = DataType::Number(NumberDataType::UInt64).wrap_nullable(); + let expr = databend_common_expression::type_check::check_function( + None, + "fallible_identity", + &[], + &[Expr::ColumnRef(ColumnRef { + span: None, + id: 0, + data_type, + display_name: "a".to_string(), + })], + ®istry, + ) + .unwrap(); + let input_domain = Domain::Nullable(NullableDomain { + has_null: false, + value: Some(Box::new(Domain::Number(NumberDomain::UInt64( + SimpleDomain { min: 10, max: 20 }, + )))), + }); + + let (_, output_domain) = ConstantFolder::fold_with_domain( + &expr, + &HashMap::from([(0, input_domain)]), + &FunctionContext::default(), + ®istry, + ); + + assert_eq!( + output_domain, + Some(Domain::Nullable(NullableDomain { + has_null: true, + value: Some(Box::new(Domain::Number(NumberDomain::UInt64( + SimpleDomain { min: 0, max: 20 }, + )))), + })) + ); } #[test] diff --git a/src/query/expression/tests/it/main.rs b/src/query/expression/tests/it/main.rs index 0ca0e0489d166..e47a5e2f4bf90 100644 --- a/src/query/expression/tests/it/main.rs +++ b/src/query/expression/tests/it/main.rs @@ -39,6 +39,7 @@ mod group_by; mod hilbert; mod kernel; mod meta_scalar; +mod min_max; mod schema; mod serde; diff --git a/src/query/expression/tests/it/min_max.rs b/src/query/expression/tests/it/min_max.rs new file mode 100644 index 0000000000000..6de167c16df8e --- /dev/null +++ b/src/query/expression/tests/it/min_max.rs @@ -0,0 +1,262 @@ +// Copyright 2026 Datafuse Labs. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use databend_common_column::bitmap::Bitmap; +use databend_common_expression::Column; +use databend_common_expression::ColumnMinMax; +use databend_common_expression::Domain; +use databend_common_expression::FromData; +use databend_common_expression::MinMax; +use databend_common_expression::Scalar; +use databend_common_expression::types::AnyType; +use databend_common_expression::types::ArrayColumn; +use databend_common_expression::types::BinaryType; +use databend_common_expression::types::BooleanType; +use databend_common_expression::types::DataType; +use databend_common_expression::types::Int32Type; +use databend_common_expression::types::NumberDataType; +use databend_common_expression::types::SimpleDomain; +use databend_common_expression::types::StringType; +use databend_common_expression::types::nullable::NullableColumn; +use databend_common_expression::types::nullable::NullableDomain; +use databend_common_expression::types::number::NumberDomain; + +#[test] +fn test_nullable_column_min_max_excludes_null_placeholders() { + let column = NullableColumn::new_column( + Int32Type::from_data(vec![99, 4, 1]), + Bitmap::from([false, true, true]), + ); + + let min_max = column.min_max().unwrap().into_option().unwrap(); + assert!(min_max.has_null()); + assert_eq!(min_max.scalars().0, 1i32.into()); + assert_eq!(min_max.scalars().1, 4i32.into()); +} + +#[test] +fn test_nullable_column_min_max_filters_without_materializing() { + let boolean = BooleanType::from_opt_data(vec![None, Some(true), Some(true)]); + let boolean = boolean.min_max().unwrap().into_option().unwrap(); + assert_eq!( + boolean.scalars(), + (Scalar::Boolean(true), Scalar::Boolean(true)) + ); + assert!(boolean.has_null()); + + let boolean = BooleanType::from_opt_data(vec![Some(false), None, Some(true)]); + let boolean = boolean.min_max().unwrap().into_option().unwrap(); + assert_eq!( + boolean.scalars(), + (Scalar::Boolean(false), Scalar::Boolean(true)) + ); + assert!(boolean.has_null()); + + let string = StringType::from_opt_data(vec![Some("z"), None, Some("a")]); + let string = string.min_max().unwrap().into_option().unwrap(); + assert_eq!( + string.scalars(), + ( + Scalar::String("a".to_string()), + Scalar::String("z".to_string()) + ) + ); + assert!(string.has_null()); +} + +#[test] +fn test_nullable_array_domain_excludes_values_from_null_rows() { + let array = Column::Array(Box::new(ArrayColumn::::new( + Int32Type::from_data(vec![99, 100, 1, 4]), + vec![0_u64, 2, 4].into(), + ))); + let nullable = NullableColumn::new_column(array, Bitmap::from([false, true])); + + assert_eq!( + nullable.domain(), + Domain::Nullable(NullableDomain { + has_null: true, + value: Some(Box::new(Domain::Array(Some(Box::new(Domain::Number( + NumberDomain::Int32(SimpleDomain { min: 1, max: 4 }), + )))))), + }) + ); +} + +#[test] +fn test_nullable_domain_preserves_legacy_empty_inner_boundaries() { + let int32_type = DataType::Number(NumberDataType::Int32); + let full_int32 = Domain::full(&int32_type); + + let all_null = NullableColumn::new_column( + Int32Type::from_data(vec![9, 1]), + Bitmap::from([false, false]), + ); + assert_eq!( + all_null.domain(), + Domain::Nullable(NullableDomain { + has_null: true, + value: Some(Box::new(full_int32.clone())), + }) + ); + + let values_only_in_null_row = NullableColumn::new_column( + Column::Array(Box::new(ArrayColumn::::new( + Int32Type::from_data(vec![9, 1]), + vec![0_u64, 2, 2].into(), + ))), + Bitmap::from([false, true]), + ); + assert_eq!( + values_only_in_null_row.domain(), + Domain::Nullable(NullableDomain { + has_null: true, + value: Some(Box::new(Domain::Array(Some(Box::new(full_int32))))), + }) + ); +} + +#[test] +fn test_nullable_tuple_domain_uses_only_parent_visible_rows() { + let column = NullableColumn::new_column( + Column::Tuple(vec![NullableColumn::new_column( + Int32Type::from_data(vec![99, 1, 4]), + Bitmap::from([false, true, false]), + )]), + Bitmap::from([false, true, true]), + ); + + assert_eq!( + column.domain(), + Domain::Nullable(NullableDomain { + has_null: true, + value: Some(Box::new(Domain::Tuple(vec![Domain::Nullable( + NullableDomain { + has_null: true, + value: Some(Box::new(Domain::Number(NumberDomain::Int32( + SimpleDomain { min: 1, max: 1 }, + )))), + }, + )]))), + }) + ); +} + +#[test] +fn test_nullable_sliced_array_domain_uses_underlying_offsets() { + let array = Column::Array(Box::new( + ArrayColumn::::new( + Int32Type::from_data(vec![99, 98, 4, 1, 97]), + vec![0_u64, 2, 4, 5].into(), + ) + .slice(1..3), + )); + let column = NullableColumn::new_column(array, Bitmap::from([true, false])); + + assert_eq!( + column.domain(), + Domain::Nullable(NullableDomain { + has_null: true, + value: Some(Box::new(Domain::Array(Some(Box::new(Domain::Number( + NumberDomain::Int32(SimpleDomain { min: 1, max: 4 }), + )))))), + }) + ); +} + +#[test] +fn test_empty_tuple_domain_remains_full() { + let column = Column::Tuple(vec![Column::Array(Box::new(ArrayColumn::::new( + Int32Type::from_data(vec![]), + vec![0_u64].into(), + )))]); + + assert_eq!(column.domain(), Domain::full(&column.data_type())); +} + +#[test] +fn test_empty_and_all_null_min_max_are_distinct_outer_states() { + assert_eq!( + Int32Type::from_data(vec![]).min_max().unwrap(), + ColumnMinMax::Empty + ); + + let all_null = NullableColumn::new_column( + Int32Type::from_data(vec![7, 8]), + Bitmap::from([false, false]), + ) + .min_max() + .unwrap(); + assert_eq!(all_null, ColumnMinMax::AllNull); + + let values = Int32Type::from_data(vec![3, 5]) + .min_max() + .unwrap() + .into_option() + .unwrap(); + assert!(!values.has_null()); + assert_eq!(values.scalars().0, 3i32.into()); + assert_eq!(values.scalars().1, 5i32.into()); + + assert!(matches!( + values, + MinMax::Number(NumberDomain::Int32(_), false) + )); +} + +#[test] +fn test_min_max_rejects_unsupported_type() { + let binary = BinaryType::from_data(vec![b"a".as_slice(), b"b".as_slice()]); + assert!(binary.min_max().is_err()); +} + +#[test] +fn test_min_max_merge_and_serde_reuse_typed_range() { + let mut lhs = MinMax::Number(NumberDomain::Int32(SimpleDomain { min: 3, max: 5 }), false); + let rhs = MinMax::Number(NumberDomain::Int32(SimpleDomain { min: 1, max: 4 }), true); + + lhs.merge(&rhs).unwrap(); + assert_eq!(lhs.scalars(), (1i32.into(), 5i32.into())); + assert!(lhs.has_null()); + + let encoded = serde_json::to_vec(&lhs).unwrap(); + let decoded: MinMax = serde_json::from_slice(&encoded).unwrap(); + assert_eq!(decoded, lhs); +} + +#[test] +fn test_min_max_merge_rejects_incompatible_other() { + let mut lhs = MinMax::Number(NumberDomain::Int32(SimpleDomain { min: 3, max: 5 }), false); + let original = lhs.clone(); + let rhs = MinMax::Number(NumberDomain::Int64(SimpleDomain { min: 1, max: 4 }), true); + + assert!(lhs.merge(&rhs).is_err()); + assert_eq!(lhs, original); +} + +#[test] +fn test_column_min_max_merge_preserves_all_null_state() { + let mut merged = ColumnMinMax::Empty; + merged.merge(&ColumnMinMax::AllNull).unwrap(); + merged + .merge(&ColumnMinMax::Values(MinMax::Number( + NumberDomain::Int32(SimpleDomain { min: 3, max: 5 }), + false, + ))) + .unwrap(); + + let min_max = merged.into_option().unwrap(); + assert!(min_max.has_null()); + assert_eq!(min_max.scalars(), (3i32.into(), 5i32.into())); +} diff --git a/src/query/formats/src/output_format/json.rs b/src/query/formats/src/output_format/json.rs index cca787a468344..34a5c38a6ac28 100644 --- a/src/query/formats/src/output_format/json.rs +++ b/src/query/formats/src/output_format/json.rs @@ -15,10 +15,11 @@ use databend_common_expression::DataBlock; use databend_common_expression::ScalarRef; use databend_common_expression::TableSchemaRef; -use databend_common_expression::date_helper::DateConverter; use databend_common_expression::types::VectorScalarRef; +use databend_common_expression::types::date::date_from_days; use databend_common_expression::types::interval::interval_to_string; use databend_common_expression::types::number::NumberScalar; +use databend_common_expression::types::timestamp::timestamp_from_micros; use databend_common_io::deserialize_bitmap; use databend_common_io::prelude::OutputFormatSettings; use geozero::ToJson; @@ -94,14 +95,14 @@ fn scalar_to_json( }), ScalarRef::Decimal(x) => Ok(serde_json::to_value(x.to_string()).unwrap()), ScalarRef::Date(v) => { - let dt = DateConverter::to_date(&v, &format.jiff_timezone); + let dt = date_from_days(v); Ok(serde_json::to_value(strtime::format("%Y-%m-%d", dt).unwrap()).unwrap()) } ScalarRef::Interval(v) => { Ok(serde_json::to_value(interval_to_string(&v).to_string()).unwrap()) } ScalarRef::Timestamp(v) => { - let dt = DateConverter::to_timestamp(&v, &format.jiff_timezone); + let dt = timestamp_from_micros(v, &format.jiff_timezone); Ok(serde_json::to_value(strtime::format("%Y-%m-%d %H:%M:%S", &dt).unwrap()).unwrap()) } ScalarRef::TimestampTz(v) => Ok(serde_json::to_value(v.to_string()).unwrap()), diff --git a/src/query/functions/benches/bench.rs b/src/query/functions/benches/bench.rs index 37109aa2b7097..647208de53f99 100644 --- a/src/query/functions/benches/bench.rs +++ b/src/query/functions/benches/bench.rs @@ -318,12 +318,12 @@ mod datetime_fast_path { use databend_common_expression::Evaluator; use databend_common_expression::Expr; use databend_common_expression::FunctionContext; - use databend_common_expression::date_helper::DateConverter; use databend_common_expression::type_check; use databend_common_expression::types::DataType; use databend_common_expression::types::string::StringColumn; use databend_common_expression::types::string::StringColumnBuilder; use databend_common_expression::types::timestamp::microseconds_to_days; + use databend_common_expression::types::timestamp::timestamp_from_micros; use databend_common_expression::types::timestamp::timestamp_to_string; use databend_common_expression_test_support as parser; use databend_common_functions::BUILTIN_FUNCTIONS; @@ -360,7 +360,7 @@ mod datetime_fast_path { let formatted = timestamp_to_string(micros, &tz_sh).to_string(); string_builder.put_and_commit(formatted); - let zoned = micros.to_timestamp(&tz_sh); + let zoned = timestamp_from_micros(micros, &tz_sh); let offset_secs = zoned.offset().seconds(); let offset_hours = offset_secs / 3600; let offset_minutes = (offset_secs.abs() % 3600) / 60; diff --git a/src/query/expression/src/utils/date_helper.rs b/src/query/functions/src/scalars/timestamp/src/date_helper.rs similarity index 91% rename from src/query/expression/src/utils/date_helper.rs rename to src/query/functions/src/scalars/timestamp/src/date_helper.rs index 0a18e49250e58..be66a0293cc10 100644 --- a/src/query/expression/src/utils/date_helper.rs +++ b/src/query/functions/src/scalars/timestamp/src/date_helper.rs @@ -15,7 +15,11 @@ use std::sync::LazyLock; use databend_common_column::types::timestamp_tz; -use databend_common_exception::Result; +use databend_common_expression::types::date::clamp_date; +use databend_common_expression::types::date::date_from_days; +use databend_common_expression::types::timestamp::MICROS_PER_SEC; +use databend_common_expression::types::timestamp::clamp_timestamp; +use databend_common_expression::types::timestamp::timestamp_from_micros; use databend_common_timezone::DateTimeComponents; use databend_common_timezone::fast_components_from_timestamp; use databend_common_timezone::fast_utc_from_local; @@ -34,50 +38,6 @@ use jiff::civil::datetime; use jiff::tz::TimeZone; use num_traits::AsPrimitive; -use crate::types::date::clamp_date; -use crate::types::timestamp::MICROS_PER_SEC; -use crate::types::timestamp::clamp_timestamp; - -// jiff's `Timestamp` only accepts UTC seconds in -// [-377705023201, 253402207200] so that any ±25:59:59 offset still -// yields a valid civil datetime. Clamp after splitting into seconds -// and sub-second nanoseconds to avoid constructing out-of-range values. -const JIFF_TIMESTAMP_MIN_SEC: i64 = -377705023201; -const JIFF_TIMESTAMP_MAX_SEC: i64 = 253402207200; - -pub trait DateConverter { - fn to_date(&self, tz: &TimeZone) -> Date; - fn to_timestamp(&self, tz: &TimeZone) -> Zoned; -} - -impl DateConverter for T -where T: AsPrimitive -{ - fn to_date(&self, _tz: &TimeZone) -> Date { - let dur = SignedDuration::from_hours(self.as_() * 24); - date(1970, 1, 1).checked_add(dur).unwrap() - } - - fn to_timestamp(&self, tz: &TimeZone) -> Zoned { - // Can't use `tz.timestamp_nanos(self.as_() * 1000)` directly, is may cause multiply with overflow. - let micros = self.as_(); - let (mut secs, mut nanos) = (micros / MICROS_PER_SEC, (micros % MICROS_PER_SEC) * 1_000); - if nanos < 0 { - secs -= 1; - nanos += 1_000_000_000; - } - if secs > JIFF_TIMESTAMP_MAX_SEC { - secs = JIFF_TIMESTAMP_MAX_SEC; - nanos = 0; - } else if secs < JIFF_TIMESTAMP_MIN_SEC { - secs = JIFF_TIMESTAMP_MIN_SEC; - nanos = 0; - } - let ts = Timestamp::new(secs, nanos as i32).unwrap(); - ts.to_zoned(tz.clone()) - } -} - pub const MICROSECS_PER_DAY: i64 = 86_400_000_000; // Timestamp arithmetic factors. @@ -153,11 +113,10 @@ macro_rules! impl_interval_year_month { impl $name { pub fn eval_date( date: i32, - tz: &TimeZone, delta: impl AsPrimitive, add_months: bool, ) -> std::result::Result { - let date = date.to_date(tz); + let date = date_from_days(date); let new_date = $op( date.year(), date.month(), @@ -180,7 +139,7 @@ macro_rules! impl_interval_year_month { delta: impl AsPrimitive, add_months: bool, ) -> std::result::Result { - let ts = us.to_timestamp(tz); + let ts = timestamp_from_micros(us, tz); let original_offset = ts.offset().seconds(); if let Some(components) = fast_components_from_timestamp(us, tz) { @@ -284,22 +243,22 @@ fn datetime_from_components(c: &DateTimeComponents) -> Option { } impl EvalYearsImpl { - pub fn eval_date_diff(date_start: i32, date_end: i32, tz: &TimeZone) -> i32 { - let date_start = date_start.to_date(tz); - let date_end = date_end.to_date(tz); + pub fn eval_date_diff(date_start: i32, date_end: i32) -> i32 { + let date_start = date_from_days(date_start); + let date_end = date_from_days(date_end); (date_end.year() - date_start.year()) as i32 } - pub fn eval_date_between(date_start: i32, date_end: i32, tz: &TimeZone) -> i32 { + pub fn eval_date_between(date_start: i32, date_end: i32) -> i32 { if date_start == date_end { return 0; } if date_start > date_end { - return -Self::eval_date_between(date_end, date_start, tz); + return -Self::eval_date_between(date_end, date_start); } - let date_start = date_start.to_date(tz); - let date_end = date_end.to_date(tz); + let date_start = date_from_days(date_start); + let date_end = date_from_days(date_end); let mut years = date_end.year() - date_start.year(); @@ -322,8 +281,8 @@ impl EvalYearsImpl { ) { return (end.year as i64) - (start.year as i64); } - let date_start = date_start.to_timestamp(tz); - let date_end = date_end.to_timestamp(tz); + let date_start = timestamp_from_micros(date_start, tz); + let date_end = timestamp_from_micros(date_end, tz); date_end.year() as i64 - date_start.year() as i64 } @@ -351,8 +310,8 @@ impl EvalYearsImpl { } return years as i64; } - let start = date_start.to_timestamp(tz); - let end = date_end.to_timestamp(tz); + let start = timestamp_from_micros(date_start, tz); + let end = timestamp_from_micros(date_end, tz); let mut years = end.year() - start.year(); @@ -383,21 +342,21 @@ impl EvalYearsImpl { pub struct EvalISOYearsImpl; impl EvalISOYearsImpl { - pub fn eval_date_diff(date_start: i32, date_end: i32, tz: &TimeZone) -> i32 { - let date_start = date_start.to_date(tz); - let date_end = date_end.to_date(tz); + pub fn eval_date_diff(date_start: i32, date_end: i32) -> i32 { + let date_start = date_from_days(date_start); + let date_end = date_from_days(date_end); date_end.iso_week_date().year() as i32 - date_start.iso_week_date().year() as i32 } - pub fn eval_date_between(date_start: i32, date_end: i32, tz: &TimeZone) -> i32 { + pub fn eval_date_between(date_start: i32, date_end: i32) -> i32 { if date_start == date_end { return 0; } if date_start > date_end { - return -Self::eval_date_between(date_end, date_start, tz); + return -Self::eval_date_between(date_end, date_start); } - let date_start = date_start.to_date(tz); - let date_end = date_end.to_date(tz); + let date_start = date_from_days(date_start); + let date_end = date_from_days(date_end); let mut years = date_end.iso_week_date().year() - date_start.iso_week_date().year(); if (date_end.month() < date_start.month()) || (date_end.month() == date_start.month() && date_end.day() < date_start.day()) @@ -417,8 +376,8 @@ impl EvalISOYearsImpl { let (end_year, _) = end.iso_year_week(); return (end_year - start_year) as i64; } - let date_start = date_start.to_timestamp(tz); - let date_end = date_end.to_timestamp(tz); + let date_start = timestamp_from_micros(date_start, tz); + let date_end = timestamp_from_micros(date_end, tz); date_end.date().iso_week_date().year() as i64 - date_start.iso_week_date().year() as i64 } @@ -447,8 +406,8 @@ impl EvalISOYearsImpl { return years as i64; } - let start = date_start.to_timestamp(tz); - let end = date_end.to_timestamp(tz); + let start = timestamp_from_micros(date_start, tz); + let end = timestamp_from_micros(date_end, tz); let mut years = end.date().iso_week_date().year() as i64 - start.date().iso_week_date().year() as i64; let start_month = start.month(); @@ -485,9 +444,9 @@ impl EvalYearWeeksImpl { year * 100 + week as i32 } - pub fn eval_date_diff(date_start: i32, date_end: i32, tz: &TimeZone) -> i32 { - let date_start = date_start.to_date(tz); - let date_end = date_end.to_date(tz); + pub fn eval_date_diff(date_start: i32, date_end: i32) -> i32 { + let date_start = date_from_days(date_start); + let date_end = date_from_days(date_end); let end = Self::yearweek(date_end); let start = Self::yearweek(date_start); @@ -503,8 +462,8 @@ impl EvalYearWeeksImpl { let end_yw = Self::yearweek_from_components(&end) as i64; return end_yw - start_yw; } - let date_start = date_start.to_timestamp(tz); - let date_end = date_end.to_timestamp(tz); + let date_start = timestamp_from_micros(date_start, tz); + let date_end = timestamp_from_micros(date_end, tz); let end = Self::yearweek(date_end.date()) as i64; let start = Self::yearweek(date_start.date()) as i64; @@ -530,8 +489,8 @@ impl EvalYearWeeksImpl { // (end, start, -1) // }; // - // let earlier = earlier.to_date(tz); - // let later = later.to_date(tz); + // let earlier = date_from_days(earlier); + // let later = date_from_days(later); // // let start_yw = Self::yearweek(earlier); // let end_yw = Self::yearweek(later); @@ -556,8 +515,8 @@ impl EvalYearWeeksImpl { // (end, start, -1) // }; // - // let earlier = earlier.to_timestamp(tz); - // let later = later.to_timestamp(tz); + // let earlier = timestamp_from_micros(earlier, tz); + // let later = timestamp_from_micros(later, tz); // // let start_yw = Self::yearweek(earlier.date()); // let end_yw = Self::yearweek(later.date()); @@ -576,12 +535,12 @@ impl EvalYearWeeksImpl { pub struct EvalQuartersImpl; impl EvalQuartersImpl { - pub fn eval_date_diff(date_start: i32, date_end: i32, tz: &TimeZone) -> i32 { - EvalQuartersImpl::eval_timestamp_diff( - date_start as i64 * MICROSECS_PER_DAY, - date_end as i64 * MICROSECS_PER_DAY, - tz, - ) as i32 + pub fn eval_date_diff(date_start: i32, date_end: i32) -> i32 { + let date_start = date_from_days(date_start); + let date_end = date_from_days(date_end); + let start_quarter = (date_start.month() as i32 - 1) / 3 + 1; + let end_quarter = (date_end.month() as i32 - 1) / 3 + 1; + (date_end.year() - date_start.year()) as i32 * 4 + end_quarter - start_quarter } pub fn eval_timestamp_diff(date_start: i64, date_end: i64, tz: &TimeZone) -> i64 { @@ -593,8 +552,8 @@ impl EvalQuartersImpl { let end_quarter = ((end.month as i64 - 1) / 3) + 1; return (end.year as i64 - start.year as i64) * 4 + end_quarter - start_quarter; } - let date_start = date_start.to_timestamp(tz); - let date_end = date_end.to_timestamp(tz); + let date_start = timestamp_from_micros(date_start, tz); + let date_end = timestamp_from_micros(date_end, tz); (date_end.year() - date_start.year()) as i64 * 4 + ToQuarter::to_number(&date_end) as i64 - ToQuarter::to_number(&date_start) as i64 } @@ -622,8 +581,8 @@ impl EvalQuartersImpl { // (end, start, -1) // }; // - // let earlier = earlier.to_date(tz); - // let later = later.to_date(tz); + // let earlier = date_from_days(earlier); + // let later = date_from_days(later); // // let start_year = earlier.year(); // let start_quarter = Self::quarter(earlier.month()); @@ -655,8 +614,8 @@ impl EvalQuartersImpl { // (end, start, -1) // }; // - // let earlier = earlier.to_timestamp(tz); - // let later = later.to_timestamp(tz); + // let earlier = timestamp_from_micros(earlier, tz); + // let later = timestamp_from_micros(later, tz); // // let start_year = earlier.year(); // let start_quarter = Self::quarter(earlier.month()); @@ -679,23 +638,23 @@ impl EvalQuartersImpl { } impl EvalMonthsImpl { - pub fn eval_date_diff(date_start: i32, date_end: i32, tz: &TimeZone) -> i32 { - let date_start = date_start.to_date(tz); - let date_end = date_end.to_date(tz); + pub fn eval_date_diff(date_start: i32, date_end: i32) -> i32 { + let date_start = date_from_days(date_start); + let date_end = date_from_days(date_end); (date_end.year() - date_start.year()) as i32 * 12 + date_end.month() as i32 - date_start.month() as i32 } - pub fn eval_date_between(start: i32, end: i32, tz: &TimeZone) -> i32 { + pub fn eval_date_between(start: i32, end: i32) -> i32 { if start == end { return 0; } if start > end { - return -Self::eval_date_between(end, start, tz); + return -Self::eval_date_between(end, start); } - let start = start.to_date(tz); - let end = end.to_date(tz); + let start = date_from_days(start); + let end = date_from_days(end); let year_diff = end.year() - start.year(); let month_diff = end.month() as i32 - start.month() as i32; @@ -708,11 +667,10 @@ impl EvalMonthsImpl { months } - pub fn eval_timestamp_diff(date_start: i64, date_end: i64, tz: &TimeZone) -> i64 { + pub fn eval_timestamp_diff(date_start: i64, date_end: i64) -> i64 { EvalMonthsImpl::eval_date_diff( (date_start / MICROSECS_PER_DAY) as i32, (date_end / MICROSECS_PER_DAY) as i32, - tz, ) as i64 } @@ -738,8 +696,8 @@ impl EvalMonthsImpl { return months; } - let start = start.to_timestamp(tz); - let end = end.to_timestamp(tz); + let start = timestamp_from_micros(start, tz); + let end = timestamp_from_micros(end, tz); let year_diff = end.year() - start.year(); let month_diff = end.month() as i64 - start.month() as i64; let mut months = year_diff as i64 * 12 + month_diff; @@ -849,16 +807,16 @@ impl EvalWeeksImpl { weeks } - pub fn eval_date_between(start: i32, end: i32, tz: &TimeZone) -> i32 { + pub fn eval_date_between(start: i32, end: i32) -> i32 { if start == end { return 0; } if start > end { - return -Self::eval_date_between(end, start, tz); + return -Self::eval_date_between(end, start); } - let earlier = start.to_date(tz); - let later = end.to_date(tz); + let earlier = date_from_days(start); + let later = date_from_days(end); let mut weeks = Self::calculate_weeks_between_years( earlier.year() as i32, later.year() as i32, @@ -911,8 +869,8 @@ impl EvalWeeksImpl { } } - let earlier = start.to_timestamp(tz); - let later = end.to_timestamp(tz); + let earlier = timestamp_from_micros(start, tz); + let later = timestamp_from_micros(end, tz); let mut weeks = Self::calculate_weeks_between_years( earlier.year() as i32, @@ -967,8 +925,8 @@ impl EvalDaysImpl { return -Self::eval_timestamp_between(end, start, tz); } - let start = start.to_timestamp(tz); - let end = end.to_timestamp(tz); + let start = timestamp_from_micros(start, tz); + let end = timestamp_from_micros(end, tz); let mut full_days = (end.date() - start.date()) .to_duration(SpanRelativeTo::days_are_24_hours()) .unwrap() @@ -1038,7 +996,7 @@ pub fn today_date(now: &Zoned, tz: &TimeZone) -> i32 { // The working hours of all departments of The State Council are from 8 a.m. to 12 p.m. and from 1:30 p.m. to 5:30 p.m. The winter working hours will be implemented after September 17th. pub fn calc_date_to_timestamp(val: i32, tz: &TimeZone) -> std::result::Result { let ts = (val as i64) * 24 * 3600 * MICROS_PER_SEC; - let local_date = val.to_date(tz); + let local_date = date_from_days(val); let year = i32::from(local_date.year()); let month = local_date.month() as u8; let day = local_date.day() as u8; @@ -1082,6 +1040,10 @@ pub trait ToNumber { } } +pub trait DateToNumber { + fn to_number_from_date(date: &Date) -> N; +} + pub struct ToNumberImpl; impl ToNumberImpl { @@ -1091,17 +1053,12 @@ impl ToNumberImpl { return value; } } - let dt = us.to_timestamp(tz); + let dt = timestamp_from_micros(us, tz); T::to_number(&dt) } - pub fn eval_date, R>(date: i32, tz: &TimeZone) -> Result { - let dt = date - .to_date(tz) - .at(0, 0, 0, 0) - .to_zoned(tz.clone()) - .unwrap(); - Ok(T::to_number(&dt)) + pub fn eval_date, R>(days: i32) -> R { + T::to_number_from_date(&date_from_days(days as i64)) } } @@ -1138,6 +1095,12 @@ impl ToNumber for ToYYYYMM { } } +impl DateToNumber for ToYYYYMM { + fn to_number_from_date(date: &Date) -> u32 { + date.year() as u32 * 100 + date.month() as u32 + } +} + impl ToNumber for ToMillennium { fn to_number(dt: &Zoned) -> u16 { dt.year() as u16 / 1000 + 1 @@ -1148,6 +1111,12 @@ impl ToNumber for ToMillennium { } } +impl DateToNumber for ToMillennium { + fn to_number_from_date(date: &Date) -> u16 { + date.year() as u16 / 1000 + 1 + } +} + impl ToNumber for ToWeekOfYear { fn to_number(dt: &Zoned) -> u32 { dt.date().iso_week_date().week() as u32 @@ -1158,6 +1127,12 @@ impl ToNumber for ToWeekOfYear { } } +impl DateToNumber for ToWeekOfYear { + fn to_number_from_date(date: &Date) -> u32 { + date.iso_week_date().week() as u32 + } +} + impl ToNumber for ToYYYYMMDD { fn to_number(dt: &Zoned) -> u32 { dt.year() as u32 * 10_000 + dt.month() as u32 * 100 + dt.day() as u32 @@ -1170,6 +1145,12 @@ impl ToNumber for ToYYYYMMDD { } } +impl DateToNumber for ToYYYYMMDD { + fn to_number_from_date(date: &Date) -> u32 { + date.year() as u32 * 10_000 + date.month() as u32 * 100 + date.day() as u32 + } +} + impl ToNumber for ToYYYYMMDDHH { fn to_number(dt: &Zoned) -> u64 { dt.year() as u64 * 1_000_000 @@ -1220,6 +1201,12 @@ impl ToNumber for ToYear { } } +impl DateToNumber for ToYear { + fn to_number_from_date(date: &Date) -> u16 { + date.year() as u16 + } +} + impl ToNumber for ToTimezoneHour { fn to_number(dt: &Zoned) -> i16 { dt.offset().seconds().div_ceil(3600) as i16 @@ -1250,6 +1237,12 @@ impl ToNumber for ToISOYear { } } +impl DateToNumber for ToISOYear { + fn to_number_from_date(date: &Date) -> u16 { + date.iso_week_date().year() as u16 + } +} + impl ToNumber for ToYYYYWW { fn to_number(dt: &Zoned) -> u32 { let week_date = dt.date().iso_week_date(); @@ -1263,6 +1256,13 @@ impl ToNumber for ToYYYYWW { } } +impl DateToNumber for ToYYYYWW { + fn to_number_from_date(date: &Date) -> u32 { + let week_date = date.iso_week_date(); + week_date.year() as u32 * 100 + week_date.week() as u32 + } +} + impl ToNumber for ToQuarter { fn to_number(dt: &Zoned) -> u8 { // begin with 0 @@ -1274,6 +1274,12 @@ impl ToNumber for ToQuarter { } } +impl DateToNumber for ToQuarter { + fn to_number_from_date(date: &Date) -> u8 { + (date.month() as u8 - 1) / 3 + 1 + } +} + impl ToNumber for ToMonth { fn to_number(dt: &Zoned) -> u8 { dt.month() as u8 @@ -1284,6 +1290,12 @@ impl ToNumber for ToMonth { } } +impl DateToNumber for ToMonth { + fn to_number_from_date(date: &Date) -> u8 { + date.month() as u8 + } +} + impl ToNumber for ToDayOfYear { fn to_number(dt: &Zoned) -> u16 { dt.day_of_year() as u16 @@ -1294,6 +1306,12 @@ impl ToNumber for ToDayOfYear { } } +impl DateToNumber for ToDayOfYear { + fn to_number_from_date(date: &Date) -> u16 { + date.day_of_year() as u16 + } +} + impl ToNumber for ToDayOfMonth { fn to_number(dt: &Zoned) -> u8 { dt.day() as u8 @@ -1304,6 +1322,12 @@ impl ToNumber for ToDayOfMonth { } } +impl DateToNumber for ToDayOfMonth { + fn to_number_from_date(date: &Date) -> u8 { + date.day() as u8 + } +} + impl ToNumber for ToDayOfWeek { fn to_number(dt: &Zoned) -> u8 { dt.weekday().to_monday_one_offset() as u8 @@ -1314,6 +1338,12 @@ impl ToNumber for ToDayOfWeek { } } +impl DateToNumber for ToDayOfWeek { + fn to_number_from_date(date: &Date) -> u8 { + date.weekday().to_monday_one_offset() as u8 + } +} + impl ToNumber for DayOfWeek { fn to_number(dt: &Zoned) -> u8 { dt.weekday().to_sunday_zero_offset() as u8 @@ -1324,6 +1354,12 @@ impl ToNumber for DayOfWeek { } } +impl DateToNumber for DayOfWeek { + fn to_number_from_date(date: &Date) -> u8 { + date.weekday().to_sunday_zero_offset() as u8 + } +} + impl ToNumber for ToUnixTimestamp { fn to_number(dt: &Zoned) -> i64 { dt.with_time_zone(TimeZone::UTC).timestamp().as_second() @@ -1347,7 +1383,7 @@ pub enum Round { } pub fn round_timestamp(ts: i64, tz: &TimeZone, round: Round) -> i64 { - let dtz = ts.to_timestamp(tz); + let dtz = timestamp_from_micros(ts, tz); let res = match round { Round::Second => tz .to_zoned(datetime( @@ -1533,7 +1569,7 @@ pub fn time_slice_timestamp( ) -> i64 { let slice_length = slice_length as i64; - let ts = ts.to_timestamp(tz); + let ts = timestamp_from_micros(ts, tz); let dt = ts.datetime(); let start = match part { @@ -1681,17 +1717,16 @@ pub struct DateRounder; impl DateRounder { pub fn eval_timestamp>(us: i64, tz: &TimeZone) -> i32 { - let dt = us.to_timestamp(tz); + let dt = timestamp_from_micros(us, tz); T::to_number(&dt) } - pub fn eval_date>(date: i32, tz: &TimeZone) -> Result { - let naive_dt = date - .to_date(tz) + pub fn eval_date>(date: i32, tz: &TimeZone) -> i32 { + let naive_dt = date_from_days(date) .at(0, 0, 0, 0) .to_zoned(tz.clone()) .unwrap(); - Ok(T::to_number(&naive_dt)) + T::to_number(&naive_dt) } } diff --git a/src/query/functions/src/scalars/timestamp/src/datetime.rs b/src/query/functions/src/scalars/timestamp/src/datetime.rs index 1aa3f806a9954..7c52d280eb43e 100644 --- a/src/query/functions/src/scalars/timestamp/src/datetime.rs +++ b/src/query/functions/src/scalars/timestamp/src/datetime.rs @@ -70,6 +70,7 @@ use databend_common_expression::types::timestamp::TIMESTAMP_MAX; use databend_common_expression::types::timestamp::TIMESTAMP_MIN; use databend_common_expression::types::timestamp::clamp_timestamp; use databend_common_expression::types::timestamp::string_to_timestamp; +use databend_common_expression::types::timestamp::timestamp_from_micros; use databend_common_expression::types::timestamp::timestamp_to_string; use databend_common_expression::types::timestamp_tz::TimestampTzType; use databend_common_expression::utils::auto_detect_datetime::auto_detect_date; @@ -79,7 +80,7 @@ use databend_common_expression::utils::auto_detect_datetime::fast_timestamp_from use databend_common_expression::utils::auto_detect_datetime::int64_to_timestamp; use databend_common_expression::utils::auto_detect_datetime::parse_epoch_str; use databend_common_expression::utils::auto_detect_datetime::parse_timestamp_tz_with_auto; -use databend_common_expression::utils::date_helper::*; +use databend_common_expression::vectorize_1_arg; use databend_common_expression::vectorize_2_arg; use databend_common_expression::vectorize_4_arg; use databend_common_expression::vectorize_with_builder_1_arg; @@ -101,6 +102,8 @@ use jiff::tz::Offset; use jiff::tz::TimeZone; use num_traits::AsPrimitive; +use crate::date_helper::*; + const MONTHS_PER_YEAR: i64 = 12; pub fn register(registry: &mut FunctionRegistry) { @@ -262,7 +265,7 @@ fn register_convert_timezone(registry: &mut FunctionRegistry) { } else { // Fall back to the slower Jiff conversion for timestamps // outside the LUT coverage (e.g. <1900 or >2299). - let src_zoned = src_timestamp.to_timestamp(source_tz); + let src_zoned = timestamp_from_micros(src_timestamp, source_tz); let target_zoned = src_zoned.with_time_zone(t_tz.clone()); ( target_zoned.timestamp().as_microsecond(), @@ -1042,8 +1045,7 @@ fn days_from_components(year: i32, month: u8, day: u8) -> Option { } fn timestamp_days_via_jiff(value: i64, tz: &TimeZone) -> i32 { - value - .to_timestamp(tz) + timestamp_from_micros(value, tz) .date() .since((Unit::Day, Date::new(1970, 1, 1).unwrap())) .unwrap() @@ -1108,13 +1110,13 @@ fn register_timestamp_tz_to_date(registry: &mut FunctionRegistry) { let offset = Offset::from_seconds(val.seconds_offset()).map_err(|err| err.to_string())?; - Ok(val - .timestamp() - .to_timestamp(&TimeZone::fixed(offset)) - .date() - .since((Unit::Day, Date::new(1970, 1, 1).unwrap())) - .unwrap() - .get_days()) + Ok( + timestamp_from_micros(val.timestamp(), &TimeZone::fixed(offset)) + .date() + .since((Unit::Day, Date::new(1970, 1, 1).unwrap())) + .unwrap() + .get_days(), + ) } } } @@ -1158,7 +1160,7 @@ fn register_to_string(registry: &mut FunctionRegistry) { |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::>( |micros, format, output, ctx| { - let ts = micros.to_timestamp(&ctx.func_ctx.tz); + let ts = timestamp_from_micros(micros, &ctx.func_ctx.tz); let format = prepare_format_string(format, &ctx.func_ctx.date_format_style); let mut buf = String::new(); let mut formatter = fmt::Formatter::new(&mut buf, FormattingOptions::new()); @@ -1334,7 +1336,6 @@ fn register_year_arith_function( vectorize_with_builder_2_arg::( move |date, delta, builder, ctx| match EvalYearsImpl::eval_date( date, - &ctx.func_ctx.tz, delta * delta_sign, false, ) { @@ -1378,7 +1379,6 @@ fn register_month_based_arith_function( vectorize_with_builder_2_arg::( move |date, delta, builder, ctx| match EvalMonthsImpl::eval_date( date, - &ctx.func_ctx.tz, delta * month_multiplier, keep_end_of_month, ) { @@ -1481,9 +1481,8 @@ fn register_diff_functions(registry: &mut FunctionRegistry) { "diff_years", |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( - |date_end, date_start, builder, ctx| { - let diff_years = - EvalYearsImpl::eval_date_diff(date_start, date_end, &ctx.func_ctx.tz); + |date_end, date_start, builder, _| { + let diff_years = EvalYearsImpl::eval_date_diff(date_start, date_end); builder.push(diff_years as i64); }, ), @@ -1505,9 +1504,8 @@ fn register_diff_functions(registry: &mut FunctionRegistry) { "diff_quarters", |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( - |date_end, date_start, builder, ctx| { - let diff_years = - EvalQuartersImpl::eval_date_diff(date_start, date_end, &ctx.func_ctx.tz); + |date_end, date_start, builder, _| { + let diff_years = EvalQuartersImpl::eval_date_diff(date_start, date_end); builder.push(diff_years as i64); }, ), @@ -1529,9 +1527,8 @@ fn register_diff_functions(registry: &mut FunctionRegistry) { "diff_months", |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( - |date_end, date_start, builder, ctx| { - let diff_months = - EvalMonthsImpl::eval_date_diff(date_start, date_end, &ctx.func_ctx.tz); + |date_end, date_start, builder, _| { + let diff_months = EvalMonthsImpl::eval_date_diff(date_start, date_end); builder.push(diff_months as i64); }, ), @@ -1541,9 +1538,8 @@ fn register_diff_functions(registry: &mut FunctionRegistry) { "diff_months", |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( - |date_end, date_start, builder, ctx| { - let diff_months = - EvalMonthsImpl::eval_timestamp_diff(date_start, date_end, &ctx.func_ctx.tz); + |date_end, date_start, builder, _| { + let diff_months = EvalMonthsImpl::eval_timestamp_diff(date_start, date_end); builder.push(diff_months); }, ), @@ -1655,9 +1651,8 @@ fn register_diff_functions(registry: &mut FunctionRegistry) { "diff_yearweeks", |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( - |date_end, date_start, builder, ctx| { - let diff = - EvalYearWeeksImpl::eval_date_diff(date_start, date_end, &ctx.func_ctx.tz); + |date_end, date_start, builder, _| { + let diff = EvalYearWeeksImpl::eval_date_diff(date_start, date_end); builder.push(diff as i64); }, ), @@ -1679,8 +1674,8 @@ fn register_diff_functions(registry: &mut FunctionRegistry) { "diff_isoyears", |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( - |date_end, date_start, builder, ctx| { - let diff = EvalISOYearsImpl::eval_date_diff(date_start, date_end, &ctx.func_ctx.tz); + |date_end, date_start, builder, _| { + let diff = EvalISOYearsImpl::eval_date_diff(date_start, date_end); builder.push(diff as i64); }, ), @@ -1702,9 +1697,8 @@ fn register_diff_functions(registry: &mut FunctionRegistry) { "diff_millenniums", |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( - |date_end, date_start, builder, ctx| { - let diff_years = - EvalYearsImpl::eval_date_diff(date_start, date_end, &ctx.func_ctx.tz); + |date_end, date_start, builder, _| { + let diff_years = EvalYearsImpl::eval_date_diff(date_start, date_end); builder.push((diff_years / 1000) as i64); }, ), @@ -1814,9 +1808,8 @@ fn register_between_functions(registry: &mut FunctionRegistry) { "between_years", |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( - |date_end, date_start, builder, ctx| { - let between_years = - EvalYearsImpl::eval_date_between(date_start, date_end, &ctx.func_ctx.tz); + |date_end, date_start, builder, _| { + let between_years = EvalYearsImpl::eval_date_between(date_start, date_end); builder.push(between_years as i64); }, ), @@ -1838,9 +1831,8 @@ fn register_between_functions(registry: &mut FunctionRegistry) { "between_quarters", |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( - |date_end, date_start, builder, ctx| { - let between_quarters = - EvalMonthsImpl::eval_date_between(date_start, date_end, &ctx.func_ctx.tz) / 3; + |date_end, date_start, builder, _| { + let between_quarters = EvalMonthsImpl::eval_date_between(date_start, date_end) / 3; builder.push(between_quarters as i64); }, ), @@ -1863,9 +1855,8 @@ fn register_between_functions(registry: &mut FunctionRegistry) { "between_months", |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( - |date_end, date_start, builder, ctx| { - let between_months = - EvalMonthsImpl::eval_date_between(date_start, date_end, &ctx.func_ctx.tz); + |date_end, date_start, builder, _| { + let between_months = EvalMonthsImpl::eval_date_between(date_start, date_end); builder.push(between_months as i64); }, ), @@ -1887,9 +1878,8 @@ fn register_between_functions(registry: &mut FunctionRegistry) { "between_weeks", |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( - |date_end, date_start, builder, ctx| { - let between_weeks = - EvalWeeksImpl::eval_date_between(date_start, date_end, &ctx.func_ctx.tz); + |date_end, date_start, builder, _| { + let between_weeks = EvalWeeksImpl::eval_date_between(date_start, date_end); builder.push(between_weeks as i64); }, ), @@ -2028,9 +2018,8 @@ fn register_between_functions(registry: &mut FunctionRegistry) { "between_isoyears", |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( - |date_end, date_start, builder, ctx| { - let between_isoyears = - EvalISOYearsImpl::eval_date_between(date_start, date_end, &ctx.func_ctx.tz); + |date_end, date_start, builder, _| { + let between_isoyears = EvalISOYearsImpl::eval_date_between(date_start, date_end); builder.push(between_isoyears as i64); }, ), @@ -2055,9 +2044,8 @@ fn register_between_functions(registry: &mut FunctionRegistry) { "between_millenniums", |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::( - |date_end, date_start, builder, ctx| { - let between_millenniums = - EvalYearsImpl::eval_date_between(date_start, date_end, &ctx.func_ctx.tz); + |date_end, date_start, builder, _| { + let between_millenniums = EvalYearsImpl::eval_date_between(date_start, date_end); builder.push((between_millenniums / 1000) as i64); }, ), @@ -2219,172 +2207,90 @@ fn register_to_number_functions(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_1_arg::( "to_yyyymm", |_, _| FunctionDomain::Full, - vectorize_with_builder_1_arg::(|val, output, ctx| { - match ToNumberImpl::eval_date::(val, &ctx.func_ctx.tz) { - Ok(t) => output.push(t), - Err(e) => { - ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); - output.push(0); - } - } + vectorize_1_arg::(|val, _| { + ToNumberImpl::eval_date::(val) }), ); registry.register_passthrough_nullable_1_arg::( "to_yyyymmdd", |_, _| FunctionDomain::Full, - vectorize_with_builder_1_arg::(|val, output, ctx| { - match ToNumberImpl::eval_date::(val, &ctx.func_ctx.tz) { - Ok(t) => output.push(t), - Err(e) => { - ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); - output.push(0); - } - } + vectorize_1_arg::(|val, _| { + ToNumberImpl::eval_date::(val) }), ); registry.register_passthrough_nullable_1_arg::( "to_year", |_, _| FunctionDomain::Full, - vectorize_with_builder_1_arg::(|val, output, ctx| { - match ToNumberImpl::eval_date::(val, &ctx.func_ctx.tz) { - Ok(t) => output.push(t), - Err(e) => { - ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); - output.push(0); - } - } - }), + vectorize_1_arg::(|val, _| ToNumberImpl::eval_date::(val)), ); registry.register_passthrough_nullable_1_arg::( "to_iso_year", |_, _| FunctionDomain::Full, - vectorize_with_builder_1_arg::(|val, output, ctx| { - match ToNumberImpl::eval_date::(val, &ctx.func_ctx.tz) { - Ok(t) => output.push(t), - Err(e) => { - ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); - output.push(0); - } - } + vectorize_1_arg::(|val, _| { + ToNumberImpl::eval_date::(val) }), ); registry.register_passthrough_nullable_1_arg::( "to_quarter", |_, _| FunctionDomain::Full, - vectorize_with_builder_1_arg::(|val, output, ctx| { - match ToNumberImpl::eval_date::(val, &ctx.func_ctx.tz) { - Ok(t) => output.push(t), - Err(e) => { - ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); - output.push(0); - } - } + vectorize_1_arg::(|val, _| { + ToNumberImpl::eval_date::(val) }), ); registry.register_passthrough_nullable_1_arg::( "to_month", |_, _| FunctionDomain::Full, - vectorize_with_builder_1_arg::(|val, output, ctx| { - match ToNumberImpl::eval_date::(val, &ctx.func_ctx.tz) { - Ok(t) => output.push(t), - Err(e) => { - ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); - output.push(0); - } - } - }), + vectorize_1_arg::(|val, _| ToNumberImpl::eval_date::(val)), ); registry.register_passthrough_nullable_1_arg::( "to_day_of_year", |_, _| FunctionDomain::Full, - vectorize_with_builder_1_arg::(|val, output, ctx| { - match ToNumberImpl::eval_date::(val, &ctx.func_ctx.tz) { - Ok(t) => output.push(t), - Err(e) => { - ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); - output.push(0); - } - } + vectorize_1_arg::(|val, _| { + ToNumberImpl::eval_date::(val) }), ); registry.register_passthrough_nullable_1_arg::( "to_day_of_month", |_, _| FunctionDomain::Full, - vectorize_with_builder_1_arg::(|val, output, ctx| { - match ToNumberImpl::eval_date::(val, &ctx.func_ctx.tz) { - Ok(t) => output.push(t), - Err(e) => { - ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); - output.push(0); - } - } + vectorize_1_arg::(|val, _| { + ToNumberImpl::eval_date::(val) }), ); registry.register_passthrough_nullable_1_arg::( "to_day_of_week", |_, _| FunctionDomain::Full, - vectorize_with_builder_1_arg::(|val, output, ctx| { - match ToNumberImpl::eval_date::(val, &ctx.func_ctx.tz) { - Ok(t) => output.push(t), - Err(e) => { - ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); - output.push(0); - } - } + vectorize_1_arg::(|val, _| { + ToNumberImpl::eval_date::(val) }), ); registry.register_passthrough_nullable_1_arg::( "dayofweek", |_, _| FunctionDomain::Full, - vectorize_with_builder_1_arg::(|val, output, ctx| { - match ToNumberImpl::eval_date::(val, &ctx.func_ctx.tz) { - Ok(t) => output.push(t), - Err(e) => { - ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); - output.push(0); - } - } + vectorize_1_arg::(|val, _| { + ToNumberImpl::eval_date::(val) }), ); registry.register_passthrough_nullable_1_arg::( "yearweek", |_, _| FunctionDomain::Full, - vectorize_with_builder_1_arg::(|val, output, ctx| { - match ToNumberImpl::eval_date::(val, &ctx.func_ctx.tz) { - Ok(t) => output.push(t), - Err(e) => { - ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); - output.push(0); - } - } + vectorize_1_arg::(|val, _| { + ToNumberImpl::eval_date::(val) }), ); registry.register_passthrough_nullable_1_arg::( "millennium", |_, _| FunctionDomain::Full, - vectorize_with_builder_1_arg::(|val, output, ctx| { - match ToNumberImpl::eval_date::(val, &ctx.func_ctx.tz) { - Ok(t) => output.push(t), - Err(e) => { - ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); - output.push(0); - } - } + vectorize_1_arg::(|val, _| { + ToNumberImpl::eval_date::(val) }), ); registry.register_passthrough_nullable_1_arg::( "to_week_of_year", |_, _| FunctionDomain::Full, - vectorize_with_builder_1_arg::(|val, output, ctx| { - match ToNumberImpl::eval_date::(val, &ctx.func_ctx.tz) { - Ok(t) => output.push(t), - Err(e) => { - ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); - output.push(0); - } - } + vectorize_1_arg::(|val, _| { + ToNumberImpl::eval_date::(val) }), ); // timestamp @@ -2484,7 +2390,7 @@ fn register_to_number_functions(registry: &mut FunctionRegistry) { "to_hour", |_, _| FunctionDomain::Full, |val, ctx| { - let datetime = val.to_timestamp(&ctx.func_ctx.tz); + let datetime = timestamp_from_micros(val, &ctx.func_ctx.tz); datetime.hour() as u8 }, ); @@ -2492,7 +2398,7 @@ fn register_to_number_functions(registry: &mut FunctionRegistry) { "to_minute", |_, _| FunctionDomain::Full, |val, ctx| { - let datetime = val.to_timestamp(&ctx.func_ctx.tz); + let datetime = timestamp_from_micros(val, &ctx.func_ctx.tz); datetime.minute() as u8 }, ); @@ -2500,7 +2406,7 @@ fn register_to_number_functions(registry: &mut FunctionRegistry) { "to_second", |_, _| FunctionDomain::Full, |val, ctx| { - let datetime = val.to_timestamp(&ctx.func_ctx.tz); + let datetime = timestamp_from_micros(val, &ctx.func_ctx.tz); datetime.second() as u8 }, ); @@ -2747,23 +2653,11 @@ fn register_rounder_functions(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_2_arg::( "to_start_of_week", |_, _, _| FunctionDomain::Full, - vectorize_with_builder_2_arg::(|val, mode, output, ctx| { + vectorize_2_arg::(|val, mode, ctx| { if mode == 0 { - match DateRounder::eval_date::(val, &ctx.func_ctx.tz) { - Ok(t) => output.push(t), - Err(e) => { - ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); - output.push(0); - } - } + DateRounder::eval_date::(val, &ctx.func_ctx.tz) } else { - match DateRounder::eval_date::(val, &ctx.func_ctx.tz) { - Ok(t) => output.push(t), - Err(e) => { - ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); - output.push(0); - } - } + DateRounder::eval_date::(val, &ctx.func_ctx.tz) } }), ); @@ -2785,14 +2679,8 @@ where T: ToNumber { registry.register_passthrough_nullable_1_arg::( name, |_, _| FunctionDomain::Full, - vectorize_with_builder_1_arg::(|val, output, ctx| { - match DateRounder::eval_date::(val, &ctx.func_ctx.tz) { - Ok(t) => output.push(t), - Err(e) => { - ctx.set_error(output.len(), format!("cannot parse to type `Date`. {}", e)); - output.push(0); - } - } + vectorize_1_arg::(|val, ctx| { + DateRounder::eval_date::(val, &ctx.func_ctx.tz) }), ); registry.register_1_arg::( diff --git a/src/query/functions/src/scalars/timestamp/src/interval.rs b/src/query/functions/src/scalars/timestamp/src/interval.rs index b39015dfb17f1..24e0458819c46 100644 --- a/src/query/functions/src/scalars/timestamp/src/interval.rs +++ b/src/query/functions/src/scalars/timestamp/src/interval.rs @@ -19,12 +19,6 @@ use databend_common_expression::EvalContext; use databend_common_expression::FunctionDomain; use databend_common_expression::FunctionRegistry; use databend_common_expression::Value; -use databend_common_expression::date_helper::DateConverter; -use databend_common_expression::date_helper::EvalDaysImpl; -use databend_common_expression::date_helper::EvalMonthsImpl; -use databend_common_expression::date_helper::calc_date_to_timestamp; -use databend_common_expression::date_helper::timestamp_tz_components_via_lut; -use databend_common_expression::date_helper::today_date; use databend_common_expression::error_to_null; use databend_common_expression::types::AccessType; use databend_common_expression::types::DateType; @@ -35,6 +29,7 @@ use databend_common_expression::types::StringType; use databend_common_expression::types::TimestampType; use databend_common_expression::types::interval::interval_to_string; use databend_common_expression::types::interval::string_to_interval; +use databend_common_expression::types::timestamp::timestamp_from_micros; use databend_common_expression::types::timestamp_tz::TimestampTzType; use databend_common_expression::vectorize_2_arg; use databend_common_expression::vectorize_with_builder_1_arg; @@ -46,6 +41,12 @@ use jiff::Zoned; use jiff::tz::Offset; use jiff::tz::TimeZone; +use crate::date_helper::EvalDaysImpl; +use crate::date_helper::EvalMonthsImpl; +use crate::date_helper::calc_date_to_timestamp; +use crate::date_helper::timestamp_tz_components_via_lut; +use crate::date_helper::today_date; + pub fn register(registry: &mut FunctionRegistry) { // cast(xx AS interval) // to_interval(xx) @@ -333,8 +334,8 @@ fn register_interval_add_sub_mul(registry: &mut FunctionRegistry) { ) { output.push(calc_age_from_components(&c1, &c2, is_negative)); } else { - let t1 = t1.to_timestamp(tz); - let t2 = t2.to_timestamp(tz); + let t1 = timestamp_from_micros(t1, tz); + let t2 = timestamp_from_micros(t2, tz); output.push(calc_age(t1, t2, is_negative)); } }, @@ -435,8 +436,8 @@ fn register_interval_add_sub_mul(registry: &mut FunctionRegistry) { ) { output.push(calc_age_from_components(&c1, &c2, is_negative)); } else { - let mut t1 = t1.to_timestamp(tz); - let mut t2 = t2_val.to_timestamp(tz); + let mut t1 = timestamp_from_micros(t1, tz); + let mut t2 = timestamp_from_micros(t2_val, tz); if t1 < t2 { std::mem::swap(&mut t1, &mut t2); @@ -597,7 +598,7 @@ fn eval_date_plus( output: &mut Vec, ctx: &mut EvalContext, ) { - match apply_interval_to_date(date, interval, &ctx.func_ctx.tz, true) { + match apply_interval_to_date(date, interval, true) { Ok(result) => output.push(result), Err(err) => { ctx.set_error(output.len(), err); @@ -612,7 +613,7 @@ fn eval_date_minus( output: &mut Vec, ctx: &mut EvalContext, ) { - match apply_interval_to_date(date, interval, &ctx.func_ctx.tz, false) { + match apply_interval_to_date(date, interval, false) { Ok(result) => output.push(result), Err(err) => { ctx.set_error(output.len(), err); @@ -624,7 +625,6 @@ fn eval_date_minus( fn apply_interval_to_date( mut date: i32, interval: months_days_micros, - tz: &TimeZone, is_addition: bool, ) -> std::result::Result { if interval.microseconds() != 0 { @@ -643,7 +643,7 @@ fn apply_interval_to_date( date = EvalDaysImpl::eval_date(date, days); } if months != 0 { - date = EvalMonthsImpl::eval_date(date, tz, months, false)?; + date = EvalMonthsImpl::eval_date(date, months, false)?; } Ok(date) diff --git a/src/query/functions/src/scalars/timestamp/src/lib.rs b/src/query/functions/src/scalars/timestamp/src/lib.rs index b316e75941ac9..df6a31a08f4ee 100644 --- a/src/query/functions/src/scalars/timestamp/src/lib.rs +++ b/src/query/functions/src/scalars/timestamp/src/lib.rs @@ -27,7 +27,9 @@ #![feature(str_internals)] #![feature(fmt_internals)] #![feature(formatting_options)] +#![feature(int_roundings)] extern crate core; +pub mod date_helper; pub mod datetime; pub mod interval; diff --git a/src/query/functions/tests/it/scalars/datetime.rs b/src/query/functions/tests/it/scalars/datetime.rs index 7d72cd750fe85..ff81b25062d2c 100644 --- a/src/query/functions/tests/it/scalars/datetime.rs +++ b/src/query/functions/tests/it/scalars/datetime.rs @@ -632,6 +632,7 @@ fn test_date_domain_overflow(file: &mut impl Write) { fn test_to_number(file: &mut impl Write) { // date run_ast(file, "to_yyyymm(to_date(18875))", &[]); + run_ast(file, "to_yyyymm(to_date(2932896))", &[]); run_ast(file, "to_yyyymmdd(to_date(18875))", &[]); run_ast(file, "to_yyyymmddhhmmss(to_date(18875))", &[]); run_ast(file, "to_year(to_date(18875))", &[]); diff --git a/src/query/functions/tests/it/scalars/testdata/datetime.txt b/src/query/functions/tests/it/scalars/testdata/datetime.txt index 24302d473e228..e4e2f1cd48d64 100644 --- a/src/query/functions/tests/it/scalars/testdata/datetime.txt +++ b/src/query/functions/tests/it/scalars/testdata/datetime.txt @@ -2708,6 +2708,15 @@ output domain : {202109..=202109} output : 202109 +ast : to_yyyymm(to_date(2932896)) +raw expr : to_yyyymm(to_date(2932896)) +checked expr : to_yyyymm(CAST(CAST(2932896_u32 AS Int64) AS Date)) +optimized expr : 999912_u32 +output type : UInt32 +output domain : {999912..=999912} +output : 999912 + + ast : to_yyyymmdd(to_date(18875)) raw expr : to_yyyymmdd(to_date(18875)) checked expr : to_yyyymmdd(CAST(CAST(18875_u16 AS Int64) AS Date)) diff --git a/src/query/pipeline/transforms/src/processors/transforms/window/transform_window.rs b/src/query/pipeline/transforms/src/processors/transforms/window/transform_window.rs index 35fa89e9d7345..91d37ec3bf502 100644 --- a/src/query/pipeline/transforms/src/processors/transforms/window/transform_window.rs +++ b/src/query/pipeline/transforms/src/processors/transforms/window/transform_window.rs @@ -32,8 +32,6 @@ use databend_common_expression::Scalar; use databend_common_expression::ScalarRef; use databend_common_expression::SortColumnDescription; use databend_common_expression::arithmetics_type::ResultTypeOfUnary; -use databend_common_expression::date_helper::EvalMonthsImpl; -use databend_common_expression::date_helper::calc_date_to_timestamp; use databend_common_expression::types::AccessType; use databend_common_expression::types::DataType; use databend_common_expression::types::DateType; @@ -44,6 +42,8 @@ use databend_common_expression::types::NumberScalar; use databend_common_expression::types::NumberType; use databend_common_expression::types::TimestampType; use databend_common_expression::with_number_mapped_type; +use databend_common_functions::scalars::dt_func::date_helper::EvalMonthsImpl; +use databend_common_functions::scalars::dt_func::date_helper::calc_date_to_timestamp; use databend_common_pipeline::core::Event; use databend_common_pipeline::core::InputPort; use databend_common_pipeline::core::OutputPort; diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/convert.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/convert.rs index 96ae0e2833dc7..255fbdb417521 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/convert.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/convert.rs @@ -26,19 +26,16 @@ use databend_common_exception::Result; use databend_common_expression::Column; use databend_common_expression::ColumnRef; use databend_common_expression::Constant; -use databend_common_expression::Domain; use databend_common_expression::Expr; +use databend_common_expression::MinMax; use databend_common_expression::RawExpr; use databend_common_expression::Scalar; use databend_common_expression::type_check; use databend_common_expression::types::DataType; -use databend_common_expression::types::NumberDomain; -use databend_common_expression::types::NumberScalar; use databend_common_functions::BUILTIN_FUNCTIONS; use super::builder::should_enable_runtime_filter; use super::packet::JoinRuntimeFilterPacket; -use super::packet::SerializableDomain; use crate::pipelines::processors::transforms::hash_join::desc::RuntimeFilterDesc; use crate::pipelines::processors::transforms::hash_join::util::min_max_filter; @@ -89,11 +86,7 @@ pub async fn build_runtime_filter_infos( }; let min_max = if let Some(ref min_max) = packet.min_max { - Some(build_min_max_filter( - min_max.clone(), - probe_key, - &desc.build_key, - )?) + Some(build_min_max_filter(min_max.clone(), probe_key)?) } else { None }; @@ -169,87 +162,9 @@ fn build_inlist_filter(inlist: Column, probe_key: &Expr) -> Result<(Expr Ok((expr, inlist_value_count)) } -fn build_min_max_filter( - min_max: SerializableDomain, - probe_key: &Expr, - build_key: &Expr, -) -> Result> { - let min_max = Domain::from_min_max( - min_max.min, - min_max.max, - &build_key.data_type().remove_nullable(), - ); - let min_max_filter = match min_max { - Domain::Number(domain) => match domain { - NumberDomain::UInt8(simple_domain) => { - let min = Scalar::Number(NumberScalar::from(simple_domain.min)); - let max = Scalar::Number(NumberScalar::from(simple_domain.max)); - min_max_filter(min, max, probe_key)? - } - NumberDomain::UInt16(simple_domain) => { - let min = Scalar::Number(NumberScalar::from(simple_domain.min)); - let max = Scalar::Number(NumberScalar::from(simple_domain.max)); - min_max_filter(min, max, probe_key)? - } - NumberDomain::UInt32(simple_domain) => { - let min = Scalar::Number(NumberScalar::from(simple_domain.min)); - let max = Scalar::Number(NumberScalar::from(simple_domain.max)); - min_max_filter(min, max, probe_key)? - } - NumberDomain::UInt64(simple_domain) => { - let min = Scalar::Number(NumberScalar::from(simple_domain.min)); - let max = Scalar::Number(NumberScalar::from(simple_domain.max)); - min_max_filter(min, max, probe_key)? - } - NumberDomain::Int8(simple_domain) => { - let min = Scalar::Number(NumberScalar::from(simple_domain.min)); - let max = Scalar::Number(NumberScalar::from(simple_domain.max)); - min_max_filter(min, max, probe_key)? - } - NumberDomain::Int16(simple_domain) => { - let min = Scalar::Number(NumberScalar::from(simple_domain.min)); - let max = Scalar::Number(NumberScalar::from(simple_domain.max)); - min_max_filter(min, max, probe_key)? - } - NumberDomain::Int32(simple_domain) => { - let min = Scalar::Number(NumberScalar::from(simple_domain.min)); - let max = Scalar::Number(NumberScalar::from(simple_domain.max)); - min_max_filter(min, max, probe_key)? - } - NumberDomain::Int64(simple_domain) => { - let min = Scalar::Number(NumberScalar::from(simple_domain.min)); - let max = Scalar::Number(NumberScalar::from(simple_domain.max)); - min_max_filter(min, max, probe_key)? - } - NumberDomain::Float32(simple_domain) => { - let min = Scalar::Number(NumberScalar::from(simple_domain.min)); - let max = Scalar::Number(NumberScalar::from(simple_domain.max)); - min_max_filter(min, max, probe_key)? - } - NumberDomain::Float64(simple_domain) => { - let min = Scalar::Number(NumberScalar::from(simple_domain.min)); - let max = Scalar::Number(NumberScalar::from(simple_domain.max)); - min_max_filter(min, max, probe_key)? - } - }, - Domain::String(domain) => { - let min = Scalar::String(domain.min); - let max = Scalar::String(domain.max.unwrap()); - min_max_filter(min, max, probe_key)? - } - Domain::Date(date_domain) => { - let min = Scalar::Date(date_domain.min); - let max = Scalar::Date(date_domain.max); - min_max_filter(min, max, probe_key)? - } - _ => { - return Err(ErrorCode::UnsupportedDataType(format!( - "Unsupported domain {:?} for runtime filter", - min_max, - ))); - } - }; - Ok(min_max_filter) +fn build_min_max_filter(min_max: MinMax, probe_key: &Expr) -> Result> { + let (min, max) = min_max.scalars(); + min_max_filter(min, max, probe_key) } async fn build_bloom_filter( @@ -312,9 +227,12 @@ mod tests { use databend_common_expression::Domain; use databend_common_expression::Expr; use databend_common_expression::FunctionContext; + use databend_common_expression::MinMax; use databend_common_expression::Scalar; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; + use databend_common_expression::types::NumberDomain; + use databend_common_expression::types::SimpleDomain; use databend_common_functions::BUILTIN_FUNCTIONS; use super::build_inlist_filter; @@ -322,7 +240,10 @@ mod tests { use crate::pipelines::processors::transforms::hash_join::desc::RuntimeFilterDesc; use crate::pipelines::processors::transforms::hash_join::runtime_filter::packet::JoinRuntimeFilterPacket; use crate::pipelines::processors::transforms::hash_join::runtime_filter::packet::RuntimeFilterPacket; - use crate::pipelines::processors::transforms::hash_join::runtime_filter::packet::SerializableDomain; + + fn int32_domain(min: i32, max: i32) -> Domain { + Domain::Number(NumberDomain::Int32(SimpleDomain { min, max })) + } #[tokio::test(flavor = "current_thread")] async fn test_build_runtime_filter_infos_selectivity_threshold_only_disables_bloom() { @@ -358,10 +279,10 @@ mod tests { packets.insert(0, RuntimeFilterPacket { id: 0, inlist: Some(inlist), - min_max: Some(SerializableDomain { - min: Scalar::Number(1i32.into()), - max: Scalar::Number(10i32.into()), - }), + min_max: Some(MinMax::Number( + NumberDomain::Int32(SimpleDomain { min: 1, max: 10 }), + false, + )), bloom: Some(vec![11, 22]), }); @@ -407,11 +328,7 @@ mod tests { // Test with ConstantFolder - case where column_a in [2,10] (can be folded to constant) let mut input_domains = HashMap::new(); - let domain_value_2_10 = Domain::from_min_max( - Scalar::Number(2i32.into()), - Scalar::Number(10i32.into()), - &data_type, - ); + let domain_value_2_10 = int32_domain(2, 10); input_domains.insert("column_a".to_string(), domain_value_2_10); let (folded_expr, _) = ConstantFolder::fold_with_domain( @@ -426,11 +343,7 @@ mod tests { // Test with ConstantFolder - case where column_a in [2,9] (should evaluate to false) let mut input_domains_false = HashMap::new(); - let domain_value_2_9 = Domain::from_min_max( - Scalar::Number(2i32.into()), - Scalar::Number(9i32.into()), - &data_type, - ); + let domain_value_2_9 = int32_domain(2, 9); input_domains_false.insert("column_a".to_string(), domain_value_2_9); let (folded_expr_false, _) = ConstantFolder::fold_with_domain( @@ -536,11 +449,7 @@ mod tests { // Test with ConstantFolder - case where column_b in [500, 600] // (should intersect with our range [0, 1023]) let mut input_domains = HashMap::new(); - let domain_value_500_600 = Domain::from_min_max( - Scalar::Number(500i32.into()), - Scalar::Number(600i32.into()), - &data_type, - ); + let domain_value_500_600 = int32_domain(500, 600); input_domains.insert("column_b".to_string(), domain_value_500_600); let (folded_expr, _) = ConstantFolder::fold_with_domain( @@ -559,11 +468,7 @@ mod tests { // Test with ConstantFolder - case where column_b in [2000, 3000] // (should NOT intersect with our range [0, 1023]) let mut input_domains_no_intersect = HashMap::new(); - let domain_value_2000_3000 = Domain::from_min_max( - Scalar::Number(2000i32.into()), - Scalar::Number(3000i32.into()), - &data_type, - ); + let domain_value_2000_3000 = int32_domain(2000, 3000); input_domains_no_intersect.insert("column_b".to_string(), domain_value_2000_3000); let (folded_expr_false, _) = ConstantFolder::fold_with_domain( diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/local_builder.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/local_builder.rs index 436c700215eb3..cbbacadfd1c91 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/local_builder.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/local_builder.rs @@ -16,8 +16,8 @@ use databend_common_exception::Result; use databend_common_expression::BlockEntry; use databend_common_expression::Column; use databend_common_expression::ColumnBuilder; +use databend_common_expression::ColumnMinMax; use databend_common_expression::DataBlock; -use databend_common_expression::Domain; use databend_common_expression::Evaluator; use databend_common_expression::FunctionContext; use databend_common_expression::HashMethodKind; @@ -29,7 +29,6 @@ use databend_common_functions::BUILTIN_FUNCTIONS; use crate::pipelines::processors::transforms::hash_join::desc::RuntimeFilterDesc; use crate::pipelines::processors::transforms::hash_join::runtime_filter::packet::JoinRuntimeFilterPacket; use crate::pipelines::processors::transforms::hash_join::runtime_filter::packet::RuntimeFilterPacket; -use crate::pipelines::processors::transforms::hash_join::runtime_filter::packet::SerializableDomain; use crate::pipelines::processors::transforms::hash_join::util::hash_by_method_for_bloom; struct SingleFilterBuilder { @@ -37,7 +36,7 @@ struct SingleFilterBuilder { inlist_data_type: DataType, hash_method: HashMethodKind, - min_max_domain: Option, + min_max: Option, min_max_threshold: usize, inlist_builder: Option, @@ -62,12 +61,9 @@ impl SingleFilterBuilder { id: desc.id, inlist_data_type: bloom_data_type, hash_method, - min_max_domain: None, - min_max_threshold: if desc.enable_min_max_runtime_filter { - min_max_threshold - } else { - 0 - }, + min_max: (desc.enable_min_max_runtime_filter && min_max_threshold > 0) + .then_some(ColumnMinMax::Empty), + min_max_threshold, inlist_builder: None, inlist_threshold: if desc.enable_inlist_runtime_filter { inlist_threshold @@ -85,22 +81,22 @@ impl SingleFilterBuilder { fn add_column(&mut self, column: &Column, total_rows: usize) -> Result<()> { let new_total = total_rows + column.len(); - self.add_min_max(column, new_total); + self.add_min_max(column, new_total)?; self.add_inlist(column, new_total); self.add_bloom(column, new_total)?; Ok(()) } - fn add_min_max(&mut self, column: &Column, new_total: usize) { + fn add_min_max(&mut self, column: &Column, new_total: usize) -> Result<()> { if new_total > self.min_max_threshold { - self.min_max_domain = None; - return; + self.min_max = None; + return Ok(()); + } + + if let Some(min_max) = self.min_max.as_mut() { + min_max.merge(&column.min_max()?)?; } - let col_domain = column.remove_nullable().domain(); - self.min_max_domain = Some(match self.min_max_domain.take() { - Some(d) => d.merge(&col_domain), - None => col_domain, - }); + Ok(()) } fn add_inlist(&mut self, column: &Column, new_total: usize) { @@ -139,10 +135,7 @@ impl SingleFilterBuilder { } fn finish(mut self, func_ctx: &FunctionContext) -> Result { - let min_max = self.min_max_domain.take().map(|domain| { - let (min, max) = domain.to_minmax(); - SerializableDomain { min, max } - }); + let min_max = self.min_max.take().and_then(ColumnMinMax::into_option); let inlist = if let Some(builder) = self.inlist_builder.take() { let column = builder.build(); @@ -278,6 +271,7 @@ mod tests { use databend_common_expression::ColumnRef; use databend_common_expression::Expr; use databend_common_expression::FromData; + use databend_common_expression::types::Int32Type; use databend_common_expression::types::StringType; use super::*; @@ -309,4 +303,36 @@ mod tests { assert_eq!(inlist.len(), 2); Ok(()) } + + #[test] + fn test_min_max_retains_null_from_an_all_null_batch() -> Result<()> { + let desc = RuntimeFilterDesc { + id: 0, + build_key: Expr::ColumnRef(ColumnRef { + span: None, + id: 0, + data_type: DataType::Nullable(Box::new(DataType::Number( + databend_common_expression::types::NumberDataType::Int32, + ))), + display_name: "build_key".to_string(), + }), + probe_targets: vec![], + build_table_rows: Some(4), + enable_bloom_runtime_filter: false, + enable_inlist_runtime_filter: false, + enable_min_max_runtime_filter: true, + }; + + let mut builder = SingleFilterBuilder::new(&desc, 0, 0, 10)?; + builder.add_column(&Int32Type::from_opt_data(vec![None, None]), 0)?; + builder.add_column(&Int32Type::from_opt_data(vec![Some(3), Some(5)]), 2)?; + + let min_max = builder + .finish(&FunctionContext::default())? + .min_max + .unwrap(); + assert!(min_max.has_null()); + assert_eq!(min_max.scalars(), (3i32.into(), 5i32.into())); + Ok(()) + } } diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/merge.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/merge.rs index 0a19c75dae755..202c3e60bdd9a 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/merge.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/merge.rs @@ -16,10 +16,10 @@ use std::collections::HashMap; use databend_common_exception::Result; use databend_common_expression::Column; +use databend_common_expression::MinMax; use super::packet::JoinRuntimeFilterPacket; use super::packet::RuntimeFilterPacket; -use super::packet::SerializableDomain; pub fn merge_join_runtime_filter_packets( packets: Vec, @@ -43,10 +43,6 @@ pub fn merge_join_runtime_filter_packets( return Ok(result); } - let should_merge_inlist = total_build_rows < inlist_threshold; - let should_merge_bloom = total_build_rows < bloom_threshold; - let should_merge_min_max = total_build_rows < min_max_threshold; - let packets = packets .into_iter() .filter_map(|packet| packet.packets) @@ -63,17 +59,17 @@ pub fn merge_join_runtime_filter_packets( for id in packets[0].keys() { result.insert(*id, RuntimeFilterPacket { id: *id, - inlist: if should_merge_inlist { + inlist: if total_build_rows <= inlist_threshold { merge_inlist(&packets, *id)? } else { None }, - min_max: if should_merge_min_max { - merge_min_max(&packets, *id) + min_max: if total_build_rows <= min_max_threshold { + merge_min_max(&packets, *id)? } else { None }, - bloom: if should_merge_bloom { + bloom: if total_build_rows <= bloom_threshold { merge_bloom(&packets, *id) } else { None @@ -116,42 +112,22 @@ fn merge_inlist( fn merge_min_max( packets: &[HashMap], rf_id: usize, -) -> Option { - if packets - .iter() - .any(|packet| packet.get(&rf_id).unwrap().min_max.is_none()) +) -> Result> { + let mut iter = packets.iter().map(|packet| packet.get(&rf_id).unwrap()); + if let Some(first) = iter.next() + && let Some(first) = &first.min_max { - return None; + let mut acc = first.clone(); + for item in iter { + let Some(min_max) = &item.min_max else { + return Ok(None); + }; + acc.merge(min_max)?; + } + Ok(Some(acc)) + } else { + Ok(None) } - let min = packets - .iter() - .map(|packet| { - packet - .get(&rf_id) - .unwrap() - .min_max - .as_ref() - .unwrap() - .min - .clone() - }) - .min() - .unwrap(); - let max = packets - .iter() - .map(|packet| { - packet - .get(&rf_id) - .unwrap() - .min_max - .as_ref() - .unwrap() - .max - .clone() - }) - .max() - .unwrap(); - Some(SerializableDomain { min, max }) } fn merge_bloom(packets: &[HashMap], rf_id: usize) -> Option> { @@ -183,7 +159,9 @@ mod tests { use databend_common_expression::Scalar; use databend_common_expression::types::DataType; use databend_common_expression::types::NumberDataType; + use databend_common_expression::types::NumberDomain; use databend_common_expression::types::NumberScalar; + use databend_common_expression::types::SimpleDomain; use super::*; @@ -196,16 +174,17 @@ mod tests { builder.build() } + fn int_min_max(min: i32, max: i32) -> MinMax { + MinMax::Number(NumberDomain::Int32(SimpleDomain { min, max }), false) + } + #[test] fn test_merge_short_circuit_all_types() -> Result<()> { let mut runtime_filters = HashMap::new(); runtime_filters.insert(1, RuntimeFilterPacket { id: 1, inlist: Some(int_column(&[1, 2, 3])), - min_max: Some(SerializableDomain { - min: Scalar::Number(NumberScalar::Int32(1)), - max: Scalar::Number(NumberScalar::Int32(3)), - }), + min_max: Some(int_min_max(1, 3)), bloom: Some(vec![11, 22, 33]), }); @@ -227,20 +206,14 @@ mod tests { runtime_filters_1.insert(7, RuntimeFilterPacket { id: 7, inlist: Some(int_column(&[1, 2])), - min_max: Some(SerializableDomain { - min: Scalar::Number(NumberScalar::Int32(1)), - max: Scalar::Number(NumberScalar::Int32(5)), - }), + min_max: Some(int_min_max(1, 5)), bloom: Some(vec![1, 2]), }); let mut runtime_filters_2 = HashMap::new(); runtime_filters_2.insert(7, RuntimeFilterPacket { id: 7, inlist: Some(int_column(&[3, 4])), - min_max: Some(SerializableDomain { - min: Scalar::Number(NumberScalar::Int32(-1)), - max: Scalar::Number(NumberScalar::Int32(8)), - }), + min_max: Some(int_min_max(-1, 8)), bloom: Some(vec![3, 4]), }); @@ -258,13 +231,7 @@ mod tests { assert_eq!(merged.build_rows, 11); assert!(packet.inlist.is_none()); assert_eq!(packet.bloom, Some(vec![1, 2, 3, 4])); - assert_eq!( - packet.min_max, - Some(SerializableDomain { - min: Scalar::Number(NumberScalar::Int32(-1)), - max: Scalar::Number(NumberScalar::Int32(8)), - }) - ); + assert_eq!(packet.min_max, Some(int_min_max(-1, 8))); Ok(()) } diff --git a/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/packet.rs b/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/packet.rs index e5ac320e2d582..960b41da75a0e 100644 --- a/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/packet.rs +++ b/src/query/service/src/pipelines/processors/transforms/hash_join/runtime_filter/packet.rs @@ -25,7 +25,7 @@ use databend_common_expression::Column; use databend_common_expression::ColumnBuilder; use databend_common_expression::DataBlock; use databend_common_expression::DataSchemaRef; -use databend_common_expression::Scalar; +use databend_common_expression::MinMax; use databend_common_expression::types::ArrayColumn; use databend_common_expression::types::NumberColumn; use databend_common_expression::types::NumberColumnBuilder; @@ -43,7 +43,7 @@ use databend_common_expression::types::array::ArrayColumnBuilder; pub struct RuntimeFilterPacket { pub id: usize, pub inlist: Option, - pub min_max: Option, + pub min_max: Option, pub bloom: Option>, } @@ -108,7 +108,7 @@ struct FlightRuntimeFilterPacket { pub id: usize, pub bloom: Option, pub inlist: Option, - pub min_max: Option, + pub min_max: Option, } #[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Default, PartialEq)] @@ -256,9 +256,3 @@ impl BlockMetaInfo for FlightJoinRuntimeFilterPacket { Some(self.schema.clone()) } } - -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, PartialEq)] -pub struct SerializableDomain { - pub min: Scalar, - pub max: Scalar, -} diff --git a/src/query/storages/common/index/tests/it/bloom_pruner.rs b/src/query/storages/common/index/tests/it/bloom_pruner.rs index 12dee8ca5de4c..d6fc8f7576cca 100644 --- a/src/query/storages/common/index/tests/it/bloom_pruner.rs +++ b/src/query/storages/common/index/tests/it/bloom_pruner.rs @@ -19,6 +19,7 @@ use std::sync::Arc; use databend_common_expression::BlockEntry; use databend_common_expression::Column; +use databend_common_expression::ColumnMinMax; use databend_common_expression::ColumnRef; use databend_common_expression::Constant; use databend_common_expression::ConstantFolder; @@ -169,7 +170,7 @@ fn test_bloom_filter_rewrites_string_literal_integer_comparison() { .filter_map(|(i, entry)| { let field = bloom_columns.get(&i)?; let column = entry.as_column().unwrap(); - let (min, max) = column.domain().to_minmax(); + let (min, max) = column.min_max().unwrap().into_option().unwrap().scalars(); Some((field.column_id, ColumnStatistics { min, max, @@ -744,7 +745,11 @@ fn eval_index_expr( .as_nullable() .map(|nullable| nullable.validity.null_count()) .unwrap_or_default() as u64; - let (min, max) = column.domain().to_minmax(); + let (min, max) = column + .min_max() + .ok() + .and_then(ColumnMinMax::into_option) + .map_or((Scalar::Null, Scalar::Null), |min_max| min_max.scalars()); Some((field.column_id, ColumnStatistics { min, max, diff --git a/src/query/storages/common/index/tests/it/eliminate_cast.rs b/src/query/storages/common/index/tests/it/eliminate_cast.rs index a6541a748daa3..b5f3a058254ef 100644 --- a/src/query/storages/common/index/tests/it/eliminate_cast.rs +++ b/src/query/storages/common/index/tests/it/eliminate_cast.rs @@ -16,11 +16,12 @@ use std::io::Write; use databend_common_expression::Domain; use databend_common_expression::Expr; -use databend_common_expression::Scalar; use databend_common_expression::type_check; use databend_common_expression::types::ArgType; use databend_common_expression::types::DataType; use databend_common_expression::types::Int32Type; +use databend_common_expression::types::NumberDomain; +use databend_common_expression::types::SimpleDomain; use databend_common_expression_test_support::parse_raw_expr; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_storages_common_index::eliminate_cast; @@ -38,14 +39,10 @@ fn test_eliminate_cast() { let mut mint = Mint::new("tests/it/testdata"); let file = &mut mint.new_goldenfile("test_eliminate_cast.txt").unwrap(); - fn n(n: i32) -> Scalar { - Scalar::Number(n.into()) - } - run_text(file, "a::string = '2'", &[( "a", Int32Type::data_type(), - Domain::from_min_max(n(-2), n(3), &Int32Type::data_type()), + Domain::Number(NumberDomain::Int32(SimpleDomain { min: -2, max: 3 })), )]); } diff --git a/src/query/storages/fuse/src/statistics/cluster_statistics.rs b/src/query/storages/fuse/src/statistics/cluster_statistics.rs index bb8bdf5b87531..19cd3dfdf1612 100644 --- a/src/query/storages/fuse/src/statistics/cluster_statistics.rs +++ b/src/query/storages/fuse/src/statistics/cluster_statistics.rs @@ -30,6 +30,16 @@ use databend_common_expression::SortColumnDescription; use databend_common_expression::TableSchema; use databend_common_expression::compare_scalars; use databend_common_expression::types::DataType; +use databend_common_expression::types::boolean::BooleanDomain; +use databend_common_expression::types::decimal::DecimalDomain; +use databend_common_expression::types::decimal::DecimalScalar; +use databend_common_expression::types::nullable::NullableDomain; +use databend_common_expression::types::number::NumberDomain; +use databend_common_expression::types::number::NumberScalar; +use databend_common_expression::types::number::SimpleDomain; +use databend_common_expression::types::string::StringDomain; +use databend_common_expression::with_decimal_type; +use databend_common_expression::with_number_type; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_functions::aggregates::eval_aggr; use databend_common_meta_app::schema::TableIndex; @@ -489,6 +499,57 @@ pub(crate) fn prepare_cluster_key_exprs( .collect() } +fn domain_to_cluster_boundaries(domain: &Domain) -> (Scalar, Scalar) { + match domain { + Domain::Number(domain) => with_number_type!(|NUM| match domain { + NumberDomain::NUM(SimpleDomain { min, max }) => ( + Scalar::Number(NumberScalar::NUM(*min)), + Scalar::Number(NumberScalar::NUM(*max)), + ), + }), + Domain::Decimal(domain) => with_decimal_type!(|DECIMAL| match domain { + DecimalDomain::DECIMAL(SimpleDomain { min, max }, size) => ( + Scalar::Decimal(DecimalScalar::DECIMAL(*min, *size)), + Scalar::Decimal(DecimalScalar::DECIMAL(*max, *size)), + ), + }), + Domain::Boolean(BooleanDomain { + has_false, + has_true, + }) => (Scalar::Boolean(!*has_false), Scalar::Boolean(*has_true)), + Domain::String(StringDomain { min, max }) => ( + Scalar::String(min.clone()), + max.clone().map(Scalar::String).unwrap_or(Scalar::Null), + ), + Domain::Timestamp(SimpleDomain { min, max }) => { + (Scalar::Timestamp(*min), Scalar::Timestamp(*max)) + } + Domain::TimestampTz(SimpleDomain { min, max }) => { + (Scalar::TimestampTz(*min), Scalar::TimestampTz(*max)) + } + Domain::Date(SimpleDomain { min, max }) => (Scalar::Date(*min), Scalar::Date(*max)), + Domain::Interval(SimpleDomain { min, max }) => { + (Scalar::Interval(*min), Scalar::Interval(*max)) + } + Domain::Nullable(NullableDomain { + has_null, + value: Some(value), + }) => { + let (min, mut max) = domain_to_cluster_boundaries(value); + if *has_null { + max = Scalar::Null; + } + (min, max) + } + Domain::Nullable(NullableDomain { value: None, .. }) => (Scalar::Null, Scalar::Null), + Domain::Tuple(fields) => { + let (mins, maxs) = fields.iter().map(domain_to_cluster_boundaries).unzip(); + (Scalar::Tuple(mins), Scalar::Tuple(maxs)) + } + _ => (Scalar::Null, Scalar::Null), + } +} + pub(crate) fn get_min_max_stats( prepared_exprs: &[PreparedClusterKeyExpr], col_stats: &StatisticsOfColumns, @@ -535,10 +596,10 @@ pub(crate) fn get_min_max_stats( &BUILTIN_FUNCTIONS, ); let domain = domain_opt.unwrap_or_else(|| Domain::full(&prepared_expr.data_type)); - let (mut min, mut max) = domain.to_minmax(); + let (mut min, mut max) = domain_to_cluster_boundaries(&domain); if min.as_ref().cmp(&max.as_ref()) == Ordering::Greater { warn!("invalid cluster key expression range, fallback to full domain"); - (min, max) = Domain::full(&prepared_expr.data_type).to_minmax(); + (min, max) = domain_to_cluster_boundaries(&Domain::full(&prepared_expr.data_type)); } mins.push(min); maxs.push(max); diff --git a/src/query/task_support/src/system_tables/task_history.rs b/src/query/task_support/src/system_tables/task_history.rs index 1bf6d92e12215..cb0264220a295 100644 --- a/src/query/task_support/src/system_tables/task_history.rs +++ b/src/query/task_support/src/system_tables/task_history.rs @@ -28,7 +28,6 @@ use databend_common_expression::DataBlock; use databend_common_expression::FromData; use databend_common_expression::FunctionContext; use databend_common_expression::Scalar; -use databend_common_expression::date_helper::DateConverter; use databend_common_expression::expr::*; use databend_common_expression::filter_helper::FilterHelpers; use databend_common_expression::infer_table_schema; @@ -39,6 +38,7 @@ use databend_common_expression::types::StringType; use databend_common_expression::types::TimestampType; use databend_common_expression::types::UInt64Type; use databend_common_expression::types::VariantType; +use databend_common_expression::types::timestamp::timestamp_from_micros; use databend_common_functions::BUILTIN_FUNCTIONS; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; @@ -162,16 +162,22 @@ impl AsyncSystemTable for TaskHistoryTable { if col_name == "scheduled_time" && let Scalar::Timestamp(s) = scalar { - scheduled_time_end = - Some(s.to_timestamp(&TimeZone::UTC).timestamp().to_string()); + scheduled_time_end = Some( + timestamp_from_micros(*s, &TimeZone::UTC) + .timestamp() + .to_string(), + ); } }); find_gt_filter(&expr, &mut |col_name, scalar| { if col_name == "scheduled_time" && let Scalar::Timestamp(s) = scalar { - scheduled_time_start = - Some(s.to_timestamp(&TimeZone::UTC).timestamp().to_string()); + scheduled_time_start = Some( + timestamp_from_micros(*s, &TimeZone::UTC) + .timestamp() + .to_string(), + ); } }); } diff --git a/src/query/task_support/src/table_functions/task_history.rs b/src/query/task_support/src/table_functions/task_history.rs index 55833a887e9e7..b9e753f644abb 100644 --- a/src/query/task_support/src/table_functions/task_history.rs +++ b/src/query/task_support/src/table_functions/task_history.rs @@ -32,8 +32,9 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; use databend_common_expression::DataBlock; use databend_common_expression::Scalar; -use databend_common_expression::date_helper::DateConverter; use databend_common_expression::infer_table_schema; +use databend_common_expression::types::date::date_from_days; +use databend_common_expression::types::timestamp::timestamp_from_micros; use databend_common_meta_app::schema::TableIdent; use databend_common_meta_app::schema::TableInfo; use databend_common_meta_app::schema::TableMeta; @@ -256,14 +257,14 @@ fn parse_date_or_timestamp(v: &Scalar) -> Option { if v.as_timestamp().is_some() { Some( v.as_timestamp() - .map(|s| s.to_timestamp(&TimeZone::UTC).to_string()) + .map(|s| timestamp_from_micros(*s, &TimeZone::UTC).to_string()) .unwrap(), ) } else if v.as_date().is_some() { Some( v.as_date() .map(|s| { - s.to_date(&TimeZone::UTC) + date_from_days(*s) .at(0, 0, 0, 0) .in_tz("UTC") .unwrap()