diff --git a/src/query/sql/src/planner/optimizer/ir/stats/selectivity.rs b/src/query/sql/src/planner/optimizer/ir/stats/selectivity.rs index 9f234769b4125..20f7351c19897 100644 --- a/src/query/sql/src/planner/optimizer/ir/stats/selectivity.rs +++ b/src/query/sql/src/planner/optimizer/ir/stats/selectivity.rs @@ -52,6 +52,7 @@ use crate::plans::ScalarExpr; pub const DEFAULT_SELECTIVITY: f64 = 1f64 / 5f64; pub const UNKNOWN_COL_STATS_FILTER_SEL_LOWER_BOUND: f64 = 0.5_f64; pub const MAX_SELECTIVITY: f64 = 1f64; +const BOOLEAN_VALUE_SELECTIVITY: f64 = 0.5; const HISTOGRAM_ROW_COUNT_TOLERANCE: f64 = 1e-9; /// Some constants for like predicate selectivity estimation. @@ -633,14 +634,23 @@ impl SelectivityVisitor<'_> { (Expr::ColumnRef(column_ref), Expr::Constant(constant)) | (Expr::Constant(constant), Expr::ColumnRef(column_ref)) => { let column_index = column_ref.id.index; + let op = if left.is_constant() { op.reverse() } else { op }; if !self.column_stats.contains_key(&column_index) { + if matches!(column_ref.data_type.remove_nullable(), DataType::Boolean) + && let Scalar::Boolean(value) = constant.scalar + { + return if column_ref.data_type.is_nullable() { + Ok(Selectivity::Unknown) + } else { + Ok(boolean_comparison_selectivity(op, value)) + }; + } // The column is derived column, give a small selectivity currently. // Need to improve it later. // Another case: column is from system table, such as numbers. We shouldn't use numbers() table to test cardinality estimation. return Ok(Selectivity::LowerBound); } let column_stat = &self.column_stats[&column_index]; - let op = if left.is_constant() { op.reverse() } else { op }; let can_apply_constant_constraint = { use DataType::*; @@ -972,8 +982,13 @@ impl SelectivityVisitor<'_> { .unwrap_or(Selectivity::Unknown); Ok(()) } - Expr::ColumnRef(_) => { - self.selectivity = Selectivity::LowerBound; + Expr::ColumnRef(column_ref) => { + self.selectivity = + if matches!(column_ref.data_type.remove_nullable(), DataType::Boolean) { + Selectivity::N(BOOLEAN_VALUE_SELECTIVITY) + } else { + Selectivity::LowerBound + }; Ok(()) } Expr::Cast(cast) => self.visit_expr(&cast.expr), @@ -1029,16 +1044,14 @@ impl SelectivityVisitor<'_> { self.selectivity = if has_zero { Selectivity::Zero - } else if !has_unknown && !has_lower_bound && !has_n { - Selectivity::All - } else if (!has_unknown && !has_lower_bound) || acc < DEFAULT_SELECTIVITY { + } else if has_n { Selectivity::N(acc) } else if has_unknown { Selectivity::Unknown } else if has_lower_bound { Selectivity::LowerBound } else { - Selectivity::Unknown + Selectivity::All }; } @@ -1124,3 +1137,16 @@ impl SelectivityVisitor<'_> { Ok(()) } } + +fn boolean_comparison_selectivity(op: ComparisonOp, constant: bool) -> Selectivity { + let selectivity = match (op, constant) { + (ComparisonOp::Equal | ComparisonOp::NotEqual, _) => BOOLEAN_VALUE_SELECTIVITY, + (ComparisonOp::GT, false) + | (ComparisonOp::LT, true) + | (ComparisonOp::GTE, true) + | (ComparisonOp::LTE, false) => BOOLEAN_VALUE_SELECTIVITY, + (ComparisonOp::GT, true) | (ComparisonOp::LT, false) => 0.0, + (ComparisonOp::GTE, false) | (ComparisonOp::LTE, true) => MAX_SELECTIVITY, + }; + Selectivity::N(selectivity) +} diff --git a/src/query/sql/tests/it/optimizer/selectivity.rs b/src/query/sql/tests/it/optimizer/selectivity.rs index 634bbe9f9b5e2..d6eeae434eda1 100644 --- a/src/query/sql/tests/it/optimizer/selectivity.rs +++ b/src/query/sql/tests/it/optimizer/selectivity.rs @@ -1196,6 +1196,39 @@ fn test_selectivity_logical_outcomes() -> Result<()> { StatCardinality::estimate(100.0), )?; + write_case_title( + &mut file, + "missing_stats_logical_predicates", + "Boolean predicates should use an equal true/false distribution, while numeric estimates should take priority over AND fallbacks.", + )?; + let partial_stats = ColumnStatSet::from_iter([(Symbol::new(1), ColumnStat { + min: Datum::UInt(0), + max: Datum::UInt(3), + ndv: NdvEstimate::exact(4.0), + null_count: StatCount::exact(0), + histogram: None, + })]); + let partial_columns = [ + ("flag", BooleanType::data_type()), + ("number", UInt64Type::data_type()), + ("missing", UInt64Type::data_type()), + ("nullable_missing", UInt64Type::data_type().wrap_nullable()), + ("nullable_flag", BooleanType::data_type().wrap_nullable()), + ]; + for expr in [ + "and_filters(flag, number = 1)", + "and_filters(flag = true, number = 1)", + "or_filters(flag, number = 1)", + "or_filters(flag = true, number = 1)", + "flag > true", + "flag >= false", + "nullable_flag >= false", + "and_filters(missing = 1, number = 1)", + "and_filters(is_not_null(nullable_missing), number = 1)", + ] { + run_case(&mut file, expr, &partial_columns, partial_stats.clone())?; + } + write_case_title( &mut file, "histogram_logical_predicates", diff --git a/src/query/sql/tests/it/optimizer/selectivity_logical.txt b/src/query/sql/tests/it/optimizer/selectivity_logical.txt index 64ff3283ca619..d61977812d672 100644 --- a/src/query/sql/tests/it/optimizer/selectivity_logical.txt +++ b/src/query/sql/tests/it/optimizer/selectivity_logical.txt @@ -110,6 +110,80 @@ out stats : 0 ColumnStat { min: UInt(0), max: UInt(9), ndv: 0.0, null_count: 0, histogram: None } 1 ColumnStat { min: UInt(0), max: UInt(9), ndv: 0.0, null_count: 0, histogram: None } +=== missing_stats_logical_predicates === +description: Boolean predicates should use an equal true/false distribution, while numeric estimates should take priority over AND fallbacks. +expr : and_filters(flag, number = 1) +cardinality : 100 +estimated : 25 +in stats : +1 ColumnStat { min: UInt(0), max: UInt(3), ndv: 4.0, null_count: 0, histogram: None } +out stats : +1 ColumnStat { min: UInt(1), max: UInt(1), ndv: 1.0, null_count: 0, histogram: None } + +expr : and_filters(flag = true, number = 1) +cardinality : 100 +estimated : 25 +in stats : +1 ColumnStat { min: UInt(0), max: UInt(3), ndv: 4.0, null_count: 0, histogram: None } +out stats : +1 ColumnStat { min: UInt(1), max: UInt(1), ndv: 1.0, null_count: 0, histogram: None } + +expr : or_filters(flag, number = 1) +cardinality : 100 +estimated : 62.5 +in stats : +1 ColumnStat { min: UInt(0), max: UInt(3), ndv: 4.0, null_count: 0, histogram: None } +out stats : +1 ColumnStat { min: UInt(0), max: UInt(3), ndv: ~3.99999999991029[..4.0], null_count: 0, histogram: None } + +expr : or_filters(flag = true, number = 1) +cardinality : 100 +estimated : 62.5 +in stats : +1 ColumnStat { min: UInt(0), max: UInt(3), ndv: 4.0, null_count: 0, histogram: None } +out stats : +1 ColumnStat { min: UInt(0), max: UInt(3), ndv: ~3.99999999991029[..4.0], null_count: 0, histogram: None } + +expr : flag > true +cardinality : 100 +estimated : 0 +in stats : +1 ColumnStat { min: UInt(0), max: UInt(3), ndv: 4.0, null_count: 0, histogram: None } +out stats : +1 ColumnStat { min: UInt(0), max: UInt(3), ndv: 4.0, null_count: 0, histogram: None } + +expr : flag >= false +cardinality : 100 +estimated : 100 +in stats : +1 ColumnStat { min: UInt(0), max: UInt(3), ndv: 4.0, null_count: 0, histogram: None } +out stats : +1 ColumnStat { min: UInt(0), max: UInt(3), ndv: 4.0, null_count: 0, histogram: None } + +expr : nullable_flag >= false +cardinality : 100 +estimated : 20 +in stats : +1 ColumnStat { min: UInt(0), max: UInt(3), ndv: 4.0, null_count: 0, histogram: None } +out stats : +1 ColumnStat { min: UInt(0), max: UInt(3), ndv: ~3.984888427254817[..4.0], null_count: 0, histogram: None } + +expr : and_filters(missing = 1, number = 1) +cardinality : 100 +estimated : 25 +in stats : +1 ColumnStat { min: UInt(0), max: UInt(3), ndv: 4.0, null_count: 0, histogram: None } +out stats : +1 ColumnStat { min: UInt(1), max: UInt(1), ndv: 1.0, null_count: 0, histogram: None } + +expr : and_filters(is_not_null(nullable_missing), number = 1) +cardinality : 100 +estimated : 25 +in stats : +1 ColumnStat { min: UInt(0), max: UInt(3), ndv: 4.0, null_count: 0, histogram: None } +out stats : +1 ColumnStat { min: UInt(1), max: UInt(1), ndv: 1.0, null_count: 0, histogram: None } + === histogram_logical_predicates === description: AND constraints should be visible to later predicates, while OR and NOT should only affect final selectivity. expr : a > 3, a > 4 @@ -194,13 +268,13 @@ out stats : expr : and_filters(a > 4, a + b > 10) cardinality : 100 -estimated : 20 +estimated : 50 in stats : 0 ColumnStat { min: UInt(0), max: UInt(9), ndv: 10.0, null_count: 0, histogram: Some(UInt(TypedHistogram { accuracy: true, buckets: [TypedHistogramBucket { lower_bound: 0, upper_bound: 9, num_values: 100.0, num_distinct: 10.0 }], avg_spacing: None })) } 1 ColumnStat { min: UInt(0), max: UInt(4), ndv: 5.0, null_count: 0, histogram: Some(UInt(TypedHistogram { accuracy: true, buckets: [TypedHistogramBucket { lower_bound: 0, upper_bound: 4, num_values: 50.0, num_distinct: 5.0 }], avg_spacing: None })) } out stats : -0 ColumnStat { min: UInt(5), max: UInt(9), ndv: ~4.969766912[..5.0], null_count: 0, histogram: Some(UInt(TypedHistogram { accuracy: false, row_scale: 0.4, buckets: [TypedHistogramBucket { lower_bound: 5, upper_bound: 9, num_values: 50.0, num_distinct: 5.0 }], avg_spacing: None })) } -1 ColumnStat { min: UInt(0), max: UInt(4), ndv: ~4.463129088[..5.0], null_count: 0, histogram: Some(UInt(TypedHistogram { accuracy: false, row_scale: 0.2, buckets: [TypedHistogramBucket { lower_bound: 0, upper_bound: 4, num_values: 50.0, num_distinct: 5.0 }], avg_spacing: None })) } +0 ColumnStat { min: UInt(5), max: UInt(9), ndv: 5.0, null_count: 0, histogram: Some(UInt(TypedHistogram { accuracy: true, buckets: [TypedHistogramBucket { lower_bound: 5, upper_bound: 9, num_values: 50.0, num_distinct: 5.0 }], avg_spacing: None })) } +1 ColumnStat { min: UInt(0), max: UInt(4), ndv: ~4.9951171875[..5.0], null_count: 0, histogram: Some(UInt(TypedHistogram { accuracy: false, row_scale: 0.5, buckets: [TypedHistogramBucket { lower_bound: 0, upper_bound: 4, num_values: 50.0, num_distinct: 5.0 }], avg_spacing: None })) } === constant_logical_predicates === description: Constant predicates should preserve Zero and All through logical composition. diff --git a/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_project_set.test b/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_project_set.test index 276dbd62e9aa1..6ba12dbdc9a50 100644 --- a/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_project_set.test +++ b/tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_project_set.test @@ -14,14 +14,14 @@ explain select name, json_path_query(details, '$.features.*') as all_features, j EvalScalar ├── output columns: [products.name (#0), all_features (#3), first_feature (#4)] ├── expressions: [get(1)(json_path_query(products.details (#1), '$.features.*') (#2)), json_path_query_first(products.details (#1), '$.features.*')] -├── estimated rows: 0.36 +├── estimated rows: 0.60 └── Filter ├── output columns: [products.name (#0), products.details (#1), json_path_query(products.details (#1), '$.features.*') (#2)] ├── filters: [is_true(get(1)(json_path_query(products.details (#1), '$.features.*') (#2)) = '"512GB"')] - ├── estimated rows: 0.36 + ├── estimated rows: 0.60 └── ProjectSet ├── output columns: [products.name (#0), products.details (#1), json_path_query(products.details (#1), '$.features.*') (#2)] - ├── estimated rows: 1.80 + ├── estimated rows: 3.00 ├── set returning functions: json_path_query(products.details (#1), '$.features.*') └── TableScan ├── table: default.default.products @@ -33,7 +33,7 @@ EvalScalar ├── partitions scanned: 1 ├── pruning stats: [segments: , decompress cost: , range pruning: 1 to 1 cost: >, blocks: , bloom index read cost: , bloom pruning: 1 to 1 cost: >] ├── push downs: [filters: [products.name (#0) = 'Laptop' and json_path_query_first(products.details (#1), '$.features.*') = '"16GB"'], limit: NONE] - └── estimated rows: 0.60 + └── estimated rows: 1.00 query T?? select name, json_path_query(details, '$.features.*') as all_features, json_path_query_first(details, '$.features.*') as first_feature from products where name = 'Laptop' and first_feature = '16GB' and all_features = '512GB'; @@ -46,10 +46,10 @@ explain select name, json_path_query(details, '$.features.*') as all_features, j EvalScalar ├── output columns: [products.name (#0), all_features (#3), first_feature (#4)] ├── expressions: [get(1)(json_path_query(products.details (#1), '$.features.*') (#2)), json_path_query_first(products.details (#1), '$.features.*')] -├── estimated rows: 1.80 +├── estimated rows: 3.00 └── ProjectSet ├── output columns: [products.name (#0), products.details (#1), json_path_query(products.details (#1), '$.features.*') (#2)] - ├── estimated rows: 1.80 + ├── estimated rows: 3.00 ├── set returning functions: json_path_query(products.details (#1), '$.features.*') └── TableScan ├── table: default.default.products @@ -61,7 +61,7 @@ EvalScalar ├── partitions scanned: 1 ├── pruning stats: [segments: , decompress cost: , range pruning: 1 to 1 cost: >, blocks: , bloom index read cost: , bloom pruning: 1 to 1 cost: >] ├── push downs: [filters: [products.name (#0) = 'Laptop' and json_path_query_first(products.details (#1), '$.features.*') = '"16GB"'], limit: NONE] - └── estimated rows: 0.60 + └── estimated rows: 1.00 query T?? select name, json_path_query(details, '$.features.*') as all_features, json_path_query_first(details, '$.features.*') as first_feature from products where name = 'Laptop' and first_feature = '16GB'; diff --git a/tests/sqllogictests/suites/mode/standalone/explain/selectivity/boolean.test b/tests/sqllogictests/suites/mode/standalone/explain/selectivity/boolean.test new file mode 100644 index 0000000000000..46e2ff7883b7c --- /dev/null +++ b/tests/sqllogictests/suites/mode/standalone/explain/selectivity/boolean.test @@ -0,0 +1,33 @@ +statement ok +settings (ddl_column_type_nullable=0) CREATE OR REPLACE TABLE t1 AS +SELECT + number % 5 AS c39, + number % 2 = 0 AS c13, + if(number % 40 = 0, NULL, number % 40) AS c40, + number AS c45 +FROM numbers(1000) + +statement ok +ANALYZE TABLE t1 + +query T +EXPLAIN SELECT c39, c13, c40, c45 +FROM t1 +WHERE c39 != 0 + AND c13 = FALSE + AND (c40 = 1 OR c40 IS NULL OR c40 = 2); +---- +TableScan +├── table: default.default.t1 +├── scan id: 0 +├── output columns: [c39 (#0), c13 (#1), c40 (#2), c45 (#3)] +├── read rows: 1000 +├── read size: 1.91 KiB +├── partitions total: 1 +├── partitions scanned: 1 +├── pruning stats: [segments: , decompress cost: , range pruning: 1 to 1 cost: >, blocks: , bloom index read cost: , bloom pruning: 1 to 1 cost: >] +├── push downs: [filters: [t1.c39 (#0) <> 0 and t1.c13 (#1) = false and (t1.c40 (#2) = 1 or NOT is_not_null(t1.c40 (#2)) or t1.c40 (#2) = 2)], limit: NONE] +└── estimated rows: 73.14 + +statement ok +DROP TABLE t1