fix(query): improve selectivity estimation for boolean expressions

src/query/sql/src/planner/optimizer/ir/stats/selectivity.rs

@@ -52,6 +52,7 @@ use crate::plans::ScalarExpr;
 pub const DEFAULT_SELECTIVITY: f64 = 1f64 / 5f64;
 pub const UNKNOWN_COL_STATS_FILTER_SEL_LOWER_BOUND: f64 = 0.5_f64;
 pub const MAX_SELECTIVITY: f64 = 1f64;
+const BOOLEAN_VALUE_SELECTIVITY: f64 = 0.5;
 const HISTOGRAM_ROW_COUNT_TOLERANCE: f64 = 1e-9;
 
 /// Some constants for like predicate selectivity estimation.
@@ -633,14 +634,23 @@ impl SelectivityVisitor<'_> {
             (Expr::ColumnRef(column_ref), Expr::Constant(constant))
             | (Expr::Constant(constant), Expr::ColumnRef(column_ref)) => {
                 let column_index = column_ref.id.index;
+                let op = if left.is_constant() { op.reverse() } else { op };
                 if !self.column_stats.contains_key(&column_index) {
+                    if matches!(column_ref.data_type.remove_nullable(), DataType::Boolean)
+                        && let Scalar::Boolean(value) = constant.scalar
+                    {
+                        return if column_ref.data_type.is_nullable() {
+                            Ok(Selectivity::Unknown)
+                        } else {
+                            Ok(boolean_comparison_selectivity(op, value))
+                        };
+                    }
                     // The column is derived column, give a small selectivity currently.
                     // Need to improve it later.
                     // Another case: column is from system table, such as numbers. We shouldn't use numbers() table to test cardinality estimation.
                     return Ok(Selectivity::LowerBound);
                 }
                 let column_stat = &self.column_stats[&column_index];
-                let op = if left.is_constant() { op.reverse() } else { op };
 
                 let can_apply_constant_constraint = {
                     use DataType::*;
@@ -972,8 +982,13 @@ impl SelectivityVisitor<'_> {
                     .unwrap_or(Selectivity::Unknown);
                 Ok(())
             }
-            Expr::ColumnRef(_) => {
-                self.selectivity = Selectivity::LowerBound;
+            Expr::ColumnRef(column_ref) => {
+                self.selectivity =
+                    if matches!(column_ref.data_type.remove_nullable(), DataType::Boolean) {
+                        Selectivity::N(BOOLEAN_VALUE_SELECTIVITY)
+                    } else {
+                        Selectivity::LowerBound
+                    };
                 Ok(())
             }
             Expr::Cast(cast) => self.visit_expr(&cast.expr),
@@ -1029,16 +1044,14 @@ impl SelectivityVisitor<'_> {
 
                 self.selectivity = if has_zero {
                     Selectivity::Zero
-                } else if !has_unknown && !has_lower_bound && !has_n {
-                    Selectivity::All
-                } else if (!has_unknown && !has_lower_bound) || acc < DEFAULT_SELECTIVITY {
+                } else if has_n {
                     Selectivity::N(acc)
                 } else if has_unknown {
                     Selectivity::Unknown
                 } else if has_lower_bound {
                     Selectivity::LowerBound
                 } else {
-                    Selectivity::Unknown
+                    Selectivity::All
                 };
             }
 
@@ -1124,3 +1137,16 @@ impl SelectivityVisitor<'_> {
         Ok(())
     }
 }
+
+fn boolean_comparison_selectivity(op: ComparisonOp, constant: bool) -> Selectivity {
+    let selectivity = match (op, constant) {
+        (ComparisonOp::Equal | ComparisonOp::NotEqual, _) => BOOLEAN_VALUE_SELECTIVITY,
+        (ComparisonOp::GT, false)
+        | (ComparisonOp::LT, true)
+        | (ComparisonOp::GTE, true)
+        | (ComparisonOp::LTE, false) => BOOLEAN_VALUE_SELECTIVITY,
+        (ComparisonOp::GT, true) | (ComparisonOp::LT, false) => 0.0,
+        (ComparisonOp::GTE, false) | (ComparisonOp::LTE, true) => MAX_SELECTIVITY,
+    };
+    Selectivity::N(selectivity)
+}

src/query/sql/tests/it/optimizer/selectivity.rs

@@ -1196,6 +1196,39 @@ fn test_selectivity_logical_outcomes() -> Result<()> {
         StatCardinality::estimate(100.0),
     )?;
 
+    write_case_title(
+        &mut file,
+        "missing_stats_logical_predicates",
+        "Boolean predicates should use an equal true/false distribution, while numeric estimates should take priority over AND fallbacks.",
+    )?;
+    let partial_stats = ColumnStatSet::from_iter([(Symbol::new(1), ColumnStat {
+        min: Datum::UInt(0),
+        max: Datum::UInt(3),
+        ndv: NdvEstimate::exact(4.0),
+        null_count: StatCount::exact(0),
+        histogram: None,
+    })]);
+    let partial_columns = [
+        ("flag", BooleanType::data_type()),
+        ("number", UInt64Type::data_type()),
+        ("missing", UInt64Type::data_type()),
+        ("nullable_missing", UInt64Type::data_type().wrap_nullable()),
+        ("nullable_flag", BooleanType::data_type().wrap_nullable()),
+    ];
+    for expr in [
+        "and_filters(flag, number = 1)",
+        "and_filters(flag = true, number = 1)",
+        "or_filters(flag, number = 1)",
+        "or_filters(flag = true, number = 1)",
+        "flag > true",
+        "flag >= false",
+        "nullable_flag >= false",
+        "and_filters(missing = 1, number = 1)",
+        "and_filters(is_not_null(nullable_missing), number = 1)",
+    ] {
+        run_case(&mut file, expr, &partial_columns, partial_stats.clone())?;
+    }
+
     write_case_title(
         &mut file,
         "histogram_logical_predicates",

src/query/sql/tests/it/optimizer/selectivity_logical.txt

@@ -110,6 +110,80 @@ out stats     :
 0 ColumnStat { min: UInt(0), max: UInt(9), ndv: 0.0, null_count: 0, histogram: None }
 1 ColumnStat { min: UInt(0), max: UInt(9), ndv: 0.0, null_count: 0, histogram: None }
 
+=== missing_stats_logical_predicates ===
+description: Boolean predicates should use an equal true/false distribution, while numeric estimates should take priority over AND fallbacks.
+expr          : and_filters(flag, number = 1)
+cardinality   : 100
+estimated     : 25
+in stats      :
+1 ColumnStat { min: UInt(0), max: UInt(3), ndv: 4.0, null_count: 0, histogram: None }
+out stats     :
+1 ColumnStat { min: UInt(1), max: UInt(1), ndv: 1.0, null_count: 0, histogram: None }
+
+expr          : and_filters(flag = true, number = 1)
+cardinality   : 100
+estimated     : 25
+in stats      :
+1 ColumnStat { min: UInt(0), max: UInt(3), ndv: 4.0, null_count: 0, histogram: None }
+out stats     :
+1 ColumnStat { min: UInt(1), max: UInt(1), ndv: 1.0, null_count: 0, histogram: None }
+
+expr          : or_filters(flag, number = 1)
+cardinality   : 100
+estimated     : 62.5
+in stats      :
+1 ColumnStat { min: UInt(0), max: UInt(3), ndv: 4.0, null_count: 0, histogram: None }
+out stats     :
+1 ColumnStat { min: UInt(0), max: UInt(3), ndv: ~3.99999999991029[..4.0], null_count: 0, histogram: None }
+
+expr          : or_filters(flag = true, number = 1)
+cardinality   : 100
+estimated     : 62.5
+in stats      :
+1 ColumnStat { min: UInt(0), max: UInt(3), ndv: 4.0, null_count: 0, histogram: None }
+out stats     :
+1 ColumnStat { min: UInt(0), max: UInt(3), ndv: ~3.99999999991029[..4.0], null_count: 0, histogram: None }
+
+expr          : flag > true
+cardinality   : 100
+estimated     : 0
+in stats      :
+1 ColumnStat { min: UInt(0), max: UInt(3), ndv: 4.0, null_count: 0, histogram: None }
+out stats     :
+1 ColumnStat { min: UInt(0), max: UInt(3), ndv: 4.0, null_count: 0, histogram: None }
+
+expr          : flag >= false
+cardinality   : 100
+estimated     : 100
+in stats      :
+1 ColumnStat { min: UInt(0), max: UInt(3), ndv: 4.0, null_count: 0, histogram: None }
+out stats     :
+1 ColumnStat { min: UInt(0), max: UInt(3), ndv: 4.0, null_count: 0, histogram: None }
+
+expr          : nullable_flag >= false
+cardinality   : 100
+estimated     : 20
+in stats      :
+1 ColumnStat { min: UInt(0), max: UInt(3), ndv: 4.0, null_count: 0, histogram: None }
+out stats     :
+1 ColumnStat { min: UInt(0), max: UInt(3), ndv: ~3.984888427254817[..4.0], null_count: 0, histogram: None }
+
+expr          : and_filters(missing = 1, number = 1)
+cardinality   : 100
+estimated     : 25
+in stats      :
+1 ColumnStat { min: UInt(0), max: UInt(3), ndv: 4.0, null_count: 0, histogram: None }
+out stats     :
+1 ColumnStat { min: UInt(1), max: UInt(1), ndv: 1.0, null_count: 0, histogram: None }
+
+expr          : and_filters(is_not_null(nullable_missing), number = 1)
+cardinality   : 100
+estimated     : 25
+in stats      :
+1 ColumnStat { min: UInt(0), max: UInt(3), ndv: 4.0, null_count: 0, histogram: None }
+out stats     :
+1 ColumnStat { min: UInt(1), max: UInt(1), ndv: 1.0, null_count: 0, histogram: None }
+
 === histogram_logical_predicates ===
 description: AND constraints should be visible to later predicates, while OR and NOT should only affect final selectivity.
 expr          : a > 3, a > 4
@@ -194,13 +268,13 @@ out stats     :
 
 expr          : and_filters(a > 4, a + b > 10)
 cardinality   : 100
-estimated     : 20
+estimated     : 50
 in stats      :
 0 ColumnStat { min: UInt(0), max: UInt(9), ndv: 10.0, null_count: 0, histogram: Some(UInt(TypedHistogram { accuracy: true, buckets: [TypedHistogramBucket { lower_bound: 0, upper_bound: 9, num_values: 100.0, num_distinct: 10.0 }], avg_spacing: None })) }
 1 ColumnStat { min: UInt(0), max: UInt(4), ndv: 5.0, null_count: 0, histogram: Some(UInt(TypedHistogram { accuracy: true, buckets: [TypedHistogramBucket { lower_bound: 0, upper_bound: 4, num_values: 50.0, num_distinct: 5.0 }], avg_spacing: None })) }
 out stats     :
-0 ColumnStat { min: UInt(5), max: UInt(9), ndv: ~4.969766912[..5.0], null_count: 0, histogram: Some(UInt(TypedHistogram { accuracy: false, row_scale: 0.4, buckets: [TypedHistogramBucket { lower_bound: 5, upper_bound: 9, num_values: 50.0, num_distinct: 5.0 }], avg_spacing: None })) }
-1 ColumnStat { min: UInt(0), max: UInt(4), ndv: ~4.463129088[..5.0], null_count: 0, histogram: Some(UInt(TypedHistogram { accuracy: false, row_scale: 0.2, buckets: [TypedHistogramBucket { lower_bound: 0, upper_bound: 4, num_values: 50.0, num_distinct: 5.0 }], avg_spacing: None })) }
+0 ColumnStat { min: UInt(5), max: UInt(9), ndv: 5.0, null_count: 0, histogram: Some(UInt(TypedHistogram { accuracy: true, buckets: [TypedHistogramBucket { lower_bound: 5, upper_bound: 9, num_values: 50.0, num_distinct: 5.0 }], avg_spacing: None })) }
+1 ColumnStat { min: UInt(0), max: UInt(4), ndv: ~4.9951171875[..5.0], null_count: 0, histogram: Some(UInt(TypedHistogram { accuracy: false, row_scale: 0.5, buckets: [TypedHistogramBucket { lower_bound: 0, upper_bound: 4, num_values: 50.0, num_distinct: 5.0 }], avg_spacing: None })) }
 
 === constant_logical_predicates ===
 description: Constant predicates should preserve Zero and All through logical composition.

tests/sqllogictests/suites/mode/standalone/explain/push_down_filter/push_down_filter_project_set.test

@@ -14,14 +14,14 @@ explain select name, json_path_query(details, '$.features.*') as all_features, j
 EvalScalar
 ├── output columns: [products.name (#0), all_features (#3), first_feature (#4)]
 ├── expressions: [get(1)(json_path_query(products.details (#1), '$.features.*') (#2)), json_path_query_first(products.details (#1), '$.features.*')]
-├── estimated rows: 0.36
+├── estimated rows: 0.60
 └── Filter
     ├── output columns: [products.name (#0), products.details (#1), json_path_query(products.details (#1), '$.features.*') (#2)]
     ├── filters: [is_true(get(1)(json_path_query(products.details (#1), '$.features.*') (#2)) = '"512GB"')]
-    ├── estimated rows: 0.36
+    ├── estimated rows: 0.60
     └── ProjectSet
         ├── output columns: [products.name (#0), products.details (#1), json_path_query(products.details (#1), '$.features.*') (#2)]
-        ├── estimated rows: 1.80
+        ├── estimated rows: 3.00
         ├── set returning functions: json_path_query(products.details (#1), '$.features.*')
         └── TableScan
             ├── table: default.default.products
@@ -33,7 +33,7 @@ EvalScalar
             ├── partitions scanned: 1
             ├── pruning stats: [segments: <read cost: <slt:ignore>, decompress cost: <slt:ignore>, range pruning: 1 to 1 cost: <slt:ignore>>, blocks: <range pruning: 1 to 1 cost: <slt:ignore>, bloom index read cost: <slt:ignore>, bloom pruning: 1 to 1 cost: <slt:ignore>>]
             ├── push downs: [filters: [products.name (#0) = 'Laptop' and json_path_query_first(products.details (#1), '$.features.*') = '"16GB"'], limit: NONE]
-            └── estimated rows: 0.60
+            └── estimated rows: 1.00
 
 query T??
 select name, json_path_query(details, '$.features.*') as all_features, json_path_query_first(details, '$.features.*') as first_feature from products where name = 'Laptop' and first_feature = '16GB' and all_features = '512GB';
@@ -46,10 +46,10 @@ explain select name, json_path_query(details, '$.features.*') as all_features, j
 EvalScalar
 ├── output columns: [products.name (#0), all_features (#3), first_feature (#4)]
 ├── expressions: [get(1)(json_path_query(products.details (#1), '$.features.*') (#2)), json_path_query_first(products.details (#1), '$.features.*')]
-├── estimated rows: 1.80
+├── estimated rows: 3.00
 └── ProjectSet
     ├── output columns: [products.name (#0), products.details (#1), json_path_query(products.details (#1), '$.features.*') (#2)]
-    ├── estimated rows: 1.80
+    ├── estimated rows: 3.00
     ├── set returning functions: json_path_query(products.details (#1), '$.features.*')
     └── TableScan
         ├── table: default.default.products
@@ -61,7 +61,7 @@ EvalScalar
         ├── partitions scanned: 1
         ├── pruning stats: [segments: <read cost: <slt:ignore>, decompress cost: <slt:ignore>, range pruning: 1 to 1 cost: <slt:ignore>>, blocks: <range pruning: 1 to 1 cost: <slt:ignore>, bloom index read cost: <slt:ignore>, bloom pruning: 1 to 1 cost: <slt:ignore>>]
         ├── push downs: [filters: [products.name (#0) = 'Laptop' and json_path_query_first(products.details (#1), '$.features.*') = '"16GB"'], limit: NONE]
-        └── estimated rows: 0.60
+        └── estimated rows: 1.00
 
 query T??
 select name, json_path_query(details, '$.features.*') as all_features, json_path_query_first(details, '$.features.*') as first_feature from products where name = 'Laptop' and first_feature = '16GB';

tests/sqllogictests/suites/mode/standalone/explain/selectivity/boolean.test

@@ -0,0 +1,33 @@
+statement ok
+settings (ddl_column_type_nullable=0) CREATE OR REPLACE TABLE t1 AS
+SELECT
+    number % 5 AS c39,
+    number % 2 = 0 AS c13,
+    if(number % 40 = 0, NULL, number % 40) AS c40,
+    number AS c45
+FROM numbers(1000)
+
+statement ok
+ANALYZE TABLE t1
+
+query T
+EXPLAIN SELECT c39, c13, c40, c45
+FROM t1
+WHERE c39 != 0
+  AND c13 = FALSE
+  AND (c40 = 1 OR c40 IS NULL OR c40 = 2);
+----
+TableScan
+├── table: default.default.t1
+├── scan id: 0
+├── output columns: [c39 (#0), c13 (#1), c40 (#2), c45 (#3)]
+├── read rows: 1000
+├── read size: 1.91 KiB
+├── partitions total: 1
+├── partitions scanned: 1
+├── pruning stats: [segments: <read cost: <slt:ignore>, decompress cost: <slt:ignore>, range pruning: 1 to 1 cost: <slt:ignore>>, blocks: <range pruning: 1 to 1 cost: <slt:ignore>, bloom index read cost: <slt:ignore>, bloom pruning: 1 to 1 cost: <slt:ignore>>]
+├── push downs: [filters: [t1.c39 (#0) <> 0 and t1.c13 (#1) = false and (t1.c40 (#2) = 1 or NOT is_not_null(t1.c40 (#2)) or t1.c40 (#2) = 2)], limit: NONE]
+└── estimated rows: 73.14
+
+statement ok
+DROP TABLE t1