feat: implement IsNotNull expression in vortex expression library

Add a first-class IsNotNull scalar function instead of composing
Not(IsNull(...)). This simplifies the expression tree, enables direct
stat_falsification for zone map pruning, and updates all integration
points (DataFusion, DuckDB, Python/Substrait).

The stat_falsification uses is_constant && null_count > 0 as an
approximation since there is no RowCount stat yet.

Closes: #6040

Author: xiaoxuandev

vortex-array/src/builtins.rs

@@ -28,6 +28,7 @@ use crate::scalar_fn::fns::binary::Binary;
 use crate::scalar_fn::fns::cast::Cast;
 use crate::scalar_fn::fns::fill_null::FillNull;
 use crate::scalar_fn::fns::get_item::GetItem;
+use crate::scalar_fn::fns::is_not_null::IsNotNull;
 use crate::scalar_fn::fns::is_null::IsNull;
 use crate::scalar_fn::fns::list_contains::ListContains;
 use crate::scalar_fn::fns::mask::Mask;
@@ -49,6 +50,9 @@ pub trait ExprBuiltins: Sized {
     /// Is null check.
     fn is_null(&self) -> VortexResult<Expression>;
 
+    /// Is not null check.
+    fn is_not_null(&self) -> VortexResult<Expression>;
+
     /// Mask the expression using the given boolean mask.
     /// The resulting expression's validity is the intersection of the original expression's
     /// validity.
@@ -84,6 +88,10 @@ impl ExprBuiltins for Expression {
         IsNull.try_new_expr(EmptyOptions, [self.clone()])
     }
 
+    fn is_not_null(&self) -> VortexResult<Expression> {
+        IsNotNull.try_new_expr(EmptyOptions, [self.clone()])
+    }
+
     fn mask(&self, mask: Expression) -> VortexResult<Expression> {
         Mask.try_new_expr(EmptyOptions, [self.clone(), mask])
     }
@@ -118,6 +126,9 @@ pub trait ArrayBuiltins: Sized {
     /// Is null check.
     fn is_null(&self) -> VortexResult<ArrayRef>;
 
+    /// Is not null check.
+    fn is_not_null(&self) -> VortexResult<ArrayRef>;
+
     /// Mask the array using the given boolean mask.
     /// The resulting array's validity is the intersection of the original array's validity
     /// and the mask's validity.
@@ -182,6 +193,12 @@ impl ArrayBuiltins for ArrayRef {
             .optimize()
     }
 
+    fn is_not_null(&self) -> VortexResult<ArrayRef> {
+        IsNotNull
+            .try_new_array(self.len(), EmptyOptions, [self.clone()])?
+            .optimize()
+    }
+
     fn mask(self, mask: ArrayRef) -> VortexResult<ArrayRef> {
         Mask.try_new_array(self.len(), EmptyOptions, [self, mask])?
             .optimize()

vortex-array/src/expr/exprs.rs

@@ -29,6 +29,7 @@ use crate::scalar_fn::fns::dynamic::DynamicComparisonExpr;
 use crate::scalar_fn::fns::dynamic::Rhs;
 use crate::scalar_fn::fns::fill_null::FillNull;
 use crate::scalar_fn::fns::get_item::GetItem;
+use crate::scalar_fn::fns::is_not_null::IsNotNull;
 use crate::scalar_fn::fns::is_null::IsNull;
 use crate::scalar_fn::fns::like::Like;
 use crate::scalar_fn::fns::like::LikeOptions;
@@ -547,6 +548,20 @@ pub fn is_null(child: Expression) -> Expression {
     IsNull.new_expr(EmptyOptions, vec![child])
 }
 
+// ---- IsNotNull ----
+
+/// Creates an expression that checks for non-null values.
+///
+/// Returns a boolean array indicating which positions contain non-null values.
+///
+/// ```rust
+/// # use vortex_array::expr::{is_not_null, root};
+/// let expr = is_not_null(root());
+/// ```
+pub fn is_not_null(child: Expression) -> Expression {
+    IsNotNull.new_expr(EmptyOptions, vec![child])
+}
+
 // ---- Like ----
 
 /// Creates a SQL LIKE expression.

vortex-array/src/scalar_fn/erased.rs

@@ -31,8 +31,7 @@ use crate::scalar_fn::ScalarFnId;
 use crate::scalar_fn::ScalarFnVTable;
 use crate::scalar_fn::ScalarFnVTableExt;
 use crate::scalar_fn::SimplifyCtx;
-use crate::scalar_fn::fns::is_null::IsNull;
-use crate::scalar_fn::fns::not::Not;
+use crate::scalar_fn::fns::is_not_null::IsNotNull;
 use crate::scalar_fn::options::ScalarFnOptions;
 use crate::scalar_fn::signature::ScalarFnSignature;
 use crate::scalar_fn::typed::DynScalarFn;
@@ -135,11 +134,7 @@ impl ScalarFnRef {
     pub fn validity(&self, expr: &Expression) -> VortexResult<Expression> {
         Ok(self.0.validity(expr)?.unwrap_or_else(|| {
             // TODO(ngates): make validity a mandatory method on VTable to avoid this fallback.
-            // TODO(ngates): add an IsNotNull expression.
-            Not.new_expr(
-                EmptyOptions,
-                [IsNull.new_expr(EmptyOptions, [expr.clone()])],
-            )
+            IsNotNull.new_expr(EmptyOptions, [expr.clone()])
         }))
     }