feat: implement IsNotNull expression in vortex expression library (#6969)

### Summary
Closes: #6040

Add a first-class IsNotNull scalar function, replacing the previous
Not(IsNull(...)) composition pattern. This simplifies the expression
tree and enables direct stat_falsification for zone map pruning.

Changes:

New is_not_null.rs with ScalarFnVTable implementation, including
stat_falsification using is_constant && null_count > 0 (with TODO for
future RowCount stat)
Updated all integration points: DataFusion, DuckDB, Python/Substrait to
use is_not_null(...) directly
Replaced the Not(IsNull(...)) fallback in erased.rs validity with
IsNotNull
Registered IsNotNull in ScalarFnSession and ExprBuiltins/ArrayBuiltins

### AI Assistance Disclosure
This PR was developed with AI assistance (Kiro). AI was used for code
review, implementing stat_falsification, writing tests, and drafting the
PR description. All output was reviewed and validated by the author.

API Changes
New public APIs:

vortex_array::expr::is_not_null(child) — creates an IsNotNull expression
Expression::is_not_null() / ArrayRef::is_not_null() via
ExprBuiltins/ArrayBuiltins traits
Python: vortex._lib.expr.is_not_null(child)

### Testing
9 unit tests covering: return dtype, child replacement,
mixed/all-valid/all-invalid evaluation, struct field access, display
formatting, null sensitivity, and stat falsification pruning expression
generation.

---------

Signed-off-by: Xiaoxuan Li <xioxuan@amazon.com>
Signed-off-by: Robert Kruszewski <github@robertk.io>
Co-authored-by: Robert Kruszewski <github@robertk.io>

Author: xiaoxuandev

java/vortex-jni/src/main/java/dev/vortex/api/Expression.java

@@ -95,6 +95,14 @@ interface Visitor<T> {
          */
         T visitIsNull(IsNull isNull);
 
+        /**
+         * Visits an is not null expression (non-null check).
+         *
+         * @param isNotNull the is not null expression to visit
+         * @return the result of visiting the is not null expression
+         */
+        T visitIsNotNull(IsNotNull isNotNull);
+
         /**
          * For expressions that do not have a specific visitor method.
          */

java/vortex-jni/src/main/java/dev/vortex/api/expressions/IsNotNull.java

@@ -0,0 +1,99 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+package dev.vortex.api.expressions;
+
+import dev.vortex.api.Expression;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * Represents an IS NOT NULL expression that checks whether values are non-null.
+ * This expression returns true for non-null values and false for null values.
+ */
+public final class IsNotNull implements Expression {
+    private final Expression child;
+
+    private IsNotNull(Expression child) {
+        this.child = child;
+    }
+
+    /**
+     * Parses an IsNotNull expression from serialized metadata and child expressions.
+     * This method is used during deserialization of Vortex expressions.
+     *
+     * @param metadata the serialized metadata, must be empty for IsNotNull expressions
+     * @param children the child expressions, must contain exactly one element
+     * @return a new IsNotNull expression parsed from the provided data
+     * @throws IllegalArgumentException if the number of children is not exactly one,
+     *                                  or if metadata is not empty
+     */
+    public static IsNotNull parse(byte[] metadata, List<Expression> children) {
+        if (children.size() != 1) {
+            throw new IllegalArgumentException(
+                    "IsNotNull expression must have exactly one child, found: " + children.size());
+        }
+        if (metadata.length > 0) {
+            throw new IllegalArgumentException(
+                    "IsNotNull expression must not have metadata, found: " + metadata.length);
+        }
+        return new IsNotNull(children.get(0));
+    }
+
+    /**
+     * Creates a new IsNotNull expression that checks non-nullity of the given child expression.
+     *
+     * @param child the expression to check for non-null values
+     * @return a new IsNotNull expression
+     */
+    public static IsNotNull of(Expression child) {
+        return new IsNotNull(child);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == null || getClass() != o.getClass()) return false;
+        IsNotNull other = (IsNotNull) o;
+        return Objects.equals(child, other.child);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(child);
+    }
+
+    @Override
+    public String id() {
+        return "vortex.is_not_null";
+    }
+
+    @Override
+    public List<Expression> children() {
+        return List.of(child);
+    }
+
+    @Override
+    public Optional<byte[]> metadata() {
+        return Optional.of(new byte[] {});
+    }
+
+    @Override
+    public String toString() {
+        return "vortex.is_not_null(" + child + ")";
+    }
+
+    /**
+     * Returns the child expression that will be checked for non-null values.
+     *
+     * @return the child expression
+     */
+    public Expression getChild() {
+        return child;
+    }
+
+    @Override
+    public <T> T accept(Visitor<T> visitor) {
+        return visitor.visitIsNotNull(this);
+    }
+}

java/vortex-jni/src/main/java/dev/vortex/api/proto/Expressions.java

@@ -34,7 +34,8 @@ public static ExprProtos.Expr serialize(Expression expression) {
     /**
      * Deserialize a protocol buffer representation back into an {@link Expression} object.
      * The method examines the expression ID and creates the appropriate concrete expression type
-     * based on the registered expression types (binary, get_item, root, literal, not).
+     * based on the registered expression types (binary, get_item, root, literal, not, is null,
+     * is not null).
      * If the expression ID is not recognized, an {@link Unknown} expression is created.
      *
      * @param expr the protocol buffer expression to deserialize
@@ -58,6 +59,8 @@ public static Expression deserialize(ExprProtos.Expr expr) {
                 return Not.parse(metadata, children);
             case "vortex.is_null":
                 return IsNull.parse(metadata, children);
+            case "vortex.is_not_null":
+                return IsNotNull.parse(metadata, children);
             default:
                 return new Unknown(expr.getId(), children, expr.getMetadata().toByteArray());
         }

java/vortex-jni/src/test/java/dev/vortex/api/expressions/proto/TestExpressionProtos.java

@@ -30,4 +30,12 @@ public void testIsNullRoundTrip() {
         Expression deserialized = Expressions.deserialize(proto);
         assertEquals(expression, deserialized);
     }
+
+    @Test
+    public void testIsNotNullRoundTrip() {
+        Expression expression = IsNotNull.of(GetItem.of(Root.INSTANCE, "a.b.c"));
+        ExprProtos.Expr proto = Expressions.serialize(expression);
+        Expression deserialized = Expressions.deserialize(proto);
+        assertEquals(expression, deserialized);
+    }
 }

vortex-array/public-api.lock

@@ -8440,6 +8440,8 @@ pub fn vortex_array::builtins::ArrayBuiltins::fill_null(&self, fill_value: impl
 
 pub fn vortex_array::builtins::ArrayBuiltins::get_item(&self, field_name: impl core::convert::Into<vortex_array::dtype::FieldName>) -> vortex_error::VortexResult<vortex_array::ArrayRef>
 
+pub fn vortex_array::builtins::ArrayBuiltins::is_not_null(&self) -> vortex_error::VortexResult<vortex_array::ArrayRef>
+
 pub fn vortex_array::builtins::ArrayBuiltins::is_null(&self) -> vortex_error::VortexResult<vortex_array::ArrayRef>
 
 pub fn vortex_array::builtins::ArrayBuiltins::list_contains(&self, value: vortex_array::ArrayRef) -> vortex_error::VortexResult<vortex_array::ArrayRef>
@@ -8462,6 +8464,8 @@ pub fn vortex_array::ArrayRef::fill_null(&self, fill_value: impl core::convert::
 
 pub fn vortex_array::ArrayRef::get_item(&self, field_name: impl core::convert::Into<vortex_array::dtype::FieldName>) -> vortex_error::VortexResult<vortex_array::ArrayRef>
 
+pub fn vortex_array::ArrayRef::is_not_null(&self) -> vortex_error::VortexResult<vortex_array::ArrayRef>
+
 pub fn vortex_array::ArrayRef::is_null(&self) -> vortex_error::VortexResult<vortex_array::ArrayRef>
 
 pub fn vortex_array::ArrayRef::list_contains(&self, value: vortex_array::ArrayRef) -> vortex_error::VortexResult<vortex_array::ArrayRef>
@@ -8482,6 +8486,8 @@ pub fn vortex_array::builtins::ExprBuiltins::fill_null(&self, fill_value: vortex
 
 pub fn vortex_array::builtins::ExprBuiltins::get_item(&self, field_name: impl core::convert::Into<vortex_array::dtype::FieldName>) -> vortex_error::VortexResult<vortex_array::expr::Expression>
 
+pub fn vortex_array::builtins::ExprBuiltins::is_not_null(&self) -> vortex_error::VortexResult<vortex_array::expr::Expression>
+
 pub fn vortex_array::builtins::ExprBuiltins::is_null(&self) -> vortex_error::VortexResult<vortex_array::expr::Expression>
 
 pub fn vortex_array::builtins::ExprBuiltins::list_contains(&self, value: vortex_array::expr::Expression) -> vortex_error::VortexResult<vortex_array::expr::Expression>
@@ -8502,6 +8508,8 @@ pub fn vortex_array::expr::Expression::fill_null(&self, fill_value: vortex_array
 
 pub fn vortex_array::expr::Expression::get_item(&self, field_name: impl core::convert::Into<vortex_array::dtype::FieldName>) -> vortex_error::VortexResult<vortex_array::expr::Expression>
 
+pub fn vortex_array::expr::Expression::is_not_null(&self) -> vortex_error::VortexResult<vortex_array::expr::Expression>
+
 pub fn vortex_array::expr::Expression::is_null(&self) -> vortex_error::VortexResult<vortex_array::expr::Expression>
 
 pub fn vortex_array::expr::Expression::list_contains(&self, value: vortex_array::expr::Expression) -> vortex_error::VortexResult<vortex_array::expr::Expression>
@@ -12426,6 +12434,8 @@ pub fn vortex_array::expr::Expression::fill_null(&self, fill_value: vortex_array
 
 pub fn vortex_array::expr::Expression::get_item(&self, field_name: impl core::convert::Into<vortex_array::dtype::FieldName>) -> vortex_error::VortexResult<vortex_array::expr::Expression>
 
+pub fn vortex_array::expr::Expression::is_not_null(&self) -> vortex_error::VortexResult<vortex_array::expr::Expression>
+
 pub fn vortex_array::expr::Expression::is_null(&self) -> vortex_error::VortexResult<vortex_array::expr::Expression>
 
 pub fn vortex_array::expr::Expression::list_contains(&self, value: vortex_array::expr::Expression) -> vortex_error::VortexResult<vortex_array::expr::Expression>
@@ -12520,6 +12530,8 @@ pub fn vortex_array::expr::immediate_scope_access<'a>(expr: &'a vortex_array::ex
 
 pub fn vortex_array::expr::immediate_scope_accesses<'a>(expr: &'a vortex_array::expr::Expression, scope: &'a vortex_array::dtype::StructFields) -> vortex_array::expr::FieldAccesses<'a>
 
+pub fn vortex_array::expr::is_not_null(child: vortex_array::expr::Expression) -> vortex_array::expr::Expression
+
 pub fn vortex_array::expr::is_null(child: vortex_array::expr::Expression) -> vortex_array::expr::Expression
 
 pub fn vortex_array::expr::is_root(expr: &vortex_array::expr::Expression) -> bool
@@ -16302,6 +16314,52 @@ pub fn vortex_array::scalar_fn::fns::get_item::GetItem::stat_falsification(&self
 
 pub fn vortex_array::scalar_fn::fns::get_item::GetItem::validity(&self, options: &Self::Options, expression: &vortex_array::expr::Expression) -> vortex_error::VortexResult<core::option::Option<vortex_array::expr::Expression>>
 
+pub mod vortex_array::scalar_fn::fns::is_not_null
+
+pub struct vortex_array::scalar_fn::fns::is_not_null::IsNotNull
+
+impl core::clone::Clone for vortex_array::scalar_fn::fns::is_not_null::IsNotNull
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::clone(&self) -> vortex_array::scalar_fn::fns::is_not_null::IsNotNull
+
+impl vortex_array::scalar_fn::ScalarFnVTable for vortex_array::scalar_fn::fns::is_not_null::IsNotNull
+
+pub type vortex_array::scalar_fn::fns::is_not_null::IsNotNull::Options = vortex_array::scalar_fn::EmptyOptions
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::arity(&self, _options: &Self::Options) -> vortex_array::scalar_fn::Arity
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::child_name(&self, _instance: &Self::Options, child_idx: usize) -> vortex_array::scalar_fn::ChildName
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::coerce_args(&self, options: &Self::Options, args: &[vortex_array::dtype::DType]) -> vortex_error::VortexResult<alloc::vec::Vec<vortex_array::dtype::DType>>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::deserialize(&self, _metadata: &[u8], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Options>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::execute(&self, _data: &Self::Options, args: &dyn vortex_array::scalar_fn::ExecutionArgs, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ArrayRef>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::fmt_sql(&self, _options: &Self::Options, expr: &vortex_array::expr::Expression, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::id(&self) -> vortex_array::scalar_fn::ScalarFnId
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::is_fallible(&self, _instance: &Self::Options) -> bool
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::is_null_sensitive(&self, _instance: &Self::Options) -> bool
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::reduce(&self, options: &Self::Options, node: &dyn vortex_array::scalar_fn::ReduceNode, ctx: &dyn vortex_array::scalar_fn::ReduceCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::scalar_fn::ReduceNodeRef>>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::return_dtype(&self, _options: &Self::Options, _arg_dtypes: &[vortex_array::dtype::DType]) -> vortex_error::VortexResult<vortex_array::dtype::DType>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::serialize(&self, _instance: &Self::Options) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::simplify(&self, options: &Self::Options, expr: &vortex_array::expr::Expression, ctx: &dyn vortex_array::scalar_fn::SimplifyCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::expr::Expression>>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::simplify_untyped(&self, options: &Self::Options, expr: &vortex_array::expr::Expression) -> vortex_error::VortexResult<core::option::Option<vortex_array::expr::Expression>>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::stat_expression(&self, options: &Self::Options, expr: &vortex_array::expr::Expression, stat: vortex_array::expr::stats::Stat, catalog: &dyn vortex_array::expr::pruning::StatsCatalog) -> core::option::Option<vortex_array::expr::Expression>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::stat_falsification(&self, _options: &Self::Options, expr: &vortex_array::expr::Expression, catalog: &dyn vortex_array::expr::pruning::StatsCatalog) -> core::option::Option<vortex_array::expr::Expression>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::validity(&self, options: &Self::Options, expression: &vortex_array::expr::Expression) -> vortex_error::VortexResult<core::option::Option<vortex_array::expr::Expression>>
+
 pub mod vortex_array::scalar_fn::fns::is_null
 
 pub struct vortex_array::scalar_fn::fns::is_null::IsNull
@@ -18084,6 +18142,44 @@ pub fn vortex_array::scalar_fn::fns::get_item::GetItem::stat_falsification(&self
 
 pub fn vortex_array::scalar_fn::fns::get_item::GetItem::validity(&self, options: &Self::Options, expression: &vortex_array::expr::Expression) -> vortex_error::VortexResult<core::option::Option<vortex_array::expr::Expression>>
 
+impl vortex_array::scalar_fn::ScalarFnVTable for vortex_array::scalar_fn::fns::is_not_null::IsNotNull
+
+pub type vortex_array::scalar_fn::fns::is_not_null::IsNotNull::Options = vortex_array::scalar_fn::EmptyOptions
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::arity(&self, _options: &Self::Options) -> vortex_array::scalar_fn::Arity
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::child_name(&self, _instance: &Self::Options, child_idx: usize) -> vortex_array::scalar_fn::ChildName
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::coerce_args(&self, options: &Self::Options, args: &[vortex_array::dtype::DType]) -> vortex_error::VortexResult<alloc::vec::Vec<vortex_array::dtype::DType>>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::deserialize(&self, _metadata: &[u8], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Options>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::execute(&self, _data: &Self::Options, args: &dyn vortex_array::scalar_fn::ExecutionArgs, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ArrayRef>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::fmt_sql(&self, _options: &Self::Options, expr: &vortex_array::expr::Expression, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::id(&self) -> vortex_array::scalar_fn::ScalarFnId
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::is_fallible(&self, _instance: &Self::Options) -> bool
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::is_null_sensitive(&self, _instance: &Self::Options) -> bool
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::reduce(&self, options: &Self::Options, node: &dyn vortex_array::scalar_fn::ReduceNode, ctx: &dyn vortex_array::scalar_fn::ReduceCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::scalar_fn::ReduceNodeRef>>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::return_dtype(&self, _options: &Self::Options, _arg_dtypes: &[vortex_array::dtype::DType]) -> vortex_error::VortexResult<vortex_array::dtype::DType>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::serialize(&self, _instance: &Self::Options) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::simplify(&self, options: &Self::Options, expr: &vortex_array::expr::Expression, ctx: &dyn vortex_array::scalar_fn::SimplifyCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::expr::Expression>>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::simplify_untyped(&self, options: &Self::Options, expr: &vortex_array::expr::Expression) -> vortex_error::VortexResult<core::option::Option<vortex_array::expr::Expression>>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::stat_expression(&self, options: &Self::Options, expr: &vortex_array::expr::Expression, stat: vortex_array::expr::stats::Stat, catalog: &dyn vortex_array::expr::pruning::StatsCatalog) -> core::option::Option<vortex_array::expr::Expression>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::stat_falsification(&self, _options: &Self::Options, expr: &vortex_array::expr::Expression, catalog: &dyn vortex_array::expr::pruning::StatsCatalog) -> core::option::Option<vortex_array::expr::Expression>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::validity(&self, options: &Self::Options, expression: &vortex_array::expr::Expression) -> vortex_error::VortexResult<core::option::Option<vortex_array::expr::Expression>>
+
 impl vortex_array::scalar_fn::ScalarFnVTable for vortex_array::scalar_fn::fns::is_null::IsNull
 
 pub type vortex_array::scalar_fn::fns::is_null::IsNull::Options = vortex_array::scalar_fn::EmptyOptions
@@ -22410,6 +22506,8 @@ pub fn vortex_array::ArrayRef::fill_null(&self, fill_value: impl core::convert::
 
 pub fn vortex_array::ArrayRef::get_item(&self, field_name: impl core::convert::Into<vortex_array::dtype::FieldName>) -> vortex_error::VortexResult<vortex_array::ArrayRef>
 
+pub fn vortex_array::ArrayRef::is_not_null(&self) -> vortex_error::VortexResult<vortex_array::ArrayRef>
+
 pub fn vortex_array::ArrayRef::is_null(&self) -> vortex_error::VortexResult<vortex_array::ArrayRef>
 
 pub fn vortex_array::ArrayRef::list_contains(&self, value: vortex_array::ArrayRef) -> vortex_error::VortexResult<vortex_array::ArrayRef>