Merge branch 'develop' into ji/sub-segment-read

Author: joseph-isaacs

.github/workflows/bench.yml

@@ -38,6 +38,7 @@ jobs:
           && format('runs-on={0}/runner=bench-dedicated/extras=s3-cache/tag={1}', github.run_id, matrix.benchmark.id)
           || 'ubuntu-latest' }}
     strategy:
+      fail-fast: false
       matrix:
         benchmark:
           - id: random-access-bench

java/vortex-jni/src/main/java/dev/vortex/api/Expression.java

@@ -95,6 +95,14 @@ interface Visitor<T> {
          */
         T visitIsNull(IsNull isNull);
 
+        /**
+         * Visits an is not null expression (non-null check).
+         *
+         * @param isNotNull the is not null expression to visit
+         * @return the result of visiting the is not null expression
+         */
+        T visitIsNotNull(IsNotNull isNotNull);
+
         /**
          * For expressions that do not have a specific visitor method.
          */

java/vortex-jni/src/main/java/dev/vortex/api/expressions/IsNotNull.java

@@ -0,0 +1,99 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: Copyright the Vortex contributors
+
+package dev.vortex.api.expressions;
+
+import dev.vortex.api.Expression;
+import java.util.List;
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * Represents an IS NOT NULL expression that checks whether values are non-null.
+ * This expression returns true for non-null values and false for null values.
+ */
+public final class IsNotNull implements Expression {
+    private final Expression child;
+
+    private IsNotNull(Expression child) {
+        this.child = child;
+    }
+
+    /**
+     * Parses an IsNotNull expression from serialized metadata and child expressions.
+     * This method is used during deserialization of Vortex expressions.
+     *
+     * @param metadata the serialized metadata, must be empty for IsNotNull expressions
+     * @param children the child expressions, must contain exactly one element
+     * @return a new IsNotNull expression parsed from the provided data
+     * @throws IllegalArgumentException if the number of children is not exactly one,
+     *                                  or if metadata is not empty
+     */
+    public static IsNotNull parse(byte[] metadata, List<Expression> children) {
+        if (children.size() != 1) {
+            throw new IllegalArgumentException(
+                    "IsNotNull expression must have exactly one child, found: " + children.size());
+        }
+        if (metadata.length > 0) {
+            throw new IllegalArgumentException(
+                    "IsNotNull expression must not have metadata, found: " + metadata.length);
+        }
+        return new IsNotNull(children.get(0));
+    }
+
+    /**
+     * Creates a new IsNotNull expression that checks non-nullity of the given child expression.
+     *
+     * @param child the expression to check for non-null values
+     * @return a new IsNotNull expression
+     */
+    public static IsNotNull of(Expression child) {
+        return new IsNotNull(child);
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (o == null || getClass() != o.getClass()) return false;
+        IsNotNull other = (IsNotNull) o;
+        return Objects.equals(child, other.child);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(child);
+    }
+
+    @Override
+    public String id() {
+        return "vortex.is_not_null";
+    }
+
+    @Override
+    public List<Expression> children() {
+        return List.of(child);
+    }
+
+    @Override
+    public Optional<byte[]> metadata() {
+        return Optional.of(new byte[] {});
+    }
+
+    @Override
+    public String toString() {
+        return "vortex.is_not_null(" + child + ")";
+    }
+
+    /**
+     * Returns the child expression that will be checked for non-null values.
+     *
+     * @return the child expression
+     */
+    public Expression getChild() {
+        return child;
+    }
+
+    @Override
+    public <T> T accept(Visitor<T> visitor) {
+        return visitor.visitIsNotNull(this);
+    }
+}

java/vortex-jni/src/main/java/dev/vortex/api/proto/Expressions.java

@@ -34,7 +34,8 @@ public static ExprProtos.Expr serialize(Expression expression) {
     /**
      * Deserialize a protocol buffer representation back into an {@link Expression} object.
      * The method examines the expression ID and creates the appropriate concrete expression type
-     * based on the registered expression types (binary, get_item, root, literal, not).
+     * based on the registered expression types (binary, get_item, root, literal, not, is null,
+     * is not null).
      * If the expression ID is not recognized, an {@link Unknown} expression is created.
      *
      * @param expr the protocol buffer expression to deserialize
@@ -58,6 +59,8 @@ public static Expression deserialize(ExprProtos.Expr expr) {
                 return Not.parse(metadata, children);
             case "vortex.is_null":
                 return IsNull.parse(metadata, children);
+            case "vortex.is_not_null":
+                return IsNotNull.parse(metadata, children);
             default:
                 return new Unknown(expr.getId(), children, expr.getMetadata().toByteArray());
         }

java/vortex-jni/src/test/java/dev/vortex/api/expressions/proto/TestExpressionProtos.java

@@ -30,4 +30,12 @@ public void testIsNullRoundTrip() {
         Expression deserialized = Expressions.deserialize(proto);
         assertEquals(expression, deserialized);
     }
+
+    @Test
+    public void testIsNotNullRoundTrip() {
+        Expression expression = IsNotNull.of(GetItem.of(Root.INSTANCE, "a.b.c"));
+        ExprProtos.Expr proto = Expressions.serialize(expression);
+        Expression deserialized = Expressions.deserialize(proto);
+        assertEquals(expression, deserialized);
+    }
 }

vortex-array/public-api.lock

@@ -8668,6 +8668,8 @@ pub fn vortex_array::builtins::ArrayBuiltins::fill_null(&self, fill_value: impl
 
 pub fn vortex_array::builtins::ArrayBuiltins::get_item(&self, field_name: impl core::convert::Into<vortex_array::dtype::FieldName>) -> vortex_error::VortexResult<vortex_array::ArrayRef>
 
+pub fn vortex_array::builtins::ArrayBuiltins::is_not_null(&self) -> vortex_error::VortexResult<vortex_array::ArrayRef>
+
 pub fn vortex_array::builtins::ArrayBuiltins::is_null(&self) -> vortex_error::VortexResult<vortex_array::ArrayRef>
 
 pub fn vortex_array::builtins::ArrayBuiltins::list_contains(&self, value: vortex_array::ArrayRef) -> vortex_error::VortexResult<vortex_array::ArrayRef>
@@ -8690,6 +8692,8 @@ pub fn vortex_array::ArrayRef::fill_null(&self, fill_value: impl core::convert::
 
 pub fn vortex_array::ArrayRef::get_item(&self, field_name: impl core::convert::Into<vortex_array::dtype::FieldName>) -> vortex_error::VortexResult<vortex_array::ArrayRef>
 
+pub fn vortex_array::ArrayRef::is_not_null(&self) -> vortex_error::VortexResult<vortex_array::ArrayRef>
+
 pub fn vortex_array::ArrayRef::is_null(&self) -> vortex_error::VortexResult<vortex_array::ArrayRef>
 
 pub fn vortex_array::ArrayRef::list_contains(&self, value: vortex_array::ArrayRef) -> vortex_error::VortexResult<vortex_array::ArrayRef>
@@ -8710,6 +8714,8 @@ pub fn vortex_array::builtins::ExprBuiltins::fill_null(&self, fill_value: vortex
 
 pub fn vortex_array::builtins::ExprBuiltins::get_item(&self, field_name: impl core::convert::Into<vortex_array::dtype::FieldName>) -> vortex_error::VortexResult<vortex_array::expr::Expression>
 
+pub fn vortex_array::builtins::ExprBuiltins::is_not_null(&self) -> vortex_error::VortexResult<vortex_array::expr::Expression>
+
 pub fn vortex_array::builtins::ExprBuiltins::is_null(&self) -> vortex_error::VortexResult<vortex_array::expr::Expression>
 
 pub fn vortex_array::builtins::ExprBuiltins::list_contains(&self, value: vortex_array::expr::Expression) -> vortex_error::VortexResult<vortex_array::expr::Expression>
@@ -8730,6 +8736,8 @@ pub fn vortex_array::expr::Expression::fill_null(&self, fill_value: vortex_array
 
 pub fn vortex_array::expr::Expression::get_item(&self, field_name: impl core::convert::Into<vortex_array::dtype::FieldName>) -> vortex_error::VortexResult<vortex_array::expr::Expression>
 
+pub fn vortex_array::expr::Expression::is_not_null(&self) -> vortex_error::VortexResult<vortex_array::expr::Expression>
+
 pub fn vortex_array::expr::Expression::is_null(&self) -> vortex_error::VortexResult<vortex_array::expr::Expression>
 
 pub fn vortex_array::expr::Expression::list_contains(&self, value: vortex_array::expr::Expression) -> vortex_error::VortexResult<vortex_array::expr::Expression>
@@ -9394,7 +9402,7 @@ impl core::cmp::Eq for vortex_array::dtype::DType
 
 impl core::cmp::PartialEq for vortex_array::dtype::DType
 
-pub fn vortex_array::dtype::DType::eq(&self, other: &vortex_array::dtype::DType) -> bool
+pub fn vortex_array::dtype::DType::eq(&self, other: &Self) -> bool
 
 impl core::convert::From<vortex_array::dtype::DType> for vortex_array::dtype::FieldDType
 
@@ -9444,8 +9452,6 @@ impl core::hash::Hash for vortex_array::dtype::DType
 
 pub fn vortex_array::dtype::DType::hash<__H: core::hash::Hasher>(&self, state: &mut __H)
 
-impl core::marker::StructuralPartialEq for vortex_array::dtype::DType
-
 impl vortex_array::dtype::arrow::FromArrowType<&arrow_schema::field::Field> for vortex_array::dtype::DType
 
 pub fn vortex_array::dtype::DType::from_arrow(field: &arrow_schema::field::Field) -> Self
@@ -12656,6 +12662,8 @@ pub fn vortex_array::expr::Expression::fill_null(&self, fill_value: vortex_array
 
 pub fn vortex_array::expr::Expression::get_item(&self, field_name: impl core::convert::Into<vortex_array::dtype::FieldName>) -> vortex_error::VortexResult<vortex_array::expr::Expression>
 
+pub fn vortex_array::expr::Expression::is_not_null(&self) -> vortex_error::VortexResult<vortex_array::expr::Expression>
+
 pub fn vortex_array::expr::Expression::is_null(&self) -> vortex_error::VortexResult<vortex_array::expr::Expression>
 
 pub fn vortex_array::expr::Expression::list_contains(&self, value: vortex_array::expr::Expression) -> vortex_error::VortexResult<vortex_array::expr::Expression>
@@ -12750,6 +12758,8 @@ pub fn vortex_array::expr::immediate_scope_access<'a>(expr: &'a vortex_array::ex
 
 pub fn vortex_array::expr::immediate_scope_accesses<'a>(expr: &'a vortex_array::expr::Expression, scope: &'a vortex_array::dtype::StructFields) -> vortex_array::expr::FieldAccesses<'a>
 
+pub fn vortex_array::expr::is_not_null(child: vortex_array::expr::Expression) -> vortex_array::expr::Expression
+
 pub fn vortex_array::expr::is_null(child: vortex_array::expr::Expression) -> vortex_array::expr::Expression
 
 pub fn vortex_array::expr::is_root(expr: &vortex_array::expr::Expression) -> bool
@@ -16532,6 +16542,52 @@ pub fn vortex_array::scalar_fn::fns::get_item::GetItem::stat_falsification(&self
 
 pub fn vortex_array::scalar_fn::fns::get_item::GetItem::validity(&self, options: &Self::Options, expression: &vortex_array::expr::Expression) -> vortex_error::VortexResult<core::option::Option<vortex_array::expr::Expression>>
 
+pub mod vortex_array::scalar_fn::fns::is_not_null
+
+pub struct vortex_array::scalar_fn::fns::is_not_null::IsNotNull
+
+impl core::clone::Clone for vortex_array::scalar_fn::fns::is_not_null::IsNotNull
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::clone(&self) -> vortex_array::scalar_fn::fns::is_not_null::IsNotNull
+
+impl vortex_array::scalar_fn::ScalarFnVTable for vortex_array::scalar_fn::fns::is_not_null::IsNotNull
+
+pub type vortex_array::scalar_fn::fns::is_not_null::IsNotNull::Options = vortex_array::scalar_fn::EmptyOptions
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::arity(&self, _options: &Self::Options) -> vortex_array::scalar_fn::Arity
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::child_name(&self, _instance: &Self::Options, child_idx: usize) -> vortex_array::scalar_fn::ChildName
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::coerce_args(&self, options: &Self::Options, args: &[vortex_array::dtype::DType]) -> vortex_error::VortexResult<alloc::vec::Vec<vortex_array::dtype::DType>>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::deserialize(&self, _metadata: &[u8], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Options>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::execute(&self, _data: &Self::Options, args: &dyn vortex_array::scalar_fn::ExecutionArgs, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ArrayRef>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::fmt_sql(&self, _options: &Self::Options, expr: &vortex_array::expr::Expression, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::id(&self) -> vortex_array::scalar_fn::ScalarFnId
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::is_fallible(&self, _instance: &Self::Options) -> bool
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::is_null_sensitive(&self, _instance: &Self::Options) -> bool
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::reduce(&self, options: &Self::Options, node: &dyn vortex_array::scalar_fn::ReduceNode, ctx: &dyn vortex_array::scalar_fn::ReduceCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::scalar_fn::ReduceNodeRef>>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::return_dtype(&self, _options: &Self::Options, _arg_dtypes: &[vortex_array::dtype::DType]) -> vortex_error::VortexResult<vortex_array::dtype::DType>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::serialize(&self, _instance: &Self::Options) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::simplify(&self, options: &Self::Options, expr: &vortex_array::expr::Expression, ctx: &dyn vortex_array::scalar_fn::SimplifyCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::expr::Expression>>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::simplify_untyped(&self, options: &Self::Options, expr: &vortex_array::expr::Expression) -> vortex_error::VortexResult<core::option::Option<vortex_array::expr::Expression>>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::stat_expression(&self, options: &Self::Options, expr: &vortex_array::expr::Expression, stat: vortex_array::expr::stats::Stat, catalog: &dyn vortex_array::expr::pruning::StatsCatalog) -> core::option::Option<vortex_array::expr::Expression>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::stat_falsification(&self, _options: &Self::Options, expr: &vortex_array::expr::Expression, catalog: &dyn vortex_array::expr::pruning::StatsCatalog) -> core::option::Option<vortex_array::expr::Expression>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::validity(&self, options: &Self::Options, expression: &vortex_array::expr::Expression) -> vortex_error::VortexResult<core::option::Option<vortex_array::expr::Expression>>
+
 pub mod vortex_array::scalar_fn::fns::is_null
 
 pub struct vortex_array::scalar_fn::fns::is_null::IsNull
@@ -18314,6 +18370,44 @@ pub fn vortex_array::scalar_fn::fns::get_item::GetItem::stat_falsification(&self
 
 pub fn vortex_array::scalar_fn::fns::get_item::GetItem::validity(&self, options: &Self::Options, expression: &vortex_array::expr::Expression) -> vortex_error::VortexResult<core::option::Option<vortex_array::expr::Expression>>
 
+impl vortex_array::scalar_fn::ScalarFnVTable for vortex_array::scalar_fn::fns::is_not_null::IsNotNull
+
+pub type vortex_array::scalar_fn::fns::is_not_null::IsNotNull::Options = vortex_array::scalar_fn::EmptyOptions
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::arity(&self, _options: &Self::Options) -> vortex_array::scalar_fn::Arity
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::child_name(&self, _instance: &Self::Options, child_idx: usize) -> vortex_array::scalar_fn::ChildName
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::coerce_args(&self, options: &Self::Options, args: &[vortex_array::dtype::DType]) -> vortex_error::VortexResult<alloc::vec::Vec<vortex_array::dtype::DType>>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::deserialize(&self, _metadata: &[u8], _session: &vortex_session::VortexSession) -> vortex_error::VortexResult<Self::Options>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::execute(&self, _data: &Self::Options, args: &dyn vortex_array::scalar_fn::ExecutionArgs, _ctx: &mut vortex_array::ExecutionCtx) -> vortex_error::VortexResult<vortex_array::ArrayRef>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::fmt_sql(&self, _options: &Self::Options, expr: &vortex_array::expr::Expression, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::id(&self) -> vortex_array::scalar_fn::ScalarFnId
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::is_fallible(&self, _instance: &Self::Options) -> bool
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::is_null_sensitive(&self, _instance: &Self::Options) -> bool
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::reduce(&self, options: &Self::Options, node: &dyn vortex_array::scalar_fn::ReduceNode, ctx: &dyn vortex_array::scalar_fn::ReduceCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::scalar_fn::ReduceNodeRef>>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::return_dtype(&self, _options: &Self::Options, _arg_dtypes: &[vortex_array::dtype::DType]) -> vortex_error::VortexResult<vortex_array::dtype::DType>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::serialize(&self, _instance: &Self::Options) -> vortex_error::VortexResult<core::option::Option<alloc::vec::Vec<u8>>>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::simplify(&self, options: &Self::Options, expr: &vortex_array::expr::Expression, ctx: &dyn vortex_array::scalar_fn::SimplifyCtx) -> vortex_error::VortexResult<core::option::Option<vortex_array::expr::Expression>>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::simplify_untyped(&self, options: &Self::Options, expr: &vortex_array::expr::Expression) -> vortex_error::VortexResult<core::option::Option<vortex_array::expr::Expression>>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::stat_expression(&self, options: &Self::Options, expr: &vortex_array::expr::Expression, stat: vortex_array::expr::stats::Stat, catalog: &dyn vortex_array::expr::pruning::StatsCatalog) -> core::option::Option<vortex_array::expr::Expression>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::stat_falsification(&self, _options: &Self::Options, expr: &vortex_array::expr::Expression, catalog: &dyn vortex_array::expr::pruning::StatsCatalog) -> core::option::Option<vortex_array::expr::Expression>
+
+pub fn vortex_array::scalar_fn::fns::is_not_null::IsNotNull::validity(&self, options: &Self::Options, expression: &vortex_array::expr::Expression) -> vortex_error::VortexResult<core::option::Option<vortex_array::expr::Expression>>
+
 impl vortex_array::scalar_fn::ScalarFnVTable for vortex_array::scalar_fn::fns::is_null::IsNull
 
 pub type vortex_array::scalar_fn::fns::is_null::IsNull::Options = vortex_array::scalar_fn::EmptyOptions
@@ -22820,6 +22914,8 @@ pub fn vortex_array::ArrayRef::fill_null(&self, fill_value: impl core::convert::
 
 pub fn vortex_array::ArrayRef::get_item(&self, field_name: impl core::convert::Into<vortex_array::dtype::FieldName>) -> vortex_error::VortexResult<vortex_array::ArrayRef>
 
+pub fn vortex_array::ArrayRef::is_not_null(&self) -> vortex_error::VortexResult<vortex_array::ArrayRef>
+
 pub fn vortex_array::ArrayRef::is_null(&self) -> vortex_error::VortexResult<vortex_array::ArrayRef>
 
 pub fn vortex_array::ArrayRef::list_contains(&self, value: vortex_array::ArrayRef) -> vortex_error::VortexResult<vortex_array::ArrayRef>