duckdb: Exit earlier on Validity::AllFalse (#7411)

1. Make duckdb all-invalid exporter a unit type:
  If we set underlying vector's validity to "all false",
  duckdb won't read underlying values so you don't need to
  fill them
2. Exit early on exporter branches if validity is all false.
  Saves a ConstantArray creation + execute

Signed-off-by: Mikhail Kot <to@myrrc.dev>

Author: myrrc

vortex-duckdb/src/exporter/all_invalid.rs

@@ -3,47 +3,32 @@
 
 use vortex::array::ExecutionCtx;
 use vortex::error::VortexResult;
-use vortex::error::vortex_ensure;
 
-use crate::duckdb::LogicalTypeRef;
-use crate::duckdb::Value;
 use crate::duckdb::VectorRef;
 use crate::exporter::ColumnExporter;
 
-struct AllInvalidExporter {
-    len: usize,
-    null_value: Value,
-}
+struct AllInvalidExporter;
 
-pub(crate) fn new_exporter(len: usize, logical_type: &LogicalTypeRef) -> Box<dyn ColumnExporter> {
-    Box::new(AllInvalidExporter {
-        len,
-        null_value: Value::null(logical_type),
-    })
+pub(crate) fn new_exporter() -> Box<dyn ColumnExporter> {
+    Box::new(AllInvalidExporter {})
 }
 
 impl ColumnExporter for AllInvalidExporter {
     fn export(
         &self,
-        offset: usize,
-        len: usize,
+        _offset: usize,
+        _len: usize,
         vector: &mut VectorRef,
         _ctx: &mut ExecutionCtx,
     ) -> VortexResult<()> {
-        vortex_ensure!(
-            offset + len <= self.len,
-            "invalid exporter: offset + len must be less than or equal to len"
-        );
-
-        vector.reference_value(&self.null_value);
+        vector.set_all_false_validity();
         Ok(())
     }
 }
 
 #[cfg(test)]
 mod tests {
     use vortex::array::VortexSessionExecute;
-    use vortex::array::arrays::PrimitiveArray;
 
     use super::*;
     use crate::SESSION;
@@ -52,12 +37,10 @@ mod tests {
 
     #[test]
     fn all_null_array() {
-        let arr = PrimitiveArray::from_option_iter::<i32, _>([None, None, None]);
         let ltype = LogicalType::int32();
+        let mut chunk = DataChunk::new([ltype]);
 
-        let mut chunk = DataChunk::new([ltype.clone()]);
-
-        new_exporter(arr.len(), &ltype)
+        new_exporter()
             .export(
                 0,
                 3,

vortex-duckdb/src/exporter/bool.rs

@@ -4,11 +4,11 @@
 use vortex::array::ExecutionCtx;
 use vortex::array::arrays::BoolArray;
 use vortex::array::arrays::bool::BoolArrayExt;
+use vortex::array::validity::Validity;
 use vortex::buffer::BitBuffer;
 use vortex::error::VortexResult;
 use vortex::mask::Mask;
 
-use crate::duckdb::LogicalType;
 use crate::duckdb::VectorRef;
 use crate::exporter::ColumnExporter;
 use crate::exporter::all_invalid;
@@ -24,11 +24,12 @@ pub(crate) fn new_exporter(
 ) -> VortexResult<Box<dyn ColumnExporter>> {
     let len = array.len();
     let bits = array.to_bit_buffer();
-    let validity = array.validity()?.to_array(len).execute::<Mask>(ctx)?;
 
-    if validity.all_false() {
-        return Ok(all_invalid::new_exporter(len, &LogicalType::bool()));
+    let validity = array.validity()?;
+    if matches!(validity, Validity::AllInvalid) {
+        return Ok(all_invalid::new_exporter());
     }
+    let validity = validity.to_array(len).execute::<Mask>(ctx)?;
 
     Ok(validity::new_exporter(
         validity,

vortex-duckdb/src/exporter/constant.rs

@@ -65,14 +65,14 @@ impl ColumnExporter for ConstantExporter {
     fn export(
         &self,
         _offset: usize,
-        len: usize,
+        _len: usize,
         vector: &mut VectorRef,
         _ctx: &mut ExecutionCtx,
     ) -> VortexResult<()> {
         match self.value.as_ref() {
             None => {
                 // TODO(ngates): would be good if DuckDB supported constant null vectors.
-                unsafe { vector.set_validity(&Mask::AllFalse(len), 0, len) };
+                vector.set_all_false_validity();
             }
             Some(value) => {
                 vector.reference_value(value);

vortex-duckdb/src/exporter/decimal.rs

@@ -8,9 +8,9 @@ use vortex::array::ExecutionCtx;
 use vortex::array::arrays::DecimalArray;
 use vortex::array::arrays::decimal::DecimalDataParts;
 use vortex::array::match_each_decimal_value_type;
+use vortex::array::validity::Validity;
 use vortex::buffer::Buffer;
 use vortex::dtype::BigCast;
-use vortex::dtype::DType;
 use vortex::dtype::DecimalDType;
 use vortex::dtype::DecimalType;
 use vortex::dtype::NativeDecimalType;
@@ -19,7 +19,6 @@ use vortex::error::VortexResult;
 use vortex::error::vortex_bail;
 use vortex::mask::Mask;
 
-use crate::duckdb::LogicalType;
 use crate::duckdb::VectorBuffer;
 use crate::duckdb::VectorRef;
 use crate::exporter::ColumnExporter;
@@ -49,13 +48,11 @@ pub(crate) fn new_exporter(
         values,
     } = array.into_data_parts();
     let dest_values_type = precision_to_duckdb_storage_size(&decimal_dtype)?;
-    let nullability = validity.nullability();
-    let validity = validity.to_array(len).execute::<Mask>(ctx)?;
 
-    if validity.all_false() {
-        let ltype = LogicalType::try_from(DType::Decimal(decimal_dtype, nullability))?;
-        return Ok(all_invalid::new_exporter(len, &ltype));
+    if matches!(validity, Validity::AllInvalid) {
+        return Ok(all_invalid::new_exporter());
     }
+    let validity = validity.to_array(len).execute::<Mask>(ctx)?;
 
     let exporter = if values_type == dest_values_type {
         match_each_decimal_value_type!(values_type, |D| {

vortex-duckdb/src/exporter/dict.rs

@@ -17,7 +17,6 @@ use vortex::dtype::IntegerPType;
 use vortex::error::VortexResult;
 use vortex::mask::Mask;
 
-use crate::duckdb::LogicalType;
 use crate::duckdb::ReusableDict;
 use crate::duckdb::SelectionVector;
 use crate::duckdb::VectorRef;
@@ -43,7 +42,6 @@ pub(crate) fn new_exporter_with_flatten(
 ) -> VortexResult<Box<dyn ColumnExporter>> {
     // Grab the cache dictionary values.
     let values = array.values();
-    let values_type: LogicalType = values.dtype().try_into()?;
     if let Some(constant) = values.as_opt::<Constant>() {
         return constant::new_exporter_with_mask(
             ConstantArray::new(constant.scalar().clone(), array.codes().len()),
@@ -57,7 +55,7 @@ pub(crate) fn new_exporter_with_flatten(
 
     match codes_mask {
         Mask::AllTrue(_) => {}
-        Mask::AllFalse(len) => return Ok(all_invalid::new_exporter(len, &values_type)),
+        Mask::AllFalse(_) => return Ok(all_invalid::new_exporter()),
         Mask::Values(_) => {
             // duckdb cannot have a dictionary with validity in the codes, so flatten the array and
             // apply the validity mask there.

vortex-duckdb/src/exporter/fixed_size_list.rs

@@ -11,14 +11,14 @@
 use vortex::array::ExecutionCtx;
 use vortex::array::arrays::FixedSizeListArray;
 use vortex::array::arrays::fixed_size_list::FixedSizeListArrayExt;
+use vortex::array::validity::Validity;
 use vortex::error::VortexResult;
 use vortex::mask::Mask;
 
 use super::ConversionCache;
 use super::all_invalid;
 use super::new_array_exporter_with_flatten;
 use super::validity;
-use crate::duckdb::LogicalType;
 use crate::duckdb::VectorRef;
 use crate::exporter::ColumnExporter;
 
@@ -42,15 +42,14 @@ pub(crate) fn new_exporter(
     let parts = array.into_data_parts();
     let elements = parts.elements;
     let validity = parts.validity;
-    let dtype = parts.dtype;
-    let mask = validity.to_array(len).execute::<Mask>(ctx)?;
-    let elements_exporter = new_array_exporter_with_flatten(elements, cache, ctx, true)?;
 
-    if mask.all_false() {
-        let ltype = LogicalType::try_from(dtype)?;
-        return Ok(all_invalid::new_exporter(len, &ltype));
+    if matches!(validity, Validity::AllInvalid) {
+        return Ok(all_invalid::new_exporter());
     }
 
+    let mask = validity.to_array(len).execute::<Mask>(ctx)?;
+    let elements_exporter = new_array_exporter_with_flatten(elements, cache, ctx, true)?;
+
     Ok(validity::new_exporter(
         mask,
         Box::new(FixedSizeListExporter {

vortex-duckdb/src/exporter/list.rs

@@ -10,6 +10,7 @@ use vortex::array::arrays::ListArray;
 use vortex::array::arrays::PrimitiveArray;
 use vortex::array::arrays::list::ListDataParts;
 use vortex::array::match_each_integer_ptype;
+use vortex::array::validity::Validity;
 use vortex::dtype::IntegerPType;
 use vortex::error::VortexResult;
 use vortex::error::vortex_err;
@@ -49,15 +50,14 @@ pub(crate) fn new_exporter(
         elements,
         offsets,
         validity,
-        dtype,
+        dtype: _dtype,
     } = array.into_data_parts();
     let num_elements = elements.len();
-    let validity = validity.to_array(array_len).execute::<Mask>(ctx)?;
 
-    if validity.all_false() {
-        let ltype = LogicalType::try_from(dtype)?;
-        return Ok(all_invalid::new_exporter(array_len, &ltype));
+    if matches!(validity, Validity::AllInvalid) {
+        return Ok(all_invalid::new_exporter());
     }
+    let validity = validity.to_array(array_len).execute::<Mask>(ctx)?;
 
     let values_key = elements.addr();
     // Check if we have a cached vector and extract it if we do.

vortex-duckdb/src/exporter/list_view.rs

@@ -10,7 +10,7 @@ use vortex::array::arrays::ListViewArray;
 use vortex::array::arrays::PrimitiveArray;
 use vortex::array::arrays::listview::ListViewDataParts;
 use vortex::array::match_each_integer_ptype;
-use vortex::dtype::DType;
+use vortex::array::validity::Validity;
 use vortex::dtype::IntegerPType;
 use vortex::error::VortexResult;
 use vortex::error::vortex_err;
@@ -56,13 +56,11 @@ pub(crate) fn new_exporter(
     } = array.into_data_parts();
     // Cache an `elements` vector up front so that future exports can reference it.
     let num_elements = elements.len();
-    let nullability = validity.nullability();
-    let validity = validity.to_array(len).execute::<Mask>(ctx)?;
 
-    if validity.all_false() {
-        let ltype = LogicalType::try_from(DType::List(elements_dtype, nullability))?;
-        return Ok(all_invalid::new_exporter(len, &ltype));
+    if matches!(validity, Validity::AllInvalid) {
+        return Ok(all_invalid::new_exporter());
     }
+    let validity = validity.to_array(len).execute::<Mask>(ctx)?;
 
     let values_key = elements.addr();
     // Check if we have a cached vector and extract it if we do.

vortex-duckdb/src/exporter/mod.rs

@@ -37,7 +37,6 @@ use vortex::error::VortexResult;
 use vortex::error::vortex_bail;
 
 use crate::duckdb::DataChunkRef;
-use crate::duckdb::LogicalType;
 use crate::duckdb::VectorRef;
 use crate::duckdb::duckdb_vector_size;
 
@@ -166,7 +165,7 @@ fn new_array_exporter_with_flatten(
 
     // Otherwise, we fall back to canonical
     match array.execute::<Canonical>(ctx)? {
-        Canonical::Null(array) => Ok(all_invalid::new_exporter(array.len(), &LogicalType::null())),
+        Canonical::Null(_) => Ok(all_invalid::new_exporter()),
         Canonical::Bool(array) => bool::new_exporter(array, ctx),
         Canonical::Primitive(array) => primitive::new_exporter(array, ctx),
         Canonical::Decimal(array) => decimal::new_exporter(array, ctx),

vortex-duckdb/src/exporter/primitive.rs

@@ -6,11 +6,11 @@ use std::marker::PhantomData;
 use vortex::array::ExecutionCtx;
 use vortex::array::arrays::PrimitiveArray;
 use vortex::array::match_each_native_ptype;
+use vortex::array::validity::Validity;
 use vortex::dtype::NativePType;
 use vortex::error::VortexResult;
 use vortex::mask::Mask;
 
-use crate::duckdb::LogicalType;
 use crate::duckdb::VectorBuffer;
 use crate::duckdb::VectorRef;
 use crate::exporter::ColumnExporter;
@@ -28,15 +28,11 @@ pub fn new_exporter(
     array: PrimitiveArray,
     ctx: &mut ExecutionCtx,
 ) -> VortexResult<Box<dyn ColumnExporter>> {
-    let validity = array
-        .validity()?
-        .to_array(array.len())
-        .execute::<Mask>(ctx)?;
-
-    if validity.all_false() {
-        let ltype = LogicalType::try_from(array.ptype())?;
-        return Ok(all_invalid::new_exporter(array.len(), &ltype));
-    }
+    let validity = array.validity()?;
+    if matches!(validity, Validity::AllInvalid) {
+        return Ok(all_invalid::new_exporter());
+    };
+    let validity = validity.to_array(array.len()).execute::<Mask>(ctx)?;
 
     match_each_native_ptype!(array.ptype(), |T| {
         let buffer = array.to_buffer::<T>();