initial

Signed-off-by: Mikhail Kot <to@myrrc.dev>

Author: myrrc

vortex-array/src/scalar/mod.rs

@@ -38,12 +38,12 @@ use crate::dtype::DType;
 #[derive(Clone, Debug, Eq)]
 pub struct Scalar {
     /// The type of the scalar.
-    dtype: DType,
+    pub dtype: DType,
 
     /// The value of the scalar. This is [`None`] if the value is null, otherwise it is [`Some`].
     ///
     /// Invariant: If the [`DType`] is non-nullable, then this value _cannot_ be [`None`].
-    value: Option<ScalarValue>,
+    pub value: Option<ScalarValue>,
 }
 
 #[cfg(test)]

vortex-duckdb/cpp/include/duckdb_vx/table_function.h

@@ -97,6 +97,18 @@ typedef struct {
     bool has_max_cardinality;
 } duckdb_vx_node_statistics;
 
+typedef struct {
+    duckdb_value min;
+    duckdb_value max;
+    // Duckdb's max_string length, uint32_t, is stored in lower bits.
+    // upper bit is "max string length is set"
+    uint64_t max_string_length;
+} duckdb_column_statistics;
+
+#define U64_HIGHEST_BIT_SET(x) (((x) & (UINT64_C(1) << 63)) != 0)
+
+typedef idx_t column_t;
+
 // A transparent DuckDB table function vtable, which can be used to configure a table function.
 // See duckdb/include/function/tfunc.hpp for details on each field.
 typedef struct {
@@ -137,7 +149,11 @@ typedef struct {
 
     // void *in_out_function;
     // void *in_out_function_final;
-    void *statistics;
+
+    void (*statistics)(duckdb_client_context context,
+                       const void *bind_data,
+                       size_t column_index,
+                       duckdb_column_statistics *stats_out);
 
     // void *dependency;
     void (*cardinality)(void *bind_data, duckdb_vx_node_statistics *node_stats_out);

vortex-duckdb/cpp/table_function.cpp

@@ -1,6 +1,7 @@
 // SPDX-License-Identifier: Apache-2.0
 // SPDX-FileCopyrightText: Copyright the Vortex contributors
 
+#include "duckdb_vx/table_function.h"
 #include "duckdb_vx/duckdb_diagnostics.h"
 
 DUCKDB_INCLUDES_BEGIN
@@ -30,8 +31,10 @@ struct CTableFunctionInfo final : TableFunctionInfo {
 };
 
 struct CTableBindData final : TableFunctionData {
-    CTableBindData(unique_ptr<CTableFunctionInfo> info_p, unique_ptr<vortex::CData> ffi_data_p)
-        : info(std::move(info_p)), ffi_data(std::move(ffi_data_p)) {
+    CTableBindData(unique_ptr<CTableFunctionInfo> info_p,
+                   unique_ptr<vortex::CData> ffi_data_p,
+                   const vector<LogicalType> &types)
+        : info(std::move(info_p)), ffi_data(std::move(ffi_data_p)), types(types) {
     }
 
     unique_ptr<FunctionData> Copy() const override {
@@ -43,11 +46,13 @@ struct CTableBindData final : TableFunctionData {
             throw BinderException(IntoErrString(error_out));
         }
         return make_uniq<CTableBindData>(make_uniq<CTableFunctionInfo>(info->vtab),
-                                         unique_ptr<CData>(reinterpret_cast<CData *>(copied_ffi_data)));
+                                         unique_ptr<CData>(reinterpret_cast<CData *>(copied_ffi_data)),
+                                         types);
     }
 
     unique_ptr<CTableFunctionInfo> info;
     unique_ptr<CData> ffi_data;
+    vector<LogicalType> types;
 };
 
 struct CTableGlobalData final : GlobalTableFunctionState {
@@ -88,6 +93,75 @@ double c_table_scan_progress(ClientContext &context,
     return bind.info->vtab.table_scan_progress(c_ctx, c_bind_data, c_global_state);
 }
 
+static Value &UnwrapValue(duckdb_value value) {
+    return *(reinterpret_cast<Value *>(value));
+}
+
+unique_ptr<BaseStatistics> numeric_stats(duckdb_column_statistics &stats, LogicalType type) {
+    BaseStatistics out = StringStats::CreateUnknown(type);
+    if (stats.min) {
+        NumericStats::SetMin(out, UnwrapValue(stats.min));
+        duckdb_destroy_value(&stats.min);
+    }
+    if (stats.max) {
+        NumericStats::SetMax(out, UnwrapValue(stats.max));
+        duckdb_destroy_value(&stats.max);
+    }
+    return out.ToUnique();
+}
+
+unique_ptr<BaseStatistics> string_stats(duckdb_column_statistics &stats, LogicalType type) {
+    BaseStatistics out = StringStats::CreateUnknown(type);
+    if (stats.min) {
+        StringStats::SetMin(out, StringValue::Get(UnwrapValue(stats.min)));
+        duckdb_destroy_value(&stats.min);
+    }
+    if (stats.max) {
+        StringStats::SetMax(out, StringValue::Get(UnwrapValue(stats.max)));
+        duckdb_destroy_value(&stats.max);
+    }
+    if (U64_HIGHEST_BIT_SET(stats.max_string_length)) {
+        StringStats::SetMaxStringLength(out, uint32_t(stats.max_string_length));
+    }
+    return out.ToUnique();
+}
+
+unique_ptr<BaseStatistics>
+c_statistics(ClientContext &context, const FunctionData *bind_data, column_t column_index) {
+    const auto &bind = bind_data->Cast<CTableBindData>();
+    void *const ffi_bind = bind.ffi_data->DataPtr();
+
+    duckdb_client_context c_ctx = reinterpret_cast<duckdb_client_context>(&context);
+    duckdb_column_statistics statistics = {};
+    const LogicalType type = bind.types[column_index];
+
+    switch (type.id()) {
+    case LogicalTypeId::BOOLEAN:
+    case LogicalTypeId::TINYINT:
+    case LogicalTypeId::SMALLINT:
+    case LogicalTypeId::INTEGER:
+    case LogicalTypeId::BIGINT:
+    case LogicalTypeId::FLOAT:
+    case LogicalTypeId::DOUBLE:
+    case LogicalTypeId::UTINYINT:
+    case LogicalTypeId::USMALLINT:
+    case LogicalTypeId::UINTEGER:
+    case LogicalTypeId::UBIGINT:
+    case LogicalTypeId::UHUGEINT:
+    case LogicalTypeId::HUGEINT: {
+        bind.info->vtab.statistics(c_ctx, ffi_bind, column_index, &statistics);
+        return numeric_stats(statistics, type);
+    }
+    case LogicalTypeId::VARCHAR:
+    case LogicalTypeId::BLOB: {
+        bind.info->vtab.statistics(c_ctx, ffi_bind, column_index, &statistics);
+        return string_stats(statistics, type);
+    }
+    default:
+        return BaseStatistics::CreateUnknown(type).ToUnique();
+    }
+}
+
 unique_ptr<FunctionData> c_bind(ClientContext &context,
                                 TableFunctionBindInput &input,
                                 vector<LogicalType> &return_types,
@@ -111,7 +185,8 @@ unique_ptr<FunctionData> c_bind(ClientContext &context,
     }
 
     return make_uniq<CTableBindData>(make_uniq<CTableFunctionInfo>(info.vtab),
-                                     unique_ptr<CData>(reinterpret_cast<CData *>(ffi_bind_data)));
+                                     unique_ptr<CData>(reinterpret_cast<CData *>(ffi_bind_data)),
+                                     return_types);
 }
 
 unique_ptr<GlobalTableFunctionState> c_init_global(ClientContext &context, TableFunctionInitInput &input) {
@@ -363,6 +438,7 @@ extern "C" duckdb_state duckdb_vx_tfunc_register(duckdb_database ffi_db, const d
     tf.get_virtual_columns = c_get_virtual_columns;
     tf.to_string = c_to_string;
     tf.table_scan_progress = c_table_scan_progress;
+    tf.statistics = c_statistics;
 
     // Set up the parameters
     tf.arguments.reserve(vtab->parameter_count);

vortex-duckdb/src/datasource.rs

@@ -37,6 +37,7 @@ use vortex::expr::col;
 use vortex::expr::root;
 use vortex::expr::select;
 use vortex::expr::stats::Precision;
+use vortex::file::FileStatistics;
 use vortex::io::kanal_ext::KanalExt;
 use vortex::io::runtime::BlockingRuntime;
 use vortex::io::runtime::current::ThreadSafeIterator;
@@ -54,6 +55,7 @@ use crate::duckdb::BindInputRef;
 use crate::duckdb::BindResultRef;
 use crate::duckdb::Cardinality;
 use crate::duckdb::ClientContextRef;
+use crate::duckdb::ColumnStatistics;
 use crate::duckdb::DataChunkRef;
 use crate::duckdb::ExpressionRef;
 use crate::duckdb::LogicalType;
@@ -97,7 +99,10 @@ pub(crate) trait DataSourceTableFunction: Sized + Debug {
     }
 
     /// Bind the table function and return a [`DataSourceRef`].
-    fn bind(ctx: &ClientContextRef, input: &BindInputRef) -> VortexResult<DataSourceRef>;
+    fn bind(
+        ctx: &ClientContextRef,
+        input: &BindInputRef,
+    ) -> VortexResult<(DataSourceRef, FileStatistics)>;
 }
 
 /// Bind data produced by a [`DataSourceTableFunction`].
@@ -106,6 +111,7 @@ pub struct DataSourceBindData {
     filter_exprs: Vec<Expression>,
     column_names: Vec<String>,
     column_types: Vec<LogicalType>,
+    stats: Vec<ColumnStatistics>,
 }
 
 impl Clone for DataSourceBindData {
@@ -116,6 +122,7 @@ impl Clone for DataSourceBindData {
             filter_exprs: vec![],
             column_names: self.column_names.clone(),
             column_types: self.column_types.clone(),
+            stats: self.stats.clone(),
         }
     }
 }
@@ -189,19 +196,24 @@ impl<T: DataSourceTableFunction> TableFunction for T {
         input: &BindInputRef,
         result: &mut BindResultRef,
     ) -> VortexResult<Self::BindData> {
-        let data_source = T::bind(ctx, input)?;
+        let (data_source, file_stats) = T::bind(ctx, input)?;
 
         let (column_names, column_types) = extract_schema_from_dtype(data_source.dtype())?;
 
-        for (column_name, column_type) in column_names.iter().zip(&column_types) {
+        let mut stats = Vec::new();
+        for (i, (column_name, column_type)) in column_names.iter().zip(&column_types).enumerate() {
             result.add_result_column(column_name, column_type);
+            let stats_set = &file_stats.stats_sets()[i];
+            let dtype = &file_stats.dtypes()[i];
+            stats.push(ColumnStatistics::new(&stats_set, dtype.clone()));
         }
 
         Ok(DataSourceBindData {
             data_source,
             filter_exprs: vec![],
             column_names,
             column_types,
+            stats,
         })
     }
 
@@ -412,6 +424,14 @@ impl<T: DataSourceTableFunction> TableFunction for T {
         Ok(false)
     }
 
+    fn statistics(
+        _client_context: &ClientContextRef,
+        bind_data: &Self::BindData,
+        column_index: usize,
+    ) -> ColumnStatistics {
+        bind_data.stats[column_index].clone()
+    }
+
     fn cardinality(bind_data: &Self::BindData) -> Cardinality {
         match bind_data.data_source.row_count() {
             Some(Precision::Exact(v)) => Cardinality::Maximum(v),

vortex-duckdb/src/duckdb/mod.rs

@@ -19,7 +19,8 @@ mod reusable_dict;
 mod scalar_function;
 mod selection_vector;
 mod table_filter;
-mod table_function;
+// TODO(myrrc) should it be private?
+pub mod table_function;
 mod value;
 mod vector;
 mod vector_buffer;

vortex-duckdb/src/duckdb/table_function/mod.rs

@@ -7,20 +7,26 @@ use std::ffi::c_void;
 use std::fmt::Debug;
 use std::ptr;
 
+use vortex::dtype::DType;
 use vortex::error::VortexExpect;
 use vortex::error::VortexResult;
 mod bind;
 mod cardinality;
 mod init;
 mod partition;
 mod pushdown_complex_filter;
+mod statistics;
 mod table_scan_progress;
 mod virtual_columns;
 
 pub use bind::*;
 pub use init::*;
 pub use virtual_columns::VirtualColumnsResult;
 pub use virtual_columns::VirtualColumnsResultRef;
+use vortex::array::stats::StatsSet;
+use vortex::expr::stats::Precision;
+use vortex::expr::stats::Stat;
+use vortex::scalar::ScalarValue;
 
 use crate::cpp;
 use crate::cpp::duckdb_client_context;
@@ -34,10 +40,50 @@ use crate::duckdb::expr::ExpressionRef;
 use crate::duckdb::table_function::cardinality::cardinality_callback;
 use crate::duckdb::table_function::partition::get_partition_data_callback;
 use crate::duckdb::table_function::pushdown_complex_filter::pushdown_complex_filter_callback;
+use crate::duckdb::table_function::statistics::statistics;
 use crate::duckdb::table_function::table_scan_progress::table_scan_progress_callback;
 use crate::duckdb::table_function::virtual_columns::get_virtual_columns_callback;
 use crate::duckdb_try;
 
+#[derive(Clone)]
+pub struct ColumnStatistics {
+    pub minmax_dtype: DType,
+    pub min: Option<ScalarValue>,
+    pub max: Option<ScalarValue>,
+    pub max_string_length: Option<u32>,
+}
+
+impl ColumnStatistics {
+    pub fn new(stats: &StatsSet, dtype: DType) -> Self {
+        let min = if let Some(Precision::Exact(value)) = stats.get(Stat::Min) {
+            Some(value)
+        } else {
+            None
+        };
+
+        let max = if let Some(Precision::Exact(value)) = stats.get(Stat::Max) {
+            Some(value)
+        } else {
+            None
+        };
+
+        // TODO(myrrc): does it calculate string length?
+        let max_string_length =
+            if let Some(Precision::Exact(value)) = stats.get(Stat::UncompressedSizeInBytes) {
+                Some(value.as_primitive().as_u64().expect("not a u64") as u32)
+            } else {
+                None
+            };
+
+        Self {
+            minmax_dtype: dtype,
+            min,
+            max,
+            max_string_length,
+        }
+    }
+}
+
 /// A trait that defines the supported operations for a table function in DuckDB.
 ///
 /// This trait does not yet cover the full C++ API, see table_function.hpp.
@@ -84,6 +130,12 @@ pub trait TableFunction: Sized + Debug {
         result: &mut BindResultRef,
     ) -> VortexResult<Self::BindData>;
 
+    fn statistics(
+        client_context: &ClientContextRef,
+        bind_data: &Self::BindData,
+        column_index: usize,
+    ) -> ColumnStatistics;
+
     /// The function is called during query execution and is responsible for producing the output
     fn scan(
         client_context: &ClientContextRef,
@@ -188,7 +240,7 @@ impl DatabaseRef {
             init_global: Some(init_global_callback::<T>),
             init_local: Some(init_local_callback::<T>),
             function: Some(function::<T>),
-            statistics: ptr::null_mut::<c_void>(),
+            statistics: Some(statistics::<T>),
             cardinality: Some(cardinality_callback::<T>),
             pushdown_complex_filter: Some(pushdown_complex_filter_callback::<T>),
             pushdown_expression: ptr::null_mut::<c_void>(),