Merge branch 'main' into feature/ffi-executionplan-metrics

Author: mailmindlin

.github/workflows/codeql.yml

@@ -45,11 +45,11 @@ jobs:
         persist-credentials: false
 
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@e46ed2cbd01164d986452f91f178727624ae40d7 # v4
+      uses: github/codeql-action/init@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v4
       with:
         languages: actions
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@e46ed2cbd01164d986452f91f178727624ae40d7 # v4
+      uses: github/codeql-action/analyze@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v4
       with:
         category: "/language:actions"

Cargo.lock

@@ -192,7 +192,7 @@ regex = "1.12"
 rstest = "0.26.1"
 serde_json = "1"
 sha2 = "^0.11.0"
-sqlparser = { version = "0.61.0", default-features = false, features = ["std", "visitor"] }
+sqlparser = { version = "0.62.0", default-features = false, features = ["std", "visitor"] }
 strum = "0.28.0"
 strum_macros = "0.28.0"
 tempfile = "3"

Cargo.toml

@@ -19,6 +19,7 @@
 //! help with allocation accounting.
 
 use datafusion_common::{Result, internal_datafusion_err};
+use std::any::Any;
 use std::fmt::Display;
 use std::hash::{Hash, Hasher};
 use std::{cmp::Ordering, sync::Arc, sync::atomic};
@@ -182,7 +183,7 @@ pub use pool::*;
 ///
 /// * [`TrackConsumersPool`]: Wraps another [`MemoryPool`] and tracks consumers,
 ///   providing better error messages on the largest memory users.
-pub trait MemoryPool: Send + Sync + std::fmt::Debug + Display {
+pub trait MemoryPool: Any + Send + Sync + std::fmt::Debug + Display {
     /// Return pool name
     fn name(&self) -> &str;
 
@@ -224,6 +225,18 @@ pub trait MemoryPool: Send + Sync + std::fmt::Debug + Display {
     }
 }
 
+impl dyn MemoryPool {
+    /// Returns `true` if this pool is of type `T`.
+    pub fn is<T: MemoryPool>(&self) -> bool {
+        (self as &dyn Any).is::<T>()
+    }
+
+    /// Attempts to downcast this pool to a concrete type `T`.
+    pub fn downcast_ref<T: MemoryPool>(&self) -> Option<&T> {
+        (self as &dyn Any).downcast_ref()
+    }
+}
+
 /// Memory limit of `MemoryPool`
 pub enum MemoryLimit {
     Infinite,
@@ -603,6 +616,18 @@ mod tests {
         assert_eq!(pool.reserved(), 28);
     }
 
+    #[test]
+    fn test_downcast() {
+        let pool: Arc<dyn MemoryPool> = Arc::new(GreedyMemoryPool::new(50));
+
+        assert!(pool.is::<GreedyMemoryPool>());
+        assert!(!pool.is::<UnboundedMemoryPool>());
+
+        let greedy: &GreedyMemoryPool = pool.downcast_ref().unwrap();
+        assert_eq!(greedy.reserved(), 0);
+        assert!(pool.downcast_ref::<UnboundedMemoryPool>().is_none());
+    }
+
     #[test]
     fn test_try_shrink() {
         let pool = Arc::new(GreedyMemoryPool::new(100)) as _;

datafusion/execution/src/memory_pool/mod.rs

@@ -15,6 +15,16 @@
 // specific language governing permissions and limitations
 // under the License.
 
+//! Probabilistic distributions for expression-level statistics (unused).
+//!
+//! Note: All public items in this module are **deprecated** as of `54.0.0`.
+//!
+//! See <https://github.com/apache/datafusion/pull/22071> for details.
+
+// The whole module is deprecated; suppress warnings from intra-module uses
+// of the deprecated types so the module continues to compile.
+#![allow(deprecated)]
+
 use std::f64::consts::LN_2;
 
 use crate::interval_arithmetic::{Interval, apply_operator};
@@ -37,6 +47,10 @@ use datafusion_common::{
 /// is the main unit of calculus when evaluating expressions in a statistical
 /// context. Notions like column and table statistics are built on top of this
 /// object and the operations it supports.
+#[deprecated(
+    since = "54.0.0",
+    note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"
+)]
 #[derive(Clone, Debug, PartialEq)]
 pub enum Distribution {
     Uniform(UniformDistribution),
@@ -214,6 +228,10 @@ impl Distribution {
 ///
 /// <https://en.wikipedia.org/wiki/Continuous_uniform_distribution>
 /// <https://en.wikipedia.org/wiki/Prior_probability#Improper_priors>
+#[deprecated(
+    since = "54.0.0",
+    note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"
+)]
 #[derive(Clone, Debug, PartialEq)]
 pub struct UniformDistribution {
     interval: Interval,
@@ -236,6 +254,10 @@ pub struct UniformDistribution {
 /// For more information, see:
 ///
 /// <https://en.wikipedia.org/wiki/Exponential_distribution>
+#[deprecated(
+    since = "54.0.0",
+    note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"
+)]
 #[derive(Clone, Debug, PartialEq)]
 pub struct ExponentialDistribution {
     rate: ScalarValue,
@@ -249,6 +271,10 @@ pub struct ExponentialDistribution {
 /// For a more in-depth discussion, see:
 ///
 /// <https://en.wikipedia.org/wiki/Normal_distribution>
+#[deprecated(
+    since = "54.0.0",
+    note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"
+)]
 #[derive(Clone, Debug, PartialEq)]
 pub struct GaussianDistribution {
     mean: ScalarValue,
@@ -259,6 +285,10 @@ pub struct GaussianDistribution {
 /// the success probability is unknown. For a more in-depth discussion, see:
 ///
 /// <https://en.wikipedia.org/wiki/Bernoulli_distribution>
+#[deprecated(
+    since = "54.0.0",
+    note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"
+)]
 #[derive(Clone, Debug, PartialEq)]
 pub struct BernoulliDistribution {
     p: ScalarValue,
@@ -268,6 +298,10 @@ pub struct BernoulliDistribution {
 /// approximated via some summary statistics. For a more in-depth discussion, see:
 ///
 /// <https://en.wikipedia.org/wiki/Summary_statistics>
+#[deprecated(
+    since = "54.0.0",
+    note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"
+)]
 #[derive(Clone, Debug, PartialEq)]
 pub struct GenericDistribution {
     mean: ScalarValue,
@@ -594,6 +628,10 @@ impl GenericDistribution {
 /// This function takes a logical operator and two Bernoulli distributions,
 /// and it returns a new Bernoulli distribution that represents the result of
 /// the operation. Currently, only `AND` and `OR` operations are supported.
+#[deprecated(
+    since = "54.0.0",
+    note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"
+)]
 pub fn combine_bernoullis(
     op: &Operator,
     left: &BernoulliDistribution,
@@ -649,6 +687,10 @@ pub fn combine_bernoullis(
 /// see:
 ///
 /// <https://en.wikipedia.org/wiki/Sum_of_normally_distributed_random_variables>
+#[deprecated(
+    since = "54.0.0",
+    note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"
+)]
 pub fn combine_gaussians(
     op: &Operator,
     left: &GaussianDistribution,
@@ -673,6 +715,10 @@ pub fn combine_gaussians(
 /// Expects `op` to be a comparison operator, with `left` and `right` having
 /// numeric distributions. The resulting distribution has the `Float64` data
 /// type.
+#[deprecated(
+    since = "54.0.0",
+    note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"
+)]
 pub fn create_bernoulli_from_comparison(
     op: &Operator,
     left: &Distribution,
@@ -751,6 +797,10 @@ pub fn create_bernoulli_from_comparison(
 /// given binary operation on two unknown quantities represented by their
 /// [`Distribution`] objects. The function computes the mean, median and
 /// variance if possible.
+#[deprecated(
+    since = "54.0.0",
+    note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"
+)]
 pub fn new_generic_from_binary_op(
     op: &Operator,
     left: &Distribution,
@@ -766,6 +816,10 @@ pub fn new_generic_from_binary_op(
 
 /// Computes the mean value for the result of the given binary operation on
 /// two unknown quantities represented by their [`Distribution`] objects.
+#[deprecated(
+    since = "54.0.0",
+    note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"
+)]
 pub fn compute_mean(
     op: &Operator,
     left: &Distribution,
@@ -798,6 +852,10 @@ pub fn compute_mean(
 /// the median is calculable only for addition and subtraction operations on:
 /// - [`Uniform`] and [`Uniform`] distributions, and
 /// - [`Gaussian`] and [`Gaussian`] distributions.
+#[deprecated(
+    since = "54.0.0",
+    note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"
+)]
 pub fn compute_median(
     op: &Operator,
     left: &Distribution,
@@ -835,6 +893,10 @@ pub fn compute_median(
 
 /// Computes the variance value for the result of the given binary operation on
 /// two unknown quantities represented by their [`Distribution`] objects.
+#[deprecated(
+    since = "54.0.0",
+    note = "Part of the unused Statistics V2 framework; see https://github.com/apache/datafusion/pull/22071"
+)]
 pub fn compute_variance(
     op: &Operator,
     left: &Distribution,

datafusion/expr-common/src/statistics.rs

@@ -4014,8 +4014,8 @@ mod test {
                 wildcard_with_options(wildcard_options(
                     None,
                     Some(ExcludeSelectItem::Multiple(vec![
-                        Ident::from("c1"),
-                        Ident::from("c2")
+                        Ident::from("c1").into(),
+                        Ident::from("c2").into()
                     ])),
                     None,
                     None,

datafusion/expr/src/expr.rs

@@ -45,8 +45,8 @@ impl Display for IlikeSelectItem {
 
 #[derive(Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]
 pub enum ExcludeSelectItem {
-    Single(Ident),
-    Multiple(Vec<Ident>),
+    Single(ObjectName),
+    Multiple(Vec<ObjectName>),
 }
 
 impl Display for ExcludeSelectItem {
@@ -64,6 +64,37 @@ impl Display for ExcludeSelectItem {
     }
 }
 
+#[derive(Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]
+pub struct ObjectName(pub Vec<ObjectNamePart>);
+
+impl Display for ObjectName {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        let parts: Vec<String> = self.0.iter().map(|p| format!("{p}")).collect();
+        write!(f, "{}", parts.join("."))
+    }
+}
+
+#[derive(Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]
+pub enum ObjectNamePart {
+    Identifier(Ident),
+}
+
+impl ObjectNamePart {
+    pub fn as_ident(&self) -> Option<&Ident> {
+        match self {
+            ObjectNamePart::Identifier(ident) => Some(ident),
+        }
+    }
+}
+
+impl Display for ObjectNamePart {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        match self {
+            ObjectNamePart::Identifier(ident) => write!(f, "{ident}"),
+        }
+    }
+}
+
 #[derive(Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]
 pub struct ExceptSelectItem {
     pub first_element: Ident,

datafusion/expr/src/sql.rs

@@ -39,10 +39,10 @@ use datafusion_common::{
 };
 
 #[cfg(not(feature = "sql"))]
-use crate::sql::{ExceptSelectItem, ExcludeSelectItem};
+use crate::sql::{ExceptSelectItem, ExcludeSelectItem, Ident, ObjectName};
 use indexmap::IndexSet;
 #[cfg(feature = "sql")]
-use sqlparser::ast::{ExceptSelectItem, ExcludeSelectItem};
+use sqlparser::ast::{ExceptSelectItem, ExcludeSelectItem, Ident, ObjectName};
 
 pub use datafusion_functions_aggregate_common::order::AggregateOrderSensitivity;
 
@@ -339,11 +339,32 @@ fn get_excluded_columns(
         idents.push(&excepts.first_element);
         idents.extend(&excepts.additional_elements);
     }
+    // Declared outside the `if let` so `idents.extend(exclude_owned.iter())`
+    // below can borrow references that outlive the inner scope.
+    let exclude_owned: Vec<Ident>;
     if let Some(exclude) = opt_exclude {
-        match exclude {
-            ExcludeSelectItem::Single(ident) => idents.push(ident),
-            ExcludeSelectItem::Multiple(idents_inner) => idents.extend(idents_inner),
-        }
+        let object_name_to_ident = |name: &ObjectName| -> Result<Ident> {
+            if name.0.len() != 1 {
+                return plan_err!(
+                    "EXCLUDE with multi-part identifiers is not supported: {name}"
+                );
+            }
+            let part = &name.0[0];
+            let Some(ident) = part.as_ident() else {
+                return plan_err!(
+                    "EXCLUDE with non-identifier name part is not supported: {part}"
+                );
+            };
+            Ok(ident.clone())
+        };
+        exclude_owned = match exclude {
+            ExcludeSelectItem::Single(name) => vec![object_name_to_ident(name)?],
+            ExcludeSelectItem::Multiple(names) => names
+                .iter()
+                .map(object_name_to_ident)
+                .collect::<Result<Vec<_>>>()?,
+        };
+        idents.extend(exclude_owned.iter());
     }
     // Excluded columns should be unique
     let n_elem = idents.len();

datafusion/expr/src/utils.rs

@@ -15,6 +15,13 @@
 // specific language governing permissions and limitations
 // under the License.
 
+//! FFI types for the deprecated Statistics V2 [`Distribution`] framework.
+//!
+//! These FFI types mirror the deprecated probabilistic distribution types.
+//! See <https://github.com/apache/datafusion/pull/22071> for details.
+
+#![allow(deprecated)]
+
 use datafusion_common::DataFusionError;
 use datafusion_expr::statistics::{
     BernoulliDistribution, Distribution, ExponentialDistribution, GaussianDistribution,