Merge branch 'main' into file-stats-cache

Author: mkleen

.github/workflows/audit.yml

@@ -45,7 +45,7 @@ jobs:
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
       - name: Install cargo-audit
-        uses: taiki-e/install-action@7ea35f098a7369cd23488403f58be9c491a6c55f  # v2.77.0
+        uses: taiki-e/install-action@c070f87102a1c75b3183910f391c1cb887fe13c8  # v2.77.6
         with:
           tool: cargo-audit
       - name: Run audit check

.github/workflows/breaking_changes_detector.yml

@@ -89,7 +89,7 @@ jobs:
 
       - name: Install cargo-semver-checks
         if: steps.changed_crates.outputs.packages != ''
-        uses: taiki-e/install-action@7ea35f098a7369cd23488403f58be9c491a6c55f  # v2.77.0
+        uses: taiki-e/install-action@c070f87102a1c75b3183910f391c1cb887fe13c8  # v2.77.6
         with:
           tool: cargo-semver-checks
 

.github/workflows/dev.yml

@@ -64,7 +64,7 @@ jobs:
           source ci/scripts/utils/tool_versions.sh
           echo "LYCHEE_VERSION=${LYCHEE_VERSION}" >> "$GITHUB_ENV"
       - name: Install lychee
-        uses: taiki-e/install-action@7ea35f098a7369cd23488403f58be9c491a6c55f  # v2.77.0
+        uses: taiki-e/install-action@c070f87102a1c75b3183910f391c1cb887fe13c8  # v2.77.6
         with:
           tool: lychee@${{ env.LYCHEE_VERSION }}
       - name: Run markdown link check

.github/workflows/rust.yml

@@ -429,7 +429,7 @@ jobs:
           sudo apt-get update -qq
           sudo apt-get install -y -qq clang
       - name: Setup wasm-pack
-        uses: taiki-e/install-action@7ea35f098a7369cd23488403f58be9c491a6c55f  # v2.77.0
+        uses: taiki-e/install-action@c070f87102a1c75b3183910f391c1cb887fe13c8  # v2.77.6
         with:
           tool: wasm-pack
       - name: Run tests with headless mode
@@ -774,7 +774,7 @@ jobs:
       - name: Setup Rust toolchain
         uses: ./.github/actions/setup-builder
       - name: Install cargo-msrv
-        uses: taiki-e/install-action@7ea35f098a7369cd23488403f58be9c491a6c55f  # v2.77.0
+        uses: taiki-e/install-action@c070f87102a1c75b3183910f391c1cb887fe13c8  # v2.77.6
         with:
           tool: cargo-msrv
 

datafusion/core/tests/parquet/row_group_pruning.rs

@@ -1777,6 +1777,15 @@ fn make_i32_batch(
     RecordBatch::try_new(schema, vec![array]).map_err(DataFusionError::from)
 }
 
+fn make_nullable_i32_batch(
+    name: &str,
+    values: Vec<Option<i32>>,
+) -> datafusion_common::error::Result<RecordBatch> {
+    let schema = Arc::new(Schema::new(vec![Field::new(name, DataType::Int32, true)]));
+    let array: ArrayRef = Arc::new(Int32Array::from(values));
+    RecordBatch::try_new(schema, vec![array]).map_err(DataFusionError::from)
+}
+
 // Helper function to create a batch with two Int32 columns
 fn make_two_col_i32_batch(
     name_a: &str,
@@ -1793,6 +1802,72 @@ fn make_two_col_i32_batch(
     RecordBatch::try_new(schema, vec![array_a, array_b]).map_err(DataFusionError::from)
 }
 
+#[tokio::test]
+async fn prune_is_not_distinct_from_i32() -> datafusion_common::error::Result<()> {
+    let schema = Arc::new(Schema::new(vec![Field::new("a", DataType::Int32, true)]));
+    let batches = vec![
+        make_nullable_i32_batch("a", vec![None, None])?,
+        make_nullable_i32_batch("a", vec![Some(0), Some(0)])?,
+        make_nullable_i32_batch("a", vec![Some(0), Some(1)])?,
+        make_nullable_i32_batch("a", vec![Some(2), Some(3)])?,
+        make_nullable_i32_batch("a", vec![None, Some(5)])?,
+    ];
+
+    RowGroupPruningTest::new()
+        .with_scenario(Scenario::Int)
+        .with_query("SELECT a FROM t WHERE a IS NOT DISTINCT FROM 0")
+        .with_expected_errors(Some(0))
+        .with_expected_rows(3)
+        .with_pruned_files(Some(0))
+        .with_matched_by_stats(Some(2))
+        .with_fully_matched_by_stats(Some(1))
+        .with_pruned_by_stats(Some(3))
+        .with_limit_pruned_row_groups(Some(0))
+        .test_row_group_prune_with_custom_data(schema.clone(), batches.clone(), 2)
+        .await;
+
+    RowGroupPruningTest::new()
+        .with_scenario(Scenario::Int)
+        .with_query("SELECT a FROM t WHERE a IS NOT DISTINCT FROM NULL")
+        .with_expected_errors(Some(0))
+        .with_expected_rows(3)
+        .with_pruned_files(Some(0))
+        .with_matched_by_stats(Some(2))
+        .with_fully_matched_by_stats(Some(0))
+        .with_pruned_by_stats(Some(3))
+        .with_limit_pruned_row_groups(Some(0))
+        .test_row_group_prune_with_custom_data(schema.clone(), batches.clone(), 2)
+        .await;
+
+    RowGroupPruningTest::new()
+        .with_scenario(Scenario::Int)
+        .with_query("SELECT a FROM t WHERE a IS DISTINCT FROM 0")
+        .with_expected_errors(Some(0))
+        .with_expected_rows(7)
+        .with_pruned_files(Some(0))
+        .with_matched_by_stats(Some(4))
+        .with_fully_matched_by_stats(Some(1))
+        .with_pruned_by_stats(Some(1))
+        .with_limit_pruned_row_groups(Some(0))
+        .test_row_group_prune_with_custom_data(schema.clone(), batches.clone(), 2)
+        .await;
+
+    RowGroupPruningTest::new()
+        .with_scenario(Scenario::Int)
+        .with_query("SELECT a FROM t WHERE a IS DISTINCT FROM NULL")
+        .with_expected_errors(Some(0))
+        .with_expected_rows(7)
+        .with_pruned_files(Some(0))
+        .with_matched_by_stats(Some(4))
+        .with_fully_matched_by_stats(Some(3))
+        .with_pruned_by_stats(Some(1))
+        .with_limit_pruned_row_groups(Some(0))
+        .test_row_group_prune_with_custom_data(schema, batches, 2)
+        .await;
+
+    Ok(())
+}
+
 #[tokio::test]
 async fn test_limit_pruning_basic() -> datafusion_common::error::Result<()> {
     // Scenario: Simple integer column, multiple row groups

datafusion/expr-common/src/operator.rs

@@ -255,9 +255,9 @@ impl Operator {
             Operator::GtEq => Some(Operator::LtEq),
             Operator::AtArrow => Some(Operator::ArrowAt),
             Operator::ArrowAt => Some(Operator::AtArrow),
-            Operator::IsDistinctFrom
-            | Operator::IsNotDistinctFrom
-            | Operator::Plus
+            Operator::IsDistinctFrom => Some(Operator::IsDistinctFrom),
+            Operator::IsNotDistinctFrom => Some(Operator::IsNotDistinctFrom),
+            Operator::Plus
             | Operator::Minus
             | Operator::Multiply
             | Operator::Divide

datafusion/expr/src/expr.rs

@@ -46,7 +46,13 @@ use datafusion_common::{
 use datafusion_expr_common::placement::ExpressionPlacement;
 use datafusion_functions_window_common::field::WindowUDFFieldArgs;
 #[cfg(feature = "sql")]
-use sqlparser::ast::{
+pub use sqlparser::ast::{
+    ExceptSelectItem, ExcludeSelectItem, IlikeSelectItem, RenameSelectItem,
+    ReplaceSelectElement,
+};
+// Use shims for sqlparser types when the sql feature is disabled.
+#[cfg(not(feature = "sql"))]
+pub use crate::sql::{
     ExceptSelectItem, ExcludeSelectItem, IlikeSelectItem, RenameSelectItem,
     ReplaceSelectElement,
 };
@@ -1416,63 +1422,6 @@ impl Lambda {
     }
 }
 
-#[derive(Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]
-#[cfg(not(feature = "sql"))]
-pub struct IlikeSelectItem {
-    pub pattern: String,
-}
-#[cfg(not(feature = "sql"))]
-impl Display for IlikeSelectItem {
-    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
-        write!(f, "ILIKE '{}'", &self.pattern)?;
-        Ok(())
-    }
-}
-#[derive(Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]
-#[cfg(not(feature = "sql"))]
-pub enum ExcludeSelectItem {
-    Single(Ident),
-    Multiple(Vec<Ident>),
-}
-#[cfg(not(feature = "sql"))]
-impl Display for ExcludeSelectItem {
-    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
-        write!(f, "EXCLUDE")?;
-        match self {
-            Self::Single(column) => {
-                write!(f, " {column}")?;
-            }
-            Self::Multiple(columns) => {
-                write!(f, " ({})", display_comma_separated(columns))?;
-            }
-        }
-        Ok(())
-    }
-}
-#[derive(Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]
-#[cfg(not(feature = "sql"))]
-pub struct ExceptSelectItem {
-    pub first_element: Ident,
-    pub additional_elements: Vec<Ident>,
-}
-#[cfg(not(feature = "sql"))]
-impl Display for ExceptSelectItem {
-    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
-        write!(f, "EXCEPT ")?;
-        if self.additional_elements.is_empty() {
-            write!(f, "({})", self.first_element)?;
-        } else {
-            write!(
-                f,
-                "({}, {})",
-                self.first_element,
-                display_comma_separated(&self.additional_elements)
-            )?;
-        }
-        Ok(())
-    }
-}
-
 pub fn display_comma_separated<T>(slice: &[T]) -> String
 where
     T: Display,
@@ -1481,64 +1430,6 @@ where
     slice.iter().map(|v| format!("{v}")).join(", ")
 }
 
-#[derive(Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]
-#[cfg(not(feature = "sql"))]
-pub enum RenameSelectItem {
-    Single(String),
-    Multiple(Vec<String>),
-}
-#[cfg(not(feature = "sql"))]
-impl Display for RenameSelectItem {
-    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
-        write!(f, "RENAME")?;
-        match self {
-            Self::Single(column) => {
-                write!(f, " {column}")?;
-            }
-            Self::Multiple(columns) => {
-                write!(f, " ({})", display_comma_separated(columns))?;
-            }
-        }
-        Ok(())
-    }
-}
-
-#[derive(Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]
-#[cfg(not(feature = "sql"))]
-pub struct Ident {
-    /// The value of the identifier without quotes.
-    pub value: String,
-    /// The starting quote if any. Valid quote characters are the single quote,
-    /// double quote, backtick, and opening square bracket.
-    pub quote_style: Option<char>,
-    /// The span of the identifier in the original SQL string.
-    pub span: String,
-}
-#[cfg(not(feature = "sql"))]
-impl Display for Ident {
-    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
-        write!(f, "[{}]", self.value)
-    }
-}
-
-#[derive(Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]
-#[cfg(not(feature = "sql"))]
-pub struct ReplaceSelectElement {
-    pub expr: String,
-    pub column_name: Ident,
-    pub as_keyword: bool,
-}
-#[cfg(not(feature = "sql"))]
-impl Display for ReplaceSelectElement {
-    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
-        if self.as_keyword {
-            write!(f, "{} AS {}", self.expr, self.column_name)
-        } else {
-            write!(f, "{} {}", self.expr, self.column_name)
-        }
-    }
-}
-
 /// Additional options for wildcards, e.g. Snowflake `EXCLUDE`/`RENAME` and Bigquery `EXCEPT`.
 #[derive(Clone, PartialEq, Eq, PartialOrd, Hash, Debug, Default)]
 pub struct WildcardOptions {

datafusion/expr/src/lib.rs

@@ -80,6 +80,8 @@ pub mod statistics {
 mod predicate_bounds;
 pub mod preimage;
 pub mod ptr_eq;
+#[cfg(not(feature = "sql"))]
+pub mod sql;
 pub mod test;
 pub mod tree_node;
 pub mod type_coercion;

datafusion/expr/src/logical_plan/ddl.rs

@@ -24,9 +24,9 @@ use std::{
     hash::{Hash, Hasher},
 };
 
-#[cfg(not(feature = "sql"))]
-use crate::expr::Ident;
 use crate::expr::Sort;
+#[cfg(not(feature = "sql"))]
+use crate::sql::Ident;
 use arrow::datatypes::DataType;
 use datafusion_common::tree_node::{Transformed, TreeNodeContainer, TreeNodeRecursion};
 use datafusion_common::{