Merge branch 'main' into implement_additional_benchmarks_hashjoin

Author: coderfender

Cargo.toml

@@ -104,8 +104,11 @@ arrow-data = { version = "58.1.0", default-features = false }
 arrow-flight = { version = "58.1.0", features = [
     "flight-sql-experimental",
 ] }
+# Both codecs are required here to make sure that code paths like
+# file-spilling have access to all compression codecs.
 arrow-ipc = { version = "58.1.0", default-features = false, features = [
     "lz4",
+    "zstd",
 ] }
 arrow-ord = { version = "58.1.0", default-features = false }
 arrow-schema = { version = "58.1.0", default-features = false }

datafusion/datasource-arrow/Cargo.toml

@@ -62,6 +62,6 @@ name = "datafusion_datasource_arrow"
 path = "src/mod.rs"
 
 [features]
-compression = [
-    "arrow-ipc/zstd",
-]
+# This feature is deprecated, as core functionality in the SpillManager requires all features
+# it enabled, and will be removed in a future version.
+compression = []

datafusion/datasource-arrow/src/mod.rs

@@ -21,6 +21,10 @@
 #![cfg_attr(not(test), deny(clippy::clone_on_ref_ptr))]
 
 //! [`ArrowFormat`]: Apache Arrow file format abstractions
+//!
+//! Note: As of DataFusion 54.0.0, the `compression` feature of this crate
+//! is a no-op, only kept for backwards compatibility purposes, and it will
+//! be removed in a future release.
 
 pub mod file_format;
 pub mod source;

datafusion/sqllogictest/test_files/array/array_union.slt

@@ -114,11 +114,10 @@ select array_union(arrow_cast([], 'LargeList(Int64)'), arrow_cast([], 'LargeList
 []
 
 # array_union scalar function #7
-# re-enable when https://github.com/apache/arrow-rs/issues/9227 is fixed
-# query ?
-# select array_union([[null]], []);
-# ----
-# [[]]
+query ?
+select array_union([[null]], []);
+----
+[[NULL]]
 
 query error DataFusion error: Error during planning: Failed to coerce arguments to satisfy a call to 'array_union' function:
 select array_union(arrow_cast([[null]], 'LargeList(List(Int64))'), arrow_cast([], 'LargeList(Int64)'));