Merge branch 'main' into feat_migrate_ffi_to_stabby

Author: coderfender

.github/workflows/stale.yml

@@ -37,3 +37,4 @@ jobs:
           days-before-issue-stale: -1
           days-before-issue-close: -1
           repo-token: ${{ secrets.GITHUB_TOKEN }}
+          operations-per-run: 150

Cargo.lock

@@ -190,7 +190,7 @@ recursive = "0.1.1"
 regex = "1.12"
 rstest = "0.26.1"
 serde_json = "1"
-sha2 = "^0.10.9"
+sha2 = "^0.11.0"
 sqlparser = { version = "0.61.0", default-features = false, features = ["std", "visitor"] }
 strum = "0.28.0"
 strum_macros = "0.28.0"

Cargo.toml

@@ -25,6 +25,7 @@ homepage = { workspace = true }
 repository = { workspace = true }
 license = { workspace = true }
 rust-version = { workspace = true }
+publish = false
 
 # Note: add additional linter rules in lib.rs.
 # Rust does not support workspace + new linter rules in subcrates yet

benchmarks/Cargo.toml

@@ -124,6 +124,10 @@ impl Highlighter for CliHelper {
     fn highlight_char(&self, line: &str, pos: usize, kind: CmdKind) -> bool {
         self.highlighter.highlight_char(line, pos, kind)
     }
+
+    fn highlight_hint<'h>(&self, hint: &'h str) -> Cow<'h, str> {
+        Color::gray(hint).into()
+    }
 }
 
 impl Hinter for CliHelper {
@@ -134,7 +138,7 @@ impl Hinter for CliHelper {
             self.show_hint.set(false);
         }
         (self.show_hint.get() && line.trim().is_empty())
-            .then(|| Color::gray(DEFAULT_HINT_SUGGESTION))
+            .then(|| DEFAULT_HINT_SUGGESTION.to_owned())
     }
 }
 

datafusion-cli/src/helper.rs

@@ -81,7 +81,7 @@ datafusion-macros = { workspace = true }
 hex = { workspace = true, optional = true }
 itertools = { workspace = true }
 log = { workspace = true }
-md-5 = { version = "^0.10.0", optional = true }
+md-5 = { version = "^0.11.0", optional = true }
 memchr = { workspace = true }
 num-traits = { workspace = true }
 rand = { workspace = true }

datafusion/functions/Cargo.toml

@@ -19,7 +19,7 @@
 
 use arrow::array::{Array, ArrayRef, AsArray, BinaryArray, BinaryArrayType};
 use arrow::datatypes::DataType;
-use blake2::{Blake2b512, Blake2s256, Digest};
+use blake2::{Blake2b512, Blake2s256};
 use blake3::Hasher as Blake3;
 
 use arrow::compute::StringArrayType;
@@ -85,7 +85,8 @@ impl fmt::Display for DigestAlgorithm {
 }
 
 macro_rules! digest_to_array {
-    ($METHOD:ident, $INPUT:expr) => {{
+    ($MODULE:ident, $METHOD:ident, $INPUT:expr) => {{
+        use $MODULE::Digest;
         let binary_array: BinaryArray = $INPUT
             .iter()
             .map(|x| x.map(|x| $METHOD::digest(x)))
@@ -95,20 +96,23 @@ macro_rules! digest_to_array {
 }
 
 macro_rules! digest_to_scalar {
-    ($METHOD: ident, $INPUT:expr) => {{ ScalarValue::Binary($INPUT.map(|v| $METHOD::digest(v).as_slice().to_vec())) }};
+    ($MODULE: ident, $METHOD: ident, $INPUT:expr) => {{
+        use $MODULE::Digest;
+        ScalarValue::Binary($INPUT.map(|v| $METHOD::digest(v).as_slice().to_vec()))
+    }};
 }
 
 impl DigestAlgorithm {
     /// digest an optional string to its hash value, null values are returned as is
     fn digest_scalar(self, value: Option<&[u8]>) -> ColumnarValue {
         ColumnarValue::Scalar(match self {
-            Self::Md5 => digest_to_scalar!(Md5, value),
-            Self::Sha224 => digest_to_scalar!(Sha224, value),
-            Self::Sha256 => digest_to_scalar!(Sha256, value),
-            Self::Sha384 => digest_to_scalar!(Sha384, value),
-            Self::Sha512 => digest_to_scalar!(Sha512, value),
-            Self::Blake2b => digest_to_scalar!(Blake2b512, value),
-            Self::Blake2s => digest_to_scalar!(Blake2s256, value),
+            Self::Md5 => digest_to_scalar!(md5, Md5, value),
+            Self::Sha224 => digest_to_scalar!(sha2, Sha224, value),
+            Self::Sha256 => digest_to_scalar!(sha2, Sha256, value),
+            Self::Sha384 => digest_to_scalar!(sha2, Sha384, value),
+            Self::Sha512 => digest_to_scalar!(sha2, Sha512, value),
+            Self::Blake2b => digest_to_scalar!(blake2, Blake2b512, value),
+            Self::Blake2s => digest_to_scalar!(blake2, Blake2s256, value),
             Self::Blake3 => ScalarValue::Binary(value.map(|v| {
                 let mut digest = Blake3::default();
                 digest.update(v);
@@ -125,13 +129,13 @@ impl DigestAlgorithm {
         StringArrType: StringArrayType<'a>,
     {
         match self {
-            Self::Md5 => digest_to_array!(Md5, input_value),
-            Self::Sha224 => digest_to_array!(Sha224, input_value),
-            Self::Sha256 => digest_to_array!(Sha256, input_value),
-            Self::Sha384 => digest_to_array!(Sha384, input_value),
-            Self::Sha512 => digest_to_array!(Sha512, input_value),
-            Self::Blake2b => digest_to_array!(Blake2b512, input_value),
-            Self::Blake2s => digest_to_array!(Blake2s256, input_value),
+            Self::Md5 => digest_to_array!(md5, Md5, input_value),
+            Self::Sha224 => digest_to_array!(sha2, Sha224, input_value),
+            Self::Sha256 => digest_to_array!(sha2, Sha256, input_value),
+            Self::Sha384 => digest_to_array!(sha2, Sha384, input_value),
+            Self::Sha512 => digest_to_array!(sha2, Sha512, input_value),
+            Self::Blake2b => digest_to_array!(blake2, Blake2b512, input_value),
+            Self::Blake2s => digest_to_array!(blake2, Blake2s256, input_value),
             Self::Blake3 => {
                 let binary_array: BinaryArray = input_value
                     .iter()
@@ -156,13 +160,13 @@ impl DigestAlgorithm {
         BinaryArrType: BinaryArrayType<'a>,
     {
         match self {
-            Self::Md5 => digest_to_array!(Md5, input_value),
-            Self::Sha224 => digest_to_array!(Sha224, input_value),
-            Self::Sha256 => digest_to_array!(Sha256, input_value),
-            Self::Sha384 => digest_to_array!(Sha384, input_value),
-            Self::Sha512 => digest_to_array!(Sha512, input_value),
-            Self::Blake2b => digest_to_array!(Blake2b512, input_value),
-            Self::Blake2s => digest_to_array!(Blake2s256, input_value),
+            Self::Md5 => digest_to_array!(md5, Md5, input_value),
+            Self::Sha224 => digest_to_array!(sha2, Sha224, input_value),
+            Self::Sha256 => digest_to_array!(sha2, Sha256, input_value),
+            Self::Sha384 => digest_to_array!(sha2, Sha384, input_value),
+            Self::Sha512 => digest_to_array!(sha2, Sha512, input_value),
+            Self::Blake2b => digest_to_array!(blake2, Blake2b512, input_value),
+            Self::Blake2s => digest_to_array!(blake2, Blake2s256, input_value),
             Self::Blake3 => {
                 let binary_array: BinaryArray = input_value
                     .iter()

datafusion/functions/src/crypto/basic.rs

@@ -604,7 +604,17 @@ impl StringViewArrayBuilder {
         let offset: u32 = i32::try_from(self.in_progress.len())
             .expect("offset exceeds i32::MAX") as u32;
         self.in_progress.extend_from_slice(v);
-        self.views.push(make_view(v, buffer_index, offset));
+
+        // Build the ByteView inline rather than going through `make_view`,
+        // which is marked as `[inline(never)]`.
+        let view = ByteView {
+            length,
+            // SAFETY: length > 12 here, so v has at least 4 bytes.
+            prefix: u32::from_le_bytes(v[0..4].try_into().unwrap()),
+            buffer_index,
+            offset,
+        };
+        self.views.push(view.into());
     }
 
     /// Append an empty placeholder row. The corresponding slot must be

datafusion/functions/src/strings.rs

@@ -17,13 +17,11 @@
 
 use std::sync::Arc;
 
-use arrow::array::{
-    Array, ArrayRef, GenericStringArray, GenericStringBuilder, OffsetSizeTrait,
-    StringViewBuilder,
-};
+use arrow::array::{Array, ArrayRef, GenericStringArray, OffsetSizeTrait};
 use arrow::buffer::{Buffer, OffsetBuffer};
 use arrow::datatypes::DataType;
 
+use crate::strings::{GenericStringArrayBuilder, StringViewArrayBuilder};
 use crate::utils::{make_scalar_function, utf8_to_str_type};
 use datafusion_common::cast::{as_generic_string_array, as_string_view_array};
 use datafusion_common::types::logical_string;
@@ -157,21 +155,35 @@ fn initcap<T: OffsetSizeTrait>(args: &[ArrayRef]) -> Result<ArrayRef> {
         return Ok(initcap_ascii_array(string_array));
     }
 
-    let mut builder = GenericStringBuilder::<T>::with_capacity(
-        string_array.len(),
+    let len = string_array.len();
+    let mut builder = GenericStringArrayBuilder::<T>::with_capacity(
+        len,
         string_array.value_data().len(),
     );
 
     let mut container = String::new();
-    string_array.iter().for_each(|str| match str {
-        Some(s) => {
+    let nulls = string_array.nulls().cloned();
+    if let Some(ref n) = nulls {
+        for i in 0..len {
+            if n.is_null(i) {
+                builder.append_placeholder();
+            } else {
+                // SAFETY: not null per check above.
+                let s = unsafe { string_array.value_unchecked(i) };
+                initcap_string(s, &mut container);
+                builder.append_value(&container);
+            }
+        }
+    } else {
+        for i in 0..len {
+            // SAFETY: no null buffer means every index is valid.
+            let s = unsafe { string_array.value_unchecked(i) };
             initcap_string(s, &mut container);
             builder.append_value(&container);
         }
-        None => builder.append_null(),
-    });
+    }
 
-    Ok(Arc::new(builder.finish()) as ArrayRef)
+    Ok(Arc::new(builder.finish(nulls)?) as ArrayRef)
 }
 
 /// Fast path for `Utf8` or `LargeUtf8` arrays that are ASCII-only. We can use a
@@ -232,18 +244,32 @@ fn initcap_ascii_array<T: OffsetSizeTrait>(
 
 fn initcap_utf8view(args: &[ArrayRef]) -> Result<ArrayRef> {
     let string_view_array = as_string_view_array(&args[0])?;
-    let mut builder = StringViewBuilder::with_capacity(string_view_array.len());
+    let len = string_view_array.len();
+    let mut builder = StringViewArrayBuilder::with_capacity(len);
     let mut container = String::new();
 
-    string_view_array.iter().for_each(|str| match str {
-        Some(s) => {
+    let nulls = string_view_array.nulls().cloned();
+    if let Some(ref n) = nulls {
+        for i in 0..len {
+            if n.is_null(i) {
+                builder.append_placeholder();
+            } else {
+                // SAFETY: not null per check above.
+                let s = unsafe { string_view_array.value_unchecked(i) };
+                initcap_string(s, &mut container);
+                builder.append_value(&container);
+            }
+        }
+    } else {
+        for i in 0..len {
+            // SAFETY: no null buffer means every index is valid.
+            let s = unsafe { string_view_array.value_unchecked(i) };
             initcap_string(s, &mut container);
             builder.append_value(&container);
         }
-        None => builder.append_null(),
-    });
+    }
 
-    Ok(Arc::new(builder.finish()) as ArrayRef)
+    Ok(Arc::new(builder.finish(nulls)?) as ArrayRef)
 }
 
 fn initcap_string(input: &str, container: &mut String) {