feat: Support IEEE 754 for SQL ops

Author: comphead

datafusion/common/src/utils/mod.rs

@@ -1395,24 +1395,55 @@ fn fsl_values_row_number(list_size: i32, array_len: usize) -> Result<Int32Array>
 /// semantics, which treats `-0.0` and `+0.0` as distinct. SQL semantics
 /// (PostgreSQL / IEEE 754 equality) require them to compare equal, so
 /// callers normalize before invoking those kernels.
+///
+/// The common case — no `-0.0` present — is allocation-free: a single
+/// read-only scan of the underlying buffer (auto-vectorizable to an
+/// OR-reduction) decides whether to fall through to the rewriting path.
+/// Only arrays that actually contain `-0.0` pay for a new buffer.
 pub fn normalize_float_zero(array: &ArrayRef) -> ArrayRef {
     use arrow::array::{Float16Array, Float32Array, Float64Array};
     use arrow::datatypes::{Float16Type, Float32Type, Float64Type};
+    // -0.0 has only the sign bit set; no other finite or NaN value shares
+    // this bit pattern, so a strict-equality scan reliably gates the rewrite.
+    const NEG_ZERO_F16_BITS: u16 = half::f16::NEG_ZERO.to_bits();
+    const NEG_ZERO_F32_BITS: u32 = (-0.0_f32).to_bits();
+    const NEG_ZERO_F64_BITS: u64 = (-0.0_f64).to_bits();
     match array.data_type() {
         DataType::Float32 => {
             let arr: &Float32Array = array.as_primitive::<Float32Type>();
+            if !arr
+                .values()
+                .iter()
+                .any(|v| v.to_bits() == NEG_ZERO_F32_BITS)
+            {
+                return Arc::clone(array);
+            }
             let normalized: Float32Array =
                 arr.unary(|v| if v.to_bits() << 1 == 0 { 0.0_f32 } else { v });
             Arc::new(normalized)
         }
         DataType::Float64 => {
             let arr: &Float64Array = array.as_primitive::<Float64Type>();
+            if !arr
+                .values()
+                .iter()
+                .any(|v| v.to_bits() == NEG_ZERO_F64_BITS)
+            {
+                return Arc::clone(array);
+            }
             let normalized: Float64Array =
                 arr.unary(|v| if v.to_bits() << 1 == 0 { 0.0_f64 } else { v });
             Arc::new(normalized)
         }
         DataType::Float16 => {
             let arr: &Float16Array = array.as_primitive::<Float16Type>();
+            if !arr
+                .values()
+                .iter()
+                .any(|v| v.to_bits() == NEG_ZERO_F16_BITS)
+            {
+                return Arc::clone(array);
+            }
             let normalized: Float16Array = arr.unary(|v| {
                 if v.to_bits() << 1 == 0 {
                     half::f16::from_bits(0)

datafusion/physical-plan/src/joins/utils.rs

@@ -2262,7 +2262,16 @@ impl JoinKeyComparator {
             .zip(right_arrays.iter())
             .zip(sort_options.iter())
             .map(|((l, r), opts)| {
-                let inner = make_comparator(l.as_ref(), r.as_ref(), *opts)?;
+                // `make_comparator` uses IEEE 754 totalOrder for floats and
+                // treats `-0.0` / `+0.0` as distinct. Normalize float arrays
+                // so SMJ / piecewise-merge equi-keys honor SQL equality;
+                // no-op (Arc::clone) for non-floats and for float arrays
+                // that contain no `-0.0`. `normalize_float_zero` preserves
+                // null positions, so the original null masks below remain
+                // valid.
+                let l_norm = normalize_float_zero(l);
+                let r_norm = normalize_float_zero(r);
+                let inner = make_comparator(l_norm.as_ref(), r_norm.as_ref(), *opts)?;
                 if null_equality == NullEquality::NullEqualsNothing {
                     let ln = l.logical_nulls().filter(|n| n.null_count() > 0);
                     let rn = r.logical_nulls().filter(|n| n.null_count() > 0);

datafusion/sqllogictest/test_files/negative_zero.slt

@@ -204,3 +204,28 @@ FROM (SELECT 0.0 AS a) t1
 JOIN (SELECT -0.0 AS b) t2 ON t1.a = t2.b;
 ----
 0 0
+
+# Sort-merge join must also match +0.0 against -0.0. SMJ builds equi-key
+# matchers via `JoinKeyComparator`, which calls Arrow's `make_comparator`
+# (IEEE 754 totalOrder); without normalization, +0.0 and -0.0 produce
+# different orderings and miss the match.
+statement ok
+set datafusion.optimizer.prefer_hash_join = false;
+
+query RR
+SELECT t1.a, t2.b
+FROM (SELECT 0.0 AS a) t1
+JOIN (SELECT -0.0 AS b) t2 ON t1.a = t2.b;
+----
+0 0
+
+# Float32 SMJ equi-join.
+query RR
+SELECT t1.a, t2.b
+FROM (SELECT arrow_cast(0.0, 'Float32') AS a) t1
+JOIN (SELECT arrow_cast(-0.0, 'Float32') AS b) t2 ON t1.a = t2.b;
+----
+0 0
+
+statement ok
+reset datafusion.optimizer.prefer_hash_join;