feat: expose array_compact, array_normalize, cosine_distance, inner_product

Adds Python bindings for four scalar functions from
datafusion::functions_nested::expr_fn that were not previously surfaced:

- array_compact / list_compact: drop NULLs from an array.
- array_normalize / list_normalize: L2-normalize a numeric array.
- cosine_distance: 1 - cosine_similarity(a, b).
- inner_product: dot product of two numeric arrays.

Implementation routes each through the existing array_fn! macro in
crates/core/src/functions.rs, mirroring the other functions_nested
wrappers. Python wrappers in python/datafusion/functions.py follow the
established pattern with doctest examples; list_* aliases use the
one-line + See Also form per project convention.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: timsaucer

crates/core/src/functions.rs

@@ -654,6 +654,10 @@ array_fn!(array_replace, array from to);
 array_fn!(array_replace_n, array from to max);
 array_fn!(array_replace_all, array from to);
 array_fn!(array_sort, array desc null_first);
+array_fn!(array_compact, array);
+array_fn!(array_normalize, array);
+array_fn!(cosine_distance, array1 array2);
+array_fn!(inner_product, array1 array2);
 array_fn!(array_intersect, first_array second_array);
 array_fn!(array_union, array1 array2);
 array_fn!(array_except, first_array second_array);
@@ -1133,6 +1137,10 @@ pub(crate) fn init_module(m: &Bound<'_, PyModule>) -> PyResult<()> {
     m.add_wrapped(wrap_pyfunction!(array_cat))?;
     m.add_wrapped(wrap_pyfunction!(array_dims))?;
     m.add_wrapped(wrap_pyfunction!(array_distinct))?;
+    m.add_wrapped(wrap_pyfunction!(array_compact))?;
+    m.add_wrapped(wrap_pyfunction!(array_normalize))?;
+    m.add_wrapped(wrap_pyfunction!(cosine_distance))?;
+    m.add_wrapped(wrap_pyfunction!(inner_product))?;
     m.add_wrapped(wrap_pyfunction!(array_element))?;
     m.add_wrapped(wrap_pyfunction!(array_empty))?;
     m.add_wrapped(wrap_pyfunction!(array_length))?;

python/datafusion/functions.py

@@ -76,6 +76,7 @@
     "array_any_value",
     "array_append",
     "array_cat",
+    "array_compact",
     "array_concat",
     "array_contains",
     "array_dims",
@@ -96,6 +97,7 @@
     "array_max",
     "array_min",
     "array_ndims",
+    "array_normalize",
     "array_pop_back",
     "array_pop_front",
     "array_position",
@@ -151,6 +153,7 @@
     "corr",
     "cos",
     "cosh",
+    "cosine_distance",
     "cot",
     "count",
     "count_star",
@@ -192,6 +195,7 @@
     "ifnull",
     "in_list",
     "initcap",
+    "inner_product",
     "instr",
     "isnan",
     "iszero",
@@ -209,6 +213,7 @@
     "list_any_value",
     "list_append",
     "list_cat",
+    "list_compact",
     "list_concat",
     "list_contains",
     "list_dims",
@@ -229,6 +234,7 @@
     "list_max",
     "list_min",
     "list_ndims",
+    "list_normalize",
     "list_overlap",
     "list_pop_back",
     "list_pop_front",
@@ -3204,6 +3210,78 @@ def array_distinct(array: Expr) -> Expr:
     return Expr(f.array_distinct(array.expr))
 
 
+def array_compact(array: Expr) -> Expr:
+    """Removes NULL values from the array.
+
+    Examples:
+        >>> ctx = dfn.SessionContext()
+        >>> df = ctx.from_pydict({"a": [[1, None, 2, None, 3]]})
+        >>> result = df.select(
+        ...     dfn.functions.array_compact(dfn.col("a")).alias("result")
+        ... )
+        >>> result.collect_column("result")[0].as_py()
+        [1, 2, 3]
+    """
+    return Expr(f.array_compact(array.expr))
+
+
+def array_normalize(array: Expr) -> Expr:
+    """Returns the L2-normalized vector for a numeric array.
+
+    Examples:
+        >>> ctx = dfn.SessionContext()
+        >>> df = ctx.from_pydict({"a": [[3.0, 4.0]]})
+        >>> result = df.select(
+        ...     dfn.functions.array_normalize(dfn.col("a")).alias("result")
+        ... )
+        >>> result.collect_column("result")[0].as_py()
+        [0.6, 0.8]
+    """
+    return Expr(f.array_normalize(array.expr))
+
+
+def cosine_distance(array1: Expr, array2: Expr) -> Expr:
+    """Returns the cosine distance between two numeric arrays.
+
+    Computed as ``1 - cosine_similarity(array1, array2)``.
+
+    Examples:
+        >>> ctx = dfn.SessionContext()
+        >>> df = ctx.from_pydict(
+        ...     {"a": [[1.0, 2.0, 3.0]], "b": [[1.0, 2.0, 3.0]]}
+        ... )
+        >>> result = df.select(
+        ...     dfn.functions.cosine_distance(
+        ...         dfn.col("a"), dfn.col("b")
+        ...     ).alias("result")
+        ... )
+        >>> result.collect_column("result")[0].as_py()
+        0.0
+    """
+    return Expr(f.cosine_distance(array1.expr, array2.expr))
+
+
+def inner_product(array1: Expr, array2: Expr) -> Expr:
+    """Returns the inner (dot) product of two numeric arrays.
+
+    The SQL name ``dot_product`` is an alias for this function in raw SQL.
+
+    Examples:
+        >>> ctx = dfn.SessionContext()
+        >>> df = ctx.from_pydict(
+        ...     {"a": [[1.0, 2.0, 3.0]], "b": [[4.0, 5.0, 6.0]]}
+        ... )
+        >>> result = df.select(
+        ...     dfn.functions.inner_product(
+        ...         dfn.col("a"), dfn.col("b")
+        ...     ).alias("result")
+        ... )
+        >>> result.collect_column("result")[0].as_py()
+        32.0
+    """
+    return Expr(f.inner_product(array1.expr, array2.expr))
+
+
 def list_cat(*args: Expr) -> Expr:
     """Concatenates the input arrays.
 
@@ -3231,6 +3309,24 @@ def list_distinct(array: Expr) -> Expr:
     return array_distinct(array)
 
 
+def list_compact(array: Expr) -> Expr:
+    """Removes NULL values from the array.
+
+    See Also:
+        This is an alias for :py:func:`array_compact`.
+    """
+    return array_compact(array)
+
+
+def list_normalize(array: Expr) -> Expr:
+    """Returns the L2-normalized vector for a numeric array.
+
+    See Also:
+        This is an alias for :py:func:`array_normalize`.
+    """
+    return array_normalize(array)
+
+
 def list_dims(array: Expr) -> Expr:
     """Returns an array of the array's dimensions.