Check arrow for nulls (#629)

Author: hombit

CHANGELOG.md

@@ -25,7 +25,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
---
+- `.many(arrow_array)` now checks for nulls, and raises an arrow if any. Previously we used all the values, even masked, which may cause unexpected results https://github.com/light-curve/light-curve-python/pull/629
 
 ### Security
 

light-curve/src/features.rs

@@ -8,6 +8,7 @@ use crate::ln_prior::LnPrior1D;
 use crate::np_array::Arr;
 use crate::transform::{StockTransformer, parse_transform};
 
+use arrow_array::Array;
 use arrow_array::cast::AsArray;
 use const_format::formatcp;
 use conv::ConvUtil;
@@ -593,13 +594,31 @@ impl PyFeatureEvaluator {
         fill_value: Option<T>,
         n_jobs: i64,
     ) -> Res<ndarray::Array2<T>> {
-        let chunks = chunked.chunks();
-
-        let tss = chunks
+        let tss = chunked
+            .chunks()
             .iter()
-            .flat_map(|chunk| {
-                let list = chunk.as_list::<O>();
+            .map(|chunk| {
+                let list: &arrow_array::GenericListArray<O> = chunk.as_list::<O>();
+                // O(1) null checks — nulls are not supported yet
+                if list.null_count() > 0 {
+                    return Err(Exception::NotImplementedError(
+                        "Null entries in the list array are not supported".to_string(),
+                    ));
+                }
                 let struct_arr = list.values().as_struct();
+                if struct_arr.null_count() > 0 {
+                    return Err(Exception::NotImplementedError(
+                        "Null entries in the struct array are not supported".to_string(),
+                    ));
+                }
+                for i in 0..struct_arr.num_columns() {
+                    if struct_arr.column(i).null_count() > 0 {
+                        return Err(Exception::NotImplementedError(
+                            "Null values in data columns are not supported".to_string(),
+                        ));
+                    }
+                }
+
                 let t_vals: &[T] = struct_arr
                     .column(0)
                     .as_primitive::<T::ArrowType>()
@@ -618,19 +637,24 @@ impl PyFeatureEvaluator {
                         .as_ref()
                 });
                 let offsets = list.value_offsets();
-                offsets.iter().tuple_windows().map(move |(&start, &end)| {
-                    let (start, end) = (start.as_usize(), end.as_usize());
-                    Self::ts_from_views(
-                        feature_evaluator,
-                        ndarray::ArrayView1::from(&t_vals[start..end]),
-                        ndarray::ArrayView1::from(&m_vals[start..end]),
-                        sigma_vals.map(|s| ndarray::ArrayView1::from(&s[start..end])),
-                        sorted,
-                        check,
-                        is_t_required,
-                    )
-                })
+                offsets
+                    .iter()
+                    .tuple_windows()
+                    .map(move |(&start, &end)| {
+                        let (start, end) = (start.as_usize(), end.as_usize());
+                        Self::ts_from_views(
+                            feature_evaluator,
+                            ndarray::ArrayView1::from(&t_vals[start..end]),
+                            ndarray::ArrayView1::from(&m_vals[start..end]),
+                            sigma_vals.map(|s| ndarray::ArrayView1::from(&s[start..end])),
+                            sorted,
+                            check,
+                            is_t_required,
+                        )
+                    })
+                    .collect::<Res<Vec<_>>>()
             })
+            .flatten_ok()
             .collect::<Res<Vec<_>>>()?;
 
         if tss.is_empty() {

light-curve/tests/light_curve_ext/test_feature.py

@@ -668,3 +668,44 @@ def test_many_arro3():
     arro3_arr = arro3.core.Array.from_arrow(arrow_arr)
     result = feature.many(arro3_arr, sorted=True, n_jobs=1)
     assert_array_equal(expected, result)
+
+
+@pytest.mark.parametrize(
+    "arrow_arr, match_msg",
+    [
+        # Null list entry (whole light curve is null)
+        (
+            pa.array(
+                [[{"t": 1.0, "m": 2.0, "sigma": 0.1}], None],
+                type=pa.list_(pa.struct([("t", pa.float64()), ("m", pa.float64()), ("sigma", pa.float64())])),
+            ),
+            "list array",
+        ),
+        # Null struct entry (observation within a light curve is null)
+        (
+            pa.ListArray.from_arrays(
+                pa.array([0, 2], type=pa.int32()),
+                pa.StructArray.from_arrays(
+                    [pa.array([1.0, 2.0]), pa.array([3.0, 4.0]), pa.array([0.1, 0.2])],
+                    names=["t", "m", "sigma"],
+                    mask=pa.array([False, True]),
+                ),
+            ),
+            "struct array",
+        ),
+        # Null value in a data column
+        (
+            pa.array(
+                [[{"t": 1.0, "m": None, "sigma": 0.1}, {"t": 2.0, "m": 5.0, "sigma": 0.2}]],
+                type=pa.list_(pa.struct([("t", pa.float64()), ("m", pa.float64()), ("sigma", pa.float64())])),
+            ),
+            "data columns",
+        ),
+    ],
+    ids=["null_list_entry", "null_struct_entry", "null_value"],
+)
+def test_many_arrow_nulls_rejected(arrow_arr, match_msg):
+    """Null values at any level of the Arrow array raise NotImplementedError."""
+    feature = lc.Amplitude()
+    with pytest.raises(NotImplementedError, match=match_msg):
+        feature.many(arrow_arr, sorted=True, n_jobs=1)