Merge pull request #416 from lincc-frameworks/no-fixed-lists

Hot fix of the fixed-length list-column reading

Author: hombit

src/nested_pandas/series/utils.py

@@ -32,8 +32,10 @@ def struct_fields(struct_type: pa.StructType) -> list[pa.Field]:
 def is_pa_type_a_list(pa_type: pa.DataType) -> bool:
     """Check if the given pyarrow type is a list type.
 
-    I.e., one of the following types: ListArray, LargeListArray,
-    FixedSizeListArray.
+    Currently, it has to be ListArray, but we plan to support
+    FixedSizeListArray and LargeListArray as well:
+    https://github.com/lincc-frameworks/nested-pandas/issues/276
+    https://github.com/lincc-frameworks/nested-pandas/issues/95
 
     Parameters
     ----------
@@ -45,9 +47,7 @@ def is_pa_type_a_list(pa_type: pa.DataType) -> bool:
     bool
         True if the given type is a list type, False otherwise.
     """
-    return (
-        pa.types.is_list(pa_type) or pa.types.is_large_list(pa_type) or pa.types.is_fixed_size_list(pa_type)
-    )
+    return pa.types.is_list(pa_type)
 
 
 def is_pa_type_is_list_struct(pa_type: pa.DataType) -> bool:

tests/fixed_length_data/mmu-desi.parquet

@@ -473,3 +473,9 @@ def test_normal_loading_error():
     # Load in the example file
     with pytest.raises(ValueError, match="No match for*"):
         read_parquet("tests/test_data/nested.parquet", columns=["not_a_column"])
+
+
+def test_read_parquet_with_fixed_length_nested():
+    """Test reading a parquet file with fixed-length nested arrays"""
+    nf = read_parquet("tests/fixed_length_data/mmu-desi.parquet")
+    assert nf.shape == (2, 18)