Base implementation of Parquet file writing (#2583)

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: VeckoTheGecko

.github/ci/recipe.yaml

@@ -36,6 +36,8 @@ requirements:
     - numpy >=2.1.0
     - tqdm >=4.50.0
     - xarray >=2025.8.0,<2026.4.0 # TODO: remove upper pin when https://github.com/UXARRAY/uxarray/issues/1490 is resolved
+    - pandas >=2.2
+    - pyarrow >=20.0.0
     - cf_xarray >=0.8.6
     - xgcm >=0.9.0
     - zarr >=2.15.0,!=2.18.0,<3

.gitignore

@@ -14,6 +14,7 @@ out-*
 *.pyc
 **/*.zarr/*
 .DS_Store
+*.parquet
 
 .vscode
 .env

docs/user_guide/v4-migration.md

@@ -36,7 +36,9 @@ Version 4 of Parcels is unreleased at the moment. The information in this migrat
 ## ParticleFile
 
 - Particlefiles should be created by `ParticleFile(...)` instead of `pset.ParticleFile(...)`
-- The `name` argument in `ParticleFile` has been replaced by `store` and can now be a string, a Path or a zarr store.
+- `ParticleFile` output is now in Parquet format
+- `ParticleFile` writing behaviour now errors out if there's existing output (this be being further discussed in https://github.com/Parcels-code/Parcels/issues/2593 )
+- A utility to read in ParticleFile output is now available. `parcels.read_particlefile()`
 
 ## Field
 

pixi.toml

@@ -24,6 +24,8 @@ netcdf4 = ">=1.6.0"
 numpy = ">=2.1.0"
 tqdm = ">=4.50.0"
 xarray = ">=2024.5.0,<2026.4.0" # TODO: remove upper pin when https://github.com/UXARRAY/uxarray/issues/1490 is resolved
+pandas = ">=2.2"
+pyarrow = ">=20.0.0"
 holoviews = ">=1.22.0" # https://github.com/prefix-dev/rattler-build/issues/2326
 uxarray = ">=2025.3.0"
 dask = ">=2024.5.1"
@@ -51,6 +53,8 @@ netcdf4 = "1.6.*"
 numpy = "2.1.*"
 tqdm = "4.50.*"
 xarray = "2025.8.*"
+pandas = "2.2.*"
+pyarrow = "20.0.*"
 uxarray = "2025.3.*"
 dask = "2024.6.*"
 zarr = "2.18.*"

pyproject.toml

@@ -27,6 +27,8 @@ dependencies = [
   "zarr >=2.15.0,!=2.18.0,<3",
   "tqdm >=4.50.0",
   "xarray >=2024.5.0,<2026.4.0", # TODO: remove upper pin when https://github.com/UXARRAY/uxarray/issues/1490 is resolved
+  "pandas >= 2.2",
+  "pyarrow >=20.0.0",
   "uxarray >=2025.3.0",
   "pooch >=1.8.0",
   "xgcm >=0.9.0",

src/parcels/__init__.py

@@ -11,7 +11,7 @@
 
 from parcels._core.fieldset import FieldSet
 from parcels._core.particleset import ParticleSet
-from parcels._core.particlefile import ParticleFile
+from parcels._core.particlefile import ParticleFile, read_particlefile
 from parcels._core.particle import (
     Variable,
     Particle,
@@ -67,6 +67,7 @@
     "ParticleSetWarning",
     # Utilities
     "logger",
+    "read_particlefile",
 ]
 
 _stdlib_warnings.warn(

src/parcels/_core/particle.py

@@ -8,7 +8,6 @@
 from parcels._compat import _attrgetter_helper
 from parcels._core.statuscodes import StatusCode
 from parcels._core.utils.string import _assert_str_and_python_varname
-from parcels._core.utils.time import TimeInterval
 from parcels._reprs import particleclass_repr, variable_repr
 
 __all__ = ["Particle", "ParticleClass", "Variable"]
@@ -149,7 +148,11 @@ def get_default_particle(spatial_dtype: type[np.float32] | type[np.float64]) ->
             Variable(
                 "time",
                 dtype=np.float64,
-                attrs={"standard_name": "time", "units": "seconds", "axis": "T"},
+                attrs={
+                    "standard_name": "time",
+                    "units": "seconds",
+                    "axis": "T",
+                },  # "units" and "calendar" gets updated/set if working with cftime time domain
             ),
             Variable(
                 "trajectory",
@@ -160,7 +163,6 @@ def get_default_particle(spatial_dtype: type[np.float32] | type[np.float64]) ->
                     "cf_role": "trajectory_id",
                 },
             ),
-            Variable("obs_written", dtype=np.int32, initial=0, to_write=False),
             Variable("dt", dtype=np.float64, initial=1.0, to_write=False),
             Variable("state", dtype=np.int32, initial=StatusCode.Evaluate, to_write=False),
         ]
@@ -176,7 +178,6 @@ def create_particle_data(
     pclass: ParticleClass,
     nparticles: int,
     ngrids: int,
-    time_interval: TimeInterval,
     initial: dict[str, np.ndarray] | None = None,
 ):
     if initial is None:
@@ -207,16 +208,9 @@ def create_particle_data(
             name_to_copy = var.initial(_attrgetter_helper)
             data[var.name] = data[name_to_copy].copy()
         else:
-            data[var.name] = _create_array_for_variable(var, nparticles, time_interval)
+            data[var.name] = np.full(
+                shape=(nparticles,),
+                fill_value=var.initial,
+                dtype=var.dtype,
+            )
     return data
-
-
-def _create_array_for_variable(variable: Variable, nparticles: int, time_interval: TimeInterval):
-    assert not isinstance(variable.initial, operator.attrgetter), (
-        "This function cannot handle attrgetter initial values."
-    )
-    return np.full(
-        shape=(nparticles,),
-        fill_value=variable.initial,
-        dtype=variable.dtype,
-    )