Support dask 2025.4.0

Dask 2025.4.0 optimizes multiple DataFrames together, which exposes division mismatches and causes dask to throw an error. This change removes a source of repartitioning, preserving the divisions when assigning a pandas series to a dask dataframe

Author: filippsatverily

cdisc_rules_engine/models/dataset/dask_dataset.py

@@ -81,9 +81,9 @@ def __setitem__(self, key, value):
             array_values = da.from_array(value, chunks=tuple(chunks))
             self._data[key] = array_values
         elif isinstance(value, pd.Series):
-            self._data = self._data.reset_index()
-            self._data = self._data.set_index("index")
-            self._data[key] = value
+            chunks = self._data.map_partitions(lambda x: len(x)).compute().to_numpy()
+            array_values = da.from_array(value.values, chunks=tuple(chunks))
+            self._data[key] = array_values
         elif isinstance(value, dd.DataFrame):
             for column in value:
                 self._data[column] = value[column]

pyproject.toml

@@ -15,7 +15,7 @@ dependencies = [
     "cachetools >=6.1.0",
     "cdisc-library-client >=0.1.6",
     "click >=8.1.7",
-    "dask[dataframe,array] >=2024.6.0, <2025.4.0",
+    "dask[dataframe,array] >=2024.6.0",
     "fastparquet >=2024.2.0",
     "importlib-metadata >=8.5.0",
     "jsonata-python >=0.6.0",