Relax dependency constraints (#1713)

* Define dependencies in pyproject.toml
Moves dependency constraints to pyproject.toml.
Makes requirements.txt a lockfile.

* Support click 8.3.0
Fixes an incompatibility caused by click 8.3.0, which passes the default value as-is.

* Support pyreadstat 1.2.9
Fixes an incompatibility caused by pyreadstat 1.2.9, which changed original_variable_type from 'NULL' to None

* Support jsonpath-ng 1.8.0
Works around an behavior change in jsonpath-ng 1.8.0 where Child.str gets wrapped in parenthesis.

* Suport dask 2024.8.1

Fixes tokenization errors when using dask 2024.8.1+. Starting with this
version, dask enforces that tokens remain stable across pickle
round-trips (dask/dask#11320). Capturing self in a lambda fails this
check because instance objects can have non-deterministic pickle
representations. Since calculate_variable_value_length is already a
static method, replacing self with the class name is enough to remove
the capture.

* Support dask 2024.12.1

Fixes an import error caused by dask 2024.12.1, which removed the
legacy dask.dataframe.dd submodule (dask/dask#11604). Changes the
import to `import dask.dataframe as dd`, consistent with every other
file in the codebase.

* Support dask 2025.4.0

Dask 2025.4.0 optimizes multiple DataFrames together, which exposes
division mismatches when assigning a pandas Series to a dask DataFrame
column. The old reset_index/set_index workaround no longer avoids this.
Replacing it with compute-assign-rewrap via dd.from_pandas, which builds
a clean expression graph. This is safe because __getitem__ already
computes the DataFrame to produce the Series being assigned.

* Support pandas 2.2.0

Fixes a unit test to support pandas 2.2.0+. The pandas release fixes a
sorting bug with pandas-dev/pandas#54611. This
commit changes the expected results accordingly.

Also fixes a merge type mismatch introduced by upstream #1709: the
codelist metadata side was cast to StringDtype but the evaluation
dataset side was not. With pandas 2.2.0, empty columns infer as float64,
and merging float64 with string is rejected. Casting both sides to
string before the merge resolves this.

---------

Co-authored-by: Samuel Johnson <96841389+SFJohnson24@users.noreply.github.com>

Author: filippsatverily

.github/workflows/lint-format.yml

@@ -49,7 +49,7 @@ jobs:
           python-version: "3.12"
       - name: Install linters
         run: |
-          pip install black flake8 -c requirements-dev.txt
+          pip install black flake8
       - name: Run flake8
         run: |
           flake8 ${{needs.get_changed_files.outputs.py}} --count --select=E9,F63,F7,F82 --show-source --statistics

.github/workflows/test-unit.yml

@@ -16,8 +16,8 @@ jobs:
           python-version: "3.12"
       - name: Install requirements
         run: |
-          pip install -r requirements-dev.txt
-          pip install -e .
+          pip install -r requirements.txt
+          pip install --group dev -e .
       - name: Running Tests
         env:
           CDISC_LIBRARY_API_KEY: fakekey12341234

cdisc_rules_engine/dataset_builders/contents_define_vlm_dataset_builder.py

@@ -66,7 +66,7 @@ def build(self):
         data_contents_with_vlm["variable_value_length"] = data_contents_with_vlm.data[
             ["variable_value", "define_vlm_data_type"]
         ].apply(
-            lambda row: self.calculate_variable_value_length(
+            lambda row: ValuesDatasetBuilder.calculate_variable_value_length(
                 row["variable_value"], row["define_vlm_data_type"]
             ),
             axis=1,

cdisc_rules_engine/models/dataset/dask_dataset.py

@@ -81,9 +81,9 @@ def __setitem__(self, key, value):
             array_values = da.from_array(value, chunks=tuple(chunks))
             self._data[key] = array_values
         elif isinstance(value, pd.Series):
-            self._data = self._data.reset_index()
-            self._data = self._data.set_index("index")
-            self._data[key] = value
+            pdf = self._data.compute()
+            pdf[key] = value.values
+            self._data = dd.from_pandas(pdf, npartitions=1)
         elif isinstance(value, dd.DataFrame):
             for column in value:
                 self._data[column] = value[column]

cdisc_rules_engine/operations/codelist_extensible.py

@@ -32,6 +32,10 @@ def _handle_multiple_versions(self) -> pd.Series:
                 "codelist_code": "string",
             }
         )
+        cast_cols = {self.params.ct_version: "string"}
+        if self.params.codelist_code in self.evaluation_dataset.columns:
+            cast_cols[self.params.codelist_code] = "string"
+        self.evaluation_dataset = self.evaluation_dataset.astype(cast_cols)
         if self.params.codelist_code in self.evaluation_dataset.columns:
             is_extensible = self.evaluation_dataset.merge(
                 ct_df.data,

cdisc_rules_engine/operations/codelist_terms.py

@@ -64,6 +64,10 @@ def _handle_multiple_versions(self) -> pd.Series:
                 "codelist_code": "string",
             }
         )
+        cast_cols = {self.params.ct_version: "string"}
+        if self.params.codelist_code in self.evaluation_dataset.columns:
+            cast_cols[self.params.codelist_code] = "string"
+        self.evaluation_dataset = self.evaluation_dataset.astype(cast_cols)
         if self.params.codelist_code in self.evaluation_dataset.columns:
             result = self.evaluation_dataset.merge(
                 ct_df.data,

cdisc_rules_engine/services/data_readers/csv_reader.py

@@ -1,6 +1,6 @@
 import tempfile
 
-from dask.dataframe import dd
+import dask.dataframe as dd
 
 from cdisc_rules_engine.exceptions.custom_exceptions import InvalidCSVFile
 from cdisc_rules_engine.interfaces import DataReaderInterface

cdisc_rules_engine/services/data_services/usdm_data_service.py

@@ -399,8 +399,19 @@ def __read_node_metadata(
         }
 
     @staticmethod
-    def __get_full_path(node: DatumInContext):
-        return f"{node.full_path}".replace(".[", "[")
+    def __get_full_path(node: DatumInContext) -> str:
+        parts = []
+        current = node
+        while current is not None and current.context is not None:
+            parts.append(str(current.path))
+            current = current.context
+        result = ""
+        for part in reversed(parts):
+            if part.startswith("["):
+                result += part
+            else:
+                result = (result + "." if result else "") + part
+        return result
 
     def __get_datasets_content_index(self) -> List[dict]:
         """

cdisc_rules_engine/services/datasetxpt_metadata_reader.py

@@ -61,7 +61,7 @@ def read(self) -> dict:
             "variable_labels": list(metadata.column_labels),
             "variable_names": list(metadata.column_names),
             "variable_formats": [
-                "" if data_type == "NULL" else data_type
+                "" if (data_type == "NULL" or data_type is None) else data_type
                 for data_type in metadata.original_variable_types.values()
             ],
             "variable_name_to_label_map": metadata.column_names_to_labels,

core.py

@@ -357,15 +357,13 @@ def load_custom_dotenv_from_data_options(_ctx, _param, value):
     "-s",
     "--standard",
     required=True,
-    default=None,
     help="CDISC standard to validate against",
     envvar="PRODUCT",
 )
 @click.option(
     "-v",
     "--version",
     required=True,
-    default=None,
     help="Standard version to validate against",
     envvar="VERSION",
 )