Filipps/update deps main

.github/workflows/lint-format.yml

@@ -49,7 +49,7 @@ jobs:
           python-version: "3.12"
       - name: Install linters
         run: |
-          pip install black flake8 -c requirements-dev.txt
+          pip install black flake8
       - name: Run flake8
         run: |
           flake8 ${{needs.get_changed_files.outputs.py}} --count --select=E9,F63,F7,F82 --show-source --statistics

.github/workflows/test-unit.yml

@@ -16,8 +16,8 @@ jobs:
           python-version: "3.12"
       - name: Install requirements
         run: |
-          pip install -r requirements-dev.txt
-          pip install -e .
+          pip install -r requirements.txt
+          pip install --group dev -e .
       - name: Running Tests
         env:
           CDISC_LIBRARY_API_KEY: fakekey12341234

cdisc_rules_engine/dataset_builders/contents_define_vlm_dataset_builder.py

@@ -66,7 +66,7 @@ def build(self):
         data_contents_with_vlm["variable_value_length"] = data_contents_with_vlm.data[
             ["variable_value", "define_vlm_data_type"]
         ].apply(
-            lambda row: self.calculate_variable_value_length(
+            lambda row: ValuesDatasetBuilder.calculate_variable_value_length(
                 row["variable_value"], row["define_vlm_data_type"]
             ),
             axis=1,

cdisc_rules_engine/models/dataset/dask_dataset.py

@@ -81,9 +81,9 @@ def __setitem__(self, key, value):
             array_values = da.from_array(value, chunks=tuple(chunks))
             self._data[key] = array_values
         elif isinstance(value, pd.Series):
-            self._data = self._data.reset_index()
-            self._data = self._data.set_index("index")
-            self._data[key] = value
+            pdf = self._data.compute()
+            pdf[key] = value.reindex(pdf.index)
+            self._data = dd.from_pandas(pdf, npartitions=self._data.npartitions)
         elif isinstance(value, dd.DataFrame):
             for column in value:
                 self._data[column] = value[column]

cdisc_rules_engine/operations/codelist_extensible.py

@@ -32,6 +32,10 @@ def _handle_multiple_versions(self) -> pd.Series:
                 "codelist_code": "string",
             }
         )
+        cast_cols = {self.params.ct_version: "string"}
+        if self.params.codelist_code in self.evaluation_dataset.columns:
+            cast_cols[self.params.codelist_code] = "string"
+        self.evaluation_dataset = self.evaluation_dataset.astype(cast_cols)
         if self.params.codelist_code in self.evaluation_dataset.columns:
             is_extensible = self.evaluation_dataset.merge(
                 ct_df.data,

cdisc_rules_engine/operations/codelist_terms.py

@@ -64,6 +64,10 @@ def _handle_multiple_versions(self) -> pd.Series:
                 "codelist_code": "string",
             }
         )
+        cast_cols = {self.params.ct_version: "string"}
+        if self.params.codelist_code in self.evaluation_dataset.columns:
+            cast_cols[self.params.codelist_code] = "string"
+        self.evaluation_dataset = self.evaluation_dataset.astype(cast_cols)
         if self.params.codelist_code in self.evaluation_dataset.columns:
             result = self.evaluation_dataset.merge(
                 ct_df.data,

cdisc_rules_engine/services/data_readers/csv_reader.py

@@ -1,6 +1,6 @@
 import tempfile
 
-from dask.dataframe import dd
+import dask.dataframe as dd
 
 from cdisc_rules_engine.exceptions.custom_exceptions import InvalidCSVFile
 from cdisc_rules_engine.interfaces import DataReaderInterface

cdisc_rules_engine/services/data_services/usdm_data_service.py

@@ -399,8 +399,19 @@ def __read_node_metadata(
         }
 
     @staticmethod
-    def __get_full_path(node: DatumInContext):
-        return f"{node.full_path}".replace(".[", "[")
+    def __get_full_path(node: DatumInContext) -> str:
+        parts = []
+        current = node
+        while current is not None and current.context is not None:
+            parts.append(str(current.path))
+            current = current.context
+        result = ""
+        for part in reversed(parts):
+            if part.startswith("["):
+                result += part
+            else:
+                result = (result + "." if result else "") + part
+        return result
 
     def __get_datasets_content_index(self) -> List[dict]:
         """

cdisc_rules_engine/services/datasetxpt_metadata_reader.py

@@ -61,7 +61,7 @@ def read(self) -> dict:
             "variable_labels": list(metadata.column_labels),
             "variable_names": list(metadata.column_names),
             "variable_formats": [
-                "" if data_type == "NULL" else data_type
+                "" if (data_type == "NULL" or data_type is None) else data_type
                 for data_type in metadata.original_variable_types.values()
             ],
             "variable_name_to_label_map": metadata.column_names_to_labels,

core.py

@@ -357,15 +357,13 @@ def load_custom_dotenv_from_data_options(_ctx, _param, value):
     "-s",
     "--standard",
     required=True,
-    default=None,
     help="CDISC standard to validate against",
     envvar="PRODUCT",
 )
 @click.option(
     "-v",
     "--version",
     required=True,
-    default=None,
     help="Standard version to validate against",
     envvar="VERSION",
 )

docs/contributing.md

@@ -34,7 +34,7 @@ This project enforces consistent formatting and linting via pre-commit hooks.
 - [`flake8`](https://flake8.pycqa.org/) — Python linter
 - [`prettier`](https://prettier.io/) — JSON, YAML, and Markdown formatter
 
-Both `black` and `flake8` are included in `requirements-dev.txt`. After installing dependencies, install the pre-commit hooks:
+Both `black` and `flake8` are included in the `dev` dependency group in `pyproject.toml`. After installing dependencies, install the pre-commit hooks:
 
 ```bash
 pre-commit install

docs/development.md

@@ -47,7 +47,7 @@ source venv/bin/activate
 .\venv\Scripts\Activate
 
 # Install dependencies
-python -m pip install -r requirements-dev.txt
+pip install -e . && pip install --group dev
 ```
 
 ---

pyproject.toml

@@ -4,12 +4,49 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "cdisc-rules-engine"
-dynamic = ["version", "dependencies"]
+dynamic = ["version"]
 description = "Open source offering of the cdisc rules engine"
 readme = "docs/PYPI.md"
 requires-python = ">=3.12, <3.13"
 license = { text = "MIT" }
 authors = [{ name = "cdisc-org", email = "info@cdisc.org" }]
+dependencies = [
+    "business_rules_enhanced >=1.4.8",
+    "cachetools >=6.1.0",
+    "cdisc-library-client >=0.1.6",
+    "click >=8.1.7",
+    "dask[dataframe,array] >=2024.6.0",
+    "fastparquet >=2024.2.0",
+    "importlib-metadata >=8.5.0",
+    "jsonata-python >=0.6.0",
+    "jsonpath-ng >=1.6.1",
+    "jsonschema >=4.18.5",
+    "lxml >=5.2.1",
+    "numpy >=1.26.0",
+    "odmlib >=0.1.4",
+    "openpyxl >=3.1.5",
+    "pandas >=2.1.4, <3.0.0",
+    "psutil >=6.1.1",
+    "pyinstaller >=6.11.0",
+    "pympler >=1.1",
+    "pyreadstat >=1.2.7",
+    "python-dotenv >=1.0.0",
+    "pyyaml >=6.0.2",
+    "redis >=4.5.0",
+    "requests >=2.32.3",
+    "setuptools >=75.6.0",
+    "titlecase >=2.4.1",
+]
+
+[dependency-groups]
+dev = [
+    "black >=24.10.0",
+    "flake8 >=6.1.0",
+    "pre-commit >=2.20.0",
+    "pytest >=7.4.0, <8.0.0",
+    "pytest-asyncio >=0.21.0",
+    "pytest-cov >=6.0.0",
+]
 
 [project.urls]
 "Homepage" = "https://github.com/cdisc-org/cdisc-rules-engine"
@@ -26,5 +63,4 @@ include-package-data = true
 py-modules = ["version"]
 
 [tool.setuptools.dynamic]
-version = { attr = "version.__version__" }
-dependencies = {file = ["requirements.txt"]}
+version = { attr = "version.__version__" }

requirements.txt

@@ -1,3 +1,5 @@
+# Lockfile: exact pinned versions for reproducible installs.
+# Dependency constraints are defined in pyproject.toml.
 business_rules_enhanced==1.4.8
 cachetools==6.1.0
 cdisc-library-client==0.1.6

tests/unit/test_dataset_builders/test_dataset_metadata_define_dataset_builder.py

@@ -149,7 +149,7 @@ def test_dataset_metadata_define_dataset_builder(dataset_path):
             expected_results["dm.xpt"],
             expected_results["ae.xpt"],
         ]
-    ).astype(object)
+    ).astype(object).sort_values("dataset_location").reset_index(drop=True)
 
     result_df = result.data[expected_df.columns].reset_index(drop=True)