Variable value (#1148)

* started dataset builder

* new rule types

* progress

* updated rule type json

* updating builders

* update

* added split dataset method

* split dataset synthesis

* merged logic

* updated tests

* update

* update test

* updated test

Author: SFJohnson24

cdisc_rules_engine/dataset_builders/base_dataset_builder.py

@@ -56,12 +56,18 @@ def build(self) -> DatasetInterface:
         """
         pass
 
-    @abstractmethod
-    def build_split_datasets(self, dataset_name) -> DatasetInterface:
+    def build_split_datasets(self, dataset_name, **kwargs) -> DatasetInterface:
         """
-        Returns correct dataframe to operate on
+        Returns correct dataframe to operate on.
+        Default implementation that temporarily sets dataset_path to dataset_name and calls build().
         """
-        pass
+        original_path = self.dataset_path
+        try:
+            self.dataset_path = dataset_name
+            result = self.build(**kwargs)
+            return result
+        finally:
+            self.dataset_path = original_path
 
     def get_dataset(self, **kwargs):
         # If validating dataset content, ensure split datasets are handled.

cdisc_rules_engine/dataset_builders/content_metadata_dataset_builder.py

@@ -10,12 +10,9 @@ def build(self):
         dataset_name - Name of the dataset
         dataset_label - Label for the dataset
         """
-        return self.build_split_datasets(self.dataset_path)
-
-    def build_split_datasets(self, dataset_name, **kwargs):
         size_unit: str = self.rule_processor.get_size_unit_from_rule(self.rule)
         return self.data_service.get_dataset_metadata(
-            dataset_name=dataset_name,
+            dataset_name=self.dataset_path,
             size_unit=size_unit,
             datasets=self.datasets,
         )

cdisc_rules_engine/dataset_builders/contents_define_dataset_builder.py

@@ -27,19 +27,13 @@ def build(self):
 
         ...,
         """
-        return self.build_split_datasets(self.dataset_metadata.filename)
-
-    def build_split_datasets(self, dataset_name, **kwargs):
-        """
-        Returns the contents of a file as a dataframe for evaluation.
-        """
         data_contents_df = self.data_service.get_dataset(
-            dataset_name=dataset_name, datasets=self.datasets
+            dataset_name=self.dataset_path, datasets=self.datasets
         )
         # Build dataset metadata dataframe
         size_unit: str = self.rule_processor.get_size_unit_from_rule(self.rule)
         dataset_metadata = self.data_service.get_dataset_metadata(
-            dataset_name=dataset_name, size_unit=size_unit, datasets=self.datasets
+            dataset_name=self.dataset_path, size_unit=size_unit, datasets=self.datasets
         ).to_dict(orient="records")[0]
         # Build define xml dataframe
         define = self.get_define_xml_item_group_metadata_for_dataset(dataset_metadata)

cdisc_rules_engine/dataset_builders/dataset_builder_factory.py

@@ -44,6 +44,12 @@
 from cdisc_rules_engine.dataset_builders.variables_metadata_with_define_and_library_dataset_builder import (
     VariablesMetadataWithDefineAndLibraryDatasetBuilder,
 )
+from cdisc_rules_engine.dataset_builders.dataset_metadata_values_builder import (
+    ValueCheckDatasetMetadataDatasetBuilder,
+)
+from cdisc_rules_engine.dataset_builders.variables_metadata_values_dataset_builder import (
+    ValueCheckVariableMetadataDatasetBuilder,
+)
 from cdisc_rules_engine.dataset_builders.base_dataset_builder import BaseDatasetBuilder
 from cdisc_rules_engine.enums.rule_types import RuleTypes
 
@@ -65,6 +71,8 @@ class DatasetBuilderFactory(FactoryInterface):
         RuleTypes.VARIABLE_METADATA_CHECK_AGAINST_LIBRARY.value: VariablesMetadataWithLibraryMetadataDatasetBuilder,
         RuleTypes.DEFINE_ITEM_METADATA_CHECK_AGAINST_LIBRARY.value: DefineVariablesWithLibraryMetadataDatasetBuilder,
         RuleTypes.VARIABLE_METADATA_CHECK_AGAINST_DEFINE_XML_AND_LIBRARY.value: VariablesMetadataWithDefineAndLibraryDatasetBuilder,
+        RuleTypes.VALUE_CHECK_WITH_DATASET_METADATA.value: ValueCheckDatasetMetadataDatasetBuilder,
+        RuleTypes.VALUE_CHECK_WITH_VARIABLE_METADATA.value: ValueCheckVariableMetadataDatasetBuilder,
     }
 
     @classmethod

cdisc_rules_engine/dataset_builders/dataset_metadata_values_builder.py

@@ -0,0 +1,31 @@
+from cdisc_rules_engine.dataset_builders.values_dataset_builder import (
+    ValuesDatasetBuilder,
+)
+
+
+class ValueCheckDatasetMetadataDatasetBuilder(ValuesDatasetBuilder):
+    def build(self):
+        """
+        Returns a long dataset where each value in each row of the original dataset is
+        a row in the new dataset with dataset metadata attached.
+        Columns available in the dataset include:
+        - "row_number"
+        - "variable_name"
+        - "variable_value"
+        - dataset_size - File size
+        - dataset_location - Path to file
+        - dataset_name - Name of the dataset
+        - dataset_label - Label for the dataset
+        """
+        size_unit: str = self.rule_processor.get_size_unit_from_rule(self.rule)
+        dataset_metadata = self.data_service.get_dataset_metadata(
+            dataset_name=self.dataset_path,
+            size_unit=size_unit,
+            datasets=self.datasets,
+        )
+        dataset_metadata = dataset_metadata.to_dict(orient="records")[0]
+        data_contents_long_df = super().build()
+        row_count = len(data_contents_long_df)
+        for key, value in dataset_metadata.items():
+            data_contents_long_df[key] = [value] * row_count
+        return data_contents_long_df

cdisc_rules_engine/dataset_builders/variables_metadata_dataset_builder.py

@@ -16,8 +16,3 @@ def build(self):
         return self.data_service.get_variables_metadata(
             dataset_name=self.dataset_path, datasets=self.datasets, drop_duplicates=True
         )
-
-    def build_split_datasets(self, dataset_name, **kwargs):
-        return self.data_service.get_variables_metadata(
-            dataset_name=dataset_name, datasets=self.datasets, drop_duplicates=True
-        )

cdisc_rules_engine/dataset_builders/variables_metadata_values_dataset_builder.py

@@ -0,0 +1,36 @@
+from cdisc_rules_engine.dataset_builders.values_dataset_builder import (
+    ValuesDatasetBuilder,
+)
+
+
+class ValueCheckVariableMetadataDatasetBuilder(ValuesDatasetBuilder):
+    def build(self):
+        """
+        Returns a long dataset where each value in each row of the original dataset is
+        a row in the new dataset, with variable metadata attached.
+
+        Columns available in the dataset include:
+        - "row_number"
+        - "variable_name"
+        - "variable_value"
+        - "variable_order_number"
+        - "variable_label"
+        - "variable_size"
+        - "variable_data_type"
+        - "variable_format"
+        - "variable_value_length"
+        """
+        data_contents_long_df = super().build()
+        variable_metadata = self.data_service.get_variables_metadata(
+            dataset_name=self.dataset_path, datasets=self.datasets, drop_duplicates=True
+        )
+        merged_df = data_contents_long_df.merge(
+            variable_metadata, how="left", on="variable_name"
+        )
+        merged_df["variable_value_length"] = merged_df.apply(
+            lambda row: ValuesDatasetBuilder.calculate_variable_value_length(
+                row["variable_value"], row["variable_data_type"]
+            ),
+            axis=1,
+        )
+        return merged_df

cdisc_rules_engine/enums/rule_types.py

@@ -29,3 +29,5 @@ class RuleTypes(BaseEnum):
     DEFINE_ITEM_METADATA_CHECK_AGAINST_LIBRARY = (
         "Define Item Metadata Check against Library Metadata"
     )
+    VALUE_CHECK_WITH_DATASET_METADATA = "Value Check with Dataset Metadata"
+    VALUE_CHECK_WITH_VARIABLE_METADATA = "Value Check with Variable Metadata"

cdisc_rules_engine/models/dataset/dask_dataset.py

@@ -360,6 +360,24 @@ def fillna(
     def to_dict(self, **kwargs) -> dict:
         return list(self._data.map_partitions(lambda x: x.to_dict(orient="records")))
 
+    def items(self, **kwargs):
+        computed_df = self._data.compute()
+        return computed_df.to_dict(**kwargs).items()
+
+    def keys(self, **kwargs):
+        """
+        Returns a object containing the keys in the dataset dictionary.
+        """
+        computed_df = self._data.compute()
+        return computed_df.to_dict(**kwargs).keys()
+
+    def values(self, **kwargs):
+        """
+        Returns a object containing the values in the dataset dictionary.
+        """
+        computed_df = self._data.compute()
+        return computed_df.to_dict(**kwargs).values()
+
     def isin(self, values):
         values_set = set(values)
 

cdisc_rules_engine/models/dataset/dataset_interface.py

@@ -251,3 +251,22 @@ def to_dict(self, **kwargs) -> dict:
         """
         Convert the dataset to a dictionary.
         """
+
+    @abstractmethod
+    def items(self, **kwargs):
+        """
+        Convert the dataset to dictionary items.
+        Returns a view object displaying a list of (key, value) tuple pairs.
+        """
+
+    @abstractmethod
+    def keys(self, **kwargs):
+        """
+        Returns a view object containing the keys in the dataset dictionary.
+        """
+
+    @abstractmethod
+    def values(self, **kwargs):
+        """
+        Returns a view object containing the values in the dataset dictionary.
+        """