Define metadata (#1273)

* work

* update

* builder

* tests

* ok

* error handling

* update docs

---------

Co-authored-by: RamilCDISC <113539111+RamilCDISC@users.noreply.github.com>

Author: SFJohnson24

cdisc_rules_engine/dataset_builders/dataset_metadata_define_dataset_builder.py

@@ -25,6 +25,8 @@ def build(self):
         define_dataset_class - dataset class
         define_dataset_structure - dataset structure
         define_dataset_is_non_standard - whether a dataset is a standard
+        define_dataset_variables - dataset variables
+        define_dataset_key_sequence - dataset key sequence
 
         ...,
         """

cdisc_rules_engine/dataset_builders/define_variables_dataset_builder.py

@@ -22,6 +22,7 @@ def build(self):
         "define_variable_order_number",
         "define_variable_has_codelist",
         "define_variable_codelist_coded_values",
+        "define_variable_codelist_coded_codes",
         "define_variable_mandatory",
         """
         # get Define XML metadata for domain and use it as a rule comparator

cdisc_rules_engine/dataset_builders/define_variables_with_library_metadata.py

@@ -21,6 +21,7 @@ def build(self):
         "define_variable_order_number",
         "define_variable_has_codelist",
         "define_variable_codelist_coded_values",
+        "define_variable_codelist_coded_codes",
         "define_variable_mandatory",
         "define_variable_has_comment",
         "library_variable_name",

cdisc_rules_engine/dataset_builders/variables_metadata_with_define_and_library_dataset_builder.py

@@ -26,8 +26,9 @@ def build(self):
         define_variable_has_no_data,
         define_variable_order_number,
         define_variable_has_codelist,
-        define_variable_codelist_coded_values
-        define_variable_mandatory
+        define_variable_codelist_coded_values,
+        define_variable_codelist_coded_codes,
+        define_variable_mandatory,
         library_variable_name,
         library_variable_label,
         library_variable_data_type,

cdisc_rules_engine/dataset_builders/variables_metadata_with_define_dataset_builder.py

@@ -26,7 +26,8 @@ def build(self):
         define_variable_has_no_data,
         define_variable_order_number,
         define_variable_has_codelist,
-        define_variable_codelist_coded_values
+        define_variable_codelist_coded_values,
+        define_variable_codelist_coded_codes,
         define_variable_mandatory
         """
         # get Define XML metadata for domain and use it as a rule comparator

cdisc_rules_engine/operations/codelist_terms.py

@@ -86,9 +86,18 @@ def _handle_single_version(self) -> pd.Series:
             ct_packages = self.library_metadata._ct_package_metadata
             if "define_XML_merged_CT" in ct_packages:
                 ct_package_data = ct_packages["define_XML_merged_CT"]
+            elif not ct_packages:
+                raise MissingDataError(
+                    "CT package data is not populated. "
+                    "A valid define.xml file or -ct command is required to execute."
+                )
             else:
                 ct_package_data = next(
-                    (pkg for name, pkg in ct_packages.items() if name != "extensible")
+                    (
+                        pkg
+                        for name, pkg in ct_packages.items()
+                        if name != "extensible" and not name.startswith("define-xml")
+                    )
                 )
         except AttributeError as e:
             logger.warning(

cdisc_rules_engine/rules_engine.py

@@ -454,6 +454,22 @@ def handle_validation_exceptions(  # noqa
                 message=message,
                 status=ExecutionStatus.SKIPPED.value,
             )
+        elif isinstance(
+            exception, AttributeError
+        ) and "'NoneType' object has no attribute" in str(exception):
+            error_obj = ValidationErrorContainer(
+                dataset=os.path.basename(dataset_path),
+                message="Missing field during execution, rule may not be applicable- unable to process dataset",
+                status=ExecutionStatus.SKIPPED.value,
+            )
+            message = "rule evaluation skipped - missing metadata"
+            errors = [error_obj]
+            return ValidationErrorContainer(
+                dataset=os.path.basename(dataset_path),
+                errors=errors,
+                message=message,
+                status=ExecutionStatus.SKIPPED.value,
+            )
         else:
             error_obj = FailedValidationEntity(
                 dataset=os.path.basename(dataset_path),

cdisc_rules_engine/services/define_xml/base_define_xml_reader.py

@@ -349,6 +349,7 @@ def _get_item_def_representation(self, itemdef, itemref, codelists, index) -> di
             "define_variable_length": None,
             "define_variable_has_codelist": False,
             "define_variable_codelist_coded_values": [],
+            "define_variable_codelist_coded_codes": [],
             "define_variable_mandatory": None,
             "define_variable_has_comment": False,
         }
@@ -377,6 +378,9 @@ def _get_item_def_representation(self, itemdef, itemref, codelists, index) -> di
                 data["define_variable_codelist_coded_values"].extend(
                     self._get_codelist_coded_values(codelist)
                 )
+                data["define_variable_codelist_coded_codes"].extend(
+                    self._get_codelist_coded_codes(codelist)
+                )
             if itemdef.Origin:
                 data["define_variable_origin_type"] = self._get_origin_type(itemdef)
             data["define_variable_has_no_data"] = getattr(itemref, "HasNoData", "")
@@ -405,6 +409,14 @@ def _get_codelist_coded_values(self, codelist):
             for codelist_item in codelist.CodeListItem + codelist.EnumeratedItem:
                 yield codelist_item.CodedValue
 
+    def _get_codelist_coded_codes(self, codelist):
+        if codelist:
+            for codelist_item in codelist.CodeListItem + codelist.EnumeratedItem:
+                if hasattr(codelist_item, "Alias") and codelist_item.Alias:
+                    for alias in codelist_item.Alias:
+                        if hasattr(alias, "Name"):
+                            yield alias.Name
+
     @abstractmethod
     def _get_origin_type(self, itemdef):
         pass

resources/schema/Rule_Type.md

@@ -119,6 +119,7 @@ any:
 - `define_variable_order_number`
 - `define_variable_has_codelist`
 - `define_variable_codelist_coded_values`
+- `define_variable_codelist_coded_values`
 - `define_variable_has_comment`
 
 #### Rule Macro
@@ -143,6 +144,8 @@ Attach variable codelist and codelist terms
 - `define_variable_order_number`
 - `define_variable_has_codelist`
 - `define_variable_codelist_coded_values`
+- `define_variable_codelist_coded_codes`
+- `define_variable_mandatory`
 - `define_variable_has_comment`
 - `library_variable_name`
 - `library_variable_order_number`
@@ -363,7 +366,22 @@ Attach define xml metadata at value level
 - `variable_`...
 - `define_variable_name`
 - `define_variable_label`
-- `define_variable_`...
+- `define_variable_data_type`
+- `define_variable_is_collected`
+- `define_variable_role`
+- `define_variable_size`
+- `define_variable_ccode`
+- `define_variable_format`
+- `define_variable_allowed_terms`
+- `define_variable_origin_type`
+- `define_variable_has_no_data`
+- `define_variable_order_number`
+- `define_variable_length`
+- `define_variable_has_codelist`
+- `define_variable_codelist_coded_values`
+- `define_variable_codelist_coded_codes`
+- `define_variable_mandatory`
+- `define_variable_has_comment`
 
 #### Rule Macro
 
@@ -420,6 +438,7 @@ Attach define xml metadata at variable level
 - `define_variable_length`
 - `define_variable_has_codelist`
 - `define_variable_codelist_coded_values`
+- `define_variable_codelist_coded_codes`
 - `define_variable_mandatory`
 - `define_variable_has_comment`
 - `library_variable_name`

scripts/script_utils.py

@@ -367,7 +367,7 @@ def load_rules_from_local(args) -> List[dict]:
     else:
         engine_logger.info(
             "No rules specified with -r rules flag. "
-            "Validating with all rules in local directory"
+            "Validating with rules in local directory"
         )
         keys = None