Merge pull request #74 from cdisc-org/main

Update sdtm-define.xml branch from main

Author: dostiep

src/generators/define/__init__.py

@@ -3,7 +3,7 @@
 
 
 class AnnotatedCRF(define_object.DefineObject):
-    """ create a Define-XML v2.1 leaf element template """
+    """ create Define-XML v2.1 AnnotatedCRF and leaf element objects """
     def __init__(self):
         super().__init__()
 
@@ -15,15 +15,13 @@ def create_define_objects(self, template, define_objects, lang, acrf):
         :param lang: xml:lang setting for TranslatedText
         :param acrf: part of the common interface but not used by this class
         """
-        self.logger.info("in annotatedCRF...")
         self.lang = lang
-        define_objects["AnnotatedCRF"] = []
-        define_objects["leaf"] = []
         for doc in template:
             a_crf = self._create_acrf_object(doc)
             define_objects["AnnotatedCRF"].append(a_crf)
             leaf = self._create_leaf_object(doc)
-            define_objects["leaf"].append(leaf)
+            if self.find_object(define_objects["leaf"], leaf.ID) is None:
+                define_objects["leaf"].append(leaf)
 
     @staticmethod
     def _create_acrf_object(doc):
@@ -33,7 +31,7 @@ def _create_acrf_object(doc):
         :return: a leaf odmlib template
         """
         acrf = DEFINE.AnnotatedCRF()
-        doc_ref = DEFINE.DocumentRef(leafID=doc["leafID"])
+        doc_ref = DEFINE.DocumentRef(leafID=doc.get("leafID", "LF.acrf"))
         acrf.DocumentRef = doc_ref
         return acrf
 
@@ -45,9 +43,7 @@ def _create_leaf_object(doc):
         :return: a leaf odmlib template
         """
         href = doc.get("href", "acrf.pdf")
-        id = "LF.acrf"
-        # TODO how generate the ID correctly?
-        lf = DEFINE.leaf(ID=id, href=href)
-        title = DEFINE.title(_content=doc["title"])
-        lf.title = title
-        return lf
+        leaf_id = doc.get("leafID", "LF.acrf")
+        lf = DEFINE.leaf(ID=leaf_id, href=href)
+        lf.title = DEFINE.title(_content=doc.get("title", "Annotated CRF"))
+        return lf

src/generators/define/annotatedCRF.py

@@ -26,9 +26,13 @@ def create_define_objects(
         :param acrf: part of the common interface but not used by this class
         """
         self.lang = lang
-        define_objects["CodeList"] = []
         for cl in template:
             # TODO template missing the NCI c-codes for codelists and terms
+            cl_oid = self.require_key(cl, "OID", "CodeList")
+            # Dedup CodeLists by OID so two datasets that reference the same codelist
+            # don't land duplicate definitions in the output.
+            if self.find_object(define_objects["CodeList"], cl_oid) is not None:
+                continue
             cl_defn = self._create_codelist_object(cl)
             coding = cl.get("coding", [])
             cl_c_code = coding[0].get("code") if coding else None
@@ -58,9 +62,7 @@ def _add_codelist_to_objects(cl_c_code, cl, objects):
         if cl_c_code:
             alias = DEFINE.Alias(Context="nci:ExtCodeID", Name=cl_c_code)
             cl.Alias.append(alias)
-        # add the code list to the list of code list define_objects
-        if cl:
-            objects["CodeList"].append(cl)
+        objects["CodeList"].append(cl)
 
     def _create_codelist_object(self, obj):
         oid = self.require_key(obj, "OID", "CodeList")

src/generators/define/codeLists.py

@@ -2,7 +2,6 @@
 import define_object
 
 
-""" Note: Comments have not yet been implemented in the template """
 class Comments(define_object.DefineObject):
     """ create a Define-XML v2.1 CommentDef element template """
     def __init__(self):
@@ -17,13 +16,14 @@ def create_define_objects(self, template, define_objects, lang, acrf):
         :param define_objects: dictionary of odmlib define_objects updated by this method
         :param lang: xml:lang setting for TranslatedText
         """
-        self.logger.info("in Comments...")
         self.lang = lang
-        define_objects["CommentDef"] = []
         for comment in template:
-            com_oid = self.generate_oid(["COM", comment.Name])
-            comment = self._create_commentdef_object(com_oid, comment)
-            define_objects["CommentDef"].append(comment)
+            name = self.require_key(comment, "name", "CommentDef")
+            com_oid = comment.get("OID") or self.generate_oid(["COM", name])
+            if self.find_object(define_objects["CommentDef"], com_oid) is not None:
+                continue
+            com_def = self._create_commentdef_object(com_oid, comment)
+            define_objects["CommentDef"].append(com_def)
 
     def _create_commentdef_object(self, com_oid, comment):
         """
@@ -32,11 +32,12 @@ def _create_commentdef_object(self, com_oid, comment):
         :param comment: comment dictionary from the DDS JSON
         :return: a CommentDef odmlib template
         """
-        com = DEFINE.CommentDef(OID=com_oid, CommentType="FreeText")
-        tt = DEFINE.TranslatedText(_content=comment["Description"], lang=self.lang)
+        com = DEFINE.CommentDef(OID=com_oid)
+        description = self.require_key(comment, "description", f"CommentDef {com_oid}")
+        tt = DEFINE.TranslatedText(_content=description, lang=self.lang)
         com.Description = DEFINE.Description()
         com.Description.TranslatedText.append(tt)
-        if comment.get("Document"):
+        if comment.get("document"):
             self._add_document(comment, com)
         return com
 
@@ -47,8 +48,8 @@ def _add_document(comment, com):
         :param comment: comment dictionary from the DDS JSON
         :param com: define comment template
         """
-        dr = DEFINE.DocumentRef(leafID=comment["Document"])
-        if comment.get("Pages"):
-            pdf = DEFINE.PDFPageRef(PageRefs=comment["Pages"], Type="NamedDestination")
+        dr = DEFINE.DocumentRef(leafID=comment["document"])
+        if comment.get("pages"):
+            pdf = DEFINE.PDFPageRef(PageRefs=comment["pages"], Type="NamedDestination")
             dr.PDFPageRef.append(pdf)
         com.DocumentRef.append(dr)

src/generators/define/comments.py

@@ -2,26 +2,27 @@
 
 
 class ConceptProperties(define_object.DefineObject):
-    """ create a Define-XML v2.1 ExternalCodeList element template """
+    """Create Define-XML v2.1 CodeList elements for concept properties."""
     def __init__(self):
         super().__init__()
 
     def create_define_objects(self, template, objects, lang, acrf):
         """
         parse the define-template and create odmlib define_objects to return in the define_objects dictionary
-        :param template: DDS template dictionary section
-        :param objects: dictionary of odmlib define_objects updated by this method
-        :param lang: xml:lang setting for TranslatedText
-        :param acrf: part of the common interface but not used by this class
         """
         self.lang = lang
         for concept in template:
-            cl_oid = self.generate_oid(["CL", concept["Short Name"]])
+            short_name = concept.get("shortName") or concept.get("Short Name")
+            if not short_name:
+                raise ValueError("Required field 'shortName' missing in ConceptProperties")
+            cl_oid = self.generate_oid(["CL", short_name])
+            if self.find_object(objects["CodeList"], cl_oid) is not None:
+                continue
             cl = self.create_external_codelist(
                 cl_oid=cl_oid,
-                name=concept["Name"],
-                data_type=concept["Data Type"],
-                dictionary=concept["Dictionary"],
-                version=concept.get("Version")
+                name=concept.get("name") or concept.get("Name"),
+                data_type=concept.get("dataType") or concept.get("Data Type"),
+                dictionary=concept.get("dictionary") or concept.get("Dictionary"),
+                version=concept.get("version") or concept.get("Version"),
             )
             objects["CodeList"].append(cl)

src/generators/define/conceptProperties.py

@@ -2,26 +2,27 @@
 
 
 class Concepts(define_object.DefineObject):
-    """ create a Define-XML v2.1 ExternalCodeList element template """
+    """Create Define-XML v2.1 CodeList elements for biomedical concepts."""
     def __init__(self):
         super().__init__()
 
     def create_define_objects(self, template, objects, lang, acrf):
         """
         parse the define-template and create odmlib define_objects to return in the define_objects dictionary
-        :param template: define-template dictionary section
-        :param objects: dictionary of odmlib define_objects updated by this method
-        :param lang: xml:lang setting for TranslatedText
-        :param acrf: part of the common interface but not used by this class
         """
         self.lang = lang
         for concept in template:
-            cl_oid = self.generate_oid(["CL", concept["Short Name"]])
+            short_name = concept.get("shortName") or concept.get("Short Name")
+            if not short_name:
+                raise ValueError("Required field 'shortName' missing in Concepts")
+            cl_oid = self.generate_oid(["CL", short_name])
+            if self.find_object(objects["CodeList"], cl_oid) is not None:
+                continue
             cl = self.create_external_codelist(
                 cl_oid=cl_oid,
-                name=concept["Name"],
-                data_type=concept["Data Type"],
-                dictionary=concept["Dictionary"],
-                version=concept.get("Version")
+                name=concept.get("name") or concept.get("Name"),
+                data_type=concept.get("dataType") or concept.get("Data Type"),
+                dictionary=concept.get("dictionary") or concept.get("Dictionary"),
+                version=concept.get("version") or concept.get("Version"),
             )
             objects["CodeList"].append(cl)

src/generators/define/concepts.py

@@ -1,8 +1,8 @@
-from odmlib.define_2_1 import model as DEFINE
 import define_object
 
+
 class Conditions(define_object.DefineObject):
-    """ create a Define-XML v2.1 WhereClauseDef element objects """
+    """ cache DDS conditions for use by WhereClauses to build RangeCheck elements """
     def __init__(self):
         super().__init__()
 
@@ -16,38 +16,22 @@ def create_define_objects(self, template, define_objects, lang, acrf):
         """
         self.lang = lang
         conditions = []
-        # store the conditions in define_objects for use when generating WhereClauseDef
         for condition in template:
             rc = self._create_condition(condition)
             conditions.append({rc["OID"]: rc})
-        # store in define_objects with underscore prefix to indicate internal use
         define_objects["_conditions"] = conditions
 
     @staticmethod
     def _create_condition(condition):
         condition_obj = {"OID": condition["OID"]}
         range_checks = []
         for rc in condition["rangeChecks"]:
-            rc_attr = {"SoftHard": "Soft", "ItemOID": rc["item"], "Comparator": rc["comparator"]}
-            check_values = []
-            for value in rc["checkValues"]:
-                check_values.append(value)
-            rc_attr["CheckValue"] = check_values
+            rc_attr = {
+                "SoftHard": rc.get("softHard", "Soft"),
+                "ItemOID": rc["item"],
+                "Comparator": rc["comparator"],
+                "CheckValue": list(rc.get("checkValues", [])),
+            }
             range_checks.append(rc_attr)
         condition_obj["RangeCheck"] = range_checks
         return condition_obj
-
-    def _create_rangecheck(self, wc, dataset):
-        """
-        use the values from the conditions section of the DDS JSON to create a RangeCheck odmlib template
-        :param wc: WhereClause dictionary from the DDS JSON
-        :param dataset: dataset name
-        :return: a RangeCheck odmlib template
-        """
-        item_oid = self.generate_oid(["IT", dataset, wc["Variable"]])
-        rc_attr = {"SoftHard": "Soft", "ItemOID": item_oid, "Comparator": wc["Comparator"]}
-        rc = DEFINE.RangeCheck(**rc_attr)
-        for value in wc["Values"]:
-            cv = DEFINE.CheckValue(_content=value)
-            rc.CheckValue.append(cv)
-        return rc