Merge pull request #55 from cdisc-org/sdtm-define-xml

Updates to create_define_json programm

Author: dostiep

src/define-xml/create_define_json.py

@@ -1800,6 +1800,37 @@ def update_datasets_dict(self):
         # Add TS dataset
         if "TS" not in self.datasets_dict:
             self.datasets_dict["TS"] = {}
+
+            # Add TSPARMCD values extracted from vlm_lookup["TSPARMCD"] WhereClause Values
+            tsparmcd_values = set()
+            for entry in self.vlm_lookup.get("TSPARMCD", []):
+                for where_clause in entry.get("WhereClause", []):
+                    for clause in where_clause.get("Clause", []):
+                        for value in clause.get("Values", []):
+                            if value:
+                                tsparmcd_values.add(value)
+
+            if tsparmcd_values:
+                self.datasets_dict["TS"]["TSPARMCD"] = {"codelist": {
+                    "C66738": sorted(tsparmcd_values)
+                }}
+
+            # Add TSPARM values by matching conceptIds from filtered TSPARMCD terms against C67152
+            kept_concept_ids = {term.get('conceptId') for term in [
+                t for t in self.client.get_api_json(f"/mdr/ct/packages/sdtmct-{self.sdtmct}/codelists/C66738")['terms']
+                if not tsparmcd_values or t.get('submissionValue') in tsparmcd_values
+            ]}
+
+            tsparm_values = {
+                term.get('submissionValue')
+                for term in self.client.get_api_json(f"/mdr/ct/packages/sdtmct-{self.sdtmct}/codelists/C67152")['terms']
+                if term.get('conceptId') in kept_concept_ids and term.get('submissionValue')
+            }
+
+            if tsparm_values:
+                self.datasets_dict["TS"]["TSPARM"] = {"codelist": {
+                    "C67152": sorted(tsparm_values)
+                }}
 
         # Add TA dataset from arms
         if self.studyDesignData.get('arms', []):
@@ -2060,7 +2091,7 @@ def update_datasets_dict(self):
                         "OID": f"CL.{epoch_codelist_name}",
                         "name": "Epoch",
                         "dataType": "text",
-                        "standard": "ST.SDTMCT",
+                        "standard": "STD.SDTMCT",
                         "codeListItems": epoch_terms,
                     }
 
@@ -2106,8 +2137,13 @@ def _process_standard_dataset(self, dataset, dataset_data):
             "purpose": "Tabulation",
             "structure": dataset_data['datasetStructure'],
             "isReferenceData": (dataset_data.get('_links', {}).get('parentClass', {}).get('title') == "Trial Design") or not any(v.get('name') in ("USUBJID", "POOLID") for v in all_vars),
-            # Here are we always sure to use ST.SDTMIG?
-            "standard": "ST.SDTMIG",
+            "keySequence": ["__PLACEHOLDER__"],
+            # Here are we always sure to use STD.SDTMIG?
+            "standard": "STD.SDTMIG",
+            "observationClass": {
+                "name": dataset_data.get("_links", {}).get("parentClass", {}).get("title", "").upper()
+            },
+            # Here add future code for subClass in ADaM if needed
             "items": []
         }
 
@@ -2131,6 +2167,8 @@ def _process_standard_dataset(self, dataset, dataset_data):
 
             if 'length' in var_data:
                 item_dict['length'] = var_data['length']
+            else:
+                item_dict['length'] = None
 
             if 'format' in var_data:
                 item_dict['displayFormat'] = var_data['format']
@@ -2143,6 +2181,11 @@ def _process_standard_dataset(self, dataset, dataset_data):
                     "type": var_data['originType'],
                     "source": var_data['originSource']
                 }
+            else:
+                item_dict['origin'] = {
+                    "type": "__PLACEHOLDER__",
+                    "source": "__PLACEHOLDER__"
+                }
 
             # Add codelist references for TA variables
             if dataset == "TA":
@@ -2281,6 +2324,7 @@ def _process_standard_dataset(self, dataset, dataset_data):
                     "OID": f"VL.{dataset}.{var_name}",
                     "name": f"VL_{dataset}_{var_name}",
                     "type": "ValueList",
+                    "wasDerivedFrom": f"IT.{dataset}.{var_name}",
                     "items": self.vlm_items_by_variable[vlm_key]
                 }
                 slices.append(slice_dict)
@@ -2772,6 +2816,7 @@ def process(self):
         self.process_biomedical_concepts()
         self.build_vlm_lookup()
         self.update_datasets_dict()
+        self._build_global_codelist_terms()
         self.populate_study_elements()
         self.process_datasets()
         self.add_standards()
@@ -2780,6 +2825,27 @@ def process(self):
         if self.debug:
             self.save_debug_files()
 
+    def _build_global_codelist_terms(self):
+        """
+        Pre-scan datasets_dict to build a union of all explicitly restricted
+        terms per codelist C-Code. Used to decide placeholder vs. real terms.
+
+        For each codelist C-Code encountered across all datasets/variables:
+        - Non-empty list: contributes those values to the union
+        - Empty list: contributes nothing (means "unrestricted" for that variable)
+
+        Result stored in self.global_codelist_terms = {codelist_id: set_of_values}.
+        A codelist with an empty set means no dataset ever restricted it → placeholder.
+        """
+        self.global_codelist_terms = {}
+        for variables in self.datasets_dict.values():
+            for var_data in variables.values():
+                for codelist_id, terms in var_data.get('codelist', {}).items():
+                    if codelist_id not in self.global_codelist_terms:
+                        self.global_codelist_terms[codelist_id] = set()
+                    for term in terms:
+                        self.global_codelist_terms[codelist_id].add(term)
+
     def _get_or_create_condition_from_vlm(self, where_clause_data, dataset, variable_name):
         """
         Get existing or create new conditions from VLM where clause data.
@@ -2932,7 +2998,15 @@ def create_term_dict(term):
                 restriction_list = restriction_codes[codelist_id]
 
                 if not restriction_list:
-                    final_terms = [create_term_dict(term) for term in all_terms]
+                    global_terms = self.global_codelist_terms.get(codelist_id, set())
+                    if global_terms:
+                        final_terms = [
+                            create_term_dict(term)
+                            for term in all_terms
+                            if term.get("submissionValue") in global_terms
+                        ]
+                    else:
+                        final_terms = [{"NCI Term Code": None, "Term": "__PLACEHOLDER__", "Decoded Value": ["__PLACEHOLDER__"]}]
                 else:
                     # Find matching terms from standard codelist
                     final_terms = [
@@ -2962,7 +3036,15 @@ def create_term_dict(term):
                 ]
 
             else:
-                final_terms = [create_term_dict(term) for term in all_terms]
+                global_terms = self.global_codelist_terms.get(codelist_id, set())
+                if global_terms:
+                    final_terms = [
+                        create_term_dict(term)
+                        for term in all_terms
+                        if term.get("submissionValue") in global_terms
+                    ]
+                else:
+                    final_terms = [{"NCI Term Code": None, "Term": "__PLACEHOLDER__", "Decoded Value": ["__PLACEHOLDER__"]}]
 
             codelist_entry = {
                 "NCI Codelist Code": codelist_data.get("conceptId"),
@@ -2990,7 +3072,7 @@ def create_term_dict(term):
                     "OID": oid,
                     "name": name,
                     "dataType": "text",
-                    "standard": "ST.SDTMCT",
+                    "standard": "STD.SDTMCT",
                 }
 
                 # Add coding if NCI Codelist Code exists
@@ -3038,14 +3120,14 @@ def add_standards(self):
         """
         self.template["standards"] = [
             {
-                "OID": "ST.SDTMIG",
+                "OID": "STD.SDTMIG",
                 "name": "SDTMIG",
                 "type": "IG",
                 "version": self.sdtmig,
                 "status": "FINAL"
             },
             {
-                "OID": "ST.SDTMCT",
+                "OID": "STD.SDTMCT",
                 "name": "CDISC/NCI",
                 "type": "CT",
                 "version": self.sdtmct,

src/define-xml/define.yaml

@@ -634,6 +634,52 @@ classes:
         description: >-
           Used to indicate that this ItemGroup has no data, e.g. for a manifest.
         range: boolean
+      observationClass:
+        description: >-
+          Identifies the predefined CDISC model Class.
+        range: DefClass
+        required: false
+
+  DefClass:
+    description: >-
+      The predefined CDISC model Class that applies to an ItemGroupDef.
+    attributes:
+      name:
+        description: >-
+          Name of the General Observation Class following CDISC Controlled Terminology.
+        range: string
+        required: true
+      subClasses:
+        description: >-
+          One or more SubClasses that further identify the specific SubClass within a Class.
+        range: SubClass
+        multivalued: true
+        required: false
+        inlined: true
+        inlined_as_list: true
+
+  SubClass:
+    description: >-
+      A specific SubClass within a CDISC model Class.
+    attributes:
+      name:
+        description: >-
+          Name of the SubClass following CDISC Controlled Terminology for SubClass.
+        range: string
+        required: true
+      parentClass:
+        description: >-
+          Name of the parent Class or SubClass following CDISC Controlled Terminology.
+        range: string
+        required: false
+      subClasses:
+        description: >-
+          Nested SubClass(es) for multi-level SubClass hierarchy.
+        range: SubClass
+        multivalued: true
+        required: false
+        inlined: true
+        inlined_as_list: true
 
   Relationship:
     description: >-
@@ -1848,7 +1894,8 @@ classes:
   # DPROD to complement Dataset class and link to DCAT ontology
   DataProduct:
     description: >-
-      A governed collection that represents a purpose-driven assembly of datasets and services with an owning team and lifecycle
+      A governed collection that represents a purpose-driven assembly of datasets and services with an owning team and lifecycle.
+      The DataProduct defines the boundary of accountability between data producers and consumers.
     exact_mappings:
       - dprod:DataProduct
       - dcat:DataService
@@ -1881,6 +1928,26 @@ classes:
         multivalued: true
         inlined: true
         inlined_as_list: true
+      inputDataflow:
+        description: >-
+          Description of the input interface before concrete Datasets exist.
+          Dataflows referenced here represent the demand side of a ProvisionAgreement.
+        range: Dataflow
+        multivalued: true
+        inlined: true
+        inlined_as_list: true
+        close_mappings:
+          - dcat:distribution
+      outputDataflow:
+        description: >-
+          Description of the output interface before concrete Datasets exist.
+          Dataflows referenced here represent the supply side of a ProvisionAgreement.
+        range: Dataflow
+        multivalued: true
+        inlined: true
+        inlined_as_list: true
+        close_mappings:
+          - dcat:distribution
       inputDataset:
         description: Source datasets used by the data product
         range: Dataset
@@ -1891,6 +1958,7 @@ classes:
         description: Output datasets produced by the data product
         range: Dataset
         multivalued: true
+        inlined: true
         inlined_as_list: true
       hasPolicy:
         description: Policies governing the use and access of the data product
@@ -1975,6 +2043,13 @@ classes:
         description: >-
           The Data Provider that is part of this agreement
         range: DataProvider
+      consumer:
+        description: >-
+          The Data Consumer that is part of this agreement
+        any_of:
+          - range: DataProduct
+          - range: Organization
+          - range: string
       dataFlow:
         description: >-
           The Dataflow that is covered by this agreement