Merge pull request #45 from cdisc-org/sdtm-define-xml

Added Readme and Requirements files

Author: dostiep

src/define-xml/README.md

@@ -0,0 +1,141 @@
+# USDM to Define-JSON Processor
+
+This script transforms a [USDM](https://www.cdisc.org/standards/foundational/usdm) (Unified Study Data Model) JSON file into a Define-JSON structure. 
+
+---
+
+## Prerequisites
+
+### Python version
+
+Python 3.9 or later is recommended.
+
+### Required packages
+
+Install dependencies with:
+
+```bash
+pip install jsonata cdisc-library-client python-dotenv pyyaml openpyxl
+```
+
+| Package | Purpose |
+|---|---|
+| `jsonata` | JSONata expression evaluation for USDM navigation |
+| `cdisc-library-client` | CDISC Library REST API client |
+| `python-dotenv` | Load API key from a `.env` file |
+| `pyyaml` | Parse the Define-JSON YAML schema for validation |
+| `openpyxl` | Write the Excel validation report |
+
+### CDISC Library API key
+
+The script requires a valid CDISC Library API key. Provide it in one of two ways:
+
+1. **Environment variable** – set `CDISC_API_KEY` in your shell or system environment.
+2. **`.env` file** – create a `.env` file in the working directory containing:
+   ```
+   CDISC_API_KEY=your_api_key_here
+   ```
+3. **Command-line argument** – pass `--cdisc_api_key <key>` at runtime.
+
+---
+
+## Usage
+
+Run the script from the `src/define-xml/` directory:
+
+```
+python create_define_json.py --usdm_file <path> --output_template <path> --sdtmct <date> [options]
+```
+
+### Arguments
+
+| Argument | Required | Default | Description |
+|---|---|---|---|
+| `--usdm_file` | **Yes** | — | Path to the input USDM JSON file |
+| `--output_template` | **Yes** | — | Path for the output Define-JSON file |
+| `--sdtmct` | **Yes** | — | SDTM Controlled Terminology date (`yyyy-mm-dd`) |
+| `--sdtmig` | No | `3.4` | SDTM Implementation Guide version |
+| `--studyversion` | No | `0` | Index of the study version within the USDM file |
+| `--studydesign` | No | `0` | Index of the study design within the USDM file |
+| `--docversion` | No | `0` | Index of the document version within the USDM file |
+| `--cdisc_api_key` | No | env var | CDISC Library API key (overrides `CDISC_API_KEY` env var) |
+| `--cosmosversion` | No | `v2` | CDISC Cosmos API version |
+| `--validate [schema]` | No | `define.yaml` | Validate output against a YAML schema file. If omitted the default `define.yaml` is used. Requires `--validation_report` when specified. |
+| `--validation_report` | Conditional | — | Path to the Excel validation report (required when `--validate` is used) |
+| `--debug` | No | `false` | Save intermediate dictionaries as JSON debug files |
+
+---
+
+## Examples
+
+### Minimal run (no validation)
+
+```bash
+python create_define_json.py \
+  --usdm_file ..\..\data\protocol\LZZT\usdm\pilot_LLZT_protocol.json \
+  --output_template define.json \
+  --sdtmct 2025-03-28
+```
+
+### With schema validation and debug output
+
+```bash
+python create_define_json.py \
+  --usdm_file ..\..\data\protocol\LZZT\usdm\pilot_LLZT_protocol.json \
+  --output_template define.json \
+  --sdtmct 2025-03-28 \
+  --validate \
+  --validation_report validation_report.xlsx \
+  --debug
+```
+
+### Specifying a custom SDTMIG version and API key
+
+```bash
+python create_define_json.py \
+  --usdm_file ..\..\data\protocol\LZZT\usdm\pilot_LLZT_protocol.json \
+  --output_template define.json \
+  --sdtmct 2025-03-28 \
+  --sdtmig 3.3 \
+  --cdisc_api_key <your_api_key> \
+  --validate \
+  --validation_report validation_report.xlsx
+```
+
+---
+
+## Output files
+
+| File | Description |
+|---|---|
+| `<output_template>` (e.g. `define.json`) | Define-JSON with itemGroups, slices, conditions, whereClauses, and codeLists |
+| `<validation_report>` (e.g. `validation_report.xlsx`) | Excel report listing schema validation results (pass/fail per field) |
+
+### Debug files (created with `--debug`)
+
+When `--debug` is enabled the following intermediate JSON files are written to the working directory:
+
+| File | Contents |
+|---|---|
+| `debug_datasets_dict.json` | Variable metadata per dataset |
+| `debug_bc_dict.json` | Biomedical concept VLM metadata |
+| `debug_vlm_lookup.json` | VLM entries keyed by variable name |
+| `debug_vlm_items_by_variable.json` | VLM items grouped for slice creation |
+| `debug_test_dict.json` | TEST/TESTCD concept mappings |
+| `debug_code_lists_map.json` | Deduplicated codelists by short name |
+| `debug_condition_lookup.json` | Condition deduplication cache |
+| `debug_dataset_data.json` | Raw dataset data accumulated during processing |
+
+---
+
+## Schema validation
+
+The `--validate` flag runs the output JSON against the `define.yaml` schema (or a custom schema file you provide). The validation results are written to the Excel file specified by `--validation_report`. The script exits with code `1` if validation fails, making it suitable for use in CI pipelines.
+
+---
+
+## Notes
+
+- The script requires an active internet connection to call the CDISC Library API.
+- Windows terminals that default to `cp1252` encoding are handled automatically; Unicode status symbols are displayed correctly.
+- The `--studyversion`, `--studydesign`, and `--docversion` arguments are zero-based indices into the corresponding arrays in the USDM JSON. Most studies use the default value of `0`.

src/define-xml/create_define_json.py

@@ -1,9 +1,9 @@
 """
-USDM to Define-360i JSON Processor
+USDM to Define JSON Processor
 ===================================
 
 This module processes USDM (Unified Study Data Model) JSON files and generates
-Define-360i JSON structures with value-level metadata organized in "slices".
+Define-JSON structures with value-level metadata organized in "slices".
 
 The major difference from the standard Define-JSON format is that variable-level
 metadata (VLM) is organized under "slices" within each itemGroup rather than as
@@ -16,26 +16,25 @@
 import argparse
 import hashlib
 import yaml
-import copy
 from jsonata import Jsonata
 from cdisc_library_client import CDISCLibraryClient
 from collections import defaultdict
 from dotenv import load_dotenv
 from datetime import datetime
 
 
-class USDMDefine360iProcessor:
+class USDMDefineJSONProcessor:
     """
-    Processes USDM JSON files and generates Define-360i JSON structure.
+    Processes USDM JSON files and generates Define-JSON structure.
     
-    This class transforms USDM data into Define-360i format where VLM is
+    This class transforms USDM data into Define-JSON format where VLM is
     organized as "slices" within itemGroups instead of separate DataSpecialization
     itemGroups.
     
     Attributes:
         client (CDISCLibraryClient): CDISC Library API client
         usdm_data (dict): Loaded USDM JSON data
-        template (dict): Output Define-360i JSON structure
+        template (dict): Output Define-JSON structure
         datasets_dict (dict): Dataset variables and restricted values
         bc_dict (dict): Biomedical concepts with VLM metadata
         vlm_lookup (dict): VLM metadata keyed by variable name
@@ -60,7 +59,7 @@ def __init__(self, usdm_file, output_template, sdtmig, sdtmct,
         
         Args:
             usdm_file (str): Path to USDM JSON file
-            output_template (str): Path to output Define-360i JSON file
+            output_template (str): Path to output Define-JSON file
             sdtmig (str): SDTM Implementation Guide version
             sdtmct (str): SDTM Controlled Terminology date in yyyy-mm-dd format
             studyversion (int): Study version index in USDM
@@ -101,7 +100,7 @@ def __init__(self, usdm_file, output_template, sdtmig, sdtmct,
             'creationDateTime': '',
             'odmVersion': '1.3.2',
             'fileType': 'Snapshot',
-            'originator': 'Define-360i Processor',
+            'originator': 'Define-JSON Processor',
             'context': 'Other',
             'defineVersion': '2.1.0',
             'studyOID': '',
@@ -2317,7 +2316,7 @@ def save_debug_files(self, prefix="debug"):
 
     def save_output(self):
         """
-        Save the generated Define-360i JSON template to file.
+        Save the generated Define-JSON template to file.
         
         Assembles all processed components (itemGroups, conditions,
         whereClauses, codeLists) into the template and writes to output file.
@@ -2332,7 +2331,7 @@ def save_output(self):
 
     def validate_against_schema(self, schema_file, excel_output=None):
         """
-        Validate the generated Define-360i file against a YAML schema.
+        Validate the generated Define-JSON file against a YAML schema.
         
         This method loads the YAML schema file and validates the generated
         JSON output against it using LinkML validation if available, or
@@ -2765,7 +2764,7 @@ def process(self):
         2. Build VLM lookup tables
         3. Update datasets dictionary with VLM data
         4. Populate study metadata
-        5. Process datasets to generate Define-360i structures
+        5. Process datasets to generate Define-JSON structures
         6. Add CDISC standards information
         7. Save output to JSON file
         8. Save debug files if debug mode is enabled
@@ -2899,7 +2898,7 @@ def _process_variable_codelist(self, var, dataset, restriction_codes=None):
         
         Fetches controlled terminology from CDISC Library, applies restrictions
         if specified, and creates deduplicated codelist entries. Returns single
-        OID string instead of list for 360i format.
+        OID string instead of list for format.
         
         Args:
             var (dict): Variable metadata from SDTMIG
@@ -3086,7 +3085,7 @@ def populate_study_elements(self):
         version_display = self.studyversion + 1
         design_display = self.studydesign + 1
 
-        self.template['fileOID'] = f"ODM.DEFINE-360i.{study_name}.Version{version_display}.Design{design_display}"
+        self.template['fileOID'] = f"ODM.DEFINE-JSON.{study_name}.Version{version_display}.Design{design_display}"
         self.template['creationDateTime'] = time_str
         self.template['studyOID'] = f"ODM.{study_name}.Version{version_display}.Design{design_display}"
         self.template['studyName'] = study_name
@@ -3101,7 +3100,7 @@ def main():
     """
     Main entry point for command-line execution.
     
-    Parses command-line arguments and orchestrates the USDM to Define-360i
+    Parses command-line arguments and orchestrates the USDM to Define-JSON
     transformation process.
     """
     # Reconfigure stdout to UTF-8 so Unicode symbols (✓, ✅, ❌, ⚠️) don't
@@ -3111,7 +3110,7 @@ def main():
         sys.stdout.reconfigure(encoding='utf-8', errors='replace')
 
     parser = argparse.ArgumentParser(
-        description="Process USDM JSON file to generate Define-360i JSON with slices.",
+        description="Process USDM JSON file to generate Define-JSON with slices.",
         epilog="Example: python create_define_json.py --usdm_file data/study.json --output_template output.json --sdtmct 2025-03-28 --validate define.yaml --validation_report validation_report.xlsx"
     )
     parser.add_argument(
@@ -3122,7 +3121,7 @@ def main():
     parser.add_argument(
         "--output_template",
         required=True,
-        help="Path to output Define-360i JSON file"
+        help="Path to output Define-JSON file"
     )
     parser.add_argument(
         "--sdtmct",
@@ -3185,7 +3184,7 @@ def main():
     if args.validate and not args.validation_report:
         parser.error("--validation_report is required when --validate is used")
 
-    processor = USDMDefine360iProcessor(
+    processor = USDMDefineJSONProcessor(
         usdm_file=args.usdm_file,
         output_template=args.output_template,
         sdtmig=args.sdtmig,
@@ -3201,7 +3200,7 @@ def main():
     processor.output_template = args.output_template
     processor.process()
 
-    print(f"\n✅ Define-360i JSON file created successfully: {args.output_template}")
+    print(f"\n✅ Define-JSON file created successfully: {args.output_template}")
 
     # Validate if requested
     if args.validate:

src/define-xml/requirements.txt

@@ -0,0 +1,10 @@
+cdisc_library_client==0.1.6
+jsonata==0.2.5
+jsonschema==4.26.0
+linkml==1.10.0
+linkml_runtime==1.10.0
+openpyxl==3.1.5
+pandas==3.0.1
+python-dotenv==1.2.2
+PyYAML==6.0.2
+PyYAML==6.0.3