change csv conversion to an engine output format

Author: gerrycampion

.github/workflows/validate-published-rules.yml

@@ -98,7 +98,7 @@ jobs:
             CORE_IDS_ARG="--core-ids ${{ inputs.core_ids }}"
           fi
 
-          # CORE_IDS_ARG="--core-ids CORE-000001 CORE-000002 CORE-000003 CORE-000004" # TODO: Remove this before merging to main
+          CORE_IDS_ARG="--core-ids CORE-000001 CORE-000002 CORE-000003 CORE-000004 CORE-000005 CORE-000006" # TODO: Remove this before merging to main
 
           ./venv/bin/python engine/scripts/validate_published_rules.py \
             --rules-root "$(pwd)/open-rules" \
@@ -116,7 +116,7 @@ jobs:
         with:
           name: published-rules-validation-${{ github.run_id }}
           path: |
-            open-rules/Published/**/results/results.json
+            open-rules/Published/**/results/results.csv
             summary_table.md
             detail_report.md
           if-no-files-found: warn

cdisc_rules_engine/constants/__init__.py

@@ -5,7 +5,7 @@
 
 NULL_FLAVORS = ["", None, {}, {None}, [], [None], np.nan]
 
-KNOWN_REPORT_EXTENSIONS = [".json", ".xlsx", ".xls"]
+KNOWN_REPORT_EXTENSIONS = [".json", ".xlsx", ".xls", ".csv"]
 
 VALIDATION_FORMATS_MESSAGE = (
     "SAS V5 XPT, Dataset-JSON (JSON or NDJSON), or Excel (XLSX)"

cdisc_rules_engine/enums/report_types.py

@@ -4,3 +4,4 @@
 class ReportTypes(BaseEnum):
     XLSX = "XLSX"
     JSON = "JSON"
+    CSV = "CSV"

cdisc_rules_engine/services/reporting/base_report_data.py

@@ -1,4 +1,4 @@
-from abc import ABC
+from abc import ABC, abstractmethod
 from io import IOBase
 from typing import Iterable
 
@@ -53,3 +53,11 @@ def process_values(values: list[str]) -> list[str]:
             else:
                 processed_values.append(value)
         return processed_values
+
+    @abstractmethod
+    def get_csv_rows(self) -> tuple[list[str], list[list[str]]]:
+        """
+        Return (header, rows) for the CSV output format.
+        Each row is a list of string values matching the header columns.
+        """
+        pass

cdisc_rules_engine/services/reporting/csv_report.py

@@ -0,0 +1,43 @@
+import csv
+import os
+from io import IOBase
+from typing import override
+
+from cdisc_rules_engine.enums.report_types import ReportTypes
+from cdisc_rules_engine.models.validation_args import Validation_args
+from cdisc_rules_engine.services.reporting.base_report_data import BaseReportData
+
+from .base_report import BaseReport
+
+
+class CsvReport(BaseReport):
+    """
+    Writes a results.csv file in the format defined by the report standard,
+    compatible with the cdisc-open-rules test harness baselines.
+    """
+
+    def __init__(
+        self,
+        report_standard: BaseReportData,
+        args: Validation_args,
+        template: IOBase | None = None,
+    ):
+        super().__init__(report_standard, args, template)
+
+    @property
+    @override
+    def _file_ext(self) -> str:
+        return ReportTypes.CSV.value.lower()
+
+    @override
+    def write_report(self) -> None:
+        output_dir = os.path.dirname(self._output_name)
+        if output_dir:
+            os.makedirs(output_dir, exist_ok=True)
+
+        header, rows = self._report_standard.get_csv_rows()
+
+        with open(self._output_name, "w", newline="", encoding="utf-8") as fh:
+            writer = csv.writer(fh)
+            writer.writerow(header)
+            writer.writerows(rows)

cdisc_rules_engine/services/reporting/report_factory.py

@@ -13,6 +13,7 @@
 from .base_report import BaseReport
 from .excel_report import ExcelReport
 from .json_report import JsonReport
+from .csv_report import CsvReport
 
 
 class ReportFactory:
@@ -46,6 +47,7 @@ def __init__(
         self._output_type_service_map: dict[str, Type[BaseReport]] = {
             ReportTypes.XLSX.value: ExcelReport,
             ReportTypes.JSON.value: JsonReport,
+            ReportTypes.CSV.value: CsvReport,
         }
         self._standard_type_map: dict[str, Type[BaseReportData]] = {
             "usdm": USDMReportData,

cdisc_rules_engine/services/reporting/sdtm_report_data.py

@@ -347,6 +347,18 @@ def _generate_error_details(
             )
         return errors
 
+    def get_csv_rows(self) -> tuple[list[str], list[list[str]]]:
+        header = ["Dataset", "Record", "Variable", "Value"]
+        rows = []
+        for issue in self.data_sheets.get("Issue Details", []):
+            dataset = (issue.get("dataset") or "").removesuffix(".csv")
+            record = str(issue.get("row", ""))
+            variables = issue.get("variables") or []
+            values = issue.get("values") or []
+            for variable, value in zip(variables, values):
+                rows.append([dataset, record, variable, str(value)])
+        return header, rows
+
     def get_rules_report_data(self) -> list[dict]:
         """
         Generates the rules report data that goes into the excel export.

cdisc_rules_engine/services/reporting/usdm_report_data.py

@@ -245,6 +245,17 @@ def _generate_error_details(
             )
         return errors
 
+    def get_csv_rows(self) -> tuple[list[str], list[list[str]]]:
+        header = ["path", "attribute", "value"]
+        rows = []
+        for issue in self.data_sheets.get("Issue Details", []):
+            path = issue.get("path") or ""
+            attributes = issue.get("attributes") or []
+            values = issue.get("values") or []
+            for attribute, value in zip(attributes, values):
+                rows.append([path, attribute, str(value)])
+        return header, rows
+
     def get_rules_report_data(self) -> list[dict]:
         """
         Generates the rules report data that goes into the excel export.

docs/cli-reference.md

@@ -69,14 +69,14 @@ python core.py validate --help
 
 ### Output
 
-| Flag                                         | Description                                                                                                           |
-| -------------------------------------------- | --------------------------------------------------------------------------------------------------------------------- |
-| `-o, --output TEXT`                          | Output file path (without extension). Extension is added automatically based on format.                               |
-| `-of, --output-format [JSON\|XLSX]`          | Output format.                                                                                                        |
-| `-rr, --raw-report`                          | Raw output format (JSON only).                                                                                        |
-| `-mr, --max-report-rows INTEGER`             | Max rows in the Issue Details tab of Excel output (default: 1000; 0 = unlimited). Also via `MAX_REPORT_ROWS` env var. |
-| `-me, --max-errors-per-rule INTEGER BOOLEAN` | Limit errors per rule. Format: `-me <limit> <per_dataset_flag>`. See below.                                           |
-| `-rt, --report-template TEXT`                | Path to a custom Excel report template.                                                                               |
+| Flag                                         | Description                                                                                                                     |
+| -------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- |
+| `-o, --output TEXT`                          | Output file path (without extension). Extension is added automatically based on format.                                         |
+| `-of, --output-format [JSON\|XLSX\|CSV]`     | Output format. `CSV` writes issue rows directly (Dataset, Record, Variable, Value) compatible with the open-rules test harness. |
+| `-rr, --raw-report`                          | Raw output format (JSON only).                                                                                                  |
+| `-mr, --max-report-rows INTEGER`             | Max rows in the Issue Details tab of Excel output (default: 1000; 0 = unlimited). Also via `MAX_REPORT_ROWS` env var.           |
+| `-me, --max-errors-per-rule INTEGER BOOLEAN` | Limit errors per rule. Format: `-me <limit> <per_dataset_flag>`. See below.                                                     |
+| `-rt, --report-template TEXT`                | Path to a custom Excel report template.                                                                                         |
 
 #### `--max-errors-per-rule` Detail