Merge pull request #883 from cdisc-org/datasets

RamilCDISC · web-flow · commit bd1866982ea8 · 2024-11-04T17:16:57.000-06:00
test XPT and logger
diff --git a/README.md b/README.md
@@ -206,6 +206,10 @@ To obtain an api key, please follow the instructions found here: <https://wiki.c
   -ca, --cache TEXT               Relative path to cache files containing pre
                                   loaded metadata and rules
   -dp, --dataset-path TEXT        Absolute path to dataset file
+  -d, --data TEXT                 Path to directory containing data files
+  -l, --log-level [info|debug|error|critical|disabled|warn]
+                                  Sets log level for engine logs, logs are
+                                  disabled by default
   -s, --standard TEXT             CDISC standard to validate against
                                   [required]
   -v, --version TEXT              Standard version to validate against
diff --git a/cdisc_rules_engine/enums/dataformat_test_types.py b/cdisc_rules_engine/enums/dataformat_test_types.py
@@ -0,0 +1,6 @@
+from cdisc_rules_engine.enums.base_enum import BaseEnum
+
+
+class TestDataFormatTypes(BaseEnum):
+    JSON = "JSON"
+    XPT = "XPT"
diff --git a/cdisc_rules_engine/models/test_args.py b/cdisc_rules_engine/models/test_args.py
@@ -4,7 +4,8 @@
     "TestArgs",
     [
         "cache",
-        "dataset_path",
+        "dataset_paths",
+        "log_level",
         "rule",
         "standard",
         "version",
diff --git a/core.py b/core.py
@@ -13,6 +13,7 @@
 from cdisc_rules_engine.enums.default_file_paths import DefaultFilePaths
 from cdisc_rules_engine.enums.progress_parameter_options import ProgressParameterOptions
 from cdisc_rules_engine.enums.report_types import ReportTypes
+from cdisc_rules_engine.enums.dataformat_test_types import TestDataFormatTypes
 from cdisc_rules_engine.enums.dataformat_types import DataFormatTypes
 from cdisc_rules_engine.models.validation_args import Validation_args
 from cdisc_rules_engine.models.test_args import TestArgs
@@ -34,8 +35,11 @@
 from version import __version__
 
 
-def valid_data_file(data_path: list) -> Tuple[list, set]:
-    allowed_formats = [format.value for format in DataFormatTypes]
+def valid_data_file(data_path: list, test: bool = False) -> Tuple[list, set]:
+    if test:
+        allowed_formats = [format.value for format in TestDataFormatTypes]
+    else:
+        allowed_formats = [format.value for format in DataFormatTypes]
     found_formats = set()
     file_list = []
     for file in data_path:
@@ -467,9 +471,22 @@ def list_rules(
 @click.option(
     "-dp",
     "--dataset-path",
-    required=True,
+    required=False,
     help="Absolute path to dataset file",
 )
+@click.option(
+    "-d",
+    "--data",
+    required=False,
+    help="Path to directory containing data files",
+)
+@click.option(
+    "-l",
+    "--log-level",
+    default="disabled",
+    type=click.Choice(["info", "debug", "error", "critical", "disabled", "warn"]),
+    help="Sets log level for engine logs, logs are disabled by default",
+)
 @click.option(
     "-r",
     "--rule",
@@ -514,6 +531,8 @@ def test(
     ctx,
     cache_path: str,
     dataset_path: Tuple[str],
+    data: str,
+    log_level: str,
     rule: str,
     standard: str,
     version: str,
@@ -527,6 +546,34 @@ def test(
     validate_xml,
     define_xml_path: str,
 ):
+    logger = logging.getLogger("tester")
+    if data:
+        if dataset_path:
+            logger.error(
+                "Argument --dataset-path cannot be used together with argument --data"
+            )
+            ctx.exit()
+        dataset_paths, found_formats = valid_data_file(
+            [str(Path(data).joinpath(fn)) for fn in os.listdir(data)]
+        )
+        if len(found_formats) > 1:
+            logger.error(
+                f"Argument --data contains more than one allowed file format ({', '.join(found_formats)})."  # noqa: E501
+            )
+            ctx.exit()
+    elif dataset_path:
+        dataset_paths, found_formats = valid_data_file([dataset_path])
+        if len(found_formats) > 1:
+            logger.error(
+                f"Argument --dataset_path contains more than one allowed file format ({', '.join(found_formats)})."  # noqa: E501
+            )
+            ctx.exit()
+    else:
+        logger.error(
+            "You must pass one of the following arguments: --dataset-path, --data"
+        )
+        # no need to define dataset_paths here, the program execution will stop
+        ctx.exit()
     external_dictionaries = ExternalDictionariesContainer(
         {
             DictionaryTypes.MEDDRA.value: meddra,
@@ -539,7 +586,8 @@ def test(
     validate_xml = True if validate_xml.lower() in ("y", "yes") else False
     args = TestArgs(
         cache_path,
-        dataset_path,
+        dataset_paths,
+        log_level,
         rule,
         standard,
         version,
diff --git a/scripts/test_rule.py b/scripts/test_rule.py
@@ -28,11 +28,11 @@
 from cdisc_rules_engine.models.rule import Rule
 from cdisc_rules_engine.utilities.utils import generate_report_filename
 from scripts.script_utils import (
+    get_datasets,
     fill_cache_with_dictionaries,
     get_cache_service,
     get_library_metadata_from_cache,
 )
-from cdisc_rules_engine.utilities.utils import get_directory_path
 from cdisc_rules_engine.enums.progress_parameter_options import ProgressParameterOptions
 
 """
@@ -54,7 +54,7 @@ def validate_single_rule(
     library_metadata: LibraryMetadataContainer,
     rule: dict = None,
 ):
-    set_log_level("ERROR")
+    set_log_level(args)
     rule["conditions"] = ConditionCompositeFactory.get_condition_composite(
         rule["conditions"]
     )
@@ -68,11 +68,11 @@ def validate_single_rule(
         define_xml_path=args.define_xml_path,
         library_metadata=library_metadata,
         validate_xml=args.validate_xml,
-        dataset_paths=args.dataset_path,
+        dataset_paths=args.dataset_paths,
     )
     validated_domains = set()
     results = []
-    directory = get_directory_path(args.dataset_path)
+    directory = os.path.dirname(args.dataset_paths[0])
 
     if rule.get("sensitivity").lower() == "study":
         results.append(
@@ -103,39 +103,78 @@ def validate_single_rule(
     return RuleValidationResult(rule, results)
 
 
-def set_log_level(level: str):
-    if level == "disabled":
+def initialize_logger(disabled, log_level):
+    if disabled:
         engine_logger.disabled = True
     else:
-        engine_logger.setLevel(level)
+        engine_logger.disabled = False
+        engine_logger.setLevel(log_level)
+
+
+def set_log_level(args):
+    if args.log_level.lower() == "disabled":
+        engine_logger.disabled = True
+    else:
+        engine_logger.setLevel(args.log_level.lower())
 
 
 def test(args: TestArgs):
-    set_log_level("ERROR")
+    set_log_level(args)
     # fill cache
     CacheManager.register("RedisCacheService", RedisCacheService)
     CacheManager.register("InMemoryCacheService", InMemoryCacheService)
     manager = CacheManager()
     manager.start()
     shared_cache = get_cache_service(manager)
+    engine_logger.info(f"Populating cache, cache path: {args.cache}")
     library_metadata: LibraryMetadataContainer = get_library_metadata_from_cache(args)
     # install dictionaries if needed
     dictionary_versions = fill_cache_with_dictionaries(shared_cache, args)
     with open(args.rule, "r", encoding="utf-8") as f:
         rules = [Rule.from_cdisc_metadata(json.load(f))]
-    with open(args.dataset_path, "r") as f:
-        data_json = json.load(f)
-    datasets = [DummyDataset(data) for data in data_json.get("datasets", [])]
     data_service_factory = DataServiceFactory(
         config, shared_cache, args.standard, args.version
     )
-    dummy_data_service = data_service_factory.get_dummy_data_service(datasets)
     data_service = data_service_factory.get_data_service()
-
+    datasets = []
+    for dataset_path in args.dataset_paths:
+        try:
+            with open(dataset_path, "r") as f:
+                data_json = json.load(f)
+                datasets.extend(
+                    [DummyDataset(data) for data in data_json.get("datasets", [])]
+                )
+        except Exception as e:
+            engine_logger.info(f"Dataset {dataset_path} is not encoded in {e}")
+    if not datasets:
+        engine_logger.info(
+            "No datasets loaded from JSON files, attempting to load using data service"
+        )
+        try:
+            datasets = [
+                DummyDataset(dataset)
+                for dataset in get_datasets(data_service, args.dataset_paths)
+            ]
+            for dataset_path in args.dataset_paths:
+                filename = os.path.basename(dataset_path).lower()
+                matching_dataset = next(
+                    dataset
+                    for dataset in datasets
+                    if dataset.filename.lower() == filename
+                )
+                df = data_service.get_dataset(dataset_name=dataset_path)
+                matching_dataset.data = df.data
+        except Exception as e:
+            engine_logger.error(f"Data service failed to load datasets: {e}")
+    dummy_data_service = data_service_factory.get_dummy_data_service(datasets)
     start = time.time()
     results = []
+    # instantiate logger in each child process to maintain log level
+    initializer = partial(
+        initialize_logger, engine_logger.disabled, engine_logger._logger.level
+    )
     # run each rule in a separate process
-    with Pool(10) as pool:
+    with Pool(10, initializer=initializer) as pool:
         with click.progressbar(
             length=len(rules),
             fill_char=click.style("\u2588", fg="green"),
@@ -163,7 +202,7 @@ def test(args: TestArgs):
     validation_args = Validation_args(
         None,
         None,
-        [args.dataset_path],
+        args.dataset_paths,
         None,
         os.path.join("resources", "templates", "report-template.xlsx"),
         args.standard,
diff --git a/tests/QARegressionTests/test_core/test_test_command.py b/tests/QARegressionTests/test_core/test_test_command.py
@@ -1,4 +1,5 @@
 import os
+import re
 import subprocess
 import unittest
 
@@ -19,24 +20,57 @@ def run_command(self, command):
         exit_code = process.returncode
         return exit_code, stdout.lower(), stderr.lower()
 
+    def test_test_command_with_all_options_one_data_source(self):
+        command = (
+            f"python core.py test "
+            f"-c {os.path.join('resources', 'cache')} "
+            f"-dp {os.path.join('tests', 'resources', 'CoreIssue164', 'Positive_Dataset.json')} "
+            f"-r {os.path.join('tests', 'resources', 'Rule-CG0027.json')} "
+            f"--whodrug "
+            f"{os.path.join('tests', 'resources', 'dictionaries', 'whodrug')} "
+            f"--meddra {os.path.join('tests', 'resources', 'dictionaries', 'meddra')} "
+            f"-s sdtmig "
+            f"-v 3.4 "
+            f"-dv 2.1 "
+            f"-dxp {os.path.join('tests', 'resources','define.xml')} "
+            f"-l error"
+        )
+        exit_code, stdout, stderr = self.run_command(command)
+        self.assertEqual(exit_code, 0)
+        self.assertFalse(self.error_keyword in stdout)
+        self.assertEqual(stderr, "", f"Error while executing command:\n{stderr}")
+
     def test_test_command_with_all_options(self):
         command = (
             f"python core.py test "
             f"-c {os.path.join('resources', 'cache')} "
             f"-dp {os.path.join('tests', 'resources', 'CG0027-positive.json')} "
+            f"-d {os.path.join('tests', 'resources', 'report_test_data')} "
             f"-r {os.path.join('tests', 'resources', 'Rule-CG0027.json')} "
             f"--whodrug "
             f"{os.path.join('tests', 'resources', 'dictionaries', 'whodrug')} "
             f"--meddra {os.path.join('tests', 'resources', 'dictionaries', 'meddra')} "
             f"-s sdtmig "
             f"-v 3.4 "
             f"-dv 2.1 "
-            f"-dxp {os.path.join('tests', 'resources','define.xml')}"
+            f"-dxp {os.path.join('tests', 'resources','define.xml')} "
+            f"-l error"
         )
         exit_code, stdout, stderr = self.run_command(command)
         self.assertEqual(exit_code, 0)
         self.assertFalse(self.error_keyword in stdout)
-        self.assertEqual(stderr, "", f"Error while executing command:\n{stderr}")
+        self.assertFalse(self.error_keyword in stdout)
+        expected_pattern = (
+            r"\[error \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} - "
+            r"core\.py:\d+\] - argument --dataset-path cannot be used together "
+            r"with argument --data\n"
+        )
+        error_msg = (
+            f"Error message format doesn't match expected pattern.\n"
+            f"Actual: {stderr}\n"
+            f"Expected pattern: {expected_pattern}"
+        )
+        self.assertTrue(re.match(expected_pattern, stderr), error_msg)
 
     def test_test_command_without_dataset_path(self):
         command = (
@@ -45,10 +79,18 @@ def test_test_command_without_dataset_path(self):
             f"-r {os.path.join('tests', 'resources', 'Rule-CG0027.json')}"
         )
         exit_code, stdout, stderr = self.run_command(command)
-        self.assertNotEqual(exit_code, 0)
-        self.assertNotEqual(
-            stderr, "", f"Error not raised while executing invalid command:\n{stderr}"
+        self.assertEqual(exit_code, 0)
+        expected_pattern = (
+            r"\[error \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3} - "
+            r"core\.py:\d+\] - you must pass one of the following arguments: "
+            r"--dataset-path, --data\n"
+        )
+        error_msg = (
+            f"Error message format doesn't match expected pattern.\n"
+            f"Actual: {stderr}\n"
+            f"Expected pattern: {expected_pattern}"
         )
+        self.assertTrue(re.match(expected_pattern, stderr), error_msg)
 
     def test_test_command_without_rule(self):
         command = (
@@ -126,7 +168,8 @@ def test_test_command_with_vx_as_yes(self):
             f"-vx y"
         )
         exit_code, stdout, stderr = self.run_command(command)
-        self.assertFalse(stderr == "")
+        self.assertEqual(exit_code, 0)
+        self.assertTrue(stderr == "")
 
     def tearDown(self):
         for file_name in os.listdir("."):
