Update pre-commit hook and failed unittest

Author: lbesnard

.pre-commit-config.yaml

@@ -59,7 +59,7 @@ repos:
     hooks:
       - id: validate-json-configs
         name: Validate dataset JSON configs with Pydantic
-        entry: python3 aodn_cloud_optimised/bin/generic_cloud_optimised_creation.py --validate-configs aodn_cloud_optimised/config/dataset
+        entry: python3 aodn_cloud_optimised/bin/pydantic_precommit_hook.py --validate-configs aodn_cloud_optimised/config/dataset
         language: system
         types: [json]
         pass_filenames: false

aodn_cloud_optimised/bin/generic_cloud_optimised_creation.py

@@ -409,58 +409,6 @@ def resolve_dataset_config_path(config_arg: str) -> str:
     return str(config_path)
 
 
-def validate_all_configs(config_dir: str, exclude_regex: Optional[str] = None) -> int:
-    """Validate all JSON config files in a directory using the DatasetConfig schema.
-
-    Args:
-        config_dir: Path to the directory containing JSON configuration files.
-        exclude_regex: Optional regex pattern to exclude files.
-
-    Returns:
-        Number of invalid configurations (0 if all valid).
-    """
-    config_path = Path(config_dir)
-    if not config_path.is_dir():
-        print(f"❌ Provided path is not a directory: {config_dir}")
-        return 1
-
-    exclude_pattern = re.compile(exclude_regex) if exclude_regex else None
-
-    json_files = sorted(
-        p
-        for p in config_path.glob("*.json")
-        if p.is_file() and not (exclude_pattern and exclude_pattern.match(str(p)))
-    )
-
-    if not json_files:
-        print(f"ℹ️ No JSON files to validate in {config_dir}")
-        return 0
-
-    print(f"🔍 Validating {len(json_files)} config file(s) in {config_dir}")
-    errors = 0
-    for json_file in json_files:
-        try:
-            with open(json_file, "r") as f:
-                raw = json.load(f)
-            DatasetConfig.model_validate(raw)
-        except ValidationError as e:
-            print(f"\n❌ Validation failed in: {json_file}")
-            print("─" * 80)
-            print(e)
-            print("─" * 80)
-            errors += 1
-        except Exception as e:
-            print(f"\n❌ Error reading {json_file}: {e}")
-            errors += 1
-
-    if errors > 0:
-        print(f"\n❌ {errors} configuration file(s) failed validation.")
-    else:
-        print("✅ All configurations are valid.")
-
-    return errors
-
-
 def main():
     parser = argparse.ArgumentParser(
         description="Run cloud-optimised creation using config."
@@ -473,22 +421,9 @@ def main():
         type=str,
         help='JSON string to override config fields. Example:  \'{"run_settings": {"cluster": {"mode": null}, "raise_error": true}}\' ',
     )
-    parser.add_argument(
-        "--validate-configs",
-        type=str,
-        help="Validate all JSON configs in a directory (no processing is done).",
-    )
 
     args = parser.parse_args()
 
-    if args.validate_configs:
-        # Pre-commit pattern: match full path string
-        exclude_pattern = r"^.*dataset_template\.json$"
-        exit_code = validate_all_configs(
-            args.validate_configs, exclude_regex=exclude_pattern
-        )
-        sys.exit(exit_code)
-
     try:
         config = load_config_and_validate(args.config)
     except ValidationError as e:

aodn_cloud_optimised/bin/pydantic_precommit_hook.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+import argparse
+import json
+import logging
+import re
+import sys
+from pathlib import Path
+from typing import Optional
+
+from pydantic import (
+    ValidationError,
+)
+
+from aodn_cloud_optimised.bin.generic_cloud_optimised_creation import DatasetConfig
+
+logger = logging.getLogger(__name__)
+
+
+def validate_all_configs(config_dir: str, exclude_regex: Optional[str] = None) -> int:
+    """Validate all JSON config files in a directory using the DatasetConfig schema.
+
+    Args:
+        config_dir: Path to the directory containing JSON configuration files.
+        exclude_regex: Optional regex pattern to exclude files.
+
+    Returns:
+        Number of invalid configurations (0 if all valid).
+    """
+    config_path = Path(config_dir)
+    if not config_path.is_dir():
+        print(f"❌ Provided path is not a directory: {config_dir}")
+        return 1
+
+    exclude_pattern = re.compile(exclude_regex) if exclude_regex else None
+
+    json_files = sorted(
+        p
+        for p in config_path.glob("*.json")
+        if p.is_file() and not (exclude_pattern and exclude_pattern.match(str(p)))
+    )
+
+    if not json_files:
+        print(f"ℹ️ No JSON files to validate in {config_dir}")
+        return 0
+
+    print(f"🔍 Validating {len(json_files)} config file(s) in {config_dir}")
+    errors = 0
+    for json_file in json_files:
+        try:
+            with open(json_file, "r") as f:
+                raw = json.load(f)
+            DatasetConfig.model_validate(raw)
+        except ValidationError as e:
+            print(f"\n❌ Validation failed in: {json_file}")
+            print("─" * 80)
+            print(e)
+            print("─" * 80)
+            errors += 1
+        except Exception as e:
+            print(f"\n❌ Error reading {json_file}: {e}")
+            errors += 1
+
+    if errors > 0:
+        print(f"\n❌ {errors} configuration file(s) failed validation.")
+    else:
+        print("✅ All configurations are valid.")
+
+    return errors
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Pydantic validator with pre-commit.")
+    parser.add_argument(
+        "--validate-configs",
+        type=str,
+        help="Validate all JSON configs in a directory (no processing is done).",
+    )
+
+    args = parser.parse_args()
+
+    if args.validate_configs:
+        # Pre-commit pattern: match full path string
+        exclude_pattern = r"^.*dataset_template\.json$"
+        exit_code = validate_all_configs(
+            args.validate_configs, exclude_regex=exclude_pattern
+        )
+        sys.exit(exit_code)
+
+
+if __name__ == "__main__":
+    main()

pyproject.toml

@@ -153,6 +153,7 @@ cloud_optimised_vessel_xbt_delayed_qc = "aodn_cloud_optimised.bin.vessel_xbt_del
 cloud_optimised_vessel_xbt_realtime_nonqc = "aodn_cloud_optimised.bin.vessel_xbt_realtime_nonqc:main"
 cloud_optimised_wave_buoy_realtime_nonqc = "aodn_cloud_optimised.bin.wave_buoy_realtime_nonqc:main"
 generic_cloud_optimised_creation = "aodn_cloud_optimised.bin.generic_cloud_optimised_creation:main"
+pydantic_precommit_hook = "aodn_cloud_optimised.bin.pydantic_precommit_hook:main"
 
 #[tool.poetry.include]
 #data = ["aodn_cloud_optimised/config/*.json", "aodn_cloud_optimised/config/dataset/*.json"]

test_aodn_cloud_optimised/resources/wave_buoy_realtime_nonqc.json

@@ -17,6 +17,12 @@
         "memory_limit": "4GB"
       }
     },
+    "paths": [
+      {
+        "s3_uri": "s3://imos-data/Department_of_Transport-Western_Australia/WAVE-BUOYS/REALTIME/",
+        "filter": []
+      }
+    ],
     "batch_size": 300
   },
   "metadata_uuid": "b299cdcd-3dee-48aa-abdd-e0fcdbb9cadc",

test_aodn_cloud_optimised/test_bin_generic_error.py

@@ -1,18 +1,23 @@
 import json
 import logging
 import os
-import sys
 import unittest
 from io import StringIO
-from unittest.mock import patch, MagicMock
+from unittest.mock import MagicMock, patch
 
 import boto3
 import s3fs
+from botocore import UNSIGNED
+from botocore.session import get_session
+from botocore.client import Config as BotoConfig  # avoid conflict with json config
 from moto import mock_aws
 from moto.moto_server.threaded_moto_server import ThreadedMotoServer
 
-from aodn_cloud_optimised.bin.generic_cloud_optimised_creation import main
-from aodn_cloud_optimised.lib.clusterLib import ClusterMode
+from aodn_cloud_optimised.bin.generic_cloud_optimised_creation import (
+    DatasetConfig,
+    main,
+)
+from aodn_cloud_optimised.lib.config import load_dataset_config
 
 ROOT_DIR = os.path.dirname(os.path.abspath(__file__))
 
@@ -28,10 +33,10 @@
     os.path.join(ROOT_DIR, "resources", file_name) for file_name in filenames
 ]
 
+# On purpose wrong!
 DATASET_CONFIG_NC_ACORN_JSON = os.path.join(
     ROOT_DIR,
     "resources",
-    # "radar_TurquoiseCoast_velocity_hourly_averaged_delayed_qc.json",
     "wave_buoy_realtime_nonqc.json",
 )
 
@@ -120,41 +125,75 @@ def tearDown(self):
         self.server.stop()
         del os.environ["RUNNING_UNDER_UNITTEST"]
 
-    @patch("argparse.ArgumentParser.parse_args")
-    def test_main(self, mock_parse_args):
-        # Prepare mock arguments
-        mock_parse_args.return_value = MagicMock(
-            paths=["IMOS/ACORN/gridded_1h-avg-current-map_QC"],
-            # filters=["TURQ"],
-            filters=[".nc"],
-            exclude="FV02",
-            suffix=".nc",
-            raise_error=True,
-            dataset_config=DATASET_CONFIG_NC_ACORN_JSON,
-            clear_existing_data=True,
-            force_previous_parquet_deletion=False,
-            # cluster_mode=ClusterMode.LOCAL,
-            cluster_mode=ClusterMode.NONE,
-            optimised_bucket_name=self.BUCKET_OPTIMISED_NAME,
-            root_prefix_cloud_optimised_path="testing",
-            bucket_raw="imos-data",
-        )
+    def test_main_with_config_and_json_overwrite_fail(self):
+        dataset_config = load_dataset_config(DATASET_CONFIG_NC_ACORN_JSON)
+        config_validated = DatasetConfig.model_validate(dataset_config)
+
+        with open(DATASET_CONFIG_NC_ACORN_JSON) as f:
+            raw_json = f.read()
+            config_validated = DatasetConfig.model_validate_json(raw_json)
+
+        def _mock_boto3_client(service_name, *args, **kwargs):
+            if service_name == "s3":
+                session = get_session()
+                return session.create_client(
+                    "s3",
+                    endpoint_url="http://127.0.0.1:5555",
+                    region_name="us-east-1",
+                    config=BotoConfig(signature_version=UNSIGNED),
+                )
+            raise NotImplementedError(f"Unhandled boto3 service: {service_name}")
+
+        with (
+            patch(
+                "aodn_cloud_optimised.bin.generic_cloud_optimised_creation.load_config_and_validate",
+                new=lambda _: config_validated,
+            ),
+            patch("argparse.ArgumentParser.parse_args") as mock_parse_args,
+            patch("sys.exit") as mock_sys_exit,
+            patch(
+                "aodn_cloud_optimised.lib.s3Tools.boto3.client", new=_mock_boto3_client
+            ),
+        ):
+            mock_parse_args.return_value = MagicMock(
+                config=DATASET_CONFIG_NC_ACORN_JSON,
+                json_overwrite=json.dumps(
+                    {
+                        "run_settings": {
+                            "cluster": {"mode": None},
+                            "raise_error": True,
+                            "force_previous_parquet_deletion": False,
+                            "clear_existing_data": False,
+                            "paths": [
+                                {
+                                    "s3_uri": "s3://imos-data/IMOS/ACORN/gridded_1h-avg-current-map_QC",
+                                    "filter": [],
+                                },
+                            ],
+                            "optimised_bucket_name": self.BUCKET_OPTIMISED_NAME,
+                            "root_prefix_cloud_optimised_path": self.ROOT_PREFIX_CLOUD_OPTIMISED_PATH,
+                        }
+                    }
+                ),
+            )
 
-        # Capture logs
-        log_stream = StringIO()
-        log_handler = logging.StreamHandler(log_stream)
-        logger = logging.getLogger()
-        logger.addHandler(log_handler)
+            log_stream = StringIO()
+            log_handler = logging.StreamHandler(log_stream)
+            logger = logging.getLogger()
+            logger.addHandler(log_handler)
 
-        with self.assertRaises(Exception) as context:
-            main()
+            with self.assertRaises(Exception) as context:
+                main()
+                mock_sys_exit.assert_called_with(0)
 
-        # with self.assertRaises(SystemExit) as cm:
-        #     main()
-        # self.assertEqual(cm.exception.code, 1)  # Verify exit code
+            log_handler.flush()
+            captured_logs = log_stream.getvalue().strip().split("\n")
 
-        # Restore stdout
-        # sys.stdout = sys.__stdout__
+            assert any(
+                "Exception: Error in Cloud Optimised process. Forcing script exit"
+                in log
+                for log in captured_logs
+            )
 
 
 if __name__ == "__main__":