Merge pull request #3 from xylar/add-ci-and-unit-tests

Add unit tests and continuous integration

Author: xylar

.github/workflows/pytest.yml

@@ -0,0 +1,28 @@
+name: Pytest
+
+on:
+  push:
+  pull_request:
+
+jobs:
+  tests:
+    name: Run Pytest
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+
+      - name: Set up micromamba environment
+        uses: mamba-org/setup-micromamba@v2
+        with:
+          environment-file: isschecker_env.yml
+          environment-name: isschecker
+          cache-environment: true
+          cache-downloads: true
+          init-shell: >-
+            bash
+
+      - name: Run test suite
+        shell: micromamba-shell {0}
+        run: pytest -v tests/test_compliance_checker.py

.gitignore

@@ -0,0 +1,2 @@
+__pycache__
+.pytest_cache

README.md

@@ -47,17 +47,33 @@ python compliance_checker.py --source-path ./Models/GrIS/ISMIP7/SYNTH1/CORE --va
 
 ## Generating synthetic test files
 
-`test/generate_test_files.py` creates ISMIP7-style NetCDF test files with synthetic data. See [test/README.md](test/README.md) for full options and examples.
+`generate/generate_test_files.py` creates ISMIP7-style NetCDF test files with synthetic data. See [generate/README.md](generate/README.md) for full options and examples.
 
 ```bash
 conda activate isschecker
 
 # Generate 286-year GrIS ctrl xyt variables
-python test/generate_test_files.py --grid GrIS_16000m --scenario ctrl --xyt --nyears 286 --start-year 2015
+python generate/generate_test_files.py --grid GrIS_16000m --scenario ctrl --xyt --nyears 286 --start-year 2015
 
 # Generate 286-year AIS ctrl scalar variables
-python test/generate_test_files.py --grid AIS_16000m --scenario ctrl --scalars --nyears 286 --start-year 2015
+python generate/generate_test_files.py --grid AIS_16000m --scenario ctrl --scalars --nyears 286 --start-year 2015
 
 # List available grids
-python test/generate_test_files.py --list-grids
+python generate/generate_test_files.py --list-grids
 ```
+
+---
+
+## Running Tests
+
+The regression suite uses `pytest` and creates temporary synthetic datasets, then mutates them to verify expected checker failures for naming, missing variables, time-axis problems, and missing attributes.
+
+```bash
+pytest -v tests/test_compliance_checker.py
+```
+
+If you want to retain the files generated during testing you can use:
+```
+pytest -v tests/test_compliance_checker.py --basetemp=/tmp/pytest_tmp
+```
+The files will then be left in `/tmp/pytest_tmp`.  Otherwise, they are cleaned up once tests pass.

compliance_checker.py

@@ -134,15 +134,28 @@ def main() -> None:
     args = _parse_args()
     source_path = args.source_path
     variable_list = args.variable_list
-    workdir = os.getcwd()
 
-    commit_num = _get_commit_number()
+    run_checker(
+        source_path=source_path,
+        variable_list=variable_list,
+        workdir=os.getcwd(),
+    )
+
+
+def run_checker(
+    source_path: str,
+    variable_list: str = DEFAULT_VARIABLE_LIST,
+    workdir: str | None = None,
+    commit_num: str | None = None,
+):
+    workdir = os.path.abspath(workdir or os.getcwd())
+    commit_num = _get_commit_number() if commit_num is None else commit_num
     experiments_ismip7 = _load_experiments_csv(
         os.path.join(workdir, EXPERIMENTS_ISMIP7_CSV_FILENAME)
     )
     ismip_meta, ismip_var, mandatory_variables = _load_criteria(workdir, variable_list)
 
-    _run_compliance_checker(
+    summary = _run_compliance_checker(
         source_path=source_path,
         commit_num=commit_num,
         ismip_meta=ismip_meta,
@@ -152,6 +165,16 @@ def main() -> None:
         criteria_file=VARIABLE_REQUEST_XLSX,
     )
 
+    log_path = os.path.join(source_path, "compliance_checker_log.txt")
+    log_text = ""
+    if os.path.exists(log_path):
+        with open(log_path, "r") as log_file:
+            log_text = log_file.read()
+
+    summary["log_path"] = log_path
+    summary["log_text"] = log_text
+    return summary
+
 
 def _get_commit_number() -> str:
     try:
@@ -250,10 +273,10 @@ def _run_compliance_checker(
     mandatory_variables,
     experiments,
     criteria_file,
-) -> None:
+):
     if not os.path.isdir(source_path):
         print(f"ERROR: Directory not found: '{source_path}'. Please check your --source-path argument.")
-        return
+        return _empty_summary()
 
     try:
         with open(os.path.join(source_path, "compliance_checker_log.txt"), "w") as f:
@@ -267,7 +290,7 @@ def _run_compliance_checker(
                 msg = f"No .nc files found in directory '{source_path}'. Please check your --source-path argument."
                 print(f"ERROR: {msg}")
                 f.write(f"ERROR: {msg}\n")
-                return
+                return _empty_summary()
 
             summary = _process_experiments(
                 log_file=f,
@@ -292,12 +315,29 @@ def _run_compliance_checker(
             total_attr_errors=summary["total_attr_errors"],
             report_naming_issues=summary["report_naming_issues"],
         )
+        return summary
 
     except TypeError as err:
         print(
             "Something went wrong with your dataset. Please, check your file(s) carefully. Error:",
             err,
         )
+        return _empty_summary()
+
+
+def _empty_summary() -> dict:
+    return {
+        "exp_counter": 0,
+        "file_counter": 0,
+        "total_errors": 0,
+        "total_naming_errors": 0,
+        "total_num_errors": 0,
+        "total_spatial_errors": 0,
+        "total_time_errors": 0,
+        "total_attr_errors": 0,
+        "total_file_errors": 0,
+        "report_naming_issues": [],
+    }
 
 
 def _group_files_by_experiment(source_path: str) -> dict:

test/README.md generate/README.mdtest/README.md renamed to generate/README.md

@@ -1,14 +1,14 @@
 # ISMIP7 NetCDF generator
 
-`test/generate_test_files.py` creates ISMIP7-style NetCDF test files with synthetic data, one file per variable, following the naming convention and grid definitions used by the compliance checker.
+`generate/generate_test_files.py` creates ISMIP7-style NetCDF test files with synthetic data, one file per variable, following the naming convention and grid definitions used by the compliance checker.
 
 Files are written to `Models/{GrIS|AIS}/ISMIP7/SYNTH1/CORE/`.
 
 ## Usage
 
 ```bash
 conda activate isschecker
-python test/generate_test_files.py [OPTIONS]
+python generate/generate_test_files.py [OPTIONS]
 ```
 
 ## Key options
@@ -32,22 +32,22 @@ python test/generate_test_files.py [OPTIONS]
 
 ```bash
 # List available grids
-python test/generate_test_files.py --list-grids
+python generate/generate_test_files.py --list-grids
 
 # Generate 286-year GrIS ctrl files (x,y,t variables)
-python test/generate_test_files.py --grid GrIS_16000m --scenario ctrl \
+python generate/generate_test_files.py --grid GrIS_16000m --scenario ctrl \
   --xyt --nyears 286 --start-year 2015
 
 # Generate 286-year AIS ssp370 files
-python test/generate_test_files.py --grid AIS_08000m --scenario ssp370 \
+python generate/generate_test_files.py --grid AIS_08000m --scenario ssp370 \
   --xyt --nyears 286 --start-year 2015
 
 # Generate scalar-only variables
-python test/generate_test_files.py --grid GrIS_16000m --scenario ctrl \
+python generate/generate_test_files.py --grid GrIS_16000m --scenario ctrl \
   --scalars --nyears 286 --start-year 2015
 
 # Generate both 3D and scalar variables, including non-mandatory ones
-python test/generate_test_files.py --grid GrIS_16000m --scenario ctrl \
+python generate/generate_test_files.py --grid GrIS_16000m --scenario ctrl \
   --xyt --scalars --include-non-mandatory --nyears 286 --start-year 2015
 ```