Skip to content

Commit b17091f

Browse files
rbeucherrbeucher
committed
Add comprehensive tests for ACCESS-MOPPeR functionality
- Implement integration tests for CMORiser workflows, including full workflow, memory efficiency, and multiple variable handling. - Create mock datasets for testing atmospheric and oceanic data. - Introduce a mock file system to simulate file operations during tests. - Develop a mock PBS manager to simulate job submission and tracking. - Add performance tests to evaluate memory usage and chunking strategies. - Generate test data scripts for creating mock NetCDF files. - Establish unit tests for BaseCMORiser, batch processing functions, Jinja2 template rendering, and task tracking. - Include pytest configuration for test discovery and management.
1 parent 885e6b9 commit b17091f

23 files changed

Lines changed: 1631 additions & 50 deletions

‎.pre-commit-config.yaml‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,10 @@ repos:
1212
- id: debug-statements # Detects print() and pdb in code
1313

1414
- repo: https://github.com/astral-sh/ruff-pre-commit
15-
rev: v0.3.0 # Check the latest version
15+
rev: v0.8.4 # Use a specific version
1616
hooks:
17-
- id: ruff # Linting
18-
exclude: "^(src/access_mopper/cmor_tables|src/access_mopper/_version.py|src/access_mopper/calc_ocean.py|src/access_mopper/calc_seaice.py)$" # Exclude files or directories
19-
- id: ruff-format # Auto-formatting
20-
exclude: "^(src/access_mopper/cmor_tables|src/access_mopper/_version.py)$"
17+
- id: ruff
18+
args: [--fix, --exit-non-zero-on-fix]
19+
exclude: "_version.py"
20+
- id: ruff-format
21+
exclude: "_version.py"

‎notebooks/Getting_started.ipynb‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -765,7 +765,7 @@
765765
"source": [
766766
"import dask.distributed as dask\n",
767767
"\n",
768-
"client = dask.Client(threads_per_worker = 1)\n",
768+
"client = dask.Client(threads_per_worker=1)\n",
769769
"client"
770770
]
771771
},
@@ -790,6 +790,7 @@
790790
"source": [
791791
"# Here we use netcdf file from a raw ACCESS-ESM run.\n",
792792
"import glob\n",
793+
"\n",
793794
"files = glob.glob(\"../../Test_data/esm1-6/atmosphere/aiihca.pa-0961*_mon.nc\")"
794795
]
795796
},
@@ -815,15 +816,15 @@
815816
"outputs": [],
816817
"source": [
817818
"parent_experiment_config = {\n",
818-
" \"parent_experiment_id\": \"piControl\",\n",
819-
" \"parent_activity_id\": \"CMIP\",\n",
820-
" \"parent_source_id\": \"ACCESS-ESM1-5\",\n",
821-
" \"parent_variant_label\": \"r1i1p1f1\",\n",
822-
" \"parent_time_units\": \"days since 0001-01-01 00:00:00\",\n",
823-
" \"parent_mip_era\": \"CMIP6\",\n",
824-
" \"branch_time_in_child\": 0.0,\n",
825-
" \"branch_time_in_parent\": 54786.0,\n",
826-
" \"branch_method\": \"standard\"\n",
819+
" \"parent_experiment_id\": \"piControl\",\n",
820+
" \"parent_activity_id\": \"CMIP\",\n",
821+
" \"parent_source_id\": \"ACCESS-ESM1-5\",\n",
822+
" \"parent_variant_label\": \"r1i1p1f1\",\n",
823+
" \"parent_time_units\": \"days since 0001-01-01 00:00:00\",\n",
824+
" \"parent_mip_era\": \"CMIP6\",\n",
825+
" \"branch_time_in_child\": 0.0,\n",
826+
" \"branch_time_in_parent\": 54786.0,\n",
827+
" \"branch_method\": \"standard\",\n",
827828
"}"
828829
]
829830
},
@@ -860,8 +861,8 @@
860861
" variant_label=\"r1i1p1f1\",\n",
861862
" grid_label=\"gn\",\n",
862863
" activity_id=\"CMIP\",\n",
863-
" parent_info=parent_experiment_config # <-- This is optional, can be skipped if not needed\n",
864-
" )"
864+
" parent_info=parent_experiment_config, # <-- This is optional, can be skipped if not needed\n",
865+
")"
865866
]
866867
},
867868
{

‎notebooks/Tutorial1_CMORisation_ENSO_Recipes.ipynb‎

Lines changed: 40 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -857,7 +857,7 @@
857857
"source": [
858858
"import dask.distributed as dask\n",
859859
"\n",
860-
"client = dask.Client(threads_per_worker = 1)\n",
860+
"client = dask.Client(threads_per_worker=1)\n",
861861
"client"
862862
]
863863
},
@@ -884,7 +884,9 @@
884884
"metadata": {},
885885
"outputs": [],
886886
"source": [
887-
"ROOT_FOLDER = \"/g/data/p73/archive/CMIP7/ACCESS-ESM1-6/spinup/JuneSpinUp-JuneSpinUp-bfaa9c5b/\""
887+
"ROOT_FOLDER = (\n",
888+
" \"/g/data/p73/archive/CMIP7/ACCESS-ESM1-6/spinup/JuneSpinUp-JuneSpinUp-bfaa9c5b/\"\n",
889+
")"
888890
]
889891
},
890892
{
@@ -905,6 +907,7 @@
905907
"outputs": [],
906908
"source": [
907909
"import glob\n",
910+
"\n",
908911
"FILES = glob.glob(ROOT_FOLDER + \"output[0-4][0-9][0-9]/atmosphere/netCDF/*mon.nc\")"
909912
]
910913
},
@@ -937,15 +940,15 @@
937940
"outputs": [],
938941
"source": [
939942
"parent_experiment_config = {\n",
940-
" \"parent_experiment_id\": \"piControl\",\n",
941-
" \"parent_activity_id\": \"CMIP\",\n",
942-
" \"parent_source_id\": \"ACCESS-ESM1-5\",\n",
943-
" \"parent_variant_label\": \"r1i1p1f1\",\n",
944-
" \"parent_time_units\": \"days since 0001-01-01 00:00:00\",\n",
945-
" \"parent_mip_era\": \"CMIP6\",\n",
946-
" \"branch_time_in_child\": 0.0,\n",
947-
" \"branch_time_in_parent\": 54786.0,\n",
948-
" \"branch_method\": \"standard\"\n",
943+
" \"parent_experiment_id\": \"piControl\",\n",
944+
" \"parent_activity_id\": \"CMIP\",\n",
945+
" \"parent_source_id\": \"ACCESS-ESM1-5\",\n",
946+
" \"parent_variant_label\": \"r1i1p1f1\",\n",
947+
" \"parent_time_units\": \"days since 0001-01-01 00:00:00\",\n",
948+
" \"parent_mip_era\": \"CMIP6\",\n",
949+
" \"branch_time_in_child\": 0.0,\n",
950+
" \"branch_time_in_parent\": 54786.0,\n",
951+
" \"branch_method\": \"standard\",\n",
949952
"}"
950953
]
951954
},
@@ -983,8 +986,8 @@
983986
" variant_label=\"r1i1p1f1\",\n",
984987
" grid_label=\"gn\",\n",
985988
" activity_id=\"CMIP\",\n",
986-
" parent_info=parent_experiment_config # <-- This is optional, can be skipped if not needed\n",
987-
" )"
989+
" parent_info=parent_experiment_config, # <-- This is optional, can be skipped if not needed\n",
990+
")"
988991
]
989992
},
990993
{
@@ -1022,7 +1025,9 @@
10221025
"metadata": {},
10231026
"outputs": [],
10241027
"source": [
1025-
"FILES = glob.glob(ROOT_FOLDER + \"output[0-4][0-9][0-9]/ocean/ocean-2d-surface_temp-1monthly-mean*.nc\")\n",
1028+
"FILES = glob.glob(\n",
1029+
" ROOT_FOLDER + \"output[0-4][0-9][0-9]/ocean/ocean-2d-surface_temp-1monthly-mean*.nc\"\n",
1030+
")\n",
10261031
"len(FILES)"
10271032
]
10281033
},
@@ -1041,8 +1046,8 @@
10411046
" variant_label=\"r1i1p1f1\",\n",
10421047
" grid_label=\"gn\",\n",
10431048
" activity_id=\"CMIP\",\n",
1044-
" parent_info=parent_experiment_config # <-- This is optional, can be skipped if not needed\n",
1045-
" )"
1049+
" parent_info=parent_experiment_config, # <-- This is optional, can be skipped if not needed\n",
1050+
")"
10461051
]
10471052
},
10481053
{
@@ -1099,8 +1104,8 @@
10991104
" variant_label=\"r1i1p1f1\",\n",
11001105
" grid_label=\"gn\",\n",
11011106
" activity_id=\"CMIP\",\n",
1102-
" parent_info=parent_experiment_config # <-- This is optional, can be skipped if not needed\n",
1103-
" )"
1107+
" parent_info=parent_experiment_config, # <-- This is optional, can be skipped if not needed\n",
1108+
")"
11041109
]
11051110
},
11061111
{
@@ -1157,8 +1162,8 @@
11571162
" variant_label=\"r1i1p1f1\",\n",
11581163
" grid_label=\"gn\",\n",
11591164
" activity_id=\"CMIP\",\n",
1160-
" parent_info=parent_experiment_config # <-- This is optional, can be skipped if not needed\n",
1161-
" )"
1165+
" parent_info=parent_experiment_config, # <-- This is optional, can be skipped if not needed\n",
1166+
")"
11621167
]
11631168
},
11641169
{
@@ -1196,7 +1201,9 @@
11961201
"metadata": {},
11971202
"outputs": [],
11981203
"source": [
1199-
"FILES = glob.glob(ROOT_FOLDER + \"output[0-4][0-9][0-9]/ocean/ocean-2d-sea_level-1monthly-mean*.nc\")\n",
1204+
"FILES = glob.glob(\n",
1205+
" ROOT_FOLDER + \"output[0-4][0-9][0-9]/ocean/ocean-2d-sea_level-1monthly-mean*.nc\"\n",
1206+
")\n",
12001207
"len(FILES)"
12011208
]
12021209
},
@@ -1215,8 +1222,8 @@
12151222
" variant_label=\"r1i1p1f1\",\n",
12161223
" grid_label=\"gn\",\n",
12171224
" activity_id=\"CMIP\",\n",
1218-
" parent_info=parent_experiment_config # <-- This is optional, can be skipped if not needed\n",
1219-
" )"
1225+
" parent_info=parent_experiment_config, # <-- This is optional, can be skipped if not needed\n",
1226+
")"
12201227
]
12211228
},
12221229
{
@@ -1273,8 +1280,8 @@
12731280
" variant_label=\"r1i1p1f1\",\n",
12741281
" grid_label=\"gn\",\n",
12751282
" activity_id=\"CMIP\",\n",
1276-
" parent_info=parent_experiment_config # <-- This is optional, can be skipped if not needed\n",
1277-
" )"
1283+
" parent_info=parent_experiment_config, # <-- This is optional, can be skipped if not needed\n",
1284+
")"
12781285
]
12791286
},
12801287
{
@@ -1331,8 +1338,8 @@
13311338
" variant_label=\"r1i1p1f1\",\n",
13321339
" grid_label=\"gn\",\n",
13331340
" activity_id=\"CMIP\",\n",
1334-
" parent_info=parent_experiment_config # <-- This is optional, can be skipped if not needed\n",
1335-
" )"
1341+
" parent_info=parent_experiment_config, # <-- This is optional, can be skipped if not needed\n",
1342+
")"
13361343
]
13371344
},
13381345
{
@@ -1389,8 +1396,8 @@
13891396
" variant_label=\"r1i1p1f1\",\n",
13901397
" grid_label=\"gn\",\n",
13911398
" activity_id=\"CMIP\",\n",
1392-
" parent_info=parent_experiment_config # <-- This is optional, can be skipped if not needed\n",
1393-
" )"
1399+
" parent_info=parent_experiment_config, # <-- This is optional, can be skipped if not needed\n",
1400+
")"
13941401
]
13951402
},
13961403
{
@@ -1447,8 +1454,8 @@
14471454
" variant_label=\"r1i1p1f1\",\n",
14481455
" grid_label=\"gn\",\n",
14491456
" activity_id=\"CMIP\",\n",
1450-
" parent_info=parent_experiment_config # <-- This is optional, can be skipped if not needed\n",
1451-
" )"
1457+
" parent_info=parent_experiment_config, # <-- This is optional, can be skipped if not needed\n",
1458+
")"
14521459
]
14531460
},
14541461
{
@@ -1505,8 +1512,8 @@
15051512
" variant_label=\"r1i1p1f1\",\n",
15061513
" grid_label=\"gn\",\n",
15071514
" activity_id=\"CMIP\",\n",
1508-
" parent_info=parent_experiment_config # <-- This is optional, can be skipped if not needed\n",
1509-
" )"
1515+
" parent_info=parent_experiment_config, # <-- This is optional, can be skipped if not needed\n",
1516+
")"
15101517
]
15111518
},
15121519
{

‎tests/conftest.py‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
import importlib.resources as resources
2+
import shutil
3+
import tempfile
4+
from pathlib import Path
5+
6+
import numpy as np
7+
import pandas as pd
8+
import pytest
9+
import xarray as xr
10+
11+
# Use your existing DATA_DIR
12+
DATA_DIR = Path(__file__).parent / "data"
13+
14+
15+
@pytest.fixture
16+
def temp_dir():
17+
"""Create a temporary directory for test outputs."""
18+
temp_path = Path(tempfile.mkdtemp())
19+
yield temp_path
20+
shutil.rmtree(temp_path)
21+
22+
23+
@pytest.fixture
24+
def parent_experiment_config():
25+
"""Parent experiment configuration - same as your existing one."""
26+
return {
27+
"parent_experiment_id": "piControl",
28+
"parent_activity_id": "CMIP",
29+
"parent_source_id": "ACCESS-ESM1-5",
30+
"parent_variant_label": "r1i1p1f1",
31+
"parent_time_units": "days since 0001-01-01 00:00:00",
32+
"parent_mip_era": "CMIP6",
33+
"branch_time_in_child": 0.0,
34+
"branch_time_in_parent": 54786.0,
35+
"branch_method": "standard",
36+
}
37+
38+
39+
@pytest.fixture
40+
def mock_netcdf_dataset():
41+
"""Create a mock xarray dataset that mimics ACCESS model output."""
42+
time = pd.date_range("2000-01-01", periods=12, freq="M")
43+
lat = np.linspace(-90, 90, 10)
44+
lon = np.linspace(0, 360, 20)
45+
46+
# Create realistic test data
47+
temp_data = 273.15 + 15 + 10 * np.random.random((12, 10, 20))
48+
precip_data = np.abs(5e-5 * np.random.random((12, 10, 20)))
49+
50+
ds = xr.Dataset(
51+
{
52+
"temp": (
53+
["time", "lat", "lon"],
54+
temp_data,
55+
{
56+
"units": "K",
57+
"standard_name": "air_temperature",
58+
"long_name": "Near-Surface Air Temperature",
59+
},
60+
),
61+
"precip": (
62+
["time", "lat", "lon"],
63+
precip_data,
64+
{
65+
"units": "kg m-2 s-1",
66+
"standard_name": "precipitation_flux",
67+
"long_name": "Precipitation",
68+
},
69+
),
70+
},
71+
coords={
72+
"time": ("time", time),
73+
"lat": ("lat", lat, {"units": "degrees_north"}),
74+
"lon": ("lon", lon, {"units": "degrees_east"}),
75+
},
76+
)
77+
78+
return ds
79+
80+
81+
@pytest.fixture
82+
def mock_config():
83+
"""Standard configuration for testing."""
84+
return {
85+
"experiment_id": "historical",
86+
"source_id": "ACCESS-ESM1-5",
87+
"variant_label": "r1i1p1f1",
88+
"grid_label": "gn",
89+
"activity_id": "CMIP",
90+
}
91+
92+
93+
@pytest.fixture
94+
def batch_config():
95+
"""Sample batch configuration for testing."""
96+
return {
97+
"variables": ["Amon.pr", "Amon.tas"],
98+
"experiment_id": "historical",
99+
"source_id": "ACCESS-ESM1-5",
100+
"variant_label": "r1i1p1f1",
101+
"grid_label": "gn",
102+
"activity_id": "CMIP",
103+
"input_folder": "/test/input",
104+
"output_folder": "/test/output",
105+
"file_patterns": {
106+
"Amon.pr": "/output[0-4][0-9][0-9]/atmosphere/netCDF/*mon.nc",
107+
"Amon.tas": "/output[0-4][0-9][0-9]/atmosphere/netCDF/*mon.nc",
108+
},
109+
"cpus_per_node": 4,
110+
"mem": "16GB",
111+
"walltime": "01:00:00",
112+
"queue": "normal",
113+
}
114+
115+
116+
def load_filtered_variables(mappings):
117+
"""Load variables from mapping files - keeping your existing function."""
118+
with resources.files("access_mopper.mappings").joinpath(mappings).open() as f:
119+
df = pd.read_json(f, orient="index")
120+
return df.index.tolist()

‎tests/e2e/__init__.py‎

Whitespace-only changes.

0 commit comments

Comments
 (0)