Merge branch 'main' of https://github.com/ESMValGroup/ESMValTool

valeriupredoi · valeriupredoi · commit c14443156b37 · 2026-06-09T13:57:23.000+01:00
diff --git a/.github/workflows/run-tests-monitor.yml b/.github/workflows/run-tests-monitor.yml
@@ -33,13 +33,14 @@ jobs:
         environment:
           - test-py312
           - test-py313
+          - test-py314
     name: ${{ matrix.os }} ${{ matrix.environment }}
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           fetch-depth: 0
           persist-credentials: false
-      - uses: prefix-dev/setup-pixi@1b2de7f3351f171c8b4dfeb558c639cb58ed4ec0  # v0.9.5
+      - uses: prefix-dev/setup-pixi@5185adfbffb4bd703da3010310260805d89ebb11  # v0.9.6
         with:
           frozen: true
           environments: ${{ matrix.environment }}
diff --git a/.github/workflows/test-development.yml b/.github/workflows/test-development.yml
@@ -32,7 +32,7 @@ jobs:
         with:
           fetch-depth: 0
           persist-credentials: false
-      - uses: prefix-dev/setup-pixi@1b2de7f3351f171c8b4dfeb558c639cb58ed4ec0  # v0.9.5
+      - uses: prefix-dev/setup-pixi@5185adfbffb4bd703da3010310260805d89ebb11  # v0.9.6
         with:
           frozen: true
           environments: esmvalcore-dev
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -29,13 +29,14 @@ jobs:
         environment:
           - test-py312
           - test-py313
+          - test-py314
     name: ${{ matrix.os }} ${{ matrix.environment }}
     steps:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           fetch-depth: 0
           persist-credentials: false
-      - uses: prefix-dev/setup-pixi@1b2de7f3351f171c8b4dfeb558c639cb58ed4ec0  # v0.9.5
+      - uses: prefix-dev/setup-pixi@5185adfbffb4bd703da3010310260805d89ebb11  # v0.9.6
         with:
           frozen: true
           environments: ${{ matrix.environment }}
diff --git a/.github/workflows/update-lockfiles.yml b/.github/workflows/update-lockfiles.yml
@@ -19,7 +19,7 @@ jobs:
           fetch-depth: 0
           persist-credentials: false
       - name: Set up pixi
-        uses: prefix-dev/setup-pixi@1b2de7f3351f171c8b4dfeb558c639cb58ed4ec0  # v0.9.5
+        uses: prefix-dev/setup-pixi@5185adfbffb4bd703da3010310260805d89ebb11  # v0.9.6
         with:
           run-install: false
       - name: Update lockfiles
diff --git a/doc/sphinx/source/input.rst b/doc/sphinx/source/input.rst
@@ -44,7 +44,7 @@ but many more exist around the world.
 
 If you do not have access to such a facility through your institute or the
 project you are working on, you can request access by applying for the
-`ENES Climate Analytics Service <https://is.enes.org/sdm-climate-analytics-data/>`.
+`ENES Climate Analytics Service <https://is.enes.org/sdm-climate-analytics-data/>`__.
 
 If the options above are not available to you, ESMValTool also offers a feature
 to make it easy to download CMIP6, CMIP5, CMIP3, CORDEX, and obs4MIPs from ESGF.
@@ -66,18 +66,9 @@ ESMValTool can automatically download any required data that is available on
 ESGF.
 This is the recommended approach for first-time users to obtain some data for
 running ESMValTool.
-For example, run
-
-.. code-block:: bash
-
-    esmvaltool run --search_esgf=when_missing examples/recipe_python.yml
-
-to run the default example recipe and automatically download the required data
-to the directory ``~/climate_data``.
-The data only needs to be downloaded once, every following run will reuse
-previously downloaded data stored in this directory.
-See :ref:`esmvalcore:config-esgf` for a more in depth explanation and the
-available configuration options.
+See the section on configuring :ref:`esmvalcore:config-data-sources` for
+details on how to configure ESMValTool to download data from ESGF using
+`intake-esgf <https://intake-esgf.readthedocs.io>`__.
 
 Alternatively, you can use an external tool called
 `esgpull <https://github.com/ESGF/esgf-download>`__
@@ -91,7 +82,7 @@ Observations
 
 Observational and reanalysis products in the standard CF/CMOR format used in
 CMIP and required by ESMValTool are available via the obs4MIPs and ana4mips
-projects at the ESGF (e.g., https://esgf-data.dkrz.de/projects/esgf-dkrz/).
+projects hosted by ESGF (e.g., https://pcmdi.github.io/obs4MIPs/).
 Their use is strongly recommended, when possible.
 
 Other datasets not available in these archives can be obtained by the user from
@@ -119,9 +110,8 @@ All observational datasets are grouped into in three tiers:
 * **Tier 2** other freely available datasets that are not obs4mips. There are no license restrictions. These datasets need to be reformatted to be used with ESMValTool ('CMORization', see above).
 * **Tier 3** restricted datasets. Datasets which require registration to be downloaded or that can only be obtained upon request from the respective authors. License restrictions do not allow us to redistribute Tier 3 datasets. The data have to be obtained and reformatted by the user ('CMORization', see above).
 
-[!NOTE]
-.. _tier3_note:
-For some of the Tier 3 datasets, we obtained permission from the dataset providers to share the data among ESMValTool users on HPC systems. These Tier 3 datasets are marked with an asterisk in the table in section :ref:`supported datasets below<supported_datasets>`.
+.. note::
+    For some of the Tier 3 datasets, we obtained permission from the dataset providers to share the data among ESMValTool users on HPC systems. These Tier 3 datasets are marked with an asterisk in the table in section :ref:`supported datasets below<supported_datasets>`.
 
 An overview of the Tier 2 and Tier 3 datasets for which a CMORizing script is available in ESMValTool v2.0 is given in section :ref:`supported datasets below<supported_datasets>`.
 
diff --git a/esmvaltool/cmorizers/data/datasets.yml b/esmvaltool/cmorizers/data/datasets.yml
@@ -8,6 +8,10 @@ datasets:
     info: |
       Aerosol Optical Depth information from a worldwide network of stations.
 
+      NOTE this formatter is currently not working for Python 3.14 since pys2index is
+      not supported under Python 3.14! You will have to use Python<3.14, and install pys2index
+      from conda-forge.
+
   AGCD:
     tier: 2
     source: "http://dx.doi.org/10.25914/6009600786063"
@@ -1261,6 +1265,10 @@ datasets:
       Download the following archive:
       ch4_surface-flask_ccgg_text.tar.gz
 
+      NOTE this formatter is currently not working for Python 3.14 since pys2index is
+      not supported under Python 3.14! You will have to use Python<3.14, and install pys2index
+      from conda-forge.
+
   NOAA-GML-SURFACE-FLASK-CO2:
     tier: 2
     source: https://gml.noaa.gov/aftp/data/trace_gases/co2/flask/surface/
@@ -1269,6 +1277,10 @@ datasets:
       Download the following archive:
       co2_surface-flask_ccgg_text.tar.gz
 
+      NOTE this formatter is currently not working for Python 3.14 since pys2index is
+      not supported under Python 3.14! You will have to use Python<3.14, and install pys2index
+      from conda-forge.
+
   NOAAGlobalTemp:
     tier: 2
     source: https://www.ncei.noaa.gov/data/noaa-global-surface-temperature/v5/access/
@@ -1293,6 +1305,10 @@ datasets:
       Download the following archive:
       n2o_surface-flask_ccgg_text.tar.gz
 
+      NOTE this formatter is currently not working for Python 3.14 since pys2index is
+      not supported under Python 3.14! You will have to use Python<3.14, and install pys2index
+      from conda-forge.
+
   NSIDC-0116-sh:
     tier: 3
     source: https://nsidc.org/data/NSIDC-0116
diff --git a/esmvaltool/cmorizers/data/formatters/datasets/aeronet.py b/esmvaltool/cmorizers/data/formatters/datasets/aeronet.py
@@ -12,6 +12,10 @@
 Download and processing instructions
     Download the following file:
     https://aeronet.gsfc.nasa.gov/data_push/V3/AOD/AOD_Level20_Monthly_V3.tar.gz
+
+NOTE this formatter is currently not working for Python 3.14 since pys2index is
+not supported under Python 3.14! You will have to use Python<3.14, and install pys2index
+from conda-forge.
 """
 
 import logging
diff --git a/esmvaltool/cmorizers/data/formatters/datasets/noaa_gml_surface_flask.py b/esmvaltool/cmorizers/data/formatters/datasets/noaa_gml_surface_flask.py
@@ -14,6 +14,10 @@
     https://gml.noaa.gov/aftp/data/trace_gases/ch4/flask/surface/ch4_surface-flask_ccgg_text.tar.gz
     https://gml.noaa.gov/aftp/data/trace_gases/co2/flask/surface/co2_surface-flask_ccgg_text.tar.gz
     https://gml.noaa.gov/aftp/data/trace_gases/n2o/flask/surface/n2o_surface-flask_ccgg_text.tar.gz
+
+NOTE this formatter is currently not working for Python 3.14 since pys2index is
+not supported under Python 3.14! You will have to use Python<3.14, and install pys2index
+from conda-forge.
 """
 
 import logging
diff --git a/esmvaltool/cmorizers/data/formatters/datasets/noaa_gml_surface_flask_ch4.py b/esmvaltool/cmorizers/data/formatters/datasets/noaa_gml_surface_flask_ch4.py
@@ -12,6 +12,10 @@
 Download and processing instructions
     Download the following file:
     https://gml.noaa.gov/aftp/data/trace_gases/ch4/flask/surface/ch4_surface-flask_ccgg_text.tar.gz
+
+NOTE this formatter is currently not working for Python 3.14 since pys2index is
+not supported under Python 3.14! You will have to use Python<3.14, and install pys2index
+from conda-forge.
 """
 
 from esmvaltool.cmorizers.data.formatters.datasets import (
diff --git a/esmvaltool/cmorizers/data/formatters/datasets/noaa_gml_surface_flask_co2.py b/esmvaltool/cmorizers/data/formatters/datasets/noaa_gml_surface_flask_co2.py
@@ -12,6 +12,10 @@
 Download and processing instructions
     Download the following file:
     https://gml.noaa.gov/aftp/data/trace_gases/co2/flask/surface/co2_surface-flask_ccgg_text.tar.gz
+
+NOTE this formatter is currently not working for Python 3.14 since pys2index is
+not supported under Python 3.14! You will have to use Python<3.14, and install pys2index
+from conda-forge.
 """
 
 from esmvaltool.cmorizers.data.formatters.datasets import (
diff --git a/esmvaltool/cmorizers/data/formatters/datasets/noaa_gml_surface_flask_n2o.py b/esmvaltool/cmorizers/data/formatters/datasets/noaa_gml_surface_flask_n2o.py
@@ -12,6 +12,10 @@
 Download and processing instructions
     Download the following file:
     https://gml.noaa.gov/aftp/data/trace_gases/n2o/flask/surface/n2o_surface-flask_ccgg_text.tar.gz
+
+NOTE this formatter is currently not working for Python 3.14 since pys2index is
+not supported under Python 3.14! You will have to use Python<3.14, and install pys2index
+from conda-forge.
 """
 
 from esmvaltool.cmorizers.data.formatters.datasets import (
diff --git a/esmvaltool/recipes/ref/recipe_calculate_gwl_exceedance_stats_cmip7.yml b/esmvaltool/recipes/ref/recipe_calculate_gwl_exceedance_stats_cmip7.yml
@@ -0,0 +1,146 @@
+---
+documentation:
+  title: Diagnostic to calculate Global Warming Level exceedance years.
+
+  description: |
+    Recipe for calculating global warming level exceedances for individual model
+    and ensemble members. This recipe tests CMIP7 like data.
+
+  authors:
+    - swaminathan_ranjini
+
+  maintainer:
+    - swaminathan_ranjini
+
+  references:
+    - swaminathan22jclim
+
+preprocessors:
+  calculate_anomalies:
+    custom_order: true
+    area_statistics:
+      operator: mean
+    annual_statistics:
+      operator: mean
+      keep_group_coordinates: true
+    anomalies:
+      period: full
+      reference: &ref_period
+        start_year: 1850
+        start_month: 1
+        start_day: 1
+        end_year: 1900
+        end_month: 12
+        end_day: 31
+      standardize: false
+    extract_time:  # Future period
+      start_year: 2000
+      start_month: 1
+      start_day: 1
+      end_year: 2100
+      end_month: 12
+      end_day: 31
+
+  multi_model_gwl_stats:
+    custom_order: true
+    extract_time:  # Future period
+      start_year: 2000
+      start_month: 1
+      start_day: 1
+      end_year: 2100
+      end_month: 12
+      end_day: 31
+    annual_statistics:
+      operator: mean
+      keep_group_coordinates: true
+    regrid:
+      target_grid: 1x1
+      scheme: linear
+
+GWL_PARAMS: &gwl_calc_params
+  window_size: 21
+  gwls: [1.5, 2.0, 3.0, 4.0, 5.0]
+
+GWL: &gwl_exceedance_script
+  <<: *gwl_calc_params
+  script: gwls/calculate_gwl_exceedance_years.py
+
+
+PLOT_STATS: &plot_gwl_mm_stats_script
+  <<: *gwl_calc_params
+  script: gwls/plot_gwl_exceedance_mm_stats.py
+  pattern: 'GWL_exceedance_years.csv'
+
+diagnostics:
+  calculate_gwl_exceedance_years:
+    description: Calculate Global Warming Level exceedance years from smoothed temperature anomalies.
+    variables:
+      tas_anomaly:
+        short_name: tas
+        branding_suffix: tavg-h2m-hxy-u
+        mip: atmos
+        preprocessor: calculate_anomalies
+    scripts:
+      gwl_exceedance_calculation:
+        <<: *gwl_exceedance_script
+    additional_datasets: &gwl_datasets
+    - dataset: DUMMY-MODEL
+      drs_specs: MIP-DRS7
+      project: CMIP7
+      exp: [historical, scen7-l]
+      ensemble: r1i1p1f3
+      grid: g999
+      activity: [CMIP,ScenarioMIP]
+      institute: MOHC
+      mip: atmos
+      frequency: mon
+      region: glb
+
+      #- dataset: DUMMY-MODEL, project: CMIP6, exp: [historical, ssp126], grid: gn, ensemble: r1i1p1f1}
+      #- {project: CMIP6, dataset: CanESM5, ensemble: r1i1p1f1, exp: [historical, ssp126], grid: gn}
+      #- {dataset: CNRM-CM6-1, project: CMIP6, exp: [historical, ssp126], grid: gr, ensemble: r1i1p1f2}
+      #- {dataset: FGOALS-f3-L, project: CMIP6, exp: [historical, ssp126], grid: gr, ensemble: r1i1p1f1}
+      #- {dataset: MIROC6, project: CMIP6, exp: [historical, ssp126], grid: gn, ensemble: r1i1p1f1}
+      #- {project: CMIP6, dataset: UKESM1-0-LL, ensemble: r1i1p1f2, exp: [historical, ssp126], grid: gn}
+
+
+  gwl_mean_plots_tas:
+    description: Plot multimodel mean and standard deviation for surface temperature at specific Global Warming Levels.
+    variables:
+      tas:
+        branding_suffix: tavg-h2m-hxy-u
+        mip: atmos
+        preprocessor: multi_model_gwl_stats
+    scripts:
+      plot_gwl_stats:
+        <<: *plot_gwl_mm_stats_script
+        ancestors: ['calculate_gwl_exceedance_years/gwl_exceedance_calculation', 'gwl_mean_plots_tas/tas']
+        quickplot:
+          plot_type: map
+          cmap_mean: RdBu_r
+          cmap_stdev: YlOrRd
+          title_var: 'Temperature'
+          mean_level_params: [-60.0, 55.0, 5.0]
+          stdev_level_params: [0.0, 11.0, 1.0]
+    additional_datasets: *gwl_datasets
+
+  gwl_mean_plots_pr:
+    description: Plot multimodel mean and standard deviation for precipitation at specific Global Warming Levels.
+    variables:
+      pr:
+        short_name: pr
+        branding_suffix: tavg-u-hxy-u
+        mip: atmos
+        preprocessor: multi_model_gwl_stats
+    scripts:
+      plot_gwl_stats:
+        <<: *plot_gwl_mm_stats_script
+        ancestors: ['calculate_gwl_exceedance_years/gwl_exceedance_calculation', 'gwl_mean_plots_pr/pr']
+        quickplot:
+          plot_type: map
+          cmap_mean: YlGnBu
+          cmap_stdev: YlGnBu
+          title_var: 'Precipitation'
+          mean_level_params: [0.0, 18.0, 2.0]
+          stdev_level_params: [0.0, 11.0, 1.0]
+    additional_datasets: *gwl_datasets
diff --git a/pixi.lock b/pixi.lock
diff --git a/pyproject.toml b/pyproject.toml
diff --git a/tests/unit/cmorizers/test_cmorization_interface.py b/tests/unit/cmorizers/test_cmorization_interface.py
