Merge branch 'main' into 4GST_for_CMUG

Author: morobking

.circleci/config.yml

@@ -1,56 +1,49 @@
 ---
 version: 2.1
 
-commands:
-  check_changes:
-    steps:
-      - run:
-          name: Check whether or not installation tests are needed
-          command: |
-            if (test "$CIRCLE_BRANCH" = main ||
-              git --no-pager diff --name-only origin/main... |
-              grep -q -E -f .circleci/install_triggers)
-            then
-              echo Running installation tests
-            else
-              echo Skipping installation tests
-              circleci step halt
-            fi
+orbs:
+  pixi: prefix-dev/pixi@0.1.0
 
-  log_versions:
+jobs:
+  tests:
+    # Run tests
+    docker:
+      - image: cimg/base:current
+    resource_class: large
     steps:
+      - checkout
+      - pixi/pixi_setup
       - run:
-          name: Log versions
+          name: Check dependencies are compatible and installed
           command: |
-            . /opt/conda/etc/profile.d/conda.sh
-            conda env export --name base > /logs/base_environment.yml
-            conda activate esmvaltool
-            esmvaltool version
-            dpkg -l > /logs/versions.txt
-            conda env export > /logs/environment.yml
-            pip freeze > /logs/requirements.txt
-
-  test_and_report:
-    steps:
+            pixi run --as-is pip check
+            # The following command should not install any new dependencies
+            # because they are already in the conda environment, but this is
+            # not checked.
+            pixi run --as-is pip install --dry-run .[all]
+      - run:
+          name: Check code formatting and mistakes
+          command: pixi run --as-is pre-commit run --all
       - run:
           name: Run tests
           command: |
-            . /opt/conda/etc/profile.d/conda.sh
-            conda activate esmvaltool
-            pip check
-            pytest -n 4 --junitxml=test-reports/report.xml
+            pixi run --as-is test -n 4 --durations=10 --junitxml=test-reports/report.xml
+      - run:
+          name: Check the command line interface
+          command: |
+            source <(pixi shell-hook --as-is)
             esmvaltool version
             esmvaltool -- --help
             esmvaltool develop compare -- --help
-            ncl -V
-            cdo --version
       - store_test_results:
           path: test-reports/report.xml
-      - store_artifacts:
-          path: /logs
+      - run:
+          # Documentation available at https://docs.codacy.com/coverage-reporter/
+          name: Upload coverage report to Codacy
+          command: bash <(curl -Ls https://coverage.codacy.com/get.sh) report -s -r test-reports/coverage.xml
       - run:
           name: Compress pytest artifacts
-          command: tar -cvzf pytest.tar.gz -C /tmp/pytest-of-root/ .
+          command: tar -cvzf pytest.tar.gz -C /tmp/pytest-of-$(whoami)/ .
           when: always
       - store_artifacts:
           path: pytest.tar.gz
@@ -61,161 +54,72 @@ commands:
       - store_artifacts:
           path: test-reports.tar.gz
 
-  test_installation_from_source:
-    parameters:
-      extra:
-        type: string
-      flags:
-        type: string
-        default: ""
-    steps:
-      - run:
-          name: Install git+ssh
-          environment:
-            DEBIAN_FRONTEND: noninteractive # needed to install tzdata
-          command: apt update && apt install -y git ssh
-      - checkout
-      - check_changes
-      - run:
-          name: Generate cache key
-          command: date '+%Y-%V' | tee cache_key.txt
-      - restore_cache:
-          key: install-<< parameters.extra >>-{{ .Branch }}-{{ checksum "cache_key.txt" }}
-      - run:
-          name: Install dependencies
-          no_output_timeout: 30m
-          command: |
-            # Install
-            . /opt/conda/etc/profile.d/conda.sh
-            mkdir /logs
-            mamba --version
-            conda env create -n esmvaltool -f environment.yml --verbose
-            conda activate esmvaltool
-            mamba list >> /logs/conda.txt
-            pip install --no-deps << parameters.flags >> ".[<<parameters.extra>>]"> /logs/install.txt 2>&1
-            if [[ "<<parameters.flags>>" != *'--editable'* ]]
-            then
-              rm -r esmvaltool
-            fi
-      - log_versions
-      - run:
-          name: Lint source code
-          command: |
-            . /opt/conda/etc/profile.d/conda.sh
-            conda activate esmvaltool
-            pre-commit run --all
-      - test_and_report
-      - save_cache:
-          key: install-<< parameters.extra >>-{{ .Branch }}-{{ checksum "cache_key.txt" }}
-          paths:
-            - /opt/conda/pkgs
-            - /root/.cache/pip
-            - .pytest_cache
-
-jobs:
-  run_tests:
-    # Run tests
+  test_esmvaltool_base_installation:
+    # Test running the example recipe with minimal dependencies.
     docker:
-      - image: esmvalgroup/esmvaltool:development
-    resource_class: large
+      - image: cimg/base:current
+    resource_class: medium
     steps:
       - checkout
       - run:
           name: Generate cache key
-          command: date '+%Y-%V' | tee cache_key.txt
+          # Refresh the cache every month.
+          command: echo $(date '+%Y-%m') | tee cache_key.txt
       - restore_cache:
-          key: test-{{ .Branch }}-{{ checksum "cache_key.txt" }}
+          key: test-base-{{ .Branch }}-{{ checksum "cache_key.txt" }}
+      - pixi/pixi_setup:
+          env: esmvaltool-base
       - run:
-          name: Install dependencies
-          command: |
-            set -x
-            . /opt/conda/etc/profile.d/conda.sh
-            conda activate esmvaltool
-            mkdir /logs
-            pip install --no-deps .[test] > /logs/install.txt 2>&1
+          name: Check dependencies are compatible
+          command: pixi run --as-is --environment esmvaltool-base pip check
       - run:
-          name: Check Python code style and mistakes
+          name: Run example recipe
           command: |
-            . /opt/conda/etc/profile.d/conda.sh
-            conda activate esmvaltool
-            pre-commit run --all
-      - run:
-          name: Remove source code to test the installed software
-          command: rm -r esmvaltool
-      - test_and_report
-      - run:
-          # Documentation available at https://docs.codacy.com/coverage-reporter/
-          name: Upload coverage report to Codacy
-          command: bash <(curl -Ls https://coverage.codacy.com/get.sh) report -s -r test-reports/coverage.xml
+            source <(pixi shell-hook --as-is --environment esmvaltool-base)
+            esmvaltool version
+            esmvaltool config copy data-intake-esgf.yml
+            python -c '
+            import intake_esgf
+            intake_esgf.conf["solr_indices"]["esgf.ceda.ac.uk"] = True
+            intake_esgf.conf["solr_indices"]["esgf-data.dkrz.de"] = True
+            intake_esgf.conf.save()
+            '
+            esmvaltool run --max-parallel-tasks=2 testing/recipe_python_for_CI.yml
+      - store_artifacts:
+          path: /root/esmvaltool_output
       - save_cache:
-          key: test-{{ .Branch }}-{{ checksum "cache_key.txt" }}
+          key: test-base-{{ .Branch }}-{{ checksum "cache_key.txt" }}
           paths:
-            - /root/.cache/pip
-            - .pytest_cache
-
-  test_installation_from_source_test_mode:
-    # Test installation from source
-    docker:
-      - image: condaforge/miniforge3:latest
-    resource_class: large
-    steps:
-      - test_installation_from_source:
-          extra: test
-
-  test_installation_from_source_develop_mode:
-    # Test development installation
-    docker:
-      - image: condaforge/miniforge3:latest
-    resource_class: large
-    steps:
-      - test_installation_from_source:
-          extra: develop
-          flags: "--editable"
+            - /root/climate_data
 
   test_upstream_development:
     # Test running recipes with the development version of ESMValCore. The
     # purpose of this test to discover backward-incompatible changes early on in
     # the development cycle.
     docker:
-      - image: condaforge/miniforge3:latest
-    resource_class: large
+      - image: cimg/base:current
+    resource_class: medium
     steps:
-      - run:
-          name: Install git and ssh
-          environment:
-            DEBIAN_FRONTEND: noninteractive # needed to install tzdata
-          command: apt update && apt install -y git ssh
       - checkout
       - run:
           name: Generate cache key
-          command: echo $(date '+%Y')-$(expr $(date '+%V') / 2) | tee cache_key.txt
+          # Refresh the cache every month.
+          command: echo $(date '+%Y-%m') | tee cache_key.txt
       - restore_cache:
           key: test-upstream-{{ .Branch }}-{{ checksum "cache_key.txt" }}
+      - pixi/pixi_setup:
+          env: esmvalcore-dev
       - run:
-          name: Install
-          no_output_timeout: 30m
+          name: Run tests
           command: |
-            # Install according to instructions on readthedocs with the
-            # development version of ESMValTool and ESMValCore:
-            # https://docs.esmvaltool.org/en/latest/quickstart/installation.html#install-from-source
-            . /opt/conda/etc/profile.d/conda.sh
-            mkdir /logs
-            conda env create -n esmvaltool -f environment.yml --verbose
-            conda activate esmvaltool
-            # Temporarily install intake-esgf here until there is a new release
-            # that includes https://github.com/esgf2-us/intake-esgf/pull/160.
-            pip install git+https://github.com/esgf2-us/intake-esgf@main
-            mamba list >> /logs/conda.txt
-            pip install --no-deps --editable .[develop]
-            git clone https://github.com/ESMValGroup/ESMValCore $HOME/ESMValCore
-            pip install --no-deps --editable $HOME/ESMValCore[develop]
-      - log_versions
-      - test_and_report
+            pixi run --as-is --environment esmvalcore-dev test -n 2 --junitxml=test-reports/report.xml
+      - store_test_results:
+          path: test-reports/report.xml
       - run:
           name: Run recipes
           command: |
-            . /opt/conda/etc/profile.d/conda.sh
-            conda activate esmvaltool
+            source <(pixi shell-hook --as-is --environment esmvalcore-dev)
+            esmvaltool version
             esmvaltool config copy data-intake-esgf.yml
             python -c '
             import intake_esgf
@@ -231,34 +135,19 @@ jobs:
       - save_cache:
           key: test-upstream-{{ .Branch }}-{{ checksum "cache_key.txt" }}
           paths:
-            - /opt/conda/pkgs
-            - /root/.cache/pip
             - /root/climate_data
 
   build_documentation:
     # Test building documentation
     docker:
-      - image: condaforge/miniforge3:latest
-    resource_class: medium+
+      - image: cimg/base:current
+    resource_class: medium
     steps:
       - checkout
+      - pixi/pixi_setup
       - run:
-          command: |
-            mkdir /logs
-            . /opt/conda/etc/profile.d/conda.sh
-            # Install
-            conda env create -n esmvaltool -f environment_python.yml
-            conda activate esmvaltool
-            pip install --no-deps .[doc]
-            pip check
-            # Log versions
-            dpkg -l > /logs/versions.txt
-            conda env export > /logs/environment.yml
-            pip freeze > /logs/requirements.txt
-            # Test building documentation
-            MPLBACKEND=Agg sphinx-build -W doc/sphinx/source doc/spinx/build
-      - store_artifacts:
-          path: /logs
+          name: Build documentation
+          command: pixi run --as-is doc --fail-on-warning --show-traceback
 
   test_installation_from_conda:
     # Test conda package installation
@@ -273,7 +162,7 @@ jobs:
             # Install prerequisites
             mkdir /logs
             # Install ESMValTool in a new conda environment
-            mamba create -y --name esmvaltool -c conda-forge esmvaltool 'python=3.12' >> /logs/conda.txt 2>&1
+            mamba create -y --name esmvaltool -c conda-forge esmvaltool 'python=3.13' >> /logs/conda.txt 2>&1
             # Activate the environment
             set +x; conda activate esmvaltool; set -x
             # Log versions
@@ -289,9 +178,7 @@ jobs:
 workflows:
   commit:
     jobs:
-      - run_tests
-      - test_installation_from_source_test_mode
-      - test_installation_from_source_develop_mode
+      - tests
   nightly:
     triggers:
       - schedule:
@@ -301,9 +188,8 @@ workflows:
               only:
                 - main
     jobs:
-      - run_tests
-      - test_installation_from_source_test_mode
-      - test_installation_from_source_develop_mode
+      - tests
+      - test_esmvaltool_base_installation
       - test_upstream_development
       - build_documentation
       - test_installation_from_conda

.circleci/install_triggers

@@ -0,0 +1,2 @@
+# SCM syntax highlighting & preventing 3-way merges
+pixi.lock merge=binary linguist-language=YAML linguist-generated=true -diff

.gitattributes

@@ -12,6 +12,8 @@ updates:
     directory: "/"
     schedule:
       interval: "weekly"
+    cooldown:
+      default-days: 7
     labels:
       - dependencies
       - github_actions