Bring to master

Author: sergisiso

.github/workflows/compilation.yml

@@ -56,7 +56,7 @@ env:
   HDF5_VERSION: "2.1"
   NETCDF_C_VERSION: "4.10"
   NETCDF_FORTRAN_VERSION: "4.6"
-  NVFORTRAN_VERSION: "26.3"
+  NVFORTRAN_VERSION: "26.5"
   OPENMPI_VERSION: "5.0"
   PYTHON_VERSION: "3.14"
   NUM_PARALLEL: 16
@@ -87,7 +87,7 @@ jobs:
         python -m pip install --upgrade pip
         # Uncomment the below to use the submodule version of fparser rather
         # than the latest release from pypi.
-        pip install external/fparser
+        # pip install external/fparser
         pip install .[test,psydata,doc,treesitter]
     - name: Unit tests with compilation - gfortran
       run: |

.github/workflows/extraction_test.yml

@@ -74,7 +74,7 @@ jobs:
         python -m pip install --upgrade pip
         # Uncomment the below to use the submodule version of fparser rather
         # than the latest release from pypi.
-        pip install external/fparser
+        # pip install external/fparser
         pip install .[test]
         # Install BAF, and its version of Fab:
         # ------------------------------------

.github/workflows/lfric_test.yml

@@ -42,7 +42,7 @@ on:
   push
 
 env:
-  LFRIC_APPS_HASH: d8cab8417d873eb686e7fe74707da03742eaa9d9
+  LFRIC_APPS_HASH: 56054ba955cc97a8529afc3844139ab58bd7d359
   PYTHON_VERSION: "3.14"
   GNU_TOOLCHAIN: gnu14_openmpi
   NVHPC_TOOLCHAIN: nvhpc26_3_openmpi
@@ -85,7 +85,7 @@ jobs:
         python -m pip install --upgrade pip
         # Uncomment the below to use the submodule version of fparser rather
         # than the latest release from pypi.
-        pip install external/fparser
+        # pip install external/fparser
         pip install .[test]
         # Fetch the specified version of LFRic apps
         cd /archive/psyclone-tests/latest-run
@@ -116,7 +116,7 @@ jobs:
         cp ${PSYCLONE_LFRIC_DIR}/does_nothing.py ${OPT_DIR}/global.py
         # Clean previous version and compile again
         rm -rf working-gh-passthrough
-        ./build/local_build.py -j ${NUM_PARALLEL} -v -p psyclone-test -w working-gh-passthrough gungho_model
+        ./build/local_build.py -j ${NUM_PARALLEL} -p psyclone-test -w working-gh-passthrough gungho_model
         # Run
         cd applications/gungho_model/example
         rm -f timer.txt gungho_model-checksums.txt  # In case there were from a previous run
@@ -148,9 +148,14 @@ jobs:
         # Clean previous version and compile again
         rm -rf working-gh-ompoffload
         export BUILD_START="${SECONDS}"
-        LFRIC_OFFLOAD_DIRECTIVES=omp ./build/local_build.py -v -j ${NUM_PARALLEL} -p psyclone-test \
-          -w working-gh-ompoffload gungho_model
+        LFRIC_OFFLOAD_DIRECTIVES=omp ./build/local_build.py -j ${NUM_PARALLEL} -p psyclone-test \
+          -w working-gh-ompoffload gungho_model |& tee output.txt
+        # Piping to tee ignores the errcode of the first command, make sure we account for it
+        if [[ ${PIPESTATUS[0]} -ne 0 ]]; then
+            exit ${PIPESTATUS[0]}
+        fi
         export BUILD_ELAPSED=$((${SECONDS}-${BUILD_START}))
+        ${PSYCLONE_LFRIC_DIR}/aggregate_gpu_stats.sh output.txt
         cd applications/gungho_model/example
         rm -f timer.txt gungho_model-checksums.txt  # In case there were from a previous run
         export OMP_NUM_THREADS=12
@@ -180,7 +185,7 @@ jobs:
         cp ${PSYCLONE_LFRIC_DIR}/gpu_offloading.py ${OPT_DIR}/global.py
         # Clean previous version and compile again
         rm -rf working-gh-accoffload
-        LFRIC_OFFLOAD_DIRECTIVES=acc ./build/local_build.py -v -j ${NUM_PARALLEL} -p psyclone-test \
+        LFRIC_OFFLOAD_DIRECTIVES=acc ./build/local_build.py -j ${NUM_PARALLEL} -p psyclone-test \
           -w working-gh-accoffload gungho_model
         cd applications/gungho_model/example
         rm -f timer.txt gungho_model-checksums.txt  # In case there were from a previous run
@@ -210,7 +215,7 @@ jobs:
         cp ${PSYCLONE_LFRIC_DIR}/everything_everywhere_all_at_once.py ${OPT_DIR}/global.py
         # Clean previous version and compile again
         rm -rf working-gh_alltrans
-        ./build/local_build.py -j ${NUM_PARALLEL} -p psyclone-test -v -w working-gh-alltrans gungho_model
+        ./build/local_build.py -j ${NUM_PARALLEL} -p psyclone-test -w working-gh-alltrans gungho_model
         # Run
         cd applications/gungho_model/example
         rm -f timer.txt gungho_model-checksums.txt  # In case there were from a previous run
@@ -239,7 +244,7 @@ jobs:
         cp ${PSYCLONE_LFRIC_DIR}/tiledcolouring_and_omp.py ${OPT_DIR}/global.py
         # Clean previous version and compile again
         rm -rf working-gh-omptiling
-        ./build/local_build.py -j ${NUM_PARALLEL} -p psyclone-test -v -w working-gh-omptiling gungho_model
+        ./build/local_build.py -j ${NUM_PARALLEL} -p psyclone-test -w working-gh-omptiling gungho_model
         # Run
         cd applications/gungho_model/example
         rm -f timer.txt gungho_model-checksums.txt  # In case there were from a previous run
@@ -264,7 +269,7 @@ jobs:
         # Build the 'meto-ex1a' platform, that includes psyclone-transmutation
         rm -rf working-atm-transmuted
         export BUILD_START="${SECONDS}"
-        ./build/local_build.py -v -p meto-ex1a -j 16 -w working-atm-transmuted lfric_atm
+        ./build/local_build.py -p meto-ex1a -j 16 -w working-atm-transmuted lfric_atm
         export BUILD_ELAPSED=$((${SECONDS}-${BUILD_START}))
 
         # Run the app with 8 threads and no MPI
@@ -279,6 +284,26 @@ jobs:
         echo "time=${VAR_TIME}" >> "${GITHUB_OUTPUT}"
         echo "build_time=${BUILD_ELAPSED}" >> "${GITHUB_OUTPUT}"
 
+    - name: LFRic Atm nvfortran
+      id: lfric_atm_nvfortran
+      run: |
+        # Set up environment
+        source /archive/psyclone-spack/psyclone-spack-Dec25/spack-repo/share/spack/setup-env.sh
+        spack unload && spack load lfric-build-environment %${NVHPC_TOOLCHAIN}
+        source .runner_venv/bin/activate
+        export PSYCLONE_LFRIC_DIR=${GITHUB_WORKSPACE}/examples/lfric/scripts
+        export LFRIC_DIR=/archive/psyclone-tests/latest-run/lfric_apps
+        cd ${LFRIC_DIR}
+
+        # Currently we just compile the 'meto-ex1a' with the nvidia compiler, this won't
+        # create offloading directives and fails at runtime because lfric_atm needs XIOS
+        # enabled, but we want to make sure we keep the nvfortran build working.
+        rm -rf working-atm-gpu
+        export BUILD_START="${SECONDS}"
+        ./build/local_build.py -p meto-ex1a -j 16 -w working-atm-gpu lfric_atm
+        export BUILD_ELAPSED=$((${SECONDS}-${BUILD_START}))
+
+
   upload_if_on_mirror:
     if: ${{ github.repository == 'stfc/PSyclone-mirror' }}
     runs-on: ubuntu-latest

.github/workflows/linkcheck.yml

@@ -6,7 +6,7 @@ jobs:
     name: Run linkspector
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v5
       - name: Run linkspector
         uses: umbrelladocs/action-linkspector@v1
         with:

.github/workflows/nemo_tests.yml

@@ -45,7 +45,8 @@ env:
   HDF5_VERSION: "2.1"
   NETCDF_C_VERSION: "4.10"
   NETCDF_FORTRAN_VERSION: "4.6"
-  NVFORTRAN_VERSION: "26.3"
+  NVFORTRAN_VERSION: "26.5"
+  NVHPC_TOOLCHAIN: nvhpc26_3_openmpi
   INTEL_TOOLCHAIN: intel25_3_openmpi
   PERL_VERSION: "5.42"
   PYTHON_VERSION: "3.14"
@@ -85,7 +86,7 @@ jobs:
         python -m pip install --upgrade pip
         # Uncomment the below to use the submodule version of fparser rather
         # than the latest release from pypi.
-        pip install external/fparser
+        # pip install external/fparser
         pip install .[test]
         # Compile nvidia profiling tools
         module load nvidia-hpcsdk/${NVFORTRAN_VERSION}
@@ -130,6 +131,8 @@ jobs:
         module load perl/${PERL_VERSION}
         make clean
         export NEMOV4=1  # Enables specific NEMOV4 exclusions in the PSyclone transformation script
+        export PARALLEL_DIRECTIVES="omp_offloading+omp_threading"
+        export REPRODUCIBLE=1
         make -j ${NUM_PARALLEL} openmp_gpu
         make -j ${NUM_PARALLEL} compile-openmp_gpu
         export NV_ACC_POOL_THRESHOLD=75
@@ -171,27 +174,39 @@ jobs:
     - name: NEMO MetOffice OpenACC loops for GPU
       id: nemo_acc_loops
       run: |
+        source /archive/psyclone-spack/psyclone-spack-Dec25/spack-repo/share/spack/setup-env.sh
+        spack unload && spack load nemo-build-environment %${NVHPC_TOOLCHAIN}
         . .runner_venv/bin/activate
+
+        # Set envvars
         export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts
         export PROFILE_HOME=${GITHUB_WORKSPACE}/lib/profiling/nvidia/
         export NEMO_DIR=${PREFIX}/UKMO-NEMOv4
-        cd $PSYCLONE_NEMO_DIR
-        module load nvidia-hpcsdk/${NVFORTRAN_VERSION}
-        module load hdf5/${HDF5_VERSION} netcdf-c/${NETCDF_C_VERSION} netcdf-fortran/${NETCDF_FORTRAN_VERSION}
-        module load perl/${PERL_VERSION}
-        make clean
-        make -j ${NUM_PARALLEL} openacc_loops
-        COMPILER_ARCH=linux_nvidia_acc_gpu make -j ${NUM_PARALLEL} compile-openacc_loops
+        export TEST_DIR=SPITZ12_ACC_LOOPS_NVHPC
+        export PSYCLONE_COMPILER=$MPIF90
+        export MPIF90=psyclonefc
+        export PARALLEL_DIRECTIVES="acc_offloading"
+        export REPRODUCIBLE=1
+        export PSYCLONE_OPTS="--enable-cache -l output -s ${PSYCLONE_NEMO_DIR}/insert_loop_parallelism.py"
+        export FCFLAGS="-i4 -Mr8 -O2 -Mnovect -Mnofma -g -acc -mp=gpu -gpu=mem:managed,math_uniform"
+        export NEMOV4=1  # Enables specific NEMOV4 exclusions in the PSyclone transformation script
+
+        # Compile
+        cd ${PREFIX}/UKMO-NEMOv4
+        cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm
+        rm -rf cfgs/${TEST_DIR}
+        ./makenemo -r SPITZ12 -m linux_spack -n ${TEST_DIR} \
+          add_key "IEEE_IS_NAN=ieee_is_nan key_nosignedzero" \
+          del_key "key_iomput key_mpp_mpi key_si3" -j ${NUM_PARALLEL}
+
+        # Run test
         export NV_ACC_POOL_THRESHOLD=75
         export CUDA_VISIBLE_DEVICES=1
-        make run-openacc_loops
-        # Check the output is as expected (TODO #2895: improve numerical reproducibility)
-        make output-openacc_loops | grep -q " it :      10"  || (echo "Error: 'it :      10' not found!" & false)
-        make output-openacc_loops | grep -q "|ssh|_max:  0.259483" || (echo "Error: '|ssh|_max:  0.259483' not found!" & false)
-        make output-openacc_loops | grep -q "|U|_max:  0.458515" || (echo "Error: '|U|_max:  0.458515' not found!" & false)
-        make output-openacc_loops | grep -q "S_min:  0.482686" || (echo "Error: 'S_min:  0.482686' not found!" & false)
-        make output-openacc_loops | grep -q "S_max:  0.407622" || (echo "Error: 'S_max:  0.407622' not found!" & false)
-        export VAR_TIME=$(grep -A 1 "Elapsed Time" <(make -s time-openacc_loops) | head -n 2 | tail -n 1 | awk '{print $1}')
+        cd cfgs/${TEST_DIR}/EXP00/
+        ln -sf /archive/psyclone-tests/nemo-inputs/UKMO-eORCA1_GO8_NEMOv4/*.nc .
+        ./nemo
+        diff run.stat $PSYCLONE_NEMO_DIR/KGOs/run.stat.nemo4.spitz12.nvhpc.10steps
+        export VAR_TIME=$(grep -A 1 "Elapsed Time" timing.output | head -n 2 | tail -n 1 | awk '{print $1}')
         echo "time=${VAR_TIME}" >> "${GITHUB_OUTPUT}"
       
     # PSyclone, compile and run ECMWF NEMO with OpenMP for CPUs. This uses
@@ -205,21 +220,25 @@ jobs:
         source .runner_venv/bin/activate
 
         # Set up envvars
+        export TEST_DIR=SPITZ12_OMP_CPU
         export PSYCLONE_NEMO_DIR=${GITHUB_WORKSPACE}/examples/nemo/scripts
         export PSYCLONE_COMPILER=$MPIF90
         export MPIF90=psyclonefc
-        export PSYCLONE_OPTS="--enable-cache -l output -s ${PSYCLONE_NEMO_DIR}/omp_cpu_trans.py -I ${MPI_HOME}/include"
+        export PARALLEL_DIRECTIVES="omp_threading"
+        export PSYCLONE_OPTS="--enable-cache -l output -s ${PSYCLONE_NEMO_DIR}/insert_loop_parallelism.py -I ${MPI_HOME}/include"
         export FCFLAGS="-i4 -r8 -O2 -heap-arrays -fp-model=precise -g -qopenmp"
         export NEMOV4=1  # Enables specific NEMOV4 exclusions in the PSyclone transformation script
 
         # Compile
         cd ${PREFIX}/ECMWF-NEMOv4
-        ./makenemo -r SPITZ12 -m linux_spack -n SPITZ12_psyclone -j ${NUM_PARALLEL} \
+        cp $PSYCLONE_NEMO_DIR/KGOs/arch-linux_spack.fcm arch/arch-linux_spack.fcm
+        rm -rf cfgs/${TEST_DIR}
+        ./makenemo -r SPITZ12 -m linux_spack -n ${TEST_DIR} -j ${NUM_PARALLEL} \
           add_key "IEEE_IS_NAN=ieee_is_nan key_nosignedzero" \
           del_key "key_iomput"
         
         # Run NEMO
-        cd cfgs/SPITZ12_psyclone/EXP00/
+        cd cfgs/${TEST_DIR}/EXP00/
         ln -sf /archive/psyclone-tests/nemo-inputs/ECMWF-eORCA1_GO8/* .
         export OMP_NUM_THREADS=12
         ./nemo
@@ -240,6 +259,8 @@ jobs:
         module load perl/${PERL_VERSION}
         make clean
         export NEMOV4=1  # Enables specific NEMOV4 exclusions in the PSyclone transformation script
+        export PARALLEL_DIRECTIVES="omp_offloading+omp_threading"
+        export REPRODUCIBLE=1
         export ASYNC_PARALLEL=1
         make -j ${NUM_PARALLEL} openmp_gpu
         make -j ${NUM_PARALLEL} compile-openmp_gpu
@@ -290,7 +311,7 @@ jobs:
             {
               ci_test: "NEMOv4 OpenACC loops for GPU",
               nemo_version: "MEMO MO 4.0.2",
-              compiler:"nvhpc-'"$NVFORTRAN_VERSION"'",
+              compiler:"'"${NVHPC_TOOLCHAIN}"'",
               elapsed_time: '"${{needs.run_if_on_mirror.outputs.nemo_acc_loops}}"',
               '"$COMMON_FIELDS"'
             },