Merge branch 'master' into local-aware-ibm

Author: sbryngelson

.coderabbit.yaml

@@ -1,15 +1,3 @@
 # Spencer H. Bryngelson (sbryngelson) is the
 # default owner of all files in the repository.
 *                  @sbryngelson
-
-# @core-devs
-src/               @MFlowCode/core-devs
-docs/              @MFlowCode/core-devs
-toolchain/         @MFlowCode/core-devs
-tests/             @MFlowCode/core-devs
-benchmarks/        @MFlowCode/core-devs
-mfc.sh             @MFlowCode/core-devs
-CMakeLists.txt     @MFlowCode/core-devs
-
-# @physics-devs
-src/               @MFlowCode/physics-devs

.github/CODEOWNERS

@@ -7,8 +7,6 @@ ARG CXX_COMPILER
 ARG FC_COMPILER
 ARG COMPILER_PATH
 ARG COMPILER_LD_LIBRARY_PATH
-ARG AFAR_VERSION
-ARG OLCF_AFAR_ROOT=""
 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV TZ=UTC
@@ -29,47 +27,10 @@ RUN apt-get update -y && \
             python3.12 python3.12-venv python3-pip \
             libfftw3-dev \
             openmpi-bin libopenmpi-dev; \
-    elif [ "$TARGET" = "amd" ]; then \
-        apt-get install -y \
-            build-essential git make gcc g++ gfortran bc \
-            python3.12 python3.12-venv python3-pip \
-            libfftw3-dev libnuma1 libdrm2 libdrm-amdgpu1; \
     fi && \
     update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 2 && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
-# AMD: download AFAR, install cmake 3.28 (Ubuntu 22.04 ships 3.22 which doesn't
-# recognize amdflang as LLVMFlang), then build MPICH with amdflang so the
-# generated mpi.mod is compiler-compatible.
-RUN if [ "$TARGET" = "amd" ] && [ -n "$AFAR_VERSION" ]; then \
-        OLCF_AFAR_ROOT="/opt/${AFAR_VERSION}" && \
-        wget -q "https://repo.radeon.com/rocm/misc/flang/${AFAR_VERSION}-ubuntu.tar.bz2" -O /tmp/afar.tar.bz2 && \
-        tar -xjf /tmp/afar.tar.bz2 -C /opt/ && \
-        rm /tmp/afar.tar.bz2 && \
-        CMAKE_VER=3.28.6 && \
-        wget -q "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VER}/cmake-${CMAKE_VER}-linux-x86_64.sh" \
-            -O /tmp/cmake-install.sh && \
-        chmod +x /tmp/cmake-install.sh && \
-        /tmp/cmake-install.sh --prefix=/usr/local --skip-license --exclude-subdir && \
-        rm /tmp/cmake-install.sh && \
-        printf '#!/bin/bash\nargs=()\nwhile [ "$#" -gt 0 ]; do\n    if [ "$1" = "-soname" ]; then\n        args+=("-Wl,-soname,$2"); shift 2\n    else\n        args+=("$1"); shift\n    fi\ndone\nexec '"${OLCF_AFAR_ROOT}"'/bin/amdflang "${args[@]}"\n' \
-            > /usr/local/bin/amdflang-ld-wrap && \
-        chmod +x /usr/local/bin/amdflang-ld-wrap && \
-        MPICH_VER=3.4.3 && \
-        wget -q "https://www.mpich.org/static/downloads/${MPICH_VER}/mpich-${MPICH_VER}.tar.gz" \
-            -O /tmp/mpich.tar.gz && \
-        mkdir -p /tmp/mpich-src && \
-        tar -xzf /tmp/mpich.tar.gz -C /tmp/mpich-src --strip-components=1 && \
-        cd /tmp/mpich-src && \
-        FC=/usr/local/bin/amdflang-ld-wrap CC=gcc CXX=g++ \
-            ./configure --prefix=/opt/mpich --enable-shared --disable-static \
-            --with-device=ch3 2>&1 && \
-        make -j$(nproc) 2>&1 && \
-        make install 2>&1 && \
-        cd / && \
-        rm -rf /tmp/mpich-src /tmp/mpich.tar.gz; \
-    fi
-
 ENV OMPI_ALLOW_RUN_AS_ROOT=1
 ENV OMPI_ALLOW_RUN_AS_ROOT_CONFIRM=1
 ENV HYDRA_LAUNCHER=fork
@@ -80,7 +41,6 @@ COPY ../ /opt/MFC
 ENV CC=${CC_COMPILER}
 ENV CXX=${CXX_COMPILER}
 ENV FC=${FC_COMPILER}
-ENV OLCF_AFAR_ROOT=${OLCF_AFAR_ROOT}
 ENV PATH="${COMPILER_PATH}:/opt/mpich/bin:$PATH"
 ENV LD_LIBRARY_PATH="${COMPILER_LD_LIBRARY_PATH}:/opt/mpich/lib:${LD_LIBRARY_PATH:-}"
 
@@ -95,17 +55,13 @@ RUN echo "TARGET=$TARGET CC=$CC_COMPILER FC=$FC_COMPILER" && \
     cd /opt/MFC && \
     if [ "$TARGET" = "gpu" ]; then \
       ./mfc.sh build --gpu acc -j $(nproc); \
-    elif [ "$TARGET" = "amd" ]; then \
-      ./mfc.sh build --gpu mp -j $(nproc); \
     else \
       ./mfc.sh build -j $(nproc); \
     fi
 
 RUN cd /opt/MFC && \
     if [ "$TARGET" = "gpu" ]; then \
       ./mfc.sh test -a --dry-run --gpu acc -j $(nproc); \
-    elif [ "$TARGET" = "amd" ]; then \
-      ./mfc.sh test -a --dry-run --gpu mp -j $(nproc); \
     else \
       ./mfc.sh test -a --dry-run -j $(nproc); \
     fi

.github/Dockerfile

@@ -1,10 +1,10 @@
 #!/bin/bash
 
-# Pre-builds all benchmark cases with --case-optimization.
-# No GPU hardware needed — compilation only.
+# Pre-builds all benchmark cases with --case-optimization using --dry-run so
+# binaries are cached before the GPU run job. No simulation is executed.
 # Can run in two modes:
 #   1. Direct (Frontier login nodes): pass cluster/device/interface as args
-#   2. Inside SLURM (Phoenix): uses $job_device/$job_interface from submit-slurm-job.sh
+#   2. Inside SLURM (Phoenix/frontier_amd): uses $job_device/$job_interface
 # Usage: bash prebuild-case-optimization.sh [<cluster> <device> <interface>]
 
 set -e
@@ -22,14 +22,18 @@ case "$cluster" in
     *) echo "ERROR: Unknown cluster '$cluster'"; exit 1 ;;
 esac
 
-source .github/scripts/clean-build.sh
-clean_build
+# Phoenix starts fresh (no prior dep build); other clusters pre-build deps via
+# build.sh first, so we must preserve them and only clean MFC target staging.
+if [ "$cluster" = "phoenix" ]; then
+    source .github/scripts/clean-build.sh
+    clean_build
+else
+    find build/staging -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true
+    find build/install -maxdepth 1 -regex '.*/[0-9a-f]+' -type d -exec rm -rf {} + 2>/dev/null || true
+fi
 
 . ./mfc.sh load -c "$flag" -m g
 
-# Set GPU build flags from interface — this is always a GPU build.
-# Don't use gpu-opts.sh since $job_device may be "cpu" when submitted
-# to a CPU SLURM partition (no GPU hardware needed for compilation).
 case "$job_interface" in
     acc) gpu_opts="--gpu acc" ;;
     omp) gpu_opts="--gpu mp" ;;
@@ -38,5 +42,5 @@ esac
 
 for case in benchmarks/*/case.py; do
     echo "=== Pre-building: $case ==="
-    ./mfc.sh build -i "$case" --case-optimization $gpu_opts -j 8
+    ./mfc.sh run "$case" --case-optimization $gpu_opts -j 8 --dry-run
 done

.github/scripts/prebuild-case-optimization.sh

@@ -23,13 +23,14 @@ benchmarks=(
 
 # For Frontier/Frontier AMD: deps were fetched on the login node via --deps-only;
 # build case-optimized binaries here on the compute node before running.
-# For Phoenix: prebuild-case-optimization.sh already built everything in a prior SLURM job.
+# For Phoenix and frontier_amd: prebuild-case-optimization.sh already built
+# everything in a prior SLURM job (via --dry-run), so skip the build here.
 #
 # Clean stale MFC target staging before building. On self-hosted CI runners,
 # corrupted intermediate files from a prior failed build (e.g. CCE optcg crash)
 # can persist and poison subsequent builds. Each case-opt config gets its own
 # hash-named staging dir, but install dirs and other artifacts may be stale.
-if [ "$job_cluster" != "phoenix" ]; then
+if [ "$job_cluster" != "phoenix" ] && [ "$job_cluster" != "frontier_amd" ]; then
     # Clean stale MFC target dirs (hash-named) from prior builds, but
     # preserve dependency dirs (hipfort, fftw, etc.) since the compute
     # node has no internet to re-fetch them.

.github/scripts/run_case_optimization.sh

@@ -28,7 +28,6 @@ jobs:
             - { name: 'cpu',  tag: 'cpu',       runner: 'ubuntu-22.04-arm', base_image: 'ubuntu:22.04' }
             - { name: 'gpu',  tag: 'gpu-nvidia', runner: 'ubuntu-22.04',     base_image: 'nvcr.io/nvidia/nvhpc:24.5-devel-cuda_multi-ubuntu22.04', compiler_arch: 'Linux_x86_64' }
             - { name: 'gpu',  tag: 'gpu-nvidia', runner: 'ubuntu-22.04-arm', base_image: 'nvcr.io/nvidia/nvhpc:24.5-devel-cuda_multi-ubuntu22.04', compiler_arch: 'Linux_aarch64' }
-            - { name: 'amd',  tag: 'gpu-amd',   runner: 'ubuntu-22.04',     base_image: 'ubuntu:22.04' }
     runs-on: ${{ matrix.config.runner }}
     outputs:
       tag: ${{ steps.clone.outputs.tag }}
@@ -135,37 +134,6 @@ jobs:
             ${{ env.GH_REGISTRY }}:${{ env.TAG }}-${{ matrix.config.tag }}-${{ matrix.config.runner }}
           push: true
 
-      - name: Set AMD AFAR vars
-        if: ${{ matrix.config.name == 'amd' }}
-        run: |
-          AFAR=rocm-afar-8873-drop-22.2.0
-          ROOT="/opt/${AFAR}"
-          echo "AFAR_VERSION=${AFAR}"      >> $GITHUB_ENV
-          echo "AFAR_ROOT=${ROOT}"         >> $GITHUB_ENV
-
-      - name: Build and push image (amd)
-        if: ${{ matrix.config.name == 'amd' }}
-        uses: docker/build-push-action@v6
-        with:
-          context: /mnt/share
-          file: /mnt/share/Dockerfile
-          build-args: |
-            BASE_IMAGE=${{ matrix.config.base_image }}
-            TARGET=amd
-            AFAR_VERSION=${{ env.AFAR_VERSION }}
-            OLCF_AFAR_ROOT=${{ env.AFAR_ROOT }}
-            CC_COMPILER=gcc
-            CXX_COMPILER=g++
-            FC_COMPILER=${{ env.AFAR_ROOT }}/bin/amdflang
-            COMPILER_PATH=${{ env.AFAR_ROOT }}/lib/llvm/bin:${{ env.AFAR_ROOT }}/bin
-            COMPILER_LD_LIBRARY_PATH=${{ env.AFAR_ROOT }}/lib:${{ env.AFAR_ROOT }}/lib/llvm/lib
-          labels: |
-            org.opencontainers.image.source=https://github.com/${{ github.repository }}
-          tags: |
-            ${{ secrets.DOCKERHUB_USERNAME }}/mfc:${{ env.TAG }}-gpu-amd-ubuntu-22.04
-            ${{ env.GH_REGISTRY }}:${{ env.TAG }}-gpu-amd-ubuntu-22.04
-          push: true
-
   manifests:
     runs-on: ubuntu-latest
     needs: Container
@@ -201,9 +169,8 @@ jobs:
         run: |
           GH="${{ env.GH_REGISTRY }}"
           for R in "$DH" "$GH"; do
-            docker buildx imagetools create -t $R:$TAG-cpu       $R:$TAG-cpu-ubuntu-22.04 $R:$TAG-cpu-ubuntu-22.04-arm
+            docker buildx imagetools create -t $R:$TAG-cpu        $R:$TAG-cpu-ubuntu-22.04 $R:$TAG-cpu-ubuntu-22.04-arm
             docker buildx imagetools create -t $R:$TAG-gpu-nvidia $R:$TAG-gpu-nvidia-ubuntu-22.04 $R:$TAG-gpu-nvidia-ubuntu-22.04-arm
-            docker buildx imagetools create -t $R:$TAG-gpu-amd   $R:$TAG-gpu-amd-ubuntu-22.04
           done
 
       - name: Update latest tags
@@ -214,7 +181,6 @@ jobs:
         run: |
           GH="${{ env.GH_REGISTRY }}"
           for R in "$DH" "$GH"; do
-            docker buildx imagetools create -t $R:latest-cpu       $R:$TAG-cpu-ubuntu-22.04 $R:$TAG-cpu-ubuntu-22.04-arm
+            docker buildx imagetools create -t $R:latest-cpu        $R:$TAG-cpu-ubuntu-22.04 $R:$TAG-cpu-ubuntu-22.04-arm
             docker buildx imagetools create -t $R:latest-gpu-nvidia $R:$TAG-gpu-nvidia-ubuntu-22.04 $R:$TAG-gpu-nvidia-ubuntu-22.04-arm
-            docker buildx imagetools create -t $R:latest-gpu-amd   $R:$TAG-gpu-amd-ubuntu-22.04
           done

.github/workflows/docker.yml

@@ -101,6 +101,12 @@ jobs:
         run: ~/.local/verrou/bin/valgrind --version
 
       - name: Build MFC (debug, serial)
+        # FFLAGS=-fno-inline prevents gfortran from inlining small functions into
+        # their callers.  Without it, DWARF debug info attributes inlined ops to
+        # the caller's line (often a do-loop header), making Verrou dd_line point
+        # to loop boundaries instead of the actual arithmetic.
+        env:
+          FFLAGS: "-fno-inline"
         run: ./mfc.sh build -t pre_process simulation --no-mpi --debug -j"$(nproc)"
 
       - name: Run FP Stability Suite

.github/workflows/fp-stability.yml

@@ -659,12 +659,16 @@ jobs:
         if:   matrix.cluster == 'phoenix'
         run:  bash .github/scripts/submit-slurm-job.sh .github/scripts/prebuild-case-optimization.sh cpu ${{ matrix.interface }} ${{ matrix.cluster }}
 
+      - name: Pre-Build (SLURM)
+        if:   matrix.cluster == 'frontier_amd'
+        run:  bash .github/scripts/submit-slurm-job.sh .github/scripts/prebuild-case-optimization.sh gpu ${{ matrix.interface }} ${{ matrix.cluster }}
+
       - name: Build & Run Case-Optimization Tests
-        if:   matrix.cluster != 'phoenix'
+        if:   matrix.cluster != 'phoenix' && matrix.cluster != 'frontier_amd'
         run:  bash .github/scripts/submit-slurm-job.sh .github/scripts/run_case_optimization.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }}
 
       - name: Run Case-Optimization Tests
-        if:   matrix.cluster == 'phoenix'
+        if:   matrix.cluster == 'phoenix' || matrix.cluster == 'frontier_amd'
         run:  bash .github/scripts/submit-slurm-job.sh .github/scripts/run_case_optimization.sh ${{ matrix.device }} ${{ matrix.interface }} ${{ matrix.cluster }}
 
       - name: Cancel SLURM Jobs

.github/workflows/test.yml

@@ -114,3 +114,5 @@ cce_*/
 cce_*.log
 run_cce_*.sh
 .ffmt_cache/
+# FP-stability log artifacts (generated by ./mfc.sh fp-stability)
+fp-stability-logs/

.gitignore

@@ -31,4 +31,4 @@ tru = "tru"           # typo for "true" in "when_tru" - tests dependency keys
 PNGs = "PNGs"
 
 [files]
-extend-exclude = ["docs/documentation/references*", "docs/references.bib", "tests/", "toolchain/cce_simulation_workgroup_256.sh", "build-docs/", "build/", "build_test/"]
+extend-exclude = ["docs/documentation/references*", "docs/references.bib", "tests/", "toolchain/cce_simulation_workgroup_256.sh", "build-docs/", "build/", "build_test/", "fp-stability-logs/"]