Add cuSolverMp RT-TDDFT smoke coverage and version notes

.github/workflows/sai-gpu-full.yml

@@ -0,0 +1,181 @@
+name: SAI GPU Full Validation
+
+on:
+  workflow_dispatch:
+    inputs:
+      mp_version:
+        description: "cuSolverMp/cuBLASMp runtime: project (0.9.x) or system (NVHPC 0.7.x)"
+        required: true
+        default: project
+        type: choice
+        options:
+          - project
+          - system
+      run_cusolvermp:
+        description: "Run the O3 RT-TDDFT cuSolverMp smoke case"
+        required: true
+        default: true
+        type: boolean
+
+permissions:
+  contents: read
+
+concurrency:
+  group: sai-gpu-full
+  cancel-in-progress: false
+
+jobs:
+  gpu-full:
+    name: SAI GPU full suite (${{ inputs.mp_version }})
+    runs-on: [self-hosted, linux, x64, sai-slurm, v100, cusolvermp]
+    timeout-minutes: 75
+    environment: sai-gpu
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Stage checkout on shared SAI filesystem
+        run: |
+          set -euo pipefail
+          export CI_SRC=/home/sai-partner/saiuser01/agent/abacus_beta6_build/ci-checkouts/${GITHUB_RUN_ID}
+          export RESULT_ROOT=/home/sai-partner/saiuser01/agent/abacus_beta6_build/ci-results/${GITHUB_RUN_ID}
+          rm -rf "$CI_SRC"
+          rm -rf "$RESULT_ROOT"
+          mkdir -p "$CI_SRC" "$RESULT_ROOT"
+          rsync -a --delete --exclude .git/ "$GITHUB_WORKSPACE/" "$CI_SRC/"
+          echo "CI_SRC=$CI_SRC" >> "$GITHUB_ENV"
+          echo "RESULT_ROOT=$RESULT_ROOT" >> "$GITHUB_ENV"
+
+      - name: Submit Slurm GPU validation
+        env:
+          MP_VERSION: ${{ inputs.mp_version }}
+          RUN_CUSOLVERMP: ${{ inputs.run_cusolvermp }}
+        run: |
+          set -euo pipefail
+          script="$RUNNER_TEMP/sai-gpu-${GITHUB_RUN_ID}.sbatch"
+          log="$CI_SRC/sai-gpu-${GITHUB_RUN_ID}.log"
+          cat > "$script" <<'SBATCH'
+          #!/usr/bin/env bash
+          #SBATCH --job-name=abacus-sai-ci
+          #SBATCH --partition=16V100
+          #SBATCH --qos=huge-gpu
+          #SBATCH --nodes=1
+          #SBATCH --ntasks=4
+          #SBATCH --gpus-per-node=4
+          #SBATCH --time=01:00:00
+          #SBATCH --output=__RESULT_ROOT__/sai-gpu-%j.out
+
+          set -euo pipefail
+          : "${CI_SRC:?}"
+          : "${ABACUS:?}"
+          : "${MP_VERSION:?}"
+          export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-}
+          export LD_PRELOAD=${LD_PRELOAD:-}
+          source /etc/profile.d/lmod.sh
+          module purge
+          module load nvhpc/26.3-gnu-cuda12-tuned
+          module load openmpi/5.0.10-nvhpc26.3-gnu-cuda12-auto
+          module load fftw/3.3.10 libxc/7.0.0-auto saiblas/2603-gnu-auto
+          module load elpa/2026.02.001-2603-gnu cuda/12.9.1
+          module unload nccl
+          source /opt/sai_config/mps_mapping.d/16V100.bash
+          export NCCL_IB_DISABLE=1
+          export OMP_NUM_THREADS=1
+          export OMPI_MCA_rmaps_base_mapping_policy="$MAP_OPT"
+
+          MPI=/opt/devtools/openmpi/openmpi-5.0.10-nvhpc263-gnu-cuda12-avx2
+          NV=/opt/devtools/nvidia/hpc_sdk/Linux_x86_64/26.3
+          CUDA=/opt/devtools/nvidia/cuda-12.9.1
+          MATH="$NV/math_libs/12.9"
+          NCCL="$NV/comm_libs/12.9/nccl-2.29"
+          if [[ "$MP_VERSION" == project ]]; then
+            CUSOLVERMP=/home/sai-partner/saiuser01/agent/abacus_beta6_build/deps-mp-0.9/nvidia/cu12
+            CUBLASMP=/home/sai-partner/saiuser01/agent/abacus_beta6_build/deps-mp-0.9/nvidia/cublasmp/cu12
+          else
+            CUSOLVERMP="$MATH/targets/x86_64-linux"
+            CUBLASMP="$MATH/targets/x86_64-linux"
+          fi
+          export PATH="$MPI/bin:$PATH"
+          echo "MPI=$MPI"
+          echo "CUDA=$CUDA"
+          echo "CUSOLVERMP=$CUSOLVERMP"
+          echo "CUBLASMP=$CUBLASMP"
+          ls -l "$CUSOLVERMP"/lib/libcusolverMp.so* "$CUBLASMP"/lib/libcublasMp.so* 2>/dev/null || true
+          export LD_LIBRARY_PATH="$MPI/lib:$CUDA/lib64:$CUSOLVERMP/lib:$CUBLASMP/lib:$MATH/lib64:$MATH/targets/x86_64-linux/lib:$NCCL/lib:/home/sai-partner/saiuser01/agent/abacus_beta6_build/install-full-sm70-avx2-mp09/lib:/home/sai-partner/saiuser01/agent/abacus_beta6_build/deps-full/lib:/home/sai-partner/saiuser01/agent/abacus_beta6_build/deps-full/lib64:/home/sai-partner/saiuser01/agent/abacus_beta6_build/deps-deepmd-pt-only/lib:/opt/devtools/elpa/elpa-2026.02.001-2603-gnu/nvidia/lib:/opt/devtools/libxc/libxc-7.0.0-avx2/lib:/opt/devtools/saiblas/2603-gnu-avx2/lib:/opt/devtools/fftw/fftw-3.3.10/lib:/opt/apps/conda_env/deepmd-kit-3.1.2/lib:/opt/apps/abacus/abacus-develop-3.9.0.26_nvhpc263gnu/toolchain/install/NEP_CPU-main/lib:${LD_LIBRARY_PATH:-}"
+
+          suite_status=0
+          run_checked() {
+            if ! "$@"; then
+              suite_status=1
+              echo "[WARN] test command failed; continuing to collect remaining GPU diagnostics"
+            fi
+          }
+
+          run_cases() {
+            local suite="$1"
+            local ranks="$2"
+            local cases_file="$3"
+            local regex="$4"
+            if [[ "$ranks" -lt 4 ]]; then
+              (unset OMPI_MCA_rmaps_base_mapping_policy; cd "$CI_SRC/tests/$suite" && \
+                bash ../integrate/Autotest.sh -a "$ABACUS" -n "$ranks" -o 1 \
+                  -f "$cases_file" -r "$regex")
+            else
+              (cd "$CI_SRC/tests/$suite" && bash ../integrate/Autotest.sh \
+                -a "$ABACUS" -n "$ranks" -o 1 -f "$cases_file" -r "$regex")
+            fi
+          }
+
+          cases_11="$CI_SRC/tests/11_PW_GPU/CASES_CI_GPU.txt"
+          grep -v -E '^(scf_bpcg|scf_out_wf)$' "$CI_SRC/tests/11_PW_GPU/CASES_GPU.txt" > "$cases_11"
+          run_checked run_cases 11_PW_GPU 4 "$cases_11" '^[^#].*_.*$'
+          run_checked run_cases 11_PW_GPU 2 "$CI_SRC/tests/11_PW_GPU/CASES_GPU.txt" '^scf_bpcg$'
+          run_checked run_cases 11_PW_GPU 1 "$CI_SRC/tests/11_PW_GPU/CASES_GPU.txt" '^scf_out_wf$'
+
+          for suite in 12_NAO_Gamma_GPU 13_NAO_multik_GPU 15_rtTDDFT_GPU; do
+            run_checked run_cases "$suite" 4 CASES_GPU.txt '^[^#].*_.*$'
+          done
+          run_checked run_cases 16_SDFT_GPU 2 CASES_GPU.txt '^[^#].*_.*$'
+
+          if [[ "${RUN_CUSOLVERMP:-false}" == true ]]; then
+            tmp=$(mktemp -d /tmp/abacus-sai-cusolvermp.XXXXXX)
+            trap 'rm -rf "$tmp"' EXIT
+            cp -a "$CI_SRC/tests/15_rtTDDFT_GPU" "$tmp/"
+            ln -s "$CI_SRC/tests/integrate" "$tmp/integrate"
+            ln -s "$CI_SRC/tests/PP_ORB" "$tmp/PP_ORB"
+            sed -i 's/^ks_solver[[:space:]]\+cusolver[[:space:]]*$/ks_solver          cusolvermp/' \
+              "$tmp/15_rtTDDFT_GPU/11_NO_O3_TDDFT_GPU/INPUT"
+            if ! (cd "$tmp/15_rtTDDFT_GPU" && timeout --signal=TERM --kill-after=30s 10m \
+              bash ../integrate/Autotest.sh -a "$ABACUS" -n 4 -o 1 \
+              -f CASES_GPU.txt -r '^11_NO_O3_TDDFT_GPU$'); then
+              suite_status=1
+              echo "[WARN] cuSolverMp RT-TDDFT smoke failed"
+            fi
+          fi
+          echo "SAI_GPU_VALIDATION_PASSED mp_version=$MP_VERSION"
+          exit "$suite_status"
+          SBATCH
+          sed -i "s|__RESULT_ROOT__|$RESULT_ROOT|g" "$script"
+          export CI_SRC RESULT_ROOT ABACUS=/home/sai-partner/saiuser01/agent/abacus_beta6_build/install-full-sm70-avx2-mp09/bin/abacus MP_VERSION RUN_CUSOLVERMP
+          set +e
+          sbatch --wait --parsable --export=ALL "$script" | tee "$log"
+          status=${PIPESTATUS[0]}
+          set -e
+          cat "$RESULT_ROOT"/sai-gpu-*.out >> "$log" 2>/dev/null || true
+          exit "$status"
+
+      - name: Upload SAI logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: sai-gpu-full-${{ github.run_id }}
+          path: |
+            ${{ env.CI_SRC }}/**/log.txt
+            ${{ env.CI_SRC }}/**/result.out
+            ${{ env.CI_SRC }}/sai-gpu-${{ github.run_id }}.log
+            ${{ env.RESULT_ROOT }}/sai-gpu-*.out
+          if-no-files-found: warn
+
+      - name: Cleanup shared checkout
+        if: always()
+        run: rm -rf "$CI_SRC"