ci: continue GPU diagnostics after suite failures

Author: Codex

.github/workflows/sai-gpu-full.yml

@@ -95,8 +95,22 @@ jobs:
             CUSOLVERMP="$MATH/targets/x86_64-linux"
             CUBLASMP="$MATH/targets/x86_64-linux"
           fi
+          export PATH="$MPI/bin:$PATH"
+          echo "MPI=$MPI"
+          echo "CUDA=$CUDA"
+          echo "CUSOLVERMP=$CUSOLVERMP"
+          echo "CUBLASMP=$CUBLASMP"
+          ls -l "$CUSOLVERMP"/lib/libcusolverMp.so* "$CUBLASMP"/lib/libcublasMp.so* 2>/dev/null || true
           export LD_LIBRARY_PATH="$MPI/lib:$CUDA/lib64:$CUSOLVERMP/lib:$CUBLASMP/lib:$MATH/lib64:$MATH/targets/x86_64-linux/lib:$NCCL/lib:/home/sai-partner/saiuser01/agent/abacus_beta6_build/install-full-sm70-avx2-mp09/lib:/home/sai-partner/saiuser01/agent/abacus_beta6_build/deps-full/lib:/home/sai-partner/saiuser01/agent/abacus_beta6_build/deps-full/lib64:/home/sai-partner/saiuser01/agent/abacus_beta6_build/deps-deepmd-pt-only/lib:/opt/devtools/elpa/elpa-2026.02.001-2603-gnu/nvidia/lib:/opt/devtools/libxc/libxc-7.0.0-avx2/lib:/opt/devtools/saiblas/2603-gnu-avx2/lib:/opt/devtools/fftw/fftw-3.3.10/lib:/opt/apps/conda_env/deepmd-kit-3.1.2/lib:/opt/apps/abacus/abacus-develop-3.9.0.26_nvhpc263gnu/toolchain/install/NEP_CPU-main/lib:${LD_LIBRARY_PATH:-}"
 
+          suite_status=0
+          run_checked() {
+            if ! "$@"; then
+              suite_status=1
+              echo "[WARN] test command failed; continuing to collect remaining GPU diagnostics"
+            fi
+          }
+
           run_cases() {
             local suite="$1"
             local ranks="$2"
@@ -114,14 +128,14 @@ jobs:
 
           cases_11="$CI_SRC/tests/11_PW_GPU/CASES_CI_GPU.txt"
           grep -v -E '^(scf_bpcg|scf_out_wf)$' "$CI_SRC/tests/11_PW_GPU/CASES_GPU.txt" > "$cases_11"
-          run_cases 11_PW_GPU 4 "$cases_11" '^[^#].*_.*$'
-          run_cases 11_PW_GPU 2 "$CI_SRC/tests/11_PW_GPU/CASES_GPU.txt" '^scf_bpcg$'
-          run_cases 11_PW_GPU 1 "$CI_SRC/tests/11_PW_GPU/CASES_GPU.txt" '^scf_out_wf$'
+          run_checked run_cases 11_PW_GPU 4 "$cases_11" '^[^#].*_.*$'
+          run_checked run_cases 11_PW_GPU 2 "$CI_SRC/tests/11_PW_GPU/CASES_GPU.txt" '^scf_bpcg$'
+          run_checked run_cases 11_PW_GPU 1 "$CI_SRC/tests/11_PW_GPU/CASES_GPU.txt" '^scf_out_wf$'
 
           for suite in 12_NAO_Gamma_GPU 13_NAO_multik_GPU 15_rtTDDFT_GPU; do
-            run_cases "$suite" 4 CASES_GPU.txt '^[^#].*_.*$'
+            run_checked run_cases "$suite" 4 CASES_GPU.txt '^[^#].*_.*$'
           done
-          run_cases 16_SDFT_GPU 2 CASES_GPU.txt '^[^#].*_.*$'
+          run_checked run_cases 16_SDFT_GPU 2 CASES_GPU.txt '^[^#].*_.*$'
 
           if [[ "${RUN_CUSOLVERMP:-false}" == true ]]; then
             tmp=$(mktemp -d /tmp/abacus-sai-cusolvermp.XXXXXX)
@@ -131,11 +145,15 @@ jobs:
             ln -s "$CI_SRC/tests/PP_ORB" "$tmp/PP_ORB"
             sed -i 's/^ks_solver[[:space:]]\+cusolver[[:space:]]*$/ks_solver          cusolvermp/' \
               "$tmp/15_rtTDDFT_GPU/11_NO_O3_TDDFT_GPU/INPUT"
-            (cd "$tmp/15_rtTDDFT_GPU" && timeout --signal=TERM --kill-after=30s 10m \
+            if ! (cd "$tmp/15_rtTDDFT_GPU" && timeout --signal=TERM --kill-after=30s 10m \
               bash ../integrate/Autotest.sh -a "$ABACUS" -n 4 -o 1 \
-              -f CASES_GPU.txt -r '^11_NO_O3_TDDFT_GPU$')
+              -f CASES_GPU.txt -r '^11_NO_O3_TDDFT_GPU$'); then
+              suite_status=1
+              echo "[WARN] cuSolverMp RT-TDDFT smoke failed"
+            fi
           fi
           echo "SAI_GPU_VALIDATION_PASSED mp_version=$MP_VERSION"
+          exit "$suite_status"
           SBATCH
           sed -i "s|__RESULT_ROOT__|$RESULT_ROOT|g" "$script"
           export CI_SRC RESULT_ROOT ABACUS=/home/sai-partner/saiuser01/agent/abacus_beta6_build/install-full-sm70-avx2-mp09/bin/abacus MP_VERSION RUN_CUSOLVERMP