From d5ee3efa058ef38feccea9223443c8b86f2b8377 Mon Sep 17 00:00:00 2001 From: Jiacheng Xu <13862180016@163.com> Date: Mon, 20 Jul 2026 06:32:47 +0800 Subject: [PATCH 1/6] ci: add SAI GPU full validation workflow --- .github/workflows/sai-gpu-full.yml | 140 +++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 .github/workflows/sai-gpu-full.yml diff --git a/.github/workflows/sai-gpu-full.yml b/.github/workflows/sai-gpu-full.yml new file mode 100644 index 0000000000..da1d3a2168 --- /dev/null +++ b/.github/workflows/sai-gpu-full.yml @@ -0,0 +1,140 @@ +name: SAI GPU Full Validation + +on: + workflow_dispatch: + inputs: + mp_version: + description: "cuSolverMp/cuBLASMp runtime: project (0.9.x) or system (NVHPC 0.7.x)" + required: true + default: project + type: choice + options: + - project + - system + run_cusolvermp: + description: "Run the O3 RT-TDDFT cuSolverMp smoke case" + required: true + default: true + type: boolean + +permissions: + contents: read + +concurrency: + group: sai-gpu-full + cancel-in-progress: false + +jobs: + gpu-full: + name: SAI GPU full suite (${{ inputs.mp_version }}) + runs-on: [self-hosted, linux, x64, sai-slurm, v100, cusolvermp] + timeout-minutes: 75 + environment: sai-gpu + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Stage checkout on shared SAI filesystem + run: | + set -euo pipefail + export CI_SRC=/home/sai-partner/saiuser01/agent/abacus_beta6_build/ci-checkouts/${GITHUB_RUN_ID} + rm -rf "$CI_SRC" + mkdir -p "$CI_SRC" + rsync -a --delete --exclude .git/ "$GITHUB_WORKSPACE/" "$CI_SRC/" + echo "CI_SRC=$CI_SRC" >> "$GITHUB_ENV" + + - name: Submit Slurm GPU validation + env: + MP_VERSION: ${{ inputs.mp_version }} + RUN_CUSOLVERMP: ${{ inputs.run_cusolvermp }} + run: | + set -euo pipefail + script="$RUNNER_TEMP/sai-gpu-${GITHUB_RUN_ID}.sbatch" + log="$CI_SRC/sai-gpu-${GITHUB_RUN_ID}.log" + cat > "$script" <<'SBATCH' + #!/usr/bin/env bash + #SBATCH --job-name=abacus-sai-ci + #SBATCH --partition=16V100 + #SBATCH --qos=huge-gpu + #SBATCH --nodes=1 + #SBATCH --ntasks=4 + #SBATCH --gpus-per-node=4 + #SBATCH --time=01:00:00 + #SBATCH --output=__CI_SRC__/sai-gpu-%j.out + + set -euo pipefail + : "${CI_SRC:?}" + : "${ABACUS:?}" + : "${MP_VERSION:?}" + export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-} + export LD_PRELOAD=${LD_PRELOAD:-} + source /etc/profile.d/lmod.sh + module purge + module load nvhpc/26.3-gnu-cuda12-tuned + module load openmpi/5.0.10-nvhpc26.3-gnu-cuda12-auto + module load fftw/3.3.10 libxc/7.0.0-auto saiblas/2603-gnu-auto + module load elpa/2026.02.001-2603-gnu cuda/12.9.1 + module unload nccl + source /opt/sai_config/mps_mapping.d/16V100.bash + export NCCL_IB_DISABLE=1 + export OMP_NUM_THREADS=1 + export OMPI_MCA_rmaps_base_mapping_policy="$MAP_OPT" + + MPI=/opt/devtools/openmpi/openmpi-5.0.10-nvhpc263-gnu-cuda12-avx2 + NV=/opt/devtools/nvidia/hpc_sdk/Linux_x86_64/26.3 + CUDA=/opt/devtools/nvidia/cuda-12.9.1 + MATH="$NV/math_libs/12.9" + NCCL="$NV/comm_libs/12.9/nccl-2.29" + if [[ "$MP_VERSION" == project ]]; then + CUSOLVERMP=/home/sai-partner/saiuser01/agent/abacus_beta6_build/deps-mp-0.9/nvidia/cu12 + CUBLASMP=/home/sai-partner/saiuser01/agent/abacus_beta6_build/deps-mp-0.9/nvidia/cublasmp/cu12 + else + CUSOLVERMP="$MATH/targets/x86_64-linux" + CUBLASMP="$MATH/targets/x86_64-linux" + fi + export LD_LIBRARY_PATH="$MPI/lib:$CUDA/lib64:$CUSOLVERMP/lib:$CUBLASMP/lib:$MATH/lib64:$MATH/targets/x86_64-linux/lib:$NCCL/lib:/home/sai-partner/saiuser01/agent/abacus_beta6_build/install-full-sm70-avx2-mp09/lib:/home/sai-partner/saiuser01/agent/abacus_beta6_build/deps-full/lib:/home/sai-partner/saiuser01/agent/abacus_beta6_build/deps-full/lib64:/home/sai-partner/saiuser01/agent/abacus_beta6_build/deps-deepmd-pt-only/lib:/opt/devtools/elpa/elpa-2026.02.001-2603-gnu/nvidia/lib:/opt/devtools/libxc/libxc-7.0.0-avx2/lib:/opt/devtools/saiblas/2603-gnu-avx2/lib:/opt/devtools/fftw/fftw-3.3.10/lib:/opt/apps/conda_env/deepmd-kit-3.1.2/lib:/opt/apps/abacus/abacus-develop-3.9.0.26_nvhpc263gnu/toolchain/install/NEP_CPU-main/lib:${LD_LIBRARY_PATH:-}" + + run_suite() { + local suite="$1" + (cd "$CI_SRC/tests/$suite" && bash ../integrate/Autotest.sh \ + -a "$ABACUS" -n 4 -o 1 -f CASES_GPU.txt) + } + + for suite in 11_PW_GPU 12_NAO_Gamma_GPU 13_NAO_multik_GPU 15_rtTDDFT_GPU 16_SDFT_GPU; do + run_suite "$suite" + done + + if [[ "${RUN_CUSOLVERMP:-false}" == true ]]; then + tmp=$(mktemp -d /tmp/abacus-sai-cusolvermp.XXXXXX) + trap 'rm -rf "$tmp"' EXIT + cp -a "$CI_SRC/tests/15_rtTDDFT_GPU" "$tmp/" + ln -s "$CI_SRC/tests/integrate" "$tmp/integrate" + ln -s "$CI_SRC/tests/PP_ORB" "$tmp/PP_ORB" + sed -i 's/^ks_solver[[:space:]]\+cusolver[[:space:]]*$/ks_solver cusolvermp/' \ + "$tmp/15_rtTDDFT_GPU/11_NO_O3_TDDFT_GPU/INPUT" + (cd "$tmp/15_rtTDDFT_GPU" && timeout --signal=TERM --kill-after=30s 10m \ + bash ../integrate/Autotest.sh -a "$ABACUS" -n 4 -o 1 \ + -f CASES_GPU.txt -r '^11_NO_O3_TDDFT_GPU$') + fi + echo "SAI_GPU_VALIDATION_PASSED mp_version=$MP_VERSION" + SBATCH + sed -i "s|__CI_SRC__|$CI_SRC|g" "$script" + export CI_SRC ABACUS=/home/sai-partner/saiuser01/agent/abacus_beta6_build/install-full-sm70-avx2-mp09/bin/abacus MP_VERSION RUN_CUSOLVERMP + sbatch --wait --parsable --export=ALL "$script" | tee "$log" + cat "$CI_SRC"/sai-gpu-*.out >> "$log" + grep -q 'SAI_GPU_VALIDATION_PASSED' "$log" + + - name: Upload SAI logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: sai-gpu-full-${{ github.run_id }} + path: | + ${{ env.CI_SRC }}/**/log.txt + ${{ env.CI_SRC }}/**/result.out + ${{ env.CI_SRC }}/sai-gpu-${{ github.run_id }}.log + if-no-files-found: warn + + - name: Cleanup shared checkout + if: always() + run: rm -rf "$CI_SRC" From e55788fd75a046a193b07f56d2e5cc0685892ae6 Mon Sep 17 00:00:00 2001 From: Jiacheng Xu <13862180016@163.com> Date: Mon, 20 Jul 2026 06:38:01 +0800 Subject: [PATCH 2/6] ci: preserve SAI Slurm logs on failure --- .github/workflows/sai-gpu-full.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/sai-gpu-full.yml b/.github/workflows/sai-gpu-full.yml index da1d3a2168..27e97e051c 100644 --- a/.github/workflows/sai-gpu-full.yml +++ b/.github/workflows/sai-gpu-full.yml @@ -38,10 +38,13 @@ jobs: run: | set -euo pipefail export CI_SRC=/home/sai-partner/saiuser01/agent/abacus_beta6_build/ci-checkouts/${GITHUB_RUN_ID} + export RESULT_ROOT=/home/sai-partner/saiuser01/agent/abacus_beta6_build/ci-results/${GITHUB_RUN_ID} rm -rf "$CI_SRC" - mkdir -p "$CI_SRC" + rm -rf "$RESULT_ROOT" + mkdir -p "$CI_SRC" "$RESULT_ROOT" rsync -a --delete --exclude .git/ "$GITHUB_WORKSPACE/" "$CI_SRC/" echo "CI_SRC=$CI_SRC" >> "$GITHUB_ENV" + echo "RESULT_ROOT=$RESULT_ROOT" >> "$GITHUB_ENV" - name: Submit Slurm GPU validation env: @@ -60,7 +63,7 @@ jobs: #SBATCH --ntasks=4 #SBATCH --gpus-per-node=4 #SBATCH --time=01:00:00 - #SBATCH --output=__CI_SRC__/sai-gpu-%j.out + #SBATCH --output=__RESULT_ROOT__/sai-gpu-%j.out set -euo pipefail : "${CI_SRC:?}" @@ -118,11 +121,14 @@ jobs: fi echo "SAI_GPU_VALIDATION_PASSED mp_version=$MP_VERSION" SBATCH - sed -i "s|__CI_SRC__|$CI_SRC|g" "$script" - export CI_SRC ABACUS=/home/sai-partner/saiuser01/agent/abacus_beta6_build/install-full-sm70-avx2-mp09/bin/abacus MP_VERSION RUN_CUSOLVERMP + sed -i "s|__RESULT_ROOT__|$RESULT_ROOT|g" "$script" + export CI_SRC RESULT_ROOT ABACUS=/home/sai-partner/saiuser01/agent/abacus_beta6_build/install-full-sm70-avx2-mp09/bin/abacus MP_VERSION RUN_CUSOLVERMP + set +e sbatch --wait --parsable --export=ALL "$script" | tee "$log" - cat "$CI_SRC"/sai-gpu-*.out >> "$log" - grep -q 'SAI_GPU_VALIDATION_PASSED' "$log" + status=${PIPESTATUS[0]} + set -e + cat "$RESULT_ROOT"/sai-gpu-*.out >> "$log" 2>/dev/null || true + exit "$status" - name: Upload SAI logs if: always() @@ -133,6 +139,7 @@ jobs: ${{ env.CI_SRC }}/**/log.txt ${{ env.CI_SRC }}/**/result.out ${{ env.CI_SRC }}/sai-gpu-${{ github.run_id }}.log + ${{ env.RESULT_ROOT }}/sai-gpu-*.out if-no-files-found: warn - name: Cleanup shared checkout From 0fa3d7a60d8ad778deca55c687c92939ed3540bc Mon Sep 17 00:00:00 2001 From: Jiacheng Xu <13862180016@163.com> Date: Mon, 20 Jul 2026 06:42:11 +0800 Subject: [PATCH 3/6] ci: use compatible MPI width for PW GPU cases --- .github/workflows/sai-gpu-full.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sai-gpu-full.yml b/.github/workflows/sai-gpu-full.yml index 27e97e051c..0bef56946e 100644 --- a/.github/workflows/sai-gpu-full.yml +++ b/.github/workflows/sai-gpu-full.yml @@ -99,8 +99,19 @@ jobs: run_suite() { local suite="$1" - (cd "$CI_SRC/tests/$suite" && bash ../integrate/Autotest.sh \ - -a "$ABACUS" -n 4 -o 1 -f CASES_GPU.txt) + local ranks=4 + if [[ "$suite" == 11_PW_GPU ]]; then + # bpcg/use_k_continuity and out_wfc cases are intentionally + # single-rank because their k-point layouts are not MPI-safe. + ranks=1 + fi + if [[ "$ranks" == 1 ]]; then + (unset OMPI_MCA_rmaps_base_mapping_policy; cd "$CI_SRC/tests/$suite" && \ + bash ../integrate/Autotest.sh -a "$ABACUS" -n 1 -o 1 -f CASES_GPU.txt) + else + (cd "$CI_SRC/tests/$suite" && bash ../integrate/Autotest.sh \ + -a "$ABACUS" -n "$ranks" -o 1 -f CASES_GPU.txt) + fi } for suite in 11_PW_GPU 12_NAO_Gamma_GPU 13_NAO_multik_GPU 15_rtTDDFT_GPU 16_SDFT_GPU; do From c5f3c39dca49d578ba3833e8ca3e7d658feffae7 Mon Sep 17 00:00:00 2001 From: Jiacheng Xu <13862180016@163.com> Date: Mon, 20 Jul 2026 06:46:19 +0800 Subject: [PATCH 4/6] ci: run PW GPU cases with compatible rank widths --- .github/workflows/sai-gpu-full.yml | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/.github/workflows/sai-gpu-full.yml b/.github/workflows/sai-gpu-full.yml index 0bef56946e..1009260684 100644 --- a/.github/workflows/sai-gpu-full.yml +++ b/.github/workflows/sai-gpu-full.yml @@ -97,25 +97,29 @@ jobs: fi export LD_LIBRARY_PATH="$MPI/lib:$CUDA/lib64:$CUSOLVERMP/lib:$CUBLASMP/lib:$MATH/lib64:$MATH/targets/x86_64-linux/lib:$NCCL/lib:/home/sai-partner/saiuser01/agent/abacus_beta6_build/install-full-sm70-avx2-mp09/lib:/home/sai-partner/saiuser01/agent/abacus_beta6_build/deps-full/lib:/home/sai-partner/saiuser01/agent/abacus_beta6_build/deps-full/lib64:/home/sai-partner/saiuser01/agent/abacus_beta6_build/deps-deepmd-pt-only/lib:/opt/devtools/elpa/elpa-2026.02.001-2603-gnu/nvidia/lib:/opt/devtools/libxc/libxc-7.0.0-avx2/lib:/opt/devtools/saiblas/2603-gnu-avx2/lib:/opt/devtools/fftw/fftw-3.3.10/lib:/opt/apps/conda_env/deepmd-kit-3.1.2/lib:/opt/apps/abacus/abacus-develop-3.9.0.26_nvhpc263gnu/toolchain/install/NEP_CPU-main/lib:${LD_LIBRARY_PATH:-}" - run_suite() { + run_cases() { local suite="$1" - local ranks=4 - if [[ "$suite" == 11_PW_GPU ]]; then - # bpcg/use_k_continuity and out_wfc cases are intentionally - # single-rank because their k-point layouts are not MPI-safe. - ranks=1 - fi - if [[ "$ranks" == 1 ]]; then + local ranks="$2" + local cases_file="$3" + local regex="$4" + if [[ "$ranks" -lt 4 ]]; then (unset OMPI_MCA_rmaps_base_mapping_policy; cd "$CI_SRC/tests/$suite" && \ - bash ../integrate/Autotest.sh -a "$ABACUS" -n 1 -o 1 -f CASES_GPU.txt) + bash ../integrate/Autotest.sh -a "$ABACUS" -n "$ranks" -o 1 \ + -f "$cases_file" -r "$regex") else (cd "$CI_SRC/tests/$suite" && bash ../integrate/Autotest.sh \ - -a "$ABACUS" -n "$ranks" -o 1 -f CASES_GPU.txt) + -a "$ABACUS" -n "$ranks" -o 1 -f "$cases_file" -r "$regex") fi } - for suite in 11_PW_GPU 12_NAO_Gamma_GPU 13_NAO_multik_GPU 15_rtTDDFT_GPU 16_SDFT_GPU; do - run_suite "$suite" + cases_11="$CI_SRC/tests/11_PW_GPU/CASES_CI_GPU.txt" + grep -v -E '^(scf_bpcg|scf_out_wf)$' "$CI_SRC/tests/11_PW_GPU/CASES_GPU.txt" > "$cases_11" + run_cases 11_PW_GPU 4 "$cases_11" '^[^#].*_.*$' + run_cases 11_PW_GPU 2 "$CI_SRC/tests/11_PW_GPU/CASES_GPU.txt" '^scf_bpcg$' + run_cases 11_PW_GPU 1 "$CI_SRC/tests/11_PW_GPU/CASES_GPU.txt" '^scf_out_wf$' + + for suite in 12_NAO_Gamma_GPU 13_NAO_multik_GPU 15_rtTDDFT_GPU 16_SDFT_GPU; do + run_cases "$suite" 4 CASES_GPU.txt '^[^#].*_.*$' done if [[ "${RUN_CUSOLVERMP:-false}" == true ]]; then From 2057c861c41a287b687aae4ddc89e266e5cbaec1 Mon Sep 17 00:00:00 2001 From: Jiacheng Xu <13862180016@163.com> Date: Mon, 20 Jul 2026 06:59:16 +0800 Subject: [PATCH 5/6] ci: use SDFT-compatible MPI width --- .github/workflows/sai-gpu-full.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sai-gpu-full.yml b/.github/workflows/sai-gpu-full.yml index 1009260684..2412c94654 100644 --- a/.github/workflows/sai-gpu-full.yml +++ b/.github/workflows/sai-gpu-full.yml @@ -118,9 +118,10 @@ jobs: run_cases 11_PW_GPU 2 "$CI_SRC/tests/11_PW_GPU/CASES_GPU.txt" '^scf_bpcg$' run_cases 11_PW_GPU 1 "$CI_SRC/tests/11_PW_GPU/CASES_GPU.txt" '^scf_out_wf$' - for suite in 12_NAO_Gamma_GPU 13_NAO_multik_GPU 15_rtTDDFT_GPU 16_SDFT_GPU; do + for suite in 12_NAO_Gamma_GPU 13_NAO_multik_GPU 15_rtTDDFT_GPU; do run_cases "$suite" 4 CASES_GPU.txt '^[^#].*_.*$' done + run_cases 16_SDFT_GPU 2 CASES_GPU.txt '^[^#].*_.*$' if [[ "${RUN_CUSOLVERMP:-false}" == true ]]; then tmp=$(mktemp -d /tmp/abacus-sai-cusolvermp.XXXXXX) From 44b5034524c04d41abc67f451699e699b2fdf9d1 Mon Sep 17 00:00:00 2001 From: Codex Date: Mon, 20 Jul 2026 07:24:47 +0800 Subject: [PATCH 6/6] ci: continue GPU diagnostics after suite failures --- .github/workflows/sai-gpu-full.yml | 32 +++++++++++++++++++++++------- 1 file changed, 25 insertions(+), 7 deletions(-) diff --git a/.github/workflows/sai-gpu-full.yml b/.github/workflows/sai-gpu-full.yml index 2412c94654..eabaa65bb8 100644 --- a/.github/workflows/sai-gpu-full.yml +++ b/.github/workflows/sai-gpu-full.yml @@ -95,8 +95,22 @@ jobs: CUSOLVERMP="$MATH/targets/x86_64-linux" CUBLASMP="$MATH/targets/x86_64-linux" fi + export PATH="$MPI/bin:$PATH" + echo "MPI=$MPI" + echo "CUDA=$CUDA" + echo "CUSOLVERMP=$CUSOLVERMP" + echo "CUBLASMP=$CUBLASMP" + ls -l "$CUSOLVERMP"/lib/libcusolverMp.so* "$CUBLASMP"/lib/libcublasMp.so* 2>/dev/null || true export LD_LIBRARY_PATH="$MPI/lib:$CUDA/lib64:$CUSOLVERMP/lib:$CUBLASMP/lib:$MATH/lib64:$MATH/targets/x86_64-linux/lib:$NCCL/lib:/home/sai-partner/saiuser01/agent/abacus_beta6_build/install-full-sm70-avx2-mp09/lib:/home/sai-partner/saiuser01/agent/abacus_beta6_build/deps-full/lib:/home/sai-partner/saiuser01/agent/abacus_beta6_build/deps-full/lib64:/home/sai-partner/saiuser01/agent/abacus_beta6_build/deps-deepmd-pt-only/lib:/opt/devtools/elpa/elpa-2026.02.001-2603-gnu/nvidia/lib:/opt/devtools/libxc/libxc-7.0.0-avx2/lib:/opt/devtools/saiblas/2603-gnu-avx2/lib:/opt/devtools/fftw/fftw-3.3.10/lib:/opt/apps/conda_env/deepmd-kit-3.1.2/lib:/opt/apps/abacus/abacus-develop-3.9.0.26_nvhpc263gnu/toolchain/install/NEP_CPU-main/lib:${LD_LIBRARY_PATH:-}" + suite_status=0 + run_checked() { + if ! "$@"; then + suite_status=1 + echo "[WARN] test command failed; continuing to collect remaining GPU diagnostics" + fi + } + run_cases() { local suite="$1" local ranks="$2" @@ -114,14 +128,14 @@ jobs: cases_11="$CI_SRC/tests/11_PW_GPU/CASES_CI_GPU.txt" grep -v -E '^(scf_bpcg|scf_out_wf)$' "$CI_SRC/tests/11_PW_GPU/CASES_GPU.txt" > "$cases_11" - run_cases 11_PW_GPU 4 "$cases_11" '^[^#].*_.*$' - run_cases 11_PW_GPU 2 "$CI_SRC/tests/11_PW_GPU/CASES_GPU.txt" '^scf_bpcg$' - run_cases 11_PW_GPU 1 "$CI_SRC/tests/11_PW_GPU/CASES_GPU.txt" '^scf_out_wf$' + run_checked run_cases 11_PW_GPU 4 "$cases_11" '^[^#].*_.*$' + run_checked run_cases 11_PW_GPU 2 "$CI_SRC/tests/11_PW_GPU/CASES_GPU.txt" '^scf_bpcg$' + run_checked run_cases 11_PW_GPU 1 "$CI_SRC/tests/11_PW_GPU/CASES_GPU.txt" '^scf_out_wf$' for suite in 12_NAO_Gamma_GPU 13_NAO_multik_GPU 15_rtTDDFT_GPU; do - run_cases "$suite" 4 CASES_GPU.txt '^[^#].*_.*$' + run_checked run_cases "$suite" 4 CASES_GPU.txt '^[^#].*_.*$' done - run_cases 16_SDFT_GPU 2 CASES_GPU.txt '^[^#].*_.*$' + run_checked run_cases 16_SDFT_GPU 2 CASES_GPU.txt '^[^#].*_.*$' if [[ "${RUN_CUSOLVERMP:-false}" == true ]]; then tmp=$(mktemp -d /tmp/abacus-sai-cusolvermp.XXXXXX) @@ -131,11 +145,15 @@ jobs: ln -s "$CI_SRC/tests/PP_ORB" "$tmp/PP_ORB" sed -i 's/^ks_solver[[:space:]]\+cusolver[[:space:]]*$/ks_solver cusolvermp/' \ "$tmp/15_rtTDDFT_GPU/11_NO_O3_TDDFT_GPU/INPUT" - (cd "$tmp/15_rtTDDFT_GPU" && timeout --signal=TERM --kill-after=30s 10m \ + if ! (cd "$tmp/15_rtTDDFT_GPU" && timeout --signal=TERM --kill-after=30s 10m \ bash ../integrate/Autotest.sh -a "$ABACUS" -n 4 -o 1 \ - -f CASES_GPU.txt -r '^11_NO_O3_TDDFT_GPU$') + -f CASES_GPU.txt -r '^11_NO_O3_TDDFT_GPU$'); then + suite_status=1 + echo "[WARN] cuSolverMp RT-TDDFT smoke failed" + fi fi echo "SAI_GPU_VALIDATION_PASSED mp_version=$MP_VERSION" + exit "$suite_status" SBATCH sed -i "s|__RESULT_ROOT__|$RESULT_ROOT|g" "$script" export CI_SRC RESULT_ROOT ABACUS=/home/sai-partner/saiuser01/agent/abacus_beta6_build/install-full-sm70-avx2-mp09/bin/abacus MP_VERSION RUN_CUSOLVERMP