Skip to content

Commit 5821e69

Browse files
Stardust0831Stardust0831
committed
cmake: guard known-affected cuSolverMp versions
1 parent 83e5099 commit 5821e69

4 files changed

Lines changed: 146 additions & 1 deletion

File tree

‎CMakeLists.txt‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,9 @@ option(ENABLE_GOOGLEBENCH "Enable GOOGLE-benchmark usage" OFF)
5353
option(ENABLE_RAPIDJSON "Enable rapid-json usage" OFF)
5454
option(ENABLE_CNPY "Enable cnpy usage" OFF)
5555
option(ENABLE_CUSOLVERMP "Enable cusolvermp" OFF)
56+
option(ALLOW_KNOWN_AFFECTED_CUSOLVERMP
57+
"Allow NCCL-backed cuSolverMp 0.7.x/0.8.x despite known Sygvd defects"
58+
OFF)
5659
option(ENABLE_NCCL_PARALLEL_DEVICE "Enable NCCL-backed collectives in parallel_device" OFF)
5760

5861
# ==============================================================================
@@ -96,6 +99,16 @@ endif()
9699
# ABACUS has declared its OFF-by-default option above.
97100
include(CTest)
98101

102+
if(BUILD_TESTING)
103+
add_test(
104+
NAME cmake_cusolvermp_version_guard
105+
COMMAND
106+
${CMAKE_COMMAND}
107+
-DABACUS_SOURCE_DIR=${PROJECT_SOURCE_DIR}
108+
-DTEST_BINARY_DIR=${PROJECT_BINARY_DIR}/cmake-tests
109+
-P ${PROJECT_SOURCE_DIR}/cmake/tests/test_cusolvermp_version_guard.cmake)
110+
endif()
111+
99112
if(NOT DEFINED NVHPC_ROOT_DIR AND DEFINED ENV{NVHPC_ROOT})
100113
set(NVHPC_ROOT_DIR
101114
"$ENV{NVHPC_ROOT}"

‎cmake/modules/SetupCuSolverMp.cmake‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,28 @@ function(abacus_setup_cusolvermp)
6868
"Unable to detect cuSOLVERMp version from header. Using NCCL backend by default.")
6969
endif()
7070

71+
if(NOT _use_cal
72+
AND CUSOLVERMP_VERSION_STR
73+
AND CUSOLVERMP_VERSION_STR VERSION_GREATER_EQUAL "0.7.0"
74+
AND CUSOLVERMP_VERSION_STR VERSION_LESS "0.9.0")
75+
string(CONCAT _unsupported_version_message
76+
"cuSOLVERMp ${CUSOLVERMP_VERSION_STR} selects the NCCL backend but is in "
77+
"the known-affected 0.7.x/0.8.x range. NVIDIA documents STEDC defects "
78+
"with non-power-of-two block sizes and certain 2D configurations; "
79+
"Syevd/Sygvd use STEDC internally. The defect exists through version 0.8.0. "
80+
"The recommended stack is cuSOLVERMp 0.9.0 with cuBLASMp 0.9.1 or newer. See "
81+
"https://docs.nvidia.com/cuda/cusolvermp/release_notes/index.html#cusolvermp-v0-9-0.")
82+
if(ALLOW_KNOWN_AFFECTED_CUSOLVERMP)
83+
message(WARNING
84+
"${_unsupported_version_message} Proceeding because "
85+
"ALLOW_KNOWN_AFFECTED_CUSOLVERMP=ON; distributed Sygvd may fail or hang.")
86+
else()
87+
message(FATAL_ERROR
88+
"${_unsupported_version_message} To proceed at your own risk, configure "
89+
"with -DALLOW_KNOWN_AFFECTED_CUSOLVERMP=ON.")
90+
endif()
91+
endif()
92+
7193
# Raise the variable to the caller's scope
7294
set(_use_cal ${_use_cal} PARENT_SCOPE)
7395

‎cmake/tests/test_cusolvermp_version_guard.cmake‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
cmake_minimum_required(VERSION 3.16)
2+
3+
if(NOT DEFINED ABACUS_SOURCE_DIR OR NOT DEFINED TEST_BINARY_DIR)
4+
message(FATAL_ERROR "ABACUS_SOURCE_DIR and TEST_BINARY_DIR are required")
5+
endif()
6+
7+
set(_test_root "${TEST_BINARY_DIR}/cusolvermp-version-guard")
8+
set(_source_dir "${_test_root}/source")
9+
set(_fake_prefix "${_test_root}/fake-prefix")
10+
file(REMOVE_RECURSE "${_test_root}")
11+
file(MAKE_DIRECTORY "${_source_dir}" "${_fake_prefix}/include" "${_fake_prefix}/lib")
12+
13+
foreach(_library IN ITEMS cusolverMp cal nccl)
14+
file(WRITE "${_fake_prefix}/lib/lib${_library}.so" "")
15+
endforeach()
16+
file(WRITE "${_fake_prefix}/include/cal.h" "")
17+
file(WRITE "${_fake_prefix}/include/nccl.h" "")
18+
19+
file(WRITE "${_source_dir}/CMakeLists.txt" [=[
20+
cmake_minimum_required(VERSION 3.16)
21+
project(cusolvermp_version_guard LANGUAGES NONE)
22+
23+
function(abacus_add_feature_definitions)
24+
endfunction()
25+
26+
include("${ABACUS_SOURCE_DIR}/cmake/modules/SetupCuSolverMp.cmake")
27+
abacus_setup_cusolvermp()
28+
file(WRITE "${CMAKE_BINARY_DIR}/backend.txt" "${_use_cal}")
29+
]=])
30+
31+
function(run_version_case case_name version expected_result expected_backend allow_unsupported)
32+
string(REPLACE "." ";" _version_parts "${version}")
33+
list(GET _version_parts 0 _major)
34+
list(GET _version_parts 1 _minor)
35+
list(GET _version_parts 2 _patch)
36+
file(WRITE "${_fake_prefix}/include/cusolverMp.h"
37+
"#define CUSOLVERMP_VER_MAJOR ${_major}\n"
38+
"#define CUSOLVERMP_VER_MINOR ${_minor}\n"
39+
"#define CUSOLVERMP_VER_PATCH ${_patch}\n")
40+
41+
set(_build_dir "${_test_root}/build-${case_name}")
42+
file(REMOVE_RECURSE "${_build_dir}")
43+
execute_process(
44+
COMMAND
45+
"${CMAKE_COMMAND}"
46+
-S "${_source_dir}"
47+
-B "${_build_dir}"
48+
"-DABACUS_SOURCE_DIR=${ABACUS_SOURCE_DIR}"
49+
"-DCAL_CUSOLVERMP_PATH=${_fake_prefix}"
50+
"-DNCCL_PATH=${_fake_prefix}"
51+
"-DALLOW_KNOWN_AFFECTED_CUSOLVERMP=${allow_unsupported}"
52+
RESULT_VARIABLE _result
53+
OUTPUT_VARIABLE _stdout
54+
ERROR_VARIABLE _stderr)
55+
set(_output "${_stdout}\n${_stderr}")
56+
57+
if(expected_result STREQUAL "success")
58+
if(NOT _result EQUAL 0)
59+
message(FATAL_ERROR "${case_name} unexpectedly failed:\n${_output}")
60+
endif()
61+
file(READ "${_build_dir}/backend.txt" _backend)
62+
if(NOT _backend STREQUAL "${expected_backend}")
63+
message(FATAL_ERROR
64+
"${case_name} selected backend '${_backend}', expected '${expected_backend}'")
65+
endif()
66+
if(allow_unsupported
67+
AND version VERSION_GREATER_EQUAL "0.7.0"
68+
AND version VERSION_LESS "0.9.0"
69+
AND (NOT _output MATCHES "ALLOW_KNOWN_AFFECTED_CUSOLVERMP=ON"
70+
OR NOT _output MATCHES "may fail or hang"))
71+
message(FATAL_ERROR
72+
"${case_name} configured without the expected risk warning:\n${_output}")
73+
endif()
74+
else()
75+
if(_result EQUAL 0)
76+
message(FATAL_ERROR "${case_name} unexpectedly configured successfully")
77+
endif()
78+
if(NOT _output MATCHES "known-affected"
79+
OR NOT _output MATCHES "0\\.7\\.x/0\\.8\\.x range")
80+
message(FATAL_ERROR "${case_name} failed without the expected diagnostic:\n${_output}")
81+
endif()
82+
endif()
83+
endfunction()
84+
85+
run_version_case(cal_060 0.6.0 success ON OFF)
86+
run_version_case(nccl_070 0.7.0 failure OFF OFF)
87+
run_version_case(nccl_072 0.7.2 failure OFF OFF)
88+
run_version_case(nccl_080 0.8.0 failure OFF OFF)
89+
run_version_case(nccl_090 0.9.0 success OFF OFF)
90+
run_version_case(nccl_072_override 0.7.2 success OFF ON)
91+
92+
message(STATUS "cuSOLVERMp version guard cases passed")

‎docs/advanced/acceleration/cuda.md‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,24 @@ To compile and use ABACUS in CUDA mode, you currently need to have an NVIDIA GPU
2424

2525
- Install a driver and toolkit appropriate for your system (SDK is not necessary)
2626

27+
For NCCL-backed multi-GPU diagonalization with `ENABLE_CUSOLVERMP=ON`, the
28+
recommended stack is cuSOLVERMp 0.9.0 with cuBLASMp 0.9.1 or newer. NVIDIA
29+
reports that cuSOLVERMp 0.4.2 through 0.8.0 contain an STEDC defect affecting
30+
non-power-of-two block sizes and certain 2D configurations; `Syevd` and `Sygvd`
31+
use STEDC internally. ABACUS selects NCCL for cuSOLVERMp 0.7.0 and newer, so
32+
versions from 0.7.0 up to, but not including, 0.9.0 stop at configure time by
33+
default. The existing cuSOLVERMp 0.4.0 minimum and CAL selection for versions
34+
older than 0.7.0 are unchanged. The known-issue check can be bypassed with
35+
`-DALLOW_KNOWN_AFFECTED_CUSOLVERMP=ON`, but affected distributed eigensolves
36+
may fail or hang. See the
37+
[cuSOLVERMp 0.9.0 release notes](https://docs.nvidia.com/cuda/cusolvermp/release_notes/index.html#cusolvermp-v0-9-0).
38+
39+
NVIDIA documents cuBLASMp 0.9.1 or newer as a runtime requirement of
40+
cuSOLVERMp 0.9.0. Its current requirements also list NCCL 2.18.5 or newer;
41+
consult the
42+
[cuSOLVERMp requirements](https://docs.nvidia.com/cuda/cusolvermp/getting_started/index.html#hardware-and-software-requirements)
43+
for the supported dependency stack.
44+
2745

2846
## Building ABACUS with the GPU support:
2947

@@ -50,4 +68,4 @@ PW basis:
5068

5169
LCAO basis:
5270
- Unless there is a specific reason, avoid using multiple GPUs, as it can be slower than using a single GPU. This is because the generalized eigenvalue solution of the LCAO basis set will incur additional communication overhead when calculated on multiple cards. When the memory limit of a GPU card makes it insufficient to complete the task, it is recommended to use multiple cards for calculation.
53-
- When using elpa on GPUs, some ELPA internal logs will be output.
71+
- When using elpa on GPUs, some ELPA internal logs will be output.

0 commit comments

Comments
 (0)