Skip to content

Commit c169a85

Browse files
evaleevevaleev
authored
Merge pull request TESSEorg#330 from TESSEorg/ci/cuda
[ci] try adding gpu tests
2 parents 0292283 + 60d6d2a commit c169a85

10 files changed

Lines changed: 231 additions & 40 deletions

File tree

‎.github/workflows/cmake.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ jobs:
7171
sudo dpkg -i cuda-keyring_1.1-1_all.deb
7272
sudo apt update
7373
sudo apt install -y cuda-toolkit-12
74-
echo "CUDA_BUILD_OPTS=-DENABLE_CUDA=ON -DTTG_ENABLE_CUDA=ON -DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-12 -DCMAKE_CUDA_COMPILER=/usr/local/cuda-12/bin/nvcc -DCMAKE_CUDA_HOST_COMPILER=${{ matrix.cxx }}" >> $GITHUB_ENV
74+
echo "CUDA_BUILD_OPTS=-DTTG_ENABLE_CUDA=ON -DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-12 -DCMAKE_CUDA_COMPILER=/usr/local/cuda-12/bin/nvcc -DCMAKE_CUDA_HOST_COMPILER=${{ matrix.cxx }}" >> $GITHUB_ENV
7575
7676
- name: Create Build Environment
7777
# Some projects don't allow in-source building, so create a separate build directory
@@ -140,7 +140,7 @@ jobs:
140140
cmake --build test_install_devsamp_helloworld
141141
$MPIEXEC -n 2 test_install_devsamp_helloworld/helloworld-parsec
142142
$MPIEXEC -n 2 test_install_devsamp_helloworld/helloworld-mad
143-
cmake -S $GITHUB_WORKSPACE/doc/dox/dev/devsamp/fibonacci -B test_install_devsamp_fibonacci -DCMAKE_PREFIX_PATH=${{github.workspace}}/install || (cat /home/runner/work/ttg/ttg/install/lib/cmake/ttg/ttg-config.cmake && cat test_install_devsamp_fibonacci/CMakeFiles/CMakeConfigureLog.yaml)
143+
cmake -S $GITHUB_WORKSPACE/doc/dox/dev/devsamp/fibonacci -B test_install_devsamp_fibonacci -DCMAKE_PREFIX_PATH=${{github.workspace}}/install $CUDA_BUILD_OPTS || (cat /home/runner/work/ttg/ttg/install/lib/cmake/ttg/ttg-config.cmake && cat test_install_devsamp_fibonacci/CMakeFiles/CMakeConfigureLog.yaml)
144144
cmake --build test_install_devsamp_fibonacci
145145
$MPIEXEC -n 2 test_install_devsamp_fibonacci/fibonacci-parsec
146146
cmake -E make_directory test_install_userexamples

‎.github/workflows/gpu.yml‎

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
name: GPU Tests
2+
3+
on:
4+
push:
5+
branches: [ master ]
6+
pull_request:
7+
branches: [ master ]
8+
workflow_dispatch:
9+
10+
concurrency:
11+
group: ${{ github.workflow }}-${{ github.ref }}
12+
cancel-in-progress: ${{ github.ref != 'refs/heads/master'}}
13+
14+
jobs:
15+
cuda-tests:
16+
name: "CUDA Device Tests"
17+
runs-on: self-hosted
18+
19+
env:
20+
CCACHE_DIR: ${{github.workspace}}/build/.ccache
21+
CCACHE_COMPRESS: true
22+
CCACHE_COMPRESSLEVEL: 6
23+
OMPI_MCA_btl_vader_single_copy_mechanism: none
24+
PARSEC_MCA_runtime_bind_threads: 0
25+
MKLROOT: /opt/intel/mkl
26+
Eigen3_DIR: /home/evaleev/code/install/eigen-master
27+
LD_LIBRARY_PATH: /home/evaleev/code/install/gcc/12/lib64
28+
29+
steps:
30+
- uses: actions/checkout@v4
31+
32+
- name: Setup ccache
33+
uses: actions/cache@v4
34+
with:
35+
path: ${{github.workspace}}/build/.ccache
36+
key: ${{ runner.os }}-cuda-ccache-${{ github.sha }}
37+
restore-keys: |
38+
${{ runner.os }}-cuda-ccache-
39+
40+
- name: Create Build Environment
41+
run: cmake -E make_directory ${{github.workspace}}/build
42+
43+
- name: Configure CMake
44+
shell: bash
45+
working-directory: ${{github.workspace}}/build
46+
run: |
47+
cmake $GITHUB_WORKSPACE \
48+
-G Ninja \
49+
-DCMAKE_CXX_COMPILER=/home/evaleev/code/install/gcc/12/bin/g++ \
50+
-DCMAKE_C_COMPILER=/home/evaleev/code/install/gcc/12/bin/gcc \
51+
-DCMAKE_CUDA_HOST_COMPILER=/home/evaleev/code/install/gcc/12/bin/g++ \
52+
-DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc \
53+
-DCMAKE_CUDA_ARCHITECTURES=50 \
54+
-DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda \
55+
-DCMAKE_BUILD_TYPE=Release \
56+
-DBUILD_SHARED_LIBS=OFF \
57+
-DCMAKE_INSTALL_PREFIX=${{github.workspace}}/install \
58+
-DTTG_EXAMPLES=ON \
59+
-DTTG_ENABLE_CUDA=ON \
60+
-DMPIEXEC_PREFLAGS='--bind-to;none;--allow-run-as-root' \
61+
|| (cat CMakeFiles/CMakeConfigureLog.yaml && exit 1)
62+
63+
- name: Build TTG
64+
working-directory: ${{github.workspace}}/build
65+
shell: bash
66+
run: ccache -p && ccache -z && cmake --build . && ccache -s
67+
68+
- name: Build CUDA Examples
69+
working-directory: ${{github.workspace}}/build
70+
shell: bash
71+
run: |
72+
echo "Building CUDA device examples..."
73+
cmake --build . --target bspmm-cuda-parsec || echo "bspmm-cuda-parsec not available"
74+
cmake --build . --target testing_dpotrf_cuda-parsec || echo "testing_dpotrf_cuda-parsec not available"
75+
cmake --build . --target chain-ttg-cuda-parsec || echo "chain-ttg-cuda-parsec not available"
76+
77+
- name: Install
78+
working-directory: ${{github.workspace}}/build
79+
shell: bash
80+
run: cmake --build . --target install
81+
82+
- name: Configure Fibonacci Example
83+
shell: bash
84+
run: |
85+
cmake -E make_directory ${{github.workspace}}/fibonacci-build
86+
cd ${{github.workspace}}/fibonacci-build
87+
cmake ${{github.workspace}}/doc/dox/dev/devsamp/fibonacci \
88+
-G Ninja \
89+
-DCMAKE_CXX_COMPILER=/home/evaleev/code/install/gcc/12/bin/g++ \
90+
-DCMAKE_C_COMPILER=/home/evaleev/code/install/gcc/12/bin/gcc \
91+
-DCMAKE_CUDA_HOST_COMPILER=/home/evaleev/code/install/gcc/12/bin/g++ \
92+
-DCMAKE_CUDA_COMPILER=/usr/local/cuda/bin/nvcc \
93+
-DCMAKE_CUDA_ARCHITECTURES=50 \
94+
-DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda \
95+
-DCMAKE_PREFIX_PATH="${{github.workspace}}/install" \
96+
|| (cat CMakeFiles/CMakeConfigureLog.yaml && exit 1)
97+
98+
- name: Build Fibonacci CUDA Example
99+
working-directory: ${{github.workspace}}/fibonacci-build
100+
shell: bash
101+
run: cmake --build . --target fibonacci_cuda-parsec
102+
103+
- name: Run GPU Tests
104+
working-directory: ${{github.workspace}}/build
105+
shell: bash
106+
run: |
107+
set -e # Exit on first error
108+
109+
echo "=== GPU Device Information ==="
110+
nvidia-smi || echo "nvidia-smi not available"
111+
112+
echo "=== Running CUDA Examples ==="
113+
114+
# Test bspmm-cuda if it exists
115+
if [ -f "examples/bspmm-cuda-parsec" ]; then
116+
echo "Testing bspmm-cuda-parsec..."
117+
mpiexec -n 1 ./examples/bspmm-cuda-parsec
118+
mpiexec -n 1 ./examples/bspmm-cuda-parsec 10 10 10
119+
fi
120+
121+
# Test testing_dpotrf_cuda if it exists
122+
if [ -f "examples/testing_dpotrf_cuda-parsec" ]; then
123+
echo "Testing testing_dpotrf_cuda-parsec..."
124+
mpiexec -n 1 ./examples/testing_dpotrf_cuda-parsec -N 100 -nb 25 -P 1
125+
fi
126+
127+
# Test chain-ttg-cuda if it exists
128+
if [ -f "examples/chain-ttg-cuda-parsec" ]; then
129+
echo "Testing chain-ttg-cuda-parsec..."
130+
mpiexec -n 1 ./examples/chain-ttg-cuda-parsec 100
131+
fi
132+
133+
# Test fibonacci_cuda-parsec example (built from installed TTG)
134+
if [ -f "${{github.workspace}}/fibonacci-build/fibonacci_cuda-parsec" ]; then
135+
echo "Testing fibonacci_cuda-parsec..."
136+
mpiexec -n 1 ${{github.workspace}}/fibonacci-build/fibonacci_cuda-parsec 20
137+
fi
138+
139+
- name: Upload build artifacts
140+
if: failure()
141+
uses: actions/upload-artifact@v4
142+
with:
143+
name: cuda-build-logs
144+
path: |
145+
${{github.workspace}}/build/CMakeFiles/CMakeConfigureLog.yaml
146+
${{github.workspace}}/build/CMakeCache.txt

‎CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -279,6 +279,8 @@ install(FILES
279279
"${PROJECT_SOURCE_DIR}/cmake/modules/AddTTGExecutable.cmake"
280280
"${PROJECT_SOURCE_DIR}/cmake/modules/AddTTGTestExecutable.cmake"
281281
"${PROJECT_SOURCE_DIR}/cmake/modules/FindCXXStdCoroutine.cmake"
282+
"${PROJECT_SOURCE_DIR}/cmake/modules/FindDPCPP.cmake"
283+
"${PROJECT_SOURCE_DIR}/cmake/modules/Findlevel-zero.cmake"
282284
DESTINATION "${CMAKE_INSTALL_CMAKEDIR}/modules"
283285
COMPONENT ttg-config)
284286

‎README.md‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -365,20 +365,20 @@ The host-only part is completely independent of the type of the device programmi
365365

366366
```cpp
367367
struct Fn : public ttg::TTValue<Fn> {
368-
std::unique_ptr<int64_t[]> F; // F[0] = F_n, F[1] = F_{n-1}
368+
std::shared_ptr<int64_t[]> F; // F[0] = F_n, F[1] = F_{n-1}
369369
ttg::Buffer<int64_t> b; // buffer managing host and device memory
370-
Fn() : F(std::make_unique<int64_t[]>(2)), b(F.get(), 2) { F[0] = 1; F[1] = 0; }
370+
Fn() : F(std::make_shared<int64_t[]>(2)), b(F.get(), 2) { F[0] = 1; F[1] = 0; }
371371
Fn(const Fn&) = delete;
372372
Fn(Fn&& other) = default;
373373
Fn& operator=(const Fn& other) = delete;
374374
Fn& operator=(Fn&& other) = default;
375375
template <typename Archive>
376376
void serialize(Archive& ar) {
377-
ttg::ttg_abort();
377+
ar & F[0] & F[1] & b;
378378
}
379379
template <typename Archive>
380380
void serialize(Archive& ar, const unsigned int) {
381-
ttg::ttg_abort();
381+
ar & F[0] & F[1] & b;
382382
}
383383
};
384384

@@ -449,7 +449,7 @@ For optimal performance, the low-level runtime that manages the data motion acro
449449
450450
##### `Buffer`
451451
`Buffer<T>` is a view of a contiguous sequence of objects of type `T` in the host memory that can be automatically moved by the runtime to/from the device memory. Here `Fn::b` is a view of the 2-element sequence pointed to by `Fn::F`; once it's constructed the content of `Fn::F` will be moved to/from the device by the runtime. The subsequent use of `Fn::b` cause the automatic transfers of data to (`device::select(f_n.b)`) and from (`ttg::device::wait(f_n.b)`) the device.
452-
A `Buffer<T>` can be either owning or non-owning. In the example above, the memory is owned by the `unique_ptr`.
452+
A `Buffer<T>` can be either freestanding or with lifetime tied to the lifetime of the host buffer; the latter is used in the example above, indicated by the use of `shared_ptr` to manage the lifetime of the host buffer.
453453
If no pointer is passed to the constructor of `Buffer<T>` the buffer allocates the necessary host-side memory.
454454
In order to guarantee relocatability of buffers, the data managed by a buffer should be located on the heap, i.e., dynamically allocated.
455455

‎cmake/modules/AddCUDAToolkit.cmake‎

Lines changed: 0 additions & 10 deletions
This file was deleted.

‎cmake/ttg-config.cmake.in‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,17 @@ set(TTG_TRACKED_BOOST_VERSION "@TTG_TRACKED_BOOST_VERSION@")
1313
set(TTG_HAS_BOOST @TTG_HAS_BOOST@)
1414
set(TTG_IGNORE_BUNDLED_EXTERNALS @TTG_IGNORE_BUNDLED_EXTERNALS@)
1515

16+
# Device runtime support
17+
set(TTG_HAVE_CUDA @TTG_HAVE_CUDA@)
18+
set(TTG_HAVE_CUDART @TTG_HAVE_CUDART@)
19+
set(TTG_HAVE_HIP @TTG_HAVE_HIP@)
20+
set(TTG_HAVE_HIPBLAS @TTG_HAVE_HIPBLAS@)
21+
set(TTG_HAVE_HIPSOLVER @TTG_HAVE_HIPSOLVER@)
22+
set(TTG_HAVE_LEVEL_ZERO @TTG_HAVE_LEVEL_ZERO@)
23+
set(TTG_HAVE_DPCPP @TTG_HAVE_DPCPP@)
24+
set(TTG_HAVE_DEVICE @TTG_HAVE_DEVICE@)
25+
set(TTG_HAVE_COROUTINE @TTG_HAVE_COROUTINE@)
26+
1627
# make TTG CMake modules discoverable + load AddTTGExecutable by default
1728
list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/modules")
1829
include(AddTTGExecutable)
@@ -43,6 +54,45 @@ if (NOT TARGET std::coroutine AND CXX_COROUTINE_COMPONENT)
4354
find_package(CXXStdCoroutine MODULE QUIET REQUIRED COMPONENTS "${CXX_COROUTINE_COMPONENT}")
4455
endif()
4556

57+
# Load device runtime support if enabled
58+
if (TTG_HAVE_CUDA)
59+
# Enable CUDA language if not already enabled
60+
include(CheckLanguage)
61+
check_language(CUDA)
62+
if(CMAKE_CUDA_COMPILER AND NOT CMAKE_CUDA_COMPILER_LOADED)
63+
enable_language(CUDA)
64+
endif()
65+
endif()
66+
67+
if (TTG_HAVE_CUDART)
68+
find_package(CUDAToolkit REQUIRED)
69+
endif()
70+
71+
if (TTG_HAVE_HIP)
72+
# Enable HIP language if not already enabled
73+
include(CheckLanguage)
74+
check_language(HIP)
75+
if(CMAKE_HIP_COMPILER AND NOT CMAKE_HIP_COMPILER_LOADED)
76+
enable_language(HIP)
77+
endif()
78+
79+
if (TTG_HAVE_HIPBLAS)
80+
find_package(hipblas REQUIRED)
81+
endif()
82+
83+
if (TTG_HAVE_HIPSOLVER)
84+
find_package(hipsolver REQUIRED)
85+
endif()
86+
endif()
87+
88+
if (TTG_HAVE_LEVEL_ZERO)
89+
find_package(level-zero REQUIRED)
90+
if (TTG_HAVE_DPCPP)
91+
find_package(DPCPP REQUIRED)
92+
find_package(MKL)
93+
endif()
94+
endif()
95+
4696
# Include library IMPORT targets
4797
if(NOT TARGET ttg)
4898
include("${CMAKE_CURRENT_LIST_DIR}/ttg-targets.cmake")

‎doc/dox/dev/devsamp/fibonacci/README.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,5 +13,5 @@ After TTG has been installed to `/path/to/ttg`, do this:
1313
- configure: `cmake -S . -B build -DCMAKE_PREFIX_PATH="/path/to/ttg"`
1414
- build:
1515
- CPU version: `cmake --build build --target fibonacci`
16-
- CUDA version (TTG must have been configured with CUDA support): `cmake --build build --target fibonacci_cuda`
17-
- run: `./build/fibonacci N` or `./build/fibonacci_cuda N`
16+
- CUDA version (TTG must have been configured with CUDA support): `cmake --build build --target fibonacci_cuda-parsec`
17+
- run: `./build/fibonacci N` or `./build/fibonacci_cuda-parsec N`

‎doc/dox/dev/devsamp/fibonacci/fibonacci_device.cc‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,24 +14,28 @@
1414
const int64_t F_n_max = 1000;
1515
/// N.B. contains values of F_n and F_{n-1}
1616
struct Fn : public ttg::TTValue<Fn> {
17-
std::unique_ptr<int64_t[]> F; // F[0] = F_n, F[1] = F_{n-1}
17+
std::shared_ptr<int64_t[]> F; // F[0] = F_n, F[1] = F_{n-1}
1818
ttg::Buffer<int64_t> b;
1919

20-
Fn() : F(std::make_unique<int64_t[]>(2)), b(F.get(), 2) { F[0] = 1; F[1] = 0; }
20+
Fn() : F(std::make_shared<int64_t[]>(2)), b(F, 2) { F[0] = 1; F[1] = 0; }
2121

2222
Fn(const Fn&) = delete;
2323
Fn(Fn&& other) = default;
2424
Fn& operator=(const Fn& other) = delete;
2525
Fn& operator=(Fn&& other) = default;
2626

27+
#ifdef TTG_SERIALIZATION_SUPPORTS_MADNESS
2728
template <typename Archive>
2829
void serialize(Archive& ar) {
29-
ttg::ttg_abort();
30+
ar & madness::archive::wrap(F.get(), 2) & b;
3031
}
32+
#endif
33+
#ifdef TTG_SERIALIZATION_SUPPORTS_BOOST
3134
template <typename Archive>
3235
void serialize(Archive& ar, const unsigned int) {
33-
ttg::ttg_abort();
36+
ar & boost::serialization::make_array(F.get(), 2) & b;
3437
}
38+
#endif
3539
};
3640

3741
auto make_ttg_fib_lt(const int64_t F_n_max = 1000) {

0 commit comments

Comments
 (0)