Merge pull request #15 from simongdg/main

AndiH · web-flow · commit 7797b80b63fb · 2022-05-25T16:05:58.000+02:00
removed MPI Warmup so that the NVSHMEM_DISABLE_CUDA_VMM=1 and extra i…
diff --git a/08-H_NCCL_NVSHMEM/.master/NVSHMEM/Instructions.ipynb b/08-H_NCCL_NVSHMEM/.master/NVSHMEM/Instructions.ipynb
@@ -64,13 +64,6 @@
         "GPUs listed. This is automatically done for the `sanitize`, `run` and\n",
         "`profile` make targets.\n",
         "\n",
-        "`NVSHMEM_DISABLE_CUDA_VMM=1` is set for the `sanitize`, `run` and\n",
-        "`profile` make targets. This is done to hide warnings and errors\n",
-        "appearing only for NVSHMEM version 2.5.0 to be fixed in the next\n",
-        "release. You might still see cuMemFree, symmetric heap, and\n",
-        "`nvshmem_finalize` errors at the end of the program execution, depending\n",
-        "on the system used to run the program. You may ignore these errors for\n",
-        "now."
       ],
       "id": "e1e833fa-16a5-4510-8e6c-7d6abd44310e"
     }
diff --git a/08-H_NCCL_NVSHMEM/.master/NVSHMEM/Instructions.md b/08-H_NCCL_NVSHMEM/.master/NVSHMEM/Instructions.md
@@ -42,4 +42,3 @@ Study the performance by glimpsing at the profile generated with
 
 The Slurm installation on JUWELS-Booster sets `CUDA_VISIBLE_DEVICES` automatically so that each spawned process only sees the GPU it should use (see [GPU Devices](https://apps.fz-juelich.de/jsc/hps/juwels/booster-overview.html#gpu-devices) in the JUWELS Booster Overview documentation). This is not supported for NVSHMEM. The automatic setting of `CUDA_VISIBLE_DEVICES` can be disabled by setting `CUDA_VISIBLE_DEVICES=0,1,2,3` in the shell that executes srun. With `CUDA_VISIBLE_DEVICES` set all spawned processes can see all GPUs listed. This is automatically done for the `sanitize`, `run` and `profile` make targets.
 
-`NVSHMEM_DISABLE_CUDA_VMM=1` is set for the `sanitize`, `run` and `profile` make targets. This is done to hide warnings and errors appearing only for NVSHMEM version 2.5.0 to be fixed in the next release. You might still see cuMemFree, symmetric heap, and `nvshmem_finalize` errors at the end of the program execution, depending on the system used to run the program. You may ignore these errors for now.
diff --git a/08-H_NCCL_NVSHMEM/.master/NVSHMEM/Makefile b/08-H_NCCL_NVSHMEM/.master/NVSHMEM/Makefile
@@ -36,10 +36,10 @@ clean:
 	rm -f jacobi jacobi.o *.nsys-rep jacobi.*.compute-sanitizer.log
 
 sanitize: jacobi
-	NVSHMEM_DISABLE_CUDA_VMM=$(N_D_C_VMM) CUDA_VISIBLE_DEVICES=$(C_V_D) $(JSC_SUBMIT_CMD) -n $(NP) compute-sanitizer --log-file jacobi.%q{SLURM_PROCID}.compute-sanitizer.log ./jacobi -niter 10
+	CUDA_VISIBLE_DEVICES=$(C_V_D) $(JSC_SUBMIT_CMD) -n $(NP) compute-sanitizer --log-file jacobi.%q{SLURM_PROCID}.compute-sanitizer.log ./jacobi -niter 10
 
 run: jacobi
-	NVSHMEM_DISABLE_CUDA_VMM=$(N_D_C_VMM) CUDA_VISIBLE_DEVICES=$(C_V_D) $(JSC_SUBMIT_CMD) -n $(NP) ./jacobi
+	CUDA_VISIBLE_DEVICES=$(C_V_D) $(JSC_SUBMIT_CMD) -n $(NP) ./jacobi
 
 profile: jacobi
-	NVSHMEM_DISABLE_CUDA_VMM=$(N_D_C_VMM) CUDA_VISIBLE_DEVICES=$(C_V_D) $(JSC_SUBMIT_CMD) -n $(NP) nsys profile --trace=mpi,cuda,nvtx -o jacobi.%q{SLURM_PROCID} ./jacobi -niter 10
+	CUDA_VISIBLE_DEVICES=$(C_V_D) $(JSC_SUBMIT_CMD) -n $(NP) nsys profile --trace=mpi,cuda,nvtx -o jacobi.%q{SLURM_PROCID} ./jacobi -niter 10
diff --git a/08-H_NCCL_NVSHMEM/.master/NVSHMEM/jacobi.cu b/08-H_NCCL_NVSHMEM/.master/NVSHMEM/jacobi.cu
@@ -317,19 +317,6 @@ int main(int argc, char* argv[]) {
     real* l2_norm_h;
     CUDA_RT_CALL(cudaMallocHost(&l2_norm_h, sizeof(real)));
 
-    PUSH_RANGE("MPI_Warmup", 5)
-    for (int i = 0; i < 10; ++i) {
-        const int top = rank > 0 ? rank - 1 : (size - 1);
-        const int bottom = (rank + 1) % size;
-        MPI_CALL(MPI_Sendrecv(a_new + iy_start * nx, nx, MPI_REAL_TYPE, top, 0,
-                              a_new + (iy_end * nx), nx, MPI_REAL_TYPE, bottom, 0, MPI_COMM_WORLD,
-                              MPI_STATUS_IGNORE));
-        MPI_CALL(MPI_Sendrecv(a_new + (iy_end - 1) * nx, nx, MPI_REAL_TYPE, bottom, 0, a_new, nx,
-                              MPI_REAL_TYPE, top, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE));
-        std::swap(a_new, a);
-    }
-    POP_RANGE
-
     CUDA_RT_CALL(cudaDeviceSynchronize());
 
     if (!csv && 0 == rank) {
diff --git a/08-H_NCCL_NVSHMEM/solutions/NVSHMEM/Instructions.ipynb b/08-H_NCCL_NVSHMEM/solutions/NVSHMEM/Instructions.ipynb
@@ -64,13 +64,6 @@
         "GPUs listed. This is automatically done for the `sanitize`, `run` and\n",
         "`profile` make targets.\n",
         "\n",
-        "`NVSHMEM_DISABLE_CUDA_VMM=1` is set for the `sanitize`, `run` and\n",
-        "`profile` make targets. This is done to hide warnings and errors\n",
-        "appearing only for NVSHMEM version 2.5.0 to be fixed in the next\n",
-        "release. You might still see cuMemFree, symmetric heap, and\n",
-        "`nvshmem_finalize` errors at the end of the program execution, depending\n",
-        "on the system used to run the program. You may ignore these errors for\n",
-        "now."
       ],
       "id": "e1e833fa-16a5-4510-8e6c-7d6abd44310e"
     }
diff --git a/08-H_NCCL_NVSHMEM/solutions/NVSHMEM/Instructions.md b/08-H_NCCL_NVSHMEM/solutions/NVSHMEM/Instructions.md
@@ -42,4 +42,3 @@ Study the performance by glimpsing at the profile generated with
 
 The Slurm installation on JUWELS-Booster sets `CUDA_VISIBLE_DEVICES` automatically so that each spawned process only sees the GPU it should use (see [GPU Devices](https://apps.fz-juelich.de/jsc/hps/juwels/booster-overview.html#gpu-devices) in the JUWELS Booster Overview documentation). This is not supported for NVSHMEM. The automatic setting of `CUDA_VISIBLE_DEVICES` can be disabled by setting `CUDA_VISIBLE_DEVICES=0,1,2,3` in the shell that executes srun. With `CUDA_VISIBLE_DEVICES` set all spawned processes can see all GPUs listed. This is automatically done for the `sanitize`, `run` and `profile` make targets.
 
-`NVSHMEM_DISABLE_CUDA_VMM=1` is set for the `sanitize`, `run` and `profile` make targets. This is done to hide warnings and errors appearing only for NVSHMEM version 2.5.0 to be fixed in the next release. You might still see cuMemFree, symmetric heap, and `nvshmem_finalize` errors at the end of the program execution, depending on the system used to run the program. You may ignore these errors for now.
diff --git a/08-H_NCCL_NVSHMEM/solutions/NVSHMEM/Makefile b/08-H_NCCL_NVSHMEM/solutions/NVSHMEM/Makefile
@@ -36,10 +36,10 @@ clean:
 	rm -f jacobi jacobi.o *.nsys-rep jacobi.*.compute-sanitizer.log
 
 sanitize: jacobi
-	NVSHMEM_DISABLE_CUDA_VMM=$(N_D_C_VMM) CUDA_VISIBLE_DEVICES=$(C_V_D) $(JSC_SUBMIT_CMD) -n $(NP) compute-sanitizer --log-file jacobi.%q{SLURM_PROCID}.compute-sanitizer.log ./jacobi -niter 10
+	CUDA_VISIBLE_DEVICES=$(C_V_D) $(JSC_SUBMIT_CMD) -n $(NP) compute-sanitizer --log-file jacobi.%q{SLURM_PROCID}.compute-sanitizer.log ./jacobi -niter 10
 
 run: jacobi
-	NVSHMEM_DISABLE_CUDA_VMM=$(N_D_C_VMM) CUDA_VISIBLE_DEVICES=$(C_V_D) $(JSC_SUBMIT_CMD) -n $(NP) ./jacobi
+	CUDA_VISIBLE_DEVICES=$(C_V_D) $(JSC_SUBMIT_CMD) -n $(NP) ./jacobi
 
 profile: jacobi
-	NVSHMEM_DISABLE_CUDA_VMM=$(N_D_C_VMM) CUDA_VISIBLE_DEVICES=$(C_V_D) $(JSC_SUBMIT_CMD) -n $(NP) nsys profile --trace=mpi,cuda,nvtx -o jacobi.%q{SLURM_PROCID} ./jacobi -niter 10
+	CUDA_VISIBLE_DEVICES=$(C_V_D) $(JSC_SUBMIT_CMD) -n $(NP) nsys profile --trace=mpi,cuda,nvtx -o jacobi.%q{SLURM_PROCID} ./jacobi -niter 10
diff --git a/08-H_NCCL_NVSHMEM/solutions/NVSHMEM/jacobi.cu b/08-H_NCCL_NVSHMEM/solutions/NVSHMEM/jacobi.cu
@@ -304,19 +304,6 @@ int main(int argc, char* argv[]) {
     real* l2_norm_h;
     CUDA_RT_CALL(cudaMallocHost(&l2_norm_h, sizeof(real)));
 
-    PUSH_RANGE("MPI_Warmup", 5)
-    for (int i = 0; i < 10; ++i) {
-        const int top = rank > 0 ? rank - 1 : (size - 1);
-        const int bottom = (rank + 1) % size;
-        MPI_CALL(MPI_Sendrecv(a_new + iy_start * nx, nx, MPI_REAL_TYPE, top, 0,
-                              a_new + (iy_end * nx), nx, MPI_REAL_TYPE, bottom, 0, MPI_COMM_WORLD,
-                              MPI_STATUS_IGNORE));
-        MPI_CALL(MPI_Sendrecv(a_new + (iy_end - 1) * nx, nx, MPI_REAL_TYPE, bottom, 0, a_new, nx,
-                              MPI_REAL_TYPE, top, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE));
-        std::swap(a_new, a);
-    }
-    POP_RANGE
-
     CUDA_RT_CALL(cudaDeviceSynchronize());
 
     if (!csv && 0 == rank) {
diff --git a/08-H_NCCL_NVSHMEM/tasks/NVSHMEM/Instructions.ipynb b/08-H_NCCL_NVSHMEM/tasks/NVSHMEM/Instructions.ipynb
@@ -64,13 +64,6 @@
         "GPUs listed. This is automatically done for the `sanitize`, `run` and\n",
         "`profile` make targets.\n",
         "\n",
-        "`NVSHMEM_DISABLE_CUDA_VMM=1` is set for the `sanitize`, `run` and\n",
-        "`profile` make targets. This is done to hide warnings and errors\n",
-        "appearing only for NVSHMEM version 2.5.0 to be fixed in the next\n",
-        "release. You might still see cuMemFree, symmetric heap, and\n",
-        "`nvshmem_finalize` errors at the end of the program execution, depending\n",
-        "on the system used to run the program. You may ignore these errors for\n",
-        "now."
       ],
       "id": "e1e833fa-16a5-4510-8e6c-7d6abd44310e"
     }
diff --git a/08-H_NCCL_NVSHMEM/tasks/NVSHMEM/Instructions.md b/08-H_NCCL_NVSHMEM/tasks/NVSHMEM/Instructions.md
@@ -42,4 +42,3 @@ Study the performance by glimpsing at the profile generated with
 
 The Slurm installation on JUWELS-Booster sets `CUDA_VISIBLE_DEVICES` automatically so that each spawned process only sees the GPU it should use (see [GPU Devices](https://apps.fz-juelich.de/jsc/hps/juwels/booster-overview.html#gpu-devices) in the JUWELS Booster Overview documentation). This is not supported for NVSHMEM. The automatic setting of `CUDA_VISIBLE_DEVICES` can be disabled by setting `CUDA_VISIBLE_DEVICES=0,1,2,3` in the shell that executes srun. With `CUDA_VISIBLE_DEVICES` set all spawned processes can see all GPUs listed. This is automatically done for the `sanitize`, `run` and `profile` make targets.
 
-`NVSHMEM_DISABLE_CUDA_VMM=1` is set for the `sanitize`, `run` and `profile` make targets. This is done to hide warnings and errors appearing only for NVSHMEM version 2.5.0 to be fixed in the next release. You might still see cuMemFree, symmetric heap, and `nvshmem_finalize` errors at the end of the program execution, depending on the system used to run the program. You may ignore these errors for now.
diff --git a/08-H_NCCL_NVSHMEM/tasks/NVSHMEM/Makefile b/08-H_NCCL_NVSHMEM/tasks/NVSHMEM/Makefile
@@ -36,10 +36,10 @@ clean:
 	rm -f jacobi jacobi.o *.nsys-rep jacobi.*.compute-sanitizer.log
 
 sanitize: jacobi
-	NVSHMEM_DISABLE_CUDA_VMM=$(N_D_C_VMM) CUDA_VISIBLE_DEVICES=$(C_V_D) $(JSC_SUBMIT_CMD) -n $(NP) compute-sanitizer --log-file jacobi.%q{SLURM_PROCID}.compute-sanitizer.log ./jacobi -niter 10
+	CUDA_VISIBLE_DEVICES=$(C_V_D) $(JSC_SUBMIT_CMD) -n $(NP) compute-sanitizer --log-file jacobi.%q{SLURM_PROCID}.compute-sanitizer.log ./jacobi -niter 10
 
 run: jacobi
-	NVSHMEM_DISABLE_CUDA_VMM=$(N_D_C_VMM) CUDA_VISIBLE_DEVICES=$(C_V_D) $(JSC_SUBMIT_CMD) -n $(NP) ./jacobi
+	CUDA_VISIBLE_DEVICES=$(C_V_D) $(JSC_SUBMIT_CMD) -n $(NP) ./jacobi
 
 profile: jacobi
-	NVSHMEM_DISABLE_CUDA_VMM=$(N_D_C_VMM) CUDA_VISIBLE_DEVICES=$(C_V_D) $(JSC_SUBMIT_CMD) -n $(NP) nsys profile --trace=mpi,cuda,nvtx -o jacobi.%q{SLURM_PROCID} ./jacobi -niter 10
+	CUDA_VISIBLE_DEVICES=$(C_V_D) $(JSC_SUBMIT_CMD) -n $(NP) nsys profile --trace=mpi,cuda,nvtx -o jacobi.%q{SLURM_PROCID} ./jacobi -niter 10
diff --git a/08-H_NCCL_NVSHMEM/tasks/NVSHMEM/jacobi.cu b/08-H_NCCL_NVSHMEM/tasks/NVSHMEM/jacobi.cu
@@ -295,19 +295,6 @@ int main(int argc, char* argv[]) {
     real* l2_norm_h;
     CUDA_RT_CALL(cudaMallocHost(&l2_norm_h, sizeof(real)));
 
-    PUSH_RANGE("MPI_Warmup", 5)
-    for (int i = 0; i < 10; ++i) {
-        const int top = rank > 0 ? rank - 1 : (size - 1);
-        const int bottom = (rank + 1) % size;
-        MPI_CALL(MPI_Sendrecv(a_new + iy_start * nx, nx, MPI_REAL_TYPE, top, 0,
-                              a_new + (iy_end * nx), nx, MPI_REAL_TYPE, bottom, 0, MPI_COMM_WORLD,
-                              MPI_STATUS_IGNORE));
-        MPI_CALL(MPI_Sendrecv(a_new + (iy_end - 1) * nx, nx, MPI_REAL_TYPE, bottom, 0, a_new, nx,
-                              MPI_REAL_TYPE, top, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE));
-        std::swap(a_new, a);
-    }
-    POP_RANGE
-
     CUDA_RT_CALL(cudaDeviceSynchronize());
 
     if (!csv && 0 == rank) {

Original file line number	Diff line number	Diff line change
`@@ -42,4 +42,3 @@ Study the performance by glimpsing at the profile generated with`
`42`	`42`
`43`	`43`	The Slurm installation on JUWELS-Booster sets `CUDA_VISIBLE_DEVICES` automatically so that each spawned process only sees the GPU it should use (see [GPU Devices](https://apps.fz-juelich.de/jsc/hps/juwels/booster-overview.html#gpu-devices) in the JUWELS Booster Overview documentation). This is not supported for NVSHMEM. The automatic setting of `CUDA_VISIBLE_DEVICES` can be disabled by setting `CUDA_VISIBLE_DEVICES=0,1,2,3` in the shell that executes srun. With `CUDA_VISIBLE_DEVICES` set all spawned processes can see all GPUs listed. This is automatically done for the `sanitize`, `run` and `profile` make targets.
`44`	`44`
`45`		-`NVSHMEM_DISABLE_CUDA_VMM=1` is set for the `sanitize`, `run` and `profile` make targets. This is done to hide warnings and errors appearing only for NVSHMEM version 2.5.0 to be fixed in the next release. You might still see cuMemFree, symmetric heap, and `nvshmem_finalize` errors at the end of the program execution, depending on the system used to run the program. You may ignore these errors for now.