Merge remote-tracking branch 'origin/olupton/reenable-ci' into magkanar/hackathon_openmp_async

Author: iomaganaris

.github/workflows/clang_cmake_format_check.yaml

@@ -0,0 +1,37 @@
+name: clang-cmake-format-check
+
+concurrency:
+  group: ${{ github.workflow }}#${{ github.ref }}
+  cancel-in-progress: true
+
+on:
+    push:
+
+jobs:
+  build:
+    name: clang-cmake-format-check
+    runs-on: ubuntu-20.04
+    steps:
+        - name: Fetch repository
+          uses: actions/checkout@v2
+        - name: Install clang-format 11
+          run: |
+              sudo apt-get update
+              sudo apt-get install clang-format-11 python3-pip libboost-all-dev libopenmpi-dev openmpi-bin
+        - name: Install cmake-format 0.6.13
+          run:  python3 -m pip install cmake-format==0.6.13
+        - name: Configure
+          shell: bash
+          working-directory: ${{runner.workspace}}/CoreNeuron
+          run: |
+              export PATH=/home/runner/.local/bin:$PATH
+              mkdir BUILD && cd BUILD
+              cmake -DCORENRN_CLANG_FORMAT=ON -DCORENRN_CMAKE_FORMAT=ON -DCORENRN_ENABLE_MPI=ON -DCORENRN_ENABLE_OPENMP=OFF -DClangFormat_EXECUTABLE=$(which clang-format-11) -DCMakeFormat_EXECUTABLE=$(which cmake-format) ..
+        - name: Run clang-format
+          shell: bash
+          working-directory: ${{runner.workspace}}/CoreNeuron/BUILD
+          run: make check-clang-format VERBOSE=1
+        - name: Run cmake-format
+          shell: bash
+          working-directory: ${{runner.workspace}}/CoreNeuron/BUILD
+          run: make check-cmake-format

.gitlab-ci.yml

@@ -35,18 +35,31 @@ spack_setup:
     - git diff
     - fi
 
+.spack_intel:
+  variables:
+    SPACK_PACKAGE_COMPILER: intel
 .spack_nvhpc:
   variables:
     SPACK_PACKAGE_COMPILER: nvhpc
 .spack_neuron:
   variables:
     SPACK_PACKAGE: neuron
     SPACK_PACKAGE_REF: ''
-    SPACK_PACKAGE_SPEC: +coreneuron+debug+tests~legacy-unit
+    SPACK_PACKAGE_SPEC: +coreneuron+debug+tests~legacy-unit model_tests=channel-benchmark,olfactory
 .gpu_node:
   variables:
     bb5_constraint: volta
 
+build:nmodl:intel:
+  stage: build_nmodl
+  variables:
+    SPACK_PACKAGE: nmodl
+    SPACK_PACKAGE_REF: ''
+    SPACK_PACKAGE_SPEC: ~legacy-unit
+  extends:
+    - .spack_build
+    - .spack_intel
+
 build:nmodl:gpu:
   stage: build_nmodl
   variables:
@@ -58,21 +71,92 @@ build:nmodl:gpu:
     - .spack_build
     - .spack_nvhpc
 
+build:coreneuron+nmodl:intel:
+  variables:
+    SPACK_PACKAGE: coreneuron
+    SPACK_PACKAGE_SPEC: +nmodl+tests~legacy-unit build_type=Debug
+  extends:
+    - .spack_build
+    - .spack_intel
+  needs: ["build:nmodl:intel"]
+
+build:coreneuron:intel:
+  variables:
+    SPACK_PACKAGE: coreneuron
+    SPACK_PACKAGE_SPEC: +tests~legacy-unit build_type=Debug
+  extends:
+    - .spack_build
+    - .spack_intel
+
 build:coreneuron+nmodl:gpu:
   variables:
     SPACK_PACKAGE: coreneuron
     # +report pulls in a lot of dependencies and the tests fail.
     # See https://github.com/BlueBrain/CoreNeuron/issues/518 re: build_type
-    SPACK_PACKAGE_SPEC: +nmodl+openmp+gpu+tests~legacy-unit~report build_type=RelWithDebInfo
+    SPACK_PACKAGE_SPEC: +nmodl+openmp+gpu+tests~legacy-unit~report~sympy build_type=RelWithDebInfo
   extends:
     - .spack_build
     - .spack_nvhpc
   needs: ["build:nmodl:gpu"]
 
+build:coreneuron+nmodl~openmp:gpu:
+  variables:
+    SPACK_PACKAGE: coreneuron
+    # +report pulls in a lot of dependencies and the tests fail.
+    # See https://github.com/BlueBrain/CoreNeuron/issues/518 re: build_type
+    # Sympy + OpenMP target offload does not currently work with NVHPC
+    SPACK_PACKAGE_SPEC: +nmodl~openmp+gpu+tests~legacy-unit~report+sympy build_type=RelWithDebInfo
+  extends:
+    - .spack_build
+    - .spack_nvhpc
+  needs: ["build:nmodl:gpu"]
+
+build:coreneuron:gpu:
+  variables:
+    SPACK_PACKAGE: coreneuron
+    # +report pulls in a lot of dependencies and the tests fail.
+    # See https://github.com/BlueBrain/CoreNeuron/issues/518 re: build_type
+    SPACK_PACKAGE_SPEC: +gpu+openmp+tests~legacy-unit~report build_type=RelWithDebInfo
+  extends:
+    - .spack_build
+    - .spack_nvhpc
+
+test:coreneuron+nmodl:intel:
+  extends: [.ctest]
+  needs: ["build:coreneuron+nmodl:intel"]
+
+test:coreneuron:intel:
+  extends: [.ctest]
+  needs: ["build:coreneuron:intel"]
+
 test:coreneuron+nmodl:gpu:
   extends: [.ctest, .gpu_node]
   needs: ["build:coreneuron+nmodl:gpu"]
 
+test:coreneuron+nmodl~openmp:gpu:
+  extends: [.ctest, .gpu_node]
+  needs: ["build:coreneuron+nmodl~openmp:gpu"]
+
+test:coreneuron:gpu:
+  extends: [.ctest, .gpu_node]
+  needs: ["build:coreneuron:gpu"]
+
+build:neuron+nmodl:intel:
+  stage: build_neuron
+  extends:
+    - .spack_build
+    - .spack_neuron
+    - .spack_intel
+  needs: ["build:coreneuron+nmodl:intel"]
+
+build:neuron:intel:
+  stage: build_neuron
+  extends:
+    - .spack_build
+    - .spack_neuron
+    - .spack_intel
+  needs: ["build:coreneuron:intel"]
+
 build:neuron+nmodl:gpu:
   stage: build_neuron
   extends:
@@ -85,7 +169,51 @@ build:neuron+nmodl:gpu:
     - !reference [.spack_build, before_script]
   needs: ["build:coreneuron+nmodl:gpu"]
 
+build:neuron+nmodl~openmp:gpu:
+  stage: build_neuron
+  extends:
+    - .spack_build
+    - .spack_neuron
+    - .spack_nvhpc
+  before_script:
+    # Build py-cython and py-numpy with GCC instead of NVHPC.
+    - SPACK_PACKAGE_DEPENDENCIES="${SPACK_PACKAGE_DEPENDENCIES}^py-cython%gcc^py-numpy%gcc"
+    - !reference [.spack_build, before_script]
+  needs: ["build:coreneuron+nmodl~openmp:gpu"]
+
+build:neuron:gpu:
+  stage: build_neuron
+  extends:
+    - .spack_build
+    - .spack_neuron
+    - .spack_nvhpc
+  before_script:
+    # Build py-cython and py-numpy with GCC instead of NVHPC.
+    - SPACK_PACKAGE_DEPENDENCIES="${SPACK_PACKAGE_DEPENDENCIES}^py-cython%gcc^py-numpy%gcc"
+    - !reference [.spack_build, before_script]
+  needs: ["build:coreneuron:gpu"]
+
+test:neuron+nmodl:intel:
+  stage: test_neuron
+  extends: [.ctest]
+  needs: ["build:neuron+nmodl:intel"]
+
+test:neuron:intel:
+  stage: test_neuron
+  extends: [.ctest]
+  needs: ["build:neuron:intel"]
+
 test:neuron+nmodl:gpu:
   stage: test_neuron
   extends: [.ctest, .gpu_node]
   needs: ["build:neuron+nmodl:gpu"]
+
+test:neuron+nmodl~openmp:gpu:
+  stage: test_neuron
+  extends: [.ctest, .gpu_node]
+  needs: ["build:neuron+nmodl~openmp:gpu"]
+
+test:neuron:gpu:
+  stage: test_neuron
+  extends: [.ctest, .gpu_node]
+  needs: ["build:neuron:gpu"]

CMake/OpenAccHelper.cmake

@@ -55,20 +55,26 @@ if(CORENRN_ENABLE_GPU)
   # due to e.g. __CUDACC__ being defined. See https://github.com/BlueBrain/CoreNeuron/issues/607 for
   # more information about this. -gpu=cudaX.Y ensures that OpenACC code is compiled with the same
   # CUDA version as is used for the explicit CUDA code.
-  set(NVHPC_ACC_COMP_FLAGS "-acc -Minfo=accel -gpu=cuda${CORENRN_CUDA_VERSION_SHORT},lineinfo")
-  set(NVHPC_ACC_LINK_FLAGS "-acc -cuda")
+  set(NVHPC_ACC_COMP_FLAGS "-Minfo=accel -gpu=cuda${CORENRN_CUDA_VERSION_SHORT},lineinfo")
+  set(NVHPC_ACC_LINK_FLAGS "-cuda")
   # Make sure that OpenACC code is generated for the same compute capabilities as the explicit CUDA
   # code. Otherwise there may be confusing linker errors. We cannot rely on nvcc and nvc++ using the
   # same default compute capabilities as each other, particularly on GPU-less build machines.
   foreach(compute_capability ${CMAKE_CUDA_ARCHITECTURES})
     string(APPEND NVHPC_ACC_COMP_FLAGS ",cc${compute_capability}")
   endforeach()
-  if(CORENRN_ENABLE_OPENMP AND CORENRN_ENABLE_OPENMP_OFFLOAD)
+  if(CORENRN_ACCELERATOR_OFFLOAD STREQUAL "OpenMP")
     # Enable OpenMP target offload to GPU and if both OpenACC and OpenMP directives are available
     # for a region then prefer OpenMP.
     add_compile_definitions(CORENEURON_PREFER_OPENMP_OFFLOAD)
     string(APPEND NVHPC_ACC_COMP_FLAGS " -mp=gpu -Minfo=mp")
     string(APPEND NVHPC_ACC_LINK_FLAGS " -mp=gpu")
+  elseif(CORENRN_ACCELERATOR_OFFLOAD STREQUAL "OpenACC")
+    # Only enable OpenACC offload for GPU
+    string(APPEND NVHPC_ACC_COMP_FLAGS " -acc")
+    string(APPEND NVHPC_ACC_LINK_FLAGS " -acc")
+  else()
+    message(FATAL_ERROR "${CORENRN_ACCELERATOR_OFFLOAD} not supported with NVHPC compilers")
   endif()
   # avoid PGI adding standard compliant "-A" flags
   set(CMAKE_CXX14_STANDARD_COMPILE_OPTION --c++14)

CMakeLists.txt

@@ -121,6 +121,7 @@ else()
   set(CORENRN_HAVE_NVHPC_COMPILER OFF)
 endif()
 
+set(CORENRN_ACCELERATOR_OFFLOAD "Disabled")
 if(CORENRN_ENABLE_GPU)
   # Older CMake versions than 3.15 have not been tested for GPU/CUDA/OpenACC support after
   # https://github.com/BlueBrain/CoreNeuron/pull/609.
@@ -189,6 +190,18 @@ if(CORENRN_ENABLE_GPU)
   set(CMAKE_CUDA_FLAGS
       "${CMAKE_CUDA_FLAGS} --expt-relaxed-constexpr -Xcudafe --diag_suppress=3057,--diag_suppress=3085"
   )
+
+  if(CORENRN_ENABLE_NMODL)
+    # NMODL supports both OpenACC and OpenMP target offload
+    if(CORENRN_ENABLE_OPENMP AND CORENRN_ENABLE_OPENMP_OFFLOAD)
+      set(CORENRN_ACCELERATOR_OFFLOAD "OpenMP")
+    else()
+      set(CORENRN_ACCELERATOR_OFFLOAD "OpenACC")
+    endif()
+  else()
+    # MOD2C only supports OpenACC offload
+    set(CORENRN_ACCELERATOR_OFFLOAD "OpenACC")
+  endif()
 endif()
 
 # =============================================================================
@@ -530,6 +543,7 @@ message(STATUS "MOD2CPP PATH        | ${CORENRN_MOD2CPP_BINARY}")
 message(STATUS "GPU Support         | ${CORENRN_ENABLE_GPU}")
 if(CORENRN_ENABLE_GPU)
   message(STATUS "  CUDA              | ${CUDAToolkit_LIBRARY_DIR}")
+  message(STATUS "  Offload           | ${CORENRN_ACCELERATOR_OFFLOAD}")
   message(STATUS "  Unified Memory    | ${CORENRN_ENABLE_CUDA_UNIFIED_MEMORY}")
 endif()
 message(STATUS "Auto Timeout        | ${CORENRN_ENABLE_TIMEOUT}")

coreneuron/CMakeLists.txt

@@ -321,13 +321,15 @@ if(EXISTS "${CORENRN_EXTERNAL_BENCHMARK_DATA}")
     COMMENT "Running nrnivmodl-core for channel-benchmark mechanisms")
   list(APPEND all_output_binaries ${output_binaries})
   string(
-    CONCAT
-      benchmark_command
-      "'${CMAKE_BINARY_DIR}/benchmark/${CMAKE_SYSTEM_PROCESSOR}/special-core'"
-      " --datpath '${CORENRN_EXTERNAL_BENCHMARK_DATA}/channel-benchmark-all-440-cells-2-ranks'"
-      " --tstop 1 --gpu &&"
-      "diff out.dat '${CORENRN_EXTERNAL_BENCHMARK_DATA}/channel-benchmark-all-440-cells-2-ranks.gpu.spikes'"
-  )
+    CONCAT benchmark_command
+           "'${CMAKE_BINARY_DIR}/benchmark/${CMAKE_SYSTEM_PROCESSOR}/special-core'"
+           " --datpath '${CORENRN_EXTERNAL_BENCHMARK_DATA}/channel-benchmark-all-440-cells-2-ranks'"
+           " --tstop 1 --mpi")
+  if(CORENRN_ENABLE_GPU)
+    string(APPEND benchmark_command " --gpu")
+  endif()
+  string(APPEND benchmark_command " && diff out.dat "
+         "'${CORENRN_EXTERNAL_BENCHMARK_DATA}/channel-benchmark-all-440-cells-2-ranks.gpu.spikes'")
   add_test(NAME benchmark COMMAND sh -c "${benchmark_command}")
 endif()
 set(modfile_directory "${CORENEURON_PROJECT_SOURCE_DIR}/tests/integration/ring_gap/mod")

coreneuron/apps/main1.cpp

@@ -193,10 +193,11 @@ void nrn_init_and_load_data(int argc,
     // precedence is: set by user, globals.dat, 34.0
     celsius = corenrn_param.celsius;
 
-#if _OPENACC
+#if CORENEURON_ENABLE_GPU
     if (!corenrn_param.gpu && corenrn_param.cell_interleave_permute == 2) {
         fprintf(stderr,
-                "compiled with _OPENACC does not allow the combination of --cell-permute=2 and "
+                "compiled with CORENEURON_ENABLE_GPU does not allow the combination of "
+                "--cell-permute=2 and "
                 "missing --gpu\n");
         exit(1);
     }
@@ -497,7 +498,7 @@ extern "C" void mk_mech_init(int argc, char** argv) {
     }
 #endif
 
-#ifdef _OPENACC
+#ifdef CORENEURON_ENABLE_GPU
     if (corenrn_param.gpu) {
         init_gpu();
     }
@@ -558,8 +559,8 @@ extern "C" int run_solve_core(int argc, char** argv) {
 #endif
     bool compute_gpu = corenrn_param.gpu;
 
-    nrn_pragma_acc(update device(celsius, secondorder, pi) if(compute_gpu))
-    nrn_pragma_omp(target update to(celsius, secondorder, pi) if(compute_gpu))
+    nrn_pragma_acc(update device(celsius, secondorder, pi) if (compute_gpu))
+    nrn_pragma_omp(target update to(celsius, secondorder, pi) if (compute_gpu))
     {
         double v = corenrn_param.voltage;
         double dt = corenrn_param.dt;