add base cscs-ci ext (#138)

* add base cscs-ci ext

Author: simonpintarelli

ci/baseimage.cuda.Dockerfile

@@ -0,0 +1,85 @@
+FROM ubuntu:22.04 as builder
+
+ARG CUDA_ARCH=60
+
+ENV DEBIAN_FRONTEND noninteractive
+
+ENV FORCE_UNSAFE_CONFIGURE 1
+
+ENV PATH="/spack/bin:${PATH}"
+
+ENV MPICH_VERSION=3.4.3
+
+ENV CMAKE_VERSION=3.27.9
+
+RUN apt-get -y update
+
+RUN apt-get install -y apt-utils
+
+# install basic tools
+RUN apt-get install -y --no-install-recommends gcc g++ gfortran clang libomp-14-dev git make unzip file \
+  vim wget pkg-config python3-pip python3-dev cython3 python3-pythran curl tcl m4 cpio automake meson \
+  xz-utils patch patchelf apt-transport-https ca-certificates gnupg software-properties-common perl tar bzip2 \
+  liblzma-dev libbz2-dev
+
+# install CMake
+RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz -O cmake.tar.gz && \
+    tar zxvf cmake.tar.gz --strip-components=1 -C /usr
+
+# get latest version of spack
+RUN git clone -b v0.21.0 https://github.com/spack/spack.git
+
+# set the location of packages built by spack
+RUN spack config add config:install_tree:root:/opt/local
+# set cuda_arch for all packages
+RUN spack config add packages:all:variants:cuda_arch=${CUDA_ARCH}
+
+# add local repo for cosma and tiled-mm
+COPY ./spack /cosma-repo
+RUN spack repo add /cosma-repo
+
+# find all external packages
+RUN spack external find --all --exclude python
+
+# find compilers
+RUN spack compiler find
+
+# install yq (utility to manipulate the yaml files)
+RUN wget -qO /usr/local/bin/yq https://github.com/mikefarah/yq/releases/latest/download/yq_linux_386 && chmod a+x /usr/local/bin/yq
+
+# change the fortran compilers: for gcc the gfortran is already properly set and the change has no effect; add it for clang
+RUN yq -i '.compilers[0].compiler.paths.f77 = "/usr/bin/gfortran"' /root/.spack/linux/compilers.yaml && \
+    yq -i '.compilers[0].compiler.paths.fc = "/usr/bin/gfortran"' /root/.spack/linux/compilers.yaml  && \
+    yq -i '.compilers[1].compiler.paths.f77 = "/usr/bin/gfortran"' /root/.spack/linux/compilers.yaml && \
+    yq -i '.compilers[1].compiler.paths.fc = "/usr/bin/gfortran"' /root/.spack/linux/compilers.yaml
+
+# install MPICH
+RUN spack install mpich@${MPICH_VERSION} %gcc
+
+# for the MPI hook
+RUN echo $(spack find --format='{prefix.lib}' mpich) > /etc/ld.so.conf.d/mpich.conf
+RUN ldconfig
+
+# create environments for several configurations and install dependencies
+RUN spack env create -d /cosma-env-cuda && \
+    spack -e /cosma-env-cuda add "cosma@master %gcc +cuda +tests +scalapack +shared ^mpich" && \
+    spack -e /cosma-env-cuda add "tiled-mm@master" && \
+    spack -e /cosma-env-cuda develop -p /src cosma@master && \
+    spack -e /cosma-env-cuda install --only=dependencies --fail-fast
+
+RUN spack env create -d /cosma-env-cuda-gpu-direct && \
+    spack -e /cosma-env-cuda-gpu-direct add "cosma@master %gcc +cuda +tests +scalapack +shared +gpu_direct ^mpich " && \
+    spack -e /cosma-env-cuda-gpu-direct add "tiled-mm@master" && \
+    spack -e /cosma-env-cuda-gpu-direct develop -p /src cosma@master && \
+    spack -e /cosma-env-cuda-gpu-direct install --only=dependencies --fail-fast
+
+RUN spack env create -d /cosma-env-cuda-nccl && \
+    spack -e /cosma-env-cuda-nccl add "cosma@master %gcc +cuda +tests +scalapack +shared +nccl ^mpich " && \
+    spack -e /cosma-env-cuda-nccl add "tiled-mm@master" && \
+    spack -e /cosma-env-cuda-nccl develop -p /src cosma@master && \
+    spack -e /cosma-env-cuda-nccl install --only=dependencies --fail-fast
+
+RUN spack env create -d /cosma-env-cpu && \
+    spack -e /cosma-env-cpu add "cosma@master %gcc ~cuda +tests +scalapack +shared ^mpich " && \
+    spack -e /cosma-env-cpu develop -p /src cosma@master && \
+    spack -e /cosma-env-cpu install --only=dependencies --fail-fast

ci/build.Dockerfile

@@ -0,0 +1,17 @@
+ARG BASE_IMAGE
+FROM $BASE_IMAGE
+
+ARG ENVPATH
+
+# copy source files of the pull request into container
+COPY . /src
+
+# build SIRIUS
+RUN spack -e $ENVPATH install
+
+# # show the spack's spec
+RUN spack -e $ENVPATH find -lcdv
+
+# we need a fixed name for the build directory
+# here is a hacky workaround to link ./spack-build-{hash} to ./spack-build
+RUN cd /src && ln -s $(spack -e $ENVPATH location -b cosma) spack-build

ci/cpu.yml

@@ -0,0 +1,119 @@
+include:
+  - remote: 'https://gitlab.com/cscs-ci/recipes/-/raw/master/templates/v2/.ci-ext.yml'
+
+stages:
+  - baseimage
+  - build
+  - test
+
+build base image:
+  extends: .container-builder-dynamic-name
+  stage: baseimage
+  timeout: 2h
+  variables:
+    DOCKERFILE: ci/baseimage.cuda.Dockerfile
+    WATCH_FILECHANGES: ci/baseimage.cuda.Dockerfile
+    PERSIST_IMAGE_NAME: $CSCS_REGISTRY_PATH/base/cosma-ci
+
+build cosma:
+  extends: .container-builder
+  needs: ["build base image"]
+  stage: build
+  variables:
+    DOCKERFILE: ci/build.Dockerfile
+    PERSIST_IMAGE_NAME: $CSCS_REGISTRY_PATH/cosma/cosma-ci:$CI_COMMIT_SHA
+    ENVPATH: "/cosma-env-cuda"
+    DOCKER_BUILD_ARGS: '["BASE_IMAGE=${BASE_IMAGE}", "ENVPATH=$ENVPATH"]'
+
+build cosma cpu:
+  extends: .container-builder
+  needs: ["build base image"]
+  stage: build
+  variables:
+    DOCKERFILE: ci/build.Dockerfile
+    PERSIST_IMAGE_NAME: discard
+    ENVPATH: "/cosma-env-cpu"
+    DOCKER_BUILD_ARGS: '["BASE_IMAGE=${BASE_IMAGE}", "ENVPATH=$ENVPATH"]'
+
+build cosma cuda gpu_direct:
+  extends: .container-builder
+  needs: ["build base image"]
+  stage: build
+  variables:
+    DOCKERFILE: ci/build.Dockerfile
+    PERSIST_IMAGE_NAME: discard
+    ENVPATH: "/cosma-env-cuda-gpu-direct"
+    DOCKER_BUILD_ARGS: '["BASE_IMAGE=${BASE_IMAGE}", "ENVPATH=$ENVPATH"]'
+
+build cosma cuda nccl:
+  extends: .container-builder
+  needs: ["build base image"]
+  stage: build
+  variables:
+    DOCKERFILE: ci/build.Dockerfile
+    PERSIST_IMAGE_NAME: discard
+    ENVPATH: "/cosma-env-cuda-nccl"
+    DOCKER_BUILD_ARGS: '["BASE_IMAGE=${BASE_IMAGE}", "ENVPATH=$ENVPATH"]'
+
+.run_tests:
+  extends: .container-runner-daint-gpu
+  needs: ["build cosma"]
+  stage: test
+  image: $CSCS_REGISTRY_PATH/cosma/cosma-ci:$CI_COMMIT_SHA
+  variables:
+    CRAY_CUDA_MPS: 1
+    GIT_STRATEGY: none
+    MPICH_MAX_THREAD_SAFETY: multiple
+    CSCS_REGISTRY_LOGIN: 'YES'
+    PULL_IMAGE: 'YES'
+    SLURM_HINT: nomultithread
+    SLURM_UNBUFFEREDIO: ''
+    SLURM_WAIT: 0
+    COSMA_GPU_MAX_TILE_K: 100
+    COSMA_GPU_MAX_TILE_M: 100
+    COSMA_GPU_MAX_TILE_N: 100
+
+mapper:
+  extends: .run_tests
+  stage: test
+  script: /cosma-env-cuda/.spack-env/view/bin/test.mapper
+  variables:
+    SLURM_JOB_NUM_NODES: 1
+    SLURM_NTASKS: 1
+    USE_MPI: 'YES'
+
+pdgemm:
+  extends: .run_tests
+  stage: test
+  script: /cosma-env-cuda/.spack-env/view/bin/test.pdgemm
+  variables:
+    SLURM_JOB_NUM_NODES: 2
+    SLURM_NTASKS: 16
+    USE_MPI: 'YES'
+
+multiply:
+  extends: .run_tests
+  stage: test
+  script: /cosma-env-cuda/.spack-env/view/bin/test.multiply
+  variables:
+    SLURM_JOB_NUM_NODES: 2
+    SLURM_NTASKS: 16
+    USE_MPI: 'YES'
+
+scalar_matmul:
+  extends: .run_tests
+  stage: test
+  script: /cosma-env-cuda/.spack-env/view/bin/test.scalar_matmul
+  variables:
+    SLURM_JOB_NUM_NODES: 1
+    SLURM_NTASKS: 8
+    USE_MPI: 'YES'
+
+multiply_using_layout:
+  extends: .run_tests
+  stage: test
+  script: /cosma-env-cuda/.spack-env/view/bin/test.multiply_using_layout
+  variables:
+    SLURM_JOB_NUM_NODES: 1
+    SLURM_NTASKS: 4
+    USE_MPI: 'YES'