Merge branch '2023.06-software.eessi.io' of github-trz:EESSI/software-layer into 2023.06-PyTorch-2.1.2-with-CUDA-foss-2023a-final

Author: truib

.github/workflows/test-software.eessi.io.yml

@@ -8,33 +8,60 @@ on:
 permissions:
   contents: read # to fetch code (actions/checkout)
 env:
+  EESSI_VERSION: '2023.06'
   EESSI_ACCELERATOR_TARGETS: |
     x86_64/amd/zen2:
       - nvidia/cc80
     x86_64/amd/zen3:
       - nvidia/cc80
 jobs:
   check_missing:
-    runs-on: ubuntu-24.04
     strategy:
       fail-fast: false
       matrix:
-        EESSI_VERSION:
-        - 2023.06
-        EESSI_SOFTWARE_SUBDIR_OVERRIDE:
-        - aarch64/generic
-        - aarch64/neoverse_n1
-        - aarch64/neoverse_v1
-        - x86_64/amd/zen2
-        - x86_64/amd/zen3
-        - x86_64/amd/zen4
-        - x86_64/intel/haswell
-        - x86_64/intel/sapphirerapids
-        - x86_64/intel/skylake_avx512
-        - x86_64/generic
+        include:
+          # Arm CPU targets
+          - runs_on: ubuntu-24.04-arm
+            EESSI_SOFTWARE_SUBDIR_OVERRIDE: aarch64/generic
+          - runs_on: ubuntu-24.04-arm
+            EESSI_SOFTWARE_SUBDIR_OVERRIDE: aarch64/neoverse_n1
+          - runs_on: ubuntu-24.04-arm
+            EESSI_SOFTWARE_SUBDIR_OVERRIDE: aarch64/neoverse_v1
+          - runs_on: ubuntu-24.04-arm
+            EESSI_SOFTWARE_SUBDIR_OVERRIDE: aarch64/nvidia/grace
+          # x86_64 CPU targets
+          - runs_on: ubuntu-24.04
+            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/amd/zen2
+          - runs_on: ubuntu-24.04
+            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/amd/zen3
+          - runs_on: ubuntu-24.04
+            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/amd/zen4
+          - runs_on: ubuntu-24.04
+            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/haswell
+          - runs_on: ubuntu-24.04
+            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/sapphirerapids
+          - runs_on: ubuntu-24.04
+            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/skylake_avx512
+          - runs_on: ubuntu-24.04
+            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/icelake
+          - runs_on: ubuntu-24.04
+            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/intel/cascadelake
+          - runs_on: ubuntu-24.04
+            EESSI_SOFTWARE_SUBDIR_OVERRIDE: x86_64/generic
+    runs-on: ${{ matrix.runs_on }}
     steps:
         - name: Check out software-layer repository
           uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+          with:
+              fetch-depth: 0  # Fetch all history for all branches and tags
+
+        - name: Show host system info
+          run: |
+            echo "/proc/cpuinfo:"
+            cat /proc/cpuinfo
+            echo
+            echo "lscpu:"
+            lscpu
 
         - name: Mount EESSI CernVM-FS pilot repository
           uses: cvmfs-contrib/github-action-cvmfs@55899ca74cf78ab874bdf47f5a804e47c198743c # v4.0
@@ -46,21 +73,23 @@ jobs:
         - name: Test check_missing_installations.sh script
           run: |
               export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
-              source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
+              source /cvmfs/software.eessi.io/versions/${EESSI_VERSION}/init/bash
               # set $EESSI_CPU_FAMILY to the CPU architecture that corresponds to $EESSI_SOFTWARE_SUBDIR_OVERRIDE (part before the first slash),
               # to prevent issues with checks in the Easybuild configuration that use this variable
               export EESSI_CPU_FAMILY=${EESSI_SOFTWARE_SUBDIR_OVERRIDE%%/*}
-              module load EasyBuild
-              which eb
-              eb --version
-              export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}
+              export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${EESSI_VERSION}
               export EESSI_OS_TYPE=linux
               env | grep ^EESSI | sort
 
               # first check the CPU-only builds for this CPU target
-              echo "just run check_missing_installations.sh (should use easystacks/software.eessi.io/${{matrix.EESSI_VERSION}}/eessi-${{matrix.EESSI_VERSION}}-*.yml with latest EasyBuild release)"
-              for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} .github/workflows/scripts/only_latest_easystacks.sh); do
-                  echo "check missing installations for ${easystack_file}..."
+              echo "first run check_missing_installations.sh for CPU-only builds"
+              for easystack_file in $(EESSI_VERSION=${EESSI_VERSION} .github/workflows/scripts/only_latest_easystacks.sh); do
+                  eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*.yml/\1/g')
+                  echo "check missing installations for ${easystack_file} with EasyBuild ${eb_version}..."
+                  module purge
+                  module load EasyBuild/${eb_version}
+                  which eb
+                  eb --version
                   ./check_missing_installations.sh ${easystack_file}
                   ec=$?
                   if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi
@@ -74,8 +103,13 @@ jobs:
                   for accel in ${accelerators}; do
                       module use ${EESSI_SOFTWARE_PATH}/accel/${accel}/modules/all
                       echo "checking missing installations for accelerator ${accel} using modulepath: ${MODULEPATH}"
-                      for easystack_file in $(EESSI_VERSION=${{matrix.EESSI_VERSION}} ACCEL_EASYSTACKS=1 .github/workflows/scripts/only_latest_easystacks.sh); do
-                          echo "check missing installations for ${easystack_file}..."
+                      for easystack_file in $(EESSI_VERSION=${EESSI_VERSION} ACCEL_EASYSTACKS=1 .github/workflows/scripts/only_latest_easystacks.sh); do
+                          eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*.yml/\1/g')
+                          echo "check missing installations for ${easystack_file} with EasyBuild ${eb_version}..."
+                          module purge
+                          module load EasyBuild/${eb_version}
+                          which eb
+                          eb --version
                           ./check_missing_installations.sh ${easystack_file}
                           ec=$?
                           if [[ ${ec} -ne 0 ]]; then echo "missing installations found for ${easystack_file}!" >&2; exit ${ec}; fi
@@ -87,14 +121,14 @@ jobs:
         - name: Test check_missing_installations.sh with missing package (GCC/8.3.0)
           run: |
               export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
-              source /cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}/init/bash
+              source /cvmfs/software.eessi.io/versions/${EESSI_VERSION}/init/bash
               # set $EESSI_CPU_FAMILY to the CPU architecture that corresponds to $EESSI_SOFTWARE_SUBDIR_OVERRIDE (part before the first slash),
               # to prevent issues with checks in the Easybuild configuration that use this variable
               export EESSI_CPU_FAMILY=${EESSI_SOFTWARE_SUBDIR_OVERRIDE%%/*}
               module load EasyBuild
               which eb
               eb --version
-              export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${{matrix.EESSI_VERSION}}
+              export EESSI_PREFIX=/cvmfs/software.eessi.io/versions/${EESSI_VERSION}
               export EESSI_OS_TYPE=linux
               env | grep ^EESSI | sort
               # create dummy easystack file with a single entry (something that is not installed in EESSI)
@@ -115,3 +149,28 @@ jobs:
                 echo "captured missing package; test PASSED"
                 exit 0
               fi
+
+        - name: Check that EasyBuild hook is up to date
+          if: ${{ github.event_name == 'pull_request' }}
+          run: |
+              FILE="eb_hooks.py"
+              TEMP_FILE="$(mktemp)"
+
+              # Fetch base branch
+              git fetch origin ${{ github.base_ref }}
+
+              # Check if the hooks has changed in the PR
+              if git diff --name-only origin/${{ github.base_ref }}...HEAD | grep -q "^$FILE$"; then
+                echo "Hooks changed in PR. Using PR version."
+                cp "$FILE" "$TEMP_FILE"
+              else
+                echo "File not changed in PR. Using default branch version."
+                git show origin/${{ github.base_ref }}:$FILE > "$TEMP_FILE"
+              fi
+
+              # Compare the hooks to what is shipped in the repository
+              # (it is overkill, but harmless, to do this for every architecture)
+              export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
+              source /cvmfs/software.eessi.io/versions/${EESSI_VERSION}/init/bash
+              module load EESSI-extend
+              diff "$TEMP_FILE" "$EASYBUILD_HOOKS"

.github/workflows/test_compare_stacks.yml

@@ -24,12 +24,15 @@ jobs:
         COMPARISON_ARCH:
         - aarch64/neoverse_n1
         - aarch64/neoverse_v1
+        - aarch64/nvidia/grace
         - x86_64/amd/zen2
         - x86_64/amd/zen3
         - x86_64/amd/zen4
         - x86_64/intel/haswell
         - x86_64/intel/skylake_avx512
         - x86_64/intel/sapphirerapids
+        - x86_64/intel/icelake
+        - x86_64/intel/cascadelake
     steps:
         - name: Check out software-layer repository
           uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1

.github/workflows/tests_archdetect.yml

@@ -23,6 +23,8 @@ jobs:
           - aarch64/neoverse_n1/Azure-Ubuntu20-Altra
           - aarch64/neoverse_n1/AWS-awslinux-graviton2
           - aarch64/neoverse_v1/AWS-awslinux-graviton3
+          - aarch64/nvidia/grace/Jureca-Rocky95
+          - aarch64/google/axion/GCP-axion
           # commented out since these targets are currently not supported in software.eessi.io repo
           # (and some tests assume that the corresponding subdirectory in software layer is there)
           # - ppc64le/power9le/unknown-power9le

.github/workflows/tests_link_nvidia_host_libraries.yml

@@ -120,6 +120,8 @@ jobs:
         # List dir with libraries
         echo "Showing content of /tmp/nvidia_libs"
         echo "$(ls -l /tmp/nvidia_libs)"
+        echo "Showing content of /tmp/nvidia_libs_duplicate"
+        echo "$(ls -l /tmp/nvidia_libs_duplicate)"
         echo "Showing content of /opt/eessi/nvidia/x86_64/host"
         echo "$(ls -l /opt/eessi/nvidia/x86_64/host)"
 
@@ -152,9 +154,9 @@ jobs:
             exit 1
           fi
           
-          # Verify it points to our mock library in /tmp/nvidia_libs
-          if [[ "$target" != "/tmp/nvidia_libs/$lib"* && "$target" != *"/tmp/nvidia_libs/"* ]]; then
-            echo "Error: Symlink $lib_path points to $target, which is not in our mock directory"
+          # Verify it points to our mock library in /tmp/nvidia_libs or /tmp/nvidia_libs_duplicate
+          if [[ "$target" != "/tmp/nvidia_libs/$lib"* && "$target" != "/tmp/nvidia_libs_duplicate/$lib"* ]]; then
+            echo "Error: Symlink $lib_path points to $target, which is not in our mock directories"
             exit 1
           fi
           
@@ -194,3 +196,4 @@ jobs:
         echo "$output" | grep "have already been linked" || { echo "Missing 'already linked' message"; exit 1; }
         
         echo "Second normal run test passed"
+        

EESSI-install-software.sh

@@ -53,6 +53,23 @@ function copy_build_log() {
     fi
 }
 
+function safe_module_use {
+    # add a given non-empty directory to $MODULEPATH if and only if it is not yet in
+    directory=${1}
+
+    if [[ -z ${directory+x} ]]; then
+        echo "safe_module_use: given directory unset or empty; not adding it to \$MODULEPATH (${MODULEPATH})"
+        return
+    fi
+    if [[ ":${MODULEPATH}:" == *":${directory}:"* ]]; then
+        echo "safe_module_use: directory '${directory}' is already in \$MODULEPATH (${MODULEPATH}); not adding it again"
+        return
+    else
+        echo "safe_module_use: directory '${directory}' is not yet in \$MODULEPATH (${MODULEPATH}); adding it"
+        module use ${directory}
+    fi
+}
+
 POSITIONAL_ARGS=()
 
 while [[ $# -gt 0 ]]; do
@@ -225,8 +242,8 @@ fi
 # order is important: these are needed to install a full CUDA SDK in host_injections
 # for now, this just reinstalls all scripts. Note the most elegant, but works
 
-# Only run install_scripts.sh if not dev.eessi.io for security
-if [[ "${EESSI_CVMFS_REPO}" != /cvmfs/dev.eessi.io ]]; then
+# Only run install_scripts.sh if not in dev.eessi.io for security
+if [[ -z ${EESSI_DEV_PROJECT} ]]; then
     ${TOPDIR}/install_scripts.sh --prefix ${EESSI_PREFIX}
 fi
 
@@ -251,7 +268,20 @@ export EESSI_CVMFS_INSTALL=1
 # NOTE 3, we have to set a default for EASYBUILD_INSTALLPATH here in cases the
 #   EESSI-extend module itself needs to be installed.
 export EASYBUILD_INSTALLPATH=${EESSI_PREFIX}/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}
-source load_eessi_extend_module.sh ${EESSI_VERSION}
+
+# If in dev.eessi.io, allow building on top of software.eessi.io via EESSI-extend
+if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then
+    # We keep track of the old install path for settings paths to SitePackage.lua and .lmodrc later
+    EASYBUILD_INSTALLPATH_STANDARD=${EASYBUILD_INSTALLPATH}
+    # Need to unset $EESSI_CVMFS_INSTALL to use $EESSI_PROJECT_INSTALL
+    unset EESSI_CVMFS_INSTALL
+    export EESSI_PROJECT_INSTALL=${EESSI_CVMFS_REPO_OVERRIDE}
+    echo ">> \$EESSI_PROJECT_INSTALL set to ${EESSI_PROJECT_INSTALL}"
+fi
+
+echo "DEBUG: before loading EESSI-extend // EASYBUILD_INSTALLPATH='${EASYBUILD_INSTALLPATH}'"
+source $TOPDIR/load_eessi_extend_module.sh ${EESSI_VERSION}
+echo "DEBUG: after loading EESSI-extend //  EASYBUILD_INSTALLPATH='${EASYBUILD_INSTALLPATH}'"
 
 # Install full CUDA SDK and cu* libraries in host_injections
 # Hardcode this for now, see if it works
@@ -271,6 +301,7 @@ fi
 
 # Install NVIDIA drivers in host_injections (if they exist)
 if command_exists "nvidia-smi"; then
+    export LD_LIBRARY_PATH="/.singularity.d/libs:${LD_LIBRARY_PATH}"
     nvidia-smi --version
     ec=$?
     if [ ${ec} -eq 0 ]; then 
@@ -299,12 +330,10 @@ if [ ! -z ${EESSI_ACCELERATOR_TARGET} ]; then
     fi
 fi
 
-# If in dev.eessi.io, allow building on top of software.eessi.io
-if [[ "${EESSI_CVMFS_REPO}" == /cvmfs/dev.eessi.io ]]; then
-    module use /cvmfs/software.eessi.io/versions/$EESSI_VERSION/software/${EESSI_OS_TYPE}/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all
-fi
-
-module use $EASYBUILD_INSTALLPATH/modules/all
+echo "DEBUG: adding path '$EASYBUILD_INSTALLPATH/modules/all' to MODULEPATH='${MODULEPATH}'"
+#module use $EASYBUILD_INSTALLPATH/modules/all
+safe_module_use $EASYBUILD_INSTALLPATH/modules/all
+echo "DEBUG: after adding module path // MODULEPATH='${MODULEPATH}'"
 
 if [[ -z ${MODULEPATH} ]]; then
     fatal_error "Failed to set up \$MODULEPATH?!"
@@ -331,7 +360,7 @@ else
         echo -e "Processing easystack file ${easystack_file}...\n\n"
 
         # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file
-        eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g')
+        eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*.yml/\1/g')
 
         # load EasyBuild module (will be installed if it's not available yet)
         source ${TOPDIR}/load_easybuild_module.sh ${eb_version}
@@ -372,8 +401,18 @@ else
     done
 fi
 
-export LMOD_CONFIG_DIR="${EASYBUILD_INSTALLPATH}/.lmod"
+echo "DEBUG: before creating/updating lmod config files // EASYBUILD_INSTALLPATH='${EASYBUILD_INSTALLPATH}'"
+if [[ ! -z ${EESSI_DEV_PROJECT} ]]; then
+    # Make sure .lmod files are not checked for dev.eeessi.io
+    export LMOD_CONFIG_DIR="${EASYBUILD_INSTALLPATH_STANDARD}/.lmod"
+    export LMOD_PACKAGE_PATH="${EASYBUILD_INSTALLPATH_STANDARD}/.lmod"
+else
+    export LMOD_CONFIG_DIR="${EASYBUILD_INSTALLPATH}/.lmod"
+    export LMOD_PACKAGE_PATH="${EASYBUILD_INSTALLPATH}/.lmod"
+fi
+
 lmod_rc_file="$LMOD_CONFIG_DIR/lmodrc.lua"
+echo "DEBUG: lmod_rc_file='${lmod_rc_file}'"
 if [[ ! -z ${EESSI_ACCELERATOR_TARGET} ]]; then
     # EESSI_ACCELERATOR_TARGET is set, so let's remove the accelerator path from $lmod_rc_file
     lmod_rc_file=$(echo ${lmod_rc_file} | sed "s@/accel/${EESSI_ACCELERATOR_TARGET}@@")
@@ -386,7 +425,6 @@ if [ ! -f $lmod_rc_file ] || [ ${lmodrc_changed} == '0' ]; then
     check_exit_code $? "$lmod_rc_file created" "Failed to create $lmod_rc_file"
 fi
 
-export LMOD_PACKAGE_PATH="${EASYBUILD_INSTALLPATH}/.lmod"
 lmod_sitepackage_file="$LMOD_PACKAGE_PATH/SitePackage.lua"
 if [[ ! -z ${EESSI_ACCELERATOR_TARGET} ]]; then
     # EESSI_ACCELERATOR_TARGET is set, so let's remove the accelerator path from $lmod_sitepackage_file

EESSI-remove-software.sh

@@ -108,7 +108,7 @@ if [ $EUID -eq 0 ]; then
     else
         for easystack_file in ${changed_easystacks_rebuilds}; do
             # determine version of EasyBuild module to load based on EasyBuild version included in name of easystack file
-            eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*/\1/g')
+            eb_version=$(echo ${easystack_file} | sed 's/.*eb-\([0-9.]*\).*.yml/\1/g')
 
             # load EasyBuild module (will be installed if it's not available yet)
             source ${TOPDIR}/load_easybuild_module.sh ${eb_version}

bot/build.sh

@@ -147,7 +147,9 @@ echo "bot/build.sh: EESSI_VERSION_OVERRIDE='${EESSI_VERSION_OVERRIDE}'"
 # determine CVMFS repo to be used from .repository.repo_name in ${JOB_CFG_FILE}
 # here, just set EESSI_CVMFS_REPO_OVERRIDE, a bit further down
 # "source init/eessi_defaults" via sourcing init/minimal_eessi_env
-export EESSI_CVMFS_REPO_OVERRIDE=/cvmfs/${REPOSITORY_NAME}
+# Note: iff ${EESSI_DEV_PROJECT} is defined (building for dev.eessi.io), then we 
+# append the project subdirectory to ${EESSI_CVMFS_REPO_OVERRIDE}
+export EESSI_CVMFS_REPO_OVERRIDE=/cvmfs/${REPOSITORY_NAME}${EESSI_DEV_PROJECT:+/$EESSI_DEV_PROJECT}
 echo "bot/build.sh: EESSI_CVMFS_REPO_OVERRIDE='${EESSI_CVMFS_REPO_OVERRIDE}'"
 
 # determine CPU architecture to be used from entry .architecture in ${JOB_CFG_FILE}
@@ -184,6 +186,15 @@ if [[ "${REPOSITORY_NAME}" == "dev.eessi.io" ]]; then
     COMMON_ARGS+=("--repository" "software.eessi.io,access=ro")
 fi
 
+# add $software_layer_dir and /dev as extra bind paths
+#  - $software_layer_dir is needed because it is used as prefix for running scripts
+#  - /dev is needed to access /dev/fuse
+COMMON_ARGS+=("--extra-bind-paths" "${software_layer_dir},/dev")
+
+# pass through '--contain' to avoid leaking in scripts into the container session
+# note, --pass-through can be used multiple times if needed
+COMMON_ARGS+=("--pass-through" "--contain")
+
 # make sure to use the same parent dir for storing tarballs of tmp
 PREVIOUS_TMP_DIR=${PWD}/previous_tmp
 
@@ -309,7 +320,15 @@ fi
 timestamp=$(date +%s)
 # to set EESSI_VERSION we need to source init/eessi_defaults now
 source $software_layer_dir/init/eessi_defaults
-export TGZ=$(printf "eessi-%s-software-%s-%s-%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${timestamp})
+# Note: iff ${EESSI_DEV_PROJECT} is defined (building for dev.eessi.io), then we 
+# append the project (subdirectory) name to the end tarball name. This is information
+# then used at the ingestion stage. If ${EESSI_DEV_PROJECT} is not defined, nothing is
+# appended
+export TGZ=$(printf "eessi-%s-software-%s-%s-%b%d.tar.gz" ${EESSI_VERSION} ${EESSI_OS_TYPE} ${EESSI_SOFTWARE_SUBDIR_OVERRIDE//\//-} ${EESSI_DEV_PROJECT:+$EESSI_DEV_PROJECT-} ${timestamp})
+
+# Export EESSI_DEV_PROJECT to use it (if needed) when making tarball
+echo "bot/build.sh: EESSI_DEV_PROJECT='${EESSI_DEV_PROJECT}'"
+export EESSI_DEV_PROJECT=${EESSI_DEV_PROJECT}
 
 # value of first parameter to create_tarball.sh - TMP_IN_CONTAINER - needs to be
 # synchronised with setting of TMP_IN_CONTAINER in eessi_container.sh