Merge branch 'main' into link_nvidia_drivers

Author: bedroge

.github/workflows/modules/append_module.lua

@@ -0,0 +1,3 @@
+setenv("INSIDE_GITHUB_ACTIONS", "true")
+-- Interfere with PATH so Lmod keeps a record
+prepend_path("PATH", "/snap/bin")

.github/workflows/modules/prepend_module.lua

@@ -0,0 +1,3 @@
+setenv("INSIDE_GITHUB_ACTIONS", "true")
+-- Interfere with PATH so Lmod keeps a record
+prepend_path("PATH", "/snap/bin")

.github/workflows/scripts/test_init_scripts.sh

@@ -11,11 +11,17 @@ if [ -z ${EXPECTED_EASYBUILD_VERSION} ]; then
   exit 1
 fi
 
+if [ -z ${EESSI_SOFTWARE_SUBDIR_OVERRIDE} ]; then
+  echo "\$EESSI_SOFTWARE_SUBDIR_OVERRIDE has to be set (e.g., x86_64/intel/haswell) so we can do well defined string comparison for the architecture."
+  exit 1
+fi
+
 # initialize assert framework
 if [ ! -d assert.sh ]; then
   echo "assert.sh not cloned."
   echo ""
   echo "run \`git clone https://github.com/lehmannro/assert.sh.git\`"
+  echo "(see workflow file that calls this script for how to only clone specific commit if you are worried about security)"
   exit 1
 fi
 . assert.sh/assert.sh
@@ -28,12 +34,16 @@ for shell in ${SHELLS[@]}; do
   echo  RUNNING TESTS FOR SHELL: $shell
   echo = | awk 'NF += (OFS = $_) + 100'
   if [[ ! " ${TEST_SHELLS[*]} " =~ [[:space:]]${shell}[[:space:]] ]]; then
-    ### EXCEPTION FOR CSH ###
     echo -e "\033[33mWe don't now how to test the shell '$shell', PRs are Welcome.\033[0m" 
   else
+    if [ "$shell" = "csh" ]; then
+      # make sure our .cshrc is empty before we begin as we will clobber it
+      [ -f ~/.cshrc ] && mv ~/.cshrc ~/.cshrc_orig
+    fi
+
     # TEST 1: Source Script and check Module Output
-    expected="Module for EESSI/$EESSI_VERSION loaded successfully"
-    assert "$shell -c 'source init/lmod/$shell' 2>&1 " "${expected}"
+    expected_pattern=".*EESSI has selected $EESSI_SOFTWARE_SUBDIR_OVERRIDE as the compatible CPU target for EESSI/$EESSI_VERSION.*"
+    assert_raises "$shell -c 'source init/lmod/$shell' 2>&1 | grep -E \"${expected_pattern}\""
 
     # TEST 2: Check if module overviews first section is the loaded EESSI module
     if [ "$shell" = "csh" ]; then
@@ -43,21 +53,22 @@ for shell in ${SHELLS[@]}; do
       echo "source init/lmod/$shell" > ~/.cshrc
       MODULE_SECTIONS=($($shell -c "module ov" 2>&1 | grep -e '---'))
     else
-      MODULE_SECTIONS=($($shell -c "source init/lmod/$shell 2>/dev/null; module ov 2>&1 | grep -e '---'"))
+      MODULE_SECTIONS=($($shell -c "source init/lmod/$shell >/dev/null 2>&1; module ov 2>&1 | grep -e '---'"))
     fi
-    PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/x86_64/(intel/haswell|amd/zen3)/modules/all"
+    PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/$EESSI_SOFTWARE_SUBDIR_OVERRIDE/modules/all"
     assert_raises 'echo "${MODULE_SECTIONS[1]}" | grep -E "$PATTERN"'
+    # echo "${MODULE_SECTIONS[1]}" "$PATTERN"
 
     # TEST 3: Check if module overviews second section is the EESSI init module
-    assert "echo ${MODULE_SECTIONS[4]}" "/cvmfs/software.eessi.io/versions/$EESSI_VERSION/init/modules"
+    assert "echo ${MODULE_SECTIONS[4]}" "/cvmfs/software.eessi.io/init/modules"
 
     # TEST 4: Load EasyBuild module and check version
     # eb --version outputs: "This is EasyBuild 5.1.1 (framework: 5.1.1, easyblocks: 5.1.1) on host ..."
     if [ "$shell" = "csh" ]; then
       echo "source init/lmod/$shell" > ~/.cshrc
-      command="$shell -c 'module load EasyBuild/${EXPECTED_EASYBUILD_VERSION}; eb --version' | cut -d \" \" -f4"
+      command="$shell -c 'module load EasyBuild/${EXPECTED_EASYBUILD_VERSION}; eb --version' | tail -n 1 | awk '{print \$4}'"
     else
-      command="$shell -c 'source init/lmod/$shell 2>/dev/null; module load EasyBuild/${EXPECTED_EASYBUILD_VERSION}; eb --version | cut -d \" \" -f4'"
+      command="$shell -c 'source init/lmod/$shell >/dev/null 2>&1; module load EasyBuild/${EXPECTED_EASYBUILD_VERSION}; eb --version' | tail -n 1 | awk '{print \$4}'"
     fi
     assert "$command" "$EXPECTED_EASYBUILD_VERSION"
 
@@ -69,12 +80,82 @@ for shell in ${SHELLS[@]}; do
       EASYBUILD_PATH=$($shell -c "source init/lmod/$shell 2>/dev/null; module load EasyBuild/${EXPECTED_EASYBUILD_VERSION}; which eb")
     fi
     # escape the dots in ${EASYBUILD_VERSION}
-    PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/x86_64/(intel/haswell|amd/zen3)/software/EasyBuild/${EXPECTED_EASYBUILD_VERSION//./\\.}/bin/eb"
-    echo "$EASYBUILD_PATH" | grep -E "$PATTERN"
+    PATTERN="/cvmfs/software\.eessi\.io/versions/$EESSI_VERSION/software/linux/$EESSI_SOFTWARE_SUBDIR_OVERRIDE/software/EasyBuild/${EXPECTED_EASYBUILD_VERSION//./\\.}/bin/eb"
+    # echo "$EASYBUILD_PATH" | grep -E "$PATTERN"
     assert_raises 'echo "$EASYBUILD_PATH" | grep -E "$PATTERN"'
+    # echo "$EASYBUILD_PATH" "$PATTERN"
+
+    # TEST 6 and 7: Check the various options (EESSI_DEFAULT_MODULES_APPEND, EESSI_DEFAULT_MODULES_APPEND, EESSI_EXTRA_MODULEPATH) all work
+    if [ "$shell" = "csh" ]; then
+      echo "setenv EESSI_DEFAULT_MODULES_APPEND append_module" > ~/.cshrc
+      echo "setenv EESSI_DEFAULT_MODULES_PREPEND prepend_module" >> ~/.cshrc
+      echo "setenv EESSI_EXTRA_MODULEPATH .github/workflows/modules" >> ~/.cshrc
+      echo "source init/lmod/$shell" >> ~/.cshrc
+      TEST_LMOD_SYSTEM_DEFAULT_MODULES=$($shell -c 'echo $LMOD_SYSTEM_DEFAULT_MODULES')
+      TEST_MODULEPATH=$($shell -c 'echo $MODULEPATH')
+    elif [ "$shell" = "fish" ]; then
+      TEST_LMOD_SYSTEM_DEFAULT_MODULES=$($shell -c 'set -x EESSI_DEFAULT_MODULES_APPEND append_module ; set -x EESSI_DEFAULT_MODULES_PREPEND prepend_module ; set -x EESSI_EXTRA_MODULEPATH .github/workflows/modules ; source init/lmod/'"$shell"' 2>/dev/null; echo $LMOD_SYSTEM_DEFAULT_MODULES')
+      TEST_MODULEPATH=$($shell -c 'set -x EESSI_DEFAULT_MODULES_APPEND append_module ; set -x EESSI_DEFAULT_MODULES_PREPEND prepend_module ; set -x EESSI_EXTRA_MODULEPATH .github/workflows/modules ; source init/lmod/'"$shell"' 2>/dev/null; echo $MODULEPATH')
+    else
+      TEST_LMOD_SYSTEM_DEFAULT_MODULES=$($shell -c 'export EESSI_DEFAULT_MODULES_APPEND=append_module ; export EESSI_DEFAULT_MODULES_PREPEND=prepend_module ; export EESSI_EXTRA_MODULEPATH=.github/workflows/modules ; source init/lmod/'"$shell"' ; echo $LMOD_SYSTEM_DEFAULT_MODULES')
+      TEST_MODULEPATH=$($shell -c 'export EESSI_DEFAULT_MODULES_APPEND=append_module ; export EESSI_DEFAULT_MODULES_PREPEND=prepend_module ; export EESSI_EXTRA_MODULEPATH=.github/workflows/modules ; source init/lmod/'"$shell"' 2>/dev/null; echo $MODULEPATH')
+    fi
+    LMOD_SYSTEM_DEFAULT_MODULES_PATTERN='^prepend_module:.*:append_module$'
+    # echo "$TEST_LMOD_SYSTEM_DEFAULT_MODULES" AND "$LMOD_SYSTEM_DEFAULT_MODULES_PATTERN"
+    assert_raises 'echo "$TEST_LMOD_SYSTEM_DEFAULT_MODULES" | grep -E "$LMOD_SYSTEM_DEFAULT_MODULES_PATTERN"'
+    if [ "$shell" = "fish" ]; then
+      MODULEPATH_PATTERN='\.github/workflows/modules$'
+    else
+      MODULEPATH_PATTERN=':\.github/workflows/modules$'
+    fi
+    # echo "$TEST_MODULEPATH" AND "$MODULEPATH_PATTERN"
+    assert_raises 'echo "$TEST_MODULEPATH" | grep -E "$MODULEPATH_PATTERN"'
+
+    # TEST 8 and 9: Add a conditional test depending on whether we have the Lmod command is available locally or not (Ubuntu-based location for CI)
+    if [ -d "$LMOD_PKG/init" ]; then
+        echo "Running check for locally available Lmod with purge"
+        if [ "$shell" = "csh" ]; then
+          echo "source $LMOD_PKG/init/$shell" > ~/.cshrc
+          echo "source init/lmod/$shell" >> ~/.cshrc
+          TEST_EESSI_WITH_PURGE=$($shell -c 'echo')
+          echo "source $LMOD_PKG/init/$shell" > ~/.cshrc
+          echo "setenv EESSI_NO_MODULE_PURGE_ON_INIT 1" >> ~/.cshrc
+          echo "source init/lmod/$shell" >> ~/.cshrc
+          TEST_EESSI_WITHOUT_PURGE=$($shell -c 'echo $EESSI_NO_MODULE_PURGE_ON_INIT')
+        elif [ "$shell" = "fish" ]; then
+          TEST_EESSI_WITH_PURGE=$($shell -c "source $LMOD_PKG/init/$shell 2>/dev/null ; source init/lmod/$shell 2>/dev/null")
+          TEST_EESSI_WITHOUT_PURGE=$($shell -c "set -x EESSI_NO_MODULE_PURGE_ON_INIT 1 ; source $LMOD_PKG/init/$shell 2>/dev/null ; source init/lmod/$shell 2>/dev/null")
+        else
+          TEST_EESSI_WITH_PURGE=$($shell -c "source $LMOD_PKG/init/$shell 2>/dev/null ; source init/lmod/$shell 2>/dev/null")
+          TEST_EESSI_WITHOUT_PURGE=$($shell -c "export EESSI_NO_MODULE_PURGE_ON_INIT=1 ; source $LMOD_PKG/init/$shell 2>/dev/null ; source init/lmod/$shell 2>/dev/null")
+        fi
+        # In the first case we should have the test and in the second case we shouldn't
+        pattern="Modules purged before initialising EESSI"
+        echo $TEST_EESSI_WITH_PURGE
+        assert_raises 'echo "$TEST_EESSI_WITH_PURGE" | grep "$pattern"'
+        # this case should raise 1
+        echo $TEST_EESSI_WITHOUT_PURGE
+        assert_raises 'echo "$TEST_EESSI_WITHOUT_PURGE" | grep "$pattern"' 1
+    fi
+
+    # Optional test 10, check if the prompt has been updated
+    if [ "$shell" = "bash" ] || [ "$shell" = "ksh" ] || [ "$shell" = "zsh" ]; then
+        # Typically this is a non-interactive shell, so manually unset PS1 and reset to a non-exported variable when testing
+        TEST_EESSI_PS1_UPDATE=$($shell -c "unset PS1 ; PS1='$ ' ; source init/lmod/$shell 2>/dev/null ; echo \"\$PS1\"")
+        TEST_EESSI_NO_PS1_UPDATE=$($shell -c "unset PS1 ; source init/lmod/$shell 2>/dev/null ; echo \"\$PS1\"")
+        pattern="{EESSI/${EESSI_VERSION}} "
+        assert_raises 'echo "$TEST_EESSI_PS1_UPDATE" | grep "$pattern"'
+        assert_raises 'echo "$TEST_EESSI_NO_PS1_UPDATE" | grep "$pattern"' 1
+    fi
 
     # End Test Suite
     assert_end "source_eessi_$shell"
+
+    if [ "$shell" = "csh" ]; then
+      # Restore our .cshrc
+      [ -f ~/.cshrc_orig ] && mv ~/.cshrc_orig ~/.cshrc
+    fi
+
   fi
 done
 

.github/workflows/tests_eessi_module.yml

@@ -109,11 +109,11 @@ jobs:
           # First do (and undo) the Lmod initialisation
           export MODULEPATH=init/modules
           # Turn on debug output in case we want to take a look
-          export EESSI_DEBUG_INIT=true
+          export EESSI_MODULE_DEBUG_INIT=true
           CPU_ARCH=$(./init/eessi_archdetect.sh -a cpupath)
           module load EESSI/${{matrix.EESSI_VERSION}}
-          # EESSI_DEBUG_INIT/EESSI_ARCHDETECT_OPTIONS only relevant for Lmod init
-          unset EESSI_DEBUG_INIT
+          # EESSI_MODULE_DEBUG_INIT/EESSI_ARCHDETECT_OPTIONS only relevant for Lmod init
+          unset EESSI_MODULE_DEBUG_INIT
           # Store all relevant environment variables
           env | grep -E '(^EESSI_|^LMOD_RC|^LMOD_PACKAGE_PATH|^MODULEPATH)' | grep -v EESSI_ARCHDETECT_OPTIONS | sort > "${moduleoutfile}"
           module unload EESSI/${{matrix.EESSI_VERSION}}
@@ -200,7 +200,7 @@ jobs:
           fi
 
           # Turn on debug output in case we want to take a look
-          export EESSI_DEBUG_INIT=true
+          export EESSI_MODULE_DEBUG_INIT=true
 
           initial_env_file="initial_env.txt"
           module_cycled_file="load_unload_cycle.txt"

.github/workflows/tests_init_module.yml

@@ -46,18 +46,34 @@ jobs:
             sed -i "s/__EESSI_VERSION_DEFAULT__/${{matrix.EESSI_VERSION}}/g" init/lmod/${shell}
           done
 
-      - name: Clone assert.sh script
-        run: git clone https://github.com/lehmannro/assert.sh.git
+      - name: Clone assert.sh at pinned commit
+        run: |
+          ASSERT_COMMIT_SHA=fe359e341670f1e8e86a3804ca00e5a3ebc30fa4
+          git clone --no-checkout https://github.com/lehmannro/assert.sh.git
+          cd assert.sh
+          git fetch --depth 1 origin $ASSERT_COMMIT_SHA
+          git checkout $ASSERT_COMMIT_SHA
 
       - name: Install missing shells
         run: |
           sudo apt update
           sudo apt install zsh ksh fish tcsh
           echo "# INIT ZSH" > ~/.zshrc
 
-      - name: Run tests for available shells
+      - name: Run tests for available shells without system Lmod
         run: |
+          export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
+          export EESSI_VERSION=${{matrix.EESSI_VERSION}}
+          export EXPECTED_EASYBUILD_VERSION=${{matrix.EXPECTED_EASYBUILD_VERSION}}
+          .github/workflows/scripts/test_init_scripts.sh "bash" "zsh" "ksh" "fish" "csh"
+      - name: Run tests for available shells with system Lmod
+        run: |
+          # We also want to perform the same test when there is an Lmod version available on the system
+          sudo apt install lmod
+          source /usr/share/lmod/lmod/init/bash
+          export MODULEPATH=/usr/share/lmod/lmod/modulefiles
+          module avail
+          export EESSI_SOFTWARE_SUBDIR_OVERRIDE=${{matrix.EESSI_SOFTWARE_SUBDIR_OVERRIDE}}
           export EESSI_VERSION=${{matrix.EESSI_VERSION}}
           export EXPECTED_EASYBUILD_VERSION=${{matrix.EXPECTED_EASYBUILD_VERSION}}
           .github/workflows/scripts/test_init_scripts.sh "bash" "zsh" "ksh" "fish" "csh"
-

create_tarball.sh

@@ -91,7 +91,7 @@ for subdir in ${sw_subdirs}; do
             echo "handling ${package_version}"
             find ${eessi_version}/software/${os}/${subdir}/software/${package_version} -maxdepth 0 -type d \! -name '.wh.*' >> ${files_list}
             # if there is a directory for this installation in the stack's reprod directory, include that too
-            if [ -d ${eessi_version}/software/${os}/${subdir}/reprod ]; then
+            if [ -d ${eessi_version}/software/${os}/${subdir}/reprod/${package_version} ]; then
                 find ${eessi_version}/software/${os}/${subdir}/reprod/${package_version} -maxdepth 0 -type d \! -name '.wh.*' >> ${files_list}
             fi
         done

eb_hooks.py

@@ -151,7 +151,6 @@ def is_gcccore_1220_based(**kwargs):
     )
 
 
-
 def get_cuda_version(ec, check_deps=True, check_builddeps=True):
     """
     Returns the CUDA version that this EasyConfig (ec) uses as a (build)dependency.
@@ -632,6 +631,24 @@ def parse_hook_qt5_check_qtwebengine_disable(ec, eprefix):
         raise EasyBuildError("Qt5-specific hook triggered for non-Qt5 easyconfig?!")
 
 
+def parse_hook_maturin(ec, eprefix):
+    """
+    Replace build dependency on Rust 1.88.0 by 1.91.1,
+    as 1.88.0 causes segmentation faults on A64FX.
+    cfr. https://github.com/EESSI/software-layer/pull/1357
+    """
+    if ec.name == 'maturin':
+        if ec.version == '1.9.1':
+            orig_rust = ('Rust', '1.88.0')
+            new_rust = ('Rust', '1.91.1')
+            if orig_rust in ec['builddependencies']:
+                rust_index = ec['builddependencies'].index(orig_rust)
+                ec['builddependencies'][rust_index] = new_rust
+                print_msg(f"Replaced {orig_rust} build dependency by {new_rust} for {ec.name} {ec.version}")
+    else:
+        raise EasyBuildError("maturin-specific hook triggered for non-maturin easyconfig?!")
+
+
 def parse_hook_ucx_eprefix(ec, eprefix):
     """Make UCX aware of compatibility layer via additional configuration options."""
     if ec.name == 'UCX':
@@ -1461,6 +1478,19 @@ def pre_test_hook_increase_max_failed_tests_arm_PyTorch(self, *args, **kwargs):
             self.cfg['max_failed_tests'] = 4
 
 
+def pre_test_hook_ignore_failing_tests_OpenBabel_a64fx(self, *args, **kwargs):
+    """
+    Pre-test hook for OpenBabel: skip timeout tests for OpenBabel 3.1.1 on aarch64/a64fx
+    see https://github.com/EESSI/software-layer/pull/1332#issuecomment-3877255228
+    the `testroundtrip.py` test reads and writes tens of thousands of small files.
+    The test works fine when manually ran with EESSI-extend either directly or inside an eessi_container, but 
+    consistently fails with the bot
+    """
+    cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')
+    if self.name == 'OpenBabel' and self.version == '3.1.1' and cpu_target == CPU_TARGET_A64FX:
+        self.cfg['testopts'] = "|| echo ignoring failing tests"
+
+
 def pre_single_extension_hook(ext, *args, **kwargs):
     """Main pre-extension: trigger custom functions based on software name."""
     if ext.name in PRE_SINGLE_EXTENSION_HOOKS:
@@ -1818,6 +1848,7 @@ def post_easyblock_hook(self, *args, **kwargs):
     'fontconfig': parse_hook_fontconfig_add_fonts,
     'FreeImage': parse_hook_freeimage_aarch64,
     'grpcio': parse_hook_grpcio_zlib,
+    'maturin': parse_hook_maturin,
     'Mesa': parse_hook_mesa_use_llvm_minimal,
     'OpenBLAS': parse_hook_openblas_relax_lapack_tests_num_errors,
     'pybind11': parse_hook_pybind11_replace_catch2,
@@ -1868,6 +1899,7 @@ def post_easyblock_hook(self, *args, **kwargs):
     'Highway': pre_test_hook_exclude_failing_test_Highway,
     'SciPy-bundle': pre_test_hook_ignore_failing_tests_SciPybundle,
     'netCDF': pre_test_hook_ignore_failing_tests_netCDF,
+    'OpenBabel': pre_test_hook_ignore_failing_tests_OpenBabel_a64fx,
     'PyTorch': pre_test_hook_increase_max_failed_tests_arm_PyTorch,
 }
 

eessi_container.sh

@@ -828,7 +828,7 @@ if [[ ! -z ${http_proxy} ]]; then
             ;;
         2)
             # target not found - safe to add
-            if [[ -z ${BIND_PATH} ]]; then
+            if [[ -z ${BIND_PATHS} ]]; then
                 BIND_PATHS="${src}:${target}"
             else
                 BIND_PATHS="${BIND_PATHS},${src}:${target}"