Merge branch 'main' into reprod_scripts

Author: Caspar van Leeuwen

.github/workflows/test_licenses.yml

@@ -244,8 +244,12 @@ if mode() == "unload" or mode() == "dependencyCk" or convertToCanonical(easybuil
   -- Set environment variables that are EESSI version specific
   if convertToCanonical(eessi_version) > convertToCanonical("2023.06") then
     setenv ("EASYBUILD_PREFER_PYTHON_SEARCH_PATH", "EBPYTHONPREFIXES")
+    -- Note: make sure to *not* use same setting for both module-search-path-headers and search-path-cpp-headers
+    -- EasyBuild configuration options, since this leads to trouble because (for example) $C_INCLUDE_PATH
+    -- only includes paths for the *direct* dependencies (not transitive dependencies)
+    -- when using EasyBuild v5.2.1 or earlier
     setenv ("EASYBUILD_MODULE_SEARCH_PATH_HEADERS", "include_paths")
-    setenv ("EASYBUILD_SEARCH_PATH_CPP_HEADERS", "include_paths")
+    setenv ("EASYBUILD_SEARCH_PATH_CPP_HEADERS", "flags")
   end
 end
 

EESSI-extend-easybuild.eb

@@ -91,7 +91,7 @@ for subdir in ${sw_subdirs}; do
             echo "handling ${package_version}"
             find ${eessi_version}/software/${os}/${subdir}/software/${package_version} -maxdepth 0 -type d \! -name '.wh.*' >> ${files_list}
             # if there is a directory for this installation in the stack's reprod directory, include that too
-            if [ -d ${eessi_version}/software/${os}/${subdir}/reprod ]; then
+            if [ -d ${eessi_version}/software/${os}/${subdir}/reprod/${package_version} ]; then
                 find ${eessi_version}/software/${os}/${subdir}/reprod/${package_version} -maxdepth 0 -type d \! -name '.wh.*' >> ${files_list}
             fi
         done

create_tarball.sh

@@ -40,6 +40,7 @@
 CPU_TARGET_ICELAKE = 'x86_64/intel/icelake'
 CPU_TARGET_SAPPHIRE_RAPIDS = 'x86_64/intel/sapphirerapids'
 CPU_TARGET_ZEN4 = 'x86_64/amd/zen4'
+CPU_TARGET_ZEN5 = 'x86_64/amd/zen5'
 
 EESSI_RPATH_OVERRIDE_ATTR = 'orig_rpath_override_dirs'
 EESSI_MODULE_ONLY_ATTR = 'orig_module_only'
@@ -697,7 +698,7 @@ def pre_fetch_hook_check_installation_path(self, *args, **kwargs):
     strict_eessi_installation = (
         bool(re.search(EESSI_INSTALLATION_REGEX, self.installdir)) or
         self.installdir.startswith(HOST_INJECTIONS_LOCATION))
-    if strict_eessi_installation:
+    if strict_eessi_installation and not os.getenv("EESSI_OVERRIDE_STRICT_INSTALLPATH_CHECK"):
         dependency_names = self.cfg.dependency_names()
         if self.cfg.name in accelerator_deps or any(dep in dependency_names for dep in accelerator_deps):
             # Make sure the path is an accelerator location
@@ -960,10 +961,11 @@ def pre_configure_hook(self, *args, **kwargs):
         self.cfg.update('configopts', 'CPPFLAGS="-DOF=_Z_OF ${CPPFLAGS}"')
 
 
-def pre_configure_hook_BLIS_a64fx(self, *args, **kwargs):
+def pre_configure_hook_BLIS(self, *args, **kwargs):
     """
-    Pre-configure hook for BLIS when building for A64FX to fix "Illegal instruction" problem
-    - add -DCACHE_SECTOR_SIZE_READONLY to $CFLAGS for BLIS 0.9.0, cfr. https://github.com/flame/blis/issues/800
+    Pre-configure hook for BLIS
+    - fix "Illegal instruction" problem on A64FX by adding -DCACHE_SECTOR_SIZE_READONLY to $CFLAGS for BLIS 0.9.0, cfr. https://github.com/flame/blis/issues/800
+    - fall back to zen3 for zen5 to solve "Unable to automatically detect hardware type" error
     """
     if self.name == 'BLIS':
         cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')
@@ -974,6 +976,8 @@ def pre_configure_hook_BLIS_a64fx(self, *args, **kwargs):
             cflags_var = 'CFLAGS="$CFLAGS -DCACHE_SECTOR_SIZE_READONLY"'
             config_target = config_opts[-1]
             self.cfg['configopts'] = ' '.join(config_opts[:-1] + [cflags_var, config_target])
+        if self.version in ('1.0', '1.1', '2.0') and cpu_target == CPU_TARGET_ZEN5:
+          self.cfg['cpu_architecture'] = 'zen3'
     else:
         raise EasyBuildError("BLIS-specific hook triggered for non-BLIS easyconfig?!")
 
@@ -1089,7 +1093,7 @@ def pre_configure_hook_graphviz(self, *args, **kwargs):
         # Replace --with-ltdl-lib and --with-zlibdir options defined in the EC to point to compat layer
         lib_dir = os.path.join(usr_dir, 'lib64')
         new_items = set()
-        # Add to the new_items all the old items except the `--with-ltdl-lib` and `--with-ltdl-lib` for 
+        # Add to the new_items all the old items except the `--with-ltdl-lib` and `--with-ltdl-lib` for
         # which we fix the lib dir to lib64 instead of lib
         for item in old_items:
             if item.startswith('--with-ltdl-lib'):
@@ -1483,7 +1487,7 @@ def pre_test_hook_ignore_failing_tests_OpenBabel_a64fx(self, *args, **kwargs):
     Pre-test hook for OpenBabel: skip timeout tests for OpenBabel 3.1.1 on aarch64/a64fx
     see https://github.com/EESSI/software-layer/pull/1332#issuecomment-3877255228
     the `testroundtrip.py` test reads and writes tens of thousands of small files.
-    The test works fine when manually ran with EESSI-extend either directly or inside an eessi_container, but 
+    The test works fine when manually ran with EESSI-extend either directly or inside an eessi_container, but
     consistently fails with the bot
     """
     cpu_target = get_eessi_envvar('EESSI_SOFTWARE_SUBDIR')
@@ -1873,7 +1877,7 @@ def post_easyblock_hook(self, *args, **kwargs):
 }
 
 PRE_CONFIGURE_HOOKS = {
-    'BLIS': pre_configure_hook_BLIS_a64fx,
+    'BLIS': pre_configure_hook_BLIS,
     'CUDA-Samples': pre_configure_hook_CUDA_Samples_test_remove,
     'GObject-Introspection': pre_configure_hook_gobject_introspection,
     'Extrae': pre_configure_hook_extrae,

eb_hooks.py

@@ -828,7 +828,7 @@ if [[ ! -z ${http_proxy} ]]; then
             ;;
         2)
             # target not found - safe to add
-            if [[ -z ${BIND_PATH} ]]; then
+            if [[ -z ${BIND_PATHS} ]]; then
                 BIND_PATHS="${src}:${target}"
             else
                 BIND_PATHS="${BIND_PATHS},${src}:${target}"

eessi_container.sh

@@ -93,11 +93,11 @@ cpupath(){
     # Identify the best matching CPU architecture from a list of supported specifications for the host CPU
     # Return the path to the installation files in EESSI of the best matching architecture
     local cpu_arch_spec=()
-  
+
     # Identify the host CPU architecture
     local machine_type=${EESSI_MACHINE_TYPE:-$(uname -m)}
     log "DEBUG" "cpupath: Host CPU architecture identified as '$machine_type'"
-  
+
     # Populate list of supported specs for this architecture
     case $machine_type in
         "x86_64") local spec_file="eessi_arch_x86.spec";;
@@ -109,14 +109,28 @@ cpupath(){
     # spec files are located in a subfolder with this script
     local base_dir=$(dirname $(readlink -f $0))
     update_arch_specs "$base_dir/arch_specs/${spec_file}"
-  
+
     # Identify the host CPU vendor
     local cpu_vendor=$(get_cpuinfo "vendor[ _]id")
     if [ "${cpu_vendor}" == "" ]; then
         cpu_vendor=$(get_cpuinfo "cpu[ _]implementer")
     fi
     log "DEBUG" "cpupath: CPU vendor of host system: '$cpu_vendor'"
-  
+    # Construct a list of known cpu vendors
+    local cpu_vendors=()
+    for spec in "${cpu_arch_spec[@]}"; do
+        eval "cols=$spec"
+        cpu_vendors+=("${cols[1]}")
+    done
+    log "DEBUG" "cpupath: Known CPU vendors: ${cpu_vendors[*]}"
+    # For ARM, if CPU vendor is as-yet-unknown fall back to a default ARM vendor 0x41
+    if [ "${machine_type}" == "aarch64" ]; then
+        if [[ " ${cpu_vendors[*]} " != *" $cpu_vendor "* ]]; then
+            log "DEBUG" "cpupath: Unknown ARM CPU vendor '$cpu_vendor', falling back to '0x41'"
+            cpu_vendor="0x41"
+        fi
+    fi
+
     # Identify the host CPU flags or features
     # cpuinfo systems print different line identifiers, eg features, instead of flags
     local cpu_flag_tag;
@@ -132,14 +146,14 @@ cpupath(){
     else
         cpu_flag_tag='flags'
     fi
-  
+
     local cpu_flags=$(get_cpuinfo "$cpu_flag_tag")
     log "DEBUG" "cpupath: CPU flags of host system: '$cpu_flags'"
-  
+
     # Default to generic CPU
     local best_arch_match="$machine_type/generic"
     local all_arch_matches=$best_arch_match
-  
+
     # Iterate over the supported CPU specifications to find the best match for host CPU
     # Order of the specifications matters, the last one to match will be selected
     for arch in "${cpu_arch_spec[@]}"; do
@@ -151,7 +165,7 @@ cpupath(){
                 log "DEBUG" "cpupath: host CPU best match updated to $best_arch_match" 
         fi
     done
-  
+
     if [ "allx" == "${CPUPATH_RESULT}x" ]; then
         log "INFO" "cpupath: all matches for host CPU: $all_arch_matches"
         echo "$all_arch_matches"
@@ -219,4 +233,3 @@ case "$ARGUMENT" in
     "accelpath") accelpath; exit;;
     *) echo "$USAGE"; log "ERROR" "Missing <action> argument (possible actions: 'cpupath', 'accelpath')";;
 esac
-

init/eessi_archdetect.sh

@@ -15,3 +15,10 @@ easyconfigs:
       options:
         accept-eula-for: cuDNN
         cuda-sanity-check-accept-missing-ptx: True
+  - CUDA-12.9.1.eb:
+      options:
+        accept-eula-for: CUDA
+  - cuDNN-9.15.0.57-CUDA-12.9.1.eb:
+      options:
+        accept-eula-for: cuDNN
+        cuda-sanity-check-accept-missing-ptx: True