Lift fallback for PYPI package names to obtain_file

Author: Flamefire

easybuild/framework/easyblock.py

@@ -680,12 +680,11 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
                         'version': ext_version,
                         'options': ext_options,
                         'github_account': ext_options.get('github_account', orig_github_account),
+                        # if a particular easyblock is specified, make sure it's used
+                        # (this is picked up by init_ext_instances)
+                        'easyblock': ext_options.get('easyblock', None),
                     }
 
-                    # if a particular easyblock is specified, make sure it's used
-                    # (this is picked up by init_ext_instances)
-                    ext_src['easyblock'] = ext_options.get('easyblock', None)
-
                     # construct dictionary with template values;
                     # inherited from parent, except for name/version templates which are specific to this extension
                     template_values = copy.deepcopy(self.cfg.template_values)
@@ -740,13 +739,9 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
 
                         # if no sources are specified via 'sources', fall back to 'source_tmpl'
                         src_fn = ext_options.get('source_tmpl')
-                        is_pypi_source = False
                         if src_fn is None:
                             # use default template for name of source file if none is specified
                             src_fn = '%(name)s-%(version)s.tar.gz'
-                            if len(source_urls) == 1 and PYPI_PKG_URL_PATTERN in source_urls[0]:
-                                # allow retrying with alternative download_filename
-                                is_pypi_source = True
                         elif not isinstance(src_fn, str):
                             error_msg = "source_tmpl value must be a string! (found value of type '%s'): %s"
                             raise EasyBuildError(error_msg, type(src_fn).__name__, src_fn)
@@ -756,17 +751,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
                         if fetch_files:
                             src_path = self.obtain_file(src_fn, extension=True, urls=source_urls,
                                                         force_download=force_download,
-                                                        download_instructions=download_instructions,
-                                                        warning_only=is_pypi_source)
-                            if not src_path and is_pypi_source:
-                                # retry with alternative download_filename
-                                alt_name = resolve_template('%(name)s', template_values).replace("-", "_")
-                                src_version = resolve_template('%(version)s', template_values)
-                                alt_download_fn = f'{alt_name}-{src_version}.tar.gz'
-                                src_path = self.obtain_file(src_fn, extension=True, urls=source_urls,
-                                                            force_download=force_download,
-                                                            download_instructions=download_instructions,
-                                                            download_filename=alt_download_fn)
+                                                        download_instructions=download_instructions)
                             if src_path:
                                 ext_src.update({'src': src_path})
                             else:
@@ -1027,15 +1012,31 @@ def obtain_file(self, filename, extension=False, urls=None, download_filename=No
                 target_path = os.path.join(targetdir, "extensions", filename)
             else:
                 target_path = os.path.join(targetdir, filename)
-            downloaded, failed_urls = self.download_file(target_path, download_filename=download_filename or filename,
-                                                         urls=source_urls,
+            if download_filename is None:
+                download_filename = filename
+            downloaded, failed_urls = self.download_file(target_path, download_filename, urls=source_urls,
                                                          # Only log when extensions have explicit URLs
                                                          log_url_in_dry_run=extension and urls)
             if downloaded:
                 return target_path
-
             failedpaths.extend(failed_urls)
 
+            # Usual PYPI sources have a format of '<name>-version.ext', with '<name>' has dashes replaced by underscores
+            num_dashes = download_filename.count('-')
+            if num_dashes >= 2 and any(PYPI_PKG_URL_PATTERN in url for url in source_urls):
+                # Last dash is the separator between name and version
+                alt_download_filename = download_filename.replace('-', '_', num_dashes - 1)
+                self.log.warning("Could not download %s (%s) from any of %s.\n"
+                                 "Retrying with alternative download filename '%s' as it looks like a PYPI source.",
+                                 filename, download_filename, ', '.join(source_urls), alt_download_filename)
+                # retry with alternative download_filename
+                downloaded, failed_urls = self.download_file(target_path, alt_download_filename, urls=source_urls,
+                                                             # Only log when extensions have explicit URLs
+                                                             log_url_in_dry_run=extension and urls)
+                if downloaded:
+                    return target_path
+                failedpaths.extend(failed_urls)
+
             if self.dry_run:
                 self.dry_run_msg("  * %s (MISSING)", filename)
                 return filename
@@ -2829,7 +2830,7 @@ def check_checksums_for(self, ent, sub='', source_cnt=None):
                 # if the filename is a dict, the actual source file name is the "filename" element
                 if isinstance(fn, dict):
                     fn = fn["filename"]
-                if fn in checksums_from_json.keys():
+                if fn in checksums_from_json:
                     checksums += [checksums_from_json[fn]]
 
         if source_cnt is None: