diff --git a/easybuild/base/frozendict.py b/easybuild/base/frozendict.py index 06ad117c99..91fa117476 100644 --- a/easybuild/base/frozendict.py +++ b/easybuild/base/frozendict.py @@ -76,7 +76,7 @@ def __init__(self, *args, **kwargs): # handle unknown keys: either ignore them or raise an exception tmpdict = dict(*args, **kwargs) - unknown_keys = [key for key in tmpdict.keys() if key not in self.KNOWN_KEYS] + unknown_keys = [key for key in tmpdict if key not in self.KNOWN_KEYS] if unknown_keys: if ignore_unknown_keys: for key in unknown_keys: diff --git a/easybuild/base/generaloption.py b/easybuild/base/generaloption.py index f0db658f3b..a2ffddda8a 100644 --- a/easybuild/base/generaloption.py +++ b/easybuild/base/generaloption.py @@ -1023,7 +1023,7 @@ def _make_debug_options(self): def _set_default_loglevel(self): """Set the default loglevel if no logging options are set""" - loglevel_set = sum([getattr(self.options, name, False) for name in self._logopts.keys()]) + loglevel_set = sum([getattr(self.options, name, False) for name in self._logopts]) if not loglevel_set and self.DEFAULT_LOGLEVEL is not None: setLogLevel(self.DEFAULT_LOGLEVEL) diff --git a/easybuild/base/optcomplete.py b/easybuild/base/optcomplete.py index 7f76f79194..e72dc71f43 100644 --- a/easybuild/base/optcomplete.py +++ b/easybuild/base/optcomplete.py @@ -169,7 +169,7 @@ def __call__(self, **kwargs): if self.CALL_ARGS_OPTIONAL is not None: all_args.extend(self.CALL_ARGS_OPTIONAL) - for arg in kwargs.keys(): + for arg in kwargs: if arg not in all_args: # remove it kwargs.pop(arg) diff --git a/easybuild/framework/easyconfig/easyconfig.py b/easybuild/framework/easyconfig/easyconfig.py index 698891a772..0343b673dd 100644 --- a/easybuild/framework/easyconfig/easyconfig.py +++ b/easybuild/framework/easyconfig/easyconfig.py @@ -2203,11 +2203,11 @@ def resolve_template(value, tmpl_dict, expect_resolved=True): orig_value = value # map old templates to new values for alternative and deprecated templates alt_map = {old_tmpl: tmpl_dict[new_tmpl] for (old_tmpl, new_tmpl) in - ALTERNATIVE_EASYCONFIG_TEMPLATES.items() if new_tmpl in tmpl_dict.keys()} + ALTERNATIVE_EASYCONFIG_TEMPLATES.items() if new_tmpl in tmpl_dict} alt_map2 = {new_tmpl: tmpl_dict[old_tmpl] for (old_tmpl, new_tmpl) in - ALTERNATIVE_EASYCONFIG_TEMPLATES.items() if old_tmpl in tmpl_dict.keys()} + ALTERNATIVE_EASYCONFIG_TEMPLATES.items() if old_tmpl in tmpl_dict} depr_map = {old_tmpl: tmpl_dict[new_tmpl] for (old_tmpl, (new_tmpl, ver)) in - DEPRECATED_EASYCONFIG_TEMPLATES.items() if new_tmpl in tmpl_dict.keys()} + DEPRECATED_EASYCONFIG_TEMPLATES.items() if new_tmpl in tmpl_dict} # try templating with alternative and deprecated templates included value = value % {**tmpl_dict, **alt_map, **alt_map2, **depr_map} diff --git a/easybuild/framework/easyconfig/tools.py b/easybuild/framework/easyconfig/tools.py index dd0e298aed..9025792ba7 100644 --- a/easybuild/framework/easyconfig/tools.py +++ b/easybuild/framework/easyconfig/tools.py @@ -46,6 +46,7 @@ import sys import tempfile from collections import OrderedDict +from typing import List from easybuild.base import fancylogger from easybuild.framework.easyconfig import EASYCONFIGS_PKG_SUBDIR @@ -60,7 +61,7 @@ from easybuild.tools.environment import restore_env from easybuild.tools.filetools import EASYBLOCK_CLASS_PREFIX, get_cwd, find_easyconfigs, is_patch_file from easybuild.tools.filetools import locate_files, read_file, resolve_path, which, write_file -from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO +from easybuild.tools.github import GITHUB_EASYCONFIGS_REPO, CategorizedPaths from easybuild.tools.github import det_pr_labels, det_pr_title, download_repo, fetch_easyconfigs_from_commit from easybuild.tools.github import fetch_easyconfigs_from_pr, fetch_pr_data from easybuild.tools.github import fetch_files_from_commit, fetch_files_from_pr @@ -617,37 +618,30 @@ def dump_env_script(easyconfigs): dump_env_easyblock(app, orig_env=orig_env, ec_path=ec.path, script_path=script_path) -def categorize_files_by_type(paths): +def categorize_files_by_type(paths: List[str]) -> CategorizedPaths: """ Splits list of filepaths into a 4 separate lists: easyconfigs, files to delete, patch files and files with extension .py """ - res = { - 'easyconfigs': [], - 'files_to_delete': [], - 'patch_files': [], - 'py_files': [], - } + res = CategorizedPaths([], [], [], []) for path in paths: if path.startswith(':'): - res['files_to_delete'].append(path[1:]) + res.files_to_delete.append(path[1:]) elif path.endswith('.py'): - res['py_files'].append(path) + res.py_files.append(path) # file must exist in order to check whether it's a patch file elif os.path.isfile(path) and is_patch_file(path): - res['patch_files'].append(path) + res.patch_files.append(path) elif path.endswith('.patch'): if not os.path.exists(path): raise EasyBuildError('File %s does not exist, did you mistype the path?', path) - elif not os.path.isfile(path): + if not os.path.isfile(path): raise EasyBuildError('File %s is expected to be a regular file, but is a folder instead', path) - else: - raise EasyBuildError('%s is not detected as a valid patch file. Please verify its contents!', - path) + raise EasyBuildError('%s is not detected as a valid patch file. Please verify its contents!', path) else: # anything else is considered to be an easyconfig file - res['easyconfigs'].append(path) + res.easyconfigs.append(path) return res diff --git a/easybuild/framework/easyconfig/types.py b/easybuild/framework/easyconfig/types.py index ad8624a154..97e11b60d0 100644 --- a/easybuild/framework/easyconfig/types.py +++ b/easybuild/framework/easyconfig/types.py @@ -103,7 +103,7 @@ def check_key_types(val, allowed_types): """Check whether type of keys for specific dict value are as expected.""" if isinstance(val, dict): res = True - for key in val.keys(): + for key in val: res &= any(is_value_of_type(key, t) for t in allowed_types) else: _log.debug("Specified value %s (type: %s) is not a dict, so key types check failed", val, type(val)) @@ -115,7 +115,7 @@ def check_key_types(val, allowed_types): def check_known_keys(val, allowed_keys): """Check whether all keys for specified dict value are known keys.""" if isinstance(val, dict): - res = all(key in allowed_keys for key in val.keys()) + res = all(key in allowed_keys for key in val) else: _log.debug("Specified value %s (type: %s) is not a dict, so known keys check failed", val, type(val)) res = False diff --git a/easybuild/main.py b/easybuild/main.py index dda45eb00d..22d14f2ee0 100755 --- a/easybuild/main.py +++ b/easybuild/main.py @@ -402,7 +402,7 @@ def process_eb_args(eb_args, eb_go, cfg_settings, modtool, testing, init_session no_ec_opts = [options.aggregate_regtest, options.regtest, any_pr_option_set, search_query] # determine paths to easyconfigs - determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs']) + determined_paths = det_easyconfig_paths(categorized_paths.easyconfigs) # only copy easyconfigs here if we're not using --try-* (that's handled below) copy_ec = options.copy_ec and not tweaked_ecs_paths diff --git a/easybuild/tools/containers/docker.py b/easybuild/tools/containers/docker.py index 274f45c7ce..358b5b4aad 100644 --- a/easybuild/tools/containers/docker.py +++ b/easybuild/tools/containers/docker.py @@ -150,7 +150,7 @@ def resolve_template_data(self): def validate(self): """Perform validation of specified container configuration.""" - if self.container_config not in DOCKER_OS_INSTALL_DEPS_TMPLS.keys(): + if self.container_config not in DOCKER_OS_INSTALL_DEPS_TMPLS: raise EasyBuildError("Unsupported container config '%s'" % self.container_config) super().validate() diff --git a/easybuild/tools/docs.py b/easybuild/tools/docs.py index 49c52865c1..1da2e35888 100644 --- a/easybuild/tools/docs.py +++ b/easybuild/tools/docs.py @@ -850,7 +850,7 @@ def list_software_md(software, detailed=True): ] # links to per-letter tables - key_letters = nub(sorted(k[0].lower() for k in software.keys())) + key_letters = nub(sorted(k[0].lower() for k in software)) letter_links = ' - '.join(['[' + x + '](#' + x + ')' for x in ascii_lowercase if x in key_letters]) lines.extend([letter_links, '']) @@ -922,7 +922,7 @@ def list_software_md(software, detailed=True): return '\n'.join(lines) -def list_software_rst(software, detailed=False): +def list_software_rst(software: dict, detailed=False): """ Return overview of supported software in RST format @@ -942,7 +942,7 @@ def list_software_rst(software, detailed=False): # links to per-letter tables letter_refs = '' - key_letters = nub(sorted(k[0].lower() for k in software.keys())) + key_letters = nub(sorted(k[0].lower() for k in software)) for letter in ascii_lowercase: if letter in key_letters: if letter_refs: @@ -1072,7 +1072,7 @@ def list_software_txt(software, detailed=False): return '\n'.join(lines) -def list_software_json(software, detailed=False): +def list_software_json(software: dict, detailed=False): """ Return overview of supported software in json @@ -1320,11 +1320,11 @@ def avail_toolchain_opts_json(name, tc_dict): raise NotImplementedError("JSON output not implemented yet for --avail-toolchain-opts") -def avail_toolchain_opts_txt(name, tc_dict): +def avail_toolchain_opts_txt(name, tc_dict: dict): """ Returns overview of toolchain options in txt format """ doc = ["Available options for %s toolchain:" % name] - for opt_name in sorted(tc_dict.keys()): - doc.append("%s%s: %s (default: %s)" % (INDENT_4SPACES, opt_name, tc_dict[opt_name][1], tc_dict[opt_name][0])) + for opt_name, value in sorted(tc_dict.items()): + doc.append("%s%s: %s (default: %s)" % (INDENT_4SPACES, opt_name, value[1], value[0])) return '\n'.join(doc) diff --git a/easybuild/tools/filetools.py b/easybuild/tools/filetools.py index 366f7b87fc..69fd9582b1 100644 --- a/easybuild/tools/filetools.py +++ b/easybuild/tools/filetools.py @@ -781,7 +781,7 @@ def parse_http_header_fields_urlpat(arg, urlpat=None, header=None, urlpat_header argline = header.strip() + ':' + argline if urlpat is not None: - if urlpat in urlpat_headers.keys(): + if urlpat in urlpat_headers: urlpat_headers[urlpat].append(argline) # add headers to the list else: urlpat_headers[urlpat] = [argline] # new list headers for this urlpat @@ -1618,9 +1618,9 @@ def create_patch_info(patch_spec): patch_info = {'name': patch_spec} elif isinstance(patch_spec, dict): patch_info = {} - for key in patch_spec.keys(): + for key, value in patch_spec.items(): if key in valid_keys: - patch_info[key] = patch_spec[key] + patch_info[key] = value else: raise EasyBuildError( "Wrong patch spec '%s', use of unknown key %s in dict (valid keys are %s)", diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index 9bb224537b..dfde885d8a 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -48,6 +48,7 @@ from http import HTTPStatus from http.client import HTTPException from string import ascii_letters +from typing import List, NamedTuple from urllib.request import HTTPError, URLError, urlopen from easybuild.base import fancylogger @@ -90,17 +91,25 @@ _log.warning("Failed to import 'git' Python module: %s", err) +class CategorizedPaths(NamedTuple): + easyconfigs: List[str] + easyconfigs: List[str] + files_to_delete: List[str] + patch_files: List[str] + py_files: List[str] + + GITHUB_URL = 'https://github.com' GITHUB_API_URL = 'https://api.github.com' GITHUB_BRANCH_MAIN = 'main' GITHUB_BRANCH_MASTER = 'master' -GITHUB_DIR_TYPE = u'dir' +GITHUB_DIR_TYPE = 'dir' GITHUB_EB_MAIN = 'easybuilders' GITHUB_EASYBLOCKS_REPO = 'easybuild-easyblocks' GITHUB_EASYCONFIGS_REPO = 'easybuild-easyconfigs' GITHUB_FRAMEWORK_REPO = 'easybuild-framework' GITHUB_DEVELOP_BRANCH = 'develop' -GITHUB_FILE_TYPE = u'file' +GITHUB_FILE_TYPE = 'file' GITHUB_PR_STATE_OPEN = 'open' GITHUB_PR_STATES = [GITHUB_PR_STATE_OPEN, 'closed', 'all'] GITHUB_PR_ORDER_CREATED = 'created' @@ -1052,7 +1061,8 @@ def setup_repo(git_repo, target_account, target_repo, branch_name, silent=False, @only_if_module_is_available('git', pkgname='GitPython') -def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_account=None, commit_msg=None): +def _easyconfigs_pr_common(paths: CategorizedPaths, ecs, start_branch=None, pr_branch=None, + start_account=None, commit_msg=None): """ Common code for new_pr and update_pr functions: * check whether all supplied paths point to existing files @@ -1072,8 +1082,8 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ # we need files to create the PR with non_existing_paths = [] ec_paths = [] - if paths['easyconfigs'] or paths['py_files']: - for path in paths['easyconfigs'] + paths['py_files']: + if paths.easyconfigs or paths.py_files: + for path in paths.easyconfigs + paths.py_files: if not os.path.exists(path): non_existing_paths.append(path) else: @@ -1085,7 +1095,7 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ exit_code=EasyBuildExit.OPTION_ERROR ) - if not any(paths.values()): + if not any(paths): raise EasyBuildError("No paths specified", exit_code=EasyBuildExit.OPTION_ERROR) pr_target_repo = det_pr_target_repo(paths) @@ -1139,7 +1149,7 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ # figure out commit message to use if commit_msg: - if (pr_target_repo == GITHUB_EASYCONFIGS_REPO and all(file_info['new']) and not paths['files_to_delete'] + if (pr_target_repo == GITHUB_EASYCONFIGS_REPO and all(file_info['new']) and not paths.files_to_delete and is_new_pr): # Only if opening a new PR msg = "When only adding new easyconfigs a PR commit msg (--pr-commit-msg) should not be used, as " msg += "the PR title will be automatically generated." @@ -1150,11 +1160,11 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ raise EasyBuildError(msg) cnt = len(file_info['paths_in_repo']) _log.debug("Using specified commit message for all %d new/modified files at once: %s", cnt, commit_msg) - elif pr_target_repo == GITHUB_EASYCONFIGS_REPO and all(file_info['new']) and not paths['files_to_delete']: + elif pr_target_repo == GITHUB_EASYCONFIGS_REPO and all(file_info['new']) and not paths.files_to_delete: # automagically derive meaningful commit message if all easyconfig files are new commit_msg = "adding easyconfigs: %s" % ', '.join(os.path.basename(p) for p in file_info['paths_in_repo']) - if paths['patch_files']: - commit_msg += " and patches: %s" % ', '.join(os.path.basename(p) for p in paths['patch_files']) + if paths.patch_files: + commit_msg += " and patches: %s" % ', '.join(os.path.basename(p) for p in paths.patch_files) elif pr_target_repo == GITHUB_EASYBLOCKS_REPO and all(file_info['new']): commit_msg = "adding easyblocks: %s" % ', '.join(os.path.basename(p) for p in file_info['paths_in_repo']) else: @@ -1163,22 +1173,22 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ if not new] if modified_files: msg += '\nModified: ' + ', '.join(modified_files) - if paths['files_to_delete']: - msg += '\nDeleted: ' + ', '.join(paths['files_to_delete']) + if paths.files_to_delete: + msg += '\nDeleted: ' + ', '.join(paths.files_to_delete) raise EasyBuildError("A meaningful commit message must be specified via --pr-commit-msg when " "modifying/deleting files or targeting the framework repo." + msg, exit_code=EasyBuildExit.OPTION_ERROR) # figure out to which software name patches relate, and copy them to the right place - if paths['patch_files']: - patch_specs = det_patch_specs(paths['patch_files'], file_info, [target_dir]) + if paths.patch_files: + patch_specs = det_patch_specs(paths.patch_files, file_info, [target_dir]) print_msg("copying patch files to %s..." % target_dir) patch_info = copy_patch_files(patch_specs, target_dir) # determine path to files to delete (if any) deleted_paths = [] - for fn in paths['files_to_delete']: + for fn in paths.files_to_delete: fullpath = os.path.join(repo_path, fn) if os.path.exists(fullpath): deleted_paths.append(fullpath) @@ -1217,8 +1227,8 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ if pr_branch is None: if ec_paths and pr_target_repo == GITHUB_EASYCONFIGS_REPO: label = file_info['ecs'][0].name + re.sub('[.-]', '', file_info['ecs'][0].version) - elif pr_target_repo == GITHUB_EASYBLOCKS_REPO and paths.get('py_files'): - label = os.path.splitext(os.path.basename(paths['py_files'][0]))[0] + elif pr_target_repo == GITHUB_EASYBLOCKS_REPO and paths.py_files: + label = os.path.splitext(os.path.basename(paths.py_files[0]))[0] else: label = ''.join(random.choice(ascii_letters) for _ in range(10)) pr_branch = '%s_new_pr_%s' % (time.strftime("%Y%m%d%H%M%S"), label) @@ -1233,7 +1243,7 @@ def _easyconfigs_pr_common(paths, ecs, start_branch=None, pr_branch=None, start_ git_repo.index.add(file_info['paths_in_repo']) git_repo.index.add(dep_info['paths_in_repo']) - if paths['patch_files']: + if paths.patch_files: _log.debug("Staging all %d new/modified patch files", len(patch_info['paths_in_repo'])) git_repo.index.add(patch_info['paths_in_repo']) @@ -1913,7 +1923,7 @@ def add_pr_labels(pr, branch=GITHUB_DEVELOP_BRANCH): @only_if_module_is_available('git', pkgname='GitPython') -def new_branch_github(paths, ecs, commit_msg=None): +def new_branch_github(paths: CategorizedPaths, ecs, commit_msg=None): """ Create new branch on GitHub using specified files @@ -1971,7 +1981,8 @@ def det_pr_title(ecs): @only_if_module_is_available('git', pkgname='GitPython') -def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None, pr_metadata=None, commit_msg=None): +def new_pr_from_branch(branch_name: CategorizedPaths, title=None, descr=None, + pr_target_repo=None, pr_metadata=None, commit_msg=None): """ Create new pull request from specified branch on GitHub. """ @@ -2152,7 +2163,7 @@ def new_pr_from_branch(branch_name, title=None, descr=None, pr_target_repo=None, print_msg("This PR should be labelled %s" % ', '.join(labels), log=_log, prefix=False) -def new_pr(paths, ecs, title=None, descr=None, commit_msg=None): +def new_pr(paths: CategorizedPaths, ecs, title=None, descr=None, commit_msg=None): """ Open new pull request using specified files @@ -2184,8 +2195,8 @@ def new_pr(paths, ecs, title=None, descr=None, commit_msg=None): raise EasyBuildError(msg, exit_code=EasyBuildExit.EASYCONFIG_ERROR) patch = patch_info['name'] - if patch not in paths['patch_files'] and not os.path.isfile(os.path.join(os.path.dirname(ec_path), - patch)): + if patch not in paths.patch_files and not os.path.isfile(os.path.join(os.path.dirname(ec_path), + patch)): print_warning("new patch file %s, referenced by %s, is not included in this PR" % (patch, ec.filename())) @@ -2226,7 +2237,7 @@ def det_account_branch_for_pr(pr_id, github_user=None, pr_target_repo=None): return account, branch -def det_pr_target_repo(paths): +def det_pr_target_repo(paths: CategorizedPaths): """Determine target repository for pull request from given cagetorized list of files :param paths: paths to categorized lists of files (easyconfigs, files to delete, patches, .py files) @@ -2234,21 +2245,18 @@ def det_pr_target_repo(paths): pr_target_repo = build_option('pr_target_repo') # determine target repository for PR based on which files are provided - # (see categorize_files_by_type function) if pr_target_repo is None: _log.info("Trying to derive target repository based on specified files...") - easyconfigs, files_to_delete, patch_files, py_files = [paths[key] for key in sorted(paths.keys())] - # Python files provided, and no easyconfig files or patches - if py_files and not (easyconfigs or patch_files): + if paths.py_files and not (paths.easyconfigs or paths.patch_files): _log.info("Only Python files provided, no easyconfig files or patches...") # if all Python files are easyblocks, target repo should be easyblocks; # otherwise, target repo is assumed to be framework - if all(get_easyblock_class_name(path) for path in py_files): + if all(get_easyblock_class_name(path) for path in paths.py_files): pr_target_repo = GITHUB_EASYBLOCKS_REPO _log.info("All Python files are easyblocks, target repository is assumed to be %s", pr_target_repo) else: @@ -2257,7 +2265,8 @@ def det_pr_target_repo(paths): # if no Python files are provided, only easyconfigs & patches, or if files to delete are .eb files, # then target repo is assumed to be easyconfigs - elif easyconfigs or patch_files or (files_to_delete and all(x.endswith('.eb') for x in files_to_delete)): + elif paths.easyconfigs or paths.patch_files or (paths.files_to_delete and all(x.endswith('.eb') + for x in paths.files_to_delete)): pr_target_repo = GITHUB_EASYCONFIGS_REPO _log.info("Only easyconfig and patch files found, target repository is assumed to be %s", pr_target_repo) @@ -2268,7 +2277,7 @@ def det_pr_target_repo(paths): @only_if_module_is_available('git', pkgname='GitPython') -def update_branch(branch_name, paths, ecs, github_account=None, commit_msg=None): +def update_branch(branch_name, paths: CategorizedPaths, ecs, github_account=None, commit_msg=None): """ Update specified branch in GitHub using specified files @@ -2303,7 +2312,7 @@ def update_branch(branch_name, paths, ecs, github_account=None, commit_msg=None) @only_if_module_is_available('git', pkgname='GitPython') -def update_pr(pr_id, paths, ecs, commit_msg=None): +def update_pr(pr_id, paths: CategorizedPaths, ecs, commit_msg=None): """ Update specified pull request using specified files diff --git a/easybuild/tools/module_naming_scheme/hierarchical_mns.py b/easybuild/tools/module_naming_scheme/hierarchical_mns.py index a562973393..18230da7b6 100644 --- a/easybuild/tools/module_naming_scheme/hierarchical_mns.py +++ b/easybuild/tools/module_naming_scheme/hierarchical_mns.py @@ -199,7 +199,7 @@ def det_modpath_extensions(self, ec): extend_comps = [] # exclude GCC for which / is used as $MODULEPATH extension excluded_comps = ['GCC'] - for comps in COMP_NAME_VERSION_TEMPLATES.keys(): + for comps in COMP_NAME_VERSION_TEMPLATES: extend_comps.extend([comp for comp in comps.split(',') if comp not in excluded_comps]) comp_name_ver = None diff --git a/easybuild/tools/modules.py b/easybuild/tools/modules.py index 41a4174650..4e72ec6451 100644 --- a/easybuild/tools/modules.py +++ b/easybuild/tools/modules.py @@ -2303,7 +2303,7 @@ def invalidate_module_caches_for(path): _log.debug("Invallidating module cache entries for path '%s'", path) for cache, subcmd in [(MODULE_AVAIL_CACHE, 'avail'), (MODULE_SHOW_CACHE, 'show')]: - for key in list(cache.keys()): + for key in list(cache): paths_in_key = '='.join(key[0].split('=')[1:]).split(os.pathsep) _log.debug("Paths for 'module %s' key '%s': %s", subcmd, key, paths_in_key) for path_in_key in paths_in_key: diff --git a/easybuild/tools/options.py b/easybuild/tools/options.py index 96bb8a484a..a052a6a266 100644 --- a/easybuild/tools/options.py +++ b/easybuild/tools/options.py @@ -2126,7 +2126,7 @@ def parse_external_modules_metadata(cfgs): # make sure name/version values are always lists, make sure they're equal length for mod, entry in parsed_metadata.items(): # make sure only known keys are used - for key in entry.keys(): + for key in entry: if key not in known_metadata_keys: unknown_keys.setdefault(mod, []).append(key) @@ -2147,8 +2147,8 @@ def parse_external_modules_metadata(cfgs): if unknown_keys: error_msg = "Found metadata entries with unknown keys:" - for mod in sorted(unknown_keys.keys()): - error_msg += "\n* %s: %s" % (mod, ', '.join(sorted(unknown_keys[mod]))) + for mod, keys in sorted(unknown_keys.items()): + error_msg += "\n* %s: %s" % (mod, ', '.join(sorted(keys))) raise EasyBuildError(error_msg, exit_code=EasyBuildExit.MODULE_ERROR) _log.debug("External modules metadata: %s", parsed_metadata) diff --git a/easybuild/tools/systemtools.py b/easybuild/tools/systemtools.py index c2cf7e3a16..9ce43e80d8 100644 --- a/easybuild/tools/systemtools.py +++ b/easybuild/tools/systemtools.py @@ -1491,7 +1491,7 @@ def pick_system_specific_value(description, options_or_value, allow_none=False): if isinstance(options_or_value, dict): if not options_or_value: raise EasyBuildError("Found empty dict as %s!", description) - other_keys = [x for x in options_or_value.keys() if not x.startswith(ARCH_KEY_PREFIX)] + other_keys = [x for x in options_or_value if not x.startswith(ARCH_KEY_PREFIX)] if other_keys: other_keys = ','.join(sorted(other_keys)) raise EasyBuildError("Unexpected keys in %s: %s (only '%s' keys are supported)", diff --git a/easybuild/tools/testing.py b/easybuild/tools/testing.py index 708932f8d8..d441b8ca5e 100644 --- a/easybuild/tools/testing.py +++ b/easybuild/tools/testing.py @@ -294,8 +294,8 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nrs=None, gist_ "```", ] + eb_config + ["````", ""]) - system_info = init_session_state['system_info'] - system_info = [" * _%s:_ %s" % (key.replace('_', ' '), system_info[key]) for key in sorted(system_info.keys())] + system_info_dict: dict = init_session_state['system_info'] + system_info = [" * _%s:_ %s" % (key.replace('_', ' '), value) for key, value in sorted(system_info_dict.items())] test_report.extend(["#### System info"] + system_info + [""]) module_list = init_session_state['module_list'] @@ -309,12 +309,11 @@ def create_test_report(msg, ecs_with_res, init_session_state, pr_nrs=None, gist_ environment = [] env_filter = build_option('test_report_env_filter') - for key in sorted(environ_dump.keys()): + for key, value in sorted(environ_dump.items()): if env_filter is not None and env_filter.search(key): continue if any(x in key.upper() for x in DEFAULT_EXCLUDE_FROM_TEST_REPORT_ENV_VAR_NAMES): continue - value = environ_dump[key] if any(re.match(rgx, value) for rgx in DEFAULT_EXCLUDE_FROM_TEST_REPORT_VALUE_REGEX): continue environment += ["%s = %s" % (key, value)] diff --git a/easybuild/tools/toolchain/compiler.py b/easybuild/tools/toolchain/compiler.py index 32a7960c1f..fa7da0d1af 100644 --- a/easybuild/tools/toolchain/compiler.py +++ b/easybuild/tools/toolchain/compiler.py @@ -280,9 +280,9 @@ def _set_compiler_flags(self): # not passing that flag. Convert the passed options to a dict for further handling if not isinstance(openmpoptions, dict): openmpoptions = {False: '', True: openmpoptions} - if self.options['openmp'] not in openmpoptions.keys(): + if self.options['openmp'] not in openmpoptions: raise EasyBuildError(f"Unknown value for openmp: {self.options['openmp']} " - f"(possibilities are {', '.join(str(key) for key in openmpoptions.keys())}).") + f"(possibilities are {', '.join(str(key) for key in openmpoptions)}).") openmpflags = openmpoptions[self.options['openmp']] # avoid double use of such flags, or e.g. -hnomp followed by -homp if isinstance(optflags[0], list): @@ -294,9 +294,9 @@ def _set_compiler_flags(self): # vectorization is disabled for noopt and lowopt, and enabled otherwise. if self.options.get('vectorize') is not None: vectoptions = self.options.option('vectorize') - if self.options['vectorize'] not in vectoptions.keys(): + if self.options['vectorize'] not in vectoptions: raise EasyBuildError(f"Unknown value for 'vectorize': {self.options['vectorize']} " - f"(possibilities are {', '.join(str(key) for key in vectoptions.keys())}).") + f"(possibilities are {', '.join(str(key) for key in vectoptions)}).") vectflags = vectoptions[self.options['vectorize']] # avoid double use of such flags, or e.g. -fno-tree-vectorize followed by -ftree-vectorize if isinstance(optflags[0], list): diff --git a/easybuild/tools/toolchain/mpi.py b/easybuild/tools/toolchain/mpi.py index 60be387d34..570bd55445 100644 --- a/easybuild/tools/toolchain/mpi.py +++ b/easybuild/tools/toolchain/mpi.py @@ -48,7 +48,7 @@ _log = fancylogger.getLogger('tools.toolchain.mpi', fname=False) -def get_mpi_cmd_template(mpi_family, params, mpi_version=None): +def get_mpi_cmd_template(mpi_family, params: dict, mpi_version=None): """ Return template for MPI command, for specified MPI family. @@ -125,7 +125,7 @@ def get_mpi_cmd_template(mpi_family, params, mpi_version=None): raise EasyBuildError("Don't know which template MPI command to use for MPI family '%s'", mpi_family) missing = [] - for key in sorted(params.keys()): + for key in sorted(params): tmpl = '%(' + key + ')s' if tmpl not in mpi_cmd_template: missing.append(tmpl) diff --git a/easybuild/tools/toolchain/options.py b/easybuild/tools/toolchain/options.py index 9b2c881063..9147651c29 100644 --- a/easybuild/tools/toolchain/options.py +++ b/easybuild/tools/toolchain/options.py @@ -67,14 +67,14 @@ def _add_options(self, options): raise EasyBuildError("_add_options: option name %s has to be 2 element list (%s)", name, value) if name in self: self.log.devel("_add_options: redefining previous name %s (previous value %s)", name, self.get(name)) - self.__setitem__(name, value[0]) - self.description.__setitem__(name, value[1]) + self[name] = value[0] + self.description[name] = value[1] - def _add_options_map(self, options_map): + def _add_options_map(self, options_map: dict): """Add map dict between options and values map names starting with _opt_ are allowed without corresponding option """ - for name in options_map.keys(): + for name in options_map: if name not in self: if name.startswith('_opt_'): self.log.devel("_add_options_map: no option with name %s defined, but allowed", name) diff --git a/easybuild/tools/toolchain/toolchain.py b/easybuild/tools/toolchain/toolchain.py index f4ce06b334..edb1cae502 100644 --- a/easybuild/tools/toolchain/toolchain.py +++ b/easybuild/tools/toolchain/toolchain.py @@ -431,7 +431,7 @@ def show_variables(self, offset='', sep='\n', verbose=False): if verbose: res.append("# type %s" % (type(self.variables[v]))) res.append("# %s" % (self.variables[v].show_el())) - res.append("# repr %s" % (self.variables[v].__repr__())) + res.append("# repr %s" % repr(self.variables[v])) if offset is None: offset = '' @@ -521,15 +521,14 @@ def _toolchain_exists(self): # check whether a matching module exists if self.mod_short_name contains a module name return self.modules_tool.exist([self.mod_full_name], skip_avail=True)[0] - def set_options(self, options): + def set_options(self, options: dict): """ Process toolchain options """ - for opt in options.keys(): + for opt in options: # Only process supported opts if opt in self.options: self.options[opt] = options[opt] else: - # used to be warning, but this is a severe error imho - known_opts = ','.join(self.options.keys()) + known_opts = ','.join(self.options) raise EasyBuildError("Undefined toolchain option %s specified (known options: %s)", opt, known_opts) def get_dependency_version(self, dependency): diff --git a/test/framework/easyblock.py b/test/framework/easyblock.py index fef50b14fe..eef9909069 100644 --- a/test/framework/easyblock.py +++ b/test/framework/easyblock.py @@ -98,9 +98,9 @@ def check_extra_options_format(extra_options): extra_options.items() extra_options.keys() extra_options.values() - for key in extra_options.keys(): - self.assertIsInstance(extra_options[key], list) - self.assertEqual(len(extra_options[key]), 3) + for opt in extra_options.values(): + self.assertIsInstance(opt, list) + self.assertEqual(len(opt), 3) name = "pi" version = "3.14" diff --git a/test/framework/easyconfig.py b/test/framework/easyconfig.py index e08465e05c..089279bd7e 100644 --- a/test/framework/easyconfig.py +++ b/test/framework/easyconfig.py @@ -60,7 +60,7 @@ from easybuild.framework.easyconfig.style import check_easyconfigs_style from easybuild.framework.easyconfig.tools import alt_easyconfig_paths, categorize_files_by_type, check_sha256_checksums from easybuild.framework.easyconfig.tools import dep_graph, det_copy_ec_specs, find_related_easyconfigs, get_paths_for -from easybuild.framework.easyconfig.tools import parse_easyconfigs +from easybuild.framework.easyconfig.tools import parse_easyconfigs, CategorizedPaths from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak, tweak_one from easybuild.framework.extension import construct_exts_filter_cmds from easybuild.toolchains.system import SystemToolchain @@ -3876,8 +3876,7 @@ def test_hidden_toolchain(self): def test_categorize_files_by_type(self): """Test categorize_files_by_type""" - self.assertEqual({'easyconfigs': [], 'files_to_delete': [], 'patch_files': [], 'py_files': []}, - categorize_files_by_type([])) + self.assertEqual(categorize_files_by_type([]), CategorizedPaths([], [], [], [])) test_dir = os.path.dirname(os.path.abspath(__file__)) test_ecs_dir = os.path.join(test_dir, 'easyconfigs') @@ -3904,10 +3903,10 @@ def test_categorize_files_by_type(self): gzip_ec, 'foo', ] - self.assertEqual(res['easyconfigs'], expected) - self.assertEqual(res['files_to_delete'], ['toy-0.0-deps.eb']) - self.assertEqual(res['patch_files'], [toy_patch]) - self.assertEqual(res['py_files'], [toy_easyblock, configuremake]) + self.assertEqual(res.easyconfigs, expected) + self.assertEqual(res.files_to_delete, ['toy-0.0-deps.eb']) + self.assertEqual(res.patch_files, [toy_patch]) + self.assertEqual(res.py_files, [toy_easyblock, configuremake]) # Error cases tmpdir = tempfile.mkdtemp() diff --git a/test/framework/environment.py b/test/framework/environment.py index 64467a9d3b..d1a5215ce8 100644 --- a/test/framework/environment.py +++ b/test/framework/environment.py @@ -91,8 +91,8 @@ def test_modify_env(self): # keys in new_env should not be set yet, keys in old_env are expected to be set for key in new_env_vars: os.environ.pop(key, None) - for key in old_env_vars: - os.environ[key] = old_env_vars[key] + for key, value in old_env_vars.items(): + os.environ[key] = value env.modify_env(os.environ, new_env_vars) @@ -147,7 +147,7 @@ def test_sanitize_env(self): env.sanitize_env() - self.assertFalse(any(x for x in os.environ.keys() if x.startswith('PYTHON'))) + self.assertFalse(any(x for x in os.environ if x.startswith('PYTHON'))) expected = { 'CPATH': self.test_prefix, diff --git a/test/framework/options.py b/test/framework/options.py index 63c923581f..cc45478bcb 100644 --- a/test/framework/options.py +++ b/test/framework/options.py @@ -70,7 +70,7 @@ from test.framework.github import ignore_rate_limit_in_pr try: - import pycodestyle # noqa + import pycodestyle # noqa pylint:disable=unused-import except ImportError: pass @@ -2642,7 +2642,7 @@ def test_deprecated(self): def test_allow_modules_tool_mismatch(self): """Test allowing mismatch of modules tool with 'module' function.""" # make sure MockModulesTool is available - from test.framework.modulestool import MockModulesTool # noqa + from test.framework.modulestool import MockModulesTool # noqa pylint:disable=unused-import # trigger that main() creates new instance of ModulesTool self.modtool = None @@ -4014,9 +4014,9 @@ def test_github_xxx_include_easyblocks_from_pr(self): del sys.modules['easybuild.easyblocks.llvm'] del sys.modules['easybuild.easyblocks.generic.cmakemake'] sys.path[:] = orig_local_sys_path - import easybuild.easyblocks + import easybuild.easyblocks # pylint:disable=reimported reload(easybuild.easyblocks) - import easybuild.easyblocks.generic + import easybuild.easyblocks.generic # pylint:disable=reimported reload(easybuild.easyblocks.generic) def mk_eb_test_cmd(self, args): diff --git a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py index 4ba99733eb..e6ce117973 100644 --- a/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py +++ b/test/framework/sandbox/easybuild/tools/module_naming_scheme/test_module_naming_scheme_more.py @@ -61,7 +61,7 @@ def det_full_module_name(self, ec): for key in self.REQUIRED_KEYS: if isinstance(ec[key], dict): res += '%s=>' % key - for item_key in sorted(ec[key].keys()): + for item_key in sorted(ec[key]): res += '%s:%s,' % (item_key, ec[key][item_key]) else: res += str(ec[key]) diff --git a/test/framework/tweak.py b/test/framework/tweak.py index 48e6f48935..cf59d22caf 100644 --- a/test/framework/tweak.py +++ b/test/framework/tweak.py @@ -155,7 +155,7 @@ def test_tweak_one_version(self): # checksums should be reset to empty list, only version should be changed, nothing else self.assertEqual(tweaked_toy_ec_parsed['checksums'], []) self.assertEqual(tweaked_toy_ec_parsed['version'], '1.2.3') - for key in [k for k in toy_ec_parsed.keys() if k not in ['checksums', 'version']]: + for key in [k for k in toy_ec_parsed if k not in ['checksums', 'version']]: val = toy_ec_parsed[key] self.assertIn(key, tweaked_toy_ec_parsed, "Parameter '%s' not defined in tweaked easyconfig file" % key) tweaked_val = tweaked_toy_ec_parsed.get(key)