Cleanup use of dicts, e.g. use of .keys()

Using `x in dict.keys()` is discouraged and should be `x in dict`
directly which is potentially more performant and easier to read/shorter.
Accesses to keys and their values should use `dict.items` to avoid the
lookup overhead.

Author: Flamefire

easybuild/base/frozendict.py

@@ -76,7 +76,7 @@ def __init__(self, *args, **kwargs):
 
         # handle unknown keys: either ignore them or raise an exception
         tmpdict = dict(*args, **kwargs)
-        unknown_keys = [key for key in tmpdict.keys() if key not in self.KNOWN_KEYS]
+        unknown_keys = [key for key in tmpdict if key not in self.KNOWN_KEYS]
         if unknown_keys:
             if ignore_unknown_keys:
                 for key in unknown_keys:

easybuild/base/generaloption.py

@@ -1023,7 +1023,7 @@ def _make_debug_options(self):
 
     def _set_default_loglevel(self):
         """Set the default loglevel if no logging options are set"""
-        loglevel_set = sum([getattr(self.options, name, False) for name in self._logopts.keys()])
+        loglevel_set = sum([getattr(self.options, name, False) for name in self._logopts])
         if not loglevel_set and self.DEFAULT_LOGLEVEL is not None:
             setLogLevel(self.DEFAULT_LOGLEVEL)
 

easybuild/base/optcomplete.py

@@ -169,7 +169,7 @@ def __call__(self, **kwargs):
         if self.CALL_ARGS_OPTIONAL is not None:
             all_args.extend(self.CALL_ARGS_OPTIONAL)
 
-        for arg in kwargs.keys():
+        for arg in kwargs:
             if arg not in all_args:
                 # remove it
                 kwargs.pop(arg)

easybuild/framework/easyconfig/easyconfig.py

@@ -41,6 +41,7 @@
 * Victor Holanda (CSCS, ETH Zurich)
 """
 
+import contextlib
 import copy
 import difflib
 import functools
@@ -199,8 +200,7 @@ def triage_easyconfig_params(variables, ec):
 
     for key in variables:
         # validations are skipped, just set in the config
-        if any(key in d for d in (ec, DEPRECATED_EASYCONFIG_PARAMETERS.keys(),
-                                  ALTERNATIVE_EASYCONFIG_PARAMETERS.keys())):
+        if any(key in d for d in (ec, DEPRECATED_EASYCONFIG_PARAMETERS, ALTERNATIVE_EASYCONFIG_PARAMETERS)):
             ec_params[key] = variables[key]
             _log.debug("setting config option %s: value %s (type: %s)", key, ec_params[key], type(ec_params[key]))
         elif key in REPLACED_PARAMETERS:
@@ -728,7 +728,7 @@ def update(self, key, value, allow_duplicate=True):
         # Overwrite easyconfig parameter value with updated value, preserving type
         self[key] = param_value
 
-    def set_keys(self, params):
+    def set_keys(self, params: dict):
         """
         Set keys in this EasyConfig instance based on supplied easyconfig parameter values.
 
@@ -739,15 +739,15 @@ def set_keys(self, params):
         # disable templating when setting easyconfig parameters
         # required to avoid problems with values that need more parsing to be done (e.g. dependencies)
         with self.disable_templating():
-            for key in sorted(params.keys()):
+            for key, value in sorted(params.items()):
                 # validations are skipped, just set in the config
-                if any(key in x.keys() for x in (self._config, ALTERNATIVE_EASYCONFIG_PARAMETERS,
-                                                 DEPRECATED_EASYCONFIG_PARAMETERS)):
-                    self[key] = params[key]
+                if any(key in x for x in (self._config, ALTERNATIVE_EASYCONFIG_PARAMETERS,
+                                          DEPRECATED_EASYCONFIG_PARAMETERS)):
+                    self[key] = value
                     self.log.info("setting easyconfig parameter %s: value %s (type: %s)",
                                   key, self[key], type(self[key]))
                 else:
-                    raise EasyBuildError("Unknown easyconfig parameter: %s (value '%s')", key, params[key])
+                    raise EasyBuildError("Unknown easyconfig parameter: %s (value '%s')", key, value)
 
     def parse(self):
         """
@@ -2202,15 +2202,18 @@ def resolve_template(value, tmpl_dict, expect_resolved=True):
                 try:
                     orig_value = value
                     # map old templates to new values for alternative and deprecated templates
-                    alt_map = {old_tmpl: tmpl_dict[new_tmpl] for (old_tmpl, new_tmpl) in
-                               ALTERNATIVE_EASYCONFIG_TEMPLATES.items() if new_tmpl in tmpl_dict.keys()}
-                    alt_map2 = {new_tmpl: tmpl_dict[old_tmpl] for (old_tmpl, new_tmpl) in
-                                ALTERNATIVE_EASYCONFIG_TEMPLATES.items() if old_tmpl in tmpl_dict.keys()}
-                    depr_map = {old_tmpl: tmpl_dict[new_tmpl] for (old_tmpl, (new_tmpl, ver)) in
-                                DEPRECATED_EASYCONFIG_TEMPLATES.items() if new_tmpl in tmpl_dict.keys()}
+                    new_tmpl_dict = tmpl_dict.copy()
+                    with contextlib.suppress(KeyError):
+                        new_tmpl_dict.update({old_tmpl: tmpl_dict[new_tmpl]
+                                              for (old_tmpl, new_tmpl) in ALTERNATIVE_EASYCONFIG_TEMPLATES.items()})
+                        new_tmpl_dict.update({new_tmpl: tmpl_dict[old_tmpl]
+                                              for (old_tmpl, new_tmpl) in ALTERNATIVE_EASYCONFIG_TEMPLATES.items()})
+                        depr_map = {old_tmpl: tmpl_dict[new_tmpl]
+                                    for (old_tmpl, new_tmpl) in DEPRECATED_EASYCONFIG_TEMPLATES.items()}
+                        new_tmpl_dict.update(depr_map)
 
                     # try templating with alternative and deprecated templates included
-                    value = value % {**tmpl_dict, **alt_map, **alt_map2, **depr_map}
+                    value = value % new_tmpl_dict
 
                     for old_tmpl, val in depr_map.items():
                         # check which deprecated templates were replaced, and issue deprecation warnings

easybuild/framework/easyconfig/types.py

@@ -103,7 +103,7 @@ def check_key_types(val, allowed_types):
     """Check whether type of keys for specific dict value are as expected."""
     if isinstance(val, dict):
         res = True
-        for key in val.keys():
+        for key in val:
             res &= any(is_value_of_type(key, t) for t in allowed_types)
     else:
         _log.debug("Specified value %s (type: %s) is not a dict, so key types check failed", val, type(val))
@@ -115,7 +115,7 @@ def check_key_types(val, allowed_types):
 def check_known_keys(val, allowed_keys):
     """Check whether all keys for specified dict value are known keys."""
     if isinstance(val, dict):
-        res = all(key in allowed_keys for key in val.keys())
+        res = all(key in allowed_keys for key in val)
     else:
         _log.debug("Specified value %s (type: %s) is not a dict, so known keys check failed", val, type(val))
         res = False

easybuild/tools/containers/docker.py

@@ -150,7 +150,7 @@ def resolve_template_data(self):
 
     def validate(self):
         """Perform validation of specified container configuration."""
-        if self.container_config not in DOCKER_OS_INSTALL_DEPS_TMPLS.keys():
+        if self.container_config not in DOCKER_OS_INSTALL_DEPS_TMPLS:
             raise EasyBuildError("Unsupported container config '%s'" % self.container_config)
         super().validate()
 

easybuild/tools/docs.py

@@ -850,7 +850,7 @@ def list_software_md(software, detailed=True):
     ]
 
     # links to per-letter tables
-    key_letters = nub(sorted(k[0].lower() for k in software.keys()))
+    key_letters = nub(sorted(k[0].lower() for k in software))
     letter_links = ' - '.join(['[' + x + '](#' + x + ')' for x in ascii_lowercase if x in key_letters])
     lines.extend([letter_links, ''])
 
@@ -922,7 +922,7 @@ def list_software_md(software, detailed=True):
     return '\n'.join(lines)
 
 
-def list_software_rst(software, detailed=False):
+def list_software_rst(software: dict, detailed=False):
     """
     Return overview of supported software in RST format
 
@@ -942,7 +942,7 @@ def list_software_rst(software, detailed=False):
 
     # links to per-letter tables
     letter_refs = ''
-    key_letters = nub(sorted(k[0].lower() for k in software.keys()))
+    key_letters = nub(sorted(k[0].lower() for k in software))
     for letter in ascii_lowercase:
         if letter in key_letters:
             if letter_refs:
@@ -1072,7 +1072,7 @@ def list_software_txt(software, detailed=False):
     return '\n'.join(lines)
 
 
-def list_software_json(software, detailed=False):
+def list_software_json(software: dict, detailed=False):
     """
     Return overview of supported software in json
 
@@ -1320,11 +1320,11 @@ def avail_toolchain_opts_json(name, tc_dict):
     raise NotImplementedError("JSON output not implemented yet for --avail-toolchain-opts")
 
 
-def avail_toolchain_opts_txt(name, tc_dict):
+def avail_toolchain_opts_txt(name, tc_dict: dict):
     """ Returns overview of toolchain options in txt format """
     doc = ["Available options for %s toolchain:" % name]
-    for opt_name in sorted(tc_dict.keys()):
-        doc.append("%s%s: %s (default: %s)" % (INDENT_4SPACES, opt_name, tc_dict[opt_name][1], tc_dict[opt_name][0]))
+    for opt_name, value in sorted(tc_dict.items()):
+        doc.append("%s%s: %s (default: %s)" % (INDENT_4SPACES, opt_name, value[1], value[0]))
 
     return '\n'.join(doc)
 

easybuild/tools/filetools.py

@@ -781,7 +781,7 @@ def parse_http_header_fields_urlpat(arg, urlpat=None, header=None, urlpat_header
             argline = header.strip() + ':' + argline
 
         if urlpat is not None:
-            if urlpat in urlpat_headers.keys():
+            if urlpat in urlpat_headers:
                 urlpat_headers[urlpat].append(argline)  # add headers to the list
             else:
                 urlpat_headers[urlpat] = [argline]  # new list headers for this urlpat
@@ -1618,9 +1618,9 @@ def create_patch_info(patch_spec):
         patch_info = {'name': patch_spec}
     elif isinstance(patch_spec, dict):
         patch_info = {}
-        for key in patch_spec.keys():
+        for key, value in patch_spec.items():
             if key in valid_keys:
-                patch_info[key] = patch_spec[key]
+                patch_info[key] = value
             else:
                 raise EasyBuildError(
                     "Wrong patch spec '%s', use of unknown key %s in dict (valid keys are %s)",

easybuild/tools/github.py

@@ -94,13 +94,13 @@
 GITHUB_API_URL = 'https://api.github.com'
 GITHUB_BRANCH_MAIN = 'main'
 GITHUB_BRANCH_MASTER = 'master'
-GITHUB_DIR_TYPE = u'dir'
+GITHUB_DIR_TYPE = 'dir'
 GITHUB_EB_MAIN = 'easybuilders'
 GITHUB_EASYBLOCKS_REPO = 'easybuild-easyblocks'
 GITHUB_EASYCONFIGS_REPO = 'easybuild-easyconfigs'
 GITHUB_FRAMEWORK_REPO = 'easybuild-framework'
 GITHUB_DEVELOP_BRANCH = 'develop'
-GITHUB_FILE_TYPE = u'file'
+GITHUB_FILE_TYPE = 'file'
 GITHUB_PR_STATE_OPEN = 'open'
 GITHUB_PR_STATES = [GITHUB_PR_STATE_OPEN, 'closed', 'all']
 GITHUB_PR_ORDER_CREATED = 'created'
@@ -2226,7 +2226,7 @@ def det_account_branch_for_pr(pr_id, github_user=None, pr_target_repo=None):
     return account, branch
 
 
-def det_pr_target_repo(paths):
+def det_pr_target_repo(paths: dict):
     """Determine target repository for pull request from given cagetorized list of files
 
     :param paths: paths to categorized lists of files (easyconfigs, files to delete, patches, .py files)
@@ -2239,7 +2239,7 @@ def det_pr_target_repo(paths):
 
         _log.info("Trying to derive target repository based on specified files...")
 
-        easyconfigs, files_to_delete, patch_files, py_files = [paths[key] for key in sorted(paths.keys())]
+        easyconfigs, files_to_delete, patch_files, py_files = sorted(paths.items())
 
         # Python files provided, and no easyconfig files or patches
         if py_files and not (easyconfigs or patch_files):

easybuild/tools/module_naming_scheme/hierarchical_mns.py

@@ -199,7 +199,7 @@ def det_modpath_extensions(self, ec):
             extend_comps = []
             # exclude GCC for which <name>/<version> is used as $MODULEPATH extension
             excluded_comps = ['GCC']
-            for comps in COMP_NAME_VERSION_TEMPLATES.keys():
+            for comps in COMP_NAME_VERSION_TEMPLATES:
                 extend_comps.extend([comp for comp in comps.split(',') if comp not in excluded_comps])
 
             comp_name_ver = None