diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ea0606d3..ab2a02a3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,6 +53,7 @@ jobs: - run: coverage run --source=dfetch --append -m behave features # Run features tests - run: coverage xml -o coverage.xml # Create XML report - run: pyroma --directory --min=10 . # Check pyproject + - run: find dfetch -name "*.py" | xargs pyupgrade --py39-plus # Check syntax - name: Run codacy-coverage-reporter uses: codacy/codacy-coverage-reporter-action@a38818475bb21847788496e9f0fddaa4e84955ba # master diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 747bad5b..bae04b9a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -113,6 +113,15 @@ repos: entry: pyright language: python types: [file, python] + - id: pyupgrade + name: pyupgrade + description: Modernize python + entry: pyupgrade + language: python + files: ^dfetch/ + args: [--py39-plus] + types: [file, python] + - repo: https://github.com/gitleaks/gitleaks rev: v8.16.3 hooks: diff --git a/dfetch/__main__.py b/dfetch/__main__.py index 7938a821..824cf355 100644 --- a/dfetch/__main__.py +++ b/dfetch/__main__.py @@ -5,7 +5,7 @@ import argparse import sys -from typing import Sequence +from collections.abc import Sequence import dfetch.commands.check import dfetch.commands.diff diff --git a/dfetch/commands/check.py b/dfetch/commands/check.py index 4f67fd4b..85c27947 100644 --- a/dfetch/commands/check.py +++ b/dfetch/commands/check.py @@ -21,7 +21,6 @@ import argparse import os -from typing import List import dfetch.commands.command import dfetch.manifest.manifest @@ -88,7 +87,7 @@ def __call__(self, args: argparse.Namespace) -> None: reporters = self._get_reporters(args, manifest) with in_directory(os.path.dirname(manifest.path)): - exceptions: List[str] = [] + exceptions: list[str] = [] for project in manifest.selected_projects(args.projects): with catch_runtime_exceptions(exceptions) as exceptions: dfetch.project.make(project).check_for_update(reporters) @@ -106,7 +105,7 @@ def __call__(self, args: argparse.Namespace) -> None: @staticmethod def _get_reporters( args: argparse.Namespace, manifest: Manifest - ) -> List[CheckReporter]: + ) -> list[CheckReporter]: """Get all reporters. Args: @@ -116,7 +115,7 @@ def _get_reporters( Returns: List[CheckReporter]: List of reporters that each provide a unique report """ - reporters: List[CheckReporter] = [CheckStdoutReporter(manifest)] + reporters: list[CheckReporter] = [CheckStdoutReporter(manifest)] if args.jenkins_json: reporters += [JenkinsReporter(manifest, args.jenkins_json)] if args.sarif: diff --git a/dfetch/commands/command.py b/dfetch/commands/command.py index 7f7729c9..b2e6d1cd 100644 --- a/dfetch/commands/command.py +++ b/dfetch/commands/command.py @@ -4,7 +4,7 @@ import sys from abc import ABC, abstractmethod from argparse import ArgumentParser # pylint: disable=unused-import -from typing import TYPE_CHECKING, Type, TypeVar +from typing import TYPE_CHECKING, TypeVar if TYPE_CHECKING and sys.version_info >= (3, 10): from typing import TypeAlias @@ -56,7 +56,7 @@ def __call__(self, args: argparse.Namespace) -> None: @staticmethod def parser( subparsers: SubparserActionType, - command: Type["Command.CHILD_TYPE"], + command: type["Command.CHILD_TYPE"], ) -> "argparse.ArgumentParser": """Generate the parser. diff --git a/dfetch/commands/common.py b/dfetch/commands/common.py index eafbd712..d73f88a8 100644 --- a/dfetch/commands/common.py +++ b/dfetch/commands/common.py @@ -1,7 +1,6 @@ """Module for common command operations.""" import os -from typing import List import yaml @@ -20,7 +19,7 @@ def check_child_manifests(manifest: Manifest, project: ProjectEntry) -> None: project (ProjectEntry): The parent project. """ for childmanifest in get_childmanifests(skip=[manifest.path]): - recommendations: List[ProjectEntry] = [] + recommendations: list[ProjectEntry] = [] for childproject in childmanifest.projects: if childproject.remote_url not in [ project.remote_url for project in manifest.projects @@ -35,7 +34,7 @@ def check_child_manifests(manifest: Manifest, project: ProjectEntry) -> None: def _make_recommendation( - project: ProjectEntry, recommendations: List[ProjectEntry], childmanifest_path: str + project: ProjectEntry, recommendations: list[ProjectEntry], childmanifest_path: str ) -> None: """Make recommendations to the user. diff --git a/dfetch/commands/diff.py b/dfetch/commands/diff.py index d0e1586b..a802ac87 100644 --- a/dfetch/commands/diff.py +++ b/dfetch/commands/diff.py @@ -52,7 +52,6 @@ import argparse import os -from typing import List import dfetch.commands.command import dfetch.manifest.manifest @@ -104,7 +103,7 @@ def __call__(self, args: argparse.Namespace) -> None: revs = [r for r in args.revs.strip(":").split(":", maxsplit=1) if r] with in_directory(os.path.dirname(manifest.path)): - exceptions: List[str] = [] + exceptions: list[str] = [] projects = manifest.selected_projects(args.projects) if not projects: raise RuntimeError( @@ -139,7 +138,7 @@ def _get_repo(path: str, project: ProjectEntry) -> VCS: ) -def _diff_from_repo(repo: VCS, project: ProjectEntry, revs: List[str]) -> str: +def _diff_from_repo(repo: VCS, project: ProjectEntry, revs: list[str]) -> str: """Generate a relative diff for a svn repo.""" if len(revs) > 2: raise RuntimeError(f"Too many revisions given! {revs}") @@ -160,7 +159,7 @@ def _diff_from_repo(repo: VCS, project: ProjectEntry, revs: List[str]) -> str: def _dump_patch( - path: str, revs: List[str], project: ProjectEntry, patch_name: str, patch: str + path: str, revs: list[str], project: ProjectEntry, patch_name: str, patch: str ) -> None: """Dump the patch to a file.""" if patch: diff --git a/dfetch/commands/freeze.py b/dfetch/commands/freeze.py index 9e305e86..0eb648c8 100644 --- a/dfetch/commands/freeze.py +++ b/dfetch/commands/freeze.py @@ -41,7 +41,6 @@ import argparse import os import shutil -from typing import List import dfetch.commands.command import dfetch.manifest.project @@ -72,8 +71,8 @@ def __call__(self, args: argparse.Namespace) -> None: manifest = get_manifest() - exceptions: List[str] = [] - projects: List[ProjectEntry] = [] + exceptions: list[str] = [] + projects: list[ProjectEntry] = [] with in_directory(os.path.dirname(manifest.path)): for project in manifest.projects: diff --git a/dfetch/commands/import_.py b/dfetch/commands/import_.py index 5496d81d..f22a087f 100644 --- a/dfetch/commands/import_.py +++ b/dfetch/commands/import_.py @@ -83,8 +83,8 @@ import argparse import os import re +from collections.abc import Sequence from itertools import combinations -from typing import List, Sequence, Set, Tuple import dfetch.commands.command from dfetch import DEFAULT_MANIFEST_NAME @@ -150,7 +150,7 @@ def _import_projects() -> Sequence[ProjectEntry]: def _import_from_svn() -> Sequence[ProjectEntry]: - projects: List[ProjectEntry] = [] + projects: list[ProjectEntry] = [] for external in SvnRepo.externals(): projects.append( @@ -172,7 +172,7 @@ def _import_from_svn() -> Sequence[ProjectEntry]: def _import_from_git() -> Sequence[ProjectEntry]: - projects: List[ProjectEntry] = [] + projects: list[ProjectEntry] = [] toplevel: str = "" for submodule in GitLocalRepo.submodules(): projects.append( @@ -241,7 +241,7 @@ def _generate_remote_name(remote_url: str) -> str: return re.sub(r"[-]{2,}", "-", filtered).strip("-") -def _determine_best_remotes(projects_urls: Set[str]) -> Tuple[str, ...]: +def _determine_best_remotes(projects_urls: set[str]) -> tuple[str, ...]: """Determine the smallest amount of remotes, that cover the most urls. Args: @@ -254,17 +254,17 @@ def _determine_best_remotes(projects_urls: Set[str]) -> Tuple[str, ...]: max_remotes = 5 # Determine all possible remotes - potential_remotes: Set[str] = set() + potential_remotes: set[str] = set() for url in projects_urls: potential_remotes.add(url[:max_remote_length].rsplit("/", maxsplit=1)[0]) potential_remotes.add(url[:max_remote_length].rsplit("/", maxsplit=2)[0]) potential_remotes.add(url[:max_remote_length].rsplit(":", maxsplit=1)[0]) - useless_potential = set(["http", "https"]) + useless_potential = {"http", "https"} potential_remotes = potential_remotes - useless_potential # For each permutation of any length, calculate the solution score - solutions: List[Tuple[int, Tuple[str, ...]]] = [] + solutions: list[tuple[int, tuple[str, ...]]] = [] for i in range(min(len(potential_remotes), max_remotes)): for solution in combinations(potential_remotes, i): score = _calculate_solution_score(solution, projects_urls) @@ -275,7 +275,7 @@ def _determine_best_remotes(projects_urls: Set[str]) -> Tuple[str, ...]: def _calculate_solution_score( - solution: Tuple[str, ...], projects_urls: Set[str] + solution: tuple[str, ...], projects_urls: set[str] ) -> int: """Calculate a score with the given solution. diff --git a/dfetch/commands/report.py b/dfetch/commands/report.py index c5b03aae..07f0b2e8 100644 --- a/dfetch/commands/report.py +++ b/dfetch/commands/report.py @@ -6,7 +6,6 @@ import argparse import glob import os -from typing import List import dfetch.commands.command import dfetch.manifest.manifest @@ -79,7 +78,7 @@ def __call__(self, args: argparse.Namespace) -> None: logger.info(f"Generated {reporter.name} report: {args.outfile}") @staticmethod - def _determine_licenses(project: ProjectEntry) -> List[License]: + def _determine_licenses(project: ProjectEntry) -> list[License]: """Try to determine license of fetched project.""" if not os.path.exists(project.destination): logger.print_warning_line( diff --git a/dfetch/commands/update.py b/dfetch/commands/update.py index c82f07ec..04e80ed8 100644 --- a/dfetch/commands/update.py +++ b/dfetch/commands/update.py @@ -22,7 +22,6 @@ import argparse import os from pathlib import Path -from typing import List import dfetch.commands.command import dfetch.manifest.manifest @@ -71,8 +70,8 @@ def __call__(self, args: argparse.Namespace) -> None: """Perform the update.""" manifest = dfetch.manifest.manifest.get_manifest() - exceptions: List[str] = [] - destinations: List[str] = [ + exceptions: list[str] = [] + destinations: list[str] = [ os.path.realpath(project.destination) for project in manifest.projects ] with in_directory(os.path.dirname(manifest.path)): @@ -92,7 +91,7 @@ def __call__(self, args: argparse.Namespace) -> None: @staticmethod def _check_destination( - project: dfetch.manifest.project.ProjectEntry, destinations: List[str] + project: dfetch.manifest.project.ProjectEntry, destinations: list[str] ) -> None: """Do some sanity checks on the destination path.""" real_path = os.path.realpath(project.destination) @@ -137,7 +136,7 @@ def _check_dst_not_in_blacklist( @staticmethod def _check_overlapping_destination( project: dfetch.manifest.project.ProjectEntry, - destinations: List[str], + destinations: list[str], real_path: str, ) -> None: """Check if project will try to write to the same destination.""" diff --git a/dfetch/manifest/manifest.py b/dfetch/manifest/manifest.py index eebe88aa..40e10b4e 100644 --- a/dfetch/manifest/manifest.py +++ b/dfetch/manifest/manifest.py @@ -23,8 +23,9 @@ import os import pathlib import re +from collections.abc import Sequence from dataclasses import dataclass -from typing import IO, Any, Dict, List, Optional, Sequence, Tuple, Union +from typing import IO, Any, Optional, Union import yaml from typing_extensions import TypedDict @@ -102,7 +103,7 @@ class ManifestDict( # pylint: disable=too-many-ancestors version: Union[int, str] remotes: Sequence[Union[RemoteDict, Remote]] - projects: Sequence[Union[ProjectEntryDict, ProjectEntry, Dict[str, str]]] + projects: Sequence[Union[ProjectEntryDict, ProjectEntry, dict[str, str]]] class Manifest: @@ -143,8 +144,8 @@ def __init__( self._projects = self._init_projects(manifest["projects"]) def _init_projects( - self, projects: Sequence[Union[ProjectEntryDict, ProjectEntry, Dict[str, str]]] - ) -> Dict[str, ProjectEntry]: + self, projects: Sequence[Union[ProjectEntryDict, ProjectEntry, dict[str, str]]] + ) -> dict[str, ProjectEntry]: """Iterate over projects from manifest and initialize ProjectEntries from it. Args: @@ -156,7 +157,7 @@ def _init_projects( Returns: Dict[str, ProjectEntry]: Dictionary with key: Name of project, Value: ProjectEntry """ - _projects: Dict[str, ProjectEntry] = {} + _projects: dict[str, ProjectEntry] = {} for project in projects: if isinstance(project, dict): @@ -184,9 +185,9 @@ def _init_projects( @staticmethod def _determine_remotes( remotes_from_manifest: Sequence[Union[RemoteDict, Remote]], - ) -> Tuple[Dict[str, Remote], List[Remote]]: - default_remotes: List[Remote] = [] - remotes: Dict[str, Remote] = {} + ) -> tuple[dict[str, Remote], list[Remote]]: + default_remotes: list[Remote] = [] + remotes: dict[str, Remote] = {} for remote in remotes_from_manifest: if isinstance(remote, dict): last_remote = remotes[remote["name"]] = Remote.from_yaml(remote) @@ -243,7 +244,7 @@ def from_file(path: str) -> "Manifest": Raises: FileNotFoundError: Given path was not a file. """ - with open(path, "r", encoding="utf-8") as opened_file: + with open(path, encoding="utf-8") as opened_file: return Manifest.from_yaml(opened_file, path) @property @@ -291,7 +292,7 @@ def __repr__(self) -> str: """Get string representing this object.""" return str(self._as_dict()) - def _as_dict(self) -> Dict[str, ManifestDict]: + def _as_dict(self) -> dict[str, ManifestDict]: """Get this manifest as dict.""" remotes: Sequence[RemoteDict] = [ remote.as_yaml() for remote in self._remotes.values() @@ -300,9 +301,9 @@ def _as_dict(self) -> Dict[str, ManifestDict]: if len(remotes) == 1: remotes[0].pop("default", None) - projects: List[Dict[str, str]] = [] + projects: list[dict[str, str]] = [] for project in self.projects: - project_yaml: Dict[str, str] = project.as_yaml() + project_yaml: dict[str, str] = project.as_yaml() if len(remotes) == 1: project_yaml.pop("remote", None) projects.append(project_yaml) @@ -381,12 +382,12 @@ def get_manifest() -> Manifest: return Manifest.from_file(manifest_path) -def get_childmanifests(skip: Optional[List[str]] = None) -> List[Manifest]: +def get_childmanifests(skip: Optional[list[str]] = None) -> list[Manifest]: """Get manifest and its path.""" skip = skip or [] logger.debug("Looking for sub-manifests") - childmanifests: List[Manifest] = [] + childmanifests: list[Manifest] = [] for path in find_file(DEFAULT_MANIFEST_NAME, "."): path = os.path.realpath(path) if path not in skip: diff --git a/dfetch/manifest/project.py b/dfetch/manifest/project.py index e4c4ecab..4d5b4d5c 100644 --- a/dfetch/manifest/project.py +++ b/dfetch/manifest/project.py @@ -247,7 +247,8 @@ """ import copy -from typing import Dict, Optional, Sequence, Union +from collections.abc import Sequence +from typing import Optional, Union from typing_extensions import TypedDict @@ -304,7 +305,7 @@ def __init__(self, kwargs: ProjectEntryDict) -> None: @classmethod def from_yaml( cls, - yamldata: Union[Dict[str, str], ProjectEntryDict], + yamldata: Union[dict[str, str], ProjectEntryDict], default_remote: str = "", ) -> "ProjectEntry": """Create a Project Entry from yaml data. @@ -431,7 +432,7 @@ def as_recommendation(self) -> "ProjectEntry": recommendation._repo_path = "" # pylint: disable=protected-access return recommendation - def as_yaml(self) -> Dict[str, str]: + def as_yaml(self) -> dict[str, str]: """Get this project as yaml dictionary.""" yamldata = { "name": self._name, diff --git a/dfetch/manifest/remote.py b/dfetch/manifest/remote.py index b544e7ff..8fda4606 100644 --- a/dfetch/manifest/remote.py +++ b/dfetch/manifest/remote.py @@ -18,7 +18,7 @@ url-base: https://github.com/ """ -from typing import Dict, Optional, Union +from typing import Optional, Union from typing_extensions import TypedDict @@ -41,7 +41,7 @@ def __init__(self, kwargs: RemoteDict) -> None: self._default: bool = bool(kwargs.get("default", False)) @classmethod - def from_yaml(cls, yamldata: Union[Dict[str, str], RemoteDict]) -> "Remote": + def from_yaml(cls, yamldata: Union[dict[str, str], RemoteDict]) -> "Remote": """Create a remote entry in the manifest from yaml data. Returns: diff --git a/dfetch/project/git.py b/dfetch/project/git.py index b5563600..86375bdb 100644 --- a/dfetch/project/git.py +++ b/dfetch/project/git.py @@ -3,7 +3,7 @@ import os import pathlib from functools import lru_cache -from typing import List, Optional +from typing import Optional from dfetch.log import get_logger from dfetch.manifest.project import ProjectEntry @@ -38,7 +38,7 @@ def _does_revision_exist(self, revision: str) -> bool: """Check if the given revision exists.""" return self._remote_repo.check_version_exists(revision) - def _list_of_tags(self) -> List[str]: + def _list_of_tags(self) -> list[str]: """Get list of all available tags.""" return [str(tag) for tag in self._remote_repo.list_of_tags()] @@ -120,7 +120,7 @@ def _determine_fetched_version(self, version: Version, fetched_sha: str) -> Vers revision=version.revision or fetched_sha, ) - @lru_cache() + @lru_cache def get_default_branch(self) -> str: # type:ignore """Get the default branch of this repository.""" return self._remote_repo.get_default_branch() diff --git a/dfetch/project/metadata.py b/dfetch/project/metadata.py index 839fa26d..b9b363cf 100644 --- a/dfetch/project/metadata.py +++ b/dfetch/project/metadata.py @@ -69,7 +69,7 @@ def from_project_entry(cls, project: ProjectEntry) -> "Metadata": @classmethod def from_file(cls, path: str) -> "Metadata": """Load metadata file.""" - with open(path, "r", encoding="utf-8") as metadata_file: + with open(path, encoding="utf-8") as metadata_file: data: Options = yaml.safe_load(metadata_file)["dfetch"] return cls(data) diff --git a/dfetch/project/svn.py b/dfetch/project/svn.py index 7b874676..549b5f45 100644 --- a/dfetch/project/svn.py +++ b/dfetch/project/svn.py @@ -4,7 +4,7 @@ import pathlib import re import urllib.parse -from typing import Dict, List, NamedTuple, Optional, Tuple +from typing import NamedTuple, Optional from dfetch.log import get_logger from dfetch.manifest.version import Version @@ -40,7 +40,7 @@ class SvnRepo(VCS): NAME = "svn" @staticmethod - def externals() -> List[External]: + def externals() -> list[External]: """Get list of externals.""" result = run_on_cmdline( logger, @@ -54,7 +54,7 @@ def externals() -> List[External]: repo_root = SvnRepo._get_info_from_target()["Repository Root"] - externals: List[External] = [] + externals: list[External] = [] path_pattern = r"([^\s^-]+)\s+-" for entry in result.stdout.decode().split(os.linesep * 2): match: Optional[re.Match[str]] = None @@ -91,7 +91,7 @@ def externals() -> List[External]: return externals @staticmethod - def _split_url(url: str, repo_root: str) -> Tuple[str, str, str, str]: + def _split_url(url: str, repo_root: str) -> tuple[str, str, str, str]: # ../ Relative to the URL of the directory on which the svn:externals property is set # ^/ Relative to the root of the repository in which the svn:externals property is versioned # // Relative to the scheme of the URL of the directory on which the svn:externals property is set @@ -163,7 +163,7 @@ def _does_revision_exist(self, revision: str) -> bool: "In SVN only a revision is NOT enough, this should not be called!" ) - def _list_of_tags(self) -> List[str]: + def _list_of_tags(self) -> list[str]: """Get list of all available tags.""" result = run_on_cmdline(logger, f"svn ls --non-interactive {self.remote}/tags") return [ @@ -186,7 +186,7 @@ def list_tool_info() -> None: tool, version = first_line.replace(",", "").split("version", maxsplit=1) VCS._log_tool(tool, version) - def _determine_what_to_fetch(self, version: Version) -> Tuple[str, str, str]: + def _determine_what_to_fetch(self, version: Version) -> tuple[str, str, str]: """Based on the given version, determine what to fetch. Args: @@ -271,7 +271,7 @@ def _fetch_impl(self, version: Version) -> Version: return Version(tag=version.tag, branch=branch, revision=revision) @staticmethod - def _parse_file_pattern(complete_path: str) -> Tuple[str, str]: + def _parse_file_pattern(complete_path: str) -> tuple[str, str]: if complete_path.count("*") > 1: raise RuntimeError("Only single * supported in 'src:'!") @@ -282,7 +282,7 @@ def _parse_file_pattern(complete_path: str) -> Tuple[str, str]: glob_filter = "*".join([before_star, after]) return complete_path, glob_filter - def _get_info(self, branch: str) -> Dict[str, str]: + def _get_info(self, branch: str) -> dict[str, str]: return self._get_info_from_target(f"{self.remote}/{branch}") @staticmethod @@ -295,7 +295,7 @@ def _export(url: str, rev: str = "", dst: str = ".") -> None: ) @staticmethod - def _files_in_path(url_path: str) -> List[str]: + def _files_in_path(url_path: str) -> list[str]: return [ str(line) for line in run_on_cmdline(logger, f"svn list --non-interactive {url_path}") @@ -304,7 +304,7 @@ def _files_in_path(url_path: str) -> List[str]: ] @staticmethod - def _license_files(url_path: str) -> List[str]: + def _license_files(url_path: str) -> list[str]: return [ str(license) for license in filter( @@ -313,7 +313,7 @@ def _license_files(url_path: str) -> List[str]: ] @staticmethod - def _get_info_from_target(target: str = "") -> Dict[str, str]: + def _get_info_from_target(target: str = "") -> dict[str, str]: try: result = run_on_cmdline( logger, f"svn info --non-interactive {target.strip()}" diff --git a/dfetch/project/vcs.py b/dfetch/project/vcs.py index 373a4c9b..9c4e5399 100644 --- a/dfetch/project/vcs.py +++ b/dfetch/project/vcs.py @@ -4,7 +4,8 @@ import os import pathlib from abc import ABC, abstractmethod -from typing import List, Optional, Sequence, Tuple +from collections.abc import Sequence +from typing import Optional from halo import Halo from patch_ng import fromfile @@ -43,7 +44,7 @@ def _running_in_ci() -> bool: ci_env_var = os.getenv("CI", "") return bool(ci_env_var) and ci_env_var[0].lower() in ("t", "1", "y") - def check_wanted_with_local(self) -> Tuple[Optional[Version], Optional[Version]]: + def check_wanted_with_local(self) -> tuple[Optional[Version], Optional[Version]]: """Given the project entry in the manifest, get the relevant version from disk. Returns: @@ -263,7 +264,7 @@ def _does_revision_exist(self, revision: str) -> bool: """Check if the given revision exists.""" @abstractmethod - def _list_of_tags(self) -> List[str]: + def _list_of_tags(self) -> list[str]: """Get list of all available tags.""" @staticmethod diff --git a/dfetch/reporting/__init__.py b/dfetch/reporting/__init__.py index 93e9f98f..8eada255 100644 --- a/dfetch/reporting/__init__.py +++ b/dfetch/reporting/__init__.py @@ -1,7 +1,6 @@ """Various reporters for generating reports.""" from enum import Enum -from typing import Dict, Type from dfetch.reporting.reporter import Reporter from dfetch.reporting.sbom_reporter import SbomReporter @@ -19,7 +18,7 @@ def __str__(self) -> str: return self.value -REPORTERS: Dict[ReportTypes, Type[Reporter]] = { +REPORTERS: dict[ReportTypes, type[Reporter]] = { ReportTypes.STDOUT: StdoutReporter, ReportTypes.SBOM: SbomReporter, } diff --git a/dfetch/reporting/check/code_climate_reporter.py b/dfetch/reporting/check/code_climate_reporter.py index 7f786c22..2e4c1146 100644 --- a/dfetch/reporting/check/code_climate_reporter.py +++ b/dfetch/reporting/check/code_climate_reporter.py @@ -58,7 +58,7 @@ import json import os from enum import Enum -from typing import Any, Dict, List +from typing import Any from dfetch.log import get_logger from dfetch.manifest.manifest import Manifest @@ -94,7 +94,7 @@ def __init__(self, manifest: Manifest, report_path: str) -> None: self._report_path = report_path - self._report: List[Dict[str, Any]] = [] + self._report: list[dict[str, Any]] = [] @staticmethod def _determine_severity(severity: IssueSeverity) -> CodeClimateSeverity: diff --git a/dfetch/reporting/check/jenkins_reporter.py b/dfetch/reporting/check/jenkins_reporter.py index 0815c8b5..5e3714e0 100644 --- a/dfetch/reporting/check/jenkins_reporter.py +++ b/dfetch/reporting/check/jenkins_reporter.py @@ -52,7 +52,7 @@ import json import os -from typing import Any, Dict +from typing import Any from dfetch.log import get_logger from dfetch.manifest.manifest import Manifest @@ -78,7 +78,7 @@ def __init__(self, manifest: Manifest, report_path: str) -> None: self._report_path = report_path - self._report: Dict[str, Any] = { + self._report: dict[str, Any] = { "_class": "io.jenkins.plugins.analysis.core.restapi.ReportApi", "issues": [], } diff --git a/dfetch/reporting/check/reporter.py b/dfetch/reporting/check/reporter.py index 422a1daa..2f216912 100644 --- a/dfetch/reporting/check/reporter.py +++ b/dfetch/reporting/check/reporter.py @@ -26,9 +26,9 @@ """ from abc import abstractmethod +from collections.abc import Sequence from dataclasses import dataclass from enum import Enum -from typing import Sequence from dfetch.manifest.manifest import Manifest from dfetch.manifest.project import ProjectEntry diff --git a/dfetch/reporting/check/sarif_reporter.py b/dfetch/reporting/check/sarif_reporter.py index e76124b9..14f2cec5 100644 --- a/dfetch/reporting/check/sarif_reporter.py +++ b/dfetch/reporting/check/sarif_reporter.py @@ -70,7 +70,7 @@ import json import os from enum import Enum -from typing import Any, Dict +from typing import Any import attr from sarif_om import ( @@ -209,7 +209,7 @@ def __init__(self, sarif: SarifLog) -> None: Args: sarif (SarifLog): Log to serialize """ - self._sarif_dict: Dict[str, Any] = {"default": "default"} + self._sarif_dict: dict[str, Any] = {"default": "default"} self._json = self._walk_sarif( attr.asdict( sarif, diff --git a/dfetch/reporting/reporter.py b/dfetch/reporting/reporter.py index 705be102..1041d85a 100644 --- a/dfetch/reporting/reporter.py +++ b/dfetch/reporting/reporter.py @@ -1,7 +1,6 @@ """Abstract reporting interface.""" from abc import ABC, abstractmethod -from typing import List from dfetch.manifest.manifest import Manifest from dfetch.manifest.project import ProjectEntry @@ -30,7 +29,7 @@ def manifest(self) -> Manifest: def add_project( self, project: ProjectEntry, - licenses: List[License], + licenses: list[License], version: str, ) -> None: """Add a project to the report.""" diff --git a/dfetch/reporting/sbom_reporter.py b/dfetch/reporting/sbom_reporter.py index 48ca5cd7..263d79dc 100644 --- a/dfetch/reporting/sbom_reporter.py +++ b/dfetch/reporting/sbom_reporter.py @@ -69,7 +69,6 @@ """ from decimal import Decimal -from typing import List from cyclonedx.builder.this import this_component as cdx_lib_component from cyclonedx.model import ExternalReference, ExternalReferenceType, XsUri @@ -163,7 +162,7 @@ def __init__(self, manifest: Manifest) -> None: def add_project( self, project: ProjectEntry, - licenses: List[License], + licenses: list[License], version: str, ) -> None: """Add a project to the report.""" diff --git a/dfetch/reporting/stdout_reporter.py b/dfetch/reporting/stdout_reporter.py index 868c2ffd..e6a90df1 100644 --- a/dfetch/reporting/stdout_reporter.py +++ b/dfetch/reporting/stdout_reporter.py @@ -4,8 +4,6 @@ from the manifest or the metadata (``.dfetch_data.yaml``). """ -from typing import List - from dfetch.log import get_logger from dfetch.manifest.project import ProjectEntry from dfetch.project.metadata import Metadata @@ -23,7 +21,7 @@ class StdoutReporter(Reporter): def add_project( self, project: ProjectEntry, - licenses: List[License], + licenses: list[License], version: str, ) -> None: """Add a project to the report.""" diff --git a/dfetch/resources/__init__.py b/dfetch/resources/__init__.py index bac7575d..5e7548c1 100644 --- a/dfetch/resources/__init__.py +++ b/dfetch/resources/__init__.py @@ -1,7 +1,6 @@ """Resources needed when dfetch is distributed.""" import importlib.resources as importlib_resources -import sys from pathlib import Path from typing import ContextManager @@ -10,13 +9,7 @@ def _resource_path(filename: str) -> ContextManager[Path]: """Get the path to the resource.""" - if sys.version_info >= (3, 9): - return importlib_resources.as_file( - importlib_resources.files(resources) / filename - ) - return importlib_resources.path( # pylint: disable=deprecated-method - resources, filename - ) + return importlib_resources.as_file(importlib_resources.files(resources) / filename) def schema_path() -> ContextManager[Path]: diff --git a/dfetch/util/cmdline.py b/dfetch/util/cmdline.py index 84c8ca71..c302e77d 100644 --- a/dfetch/util/cmdline.py +++ b/dfetch/util/cmdline.py @@ -3,7 +3,7 @@ import logging import os import subprocess # nosec -from typing import Any, List, Optional, Union # pylint: disable=unused-import +from typing import Any, Optional, Union # pylint: disable=unused-import class SubprocessCommandError(Exception): @@ -15,7 +15,7 @@ class SubprocessCommandError(Exception): def __init__( self, - cmd: Optional[List[str]] = None, + cmd: Optional[list[str]] = None, stdout: str = "", stderr: str = "", returncode: int = 0, @@ -36,7 +36,7 @@ def message(self) -> str: def run_on_cmdline( - logger: logging.Logger, cmd: Union[str, List[str]] + logger: logging.Logger, cmd: Union[str, list[str]] ) -> "subprocess.CompletedProcess[Any]": """Run a command and log the output, and raise if something goes wrong.""" logger.debug(f"Running {cmd}") @@ -45,9 +45,7 @@ def run_on_cmdline( cmd = cmd.split(" ") try: - proc = subprocess.run( # nosec - cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True - ) + proc = subprocess.run(cmd, capture_output=True, check=True) # nosec except subprocess.CalledProcessError as exc: raise SubprocessCommandError( exc.cmd, diff --git a/dfetch/util/purl.py b/dfetch/util/purl.py index d4a24fa1..1cdbb99e 100644 --- a/dfetch/util/purl.py +++ b/dfetch/util/purl.py @@ -4,7 +4,7 @@ """ import re -from typing import List, Optional, Tuple +from typing import Optional from urllib.parse import urlparse from packageurl import PackageURL @@ -40,10 +40,10 @@ DEFAULT_NAME = "unknown" -def _namespace_and_name_from_domain_and_path(domain: str, path: str) -> Tuple[str, str]: +def _namespace_and_name_from_domain_and_path(domain: str, path: str) -> tuple[str, str]: """Split the full path to a name and namespace.""" domain = NO_FETCH_EXTRACT(domain).domain - parts: List[str] = [domain] if domain not in EXCLUDED_DOMAINS else [] + parts: list[str] = [domain] if domain not in EXCLUDED_DOMAINS else [] if path: parts.extend(path.split("/")) diff --git a/dfetch/util/util.py b/dfetch/util/util.py index eabcd42b..9ac5ebc5 100644 --- a/dfetch/util/util.py +++ b/dfetch/util/util.py @@ -5,9 +5,10 @@ import os import shutil import stat +from collections.abc import Generator, Iterator, Sequence from contextlib import contextmanager from pathlib import Path -from typing import Any, Generator, Iterator, List, Optional, Sequence, Union +from typing import Any, Optional, Union from _hashlib import HASH @@ -76,8 +77,8 @@ def in_directory(path: str) -> Generator[str, None, None]: @contextmanager def catch_runtime_exceptions( - exc_list: Optional[List[str]] = None, -) -> Generator[List[str], None, None]: + exc_list: Optional[list[str]] = None, +) -> Generator[list[str], None, None]: """Catch all runtime errors and add it to list of strings.""" exc_list = exc_list or [] try: @@ -97,14 +98,14 @@ def prefix_runtime_exceptions( raise RuntimeError(f"{prefix}: {exc}") from exc -def find_file(name: str, path: str = ".") -> List[str]: +def find_file(name: str, path: str = ".") -> list[str]: """Find all files with a specific name recursively in a directory.""" return [ os.path.join(root, name) for root, _, files in os.walk(path) if name in files ] -def hash_directory(path: str, skiplist: Optional[List[str]]) -> str: +def hash_directory(path: str, skiplist: Optional[list[str]]) -> str: """Hash a directory with all its files.""" digest = hashlib.md5() # nosec skiplist = skiplist or [] diff --git a/dfetch/util/versions.py b/dfetch/util/versions.py index 2a1c5285..6142d46d 100644 --- a/dfetch/util/versions.py +++ b/dfetch/util/versions.py @@ -2,7 +2,7 @@ import re from collections import defaultdict -from typing import Dict, List, Optional, Tuple +from typing import Optional from semver.version import Version @@ -20,7 +20,7 @@ ) -def coerce(version: str) -> Tuple[str, Optional[Version], str]: +def coerce(version: str) -> tuple[str, Optional[Version], str]: """Convert an incomplete version string into a semver-compatible Version object. * Tries to detect a "basic" version string (``major.minor.patch``). @@ -49,7 +49,7 @@ def coerce(version: str) -> Tuple[str, Optional[Version], str]: ) -def latest_tag_from_list(current_tag: str, available_tags: List[str]) -> str: +def latest_tag_from_list(current_tag: str, available_tags: list[str]) -> str: """Based on the given tag string and list of tags, get the latest available.""" parsed_tags = _create_available_version_dict(available_tags) @@ -67,8 +67,8 @@ def latest_tag_from_list(current_tag: str, available_tags: List[str]) -> str: def _create_available_version_dict( - available_tags: List[str], -) -> Dict[str, List[Tuple[Version, str]]]: + available_tags: list[str], +) -> dict[str, list[tuple[Version, str]]]: """Create a dictionary where each key is a prefix with a list of versions. Args: @@ -86,7 +86,7 @@ def _create_available_version_dict( {'release/': [(Version(major=1, minor=2, patch=3, prerelease=None, build=None), 'release/v1.2.3'), (Version(major=2, minor=0, patch=0, prerelease=None, build=None), 'release/v2.0.0')]} """ - parsed_tags: Dict[str, List[Tuple[Version, str]]] = defaultdict(list) + parsed_tags: dict[str, list[tuple[Version, str]]] = defaultdict(list) for available_tag in available_tags: prefix, version, _ = coerce(available_tag) if version: diff --git a/dfetch/vcs/git.py b/dfetch/vcs/git.py index d30009f0..925eaf93 100644 --- a/dfetch/vcs/git.py +++ b/dfetch/vcs/git.py @@ -4,8 +4,9 @@ import re import shutil import tempfile +from collections.abc import Generator, Sequence from pathlib import PurePath -from typing import Dict, Generator, List, NamedTuple, Optional, Sequence, Tuple +from typing import NamedTuple, Optional from dfetch.log import get_logger from dfetch.util.cmdline import SubprocessCommandError, run_on_cmdline @@ -26,7 +27,7 @@ class Submodule(NamedTuple): tag: str -def get_git_version() -> Tuple[str, str]: +def get_git_version() -> tuple[str, str]: """Get the name and version of git.""" result = run_on_cmdline(logger, "git --version") tool, version = result.stdout.decode().strip().split("version", maxsplit=1) @@ -62,11 +63,11 @@ def last_sha_on_branch(self, branch: str) -> str: """Get the last sha of a branch.""" return self._find_sha_of_branch_or_tag(self._ls_remote(self._remote), branch) - def find_branch_tip_or_tag_from_sha(self, sha: str) -> Tuple[str, str]: + def find_branch_tip_or_tag_from_sha(self, sha: str) -> tuple[str, str]: """Find branch or tag from sha.""" return self._find_branch_tip_or_tag_from_sha(self._ls_remote(self._remote), sha) - def list_of_tags(self) -> List[str]: + def list_of_tags(self) -> list[str]: """Get list of all available tags.""" info = self._ls_remote(self._remote) @@ -97,14 +98,14 @@ def get_default_branch(self) -> str: return "master" @staticmethod - def _ls_remote(remote: str) -> Dict[str, str]: + def _ls_remote(remote: str) -> dict[str, str]: result = run_on_cmdline( logger, f"git ls-remote --heads --tags {remote}" ).stdout.decode() - info: Dict[str, str] = {} + info: dict[str, str] = {} for line in filter(lambda x: x, result.split("\n")): - sha, ref = [part.strip() for part in f"{line} ".split("\t", maxsplit=1)] + sha, ref = (part.strip() for part in f"{line} ".split("\t", maxsplit=1)) # Annotated tag commit (more important) if ref.endswith("^{}"): @@ -114,7 +115,7 @@ def _ls_remote(remote: str) -> Dict[str, str]: return info @staticmethod - def _find_sha_of_branch_or_tag(info: Dict[str, str], branch_or_tag: str) -> str: + def _find_sha_of_branch_or_tag(info: dict[str, str], branch_or_tag: str) -> str: """Find SHA of a branch tip or tag.""" for reference, sha in info.items(): if reference in [ @@ -126,8 +127,8 @@ def _find_sha_of_branch_or_tag(info: Dict[str, str], branch_or_tag: str) -> str: @staticmethod def _find_branch_tip_or_tag_from_sha( - info: Dict[str, str], rev: str - ) -> Tuple[str, str]: + info: dict[str, str], rev: str + ) -> tuple[str, str]: """Check all branch tips and tags and see if the sha is one of them.""" branch, tag = "", "" for reference, sha in info.items(): @@ -194,7 +195,7 @@ def checkout_version( # pylint: disable=too-many-arguments remote: str, version: str, src: Optional[str] = None, - must_keeps: Optional[List[str]] = None, + must_keeps: Optional[list[str]] = None, ignore: Optional[Sequence[str]] = None, ) -> str: """Checkout a specific version from a given remote. @@ -332,7 +333,7 @@ def create_diff(self, old_hash: str, new_hash: Optional[str]) -> str: return str(result.stdout.decode()) @staticmethod - def submodules() -> List[Submodule]: + def submodules() -> list[Submodule]: """Get a list of submodules in the current directory.""" result = run_on_cmdline( logger, @@ -345,8 +346,8 @@ def submodules() -> List[Submodule]: ], ) - submodules: List[Submodule] = [] - urls: Dict[str, str] = {} + submodules: list[Submodule] = [] + urls: dict[str, str] = {} for line in result.stdout.decode().split("\n"): if line: name, sm_path, sha, toplevel = line.split(" ") @@ -381,7 +382,7 @@ def submodules() -> List[Submodule]: return submodules @staticmethod - def _get_submodule_urls(toplevel: str) -> Dict[str, str]: + def _get_submodule_urls(toplevel: str) -> dict[str, str]: result = run_on_cmdline( logger, [ @@ -432,7 +433,7 @@ def find_branch_containing_sha(self, sha: str) -> str: ["git", "branch", "--contains", sha], ) - branches: List[str] = [ + branches: list[str] = [ branch.strip() for branch in result.stdout.decode().split("*") if branch.strip() and "HEAD detached at" not in branch.strip() diff --git a/pyproject.toml b/pyproject.toml index 1c37acdc..cdedf596 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -82,6 +82,7 @@ development = [ 'isort==6.1.0', 'pylint==3.3.9', 'pyright==1.1.406', + 'pyupgrade==3.21.0', "tomli; python_version < '3.11'", # Tomllib is default in 3.11, required for letting codespell read the pyproject.toml 'pre-commit==4.3.0', 'ruff==0.14.1',