Fixes

spoorcc · spoorcc · commit 978b34746f43 · 2026-03-20T22:01:28.000Z
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -50,7 +50,7 @@ jobs:
 
       - name: Update latest tag
         if: ${{ steps.release_info.outputs.tag == 'latest' }}
-        uses: EndBug/latest-tag@fabb56bc8d15d5937c76719060da2226f5c3ffa8 
+        uses: EndBug/latest-tag@fabb56bc8d15d5937c76719060da2226f5c3ffa8
         with:
           ref: latest
           description: Last state in main
diff --git a/dfetch/manifest/manifest.py b/dfetch/manifest/manifest.py
@@ -97,7 +97,9 @@ class ManifestDict(TypedDict, total=True):  # pylint: disable=too-many-ancestors
 
     version: int | str
     remotes: NotRequired[Sequence[RemoteDict | Remote]]
-    projects: Sequence[ProjectEntryDict | ProjectEntry | dict[str, str | list[str]]]
+    projects: Sequence[
+        ProjectEntryDict | ProjectEntry | dict[str, str | list[str] | dict[str, str]]
+    ]
 
 
 class Manifest:
@@ -140,14 +142,16 @@ def __init__(
     def _init_projects(
         self,
         projects: Sequence[
-            ProjectEntryDict | ProjectEntry | dict[str, str | list[str]]
+            ProjectEntryDict
+            | ProjectEntry
+            | dict[str, str | list[str] | dict[str, str]]
         ],
     ) -> dict[str, ProjectEntry]:
         """Iterate over projects from manifest and initialize ProjectEntries from it.
 
         Args:
             projects (Sequence[
-                Union[ProjectEntryDict, ProjectEntry, Dict[str, Union[str, list[str]]]]
+                Union[ProjectEntryDict, ProjectEntry, Dict[str, Union[str, list[str], dict[str, str]]]]
             ]): Iterable with projects
 
         Raises:
@@ -304,9 +308,11 @@ def _as_dict(self) -> dict[str, ManifestDict]:
         if len(remotes) == 1:
             remotes[0].pop("default", None)
 
-        projects: list[dict[str, str | list[str]]] = []
+        projects: list[dict[str, str | list[str] | dict[str, str]]] = []
         for project in self.projects:
-            project_yaml: dict[str, str | list[str]] = project.as_yaml()
+            project_yaml: dict[str, str | list[str] | dict[str, str]] = (
+                project.as_yaml()
+            )
             if len(remotes) == 1:
                 project_yaml.pop("remote", None)
             projects.append(project_yaml)
diff --git a/dfetch/manifest/project.py b/dfetch/manifest/project.py
@@ -370,7 +370,7 @@ def as_yaml(self) -> dict[str, str]:
         "repo-path": str,
         "vcs": str,
         "ignore": Sequence[str],
-        "integrity": dict,
+        "integrity": dict[str, str],
         "default_remote": str,
     },
     total=False,
@@ -398,7 +398,7 @@ def __init__(self, kwargs: ProjectEntryDict) -> None:
         self._tag: str = kwargs.get("tag", "")
         self._vcs: str = kwargs.get("vcs", "")
         self._ignore: Sequence[str] = kwargs.get("ignore", [])
-        integrity_data: dict = kwargs.get("integrity", {})
+        integrity_data: dict[str, str] = kwargs.get("integrity", {})
         self._integrity = Integrity(hash=integrity_data.get("hash", ""))
 
         if not self._remote and not self._url:
@@ -407,7 +407,7 @@ def __init__(self, kwargs: ProjectEntryDict) -> None:
     @classmethod
     def from_yaml(
         cls,
-        yamldata: dict[str, str | list[str]] | ProjectEntryDict,
+        yamldata: dict[str, str | list[str] | dict[str, str]] | ProjectEntryDict,
         default_remote: str = "",
     ) -> "ProjectEntry":
         """Create a Project Entry from yaml data.
diff --git a/dfetch/project/__init__.py b/dfetch/project/__init__.py
@@ -15,7 +15,9 @@
 from dfetch.project.svnsuperproject import SvnSuperProject
 from dfetch.util.util import resolve_absolute_path
 
-SUPPORTED_SUBPROJECT_TYPES = [ArchiveSubProject, GitSubProject, SvnSubProject]
+SUPPORTED_SUBPROJECT_TYPES: list[
+    type[ArchiveSubProject] | type[GitSubProject] | type[SvnSubProject]
+] = [ArchiveSubProject, GitSubProject, SvnSubProject]
 SUPPORTED_SUPERPROJECT_TYPES = [GitSuperProject, SvnSuperProject]
 
 logger = get_logger(__name__)
diff --git a/dfetch/project/archivesubproject.py b/dfetch/project/archivesubproject.py
@@ -44,17 +44,22 @@
 import os
 import pathlib
 import tempfile
+import urllib.request as _ur
 
 from dfetch.log import get_logger
 from dfetch.manifest.project import ProjectEntry
 from dfetch.manifest.version import Version
 from dfetch.project.subproject import SubProject
+from dfetch.vcs.archive import (
+    _safe_compare_hex,  # private helper, intentionally imported for internal use
+)
+from dfetch.vcs.archive import (
+    _suffix_for_url,  # private helper, intentionally imported for internal use
+)
 from dfetch.vcs.archive import (
     SUPPORTED_HASH_ALGORITHMS,
     ArchiveLocalRepo,
     ArchiveRemote,
-    _safe_compare_hex,  # private helper, intentionally imported for internal use
-    _suffix_for_url,    # private helper, intentionally imported for internal use
     compute_hash,
     is_archive_url,
 )
@@ -94,8 +99,6 @@ def revision_is_enough() -> bool:
     @staticmethod
     def list_tool_info() -> None:
         """Log information about the archive fetching tool (Python's urllib)."""
-        import urllib.request as _ur  # noqa: PLC0415
-
         SubProject._log_tool("urllib", _ur.__doc__ or "built-in")
 
     def get_default_branch(self) -> str:
diff --git a/dfetch/project/subproject.py b/dfetch/project/subproject.py
@@ -409,7 +409,9 @@ def freeze_project(self, project: ProjectEntry) -> str | None:
             return None
         if on_disk_version:
             project.version = on_disk_version
-            return on_disk_version.revision or on_disk_version.tag or str(on_disk_version)
+            return (
+                on_disk_version.revision or on_disk_version.tag or str(on_disk_version)
+            )
         return None
 
     @staticmethod
diff --git a/dfetch/reporting/sbom_reporter.py b/dfetch/reporting/sbom_reporter.py
@@ -107,13 +107,14 @@
 from cyclonedx.model.license import LicenseAcknowledgement
 from cyclonedx.output import make_outputter
 from cyclonedx.schema import OutputFormat, SchemaVersion
+from packageurl import PackageURL
 
 import dfetch.util.purl
-from dfetch.util.purl import DFETCH_TO_CDX_HASH_ALGORITHM
 from dfetch.manifest.manifest import Manifest
 from dfetch.manifest.project import ProjectEntry
 from dfetch.reporting.reporter import Reporter
 from dfetch.util.license import License
+from dfetch.util.purl import DFETCH_TO_CDX_HASH_ALGORITHM
 
 # PyRight is pedantic with decorators see https://github.com/madpah/serializable/issues/8
 # It might be fixable with https://github.com/microsoft/pyright/discussions/4426, would prefer
@@ -190,11 +191,8 @@ def add_project(
         purl = dfetch.util.purl.remote_url_to_purl(
             project.remote_url, version=version, subpath=project.source or None
         )
-
         name = project.name if purl.type == "generic" else purl.name
-
         location = self.manifest.find_name_in_manifest(project.name)
-
         component = Component(
             name=name,
             version=version,
@@ -250,7 +248,15 @@ def add_project(
                 ],
             ),
         )
+        self._apply_external_references(component, purl, version)
+        self._apply_licenses(component, licenses)
+        self._bom.components.add(component)
 
+    @staticmethod
+    def _apply_external_references(
+        component: Component, purl: PackageURL, version: str
+    ) -> None:
+        """Attach external references to *component* based on its PURL type."""
         if purl.type == "github":
             component.external_references.add(
                 ExternalReference(
@@ -266,53 +272,62 @@ def add_project(
                 )
             )
         elif purl.qualifiers.get("download_url"):
-            # Archive dependency: add a DISTRIBUTION external reference and,
-            # when the version encodes a cryptographic hash, record it on the component.
-            download_url = purl.qualifiers["download_url"]
-            component.group = purl.namespace or None  # type: ignore[assignment]
+            SbomReporter._apply_archive_refs(component, purl, version)
+        else:
+            SbomReporter._apply_vcs_refs(component, purl)
+
+    @staticmethod
+    def _apply_archive_refs(
+        component: Component, purl: PackageURL, version: str
+    ) -> None:
+        """Add DISTRIBUTION reference and optional hash for an archive dependency."""
+        download_url = purl.qualifiers["download_url"]
+        component.group = purl.namespace or None  # type: ignore[assignment]
+        component.external_references.add(
+            ExternalReference(
+                type=ExternalReferenceType.DISTRIBUTION,
+                url=XsUri(download_url),
+            )
+        )
+        if version and ":" in version:
+            algo_prefix, hex_value = version.split(":", 1)
+            cdx_algo_name = DFETCH_TO_CDX_HASH_ALGORITHM.get(algo_prefix)
+            if cdx_algo_name:
+                component.hashes.add(
+                    HashType(
+                        alg=HashAlgorithm(cdx_algo_name),
+                        content=hex_value,
+                    )
+                )
+
+    @staticmethod
+    def _apply_vcs_refs(component: Component, purl: PackageURL) -> None:
+        """Add VCS external reference and group for a generic VCS dependency."""
+        component.group = purl.namespace
+        vcs_url = purl.qualifiers.get("vcs_url", "")
+        # ExternalReferenceType.VCS does not support ssh:// urls
+        if vcs_url and "ssh://" not in vcs_url:
             component.external_references.add(
                 ExternalReference(
-                    type=ExternalReferenceType.DISTRIBUTION,
-                    url=XsUri(download_url),
+                    type=ExternalReferenceType.VCS,
+                    url=XsUri(vcs_url),
                 )
             )
-            if version and ":" in version:
-                algo_prefix, hex_value = version.split(":", 1)
-                cdx_algo_name = DFETCH_TO_CDX_HASH_ALGORITHM.get(algo_prefix)
-                if cdx_algo_name:
-                    component.hashes.add(
-                        HashType(
-                            alg=HashAlgorithm(cdx_algo_name),
-                            content=hex_value,
-                        )
-                    )
-        else:
-            component.group = purl.namespace
-
-            vcs_url = purl.qualifiers.get("vcs_url", "")
-            # ExternalReferenceType.VCS does not support ssh:// urls
-            if vcs_url and "ssh://" not in vcs_url:
-                component.external_references.add(
-                    ExternalReference(
-                        type=ExternalReferenceType.VCS,
-                        url=XsUri(vcs_url),
-                    )
-                )
 
+    @staticmethod
+    def _apply_licenses(component: Component, licenses: list[License]) -> None:
+        """Attach *licenses* to *component* and its evidence block."""
         for lic in licenses:
-            # License wants either an SPDX id or a name, prefer SPDX id when available
+            # Prefer SPDX id when available
             cdx_license = (
                 CycloneDxLicense(id=lic.spdx_id)
                 if lic.spdx_id
                 else CycloneDxLicense(name=lic.name)
             )
-
             component.licenses.add(cdx_license)
             if component.evidence:
                 component.evidence.licenses.add(cdx_license)
 
-        self._bom.components.add(component)
-
     def dump_to_file(self, outfile: str) -> bool:
         """Dump the SBoM to file."""
         output_format = OutputFormat(
diff --git a/dfetch/util/purl.py b/dfetch/util/purl.py
@@ -101,59 +101,67 @@ def _known_purl_types(
     return None
 
 
-def remote_url_to_purl(
-    remote_url: str, version: str | None = None, subpath: str | None = None
+def _archive_purl(
+    remote_url: str, version: str | None, subpath: str | None
 ) -> PackageURL:
-    """Convert a remote URL to a valid PackageURL object.
+    """Build a generic PURL for an archive URL."""
+    parsed = urlparse(remote_url)
+    basename = os.path.basename(parsed.path)
+    name = _strip_archive_extension(basename) or DEFAULT_NAME
+    namespace = parsed.hostname or ""
+    return PackageURL(
+        type="generic",
+        namespace=namespace or None,
+        name=name,
+        version=version,
+        qualifiers={"download_url": remote_url},
+        subpath=subpath,
+    )
 
-    Supports GitHub, Bitbucket, SVN, SSH paths, and archive downloads.
-    Optionally specify version and subpath.
-    """
-    purl = _known_purl_types(remote_url, version, subpath)
-    if purl:
-        return purl
 
-    # Archive URLs (tar.gz, zip, …) get a generic PURL with a download_url qualifier.
-    # The name is derived from the archive filename (extension stripped); the
-    # namespace is the hostname (empty for file:// URLs).
-    if _is_archive_url(remote_url):
-        parsed = urlparse(remote_url)
-        basename = os.path.basename(parsed.path)
-        name = _strip_archive_extension(basename) or DEFAULT_NAME
-        namespace = parsed.hostname or ""
-        return PackageURL(
-            type="generic",
-            namespace=namespace or None,
-            name=name,
-            version=version,
-            qualifiers={"download_url": remote_url},
-            subpath=subpath,
-        )
+def _vcs_namespace_and_name(remote_url: str) -> tuple[str, str, str]:
+    """Derive namespace, name, and normalised URL for a generic VCS remote URL.
 
+    Returns:
+        A ``(namespace, name, remote_url)`` tuple where *remote_url* may have
+        been normalised (e.g. SSH short-form converted to ``ssh://`` scheme).
+    """
     parsed = urlparse(remote_url)
     path = parsed.path.lstrip("/")
-
     if "svn" in parsed.scheme or "svn." in parsed.netloc:
         namespace, name = _namespace_and_name_from_domain_and_path(parsed.netloc, path)
         if namespace.startswith("p/"):
             namespace = namespace[len("p/") :]
         namespace = namespace.replace("/svn/", "/")
-
     else:
         match = SSH_REGEX.match(remote_url)
         if match:
             namespace, name = _namespace_and_name_from_domain_and_path(
-                match.group("host"),
-                match.group("path"),
+                match.group("host"), match.group("path")
             )
-
             if not parsed.scheme:
                 remote_url = f"ssh://{parsed.path.replace(':', '/')}"
         else:
             namespace, name = _namespace_and_name_from_domain_and_path(
                 remote_url, path.replace(".git", "")
             )
+    return namespace, name, remote_url
+
 
+def remote_url_to_purl(
+    remote_url: str, version: str | None = None, subpath: str | None = None
+) -> PackageURL:
+    """Convert a remote URL to a valid PackageURL object.
+
+    Supports GitHub, Bitbucket, SVN, SSH paths, and archive downloads.
+    Optionally specify version and subpath.
+    """
+    purl = _known_purl_types(remote_url, version, subpath)
+    if purl:
+        return purl
+    if _is_archive_url(remote_url):
+        return _archive_purl(remote_url, version, subpath)
+    namespace, name, remote_url = _vcs_namespace_and_name(remote_url)
     return PackageURL(
         type="generic",
         namespace=namespace,
diff --git a/dfetch/vcs/archive.py b/dfetch/vcs/archive.py
diff --git a/tests/test_archive.py b/tests/test_archive.py
diff --git a/tests/test_integrity.py b/tests/test_integrity.py
