Update the aosp importer to correctly parse CodeCommit using packageurl-python

Signed-off-by: ziad hany <ziadhany2016@gmail.com>

Author: ziadhany

vulnerabilities/importer.py

@@ -194,6 +194,12 @@ def from_url(cls, url):
         return cls(url=url)
 
 
+"""
+
+"""
+VCS_URLS_SUPPORTED_TYPES = {"github", "bitbucket", "gitlab"}
+
+
 @dataclasses.dataclass(eq=True)
 @functools.total_ordering
 class CodePatchData:

vulnerabilities/pipelines/v2_importers/aosp_importer.py

@@ -14,18 +14,18 @@
 
 import dateparser
 from fetchcode.vcs import fetch_via_vcs
+from packageurl.contrib.purl2url import get_repo_url
 from packageurl.contrib.url2purl import url2purl
 
 from aboutcode.hashid import get_core_purl
+from vulnerabilities.importer import VCS_URLS_SUPPORTED_TYPES
 from vulnerabilities.importer import AdvisoryData
 from vulnerabilities.importer import AffectedPackageV2
 from vulnerabilities.importer import CodePatchData
 from vulnerabilities.importer import ReferenceV2
 from vulnerabilities.importer import VulnerabilitySeverity
 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
 from vulnerabilities.severity_systems import GENERIC
-from vulnerabilities.utils import VCS_URLS_SUPPORTED_TYPES
-from vulnerabilities.utils import parse_commit_url
 
 
 class AospImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
@@ -96,24 +96,29 @@ def collect_advisories(self):
                     purl = url2purl(commit_url)
                     base_purl = get_core_purl(purl)
 
-                    if base_purl and base_purl.type in VCS_URLS_SUPPORTED_TYPES:
-                        vcs_url, commit_hash = parse_commit_url(url=commit_url)
+                    purl_string = base_purl.to_string()
+                    vcs_url = get_repo_url(purl_string)
 
-                        fixed_commit = CodePatchData(
-                            commit_hash=commit_hash,
-                            vcs_url=vcs_url,
+                    if not base_purl or base_purl.type not in VCS_URLS_SUPPORTED_TYPES:
+                        references.append(
+                            ReferenceV2(
+                                reference_id=commit_id,
+                                reference_type="commit",
+                                url=commit_url,
+                            )
                         )
+                        continue
 
-                        affected_package = AffectedPackageV2(
-                            package=base_purl,
-                            fixed_by_commits=[fixed_commit],
-                        )
-                        affected_packages.append(affected_package)
-                    else:
-                        ref = ReferenceV2(
-                            reference_id=commit_id, reference_type="commit", url=commit_url
-                        )
-                        references.append(ref)
+                    fixed_commit = CodePatchData(
+                        commit_hash=purl.version,
+                        vcs_url=vcs_url,
+                    )
+
+                    affected_package = AffectedPackageV2(
+                        package=base_purl,
+                        fixed_by_commits=[fixed_commit],
+                    )
+                    affected_packages.append(affected_package)
 
                 url = (
                     "https://raw.githubusercontent.com/quarkslab/aosp_dataset/refs/heads/master/cves/"

vulnerabilities/tests/test_data/aosp/CVE-aosp_test3-expected.json

@@ -19,7 +19,7 @@
         "fixed_by_commits": [
           {
             "commit_hash": "0048b4837affd153897ed1222283492070027aa9",
-            "vcs_url": "https://github.com/torvalds/linux.git",
+            "vcs_url": "https://github.com/torvalds/linux",
             "commit_patch": null
           }
         ]

vulnerabilities/tests/test_utils.py

@@ -17,7 +17,6 @@
 from vulnerabilities.utils import get_item
 from vulnerabilities.utils import get_severity_range
 from vulnerabilities.utils import nearest_patched_package
-from vulnerabilities.utils import parse_commit_url
 from vulnerabilities.utils import resolve_version_range
 from vulnerabilities.utils import split_markdown_front_matter
 
@@ -152,33 +151,3 @@ def test_resolve_version_range_without_ignorable_versions():
 def test_get_severity_range():
     assert get_severity_range({""}) is None
     assert get_severity_range({}) is None
-
-
-@pytest.mark.parametrize(
-    "url,expected_repo,expected_commit",
-    [
-        (
-            "https://github.com/aboutcode-org/vulnerablecode/commit/98e516011d6e096e25247b82fc5f196bbeecff10",
-            "https://github.com/aboutcode-org/vulnerablecode.git",
-            "98e516011d6e096e25247b82fc5f196bbeecff10",
-        ),
-        (
-            "https://gitlab.com/gitlab-org/gitlab-development-kit/-/commit/1bd329f740c39ef97f1a2a5b2a760a4fe544ca14",
-            "https://gitlab.com/gitlab-org/gitlab-development-kit.git",
-            "1bd329f740c39ef97f1a2a5b2a760a4fe544ca14",
-        ),
-        (
-            "https://bitbucket.org/cpointe/fermenter/commits/6ba6f9fb354117bbf0d5f78cb8a80804bae694dd",
-            "https://bitbucket.org/cpointe/fermenter.git",
-            "6ba6f9fb354117bbf0d5f78cb8a80804bae694dd",
-        ),
-    ],
-)
-def test_valid_commit_urls(url, expected_repo, expected_commit):
-    result = parse_commit_url(url)
-    assert result == (expected_repo, expected_commit)
-
-
-def test_invalid_url():
-    with pytest.raises(ValueError):
-        parse_commit_url("https://example.com/not/a/commit/url")

vulnerabilities/utils.py

@@ -681,36 +681,3 @@ def create_registry(pipelines):
         registry[key] = pipeline
 
     return registry
-
-
-VCS_URLS_SUPPORTED_TYPES = {"github", "bitbucket", "gitlab"}
-
-
-def parse_commit_url(url: str):
-    """Parse a GitHub/GitLab/Bitbucket commit URL and return a repo_url and commit."""
-
-    patterns = [
-        # GitHub: https://github.com/user/repo/commit/<sha>
-        (
-            r"^https?://github\.com/([^/]+)/([^/]+)/commit/([a-f0-9]+)$",
-            "https://github.com/{user}/{repo}.git",
-        ),
-        # GitLab: https://gitlab.com/user/repo/-/commit/<sha>
-        (
-            r"^https?://gitlab\.com/([^/]+)/([^/]+)/-/commit/([a-f0-9]+)$",
-            "https://gitlab.com/{user}/{repo}.git",
-        ),
-        # Bitbucket: https://bitbucket.org/user/repo/commits/<sha>
-        (
-            r"^https?://bitbucket\.org/([^/]+)/([^/]+)/commits/([a-f0-9]+)$",
-            "https://bitbucket.org/{user}/{repo}.git",
-        ),
-    ]
-
-    for regex, repo_template in patterns:
-        m = re.match(regex, url, re.IGNORECASE)
-        if m:
-            user, repo, commit = m.groups()
-            return repo_template.format(user=user, repo=repo), commit
-
-    raise ValueError(f"Unsupported commit URL format {url}")