Skip to content

Commit 64e477b

Browse files
ziadhanyziadhany
committed
Add support for aosp importer
Add a test Signed-off-by: ziad hany <ziadhany2016@gmail.com>
1 parent 6fc641b commit 64e477b

File tree

8 files changed

+370
-62
lines changed

8 files changed

+370
-62
lines changed

‎vulnerabilities/importers/__init__.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@
4141
from vulnerabilities.pipelines import nvd_importer
4242
from vulnerabilities.pipelines import pypa_importer
4343
from vulnerabilities.pipelines import pysec_importer
44-
from vulnerabilities.pipelines.v2_importers import aosp_importer
44+
from vulnerabilities.pipelines.v2_importers import aosp_importer as aosp_importer_v2
4545
from vulnerabilities.pipelines.v2_importers import apache_httpd_importer as apache_httpd_v2
4646
from vulnerabilities.pipelines.v2_importers import archlinux_importer as archlinux_importer_v2
4747
from vulnerabilities.pipelines.v2_importers import curl_importer as curl_importer_v2
@@ -82,7 +82,7 @@
8282
mozilla_importer_v2.MozillaImporterPipeline,
8383
github_osv_importer_v2.GithubOSVImporterPipeline,
8484
redhat_importer_v2.RedHatImporterPipeline,
85-
aosp_importer.AospImporterPipeline,
85+
aosp_importer_v2.AospImporterPipeline,
8686
nvd_importer.NVDImporterPipeline,
8787
github_importer.GitHubAPIImporterPipeline,
8888
gitlab_importer.GitLabImporterPipeline,

‎vulnerabilities/pipelines/v2_importers/aosp_importer.py‎

Lines changed: 68 additions & 58 deletions
Original file line numberDiff line numberDiff line change
@@ -8,18 +8,24 @@
88
#
99

1010
import json
11-
import shutil
1211
from datetime import timezone
1312
from pathlib import Path
13+
from urllib.parse import quote
1414

1515
import dateparser
1616
from fetchcode.vcs import fetch_via_vcs
17+
from packageurl.contrib.url2purl import url2purl
1718

19+
from aboutcode.hashid import get_core_purl
1820
from vulnerabilities.importer import AdvisoryData
21+
from vulnerabilities.importer import AffectedPackageV2
22+
from vulnerabilities.importer import CodePatchData
1923
from vulnerabilities.importer import ReferenceV2
2024
from vulnerabilities.importer import VulnerabilitySeverity
2125
from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2
2226
from vulnerabilities.severity_systems import GENERIC
27+
from vulnerabilities.utils import VCS_URLS_SUPPORTED_TYPES
28+
from vulnerabilities.utils import parse_commit_url
2329

2430

2531
class AospImporterPipeline(VulnerableCodeBaseImporterPipelineV2):
@@ -60,71 +66,75 @@ def collect_advisories(self):
6066
with open(file_path) as f:
6167
vulnerability_data = json.load(f)
6268

63-
vulnerability_id = vulnerability_data.get("cveId", [])
64-
if (
65-
not vulnerability_id or "," in vulnerability_id
66-
): # escape invalid multiple CVE-2017-13077, CVE-2017-13078
67-
continue
68-
69-
summary = vulnerability_data.get("vulnerabilityType")
70-
date_reported = vulnerability_data.get("dateReported")
71-
date_published = dateparser.parse(date_reported) if date_reported else None
72-
if date_published and not date_published.tzinfo:
73-
date_published = date_published.replace(tzinfo=timezone.utc)
74-
75-
severities = []
76-
severity_value = vulnerability_data.get("severity")
77-
if severity_value:
78-
severities.append(
79-
VulnerabilitySeverity(
80-
system=GENERIC,
81-
value=severity_value,
82-
)
83-
)
84-
85-
references = []
86-
for commit_data in vulnerability_data.get("fixes", []):
87-
vcs_url = commit_data.get("patchUrl")
88-
commit_id = commit_data.get("commitId")
89-
90-
"""
91-
https://us.codeaurora.org/cgit/quic/la/kernel/msm/commit/?id=17bfaf64ad503d2e6607d2d3e0956f25bf07eb43
92-
http://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/commit/?id=f54e18f1b831c92f6512d2eedb224cd63d607d3d
93-
https://android.googlesource.com/platform/system/bt/+/514139f4b40cbb035bb92f3e24d5a389d75db9e6
94-
https://source.codeaurora.org/quic/la//kernel/msm-4.4/commit/?id=b108c651cae9913da1ab163cb4e5f7f2db87b747
95-
96-
Check if commit in the url split based on it to get the commit hash and the vcs url
97-
if not split base on /+/
98-
"""
99-
100-
if not vcs_url:
69+
vulnerability_ids = vulnerability_data.get("cveId", "")
70+
for vulnerability_id in vulnerability_ids.split(","):
71+
if not vulnerability_id:
10172
continue
10273

103-
fixed_by_commits = []
104-
repo_url, commit_id = url.split("/+/")
74+
summary = vulnerability_data.get("vulnerabilityType")
75+
date_reported = vulnerability_data.get("dateReported")
76+
date_published = dateparser.parse(date_reported) if date_reported else None
77+
if date_published and not date_published.tzinfo:
78+
date_published = date_published.replace(tzinfo=timezone.utc)
79+
80+
severities = []
81+
severity_value = vulnerability_data.get("severity")
82+
if severity_value:
83+
severities.append(
84+
VulnerabilitySeverity(
85+
system=GENERIC,
86+
value=severity_value,
87+
)
88+
)
10589

106-
fixed_commit = CodeCommitData(
107-
commit_hash=commit_hash,
108-
url=vcs_url,
90+
references = []
91+
affected_packages = []
92+
for commit_data in vulnerability_data.get("fixes", []):
93+
commit_url = commit_data.get("patchUrl")
94+
commit_id = commit_data.get("commitId")
95+
96+
purl = url2purl(commit_url)
97+
base_purl = get_core_purl(purl)
98+
99+
if base_purl and base_purl.type in VCS_URLS_SUPPORTED_TYPES:
100+
vcs_url, commit_hash = parse_commit_url(url=commit_url)
101+
102+
fixed_commit = CodePatchData(
103+
commit_hash=commit_hash,
104+
vcs_url=vcs_url,
105+
)
106+
107+
affected_package = AffectedPackageV2(
108+
package=base_purl,
109+
fixed_by_commits=[fixed_commit],
110+
)
111+
affected_packages.append(affected_package)
112+
else:
113+
ref = ReferenceV2(
114+
reference_id=commit_id, reference_type="commit", url=commit_url
115+
)
116+
references.append(ref)
117+
118+
url = (
119+
"https://raw.githubusercontent.com/quarkslab/aosp_dataset/refs/heads/master/cves/"
120+
f"{quote(file_path.name)}"
109121
)
110122

111-
fixed_by_commits.append(fixed_commit)
112-
113-
yield AdvisoryData(
114-
advisory_id=vulnerability_id,
115-
summary=summary,
116-
references_v2=references,
117-
severities=severities,
118-
fixed_by_commits=fixed_by_commits,
119-
date_published=date_published,
120-
url=f"https://raw.githubusercontent.com/quarkslab/aosp_dataset/refs/heads/master/cves/{file_path.name}",
121-
)
123+
yield AdvisoryData(
124+
advisory_id=vulnerability_id,
125+
summary=summary,
126+
affected_packages=affected_packages,
127+
references_v2=references,
128+
severities=severities,
129+
date_published=date_published,
130+
url=url,
131+
)
122132

123133
def clean_downloads(self):
124134
"""Cleanup any temporary repository data."""
125-
self.log("Cleaning up local repository resources.")
126-
if hasattr(self, "repo") and self.repo.working_dir:
127-
shutil.rmtree(path=self.repo.working_dir)
135+
if self.vcs_response:
136+
self.log(f"Removing cloned repository")
137+
self.vcs_response.delete()
128138

129139
def on_failure(self):
130140
"""Ensure cleanup is always performed on failure."""

‎vulnerabilities/tests/test_data/aosp/aosp_advisoryv2-expected.json‎

Lines changed: 145 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,14 +22,120 @@
2222
"weaknesses": [],
2323
"url": "https://raw.githubusercontent.com/quarkslab/aosp_dataset/refs/heads/master/cves/CVE-aosp_test1.json"
2424
},
25+
{
26+
"advisory_id": "CVE-2017-13077",
27+
"aliases": [],
28+
"summary": "Elevation of Privilege Vulnerability",
29+
"affected_packages": [],
30+
"references_v2": [
31+
{
32+
"reference_id": "c66556ca2473620df9751e73eb97ec50a40ffd3e",
33+
"reference_type": "commit",
34+
"url": "https://android.googlesource.com/platform/external/wpa_supplicant_8/+/c66556ca2473620df9751e73eb97ec50a40ffd3e"
35+
},
36+
{
37+
"reference_id": "",
38+
"reference_type": "commit",
39+
"url": "https://source.codeaurora.org/quic/la/platform/vendor/qcom-opensource/wlan/qcacld-3.0/commit/?id=776f17c87599fae3202e69bb5718ac9062f14695"
40+
},
41+
{
42+
"reference_id": "",
43+
"reference_type": "commit",
44+
"url": "https://source.codeaurora.org/quic/la/platform/vendor/qcom-opensource/wlan/qcacld-2.0/commit/?id=edb507885fc47cf3cdf061bfba1dc77451a6a332"
45+
}
46+
],
47+
"severities": [
48+
{
49+
"system": "generic_textual",
50+
"value": "High",
51+
"scoring_elements": ""
52+
}
53+
],
54+
"date_published": "2017-10-17T00:00:00+00:00",
55+
"weaknesses": [],
56+
"url": "https://raw.githubusercontent.com/quarkslab/aosp_dataset/refs/heads/master/cves/CVE-aosp_test5.json"
57+
},
58+
{
59+
"advisory_id": "CVE-2014-9322",
60+
"aliases": [],
61+
"summary": "Elevation of Privilege Vulnerability",
62+
"affected_packages": [],
63+
"references_v2": [
64+
{
65+
"reference_id": "c22e479e335628ce8766cfbf06e2ba17e8f9a1bb",
66+
"reference_type": "commit",
67+
"url": "https://android.googlesource.com/kernel/common/+/c22e479e335628ce8766cfbf06e2ba17e8f9a1bb"
68+
},
69+
{
70+
"reference_id": "1b627d4e5e61e89b840f77abb3ca6711ad6ffbeb",
71+
"reference_type": "commit",
72+
"url": "https://android.googlesource.com/kernel/common/+/1b627d4e5e61e89b840f77abb3ca6711ad6ffbeb"
73+
},
74+
{
75+
"reference_id": "4c941665c7368a34b146929b31949555e680a4ee",
76+
"reference_type": "commit",
77+
"url": "https://android.googlesource.com/kernel/common/+/4c941665c7368a34b146929b31949555e680a4ee"
78+
},
79+
{
80+
"reference_id": "758f0dac9104b46016af98304656a0268ac3e105",
81+
"reference_type": "commit",
82+
"url": "https://android.googlesource.com/kernel/common/+/758f0dac9104b46016af98304656a0268ac3e105"
83+
},
84+
{
85+
"reference_id": "44d057a37868a60bc2eb6e7d1dcea701f234d56a",
86+
"reference_type": "commit",
87+
"url": "https://android.googlesource.com/kernel/common/+/44d057a37868a60bc2eb6e7d1dcea701f234d56a"
88+
},
89+
{
90+
"reference_id": "b9b9f908c8ae82b73b9d75181982028b6bc06c2b",
91+
"reference_type": "commit",
92+
"url": "https://android.googlesource.com/kernel/common/+/b9b9f908c8ae82b73b9d75181982028b6bc06c2b"
93+
},
94+
{
95+
"reference_id": "e068734f9e7344997a61022629b92d142a985ab3",
96+
"reference_type": "commit",
97+
"url": "https://android.googlesource.com/kernel/common/+/e068734f9e7344997a61022629b92d142a985ab3"
98+
},
99+
{
100+
"reference_id": "fdc6c1052bc7d89a5826904fbb4318677e8442ce",
101+
"reference_type": "commit",
102+
"url": "https://android.googlesource.com/kernel/common/+/fdc6c1052bc7d89a5826904fbb4318677e8442ce"
103+
},
104+
{
105+
"reference_id": "211d59c0034ec9d88690c750ccd6da27f6952dc5",
106+
"reference_type": "commit",
107+
"url": "https://android.googlesource.com/kernel/common/+/211d59c0034ec9d88690c750ccd6da27f6952dc5"
108+
},
109+
{
110+
"reference_id": "c9e31d5a4747e9967ace6d05896c78516c4c0850",
111+
"reference_type": "commit",
112+
"url": "https://android.googlesource.com/kernel/common/+/c9e31d5a4747e9967ace6d05896c78516c4c0850"
113+
},
114+
{
115+
"reference_id": "e01834bfbafd25fd392bf10014451c4e5f34f829",
116+
"reference_type": "commit",
117+
"url": "https://android.googlesource.com/kernel/common/+/e01834bfbafd25fd392bf10014451c4e5f34f829"
118+
}
119+
],
120+
"severities": [
121+
{
122+
"system": "generic_textual",
123+
"value": "Critical",
124+
"scoring_elements": ""
125+
}
126+
],
127+
"date_published": "2015-12-25T00:00:00+00:00",
128+
"weaknesses": [],
129+
"url": "https://raw.githubusercontent.com/quarkslab/aosp_dataset/refs/heads/master/cves/CVE-aosp_test4.json"
130+
},
25131
{
26132
"advisory_id": "CVE-2017-13282",
27133
"aliases": [],
28134
"summary": "Remote Code Execution Vulnerability",
29135
"affected_packages": [],
30136
"references_v2": [
31137
{
32-
"reference_id": "",
138+
"reference_id": "6ecbbc093f4383e90cbbf681cd55da1303a8ef94",
33139
"reference_type": "commit",
34140
"url": "https://android.googlesource.com/platform/system/bt/+/6ecbbc093f4383e90cbbf681cd55da1303a8ef94"
35141
}
@@ -44,5 +150,43 @@
44150
"date_published": "2018-04-04T00:00:00+00:00",
45151
"weaknesses": [],
46152
"url": "https://raw.githubusercontent.com/quarkslab/aosp_dataset/refs/heads/master/cves/CVE-aosp_test2.json"
153+
},
154+
{
155+
"advisory_id": "CVE-2015-9016",
156+
"aliases": [],
157+
"summary": "Elevation of Privilege Vulnerability",
158+
"affected_packages": [
159+
{
160+
"package": {
161+
"type": "github",
162+
"namespace": "torvalds",
163+
"name": "linux",
164+
"version": "",
165+
"qualifiers": "",
166+
"subpath": ""
167+
},
168+
"affected_version_range": null,
169+
"fixed_version_range": null,
170+
"introduced_by_commits": [],
171+
"fixed_by_commits": [
172+
{
173+
"commit_hash": "0048b4837affd153897ed1222283492070027aa9",
174+
"vcs_url": "https://github.com/torvalds/linux.git",
175+
"commit_patch": null
176+
}
177+
]
178+
}
179+
],
180+
"references_v2": [],
181+
"severities": [
182+
{
183+
"system": "generic_textual",
184+
"value": "High",
185+
"scoring_elements": ""
186+
}
187+
],
188+
"date_published": "2018-04-05T00:00:00+00:00",
189+
"weaknesses": [],
190+
"url": "https://raw.githubusercontent.com/quarkslab/aosp_dataset/refs/heads/master/cves/CVE-aosp_test3.json"
47191
}
48192
]

‎vulnerabilities/tests/test_data/aosp/cves/CVE-aosp_test3.json‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"cveId": "CVE-2015-9016",
3+
"dateReported": "2018-04-05",
4+
"vulnerabilityType": "Elevation of Privilege Vulnerability",
5+
"language": "unknown",
6+
"fixes": [
7+
{
8+
"commitId": "",
9+
"patchUrl": "https://github.com/torvalds/linux/commit/0048b4837affd153897ed1222283492070027aa9"
10+
}
11+
],
12+
"severity": "High",
13+
"component": "Kernel Multi-queue block IO"
14+
}

0 commit comments

Comments
 (0)