[SP-4275] feat: extract and add cpe's to spdx output (#204)

* [SP-4275] feat: extract and add cpe's to spdx output

* [SP-4275] chore: normalize cpe to lowercase

Author: matiasdaloia

.gitignore

@@ -34,3 +34,4 @@ docs/build
 .DS_Store
 !scanoss.json
 examples/output/
+!spdx-*.json

CHANGELOG.md

@@ -8,6 +8,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 
+## [1.52.1] - 2026-04-14
+### Fixed
+- Fixed CPE identifiers missing from SPDX Lite output (`--format spdxlite`)
+  - CPEs are now emitted as SPDX 2.2 `externalRefs` with `referenceCategory: SECURITY`
+  - CPE 2.3 strings use `referenceType: cpe23Type`; legacy `cpe:/...` and `cpe:2.2:...` use `cpe22Type`
+  - Multiple CPEs per component are preserved and deduplicated
+
 ## [1.52.0] - 2026-04-09
 ### Added
 - Added `status` subcommand query to `component` command to retrieve development life-cycle status:
@@ -877,3 +884,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 [1.51.0]: https://github.com/scanoss/scanoss.py/compare/v1.50.1...v1.51.0
 [1.51.1]: https://github.com/scanoss/scanoss.py/compare/v1.51.0...v1.51.1
 [1.52.0]: https://github.com/scanoss/scanoss.py/compare/v1.51.1...v1.52.0
+[1.52.0]: https://github.com/scanoss/scanoss.py/compare/v1.51.1...v1.52.1

src/scanoss/__init__.py

@@ -22,4 +22,4 @@
   THE SOFTWARE.
 """
 
-__version__ = '1.52.0'
+__version__ = '1.52.1'

src/scanoss/spdxlite.py

@@ -219,8 +219,41 @@ def _create_file_summary(self, entry: dict) -> dict:
         for field in fields:
             summary[field] = entry.get(field)
         summary['licenses'] = self._process_licenses(entry.get('licenses'))
+        summary['cpes'] = self._extract_cpes(entry.get('vulnerabilities'))
         return summary
 
+    def _extract_cpes(self, vulnerabilities: list) -> list:
+        """
+        Extract CPE identifiers from a file entry's vulnerabilities array.
+
+        Raw scan results deliver CPEs embedded as vulnerability IDs prefixed with "CPE:"
+        (case-insensitive). Everything else in the array is a real vulnerability record
+        (CVE/GHSA) and must be ignored here.
+
+        Args:
+            vulnerabilities (list): The 'vulnerabilities' list from a file match entry.
+                                    May be None or empty.
+
+        Returns:
+            list: Deduplicated list of CPE strings in source order (e.g.
+                  ['cpe:2.3:a:postgresql:postgresql:17.0:*:*:*:*:*:*:*']).
+                  Returns an empty list when there are no CPE entries.
+        """
+        if not vulnerabilities:
+            return []
+        cpes = []
+        seen = set()
+        for vuln in vulnerabilities:
+            vuln_id = vuln.get('ID') or vuln.get('id') or ''
+            if not vuln_id.upper().startswith('CPE:'):
+                continue
+            normalized = vuln_id.upper()
+            if normalized in seen:
+                continue
+            seen.add(normalized)
+            cpes.append(vuln_id)
+        return cpes
+
     def _process_licenses(self, licenses: list) -> list:
         """
             Process license information and remove duplicates.
@@ -426,6 +459,15 @@ def _create_package_info(self, purl: str, comp: dict, lic_refs: set) -> dict:
         purl_ver = f'{purl}@{comp_ver}'
         purl_hash = hashlib.md5(purl_ver.encode('utf-8')).hexdigest()
 
+        external_refs = [
+            {
+                'referenceCategory': 'PACKAGE-MANAGER',
+                'referenceLocator': PackageURL.from_string(purl_ver).to_string(),
+                'referenceType': 'purl'
+            }
+        ]
+        external_refs.extend(self._create_cpe_external_refs(comp.get('cpes', [])))
+
         return {
             'name': comp.get('component'),
             'SPDXID': f'SPDXRef-{purl_hash}',
@@ -437,13 +479,7 @@ def _create_package_info(self, purl: str, comp: dict, lic_refs: set) -> dict:
             'filesAnalyzed': False,
             'copyrightText': 'NOASSERTION',
             'supplier': f'Organization: {comp.get("vendor", "NOASSERTION")}',
-            'externalRefs': [
-                {
-                    'referenceCategory': 'PACKAGE-MANAGER',
-                    'referenceLocator': PackageURL.from_string(purl_ver).to_string(),
-                    'referenceType': 'purl'
-                }
-            ],
+            'externalRefs': external_refs,
             'checksums': [
                 {
                     'algorithm': 'MD5',
@@ -452,6 +488,48 @@ def _create_package_info(self, purl: str, comp: dict, lic_refs: set) -> dict:
             ],
         }
 
+    def _create_cpe_external_refs(self, cpes: list) -> list:
+        """
+        Build SPDX externalRefs entries for a component's CPE identifiers.
+
+        SPDX 2.2 models CPEs under the SECURITY reference category. Each CPE string
+        must be emitted as its own externalRef dict with the shape:
+
+            {
+                'referenceCategory': 'SECURITY',
+                'referenceType': 'cpe23Type' | 'cpe22Type',
+                'referenceLocator': '<cpe string>',
+            }
+
+        Args:
+            cpes (list): CPE strings extracted from the raw scan results. The list is
+                         already deduplicated by `_extract_cpes`. Values look like
+                         'cpe:2.3:a:vendor:product:version:...' (CPE 2.3) or
+                         'cpe:/a:vendor:product:version' (legacy CPE 2.2). May be empty.
+
+        Returns:
+            list: A list of SPDX externalRef dicts ready to be appended to a package's
+                  `externalRefs`. Return an empty list when `cpes` is empty.
+        """
+        if not cpes:
+            return []
+        refs = []
+        for cpe in cpes:
+            normalized = cpe.lower()
+            if normalized.startswith('cpe:2.3:'):
+                ref_type = 'cpe23Type'
+            elif normalized.startswith('cpe:/') or normalized.startswith('cpe:2.2:'):
+                ref_type = 'cpe22Type'
+            else:
+                self.print_debug(f'Warning: Unrecognized CPE format, defaulting to cpe23Type: {cpe}')
+                ref_type = 'cpe23Type'
+            refs.append({
+                'referenceCategory': 'SECURITY',
+                'referenceType': ref_type,
+                'referenceLocator': cpe,
+            })
+        return refs
+
     def _process_package_licenses(self, licenses: list, lic_refs: set) -> str:
         """
            Process licenses and return license text formatted for SPDX.

tests/test_spdxlite.py

@@ -67,4 +67,165 @@ def testSpdxLite(self):
                     self.assertEqual(len(checksum.get("checksumValue")), md5_length) #Check checksum length value be 32
 
 
-        os.remove(spdx_lite_output) #Removes tmp spdxlite.json file
+        os.remove(spdx_lite_output) #Removes tmp spdxlite.json file
+
+
+class SpdxLiteCpeTests(unittest.TestCase):
+    """
+    Exercise CPE extraction and SPDX externalRefs emission.
+    """
+
+    @staticmethod
+    def _build_raw(vulnerabilities, purl='pkg:github/postgres/postgres'):
+        return {
+            'src/main.c': [{
+                'id': 'file',
+                'component': 'postgresql',
+                'vendor': 'postgresql',
+                'version': '17.0',
+                'latest': '17.0',
+                'url': 'https://www.postgresql.org',
+                'url_hash': 'abc123',
+                'download_url': 'https://example.com/pg.tar.gz',
+                'purl': [purl],
+                'licenses': [{'name': 'PostgreSQL', 'source': 'component_declared'}],
+                'vulnerabilities': vulnerabilities,
+            }]
+        }
+
+    def _run(self, raw):
+        fd, out_path = tempfile.mkstemp(prefix='spdxlite_cpe_', suffix='.json')
+        os.close(fd)  # SpdxLite re-opens the path itself for writing
+        try:
+            spdx = SpdxLite(debug=False, output_file=out_path)
+            spdx.produce_from_json(raw)
+            with open(out_path, 'r') as f:
+                return json.load(f)
+        finally:
+            if os.path.exists(out_path):
+                os.remove(out_path)
+
+    def _security_refs(self, doc):
+        refs = doc['packages'][0]['externalRefs']
+        return [r for r in refs if r['referenceCategory'] == 'SECURITY']
+
+    def test_cpe23_emits_cpe23Type(self):
+        cpe = 'cpe:2.3:a:postgresql:postgresql:17.0:*:*:*:*:*:*:*'
+        doc = self._run(self._build_raw([{'ID': cpe, 'source': 'nvd'}]))
+        refs = self._security_refs(doc)
+        self.assertEqual(len(refs), 1)
+        self.assertEqual(refs[0]['referenceType'], 'cpe23Type')
+        self.assertEqual(refs[0]['referenceLocator'], cpe)
+
+    def test_legacy_cpe22_slash_emits_cpe22Type(self):
+        cpe = 'cpe:/a:postgresql:postgresql:17.0'
+        doc = self._run(self._build_raw([{'ID': cpe, 'source': 'nvd'}]))
+        refs = self._security_refs(doc)
+        self.assertEqual(len(refs), 1)
+        self.assertEqual(refs[0]['referenceType'], 'cpe22Type')
+        self.assertEqual(refs[0]['referenceLocator'], cpe)
+
+    def test_explicit_cpe22_prefix_emits_cpe22Type(self):
+        cpe = 'cpe:2.2:a:postgresql:postgresql:17.0'
+        doc = self._run(self._build_raw([{'ID': cpe, 'source': 'nvd'}]))
+        refs = self._security_refs(doc)
+        self.assertEqual(len(refs), 1)
+        self.assertEqual(refs[0]['referenceType'], 'cpe22Type')
+
+    def test_case_insensitive_prefix_detection(self):
+        cpe = 'CPE:2.3:a:postgresql:postgresql:17.0:*:*:*:*:*:*:*'
+        doc = self._run(self._build_raw([{'ID': cpe, 'source': 'nvd'}]))
+        refs = self._security_refs(doc)
+        self.assertEqual(len(refs), 1)
+        self.assertEqual(refs[0]['referenceType'], 'cpe23Type')
+        self.assertEqual(refs[0]['referenceLocator'], cpe)  # casing preserved in locator
+
+    def test_duplicate_cpes_are_deduplicated(self):
+        cpe = 'cpe:2.3:a:postgresql:postgresql:17.0:*:*:*:*:*:*:*'
+        doc = self._run(self._build_raw([
+            {'ID': cpe, 'source': 'nvd'},
+            {'ID': cpe, 'source': 'nvd'},
+            {'ID': cpe, 'source': 'nvd'},
+        ]))
+        refs = self._security_refs(doc)
+        self.assertEqual(len(refs), 1)
+
+    def test_dedup_is_case_insensitive_and_preserves_first_locator(self):
+        lower = 'cpe:2.3:a:postgresql:postgresql:17.0:*:*:*:*:*:*:*'
+        upper = 'CPE:2.3:A:POSTGRESQL:POSTGRESQL:17.0:*:*:*:*:*:*:*'
+        doc = self._run(self._build_raw([
+            {'ID': lower, 'source': 'nvd'},
+            {'ID': upper, 'source': 'nvd'},
+        ]))
+        refs = self._security_refs(doc)
+        self.assertEqual(len(refs), 1)
+        self.assertEqual(refs[0]['referenceLocator'], lower)  # first-seen wins
+
+    def test_cve_entries_are_ignored(self):
+        doc = self._run(self._build_raw([
+            {'ID': 'CVE-2024-12345', 'CVE': 'CVE-2024-12345',
+             'source': 'nvd', 'severity': 'high'},
+            {'ID': 'GHSA-xxxx-yyyy-zzzz', 'source': 'github'},
+        ]))
+        refs = self._security_refs(doc)
+        self.assertEqual(refs, [])
+
+    def test_mixed_cpe_versions_in_same_component(self):
+        cpe23 = 'cpe:2.3:a:postgresql:postgresql:17.0:*:*:*:*:*:*:*'
+        cpe22 = 'cpe:/a:postgresql:postgresql:17.0'
+        doc = self._run(self._build_raw([
+            {'ID': cpe23, 'source': 'nvd'},
+            {'ID': cpe22, 'source': 'nvd'},
+        ]))
+        refs = self._security_refs(doc)
+        self.assertEqual(len(refs), 2)
+        types = {r['referenceType']: r['referenceLocator'] for r in refs}
+        self.assertEqual(types['cpe23Type'], cpe23)
+        self.assertEqual(types['cpe22Type'], cpe22)
+
+    def test_unknown_cpe_format_falls_back_to_cpe23Type(self):
+        odd_cpe = 'cpe:weird-format:postgresql:17.0'
+        doc = self._run(self._build_raw([{'ID': odd_cpe, 'source': 'nvd'}]))
+        refs = self._security_refs(doc)
+        self.assertEqual(len(refs), 1)
+        self.assertEqual(refs[0]['referenceType'], 'cpe23Type')
+        self.assertEqual(refs[0]['referenceLocator'], odd_cpe)
+
+    def test_no_vulnerabilities_field_produces_no_security_refs(self):
+        raw = self._build_raw([])
+        # Drop the key entirely to simulate entries without a vulnerabilities block
+        del raw['src/main.c'][0]['vulnerabilities']
+        doc = self._run(raw)
+        self.assertEqual(self._security_refs(doc), [])
+        # PURL externalRef must still be present
+        refs = doc['packages'][0]['externalRefs']
+        self.assertEqual(len(refs), 1)
+        self.assertEqual(refs[0]['referenceType'], 'purl')
+
+    def test_empty_vulnerabilities_list_produces_no_security_refs(self):
+        doc = self._run(self._build_raw([]))
+        self.assertEqual(self._security_refs(doc), [])
+
+    def test_dependency_entries_do_not_emit_cpes(self):
+        raw = {
+            'package.json': [{
+                'id': 'dependency',
+                'dependencies': [{
+                    'purl': 'pkg:npm/left-pad',
+                    'component': 'left-pad',
+                    'version': '1.3.0',
+                    'url': 'https://npmjs.com/package/left-pad',
+                    'licenses': [{'name': 'MIT', 'source': 'component_declared'}],
+                }]
+            }]
+        }
+        doc = self._run(raw)
+        self.assertEqual(self._security_refs(doc), [])
+
+    def test_lowercase_id_key_is_also_supported(self):
+        cpe = 'cpe:2.3:a:postgresql:postgresql:17.0:*:*:*:*:*:*:*'
+        # Raw scan output has been known to use 'id' (lowercase) occasionally
+        doc = self._run(self._build_raw([{'id': cpe, 'source': 'nvd'}]))
+        refs = self._security_refs(doc)
+        self.assertEqual(len(refs), 1)
+        self.assertEqual(refs[0]['referenceType'], 'cpe23Type')