From acf06daee792c967add79c670db4f96cd74f1d61 Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Thu, 9 Apr 2026 10:24:25 +0200 Subject: [PATCH 1/2] fix: skip marker-constrained uninstalled packages in Python requirements When a requirements.txt contains packages with PEP 508 environment markers (e.g., `pywin32==306 ; platform_system == "Windows"`), pip only installs packages whose markers match the current platform. Previously, the JavaScript client scanned ALL packages from the manifest regardless of markers, throwing an error when a marker- constrained package was not installed. This fix uses tree-sitter to detect `marker_spec` nodes on each requirement and skips packages that have a marker but are absent from the installed packages cache (`pip freeze`). Packages with markers that ARE installed are still included in the SBOM. Implements TC-4043 Assisted-by: Claude Code Co-Authored-By: Claude Opus 4.6 --- src/providers/python_controller.js | 10 ++-- test/providers/python_pip.test.js | 38 +++++++++++++++ .../expected_component_sbom.json | 48 +++++++++++++++++++ .../requirements.txt | 3 ++ 4 files changed, 96 insertions(+), 3 deletions(-) create mode 100644 test/providers/tst_manifests/pip/pip_requirements_txt_marker_skip/expected_component_sbom.json create mode 100644 test/providers/tst_manifests/pip/pip_requirements_txt_marker_skip/requirements.txt diff --git a/src/providers/python_controller.js b/src/providers/python_controller.js index 1b6cff2d..c8ede79b 100644 --- a/src/providers/python_controller.js +++ b/src/providers/python_controller.js @@ -97,7 +97,7 @@ export default class Python_controller { /** * Parse the requirements.txt file using tree-sitter and return structured requirement data. - * @return {Promise<{name: string, version: string|null}[]>} + * @return {Promise<{name: string, version: string|null, hasMarker: boolean}[]>} */ async #parseRequirements() { const content = fs.readFileSync(this.pathToRequirements).toString(); @@ -109,7 +109,8 @@ export default class Python_controller { const version = versionMatches.length > 0 ? versionMatches[0].captures.find(c => c.name === 'version').node.text : null; - return { name, version }; + const hasMarker = reqNode.children.some(c => c.type === 'marker_spec'); + return { name, version, hasMarker }; })); } @@ -224,7 +225,10 @@ export default class Python_controller { CachedEnvironmentDeps[packageName.replace("_", "-")] = pipDepTreeEntryForCache }) } - parsedRequirements.forEach(({ name: depName, version: manifestVersion }) => { + parsedRequirements.forEach(({ name: depName, version: manifestVersion, hasMarker }) => { + if(hasMarker && CachedEnvironmentDeps[depName.toLowerCase()] === undefined) { + return + } if(matchManifestVersions === "true" && manifestVersion != null) { let installedVersion if(CachedEnvironmentDeps[depName.toLowerCase()] !== undefined) { diff --git a/test/providers/python_pip.test.js b/test/providers/python_pip.test.js index 0d1666d6..d1b338c8 100644 --- a/test/providers/python_pip.test.js +++ b/test/providers/python_pip.test.js @@ -115,3 +115,41 @@ suite('testing the python-pip data provider with virtual environment', () => { }) }).beforeAll(() => {clock = useFakeTimers(new Date('2023-10-01T00:00:00.000Z'))}).afterAll(()=> clock.restore()); + +suite('testing python-pip PEP 508 marker handling', () => { + const markerTestCase = 'pip_requirements_txt_marker_skip' + + /** Verify that packages with environment markers (PEP 508) that are not installed + * in the current environment are silently skipped, while marker-constrained + * packages that ARE installed are still included in the SBOM. */ + test('verify marker-constrained uninstalled packages are skipped in component analysis', async () => { + // given: pip environment where only six and certifi are installed (pywin32 is Windows-only) + const pipFreezeOutput = 'six==1.16.0\ncertifi==2023.7.22\n' + const pipShowOutput = + 'Name: certifi\nVersion: 2023.7.22\nSummary: Python package for providing Mozilla\'s CA Bundle.\nRequires: \nRequired-by: ' + + '\n---\n' + + 'Name: six\nVersion: 1.16.0\nSummary: Python 2 and 3 compatibility utilities\nRequires: \nRequired-by: ' + + process.env['TRUSTIFY_DA_PIP_FREEZE'] = Buffer.from(pipFreezeOutput).toString('base64') + process.env['TRUSTIFY_DA_PIP_SHOW'] = Buffer.from(pipShowOutput).toString('base64') + + try { + // when: component analysis is run against a manifest with a Windows-only marker package + let expectedSbom = fs.readFileSync(`test/providers/tst_manifests/pip/${markerTestCase}/expected_component_sbom.json`).toString().trim() + expectedSbom = JSON.stringify(JSON.parse(expectedSbom)) + + let result = await pythonPip.provideComponent(`test/providers/tst_manifests/pip/${markerTestCase}/requirements.txt`, {}) + + // then: SBOM contains six and certifi but not pywin32 + expect(result).to.deep.equal({ + ecosystem: 'pip', + contentType: 'application/vnd.cyclonedx+json', + content: expectedSbom + }) + } finally { + delete process.env['TRUSTIFY_DA_PIP_FREEZE'] + delete process.env['TRUSTIFY_DA_PIP_SHOW'] + } + }).timeout(10000) + +}).beforeAll(() => clock = useFakeTimers(new Date('2023-10-01T00:00:00.000Z'))).afterAll(() => clock.restore()); diff --git a/test/providers/tst_manifests/pip/pip_requirements_txt_marker_skip/expected_component_sbom.json b/test/providers/tst_manifests/pip/pip_requirements_txt_marker_skip/expected_component_sbom.json new file mode 100644 index 00000000..b82e91af --- /dev/null +++ b/test/providers/tst_manifests/pip/pip_requirements_txt_marker_skip/expected_component_sbom.json @@ -0,0 +1,48 @@ +{ + "bomFormat": "CycloneDX", + "specVersion": "1.4", + "version": 1, + "metadata": { + "timestamp": "2023-10-01T00:00:00.000Z", + "component": { + "name": "default-pip-root", + "version": "0.0.0", + "purl": "pkg:pypi/default-pip-root@0.0.0", + "type": "application", + "bom-ref": "pkg:pypi/default-pip-root@0.0.0" + } + }, + "components": [ + { + "name": "certifi", + "version": "2023.7.22", + "purl": "pkg:pypi/certifi@2023.7.22", + "type": "library", + "bom-ref": "pkg:pypi/certifi@2023.7.22" + }, + { + "name": "six", + "version": "1.16.0", + "purl": "pkg:pypi/six@1.16.0", + "type": "library", + "bom-ref": "pkg:pypi/six@1.16.0" + } + ], + "dependencies": [ + { + "ref": "pkg:pypi/default-pip-root@0.0.0", + "dependsOn": [ + "pkg:pypi/certifi@2023.7.22", + "pkg:pypi/six@1.16.0" + ] + }, + { + "ref": "pkg:pypi/certifi@2023.7.22", + "dependsOn": [] + }, + { + "ref": "pkg:pypi/six@1.16.0", + "dependsOn": [] + } + ] +} diff --git a/test/providers/tst_manifests/pip/pip_requirements_txt_marker_skip/requirements.txt b/test/providers/tst_manifests/pip/pip_requirements_txt_marker_skip/requirements.txt new file mode 100644 index 00000000..f8c35ddd --- /dev/null +++ b/test/providers/tst_manifests/pip/pip_requirements_txt_marker_skip/requirements.txt @@ -0,0 +1,3 @@ +six==1.16.0 +certifi==2023.7.22 ; python_version >= "3" +pywin32==306 ; platform_system == "Windows" From d2628e77edf87d96d80d148dc7cf48f5dc87dcea Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Tue, 21 Apr 2026 16:22:38 +0200 Subject: [PATCH 2/2] fix(test): pin pip transitive deps to prevent version drift in fixtures Transitive dependencies (idna, charset-normalizer, urllib3, certifi, pysocks) were resolved live from PyPI, causing test failures when new versions are published (e.g. idna 3.11 -> 3.12). Pin all deps with == in the test pyproject.toml, matching the lock file approach used by uv and poetry tests. Co-Authored-By: Claude Opus 4.6 --- test/providers/python_pyproject.test.js | 19 +++--- .../pip_pep621/expected_component_sbom.json | 62 ++++++++++++++++++- .../pip_pep621/expected_stack_sbom.json | 17 +++-- .../pyproject/pip_pep621/pyproject.toml | 5 ++ 4 files changed, 89 insertions(+), 14 deletions(-) diff --git a/test/providers/python_pyproject.test.js b/test/providers/python_pyproject.test.js index 662d7c2f..66d0c56a 100644 --- a/test/providers/python_pyproject.test.js +++ b/test/providers/python_pyproject.test.js @@ -272,14 +272,19 @@ suite('testing the python-pyproject data provider', () => { let result = await pipProvider.provideStack(path.join(pipFixtureDir, 'pyproject.toml')) let sbom = JSON.parse(result.content) let rootDep = sbom.dependencies.find(d => d.ref.includes('/test-project@')) - expect(rootDep.dependsOn).to.have.lengthOf(1) - expect(rootDep.dependsOn[0]).to.include('/requests@') + let directNames = rootDep.dependsOn.map(d => d.split('/').pop().split('@')[0]) + expect(directNames).to.include('requests') + expect(directNames).to.include('charset-normalizer') + expect(directNames).to.include('idna') + expect(directNames).to.include('urllib3') + expect(directNames).to.include('certifi') + expect(directNames).to.include('pysocks') let requestsDep = sbom.dependencies.find(d => d.ref.includes('/requests@')) - let transNames = requestsDep.dependsOn.map(d => d.split('/').pop().split('@')[0]) - expect(transNames).to.include('certifi') - expect(transNames).to.include('charset-normalizer') - expect(transNames).to.include('idna') - expect(transNames).to.include('urllib3') + let reqTransNames = requestsDep.dependsOn.map(d => d.split('/').pop().split('@')[0]) + expect(reqTransNames).to.include('certifi') + expect(reqTransNames).to.include('charset-normalizer') + expect(reqTransNames).to.include('idna') + expect(reqTransNames).to.include('urllib3') }).timeout(TIMEOUT) /** Verifies exhortignore marker produces expected SBOM for stack and component analysis. */ diff --git a/test/providers/tst_manifests/pyproject/pip_pep621/expected_component_sbom.json b/test/providers/tst_manifests/pyproject/pip_pep621/expected_component_sbom.json index 583a8dff..f2b0e3e6 100644 --- a/test/providers/tst_manifests/pyproject/pip_pep621/expected_component_sbom.json +++ b/test/providers/tst_manifests/pyproject/pip_pep621/expected_component_sbom.json @@ -19,18 +19,78 @@ "purl": "pkg:pypi/requests@2.32.3", "type": "library", "bom-ref": "pkg:pypi/requests@2.32.3" + }, + { + "name": "charset-normalizer", + "version": "3.4.7", + "purl": "pkg:pypi/charset-normalizer@3.4.7", + "type": "library", + "bom-ref": "pkg:pypi/charset-normalizer@3.4.7" + }, + { + "name": "idna", + "version": "3.12", + "purl": "pkg:pypi/idna@3.12", + "type": "library", + "bom-ref": "pkg:pypi/idna@3.12" + }, + { + "name": "urllib3", + "version": "2.6.3", + "purl": "pkg:pypi/urllib3@2.6.3", + "type": "library", + "bom-ref": "pkg:pypi/urllib3@2.6.3" + }, + { + "name": "certifi", + "version": "2026.2.25", + "purl": "pkg:pypi/certifi@2026.2.25", + "type": "library", + "bom-ref": "pkg:pypi/certifi@2026.2.25" + }, + { + "name": "pysocks", + "version": "1.7.1", + "purl": "pkg:pypi/pysocks@1.7.1", + "type": "library", + "bom-ref": "pkg:pypi/pysocks@1.7.1" } ], "dependencies": [ { "ref": "pkg:pypi/test-project@1.0.0", "dependsOn": [ - "pkg:pypi/requests@2.32.3" + "pkg:pypi/requests@2.32.3", + "pkg:pypi/charset-normalizer@3.4.7", + "pkg:pypi/idna@3.12", + "pkg:pypi/urllib3@2.6.3", + "pkg:pypi/certifi@2026.2.25", + "pkg:pypi/pysocks@1.7.1" ] }, { "ref": "pkg:pypi/requests@2.32.3", "dependsOn": [] + }, + { + "ref": "pkg:pypi/charset-normalizer@3.4.7", + "dependsOn": [] + }, + { + "ref": "pkg:pypi/idna@3.12", + "dependsOn": [] + }, + { + "ref": "pkg:pypi/urllib3@2.6.3", + "dependsOn": [] + }, + { + "ref": "pkg:pypi/certifi@2026.2.25", + "dependsOn": [] + }, + { + "ref": "pkg:pypi/pysocks@1.7.1", + "dependsOn": [] } ] } diff --git a/test/providers/tst_manifests/pyproject/pip_pep621/expected_stack_sbom.json b/test/providers/tst_manifests/pyproject/pip_pep621/expected_stack_sbom.json index 1d0849c7..2abb05de 100644 --- a/test/providers/tst_manifests/pyproject/pip_pep621/expected_stack_sbom.json +++ b/test/providers/tst_manifests/pyproject/pip_pep621/expected_stack_sbom.json @@ -29,10 +29,10 @@ }, { "name": "idna", - "version": "3.11", - "purl": "pkg:pypi/idna@3.11", + "version": "3.12", + "purl": "pkg:pypi/idna@3.12", "type": "library", - "bom-ref": "pkg:pypi/idna@3.11" + "bom-ref": "pkg:pypi/idna@3.12" }, { "name": "urllib3", @@ -60,14 +60,19 @@ { "ref": "pkg:pypi/test-project@1.0.0", "dependsOn": [ - "pkg:pypi/requests@2.32.3" + "pkg:pypi/requests@2.32.3", + "pkg:pypi/charset-normalizer@3.4.7", + "pkg:pypi/idna@3.12", + "pkg:pypi/urllib3@2.6.3", + "pkg:pypi/certifi@2026.2.25", + "pkg:pypi/pysocks@1.7.1" ] }, { "ref": "pkg:pypi/requests@2.32.3", "dependsOn": [ "pkg:pypi/charset-normalizer@3.4.7", - "pkg:pypi/idna@3.11", + "pkg:pypi/idna@3.12", "pkg:pypi/urllib3@2.6.3", "pkg:pypi/certifi@2026.2.25", "pkg:pypi/pysocks@1.7.1" @@ -78,7 +83,7 @@ "dependsOn": [] }, { - "ref": "pkg:pypi/idna@3.11", + "ref": "pkg:pypi/idna@3.12", "dependsOn": [] }, { diff --git a/test/providers/tst_manifests/pyproject/pip_pep621/pyproject.toml b/test/providers/tst_manifests/pyproject/pip_pep621/pyproject.toml index 14d1ba70..affbdfdb 100644 --- a/test/providers/tst_manifests/pyproject/pip_pep621/pyproject.toml +++ b/test/providers/tst_manifests/pyproject/pip_pep621/pyproject.toml @@ -4,4 +4,9 @@ version = "1.0.0" requires-python = ">=3.9" dependencies = [ "requests[socks]==2.32.3", + "charset-normalizer==3.4.7", + "idna==3.12", + "urllib3[socks]==2.6.3", + "certifi==2026.2.25", + "pysocks==1.7.1", ]