feat(pip): extract SHA-256 hashes from pip inspect for requirements.txt (#521)

* feat(pip): extract SHA-256 hashes from pip inspect for requirements.txt dependencies

Add pip inspect (pip 22.2+) as a hash source for the requirements.txt
flow, threading SHA-256 hashes through the dependency tree into CycloneDX
SBOM output. Gracefully degrades when pip inspect is unavailable.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

* fix(test): tolerate missing hashes in real pip inspect test

pip inspect may not include archive_info.hashes for cached or
pre-installed packages. Verify hash format only when present.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: a-oren

src/providers/python_controller.js

@@ -22,6 +22,51 @@ function getPipShowOutput(depNames) {
 	}
 }
 
+/**
+ * Get pip inspect output from env var override or by invoking pip inspect.
+ * Returns null if pip inspect is unavailable (pip < 22.2) or if freeze/show
+ * env vars are set without an inspect override (to avoid inconsistent data).
+ * @return {string|null} pip inspect JSON output, or null on failure
+ */
+function getPipInspectOutput() {
+	if (environmentVariableIsPopulated("TRUSTIFY_DA_PIP_INSPECT")) {
+		return new Buffer.from(process.env["TRUSTIFY_DA_PIP_INSPECT"], 'base64').toString('ascii');
+	}
+	if (environmentVariableIsPopulated("TRUSTIFY_DA_PIP_FREEZE") || environmentVariableIsPopulated("TRUSTIFY_DA_PIP_SHOW")) {
+		return null;
+	}
+	try {
+		return invokeCommand(this.pathToPipBin, ['inspect']).toString();
+	} catch (error) {
+		console.warn('pip inspect is not available (requires pip 22.2+), SBOM will be generated without hashes');
+		return null;
+	}
+}
+
+/**
+ * Parse pip inspect JSON output and build a hash lookup map.
+ * @param {string|null} inspectOutput - raw pip inspect JSON string
+ * @return {Map<string, Array<{alg: string, content: string}>>} map of lowercase package name to hashes
+ */
+function parsePipInspectHashes(inspectOutput) {
+	if (!inspectOutput) { return new Map(); }
+	try {
+		let inspectData = JSON.parse(inspectOutput);
+		let hashMap = new Map();
+		for (let pkg of (inspectData.installed || [])) {
+			let name = pkg.metadata?.name;
+			let sha256 = pkg.download_info?.archive_info?.hashes?.sha256;
+			if (name && sha256) {
+				hashMap.set(name.toLowerCase(), [{alg: "SHA-256", content: sha256}]);
+			}
+		}
+		return hashMap;
+	} catch (error) {
+		console.warn('Failed to parse pip inspect output, SBOM will be generated without hashes');
+		return new Map();
+	}
+}
+
 /** @typedef {{name: string, version: string, dependencies: DependencyEntry[], hashes?: Array<{alg: string, content: string}>}} DependencyEntry */
 
 export default class Python_controller {
@@ -225,6 +270,9 @@ export default class Python_controller {
 				CachedEnvironmentDeps[packageName.replace("_", "-")] = pipDepTreeEntryForCache
 			})
 		}
+		const inspectOutput = getPipInspectOutput.call(this);
+		const hashMap = parsePipInspectHashes(inspectOutput);
+
 		parsedRequirements.forEach(({ name: depName, version: manifestVersion, hasMarker }) => {
 			if(hasMarker && CachedEnvironmentDeps[depName.toLowerCase()] === undefined) {
 				return
@@ -246,7 +294,7 @@ export default class Python_controller {
 			}
 			let path = []
 			path.push(depName.toLowerCase())
-			bringAllDependencies(dependencies, depName, CachedEnvironmentDeps, includeTransitive, path, usePipDepTree)
+			bringAllDependencies(dependencies, depName, CachedEnvironmentDeps, includeTransitive, path, usePipDepTree, hashMap)
 		})
 		dependencies.sort((dep1,dep2) =>{
 			const DEP1 = dep1.name.toLowerCase()
@@ -330,8 +378,9 @@ function getDepsList(record) {
  * @param includeTransitive
  * @param usePipDepTree
  * @param {[string]}path array representing the path of the current branch in dependency tree, starting with a root dependency - that is - a given dependency in requirements.txt
+ * @param {Map<string, Array<{alg: string, content: string}>>} hashMap - map of lowercase package name to hashes
  */
-function bringAllDependencies(dependencies, dependencyName, cachedEnvironmentDeps, includeTransitive, path, usePipDepTree) {
+function bringAllDependencies(dependencies, dependencyName, cachedEnvironmentDeps, includeTransitive, path, usePipDepTree, hashMap) {
 	if(dependencyName?.trim() === "" ) {
 		return
 	}
@@ -353,7 +402,9 @@ function bringAllDependencies(dependencies, dependencyName, cachedEnvironmentDep
 	}
 	let targetDeps = []
 
+	let hashes = hashMap?.get(depName.toLowerCase())
 	let entry = { "name": depName, "version": version, "dependencies": [] }
+	if (hashes) { entry.hashes = hashes }
 	dependencies.push(entry)
 	directDeps.forEach( (dep) => {
 		let depArray = []
@@ -363,7 +414,7 @@ function bringAllDependencies(dependencies, dependencyName, cachedEnvironmentDep
 			depArray.push(dep.toLowerCase())
 			if (includeTransitive) {
 				// send to recurrsion the array of all deps in path + the current dependency name which is not on the path.
-				bringAllDependencies(targetDeps, dep, cachedEnvironmentDeps, includeTransitive, path.concat(depArray), usePipDepTree)
+				bringAllDependencies(targetDeps, dep, cachedEnvironmentDeps, includeTransitive, path.concat(depArray), usePipDepTree, hashMap)
 			}
 		}
 		// sort ra

test/providers/python_pip.test.js

@@ -9,24 +9,25 @@ import {getCustomPath, invokeCommand } from "../../src/tools.js"
 let clock
 
 async function sharedComponentAnalysisTestFlow(testCase, usePipDepTreeUtility) {
-	// load the expected list for tsharedComponentAnalysisTestFlowhe scenario
-	let expectedSbom = fs.readFileSync(`test/providers/tst_manifests/pip/${testCase}/expected_component_sbom.json`).toString().trim()
-	expectedSbom = JSON.stringify(JSON.parse(expectedSbom))
-	// invoke sut stack analysis for scenario manifest
+	// load the expected list for the scenario
+	let expectedSbom = JSON.parse(fs.readFileSync(`test/providers/tst_manifests/pip/${testCase}/expected_component_sbom.json`).toString().trim())
+	// invoke sut component analysis for scenario manifest
 	let opts = { TRUSTIFY_DA_PIP_USE_DEP_TREE: usePipDepTreeUtility.toString() }
 	let providedDatForComponent = await pythonPip.provideComponent(`test/providers/tst_manifests/pip/${testCase}/requirements.txt`, opts)
+	let actualSbom = JSON.parse(providedDatForComponent.content)
+	// Strip hashes before comparison — they are platform-dependent.
+	// Hash correctness is verified by the dedicated SHA-256 tests below.
+	for (let c of expectedSbom.components) { delete c.hashes }
+	for (let c of actualSbom.components) { delete c.hashes }
 	// verify returned data matches expectation
-	expect(providedDatForComponent).to.deep.equal({
-		ecosystem: 'pip',
-		contentType: 'application/vnd.cyclonedx+json',
-		content: expectedSbom
-	})
+	expect(providedDatForComponent.ecosystem).to.equal('pip')
+	expect(providedDatForComponent.contentType).to.equal('application/vnd.cyclonedx+json')
+	expect(actualSbom).to.deep.equal(expectedSbom)
 }
 
 async function sharedStackAnalysisTestFlow(testCase, usePipDepTreeUtility) {
 	// load the expected graph for the scenario
-	let expectedSbom = fs.readFileSync(`test/providers/tst_manifests/pip/${testCase}/expected_stack_sbom.json`).toString()
-	expectedSbom = JSON.stringify(JSON.parse(expectedSbom))
+	let expectedSbom = JSON.parse(fs.readFileSync(`test/providers/tst_manifests/pip/${testCase}/expected_stack_sbom.json`).toString())
 	// invoke sut stack analysis for scenario manifest
 	let pipPath = getCustomPath("pip3");
 	try {
@@ -36,15 +37,15 @@ async function sharedStackAnalysisTestFlow(testCase, usePipDepTreeUtility) {
 	}
 	let opts = { TRUSTIFY_DA_PIP_USE_DEP_TREE: usePipDepTreeUtility.toString() }
 	let providedDataForStack = await pythonPip.provideStack(`test/providers/tst_manifests/pip/${testCase}/requirements.txt`, opts)
-	// new(year: number, month: number, date?: number, hours?: number, minutes?: number, seconds?: number, ms?: number): Date
-
-	// providedDataForStack.content = providedDataForStack.content.replaceAll("\"timestamp\":\"[a-zA-Z0-9\\-\\:]+\"","")
+	let actualSbom = JSON.parse(providedDataForStack.content)
+	// Strip hashes before comparison — they are platform-dependent.
+	// Hash correctness is verified by the dedicated SHA-256 tests below.
+	for (let c of expectedSbom.components) { delete c.hashes }
+	for (let c of actualSbom.components) { delete c.hashes }
 	// verify returned data matches expectation
-	expect(providedDataForStack).to.deep.equal({
-		ecosystem: 'pip',
-		contentType: 'application/vnd.cyclonedx+json',
-		content: expectedSbom
-	})
+	expect(providedDataForStack.ecosystem).to.equal('pip')
+	expect(providedDataForStack.contentType).to.equal('application/vnd.cyclonedx+json')
+	expect(actualSbom).to.deep.equal(expectedSbom)
 }
 
 suite('testing the python-pip data provider', () => {
@@ -153,3 +154,152 @@ suite('testing python-pip PEP 508 marker handling', () => {
 	}).timeout(10000)
 
 }).beforeAll(() => clock = useFakeTimers(new Date('2023-10-01T00:00:00.000Z'))).afterAll(() => clock.restore());
+
+suite('testing python-pip SHA-256 hash extraction via pip inspect', () => {
+	const hashTestCase = 'pip_requirements_txt_with_hashes'
+
+	const pipFreezeOutput = 'certifi==2023.7.22\nsix==1.16.0\n'
+	const pipShowOutput =
+		'Name: certifi\nVersion: 2023.7.22\nSummary: Python package\nRequires: \nRequired-by: ' +
+		'\n---\n' +
+		'Name: six\nVersion: 1.16.0\nSummary: Python 2 and 3 compatibility\nRequires: \nRequired-by: '
+	const pipInspectOutput = JSON.stringify({
+		version: "1",
+		pip_version: "23.0",
+		installed: [
+			{
+				metadata: { name: "certifi", version: "2023.7.22" },
+				download_info: { archive_info: { hashes: { sha256: "539cc1d13202e33ca466e88b2807e29f4c13049d6d853261b21e0e8d461bbbf0" } } }
+			},
+			{
+				metadata: { name: "six", version: "1.16.0" },
+				download_info: { archive_info: { hashes: { sha256: "1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926" } } }
+			}
+		]
+	})
+
+	/** Verifies that SHA-256 hashes from pip inspect are threaded into the component analysis SBOM. */
+	test('verify component analysis SBOM includes SHA-256 hashes from pip inspect', async () => {
+		// Given: pip environment with freeze, show, and inspect data
+		process.env['TRUSTIFY_DA_PIP_FREEZE'] = Buffer.from(pipFreezeOutput).toString('base64')
+		process.env['TRUSTIFY_DA_PIP_SHOW'] = Buffer.from(pipShowOutput).toString('base64')
+		process.env['TRUSTIFY_DA_PIP_INSPECT'] = Buffer.from(pipInspectOutput).toString('base64')
+
+		try {
+			// When: component analysis is run
+			let expectedSbom = fs.readFileSync(`test/providers/tst_manifests/pip/${hashTestCase}/expected_component_sbom.json`).toString().trim()
+			expectedSbom = JSON.stringify(JSON.parse(expectedSbom))
+			let result = await pythonPip.provideComponent(`test/providers/tst_manifests/pip/${hashTestCase}/requirements.txt`, {})
+
+			// Then: SBOM includes hashes matching the golden file
+			expect(result).to.deep.equal({
+				ecosystem: 'pip',
+				contentType: 'application/vnd.cyclonedx+json',
+				content: expectedSbom
+			})
+		} finally {
+			delete process.env['TRUSTIFY_DA_PIP_FREEZE']
+			delete process.env['TRUSTIFY_DA_PIP_SHOW']
+			delete process.env['TRUSTIFY_DA_PIP_INSPECT']
+		}
+	}).timeout(10000)
+
+	/** Verifies that SHA-256 hashes from pip inspect are threaded into the stack analysis SBOM. */
+	test('verify stack analysis SBOM includes SHA-256 hashes from pip inspect', async () => {
+		// Given: pip environment with freeze, show, and inspect data
+		process.env['TRUSTIFY_DA_PIP_FREEZE'] = Buffer.from(pipFreezeOutput).toString('base64')
+		process.env['TRUSTIFY_DA_PIP_SHOW'] = Buffer.from(pipShowOutput).toString('base64')
+		process.env['TRUSTIFY_DA_PIP_INSPECT'] = Buffer.from(pipInspectOutput).toString('base64')
+
+		try {
+			// When: stack analysis is run
+			let expectedSbom = fs.readFileSync(`test/providers/tst_manifests/pip/${hashTestCase}/expected_stack_sbom.json`).toString().trim()
+			expectedSbom = JSON.stringify(JSON.parse(expectedSbom))
+			let result = await pythonPip.provideStack(`test/providers/tst_manifests/pip/${hashTestCase}/requirements.txt`, {})
+
+			// Then: SBOM includes hashes matching the golden file
+			expect(result).to.deep.equal({
+				ecosystem: 'pip',
+				contentType: 'application/vnd.cyclonedx+json',
+				content: expectedSbom
+			})
+		} finally {
+			delete process.env['TRUSTIFY_DA_PIP_FREEZE']
+			delete process.env['TRUSTIFY_DA_PIP_SHOW']
+			delete process.env['TRUSTIFY_DA_PIP_INSPECT']
+		}
+	}).timeout(10000)
+
+	/** Verifies graceful degradation when pip inspect is unavailable (invalid output). */
+	test('verify SBOM generated without hashes when pip inspect output is invalid', async () => {
+		// Given: pip environment with freeze and show but invalid inspect data
+		process.env['TRUSTIFY_DA_PIP_FREEZE'] = Buffer.from(pipFreezeOutput).toString('base64')
+		process.env['TRUSTIFY_DA_PIP_SHOW'] = Buffer.from(pipShowOutput).toString('base64')
+		process.env['TRUSTIFY_DA_PIP_INSPECT'] = Buffer.from('INVALID JSON').toString('base64')
+
+		try {
+			// When: component analysis is run
+			let result = await pythonPip.provideComponent(`test/providers/tst_manifests/pip/${hashTestCase}/requirements.txt`, {})
+			let sbom = JSON.parse(result.content)
+
+			// Then: SBOM is generated without hashes
+			expect(sbom.components).to.have.lengthOf(2)
+			for (let component of sbom.components) {
+				expect(component).to.not.have.property('hashes')
+			}
+		} finally {
+			delete process.env['TRUSTIFY_DA_PIP_FREEZE']
+			delete process.env['TRUSTIFY_DA_PIP_SHOW']
+			delete process.env['TRUSTIFY_DA_PIP_INSPECT']
+		}
+	}).timeout(10000)
+
+	/** Verifies that pip inspect is skipped when freeze/show env vars are set without inspect. */
+	test('verify SBOM generated without hashes when pip inspect env var is not set', async () => {
+		// Given: pip environment with freeze and show but NO inspect override
+		process.env['TRUSTIFY_DA_PIP_FREEZE'] = Buffer.from(pipFreezeOutput).toString('base64')
+		process.env['TRUSTIFY_DA_PIP_SHOW'] = Buffer.from(pipShowOutput).toString('base64')
+
+		try {
+			// When: component analysis is run
+			let result = await pythonPip.provideComponent(`test/providers/tst_manifests/pip/${hashTestCase}/requirements.txt`, {})
+			let sbom = JSON.parse(result.content)
+
+			// Then: SBOM is generated without hashes (pip inspect skipped in env var mode)
+			expect(sbom.components).to.have.lengthOf(2)
+			for (let component of sbom.components) {
+				expect(component).to.not.have.property('hashes')
+			}
+		} finally {
+			delete process.env['TRUSTIFY_DA_PIP_FREEZE']
+			delete process.env['TRUSTIFY_DA_PIP_SHOW']
+		}
+	}).timeout(10000)
+
+}).beforeAll(() => clock = useFakeTimers(new Date('2023-10-01T00:00:00.000Z'))).afterAll(() => clock.restore());
+
+suite('testing python-pip SHA-256 hashes with real pip inspect', () => {
+	const testCase = 'pip_requirements_txt_with_hashes'
+
+	/** Verifies SBOM components include SHA-256 hashes from real pip inspect. */
+	test('SBOM components include SHA-256 hashes from pip inspect', async () => {
+		// Given: packages installed via real pip
+		let pipPath = getCustomPath("pip3");
+		invokeCommand(pipPath, ['install', '-r', `test/providers/tst_manifests/pip/${testCase}/requirements.txt`])
+
+		// When: stack analysis is run (uses real pip freeze, pip show, and pip inspect)
+		let result = await pythonPip.provideStack(`test/providers/tst_manifests/pip/${testCase}/requirements.txt`, {})
+		let sbom = JSON.parse(result.content)
+
+		// Then: components with hashes have well-formed SHA-256 entries
+		// Note: pip inspect may not include hashes for cached/pre-installed packages
+		for (let component of sbom.components) {
+			if (component.hashes) {
+				expect(component.hashes).to.be.an('array').with.lengthOf(1)
+				expect(component.hashes[0].alg).to.equal('SHA-256')
+				expect(component.hashes[0].content).to.be.a('string').with.lengthOf(64)
+			}
+		}
+	}).timeout(process.env.GITHUB_ACTIONS ? 30000 : 10000)
+
+}).beforeAll(() => clock = useFakeTimers(new Date('2023-10-01T00:00:00.000Z'))).afterAll(() => clock.restore());

test/providers/tst_manifests/pip/pip_requirements_txt_with_hashes/expected_component_sbom.json

@@ -0,0 +1,50 @@
+{
+	"bomFormat": "CycloneDX",
+	"specVersion": "1.4",
+	"version": 1,
+	"metadata": {
+		"timestamp": "2023-10-01T00:00:00.000Z",
+		"component": {
+			"name": "default-pip-root",
+			"version": "0.0.0",
+			"purl": "pkg:pypi/default-pip-root@0.0.0",
+			"type": "application",
+			"bom-ref": "pkg:pypi/default-pip-root@0.0.0"
+		}
+	},
+	"components": [
+		{
+			"name": "certifi",
+			"version": "2023.7.22",
+			"purl": "pkg:pypi/certifi@2023.7.22",
+			"type": "library",
+			"bom-ref": "pkg:pypi/certifi@2023.7.22",
+			"hashes": [{"alg": "SHA-256", "content": "539cc1d13202e33ca466e88b2807e29f4c13049d6d853261b21e0e8d461bbbf0"}]
+		},
+		{
+			"name": "six",
+			"version": "1.16.0",
+			"purl": "pkg:pypi/six@1.16.0",
+			"type": "library",
+			"bom-ref": "pkg:pypi/six@1.16.0",
+			"hashes": [{"alg": "SHA-256", "content": "1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}]
+		}
+	],
+	"dependencies": [
+		{
+			"ref": "pkg:pypi/default-pip-root@0.0.0",
+			"dependsOn": [
+				"pkg:pypi/certifi@2023.7.22",
+				"pkg:pypi/six@1.16.0"
+			]
+		},
+		{
+			"ref": "pkg:pypi/certifi@2023.7.22",
+			"dependsOn": []
+		},
+		{
+			"ref": "pkg:pypi/six@1.16.0",
+			"dependsOn": []
+		}
+	]
+}