From dbdaef0152401489804ef462717f441203ea2d1b Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Thu, 9 Apr 2026 13:01:11 +0200 Subject: [PATCH 1/3] fix: skip PEP 508 marker-constrained packages not installed in pip environment When a requirements.txt contains packages with environment markers (e.g., `pywin32==306 ; platform_system == "Windows"`), pip only installs packages whose markers match the current platform. The component analysis now detects the semicolon marker separator and silently skips packages that have markers but are not present in the pip freeze cache, instead of throwing a PackageNotInstalledException. Also strips marker suffixes from version strings before version-match comparison. Co-Authored-By: Claude Opus 4.6 --- .../utils/PythonControllerBase.java | 7 +++ .../providers/Python_Provider_Test.java | 38 +++++++++++++++ .../expected_component_sbom.json | 48 +++++++++++++++++++ .../requirements.txt | 3 ++ 4 files changed, 96 insertions(+) create mode 100644 src/test/resources/tst_manifests/pip/pip_requirements_txt_marker_skip/expected_component_sbom.json create mode 100644 src/test/resources/tst_manifests/pip/pip_requirements_txt_marker_skip/requirements.txt diff --git a/src/main/java/io/github/guacsec/trustifyda/utils/PythonControllerBase.java b/src/main/java/io/github/guacsec/trustifyda/utils/PythonControllerBase.java index 68ff5c86..8103f9fc 100644 --- a/src/main/java/io/github/guacsec/trustifyda/utils/PythonControllerBase.java +++ b/src/main/java/io/github/guacsec/trustifyda/utils/PythonControllerBase.java @@ -181,6 +181,9 @@ private List> getDependenciesImpl( if (dep.contains("==")) { doubleEqualSignPosition = dep.indexOf("=="); manifestVersion = dep.substring(doubleEqualSignPosition + 2).trim(); + if (manifestVersion.contains(";")) { + manifestVersion = manifestVersion.substring(0, manifestVersion.indexOf(";")).trim(); + } if (manifestVersion.contains("#")) { var hashCharIndex = manifestVersion.indexOf("#"); manifestVersion = manifestVersion.substring(0, hashCharIndex); @@ -210,6 +213,10 @@ private List> getDependenciesImpl( } List path = new ArrayList<>(); String selectedDepName = getDependencyName(dep.toLowerCase()); + boolean hasMarker = dep.contains(";"); + if (hasMarker && cachedEnvironmentDeps.get(new StringInsensitive(selectedDepName)) == null) { + continue; + } path.add(selectedDepName); bringAllDependencies( dependencies, selectedDepName, cachedEnvironmentDeps, includeTransitive, path); diff --git a/src/test/java/io/github/guacsec/trustifyda/providers/Python_Provider_Test.java b/src/test/java/io/github/guacsec/trustifyda/providers/Python_Provider_Test.java index 109b0a65..9c87fb19 100644 --- a/src/test/java/io/github/guacsec/trustifyda/providers/Python_Provider_Test.java +++ b/src/test/java/io/github/guacsec/trustifyda/providers/Python_Provider_Test.java @@ -211,6 +211,44 @@ void test_the_provideComponent_with_properties(String testFolder) throws IOExcep assertThat(dropIgnored(new String(content.buffer))).isEqualTo(dropIgnored(expectedSbom)); } + @Test + @RestoreSystemProperties + void test_marker_constrained_uninstalled_packages_are_skipped_in_component_analysis() + throws IOException { + var testFolder = "pip_requirements_txt_marker_skip"; + var targetRequirements = + String.format("src/test/resources/tst_manifests/pip/%s/requirements.txt", testFolder); + + // load expected SBOM + String expectedSbom; + try (var is = + getResourceAsStreamDecision( + this.getClass(), + String.format("tst_manifests/pip/%s/expected_component_sbom.json", testFolder))) { + expectedSbom = new String(is.readAllBytes()); + } + + // pip environment where only six and certifi are installed (pywin32 is Windows-only) + String pipFreezeContent = "six==1.16.0\ncertifi==2023.7.22\n"; + String pipShowContent = + "Name: certifi\nVersion: 2023.7.22\nSummary: Python package for providing Mozilla's CA" + + " Bundle.\nRequires: \nRequired-by: \n---\nName: six\nVersion: 1.16.0\nSummary:" + + " Python 2 and 3 compatibility utilities\nRequires: \nRequired-by: "; + System.setProperty( + PROP_TRUSTIFY_DA_PIP_FREEZE, + new String(Base64.getEncoder().encode(pipFreezeContent.getBytes()))); + System.setProperty( + PROP_TRUSTIFY_DA_PIP_SHOW, + new String(Base64.getEncoder().encode(pipShowContent.getBytes()))); + + // when providing component content for a manifest with a Windows-only marker package + var content = new PythonPipProvider(Path.of(targetRequirements)).provideComponent(); + + // then SBOM contains six and certifi but not pywin32 + assertThat(content.type).isEqualTo(Api.CYCLONEDX_MEDIA_TYPE); + assertThat(dropIgnored(new String(content.buffer))).isEqualTo(dropIgnored(expectedSbom)); + } + @Test void Test_The_ProvideComponent_Path_Should_Throw_Exception() { assertThatIllegalArgumentException() diff --git a/src/test/resources/tst_manifests/pip/pip_requirements_txt_marker_skip/expected_component_sbom.json b/src/test/resources/tst_manifests/pip/pip_requirements_txt_marker_skip/expected_component_sbom.json new file mode 100644 index 00000000..af333b2a --- /dev/null +++ b/src/test/resources/tst_manifests/pip/pip_requirements_txt_marker_skip/expected_component_sbom.json @@ -0,0 +1,48 @@ +{ + "bomFormat" : "CycloneDX", + "specVersion" : "1.4", + "version" : 1, + "metadata" : { + "timestamp" : "2025-04-09T12:38:18Z", + "component" : { + "type" : "application", + "bom-ref" : "pkg:pypi/default-pip-root@0.0.0", + "name" : "default-pip-root", + "version" : "0.0.0", + "purl" : "pkg:pypi/default-pip-root@0.0.0" + } + }, + "components" : [ + { + "type" : "library", + "bom-ref" : "pkg:pypi/six@1.16.0", + "name" : "six", + "version" : "1.16.0", + "purl" : "pkg:pypi/six@1.16.0" + }, + { + "type" : "library", + "bom-ref" : "pkg:pypi/certifi@2023.7.22", + "name" : "certifi", + "version" : "2023.7.22", + "purl" : "pkg:pypi/certifi@2023.7.22" + } + ], + "dependencies" : [ + { + "ref" : "pkg:pypi/default-pip-root@0.0.0", + "dependsOn" : [ + "pkg:pypi/six@1.16.0", + "pkg:pypi/certifi@2023.7.22" + ] + }, + { + "ref" : "pkg:pypi/six@1.16.0", + "dependsOn" : [ ] + }, + { + "ref" : "pkg:pypi/certifi@2023.7.22", + "dependsOn" : [ ] + } + ] +} diff --git a/src/test/resources/tst_manifests/pip/pip_requirements_txt_marker_skip/requirements.txt b/src/test/resources/tst_manifests/pip/pip_requirements_txt_marker_skip/requirements.txt new file mode 100644 index 00000000..f8c35ddd --- /dev/null +++ b/src/test/resources/tst_manifests/pip/pip_requirements_txt_marker_skip/requirements.txt @@ -0,0 +1,3 @@ +six==1.16.0 +certifi==2023.7.22 ; python_version >= "3" +pywin32==306 ; platform_system == "Windows" From 7432a6c8e44a71e745daf498a9b8ec6c271677ad Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Fri, 10 Apr 2026 19:57:12 +0200 Subject: [PATCH 2/3] fix: strip PEP 508 marker suffix in getDependencyName() for versionless requirements getDependencyName() picked up operators from marker expressions (e.g., == in sys_platform == "win32") when a requirement had no version operator, producing malformed names and cache misses. Strip the marker suffix (;) before scanning for version operators in both getDependencyName() and the version matching block in getDependenciesImpl(). Implements TC-4085 Assisted-by: Claude Code --- .../utils/PythonControllerBase.java | 28 +++++------ .../providers/Python_Provider_Test.java | 39 +++++++++++++++ .../utils/PythonControllerRealEnvTest.java | 13 +++++ .../expected_component_sbom.json | 48 +++++++++++++++++++ .../requirements.txt | 2 + 5 files changed, 116 insertions(+), 14 deletions(-) create mode 100644 src/test/resources/tst_manifests/pip/pip_requirements_txt_marker_installed/expected_component_sbom.json create mode 100644 src/test/resources/tst_manifests/pip/pip_requirements_txt_marker_installed/requirements.txt diff --git a/src/main/java/io/github/guacsec/trustifyda/utils/PythonControllerBase.java b/src/main/java/io/github/guacsec/trustifyda/utils/PythonControllerBase.java index 8103f9fc..b08c204e 100644 --- a/src/main/java/io/github/guacsec/trustifyda/utils/PythonControllerBase.java +++ b/src/main/java/io/github/guacsec/trustifyda/utils/PythonControllerBase.java @@ -173,22 +173,21 @@ private List> getDependenciesImpl( boolean matchManifestVersions = Environment.getBoolean(PROP_MATCH_MANIFEST_VERSIONS, true); for (String dep : linesOfRequirements) { + boolean hasMarker = dep.contains(";"); + String requirementSpec = hasMarker ? dep.substring(0, dep.indexOf(";")).trim() : dep; if (matchManifestVersions) { String dependencyName; String manifestVersion; String installedVersion = ""; int doubleEqualSignPosition; - if (dep.contains("==")) { - doubleEqualSignPosition = dep.indexOf("=="); - manifestVersion = dep.substring(doubleEqualSignPosition + 2).trim(); - if (manifestVersion.contains(";")) { - manifestVersion = manifestVersion.substring(0, manifestVersion.indexOf(";")).trim(); - } + if (requirementSpec.contains("==")) { + doubleEqualSignPosition = requirementSpec.indexOf("=="); + manifestVersion = requirementSpec.substring(doubleEqualSignPosition + 2).trim(); if (manifestVersion.contains("#")) { var hashCharIndex = manifestVersion.indexOf("#"); manifestVersion = manifestVersion.substring(0, hashCharIndex); } - dependencyName = getDependencyName(dep); + dependencyName = getDependencyName(requirementSpec); PythonDependency pythonDependency = cachedEnvironmentDeps.get(new StringInsensitive(dependencyName)); if (pythonDependency != null) { @@ -212,8 +211,7 @@ private List> getDependenciesImpl( } } List path = new ArrayList<>(); - String selectedDepName = getDependencyName(dep.toLowerCase()); - boolean hasMarker = dep.contains(";"); + String selectedDepName = getDependencyName(requirementSpec.toLowerCase()); if (hasMarker && cachedEnvironmentDeps.get(new StringInsensitive(selectedDepName)) == null) { continue; } @@ -380,15 +378,17 @@ protected String getDependencyNameShow(String pipShowOutput) { } public static String getDependencyName(String dep) { - int rightTriangleBracket = dep.indexOf(">"); - int leftTriangleBracket = dep.indexOf("<"); - int equalsSign = dep.indexOf("="); + int markerSeparator = dep.indexOf(";"); + String requirement = markerSeparator == -1 ? dep : dep.substring(0, markerSeparator); + int rightTriangleBracket = requirement.indexOf(">"); + int leftTriangleBracket = requirement.indexOf("<"); + int equalsSign = requirement.indexOf("="); int minimumIndex = getFirstSign(rightTriangleBracket, leftTriangleBracket, equalsSign); String depName; if (rightTriangleBracket == -1 && leftTriangleBracket == -1 && equalsSign == -1) { - depName = dep; + depName = requirement; } else { - depName = dep.substring(0, minimumIndex); + depName = requirement.substring(0, minimumIndex); } return depName.trim(); } diff --git a/src/test/java/io/github/guacsec/trustifyda/providers/Python_Provider_Test.java b/src/test/java/io/github/guacsec/trustifyda/providers/Python_Provider_Test.java index 9c87fb19..01786f49 100644 --- a/src/test/java/io/github/guacsec/trustifyda/providers/Python_Provider_Test.java +++ b/src/test/java/io/github/guacsec/trustifyda/providers/Python_Provider_Test.java @@ -249,6 +249,45 @@ void test_marker_constrained_uninstalled_packages_are_skipped_in_component_analy assertThat(dropIgnored(new String(content.buffer))).isEqualTo(dropIgnored(expectedSbom)); } + /** + * Verifies that marker-only packages (no version operator) that ARE installed appear in the SBOM. + */ + @Test + @RestoreSystemProperties + void test_marker_only_installed_packages_are_included_in_component_analysis() throws IOException { + // Given a requirements.txt with a marker-only dep that IS installed + var testFolder = "pip_requirements_txt_marker_installed"; + var targetRequirements = + String.format("src/test/resources/tst_manifests/pip/%s/requirements.txt", testFolder); + + String expectedSbom; + try (var is = + getResourceAsStreamDecision( + this.getClass(), + String.format("tst_manifests/pip/%s/expected_component_sbom.json", testFolder))) { + expectedSbom = new String(is.readAllBytes()); + } + + // When pip freeze and pip show include colorama (the marker-only package) + String pipFreezeContent = "six==1.16.0\ncolorama==0.4.6\n"; + String pipShowContent = + "Name: six\nVersion: 1.16.0\nSummary: Python 2 and 3 compatibility utilities\nRequires:" + + " \nRequired-by: \n---\nName: colorama\nVersion: 0.4.6\nSummary: Cross-platform" + + " colored terminal text\nRequires: \nRequired-by: "; + System.setProperty( + PROP_TRUSTIFY_DA_PIP_FREEZE, + new String(Base64.getEncoder().encode(pipFreezeContent.getBytes()))); + System.setProperty( + PROP_TRUSTIFY_DA_PIP_SHOW, + new String(Base64.getEncoder().encode(pipShowContent.getBytes()))); + + var content = new PythonPipProvider(Path.of(targetRequirements)).provideComponent(); + + // Then SBOM contains both six and colorama + assertThat(content.type).isEqualTo(Api.CYCLONEDX_MEDIA_TYPE); + assertThat(dropIgnored(new String(content.buffer))).isEqualTo(dropIgnored(expectedSbom)); + } + @Test void Test_The_ProvideComponent_Path_Should_Throw_Exception() { assertThatIllegalArgumentException() diff --git a/src/test/java/io/github/guacsec/trustifyda/utils/PythonControllerRealEnvTest.java b/src/test/java/io/github/guacsec/trustifyda/utils/PythonControllerRealEnvTest.java index 378cd1da..2b382110 100644 --- a/src/test/java/io/github/guacsec/trustifyda/utils/PythonControllerRealEnvTest.java +++ b/src/test/java/io/github/guacsec/trustifyda/utils/PythonControllerRealEnvTest.java @@ -288,6 +288,19 @@ void get_Dependency_Name_requirements() { assertEquals("something", PythonControllerRealEnv.getDependencyName("something>=2.0.5")); } + /** Verifies getDependencyName strips PEP 508 marker suffix from requirements. */ + @Test + void get_Dependency_Name_with_markers() { + assertEquals( + "colorama", + PythonControllerRealEnv.getDependencyName("colorama ; sys_platform == \"win32\"")); + assertEquals( + "colorama", PythonControllerRealEnv.getDependencyName("colorama;sys_platform==\"win32\"")); + assertEquals( + "certifi", + PythonControllerRealEnv.getDependencyName("certifi==2023.7.22 ; python_version >= \"3\"")); + } + @Test void automaticallyInstallPackageOnEnvironment() { assertFalse(pythonControllerRealEnv.automaticallyInstallPackageOnEnvironment()); diff --git a/src/test/resources/tst_manifests/pip/pip_requirements_txt_marker_installed/expected_component_sbom.json b/src/test/resources/tst_manifests/pip/pip_requirements_txt_marker_installed/expected_component_sbom.json new file mode 100644 index 00000000..072b60c7 --- /dev/null +++ b/src/test/resources/tst_manifests/pip/pip_requirements_txt_marker_installed/expected_component_sbom.json @@ -0,0 +1,48 @@ +{ + "bomFormat" : "CycloneDX", + "specVersion" : "1.4", + "version" : 1, + "metadata" : { + "timestamp" : "2025-04-09T12:38:18Z", + "component" : { + "type" : "application", + "bom-ref" : "pkg:pypi/default-pip-root@0.0.0", + "name" : "default-pip-root", + "version" : "0.0.0", + "purl" : "pkg:pypi/default-pip-root@0.0.0" + } + }, + "components" : [ + { + "type" : "library", + "bom-ref" : "pkg:pypi/six@1.16.0", + "name" : "six", + "version" : "1.16.0", + "purl" : "pkg:pypi/six@1.16.0" + }, + { + "type" : "library", + "bom-ref" : "pkg:pypi/colorama@0.4.6", + "name" : "colorama", + "version" : "0.4.6", + "purl" : "pkg:pypi/colorama@0.4.6" + } + ], + "dependencies" : [ + { + "ref" : "pkg:pypi/default-pip-root@0.0.0", + "dependsOn" : [ + "pkg:pypi/six@1.16.0", + "pkg:pypi/colorama@0.4.6" + ] + }, + { + "ref" : "pkg:pypi/six@1.16.0", + "dependsOn" : [ ] + }, + { + "ref" : "pkg:pypi/colorama@0.4.6", + "dependsOn" : [ ] + } + ] +} diff --git a/src/test/resources/tst_manifests/pip/pip_requirements_txt_marker_installed/requirements.txt b/src/test/resources/tst_manifests/pip/pip_requirements_txt_marker_installed/requirements.txt new file mode 100644 index 00000000..7cf1e7e2 --- /dev/null +++ b/src/test/resources/tst_manifests/pip/pip_requirements_txt_marker_installed/requirements.txt @@ -0,0 +1,2 @@ +six==1.16.0 +colorama ; sys_platform == "win32" From 00e4d23c03fd048231d707b166c33d328572c9b5 Mon Sep 17 00:00:00 2001 From: Ruben Romero Montes Date: Mon, 13 Apr 2026 19:46:22 +0200 Subject: [PATCH 3/3] chore(refactor): use parametrized tests Signed-off-by: Ruben Romero Montes --- .../providers/Python_Provider_Test.java | 69 ++++++------------- 1 file changed, 21 insertions(+), 48 deletions(-) diff --git a/src/test/java/io/github/guacsec/trustifyda/providers/Python_Provider_Test.java b/src/test/java/io/github/guacsec/trustifyda/providers/Python_Provider_Test.java index 01786f49..2f6cce69 100644 --- a/src/test/java/io/github/guacsec/trustifyda/providers/Python_Provider_Test.java +++ b/src/test/java/io/github/guacsec/trustifyda/providers/Python_Provider_Test.java @@ -35,6 +35,7 @@ import org.junit.jupiter.api.condition.EnabledIfEnvironmentVariable; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.MethodSource; import org.junitpioneer.jupiter.RestoreSystemProperties; import org.junitpioneer.jupiter.SetSystemProperty; @@ -211,52 +212,31 @@ void test_the_provideComponent_with_properties(String testFolder) throws IOExcep assertThat(dropIgnored(new String(content.buffer))).isEqualTo(dropIgnored(expectedSbom)); } - @Test - @RestoreSystemProperties - void test_marker_constrained_uninstalled_packages_are_skipped_in_component_analysis() - throws IOException { - var testFolder = "pip_requirements_txt_marker_skip"; - var targetRequirements = - String.format("src/test/resources/tst_manifests/pip/%s/requirements.txt", testFolder); - - // load expected SBOM - String expectedSbom; - try (var is = - getResourceAsStreamDecision( - this.getClass(), - String.format("tst_manifests/pip/%s/expected_component_sbom.json", testFolder))) { - expectedSbom = new String(is.readAllBytes()); - } - - // pip environment where only six and certifi are installed (pywin32 is Windows-only) - String pipFreezeContent = "six==1.16.0\ncertifi==2023.7.22\n"; - String pipShowContent = - "Name: certifi\nVersion: 2023.7.22\nSummary: Python package for providing Mozilla's CA" - + " Bundle.\nRequires: \nRequired-by: \n---\nName: six\nVersion: 1.16.0\nSummary:" - + " Python 2 and 3 compatibility utilities\nRequires: \nRequired-by: "; - System.setProperty( - PROP_TRUSTIFY_DA_PIP_FREEZE, - new String(Base64.getEncoder().encode(pipFreezeContent.getBytes()))); - System.setProperty( - PROP_TRUSTIFY_DA_PIP_SHOW, - new String(Base64.getEncoder().encode(pipShowContent.getBytes()))); - - // when providing component content for a manifest with a Windows-only marker package - var content = new PythonPipProvider(Path.of(targetRequirements)).provideComponent(); - - // then SBOM contains six and certifi but not pywin32 - assertThat(content.type).isEqualTo(Api.CYCLONEDX_MEDIA_TYPE); - assertThat(dropIgnored(new String(content.buffer))).isEqualTo(dropIgnored(expectedSbom)); + static Stream markerTestCases() { + return Stream.of( + Arguments.of( + "pip_requirements_txt_marker_skip", + "six==1.16.0\ncertifi==2023.7.22\n", + "Name: certifi\nVersion: 2023.7.22\nSummary: Python package for providing Mozilla's CA" + + " Bundle.\nRequires: \nRequired-by: \n---\nName: six\nVersion: 1.16.0\nSummary:" + + " Python 2 and 3 compatibility utilities\nRequires: \nRequired-by: "), + Arguments.of( + "pip_requirements_txt_marker_installed", + "six==1.16.0\ncolorama==0.4.6\n", + "Name: six\nVersion: 1.16.0\nSummary: Python 2 and 3 compatibility utilities\nRequires:" + + " \nRequired-by: \n---\nName: colorama\nVersion: 0.4.6\nSummary: Cross-platform" + + " colored terminal text\nRequires: \nRequired-by: ")); } /** - * Verifies that marker-only packages (no version operator) that ARE installed appear in the SBOM. + * Verifies that PEP 508 marker-constrained packages are handled correctly: skipped when not + * installed (marker didn't match) and included when installed (marker matched or marker-only). */ - @Test + @ParameterizedTest + @MethodSource("markerTestCases") @RestoreSystemProperties - void test_marker_only_installed_packages_are_included_in_component_analysis() throws IOException { - // Given a requirements.txt with a marker-only dep that IS installed - var testFolder = "pip_requirements_txt_marker_installed"; + void test_marker_constrained_packages_in_component_analysis( + String testFolder, String pipFreezeContent, String pipShowContent) throws IOException { var targetRequirements = String.format("src/test/resources/tst_manifests/pip/%s/requirements.txt", testFolder); @@ -268,12 +248,6 @@ void test_marker_only_installed_packages_are_included_in_component_analysis() th expectedSbom = new String(is.readAllBytes()); } - // When pip freeze and pip show include colorama (the marker-only package) - String pipFreezeContent = "six==1.16.0\ncolorama==0.4.6\n"; - String pipShowContent = - "Name: six\nVersion: 1.16.0\nSummary: Python 2 and 3 compatibility utilities\nRequires:" - + " \nRequired-by: \n---\nName: colorama\nVersion: 0.4.6\nSummary: Cross-platform" - + " colored terminal text\nRequires: \nRequired-by: "; System.setProperty( PROP_TRUSTIFY_DA_PIP_FREEZE, new String(Base64.getEncoder().encode(pipFreezeContent.getBytes()))); @@ -283,7 +257,6 @@ void test_marker_only_installed_packages_are_included_in_component_analysis() th var content = new PythonPipProvider(Path.of(targetRequirements)).provideComponent(); - // Then SBOM contains both six and colorama assertThat(content.type).isEqualTo(Api.CYCLONEDX_MEDIA_TYPE); assertThat(dropIgnored(new String(content.buffer))).isEqualTo(dropIgnored(expectedSbom)); }