From 77683ef08677a1090fdd6d6096d41334ce9dc4d3 Mon Sep 17 00:00:00 2001 From: zahidblackduck Date: Mon, 9 Mar 2026 17:12:58 +0600 Subject: [PATCH 01/10] update setuptools cfg parser for extras --- .../setuptools/SetupToolsExtractUtils.java | 6 +- .../setuptools/parse/SetupToolsCfgParser.java | 139 +++++++++++++++--- .../parse/SetupToolsParsedResult.java | 12 ++ .../transform/SetupToolsGraphTransformer.java | 19 ++- 4 files changed, 150 insertions(+), 26 deletions(-) diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/SetupToolsExtractUtils.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/SetupToolsExtractUtils.java index e39c25618e..c2ffa0cfec 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/SetupToolsExtractUtils.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/SetupToolsExtractUtils.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.List; +import java.util.Map; import org.apache.commons.io.FileUtils; import org.tomlj.Toml; @@ -77,13 +78,14 @@ public static SetupToolsParser resolveSetupToolsParser(TomlParseResult parsedTom // Step 3: Check the setup.cfg fileResolver = new Requirements(fileFinder, environment); File cfgFile = fileResolver.file(SETUP_CFG); - + if (cfgFile != null) { SetupToolsCfgParser cfgParser = new SetupToolsCfgParser(parsedToml); List cfgDependencies = cfgParser.load(cfgFile.toString()); + Map> extrasMap = cfgParser.loadExtrasRequire(cfgFile.toString()); - if (cfgDependencies != null && !cfgDependencies.isEmpty()) { + if ((cfgDependencies != null && !cfgDependencies.isEmpty()) || (extrasMap != null && !extrasMap.isEmpty())) { return cfgParser; } } diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsCfgParser.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsCfgParser.java index 1aa8c1ef76..6d29753baf 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsCfgParser.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsCfgParser.java @@ -5,8 +5,10 @@ import java.io.FileReader; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; import org.tomlj.TomlParseResult; @@ -14,36 +16,60 @@ import com.blackduck.integration.detectable.python.util.PythonDependencyTransformer; public class SetupToolsCfgParser implements SetupToolsParser { - + private TomlParseResult parsedToml; - + private String projectName; - + private List dependencies; - + + private Map> extrasRequireMap; + public SetupToolsCfgParser(TomlParseResult parsedToml) { this.parsedToml = parsedToml; this.dependencies = new ArrayList<>(); + this.extrasRequireMap = new HashMap<>(); } @Override public SetupToolsParsedResult parse() throws IOException { String tomlProjectName = parsedToml.getString("project.name"); String projectVersion = parsedToml.getString("project.version"); - + // If we have multiple project names the name from the toml wins // I've only seen version information in the toml so use that. String finalProjectName = (tomlProjectName != null && !tomlProjectName.isEmpty()) ? tomlProjectName : projectName; - + + PythonDependencyTransformer dependencyTransformer = new PythonDependencyTransformer(); List parsedDirectDependencies = parseDirectDependencies(); - - return new SetupToolsParsedResult(finalProjectName, projectVersion, parsedDirectDependencies); + + // Build extras transitives map: base package name -> list of transitive deps + Map> extrasTransitives = new HashMap<>(); + for (String rawDep : dependencies) { + String extrasName = extractExtrasName(rawDep); + if (extrasName != null && extrasRequireMap.containsKey(extrasName)) { + // Extract the base package name (everything before '[') + int bracketIndex = rawDep.indexOf('['); + String baseName = rawDep.substring(0, bracketIndex).trim(); + + List transitives = new LinkedList<>(); + for (String transitiveLine : extrasRequireMap.get(extrasName)) { + PythonDependency dep = dependencyTransformer.transformLine(transitiveLine); + if (dep != null) { + transitives.add(dep); + } + } + extrasTransitives.put(baseName, transitives); + } + } + + return new SetupToolsParsedResult(finalProjectName, projectVersion, parsedDirectDependencies, extrasTransitives); } /** * Extracts, does not parse, any entries in the install_requires section of the * setup.cfg - * + * * @param filePath path to the setup.cfg file * @return a list of dependencies extracted from the install_requires section * @throws FileNotFoundException @@ -58,14 +84,14 @@ public List load(String filePath) throws FileNotFoundException, IOExcept while ((line = reader.readLine()) != null) { line = line.trim(); - + if (line.startsWith("name")) { parseProjectName(line); } // Remove all whitespace from the line for key searching String keySearch = line.replaceAll("\\s", ""); - + // If the line starts with "install_requires=", we've found the key we're interested in if (keySearch.startsWith("install_requires=")) { isInstallRequiresSection = true; @@ -77,7 +103,7 @@ public List load(String filePath) throws FileNotFoundException, IOExcept } } else if (isInstallRequiresSection) { - if (isEndofInstallRequiresSection(line)) { + if (isEndofInstallRequiresSection(line)) { break; } // If the line is not empty, add it to the dependencies list @@ -90,15 +116,15 @@ else if (!line.isEmpty()) { return dependencies; } - + private List parseDirectDependencies() { List results = new LinkedList<>(); - + PythonDependencyTransformer dependencyTransformer = new PythonDependencyTransformer(); - for (String dependencyLine : dependencies) { + for (String dependencyLine : dependencies) { PythonDependency dependency = dependencyTransformer.transformLine(dependencyLine); - + // If we have a ; in our requirements line then there is a condition on this dependency. // We want to know this so we don't consider it a failure later if we try to run pip show // on it and we don't find it. @@ -110,7 +136,7 @@ private List parseDirectDependencies() { results.add(dependency); } } - + return results; } @@ -121,16 +147,85 @@ public void parseProjectName(String line) { } } + /** + * Extracts entries in the [options.extras_require] section of the setup.cfg, + * grouped by extras group name. + * + * @param filePath path to the setup.cfg file + * @return a map of group name to list of dependency strings + * @throws FileNotFoundException + * @throws IOException + */ + public Map> loadExtrasRequire(String filePath) throws FileNotFoundException, IOException { + try (BufferedReader reader = new BufferedReader(new FileReader(filePath))) { + String line; + boolean isExtrasRequireSection = false; + String currentGroup = null; + + while ((line = reader.readLine()) != null) { + String trimmedLine = line.trim(); + + if (trimmedLine.equals("[options.extras_require]")) { + isExtrasRequireSection = true; + continue; + } + + if (isExtrasRequireSection) { + // A new section header means we've left [options.extras_require] + if (trimmedLine.startsWith("[")) { + break; + } + + // Skip empty lines + if (trimmedLine.isEmpty()) { + continue; + } + + // In INI format, continuation lines (dependencies) are always indented. + // Group key lines (e.g., "http2 =", "security =") are not indented. + if (line.length() > 0 && (line.charAt(0) == ' ' || line.charAt(0) == '\t')) { + // Indented line — it's a dependency under the current group + if (currentGroup != null) { + extrasRequireMap.computeIfAbsent(currentGroup, k -> new ArrayList<>()).add(trimmedLine); + } + } else { + // Non-indented line — it's a group key (e.g., "security =" or "http2 =") + int equalsIndex = trimmedLine.indexOf('='); + if (equalsIndex >= 0) { + currentGroup = trimmedLine.substring(0, equalsIndex).trim(); + } + } + } + } + } + + return extrasRequireMap; + } + + /** + * Extracts the extras specifier name from a raw dependency string. + * For example, "requests[security]==2.28.2" returns "security". + * Returns null if no extras specifier is present. + */ + private String extractExtrasName(String rawDep) { + int openBracket = rawDep.indexOf('['); + int closeBracket = rawDep.indexOf(']'); + if (openBracket >= 0 && closeBracket > openBracket) { + return rawDep.substring(openBracket + 1, closeBracket).trim(); + } + return null; + } + private boolean isEndofInstallRequiresSection(String line) { /* * If the line starts with a [ we have reached a new section and want to exit. - * - * The line.matches call looks for a new key. - * It will return true if the string starts with optional whitespace, - * followed by one or more alphanumeric characters, periods, underscores, or hyphens, + * + * The line.matches call looks for a new key. + * It will return true if the string starts with optional whitespace, + * followed by one or more alphanumeric characters, periods, underscores, or hyphens, * (which is the allowed set of characters for a key), followed by * optional whitespace, an equal sign, optional whitespace, and then any - * character that is not another =, !, <, >, or ~ which would indicate a requirement + * character that is not another =, !, <, >, or ~ which would indicate a requirement * operator and not a new key. */ if (line.startsWith("[") || line.matches("^\\s*[a-zA-Z0-9_.-]+\\s*=\\s*(?![=!<>~]).*$")) { diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsParsedResult.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsParsedResult.java index 8bb84d661b..e9682f1968 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsParsedResult.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsParsedResult.java @@ -1,6 +1,8 @@ package com.blackduck.integration.detectable.detectables.setuptools.parse; +import java.util.Collections; import java.util.List; +import java.util.Map; import com.blackduck.integration.detectable.python.util.PythonDependency; @@ -9,11 +11,17 @@ public class SetupToolsParsedResult { private String projectName; private String projectVersion; private List directDependencies; + private Map> extrasTransitives; public SetupToolsParsedResult(String projectName, String projectVersion, List parsedDirectDependencies) { + this(projectName, projectVersion, parsedDirectDependencies, Collections.emptyMap()); + } + + public SetupToolsParsedResult(String projectName, String projectVersion, List parsedDirectDependencies, Map> extrasTransitives) { this.projectName = projectName; this.projectVersion = projectVersion; this.directDependencies = parsedDirectDependencies; + this.extrasTransitives = extrasTransitives; } public String getProjectName() { @@ -27,4 +35,8 @@ public String getProjectVersion() { public List getDirectDependencies() { return directDependencies; } + + public Map> getExtrasTransitives() { + return extrasTransitives; + } } diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/transform/SetupToolsGraphTransformer.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/transform/SetupToolsGraphTransformer.java index 80ae7b14b6..916efac673 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/transform/SetupToolsGraphTransformer.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/transform/SetupToolsGraphTransformer.java @@ -61,11 +61,12 @@ public DependencyGraph transform(ExecutableTarget pipExe, SetupToolsParsedResult private void handleParsedDependencies(SetupToolsParsedResult parsedResult, DependencyGraph dependencyGraph) { List directDependencies = parsedResult.getDirectDependencies(); - + Map> extrasTransitives = parsedResult.getExtrasTransitives(); + for (PythonDependency directDependency : directDependencies) { String name = directDependency.getName(); String version = directDependency.getVersion(); - + Dependency currentDependency; if (StringUtils.isEmpty(version)) { currentDependency = entryToDependency(name); @@ -73,6 +74,20 @@ private void handleParsedDependencies(SetupToolsParsedResult parsedResult, Depen currentDependency = entryToDependency(name, version); } dependencyGraph.addChildrenToRoot(currentDependency); + + if (extrasTransitives.containsKey(name)) { + for (PythonDependency transitiveDep : extrasTransitives.get(name)) { + String transName = transitiveDep.getName(); + String transVersion = transitiveDep.getVersion(); + Dependency transitiveDependency; + if (StringUtils.isEmpty(transVersion)) { + transitiveDependency = entryToDependency(transName); + } else { + transitiveDependency = entryToDependency(transName, transVersion); + } + dependencyGraph.addChildWithParent(transitiveDependency, currentDependency); + } + } } } From 7a3eacc7442e027085c1a541ccd31cfa583388fe Mon Sep 17 00:00:00 2001 From: zahidblackduck Date: Mon, 9 Mar 2026 17:27:35 +0600 Subject: [PATCH 02/10] add unit tests for cfg extra parsing --- .../unit/SetupToolsCfgParserTest.java | 158 ++++++++++++++++++ .../unit/SetupToolsGraphTransformerTest.java | 45 +++++ 2 files changed, 203 insertions(+) diff --git a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsCfgParserTest.java b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsCfgParserTest.java index ed1267eaac..a7794e56d7 100644 --- a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsCfgParserTest.java +++ b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsCfgParserTest.java @@ -1,6 +1,7 @@ package com.blackduck.integration.detectable.detectables.setuptools.unit; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -8,6 +9,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Test; import org.tomlj.Toml; @@ -62,4 +64,160 @@ public void testParse() throws IOException { Files.delete(cfgFile); // delete the temporary file } + + @Test + public void testLoadExtrasRequireReturnsGroupedMap() throws IOException { + String cfgContent = "[metadata]\n" + + "name = sample\n" + + "\n" + + "[options.extras_require]\n" + + "http2 =\n" + + " h2>=4,<5\n" + + " hpack>=4,<5\n" + + "security =\n" + + " pyOpenSSL>=23.0\n" + + " cryptography>=41.0\n" + + "cli =\n" + + " click==8.*\n" + + " rich>=10,<13\n"; + Path tempFile = Files.createTempFile("setup", ".cfg"); + Files.write(tempFile, cfgContent.getBytes()); + + TomlParseResult result = Toml.parse("[build-system]\nrequires = [\"setuptools\"]"); + + SetupToolsCfgParser cfgParser = new SetupToolsCfgParser(result); + Map> extrasMap = cfgParser.loadExtrasRequire(tempFile.toString()); + + assertEquals(3, extrasMap.size()); + + assertTrue(extrasMap.containsKey("http2")); + assertEquals(2, extrasMap.get("http2").size()); + assertTrue(extrasMap.get("http2").contains("h2>=4,<5")); + assertTrue(extrasMap.get("http2").contains("hpack>=4,<5")); + + assertTrue(extrasMap.containsKey("security")); + assertEquals(2, extrasMap.get("security").size()); + assertTrue(extrasMap.get("security").contains("pyOpenSSL>=23.0")); + assertTrue(extrasMap.get("security").contains("cryptography>=41.0")); + + assertTrue(extrasMap.containsKey("cli")); + assertEquals(2, extrasMap.get("cli").size()); + assertTrue(extrasMap.get("cli").contains("click==8.*")); + assertTrue(extrasMap.get("cli").contains("rich>=10,<13")); + + Files.delete(tempFile); + } + + @Test + public void testParseWithExtrasTransitives() throws IOException { + String cfgContent = "[metadata]\n" + + "name = sample\n" + + "\n" + + "[options]\n" + + "install_requires =\n" + + " requests[security]==2.28.2\n" + + " httpx[http2]==0.23.3\n" + + "\n" + + "[options.extras_require]\n" + + "http2 =\n" + + " certifi>=2025.11.12\n" + + " httpcore>=1.0.9\n" + + "security =\n" + + " charset-normalizer>=3.3.1\n" + + " idna>=3.11\n"; + Path tempFile = Files.createTempFile("setup", ".cfg"); + Files.write(tempFile, cfgContent.getBytes()); + + TomlParseResult result = Toml.parse("[build-system]\nrequires = [\"setuptools\"]"); + + SetupToolsCfgParser cfgParser = new SetupToolsCfgParser(result); + cfgParser.load(tempFile.toString()); + cfgParser.loadExtrasRequire(tempFile.toString()); + SetupToolsParsedResult parsedResult = cfgParser.parse(); + + // Direct dependencies + assertEquals(2, parsedResult.getDirectDependencies().size()); + + // Extras transitives + Map> extrasTransitives = parsedResult.getExtrasTransitives(); + assertNotNull(extrasTransitives); + assertEquals(2, extrasTransitives.size()); + + // requests -> security group + assertTrue(extrasTransitives.containsKey("requests")); + List requestsTransitives = extrasTransitives.get("requests"); + assertEquals(2, requestsTransitives.size()); + assertEquals("charset-normalizer", requestsTransitives.get(0).getName()); + assertEquals("3.3.1", requestsTransitives.get(0).getVersion()); + assertEquals("idna", requestsTransitives.get(1).getName()); + assertEquals("3.11", requestsTransitives.get(1).getVersion()); + + // httpx -> http2 group + assertTrue(extrasTransitives.containsKey("httpx")); + List httpxTransitives = extrasTransitives.get("httpx"); + assertEquals(2, httpxTransitives.size()); + assertEquals("certifi", httpxTransitives.get(0).getName()); + assertEquals("2025.11.12", httpxTransitives.get(0).getVersion()); + assertEquals("httpcore", httpxTransitives.get(1).getName()); + assertEquals("1.0.9", httpxTransitives.get(1).getVersion()); + + Files.delete(tempFile); + } + + @Test + public void testExtrasGroupNotMatchingInstallRequiresIsIgnored() throws IOException { + String cfgContent = "[metadata]\n" + + "name = sample\n" + + "\n" + + "[options]\n" + + "install_requires =\n" + + " requests==2.28.2\n" + + "\n" + + "[options.extras_require]\n" + + "dev =\n" + + " pytest>=7.0\n" + + " flake8>=5.0\n"; + Path tempFile = Files.createTempFile("setup", ".cfg"); + Files.write(tempFile, cfgContent.getBytes()); + + TomlParseResult result = Toml.parse("[build-system]\nrequires = [\"setuptools\"]"); + + SetupToolsCfgParser cfgParser = new SetupToolsCfgParser(result); + cfgParser.load(tempFile.toString()); + cfgParser.loadExtrasRequire(tempFile.toString()); + SetupToolsParsedResult parsedResult = cfgParser.parse(); + + // Direct dependency has no extras specifier, so no match + assertEquals(1, parsedResult.getDirectDependencies().size()); + assertEquals("requests", parsedResult.getDirectDependencies().get(0).getName()); + + // Extras transitives should be empty since "dev" doesn't match any install_requires extras + assertTrue(parsedResult.getExtrasTransitives().isEmpty()); + + Files.delete(tempFile); + } + + @Test + public void testLoadExtrasRequireStopsAtNextSection() throws IOException { + String cfgContent = "[options.extras_require]\n" + + "dev =\n" + + " pytest>=7.0\n" + + "\n" + + "[options.package_data]\n" + + "* = *.txt\n"; + Path tempFile = Files.createTempFile("setup", ".cfg"); + Files.write(tempFile, cfgContent.getBytes()); + + TomlParseResult result = Toml.parse("[build-system]\nrequires = [\"setuptools\"]"); + + SetupToolsCfgParser cfgParser = new SetupToolsCfgParser(result); + Map> extrasMap = cfgParser.loadExtrasRequire(tempFile.toString()); + + assertEquals(1, extrasMap.size()); + assertTrue(extrasMap.containsKey("dev")); + assertEquals(1, extrasMap.get("dev").size()); + assertTrue(extrasMap.get("dev").contains("pytest>=7.0")); + + Files.delete(tempFile); + } } diff --git a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsGraphTransformerTest.java b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsGraphTransformerTest.java index f7af7289f1..e47a5e10a2 100644 --- a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsGraphTransformerTest.java +++ b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsGraphTransformerTest.java @@ -6,7 +6,9 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -85,4 +87,47 @@ public void testPipTransform() throws ExecutableRunnerException { graphAssert.hasRootDependency("certifi", "2022.6.15"); } + + @Test + public void testNoPipTransformWithExtrasTransitives() throws ExecutableRunnerException { + List dependencies = new ArrayList<>(); + dependencies.add(new PythonDependency("requests", "2.28.2")); + dependencies.add(new PythonDependency("httpx", "0.23.3")); + + Map> extrasTransitives = new HashMap<>(); + extrasTransitives.put("requests", Arrays.asList( + new PythonDependency("charset-normalizer", "3.3.1"), + new PythonDependency("idna", "3.11") + )); + extrasTransitives.put("httpx", Arrays.asList( + new PythonDependency("certifi", "2025.11.12"), + new PythonDependency("httpcore", "1.0.9") + )); + + SetupToolsParsedResult parsedResult = new SetupToolsParsedResult("sample", "1.0.0", dependencies, extrasTransitives); + + SetupToolsGraphTransformer graphTransformer = new SetupToolsGraphTransformer(null, new ExternalIdFactory(), null); + DependencyGraph dependencyGraph = graphTransformer.transform(null, parsedResult); + + assertNotNull(dependencyGraph); + assertEquals(2, dependencyGraph.getRootDependencies().size()); + + NameVersionGraphAssert graphAssert = new NameVersionGraphAssert(Forge.PYPI, dependencyGraph); + + // Direct dependencies are at root + graphAssert.hasRootDependency("requests", "2.28.2"); + graphAssert.hasRootDependency("httpx", "0.23.3"); + + // Transitives exist in graph + graphAssert.hasDependency("charset-normalizer", "3.3.1"); + graphAssert.hasDependency("idna", "3.11"); + graphAssert.hasDependency("certifi", "2025.11.12"); + graphAssert.hasDependency("httpcore", "1.0.9"); + + // Verify parent-child relationships + graphAssert.hasParentChildRelationship("requests", "2.28.2", "charset-normalizer", "3.3.1"); + graphAssert.hasParentChildRelationship("requests", "2.28.2", "idna", "3.11"); + graphAssert.hasParentChildRelationship("httpx", "0.23.3", "certifi", "2025.11.12"); + graphAssert.hasParentChildRelationship("httpx", "0.23.3", "httpcore", "1.0.9"); + } } From ee0709fe1c50009ce8a9c8e1e00f2b4172865b13 Mon Sep 17 00:00:00 2001 From: zahidblackduck Date: Mon, 9 Mar 2026 17:51:42 +0600 Subject: [PATCH 03/10] add setuptools toml parser extra support --- .../setuptools/parse/SetupToolsCfgParser.java | 37 +-------- .../parse/SetupToolsExtrasUtils.java | 75 +++++++++++++++++++ .../parse/SetupToolsTomlParser.java | 49 ++++++++++-- 3 files changed, 120 insertions(+), 41 deletions(-) create mode 100644 detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsExtrasUtils.java diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsCfgParser.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsCfgParser.java index 6d29753baf..351a1b2f16 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsCfgParser.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsCfgParser.java @@ -15,6 +15,8 @@ import com.blackduck.integration.detectable.python.util.PythonDependency; import com.blackduck.integration.detectable.python.util.PythonDependencyTransformer; +import static com.blackduck.integration.detectable.detectables.setuptools.parse.SetupToolsExtrasUtils.buildExtrasTransitives; + public class SetupToolsCfgParser implements SetupToolsParser { private TomlParseResult parsedToml; @@ -40,28 +42,9 @@ public SetupToolsParsedResult parse() throws IOException { // I've only seen version information in the toml so use that. String finalProjectName = (tomlProjectName != null && !tomlProjectName.isEmpty()) ? tomlProjectName : projectName; - PythonDependencyTransformer dependencyTransformer = new PythonDependencyTransformer(); List parsedDirectDependencies = parseDirectDependencies(); - // Build extras transitives map: base package name -> list of transitive deps - Map> extrasTransitives = new HashMap<>(); - for (String rawDep : dependencies) { - String extrasName = extractExtrasName(rawDep); - if (extrasName != null && extrasRequireMap.containsKey(extrasName)) { - // Extract the base package name (everything before '[') - int bracketIndex = rawDep.indexOf('['); - String baseName = rawDep.substring(0, bracketIndex).trim(); - - List transitives = new LinkedList<>(); - for (String transitiveLine : extrasRequireMap.get(extrasName)) { - PythonDependency dep = dependencyTransformer.transformLine(transitiveLine); - if (dep != null) { - transitives.add(dep); - } - } - extrasTransitives.put(baseName, transitives); - } - } + Map> extrasTransitives = buildExtrasTransitives(dependencies, extrasRequireMap); return new SetupToolsParsedResult(finalProjectName, projectVersion, parsedDirectDependencies, extrasTransitives); } @@ -202,20 +185,6 @@ public Map> loadExtrasRequire(String filePath) throws FileN return extrasRequireMap; } - /** - * Extracts the extras specifier name from a raw dependency string. - * For example, "requests[security]==2.28.2" returns "security". - * Returns null if no extras specifier is present. - */ - private String extractExtrasName(String rawDep) { - int openBracket = rawDep.indexOf('['); - int closeBracket = rawDep.indexOf(']'); - if (openBracket >= 0 && closeBracket > openBracket) { - return rawDep.substring(openBracket + 1, closeBracket).trim(); - } - return null; - } - private boolean isEndofInstallRequiresSection(String line) { /* * If the line starts with a [ we have reached a new section and want to exit. diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsExtrasUtils.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsExtrasUtils.java new file mode 100644 index 0000000000..8a25485876 --- /dev/null +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsExtrasUtils.java @@ -0,0 +1,75 @@ +package com.blackduck.integration.detectable.detectables.setuptools.parse; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import com.blackduck.integration.detectable.python.util.PythonDependency; +import com.blackduck.integration.detectable.python.util.PythonDependencyTransformer; + +public class SetupToolsExtrasUtils { + + /** + * Extracts the extras specifier names from a raw dependency string. + * For example, "requests[security,socks]==2.28.2" returns ["security", "socks"]. + * Returns an empty list if no extras specifier is present. + */ + public static List extractExtrasNames(String rawDep) { + List names = new ArrayList<>(); + int openBracket = rawDep.indexOf('['); + int closeBracket = rawDep.indexOf(']'); + if (openBracket >= 0 && closeBracket > openBracket) { + String extrasContent = rawDep.substring(openBracket + 1, closeBracket); + for (String name : extrasContent.split(",")) { + String trimmed = name.trim(); + if (!trimmed.isEmpty()) { + names.add(trimmed); + } + } + } + return names; + } + + /** + * Builds a map of base package name to transitive dependencies by matching + * extras specifiers in raw dependency lines against the extras group map. + * + * @param rawDependencyLines the raw dependency strings (e.g., "requests[security]==2.28.2") + * @param extrasGroupMap map of extras group name to list of dependency strings in that group + * @return map of base package name to list of transitive PythonDependency objects + */ + public static Map> buildExtrasTransitives( + List rawDependencyLines, + Map> extrasGroupMap) { + + Map> extrasTransitives = new HashMap<>(); + PythonDependencyTransformer dependencyTransformer = new PythonDependencyTransformer(); + + for (String rawDep : rawDependencyLines) { + List extrasNames = extractExtrasNames(rawDep); + if (extrasNames.isEmpty()) { + continue; + } + + // Extract the base package name (everything before '[') + int bracketIndex = rawDep.indexOf('['); + String baseName = rawDep.substring(0, bracketIndex).trim(); + + for (String extrasName : extrasNames) { + if (extrasGroupMap.containsKey(extrasName)) { + List transitives = extrasTransitives.computeIfAbsent(baseName, k -> new LinkedList<>()); + for (String transitiveLine : extrasGroupMap.get(extrasName)) { + PythonDependency dep = dependencyTransformer.transformLine(transitiveLine); + if (dep != null) { + transitives.add(dep); + } + } + } + } + } + + return extrasTransitives; + } +} diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsTomlParser.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsTomlParser.java index d8bfd97fa9..7b4b0c852f 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsTomlParser.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsTomlParser.java @@ -1,21 +1,29 @@ package com.blackduck.integration.detectable.detectables.setuptools.parse; import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; import org.tomlj.TomlArray; import org.tomlj.TomlParseResult; +import org.tomlj.TomlTable; import com.blackduck.integration.detectable.python.util.PythonDependency; import com.blackduck.integration.detectable.python.util.PythonDependencyTransformer; +import static com.blackduck.integration.detectable.detectables.setuptools.parse.SetupToolsExtrasUtils.buildExtrasTransitives; + public class SetupToolsTomlParser implements SetupToolsParser { - + private TomlParseResult parsedToml; - + private List rawDependencyLines; + public SetupToolsTomlParser(TomlParseResult parsedToml) { this.parsedToml = parsedToml; + this.rawDependencyLines = new ArrayList<>(); } @Override @@ -23,8 +31,12 @@ public SetupToolsParsedResult parse() throws IOException { List parsedDirectDependencies = parseDirectDependencies(parsedToml); String projectName = parsedToml.getString("project.name"); String projectVersion = parsedToml.getString("project.version"); - - return new SetupToolsParsedResult(projectName, projectVersion, parsedDirectDependencies); + + // Parse optional-dependencies and build extras transitives + Map> optionalDepsMap = parseOptionalDependencies(parsedToml); + Map> extrasTransitives = buildExtrasTransitives(rawDependencyLines, optionalDepsMap); + + return new SetupToolsParsedResult(projectName, projectVersion, parsedDirectDependencies, extrasTransitives); } public List parseDirectDependencies(TomlParseResult tomlParseResult) throws IOException { @@ -35,9 +47,10 @@ public List parseDirectDependencies(TomlParseResult tomlParseR for (int i = 0; i < dependencies.size(); i++) { String dependencyLine = dependencies.getString(i); - + rawDependencyLines.add(dependencyLine); + PythonDependency dependency = dependencyTransformer.transformLine(dependencyLine); - + // If we have a ; in our requirements line then there is a condition on this dependency. // We want to know this so we don't consider it a failure later if we try to run pip show // on it and we don't find it. @@ -49,7 +62,29 @@ public List parseDirectDependencies(TomlParseResult tomlParseR results.add(dependency); } } - + return results; } + + private Map> parseOptionalDependencies(TomlParseResult tomlParseResult) { + Map> optionalDepsMap = new HashMap<>(); + + TomlTable optionalDepsTable = tomlParseResult.getTable("project.optional-dependencies"); + if (optionalDepsTable == null) { + return optionalDepsMap; + } + + for (String groupName : optionalDepsTable.keySet()) { + TomlArray groupArray = optionalDepsTable.getArray(groupName); + if (groupArray != null) { + List groupDeps = new ArrayList<>(); + for (int i = 0; i < groupArray.size(); i++) { + groupDeps.add(groupArray.getString(i)); + } + optionalDepsMap.put(groupName, groupDeps); + } + } + + return optionalDepsMap; + } } From fc2ce82693662a7a09b948f4e0dcd845bffec102 Mon Sep 17 00:00:00 2001 From: zahidblackduck Date: Mon, 9 Mar 2026 17:52:18 +0600 Subject: [PATCH 04/10] add setuptools toml parser test, update cfg parser test --- .../unit/PyprojectTomlParserTest.java | 12 +++ .../unit/SetupToolsCfgParserTest.java | 43 ++++++++++ .../unit/SetupToolsTomlParserTest.java | 83 ++++++++++++++++++- 3 files changed, 137 insertions(+), 1 deletion(-) diff --git a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/PyprojectTomlParserTest.java b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/PyprojectTomlParserTest.java index 60e4f9a9b8..55b5d28a8e 100644 --- a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/PyprojectTomlParserTest.java +++ b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/PyprojectTomlParserTest.java @@ -5,6 +5,8 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Test; import org.tomlj.Toml; @@ -12,6 +14,7 @@ import com.blackduck.integration.detectable.detectables.setuptools.parse.SetupToolsParsedResult; import com.blackduck.integration.detectable.detectables.setuptools.parse.SetupToolsTomlParser; +import com.blackduck.integration.detectable.python.util.PythonDependency; class PyprojectTomlParserTest { @@ -84,6 +87,15 @@ void testParseComplexPyprojectToml() throws IOException { assertTrue(result.contains("project.optional-dependencies.dev")); assertTrue(result.contains("project.optional-dependencies.docs")); + // Verify extras transitives: requests[security,socks] references "security" and "socks" extras, + // but neither exists as an optional-dependencies group (only "dev" and "docs" exist). + // Similarly, pandas[all] references "all" which also doesn't exist as a group. + // So the extras transitives map should be empty. + Map> extrasTransitives = parsedResult.getExtrasTransitives(); + assertNotNull(extrasTransitives); + assertTrue(extrasTransitives.isEmpty(), + "No extras transitives expected because no dependency extras names match any optional-dependencies group"); + Files.delete(pyProjectFile); // Clean up the temporary file } } \ No newline at end of file diff --git a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsCfgParserTest.java b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsCfgParserTest.java index a7794e56d7..86a10463e6 100644 --- a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsCfgParserTest.java +++ b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsCfgParserTest.java @@ -197,6 +197,49 @@ public void testExtrasGroupNotMatchingInstallRequiresIsIgnored() throws IOExcept Files.delete(tempFile); } + @Test + public void testParseWithMultiExtrasTransitives() throws IOException { + String cfgContent = "[metadata]\n" + + "name = sample\n" + + "\n" + + "[options]\n" + + "install_requires =\n" + + " requests[security,socks]==2.28.2\n" + + "\n" + + "[options.extras_require]\n" + + "security =\n" + + " pyOpenSSL>=23.0\n" + + " cryptography>=41.0\n" + + "socks =\n" + + " PySocks>=1.5.6\n"; + Path tempFile = Files.createTempFile("setup", ".cfg"); + Files.write(tempFile, cfgContent.getBytes()); + + TomlParseResult result = Toml.parse("[build-system]\nrequires = [\"setuptools\"]"); + + SetupToolsCfgParser cfgParser = new SetupToolsCfgParser(result); + cfgParser.load(tempFile.toString()); + cfgParser.loadExtrasRequire(tempFile.toString()); + SetupToolsParsedResult parsedResult = cfgParser.parse(); + + // Direct dependencies + assertEquals(1, parsedResult.getDirectDependencies().size()); + + // Extras transitives: both security and socks groups should be merged under "requests" + Map> extrasTransitives = parsedResult.getExtrasTransitives(); + assertNotNull(extrasTransitives); + assertEquals(1, extrasTransitives.size()); + assertTrue(extrasTransitives.containsKey("requests")); + + List requestsTransitives = extrasTransitives.get("requests"); + assertEquals(3, requestsTransitives.size()); + assertEquals("pyOpenSSL", requestsTransitives.get(0).getName()); + assertEquals("cryptography", requestsTransitives.get(1).getName()); + assertEquals("PySocks", requestsTransitives.get(2).getName()); + + Files.delete(tempFile); + } + @Test public void testLoadExtrasRequireStopsAtNextSection() throws IOException { String cfgContent = "[options.extras_require]\n" + diff --git a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsTomlParserTest.java b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsTomlParserTest.java index 85e2eabfc4..6927fb1b6b 100644 --- a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsTomlParserTest.java +++ b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsTomlParserTest.java @@ -1,10 +1,14 @@ package com.blackduck.integration.detectable.detectables.setuptools.unit; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Test; import org.tomlj.Toml; @@ -12,9 +16,10 @@ import com.blackduck.integration.detectable.detectables.setuptools.parse.SetupToolsParsedResult; import com.blackduck.integration.detectable.detectables.setuptools.parse.SetupToolsTomlParser; +import com.blackduck.integration.detectable.python.util.PythonDependency; public class SetupToolsTomlParserTest { - + @Test public void testParse() throws IOException { String tomlContent = "[project]\nname = \"setuptools\"\nversion = \"1.0.0\"\ndependencies = [\n \"requests\",\n \"numpy\"\n]"; @@ -32,4 +37,80 @@ public void testParse() throws IOException { Files.delete(pyProjectFile); // delete the temporary file } + + @Test + public void testParseWithExtrasTransitives() throws IOException { + String tomlContent = "[project]\n" + + "name = \"myproject\"\n" + + "version = \"1.0.0\"\n" + + "dependencies = [\n" + + " \"requests[security]==2.28.2\",\n" + + " \"httpx[http2]==0.23.3\"\n" + + "]\n\n" + + "[project.optional-dependencies]\n" + + "security = [\n" + + " \"charset-normalizer>=3.3.1\",\n" + + " \"idna>=3.11\"\n" + + "]\n" + + "http2 = [\n" + + " \"certifi>=2025.11.12\",\n" + + " \"httpcore>=1.0.9\"\n" + + "]\n"; + + TomlParseResult result = Toml.parse(tomlContent); + SetupToolsTomlParser tomlParser = new SetupToolsTomlParser(result); + SetupToolsParsedResult parsedResult = tomlParser.parse(); + + // Direct dependencies + assertEquals(2, parsedResult.getDirectDependencies().size()); + + // Extras transitives + Map> extrasTransitives = parsedResult.getExtrasTransitives(); + assertNotNull(extrasTransitives); + assertEquals(2, extrasTransitives.size()); + + // requests -> security group + assertTrue(extrasTransitives.containsKey("requests")); + List requestsTransitives = extrasTransitives.get("requests"); + assertEquals(2, requestsTransitives.size()); + assertEquals("charset-normalizer", requestsTransitives.get(0).getName()); + assertEquals("3.3.1", requestsTransitives.get(0).getVersion()); + assertEquals("idna", requestsTransitives.get(1).getName()); + assertEquals("3.11", requestsTransitives.get(1).getVersion()); + + // httpx -> http2 group + assertTrue(extrasTransitives.containsKey("httpx")); + List httpxTransitives = extrasTransitives.get("httpx"); + assertEquals(2, httpxTransitives.size()); + assertEquals("certifi", httpxTransitives.get(0).getName()); + assertEquals("2025.11.12", httpxTransitives.get(0).getVersion()); + assertEquals("httpcore", httpxTransitives.get(1).getName()); + assertEquals("1.0.9", httpxTransitives.get(1).getVersion()); + } + + @Test + public void testExtrasGroupNotMatchingDependenciesIsIgnored() throws IOException { + String tomlContent = "[project]\n" + + "name = \"myproject\"\n" + + "version = \"1.0.0\"\n" + + "dependencies = [\n" + + " \"requests==2.28.2\"\n" + + "]\n\n" + + "[project.optional-dependencies]\n" + + "dev = [\n" + + " \"pytest>=7.0\",\n" + + " \"flake8>=5.0\"\n" + + "]\n"; + + TomlParseResult result = Toml.parse(tomlContent); + SetupToolsTomlParser tomlParser = new SetupToolsTomlParser(result); + SetupToolsParsedResult parsedResult = tomlParser.parse(); + + // Direct dependency has no extras specifier, so no match + assertEquals(1, parsedResult.getDirectDependencies().size()); + assertEquals("requests", parsedResult.getDirectDependencies().get(0).getName()); + + // Extras transitives should be empty since "dev" doesn't match any dependency extras + assertTrue(parsedResult.getExtrasTransitives().isEmpty()); + } } From 415a07d99112060b1130c1d6f2e50aa1b0088724 Mon Sep 17 00:00:00 2001 From: zahidblackduck Date: Mon, 9 Mar 2026 18:27:53 +0600 Subject: [PATCH 05/10] add setuptools py parser extra support --- .../setuptools/SetupToolsExtractUtils.java | 5 +- .../setuptools/parse/SetupToolsPyParser.java | 108 +++++++++++++++++- 2 files changed, 107 insertions(+), 6 deletions(-) diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/SetupToolsExtractUtils.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/SetupToolsExtractUtils.java index c2ffa0cfec..44cfe9d127 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/SetupToolsExtractUtils.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/SetupToolsExtractUtils.java @@ -69,8 +69,9 @@ public static SetupToolsParser resolveSetupToolsParser(TomlParseResult parsedTom SetupToolsPyParser pyParser = new SetupToolsPyParser(parsedToml); List pyDependencies = pyParser.load(pyFile.toString()); - - if (pyDependencies != null && !pyDependencies.isEmpty()) { + Map> extrasMap = pyParser.loadExtrasRequire(pyFile.toString()); + + if ((pyDependencies != null && !pyDependencies.isEmpty()) || (extrasMap != null && !extrasMap.isEmpty())) { return pyParser; } } diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsPyParser.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsPyParser.java index cdfbe4017d..1589052225 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsPyParser.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsPyParser.java @@ -4,8 +4,10 @@ import java.io.FileReader; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -14,15 +16,20 @@ import com.blackduck.integration.detectable.python.util.PythonDependency; import com.blackduck.integration.detectable.python.util.PythonDependencyTransformer; +import static com.blackduck.integration.detectable.detectables.setuptools.parse.SetupToolsExtrasUtils.buildExtrasTransitives; + public class SetupToolsPyParser implements SetupToolsParser { private TomlParseResult parsedToml; private List dependencies; - + + private Map> extrasRequireMap; + public SetupToolsPyParser(TomlParseResult parsedToml) { this.parsedToml = parsedToml; this.dependencies = new ArrayList<>(); + this.extrasRequireMap = new HashMap<>(); } @Override @@ -33,8 +40,10 @@ public SetupToolsParsedResult parse() throws IOException { String projectVersion = parsedToml.getString("project.version"); List parsedDirectDependencies = parseDirectDependencies(); - - return new SetupToolsParsedResult(tomlProjectName, projectVersion, parsedDirectDependencies); + + Map> extrasTransitives = buildExtrasTransitives(dependencies, extrasRequireMap); + + return new SetupToolsParsedResult(tomlProjectName, projectVersion, parsedDirectDependencies, extrasTransitives); } public List load(String setupFile) throws IOException { @@ -79,7 +88,98 @@ public List load(String setupFile) throws IOException { return dependencies; } - + + public Map> loadExtrasRequire(String setupFile) throws IOException { + Pattern patternSingleQuotes = Pattern.compile("'(.*?)'"); + Pattern patternDoubleQuotes = Pattern.compile("\"(.*?)\""); + // Pattern for group key lines like "security": [ or 'http2': [ + Pattern groupKeyDoubleQuotes = Pattern.compile("\"(.*?)\"\\s*:"); + Pattern groupKeySingleQuotes = Pattern.compile("'(.*?)'\\s*:"); + + try (BufferedReader reader = new BufferedReader(new FileReader(setupFile))) { + String line; + boolean isExtrasRequireSection = false; + String currentGroup = null; + + while ((line = reader.readLine()) != null) { + String trimmedLine = line.trim(); + + if (trimmedLine.replaceAll("\\s+", "").startsWith("extras_require=")) { + isExtrasRequireSection = true; + continue; + } + + if (isExtrasRequireSection) { + // Skip lines that are just { or contain only whitespace + if (trimmedLine.equals("{")) { + continue; + } + + // End of extras_require section + if (trimmedLine.equals("}") || trimmedLine.equals("},")) { + break; + } + + // Check if this line is a group key like "security": [ or "http2": ["dep1"] + Matcher groupMatcherDouble = groupKeyDoubleQuotes.matcher(trimmedLine); + Matcher groupMatcherSingle = groupKeySingleQuotes.matcher(trimmedLine); + + if (groupMatcherDouble.find()) { + currentGroup = groupMatcherDouble.group(1); + // After the colon, there may be dependencies on the same line + String afterColon = trimmedLine.substring(trimmedLine.indexOf(':') + 1).trim(); + extractDepsFromExtrasLine(afterColon, currentGroup, patternDoubleQuotes, patternSingleQuotes); + + // If the line contains ], the group ends on this line + if (afterColon.contains("]")) { + currentGroup = null; + } + } else if (groupMatcherSingle.find()) { + currentGroup = groupMatcherSingle.group(1); + String afterColon = trimmedLine.substring(trimmedLine.indexOf(':') + 1).trim(); + extractDepsFromExtrasLine(afterColon, currentGroup, patternDoubleQuotes, patternSingleQuotes); + + if (afterColon.contains("]")) { + currentGroup = null; + } + } else if (currentGroup != null) { + // Continuation line within a group's dependency list + extractDepsFromExtrasLine(trimmedLine, currentGroup, patternDoubleQuotes, patternSingleQuotes); + + // If the line ends with ] or ], the current group ends + if (trimmedLine.endsWith("]") || trimmedLine.endsWith("],")) { + currentGroup = null; + } + } + } + } + } + + return extrasRequireMap; + } + + private void extractDepsFromExtrasLine(String line, String group, Pattern patternDoubleQuotes, Pattern patternSingleQuotes) { + // Try double quotes first, then single quotes + Matcher matcherDouble = patternDoubleQuotes.matcher(line); + boolean found = false; + while (matcherDouble.find()) { + String dep = matcherDouble.group(1); + if (!dep.isEmpty()) { + extrasRequireMap.computeIfAbsent(group, k -> new ArrayList<>()).add(dep); + found = true; + } + } + if (!found) { + Matcher matcherSingle = patternSingleQuotes.matcher(line); + while (matcherSingle.find()) { + String dep = matcherSingle.group(1); + if (!dep.isEmpty()) { + extrasRequireMap.computeIfAbsent(group, k -> new ArrayList<>()).add(dep); + } + } + } + } + private void checkLineForDependency(String line, Pattern patternSingleQuotes, Pattern patternDoubleQuotes) { // Using the pattern for double quotes to match the dependencies in the current line. Matcher matcherDoubleQuotes = patternDoubleQuotes.matcher(line); From 6dd512d9c99f64ca95fe97781eeb2f6019c89105 Mon Sep 17 00:00:00 2001 From: zahidblackduck Date: Mon, 9 Mar 2026 18:28:18 +0600 Subject: [PATCH 06/10] add py parser extra support test --- .../unit/SetupToolsPyParserTest.java | 149 +++++++++++++++++- 1 file changed, 148 insertions(+), 1 deletion(-) diff --git a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsPyParserTest.java b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsPyParserTest.java index f0622cebf3..cfd63526b1 100644 --- a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsPyParserTest.java +++ b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsPyParserTest.java @@ -1,6 +1,7 @@ package com.blackduck.integration.detectable.detectables.setuptools.unit; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -8,12 +9,12 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Test; import org.tomlj.Toml; import org.tomlj.TomlParseResult; -import com.blackduck.integration.detectable.detectables.setuptools.parse.SetupToolsCfgParser; import com.blackduck.integration.detectable.detectables.setuptools.parse.SetupToolsParsedResult; import com.blackduck.integration.detectable.detectables.setuptools.parse.SetupToolsPyParser; import com.blackduck.integration.detectable.python.util.PythonDependency; @@ -60,4 +61,150 @@ public void testParse() throws IOException { Files.delete(pyFile); // delete the temporary file } + + @Test + public void testLoadExtrasRequire() throws IOException { + String pyContent = "from setuptools import setup\n\n" + + "setup(\n" + + " install_requires=[\n" + + " 'requests==2.28.2',\n" + + " ],\n" + + " extras_require={\n" + + " \"http2\": [\n" + + " \"certifi>=2025.11.12\",\n" + + " \"httpcore>=1.0.9\",\n" + + " ],\n" + + " \"security\": [\n" + + " \"certifi>=2025.11.12\",\n" + + " \"charset-normalizer>=3.3.1\",\n" + + " ],\n" + + " },\n" + + ")\n"; + + Path tempFile = Files.createTempFile("setup", ".py"); + Files.write(tempFile, pyContent.getBytes()); + + TomlParseResult toml = Toml.parse(""); + SetupToolsPyParser pyParser = new SetupToolsPyParser(toml); + Map> extrasMap = pyParser.loadExtrasRequire(tempFile.toString()); + + assertEquals(2, extrasMap.size()); + assertTrue(extrasMap.containsKey("http2")); + assertTrue(extrasMap.containsKey("security")); + assertEquals(2, extrasMap.get("http2").size()); + assertTrue(extrasMap.get("http2").contains("certifi>=2025.11.12")); + assertTrue(extrasMap.get("http2").contains("httpcore>=1.0.9")); + assertEquals(2, extrasMap.get("security").size()); + assertTrue(extrasMap.get("security").contains("certifi>=2025.11.12")); + assertTrue(extrasMap.get("security").contains("charset-normalizer>=3.3.1")); + + Files.delete(tempFile); + } + + @Test + public void testParseWithExtrasTransitives() throws IOException { + String pyContent = "from setuptools import setup\n\n" + + "setup(\n" + + " install_requires=[\n" + + " 'requests[security]==2.28.2',\n" + + " ],\n" + + " extras_require={\n" + + " \"security\": [\n" + + " \"certifi>=2025.11.12\",\n" + + " \"charset-normalizer>=3.3.1\",\n" + + " ],\n" + + " },\n" + + ")\n"; + + Path tempFile = Files.createTempFile("setup", ".py"); + Files.write(tempFile, pyContent.getBytes()); + + String tomlContent = "[build-system]\nrequires = [\"setuptools\"]"; + TomlParseResult toml = Toml.parse(tomlContent); + + SetupToolsPyParser pyParser = new SetupToolsPyParser(toml); + pyParser.load(tempFile.toString()); + pyParser.loadExtrasRequire(tempFile.toString()); + SetupToolsParsedResult parsedResult = pyParser.parse(); + + assertEquals(1, parsedResult.getDirectDependencies().size()); + assertEquals("requests", parsedResult.getDirectDependencies().get(0).getName()); + + Map> extrasTransitives = parsedResult.getExtrasTransitives(); + assertNotNull(extrasTransitives); + assertTrue(extrasTransitives.containsKey("requests")); + assertEquals(2, extrasTransitives.get("requests").size()); + assertEquals("certifi", extrasTransitives.get("requests").get(0).getName()); + assertEquals("charset-normalizer", extrasTransitives.get("requests").get(1).getName()); + + Files.delete(tempFile); + } + + @Test + public void testExtrasGroupNotMatchingInstallRequiresIsIgnored() throws IOException { + String pyContent = "from setuptools import setup\n\n" + + "setup(\n" + + " install_requires=[\n" + + " 'requests==2.28.2',\n" + + " ],\n" + + " extras_require={\n" + + " \"dev\": [\n" + + " \"pytest>=7.0\",\n" + + " ],\n" + + " },\n" + + ")\n"; + + Path tempFile = Files.createTempFile("setup", ".py"); + Files.write(tempFile, pyContent.getBytes()); + + String tomlContent = "[build-system]\nrequires = [\"setuptools\"]"; + TomlParseResult toml = Toml.parse(tomlContent); + + SetupToolsPyParser pyParser = new SetupToolsPyParser(toml); + pyParser.load(tempFile.toString()); + pyParser.loadExtrasRequire(tempFile.toString()); + SetupToolsParsedResult parsedResult = pyParser.parse(); + + assertTrue(parsedResult.getExtrasTransitives().isEmpty()); + + Files.delete(tempFile); + } + + @Test + public void testParseWithMultiExtrasTransitives() throws IOException { + String pyContent = "from setuptools import setup\n\n" + + "setup(\n" + + " install_requires=[\n" + + " 'requests[security,socks]==2.28.2',\n" + + " ],\n" + + " extras_require={\n" + + " \"security\": [\n" + + " \"certifi>=2025.11.12\",\n" + + " ],\n" + + " \"socks\": [\n" + + " \"PySocks>=1.5.6\",\n" + + " ],\n" + + " },\n" + + ")\n"; + + Path tempFile = Files.createTempFile("setup", ".py"); + Files.write(tempFile, pyContent.getBytes()); + + String tomlContent = "[build-system]\nrequires = [\"setuptools\"]"; + TomlParseResult toml = Toml.parse(tomlContent); + + SetupToolsPyParser pyParser = new SetupToolsPyParser(toml); + pyParser.load(tempFile.toString()); + pyParser.loadExtrasRequire(tempFile.toString()); + SetupToolsParsedResult parsedResult = pyParser.parse(); + + Map> extrasTransitives = parsedResult.getExtrasTransitives(); + assertNotNull(extrasTransitives); + assertTrue(extrasTransitives.containsKey("requests")); + assertEquals(2, extrasTransitives.get("requests").size()); + assertEquals("certifi", extrasTransitives.get("requests").get(0).getName()); + assertEquals("PySocks", extrasTransitives.get("requests").get(1).getName()); + + Files.delete(tempFile); + } } From 00758bedc34a07268d8d41aebbe4dd3fc00b634c Mon Sep 17 00:00:00 2001 From: zahidblackduck Date: Mon, 9 Mar 2026 18:43:57 +0600 Subject: [PATCH 07/10] refactor to reduce cognitive complexity --- .../setuptools/parse/SetupToolsPyParser.java | 123 +++++++++--------- 1 file changed, 59 insertions(+), 64 deletions(-) diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsPyParser.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsPyParser.java index 1589052225..df5611ddad 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsPyParser.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsPyParser.java @@ -90,11 +90,8 @@ public List load(String setupFile) throws IOException { } public Map> loadExtrasRequire(String setupFile) throws IOException { - Pattern patternSingleQuotes = Pattern.compile("'(.*?)'"); - Pattern patternDoubleQuotes = Pattern.compile("\"(.*?)\""); // Pattern for group key lines like "security": [ or 'http2': [ - Pattern groupKeyDoubleQuotes = Pattern.compile("\"(.*?)\"\\s*:"); - Pattern groupKeySingleQuotes = Pattern.compile("'(.*?)'\\s*:"); + Pattern groupKeyPattern = Pattern.compile("[\"'](.*?)[\"']\\s*:"); try (BufferedReader reader = new BufferedReader(new FileReader(setupFile))) { String line; @@ -109,95 +106,93 @@ public Map> loadExtrasRequire(String setupFile) throws IOEx continue; } - if (isExtrasRequireSection) { - // Skip lines that are just { or contain only whitespace - if (trimmedLine.equals("{")) { - continue; - } + if (!isExtrasRequireSection) { + continue; + } - // End of extras_require section - if (trimmedLine.equals("}") || trimmedLine.equals("},")) { - break; - } + if (trimmedLine.equals("{")) { + continue; + } - // Check if this line is a group key like "security": [ or "http2": ["dep1"] - Matcher groupMatcherDouble = groupKeyDoubleQuotes.matcher(trimmedLine); - Matcher groupMatcherSingle = groupKeySingleQuotes.matcher(trimmedLine); - - if (groupMatcherDouble.find()) { - currentGroup = groupMatcherDouble.group(1); - // After the colon, there may be dependencies on the same line - String afterColon = trimmedLine.substring(trimmedLine.indexOf(':') + 1).trim(); - extractDepsFromExtrasLine(afterColon, currentGroup, patternDoubleQuotes, patternSingleQuotes); - - // If the line contains ], the group ends on this line - if (afterColon.contains("]")) { - currentGroup = null; - } - } else if (groupMatcherSingle.find()) { - currentGroup = groupMatcherSingle.group(1); - String afterColon = trimmedLine.substring(trimmedLine.indexOf(':') + 1).trim(); - extractDepsFromExtrasLine(afterColon, currentGroup, patternDoubleQuotes, patternSingleQuotes); - - if (afterColon.contains("]")) { - currentGroup = null; - } - } else if (currentGroup != null) { - // Continuation line within a group's dependency list - extractDepsFromExtrasLine(trimmedLine, currentGroup, patternDoubleQuotes, patternSingleQuotes); - - // If the line ends with ] or ], the current group ends - if (trimmedLine.endsWith("]") || trimmedLine.endsWith("],")) { - currentGroup = null; - } - } + if (trimmedLine.equals("}") || trimmedLine.equals("},")) { + break; } + + currentGroup = processExtrasLine(trimmedLine, currentGroup, groupKeyPattern); } } return extrasRequireMap; } - private void extractDepsFromExtrasLine(String line, String group, Pattern patternDoubleQuotes, Pattern patternSingleQuotes) { - // Try double quotes first, then single quotes + private String processExtrasLine(String trimmedLine, String currentGroup, Pattern groupKeyPattern) { + Matcher groupMatcher = groupKeyPattern.matcher(trimmedLine); + + if (groupMatcher.find()) { + currentGroup = groupMatcher.group(1); + String afterColon = trimmedLine.substring(trimmedLine.indexOf(':') + 1).trim(); + addExtrasLineDeps(afterColon, currentGroup); + if (afterColon.contains("]")) { + return null; + } + } else if (currentGroup != null) { + addExtrasLineDeps(trimmedLine, currentGroup); + if (trimmedLine.endsWith("]") || trimmedLine.endsWith("],")) { + return null; + } + } + + return currentGroup; + } + + private void addExtrasLineDeps(String line, String group) { + List deps = extractQuotedStrings(line); + for (String dep : deps) { + extrasRequireMap.computeIfAbsent(group, k -> new ArrayList<>()).add(dep); + } + } + + /** + * Extracts the first quoted string from a line, trying double quotes first, + * then single quotes. Used by both install_requires and extras_require parsing. + */ + private List extractQuotedStrings(String line) { + List results = new ArrayList<>(); + Pattern patternDoubleQuotes = Pattern.compile("\"(.*?)\""); + Pattern patternSingleQuotes = Pattern.compile("'(.*?)'"); + Matcher matcherDouble = patternDoubleQuotes.matcher(line); boolean found = false; while (matcherDouble.find()) { - String dep = matcherDouble.group(1); - if (!dep.isEmpty()) { - extrasRequireMap.computeIfAbsent(group, k -> new ArrayList<>()).add(dep); + String value = matcherDouble.group(1); + if (!value.isEmpty()) { + results.add(value); found = true; } } if (!found) { Matcher matcherSingle = patternSingleQuotes.matcher(line); while (matcherSingle.find()) { - String dep = matcherSingle.group(1); - if (!dep.isEmpty()) { - extrasRequireMap.computeIfAbsent(group, k -> new ArrayList<>()).add(dep); + String value = matcherSingle.group(1); + if (!value.isEmpty()) { + results.add(value); } } } + return results; } private void checkLineForDependency(String line, Pattern patternSingleQuotes, Pattern patternDoubleQuotes) { - // Using the pattern for double quotes to match the dependencies in the current line. + // Try double quotes first, then fall back to single quotes. + // Double quotes are preferred as lines sometimes use double quotes with + // single quotes inside them to specify conditionals. Matcher matcherDoubleQuotes = patternDoubleQuotes.matcher(line); if (matcherDoubleQuotes.find()) { - // Extracting the dependency from the matched group. - String dependency = matcherDoubleQuotes.group(1); - // Adding the dependency to the list. - dependencies.add(dependency); + dependencies.add(matcherDoubleQuotes.group(1)); } else { - // Fallback to use the pattern for single quotes to match the dependencies in the current - // line. We do this second as there are sometimes lines that use double quotes and then - // single quotes inside them to specify conditionals Matcher matcherSingleQuotes = patternSingleQuotes.matcher(line); if (matcherSingleQuotes.find()) { - // Extracting the dependency from the matched group. - String dependency = matcherSingleQuotes.group(1); - // Adding the dependency to the list. - dependencies.add(dependency); + dependencies.add(matcherSingleQuotes.group(1)); } } } From c3a03dcf2a1d86ac4db7bcf8e99f30631f35487d Mon Sep 17 00:00:00 2001 From: zahidblackduck Date: Mon, 9 Mar 2026 19:14:52 +0600 Subject: [PATCH 08/10] refactor constants to static final fields --- .../setuptools/parse/SetupToolsCfgParser.java | 41 ++++---- .../parse/SetupToolsExtrasUtils.java | 49 +++++++--- .../setuptools/parse/SetupToolsPyParser.java | 96 +++++++++++-------- .../parse/SetupToolsTomlParser.java | 21 ++-- 4 files changed, 130 insertions(+), 77 deletions(-) diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsCfgParser.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsCfgParser.java index 351a1b2f16..697e8fb0d0 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsCfgParser.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsCfgParser.java @@ -19,13 +19,26 @@ public class SetupToolsCfgParser implements SetupToolsParser { - private TomlParseResult parsedToml; + private static final String TOML_PROJECT_NAME_KEY = "project.name"; + private static final String TOML_PROJECT_VERSION_KEY = "project.version"; + + private static final String CFG_NAME_PREFIX = "name"; + private static final String CFG_INSTALL_REQUIRES_PREFIX = "install_requires="; + private static final String CFG_EXTRAS_REQUIRE_SECTION = "[options.extras_require]"; + + private static final String CONDITIONAL_MARKER = ";"; + private static final String KEY_VALUE_SEPARATOR = "="; + + // Matches a new cfg key line: e.g. "python_requires = >=3.10" + private static final String NEW_KEY_LINE_REGEX = "^\\s*[a-zA-Z0-9_.-]+\\s*=\\s*(?![=!<>~]).*$"; + + private final TomlParseResult parsedToml; private String projectName; - private List dependencies; + private final List dependencies; - private Map> extrasRequireMap; + private final Map> extrasRequireMap; public SetupToolsCfgParser(TomlParseResult parsedToml) { this.parsedToml = parsedToml; @@ -35,8 +48,8 @@ public SetupToolsCfgParser(TomlParseResult parsedToml) { @Override public SetupToolsParsedResult parse() throws IOException { - String tomlProjectName = parsedToml.getString("project.name"); - String projectVersion = parsedToml.getString("project.version"); + String tomlProjectName = parsedToml.getString(TOML_PROJECT_NAME_KEY); + String projectVersion = parsedToml.getString(TOML_PROJECT_VERSION_KEY); // If we have multiple project names the name from the toml wins // I've only seen version information in the toml so use that. @@ -68,7 +81,7 @@ public List load(String filePath) throws FileNotFoundException, IOExcept while ((line = reader.readLine()) != null) { line = line.trim(); - if (line.startsWith("name")) { + if (line.startsWith(CFG_NAME_PREFIX)) { parseProjectName(line); } @@ -76,9 +89,9 @@ public List load(String filePath) throws FileNotFoundException, IOExcept String keySearch = line.replaceAll("\\s", ""); // If the line starts with "install_requires=", we've found the key we're interested in - if (keySearch.startsWith("install_requires=")) { + if (keySearch.startsWith(CFG_INSTALL_REQUIRES_PREFIX)) { isInstallRequiresSection = true; - String[] parts = line.split("=", 2); + String[] parts = line.split(KEY_VALUE_SEPARATOR, 2); // If there is a value and it's not empty, add it to the dependencies list if (parts.length > 1 && !parts[1].trim().isEmpty()) { @@ -111,7 +124,7 @@ private List parseDirectDependencies() { // If we have a ; in our requirements line then there is a condition on this dependency. // We want to know this so we don't consider it a failure later if we try to run pip show // on it and we don't find it. - if (dependencyLine.contains(";")) { + if (dependencyLine.contains(CONDITIONAL_MARKER)) { dependency.setConditional(true); } @@ -124,7 +137,7 @@ private List parseDirectDependencies() { } public void parseProjectName(String line) { - String[] parts = line.split("=", 2); + String[] parts = line.split(KEY_VALUE_SEPARATOR, 2); if (parts.length > 1 && !parts[1].trim().isEmpty()) { projectName = parts[1].trim(); } @@ -148,7 +161,7 @@ public Map> loadExtrasRequire(String filePath) throws FileN while ((line = reader.readLine()) != null) { String trimmedLine = line.trim(); - if (trimmedLine.equals("[options.extras_require]")) { + if (trimmedLine.equals(CFG_EXTRAS_REQUIRE_SECTION)) { isExtrasRequireSection = true; continue; } @@ -197,10 +210,6 @@ private boolean isEndofInstallRequiresSection(String line) { * character that is not another =, !, <, >, or ~ which would indicate a requirement * operator and not a new key. */ - if (line.startsWith("[") || line.matches("^\\s*[a-zA-Z0-9_.-]+\\s*=\\s*(?![=!<>~]).*$")) { - return true; - } else { - return false; - } + return line.startsWith("[") || line.matches(NEW_KEY_LINE_REGEX); } } diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsExtrasUtils.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsExtrasUtils.java index 8a25485876..723e941651 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsExtrasUtils.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsExtrasUtils.java @@ -11,6 +11,12 @@ public class SetupToolsExtrasUtils { + private static final char OPEN_BRACKET = '['; + private static final char CLOSE_BRACKET = ']'; + private static final String EXTRAS_DELIMITER = ","; + + private SetupToolsExtrasUtils() {} + /** * Extracts the extras specifier names from a raw dependency string. * For example, "requests[security,socks]==2.28.2" returns ["security", "socks"]. @@ -18,11 +24,11 @@ public class SetupToolsExtrasUtils { */ public static List extractExtrasNames(String rawDep) { List names = new ArrayList<>(); - int openBracket = rawDep.indexOf('['); - int closeBracket = rawDep.indexOf(']'); + int openBracket = rawDep.indexOf(OPEN_BRACKET); + int closeBracket = rawDep.indexOf(CLOSE_BRACKET); if (openBracket >= 0 && closeBracket > openBracket) { String extrasContent = rawDep.substring(openBracket + 1, closeBracket); - for (String name : extrasContent.split(",")) { + for (String name : extrasContent.split(EXTRAS_DELIMITER)) { String trimmed = name.trim(); if (!trimmed.isEmpty()) { names.add(trimmed); @@ -54,22 +60,35 @@ public static Map> buildExtrasTransitives( } // Extract the base package name (everything before '[') - int bracketIndex = rawDep.indexOf('['); + int bracketIndex = rawDep.indexOf(OPEN_BRACKET); String baseName = rawDep.substring(0, bracketIndex).trim(); - for (String extrasName : extrasNames) { - if (extrasGroupMap.containsKey(extrasName)) { - List transitives = extrasTransitives.computeIfAbsent(baseName, k -> new LinkedList<>()); - for (String transitiveLine : extrasGroupMap.get(extrasName)) { - PythonDependency dep = dependencyTransformer.transformLine(transitiveLine); - if (dep != null) { - transitives.add(dep); - } - } - } - } + resolveExtrasForDependency(baseName, extrasNames, extrasGroupMap, extrasTransitives, dependencyTransformer); } return extrasTransitives; } + + private static void resolveExtrasForDependency( + String baseName, + List extrasNames, + Map> extrasGroupMap, + Map> extrasTransitives, + PythonDependencyTransformer dependencyTransformer) { + + for (String extrasName : extrasNames) { + List groupLines = extrasGroupMap.get(extrasName); + if (groupLines == null) { + continue; + } + List transitives = extrasTransitives.computeIfAbsent(baseName, k -> new LinkedList<>()); + for (String transitiveLine : groupLines) { + PythonDependency dep = dependencyTransformer.transformLine(transitiveLine); + if (dep != null) { + transitives.add(dep); + } + } + } + } } + diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsPyParser.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsPyParser.java index df5611ddad..a4f9dddf00 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsPyParser.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsPyParser.java @@ -19,12 +19,40 @@ import static com.blackduck.integration.detectable.detectables.setuptools.parse.SetupToolsExtrasUtils.buildExtrasTransitives; public class SetupToolsPyParser implements SetupToolsParser { - - private TomlParseResult parsedToml; - - private List dependencies; - private Map> extrasRequireMap; + // Regex pattern for dependencies enclosed in single quotes: ['requests==2.31.0'], + private static final Pattern PATTERN_SINGLE_QUOTES = Pattern.compile("\\[?'(.*)'\\s*\\]?,?"); + + // Regex pattern for dependencies enclosed in double quotes: ["requests==2.31.0"], + private static final Pattern PATTERN_DOUBLE_QUOTES = Pattern.compile("\\[?\"(.*)\"\\s*\\]?,?"); + + // Regex pattern for extras_require group key lines like "security": [ or 'http2': [ + private static final Pattern EXTRAS_GROUP_KEY_PATTERN = Pattern.compile("[\"'](.*?)[\"']\\s*:"); + + // Regex pattern for extracting any double-quoted string value + private static final Pattern EXTRACT_DOUBLE_QUOTES = Pattern.compile("\"(.*?)\""); + + // Regex pattern for extracting any single-quoted string value + private static final Pattern EXTRACT_SINGLE_QUOTES = Pattern.compile("'(.*?)'"); + + private static final String INSTALL_REQUIRES_PREFIX = "install_requires="; + private static final String EXTRAS_REQUIRE_PREFIX = "extras_require="; + + private static final String TOML_PROJECT_NAME_KEY = "project.name"; + private static final String TOML_PROJECT_VERSION_KEY = "project.version"; + + private static final String OPEN_BRACKET = "["; + private static final String OPEN_BRACE = "{"; + private static final String CLOSE_BRACE = "}"; + private static final String CLOSE_BRACE_COMMA = "},"; + private static final String CLOSE_BRACKET = "]"; + private static final String CLOSE_BRACKET_COMMA = "],"; + + private final TomlParseResult parsedToml; + + private final List dependencies; + + private final Map> extrasRequireMap; public SetupToolsPyParser(TomlParseResult parsedToml) { this.parsedToml = parsedToml; @@ -36,9 +64,9 @@ public SetupToolsPyParser(TomlParseResult parsedToml) { public SetupToolsParsedResult parse() throws IOException { // Use a name from the toml if we have it. Do not parse names and versions from the setup.py // as the project will not always have a string (it could have variables or method calls) - String tomlProjectName = parsedToml.getString("project.name"); - String projectVersion = parsedToml.getString("project.version"); - + String tomlProjectName = parsedToml.getString(TOML_PROJECT_NAME_KEY); + String projectVersion = parsedToml.getString(TOML_PROJECT_VERSION_KEY); + List parsedDirectDependencies = parseDirectDependencies(); Map> extrasTransitives = buildExtrasTransitives(dependencies, extrasRequireMap); @@ -51,35 +79,30 @@ public List load(String setupFile) throws IOException { // - "\\[?'(.*)'\\s*\\]?,?" matches dependencies that start with an optional '[' followed by a mandatory single quote, // then any characters (the dependency name), ending with a single quote followed by optional whitespace and an optional ',' or ']'. // - "\\[?\"(.*)\"\\s*\\]?,?" is similar to the first part but for dependencies enclosed in double quotes. - // Pattern for single quotes - Pattern patternSingleQuotes = Pattern.compile("\\[?'(.*)'\\s*\\]?,?"); - - // Pattern for double quotes - Pattern patternDoubleQuotes = Pattern.compile("\\[?\"(.*)\"\\s*\\]?,?"); try (BufferedReader reader = new BufferedReader(new FileReader(setupFile))) { String line; boolean isInstallRequiresSection = false; while ((line = reader.readLine()) != null) { - line = line.trim(); - + line = line.trim(); + // If after removing all whitespace the line starts with install_requires= // then we have found the section we are after. - if (line.replaceAll("\\s+","").startsWith("install_requires=")) { + if (line.replaceAll("\\s+", "").startsWith(INSTALL_REQUIRES_PREFIX)) { isInstallRequiresSection = true; continue; } if (isInstallRequiresSection) { // If the [ is on its own line skip it, it doesn't contain a dependency - if (line.equals("[")) { + if (line.equals(OPEN_BRACKET)) { continue; } - - checkLineForDependency(line, patternSingleQuotes, patternDoubleQuotes); - + + checkLineForDependency(line); + // If the line ends with ] or ], it means we have reached the end of the dependencies list. - if (line.endsWith("]") || line.endsWith("],")) { + if (line.endsWith(CLOSE_BRACKET) || line.endsWith(CLOSE_BRACKET_COMMA)) { break; } } @@ -90,9 +113,6 @@ public List load(String setupFile) throws IOException { } public Map> loadExtrasRequire(String setupFile) throws IOException { - // Pattern for group key lines like "security": [ or 'http2': [ - Pattern groupKeyPattern = Pattern.compile("[\"'](.*?)[\"']\\s*:"); - try (BufferedReader reader = new BufferedReader(new FileReader(setupFile))) { String line; boolean isExtrasRequireSection = false; @@ -101,7 +121,7 @@ public Map> loadExtrasRequire(String setupFile) throws IOEx while ((line = reader.readLine()) != null) { String trimmedLine = line.trim(); - if (trimmedLine.replaceAll("\\s+", "").startsWith("extras_require=")) { + if (trimmedLine.replaceAll("\\s+", "").startsWith(EXTRAS_REQUIRE_PREFIX)) { isExtrasRequireSection = true; continue; } @@ -110,34 +130,34 @@ public Map> loadExtrasRequire(String setupFile) throws IOEx continue; } - if (trimmedLine.equals("{")) { + if (trimmedLine.equals(OPEN_BRACE)) { continue; } - if (trimmedLine.equals("}") || trimmedLine.equals("},")) { + if (trimmedLine.equals(CLOSE_BRACE) || trimmedLine.equals(CLOSE_BRACE_COMMA)) { break; } - currentGroup = processExtrasLine(trimmedLine, currentGroup, groupKeyPattern); + currentGroup = processExtrasLine(trimmedLine, currentGroup); } } return extrasRequireMap; } - private String processExtrasLine(String trimmedLine, String currentGroup, Pattern groupKeyPattern) { - Matcher groupMatcher = groupKeyPattern.matcher(trimmedLine); + private String processExtrasLine(String trimmedLine, String currentGroup) { + Matcher groupMatcher = EXTRAS_GROUP_KEY_PATTERN.matcher(trimmedLine); if (groupMatcher.find()) { currentGroup = groupMatcher.group(1); String afterColon = trimmedLine.substring(trimmedLine.indexOf(':') + 1).trim(); addExtrasLineDeps(afterColon, currentGroup); - if (afterColon.contains("]")) { + if (afterColon.contains(CLOSE_BRACKET)) { return null; } } else if (currentGroup != null) { addExtrasLineDeps(trimmedLine, currentGroup); - if (trimmedLine.endsWith("]") || trimmedLine.endsWith("],")) { + if (trimmedLine.endsWith(CLOSE_BRACKET) || trimmedLine.endsWith(CLOSE_BRACKET_COMMA)) { return null; } } @@ -158,10 +178,8 @@ private void addExtrasLineDeps(String line, String group) { */ private List extractQuotedStrings(String line) { List results = new ArrayList<>(); - Pattern patternDoubleQuotes = Pattern.compile("\"(.*?)\""); - Pattern patternSingleQuotes = Pattern.compile("'(.*?)'"); - Matcher matcherDouble = patternDoubleQuotes.matcher(line); + Matcher matcherDouble = EXTRACT_DOUBLE_QUOTES.matcher(line); boolean found = false; while (matcherDouble.find()) { String value = matcherDouble.group(1); @@ -171,7 +189,7 @@ private List extractQuotedStrings(String line) { } } if (!found) { - Matcher matcherSingle = patternSingleQuotes.matcher(line); + Matcher matcherSingle = EXTRACT_SINGLE_QUOTES.matcher(line); while (matcherSingle.find()) { String value = matcherSingle.group(1); if (!value.isEmpty()) { @@ -182,15 +200,15 @@ private List extractQuotedStrings(String line) { return results; } - private void checkLineForDependency(String line, Pattern patternSingleQuotes, Pattern patternDoubleQuotes) { + private void checkLineForDependency(String line) { // Try double quotes first, then fall back to single quotes. // Double quotes are preferred as lines sometimes use double quotes with // single quotes inside them to specify conditionals. - Matcher matcherDoubleQuotes = patternDoubleQuotes.matcher(line); + Matcher matcherDoubleQuotes = PATTERN_DOUBLE_QUOTES.matcher(line); if (matcherDoubleQuotes.find()) { dependencies.add(matcherDoubleQuotes.group(1)); } else { - Matcher matcherSingleQuotes = patternSingleQuotes.matcher(line); + Matcher matcherSingleQuotes = PATTERN_SINGLE_QUOTES.matcher(line); if (matcherSingleQuotes.find()) { dependencies.add(matcherSingleQuotes.group(1)); } diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsTomlParser.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsTomlParser.java index 7b4b0c852f..22818743a6 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsTomlParser.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsTomlParser.java @@ -18,8 +18,15 @@ public class SetupToolsTomlParser implements SetupToolsParser { - private TomlParseResult parsedToml; - private List rawDependencyLines; + private static final String TOML_PROJECT_NAME_KEY = "project.name"; + private static final String TOML_PROJECT_VERSION_KEY = "project.version"; + private static final String TOML_PROJECT_DEPENDENCIES_KEY = "project.dependencies"; + private static final String TOML_OPTIONAL_DEPENDENCIES_KEY = "project.optional-dependencies"; + + private static final String CONDITIONAL_MARKER = ";"; + + private final TomlParseResult parsedToml; + private final List rawDependencyLines; public SetupToolsTomlParser(TomlParseResult parsedToml) { this.parsedToml = parsedToml; @@ -29,8 +36,8 @@ public SetupToolsTomlParser(TomlParseResult parsedToml) { @Override public SetupToolsParsedResult parse() throws IOException { List parsedDirectDependencies = parseDirectDependencies(parsedToml); - String projectName = parsedToml.getString("project.name"); - String projectVersion = parsedToml.getString("project.version"); + String projectName = parsedToml.getString(TOML_PROJECT_NAME_KEY); + String projectVersion = parsedToml.getString(TOML_PROJECT_VERSION_KEY); // Parse optional-dependencies and build extras transitives Map> optionalDepsMap = parseOptionalDependencies(parsedToml); @@ -43,7 +50,7 @@ public List parseDirectDependencies(TomlParseResult tomlParseR List results = new LinkedList<>(); PythonDependencyTransformer dependencyTransformer = new PythonDependencyTransformer(); - TomlArray dependencies = tomlParseResult.getArray("project.dependencies"); + TomlArray dependencies = tomlParseResult.getArray(TOML_PROJECT_DEPENDENCIES_KEY); for (int i = 0; i < dependencies.size(); i++) { String dependencyLine = dependencies.getString(i); @@ -54,7 +61,7 @@ public List parseDirectDependencies(TomlParseResult tomlParseR // If we have a ; in our requirements line then there is a condition on this dependency. // We want to know this so we don't consider it a failure later if we try to run pip show // on it and we don't find it. - if (dependencyLine.contains(";")) { + if (dependencyLine.contains(CONDITIONAL_MARKER)) { dependency.setConditional(true); } @@ -69,7 +76,7 @@ public List parseDirectDependencies(TomlParseResult tomlParseR private Map> parseOptionalDependencies(TomlParseResult tomlParseResult) { Map> optionalDepsMap = new HashMap<>(); - TomlTable optionalDepsTable = tomlParseResult.getTable("project.optional-dependencies"); + TomlTable optionalDepsTable = tomlParseResult.getTable(TOML_OPTIONAL_DEPENDENCIES_KEY); if (optionalDepsTable == null) { return optionalDepsMap; } From 2fc2115e2c6d97ccb7aadceed63efde6033ddb34 Mon Sep 17 00:00:00 2001 From: zahidblackduck Date: Fri, 13 Mar 2026 15:03:33 +0600 Subject: [PATCH 09/10] update release notes entry and python doc for the change --- documentation/src/main/markdown/currentreleasenotes.md | 4 +++- documentation/src/main/markdown/packagemgrs/python.md | 6 ++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/documentation/src/main/markdown/currentreleasenotes.md b/documentation/src/main/markdown/currentreleasenotes.md index 01af7d1bc1..9b4574e025 100644 --- a/documentation/src/main/markdown/currentreleasenotes.md +++ b/documentation/src/main/markdown/currentreleasenotes.md @@ -22,4 +22,6 @@ * Support for the Conda Tree–based detector has been added. For more details, see [Conda Tree](packagemgrs/conda.md#conda-tree-detector). -### Resolved issues \ No newline at end of file +### Resolved issues + +* (IDETECT-4981) The Setuptools Parse detector now resolves Python package extras (optional dependency groups) declared in `pyproject.toml`, `setup.cfg`, and `setup.py`. When a direct dependency references an extras group via bracket notation (e.g., `requests[security]`), the dependencies defined in the matching `[project.optional-dependencies]`, `[options.extras_require]`, or `extras_require` section are added as transitive children of the base package in the SBOM. See [Python support](packagemgrs/python.md) for details. \ No newline at end of file diff --git a/documentation/src/main/markdown/packagemgrs/python.md b/documentation/src/main/markdown/packagemgrs/python.md index 3144ea4fb7..0db7abb7e7 100644 --- a/documentation/src/main/markdown/packagemgrs/python.md +++ b/documentation/src/main/markdown/packagemgrs/python.md @@ -37,6 +37,12 @@ For setup.cfg and setup.py file parsing, the Setuptools detectors support direct The `--detect.pip.only.project.tree`, `--detect.pip.project.name`, and `--detect.pip.project.version.name` properties do not apply to the Setuptools detectors. +### Extras support in Setuptools Parse detector + +When a direct dependency uses bracket notation to reference an extras group (e.g., `requests[security]`), the Setuptools Parse detector matches the extras name against the project's own optional dependency definitions in `[project.optional-dependencies]` (pyproject.toml), `[options.extras_require]` (setup.cfg), or `extras_require` (setup.py, static dict literals only), and adds the matching group's dependencies as transitive children of the base package in the SBOM. The base package name is always used for Knowledge Base matching (e.g., `requests`, not `requests[security]`). + +Extras resolution in the buildless detector is limited to extras groups declared within the scanned project's own configuration files. Build detectors (Setuptools CLI, Pip Native Inspector, Pipenv, UV CLI) resolve upstream extras by downloading and installing packages, while other buildless detectors (Pipfile Lock, Poetry, UV Lock) read extras that were already resolved during lock file generation. + ## PIPENV Detectors ## Pipenv lock detector From fc4b3bf09e888674a29346c31c49c5b295ca51a4 Mon Sep 17 00:00:00 2001 From: zahidblackduck Date: Fri, 13 Mar 2026 19:35:24 +0600 Subject: [PATCH 10/10] update release note as per review --- documentation/src/main/markdown/currentreleasenotes.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/documentation/src/main/markdown/currentreleasenotes.md b/documentation/src/main/markdown/currentreleasenotes.md index 9b4574e025..2bfc8e8f93 100644 --- a/documentation/src/main/markdown/currentreleasenotes.md +++ b/documentation/src/main/markdown/currentreleasenotes.md @@ -21,7 +21,6 @@ ### New features * Support for the Conda Tree–based detector has been added. For more details, see [Conda Tree](packagemgrs/conda.md#conda-tree-detector). +* The Setuptools Parse detector now resolves Python package extras (optional dependency groups) declared in `pyproject.toml`, `setup.cfg`, and `setup.py`. When a direct dependency references an extras group via bracket notation (e.g., `requests[security]`), the dependencies defined in the matching `[project.optional-dependencies]`, `[options.extras_require]`, or `extras_require` section are added as transitive children of the base package in the SBOM. See [Python support](packagemgrs/python.md) for details. ### Resolved issues - -* (IDETECT-4981) The Setuptools Parse detector now resolves Python package extras (optional dependency groups) declared in `pyproject.toml`, `setup.cfg`, and `setup.py`. When a direct dependency references an extras group via bracket notation (e.g., `requests[security]`), the dependencies defined in the matching `[project.optional-dependencies]`, `[options.extras_require]`, or `extras_require` section are added as transitive children of the base package in the SBOM. See [Python support](packagemgrs/python.md) for details. \ No newline at end of file