diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/SetupToolsExtractUtils.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/SetupToolsExtractUtils.java index e39c25618e..44cfe9d127 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/SetupToolsExtractUtils.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/SetupToolsExtractUtils.java @@ -4,6 +4,7 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.List; +import java.util.Map; import org.apache.commons.io.FileUtils; import org.tomlj.Toml; @@ -68,8 +69,9 @@ public static SetupToolsParser resolveSetupToolsParser(TomlParseResult parsedTom SetupToolsPyParser pyParser = new SetupToolsPyParser(parsedToml); List pyDependencies = pyParser.load(pyFile.toString()); - - if (pyDependencies != null && !pyDependencies.isEmpty()) { + Map> extrasMap = pyParser.loadExtrasRequire(pyFile.toString()); + + if ((pyDependencies != null && !pyDependencies.isEmpty()) || (extrasMap != null && !extrasMap.isEmpty())) { return pyParser; } } @@ -77,13 +79,14 @@ public static SetupToolsParser resolveSetupToolsParser(TomlParseResult parsedTom // Step 3: Check the setup.cfg fileResolver = new Requirements(fileFinder, environment); File cfgFile = fileResolver.file(SETUP_CFG); - + if (cfgFile != null) { SetupToolsCfgParser cfgParser = new SetupToolsCfgParser(parsedToml); List cfgDependencies = cfgParser.load(cfgFile.toString()); + Map> extrasMap = cfgParser.loadExtrasRequire(cfgFile.toString()); - if (cfgDependencies != null && !cfgDependencies.isEmpty()) { + if ((cfgDependencies != null && !cfgDependencies.isEmpty()) || (extrasMap != null && !extrasMap.isEmpty())) { return cfgParser; } } diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsCfgParser.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsCfgParser.java index 1aa8c1ef76..697e8fb0d0 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsCfgParser.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsCfgParser.java @@ -5,45 +5,67 @@ import java.io.FileReader; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; import org.tomlj.TomlParseResult; import com.blackduck.integration.detectable.python.util.PythonDependency; import com.blackduck.integration.detectable.python.util.PythonDependencyTransformer; +import static com.blackduck.integration.detectable.detectables.setuptools.parse.SetupToolsExtrasUtils.buildExtrasTransitives; + public class SetupToolsCfgParser implements SetupToolsParser { - - private TomlParseResult parsedToml; - + + private static final String TOML_PROJECT_NAME_KEY = "project.name"; + private static final String TOML_PROJECT_VERSION_KEY = "project.version"; + + private static final String CFG_NAME_PREFIX = "name"; + private static final String CFG_INSTALL_REQUIRES_PREFIX = "install_requires="; + private static final String CFG_EXTRAS_REQUIRE_SECTION = "[options.extras_require]"; + + private static final String CONDITIONAL_MARKER = ";"; + private static final String KEY_VALUE_SEPARATOR = "="; + + // Matches a new cfg key line: e.g. "python_requires = >=3.10" + private static final String NEW_KEY_LINE_REGEX = "^\\s*[a-zA-Z0-9_.-]+\\s*=\\s*(?![=!<>~]).*$"; + + private final TomlParseResult parsedToml; + private String projectName; - - private List dependencies; - + + private final List dependencies; + + private final Map> extrasRequireMap; + public SetupToolsCfgParser(TomlParseResult parsedToml) { this.parsedToml = parsedToml; this.dependencies = new ArrayList<>(); + this.extrasRequireMap = new HashMap<>(); } @Override public SetupToolsParsedResult parse() throws IOException { - String tomlProjectName = parsedToml.getString("project.name"); - String projectVersion = parsedToml.getString("project.version"); - + String tomlProjectName = parsedToml.getString(TOML_PROJECT_NAME_KEY); + String projectVersion = parsedToml.getString(TOML_PROJECT_VERSION_KEY); + // If we have multiple project names the name from the toml wins // I've only seen version information in the toml so use that. String finalProjectName = (tomlProjectName != null && !tomlProjectName.isEmpty()) ? tomlProjectName : projectName; - + List parsedDirectDependencies = parseDirectDependencies(); - - return new SetupToolsParsedResult(finalProjectName, projectVersion, parsedDirectDependencies); + + Map> extrasTransitives = buildExtrasTransitives(dependencies, extrasRequireMap); + + return new SetupToolsParsedResult(finalProjectName, projectVersion, parsedDirectDependencies, extrasTransitives); } /** * Extracts, does not parse, any entries in the install_requires section of the * setup.cfg - * + * * @param filePath path to the setup.cfg file * @return a list of dependencies extracted from the install_requires section * @throws FileNotFoundException @@ -58,18 +80,18 @@ public List load(String filePath) throws FileNotFoundException, IOExcept while ((line = reader.readLine()) != null) { line = line.trim(); - - if (line.startsWith("name")) { + + if (line.startsWith(CFG_NAME_PREFIX)) { parseProjectName(line); } // Remove all whitespace from the line for key searching String keySearch = line.replaceAll("\\s", ""); - + // If the line starts with "install_requires=", we've found the key we're interested in - if (keySearch.startsWith("install_requires=")) { + if (keySearch.startsWith(CFG_INSTALL_REQUIRES_PREFIX)) { isInstallRequiresSection = true; - String[] parts = line.split("=", 2); + String[] parts = line.split(KEY_VALUE_SEPARATOR, 2); // If there is a value and it's not empty, add it to the dependencies list if (parts.length > 1 && !parts[1].trim().isEmpty()) { @@ -77,7 +99,7 @@ public List load(String filePath) throws FileNotFoundException, IOExcept } } else if (isInstallRequiresSection) { - if (isEndofInstallRequiresSection(line)) { + if (isEndofInstallRequiresSection(line)) { break; } // If the line is not empty, add it to the dependencies list @@ -90,19 +112,19 @@ else if (!line.isEmpty()) { return dependencies; } - + private List parseDirectDependencies() { List results = new LinkedList<>(); - + PythonDependencyTransformer dependencyTransformer = new PythonDependencyTransformer(); - for (String dependencyLine : dependencies) { + for (String dependencyLine : dependencies) { PythonDependency dependency = dependencyTransformer.transformLine(dependencyLine); - + // If we have a ; in our requirements line then there is a condition on this dependency. // We want to know this so we don't consider it a failure later if we try to run pip show // on it and we don't find it. - if (dependencyLine.contains(";")) { + if (dependencyLine.contains(CONDITIONAL_MARKER)) { dependency.setConditional(true); } @@ -110,33 +132,84 @@ private List parseDirectDependencies() { results.add(dependency); } } - + return results; } public void parseProjectName(String line) { - String[] parts = line.split("=", 2); + String[] parts = line.split(KEY_VALUE_SEPARATOR, 2); if (parts.length > 1 && !parts[1].trim().isEmpty()) { projectName = parts[1].trim(); } } + /** + * Extracts entries in the [options.extras_require] section of the setup.cfg, + * grouped by extras group name. + * + * @param filePath path to the setup.cfg file + * @return a map of group name to list of dependency strings + * @throws FileNotFoundException + * @throws IOException + */ + public Map> loadExtrasRequire(String filePath) throws FileNotFoundException, IOException { + try (BufferedReader reader = new BufferedReader(new FileReader(filePath))) { + String line; + boolean isExtrasRequireSection = false; + String currentGroup = null; + + while ((line = reader.readLine()) != null) { + String trimmedLine = line.trim(); + + if (trimmedLine.equals(CFG_EXTRAS_REQUIRE_SECTION)) { + isExtrasRequireSection = true; + continue; + } + + if (isExtrasRequireSection) { + // A new section header means we've left [options.extras_require] + if (trimmedLine.startsWith("[")) { + break; + } + + // Skip empty lines + if (trimmedLine.isEmpty()) { + continue; + } + + // In INI format, continuation lines (dependencies) are always indented. + // Group key lines (e.g., "http2 =", "security =") are not indented. + if (line.length() > 0 && (line.charAt(0) == ' ' || line.charAt(0) == '\t')) { + // Indented line — it's a dependency under the current group + if (currentGroup != null) { + extrasRequireMap.computeIfAbsent(currentGroup, k -> new ArrayList<>()).add(trimmedLine); + } + } else { + // Non-indented line — it's a group key (e.g., "security =" or "http2 =") + int equalsIndex = trimmedLine.indexOf('='); + if (equalsIndex >= 0) { + currentGroup = trimmedLine.substring(0, equalsIndex).trim(); + } + } + } + } + } + + return extrasRequireMap; + } + private boolean isEndofInstallRequiresSection(String line) { /* * If the line starts with a [ we have reached a new section and want to exit. - * - * The line.matches call looks for a new key. - * It will return true if the string starts with optional whitespace, - * followed by one or more alphanumeric characters, periods, underscores, or hyphens, + * + * The line.matches call looks for a new key. + * It will return true if the string starts with optional whitespace, + * followed by one or more alphanumeric characters, periods, underscores, or hyphens, * (which is the allowed set of characters for a key), followed by * optional whitespace, an equal sign, optional whitespace, and then any - * character that is not another =, !, <, >, or ~ which would indicate a requirement + * character that is not another =, !, <, >, or ~ which would indicate a requirement * operator and not a new key. */ - if (line.startsWith("[") || line.matches("^\\s*[a-zA-Z0-9_.-]+\\s*=\\s*(?![=!<>~]).*$")) { - return true; - } else { - return false; - } + return line.startsWith("[") || line.matches(NEW_KEY_LINE_REGEX); } } diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsExtrasUtils.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsExtrasUtils.java new file mode 100644 index 0000000000..723e941651 --- /dev/null +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsExtrasUtils.java @@ -0,0 +1,94 @@ +package com.blackduck.integration.detectable.detectables.setuptools.parse; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; + +import com.blackduck.integration.detectable.python.util.PythonDependency; +import com.blackduck.integration.detectable.python.util.PythonDependencyTransformer; + +public class SetupToolsExtrasUtils { + + private static final char OPEN_BRACKET = '['; + private static final char CLOSE_BRACKET = ']'; + private static final String EXTRAS_DELIMITER = ","; + + private SetupToolsExtrasUtils() {} + + /** + * Extracts the extras specifier names from a raw dependency string. + * For example, "requests[security,socks]==2.28.2" returns ["security", "socks"]. + * Returns an empty list if no extras specifier is present. + */ + public static List extractExtrasNames(String rawDep) { + List names = new ArrayList<>(); + int openBracket = rawDep.indexOf(OPEN_BRACKET); + int closeBracket = rawDep.indexOf(CLOSE_BRACKET); + if (openBracket >= 0 && closeBracket > openBracket) { + String extrasContent = rawDep.substring(openBracket + 1, closeBracket); + for (String name : extrasContent.split(EXTRAS_DELIMITER)) { + String trimmed = name.trim(); + if (!trimmed.isEmpty()) { + names.add(trimmed); + } + } + } + return names; + } + + /** + * Builds a map of base package name to transitive dependencies by matching + * extras specifiers in raw dependency lines against the extras group map. + * + * @param rawDependencyLines the raw dependency strings (e.g., "requests[security]==2.28.2") + * @param extrasGroupMap map of extras group name to list of dependency strings in that group + * @return map of base package name to list of transitive PythonDependency objects + */ + public static Map> buildExtrasTransitives( + List rawDependencyLines, + Map> extrasGroupMap) { + + Map> extrasTransitives = new HashMap<>(); + PythonDependencyTransformer dependencyTransformer = new PythonDependencyTransformer(); + + for (String rawDep : rawDependencyLines) { + List extrasNames = extractExtrasNames(rawDep); + if (extrasNames.isEmpty()) { + continue; + } + + // Extract the base package name (everything before '[') + int bracketIndex = rawDep.indexOf(OPEN_BRACKET); + String baseName = rawDep.substring(0, bracketIndex).trim(); + + resolveExtrasForDependency(baseName, extrasNames, extrasGroupMap, extrasTransitives, dependencyTransformer); + } + + return extrasTransitives; + } + + private static void resolveExtrasForDependency( + String baseName, + List extrasNames, + Map> extrasGroupMap, + Map> extrasTransitives, + PythonDependencyTransformer dependencyTransformer) { + + for (String extrasName : extrasNames) { + List groupLines = extrasGroupMap.get(extrasName); + if (groupLines == null) { + continue; + } + List transitives = extrasTransitives.computeIfAbsent(baseName, k -> new LinkedList<>()); + for (String transitiveLine : groupLines) { + PythonDependency dep = dependencyTransformer.transformLine(transitiveLine); + if (dep != null) { + transitives.add(dep); + } + } + } + } +} + diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsParsedResult.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsParsedResult.java index 8bb84d661b..e9682f1968 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsParsedResult.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsParsedResult.java @@ -1,6 +1,8 @@ package com.blackduck.integration.detectable.detectables.setuptools.parse; +import java.util.Collections; import java.util.List; +import java.util.Map; import com.blackduck.integration.detectable.python.util.PythonDependency; @@ -9,11 +11,17 @@ public class SetupToolsParsedResult { private String projectName; private String projectVersion; private List directDependencies; + private Map> extrasTransitives; public SetupToolsParsedResult(String projectName, String projectVersion, List parsedDirectDependencies) { + this(projectName, projectVersion, parsedDirectDependencies, Collections.emptyMap()); + } + + public SetupToolsParsedResult(String projectName, String projectVersion, List parsedDirectDependencies, Map> extrasTransitives) { this.projectName = projectName; this.projectVersion = projectVersion; this.directDependencies = parsedDirectDependencies; + this.extrasTransitives = extrasTransitives; } public String getProjectName() { @@ -27,4 +35,8 @@ public String getProjectVersion() { public List getDirectDependencies() { return directDependencies; } + + public Map> getExtrasTransitives() { + return extrasTransitives; + } } diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsPyParser.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsPyParser.java index cdfbe4017d..a4f9dddf00 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsPyParser.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsPyParser.java @@ -4,8 +4,10 @@ import java.io.FileReader; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -14,27 +16,62 @@ import com.blackduck.integration.detectable.python.util.PythonDependency; import com.blackduck.integration.detectable.python.util.PythonDependencyTransformer; +import static com.blackduck.integration.detectable.detectables.setuptools.parse.SetupToolsExtrasUtils.buildExtrasTransitives; + public class SetupToolsPyParser implements SetupToolsParser { - - private TomlParseResult parsedToml; - - private List dependencies; - + + // Regex pattern for dependencies enclosed in single quotes: ['requests==2.31.0'], + private static final Pattern PATTERN_SINGLE_QUOTES = Pattern.compile("\\[?'(.*)'\\s*\\]?,?"); + + // Regex pattern for dependencies enclosed in double quotes: ["requests==2.31.0"], + private static final Pattern PATTERN_DOUBLE_QUOTES = Pattern.compile("\\[?\"(.*)\"\\s*\\]?,?"); + + // Regex pattern for extras_require group key lines like "security": [ or 'http2': [ + private static final Pattern EXTRAS_GROUP_KEY_PATTERN = Pattern.compile("[\"'](.*?)[\"']\\s*:"); + + // Regex pattern for extracting any double-quoted string value + private static final Pattern EXTRACT_DOUBLE_QUOTES = Pattern.compile("\"(.*?)\""); + + // Regex pattern for extracting any single-quoted string value + private static final Pattern EXTRACT_SINGLE_QUOTES = Pattern.compile("'(.*?)'"); + + private static final String INSTALL_REQUIRES_PREFIX = "install_requires="; + private static final String EXTRAS_REQUIRE_PREFIX = "extras_require="; + + private static final String TOML_PROJECT_NAME_KEY = "project.name"; + private static final String TOML_PROJECT_VERSION_KEY = "project.version"; + + private static final String OPEN_BRACKET = "["; + private static final String OPEN_BRACE = "{"; + private static final String CLOSE_BRACE = "}"; + private static final String CLOSE_BRACE_COMMA = "},"; + private static final String CLOSE_BRACKET = "]"; + private static final String CLOSE_BRACKET_COMMA = "],"; + + private final TomlParseResult parsedToml; + + private final List dependencies; + + private final Map> extrasRequireMap; + public SetupToolsPyParser(TomlParseResult parsedToml) { this.parsedToml = parsedToml; this.dependencies = new ArrayList<>(); + this.extrasRequireMap = new HashMap<>(); } @Override public SetupToolsParsedResult parse() throws IOException { // Use a name from the toml if we have it. Do not parse names and versions from the setup.py // as the project will not always have a string (it could have variables or method calls) - String tomlProjectName = parsedToml.getString("project.name"); - String projectVersion = parsedToml.getString("project.version"); - + String tomlProjectName = parsedToml.getString(TOML_PROJECT_NAME_KEY); + String projectVersion = parsedToml.getString(TOML_PROJECT_VERSION_KEY); + List parsedDirectDependencies = parseDirectDependencies(); - - return new SetupToolsParsedResult(tomlProjectName, projectVersion, parsedDirectDependencies); + + Map> extrasTransitives = buildExtrasTransitives(dependencies, extrasRequireMap); + + return new SetupToolsParsedResult(tomlProjectName, projectVersion, parsedDirectDependencies, extrasTransitives); } public List load(String setupFile) throws IOException { @@ -42,35 +79,30 @@ public List load(String setupFile) throws IOException { // - "\\[?'(.*)'\\s*\\]?,?" matches dependencies that start with an optional '[' followed by a mandatory single quote, // then any characters (the dependency name), ending with a single quote followed by optional whitespace and an optional ',' or ']'. // - "\\[?\"(.*)\"\\s*\\]?,?" is similar to the first part but for dependencies enclosed in double quotes. - // Pattern for single quotes - Pattern patternSingleQuotes = Pattern.compile("\\[?'(.*)'\\s*\\]?,?"); - - // Pattern for double quotes - Pattern patternDoubleQuotes = Pattern.compile("\\[?\"(.*)\"\\s*\\]?,?"); try (BufferedReader reader = new BufferedReader(new FileReader(setupFile))) { String line; boolean isInstallRequiresSection = false; while ((line = reader.readLine()) != null) { - line = line.trim(); - + line = line.trim(); + // If after removing all whitespace the line starts with install_requires= // then we have found the section we are after. - if (line.replaceAll("\\s+","").startsWith("install_requires=")) { + if (line.replaceAll("\\s+", "").startsWith(INSTALL_REQUIRES_PREFIX)) { isInstallRequiresSection = true; continue; } if (isInstallRequiresSection) { // If the [ is on its own line skip it, it doesn't contain a dependency - if (line.equals("[")) { + if (line.equals(OPEN_BRACKET)) { continue; } - - checkLineForDependency(line, patternSingleQuotes, patternDoubleQuotes); - + + checkLineForDependency(line); + // If the line ends with ] or ], it means we have reached the end of the dependencies list. - if (line.endsWith("]") || line.endsWith("],")) { + if (line.endsWith(CLOSE_BRACKET) || line.endsWith(CLOSE_BRACKET_COMMA)) { break; } } @@ -79,25 +111,106 @@ public List load(String setupFile) throws IOException { return dependencies; } - - private void checkLineForDependency(String line, Pattern patternSingleQuotes, Pattern patternDoubleQuotes) { - // Using the pattern for double quotes to match the dependencies in the current line. - Matcher matcherDoubleQuotes = patternDoubleQuotes.matcher(line); + + public Map> loadExtrasRequire(String setupFile) throws IOException { + try (BufferedReader reader = new BufferedReader(new FileReader(setupFile))) { + String line; + boolean isExtrasRequireSection = false; + String currentGroup = null; + + while ((line = reader.readLine()) != null) { + String trimmedLine = line.trim(); + + if (trimmedLine.replaceAll("\\s+", "").startsWith(EXTRAS_REQUIRE_PREFIX)) { + isExtrasRequireSection = true; + continue; + } + + if (!isExtrasRequireSection) { + continue; + } + + if (trimmedLine.equals(OPEN_BRACE)) { + continue; + } + + if (trimmedLine.equals(CLOSE_BRACE) || trimmedLine.equals(CLOSE_BRACE_COMMA)) { + break; + } + + currentGroup = processExtrasLine(trimmedLine, currentGroup); + } + } + + return extrasRequireMap; + } + + private String processExtrasLine(String trimmedLine, String currentGroup) { + Matcher groupMatcher = EXTRAS_GROUP_KEY_PATTERN.matcher(trimmedLine); + + if (groupMatcher.find()) { + currentGroup = groupMatcher.group(1); + String afterColon = trimmedLine.substring(trimmedLine.indexOf(':') + 1).trim(); + addExtrasLineDeps(afterColon, currentGroup); + if (afterColon.contains(CLOSE_BRACKET)) { + return null; + } + } else if (currentGroup != null) { + addExtrasLineDeps(trimmedLine, currentGroup); + if (trimmedLine.endsWith(CLOSE_BRACKET) || trimmedLine.endsWith(CLOSE_BRACKET_COMMA)) { + return null; + } + } + + return currentGroup; + } + + private void addExtrasLineDeps(String line, String group) { + List deps = extractQuotedStrings(line); + for (String dep : deps) { + extrasRequireMap.computeIfAbsent(group, k -> new ArrayList<>()).add(dep); + } + } + + /** + * Extracts the first quoted string from a line, trying double quotes first, + * then single quotes. Used by both install_requires and extras_require parsing. + */ + private List extractQuotedStrings(String line) { + List results = new ArrayList<>(); + + Matcher matcherDouble = EXTRACT_DOUBLE_QUOTES.matcher(line); + boolean found = false; + while (matcherDouble.find()) { + String value = matcherDouble.group(1); + if (!value.isEmpty()) { + results.add(value); + found = true; + } + } + if (!found) { + Matcher matcherSingle = EXTRACT_SINGLE_QUOTES.matcher(line); + while (matcherSingle.find()) { + String value = matcherSingle.group(1); + if (!value.isEmpty()) { + results.add(value); + } + } + } + return results; + } + + private void checkLineForDependency(String line) { + // Try double quotes first, then fall back to single quotes. + // Double quotes are preferred as lines sometimes use double quotes with + // single quotes inside them to specify conditionals. + Matcher matcherDoubleQuotes = PATTERN_DOUBLE_QUOTES.matcher(line); if (matcherDoubleQuotes.find()) { - // Extracting the dependency from the matched group. - String dependency = matcherDoubleQuotes.group(1); - // Adding the dependency to the list. - dependencies.add(dependency); + dependencies.add(matcherDoubleQuotes.group(1)); } else { - // Fallback to use the pattern for single quotes to match the dependencies in the current - // line. We do this second as there are sometimes lines that use double quotes and then - // single quotes inside them to specify conditionals - Matcher matcherSingleQuotes = patternSingleQuotes.matcher(line); + Matcher matcherSingleQuotes = PATTERN_SINGLE_QUOTES.matcher(line); if (matcherSingleQuotes.find()) { - // Extracting the dependency from the matched group. - String dependency = matcherSingleQuotes.group(1); - // Adding the dependency to the list. - dependencies.add(dependency); + dependencies.add(matcherSingleQuotes.group(1)); } } } diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsTomlParser.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsTomlParser.java index d8bfd97fa9..22818743a6 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsTomlParser.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/parse/SetupToolsTomlParser.java @@ -1,47 +1,67 @@ package com.blackduck.integration.detectable.detectables.setuptools.parse; import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; import java.util.LinkedList; import java.util.List; +import java.util.Map; import org.tomlj.TomlArray; import org.tomlj.TomlParseResult; +import org.tomlj.TomlTable; import com.blackduck.integration.detectable.python.util.PythonDependency; import com.blackduck.integration.detectable.python.util.PythonDependencyTransformer; +import static com.blackduck.integration.detectable.detectables.setuptools.parse.SetupToolsExtrasUtils.buildExtrasTransitives; + public class SetupToolsTomlParser implements SetupToolsParser { - - private TomlParseResult parsedToml; - + + private static final String TOML_PROJECT_NAME_KEY = "project.name"; + private static final String TOML_PROJECT_VERSION_KEY = "project.version"; + private static final String TOML_PROJECT_DEPENDENCIES_KEY = "project.dependencies"; + private static final String TOML_OPTIONAL_DEPENDENCIES_KEY = "project.optional-dependencies"; + + private static final String CONDITIONAL_MARKER = ";"; + + private final TomlParseResult parsedToml; + private final List rawDependencyLines; + public SetupToolsTomlParser(TomlParseResult parsedToml) { this.parsedToml = parsedToml; + this.rawDependencyLines = new ArrayList<>(); } @Override public SetupToolsParsedResult parse() throws IOException { List parsedDirectDependencies = parseDirectDependencies(parsedToml); - String projectName = parsedToml.getString("project.name"); - String projectVersion = parsedToml.getString("project.version"); - - return new SetupToolsParsedResult(projectName, projectVersion, parsedDirectDependencies); + String projectName = parsedToml.getString(TOML_PROJECT_NAME_KEY); + String projectVersion = parsedToml.getString(TOML_PROJECT_VERSION_KEY); + + // Parse optional-dependencies and build extras transitives + Map> optionalDepsMap = parseOptionalDependencies(parsedToml); + Map> extrasTransitives = buildExtrasTransitives(rawDependencyLines, optionalDepsMap); + + return new SetupToolsParsedResult(projectName, projectVersion, parsedDirectDependencies, extrasTransitives); } public List parseDirectDependencies(TomlParseResult tomlParseResult) throws IOException { List results = new LinkedList<>(); PythonDependencyTransformer dependencyTransformer = new PythonDependencyTransformer(); - TomlArray dependencies = tomlParseResult.getArray("project.dependencies"); + TomlArray dependencies = tomlParseResult.getArray(TOML_PROJECT_DEPENDENCIES_KEY); for (int i = 0; i < dependencies.size(); i++) { String dependencyLine = dependencies.getString(i); - + rawDependencyLines.add(dependencyLine); + PythonDependency dependency = dependencyTransformer.transformLine(dependencyLine); - + // If we have a ; in our requirements line then there is a condition on this dependency. // We want to know this so we don't consider it a failure later if we try to run pip show // on it and we don't find it. - if (dependencyLine.contains(";")) { + if (dependencyLine.contains(CONDITIONAL_MARKER)) { dependency.setConditional(true); } @@ -49,7 +69,29 @@ public List parseDirectDependencies(TomlParseResult tomlParseR results.add(dependency); } } - + return results; } + + private Map> parseOptionalDependencies(TomlParseResult tomlParseResult) { + Map> optionalDepsMap = new HashMap<>(); + + TomlTable optionalDepsTable = tomlParseResult.getTable(TOML_OPTIONAL_DEPENDENCIES_KEY); + if (optionalDepsTable == null) { + return optionalDepsMap; + } + + for (String groupName : optionalDepsTable.keySet()) { + TomlArray groupArray = optionalDepsTable.getArray(groupName); + if (groupArray != null) { + List groupDeps = new ArrayList<>(); + for (int i = 0; i < groupArray.size(); i++) { + groupDeps.add(groupArray.getString(i)); + } + optionalDepsMap.put(groupName, groupDeps); + } + } + + return optionalDepsMap; + } } diff --git a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/transform/SetupToolsGraphTransformer.java b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/transform/SetupToolsGraphTransformer.java index 80ae7b14b6..916efac673 100644 --- a/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/transform/SetupToolsGraphTransformer.java +++ b/detectable/src/main/java/com/blackduck/integration/detectable/detectables/setuptools/transform/SetupToolsGraphTransformer.java @@ -61,11 +61,12 @@ public DependencyGraph transform(ExecutableTarget pipExe, SetupToolsParsedResult private void handleParsedDependencies(SetupToolsParsedResult parsedResult, DependencyGraph dependencyGraph) { List directDependencies = parsedResult.getDirectDependencies(); - + Map> extrasTransitives = parsedResult.getExtrasTransitives(); + for (PythonDependency directDependency : directDependencies) { String name = directDependency.getName(); String version = directDependency.getVersion(); - + Dependency currentDependency; if (StringUtils.isEmpty(version)) { currentDependency = entryToDependency(name); @@ -73,6 +74,20 @@ private void handleParsedDependencies(SetupToolsParsedResult parsedResult, Depen currentDependency = entryToDependency(name, version); } dependencyGraph.addChildrenToRoot(currentDependency); + + if (extrasTransitives.containsKey(name)) { + for (PythonDependency transitiveDep : extrasTransitives.get(name)) { + String transName = transitiveDep.getName(); + String transVersion = transitiveDep.getVersion(); + Dependency transitiveDependency; + if (StringUtils.isEmpty(transVersion)) { + transitiveDependency = entryToDependency(transName); + } else { + transitiveDependency = entryToDependency(transName, transVersion); + } + dependencyGraph.addChildWithParent(transitiveDependency, currentDependency); + } + } } } diff --git a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/PyprojectTomlParserTest.java b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/PyprojectTomlParserTest.java index 60e4f9a9b8..55b5d28a8e 100644 --- a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/PyprojectTomlParserTest.java +++ b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/PyprojectTomlParserTest.java @@ -5,6 +5,8 @@ import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Test; import org.tomlj.Toml; @@ -12,6 +14,7 @@ import com.blackduck.integration.detectable.detectables.setuptools.parse.SetupToolsParsedResult; import com.blackduck.integration.detectable.detectables.setuptools.parse.SetupToolsTomlParser; +import com.blackduck.integration.detectable.python.util.PythonDependency; class PyprojectTomlParserTest { @@ -84,6 +87,15 @@ void testParseComplexPyprojectToml() throws IOException { assertTrue(result.contains("project.optional-dependencies.dev")); assertTrue(result.contains("project.optional-dependencies.docs")); + // Verify extras transitives: requests[security,socks] references "security" and "socks" extras, + // but neither exists as an optional-dependencies group (only "dev" and "docs" exist). + // Similarly, pandas[all] references "all" which also doesn't exist as a group. + // So the extras transitives map should be empty. + Map> extrasTransitives = parsedResult.getExtrasTransitives(); + assertNotNull(extrasTransitives); + assertTrue(extrasTransitives.isEmpty(), + "No extras transitives expected because no dependency extras names match any optional-dependencies group"); + Files.delete(pyProjectFile); // Clean up the temporary file } } \ No newline at end of file diff --git a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsCfgParserTest.java b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsCfgParserTest.java index ed1267eaac..86a10463e6 100644 --- a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsCfgParserTest.java +++ b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsCfgParserTest.java @@ -1,6 +1,7 @@ package com.blackduck.integration.detectable.detectables.setuptools.unit; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -8,6 +9,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Test; import org.tomlj.Toml; @@ -62,4 +64,203 @@ public void testParse() throws IOException { Files.delete(cfgFile); // delete the temporary file } + + @Test + public void testLoadExtrasRequireReturnsGroupedMap() throws IOException { + String cfgContent = "[metadata]\n" + + "name = sample\n" + + "\n" + + "[options.extras_require]\n" + + "http2 =\n" + + " h2>=4,<5\n" + + " hpack>=4,<5\n" + + "security =\n" + + " pyOpenSSL>=23.0\n" + + " cryptography>=41.0\n" + + "cli =\n" + + " click==8.*\n" + + " rich>=10,<13\n"; + Path tempFile = Files.createTempFile("setup", ".cfg"); + Files.write(tempFile, cfgContent.getBytes()); + + TomlParseResult result = Toml.parse("[build-system]\nrequires = [\"setuptools\"]"); + + SetupToolsCfgParser cfgParser = new SetupToolsCfgParser(result); + Map> extrasMap = cfgParser.loadExtrasRequire(tempFile.toString()); + + assertEquals(3, extrasMap.size()); + + assertTrue(extrasMap.containsKey("http2")); + assertEquals(2, extrasMap.get("http2").size()); + assertTrue(extrasMap.get("http2").contains("h2>=4,<5")); + assertTrue(extrasMap.get("http2").contains("hpack>=4,<5")); + + assertTrue(extrasMap.containsKey("security")); + assertEquals(2, extrasMap.get("security").size()); + assertTrue(extrasMap.get("security").contains("pyOpenSSL>=23.0")); + assertTrue(extrasMap.get("security").contains("cryptography>=41.0")); + + assertTrue(extrasMap.containsKey("cli")); + assertEquals(2, extrasMap.get("cli").size()); + assertTrue(extrasMap.get("cli").contains("click==8.*")); + assertTrue(extrasMap.get("cli").contains("rich>=10,<13")); + + Files.delete(tempFile); + } + + @Test + public void testParseWithExtrasTransitives() throws IOException { + String cfgContent = "[metadata]\n" + + "name = sample\n" + + "\n" + + "[options]\n" + + "install_requires =\n" + + " requests[security]==2.28.2\n" + + " httpx[http2]==0.23.3\n" + + "\n" + + "[options.extras_require]\n" + + "http2 =\n" + + " certifi>=2025.11.12\n" + + " httpcore>=1.0.9\n" + + "security =\n" + + " charset-normalizer>=3.3.1\n" + + " idna>=3.11\n"; + Path tempFile = Files.createTempFile("setup", ".cfg"); + Files.write(tempFile, cfgContent.getBytes()); + + TomlParseResult result = Toml.parse("[build-system]\nrequires = [\"setuptools\"]"); + + SetupToolsCfgParser cfgParser = new SetupToolsCfgParser(result); + cfgParser.load(tempFile.toString()); + cfgParser.loadExtrasRequire(tempFile.toString()); + SetupToolsParsedResult parsedResult = cfgParser.parse(); + + // Direct dependencies + assertEquals(2, parsedResult.getDirectDependencies().size()); + + // Extras transitives + Map> extrasTransitives = parsedResult.getExtrasTransitives(); + assertNotNull(extrasTransitives); + assertEquals(2, extrasTransitives.size()); + + // requests -> security group + assertTrue(extrasTransitives.containsKey("requests")); + List requestsTransitives = extrasTransitives.get("requests"); + assertEquals(2, requestsTransitives.size()); + assertEquals("charset-normalizer", requestsTransitives.get(0).getName()); + assertEquals("3.3.1", requestsTransitives.get(0).getVersion()); + assertEquals("idna", requestsTransitives.get(1).getName()); + assertEquals("3.11", requestsTransitives.get(1).getVersion()); + + // httpx -> http2 group + assertTrue(extrasTransitives.containsKey("httpx")); + List httpxTransitives = extrasTransitives.get("httpx"); + assertEquals(2, httpxTransitives.size()); + assertEquals("certifi", httpxTransitives.get(0).getName()); + assertEquals("2025.11.12", httpxTransitives.get(0).getVersion()); + assertEquals("httpcore", httpxTransitives.get(1).getName()); + assertEquals("1.0.9", httpxTransitives.get(1).getVersion()); + + Files.delete(tempFile); + } + + @Test + public void testExtrasGroupNotMatchingInstallRequiresIsIgnored() throws IOException { + String cfgContent = "[metadata]\n" + + "name = sample\n" + + "\n" + + "[options]\n" + + "install_requires =\n" + + " requests==2.28.2\n" + + "\n" + + "[options.extras_require]\n" + + "dev =\n" + + " pytest>=7.0\n" + + " flake8>=5.0\n"; + Path tempFile = Files.createTempFile("setup", ".cfg"); + Files.write(tempFile, cfgContent.getBytes()); + + TomlParseResult result = Toml.parse("[build-system]\nrequires = [\"setuptools\"]"); + + SetupToolsCfgParser cfgParser = new SetupToolsCfgParser(result); + cfgParser.load(tempFile.toString()); + cfgParser.loadExtrasRequire(tempFile.toString()); + SetupToolsParsedResult parsedResult = cfgParser.parse(); + + // Direct dependency has no extras specifier, so no match + assertEquals(1, parsedResult.getDirectDependencies().size()); + assertEquals("requests", parsedResult.getDirectDependencies().get(0).getName()); + + // Extras transitives should be empty since "dev" doesn't match any install_requires extras + assertTrue(parsedResult.getExtrasTransitives().isEmpty()); + + Files.delete(tempFile); + } + + @Test + public void testParseWithMultiExtrasTransitives() throws IOException { + String cfgContent = "[metadata]\n" + + "name = sample\n" + + "\n" + + "[options]\n" + + "install_requires =\n" + + " requests[security,socks]==2.28.2\n" + + "\n" + + "[options.extras_require]\n" + + "security =\n" + + " pyOpenSSL>=23.0\n" + + " cryptography>=41.0\n" + + "socks =\n" + + " PySocks>=1.5.6\n"; + Path tempFile = Files.createTempFile("setup", ".cfg"); + Files.write(tempFile, cfgContent.getBytes()); + + TomlParseResult result = Toml.parse("[build-system]\nrequires = [\"setuptools\"]"); + + SetupToolsCfgParser cfgParser = new SetupToolsCfgParser(result); + cfgParser.load(tempFile.toString()); + cfgParser.loadExtrasRequire(tempFile.toString()); + SetupToolsParsedResult parsedResult = cfgParser.parse(); + + // Direct dependencies + assertEquals(1, parsedResult.getDirectDependencies().size()); + + // Extras transitives: both security and socks groups should be merged under "requests" + Map> extrasTransitives = parsedResult.getExtrasTransitives(); + assertNotNull(extrasTransitives); + assertEquals(1, extrasTransitives.size()); + assertTrue(extrasTransitives.containsKey("requests")); + + List requestsTransitives = extrasTransitives.get("requests"); + assertEquals(3, requestsTransitives.size()); + assertEquals("pyOpenSSL", requestsTransitives.get(0).getName()); + assertEquals("cryptography", requestsTransitives.get(1).getName()); + assertEquals("PySocks", requestsTransitives.get(2).getName()); + + Files.delete(tempFile); + } + + @Test + public void testLoadExtrasRequireStopsAtNextSection() throws IOException { + String cfgContent = "[options.extras_require]\n" + + "dev =\n" + + " pytest>=7.0\n" + + "\n" + + "[options.package_data]\n" + + "* = *.txt\n"; + Path tempFile = Files.createTempFile("setup", ".cfg"); + Files.write(tempFile, cfgContent.getBytes()); + + TomlParseResult result = Toml.parse("[build-system]\nrequires = [\"setuptools\"]"); + + SetupToolsCfgParser cfgParser = new SetupToolsCfgParser(result); + Map> extrasMap = cfgParser.loadExtrasRequire(tempFile.toString()); + + assertEquals(1, extrasMap.size()); + assertTrue(extrasMap.containsKey("dev")); + assertEquals(1, extrasMap.get("dev").size()); + assertTrue(extrasMap.get("dev").contains("pytest>=7.0")); + + Files.delete(tempFile); + } } diff --git a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsGraphTransformerTest.java b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsGraphTransformerTest.java index f7af7289f1..e47a5e10a2 100644 --- a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsGraphTransformerTest.java +++ b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsGraphTransformerTest.java @@ -6,7 +6,9 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -85,4 +87,47 @@ public void testPipTransform() throws ExecutableRunnerException { graphAssert.hasRootDependency("certifi", "2022.6.15"); } + + @Test + public void testNoPipTransformWithExtrasTransitives() throws ExecutableRunnerException { + List dependencies = new ArrayList<>(); + dependencies.add(new PythonDependency("requests", "2.28.2")); + dependencies.add(new PythonDependency("httpx", "0.23.3")); + + Map> extrasTransitives = new HashMap<>(); + extrasTransitives.put("requests", Arrays.asList( + new PythonDependency("charset-normalizer", "3.3.1"), + new PythonDependency("idna", "3.11") + )); + extrasTransitives.put("httpx", Arrays.asList( + new PythonDependency("certifi", "2025.11.12"), + new PythonDependency("httpcore", "1.0.9") + )); + + SetupToolsParsedResult parsedResult = new SetupToolsParsedResult("sample", "1.0.0", dependencies, extrasTransitives); + + SetupToolsGraphTransformer graphTransformer = new SetupToolsGraphTransformer(null, new ExternalIdFactory(), null); + DependencyGraph dependencyGraph = graphTransformer.transform(null, parsedResult); + + assertNotNull(dependencyGraph); + assertEquals(2, dependencyGraph.getRootDependencies().size()); + + NameVersionGraphAssert graphAssert = new NameVersionGraphAssert(Forge.PYPI, dependencyGraph); + + // Direct dependencies are at root + graphAssert.hasRootDependency("requests", "2.28.2"); + graphAssert.hasRootDependency("httpx", "0.23.3"); + + // Transitives exist in graph + graphAssert.hasDependency("charset-normalizer", "3.3.1"); + graphAssert.hasDependency("idna", "3.11"); + graphAssert.hasDependency("certifi", "2025.11.12"); + graphAssert.hasDependency("httpcore", "1.0.9"); + + // Verify parent-child relationships + graphAssert.hasParentChildRelationship("requests", "2.28.2", "charset-normalizer", "3.3.1"); + graphAssert.hasParentChildRelationship("requests", "2.28.2", "idna", "3.11"); + graphAssert.hasParentChildRelationship("httpx", "0.23.3", "certifi", "2025.11.12"); + graphAssert.hasParentChildRelationship("httpx", "0.23.3", "httpcore", "1.0.9"); + } } diff --git a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsPyParserTest.java b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsPyParserTest.java index f0622cebf3..cfd63526b1 100644 --- a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsPyParserTest.java +++ b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsPyParserTest.java @@ -1,6 +1,7 @@ package com.blackduck.integration.detectable.detectables.setuptools.unit; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; @@ -8,12 +9,12 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Test; import org.tomlj.Toml; import org.tomlj.TomlParseResult; -import com.blackduck.integration.detectable.detectables.setuptools.parse.SetupToolsCfgParser; import com.blackduck.integration.detectable.detectables.setuptools.parse.SetupToolsParsedResult; import com.blackduck.integration.detectable.detectables.setuptools.parse.SetupToolsPyParser; import com.blackduck.integration.detectable.python.util.PythonDependency; @@ -60,4 +61,150 @@ public void testParse() throws IOException { Files.delete(pyFile); // delete the temporary file } + + @Test + public void testLoadExtrasRequire() throws IOException { + String pyContent = "from setuptools import setup\n\n" + + "setup(\n" + + " install_requires=[\n" + + " 'requests==2.28.2',\n" + + " ],\n" + + " extras_require={\n" + + " \"http2\": [\n" + + " \"certifi>=2025.11.12\",\n" + + " \"httpcore>=1.0.9\",\n" + + " ],\n" + + " \"security\": [\n" + + " \"certifi>=2025.11.12\",\n" + + " \"charset-normalizer>=3.3.1\",\n" + + " ],\n" + + " },\n" + + ")\n"; + + Path tempFile = Files.createTempFile("setup", ".py"); + Files.write(tempFile, pyContent.getBytes()); + + TomlParseResult toml = Toml.parse(""); + SetupToolsPyParser pyParser = new SetupToolsPyParser(toml); + Map> extrasMap = pyParser.loadExtrasRequire(tempFile.toString()); + + assertEquals(2, extrasMap.size()); + assertTrue(extrasMap.containsKey("http2")); + assertTrue(extrasMap.containsKey("security")); + assertEquals(2, extrasMap.get("http2").size()); + assertTrue(extrasMap.get("http2").contains("certifi>=2025.11.12")); + assertTrue(extrasMap.get("http2").contains("httpcore>=1.0.9")); + assertEquals(2, extrasMap.get("security").size()); + assertTrue(extrasMap.get("security").contains("certifi>=2025.11.12")); + assertTrue(extrasMap.get("security").contains("charset-normalizer>=3.3.1")); + + Files.delete(tempFile); + } + + @Test + public void testParseWithExtrasTransitives() throws IOException { + String pyContent = "from setuptools import setup\n\n" + + "setup(\n" + + " install_requires=[\n" + + " 'requests[security]==2.28.2',\n" + + " ],\n" + + " extras_require={\n" + + " \"security\": [\n" + + " \"certifi>=2025.11.12\",\n" + + " \"charset-normalizer>=3.3.1\",\n" + + " ],\n" + + " },\n" + + ")\n"; + + Path tempFile = Files.createTempFile("setup", ".py"); + Files.write(tempFile, pyContent.getBytes()); + + String tomlContent = "[build-system]\nrequires = [\"setuptools\"]"; + TomlParseResult toml = Toml.parse(tomlContent); + + SetupToolsPyParser pyParser = new SetupToolsPyParser(toml); + pyParser.load(tempFile.toString()); + pyParser.loadExtrasRequire(tempFile.toString()); + SetupToolsParsedResult parsedResult = pyParser.parse(); + + assertEquals(1, parsedResult.getDirectDependencies().size()); + assertEquals("requests", parsedResult.getDirectDependencies().get(0).getName()); + + Map> extrasTransitives = parsedResult.getExtrasTransitives(); + assertNotNull(extrasTransitives); + assertTrue(extrasTransitives.containsKey("requests")); + assertEquals(2, extrasTransitives.get("requests").size()); + assertEquals("certifi", extrasTransitives.get("requests").get(0).getName()); + assertEquals("charset-normalizer", extrasTransitives.get("requests").get(1).getName()); + + Files.delete(tempFile); + } + + @Test + public void testExtrasGroupNotMatchingInstallRequiresIsIgnored() throws IOException { + String pyContent = "from setuptools import setup\n\n" + + "setup(\n" + + " install_requires=[\n" + + " 'requests==2.28.2',\n" + + " ],\n" + + " extras_require={\n" + + " \"dev\": [\n" + + " \"pytest>=7.0\",\n" + + " ],\n" + + " },\n" + + ")\n"; + + Path tempFile = Files.createTempFile("setup", ".py"); + Files.write(tempFile, pyContent.getBytes()); + + String tomlContent = "[build-system]\nrequires = [\"setuptools\"]"; + TomlParseResult toml = Toml.parse(tomlContent); + + SetupToolsPyParser pyParser = new SetupToolsPyParser(toml); + pyParser.load(tempFile.toString()); + pyParser.loadExtrasRequire(tempFile.toString()); + SetupToolsParsedResult parsedResult = pyParser.parse(); + + assertTrue(parsedResult.getExtrasTransitives().isEmpty()); + + Files.delete(tempFile); + } + + @Test + public void testParseWithMultiExtrasTransitives() throws IOException { + String pyContent = "from setuptools import setup\n\n" + + "setup(\n" + + " install_requires=[\n" + + " 'requests[security,socks]==2.28.2',\n" + + " ],\n" + + " extras_require={\n" + + " \"security\": [\n" + + " \"certifi>=2025.11.12\",\n" + + " ],\n" + + " \"socks\": [\n" + + " \"PySocks>=1.5.6\",\n" + + " ],\n" + + " },\n" + + ")\n"; + + Path tempFile = Files.createTempFile("setup", ".py"); + Files.write(tempFile, pyContent.getBytes()); + + String tomlContent = "[build-system]\nrequires = [\"setuptools\"]"; + TomlParseResult toml = Toml.parse(tomlContent); + + SetupToolsPyParser pyParser = new SetupToolsPyParser(toml); + pyParser.load(tempFile.toString()); + pyParser.loadExtrasRequire(tempFile.toString()); + SetupToolsParsedResult parsedResult = pyParser.parse(); + + Map> extrasTransitives = parsedResult.getExtrasTransitives(); + assertNotNull(extrasTransitives); + assertTrue(extrasTransitives.containsKey("requests")); + assertEquals(2, extrasTransitives.get("requests").size()); + assertEquals("certifi", extrasTransitives.get("requests").get(0).getName()); + assertEquals("PySocks", extrasTransitives.get("requests").get(1).getName()); + + Files.delete(tempFile); + } } diff --git a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsTomlParserTest.java b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsTomlParserTest.java index 85e2eabfc4..6927fb1b6b 100644 --- a/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsTomlParserTest.java +++ b/detectable/src/test/java/com/blackduck/integration/detectable/detectables/setuptools/unit/SetupToolsTomlParserTest.java @@ -1,10 +1,14 @@ package com.blackduck.integration.detectable.detectables.setuptools.unit; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; +import java.util.Map; import org.junit.jupiter.api.Test; import org.tomlj.Toml; @@ -12,9 +16,10 @@ import com.blackduck.integration.detectable.detectables.setuptools.parse.SetupToolsParsedResult; import com.blackduck.integration.detectable.detectables.setuptools.parse.SetupToolsTomlParser; +import com.blackduck.integration.detectable.python.util.PythonDependency; public class SetupToolsTomlParserTest { - + @Test public void testParse() throws IOException { String tomlContent = "[project]\nname = \"setuptools\"\nversion = \"1.0.0\"\ndependencies = [\n \"requests\",\n \"numpy\"\n]"; @@ -32,4 +37,80 @@ public void testParse() throws IOException { Files.delete(pyProjectFile); // delete the temporary file } + + @Test + public void testParseWithExtrasTransitives() throws IOException { + String tomlContent = "[project]\n" + + "name = \"myproject\"\n" + + "version = \"1.0.0\"\n" + + "dependencies = [\n" + + " \"requests[security]==2.28.2\",\n" + + " \"httpx[http2]==0.23.3\"\n" + + "]\n\n" + + "[project.optional-dependencies]\n" + + "security = [\n" + + " \"charset-normalizer>=3.3.1\",\n" + + " \"idna>=3.11\"\n" + + "]\n" + + "http2 = [\n" + + " \"certifi>=2025.11.12\",\n" + + " \"httpcore>=1.0.9\"\n" + + "]\n"; + + TomlParseResult result = Toml.parse(tomlContent); + SetupToolsTomlParser tomlParser = new SetupToolsTomlParser(result); + SetupToolsParsedResult parsedResult = tomlParser.parse(); + + // Direct dependencies + assertEquals(2, parsedResult.getDirectDependencies().size()); + + // Extras transitives + Map> extrasTransitives = parsedResult.getExtrasTransitives(); + assertNotNull(extrasTransitives); + assertEquals(2, extrasTransitives.size()); + + // requests -> security group + assertTrue(extrasTransitives.containsKey("requests")); + List requestsTransitives = extrasTransitives.get("requests"); + assertEquals(2, requestsTransitives.size()); + assertEquals("charset-normalizer", requestsTransitives.get(0).getName()); + assertEquals("3.3.1", requestsTransitives.get(0).getVersion()); + assertEquals("idna", requestsTransitives.get(1).getName()); + assertEquals("3.11", requestsTransitives.get(1).getVersion()); + + // httpx -> http2 group + assertTrue(extrasTransitives.containsKey("httpx")); + List httpxTransitives = extrasTransitives.get("httpx"); + assertEquals(2, httpxTransitives.size()); + assertEquals("certifi", httpxTransitives.get(0).getName()); + assertEquals("2025.11.12", httpxTransitives.get(0).getVersion()); + assertEquals("httpcore", httpxTransitives.get(1).getName()); + assertEquals("1.0.9", httpxTransitives.get(1).getVersion()); + } + + @Test + public void testExtrasGroupNotMatchingDependenciesIsIgnored() throws IOException { + String tomlContent = "[project]\n" + + "name = \"myproject\"\n" + + "version = \"1.0.0\"\n" + + "dependencies = [\n" + + " \"requests==2.28.2\"\n" + + "]\n\n" + + "[project.optional-dependencies]\n" + + "dev = [\n" + + " \"pytest>=7.0\",\n" + + " \"flake8>=5.0\"\n" + + "]\n"; + + TomlParseResult result = Toml.parse(tomlContent); + SetupToolsTomlParser tomlParser = new SetupToolsTomlParser(result); + SetupToolsParsedResult parsedResult = tomlParser.parse(); + + // Direct dependency has no extras specifier, so no match + assertEquals(1, parsedResult.getDirectDependencies().size()); + assertEquals("requests", parsedResult.getDirectDependencies().get(0).getName()); + + // Extras transitives should be empty since "dev" doesn't match any dependency extras + assertTrue(parsedResult.getExtrasTransitives().isEmpty()); + } } diff --git a/documentation/src/main/markdown/currentreleasenotes.md b/documentation/src/main/markdown/currentreleasenotes.md index 01af7d1bc1..2bfc8e8f93 100644 --- a/documentation/src/main/markdown/currentreleasenotes.md +++ b/documentation/src/main/markdown/currentreleasenotes.md @@ -21,5 +21,6 @@ ### New features * Support for the Conda Tree–based detector has been added. For more details, see [Conda Tree](packagemgrs/conda.md#conda-tree-detector). +* The Setuptools Parse detector now resolves Python package extras (optional dependency groups) declared in `pyproject.toml`, `setup.cfg`, and `setup.py`. When a direct dependency references an extras group via bracket notation (e.g., `requests[security]`), the dependencies defined in the matching `[project.optional-dependencies]`, `[options.extras_require]`, or `extras_require` section are added as transitive children of the base package in the SBOM. See [Python support](packagemgrs/python.md) for details. -### Resolved issues \ No newline at end of file +### Resolved issues diff --git a/documentation/src/main/markdown/packagemgrs/python.md b/documentation/src/main/markdown/packagemgrs/python.md index 3144ea4fb7..0db7abb7e7 100644 --- a/documentation/src/main/markdown/packagemgrs/python.md +++ b/documentation/src/main/markdown/packagemgrs/python.md @@ -37,6 +37,12 @@ For setup.cfg and setup.py file parsing, the Setuptools detectors support direct The `--detect.pip.only.project.tree`, `--detect.pip.project.name`, and `--detect.pip.project.version.name` properties do not apply to the Setuptools detectors. +### Extras support in Setuptools Parse detector + +When a direct dependency uses bracket notation to reference an extras group (e.g., `requests[security]`), the Setuptools Parse detector matches the extras name against the project's own optional dependency definitions in `[project.optional-dependencies]` (pyproject.toml), `[options.extras_require]` (setup.cfg), or `extras_require` (setup.py, static dict literals only), and adds the matching group's dependencies as transitive children of the base package in the SBOM. The base package name is always used for Knowledge Base matching (e.g., `requests`, not `requests[security]`). + +Extras resolution in the buildless detector is limited to extras groups declared within the scanned project's own configuration files. Build detectors (Setuptools CLI, Pip Native Inspector, Pipenv, UV CLI) resolve upstream extras by downloading and installing packages, while other buildless detectors (Pipfile Lock, Poetry, UV Lock) read extras that were already resolved during lock file generation. + ## PIPENV Detectors ## Pipenv lock detector