diff --git a/README.md b/README.md index b4e86e6f..9e8eb998 100644 --- a/README.md +++ b/README.md @@ -182,7 +182,7 @@ public class TrustifyExample {
  • Java - Maven
  • JavaScript - Npm
  • Golang - Go Modules
  • -
  • Python - pip Installer (requirements.txt, pyproject.toml)
  • +
  • Python - pip Installer (requirements.txt, pyproject.toml with PEP 621 format). Note: Poetry-style dependencies ([tool.poetry.dependencies]) are not supported.
  • Gradle - Gradle Installation
  • Rust - Cargo
  • diff --git a/src/main/java/io/github/guacsec/trustifyda/providers/PythonPipProvider.java b/src/main/java/io/github/guacsec/trustifyda/providers/PythonPipProvider.java index ab44ceac..7d12c540 100644 --- a/src/main/java/io/github/guacsec/trustifyda/providers/PythonPipProvider.java +++ b/src/main/java/io/github/guacsec/trustifyda/providers/PythonPipProvider.java @@ -16,27 +16,79 @@ */ package io.github.guacsec.trustifyda.providers; +import static io.github.guacsec.trustifyda.impl.ExhortApi.debugLoggingIsNeeded; + +import com.fasterxml.jackson.core.JsonProcessingException; import com.github.packageurl.PackageURL; +import io.github.guacsec.trustifyda.Api; +import io.github.guacsec.trustifyda.logging.LoggersFactory; +import io.github.guacsec.trustifyda.sbom.Sbom; +import io.github.guacsec.trustifyda.sbom.SbomFactory; +import io.github.guacsec.trustifyda.tools.Operations; +import io.github.guacsec.trustifyda.utils.Environment; import io.github.guacsec.trustifyda.utils.PythonControllerBase; +import io.github.guacsec.trustifyda.utils.PythonControllerRealEnv; +import io.github.guacsec.trustifyda.utils.PythonControllerVirtualEnv; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; +import java.util.List; +import java.util.Map; import java.util.Set; +import java.util.logging.Logger; import java.util.stream.Collectors; public final class PythonPipProvider extends PythonProvider { + private static final Logger log = LoggersFactory.getLogger(PythonPipProvider.class.getName()); + + private PythonControllerBase pythonController; + public PythonPipProvider(Path manifest) { super(manifest); } + public void setPythonController(PythonControllerBase pythonController) { + this.pythonController = pythonController; + } + @Override - protected Path getRequirementsPath() { - return manifest; + public Content provideStack() throws IOException { + PythonControllerBase controller = getPythonController(); + List> dependencies = controller.getDependencies(manifest.toString(), true); + printDependenciesTree(dependencies); + Sbom sbom = SbomFactory.newInstance(Sbom.BelongingCondition.PURL, "sensitive"); + sbom.addRoot( + toPurl(getRootComponentName(), getRootComponentVersion()), readLicenseFromManifest()); + for (Map component : dependencies) { + addAllDependencies(sbom.getRoot(), component, sbom); + } + String manifestContent = Files.readString(manifest); + handleIgnoredDependencies(manifestContent, sbom); + return new Content( + sbom.getAsJsonString().getBytes(StandardCharsets.UTF_8), Api.CYCLONEDX_MEDIA_TYPE); } @Override - protected void cleanupRequirementsPath(Path requirementsPath) { - // No cleanup needed — the manifest is the requirements file itself. + public Content provideComponent() throws IOException { + PythonControllerBase controller = getPythonController(); + List> dependencies = controller.getDependencies(manifest.toString(), false); + printDependenciesTree(dependencies); + Sbom sbom = SbomFactory.newInstance(); + sbom.addRoot( + toPurl(getRootComponentName(), getRootComponentVersion()), readLicenseFromManifest()); + dependencies.forEach( + (component) -> + sbom.addDependency( + sbom.getRoot(), + toPurl((String) component.get("name"), (String) component.get("version")), + null)); + String manifestContent = Files.readString(manifest); + handleIgnoredDependencies(manifestContent, sbom); + return new Content( + sbom.getAsJsonString().getBytes(StandardCharsets.UTF_8), Api.CYCLONEDX_MEDIA_TYPE); } @Override @@ -50,6 +102,107 @@ protected Set getIgnoredDependencies(String manifestContent) { .collect(Collectors.toSet()); } + @SuppressWarnings("unchecked") + private void addAllDependencies(PackageURL source, Map component, Sbom sbom) { + PackageURL packageURL = + toPurl((String) component.get("name"), (String) component.get("version")); + sbom.addDependency(source, packageURL, null); + + List> directDeps = + (List>) component.get("dependencies"); + if (directDeps != null) { + for (Map dep : directDeps) { + addAllDependencies(packageURL, dep, sbom); + } + } + } + + private void printDependenciesTree(List> dependencies) + throws JsonProcessingException { + if (debugLoggingIsNeeded()) { + String pythonControllerTree = + objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(dependencies); + log.info( + String.format( + "Python Generated Dependency Tree in Json Format: %s %s %s", + System.lineSeparator(), pythonControllerTree, System.lineSeparator())); + } + } + + private PythonControllerBase getPythonController() { + String pythonPipBinaries; + boolean useVirtualPythonEnv; + if (!Environment.get(PythonControllerBase.PROP_TRUSTIFY_DA_PIP_SHOW, "").trim().isEmpty() + && !Environment.get(PythonControllerBase.PROP_TRUSTIFY_DA_PIP_FREEZE, "") + .trim() + .isEmpty()) { + pythonPipBinaries = "python;;pip"; + useVirtualPythonEnv = false; + } else { + pythonPipBinaries = getExecutable("python", "--version"); + useVirtualPythonEnv = + Environment.getBoolean(PythonControllerBase.PROP_TRUSTIFY_DA_PYTHON_VIRTUAL_ENV, false); + } + + String[] parts = pythonPipBinaries.split(";;"); + var python = parts[0]; + var pip = parts[1]; + PythonControllerBase controller; + if (this.pythonController == null) { + if (useVirtualPythonEnv) { + controller = new PythonControllerVirtualEnv(python); + } else { + controller = new PythonControllerRealEnv(python, pip); + } + } else { + controller = this.pythonController; + } + return controller; + } + + private String getExecutable(String command, String args) { + String python = Operations.getCustomPathOrElse("python3"); + String pip = Operations.getCustomPathOrElse("pip3"); + try { + Operations.runProcess(python, args); + Operations.runProcess(pip, args); + } catch (Exception e) { + python = Operations.getCustomPathOrElse("python"); + pip = Operations.getCustomPathOrElse("pip"); + try { + Process process = new ProcessBuilder(command, args).redirectErrorStream(true).start(); + int exitCode = process.waitFor(); + if (exitCode != 0) { + throw new IOException( + "Python executable found, but it exited with error code " + exitCode); + } + } catch (IOException | InterruptedException ex) { + throw new RuntimeException( + String.format( + "Unable to find or run Python executable '%s'. Please ensure Python is installed" + + " and available in your PATH.", + command), + ex); + } + + try { + Process process = new ProcessBuilder("pip", args).redirectErrorStream(true).start(); + int exitCode = process.waitFor(); + if (exitCode != 0) { + throw new IOException("Pip executable found, but it exited with error code " + exitCode); + } + } catch (IOException | InterruptedException ex) { + throw new RuntimeException( + String.format( + "Unable to find or run Pip executable '%s'. Please ensure Pip is installed and" + + " available in your PATH.", + command), + ex); + } + } + return String.format("%s;;%s", python, pip); + } + private static String extractDepFull(String requirementLine) { return requirementLine.substring(0, requirementLine.indexOf("#")).trim(); } diff --git a/src/main/java/io/github/guacsec/trustifyda/providers/PythonProvider.java b/src/main/java/io/github/guacsec/trustifyda/providers/PythonProvider.java index 5647d6ad..aaa69924 100644 --- a/src/main/java/io/github/guacsec/trustifyda/providers/PythonProvider.java +++ b/src/main/java/io/github/guacsec/trustifyda/providers/PythonProvider.java @@ -16,54 +16,31 @@ */ package io.github.guacsec.trustifyda.providers; -import static io.github.guacsec.trustifyda.impl.ExhortApi.debugLoggingIsNeeded; - -import com.fasterxml.jackson.core.JsonProcessingException; import com.github.packageurl.MalformedPackageURLException; import com.github.packageurl.PackageURL; -import io.github.guacsec.trustifyda.Api; import io.github.guacsec.trustifyda.Provider; import io.github.guacsec.trustifyda.license.LicenseUtils; -import io.github.guacsec.trustifyda.logging.LoggersFactory; import io.github.guacsec.trustifyda.sbom.Sbom; -import io.github.guacsec.trustifyda.sbom.SbomFactory; import io.github.guacsec.trustifyda.tools.Ecosystem; -import io.github.guacsec.trustifyda.tools.Operations; import io.github.guacsec.trustifyda.utils.Environment; import io.github.guacsec.trustifyda.utils.IgnorePatternDetector; -import io.github.guacsec.trustifyda.utils.PythonControllerBase; -import io.github.guacsec.trustifyda.utils.PythonControllerRealEnv; -import io.github.guacsec.trustifyda.utils.PythonControllerVirtualEnv; -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; import java.nio.file.Path; -import java.util.List; -import java.util.Map; import java.util.Set; -import java.util.logging.Logger; import java.util.stream.Collectors; /** * Abstract base class for Python providers. Encapsulates shared Python infrastructure including - * controller resolution, executable discovery, SBOM construction, and ignore-pattern handling. + * PURL construction, ignore-pattern handling, and root component defaults. */ public abstract class PythonProvider extends Provider { - private static final Logger log = LoggersFactory.getLogger(PythonProvider.class.getName()); static final String DEFAULT_PIP_ROOT_COMPONENT_NAME = "default-pip-root"; static final String DEFAULT_PIP_ROOT_COMPONENT_VERSION = "0.0.0"; - private PythonControllerBase pythonController; - protected PythonProvider(Path manifest) { super(Ecosystem.Type.PYTHON, manifest); } - public void setPythonController(PythonControllerBase pythonController) { - this.pythonController = pythonController; - } - @Override public String readLicenseFromManifest() { return LicenseUtils.readLicenseFile(manifest); @@ -77,104 +54,10 @@ protected String getRootComponentVersion() { return DEFAULT_PIP_ROOT_COMPONENT_VERSION; } - /** - * Returns the path to a requirements-format file that the {@link PythonControllerBase} can - * consume. For requirements.txt this is the manifest itself; for pyproject.toml a temporary file - * is generated. - */ - protected abstract Path getRequirementsPath() throws IOException; - - /** Clean up any temporary files created by {@link #getRequirementsPath()}. */ - protected abstract void cleanupRequirementsPath(Path requirementsPath) throws IOException; - /** Parse ignored dependencies from the raw manifest content. */ protected abstract Set getIgnoredDependencies(String manifestContent); - @Override - public Content provideStack() throws IOException { - Path requirementsPath = getRequirementsPath(); - try { - PythonControllerBase controller = getPythonController(); - List> dependencies = - controller.getDependencies(requirementsPath.toString(), true); - printDependenciesTree(dependencies); - Sbom sbom = SbomFactory.newInstance(Sbom.BelongingCondition.PURL, "sensitive"); - sbom.addRoot( - toPurl(getRootComponentName(), getRootComponentVersion()), readLicenseFromManifest()); - for (Map component : dependencies) { - addAllDependencies(sbom.getRoot(), component, sbom); - } - String manifestContent = Files.readString(manifest); - handleIgnoredDependencies(manifestContent, sbom); - return new Content( - sbom.getAsJsonString().getBytes(StandardCharsets.UTF_8), Api.CYCLONEDX_MEDIA_TYPE); - } finally { - try { - cleanupRequirementsPath(requirementsPath); - } catch (IOException e) { - log.warning("Failed to clean up temporary requirements file: " + e.getMessage()); - } - } - } - - @Override - public Content provideComponent() throws IOException { - Path requirementsPath = getRequirementsPath(); - try { - PythonControllerBase controller = getPythonController(); - List> dependencies = - controller.getDependencies(requirementsPath.toString(), false); - printDependenciesTree(dependencies); - Sbom sbom = SbomFactory.newInstance(); - sbom.addRoot( - toPurl(getRootComponentName(), getRootComponentVersion()), readLicenseFromManifest()); - dependencies.forEach( - (component) -> - sbom.addDependency( - sbom.getRoot(), - toPurl((String) component.get("name"), (String) component.get("version")), - null)); - String manifestContent = Files.readString(manifest); - handleIgnoredDependencies(manifestContent, sbom); - return new Content( - sbom.getAsJsonString().getBytes(StandardCharsets.UTF_8), Api.CYCLONEDX_MEDIA_TYPE); - } finally { - try { - cleanupRequirementsPath(requirementsPath); - } catch (IOException e) { - log.warning("Failed to clean up temporary requirements file: " + e.getMessage()); - } - } - } - - @SuppressWarnings("unchecked") - private void addAllDependencies(PackageURL source, Map component, Sbom sbom) { - PackageURL packageURL = - toPurl((String) component.get("name"), (String) component.get("version")); - sbom.addDependency(source, packageURL, null); - - List> directDeps = - (List>) component.get("dependencies"); - if (directDeps != null) { - for (Map dep : directDeps) { - addAllDependencies(packageURL, dep, sbom); - } - } - } - - private void printDependenciesTree(List> dependencies) - throws JsonProcessingException { - if (debugLoggingIsNeeded()) { - String pythonControllerTree = - objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(dependencies); - log.info( - String.format( - "Python Generated Dependency Tree in Json Format: %s %s %s", - System.lineSeparator(), pythonControllerTree, System.lineSeparator())); - } - } - - private void handleIgnoredDependencies(String manifestContent, Sbom sbom) { + protected void handleIgnoredDependencies(String manifestContent, Sbom sbom) { Set ignoredDeps = getIgnoredDependencies(manifestContent); Set ignoredDepsVersions = ignoredDeps.stream() @@ -224,78 +107,4 @@ protected PackageURL toPurl(String name, String version) { throw new RuntimeException(e); } } - - protected PythonControllerBase getPythonController() { - String pythonPipBinaries; - boolean useVirtualPythonEnv; - if (!Environment.get(PythonControllerBase.PROP_TRUSTIFY_DA_PIP_SHOW, "").trim().isEmpty() - && !Environment.get(PythonControllerBase.PROP_TRUSTIFY_DA_PIP_FREEZE, "") - .trim() - .isEmpty()) { - pythonPipBinaries = "python;;pip"; - useVirtualPythonEnv = false; - } else { - pythonPipBinaries = getExecutable("python", "--version"); - useVirtualPythonEnv = - Environment.getBoolean(PythonControllerBase.PROP_TRUSTIFY_DA_PYTHON_VIRTUAL_ENV, false); - } - - String[] parts = pythonPipBinaries.split(";;"); - var python = parts[0]; - var pip = parts[1]; - PythonControllerBase controller; - if (this.pythonController == null) { - if (useVirtualPythonEnv) { - controller = new PythonControllerVirtualEnv(python); - } else { - controller = new PythonControllerRealEnv(python, pip); - } - } else { - controller = this.pythonController; - } - return controller; - } - - private String getExecutable(String command, String args) { - String python = Operations.getCustomPathOrElse("python3"); - String pip = Operations.getCustomPathOrElse("pip3"); - try { - Operations.runProcess(python, args); - Operations.runProcess(pip, args); - } catch (Exception e) { - python = Operations.getCustomPathOrElse("python"); - pip = Operations.getCustomPathOrElse("pip"); - try { - Process process = new ProcessBuilder(command, args).redirectErrorStream(true).start(); - int exitCode = process.waitFor(); - if (exitCode != 0) { - throw new IOException( - "Python executable found, but it exited with error code " + exitCode); - } - } catch (IOException | InterruptedException ex) { - throw new RuntimeException( - String.format( - "Unable to find or run Python executable '%s'. Please ensure Python is installed" - + " and available in your PATH.", - command), - ex); - } - - try { - Process process = new ProcessBuilder("pip", args).redirectErrorStream(true).start(); - int exitCode = process.waitFor(); - if (exitCode != 0) { - throw new IOException("Pip executable found, but it exited with error code " + exitCode); - } - } catch (IOException | InterruptedException ex) { - throw new RuntimeException( - String.format( - "Unable to find or run Pip executable '%s'. Please ensure Pip is installed and" - + " available in your PATH.", - command), - ex); - } - } - return String.format("%s;;%s", python, pip); - } } diff --git a/src/main/java/io/github/guacsec/trustifyda/providers/PythonPyprojectProvider.java b/src/main/java/io/github/guacsec/trustifyda/providers/PythonPyprojectProvider.java index c53d91ac..fd8d2cba 100644 --- a/src/main/java/io/github/guacsec/trustifyda/providers/PythonPyprojectProvider.java +++ b/src/main/java/io/github/guacsec/trustifyda/providers/PythonPyprojectProvider.java @@ -16,29 +16,59 @@ */ package io.github.guacsec.trustifyda.providers; +import com.fasterxml.jackson.databind.JsonNode; import com.github.packageurl.PackageURL; +import io.github.guacsec.trustifyda.Api; import io.github.guacsec.trustifyda.license.LicenseUtils; import io.github.guacsec.trustifyda.logging.LoggersFactory; +import io.github.guacsec.trustifyda.sbom.Sbom; +import io.github.guacsec.trustifyda.sbom.SbomFactory; +import io.github.guacsec.trustifyda.tools.Operations; +import io.github.guacsec.trustifyda.utils.Environment; import io.github.guacsec.trustifyda.utils.PythonControllerBase; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; +import java.util.Base64; +import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.logging.Logger; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.stream.Collectors; import org.tomlj.Toml; import org.tomlj.TomlArray; import org.tomlj.TomlParseResult; import org.tomlj.TomlTable; +/** + * Provider for Python projects using {@code pyproject.toml} with PEP 621 {@code [project.dependencies]}. + * + *

    Dependency resolution is performed via {@code pip install --dry-run --ignore-installed + * --report - .}, which produces a JSON installation report containing the full dependency graph. + * + *

    Poetry is not supported. If the manifest contains {@code + * [tool.poetry.dependencies]}, both {@link #provideStack()} and {@link #provideComponent()} will + * throw an {@link IllegalStateException}. + */ public final class PythonPyprojectProvider extends PythonProvider { private static final Logger log = LoggersFactory.getLogger(PythonPyprojectProvider.class.getName()); + static final String PROP_TRUSTIFY_DA_PIP_REPORT = "TRUSTIFY_DA_PIP_REPORT"; + + private static final Pattern DEP_NAME_PATTERN = + Pattern.compile("^([A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?)"); + private static final Pattern EXTRA_MARKER_PATTERN = Pattern.compile(";\\s*.*extra\\s*=="); + private Set collectedIgnoredDeps; private TomlParseResult cachedToml; @@ -47,18 +77,267 @@ public PythonPyprojectProvider(Path manifest) { } @Override - protected Path getRequirementsPath() throws IOException { - List depStrings = parseDependencyStrings(); - Path tmpDir = Files.createTempDirectory("trustify_da_pyproject_"); - Path tmpFile = Files.createFile(tmpDir.resolve("requirements.txt")); - Files.write(tmpFile, depStrings); - return tmpFile; + public Content provideStack() throws IOException { + rejectPoetryDependencies(); + collectIgnoredDeps(); + String reportJson = getPipReportOutput(manifest.toAbsolutePath().getParent()); + PipReportData data = parsePipReport(reportJson); + + Sbom sbom = SbomFactory.newInstance(Sbom.BelongingCondition.PURL, "sensitive"); + sbom.addRoot( + toPurl(getRootComponentName(), getRootComponentVersion()), readLicenseFromManifest()); + + for (String directKey : data.directDeps) { + PipPackage pkg = data.graph.get(directKey); + if (pkg != null) { + addDependencyTree(sbom.getRoot(), pkg, data.graph, sbom, new HashSet<>()); + } + } + + String manifestContent = Files.readString(manifest); + handleIgnoredDependencies(manifestContent, sbom); + return new Content( + sbom.getAsJsonString().getBytes(StandardCharsets.UTF_8), Api.CYCLONEDX_MEDIA_TYPE); } @Override - protected void cleanupRequirementsPath(Path requirementsPath) throws IOException { - Files.deleteIfExists(requirementsPath); - Files.deleteIfExists(requirementsPath.getParent()); + public Content provideComponent() throws IOException { + rejectPoetryDependencies(); + collectIgnoredDeps(); + String reportJson = getPipReportOutput(manifest.toAbsolutePath().getParent()); + PipReportData data = parsePipReport(reportJson); + + Sbom sbom = SbomFactory.newInstance(); + sbom.addRoot( + toPurl(getRootComponentName(), getRootComponentVersion()), readLicenseFromManifest()); + + for (String directKey : data.directDeps) { + PipPackage pkg = data.graph.get(directKey); + if (pkg != null) { + sbom.addDependency(sbom.getRoot(), toPurl(pkg.name, pkg.version), null); + } + } + + String manifestContent = Files.readString(manifest); + handleIgnoredDependencies(manifestContent, sbom); + return new Content( + sbom.getAsJsonString().getBytes(StandardCharsets.UTF_8), Api.CYCLONEDX_MEDIA_TYPE); + } + + private void addDependencyTree( + PackageURL source, + PipPackage pkg, + Map graph, + Sbom sbom, + Set visited) { + PackageURL packageURL = toPurl(pkg.name, pkg.version); + sbom.addDependency(source, packageURL, null); + + String key = canonicalize(pkg.name); + if (!visited.add(key)) { + return; + } + + for (String childKey : pkg.children) { + PipPackage child = graph.get(childKey); + if (child != null) { + addDependencyTree(packageURL, child, graph, sbom, visited); + } + } + } + + String getPipReportOutput(Path manifestDir) { + String envValue = Environment.get(PROP_TRUSTIFY_DA_PIP_REPORT); + if (envValue != null && !envValue.isBlank()) { + return new String(Base64.getDecoder().decode(envValue), StandardCharsets.UTF_8); + } + + String pip = findPipBinary(); + String[] cmd = {pip, "install", "--dry-run", "--ignore-installed", "--report", "-", "."}; + Operations.ProcessExecOutput result = + Operations.runProcessGetFullOutput(manifestDir, cmd, null); + if (result.getExitCode() != 0) { + throw new RuntimeException( + String.format( + "pip report command failed with exit code %d: %s", + result.getExitCode(), result.getError())); + } + return result.getOutput(); + } + + private String findPipBinary() { + String pip = Operations.getCustomPathOrElse("pip3"); + try { + Operations.runProcess(pip, "--version"); + return pip; + } catch (Exception e) { + pip = Operations.getCustomPathOrElse("pip"); + Operations.runProcess(pip, "--version"); + return pip; + } + } + + /** + * Parses the JSON document produced by {@code pip install --dry-run --ignore-installed --report}. + * That output is pip’s installation report: a single object describing the resolved + * dependency set, not a log file. + * + *

    Top-level shape (fields this code uses) + * + *

      + *
    • {@code install} — JSON array of one object per distribution pip would install. Other + * top-level keys (e.g. {@code version}, {@code pip_version}) are ignored here. + *
    + * + *

    Each element of {@code install} + * + *

      + *
    • {@code download_info} — Where the distribution comes from. The project root (the + * {@code .} passed to pip) is identified by the presence of {@code dir_info} (often {@code + * {}}) under {@code download_info}. Every other entry is treated as a resolved dependency + * package (wheels/sdists typically use {@code archive_info} instead). + *
    • {@code metadata} — Core package metadata, aligned with core metadata fields: + *
        + *
      • {@code name}, {@code version} — Distribution name and version (strings). + *
      • {@code requires_dist} — Optional array of PEP 508 requirement strings (e.g. {@code + * "requests>=2.0"}, {@code "foo; extra == \"bar\""}). Version specifiers and + * environment markers appear after the name; optional dependencies use {@code extra + * == "..."} markers, which this parser skips when building edges. + *
      + *
    + * + *

    How this method interprets the report + * + *

      + *
    • The entry whose {@code download_info} contains {@code dir_info} is the root + * package. Its {@code metadata.requires_dist} names the direct dependencies. + *
    • All other {@code install} entries contribute nodes in the dependency graph ({@code + * metadata.name} / {@code version}). + *
    • Edges are derived from each node’s {@code requires_dist}: the first token of each + * requirement is taken as the dependency name; the edge is kept only if that name resolves + * to another node in the graph (after canonicalization). + *
    + * + * @param reportJson raw UTF-8 JSON text from pip’s {@code --report} output + * @return direct dependency keys (canonicalized names) and a map of graph nodes; empty if {@code + * install} is missing or not an array + * @throws IOException if {@code reportJson} is not valid JSON + * @see pip installation + * report + */ + PipReportData parsePipReport(String reportJson) throws IOException { + JsonNode report = objectMapper.readTree(reportJson); + JsonNode installArray = report.get("install"); + if (installArray == null || !installArray.isArray()) { + return new PipReportData(List.of(), Map.of()); + } + + // Find root entry (has dir_info in download_info) and collect non-root packages + JsonNode rootEntry = null; + List nonRootPackages = new ArrayList<>(); + for (JsonNode entry : installArray) { + JsonNode downloadInfo = entry.get("download_info"); + if (rootEntry == null && downloadInfo != null && downloadInfo.has("dir_info")) { + rootEntry = entry; + } else { + nonRootPackages.add(entry); + } + } + + if (rootEntry == null && !nonRootPackages.isEmpty()) { + log.warning( + "pip report contains packages but no root entry (dir_info);" + + " dependency tree will be empty"); + } + + // Extract direct dependency names from root's requires_dist (LinkedHashSet preserves order) + Set directDepNames = new LinkedHashSet<>(); + if (rootEntry != null) { + JsonNode metadata = rootEntry.get("metadata"); + if (metadata != null) { + JsonNode requiresDist = metadata.get("requires_dist"); + if (requiresDist != null && requiresDist.isArray()) { + for (JsonNode req : requiresDist) { + String reqStr = req.asText(); + if (hasExtraMarker(reqStr)) { + continue; + } + String name = extractDepName(reqStr); + if (name != null) { + directDepNames.add(canonicalize(name)); + } + } + } + } + } + + // Build graph from non-root packages + Map graph = new HashMap<>(); + for (JsonNode pkg : nonRootPackages) { + JsonNode metadata = pkg.get("metadata"); + if (metadata == null) { + continue; + } + String name = metadata.has("name") ? metadata.get("name").asText() : null; + String version = metadata.has("version") ? metadata.get("version").asText() : null; + if (name == null) { + continue; + } + String key = canonicalize(name); + graph.put(key, new PipPackage(name, version, new ArrayList<>())); + } + + // Build children from each package's requires_dist + for (JsonNode pkg : nonRootPackages) { + JsonNode metadata = pkg.get("metadata"); + if (metadata == null) { + continue; + } + String name = metadata.has("name") ? metadata.get("name").asText() : null; + if (name == null) { + continue; + } + String key = canonicalize(name); + PipPackage pipPkg = graph.get(key); + if (pipPkg == null) { + continue; + } + JsonNode requiresDist = metadata.get("requires_dist"); + if (requiresDist == null || !requiresDist.isArray()) { + continue; + } + for (JsonNode req : requiresDist) { + String reqStr = req.asText(); + if (hasExtraMarker(reqStr)) { + continue; + } + String depName = extractDepName(reqStr); + if (depName == null) { + continue; + } + String depKey = canonicalize(depName); + if (graph.containsKey(depKey)) { + pipPkg.children.add(depKey); + } + } + } + + List directDeps = + directDepNames.stream().filter(graph::containsKey).collect(Collectors.toList()); + return new PipReportData(directDeps, graph); + } + + static boolean hasExtraMarker(String req) { + return EXTRA_MARKER_PATTERN.matcher(req).find(); + } + + static String extractDepName(String req) { + Matcher m = DEP_NAME_PATTERN.matcher(req); + return m.find() ? m.group(1) : null; + } + + static String canonicalize(String name) { + return name.toLowerCase().replaceAll("[-_.]+", "-"); } private TomlParseResult getToml() throws IOException { @@ -81,10 +360,6 @@ protected String getRootComponentName() { if (name != null && !name.isBlank()) { return name; } - String poetryName = toml.getString("tool.poetry.name"); - if (poetryName != null && !poetryName.isBlank()) { - return poetryName; - } } catch (IOException e) { log.fine("Failed to parse pyproject.toml for root component name: " + e.getMessage()); } @@ -99,10 +374,6 @@ protected String getRootComponentVersion() { if (version != null && !version.isBlank()) { return version; } - String poetryVersion = toml.getString("tool.poetry.version"); - if (poetryVersion != null && !poetryVersion.isBlank()) { - return poetryVersion; - } } catch (IOException e) { log.fine("Failed to parse pyproject.toml for root component version: " + e.getMessage()); } @@ -122,10 +393,6 @@ public String readLicenseFromManifest() { if (licenseText != null && !licenseText.isBlank()) { return licenseText; } - String poetryLicense = toml.getString("tool.poetry.license"); - if (poetryLicense != null && !poetryLicense.isBlank()) { - return poetryLicense; - } } catch (IOException e) { log.fine("Failed to parse pyproject.toml for license: " + e.getMessage()); } @@ -146,31 +413,40 @@ protected Set getIgnoredDependencies(String manifestContent) { .collect(Collectors.toSet()); } - List parseDependencyStrings() throws IOException { + private void rejectPoetryDependencies() throws IOException { TomlParseResult toml = getToml(); + TomlTable poetryDeps = toml.getTable("tool.poetry.dependencies"); + if (poetryDeps != null) { + throw new IllegalStateException( + "Poetry dependencies in pyproject.toml are not supported." + + " Please use PEP 621 [project.dependencies] format instead."); + } + } + private void collectIgnoredDeps() throws IOException { + TomlParseResult toml = getToml(); List rawLines = Files.readAllLines(manifest); collectedIgnoredDeps = new HashSet<>(); - List deps = new ArrayList<>(); // [project.dependencies] - PEP 621 TomlArray projectDeps = toml.getArray("project.dependencies"); if (projectDeps != null) { for (int i = 0; i < projectDeps.size(); i++) { String dep = projectDeps.getString(i); - deps.add(dep); checkIgnored(rawLines, dep, dep); } } + } - // [tool.poetry.dependencies] - production only - TomlTable poetryDeps = toml.getTable("tool.poetry.dependencies"); - if (poetryDeps != null) { - for (String name : poetryDeps.keySet()) { - if (!"python".equalsIgnoreCase(name)) { - deps.add(poetryDepToRequirement(name, poetryDeps, name)); - checkIgnored(rawLines, name, name); - } + List parseDependencyStrings() throws IOException { + collectIgnoredDeps(); + TomlParseResult toml = getToml(); + List deps = new ArrayList<>(); + + TomlArray projectDeps = toml.getArray("project.dependencies"); + if (projectDeps != null) { + for (int i = 0; i < projectDeps.size(); i++) { + deps.add(projectDeps.getString(i)); } } @@ -186,77 +462,25 @@ private void checkIgnored(List rawLines, String searchToken, String depI } } - /** - * Converts a Poetry dependency entry to a pip-compatible requirement string. Poetry uses {@code - * ^} and {@code ~} operators which are not PEP 440, so they must be converted to PEP 440 ranges. - */ - static String poetryDepToRequirement(String name, TomlTable table, String key) { - String version = null; - if (table.isString(key)) { - version = table.getString(key); - } else if (table.isTable(key)) { - TomlTable depTable = table.getTable(key); - if (depTable != null) { - version = depTable.getString("version"); - } - } - if (version == null || version.isEmpty() || "*".equals(version)) { - return name; - } - return name + convertPoetryVersion(version); - } + static final class PipPackage { + final String name; + final String version; + final List children; - /** - * Converts a Poetry version constraint to PEP 440 format. - * - *
      - *
    • {@code ^X.Y.Z} → {@code >=X.Y.Z,<(X+1).0.0} (when X > 0) - *
    • {@code ^0.Y.Z} → {@code >=0.Y.Z,<0.(Y+1).0} (when Y > 0) - *
    • {@code ^0.0.Z} → {@code >=0.0.Z,<0.0.(Z+1)} - *
    • {@code ~X.Y.Z} → {@code >=X.Y.Z,PEP 440 operators ({@code >=}, {@code ==}, etc.) are passed through unchanged - *
    - */ - static String convertPoetryVersion(String version) { - if (version.startsWith("^")) { - return convertCaret(version.substring(1)); + PipPackage(String name, String version, List children) { + this.name = name; + this.version = version; + this.children = children; } - if (version.startsWith("~") && !version.startsWith("~=")) { - return convertTilde(version.substring(1)); - } - if (version.matches("^\\d.*")) { - return "==" + version; - } - // Already PEP 440 compatible (>=, ==, ~=, !=, etc.) - return version; - } - - private static int parseNumericPart(String part) { - return Integer.parseInt(part.replaceAll("[^0-9].*", "")); } - private static String convertCaret(String ver) { - String[] parts = ver.split("\\."); - int major = parseNumericPart(parts[0]); - int minor = parts.length > 1 ? parseNumericPart(parts[1]) : 0; - int patch = parts.length > 2 ? parseNumericPart(parts[2]) : 0; - String fullVer = major + "." + minor + "." + patch; + static final class PipReportData { + final List directDeps; + final Map graph; - if (major > 0) { - return ">=" + fullVer + ",<" + (major + 1) + ".0.0"; + PipReportData(List directDeps, Map graph) { + this.directDeps = directDeps; + this.graph = graph; } - if (minor > 0) { - return ">=" + fullVer + ",<0." + (minor + 1) + ".0"; - } - return ">=" + fullVer + ",<0.0." + (patch + 1); - } - - private static String convertTilde(String ver) { - String[] parts = ver.split("\\."); - int major = parseNumericPart(parts[0]); - int minor = parts.length > 1 ? parseNumericPart(parts[1]) : 0; - int patch = parts.length > 2 ? parseNumericPart(parts[2]) : 0; - String fullVer = major + "." + minor + "." + patch; - return ">=" + fullVer + ",<" + major + "." + (minor + 1) + ".0"; } } diff --git a/src/test/java/io/github/guacsec/trustifyda/providers/Python_Pyproject_Provider_Test.java b/src/test/java/io/github/guacsec/trustifyda/providers/Python_Pyproject_Provider_Test.java index 54f9f1ef..51cce7b8 100644 --- a/src/test/java/io/github/guacsec/trustifyda/providers/Python_Pyproject_Provider_Test.java +++ b/src/test/java/io/github/guacsec/trustifyda/providers/Python_Pyproject_Provider_Test.java @@ -17,15 +17,16 @@ package io.github.guacsec.trustifyda.providers; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatNoException; +import static org.assertj.core.api.Assertions.assertThatIllegalStateException; import io.github.guacsec.trustifyda.Api; import io.github.guacsec.trustifyda.ExhortTest; import io.github.guacsec.trustifyda.tools.Ecosystem; -import io.github.guacsec.trustifyda.utils.PythonControllerTestEnv; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Base64; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -63,90 +64,23 @@ void test_parse_pep621_excludes_optional_dependencies() throws IOException { } @Test - void test_parse_poetry_dependencies_converts_to_pep440() throws IOException { + void test_provideStack_rejects_poetry_dependencies() { Path pyprojectPath = Path.of("src/test/resources/tst_manifests/pip/pip_pyproject_toml_poetry/pyproject.toml"); var provider = new PythonPyprojectProvider(pyprojectPath); - List deps = provider.parseDependencyStrings(); - assertThat(deps) - .contains("anyio>=3.6.2,<4.0.0", "flask>=2.0.3,<3.0.0", "requests>=2.25.1,<3.0.0"); - } - - @Test - void test_parse_poetry_excludes_python() throws IOException { - Path pyprojectPath = - Path.of("src/test/resources/tst_manifests/pip/pip_pyproject_toml_poetry/pyproject.toml"); - var provider = new PythonPyprojectProvider(pyprojectPath); - List deps = provider.parseDependencyStrings(); - assertThat(deps).doesNotContain("python"); + assertThatIllegalStateException() + .isThrownBy(provider::provideStack) + .withMessageContaining("Poetry dependencies in pyproject.toml are not supported"); } @Test - void test_parse_poetry_excludes_dev_group_dependencies() throws IOException { + void test_provideComponent_rejects_poetry_dependencies() { Path pyprojectPath = Path.of("src/test/resources/tst_manifests/pip/pip_pyproject_toml_poetry/pyproject.toml"); var provider = new PythonPyprojectProvider(pyprojectPath); - List deps = provider.parseDependencyStrings(); - assertThat(deps).doesNotContain("click", "click>=8.0.4,<9.0.0"); - } - - @Test - void test_convert_caret_major() { - assertThat(PythonPyprojectProvider.convertPoetryVersion("^3.6.2")).isEqualTo(">=3.6.2,<4.0.0"); - } - - @Test - void test_convert_caret_zero_major() { - assertThat(PythonPyprojectProvider.convertPoetryVersion("^0.5.1")).isEqualTo(">=0.5.1,<0.6.0"); - } - - @Test - void test_convert_caret_zero_zero() { - assertThat(PythonPyprojectProvider.convertPoetryVersion("^0.0.3")).isEqualTo(">=0.0.3,<0.0.4"); - } - - @Test - void test_convert_caret_two_parts() { - assertThat(PythonPyprojectProvider.convertPoetryVersion("^2.0")).isEqualTo(">=2.0.0,<3.0.0"); - } - - @Test - void test_convert_tilde() { - assertThat(PythonPyprojectProvider.convertPoetryVersion("~1.2.3")).isEqualTo(">=1.2.3,<1.3.0"); - } - - @Test - void test_convert_tilde_two_parts() { - assertThat(PythonPyprojectProvider.convertPoetryVersion("~1.2")).isEqualTo(">=1.2.0,<1.3.0"); - } - - @Test - void test_pep440_passthrough() { - assertThat(PythonPyprojectProvider.convertPoetryVersion(">=2.0")).isEqualTo(">=2.0"); - assertThat(PythonPyprojectProvider.convertPoetryVersion("==1.0.0")).isEqualTo("==1.0.0"); - assertThat(PythonPyprojectProvider.convertPoetryVersion("~=1.4")).isEqualTo("~=1.4"); - } - - @Test - void test_convert_bare_version_prepends_equals() { - assertThat(PythonPyprojectProvider.convertPoetryVersion("1.2.3")).isEqualTo("==1.2.3"); - assertThat(PythonPyprojectProvider.convertPoetryVersion("2.0")).isEqualTo("==2.0"); - } - - @Test - void test_convert_caret_prerelease_does_not_crash() { - assertThatNoException() - .isThrownBy(() -> PythonPyprojectProvider.convertPoetryVersion("^1.2.3b1")); - assertThat(PythonPyprojectProvider.convertPoetryVersion("^1.2.3b1")) - .isEqualTo(">=1.2.3,<2.0.0"); - } - - @Test - void test_convert_tilde_prerelease_does_not_crash() { - assertThatNoException() - .isThrownBy(() -> PythonPyprojectProvider.convertPoetryVersion("~1.2.3rc1")); - assertThat(PythonPyprojectProvider.convertPoetryVersion("~1.2.3rc1")) - .isEqualTo(">=1.2.3,<1.3.0"); + assertThatIllegalStateException() + .isThrownBy(provider::provideComponent) + .withMessageContaining("Poetry dependencies in pyproject.toml are not supported"); } @Test @@ -171,14 +105,6 @@ void test_getRootComponentName_reads_pep621_name() { assertThat(provider.getRootComponentName()).isEqualTo("test-project"); } - @Test - void test_getRootComponentName_reads_poetry_name() { - Path pyprojectPath = - Path.of("src/test/resources/tst_manifests/pip/pip_pyproject_toml_poetry/pyproject.toml"); - var provider = new PythonPyprojectProvider(pyprojectPath); - assertThat(provider.getRootComponentName()).isEqualTo("test-project"); - } - @Test void test_getRootComponentName_falls_back_to_default() { Path pyprojectPath = @@ -197,14 +123,6 @@ void test_getRootComponentVersion_reads_pep621_version() { assertThat(provider.getRootComponentVersion()).isEqualTo("2.0.0"); } - @Test - void test_getRootComponentVersion_reads_poetry_version() { - Path pyprojectPath = - Path.of("src/test/resources/tst_manifests/pip/pip_pyproject_toml_poetry/pyproject.toml"); - var provider = new PythonPyprojectProvider(pyprojectPath); - assertThat(provider.getRootComponentVersion()).isEqualTo("0.1.0"); - } - @Test void test_getRootComponentVersion_falls_back_to_default() { Path pyprojectPath = @@ -223,40 +141,202 @@ void test_readLicenseFromManifest_reads_pep621_license() { assertThat(provider.readLicenseFromManifest()).isEqualTo("MIT"); } + // --- pip report parsing tests --- + @Test - void test_readLicenseFromManifest_reads_poetry_license() { + void test_parsePipReport_identifies_direct_deps() throws IOException { Path pyprojectPath = + Path.of("src/test/resources/tst_manifests/pip/pip_pyproject_toml_no_ignore/pyproject.toml"); + Path reportPath = Path.of( - "src/test/resources/tst_manifests/pip/pip_pyproject_toml_poetry_license/pyproject.toml"); + "src/test/resources/tst_manifests/pip/pip_pyproject_toml_no_ignore/pip_report.json"); var provider = new PythonPyprojectProvider(pyprojectPath); - assertThat(provider.readLicenseFromManifest()).isEqualTo("Apache-2.0"); + String reportJson = Files.readString(reportPath); + var data = provider.parsePipReport(reportJson); + assertThat(data.directDeps).containsExactlyInAnyOrder("anyio", "flask", "requests"); } @Test - void test_provideComponent_generates_correct_media_type() throws IOException { + void test_parsePipReport_builds_transitive_graph() throws IOException { Path pyprojectPath = Path.of("src/test/resources/tst_manifests/pip/pip_pyproject_toml_no_ignore/pyproject.toml"); - var tmpDir = Files.createTempDirectory("trustify_da_test_"); - var tmpFile = Files.createFile(tmpDir.resolve("pyproject.toml")); - Files.copy(pyprojectPath, tmpFile, java.nio.file.StandardCopyOption.REPLACE_EXISTING); - var provider = new PythonPyprojectProvider(tmpFile); - var mockController = - new PythonControllerTestEnv( - io.github.guacsec.trustifyda.tools.Operations.getCustomPathOrElse("python3"), - io.github.guacsec.trustifyda.tools.Operations.getCustomPathOrElse("pip3")); - provider.setPythonController(mockController); + Path reportPath = + Path.of( + "src/test/resources/tst_manifests/pip/pip_pyproject_toml_no_ignore/pip_report.json"); + var provider = new PythonPyprojectProvider(pyprojectPath); + String reportJson = Files.readString(reportPath); + var data = provider.parsePipReport(reportJson); + + var requestsPkg = data.graph.get("requests"); + assertThat(requestsPkg).isNotNull(); + assertThat(requestsPkg.children) + .containsExactlyInAnyOrder("charset-normalizer", "idna", "urllib3", "certifi"); + + var anyioPkg = data.graph.get("anyio"); + assertThat(anyioPkg).isNotNull(); + assertThat(anyioPkg.children).containsExactlyInAnyOrder("idna", "sniffio"); + } + + @Test + void test_parsePipReport_filters_extras() throws IOException { + Path pyprojectPath = + Path.of("src/test/resources/tst_manifests/pip/pip_pyproject_toml_no_ignore/pyproject.toml"); + Path reportPath = + Path.of( + "src/test/resources/tst_manifests/pip/pip_pyproject_toml_no_ignore/pip_report.json"); + var provider = new PythonPyprojectProvider(pyprojectPath); + String reportJson = Files.readString(reportPath); + var data = provider.parsePipReport(reportJson); + + assertThat(data.graph.containsKey("pysocks")).isFalse(); + var requestsPkg = data.graph.get("requests"); + assertThat(requestsPkg.children).doesNotContain("pysocks"); + } + + @Test + void test_parsePipReport_excludes_root_from_graph() throws IOException { + Path pyprojectPath = + Path.of("src/test/resources/tst_manifests/pip/pip_pyproject_toml_no_ignore/pyproject.toml"); + Path reportPath = + Path.of( + "src/test/resources/tst_manifests/pip/pip_pyproject_toml_no_ignore/pip_report.json"); + var provider = new PythonPyprojectProvider(pyprojectPath); + String reportJson = Files.readString(reportPath); + var data = provider.parsePipReport(reportJson); + + assertThat(data.graph.containsKey("test-project")).isFalse(); + } + + @Test + void test_parsePipReport_name_canonicalization() throws IOException { + Path pyprojectPath = + Path.of("src/test/resources/tst_manifests/pip/pip_pyproject_toml_no_ignore/pyproject.toml"); + Path reportPath = + Path.of( + "src/test/resources/tst_manifests/pip/pip_pyproject_toml_no_ignore/pip_report.json"); + var provider = new PythonPyprojectProvider(pyprojectPath); + String reportJson = Files.readString(reportPath); + var data = provider.parsePipReport(reportJson); + + assertThat(data.graph.containsKey("charset-normalizer")).isTrue(); + assertThat(data.graph.containsKey("werkzeug")).isTrue(); + assertThat(data.graph.containsKey("jinja2")).isTrue(); + assertThat(data.graph.containsKey("markupsafe")).isTrue(); + } + + @Test + void test_hasExtraMarker() { + assertThat(PythonPyprojectProvider.hasExtraMarker("PySocks!=1.5.7,>=1.5.6; extra == \"socks\"")) + .isTrue(); + assertThat(PythonPyprojectProvider.hasExtraMarker("charset_normalizer<4,>=2")).isFalse(); + assertThat( + PythonPyprojectProvider.hasExtraMarker( + "importlib-metadata>=3.6.0; python_version < \"3.10\"")) + .isFalse(); + } + + @Test + void test_extractDepName() { + assertThat(PythonPyprojectProvider.extractDepName("charset_normalizer<4,>=2")) + .isEqualTo("charset_normalizer"); + assertThat(PythonPyprojectProvider.extractDepName("idna<4,>=2.5")).isEqualTo("idna"); + assertThat(PythonPyprojectProvider.extractDepName("PySocks!=1.5.7,>=1.5.6; extra == \"socks\"")) + .isEqualTo("PySocks"); + assertThat(PythonPyprojectProvider.extractDepName("requests>=2.32")).isEqualTo("requests"); + } + + @Test + void test_canonicalize() { + assertThat(PythonPyprojectProvider.canonicalize("charset_normalizer")) + .isEqualTo("charset-normalizer"); + assertThat(PythonPyprojectProvider.canonicalize("Jinja2")).isEqualTo("jinja2"); + assertThat(PythonPyprojectProvider.canonicalize("MarkupSafe")).isEqualTo("markupsafe"); + assertThat(PythonPyprojectProvider.canonicalize("my.package_name")) + .isEqualTo("my-package-name"); + } + + @Test + void test_provideStack_with_pip_report() throws IOException { + Path pyprojectPath = + Path.of("src/test/resources/tst_manifests/pip/pip_pyproject_toml_no_ignore/pyproject.toml"); + Path reportPath = + Path.of( + "src/test/resources/tst_manifests/pip/pip_pyproject_toml_no_ignore/pip_report.json"); + String reportJson = Files.readString(reportPath); + String encodedReport = + Base64.getEncoder().encodeToString(reportJson.getBytes(StandardCharsets.UTF_8)); + + System.setProperty(PythonPyprojectProvider.PROP_TRUSTIFY_DA_PIP_REPORT, encodedReport); try { - var content = provider.provideComponent(); + var provider = new PythonPyprojectProvider(pyprojectPath); + var content = provider.provideStack(); assertThat(content.type).isEqualTo(Api.CYCLONEDX_MEDIA_TYPE); String sbomJson = new String(content.buffer); assertThat(sbomJson).contains("CycloneDX"); assertThat(sbomJson).contains("pkg:pypi/"); - } catch (RuntimeException e) { - Assumptions.assumeTrue( - false, "Skipping: Python/pip environment not usable - " + e.getMessage()); + assertThat(sbomJson).contains("pkg:pypi/anyio@3.6.2"); + assertThat(sbomJson).contains("pkg:pypi/flask@2.0.3"); + assertThat(sbomJson).contains("pkg:pypi/requests@2.25.1"); + assertThat(sbomJson).contains("pkg:pypi/idna@3.4"); + assertThat(sbomJson).contains("pkg:pypi/sniffio@1.3.0"); + assertThat(sbomJson).contains("pkg:pypi/certifi@2023.5.7"); + } catch (RuntimeException | NoClassDefFoundError e) { + Assumptions.assumeTrue(false, "Skipping: SBOM serialization unavailable - " + e.getMessage()); + } finally { + System.clearProperty(PythonPyprojectProvider.PROP_TRUSTIFY_DA_PIP_REPORT); + } + } + + @Test + void test_provideComponent_with_pip_report() throws IOException { + Path pyprojectPath = + Path.of("src/test/resources/tst_manifests/pip/pip_pyproject_toml_no_ignore/pyproject.toml"); + Path reportPath = + Path.of( + "src/test/resources/tst_manifests/pip/pip_pyproject_toml_no_ignore/pip_report.json"); + String reportJson = Files.readString(reportPath); + String encodedReport = + Base64.getEncoder().encodeToString(reportJson.getBytes(StandardCharsets.UTF_8)); + + System.setProperty(PythonPyprojectProvider.PROP_TRUSTIFY_DA_PIP_REPORT, encodedReport); + try { + var provider = new PythonPyprojectProvider(pyprojectPath); + var content = provider.provideComponent(); + assertThat(content.type).isEqualTo(Api.CYCLONEDX_MEDIA_TYPE); + String sbomJson = new String(content.buffer); + assertThat(sbomJson).contains("CycloneDX"); + assertThat(sbomJson).contains("pkg:pypi/anyio@3.6.2"); + assertThat(sbomJson).contains("pkg:pypi/flask@2.0.3"); + assertThat(sbomJson).contains("pkg:pypi/requests@2.25.1"); + } catch (RuntimeException | NoClassDefFoundError e) { + Assumptions.assumeTrue(false, "Skipping: SBOM serialization unavailable - " + e.getMessage()); + } finally { + System.clearProperty(PythonPyprojectProvider.PROP_TRUSTIFY_DA_PIP_REPORT); + } + } + + @Test + void test_provideStack_with_exhortignore() throws IOException { + Path pyprojectPath = + Path.of("src/test/resources/tst_manifests/pip/pip_pyproject_toml_ignore/pyproject.toml"); + Path reportPath = + Path.of("src/test/resources/tst_manifests/pip/pip_pyproject_toml_ignore/pip_report.json"); + String reportJson = Files.readString(reportPath); + String encodedReport = + Base64.getEncoder().encodeToString(reportJson.getBytes(StandardCharsets.UTF_8)); + + System.setProperty(PythonPyprojectProvider.PROP_TRUSTIFY_DA_PIP_REPORT, encodedReport); + try { + var provider = new PythonPyprojectProvider(pyprojectPath); + var content = provider.provideStack(); + String sbomJson = new String(content.buffer); + assertThat(sbomJson).doesNotContain("pkg:pypi/flask@"); + assertThat(sbomJson).contains("pkg:pypi/anyio@3.6.2"); + assertThat(sbomJson).contains("pkg:pypi/requests@2.25.1"); + } catch (RuntimeException | NoClassDefFoundError e) { + Assumptions.assumeTrue(false, "Skipping: SBOM serialization unavailable - " + e.getMessage()); } finally { - Files.deleteIfExists(tmpFile); - Files.deleteIfExists(tmpDir); + System.clearProperty(PythonPyprojectProvider.PROP_TRUSTIFY_DA_PIP_REPORT); } } } diff --git a/src/test/resources/tst_manifests/pip/pip_pyproject_toml_ignore/pip_report.json b/src/test/resources/tst_manifests/pip/pip_pyproject_toml_ignore/pip_report.json new file mode 100644 index 00000000..a6b3c609 --- /dev/null +++ b/src/test/resources/tst_manifests/pip/pip_pyproject_toml_ignore/pip_report.json @@ -0,0 +1,144 @@ +{ + "version": "1", + "pip_version": "24.0", + "install": [ + { + "download_info": {"url": "file:///project", "dir_info": {}}, + "requested": true, + "metadata": { + "name": "test-project", + "version": "0.1.0", + "requires_dist": [ + "anyio==3.6.2", + "flask==2.0.3", + "requests==2.25.1" + ] + } + }, + { + "download_info": {"url": "https://pypi.org/anyio-3.6.2.whl", "archive_info": {}}, + "requested": false, + "metadata": { + "name": "anyio", + "version": "3.6.2", + "requires_dist": [ + "idna>=2.8", + "sniffio>=1.1" + ] + } + }, + { + "download_info": {"url": "https://pypi.org/flask-2.0.3.whl", "archive_info": {}}, + "requested": false, + "metadata": { + "name": "flask", + "version": "2.0.3", + "requires_dist": [ + "Werkzeug>=2.0", + "Jinja2>=3.0", + "itsdangerous>=2.0", + "click>=7.1.2", + "importlib-metadata>=3.6.0; python_version < \"3.10\"" + ] + } + }, + { + "download_info": {"url": "https://pypi.org/requests-2.25.1.whl", "archive_info": {}}, + "requested": false, + "metadata": { + "name": "requests", + "version": "2.25.1", + "requires_dist": [ + "charset_normalizer<4,>=2", + "idna<4,>=2.5", + "urllib3<3,>=1.21.1", + "certifi>=2017.4.17", + "PySocks!=1.5.7,>=1.5.6; extra == \"socks\"" + ] + } + }, + { + "download_info": {"url": "https://pypi.org/idna-3.4.whl", "archive_info": {}}, + "requested": false, + "metadata": { + "name": "idna", + "version": "3.4" + } + }, + { + "download_info": {"url": "https://pypi.org/sniffio-1.3.0.whl", "archive_info": {}}, + "requested": false, + "metadata": { + "name": "sniffio", + "version": "1.3.0" + } + }, + { + "download_info": {"url": "https://pypi.org/Werkzeug-2.2.3.whl", "archive_info": {}}, + "requested": false, + "metadata": { + "name": "Werkzeug", + "version": "2.2.3" + } + }, + { + "download_info": {"url": "https://pypi.org/Jinja2-3.1.2.whl", "archive_info": {}}, + "requested": false, + "metadata": { + "name": "Jinja2", + "version": "3.1.2", + "requires_dist": [ + "MarkupSafe>=2.0" + ] + } + }, + { + "download_info": {"url": "https://pypi.org/itsdangerous-2.1.2.whl", "archive_info": {}}, + "requested": false, + "metadata": { + "name": "itsdangerous", + "version": "2.1.2" + } + }, + { + "download_info": {"url": "https://pypi.org/click-8.1.3.whl", "archive_info": {}}, + "requested": false, + "metadata": { + "name": "click", + "version": "8.1.3" + } + }, + { + "download_info": {"url": "https://pypi.org/charset_normalizer-3.1.0.whl", "archive_info": {}}, + "requested": false, + "metadata": { + "name": "charset-normalizer", + "version": "3.1.0" + } + }, + { + "download_info": {"url": "https://pypi.org/urllib3-1.26.15.whl", "archive_info": {}}, + "requested": false, + "metadata": { + "name": "urllib3", + "version": "1.26.15" + } + }, + { + "download_info": {"url": "https://pypi.org/certifi-2023.5.7.whl", "archive_info": {}}, + "requested": false, + "metadata": { + "name": "certifi", + "version": "2023.5.7" + } + }, + { + "download_info": {"url": "https://pypi.org/MarkupSafe-2.1.2.whl", "archive_info": {}}, + "requested": false, + "metadata": { + "name": "MarkupSafe", + "version": "2.1.2" + } + } + ] +} diff --git a/src/test/resources/tst_manifests/pip/pip_pyproject_toml_no_ignore/pip_report.json b/src/test/resources/tst_manifests/pip/pip_pyproject_toml_no_ignore/pip_report.json new file mode 100644 index 00000000..a6b3c609 --- /dev/null +++ b/src/test/resources/tst_manifests/pip/pip_pyproject_toml_no_ignore/pip_report.json @@ -0,0 +1,144 @@ +{ + "version": "1", + "pip_version": "24.0", + "install": [ + { + "download_info": {"url": "file:///project", "dir_info": {}}, + "requested": true, + "metadata": { + "name": "test-project", + "version": "0.1.0", + "requires_dist": [ + "anyio==3.6.2", + "flask==2.0.3", + "requests==2.25.1" + ] + } + }, + { + "download_info": {"url": "https://pypi.org/anyio-3.6.2.whl", "archive_info": {}}, + "requested": false, + "metadata": { + "name": "anyio", + "version": "3.6.2", + "requires_dist": [ + "idna>=2.8", + "sniffio>=1.1" + ] + } + }, + { + "download_info": {"url": "https://pypi.org/flask-2.0.3.whl", "archive_info": {}}, + "requested": false, + "metadata": { + "name": "flask", + "version": "2.0.3", + "requires_dist": [ + "Werkzeug>=2.0", + "Jinja2>=3.0", + "itsdangerous>=2.0", + "click>=7.1.2", + "importlib-metadata>=3.6.0; python_version < \"3.10\"" + ] + } + }, + { + "download_info": {"url": "https://pypi.org/requests-2.25.1.whl", "archive_info": {}}, + "requested": false, + "metadata": { + "name": "requests", + "version": "2.25.1", + "requires_dist": [ + "charset_normalizer<4,>=2", + "idna<4,>=2.5", + "urllib3<3,>=1.21.1", + "certifi>=2017.4.17", + "PySocks!=1.5.7,>=1.5.6; extra == \"socks\"" + ] + } + }, + { + "download_info": {"url": "https://pypi.org/idna-3.4.whl", "archive_info": {}}, + "requested": false, + "metadata": { + "name": "idna", + "version": "3.4" + } + }, + { + "download_info": {"url": "https://pypi.org/sniffio-1.3.0.whl", "archive_info": {}}, + "requested": false, + "metadata": { + "name": "sniffio", + "version": "1.3.0" + } + }, + { + "download_info": {"url": "https://pypi.org/Werkzeug-2.2.3.whl", "archive_info": {}}, + "requested": false, + "metadata": { + "name": "Werkzeug", + "version": "2.2.3" + } + }, + { + "download_info": {"url": "https://pypi.org/Jinja2-3.1.2.whl", "archive_info": {}}, + "requested": false, + "metadata": { + "name": "Jinja2", + "version": "3.1.2", + "requires_dist": [ + "MarkupSafe>=2.0" + ] + } + }, + { + "download_info": {"url": "https://pypi.org/itsdangerous-2.1.2.whl", "archive_info": {}}, + "requested": false, + "metadata": { + "name": "itsdangerous", + "version": "2.1.2" + } + }, + { + "download_info": {"url": "https://pypi.org/click-8.1.3.whl", "archive_info": {}}, + "requested": false, + "metadata": { + "name": "click", + "version": "8.1.3" + } + }, + { + "download_info": {"url": "https://pypi.org/charset_normalizer-3.1.0.whl", "archive_info": {}}, + "requested": false, + "metadata": { + "name": "charset-normalizer", + "version": "3.1.0" + } + }, + { + "download_info": {"url": "https://pypi.org/urllib3-1.26.15.whl", "archive_info": {}}, + "requested": false, + "metadata": { + "name": "urllib3", + "version": "1.26.15" + } + }, + { + "download_info": {"url": "https://pypi.org/certifi-2023.5.7.whl", "archive_info": {}}, + "requested": false, + "metadata": { + "name": "certifi", + "version": "2023.5.7" + } + }, + { + "download_info": {"url": "https://pypi.org/MarkupSafe-2.1.2.whl", "archive_info": {}}, + "requested": false, + "metadata": { + "name": "MarkupSafe", + "version": "2.1.2" + } + } + ] +}