diff --git a/README.md b/README.md index 4457b361..de7ddb9f 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,11 @@ let stackAnalysis = await client.stackAnalysis('/path/to/pom.xml') let stackAnalysisHtml = await client.stackAnalysis('/path/to/pom.xml', true) // Get component analysis in JSON format let componentAnalysis = await client.componentAnalysis('/path/to/pom.xml') +// For monorepos, pass workspace root so the client finds the lock file +let monorepoOpts = { workspaceDir: '/path/to/workspace-root' } +let stackAnalysisMonorepo = await client.stackAnalysis('/path/to/package.json', false, monorepoOpts) +// Batch analysis for entire workspace (Cargo or JS/TS); optional parallel SBOM generation +let batchReport = await client.stackAnalysisBatch('/path/to/workspace-root', false, { batchConcurrency: 10 }) // Get image analysis in JSON format let imageAnalysis = await client.imageAnalysis(['docker.io/library/node:18']) // Get image analysis in HTML format (string) @@ -101,8 +106,9 @@ $ npx @trustify-da/trustify-da-javascript-client help Usage: trustify-da-javascript-client {component|stack|image|validate-token|license} Commands: - trustify-da-javascript-client stack [--html|--summary] produce stack report for manifest path - trustify-da-javascript-client component [--summary] produce component report for a manifest type and content + trustify-da-javascript-client stack [--workspace-dir ] [--html|--summary] produce stack report for manifest path + trustify-da-javascript-client stack-batch [--html|--summary] produce stack report for all packages/crates in workspace + trustify-da-javascript-client component [--workspace-dir ] produce component report for a manifest type and content trustify-da-javascript-client image [--html|--summary] produce image analysis report for OCI image references trustify-da-javascript-client license display project license information from manifest and LICENSE file in JSON format @@ -121,9 +127,21 @@ $ npx @trustify-da/trustify-da-javascript-client stack /path/to/pom.xml --summar # get stack analysis in html format format $ npx @trustify-da/trustify-da-javascript-client stack /path/to/pom.xml --html +# get stack analysis for monorepo (lock file at workspace root) +$ npx @trustify-da/trustify-da-javascript-client stack /path/to/package.json --workspace-dir /path/to/workspace-root + # get component analysis $ npx @trustify-da/trustify-da-javascript-client component /path/to/pom.xml +# get component analysis for monorepo +$ npx @trustify-da/trustify-da-javascript-client component /path/to/package.json -w /path/to/workspace-root + +# batch stack analysis for entire workspace (Cargo or JS/TS) +$ npx @trustify-da/trustify-da-javascript-client stack-batch /path/to/workspace-root + +# optional: extra discovery excludes (merged with defaults); repeat --ignore or use TRUSTIFY_DA_WORKSPACE_DISCOVERY_IGNORE +$ npx @trustify-da/trustify-da-javascript-client stack-batch /path/to/workspace-root --ignore '**/fixtures/**' + # get image analysis in json format $ npx @trustify-da/trustify-da-javascript-client image docker.io/library/node:18 @@ -162,9 +180,21 @@ $ trustify-da-javascript-client stack /path/to/pom.xml --summary # get stack analysis in html format format $ trustify-da-javascript-client stack /path/to/pom.xml --html +# get stack analysis for monorepo (lock file at workspace root) +$ trustify-da-javascript-client stack /path/to/package.json --workspace-dir /path/to/workspace-root + # get component analysis $ trustify-da-javascript-client component /path/to/pom.xml +# get component analysis for monorepo +$ trustify-da-javascript-client component /path/to/package.json -w /path/to/workspace-root + +# batch stack analysis for entire workspace +$ trustify-da-javascript-client stack-batch /path/to/workspace-root + +# with extra discovery excludes +$ trustify-da-javascript-client stack-batch /path/to/workspace-root -i '**/vendor/**' + # get image analysis in json format $ trustify-da-javascript-client image docker.io/library/node:18 @@ -382,6 +412,8 @@ let options = { 'TRUSTIFY_DA_PIP_PATH' : '/path/to/pip', 'TRUSTIFY_DA_GRADLE_PATH' : '/path/to/gradle', 'TRUSTIFY_DA_CARGO_PATH' : '/path/to/cargo', + // Workspace root for monorepos (Cargo, npm/pnpm/yarn); lock file expected here + 'workspaceDir': '/path/to/workspace-root', // Configure proxy for all requests 'TRUSTIFY_DA_PROXY_URL': 'http://proxy.example.com:8080' } @@ -404,6 +436,21 @@ let imageAnalysisWithArch = await client.imageAnalysis(['httpd:2.4.49^^amd64'], **_Environment variables takes precedence._**

+

Monorepo / Workspace Support

+

+For monorepos (Cargo workspaces, npm/pnpm/yarn workspaces) where the lock file lives at the workspace root rather than next to the manifest, pass the workspace root via workspaceDir or TRUSTIFY_DA_WORKSPACE_DIR: +

+ +

+Use stackAnalysisBatch(workspaceRoot, html, opts) to analyze all packages/crates in a workspace in one request. Supports Cargo workspaces and JS/TS workspaces (pnpm, npm, yarn). Optional batchConcurrency (or TRUSTIFY_DA_BATCH_CONCURRENCY) limits parallel SBOM generation (default 10). For JS/TS, each package.json must have non-empty name and version; invalid manifests are skipped (warnings). Per-manifest SBOM failures are skipped if at least one SBOM succeeds (unless continueOnError: false). Set batchMetadata: true (or TRUSTIFY_DA_BATCH_METADATA) to receive { analysis, metadata } with errors[]. CLI: stack-batch --metadata, --fail-fast. See monorepo implementation plan §2.3 and §3.5. +

+

+See VS Code Extension Integration Requirements for integration details. +

+

Proxy Configuration

You can configure a proxy for all HTTP/HTTPS requests made by the API. This is useful when your environment requires going through a proxy to access external services. @@ -508,6 +555,11 @@ following keys for setting custom paths for the said executables. cargo TRUSTIFY_DA_CARGO_PATH + +Workspace root (monorepos) +— +workspaceDir / TRUSTIFY_DA_WORKSPACE_DIR + #### Match Manifest Versions Feature diff --git a/package-lock.json b/package-lock.json index 9b3cef0d..05d039ed 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,11 +12,15 @@ "@babel/core": "^7.23.2", "@cyclonedx/cyclonedx-library": "^6.13.0", "eslint-import-resolver-typescript": "^4.4.4", + "fast-glob": "^3.3.3", "fast-toml": "^0.5.4", "fast-xml-parser": "^5.3.4", "help": "^3.0.2", "https-proxy-agent": "^7.0.6", + "js-yaml": "^4.1.1", + "micromatch": "^4.0.8", "node-fetch": "^3.3.2", + "p-limit": "^5.0.0", "packageurl-js": "~1.0.2", "smol-toml": "^1.6.0", "tree-sitter-requirements": "github:Strum355/tree-sitter-requirements#d0261ee76b84253997fe70d7d397e78c006c3801", @@ -772,7 +776,6 @@ "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", - "dev": true, "license": "MIT", "dependencies": { "@nodelib/fs.stat": "2.0.5", @@ -786,7 +789,6 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", - "dev": true, "license": "MIT", "engines": { "node": ">= 8" @@ -796,7 +798,6 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", - "dev": true, "license": "MIT", "dependencies": { "@nodelib/fs.scandir": "2.1.5", @@ -1498,7 +1499,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", - "dev": true, "license": "Python-2.0" }, "node_modules/array-buffer-byte-length": { @@ -1754,7 +1754,6 @@ "version": "3.0.3", "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "dev": true, "license": "MIT", "dependencies": { "fill-range": "^7.1.1" @@ -3323,6 +3322,34 @@ "devOptional": true, "license": "MIT" }, + "node_modules/fast-glob": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", + "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.8" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/fast-glob/node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", @@ -3382,7 +3409,6 @@ "version": "1.20.1", "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==", - "dev": true, "license": "ISC", "dependencies": { "reusify": "^1.0.4" @@ -3435,7 +3461,6 @@ "version": "7.1.1", "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "dev": true, "license": "MIT", "dependencies": { "to-regex-range": "^5.0.1" @@ -4341,7 +4366,6 @@ "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -4397,7 +4421,6 @@ "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", - "dev": true, "license": "MIT", "dependencies": { "is-extglob": "^2.1.1" @@ -4450,7 +4473,6 @@ "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, "license": "MIT", "engines": { "node": ">=0.12.0" @@ -4734,7 +4756,6 @@ "version": "4.1.1", "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz", "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==", - "dev": true, "license": "MIT", "dependencies": { "argparse": "^2.0.1" @@ -4971,6 +4992,28 @@ "node": ">= 0.4" } }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, "node_modules/mimic-response": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", @@ -5941,16 +5984,15 @@ } }, "node_modules/p-limit": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", - "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", - "dev": true, + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-5.0.0.tgz", + "integrity": "sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ==", "license": "MIT", "dependencies": { - "yocto-queue": "^0.1.0" + "yocto-queue": "^1.0.0" }, "engines": { - "node": ">=10" + "node": ">=18" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" @@ -5972,6 +6014,35 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/p-locate/node_modules/p-limit": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-3.1.0.tgz", + "integrity": "sha512-TYOanM3wGwNGsZN2cVTYPArw454xnXj5qmWF1bEoAc4+cU/ol7GVh7odevjp1FNHduHc3KZMcFduxU5Xc6uJRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "yocto-queue": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-locate/node_modules/yocto-queue": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", + "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-map": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/p-map/-/p-map-4.0.0.tgz", @@ -6103,7 +6174,6 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", - "dev": true, "license": "MIT", "engines": { "node": ">=8.6" @@ -6208,7 +6278,6 @@ "version": "1.2.3", "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", - "dev": true, "funding": [ { "type": "github", @@ -6446,7 +6515,6 @@ "version": "1.1.0", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", - "dev": true, "license": "MIT", "engines": { "iojs": ">=1.0.0", @@ -6496,7 +6564,6 @@ "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", - "dev": true, "funding": [ { "type": "github", @@ -7495,7 +7562,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, "license": "MIT", "dependencies": { "is-number": "^7.0.0" @@ -8210,13 +8276,12 @@ } }, "node_modules/yocto-queue": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-0.1.0.tgz", - "integrity": "sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q==", - "dev": true, + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.2.2.tgz", + "integrity": "sha512-4LCcse/U2MHZ63HAJVE+v71o7yOdIe4cZ70Wpf8D/IyjDKYQLV5GD46B+hSTjJsvV5PztjvHoU580EftxjDZFQ==", "license": "MIT", "engines": { - "node": ">=10" + "node": ">=12.20" }, "funding": { "url": "https://github.com/sponsors/sindresorhus" diff --git a/package.json b/package.json index 31d072fd..e21a9540 100644 --- a/package.json +++ b/package.json @@ -50,11 +50,15 @@ "@babel/core": "^7.23.2", "@cyclonedx/cyclonedx-library": "^6.13.0", "eslint-import-resolver-typescript": "^4.4.4", + "fast-glob": "^3.3.3", "fast-toml": "^0.5.4", "fast-xml-parser": "^5.3.4", "help": "^3.0.2", "https-proxy-agent": "^7.0.6", + "js-yaml": "^4.1.1", + "micromatch": "^4.0.8", "node-fetch": "^3.3.2", + "p-limit": "^5.0.0", "packageurl-js": "~1.0.2", "smol-toml": "^1.6.0", "tree-sitter-requirements": "github:Strum355/tree-sitter-requirements#d0261ee76b84253997fe70d7d397e78c006c3801", diff --git a/src/analysis.js b/src/analysis.js index 2d56fad5..f25274dd 100644 --- a/src/analysis.js +++ b/src/analysis.js @@ -6,7 +6,7 @@ import { runLicenseCheck } from "./license/index.js"; import { generateImageSBOM, parseImageRef } from "./oci_image/utils.js"; import { addProxyAgent, getCustom, getTokenHeaders , TRUSTIFY_DA_OPERATION_TYPE_HEADER, TRUSTIFY_DA_PACKAGE_MANAGER_HEADER } from "./tools.js"; -export default { requestComponent, requestStack, requestImages, validateToken } +export default { requestComponent, requestStack, requestStackBatch, requestImages, validateToken } /** * Send a stack analysis request and get the report as 'text/html' or 'application/json'. @@ -137,6 +137,54 @@ async function requestComponent(provider, manifest, url, opts = {}) { return Promise.resolve(result) } +/** + * Send a batch stack analysis request for multiple manifests (SBOMs keyed by purl). + * @param {Object.} sbomByPurl - Map of root purl to CycloneDX SBOM object + * @param {string} url - the backend url + * @param {boolean} [html=false] - true returns HTML, false returns JSON + * @param {import("index.js").Options} [opts={}] + * @returns {Promise>} + */ +async function requestStackBatch(sbomByPurl, url, html = false, opts = {}) { + const finalUrl = new URL(`${url}/api/v5/batch-analysis`) + if (opts['TRUSTIFY_DA_RECOMMENDATIONS_ENABLED'] === 'false') { + finalUrl.searchParams.append('recommend', 'false') + } + + const fetchOptions = addProxyAgent({ + method: 'POST', + headers: { + 'Accept': html ? 'text/html' : 'application/json', + 'Content-Type': 'application/json', + ...getTokenHeaders(opts) + }, + body: JSON.stringify(sbomByPurl) + }, opts) + + const resp = await fetch(finalUrl, fetchOptions) + + if (resp.status === 200) { + let result + if (!html) { + result = await resp.json() + } else { + result = await resp.text() + } + if (process.env["TRUSTIFY_DA_DEBUG"] === "true") { + const exRequestId = resp.headers.get("ex-request-id") + if (exRequestId) { + console.log("Unique Identifier associated with this request - ex-request-id=" + exRequestId) + } + console.log("Response body received from Trustify DA backend server : " + EOL + EOL) + console.log(JSON.stringify(result, null, 4)) + console.log("Ending time of sending batch stack analysis request to Trustify DA backend server= " + new Date()) + } + return result + } else { + throw new Error(`Got error response from Trustify DA backend - http return code : ${resp.status}, ex-request-id: ${resp.headers.get("ex-request-id")} error message => ${await resp.text()}`) + } +} + /** * * @param {Array} imageRefs diff --git a/src/batch_opts.js b/src/batch_opts.js new file mode 100644 index 00000000..9c18d841 --- /dev/null +++ b/src/batch_opts.js @@ -0,0 +1,37 @@ +import { getCustom } from './tools.js' + +/** + * Whether to skip failed manifests and continue (default), or fail on first SBOM/validation error. + * `opts.continueOnError` overrides; env `TRUSTIFY_DA_CONTINUE_ON_ERROR=false` disables continuation. + * + * @param {{ continueOnError?: boolean, TRUSTIFY_DA_CONTINUE_ON_ERROR?: string, [key: string]: unknown }} [opts={}] + * @returns {boolean} true = collect errors (default), false = fail-fast + */ +export function resolveContinueOnError(opts = {}) { + if (typeof opts.continueOnError === 'boolean') { + return opts.continueOnError + } + const v = getCustom('TRUSTIFY_DA_CONTINUE_ON_ERROR', null, opts) + if (v != null && String(v).trim() !== '') { + return String(v).toLowerCase() !== 'false' + } + return true +} + +/** + * When true, `stackAnalysisBatch` returns `{ analysis, metadata }` instead of the backend response only. + * `opts.batchMetadata` overrides; env `TRUSTIFY_DA_BATCH_METADATA=true` enables. + * + * @param {{ batchMetadata?: boolean, TRUSTIFY_DA_BATCH_METADATA?: string, [key: string]: unknown }} [opts={}] + * @returns {boolean} + */ +export function resolveBatchMetadata(opts = {}) { + if (typeof opts.batchMetadata === 'boolean') { + return opts.batchMetadata + } + const v = getCustom('TRUSTIFY_DA_BATCH_METADATA', null, opts) + if (v != null && String(v).trim() !== '') { + return String(v).toLowerCase() === 'true' + } + return false +} diff --git a/src/cli.js b/src/cli.js index 3e289c6e..1550c4aa 100644 --- a/src/cli.js +++ b/src/cli.js @@ -21,10 +21,18 @@ const component = { type: 'string', normalize: true, } - ), + ).options({ + workspaceDir: { + alias: 'w', + desc: 'Workspace root directory (for monorepos; lock file is expected here)', + type: 'string', + normalize: true, + } + }), handler: async args => { let manifestName = args['/path/to/manifest'] - let res = await client.componentAnalysis(manifestName) + const opts = args.workspaceDir ? { TRUSTIFY_DA_WORKSPACE_DIR: args.workspaceDir } : {} + let res = await client.componentAnalysis(manifestName, opts) console.log(JSON.stringify(res, null, 2)) } } @@ -137,15 +145,22 @@ const stack = { desc: 'For JSON report, get only the \'summary\'', type: 'boolean', conflicts: 'html' + }, + workspaceDir: { + alias: 'w', + desc: 'Workspace root directory (for monorepos; lock file is expected here)', + type: 'string', + normalize: true, } }), handler: async args => { let manifest = args['/path/to/manifest'] let html = args['html'] let summary = args['summary'] + const opts = args.workspaceDir ? { TRUSTIFY_DA_WORKSPACE_DIR: args.workspaceDir } : {} let theProvidersSummary = new Map(); let theProvidersObject ={} - let res = await client.stackAnalysis(manifest, html) + let res = await client.stackAnalysis(manifest, html, opts) if(summary) { for (let provider in res.providers ) { @@ -169,6 +184,111 @@ const stack = { } } +// command for batch stack analysis (workspace) +const stackBatch = { + command: 'stack-batch [--html|--summary] [--concurrency ] [--ignore ...] [--metadata] [--fail-fast]', + desc: 'produce stack report for all packages/crates in a workspace (Cargo or JS/TS)', + builder: yargs => yargs.positional( + '/path/to/workspace-root', + { + desc: 'workspace root directory (containing Cargo.toml+Cargo.lock or package.json+lock file)', + type: 'string', + normalize: true, + } + ).options({ + html: { + alias: 'r', + desc: 'Get the report as HTML instead of JSON', + type: 'boolean', + conflicts: 'summary' + }, + summary: { + alias: 's', + desc: 'For JSON report, get only the \'summary\' per package', + type: 'boolean', + conflicts: 'html' + }, + concurrency: { + alias: 'c', + desc: 'Max parallel SBOM generations (default: 10, env: TRUSTIFY_DA_BATCH_CONCURRENCY)', + type: 'number', + }, + ignore: { + alias: 'i', + desc: 'Extra glob patterns excluded from workspace discovery (merged with defaults). Repeat flag per pattern. Env: TRUSTIFY_DA_WORKSPACE_DISCOVERY_IGNORE (comma-separated)', + type: 'string', + array: true, + }, + metadata: { + alias: 'm', + desc: 'Return { analysis, metadata } with per-manifest errors (env: TRUSTIFY_DA_BATCH_METADATA=true)', + type: 'boolean', + default: false, + }, + failFast: { + desc: 'Stop on first invalid package.json or SBOM error (env: TRUSTIFY_DA_CONTINUE_ON_ERROR=false)', + type: 'boolean', + default: false, + } + }), + handler: async args => { + const workspaceRoot = args['/path/to/workspace-root'] + const html = args['html'] + const summary = args['summary'] + const opts = {} + if (args.concurrency != null) { + opts.batchConcurrency = args.concurrency + } + const extraIgnores = Array.isArray(args.ignore) ? args.ignore.filter(p => p != null && String(p).trim()) : [] + if (extraIgnores.length > 0) { + opts.workspaceDiscoveryIgnore = extraIgnores + } + if (args.metadata) { + opts.batchMetadata = true + } + if (args.failFast) { + opts.continueOnError = false + } + let res = await client.stackAnalysisBatch(workspaceRoot, html, opts) + const batchAnalysis = + res && typeof res === 'object' && res != null && 'analysis' in res ? res.analysis : res + if (summary && !html && typeof batchAnalysis === 'object') { + const summaries = {} + for (const [purl, report] of Object.entries(batchAnalysis)) { + if (report?.providers) { + for (const provider of Object.keys(report.providers)) { + const sources = report.providers[provider]?.sources + if (sources) { + for (const [source, data] of Object.entries(sources)) { + if (data?.summary) { + if (!summaries[purl]) { + summaries[purl] = {} + } + if (!summaries[purl][provider]) { + summaries[purl][provider] = {} + } + summaries[purl][provider][source] = data.summary + } + } + } + } + } + } + if (res && typeof res === 'object' && res != null && 'metadata' in res) { + res = { analysis: summaries, metadata: res.metadata } + } else { + res = summaries + } + } + if (html) { + const htmlContent = res && typeof res === 'object' && 'analysis' in res ? res.analysis : res + console.log(htmlContent) + } else { + console.log(JSON.stringify(res, null, 2)) + } + } +} + // command for license checking const license = { command: 'license ', @@ -243,8 +363,9 @@ const license = { // parse and invoke the command yargs(hideBin(process.argv)) - .usage(`Usage: ${process.argv[0].includes("node") ? path.parse(process.argv[1]).base : path.parse(process.argv[0]).base} {component|stack|image|validate-token|license}`) + .usage(`Usage: ${process.argv[0].includes("node") ? path.parse(process.argv[1]).base : path.parse(process.argv[0]).base} {component|stack|stack-batch|image|validate-token|license}`) .command(stack) + .command(stackBatch) .command(component) .command(image) .command(validateToken) diff --git a/src/index.js b/src/index.js index 3dfdf976..a3980992 100644 --- a/src/index.js +++ b/src/index.js @@ -1,9 +1,19 @@ import path from "node:path"; import { EOL } from "os"; +import pLimit from 'p-limit' + import { availableProviders, match } from './provider.js' import analysis from './analysis.js' import fs from 'node:fs' import { getCustom } from "./tools.js"; +import { resolveBatchMetadata, resolveContinueOnError } from './batch_opts.js' +import { + discoverWorkspaceCrates, + discoverWorkspacePackages, + filterManifestPathsByDiscoveryIgnore, + resolveWorkspaceDiscoveryIgnore, + validatePackageJson, +} from './workspace.js' import.meta.dirname import * as url from 'url'; @@ -11,7 +21,16 @@ export { parseImageRef } from "./oci_image/utils.js"; export { ImageRef } from "./oci_image/images.js"; export { getProjectLicense, findLicenseFilePath, identifyLicense, getLicenseDetails, licensesFromReport, normalizeLicensesResponse, runLicenseCheck, getCompatibility } from "./license/index.js"; -export default { componentAnalysis, stackAnalysis, imageAnalysis, validateToken } +export default { componentAnalysis, stackAnalysis, stackAnalysisBatch, imageAnalysis, validateToken } +export { + discoverWorkspacePackages, + discoverWorkspaceCrates, + validatePackageJson, + resolveWorkspaceDiscoveryIgnore, + filterManifestPathsByDiscoveryIgnore, + resolveContinueOnError, + resolveBatchMetadata, +} /** * @typedef {{ @@ -40,15 +59,36 @@ export default { componentAnalysis, stackAnalysis, imageAnalysis, validateToken * TRUSTIFY_DA_SYFT_CONFIG_PATH?: string | undefined, * TRUSTIFY_DA_SYFT_PATH?: string | undefined, * TRUSTIFY_DA_YARN_PATH?: string | undefined, + * TRUSTIFY_DA_WORKSPACE_DIR?: string | undefined, * TRUSTIFY_DA_LICENSE_CHECK?: string | undefined, * MATCH_MANIFEST_VERSIONS?: string | undefined, * TRUSTIFY_DA_SOURCE?: string | undefined, * TRUSTIFY_DA_TOKEN?: string | undefined, * TRUSTIFY_DA_TELEMETRY_ID?: string | undefined, - * [key: string]: string | undefined, + * TRUSTIFY_DA_WORKSPACE_DIR?: string | undefined, + * batchConcurrency?: number | undefined, + * TRUSTIFY_DA_BATCH_CONCURRENCY?: string | undefined, + * workspaceDiscoveryIgnore?: string[] | undefined, + * TRUSTIFY_DA_WORKSPACE_DISCOVERY_IGNORE?: string | undefined, + * continueOnError?: boolean | undefined, + * TRUSTIFY_DA_CONTINUE_ON_ERROR?: string | undefined, + * batchMetadata?: boolean | undefined, + * TRUSTIFY_DA_BATCH_METADATA?: string | undefined, + * [key: string]: string | number | boolean | string[] | undefined, * }} Options */ +/** + * @typedef {{ + * workspaceRoot: string, + * ecosystem: 'javascript' | 'cargo' | 'unknown', + * total: number, + * successful: number, + * failed: number, + * errors: Array<{ manifestPath: string, phase: 'validation' | 'sbom', reason: string }> + * }} BatchAnalysisMetadata + */ + /** * Logs messages to the console if the TRUSTIFY_DA_DEBUG environment variable is set to "true". @@ -144,7 +184,7 @@ export function selectTrustifyDABackend(opts = {}) { async function stackAnalysis(manifest, html = false, opts = {}) { const theUrl = selectTrustifyDABackend(opts) fs.accessSync(manifest, fs.constants.R_OK) // throws error if file unreadable - let provider = match(manifest, availableProviders) // throws error if no matching provider + let provider = match(manifest, availableProviders, opts) // throws error if no matching provider return await analysis.requestStack(provider, manifest, theUrl, html, opts) // throws error request sending failed } @@ -159,7 +199,7 @@ async function componentAnalysis(manifest, opts = {}) { const theUrl = selectTrustifyDABackend(opts) fs.accessSync(manifest, fs.constants.R_OK) opts["manifest-type"] = path.basename(manifest) - let provider = match(manifest, availableProviders) // throws error if no matching provider + let provider = match(manifest, availableProviders, opts) // throws error if no matching provider return await analysis.requestComponent(provider, manifest, theUrl, opts) // throws error request sending failed } @@ -196,6 +236,273 @@ async function imageAnalysis(imageRefs, html = false, opts = {}) { return await analysis.requestImages(imageRefs, theUrl, html, opts) } +/** + * Max concurrent SBOM generations for batch workspace analysis. Env/opts override default 10. + * @param {Options} opts + * @returns {number} + * @private + */ +function resolveBatchConcurrency(opts) { + const fromEnv = getCustom('TRUSTIFY_DA_BATCH_CONCURRENCY', null, opts) + const raw = opts.batchConcurrency ?? fromEnv ?? '10' + const n = typeof raw === 'number' ? raw : parseInt(String(raw), 10) + if (!Number.isFinite(n) || n < 1) { + return 10 + } + return Math.min(256, n) +} + +/** + * @param {string} root + * @param {'javascript' | 'cargo' | 'unknown'} ecosystem + * @param {number} totalSbomAttempts + * @param {number} successfulSbomCount + * @param {Array<{ manifestPath: string, phase: 'validation' | 'sbom', reason: string }>} errors + * @returns {BatchAnalysisMetadata} + * @private + */ +function buildBatchAnalysisMetadata(root, ecosystem, totalSbomAttempts, successfulSbomCount, errors) { + return { + workspaceRoot: root, + ecosystem, + total: totalSbomAttempts, + successful: successfulSbomCount, + failed: errors.length, + errors: [...errors], + } +} + +/** + * @typedef {{ ok: true, purl: string, sbom: object } | { ok: false, manifestPath: string, reason: string }} SbomResult + */ + +/** + * Generate an SBOM for a single manifest, returning a normalized result. + * + * @param {string} manifestPath + * @param {Options} workspaceOpts - opts with `TRUSTIFY_DA_WORKSPACE_DIR` set + * @returns {Promise} + * @private + */ +async function generateOneSbom(manifestPath, workspaceOpts) { + const provider = match(manifestPath, availableProviders, workspaceOpts) + const provided = await provider.provideStack(manifestPath, workspaceOpts) + const sbom = JSON.parse(provided.content) + const purl = sbom?.metadata?.component?.purl || sbom?.metadata?.component?.['bom-ref'] + if (!purl) { + return { ok: false, manifestPath, reason: 'missing purl in SBOM' } + } + return { ok: true, purl, sbom } +} + +/** + * Detect the workspace ecosystem and discover manifest paths. + * + * @param {string} root - Resolved workspace root + * @param {Options} opts + * @returns {Promise<{ ecosystem: 'javascript' | 'cargo' | 'unknown', manifestPaths: string[] }>} + * @private + */ +async function detectWorkspaceManifests(root, opts) { + const cargoToml = path.join(root, 'Cargo.toml') + const cargoLock = path.join(root, 'Cargo.lock') + const packageJson = path.join(root, 'package.json') + + if (fs.existsSync(cargoToml) && fs.existsSync(cargoLock)) { + return { ecosystem: 'cargo', manifestPaths: await discoverWorkspaceCrates(root, opts) } + } + + const hasJsLock = fs.existsSync(path.join(root, 'pnpm-lock.yaml')) + || fs.existsSync(path.join(root, 'yarn.lock')) + || fs.existsSync(path.join(root, 'package-lock.json')) + + if (fs.existsSync(packageJson) && hasJsLock) { + let manifestPaths = await discoverWorkspacePackages(root, opts) + if (manifestPaths.length === 0) { + manifestPaths = [packageJson] + } + return { ecosystem: 'javascript', manifestPaths } + } + + return { ecosystem: 'unknown', manifestPaths: [] } +} + +/** + * Validate discovered JS package.json manifests, collecting errors. + * + * @param {string[]} manifestPaths + * @param {boolean} continueOnError + * @param {Array<{ manifestPath: string, phase: 'validation' | 'sbom', reason: string }>} collectedErrors - mutated in place + * @returns {{ validPaths: string[] }} + * @throws {Error} on first invalid manifest when `continueOnError` is false + * @private + */ +function validateJsManifests(manifestPaths, continueOnError, collectedErrors) { + const validPaths = [] + for (const p of manifestPaths) { + const v = validatePackageJson(p) + if (v.valid) { + validPaths.push(p) + } else { + collectedErrors.push({ manifestPath: p, phase: 'validation', reason: v.error }) + console.warn(`Skipping invalid package.json (${v.error}): ${p}`) + if (!continueOnError) { + throw new Error(`Invalid package.json (${v.error}): ${p}`) + } + } + } + return { validPaths } +} + +/** + * Generate SBOMs for all manifests. In fail-fast mode, stops on first error. + * In continue-on-error mode, runs concurrently and collects failures. + * + * @param {string[]} manifestPaths + * @param {Options} workspaceOpts + * @param {boolean} continueOnError + * @param {number} concurrency + * @param {Array<{ manifestPath: string, phase: 'validation' | 'sbom', reason: string }>} collectedErrors - mutated in place + * @returns {Promise>} sbomByPurl map + * @throws {Error} on first SBOM failure when `continueOnError` is false + * @private + */ +async function generateSboms(manifestPaths, workspaceOpts, continueOnError, concurrency, collectedErrors) { + /** @type {SbomResult[]} */ + const results = [] + + if (!continueOnError) { + for (const manifestPath of manifestPaths) { + const result = await generateOneSbom(manifestPath, workspaceOpts) + if (!result.ok) { + collectedErrors.push({ manifestPath: result.manifestPath, phase: 'sbom', reason: result.reason }) + throw new Error(`${result.manifestPath}: ${result.reason}`) + } + results.push(result) + } + } else { + const limit = pLimit(concurrency) + const settled = await Promise.all( + manifestPaths.map(manifestPath => limit(async () => { + try { + return await generateOneSbom(manifestPath, workspaceOpts) + } catch (err) { + const msg = err instanceof Error ? err.message : String(err) + if (process.env["TRUSTIFY_DA_DEBUG"] === "true") { + console.log(`Skipping ${manifestPath}: ${msg}`) + } + return { ok: false, manifestPath, reason: msg } + } + })) + ) + for (const r of settled) { + results.push(r) + if (!r.ok) { + collectedErrors.push({ manifestPath: r.manifestPath, phase: 'sbom', reason: r.reason }) + } + } + } + + const sbomByPurl = {} + for (const r of results) { + if (r.ok) { + sbomByPurl[r.purl] = r.sbom + } + } + return sbomByPurl +} + +/** + * Create an Error with optional `batchMetadata` attached. + * @param {string} message + * @param {boolean} wantMetadata + * @param {BatchAnalysisMetadata} [metadata] + * @returns {Error} + * @private + */ +function batchError(message, wantMetadata, metadata) { + const err = new Error(message) + if (wantMetadata && metadata) { + err.batchMetadata = metadata + } + return err +} + +/** + * Get stack analysis for all workspace packages/crates (batch). + * Detects ecosystem from workspace root: Cargo (Cargo.toml + Cargo.lock) or JS/TS (package.json + lock file). + * SBOMs are generated in parallel (see `batchConcurrency`) unless `continueOnError: false` (fail-fast sequential). + * With `opts.batchMetadata` / `TRUSTIFY_DA_BATCH_METADATA`, returns `{ analysis, metadata }` including validation and SBOM errors. + * + * @param {string} workspaceRoot - Path to workspace root (containing lock file and workspace config) + * @param {boolean} [html=false] - true returns HTML, false returns JSON report + * @param {Options} [opts={}] - `batchConcurrency`, discovery ignores, `continueOnError` (default true), `batchMetadata` (default false) + * @returns {Promise|{ analysis: string|Object., metadata: BatchAnalysisMetadata }>} + * @throws {Error} if workspace root invalid, no manifests found, no packages pass validation, no SBOMs produced, or backend request failed. When `opts.batchMetadata` is set, `error.batchMetadata` may be set on thrown errors. + */ +async function stackAnalysisBatch(workspaceRoot, html = false, opts = {}) { + const theUrl = selectTrustifyDABackend(opts) + const root = path.resolve(workspaceRoot) + fs.accessSync(root, fs.constants.R_OK) + + const continueOnError = resolveContinueOnError(opts) + const wantMetadata = resolveBatchMetadata(opts) + + /** @type {Array<{ manifestPath: string, phase: 'validation' | 'sbom', reason: string }>} */ + const collectedErrors = [] + + const { ecosystem, manifestPaths: discovered } = await detectWorkspaceManifests(root, opts) + let manifestPaths = discovered + + if (ecosystem === 'javascript') { + try { + const { validPaths } = validateJsManifests(manifestPaths, continueOnError, collectedErrors) + manifestPaths = validPaths + } catch (err) { + throw batchError(err.message, wantMetadata, + buildBatchAnalysisMetadata(root, ecosystem, 0, 0, collectedErrors)) + } + if (manifestPaths.length === 0 && discovered.length > 0) { + const detail = collectedErrors.map(e => `${e.manifestPath}: ${e.reason}`).join('; ') + throw batchError(`No valid packages after validation at ${root}. ${detail}`, wantMetadata, + buildBatchAnalysisMetadata(root, ecosystem, 0, 0, collectedErrors)) + } + } + + if (manifestPaths.length === 0) { + throw new Error(`No workspace manifests found at ${root}. Ensure Cargo.toml+Cargo.lock or package.json+lock file exist.`) + } + + const workspaceOpts = { ...opts, TRUSTIFY_DA_WORKSPACE_DIR: root } + const concurrency = resolveBatchConcurrency(opts) + + let sbomByPurl + try { + sbomByPurl = await generateSboms(manifestPaths, workspaceOpts, continueOnError, concurrency, collectedErrors) + } catch (err) { + throw batchError(err.message, wantMetadata, + buildBatchAnalysisMetadata(root, ecosystem, manifestPaths.length, 0, collectedErrors)) + } + + if (Object.keys(sbomByPurl).length === 0) { + throw batchError( + `No valid SBOMs produced from ${manifestPaths.length} manifest(s) at ${root}`, + wantMetadata, + buildBatchAnalysisMetadata(root, ecosystem, manifestPaths.length, 0, collectedErrors) + ) + } + + const analysisResult = await analysis.requestStackBatch(sbomByPurl, theUrl, html, opts) + const meta = buildBatchAnalysisMetadata( + root, ecosystem, manifestPaths.length, Object.keys(sbomByPurl).length, collectedErrors + ) + + if (wantMetadata) { + return { analysis: analysisResult, metadata: meta } + } + return analysisResult +} + /** * Validates the Exhort token. * @param {Options} [opts={}] - Optional parameters, potentially including token override. diff --git a/src/provider.js b/src/provider.js index 4bfa8b52..0a1fa94a 100644 --- a/src/provider.js +++ b/src/provider.js @@ -11,7 +11,7 @@ import pythonPipProvider from './providers/python_pip.js' import rustCargoProvider from './providers/rust_cargo.js' /** @typedef {{ecosystem: string, contentType: string, content: string}} Provided */ -/** @typedef {{isSupported: function(string): boolean, validateLockFile: function(string): void, provideComponent: function(string, {}): Provided | Promise, provideStack: function(string, {}): Provided | Promise, readLicenseFromManifest: function(string): string | null}} Provider */ +/** @typedef {{isSupported: function(string): boolean, validateLockFile: function(string, Object): void, provideComponent: function(string, {}): Provided | Promise, provideStack: function(string, {}): Provided | Promise, readLicenseFromManifest: function(string): string | null}} Provider */ /** * MUST include all providers here. @@ -21,9 +21,9 @@ export const availableProviders = [ new Java_maven(), new Java_gradle_groovy(), new Java_gradle_kotlin(), - new Javascript_npm(), new Javascript_pnpm(), new Javascript_yarn(), + new Javascript_npm(), golangGomodulesProvider, pythonPipProvider, rustCargoProvider] @@ -51,16 +51,17 @@ export function matchForLicense(manifestPath, providers) { * Each provider MUST export 'provideStack' taking manifest path returning a {@link Provided}. * @param {string} manifest - the name-type or path of the manifest * @param {[Provider]} providers - list of providers to iterate over + * @param {{TRUSTIFY_DA_WORKSPACE_DIR?: string}} [opts={}] - optional; TRUSTIFY_DA_WORKSPACE_DIR overrides lock file location for workspaces * @returns {Provider} * @throws {Error} when the manifest is not supported and no provider was matched */ -export function match(manifest, providers) { +export function match(manifest, providers, opts = {}) { const manifestPath = path.parse(manifest) const supported = providers.filter(prov => prov.isSupported(manifestPath.base)) if (supported.length === 0) { throw new Error(`${manifestPath.base} is not supported`) } - const provider = supported.find(prov => prov.validateLockFile(manifestPath.dir)) + const provider = supported.find(prov => prov.validateLockFile(manifestPath.dir, opts)) if(!provider) { throw new Error(`${manifestPath.base} requires a lock file. Use your preferred package manager to generate the lock file.`); } diff --git a/src/providers/base_javascript.js b/src/providers/base_javascript.js index 2d863539..64af7f22 100644 --- a/src/providers/base_javascript.js +++ b/src/providers/base_javascript.js @@ -4,7 +4,7 @@ import path from 'node:path' import { getLicense } from '../license/license_utils.js' import Sbom from '../sbom.js' -import { getCustom, getCustomPath, invokeCommand, toPurl, toPurlFromString } from "../tools.js"; +import { getCustom, getCustomPath, invokeCommand, toPurl, toPurlFromString } from '../tools.js' import Manifest from './manifest.js'; @@ -112,13 +112,18 @@ export default class Base_javascript { } /** - * Checks if a required lock file exists in the same path as the manifest + * Checks if a required lock file exists in the manifest directory or at the workspace root. + * When TRUSTIFY_DA_WORKSPACE_DIR is provided (via env var or opts), + * checks only that directory for the lock file. * @param {string} manifestDir - The base directory where the manifest is located + * @param {{TRUSTIFY_DA_WORKSPACE_DIR?: string}} [opts={}] - optional workspace root * @returns {boolean} True if the lock file exists */ - validateLockFile(manifestDir) { - const lock = path.join(manifestDir, this._lockFileName()); - return fs.existsSync(lock); + validateLockFile(manifestDir, opts = {}) { + const workspaceDir = getCustom('TRUSTIFY_DA_WORKSPACE_DIR', null, opts) + const dirToCheck = workspaceDir ? path.resolve(workspaceDir) : manifestDir + const lock = path.join(dirToCheck, this._lockFileName()) + return fs.existsSync(lock) } /** @@ -176,15 +181,18 @@ export default class Base_javascript { /** * Builds the dependency tree for the project * @param {boolean} includeTransitive - Whether to include transitive dependencies + * @param {Object} [opts={}] - Configuration options; when `TRUSTIFY_DA_WORKSPACE_DIR` is set, commands run from workspace root * @returns {Object} The dependency tree * @protected */ - _buildDependencyTree(includeTransitive) { + _buildDependencyTree(includeTransitive, opts = {}) { this._version(); - let manifestDir = path.dirname(this.#manifest.manifestPath); - this.#createLockFile(manifestDir); + const manifestDir = path.dirname(this.#manifest.manifestPath); + const workspaceDir = getCustom('TRUSTIFY_DA_WORKSPACE_DIR', null, opts) + const cmdDir = workspaceDir ? path.resolve(workspaceDir) : manifestDir; + this.#createLockFile(cmdDir); - let output = this.#executeListCmd(includeTransitive, manifestDir); + let output = this.#executeListCmd(includeTransitive, cmdDir); output = this._parseDepTreeOutput(output); return JSON.parse(output); } @@ -196,7 +204,7 @@ export default class Base_javascript { * @private */ #getSBOM(opts = {}) { - const depsObject = this._buildDependencyTree(true); + const depsObject = this._buildDependencyTree(true, opts); let mainComponent = toPurl(purlType, this.#manifest.name, this.#manifest.version); const license = this.readLicenseFromManifest(this.#manifest.manifestPath); @@ -255,7 +263,7 @@ export default class Base_javascript { * @private */ #getDirectDependencySbom(opts = {}) { - const depTree = this._buildDependencyTree(false); + const depTree = this._buildDependencyTree(false, opts); let mainComponent = toPurl(purlType, this.#manifest.name, this.#manifest.version); const license = this.readLicenseFromManifest(this.#manifest.manifestPath); diff --git a/src/providers/javascript_pnpm.js b/src/providers/javascript_pnpm.js index 9cb723f5..142add02 100644 --- a/src/providers/javascript_pnpm.js +++ b/src/providers/javascript_pnpm.js @@ -18,8 +18,8 @@ export default class Javascript_pnpm extends Base_javascript { return ['install', '--frozen-lockfile']; } - _buildDependencyTree(includeTransitive, manifest) { - const tree = super._buildDependencyTree(includeTransitive, manifest); + _buildDependencyTree(includeTransitive, opts = {}) { + const tree = super._buildDependencyTree(includeTransitive, opts); if (Array.isArray(tree) && tree.length > 0) { return tree[0]; } diff --git a/src/providers/rust_cargo.js b/src/providers/rust_cargo.js index bbe52c10..bfe62eba 100644 --- a/src/providers/rust_cargo.js +++ b/src/providers/rust_cargo.js @@ -6,7 +6,7 @@ import { parse as parseToml } from 'smol-toml' import { getLicense } from '../license/license_utils.js' import Sbom from '../sbom.js' -import { getCustomPath, invokeCommand } from '../tools.js' +import { getCustom, getCustomPath, invokeCommand } from '../tools.js' export default { isSupported, validateLockFile, provideComponent, provideStack, readLicenseFromManifest } @@ -102,10 +102,19 @@ function readLicenseFromManifest(manifestPath) { * up the directory tree looking for Cargo.lock (stopping when we find a * Cargo.toml that contains a [workspace] section, or when we reach the * filesystem root). + * When TRUSTIFY_DA_WORKSPACE_DIR is provided (via env var or opts), + * checks only that directory for Cargo.lock — no walk-up. * @param {string} manifestDir - the directory where the manifest lies + * @param {{TRUSTIFY_DA_WORKSPACE_DIR?: string}} [opts={}] - optional workspace root * @returns {boolean} true if Cargo.lock is found */ -function validateLockFile(manifestDir) { +function validateLockFile(manifestDir, opts = {}) { + const workspaceDir = getCustom('TRUSTIFY_DA_WORKSPACE_DIR', null, opts) + if (workspaceDir) { + const dir = path.resolve(workspaceDir) + return fs.existsSync(path.join(dir, 'Cargo.lock')) + } + let dir = path.resolve(manifestDir) let parent = dir diff --git a/src/workspace.js b/src/workspace.js new file mode 100644 index 00000000..f71d5e18 --- /dev/null +++ b/src/workspace.js @@ -0,0 +1,271 @@ +import fs from 'node:fs' +import path from 'node:path' + +import fg from 'fast-glob' +import { load as yamlLoad } from 'js-yaml' +import micromatch from 'micromatch' + +import { getCustom, getCustomPath, invokeCommand } from './tools.js' + +/** Default paths skipped during JS workspace discovery (merged with user patterns). */ +const DEFAULT_WORKSPACE_DISCOVERY_IGNORE = [ + '**/node_modules/**', + '**/.git/**', +] + +/** + * Resolve ignore globs for workspace discovery: defaults + `TRUSTIFY_DA_WORKSPACE_DISCOVERY_IGNORE` + `opts.workspaceDiscoveryIgnore`. + * Patterns are fast-glob / micromatch style, relative to the workspace root (forward slashes). + * + * @param {{ workspaceDiscoveryIgnore?: string[], TRUSTIFY_DA_WORKSPACE_DISCOVERY_IGNORE?: string, [key: string]: unknown }} [opts={}] + * @returns {string[]} + */ +export function resolveWorkspaceDiscoveryIgnore(opts = {}) { + const merged = [...DEFAULT_WORKSPACE_DISCOVERY_IGNORE] + const fromEnv = getCustom('TRUSTIFY_DA_WORKSPACE_DISCOVERY_IGNORE', null, opts) + if (fromEnv && String(fromEnv).trim()) { + merged.push(...String(fromEnv).split(',').map(s => s.trim()).filter(Boolean)) + } + const extra = opts.workspaceDiscoveryIgnore + if (Array.isArray(extra)) { + for (const p of extra) { + if (typeof p === 'string' && p.trim()) { + merged.push(p.trim()) + } + } + } + return [...new Set(merged)] +} + +/** + * @param {string} root - Workspace root (absolute) + * @param {string[]} ignorePatterns + */ +function buildWorkspaceDiscoveryGlobOptions(root, ignorePatterns) { + return { + cwd: root, + absolute: true, + onlyFiles: true, + ignore: ignorePatterns, + followSymbolicLinks: false, + } +} + +/** + * @param {string} filePath - Absolute path + * @param {string} root - Workspace root (absolute) + * @returns {string} Relative path with forward slashes + */ +function relativePathForGlobMatch(filePath, root) { + const rel = path.relative(root, filePath) + return rel.split(path.sep).join('/') +} + +/** + * Drop manifest paths whose location matches an ignore pattern (e.g. root unshift, Cargo paths). + * + * @param {string[]} manifestPaths + * @param {string} root + * @param {string[]} ignorePatterns + * @returns {string[]} + */ +export function filterManifestPathsByDiscoveryIgnore(manifestPaths, root, ignorePatterns) { + if (!ignorePatterns.length) { + return manifestPaths + } + const resolvedRoot = path.resolve(root) + return manifestPaths.filter(absPath => { + const rel = relativePathForGlobMatch(absPath, resolvedRoot) + if (rel === '') { + return true + } + return !ignorePatterns.some(pattern => micromatch.isMatch(rel, pattern, { dot: true })) + }) +} + +/** + * @typedef {{ valid: true, name: string, version: string } | { valid: false, error: string }} ValidatePackageJsonResult + */ + +/** + * Validate a package.json has non-empty `name` and `version` (required for stable SBOM root identity in batch). + * + * @param {string} packageJsonPath - Absolute or relative path to package.json + * @returns {ValidatePackageJsonResult} + */ +export function validatePackageJson(packageJsonPath) { + let content + try { + const raw = fs.readFileSync(packageJsonPath, 'utf-8') + content = JSON.parse(raw) + } catch (err) { + const msg = err instanceof Error ? err.message : String(err) + return { valid: false, error: `Invalid package.json: ${msg}` } + } + if (!content || typeof content !== 'object') { + return { valid: false, error: 'package.json must be a JSON object' } + } + const name = content.name + const version = content.version + if (typeof name !== 'string' || !name.trim()) { + return { valid: false, error: 'Missing or invalid name' } + } + if (typeof version !== 'string' || !version.trim()) { + return { valid: false, error: 'Missing or invalid version' } + } + return { valid: true, name: name.trim(), version: version.trim() } +} + +/** + * Discover all package.json paths in a JS/TS workspace. + * Reads pnpm-workspace.yaml or package.json workspaces. + * + * @param {string} workspaceRoot - Absolute or relative path to workspace root + * @param {{ workspaceDiscoveryIgnore?: string[], TRUSTIFY_DA_WORKSPACE_DISCOVERY_IGNORE?: string, [key: string]: unknown }} [opts={}] - optional `workspaceDiscoveryIgnore` globs (merged with defaults and env) + * @returns {Promise} Paths to package.json files (absolute) + */ +export async function discoverWorkspacePackages(workspaceRoot, opts = {}) { + const root = path.resolve(workspaceRoot) + const ignorePatterns = resolveWorkspaceDiscoveryIgnore(opts) + const globOpts = buildWorkspaceDiscoveryGlobOptions(root, ignorePatterns) + const pnpmWorkspace = path.join(root, 'pnpm-workspace.yaml') + const packageJson = path.join(root, 'package.json') + + if (fs.existsSync(pnpmWorkspace)) { + return discoverFromPnpmWorkspace(root, pnpmWorkspace, globOpts, ignorePatterns) + } + if (fs.existsSync(packageJson)) { + return discoverFromPackageJsonWorkspaces(root, packageJson, globOpts, ignorePatterns) + } + return [] +} + +/** + * @param {string} root + * @param {string} pnpmWorkspacePath + * @param {object} globOpts - fast-glob options (cwd, ignore, followSymbolicLinks, …) + * @param {string[]} ignorePatterns - for post-filter + * @returns {Promise} + */ +async function discoverFromPnpmWorkspace(root, pnpmWorkspacePath, globOpts, ignorePatterns) { + const content = fs.readFileSync(pnpmWorkspacePath, 'utf-8') + const packages = parsePnpmPackages(content) + if (packages.length === 0) { + return [] + } + const patterns = toManifestGlobPatterns(packages, 'package.json') + const manifestPaths = await fg(patterns, globOpts) + return filterManifestPathsByDiscoveryIgnore(manifestPaths, root, ignorePatterns) +} + +/** + * Parse the `packages` array from pnpm-workspace.yaml content. + * @param {string} content - Raw YAML content + * @returns {string[]} + */ +function parsePnpmPackages(content) { + let doc + try { + doc = yamlLoad(content) + } catch { + return [] + } + if (!doc || typeof doc !== 'object' || !Array.isArray(doc.packages)) { + return [] + } + return doc.packages.filter(p => typeof p === 'string' && p.trim()).map(p => p.trim()) +} + +/** + * Convert workspace glob patterns to manifest-file glob patterns, + * correctly handling negation prefixes. + * + * @param {string[]} patterns - Workspace glob patterns (may include negations) + * @param {string} manifestFileName - e.g. 'package.json' or 'Cargo.toml' + * @returns {string[]} + */ +function toManifestGlobPatterns(patterns, manifestFileName) { + return patterns.map(p => { + if (p.startsWith('!')) { + return `!${p.slice(1)}/${manifestFileName}` + } + return `${p}/${manifestFileName}` + }) +} + +/** + * @param {string} root + * @param {string} packageJsonPath + * @param {object} globOpts + * @param {string[]} ignorePatterns + * @returns {Promise} + */ +async function discoverFromPackageJsonWorkspaces(root, packageJsonPath, globOpts, ignorePatterns) { + let pkg + try { + pkg = JSON.parse(fs.readFileSync(packageJsonPath, 'utf-8')) + } catch { + return [] + } + const workspaces = pkg.workspaces + if (!workspaces) { + return [] + } + const raw = Array.isArray(workspaces) ? workspaces : workspaces.packages || [] + const patterns = toManifestGlobPatterns(raw.filter(p => typeof p === 'string'), 'package.json') + if (patterns.length === 0) { + return [] + } + const manifestPaths = await fg(patterns, globOpts) + const rootPkg = path.join(root, 'package.json') + if (fs.existsSync(rootPkg) && !manifestPaths.includes(rootPkg)) { + manifestPaths.unshift(rootPkg) + } + return filterManifestPathsByDiscoveryIgnore(manifestPaths, root, ignorePatterns) +} + +/** + * Discover all Cargo.toml manifest paths in a Cargo workspace. + * Uses `cargo metadata` to get workspace members. + * + * @param {string} workspaceRoot - Absolute or relative path to workspace root (must contain Cargo.toml and Cargo.lock) + * @param {import('./index.js').Options} [opts={}] + * @returns {Promise} Paths to Cargo.toml files (absolute) + */ +export async function discoverWorkspaceCrates(workspaceRoot, opts = {}) { + const root = path.resolve(workspaceRoot) + const cargoToml = path.join(root, 'Cargo.toml') + const cargoLock = path.join(root, 'Cargo.lock') + + if (!fs.existsSync(cargoToml) || !fs.existsSync(cargoLock)) { + return [] + } + + const cargoBin = getCustomPath('cargo', opts) + let output + try { + output = invokeCommand(cargoBin, ['metadata', '--format-version', '1', '--no-deps'], { cwd: root }) + } catch { + return [] + } + + let metadata + try { + metadata = JSON.parse(output.toString().trim()) + } catch { + return [] + } + + const memberIds = new Set(metadata.workspace_members || []) + const manifestPaths = [] + for (const pkg of metadata.packages || []) { + if (memberIds.has(pkg.id) && pkg.manifest_path) { + const manifestPath = path.resolve(pkg.manifest_path) + if (fs.existsSync(manifestPath)) { + manifestPaths.push(manifestPath) + } + } + } + const ignorePatterns = resolveWorkspaceDiscoveryIgnore(opts) + return filterManifestPathsByDiscoveryIgnore(manifestPaths, root, ignorePatterns) +} diff --git a/test/batch_opts.test.js b/test/batch_opts.test.js new file mode 100644 index 00000000..b3f1b97f --- /dev/null +++ b/test/batch_opts.test.js @@ -0,0 +1,51 @@ +import { expect } from 'chai' + +import { resolveBatchMetadata, resolveContinueOnError } from '../src/batch_opts.js' + +suite('resolveContinueOnError', () => { + test('defaults to true', () => { + expect(resolveContinueOnError({})).to.be.true + }) + + test('opts false disables', () => { + expect(resolveContinueOnError({ continueOnError: false })).to.be.false + }) + + test('env false disables', () => { + const prev = process.env.TRUSTIFY_DA_CONTINUE_ON_ERROR + process.env.TRUSTIFY_DA_CONTINUE_ON_ERROR = 'false' + try { + expect(resolveContinueOnError({})).to.be.false + } finally { + if (prev === undefined) { + delete process.env.TRUSTIFY_DA_CONTINUE_ON_ERROR + } else { + process.env.TRUSTIFY_DA_CONTINUE_ON_ERROR = prev + } + } + }) +}) + +suite('resolveBatchMetadata', () => { + test('defaults to false', () => { + expect(resolveBatchMetadata({})).to.be.false + }) + + test('opts true enables', () => { + expect(resolveBatchMetadata({ batchMetadata: true })).to.be.true + }) + + test('env true enables', () => { + const prev = process.env.TRUSTIFY_DA_BATCH_METADATA + process.env.TRUSTIFY_DA_BATCH_METADATA = 'true' + try { + expect(resolveBatchMetadata({})).to.be.true + } finally { + if (prev === undefined) { + delete process.env.TRUSTIFY_DA_BATCH_METADATA + } else { + process.env.TRUSTIFY_DA_BATCH_METADATA = prev + } + } + }) +}) diff --git a/test/providers/javascript.test.js b/test/providers/javascript.test.js index 861166d4..59708b08 100644 --- a/test/providers/javascript.test.js +++ b/test/providers/javascript.test.js @@ -152,5 +152,20 @@ suite('testing the javascript-npm data provider', async () => { expect(() => new Manifest(manifestPath)).to.throw(Error); }); + test('verify match with opts.TRUSTIFY_DA_WORKSPACE_DIR finds npm provider when lock is at workspace root', () => { + const manifest = 'test/providers/provider_manifests/npm/with_lock_file/package.json' + const opts = { TRUSTIFY_DA_WORKSPACE_DIR: 'test/providers/provider_manifests/npm/with_lock_file' } + const provider = match(manifest, availableProviders, opts) + expect(provider).to.not.be.null + expect(provider.isSupported('package.json')).to.be.true + }) + + test('verify match with opts.TRUSTIFY_DA_WORKSPACE_DIR finds pnpm provider when lock is at workspace root', () => { + const manifest = 'test/providers/provider_manifests/pnpm/with_lock_file/package.json' + const opts = { TRUSTIFY_DA_WORKSPACE_DIR: 'test/providers/provider_manifests/pnpm/with_lock_file' } + const provider = match(manifest, availableProviders, opts) + expect(provider).to.not.be.null + expect(provider.isSupported('package.json')).to.be.true + }) }).beforeAll(() => clock = useFakeTimers(new Date('2023-08-07T00:00:00.000Z'))).afterAll(() => clock.restore()); diff --git a/test/providers/rust_cargo.test.js b/test/providers/rust_cargo.test.js index dda73448..edcd01b7 100644 --- a/test/providers/rust_cargo.test.js +++ b/test/providers/rust_cargo.test.js @@ -125,6 +125,23 @@ suite('testing the rust-cargo data provider', () => { test('verify validateLockFile returns false for workspace member when workspace root has no Cargo.lock', () => { expect(rustCargo.validateLockFile('test/providers/provider_manifests/cargo/workspace_member_without_lock/member1')).to.equal(false) }) + + test('verify validateLockFile with opts.TRUSTIFY_DA_WORKSPACE_DIR returns true when Cargo.lock exists at workspace root', () => { + const workspaceRoot = 'test/providers/provider_manifests/cargo/workspace_member_with_lock' + expect(rustCargo.validateLockFile('test/providers/provider_manifests/cargo/workspace_member_with_lock/member1', { TRUSTIFY_DA_WORKSPACE_DIR: workspaceRoot })).to.equal(true) + }) + + test('verify validateLockFile with opts.TRUSTIFY_DA_WORKSPACE_DIR returns false when Cargo.lock is not at workspace root', () => { + const wrongDir = 'test/providers/provider_manifests/cargo/workspace_member_without_lock' + expect(rustCargo.validateLockFile('test/providers/provider_manifests/cargo/workspace_member_with_lock/member1', { TRUSTIFY_DA_WORKSPACE_DIR: wrongDir })).to.equal(false) + }) + + test('verify match with opts.TRUSTIFY_DA_WORKSPACE_DIR finds Cargo provider when lock is at workspace root', () => { + const opts = { TRUSTIFY_DA_WORKSPACE_DIR: 'test/providers/provider_manifests/cargo/workspace_member_with_lock' } + const provider = match('test/providers/provider_manifests/cargo/workspace_member_with_lock/member1/Cargo.toml', availableProviders, opts) + expect(provider).to.not.be.null + expect(provider.isSupported('Cargo.toml')).to.be.true + }) }).beforeAll(() => clock = useFakeTimers(new Date('2023-08-07T00:00:00.000Z'))).afterAll(() => clock.restore()); suite('testing the rust-cargo single crate without ignore', () => { diff --git a/test/providers/workspace.test.js b/test/providers/workspace.test.js new file mode 100644 index 00000000..5c06b68a --- /dev/null +++ b/test/providers/workspace.test.js @@ -0,0 +1,189 @@ +import fs from 'node:fs' +import path from 'node:path' + +import { expect } from 'chai' +import esmock from 'esmock' + +import { + discoverWorkspaceCrates, + discoverWorkspacePackages, + filterManifestPathsByDiscoveryIgnore, + resolveWorkspaceDiscoveryIgnore, + validatePackageJson, +} from '../../src/workspace.js' + +suite('discoverWorkspacePackages', () => { + test('returns empty when no pnpm-workspace.yaml or package.json workspaces', async () => { + const root = 'test/providers/tst_manifests/npm/package_json_deps_without_exhortignore_object' + const result = await discoverWorkspacePackages(root) + expect(result).to.be.an('array') + expect(result).to.have.lengthOf(0) + }) + + test('excludes paths matching workspaceDiscoveryIgnore', async () => { + const pkgContent = { name: 'root', version: '1.0.0', workspaces: ['packages/*'] } + const memberA = { name: 'a', version: '1.0.0' } + const memberB = { name: 'b', version: '1.0.0' } + const tmpDir = path.join(process.cwd(), 'test/providers/tst_manifests/tmp_workspace_ignore') + fs.mkdirSync(path.join(tmpDir, 'packages/a'), { recursive: true }) + fs.mkdirSync(path.join(tmpDir, 'packages/b'), { recursive: true }) + fs.writeFileSync(path.join(tmpDir, 'package.json'), JSON.stringify(pkgContent)) + fs.writeFileSync(path.join(tmpDir, 'packages/a/package.json'), JSON.stringify(memberA)) + fs.writeFileSync(path.join(tmpDir, 'packages/b/package.json'), JSON.stringify(memberB)) + try { + const result = await discoverWorkspacePackages(tmpDir, { + workspaceDiscoveryIgnore: ['**/packages/b/**'], + }) + expect(result.some(p => p.endsWith('packages/a/package.json'))).to.be.true + expect(result.some(p => p.endsWith('packages/b/package.json'))).to.be.false + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }) + } + }) + + test('discovers packages from package.json workspaces array', async () => { + const pkgContent = { name: 'root', version: '1.0.0', workspaces: ['packages/*'] } + const memberPkg = { name: 'member', version: '1.0.0' } + const tmpDir = path.join(process.cwd(), 'test/providers/tst_manifests/tmp_workspace_test') + fs.mkdirSync(path.join(tmpDir, 'packages/member'), { recursive: true }) + fs.writeFileSync(path.join(tmpDir, 'package.json'), JSON.stringify(pkgContent)) + fs.writeFileSync(path.join(tmpDir, 'packages/member/package.json'), JSON.stringify(memberPkg)) + try { + const result = await discoverWorkspacePackages(tmpDir) + expect(result).to.be.an('array') + expect(result.length).to.be.at.least(1) + expect(result.some(p => p.endsWith('packages/member/package.json'))).to.be.true + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }) + } + }) + + test('returns empty when package.json has no workspaces (single package)', async () => { + const root = 'test/providers/provider_manifests/npm/with_lock_file' + const result = await discoverWorkspacePackages(root) + expect(result).to.be.an('array') + expect(result).to.have.lengthOf(0) + }) +}) + +suite('resolveWorkspaceDiscoveryIgnore', () => { + test('merges defaults with opts and dedupes', () => { + const r = resolveWorkspaceDiscoveryIgnore({ + workspaceDiscoveryIgnore: ['**/vendor/**', '**/node_modules/**'], + }) + expect(r).to.include('**/node_modules/**') + expect(r).to.include('**/.git/**') + expect(r).to.include('**/vendor/**') + }) +}) + +suite('filterManifestPathsByDiscoveryIgnore', () => { + test('removes paths matching a pattern', () => { + const root = path.resolve('test/providers/tst_manifests') + const paths = [ + path.join(root, 'a/package.json'), + path.join(root, 'node_modules/x/package.json'), + ] + const filtered = filterManifestPathsByDiscoveryIgnore(paths, root, ['**/node_modules/**']) + expect(filtered).to.have.lengthOf(1) + expect(filtered[0]).to.include('a/package.json') + }) +}) + +suite('validatePackageJson', () => { + test('accepts valid name and version', () => { + const tmpDir = path.join(process.cwd(), 'test/providers/tst_manifests/tmp_validate_pkg') + fs.mkdirSync(tmpDir, { recursive: true }) + const p = path.join(tmpDir, 'package.json') + fs.writeFileSync(p, JSON.stringify({ name: 'foo', version: '1.0.0' })) + try { + const r = validatePackageJson(p) + expect(r.valid).to.be.true + if (r.valid) { + expect(r.name).to.equal('foo') + expect(r.version).to.equal('1.0.0') + } + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }) + } + }) + + test('rejects missing name', () => { + const tmpDir = path.join(process.cwd(), 'test/providers/tst_manifests/tmp_validate_pkg2') + fs.mkdirSync(tmpDir, { recursive: true }) + const p = path.join(tmpDir, 'package.json') + fs.writeFileSync(p, JSON.stringify({ version: '1.0.0' })) + try { + const r = validatePackageJson(p) + expect(r.valid).to.be.false + if (!r.valid) { + expect(r.error).to.match(/name/i) + } + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }) + } + }) + + test('rejects missing version', () => { + const tmpDir = path.join(process.cwd(), 'test/providers/tst_manifests/tmp_validate_pkg3') + fs.mkdirSync(tmpDir, { recursive: true }) + const p = path.join(tmpDir, 'package.json') + fs.writeFileSync(p, JSON.stringify({ name: 'foo' })) + try { + const r = validatePackageJson(p) + expect(r.valid).to.be.false + if (!r.valid) { + expect(r.error).to.match(/version/i) + } + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }) + } + }) + + test('rejects invalid JSON', () => { + const tmpDir = path.join(process.cwd(), 'test/providers/tst_manifests/tmp_validate_pkg4') + fs.mkdirSync(tmpDir, { recursive: true }) + const p = path.join(tmpDir, 'package.json') + fs.writeFileSync(p, '{ not json') + try { + const r = validatePackageJson(p) + expect(r.valid).to.be.false + if (!r.valid) { + expect(r.error).to.match(/Invalid package\.json/i) + } + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }) + } + }) +}) + +suite('discoverWorkspaceCrates', () => { + test('returns empty when no Cargo.toml or Cargo.lock at root', async () => { + const result = await discoverWorkspaceCrates('test/providers/tst_manifests/npm') + expect(result).to.be.an('array') + expect(result).to.have.lengthOf(0) + }) + + test('discovers workspace members from Cargo workspace', async () => { + const root = path.resolve('test/providers/tst_manifests/cargo/cargo_virtual_workspace') + const metadata = { + packages: [ + { id: `path+file://${root}/crate-a#0.1.0`, manifest_path: path.join(root, 'crate-a/Cargo.toml') }, + { id: `path+file://${root}/crate-b#0.2.0`, manifest_path: path.join(root, 'crate-b/Cargo.toml') } + ], + workspace_members: [`path+file://${root}/crate-a#0.1.0`, `path+file://${root}/crate-b#0.2.0`] + } + const { discoverWorkspaceCrates } = await esmock('../../src/workspace.js', { + '../../src/tools.js': { + getCustomPath: () => 'cargo', + invokeCommand: () => Buffer.from(JSON.stringify(metadata)) + } + }) + const result = await discoverWorkspaceCrates(root) + expect(result).to.be.an('array') + expect(result).to.have.lengthOf(2) + expect(result.every(p => p.endsWith('Cargo.toml'))).to.be.true + expect(result.some(p => p.includes('crate-a'))).to.be.true + expect(result.some(p => p.includes('crate-b'))).to.be.true + }) +}) diff --git a/test/stack_analysis_batch.test.js b/test/stack_analysis_batch.test.js new file mode 100644 index 00000000..c9bcbc69 --- /dev/null +++ b/test/stack_analysis_batch.test.js @@ -0,0 +1,289 @@ +import fs from 'node:fs' +import path from 'node:path' + +import { expect } from 'chai' +import esmock from 'esmock' +import { http, HttpResponse } from 'msw' +import { setupServer } from 'msw/node' + +const BACKEND_URL = 'http://localhost:9999' +const BACKEND_OPTS = { TRUSTIFY_DA_BACKEND_URL: BACKEND_URL } + +/** + * Helper: create a temporary monorepo workspace on disk. + * Returns the root path and a cleanup function. + */ +function createTmpWorkspace(name, { packages = [], pnpmWorkspace = null, rootPkg = null, lockFile = 'package-lock.json' } = {}) { + const tmpDir = path.resolve(`test/tst_manifests_batch_${name}`) + fs.mkdirSync(tmpDir, { recursive: true }) + + if (rootPkg) { + fs.writeFileSync(path.join(tmpDir, 'package.json'), JSON.stringify(rootPkg)) + } + if (pnpmWorkspace) { + fs.writeFileSync(path.join(tmpDir, 'pnpm-workspace.yaml'), pnpmWorkspace) + } + if (lockFile) { + fs.writeFileSync(path.join(tmpDir, lockFile), '{}') + } + + for (const pkg of packages) { + const pkgDir = path.join(tmpDir, pkg.dir) + fs.mkdirSync(pkgDir, { recursive: true }) + fs.writeFileSync(path.join(pkgDir, 'package.json'), JSON.stringify(pkg.content)) + } + + return { root: tmpDir, cleanup: () => fs.rmSync(tmpDir, { recursive: true, force: true }) } +} + +/** + * Build a mock provider module that returns a deterministic SBOM for each manifest. + */ +function buildMockProviders(sbomMap) { + const mockProvider = { + isSupported: () => true, + validateLockFile: () => true, + provideStack: (manifestPath) => { + const sbom = sbomMap[manifestPath] + if (!sbom) { + throw new Error(`No mock SBOM for ${manifestPath}`) + } + return { + ecosystem: 'npm', + content: JSON.stringify(sbom), + contentType: 'application/vnd.cyclonedx+json', + } + }, + } + return { + availableProviders: [mockProvider], + match: () => mockProvider, + } +} + +function makeSbom(name, version) { + return { + metadata: { + component: { + purl: `pkg:npm/${name}@${version}`, + 'bom-ref': `pkg:npm/${name}@${version}`, + }, + }, + components: [], + } +} + +suite('stackAnalysisBatch', () => { + let server + let capturedBody + + suiteSetup(() => { + server = setupServer( + http.post(`${BACKEND_URL}/api/v5/batch-analysis`, async ({ request }) => { + capturedBody = await request.json() + const report = {} + for (const purl of Object.keys(capturedBody)) { + report[purl] = { providers: {} } + } + return HttpResponse.json(report) + }) + ) + server.listen() + }) + + suiteTeardown(() => { + server.close() + }) + + setup(() => { + capturedBody = null + }) + + test('discovers JS workspace packages, generates SBOMs, and sends batch request', async () => { + const { root, cleanup } = createTmpWorkspace('js_batch', { + rootPkg: { name: 'root', version: '1.0.0', workspaces: ['packages/*'] }, + packages: [ + { dir: 'packages/app-a', content: { name: 'app-a', version: '1.0.0' } }, + { dir: 'packages/app-b', content: { name: 'app-b', version: '2.0.0' } }, + ], + }) + + try { + const appASbom = makeSbom('app-a', '1.0.0') + const appBSbom = makeSbom('app-b', '2.0.0') + const rootSbom = makeSbom('root', '1.0.0') + + const sbomMap = {} + sbomMap[path.join(root, 'package.json')] = rootSbom + sbomMap[path.join(root, 'packages/app-a/package.json')] = appASbom + sbomMap[path.join(root, 'packages/app-b/package.json')] = appBSbom + + const mockProviders = buildMockProviders(sbomMap) + const client = await esmock('../src/index.js', { + '../src/provider.js': mockProviders, + }) + + const result = await client.stackAnalysisBatch(root, false, BACKEND_OPTS) + + expect(result).to.be.an('object') + expect(capturedBody).to.be.an('object') + const purls = Object.keys(capturedBody) + expect(purls).to.include('pkg:npm/app-a@1.0.0') + expect(purls).to.include('pkg:npm/app-b@2.0.0') + expect(purls).to.include('pkg:npm/root@1.0.0') + } finally { + cleanup() + } + }) + + test('returns metadata when batchMetadata option is set', async () => { + const { root, cleanup } = createTmpWorkspace('js_meta', { + rootPkg: { name: 'meta-root', version: '1.0.0', workspaces: ['packages/*'] }, + packages: [ + { dir: 'packages/pkg-a', content: { name: 'pkg-a', version: '1.0.0' } }, + ], + }) + + try { + const sbomMap = {} + sbomMap[path.join(root, 'package.json')] = makeSbom('meta-root', '1.0.0') + sbomMap[path.join(root, 'packages/pkg-a/package.json')] = makeSbom('pkg-a', '1.0.0') + + const mockProviders = buildMockProviders(sbomMap) + const client = await esmock('../src/index.js', { + '../src/provider.js': mockProviders, + }) + + const result = await client.stackAnalysisBatch(root, false, { + ...BACKEND_OPTS, + batchMetadata: true, + }) + + expect(result).to.have.property('analysis') + expect(result).to.have.property('metadata') + expect(result.metadata.ecosystem).to.equal('javascript') + expect(result.metadata.successful).to.equal(2) + expect(result.metadata.failed).to.equal(0) + } finally { + cleanup() + } + }) + + test('skips invalid package.json and continues in default mode', async () => { + const { root, cleanup } = createTmpWorkspace('js_invalid', { + rootPkg: { name: 'root-inv', version: '1.0.0', workspaces: ['packages/*'] }, + packages: [ + { dir: 'packages/good', content: { name: 'good', version: '1.0.0' } }, + { dir: 'packages/bad', content: { version: '1.0.0' } }, // missing name + ], + }) + + try { + const sbomMap = {} + sbomMap[path.join(root, 'package.json')] = makeSbom('root-inv', '1.0.0') + sbomMap[path.join(root, 'packages/good/package.json')] = makeSbom('good', '1.0.0') + + const mockProviders = buildMockProviders(sbomMap) + const client = await esmock('../src/index.js', { + '../src/provider.js': mockProviders, + }) + + const result = await client.stackAnalysisBatch(root, false, { + ...BACKEND_OPTS, + batchMetadata: true, + }) + + expect(result.metadata.failed).to.be.at.least(1) + expect(result.metadata.errors.some(e => e.phase === 'validation')).to.be.true + expect(capturedBody).to.be.an('object') + expect(Object.keys(capturedBody)).to.include('pkg:npm/good@1.0.0') + } finally { + cleanup() + } + }) + + test('throws on first invalid package.json in fail-fast mode', async () => { + const { root, cleanup } = createTmpWorkspace('js_failfast', { + rootPkg: { name: 'root-ff', version: '1.0.0', workspaces: ['packages/*'] }, + packages: [ + { dir: 'packages/bad', content: { version: '1.0.0' } }, // missing name + { dir: 'packages/good', content: { name: 'good', version: '1.0.0' } }, + ], + }) + + try { + const sbomMap = {} + sbomMap[path.join(root, 'package.json')] = makeSbom('root-ff', '1.0.0') + sbomMap[path.join(root, 'packages/good/package.json')] = makeSbom('good', '1.0.0') + + const mockProviders = buildMockProviders(sbomMap) + const client = await esmock('../src/index.js', { + '../src/provider.js': mockProviders, + }) + + try { + await client.stackAnalysisBatch(root, false, { + ...BACKEND_OPTS, + continueOnError: false, + batchMetadata: true, + }) + expect.fail('should have thrown') + } catch (err) { + expect(err.message).to.match(/Invalid package\.json/i) + expect(err.batchMetadata).to.be.an('object') + } + } finally { + cleanup() + } + }) + + test('throws when no workspace manifests found', async () => { + const tmpDir = path.resolve('test/tst_manifests_batch_empty') + fs.mkdirSync(tmpDir, { recursive: true }) + + try { + const client = await esmock('../src/index.js', { + '../src/provider.js': buildMockProviders({}), + }) + + try { + await client.stackAnalysisBatch(tmpDir, false, BACKEND_OPTS) + expect.fail('should have thrown') + } catch (err) { + expect(err.message).to.match(/No workspace manifests found/) + } + } finally { + fs.rmSync(tmpDir, { recursive: true, force: true }) + } + }) + + test('discovers pnpm workspace packages', async () => { + const pnpmYaml = `packages:\n - 'apps/*'\n` + const { root, cleanup } = createTmpWorkspace('pnpm_batch', { + rootPkg: { name: 'pnpm-root', version: '1.0.0' }, + pnpmWorkspace: pnpmYaml, + lockFile: 'pnpm-lock.yaml', + packages: [ + { dir: 'apps/web', content: { name: 'web', version: '1.0.0' } }, + ], + }) + + try { + const sbomMap = {} + sbomMap[path.join(root, 'apps/web/package.json')] = makeSbom('web', '1.0.0') + + const mockProviders = buildMockProviders(sbomMap) + const client = await esmock('../src/index.js', { + '../src/provider.js': mockProviders, + }) + + const result = await client.stackAnalysisBatch(root, false, BACKEND_OPTS) + + expect(result).to.be.an('object') + expect(capturedBody).to.be.an('object') + expect(Object.keys(capturedBody)).to.include('pkg:npm/web@1.0.0') + } finally { + cleanup() + } + }) +})