diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 6d95dfeda7..ecc7380028 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -149,6 +149,7 @@ /integration-tests/mocha/ @DataDog/ci-app-libraries /integration-tests/playwright/ @DataDog/ci-app-libraries /integration-tests/vitest/ @DataDog/ci-app-libraries +/integration-tests/webdriverio/ @DataDog/ci-app-libraries /integration-tests/vitest.config.mjs @DataDog/ci-app-libraries /integration-tests/vitest.second-project.config.mjs @DataDog/ci-app-libraries /integration-tests/vitest.typecheck.config.mjs @DataDog/ci-app-libraries @@ -279,7 +280,10 @@ /scripts/generate-config-types.js @DataDog/apm-sdk-capabilities-js /scripts/generate-supported-integrations.js @DataDog/apm-sdk-capabilities-js +/scripts/helpers/test-file-index.js @DataDog/apm-sdk-capabilities-js +/scripts/helpers/test-file-index.spec.js @DataDog/apm-sdk-capabilities-js /scripts/verify-exercised-tests.js @DataDog/apm-sdk-capabilities-js +/scripts/verify-exercised-tests.spec.mjs @DataDog/apm-sdk-capabilities-js /supported_versions_output.json @DataDog/apm-sdk-capabilities-js /supported_versions_table.csv @DataDog/apm-sdk-capabilities-js diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 3c71b45f8d..631bf33d3e 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -45,7 +45,7 @@ jobs: - name: Initialize CodeQL id: init-codeql - uses: github/codeql-action/init@7188fc363630916deb702c7fdcf4e481b751f97a # v4.37.1 + uses: github/codeql-action/init@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3 with: languages: ${{ matrix.language }} config-file: .github/codeql_config.yml @@ -57,7 +57,7 @@ jobs: - name: Perform CodeQL Analysis id: analyze - uses: github/codeql-action/analyze@7188fc363630916deb702c7fdcf4e481b751f97a # v4.37.1 + uses: github/codeql-action/analyze@e4fba868fa4b1b91e1fdab776edc8cfbe6e9fb81 # v4.37.3 with: token: ${{ github.token }} wait-for-processing: false diff --git a/.github/workflows/llmobs.yml b/.github/workflows/llmobs.yml index 370a87db33..d24c232d04 100644 --- a/.github/workflows/llmobs.yml +++ b/.github/workflows/llmobs.yml @@ -311,7 +311,7 @@ jobs: env: PLUGINS: openai-agents steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: ./.github/actions/testagent/start - uses: ./.github/actions/node/oldest-maintenance-lts - uses: ./.github/actions/install diff --git a/.github/workflows/mirror-image.yml b/.github/workflows/mirror-image.yml index 7a68f56962..5454d6167d 100644 --- a/.github/workflows/mirror-image.yml +++ b/.github/workflows/mirror-image.yml @@ -28,7 +28,7 @@ jobs: policy: self.mirror-image - name: Log in to GHCR - uses: docker/login-action@v4 + uses: docker/login-action@v4.5.1 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/serverless.yml b/.github/workflows/serverless.yml index 63831a2223..6bc8ecc696 100644 --- a/.github/workflows/serverless.yml +++ b/.github/workflows/serverless.yml @@ -52,6 +52,7 @@ jobs: spec: - client - aws-sdk + - base-inject-field - bedrockruntime - dynamodb - eventbridge diff --git a/.github/workflows/test-optimization.yml b/.github/workflows/test-optimization.yml index 3ce3678890..6d1dce1bcb 100644 --- a/.github/workflows/test-optimization.yml +++ b/.github/workflows/test-optimization.yml @@ -56,7 +56,11 @@ jobs: - name: Get Playwright version from versions package.json id: playwright-version run: | - PLAYWRIGHT_VERSION=$(node -p "require('./packages/dd-trace/test/plugins/versions/package.json').dependencies['@playwright/test']") + PLAYWRIGHT_VERSION=$(node -p \ + "const versions = require('./integration-tests/playwright/versions'); \ + versions.getLatestPlaywrightSpecifier() === 'latest' \ + ? versions.latest \ + : versions.latestSupportedByNode18") echo "version=$PLAYWRIGHT_VERSION" >> $GITHUB_OUTPUT echo "Playwright version: $PLAYWRIGHT_VERSION" - name: Cache Playwright browsers @@ -73,14 +77,14 @@ jobs: strategy: fail-fast: false matrix: - playwright-version: [oldest, latest] + playwright-version: [oldest, latest, latest-node18] name: Ensure Playwright Docker image (${{ matrix.playwright-version }}) runs-on: ubuntu-latest permissions: contents: read packages: write outputs: - # Both matrix jobs set this to the same value, so the last-writer-wins + # All matrix jobs set this to the same value, so the last-writer-wins # behaviour of matrix outputs is safe here. images: ${{ steps.versions.outputs.images }} steps: @@ -89,19 +93,35 @@ jobs: id: versions run: | LATEST=$(node -p "require('./integration-tests/playwright/versions').latest") + LATEST_NODE18=$(node -p "require('./integration-tests/playwright/versions').latestSupportedByNode18") OLDEST=$(node -p "require('./integration-tests/playwright/versions').oldest") DOCKER_HASH=$(sha256sum .github/playwright/Dockerfile | cut -c1-8) LATEST_TAG="${LATEST}-${DOCKER_HASH}" + LATEST_NODE18_TAG="${LATEST_NODE18}-${DOCKER_HASH}" OLDEST_TAG="${OLDEST}-${DOCKER_HASH}" BASE="ghcr.io/datadog/dd-trace-js/playwright-tools" - IMAGES=$(printf '{"latest":"%s:%s","oldest":"%s:%s"}' "$BASE" "$LATEST_TAG" "$BASE" "$OLDEST_TAG") - PW_VERSION=$([ "${{ matrix.playwright-version }}" = "latest" ] && echo "$LATEST" || echo "$OLDEST") - IMAGE_TAG=$([ "${{ matrix.playwright-version }}" = "latest" ] && echo "$LATEST_TAG" || echo "$OLDEST_TAG") + IMAGES=$(printf \ + '{"latest":{"latest":"%s:%s","oldest":"%s:%s"},"oldest":{"latest":"%s:%s","oldest":"%s:%s"}}' \ + "$BASE" "$LATEST_TAG" "$BASE" "$OLDEST_TAG" "$BASE" "$LATEST_NODE18_TAG" "$BASE" "$OLDEST_TAG") + case "${{ matrix.playwright-version }}" in + latest) + PW_VERSION="$LATEST" + IMAGE_TAG="$LATEST_TAG" + ;; + latest-node18) + PW_VERSION="$LATEST_NODE18" + IMAGE_TAG="$LATEST_NODE18_TAG" + ;; + *) + PW_VERSION="$OLDEST" + IMAGE_TAG="$OLDEST_TAG" + ;; + esac echo "images=$IMAGES" >> $GITHUB_OUTPUT echo "pw-version=$PW_VERSION" >> $GITHUB_OUTPUT echo "image-tag=$IMAGE_TAG" >> $GITHUB_OUTPUT - name: Log in to GHCR - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1 with: registry: ghcr.io username: ${{ github.actor }} @@ -140,7 +160,7 @@ jobs: permissions: id-token: write container: - image: ${{ fromJson(needs.playwright-image.outputs.images)[matrix.playwright-version] }} + image: ${{ fromJson(needs.playwright-image.outputs.images)[matrix.node-version][matrix.playwright-version] }} credentials: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} @@ -202,6 +222,33 @@ jobs: with: flags: test-optimization-mocha-${{ matrix.version }}-${{ matrix.mocha-version }} + integration-webdriverio: + strategy: + fail-fast: false + matrix: + include: + - node-version: oldest + webdriverio-version: oldest + - node-version: latest + webdriverio-version: latest + name: integration-webdriverio (${{ matrix.webdriverio-version }}, node-${{ matrix.node-version }}) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: ./.github/actions/node + with: + version: ${{ matrix.node-version }} + - uses: ./.github/actions/install + - run: npm run test:instrumentations + env: + PLUGINS: webdriverio + - run: npm run test:integration:webdriverio:coverage + env: + WEBDRIVERIO_VERSION: ${{ matrix.webdriverio-version }} + - uses: ./.github/actions/coverage + with: + flags: test-optimization-webdriverio-${{ matrix.node-version }}-${{ matrix.webdriverio-version }} + integration-jest: strategy: fail-fast: false @@ -252,6 +299,14 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: ./.github/actions/plugins/test + dependency-helpers: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: ./.github/actions/node + - uses: ./.github/actions/install + - run: npm run test:integration:test-optimization:helpers + integration-cucumber: strategy: fail-fast: false @@ -299,7 +354,7 @@ jobs: IMAGE="ghcr.io/datadog/dd-trace-js/selenium-tools:${DOCKER_HASH}" echo "image=$IMAGE" >> $GITHUB_OUTPUT - name: Log in to GHCR - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + uses: docker/login-action@abd2ef45e78c5afb21d64d4ca52ee8550d9572c7 # v4.5.1 with: registry: ghcr.io username: ${{ github.actor }} diff --git a/.github/workflows/update-3rdparty-licenses.yml b/.github/workflows/update-3rdparty-licenses.yml index c987158786..dd2399eb7a 100644 --- a/.github/workflows/update-3rdparty-licenses.yml +++ b/.github/workflows/update-3rdparty-licenses.yml @@ -32,7 +32,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Set up Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.14" diff --git a/ci/test-optimization-validation/generated-files.js b/ci/test-optimization-validation/generated-files.js index 1433416e8f..a8d0e5af6b 100644 --- a/ci/test-optimization-validation/generated-files.js +++ b/ci/test-optimization-validation/generated-files.js @@ -132,7 +132,7 @@ function getMissingDirectories (root, directory) { function cleanupCreatedDirectories (root) { const outcome = { removed: 0, retained: 0 } const resolvedRoot = path.resolve(root) - const directories = [...createdGeneratedDirectories.entries()] + const directories = [...createdGeneratedDirectories] .filter(([directory]) => isPathInside(resolvedRoot, directory)) .sort(([left], [right]) => right.length - left.length) diff --git a/ci/test-optimization-validation/generated-test-contract.js b/ci/test-optimization-validation/generated-test-contract.js index 8db9b1400c..da6fcaf405 100644 --- a/ci/test-optimization-validation/generated-test-contract.js +++ b/ci/test-optimization-validation/generated-test-contract.js @@ -192,9 +192,7 @@ function getGeneratedTestContractError (framework) { if (!cleanupPaths.has(path.normalize(stepsFile))) { return 'must include the isolated Cucumber step definitions in cleanupPaths.' } - } - - if (framework.framework === 'playwright') { + } else if (framework.framework === 'playwright') { const configPath = playwrightAdapter.getGeneratedConfigPath(strategy.testDirectory) const config = files.find(file => path.normalize(file.path) === path.normalize(configPath)) if (config?.contentLines?.join('\n') !== playwrightAdapter.getGeneratedConfigContent()) { diff --git a/ci/test-optimization-validation/manifest-scaffold.js b/ci/test-optimization-validation/manifest-scaffold.js index 272f6b7ac8..33cbdc4aa3 100644 --- a/ci/test-optimization-validation/manifest-scaffold.js +++ b/ci/test-optimization-validation/manifest-scaffold.js @@ -477,8 +477,7 @@ function buildGeneratedTestStrategy ({ framework, projectRoot, representative }) const stepsFile = cucumber.getGeneratedStepsPath(testDirectory) files.push({ path: stepsFile, contentLines: cucumber.getGeneratedStepsContent().split('\n') }) cleanupPaths.push(stepsFile) - } - if (framework === 'playwright') { + } else if (framework === 'playwright') { const configFile = playwright.getGeneratedConfigPath(testDirectory) files.push({ path: configFile, contentLines: playwright.getGeneratedConfigContent().split('\n') }) cleanupPaths.push(configFile) diff --git a/ci/test-optimization-validation/runner-command.js b/ci/test-optimization-validation/runner-command.js index 55517766c0..5e29341762 100644 --- a/ci/test-optimization-validation/runner-command.js +++ b/ci/test-optimization-validation/runner-command.js @@ -211,7 +211,7 @@ function getRunnerArgs (framework, testFile, generated) { */ function getRunnerEnv (framework) { return { - ...(framework.validation.environment || {}), + ...framework.validation.environment, ...(framework.framework === 'cucumber' ? { CUCUMBER_PUBLISH_QUIET: 'true' } : {}), } } diff --git a/ci/test-optimization-validation/scenarios/basic-reporting.js b/ci/test-optimization-validation/scenarios/basic-reporting.js index fa19cfccf3..fabe2a6566 100644 --- a/ci/test-optimization-validation/scenarios/basic-reporting.js +++ b/ci/test-optimization-validation/scenarios/basic-reporting.js @@ -58,7 +58,7 @@ async function runBasicReporting ({ framework, out, options }) { selector, settingsLoadedFromCache: run.offline.inputs.settings?.status === 'loaded', } - evidence.foundationalReportingEstablished = evidence.foundationalReportingEstablished && selector.verified + evidence.foundationalReportingEstablished &&= selector.verified if (run.result.timedOut) { return inconclusive( diff --git a/ci/test-optimization-validation/scenarios/test-management.js b/ci/test-optimization-validation/scenarios/test-management.js index d2bdde91d8..8c98ee8452 100644 --- a/ci/test-optimization-validation/scenarios/test-management.js +++ b/ci/test-optimization-validation/scenarios/test-management.js @@ -149,7 +149,7 @@ function buildQuarantinedResponse (framework, scenario, discoveredIdentities = [ for (const identity of identities) { for (const suite of getSuiteCandidates(identity, scenario)) { for (const name of getNameCandidates(identity)) { - suites[suite] = suites[suite] || { tests: {} } + suites[suite] ||= { tests: {} } suites[suite].tests[name] = { properties: { quarantined: true, @@ -210,7 +210,7 @@ function summarizeManagedTests (testManagementTests) { } summary.set(displaySuite, testNames) } - return [...summary.entries()].slice(0, 5).map(([suite, tests]) => ({ + return [...summary].slice(0, 5).map(([suite, tests]) => ({ suite, tests: [...tests].slice(0, 5), })) diff --git a/eslint.config.mjs b/eslint.config.mjs index 4a869ca1b0..af77ab3c79 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -586,60 +586,57 @@ export default [ ...eslintPluginUnicorn.configs.recommended.rules, // Overriding recommended unicorn rules. - // Rules not listed here are left at the `recommended` default. The v65→v68 bump - // turned ~130 rules on in `recommended`; the entries below are the ones that fire - // on our source and are intentionally kept off (counts are from the v68 run). - 'unicorn/catch-error-name': ['off', { name: 'err' }], // Many errors + // Rules not listed here are left at the `recommended` default. The entries below + // document deliberate exceptions. Volume markers stay coarse so they do not drift: + // `few` is under ten sites, `many` is tens, `lots` is hundreds or more. + 'unicorn/catch-error-name': ['off', { name: 'err' }], // lots 'unicorn/expiring-todo-comments': 'off', - 'unicorn/filename-case': ['off', { case: 'kebabCase' }], // Many errors - 'unicorn/name-replacements': 'off', // Many errors | naming churn (split out of prevent-abbreviations) - 'unicorn/prevent-abbreviations': 'off', // Many errors + 'unicorn/filename-case': ['off', { case: 'kebabCase' }], // lots + 'unicorn/name-replacements': 'off', // lots | naming churn (split out of prevent-abbreviations) + 'unicorn/prevent-abbreviations': 'off', // Its replacements moved to name-replacements // These rules require a newer Node.js version than we support 'unicorn/no-array-reverse': 'off', // Node.js 20 'unicorn/no-array-sort': 'off', // Node.js 20 + 'unicorn/prefer-abort-signal-any': 'off', // Node.js 18.17 'unicorn/prefer-dispose': 'off', // Explicit resource management (newer Node.js) + 'unicorn/prefer-group-by': 'off', // Node.js 21 + 'unicorn/prefer-iterator-helpers': 'off', // Iterator helpers (Node.js 22) 'unicorn/prefer-iterator-to-array': 'off', // Iterator helpers (Node.js 22) 'unicorn/prefer-iterator-to-array-at-end': 'off', // Iterator helpers (Node.js 22) - 'unicorn/prefer-promise-with-resolvers': 'off', // 6 errors | Promise.withResolvers (Node.js 22) + 'unicorn/prefer-promise-try': 'off', // Promise.try (Node.js 24) + 'unicorn/prefer-promise-with-resolvers': 'off', // few | Promise.withResolvers (Node.js 22) + 'unicorn/prefer-set-methods': 'off', // Set methods (Node.js 22) 'unicorn/prefer-temporal': 'off', // Temporal is not stable on supported Node.js 'unicorn/prefer-uint8array-base64': 'off', // Uint8Array base64 (Node.js 22) // These rules could potentially be evaluated again at a much later point - 'unicorn/class-reference-in-static-methods': 'off', // 6 errors - 'unicorn/consistent-class-member-order': 'off', // 55 errors | ordering churn - 'unicorn/consistent-conditional-object-spread': 'off', // 3 errors - 'unicorn/consistent-optional-chaining': 'off', // 3 errors + 'unicorn/class-reference-in-static-methods': 'off', // few + 'unicorn/consistent-class-member-order': 'off', // many | ordering churn + 'unicorn/consistent-conditional-object-spread': 'off', // few 'unicorn/explicit-length-check': 'off', // Not a big advantage 'unicorn/explicit-timer-delay': 'off', // Covered by our own timer lint rules 'unicorn/no-array-callback-reference': 'off', - 'unicorn/no-computed-property-existence-check': 'off', // 160 errors | needs an audit - 'unicorn/no-declarations-before-early-exit': 'off', // 62 errors - 'unicorn/no-duplicate-if-branches': 'off', // 1 error | may surface real bugs - 'unicorn/no-duplicate-logical-operands': 'off', // 2 errors | may surface real bugs - 'unicorn/no-error-property-assignment': 'off', // 6 errors + 'unicorn/no-computed-property-existence-check': 'off', // lots | needs an audit + 'unicorn/no-declarations-before-early-exit': 'off', // many + 'unicorn/no-error-property-assignment': 'off', // few | all preserve upstream error metadata 'unicorn/no-for-loop': 'off', // Activate if this is resolved https://github.com/sindresorhus/eslint-plugin-unicorn/issues/2664 - 'unicorn/no-incorrect-template-string-interpolation': 'off', // 3 errors | audit for real bugs first - 'unicorn/no-invalid-argument-count': 'off', // 98 errors | high false-positive risk, worth a focused pass - 'unicorn/no-loop-iterable-mutation': 'off', // 2 errors | may surface real bugs - 'unicorn/no-nonstandard-builtin-properties': 'off', // 34 errors | needs an audit + 'unicorn/no-nonstandard-builtin-properties': 'off', // many | needs an audit 'unicorn/no-this-assignment': 'off', // This would need some further refactoring and the benefit is small - 'unicorn/no-undeclared-class-members': 'off', // 272 errors | requires declaring every field - 'unicorn/no-unreadable-array-destructuring': 'off', // 4 errors | not autofixable, needs manual rewrite - 'unicorn/no-unreadable-for-of-expression': 'off', // 32 errors - 'unicorn/no-unreadable-object-destructuring': 'off', // 57 errors - 'unicorn/no-unsafe-string-replacement': 'off', // 6 errors - 'unicorn/no-useless-recursion': 'off', // 2 errors + 'unicorn/no-undeclared-class-members': 'off', // lots | requires declaring every field + 'unicorn/no-unreadable-array-destructuring': 'off', // few | not autofixable, needs manual rewrite + 'unicorn/no-unreadable-for-of-expression': 'off', // many + 'unicorn/no-unreadable-object-destructuring': 'off', // many + 'unicorn/no-unsafe-string-replacement': 'off', // many | replacement callbacks reduce readability + 'unicorn/no-useless-recursion': 'off', // few | iterative rewrites add substantial nesting 'unicorn/prefer-code-point': 'off', // Should be activated, but needs a refactor of some code - 'unicorn/prefer-early-return': 'off', // 67 errors | tension with our positive-`if` style - 'unicorn/prefer-hoisting-branch-code': 'off', // 2 errors | reshapes branch bodies - 'unicorn/prefer-minimal-ternary': 'off', // 24 errors - 'unicorn/prefer-number-is-safe-integer': 'off', // 17 errors - 'unicorn/prefer-object-iterable-methods': 'off', // 56 errors + 'unicorn/prefer-early-return': 'off', // many | tension with our positive-`if` style + 'unicorn/prefer-number-is-safe-integer': 'off', // many + 'unicorn/prefer-object-iterable-methods': 'off', // many 'unicorn/prefer-queue-microtask': 'off', // process.nextTick semantics differ - 'unicorn/prefer-smaller-scope': 'off', // 3 errors - 'unicorn/prefer-split-limit': 'off', // 23 errors - 'unicorn/require-array-sort-compare': 'off', // 8 errors | may surface real default-sort bugs + 'unicorn/prefer-simple-condition-first': 'off', // lots | needs a short-circuit behavior audit + 'unicorn/prefer-then-catch': 'off', // many | broadens rejection boundaries + 'unicorn/require-array-sort-compare': 'off', // many | many intentional lexicographic sorts // The following rules should not be activated! 'unicorn/consistent-boolean-name': 'off', // Would rename public API and config booleans @@ -649,6 +646,8 @@ export default [ 'unicorn/no-array-splice': 'off', // toSpliced copies the whole array (perf) 'unicorn/no-break-in-nested-loop': 'off', // Conflicts with our performance-oriented loops 'unicorn/no-global-object-property-assignment': 'off', // We use globalThis[Symbol.for('dd-trace')] + 'unicorn/no-negated-array-predicate': 'off', // Predicate inversion is harder to read and creates churn + 'unicorn/no-negated-comparison': 'off', // Opposite comparisons do not preserve NaN handling 'unicorn/no-nested-ternary': 'off', // Not really an issue in the code and the benefit is small 'unicorn/no-new-array': 'off', // new Array is often used for performance reasons 'unicorn/no-null': 'off', // We do not control external APIs and it is hard to differentiate these @@ -659,47 +658,32 @@ export default [ 'unicorn/operator-assignment': 'off', // Covered by core operator-assignment 'unicorn/prefer-array-last-methods': 'off', // Questionable benefit 'unicorn/prefer-await': 'off', // We avoid async/await in production hot paths + 'unicorn/prefer-dom-node-html-methods': 'off', // Browser compatibility and different serialization semantics 'unicorn/prefer-event-target': 'off', // Benefit only outside of Node.js 'unicorn/prefer-global-this': 'off', // Questionable benefit in Node.js alone 'unicorn/prefer-includes-over-repeated-comparisons': 'off', // Bad for performance 'unicorn/prefer-math-trunc': 'off', // Math.trunc is not a 1-to-1 replacement for most of our usage + 'unicorn/prefer-minimal-ternary': 'off', // Conflicts with our restricted-syntax rule on require(cond ? a : b) 'unicorn/prefer-module': 'off', // We use CJS 'unicorn/prefer-node-protocol': 'off', // May not be used due to guardrails 'unicorn/prefer-number-coercion': 'off', // Number() is not a 1-to-1 replacement for parseInt/parseFloat 'unicorn/prefer-private-class-fields': 'off', // Many `_underscore` fields cross module boundaries - 'unicorn/prefer-reflect-apply': 'off', // Questionable benefit and more than 500 matches + 'unicorn/prefer-reflect-apply': 'off', // lots | questionable benefit 'unicorn/prefer-short-arrow-method': 'off', // Method shorthand is intentional; arrow properties change `this` + 'unicorn/prefer-split-limit': 'off', // A limit is slower than getSegment; the rest read every segment 'unicorn/prefer-switch': 'off', // Questionable benefit 'unicorn/prefer-top-level-await': 'off', // Only useful when using ESM 'unicorn/prefer-unicode-code-point-escapes': 'off', // Replaces the dropped no-hex-escape; questionable benefit 'unicorn/switch-case-braces': 'off', // Questionable benefit - // Safe to enable in a follow-up: autofixable and aligned with our style. Kept off - // here only to keep this version bump free of source churn (counts from the v68 run). - 'unicorn/logical-assignment-operators': 'off', // 42 errors | matches our ??=/||= usage - 'unicorn/no-confusing-array-splice': 'off', // 1 error - 'unicorn/no-for-each': 'off', // 10 errors | we already prefer for-of in production - 'unicorn/no-negated-array-predicate': 'off', // 2 errors - 'unicorn/no-negated-comparison': 'off', // 1 error - 'unicorn/no-subtraction-comparison': 'off', // 2 errors - 'unicorn/no-unnecessary-boolean-comparison': 'off', // 6 errors - 'unicorn/no-unnecessary-global-this': 'off', // 4 errors - 'unicorn/no-unnecessary-splice': 'off', // 2 errors - 'unicorn/no-useless-concat': 'off', // 4 errors - 'unicorn/no-useless-continue': 'off', // 1 error - 'unicorn/no-useless-delete-check': 'off', // 1 error - 'unicorn/no-useless-fallback-in-spread': 'off', // 5 errors - 'unicorn/no-useless-override': 'off', // 1 error - 'unicorn/no-useless-template-literals': 'off', // 13 errors - 'unicorn/prefer-array-from-map': 'off', // 6 errors - 'unicorn/prefer-boolean-return': 'off', // 1 error - 'unicorn/prefer-continue': 'off', // 52 errors - 'unicorn/prefer-direct-iteration': 'off', // 5 errors - 'unicorn/prefer-else-if': 'off', // 6 errors - 'unicorn/prefer-global-number-constants': 'off', // 3 errors - 'unicorn/prefer-logical-operator-over-ternary': 'off', // 3 errors - 'unicorn/prefer-ternary': 'off', // 16 errors - 'unicorn/prefer-unary-minus': 'off', // 1 error + // These remaining rules need focused rewrites before activation. + 'unicorn/no-confusing-array-splice': 'off', // few + 'unicorn/no-for-each': 'off', // many | we already prefer for-of in production + 'unicorn/no-unnecessary-global-this': 'off', // few | explicit globals are clearer + 'unicorn/no-useless-continue': 'off', // few + 'unicorn/prefer-array-from-map': 'off', // few | loops avoid callback allocation + 'unicorn/prefer-continue': 'off', // many + 'unicorn/prefer-ternary': 'off', // many }, }, { diff --git a/index.d.ts b/index.d.ts index 6f703250bd..0048417a75 100644 --- a/index.d.ts +++ b/index.d.ts @@ -3734,16 +3734,36 @@ declare namespace tracer { flush (): void } + /** JSON-serializable value accepted by LLMObs Experiments. */ + type JSONType = string | number | boolean | null | JSONType[] | { [key: string]: JSONType } + /** * A task run over each dataset record during an experiment. */ - type ExperimentTask = (input: any, config: Record) => any | Promise + type ExperimentTask = ( + input: JSONType, + config: Record + ) => JSONType | Promise /** * Scores a single task output. The return type selects the metric: * `boolean` -> boolean, `number` -> score, anything else -> categorical. */ - type ExperimentEvaluator = (input: any, output: any, expectedOutput: any) => any | Promise + type ExperimentEvaluator = ( + input: JSONType, + output: JSONType, + expectedOutput: JSONType + ) => JSONType | Promise + + interface CreateDatasetOptions { + description?: string + records?: Array<{ + id?: string, + inputData: JSONType, + expectedOutput?: JSONType, + metadata?: Record + }> + } interface ExperimentOptions { name: string @@ -3752,11 +3772,13 @@ declare namespace tracer { /** Evaluators keyed by metric label. */ evaluators?: Record description?: string - config?: Record + config?: Record tags?: Record } interface PullDatasetOptions { + /** Dataset version to pull. Defaults to latest. */ + version?: number /** Wait until at least this many records are readable (absorbs write lag). */ expectedRecordCount?: number /** Maximum total time to wait, in ms. Default 30000. */ @@ -3769,13 +3791,13 @@ declare namespace tracer { traceId: string startNs: number durationNs: number - input: any - output: any - expectedOutput: any + input: JSONType + output: JSONType + expectedOutput: JSONType readonly isError: boolean errorType: string | null errorMessage: string | null - evaluations: Record + evaluations: Record evaluationErrors: Record } @@ -3794,13 +3816,20 @@ declare namespace tracer { } interface Dataset { - addRecord (input: any, expectedOutput?: any, metadata?: Record): Dataset + addRecord (input: JSONType, expectedOutput?: JSONType, metadata?: Record): Dataset /** Creates the dataset remotely if needed and pushes any unpushed records. */ push (): Promise name (): string id (): string | null projectId (): string | null - records (): Array<{ input: any, expectedOutput: any, metadata: Record }> + version (): number | null + latestVersion (): number | null + records (): Array<{ + id: string | null, + input: JSONType, + expectedOutput: JSONType, + metadata: Record + }> /** Dashboard URL for the dataset, or null until pushed. */ url (): string | null } @@ -3815,6 +3844,7 @@ declare namespace tracer { interface Experiments { /** Create a local dataset buffer; pushed on the first experiment run. */ createDataset (name: string, description?: string): Dataset + createDataset (name: string, options?: CreateDatasetOptions): Dataset /** Pull an existing dataset (with records) by name. */ pullDataset (name: string, options?: PullDatasetOptions): Promise /** Build an experiment to run over a dataset. */ diff --git a/index.d.v5.ts b/index.d.v5.ts index 14a7680153..b5041d9116 100644 --- a/index.d.v5.ts +++ b/index.d.v5.ts @@ -3931,16 +3931,36 @@ declare namespace tracer { flush (): void } + /** JSON-serializable value accepted by LLMObs Experiments. */ + type JSONType = string | number | boolean | null | JSONType[] | { [key: string]: JSONType } + /** * A task run over each dataset record during an experiment. */ - type ExperimentTask = (input: any, config: Record) => any | Promise + type ExperimentTask = ( + input: JSONType, + config: Record + ) => JSONType | Promise /** * Scores a single task output. The return type selects the metric: * `boolean` -> boolean, `number` -> score, anything else -> categorical. */ - type ExperimentEvaluator = (input: any, output: any, expectedOutput: any) => any | Promise + type ExperimentEvaluator = ( + input: JSONType, + output: JSONType, + expectedOutput: JSONType + ) => JSONType | Promise + + interface CreateDatasetOptions { + description?: string + records?: Array<{ + id?: string, + inputData: JSONType, + expectedOutput?: JSONType, + metadata?: Record + }> + } interface ExperimentOptions { name: string @@ -3949,11 +3969,13 @@ declare namespace tracer { /** Evaluators keyed by metric label. */ evaluators?: Record description?: string - config?: Record + config?: Record tags?: Record } interface PullDatasetOptions { + /** Dataset version to pull. Defaults to latest. */ + version?: number /** Wait until at least this many records are readable (absorbs write lag). */ expectedRecordCount?: number /** Maximum total time to wait, in ms. Default 30000. */ @@ -3966,13 +3988,13 @@ declare namespace tracer { traceId: string startNs: number durationNs: number - input: any - output: any - expectedOutput: any + input: JSONType + output: JSONType + expectedOutput: JSONType readonly isError: boolean errorType: string | null errorMessage: string | null - evaluations: Record + evaluations: Record evaluationErrors: Record } @@ -3991,13 +4013,20 @@ declare namespace tracer { } interface Dataset { - addRecord (input: any, expectedOutput?: any, metadata?: Record): Dataset + addRecord (input: JSONType, expectedOutput?: JSONType, metadata?: Record): Dataset /** Creates the dataset remotely if needed and pushes any unpushed records. */ push (): Promise name (): string id (): string | null projectId (): string | null - records (): Array<{ input: any, expectedOutput: any, metadata: Record }> + version (): number | null + latestVersion (): number | null + records (): Array<{ + id: string | null, + input: JSONType, + expectedOutput: JSONType, + metadata: Record + }> /** Dashboard URL for the dataset, or null until pushed. */ url (): string | null } @@ -4012,6 +4041,7 @@ declare namespace tracer { interface Experiments { /** Create a local dataset buffer; pushed on the first experiment run. */ createDataset (name: string, description?: string): Dataset + createDataset (name: string, options?: CreateDatasetOptions): Dataset /** Pull an existing dataset (with records) by name. */ pullDataset (name: string, options?: PullDatasetOptions): Promise /** Build an experiment to run over a dataset. */ diff --git a/integration-tests/appsec/iast-stack-traces-ts-with-sourcemaps/tsconfig.json b/integration-tests/appsec/iast-stack-traces-ts-with-sourcemaps/tsconfig.json index b48221fb50..a9fe1ab703 100644 --- a/integration-tests/appsec/iast-stack-traces-ts-with-sourcemaps/tsconfig.json +++ b/integration-tests/appsec/iast-stack-traces-ts-with-sourcemaps/tsconfig.json @@ -11,8 +11,7 @@ "esModuleInterop": true, "skipLibCheck": true, "forceConsistentCasingInFileNames": true, - "moduleResolution": "node", - "ignoreDeprecations": "6.0", + "moduleResolution": "bundler", "types": ["node"], "resolveJsonModule": true }, diff --git a/integration-tests/ci-visibility/automatic-log-submission.spec.js b/integration-tests/ci-visibility/automatic-log-submission.spec.js index 13ac26708c..9619eb3f6f 100644 --- a/integration-tests/ci-visibility/automatic-log-submission.spec.js +++ b/integration-tests/ci-visibility/automatic-log-submission.spec.js @@ -14,9 +14,11 @@ const { } = require('../helpers') const { FakeCiVisIntake } = require('../ci-visibility-intake') const { NODE_MAJOR } = require('../../version') +const { getLatestPlaywrightSpecifier } = require('../playwright/versions') const webAppServer = require('./web-app-server') const isLatestCucumberSupported = NODE_MAJOR === 22 || NODE_MAJOR === 24 || NODE_MAJOR >= 26 +const playwrightDependency = `@playwright/test@${getLatestPlaywrightSpecifier()}` describe('test optimization automatic log submission', () => { let cwd, receiver, childProcess, webAppPort @@ -27,7 +29,7 @@ describe('test optimization automatic log submission', () => { ...(isLatestCucumberSupported ? ['@cucumber/cucumber'] : []), 'jest', 'winston', - '@playwright/test', + playwrightDependency, ], true) before(async () => { diff --git a/integration-tests/cypress/cypress-atr.spec.js b/integration-tests/cypress/cypress-atr.spec.js index 6dde157cfc..a55f4d5d64 100644 --- a/integration-tests/cypress/cypress-atr.spec.js +++ b/integration-tests/cypress/cypress-atr.spec.js @@ -31,6 +31,7 @@ const { TEST_HAS_DYNAMIC_NAME, } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_MAJOR, NODE_MAJOR } = require('../../version') +const { getCypressDependencies } = require('./dependencies') const requestedVersion = process.env.CYPRESS_VERSION const oldestVersion = DD_MAJOR >= 6 ? '12.0.0' : '6.7.0' @@ -97,9 +98,7 @@ moduleTypes.forEach(({ this.timeout(80_000) let cwd, receiver, childProcess, webAppBaseUrl, webAppServer - // cypress-fail-fast is required as an incompatible plugin. - // typescript is required to compile .cy.ts spec files in the pre-compiled JS tests. - useSandbox([`cypress@${version}`, 'cypress-fail-fast@7.1.0', 'typescript'], true) + useSandbox(getCypressDependencies(version), true) before(async function () { this.timeout(180_000) diff --git a/integration-tests/cypress/cypress-efd.spec.js b/integration-tests/cypress/cypress-efd.spec.js index 2a8f261d05..6a2832bdca 100644 --- a/integration-tests/cypress/cypress-efd.spec.js +++ b/integration-tests/cypress/cypress-efd.spec.js @@ -28,6 +28,7 @@ const { TEST_FINAL_STATUS, } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_MAJOR, NODE_MAJOR } = require('../../version') +const { getCypressDependencies } = require('./dependencies') const requestedVersion = process.env.CYPRESS_VERSION const oldestVersion = DD_MAJOR >= 6 ? '12.0.0' : '6.7.0' @@ -95,9 +96,7 @@ moduleTypes.forEach(({ this.timeout(80_000) let cwd, receiver, childProcess, webAppBaseUrl, webAppServer - // cypress-fail-fast is required as an incompatible plugin. - // typescript is required to compile .cy.ts spec files in the pre-compiled JS tests. - useSandbox([`cypress@${version}`, 'cypress-fail-fast@7.1.0', 'typescript'], true) + useSandbox(getCypressDependencies(version), true) before(async function () { this.timeout(180_000) diff --git a/integration-tests/cypress/cypress-final-status-impacted-tests.spec.js b/integration-tests/cypress/cypress-final-status-impacted-tests.spec.js index 6929acf4d2..e8917d0261 100644 --- a/integration-tests/cypress/cypress-final-status-impacted-tests.spec.js +++ b/integration-tests/cypress/cypress-final-status-impacted-tests.spec.js @@ -43,6 +43,7 @@ const { TEST_IS_MODIFIED, } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_MAJOR, NODE_MAJOR } = require('../../version') +const { getCypressDependencies } = require('./dependencies') const requestedVersion = process.env.CYPRESS_VERSION const oldestVersion = DD_MAJOR >= 6 ? '12.0.0' : '6.7.0' @@ -110,9 +111,7 @@ moduleTypes.forEach(({ this.timeout(80_000) let cwd, receiver, childProcess, webAppBaseUrl, webAppServer - // cypress-fail-fast is required as an incompatible plugin. - // typescript is required to compile .cy.ts spec files in the pre-compiled JS tests. - useSandbox([`cypress@${version}`, 'cypress-fail-fast@7.1.0', 'typescript'], true) + useSandbox(getCypressDependencies(version), true) before(async function () { this.timeout(180_000) diff --git a/integration-tests/cypress/cypress-itr.spec.js b/integration-tests/cypress/cypress-itr.spec.js index 7c66cf4157..9311decfcf 100644 --- a/integration-tests/cypress/cypress-itr.spec.js +++ b/integration-tests/cypress/cypress-itr.spec.js @@ -28,6 +28,7 @@ const { TEST_ITR_FORCED_RUN, } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_MAJOR, NODE_MAJOR } = require('../../version') +const { getCypressDependencies } = require('./dependencies') const requestedVersion = process.env.CYPRESS_VERSION const oldestVersion = DD_MAJOR >= 6 ? '12.0.0' : '6.7.0' @@ -111,9 +112,7 @@ moduleTypes.forEach(({ this.timeout(80_000) let cwd, receiver, childProcess, webAppBaseUrl, webAppServer - // cypress-fail-fast is required as an incompatible plugin. - // typescript is required to compile .cy.ts spec files in the pre-compiled JS tests. - useSandbox([`cypress@${version}`, 'cypress-fail-fast@7.1.0', 'typescript'], true) + useSandbox(getCypressDependencies(version), true) before(async function () { this.timeout(180_000) diff --git a/integration-tests/cypress/cypress-reporting-instrumentation.spec.js b/integration-tests/cypress/cypress-reporting-instrumentation.spec.js index 38ce71728f..b6daffcafa 100644 --- a/integration-tests/cypress/cypress-reporting-instrumentation.spec.js +++ b/integration-tests/cypress/cypress-reporting-instrumentation.spec.js @@ -45,6 +45,7 @@ const { resolveOriginalSourceFile, resolveSourceLineForTest, } = require('../../packages/datadog-plugin-cypress/src/source-map-utils') +const { getCypressDependencies } = require('./dependencies') const requestedVersion = process.env.CYPRESS_VERSION const oldestVersion = DD_MAJOR >= 6 ? '12.0.0' : '6.7.0' @@ -165,9 +166,7 @@ moduleTypes.forEach(({ this.timeout(80_000) let cwd, receiver, childProcess, webAppBaseUrl, webAppServer - // cypress-fail-fast is required as an incompatible plugin. - // typescript is required to compile .cy.ts spec files in the pre-compiled JS tests. - const sandboxDependencies = [`cypress@${version}`, 'cypress-fail-fast@7.1.0', 'typescript'] + const sandboxDependencies = getCypressDependencies(version) if (type === 'commonJS' && version === 'latest') { // These dependencies are only needed by the component/Vite regression test below. sandboxDependencies.push( diff --git a/integration-tests/cypress/cypress-reporting.spec.js b/integration-tests/cypress/cypress-reporting.spec.js index 576207a229..d1b616ad62 100644 --- a/integration-tests/cypress/cypress-reporting.spec.js +++ b/integration-tests/cypress/cypress-reporting.spec.js @@ -34,6 +34,7 @@ const { } = require('../../packages/dd-trace/src/plugins/util/test') const { ERROR_MESSAGE, ERROR_TYPE, COMPONENT } = require('../../packages/dd-trace/src/constants') const { DD_MAJOR, NODE_MAJOR } = require('../../version') +const { getCypressDependencies } = require('./dependencies') const requestedVersion = process.env.CYPRESS_VERSION const oldestVersion = DD_MAJOR >= 6 ? '12.0.0' : '6.7.0' @@ -102,9 +103,7 @@ moduleTypes.forEach(({ this.timeout(80_000) let cwd, receiver, childProcess, webAppBaseUrl, webAppServer - // cypress-fail-fast is required as an incompatible plugin. - // typescript is required to compile .cy.ts spec files in the pre-compiled JS tests. - useSandbox([`cypress@${version}`, 'cypress-fail-fast@7.1.0', 'typescript'], true) + useSandbox(getCypressDependencies(version), true) before(async function () { this.timeout(180_000) diff --git a/integration-tests/cypress/cypress-test-management.spec.js b/integration-tests/cypress/cypress-test-management.spec.js index 9a0203e827..f2b533663e 100644 --- a/integration-tests/cypress/cypress-test-management.spec.js +++ b/integration-tests/cypress/cypress-test-management.spec.js @@ -33,6 +33,7 @@ const { TEST_RETRY_REASON_TYPES, } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_MAJOR, NODE_MAJOR } = require('../../version') +const { getCypressDependencies } = require('./dependencies') const requestedVersion = process.env.CYPRESS_VERSION const oldestVersion = DD_MAJOR >= 6 ? '12.0.0' : '6.7.0' @@ -100,9 +101,7 @@ moduleTypes.forEach(({ this.timeout(80_000) let cwd, receiver, childProcess, webAppBaseUrl, webAppServer, secondWebAppBaseUrl, secondWebAppServer - // cypress-fail-fast is required as an incompatible plugin. - // typescript is required to compile .cy.ts spec files in the pre-compiled JS tests. - useSandbox([`cypress@${version}`, 'cypress-fail-fast@7.1.0', 'typescript'], true) + useSandbox(getCypressDependencies(version), true) before(async function () { this.timeout(180_000) diff --git a/integration-tests/cypress/cypress-tia-code-coverage.spec.js b/integration-tests/cypress/cypress-tia-code-coverage.spec.js index 29693ccd20..9318440686 100644 --- a/integration-tests/cypress/cypress-tia-code-coverage.spec.js +++ b/integration-tests/cypress/cypress-tia-code-coverage.spec.js @@ -24,6 +24,7 @@ const { getLineCoverageBitmap, } = require('../../packages/dd-trace/src/plugins/util/test') const { DD_MAJOR, NODE_MAJOR } = require('../../version') +const { getCypressDependencies } = require('./dependencies') const requestedVersion = process.env.CYPRESS_VERSION const oldestVersion = DD_MAJOR >= 6 ? '12.0.0' : '6.7.0' @@ -122,7 +123,7 @@ moduleTypes.forEach(({ let cwd, childProcess, webAppBaseUrl, webAppServer - useSandbox([`cypress@${version}`, 'cypress-fail-fast@7.1.0', 'typescript'], true) + useSandbox(getCypressDependencies(version), true) before(async function () { cwd = sandboxCwd() diff --git a/integration-tests/cypress/dependencies.js b/integration-tests/cypress/dependencies.js new file mode 100644 index 0000000000..5d29e9c343 --- /dev/null +++ b/integration-tests/cypress/dependencies.js @@ -0,0 +1,24 @@ +'use strict' + +const BABEL_7_PRESET_TYPESCRIPT_VERSION = '7.28.5' +const TYPESCRIPT_6_VERSION = '6.0.3' + +/** + * @param {string} cypressVersion + * @returns {string[]} + */ +function getCypressDependencies (cypressVersion) { + const dependencies = [ + `cypress@${cypressVersion}`, + 'cypress-fail-fast@7.1.0', + `typescript@${TYPESCRIPT_6_VERSION}`, + ] + + if (cypressVersion === 'latest') { + dependencies.push(`@babel/preset-typescript@${BABEL_7_PRESET_TYPESCRIPT_VERSION}`) + } + + return dependencies +} + +module.exports = { getCypressDependencies } diff --git a/integration-tests/cypress/dependencies.spec.js b/integration-tests/cypress/dependencies.spec.js new file mode 100644 index 0000000000..07122e318b --- /dev/null +++ b/integration-tests/cypress/dependencies.spec.js @@ -0,0 +1,27 @@ +'use strict' + +const assert = require('node:assert/strict') + +const { getCypressDependencies } = require('./dependencies') + +const sharedDependencies = [ + 'cypress-fail-fast@7.1.0', + 'typescript@6.0.3', +] + +describe('getCypressDependencies', () => { + it('pins TypeScript 6 for Cypress versions using the bundled TypeScript loader', () => { + assert.deepStrictEqual(getCypressDependencies('12.0.0'), [ + 'cypress@12.0.0', + ...sharedDependencies, + ]) + }) + + it('installs the Babel TypeScript preset required by latest Cypress', () => { + assert.deepStrictEqual(getCypressDependencies('latest'), [ + 'cypress@latest', + ...sharedDependencies, + '@babel/preset-typescript@7.28.5', + ]) + }) +}) diff --git a/integration-tests/esbuild/package.json b/integration-tests/esbuild/package.json index 3aebfa547e..bdfa32b18c 100644 --- a/integration-tests/esbuild/package.json +++ b/integration-tests/esbuild/package.json @@ -22,7 +22,7 @@ "dependencies": { "@apollo/server": "5.5.1", "@koa/router": "15.7.0", - "@smithy/smithy-client": "4.14.13", + "@smithy/smithy-client": "4.14.14", "aws-sdk": "2.1693.0", "axios": "1.18.1", "esbuild": "^0.28.0", diff --git a/integration-tests/jest/babel-dependencies.js b/integration-tests/jest/babel-dependencies.js new file mode 100644 index 0000000000..b60ba90bc4 --- /dev/null +++ b/integration-tests/jest/babel-dependencies.js @@ -0,0 +1,30 @@ +'use strict' + +const { satisfies } = require('semver') + +const { NODE_VERSION } = require('../../version') + +const BABEL_7_CORE_VERSION = '7.29.0' +const BABEL_7_PRESET_TYPESCRIPT_VERSION = '7.28.5' +const BABEL_8_NODE_RANGE = '^22.18.0 || >=24.11.0' + +/** + * @param {string} jestVersion + * @param {string} [nodeVersion] + * @returns {string[]} + */ +function getBabelDependencies (jestVersion, nodeVersion = NODE_VERSION) { + if (jestVersion === 'latest' && satisfies(nodeVersion, BABEL_8_NODE_RANGE)) { + return [ + '@babel/core', + '@babel/preset-typescript', + ] + } + + return [ + `@babel/core@${BABEL_7_CORE_VERSION}`, + `@babel/preset-typescript@${BABEL_7_PRESET_TYPESCRIPT_VERSION}`, + ] +} + +module.exports = { getBabelDependencies } diff --git a/integration-tests/jest/jest.babel-dependencies.spec.js b/integration-tests/jest/jest.babel-dependencies.spec.js new file mode 100644 index 0000000000..d7dc1469ad --- /dev/null +++ b/integration-tests/jest/jest.babel-dependencies.spec.js @@ -0,0 +1,33 @@ +'use strict' + +const assert = require('node:assert/strict') + +const { describe, it } = require('mocha') + +const { getBabelDependencies } = require('./babel-dependencies') + +const babel7Dependencies = [ + '@babel/core@7.29.0', + '@babel/preset-typescript@7.28.5', +] + +describe('getBabelDependencies', () => { + it('uses Babel 7 pins when Node.js does not support Babel 8', () => { + assert.deepStrictEqual(getBabelDependencies('latest', '18.20.8'), babel7Dependencies) + }) + + it('uses Babel 7 pins when Jest is not latest', () => { + assert.deepStrictEqual(getBabelDependencies('28.0.0', '22.22.3'), babel7Dependencies) + }) + + it('uses Babel 7 pins before the Node.js 24 version supported by Babel 8', () => { + assert.deepStrictEqual(getBabelDependencies('latest', '24.10.0'), babel7Dependencies) + }) + + it('uses the unversioned Babel entries when Node.js and Jest support Babel 8', () => { + assert.deepStrictEqual(getBabelDependencies('latest', '22.22.3'), [ + '@babel/core', + '@babel/preset-typescript', + ]) + }) +}) diff --git a/integration-tests/jest/jest.core.spec.js b/integration-tests/jest/jest.core.spec.js index 05f9d1a7b1..0f15997961 100644 --- a/integration-tests/jest/jest.core.spec.js +++ b/integration-tests/jest/jest.core.spec.js @@ -56,6 +56,7 @@ const { DD_HOST_CPU_COUNT } = require('../../packages/dd-trace/src/plugins/util/ const { ERROR_MESSAGE, ERROR_TYPE, ORIGIN_KEY, COMPONENT } = require('../../packages/dd-trace/src/constants') const { DD_MAJOR } = require('../../version') const { version: ddTraceVersion } = require('../../package.json') +const { getBabelDependencies } = require('./babel-dependencies') const testFile = 'ci-visibility/run-jest.js' const expectedStdout = 'Test Suites: 2 passed' @@ -83,8 +84,7 @@ describe(`jest@${JEST_VERSION} commonJS`, () => { shouldInstallJestEnvironmentJsdom ? `jest-environment-jsdom@${JEST_VERSION}` : '', // jest-circus is not included in older versions of jest JEST_VERSION !== 'latest' ? `jest-circus@${JEST_VERSION}` : '', - '@babel/core', - '@babel/preset-typescript', + ...getBabelDependencies(JEST_VERSION), '@happy-dom/jest-environment', 'office-addin-mock', 'winston', diff --git a/integration-tests/jest/jest.test-management.spec.js b/integration-tests/jest/jest.test-management.spec.js index 42fa34d04c..77a3e5ff23 100644 --- a/integration-tests/jest/jest.test-management.spec.js +++ b/integration-tests/jest/jest.test-management.spec.js @@ -53,6 +53,7 @@ const { const { TELEMETRY_COVERAGE_UPLOAD } = require('../../packages/dd-trace/src/ci-visibility/telemetry') const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') const { DD_MAJOR } = require('../../version') +const { getBabelDependencies } = require('./babel-dependencies') const runTestsCommand = 'node ./ci-visibility/run-jest.js' @@ -75,8 +76,7 @@ describe(`jest@${JEST_VERSION} commonJS`, () => { shouldInstallJestEnvironmentJsdom ? `jest-environment-jsdom@${JEST_VERSION}` : '', // jest-circus is not included in older versions of jest JEST_VERSION !== 'latest' ? `jest-circus@${JEST_VERSION}` : '', - '@babel/core', - '@babel/preset-typescript', + ...getBabelDependencies(JEST_VERSION), '@happy-dom/jest-environment', 'office-addin-mock', 'winston', diff --git a/integration-tests/jest/jest.tia-efd.spec.js b/integration-tests/jest/jest.tia-efd.spec.js index 2fef99441e..b5a5a5ac8c 100644 --- a/integration-tests/jest/jest.tia-efd.spec.js +++ b/integration-tests/jest/jest.tia-efd.spec.js @@ -51,6 +51,7 @@ const { } = require('../../packages/dd-trace/src/plugins/util/test') const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') const { DD_MAJOR, NODE_MAJOR } = require('../../version') +const { getBabelDependencies } = require('./babel-dependencies') const testFile = 'ci-visibility/run-jest.js' const expectedCoverageFiles = [ @@ -98,8 +99,7 @@ describe(`jest@${JEST_VERSION} commonJS`, () => { shouldInstallJestEnvironmentJsdom ? `jest-environment-jsdom@${JEST_VERSION}` : '', // jest-circus is not included in older versions of jest JEST_VERSION !== 'latest' ? `jest-circus@${JEST_VERSION}` : '', - '@babel/core', - '@babel/preset-typescript', + ...getBabelDependencies(JEST_VERSION), '@happy-dom/jest-environment', 'office-addin-mock', 'winston', diff --git a/integration-tests/playwright/playwright-active-test-span.spec.js b/integration-tests/playwright/playwright-active-test-span.spec.js index f979984d74..3d985b2bbb 100644 --- a/integration-tests/playwright/playwright-active-test-span.spec.js +++ b/integration-tests/playwright/playwright-active-test-span.spec.js @@ -33,8 +33,8 @@ const { PLAYWRIGHT_VERSION } = process.env const NUM_RETRIES_EFD = 3 -const latest = 'latest' -const { oldest } = require('./versions') +const { getLatestPlaywrightSpecifier, oldest } = require('./versions') +const latest = getLatestPlaywrightSpecifier() const versions = [oldest, latest] versions.forEach((version) => { diff --git a/integration-tests/playwright/playwright-atr.spec.js b/integration-tests/playwright/playwright-atr.spec.js index 8fec0d3a3b..eb8ad75b07 100644 --- a/integration-tests/playwright/playwright-atr.spec.js +++ b/integration-tests/playwright/playwright-atr.spec.js @@ -23,8 +23,8 @@ const { const { PLAYWRIGHT_VERSION } = process.env -const latest = 'latest' -const { oldest } = require('./versions') +const { getLatestPlaywrightSpecifier, oldest } = require('./versions') +const latest = getLatestPlaywrightSpecifier() const versions = [oldest, latest] versions.forEach((version) => { diff --git a/integration-tests/playwright/playwright-efd.spec.js b/integration-tests/playwright/playwright-efd.spec.js index 7c723ab642..7840efcd5b 100644 --- a/integration-tests/playwright/playwright-efd.spec.js +++ b/integration-tests/playwright/playwright-efd.spec.js @@ -31,8 +31,8 @@ const { PLAYWRIGHT_VERSION } = process.env const NUM_RETRIES_EFD = 3 const PLAYWRIGHT_EFD_GATHER_TIMEOUT = 60000 -const latest = 'latest' -const { oldest } = require('./versions') +const { getLatestPlaywrightSpecifier, oldest } = require('./versions') +const latest = getLatestPlaywrightSpecifier() const versions = [oldest, latest] versions.forEach((version) => { diff --git a/integration-tests/playwright/playwright-final-status.spec.js b/integration-tests/playwright/playwright-final-status.spec.js index 8652832d49..e152ff4f43 100644 --- a/integration-tests/playwright/playwright-final-status.spec.js +++ b/integration-tests/playwright/playwright-final-status.spec.js @@ -27,8 +27,8 @@ const { PLAYWRIGHT_VERSION } = process.env const NUM_RETRIES_EFD = 3 const RETRY_FINAL_STATUS_TIMEOUT = 60000 -const latest = 'latest' -const { oldest } = require('./versions') +const { getLatestPlaywrightSpecifier, oldest } = require('./versions') +const latest = getLatestPlaywrightSpecifier() const versions = [oldest, latest] versions.forEach((version) => { diff --git a/integration-tests/playwright/playwright-impacted-tests.spec.js b/integration-tests/playwright/playwright-impacted-tests.spec.js index 2607ac35ad..f180f3d1c7 100644 --- a/integration-tests/playwright/playwright-impacted-tests.spec.js +++ b/integration-tests/playwright/playwright-impacted-tests.spec.js @@ -30,8 +30,8 @@ const { PLAYWRIGHT_VERSION } = process.env const NUM_RETRIES_EFD = 3 -const latest = 'latest' -const { oldest } = require('./versions') +const { getLatestPlaywrightSpecifier, oldest } = require('./versions') +const latest = getLatestPlaywrightSpecifier() const versions = [oldest, latest] const DEFAULT_IMPACTED_KNOWN_TESTS = { diff --git a/integration-tests/playwright/playwright-reporting.spec.js b/integration-tests/playwright/playwright-reporting.spec.js index 5644bdef0d..f2cb857fd1 100644 --- a/integration-tests/playwright/playwright-reporting.spec.js +++ b/integration-tests/playwright/playwright-reporting.spec.js @@ -48,8 +48,8 @@ const { ERROR_MESSAGE } = require('../../packages/dd-trace/src/constants') const { PLAYWRIGHT_VERSION } = process.env -const latest = 'latest' -const { oldest } = require('./versions') +const { getLatestPlaywrightSpecifier, oldest } = require('./versions') +const latest = getLatestPlaywrightSpecifier() const versions = [oldest, latest] const REQUEST_ERROR_TAG_TEST_DIR = './ci-visibility/playwright-tests-request-error-tag' const SCREENSHOT_CAPTURE_DISABLED_WARNING = diff --git a/integration-tests/playwright/playwright-test-management.spec.js b/integration-tests/playwright/playwright-test-management.spec.js index 0f01cb036c..e39f45a1fe 100644 --- a/integration-tests/playwright/playwright-test-management.spec.js +++ b/integration-tests/playwright/playwright-test-management.spec.js @@ -36,8 +36,8 @@ const { PLAYWRIGHT_VERSION } = process.env const PLAYWRIGHT_TEST_MANAGEMENT_GATHER_TIMEOUT = 60000 -const latest = 'latest' -const { oldest } = require('./versions') +const { getLatestPlaywrightSpecifier, oldest } = require('./versions') +const latest = getLatestPlaywrightSpecifier() const versions = [oldest, latest] const ATF_MANAGEMENT_TESTS = { diff --git a/integration-tests/playwright/versions.js b/integration-tests/playwright/versions.js index 90ee673f79..69ca5f59c0 100644 --- a/integration-tests/playwright/versions.js +++ b/integration-tests/playwright/versions.js @@ -1,9 +1,23 @@ 'use strict' -const { DD_MAJOR } = require('../../version') +const { DD_MAJOR, NODE_MAJOR } = require('../../version') const oldest = DD_MAJOR >= 6 ? '1.38.0' : '1.18.0' const latest = require('../../packages/dd-trace/test/plugins/versions/package.json') .dependencies['@playwright/test'] +const latestSupportedByNode18 = '1.61.0' -module.exports = { oldest, latest } +/** + * @param {number} [nodeMajor] + * @returns {string} + */ +function getLatestPlaywrightSpecifier (nodeMajor = NODE_MAJOR) { + return nodeMajor < 20 ? latestSupportedByNode18 : 'latest' +} + +module.exports = { + getLatestPlaywrightSpecifier, + latest, + latestSupportedByNode18, + oldest, +} diff --git a/integration-tests/playwright/versions.spec.js b/integration-tests/playwright/versions.spec.js new file mode 100644 index 0000000000..d5ab14f614 --- /dev/null +++ b/integration-tests/playwright/versions.spec.js @@ -0,0 +1,18 @@ +'use strict' + +const assert = require('node:assert/strict') + +const { + getLatestPlaywrightSpecifier, + latestSupportedByNode18, +} = require('./versions') + +describe('getLatestPlaywrightSpecifier', () => { + it('uses the last Playwright version supporting Node.js 18', () => { + assert.strictEqual(getLatestPlaywrightSpecifier(18), latestSupportedByNode18) + }) + + it('uses latest Playwright on supported Node.js versions', () => { + assert.strictEqual(getLatestPlaywrightSpecifier(20), 'latest') + }) +}) diff --git a/integration-tests/vitest/vitest.core.spec.js b/integration-tests/vitest/vitest.core.spec.js index 7a3be13442..45b286875a 100644 --- a/integration-tests/vitest/vitest.core.spec.js +++ b/integration-tests/vitest/vitest.core.spec.js @@ -121,7 +121,7 @@ versions.forEach((version) => { `@vitest/coverage-v8@${version}`, '@types/node', 'tinypool', - 'typescript', + 'typescript@6.0.3', ], true) before(function () { diff --git a/integration-tests/webdriverio/fixtures/delay.e2e.js b/integration-tests/webdriverio/fixtures/delay.e2e.js new file mode 100644 index 0000000000..1a09b24f99 --- /dev/null +++ b/integration-tests/webdriverio/fixtures/delay.e2e.js @@ -0,0 +1,16 @@ +'use strict' + +const tracer = require('dd-trace') + +const assert = require('node:assert/strict') + +describe('WebdriverIO delayed root suite', () => { + it('waits for the user and Test Optimization configuration', () => { + const activeSpan = tracer.scope().active() + + assert.ok(activeSpan) + activeSpan.setTag('test.webdriverio.worker', 'delay') + }) +}) + +setTimeout(run, 1_000) diff --git a/integration-tests/webdriverio/fixtures/fail.e2e.js b/integration-tests/webdriverio/fixtures/fail.e2e.js new file mode 100644 index 0000000000..76057b6945 --- /dev/null +++ b/integration-tests/webdriverio/fixtures/fail.e2e.js @@ -0,0 +1,9 @@ +'use strict' + +const assert = require('node:assert/strict') + +describe('WebdriverIO failing worker', () => { + it('fails before the next grouped spec', () => { + assert.fail('expected WebdriverIO integration failure') + }) +}) diff --git a/integration-tests/webdriverio/fixtures/first.e2e.js b/integration-tests/webdriverio/fixtures/first.e2e.js new file mode 100644 index 0000000000..a21ea0b265 --- /dev/null +++ b/integration-tests/webdriverio/fixtures/first.e2e.js @@ -0,0 +1,14 @@ +'use strict' + +const tracer = require('dd-trace') + +const assert = require('node:assert/strict') + +describe('WebdriverIO first worker', () => { + it('runs with an active Test Optimization span', () => { + const activeSpan = tracer.scope().active() + + assert.ok(activeSpan) + activeSpan.setTag('test.webdriverio.worker', 'first') + }) +}) diff --git a/integration-tests/webdriverio/fixtures/hook-fail.e2e.js b/integration-tests/webdriverio/fixtures/hook-fail.e2e.js new file mode 100644 index 0000000000..6daa199119 --- /dev/null +++ b/integration-tests/webdriverio/fixtures/hook-fail.e2e.js @@ -0,0 +1,9 @@ +'use strict' + +describe('WebdriverIO hook failure', () => { + before(() => { + throw new Error('expected WebdriverIO hook failure') + }) + + it('does not run', () => {}) +}) diff --git a/integration-tests/webdriverio/fixtures/load-fail.e2e.js b/integration-tests/webdriverio/fixtures/load-fail.e2e.js new file mode 100644 index 0000000000..fef4ed6b50 --- /dev/null +++ b/integration-tests/webdriverio/fixtures/load-fail.e2e.js @@ -0,0 +1,3 @@ +'use strict' + +throw new Error('expected WebdriverIO spec load failure') diff --git a/integration-tests/webdriverio/fixtures/retry.e2e.js b/integration-tests/webdriverio/fixtures/retry.e2e.js new file mode 100644 index 0000000000..264c6e3b0d --- /dev/null +++ b/integration-tests/webdriverio/fixtures/retry.e2e.js @@ -0,0 +1,18 @@ +'use strict' + +const tracer = require('dd-trace') + +const assert = require('node:assert/strict') + +let attempt = 0 + +describe('WebdriverIO Mocha retries', () => { + it('reports every Mocha attempt', () => { + const activeSpan = tracer.scope().active() + + assert.ok(activeSpan) + activeSpan.setTag('test.webdriverio.worker', 'retry') + attempt++ + assert.strictEqual(attempt, 2) + }) +}) diff --git a/integration-tests/webdriverio/fixtures/runner-env-preload.js b/integration-tests/webdriverio/fixtures/runner-env-preload.js new file mode 100644 index 0000000000..feb4722834 --- /dev/null +++ b/integration-tests/webdriverio/fixtures/runner-env-preload.js @@ -0,0 +1,3 @@ +'use strict' + +globalThis.webdriverioRunnerEnvPreloaded = true diff --git a/integration-tests/webdriverio/fixtures/runner-env.e2e.js b/integration-tests/webdriverio/fixtures/runner-env.e2e.js new file mode 100644 index 0000000000..3ec46ea951 --- /dev/null +++ b/integration-tests/webdriverio/fixtures/runner-env.e2e.js @@ -0,0 +1,16 @@ +'use strict' + +const tracer = require('dd-trace') + +const assert = require('node:assert/strict') + +describe('WebdriverIO runner environment', () => { + it('preserves the launcher and worker NODE_OPTIONS', () => { + assert.strictEqual(globalThis.webdriverioRunnerEnvPreloaded, true) + + const activeSpan = tracer.scope().active() + + assert.ok(activeSpan) + activeSpan.setTag('test.webdriverio.worker', 'runner-env-node-options') + }) +}) diff --git a/integration-tests/webdriverio/fixtures/second.e2e.js b/integration-tests/webdriverio/fixtures/second.e2e.js new file mode 100644 index 0000000000..72774856ba --- /dev/null +++ b/integration-tests/webdriverio/fixtures/second.e2e.js @@ -0,0 +1,14 @@ +'use strict' + +const tracer = require('dd-trace') + +const assert = require('node:assert/strict') + +describe('WebdriverIO second worker', () => { + it('runs with an active Test Optimization span', () => { + const activeSpan = tracer.scope().active() + + assert.ok(activeSpan) + activeSpan.setTag('test.webdriverio.worker', 'second') + }) +}) diff --git a/integration-tests/webdriverio/fixtures/spec-file-retry.e2e.js b/integration-tests/webdriverio/fixtures/spec-file-retry.e2e.js new file mode 100644 index 0000000000..eca5fd15e2 --- /dev/null +++ b/integration-tests/webdriverio/fixtures/spec-file-retry.e2e.js @@ -0,0 +1,25 @@ +'use strict' + +const tracer = require('dd-trace') + +const assert = require('node:assert/strict') +const fs = require('node:fs') +const path = require('node:path') + +const retryMarker = path.join(__dirname, '.webdriverio-spec-file-retry') + +describe('WebdriverIO spec file retries', () => { + it('reports the retried worker', () => { + const activeSpan = tracer.scope().active() + + assert.ok(activeSpan) + activeSpan.setTag('test.webdriverio.worker', 'spec-file-retry') + + if (!fs.existsSync(retryMarker)) { + fs.writeFileSync(retryMarker, '') + assert.fail('fail the first spec file execution') + } + + fs.unlinkSync(retryMarker) + }) +}) diff --git a/integration-tests/webdriverio/fixtures/tdd.e2e.js b/integration-tests/webdriverio/fixtures/tdd.e2e.js new file mode 100644 index 0000000000..82b2fefbaf --- /dev/null +++ b/integration-tests/webdriverio/fixtures/tdd.e2e.js @@ -0,0 +1,14 @@ +'use strict' + +const tracer = require('dd-trace') + +const assert = require('node:assert/strict') + +suite('WebdriverIO TDD interface', () => { + test('reports a TDD test', () => { + const activeSpan = tracer.scope().active() + + assert.ok(activeSpan) + activeSpan.setTag('test.webdriverio.worker', 'tdd') + }) +}) diff --git a/integration-tests/webdriverio/fixtures/wdio.conf.js b/integration-tests/webdriverio/fixtures/wdio.conf.js new file mode 100644 index 0000000000..088102e9ca --- /dev/null +++ b/integration-tests/webdriverio/fixtures/wdio.conf.js @@ -0,0 +1,132 @@ +'use strict' + +const scenario = process.env.WEBDRIVERIO_SCENARIO || 'parallel' + +const baseConfig = { + runner: 'local', + specs: [ + './first.e2e.js', + './second.e2e.js', + ], + maxInstances: 2, + capabilities: [{ + browserName: 'chrome', + }], + protocol: 'http', + hostname: '127.0.0.1', + port: Number(process.env.WEBDRIVER_PORT), + path: '/', + connectionRetryCount: 0, + services: [], + framework: 'mocha', + reporters: [], + mochaOpts: { + ui: 'bdd', + timeout: 10_000, + }, +} + +const scenarioConfig = { + bail: { + maxInstances: 1, + mochaOpts: { + bail: true, + }, + specs: [[ + './fail.e2e.js', + './second.e2e.js', + ]], + }, + preFrameworkFailure: { + maxInstances: 1, + reporters: ['webdriverio-missing-reporter'], + specs: ['./first.e2e.js'], + }, + delay: { + maxInstances: 1, + mochaOpts: { + delay: true, + }, + specs: ['./delay.e2e.js'], + }, + grep: { + maxInstances: 1, + mochaOpts: { + grep: 'first worker', + }, + specs: [[ + './first.e2e.js', + './second.e2e.js', + ]], + }, + grouped: { + maxInstances: 1, + specs: [[ + './first.e2e.js', + './second.e2e.js', + ]], + }, + hookFailure: { + maxInstances: 1, + specs: [[ + './hook-fail.e2e.js', + './first.e2e.js', + ]], + }, + loadFailure: { + maxInstances: 1, + specs: ['./load-fail.e2e.js'], + }, + multipleCapabilities: { + capabilities: [ + { browserName: 'chrome' }, + { browserName: 'firefox' }, + ], + specs: ['./first.e2e.js'], + }, + parallel: {}, + retries: { + maxInstances: 1, + mochaOpts: { + retries: 1, + }, + specs: ['./retry.e2e.js'], + }, + runnerEnvNodeOptions: { + maxInstances: 1, + runnerEnv: { + NODE_OPTIONS: '--require ./runner-env-preload.js', + }, + specs: ['./runner-env.e2e.js'], + }, + serial: { + maxInstances: 1, + }, + specFileRetries: { + maxInstances: 1, + specFileRetries: 1, + specFileRetriesDelay: 0, + specs: ['./spec-file-retry.e2e.js'], + }, + tdd: { + maxInstances: 1, + mochaOpts: { + ui: 'tdd', + }, + specs: ['./tdd.e2e.js'], + }, +} + +const selectedScenario = scenarioConfig[scenario] +if (!selectedScenario) { + throw new Error(`Unknown WebdriverIO integration scenario: ${scenario}`) +} + +exports.config = { + ...baseConfig, + ...selectedScenario, + mochaOpts: { + ...baseConfig.mochaOpts, + ...selectedScenario.mochaOpts, + }, +} diff --git a/integration-tests/webdriverio/webdriverio.spec.js b/integration-tests/webdriverio/webdriverio.spec.js new file mode 100644 index 0000000000..06886fb789 --- /dev/null +++ b/integration-tests/webdriverio/webdriverio.spec.js @@ -0,0 +1,489 @@ +'use strict' + +const assert = require('node:assert/strict') +const { exec } = require('node:child_process') +const { once } = require('node:events') +const http = require('node:http') + +const { + getCiVisAgentlessConfig, + sandboxCwd, + useSandbox, +} = require('../helpers') +const { FakeCiVisIntake } = require('../ci-visibility-intake') +const { + MOCHA_IS_PARALLEL, + TEST_CODE_COVERAGE_ENABLED, + TEST_EARLY_FLAKE_ENABLED, + TEST_FRAMEWORK, + TEST_FRAMEWORK_ADAPTER, + TEST_FRAMEWORK_VERSION, + TEST_IS_RETRY, + TEST_ITR_SKIPPING_ENABLED, + TEST_MANAGEMENT_ENABLED, + TEST_MODULE, + TEST_STATUS, + TEST_SUITE, + TEST_TYPE, +} = require('../../packages/dd-trace/src/plugins/util/test') + +const OLDEST_WEBDRIVERIO_VERSION = '9.0.0' +const requestedVersion = process.env.WEBDRIVERIO_VERSION +const versions = requestedVersion + ? [requestedVersion === 'oldest' ? OLDEST_WEBDRIVERIO_VERSION : requestedVersion] + : [OLDEST_WEBDRIVERIO_VERSION, 'latest'] + +const advancedSettings = { + code_coverage: true, + tests_skipping: true, + itr_enabled: true, + require_git: false, + early_flake_detection: { + enabled: true, + slow_test_retries: { + '5s': 5, + }, + }, + flaky_test_retries_enabled: true, + di_enabled: true, + known_tests_enabled: true, + test_management: { + enabled: true, + attempt_to_fix_retries: 5, + }, + impacted_tests_enabled: true, + coverage_report_upload_enabled: true, +} + +const advancedRequestPaths = [ + '/api/v2/ci/libraries/tests', + '/api/v2/ci/tests/skippable', + '/api/v2/test/libraries/test-management/tests', +] + +/** + * Starts the minimal W3C WebDriver endpoint required by WebdriverIO workers. + * + * @returns {Promise<{port: number, server: import('node:http').Server, getSessionCount: () => number}>} + */ +function startWebDriverServer () { + let sessionCount = 0 + const server = http.createServer((request, response) => { + request.resume() + request.once('end', () => { + const isNewSession = request.method === 'POST' && request.url === '/session' + let value = null + + if (isNewSession) { + sessionCount++ + value = { + sessionId: `webdriverio-${sessionCount}`, + capabilities: { + browserName: 'chrome', + browserVersion: 'test', + platformName: process.platform, + }, + } + } else if (request.method === 'GET' && request.url === '/status') { + value = { ready: true, message: '' } + } + + response.writeHead(200, { 'content-type': 'application/json' }) + response.end(JSON.stringify({ value })) + }) + }) + + return new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(0, '127.0.0.1', () => { + server.removeListener('error', reject) + const address = server.address() + + if (!address || typeof address === 'string') { + reject(new Error('WebDriver server did not bind to a TCP port')) + return + } + + resolve({ + port: address.port, + server, + getSessionCount: () => sessionCount, + }) + }) + }) +} + +/** + * Stops an HTTP server if it is listening. + * + * @param {import('node:http').Server|undefined} server + * @returns {Promise} + */ +function stopServer (server) { + if (!server?.listening) { + return Promise.resolve() + } + return new Promise(resolve => server.close(resolve)) +} + +/** + * Asserts every child event belongs to the coordinator-owned session and module. + * + * @param {object} session + * @param {object} module + * @param {object[]} suites + * @param {object[]} tests + * @returns {void} + */ +function assertEventHierarchy (session, module, suites, tests) { + const sessionId = session.test_session_id.toString(10) + const moduleId = module.test_module_id.toString(10) + + assert.strictEqual(module.test_session_id.toString(10), sessionId) + + for (const event of [...suites, ...tests]) { + assert.strictEqual(event.test_session_id.toString(10), sessionId) + assert.strictEqual(event.test_module_id.toString(10), moduleId) + } + + const suiteIds = new Set(suites.map(suite => suite.test_suite_id.toString(10))) + for (const test of tests) { + assert.ok(suiteIds.has(test.test_suite_id.toString(10))) + } +} + +/** + * Asserts each repeated suite execution owns exactly one test event. + * + * @param {object[]} suites + * @param {object[]} tests + * @returns {void} + */ +function assertOneTestPerSuiteExecution (suites, tests) { + assert.deepStrictEqual( + tests.map(test => test.test_suite_id.toString(10)).sort(), + suites.map(suite => suite.test_suite_id.toString(10)).sort() + ) +} + +/** + * Extracts events and verifies the WebdriverIO run stayed in basic-reporting mode. + * + * @param {object[]} payloads + * @param {string} requestedVersion + * @returns {{session: object, module: object, suites: object[], tests: object[]}} + */ +function getBasicReportingEvents (payloads, requestedVersion) { + const settingsRequests = payloads.filter(({ url }) => + url.endsWith('/api/v2/libraries/tests/services/setting')) + const advancedRequests = payloads.filter(({ url }) => + advancedRequestPaths.some(path => url.endsWith(path))) + const cyclePayloads = payloads.filter(({ url }) => url.endsWith('/api/v2/citestcycle')) + const events = cyclePayloads.flatMap(({ payload }) => payload.events) + const sessions = events.filter(event => event.type === 'test_session_end').map(event => event.content) + const modules = events.filter(event => event.type === 'test_module_end').map(event => event.content) + const suites = events.filter(event => event.type === 'test_suite_end').map(event => event.content) + const tests = events.filter(event => event.type === 'test').map(event => event.content) + + assert.strictEqual(settingsRequests.length, 1) + assert.strictEqual(advancedRequests.length, 0) + assert.strictEqual(sessions.length, 1, JSON.stringify({ + events: events.map(event => ({ + name: event.content?.name, + spanType: event.content?.meta?.['span.type'], + type: event.type, + })), + urls: payloads.map(({ url }) => url), + })) + assert.strictEqual(modules.length, 1) + assert.strictEqual(sessions[0].meta[TEST_ITR_SKIPPING_ENABLED], 'false') + assert.strictEqual(sessions[0].meta[TEST_CODE_COVERAGE_ENABLED], 'false') + assert.strictEqual(sessions[0].meta[TEST_EARLY_FLAKE_ENABLED], undefined) + assert.strictEqual(sessions[0].meta[TEST_MANAGEMENT_ENABLED], undefined) + + const metadata = cyclePayloads.flatMap(({ payload }) => payload.metadata || []) + assert.ok(metadata.length > 0) + for (const metadataEntry of metadata) { + const capabilities = Object.keys(metadataEntry.test || {}) + .filter(tag => tag.startsWith('_dd.library_capabilities.')) + assert.deepStrictEqual(capabilities, []) + } + + for (const event of [sessions[0], modules[0], ...suites, ...tests]) { + assert.strictEqual(event.meta[TEST_FRAMEWORK], 'webdriverio') + assert.strictEqual(event.meta[TEST_MODULE], 'webdriverio') + assert.strictEqual(event.meta[TEST_TYPE], 'browser') + assert.ok(event.meta[TEST_FRAMEWORK_VERSION]) + if (requestedVersion !== 'latest') { + assert.strictEqual(event.meta[TEST_FRAMEWORK_VERSION], requestedVersion) + } + } + for (const event of [...suites, ...tests]) { + assert.strictEqual(event.meta[TEST_FRAMEWORK_ADAPTER], 'mocha') + } + assertEventHierarchy(sessions[0], modules[0], suites, tests) + + return { + session: sessions[0], + module: modules[0], + suites, + tests, + } +} + +for (const version of versions) { + describe(`webdriverio@${version}`, function () { + this.timeout(60_000) + + let childProcess + let cwd + let receiver + let testOutput = '' + let webDriver + + useSandbox([ + `@wdio/cli@${version}`, + `@wdio/local-runner@${version}`, + `@wdio/mocha-framework@${version}`, + ], true, ['./integration-tests/webdriverio/fixtures/*']) + + before(async function () { + cwd = sandboxCwd() + webDriver = await startWebDriverServer() + }) + + after(async function () { + await stopServer(webDriver?.server) + }) + + beforeEach(async function () { + receiver = await new FakeCiVisIntake().start() + receiver.setSettings(advancedSettings) + }) + + afterEach(async function () { + childProcess?.kill() + testOutput = '' + await receiver.stop() + }) + + /** + * Runs one WebdriverIO configuration scenario. + * + * @param {string} scenario + * @param {number} expectedWebDriverSessions + * @param {(events: ReturnType) => void} assertEvents + * @param {number} [expectedExitCode] + * @returns {Promise} + */ + async function runScenario (scenario, expectedWebDriverSessions, assertEvents, expectedExitCode = 0) { + const initialWebDriverSessionCount = webDriver.getSessionCount() + childProcess = exec('./node_modules/.bin/wdio run ./wdio.conf.js', { + cwd, + env: { + ...getCiVisAgentlessConfig(receiver.port), + NODE_OPTIONS: '-r dd-trace/ci/init --import dd-trace/register.js', + DD_TEST_SESSION_NAME: 'webdriverio-integration-test', + WEBDRIVERIO_SCENARIO: scenario, + WEBDRIVER_PORT: String(webDriver.port), + }, + }) + childProcess.stdout?.on('data', chunk => { + testOutput += chunk.toString() + }) + childProcess.stderr?.on('data', chunk => { + testOutput += chunk.toString() + }) + + const payloadsPromise = receiver.gatherPayloadsUntilChildExit( + childProcess, + undefined, + payloads => assertEvents(getBasicReportingEvents(payloads, version)), + { hardTimeout: 45_000 } + ) + + let exitCode + try { + [[exitCode]] = await Promise.all([ + once(childProcess, 'exit'), + payloadsPromise, + ]) + } catch (error) { + error.message += `\n${testOutput}` + throw error + } + + assert.strictEqual(exitCode, expectedExitCode, testOutput) + assert.doesNotMatch(testOutput, /dd:test-optimization:webdriverio:/) + assert.doesNotMatch(testOutput, /\bundefined undefined undefined\b/) + assert.strictEqual( + webDriver.getSessionCount() - initialWebDriverSessionCount, + expectedWebDriverSessions + ) + } + + it('reports parallel workers as one basic-reporting session', async () => { + await runScenario('parallel', 2, ({ session, suites, tests }) => { + assert.strictEqual(suites.length, 2) + assert.strictEqual(tests.length, 2) + assert.strictEqual(session.meta[MOCHA_IS_PARALLEL], 'true') + assert.strictEqual(session.meta[TEST_STATUS], 'pass') + assert.deepStrictEqual( + suites.map(suite => suite.meta[TEST_SUITE]).sort(), + ['first.e2e.js', 'second.e2e.js'] + ) + assert.deepStrictEqual( + tests.map(test => test.meta['test.webdriverio.worker']).sort(), + ['first', 'second'] + ) + assert.strictEqual(new Set(tests.map(test => test.metrics.process_id)).size, 2) + }) + }) + + it('reports failures before Mocha loads', async () => { + await runScenario('preFrameworkFailure', 0, ({ session, suites, tests }) => { + assert.strictEqual(session.meta[TEST_STATUS], 'fail') + assert.strictEqual(suites.length, 0) + assert.strictEqual(tests.length, 0) + }, 1) + }) + + it('reports specs that fail while loading', async () => { + await runScenario('loadFailure', 1, ({ session, suites, tests }) => { + assert.strictEqual(session.meta[TEST_STATUS], 'fail') + assert.strictEqual(suites.length, 1) + assert.strictEqual(suites[0].meta[TEST_STATUS], 'fail') + assert.strictEqual(suites[0].meta[TEST_SUITE], 'load-fail.e2e.js') + assert.strictEqual(tests.length, 0) + }, 1) + }) + + it('reports sequential workers as one session', async () => { + await runScenario('serial', 2, ({ session, suites, tests }) => { + assert.strictEqual(suites.length, 2) + assert.strictEqual(tests.length, 2) + assert.strictEqual(session.meta[MOCHA_IS_PARALLEL], undefined) + assert.strictEqual(new Set(tests.map(test => test.metrics.process_id)).size, 2) + }) + }) + + it('reports grouped specs from one worker', async () => { + await runScenario('grouped', 1, ({ suites, tests }) => { + assert.strictEqual(suites.length, 2) + assert.strictEqual(tests.length, 2) + assert.strictEqual(new Set(tests.map(test => test.metrics.process_id)).size, 1) + }) + }) + + it('attributes a hook-only failure to its grouped spec', async () => { + await runScenario('hookFailure', 1, ({ session, suites, tests }) => { + assert.strictEqual(session.meta[TEST_STATUS], 'fail') + assert.strictEqual(suites.length, 2) + assert.strictEqual(tests.length, 1) + assert.deepStrictEqual( + suites.map(suite => [suite.meta[TEST_SUITE], suite.meta[TEST_STATUS]]).sort(), + [ + ['first.e2e.js', 'pass'], + ['hook-fail.e2e.js', 'fail'], + ] + ) + }, 1) + }) + + it('marks a spec filtered by mochaOpts.grep as skipped', async () => { + await runScenario('grep', 1, ({ suites, tests }) => { + assert.strictEqual(suites.length, 2) + assert.strictEqual(tests.length, 1) + assert.deepStrictEqual( + suites.map(suite => [suite.meta[TEST_SUITE], suite.meta[TEST_STATUS]]).sort(), + [ + ['first.e2e.js', 'pass'], + ['second.e2e.js', 'skip'], + ] + ) + }) + }) + + it('reports mochaOpts.bail without leaving grouped suites open', async () => { + await runScenario('bail', 1, ({ session, suites, tests }) => { + assert.strictEqual(session.meta[TEST_STATUS], 'fail') + assert.strictEqual(suites.length, 2) + assert.strictEqual(tests.length, 1) + assert.deepStrictEqual( + suites.map(suite => [suite.meta[TEST_SUITE], suite.meta[TEST_STATUS]]).sort(), + [ + ['fail.e2e.js', 'fail'], + ['second.e2e.js', 'skip'], + ] + ) + }, 1) + }) + + it('coordinates mochaOpts.delay with configuration loading', async () => { + await runScenario('delay', 1, ({ suites, tests }) => { + assert.strictEqual(suites.length, 1) + assert.strictEqual(tests.length, 1) + assert.strictEqual(tests[0].meta['test.webdriverio.worker'], 'delay') + }) + }) + + it('reports native Mocha retries', async () => { + await runScenario('retries', 1, ({ session, suites, tests }) => { + assert.strictEqual(session.meta[TEST_STATUS], 'pass') + assert.strictEqual(suites.length, 1) + assert.strictEqual(suites[0].meta[TEST_STATUS], 'pass') + assert.strictEqual(tests.length, 2) + assert.deepStrictEqual(tests.map(test => test.meta[TEST_STATUS]).sort(), ['fail', 'pass']) + assert.strictEqual(tests.filter(test => test.meta[TEST_IS_RETRY] === 'true').length, 1) + }) + }) + + it('preserves tracer preload with runnerEnv.NODE_OPTIONS', async () => { + await runScenario('runnerEnvNodeOptions', 1, ({ suites, tests }) => { + assert.strictEqual(suites.length, 1) + assert.strictEqual(tests.length, 1) + assert.strictEqual(tests[0].meta['test.webdriverio.worker'], 'runner-env-node-options') + }) + }) + + it('supports the Mocha TDD interface', async () => { + await runScenario('tdd', 1, ({ suites, tests }) => { + assert.strictEqual(suites.length, 1) + assert.strictEqual(tests.length, 1) + assert.strictEqual(tests[0].meta['test.webdriverio.worker'], 'tdd') + }) + }) + + it('reports whole-spec retries in one session', async () => { + await runScenario('specFileRetries', 2, ({ session, suites, tests }) => { + assert.strictEqual(session.meta[TEST_STATUS], 'pass') + assert.strictEqual(session.meta[MOCHA_IS_PARALLEL], undefined) + assert.strictEqual(suites.length, 2) + assert.strictEqual(tests.length, 2) + assert.deepStrictEqual(tests.map(test => test.meta[TEST_STATUS]).sort(), ['fail', 'pass']) + assertOneTestPerSuiteExecution(suites, tests) + + const suiteStatusById = new Map(suites.map(suite => [ + suite.test_suite_id.toString(10), + suite.meta[TEST_STATUS], + ])) + for (const test of tests) { + assert.strictEqual( + test.meta[TEST_STATUS], + suiteStatusById.get(test.test_suite_id.toString(10)) + ) + } + }) + }) + + it('reports multiple capabilities in one session', async () => { + await runScenario('multipleCapabilities', 2, ({ session, suites, tests }) => { + assert.strictEqual(suites.length, 2) + assert.strictEqual(tests.length, 2) + assert.strictEqual(session.meta[MOCHA_IS_PARALLEL], 'true') + assert.strictEqual(new Set(tests.map(test => test.metrics.process_id)).size, 2) + assertOneTestPerSuiteExecution(suites, tests) + }) + }) + }) +} diff --git a/package.json b/package.json index dcfff5dfa5..9dde20e592 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dd-trace", - "version": "6.7.0", + "version": "6.8.0", "description": "Datadog APM tracing client for JavaScript", "main": "index.js", "typings": "index.d.ts", @@ -96,6 +96,9 @@ "test:integration:playwright:coverage": "node ./integration-tests/coverage/run-suite.js --timeout 60000 \"integration-tests/playwright/${SPEC:-playwright-*}.spec.js\"", "test:integration:selenium": "mocha --timeout 60000 \"integration-tests/selenium/*.spec.js\"", "test:integration:selenium:coverage": "node ./integration-tests/coverage/run-suite.js --timeout 60000 \"integration-tests/selenium/*.spec.js\"", + "test:integration:test-optimization:helpers": "mocha \"integration-tests/cypress/*dependencies.spec.js\" \"integration-tests/jest/*dependencies.spec.js\" \"integration-tests/playwright/version*.spec.js\"", + "test:integration:webdriverio": "mocha --timeout 60000 \"integration-tests/webdriverio/*.spec.js\"", + "test:integration:webdriverio:coverage": "node ./integration-tests/coverage/run-suite.js --timeout 60000 \"integration-tests/webdriverio/*.spec.js\"", "test:integration:vitest": "mocha --timeout 60000 \"integration-tests/vitest/${SPEC:-vitest*}.spec.js\"", "test:integration:vitest:coverage": "node ./integration-tests/coverage/run-suite.js --timeout 60000 \"integration-tests/vitest/${SPEC:-vitest*}.spec.js\"", "test:integration:testopt": "mocha --timeout 120000 \"integration-tests/ci-visibility/*.spec.js\"", @@ -178,7 +181,7 @@ "@datadog/native-appsec": "11.0.1", "@datadog/native-iast-taint-tracking": "4.2.0", "@datadog/native-metrics": "3.1.2", - "@datadog/openfeature-node-server": "2.0.0", + "@datadog/openfeature-node-server": "2.0.2", "@datadog/pprof": "5.17.0", "@datadog/wasm-js-rewriter": "5.0.1", "@opentelemetry/api": ">=1.0.0 <1.10.0", @@ -199,6 +202,7 @@ "@types/node": "^18.19.106", "@types/sinon": "^22.0.0", "@vercel/nft": "^1.10.2", + "@yarnpkg/lockfile": "^1.1.0", "axios": "^1.18.1", "benchmark": "^2.1.4", "body-parser": "^2.3.0", @@ -213,7 +217,7 @@ "eslint-plugin-n": "^18.2.1", "eslint-plugin-promise": "^7.3.0", "eslint-plugin-sonarjs": "^4.2.0", - "eslint-plugin-unicorn": "^68.0.0", + "eslint-plugin-unicorn": "^72.0.0", "express": "^5.1.0", "glob": "^10.4.5", "globals": "^17.7.0", @@ -221,7 +225,9 @@ "husky": "^9.1.7", "istanbul-lib-report": "^3.0.0", "istanbul-reports": "^3.0.2", + "jsonc-parser": "^3.3.1", "jszip": "^3.10.1", + "minimatch": "^10.2.5", "mocha": "^11.7.6", "mocha-junit-reporter": "^2.2.1", "mocha-multi-reporters": "^1.5.1", diff --git a/packages/datadog-esbuild/index.js b/packages/datadog-esbuild/index.js index 675f6d5abc..e819ce7f70 100644 --- a/packages/datadog-esbuild/index.js +++ b/packages/datadog-esbuild/index.js @@ -265,10 +265,12 @@ ${build.initialOptions.banner.js}` } } // The file namespace is used when requiring files from disk in userland + if (extracted.pkg === null) return + let pathToPackageJson try { // we can't use require.resolve('pkg/package.json') as ESM modules don't make the file available - pathToPackageJson = require.resolve(`${extracted.pkg}`, { paths: [args.resolveDir] }) + pathToPackageJson = require.resolve(extracted.pkg, { paths: [args.resolveDir] }) pathToPackageJson = extractPackageAndModulePath(pathToPackageJson).pkgJson } catch (err) { if (err.code === 'MODULE_NOT_FOUND') { @@ -340,7 +342,7 @@ ${build.initialOptions.banner.js}` if (data.isESM) { if (args.path.endsWith(ESM_INTERCEPTED_SUFFIX)) { - args.path = args.path.slice(0, -1 * ESM_INTERCEPTED_SUFFIX.length) + args.path = args.path.slice(0, -ESM_INTERCEPTED_SUFFIX.length) if (data.internal) { args.path = args.path.slice(INTERNAL_ESM_INTERCEPTED_PREFIX.length) diff --git a/packages/datadog-esbuild/src/utils.js b/packages/datadog-esbuild/src/utils.js index 9289e88e11..b61db16959 100644 --- a/packages/datadog-esbuild/src/utils.js +++ b/packages/datadog-esbuild/src/utils.js @@ -174,7 +174,7 @@ async function processModule ({ path, internal = false, context, excludeDefault for (const n of exportNames) { if (n === 'default' && excludeDefault) continue - if (isStarExportLine(n) === true) { + if (isStarExportLine(n)) { // export * from 'wherever' const [, modFile] = n.split('* from ') diff --git a/packages/datadog-esbuild/test/plugin.spec.js b/packages/datadog-esbuild/test/plugin.spec.js index cf3cb48a35..a405a407f8 100644 --- a/packages/datadog-esbuild/test/plugin.spec.js +++ b/packages/datadog-esbuild/test/plugin.spec.js @@ -17,7 +17,37 @@ function captureOptionalPeerOnLoad () { return onLoad } +function captureOnResolve () { + let onResolve + ddPlugin.setup({ + initialOptions: {}, + /** + * @param {object} options + * @param {Function} callback + */ + onResolve (options, callback) { + onResolve = callback + }, + onLoad () {}, + }) + return onResolve +} + describe('datadog-esbuild plugin', () => { + it('ignores builtins without a package path', () => { + const onResolve = captureOnResolve() + + const result = onResolve({ + path: 'fs', + resolveDir: process.cwd(), + kind: 'require-call', + namespace: 'file', + importer: '/app/index.js', + }) + + assert.strictEqual(result, undefined) + }) + describe('optional peer bundling', () => { it('rewrites the installed peer load in require-provider into a literal require', () => { const onLoad = captureOptionalPeerOnLoad() diff --git a/packages/datadog-instrumentations/src/claude-agent-sdk.js b/packages/datadog-instrumentations/src/claude-agent-sdk.js index b512c3794d..0b64f3112e 100644 --- a/packages/datadog-instrumentations/src/claude-agent-sdk.js +++ b/packages/datadog-instrumentations/src/claude-agent-sdk.js @@ -54,7 +54,7 @@ function buildTracerHooks (sessionCtx) { sessionCtx.cwd = input.cwd sessionCtx.transcriptPath = input.transcript_path sessionCtx.agentType = input.agent_type - sessionCtx.permissionMode = sessionCtx.permissionMode || input.permission_mode + sessionCtx.permissionMode ||= input.permission_mode return {} } @@ -64,8 +64,8 @@ function buildTracerHooks (sessionCtx) { } function onUserPromptSubmit (input) { - sessionCtx.sessionId = sessionCtx.sessionId || input.session_id - sessionCtx.prompt = sessionCtx.prompt || input.prompt + sessionCtx.sessionId ||= input.session_id + sessionCtx.prompt ||= input.prompt return {} } @@ -254,7 +254,7 @@ function createStreamLookup (chunks) { scanLocalLifecycle(chunks, startIndex, toolUseId, localLifecycle) if (localLifecycle.toolResultIndex !== undefined) return localLifecycle - streamIndex = streamIndex || buildStreamIndex(chunks) + streamIndex ||= buildStreamIndex(chunks) const indexedLifecycle = streamIndex.get(toolUseId) return { taskStartedChunk: localLifecycle.taskStartedChunk || indexedLifecycle?.taskStartedChunk, diff --git a/packages/datadog-instrumentations/src/cucumber.js b/packages/datadog-instrumentations/src/cucumber.js index e15928078d..0971c18e02 100644 --- a/packages/datadog-instrumentations/src/cucumber.js +++ b/packages/datadog-instrumentations/src/cucumber.js @@ -6,6 +6,7 @@ const { createCoverageMap } = require('../../../vendor/dist/istanbul-lib-coverag const shimmer = require('../../datadog-shimmer') const log = require('../../dd-trace/src/log') const { getEnvironmentVariable } = require('../../dd-trace/src/config/helper') +const { getSegment } = require('../../dd-trace/src/util') const { getCoveredFilesFromCoverage, getExecutableFilesFromCoverage, @@ -342,7 +343,7 @@ function handleParallelTestCaseFinished (pickle, worstTestStepResult) { } const testFileAbsolutePath = pickle.uri - const finished = pickleResultByFile[testFileAbsolutePath] || (pickleResultByFile[testFileAbsolutePath] = []) + const finished = (pickleResultByFile[testFileAbsolutePath] ||= []) if (isEarlyFlakeDetectionEnabled && isNew) { const testFullname = `${pickle.uri}:${pickle.name}` @@ -555,8 +556,7 @@ function getErrorFromCucumberResult (cucumberResult) { return } - const [message] = cucumberResult.message.split('\n') - const error = new Error(message) + const error = new Error(getSegment(cucumberResult.message, '\n', 0)) if (cucumberResult.exception) { error.type = cucumberResult.exception.type } @@ -793,7 +793,7 @@ function wrapRun (pl, isLatestVersion, version) { testFnCh.runStores(ctx, () => { promise = run.apply(this, args) }) - promise.finally(async () => { + const finalize = async () => { if (!canAwaitRetries) { this.eventBroadcaster.removeListener('envelope', onEnvelope) } @@ -966,6 +966,9 @@ function wrapRun (pl, isLatestVersion, version) { ...attemptCtx.currentStore, finalStatus, }) + } + promise.then(finalize, finalize).catch(error => { + log.error('Cucumber test finalization error', error) }) return promise } catch (err) { diff --git a/packages/datadog-instrumentations/src/helpers/extract-package-and-module-path.js b/packages/datadog-instrumentations/src/helpers/extract-package-and-module-path.js index 0c79beebbf..b1b688560e 100644 --- a/packages/datadog-instrumentations/src/helpers/extract-package-and-module-path.js +++ b/packages/datadog-instrumentations/src/helpers/extract-package-and-module-path.js @@ -2,11 +2,21 @@ const NM = 'node_modules/' +/** + * @typedef {object} PackageAndModulePath + * @property {string|null} pkg + * @property {string|null} path + * @property {string} [pkgJson] + */ + /** * For a given full path to a module, * return the package name it belongs to and the local path to the module * input: '/foo/node_modules/@co/stuff/foo/bar/baz.js' * output: { pkg: '@co/stuff', path: 'foo/bar/baz.js', pkgJson: '/foo/node_modules/@co/stuff/package.json' } + * + * @param {string} fullPath + * @returns {PackageAndModulePath} */ module.exports = function extractPackageAndModulePath (fullPath) { const nm = fullPath.lastIndexOf(NM) diff --git a/packages/datadog-instrumentations/src/helpers/hook.js b/packages/datadog-instrumentations/src/helpers/hook.js index 32ccbc8853..9b0e57ac3d 100644 --- a/packages/datadog-instrumentations/src/helpers/hook.js +++ b/packages/datadog-instrumentations/src/helpers/hook.js @@ -82,7 +82,7 @@ function Hook (modules, hookOptions, onrequire) { moduleVersion ||= getVersion(moduleBaseDir) } catch (error) { log.error('Error getting version for "%s": %s', moduleName, error.message, error) - return + return moduleExports } if ( diff --git a/packages/datadog-instrumentations/src/helpers/hooks.js b/packages/datadog-instrumentations/src/helpers/hooks.js index f22a3b629d..7b2e2bfe52 100644 --- a/packages/datadog-instrumentations/src/helpers/hooks.js +++ b/packages/datadog-instrumentations/src/helpers/hooks.js @@ -58,6 +58,7 @@ module.exports = { '@smithy/core': () => require('../aws-sdk'), '@smithy/smithy-client': () => require('../aws-sdk'), '@vitest/runner': { esmFirst: true, fn: () => require('../vitest') }, + '@wdio/local-runner': { esmFirst: true, fn: () => require('../webdriverio') }, aerospike: () => require('../aerospike'), ai: () => require('../ai'), amqp10: () => require('../amqp10'), diff --git a/packages/datadog-instrumentations/src/helpers/register.js b/packages/datadog-instrumentations/src/helpers/register.js index bbf2a56526..fe3559cf5a 100644 --- a/packages/datadog-instrumentations/src/helpers/register.js +++ b/packages/datadog-instrumentations/src/helpers/register.js @@ -57,19 +57,23 @@ const alreadyLoggedIncompatibleIntegrations = new Set() // Always disable prefixed and unprefixed node modules if one is disabled. if (disabledInstrumentations.size) { const builtinsSet = new Set(builtinModules) + const disabledBuiltinCounterparts = [] for (const name of disabledInstrumentations) { const hasPrefix = name.startsWith('node:') if (hasPrefix || builtinsSet.has(name)) { if (hasPrefix) { const unprefixedName = name.slice(5) if (!disabledInstrumentations.has(unprefixedName)) { - disabledInstrumentations.add(unprefixedName) + disabledBuiltinCounterparts.push(unprefixedName) } } else if (!disabledInstrumentations.has(`node:${name}`)) { - disabledInstrumentations.add(`node:${name}`) + disabledBuiltinCounterparts.push(`node:${name}`) } } } + for (const name of disabledBuiltinCounterparts) { + disabledInstrumentations.add(name) + } builtinsSet.clear() } @@ -163,7 +167,9 @@ function logAbortedIntegrations () { for (const [nameVersion, success] of instrumentedIntegrationsSuccess) { // Only ever log a single version of an integration, even if it is loaded later. if (!success && !alreadyLoggedIncompatibleIntegrations.has(nameVersion)) { - const [name, version] = nameVersion.split('@') + const lastAtPosition = nameVersion.lastIndexOf('@') + const name = nameVersion.slice(0, lastAtPosition) + const version = nameVersion.slice(lastAtPosition + 1) telemetry('abort.integration', [ `integration:${name}`, `integration_version:${version}`, diff --git a/packages/datadog-instrumentations/src/helpers/rewriter/index.js b/packages/datadog-instrumentations/src/helpers/rewriter/index.js index aab0046ccf..8cc736ff6f 100644 --- a/packages/datadog-instrumentations/src/helpers/rewriter/index.js +++ b/packages/datadog-instrumentations/src/helpers/rewriter/index.js @@ -39,6 +39,7 @@ for (const matcher of [matcherCjs, matcherEsm]) { // Keep the marker split: source-map scanners can read a contiguous token in // string literals as this file's own inline map. +// eslint-disable-next-line unicorn/no-useless-concat -- Keep the source-map marker non-contiguous. const SOURCE_MAP_PREFIX = '//# sourceMapping' + 'URL=data:application/json;base64,' function rewrite (content, filename, format) { diff --git a/packages/datadog-instrumentations/src/helpers/rewriter/instrumentations/index.js b/packages/datadog-instrumentations/src/helpers/rewriter/instrumentations/index.js index e7f19fbb46..72f44da8ec 100644 --- a/packages/datadog-instrumentations/src/helpers/rewriter/instrumentations/index.js +++ b/packages/datadog-instrumentations/src/helpers/rewriter/instrumentations/index.js @@ -12,5 +12,6 @@ module.exports = [ ...require('./modelcontextprotocol-sdk'), ...require('./openai-agents'), ...require('./playwright'), + ...require('./webdriverio'), ...require('./aws-durable-execution-sdk-js'), ] diff --git a/packages/datadog-instrumentations/src/helpers/rewriter/instrumentations/webdriverio.js b/packages/datadog-instrumentations/src/helpers/rewriter/instrumentations/webdriverio.js new file mode 100644 index 0000000000..cf73ac7412 --- /dev/null +++ b/packages/datadog-instrumentations/src/helpers/rewriter/instrumentations/webdriverio.js @@ -0,0 +1,44 @@ +'use strict' + +module.exports = [ + { + module: { + name: '@wdio/local-runner', + versionRange: '>=9.0.0', + filePath: 'build/index.js', + }, + functionQuery: { + className: 'LocalRunner', + methodName: 'run', + kind: 'Async', + }, + channelName: 'LocalRunner_run', + }, + { + module: { + name: '@wdio/local-runner', + versionRange: '>=9.0.0', + filePath: 'build/index.js', + }, + functionQuery: { + className: 'LocalRunner', + methodName: 'shutdown', + kind: 'Async', + }, + channelName: 'LocalRunner_shutdown', + }, + { + module: { + name: '@wdio/local-runner', + versionRange: '>=9.0.0', + filePath: 'build/index.js', + }, + astQuery: 'VariableDeclarator[id.name="LocalRunner"] > ClassExpression > ClassBody > ' + + 'MethodDefinition[key.name="shutdown"] ReturnStatement > ' + + 'CallExpression[callee.object.name="promise"][callee.property.name="then"], ' + + 'ClassDeclaration[id.name="LocalRunner"] > ClassBody > MethodDefinition[key.name="shutdown"] ReturnStatement > ' + + 'CallExpression[callee.object.name="promise"][callee.property.name="then"]', + channelName: 'LocalRunner_shutdown', + transform: 'waitForAsyncEnd', + }, +] diff --git a/packages/datadog-instrumentations/src/helpers/rewriter/transforms.js b/packages/datadog-instrumentations/src/helpers/rewriter/transforms.js index b9c5b4c2ae..95cb988692 100644 --- a/packages/datadog-instrumentations/src/helpers/rewriter/transforms.js +++ b/packages/datadog-instrumentations/src/helpers/rewriter/transforms.js @@ -17,8 +17,8 @@ const { parse, query } = require('./compiler') module.exports = { waitForAsyncEnd } /** - * Injects a wait for `ctx.asyncEndPromise` into a generated `tracePromise` - * wrapper's native-Promise fulfillment handler. + * Injects settlement-specific asyncEnd waits into a generated `tracePromise` + * wrapper's native-Promise handlers. * * @param {object} _state * @param {import('estree').CallExpression} node @@ -26,36 +26,68 @@ module.exports = { waitForAsyncEnd } */ function waitForAsyncEnd (_state, node) { const onFulfilled = node.arguments[0] - const statements = onFulfilled?.body?.body + const onRejected = node.arguments[1] - if (!statements || query(onFulfilled.body, '[id.name=__apm$asyncEndPromise]').length > 0) { + if (!onFulfilled?.body || !onRejected?.body) { return } - const returnIndex = statements.findIndex(statement => - statement.type === 'ReturnStatement' && statement.argument + injectAsyncEndCallbackWait(onFulfilled.body, 'ReturnStatement', 'resolveCallback') + injectAsyncEndCallbackWait(onRejected.body, 'ThrowStatement', 'rejectCallback') +} + +/** + * Injects a settlement-specific callback wait before an exit. + * + * @param {import('estree').BlockStatement} body + * @param {'ReturnStatement'|'ThrowStatement'} exitType + * @param {'resolveCallback'|'rejectCallback'} callbackProperty + * @returns {void} + */ +function injectAsyncEndCallbackWait (body, exitType, callbackProperty) { + const callbackVariable = `__apm$${callbackProperty}` + if (query(body, `[id.name=${callbackVariable}]`).length > 0) { + return + } + + const exitIndex = body.body.findIndex(statement => + statement.type === exitType && statement.argument ) - // The generated fulfillment handler always ends in a return; a miss means the + // The generated settlement handlers always end in a return or throw; a miss means the // upstream template changed and the caller's try/catch falls back to the // unwrapped source. - assert(returnIndex !== -1, 'waitForAsyncEnd: no return statement to wait on') + assert(exitIndex !== -1, `waitForAsyncEnd: no ${exitType} to wait on`) + // This runs inside tracePromise's native-Promise settlement handler. The + // Promise adapts subscriber callback completion to that existing chain. const waitStatements = parse(` function wrapper () { - const __apm$asyncEndPromise = __apm$ctx.asyncEndPromise; - if (__apm$asyncEndPromise && typeof __apm$asyncEndPromise.then === 'function') { - return __apm$asyncEndPromise.then(() => __apm$result, () => __apm$result); + const ${callbackVariable} = __apm$ctx.${callbackProperty}; + if (typeof ${callbackVariable} === 'function') { + return new Promise(${callbackVariable}).then(() => __apm$result, () => __apm$result); } } `).body[0].body.body - // Resolve to whatever the fulfillment handler returns (its return argument), - // so a subscriber that reassigned `__apm$ctx.result` in `asyncEnd` still wins. - const returnArgument = statements[returnIndex].argument - const { arguments: onSettled } = waitStatements[1].consequent.body[0].argument - onSettled[0].body = clone(returnArgument) - onSettled[1].body = clone(returnArgument) + const exitArgument = body.body[exitIndex].argument + const callbackIf = waitStatements[1] + const { arguments: onCallbackSettled } = callbackIf.consequent.body[0].argument + + if (exitType === 'ThrowStatement') { + for (const handler of onCallbackSettled) { + handler.body = { + type: 'BlockStatement', + body: [{ + type: 'ThrowStatement', + argument: clone(exitArgument), + }], + } + } + } else { + onCallbackSettled[0].body = clone(exitArgument) + onCallbackSettled[1].body = clone(exitArgument) + } - statements.splice(returnIndex, 0, ...waitStatements) + body.body.splice(exitIndex, 0, ...waitStatements) } diff --git a/packages/datadog-instrumentations/src/helpers/router-helper.js b/packages/datadog-instrumentations/src/helpers/router-helper.js index 173e4ebdbd..73e2d6781a 100644 --- a/packages/datadog-instrumentations/src/helpers/router-helper.js +++ b/packages/datadog-instrumentations/src/helpers/router-helper.js @@ -88,7 +88,7 @@ function collectRoutesFromRouter (router, prefix) { function normalizeRoutePaths (path) { if (path == null) return [] - if (Array.isArray(path) === false) { + if (!Array.isArray(path)) { const normalized = normalizeRoutePath(path) return [normalized] } diff --git a/packages/datadog-instrumentations/src/http/client.js b/packages/datadog-instrumentations/src/http/client.js index c266c84479..48d98980f6 100644 --- a/packages/datadog-instrumentations/src/http/client.js +++ b/packages/datadog-instrumentations/src/http/client.js @@ -89,9 +89,7 @@ function setupResponseInstrumentation (ctx, res) { const collectChunk = chunk => { if (!bodyChunks || !chunk) return - if (typeof chunk === 'string') { - bodyChunks.push(chunk) - } else if (Buffer.isBuffer(chunk)) { + if (typeof chunk === 'string' || Buffer.isBuffer(chunk)) { bodyChunks.push(chunk) } else { // Handle Uint8Array or other array-like types diff --git a/packages/datadog-instrumentations/src/jest.js b/packages/datadog-instrumentations/src/jest.js index 0cd20675b4..d683e9c630 100644 --- a/packages/datadog-instrumentations/src/jest.js +++ b/packages/datadog-instrumentations/src/jest.js @@ -96,7 +96,7 @@ const DD_JEST_HANDLE_TEST_EVENT_WRAPPED = Symbol('dd-trace:jest:handle-test-even const DD_JEST_HANDLE_TEST_EVENT_DATADOG = Symbol('dd-trace:jest:handle-test-event-datadog') const DD_JEST_CONCURRENT_TEST_ORIGINAL = Symbol('dd-trace:jest:concurrent-test-original') const isJestWorker = !!getEnvironmentVariable('JEST_WORKER_ID') -const jestSessionState = globalThis[JEST_SESSION_STATE] || (globalThis[JEST_SESSION_STATE] = {}) +const jestSessionState = (globalThis[JEST_SESSION_STATE] ||= {}) // https://github.com/jestjs/jest/blob/41f842a46bb2691f828c3a5f27fc1d6290495b82/packages/jest-circus/src/types.ts#L9C8-L9C54 const RETRY_TIMES = Symbol.for('RETRY_TIMES') @@ -1192,9 +1192,9 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { ctx.testParameters = testParameters ctx.frameworkVersion = jestVersion ctx.isNew = isNewTest - ctx.isEfdRetry = ctx.isEfdRetry || numEfdRetry > 0 + ctx.isEfdRetry ||= numEfdRetry > 0 ctx.isAttemptToFix = isAttemptToFix - ctx.isAttemptToFixRetry = ctx.isAttemptToFixRetry || numOfAttemptsToFixRetries > 0 + ctx.isAttemptToFixRetry ||= numOfAttemptsToFixRetries > 0 ctx.isJestRetry = isJestRetry ctx.isDisabled = isDisabled ctx.isQuarantined = isQuarantined @@ -1361,8 +1361,7 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { } } } - } - if (event.name === 'test_done') { + } else if (event.name === 'test_done') { const originalError = event.test?.errors?.[0] let status = 'pass' if (event.test.errors && event.test.errors.length) { @@ -1565,8 +1564,7 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { if (ctx.concurrentTestState) { ctx.currentStore = undefined } - } - if (event.name === 'run_finish') { + } else if (event.name === 'run_finish') { for (const [test, errors] of atrSuppressedErrors) { // Do not restore errors for non-ATF quarantined tests — they should stay suppressed // so Jest doesn't see the failure (prevents --bail from stopping the run). @@ -1595,8 +1593,7 @@ function getWrappedEnvironment (BaseEnvironment, jestVersion) { attemptToFixRetriedTestsStatuses.clear() testsToBeRetried.clear() testSuiteDatadogEnvironments.delete(this.testSuiteAbsolutePath) - } - if (event.name === 'test_skip' || event.name === 'test_todo') { + } else if (event.name === 'test_skip' || event.name === 'test_todo') { const testName = getJestTestName(event.test) testSkippedCh.publish({ test: { @@ -2298,12 +2295,11 @@ function getCliWrapper (isNewJestVersion) { } = skippableSuitesResponse || await getChannelPromise(skippableSuitesCh) if (err) { skippableSuitesCoverage = {} - skippedSuitesCoverage = {} } else { skippableSuites = receivedSkippableSuites skippableSuitesCoverage = receivedSkippableSuitesCoverage || {} - skippedSuitesCoverage = {} } + skippedSuitesCoverage = {} } catch (err) { log.error('Jest test-suite skippable error', err) } @@ -3037,9 +3033,7 @@ function wrapJestObject (jestObject, suiteFilePath) { shimmer.wrap(jestObject, 'mock', mock => function (moduleName) { // If the library is mocked with `jest.mock`, we don't want to bypass jest's own require engine - if (LIBRARIES_BYPASSING_JEST_REQUIRE_ENGINE.has(moduleName)) { - LIBRARIES_BYPASSING_JEST_REQUIRE_ENGINE.delete(moduleName) - } + LIBRARIES_BYPASSING_JEST_REQUIRE_ENGINE.delete(moduleName) recordMockedFile(suiteFilePath, moduleName) return mock.apply(this, arguments) }) diff --git a/packages/datadog-instrumentations/src/mocha/common.js b/packages/datadog-instrumentations/src/mocha/common.js index 975e5c1cff..6e77efc44a 100644 --- a/packages/datadog-instrumentations/src/mocha/common.js +++ b/packages/datadog-instrumentations/src/mocha/common.js @@ -41,11 +41,9 @@ addHook({ }, (Suite) => { shimmer.wrap(Suite.prototype, 'addTest', addTest => function (test) { const callSites = getCallSites() - let startLine const testCallSite = callSites.find(site => site.getFileName() === test.file) if (testCallSite) { - startLine = testCallSite.getLineNumber() - testToStartLine.set(test, startLine) + testToStartLine.set(test, testCallSite.getLineNumber()) } return addTest.apply(this, arguments) }) diff --git a/packages/datadog-instrumentations/src/mocha/webdriverio-protocol.js b/packages/datadog-instrumentations/src/mocha/webdriverio-protocol.js new file mode 100644 index 0000000000..c7a7353b2f --- /dev/null +++ b/packages/datadog-instrumentations/src/mocha/webdriverio-protocol.js @@ -0,0 +1,19 @@ +'use strict' + +const { + createWebdriverioWorkerMessage, + WEBDRIVERIO_WORKER_ENV, + WEBDRIVERIO_WORKER_EVENT, + WEBDRIVERIO_WORKER_ORIGIN, +} = require('../../../dd-trace/src/ci-visibility/exporters/test-worker/webdriverio') + +module.exports = { + CONFIGURATION_REQUEST: 'dd:test-optimization:webdriverio:configuration:request', + CONFIGURATION_RESPONSE: 'dd:test-optimization:webdriverio:configuration:response', + createWebdriverioWorkerMessage, + SUITE_FINISH: 'dd:test-optimization:webdriverio:test-suite:finish', + WORKER_READY: 'dd:test-optimization:webdriverio:worker:ready', + WEBDRIVERIO_WORKER_ENV, + WEBDRIVERIO_WORKER_EVENT, + WEBDRIVERIO_WORKER_ORIGIN, +} diff --git a/packages/datadog-instrumentations/src/mocha/worker.js b/packages/datadog-instrumentations/src/mocha/worker.js index e14c321922..112c4479ca 100644 --- a/packages/datadog-instrumentations/src/mocha/worker.js +++ b/packages/datadog-instrumentations/src/mocha/worker.js @@ -2,6 +2,8 @@ const { addHook, channel } = require('../helpers/instrument') const shimmer = require('../../../datadog-shimmer') +const { getEnvironmentVariable } = require('../../../dd-trace/src/config/helper') +const log = require('../../../dd-trace/src/log') const { DD_MAJOR } = require('../../../../version') const { @@ -15,6 +17,14 @@ const { getRunTestsWrapper, patchFailedTestReplayHookUp, } = require('./utils') +const { + CONFIGURATION_REQUEST, + CONFIGURATION_RESPONSE, + createWebdriverioWorkerMessage, + SUITE_FINISH, + WEBDRIVERIO_WORKER_ENV, + WORKER_READY, +} = require('./webdriverio-protocol') require('./common') const MINIMUM_MOCHA_VERSION = DD_MAJOR >= 6 ? '>=8.0.0' : '>=5.2.0' @@ -22,6 +32,284 @@ const MINIMUM_MOCHA_VERSION = DD_MAJOR >= 6 ? '>=8.0.0' : '>=5.2.0' const workerFinishCh = channel('ci:mocha:worker:finish') const config = {} +const runnerToFiles = new WeakMap() +const runnerToFailedHookFiles = new WeakMap() +const isWebdriverioWorker = !!getEnvironmentVariable(WEBDRIVERIO_WORKER_ENV) +let configurationRequestId = 0 + +/** + * Sends a message to the WebdriverIO launcher without surfacing closed IPC channels. + * + * @param {object} message + * @param {() => void} [onError] + * @param {() => void} [onDone] + * @returns {void} + */ +function sendWebdriverioMessage (message, onError, onDone) { + if (!process.send || !process.connected) { + onError?.() + onDone?.() + return + } + + process.send(createWebdriverioWorkerMessage(message), (error) => { + if (error) { + log.error('WebdriverIO Test Optimization IPC error', error) + onError?.() + } + onDone?.() + }) +} + +/** + * Applies configuration encoded as private Mocha options by its parallel runner. + * + * @param {object} options + * @returns {void} + */ +function applyMochaOptions (options) { + if (options._ddIsKnownTestsEnabled) { + config.isKnownTestsEnabled = true + config.isEarlyFlakeDetectionEnabled = options._ddIsEfdEnabled + config.knownTests = options._ddKnownTests + config.earlyFlakeDetectionNumRetries = options._ddEfdNumRetries + config.earlyFlakeDetectionSlowTestRetries = options._ddEfdSlowTestRetries ?? {} + delete options._ddIsEfdEnabled + delete options._ddKnownTests + delete options._ddEfdNumRetries + delete options._ddEfdSlowTestRetries + delete options._ddIsKnownTestsEnabled + } + if (options._ddIsImpactedTestsEnabled) { + config.isImpactedTestsEnabled = true + config.modifiedFiles = options._ddModifiedFiles + delete options._ddIsImpactedTestsEnabled + delete options._ddModifiedFiles + } + if (options._ddIsTestManagementTestsEnabled) { + config.isTestManagementTestsEnabled = true + config.testManagementAttemptToFixRetries = options._ddTestManagementAttemptToFixRetries + config.testManagementTests = options._ddTestManagementTests + delete options._ddIsTestManagementTestsEnabled + delete options._ddTestManagementAttemptToFixRetries + delete options._ddTestManagementTests + } + if (options._ddIsFlakyTestRetriesEnabled) { + config.isFlakyTestRetriesEnabled = true + config.flakyTestRetriesCount = options._ddFlakyTestRetriesCount + delete options._ddIsFlakyTestRetriesEnabled + delete options._ddFlakyTestRetriesCount + } + if (options._ddIsFailedTestReplayEnabled) { + config.isTestDynamicInstrumentationEnabled = true + config.isDiEnabled = true + delete options._ddIsFailedTestReplayEnabled + } +} + +/** + * Removes files selected for suite-level skipping from an already loaded Mocha root suite. + * + * @param {object} runner + * @param {string[]} skippedFiles + * @returns {void} + */ +function filterSkippedFiles (runner, skippedFiles) { + if (!skippedFiles.length) { + return + } + + const skippedFilesSet = new Set(skippedFiles) + runner.suite.suites = runner.suite.suites.filter(suite => !skippedFilesSet.has(suite.file)) + runner.suite.tests = runner.suite.tests.filter(test => !skippedFilesSet.has(test.file)) +} + +/** + * Requests configuration from the WebdriverIO launcher before releasing Mocha's delayed root suite. + * + * @param {string} frameworkVersion + * @param {string[]} files + * @param {(response: object) => void} onDone + * @returns {void} + */ +function requestWebdriverioConfiguration (frameworkVersion, files, onDone) { + const requestId = `${process.pid}-${++configurationRequestId}` + let finished = false + + /** + * Finishes the configuration request exactly once. + * + * @param {object} response + * @returns {void} + */ + function finish (response) { + if (finished) { + return + } + finished = true + clearTimeout(timeout) + process.off('message', onMessage) + process.off('disconnect', onDisconnect) + onDone(response) + } + + /** + * Receives the matching coordinator response. + * + * @param {object} message + * @returns {void} + */ + function onMessage (message) { + if (message?.name === CONFIGURATION_RESPONSE && message.content?.requestId === requestId) { + finish(message.content) + } + } + + /** + * Releases the runner if its parent disconnects. + * + * @returns {void} + */ + function onDisconnect () { + finish({}) + } + + const timeout = setTimeout(() => finish({}), 30_000) + process.on('message', onMessage) + process.once('disconnect', onDisconnect) + sendWebdriverioMessage({ + origin: 'datadog', + name: CONFIGURATION_REQUEST, + content: { + files, + frameworkVersion, + requestId, + }, + }, () => finish({})) +} + +/** + * Reports the Mocha version as soon as WebdriverIO loads its framework adapter. + * + * @param {string} frameworkVersion + * @returns {void} + */ +function reportWebdriverioWorkerReady (frameworkVersion) { + if (!isWebdriverioWorker) { + return + } + + sendWebdriverioMessage({ + origin: 'datadog', + name: WORKER_READY, + content: { frameworkVersion }, + }) +} + +/** + * Computes a final result for every file loaded into one Mocha worker. + * + * @param {object} runner + * @returns {object[]} + */ +function getWebdriverioSuiteResults (runner) { + const resultsByFile = new Map() + const files = runnerToFiles.get(runner) || [] + + for (const file of files) { + resultsByFile.set(file, { + file, + hasPassingTest: false, + status: 'skip', + }) + } + + runner.suite.eachTest(test => { + const result = resultsByFile.get(test.file) + if (!result) { + return + } + if (test.state === 'failed' || test.timedOut || test._ddHookFailed) { + result.status = 'fail' + } else if (test.state === 'passed') { + result.hasPassingTest = true + } + }) + + for (const file of runnerToFailedHookFiles.get(runner) || []) { + const result = resultsByFile.get(file) + if (result) { + result.status = 'fail' + } + } + + const results = [] + let hasFailedSuite = false + for (const result of resultsByFile.values()) { + if (result.status === 'fail') { + hasFailedSuite = true + } else if (result.hasPassingTest) { + result.status = 'pass' + } + delete result.hasPassingTest + results.push(result) + } + + if (runner.failures > 0 && !hasFailedSuite) { + for (const result of results) { + result.status = 'fail' + } + } + + return results +} + +/** + * Sends suite results to the WebdriverIO launcher. + * + * @param {object} runner + * @param {() => void} [onDone] + * @returns {void} + */ +function reportWebdriverioSuiteResults (runner, onDone) { + if (!isWebdriverioWorker) { + onDone?.() + return + } + + sendWebdriverioMessage({ + origin: 'datadog', + name: SUITE_FINISH, + content: { + results: getWebdriverioSuiteResults(runner), + }, + }, undefined, onDone) +} + +/** + * Flushes worker payloads before reporting suite results and completing Mocha. + * + * @param {object} runner + * @param {() => void} onDone + * @returns {void} + */ +function finishWebdriverioWorker (runner, onDone) { + try { + workerFinishCh.publish({ + onDone: () => { + try { + reportWebdriverioSuiteResults(runner, onDone) + } catch (error) { + log.error('WebdriverIO Test Optimization worker completion error', error) + onDone() + } + }, + }) + } catch (error) { + log.error('WebdriverIO Test Optimization worker completion error', error) + onDone() + } +} function isFailedTestReplayEnabled () { return config.isTestDynamicInstrumentationEnabled && config.isDiEnabled @@ -31,46 +319,72 @@ addHook({ name: 'mocha', versions: ['>=8.0.0'], file: 'lib/mocha.js', -}, (Mocha) => { +}, (Mocha, frameworkVersion) => { + reportWebdriverioWorkerReady(frameworkVersion) + shimmer.wrap(Mocha.prototype, 'run', run => function (...args) { - if (this.options._ddIsKnownTestsEnabled) { - config.isKnownTestsEnabled = true - config.isEarlyFlakeDetectionEnabled = this.options._ddIsEfdEnabled - config.knownTests = this.options._ddKnownTests - config.earlyFlakeDetectionNumRetries = this.options._ddEfdNumRetries - config.earlyFlakeDetectionSlowTestRetries = this.options._ddEfdSlowTestRetries ?? {} - delete this.options._ddIsEfdEnabled - delete this.options._ddKnownTests - delete this.options._ddEfdNumRetries - delete this.options._ddEfdSlowTestRetries - delete this.options._ddIsKnownTestsEnabled - } - if (this.options._ddIsImpactedTestsEnabled) { - config.isImpactedTestsEnabled = true - config.modifiedFiles = this.options._ddModifiedFiles - delete this.options._ddIsImpactedTestsEnabled - delete this.options._ddModifiedFiles - } - if (this.options._ddIsTestManagementTestsEnabled) { - config.isTestManagementTestsEnabled = true - config.testManagementAttemptToFixRetries = this.options._ddTestManagementAttemptToFixRetries - config.testManagementTests = this.options._ddTestManagementTests - delete this.options._ddIsTestManagementTestsEnabled - delete this.options._ddTestManagementAttemptToFixRetries - delete this.options._ddTestManagementTests + applyMochaOptions(this.options) + if (!isWebdriverioWorker || !workerFinishCh.hasSubscribers) { + return run.apply(this, args) } - if (this.options._ddIsFlakyTestRetriesEnabled) { - config.isFlakyTestRetriesEnabled = true - config.flakyTestRetriesCount = this.options._ddFlakyTestRetriesCount - delete this.options._ddIsFlakyTestRetriesEnabled - delete this.options._ddFlakyTestRetriesCount + + const isUserDelayed = this.options.delay + const rootSuite = this.suite + const rootSuiteRun = rootSuite.run + const hasOwnRootSuiteRun = Object.hasOwn(rootSuite, 'run') + let configurationReady = false + let userReady = !isUserDelayed + + /** + * Restores the root suite method after both delayed-mode gates are open. + * + * @returns {void} + */ + function restoreRootSuiteRun () { + if (hasOwnRootSuiteRun) { + rootSuite.run = rootSuiteRun + } else { + delete rootSuite.run + } } - if (this.options._ddIsFailedTestReplayEnabled) { - config.isTestDynamicInstrumentationEnabled = true - config.isDiEnabled = true - delete this.options._ddIsFailedTestReplayEnabled + + if (isUserDelayed) { + rootSuite.run = function (...args) { + userReady = true + if (!configurationReady) { + return + } + restoreRootSuiteRun() + return rootSuiteRun.apply(this, args) + } } - return run.apply(this, args) + + this.options.delay = true + const files = [...this.files] + const runner = run.apply(this, args) + runnerToFiles.set(runner, files) + + requestWebdriverioConfiguration(frameworkVersion, files, ({ + configuration, + skippedFiles = [], + }) => { + if (configuration) { + Object.assign(config, configuration) + } + filterSkippedFiles(runner, skippedFiles) + if (isFailedTestReplayEnabled()) { + patchFailedTestReplayHookUp(runner.constructor) + } + configurationReady = true + if (userReady) { + if (isUserDelayed) { + restoreRootSuiteRun() + } + rootSuite.run() + } + }) + + return runner }) return Mocha @@ -91,10 +405,22 @@ addHook({ if (isFailedTestReplayEnabled()) { patchFailedTestReplayHookUp(Runner) } - // We flush when the worker ends with its test file (a mocha instance in a worker runs a single test file) - this.once('end', () => { - workerFinishCh.publish() - }) + const onRunDone = args[0] + if (isWebdriverioWorker && typeof onRunDone === 'function') { + args[0] = (...onRunDoneArgs) => { + finishWebdriverioWorker(this, () => onRunDone(...onRunDoneArgs)) + } + } else { + // Flush after the worker finishes its Mocha run, including grouped spec files. + this.once('end', () => { + try { + workerFinishCh.publish() + reportWebdriverioSuiteResults(this) + } catch (error) { + log.error('WebdriverIO Test Optimization worker completion error', error) + } + }) + } this.on('test', getOnTestHandler(false)) this.on('test end', getOnTestEndHandler(config)) @@ -104,6 +430,15 @@ addHook({ // If the hook passes, 'hook end' will be emitted. Otherwise, 'fail' will be emitted this.on('hook end', getOnHookEndHandler(config)) + if (isWebdriverioWorker) { + const failedHookFiles = new Set() + runnerToFailedHookFiles.set(this, failedHookFiles) + this.on('fail', runnable => { + if (runnable.type === 'hook' && runnable.file) { + failedHookFiles.add(runnable.file) + } + }) + } this.on('fail', getOnFailHandler(false, config)) this.on('pending', getOnPendingHandler()) diff --git a/packages/datadog-instrumentations/src/moleculer/client.js b/packages/datadog-instrumentations/src/moleculer/client.js index eb0c0d72fc..9d31bb2bc6 100644 --- a/packages/datadog-instrumentations/src/moleculer/client.js +++ b/packages/datadog-instrumentations/src/moleculer/client.js @@ -10,7 +10,7 @@ const errorChannel = channel('apm:moleculer:call:error') function wrapCall (call) { return function (actionName, params, opts) { opts = arguments[2] = opts || {} - opts.meta = opts.meta || {} + opts.meta ||= {} arguments.length = Math.max(3, arguments.length) diff --git a/packages/datadog-instrumentations/src/playwright.js b/packages/datadog-instrumentations/src/playwright.js index 8d153b083e..7d46ddec6f 100644 --- a/packages/datadog-instrumentations/src/playwright.js +++ b/packages/datadog-instrumentations/src/playwright.js @@ -1350,7 +1350,7 @@ function runAllTestsWrapper (runAllTests, playwrightVersion) { let totalFailedTestCount = 0 let totalPureQuarantinedFailedTestCount = 0 - for (const [fqn, testStatuses] of testsToTestStatuses.entries()) { + for (const [fqn, testStatuses] of testsToTestStatuses) { // Only count as failed if the final status (after retries) is 'fail' const lastStatus = testStatuses.at(-1) if (lastStatus === 'fail') { @@ -1494,7 +1494,8 @@ createRootSuiteCh.subscribe({ pageGotoCh.subscribe({ asyncEnd (ctx) { // The Page.goto rewriter waits for this so tests closing immediately after navigation still get RUM tags. - ctx.asyncEndPromise = handlePageGoto(ctx.self) + const rumDetectionPromise = handlePageGoto(ctx.self) + ctx.resolveCallback = onDone => rumDetectionPromise.then(onDone, onDone) }, }) diff --git a/packages/datadog-instrumentations/src/rhea.js b/packages/datadog-instrumentations/src/rhea.js index 135b381866..2efb910eb2 100644 --- a/packages/datadog-instrumentations/src/rhea.js +++ b/packages/datadog-instrumentations/src/rhea.js @@ -222,7 +222,7 @@ function beforeFinish (delivery, state) { } function getStateFromData (stateData) { - if (stateData && stateData.descriptor && stateData.descriptor) { + if (stateData?.descriptor) { switch (stateData.descriptor.value) { case 0x24: return 'accepted' case 0x25: return 'rejected' diff --git a/packages/datadog-instrumentations/src/webdriverio.js b/packages/datadog-instrumentations/src/webdriverio.js new file mode 100644 index 0000000000..ca92df8c4d --- /dev/null +++ b/packages/datadog-instrumentations/src/webdriverio.js @@ -0,0 +1,691 @@ +'use strict' + +const { AsyncResource } = require('node:async_hooks') +const { fileURLToPath } = require('node:url') + +const { getEnvironmentVariable } = require('../../dd-trace/src/config/helper') +const log = require('../../dd-trace/src/log') +const { + MOCHA_WORKER_TRACE_PAYLOAD_CODE, + TEST_SUITE_EXECUTION_ID, +} = require('../../dd-trace/src/plugins/util/test') +const { addHook, channel, tracingChannel } = require('./helpers/instrument') +const { + CONFIGURATION_REQUEST, + CONFIGURATION_RESPONSE, + SUITE_FINISH, + WEBDRIVERIO_WORKER_ENV, + WEBDRIVERIO_WORKER_EVENT, + WEBDRIVERIO_WORKER_ORIGIN, + WORKER_READY, +} = require('./mocha/webdriverio-protocol') + +const testFinishCh = channel('ci:mocha:test:finish') +const testSessionStartCh = channel('ci:mocha:session:start') +const testSessionFinishCh = channel('ci:mocha:session:finish') +const testSuiteStartCh = channel('ci:mocha:test-suite:start') +const testSuiteFinishCh = channel('ci:mocha:test-suite:finish') +const libraryConfigurationCh = channel('ci:mocha:library-configuration') +const workerReportTraceCh = channel('ci:mocha:worker-report:trace') + +const localRunnerRunCh = tracingChannel('orchestrion:@wdio/local-runner:LocalRunner_run') +const localRunnerShutdownCh = tracingChannel('orchestrion:@wdio/local-runner:LocalRunner_shutdown') + +const NODE_OPTIONS_SEPARATOR_RE = /\s/ +const TEST_FRAMEWORK = 'webdriverio' + +const loadCh = channel('dd-trace:instrumentation:load') +if (loadCh.hasSubscribers) { + loadCh.publish({ name: '@wdio/local-runner' }) +} + +const coordinatorStates = new WeakMap() +const localRunnerVersions = new WeakMap() + +addHook({ + name: '@wdio/local-runner', + versions: ['>=9.0.0'], + file: 'build/index.js', + patchDefault: true, +}, (LocalRunner, version) => { + localRunnerVersions.set(LocalRunner, version) + return LocalRunner +}) + +/** + * @typedef {object} WebdriverioRunnerConfig + * @property {string} framework + * @property {string|undefined} rootDir + * @property {NodeJS.ProcessEnv|undefined} runnerEnv + */ + +/** + * @typedef {object} WebdriverioLocalRunner + * @property {WebdriverioRunnerConfig|undefined} config + * @property {WebdriverioRunnerConfig|undefined} _config + */ + +/** + * @typedef {object} WorkerRecord + * @property {object} worker + * @property {string[]} specs + * @property {Map} suiteContexts + * @property {string} testSuiteExecutionId + * @property {boolean|undefined} hasTests + * @property {number|undefined} exitCode + * @property {number|undefined} retries + */ + +/** + * @typedef {object} WebdriverioSuiteContext + * @property {object|undefined} currentStore + * @property {string|undefined} status + * @property {string} testSuiteAbsolutePath + * @property {string} testSuiteExecutionId + */ + +/** + * @typedef {object} CoordinatorState + * @property {WebdriverioLocalRunner} localRunner + * @property {AsyncResource} asyncResource + * @property {object} configuration + * @property {boolean} initialized + * @property {boolean} initializing + * @property {Array<(configuration: object) => void>} initializationCallbacks + * @property {boolean} sessionStarted + * @property {boolean} finished + * @property {string|undefined} frameworkVersion + * @property {string} testFrameworkAdapter + * @property {number} activeWorkers + * @property {number} maxActiveWorkers + * @property {number} nextWorkerId + * @property {unknown} runError + * @property {Set} workers + * @property {Map} suiteStatuses + */ + +/** + * Creates the basic-reporting configuration consumed by Mocha workers. + * + * Advanced Test Optimization features remain disabled until they have WebdriverIO-specific coverage. + * + * @returns {object} + */ +function createWorkerConfiguration () { + return { + earlyFlakeDetectionNumRetries: 0, + earlyFlakeDetectionSlowTestRetries: {}, + flakyTestRetriesCount: 0, + isCodeCoverageEnabled: false, + isCoverageReportUploadEnabled: false, + isDiEnabled: false, + isEarlyFlakeDetectionEnabled: false, + isFlakyTestRetriesEnabled: false, + isImpactedTestsEnabled: false, + isItrEnabled: false, + isKnownTestsEnabled: false, + isSuitesSkippingEnabled: false, + isTestDynamicInstrumentationEnabled: false, + isTestManagementTestsEnabled: false, + knownTests: {}, + modifiedFiles: [], + repositoryRoot: undefined, + testManagementAttemptToFixRetries: 0, + testManagementTests: {}, + } +} + +/** + * Gets the public runner configuration, or the private equivalent used by older releases. + * + * @param {WebdriverioLocalRunner} localRunner + * @returns {WebdriverioRunnerConfig|undefined} + */ +function getRunnerConfiguration (localRunner) { + return localRunner.config || localRunner._config +} + +/** + * Gets or creates coordinator state for a WebdriverIO local runner. + * + * @param {WebdriverioLocalRunner} localRunner + * @returns {CoordinatorState} + */ +function getCoordinatorState (localRunner) { + let state = coordinatorStates.get(localRunner) + if (state) { + return state + } + + state = { + localRunner, + asyncResource: new AsyncResource('dd-trace-webdriverio-coordinator'), + configuration: createWorkerConfiguration(), + initialized: false, + initializing: false, + initializationCallbacks: [], + sessionStarted: false, + finished: false, + frameworkVersion: localRunnerVersions.get(localRunner.constructor), + testFrameworkAdapter: getRunnerConfiguration(localRunner)?.framework, + activeWorkers: 0, + maxActiveWorkers: 0, + nextWorkerId: 0, + runError: undefined, + workers: new Set(), + suiteStatuses: new Map(), + } + coordinatorStates.set(localRunner, state) + + return state +} + +/** + * Normalizes a WebdriverIO spec identifier to a filesystem path. + * + * @param {string} file + * @returns {string} + */ +function normalizeFile (file) { + return file.startsWith('file://') ? fileURLToPath(file) : file +} + +/** + * Checks whether worker NODE_OPTIONS contain the complete launcher options. + * + * @param {string|undefined} workerNodeOptions + * @param {string} launcherNodeOptions + * @returns {boolean} + */ +function includesNodeOptions (workerNodeOptions, launcherNodeOptions) { + if (!workerNodeOptions) { + return false + } + + let index = workerNodeOptions.indexOf(launcherNodeOptions) + while (index !== -1) { + const endIndex = index + launcherNodeOptions.length + const startsAtBoundary = index === 0 || NODE_OPTIONS_SEPARATOR_RE.test(workerNodeOptions[index - 1]) + const endsAtBoundary = endIndex === workerNodeOptions.length || + NODE_OPTIONS_SEPARATOR_RE.test(workerNodeOptions[endIndex]) + + if (startsAtBoundary && endsAtBoundary) { + return true + } + index = workerNodeOptions.indexOf(launcherNodeOptions, index + 1) + } + return false +} + +/** + * Starts the single Mocha session owned by the WebdriverIO launcher. + * + * @param {CoordinatorState} state + * @returns {void} + */ +function startSession (state) { + if (state.sessionStarted) { + return + } + const processArgv = process.argv.slice(2).join(' ') + const command = processArgv ? `wdio ${processArgv}` : 'wdio' + const rootDir = getRunnerConfiguration(state.localRunner)?.rootDir || process.cwd() + + testSessionStartCh.publish({ + command, + frameworkVersion: state.frameworkVersion, + rootDir, + testFramework: TEST_FRAMEWORK, + testFrameworkAdapter: state.testFrameworkAdapter, + }) + state.sessionStarted = true +} + +/** + * Completes coordinator initialization and releases waiting workers. + * + * @param {CoordinatorState} state + * @param {object|undefined} response + * @returns {void} + */ +function completeCoordinatorInitialization (state, response) { + if (state.initialized) { + return + } + + state.configuration.repositoryRoot = response?.repositoryRoot + state.initialized = true + state.initializing = false + startSession(state) + + const callbacks = state.initializationCallbacks + state.initializationCallbacks = [] + for (const callback of callbacks) { + callback(state.configuration) + } +} + +/** + * Requests settings once so the launcher owns initialization for every worker. + * + * Settings for advanced features are intentionally ignored while WebdriverIO support is basic-reporting only. + * + * @param {CoordinatorState} state + * @param {(configuration: object) => void} [onDone] + * @returns {void} + */ +function initializeCoordinator (state, onDone) { + if (state.initialized) { + onDone?.(state.configuration) + return + } + if (onDone) { + state.initializationCallbacks.push(onDone) + } + if (state.initializing) { + return + } + + state.initializing = true + if (!libraryConfigurationCh.hasSubscribers) { + completeCoordinatorInitialization(state) + return + } + + try { + libraryConfigurationCh.runStores({ + basicReportingOnly: true, + frameworkVersion: state.frameworkVersion, + isParallel: state.maxActiveWorkers > 1, + onDone: response => state.asyncResource.runInAsyncScope( + completeCoordinatorInitialization, + undefined, + state, + response + ), + }, () => {}) + } catch (error) { + log.error('WebdriverIO Test Optimization configuration error', error) + completeCoordinatorInitialization(state) + } +} + +/** + * Starts basic-reporting suites for one worker. + * + * @param {WorkerRecord} workerRecord + * @param {string[]} files + * @returns {void} + */ +function startWorkerSuites (workerRecord, files) { + for (const rawFile of files) { + const file = normalizeFile(rawFile) + if (workerRecord.suiteContexts.has(file)) { + continue + } + + const suiteContext = { + testSuiteAbsolutePath: file, + testSuiteExecutionId: workerRecord.testSuiteExecutionId, + } + testSuiteStartCh.runStores(suiteContext, () => {}) + workerRecord.suiteContexts.set(file, suiteContext) + } +} + +/** + * Finishes one suite if it is still active. + * + * @param {CoordinatorState} state + * @param {WorkerRecord} workerRecord + * @param {string} rawFile + * @param {string} status + * @returns {void} + */ +function finishWorkerSuite (state, workerRecord, rawFile, status) { + const file = normalizeFile(rawFile) + const suiteContext = workerRecord.suiteContexts.get(file) + if (!suiteContext || state.suiteStatuses.has(suiteContext)) { + return + } + + state.suiteStatuses.set(suiteContext, status) + testSuiteFinishCh.publish({ status, ...suiteContext.currentStore }) +} + +/** + * Finishes every active suite belonging to a worker. + * + * @param {CoordinatorState} state + * @param {WorkerRecord} workerRecord + * @param {string} status + * @returns {void} + */ +function finishAllWorkerSuites (state, workerRecord, status) { + let hasFailedSuite = false + for (const suiteContext of workerRecord.suiteContexts.values()) { + if (suiteContext.status === 'fail') { + hasFailedSuite = true + break + } + } + + for (const [file, suiteContext] of workerRecord.suiteContexts) { + let suiteStatus = suiteContext.status ?? status + if (status === 'fail' && !hasFailedSuite && suiteStatus === 'skip') { + suiteStatus = 'fail' + } + finishWorkerSuite(state, workerRecord, file, suiteStatus) + } +} + +/** + * Sends a coordinator message to a WebdriverIO child process. + * + * @param {WorkerRecord} workerRecord + * @param {object} message + * @returns {void} + */ +function sendWorkerMessage (workerRecord, message) { + const childProcess = workerRecord.worker.childProcess + if (!childProcess?.connected) { + return + } + + childProcess.send(message, (error) => { + if (error) { + log.error('WebdriverIO Test Optimization IPC error', error) + } + }) +} + +/** + * Handles a worker request for its Mocha execution configuration. + * + * @param {CoordinatorState} state + * @param {WorkerRecord} workerRecord + * @param {object} message + * @returns {void} + */ +function handleConfigurationRequest (state, workerRecord, message) { + const { files = [], requestId } = message.content || {} + + initializeCoordinator(state, (configuration) => { + startWorkerSuites(workerRecord, files) + sendWorkerMessage(workerRecord, { + origin: 'datadog', + name: CONFIGURATION_RESPONSE, + content: { + configuration, + requestId, + }, + }) + }) +} + +/** + * Handles suite results reported by a Mocha worker. + * + * @param {WorkerRecord} workerRecord + * @param {object} message + * @returns {void} + */ +function handleSuiteResults (workerRecord, message) { + const { results = [] } = message.content || {} + for (const { file, status } of results) { + const suiteContext = workerRecord.suiteContexts.get(normalizeFile(file)) + if (suiteContext) { + suiteContext.status = status + } + } +} + +/** + * Handles all messages emitted by one WebdriverIO child process. + * + * @param {CoordinatorState} state + * @param {WorkerRecord} workerRecord + * @param {object|unknown[]} message + * @returns {void} + */ +function handleWorkerMessage (state, workerRecord, message) { + if (message?.origin === WEBDRIVERIO_WORKER_ORIGIN && message.name === WEBDRIVERIO_WORKER_EVENT) { + message = message.args + } + + if (Array.isArray(message)) { + const [messageCode, payload] = message + if (messageCode === MOCHA_WORKER_TRACE_PAYLOAD_CODE) { + workerReportTraceCh.publish({ + traces: payload, + [TEST_SUITE_EXECUTION_ID]: workerRecord.testSuiteExecutionId, + }) + } + return + } + + if (!message || typeof message !== 'object') { + return + } + + if (message.name === WORKER_READY) { + initializeCoordinator(state) + return + } + if (message.name === CONFIGURATION_REQUEST) { + handleConfigurationRequest(state, workerRecord, message) + return + } + if (message.name === SUITE_FINISH) { + handleSuiteResults(workerRecord, message) + return + } + if (message.name === 'testFrameworkInit') { + workerRecord.hasTests = message.content?.hasTests + if (!workerRecord.hasTests && state.frameworkVersion) { + initializeCoordinator(state, () => { + startWorkerSuites(workerRecord, workerRecord.specs) + finishAllWorkerSuites(state, workerRecord, 'skip') + }) + } + } +} + +/** + * Handles child-process exit and closes suites missing an explicit result. + * + * @param {CoordinatorState} state + * @param {WorkerRecord} workerRecord + * @param {object} exit + * @returns {void} + */ +function handleWorkerExit (state, workerRecord, exit) { + state.activeWorkers-- + workerRecord.exitCode = exit.exitCode + workerRecord.retries = exit.retries + if (!state.sessionStarted) { + return + } + + const status = workerRecord.hasTests === false ? 'skip' : exit.exitCode === 0 ? 'pass' : 'fail' + if (status === 'fail' && workerRecord.suiteContexts.size === 0) { + startWorkerSuites(workerRecord, workerRecord.specs) + } + finishAllWorkerSuites(state, workerRecord, status) +} + +/** + * Registers a newly created WebdriverIO worker with the coordinator. + * + * @param {CoordinatorState} state + * @param {object} worker + * @param {string[]} specs + * @returns {void} + */ +function registerWorker (state, worker, specs) { + const normalizedSpecs = [] + for (const spec of specs) { + normalizedSpecs.push(normalizeFile(spec)) + } + + const workerRecord = { + worker, + specs: normalizedSpecs, + suiteContexts: new Map(), + testSuiteExecutionId: String(++state.nextWorkerId), + hasTests: undefined, + exitCode: undefined, + retries: undefined, + } + state.activeWorkers++ + if (state.activeWorkers > state.maxActiveWorkers) { + state.maxActiveWorkers = state.activeWorkers + } + state.workers.add(workerRecord) + + worker.on('message', message => handleWorkerMessage(state, workerRecord, message)) + worker.once('exit', exit => handleWorkerExit(state, workerRecord, exit)) +} + +/** + * Calculates the final status for the coordinated session. + * + * @param {CoordinatorState} state + * @returns {string} + */ +function getSessionStatus (state) { + let hasPassingSuite = false + + for (const workerRecord of state.workers) { + if (workerRecord.exitCode !== undefined && workerRecord.exitCode !== 0) { + if (workerRecord.retries > 0) { + continue + } + return 'fail' + } + for (const suiteContext of workerRecord.suiteContexts.values()) { + const status = state.suiteStatuses.get(suiteContext) + if (status === 'fail') { + return 'fail' + } + if (status === 'pass') { + hasPassingSuite = true + } + } + } + + return hasPassingSuite ? 'pass' : 'skip' +} + +/** + * Finishes the single WebdriverIO-owned Mocha session. + * + * @param {CoordinatorState} state + * @param {unknown} error + * @param {() => void} onDone + * @returns {void} + */ +function finishCoordinator (state, error, onDone) { + if (state.finished) { + onDone() + return + } + if (!state.sessionStarted) { + if (!error && getSessionStatus(state) !== 'fail') { + onDone() + return + } + initializeCoordinator(state, () => finishCoordinator(state, error, onDone)) + return + } + state.finished = true + + for (const workerRecord of state.workers) { + const status = workerRecord.hasTests === false + ? 'skip' + : workerRecord.exitCode === 0 ? 'pass' : 'fail' + finishAllWorkerSuites(state, workerRecord, status) + } + + if (!testSessionFinishCh.hasSubscribers) { + onDone() + return + } + + testSessionFinishCh.publish({ + status: error ? 'fail' : getSessionStatus(state), + error, + isParallel: state.maxActiveWorkers > 1, + onDone, + }) +} + +// dc-polyfill supports partial tracing-channel subscribers, unlike the Node.js type definition. +// @ts-expect-error +localRunnerRunCh.subscribe({ + start (context) { + const runnerConfiguration = getRunnerConfiguration(context.self) + if (!testFinishCh.hasSubscribers || runnerConfiguration?.framework !== 'mocha') { + return + } + + const state = getCoordinatorState(context.self) + const workerOptions = context.arguments?.[0] + let workerEnvironment = runnerConfiguration.runnerEnv || {} + const launcherNodeOptions = getEnvironmentVariable('NODE_OPTIONS') + const workerNodeOptions = workerEnvironment.NODE_OPTIONS + + if (launcherNodeOptions && !includesNodeOptions(workerNodeOptions, launcherNodeOptions)) { + workerEnvironment = { + ...workerEnvironment, + NODE_OPTIONS: workerNodeOptions + ? `${launcherNodeOptions} ${workerNodeOptions}` + : launcherNodeOptions, + } + } + + runnerConfiguration.runnerEnv = { + ...workerEnvironment, + MOCHA_WORKER_ID: 'webdriverio', + [WEBDRIVERIO_WORKER_ENV]: 'true', + } + context.ddCoordinatorState = state + context.ddWorkerSpecs = workerOptions?.specs || [] + }, + asyncEnd (context) { + const state = context.ddCoordinatorState + if (!state) { + return + } + if (context.error) { + state.runError ??= context.error + return + } + if (!context.result) { + return + } + registerWorker(state, context.result, context.ddWorkerSpecs) + }, +}) + +// @ts-expect-error See the partial tracing-channel subscriber above. +localRunnerShutdownCh.subscribe({ + asyncEnd (context) { + const state = coordinatorStates.get(context.self) + if (!state) { + return + } + + // Orchestrion uses the callback for the matching settlement path to delay LocalRunner.shutdown. + const waitForCoordinator = onDone => { + const error = context.error ?? state.runError + if (state.initializing) { + state.initializationCallbacks.push(() => finishCoordinator(state, error, onDone)) + } else { + finishCoordinator(state, error, onDone) + } + } + context.resolveCallback = waitForCoordinator + context.rejectCallback = waitForCoordinator + }, +}) diff --git a/packages/datadog-instrumentations/test/fixtures/mocha-regular-worker.js b/packages/datadog-instrumentations/test/fixtures/mocha-regular-worker.js new file mode 100644 index 0000000000..0a0e265cd9 --- /dev/null +++ b/packages/datadog-instrumentations/test/fixtures/mocha-regular-worker.js @@ -0,0 +1,46 @@ +'use strict' + +const assert = require('node:assert/strict') +const { EventEmitter } = require('node:events') + +const { channel } = require('../../src/helpers/instrument') +const instrumentations = require('../../src/helpers/instrumentations') +const { WEBDRIVERIO_WORKER_ENV } = require('../../src/mocha/webdriverio-protocol') + +delete process.env[WEBDRIVERIO_WORKER_ENV] + +const existingMochaHookCount = instrumentations.mocha?.length || 0 +require('../../src/mocha/worker') + +const runnerHook = instrumentations.mocha + .slice(existingMochaHookCount) + .find(({ file }) => file === 'lib/runner.js') + +assert.ok(runnerHook) + +const workerFinishCh = channel('ci:mocha:worker:finish') + +function onWorkerFinish () {} + +workerFinishCh.subscribe(onWorkerFinish) + +class FakeRunner extends EventEmitter { + constructor () { + super() + this.suite = { + eachTest () {}, + } + } + + runTests () {} + + run () {} +} + +runnerHook.hook(FakeRunner) +const runner = new FakeRunner() +runner.run() + +assert.strictEqual(runner.listenerCount('fail'), 1) + +workerFinishCh.unsubscribe(onWorkerFinish) diff --git a/packages/datadog-instrumentations/test/fixtures/node_modules/@wdio/local-runner/package.json b/packages/datadog-instrumentations/test/fixtures/node_modules/@wdio/local-runner/package.json new file mode 100644 index 0000000000..acb08d99a7 --- /dev/null +++ b/packages/datadog-instrumentations/test/fixtures/node_modules/@wdio/local-runner/package.json @@ -0,0 +1,5 @@ +{ + "name": "@wdio/local-runner", + "version": "9.30.0", + "type": "module" +} diff --git a/packages/datadog-instrumentations/test/fixtures/webdriverio-delayed-worker.js b/packages/datadog-instrumentations/test/fixtures/webdriverio-delayed-worker.js new file mode 100644 index 0000000000..b730a9ecec --- /dev/null +++ b/packages/datadog-instrumentations/test/fixtures/webdriverio-delayed-worker.js @@ -0,0 +1,94 @@ +'use strict' + +const assert = require('node:assert/strict') +const { EventEmitter } = require('node:events') + +const { channel } = require('../../src/helpers/instrument') +const instrumentations = require('../../src/helpers/instrumentations') +const { + SUITE_FINISH, + WEBDRIVERIO_WORKER_ENV, + WEBDRIVERIO_WORKER_EVENT, + WEBDRIVERIO_WORKER_ORIGIN, +} = require('../../src/mocha/webdriverio-protocol') + +process.env[WEBDRIVERIO_WORKER_ENV] = 'true' +const existingMochaHookCount = instrumentations.mocha?.length || 0 +require('../../src/mocha/worker') + +const runnerHook = instrumentations.mocha + .slice(existingMochaHookCount) + .find(({ file }) => file === 'lib/runner.js') + +assert.ok(runnerHook) + +const workerFinishCh = channel('ci:mocha:worker:finish') +let flushDone + +function onWorkerFinish ({ onDone }) { + flushDone = onDone +} + +workerFinishCh.subscribe(onWorkerFinish) + +class FakeRunner extends EventEmitter { + constructor () { + super() + this.failures = 0 + this.suite = { + eachTest () {}, + } + } + + runTests () {} + + run (onDone) { + this.emit('end') + onDone() + } +} + +runnerHook.hook(FakeRunner) + +let runDone = false +let suiteMessageDone +process.connected = true +process.send = (message, onDone) => { + assert.strictEqual(message.origin, WEBDRIVERIO_WORKER_ORIGIN) + assert.strictEqual(message.name, WEBDRIVERIO_WORKER_EVENT) + assert.strictEqual(message.args.name, SUITE_FINISH) + suiteMessageDone = onDone +} + +new FakeRunner().run(() => { + runDone = true +}) + +assert.strictEqual(runDone, false) +assert.strictEqual(typeof flushDone, 'function') +assert.strictEqual(suiteMessageDone, undefined) + +flushDone() +assert.strictEqual(runDone, false) +assert.strictEqual(typeof suiteMessageDone, 'function') + +suiteMessageDone() +assert.strictEqual(runDone, true) + +runDone = false +flushDone = undefined +suiteMessageDone = undefined +process.connected = false + +new FakeRunner().run(() => { + runDone = true +}) + +assert.strictEqual(runDone, false) +assert.strictEqual(typeof flushDone, 'function') + +flushDone() +assert.strictEqual(runDone, true) +assert.strictEqual(suiteMessageDone, undefined) + +workerFinishCh.unsubscribe(onWorkerFinish) diff --git a/packages/datadog-instrumentations/test/fixtures/webdriverio-disconnected-worker.js b/packages/datadog-instrumentations/test/fixtures/webdriverio-disconnected-worker.js new file mode 100644 index 0000000000..b11abc941a --- /dev/null +++ b/packages/datadog-instrumentations/test/fixtures/webdriverio-disconnected-worker.js @@ -0,0 +1,128 @@ +'use strict' + +const assert = require('node:assert/strict') +const { EventEmitter } = require('node:events') + +const { channel } = require('../../src/helpers/instrument') +const instrumentations = require('../../src/helpers/instrumentations') +const { + CONFIGURATION_REQUEST, + CONFIGURATION_RESPONSE, + WEBDRIVERIO_WORKER_ENV, + WEBDRIVERIO_WORKER_EVENT, + WEBDRIVERIO_WORKER_ORIGIN, +} = require('../../src/mocha/webdriverio-protocol') + +process.env[WEBDRIVERIO_WORKER_ENV] = 'true' +const existingMochaHookCount = instrumentations.mocha?.length || 0 +require('../../src/mocha/worker') + +const webdriverioMochaHooks = instrumentations.mocha.slice(existingMochaHookCount) +const mochaHook = webdriverioMochaHooks.find(({ file }) => file === 'lib/mocha.js') +const runnerHook = webdriverioMochaHooks.find(({ file }) => file === 'lib/runner.js') + +assert.ok(mochaHook) +assert.ok(runnerHook) + +const workerFinishCh = channel('ci:mocha:worker:finish') + +function onWorkerFinish () {} + +workerFinishCh.subscribe(onWorkerFinish) + +/** + * Exercises worker-ready and suite-finish messages. + * + * @returns {void} + */ +function exerciseWorkerMessages () { + class FakeMocha { + constructor () { + this.files = [] + this.options = {} + this.suite = { + run () {}, + } + } + + /** + * @returns {object} + */ + run () { + return {} + } + } + + class FakeRunner extends EventEmitter { + constructor () { + super() + this.failures = 0 + this.suite = { + eachTest () {}, + } + } + + /** + * @returns {void} + */ + runTests () {} + + /** + * @returns {void} + */ + run () { + this.emit('fail', { file: 'hook-fail.e2e.js', type: 'hook' }) + this.emit('end') + } + } + + mochaHook.hook(FakeMocha, '10.8.2') + runnerHook.hook(FakeRunner) + new FakeMocha().run() + new FakeRunner().run() +} + +let sendCalls = 0 +process.connected = false +process.send = () => { + sendCalls++ + throw new Error('send called after disconnect') +} + +exerciseWorkerMessages() +assert.strictEqual(sendCalls, 0) + +process.connected = true +process.send = (message, onDone) => { + sendCalls++ + assert.strictEqual(message.origin, WEBDRIVERIO_WORKER_ORIGIN) + assert.strictEqual(message.name, WEBDRIVERIO_WORKER_EVENT) + assert.ok(message.args) + assert.strictEqual(typeof onDone, 'function') + onDone(new Error('IPC channel closed during send')) +} + +exerciseWorkerMessages() +assert.strictEqual(sendCalls, 3) + +process.send = (message, onDone) => { + sendCalls++ + assert.strictEqual(message.origin, WEBDRIVERIO_WORKER_ORIGIN) + assert.strictEqual(message.name, WEBDRIVERIO_WORKER_EVENT) + assert.ok(message.args) + assert.strictEqual(typeof onDone, 'function') + onDone() + if (message.args.name === CONFIGURATION_REQUEST) { + process.emit('message', { + name: CONFIGURATION_RESPONSE, + content: { + requestId: message.args.content.requestId, + }, + }) + } +} + +exerciseWorkerMessages() +assert.strictEqual(sendCalls, 6) + +workerFinishCh.unsubscribe(onWorkerFinish) diff --git a/packages/datadog-instrumentations/test/fixtures/webdriverio-local-runner.mjs b/packages/datadog-instrumentations/test/fixtures/webdriverio-local-runner.mjs new file mode 100644 index 0000000000..b0259ffd30 --- /dev/null +++ b/packages/datadog-instrumentations/test/fixtures/webdriverio-local-runner.mjs @@ -0,0 +1,13 @@ +const LocalRunner = class { + async run (workerOptions) { + return workerOptions + } + + async shutdown (error) { + if (error) { + throw error + } + } +} + +export { LocalRunner } diff --git a/packages/datadog-instrumentations/test/helpers/register.spec.js b/packages/datadog-instrumentations/test/helpers/register.spec.js index 56c0f5afea..af026dadf3 100644 --- a/packages/datadog-instrumentations/test/helpers/register.spec.js +++ b/packages/datadog-instrumentations/test/helpers/register.spec.js @@ -3,12 +3,14 @@ const Module = require('module') const assert = require('node:assert/strict') +const { channel } = require('dc-polyfill') const sinon = require('sinon') describe('register', () => { let hooksMock let HookMock let originalModuleProtoRequire + let telemetryMock const clearRegisterCache = () => { const registerPath = require.resolve('../../src/helpers/register') @@ -29,6 +31,7 @@ describe('register', () => { } HookMock = sinon.stub() + telemetryMock = sinon.stub() const registerPath = require.resolve('../../src/helpers/register') originalModuleProtoRequire = Module.prototype.require @@ -38,6 +41,7 @@ describe('register', () => { const stubs = { './hooks': hooksMock, './hook': HookMock, + '../../../dd-trace/src/guardrails/telemetry': telemetryMock, } return stubs[request] || originalModuleProtoRequire.call(this, request) } @@ -82,4 +86,40 @@ describe('register', () => { sinon.assert.notCalled(hooksMock['@confluentinc/kafka-javascript'].fn) sinon.assert.notCalled(hooksMock['mongodb-core'].fn) }) + + for (const disabledName of ['fs', 'node:fs']) { + it(`should disable both builtin hook names when ${disabledName} is disabled`, () => { + hooksMock.fs = { fn: sinon.stub() } + hooksMock['node:fs'] = { fn: sinon.stub() } + + loadRegisterWithEnv({ DD_TRACE_DISABLED_INSTRUMENTATIONS: disabledName }) + + const registeredNames = [] + for (const [names] of HookMock.args) { + registeredNames.push(names[0]) + } + assert.deepStrictEqual(registeredNames.sort(), ['@confluentinc/kafka-javascript', 'mongodb-core']) + }) + } + + it('should report the name and version correctly for scoped integration names', () => { + loadRegisterWithEnv() + + const integrationName = '@confluentinc/kafka-javascript' + const moduleVersion = '0.1.0' + const hookCall = HookMock.getCalls().find(({ args }) => args[0][0] === integrationName) + const hook = hookCall.args[2] + + hook('original', integrationName, '/path/to/module', moduleVersion) + channel('dd-trace:exporter:first-flush').publish() + + sinon.assert.calledOnceWithExactly(telemetryMock, 'abort.integration', [ + `integration:${integrationName}`, + `integration_version:${moduleVersion}`, + ], { + result: 'abort', + result_class: 'incompatible_library', + result_reason: `Incompatible integration version: ${integrationName}@${moduleVersion}`, + }) + }) }) diff --git a/packages/datadog-instrumentations/test/helpers/rewriter/index.spec.js b/packages/datadog-instrumentations/test/helpers/rewriter/index.spec.js index bdb6593ace..474b1c12c0 100644 --- a/packages/datadog-instrumentations/test/helpers/rewriter/index.spec.js +++ b/packages/datadog-instrumentations/test/helpers/rewriter/index.spec.js @@ -562,19 +562,22 @@ describe('check-require-cache', () => { assert.ok(subs.start.called) }) - it('should wait for an asyncEnd promise when configured', async () => { + it('should wait for the resolve callback only before resolving', async () => { const { test } = compileFile('trace-promise-async-end') const steps = [] subs = { asyncEnd (ctx) { steps.push('asyncEnd') - ctx.asyncEndPromise = new Promise(resolve => { + ctx.resolveCallback = onDone => { setImmediate(() => { - steps.push('asyncEndPromise') - resolve() + steps.push('resolveCallback') + onDone() }) - }) + } + ctx.rejectCallback = () => { + assert.fail('reject callback called for a fulfilled promise') + } }, } @@ -593,7 +596,68 @@ describe('check-require-cache', () => { const result = await resultPromise assert.equal(result, 'result') - assert.deepStrictEqual(steps, ['asyncEnd', 'asyncEndPromise', 'resolved']) + assert.deepStrictEqual(steps, ['asyncEnd', 'resolveCallback', 'resolved']) + }) + + it('should wait for the reject callback only before preserving a rejection', async () => { + const { test } = compileFile('trace-promise-async-end') + const error = new Error('test rejection') + const steps = [] + + subs = { + asyncEnd (ctx) { + steps.push('asyncEnd') + ctx.resolveCallback = () => { + assert.fail('resolve callback called for a rejected promise') + } + ctx.rejectCallback = onDone => { + setImmediate(() => { + steps.push('rejectCallback') + onDone() + }) + } + }, + } + + ch = tracingChannel('orchestrion:test:trace_promise_async_end') + ch.subscribe(subs) + + const resultPromise = test(error) + + await Promise.resolve() + + assert.deepStrictEqual(steps, ['asyncEnd']) + await assert.rejects(resultPromise, actualError => { + steps.push('rejected') + return actualError === error + }) + assert.deepStrictEqual(steps, ['asyncEnd', 'rejectCallback', 'rejected']) + }) + + it('should preserve promise settlement when its callback throws', async () => { + const { test } = compileFile('trace-promise-async-end') + const error = new Error('test rejection') + + subs = { + asyncEnd (ctx) { + ctx.resolveCallback = () => { + throw new Error('resolve callback error') + } + ctx.rejectCallback = () => { + throw new Error('reject callback error') + } + }, + } + + ch = tracingChannel('orchestrion:test:trace_promise_async_end') + ch.subscribe(subs) + + const [result] = await Promise.all([ + test(), + assert.rejects(test(error), actualError => actualError === error), + ]) + + assert.equal(result, 'result') }) it('should use import when rewriting esm modules', () => { diff --git a/packages/datadog-instrumentations/test/helpers/rewriter/node_modules/test/trace-promise-async-end.js b/packages/datadog-instrumentations/test/helpers/rewriter/node_modules/test/trace-promise-async-end.js index 32d089f101..9685f5d5cb 100644 --- a/packages/datadog-instrumentations/test/helpers/rewriter/node_modules/test/trace-promise-async-end.js +++ b/packages/datadog-instrumentations/test/helpers/rewriter/node_modules/test/trace-promise-async-end.js @@ -1,6 +1,10 @@ 'use strict' -async function test () { +async function test (error) { + if (error) { + throw error + } + return 'result' } diff --git a/packages/datadog-instrumentations/test/webdriverio.spec.js b/packages/datadog-instrumentations/test/webdriverio.spec.js new file mode 100644 index 0000000000..b1a025adca --- /dev/null +++ b/packages/datadog-instrumentations/test/webdriverio.spec.js @@ -0,0 +1,779 @@ +'use strict' + +const assert = require('node:assert/strict') +const { execFile } = require('node:child_process') +const { EventEmitter } = require('node:events') +const fs = require('node:fs') +const os = require('node:os') +const path = require('node:path') +const { pathToFileURL } = require('node:url') +const { promisify } = require('node:util') + +const { channel, tracingChannel } = require('../src/helpers/instrument') +const rewriter = require('../src/helpers/rewriter') +const { + MOCHA_WORKER_TRACE_PAYLOAD_CODE, + TEST_SUITE_EXECUTION_ID, +} = require('../../dd-trace/src/plugins/util/test') +const { + CONFIGURATION_REQUEST, + CONFIGURATION_RESPONSE, + SUITE_FINISH, + WEBDRIVERIO_WORKER_ENV, + WORKER_READY, +} = require('../src/mocha/webdriverio-protocol') + +const fixturePath = path.join(__dirname, 'fixtures', 'webdriverio-local-runner.mjs') +const delayedWorkerFixturePath = path.join(__dirname, 'fixtures', 'webdriverio-delayed-worker.js') +const disconnectedWorkerFixturePath = path.join(__dirname, 'fixtures', 'webdriverio-disconnected-worker.js') +const regularMochaWorkerFixturePath = path.join(__dirname, 'fixtures', 'mocha-regular-worker.js') +const fixtureModulePath = path.join( + __dirname, + 'fixtures', + 'node_modules', + '@wdio', + 'local-runner', + 'build', + 'index.js' +) +const execFileAsync = promisify(execFile) + +describe('webdriverio instrumentation', () => { + it('rewrites the ESM local runner and waits for coordinator shutdown', () => { + const source = fs.readFileSync(fixturePath, 'utf8') + const rewrittenSource = rewriter.rewrite(source, fixtureModulePath, 'module') + + assert.notStrictEqual(rewrittenSource, source) + assert.match(rewrittenSource, /orchestrion:@wdio\/local-runner:LocalRunner_run/) + assert.match(rewrittenSource, /orchestrion:@wdio\/local-runner:LocalRunner_shutdown/) + assert.match(rewrittenSource, /__apm\$ctx\.resolveCallback/) + assert.match(rewrittenSource, /__apm\$ctx\.rejectCallback/) + }) + + it('waits for coordinator shutdown before preserving a LocalRunner.shutdown rejection', async () => { + const source = fs.readFileSync(fixturePath, 'utf8') + const rewrittenSource = rewriter.rewrite(source, fixtureModulePath, 'module') + const outputDirectory = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-webdriverio-rewriter-')) + const outputPath = path.join(outputDirectory, 'index.mjs') + const shutdownCh = tracingChannel('orchestrion:@wdio/local-runner:LocalRunner_shutdown') + const shutdownError = new Error('shutdown failed') + const steps = [] + const subscriber = { + asyncEnd (context) { + steps.push('asyncEnd') + context.rejectCallback = onDone => { + setImmediate(() => { + steps.push('coordinator') + onDone() + }) + } + }, + } + + fs.writeFileSync(outputPath, rewrittenSource) + shutdownCh.subscribe(subscriber) + + try { + const { LocalRunner } = await import(pathToFileURL(outputPath)) + const resultPromise = new LocalRunner().shutdown(shutdownError) + + await Promise.resolve() + + assert.deepStrictEqual(steps, ['asyncEnd']) + await assert.rejects(resultPromise, error => { + steps.push('rejected') + return error === shutdownError + }) + assert.deepStrictEqual(steps, ['asyncEnd', 'coordinator', 'rejected']) + } finally { + shutdownCh.unsubscribe(subscriber) + } + }) + + it('propagates complete launcher NODE_OPTIONS to worker environments', () => { + const testFinishCh = channel('ci:mocha:test:finish') + const originalNodeOptions = process.env.NODE_OPTIONS + process.env.NODE_OPTIONS = '--require dd-trace/ci/init' + + function onTestFinish () {} + + testFinishCh.subscribe(onTestFinish) + + try { + require('../src/webdriverio') + + const cases = [ + { + runnerEnv: undefined, + expectedNodeOptions: '--require dd-trace/ci/init', + }, + { + runnerEnv: { NODE_OPTIONS: '--require dd-trace/ci/init-custom' }, + expectedNodeOptions: '--require dd-trace/ci/init --require dd-trace/ci/init-custom', + }, + { + runnerEnv: { NODE_OPTIONS: '--no-warnings --require dd-trace/ci/init' }, + expectedNodeOptions: '--no-warnings --require dd-trace/ci/init', + }, + ] + + for (const { runnerEnv, expectedNodeOptions } of cases) { + const localRunner = { + config: { + framework: 'mocha', + runnerEnv, + }, + } + const runContext = { + self: localRunner, + arguments: [{ specs: [] }], + } + + tracingChannel('orchestrion:@wdio/local-runner:LocalRunner_run').start.publish(runContext) + + assert.deepStrictEqual(localRunner.config.runnerEnv, { + NODE_OPTIONS: expectedNodeOptions, + MOCHA_WORKER_ID: 'webdriverio', + [WEBDRIVERIO_WORKER_ENV]: 'true', + }) + } + } finally { + testFinishCh.unsubscribe(onTestFinish) + if (originalNodeOptions === undefined) { + delete process.env.NODE_OPTIONS + } else { + process.env.NODE_OPTIONS = originalNodeOptions + } + } + }) + + it('does not send Mocha worker messages over disconnected IPC', async () => { + await execFileAsync(process.execPath, [disconnectedWorkerFixturePath]) + }) + + it('waits for worker payloads before completing Mocha', async () => { + await execFileAsync(process.execPath, [delayedWorkerFixturePath]) + }) + + it('does not track WebdriverIO hook failures in regular Mocha workers', async () => { + await execFileAsync(process.execPath, [regularMochaWorkerFixturePath]) + }) + + it('coordinates two Mocha workers under one session', async () => { + const testFinishCh = channel('ci:mocha:test:finish') + const knownTestsCh = channel('ci:mocha:known-tests') + const libraryConfigurationCh = channel('ci:mocha:library-configuration') + const modifiedFilesCh = channel('ci:mocha:modified-files') + const skippableSuitesCh = channel('ci:mocha:test-suite:skippable') + const testSessionStartCh = channel('ci:mocha:session:start') + const testSessionFinishCh = channel('ci:mocha:session:finish') + const testSuiteStartCh = channel('ci:mocha:test-suite:start') + const testSuiteFinishCh = channel('ci:mocha:test-suite:finish') + const testManagementTestsCh = channel('ci:mocha:test-management-tests') + const workerReportTraceCh = channel('ci:mocha:worker-report:trace') + + const sessionStarts = [] + const sessionFinishes = [] + const suiteStarts = [] + const suiteFinishes = [] + const workerTracePayloads = [] + let advancedFeatureRequests = 0 + let configurationRequests = 0 + const originalNodeOptions = process.env.NODE_OPTIONS + process.env.NODE_OPTIONS = '--require dd-trace/ci/init' + + function onTestFinish () {} + function onAdvancedFeatureRequest (request) { + advancedFeatureRequests++ + request.onDone({}) + } + function onLibraryConfiguration (request) { + configurationRequests++ + request.onDone({ + isTestDynamicInstrumentationEnabled: true, + libraryConfig: { + earlyFlakeDetectionNumRetries: 5, + earlyFlakeDetectionSlowTestRetries: { '5s': 5 }, + flakyTestRetriesCount: 5, + isCodeCoverageEnabled: true, + isCoverageReportUploadEnabled: true, + isDiEnabled: true, + isEarlyFlakeDetectionEnabled: true, + isFlakyTestRetriesEnabled: true, + isImpactedTestsEnabled: true, + isItrEnabled: true, + isKnownTestsEnabled: true, + isSuitesSkippingEnabled: true, + isTestManagementEnabled: true, + testManagementAttemptToFixRetries: 5, + }, + repositoryRoot: process.cwd(), + }) + } + function onSessionStart (event) { + sessionStarts.push(event) + } + function onSessionFinish (event) { + sessionFinishes.push(event) + event.onDone() + } + function onSuiteStart (event) { + suiteStarts.push(event) + } + function onSuiteFinish (event) { + suiteFinishes.push(event) + } + function onWorkerTrace (event) { + workerTracePayloads.push(event) + } + + testFinishCh.subscribe(onTestFinish) + knownTestsCh.subscribe(onAdvancedFeatureRequest) + libraryConfigurationCh.subscribe(onLibraryConfiguration) + modifiedFilesCh.subscribe(onAdvancedFeatureRequest) + skippableSuitesCh.subscribe(onAdvancedFeatureRequest) + testSessionStartCh.subscribe(onSessionStart) + testSessionFinishCh.subscribe(onSessionFinish) + testSuiteStartCh.subscribe(onSuiteStart) + testSuiteFinishCh.subscribe(onSuiteFinish) + testManagementTestsCh.subscribe(onAdvancedFeatureRequest) + workerReportTraceCh.subscribe(onWorkerTrace) + + try { + require('../src/webdriverio') + + const localRunner = { + _config: { + framework: 'mocha', + rootDir: process.cwd(), + runnerEnv: { + NODE_OPTIONS: '--no-warnings', + USER_ENV: 'preserved', + }, + }, + } + const firstFile = path.join(process.cwd(), 'first.spec.js') + const secondFile = path.join(process.cwd(), 'second.spec.js') + const firstWorker = createWorker() + const secondWorker = createWorker() + + registerWorker(localRunner, firstWorker, firstFile) + registerWorker(localRunner, secondWorker, secondFile) + + assert.deepStrictEqual(localRunner._config.runnerEnv, { + USER_ENV: 'preserved', + NODE_OPTIONS: '--require dd-trace/ci/init --no-warnings', + MOCHA_WORKER_ID: 'webdriverio', + [WEBDRIVERIO_WORKER_ENV]: 'true', + }) + + firstWorker.emit('message', { + origin: 'datadog', + name: 'workerEvent', + args: { + name: WORKER_READY, + content: { frameworkVersion: '10.8.2' }, + }, + }) + secondWorker.emit('message', { + origin: 'datadog', + name: 'workerEvent', + args: { + name: WORKER_READY, + content: { frameworkVersion: '10.8.2' }, + }, + }) + await new Promise(setImmediate) + + requestConfiguration(firstWorker, firstFile, 'first-request') + requestConfiguration(secondWorker, secondFile, 'second-request') + await new Promise(setImmediate) + + firstWorker.emit('message', { + origin: 'datadog', + name: 'workerEvent', + args: [MOCHA_WORKER_TRACE_PAYLOAD_CODE, 'first-trace'], + }) + secondWorker.emit('message', { + origin: 'datadog', + name: 'workerEvent', + args: [MOCHA_WORKER_TRACE_PAYLOAD_CODE, 'second-trace'], + }) + + assert.strictEqual(firstWorker.sentMessages[0].name, CONFIGURATION_RESPONSE) + assert.strictEqual(firstWorker.sentMessages[0].content.requestId, 'first-request') + assert.strictEqual(secondWorker.sentMessages[0].name, CONFIGURATION_RESPONSE) + assert.strictEqual(secondWorker.sentMessages[0].content.requestId, 'second-request') + assert.deepStrictEqual(firstWorker.sentMessages[0].content.configuration, { + earlyFlakeDetectionNumRetries: 0, + earlyFlakeDetectionSlowTestRetries: {}, + flakyTestRetriesCount: 0, + isCodeCoverageEnabled: false, + isCoverageReportUploadEnabled: false, + isDiEnabled: false, + isEarlyFlakeDetectionEnabled: false, + isFlakyTestRetriesEnabled: false, + isImpactedTestsEnabled: false, + isItrEnabled: false, + isKnownTestsEnabled: false, + isSuitesSkippingEnabled: false, + isTestDynamicInstrumentationEnabled: false, + isTestManagementTestsEnabled: false, + knownTests: {}, + modifiedFiles: [], + repositoryRoot: process.cwd(), + testManagementAttemptToFixRetries: 0, + testManagementTests: {}, + }) + + reportSuiteFinish(firstWorker, firstFile, 'fail') + reportSuiteFinish(secondWorker, secondFile) + firstWorker.emit('exit', { exitCode: 1, retries: 1 }) + secondWorker.emit('exit', { exitCode: 0, retries: 0 }) + + await finishLocalRunner(localRunner) + + assert.strictEqual(configurationRequests, 1) + assert.strictEqual(advancedFeatureRequests, 0) + assert.strictEqual(sessionStarts.length, 1) + assert.strictEqual(sessionStarts[0].testFramework, 'webdriverio') + assert.strictEqual(sessionStarts[0].testFrameworkAdapter, 'mocha') + assert.strictEqual(sessionFinishes.length, 1) + assert.strictEqual(sessionFinishes[0].status, 'pass') + assert.strictEqual(sessionFinishes[0].isParallel, true) + assert.strictEqual('isEarlyFlakeDetectionEnabled' in sessionFinishes[0], false) + assert.strictEqual('isSuitesSkipped' in sessionFinishes[0], false) + assert.strictEqual('isTestManagementEnabled' in sessionFinishes[0], false) + assert.deepStrictEqual(suiteStarts.map(({ testSuiteAbsolutePath }) => testSuiteAbsolutePath), [ + firstFile, + secondFile, + ]) + assert.strictEqual(new Set(suiteStarts.map(({ testSuiteExecutionId }) => testSuiteExecutionId)).size, 2) + assert.deepStrictEqual(workerTracePayloads, [ + { + traces: 'first-trace', + [TEST_SUITE_EXECUTION_ID]: suiteStarts[0].testSuiteExecutionId, + }, + { + traces: 'second-trace', + [TEST_SUITE_EXECUTION_ID]: suiteStarts[1].testSuiteExecutionId, + }, + ]) + assert.deepStrictEqual(suiteFinishes.map(({ status }) => status), ['fail', 'pass']) + } finally { + testFinishCh.unsubscribe(onTestFinish) + knownTestsCh.unsubscribe(onAdvancedFeatureRequest) + libraryConfigurationCh.unsubscribe(onLibraryConfiguration) + modifiedFilesCh.unsubscribe(onAdvancedFeatureRequest) + skippableSuitesCh.unsubscribe(onAdvancedFeatureRequest) + testSessionStartCh.unsubscribe(onSessionStart) + testSessionFinishCh.unsubscribe(onSessionFinish) + testSuiteStartCh.unsubscribe(onSuiteStart) + testSuiteFinishCh.unsubscribe(onSuiteFinish) + testManagementTestsCh.unsubscribe(onAdvancedFeatureRequest) + workerReportTraceCh.unsubscribe(onWorkerTrace) + if (originalNodeOptions === undefined) { + delete process.env.NODE_OPTIONS + } else { + process.env.NODE_OPTIONS = originalNodeOptions + } + } + }) + + it('fails a terminal worker exit without marking sequential workers as parallel', async () => { + const testFinishCh = channel('ci:mocha:test:finish') + const testSessionFinishCh = channel('ci:mocha:session:finish') + const sessionFinishes = [] + + function onTestFinish () {} + function onSessionFinish (event) { + sessionFinishes.push(event) + event.onDone() + } + + testFinishCh.subscribe(onTestFinish) + testSessionFinishCh.subscribe(onSessionFinish) + + try { + require('../src/webdriverio') + + const localRunner = { + config: { + framework: 'mocha', + rootDir: process.cwd(), + }, + } + const firstFile = path.join(process.cwd(), 'first.spec.js') + const secondFile = path.join(process.cwd(), 'second.spec.js') + const firstWorker = createWorker() + const secondWorker = createWorker() + + registerWorker(localRunner, firstWorker, firstFile) + requestConfiguration(firstWorker, firstFile, 'first-request') + reportSuiteFinish(firstWorker, firstFile) + firstWorker.emit('exit', { exitCode: 0, retries: 0 }) + + registerWorker(localRunner, secondWorker, secondFile) + requestConfiguration(secondWorker, secondFile, 'second-request') + reportSuiteFinish(secondWorker, secondFile) + secondWorker.emit('exit', { exitCode: 1, retries: 0 }) + + await finishLocalRunner(localRunner) + + assert.strictEqual(sessionFinishes.length, 1) + assert.strictEqual(sessionFinishes[0].status, 'fail') + assert.strictEqual(sessionFinishes[0].isParallel, false) + } finally { + testFinishCh.unsubscribe(onTestFinish) + testSessionFinishCh.unsubscribe(onSessionFinish) + } + }) + + it('reports a worker failure before Mocha loads', async () => { + const testFinishCh = channel('ci:mocha:test:finish') + const libraryConfigurationCh = channel('ci:mocha:library-configuration') + const testSessionStartCh = channel('ci:mocha:session:start') + const testSessionFinishCh = channel('ci:mocha:session:finish') + const sessionStarts = [] + const sessionFinishes = [] + let configurationRequests = 0 + + function onTestFinish () {} + function onLibraryConfiguration (request) { + configurationRequests++ + setImmediate(() => request.onDone({ repositoryRoot: process.cwd() })) + } + function onSessionStart (event) { + sessionStarts.push(event) + } + function onSessionFinish (event) { + sessionFinishes.push(event) + event.onDone() + } + + testFinishCh.subscribe(onTestFinish) + libraryConfigurationCh.subscribe(onLibraryConfiguration) + testSessionStartCh.subscribe(onSessionStart) + testSessionFinishCh.subscribe(onSessionFinish) + + try { + require('../src/webdriverio') + + const localRunner = { + config: { + framework: 'mocha', + rootDir: process.cwd(), + }, + } + const worker = createWorker() + + registerWorker(localRunner, worker, path.join(process.cwd(), 'first.spec.js')) + worker.emit('exit', { exitCode: 1, retries: 0 }) + + await finishLocalRunner(localRunner) + + assert.strictEqual(configurationRequests, 1) + assert.strictEqual(sessionStarts.length, 1) + assert.strictEqual(sessionStarts[0].frameworkVersion, undefined) + assert.strictEqual(sessionFinishes.length, 1) + assert.strictEqual(sessionFinishes[0].status, 'fail') + assert.strictEqual(sessionFinishes[0].isParallel, false) + } finally { + testFinishCh.unsubscribe(onTestFinish) + libraryConfigurationCh.unsubscribe(onLibraryConfiguration) + testSessionStartCh.unsubscribe(onSessionStart) + testSessionFinishCh.unsubscribe(onSessionFinish) + } + }) + + it('reports a suite that fails after Mocha loads but before requesting configuration', async () => { + const testFinishCh = channel('ci:mocha:test:finish') + const testSessionFinishCh = channel('ci:mocha:session:finish') + const testSuiteStartCh = channel('ci:mocha:test-suite:start') + const testSuiteFinishCh = channel('ci:mocha:test-suite:finish') + const sessionFinishes = [] + const suiteStarts = [] + const suiteFinishes = [] + + function onTestFinish () {} + function onSessionFinish (event) { + sessionFinishes.push(event) + event.onDone() + } + function onSuiteStart (event) { + suiteStarts.push(event) + } + function onSuiteFinish (event) { + suiteFinishes.push(event) + } + + testFinishCh.subscribe(onTestFinish) + testSessionFinishCh.subscribe(onSessionFinish) + testSuiteStartCh.subscribe(onSuiteStart) + testSuiteFinishCh.subscribe(onSuiteFinish) + + try { + require('../src/webdriverio') + + const localRunner = { + config: { + framework: 'mocha', + rootDir: process.cwd(), + }, + } + const file = path.join(process.cwd(), 'load-fail.spec.js') + const worker = createWorker() + + registerWorker(localRunner, worker, file) + worker.emit('message', { + name: WORKER_READY, + content: { frameworkVersion: '10.8.2' }, + }) + worker.emit('exit', { exitCode: 1, retries: 0 }) + + await finishLocalRunner(localRunner) + + assert.strictEqual(sessionFinishes.length, 1) + assert.strictEqual(sessionFinishes[0].status, 'fail') + assert.deepStrictEqual(suiteStarts.map(event => event.testSuiteAbsolutePath), [file]) + assert.deepStrictEqual(suiteFinishes.map(event => event.status), ['fail']) + } finally { + testFinishCh.unsubscribe(onTestFinish) + testSessionFinishCh.unsubscribe(onSessionFinish) + testSuiteStartCh.unsubscribe(onSuiteStart) + testSuiteFinishCh.unsubscribe(onSuiteFinish) + } + }) + + it('reports LocalRunner.run rejections before a worker exists', async () => { + const testFinishCh = channel('ci:mocha:test:finish') + const testSessionFinishCh = channel('ci:mocha:session:finish') + const sessionFinishes = [] + const runError = new Error('worker spawn failed') + + function onTestFinish () {} + function onSessionFinish (event) { + sessionFinishes.push(event) + event.onDone() + } + + testFinishCh.subscribe(onTestFinish) + testSessionFinishCh.subscribe(onSessionFinish) + + try { + require('../src/webdriverio') + + const localRunner = { + config: { + framework: 'mocha', + rootDir: process.cwd(), + }, + } + const runContext = { + self: localRunner, + arguments: [{ specs: [path.join(process.cwd(), 'first.spec.js')] }], + } + const runCh = tracingChannel('orchestrion:@wdio/local-runner:LocalRunner_run') + + runCh.start.publish(runContext) + runContext.error = runError + runCh.asyncEnd.publish(runContext) + + await finishLocalRunner(localRunner) + + assert.strictEqual(sessionFinishes.length, 1) + assert.strictEqual(sessionFinishes[0].status, 'fail') + assert.strictEqual(sessionFinishes[0].error, runError) + assert.strictEqual(sessionFinishes[0].isParallel, false) + } finally { + testFinishCh.unsubscribe(onTestFinish) + testSessionFinishCh.unsubscribe(onSessionFinish) + } + }) + + it('reports LocalRunner.shutdown rejections after coordinator completion', async () => { + const testFinishCh = channel('ci:mocha:test:finish') + const testSessionFinishCh = channel('ci:mocha:session:finish') + const sessionFinishes = [] + const shutdownError = new Error('shutdown failed') + + function onTestFinish () {} + function onSessionFinish (event) { + sessionFinishes.push(event) + event.onDone() + } + + testFinishCh.subscribe(onTestFinish) + testSessionFinishCh.subscribe(onSessionFinish) + + try { + require('../src/webdriverio') + + const localRunner = { + config: { + framework: 'mocha', + rootDir: process.cwd(), + }, + } + const runContext = { + self: localRunner, + arguments: [{ specs: [] }], + } + + tracingChannel('orchestrion:@wdio/local-runner:LocalRunner_run').start.publish(runContext) + await finishLocalRunner(localRunner, shutdownError) + + assert.strictEqual(sessionFinishes.length, 1) + assert.strictEqual(sessionFinishes[0].status, 'fail') + assert.strictEqual(sessionFinishes[0].error, shutdownError) + assert.strictEqual(sessionFinishes[0].isParallel, false) + } finally { + testFinishCh.unsubscribe(onTestFinish) + testSessionFinishCh.unsubscribe(onSessionFinish) + } + }) + + it('waits for in-flight coordinator initialization during shutdown', async () => { + const testFinishCh = channel('ci:mocha:test:finish') + const libraryConfigurationCh = channel('ci:mocha:library-configuration') + const testSessionFinishCh = channel('ci:mocha:session:finish') + const sessionFinishes = [] + let completeConfiguration + + function onTestFinish () {} + function onLibraryConfiguration (request) { + completeConfiguration = request.onDone + } + function onSessionFinish (event) { + sessionFinishes.push(event) + event.onDone() + } + + testFinishCh.subscribe(onTestFinish) + libraryConfigurationCh.subscribe(onLibraryConfiguration) + testSessionFinishCh.subscribe(onSessionFinish) + + try { + require('../src/webdriverio') + + const localRunner = { + config: { + framework: 'mocha', + rootDir: process.cwd(), + }, + } + const worker = createWorker() + + registerWorker(localRunner, worker, path.join(process.cwd(), 'first.spec.js')) + worker.emit('message', { name: WORKER_READY }) + + const shutdownPromise = finishLocalRunner(localRunner) + + assert.strictEqual(sessionFinishes.length, 0) + completeConfiguration({ repositoryRoot: process.cwd() }) + await shutdownPromise + + assert.strictEqual(sessionFinishes.length, 1) + assert.strictEqual(sessionFinishes[0].status, 'skip') + } finally { + testFinishCh.unsubscribe(onTestFinish) + libraryConfigurationCh.unsubscribe(onLibraryConfiguration) + testSessionFinishCh.unsubscribe(onSessionFinish) + } + }) +}) + +/** + * Creates a fake WebdriverIO worker instance. + * + * @returns {EventEmitter & {childProcess: object, sentMessages: object[]}} + */ +function createWorker () { + const worker = new EventEmitter() + worker.sentMessages = [] + worker.childProcess = { + connected: true, + send (message, onDone) { + worker.sentMessages.push(message) + onDone?.() + }, + } + return worker +} + +/** + * Publishes the LocalRunner.run lifecycle for one worker. + * + * @param {object} localRunner + * @param {object} worker + * @param {string} file + * @returns {void} + */ +function registerWorker (localRunner, worker, file) { + const context = { + self: localRunner, + arguments: [{ specs: [file] }], + } + const runCh = tracingChannel('orchestrion:@wdio/local-runner:LocalRunner_run') + runCh.start.publish(context) + context.result = worker + runCh.asyncEnd.publish(context) +} + +/** + * Publishes LocalRunner.shutdown completion and waits for the coordinator callback. + * + * @param {object} localRunner + * @param {unknown} [error] + * @returns {Promise} + */ +function finishLocalRunner (localRunner, error) { + const context = { self: localRunner, error } + tracingChannel('orchestrion:@wdio/local-runner:LocalRunner_shutdown').asyncEnd.publish(context) + const callback = error ? context.rejectCallback : context.resolveCallback + return new Promise(callback) +} + +/** + * Requests execution configuration from the coordinator. + * + * @param {EventEmitter} worker + * @param {string} file + * @param {string} requestId + * @returns {void} + */ +function requestConfiguration (worker, file, requestId) { + worker.emit('message', { + origin: 'datadog', + name: 'workerEvent', + args: { + name: CONFIGURATION_REQUEST, + content: { + files: [file], + frameworkVersion: '10.8.2', + requestId, + }, + }, + }) +} + +/** + * Reports a suite result to the coordinator. + * + * @param {EventEmitter} worker + * @param {string} file + * @param {string} [status] + * @returns {void} + */ +function reportSuiteFinish (worker, file, status = 'pass') { + worker.emit('message', { + origin: 'datadog', + name: 'workerEvent', + args: { + name: SUITE_FINISH, + content: { + results: [{ file, status }], + }, + }, + }) +} diff --git a/packages/datadog-plugin-aerospike/src/index.js b/packages/datadog-plugin-aerospike/src/index.js index d7f70f574b..d0676f6709 100644 --- a/packages/datadog-plugin-aerospike/src/index.js +++ b/packages/datadog-plugin-aerospike/src/index.js @@ -67,9 +67,7 @@ function getMeta (resourceName, commandArgs) { const [ns, set, bin, exp, index] = commandArgs // The `ext` argument was added to IndexCreate in 6.3.0 - meta = commandArgs.length > 8 - ? getMetaForIndex(ns, set, bin, index) - : getMetaForIndex(ns, set, bin, exp) + meta = getMetaForIndex(ns, set, bin, commandArgs.length > 8 ? index : exp) } else if (resourceName === 'Query') { const { ns, set } = commandArgs[2] meta = getMetaForQuery({ ns, set }) diff --git a/packages/datadog-plugin-amqplib/src/client.js b/packages/datadog-plugin-amqplib/src/client.js index 943cfd0f31..0985d6389b 100644 --- a/packages/datadog-plugin-amqplib/src/client.js +++ b/packages/datadog-plugin-amqplib/src/client.js @@ -33,7 +33,7 @@ class AmqplibClientPlugin extends ClientPlugin { }, }, ctx) - fields.headers = fields.headers || {} + fields.headers ||= {} this.tracer.inject(span, TEXT_MAP, fields.headers) diff --git a/packages/datadog-plugin-amqplib/src/producer.js b/packages/datadog-plugin-amqplib/src/producer.js index e902c0e852..75ee015d48 100644 --- a/packages/datadog-plugin-amqplib/src/producer.js +++ b/packages/datadog-plugin-amqplib/src/producer.js @@ -52,7 +52,7 @@ class AmqplibProducerPlugin extends ProducerPlugin { }, }, ctx) - fields.headers = fields.headers || {} + fields.headers ||= {} this.tracer.inject(span, TEXT_MAP, fields.headers) diff --git a/packages/datadog-plugin-aws-sdk/src/services/eventbridge.js b/packages/datadog-plugin-aws-sdk/src/services/eventbridge.js index 471d8f109c..e9e6b2b879 100644 --- a/packages/datadog-plugin-aws-sdk/src/services/eventbridge.js +++ b/packages/datadog-plugin-aws-sdk/src/services/eventbridge.js @@ -39,9 +39,9 @@ class EventBridge extends BaseAwsSdkPlugin { const rulename = params.Name ?? '' return { 'resource.name': operation ? `${operation} ${params.source}` : params.source, - 'aws.eventbridge.source': `${params.source}`, + 'aws.eventbridge.source': params.source, 'messaging.system': 'aws_eventbridge', - rulename: `${rulename}`, + rulename, } } diff --git a/packages/datadog-plugin-aws-sdk/src/services/stepfunctions.js b/packages/datadog-plugin-aws-sdk/src/services/stepfunctions.js index 283ccd22be..28a5a6a844 100644 --- a/packages/datadog-plugin-aws-sdk/src/services/stepfunctions.js +++ b/packages/datadog-plugin-aws-sdk/src/services/stepfunctions.js @@ -29,9 +29,9 @@ class Stepfunctions extends BaseAwsSdkPlugin { generateTags (params, operation, response) { if (!params) return - const tags = { 'resource.name': params.name ? `${operation} ${params.name}` : `${operation}` } + const tags = { 'resource.name': params.name ? `${operation} ${params.name}` : operation } if (operation === 'startExecution' || operation === 'startSyncExecution') { - tags.statemachinearn = `${params.stateMachineArn}` + tags.statemachinearn = params.stateMachineArn } return tags } diff --git a/packages/datadog-plugin-azure-event-hubs/src/producer.js b/packages/datadog-plugin-azure-event-hubs/src/producer.js index 4c17ff6867..22ad0b1528 100644 --- a/packages/datadog-plugin-azure-event-hubs/src/producer.js +++ b/packages/datadog-plugin-azure-event-hubs/src/producer.js @@ -46,9 +46,7 @@ class AzureEventHubsProducerPlugin extends ProducerPlugin { } injectTraceContext(this.tracer, span, ctx.eventData) } - } - - if (ctx.functionName === 'sendBatch') { + } else if (ctx.functionName === 'sendBatch') { const eventData = ctx.eventData const eventDataLength = eventData.length || eventData._context.connection._eventsCount span.setTag('messaging.operation', 'send') diff --git a/packages/datadog-plugin-azure-service-bus/src/producer.js b/packages/datadog-plugin-azure-service-bus/src/producer.js index 83f8a1a3b0..32085fc1c9 100644 --- a/packages/datadog-plugin-azure-service-bus/src/producer.js +++ b/packages/datadog-plugin-azure-service-bus/src/producer.js @@ -45,9 +45,11 @@ class AzureServiceBusProducerPlugin extends ProducerPlugin { } injectTraceContext(this.tracer, span, ctx.msg) } - } - - if (ctx.functionName === 'send' || ctx.functionName === 'sendBatch' || ctx.functionName === 'scheduleMessages') { + } else if ( + ctx.functionName === 'send' || + ctx.functionName === 'sendBatch' || + ctx.functionName === 'scheduleMessages' + ) { const messages = ctx.msg const isBatch = messages.constructor?.name === 'ServiceBusMessageBatchImpl' if (isBatch) { diff --git a/packages/datadog-plugin-bullmq/src/producer.js b/packages/datadog-plugin-bullmq/src/producer.js index 529620a713..96c63d948d 100644 --- a/packages/datadog-plugin-bullmq/src/producer.js +++ b/packages/datadog-plugin-bullmq/src/producer.js @@ -222,7 +222,7 @@ class QueueAddBulkPlugin extends BaseBullmqProducerPlugin { for (let i = 0; i < jobs.length; i++) { const job = jobs[i] if (!job) continue - job.opts = job.opts || {} + job.opts ||= {} cache[i] = this._injectIntoOpts(span, job.opts) } ctx._ddMetadata = cache @@ -274,7 +274,7 @@ class FlowProducerAddPlugin extends BaseBullmqProducerPlugin { injectTraceContext (span, ctx) { const flow = ctx.arguments?.[0] if (!flow) return - flow.opts = flow.opts || {} + flow.opts ||= {} ctx._ddMetadata = this._injectIntoOpts(span, flow.opts) } @@ -283,7 +283,7 @@ class FlowProducerAddPlugin extends BaseBullmqProducerPlugin { if (!flow) { return { queueName: 'bullmq', payloadSize: 0, optsTarget: undefined } } - flow.opts = flow.opts || {} + flow.opts ||= {} return { queueName: flow.queueName || 'bullmq', payloadSize: flow.data ? getMessageSize(flow.data) : 0, diff --git a/packages/datadog-plugin-child_process/src/index.js b/packages/datadog-plugin-child_process/src/index.js index 44ae3d7c2f..8036a0fecd 100644 --- a/packages/datadog-plugin-child_process/src/index.js +++ b/packages/datadog-plugin-child_process/src/index.js @@ -52,7 +52,7 @@ class ChildProcessPlugin extends TracingPlugin { } if (truncated) { - meta['cmd.truncated'] = `${truncated}` + meta['cmd.truncated'] = 'true' } this.startSpan('command_execution', { @@ -81,7 +81,7 @@ class ChildProcessPlugin extends TracingPlugin { const span = ctx.currentStore?.span || this.activeSpan - span?.setTag('cmd.exit_code', `${exitCode}`) + span?.setTag('cmd.exit_code', String(exitCode)) span?.finish() return ctx.parentStore @@ -112,7 +112,7 @@ class ChildProcessPlugin extends TracingPlugin { const span = ctx.currentStore?.span || this.activeSpan - span?.setTag('cmd.exit_code', `${exitCode}`) + span?.setTag('cmd.exit_code', String(exitCode)) span?.finish() return ctx.parentStore diff --git a/packages/datadog-plugin-child_process/src/scrub-cmd-params.js b/packages/datadog-plugin-child_process/src/scrub-cmd-params.js index 29f3b62aa7..0ee58361ec 100644 --- a/packages/datadog-plugin-child_process/src/scrub-cmd-params.js +++ b/packages/datadog-plugin-child_process/src/scrub-cmd-params.js @@ -14,18 +14,15 @@ const envVarRegex = new RegExp(ENV_PATTERN) const REDACTED = '?' function extractVarNames (expression) { - const varNames = new Set() + const varNames = {} let match while ((match = VARNAMES_REGEX.exec(expression))) { - varNames.add(match[1]) + const name = match[1] + varNames[name] = `$${name}` } - const varNamesObject = {} - for (const varName of varNames.keys()) { - varNamesObject[varName] = `$${varName}` - } - return varNamesObject + return varNames } function getTokensByExpression (expressionTokens) { diff --git a/packages/datadog-plugin-fastify/src/tracing.js b/packages/datadog-plugin-fastify/src/tracing.js index debe8cf61b..bf6656e2f3 100644 --- a/packages/datadog-plugin-fastify/src/tracing.js +++ b/packages/datadog-plugin-fastify/src/tracing.js @@ -27,7 +27,7 @@ class FastifyTracingPlugin extends RouterPlugin { } function getParentStore (ctx) { - ctx.parentStore = ctx.parentStore ?? storage('legacy').getStore() + ctx.parentStore ??= storage('legacy').getStore() return ctx.parentStore } diff --git a/packages/datadog-plugin-fs/src/index.js b/packages/datadog-plugin-fs/src/index.js index c7d720669f..bbf6a620f0 100644 --- a/packages/datadog-plugin-fs/src/index.js +++ b/packages/datadog-plugin-fs/src/index.js @@ -7,10 +7,6 @@ class FsPlugin extends TracingPlugin { static operation = 'operation' static experimental = true - configure (...args) { - return super.configure(...args) - } - bindStart (ctx) { if (!this.activeSpan) return { noop: true } diff --git a/packages/datadog-plugin-graphql/src/execute.js b/packages/datadog-plugin-graphql/src/execute.js index 438c8de828..5e54fcee30 100644 --- a/packages/datadog-plugin-graphql/src/execute.js +++ b/packages/datadog-plugin-graphql/src/execute.js @@ -495,6 +495,7 @@ function wrapResolve (resolve) { // run in the parent store, not field.currentStore: the first sibling's // synchronous resolver already finished the shared graphql.resolve span, so // re-entering its store would parent user spans to a closed span. + // eslint-disable-next-line unicorn/prefer-else-if -- Keep sibling early return outside first-field setup. if (!isFirst) { return callInAsyncScope(resolve, this, arguments, rootCtx.abortController, field.parentStore, (err) => { if (updateFieldCh.hasSubscribers) { diff --git a/packages/datadog-plugin-graphql/src/tools/signature.js b/packages/datadog-plugin-graphql/src/tools/signature.js index e36bda9d2f..01015d0ba5 100644 --- a/packages/datadog-plugin-graphql/src/tools/signature.js +++ b/packages/datadog-plugin-graphql/src/tools/signature.js @@ -11,7 +11,7 @@ const transforms = require('./transforms') const cache = new WeakMap() function defaultEngineReportingSignature (ast, operationName) { - const key = operationName == null ? '' : operationName + const key = operationName ?? '' let inner = cache.get(ast) if (inner !== undefined) { const cached = inner.get(key) diff --git a/packages/datadog-plugin-http/src/client.js b/packages/datadog-plugin-http/src/client.js index 46f143915c..13f2b7a9af 100644 --- a/packages/datadog-plugin-http/src/client.js +++ b/packages/datadog-plugin-http/src/client.js @@ -10,6 +10,7 @@ const HTTP_HEADERS = formats.HTTP_HEADERS const urlFilter = require('../../dd-trace/src/plugins/util/urlfilter') const { buildClientHttpUrl } = require('../../dd-trace/src/plugins/util/url') const log = require('../../dd-trace/src/log') +const { stripQueryAndFragment } = require('../../dd-trace/src/util') const { CLIENT_PORT_KEY, COMPONENT, ERROR_MESSAGE, ERROR_TYPE, ERROR_STACK } = require('../../dd-trace/src/constants') const HTTP_STATUS_CODE = tags.HTTP_STATUS_CODE @@ -32,7 +33,7 @@ class HttpClientPlugin extends ClientPlugin { // A URL object (e.g. from the fetch integration) carries the query in // `options.search`, not `options.path`; keep it so url.full retains the query. const pathname = options.path || `${options.pathname || ''}${options.search || ''}` - const path = pathname ? pathname.split(/[?#]/)[0] : '/' + const path = pathname ? stripQueryAndFragment(pathname) : '/' const uri = `${base}${path}` const allowed = this.config.filter(uri) diff --git a/packages/datadog-plugin-jest/src/index.js b/packages/datadog-plugin-jest/src/index.js index 9b0d7e5f32..069e1bc0dd 100644 --- a/packages/datadog-plugin-jest/src/index.js +++ b/packages/datadog-plugin-jest/src/index.js @@ -237,9 +237,8 @@ class JestPlugin extends CiPlugin { const testSuiteMetadata = { ...getTestSuiteCommonTags(testCommand, frameworkVersion, testSuite, 'jest'), - // requestErrorTags from test env options may be undefined - ...(requestErrorTags !== undefined && requestErrorTags !== null ? requestErrorTags : {}), - ...(itrSkippingEnabledTags !== undefined && itrSkippingEnabledTags !== null ? itrSkippingEnabledTags : {}), + ...requestErrorTags, + ...itrSkippingEnabledTags, } if (_ddUnskippable) { diff --git a/packages/datadog-plugin-langchain/src/tokens.js b/packages/datadog-plugin-langchain/src/tokens.js index 856115e4b3..5c66b0d4b6 100644 --- a/packages/datadog-plugin-langchain/src/tokens.js +++ b/packages/datadog-plugin-langchain/src/tokens.js @@ -9,7 +9,7 @@ function getTokensFromLlmOutput (result) { const { llmOutput } = result if (!llmOutput) return tokens - const tokenUsage = llmOutput.tokenUsage || llmOutput.usage_metadata || llmOutput.usage_metadata + const tokenUsage = llmOutput.tokenUsage || llmOutput.usage_metadata if (!tokenUsage) return tokens for (const tokenNames of [['input', 'prompt'], ['output', 'completion'], ['total']]) { @@ -25,7 +25,7 @@ function getTokensFromLlmOutput (result) { } // assign total_tokens again in case it was improperly set the first time, or was not on tokenUsage - tokens.total = tokens.total || tokens.input + tokens.output + tokens.total ||= tokens.input + tokens.output return tokens } diff --git a/packages/datadog-plugin-mocha/src/index.js b/packages/datadog-plugin-mocha/src/index.js index 3275901d0f..32eb30a9ed 100644 --- a/packages/datadog-plugin-mocha/src/index.js +++ b/packages/datadog-plugin-mocha/src/index.js @@ -34,6 +34,8 @@ const { TEST_IS_MODIFIED, TEST_FINAL_STATUS, TEST_HAS_DYNAMIC_NAME, + TEST_FRAMEWORK_ADAPTER, + getTestSuiteExecutionKey, isModifiedTest, } = require('../../dd-trace/src/plugins/util/test') const { COMPONENT } = require('../../dd-trace/src/constants') @@ -88,23 +90,33 @@ class MochaPlugin extends CiPlugin { }) this.addBind('ci:mocha:test-suite:start', (ctx) => { - const { testSuiteAbsolutePath, isUnskippable, isForcedToRun, itrCorrelationId } = ctx + const { + testSuiteAbsolutePath, + testSuiteExecutionId, + isUnskippable, + isForcedToRun, + itrCorrelationId, + } = ctx // If the test module span is undefined, the plugin has not been initialized correctly and we bail out if (!this.testModuleSpan) { return } const testSuite = getTestSuitePath(testSuiteAbsolutePath, this.sourceRoot) + const testFramework = this.testFramework || this.constructor.id const testSuiteMetadata = { ...getTestSuiteCommonTags( this.command, this.frameworkVersion, testSuite, - 'mocha' + testFramework ), ...this.getSessionRequestErrorTags(), ...this.getSessionItrSkippingEnabledTags(), } + if (this.testFrameworkAdapter) { + testSuiteMetadata[TEST_FRAMEWORK_ADAPTER] = this.testFrameworkAdapter + } if (isUnskippable) { testSuiteMetadata[TEST_ITR_UNSKIPPABLE] = 'true' this.telemetry.count(TELEMETRY_ITR_UNSKIPPABLE, { testLevel: 'suite' }) @@ -144,8 +156,9 @@ class MochaPlugin extends CiPlugin { const store = storage('legacy').getStore() ctx.parentStore = store ctx.currentStore = { ...store, testSuiteSpan } - this._testSuiteSpansByTestSuite.set(testSuite, testSuiteSpan) - this._exportPendingWorkerTracesForTestSuite(testSuite) + const testSuiteKey = getTestSuiteExecutionKey(testSuite, testSuiteExecutionId) + this._testSuiteSpansByTestSuite.set(testSuiteKey, testSuiteSpan) + this._exportPendingWorkerTracesForTestSuite(testSuiteKey) }) this.addSub('ci:mocha:test-suite:finish', ({ testSuiteSpan, status }) => { @@ -203,8 +216,8 @@ class MochaPlugin extends CiPlugin { return ctx.currentStore }) - this.addSub('ci:mocha:worker:finish', () => { - this.tracer._exporter.flush() + this.addSub('ci:mocha:worker:finish', ({ onDone } = {}) => { + this.tracer._exporter.flush(onDone) }) this.addSub('ci:mocha:test:finish', ({ diff --git a/packages/datadog-plugin-prisma/src/index.js b/packages/datadog-plugin-prisma/src/index.js index 313851bf27..96572e3c9d 100644 --- a/packages/datadog-plugin-prisma/src/index.js +++ b/packages/datadog-plugin-prisma/src/index.js @@ -125,9 +125,13 @@ class PrismaPlugin extends DatabasePlugin { } } +/** + * @param {string} resource + * @param {{ name?: string, model?: string, method?: string }} [attributes] + */ function formatResourceName (resource, attributes) { if (attributes?.name) { - return `${attributes.name}`.trim() + return attributes.name.trim() } if (attributes?.model && attributes.method) { return `${attributes.model}.${attributes.method}`.trim() diff --git a/packages/datadog-plugin-rhea/src/producer.js b/packages/datadog-plugin-rhea/src/producer.js index 70ca0058d5..e5bbccebbc 100644 --- a/packages/datadog-plugin-rhea/src/producer.js +++ b/packages/datadog-plugin-rhea/src/producer.js @@ -37,7 +37,7 @@ class RheaProducerPlugin extends ProducerPlugin { function addDeliveryAnnotations (msg, tracer, span) { if (msg) { - msg.delivery_annotations = msg.delivery_annotations || {} + msg.delivery_annotations ||= {} tracer.inject(span, 'text_map', msg.delivery_annotations) diff --git a/packages/datadog-plugin-undici/src/index.js b/packages/datadog-plugin-undici/src/index.js index cb164b28a2..90f50f077e 100644 --- a/packages/datadog-plugin-undici/src/index.js +++ b/packages/datadog-plugin-undici/src/index.js @@ -7,6 +7,7 @@ const formats = require('../../../ext/formats') const HTTP_HEADERS = formats.HTTP_HEADERS const log = require('../../dd-trace/src/log') const { buildClientHttpUrl } = require('../../dd-trace/src/plugins/util/url') +const { stripQueryAndFragment } = require('../../dd-trace/src/util') const { CLIENT_PORT_KEY } = require('../../dd-trace/src/constants') const { @@ -62,7 +63,7 @@ class UndiciPlugin extends HttpClientPlugin { const host = port ? `${hostname}:${port}` : hostname const base = `${protocol}//${host}` - const pathname = path.split(/[?#]/)[0] + const pathname = stripQueryAndFragment(path) const uri = `${base}${pathname}` const allowed = this.config.filter(uri) diff --git a/packages/datadog-webpack/index.js b/packages/datadog-webpack/index.js index 18e06db5cf..728a29b670 100644 --- a/packages/datadog-webpack/index.js +++ b/packages/datadog-webpack/index.js @@ -141,7 +141,7 @@ class DatadogWebpackPlugin { // (#8980); absent peers stay opaque, so a build that does not opt into the feature does // not follow their dependency chain (#8635). if (matchesOptionalPeerFile(normalizedResource)) { - createData.loaders = createData.loaders || [] + createData.loaders ||= [] createData.loaders.push({ loader: require.resolve('./src/optional-peer-loader') }) log.debug('INLINE: optional-peer loader applied to %s', normalizedResource) return @@ -188,7 +188,7 @@ class DatadogWebpackPlugin { const version = packageJson.version const pkgPath = request === pkg ? pkg : `${pkg}/${modulePath}` - createData.loaders = createData.loaders || [] + createData.loaders ||= [] createData.loaders.unshift({ loader: require.resolve('./src/loader'), options: { pkg, version, path: pkgPath }, diff --git a/packages/dd-trace/src/appsec/downstream_requests.js b/packages/dd-trace/src/appsec/downstream_requests.js index 893bb55a9f..a2b50ad1c5 100644 --- a/packages/dd-trace/src/appsec/downstream_requests.js +++ b/packages/dd-trace/src/appsec/downstream_requests.js @@ -307,7 +307,7 @@ function parseBody (body, contentType) { const formBody = Buffer.isBuffer(body) ? body.toString('utf8') : String(body) const params = new URLSearchParams(formBody) const result = {} - for (const [key, value] of params.entries()) { + for (const [key, value] of params) { if (key in result) { const existing = result[key] if (Array.isArray(existing)) { diff --git a/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-base-analyzer.js b/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-base-analyzer.js index 79e7a8dbcd..6cb6f9f554 100644 --- a/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-base-analyzer.js +++ b/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-base-analyzer.js @@ -56,7 +56,7 @@ class HardcodedBaseAnalyzer extends Analyzer { } _getEvidence (value) { - return { value: `${value.data}` } + return { value: value.data } } _getLocation (value) { diff --git a/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-password-analyzer.js b/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-password-analyzer.js index 509b292291..34eb922382 100644 --- a/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-password-analyzer.js +++ b/packages/dd-trace/src/appsec/iast/analyzers/hardcoded-password-analyzer.js @@ -11,7 +11,7 @@ class HardcodedPasswordAnalyzer extends HardcodedBaseAnalyzer { } _getEvidence (value) { - return { value: `${value.ident}` } + return { value: value.ident } } } diff --git a/packages/dd-trace/src/appsec/iast/analyzers/sql-injection-analyzer.js b/packages/dd-trace/src/appsec/iast/analyzers/sql-injection-analyzer.js index 2f2f5434b1..9c88f74344 100644 --- a/packages/dd-trace/src/appsec/iast/analyzers/sql-injection-analyzer.js +++ b/packages/dd-trace/src/appsec/iast/analyzers/sql-injection-analyzer.js @@ -74,7 +74,7 @@ class SqlInjectionAnalyzer extends StoredInjectionAnalyzer { } analyze (value, store, dialect) { - store = store || storage('legacy').getStore() + store ||= storage('legacy').getStore() if (!(store && store.sqlAnalyzed)) { super.analyze(value, store, dialect) } diff --git a/packages/dd-trace/src/appsec/iast/taint-tracking/taint-tracking-impl.js b/packages/dd-trace/src/appsec/iast/taint-tracking/taint-tracking-impl.js index 15d70e3db6..70ea25a731 100644 --- a/packages/dd-trace/src/appsec/iast/taint-tracking/taint-tracking-impl.js +++ b/packages/dd-trace/src/appsec/iast/taint-tracking/taint-tracking-impl.js @@ -215,9 +215,7 @@ function getTaintTrackingImpl (telemetryVerbosity, dummy = false) { if (dummy) return TaintTrackingNoop // with Verbosity.DEBUG every invocation of a TaintedUtils method increases the EXECUTED_PROPAGATION metric - return isDebugAllowed(telemetryVerbosity) - ? createImplWith(getContextDebug) - : createImplWith(getContextDefault) + return createImplWith(isDebugAllowed(telemetryVerbosity) ? getContextDebug : getContextDefault) } function getTaintTrackingNoop () { diff --git a/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/utils.js b/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/utils.js index 882da0e028..736f127740 100644 --- a/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/utils.js +++ b/packages/dd-trace/src/appsec/iast/vulnerabilities-formatter/utils.js @@ -35,7 +35,7 @@ function stringifyWithRanges (obj, objRanges, loadSensitiveRanges = false) { let value const ranges = [] const sensitiveRanges = [] - objRanges = objRanges || {} + objRanges ||= {} if (objRanges || loadSensitiveRanges) { const cloneObj = Array.isArray(obj) ? [] : {} diff --git a/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js b/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js index 34de46a5a8..22beee3ca1 100644 --- a/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js +++ b/packages/dd-trace/src/appsec/iast/vulnerability-reporter.js @@ -21,7 +21,7 @@ let stackTraceMaxDepth let maxStackTraces function canAddVulnerability (vulnerability) { - const hasRequiredFields = vulnerability?.evidence && vulnerability?.type && vulnerability?.location + const hasRequiredFields = vulnerability?.evidence && vulnerability.type && vulnerability.location if (!hasRequiredFields) return false const isDuplicated = deduplicationEnabled && isDuplicatedVulnerability(vulnerability) @@ -64,7 +64,7 @@ function addVulnerability (iastContext, vulnerability, callSiteFrames) { } if (iastContext?.rootSpan) { - iastContext[VULNERABILITIES_KEY] = iastContext[VULNERABILITIES_KEY] || [] + iastContext[VULNERABILITIES_KEY] ||= [] iastContext[VULNERABILITIES_KEY].push(vulnerability) } else { sendVulnerabilities([vulnerability], span) diff --git a/packages/dd-trace/src/appsec/sdk/track_event.js b/packages/dd-trace/src/appsec/sdk/track_event.js index 10388a51ae..cbd7685d34 100644 --- a/packages/dd-trace/src/appsec/sdk/track_event.js +++ b/packages/dd-trace/src/appsec/sdk/track_event.js @@ -160,7 +160,7 @@ function flattenFields (fields, depth = 0) { if (value && typeof value === 'object') { const { result: flatValue, truncated: inheritTruncated } = flattenFields(value, depth + 1) - truncated = truncated || inheritTruncated + truncated ||= inheritTruncated if (flatValue) { for (const flatKey of Object.keys(flatValue)) { diff --git a/packages/dd-trace/src/appsec/waf/waf_manager.js b/packages/dd-trace/src/appsec/waf/waf_manager.js index f1379b2d46..1620661df1 100644 --- a/packages/dd-trace/src/appsec/waf/waf_manager.js +++ b/packages/dd-trace/src/appsec/waf/waf_manager.js @@ -39,7 +39,7 @@ class WAFManager { const { obfuscatorKeyRegex, obfuscatorValueRegex } = this.config return new DDWAF(rules, WAFManager.defaultWafConfigPath, { obfuscatorKeyRegex, obfuscatorValueRegex }) } catch (err) { - this.ddwafVersion = this.ddwafVersion || 'unknown' + this.ddwafVersion ||= 'unknown' Reporter.reportWafInit(this.ddwafVersion, 'unknown') log.error('[ASM] AppSec could not load native package. In-app WAF features will not be available.', err) diff --git a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js index 0893ab8d1d..a502df8990 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js +++ b/packages/dd-trace/src/ci-visibility/exporters/ci-visibility-exporter.js @@ -15,6 +15,7 @@ const { uploadCoverageReport: uploadCoverageReportRequest } = require('../reques const { uploadTestScreenshot: uploadTestScreenshotRequest } = require('../requests/upload-test-screenshot') const { parsers } = require('../../config/parsers') const log = require('../../log') +const { getSegment } = require('../../util') const BufferingExporter = require('../../exporters/common/buffering-exporter') const { GIT_REPOSITORY_URL, GIT_COMMIT_SHA } = require('../../plugins/util/tags') const { sendGitMetadata: sendGitMetadataRequest } = require('./git/git_metadata') @@ -28,8 +29,7 @@ function getTestConfigurationTags (tags) { } return Object.keys(tags).reduce((acc, key) => { if (key.startsWith('test.configuration.')) { - const [, configKey] = key.split('test.configuration.') - acc[configKey] = tags[key] + acc[getSegment(key, 'test.configuration.', 1)] = tags[key] } return acc }, {}) diff --git a/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js b/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js index 4fcde1070d..987c39ba3c 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js +++ b/packages/dd-trace/src/ci-visibility/exporters/git/git_metadata.js @@ -8,6 +8,7 @@ const FormData = require('../../../exporters/common/form-data') const request = require('../../../exporters/common/request') const log = require('../../../log') +const { getSegment } = require('../../../util') const { getLatestCommits, getRepositoryUrl, @@ -145,7 +146,7 @@ function uploadPackFile ({ url, isEvpProxy, evpProxyPrefix, packFileToUpload, re try { const packFileContent = fs.readFileSync(packFileToUpload) // The original filename includes a random prefix, so we remove it here - const [, filename] = path.basename(packFileToUpload).split('-') + const filename = getSegment(path.basename(packFileToUpload), '-', 1) form.append('packfile', packFileContent, { filename, contentType: 'application/octet-stream', diff --git a/packages/dd-trace/src/ci-visibility/exporters/test-worker/webdriverio.js b/packages/dd-trace/src/ci-visibility/exporters/test-worker/webdriverio.js new file mode 100644 index 0000000000..d34b1ce284 --- /dev/null +++ b/packages/dd-trace/src/ci-visibility/exporters/test-worker/webdriverio.js @@ -0,0 +1,26 @@ +'use strict' + +const WEBDRIVERIO_WORKER_ENV = '_DD_TEST_OPTIMIZATION_WEBDRIVERIO_WORKER' +const WEBDRIVERIO_WORKER_EVENT = 'workerEvent' +const WEBDRIVERIO_WORKER_ORIGIN = 'datadog' + +/** + * Wraps an internal worker payload in the event WebdriverIO reserves for worker messages. + * + * @param {unknown} args + * @returns {{args: unknown, name: string, origin: string}} + */ +function createWebdriverioWorkerMessage (args) { + return { + origin: WEBDRIVERIO_WORKER_ORIGIN, + name: WEBDRIVERIO_WORKER_EVENT, + args, + } +} + +module.exports = { + createWebdriverioWorkerMessage, + WEBDRIVERIO_WORKER_ENV, + WEBDRIVERIO_WORKER_EVENT, + WEBDRIVERIO_WORKER_ORIGIN, +} diff --git a/packages/dd-trace/src/ci-visibility/exporters/test-worker/writer.js b/packages/dd-trace/src/ci-visibility/exporters/test-worker/writer.js index 02379398e3..ba0451df00 100644 --- a/packages/dd-trace/src/ci-visibility/exporters/test-worker/writer.js +++ b/packages/dd-trace/src/ci-visibility/exporters/test-worker/writer.js @@ -2,6 +2,10 @@ const { JSONEncoder } = require('../../encode/json-encoder') const { getEnvironmentVariable } = require('../../../config/helper') const log = require('../../../log') +const { + createWebdriverioWorkerMessage, + WEBDRIVERIO_WORKER_ENV, +} = require('./webdriverio') function getVitestWorkerPort () { const port = globalThis.__vitest_worker__?.ctx?.port @@ -13,6 +17,7 @@ class Writer { this._encoder = new JSONEncoder() // Code used to identify the type of payload being sent to the main process this._interprocessCode = interprocessCode + this._isWebdriverioWorker = !!getEnvironmentVariable(WEBDRIVERIO_WORKER_ENV) } /** @@ -43,9 +48,12 @@ class Writer { // Old because vitest@>=4 uses `DD_VITEST_WORKER` and reports arrays just like other frameworks // Before vitest@>=4, we need the `__tinypool_worker_message__` property, or tinypool will crash const isVitestWorkerOld = !!getEnvironmentVariable('TINYPOOL_WORKER_ID') - const payload = isVitestWorkerOld + let payload = isVitestWorkerOld ? { __tinypool_worker_message__: true, interprocessCode: this._interprocessCode, data } : [this._interprocessCode, data] + if (this._isWebdriverioWorker) { + payload = createWebdriverioWorkerMessage(payload) + } const vitestWorkerPort = getVitestWorkerPort() if (vitestWorkerPort) { diff --git a/packages/dd-trace/src/ci-visibility/requests/request.js b/packages/dd-trace/src/ci-visibility/requests/request.js index ef6e6e699a..bd3b194e7f 100644 --- a/packages/dd-trace/src/ci-visibility/requests/request.js +++ b/packages/dd-trace/src/ci-visibility/requests/request.js @@ -42,7 +42,7 @@ function request (data, options, callback) { if (url.protocol === 'unix:') { opts.socketPath = url.pathname } else { - opts.path = opts.path ?? url.path + opts.path ??= url.path opts.protocol = url.protocol opts.hostname = url.hostname opts.port = url.port @@ -100,9 +100,9 @@ function request (data, options, callback) { if (res.statusCode === 429 && !hasRetried) { const resetHeader = res.headers['x-ratelimit-reset'] const resetTs = (resetHeader === null || resetHeader === undefined) - ? Number.NaN + ? NaN : Number.parseInt(resetHeader, 10) - const waitMs = Number.isFinite(resetTs) ? Math.max(0, resetTs * 1000 - Date.now()) : Number.NaN + const waitMs = Number.isFinite(resetTs) ? Math.max(0, resetTs * 1000 - Date.now()) : NaN if (Number.isFinite(waitMs) && waitMs <= RATE_LIMIT_MAX_WAIT_MS) { hasRetried = true diff --git a/packages/dd-trace/src/datastreams/encoding.js b/packages/dd-trace/src/datastreams/encoding.js index e91b184dd4..e9eab5d67b 100644 --- a/packages/dd-trace/src/datastreams/encoding.js +++ b/packages/dd-trace/src/datastreams/encoding.js @@ -40,7 +40,7 @@ function encodeVarintInto (target, offset, value) { let i = offset const limit = offset + maxVarLen64 - 1 // if first byte is 1, the number is negative in javascript, but we want to interpret it as positive - while ((high !== 0 || low < 0 || low > 0x80) && i < limit) { + while ((high !== 0 || low < 0 || low > 0x7F) && i < limit) { target[i] = (low & 0x7F) | 0x80 low >>>= 7 low |= (high & 0x7F) << 25 @@ -88,7 +88,7 @@ function decodeUvarint64 ( low |= n << s } if (s > 0) { - high |= s - 32 > 0 ? n << (s - 32) : n >> (32 - s) + high |= s > 32 ? n << (s - 32) : n >> (32 - s) } return [low, high, bytes] } @@ -96,8 +96,7 @@ function decodeUvarint64 ( low |= (n & 0x7F) << s } if (s > 0) { - high |= - s - 32 > 0 ? (n & 0x7F) << (s - 32) : (n & 0x7F) >> (32 - s) + high |= s > 32 ? (n & 0x7F) << (s - 32) : (n & 0x7F) >> (32 - s) } s += 7 } diff --git a/packages/dd-trace/src/debugger/devtools_client/probe_sampler.js b/packages/dd-trace/src/debugger/devtools_client/probe_sampler.js index 001d05efca..bc447a56e0 100644 --- a/packages/dd-trace/src/debugger/devtools_client/probe_sampler.js +++ b/packages/dd-trace/src/debugger/devtools_client/probe_sampler.js @@ -36,10 +36,7 @@ function getRemoveProbeExpression (id) { * @returns {string} */ function compileBreakpointCondition (probes) { - const probeConditions = [] - for (const probe of probes) { - probeConditions.push(compileProbeCondition(probe)) - } + const probeConditions = probes.map(compileProbeCondition) // NOTE: $dd_sampler is read from the realm-local `globalThis` where it was installed (the main // realm). A probe whose code runs in a different V8 realm (e.g. a `vm.createContext` script with a diff --git a/packages/dd-trace/src/debugger/devtools_client/send.js b/packages/dd-trace/src/debugger/devtools_client/send.js index ec4aeead80..460af4f268 100644 --- a/packages/dd-trace/src/debugger/devtools_client/send.js +++ b/packages/dd-trace/src/debugger/devtools_client/send.js @@ -67,14 +67,13 @@ function send (message, logger, dd, snapshot, processTags) { if (pruned) { json = pruned - size = Buffer.byteLength(json) } else { // Fallback if pruning fails const line = Object.keys(snapshot.captures.lines)[0] snapshot.captures.lines[line] = { pruned: true } json = JSON.stringify(payload) - size = Buffer.byteLength(json) } + size = Buffer.byteLength(json) } jsonBuffer.write(json, size) diff --git a/packages/dd-trace/src/debugger/devtools_client/snapshot/index.js b/packages/dd-trace/src/debugger/devtools_client/snapshot/index.js index ac0c5824ad..938331d792 100644 --- a/packages/dd-trace/src/debugger/devtools_client/snapshot/index.js +++ b/packages/dd-trace/src/debugger/devtools_client/snapshot/index.js @@ -61,7 +61,7 @@ async function getLocalStateForCallFrame (callFrame, limits, deadlineNs = BIGINT // Delay calling `processRawState` so caller can resume the main thread before processing `rawState` return { processLocalState () { - processedState = processedState ?? processRawState(rawState, maxLength) + processedState ??= processRawState(rawState, maxLength) return processedState }, fatalErrors: ctx.fatalErrors, @@ -155,7 +155,7 @@ async function evaluateCaptureExpressions (callFrame, expressions, deadlineNs = fatalErrors.push(...ctx.fatalErrors) } - if (ctx.deadlineReached === true) { + if (ctx.deadlineReached) { // Add the current expression (properties may be incomplete due to timeout) rawResults.push({ name, remoteObject: result, maxLength }) // Add stub entries for remaining uncaptured expressions diff --git a/packages/dd-trace/src/debugger/devtools_client/snapshot/redaction.js b/packages/dd-trace/src/debugger/devtools_client/snapshot/redaction.js index 020f94150a..af573a69f9 100644 --- a/packages/dd-trace/src/debugger/devtools_client/snapshot/redaction.js +++ b/packages/dd-trace/src/debugger/devtools_client/snapshot/redaction.js @@ -97,10 +97,24 @@ const REDACTED_IDENTIFIERS = new Set( 'XSRF-TOKEN', ...config.dynamicInstrumentation.redactedIdentifiers, ] - .map((name) => normalizeName(name)) - .filter((name) => excludedIdentifiers.has(name) === false) + .map(normalizeIdentifier) + .filter(isIncludedIdentifier) ) +/** + * @param {string} name + */ +function normalizeIdentifier (name) { + return normalizeName(name) +} + +/** + * @param {string} name + */ +function isIncludedIdentifier (name) { + return !excludedIdentifiers.has(name) +} + function normalizeName (name, isSymbol) { if (isSymbol) name = name.slice(7, -1) // Remove `Symbol(` and `)` return name.toLowerCase().replaceAll(/[-_@$.]/g, '') diff --git a/packages/dd-trace/src/encode/tags-processors.js b/packages/dd-trace/src/encode/tags-processors.js index 99c15ff747..22f7386033 100644 --- a/packages/dd-trace/src/encode/tags-processors.js +++ b/packages/dd-trace/src/encode/tags-processors.js @@ -60,11 +60,11 @@ function eventTimeNano (event) { } function normalizeSpan (span) { - span.service = span.service || DEFAULT_SERVICE_NAME + span.service ||= DEFAULT_SERVICE_NAME if (span.service.length > MAX_SERVICE_LENGTH) { span.service = span.service.slice(0, MAX_SERVICE_LENGTH) } - span.name = span.name || DEFAULT_SPAN_NAME + span.name ||= DEFAULT_SPAN_NAME if (span.name.length > MAX_NAME_LENGTH) { span.name = span.name.slice(0, MAX_NAME_LENGTH) } diff --git a/packages/dd-trace/src/exporters/common/url.js b/packages/dd-trace/src/exporters/common/url.js index 7bf2ab9b82..bd381c996d 100644 --- a/packages/dd-trace/src/exporters/common/url.js +++ b/packages/dd-trace/src/exporters/common/url.js @@ -32,11 +32,9 @@ function parseUrl (urlObjOrString) { // `urlToHttpOptions` returns `pathname` at runtime, but @types/node narrows it // to `ClientRequestArgs`, which omits it; cast so the named-pipe fold below can // read and rewrite it. - const url = /** @type {import('node:http').ClientRequestArgs & { pathname: string }} */ ( - urlObjOrString !== null && typeof urlObjOrString === 'object' - ? urlToHttpOptions(urlObjOrString) - : urlToHttpOptions(new URL(urlObjOrString)) - ) + const url = /** @type {import('node:http').ClientRequestArgs & { pathname: string }} */ (urlToHttpOptions( + urlObjOrString !== null && typeof urlObjOrString === 'object' ? urlObjOrString : new URL(urlObjOrString) + )) if (url.protocol === 'unix:' && url.hostname === '.') { url.path = url.pathname = `//.${url.pathname}` diff --git a/packages/dd-trace/src/exporters/electron/index.js b/packages/dd-trace/src/exporters/electron/index.js index 813c0da2f0..88ef3e0276 100644 --- a/packages/dd-trace/src/exporters/electron/index.js +++ b/packages/dd-trace/src/exporters/electron/index.js @@ -16,6 +16,8 @@ class ElectronExporter { } export (spans) { + if (!traceChannel.hasSubscribers) return + this.#traces.push(spans) const { flushInterval } = this._config @@ -35,7 +37,8 @@ class ElectronExporter { clearTimeout(this.#timer) this.#timer = undefined - const traces = this.#traces.splice(0) + const traces = this.#traces + this.#traces = [] if (traces.length > 0 && traceChannel.hasSubscribers) { const formattedTraces = traces.map(spans => spans.map(span => normalizeSpan(truncateSpan(span)))) diff --git a/packages/dd-trace/src/external-logger/src/index.js b/packages/dd-trace/src/external-logger/src/index.js index c00e519f5e..cd3532a7bb 100644 --- a/packages/dd-trace/src/external-logger/src/index.js +++ b/packages/dd-trace/src/external-logger/src/index.js @@ -78,7 +78,6 @@ class ExternalLogger { // Flushes logs with optional callback for when the call is complete flush (cb = () => {}) { - let logs let numLogs let encodedLogs @@ -88,7 +87,7 @@ class ExternalLogger { } try { - logs = this.queue + const logs = this.queue this.queue = [] numLogs = logs.length diff --git a/packages/dd-trace/src/flare/index.js b/packages/dd-trace/src/flare/index.js index b270bbc8ce..584a861661 100644 --- a/packages/dd-trace/src/flare/index.js +++ b/packages/dd-trace/src/flare/index.js @@ -39,8 +39,8 @@ const flare = { logChannel?.unsubscribe(logger) logChannel = new LogChannel(logLevel) logChannel.subscribe(logger) - tracerLogs = tracerLogs || new FlareFile() - timer = timer || setTimeout(flare.cleanup, TIMEOUT) + tracerLogs ||= new FlareFile() + timer ||= setTimeout(flare.cleanup, TIMEOUT) }, send (task) { diff --git a/packages/dd-trace/src/id.js b/packages/dd-trace/src/id.js index ff7bbc9d88..b0006e44ba 100644 --- a/packages/dd-trace/src/id.js +++ b/packages/dd-trace/src/id.js @@ -185,7 +185,7 @@ function toNumberString (buffer, radix) { let low = readInt32(buffer, buffer.length - 4) let str = '' - radix = radix || 10 + radix ||= 10 while (1) { const mod = (high % radix) * UINT_MAX + low diff --git a/packages/dd-trace/src/llmobs/experiments/client.js b/packages/dd-trace/src/llmobs/experiments/client.js index 8477ae6b96..eef517cc08 100644 --- a/packages/dd-trace/src/llmobs/experiments/client.js +++ b/packages/dd-trace/src/llmobs/experiments/client.js @@ -14,9 +14,11 @@ function apiHost (site) { } // Web-app host for dashboard URLs. Single-level sites (datadoghq.com, -// ddog-gov.com) are served from the `app.` subdomain; regional sites -// (us3.datadoghq.com, ap1.datadoghq.com) are used as-is. +// ddog-gov.com) are served from the `app.` subdomain; staging uses +// dd.datad0g.com; regional sites (us3.datadoghq.com, ap1.datadoghq.com) +// are used as-is. function appHost (site) { + if (site === 'datad0g.com') return 'dd.datad0g.com' return site.split('.').length === 2 ? `app.${site}` : site } diff --git a/packages/dd-trace/src/llmobs/experiments/dataset.js b/packages/dd-trace/src/llmobs/experiments/dataset.js index db9bf4f872..a3b3b93761 100644 --- a/packages/dd-trace/src/llmobs/experiments/dataset.js +++ b/packages/dd-trace/src/llmobs/experiments/dataset.js @@ -2,15 +2,35 @@ const { API_BASE_PATH } = require('./client') -// Immutable dataset record: { input, expectedOutput?, metadata? }. +// Dataset record: { input, expectedOutput?, metadata?, id? }. +// `id` may be user-provided before push or filled from the backend-created record. class DatasetRecord { - constructor (input, expectedOutput = null, metadata = {}) { + constructor (input, expectedOutput = null, metadata = {}, id = null) { this.input = input this.expectedOutput = expectedOutput ?? null this.metadata = metadata ?? {} + this.id = id ?? null } } +function createdRecordsFromResponse (response) { + if (Array.isArray(response?.records)) return response.records + if (Array.isArray(response?.data)) return response.data + return [] +} + +function recordIdFromCreatedRecord (record) { + return String(record?.id ?? record?.attributes?.id ?? '') +} + +function versionFromCreatedRecords (records) { + const versions = records + .map(record => Number(record?.attributes?.valid_from_version ?? record?.attributes?.version)) + .filter(Number.isFinite) + if (versions.length === 0) return null + return Math.max(...versions) +} + // A local buffer of dataset records, created remotely and pushed on first run // (or eagerly via push()). Pushes are incremental. class Dataset { @@ -22,6 +42,8 @@ class Dataset { #id #projectId #pushedCount + #version + #latestVersion constructor (client, name, description = '') { this.#client = client @@ -32,16 +54,20 @@ class Dataset { this.#id = null this.#projectId = null this.#pushedCount = 0 + this.#version = null + this.#latestVersion = null } // Build a Dataset that already exists remotely (used by pullDataset). - static fromExisting (client, name, description, id, projectId, records, recordIds) { + static fromExisting (client, name, description, id, projectId, records, recordIds, version, latestVersion) { const dataset = new Dataset(client, name, description) dataset.#id = id dataset.#projectId = projectId dataset.#records.push(...records) dataset.#recordIds.push(...recordIds) dataset.#pushedCount = records.length + dataset.#version = version ?? null + dataset.#latestVersion = latestVersion ?? version ?? null return dataset } @@ -74,6 +100,14 @@ class Dataset { return this.#projectId } + version () { + return this.#version + } + + latestVersion () { + return this.#latestVersion + } + // Dashboard URL for this dataset, or null until pushed/pulled. url () { if (this.#id === null) return null @@ -100,7 +134,12 @@ class Dataset { throw new Error(`Failed to create dataset '${this.#name}': ${err.message}`) } this.#id = response?.data?.id ?? null + if (this.#id === null) { + throw new Error(`Failed to create dataset '${this.#name}': backend response is missing dataset id`) + } this.#projectId = projectId + this.#version = response?.data?.attributes?.current_version ?? this.#version + this.#latestVersion = response?.data?.attributes?.current_version ?? this.#latestVersion } if (this.#pushedCount >= this.#records.length) return { pushedCount: 0, totalCount: 0 } @@ -108,6 +147,9 @@ class Dataset { const pending = this.#records.slice(this.#pushedCount) const records = pending.map((rec) => { const out = { input: rec.input } + if (rec.id != null) { + out.id = rec.id + } if (rec.expectedOutput !== null && rec.expectedOutput !== undefined) { out.expected_output = rec.expectedOutput } @@ -128,20 +170,30 @@ class Dataset { throw new Error(`Failed to push records to dataset '${this.#name}': ${err.message}`) } - // The append-records response returns created records under a top-level - // `records` field, not the usual `data` envelope. - const created = response?.records + // The append-records response has used both a top-level `records` array + // and JSON:API `data` resources. Accept either so generated/custom record + // ids are preserved for experiment row tagging. + const created = createdRecordsFromResponse(response) + const pushedVersion = versionFromCreatedRecords(created) + if (pushedVersion === null) { + // The dataset contents changed, but the backend did not report the new + // version. Avoid pinning later experiments to the pre-append create version. + this.#version = null + } else { + this.#version = pushedVersion + this.#latestVersion = Math.max(Number(this.#latestVersion ?? pushedVersion), pushedVersion) + } + let pushedCount = 0 - if (Array.isArray(created)) { - for (const node of created) { - const recordId = String(node?.id ?? '') - if (recordId !== '') pushedCount++ - this.#recordIds.push(recordId) + for (const [index, node] of created.entries()) { + const recordId = recordIdFromCreatedRecord(node) + if (recordId !== '') { + pushedCount++ + pending[index].id = recordId } - for (let i = created.length; i < pending.length; i++) this.#recordIds.push('') - } else { - for (let i = 0; i < pending.length; i++) this.#recordIds.push('') + this.#recordIds.push(recordId) } + for (let i = created.length; i < pending.length; i++) this.#recordIds.push('') // Advance by the snapshotted pending count, not the live records length, // so records added while this push was in flight aren't skipped by the next push. diff --git a/packages/dd-trace/src/llmobs/experiments/experiment.js b/packages/dd-trace/src/llmobs/experiments/experiment.js index d415f00a8f..8a2c603468 100644 --- a/packages/dd-trace/src/llmobs/experiments/experiment.js +++ b/packages/dd-trace/src/llmobs/experiments/experiment.js @@ -149,6 +149,8 @@ class Experiment { description: this.#description, ensure_unique: true, } + const datasetVersion = this.#dataset.version() + if (datasetVersion !== null) attributes.dataset_version = datasetVersion if (Object.keys(this.#config).length > 0) attributes.config = this.#config let created diff --git a/packages/dd-trace/src/llmobs/experiments/index.js b/packages/dd-trace/src/llmobs/experiments/index.js index ca9cd2f05c..b7b34edcb5 100644 --- a/packages/dd-trace/src/llmobs/experiments/index.js +++ b/packages/dd-trace/src/llmobs/experiments/index.js @@ -39,21 +39,38 @@ class Experiments { } // Create a local dataset buffer. Pushed remotely on first experiment run. - createDataset (name, description = '') { - return new Dataset(this.#client, name, description) + createDataset (name, descriptionOrOptions = '') { + const options = typeof descriptionOrOptions === 'string' + ? { description: descriptionOrOptions } + : (descriptionOrOptions ?? {}) + const dataset = new Dataset(this.#client, name, options.description ?? '') + const recordIds = new Set() + for (const record of options.records ?? []) { + if (record.id !== undefined && (typeof record.id !== 'string' || record.id.length === 0)) { + throw new Error('record id must be a non-empty string') + } + if (record.id !== undefined) { + if (recordIds.has(record.id)) throw new Error(`Duplicate record id '${record.id}'`) + recordIds.add(record.id) + } + dataset.addRecord(new DatasetRecord(record.inputData, record.expectedOutput, record.metadata, record.id)) + } + return dataset } // Pull an existing dataset by name (with its records). Polls with exponential // backoff to absorb read-after-write lag; pass `expectedRecordCount` to also // wait until that many records are readable. async pullDataset (name, options = {}) { - const { expectedRecordCount, maxWaitMs = 30_000 } = options + const { expectedRecordCount, maxWaitMs = 30_000, version } = options const projectId = await this.#client.ensureProjectId() let datasetId = null let description = '' let records = [] let recordIds = [] + let datasetVersion = version ?? null + let latestVersion = null let lastError = '' const succeeded = await retryWithBackoff(async () => { @@ -67,6 +84,8 @@ class Experiments { if (item?.attributes?.name === name) { datasetId = String(item?.id ?? '') description = String(item?.attributes?.description ?? '') + latestVersion = item?.attributes?.current_version ?? null + datasetVersion = version ?? latestVersion break } } @@ -78,16 +97,24 @@ class Experiments { let cursor = '' // Follow the meta.after / page[cursor] pagination until the last page. for (;;) { - const query = cursor ? `?page[cursor]=${encodeURIComponent(cursor)}` : '' + const query = new URLSearchParams() + if (cursor) query.set('page[cursor]', cursor) + if (datasetVersion !== null) query.set('filter[version]', String(datasetVersion)) // eslint-disable-next-line no-await-in-loop const resp = await this.#client.request( 'GET', - `${API_BASE_PATH}/${projectId}/datasets/${datasetId}/records${query}` + `${API_BASE_PATH}/${projectId}/datasets/${datasetId}/records?${query.toString()}` ) for (const item of resp?.data ?? []) { const attrs = item?.attributes ?? item - recs.push(new DatasetRecord(attrs?.input ?? null, attrs?.expected_output ?? null, attrs?.metadata ?? {})) - ids.push(String(item?.id ?? '')) + const recordId = String(item?.id ?? attrs?.id ?? '') + recs.push(new DatasetRecord( + attrs?.input ?? null, + attrs?.expected_output ?? null, + attrs?.metadata ?? {}, + recordId === '' ? null : recordId + )) + ids.push(recordId) } cursor = resp?.meta?.after ?? '' if (!cursor) break @@ -119,7 +146,17 @@ class Experiments { ) } - return Dataset.fromExisting(this.#client, name, description, datasetId, projectId, records, recordIds) + return Dataset.fromExisting( + this.#client, + name, + description, + datasetId, + projectId, + records, + recordIds, + datasetVersion, + latestVersion + ) } // Build an experiment: { name, dataset, task, evaluators, description?, config?, tags? }. diff --git a/packages/dd-trace/src/llmobs/experiments/noop.js b/packages/dd-trace/src/llmobs/experiments/noop.js index bb80c4142c..ae32172f6b 100644 --- a/packages/dd-trace/src/llmobs/experiments/noop.js +++ b/packages/dd-trace/src/llmobs/experiments/noop.js @@ -1,8 +1,90 @@ 'use strict' +const log = require('../../log') + +class NoopDataset { + #name + #records + + constructor (name = '', options = {}) { + this.#name = name + this.#records = (typeof options === 'string' ? [] : (options.records ?? [])).map(record => ({ + id: record.id ?? null, + input: record.inputData, + expectedOutput: record.expectedOutput ?? null, + metadata: record.metadata ?? {}, + })) + } + + addRecord (input, expectedOutput, metadata) { + this.#records.push({ id: null, input, expectedOutput: expectedOutput ?? null, metadata: metadata ?? {} }) + return this + } + + push () { + return Promise.resolve({ pushedCount: 0, totalCount: 0 }) + } + + name () { + return this.#name + } + + id () { + return null + } + + projectId () { + return null + } + + version () { + return null + } + + latestVersion () { + return null + } + + records () { + return [...this.#records] + } + + recordIds () { + return [] + } + + url () { + return null + } +} + +class NoopExperiment { + #name + + constructor (name = '') { + this.#name = name + } + + name () { + return this.#name + } + + experimentId () { + return null + } + + url () { + return null + } + + run () { + return Promise.resolve({ experimentId: null, rows: [], url: null }) + } +} + // No-op Experiments used when LLM Observability is disabled or the API/APP keys -// are not configured. Every operation throws a clear, actionable error rather -// than silently doing nothing, so misconfiguration surfaces immediately. +// are not configured. Operations warn and return inert objects rather than +// throwing, so intentionally disabled experiments remain graceful. class NoopExperiments { #reason @@ -10,20 +92,23 @@ class NoopExperiments { this.#reason = reason || 'LLMObs experiments are not available' } - #unavailable () { - return new Error(`LLMObs experiments unavailable: ${this.#reason}`) + #warn () { + log.warn('LLMObs experiments unavailable: %s', this.#reason) } - createDataset () { - throw this.#unavailable() + createDataset (name, options = {}) { + this.#warn() + return new NoopDataset(name, options) } - pullDataset () { - return Promise.reject(this.#unavailable()) + pullDataset (name) { + this.#warn() + return Promise.resolve(new NoopDataset(name)) } - experiment () { - throw this.#unavailable() + experiment (options = {}) { + this.#warn() + return new NoopExperiment(options.name) } } diff --git a/packages/dd-trace/src/llmobs/plugins/anthropic/index.js b/packages/dd-trace/src/llmobs/plugins/anthropic/index.js index db2d652d45..1535859e44 100644 --- a/packages/dd-trace/src/llmobs/plugins/anthropic/index.js +++ b/packages/dd-trace/src/llmobs/plugins/anthropic/index.js @@ -93,7 +93,7 @@ class AnthropicLLMObsPlugin extends LLMObsPlugin { const { usage } = chunk if (usage) { - const responseUsage = response.usage ?? (response.usage = { input_tokens: 0, output_tokens: 0 }) + const responseUsage = (response.usage ??= { input_tokens: 0, output_tokens: 0 }) responseUsage.output_tokens = usage.output_tokens const cacheCreationTokens = usage.cache_creation_input_tokens diff --git a/packages/dd-trace/src/llmobs/plugins/genai/index.js b/packages/dd-trace/src/llmobs/plugins/genai/index.js index 69c5d63cab..12da7da905 100644 --- a/packages/dd-trace/src/llmobs/plugins/genai/index.js +++ b/packages/dd-trace/src/llmobs/plugins/genai/index.js @@ -24,7 +24,7 @@ class GenAiLLMObsPlugin extends LLMObsPlugin { // Subscribe to streaming chunk events this.addSub('apm:google:genai:request:chunk', ({ ctx, chunk, done }) => { ctx.isStreaming = true - ctx.chunks = ctx.chunks || [] + ctx.chunks ||= [] if (chunk) ctx.chunks.push(chunk) if (!done) return diff --git a/packages/dd-trace/src/llmobs/plugins/langchain/handlers/index.js b/packages/dd-trace/src/llmobs/plugins/langchain/handlers/index.js index 2438e9ed8e..fd10a858a7 100644 --- a/packages/dd-trace/src/llmobs/plugins/langchain/handlers/index.js +++ b/packages/dd-trace/src/llmobs/plugins/langchain/handlers/index.js @@ -36,7 +36,7 @@ class LangChainLLMObsHandler { const runIdBase = runId ? runId.split('-').slice(0, -1).join('-') : '' const responseMetadata = message.response_metadata || {} - usage = usage || responseMetadata.usage || responseMetadata.tokenUsage || {} + usage ||= responseMetadata.usage || responseMetadata.tokenUsage || {} const inputTokens = usage.promptTokens || usage.inputTokens || usage.prompt_tokens || usage.input_tokens || 0 const outputTokens = diff --git a/packages/dd-trace/src/llmobs/sdk.js b/packages/dd-trace/src/llmobs/sdk.js index a93bcaf05a..0e2b2a1a19 100644 --- a/packages/dd-trace/src/llmobs/sdk.js +++ b/packages/dd-trace/src/llmobs/sdk.js @@ -318,14 +318,14 @@ class LLMObs extends NoopLLMObs { } throw e } finally { - if (autoinstrumented === false) { + if (!autoinstrumented) { telemetry.recordLLMObsAnnotate(span, err) } } } exportSpan (span) { - span = span || this._active() + span ||= this._active() let err = '' try { if (!span) { @@ -426,7 +426,7 @@ class LLMObs extends NoopLLMObs { err = 'invalid_metric_value' throw new Error('value must be a boolean for a boolean metric') } - if (metricType === 'json' && !(typeof value === 'object' && value != null && !Array.isArray(value))) { + if (metricType === 'json' && (typeof value !== 'object' || value == null || Array.isArray(value))) { err = 'invalid_metric_value' throw new Error('value must be a JSON object for a json metric') } diff --git a/packages/dd-trace/src/llmobs/tagger.js b/packages/dd-trace/src/llmobs/tagger.js index a1ec7a1bb7..6873208ba1 100644 --- a/packages/dd-trace/src/llmobs/tagger.js +++ b/packages/dd-trace/src/llmobs/tagger.js @@ -158,8 +158,7 @@ class LLMObsTagger { if (modelName) this.tagModelName(span, modelName) if (modelProvider) this._setTag(span, MODEL_PROVIDER, modelProvider) - sessionId = sessionId || - registry.get(parent)?.[SESSION_ID] || + sessionId ||= registry.get(parent)?.[SESSION_ID] || traceTags[SESSION_ID_TRACE_DEFAULT_KEY] || traceTags[PROPAGATED_SESSION_ID_KEY] if (sessionId) this._setTag(span, SESSION_ID, sessionId) diff --git a/packages/dd-trace/src/noop/proxy.js b/packages/dd-trace/src/noop/proxy.js index 47969c1ea5..6f6feaa1d9 100644 --- a/packages/dd-trace/src/noop/proxy.js +++ b/packages/dd-trace/src/noop/proxy.js @@ -55,7 +55,7 @@ class NoopProxy { if (typeof fn !== 'function') return - options = options || {} + options ||= {} return this._tracer.trace(name, options, fn) } @@ -68,7 +68,7 @@ class NoopProxy { if (typeof fn !== 'function') return fn - options = options || {} + options ||= {} return this._tracer.wrap(name, options, fn) } diff --git a/packages/dd-trace/src/opentelemetry/metrics/otlp_transformer.js b/packages/dd-trace/src/opentelemetry/metrics/otlp_transformer.js index c383d1e85a..a49e1e59cc 100644 --- a/packages/dd-trace/src/opentelemetry/metrics/otlp_transformer.js +++ b/packages/dd-trace/src/opentelemetry/metrics/otlp_transformer.js @@ -198,7 +198,7 @@ class OtlpTransformer extends OtlpTransformerBase { if (isJson) { dataPoint.startTimeUnixNano = String(dataPoint.startTimeUnixNano) dataPoint.timeUnixNano = String(dataPoint.timeUnixNano) - dataPoint.count = dataPoint.count || 0 + dataPoint.count ||= 0 } return dataPoint diff --git a/packages/dd-trace/src/opentelemetry/metrics/periodic_metric_reader.js b/packages/dd-trace/src/opentelemetry/metrics/periodic_metric_reader.js index a192969103..ccfd8ac151 100644 --- a/packages/dd-trace/src/opentelemetry/metrics/periodic_metric_reader.js +++ b/packages/dd-trace/src/opentelemetry/metrics/periodic_metric_reader.js @@ -253,7 +253,8 @@ class PeriodicMetricReader { #collectAndExport (callback = () => {}) { // Atomically drain measurements for export. New measurements can be recorded // during export without interfering with this batch. - const allMeasurements = this.#measurements.splice(0) + const allMeasurements = this.#measurements + this.#measurements = [] for (const instrument of this.observableInstruments) { const observableMeasurements = instrument.collect() diff --git a/packages/dd-trace/src/opentracing/span.js b/packages/dd-trace/src/opentracing/span.js index b5f20d61b0..6cfee05656 100644 --- a/packages/dd-trace/src/opentracing/span.js +++ b/packages/dd-trace/src/opentracing/span.js @@ -440,7 +440,7 @@ class DatadogSpan { } } - spanContext._trace.ticks = spanContext._trace.ticks || now() + spanContext._trace.ticks ||= now() if (startTime) { spanContext._trace.startTime = startTime } diff --git a/packages/dd-trace/src/opentracing/span_context.js b/packages/dd-trace/src/opentracing/span_context.js index 28fdd29123..6daee283b4 100644 --- a/packages/dd-trace/src/opentracing/span_context.js +++ b/packages/dd-trace/src/opentracing/span_context.js @@ -8,7 +8,7 @@ const TRACE_ID_128 = '_dd.p.tid' class DatadogSpanContext { constructor (props) { - props = props || {} + props ||= {} this._traceId = props.traceId this._spanId = props.spanId diff --git a/packages/dd-trace/src/plugins/ci_plugin.js b/packages/dd-trace/src/plugins/ci_plugin.js index 1fec03d941..44084e4a2b 100644 --- a/packages/dd-trace/src/plugins/ci_plugin.js +++ b/packages/dd-trace/src/plugins/ci_plugin.js @@ -48,12 +48,16 @@ const { CI_APP_ORIGIN, getTestSessionCommonTags, getTestModuleCommonTags, + TEST_FRAMEWORK, + TEST_FRAMEWORK_ADAPTER, + TEST_FRAMEWORK_VERSION, TEST_SUITE_ID, TEST_MODULE_ID, TEST_SESSION_ID, TEST_COMMAND, TEST_LEVELS_METADATA, TEST_MODULE, + TEST_TYPE, TEST_SESSION_NAME, getTestSuiteCommonTags, TEST_STATUS, @@ -62,6 +66,8 @@ const { ITR_CORRELATION_ID, TEST_SOURCE_FILE, TEST_SUITE, + TEST_SUITE_EXECUTION_ID, + getTestSuiteExecutionKey, getFileAndLineNumberFromError, DI_ERROR_DEBUG_INFO_CAPTURED, DI_DEBUG_ERROR_PREFIX, @@ -101,6 +107,7 @@ const FRAMEWORK_TO_TRIMMED_COMMAND = { cucumber: 'cucumber-js', playwright: 'playwright test', jest: 'jest', + webdriverio: 'wdio', } const WORKER_EXPORTER_TO_TEST_FRAMEWORK = { @@ -131,7 +138,8 @@ function withTimeout (promise, timeoutMs) { } function setItrSkippingEnabledTagFromLibraryConfig (plugin, frameworkVersion) { - const libraryCapabilitiesTags = getDefaultLibraryCapabilitiesTags(plugin.constructor.id, frameworkVersion) + const testFramework = plugin.testFramework || plugin.constructor.id + const libraryCapabilitiesTags = getDefaultLibraryCapabilitiesTags(testFramework, frameworkVersion) if (!libraryCapabilitiesTags[DD_CAPABILITIES_TEST_IMPACT_ANALYSIS] || !plugin.libraryConfig || @@ -170,16 +178,18 @@ module.exports = class CiPlugin extends Plugin { this.rootDir = process.cwd() // fallback in case :session:start events are not emitted this._testSuiteSpansByTestSuite = new Map() this._pendingWorkerTracesByTestSuite = new Map() + this._workerTraceExecutionIds = new WeakMap() this._pendingRequestErrorTags = [] this.addSub(`ci:${this.constructor.id}:library-configuration`, (ctx) => { - const { onDone, frameworkVersion } = ctx + const { basicReportingOnly, onDone, frameworkVersion } = ctx ctx.currentStore = legacyStorage.getStore() if (!this.tracer._exporter || !this.tracer._exporter.getLibraryConfiguration) { return onDone({ err: new Error('Test optimization was not initialized correctly') }) } this.tracer._exporter.getLibraryConfiguration(this.testConfiguration, (err, libraryConfig) => { + const effectiveLibraryConfig = err ? undefined : basicReportingOnly ? {} : libraryConfig if (err) { this.libraryConfig = undefined this.itrCorrelationId = undefined @@ -187,7 +197,7 @@ module.exports = class CiPlugin extends Plugin { log.error('Library configuration could not be fetched. %s', err.message) this._addRequestErrorTag(DD_CI_LIBRARY_CONFIGURATION_ERROR_SETTINGS, err) } else { - this.libraryConfig = libraryConfig + this.libraryConfig = effectiveLibraryConfig setItrSkippingEnabledTagFromLibraryConfig(this, frameworkVersion) } @@ -201,7 +211,7 @@ module.exports = class CiPlugin extends Plugin { onDone({ err, isTestDynamicInstrumentationEnabled: this.config.isTestDynamicInstrumentationEnabled, - libraryConfig, + libraryConfig: effectiveLibraryConfig, repositoryRoot: this.repositoryRoot, requestErrorTags: this._getCurrentRequestErrorTags(), }) @@ -234,19 +244,28 @@ module.exports = class CiPlugin extends Plugin { ) }) - this.addSub(`ci:${this.constructor.id}:session:start`, ({ command, frameworkVersion, rootDir }) => { + this.addSub(`ci:${this.constructor.id}:session:start`, ({ + command, + frameworkVersion, + rootDir, + testFramework, + testFrameworkAdapter, + }) => { + const effectiveTestFramework = testFramework || this.constructor.id const childOf = getTestParentSpan(this.tracer) - const testSessionSpanMetadata = getTestSessionCommonTags(command, frameworkVersion, this.constructor.id) - const testModuleSpanMetadata = getTestModuleCommonTags(command, frameworkVersion, this.constructor.id) + const testSessionSpanMetadata = getTestSessionCommonTags(command, frameworkVersion, effectiveTestFramework) + const testModuleSpanMetadata = getTestModuleCommonTags(command, frameworkVersion, effectiveTestFramework) this.command = command this.frameworkVersion = frameworkVersion + this.testFramework = effectiveTestFramework + this.testFrameworkAdapter = testFrameworkAdapter // only for playwright this.rootDir = rootDir const testSessionName = getTestSessionName( this.config, - DD_MAJOR < 6 ? this.command : FRAMEWORK_TO_TRIMMED_COMMAND[this.constructor.id], + DD_MAJOR < 6 ? this.command : FRAMEWORK_TO_TRIMMED_COMMAND[effectiveTestFramework], this.testEnvironmentMetadata ) @@ -296,7 +315,7 @@ module.exports = class CiPlugin extends Plugin { const testCommand = this.command for (const testSuite of skippedSuites) { const testSuiteMetadata = { - ...getTestSuiteCommonTags(testCommand, frameworkVersion, testSuite, this.constructor.id), + ...getTestSuiteCommonTags(testCommand, frameworkVersion, testSuite, this.testFramework), ...getSessionRequestErrorTags(this.testSessionSpan), ...getSessionItrSkippingEnabledTags(this.testSessionSpan), } @@ -392,10 +411,16 @@ module.exports = class CiPlugin extends Plugin { return onDone({ err: new Error('No modified tests could have been retrieved') }) }) - this.addSub(`ci:${this.constructor.id}:worker-report:trace`, traces => { + this.addSub(`ci:${this.constructor.id}:worker-report:trace`, payload => { + const { traces, [TEST_SUITE_EXECUTION_ID]: testSuiteExecutionId } = typeof payload === 'string' + ? { traces: payload } + : payload const formattedTraces = JSON.parse(traces) for (const trace of formattedTraces) { + if (testSuiteExecutionId) { + this._workerTraceExecutionIds.set(trace, testSuiteExecutionId) + } this._prepareWorkerTrace(trace) this._exportWorkerTraceOrBuffer(trace) } @@ -409,7 +434,7 @@ module.exports = class CiPlugin extends Plugin { } get telemetry () { - const testFramework = this.constructor.id + const testFramework = this.testFramework || this.constructor.id const exporter = this.tracer?._exporter // TODO: only jest worker supported yet const isSupportedWorker = exporter && typeof exporter.exportTelemetry === 'function' @@ -469,9 +494,14 @@ module.exports = class CiPlugin extends Plugin { /** * Returns library capability metadata tags for this test framework. * @param {string} frameworkVersion - The test framework version. + * @param {object} [ctx] - Diagnostic channel context. + * @param {boolean} [ctx.basicReportingOnly] - Whether advanced capabilities must be omitted. * @returns {Record} */ - getLibraryCapabilitiesTags (frameworkVersion) { + getLibraryCapabilitiesTags (frameworkVersion, ctx = {}) { + if (ctx.basicReportingOnly) { + return {} + } return getDefaultLibraryCapabilitiesTags(this.constructor.id, frameworkVersion) } @@ -592,17 +622,34 @@ module.exports = class CiPlugin extends Plugin { // test session, test module and test suite ids. We have to update them in the main process. if (span.name !== 'cucumber.test' && span.name !== 'mocha.test') continue + const testFramework = this.testFramework || this.constructor.id + span.meta[TEST_FRAMEWORK] = testFramework + span.meta[TEST_MODULE] = testFramework + const testType = this.testSessionSpan?.context().getTag(TEST_TYPE) + if (testType) { + span.meta[TEST_TYPE] = testType + } + if (this.frameworkVersion) { + span.meta[TEST_FRAMEWORK_VERSION] = this.frameworkVersion + } + if (this.testFrameworkAdapter) { + span.meta[TEST_FRAMEWORK_ADAPTER] = this.testFrameworkAdapter + } + const testSuite = span.meta[TEST_SUITE] - const testSuiteSpan = this._testSuiteSpansByTestSuite.get(testSuite) - if (!testSuiteSpan) return testSuite + const testSuiteExecutionId = this._workerTraceExecutionIds.get(trace) + const testSuiteKey = getTestSuiteExecutionKey(testSuite, testSuiteExecutionId) + const testSuiteSpan = this._testSuiteSpansByTestSuite.get(testSuiteKey) + if (!testSuiteSpan) return testSuiteKey - const testSuiteTags = getTestSuiteLevelVisibilityTags(testSuiteSpan, this.constructor.id) + const testSuiteTags = getTestSuiteLevelVisibilityTags(testSuiteSpan, testFramework) span.meta = { ...span.meta, ...testSuiteTags, ...getSessionRequestErrorTags(this.testSessionSpan), } } + this._workerTraceExecutionIds.delete(trace) } /** @@ -769,11 +816,14 @@ module.exports = class CiPlugin extends Plugin { testName, testSuite, this.frameworkVersion, - this.constructor.id + this.testFramework || this.constructor.id ), [COMPONENT]: this.constructor.id, ...extraTags, } + if (this.testFrameworkAdapter) { + testTags[TEST_FRAMEWORK_ADAPTER] = this.testFrameworkAdapter + } const codeOwners = this.getCodeOwners(testTags) if (codeOwners) { @@ -790,7 +840,7 @@ module.exports = class CiPlugin extends Plugin { [TEST_SUITE_ID]: testSuiteSpan.context().toSpanId(), [TEST_SESSION_ID]: testSuiteSpan.context().toTraceId(), [TEST_COMMAND]: testSuiteSpan.context().getTag(TEST_COMMAND), - [TEST_MODULE]: this.constructor.id, + [TEST_MODULE]: this.testFramework || this.constructor.id, ...getSessionRequestErrorTags(this.testSessionSpan), } if (testSuiteSpan.context()._parentId) { diff --git a/packages/dd-trace/src/plugins/index.js b/packages/dd-trace/src/plugins/index.js index 591df8171d..7d6fa6c12e 100644 --- a/packages/dd-trace/src/plugins/index.js +++ b/packages/dd-trace/src/plugins/index.js @@ -36,6 +36,7 @@ const plugins = { get '@smithy/core' () { return require('../../../datadog-plugin-aws-sdk/src') }, get '@smithy/smithy-client' () { return require('../../../datadog-plugin-aws-sdk/src') }, get '@vitest/runner' () { return require('../../../datadog-plugin-vitest/src') }, + get '@wdio/local-runner' () { return require('../../../datadog-plugin-mocha/src') }, get '@langchain/langgraph' () { return require('../../../datadog-plugin-langgraph/src') }, get '@openai/agents' () { return require('../../../datadog-plugin-openai-agents/src') }, get aerospike () { return require('../../../datadog-plugin-aerospike/src') }, diff --git a/packages/dd-trace/src/plugins/plugin.js b/packages/dd-trace/src/plugins/plugin.js index 6a117dde41..d5e75fb67d 100644 --- a/packages/dd-trace/src/plugins/plugin.js +++ b/packages/dd-trace/src/plugins/plugin.js @@ -103,7 +103,7 @@ module.exports = class Plugin { * @returns {void} */ enter (span, store) { - store = store || legacyStorage.getStore() + store ||= legacyStorage.getStore() legacyStorage.enterWith({ ...store, span }) } diff --git a/packages/dd-trace/src/plugins/tracing.js b/packages/dd-trace/src/plugins/tracing.js index 0f4e2dd349..aee18b069a 100644 --- a/packages/dd-trace/src/plugins/tracing.js +++ b/packages/dd-trace/src/plugins/tracing.js @@ -265,6 +265,7 @@ class TracingPlugin extends Plugin { analyticsSampler.sample(span, config.measured) // TODO: Remove this after migration to TracingChannel is done. + // eslint-disable-next-line unicorn/no-unnecessary-boolean-comparison -- Context objects use the other branch. if (enterOrCtx === true) { legacyStorage.enterWith({ ...store, span }) } else if (enterOrCtx) { diff --git a/packages/dd-trace/src/plugins/util/ci.js b/packages/dd-trace/src/plugins/util/ci.js index ef7cf9f377..1160ecae3d 100644 --- a/packages/dd-trace/src/plugins/util/ci.js +++ b/packages/dd-trace/src/plugins/util/ci.js @@ -18,6 +18,7 @@ const { GIT_REPOSITORY_URL, CI_PIPELINE_ID, CI_PIPELINE_NAME, + CI_PIPELINE_DISPLAY_NAME, CI_PIPELINE_NUMBER, CI_PIPELINE_URL, CI_PROVIDER_NAME, @@ -316,10 +317,8 @@ module.exports = { } if (NODE_LABELS) { - let nodeLabels try { - nodeLabels = JSON.stringify(NODE_LABELS.split(' ')) - tags[CI_NODE_LABELS] = nodeLabels + tags[CI_NODE_LABELS] = JSON.stringify(NODE_LABELS.split(' ')) } catch { // ignore errors } @@ -699,6 +698,7 @@ module.exports = { BUILDKITE_TAG, BUILDKITE_BUILD_ID, BUILDKITE_PIPELINE_SLUG, + BUILDKITE_PIPELINE_NAME, BUILDKITE_BUILD_NUMBER, BUILDKITE_BUILD_URL, BUILDKITE_JOB_ID, @@ -722,6 +722,7 @@ module.exports = { [CI_PROVIDER_NAME]: 'buildkite', [CI_PIPELINE_ID]: BUILDKITE_BUILD_ID, [CI_PIPELINE_NAME]: BUILDKITE_PIPELINE_SLUG, + [CI_PIPELINE_DISPLAY_NAME]: BUILDKITE_PIPELINE_NAME, [CI_PIPELINE_NUMBER]: BUILDKITE_BUILD_NUMBER, [CI_PIPELINE_URL]: BUILDKITE_BUILD_URL, [CI_JOB_URL]: `${BUILDKITE_BUILD_URL}#${BUILDKITE_JOB_ID}`, diff --git a/packages/dd-trace/src/plugins/util/http-otel-semantics.js b/packages/dd-trace/src/plugins/util/http-otel-semantics.js index e139affd16..690ba3b607 100644 --- a/packages/dd-trace/src/plugins/util/http-otel-semantics.js +++ b/packages/dd-trace/src/plugins/util/http-otel-semantics.js @@ -258,7 +258,7 @@ function applyHttpOtelSemantics (formattedSpan) { // type): server spans are errors on 5xx only (4xx MUST be left unset per the // spec); client spans on any status >= 400. if (status !== undefined && newMeta[ERROR_TYPE] === undefined) { - const isError = kind === 'server' ? statusCode >= 500 : statusCode >= 400 + const isError = statusCode >= (kind === 'server' ? 500 : 400) if (isError) { newMeta[ERROR_TYPE] = status formattedSpan.error = 1 diff --git a/packages/dd-trace/src/plugins/util/tags.js b/packages/dd-trace/src/plugins/util/tags.js index f0ea40f7bd..f15128bcaf 100644 --- a/packages/dd-trace/src/plugins/util/tags.js +++ b/packages/dd-trace/src/plugins/util/tags.js @@ -26,6 +26,7 @@ const GIT_PULL_REQUEST_BASE_BRANCH = 'git.pull_request.base_branch' const CI_PIPELINE_ID = 'ci.pipeline.id' const CI_PIPELINE_NAME = 'ci.pipeline.name' +const CI_PIPELINE_DISPLAY_NAME = 'ci.pipeline.display_name' const CI_PIPELINE_NUMBER = 'ci.pipeline.number' const CI_PIPELINE_URL = 'ci.pipeline.url' const CI_PROVIDER_NAME = 'ci.provider.name' @@ -66,6 +67,7 @@ module.exports = { GIT_PULL_REQUEST_BASE_BRANCH, CI_PIPELINE_ID, CI_PIPELINE_NAME, + CI_PIPELINE_DISPLAY_NAME, CI_PIPELINE_NUMBER, CI_PIPELINE_URL, CI_PROVIDER_NAME, diff --git a/packages/dd-trace/src/plugins/util/test.js b/packages/dd-trace/src/plugins/util/test.js index 3345540218..38adcdbc7d 100644 --- a/packages/dd-trace/src/plugins/util/test.js +++ b/packages/dd-trace/src/plugins/util/test.js @@ -45,6 +45,7 @@ const { CI_WORKSPACE_PATH, CI_PIPELINE_ID, CI_PIPELINE_NAME, + CI_PIPELINE_DISPLAY_NAME, CI_PIPELINE_NUMBER, CI_PIPELINE_URL, CI_JOB_NAME, @@ -89,10 +90,12 @@ const { const TEST_SESSION_NAME = 'test_session.name' const TEST_FRAMEWORK = 'test.framework' +const TEST_FRAMEWORK_ADAPTER = 'test.framework_adapter' const TEST_FRAMEWORK_VERSION = 'test.framework_version' const TEST_TYPE = 'test.type' const TEST_NAME = 'test.name' const TEST_SUITE = 'test.suite' +const TEST_SUITE_EXECUTION_ID = '_dd.test_suite_execution_id' const TEST_STATUS = 'test.status' const TEST_FINAL_STATUS = 'test.final_status' const TEST_PARAMETERS = 'test.parameters' @@ -244,6 +247,7 @@ const TEST_LEVELS_METADATA_TAGS = [ CI_NODE_NAME, CI_PIPELINE_ID, CI_PIPELINE_NAME, + CI_PIPELINE_DISPLAY_NAME, CI_PIPELINE_NUMBER, CI_PIPELINE_URL, CI_PROVIDER_NAME, @@ -422,10 +426,22 @@ function addTestOptimizationRequest (requestPromises, responseNames, responseNam } } +/** + * Builds the internal key used to correlate a worker test with one suite execution. + * + * @param {string} testSuite + * @param {string|undefined} testSuiteExecutionId + * @returns {string} + */ +function getTestSuiteExecutionKey (testSuite, testSuiteExecutionId) { + return testSuiteExecutionId ? `${testSuite}\0${testSuiteExecutionId}` : testSuite +} + module.exports = { TEST_CODE_OWNERS, TEST_SESSION_NAME, TEST_FRAMEWORK, + TEST_FRAMEWORK_ADAPTER, TEST_FRAMEWORK_VERSION, JEST_TEST_RUNNER, JEST_DISPLAY_NAME, @@ -435,6 +451,8 @@ module.exports = { TEST_TYPE, TEST_NAME, TEST_SUITE, + TEST_SUITE_EXECUTION_ID, + getTestSuiteExecutionKey, TEST_STATUS, TEST_FINAL_STATUS, TEST_PARAMETERS, @@ -800,7 +818,7 @@ function getTestLevelsMetadataTags (testEnvironmentMetadata) { } function getTestTypeFromFramework (testFramework) { - if (testFramework === 'playwright' || testFramework === 'cypress') { + if (testFramework === 'playwright' || testFramework === 'cypress' || testFramework === 'webdriverio') { return 'browser' } return 'test' @@ -871,6 +889,7 @@ function getTestCommonTags (name, suite, version, testFramework) { return { [SPAN_TYPE]: 'test', [TEST_TYPE]: getTestTypeFromFramework(testFramework), + [TEST_FRAMEWORK]: testFramework, [SAMPLING_RULE_DECISION]: 1, [SAMPLING_PRIORITY]: AUTO_KEEP, [TEST_NAME]: name, @@ -941,8 +960,7 @@ function getCodeOwnersFileEntries (rootDir) { const lines = codeOwnersContent.split('\n') for (const line of lines) { - const [content] = line.split('#') - const trimmed = content.trim() + const trimmed = getSegment(line, '#', 0).trim() if (trimmed === '') continue const [pattern, ...owners] = trimmed.split(/\s+/) entries.push(setCodeOwnersPatternRegex({ pattern, owners })) @@ -988,9 +1006,7 @@ function codeOwnersPatternToRegexSource (pattern) { if (character === '\\') { const escapedCharacter = pattern[i + 1] - source += escapedCharacter === undefined - ? escapeRegexCharacter(character) - : escapeRegexCharacter(escapedCharacter) + source += escapeRegexCharacter(escapedCharacter ?? character) i++ } else if (character === '*') { if (pattern[i + 1] === '*') { @@ -1101,6 +1117,7 @@ function getCodeOwnersForFilename (filename, entries) { function getTestLevelCommonTags (command, testFrameworkVersion, testFramework) { return { + [TEST_FRAMEWORK]: testFramework, [TEST_FRAMEWORK_VERSION]: testFrameworkVersion, [LIBRARY_VERSION]: ddTraceVersion, [TEST_TYPE]: getTestTypeFromFramework(testFramework), @@ -1184,7 +1201,7 @@ function getCoveredFilenamesFromCoverage (coverage) { } function getCoverageMap (coverage) { - if (coverage?.files && coverage?.fileCoverageFor) { + if (coverage?.files && coverage.fileCoverageFor) { return coverage } return istanbul.createCoverageMap(coverage) @@ -1917,8 +1934,8 @@ function recordAttemptToFixExecution (attemptToFixExecutions, execution) { } result.executions++ - result.isDisabled = result.isDisabled || !!isDisabled - result.isQuarantined = result.isQuarantined || !!isQuarantined + result.isDisabled ||= !!isDisabled + result.isQuarantined ||= !!isQuarantined if (status === 'fail') { result.failedCount++ diff --git a/packages/dd-trace/src/priority_sampler.js b/packages/dd-trace/src/priority_sampler.js index c883158e77..c46e166e33 100644 --- a/packages/dd-trace/src/priority_sampler.js +++ b/packages/dd-trace/src/priority_sampler.js @@ -140,7 +140,7 @@ class PrioritySampler { samplers[key] = new Sampler(rates[key]) } - samplers[DEFAULT_KEY] = samplers[DEFAULT_KEY] || defaultSampler + samplers[DEFAULT_KEY] ||= defaultSampler this._samplers = samplers @@ -257,8 +257,11 @@ class PrioritySampler { context._trace[SAMPLING_RULE_DECISION] = rule.sampleRate context._trace.tags[SAMPLING_KNUTH_RATE] = formatKnuthRate(rule.sampleRate) context._sampling.mechanism = SAMPLING_MECHANISM_RULE - if (rule.provenance === 'customer') context._sampling.mechanism = SAMPLING_MECHANISM_REMOTE_USER - if (rule.provenance === 'dynamic') context._sampling.mechanism = SAMPLING_MECHANISM_REMOTE_DYNAMIC + if (rule.provenance === 'customer') { + context._sampling.mechanism = SAMPLING_MECHANISM_REMOTE_USER + } else if (rule.provenance === 'dynamic') { + context._sampling.mechanism = SAMPLING_MECHANISM_REMOTE_DYNAMIC + } return rule.sample(context) && this._isSampledByRateLimit(context) ? USER_KEEP diff --git a/packages/dd-trace/src/profiling/profilers/poisson.js b/packages/dd-trace/src/profiling/profilers/poisson.js index aad121b7ec..2adf3a1875 100644 --- a/packages/dd-trace/src/profiling/profilers/poisson.js +++ b/packages/dd-trace/src/profiling/profilers/poisson.js @@ -6,7 +6,7 @@ class PoissonProcessSamplingFilter { #samplingInterval #resetInterval #now - #lastNow = Number.NEGATIVE_INFINITY + #lastNow = -Infinity #samplingInstantCount = 0 constructor ({ samplingInterval, now, resetInterval }) { diff --git a/packages/dd-trace/src/profiling/profilers/shared.js b/packages/dd-trace/src/profiling/profilers/shared.js index 9fbe37e066..916194099f 100644 --- a/packages/dd-trace/src/profiling/profilers/shared.js +++ b/packages/dd-trace/src/profiling/profilers/shared.js @@ -17,8 +17,8 @@ function getThreadLabels () { const nativeThreadId = pprof.getNativeThreadId() return { [THREAD_NAME_LABEL]: eventLoopThreadName, - [THREAD_ID_LABEL]: `${threadId}`, - [OS_THREAD_ID_LABEL]: `${nativeThreadId}`, + [THREAD_ID_LABEL]: threadId.toString(), + [OS_THREAD_ID_LABEL]: nativeThreadId.toString(), } } diff --git a/packages/dd-trace/src/startup-log.js b/packages/dd-trace/src/startup-log.js index 7e74e3d508..52fafff7f5 100644 --- a/packages/dd-trace/src/startup-log.js +++ b/packages/dd-trace/src/startup-log.js @@ -108,6 +108,9 @@ function configInfo () { profiling_enabled: profilingEnabled === 'true' || profilingEnabled === 'auto', appsec_enabled: config.appsec.enabled, data_streams_enabled: !!config.dsmEnabled, + otlp_traces_export_enabled: config.OTEL_TRACES_EXPORTER === 'otlp' && !config.isCiVisibility, + otlp_metrics_export_enabled: !!config.DD_METRICS_OTEL_ENABLED, + otlp_logs_export_enabled: !!config.DD_LOGS_OTEL_ENABLED, } } diff --git a/packages/dd-trace/src/util.js b/packages/dd-trace/src/util.js index 582bfafc06..cc7fbe4198 100644 --- a/packages/dd-trace/src/util.js +++ b/packages/dd-trace/src/util.js @@ -102,6 +102,21 @@ function getSegment (string, separator, index, fallback) { return end === -1 ? string.slice(start) : string.slice(start, end) } +/** + * Return the path portion of a request target, dropping any query string and fragment. + * The two scans avoid the array and substring a `split(/[?#]/)` allocates per call. + * + * @param {string} target + */ +function stripQueryAndFragment (target) { + let cut = target.indexOf('?') + const fragment = target.indexOf('#') + if (cut === -1 || (fragment !== -1 && fragment < cut)) { + cut = fragment + } + return cut === -1 ? target : target.slice(0, cut) +} + function calculateDDBasePath (dirname) { const dirSteps = dirname.split(path.sep) const packagesIndex = dirSteps.lastIndexOf('packages') @@ -136,6 +151,7 @@ module.exports = { isError, globMatch, getSegment, + stripQueryAndFragment, ddBasePath: globalThis.__DD_ESBUILD_BASEPATH || calculateDDBasePath(__dirname), normalizePluginEnvName, formatKnuthRate, diff --git a/packages/dd-trace/test/appsec/rasp/ssrf.express.plugin.spec.js b/packages/dd-trace/test/appsec/rasp/ssrf.express.plugin.spec.js index 1fb382e5c9..0f6adbf1e6 100644 --- a/packages/dd-trace/test/appsec/rasp/ssrf.express.plugin.spec.js +++ b/packages/dd-trace/test/appsec/rasp/ssrf.express.plugin.spec.js @@ -15,7 +15,7 @@ const { checkRaspExecutedAndNotThreat, checkRaspExecutedAndHasThreat } = require function noop () {} -const UNRESOLVABLE_HOST = 'not-a-threat.invalid' +const NON_ROUTABLE_HOST = '192.0.2.1' describe('RASP - ssrf', () => { withVersions('express', 'express', expressVersion => { @@ -81,17 +81,18 @@ describe('RASP - ssrf', () => { const module = require(protocol) app = (req, res) => { - const clientRequest = module.get(`${protocol}://${req.query.host}`, function (incomingResponse) { - incomingResponse.resume() + const clientRequest = module.get(`${protocol}://${req.query.host}`) + + clientRequest.on('error', noop) + setImmediate(() => { + clientRequest.destroy() res.end('end') }) - - clientRequest.on('error', () => res.end('end')) } await Promise.all([ checkRaspExecutedAndNotThreat(agent), - axios.get(`/?host=${UNRESOLVABLE_HOST}`), + axios.get(`/?host=${NON_ROUTABLE_HOST}`), ]) }) @@ -148,13 +149,18 @@ describe('RASP - ssrf', () => { it('Should not detect threat', async () => { app = (req, res) => { - axiosToTest.get(`https://${req.query.host}`, { proxy: false }) + const abortController = new AbortController() + axiosToTest.get(`https://${req.query.host}`, { proxy: false, signal: abortController.signal }) .catch(noop) // swallow network error - .then(() => res.end('end')) + + setImmediate(() => { + abortController.abort() + res.end('end') + }) } await Promise.all([ - axios.get(`/?host=${UNRESOLVABLE_HOST}`), + axios.get(`/?host=${NON_ROUTABLE_HOST}`), checkRaspExecutedAndNotThreat(agent), ]) }) @@ -202,13 +208,17 @@ describe('RASP - ssrf', () => { it('Should not detect threat', async () => { app = (req, res) => { - requestToTest.get(`https://${req.query.host}`, { proxy: false }) - .on('response', () => res.end('end')) - .on('error', () => res.end('end')) + const clientRequest = requestToTest.get(`https://${req.query.host}`, { proxy: false }) + clientRequest.on('error', noop) + + setImmediate(() => { + clientRequest.abort() + res.end('end') + }) } await Promise.all([ - axios.get(`/?host=${UNRESOLVABLE_HOST}`), + axios.get(`/?host=${NON_ROUTABLE_HOST}`), checkRaspExecutedAndNotThreat(agent), ]) }) diff --git a/packages/dd-trace/test/ci-visibility/ci-plugin.spec.js b/packages/dd-trace/test/ci-visibility/ci-plugin.spec.js index 9aa8d186ca..299dc32752 100644 --- a/packages/dd-trace/test/ci-visibility/ci-plugin.spec.js +++ b/packages/dd-trace/test/ci-visibility/ci-plugin.spec.js @@ -157,6 +157,56 @@ describe('CiPlugin', () => { } }) + it('disables advanced features for basic-reporting library configuration requests', () => { + const libraryConfig = { + isEarlyFlakeDetectionEnabled: true, + isFlakyTestRetriesEnabled: true, + isSuitesSkippingEnabled: true, + isTestManagementEnabled: true, + } + const getLibraryConfiguration = sinon.stub().callsArgWith(1, null, libraryConfig) + const addMetadataTags = sinon.stub() + const onDone = sinon.stub() + const plugin = createPlugin('jest_worker') + plugin.tracer._exporter = { + addMetadataTags, + getLibraryConfiguration, + } + plugin.configure({ + enabled: true, + experimental: { + exporter: 'jest_worker', + }, + }) + + dc.channel('ci:vitest:library-configuration').publish({ + basicReportingOnly: true, + frameworkVersion: '1.0.0', + onDone, + }) + plugin.configure(false) + + assert.deepStrictEqual(plugin.libraryConfig, {}) + assert.deepStrictEqual(plugin.getLibraryCapabilitiesTags('1.0.0', { basicReportingOnly: true }), {}) + assert.deepStrictEqual(onDone.firstCall.args[0].libraryConfig, {}) + assert.deepStrictEqual(addMetadataTags.firstCall.args[0], { test: {} }) + sinon.assert.calledOnce(getLibraryConfiguration) + sinon.assert.calledOnce(onDone) + }) + + it('tags telemetry with the effective test framework', () => { + const exportTelemetry = sinon.stub() + const plugin = createPlugin('mocha') + plugin.tracer._exporter.exportTelemetry = exportTelemetry + + plugin.telemetry.ciVisEvent('event_created', 'session') + plugin.testFramework = 'webdriverio' + plugin.telemetry.ciVisEvent('event_finished', 'session') + + assert.strictEqual(exportTelemetry.firstCall.args[0].testFramework, 'vitest') + assert.strictEqual(exportTelemetry.secondCall.args[0].testFramework, 'webdriverio') + }) + it('starts the DI breakpoint-hit timeout when waiting, not when preparing', async () => { const plugin = createPlugin('jest_worker') const waitForDiOperation = sinon.stub(plugin, 'waitForDiOperation').resolves() diff --git a/packages/dd-trace/test/ci-visibility/exporters/test-worker/exporter.spec.js b/packages/dd-trace/test/ci-visibility/exporters/test-worker/exporter.spec.js index 1acc89806e..9d1ee8acf2 100644 --- a/packages/dd-trace/test/ci-visibility/exporters/test-worker/exporter.spec.js +++ b/packages/dd-trace/test/ci-visibility/exporters/test-worker/exporter.spec.js @@ -190,6 +190,27 @@ describe('CI Visibility Test Worker Exporter', () => { }) }) + context('when writing from a WebdriverIO worker', () => { + it('wraps traces in a filtered WebdriverIO worker event', () => { + const trace = [{ type: 'test' }] + const WebdriverioWriter = proxyquire('../../../../src/ci-visibility/exporters/test-worker/writer', { + '../../../config/helper': { + getEnvironmentVariable: name => + name === '_DD_TEST_OPTIMIZATION_WEBDRIVERIO_WORKER' ? 'true' : undefined, + }, + }) + const writer = new WebdriverioWriter(MOCHA_WORKER_TRACE_PAYLOAD_CODE) + writer.append(trace) + writer.flush() + + sinon.assert.calledWith(send, { + origin: 'datadog', + name: 'workerEvent', + args: [MOCHA_WORKER_TRACE_PAYLOAD_CODE, JSON.stringify([trace])], + }) + }) + }) + context('when the process is a playwright worker', () => { beforeEach(() => { process.env.DD_PLAYWRIGHT_WORKER = '1' diff --git a/packages/dd-trace/test/datastreams/encoding.spec.js b/packages/dd-trace/test/datastreams/encoding.spec.js index 19fba5176f..408c8853c2 100644 --- a/packages/dd-trace/test/datastreams/encoding.spec.js +++ b/packages/dd-trace/test/datastreams/encoding.spec.js @@ -37,6 +37,41 @@ describe('encoding', () => { assert.strictEqual(bytes2.length, 0) }) + it('encoding then decoding should be a no op at the largest encodable magnitude', () => { + const n = Math.floor(Number.MAX_SAFE_INTEGER / 2) + + const encoded = encodeVarint(n) + assert.deepStrictEqual([...encoded], [254, 255, 255, 255, 255, 255, 255, 15]) + const [decoded, bytes] = decodeVarint(encoded) + assert.strictEqual(decoded, n) + assert.strictEqual(bytes.length, 0) + + const encodedNegative = encodeVarint(-n) + assert.deepStrictEqual([...encodedNegative], [255, 255, 255, 255, 255, 255, 255, 15]) + const [decodedNegative, negativeBytes] = decodeVarint(encodedNegative) + assert.strictEqual(decodedNegative, -n) + assert.strictEqual(negativeBytes.length, 0) + }) + + it('encoding then decoding should be a no op when a group shifts down to 0x80', () => { + // The zig-zag value shifts down to exactly 0x80 for `2 ** exponent` and the + // `2 ** (exponent - 7)` values above it. Math.floor(Number.MAX_SAFE_INTEGER / 2) + // spans seven 7-bit groups, so there are seven such bands. + for (let exponent = 6; exponent <= 48; exponent += 7) { + const base = 2 ** exponent + const bandTop = base + 2 ** Math.max(exponent - 7, 0) - 1 + + for (const n of new Set([base, bandTop, -base, -bandTop])) { + const encoded = encodeVarint(n) + const [decoded, bytes] = decodeVarint(encoded) + assert.strictEqual(decoded, n) + assert.strictEqual(bytes.length, 0) + } + } + + assert.deepStrictEqual([...encodeVarint(2 ** 6)], [128, 1]) + }) + it('encoding a number bigger than Max safe int fails.', () => { const n = Number.MAX_SAFE_INTEGER + 10 const encoded = encodeVarint(n) diff --git a/packages/dd-trace/test/exporters/electron/exporter.spec.js b/packages/dd-trace/test/exporters/electron/exporter.spec.js index 185ba2449a..f63b5b3ba1 100644 --- a/packages/dd-trace/test/exporters/electron/exporter.spec.js +++ b/packages/dd-trace/test/exporters/electron/exporter.spec.js @@ -80,10 +80,11 @@ describe('ElectronExporter', () => { sinon.assert.notCalled(traceChannel.publish) }) - it('should not publish when there are no subscribers', () => { + it('should not buffer when there are no subscribers', () => { traceChannel.hasSubscribers = false exporter.export([span]) + traceChannel.hasSubscribers = true exporter.flush() sinon.assert.notCalled(traceChannel.publish) diff --git a/packages/dd-trace/test/llmobs/experiments/client.spec.js b/packages/dd-trace/test/llmobs/experiments/client.spec.js index eb0d7561b3..721ecf8785 100644 --- a/packages/dd-trace/test/llmobs/experiments/client.spec.js +++ b/packages/dd-trace/test/llmobs/experiments/client.spec.js @@ -7,11 +7,20 @@ const sinon = require('sinon') const { ExperimentsClient, apiHost, appHost } = require('../../../src/llmobs/experiments/client') describe('LLMObs Experiments control-plane client', () => { + let fetchError + let fetchResponse + let fetchStub let originalFetch beforeEach(() => { originalFetch = global.fetch - global.fetch = sinon.stub() + fetchError = undefined + fetchResponse = undefined + fetchStub = sinon.stub().callsFake(async () => { + if (fetchError) throw fetchError + return fetchResponse + }) + global.fetch = fetchStub }) afterEach(() => { @@ -19,12 +28,16 @@ describe('LLMObs Experiments control-plane client', () => { sinon.restore() }) + const rejectWith = (error) => { + fetchError = error + } + const resolveWith = (status, body) => { - global.fetch.resolves({ + fetchResponse = { ok: status >= 200 && status < 300, status, text: sinon.stub().resolves(JSON.stringify(body)), - }) + } } it('resolves the control-plane host from the site', () => { @@ -33,12 +46,13 @@ describe('LLMObs Experiments control-plane client', () => { assert.equal(apiHost('datad0g.com'), 'api.datad0g.com') }) - it('resolves the web-app host (app. for single-level, regional as-is)', () => { + it('resolves the web-app host (app. for single-level, staging override, regional as-is)', () => { assert.equal(appHost('datadoghq.com'), 'app.datadoghq.com') + assert.equal(appHost('datad0g.com'), 'dd.datad0g.com') assert.equal(appHost('us3.datadoghq.com'), 'us3.datadoghq.com') - const client = new ExperimentsClient({ apiKey: 'k', appKey: 'a', site: 'datadoghq.com' }) - assert.equal(client.appBase, 'https://app.datadoghq.com') - assert.equal(client.site, 'datadoghq.com') + const client = new ExperimentsClient({ apiKey: 'k', appKey: 'a', site: 'datad0g.com' }) + assert.equal(client.appBase, 'https://dd.datad0g.com') + assert.equal(client.site, 'datad0g.com') }) it('ensureProjectId resolves the configured project name', async () => { @@ -46,7 +60,7 @@ describe('LLMObs Experiments control-plane client', () => { const client = new ExperimentsClient({ apiKey: 'k', appKey: 'a', site: 'datadoghq.com', projectName: 'cfg-app' }) const id = await client.ensureProjectId() assert.equal(id, 'proj-9') - assert.deepEqual(JSON.parse(global.fetch.firstCall.args[1].body), { + assert.deepEqual(JSON.parse(fetchStub.firstCall.args[1].body), { data: { type: 'projects', attributes: { name: 'cfg-app' } }, }) }) @@ -64,9 +78,9 @@ describe('LLMObs Experiments control-plane client', () => { const id = await client.getOrCreateProject('my-project') assert.equal(id, 'proj-123') - sinon.assert.calledOnce(global.fetch) + sinon.assert.calledOnce(fetchStub) - const [url, opts] = global.fetch.firstCall.args + const [url, opts] = fetchStub.firstCall.args assert.equal(url, 'https://api.datadoghq.com/api/v2/llm-obs/v1/projects') assert.equal(opts.method, 'POST') assert.equal(opts.headers['DD-API-KEY'], 'key') @@ -86,7 +100,7 @@ describe('LLMObs Experiments control-plane client', () => { assert.equal(first, 'proj-123') assert.equal(second, 'proj-123') - sinon.assert.calledOnce(global.fetch) + sinon.assert.calledOnce(fetchStub) }) it('routes regional sites to api..', async () => { @@ -95,7 +109,7 @@ describe('LLMObs Experiments control-plane client', () => { const client = new ExperimentsClient({ apiKey: 'k', appKey: 'a', site: 'us3.datadoghq.com' }) await client.getOrCreateProject('p') - assert.equal(global.fetch.firstCall.args[0], 'https://api.us3.datadoghq.com/api/v2/llm-obs/v1/projects') + assert.equal(fetchStub.firstCall.args[0], 'https://api.us3.datadoghq.com/api/v2/llm-obs/v1/projects') }) it('throws a clear error on a non-2xx response', async () => { @@ -109,7 +123,7 @@ describe('LLMObs Experiments control-plane client', () => { }) it('throws a clear error when the request itself fails', async () => { - global.fetch.rejects(new Error('network down')) + rejectWith(new Error('network down')) const client = new ExperimentsClient({ apiKey: 'k', appKey: 'a', site: 'datadoghq.com' }) await assert.rejects( diff --git a/packages/dd-trace/test/llmobs/experiments/experiment.spec.js b/packages/dd-trace/test/llmobs/experiments/experiment.spec.js index 5ce9adb2d3..2a7fd6b82a 100644 --- a/packages/dd-trace/test/llmobs/experiments/experiment.spec.js +++ b/packages/dd-trace/test/llmobs/experiments/experiment.spec.js @@ -1,82 +1,70 @@ 'use strict' const assert = require('node:assert/strict') -const { afterEach, beforeEach, describe, it } = require('mocha') -const sinon = require('sinon') -const { ExperimentsClient } = require('../../../src/llmobs/experiments/client') -const { Dataset } = require('../../../src/llmobs/experiments/dataset') -const { Experiment } = require('../../../src/llmobs/experiments/experiment') +const { describe, it } = require('mocha') -// Routes the control-plane + events calls a run makes, recording each request. -function installFetch (calls, overrides = {}) { - const route = (method, path, body) => { - if (method === 'POST' && path === '/api/v2/llm-obs/v1/projects') return { data: { id: 'proj' } } - if (method === 'POST' && path.endsWith('/proj/datasets/ds/records')) { - return { records: body.data.attributes.records.map((_, i) => ({ id: `rec-${i}` })) } - } - if (method === 'POST' && path.endsWith('/proj/datasets')) return { data: { id: 'ds' } } - if (method === 'POST' && path.endsWith('/experiments/exp/events')) return {} - if (method === 'PATCH' && path.endsWith('/experiments/exp')) return {} - if (method === 'POST' && path.endsWith('/experiments')) return { data: { id: 'exp' } } - return {} - } +const { API_BASE_PATH, ExperimentsClient } = require('../../../src/llmobs/experiments/client') +const { Dataset, DatasetRecord } = require('../../../src/llmobs/experiments/dataset') +const { Experiment } = require('../../../src/llmobs/experiments/experiment') - global.fetch = sinon.stub().callsFake(async (url, opts) => { - const u = new URL(url) - const body = opts.body ? JSON.parse(opts.body) : undefined - calls.push({ method: opts.method, host: u.host, path: u.pathname, body }) - const payload = (overrides[`${opts.method} ${u.pathname}`]) ?? route(opts.method, u.pathname, body) - return { ok: true, status: 200, text: sinon.stub().resolves(JSON.stringify(payload)) } - }) +function defaultAppendRecordAttributes (_record, index) { + return { id: `rec-${index}`, valid_from_version: 2 } } -// Like installFetch, but returns a non-2xx for the given `METHOD /pathname`. -function installFetchFailing (calls, failKey) { - installFetch(calls) - const stub = global.fetch - global.fetch = sinon.stub().callsFake(async (url, opts) => { - const u = new URL(url) - if (`${opts.method} ${u.pathname}` === failKey) { - calls.push({ method: opts.method, path: u.pathname, failed: true }) - return { ok: false, status: 500, text: sinon.stub().resolves('boom') } - } - return stub(url, opts) - }) +function stubClient ({ appendRecordAttributes = defaultAppendRecordAttributes, createDatasetError } = {}) { + const requests = [] + return { + appBase: 'https://app.datadoghq.com', + requests, + ensureProjectId: async () => 'proj', + request: async (method, requestPath, body) => { + requests.push({ method, path: requestPath, body }) + if (method === 'POST' && requestPath === `${API_BASE_PATH}/proj/datasets`) { + if (createDatasetError) throw createDatasetError + return { data: { id: 'ds', attributes: { current_version: 1 } } } + } + if (method === 'POST' && requestPath === `${API_BASE_PATH}/proj/datasets/ds/records`) { + return { + data: body.data.attributes.records.map((record, index) => ({ + id: record.id ?? `rec-${index}`, + attributes: appendRecordAttributes(record, index), + })), + } + } + if (method === 'POST' && requestPath === `${API_BASE_PATH}/experiments`) { + return { data: { id: 'exp' } } + } + if (method === 'POST' && requestPath === `${API_BASE_PATH}/experiments/exp/events`) return {} + if (method === 'PATCH' && requestPath === `${API_BASE_PATH}/experiments/exp`) return {} + throw new Error(`Unexpected request ${method} ${requestPath}`) + }, + } } describe('LLMObs Experiments — dataset + experiment run', () => { - let originalFetch - let calls - let client - - beforeEach(() => { - originalFetch = global.fetch - calls = [] - installFetch(calls) - client = new ExperimentsClient({ apiKey: 'k', appKey: 'a', site: 'datadoghq.com', projectName: 'my-app' }) - }) - - afterEach(() => { - global.fetch = originalFetch - sinon.restore() + const client = () => new ExperimentsClient({ + apiKey: 'k', + appKey: 'a', + site: 'datadoghq.com', + projectName: 'my-app', }) - const eventsBody = () => calls.find(c => c.path.endsWith('/experiments/exp/events')).body - - it('runs an experiment and returns rows, id and dashboard url', async () => { - const dataset = new Dataset(client, 'demo', 'desc') + it('runs an experiment and returns rows, ids and dashboard URLs', async () => { + const c = stubClient() + const dataset = new Dataset(c, 'demo', 'desc') .addRecord({ q: 'apple' }, 'true', { row: 0 }) .addRecord({ q: 'car' }, 'false', { row: 1 }) - const result = await new Experiment(client, { + const result = await new Experiment(c, { name: 'exp-demo', + description: 'desc exp', dataset, task: (input) => ({ answer: input.q.toUpperCase() }), evaluators: { - nonempty: (_i, o) => o.answer.length > 0, - len: (_i, o) => o.answer.length, - label: (_i, o) => (o.answer === 'APPLE' ? 'match' : 'miss'), + nonempty: (_input, output) => output.answer.length > 0, + len: (_input, output) => output.answer.length, + label: (_input, output) => (output.answer === 'APPLE' ? 'match' : 'miss'), }, config: { temperature: 0 }, tags: { env: 'test' }, @@ -89,182 +77,45 @@ describe('LLMObs Experiments — dataset + experiment run', () => { assert.equal(result.rows[0].evaluations.nonempty, true) assert.equal(result.rows[0].evaluations.len, 5) assert.equal(result.rows[0].evaluations.label, 'match') + assert.equal(dataset.id(), 'ds') + assert.equal(dataset.version(), 2) + assert.deepEqual(dataset.recordIds(), ['rec-0', 'rec-1']) assert.equal(dataset.url(), 'https://app.datadoghq.com/llm/datasets/ds') }) - it('sends records with type "datasets" and only new records on re-push', async () => { - const dataset = new Dataset(client, 'demo').addRecord('a') - await dataset.push() - dataset.addRecord('b') - await dataset.push() - - const recordPosts = calls.filter(c => c.method === 'POST' && c.path.endsWith('/proj/datasets/ds/records')) - assert.equal(recordPosts.length, 2) - assert.equal(recordPosts[0].body.data.type, 'datasets') - assert.deepEqual(recordPosts[1].body.data.attributes.records.map(r => r.input), ['b']) - }) - - it('does not drop records added while a push is in flight', async () => { - const dataset = new Dataset(client, 'demo').addRecord('a') - - let resolveRecordsFetch - const routingFetch = global.fetch - global.fetch = sinon.stub().callsFake((url, opts) => { - const u = new URL(url) - if (opts.method === 'POST' && u.pathname === '/api/v2/llm-obs/v1/proj/datasets/ds/records') { - return new Promise((resolve) => { - resolveRecordsFetch = () => resolve({ - ok: true, - status: 200, - text: sinon.stub().resolves(JSON.stringify({ records: [{ id: 'rec-0' }] })), - }) - }) - } - return routingFetch(url, opts) - }) - - const pushPromise = dataset.push() - // Let push() get past dataset creation and snapshot pending records, up to the - // in-flight records POST, before adding a second record. - await new Promise((resolve) => setTimeout(resolve, 0)) - dataset.addRecord('b') - resolveRecordsFetch() - await pushPromise - - global.fetch = routingFetch - await dataset.push() - - // Only the second push's records POST goes through the instrumented `calls` route - // (the first push's records POST was intercepted above to control its timing). - const recordPosts = calls.filter(c => c.method === 'POST' && c.path.endsWith('/proj/datasets/ds/records')) - assert.equal(recordPosts.length, 1, 'expected a second push to send the record dropped by the race') - assert.deepEqual(recordPosts[0].body.data.attributes.records.map((r) => r.input), ['b']) - assert.deepEqual(dataset.records().map((r) => r.input), ['a', 'b']) - assert.equal(dataset.recordIds().length, 2) - }) - - it('posts events with type "experiments", one span per row, auto tags and raw metadata', async () => { - const dataset = new Dataset(client, 'demo').addRecord({ q: 'apple' }, 'true', { row: 0 }) - await new Experiment(client, { - name: 'exp-demo', - dataset, - task: (i) => ({ answer: i.q }), - evaluators: { ok: () => true }, - tags: { env: 'test' }, - }).run() - - const body = eventsBody() - assert.equal(body.data.type, 'experiments') - const span = body.data.attributes.spans[0] - assert.match(span.span_id, /^[0-9a-f]{16}$/) - assert.match(span.trace_id, /^[0-9a-f]{32}$/) - // Root-span convention: the trace id's low 64 bits are the span id itself. - assert.equal(span.trace_id.slice(16), span.span_id) - assert.equal(span.status, 'ok') - assert.ok(span.tags.includes('experiment_id:exp')) - assert.ok(span.tags.includes('dataset_id:ds')) - assert.ok(span.tags.includes('dataset_record_id:rec-0')) - assert.ok(span.tags.includes('env:test')) - assert.deepEqual(span.meta.metadata, { row: 0 }) - }) - - it('infers metric type from the evaluator return value', async () => { - const dataset = new Dataset(client, 'demo').addRecord('x') - await new Experiment(client, { - name: 'exp-demo', - dataset, - task: () => 'out', - evaluators: { b: () => true, s: () => 0.5, c: () => 'label' }, - }).run() - - const metrics = eventsBody().data.attributes.metrics - const byLabel = (l) => metrics.find(m => m.label === l) - assert.equal(byLabel('b').metric_type, 'boolean') - assert.equal(byLabel('b').boolean_value, true) - assert.equal(byLabel('s').metric_type, 'score') - assert.equal(byLabel('s').score_value, 0.5) - assert.equal(byLabel('c').metric_type, 'categorical') - assert.equal(byLabel('c').categorical_value, 'label') - }) - - it('captures a task error per row without aborting the run, but marks the experiment failed', async () => { - const dataset = new Dataset(client, 'demo').addRecord('good').addRecord('bad').addRecord('good2') - const result = await new Experiment(client, { - name: 'exp-demo', - dataset, - task: (input) => { if (input === 'bad') throw new Error('boom'); return `ok:${input}` }, - evaluators: { len: (_i, o) => String(o ?? '').length }, - }).run() - - assert.equal(result.rows.length, 3) - assert.equal(result.rows[1].isError, true) - assert.equal(result.rows[1].errorMessage, 'boom') - assert.equal(eventsBody().data.attributes.spans[1].status, 'error') - const patch = calls.find(c => c.method === 'PATCH') - assert.equal(patch.body.data.attributes.status, 'failed') - assert.equal(patch.body.data.attributes.error, 'one or more rows failed') - }) - - it('captures an evaluator error per evaluation without aborting, and marks the experiment failed', async () => { - const dataset = new Dataset(client, 'demo').addRecord('x') - const result = await new Experiment(client, { - name: 'exp-demo', - dataset, - task: (i) => i, - evaluators: { - explodes: () => { throw new Error('eval-fail') }, - fine: () => true, - }, - }).run() + it('submits custom record ids with append responses', async () => { + const dataset = new Dataset(stubClient(), 'demo') + .addRecord(new DatasetRecord('a', null, {}, 'custom-a')) + .addRecord(new DatasetRecord('b', null, {}, 'custom-b')) - assert.equal(result.rows[0].evaluationErrors.explodes, 'eval-fail') - assert.equal(result.rows[0].evaluations.fine, true) - const errored = eventsBody().data.attributes.metrics.find(m => m.label === 'explodes') - assert.equal(errored.error.message, 'eval-fail') - const patch = calls.find(c => c.method === 'PATCH') - assert.equal(patch.body.data.attributes.status, 'failed') - }) - - it('marks the experiment completed when every row succeeds', async () => { - const dataset = new Dataset(client, 'demo').addRecord('x') - await new Experiment(client, { - name: 'exp-demo', - dataset, - task: (i) => i, - evaluators: { fine: () => true }, - }).run() + const result = await dataset.push() - const patch = calls.find(c => c.method === 'PATCH') - assert.equal(patch.body.data.attributes.status, 'completed') - assert.equal(patch.body.data.attributes.error, undefined) + assert.deepEqual(result, { pushedCount: 2, totalCount: 2 }) + assert.deepEqual(dataset.recordIds(), ['custom-a', 'custom-b']) + assert.deepEqual(dataset.records().map(record => record.id), ['custom-a', 'custom-b']) + assert.equal(dataset.version(), 2) + assert.equal(dataset.latestVersion(), 2) }) - it('experiment create body carries project_id, dataset_id, config and ensure_unique', async () => { - const dataset = new Dataset(client, 'demo').addRecord('x') - await new Experiment(client, { - name: 'exp-demo', dataset, task: (i) => i, config: { approach: 'kw' }, - }).run() + it('surfaces backend failures', async () => { + const c = stubClient({ createDatasetError: new Error('HTTP 500 boom') }) + const dataset = new Dataset(c, 'demo').addRecord('a') - const create = calls.find(c => - c.method === 'POST' && c.path.endsWith('/experiments') && !c.path.includes('/events') + await assert.rejects( + () => dataset.push(), + /Failed to create dataset 'demo'.*HTTP 500 boom/ ) - assert.equal(create.body.data.type, 'experiments') - assert.equal(create.body.data.attributes.project_id, 'proj') - assert.equal(create.body.data.attributes.dataset_id, 'ds') - assert.equal(create.body.data.attributes.ensure_unique, true) - assert.deepEqual(create.body.data.attributes.config, { approach: 'kw' }) }) it('validates required options', () => { - const dataset = new Dataset(client, 'demo') - assert.throws(() => new Experiment(client, { dataset, task: (i) => i }), /name/) - assert.throws(() => new Experiment(client, { name: 'n', task: (i) => i }), /dataset/) - assert.throws(() => new Experiment(client, { name: 'n', dataset }), /task/) + const dataset = new Dataset(client(), 'demo') + assert.throws(() => new Experiment(client(), { dataset, task: (input) => input }), /name/) + assert.throws(() => new Experiment(client(), { name: 'n', task: (input) => input }), /dataset/) + assert.throws(() => new Experiment(client(), { name: 'n', dataset }), /task/) }) it('exposes dataset getters and accepts a DatasetRecord instance', () => { - const { DatasetRecord } = require('../../../src/llmobs/experiments/dataset') - const dataset = new Dataset(client, 'my-name', 'desc').addRecord(new DatasetRecord('in', 'out', { m: 1 })) + const dataset = new Dataset(client(), 'my-name', 'desc').addRecord(new DatasetRecord('in', 'out', { m: 1 })) assert.equal(dataset.name(), 'my-name') assert.equal(dataset.id(), null) assert.equal(dataset.url(), null) @@ -273,113 +124,4 @@ describe('LLMObs Experiments — dataset + experiment run', () => { assert.equal(record.expectedOutput, 'out') assert.deepEqual(record.metadata, { m: 1 }) }) - - it('pads record ids when the push response is not an array', async () => { - installFetch(calls, { 'POST /api/v2/llm-obs/v1/proj/datasets/ds/records': { data: { ok: true } } }) - const dataset = new Dataset(client, 'demo').addRecord('a').addRecord('b') - const result = await dataset.push() - assert.deepEqual(dataset.recordIds(), ['', '']) - assert.deepEqual(result, { pushedCount: 0, totalCount: 2 }) - }) - - it('resolves with the pushed/total record counts on a successful push', async () => { - const dataset = new Dataset(client, 'demo').addRecord('a').addRecord('b') - const result = await dataset.push() - assert.deepEqual(result, { pushedCount: 2, totalCount: 2 }) - }) - - it('resolves with a lower pushedCount when the backend confirms fewer records than sent', async () => { - installFetch(calls, { - 'POST /api/v2/llm-obs/v1/proj/datasets/ds/records': { records: [{ id: 'rec-0' }] }, - }) - const dataset = new Dataset(client, 'demo').addRecord('a').addRecord('b') - const result = await dataset.push() - assert.deepEqual(result, { pushedCount: 1, totalCount: 2 }) - assert.deepEqual(dataset.recordIds(), ['rec-0', '']) - }) - - it('resolves with zero counts when there is nothing new to push', async () => { - const dataset = new Dataset(client, 'demo').addRecord('a') - await dataset.push() - const result = await dataset.push() - assert.deepEqual(result, { pushedCount: 0, totalCount: 0 }) - }) - - it('throws a clear error when dataset creation fails', async () => { - installFetchFailing(calls, 'POST /api/v2/llm-obs/v1/proj/datasets') - const dataset = new Dataset(client, 'demo').addRecord('a') - await assert.rejects(() => dataset.push(), /Failed to create dataset 'demo'/) - }) - - it('throws a clear error when pushing records fails', async () => { - installFetchFailing(calls, 'POST /api/v2/llm-obs/v1/proj/datasets/ds/records') - const dataset = new Dataset(client, 'demo').addRecord('a') - await assert.rejects(() => dataset.push(), /Failed to push records to dataset 'demo'/) - }) - - it('exposes experiment getters before and after run', async () => { - const dataset = new Dataset(client, 'demo').addRecord('x') - const experiment = new Experiment(client, { name: 'exp-demo', dataset, task: (i) => i }) - assert.equal(experiment.name(), 'exp-demo') - assert.equal(experiment.experimentId(), null) - assert.equal(experiment.url(), null) - await experiment.run() - assert.equal(experiment.experimentId(), 'exp') - assert.equal(experiment.url(), 'https://app.datadoghq.com/llm/experiments/exp') - }) - - it('throws when the dataset has no id after push', async () => { - installFetch(calls, { 'POST /api/v2/llm-obs/v1/proj/datasets': { data: {} } }) - const dataset = new Dataset(client, 'demo').addRecord('x') - await assert.rejects( - () => new Experiment(client, { name: 'exp-demo', dataset, task: (i) => i }).run(), - /has no id after push/ - ) - }) - - it('throws a clear error when experiment creation fails', async () => { - installFetchFailing(calls, 'POST /api/v2/llm-obs/v1/experiments') - const dataset = new Dataset(client, 'demo').addRecord('x') - await assert.rejects( - () => new Experiment(client, { name: 'exp-demo', dataset, task: (i) => i }).run(), - /Failed to create experiment 'exp-demo'/ - ) - }) - - it('marks the experiment failed and rethrows if posting events fails', async () => { - installFetchFailing(calls, 'POST /api/v2/llm-obs/v1/experiments/exp/events') - const dataset = new Dataset(client, 'demo').addRecord('x') - await assert.rejects(() => new Experiment(client, { name: 'exp-demo', dataset, task: (i) => i }).run()) - assert.ok(calls.some(c => c.method === 'PATCH' && c.body?.data?.attributes?.status === 'failed')) - }) - - it('classifies object-valued evaluator results as json (and empties null categorical values)', async () => { - const dataset = new Dataset(client, 'demo').addRecord('x') - await new Experiment(client, { - name: 'exp-demo', - dataset, - task: () => 'out', - evaluators: { obj: () => ({ x: 1 }), nul: () => null }, - }).run() - const metrics = eventsBody().data.attributes.metrics - const objMetric = metrics.find(m => m.label === 'obj') - assert.equal(objMetric.metric_type, 'json') - assert.deepEqual(objMetric.json_value, { x: 1 }) - assert.equal(metrics.find(m => m.label === 'nul').categorical_value, '') - }) - - it('keeps array-valued evaluator results categorical, lowercased like dd-trace-py', async () => { - const dataset = new Dataset(client, 'demo').addRecord('x') - await new Experiment(client, { - name: 'exp-demo', - dataset, - task: () => 'out', - evaluators: { arr: () => ['Pass', 'FAIL'], str: () => 'MATCH' }, - }).run() - const metrics = eventsBody().data.attributes.metrics - const arrMetric = metrics.find(m => m.label === 'arr') - assert.equal(arrMetric.metric_type, 'categorical') - assert.equal(arrMetric.categorical_value, '["pass","fail"]') - assert.equal(metrics.find(m => m.label === 'str').categorical_value, 'match') - }) }) diff --git a/packages/dd-trace/test/llmobs/experiments/index.spec.js b/packages/dd-trace/test/llmobs/experiments/index.spec.js index ed789551f6..9acffe4502 100644 --- a/packages/dd-trace/test/llmobs/experiments/index.spec.js +++ b/packages/dd-trace/test/llmobs/experiments/index.spec.js @@ -4,6 +4,7 @@ const assert = require('node:assert/strict') const { afterEach, beforeEach, describe, it } = require('mocha') const sinon = require('sinon') +const log = require('../../../src/log') const { createExperiments } = require('../../../src/llmobs/experiments') const NoopExperiments = require('../../../src/llmobs/experiments/noop') @@ -16,11 +17,17 @@ const enabledConfig = (overrides = {}) => ({ }) describe('LLMObs Experiments facade', () => { + let fetchHandler + let fetchStub let originalFetch beforeEach(() => { originalFetch = global.fetch - global.fetch = sinon.stub() + fetchHandler = async (url) => { + throw new Error(`Unexpected fetch ${url}`) + } + fetchStub = sinon.stub().callsFake((...args) => fetchHandler(...args)) + global.fetch = fetchStub }) afterEach(() => { @@ -28,11 +35,21 @@ describe('LLMObs Experiments facade', () => { sinon.restore() }) + const resolveFetchWith = (handler) => { + fetchHandler = handler + } + describe('createExperiments gating', () => { it('returns a no-op when LLM Obs is disabled', () => { + const warn = sinon.spy(log, 'warn') const exp = createExperiments({ llmobs: { DD_LLMOBS_ENABLED: false } }) assert.ok(exp instanceof NoopExperiments) - assert.throws(() => exp.createDataset('d'), /unavailable/) + + const dataset = exp.createDataset('d', { records: [{ inputData: 'in' }] }) + + assert.equal(dataset.name(), 'd') + assert.equal(dataset.records()[0].input, 'in') + sinon.assert.calledWith(warn, sinon.match(/LLMObs experiments unavailable/)) }) it('returns a no-op when app key is missing', () => { @@ -42,14 +59,27 @@ describe('LLMObs Experiments facade', () => { it('returns a working facade when enabled and credentialed', () => { const exp = createExperiments(enabledConfig()) - const dataset = exp.createDataset('d', 'desc') + const dataset = exp.createDataset('d', { + description: 'desc', + records: [{ inputData: 'in', expectedOutput: 'out', metadata: { source: 'test' } }], + }) assert.equal(typeof dataset.addRecord, 'function') + assert.equal(dataset.records()[0].input, 'in') const experiment = exp.experiment({ name: 'n', dataset, task: (i) => i }) assert.equal(typeof experiment.run, 'function') }) + it('rejects duplicate custom record ids', () => { + assert.throws( + () => createExperiments(enabledConfig()).createDataset('d', { + records: [{ id: 'r1', inputData: 'a' }, { id: 'r1', inputData: 'b' }], + }), + /Duplicate record id 'r1'/ + ) + }) + it('falls back to config.service for the project name when llmobs.mlApp is not set', async () => { - global.fetch.callsFake(async () => ({ + resolveFetchWith(async () => ({ ok: true, status: 200, text: sinon.stub().resolves(JSON.stringify({ data: { id: 'proj' } })), @@ -58,31 +88,91 @@ describe('LLMObs Experiments facade', () => { const exp = createExperiments(enabledConfig({ service: 'my-service', llmobs: { DD_LLMOBS_ENABLED: true } })) await exp.createDataset('d').push() - const [url, opts] = global.fetch.getCall(0).args + const [url, opts] = fetchStub.getCall(0).args assert.equal(new URL(url).pathname, '/api/v2/llm-obs/v1/projects') assert.equal(JSON.parse(opts.body).data.attributes.name, 'my-service') }) it('returns a no-op with actionable steps when neither mlApp nor service is set', () => { + const warn = sinon.spy(log, 'warn') const exp = createExperiments(enabledConfig({ service: undefined, llmobs: { DD_LLMOBS_ENABLED: true } })) assert.ok(exp instanceof NoopExperiments) - assert.throws(() => exp.createDataset('d'), /DD_LLMOBS_ML_APP.*DD_SERVICE/) + + exp.createDataset('d') + + sinon.assert.calledWith(warn, sinon.match(/DD_LLMOBS_ML_APP.*DD_SERVICE/)) }) }) describe('no-op (disabled / missing keys)', () => { - it('throws on every operation with a clear message', async () => { + it('warns and returns inert objects for every operation', async () => { + const warn = sinon.spy(log, 'warn') const exp = createExperiments({ llmobs: { DD_LLMOBS_ENABLED: false } }) - assert.throws(() => exp.createDataset('d'), /unavailable/) - assert.throws(() => exp.experiment({}), /unavailable/) - await assert.rejects(() => exp.pullDataset('d'), /unavailable/) + + const dataset = exp.createDataset('d') + assert.deepEqual(await dataset.push(), { pushedCount: 0, totalCount: 0 }) + assert.equal(dataset.url(), null) + + const pulled = await exp.pullDataset('d') + assert.equal(pulled.name(), 'd') + + const experiment = exp.experiment({ name: 'exp' }) + assert.equal(experiment.name(), 'exp') + assert.deepEqual(await experiment.run(), { experimentId: null, rows: [], url: null }) + sinon.assert.calledThrice(warn) + }) + + it('models inert datasets and experiments with stable accessors', async () => { + const warn = sinon.spy(log, 'warn') + const exp = new NoopExperiments() + + const ignoredDescriptionDataset = exp.createDataset('legacy description', 'ignored') + assert.deepEqual(ignoredDescriptionDataset.records(), []) + + const dataset = exp.createDataset('d', { + records: [{ + id: 'r1', + inputData: { question: 'q' }, + expectedOutput: { answer: 'a' }, + metadata: { source: 'test' }, + }], + }) + dataset.addRecord('input only') + + assert.equal(dataset.name(), 'd') + assert.equal(dataset.id(), null) + assert.equal(dataset.projectId(), null) + assert.equal(dataset.version(), null) + assert.equal(dataset.latestVersion(), null) + assert.deepEqual(dataset.recordIds(), []) + assert.equal(dataset.url(), null) + assert.deepEqual(dataset.records(), [ + { + id: 'r1', + input: { question: 'q' }, + expectedOutput: { answer: 'a' }, + metadata: { source: 'test' }, + }, + { id: null, input: 'input only', expectedOutput: null, metadata: {} }, + ]) + assert.deepEqual(await dataset.push(), { pushedCount: 0, totalCount: 0 }) + + const pulled = await exp.pullDataset('pulled') + assert.equal(pulled.name(), 'pulled') + + const experiment = exp.experiment() + assert.equal(experiment.name(), '') + assert.equal(experiment.experimentId(), null) + assert.equal(experiment.url(), null) + assert.deepEqual(await experiment.run(), { experimentId: null, rows: [], url: null }) + sinon.assert.callCount(warn, 4) }) }) describe('pullDataset', () => { const resolveRoutes = (recordsResponses) => { let recordsCall = 0 - global.fetch.callsFake(async (url) => { + resolveFetchWith(async (url) => { const u = new URL(url) let payload if (u.pathname === '/api/v2/llm-obs/v1/projects') { @@ -113,6 +203,52 @@ describe('LLMObs Experiments facade', () => { assert.deepEqual(ds.records()[0].input, { q: '2+2' }) assert.equal(ds.records()[0].expectedOutput, '4') assert.deepEqual(ds.records()[0].metadata, { a: 1 }) + assert.equal(ds.records()[0].id, 'r1') + assert.equal(ds.records()[1].id, 'r2') + }) + + it('passes explicit dataset version when reading records', async () => { + resolveFetchWith(async (url) => { + const u = new URL(url) + let payload + if (u.pathname === '/api/v2/llm-obs/v1/projects') { + payload = { data: { id: 'proj' } } + } else if (u.pathname === '/api/v2/llm-obs/v1/proj/datasets') { + payload = { data: [{ id: 'ds9', attributes: { name: 'wanted', description: 'd', current_version: 7 } }] } + } else if (u.pathname === '/api/v2/llm-obs/v1/proj/datasets/ds9/records') { + assert.equal(u.searchParams.get('filter[version]'), '3') + payload = { data: [{ id: 'r1', attributes: { input: 'i1' } }] } + } else { + payload = {} + } + return { ok: true, status: 200, text: sinon.stub().resolves(JSON.stringify(payload)) } + }) + + const ds = await createExperiments(enabledConfig()).pullDataset('wanted', { version: 3 }) + assert.equal(ds.version(), 3) + assert.equal(ds.latestVersion(), 7) + }) + + it('pins the current version when pulling latest records', async () => { + resolveFetchWith(async (url) => { + const u = new URL(url) + let payload + if (u.pathname === '/api/v2/llm-obs/v1/projects') { + payload = { data: { id: 'proj' } } + } else if (u.pathname === '/api/v2/llm-obs/v1/proj/datasets') { + payload = { data: [{ id: 'ds9', attributes: { name: 'wanted', description: 'd', current_version: 7 } }] } + } else if (u.pathname === '/api/v2/llm-obs/v1/proj/datasets/ds9/records') { + assert.equal(u.searchParams.get('filter[version]'), '7') + payload = { data: [{ id: 'r1', attributes: { input: 'i1' } }] } + } else { + payload = {} + } + return { ok: true, status: 200, text: sinon.stub().resolves(JSON.stringify(payload)) } + }) + + const ds = await createExperiments(enabledConfig()).pullDataset('wanted') + assert.equal(ds.version(), 7) + assert.equal(ds.records().length, 1) }) it('waits (backoff) until the expected record count is readable', async () => { @@ -128,7 +264,7 @@ describe('LLMObs Experiments facade', () => { }) it('throws when the dataset is absent (no wait)', async () => { - global.fetch.callsFake(async (url) => { + resolveFetchWith(async (url) => { const u = new URL(url) const payload = u.pathname === '/api/v2/llm-obs/v1/projects' ? { data: { id: 'proj' } } : { data: [] } return { ok: true, status: 200, text: sinon.stub().resolves(JSON.stringify(payload)) } @@ -140,7 +276,7 @@ describe('LLMObs Experiments facade', () => { }) it('throws with the underlying error when listing datasets fails', async () => { - global.fetch.callsFake(async (url) => { + resolveFetchWith(async (url) => { const u = new URL(url) if (u.pathname === '/api/v2/llm-obs/v1/projects') { return { ok: true, status: 200, text: sinon.stub().resolves(JSON.stringify({ data: { id: 'proj' } })) } @@ -162,7 +298,7 @@ describe('LLMObs Experiments facade', () => { }) it('throws the underlying error when fetching records fails, even without expectedRecordCount', async () => { - global.fetch.callsFake(async (url) => { + resolveFetchWith(async (url) => { const u = new URL(url) if (u.pathname === '/api/v2/llm-obs/v1/projects') { return { ok: true, status: 200, text: sinon.stub().resolves(JSON.stringify({ data: { id: 'proj' } })) } @@ -184,7 +320,7 @@ describe('LLMObs Experiments facade', () => { '': { data: [{ id: 'r1', attributes: { input: 'i1' } }], meta: { after: 'cursor1' } }, cursor1: { data: [{ id: 'r2', attributes: { input: 'i2' } }], meta: { after: '' } }, } - global.fetch.callsFake(async (url) => { + resolveFetchWith(async (url) => { const u = new URL(url) let payload if (u.pathname === '/api/v2/llm-obs/v1/projects') { diff --git a/packages/dd-trace/test/llmobs/sdk/index.spec.js b/packages/dd-trace/test/llmobs/sdk/index.spec.js index 55720f9448..c622b9c098 100644 --- a/packages/dd-trace/test/llmobs/sdk/index.spec.js +++ b/packages/dd-trace/test/llmobs/sdk/index.spec.js @@ -90,6 +90,13 @@ describe('sdk', () => { } }) + describe('experiments', () => { + it('exposes dataset operations only through the experiments facade', () => { + assert.strictEqual(typeof llmobs.experiments.createDataset, 'function') + assert.strictEqual(typeof llmobs.experiments.pullDataset, 'function') + }) + }) + describe('enable', () => { it('enables llmobs if it is disabled', () => { const config = getConfigFresh({}) diff --git a/packages/dd-trace/test/plugins/util/ci-env/buildkite.json b/packages/dd-trace/test/plugins/util/ci-env/buildkite.json index 8013574b20..ee83765f8e 100644 --- a/packages/dd-trace/test/plugins/util/ci-env/buildkite.json +++ b/packages/dd-trace/test/plugins/util/ci-env/buildkite.json @@ -12,6 +12,7 @@ "BUILDKITE_COMMIT": "b9f0fb3fdbb94c9d24b2c75b49663122a529e123", "BUILDKITE_JOB_ID": "buildkite-job-id", "BUILDKITE_MESSAGE": "buildkite-git-commit-message", + "BUILDKITE_PIPELINE_NAME": "buildkite-pipeline-display-name", "BUILDKITE_PIPELINE_SLUG": "buildkite-pipeline-name", "BUILDKITE_PULL_REQUEST": "false", "BUILDKITE_PULL_REQUEST_BASE_BRANCH": "", @@ -24,6 +25,7 @@ "ci.job.url": "https://buildkite-build-url.com#buildkite-job-id", "ci.pipeline.id": "buildkite-pipeline-id", "ci.pipeline.name": "buildkite-pipeline-name", + "ci.pipeline.display_name": "buildkite-pipeline-display-name", "ci.pipeline.number": "buildkite-pipeline-number", "ci.pipeline.url": "https://buildkite-build-url.com", "ci.provider.name": "buildkite", diff --git a/packages/dd-trace/test/plugins/util/test.spec.js b/packages/dd-trace/test/plugins/util/test.spec.js index df98f2c7c5..6d9ac805f4 100644 --- a/packages/dd-trace/test/plugins/util/test.spec.js +++ b/packages/dd-trace/test/plugins/util/test.spec.js @@ -54,6 +54,7 @@ const { const { CI_JOB_NAME, + CI_PIPELINE_DISPLAY_NAME, CI_PIPELINE_URL, CI_PROVIDER_NAME, GIT_COMMIT_SHA, @@ -297,6 +298,7 @@ describe('getTestLevelsMetadataTags', () => { it('keeps only allowlisted CI and Git tags', () => { const testLevelsMetadataTags = getTestLevelsMetadataTags({ [CI_JOB_NAME]: 'test', + [CI_PIPELINE_DISPLAY_NAME]: 'Pipeline Display Name', [CI_PIPELINE_URL]: 'https://github.com/DataDog/dd-trace-js/actions/runs/1', [CI_PROVIDER_NAME]: 'github', [GIT_COMMIT_SHA]: '1234567890abcdef', @@ -308,6 +310,7 @@ describe('getTestLevelsMetadataTags', () => { assert.deepStrictEqual(testLevelsMetadataTags, { [CI_JOB_NAME]: 'test', + [CI_PIPELINE_DISPLAY_NAME]: 'Pipeline Display Name', [CI_PIPELINE_URL]: 'https://github.com/DataDog/dd-trace-js/actions/runs/1', [CI_PROVIDER_NAME]: 'github', [GIT_COMMIT_SHA]: '1234567890abcdef', @@ -761,6 +764,11 @@ describe('getCodeOwnersForFilename', () => { matches: ['file1.js', 'packages/dd-trace/fileA.js'], misses: ['file10.js', 'packages/dd-trace/file/name.js'], }, + { + pattern: String.raw`file\*.js`, + matches: ['file*.js', 'packages/dd-trace/file*.js'], + misses: ['file1.js', 'packages/dd-trace/fileA.js'], + }, ] for (const { pattern, matches = [], misses = [] } of patternTests) { @@ -777,6 +785,14 @@ describe('getCodeOwnersForFilename', () => { } }) + it('treats a trailing backslash as a literal instead of throwing', () => { + const codeOwnersFileEntries = [ + { pattern: 'docs\\', owners: ['@datadog-docs'] }, + ] + + assert.strictEqual(getCodeOwnersForFilename('docs/README.md', codeOwnersFileEntries), null) + }) + it('keeps CODEOWNERS matching case-sensitive', () => { const codeOwnersFileEntries = [ { pattern: '/Docs/', owners: ['@datadog-docs'] }, diff --git a/packages/dd-trace/test/plugins/versions/package.json b/packages/dd-trace/test/plugins/versions/package.json index bac5fffcca..9a297197f5 100644 --- a/packages/dd-trace/test/plugins/versions/package.json +++ b/packages/dd-trace/test/plugins/versions/package.json @@ -4,39 +4,39 @@ "license": "BSD-3-Clause", "private": true, "dependencies": { - "@ai-sdk/amazon-bedrock": "5.0.30", - "@ai-sdk/anthropic": "4.0.19", - "@ai-sdk/google": "4.0.23", + "@ai-sdk/amazon-bedrock": "5.0.31", + "@ai-sdk/anthropic": "4.0.20", + "@ai-sdk/google": "4.0.24", "@ai-sdk/openai": "4.0.20", - "@anthropic-ai/claude-agent-sdk": "0.3.215", - "@anthropic-ai/sdk": "0.114.0", - "@apollo/gateway": "2.14.2", + "@anthropic-ai/claude-agent-sdk": "0.3.220", + "@anthropic-ai/sdk": "0.115.0", + "@apollo/gateway": "2.14.3", "@apollo/server": "5.5.1", - "@apollo/subgraph": "2.14.2", + "@apollo/subgraph": "2.14.3", "@aws/durable-execution-sdk-js": "2.2.0", "@aws/durable-execution-sdk-js-testing": "1.1.3", - "@aws-sdk/client-bedrock-runtime": "3.1094.0", - "@aws-sdk/client-dynamodb": "3.1094.0", - "@aws-sdk/client-kinesis": "3.1094.0", - "@aws-sdk/client-lambda": "3.1094.0", - "@aws-sdk/client-s3": "3.1094.0", - "@aws-sdk/client-sfn": "3.1094.0", - "@aws-sdk/client-sns": "3.1094.0", - "@aws-sdk/client-sqs": "3.1094.0", + "@aws-sdk/client-bedrock-runtime": "3.1095.0", + "@aws-sdk/client-dynamodb": "3.1095.0", + "@aws-sdk/client-kinesis": "3.1095.0", + "@aws-sdk/client-lambda": "3.1095.0", + "@aws-sdk/client-s3": "3.1095.0", + "@aws-sdk/client-sfn": "3.1095.0", + "@aws-sdk/client-sns": "3.1095.0", + "@aws-sdk/client-sqs": "3.1095.0", "@aws-sdk/node-http-handler": "3.374.0", "@aws-sdk/smithy-client": "3.374.0", "@azure/cosmos": "4.10.0", "@azure/event-hubs": "6.0.4", "@azure/functions": "4.16.2", "@azure/service-bus": "7.9.5", - "@babel/core": "7.29.0", - "@babel/preset-typescript": "7.28.5", + "@babel/core": "8.0.1", + "@babel/preset-typescript": "8.0.1", "@confluentinc/kafka-javascript": "1.10.0", "@cucumber/cucumber": "13.2.0", - "@datadog/openfeature-node-server": "2.0.1", + "@datadog/openfeature-node-server": "2.0.2", "@elastic/elasticsearch": "9.4.2", "@elastic/transport": "9.3.7", - "@electron/packager": "20.0.0", + "@electron/packager": "20.0.4", "@fastify/cookie": "11.1.2", "@fastify/multipart": "10.1.0", "@google-cloud/pubsub": "5.3.1", @@ -46,16 +46,16 @@ "@grpc/grpc-js": "1.14.4", "@grpc/proto-loader": "0.8.1", "@hapi/boom": "10.0.1", - "@hapi/hapi": "21.4.9", - "@happy-dom/jest-environment": "20.10.6", - "@hono/node-server": "2.0.10", + "@hapi/hapi": "21.4.10", + "@happy-dom/jest-environment": "20.11.1", + "@hono/node-server": "2.0.11", "@jest/core": "30.4.2", "@jest/globals": "30.4.1", "@jest/reporters": "30.4.1", "@jest/test-sequencer": "30.4.1", "@jest/transform": "30.4.1", "@koa/router": "15.7.0", - "@langchain/anthropic": "1.5.1", + "@langchain/anthropic": "1.5.2", "@langchain/classic": "1.0.40", "@langchain/cohere": "1.1.0", "@langchain/core": "1.2.3", @@ -73,25 +73,28 @@ "@openfeature/server-sdk": "1.22.0", "@opensearch-project/opensearch": "3.6.0", "@opentelemetry/api": "1.9.1", - "@opentelemetry/api-logs": "0.220.0", - "@opentelemetry/exporter-jaeger": "2.9.0", - "@opentelemetry/instrumentation": "0.220.0", - "@opentelemetry/instrumentation-express": "0.68.0", - "@opentelemetry/instrumentation-http": "0.220.0", - "@opentelemetry/sdk-node": "0.220.0", - "@playwright/test": "1.61.0", + "@opentelemetry/api-logs": "0.221.0", + "@opentelemetry/exporter-jaeger": "2.10.0", + "@opentelemetry/instrumentation": "0.221.0", + "@opentelemetry/instrumentation-express": "0.69.0", + "@opentelemetry/instrumentation-http": "0.221.0", + "@opentelemetry/sdk-node": "0.221.0", + "@playwright/test": "1.62.0", "@prisma/adapter-mariadb": "7.8.0", "@prisma/adapter-mssql": "7.8.0", "@prisma/adapter-pg": "7.8.0", "@prisma/client": "7.8.0", "@redis/client": "6.1.0", - "@smithy/core": "3.29.8", - "@smithy/smithy-client": "4.14.13", + "@smithy/core": "3.30.0", + "@smithy/smithy-client": "4.14.14", "@types/node": "26.1.1", - "@vitest/coverage-istanbul": "4.1.9", - "@vitest/coverage-v8": "4.1.9", - "@vitest/runner": "4.1.9", + "@vitest/coverage-istanbul": "4.1.10", + "@vitest/coverage-v8": "4.1.10", + "@vitest/runner": "4.1.10", "@vscode/sqlite3": "5.1.12-vscode", + "@wdio/cli": "9.30.0", + "@wdio/local-runner": "9.30.0", + "@wdio/mocha-framework": "9.30.0", "aerospike": "6.7.1", "ai": "7.0.37", "amqp10": "3.6.0", @@ -107,7 +110,7 @@ "bluebird": "3.7.2", "body-parser": "2.3.0", "bson": "7.3.1", - "bullmq": "5.81.0", + "bullmq": "5.81.2", "bunyan": "2.0.5", "cassandra-driver": "4.9.0", "collections": "5.1.13", @@ -115,29 +118,29 @@ "cookie": "2.0.1", "cookie-parser": "1.4.7", "couchbase": "4.7.1", - "cypress": "15.16.0", + "cypress": "15.19.0", "cypress-fail-fast": "8.1.0", "dd-trace-api": "1.0.1", "durable-functions": "3.5.0", "ejs": "6.0.1", "elasticsearch": "16.7.3", - "electron": "42.1.0", - "esbuild": "0.28.0", + "electron": "43.2.0", + "esbuild": "0.28.1", "express": "5.2.1", "express-mongo-sanitize": "2.2.0", "express-session": "1.19.0", "fastify": "5.10.0", - "find-my-way": "9.6.0", + "find-my-way": "9.7.0", "fs": "0.0.1-security", "generic-pool": "3.9.0", - "google-gax": "5.0.7", + "google-gax": "5.0.8", "graphql": "16.14.2", "graphql-tag": "2.12.7", "graphql-tools": "9.0.33", "graphql-yoga": "5.21.2", "handlebars": "4.7.9", "hapi": "18.1.0", - "hono": "4.12.30", + "hono": "4.12.32", "ioredis": "5.11.1", "iovalkey": "0.3.3", "jest": "30.4.2", @@ -156,7 +159,7 @@ "koa-route": "4.0.1", "koa-router": "14.0.0", "koa-websocket": "7.0.0", - "langchain": "1.5.3", + "langchain": "1.5.4", "ldapjs": "3.0.7", "ldapjs-promise": "3.0.8", "light-my-request": "6.6.0", @@ -170,7 +173,7 @@ "middie": "7.1.0", "mocha": "11.7.6", "mocha-each": "2.0.1", - "moleculer": "0.15.0", + "moleculer": "0.15.1", "mongodb": "7.5.0", "mongodb-core": "3.2.7", "mongoose": "9.7.4", @@ -178,8 +181,8 @@ "multer": "2.2.0", "mysql": "2.18.1", "mysql2": "3.23.0", - "next": "16.2.6", - "nock": "14.0.15", + "next": "16.2.12", + "nock": "14.0.16", "node-18": "npm:node@18.20.8", "node-20": "npm:node@20.20.2", "node-22": "npm:node@22.22.3", @@ -200,36 +203,36 @@ "pg-query-stream": "4.16.0", "pino": "10.3.1", "pino-pretty": "13.1.3", - "playwright": "1.61.0", - "playwright-core": "1.61.0", - "pnpm": "11.15.1", + "playwright": "1.62.0", + "playwright-core": "1.62.0", + "pnpm": "11.17.0", "prisma": "7.8.0", "promise": "8.3.0", "promise-js": "0.0.7", "protobufjs": "8.7.1", "pug": "3.0.4", "q": "2.0.3", - "react": "19.2.6", - "react-dom": "19.2.6", + "react": "19.2.8", + "react-dom": "19.2.8", "redis": "6.1.0", "request": "2.88.2", "restify": "11.1.0", "rhea": "3.0.5", "router": "2.2.0", - "selenium-webdriver": "4.44.0", + "selenium-webdriver": "4.46.0", "sequelize": "6.37.8", "sharedb": "6.0.1", - "sinon": "22.0.0", + "sinon": "22.1.0", "sqlite3": "6.0.1", "stripe": "22.3.2", "tedious": "20.0.0", "tinypool": "2.1.0", - "typescript": "6.0.3", - "undici": "8.7.0", - "vitest": "4.1.9", + "typescript": "7.0.2", + "undici": "8.9.0", + "vitest": "4.1.10", "when": "3.7.8", "winston": "3.19.0", - "workerpool": "10.0.2", + "workerpool": "10.0.3", "ws": "8.21.1", "yarn": "1.22.22", "zod": "4.4.3", diff --git a/packages/dd-trace/test/startup-log.spec.js b/packages/dd-trace/test/startup-log.spec.js index aa0df80865..47f6a0b732 100644 --- a/packages/dd-trace/test/startup-log.spec.js +++ b/packages/dd-trace/test/startup-log.spec.js @@ -3,7 +3,7 @@ const assert = require('node:assert') const os = require('node:os') -const { describe, it, before, afterEach } = require('mocha') +const { describe, it, before, beforeEach, afterEach } = require('mocha') const sinon = require('sinon') require('./setup/core') @@ -80,6 +80,9 @@ describe('startup logging', () => { assert.strictEqual(logObj.debug, true) assert.strictEqual(logObj.appsec_enabled, true) assert.strictEqual(logObj.data_streams_enabled, true) + assert.strictEqual('otlp_traces_export_enabled' in logObj, true) + assert.strictEqual('otlp_metrics_export_enabled' in logObj, true) + assert.strictEqual('otlp_logs_export_enabled' in logObj, true) }) it('logIntegrations should output loaded integrations', () => { @@ -121,6 +124,9 @@ describe('startup logging', () => { integrations_loaded: ['http', 'fs', 'semver'], appsec_enabled: true, data_streams_enabled: true, + otlp_traces_export_enabled: false, + otlp_metrics_export_enabled: false, + otlp_logs_export_enabled: false, }) }) }) @@ -314,3 +320,70 @@ describe('profiling_enabled', () => { }) }) }) + +describe('otlp export flags', () => { + function clearOtlpEnv () { + delete process.env.OTEL_TRACES_EXPORTER + delete process.env.OTEL_METRICS_EXPORTER + delete process.env.OTEL_LOGS_EXPORTER + delete process.env.DD_METRICS_OTEL_ENABLED + delete process.env.DD_LOGS_OTEL_ENABLED + } + + // Datadog-instrumented dev shells export the OTEL_*_EXPORTER selectors: a leaked + // OTEL_TRACES_EXPORTER=otlp corrupts the default-state assertion, and OTEL_METRICS_EXPORTER=none + // makes config force DD_METRICS_OTEL_ENABLED back to false, breaking the metrics positive case. + beforeEach(clearOtlpEnv) + afterEach(clearOtlpEnv) + + function startupLogObj (configOptions) { + sinon.stub(console, 'warn') + delete require.cache[require.resolve('../src/startup-log')] + const { + setStartupLogConfig, + startupLog, + } = require('../src/startup-log') + process.env.DD_TRACE_STARTUP_LOGS = 'true' + setStartupLogConfig(getConfigFresh(configOptions)) + startupLog() + /* eslint-disable-next-line no-console */ + const warnStub = /** @type {sinon.SinonStub} */ (console.warn) + const logObj = JSON.parse(warnStub.firstCall.args[0].replace('DATADOG TRACER CONFIGURATION - ', '')) + warnStub.restore() + return logObj + } + + it('should default to false when no OTLP env vars are set', () => { + const logObj = startupLogObj() + assert.strictEqual(logObj.otlp_traces_export_enabled, false) + assert.strictEqual(logObj.otlp_metrics_export_enabled, false) + assert.strictEqual(logObj.otlp_logs_export_enabled, false) + }) + + it('otlp_traces_export_enabled should be true when OTEL_TRACES_EXPORTER is otlp', () => { + process.env.OTEL_TRACES_EXPORTER = 'otlp' + assert.strictEqual(startupLogObj().otlp_traces_export_enabled, true) + }) + + it('otlp_traces_export_enabled should be false when OTEL_TRACES_EXPORTER is none', () => { + process.env.OTEL_TRACES_EXPORTER = 'none' + assert.strictEqual(startupLogObj().otlp_traces_export_enabled, false) + }) + + it('otlp_traces_export_enabled should be false in Test Optimization mode even when exporter is otlp', () => { + // Test Optimization keeps test spans on the citestcycle endpoint, so the OTLP + // trace exporter is not used regardless of OTEL_TRACES_EXPORTER (see opentracing/tracer.js). + process.env.OTEL_TRACES_EXPORTER = 'otlp' + assert.strictEqual(startupLogObj({ isCiVisibility: true }).otlp_traces_export_enabled, false) + }) + + it('otlp_metrics_export_enabled should be true when DD_METRICS_OTEL_ENABLED is true', () => { + process.env.DD_METRICS_OTEL_ENABLED = 'true' + assert.strictEqual(startupLogObj().otlp_metrics_export_enabled, true) + }) + + it('otlp_logs_export_enabled should be true when DD_LOGS_OTEL_ENABLED is true', () => { + process.env.DD_LOGS_OTEL_ENABLED = 'true' + assert.strictEqual(startupLogObj().otlp_logs_export_enabled, true) + }) +}) diff --git a/packages/dd-trace/test/util.spec.js b/packages/dd-trace/test/util.spec.js index 72e664613c..daeef948b3 100644 --- a/packages/dd-trace/test/util.spec.js +++ b/packages/dd-trace/test/util.spec.js @@ -5,7 +5,7 @@ const assert = require('node:assert/strict') const { describe, it } = require('mocha') require('./setup/core') -const { isEmpty, isTrue, isFalse, globMatch, getSegment } = require('../src/util') +const { isEmpty, isTrue, isFalse, globMatch, getSegment, stripQueryAndFragment } = require('../src/util') const TRUES = [ 1, @@ -120,4 +120,31 @@ describe('util', () => { assert.strictEqual(getSegment('whole', '.', 0), 'whole') }) }) + + describe('stripQueryAndFragment', () => { + const cases = [ + '/api/v1/users', + '/api/v1/users?page=2&limit=50', + '/search?q=hello+world#results', + '/docs#section-3', + '/docs#section?not-a-query', + '/redirect?to=/other%3Fnested#frag', + '/?only-query', + '/#only-fragment', + '?leading-query', + '#leading-fragment', + '/', + '', + ] + + it('matches splitting on the query and fragment delimiters', () => { + for (const target of cases) { + assert.strictEqual( + stripQueryAndFragment(target), + target.split(/[?#]/)[0], + `stripQueryAndFragment(${JSON.stringify(target)})` + ) + } + }) + }) }) diff --git a/scripts/agentless-stress-test.js b/scripts/agentless-stress-test.js index 181aee772b..6b043c56b6 100644 --- a/scripts/agentless-stress-test.js +++ b/scripts/agentless-stress-test.js @@ -21,9 +21,9 @@ if (!process.env.DD_API_KEY) { } process.env._DD_APM_TRACING_AGENTLESS_ENABLED = 'true' -process.env.DD_TRACE_DEBUG = process.env.DD_TRACE_DEBUG || 'false' -process.env.DD_ENV = process.env.DD_ENV || 'agentless-stress-test' -process.env.DD_SERVICE = process.env.DD_SERVICE || 'agentless-stress-test' +process.env.DD_TRACE_DEBUG ||= 'false' +process.env.DD_ENV ||= 'agentless-stress-test' +process.env.DD_SERVICE ||= 'agentless-stress-test' process.env.DD_TRACE_FLUSH_INTERVAL = '2000' const tracer = require('../packages/dd-trace').init() diff --git a/scripts/check_licenses.js b/scripts/check_licenses.js index bf968514b8..c54f95b30f 100644 --- a/scripts/check_licenses.js +++ b/scripts/check_licenses.js @@ -1,199 +1,291 @@ /* eslint-disable no-console */ 'use strict' -const { createReadStream, existsSync } = require('node:fs') +const { existsSync, readFileSync } = require('node:fs') const { join } = require('node:path') -const readline = require('node:readline') -const { execSync } = require('node:child_process') -const { name: rootPackageName } = require('../package.json') + +const { parse: parseYarnLock } = require('@yarnpkg/lockfile') +const { parse: parseJsonc, printParseErrorCode } = require('jsonc-parser') + +/** + * @typedef {object} DependencyManifest + * @property {string} [name] + * @property {Record} [dependencies] + * @property {Record} [optionalDependencies] + * @property {Record} [peerDependencies] + * @property {string[]} [optionalPeers] + * @property {boolean} [dev] + * @property {boolean} [devOptional] + * @property {boolean} [link] + */ /** - * @typedef {object} NpmDependency - * @property {string} [resolved] - * @property {Record} [dependencies] + * @typedef {object} BunPackageContext + * @property {string} packagePath + * @property {BunPackageContext} [parent] */ -const filePath = join(__dirname, '..', 'LICENSE-3rdparty.csv') -const aliasMap = getAliasMap() -const deps = getProdDeps() +/** + * @typedef {object} DependencyPattern + * @property {boolean} [allowMissing] + * @property {BunPackageContext} [context] + * @property {string} name + * @property {string} range + */ + +const rootDirectory = process.cwd() +const dependencies = getProductionDependencies(rootDirectory) const licenses = new Set() -let isHeader = true -const lineReader = readline.createInterface({ - input: createReadStream(filePath), -}) +addCsvComponents(licenses, join(rootDirectory, 'LICENSE-3rdparty.csv'), true) + +if (!checkLicenses(dependencies)) process.exitCode = 1 -lineReader.on('line', line => { - if (isHeader) { - isHeader = false - return +/** + * @param {string} directory + */ +function getProductionDependencies (directory) { + const packageJson = require(join(directory, 'package.json')) + const dependencies = new Set([packageJson.name]) + const bunLockPath = join(directory, 'bun.lock') + + if (existsSync(bunLockPath)) { + addBunProductionDependencies(dependencies, bunLockPath, packageJson) + } else { + addYarnProductionDependencies(dependencies, directory, packageJson) } + addNpmProductionDependencies(dependencies, join(directory, 'vendor', 'package-lock.json')) - const trimmed = line.trim() - if (!trimmed) return // Skip empty lines - const columns = line.split(',') - const component = columns[0] + const vendoredDependenciesPath = join(directory, '.github', 'vendored-dependencies.csv') + if (existsSync(vendoredDependenciesPath)) addCsvComponents(dependencies, vendoredDependenciesPath, false) - // Strip quotes from the component name - licenses.add(component.replaceAll(/^"|"$/g, '')) -}) + return dependencies +} -lineReader.on('close', () => { - if (!checkLicenses(deps)) { - process.exit(1) +/** + * @param {Set} dependencies + * @param {string} directory + * @param {DependencyManifest} packageJson + */ +function addYarnProductionDependencies (dependencies, directory, packageJson) { + const { object: lock, type } = parseYarnLock(readFileSync(join(directory, 'yarn.lock'), 'utf8')) + if (type !== 'success') throw new Error(`Cannot parse yarn.lock: ${type}`) + + /** @type {DependencyPattern[]} */ + const patterns = [] + const visited = new Set() + addDependencyPatterns(patterns, packageJson) + + for (let i = 0; i < patterns.length; i++) { + const { name, range } = patterns[i] + const pattern = `${name}@${range}` + if (visited.has(pattern)) continue + + const dependency = lock[pattern] + if (!dependency) throw new Error(`Missing yarn.lock entry for ${pattern}`) + + visited.add(pattern) + dependencies.add(normalizeDependencyName(name, range)) + addDependencyPatterns(patterns, dependency) } -}) +} -function getProdDeps () { - // Add root package (dd-trace) to the set of dependencies manually as it is not included in the yarn list output. - const deps = new Set([normalizeDepName(rootPackageName)]) +/** + * @param {Set} dependencies + * @param {string} bunLockPath + * @param {DependencyManifest} packageJson + */ +function addBunProductionDependencies (dependencies, bunLockPath, packageJson) { + const parseErrors = [] + const lock = parseJsonc(readFileSync(bunLockPath, 'utf8'), parseErrors, { allowTrailingComma: true }) + if (parseErrors.length) { + const { error, offset } = parseErrors[0] + throw new Error(`Cannot parse bun.lock: ${printParseErrorCode(error)} at offset ${offset}`) + } + if (!lock || typeof lock !== 'object' || Array.isArray(lock)) { + throw new Error('bun.lock does not contain an object') + } + + const { lockfileVersion, packages, workspaces } = lock + if (!Number.isInteger(lockfileVersion) || lockfileVersion < 0 || lockfileVersion > 2) { + throw new Error(`Unsupported bun.lock version: ${lockfileVersion}`) + } + if (!packages || typeof packages !== 'object' || Array.isArray(packages)) { + throw new Error('bun.lock does not contain package metadata') + } + const rootWorkspace = workspaces?.[''] + if (!rootWorkspace || typeof rootWorkspace !== 'object' || Array.isArray(rootWorkspace)) { + throw new Error('bun.lock does not contain a root workspace') + } - addYarnProdDeps(deps, process.cwd()) - addNpmProdDeps(deps, join(process.cwd(), 'vendor')) + /** @type {DependencyPattern[]} */ + const patterns = [] + const visited = new Set() + addDependencyPatterns(patterns, packageJson) + + for (let i = 0; i < patterns.length; i++) { + const { allowMissing, context, name } = patterns[i] + const packagePath = getBunPackagePath(packages, name, context) + if (packagePath === undefined) { + if (allowMissing) continue + throw new Error(`Missing bun.lock entry for ${name}`) + } + if (visited.has(packagePath)) continue - // Add vendored dependencies - addVendoredDeps(deps) + const entry = packages[packagePath] + if (!Array.isArray(entry) || typeof entry[0] !== 'string') { + throw new TypeError(`Invalid bun.lock entry for ${packagePath}`) + } - return deps -} + visited.add(packagePath) + const resolution = entry[0] + const versionIndex = resolution.indexOf('@', resolution.startsWith('@') ? 1 : 0) + if (versionIndex <= 0) throw new Error(`Invalid bun.lock resolution for ${packagePath}`) + + const packageName = resolution.slice(0, versionIndex) + const source = resolution.slice(versionIndex + 1) + const childContext = { packagePath, parent: context } + if (source.startsWith('workspace:')) { + const workspace = workspaces[source.slice('workspace:'.length)] + if (!workspace || typeof workspace !== 'object' || Array.isArray(workspace)) { + throw new Error(`Missing bun.lock workspace for ${packagePath}`) + } + + addDependencyPatterns(patterns, workspace, childContext, true) + continue + } -function addYarnProdDeps (deps, cwd) { - // Use yarn to get full tree of production (non-dev) dependencies (format is ndjson) - const stdout = execSync('yarn list --production --json', { - encoding: 'utf8', - stdio: ['ignore', 'pipe', 'inherit'], - cwd, - }) - - for (const line of stdout.split('\n')) { - if (!line) continue - const parsed = JSON.parse(line) - if (parsed.type === 'tree' && Array.isArray(parsed.data?.trees)) { - collectFromTrees(parsed.data.trees, deps) + if (!source.startsWith('link:')) dependencies.add(packageName) + for (let entryIndex = 1; entryIndex < entry.length; entryIndex++) { + const manifest = entry[entryIndex] + if (typeof manifest === 'object' && manifest !== null && !Array.isArray(manifest)) { + addDependencyPatterns(patterns, manifest, childContext, true) + break + } } } } -function addNpmProdDeps (deps, cwd) { - // Use npm to get full tree of production (non-dev) dependencies - const stdout = execSync('npm list --omit=dev --json --depth=10', { - encoding: 'utf8', - stdio: ['ignore', 'pipe', 'inherit'], - cwd, - }) - - const parsed = JSON.parse(stdout) - - collectDependencies(deps, parsed) +/** + * @param {Record} packages + * @param {string} name + * @param {BunPackageContext} [context] + */ +function getBunPackagePath (packages, name, context) { + for (let current = context; current !== undefined; current = current.parent) { + const packagePath = `${current.packagePath}/${name}` + if (Object.hasOwn(packages, packagePath)) return packagePath + } + if (Object.hasOwn(packages, name)) return name } /** - * @param {Set} deps - * @param {NpmDependency} obj + * @param {Set} dependencies + * @param {string} packageLockPath */ -function collectDependencies (deps, obj) { - if (!obj.dependencies) return +function addNpmProductionDependencies (dependencies, packageLockPath) { + const { packages } = require(packageLockPath) + if (!packages) throw new Error('package-lock.json does not contain package metadata') - for (const [dep, dependency] of Object.entries(obj.dependencies)) { - const { resolved } = dependency - // Get the actual dependency name even when aliased in the package.json - const name = resolved ? resolved.split('/-', 1)[0].split('npmjs.org/').reverse()[0] : dep + for (const [packagePath, dependency] of Object.entries(packages)) { + if (!packagePath || dependency.link || (dependency.dev && !dependency.devOptional)) continue - deps.add(name) - - collectDependencies(deps, dependency) + dependencies.add(dependency.name ?? getNameFromPackagePath(packagePath)) } } -function collectFromTrees (trees, deps) { - for (const node of trees) { - if (typeof node?.name !== 'string') continue - - // Remove version from the package name (e.g. `@protobufjs/pool@1.1.0` -> `@protobufjs/pool`) - deps.add(normalizeDepName(node.name.slice(0, node.name.lastIndexOf('@')))) +/** + * @param {DependencyPattern[]} patterns + * @param {DependencyManifest} manifest + * @param {BunPackageContext} [context] + * @param {boolean} [includePeers] + */ +function addDependencyPatterns (patterns, manifest, context, includePeers = false) { + const optionalDependencies = manifest.optionalDependencies ?? {} - if (Array.isArray(node.children) && node.children.length) { - collectFromTrees(node.children, deps) + for (const [name, range] of Object.entries(manifest.dependencies ?? {})) { + if (!Object.hasOwn(optionalDependencies, name)) { + addDependencyPattern(patterns, name, range, context) } } -} - -function addVendoredDeps (deps) { - const vendoredDepsPath = join(__dirname, '..', '.github', 'vendored-dependencies.csv') - - // If the vendored dependencies file doesn't exist, skip - if (!existsSync(vendoredDepsPath)) { - return + for (const [name, range] of Object.entries(optionalDependencies)) { + addDependencyPattern(patterns, name, range, context) } - - const fs = require('node:fs') - const content = fs.readFileSync(vendoredDepsPath, 'utf8') - - for (const line of content.split('\n')) { - const trimmed = line.trim() - if (!trimmed) continue // Skip empty lines - - const columns = line.split(',') - const component = columns[0] - - // Strip quotes from the component name and add to deps - deps.add(normalizeDepName(component.replaceAll(/^"|"$/g, ''))) + if (includePeers) { + for (const [name, range] of Object.entries(manifest.peerDependencies ?? {})) { + addDependencyPattern(patterns, name, range, context, manifest.optionalPeers?.includes(name)) + } } } -function getAliasMap () { - const rootPackagePath = join(__dirname, '..', 'package.json') - const vendorPackagePath = join(__dirname, '..', 'vendor', 'package.json') - const map = new Map() +/** + * @param {DependencyPattern[]} patterns + * @param {string} name + * @param {string} range + * @param {BunPackageContext} [context] + * @param {boolean} [allowMissing] + */ +function addDependencyPattern (patterns, name, range, context, allowMissing = false) { + /** @type {DependencyPattern} */ + const pattern = { name, range } + if (context !== undefined) pattern.context = context + if (allowMissing) pattern.allowMissing = true + patterns.push(pattern) +} - collectAliasesFromPackageJson(rootPackagePath, map) - collectAliasesFromPackageJson(vendorPackagePath, map) +/** + * @param {string} name + * @param {string} range + */ +function normalizeDependencyName (name, range) { + if (!range.startsWith('npm:')) return name - return map + const target = range.slice('npm:'.length) + const versionIndex = target.lastIndexOf('@') + return versionIndex > 0 ? target.slice(0, versionIndex) : target } -function collectAliasesFromPackageJson (packagePath, map) { - if (!existsSync(packagePath)) return - - const packageJson = require(packagePath) - const deps = packageJson?.dependencies ?? {} - const optionalDeps = packageJson?.optionalDependencies ?? {} +/** + * @param {string} packagePath + */ +function getNameFromPackagePath (packagePath) { + const nodeModulesIndex = packagePath.lastIndexOf('node_modules/') + if (nodeModulesIndex === -1) throw new Error(`Cannot determine package name from ${packagePath}`) - collectAliasesFromDeps(deps, map) - collectAliasesFromDeps(optionalDeps, map) + return packagePath.slice(nodeModulesIndex + 'node_modules/'.length) } -function collectAliasesFromDeps (deps, map) { - for (const [alias, spec] of Object.entries(deps)) { - if (typeof spec !== 'string' || !spec.startsWith('npm:')) continue +/** + * @param {Set} components + * @param {string} filePath + * @param {boolean} hasHeader + */ +function addCsvComponents (components, filePath, hasHeader) { + const lines = readFileSync(filePath, 'utf8').split('\n') - const rawTarget = spec.slice('npm:'.length) - const atIndex = rawTarget.lastIndexOf('@') - const target = atIndex > 0 ? rawTarget.slice(0, atIndex) : rawTarget + for (let i = hasHeader ? 1 : 0; i < lines.length; i++) { + if (!lines[i].trim()) continue - if (target) { - map.set(alias, target) - } + components.add(lines[i].split(',', 1)[0].replaceAll(/^"|"$/g, '')) } } -function normalizeDepName (name) { - return aliasMap.get(name) ?? name -} - -function checkLicenses (typeDeps) { +/** + * @param {Set} dependencies + */ +function checkLicenses (dependencies) { const missing = [] const extraneous = [] - for (const dep of typeDeps) { - if (!licenses.has(dep)) { - missing.push(dep) + for (const dependency of dependencies) { + if (!licenses.has(dependency)) { + missing.push(dependency) } } - for (const dep of licenses) { - if (!typeDeps.has(dep)) { - extraneous.push(dep) + for (const dependency of licenses) { + if (!dependencies.has(dependency)) { + extraneous.push(dependency) } } diff --git a/scripts/check_licenses.spec.mjs b/scripts/check_licenses.spec.mjs new file mode 100644 index 0000000000..0de99d8283 --- /dev/null +++ b/scripts/check_licenses.spec.mjs @@ -0,0 +1,530 @@ +import assert from 'node:assert/strict' +import { spawnSync } from 'node:child_process' +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' + +import { afterEach, beforeEach, describe, it } from 'mocha' + +const repositoryDirectory = dirname(dirname(fileURLToPath(import.meta.url))) +const checkerPath = join(repositoryDirectory, 'scripts', 'check_licenses.js') +const expectedDependencies = [ + '@scope/child', + 'dev-optional', + 'manually-vendored', + 'optional-package', + 'platform-only', + 'regular', + 'right-source', + 'root-package', + 'source-package', + 'transitive-source', + 'unversioned-target', + 'vendor-peer', + 'vendor-regular', + 'vendor-source', +] +const validYarnLock = `# yarn lockfile v1 + +"@scope/child@^1.0.0": + version "1.0.0" + +"aliased@npm:source-package@^2.0.0": + version "2.0.0" + +development@^1.0.0: + version "1.0.0" + +optional-package@^1.0.0: + version "1.0.0" + dependencies: + "@scope/child" "^1.0.0" + +"overridden@npm:right-source@2.0.0": + version "2.0.0" + +"overridden@npm:wrong-source@1.0.0": + version "1.0.0" + +platform-only@^1.0.0: + version "1.0.0" + +regular@^1.0.0: + version "1.0.0" + dependencies: + "@scope/child" "^1.0.0" + transitive-alias "npm:transitive-source@^1.0.0" + optionalDependencies: + platform-only "^1.0.0" + +"transitive-alias@npm:transitive-source@^1.0.0": + version "1.0.0" + +"unversioned-alias@npm:unversioned-target": + version "1.0.0" +` +const validPackageLock = { + name: 'vendor', + lockfileVersion: 3, + packages: { + '': { + dependencies: { + 'vendor-alias': 'npm:vendor-source@1.0.0', + 'vendor-regular': '1.0.0', + }, + devDependencies: { + 'vendor-development': '1.0.0', + }, + }, + 'node_modules/dev-optional': { + devOptional: true, + version: '1.0.0', + }, + 'node_modules/vendor-alias': { + name: 'vendor-source', + version: '1.0.0', + }, + 'node_modules/vendor-development': { + dev: true, + version: '1.0.0', + }, + 'node_modules/vendor-link': { + link: true, + resolved: 'packages/vendor-target', + }, + 'node_modules/vendor-peer': { + peer: true, + version: '1.0.0', + }, + 'node_modules/vendor-regular': { + version: '1.0.0', + }, + }, +} +const validPackageJson = { + name: 'root-package', + dependencies: { + aliased: 'npm:source-package@^2.0.0', + overridden: 'npm:wrong-source@1.0.0', + regular: '^1.0.0', + 'unversioned-alias': 'npm:unversioned-target', + }, + devDependencies: { + development: '^1.0.0', + }, + optionalDependencies: { + 'optional-package': '^1.0.0', + overridden: 'npm:right-source@2.0.0', + }, +} +const validBunPackageJson = { + ...validPackageJson, + dependencies: { + ...validPackageJson.dependencies, + 'git-alias': 'github:example/source', + internal: 'workspace:*', + linked: 'link:../linked', + }, +} +const bunExpectedDependencies = [ + ...expectedDependencies, + 'git-source', + 'git-transitive', + 'link-transitive', + 'nested-only', + 'nested-version', + 'optional-peer', + 'peer-package', + 'top-level-only', + 'workspace-transitive', +] +const validBunLock = `{ + // Bun lockfiles are JSONC. + "lockfileVersion": 1, + "workspaces": { + "": { + "name": "root-package", + "dependencies": { + "aliased": "npm:source-package@^2.0.0", + "git-alias": "github:example/source", + "internal": "workspace:*", + "linked": "link:../linked", + "overridden": "npm:wrong-source@1.0.0", + "regular": "^1.0.0", + "unversioned-alias": "npm:unversioned-target", + }, + "devDependencies": { + "development": "^1.0.0", + }, + "optionalDependencies": { + "optional-package": "^1.0.0", + "overridden": "npm:right-source@2.0.0", + }, + }, + "packages/internal": { + "name": "internal", + "version": "1.0.0", + "dependencies": { + "workspace-transitive": "^1.0.0", + }, + }, + }, + "packages": { + "@scope/child": ["@scope/child@1.0.0", "", {}, "sha512-child"], + "aliased": ["source-package@2.0.0", "", { + "dependencies": { + "transitive-alias": "npm:transitive-source@^1.0.0", + }, + }, "sha512-source"], + "development": ["development@1.0.0", "", {}, "sha512-development"], + "git-alias": ["git-source@github:example/source#commit", { + "dependencies": { + "git-transitive": "^1.0.0", + }, + }, "example-source-commit"], + "git-transitive": ["git-transitive@1.0.0", "", {}, "sha512-git-transitive"], + "internal": ["internal@workspace:packages/internal"], + "linked": ["linked@link:../linked", { + "dependencies": { + "link-transitive": "^1.0.0", + }, + }], + "link-transitive": ["link-transitive@1.0.0", "", {}, "sha512-link-transitive"], + "nested-version": ["nested-version@1.0.0", "", { + "dependencies": { + "top-level-only": "^1.0.0", + }, + }, "sha512-nested"], + "optional-package": ["optional-package@1.0.0", "", { + "dependencies": { + "@scope/child": "^1.0.0", + "nested-version": "^1.0.0", + }, + }, "sha512-optional"], + "optional-peer": ["optional-peer@1.0.0", "", {}, "sha512-optional-peer"], + "overridden": ["right-source@2.0.0", "", {}, "sha512-overridden"], + "peer-package": ["peer-package@1.0.0", "", {}, "sha512-peer"], + "platform-only": ["platform-only@1.0.0", "", {}, "sha512-platform"], + "regular": ["regular@1.0.0", "", { + "dependencies": { + "@scope/child": "^1.0.0", + "nested-version": "^2.0.0", + }, + "optionalDependencies": { + "platform-only": "^1.0.0", + }, + "peerDependencies": { + "missing-optional-peer": "^1.0.0", + "optional-peer": "^1.0.0", + "peer-package": "^1.0.0", + }, + "optionalPeers": [ + "missing-optional-peer", + "optional-peer", + ], + }, "sha512-regular"], + "regular/nested-version": ["nested-version@2.0.0", "", { + "dependencies": { + "nested-only": "^1.0.0", + }, + }, "sha512-nested"], + "nested-only": ["nested-only@1.0.0", "", {}, "sha512-nested-only"], + "top-level-only": ["top-level-only@1.0.0", "", {}, "sha512-top-level-only"], + "transitive-alias": ["transitive-source@1.0.0", "", {}, "sha512-transitive"], + "unversioned-alias": ["unversioned-target@1.0.0", "", {}, "sha512-unversioned"], + "workspace-transitive": ["workspace-transitive@1.0.0", "", {}, "sha512-workspace-transitive"], + }, +}` + +/** + * @typedef {object} FixtureOptions + * @property {string} [bunLock] + * @property {boolean} [includeVendoredDependencies] + * @property {string[]} [licenses] + * @property {object} [packageLock] + * @property {object} [packageJson] + * @property {string} [yarnLock] + */ + +/** + * @param {string[]} dependencies + * @param {string} excluded + */ +function withoutDependency (dependencies, excluded) { + const remaining = [] + for (const dependency of dependencies) { + if (dependency !== excluded) remaining.push(dependency) + } + return remaining +} + +describe('check_licenses', () => { + /** @type {string} */ + let fixtureDirectory + + beforeEach(() => { + fixtureDirectory = mkdtempSync(join(tmpdir(), 'dd-trace-license-check-')) + }) + + afterEach(() => { + rmSync(fixtureDirectory, { force: true, recursive: true }) + }) + + /** + * @param {FixtureOptions} [options] + */ + function writeFixture ({ + bunLock, + includeVendoredDependencies = true, + licenses = expectedDependencies, + packageLock = validPackageLock, + packageJson = validPackageJson, + yarnLock = validYarnLock, + } = {}) { + const vendorDirectory = join(fixtureDirectory, 'vendor') + + mkdirSync(join(fixtureDirectory, '.github'), { recursive: true }) + mkdirSync(vendorDirectory, { recursive: true }) + if (includeVendoredDependencies) { + writeFileSync(join(fixtureDirectory, '.github', 'vendored-dependencies.csv'), '"manually-vendored","MIT"\n') + } + const licenseRows = ['"Component","License"'] + for (const name of licenses) { + licenseRows.push(`"${name}","MIT"`) + } + licenseRows.push('') + writeFileSync(join(fixtureDirectory, 'LICENSE-3rdparty.csv'), licenseRows.join('\n')) + writeFileSync(join(fixtureDirectory, 'package.json'), JSON.stringify(packageJson)) + writeFileSync(join(fixtureDirectory, 'yarn.lock'), yarnLock) + if (bunLock !== undefined) writeFileSync(join(fixtureDirectory, 'bun.lock'), bunLock) + writeFileSync(join(vendorDirectory, 'package-lock.json'), JSON.stringify(packageLock)) + writeFileSync(join(vendorDirectory, 'package.json'), JSON.stringify({ + dependencies: { + 'vendor-alias': 'npm:vendor-source@1.0.0', + 'vendor-regular': '1.0.0', + }, + })) + } + + function runChecker () { + return spawnSync(process.execPath, [checkerPath], { + cwd: fixtureDirectory, + encoding: 'utf8', + }) + } + + it('checks every production dependency from both lockfile formats', () => { + writeFixture() + + const result = runChecker() + + assert.strictEqual(result.status, 0, result.stderr) + }) + + it('prefers bun.lock when both root lockfile formats exist', () => { + writeFixture({ + bunLock: validBunLock, + licenses: bunExpectedDependencies, + packageJson: validBunPackageJson, + yarnLock: 'not a yarn lockfile', + }) + + const result = runChecker() + + assert.strictEqual(result.status, 0, result.stderr) + }) + + it('accepts every supported bun lockfile version', () => { + for (const version of [0, 2]) { + writeFixture({ + bunLock: validBunLock.replace('"lockfileVersion": 1', `"lockfileVersion": ${version}`), + licenses: bunExpectedDependencies, + packageJson: validBunPackageJson, + }) + + const result = runChecker() + + assert.strictEqual(result.status, 0, result.stderr) + } + }) + + it('reports missing and extraneous licenses', () => { + writeFixture({ + licenses: [...withoutDependency(expectedDependencies, 'regular'), 'extraneous'], + }) + + const result = runChecker() + + assert.strictEqual(result.status, 1) + assert.match(result.stderr, /Missing 3rd-party license for regular\./) + assert.match(result.stderr, /Extraneous 3rd-party license for extraneous\./) + }) + + it('checks repositories without manually vendored dependencies', () => { + writeFixture({ + includeVendoredDependencies: false, + licenses: withoutDependency(expectedDependencies, 'manually-vendored'), + }) + + const result = runChecker() + + assert.strictEqual(result.status, 0, result.stderr) + }) + + it('rejects an incomplete yarn lockfile', () => { + writeFixture({ + yarnLock: validYarnLock.replace(`optional-package@^1.0.0: + version "1.0.0" + dependencies: + "@scope/child" "^1.0.0" + +`, ''), + }) + + const result = runChecker() + + assert.notStrictEqual(result.status, 0) + assert.match(result.stderr, /optional-package@\^1\.0\.0/) + }) + + it('rejects a conflicted yarn lockfile', () => { + writeFixture({ + yarnLock: `<<<<<<< HEAD +${validYarnLock}======= +${validYarnLock}>>>>>>> branch +`, + }) + + const result = runChecker() + + assert.notStrictEqual(result.status, 0) + assert.match(result.stderr, /Cannot parse yarn\.lock: merge/) + }) + + it('rejects an invalid bun lockfile', () => { + writeFixture({ + bunLock: validBunLock.replace('"packages": {', '"packages": ['), + licenses: bunExpectedDependencies, + packageJson: validBunPackageJson, + }) + + const result = runChecker() + + assert.notStrictEqual(result.status, 0) + assert.match(result.stderr, /Cannot parse bun\.lock/) + }) + + it('rejects invalid bun lockfile metadata', () => { + const invalidLocks = [ + ['null', /bun\.lock does not contain an object/], + ['{"lockfileVersion":"1","packages":{},"workspaces":{"":{}}}', /Unsupported bun\.lock version: 1/], + ['{"lockfileVersion":-1,"packages":{},"workspaces":{"":{}}}', /Unsupported bun\.lock version: -1/], + ['{"lockfileVersion":3,"packages":{},"workspaces":{"":{}}}', /Unsupported bun\.lock version: 3/], + ['{"lockfileVersion":1,"workspaces":{"":{}}}', /bun\.lock does not contain package metadata/], + ['{"lockfileVersion":1,"packages":[],"workspaces":{"":{}}}', /bun\.lock does not contain package metadata/], + ['{"lockfileVersion":1,"packages":{}}', /bun\.lock does not contain a root workspace/], + ['{"lockfileVersion":1,"packages":{},"workspaces":[]}', /bun\.lock does not contain a root workspace/], + ['{"lockfileVersion":1,"packages":{},"workspaces":{"other":{}}}', /bun\.lock does not contain a root workspace/], + ] + + for (const [bunLock, expectedError] of invalidLocks) { + writeFixture({ bunLock }) + + const result = runChecker() + + assert.notStrictEqual(result.status, 0) + assert.match(result.stderr, expectedError) + } + }) + + it('rejects invalid bun package entries', () => { + const invalidEntries = [ + [{ broken: {} }, /Invalid bun\.lock entry for broken/], + [{ broken: ['broken'] }, /Invalid bun\.lock resolution for broken/], + [{ broken: ['broken@workspace:packages/broken'] }, /Missing bun\.lock workspace for broken/], + [{ + broken: ['broken@1.0.0', '', { + peerDependencies: { + peer: '^1.0.0', + }, + }, 'sha512-broken'], + }, /Missing bun\.lock entry for peer/], + ] + + for (const [packages, expectedError] of invalidEntries) { + writeFixture({ + bunLock: JSON.stringify({ + lockfileVersion: 1, + packages, + workspaces: { + '': { + dependencies: { + broken: '1.0.0', + }, + }, + }, + }), + packageJson: { + name: 'root-package', + dependencies: { + broken: '1.0.0', + }, + }, + }) + + const result = runChecker() + + assert.notStrictEqual(result.status, 0) + assert.match(result.stderr, expectedError) + } + }) + + it('rejects an incomplete bun lockfile', () => { + writeFixture({ + bunLock: validBunLock.replace( + ' "workspace-transitive": ["workspace-transitive@1.0.0", "", {}, "sha512-workspace-transitive"],\n', + '' + ), + licenses: bunExpectedDependencies, + packageJson: validBunPackageJson, + }) + + const result = runChecker() + + assert.notStrictEqual(result.status, 0) + assert.match(result.stderr, /Missing bun\.lock entry for workspace-transitive/) + }) + + it('rejects a package lockfile without package metadata', () => { + writeFixture({ + packageLock: { + lockfileVersion: 1, + }, + }) + + const result = runChecker() + + assert.notStrictEqual(result.status, 0) + assert.match(result.stderr, /package-lock\.json does not contain package metadata/) + }) + + it('rejects unnamed packages outside node_modules', () => { + writeFixture({ + packageLock: { + lockfileVersion: 3, + packages: { + '': {}, + 'packages/unnamed': { + version: '1.0.0', + }, + }, + }, + }) + + const result = runChecker() + + assert.notStrictEqual(result.status, 0) + assert.match(result.stderr, /Cannot determine package name from packages\/unnamed/) + }) +}) diff --git a/scripts/generate-config-types.js b/scripts/generate-config-types.js index 6376b8c86e..235a07a1f6 100644 --- a/scripts/generate-config-types.js +++ b/scripts/generate-config-types.js @@ -260,7 +260,7 @@ function generateEnvVarConfigTypes (supportedConfigurations) { } } - const body = [...envVarTypes.entries()] + const body = [...envVarTypes] .sort(([left], [right]) => left.localeCompare(right)) .map(([name, type]) => ` ${renderPropertyName(name)}: ${type};`) .join('\n') diff --git a/scripts/helpers/test-file-index.js b/scripts/helpers/test-file-index.js new file mode 100644 index 0000000000..b5801dd19f --- /dev/null +++ b/scripts/helpers/test-file-index.js @@ -0,0 +1,288 @@ +'use strict' + +const fs = require('node:fs') +const path = require('node:path') + +const { Glob } = require('glob') +const { Minimatch } = require('minimatch') + +const TEST_FILE_GLOB = '**/*.@(spec|test).@(js|mjs|cjs)' + +// Case-insensitive because `glob` matches case-insensitively on darwin and win32. Indexing a +// `FOO.SPEC.JS` that a case-sensitive platform would reject is harmless: the pattern matcher +// applies that platform's own rules and drops it again. +const TEST_FILE_NAME = /\.(?:spec|test)\.(?:js|mjs|cjs)$/i + +// Any character that can start a wildcard, brace, class, or extglob. Over-reporting only shortens +// the literal prefix, which widens the candidate set and can never drop a match. +const PATTERN_MAGIC = /[*?[\]{}()!+@\\]/ + +// `glob` walks `..` on the real filesystem before matching. The index has no tree to walk, so it +// would answer such a pattern with too few files and report exercised specs as unexercised. +const PARENT_SEGMENT = /(?:^|\/)\.\.(?:\/|$)/ + +/** + * @param {string} repoRoot + * @returns {string[]} + */ +function walkTestFiles (repoRoot) { + /** @type {string[]} */ + const found = [] + /** @type {string[]} */ + const pending = [''] + + while (pending.length) { + const relativeDir = pending.pop() + + /** @type {import('node:fs').Dirent[]} */ + let entries + try { + entries = fs.readdirSync(path.join(repoRoot, relativeDir), { withFileTypes: true }) + } catch (error) { + // `glob` skips unreadable directories without a word. Staying silent here would shrink the + // index instead, and an unexercised spec below that directory would pass the check unseen. + process.stderr.write(`test-file-index: skipped unreadable ${relativeDir || '.'} (${error.code})\n`) + continue + } + + for (const entry of entries) { + const { name } = entry + + // `glob` runs with `dot: false`, so a dot entry can never satisfy TEST_FILE_GLOB. Every + // consumer of this index also ignores `node_modules`, so descending into it is dead work. + if (name.startsWith('.') || name === 'node_modules') continue + + const relativePath = relativeDir ? `${relativeDir}/${name}` : name + + if (entry.isDirectory()) { + pending.push(relativePath) + } else if (TEST_FILE_NAME.test(name)) { + // Symlinks are kept: `nodir: true` filters directories, not links to files. + found.push(relativePath) + } + } + } + + return found +} + +/** + * Drops `.` and empty path segments the way `glob` does while building its pattern. `minimatch` + * on a bare string keeps them and would then compare one segment too many. + * + * @param {string} pattern + * @returns {string} + */ +function normalizePattern (pattern) { + if (!pattern.includes('./') && !pattern.includes('//')) return pattern + + const segments = pattern.split('/') + const kept = [] + + for (let i = 0; i < segments.length; i++) { + const segment = segments[i] + // A leading empty segment marks an absolute pattern and a trailing one a directory-only + // pattern; both change what matches, so only interior blanks are collapsed. + if (segment === '.' || (segment === '' && i !== 0 && i !== segments.length - 1)) continue + kept.push(segment) + } + + return kept.join('/') +} + +/** + * Longest leading directory path of `pattern` that contains no wildcard. + * + * @param {string} pattern + * @returns {string} + */ +function literalPrefix (pattern) { + const magic = pattern.search(PATTERN_MAGIC) + const literal = magic === -1 ? pattern : pattern.slice(0, magic) + const lastSlash = literal.lastIndexOf('/') + return lastSlash === -1 ? '' : literal.slice(0, lastSlash + 1) +} + +/** + * In-memory stand-in for repeated `globSync` walks over the repository's test files. + * + * The repository is traversed once; every later pattern is answered from the resulting list. Only + * paths named like a test file are indexed, so a pattern that also selects non-test files reports + * just the test-file subset. + * + * Results have to stay identical to the `globSync` walk they replace; `test-file-index.spec.js` + * pins that against `glob` itself, including the repository's own pattern corpus. + */ +class TestFileIndex { + /** @type {string[]} */ + files + + /** @type {boolean} */ + #nocase + + /** @type {NodeJS.Platform} */ + #platform + + /** @type {Map>} */ + #matchers = new Map() + + /** @type {Map>} */ + #ignoreMatchers = new Map() + + /** @type {Map} */ + #scopes = new Map() + + /** @type {Map} */ + #candidates = new Map() + + /** + * @param {string[]} files Sorted repository-relative POSIX paths. + * @param {{ nocase: boolean, platform: NodeJS.Platform }} options + */ + constructor (files, { nocase, platform }) { + this.files = files + this.#nocase = nocase + this.#platform = platform + } + + /** + * A literal path segment is compared case-sensitively on every platform, which is what `glob` + * does on Linux. On a case-insensitive filesystem `glob` instead resolves `foo/BAR.spec.js` to + * the differently-spelled file on disk; reproducing that would make the result depend on the + * developer's machine, so the index keeps the Linux rule everywhere. + * + * @param {string} pattern + * @param {string[]} [ignoreGlobs] + * @throws {Error} If `pattern` contains a `..` segment, which the index cannot resolve. + * @returns {string[]} Matching paths, in the index's sort order. + */ + match (pattern, ignoreGlobs = []) { + if (PARENT_SEGMENT.test(pattern)) { + throw new Error(`test-file-index cannot resolve the '..' segment in pattern '${pattern}'`) + } + + const normalized = normalizePattern(pattern) + const scope = this.#scope(ignoreGlobs) + const matcher = this.#matcher(normalized) + + /** @type {string[]} */ + const matched = [] + for (const file of this.#candidatesFor(scope, ignoreGlobs, normalized)) { + // eslint-disable-next-line unicorn/prefer-regexp-test -- Minimatch#match, not String#match. + if (matcher.match(file)) matched.push(file) + } + + return matched + } + + /** + * @param {string[]} ignoreGlobs + * @returns {{ files: string[], folded: string[] }} + */ + #scope (ignoreGlobs) { + const key = ignoreGlobs.join('\0') + + let scope = this.#scopes.get(key) + if (scope === undefined) { + const matchers = ignoreGlobs.map(glob => this.#ignoreMatcher(glob)) + const files = matchers.length === 0 + ? this.files + : this.files.filter(file => !matchers.some(matcher => matcher.match(file) || matcher.match(`${file}/`))) + + scope = { files, folded: this.#nocase ? files.map(file => file.toLowerCase()) : files } + this.#scopes.set(key, scope) + } + + return scope + } + + /** + * @param {{ files: string[], folded: string[] }} scope + * @param {string[]} ignoreGlobs + * @param {string} pattern + * @returns {string[]} + */ + #candidatesFor (scope, ignoreGlobs, pattern) { + const prefix = literalPrefix(pattern) + if (prefix === '') return scope.files + + const key = `${ignoreGlobs.join('\0')}\0${prefix}` + + let candidates = this.#candidates.get(key) + if (candidates === undefined) { + const wanted = this.#nocase ? prefix.toLowerCase() : prefix + + candidates = [] + for (let i = 0; i < scope.files.length; i++) { + if (scope.folded[i].startsWith(wanted)) candidates.push(scope.files[i]) + } + + this.#candidates.set(key, candidates) + } + + return candidates + } + + /** + * @param {string} pattern + * @returns {InstanceType} + */ + #matcher (pattern) { + let matcher = this.#matchers.get(pattern) + if (matcher === undefined) { + matcher = new Minimatch(pattern, { + dot: false, + nocase: this.#nocase, + nocaseMagicOnly: this.#platform === 'darwin' || this.#platform === 'win32', + nocomment: true, + nonegate: true, + optimizationLevel: 2, + platform: this.#platform, + windowsPathsNoEscape: true, + }) + this.#matchers.set(pattern, matcher) + } + + return matcher + } + + /** + * @param {string} glob + * @returns {InstanceType} + */ + #ignoreMatcher (glob) { + let matcher = this.#ignoreMatchers.get(glob) + if (matcher === undefined) { + // Mirrors the options `glob`'s Ignore class builds; notably `dot: true`, so `**/.git/**` + // still matches paths that the main pattern would skip. + matcher = new Minimatch(glob, { + dot: true, + nocase: this.#nocase, + nocomment: true, + nonegate: true, + optimizationLevel: 2, + platform: this.#platform, + }) + this.#ignoreMatchers.set(glob, matcher) + } + + return matcher + } +} + +/** + * @param {string} repoRoot + * @returns {TestFileIndex} + */ +function createTestFileIndex (repoRoot) { + // `glob` decides case sensitivity from the platform's filesystem; read it back rather than + // re-deriving it here, so the index cannot drift from the walker it replaces. + const probe = new Glob(TEST_FILE_GLOB, { cwd: repoRoot, windowsPathsNoEscape: true }) + + const files = walkTestFiles(repoRoot) + files.sort((a, b) => a.localeCompare(b, 'en')) + + return new TestFileIndex(files, { nocase: probe.nocase, platform: probe.platform }) +} + +module.exports = { TEST_FILE_GLOB, TestFileIndex, createTestFileIndex } diff --git a/scripts/helpers/test-file-index.spec.js b/scripts/helpers/test-file-index.spec.js new file mode 100644 index 0000000000..339e3a8be6 --- /dev/null +++ b/scripts/helpers/test-file-index.spec.js @@ -0,0 +1,366 @@ +'use strict' + +const assert = require('node:assert/strict') +const fs = require('node:fs') +const os = require('node:os') +const path = require('node:path') + +const { globSync } = require('glob') + +const { TEST_FILE_GLOB, TestFileIndex, createTestFileIndex } = require('./test-file-index') + +const REPO_ROOT = path.resolve(__dirname, '..', '..') + +const DEFAULT_IGNORE_GLOBS = [ + '**/node_modules/**', + '**/coverage/**', + '**/.git/**', + '**/.nyc_output/**', + '**/.junit-tmp/**', + 'vendor/dist/**', +] +const NODE_MODULES_ONLY = ['**/node_modules/**'] + +const FIXTURE_FILES = [ + 'a.spec.js', + 'b.test.js', + 'c.spec.mjs', + 'd.test.mjs', + 'e.spec.cjs', + 'f.test.cjs', + 'plain.js', + 'notspec.js', + 'wrong.spec.jsx', + 'wrong.spec.ts', + '.hidden.spec.js', + '.hiddendir/inner.spec.js', + 'node_modules/pkg/dep.spec.js', + 'nested/node_modules/deep/dep.spec.js', + 'packages/alpha/test/one.spec.js', + 'packages/alpha/test/deep/two.spec.js', + 'packages/beta/test/three.spec.js', + 'packages/beta/test/integration-test/four.spec.js', + 'packages/datadog-plugin-http/test/index.spec.js', + 'packages/datadog-plugin-redis/test/index.spec.js', + 'vendor/dist/bundled.spec.js', + 'coverage/report.spec.js', + 'sub/dir/with.dots.in.name.spec.js', + 'sub/dir/UPPER.SPEC.JS', + 'integration-tests/one.spec.js', + 'integration-tests/nested/two.spec.js', +] + +const FIXTURE_PATTERNS = [ + TEST_FILE_GLOB, + '**/*.spec.js', + '**/*.test.js', + '*.spec.js', + 'packages/*/test/*.spec.js', + 'packages/*/test/**/*.spec.js', + 'packages/**/*.spec.js', + 'packages/alpha/test/one.spec.js', + 'packages/alpha/test/**/*.@(spec|test).@(js|mjs|cjs)', + 'packages/datadog-plugin-@(http|redis)/test/**/*.spec.js', + 'packages/datadog-plugin-@(http)/test/**/*.spec.js', + 'packages/datadog-plugin-*/test/**/*.spec.js', + 'packages/{alpha,beta}/test/**/*.spec.js', + 'packages/{alpha}/test/**/*.spec.js', + 'integration-tests/*.spec.js', + 'integration-tests/**/*.spec.js', + 'sub/dir/*.spec.js', + 'sub/dir/with.dots.in.name.spec.js', + 'vendor/dist/*.spec.js', + 'coverage/*.spec.js', + '**/node_modules/**/*.spec.js', + '.hiddendir/*.spec.js', + 'does/not/exist/*.spec.js', + 'packages/*/test/**/**.spec.js', + '**/*.@(spec|test).@(js|mjs|cjs)', + '**/*.{spec,test}.js', + 'packages/?????/test/*.spec.js', + 'packages/[ab]*/test/*.spec.js', + './packages/alpha/test/*.spec.js', + './/packages/alpha/test/*.spec.js', + '././packages/alpha/test/*.spec.js', + 'packages/./alpha/test/*.spec.js', + 'packages/alpha/test/', + '', +] + +/** + * @param {string} root + * @param {string[]} files + */ +function writeFixture (root, files) { + for (const file of files) { + const full = path.join(root, file) + fs.mkdirSync(path.dirname(full), { recursive: true }) + fs.writeFileSync(full, '') + } +} + +/** + * The index deliberately holds only test-named files outside `node_modules`. Expectations are + * intersected with that same set so a pattern selecting other files is compared fairly. + * + * @param {string} root + * @returns {Set} + */ +function indexableFiles (root) { + return new Set(globSync(TEST_FILE_GLOB, { + cwd: root, + nodir: true, + windowsPathsNoEscape: true, + ignore: NODE_MODULES_ONLY, + })) +} + +/** + * @param {string} root + * @param {Set} indexable + * @param {string} pattern + * @param {string[]} [ignore] + * @returns {string[]} + */ +function globExpectation (root, indexable, pattern, ignore) { + return globSync(pattern, { cwd: root, nodir: true, windowsPathsNoEscape: true, ignore }) + .filter(file => indexable.has(file)) + .sort((a, b) => a.localeCompare(b, 'en')) +} + +/** + * @param {Record} scripts + * @returns {string[]} + */ +function globTokensFromScripts (scripts) { + const tokens = new Set() + + for (const command of Object.values(scripts)) { + if (typeof command !== 'string') continue + + for (const raw of command.split(/\s+/)) { + const unquoted = raw.replaceAll(/^["']+|["']+$/g, '') + if (!unquoted.includes('/') || !/[*?[\]{}()]/.test(unquoted)) continue + + tokens.add(unquoted.replaceAll(/\$\{[^}]+\}/g, '*').replaceAll(/\$[A-Za-z_][A-Za-z0-9_]*/g, '*')) + } + } + + return [...tokens] +} + +describe('test-file-index', () => { + /** @type {string} */ + let root + /** @type {ReturnType} */ + let index + /** @type {Set} */ + let indexable + + before(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-test-file-index-')) + writeFixture(root, FIXTURE_FILES) + fs.symlinkSync(path.join(root, 'packages', 'alpha', 'test'), path.join(root, 'linked-tests'), 'dir') + + index = createTestFileIndex(root) + indexable = indexableFiles(root) + }) + + after(() => { + fs.rmSync(root, { recursive: true, force: true }) + }) + + describe('inventory', () => { + it('indexes every supported test-file extension', () => { + assert.deepStrictEqual( + index.files.filter(file => !file.includes('/')), + ['a.spec.js', 'b.test.js', 'c.spec.mjs', 'd.test.mjs', 'e.spec.cjs', 'f.test.cjs'] + ) + }) + + it('excludes files that are not named like a test', () => { + for (const file of ['plain.js', 'notspec.js', 'wrong.spec.jsx', 'wrong.spec.ts']) { + assert.ok(!index.files.includes(file), `${file} should not be indexed`) + } + }) + + it('excludes node_modules at any depth', () => { + for (const file of index.files) { + assert.ok(!file.includes('node_modules/'), `${file} should not be indexed`) + } + }) + + it('excludes dot files and dot directories', () => { + for (const file of index.files) { + assert.ok(!file.split('/').some(segment => segment.startsWith('.')), `${file} should not be indexed`) + } + }) + + it('does not descend into symlinked directories', () => { + assert.ok(!index.files.some(file => file.startsWith('linked-tests/'))) + }) + + it('returns sorted repository-relative POSIX paths', () => { + assert.deepStrictEqual(index.files, [...index.files].sort((a, b) => a.localeCompare(b, 'en'))) + assert.ok(index.files.every(file => !file.startsWith('/') && !file.includes('\\'))) + }) + + it('indexes an empty directory as no files', () => { + const empty = fs.mkdtempSync(path.join(os.tmpdir(), 'dd-test-file-index-empty-')) + try { + assert.deepStrictEqual(createTestFileIndex(empty).files, []) + } finally { + fs.rmSync(empty, { recursive: true, force: true }) + } + }) + + it('warns instead of silently shrinking the index when a directory cannot be read', () => { + const { write } = process.stderr + /** @type {string[]} */ + const warnings = [] + process.stderr.write = chunk => warnings.push(String(chunk)) + + try { + assert.deepStrictEqual(createTestFileIndex(path.join(os.tmpdir(), 'dd-test-file-index-missing')).files, []) + } finally { + process.stderr.write = write + } + + assert.deepStrictEqual(warnings, ['test-file-index: skipped unreadable . (ENOENT)\n']) + }) + }) + + describe('matching', () => { + for (const pattern of FIXTURE_PATTERNS) { + it(`matches globSync for ${pattern}`, () => { + assert.deepStrictEqual(index.match(pattern), globExpectation(root, indexable, pattern)) + }) + } + + for (const pattern of FIXTURE_PATTERNS) { + it(`matches globSync for ${pattern} under the default ignore list`, () => { + assert.deepStrictEqual( + index.match(pattern, DEFAULT_IGNORE_GLOBS), + globExpectation(root, indexable, pattern, DEFAULT_IGNORE_GLOBS) + ) + }) + } + + it('matches globSync for every glob token in the real package.json', () => { + const { scripts } = require(path.join(REPO_ROOT, 'package.json')) + const repoIndex = createTestFileIndex(REPO_ROOT) + const repoIndexable = indexableFiles(REPO_ROOT) + const patterns = globTokensFromScripts(scripts) + + assert.ok(patterns.length > 20, `expected a representative corpus, got ${patterns.length}`) + + for (const pattern of patterns) { + assert.deepStrictEqual( + repoIndex.match(pattern, DEFAULT_IGNORE_GLOBS), + globExpectation(REPO_ROOT, repoIndexable, pattern, DEFAULT_IGNORE_GLOBS), + `pattern ${pattern} diverged from globSync` + ) + } + }) + + it('matches globSync for the repository-wide test glob', () => { + const repoIndex = createTestFileIndex(REPO_ROOT) + + assert.deepStrictEqual( + repoIndex.match(TEST_FILE_GLOB, DEFAULT_IGNORE_GLOBS), + globSync(TEST_FILE_GLOB, { + cwd: REPO_ROOT, + nodir: true, + windowsPathsNoEscape: true, + ignore: DEFAULT_IGNORE_GLOBS, + }).sort((a, b) => a.localeCompare(b, 'en')) + ) + }) + }) + + describe('ignore lists', () => { + it('drops paths excluded by the default ignore list', () => { + const matched = index.match('**/*.spec.js', DEFAULT_IGNORE_GLOBS) + + assert.ok(!matched.includes('vendor/dist/bundled.spec.js')) + assert.ok(!matched.includes('coverage/report.spec.js')) + assert.ok(matched.includes('packages/alpha/test/one.spec.js')) + }) + + it('keeps paths a narrower ignore list allows', () => { + const matched = index.match('**/*.spec.js', NODE_MODULES_ONLY) + + assert.ok(matched.includes('vendor/dist/bundled.spec.js')) + assert.ok(matched.includes('coverage/report.spec.js')) + }) + + it('treats an empty ignore list as no filtering', () => { + assert.deepStrictEqual(index.match('**/*.spec.js'), index.match('**/*.spec.js', [])) + }) + + it('keeps results for the same pattern independent per ignore list', () => { + const wide = index.match('**/*.spec.js', NODE_MODULES_ONLY) + const narrow = index.match('**/*.spec.js', DEFAULT_IGNORE_GLOBS) + + assert.ok(wide.length > narrow.length) + assert.deepStrictEqual(index.match('**/*.spec.js', NODE_MODULES_ONLY), wide) + }) + }) + + describe('platform semantics', () => { + // `glob` reads case sensitivity off the filesystem, so a run on one platform never exercises + // the other's rules. Both are pinned here, independent of the host this suite runs on. + const files = ['sub/dir/lower.spec.js', 'sub/dir/UPPER.SPEC.JS'] + const platforms = [ + { platform: 'darwin', nocase: true }, + { platform: 'win32', nocase: true }, + { platform: 'linux', nocase: false }, + ] + const caseInsensitivePlatforms = platforms.filter(entry => entry.nocase) + + it('matches a wildcard segment case-insensitively where the filesystem is', () => { + for (const { platform } of caseInsensitivePlatforms) { + assert.deepStrictEqual(new TestFileIndex(files, { nocase: true, platform }).match('sub/dir/*.spec.js'), + files, `platform ${platform}`) + } + }) + + it('matches a wildcard segment case-sensitively on linux', () => { + const index = new TestFileIndex(files, { nocase: false, platform: 'linux' }) + assert.deepStrictEqual(index.match('sub/dir/*.spec.js'), ['sub/dir/lower.spec.js']) + }) + + it('rejects a literal segment whose case differs, on every platform', () => { + for (const { platform, nocase } of platforms) { + assert.deepStrictEqual(new TestFileIndex(files, { nocase, platform }).match('sub/dir/upper.spec.js'), + [], `platform ${platform}`) + } + }) + + it('throws on a pattern it cannot resolve rather than matching too few files', () => { + const index = new TestFileIndex(files, { nocase: false, platform: 'linux' }) + const expected = { message: /cannot resolve the '\.\.' segment/ } + + assert.throws(() => index.match('packages/../sub/dir/lower.spec.js'), expected) + assert.throws(() => index.match('../sub/*.spec.js'), expected) + assert.throws(() => index.match('sub/..'), expected) + assert.deepStrictEqual(index.match('sub/dir/with..dots.spec.js'), []) + }) + }) + + describe('repeated queries', () => { + it('returns equal results when a pattern is matched twice', () => { + for (const pattern of ['**/*.spec.js', 'packages/*/test/**/*.spec.js', 'does/not/exist/*.spec.js']) { + assert.deepStrictEqual(index.match(pattern, DEFAULT_IGNORE_GLOBS), index.match(pattern, DEFAULT_IGNORE_GLOBS)) + } + }) + + it('does not leak matches between patterns sharing a literal prefix', () => { + const one = index.match('packages/alpha/test/one.spec.js') + const deep = index.match('packages/alpha/test/deep/*.spec.js') + + assert.deepStrictEqual(one, ['packages/alpha/test/one.spec.js']) + assert.deepStrictEqual(deep, ['packages/alpha/test/deep/two.spec.js']) + }) + }) +}) diff --git a/scripts/release/changelog.spec.js b/scripts/release/changelog.spec.js index 75934f7b6e..cc6c7376cb 100644 --- a/scripts/release/changelog.spec.js +++ b/scripts/release/changelog.spec.js @@ -95,7 +95,7 @@ describe('release changelog', () => { '- **AppSec:** Avoid the possibility of sensitive data going to the telemetry logs backend ' + `via WAF strings ${prLink(3884)}`, '- **AppSec:** Treat cleared shared memory as no-config rather than an error in AppSec helper ' + - `${prLink(3876)}`, + prLink(3876), `- **General:** Encoder JSON number type fix ${prLink(38_799)}`, '- **LLM Observability:** Revert "Fan array-valued user tags out into one wire entry per element ' + `(${prLink(8689)})" ${prLink(8790)}`, diff --git a/scripts/release/proposal.js b/scripts/release/proposal.js index 6cef054ca9..da73e83c82 100644 --- a/scripts/release/proposal.js +++ b/scripts/release/proposal.js @@ -112,10 +112,11 @@ try { // the capped commits actually included in the proposal, not deferred ones. // Excludes changes that are listed in the dedicated breaking changes section. const notesShas = capture(`${notesDiffCmd} --format=sha --reverse v${releaseLine}.x ${upperBoundRef}`) - .split('\n').filter(Boolean) + .split('\n') const contributorBySha = getContributorsBySha(`v${releaseLine}.x`, upperBoundSha) const notesEntries = [] for (const sha of notesShas) { + if (!sha) continue notesEntries.push({ sha, subject: capture(`git show -s --format=%s ${sha}`), diff --git a/scripts/verify-exercised-tests.js b/scripts/verify-exercised-tests.js index 0b72d6e440..64fac64292 100644 --- a/scripts/verify-exercised-tests.js +++ b/scripts/verify-exercised-tests.js @@ -6,8 +6,13 @@ const path = require('node:path') const { globSync } = require('glob') const YAML = require('yaml') +const { TEST_FILE_GLOB, createTestFileIndex } = require('./helpers/test-file-index') + +/** @typedef {import('./helpers/test-file-index').TestFileIndex} TestFileIndex */ + +const NODE_MODULES_IGNORE_GLOBS = ['**/node_modules/**'] const DEFAULT_IGNORE_GLOBS = [ - '**/node_modules/**', + ...NODE_MODULES_IGNORE_GLOBS, '**/coverage/**', '**/.git/**', '**/.nyc_output/**', @@ -15,12 +20,6 @@ const DEFAULT_IGNORE_GLOBS = [ 'vendor/dist/**', ] -const GLOB_CACHE_MAX = 2000 -/** @type {Map} */ -const globCache = new Map() -let globCacheHits = 0 -let globCacheMisses = 0 - const COVERAGE_ACTIONS = new Set([ './.github/actions/coverage', './.github/actions/upload-coverage-artifact', @@ -30,36 +29,6 @@ const COVERAGE_COLLECTORS = new Set([ 'scripts/c8-ci.js', ]) -/** - * @param {string} pattern - * @param {{cwd: string, nodir: boolean, windowsPathsNoEscape: boolean, ignore?: string[]}} opts - * @returns {string[]} - */ -function globSyncCached (pattern, opts) { - const ignoreKey = Array.isArray(opts.ignore) ? opts.ignore.join('\n') : '' - const key = `${opts.cwd}\0${pattern}\0${opts.nodir ? 1 : 0}\0${opts.windowsPathsNoEscape ? 1 : 0}\0${ignoreKey}` - - const cached = globCache.get(key) - if (cached) { - globCacheHits++ - // Basic LRU: refresh insertion order. - globCache.delete(key) - globCache.set(key, cached) - return cached - } - - globCacheMisses++ - const res = globSync(pattern, opts) - - globCache.set(key, res) - if (globCache.size > GLOB_CACHE_MAX) { - const first = globCache.keys().next().value - if (first !== undefined) globCache.delete(first) - } - - return res -} - /** * @param {string} s * @returns {string} @@ -350,30 +319,23 @@ function parseExportAssignments (run) { } /** - * @param {string} repoRoot + * @param {TestFileIndex} index * @returns {string[]} */ -function findTestFiles (repoRoot) { - const commonGlobOpts = { cwd: repoRoot, nodir: true, windowsPathsNoEscape: true, ignore: DEFAULT_IGNORE_GLOBS } - +function findTestFiles (index) { // Collect every test-file naming convention used in the repo so an unrun spec // can never slip through untracked. Kept deliberately wide (js/mjs/cjs) even // where no file currently uses an extension, so a future one is caught. - const files = globSyncCached('**/*.@(spec|test).@(js|mjs|cjs)', commonGlobOpts) - - files.sort((a, b) => a.localeCompare(b, 'en')) - return files + return index.match(TEST_FILE_GLOB, DEFAULT_IGNORE_GLOBS) } /** - * @param {string} repoRoot + * @param {TestFileIndex} index * @param {Record} scripts * @param {Record} [env] * @returns {{ globs: string[], matchedFiles: Set }} */ -function expandScriptGlobs (repoRoot, scripts, env = {}) { - const ignore = DEFAULT_IGNORE_GLOBS - +function expandScriptGlobs (index, scripts, env = {}) { /** @type {string[]} */ const allGlobs = [] const matchedFiles = new Set() @@ -392,7 +354,7 @@ function expandScriptGlobs (repoRoot, scripts, env = {}) { let expanded try { - expanded = globSyncCached(normalized, { cwd: repoRoot, nodir: true, windowsPathsNoEscape: true, ignore }) + expanded = index.match(normalized, DEFAULT_IGNORE_GLOBS) } catch { // If a pattern can't be parsed by glob, skip expanding it. It still counts as an extracted glob. continue @@ -466,14 +428,14 @@ function extractScriptInvocations (run, knownScripts) { /** * Expand a script and any nested `npm run`/`yarn