ci(scripts): speed up verify-exercised-tests and close detection gaps (#9542)

* test(scripts): cover verify-exercised-tests detection

The checker's failure branches are unpinned, so a rework can narrow what it
detects while still exiting 0 on a healthy tree. This adds fixture checkouts
that drive each branch.

`main` takes the repository root as an optional argument so a fixture can stand
in for the script's own checkout.

* ci(scripts): answer test-file globs from one repository walk

verify-exercised-tests expanded its script globs with 227 separate `globSync`
traversals of the same tree. Walking once and matching the patterns in memory
cuts the run from 312 ms to 178 ms, medians of ten interleaved runs on one
checkout.

Matching has to stay identical to the walk it replaces, so the index compiles
patterns with the options `glob` derives internally. Its spec cross-checks
every result against `globSync`, over hand-written pattern shapes and over the
glob corpus this repository's own package.json produces.

Answering with too few files would let an unexercised spec pass unseen, so the
index refuses instead: an unreadable directory warns, and a `..` segment, which
needs a real tree to resolve against, throws. Literal path segments are compared
case-sensitively on every platform, the rule `glob` applies on Linux, so a
result no longer depends on whether the developer's filesystem folds case.

* ci(scripts): resolve workflow steps into one ordered event stream

Two ways a suite could go unverified survived the checker, both because a job's
steps were never modeled as one ordered, per-leg-resolved sequence.

1. A coverage upload placed before the suite it collects passed the check, because
   uploads were tracked as a per-job boolean and position never entered it. Five
   composite actions run a suite and upload its report, so the order that decides
   whether a report survives usually lives inside the action, not in the workflow.
2. `SPEC: ${{ matrix.spec }}` reached the glob unresolved, degrading `${SPEC:-*}`
   to `*` and making every spec in a directory look exercised by every matrix leg.
   Resolving it per leg surfaces
   packages/datadog-plugin-aws-sdk/test/base-inject-field.spec.js, which no leg
   selects; the serverless matrix gains the entry that runs it.

One traversal now yields the `run` and `upload` events a step contributes. The
second traversal that answered only "does this job upload?" is gone, and with it
the missing `using: composite` guard that made it throw on a JavaScript or Docker
action. Such an action is reported now, so an upload hidden inside one cannot read
as "this job never uploads".

Author: BridgeAR

.github/CODEOWNERS

@@ -279,7 +279,10 @@
 
 /scripts/generate-config-types.js @DataDog/apm-sdk-capabilities-js
 /scripts/generate-supported-integrations.js @DataDog/apm-sdk-capabilities-js
+/scripts/helpers/test-file-index.js @DataDog/apm-sdk-capabilities-js
+/scripts/helpers/test-file-index.spec.js @DataDog/apm-sdk-capabilities-js
 /scripts/verify-exercised-tests.js @DataDog/apm-sdk-capabilities-js
+/scripts/verify-exercised-tests.spec.mjs @DataDog/apm-sdk-capabilities-js
 /supported_versions_output.json @DataDog/apm-sdk-capabilities-js
 /supported_versions_table.csv @DataDog/apm-sdk-capabilities-js
 

.github/workflows/serverless.yml

@@ -52,6 +52,7 @@ jobs:
         spec:
           - client
           - aws-sdk
+          - base-inject-field
           - bedrockruntime
           - dynamodb
           - eventbridge

package.json

@@ -222,6 +222,7 @@
     "istanbul-lib-report": "^3.0.0",
     "istanbul-reports": "^3.0.2",
     "jszip": "^3.10.1",
+    "minimatch": "^10.2.5",
     "mocha": "^11.7.6",
     "mocha-junit-reporter": "^2.2.1",
     "mocha-multi-reporters": "^1.5.1",

scripts/helpers/test-file-index.js

@@ -0,0 +1,288 @@
+'use strict'
+
+const fs = require('node:fs')
+const path = require('node:path')
+
+const { Glob } = require('glob')
+const { Minimatch } = require('minimatch')
+
+const TEST_FILE_GLOB = '**/*.@(spec|test).@(js|mjs|cjs)'
+
+// Case-insensitive because `glob` matches case-insensitively on darwin and win32. Indexing a
+// `FOO.SPEC.JS` that a case-sensitive platform would reject is harmless: the pattern matcher
+// applies that platform's own rules and drops it again.
+const TEST_FILE_NAME = /\.(?:spec|test)\.(?:js|mjs|cjs)$/i
+
+// Any character that can start a wildcard, brace, class, or extglob. Over-reporting only shortens
+// the literal prefix, which widens the candidate set and can never drop a match.
+const PATTERN_MAGIC = /[*?[\]{}()!+@\\]/
+
+// `glob` walks `..` on the real filesystem before matching. The index has no tree to walk, so it
+// would answer such a pattern with too few files and report exercised specs as unexercised.
+const PARENT_SEGMENT = /(?:^|\/)\.\.(?:\/|$)/
+
+/**
+ * @param {string} repoRoot
+ * @returns {string[]}
+ */
+function walkTestFiles (repoRoot) {
+  /** @type {string[]} */
+  const found = []
+  /** @type {string[]} */
+  const pending = ['']
+
+  while (pending.length) {
+    const relativeDir = pending.pop()
+
+    /** @type {import('node:fs').Dirent[]} */
+    let entries
+    try {
+      entries = fs.readdirSync(path.join(repoRoot, relativeDir), { withFileTypes: true })
+    } catch (error) {
+      // `glob` skips unreadable directories without a word. Staying silent here would shrink the
+      // index instead, and an unexercised spec below that directory would pass the check unseen.
+      process.stderr.write(`test-file-index: skipped unreadable ${relativeDir || '.'} (${error.code})\n`)
+      continue
+    }
+
+    for (const entry of entries) {
+      const { name } = entry
+
+      // `glob` runs with `dot: false`, so a dot entry can never satisfy TEST_FILE_GLOB. Every
+      // consumer of this index also ignores `node_modules`, so descending into it is dead work.
+      if (name.startsWith('.') || name === 'node_modules') continue
+
+      const relativePath = relativeDir ? `${relativeDir}/${name}` : name
+
+      if (entry.isDirectory()) {
+        pending.push(relativePath)
+      } else if (TEST_FILE_NAME.test(name)) {
+        // Symlinks are kept: `nodir: true` filters directories, not links to files.
+        found.push(relativePath)
+      }
+    }
+  }
+
+  return found
+}
+
+/**
+ * Drops `.` and empty path segments the way `glob` does while building its pattern. `minimatch`
+ * on a bare string keeps them and would then compare one segment too many.
+ *
+ * @param {string} pattern
+ * @returns {string}
+ */
+function normalizePattern (pattern) {
+  if (!pattern.includes('./') && !pattern.includes('//')) return pattern
+
+  const segments = pattern.split('/')
+  const kept = []
+
+  for (let i = 0; i < segments.length; i++) {
+    const segment = segments[i]
+    // A leading empty segment marks an absolute pattern and a trailing one a directory-only
+    // pattern; both change what matches, so only interior blanks are collapsed.
+    if (segment === '.' || (segment === '' && i !== 0 && i !== segments.length - 1)) continue
+    kept.push(segment)
+  }
+
+  return kept.join('/')
+}
+
+/**
+ * Longest leading directory path of `pattern` that contains no wildcard.
+ *
+ * @param {string} pattern
+ * @returns {string}
+ */
+function literalPrefix (pattern) {
+  const magic = pattern.search(PATTERN_MAGIC)
+  const literal = magic === -1 ? pattern : pattern.slice(0, magic)
+  const lastSlash = literal.lastIndexOf('/')
+  return lastSlash === -1 ? '' : literal.slice(0, lastSlash + 1)
+}
+
+/**
+ * In-memory stand-in for repeated `globSync` walks over the repository's test files.
+ *
+ * The repository is traversed once; every later pattern is answered from the resulting list. Only
+ * paths named like a test file are indexed, so a pattern that also selects non-test files reports
+ * just the test-file subset.
+ *
+ * Results have to stay identical to the `globSync` walk they replace; `test-file-index.spec.js`
+ * pins that against `glob` itself, including the repository's own pattern corpus.
+ */
+class TestFileIndex {
+  /** @type {string[]} */
+  files
+
+  /** @type {boolean} */
+  #nocase
+
+  /** @type {NodeJS.Platform} */
+  #platform
+
+  /** @type {Map<string, InstanceType<typeof Minimatch>>} */
+  #matchers = new Map()
+
+  /** @type {Map<string, InstanceType<typeof Minimatch>>} */
+  #ignoreMatchers = new Map()
+
+  /** @type {Map<string, { files: string[], folded: string[] }>} */
+  #scopes = new Map()
+
+  /** @type {Map<string, string[]>} */
+  #candidates = new Map()
+
+  /**
+   * @param {string[]} files Sorted repository-relative POSIX paths.
+   * @param {{ nocase: boolean, platform: NodeJS.Platform }} options
+   */
+  constructor (files, { nocase, platform }) {
+    this.files = files
+    this.#nocase = nocase
+    this.#platform = platform
+  }
+
+  /**
+   * A literal path segment is compared case-sensitively on every platform, which is what `glob`
+   * does on Linux. On a case-insensitive filesystem `glob` instead resolves `foo/BAR.spec.js` to
+   * the differently-spelled file on disk; reproducing that would make the result depend on the
+   * developer's machine, so the index keeps the Linux rule everywhere.
+   *
+   * @param {string} pattern
+   * @param {string[]} [ignoreGlobs]
+   * @throws {Error} If `pattern` contains a `..` segment, which the index cannot resolve.
+   * @returns {string[]} Matching paths, in the index's sort order.
+   */
+  match (pattern, ignoreGlobs = []) {
+    if (PARENT_SEGMENT.test(pattern)) {
+      throw new Error(`test-file-index cannot resolve the '..' segment in pattern '${pattern}'`)
+    }
+
+    const normalized = normalizePattern(pattern)
+    const scope = this.#scope(ignoreGlobs)
+    const matcher = this.#matcher(normalized)
+
+    /** @type {string[]} */
+    const matched = []
+    for (const file of this.#candidatesFor(scope, ignoreGlobs, normalized)) {
+      // eslint-disable-next-line unicorn/prefer-regexp-test -- Minimatch#match, not String#match.
+      if (matcher.match(file)) matched.push(file)
+    }
+
+    return matched
+  }
+
+  /**
+   * @param {string[]} ignoreGlobs
+   * @returns {{ files: string[], folded: string[] }}
+   */
+  #scope (ignoreGlobs) {
+    const key = ignoreGlobs.join('\0')
+
+    let scope = this.#scopes.get(key)
+    if (scope === undefined) {
+      const matchers = ignoreGlobs.map(glob => this.#ignoreMatcher(glob))
+      const files = matchers.length === 0
+        ? this.files
+        : this.files.filter(file => !matchers.some(matcher => matcher.match(file) || matcher.match(`${file}/`)))
+
+      scope = { files, folded: this.#nocase ? files.map(file => file.toLowerCase()) : files }
+      this.#scopes.set(key, scope)
+    }
+
+    return scope
+  }
+
+  /**
+   * @param {{ files: string[], folded: string[] }} scope
+   * @param {string[]} ignoreGlobs
+   * @param {string} pattern
+   * @returns {string[]}
+   */
+  #candidatesFor (scope, ignoreGlobs, pattern) {
+    const prefix = literalPrefix(pattern)
+    if (prefix === '') return scope.files
+
+    const key = `${ignoreGlobs.join('\0')}\0${prefix}`
+
+    let candidates = this.#candidates.get(key)
+    if (candidates === undefined) {
+      const wanted = this.#nocase ? prefix.toLowerCase() : prefix
+
+      candidates = []
+      for (let i = 0; i < scope.files.length; i++) {
+        if (scope.folded[i].startsWith(wanted)) candidates.push(scope.files[i])
+      }
+
+      this.#candidates.set(key, candidates)
+    }
+
+    return candidates
+  }
+
+  /**
+   * @param {string} pattern
+   * @returns {InstanceType<typeof Minimatch>}
+   */
+  #matcher (pattern) {
+    let matcher = this.#matchers.get(pattern)
+    if (matcher === undefined) {
+      matcher = new Minimatch(pattern, {
+        dot: false,
+        nocase: this.#nocase,
+        nocaseMagicOnly: this.#platform === 'darwin' || this.#platform === 'win32',
+        nocomment: true,
+        nonegate: true,
+        optimizationLevel: 2,
+        platform: this.#platform,
+        windowsPathsNoEscape: true,
+      })
+      this.#matchers.set(pattern, matcher)
+    }
+
+    return matcher
+  }
+
+  /**
+   * @param {string} glob
+   * @returns {InstanceType<typeof Minimatch>}
+   */
+  #ignoreMatcher (glob) {
+    let matcher = this.#ignoreMatchers.get(glob)
+    if (matcher === undefined) {
+      // Mirrors the options `glob`'s Ignore class builds; notably `dot: true`, so `**/.git/**`
+      // still matches paths that the main pattern would skip.
+      matcher = new Minimatch(glob, {
+        dot: true,
+        nocase: this.#nocase,
+        nocomment: true,
+        nonegate: true,
+        optimizationLevel: 2,
+        platform: this.#platform,
+      })
+      this.#ignoreMatchers.set(glob, matcher)
+    }
+
+    return matcher
+  }
+}
+
+/**
+ * @param {string} repoRoot
+ * @returns {TestFileIndex}
+ */
+function createTestFileIndex (repoRoot) {
+  // `glob` decides case sensitivity from the platform's filesystem; read it back rather than
+  // re-deriving it here, so the index cannot drift from the walker it replaces.
+  const probe = new Glob(TEST_FILE_GLOB, { cwd: repoRoot, windowsPathsNoEscape: true })
+
+  const files = walkTestFiles(repoRoot)
+  files.sort((a, b) => a.localeCompare(b, 'en'))
+
+  return new TestFileIndex(files, { nocase: probe.nocase, platform: probe.platform })
+}
+
+module.exports = { TEST_FILE_GLOB, TestFileIndex, createTestFileIndex }