test: validate E2E coverage infrastructure

scripts/instrument-plugin.sh

@@ -82,10 +82,12 @@ while IFS= read -r PROD_IMAGE; do
 
   PLUGIN_PATH=""
   if [[ -n "$PACKAGES_LABEL" && "$PACKAGES_LABEL" != "<no value>" ]]; then
-    # Decode base64 and extract first plugin name
-    # Expected JSON: [{"name":"backstage-community-plugin-acs","version":"0.2.0",...}]
-    # The "name" field is the directory path inside the container
-    PLUGIN_PATH=$(echo "$PACKAGES_LABEL" | base64 -d 2>/dev/null | jq -r '.[0].name // empty' 2>/dev/null || echo "")
+    # Decode base64 and extract the plugin directory path inside the container.
+    # Actual JSON shape: [{"<dir-name>": {"name":"@scope/pkg-dynamic","version":...}}]
+    # The directory path is the OBJECT KEY, not a "name" field. (An older `.[0].name`
+    # read returned empty and only worked because of the metadata fallback below.)
+    # Fall back to `.[0].name` for any legacy flat-shaped labels.
+    PLUGIN_PATH=$(echo "$PACKAGES_LABEL" | base64 -d 2>/dev/null | jq -r '.[0] as $p | (if ($p.name | type) == "string" then $p.name else ($p | keys[0]) end) // empty' 2>/dev/null || echo "")
     if [[ -n "$PLUGIN_PATH" ]]; then
       echo "  Plugin path (from OCI label): $PLUGIN_PATH"
     fi
@@ -113,60 +115,105 @@ while IFS= read -r PROD_IMAGE; do
     continue
   fi
 
-  # Create temp container and extract plugin bundle
+  # Create temp container to extract the plugin bundle(s)
   WORK_DIR=$(mktemp -d)
   # Plugin images are static bundles with no CMD/ENTRYPOINT, so we provide a dummy command
   CID=$(podman create "$PROD_IMAGE" /bin/true)
 
-  if ! podman cp "$CID:$PLUGIN_PATH/dist" "$WORK_DIR/dist-original"; then
-    echo "  ❌ Failed to extract plugin bundle from container - skipping"
-    podman rm "$CID" || true
-    rm -rf "$WORK_DIR"
-    SKIPPED_COUNT=$((SKIPPED_COUNT + 1))
-    continue
-  fi
+  # RHDH ships up to two frontend bundles per plugin and which one it actually
+  # serves/executes at runtime depends on the deployment: the Module Federation
+  # bundle (`dist/`) for the new frontend system and the Scalprum bundle
+  # (`dist-scalprum/`) for the legacy loader. Instrumenting only `dist/` leaves
+  # `window.__coverage__` undefined whenever the browser runs the Scalprum bundle,
+  # so we instrument every bundle that exists and overlay them all.
+  BUNDLE_DIRS=(dist dist-scalprum)
+  COPY_LINES=""
+  TOTAL_JS_COUNT=0
+
+  for BUNDLE in "${BUNDLE_DIRS[@]}"; do
+    # Not every plugin ships every bundle; quietly skip the absent ones.
+    if ! podman cp "$CID:$PLUGIN_PATH/$BUNDLE" "$WORK_DIR/orig-$BUNDLE" 2>/dev/null; then
+      echo "  No $BUNDLE/ in image - skipping that bundle"
+      continue
+    fi
 
-  podman rm "$CID"
+    # Instrument with nyc (pinned version for reproducibility).
+    # Must run from work directory to avoid "outside project root" errors.
+    echo "  Instrumenting $BUNDLE/ with Istanbul/nyc..."
+    if ! (cd "$WORK_DIR" && npx --yes nyc@18.0.0 instrument "orig-$BUNDLE" "inst-$BUNDLE" --source-map); then
+      echo "  ❌ Instrumentation of $BUNDLE/ failed - skipping that bundle"
+      continue
+    fi
 
-  # Instrument with nyc (pinned version for reproducibility)
-  # Must run from work directory to avoid "outside project root" errors
-  echo "  Instrumenting with Istanbul/nyc..."
-  if ! (cd "$WORK_DIR" && npx --yes nyc@18.0.0 instrument dist-original dist-instrumented --source-map); then
-    echo "  ❌ Instrumentation failed - skipping"
-    rm -rf "$WORK_DIR"
-    SKIPPED_COUNT=$((SKIPPED_COUNT + 1))
-    continue
-  fi
+    # Fix NYC's global access pattern for modern browsers.
+    # NYC emits `new Function("return this")()`, which RHDH's CSP (no unsafe-eval)
+    # blocks — leaving coverage uncollected. Replace it with `globalThis`.
+    find "$WORK_DIR/inst-$BUNDLE" -name "*.js" -type f -exec sed -i \
+      's/var global=new Function("return this")();/var global=globalThis;/g' {} \;
+
+    # Loudly flag any file where the global-scope fix did not apply: a silent miss
+    # means coverage runs but never reaches window.__coverage__.
+    UNFIXED_COUNT=$({ grep -rl 'new Function("return this")' "$WORK_DIR/inst-$BUNDLE/" --include="*.js" 2>/dev/null || true; } | wc -l | tr -d ' ')
+    if [[ "$UNFIXED_COUNT" -ne 0 ]]; then
+      echo "  ⚠️  $UNFIXED_COUNT file(s) in $BUNDLE/ still use new Function(\"return this\") after the fix"
+    fi
+
+    BUNDLE_JS_COUNT=$({ grep -rl "__coverage__" "$WORK_DIR/inst-$BUNDLE/" --include="*.js" 2>/dev/null || true; } | wc -l | tr -d ' ')
+    if [[ "$BUNDLE_JS_COUNT" -eq 0 ]]; then
+      echo "  ❌ No __coverage__ found in instrumented $BUNDLE/ - skipping that bundle"
+      continue
+    fi
+    echo "  ✓ Instrumented $BUNDLE_JS_COUNT JS files in $BUNDLE/"
+    TOTAL_JS_COUNT=$((TOTAL_JS_COUNT + BUNDLE_JS_COUNT))
+    COPY_LINES+="COPY inst-$BUNDLE/ $PLUGIN_PATH/$BUNDLE/"$'\n'
+  done
 
-  # Verify instrumentation
-  JS_COUNT=$(grep -r "__coverage__" "$WORK_DIR/dist-instrumented/" --include="*.js" -l 2>/dev/null | wc -l | tr -d ' ')
-  if [[ "$JS_COUNT" -eq 0 ]]; then
-    echo "  ❌ No __coverage__ found in instrumented files - skipping"
+  podman rm "$CID"
+
+  if [[ "$TOTAL_JS_COUNT" -eq 0 ]]; then
+    echo "  ❌ No bundles could be instrumented - skipping"
     rm -rf "$WORK_DIR"
     SKIPPED_COUNT=$((SKIPPED_COUNT + 1))
     continue
   fi
-  echo "  ✓ Instrumented $JS_COUNT JS files"
+  echo "  ✓ Instrumented $TOTAL_JS_COUNT JS files total"
 
-  # Build coverage image (copy instrumented files over production image)
-  cat > "$WORK_DIR/Containerfile" <<EOF
-FROM $PROD_IMAGE
-COPY dist-instrumented/ $PLUGIN_PATH/dist/
-EOF
+  # Build coverage image (overlay every instrumented bundle over the production image)
+  {
+    echo "FROM $PROD_IMAGE"
+    printf '%s' "$COPY_LINES"
+  } > "$WORK_DIR/Containerfile"
 
   # Generate coverage image tag: append __coverage suffix to tag
   # Example: plugin:pr_123__1.2.3 → plugin:pr_123__1.2.3__coverage
   IMAGE_BASE="${PROD_IMAGE%:*}"
   IMAGE_TAG="${PROD_IMAGE##*:}"
   COVERAGE_IMAGE="${IMAGE_BASE}:${IMAGE_TAG}__coverage"
 
-  if ! podman build -t "$COVERAGE_IMAGE" -f "$WORK_DIR/Containerfile" "$WORK_DIR"; then
+  # CRITICAL: --squash-all flattens the result into a SINGLE layer.
+  # RHDH's install-dynamic-plugins (image-cache.ts: downloadAndLocateTarball)
+  # only ever extracts manifest.layers[0] — it assumes dynamic-plugin images are
+  # single-layer. A plain `FROM prod + COPY` produces a multi-layer image whose
+  # FIRST layer is the original (uninstrumented) base, so RHDH would serve the
+  # original code and ignore our instrumented overlay layers. Squashing merges
+  # the overlays into one layer (instrumented files win), so layers[0] carries
+  # the instrumentation that actually reaches the browser.
+  if ! podman build --squash-all -t "$COVERAGE_IMAGE" -f "$WORK_DIR/Containerfile" "$WORK_DIR"; then
     echo "  ❌ Failed to build coverage image - skipping"
     rm -rf "$WORK_DIR"
     SKIPPED_COUNT=$((SKIPPED_COUNT + 1))
     continue
   fi
 
+  # Verify the image is single-layer so RHDH's layers[0]-only extraction sees
+  # the instrumented filesystem. Warn loudly if squashing did not collapse it.
+  LAYER_COUNT=$(podman inspect "$COVERAGE_IMAGE" --format '{{len .RootFS.Layers}}' 2>/dev/null || echo "?")
+  if [[ "$LAYER_COUNT" != "1" ]]; then
+    echo "  ⚠️  Coverage image has $LAYER_COUNT layers (expected 1) — RHDH only reads layers[0], coverage may not load"
+  else
+    echo "  ✓ Coverage image squashed to a single layer"
+  fi
+
   # Push coverage image
   if ! podman push "$COVERAGE_IMAGE"; then
     echo "  ❌ Failed to push coverage image"

workspaces/tech-radar/.coverage-bootstrap

@@ -0,0 +1,13 @@
+# E2E Coverage Infrastructure Validation
+
+This file is used to trigger E2E coverage collection for the tech-radar workspace.
+
+**Purpose**: Validate that PR #2383 coverage infrastructure works end-to-end:
+1. `/publish` command builds plugin images
+2. `instrument` job creates `__coverage` tagged images
+3. E2E tests collect coverage during execution
+4. Coverage is uploaded to Codecov with upstream repo attribution
+
+**Timeline**: This PR will be closed without merge after validation completes.
+
+**Test Date**: 2026-06-03

workspaces/tech-radar/e2e-tests/package.json

@@ -25,7 +25,7 @@
   "devDependencies": {
     "@eslint/js": "10.0.1",
     "@playwright/test": "1.59.1",
-    "@red-hat-developer-hub/e2e-test-utils": "1.1.45",
+    "@red-hat-developer-hub/e2e-test-utils": "gustavolira/rhdh-e2e-test-utils#debug/deep-coverage-investigation",
     "@types/node": "25.5.2",
     "eslint": "10.2.0",
     "eslint-plugin-check-file": "3.3.1",

workspaces/tech-radar/e2e-tests/tests/specs/tech-radar.spec.ts

@@ -43,9 +43,76 @@ test.describe("Test tech-radar plugin", () => {
     // await verifyRadarDetails(page, "Storage", "AWS S3");
     await verifyRadarDetails(page, "Frameworks", "React");
     await verifyRadarDetails(page, "Infrastructure", "GitHub Actions");
+
+    // Diagnostic only: confirm whether the JS bundles RHDH actually SERVES at
+    // runtime are the instrumented ones. The `__coverage` image instruments the
+    // plugin's `dist/` (and now `dist-scalprum/`), but if RHDH serves/regenerates
+    // a different bundle, `window.__coverage__` is never created. The probe fetches
+    // the served Module Federation and Scalprum entry points and reports whether
+    // they contain Istanbul markers. No-op outside coverage mode; never fails.
+    await probeServedAssetsForInstrumentation(page);
   });
 });
 
+async function probeServedAssetsForInstrumentation(page: Page) {
+  if (
+    process.env.E2E_COLLECT_COVERAGE !== "true" &&
+    process.env.E2E_COLLECT_COVERAGE !== "1"
+  ) {
+    return;
+  }
+
+  const origin = new URL(page.url()).origin;
+
+  // Report whether a fetched JS body carries Istanbul instrumentation.
+  const reportBody = (url: string, body: string) => {
+    const hasCoverageVar = body.includes("__coverage__");
+    const hasCovFn = /cov_[a-z0-9]+\(\)/.test(body);
+    const hasUnfixedGlobal = body.includes('new Function("return this")');
+    console.warn(
+      `[coverage-probe] instrumented=${hasCoverageVar && hasCovFn} __coverage__=${hasCoverageVar} cov_fn=${hasCovFn} unfixedGlobal=${hasUnfixedGlobal} bytes=${body.length} <- ${url}`,
+    );
+  };
+
+  const fetchText = async (url: string): Promise<string | null> => {
+    try {
+      const res = await page.request.get(url);
+      if (!res.ok()) {
+        console.warn(`[coverage-probe] HTTP ${res.status()} <- ${url}`);
+        return null;
+      }
+      return await res.text();
+    } catch (err) {
+      console.warn(`[coverage-probe] fetch failed for ${url}: ${err}`);
+      return null;
+    }
+  };
+
+  // 1) Module Federation entry point (served from the plugin's dist/).
+  const remoteEntry = `${origin}/dynamic-features/remotes/@backstage-community/plugin-tech-radar-dynamic/remoteEntry.js`;
+  const remoteEntryBody = await fetchText(remoteEntry);
+  if (remoteEntryBody !== null) reportBody(remoteEntry, remoteEntryBody);
+
+  // 2) Scalprum bundle (served from the plugin's dist-scalprum/): discover the
+  //    actual hashed script name from the plugin manifest, then fetch it.
+  const scalprumBase = `${origin}/api/scalprum/backstage-community.plugin-tech-radar`;
+  const manifestBody = await fetchText(`${scalprumBase}/plugin-manifest.json`);
+  if (manifestBody !== null) {
+    try {
+      const manifest = JSON.parse(manifestBody) as { loadScripts?: string[] };
+      for (const script of manifest.loadScripts ?? []) {
+        const scriptUrl = `${scalprumBase}/${script}`;
+        const scriptBody = await fetchText(scriptUrl);
+        if (scriptBody !== null) reportBody(scriptUrl, scriptBody);
+      }
+    } catch (err) {
+      console.warn(
+        `[coverage-probe] could not parse scalprum manifest: ${err}`,
+      );
+    }
+  }
+}
+
 async function verifyRadarDetails(page: Page, section: string, text: string) {
   const sectionLocator = page
     .locator(`h2:has-text("${section}")`)