feat: E2E coverage instrumentation with CI pipeline for all dynamic plugins

.github/workflows/auto-publish-pr.yaml

@@ -47,10 +47,12 @@ jobs:
             core.setOutput('overlay-commit', prCommit);
             let workspace = '';
 
+            // Try to extract workspace from branch name (e.g., workspaces/release-1.3__tech-radar)
             const matches = prBranch.match(/^workspaces\/release-.+__(.+)$/);
             if (matches && matches.length == 2) {
               workspace = `workspaces/${matches[1]}`;
             } else {
+              // Detect modified workspaces from PR files
               const prFiles = await github.rest.pulls.listFiles({
                 owner: context.repo.owner,
                 repo: context.repo.repo,
@@ -330,6 +332,63 @@ jobs:
       packages: write
       id-token: write
 
+  instrument:
+    name: Instrument plugin images for E2E coverage
+    needs:
+      - prepare
+      - export
+    # Only run if workspace has e2e-tests/ directory
+    if: |
+      always() &&
+      needs.export.result == 'success' &&
+      needs.export.outputs.published-exports != ''
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          repository: ${{ needs.prepare.outputs.overlay-repo }}
+          ref: ${{ needs.prepare.outputs.overlay-branch }}
+
+      - name: Check if workspace has E2E tests
+        id: check-e2e
+        env:
+          WORKSPACE: ${{ needs.prepare.outputs.workspace }}
+        run: |
+          if [[ -d "${WORKSPACE}/e2e-tests" ]]; then
+            echo "has-e2e=true" >> "$GITHUB_OUTPUT"
+            echo "Workspace has E2E tests directory"
+          else
+            echo "has-e2e=false" >> "$GITHUB_OUTPUT"
+            echo "Workspace does not have E2E tests directory - skipping instrumentation"
+          fi
+
+      - name: Log in to GitHub Container Registry
+        if: steps.check-e2e.outputs.has-e2e == 'true'
+        env:
+          REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
+        run: echo "$REGISTRY_PASSWORD" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin
+
+      - name: Setup Node.js
+        if: steps.check-e2e.outputs.has-e2e == 'true'
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: 20
+
+      - name: Instrument and publish coverage images
+        if: steps.check-e2e.outputs.has-e2e == 'true'
+        env:
+          PUBLISHED_EXPORTS: ${{ needs.export.outputs.published-exports }}
+          WORKSPACE: ${{ needs.prepare.outputs.workspace }}
+        run: |
+          echo "Published exports:"
+          echo "$PUBLISHED_EXPORTS"
+          echo ""
+          echo "$PUBLISHED_EXPORTS" | ./scripts/instrument-plugin.sh "$WORKSPACE"
+
   check-backstage-compatibility:
     name: Check workspace backstage compatibility
     needs:

run-e2e.sh

@@ -29,6 +29,11 @@ set -euo pipefail
 #
 #   # Use an unpublished git branch of e2e-test-utils (clones and builds locally)
 #   E2E_TEST_UTILS_GIT_REF=owner/rhdh-e2e-test-utils#my-branch ./run-e2e.sh -w tech-radar
+#
+#   # Coverage collection is ENABLED BY DEFAULT
+#   # Requires e2e-test-utils >= 1.x.x for automatic -coverage image swap
+#   # To disable for faster local development:
+#   E2E_COLLECT_COVERAGE=0 ./run-e2e.sh -w tech-radar
 # =============================================================================
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
@@ -58,6 +63,18 @@ export CATALOG_INDEX_IMAGE="${CATALOG_INDEX_IMAGE:-}"
 # Nightly mode
 E2E_NIGHTLY_MODE="${E2E_NIGHTLY_MODE:-false}"
 
+# Coverage collection (Istanbul) — enabled by default
+#
+# For PR checks: Works now. The auto-publish-pr.yaml workflow builds -coverage
+# images (plugin:tag__coverage) that e2e-test-utils will load when available.
+#
+# For nightly/local: Depends on e2e-test-utils automatic image swap logic
+# (PR #95, not yet released). Until that lands, coverage collection will be
+# skipped silently (no -coverage images exist).
+#
+# To disable (faster local dev): E2E_COLLECT_COVERAGE=0
+export E2E_COLLECT_COVERAGE="${E2E_COLLECT_COVERAGE:-1}"
+
 # Local e2e-test-utils: absolute path to use a local build instead of npm
 E2E_TEST_UTILS_PATH="${E2E_TEST_UTILS_PATH:-}"
 # Pin specific e2e-test-utils version.
@@ -247,6 +264,10 @@ for ws in "${E2E_WORKSPACES[@]}"; do
     done <<< "$PROJECTS_BLOCK"
 done
 
+if [[ "${E2E_COLLECT_COVERAGE:-}" == "1" ]]; then
+    echo "[INFO] Coverage collection enabled (E2E_COLLECT_COVERAGE=1)"
+fi
+
 cat > playwright.config.ts <<CONFIGEOF
 // Auto-generated by run-e2e.sh
 import { baseConfig } from '@red-hat-developer-hub/e2e-test-utils/playwright-config';
@@ -289,6 +310,16 @@ echo ""
 TEST_EXIT_CODE=0
 npx playwright test "${PLAYWRIGHT_ARGS[@]+"${PLAYWRIGHT_ARGS[@]}"}" || TEST_EXIT_CODE=$?
 
+# ── Merge coverage data ──────────────────────────────────────────────────
+if [[ "${E2E_COLLECT_COVERAGE:-}" == "1" ]]; then
+    if [[ -d "node_modules/.cache/e2e-test-results/coverage" ]]; then
+        "$SCRIPT_DIR/scripts/report-coverage.sh" "${E2E_WORKSPACES[@]}"
+    else
+        echo "[INFO] Coverage collection enabled but no coverage data found."
+        echo "[INFO] Ensure plugins are loaded from instrumented (-coverage) images."
+    fi
+fi
+
 # ── Summary ───────────────────────────────────────────────────────────────────
 
 echo ""

scripts/instrument-plugin.sh

@@ -0,0 +1,195 @@
+#!/usr/bin/env bash
+#
+# Instrument frontend plugin OCI images with Istanbul coverage.
+#
+# Usage:
+#   echo "image1\nimage2" | ./scripts/instrument-plugin.sh <workspace-path>
+#   ./scripts/instrument-plugin.sh <workspace-path> < images.txt
+#
+# Example:
+#   echo "ghcr.io/repo/plugin:pr_123__1.0.0" | ./scripts/instrument-plugin.sh workspaces/tech-radar
+#
+# The script:
+#   1. Reads OCI image refs from stdin (one per line)
+#   2. For each frontend plugin image:
+#      - Pulls the production image
+#      - Extracts plugin path from OCI labels (io.backstage.dynamic-packages)
+#      - Extracts plugin bundle from the container
+#      - Instruments JavaScript with nyc (Istanbul)
+#      - Builds a new coverage image with instrumented files
+#      - Pushes the coverage image with __coverage tag suffix
+
+set -euo pipefail
+
+WORKSPACE="${1:?Usage: $0 <workspace-path>}"
+
+if [[ ! -d "$WORKSPACE" ]]; then
+  echo "ERROR: Workspace directory not found: $WORKSPACE" >&2
+  exit 1
+fi
+
+if [[ ! -d "$WORKSPACE/metadata" ]]; then
+  echo "ERROR: No metadata directory found in workspace: $WORKSPACE/metadata" >&2
+  exit 1
+fi
+
+echo "=== Instrumenting published plugin images for E2E coverage ==="
+echo "Workspace: $WORKSPACE"
+echo ""
+
+INSTRUMENTED_COUNT=0
+SKIPPED_COUNT=0
+
+# Process each published image (format: plain image refs, one per line)
+while IFS= read -r PROD_IMAGE; do
+  [[ -z "$PROD_IMAGE" ]] && continue
+
+  echo "--- Processing: $PROD_IMAGE ---"
+
+  # Extract plugin name from image ref
+  PLUGIN_NAME=$(basename "${PROD_IMAGE%%:*}")
+  echo "  Plugin: $PLUGIN_NAME"
+
+  # Find metadata file for this plugin
+  # The metadata filename matches the OCI image name (e.g., backstage-community-plugin-acs.yaml)
+  METADATA_FILE="${WORKSPACE}/metadata/${PLUGIN_NAME}.yaml"
+
+  if [[ ! -f "$METADATA_FILE" ]]; then
+    echo "  ⚠️  No metadata file found at $METADATA_FILE - skipping"
+    SKIPPED_COUNT=$((SKIPPED_COUNT + 1))
+    continue
+  fi
+
+  # Check if this is a frontend plugin (only frontend plugins need instrumentation)
+  PLUGIN_ROLE=$(yq -r '.spec.backstage.role // ""' "$METADATA_FILE")
+  if [[ "$PLUGIN_ROLE" != "frontend-plugin" ]]; then
+    echo "  Skipping $PLUGIN_ROLE (only frontend plugins need browser coverage)"
+    SKIPPED_COUNT=$((SKIPPED_COUNT + 1))
+    continue
+  fi
+
+  # Pull production image first (needed to inspect labels)
+  if ! podman pull "$PROD_IMAGE" 2>&1 | grep -v "WARNING: image platform"; then
+    echo "  ❌ Failed to pull image - skipping"
+    SKIPPED_COUNT=$((SKIPPED_COUNT + 1))
+    continue
+  fi
+
+  # Extract plugin path from OCI image labels (preferred method)
+  # The io.backstage.dynamic-packages label contains base64-encoded JSON
+  # with plugin metadata including the directory path inside the container
+  PACKAGES_LABEL=$(podman inspect "$PROD_IMAGE" --format '{{index .Labels "io.backstage.dynamic-packages"}}' 2>/dev/null || echo "")
+
+  PLUGIN_PATH=""
+  if [[ -n "$PACKAGES_LABEL" && "$PACKAGES_LABEL" != "<no value>" ]]; then
+    # Decode base64 and extract first plugin name
+    # Expected JSON: [{"name":"backstage-community-plugin-acs","version":"0.2.0",...}]
+    # The "name" field is the directory path inside the container
+    PLUGIN_PATH=$(echo "$PACKAGES_LABEL" | base64 -d 2>/dev/null | jq -r '.[0].name // empty' 2>/dev/null || echo "")
+    if [[ -n "$PLUGIN_PATH" ]]; then
+      echo "  Plugin path (from OCI label): $PLUGIN_PATH"
+    fi
+  fi
+
+  # Fallback: Extract from dynamicArtifact in metadata if label doesn't exist
+  if [[ -z "$PLUGIN_PATH" ]]; then
+    echo "  No io.backstage.dynamic-packages label - using metadata fallback"
+    DYNAMIC_ARTIFACT=$(yq -r '.spec.dynamicArtifact // ""' "$METADATA_FILE")
+
+    # Format: "oci://image:tag!path" or "oci://image:tag"
+    if [[ "$DYNAMIC_ARTIFACT" =~ !(.+)$ ]]; then
+      PLUGIN_PATH="${BASH_REMATCH[1]}"
+      echo "  Plugin path (from metadata): $PLUGIN_PATH"
+    else
+      # No explicit path — use plugin name as path
+      PLUGIN_PATH="$PLUGIN_NAME"
+      echo "  Plugin path (default): $PLUGIN_PATH"
+    fi
+  fi
+
+  if [[ -z "$PLUGIN_PATH" ]]; then
+    echo "  ⚠️  Could not determine plugin path - skipping"
+    SKIPPED_COUNT=$((SKIPPED_COUNT + 1))
+    continue
+  fi
+
+  # Create temp container and extract plugin bundle
+  WORK_DIR=$(mktemp -d)
+  CID=$(podman create "$PROD_IMAGE")
+
+  if ! podman cp "$CID:$PLUGIN_PATH/dist" "$WORK_DIR/dist-original"; then
+    echo "  ❌ Failed to extract plugin bundle from container - skipping"
+    podman rm "$CID" || true
+    rm -rf "$WORK_DIR"
+    SKIPPED_COUNT=$((SKIPPED_COUNT + 1))
+    continue
+  fi
+
+  podman rm "$CID"
+
+  # Instrument with nyc (pinned version for reproducibility)
+  # Must run from work directory to avoid "outside project root" errors
+  echo "  Instrumenting with Istanbul/nyc..."
+  if ! (cd "$WORK_DIR" && npx --yes nyc@18.0.0 instrument dist-original dist-instrumented --source-map); then
+    echo "  ❌ Instrumentation failed - skipping"
+    rm -rf "$WORK_DIR"
+    SKIPPED_COUNT=$((SKIPPED_COUNT + 1))
+    continue
+  fi
+
+  # Verify instrumentation
+  JS_COUNT=$(grep -r "__coverage__" "$WORK_DIR/dist-instrumented/" --include="*.js" -l 2>/dev/null | wc -l | tr -d ' ')
+  if [[ "$JS_COUNT" -eq 0 ]]; then
+    echo "  ❌ No __coverage__ found in instrumented files - skipping"
+    rm -rf "$WORK_DIR"
+    SKIPPED_COUNT=$((SKIPPED_COUNT + 1))
+    continue
+  fi
+  echo "  ✓ Instrumented $JS_COUNT JS files"
+
+  # Build coverage image (copy instrumented files over production image)
+  cat > "$WORK_DIR/Containerfile" <<EOF
+FROM $PROD_IMAGE
+COPY dist-instrumented/ $PLUGIN_PATH/dist/
+EOF
+
+  # Generate coverage image tag: append __coverage suffix to tag
+  # Example: plugin:pr_123__1.2.3 → plugin:pr_123__1.2.3__coverage
+  IMAGE_BASE="${PROD_IMAGE%:*}"
+  IMAGE_TAG="${PROD_IMAGE##*:}"
+  COVERAGE_IMAGE="${IMAGE_BASE}:${IMAGE_TAG}__coverage"
+
+  if ! podman build -t "$COVERAGE_IMAGE" -f "$WORK_DIR/Containerfile" "$WORK_DIR"; then
+    echo "  ❌ Failed to build coverage image - skipping"
+    rm -rf "$WORK_DIR"
+    SKIPPED_COUNT=$((SKIPPED_COUNT + 1))
+    continue
+  fi
+
+  # Push coverage image
+  if ! podman push "$COVERAGE_IMAGE"; then
+    echo "  ❌ Failed to push coverage image"
+    rm -rf "$WORK_DIR"
+    SKIPPED_COUNT=$((SKIPPED_COUNT + 1))
+    continue
+  fi
+
+  echo "  ✓ Published: $COVERAGE_IMAGE"
+
+  # Cleanup
+  rm -rf "$WORK_DIR"
+  echo ""
+
+  INSTRUMENTED_COUNT=$((INSTRUMENTED_COUNT + 1))
+
+done
+
+echo "=== Instrumentation complete ==="
+echo "  Instrumented: $INSTRUMENTED_COUNT plugins"
+echo "  Skipped:      $SKIPPED_COUNT plugins"
+
+if [[ $INSTRUMENTED_COUNT -eq 0 ]]; then
+  echo ""
+  echo "[WARN] No plugins were instrumented"
+  echo "[INFO] This may be expected if there are no frontend plugins in this workspace"
+fi

scripts/report-coverage.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+#
+# Merge per-test Istanbul coverage JSONs, generate lcov, and upload to Codecov.
+#
+# Usage:
+#   ./scripts/report-coverage.sh <workspace> [workspace...]
+#
+# Example:
+#   E2E_COLLECT_COVERAGE=1 ./run-e2e.sh -w tech-radar
+#   ./scripts/report-coverage.sh tech-radar
+#
+# The script:
+#   1. Merges per-test coverage JSONs (written by the _coverageCollector fixture)
+#      into a single coverage-final.json using nyc merge
+#   2. Generates lcov and text-summary reports via nyc report
+#   3. Uploads lcov to Codecov for each workspace with cross-repo attribution
+#
+# Required environment:
+#   CODECOV_TOKEN  - Codecov upload token (org-level for cross-repo uploads)
+
+set -euo pipefail
+
+if [[ $# -eq 0 ]]; then
+  echo "Usage: $0 <workspace> [workspace...]" >&2
+  exit 1
+fi
+
+SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+WORKSPACES=("$@")
+
+COVERAGE_JSON_DIR="node_modules/.cache/e2e-test-results/coverage"
+
+if ! compgen -G "$REPO_ROOT/$COVERAGE_JSON_DIR/*.json" >/dev/null 2>&1; then
+  echo "[INFO] No coverage data found (no instrumented plugins loaded?)"
+  exit 0
+fi
+
+echo ""
+echo "[INFO] Merging coverage data with nyc..."
+mkdir -p "$REPO_ROOT/.nyc_output"
+npx nyc@18.0.0 merge "$REPO_ROOT/$COVERAGE_JSON_DIR" "$REPO_ROOT/.nyc_output/out.json"
+(cd "$REPO_ROOT" && npx nyc@18.0.0 report --reporter=lcov --reporter=text-summary --report-dir coverage)
+
+if [[ ${#WORKSPACES[@]} -gt 1 ]]; then
+  echo "[WARN] Multi-workspace coverage upload is not supported." >&2
+  echo "[WARN] Coverage is merged across workspaces but uploaded with per-workspace flags." >&2
+  echo "[WARN] This produces misleading coverage percentages in Codecov." >&2
+  echo "[WARN] Skipping upload. Run report-coverage.sh once per workspace to upload." >&2
+else
+  echo "[INFO] Uploading E2E coverage to Codecov..."
+  for ws in "${WORKSPACES[@]}"; do
+    if [[ -f "$REPO_ROOT/workspaces/$ws/source.json" ]]; then
+      "$SCRIPT_DIR/upload-coverage.sh" "$ws" || \
+        echo "[WARN] Coverage upload failed for $ws (non-fatal)"
+    fi
+  done
+fi