diff --git a/.github/workflows/auto-publish-pr.yaml b/.github/workflows/auto-publish-pr.yaml index b262dfa02..444dd18ca 100644 --- a/.github/workflows/auto-publish-pr.yaml +++ b/.github/workflows/auto-publish-pr.yaml @@ -47,10 +47,12 @@ jobs: core.setOutput('overlay-commit', prCommit); let workspace = ''; + // Try to extract workspace from branch name (e.g., workspaces/release-1.3__tech-radar) const matches = prBranch.match(/^workspaces\/release-.+__(.+)$/); if (matches && matches.length == 2) { workspace = `workspaces/${matches[1]}`; } else { + // Detect modified workspaces from PR files const prFiles = await github.rest.pulls.listFiles({ owner: context.repo.owner, repo: context.repo.repo, @@ -330,6 +332,63 @@ jobs: packages: write id-token: write + instrument: + name: Instrument plugin images for E2E coverage + needs: + - prepare + - export + # Only run if workspace has e2e-tests/ directory + if: | + always() && + needs.export.result == 'success' && + needs.export.outputs.published-exports != '' + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + repository: ${{ needs.prepare.outputs.overlay-repo }} + ref: ${{ needs.prepare.outputs.overlay-branch }} + + - name: Check if workspace has E2E tests + id: check-e2e + env: + WORKSPACE: ${{ needs.prepare.outputs.workspace }} + run: | + if [[ -d "${WORKSPACE}/e2e-tests" ]]; then + echo "has-e2e=true" >> "$GITHUB_OUTPUT" + echo "Workspace has E2E tests directory" + else + echo "has-e2e=false" >> "$GITHUB_OUTPUT" + echo "Workspace does not have E2E tests directory - skipping instrumentation" + fi + + - name: Log in to GitHub Container Registry + if: steps.check-e2e.outputs.has-e2e == 'true' + env: + REGISTRY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + run: echo "$REGISTRY_PASSWORD" | podman login ghcr.io -u "${{ github.actor }}" --password-stdin + + - name: Setup Node.js + if: steps.check-e2e.outputs.has-e2e == 'true' + uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 20 + + - name: Instrument and publish coverage images + if: steps.check-e2e.outputs.has-e2e == 'true' + env: + PUBLISHED_EXPORTS: ${{ needs.export.outputs.published-exports }} + WORKSPACE: ${{ needs.prepare.outputs.workspace }} + run: | + echo "Published exports:" + echo "$PUBLISHED_EXPORTS" + echo "" + echo "$PUBLISHED_EXPORTS" | ./scripts/instrument-plugin.sh "$WORKSPACE" + check-backstage-compatibility: name: Check workspace backstage compatibility needs: diff --git a/run-e2e.sh b/run-e2e.sh index 4fa8388cb..4ece5da22 100755 --- a/run-e2e.sh +++ b/run-e2e.sh @@ -29,6 +29,11 @@ set -euo pipefail # # # Use an unpublished git branch of e2e-test-utils (clones and builds locally) # E2E_TEST_UTILS_GIT_REF=owner/rhdh-e2e-test-utils#my-branch ./run-e2e.sh -w tech-radar +# +# # Coverage collection is ENABLED BY DEFAULT +# # Requires e2e-test-utils >= 1.x.x for automatic -coverage image swap +# # To disable for faster local development: +# E2E_COLLECT_COVERAGE=0 ./run-e2e.sh -w tech-radar # ============================================================================= SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -58,6 +63,18 @@ export CATALOG_INDEX_IMAGE="${CATALOG_INDEX_IMAGE:-}" # Nightly mode E2E_NIGHTLY_MODE="${E2E_NIGHTLY_MODE:-false}" +# Coverage collection (Istanbul) — enabled by default +# +# For PR checks: Works now. The auto-publish-pr.yaml workflow builds -coverage +# images (plugin:tag__coverage) that e2e-test-utils will load when available. +# +# For nightly/local: Depends on e2e-test-utils automatic image swap logic +# (PR #95, not yet released). Until that lands, coverage collection will be +# skipped silently (no -coverage images exist). +# +# To disable (faster local dev): E2E_COLLECT_COVERAGE=0 +export E2E_COLLECT_COVERAGE="${E2E_COLLECT_COVERAGE:-1}" + # Local e2e-test-utils: absolute path to use a local build instead of npm E2E_TEST_UTILS_PATH="${E2E_TEST_UTILS_PATH:-}" # Pin specific e2e-test-utils version. @@ -247,6 +264,10 @@ for ws in "${E2E_WORKSPACES[@]}"; do done <<< "$PROJECTS_BLOCK" done +if [[ "${E2E_COLLECT_COVERAGE:-}" == "1" ]]; then + echo "[INFO] Coverage collection enabled (E2E_COLLECT_COVERAGE=1)" +fi + cat > playwright.config.ts < +# ./scripts/instrument-plugin.sh < images.txt +# +# Example: +# echo "ghcr.io/repo/plugin:pr_123__1.0.0" | ./scripts/instrument-plugin.sh workspaces/tech-radar +# +# The script: +# 1. Reads OCI image refs from stdin (one per line) +# 2. For each frontend plugin image: +# - Pulls the production image +# - Extracts plugin path from OCI labels (io.backstage.dynamic-packages) +# - Extracts plugin bundle from the container +# - Instruments JavaScript with nyc (Istanbul) +# - Builds a new coverage image with instrumented files +# - Pushes the coverage image with __coverage tag suffix + +set -euo pipefail + +WORKSPACE="${1:?Usage: $0 }" + +if [[ ! -d "$WORKSPACE" ]]; then + echo "ERROR: Workspace directory not found: $WORKSPACE" >&2 + exit 1 +fi + +if [[ ! -d "$WORKSPACE/metadata" ]]; then + echo "ERROR: No metadata directory found in workspace: $WORKSPACE/metadata" >&2 + exit 1 +fi + +echo "=== Instrumenting published plugin images for E2E coverage ===" +echo "Workspace: $WORKSPACE" +echo "" + +INSTRUMENTED_COUNT=0 +SKIPPED_COUNT=0 + +# Process each published image (format: plain image refs, one per line) +while IFS= read -r PROD_IMAGE; do + [[ -z "$PROD_IMAGE" ]] && continue + + echo "--- Processing: $PROD_IMAGE ---" + + # Extract plugin name from image ref + PLUGIN_NAME=$(basename "${PROD_IMAGE%%:*}") + echo " Plugin: $PLUGIN_NAME" + + # Find metadata file for this plugin + # The metadata filename matches the OCI image name (e.g., backstage-community-plugin-acs.yaml) + METADATA_FILE="${WORKSPACE}/metadata/${PLUGIN_NAME}.yaml" + + if [[ ! -f "$METADATA_FILE" ]]; then + echo " ⚠️ No metadata file found at $METADATA_FILE - skipping" + SKIPPED_COUNT=$((SKIPPED_COUNT + 1)) + continue + fi + + # Check if this is a frontend plugin (only frontend plugins need instrumentation) + PLUGIN_ROLE=$(yq -r '.spec.backstage.role // ""' "$METADATA_FILE") + if [[ "$PLUGIN_ROLE" != "frontend-plugin" ]]; then + echo " Skipping $PLUGIN_ROLE (only frontend plugins need browser coverage)" + SKIPPED_COUNT=$((SKIPPED_COUNT + 1)) + continue + fi + + # Pull production image first (needed to inspect labels) + if ! podman pull "$PROD_IMAGE" 2>&1 | grep -v "WARNING: image platform"; then + echo " ❌ Failed to pull image - skipping" + SKIPPED_COUNT=$((SKIPPED_COUNT + 1)) + continue + fi + + # Extract plugin path from OCI image labels (preferred method) + # The io.backstage.dynamic-packages label contains base64-encoded JSON + # with plugin metadata including the directory path inside the container + PACKAGES_LABEL=$(podman inspect "$PROD_IMAGE" --format '{{index .Labels "io.backstage.dynamic-packages"}}' 2>/dev/null || echo "") + + PLUGIN_PATH="" + if [[ -n "$PACKAGES_LABEL" && "$PACKAGES_LABEL" != "" ]]; then + # Decode base64 and extract first plugin name + # Expected JSON: [{"name":"backstage-community-plugin-acs","version":"0.2.0",...}] + # The "name" field is the directory path inside the container + PLUGIN_PATH=$(echo "$PACKAGES_LABEL" | base64 -d 2>/dev/null | jq -r '.[0].name // empty' 2>/dev/null || echo "") + if [[ -n "$PLUGIN_PATH" ]]; then + echo " Plugin path (from OCI label): $PLUGIN_PATH" + fi + fi + + # Fallback: Extract from dynamicArtifact in metadata if label doesn't exist + if [[ -z "$PLUGIN_PATH" ]]; then + echo " No io.backstage.dynamic-packages label - using metadata fallback" + DYNAMIC_ARTIFACT=$(yq -r '.spec.dynamicArtifact // ""' "$METADATA_FILE") + + # Format: "oci://image:tag!path" or "oci://image:tag" + if [[ "$DYNAMIC_ARTIFACT" =~ !(.+)$ ]]; then + PLUGIN_PATH="${BASH_REMATCH[1]}" + echo " Plugin path (from metadata): $PLUGIN_PATH" + else + # No explicit path — use plugin name as path + PLUGIN_PATH="$PLUGIN_NAME" + echo " Plugin path (default): $PLUGIN_PATH" + fi + fi + + if [[ -z "$PLUGIN_PATH" ]]; then + echo " ⚠️ Could not determine plugin path - skipping" + SKIPPED_COUNT=$((SKIPPED_COUNT + 1)) + continue + fi + + # Create temp container and extract plugin bundle + WORK_DIR=$(mktemp -d) + CID=$(podman create "$PROD_IMAGE") + + if ! podman cp "$CID:$PLUGIN_PATH/dist" "$WORK_DIR/dist-original"; then + echo " ❌ Failed to extract plugin bundle from container - skipping" + podman rm "$CID" || true + rm -rf "$WORK_DIR" + SKIPPED_COUNT=$((SKIPPED_COUNT + 1)) + continue + fi + + podman rm "$CID" + + # Instrument with nyc (pinned version for reproducibility) + # Must run from work directory to avoid "outside project root" errors + echo " Instrumenting with Istanbul/nyc..." + if ! (cd "$WORK_DIR" && npx --yes nyc@18.0.0 instrument dist-original dist-instrumented --source-map); then + echo " ❌ Instrumentation failed - skipping" + rm -rf "$WORK_DIR" + SKIPPED_COUNT=$((SKIPPED_COUNT + 1)) + continue + fi + + # Verify instrumentation + JS_COUNT=$(grep -r "__coverage__" "$WORK_DIR/dist-instrumented/" --include="*.js" -l 2>/dev/null | wc -l | tr -d ' ') + if [[ "$JS_COUNT" -eq 0 ]]; then + echo " ❌ No __coverage__ found in instrumented files - skipping" + rm -rf "$WORK_DIR" + SKIPPED_COUNT=$((SKIPPED_COUNT + 1)) + continue + fi + echo " ✓ Instrumented $JS_COUNT JS files" + + # Build coverage image (copy instrumented files over production image) + cat > "$WORK_DIR/Containerfile" < [workspace...] +# +# Example: +# E2E_COLLECT_COVERAGE=1 ./run-e2e.sh -w tech-radar +# ./scripts/report-coverage.sh tech-radar +# +# The script: +# 1. Merges per-test coverage JSONs (written by the _coverageCollector fixture) +# into a single coverage-final.json using nyc merge +# 2. Generates lcov and text-summary reports via nyc report +# 3. Uploads lcov to Codecov for each workspace with cross-repo attribution +# +# Required environment: +# CODECOV_TOKEN - Codecov upload token (org-level for cross-repo uploads) + +set -euo pipefail + +if [[ $# -eq 0 ]]; then + echo "Usage: $0 [workspace...]" >&2 + exit 1 +fi + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +WORKSPACES=("$@") + +COVERAGE_JSON_DIR="node_modules/.cache/e2e-test-results/coverage" + +if ! compgen -G "$REPO_ROOT/$COVERAGE_JSON_DIR/*.json" >/dev/null 2>&1; then + echo "[INFO] No coverage data found (no instrumented plugins loaded?)" + exit 0 +fi + +echo "" +echo "[INFO] Merging coverage data with nyc..." +mkdir -p "$REPO_ROOT/.nyc_output" +npx nyc@18.0.0 merge "$REPO_ROOT/$COVERAGE_JSON_DIR" "$REPO_ROOT/.nyc_output/out.json" +(cd "$REPO_ROOT" && npx nyc@18.0.0 report --reporter=lcov --reporter=text-summary --report-dir coverage) + +if [[ ${#WORKSPACES[@]} -gt 1 ]]; then + echo "[WARN] Multi-workspace coverage upload is not supported." >&2 + echo "[WARN] Coverage is merged across workspaces but uploaded with per-workspace flags." >&2 + echo "[WARN] This produces misleading coverage percentages in Codecov." >&2 + echo "[WARN] Skipping upload. Run report-coverage.sh once per workspace to upload." >&2 +else + echo "[INFO] Uploading E2E coverage to Codecov..." + for ws in "${WORKSPACES[@]}"; do + if [[ -f "$REPO_ROOT/workspaces/$ws/source.json" ]]; then + "$SCRIPT_DIR/upload-coverage.sh" "$ws" || \ + echo "[WARN] Coverage upload failed for $ws (non-fatal)" + fi + done +fi diff --git a/scripts/upload-coverage.sh b/scripts/upload-coverage.sh new file mode 100755 index 000000000..cc0e92471 --- /dev/null +++ b/scripts/upload-coverage.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +# +# Upload Istanbul/lcov coverage to Codecov with cross-repo attribution. +# +# Usage: +# ./scripts/upload-coverage.sh +# +# Example: +# E2E_COLLECT_COVERAGE=1 ./run-e2e.sh -w tech-radar +# ./scripts/upload-coverage.sh tech-radar +# +# The script reads source.json to determine the upstream repo and SHA, +# then uploads the lcov coverage to Codecov attributed to that repo. +# +# Required environment: +# CODECOV_TOKEN - Codecov upload token (org-level for cross-repo uploads) + +set -euo pipefail + +readonly AWK_FIRST_FIELD='{print $1}' + +WORKSPACE="${1:?Usage: $0 }" +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +WORKSPACE_DIR="$REPO_ROOT/workspaces/$WORKSPACE" +COVERAGE_DIR="$REPO_ROOT/coverage" +LCOV_FILE="$COVERAGE_DIR/lcov.info" + +if [[ ! -f "$LCOV_FILE" ]]; then + echo "ERROR: No lcov file found at $LCOV_FILE" >&2 + echo "Run tests with E2E_COLLECT_COVERAGE=1 first" >&2 + exit 1 +fi + +if [[ ! -f "$WORKSPACE_DIR/source.json" ]]; then + echo "ERROR: source.json not found at $WORKSPACE_DIR/source.json" >&2 + exit 1 +fi + +REPO_URL=$(jq -r '.repo // empty' "$WORKSPACE_DIR/source.json") +REPO_REF=$(jq -r '.["repo-ref"] // empty' "$WORKSPACE_DIR/source.json") + +if [[ -z "$REPO_URL" || "$REPO_URL" == "null" ]]; then + echo "ERROR: Invalid or missing 'repo' field in source.json" >&2 + exit 1 +fi + +if [[ -z "$REPO_REF" || "$REPO_REF" == "null" ]]; then + echo "ERROR: Invalid or missing 'repo-ref' field in source.json" >&2 + exit 1 +fi + +# Codecov --sha requires a 40-char commit SHA. source.json repo-ref can be a +# tag name (e.g., "v1.49.4") — resolve it to a commit SHA via git ls-remote. +# For annotated tags, ls-remote returns the tag object and the dereferenced +# commit (^{}); tail -1 picks the commit in both cases. +# +# OPTIMIZATION OPPORTUNITY: This network call could be eliminated by storing +# full 40-char SHAs in source.json (updated by update-plugins-repo-refs.yaml). +# For now, we resolve at upload time and cache the result in /tmp. +if [[ ! "$REPO_REF" =~ ^[0-9a-f]{40}$ ]]; then + CACHE_FILE="/tmp/codecov-sha-${WORKSPACE}.cache" + if [[ -f "$CACHE_FILE" ]]; then + RESOLVED=$(cat "$CACHE_FILE") + echo " Using cached SHA for '$REPO_REF': $RESOLVED" + else + RESOLVED=$(git ls-remote "$REPO_URL" "$REPO_REF" "${REPO_REF}^{}" 2>/dev/null | tail -1 | awk "$AWK_FIRST_FIELD") + if [[ -n "$RESOLVED" ]]; then + echo " Resolved ref '$REPO_REF' -> $RESOLVED" + echo "$RESOLVED" > "$CACHE_FILE" + else + echo "ERROR: Could not resolve '$REPO_REF' to a commit SHA" >&2 + echo "Codecov requires a valid 40-char commit SHA" >&2 + exit 1 + fi + fi + REPO_REF="$RESOLVED" +fi + +# Extract GitHub slug from repo URL (e.g., "redhat-developer/rhdh-plugins") +SLUG=$(echo "$REPO_URL" | sed 's|https://github.com/||; s|\.git$||') + +echo "=== Uploading E2E coverage to Codecov ===" +echo " Workspace: $WORKSPACE" +echo " LCOV file: $LCOV_FILE" +echo " Target repo: $SLUG" +echo " Target SHA: $REPO_REF" +echo " Flag: e2e-$WORKSPACE" + +if [[ -z "${CODECOV_TOKEN:-}" ]]; then + echo "" + echo "[WARN] CODECOV_TOKEN is not set — skipping Codecov upload" + echo "[INFO] Coverage report is still available locally at: $LCOV_FILE" + exit 0 +fi + +# Download Codecov CLI binary with SHA256 verification. +# Uses the standalone Go binary (not pip codecov-cli) for supply-chain safety. +CODECOV_VERSION="v11.2.8" +CODECOV_BIN="/tmp/codecov" +if [[ ! -x "$CODECOV_BIN" ]]; then + OS=$(uname -s | tr '[:upper:]' '[:lower:]') + case "$OS" in + linux) CODECOV_OS="linux" ;; + darwin) CODECOV_OS="macos" ;; + *) + echo "ERROR: Unsupported OS: $OS" >&2 + exit 1 + ;; + esac + + echo "" + echo "Downloading Codecov CLI $CODECOV_VERSION for ${CODECOV_OS}..." + curl -sL -o "$CODECOV_BIN" "https://cli.codecov.io/${CODECOV_VERSION}/${CODECOV_OS}/codecov" + curl -sL -o "${CODECOV_BIN}.SHA256SUM" "https://cli.codecov.io/${CODECOV_VERSION}/${CODECOV_OS}/codecov.SHA256SUM" + + EXPECTED=$(awk "$AWK_FIRST_FIELD" "${CODECOV_BIN}.SHA256SUM") + if command -v sha256sum &>/dev/null; then + ACTUAL=$(sha256sum "$CODECOV_BIN" | awk "$AWK_FIRST_FIELD") + else + ACTUAL=$(shasum -a 256 "$CODECOV_BIN" | awk "$AWK_FIRST_FIELD") + fi + rm -f "${CODECOV_BIN}.SHA256SUM" + + if [[ "$EXPECTED" != "$ACTUAL" ]]; then + echo "ERROR: Codecov CLI checksum verification failed" >&2 + echo " Expected: $EXPECTED" >&2 + echo " Actual: $ACTUAL" >&2 + rm -f "$CODECOV_BIN" + exit 1 + fi + + chmod +x "$CODECOV_BIN" + echo " Codecov CLI downloaded and verified" +fi + +echo "" +# Codecov upload failures are intentionally non-blocking (exit 0). +# Coverage is informational — CI jobs should not fail if Codecov is down or +# has transient errors. The lcov report is still available locally for review. +# This approach prioritizes CI stability while ensuring coverage visibility when +# Codecov is available. +if "$CODECOV_BIN" upload-process \ + --file "$LCOV_FILE" \ + --flag "e2e-$WORKSPACE" \ + --sha "$REPO_REF" \ + --slug "$SLUG" \ + --token "$CODECOV_TOKEN" \ + --git-service github \ + --name "overlay-e2e-$WORKSPACE" \ + --disable-search \ + --fail-on-error; then + echo "" + echo "=== Upload complete ===" + echo " View coverage at: https://app.codecov.io/gh/$SLUG/commit/$REPO_REF" + echo " Filter by flag: e2e-$WORKSPACE" +else + echo "" + echo "==================================================" + echo " ⚠️ Codecov upload failed" + echo "==================================================" + echo " This is non-fatal — coverage data is still available locally" + echo " LCOV report: $LCOV_FILE" + echo " Target repo: $SLUG" + echo " Target SHA: $REPO_REF" + echo "==================================================" + # Exit 0 (success) — upload failure should not fail the CI job + exit 0 +fi