ci: make smoke gates fail when in-scope suites are skipped (#530)

Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>

Author: ian-flores

.github/workflows/connect-smoke.yml

@@ -312,8 +312,16 @@ jobs:
             echo "Version-matrix job failed; cannot confirm smoke scope"
             exit 1
           fi
+          # If Connect smoke was in scope for this PR (paths matched) it MUST
+          # have succeeded — a skip here means it was gated out (missing license,
+          # excluded actor, or self-skip) rather than legitimately not relevant,
+          # so treat "in scope but not success" as a hard gate failure.
+          if [ "${{ needs.changes.outputs.relevant }}" = "true" ] && [ "${{ needs.connect-smoke.result }}" != "success" ]; then
+            echo "Connect smoke was in scope for this PR but did not succeed (result=${{ needs.connect-smoke.result }})"
+            exit 1
+          fi
           if [ "${{ needs.connect-smoke.result }}" = "failure" ] || [ "${{ needs.connect-smoke.result }}" = "cancelled" ]; then
             echo "Connect smoke tests failed or were cancelled"
             exit 1
           fi
-          echo "Connect smoke tests passed or were skipped"
+          echo "Connect smoke tests passed or were legitimately out of scope"

.github/workflows/mock-idp-e2e.yml

@@ -30,23 +30,6 @@ on:
       - 'Dockerfile'
       - '.github/workflows/mock-idp-e2e.yml'
   pull_request:
-    paths:
-      - 'src/vip/auth.py'
-      - 'src/vip/idp.py'
-      - 'src/vip/totp.py'
-      - 'src/vip/cli.py'
-      - 'src/vip_tests/conftest.py'
-      - 'src/vip_tests/connect/test_auth.py'
-      - 'src/vip_tests/connect/test_auth.feature'
-      - 'src/vip_tests/workbench/test_auth.py'
-      - 'src/vip_tests/workbench/test_auth.feature'
-      - 'docker/keycloak/**'
-      - 'docker/connect/**'
-      - 'docker/workbench/**'
-      - 'docker/tls/**'
-      - 'compose.mock-idp.yml'
-      - 'Dockerfile'
-      - '.github/workflows/mock-idp-e2e.yml'
   schedule:
     - cron: "30 5 * * *" # daily at 5:30am UTC
   workflow_dispatch:
@@ -59,6 +42,39 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
+  changes:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: read
+    outputs:
+      relevant: ${{ steps.filter.outputs.relevant }}
+    steps:
+      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          persist-credentials: false
+      - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706 # v4.0.2
+        id: filter
+        with:
+          filters: |
+            relevant:
+              - 'src/vip/auth.py'
+              - 'src/vip/idp.py'
+              - 'src/vip/totp.py'
+              - 'src/vip/cli.py'
+              - 'src/vip_tests/conftest.py'
+              - 'src/vip_tests/connect/test_auth.py'
+              - 'src/vip_tests/connect/test_auth.feature'
+              - 'src/vip_tests/workbench/test_auth.py'
+              - 'src/vip_tests/workbench/test_auth.feature'
+              - 'docker/keycloak/**'
+              - 'docker/connect/**'
+              - 'docker/workbench/**'
+              - 'docker/tls/**'
+              - 'compose.mock-idp.yml'
+              - 'Dockerfile'
+              - '.github/workflows/mock-idp-e2e.yml'
+
   # Emit the Workbench version list for the E2E job. PR/push runs test only the
   # latest release (single required check, unchanged). Scheduled/manual runs fan
   # out across the Workbench support window (current + 2 back) — same list as
@@ -80,7 +96,11 @@ jobs:
           fi
 
   mock-idp-e2e:
-    needs: [set-matrix]
+    needs: [set-matrix, changes]
+    if: >-
+      needs.changes.outputs.relevant == 'true'
+      || github.event_name == 'schedule'
+      || github.event_name == 'workflow_dispatch'
     # Keep the original job name on the release (PR/push) leg so the PR-time
     # check name is unchanged; suffix only the pinned scheduled/manual legs.
     # `!=` first so the release leg yields '' rather than tripping the falsy-`||` trap.
@@ -227,19 +247,29 @@ jobs:
   status:
     name: Mock-IdP E2E Status
     if: always()
-    needs: [set-matrix, mock-idp-e2e]
+    needs: [changes, set-matrix, mock-idp-e2e]
     runs-on: ubuntu-latest
     steps:
       - name: Check result
         run: |
+          if [ "${{ needs.changes.result }}" = "failure" ] || [ "${{ needs.changes.result }}" = "cancelled" ]; then
+            echo "Change-detection job failed; cannot confirm E2E scope"
+            exit 1
+          fi
           # A failed set-matrix skips (not fails) mock-idp-e2e, which the
           # check below would otherwise read as a clean skip.
           if [ "${{ needs.set-matrix.result }}" = "failure" ] || [ "${{ needs.set-matrix.result }}" = "cancelled" ]; then
             echo "Version-matrix job failed; cannot confirm E2E scope"
             exit 1
           fi
+          # If auth paths changed, the E2E suite was in scope and MUST have
+          # succeeded — a skip means it was gated out, not legitimately absent.
+          if [ "${{ needs.changes.outputs.relevant }}" = "true" ] && [ "${{ needs.mock-idp-e2e.result }}" != "success" ]; then
+            echo "Mock-IdP E2E was in scope but did not succeed (result=${{ needs.mock-idp-e2e.result }})"
+            exit 1
+          fi
           if [ "${{ needs.mock-idp-e2e.result }}" = "failure" ] || [ "${{ needs.mock-idp-e2e.result }}" = "cancelled" ]; then
             echo "Mock-IdP E2E tests failed or were cancelled"
             exit 1
           fi
-          echo "Mock-IdP E2E tests passed or were skipped"
+          echo "Mock-IdP E2E tests passed or were legitimately out of scope"

.github/workflows/packagemanager-smoke.yml

@@ -94,16 +94,19 @@ jobs:
         with:
           persist-credentials: false
 
-      # Skip the entire job if the license secret is not configured
+      # The job only runs when PM smoke is in scope (changed paths) or on a
+      # scheduled/manual run -- all cases where the suite is meant to execute.
+      # A missing license secret therefore means the required gate cannot be
+      # satisfied, so fail loudly instead of silently skipping every test step
+      # (which would otherwise report a green required check with zero coverage).
       - name: Check for RSPM_LICENSE
         id: license
         run: |
           if [ -z "${{ secrets.RSPM_LICENSE }}" ]; then
-            echo "RSPM_LICENSE secret not configured — skipping PM smoke tests"
-            echo "available=false" >> "$GITHUB_OUTPUT"
-          else
-            echo "available=true" >> "$GITHUB_OUTPUT"
+            echo "::error::RSPM_LICENSE secret is not configured; Package Manager smoke cannot run. Set the RSPM_LICENSE repository secret."
+            exit 1
           fi
+          echo "available=true" >> "$GITHUB_OUTPUT"
 
       # Start Package Manager in Docker
       - name: Start Package Manager
@@ -358,8 +361,12 @@ jobs:
             echo "Version-matrix job failed; cannot confirm smoke scope"
             exit 1
           fi
+          if [ "${{ needs.changes.outputs.relevant }}" = "true" ] && [ "${{ needs.packagemanager-smoke.result }}" != "success" ]; then
+            echo "Package Manager smoke was in scope but did not succeed (result=${{ needs.packagemanager-smoke.result }}); is RSPM_LICENSE configured?"
+            exit 1
+          fi
           if [ "${{ needs.packagemanager-smoke.result }}" = "failure" ] || [ "${{ needs.packagemanager-smoke.result }}" = "cancelled" ]; then
             echo "Package Manager smoke tests failed or were cancelled"
             exit 1
           fi
-          echo "Package Manager smoke tests passed or were skipped"
+          echo "Package Manager smoke tests passed or were legitimately out of scope"

.github/workflows/workbench-smoke.yml

@@ -329,8 +329,12 @@ jobs:
             echo "Version-matrix job failed; cannot confirm smoke scope"
             exit 1
           fi
+          if [ "${{ needs.changes.outputs.relevant }}" = "true" ] && [ "${{ needs.workbench-smoke.result }}" != "success" ]; then
+            echo "Workbench smoke was in scope for this PR but did not succeed (result=${{ needs.workbench-smoke.result }})"
+            exit 1
+          fi
           if [ "${{ needs.workbench-smoke.result }}" = "failure" ] || [ "${{ needs.workbench-smoke.result }}" = "cancelled" ]; then
             echo "Workbench smoke tests failed or were cancelled"
             exit 1
           fi
-          echo "Workbench smoke tests passed or were skipped"
+          echo "Workbench smoke tests passed or were legitimately out of scope"

AGENTS.md

@@ -229,7 +229,8 @@ The report lives in `report/` and reads `report/results.json` (written by pytest
 -   **`ci.yml`** -- on every PR/push: ruff lint/format (pinned to 0.15.0), mypy type-check, zizmor actions-lint, a runtime dependency audit, and selftests (Ubuntu + macOS, Python 3.10 and 3.12). A `changes` path-filter gates the expensive jobs, while `Lint & Format` and `Selftests Status` always run as required checks. Uses uv cache.
 -   **`preview.yml`** -- runs selftests, renders Quarto report, publishes PR preview to gh-pages via `rossjrw/pr-preview-action@v1`. Uses uv and Quarto caches.
 -   **`pr-title.yml`** -- validates PR titles follow conventional commit format. Squash merges use the PR title as the commit message.
--   **Smoke workflows** (`connect-smoke.yml`, `workbench-smoke.yml`, `packagemanager-smoke.yml`, `mock-idp-e2e.yml`) -- run the product suites against real containers. On PR/push each tests a single latest version (change-gated, no new required checks); on `schedule` (nightly, staggered hourly) and `workflow_dispatch` a `set-matrix` job fans each out across the product version support window (current + 2 back). Bump the pinned tags in each workflow's `set-matrix` step when a new product release ships.
+-   **Smoke workflows** (`connect-smoke.yml`, `workbench-smoke.yml`, `packagemanager-smoke.yml`, `mock-idp-e2e.yml`) -- run the product suites against real containers. On PR/push each tests a single latest version (change-gated via a `changes` paths-filter job); on `schedule` (nightly, staggered hourly) and `workflow_dispatch` a `set-matrix` job fans each out across the product version support window (current + 2 back). Bump the pinned tags in each workflow's `set-matrix` step when a new product release ships. Each workflow's `*-status` aggregation job is scope-aware: it passes when the suite was legitimately out of scope (the PR's paths didn't match) but **fails** when the suite was in scope (`changes.relevant == 'true'`) yet did not succeed -- so a path-gated skip, an excluded actor, or a missing license secret can no longer report a green required check without the suite having run. The Connect, Workbench, and Package Manager `*-status` jobs are required merge checks.
+    `mock-idp-e2e.yml` is structured the same way so `Mock-IdP E2E Status` can be promoted to a required check via a separate admin action.
 -   **`add-to-team-project.yml`** -- when a `team: connect`, `team: workbench`, or `team: package manager` label is added to an issue, adds it to that product team's org-level GitHub project board. Ported from rstudio/helm. Requires the cross-org `POSIT_PLATFORM_CLIENT_ID`/`POSIT_PLATFORM_PEM` app secrets.
 -   **`weekly-summary.yml`** -- Mondays (and on demand via `workflow_dispatch`) gathers the week's merged PRs, has Claude pick the highlights via Bedrock, and posts a Slack summary; `pull_request` runs are a dry run that builds and logs the payload without posting. Requires the `SLACK_WEBHOOK_VIP_WEEKLY_SUMMARY` secret and permission to assume the `claude-code-gha` AWS role.