ci(e2e): Phase 5a — nightly matrix framework + mod-miss + diagnostic cells

.changeset/e2e-phase-5a-nightly-framework.md

@@ -0,0 +1,5 @@
+---
+"playground-cli": patch
+---
+
+E2E suite now has a `test-nightly-no-publish` matrix that runs only on the daily schedule (06:00 UTC) and `workflow_dispatch`. Adds two nightly-only cells: `nightly-mod-miss` (registry-miss path for unknown domains) and `nightly-diagnostic` (DOT_DEPLOY_VERBOSE / DOT_MEMORY_TRACE coverage). Per-PR runs are unaffected.

.github/actions/setup-e2e/action.yml

@@ -0,0 +1,47 @@
+name: Setup E2E environment
+description: Install pnpm/node/bun/kubo, run pnpm install, and resolve DOT_TAG from the trigger event. The CALLER must run actions/checkout BEFORE this action — local composite actions can't be located until the repo is on disk.
+
+outputs:
+    tag:
+        description: The DOT_TAG value resolved from github.event_name (e2e-ci-{nightly,dispatch,pr}).
+        value: ${{ steps.tag.outputs.tag }}
+
+runs:
+    using: composite
+    steps:
+        - uses: pnpm/action-setup@v4
+
+        - uses: actions/setup-node@v4
+          with:
+              node-version: "22"
+
+        - uses: oven-sh/setup-bun@v2
+          with:
+              bun-version: latest
+
+        - name: Install Kubo (IPFS)
+          shell: bash
+          run: |
+              wget -q https://dist.ipfs.tech/kubo/v0.32.1/kubo_v0.32.1_linux-amd64.tar.gz
+              tar xf kubo_v0.32.1_linux-amd64.tar.gz
+              sudo mv kubo/ipfs /usr/local/bin/
+              ipfs init --profile=test
+              ipfs version
+
+        - name: pnpm install
+          shell: bash
+          run: pnpm install
+
+        - name: Resolve DOT_TAG from trigger
+          id: tag
+          shell: bash
+          run: |
+              # Phase 1 triggers: pull_request / push:main / schedule / workflow_dispatch.
+              # Phase 7 will add `release` (with TAG=e2e-ci-release).
+              case "${{ github.event_name }}" in
+                  schedule)            TAG=e2e-ci-nightly  ;;
+                  workflow_dispatch)   TAG=e2e-ci-dispatch ;;
+                  *)                   TAG=e2e-ci-pr       ;;
+              esac
+              echo "tag=$TAG" >> "$GITHUB_OUTPUT"
+              echo "Resolved DOT_TAG=$TAG (event=${{ github.event_name }})"

.github/workflows/e2e.yml

@@ -29,64 +29,40 @@ jobs:
             matrix:
                 include:
                     - cell: pr-install
+                      # source: e2e/cli/install.test.ts → describe("dot install")
                       pattern: "dot install"
                     - cell: pr-preflight
+                      # sources: e2e/cli/build.test.ts → describe("dot build")
+                      #          e2e/cli/deploy.test.ts → describe("dot deploy — preflight and validation")
                       pattern: "dot build|preflight and validation"
                     - cell: pr-mod
+                      # source: e2e/cli/mod.test.ts → describe("dot mod — clone")
                       pattern: "dot mod — clone"
                     - cell: pr-init-session
+                      # sources: e2e/cli/init.test.ts → describe("dot init …")
+                      #          e2e/cli/session.test.ts → describe("session management")
                       pattern: "dot init|session management"
         outputs:
-            tag: ${{ steps.tag.outputs.tag }}
+            tag: ${{ steps.setup.outputs.tag }}
         steps:
+            # checkout must run before the local composite action can be loaded.
             - uses: actions/checkout@v4
 
-            - uses: pnpm/action-setup@v4
-
-            - uses: actions/setup-node@v4
-              with:
-                  node-version: "22"
-
-            - uses: oven-sh/setup-bun@v2
-              with:
-                  bun-version: latest
-
-            - name: Install Kubo (IPFS)
-              run: |
-                  wget -q https://dist.ipfs.tech/kubo/v0.32.1/kubo_v0.32.1_linux-amd64.tar.gz
-                  tar xf kubo_v0.32.1_linux-amd64.tar.gz
-                  sudo mv kubo/ipfs /usr/local/bin/
-                  ipfs init --profile=test
-                  ipfs version
-
-            - run: pnpm install
-
-            - name: Resolve DOT_TAG from trigger
-              id: tag
-              shell: bash
-              run: |
-                  # Phase 1 triggers: pull_request / push:main / schedule / workflow_dispatch.
-                  # Phase 7 will add `release` (with TAG=e2e-ci-release).
-                  case "${{ github.event_name }}" in
-                      schedule)            TAG=e2e-ci-nightly  ;;
-                      workflow_dispatch)   TAG=e2e-ci-dispatch ;;
-                      *)                   TAG=e2e-ci-pr       ;;
-                  esac
-                  echo "tag=$TAG" >> "$GITHUB_OUTPUT"
-                  echo "Resolved DOT_TAG=$TAG (event=${{ github.event_name }})"
+            - id: setup
+              uses: ./.github/actions/setup-e2e
 
             - name: Run E2E cell (one retry on transient testnet failures)
               uses: nick-fields/retry@v3
               with:
                   timeout_minutes: 20
                   max_attempts: 2
                   retry_wait_seconds: 30
-                  command: pnpm exec vitest run --config e2e/vitest.config.ts -t "${{ matrix.pattern }}"
+                  command: pnpm exec vitest run --config e2e/vitest.config.ts ${{ matrix.testFile || '' }} -t "${{ matrix.pattern }}"
               env:
                   TEST_TEMPLATE_DOMAIN: dot-cli-mod-fixture.dot
                   TEST_TEMPLATE_REPO: https://github.com/paritytech/Rock-Paper-Scissors
                   DOT_DEPLOY_VERBOSE: "1"
-                  DOT_TAG: ${{ steps.tag.outputs.tag }}
+                  DOT_TAG: ${{ steps.setup.outputs.tag }}
                   DOT_TELEMETRY: "1"
 
             - name: Upload forensic artefacts
@@ -109,62 +85,83 @@ jobs:
             matrix:
                 include:
                     - cell: pr-deploy-frontend
+                      # source: e2e/cli/deploy.test.ts → describe("dot deploy --playground — full pipeline …")
                       pattern: "full pipeline"
                     - cell: pr-deploy-foundry
+                      # source: e2e/cli/deploy.test.ts → describe("dot deploy — foundry …")
                       pattern: "deploy — foundry"
                     # pr-deploy-cdm dropped pending fixture upgrade — see the
                     # describe.skip block in e2e/cli/deploy.test.ts. Phase 5
                     # follow-up will land a real flipper fixture and re-add
                     # this cell.
         steps:
+            # checkout must run before the local composite action can be loaded.
             - uses: actions/checkout@v4
 
-            - uses: pnpm/action-setup@v4
+            - id: setup
+              uses: ./.github/actions/setup-e2e
 
-            - uses: actions/setup-node@v4
+            - name: Run E2E cell (one retry on transient testnet failures)
+              uses: nick-fields/retry@v3
               with:
-                  node-version: "22"
+                  timeout_minutes: 25
+                  max_attempts: 2
+                  retry_wait_seconds: 30
+                  command: pnpm exec vitest run --config e2e/vitest.config.ts ${{ matrix.testFile || '' }} -t "${{ matrix.pattern }}"
+              env:
+                  TEST_TEMPLATE_DOMAIN: dot-cli-mod-fixture.dot
+                  TEST_TEMPLATE_REPO: https://github.com/paritytech/Rock-Paper-Scissors
+                  DOT_DEPLOY_VERBOSE: "1"
+                  DOT_TAG: ${{ steps.setup.outputs.tag }}
+                  DOT_TELEMETRY: "1"
 
-            - uses: oven-sh/setup-bun@v2
+            - name: Upload forensic artefacts
+              if: always()
+              continue-on-error: true
+              uses: actions/upload-artifact@v4
               with:
-                  bun-version: latest
-
-            - name: Install Kubo (IPFS)
-              run: |
-                  wget -q https://dist.ipfs.tech/kubo/v0.32.1/kubo_v0.32.1_linux-amd64.tar.gz
-                  tar xf kubo_v0.32.1_linux-amd64.tar.gz
-                  sudo mv kubo/ipfs /usr/local/bin/
-                  ipfs init --profile=test
-                  ipfs version
+                  name: e2e-reports-${{ matrix.cell }}
+                  path: e2e-reports/
+                  retention-days: 7
+                  if-no-files-found: ignore
 
-            - run: pnpm install
+    test-nightly-no-publish:
+        name: "E2E · ${{ matrix.cell }}"
+        if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+        runs-on: ubuntu-latest
+        timeout-minutes: 25
+        strategy:
+            fail-fast: false
+            max-parallel: 5
+            matrix:
+                include:
+                    - cell: nightly-mod-miss
+                      # source: e2e/cli/mod.test.ts → describe("dot mod — registry miss")
+                      pattern: "dot mod — registry miss"
+                    - cell: nightly-diagnostic
+                      # source: e2e/cli/diagnostic.test.ts → describe("diagnostic mode")
+                      # testFile scoping skips globalSetup chain-RPC calls (no chain access needed for diagnostic flag tests).
+                      pattern: "diagnostic mode"
+                      testFile: e2e/cli/diagnostic.test.ts
+        steps:
+            # checkout must run before the local composite action can be loaded.
+            - uses: actions/checkout@v4
 
-            - name: Resolve DOT_TAG from trigger
-              id: tag
-              shell: bash
-              run: |
-                  # Phase 1 triggers: pull_request / push:main / schedule / workflow_dispatch.
-                  # Phase 7 will add `release` (with TAG=e2e-ci-release).
-                  case "${{ github.event_name }}" in
-                      schedule)            TAG=e2e-ci-nightly  ;;
-                      workflow_dispatch)   TAG=e2e-ci-dispatch ;;
-                      *)                   TAG=e2e-ci-pr       ;;
-                  esac
-                  echo "tag=$TAG" >> "$GITHUB_OUTPUT"
-                  echo "Resolved DOT_TAG=$TAG (event=${{ github.event_name }})"
+            - id: setup
+              uses: ./.github/actions/setup-e2e
 
             - name: Run E2E cell (one retry on transient testnet failures)
               uses: nick-fields/retry@v3
               with:
-                  timeout_minutes: 25
+                  timeout_minutes: 20
                   max_attempts: 2
                   retry_wait_seconds: 30
-                  command: pnpm exec vitest run --config e2e/vitest.config.ts -t "${{ matrix.pattern }}"
+                  command: pnpm exec vitest run --config e2e/vitest.config.ts ${{ matrix.testFile || '' }} -t "${{ matrix.pattern }}"
               env:
                   TEST_TEMPLATE_DOMAIN: dot-cli-mod-fixture.dot
                   TEST_TEMPLATE_REPO: https://github.com/paritytech/Rock-Paper-Scissors
                   DOT_DEPLOY_VERBOSE: "1"
-                  DOT_TAG: ${{ steps.tag.outputs.tag }}
+                  DOT_TAG: ${{ steps.setup.outputs.tag }}
                   DOT_TELEMETRY: "1"
 
             - name: Upload forensic artefacts
@@ -179,12 +176,15 @@ jobs:
 
     report:
         name: E2E Report
-        needs: [test-no-publish, test-publish]
+        needs: [test-no-publish, test-publish, test-nightly-no-publish]
         if: always()
         runs-on: ubuntu-latest
         timeout-minutes: 5
         steps:
             - name: Download JUnit + forensic artefacts
+              # Phase 1 has one upload (e2e-reports-current). When Phase 4 adds
+              # the matrix, drop `merge-multiple: true` and parse per-cell sub-dirs
+              # so per-leg junit.xml don't collide on the same flat path.
               uses: actions/download-artifact@v4
               with:
                   pattern: e2e-reports-*
@@ -217,16 +217,22 @@ jobs:
               env:
                   AGGREGATE_PUBLISH: ${{ needs.test-publish.result }}
                   AGGREGATE_NO_PUBLISH: ${{ needs.test-no-publish.result }}
+                  AGGREGATE_NIGHTLY_NO_PUBLISH: ${{ needs.test-nightly-no-publish.result }}
                   RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
                   TAG: ${{ needs.test-no-publish.outputs.tag }}
               shell: /usr/bin/bash -euo pipefail {0}
               run: |
-                  if [ "$AGGREGATE_PUBLISH" = "success" ] && [ "$AGGREGATE_NO_PUBLISH" = "success" ]; then
-                      AGGREGATE=success
-                  elif [ "$AGGREGATE_PUBLISH" = "cancelled" ] || [ "$AGGREGATE_NO_PUBLISH" = "cancelled" ]; then
+                  # Gather results across all matrices. Add each new matrix job
+                  # here when introducing one (Phase 5b+). 'skipped' (a matrix
+                  # that didn't run because of an event-name gate) doesn't
+                  # count as failure.
+                  all_results="$AGGREGATE_PUBLISH $AGGREGATE_NO_PUBLISH $AGGREGATE_NIGHTLY_NO_PUBLISH"
+                  if echo "$all_results" | grep -q failure; then
+                      AGGREGATE=failure
+                  elif echo "$all_results" | grep -q cancelled; then
                       AGGREGATE=cancelled
                   else
-                      AGGREGATE=failure
+                      AGGREGATE=success
                   fi
 
                   emoji() { case "$1" in
@@ -358,7 +364,7 @@ jobs:
             - name: Open failure issue
               if: >
                   (github.event_name == 'schedule' || github.event_name == 'release') &&
-                  (needs.test-no-publish.result != 'success' || needs.test-publish.result != 'success')
+                  (needs.test-no-publish.result != 'success' || needs.test-publish.result != 'success' || needs.test-nightly-no-publish.result != 'success')
               env:
                   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
                   TAG: ${{ needs.test-no-publish.outputs.tag }}

CLAUDE.md

@@ -56,7 +56,7 @@ These are things that aren't self-evident from reading the code and have bitten
 
 - **Local launcher:** `tools/e2e-local.sh [smoke|pr|nightly]` — also callable via `pnpm test:e2e:smoke`, `pnpm test:e2e:pr`, `pnpm test:e2e:nightly`.
 - **CI workflow:** `.github/workflows/e2e.yml` — runs on PR / push:main / cron 06:00 UTC / workflow_dispatch.
-- **CI matrix:** 7 cells in two matrices — `test-no-publish` (parallel: pr-install, pr-preflight, pr-mod, pr-init-session) + `test-publish` (max-parallel: 1: pr-deploy-frontend, pr-deploy-foundry, pr-deploy-cdm). Each cell runs a subset via `vitest -t "<pattern>"`.
+- **CI matrix:** 8 cells across three matrices — `test-no-publish` (parallel: pr-install, pr-preflight, pr-mod, pr-init-session) + `test-publish` (max-parallel: 1: pr-deploy-frontend, pr-deploy-foundry) + `test-nightly-no-publish` (parallel, schedule/dispatch only: nightly-mod-miss, nightly-diagnostic). Each cell runs a subset via `vitest -t "<pattern>"`.
 - **Test files:** `e2e/cli/*.test.ts` (vitest, spawned via `bun run src/index.ts`).
 - **Reports directory:** `e2e-reports/junit.xml` + `e2e-reports/dot-runs.log` (gitignored).
 - **Tag prefix:** `DOT_TAG=e2e-{ci|local}-{trigger}` so Sentry dashboards filter test traffic. The CLI plumbs `DOT_TAG` into the `cli.tag` root-span attribute via `src/telemetry-config.ts`.